extractors

Documentation

Index

• type DefaultExtractor
  • func (extractor DefaultExtractor) ExtractLinks(ctx context.Context, threadID int, link string) ([]string, error)
• type DelayingExtractor
  • func NewDelayingExtractor(minimalDelay time.Duration, sleeper SleepHandler, ...) *DelayingExtractor
  • func (extractor *DelayingExtractor) ExtractLinks(ctx context.Context, threadID int, link string) ([]string, error)
• type ExtractorGroup
  • func (extractors ExtractorGroup) ExtractLinks(ctx context.Context, threadID int, link string) ([]string, error)
• type HTTPClient
• type LinkExtractor
• type LinkLoader
• type LinkTransformer
• type Logger
• type RepeatingExtractor
  • func (extractor RepeatingExtractor) ExtractLinks(ctx context.Context, threadID int, link string) ([]string, error)
• type SitemapExtractor
  • func (extractor SitemapExtractor) ExtractLinks(ctx context.Context, threadID int, link string) ([]string, error)
• type SleepHandler
• type Sleeper
• type TrimmingExtractor
  • func (extractor TrimmingExtractor) ExtractLinks(ctx context.Context, threadID int, link string) ([]string, error)

Types

type DefaultExtractor

type DefaultExtractor struct {
	HTTPClient      httputils.HTTPClient
	Filters         htmlselector.OptimizedFilterGroup
	LinkTransformer models.LinkTransformer
}

DefaultExtractor ...

func (DefaultExtractor) ExtractLinks

func (extractor DefaultExtractor) ExtractLinks(
	ctx context.Context,
	threadID int,
	link string,
) ([]string, error)

ExtractLinks ...

type DelayingExtractor

type DelayingExtractor struct {
	// contains filtered or unexported fields
}

DelayingExtractor ...

func NewDelayingExtractor

func NewDelayingExtractor(
	minimalDelay time.Duration,
	sleeper SleepHandler,
	linkExtractor models.LinkExtractor,
) *DelayingExtractor

NewDelayingExtractor ...

func (*DelayingExtractor) ExtractLinks

func (extractor *DelayingExtractor) ExtractLinks(
	ctx context.Context,
	threadID int,
	link string,
) ([]string, error)

ExtractLinks ...

type ExtractorGroup

type ExtractorGroup struct {
	Name           string
	LinkExtractors []models.LinkExtractor
	Logger         log.Logger
}

ExtractorGroup ...

func (ExtractorGroup) ExtractLinks

func (extractors ExtractorGroup) ExtractLinks(
	ctx context.Context,
	threadID int,
	link string,
) ([]string, error)

ExtractLinks ...

type HTTPClient

type HTTPClient interface {
	httputils.HTTPClient
}

HTTPClient ...

It's used only for mock generating.

type LinkExtractor

type LinkExtractor interface {
	models.LinkExtractor
}

LinkExtractor ...

It's used only for mock generating.

type LinkLoader

type LinkLoader interface {
	LoadLink(link string, options interface{}) ([]byte, error)
}

LinkLoader ...

It's used only for mock generating.

type LinkTransformer

type LinkTransformer interface {
	models.LinkTransformer
}

LinkTransformer ...

It's used only for mock generating.

type Logger

type Logger interface {
	log.Logger
}

Logger ...

It's used only for mock generating.

type RepeatingExtractor

type RepeatingExtractor struct {
	LinkExtractor models.LinkExtractor
	RepeatCount   int
	RepeatDelay   time.Duration
	Logger        log.Logger
	SleepHandler  SleepHandler
}

RepeatingExtractor ...

func (RepeatingExtractor) ExtractLinks

func (extractor RepeatingExtractor) ExtractLinks(
	ctx context.Context,
	threadID int,
	link string,
) ([]string, error)

ExtractLinks ...

type SitemapExtractor

type SitemapExtractor struct {
	SitemapRegister registers.SitemapRegister
	Logger          log.Logger
}

SitemapExtractor ...

func (SitemapExtractor) ExtractLinks

func (extractor SitemapExtractor) ExtractLinks(
	ctx context.Context,
	threadID int,
	link string,
) ([]string, error)

ExtractLinks ...

type SleepHandler

type SleepHandler func(duration time.Duration)

SleepHandler ...

type Sleeper

type Sleeper interface {
	Sleep(duration time.Duration)
}

Sleeper ...

It's used only for mock generating.

type TrimmingExtractor

type TrimmingExtractor struct {
	TrimLink      urlutils.LinkTrimming
	LinkExtractor models.LinkExtractor
}

TrimmingExtractor ...

func (TrimmingExtractor) ExtractLinks

func (extractor TrimmingExtractor) ExtractLinks(
	ctx context.Context,
	threadID int,
	link string,
) ([]string, error)

ExtractLinks ...