scraper

Documentation

Index

• Constants
• Variables
• type DocumentSection
• type ScraperService
  • func NewScraperService(config *config.JinaConfig, natsConn *nats_internal.NATSConnections, ...) ScraperService

Constants

const (
	MaxCrawlDepth   = 5
	MaxPagesToCrawl = 100
)

const (
	// queue group
	QUEUE_GROUP = "scraper-service"

	// consumer config
	CONSUMER_NAME         = "scraper-consumer"
	ACK_WAIT              = 30 * time.Second
	MAX_DELIVERY_ATTEMPTS = 5
	MAX_ACK_PENDING       = 100
	FETCH_BATCH_SIZE      = 50
	MAX_FETCH_WAIT        = 500 * time.Millisecond
	ERR_BACKOFF           = 100 * time.Millisecond
)

Variables

var (
	ErrPaymentRequired = errors.New("jina balance requires topup")
	ErrUnprocessable   = errors.New("jina cannot process webpage")
	ErrUrlNotReachable = errors.New("url is not reachable")
)

var SUBSCRIBED_SUBJECT = enums.EventWebtrackerCreated.String()

Types

type DocumentSection

type DocumentSection struct {
	Heading   string
	Content   []string
	LinkCount int
	TextCount int
	ListCount int
}

type ScraperService

type ScraperService interface {
	interfaces.NatsService
	Crawl(ctx context.Context, domain string) error
}

func NewScraperService

func NewScraperService(
	config *config.JinaConfig,
	natsConn *nats_internal.NATSConnections,
	leadsDB *database.DbConnections,
	repositories *repository.Repositories,
) ScraperService