index

package
v0.4.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 20, 2024 License: Apache-2.0 Imports: 25 Imported by: 0

Documentation

Index

Constants

View Source 
const AlreadyIndexedError indexError = "already indexed"

Variables

This section is empty.

Functions

func NewIndexer 

func NewIndexer(w RecordWriter, options ...Option) func(string)

Types

type AutoIndexOption 

type AutoIndexOption func(*AutoIndexOptions)

func WithExcludeDirs 

func WithExcludeDirs(res ...*regexp.Regexp) AutoIndexOption

func WithMaxDepth 

func WithMaxDepth(depth int) AutoIndexOption

func WithPaths 

func WithPaths(paths []string) AutoIndexOption

type AutoIndexOptions 

type AutoIndexOptions struct {
	MaxDepth int
	Paths    []string
	Options
}

type AutoIndexer 

type AutoIndexer struct {
	// contains filtered or unexported fields
}

func NewAutoIndexer 

func NewAutoIndexer(queue Queue, options ...AutoIndexOption) AutoIndexer

func (AutoIndexer) Run 

func (a AutoIndexer) Run(ctx context.Context) error

type Cdx 

type Cdx struct{}

func (Cdx) Write 

func (c Cdx) Write(rec Record) error

type CdxAPI 

type CdxAPI interface {
	Search(context.Context, Request, chan<- CdxResponse) error
}

type CdxJ 

type CdxJ struct{}

func (CdxJ) Write 

func (c CdxJ) Write(rec Record) error

type CdxPb 

type CdxPb struct{}

func (CdxPb) Write 

func (c CdxPb) Write(rec Record) error

type CdxResponse 

type CdxResponse interface {
	GetCdx() *schema.Cdx
	GetError() error
}

type DateRange 

type DateRange interface {
	Contains(int64) bool
}

type Deleter added in v0.4.0 

type Deleter interface {
	Delete(context.Context) error
}

type FileAPI 

type FileAPI interface {
	GetFileInfo(ctx context.Context, filename string) (*schema.FileInfo, error)
	ListFileInfo(context.Context, Request, chan<- FileInfoResponse) error
}

type FileInfoResponse 

type FileInfoResponse interface {
	GetFileInfo() *schema.FileInfo
	GetError() error
}

type Filter 

type Filter interface {
	Eval(*schema.Cdx) bool
}

type IdAPI 

type IdAPI interface {
	GetStorageRef(ctx context.Context, warcId string) (string, error)
	ListStorageRef(context.Context, Request, chan<- IdResponse) error
}

type IdResponse 

type IdResponse interface {
	GetId() string
	GetValue() string
	GetError() error
}

type Indexer 

type Indexer interface {
	Index(string) error
}

type KafkaIndexOption 

type KafkaIndexOption func(options *kafka.ReaderConfig)

func WithBrokers 

func WithBrokers(brokers []string) KafkaIndexOption

func WithGroupID 

func WithGroupID(groupID string) KafkaIndexOption

func WithMaxBytes 

func WithMaxBytes(maxBytes int) KafkaIndexOption

func WithMaxWait 

func WithMaxWait(maxWait time.Duration) KafkaIndexOption

func WithMinBytes 

func WithMinBytes(minBytes int) KafkaIndexOption

func WithTopic 

func WithTopic(topic string) KafkaIndexOption

type KafkaIndexer 

type KafkaIndexer struct {
	kafka.ReaderConfig
	// contains filtered or unexported fields
}

func NewKafkaIndexer 

func NewKafkaIndexer(queue Queue, options ...KafkaIndexOption) KafkaIndexer

func (KafkaIndexer) Run 

func (k KafkaIndexer) Run(ctx context.Context) (err error)

type MatchType added in v0.4.0 

type MatchType int
const (
	MatchTypeExact MatchType = iota
	MatchTypePrefix
	MatchTypeHost
	MatchTypeDomain
	MatchTypeVerbatim
)

type Option 

type Option func(*Options)

func WithExcludes 

func WithExcludes(res ...*regexp.Regexp) Option

func WithIncludes 

func WithIncludes(res ...*regexp.Regexp) Option

type Options 

type Options struct {
	Includes []*regexp.Regexp
	Excludes []*regexp.Regexp
	// contains filtered or unexported fields
}

type Queue 

type Queue interface {
	Add(path string)
	Close()
}

type Record 

type Record struct {
	*schema.Cdx
}

func (Record) Marshal 

func (r Record) Marshal() ([]byte, error)

Marshal is a wrapper around proto.Marshal that can handle invalid UTF-8 runes in the MIME type.

func (Record) String 

func (r Record) String() string

type RecordWriter 

type RecordWriter interface {
	Write(Record) error
}

type ReportAPI added in v0.4.0 

type ReportAPI interface {
	CreateReport(context.Context, Request) (*schema.Report, error)
	ListReports(context.Context, Request, chan<- ReportResponse) error
	GetReport(context.Context, string) (*schema.Report, error)
	CancelReport(context.Context, string) error
	DeleteReport(context.Context, string) error
}

type ReportGenerator added in v0.4.0 

type ReportGenerator interface {
	CdxAPI
	AddTask(string, context.CancelFunc)
	DeleteTask(string)
	SaveReport(context.Context, *schema.Report) error
}

type ReportResponse added in v0.4.0 

type ReportResponse interface {
	GetReport() *schema.Report
	GetError() error
}

type Request 

type Request interface {
	Url() *url.Url
	Ssurt() string
	Sort() Sort
	DateRange() DateRange
	Filter() Filter
	Limit() int
	Closest() string
	MatchType() MatchType
}

type Runner 

type Runner interface {
	Run(context.Context) error
}

type Sort 

type Sort int
const (
	SortNone    Sort = iota
	SortDesc         // largest to smallest alphabetically
	SortAsc          // smallest to largest alphabetically
	SortClosest      // closest in time
)

type Toc 

type Toc struct {
	*bloom.BloomFilter
	// contains filtered or unexported fields
}

func (*Toc) Write 

func (t *Toc) Write(rec Record) error

type WorkQueue 

type WorkQueue struct {
	// contains filtered or unexported fields
}

func NewWorkQueue 

func NewWorkQueue(execute Worker, concurrency int) *WorkQueue

func (*WorkQueue) Add 

func (iw *WorkQueue) Add(job string)

func (*WorkQueue) Close 

func (iw *WorkQueue) Close()

func (*WorkQueue) Wait added in v0.4.0 

func (iw *WorkQueue) Wait()

type Worker 

type Worker func(string)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.