Documentation ¶
Index ¶
- func ImportExisting(db *ImportDb, client configV1.ConfigClient, kind configV1.Kind, ...) error
- func NewHttpClient(timeout time.Duration, followRedirects bool) *http.Client
- func NewRecordReader(fileOrDir string, decoder RecordDecoder, filePattern string) (l *recordReader, err error)
- type DuplicateKindReporter
- type DuplicateReportRecord
- type ErrAlreadyExists
- type Executor
- type ExistsCode
- type ImportDb
- type Job
- type JsonYamlDecoder
- type KeyNormalizer
- type LineAsStringDecoder
- type Payload
- type Record
- type RecordDecoder
- type SeedDesc
- type SeedDuplicateReportRecord
- type SeedRecord
- type SeedReporter
- type State
- type UriChecker
- type UriKeyNormalizer
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func ImportExisting ¶
func ImportExisting(db *ImportDb, client configV1.ConfigClient, kind configV1.Kind, keyNormalizer KeyNormalizer) error
func NewHttpClient ¶ added in v0.6.0
func NewRecordReader ¶
func NewRecordReader(fileOrDir string, decoder RecordDecoder, filePattern string) (l *recordReader, err error)
Types ¶
type DuplicateKindReporter ¶
type DuplicateKindReporter struct {
*ImportDb
}
type DuplicateReportRecord ¶
type ErrAlreadyExists ¶ added in v0.6.0
type ErrAlreadyExists string
func (ErrAlreadyExists) Error ¶ added in v0.6.0
func (e ErrAlreadyExists) Error() string
type Executor ¶
Executor is a work queue that executes jobs in concurrent workers.
func NewExecutor ¶
NewExecutor creates a work queue with nrOfWorkers workers.
do is the function that will be called for each job. onError is the function that will be called for each job that fails.
To close the work queue, call Wait() after all jobs have been queued. Writing to the Queue channel after Wait() has been called will panic.
type ExistsCode ¶
type ExistsCode int
const ( Undefined ExistsCode = iota NewKey NewId Exists )
func (ExistsCode) ExistsInVeidemann ¶
func (e ExistsCode) ExistsInVeidemann() bool
func (ExistsCode) String ¶
func (e ExistsCode) String() string
type ImportDb ¶
type ImportDb struct {
// contains filtered or unexported fields
}
func (*ImportDb) Close ¶
func (d *ImportDb) Close()
Close closes the database, stops the GC ticker and waits for
func (*ImportDb) Iterate ¶
Iterate iterates over all keys in the db and calls the function with the key and value. The function is not called in parallel.
func (*ImportDb) RunValueLogGC ¶
type JsonYamlDecoder ¶ added in v0.6.0
type JsonYamlDecoder struct {
// contains filtered or unexported fields
}
JsonYamlDecoder is a decoder that reads json or yaml from the input and decodes it into a struct
func (*JsonYamlDecoder) Init ¶ added in v0.6.0
func (j *JsonYamlDecoder) Init(r io.Reader, suffix string)
func (*JsonYamlDecoder) Read ¶ added in v0.6.0
func (j *JsonYamlDecoder) Read(v interface{}) error
type KeyNormalizer ¶
type LineAsStringDecoder ¶ added in v0.6.0
type LineAsStringDecoder struct {
// contains filtered or unexported fields
}
LineAsStringDecoder is a decoder that reads a line from the input as a string
func (*LineAsStringDecoder) Init ¶ added in v0.6.0
func (l *LineAsStringDecoder) Init(r io.Reader, suffix string)
func (*LineAsStringDecoder) Read ¶ added in v0.6.0
func (l *LineAsStringDecoder) Read(v interface{}) error
type Payload ¶
type Payload interface { any }
Payload is an interface for the payload of a job in a work queue
type RecordDecoder ¶
type SeedDesc ¶ added in v0.6.0
type SeedDesc struct { EntityId string `json:"entityId,omitempty" yaml:"entityId,omitempty"` EntityName string `json:"entityName,omitempty" yaml:"entityName,omitempty"` EntityDescription string `json:"entityDescription,omitempty" yaml:"entityDescription,omitempty"` EntityLabel []*configV1.Label `json:"entityLabel,omitempty" yaml:"entityLabel,omitempty"` Uri string `json:"uri,omitempty" yaml:"uri,omitempty"` SeedDescription string `json:"seedDescription,omitempty" yaml:"seedDescription,omitempty"` SeedLabel []*configV1.Label `json:"seedLabel,omitempty" yaml:"seedLabel,omitempty"` Description string `json:"description,omitempty" yaml:"description,omitempty"` CrawlJobRef []*configV1.ConfigRef }
func (*SeedDesc) ToEntity ¶ added in v0.6.0
func (sd *SeedDesc) ToEntity() *configV1.ConfigObject
func (*SeedDesc) ToSeed ¶ added in v0.6.0
func (sd *SeedDesc) ToSeed() *configV1.ConfigObject
type SeedDuplicateReportRecord ¶
type SeedDuplicateReportRecord struct { Host string Seeds []SeedRecord }
type SeedRecord ¶
type SeedReporter ¶
type SeedReporter struct { *ImportDb Client configV1.ConfigClient }
type State ¶
type State struct {
// contains filtered or unexported fields
}
func (*State) GetFilename ¶
func (*State) GetRecordNum ¶
type UriChecker ¶ added in v0.6.0
UriChecker checks if a uri is reachable
func (*UriChecker) Check ¶ added in v0.6.0
func (uc *UriChecker) Check(uri string) (string, error)
Check checks if a uri is reachable and returns the uri if it is reachable If the uri is not reachable, it returns an error If the uri is redirected with 301, it returns the redirected uri
func (*UriChecker) GetTitle ¶ added in v0.6.0
func (uc *UriChecker) GetTitle(uri string) string
GetTitle returns the title of the uri