config

package
v1.0.1-0...-20e81f0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 17, 2023 License: Apache-2.0 Imports: 34 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	//GZipCodec gzip code
	GZipCodec = "gzip"
	//GZIPExtension gzip extension
	GZIPExtension = ".gz"

	//AVROExtension avro extension
	AVROExtension = ".avro"

	//ZipCodec zip code
	ZipCodec = "zip"
	//ZIPExtension zip extension
	ZIPExtension = ".zip"

	//TarCodec tar code
	TarCodec = "tar"
	//TarExtension tar extension
	TarExtension = ".tar"
)
View Source 
const (
	DataTypeTime    = "time"
	DataTypeFloat   = "float"
	DataTypeInt     = "int"
	DataTypeBoolean = "boolean"
	DataTypeString  = "string"
)

Variables

This section is empty.

Functions

This section is empty.

Types

type Compression 

type Compression struct {
	Codec      string `json:",omitempty"`
	Uncompress bool   `json:",omitempty"`
}

Compression represents conversion strategy

func NewCompressionForURL 

func NewCompressionForURL(URL string) *Compression

NewCompressionForURL returns compression for matched codec or nil

func (*Compression) Equals 

func (c *Compression) Equals(compression *Compression) bool

Equals returns true if compression is the same

type CustomKey 

type CustomKey struct {
	auth.Secret `json:",omitempty"`
	AES256Key   *option.AES256Key `json:"-"`
}

CustomKey represents custom key

type Field 

type Field struct {
	Name             string
	Position         *int
	DataType         string
	SourceDateFormat string
	SourceDateLayout string
	TargetDateFormat string
	TargetDateLayout string
}

Fields represents validation filed

func (*Field) AdjustText 

func (f *Field) AdjustText(value string) (string, error)

AdjustText adjust text value

func (*Field) AdjustValue 

func (f *Field) AdjustValue(value interface{}) (interface{}, error)

AdjustText adjust text value

func (*Field) Init 

func (f *Field) Init()

type Overflow 

type Overflow struct {
	SizeMB    int64
	DestURL   string
	Queue     string
	ProjectID string
	Topic     string
}

Overflow defines overflow controls

func (Overflow) MessageDest 

func (o Overflow) MessageDest() string

func (Overflow) MessageEvent 

func (o Overflow) MessageEvent(URL string) interface{}

func (Overflow) Size 

func (o Overflow) Size() int64

type Partition 

type Partition struct {
	Field      string
	FieldIndex int
	Separator  string
	Mod        int
	Hash       string
	// contains filtered or unexported fields
}

Partition represent partition split

func (*Partition) Key 

func (p *Partition) Key(data []byte) (interface{}, error)

type Replace 

type Replace struct {
	From string
	To   string
}

Replace represents replace instruction

type Resource 

type Resource struct {
	matcher.Basic
	Overflow             *Overflow
	Bucket               string                       `json:",omitempty"`
	URL                  string                       `json:",omitempty"`
	Region               string                       `json:",omitempty"`
	CustomKey            *CustomKey                   `json:",omitempty"`
	Grant                *option.Grant                `json:",omitempty"`
	ACL                  *option.ACL                  `json:",omitempty"`
	ServerSideEncryption *option.ServerSideEncryption `json:",omitempty"`
	Credentials          *auth.Credentials            `json:",omitempty"`
	Proxy                *option.Proxy
	Topic                string `json:",omitempty"`
	Queue                string `json:",omitempty"`
	Vendor               string `json:",omitempty"`
	//Optional pubsub project ID, otherwise it uses default one.
	ProjectID string `json:",omitempty"`
	Pattern   string `json:",omitempty"`

	Parameters []*pattern.Param `json:",omitempty"`
	// contains filtered or unexported fields
}

Represents a destination

func (Resource) CloneWithURL 

func (r Resource) CloneWithURL(URL string) *Resource

CloneWithURL clone resource with URL

func (*Resource) ExpandURL 

func (r *Resource) ExpandURL(sourceURL string) (string, error)

func (*Resource) Init 

func (r *Resource) Init(projectID string)

type Rule 

type Rule struct {
	Info       base.Info
	Disabled   bool `json:",omitempty"`
	Dest       *Resource
	Source     *Resource
	Replace    []*Replace   `json:",omitempty"`
	Schema     *Schema      `json:",omitempty"`
	Transcoder *Transcoding `json:",omitempty"`
	Streaming  *Streaming   `json:",omitempty"`
	Split      *Split       `json:",omitempty"`
	AllowEmpty bool         `json:",omitempty"`
	job.Actions
	*Compression
	//PreserveDepth  - preserves specified folder depth in dest URL
	PreserveDepth *int `json:",omitempty"`

	//Group defines group of rule to be matched, otherwise multi match is invalid
	Group string `json:",omitempty"`

	//Name of the file that is done flag, in that case all file will be replayed
	DoneMarker string `json:",omitempty"`
}

Rule represent matching resource route rule

func (*Rule) ArchiveWalkURL 

func (r *Rule) ArchiveWalkURL(URL string) string

func (*Rule) GetPreserveDepth 

func (r *Rule) GetPreserveDepth() int

GetPreserveDepth returns PreservceDepth

func (*Rule) HasMatch 

func (r *Rule) HasMatch(URL string) bool

Match returns true if URL matches prefix or suffix

func (*Rule) HasPreserveDepth 

func (r *Rule) HasPreserveDepth() bool

HasPreserveDepth returns true if property has been specified

func (*Rule) HasSplit 

func (r *Rule) HasSplit() bool

HasSplit returns true if rule has split defined

func (*Rule) HasTransformer 

func (r *Rule) HasTransformer() bool

HasTransformer returns true if rule has recover or replace option

func (*Rule) Init 

func (r *Rule) Init(ctx context.Context, fs afs.Service) error

Load initialises routes

func (*Rule) Name 

func (r *Rule) Name(URL string) string

Name return route dest asset name

func (*Rule) NewReplacer 

func (r *Rule) NewReplacer() *strings.Replacer

NewReplacer create a replaced for the rule

func (*Rule) Resources 

func (r *Rule) Resources() []*Resource

Resources returns rule resource

func (*Rule) ShallArchiveWalk 

func (r *Rule) ShallArchiveWalk(URL string) bool

func (*Rule) SourceCompression 

func (r *Rule) SourceCompression(URL string) (source *Compression)

SourceCompression returns compression for URL

func (*Rule) Validate 

func (r *Rule) Validate() error

Validate checks if route is valid

type Ruleset 

type Ruleset struct {
	BaseURL   string
	CheckInMs int
	Rules     []*Rule
	// contains filtered or unexported fields
}

Ruleset represents route slice

func (Ruleset) Init 

func (r Ruleset) Init(ctx context.Context, fs afs.Service) error

func (*Ruleset) Load 

func (r *Ruleset) Load(ctx context.Context, fs afs.Service) error

Load initialises resources

func (Ruleset) Match 

func (r Ruleset) Match(URL string) (matched []*Rule)

Match returns the first match route

func (*Ruleset) ReloadIfNeeded 

func (r *Ruleset) ReloadIfNeeded(ctx context.Context, fs afs.Service) (bool, error)

func (Ruleset) Rule 

func (r Ruleset) Rule(URL string) *Rule

Match returns the first match route

func (Ruleset) Validate 

func (r Ruleset) Validate() error

type Schema 

type Schema struct {
	//MaxBadRecords
	MaxBadRecords *int
	Format        string
	Delimiter     string
	LazyQuotes    bool
	FieldCount    int

	Fields []*Field
	// contains filtered or unexported fields
}

func (*Schema) IsCSV 

func (r *Schema) IsCSV() bool

func (*Schema) IsJSON 

func (r *Schema) IsJSON() bool

func (*Schema) NewCsvReader 

func (r *Schema) NewCsvReader(reader io.Reader) *csv.Reader

type Split 

type Split struct {
	//MaxLines max number lines in one split chunk
	MaxLines int
	//Template has to have %s placeholder for file name, and %d (or padded placeholder i.e. %04d) chunk number, %v is for partition
	Template string

	//Partition partition rule
	Partition *Partition

	//Schema format specific schema
	Schema string

	//SchemaURL format specific schema location
	SchemaURL string

	//MaxSize max size, if file larger then splits
	MaxSize int
}

Split represents a split rule

func (*Split) Name 

func (s *Split) Name(router *Rule, URL string, counter int32, partition interface{}) string

Name returns a chunk name for supplied URL and mirrorChunkeddAsset number

type Streaming 

type Streaming struct {
	ThresholdMb int

	PartSizeMb              int
	ChecksumSkipThresholdMb int
	// contains filtered or unexported fields
}

Streaming represents streaming option

func (*Streaming) ChecksumSkipThreshold 

func (c *Streaming) ChecksumSkipThreshold() int

ChecksumSkipThreshold upload checksum skip threshold

func (*Streaming) Init 

func (c *Streaming) Init()

Init initialises streaming

func (*Streaming) PartSize 

func (c *Streaming) PartSize() int

PartSize download part size

func (*Streaming) Threshold 

func (c *Streaming) Threshold() int

Threshold returns download/upload streaming

type Transcoding 

type Transcoding struct {
	Source        transcoding.Codec
	Dest          transcoding.Codec
	PathMapping   transcoding.Mappings
	MaxBadRecords *int
	Autodetect    bool //detect source schema
}

Transcoding represents transcoding

func (*Transcoding) Init 

func (t *Transcoding) Init(ctx context.Context, fs afs.Service) error

Init intialise transcoding

func (*Transcoding) Validate 

func (t *Transcoding) Validate() error

Load check if transcoding is valid

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.