bundledImpl

package
v0.1.43 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 17, 2022 License: MIT Imports: 40 Imported by: 1

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func FullDestinPath added in v0.1.2 

func FullDestinPath(destDir string, fullSourcePath string, extension string) string

extension example strconv.Itoa(key)+."parquet"

func GetClient added in v0.1.11 

func GetClient(namenode string, user string) (*hdfs.Client, error)

Code copied from colinmarc/hdfs

Types

type ArrayToRust added in v0.1.35 

type ArrayToRust struct {
}

func (*ArrayToRust) Process added in v0.1.35 

func (atr *ArrayToRust) Process(reader io.Reader, customParams interface{}) bool

func (*ArrayToRust) Setup added in v0.1.35 

func (atr *ArrayToRust) Setup(customParams interface{}) bool

type DummyEvent 

type DummyEvent struct {
}

func (*DummyEvent) Process 

func (de *DummyEvent) Process(reader io.Reader, customParams interface{}) bool

func (*DummyEvent) Setup 

func (de *DummyEvent) Setup(customParams interface{}) bool

type FetchAndExecEvent added in v0.1.3 

type FetchAndExecEvent struct {
	DestinationFolder string
	CmdTemplate       string
	Data              map[string]interface{}
	Tmpl              *template.Template
}

func (*FetchAndExecEvent) Process added in v0.1.3 

func (fae *FetchAndExecEvent) Process(reader io.Reader, customParams interface{}) bool

func (*FetchAndExecEvent) Setup added in v0.1.3 

func (fae *FetchAndExecEvent) Setup(customParams interface{}) bool

type HDFS added in v0.1.11 

type HDFS struct {
	ViperKey      string
	Namenode      string
	WatchFolder   string
	MarkerFolder  string
	ExcludeFolder string
	GraceMilliSec int
	HDFSClient    *hdfs.Client
	Ctx           context.Context
	User          string
	// contains filtered or unexported fields
}

func NewHDFS added in v0.1.11 

func NewHDFS(viperkey string) *HDFS

func (*HDFS) FilterListFiles added in v0.1.31 

func (hdfsStruct *HDFS) FilterListFiles() (map[string]api.FileEntity, error)

TODO use https://github.com/deckarep/golang-set

func (*HDFS) FindWorkingNamenode added in v0.1.25 

func (hdfsStruct *HDFS) FindWorkingNamenode()

func (HDFS) GetFileSet added in v0.1.11 

func (hdfsStruct HDFS) GetFileSet() map[string]int64

func (HDFS) GetFilteredFileSet added in v0.1.11 

func (hdfsStruct HDFS) GetFilteredFileSet(filter string) map[string]int64

func (HDFS) ListFiles added in v0.1.11 

func (hdfsStruct HDFS) ListFiles() (map[string]api.FileEntity, error)

func (*HDFS) Process added in v0.1.39 

func (hdfsStruct *HDFS) Process() error

func (*HDFS) Setup added in v0.1.39 

func (hdfsStruct *HDFS) Setup(customParams interface{}) error

Process all files , and for every sucessful process , add a ".done" file.

type KafkaEmitEvent 

type KafkaEmitEvent struct {
	Data     map[string]interface{}
	Tmpl     *template.Template
	Producer *kafka.Writer
	Topic    string
	// contains filtered or unexported fields
}

func (*KafkaEmitEvent) Process 

func (kee *KafkaEmitEvent) Process(reader io.Reader, customParams interface{}) bool

func (*KafkaEmitEvent) Setup 

func (kee *KafkaEmitEvent) Setup(customParams interface{}) bool

type LocalFilesystem added in v0.1.7 

type LocalFilesystem struct {
	WatchFolder   string
	ExcludeFolder string
	GraceMilliSec int
	Ctx           context.Context
}

func NewLocalFilesystem added in v0.1.7 

func NewLocalFilesystem() *LocalFilesystem

func (*LocalFilesystem) ListFiles added in v0.1.7 

func (lf *LocalFilesystem) ListFiles() map[string]api.FileEntity

func (*LocalFilesystem) Watch added in v0.1.7 

func (lf *LocalFilesystem) Watch(eventHandlers []api.IOEvent) (bool, error)

type Metrics added in v0.1.38 

type Metrics struct {
	FilesMovedOK  prometheus.Counter
	FileSizeOK    prometheus.Counter
	LineCountOK   prometheus.Counter
	AllocMib      prometheus.Gauge
	TotalAllocMib prometheus.Gauge
	SysMib        prometheus.Gauge
	NumGC         prometheus.Gauge
}

func InitMetrics added in v0.1.38 

func InitMetrics() Metrics

type MetricsEmitEvent added in v0.1.38 

type MetricsEmitEvent struct {
	Metrics   *Metrics
	WaitGroup *sync.WaitGroup
	// contains filtered or unexported fields
}

func (*MetricsEmitEvent) Process added in v0.1.38 

func (mee *MetricsEmitEvent) Process(reader io.Reader, customParams interface{}) bool

func (*MetricsEmitEvent) Setup added in v0.1.38 

func (mee *MetricsEmitEvent) Setup(customParams interface{}) bool

type RCLONE added in v0.1.39 

type RCLONE struct {
	ViperKey        string
	GraceMilliSec   int
	CollectStat     bool
	EntriesBefore   int
	EntriesAfter    int
	SourcePath      string
	DestinationPath string

	//	HDFSClient    *hdfs.Client
	Ctx context.Context
}

func NewRCLONE added in v0.1.39 

func NewRCLONE(viperKey string) *RCLONE

func (*RCLONE) Process added in v0.1.39 

func (rcloneStruct *RCLONE) Process() error

func (*RCLONE) Setup added in v0.1.39 

func (rcloneStruct *RCLONE) Setup(customParams interface{}) error

Process all files , and for every sucessful process , add a ".done" file.

type S3 added in v0.1.2 

type S3 struct {
	ViperKey      string
	Endpoint      string
	UseSSL        bool
	BucketName    string
	User          string
	Password      string
	Token         string
	WatchFolder   string
	MarkerFolder  string
	ExcludeFolder string
	GraceMilliSec int
	MinioClient   *minio.Client
	Ctx           context.Context
	// contains filtered or unexported fields
}

func NewS3 added in v0.1.2 

func NewS3(viperKey string) *S3

func (S3) GetFileSet added in v0.1.2 

func (s3 S3) GetFileSet(minioclient *minio.Client) map[string]int64

func (S3) GetFilteredFileSet added in v0.1.2 

func (s3 S3) GetFilteredFileSet(filter string, minioClient *minio.Client) map[string]int64

func (S3) ListFiles added in v0.1.2 

func (s3 S3) ListFiles(minioClient *minio.Client) map[string]api.FileEntity

func (*S3) Process added in v0.1.39 

func (s3 *S3) Process() error

func (*S3) Setup added in v0.1.39 

func (s3 *S3) Setup(customParams interface{}) error

type StreamAndExecEvent added in v0.1.7 

type StreamAndExecEvent struct {
	CmdTemplate string
	Data        map[string]interface{}
	Tmpl        *template.Template
}

func (*StreamAndExecEvent) Process added in v0.1.7 

func (sae *StreamAndExecEvent) Process(reader io.Reader, customParams interface{}) bool

func (*StreamAndExecEvent) Setup added in v0.1.7 

func (sae *StreamAndExecEvent) Setup(customParams interface{}) bool

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.