sources

Documentation

Index

• func NewAzureBlob(params *AzureBlobParams) []pipeline.SourcePipeline
• func NewHTTP(sourceURIs []string, targetAliases []string, md5 bool) pipeline.SourcePipeline
• func NewMultiFile(params *MultiFileParams) []pipeline.SourcePipeline
• func NewS3Pipeline(params *S3Params) []pipeline.SourcePipeline
• type AzureBlob
• type AzureBlobParams
• type FileInfo
• type HTTPPipeline
  • func (f *HTTPPipeline) ConstructBlockInfoQueue(blockSize uint64) (partitionsQ chan pipeline.PartsPartition, partsQ chan pipeline.Part, ...)
  • func (f *HTTPPipeline) ExecuteReader(partitionsQ chan pipeline.PartsPartition, partsQ chan pipeline.Part, ...)
  • func (f *HTTPPipeline) GetSourcesInfo() []pipeline.SourceInfo
• type MultiFileParams
• type MultiFilePipeline
  • func (f *MultiFilePipeline) ConstructBlockInfoQueue(blockSize uint64) (partitionsQ chan pipeline.PartsPartition, partsQ chan pipeline.Part, ...)
  • func (f *MultiFilePipeline) ExecuteReader(partitionsQ chan pipeline.PartsPartition, partsQ chan pipeline.Part, ...)
  • func (f *MultiFilePipeline) GetSourcesInfo() []pipeline.SourceInfo
• type S3Params
• type S3Pipeline
• type SourceParams

Functions

func NewAzureBlob

func NewAzureBlob(params *AzureBlobParams) []pipeline.SourcePipeline

NewAzureBlob creates a new instance of the HTTPPipeline for Azure Blobs

func NewHTTP

func NewHTTP(sourceURIs []string, targetAliases []string, md5 bool) pipeline.SourcePipeline

NewHTTP creates a new instance of an HTTP source To get the file size, a HTTP HEAD request is issued and the Content-Length header is inspected.

func NewMultiFile

func NewMultiFile(params *MultiFileParams) []pipeline.SourcePipeline

NewMultiFile creates a new MultiFilePipeline. If the sourcePattern results in a single file, the targetAlias, if set, will be used as the target name. Otherwise the full original file name will be used instead. sourcePatterns []string, blockSize uint64, targetAliases []string, numOfPartitions int, md5 bool

func NewS3Pipeline

func NewS3Pipeline(params *S3Params) []pipeline.SourcePipeline

NewS3Pipeline creates a new instance of the HTTPPipeline for S3

Types

type AzureBlob

type AzureBlob struct {
	HTTPPipeline
	Container string
	BlobNames []string
	// contains filtered or unexported fields
}

AzureBlob constructs parts channel and implements data readers for Azure Blobs exposed via HTTP

type AzureBlobParams

type AzureBlobParams struct {
	SourceParams
	Container           string
	BlobNames           []string
	AccountName         string
	AccountKey          string
	SasExpNumberOfHours int
}

AzureBlobParams parameters for the creation of Azure Blob source pipeline

type FileInfo

type FileInfo struct {
	FileStats   *os.FileInfo
	SourceURI   string
	TargetAlias string
	NumOfBlocks int
}

FileInfo Contains the metadata associated with a file to be transferred

type HTTPPipeline

type HTTPPipeline struct {
	Sources    []pipeline.SourceInfo
	HTTPClient *http.Client
	// contains filtered or unexported fields
}

HTTPPipeline constructs parts channel and implements data readers for file exposed via HTTP

func (*HTTPPipeline) ConstructBlockInfoQueue

func (f *HTTPPipeline) ConstructBlockInfoQueue(blockSize uint64) (partitionsQ chan pipeline.PartsPartition, partsQ chan pipeline.Part, numOfBlocks int, size uint64)

ConstructBlockInfoQueue implements GetSourcesInfo from the pipeline.SourcePipeline Interface. Constructs the Part's channel arithmetically from the size of the sources.

func (*HTTPPipeline) ExecuteReader

func (f *HTTPPipeline) ExecuteReader(partitionsQ chan pipeline.PartsPartition, partsQ chan pipeline.Part, readPartsQ chan pipeline.Part, id int, wg *sync.WaitGroup)

ExecuteReader implements ExecuteReader from the pipeline.SourcePipeline Interface. For each part the reader makes a byte range request to the source starting from the part's Offset to BytesToRead - 1 (zero based).

func (*HTTPPipeline) GetSourcesInfo

func (f *HTTPPipeline) GetSourcesInfo() []pipeline.SourceInfo

GetSourcesInfo implements GetSourcesInfo from the pipeline.SourcePipeline Interface. Returns an array of pipeline.SourceInfo[] with the files URL, alias and size.

type MultiFileParams

type MultiFileParams struct {
	SourcePatterns   []string
	BlockSize        uint64
	TargetAliases    []string
	NumOfPartitions  int
	MD5              bool
	FilesPerPipeline int
	KeepDirStructure bool
}

MultiFileParams parameters used to create a new instance of multi-file source pipeline

type MultiFilePipeline

type MultiFilePipeline struct {
	FilesInfo           map[string]FileInfo
	TotalNumberOfBlocks int
	TotalSize           uint64
	BlockSize           uint64
	NumOfPartitions     int
	// contains filtered or unexported fields
}

MultiFilePipeline Contructs blocks queue and implements data readers

func (*MultiFilePipeline) ConstructBlockInfoQueue

func (f *MultiFilePipeline) ConstructBlockInfoQueue(blockSize uint64) (partitionsQ chan pipeline.PartsPartition, partsQ chan pipeline.Part, numOfBlocks int, size uint64)

ConstructBlockInfoQueue implements ConstructBlockInfoQueue from the pipeline.SourcePipeline Interface. this implementation uses partitions to group parts into a set that can be read sequentially. This is to avoid Window's memory pressure when calling SetFilePointer numerous times on the same handle

func (*MultiFilePipeline) ExecuteReader

func (f *MultiFilePipeline) ExecuteReader(partitionsQ chan pipeline.PartsPartition, partsQ chan pipeline.Part, readPartsQ chan pipeline.Part, id int, wg *sync.WaitGroup)

ExecuteReader implements ExecuteReader from the pipeline.SourcePipeline Interface. For each file the reader will maintain a open handle from which data will be read. This implementation uses partitions (group of parts that can be read sequentially).

func (*MultiFilePipeline) GetSourcesInfo

func (f *MultiFilePipeline) GetSourcesInfo() []pipeline.SourceInfo

GetSourcesInfo implements GetSourcesInfo from the pipeline.SourcePipeline Interface. Returns an an array of SourceInfo with the name, alias and size of the files to be transferred.

type S3Params

type S3Params struct {
	SourceParams
	Bucket          string
	Endpoint        string
	Prefixes        []string
	PreSignedExpMin int
	AccessKey       string
	SecretKey       string
}

S3Params parameters used to create a new instance of a S3 source pipeline

type S3Pipeline

type S3Pipeline struct {
	HTTPPipeline
	// contains filtered or unexported fields
}

S3Pipeline S3 source HTTP based pipeline

type SourceParams

type SourceParams struct {
	CalculateMD5      bool
	UseExactNameMatch bool
	KeepDirStructure  bool
	FilesPerPipeline  int
}

SourceParams base parameters for HTTP based pipelines