ioutils

package
Version: v0.5.28 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 4, 2021 License: Apache-2.0 Imports: 13 Imported by: 0

Documentation

Index

Constants

View Source
const (
	InputsSuffix      = "inputs.pb"
	FuturesSuffix     = "futures.pb"
	OutputsSuffix     = "outputs.pb"
	ErrorsSuffix      = "error.pb"
	IndexLookupSuffix = "indexlookup.pb"
)
View Source
const (
	ErrFailedRead string = "READ_FAILED"
)

Variables

This section is empty.

Functions

func GenerateAlphabet added in v0.3.15

func GenerateAlphabet(b []rune) []rune

Generates the entire latin alphabet and appends it to the passed in array and returns the new array

func GenerateArabicNumerals added in v0.3.15

func GenerateArabicNumerals(b []rune) []rune

Generates all arabic numerals and appends to the passed in array and returns the new array/slice

func GetFuturesPath

func GetFuturesPath(ctx context.Context, store storage.ReferenceConstructor, prefix storage.DataReference) (res storage.DataReference, err error)

func GetIndexLookupPath

func GetIndexLookupPath(ctx context.Context, store storage.ReferenceConstructor, prefix storage.DataReference) (res storage.DataReference, err error)

func GetMasterOutputsPath

func GetMasterOutputsPath(ctx context.Context, store storage.ReferenceConstructor, output storage.DataReference) (res storage.DataReference, err error)

func GetOutputsPath

func GetOutputsPath(ctx context.Context, store storage.ReferenceConstructor, prefix storage.DataReference) (res storage.DataReference, err error)

func GetPath

func GetPath(ctx context.Context, store storage.ReferenceConstructor, root storage.DataReference, subNames ...string) (res storage.DataReference, err error)

func NewCachedInputReader

func NewCachedInputReader(ctx context.Context, in io.InputReader) io.InputReader

Creates a new Read-through cached Input Reader. the returned reader is not thread-safe It caches the inputs on a successful read from the underlying input reader

func NewDeterministicUniqueRawOutputPath added in v0.4.6

func NewDeterministicUniqueRawOutputPath(ctx context.Context, rawOutputPrefix, outputMetadataPrefix storage.DataReference, store storage.ReferenceConstructor) (io.RawOutputPaths, error)

Constructs an output path that is deterministic and unique within the given outputPrefix. No sharding is performed

func NewRawOutputPaths added in v0.3.15

func NewRawOutputPaths(_ context.Context, rawOutputPrefix storage.DataReference) io.RawOutputPaths

A simple Output sandbox at a given path

func NewShardedDeterministicRawOutputPath added in v0.3.15

func NewShardedDeterministicRawOutputPath(ctx context.Context, sharder ShardSelector, basePrefix, outputMetadataPrefix storage.DataReference, store storage.ReferenceConstructor) (io.RawOutputPaths, error)

Creates a deterministic RawOutputPath whose path is distributed based on the ShardSelector passed in. Determinism depends on the outputMetadataPath Potential performance problem, as creating a new RawPath creation may be expensive as it hashes the outputMetadataPath the final RawOutputPath is created in the shard selected by the sharder at the basePath and then appended by a hashed value of the outputMetadata

func NewShardedRawOutputPath added in v0.3.15

func NewShardedRawOutputPath(ctx context.Context, sharder ShardSelector, basePath storage.DataReference, uniqueID string, store storage.ReferenceConstructor) (io.RawOutputPaths, error)

Creates an OutputSandbox in the basePath using the uniqueID and a sharder This implementation is faster than the Randomized strategy

func NewTaskIDRawOutputPath added in v0.4.6

func NewTaskIDRawOutputPath(ctx context.Context, rawOutputPrefix storage.DataReference, taskID *core2.TaskExecutionIdentifier, store storage.ReferenceConstructor) (io.RawOutputPaths, error)

Generates a RawOutput Path that looks like the TaskExecutionID and can be easily cross referenced with Flyte generated TaskExecution ID

Types

type BufferedOutputWriter

type BufferedOutputWriter struct {
	io.OutputFilePaths
	// contains filtered or unexported fields
}

A Buffered outputWriter just records the io.OutputReader and can be accessed using special methods.

func NewBufferedOutputWriter

func NewBufferedOutputWriter(ctx context.Context, paths io.OutputFilePaths) *BufferedOutputWriter

Returns a new object of type BufferedOutputWriter

func (*BufferedOutputWriter) GetReader

func (o *BufferedOutputWriter) GetReader() io.OutputReader

func (*BufferedOutputWriter) Put

type InMemoryOutputReader

type InMemoryOutputReader struct {
	// contains filtered or unexported fields
}

func NewInMemoryOutputReader

func NewInMemoryOutputReader(literals *core.LiteralMap, err *io.ExecutionError) InMemoryOutputReader

func (InMemoryOutputReader) Exists

func (r InMemoryOutputReader) Exists(ctx context.Context) (bool, error)

func (InMemoryOutputReader) IsError

func (r InMemoryOutputReader) IsError(ctx context.Context) (bool, error)

func (InMemoryOutputReader) IsFile

func (r InMemoryOutputReader) IsFile(ctx context.Context) bool

func (InMemoryOutputReader) Read

func (InMemoryOutputReader) ReadError

type PrecomputedShardSelector added in v0.3.15

type PrecomputedShardSelector struct {
	// contains filtered or unexported fields
}

this sharder distributes data into one of the precomputed buckets. The bucket is deterministically determined given the input s

func (*PrecomputedShardSelector) GetShardPrefix added in v0.3.15

func (d *PrecomputedShardSelector) GetShardPrefix(_ context.Context, s []byte) (string, error)

Generates deterministic shard id for the given string s

type RemoteFileInputReader

type RemoteFileInputReader struct {
	io.InputFilePaths
	// contains filtered or unexported fields
}

func NewRemoteFileInputReader

func NewRemoteFileInputReader(_ context.Context, store storage.ProtobufStore, inputPaths io.InputFilePaths) RemoteFileInputReader

func (RemoteFileInputReader) Get

type RemoteFileOutputPaths

type RemoteFileOutputPaths struct {
	io.RawOutputPaths
	// contains filtered or unexported fields
}

func (RemoteFileOutputPaths) GetErrorPath

func (w RemoteFileOutputPaths) GetErrorPath() storage.DataReference

func (RemoteFileOutputPaths) GetFuturesPath

func (w RemoteFileOutputPaths) GetFuturesPath() storage.DataReference

func (RemoteFileOutputPaths) GetOutputPath

func (w RemoteFileOutputPaths) GetOutputPath() storage.DataReference

func (RemoteFileOutputPaths) GetOutputPrefixPath

func (w RemoteFileOutputPaths) GetOutputPrefixPath() storage.DataReference

type RemoteFileOutputReader

type RemoteFileOutputReader struct {
	// contains filtered or unexported fields
}

func NewRemoteFileOutputReader

func NewRemoteFileOutputReader(_ context.Context, store storage.ComposedProtobufStore, outPaths io.OutputFilePaths, maxDatasetSize int64) RemoteFileOutputReader

func (RemoteFileOutputReader) Exists

func (r RemoteFileOutputReader) Exists(ctx context.Context) (bool, error)

func (RemoteFileOutputReader) IsError

func (r RemoteFileOutputReader) IsError(ctx context.Context) (bool, error)

func (RemoteFileOutputReader) IsFile

func (RemoteFileOutputReader) Read

func (RemoteFileOutputReader) ReadError

type RemoteFileOutputWriter

type RemoteFileOutputWriter struct {
	io.OutputFilePaths
	// contains filtered or unexported fields
}

func NewRemoteFileOutputWriter

func NewRemoteFileOutputWriter(_ context.Context, store storage.ProtobufStore, outputFilePaths io.OutputFilePaths) RemoteFileOutputWriter

func (RemoteFileOutputWriter) Put

type ShardSelector added in v0.3.15

type ShardSelector interface {
	GetShardPrefix(ctx context.Context, s []byte) (string, error)
}

This interface allows shard selection for OutputSandbox.

func NewBase36PrefixShardSelector added in v0.3.15

func NewBase36PrefixShardSelector(ctx context.Context) (ShardSelector, error)

Creates a PrecomputedShardSelector with 36*36 unique shards. Each shard is of the format {[0-9a-z][0-9a-z]}, i.e. 2 character long.

func NewConstantShardSelector added in v0.3.15

func NewConstantShardSelector(shards []string) ShardSelector

uses the given shards to select a shard

type SimpleInputFilePath

type SimpleInputFilePath struct {
	// contains filtered or unexported fields
}

func (SimpleInputFilePath) GetInputPath

func (s SimpleInputFilePath) GetInputPath() storage.DataReference

func (SimpleInputFilePath) GetInputPrefixPath

func (s SimpleInputFilePath) GetInputPrefixPath() storage.DataReference

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL