ioutils

package
v0.2.8 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jan 17, 2020 License: Apache-2.0 Imports: 8 Imported by: 0

Documentation

Index

Constants

View Source
const (
	InputsSuffix      = "inputs.pb"
	FuturesSuffix     = "futures.pb"
	OutputsSuffix     = "outputs.pb"
	ErrorsSuffix      = "error.pb"
	IndexLookupSuffix = "indexlookup.pb"
)
View Source
const (
	ErrFailedRead string = "READ_FAILED"
)

Variables

This section is empty.

Functions

func GetFuturesPath

func GetFuturesPath(ctx context.Context, store storage.ReferenceConstructor, prefix storage.DataReference) (res storage.DataReference, err error)

func GetIndexLookupPath

func GetIndexLookupPath(ctx context.Context, store storage.ReferenceConstructor, prefix storage.DataReference) (res storage.DataReference, err error)

func GetMasterOutputsPath

func GetMasterOutputsPath(ctx context.Context, store storage.ReferenceConstructor, output storage.DataReference) (res storage.DataReference, err error)

func GetOutputsPath

func GetOutputsPath(ctx context.Context, store storage.ReferenceConstructor, prefix storage.DataReference) (res storage.DataReference, err error)

func GetPath

func GetPath(ctx context.Context, store storage.ReferenceConstructor, root storage.DataReference, subNames ...string) (res storage.DataReference, err error)

func NewCachedInputReader

func NewCachedInputReader(ctx context.Context, in io.InputReader) io.InputReader

Creates a new Read-through cached Input Reader. the returned reader is not thread-safe It caches the inputs on a successful read from the underlying input reader

Types

type BufferedOutputWriter

type BufferedOutputWriter struct {
	io.OutputFilePaths
	// contains filtered or unexported fields
}

A Buffered outputWriter just records the io.OutputReader and can be accessed using special methods.

func NewBufferedOutputWriter

func NewBufferedOutputWriter(ctx context.Context, paths io.OutputFilePaths) *BufferedOutputWriter

Returns a new object of type BufferedOutputWriter

func (*BufferedOutputWriter) GetReader

func (o *BufferedOutputWriter) GetReader() io.OutputReader

func (*BufferedOutputWriter) Put

type InMemoryOutputReader

type InMemoryOutputReader struct {
	// contains filtered or unexported fields
}

func NewInMemoryOutputReader

func NewInMemoryOutputReader(literals *core.LiteralMap, err *io.ExecutionError) InMemoryOutputReader

func (InMemoryOutputReader) Exists

func (r InMemoryOutputReader) Exists(ctx context.Context) (bool, error)

func (InMemoryOutputReader) IsError

func (r InMemoryOutputReader) IsError(ctx context.Context) (bool, error)

func (InMemoryOutputReader) IsFile

func (r InMemoryOutputReader) IsFile(ctx context.Context) bool

func (InMemoryOutputReader) Read

func (InMemoryOutputReader) ReadError

type RemoteFileInputReader

type RemoteFileInputReader struct {
	io.InputFilePaths
	// contains filtered or unexported fields
}

func NewRemoteFileInputReader

func NewRemoteFileInputReader(_ context.Context, store storage.ProtobufStore, inputPaths io.InputFilePaths) RemoteFileInputReader

func (RemoteFileInputReader) Get

type RemoteFileOutputPaths

type RemoteFileOutputPaths struct {
	// contains filtered or unexported fields
}

func (RemoteFileOutputPaths) GetErrorPath

func (w RemoteFileOutputPaths) GetErrorPath() storage.DataReference

func (RemoteFileOutputPaths) GetFuturesPath

func (w RemoteFileOutputPaths) GetFuturesPath() storage.DataReference

func (RemoteFileOutputPaths) GetOutputPath

func (w RemoteFileOutputPaths) GetOutputPath() storage.DataReference

func (RemoteFileOutputPaths) GetOutputPrefixPath

func (w RemoteFileOutputPaths) GetOutputPrefixPath() storage.DataReference

type RemoteFileOutputReader

type RemoteFileOutputReader struct {
	// contains filtered or unexported fields
}

func NewRemoteFileOutputReader

func NewRemoteFileOutputReader(_ context.Context, store storage.ComposedProtobufStore, outPaths io.OutputFilePaths, maxDatasetSize int64) RemoteFileOutputReader

func (RemoteFileOutputReader) Exists

func (r RemoteFileOutputReader) Exists(ctx context.Context) (bool, error)

func (RemoteFileOutputReader) IsError

func (r RemoteFileOutputReader) IsError(ctx context.Context) (bool, error)

func (RemoteFileOutputReader) IsFile

func (RemoteFileOutputReader) Read

func (RemoteFileOutputReader) ReadError

type RemoteFileOutputWriter

type RemoteFileOutputWriter struct {
	io.OutputFilePaths
	// contains filtered or unexported fields
}

func NewRemoteFileOutputWriter

func NewRemoteFileOutputWriter(_ context.Context, store storage.ProtobufStore, outputFilePaths io.OutputFilePaths) RemoteFileOutputWriter

func (RemoteFileOutputWriter) Put

type SimpleInputFilePath

type SimpleInputFilePath struct {
	// contains filtered or unexported fields
}

func (SimpleInputFilePath) GetInputPath

func (s SimpleInputFilePath) GetInputPath() storage.DataReference

func (SimpleInputFilePath) GetInputPrefixPath

func (s SimpleInputFilePath) GetInputPrefixPath() storage.DataReference

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL