s3v2

Documentation

Index

• func NewS3FileReader(ctx context.Context, bucket string, key string, cfgs ...*aws.Config) (source.ParquetFile, error)
• func NewS3FileReaderVersioned(ctx context.Context, bucket string, key string, version *string, ...) (source.ParquetFile, error)
• func NewS3FileReaderWithClient(ctx context.Context, s3Client S3API, bucket string, key string) (source.ParquetFile, error)
• func NewS3FileReaderWithClientVersioned(ctx context.Context, s3Client S3API, bucket string, key string, ...) (source.ParquetFile, error)
• func NewS3FileReaderWithParams(ctx context.Context, params S3FileReaderParams) (source.ParquetFile, error)
• func NewS3FileWriter(ctx context.Context, bucket string, key string, ...) (source.ParquetFile, error)
• func NewS3FileWriterWithClient(ctx context.Context, s3Client S3API, bucket string, key string, ...) (source.ParquetFile, error)
• type S3API
• type S3File
  • func (s *S3File) Close() error
  • func (s *S3File) Create(key string) (source.ParquetFile, error)
  • func (s *S3File) Open(name string) (source.ParquetFile, error)
  • func (s *S3File) Read(p []byte) (n int, err error)
  • func (s *S3File) Seek(offset int64, whence int) (int64, error)
  • func (s *S3File) Write(p []byte) (n int, err error)
• type S3FileReaderParams

Functions

func NewS3FileReader

func NewS3FileReader(ctx context.Context, bucket string, key string, cfgs ...*aws.Config) (source.ParquetFile, error)

NewS3FileReader creates an S3 FileReader, to be used with NewParquetReader

func NewS3FileReaderVersioned

func NewS3FileReaderVersioned(ctx context.Context, bucket string, key string, version *string, cfgs ...*aws.Config) (source.ParquetFile, error)

NewS3FileReaderVersioned creates an S3 FileReader for a versioned S3 object, to be used with NewParquetReader

func NewS3FileReaderWithClient

func NewS3FileReaderWithClient(ctx context.Context, s3Client S3API, bucket string, key string) (source.ParquetFile, error)

NewS3FileReaderWithClient is the same as NewS3FileReader but allows passing your own S3 client

func NewS3FileReaderWithClientVersioned

func NewS3FileReaderWithClientVersioned(ctx context.Context, s3Client S3API, bucket string, key string, version *string) (source.ParquetFile, error)

NewS3FileReaderWithClientVersioned is the same as NewS3FileReaderVersioned but allows passing your own S3 client

func NewS3FileReaderWithParams

func NewS3FileReaderWithParams(ctx context.Context, params S3FileReaderParams) (source.ParquetFile, error)

NewS3FileReaderWithParams creates an S3 FileReader for an object identified by and configured using the S3FileReaderParams object.

func NewS3FileWriter

func NewS3FileWriter(
	ctx context.Context,
	bucket string,
	key string,
	uploaderOptions []func(*manager.Uploader),
	cfgs ...*aws.Config,
) (source.ParquetFile, error)

NewS3FileWriter creates an S3 FileWriter, to be used with NewParquetWriter

func NewS3FileWriterWithClient

func NewS3FileWriterWithClient(
	ctx context.Context,
	s3Client S3API,
	bucket string,
	key string,
	uploaderOptions []func(*manager.Uploader),
	putObjectInputOptions ...func(*s3.PutObjectInput),
) (source.ParquetFile, error)

NewS3FileWriterWithClient is the same as NewS3FileWriter but allows passing your own S3 client.

Types

type S3API

type S3API interface {
	PutObject(context.Context, *s3.PutObjectInput, ...func(*s3.Options)) (*s3.PutObjectOutput, error)
	UploadPart(context.Context, *s3.UploadPartInput, ...func(*s3.Options)) (*s3.UploadPartOutput, error)
	CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput, ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error)
	CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput, ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error)
	AbortMultipartUpload(context.Context, *s3.AbortMultipartUploadInput, ...func(*s3.Options)) (*s3.AbortMultipartUploadOutput, error)
	GetObject(context.Context, *s3.GetObjectInput, ...func(*s3.Options)) (*s3.GetObjectOutput, error)
	HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
}

type S3File

type S3File struct {
	BucketName string
	Key        string
	// contains filtered or unexported fields
}

S3File is ParquetFile for AWS S3

func (*S3File) Close

func (s *S3File) Close() error

Close signals write completion and cleans up any open streams. Will block until pending uploads are complete.

func (*S3File) Create

func (s *S3File) Create(key string) (source.ParquetFile, error)

Create creates a new S3 File instance to perform writes

func (*S3File) Open

func (s *S3File) Open(name string) (source.ParquetFile, error)

Open creates a new S3 File instance to perform concurrent reads

func (*S3File) Read

func (s *S3File) Read(p []byte) (n int, err error)

Read up to len(p) bytes into p and return the number of bytes read

func (*S3File) Seek

func (s *S3File) Seek(offset int64, whence int) (int64, error)

Seek tracks the offset for the next Read. Has no effect on Write.

func (*S3File) Write

func (s *S3File) Write(p []byte) (n int, err error)

Write len(p) bytes from p to the S3 data stream

type S3FileReaderParams

type S3FileReaderParams struct {
	Bucket string
	Key    string

	// S3Client will be used to issue requests to S3. If not set, a new one will be
	// created. Optional.
	S3Client S3API
	// Version is the version of the S3 object that will be read. If not set, the newest
	// version will be read. Optional.
	Version *string
	// MinRequestSize controls the amount of data per request that the S3File will ask for
	// from S3. Optional.
	// A large MinRequestSize should improve performance and reduce AWS costs due to
	// number of request. However, in some cases it may increase AWS costs due to data
	// processing or data transfer. For best results, set it at or above the largest of
	// the footer size and the biggest chunk size in the parquet file.
	// S3File will not buffer a large amount of data in memory at one time, regardless
	// of the value of MinRequestSize.
	MinRequestSize int
}

S3FileReaderParams contains fields used to initialize and configure an S3File object for reading.