storage

package
v0.0.0-...-8b28c38 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 20, 2023 License: Apache-2.0 Imports: 35 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// SSEKMS is the name of the SSE-KMS method for objectstore encryption.
	SSEKMS = "SSE-KMS"

	// SSEC is the name of the SSE-C method for objstore encryption.
	SSEC = "SSE-C"

	// SSES3 is the name of the SSE-S3 method for objstore encryption.
	SSES3 = "SSE-S3"
)
View Source 
const DirDelim = "/"

DirDelim is the delimiter used to model a directory structure in an object store bucket.

Variables

View Source 
var DoesNotExistError = os.ErrNotExist

DoesNotExistError is used as a generic error to return when a target path does not exist in storage. Equivalent to os.ErrorNotExist such that it will work with os.IsNotExist(err)

Functions

func DefaultAzureTransport 

func DefaultAzureTransport(config AzureConfig) (*http.Transport, error)

func DefaultS3Transport 

func DefaultS3Transport(config S3Config) (*http.Transport, error)

DefaultTransport - this default transport is based on the Minio DefaultTransport up until the following commit: https://githus3.com/minio/minio-go/commit/008c7aa71fc17e11bf980c209a4f8c4d687fc884 The values have since diverged.

func IsNotExist 

func IsNotExist(err error) bool

IsNotExist returns true if the error provided from a storage object is DoesNotExist

func NewTLSConfig 

func NewTLSConfig(cfg *TLSConfig) (*tls.Config, error)

NewTLSConfig creates a new tls.Config from the given TLSConfig.

func Validate 

func Validate(storage Storage) error

Validate uses the provided storage implementation to write a test file to the store, followed by a removal.

Types

type AzureConfig 

type AzureConfig struct {
	StorageAccountName string          `yaml:"storage_account"`
	StorageAccountKey  string          `yaml:"storage_account_key"`
	ContainerName      string          `yaml:"container"`
	Endpoint           string          `yaml:"endpoint"`
	MaxRetries         int             `yaml:"max_retries"`
	MSIResource        string          `yaml:"msi_resource"`
	UserAssignedID     string          `yaml:"user_assigned_id"`
	PipelineConfig     PipelineConfig  `yaml:"pipeline_config"`
	ReaderConfig       ReaderConfig    `yaml:"reader_config"`
	HTTPConfig         AzureHTTPConfig `yaml:"http_config"`
}

AzureConfig Azure storage configuration.

type AzureHTTPConfig 

type AzureHTTPConfig struct {
	IdleConnTimeout       model.Duration `yaml:"idle_conn_timeout"`
	ResponseHeaderTimeout model.Duration `yaml:"response_header_timeout"`
	InsecureSkipVerify    bool           `yaml:"insecure_skip_verify"`

	TLSHandshakeTimeout   model.Duration `yaml:"tls_handshake_timeout"`
	ExpectContinueTimeout model.Duration `yaml:"expect_continue_timeout"`
	MaxIdleConns          int            `yaml:"max_idle_conns"`
	MaxIdleConnsPerHost   int            `yaml:"max_idle_conns_per_host"`
	MaxConnsPerHost       int            `yaml:"max_conns_per_host"`
	DisableCompression    bool           `yaml:"disable_compression"`

	TLSConfig TLSConfig `yaml:"tls_config"`
}

type AzureStorage 

type AzureStorage struct {
	// contains filtered or unexported fields
}

AzureStorage implements the storage.Storage interface against Azure APIs.

func NewAzureStorage 

func NewAzureStorage(azureConfig []byte) (*AzureStorage, error)

NewAzureStorage returns a new Storage using the provided Azure config.

func NewAzureStorageWith 

func NewAzureStorageWith(conf AzureConfig) (*AzureStorage, error)

NewAzureStorageWith returns a new Storage using the provided Azure config struct.

func (*AzureStorage) Exists 

func (b *AzureStorage) Exists(name string) (bool, error)

Exists uses the relative path of the storage combined with the provided path to determine if the file exists.

func (*AzureStorage) FullPath 

func (as *AzureStorage) FullPath(name string) string

FullPath returns the storage working path combined with the path provided

func (*AzureStorage) List 

func (b *AzureStorage) List(path string) ([]*StorageInfo, error)

List uses the relative path of the storage combined with the provided path to return storage information for the files.

func (*AzureStorage) ListDirectories 

func (b *AzureStorage) ListDirectories(path string) ([]*StorageInfo, error)

func (*AzureStorage) Name 

func (as *AzureStorage) Name() string

Name returns the bucket name for azure storage.

func (*AzureStorage) Read 

func (b *AzureStorage) Read(name string) ([]byte, error)

Read uses the relative path of the storage combined with the provided path to read the contents.

func (*AzureStorage) Remove 

func (b *AzureStorage) Remove(name string) error

Remove uses the relative path of the storage combined with the provided path to remove a file from storage permanently.

func (*AzureStorage) Stat 

func (b *AzureStorage) Stat(name string) (*StorageInfo, error)

Stat returns the StorageStats for the specific path.

func (*AzureStorage) StorageType 

func (as *AzureStorage) StorageType() StorageType

StorageType returns a string identifier for the type of storage used by the implementation.

func (*AzureStorage) Write 

func (b *AzureStorage) Write(name string, data []byte) error

Write uses the relative path of the storage combined with the provided path to write a new file or overwrite an existing file.

type FileStorage 

type FileStorage struct {
	// contains filtered or unexported fields
}

FileStorage leverages the file system to write data to disk.

func (*FileStorage) Exists 

func (fs *FileStorage) Exists(path string) (bool, error)

Exists uses the relative path of the storage combined with the provided path to determine if the file exists.

func (*FileStorage) FullPath 

func (fs *FileStorage) FullPath(path string) string

FullPath returns the storage working path combined with the path provided

func (*FileStorage) List 

func (fs *FileStorage) List(path string) ([]*StorageInfo, error)

List uses the relative path of the storage combined with the provided path to return storage information for the files.

func (*FileStorage) ListDirectories 

func (fs *FileStorage) ListDirectories(path string) ([]*StorageInfo, error)

func (*FileStorage) Read 

func (fs *FileStorage) Read(path string) ([]byte, error)

Read uses the relative path of the storage combined with the provided path to read the contents.

func (*FileStorage) Remove 

func (fs *FileStorage) Remove(path string) error

Remove uses the relative path of the storage combined with the provided path to remove a file from storage permanently.

func (*FileStorage) Stat 

func (fs *FileStorage) Stat(path string) (*StorageInfo, error)

Stat returns the StorageStats for the specific path.

func (*FileStorage) StorageType 

func (fs *FileStorage) StorageType() StorageType

StorageType returns a string identifier for the type of storage used by the implementation.

func (*FileStorage) Write 

func (fs *FileStorage) Write(path string, data []byte) error

Write uses the relative path of the storage combined with the provided path to write a new file or overwrite an existing file.

type GCSConfig 

type GCSConfig struct {
	Bucket         string `yaml:"bucket"`
	ServiceAccount string `yaml:"service_account"`
}

Config stores the configuration for gcs bucket.

type GCSStorage 

type GCSStorage struct {
	// contains filtered or unexported fields
}

GCSStorage is a storage.Storage implementation for Google Cloud Storage.

func NewGCSStorage 

func NewGCSStorage(conf []byte) (*GCSStorage, error)

NewGCSStorage creates a new GCSStorage instance using the provided GCS configuration.

func NewGCSStorageWith 

func NewGCSStorageWith(gc GCSConfig) (*GCSStorage, error)

NewGCSStorageWith creates a new GCSStorage instance using the provided GCS configuration.

func (*GCSStorage) Exists 

func (gs *GCSStorage) Exists(name string) (bool, error)

Exists uses the relative path of the storage combined with the provided path to determine if the file exists.

func (*GCSStorage) FullPath 

func (gs *GCSStorage) FullPath(name string) string

FullPath returns the storage working path combined with the path provided

func (*GCSStorage) List 

func (gs *GCSStorage) List(path string) ([]*StorageInfo, error)

List uses the relative path of the storage combined with the provided path to return storage information for the files.

func (*GCSStorage) ListDirectories 

func (gs *GCSStorage) ListDirectories(path string) ([]*StorageInfo, error)

func (*GCSStorage) Name 

func (gs *GCSStorage) Name() string

Name returns the bucket name for gcs.

func (*GCSStorage) Read 

func (gs *GCSStorage) Read(name string) ([]byte, error)

Read uses the relative path of the storage combined with the provided path to read the contents.

func (*GCSStorage) Remove 

func (gs *GCSStorage) Remove(name string) error

Remove uses the relative path of the storage combined with the provided path to remove a file from storage permanently.

func (*GCSStorage) Stat 

func (gs *GCSStorage) Stat(name string) (*StorageInfo, error)

Stat returns the StorageStats for the specific path.

func (*GCSStorage) StorageType 

func (gs *GCSStorage) StorageType() StorageType

StorageType returns a string identifier for the type of storage used by the implementation.

func (*GCSStorage) Write 

func (gs *GCSStorage) Write(name string, data []byte) error

Write uses the relative path of the storage combined with the provided path to write a new file or overwrite an existing file.

type PipelineConfig 

type PipelineConfig struct {
	MaxTries      int32          `yaml:"max_tries"`
	TryTimeout    model.Duration `yaml:"try_timeout"`
	RetryDelay    model.Duration `yaml:"retry_delay"`
	MaxRetryDelay model.Duration `yaml:"max_retry_delay"`
}

type PrefixedBucketStorage 

type PrefixedBucketStorage struct {
	// contains filtered or unexported fields
}

func (*PrefixedBucketStorage) Exists 

func (pbs *PrefixedBucketStorage) Exists(name string) (bool, error)

Exists checks if the given object exists.

func (*PrefixedBucketStorage) FullPath 

func (pbs *PrefixedBucketStorage) FullPath(name string) string

FullPath returns the storage working path combined with the path provided.

func (*PrefixedBucketStorage) List 

func (pbs *PrefixedBucketStorage) List(path string) ([]*StorageInfo, error)

List returns storage information for files on the provided path.

func (*PrefixedBucketStorage) ListDirectories 

func (pbs *PrefixedBucketStorage) ListDirectories(path string) ([]*StorageInfo, error)

ListDirectories returns storage information for only directories on the provided path.

func (*PrefixedBucketStorage) Read 

func (pbs *PrefixedBucketStorage) Read(name string) ([]byte, error)

Read returns a reader for the given object name.

func (*PrefixedBucketStorage) Remove 

func (pbs *PrefixedBucketStorage) Remove(name string) error

Remove deletes the object with the given name.

func (*PrefixedBucketStorage) Stat 

func (pbs *PrefixedBucketStorage) Stat(name string) (*StorageInfo, error)

Stat returns information about the specified object.

func (*PrefixedBucketStorage) StorageType 

func (pbs *PrefixedBucketStorage) StorageType() StorageType

func (*PrefixedBucketStorage) Write 

func (pbs *PrefixedBucketStorage) Write(name string, data []byte) error

Write uploads the contents of the reader as an object into the bucket.

type ReaderConfig 

type ReaderConfig struct {
	MaxRetryRequests int `yaml:"max_retry_requests"`
}

type S3Config 

type S3Config struct {
	Bucket             string            `yaml:"bucket"`
	Endpoint           string            `yaml:"endpoint"`
	Region             string            `yaml:"region"`
	AWSSDKAuth         bool              `yaml:"aws_sdk_auth"`
	AccessKey          string            `yaml:"access_key"`
	Insecure           bool              `yaml:"insecure"`
	SignatureV2        bool              `yaml:"signature_version2"`
	SecretKey          string            `yaml:"secret_key"`
	PutUserMetadata    map[string]string `yaml:"put_user_metadata"`
	HTTPConfig         S3HTTPConfig      `yaml:"http_config"`
	TraceConfig        TraceConfig       `yaml:"trace"`
	ListObjectsVersion string            `yaml:"list_objects_version"`
	// PartSize used for multipart upload. Only used if uploaded object size is known and larger than configured PartSize.
	// NOTE we need to make sure this number does not produce more parts than 10 000.
	PartSize    uint64    `yaml:"part_size"`
	SSEConfig   SSEConfig `yaml:"sse_config"`
	STSEndpoint string    `yaml:"sts_endpoint"`
}

Config stores the configuration for s3 bucket.

type S3HTTPConfig 

type S3HTTPConfig struct {
	IdleConnTimeout       time.Duration `yaml:"idle_conn_timeout"`
	ResponseHeaderTimeout time.Duration `yaml:"response_header_timeout"`
	InsecureSkipVerify    bool          `yaml:"insecure_skip_verify"`

	TLSHandshakeTimeout   time.Duration `yaml:"tls_handshake_timeout"`
	ExpectContinueTimeout time.Duration `yaml:"expect_continue_timeout"`
	MaxIdleConns          int           `yaml:"max_idle_conns"`
	MaxIdleConnsPerHost   int           `yaml:"max_idle_conns_per_host"`
	MaxConnsPerHost       int           `yaml:"max_conns_per_host"`

	// Allow upstream callers to inject a round tripper
	Transport http.RoundTripper `yaml:"-"`

	TLSConfig TLSConfig `yaml:"tls_config"`
}

HTTPConfig stores the http.Transport configuration for the s3 minio client.

type S3Storage 

type S3Storage struct {
	// contains filtered or unexported fields
}

S3Storage provides storage via S3

func NewS3Storage 

func NewS3Storage(conf []byte) (*S3Storage, error)

NewBucket returns a new Bucket using the provided s3 config values.

func NewS3StorageWith 

func NewS3StorageWith(config S3Config) (*S3Storage, error)

NewBucketWithConfig returns a new Bucket using the provided s3 config values.

func (*S3Storage) Exists 

func (s3 *S3Storage) Exists(name string) (bool, error)

Exists checks if the given object exists.

func (*S3Storage) FullPath 

func (s3 *S3Storage) FullPath(name string) string

FullPath returns the storage working path combined with the path provided

func (*S3Storage) List 

func (s3 *S3Storage) List(path string) ([]*StorageInfo, error)

func (*S3Storage) ListDirectories 

func (s3 *S3Storage) ListDirectories(path string) ([]*StorageInfo, error)

func (*S3Storage) Name 

func (s3 *S3Storage) Name() string

Name returns the bucket name for s3.

func (*S3Storage) Read 

func (s3 *S3Storage) Read(name string) ([]byte, error)

Get returns a reader for the given object name.

func (*S3Storage) Remove 

func (s3 *S3Storage) Remove(name string) error

Delete removes the object with the given name.

func (*S3Storage) Stat 

func (s3 *S3Storage) Stat(name string) (*StorageInfo, error)

Attributes returns information about the specified object.

func (*S3Storage) StorageType 

func (s3 *S3Storage) StorageType() StorageType

StorageType returns a string identifier for the type of storage used by the implementation.

func (*S3Storage) Write 

func (s3 *S3Storage) Write(name string, data []byte) error

Upload the contents of the reader as an object into the bucket.

type SSEConfig 

type SSEConfig struct {
	Type                 string            `yaml:"type"`
	KMSKeyID             string            `yaml:"kms_key_id"`
	KMSEncryptionContext map[string]string `yaml:"kms_encryption_context"`
	EncryptionKey        string            `yaml:"encryption_key"`
}

SSEConfig deals with the configuration of SSE for Minio. The following options are valid: kmsencryptioncontext == https://docs.aws.amazon.com/kms/latest/developerguide/services-s3.html#s3-encryption-context

type Storage 

type Storage interface {
	// StorageType returns a string identifier for the type of storage used by the implementation.
	StorageType() StorageType

	// FullPath returns the storage working path combined with the path provided
	FullPath(path string) string

	// Stat returns the StorageStats for the specific path.
	Stat(path string) (*StorageInfo, error)

	// Read uses the relative path of the storage combined with the provided path to
	// read the contents.
	Read(path string) ([]byte, error)

	// Write uses the relative path of the storage combined with the provided path
	// to write a new file or overwrite an existing file.
	Write(path string, data []byte) error

	// Remove uses the relative path of the storage combined with the provided path to
	// remove a file from storage permanently.
	Remove(path string) error

	// Exists uses the relative path of the storage combined with the provided path to
	// determine if the file exists.
	Exists(path string) (bool, error)

	// List uses the relative path of the storage combined with the provided path to return
	// storage information for the files.
	List(path string) ([]*StorageInfo, error)

	// ListDirectories uses the relative path of the storage combined with the provided path
	// to return storage information for only directories contained along the path. This
	// functions as List, but returns storage information for only directories.
	ListDirectories(path string) ([]*StorageInfo, error)
}

Storage provides an API for storing binary data

func NewBucketStorage 

func NewBucketStorage(config []byte) (Storage, error)

NewBucketStorage initializes and returns new Storage implementation leveraging the storage provider configuration. This configuration type uses the layout provided in thanos: https://thanos.io/tip/thanos/storage.md/

func NewFileStorage 

func NewFileStorage(baseDir string) Storage

NewFileStorage returns a new storage API which leverages the file system.

func NewPrefixedBucketStorage 

func NewPrefixedBucketStorage(storage Storage, prefix string) (Storage, error)

type StorageConfig 

type StorageConfig struct {
	Type   StorageProvider `yaml:"type"`
	Config interface{}     `yaml:"config"`
	Prefix string          `yaml:"prefix"`
}

StorageConfig is the configuration type used as the "parent" configuration. It contains a type, which will specify the bucket storage implementation, and a configuration object specific to that storage implementation.

type StorageInfo 

type StorageInfo struct {
	Name    string    // base name of the file
	Size    int64     // length in bytes for regular files
	ModTime time.Time // modification time
}

StorageInfo is a data object containing basic information about the path in storage.

func DirFilesToStorageInfo 

func DirFilesToStorageInfo(fileInfo []gofs.FileInfo, path string) []*StorageInfo

DirFilesToStorageInfo maps a []fs.FileInfo to []*storage.StorageInfo but only returning StorageInfo for directories

func FileToStorageInfo 

func FileToStorageInfo(fileInfo gofs.FileInfo) *StorageInfo

FileToStorageInfo maps a fs.FileInfo to *storage.StorageInfo

func FilesToStorageInfo 

func FilesToStorageInfo(fileInfo []gofs.FileInfo) []*StorageInfo

FilesToStorageInfo maps a []fs.FileInfo to []*storage.StorageInfo

type StorageProvider 

type StorageProvider string

StorageProvider is the type of provider used for storage if not leveraging a file implementation. 

const (
	S3    StorageProvider = "S3"
	GCS   StorageProvider = "GCS"
	AZURE StorageProvider = "AZURE"
)

type StorageType 

type StorageType string

StorageType is a string identifier for the type of storage used by a Storage implementation. The string format is "backend|provider" where backend is the represents the generic storage facility, and the provider is the specific implementation. 

const (
	StorageTypeFile        StorageType = "file"
	StorageTypeBucketS3    StorageType = "bucket|s3"
	StorageTypeBucketGCS   StorageType = "bucket|gcs"
	StorageTypeBucketAzure StorageType = "bucket|azure"
)

func (StorageType) BackendType 

func (st StorageType) BackendType() string

BackendType returns the backend type if applicable for the storage type.

func (StorageType) IsBucketStorage 

func (st StorageType) IsBucketStorage() bool

IsBucketStorage returns true if the StorageType is a bucket storage type.

func (StorageType) IsFileStorage 

func (st StorageType) IsFileStorage() bool

IsFileStorage returns true if the StorageType is a file storage type.

func (StorageType) MarshalJSON 

func (st StorageType) MarshalJSON() ([]byte, error)

MarshalJSON implements the json.Marshaler interface for encoding a StorageType.

func (StorageType) ProviderType 

func (st StorageType) ProviderType() string

ProviderType returns the provider type if applicable for the storage type.

func (*StorageType) UnmarshalJSON 

func (st *StorageType) UnmarshalJSON(data []byte) error

UnmarshalJSON implements the json.Unmarshaler interface for decoding a StorageType.

type TLSConfig 

type TLSConfig struct {
	// The CA cert to use for the targets.
	CAFile string `yaml:"ca_file"`
	// The client cert file for the targets.
	CertFile string `yaml:"cert_file"`
	// The client key file for the targets.
	KeyFile string `yaml:"key_file"`
	// Used to verify the hostname for the targets.
	ServerName string `yaml:"server_name"`
	// Disable target certificate validation.
	InsecureSkipVerify bool `yaml:"insecure_skip_verify"`
}

TLSConfig configures the options for TLS connections.

type TraceConfig 

type TraceConfig struct {
	Enable bool `yaml:"enable"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.