storage

package
v0.5.8 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 28, 2025 License: Apache-2.0 Imports: 41 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func CreateBucketIfNotExist added in v0.5.7

func CreateBucketIfNotExist(ctx context.Context, cli Client, prefix string) error

func DeletePrefix added in v0.5.7

func DeletePrefix(ctx context.Context, cli Client, prefix string) error

func Exist added in v0.5.7

func Exist(ctx context.Context, cli Client, prefix string) (bool, error)

func ListPrefixFlat added in v0.5.7

func ListPrefixFlat(ctx context.Context, cli Client, prefix string, recursive bool) ([]string, []int64, error)

func Read

func Read(ctx context.Context, cli Client, key string) ([]byte, error)

func Write added in v0.5.7

func Write(ctx context.Context, cli Client, key string, body []byte) error

Types

type AzureClient added in v0.5.7

type AzureClient struct {
	// contains filtered or unexported fields
}

func (*AzureClient) BucketExist added in v0.5.7

func (a *AzureClient) BucketExist(ctx context.Context, prefix string) (bool, error)

func (*AzureClient) Config added in v0.5.7

func (a *AzureClient) Config() Config

func (*AzureClient) CopyObject added in v0.5.7

func (a *AzureClient) CopyObject(ctx context.Context, i CopyObjectInput) error

func (*AzureClient) CreateBucket added in v0.5.7

func (a *AzureClient) CreateBucket(ctx context.Context) error

func (*AzureClient) DeleteObject added in v0.5.7

func (a *AzureClient) DeleteObject(ctx context.Context, prefix string) error

func (*AzureClient) GetObject added in v0.5.7

func (a *AzureClient) GetObject(ctx context.Context, key string) (*Object, error)

func (*AzureClient) HeadObject added in v0.5.7

func (a *AzureClient) HeadObject(ctx context.Context, key string) (ObjectAttr, error)

func (*AzureClient) ListPrefix added in v0.5.7

func (a *AzureClient) ListPrefix(_ context.Context, prefix string, recursive bool) (ObjectIterator, error)

func (*AzureClient) UploadObject added in v0.5.7

func (a *AzureClient) UploadObject(ctx context.Context, i UploadObjectInput) error

type AzureObjectFlatIterator added in v0.5.7

type AzureObjectFlatIterator struct {
	// contains filtered or unexported fields
}

func (*AzureObjectFlatIterator) HasNext added in v0.5.7

func (flatIter *AzureObjectFlatIterator) HasNext() bool

func (*AzureObjectFlatIterator) Next added in v0.5.7

func (flatIter *AzureObjectFlatIterator) Next() (ObjectAttr, error)

type AzureObjectHierarchyIterator added in v0.5.7

type AzureObjectHierarchyIterator struct {
	// contains filtered or unexported fields
}

func (*AzureObjectHierarchyIterator) HasNext added in v0.5.7

func (hierIter *AzureObjectHierarchyIterator) HasNext() bool

func (*AzureObjectHierarchyIterator) Next added in v0.5.7

func (hierIter *AzureObjectHierarchyIterator) Next() (ObjectAttr, error)

type AzureReader added in v0.5.7

type AzureReader struct {
	// contains filtered or unexported fields
}

func (*AzureReader) Close added in v0.5.7

func (a *AzureReader) Close() error

func (*AzureReader) Read added in v0.5.7

func (a *AzureReader) Read(p []byte) (int, error)

type Client added in v0.5.7

type Client interface {
	Config() Config

	// CopyObject copy an object from src to dest, call on dest client.
	// The implementation of CopyObject must directly use the copy API provided by the service provider,
	CopyObject(ctx context.Context, i CopyObjectInput) error
	// HeadObject determine if an object exists, and you have permission to access it.
	HeadObject(ctx context.Context, key string) (ObjectAttr, error)
	// GetObject get an object
	GetObject(ctx context.Context, key string) (*Object, error)
	// UploadObject stream upload an object
	UploadObject(ctx context.Context, i UploadObjectInput) error
	// DeleteObject delete an object
	DeleteObject(ctx context.Context, key string) error

	// ListPrefix list all objects with same prefix, and call WalkFunc for each object.
	ListPrefix(ctx context.Context, prefix string, recursive bool) (ObjectIterator, error)

	// BucketExist use a prefix to chack if bucket exist.
	// Using a prefix to confirm whether a bucket exists can avoid requesting the head Bucket permission.
	BucketExist(ctx context.Context, prefix string) (bool, error)
	// CreateBucket create a bucket.
	CreateBucket(ctx context.Context) error
}

Client is the interface for storage service. All implementations should include retry logic internally for idempotent operations.

func NewBackupStorage added in v0.5.7

func NewBackupStorage(ctx context.Context, params *paramtable.BackupParams) (Client, error)

func NewClient added in v0.5.7

func NewClient(ctx context.Context, cfg Config) (Client, error)

func NewMilvusStorage added in v0.5.7

func NewMilvusStorage(ctx context.Context, params *paramtable.BackupParams) (Client, error)

type Config added in v0.5.7

type Config struct {
	Provider string

	Endpoint string
	UseSSL   bool
	Region   string

	Credential Credential

	Bucket string
}

type CopyAttr added in v0.5.4

type CopyAttr struct {
	Src     ObjectAttr
	DestKey string
}

type CopyObjectInput added in v0.4.21

type CopyObjectInput struct {
	SrcCli  Client
	SrcKey  string
	DestKey string
}

type CopyObjectsOpt added in v0.5.7

type CopyObjectsOpt struct {
	Src  Client
	Dest Client

	Attrs []CopyAttr

	Sem *semaphore.Weighted

	CopyByServer bool
}

type CopyObjectsTask added in v0.5.7

type CopyObjectsTask struct {
	// contains filtered or unexported fields
}

func NewCopyObjectsTask added in v0.5.7

func NewCopyObjectsTask(opt CopyObjectsOpt) *CopyObjectsTask

func (*CopyObjectsTask) Execute added in v0.5.7

func (c *CopyObjectsTask) Execute(ctx context.Context) error

func (*CopyObjectsTask) Prepare added in v0.5.7

func (c *CopyObjectsTask) Prepare(_ context.Context) error

type CopyPrefixOpt added in v0.5.7

type CopyPrefixOpt struct {
	Src  Client
	Dest Client

	SrcPrefix  string
	DestPrefix string

	Sem *semaphore.Weighted

	TraceFn TraceFn

	CopyByServer bool
}

type CopyPrefixTask added in v0.5.7

type CopyPrefixTask struct {
	// contains filtered or unexported fields
}

func NewCopyPrefixTask added in v0.5.7

func NewCopyPrefixTask(opt CopyPrefixOpt) *CopyPrefixTask

func (*CopyPrefixTask) Execute added in v0.5.7

func (c *CopyPrefixTask) Execute(ctx context.Context) error

type Credential added in v0.5.7

type Credential struct {
	Type CredentialType

	// Static credential
	AK    string
	SK    string
	Token string

	// IAM
	IAMEndpoint string

	// GCPCredJSON
	GCPCredJSON string

	// MinioCredential
	MinioCredProvider minioCred.Provider

	// Azure Specific
	AzureAccountName string
}

type CredentialType added in v0.5.7

type CredentialType uint8
const (
	Unknown CredentialType = iota
	Static
	IAM

	// GCPCredJSON For GCPNative storage. pass the json file path to the GCPNative storage.
	GCPCredJSON
	// MinioCredProvider For S3 compatible storage (now only support minio, aws),
	// pass a struct which implements minioCred.Provider
	MinioCredProvider
)

func (CredentialType) String added in v0.5.7

func (c CredentialType) String() string

type GCPNativeClient added in v0.5.7

type GCPNativeClient struct {
	// contains filtered or unexported fields
}

func (*GCPNativeClient) BucketExist added in v0.5.7

func (gcm *GCPNativeClient) BucketExist(ctx context.Context, _ string) (bool, error)

func (*GCPNativeClient) Config added in v0.5.7

func (gcm *GCPNativeClient) Config() Config

func (*GCPNativeClient) CopyObject added in v0.5.7

func (gcm *GCPNativeClient) CopyObject(ctx context.Context, i CopyObjectInput) error

func (*GCPNativeClient) CreateBucket added in v0.5.7

func (gcm *GCPNativeClient) CreateBucket(ctx context.Context) error

func (*GCPNativeClient) DeleteObject added in v0.5.7

func (gcm *GCPNativeClient) DeleteObject(ctx context.Context, key string) error

func (*GCPNativeClient) GetObject added in v0.5.7

func (gcm *GCPNativeClient) GetObject(ctx context.Context, key string) (*Object, error)

func (*GCPNativeClient) HeadObject added in v0.5.7

func (gcm *GCPNativeClient) HeadObject(ctx context.Context, key string) (ObjectAttr, error)

func (*GCPNativeClient) ListPrefix added in v0.5.7

func (gcm *GCPNativeClient) ListPrefix(ctx context.Context, prefix string, recursive bool) (ObjectIterator, error)

func (*GCPNativeClient) UploadObject added in v0.5.7

func (gcm *GCPNativeClient) UploadObject(ctx context.Context, i UploadObjectInput) error

type GcpNativeObjectIterator added in v0.5.7

type GcpNativeObjectIterator struct {
	// contains filtered or unexported fields
}

func (*GcpNativeObjectIterator) HasNext added in v0.5.7

func (g *GcpNativeObjectIterator) HasNext() bool

func (*GcpNativeObjectIterator) Next added in v0.5.7

type LocalClient added in v0.5.7

type LocalClient struct {
	// contains filtered or unexported fields
}

func (*LocalClient) BucketExist added in v0.5.7

func (l *LocalClient) BucketExist(_ context.Context, _ string) (bool, error)

func (*LocalClient) Config added in v0.5.7

func (l *LocalClient) Config() Config

func (*LocalClient) CopyObject added in v0.5.7

func (l *LocalClient) CopyObject(_ context.Context, i CopyObjectInput) error

func (*LocalClient) CreateBucket added in v0.5.7

func (l *LocalClient) CreateBucket(_ context.Context) error

func (*LocalClient) DeleteObject added in v0.5.7

func (l *LocalClient) DeleteObject(_ context.Context, key string) error

func (*LocalClient) GetObject added in v0.5.7

func (l *LocalClient) GetObject(_ context.Context, key string) (*Object, error)

func (*LocalClient) HeadObject added in v0.5.7

func (l *LocalClient) HeadObject(_ context.Context, key string) (ObjectAttr, error)

func (*LocalClient) ListPrefix added in v0.5.7

func (l *LocalClient) ListPrefix(_ context.Context, prefix string, recursive bool) (ObjectIterator, error)

func (*LocalClient) UploadObject added in v0.5.7

func (l *LocalClient) UploadObject(_ context.Context, i UploadObjectInput) error

type MinioClient added in v0.5.7

type MinioClient struct {
	// contains filtered or unexported fields
}

func (*MinioClient) BucketExist added in v0.5.7

func (m *MinioClient) BucketExist(ctx context.Context, prefix string) (bool, error)

func (*MinioClient) Config added in v0.5.7

func (m *MinioClient) Config() Config

func (*MinioClient) CopyObject added in v0.5.7

func (m *MinioClient) CopyObject(ctx context.Context, i CopyObjectInput) error

func (*MinioClient) CreateBucket added in v0.5.7

func (m *MinioClient) CreateBucket(ctx context.Context) error

func (*MinioClient) DeleteObject added in v0.5.7

func (m *MinioClient) DeleteObject(ctx context.Context, key string) error

func (*MinioClient) GetObject added in v0.5.7

func (m *MinioClient) GetObject(ctx context.Context, key string) (*Object, error)

func (*MinioClient) HeadObject added in v0.5.7

func (m *MinioClient) HeadObject(ctx context.Context, key string) (ObjectAttr, error)

func (*MinioClient) ListPrefix added in v0.5.7

func (m *MinioClient) ListPrefix(ctx context.Context, prefix string, recursive bool) (ObjectIterator, error)

func (*MinioClient) UploadObject added in v0.5.7

func (m *MinioClient) UploadObject(ctx context.Context, i UploadObjectInput) error

type MinioObjectIterator added in v0.5.7

type MinioObjectIterator struct {
	// contains filtered or unexported fields
}

func (*MinioObjectIterator) HasNext added in v0.5.7

func (m *MinioObjectIterator) HasNext() bool

func (*MinioObjectIterator) Next added in v0.5.7

func (m *MinioObjectIterator) Next() (ObjectAttr, error)

type MockClient added in v0.5.7

type MockClient struct {
	mock.Mock
}

MockClient is an autogenerated mock type for the Client type

func NewMockClient added in v0.5.7

func NewMockClient(t interface {
	mock.TestingT
	Cleanup(func())
}) *MockClient

NewMockClient creates a new instance of MockClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. The first argument is typically a *testing.T value.

func (*MockClient) BucketExist added in v0.5.7

func (_mock *MockClient) BucketExist(ctx context.Context, prefix string) (bool, error)

BucketExist provides a mock function for the type MockClient

func (*MockClient) Config added in v0.5.7

func (_mock *MockClient) Config() Config

Config provides a mock function for the type MockClient

func (*MockClient) CopyObject added in v0.5.7

func (_mock *MockClient) CopyObject(ctx context.Context, i CopyObjectInput) error

CopyObject provides a mock function for the type MockClient

func (*MockClient) CreateBucket added in v0.5.7

func (_mock *MockClient) CreateBucket(ctx context.Context) error

CreateBucket provides a mock function for the type MockClient

func (*MockClient) DeleteObject added in v0.5.7

func (_mock *MockClient) DeleteObject(ctx context.Context, key string) error

DeleteObject provides a mock function for the type MockClient

func (*MockClient) EXPECT added in v0.5.7

func (_m *MockClient) EXPECT() *MockClient_Expecter

func (*MockClient) GetObject added in v0.5.7

func (_mock *MockClient) GetObject(ctx context.Context, key string) (*Object, error)

GetObject provides a mock function for the type MockClient

func (*MockClient) HeadObject added in v0.5.7

func (_mock *MockClient) HeadObject(ctx context.Context, key string) (ObjectAttr, error)

HeadObject provides a mock function for the type MockClient

func (*MockClient) ListPrefix added in v0.5.7

func (_mock *MockClient) ListPrefix(ctx context.Context, prefix string, recursive bool) (ObjectIterator, error)

ListPrefix provides a mock function for the type MockClient

func (*MockClient) UploadObject added in v0.5.7

func (_mock *MockClient) UploadObject(ctx context.Context, i UploadObjectInput) error

UploadObject provides a mock function for the type MockClient

type MockClient_BucketExist_Call added in v0.5.7

type MockClient_BucketExist_Call struct {
	*mock.Call
}

MockClient_BucketExist_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'BucketExist'

func (*MockClient_BucketExist_Call) Return added in v0.5.7

func (*MockClient_BucketExist_Call) Run added in v0.5.7

func (*MockClient_BucketExist_Call) RunAndReturn added in v0.5.7

func (_c *MockClient_BucketExist_Call) RunAndReturn(run func(ctx context.Context, prefix string) (bool, error)) *MockClient_BucketExist_Call

type MockClient_Config_Call added in v0.5.7

type MockClient_Config_Call struct {
	*mock.Call
}

MockClient_Config_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Config'

func (*MockClient_Config_Call) Return added in v0.5.7

func (*MockClient_Config_Call) Run added in v0.5.7

func (_c *MockClient_Config_Call) Run(run func()) *MockClient_Config_Call

func (*MockClient_Config_Call) RunAndReturn added in v0.5.7

func (_c *MockClient_Config_Call) RunAndReturn(run func() Config) *MockClient_Config_Call

type MockClient_CopyObject_Call added in v0.5.7

type MockClient_CopyObject_Call struct {
	*mock.Call
}

MockClient_CopyObject_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CopyObject'

func (*MockClient_CopyObject_Call) Return added in v0.5.7

func (*MockClient_CopyObject_Call) Run added in v0.5.7

func (*MockClient_CopyObject_Call) RunAndReturn added in v0.5.7

type MockClient_CreateBucket_Call added in v0.5.7

type MockClient_CreateBucket_Call struct {
	*mock.Call
}

MockClient_CreateBucket_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateBucket'

func (*MockClient_CreateBucket_Call) Return added in v0.5.7

func (*MockClient_CreateBucket_Call) Run added in v0.5.7

func (*MockClient_CreateBucket_Call) RunAndReturn added in v0.5.7

type MockClient_DeleteObject_Call added in v0.5.7

type MockClient_DeleteObject_Call struct {
	*mock.Call
}

MockClient_DeleteObject_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteObject'

func (*MockClient_DeleteObject_Call) Return added in v0.5.7

func (*MockClient_DeleteObject_Call) Run added in v0.5.7

func (*MockClient_DeleteObject_Call) RunAndReturn added in v0.5.7

type MockClient_Expecter added in v0.5.7

type MockClient_Expecter struct {
	// contains filtered or unexported fields
}

func (*MockClient_Expecter) BucketExist added in v0.5.7

func (_e *MockClient_Expecter) BucketExist(ctx interface{}, prefix interface{}) *MockClient_BucketExist_Call

BucketExist is a helper method to define mock.On call

  • ctx context.Context
  • prefix string

func (*MockClient_Expecter) Config added in v0.5.7

Config is a helper method to define mock.On call

func (*MockClient_Expecter) CopyObject added in v0.5.7

func (_e *MockClient_Expecter) CopyObject(ctx interface{}, i interface{}) *MockClient_CopyObject_Call

CopyObject is a helper method to define mock.On call

  • ctx context.Context
  • i CopyObjectInput

func (*MockClient_Expecter) CreateBucket added in v0.5.7

func (_e *MockClient_Expecter) CreateBucket(ctx interface{}) *MockClient_CreateBucket_Call

CreateBucket is a helper method to define mock.On call

  • ctx context.Context

func (*MockClient_Expecter) DeleteObject added in v0.5.7

func (_e *MockClient_Expecter) DeleteObject(ctx interface{}, key interface{}) *MockClient_DeleteObject_Call

DeleteObject is a helper method to define mock.On call

  • ctx context.Context
  • key string

func (*MockClient_Expecter) GetObject added in v0.5.7

func (_e *MockClient_Expecter) GetObject(ctx interface{}, key interface{}) *MockClient_GetObject_Call

GetObject is a helper method to define mock.On call

  • ctx context.Context
  • key string

func (*MockClient_Expecter) HeadObject added in v0.5.7

func (_e *MockClient_Expecter) HeadObject(ctx interface{}, key interface{}) *MockClient_HeadObject_Call

HeadObject is a helper method to define mock.On call

  • ctx context.Context
  • key string

func (*MockClient_Expecter) ListPrefix added in v0.5.7

func (_e *MockClient_Expecter) ListPrefix(ctx interface{}, prefix interface{}, recursive interface{}) *MockClient_ListPrefix_Call

ListPrefix is a helper method to define mock.On call

  • ctx context.Context
  • prefix string
  • recursive bool

func (*MockClient_Expecter) UploadObject added in v0.5.7

func (_e *MockClient_Expecter) UploadObject(ctx interface{}, i interface{}) *MockClient_UploadObject_Call

UploadObject is a helper method to define mock.On call

  • ctx context.Context
  • i UploadObjectInput

type MockClient_GetObject_Call added in v0.5.7

type MockClient_GetObject_Call struct {
	*mock.Call
}

MockClient_GetObject_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetObject'

func (*MockClient_GetObject_Call) Return added in v0.5.7

func (*MockClient_GetObject_Call) Run added in v0.5.7

func (*MockClient_GetObject_Call) RunAndReturn added in v0.5.7

func (_c *MockClient_GetObject_Call) RunAndReturn(run func(ctx context.Context, key string) (*Object, error)) *MockClient_GetObject_Call

type MockClient_HeadObject_Call added in v0.5.7

type MockClient_HeadObject_Call struct {
	*mock.Call
}

MockClient_HeadObject_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'HeadObject'

func (*MockClient_HeadObject_Call) Return added in v0.5.7

func (*MockClient_HeadObject_Call) Run added in v0.5.7

func (*MockClient_HeadObject_Call) RunAndReturn added in v0.5.7

type MockClient_ListPrefix_Call added in v0.5.7

type MockClient_ListPrefix_Call struct {
	*mock.Call
}

MockClient_ListPrefix_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListPrefix'

func (*MockClient_ListPrefix_Call) Return added in v0.5.7

func (*MockClient_ListPrefix_Call) Run added in v0.5.7

func (_c *MockClient_ListPrefix_Call) Run(run func(ctx context.Context, prefix string, recursive bool)) *MockClient_ListPrefix_Call

func (*MockClient_ListPrefix_Call) RunAndReturn added in v0.5.7

func (_c *MockClient_ListPrefix_Call) RunAndReturn(run func(ctx context.Context, prefix string, recursive bool) (ObjectIterator, error)) *MockClient_ListPrefix_Call

type MockClient_UploadObject_Call added in v0.5.7

type MockClient_UploadObject_Call struct {
	*mock.Call
}

MockClient_UploadObject_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UploadObject'

func (*MockClient_UploadObject_Call) Return added in v0.5.7

func (*MockClient_UploadObject_Call) Run added in v0.5.7

func (*MockClient_UploadObject_Call) RunAndReturn added in v0.5.7

type MockObjectIterator added in v0.5.7

type MockObjectIterator struct {
	// contains filtered or unexported fields
}

func NewMockObjectIterator added in v0.5.7

func NewMockObjectIterator(objs []ObjectAttr) *MockObjectIterator

func (*MockObjectIterator) HasNext added in v0.5.7

func (m *MockObjectIterator) HasNext() bool

func (*MockObjectIterator) Next added in v0.5.7

func (m *MockObjectIterator) Next() (ObjectAttr, error)

type Object added in v0.4.21

type Object struct {
	Length int64
	Body   io.ReadCloser
}

type ObjectAttr added in v0.4.21

type ObjectAttr struct {
	Key    string
	Length int64
}

func (*ObjectAttr) IsEmpty added in v0.4.21

func (o *ObjectAttr) IsEmpty() bool

type ObjectIterator added in v0.5.7

type ObjectIterator interface {
	HasNext() bool
	Next() (ObjectAttr, error)
}

type TraceFn added in v0.5.7

type TraceFn func(size int64, cost time.Duration)

type UploadObjectInput added in v0.4.21

type UploadObjectInput struct {
	Key  string
	Body io.Reader

	// The size of the file to be uploaded, if unknown, set to 0 or negative
	// Configuring this parameter can help reduce memory usage.
	Size int64
}

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL