storage

Documentation

Index

• Constants
• func CheckEndpoint(endpoint string) bool
• func IsExist(name string) (bool, error)
• type Dir
• type Downloader
• type ExternalStorage
  • func New(b *pb.Backend) (ExternalStorage, error)
• type GS
  • func NewGS(b *pb.Backend) (*GS, error)
  • func (g *GS) Download(ctx context.Context, localPath, externalUri string, recursively bool) error
  • func (g *GS) EnsureDir(ctx context.Context, uri string, recursively bool) error
  • func (g *GS) ExistDir(ctx context.Context, uri string) bool
  • func (g *GS) GetDir(ctx context.Context, uri string) (*pb.Backend, error)
  • func (g *GS) IncrUpload(ctx context.Context, externalUri, localPath string, ...) error
  • func (g *GS) ListDir(ctx context.Context, uri string) ([]string, error)
  • func (g *GS) RemoveDir(ctx context.Context, uri string) error
  • func (g *GS) Upload(ctx context.Context, externalUri, localPath string, recursively bool) error
• type Local
  • func (l *Local) Download(ctx context.Context, localPath, externalUri string, recursively bool) error
  • func (l *Local) EnsureDir(ctx context.Context, uri string, recursively bool) error
  • func (l *Local) ExistDir(ctx context.Context, uri string) bool
  • func (l *Local) GetDir(ctx context.Context, uri string) (*pb.Backend, error)
  • func (l *Local) IncrUpload(ctx context.Context, externalUri, localPath string, ...) error
  • func (l *Local) ListDir(ctx context.Context, uri string) ([]string, error)
  • func (l *Local) RemoveDir(ctx context.Context, uri string) error
  • func (l *Local) Upload(ctx context.Context, externalUri, localPath string, recursively bool) error
• type S3
  • func NewS3(b *pb.Backend) (*S3, error)
  • func (s *S3) Download(ctx context.Context, localPath, externalUri string, recursively bool) error
  • func (s *S3) EnsureDir(ctx context.Context, uri string, recursively bool) error
  • func (s *S3) ExistDir(ctx context.Context, uri string) bool
  • func (s *S3) GetDir(ctx context.Context, uri string) (*pb.Backend, error)
  • func (s *S3) GetObjectSize(bucket, key string) (int64, error)
  • func (s *S3) IncrUpload(ctx context.Context, externalUri, localPath string, ...) error
  • func (s *S3) ListDir(ctx context.Context, uri string) ([]string, error)
  • func (s *S3) RemoveDir(ctx context.Context, uri string) error
  • func (s *S3) Upload(ctx context.Context, externalUri, localPath string, recursively bool) error
• type Uploader

Constants

const (
	SessionKey = "storage-session-id"
)

Functions

func CheckEndpoint

func CheckEndpoint(endpoint string) bool

CheckEndpoint check whether endpoint is combine with ip:port example: http://127.0.0.1:9999/xxx

func IsExist

func IsExist(name string) (bool, error)

Types

type Dir

type Dir interface {
	ExistDir(ctx context.Context, uri string) bool
	EnsureDir(ctx context.Context, uri string, recursively bool) error
	// GetDir return Backend with specified uri and authentication information stored in storage
	GetDir(ctx context.Context, uri string) (*pb.Backend, error)
	// ListDir only list get dir names in given dir, not recursively
	ListDir(ctx context.Context, uri string) ([]string, error)
	// RemoveDir remove all files recursively
	RemoveDir(ctx context.Context, uri string) error
}

Dir means we treat the storage organized as tree hierarchy

type Downloader

type Downloader interface {
	Download(ctx context.Context, localPath, externalUri string, recursively bool) error
}

Downloader download data from externalUri in ExternalStorage to the localPath If localPath not exist, create it; If it exists, overwrite it when it is file, copy to it when it is folder

type ExternalStorage

type ExternalStorage interface {
	Downloader
	Uploader
	Dir
}

ExternalStorage will keep the authentication information and other configuration for the storage Then the functions only need uri as the parameter

func New

func New(b *pb.Backend) (ExternalStorage, error)

type GS

type GS struct {
	// contains filtered or unexported fields
}

func NewGS

func NewGS(b *pb.Backend) (*GS, error)

func (*GS) Download

func (g *GS) Download(ctx context.Context, localPath, externalUri string, recursively bool) error

func (*GS) EnsureDir

func (g *GS) EnsureDir(ctx context.Context, uri string, recursively bool) error

func (*GS) ExistDir

func (g *GS) ExistDir(ctx context.Context, uri string) bool

func (*GS) GetDir

func (g *GS) GetDir(ctx context.Context, uri string) (*pb.Backend, error)

func (*GS) IncrUpload

func (g *GS) IncrUpload(ctx context.Context, externalUri, localPath string, commitLogId, lastLogId int64) error

func (*GS) ListDir

func (g *GS) ListDir(ctx context.Context, uri string) ([]string, error)

func (*GS) RemoveDir

func (g *GS) RemoveDir(ctx context.Context, uri string) error

func (*GS) Upload

func (g *GS) Upload(ctx context.Context, externalUri, localPath string, recursively bool) error

type Local

type Local struct {
}

func (*Local) Download

func (l *Local) Download(ctx context.Context, localPath, externalUri string, recursively bool) error

func (*Local) EnsureDir

func (l *Local) EnsureDir(ctx context.Context, uri string, recursively bool) error

func (*Local) ExistDir

func (l *Local) ExistDir(ctx context.Context, uri string) bool

func (*Local) GetDir

func (l *Local) GetDir(ctx context.Context, uri string) (*pb.Backend, error)

func (*Local) IncrUpload

func (l *Local) IncrUpload(ctx context.Context, externalUri, localPath string, commitLogId, lastLogId int64) error

func (*Local) ListDir

func (l *Local) ListDir(ctx context.Context, uri string) ([]string, error)

func (*Local) RemoveDir

func (l *Local) RemoveDir(ctx context.Context, uri string) error

func (*Local) Upload

func (l *Local) Upload(ctx context.Context, externalUri, localPath string, recursively bool) error

type S3

type S3 struct {
	// contains filtered or unexported fields
}

func NewS3

func NewS3(b *pb.Backend) (*S3, error)

func (*S3) Download

func (s *S3) Download(ctx context.Context, localPath, externalUri string, recursively bool) error

func (*S3) EnsureDir

func (s *S3) EnsureDir(ctx context.Context, uri string, recursively bool) error

func (*S3) ExistDir

func (s *S3) ExistDir(ctx context.Context, uri string) bool

func (*S3) GetDir

func (s *S3) GetDir(ctx context.Context, uri string) (*pb.Backend, error)

func (*S3) GetObjectSize

func (s *S3) GetObjectSize(bucket, key string) (int64, error)

GetObjectSize get s3 object size for rate-limit

func (*S3) IncrUpload

func (s *S3) IncrUpload(ctx context.Context, externalUri, localPath string, commitLogId, lastLogId int64) error

func (*S3) ListDir

func (s *S3) ListDir(ctx context.Context, uri string) ([]string, error)

func (*S3) RemoveDir

func (s *S3) RemoveDir(ctx context.Context, uri string) error

func (*S3) Upload

func (s *S3) Upload(ctx context.Context, externalUri, localPath string, recursively bool) error

type Uploader

type Uploader interface {
	Upload(ctx context.Context, externalUri, localPath string, recursively bool) error
	IncrUpload(ctx context.Context, externalUri, localPath string, commitLogId, lastLogId int64) error
}

Uploader upload data from localPath to the externalUri in ExternalStorage When recursively is false, upload file from localPath to externalUri Else upload files in localPath folder to externalUri folder