storage

package
v0.0.0-...-af02802 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 23, 2025 License: Apache-2.0 Imports: 34 Imported by: 0

Documentation

Index

Constants

View Source 
const GcsStorageListObjectPageSize = 1000 // google cloud API docs' recomendation
View Source 
const InMemoryStorageObjectPerPage = 100
View Source 
const SignedURLExpireInMinutes = 20

Variables

View Source 
var ErrNoMoreObjects = fmt.Errorf("no more objects in the storage")

Functions

func AppendContentType 

func AppendContentType(mimes map[string]bool, extraMimes map[string]string, URL, path string) string

func LoadExtraMimes 

func LoadExtraMimes() map[string]string

func LoadMimes 

func LoadMimes() map[string]bool

func RunTestForAllBackends 

func RunTestForAllBackends(t *testing.T, test func(string, Client))

func SeedBucket 

func SeedBucket(bucket Bucket, seedObjects []SeedObject) error

Types

type Bucket 

type Bucket interface {
	ListPath(options ListOptions) (PathIterator, error)
	ListObjectsWithPagination(options ListOptions) (ObjectPager, error)
	DeleteObjects(paths []string) error
	CreateObject(ctx context.Context, objectName string, content []byte) error
	IsDir(ctx context.Context, path string) (bool, error)
	IsFile(ctx context.Context, path string) (bool, error)
	DeletePath(ctx context.Context, path string) error
	DeleteDir(ctx context.Context, path string) error
	DeleteFile(ctx context.Context, path string) error
	SetCORS(ctx context.Context) error
	Destroy(ctx context.Context) error
}

type BucketOptions 

type BucketOptions struct {
	Name       string
	PathPrefix string
}

type Client 

type Client interface {
	GetBucket(options BucketOptions) Bucket
	CreateBucket(ctx context.Context) (string, error)
	DestroyBucket(ctx context.Context, options BucketOptions) error
	SignURL(ctx context.Context, options SignURLOptions) (string, error)
}

func New 

func New() (Client, error)

type Gcs 

type Gcs struct {
	Client                  *gcsstorage.Client
	Credentials             ServiceAccountCredentials
	Mimes                   map[string]bool
	ExtraMimes              map[string]string
	ManageBucketPermissions bool
}

func NewGcsClient 

func NewGcsClient(credentialsFile string) (*Gcs, error)

func (*Gcs) CreateBucket 

func (c *Gcs) CreateBucket(ctx context.Context) (string, error)

func (*Gcs) DestroyBucket 

func (c *Gcs) DestroyBucket(ctx context.Context, options BucketOptions) error

func (*Gcs) GetBucket 

func (c *Gcs) GetBucket(options BucketOptions) Bucket

func (*Gcs) SignURL 

func (c *Gcs) SignURL(ctx context.Context, options SignURLOptions) (string, error)

type GcsBucket 

type GcsBucket struct {
	BucketName    string
	BucketHandler *gcsstorage.BucketHandle
}

func (*GcsBucket) AddUser 

func (b *GcsBucket) AddUser(ctx context.Context, email string) error

AddUser connects a bucket with a service account with the adequate role permission.

func (*GcsBucket) CreateObject 

func (b *GcsBucket) CreateObject(ctx context.Context, objectName string, content []byte) error

Only used in tests

func (*GcsBucket) CreatedAt 

func (b *GcsBucket) CreatedAt(ctx context.Context, path string) (time.Time, error)

func (*GcsBucket) DeleteDir 

func (b *GcsBucket) DeleteDir(ctx context.Context, dir string) error

func (*GcsBucket) DeleteFile 

func (b *GcsBucket) DeleteFile(ctx context.Context, filename string) error

func (*GcsBucket) DeleteObjects 

func (b *GcsBucket) DeleteObjects(paths []string) error

func (*GcsBucket) DeletePath 

func (b *GcsBucket) DeletePath(ctx context.Context, name string) error

DelPath deletes object or directory from a Google Cloud Storage bucket.

func (*GcsBucket) Destroy 

func (b *GcsBucket) Destroy(ctx context.Context) error

func (*GcsBucket) IsDir 

func (b *GcsBucket) IsDir(ctx context.Context, dir string) (bool, error)

func (*GcsBucket) IsFile 

func (b *GcsBucket) IsFile(ctx context.Context, path string) (bool, error)

func (*GcsBucket) ListObjectsWithPagination 

func (b *GcsBucket) ListObjectsWithPagination(options ListOptions) (ObjectPager, error)

func (*GcsBucket) ListPath 

func (b *GcsBucket) ListPath(options ListOptions) (PathIterator, error)

func (*GcsBucket) RemoveUser 

func (b *GcsBucket) RemoveUser(ctx context.Context, email string) error

RemoveUser removes bucket-service account connection.

func (*GcsBucket) SetCORS 

func (b *GcsBucket) SetCORS(ctx context.Context) error

type GcsObjectPager 

type GcsObjectPager struct {
	// contains filtered or unexported fields
}

func (*GcsObjectPager) NextPage 

func (i *GcsObjectPager) NextPage() ([]*Object, string, error)

type GcsPathIterator 

type GcsPathIterator struct {
	// contains filtered or unexported fields
}

func (*GcsPathIterator) Count 

func (i *GcsPathIterator) Count() (int, error)

func (*GcsPathIterator) Done 

func (i *GcsPathIterator) Done() bool

func (*GcsPathIterator) Next 

func (i *GcsPathIterator) Next() (*PathItem, error)

type InMemoryBucket 

type InMemoryBucket struct {
	Name    string
	Objects []*PathItem
}

func (*InMemoryBucket) Add 

func (b *InMemoryBucket) Add(path string, age time.Time) error

func (*InMemoryBucket) CreateObject 

func (b *InMemoryBucket) CreateObject(ctx context.Context, name string, content []byte) error

func (*InMemoryBucket) DeleteDir 

func (b *InMemoryBucket) DeleteDir(ctx context.Context, dir string) error

func (*InMemoryBucket) DeleteFile 

func (b *InMemoryBucket) DeleteFile(ctx context.Context, path string) error

func (*InMemoryBucket) DeleteObjects 

func (b *InMemoryBucket) DeleteObjects(paths []string) error

func (*InMemoryBucket) DeletePath 

func (b *InMemoryBucket) DeletePath(ctx context.Context, path string) error

func (*InMemoryBucket) Destroy 

func (b *InMemoryBucket) Destroy(ctx context.Context) error

func (*InMemoryBucket) IsDir 

func (b *InMemoryBucket) IsDir(ctx context.Context, dir string) (bool, error)

func (*InMemoryBucket) IsFile 

func (b *InMemoryBucket) IsFile(ctx context.Context, path string) (bool, error)

func (*InMemoryBucket) ListObjectsWithPagination 

func (b *InMemoryBucket) ListObjectsWithPagination(options ListOptions) (ObjectPager, error)

func (*InMemoryBucket) ListPath 

func (b *InMemoryBucket) ListPath(options ListOptions) (PathIterator, error)

func (*InMemoryBucket) SetCORS 

func (b *InMemoryBucket) SetCORS(ctx context.Context) error

func (*InMemoryBucket) Size 

func (b *InMemoryBucket) Size() int

type InMemoryObjectIterator 

type InMemoryObjectIterator struct {
	// contains filtered or unexported fields
}

func (*InMemoryObjectIterator) Count 

func (i *InMemoryObjectIterator) Count() (int, error)

func (*InMemoryObjectIterator) Done 

func (i *InMemoryObjectIterator) Done() bool

func (*InMemoryObjectIterator) Next 

func (i *InMemoryObjectIterator) Next() (*PathItem, error)

type InMemoryObjectPager 

type InMemoryObjectPager struct {
	// contains filtered or unexported fields
}

func (*InMemoryObjectPager) Count 

func (p *InMemoryObjectPager) Count() (int, error)

func (*InMemoryObjectPager) NextPage 

func (p *InMemoryObjectPager) NextPage() ([]*Object, string, error)

type InMemoryStorage 

type InMemoryStorage struct {
	// contains filtered or unexported fields
}

func NewInMemoryStorage 

func NewInMemoryStorage() *InMemoryStorage

func (*InMemoryStorage) CreateBucket 

func (c *InMemoryStorage) CreateBucket(ctx context.Context) (string, error)

func (*InMemoryStorage) DestroyBucket 

func (c *InMemoryStorage) DestroyBucket(ctx context.Context, options BucketOptions) error

func (*InMemoryStorage) GetBucket 

func (c *InMemoryStorage) GetBucket(options BucketOptions) Bucket

func (*InMemoryStorage) SignURL 

func (c *InMemoryStorage) SignURL(ctx context.Context, options SignURLOptions) (string, error)

type ListOptions 

type ListOptions struct {
	Path               string
	MaxKeys            int64
	WrapSubDirectories bool
	PaginationToken    string
}

func (*ListOptions) UseDelimiter 

func (o *ListOptions) UseDelimiter() bool

type Object 

type Object struct {
	Path string
	Age  *time.Duration
}

type ObjectPager 

type ObjectPager interface {
	NextPage() ([]*Object, string, error)
}

type PathItem 

type PathItem struct {
	Path        string
	IsDirectory bool
	Age         *time.Duration
	Size        int64
}

type PathIterator 

type PathIterator interface {
	Next() (*PathItem, error)
	Count() (int, error)
	Done() bool
}

type S3 

type S3 struct {
	URL        string
	Client     *s3.S3
	BucketName string
}

func NewS3Client 

func NewS3Client(options S3Options) (*S3, error)

func (*S3) CreateBucket 

func (c *S3) CreateBucket(ctx context.Context) (string, error)

S3 storage uses a single bucket, we don't create anything.

func (*S3) DestroyBucket 

func (c *S3) DestroyBucket(ctx context.Context, options BucketOptions) error

We do not delete the bucket, we only empty it

func (*S3) GetBucket 

func (c *S3) GetBucket(options BucketOptions) Bucket

S3 storage uses a single bucket, so we ignore the bucket name passed as parameter here.

func (*S3) SignURL 

func (c *S3) SignURL(ctx context.Context, options SignURLOptions) (string, error)

type S3Bucket 

type S3Bucket struct {
	Client     *s3.S3
	BucketName string
	PathPrefix string
}

func (*S3Bucket) CreateObject 

func (b *S3Bucket) CreateObject(ctx context.Context, objectName string, content []byte) error

Only used in tests

func (*S3Bucket) DeleteDir 

func (b *S3Bucket) DeleteDir(ctx context.Context, path string) error

When deleting a directory, we list all the files in that directory and delete all the listed files in 1000-object chunks using the S3 DeleteObjects operation.

func (*S3Bucket) DeleteFile 

func (b *S3Bucket) DeleteFile(ctx context.Context, path string) error

func (*S3Bucket) DeleteObjects 

func (b *S3Bucket) DeleteObjects(paths []string) error

func (*S3Bucket) DeletePath 

func (b *S3Bucket) DeletePath(ctx context.Context, path string) error

func (*S3Bucket) Destroy 

func (b *S3Bucket) Destroy(ctx context.Context) error

no-op for S3

func (*S3Bucket) IsDir 

func (b *S3Bucket) IsDir(ctx context.Context, path string) (bool, error)

func (*S3Bucket) IsFile 

func (b *S3Bucket) IsFile(ctx context.Context, path string) (bool, error)

func (*S3Bucket) ListObjectsWithPagination 

func (b *S3Bucket) ListObjectsWithPagination(options ListOptions) (ObjectPager, error)

func (*S3Bucket) ListPath 

func (b *S3Bucket) ListPath(options ListOptions) (PathIterator, error)

func (*S3Bucket) SetCORS 

func (b *S3Bucket) SetCORS(ctx context.Context) error

type S3ObjectPager 

type S3ObjectPager struct {
	Client     *s3.S3
	BucketName string
	NextMarker string
	MaxKeys    int64
	Prefix     string
	PathPrefix string
}

func (*S3ObjectPager) NextPage 

func (p *S3ObjectPager) NextPage() ([]*Object, string, error)

type S3Options 

type S3Options struct {
	URL             string
	BucketName      string
	AccessKeyID     string
	SecretAccessKey string
	Region          string
}

type S3PathIterator 

type S3PathIterator struct {
	Client          *s3.S3
	LastOutput      *s3.ListObjectsOutput
	PathPrefix      string
	NextObjectIndex int
	NextPrefixIndex int
	IsDone          bool
}

func (*S3PathIterator) Count 

func (i *S3PathIterator) Count() (int, error)

func (*S3PathIterator) Done 

func (i *S3PathIterator) Done() bool

func (*S3PathIterator) Next 

func (i *S3PathIterator) Next() (*PathItem, error)

type SeedObject 

type SeedObject struct {
	Name    string
	Content string
}

type ServiceAccountCredentials 

type ServiceAccountCredentials struct {
	ProjectID   string `json:"project_id"`
	ClientEmail string `json:"client_email"`
	PrivateKey  string `json:"private_key"`
}

ServiceAccountCredentials is an object for parsing the Google Cloud Storage credentials from the JSON file.

type SignURLOptions 

type SignURLOptions struct {
	BucketName         string
	Path               string
	Method             string
	PathPrefix         string
	IncludeContentType bool
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.