storage

Documentation

Overview

Package storage stores files for an app (ADR-0075): a Store is a bucket of objects addressed by key, with drivers for the local disk (development) and S3-compatible services (Amazon S3, DigitalOcean Spaces, Cloudflare R2, MinIO). The Dev Portal's Storage screen and the operators' API (/ops/storage) work on any Store.

Stability: experimental (ADR-0054).

Index

• Constants
• Variables
• func ClampExpiry(d time.Duration) time.Duration
• func ContentTypeFor(key string) string
• func DirectoryMarker(prefix string) string
• func IsDirectoryMarker(key string) bool
• func ValidKey(key string) bool
• func ValidPrefix(prefix string) bool
• func ValidSignedMethod(method string) bool
• type Info
• type ListOptions
• type Object
  • func Move(ctx context.Context, s Store, src, dst string) (Object, error)
• type Page
• type PutOptions
• type Store

Constants

const (
	// MaxKeyLength bounds a key, as S3 does.
	MaxKeyLength = 1024
	// MaxListLimit bounds a List page.
	MaxListLimit = 1000
	// DefaultListLimit is a List page without a limit.
	DefaultListLimit = 200
	// MaxSignedURLExpiry bounds a signed URL's life (7 days, as S3 does).
	MaxSignedURLExpiry = 7 * 24 * time.Hour
)

Limits.

Variables

var (
	// ErrNotFound reports a key with no object.
	ErrNotFound = errors.New("storage: object not found")
	// ErrInvalidKey reports a key [ValidKey] refuses.
	ErrInvalidKey = errors.New("storage: invalid key")
	// ErrUnavailable reports a driver that can't reach its service.
	ErrUnavailable = errors.New("storage: unavailable")
)

Errors a Store returns.

Functions

func ClampExpiry

func ClampExpiry(d time.Duration) time.Duration

ClampExpiry bounds a signed URL's expiry: at least a second, at most MaxSignedURLExpiry; zero means an hour.

func ContentTypeFor

func ContentTypeFor(key string) string

ContentTypeFor guesses a content type from a key's extension, falling back to application/octet-stream.

func DirectoryMarker

func DirectoryMarker(prefix string) string

DirectoryMarker is the key of the object that makes an empty "directory" exist: prefix + "/.keep". Listings hide it.

func IsDirectoryMarker

func IsDirectoryMarker(key string) bool

IsDirectoryMarker reports a key DirectoryMarker made.

func ValidKey

func ValidKey(key string) bool

ValidKey reports whether key can name an object: 1 to MaxKeyLength bytes of valid UTF-8, no leading slash, no empty or "." or ".." segment, no control characters.

func ValidPrefix

func ValidPrefix(prefix string) bool

ValidPrefix reports whether prefix can select objects: empty, or a valid key optionally ending with a slash.

func ValidSignedMethod

func ValidSignedMethod(method string) bool

ValidSignedMethod reports a method Store.SignedURL accepts.

Types

type Info

type Info struct {
	// Driver is local, s3, spaces, r2 or minio.
	Driver string `json:"driver"`
	Bucket string `json:"bucket"`
	// Endpoint is the service's address (the directory for local).
	Endpoint string `json:"endpoint,omitempty"`
	Region   string `json:"region,omitempty"`
	// Local reports a store on this machine, safe to change freely.
	Local bool `json:"local"`
	// PublicURL is where objects are reachable without a signature, when
	// the bucket is public; empty otherwise.
	PublicURL string `json:"public_url,omitempty"`
}

Info describes a store for operators and the portal.

type ListOptions

type ListOptions struct {
	// Prefix keeps keys starting with it.
	Prefix string
	// Recursive lists every key under the prefix; otherwise keys with a
	// slash after the prefix are folded into Prefixes (directories).
	Recursive bool
	// Cursor continues a previous page.
	Cursor string
	// Limit is the page size (default [DefaultListLimit], at most
	// [MaxListLimit]).
	Limit int
}

ListOptions select a page of objects.

type Object

type Object struct {
	Key          string            `json:"key"`
	Size         int64             `json:"size"`
	ContentType  string            `json:"content_type"`
	ETag         string            `json:"etag,omitempty"`
	LastModified time.Time         `json:"last_modified"`
	Metadata     map[string]string `json:"metadata,omitempty"`
}

Object describes a stored object.

func Move

func Move(ctx context.Context, s Store, src, dst string) (Object, error)

Move copies src to dst and deletes src. It is a helper for every Store, since object stores have no rename.

type Page

type Page struct {
	Objects []Object `json:"objects"`
	// Prefixes are the "directories" folded when not recursive.
	Prefixes []string `json:"prefixes"`
	// NextCursor continues the listing; empty at the end.
	NextCursor string `json:"next_cursor,omitempty"`
}

Page is one page of a listing.

type PutOptions

type PutOptions struct {
	// ContentType of the object; guessed from the key when empty.
	ContentType string
	// Metadata is kept with the object (S3 user metadata; a sidecar file
	// on disk). Keys are lowercased; values must be plain text.
	Metadata map[string]string
}

PutOptions describe an upload.

type Store

type Store interface {
	// Info describes the store.
	Info() Info
	// Ping checks the service answers; [ErrUnavailable] wraps the reason.
	Ping(ctx context.Context) error
	// Put stores r under key, replacing any object there.
	Put(ctx context.Context, key string, r io.Reader, size int64, opts PutOptions) (Object, error)
	// Get opens the object for reading; the caller closes it.
	Get(ctx context.Context, key string) (io.ReadCloser, Object, error)
	// Stat describes the object without reading it.
	Stat(ctx context.Context, key string) (Object, error)
	// Delete removes the object; a missing key isn't an error.
	Delete(ctx context.Context, key string) error
	// Copy duplicates src to dst.
	Copy(ctx context.Context, src, dst string) (Object, error)
	// List returns a page of objects under a prefix.
	List(ctx context.Context, opts ListOptions) (Page, error)
	// SignedURL returns a URL that lets its holder GET (download) or PUT
	// (upload) the key until expiry, without other credentials.
	SignedURL(ctx context.Context, key string, method string, expiry time.Duration) (string, error)
}

A Store is a bucket of objects. Implementations are safe for concurrent use.