storage

README

go-storage

Golang S3-backed object storage wrapper with byte accounting.

Usage

package main

import (
	"context"
	"fmt"
	"io"
	"strings"

	"github.com/tavocg/go-storage"
)

func example(ctx context.Context, s *storage.Storage) error {
	head, err := s.Put(
		ctx,
		strings.NewReader("hello world"),
		storage.WithKey("greeting.txt"),
		storage.WithSizeLimit(int64(len("hello world"))),
	)
	if err != nil {
		return err
	}

	body, err := s.Get(ctx, head)
	if err != nil {
		return err
	}
	defer body.Close()

	data, err := io.ReadAll(body)
	if err != nil {
		return err
	}

	fmt.Println(string(data))
	fmt.Println(s.Exists(head.Key))
	fmt.Println(s.List())

	if err := s.Delete(ctx, head); err != nil {
		return err
	}

	return nil
}

Custom Endpoints

For S3-compatible providers such as Cloudflare R2, use the S3 backend package and pass a custom endpoint:

package main

import (
	"context"

	s3backend "github.com/tavocg/go-storage/backends/s3"
)

func newStore(ctx context.Context) error {
	_, err := s3backend.New(
		ctx,
		s3backend.WithBucket("personal"),
		s3backend.WithRegion("auto"),
		s3backend.WithEndpoint("https://11bf4a9e76b5bde4ca62baa852624281.r2.cloudflarestorage.com"),
		s3backend.WithPublicEndpointURL("https://pub-11bf4a9e76b5bde4ca62baa852624281.r2.dev"),
	)
	return err
}

When WithPublicEndpointURL is set, GetBody downloads objects directly from that public base URL; otherwise it uses the configured S3 backend.

Documentation

Overview

Package storage provides a small S3-backed object storage wrapper with basic size accounting.

Index

• Constants
• type Backend
• type ObjectHead
• type PutOption
  • func WithContentType(contentType string) PutOption
  • func WithKey(key string) PutOption
  • func WithSizeLimit(size int64) PutOption
• type Storage
  • func New(backend Backend, bucket string, maxSize int64, opts ...StorageOption) *Storage
  • func (s *Storage) Delete(ctx context.Context, head *ObjectHead) error
  • func (s *Storage) Exists(key string) bool
  • func (s *Storage) Get(ctx context.Context, head *ObjectHead) (io.ReadCloser, error)
  • func (s *Storage) GetBody(ctx context.Context, head *ObjectHead) (io.ReadCloser, error)
  • func (s *Storage) List() map[string]int64
  • func (s *Storage) LoadState(ctx context.Context) error
  • func (s *Storage) Put(ctx context.Context, body io.Reader, opts ...PutOption) (*ObjectHead, error)
• type StorageOption
  • func WithPublicEndpointURL(rawURL string) StorageOption

Constants

const (
	// ErrMaxBytesReached indicates that reading more bytes would exceed the
	// configured storage limit.
	ErrMaxBytesReached = errStr("maxBytes reached")
	// ErrObjectNotFound indicates that an object does not exist.
	ErrObjectNotFound = errStr("object not found")
)

Types

type Backend

type Backend interface {
	PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
	GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
	DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error)
	ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
}

Backend is the minimal S3 API surface required by Storage.

type ObjectHead

type ObjectHead struct {
	// Key is the object identifier inside the bucket.
	Key string
	// Type is the object's content type.
	Type string
	// Size is the number of bytes stored for the object.
	Size int64
	// SHA256 is the hex-encoded SHA-256 digest of the stored body.
	SHA256 string
}

ObjectHead describes an object stored by Storage.

type PutOption

type PutOption func(*ObjectHead)

PutOption configures metadata for a Put request.

func WithContentType

func WithContentType(contentType string) PutOption

WithContentType sets the stored content type metadata for Put.

func WithKey

func WithKey(key string) PutOption

WithKey sets the object key used by Put. If omitted, Put generates a random key.

func WithSizeLimit

func WithSizeLimit(size int64) PutOption

WithSizeLimit sets the maximum number of bytes Put will read from body.

Providing a size limit lets Put reserve capacity up front and use a bounded reader, which is slightly faster than reserving per read. The final stored object may be smaller if body ends before the limit.

type Storage

type Storage struct {
	// contains filtered or unexported fields
}

Storage stores objects in a single bucket and tracks total and in-flight byte usage against maxSize.

func New

func New(backend Backend, bucket string, maxSize int64, opts ...StorageOption) *Storage

New returns a Storage that uses backend to store objects in bucket and enforces maxSize as the total allowed byte usage.

func (*Storage) Delete

func (s *Storage) Delete(ctx context.Context, head *ObjectHead) error

Delete removes an object from the bucket and subtracts its tracked size from storage usage.

func (*Storage) Exists

func (s *Storage) Exists(key string) bool

Exists reports whether key is currently tracked in storage.

func (*Storage) Get

func (s *Storage) Get(ctx context.Context, head *ObjectHead) (io.ReadCloser, error)

Get opens an object body for reading.

The caller must close the returned reader.

func (*Storage) GetBody

func (s *Storage) GetBody(ctx context.Context, head *ObjectHead) (io.ReadCloser, error)

GetBody opens an object body for reading.

The caller must close the returned reader.

func (*Storage) List

func (s *Storage) List() map[string]int64

List returns a copy of the tracked object sizes keyed by object name.

func (*Storage) LoadState

func (s *Storage) LoadState(ctx context.Context) error

LoadState refreshes Storage's tracked byte usage from the backend bucket.

Call LoadState after constructing Storage and before using it so in-memory accounting starts from the bucket's current size.

func (*Storage) Put

func (s *Storage) Put(ctx context.Context, body io.Reader, opts ...PutOption) (*ObjectHead, error)

Put stores body in the configured bucket and returns the resulting object metadata.

When WithSizeLimit is provided, Put can reserve storage up front and stream through a bounded reader, which is more efficient than reserving bytes on each read. The returned ObjectHead.Size is the number of bytes actually stored.

type StorageOption

type StorageOption func(*Storage)

StorageOption configures Storage.

func WithPublicEndpointURL

func WithPublicEndpointURL(rawURL string) StorageOption

WithPublicEndpointURL sets the base URL used by GetBody for public object downloads. When empty, GetBody uses the backend's GetObject method.