s3

README

S3 Storage Backend

This package provides an implementation of the storage.Backend interface for AWS S3 and S3-compatible storage services like MinIO.

Features

• Direct upload and download of objects
• Pre-signed URLs for client-side upload and download
• Server-side encryption support (AES256 and aws:kms)
• Support for S3-compatible services like MinIO
• Automatic bucket creation (optional)

Configuration

The S3 backend can be configured with the following options:

type Config struct {
    Region          string // AWS region
    Bucket          string // S3 bucket name
    AccessKeyID     string // AWS access key ID
    SecretAccessKey string // AWS secret access key
    Endpoint        string // Optional custom endpoint for S3-compatible services
    UseSSL          bool   // Use SSL for connections (default: true)
    UsePathStyle    bool   // Use path-style addressing (default: false)
    PresignDuration int    // Duration in seconds for presigned URLs (default: 3600)

    // Server-side encryption options
    EnableSSE    bool   // Enable server-side encryption
    SSEAlgorithm string // SSE algorithm (AES256 or aws:kms)
    SSEKMSKeyID  string // Optional KMS key ID for aws:kms algorithm

    // MinIO-specific options
    CreateBucketIfNotExist bool // Create bucket if it doesn't exist
}

Usage

Creating an S3 Backend

// AWS S3 configuration
config := s3.Config{
    Region:          "us-east-1",
    Bucket:          "my-bucket",
    AccessKeyID:     "AKIAXXXXXXXXXXXXXXXX",
    SecretAccessKey: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
    EnableSSE:       true,
    SSEAlgorithm:    "AES256",
}

// Create the backend
backend, err := s3.NewS3Backend(config)
if err != nil {
    log.Fatalf("Failed to create S3 backend: %v", err)
}

Using with MinIO

// MinIO configuration
config := s3.Config{
    Region:                "us-east-1",
    Bucket:                "my-bucket",
    AccessKeyID:           "minioadmin",
    SecretAccessKey:       "minioadmin",
    Endpoint:              "http://localhost:9000",
    UseSSL:                false,
    UsePathStyle:          true,
    CreateBucketIfNotExist: true,
}

// Create the backend
backend, err := s3.NewS3Backend(config)
if err != nil {
    log.Fatalf("Failed to create MinIO backend: %v", err)
}

Uploading an Object

// Direct upload
file, err := os.Open("example.txt")
if err != nil {
    log.Fatalf("Failed to open file: %v", err)
}
defer file.Close()

err = backend.Upload(context.Background(), "path/to/object.txt", file)
if err != nil {
    log.Fatalf("Upload failed: %v", err)
}

// Get pre-signed URL for client-side upload
uploadURL, err := backend.GetUploadURL(context.Background(), "path/to/object.txt")
if err != nil {
    log.Fatalf("Failed to get upload URL: %v", err)
}
fmt.Printf("Upload URL: %s\n", uploadURL)

Downloading an Object

// Direct download
reader, err := backend.Download(context.Background(), "path/to/object.txt")
if err != nil {
    log.Fatalf("Download failed: %v", err)
}
defer reader.Close()

// Save to file
file, err := os.Create("downloaded.txt")
if err != nil {
    log.Fatalf("Failed to create file: %v", err)
}
defer file.Close()

_, err = io.Copy(file, reader)
if err != nil {
    log.Fatalf("Failed to write file: %v", err)
}

// Get pre-signed URL for client-side download
downloadURL, err := backend.GetDownloadURL(context.Background(), "path/to/object.txt")
if err != nil {
    log.Fatalf("Failed to get download URL: %v", err)
}
fmt.Printf("Download URL: %s\n", downloadURL)

Deleting an Object

err = backend.Delete(context.Background(), "path/to/object.txt")
if err != nil {
    log.Fatalf("Delete failed: %v", err)
}

Documentation

Overview

Deprecated: This service is deprecated and will be removed in a future version. Please use the new module instead.

Index

• Constants
• func NewS3Backend(config Config) (storage.Backend, error)
• type Config
• type PresignOutput
• type S3Backend
  • func (b *S3Backend) Delete(ctx context.Context, objectKey string) error
  • func (b *S3Backend) Download(ctx context.Context, objectKey string) (io.ReadCloser, error)
  • func (b *S3Backend) GetDownloadURL(ctx context.Context, objectKey string, downloadFilename string) (string, error)
  • func (b *S3Backend) GetObjectMeta(ctx context.Context, objectKey string) (*storage.ObjectMeta, error)
  • func (b *S3Backend) GetPreviewURL(ctx context.Context, objectKey string) (string, error)
  • func (b *S3Backend) GetUploadURL(ctx context.Context, objectKey string) (string, error)
  • func (b *S3Backend) Upload(ctx context.Context, objectKey string, reader io.Reader) error
  • func (b *S3Backend) UploadWithParams(ctx context.Context, reader io.Reader, params storage.UploadParams) error
• type S3BackendForTesting
  • func (b *S3BackendForTesting) Delete(ctx context.Context, objectKey string) error
  • func (b *S3BackendForTesting) Download(ctx context.Context, objectKey string) (io.ReadCloser, error)
  • func (b *S3BackendForTesting) GetDownloadURL(ctx context.Context, objectKey string, downloadFilename string) (string, error)
  • func (b *S3BackendForTesting) GetObjectMeta(ctx context.Context, objectKey string) (*storage.ObjectMeta, error)
  • func (b *S3BackendForTesting) GetPreviewURL(ctx context.Context, objectKey string) (string, error)
  • func (b *S3BackendForTesting) GetUploadURL(ctx context.Context, objectKey string) (string, error)
  • func (b *S3BackendForTesting) Upload(ctx context.Context, objectKey string, reader io.Reader) error

Constants

const (
	LOCAL_S3_ENDPOINT = "http://localhost:9000"
)

Functions

func NewS3Backend

func NewS3Backend(config Config) (storage.Backend, error)

NewS3Backend creates a new S3 storage backend

Types

type Config

type Config struct {
	Region          string // AWS region
	Bucket          string // S3 bucket name
	AccessKeyID     string // AWS access key ID
	SecretAccessKey string // AWS secret access key
	Endpoint        string // Optional custom endpoint for S3-compatible services
	UseSSL          bool   // Use SSL for connections (default: true)
	UsePathStyle    bool   // Use path-style addressing (default: false)
	PresignDuration int    // Duration in seconds for presigned URLs (default: 3600)

	// Server-side encryption options
	EnableSSE    bool   // Enable server-side encryption
	SSEAlgorithm string // SSE algorithm (AES256 or aws:kms)
	SSEKMSKeyID  string // Optional KMS key ID for aws:kms algorithm

	// MinIO-specific options
	CreateBucketIfNotExist bool // Create bucket if it doesn't exist
}

Config options for the S3 backend

type PresignOutput

type PresignOutput struct {
	URL string
}

PresignOutput is a simplified version of the AWS SDK's PresignedHTTPRequest

type S3Backend

type S3Backend struct {
	// contains filtered or unexported fields
}

S3Backend is an AWS S3 implementation of the storage.Backend interface

func (*S3Backend) Delete

func (b *S3Backend) Delete(ctx context.Context, objectKey string) error

Delete deletes content from S3

func (*S3Backend) Download

func (b *S3Backend) Download(ctx context.Context, objectKey string) (io.ReadCloser, error)

Download downloads content directly from S3

func (*S3Backend) GetDownloadURL

func (b *S3Backend) GetDownloadURL(ctx context.Context, objectKey string, downloadFilename string) (string, error)

GetDownloadURL returns a pre-signed URL for downloading content

func (*S3Backend) GetObjectMeta

func (b *S3Backend) GetObjectMeta(ctx context.Context, objectKey string) (*storage.ObjectMeta, error)

GetObjectMeta retrieves metadata for an object in S3

func (*S3Backend) GetPreviewURL

func (b *S3Backend) GetPreviewURL(ctx context.Context, objectKey string) (string, error)

GetPreviewURL returns a URL for previewing content

func (*S3Backend) GetUploadURL

func (b *S3Backend) GetUploadURL(ctx context.Context, objectKey string) (string, error)

GetUploadURL returns a pre-signed URL for uploading content

func (*S3Backend) Upload

func (b *S3Backend) Upload(ctx context.Context, objectKey string, reader io.Reader) error

Upload uploads content directly to S3

func (*S3Backend) UploadWithParams

func (b *S3Backend) UploadWithParams(ctx context.Context, reader io.Reader, params storage.UploadParams) error

type S3BackendForTesting

type S3BackendForTesting struct {
	Client          s3ClientInterface
	PresignClient   presignClientInterface
	Bucket          string
	PresignDuration time.Duration
	EnableSSE       bool
	SSEAlgorithm    string
	SSEKMSKeyID     string
}

S3BackendForTesting is a version of S3Backend that can be used for testing It allows injecting mock clients

func (*S3BackendForTesting) Delete

func (b *S3BackendForTesting) Delete(ctx context.Context, objectKey string) error

Delete deletes content from S3

func (*S3BackendForTesting) Download

func (b *S3BackendForTesting) Download(ctx context.Context, objectKey string) (io.ReadCloser, error)

Download downloads content directly from S3

func (*S3BackendForTesting) GetDownloadURL

func (b *S3BackendForTesting) GetDownloadURL(ctx context.Context, objectKey string, downloadFilename string) (string, error)

GetDownloadURL returns a pre-signed URL for downloading content

func (*S3BackendForTesting) GetObjectMeta

func (b *S3BackendForTesting) GetObjectMeta(ctx context.Context, objectKey string) (*storage.ObjectMeta, error)

GetObjectMeta retrieves metadata for an object in S3

func (*S3BackendForTesting) GetPreviewURL

func (b *S3BackendForTesting) GetPreviewURL(ctx context.Context, objectKey string) (string, error)

GetPreviewURL returns a pre-signed URL for previewing content

func (*S3BackendForTesting) GetUploadURL

func (b *S3BackendForTesting) GetUploadURL(ctx context.Context, objectKey string) (string, error)

GetUploadURL returns a pre-signed URL for uploading content

func (*S3BackendForTesting) Upload

func (b *S3BackendForTesting) Upload(ctx context.Context, objectKey string, reader io.Reader) error

Upload uploads content directly to S3