storage

package module
v0.4.0-alpha Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 28, 2025 License: MIT Imports: 14 Imported by: 1

README

Go-Storage

Go Reference Go Report Card

⚠️ Alpha 版本 - 欢迎试用，反馈问题！

配置驱动的 Go 文件存储库。一行代码初始化，统一 API 操作多种存储后端。

从零开始测试

# 1. 创建测试项目
mkdir test-storage && cd test-storage
go mod init test-storage

# 2. 安装
go get github.com/wdcbot/go-storage@latest

# 3. 创建 main.go
cat > main.go << 'EOF'
package main

import (
    "fmt"
    "log"

    "github.com/wdcbot/go-storage"
)

func main() {
    // 初始化本地存储
    storage.MustSetup(map[string]any{
        "default": "local",
        "disks": map[string]any{
            "local": map[string]any{
                "driver":   "local",
                "root":     "./uploads",
                "base_url": "http://localhost:8080/files",
            },
        },
    })

    // 上传字符串
    result, err := storage.PutString("test.txt", "Hello Go-Storage!")
    if err != nil {
        log.Fatal(err)
    }
    fmt.Printf("✅ 上传成功: %s\n", result.Key)

    // 检查文件是否存在
    exists, _ := storage.Exists("test.txt")
    fmt.Printf("✅ 文件存在: %v\n", exists)

    // 读取内容
    content, _ := storage.GetString("test.txt")
    fmt.Printf("✅ 文件内容: %s\n", content)

    // 获取 URL
    url, _ := storage.URL("test.txt")
    fmt.Printf("✅ 文件 URL: %s\n", url)

    // 删除文件
    storage.Delete("test.txt")
    fmt.Println("✅ 文件已删除")
}
EOF

# 4. 运行
go run main.go

输出：

✅ 上传成功: test.txt
✅ 文件存在: true
✅ 文件内容: Hello Go-Storage!
✅ 文件 URL: http://localhost:8080/files/test.txt
✅ 文件已删除

安装

go get github.com/wdcbot/go-storage

# 云存储按需安装
go get github.com/wdcbot/go-storage/drivers/aliyun   # 阿里云 OSS
go get github.com/wdcbot/go-storage/drivers/tencent  # 腾讯云 COS
go get github.com/wdcbot/go-storage/drivers/s3       # AWS S3 / MinIO
go get github.com/wdcbot/go-storage/drivers/qiniu    # 七牛云

快速开始

package main

import (
    "github.com/wdcbot/go-storage"
    _ "github.com/wdcbot/go-storage/drivers/aliyun" // 按需 import
)

func main() {
    // 初始化（配合 viper 使用）
    // storage.MustSetup(viper.GetStringMap("storage"))
    
    // 或直接传 map
    storage.MustSetup(map[string]any{
        "default": "local",
        "disks": map[string]any{
            "local": map[string]any{
                "driver": "local",
                "root":   "./uploads",
            },
        },
    })

    // 上传
    storage.PutString("hello.txt", "Hello World")
    storage.PutFile("photo.jpg", "/path/to/photo.jpg")
    
    // 下载
    content, _ := storage.GetString("hello.txt")
    
    // 其他操作
    storage.Exists("hello.txt")
    storage.Delete("hello.txt")
    storage.URL("hello.txt")
    
    // 指定 disk
    storage.Disk("aliyun").PutString("cloud.txt", "Hello Cloud")
}

配置示例

# config.yaml
storage:
  default: local
  disks:
    local:
      driver: local
      root: ./uploads
      base_url: http://localhost:8080/files

    aliyun:
      driver: aliyun
      endpoint: oss-cn-hangzhou.aliyuncs.com
      bucket: my-bucket
      access_key_id: ${ALIYUN_ACCESS_KEY_ID}
      access_key_secret: ${ALIYUN_ACCESS_KEY_SECRET}

    tencent:
      driver: tencent
      region: ap-guangzhou
      bucket: my-bucket-1234567890
      secret_id: ${TENCENT_SECRET_ID}
      secret_key: ${TENCENT_SECRET_KEY}

    s3:
      driver: s3
      region: us-east-1
      bucket: my-bucket
      access_key_id: ${AWS_ACCESS_KEY_ID}
      secret_access_key: ${AWS_SECRET_ACCESS_KEY}

    minio:
      driver: s3
      endpoint: http://localhost:9000
      bucket: my-bucket
      access_key_id: minioadmin
      secret_access_key: minioadmin
      force_path_style: true

    qiniu:
      driver: qiniu
      bucket: my-bucket
      access_key: ${QINIU_ACCESS_KEY}
      secret_key: ${QINIU_SECRET_KEY}
      domain: https://cdn.example.com
      region: z0

API

// 基础操作
storage.Put(key, reader)              // 上传
storage.PutFile(key, filePath)        // 上传文件（自动检测 Content-Type）
storage.PutString(key, content)       // 上传字符串
storage.PutBytes(key, data)           // 上传 bytes

storage.Get(key)                      // 下载 -> io.ReadCloser
storage.GetString(key)                // 下载 -> string
storage.GetBytes(key)                 // 下载 -> []byte

storage.Delete(key)                   // 删除
storage.Exists(key)                   // 检查存在
storage.URL(key)                      // 获取 URL

// 指定 disk
storage.Disk("aliyun").PutString(key, content)

// 上传选项
storage.Put(key, reader,
    storage.WithContentType("image/jpeg"),
    storage.WithACL("public-read"),
    storage.WithMetadata(map[string]string{"author": "test"}),
)

支持的存储

Driver 状态 说明
local ✅ 内置 本地文件系统
aliyun ✅ 阿里云 OSS
tencent ✅ 腾讯云 COS
s3 ✅ AWS S3 / MinIO
qiniu ✅ 七牛云

License

MIT

Documentation

Overview

Package storage provides a pluggable file storage solution for Go developers. Configure once via YAML/JSON, use everywhere without writing initialization code.

Index

Constants

This section is empty.

Variables

View Source 
var (
	ErrNotFound       = errors.New("storage: file not found")
	ErrAlreadyExists  = errors.New("storage: file already exists")
	ErrPermission     = errors.New("storage: permission denied")
	ErrInvalidKey     = errors.New("storage: invalid key")
	ErrNotImplemented = errors.New("storage: not implemented")
	ErrClosed         = errors.New("storage: storage is closed")
)

Common errors.

Functions

func Debug 

func Debug() bool

Debug returns true if STORAGE_DEBUG env is set.

func Delete 

func Delete(key string) error

Delete removes from the default disk.

func DetectContentType 

func DetectContentType(filename string) string

DetectContentType detects the content type based on file extension.

func DownloadToFile 

func DownloadToFile(ctx context.Context, s Storage, key, filePath string) error

DownloadToFile is a convenience function to download a file to disk.

func Drivers 

func Drivers() []string

Drivers returns a list of registered driver names.

func EnableDebugLog 

func EnableDebugLog()

EnableDebugLog enables simple debug logging to stderr.

func Exists 

func Exists(key string) (bool, error)

Exists checks on the default disk.

func GenerateKey 

func GenerateKey(prefix, filename string) string

GenerateKey generates a unique key for a file. Format: prefix/2006/01/02/uuid.ext

func GenerateKeyFlat 

func GenerateKeyFlat(prefix, filename string) string

GenerateKeyFlat generates a unique key without date directories. Format: prefix/uuid.ext

func Get 

func Get(key string) (io.ReadCloser, error)

Get downloads from the default disk.

func GetBytes 

func GetBytes(key string) ([]byte, error)

GetBytes downloads and returns bytes.

func GetString 

func GetString(key string) (string, error)

GetString downloads and returns string.

func IsNotExist 

func IsNotExist(err error) bool

IsNotExist checks if the error indicates the file does not exist.

func IsNotFoundError 

func IsNotFoundError(err error) bool

IsNotFound checks if the error is a "not found" error.

func IsPermissionError 

func IsPermissionError(err error) bool

IsPermissionError checks if the error is a permission error.

func Must 

func Must[T any](v T, err error) T

Must panics if err is not nil. Useful for initialization.

func MustSetup 

func MustSetup(cfg map[string]any)

MustSetup is like Setup but panics on error.

func Register 

func Register(name string, driver Driver)

Register registers a storage driver. This is typically called in init() by each driver package.

func Retry 

func Retry(ctx context.Context, maxAttempts int, fn func() error) error

Retry retries a function with exponential backoff.

func SetLogger 

func SetLogger(l Logger)

SetLogger sets the global logger.

func Setup 

func Setup(cfg map[string]any) error

Setup initializes storage from a map (works with viper, koanf, etc).

Example with viper: 

viper.ReadInConfig()
storage.Setup(viper.GetStringMap("storage"))

Example with raw map: 

storage.Setup(map[string]any{
    "default": "local",
    "disks": map[string]any{
        "local": map[string]any{
            "driver": "local",
            "root":   "./uploads",
        },
    },
})

func URL 

func URL(key string) (string, error)

URL returns URL from the default disk.

Types

type AdvancedStorage 

type AdvancedStorage interface {
	Storage

	// SignedURL generates a pre-signed URL for temporary access to private files.
	// expires specifies how long the URL should be valid.
	SignedURL(ctx context.Context, key string, expires time.Duration) (string, error)

	// List lists files in the storage with the given prefix.
	List(ctx context.Context, prefix string, opts ...ListOption) (*ListResult, error)

	// Copy copies a file from src to dst within the same storage.
	Copy(ctx context.Context, src, dst string) error

	// Move moves a file from src to dst within the same storage.
	Move(ctx context.Context, src, dst string) error

	// Size returns the size of a file in bytes.
	Size(ctx context.Context, key string) (int64, error)

	// Metadata returns the metadata of a file.
	Metadata(ctx context.Context, key string) (*FileInfo, error)
}

AdvancedStorage extends Storage with optional advanced features. Not all drivers support these methods.

type BatchDeleteResult 

type BatchDeleteResult struct {
	Succeeded []string
	Failed    []BatchError
}

BatchDeleteResult contains results of a batch delete.

func BatchDelete 

func BatchDelete(ctx context.Context, s Storage, keys []string, concurrency int) *BatchDeleteResult

BatchDelete deletes multiple files concurrently.

func DeleteAll 

func DeleteAll(ctx context.Context, s Storage, prefix string, concurrency int) (*BatchDeleteResult, error)

DeleteAll deletes all files with the given prefix. Only works with AdvancedStorage that supports List.

type BatchError 

type BatchError struct {
	Key string
	Err error
}

BatchError represents an error for a single item in a batch operation.

type BatchUploadItem 

type BatchUploadItem struct {
	Key    string
	Reader io.Reader
	Opts   []UploadOption
}

BatchUploadItem represents a single item in a batch upload.

type BatchUploadResult 

type BatchUploadResult struct {
	Succeeded []*UploadResult
	Failed    []BatchError
}

BatchUploadResult contains results of a batch upload.

func BatchUpload 

func BatchUpload(ctx context.Context, s Storage, items []BatchUploadItem, concurrency int) *BatchUploadResult

BatchUpload uploads multiple files concurrently. concurrency controls how many uploads run in parallel (0 = no limit).

type Config 

type Config struct {
	Default  string                   `yaml:"default" json:"default"`
	Storages map[string]StorageConfig `yaml:"storages" json:"storages"`
}

Config represents the storage configuration structure.

type DiskWrapper 

type DiskWrapper struct {
	// contains filtered or unexported fields
}

DiskWrapper provides a fluent API for storage operations.

func Disk 

func Disk(name string) *DiskWrapper

Disk returns a storage disk by name. Returns the default disk if name is empty.

func (*DiskWrapper) Delete 

func (d *DiskWrapper) Delete(key string) error

Delete removes a file from the storage.

func (*DiskWrapper) Exists 

func (d *DiskWrapper) Exists(key string) (bool, error)

Exists checks if a file exists.

func (*DiskWrapper) Get 

func (d *DiskWrapper) Get(key string) (io.ReadCloser, error)

Get downloads data from the storage.

func (*DiskWrapper) GetBytes 

func (d *DiskWrapper) GetBytes(key string) ([]byte, error)

GetBytes downloads and returns bytes.

func (*DiskWrapper) GetString 

func (d *DiskWrapper) GetString(key string) (string, error)

GetString downloads and returns string.

func (*DiskWrapper) GetWithContext 

func (d *DiskWrapper) GetWithContext(ctx context.Context, key string) (io.ReadCloser, error)

GetWithContext downloads data with context.

func (*DiskWrapper) MustGet 

func (d *DiskWrapper) MustGet(key string) io.ReadCloser

MustGet downloads and panics on error.

func (*DiskWrapper) MustPut 

func (d *DiskWrapper) MustPut(key string, reader io.Reader, opts ...UploadOption) *UploadResult

MustPut uploads and panics on error.

func (*DiskWrapper) Put 

func (d *DiskWrapper) Put(key string, reader io.Reader, opts ...UploadOption) (*UploadResult, error)

Put uploads data to the storage.

func (*DiskWrapper) PutBytes 

func (d *DiskWrapper) PutBytes(key string, data []byte, opts ...UploadOption) (*UploadResult, error)

PutBytes uploads bytes directly.

func (*DiskWrapper) PutFile 

func (d *DiskWrapper) PutFile(key, filePath string, opts ...UploadOption) (*UploadResult, error)

PutFile uploads a file from local path.

func (*DiskWrapper) PutString 

func (d *DiskWrapper) PutString(key, content string, opts ...UploadOption) (*UploadResult, error)

PutString uploads a string directly.

func (*DiskWrapper) PutWithContext 

func (d *DiskWrapper) PutWithContext(ctx context.Context, key string, reader io.Reader, opts ...UploadOption) (*UploadResult, error)

PutWithContext uploads data with context.

func (*DiskWrapper) Storage 

func (d *DiskWrapper) Storage() (Storage, error)

Storage returns the underlying Storage interface. Use this to access AdvancedStorage features like SignedURL, List, etc.

Example: 

s, err := storage.Disk("aliyun").Storage()
if adv, ok := s.(storage.AdvancedStorage); ok {
    url, _ := adv.SignedURL(ctx, "file.txt", time.Hour)
}

func (*DiskWrapper) URL 

func (d *DiskWrapper) URL(key string) (string, error)

URL returns the public URL of a file.

type Driver 

type Driver func(cfg map[string]any) (Storage, error)

Driver is a factory function that creates a Storage instance from config.

type Error 

type Error struct {
	Op     string // Operation that failed (e.g., "upload", "download")
	Driver string // Driver name (e.g., "aliyun", "s3")
	Key    string // File key
	Err    error  // Underlying error
}

Error represents a storage error with additional context.

func NewError 

func NewError(driver, op, key string, err error) *Error

NewError creates a new storage error.

func (*Error) Error 

func (e *Error) Error() string

func (*Error) Unwrap 

func (e *Error) Unwrap() error

type FileInfo 

type FileInfo struct {
	Key          string
	Size         int64
	LastModified time.Time
	ContentType  string
	ETag         string
	Metadata     map[string]string
}

FileInfo contains metadata about a file.

type ListOption 

type ListOption func(*ListOptions)

ListOption is a functional option for List.

func WithDelimiter 

func WithDelimiter(d string) ListOption

WithDelimiter sets the delimiter for directory-like listing.

func WithMarker 

func WithMarker(marker string) ListOption

WithMarker sets the marker for pagination.

func WithMaxKeys 

func WithMaxKeys(n int) ListOption

WithMaxKeys sets the maximum number of keys to return.

type ListOptions 

type ListOptions struct {
	MaxKeys   int
	Marker    string // Start listing after this key
	Delimiter string // e.g., "/" for directory-like listing
}

ListOptions configures list behavior.

type ListResult 

type ListResult struct {
	Files       []FileInfo
	NextMarker  string // For pagination
	IsTruncated bool   // Whether there are more results
}

ListResult contains the result of a List operation.

type Logger 

type Logger interface {
	Debug(msg string, args ...any)
	Info(msg string, args ...any)
	Warn(msg string, args ...any)
	Error(msg string, args ...any)
}

Logger interface for custom logging.

type LoggingStorage 

type LoggingStorage struct {
	Storage
	// contains filtered or unexported fields
}

LoggingStorage wraps a Storage with logging.

func WrapWithLogging 

func WrapWithLogging(s Storage, name string, logger Logger) *LoggingStorage

WrapWithLogging wraps a storage with logging.

func (*LoggingStorage) Delete 

func (l *LoggingStorage) Delete(ctx context.Context, key string) error

func (*LoggingStorage) Download 

func (l *LoggingStorage) Download(ctx context.Context, key string) (io.ReadCloser, error)

func (*LoggingStorage) Upload 

func (l *LoggingStorage) Upload(ctx context.Context, key string, reader io.Reader, opts ...UploadOption) (*UploadResult, error)

type Manager 

type Manager struct {
	// contains filtered or unexported fields
}

Manager manages multiple storage backends based on configuration.

func NewManager 

func NewManager(cfg *Config) *Manager

NewManager creates a new storage manager from configuration.

func (*Manager) Close 

func (m *Manager) Close() error

Close closes all initialized storage backends.

func (*Manager) Disk 

func (m *Manager) Disk(name string) (Storage, error)

Disk returns a storage backend by name. If name is empty, returns the default storage.

type ProgressReader 

type ProgressReader struct {
	// contains filtered or unexported fields
}

ProgressReader wraps a reader to track progress.

func NewProgressReader 

func NewProgressReader(r io.Reader, total int64, fn func(uploaded, total int64)) *ProgressReader

NewProgressReader creates a new progress reader.

func (*ProgressReader) Read 

func (pr *ProgressReader) Read(p []byte) (int, error)

type SizeReader 

type SizeReader struct {
	io.Reader
	// contains filtered or unexported fields
}

SizeReader wraps a reader to get its size.

func NewSizeReader 

func NewSizeReader(r io.Reader, size int64) *SizeReader

NewSizeReader creates a reader that knows its size.

func (*SizeReader) Size 

func (sr *SizeReader) Size() int64

Size returns the total size.

type SlogAdapter 

type SlogAdapter struct {
	// contains filtered or unexported fields
}

SlogAdapter adapts slog.Logger to our Logger interface. Usage: storage.SetLogger(storage.NewSlogAdapter(slog.Default()))

func NewSlogAdapter 

func NewSlogAdapter(l interface {
	Debug(msg string, args ...any)
	Info(msg string, args ...any)
	Warn(msg string, args ...any)
	Error(msg string, args ...any)
}) *SlogAdapter

func (*SlogAdapter) Debug 

func (a *SlogAdapter) Debug(msg string, args ...any)

func (*SlogAdapter) Error 

func (a *SlogAdapter) Error(msg string, args ...any)

func (*SlogAdapter) Info 

func (a *SlogAdapter) Info(msg string, args ...any)

func (*SlogAdapter) Warn 

func (a *SlogAdapter) Warn(msg string, args ...any)

type Storage 

type Storage interface {
	// Upload uploads a file to the storage backend.
	Upload(ctx context.Context, key string, reader io.Reader, opts ...UploadOption) (*UploadResult, error)

	// Download downloads a file from the storage backend.
	Download(ctx context.Context, key string) (io.ReadCloser, error)

	// Delete deletes a file from the storage backend.
	Delete(ctx context.Context, key string) error

	// Exists checks if a file exists in the storage backend.
	Exists(ctx context.Context, key string) (bool, error)

	// URL returns the public URL of a file (if supported).
	URL(ctx context.Context, key string) (string, error)

	// Close releases any resources held by the storage backend.
	Close() error
}

Storage is the main interface that all storage backends must implement.

func Open 

func Open(driverName string, cfg map[string]any) (Storage, error)

Open creates a Storage instance using the specified driver and config.

type StorageConfig 

type StorageConfig struct {
	Driver  string         `yaml:"driver" json:"driver"`
	Options map[string]any `yaml:"options" json:"options"`
}

StorageConfig represents a single storage backend configuration.

type UploadOption 

type UploadOption func(*UploadOptions)

UploadOption is a functional option for Upload.

func WithACL 

func WithACL(acl string) UploadOption

WithACL sets the access control.

func WithContentDisposition 

func WithContentDisposition(cd string) UploadOption

WithContentDisposition sets the content disposition.

func WithContentType 

func WithContentType(ct string) UploadOption

WithContentType sets the content type.

func WithMetadata 

func WithMetadata(m map[string]string) UploadOption

WithMetadata sets custom metadata.

func WithProgress 

func WithProgress(fn func(uploaded, total int64)) UploadOption

WithProgress sets a progress callback for upload.

type UploadOptions 

type UploadOptions struct {
	ContentType        string
	ContentDisposition string
	Metadata           map[string]string
	ACL                string                      // e.g., "public-read", "private"
	ProgressFn         func(uploaded, total int64) // Progress callback
}

UploadOptions configures upload behavior.

type UploadResult 

type UploadResult struct {
	Key      string            // The key/path of the uploaded file
	URL      string            // Public URL (if available)
	Size     int64             // Size in bytes
	ETag     string            // ETag/checksum (if available)
	Metadata map[string]string // Additional metadata
}

UploadResult contains information about an uploaded file.

func Put 

func Put(key string, reader io.Reader, opts ...UploadOption) (*UploadResult, error)

Put uploads to the default disk.

func PutBytes 

func PutBytes(key string, data []byte, opts ...UploadOption) (*UploadResult, error)

PutBytes uploads bytes directly.

func PutFile 

func PutFile(key, filePath string, opts ...UploadOption) (*UploadResult, error)

PutFile uploads a file from local path.

func PutString 

func PutString(key, content string, opts ...UploadOption) (*UploadResult, error)

PutString uploads a string directly.

func UploadFile 

func UploadFile(ctx context.Context, s Storage, key, filePath string, opts ...UploadOption) (*UploadResult, error)

UploadFile is a convenience function to upload a file from disk.

Source Files

View all Source files

Directories

Path Synopsis
drivers
local module

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.