rsstorage

package
v3.0.6 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 20, 2026 License: MIT Imports: 8 Imported by: 0

README

/pkg/rsstorage

Description

A storage library. Useful for applications that need to store and retrieve data from centralized locations like NFS or AWS S3.

Examples

  • markdownRenderer demonstrates how to use rsstorage to create and retrieve files.

Implementations

See servers for information on the implementations available.

Documentation

Index

Constants

View Source 
const (
	DefaultChunkPollTimeout = time.Second * 5
	DefaultMaxChunkAttempts = 100
)
View Source 
const (
	StorageTypeFile     = types.StorageType("file")
	StorageTypePostgres = types.StorageType("postgres")
	StorageTypeS3       = types.StorageType("s3")
)

Variables

View Source 
var (
	ErrNoChunkMetadata = errors.New("metadata not found for chunked asset")
	ErrNoChunk         = errors.New("chunk not found for chunked asset")
)

Functions

This section is empty.

Types

type CacheStore 

type CacheStore interface {
	CacheObjectEnsureExists(cacheName, key string) error
	CacheObjectMarkUse(cacheName, key string, accessTime time.Time) error
}

type ChunkNotifier 

type ChunkNotifier interface {
	Notify(ctx context.Context, c *types.ChunkNotification) error
}

type ChunkUtils 

type ChunkUtils interface {
	WriteChunked(ctx context.Context, dir, address string, sz uint64, resolve types.Resolver) error
	ReadChunked(ctx context.Context, dir, address string) (io.ReadCloser, *types.ChunksInfo, int64, time.Time, error)
}

type ChunkWaiter 

type ChunkWaiter interface {
	WaitForChunk(ctx context.Context, c *types.ChunkNotification)
}

type Config 

type Config struct {
	CacheTimeout   time.Duration
	ChunkSizeBytes uint64
	S3             *ConfigS3
	File           *ConfigFile
}

type ConfigFile 

type ConfigFile struct {
	Location string
}

type ConfigS3 

type ConfigS3 struct {
	Bucket             string // S3 bucket to use
	Prefix             string // prefix to prepend to item addresses
	Profile            string // AWS profile to use
	Region             string // AWS regions to use
	Endpoint           string // S3 service endpoint to use
	KeyID              string // the AWS KMS ID to use for client-side S3 encryption
	SkipValidation     bool   // skip the validation of the S3 configuration
	DisableSSL         bool   // disable SSL for S3
	S3ForcePathStyle   bool   // force path style for URLs for S3 objects
	EnableSharedConfig bool   // overrides the AWS_SKD_LOAD_CONFIG env var and enables shared config functionality
}

type CopyPart 

type CopyPart struct {
	Dir     string
	Address string
}

func NewCopyPart 

func NewCopyPart(dir, address string) CopyPart

type DummyStorageServer 

type DummyStorageServer struct {
	GetAttempts    int
	GetReader      io.ReadCloser
	GetOk          bool
	GetChunked     *types.ChunksInfo
	GetSize        int64
	GetModTime     time.Time
	GetErr         error
	GetMap         map[string]GetResult
	RemoveErr      error
	RemoveMap      map[string]bool
	RemoveCount    int
	Flushed        int
	PutDelay       time.Duration
	PutErr         error
	PutCalled      int
	PutChunks      bool
	Address        []string
	Placed         []string
	Buffer         *bytes.Buffer
	EnumItems      []types.StoredItem
	EnumErr        error
	MoveErr        error
	Moved          []string
	CopyErr        error
	Copied         []string
	Location       string
	MockType       types.StorageType
	MockDir        string
	MockUsage      types.Usage
	MockUsageError error

	GetCheckLock sync.RWMutex
}

func (*DummyStorageServer) Base 

func (f *DummyStorageServer) Base() StorageServer

func (*DummyStorageServer) CalculateUsage 

func (f *DummyStorageServer) CalculateUsage() (types.Usage, error)

func (*DummyStorageServer) Check 

func (f *DummyStorageServer) Check(ctx context.Context, dir, address string) (bool, *types.ChunksInfo, int64, time.Time, error)

func (*DummyStorageServer) Copy 

func (f *DummyStorageServer) Copy(ctx context.Context, dir, address string, server StorageServer) error

func (*DummyStorageServer) Dir 

func (f *DummyStorageServer) Dir() string

func (*DummyStorageServer) Enumerate 

func (f *DummyStorageServer) Enumerate(ctx context.Context) ([]types.StoredItem, error)

func (*DummyStorageServer) Flush 

func (f *DummyStorageServer) Flush(ctx context.Context, dir, address string)

func (*DummyStorageServer) Get 

func (f *DummyStorageServer) Get(ctx context.Context, dir, address string) (io.ReadCloser, *types.ChunksInfo, int64, time.Time, bool, error)

func (*DummyStorageServer) Locate 

func (f *DummyStorageServer) Locate(dir, address string) string

func (*DummyStorageServer) Move 

func (f *DummyStorageServer) Move(ctx context.Context, dir, address string, server StorageServer) error

func (*DummyStorageServer) Put 

func (f *DummyStorageServer) Put(ctx context.Context, resolve types.Resolver, dir, address string) (string, string, error)

func (*DummyStorageServer) PutChunked 

func (f *DummyStorageServer) PutChunked(ctx context.Context, resolve types.Resolver, dir, address string, sz uint64) (string, string, error)

func (*DummyStorageServer) Remove 

func (f *DummyStorageServer) Remove(ctx context.Context, dir, address string) error

func (*DummyStorageServer) Type 

func (f *DummyStorageServer) Type() types.StorageType

type GetResult 

type GetResult struct {
	GetReader  io.ReadCloser
	GetOk      bool
	GetChunked *types.ChunksInfo
	GetSize    int64
	GetModTime time.Time
	GetErr     error
}

type Logger 

type Logger interface {
	Debugf(msg string, args ...interface{})
}

type MetadataStorageServer 

type MetadataStorageServer struct {
	StorageServer
	// contains filtered or unexported fields
}

MetadataStorageServer overrides the `Get` and `Put` methods for interacting with the database to record access times.

func (*MetadataStorageServer) Base 

func (s *MetadataStorageServer) Base() StorageServer

func (*MetadataStorageServer) Get 

func (s *MetadataStorageServer) Get(ctx context.Context, dir, address string) (io.ReadCloser, *types.ChunksInfo, int64, time.Time, bool, error)

func (*MetadataStorageServer) Put 

func (s *MetadataStorageServer) Put(ctx context.Context, resolve types.Resolver, dir, address string) (string, string, error)

func (*MetadataStorageServer) PutChunked 

func (s *MetadataStorageServer) PutChunked(ctx context.Context, resolve types.Resolver, dir, address string, sz uint64) (string, string, error)

type MetadataStorageServerArgs 

type MetadataStorageServerArgs struct {
	Name   string
	Server StorageServer
	Store  CacheStore
}

type MoveCopyFn 

type MoveCopyFn func(dir, address string, server StorageServer) error

type StorageServer 

type StorageServer interface {
	// Check to see if an item exists
	// Accepts:
	//  * dir     The prefix or directory in which to look
	//  * address The address of the item
	// Returns:
	//  * bool `true` if found
	//  * ChunkInfo if chunked
	//  * int64 the file size if known
	//  * time.Time the last modification time if known
	//  * error
	Check(ctx context.Context, dir, address string) (bool, *types.ChunksInfo, int64, time.Time, error)

	// Dir will present the underlying "directory" for a
	// storage server. This doesn't make sense for all server
	// types, but for interface reasons we need it.
	Dir() string

	// Type will present the "type" of server implementation.
	Type() types.StorageType

	// CalculateUsage will look at the underlying storage and
	// report information about the usage.
	CalculateUsage() (types.Usage, error)

	// Get an item if it exists
	// Accepts:
	//  * dir     The prefix or directory in which to look
	//  * address The address of the item
	// Returns:
	//  * io.ReadCloser the file
	//  * int64 The size of the file
	//  * time.Time The last modification time
	//  * bool `true` if found
	//  * error
	Get(ctx context.Context, dir, address string) (io.ReadCloser, *types.ChunksInfo, int64, time.Time, bool, error)

	// Put writes an item. Creates a file
	// named `address`, and then passes the file writer
	// to the provided `resolve` function for writing.
	// See `cache/runners` for a number of queue runners
	// that are responsible for using this method to store
	// data
	Put(ctx context.Context, resolve types.Resolver, dir, address string) (string, string, error)

	// PutChunked writes an item. Creates a directory
	// named `address`, and then passes the file writer
	// to the provided `resolve` function for writing. The
	// data will be written to the directory in a series
	// of chunk files of fixed size. The `dir`, `address`, and
	// `sz` parameters must all be included, unlike the regular
	// Put method, which allows post-determined location and size.
	PutChunked(ctx context.Context, resolve types.Resolver, dir, address string, sz uint64) (string, string, error)

	// Remove an item
	Remove(ctx context.Context, dir, address string) error

	// Flush the NFS cache while waiting for an
	// item to appear
	Flush(ctx context.Context, dir, address string)

	// Enumerate all items in storage
	Enumerate(ctx context.Context) ([]types.StoredItem, error)

	// Move an item from one storage system to another
	Move(ctx context.Context, dir, address string, server StorageServer) error

	// Copy an item from one storage system to another
	Copy(ctx context.Context, dir, address string, server StorageServer) error

	// Locate returns the storage location for a given dir, address
	Locate(dir, address string) string

	// Base returns the base storage server in case the server is wrapped
	Base() StorageServer
}

The StorageServer provides an interface to the file system for: 

(a) The FileCache in `file.go` (gets data from the cache)
(b) The Runners in `cache/runners` (put data into the cache)

func NewMetadataStorageServer 

func NewMetadataStorageServer(args MetadataStorageServerArgs) StorageServer

Source Files

View all Source files

Directories

Path Synopsis
integration_test
servertest
servers
file
postgres
s3server

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.