storage

package
v2.0.2-rc.25 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 16, 2022 License: Apache-2.0 Imports: 20 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// DefaultGCInitialDelay is the delay time from the start to the first GC execution.
	DefaultGCInitialDelay = 0 * time.Second

	// DefaultInterval is the interval time to execute the GC storage.
	DefaultInterval = 15 * time.Second
)
View Source 
const (
	// DownloadHome is the parent directory where the downloaded files are stored
	// which is a relative path.
	DownloadHome = "download"

	// UploadHome is the parent directory where the upload files are stored
	// which is a relative path
	UploadHome = "upload"
)
View Source 
const (
	// DefaultStorageMode is the default storage mode of CDN.
	DefaultStorageMode = "disk"
)

Variables

View Source 
var (
	DefaultDiskBaseDir = filepath.Join(basic.HomeDir, "ftp")

	DefaultMemoryBaseDir = "/dev/shm/dragonfly"
)
View Source 
var (
	ErrTaskNotPersisted = errors.New("task is not persisted")
)

Functions

func GetAppendPieceMetadataRaw added in v2.0.2 

func GetAppendPieceMetadataRaw(taskID string) *storedriver.Raw

func GetDownloadHomeRaw 

func GetDownloadHomeRaw() *storedriver.Raw

func GetDownloadParentRaw added in v2.0.2 

func GetDownloadParentRaw(taskID string) *storedriver.Raw

func GetDownloadRaw 

func GetDownloadRaw(taskID string) *storedriver.Raw

func GetPieceMetadataRaw added in v2.0.2 

func GetPieceMetadataRaw(taskID string) *storedriver.Raw

func GetTaskMetadataRaw added in v2.0.2 

func GetTaskMetadataRaw(taskID string) *storedriver.Raw

func GetUploadHomeRaw 

func GetUploadHomeRaw() *storedriver.Raw

func GetUploadParentRaw added in v2.0.2 

func GetUploadParentRaw(taskID string) *storedriver.Raw

func GetUploadRaw 

func GetUploadRaw(taskID string) *storedriver.Raw

func IsSupport 

func IsSupport(mode string) bool

func Register 

func Register(builder Builder)

Register defines an interface to register a storage manager builder. All storage managers should call this function to register its builder to the storage manager factory.

Types

type Builder added in v2.0.2 

type Builder interface {
	// Build creates a new balancer with the ClientConn.
	Build(cfg Config, taskManager task.Manager) (Manager, error)
	// Name returns the name of balancers built by this builder.
	// It will be used to pick balancers (for example in service config).
	Name() string
	// Validate driver configs
	Validate(map[string]*DriverConfig) []error
	// DefaultDriverConfigs default driver config
	DefaultDriverConfigs() map[string]*DriverConfig
}

Builder creates a balancer.

func Get 

func Get(name string) Builder

Get return a storage manager from manager with specified name.

type Cleaner 

type Cleaner interface {
	GC(storagePattern string, force bool) ([]string, error)
}

func NewStorageCleaner 

func NewStorageCleaner(config *DriverGCConfig, driver storedriver.Driver, storageManager Manager, taskManager task.Manager) (Cleaner, error)

type Config 

type Config struct {
	StorageMode    string                   `yaml:"storageMode"`
	GCInitialDelay time.Duration            `yaml:"gcInitialDelay"`
	GCInterval     time.Duration            `yaml:"gcInterval"`
	DriverConfigs  map[string]*DriverConfig `yaml:"driverGCConfigs"`
}

func DefaultConfig added in v2.0.2 

func DefaultConfig() Config

func (Config) Validate added in v2.0.2 

func (c Config) Validate() []error

type DriverConfig 

type DriverConfig struct {
	BaseDir        string          `yaml:"baseDir"`
	DriverGCConfig *DriverGCConfig `yaml:"driverGCConfig"`
}

type DriverGCConfig added in v2.0.2 

type DriverGCConfig struct {
	YoungGCThreshold  unit.Bytes    `yaml:"youngGCThreshold"`
	FullGCThreshold   unit.Bytes    `yaml:"fullGCThreshold"`
	CleanRatio        int           `yaml:"cleanRatio"`
	IntervalThreshold time.Duration `yaml:"intervalThreshold"`
}

DriverGCConfig driver gc config

type FileMetadata added in v2.0.2 

type FileMetadata struct {
	TaskID           string            `json:"taskID"`
	TaskURL          string            `json:"taskURL"`
	PieceSize        int32             `json:"pieceSize"`
	SourceFileLen    int64             `json:"sourceFileLen"`
	AccessTime       int64             `json:"accessTime"`
	Interval         int64             `json:"interval"`
	CdnFileLength    int64             `json:"cdnFileLength"`
	Digest           string            `json:"digest"`
	SourceRealDigest string            `json:"sourceRealDigest"`
	Tag              string            `json:"tag"`
	ExpireInfo       map[string]string `json:"expireInfo"`
	Finish           bool              `json:"finish"`
	Success          bool              `json:"success"`
	TotalPieceCount  int32             `json:"totalPieceCount"`
	PieceMd5Sign     string            `json:"pieceMd5Sign"`
	Range            string            `json:"range"`
	Filter           string            `json:"filter"`
}

FileMetadata meta data of task

type Manager 

type Manager interface {

	// ResetRepo reset the storage of task
	ResetRepo(task *task.SeedTask) error

	// StatDownloadFile stat download file info, if task file is not exist on storage, return errTaskNotPersisted
	StatDownloadFile(taskID string) (*storedriver.StorageInfo, error)

	// WriteDownloadFile write data to download file
	WriteDownloadFile(taskID string, offset int64, len int64, data io.Reader) error

	// ReadDownloadFile return reader of download file
	ReadDownloadFile(taskID string) (io.ReadCloser, error)

	// ReadFileMetadata return meta data of download file
	ReadFileMetadata(taskID string) (*FileMetadata, error)

	// WriteFileMetadata write file meta to storage
	WriteFileMetadata(taskID string, meta *FileMetadata) error

	// WritePieceMetaRecords write piece meta records to storage
	WritePieceMetaRecords(taskID string, metaRecords []*PieceMetaRecord) error

	// AppendPieceMetadata append piece meta data to storage
	AppendPieceMetadata(taskID string, metaRecord *PieceMetaRecord) error

	// ReadPieceMetaRecords read piece meta records from storage
	ReadPieceMetaRecords(taskID string) ([]*PieceMetaRecord, error)

	// DeleteTask delete task from storage
	DeleteTask(taskID string) error

	// TryFreeSpace checks if there is enough space for the file, return true while we are sure that there is enough space.
	TryFreeSpace(fileLength int64) (bool, error)
}

func NewManager added in v2.0.2 

func NewManager(config Config, taskManager task.Manager) (Manager, error)

type PieceMetaRecord 

type PieceMetaRecord struct {
	// piece Num start from 0
	PieceNum uint32 `json:"pieceNum"`
	// 存储到存储介质的真实长度
	PieceLen uint32 `json:"pieceLen"`
	// for transported piece content，不是origin source 的 md5，是真是存储到存储介质后的md5（为了读取数据文件时方便校验完整性）
	Md5 string `json:"md5"`
	// 下载存储到磁盘的range，不是origin source的range.提供给客户端发送下载请求,for transported piece content
	Range *rangeutils.Range `json:"range"`
	//  piece's real offset in the file
	OriginRange *rangeutils.Range `json:"originRange"`
	// 0: PlainUnspecified
	PieceStyle int32 `json:"pieceStyle"`
	// total time(millisecond) consumed
	DownloadCost      uint64 `json:"downloadCost"`
	BeginDownloadTime uint64 `json:"beginDownloadTime"`
	EndDownloadTime   uint64 `json:"endDownloadTime"`
}

PieceMetaRecord meta data of piece

func ParsePieceMetaRecord 

func ParsePieceMetaRecord(value string) (record *PieceMetaRecord, err error)

func (PieceMetaRecord) String 

func (record PieceMetaRecord) String() string

Source Files

View all Source files

Directories

Path Synopsis
Package mock is a generated GoMock package.
 Package mock is a generated GoMock package.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.