fstore

package
v0.1.16 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 6, 2024 License: Apache-2.0 Imports: 19 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	FileIdPrefix = "file_" // prefix of file_id

	StatusNormal    = "NORMAL"  // file.status - normal
	StatusLogicDel  = "LOG_DEL" // file.status - logically deleted
	StatusPhysicDel = "PHY_DEL" // file.status - physically deleted

	PdelStrategyDirect = "direct" // file delete strategy - direct
	PdelStrategyTrash  = "trash"  // file delete strategy - trash

	ByteRangeMaxSize = 30_000_000 // 30 mb
)
View Source 
const (
	FILE_ID_COL        = "fstore_file_id"
	FILE_INFO_MIGR_SQL = "ALTER TABLE file_info ADD COLUMN fstore_file_id VARCHAR(32) NOT NULL DEFAULT '' COMMENT 'mini-fstore file id';"
)
View Source 
const (
	PropStorageDir                = "fstore.storage.dir"                 // where files are stored
	PropTrashDir                  = "fstore.trash.dir"                   // where files are dumped to
	PropTempDir                   = "fstore.tmp.dir"                     // temp directory
	PropPDelStrategy              = "fstore.pdelete.strategy"            // strategy used to 'physically' delete files
	PropSanitizeStorageTaskDryRun = "task.sanitize-storage-task.dry-run" // Enable dry run for SanitizeStorageTask
	PropEnableFstoreBackup        = "fstore.backup.enabled"

	PropMigrFileServerEnabled       = "fstore.migr.file-miso.enabled"       // file-server migration enabled
	PropMigrFileServerDryRun        = "fstore.migr.file-miso.dry-run"       // dry run file-server migration
	PropMigrFileServerStorage       = "fstore.migr.file-miso.storage"       // location file-server's files, files are copied to mini-fstore's server manually
	PropMigrFileServerMySQLUser     = "fstore.migr.file-miso.miso.user"     // file-server's db mysql user
	PropMigrFileServerMySQLPwd      = "fstore.migr.file-miso.miso.password" // file-server's db mysql password
	PropMigrFileServerMySQLDatabase = "fstore.migr.file-miso.miso.database" // file-server's db mysql schema name
	PropMigrFileServerMySQLHost     = "fstore.migr.file-miso.miso.host"     // file-server's db mysql host
	PropMigrFileServerMySQLPort     = "fstore.migr.file-miso.miso.port"     // file-server's db mysql port

)
View Source 
const (
	DEFAULT_BUFFER_SIZE = int64(64 * 1024)
)
View Source 
const (
	PropBackupAuthSecret = "fstore.backup.secret"
)
View Source 
const (
	ResCodeFstoreUpload = "fstore-upload"
)
View Source 
const (
	UnzipPipelineEventBus = "mini-fstore.unzip.pipeline"
)
View Source 
const (
	Version = "v0.1.16"
)

Variables

View Source 
var (
	ErrFileNotFound     = miso.NewErrf("File is not found").WithCode(api.FileNotFound)
	ErrFileDeleted      = miso.NewErrf("File has been deleted already").WithCode(api.FileDeleted)
	ErrUnknownError     = miso.NewErrf("Unknown error").WithCode(api.UnknownError)
	ErrFileIdRequired   = miso.NewErrf("fileId is required").WithCode(api.InvalidRequest)
	ErrFilenameRequired = miso.NewErrf("filename is required").WithCode(api.InvalidRequest)
	ErrNotZipFile       = miso.NewErrf("Not a zip file").WithCode(api.IllegalFormat)
)
View Source 
var (
	ErrInvalidAuth = miso.NewErrf("Invalid authorization").WithCode(api.InvalidAuthorization)
)
View Source 
var (
	UnzipResultCache = miso.NewRCache[api.UnzipFileReplyEvent]("mini-fstore:file:unzip:result",
		miso.RCacheConfig{
			Exp: time.Minute * 15,
		})
)

Functions

func BackupDownFileEp 

func BackupDownFileEp(inb *miso.Inbound)

func BootstrapServer 

func BootstrapServer(args []string)

func CheckAllNormalFiles 

func CheckAllNormalFiles(fileIds []string) (bool, error)

func CheckBackupAuth 

func CheckBackupAuth(rail miso.Rail, auth string) error

func CheckFileExists 

func CheckFileExists(fileId string) (bool, error)

func CopyChkSum 

func CopyChkSum(r io.Reader, w io.Writer) (int64, string, error)

CopyChkSum copy data from reader to writer and calculate the md5 checksum on the fly.

return the transferred size in bytes and the md5 checksum

func CreateFileRec 

func CreateFileRec(rail miso.Rail, c CreateFile) error

Create file record

func DefBuf 

func DefBuf() []byte

Create buffer with default size

func DeleteFileEp 

func DeleteFileEp(inb *miso.Inbound, req DeleteFileReq) (any, error)

mark file deleted

func DirectDownloadFileEp added in v0.1.11 

func DirectDownloadFileEp(inb *miso.Inbound)

func DownloadFile 

func DownloadFile(rail miso.Rail, w http.ResponseWriter, fileId string) error

Download file by file_id

func DownloadFileKey 

func DownloadFileKey(rail miso.Rail, w http.ResponseWriter, fileKey string) error

Download file by a generated random file key

func FastCheckFileExists added in v0.1.10 

func FastCheckFileExists(rail miso.Rail, fileId string) error

func FileLockKey 

func FileLockKey(fileId string) string

Concatenate file's redis lock key

func FindDuplicateFile 

func FindDuplicateFile(rail miso.Rail, db *gorm.DB, filename string, size int64, md5 string) (string, error)

func GenFileId 

func GenFileId() string

Generate random file_id

func GenFileKeyEp 

func GenFileKeyEp(inb *miso.Inbound, req DownloadFileReq) (string, error)

generate random file key for downloading the file

func GenStoragePath 

func GenStoragePath(fileId string) string

Generate file path

Property `fstore.storage.dir` is used

func GenTrashPath 

func GenTrashPath(fileId string) string

Generate file path for trashed file

Property `fstore.trash.dir` is used

func GetFileInfoEp 

func GetFileInfoEp(inb *miso.Inbound, req FileInfoReq) (api.FstoreFile, error)

Get file's info

func InitStorageDir added in v0.1.7 

func InitStorageDir(rail miso.Rail) error

Initialize storage dir

Property `fstore.storage.dir` is used

func InitTrashDir added in v0.1.7 

func InitTrashDir(rail miso.Rail) error

Initialize trash dir

Property `fstore.trash.dir` is used

func LDelFile 

func LDelFile(rail miso.Rail, db *gorm.DB, fileId string) error

Delete file logically by changing it's status

func MigrateFileServer 

func MigrateFileServer(rail miso.Rail) error

Migrate from file-server

Files must be copied to mini-fstore's machine beforehand (at least somewhere mini-fstore can access).

The location of these files must be specified in property: 'fstore.migr.file-miso.storage'.

func MultiCopyChkSum 

func MultiCopyChkSum(r io.Reader, ws ...io.Writer) (int64, string, error)

CopyChkSum copy data from reader to multiple writer(s) and calculate the md5 checksum on the fly.

return the transferred size in bytes and the md5 checksum

func NewUploadLock added in v0.1.7 

func NewUploadLock(rail miso.Rail, filename string, size int64, md5 string) *miso.RLock

func OnUnzipFileEvent added in v0.1.7 

func OnUnzipFileEvent(rail miso.Rail, evt UnzipFileEvent) error

func PhyDelFile 

func PhyDelFile(rail miso.Rail, db *gorm.DB, fileId string, op PDelFileOp) error

Mark file as physically deleted by changing it's status

func PrepareEventBus added in v0.1.7 

func PrepareEventBus(rail miso.Rail) error

func RandFileKey 

func RandFileKey(rail miso.Rail, name string, fileId string) (string, error)

Create random file key for the file

func RefreshFileKeyExp 

func RefreshFileKeyExp(rail miso.Rail, fileKey string) error

Refresh file key's expiration

func RemoveDeletedFiles added in v0.1.11 

func RemoveDeletedFiles(rail miso.Rail, db *gorm.DB) error

List logically deleted files, and based on the configured strategy, deleted them 'physically'.

This func reads property 'fstore.pdelete.strategy'.

If strategy is 'direct', files are deleted directly. If strategy is 'trash' (default), files are moved to 'trash' directory, which is specified in property 'fstore.trash.dir'

This func should only be used during server maintenance (no one can upload file).

func RemoveDeletedFilesEp added in v0.1.11 

func RemoveDeletedFilesEp(inb *miso.Inbound) (any, error)

func SanitizeStorage 

func SanitizeStorage(rail miso.Rail) error

func SanitizeStorageEp added in v0.1.11 

func SanitizeStorageEp(inb *miso.Inbound) (any, error)

func StreamFileKey 

func StreamFileKey(rail miso.Rail, w http.ResponseWriter, fileKey string, br ByteRange) error

Stream file by a generated random file key

func TempKeyDownloadFileEp added in v0.1.11 

func TempKeyDownloadFileEp(inb *miso.Inbound)

Download file

func TempKeyStreamFileEp added in v0.1.11 

func TempKeyStreamFileEp(inb *miso.Inbound)

Stream file (support byte-range requests)

func TransferFile 

func TransferFile(rail miso.Rail, w http.ResponseWriter, ff DFile, br ByteRange) error

Transfer file

func TriggerUnzipFilePipeline added in v0.1.7 

func TriggerUnzipFilePipeline(rail miso.Rail, db *gorm.DB, req api.UnzipFileReq) error

Trigger unzip file pipeline.

Unzipping is asynchrounous, the unzipped files are saved in mini-fstore, and the final result is replied to the specified event bus.

func TryMigrateFileServer added in v0.1.7 

func TryMigrateFileServer(rail miso.Rail) error

func UnzipFileEp added in v0.1.7 

func UnzipFileEp(inb *miso.Inbound, req api.UnzipFileReq) (any, error)

func UploadFile 

func UploadFile(rail miso.Rail, rd io.Reader, filename string) (string, error)

Upload file and create file record for it

return fileId or any error occured

func UploadFileEp 

func UploadFileEp(inb *miso.Inbound) (string, error)

Types

type BackupFileInf 

type BackupFileInf struct {
	Id     int64
	FileId string
	Name   string
	Status string
	Size   int64
	Md5    string
}

type ByteRange 

type ByteRange struct {
	Start int64 // start of byte range (inclusive)
	End   int64 // end of byte range (inclusive)
	// contains filtered or unexported fields
}

func ZeroByteRange 

func ZeroByteRange() ByteRange

func (ByteRange) IsZero 

func (br ByteRange) IsZero() bool

func (ByteRange) Size 

func (br ByteRange) Size() int64

type CachedFile 

type CachedFile struct {
	FileId string `json:"fileId"`
	Name   string `json:"name"`
}

func ResolveFileKey 

func ResolveFileKey(rail miso.Rail, fileKey string) (bool, CachedFile)

Resolve CachedFile for the given fileKey

type CreateFile 

type CreateFile struct {
	FileId string
	Link   string
	Name   string
	Size   int64
	Md5    string
}

type DFile 

type DFile struct {
	FileId string
	Link   string
	Size   int64
	Status string
	Name   string
}

func (*DFile) IsDeleted 

func (df *DFile) IsDeleted() bool

Check if the file is deleted already

func (*DFile) StoragePath added in v0.1.7 

func (f *DFile) StoragePath() string

Return the actual storage path including symbolic link.

A file could be a symbolic link to another file (using field f.Link).

Be cautious if this path is used to delete/remove files (i.e., it shouldn't).

type DeleteFileReq 

type DeleteFileReq struct {
	FileId string `form:"fileId" valid:"notEmpty" desc:"actual file_id of the file record"`
}

type DownloadFileReq 

type DownloadFileReq struct {
	FileId   string `form:"fileId" desc:"actual file_id of the file record"`
	Filename string `form:"filename" desc:"the name that will be used when downloading the file"`
}

type File 

type File struct {
	Id         int64       `json:"id"`
	FileId     string      `json:"fileId"`
	Link       string      `json:"-"`
	Name       string      `json:"name"`
	Status     string      `json:"status"`
	Size       int64       `json:"size"`
	Md5        string      `json:"md5"`
	UplTime    miso.ETime  `json:"uplTime"`
	LogDelTime *miso.ETime `json:"logDelTime"`
	PhyDelTime *miso.ETime `json:"phyDelTime"`
}

func FindFile 

func FindFile(db *gorm.DB, fileId string) (File, error)

Find File

func ListLDelFile 

func ListLDelFile(rail miso.Rail, idOffset int64, limit int) ([]File, error)

List logically deleted files

func (*File) IsDeleted 

func (f *File) IsDeleted() bool

Check if the file is deleted already

func (*File) IsLogiDeleted 

func (f *File) IsLogiDeleted() bool

Check if the file is logically already

func (*File) IsZero 

func (f *File) IsZero() bool

Check whether current file is of zero value

func (*File) StoragePath added in v0.1.7 

func (f *File) StoragePath() string

Return the actual storage path including symbolic link.

A file could be a symbolic link to another file (using field f.Link).

Be cautious if this path is used to delete/remove files (i.e., it shouldn't).

type FileInfo 

type FileInfo struct {
	Id         int64
	Uuid       string
	Name       string
	UploaderId int
	FsGroupId  int64
}

type FileInfoReq 

type FileInfoReq struct {
	FileId       string `form:"fileId" desc:"actual file_id of the file record"`
	UploadFileId string `form:"uploadFileId" desc:"temporary file_id returned when uploading files"`
}

type ListBackupFileReq 

type ListBackupFileReq struct {
	Limit    int64
	IdOffset int
}

type ListBackupFileResp 

type ListBackupFileResp struct {
	Files []BackupFileInf
}

func BackupListFilesEp 

func BackupListFilesEp(inb *miso.Inbound, req ListBackupFileReq) (ListBackupFileResp, error)

func ListBackupFiles 

func ListBackupFiles(rail miso.Rail, tx *gorm.DB, req ListBackupFileReq) (ListBackupFileResp, error)

type PDelFileDirectOp 

type PDelFileDirectOp struct {
}

The 'direct' implementation of of PDelFileOp, files are deleted directly

type PDelFileOp 

type PDelFileOp interface {
	// contains filtered or unexported methods
}

func NewPDelFileOp 

func NewPDelFileOp(strategy string) PDelFileOp

type PDelFileTrashOp 

type PDelFileTrashOp struct {
}

The 'trash' implementation of of PDelFileOp, files are deleted directly

type PendingPhyDelFile 

type PendingPhyDelFile struct {
	Id     int
	FileId string
}

type SavedZipEntry added in v0.1.7 

type SavedZipEntry struct {
	Md5    string
	Name   string
	Size   int64
	FileId string
}

func SaveZipFile added in v0.1.7 

func SaveZipFile(rail miso.Rail, db *gorm.DB, entry UnpackedZipEntry) (SavedZipEntry, error)

func SaveZipFiles added in v0.1.7 

func SaveZipFiles(rail miso.Rail, db *gorm.DB, entries []UnpackedZipEntry) ([]SavedZipEntry, error)

func UnzipFile added in v0.1.7 

func UnzipFile(rail miso.Rail, db *gorm.DB, evt UnzipFileEvent) ([]SavedZipEntry, error)

type TableCol 

type TableCol struct {
	Field   string
	Type    string
	Null    string
	Key     string
	Default string
	Extra   string
}

type UnpackedZipEntry added in v0.1.7 

type UnpackedZipEntry struct {
	Md5  string
	Name string
	Path string
	Size int64
}

func UnpackZip added in v0.1.7 

func UnpackZip(rail miso.Rail, f File, tempDir string) ([]UnpackedZipEntry, error)

type UnzipFileEvent added in v0.1.7 

type UnzipFileEvent struct {
	FileId          string `valid:"notEmpty"`
	ReplyToEventBus string `valid:"notEmpty"`
	Extra           string
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.