filestorage

package
v0.0.0-...-d31700d Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 28, 2022 License: MIT Imports: 24 Imported by: 0

README

filestorage

  • upload/store/download files, and clean files that are not linked to any objects.

Features

  • Sync files to multiple machines using scp command.
  • Download files using Nginx "X-Accel-Redirect" or sent file directly in response body.
  • Clean files that are not linked to any object.

Documentation

Overview

Package filestorage upload/store/download files, and clean files that are not linked to any objects.

Index

Examples

Constants

This section is empty.

Variables

This section is empty.

Functions

func CheckHash 

func CheckHash(hashes ...string) error

CheckHash checks if hashes is in file hash format(43 urlsafe base64 characters).

func Download 

func Download(req *http.Request, resp http.ResponseWriter) error

Download file according to the requested bucket, file, link object

func FileHash 

func FileHash(str string) (string, error)

FileHash returns file hash from a url or file hash

func FileHashes 

func FileHashes(strs []string) ([]string, error)

FileHashes returns file hashes from urls or file hashes

func ImgSrcToFileHash 

func ImgSrcToFileHash(html string) (result string, hashes []string)

func IsFileNotExists 

func IsFileNotExists(err error) bool

func IsHash 

func IsHash(s string) bool

IsHash returns if string s is in file hash format(43 urlsafe base64 characters).

func IsNotLinked 

func IsNotLinked(err error) bool

IsNotLinked check if an error is the not linked Error.

func ReplaceImgSrc 

func ReplaceImgSrc(html string, fn func(src string) string) string
Example 
var html = `<img> <img/>
<img src="1">
<img src="2"/>
<imG src="3" />
<IMG Src="4" / >
<img a src="5" b>
<img a src="x"b>
<img +src="x">
<img+ src="x">
`
fmt.Println(ReplaceImgSrc(html, func(src string) string { return src + "~" }))

Output:

<img> <img/>
<img src="1~">
<img src="2~"/>
<imG src="3~" />
<IMG Src="4~" / >
<img a src="5~" b>
<img a src="x"b>
<img +src="x">
<img+ src="x">

func TryFileHash 

func TryFileHash(str string) (string, error)

TryFileHash try to returns file hash from a url or file hash.

func UploadImages 

func UploadImages(req *http.Request, lang string) ([]string, error)

Types

type Bucket 

type Bucket struct {
	Name     string
	Machines []string
	Dir      string
	DirDepth uint8
	ScpUser  string

	DownloadURLPrefix string

	// Path prefix for "X-Accel-Redirect" response header when downloading.
	// If this path prefix is present, only file path is sent in the "X-Accel-Redirect" header,
	// and nginx is responsible for file downloading for a better performance.
	// Otherwise, file is sent directly in the response body.
	RedirectPathPrefix string

	DB         DB
	FilesTable string
	LinksTable string
	// contains filtered or unexported fields
}

Bucket store file on disk and infomation in database tables.

func GetBucket 

func GetBucket(bucketName string) (*Bucket, error)

func (*Bucket) CheckFile 

func (b *Bucket) CheckFile(db DB, files ...string) error

CheckFile ensure all files exists.

func (*Bucket) Download 

func (b *Bucket) Download(db DB, resp http.ResponseWriter, file string, object string) error

Download file, if object is not empty, the file must be linked to it, otherwise an error is returned. If RedirectPathPrefix is not empty, an location like following is required in nginx virtual server config. 

location /fs/ {
  internal;
  alias /data/file-storage;
}

The location prefix and alias path should be set according to RedirectPathPrefix and Dir.

func (*Bucket) DownloadURL 

func (b *Bucket) DownloadURL(linkObject interface{}, fileHash string) string

DownloadURL make the url for file download

func (*Bucket) DownloadURLs 

func (b *Bucket) DownloadURLs(linkObject interface{}, fileHashes []string) []string

DownloadURL make the urls for files download

func (*Bucket) EnsureLinked 

func (b *Bucket) EnsureLinked(db DB, object, file string) error

EnsureLinked ensure file is linked to object.

func (*Bucket) FileDir 

func (b *Bucket) FileDir(hash string) string

func (*Bucket) FilePath 

func (b *Bucket) FilePath(hash string) string

FilePath returns the file path to store on disk.

func (*Bucket) FilesOf 

func (b *Bucket) FilesOf(db DB, object string) ([]string, error)

FilesOf get all files linked to an object.

func (*Bucket) ImgSrcToDownloadURL 

func (b *Bucket) ImgSrcToDownloadURL(linkObject interface{}, html string) string

func (*Bucket) Init 

func (b *Bucket) Init(db DB) error

Init validate storage fields and create tables if not created. 

func (b *Bucket) Link(db DB, object string, files ...string) error

Link files to object.

func (*Bucket) LinkOnly 

func (b *Bucket) LinkOnly(db DB, object string, files ...string) error

LinkOnly make sure these files and only these files are linked to object.

Example 
testBucket.insertFileRecords(nil, []fileRecord{
	{Hash: "TEaLOxaZn9lXgYlXbV93DLShatn8oOeYolHwClSofF3"},
	{Hash: "TEaLOxaZn9lXgYlXbV93DLShatn8oOeYolHwClSofF4"},
})
fmt.Println(testBucket.LinkOnly(nil, "object", testFile3, testFile4))
if files, err := testBucket.FilesOf(nil, "object"); err != nil {
	fmt.Println(err)
} else {
	for _, v := range files {
		fmt.Println(v)
	}
}

fmt.Println(testBucket.EnsureLinked(nil, "object", testFile3))
fmt.Println(testBucket.Unlink(nil, "object", testFile3, testFile4))
fmt.Println(testBucket.Linked(nil, "object", testFile3))

Output:

<nil>
TEaLOxaZn9lXgYlXbV93DLShatn8oOeYolHwClSofF3
TEaLOxaZn9lXgYlXbV93DLShatn8oOeYolHwClSofF4
<nil>
<nil>
false <nil>

func (*Bucket) Linked 

func (b *Bucket) Linked(db DB, object, file string) (bool, error)

Linked check if file is linked to object.

func (*Bucket) Save 

func (b *Bucket) Save(
	db DB, fileCheck func(string, int64) error, object string, files ...File,
) (fileHashes []string, err error)

Save save files into bucket.

func (*Bucket) SaveFiles 

func (b *Bucket) SaveFiles(
	db DB, fileCheck func(string, int64) error, object string, paths ...string,
) (fileHashes []string, err error)

SaveFiles save files specified by paths into bucket.

Example 
files, err := testBucket.SaveFiles(nil, nil, "linkObject", "README.md")
if err != nil {
	panic(err)
}
fmt.Println(len(files), len(files[0]))

Output:

1 43

func (*Bucket) StartClean 

func (b *Bucket) StartClean(cleanInterval, cleanAfter time.Duration, logger Logger)
Example 
testUpload()
testBucket.StartClean(time.Second, time.Nanosecond, logger.New(os.Stdout))
time.Sleep(time.Second)

Output:


func (b *Bucket) Unlink(db DB, object string, files ...string) error

Unlink files from object.

func (*Bucket) UnlinkAllOf 

func (b *Bucket) UnlinkAllOf(db DB, object string) error

UnlinkAllOf unlink all linked files from an object.

func (*Bucket) Upload 

func (b *Bucket) Upload(
	db DB, fileCheck func(string, int64) error, object string, fileHeaders ...*multipart.FileHeader,
) ([]string, error)

Upload files, if object is not empty, the files are linked to it.

Example 
fmt.Println(testUpload())

Output:

[TEaLOxaZn9lXgYlXbV93DLShatn8oOeYolHwClSofF0 HEbi8PV2cRPf8QeB8lesh6gWPAmiAda7xGarbjAv8v4]

type DB 

type DB interface {
	QueryRow(query string, args ...interface{}) *sql.Row
	Query(query string, args ...interface{}) (*sql.Rows, error)
	Exec(query string, args ...interface{}) (sql.Result, error)
}

DB represents *sql.DB or *sql.Tx

type File 

type File struct {
	IO   io.ReadSeeker // bytes.Reader and strings.Reader implemented this interface.
	Size int64
}

File reprents the file to store.

type LinkObject 

type LinkObject struct {
	Table string // Table name, required.
	ID    int64  // Primary key or unique key, required.
	Field string // If the table has multiple file storage fields, Field is required, Otherwise Field can be omitted.
}

LinkObject is a structured reference implementaion for link object string.

func (LinkObject) MarshalJSON 

func (o LinkObject) MarshalJSON() ([]byte, error)

MarshalJSON implements json.Marshaler

Example 
object := LinkObject{Table: "x", ID: 1}
b, err := json.Marshal(object)
fmt.Println(string(b), err)

Output:

"x|1" <nil>

func (LinkObject) String 

func (o LinkObject) String() string

func (*LinkObject) UnmarshalJSON 

func (o *LinkObject) UnmarshalJSON(b []byte) error

UnmarshalJSON implements json.Unmarshaler

Example 
object := LinkObject{}
err := json.Unmarshal([]byte(`"a.x|1|b"`), &object)
fmt.Println(err)
fmt.Printf("%#v\n", object)

Output:

<nil>
filestorage.LinkObject{Table:"a.x", ID:1, Field:"b"}

type Logger 

type Logger interface {
	Error(args ...interface{})
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.