upload

Documentation

Index

• Variables
• type FSScanner
  • func NewFSScanner(root string, db *FileDb, numWorkers int) *FSScanner
  • func (fs *FSScanner) AddExcludePattern(pat string)
  • func (fs *FSScanner) AddIncludePattern(pat string)
  • func (fs *FSScanner) Walk(out chan *FileInfo) error
  • func (fs *FSScanner) WalkWithFilters(filters []Filter)
• type FileDb
  • func NewFileDb(path string) *FileDb
  • func (db *FileDb) Check(fileinfo *FileInfo) (bool, error)
  • func (db *FileDb) Close()
  • func (db *FileDb) Update(fileinfo *FileInfo) error
• type FileInfo
  • func NewFileInfo(path string, stat os.FileInfo) *FileInfo
• type FileStat
• type Filter
  • func FilterFunc(fn func(*FileInfo) (bool, error)) Filter
  • func MinimumSizeFilter(minSize int64) Filter
  • func NonEmptyFileFilter() Filter
  • func StableFileFilter() Filter
• type FpChecker
  • func NewFingerprintChecker(httpClient *util.HttpClient, mimeTypesByExt map[string]string) *FpChecker
  • func (c *FpChecker) Handle(fileinfo *FileInfo) (bool, error)
• type RateLimitedBuffer
  • func RateLimitedOpen(path string, rl *RateLimiter) (*RateLimitedBuffer, error)
  • func (rbuf *RateLimitedBuffer) Close() error
  • func (rbuf *RateLimitedBuffer) Read(p []byte) (int, error)
• type RateLimiter
  • func NewRateLimiter(rate int) *RateLimiter
  • func (rl *RateLimiter) Tick() int
• type SinkFunc
• type Uploader
  • func NewUploader(httpClient *util.HttpClient, rateLimit int) *Uploader
  • func (u *Uploader) Handle(fileinfo *FileInfo) (bool, error)

Variables

var (
	MIN_BLOCK_SIZE = 1024
)

Types

type FSScanner

type FSScanner struct {
	Root string
	// contains filtered or unexported fields
}

FSScanner represents a filesystem scanner that can include or exclude specific patterns.

func NewFSScanner

func NewFSScanner(root string, db *FileDb, numWorkers int) *FSScanner

func (*FSScanner) AddExcludePattern

func (fs *FSScanner) AddExcludePattern(pat string)

func (*FSScanner) AddIncludePattern

func (fs *FSScanner) AddIncludePattern(pat string)

func (*FSScanner) Walk

func (fs *FSScanner) Walk(out chan *FileInfo) error

func (*FSScanner) WalkWithFilters

func (fs *FSScanner) WalkWithFilters(filters []Filter)

WalkWithFilters scans the filesystem and applies the specified filters to the files that are found.

type FileDb

type FileDb struct {
	// contains filtered or unexported fields
}

FileDb is a database that keeps track of which files have been seen and whether they have been modified.

func NewFileDb

func NewFileDb(path string) *FileDb

func (*FileDb) Check

func (db *FileDb) Check(fileinfo *FileInfo) (bool, error)

Check returns true if the file is new (has not been seen before, or the modification time or size have changed since).

func (*FileDb) Close

func (db *FileDb) Close()

func (*FileDb) Update

func (db *FileDb) Update(fileinfo *FileInfo) error

Update saves a file's data in the database.

type FileInfo

type FileInfo struct {
	Path string
	Info os.FileInfo
	Meta map[string]string
}

func NewFileInfo

func NewFileInfo(path string, stat os.FileInfo) *FileInfo

type FileStat

type FileStat struct {
	Mtime int64
	Size  int64
}

type Filter

type Filter interface {
	Handle(*FileInfo) (bool, error)
}

Filter is the generic interface for filters.

func FilterFunc

func FilterFunc(fn func(*FileInfo) (bool, error)) Filter

FilterFunc lets you wrap a plain function with a Filter interface.

func MinimumSizeFilter

func MinimumSizeFilter(minSize int64) Filter

Filter files with a minimum size.

func NonEmptyFileFilter

func NonEmptyFileFilter() Filter

A simple filter that selects files with size greater than 0.

func StableFileFilter

func StableFileFilter() Filter

type FpChecker

type FpChecker struct {
	// contains filtered or unexported fields
}

func NewFingerprintChecker

func NewFingerprintChecker(httpClient *util.HttpClient, mimeTypesByExt map[string]string) *FpChecker

func (*FpChecker) Handle

func (c *FpChecker) Handle(fileinfo *FileInfo) (bool, error)

Handle returns true if the file should be uploaded (it is missing on the server).

type RateLimitedBuffer

type RateLimitedBuffer struct {
	// contains filtered or unexported fields
}

func RateLimitedOpen

func RateLimitedOpen(path string, rl *RateLimiter) (*RateLimitedBuffer, error)

func (*RateLimitedBuffer) Close

func (rbuf *RateLimitedBuffer) Close() error

func (*RateLimitedBuffer) Read

func (rbuf *RateLimitedBuffer) Read(p []byte) (int, error)

type RateLimiter

type RateLimiter struct {
	// contains filtered or unexported fields
}

RateLimiter represents a global rate limit in bytes/sec. The time resolution of the global ticker is controlled by the base block size.

func NewRateLimiter

func NewRateLimiter(rate int) *RateLimiter

func (*RateLimiter) Tick

func (rl *RateLimiter) Tick() int

type SinkFunc

type SinkFunc func(*FileInfo)

SinkFunc is a function that processes its input.

type Uploader

type Uploader struct {
	// contains filtered or unexported fields
}

func NewUploader

func NewUploader(httpClient *util.HttpClient, rateLimit int) *Uploader

func (*Uploader) Handle

func (u *Uploader) Handle(fileinfo *FileInfo) (bool, error)

Handle uploads files to the server.