fsutil

README

Incremental file directory sync tools in golang.

BENCH_FILE_SIZE=10000 ./bench.test --test.bench .
BenchmarkCopyWithTar10-4                	    2000	    995242 ns/op
BenchmarkCopyWithTar50-4                	     300	   4710021 ns/op
BenchmarkCopyWithTar200-4               	     100	  16627260 ns/op
BenchmarkCopyWithTar1000-4              	      20	  60031459 ns/op
BenchmarkCPA10-4                        	    1000	   1678367 ns/op
BenchmarkCPA50-4                        	     500	   3690306 ns/op
BenchmarkCPA200-4                       	     200	   9495066 ns/op
BenchmarkCPA1000-4                      	      50	  29769289 ns/op
BenchmarkDiffCopy10-4                   	    2000	    943889 ns/op
BenchmarkDiffCopy50-4                   	     500	   3285950 ns/op
BenchmarkDiffCopy200-4                  	     200	   8563792 ns/op
BenchmarkDiffCopy1000-4                 	      50	  29511340 ns/op
BenchmarkDiffCopyProto10-4              	    2000	    944615 ns/op
BenchmarkDiffCopyProto50-4              	     500	   3334940 ns/op
BenchmarkDiffCopyProto200-4             	     200	   9420038 ns/op
BenchmarkDiffCopyProto1000-4            	      50	  30632429 ns/op
BenchmarkIncrementalDiffCopy10-4        	    2000	    691993 ns/op
BenchmarkIncrementalDiffCopy50-4        	    1000	   1304253 ns/op
BenchmarkIncrementalDiffCopy200-4       	     500	   3306519 ns/op
BenchmarkIncrementalDiffCopy1000-4      	     200	  10211343 ns/op
BenchmarkIncrementalDiffCopy5000-4      	      20	  55194427 ns/op
BenchmarkIncrementalDiffCopy10000-4     	      20	  91759289 ns/op
BenchmarkIncrementalCopyWithTar10-4     	    2000	   1020258 ns/op
BenchmarkIncrementalCopyWithTar50-4     	     300	   5348786 ns/op
BenchmarkIncrementalCopyWithTar200-4    	     100	  19495000 ns/op
BenchmarkIncrementalCopyWithTar1000-4   	      20	  70338507 ns/op
BenchmarkIncrementalRsync10-4           	      30	  45215754 ns/op
BenchmarkIncrementalRsync50-4           	      30	  45837260 ns/op
BenchmarkIncrementalRsync200-4          	      30	  48780614 ns/op
BenchmarkIncrementalRsync1000-4         	      20	  54801892 ns/op
BenchmarkIncrementalRsync5000-4         	      20	  84782542 ns/op
BenchmarkIncrementalRsync10000-4        	      10	 103355108 ns/op
BenchmarkRsync10-4                      	      30	  46776470 ns/op
BenchmarkRsync50-4                      	      30	  48601555 ns/op
BenchmarkRsync200-4                     	      20	  59642691 ns/op
BenchmarkRsync1000-4                    	      20	 101343010 ns/op
BenchmarkGnuTar10-4                     	     500	   3171448 ns/op
BenchmarkGnuTar50-4                     	     300	   5030296 ns/op
BenchmarkGnuTar200-4                    	     100	  10464313 ns/op
BenchmarkGnuTar1000-4                   	      50	  30375257 ns/op

Documentation

Index

• func Changes(ctx context.Context, a, b walkerFn, changeFn ChangeFunc) error
• func ComparePath(p1, p2 string) int
• func FollowLinks(fs FS, paths []string) ([]string, error)
• func Receive(ctx context.Context, conn Stream, dest string, opt ReceiveOpt) error
• func Send(ctx context.Context, conn Stream, fs FS, progressCb func(int, bool)) error
• func Stat(path string) (*types.Stat, error)
• func Walk(ctx context.Context, p string, opt *FilterOpt, fn filepath.WalkFunc) error
• func WalkDir(ctx context.Context, p string, opt *FilterOpt, fn gofs.WalkDirFunc) error
• func WriteTar(ctx context.Context, fs FS, w io.Writer) error
• type ChangeFunc
• type ChangeKind
  • func (k ChangeKind) String() string
• type ContentHasher
• type DiffType
• type Dir
• type DirEntryInfo
  • func (s *DirEntryInfo) Info() (gofs.FileInfo, error)
  • func (s *DirEntryInfo) IsDir() bool
  • func (s *DirEntryInfo) Name() string
  • func (s *DirEntryInfo) Type() gofs.FileMode
• type DiskWriter
  • func NewDiskWriter(ctx context.Context, dest string, opt DiskWriterOpt) (*DiskWriter, error)
  • func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) (retErr error)
  • func (dw *DiskWriter) Wait(ctx context.Context) error
• type DiskWriterOpt
• type FS
  • func NewFS(root string) (FS, error)
  • func NewFilterFS(fs FS, opt *FilterOpt) (FS, error)
  • func SubDirFS(dirs []Dir) (FS, error)
  • func WithHardlinkReset(fs FS) FS
• type FilterFunc
• type FilterOpt
• type HandleChangeFn
• type Hardlinks
  • func (v *Hardlinks) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) error
• type MapFunc
• type MapResult
• type ReceiveOpt
• type StatInfo
  • func (s *StatInfo) IsDir() bool
  • func (s *StatInfo) ModTime() time.Time
  • func (s *StatInfo) Mode() os.FileMode
  • func (s *StatInfo) Name() string
  • func (s *StatInfo) Size() int64
  • func (s *StatInfo) Sys() interface{}
• type Stream
• type Validator
  • func (v *Validator) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) (retErr error)
• type WriteToFunc

Functions

func Changes

func Changes(ctx context.Context, a, b walkerFn, changeFn ChangeFunc) error

func ComparePath

func ComparePath(p1, p2 string) int

func FollowLinks

func FollowLinks(fs FS, paths []string) ([]string, error)

func Receive

func Receive(ctx context.Context, conn Stream, dest string, opt ReceiveOpt) error

func Send

func Send(ctx context.Context, conn Stream, fs FS, progressCb func(int, bool)) error

func Stat

func Stat(path string) (*types.Stat, error)

func Walk

func Walk(ctx context.Context, p string, opt *FilterOpt, fn filepath.WalkFunc) error

func WalkDir

func WalkDir(ctx context.Context, p string, opt *FilterOpt, fn gofs.WalkDirFunc) error

func WriteTar

func WriteTar(ctx context.Context, fs FS, w io.Writer) error

Types

type ChangeFunc

type ChangeFunc func(ChangeKind, string, os.FileInfo, error) error

ChangeFunc is the type of function called for each change computed during a directory changes calculation.

type ChangeKind

type ChangeKind int

ChangeKind is the type of modification that a change is making.

const (
	// ChangeKindAdd represents an addition of
	// a file
	ChangeKindAdd ChangeKind = iota

	// ChangeKindModify represents a change to
	// an existing file
	ChangeKindModify

	// ChangeKindDelete represents a delete of
	// a file
	ChangeKindDelete
)

func (ChangeKind) String

func (k ChangeKind) String() string

type ContentHasher

type ContentHasher func(*types.Stat) (hash.Hash, error)

type DiffType

type DiffType int

const (
	DiffMetadata DiffType = iota
	DiffNone
	DiffContent
)

type Dir

type Dir struct {
	Stat *types.Stat
	FS   FS
}

type DirEntryInfo

type DirEntryInfo struct {
	*types.Stat
	// contains filtered or unexported fields
}

func (*DirEntryInfo) Info

func (s *DirEntryInfo) Info() (gofs.FileInfo, error)

func (*DirEntryInfo) IsDir

func (s *DirEntryInfo) IsDir() bool

func (*DirEntryInfo) Name

func (s *DirEntryInfo) Name() string

func (*DirEntryInfo) Type

func (s *DirEntryInfo) Type() gofs.FileMode

type DiskWriter

type DiskWriter struct {
	// contains filtered or unexported fields
}

func NewDiskWriter

func NewDiskWriter(ctx context.Context, dest string, opt DiskWriterOpt) (*DiskWriter, error)

func (*DiskWriter) HandleChange

func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) (retErr error)

func (*DiskWriter) Wait

func (dw *DiskWriter) Wait(ctx context.Context) error

type DiskWriterOpt

type DiskWriterOpt struct {
	AsyncDataCb   WriteToFunc
	SyncDataCb    WriteToFunc
	NotifyCb      func(ChangeKind, string, os.FileInfo, error) error
	ContentHasher ContentHasher
	Filter        FilterFunc
}

type FS

type FS interface {
	Walk(context.Context, string, gofs.WalkDirFunc) error
	Open(string) (io.ReadCloser, error)
}

func NewFS

func NewFS(root string) (FS, error)

NewFS creates a new FS from a root directory on the host filesystem.

func NewFilterFS

func NewFilterFS(fs FS, opt *FilterOpt) (FS, error)

NewFilterFS creates a new FS that filters the given FS using the given FilterOpt.

The returned FS will not contain any paths that do not match the provided include and exclude patterns, or that are are excluded using the mapping function.

The FS is assumed to be a snapshot of the filesystem at the time of the call to NewFilterFS. If the underlying filesystem changes, calls to the underlying FS may be inconsistent.

func SubDirFS

func SubDirFS(dirs []Dir) (FS, error)

func WithHardlinkReset

func WithHardlinkReset(fs FS) FS

WithHardlinkReset returns a FS that fixes hardlinks for FS that has been filtered so that original hardlink sources might be missing

type FilterFunc

type FilterFunc func(string, *types.Stat) bool

type FilterOpt

type FilterOpt struct {
	// IncludePatterns requires that the path matches at least one of the
	// specified patterns.
	IncludePatterns []string

	// ExcludePatterns requires that the path does not match any of the
	// specified patterns.
	ExcludePatterns []string

	// FollowPaths contains symlinks that are resolved into IncludePatterns
	// at the time of the call to NewFilterFS.
	FollowPaths []string

	// Map is called for each path that is included in the result.
	// The function can modify the stat info for each element, while the result
	// of the function controls both how Walk continues.
	Map MapFunc
}

type HandleChangeFn

type HandleChangeFn func(ChangeKind, string, os.FileInfo, error) error

type Hardlinks

type Hardlinks struct {
	// contains filtered or unexported fields
}

func (*Hardlinks) HandleChange

func (v *Hardlinks) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) error

type MapFunc

type MapFunc func(string, *types.Stat) MapResult

type MapResult

type MapResult int

The result of the walk function controls both how WalkDir continues and whether the path is kept.

const (
	// Keep the current path and continue.
	MapResultKeep MapResult = iota

	// Exclude the current path and continue.
	MapResultExclude

	// Exclude the current path, and skip the rest of the dir.
	// If path is a dir, skip the current directory.
	// If path is a file, skip the rest of the parent directory.
	// (This matches the semantics of fs.SkipDir.)
	MapResultSkipDir
)

type ReceiveOpt

type ReceiveOpt struct {
	NotifyHashed  ChangeFunc
	ContentHasher ContentHasher
	ProgressCb    func(int, bool)
	Merge         bool
	Filter        FilterFunc
	Differ        DiffType
}

type StatInfo

type StatInfo struct {
	*types.Stat
}

func (*StatInfo) IsDir

func (s *StatInfo) IsDir() bool

func (*StatInfo) ModTime

func (s *StatInfo) ModTime() time.Time

func (*StatInfo) Mode

func (s *StatInfo) Mode() os.FileMode

func (*StatInfo) Name

func (s *StatInfo) Name() string

func (*StatInfo) Size

func (s *StatInfo) Size() int64

func (*StatInfo) Sys

func (s *StatInfo) Sys() interface{}

type Stream

type Stream interface {
	RecvMsg(interface{}) error
	SendMsg(m interface{}) error
	Context() context.Context
}

type Validator

type Validator struct {
	// contains filtered or unexported fields
}

func (*Validator) HandleChange

func (v *Validator) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) (retErr error)

type WriteToFunc

type WriteToFunc func(context.Context, string, io.WriteCloser) error