README

Incremental file directory sync tools in golang.

BENCH_FILE_SIZE=10000 ./bench.test --test.bench .
BenchmarkCopyWithTar10-4                	    2000	    995242 ns/op
BenchmarkCopyWithTar50-4                	     300	   4710021 ns/op
BenchmarkCopyWithTar200-4               	     100	  16627260 ns/op
BenchmarkCopyWithTar1000-4              	      20	  60031459 ns/op
BenchmarkCPA10-4                        	    1000	   1678367 ns/op
BenchmarkCPA50-4                        	     500	   3690306 ns/op
BenchmarkCPA200-4                       	     200	   9495066 ns/op
BenchmarkCPA1000-4                      	      50	  29769289 ns/op
BenchmarkDiffCopy10-4                   	    2000	    943889 ns/op
BenchmarkDiffCopy50-4                   	     500	   3285950 ns/op
BenchmarkDiffCopy200-4                  	     200	   8563792 ns/op
BenchmarkDiffCopy1000-4                 	      50	  29511340 ns/op
BenchmarkDiffCopyProto10-4              	    2000	    944615 ns/op
BenchmarkDiffCopyProto50-4              	     500	   3334940 ns/op
BenchmarkDiffCopyProto200-4             	     200	   9420038 ns/op
BenchmarkDiffCopyProto1000-4            	      50	  30632429 ns/op
BenchmarkIncrementalDiffCopy10-4        	    2000	    691993 ns/op
BenchmarkIncrementalDiffCopy50-4        	    1000	   1304253 ns/op
BenchmarkIncrementalDiffCopy200-4       	     500	   3306519 ns/op
BenchmarkIncrementalDiffCopy1000-4      	     200	  10211343 ns/op
BenchmarkIncrementalDiffCopy5000-4      	      20	  55194427 ns/op
BenchmarkIncrementalDiffCopy10000-4     	      20	  91759289 ns/op
BenchmarkIncrementalCopyWithTar10-4     	    2000	   1020258 ns/op
BenchmarkIncrementalCopyWithTar50-4     	     300	   5348786 ns/op
BenchmarkIncrementalCopyWithTar200-4    	     100	  19495000 ns/op
BenchmarkIncrementalCopyWithTar1000-4   	      20	  70338507 ns/op
BenchmarkIncrementalRsync10-4           	      30	  45215754 ns/op
BenchmarkIncrementalRsync50-4           	      30	  45837260 ns/op
BenchmarkIncrementalRsync200-4          	      30	  48780614 ns/op
BenchmarkIncrementalRsync1000-4         	      20	  54801892 ns/op
BenchmarkIncrementalRsync5000-4         	      20	  84782542 ns/op
BenchmarkIncrementalRsync10000-4        	      10	 103355108 ns/op
BenchmarkRsync10-4                      	      30	  46776470 ns/op
BenchmarkRsync50-4                      	      30	  48601555 ns/op
BenchmarkRsync200-4                     	      20	  59642691 ns/op
BenchmarkRsync1000-4                    	      20	 101343010 ns/op
BenchmarkGnuTar10-4                     	     500	   3171448 ns/op
BenchmarkGnuTar50-4                     	     300	   5030296 ns/op
BenchmarkGnuTar200-4                    	     100	  10464313 ns/op
BenchmarkGnuTar1000-4                   	      50	  30375257 ns/op
Expand ▾ Collapse ▴

Documentation

Overview

Package fsutil is a generated protocol buffer package.

It is generated from these files:

stat.proto
wire.proto

It has these top-level messages:

Stat
Packet

Index

Constants

This section is empty.

Variables

View Source
var (
	ErrInvalidLengthStat = fmt.Errorf("proto: negative length found during unmarshaling")
	ErrIntOverflowStat   = fmt.Errorf("proto: integer overflow")
)
View Source
var (
	ErrInvalidLengthWire = fmt.Errorf("proto: negative length found during unmarshaling")
	ErrIntOverflowWire   = fmt.Errorf("proto: integer overflow")
)
View Source
var Packet_PacketType_name = map[int32]string{
	0: "PACKET_STAT",
	1: "PACKET_REQ",
	2: "PACKET_DATA",
	3: "PACKET_FIN",
}
View Source
var Packet_PacketType_value = map[string]int32{
	"PACKET_STAT": 0,
	"PACKET_REQ":  1,
	"PACKET_DATA": 2,
	"PACKET_FIN":  3,
}

Functions

func Changes

func Changes(ctx context.Context, a, b walkerFn, changeFn ChangeFunc) error

func ComparePath

func ComparePath(p1, p2 string) int

func GetWalkerFn

func GetWalkerFn(root string) walkerFn

func NewTarsumHash

func NewTarsumHash(p string, fi os.FileInfo) (hash.Hash, error)

func Receive

func Receive(ctx context.Context, conn Stream, dest string, notifyHashed ChangeFunc) error

func Send

func Send(ctx context.Context, conn Stream, root string, opt *WalkOpt, progressCb func(int, bool)) error

func Walk

func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) error

Types

type ChangeFunc

type ChangeFunc func(ChangeKind, string, os.FileInfo, error) error

ChangeFunc is the type of function called for each change computed during a directory changes calculation.

type ChangeKind

type ChangeKind int

ChangeKind is the type of modification that a change is making.

const (
	// ChangeKindAdd represents an addition of
	// a file
	ChangeKindAdd ChangeKind = iota

	// ChangeKindModify represents a change to
	// an existing file
	ChangeKindModify

	// ChangeKindDelete represents a delete of
	// a file
	ChangeKindDelete
)

type DiskWriter

type DiskWriter struct {

	// contains filtered or unexported fields

}

func NewDiskWriter

func NewDiskWriter(ctx context.Context, dest string, opt DiskWriterOpt) (*DiskWriter, error)

func (*DiskWriter) HandleChange

func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) (retErr error)

func (*DiskWriter) Wait

func (dw *DiskWriter) Wait(ctx context.Context) error

type DiskWriterOpt

type DiskWriterOpt struct {
	AsyncDataCb WriteToFunc
	SyncDataCb  WriteToFunc
	NotifyCb    func(ChangeKind, string, os.FileInfo, error) error
}

type HandleChangeFn

type HandleChangeFn func(ChangeKind, string, os.FileInfo, error) error
type Hardlinks struct {
	// contains filtered or unexported fields

}

func (*Hardlinks) HandleChange

func (v *Hardlinks) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) error

type Packet

type Packet struct {
	Type Packet_PacketType `protobuf:"varint,1,opt,name=type,proto3,enum=fsutil.Packet_PacketType" json:"type,omitempty"`
	Stat *Stat             `protobuf:"bytes,2,opt,name=stat" json:"stat,omitempty"`
	ID   uint32            `protobuf:"varint,3,opt,name=ID,proto3" json:"ID,omitempty"`
	Data []byte            `protobuf:"bytes,4,opt,name=data,proto3" json:"data,omitempty"`
}

func (*Packet) Descriptor

func (*Packet) Descriptor() ([]byte, []int)

func (*Packet) Equal

func (this *Packet) Equal(that interface{}) bool

func (*Packet) GetData

func (m *Packet) GetData() []byte

func (*Packet) GetID

func (m *Packet) GetID() uint32

func (*Packet) GetStat

func (m *Packet) GetStat() *Stat

func (*Packet) GetType

func (m *Packet) GetType() Packet_PacketType

func (*Packet) GoString

func (this *Packet) GoString() string

func (*Packet) Marshal

func (m *Packet) Marshal() (dAtA []byte, err error)

func (*Packet) MarshalTo

func (m *Packet) MarshalTo(dAtA []byte) (int, error)

func (*Packet) ProtoMessage

func (*Packet) ProtoMessage()

func (*Packet) Reset

func (m *Packet) Reset()

func (*Packet) Size

func (m *Packet) Size() (n int)

func (*Packet) String

func (this *Packet) String() string

func (*Packet) Unmarshal

func (m *Packet) Unmarshal(dAtA []byte) error

type Packet_PacketType

type Packet_PacketType int32
const (
	PACKET_STAT Packet_PacketType = 0
	PACKET_REQ  Packet_PacketType = 1
	PACKET_DATA Packet_PacketType = 2
	PACKET_FIN  Packet_PacketType = 3
)

func (Packet_PacketType) EnumDescriptor

func (Packet_PacketType) EnumDescriptor() ([]byte, []int)

func (Packet_PacketType) String

func (x Packet_PacketType) String() string

type Stat

type Stat struct {
	Path    string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
	Mode    uint32 `protobuf:"varint,2,opt,name=mode,proto3" json:"mode,omitempty"`
	Uid     uint32 `protobuf:"varint,3,opt,name=uid,proto3" json:"uid,omitempty"`
	Gid     uint32 `protobuf:"varint,4,opt,name=gid,proto3" json:"gid,omitempty"`
	Size_   int64  `protobuf:"varint,5,opt,name=size,proto3" json:"size,omitempty"`
	ModTime int64  `protobuf:"varint,6,opt,name=modTime,proto3" json:"modTime,omitempty"`
	// int32 typeflag = 7;
	Linkname string            `protobuf:"bytes,7,opt,name=linkname,proto3" json:"linkname,omitempty"`
	Devmajor int64             `protobuf:"varint,8,opt,name=devmajor,proto3" json:"devmajor,omitempty"`
	Devminor int64             `protobuf:"varint,9,opt,name=devminor,proto3" json:"devminor,omitempty"`
	Xattrs   map[string][]byte "" /* 147 byte string literal not displayed */
}

func (*Stat) Descriptor

func (*Stat) Descriptor() ([]byte, []int)

func (*Stat) Equal

func (this *Stat) Equal(that interface{}) bool

func (*Stat) GetDevmajor

func (m *Stat) GetDevmajor() int64

func (*Stat) GetDevminor

func (m *Stat) GetDevminor() int64

func (*Stat) GetGid

func (m *Stat) GetGid() uint32

func (*Stat) GetLinkname

func (m *Stat) GetLinkname() string

func (*Stat) GetModTime

func (m *Stat) GetModTime() int64

func (*Stat) GetMode

func (m *Stat) GetMode() uint32

func (*Stat) GetPath

func (m *Stat) GetPath() string

func (*Stat) GetSize_

func (m *Stat) GetSize_() int64

func (*Stat) GetUid

func (m *Stat) GetUid() uint32

func (*Stat) GetXattrs

func (m *Stat) GetXattrs() map[string][]byte

func (*Stat) GoString

func (this *Stat) GoString() string

func (*Stat) Marshal

func (m *Stat) Marshal() (dAtA []byte, err error)

func (*Stat) MarshalTo

func (m *Stat) MarshalTo(dAtA []byte) (int, error)

func (*Stat) ProtoMessage

func (*Stat) ProtoMessage()

func (*Stat) Reset

func (m *Stat) Reset()

func (*Stat) Size

func (m *Stat) Size() (n int)

func (*Stat) String

func (this *Stat) String() string

func (*Stat) Unmarshal

func (m *Stat) Unmarshal(dAtA []byte) error

type StatInfo

type StatInfo struct {
	*Stat
}

func (*StatInfo) IsDir

func (s *StatInfo) IsDir() bool

func (*StatInfo) ModTime

func (s *StatInfo) ModTime() time.Time

func (*StatInfo) Mode

func (s *StatInfo) Mode() os.FileMode

func (*StatInfo) Name

func (s *StatInfo) Name() string

func (*StatInfo) Size

func (s *StatInfo) Size() int64

func (*StatInfo) Sys

func (s *StatInfo) Sys() interface{}

type Stream

type Stream interface {
	RecvMsg(interface{}) error
	SendMsg(m interface{}) error
	Context() context.Context
}

type Validator

type Validator struct {
	// contains filtered or unexported fields

}

func (*Validator) HandleChange

func (v *Validator) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) (retErr error)

type WalkOpt

type WalkOpt struct {
	IncludePaths    []string // todo: remove?
	ExcludePatterns []string
}

type WriteToFunc

type WriteToFunc func(context.Context, string, io.WriteCloser) error

Directories

Path Synopsis
bench
cmd/receive
cmd/send
cmd/walk