README

Incremental file directory sync tools in golang.

BENCH_FILE_SIZE=10000 ./bench.test --test.bench .
BenchmarkCopyWithTar10-4                	    2000	    995242 ns/op
BenchmarkCopyWithTar50-4                	     300	   4710021 ns/op
BenchmarkCopyWithTar200-4               	     100	  16627260 ns/op
BenchmarkCopyWithTar1000-4              	      20	  60031459 ns/op
BenchmarkCPA10-4                        	    1000	   1678367 ns/op
BenchmarkCPA50-4                        	     500	   3690306 ns/op
BenchmarkCPA200-4                       	     200	   9495066 ns/op
BenchmarkCPA1000-4                      	      50	  29769289 ns/op
BenchmarkDiffCopy10-4                   	    2000	    943889 ns/op
BenchmarkDiffCopy50-4                   	     500	   3285950 ns/op
BenchmarkDiffCopy200-4                  	     200	   8563792 ns/op
BenchmarkDiffCopy1000-4                 	      50	  29511340 ns/op
BenchmarkDiffCopyProto10-4              	    2000	    944615 ns/op
BenchmarkDiffCopyProto50-4              	     500	   3334940 ns/op
BenchmarkDiffCopyProto200-4             	     200	   9420038 ns/op
BenchmarkDiffCopyProto1000-4            	      50	  30632429 ns/op
BenchmarkIncrementalDiffCopy10-4        	    2000	    691993 ns/op
BenchmarkIncrementalDiffCopy50-4        	    1000	   1304253 ns/op
BenchmarkIncrementalDiffCopy200-4       	     500	   3306519 ns/op
BenchmarkIncrementalDiffCopy1000-4      	     200	  10211343 ns/op
BenchmarkIncrementalDiffCopy5000-4      	      20	  55194427 ns/op
BenchmarkIncrementalDiffCopy10000-4     	      20	  91759289 ns/op
BenchmarkIncrementalCopyWithTar10-4     	    2000	   1020258 ns/op
BenchmarkIncrementalCopyWithTar50-4     	     300	   5348786 ns/op
BenchmarkIncrementalCopyWithTar200-4    	     100	  19495000 ns/op
BenchmarkIncrementalCopyWithTar1000-4   	      20	  70338507 ns/op
BenchmarkIncrementalRsync10-4           	      30	  45215754 ns/op
BenchmarkIncrementalRsync50-4           	      30	  45837260 ns/op
BenchmarkIncrementalRsync200-4          	      30	  48780614 ns/op
BenchmarkIncrementalRsync1000-4         	      20	  54801892 ns/op
BenchmarkIncrementalRsync5000-4         	      20	  84782542 ns/op
BenchmarkIncrementalRsync10000-4        	      10	 103355108 ns/op
BenchmarkRsync10-4                      	      30	  46776470 ns/op
BenchmarkRsync50-4                      	      30	  48601555 ns/op
BenchmarkRsync200-4                     	      20	  59642691 ns/op
BenchmarkRsync1000-4                    	      20	 101343010 ns/op
BenchmarkGnuTar10-4                     	     500	   3171448 ns/op
BenchmarkGnuTar50-4                     	     300	   5030296 ns/op
BenchmarkGnuTar200-4                    	     100	  10464313 ns/op
BenchmarkGnuTar1000-4                   	      50	  30375257 ns/op
Expand ▾ Collapse ▴

Documentation

Overview

Package fsutil is a generated protocol buffer package.

It is generated from these files: 

stat.proto
wire.proto

It has these top-level messages: 

Stat
Packet

Index

Constants

This section is empty.

Variables

View Source 
var (
	ErrInvalidLengthStat = fmt.Errorf("proto: negative length found during unmarshaling")
	ErrIntOverflowStat   = fmt.Errorf("proto: integer overflow")
)
View Source 
var (
	ErrInvalidLengthWire = fmt.Errorf("proto: negative length found during unmarshaling")
	ErrIntOverflowWire   = fmt.Errorf("proto: integer overflow")
)
View Source 
var Packet_PacketType_name = map[int32]string{
	0: "PACKET_STAT",
	1: "PACKET_REQ",
	2: "PACKET_DATA",
	3: "PACKET_FIN",
}
View Source 
var Packet_PacketType_value = map[string]int32{
	"PACKET_STAT": 0,
	"PACKET_REQ":  1,
	"PACKET_DATA": 2,
	"PACKET_FIN":  3,
}

Functions

func Changes 

func Changes(ctx context.Context, a, b walkerFn, changeFn ChangeFunc) error

func ComparePath 

func ComparePath(p1, p2 string) int

func GetWalkerFn 

func GetWalkerFn(root string) walkerFn

func NewTarsumHash 

func NewTarsumHash(p string, fi os.FileInfo) (hash.Hash, error)

func Receive 

func Receive(ctx context.Context, conn Stream, dest string, notifyHashed ChangeFunc) error

func Send 

func Send(ctx context.Context, conn Stream, root string, opt *WalkOpt, progressCb func(int, bool)) error

func Walk 

func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) error

Types

type ChangeFunc 

type ChangeFunc func(ChangeKind, string, os.FileInfo, error) error

ChangeFunc is the type of function called for each change computed during a directory changes calculation.

type ChangeKind 

type ChangeKind int

ChangeKind is the type of modification that a change is making. 

const (
	// ChangeKindAdd represents an addition of
	// a file
	ChangeKindAdd ChangeKind = iota

	// ChangeKindModify represents a change to
	// an existing file
	ChangeKindModify

	// ChangeKindDelete represents a delete of
	// a file
	ChangeKindDelete
)

type DiskWriter 

type DiskWriter struct {

	// contains filtered or unexported fields

}

func NewDiskWriter 

func NewDiskWriter(ctx context.Context, dest string, opt DiskWriterOpt) (*DiskWriter, error)

func (*DiskWriter) HandleChange 

func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) (retErr error)

func (*DiskWriter) Wait 

func (dw *DiskWriter) Wait(ctx context.Context) error

type DiskWriterOpt 

type DiskWriterOpt struct {
	AsyncDataCb WriteToFunc
	SyncDataCb  WriteToFunc
	NotifyCb    func(ChangeKind, string, os.FileInfo, error) error
}

type HandleChangeFn 

type HandleChangeFn func(ChangeKind, string, os.FileInfo, error) error
type Hardlinks struct {
	// contains filtered or unexported fields

}

func (*Hardlinks) HandleChange 

func (v *Hardlinks) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) error

type Packet 

type Packet struct {
	Type Packet_PacketType `protobuf:"varint,1,opt,name=type,proto3,enum=fsutil.Packet_PacketType" json:"type,omitempty"`
	Stat *Stat             `protobuf:"bytes,2,opt,name=stat" json:"stat,omitempty"`
	ID   uint32            `protobuf:"varint,3,opt,name=ID,proto3" json:"ID,omitempty"`
	Data []byte            `protobuf:"bytes,4,opt,name=data,proto3" json:"data,omitempty"`
}

func (*Packet) Descriptor 

func (*Packet) Descriptor() ([]byte, []int)

func (*Packet) Equal 

func (this *Packet) Equal(that interface{}) bool

func (*Packet) GetData 

func (m *Packet) GetData() []byte

func (*Packet) GetID 

func (m *Packet) GetID() uint32

func (*Packet) GetStat 

func (m *Packet) GetStat() *Stat

func (*Packet) GetType 

func (m *Packet) GetType() Packet_PacketType

func (*Packet) GoString 

func (this *Packet) GoString() string

func (*Packet) Marshal 

func (m *Packet) Marshal() (dAtA []byte, err error)

func (*Packet) MarshalTo 

func (m *Packet) MarshalTo(dAtA []byte) (int, error)

func (*Packet) ProtoMessage 

func (*Packet) ProtoMessage()

func (*Packet) Reset 

func (m *Packet) Reset()

func (*Packet) Size 

func (m *Packet) Size() (n int)

func (*Packet) String 

func (this *Packet) String() string

func (*Packet) Unmarshal 

func (m *Packet) Unmarshal(dAtA []byte) error

type Packet_PacketType 

type Packet_PacketType int32
const (
	PACKET_STAT Packet_PacketType = 0
	PACKET_REQ  Packet_PacketType = 1
	PACKET_DATA Packet_PacketType = 2
	PACKET_FIN  Packet_PacketType = 3
)

func (Packet_PacketType) EnumDescriptor 

func (Packet_PacketType) EnumDescriptor() ([]byte, []int)

func (Packet_PacketType) String 

func (x Packet_PacketType) String() string

type Stat 

type Stat struct {
	Path    string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
	Mode    uint32 `protobuf:"varint,2,opt,name=mode,proto3" json:"mode,omitempty"`
	Uid     uint32 `protobuf:"varint,3,opt,name=uid,proto3" json:"uid,omitempty"`
	Gid     uint32 `protobuf:"varint,4,opt,name=gid,proto3" json:"gid,omitempty"`
	Size_   int64  `protobuf:"varint,5,opt,name=size,proto3" json:"size,omitempty"`
	ModTime int64  `protobuf:"varint,6,opt,name=modTime,proto3" json:"modTime,omitempty"`
	// int32 typeflag = 7;
	Linkname string            `protobuf:"bytes,7,opt,name=linkname,proto3" json:"linkname,omitempty"`
	Devmajor int64             `protobuf:"varint,8,opt,name=devmajor,proto3" json:"devmajor,omitempty"`
	Devminor int64             `protobuf:"varint,9,opt,name=devminor,proto3" json:"devminor,omitempty"`
	Xattrs   map[string][]byte "" /* 147 byte string literal not displayed */
}

func (*Stat) Descriptor 

func (*Stat) Descriptor() ([]byte, []int)

func (*Stat) Equal 

func (this *Stat) Equal(that interface{}) bool

func (*Stat) GetDevmajor 

func (m *Stat) GetDevmajor() int64

func (*Stat) GetDevminor 

func (m *Stat) GetDevminor() int64

func (*Stat) GetGid 

func (m *Stat) GetGid() uint32

func (*Stat) GetLinkname 

func (m *Stat) GetLinkname() string

func (*Stat) GetModTime 

func (m *Stat) GetModTime() int64

func (*Stat) GetMode 

func (m *Stat) GetMode() uint32

func (*Stat) GetPath 

func (m *Stat) GetPath() string

func (*Stat) GetSize_ 

func (m *Stat) GetSize_() int64

func (*Stat) GetUid 

func (m *Stat) GetUid() uint32

func (*Stat) GetXattrs 

func (m *Stat) GetXattrs() map[string][]byte

func (*Stat) GoString 

func (this *Stat) GoString() string

func (*Stat) Marshal 

func (m *Stat) Marshal() (dAtA []byte, err error)

func (*Stat) MarshalTo 

func (m *Stat) MarshalTo(dAtA []byte) (int, error)

func (*Stat) ProtoMessage 

func (*Stat) ProtoMessage()

func (*Stat) Reset 

func (m *Stat) Reset()

func (*Stat) Size 

func (m *Stat) Size() (n int)

func (*Stat) String 

func (this *Stat) String() string

func (*Stat) Unmarshal 

func (m *Stat) Unmarshal(dAtA []byte) error

type StatInfo 

type StatInfo struct {
	*Stat
}

func (*StatInfo) IsDir 

func (s *StatInfo) IsDir() bool

func (*StatInfo) ModTime 

func (s *StatInfo) ModTime() time.Time

func (*StatInfo) Mode 

func (s *StatInfo) Mode() os.FileMode

func (*StatInfo) Name 

func (s *StatInfo) Name() string

func (*StatInfo) Size 

func (s *StatInfo) Size() int64

func (*StatInfo) Sys 

func (s *StatInfo) Sys() interface{}

type Stream 

type Stream interface {
	RecvMsg(interface{}) error
	SendMsg(m interface{}) error
	Context() context.Context
}

type Validator 

type Validator struct {
	// contains filtered or unexported fields

}

func (*Validator) HandleChange 

func (v *Validator) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) (retErr error)

type WalkOpt 

type WalkOpt struct {
	IncludePaths    []string // todo: remove?
	ExcludePatterns []string
}

type WriteToFunc 

type WriteToFunc func(context.Context, string, io.WriteCloser) error

Directories

Path Synopsis
bench
cmd/receive
cmd/send
cmd/walk