rsync

Documentation

Overview

Algorithm found at: https://rsync.samba.org/tech_report/tech_report.html Code in this file is inspired by: https://github.com/jbreiding/rsync-go

Definitions

Source: The final content.
Target: The content to be made into final content.
Signature: The sequence of hashes used to identify the content.

First create a patcher with: p = NewPatcher() Create a signature for the file you want to update using p.CreateSignatureIterator(file_to_update) Now create a Differ with the created signature d = NewDiffer() d.AddSignatureData(signature_data_from_previous_step) Now create a delta based on the signature and the reference file d.CreateDelta(reference_file) Finally, apply this delta using the patcher to produce a file identical to reference_file based ont he delta data and file_to_update p.StartDelta(output_file, file_to_update) p.UpdateDelta(...) p.FinishDelta()

Index

• Constants
• type Api
• type BlockHash
  • func (self BlockHash) Serialize(output []byte)
  • func (self *BlockHash) Unserialize(data []byte) (err error)
• type ChecksumType
• type Differ
  • func NewDiffer() *Differ
  • func (self *Differ) AddSignatureData(data []byte) (err error)
  • func (self *Differ) BlockSize() int
  • func (self *Differ) CreateDelta(src io.Reader, output io.Writer) func() error
  • func (self *Differ) FinishSignatureData() (err error)
• type GrowBufferFunction
• type OpType
• type Operation
  • func (self Operation) Serialize(ans []byte)
  • func (self Operation) SerializeSize() int
  • func (self Operation) String() string
  • func (self *Operation) Unserialize(data []byte) (n int, err error)
• type OperationWriter
  • func (self *OperationWriter) Write(p []byte) (n int, err error)
• type Patcher
  • func NewPatcher(expected_input_size int64) (ans *Patcher)
  • func (self *Patcher) CreateSignatureIterator(src io.Reader, output io.Writer) func() error
  • func (self *Patcher) FinishDelta() (err error)
  • func (self *Patcher) StartDelta(delta_output io.Writer, delta_input io.ReadSeeker)
  • func (self *Patcher) UpdateDelta(data []byte) (err error)
• type StrongHashType
• type WeakHashType

Constants

const BlockHashSize = 20

const DataSizeMultiple int = 8

const DefaultBlockSize = 1024 * 6

If no BlockSize is specified in the rsync instance, this value is used.

const MaxBlockSize int = 1024 * 1024 // sqrt of 1TB

Types

type Api

type Api struct {
	Checksum_type    ChecksumType
	Strong_hash_type StrongHashType
	Weak_hash_type   WeakHashType
	// contains filtered or unexported fields
}

type BlockHash

type BlockHash struct {
	Index      uint64
	WeakHash   uint32
	StrongHash uint64
}

Signature hash item generated from target.

func (BlockHash) Serialize

func (self BlockHash) Serialize(output []byte)

Put the serialization of this BlockHash to output

func (*BlockHash) Unserialize

func (self *BlockHash) Unserialize(data []byte) (err error)

type ChecksumType

type ChecksumType uint16

const (
	XXH3128Sum ChecksumType = iota
)

type Differ

type Differ struct {
	Api
	// contains filtered or unexported fields
}

func NewDiffer

func NewDiffer() *Differ

Use to calculate a delta based on a supplied signature, via AddSignatureData

func (*Differ) AddSignatureData

func (self *Differ) AddSignatureData(data []byte) (err error)

Add more external signature data

func (*Differ) BlockSize

func (self *Differ) BlockSize() int

func (*Differ) CreateDelta

func (self *Differ) CreateDelta(src io.Reader, output io.Writer) func() error

Create a serialized delta based on the previously loaded signature

func (*Differ) FinishSignatureData

func (self *Differ) FinishSignatureData() (err error)

type GrowBufferFunction

type GrowBufferFunction = func(slice []byte, sz int) []byte

type OpType

type OpType byte // enum

Operation Types.

const (
	OpBlock OpType = iota
	OpData
	OpHash
	OpBlockRange
)

type Operation

type Operation struct {
	Type          OpType
	BlockIndex    uint64
	BlockIndexEnd uint64
	Data          []byte
}

Instruction to mutate target to align to source.

func (Operation) Serialize

func (self Operation) Serialize(ans []byte)

func (Operation) SerializeSize

func (self Operation) SerializeSize() int

func (Operation) String

func (self Operation) String() string

func (*Operation) Unserialize

func (self *Operation) Unserialize(data []byte) (n int, err error)

type OperationWriter

type OperationWriter struct {
	Operations []Operation
	// contains filtered or unexported fields
}

func (*OperationWriter) Write

func (self *OperationWriter) Write(p []byte) (n int, err error)

type Patcher

type Patcher struct {
	Api
	// contains filtered or unexported fields
}

func NewPatcher

func NewPatcher(expected_input_size int64) (ans *Patcher)

Use to create a signature and possibly apply a delta

func (*Patcher) CreateSignatureIterator

func (self *Patcher) CreateSignatureIterator(src io.Reader, output io.Writer) func() error

Create a signature for the data source in src.

func (*Patcher) FinishDelta

func (self *Patcher) FinishDelta() (err error)

Finish applying delta data

func (*Patcher) StartDelta

func (self *Patcher) StartDelta(delta_output io.Writer, delta_input io.ReadSeeker)

Start applying serialized delta

func (*Patcher) UpdateDelta

func (self *Patcher) UpdateDelta(data []byte) (err error)

Apply a chunk of delta data

type StrongHashType

type StrongHashType uint16

const (
	XXH3 StrongHashType = iota
)

type WeakHashType

type WeakHashType uint16

const (
	Rsync WeakHashType = iota
)