dedup

Documentation

Index

• Constants
• func NewCmdDedup() *cobra.Command
• type DedupFlags
  • func NewDedupFlags() DedupFlags
  • func (f DedupFlags) AddFlags(cmd *cobra.Command)
  • func (f DedupFlags) BufferMaxMem() int64
  • func (f DedupFlags) DedupSizeGain() int64
  • func (f DedupFlags) MinIndexDiskFree() int64
  • func (f DedupFlags) ToDedupOptions() (*DedupOptions, error)
• type DedupOptions
  • func (o *DedupOptions) Complete(cmd *cobra.Command, args []string) error
  • func (o *DedupOptions) Run() error
  • func (o *DedupOptions) Validate() error

Constants

const (
	BufferMaxMem     = "max-buffer-mem"
	BufferMaxMemHelp = "the maximum bytes of memory allowed for each buffer before overflowing to disk"

	DedupSizeGain     = "min-size-gain"
	DedupSizeGainHelp = `minimum bytes one must earn to perform a deduplication`

	MinIndexDiskFree     = "min-index-disk-free"
	MinIndexDiskFreeHelp = `minimum free space on disk to allow index writing`
)

Functions

func NewCmdDedup

func NewCmdDedup() *cobra.Command

Types

type DedupFlags

type DedupFlags struct {
	WarcFileFlags         flag.WarcIteratorFlags
	InputHookFlags        *flag.InputHookFlags
	OutputHookFlags       *flag.OutputHookFlags
	FileWalkerFlags       flag.FileWalkerFlags
	WarcWriterConfigFlags *flag.WarcWriterConfigFlags
	UtilFlags             flag.UtilFlags
	RepairFlags           flag.RepairFlags
	WarcRecordOptionFlags flag.WarcRecordOptionFlags
	IndexFlags            *flag.IndexFlags
	ConcurrencyFlags      flag.ConcurrencyFlags
	ErrorFlags            flag.ErrorFlags
}

func NewDedupFlags

func NewDedupFlags() DedupFlags

func (DedupFlags) AddFlags

func (f DedupFlags) AddFlags(cmd *cobra.Command)

func (DedupFlags) BufferMaxMem

func (f DedupFlags) BufferMaxMem() int64

func (DedupFlags) DedupSizeGain

func (f DedupFlags) DedupSizeGain() int64

func (DedupFlags) MinIndexDiskFree

func (f DedupFlags) MinIndexDiskFree() int64

func (DedupFlags) ToDedupOptions

func (f DedupFlags) ToDedupOptions() (*DedupOptions, error)

type DedupOptions

type DedupOptions struct {
	Paths               []string
	Concurrency         int
	DigestIndex         *index.DigestIndex
	FileIndex           *index.FileIndex
	WarcWriterConfig    *warcwriterconfig.WarcWriterConfig
	MinimumSizeGain     int64
	MinWARCDiskFree     int64
	MinIndexDiskFree    int64
	ContinueOnError     bool
	FileWalker          *filewalker.FileWalker
	WarcRecordOptions   []gowarc.WarcRecordOption
	OpenInputFileHook   hooks.OpenInputFileHook
	CloseInputFileHook  hooks.CloseInputFileHook
	OpenOutputFileHook  hooks.OpenOutputFileHook
	CloseOutputFileHook hooks.CloseOutputFileHook
}

func (*DedupOptions) Complete

func (o *DedupOptions) Complete(cmd *cobra.Command, args []string) error

func (*DedupOptions) Run

func (o *DedupOptions) Run() error

func (*DedupOptions) Validate

func (o *DedupOptions) Validate() error