flag

package
v3.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 27, 2025 License: Apache-2.0 Imports: 22 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	Concurrency     = "concurrency"
	ConcurrencyHelp = `number of input files to process simultaneously.`
)
View Source 
const (
	ContinueOnError     = "continue-on-error"
	ContinueOnErrorHelp = `continue on error. Will continue processing files and directories in spite of errors.`
)
View Source 
const (
	Recursive     = "recursive"
	RecursiveHelp = "walk directories recursively"

	FollowSymlinks     = "symlinks"
	FollowSymlinksHelp = `follow symlinks`

	Suffixes     = "suffixes"
	SuffixesHelp = `filter files by suffix`
)
View Source 
const (
	RecordId     = "id"
	RecordIdHelp = `filter record ID's. For more than one, repeat flag or use comma separated list.`

	RecordType     = "record-type"
	RecordTypeHelp = `` /* 190-byte string literal not displayed */

	ResponseCode     = "response-code"
	ResponseCodeHelp = `` /* 247-byte string literal not displayed */

	MimeType     = "mime-type"
	MimeTypeHelp = `filter records with given mime-types. For more than one, repeat flag or use a comma separated list.`
)
View Source 
const (
	LogFileNameFlag = "log-file"
	LogFileNameHelp = `log to file`

	LogFormatFlag = "log-format"
	LogFormatHelp = `log format. Valid values: text, json`

	LogLevelFlag = "log-level"
	LogLevelHelp = `log level. Valid values: debug, info, warn, error`

	ConfigFlag = "config"
	ConfigHelp = `` /* 151-byte string literal not displayed */
)
View Source 
const (
	OpenInputFileHook     = "open-input-file-hook"
	OpenInputFileHookHelp = `` /* 309-byte string literal not displayed */

	CloseInputFileHook     = "close-input-file-hook"
	CloseInputFileHookHelp = `` /* 414-byte string literal not displayed */

	OpenOutputFileHook     = "open-output-file-hook"
	OpenOutputFileHookHelp = `` /* 423-byte string literal not displayed */

	CloseOutputFileHook     = "close-output-file-hook"
	CloseOutputFileHookHelp = `` /* 714-byte string literal not displayed */

)
View Source 
const (
	NewIndex     = "new-index"
	NewIndexHelp = `true to start from a fresh index, deleting eventual index from last run`

	KeepIndex     = "keep-index"
	KeepIndexHelp = `true to keep index on disk so that the next run will continue where the previous run left off`

	IndexDir     = "index-dir"
	IndexDirHelp = `directory to store indexes`
)
View Source 
const (
	Offset     = "offset"
	OffsetHelp = `start processing from this byte offset in file. Defaults to 0.`

	RecordNum     = "nth"
	RecordNumHelp = `only process the n'th record. Only records that are not filtered out by other options are counted.`

	Limit     = "limit"
	LimitHelp = `limit the number of records to process. If the -n option is specified the limit is ignored.`

	Force     = "force"
	ForceHelp = `force the record iterator to continue regardless of errors.`
)
View Source 
const (
	Repair     = "repair"
	RepairHelp = `try to fix errors in records`
)
View Source 
const (
	SrcFileSystem     = "input-file"
	SrcFileSystemHelp = `` /* 150-byte string literal not displayed */

	SrcFileList     = "source-file-list"
	SrcFileListHelp = `a file containing a list of files to process, one file per line`

	FtpPoolSize     = "ftp-pool-size"
	FtpPoolSizeHelp = `size of the ftp pool`
)
View Source 
const (
	MinDiskFree     = "min-disk-free"
	MinDiskFreeHelp = `minimum free space on disk to allow WARC writing`
)
View Source 
const (
	TmpDir     = "tmp-dir"
	TmpDirHelp = `directory to use for temporary files`

	StrictValidation     = "strict"
	StrictValidationHelp = `sets the parser to fail on first validation error.`

	LenientValidation      = "lenient"
	LentientValidationHelp = `sets the parser to do as little validation as possible.`
)
View Source 
const (
	ConcurrentWriters     = "concurrent-writers"
	ConcurrentWritersHelp = `` /* 195-byte string literal not displayed */

	FileSize     = "file-size"
	FileSizeHelp = `The maximum size for WARC files`

	Compress     = "compress"
	CompressHelp = `use gzip compression for WARC files`

	CompressionLevel     = "compression-level"
	CompressionLevelHelp = `the gzip compression level to use (value between 1 and 9, -1 means the gzip library default level)`

	FilePrefix     = "prefix"
	FilePrefixHelp = `filename prefix for WARC files`

	SubdirPattern     = "subdir-pattern"
	SubdirPatternHelp = `` /* 497-byte string literal not displayed */

	NameGenerator     = "name-generator"
	NameGeneratorHelp = `` /* 218-byte string literal not displayed */

	Flush     = "flush"
	FlushHelp = `if true, sync WARC file to disk after writing each record`

	WarcVersion     = "warc-version"
	WarcVersionHelp = `the WARC version to use for created files`

	DefaultDate     = "default-date"
	DefaultDateHelp = `fetch date to use for records missing date metadata. Fetchtime is set to 12:00 UTC for the date`

	OutputDir     = "output-dir"
	OutputDirHelp = `output directory for generated warc files. Directory must exist.`

	OneToOne     = "one-to-one"
	OneToOneHelp = `` /* 127-byte string literal not displayed */

)

Variables

This section is empty.

Functions

func AddPersistentFlags 

func AddPersistentFlags(cmd *cobra.Command)

func Config 

func Config() string

func LogFileName 

func LogFileName() string

func LogFormat 

func LogFormat() string

func LogLevel 

func LogLevel() string

func ReadSrcFileList 

func ReadSrcFileList(name string) ([]string, error)

func SuffixCompletionFn 

func SuffixCompletionFn(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective)

SuffixCompletionFn can be used by commands that want to restrict file completion to suffixes set by flag

func WithDefaultFilePrefix 

func WithDefaultFilePrefix(prefix string) func(*WarcWriterConfigFlags)

func WithDefaultMimeType 

func WithDefaultMimeType(mimeTypes []string) func(*FilterFlags)

func WithDefaultOneToOne 

func WithDefaultOneToOne(oneToOne bool) func(*WarcWriterConfigFlags)

func WithDefaultSuffixes 

func WithDefaultSuffixes(suffixes []string) func(*FileWalkerFlags)

Types

type ConcurrencyFlags 

type ConcurrencyFlags struct{}

func (ConcurrencyFlags) AddFlags 

func (f ConcurrencyFlags) AddFlags(cmd *cobra.Command)

func (ConcurrencyFlags) Concurrency 

func (f ConcurrencyFlags) Concurrency() int

type ErrorFlags 

type ErrorFlags struct {
}

func (ErrorFlags) AddFlags 

func (f ErrorFlags) AddFlags(cmd *cobra.Command)

func (ErrorFlags) ContinueOnError 

func (f ErrorFlags) ContinueOnError() bool

type FileWalkerFlags 

type FileWalkerFlags struct {
	SrcFileListFlags SrcFileListFlags
	// contains filtered or unexported fields
}

func (FileWalkerFlags) AddFlags 

func (f FileWalkerFlags) AddFlags(cmd *cobra.Command, options ...func(*FileWalkerFlags))

func (FileWalkerFlags) ToFileWalker 

func (f FileWalkerFlags) ToFileWalker() (*filewalker.FileWalker, error)

type FilterFlags 

type FilterFlags struct {
	// contains filtered or unexported fields
}

func (FilterFlags) AddFlags 

func (f FilterFlags) AddFlags(cmd *cobra.Command, opts ...func(*FilterFlags))

func (FilterFlags) MimeType 

func (f FilterFlags) MimeType() []string

func (FilterFlags) RecordId 

func (f FilterFlags) RecordId() []string

func (FilterFlags) RecordType 

func (f FilterFlags) RecordType() []string

func (FilterFlags) ResponseCode 

func (f FilterFlags) ResponseCode() string

func (FilterFlags) ToFilter 

func (f FilterFlags) ToFilter() (*filter.RecordFilter, error)

type IndexFlags 

type IndexFlags struct {
	// contains filtered or unexported fields
}

func (*IndexFlags) AddFlags 

func (f *IndexFlags) AddFlags(cmd *cobra.Command, options ...func(*IndexFlags))

func (*IndexFlags) IndexDir 

func (f *IndexFlags) IndexDir() string

func (*IndexFlags) KeepIndex 

func (f *IndexFlags) KeepIndex() bool

func (*IndexFlags) NewIndex 

func (f *IndexFlags) NewIndex() bool

func (*IndexFlags) ToDigestIndex 

func (f *IndexFlags) ToDigestIndex() (*index.DigestIndex, error)

func (*IndexFlags) ToFileIndex 

func (f *IndexFlags) ToFileIndex() (*index.FileIndex, error)

type InputHookFlags 

type InputHookFlags struct {
	// contains filtered or unexported fields
}

func (*InputHookFlags) AddFlags 

func (f *InputHookFlags) AddFlags(cmd *cobra.Command, opts ...func(*InputHookFlags))

func (*InputHookFlags) CloseInputFileHook 

func (f *InputHookFlags) CloseInputFileHook() string

func (*InputHookFlags) OpenInputFileHook 

func (f *InputHookFlags) OpenInputFileHook() string

func (*InputHookFlags) ToCloseInputFileHook 

func (f *InputHookFlags) ToCloseInputFileHook() (hooks.CloseInputFileHook, error)

func (*InputHookFlags) ToOpenInputFileHook 

func (f *InputHookFlags) ToOpenInputFileHook() (hooks.OpenInputFileHook, error)

type OutputHookFlags 

type OutputHookFlags struct {
	// contains filtered or unexported fields
}

func (*OutputHookFlags) AddFlags 

func (f *OutputHookFlags) AddFlags(cmd *cobra.Command, opts ...func(*OutputHookFlags))

func (*OutputHookFlags) CloseOutputFileHook 

func (f *OutputHookFlags) CloseOutputFileHook() string

func (*OutputHookFlags) OpenOutputFileHook 

func (f *OutputHookFlags) OpenOutputFileHook() string

func (*OutputHookFlags) ToCloseOutputFileHook 

func (f *OutputHookFlags) ToCloseOutputFileHook() (hooks.CloseOutputFileHook, error)

func (*OutputHookFlags) ToOpenOutputFileHook 

func (f *OutputHookFlags) ToOpenOutputFileHook() (hooks.OpenOutputFileHook, error)

type PersistentFlags 

type PersistentFlags struct {
}

type RepairFlags 

type RepairFlags struct {
}

func (RepairFlags) AddFlags 

func (u RepairFlags) AddFlags(cmd *cobra.Command)

func (RepairFlags) Repair 

func (u RepairFlags) Repair() bool

type SliceCompletion 

type SliceCompletion []string

func (SliceCompletion) CompletionFn 

func (validCompletions SliceCompletion) CompletionFn(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective)

type SrcFileListFlags 

type SrcFileListFlags struct {
}

func NewSrcFileListeFlags 

func NewSrcFileListeFlags() SrcFileListFlags

func (SrcFileListFlags) AddFlags 

func (f SrcFileListFlags) AddFlags(cmd *cobra.Command)

func (SrcFileListFlags) FtpPoolSize 

func (f SrcFileListFlags) FtpPoolSize() int32

func (SrcFileListFlags) SrcFileList 

func (f SrcFileListFlags) SrcFileList() string

func (SrcFileListFlags) SrcFilesystem 

func (f SrcFileListFlags) SrcFilesystem() string

func (SrcFileListFlags) ToFs 

func (f SrcFileListFlags) ToFs() (afero.Fs, error)

type UtilFlags 

type UtilFlags struct {
}

func (UtilFlags) AddFlags 

func (u UtilFlags) AddFlags(cmd *cobra.Command)

func (UtilFlags) MinFreeDisk 

func (u UtilFlags) MinFreeDisk() int64

type WarcIteratorFlags 

type WarcIteratorFlags struct {
}

func NewWarcIteratorFlags 

func NewWarcIteratorFlags() WarcIteratorFlags

func (WarcIteratorFlags) AddFlags 

func (f WarcIteratorFlags) AddFlags(cmd *cobra.Command)

func (WarcIteratorFlags) Force 

func (f WarcIteratorFlags) Force() bool

func (WarcIteratorFlags) Limit 

func (f WarcIteratorFlags) Limit() int

func (WarcIteratorFlags) Offset 

func (f WarcIteratorFlags) Offset() int64

func (WarcIteratorFlags) RecordNum 

func (f WarcIteratorFlags) RecordNum() int

type WarcRecordOptionFlags 

type WarcRecordOptionFlags struct {
}

func (WarcRecordOptionFlags) AddFlags 

func (f WarcRecordOptionFlags) AddFlags(cmd *cobra.Command)

func (WarcRecordOptionFlags) LenientValidation 

func (f WarcRecordOptionFlags) LenientValidation() bool

func (WarcRecordOptionFlags) StrictValidation 

func (f WarcRecordOptionFlags) StrictValidation() bool

func (WarcRecordOptionFlags) TmpDir 

func (f WarcRecordOptionFlags) TmpDir() string

func (WarcRecordOptionFlags) ToWarcRecordOptions 

func (f WarcRecordOptionFlags) ToWarcRecordOptions() []gowarc.WarcRecordOption

type WarcWriterConfigFlags 

type WarcWriterConfigFlags struct {
	// contains filtered or unexported fields
}

func (*WarcWriterConfigFlags) AddFlags 

func (f *WarcWriterConfigFlags) AddFlags(cmd *cobra.Command, options ...func(*WarcWriterConfigFlags))

func (*WarcWriterConfigFlags) Compress 

func (f *WarcWriterConfigFlags) Compress() bool

func (*WarcWriterConfigFlags) CompressionLevel 

func (f *WarcWriterConfigFlags) CompressionLevel() int

func (*WarcWriterConfigFlags) ConcurrentWriters 

func (f *WarcWriterConfigFlags) ConcurrentWriters() int

func (*WarcWriterConfigFlags) DefaultDate 

func (f *WarcWriterConfigFlags) DefaultDate() string

func (*WarcWriterConfigFlags) FilePrefix 

func (f *WarcWriterConfigFlags) FilePrefix() string

func (*WarcWriterConfigFlags) FileSize 

func (f *WarcWriterConfigFlags) FileSize() string

func (*WarcWriterConfigFlags) Flush 

func (f *WarcWriterConfigFlags) Flush() bool

func (*WarcWriterConfigFlags) NameGenerator 

func (f *WarcWriterConfigFlags) NameGenerator() string

func (*WarcWriterConfigFlags) OneToOne 

func (f *WarcWriterConfigFlags) OneToOne() bool

func (*WarcWriterConfigFlags) OutputDir 

func (f *WarcWriterConfigFlags) OutputDir() string

func (*WarcWriterConfigFlags) SubdirPattern 

func (f *WarcWriterConfigFlags) SubdirPattern() string

func (*WarcWriterConfigFlags) ToWarcWriterConfig 

func (f *WarcWriterConfigFlags) ToWarcWriterConfig() (*warcwriterconfig.WarcWriterConfig, error)

func (*WarcWriterConfigFlags) WarcVersion 

func (f *WarcWriterConfigFlags) WarcVersion() string

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.