warc

README

warc

WARNING: This project is no longer a work-in-progress, but needs to be carefully implemented and tested, but is generating valid WARCs when used correctly!

Introduction

warc provides methods for reading and writing WARC files in Go. This module is based on nlevitt's WARC module.

Install

go get github.com/CorentinB/warc

License

warc is released under CC0 license. You can find a copy of the CC0 License in the LICENSE file.

Documentation

Index

• Constants
• Variables
• func GenerateWarcFileName(prefix string, compression string, atomicSerial *int64) (fileName string)
• func GetSHA1(r io.Reader) string
• func GetSHA256(r io.Reader) string
• func GetSHA256Base16(r io.Reader) string
• func NewDecompressionReader(r io.Reader) (io.Reader, error)
• type CustomHTTPClient
  • func NewWARCWritingHTTPClient(HTTPClientSettings HTTPClientSettings) (httpClient *CustomHTTPClient, err error)
  • func (c *CustomHTTPClient) Close() error
  • func (c *CustomHTTPClient) WriteMetadataRecord(WARCTargetURI, contentType, payload string)
• type DedupeOptions
• type Error
• type HTTPClientSettings
• type Header
  • func NewHeader() Header
  • func (h Header) Del(key string)
  • func (h Header) Get(key string) string
  • func (h Header) Set(key, value string)
• type ReadSeekCloser
• type ReadWriteSeekCloser
  • func NewSpooledTempFile(filePrefix string, tempDir string, fullOnDisk bool) ReadWriteSeekCloser
• type Reader
  • func NewReader(reader io.ReadCloser) (*Reader, error)
  • func (r *Reader) ReadRecord() (*Record, bool, error)
• type ReaderAt
• type Record
  • func NewRecord(tempDir string, fullOnDisk bool) *Record
• type RecordBatch
  • func NewRecordBatch() *RecordBatch
• type RotatorSettings
  • func NewRotatorSettings() *RotatorSettings
  • func (s *RotatorSettings) NewWARCRotator() (recordWriterChan chan *RecordBatch, doneChannels []chan bool, err error)
• type WaitGroupWithCount
  • func (wg *WaitGroupWithCount) Add(delta int)
  • func (wg *WaitGroupWithCount) Done()
  • func (wg *WaitGroupWithCount) Size() int
• type Writer
  • func NewWriter(writer io.Writer, fileName string, compression string, ...) (*Writer, error)
  • func (w *Writer) CloseCompressedWriter()
  • func (w *Writer) WriteInfoRecord(payload map[string]string) (recordID string, err error)
  • func (w *Writer) WriteRecord(r *Record) (recordID string, err error)

Constants

const MaxInMemorySize = 1000000

MaxInMemorySize is the max number of bytes (currently 1MB) to hold in memory before starting to write to disk

Variables

var (
	IPv6 *availableIPs
	IPv4 *availableIPs
)

var (

	// Create a counter to keep track of the number of bytes written to WARC files
	// and the number of bytes deduped
	DataTotal         *ratecounter.Counter
	RemoteDedupeTotal *ratecounter.Counter
	LocalDedupeTotal  *ratecounter.Counter
)

var CDXHTTPClient = http.Client{
	Timeout: 10 * time.Second,
	Transport: &http.Transport{
		Dial: (&net.Dialer{
			Timeout: 5 * time.Second,
		}).Dial,
		TLSHandshakeTimeout: 5 * time.Second,
	},
}

Functions

func GenerateWarcFileName

func GenerateWarcFileName(prefix string, compression string, atomicSerial *int64) (fileName string)

GenerateWarcFileName generate a WARC file name following recommendations of the specs: Prefix-Timestamp-Serial-Crawlhost.warc.gz

func GetSHA1

func GetSHA1(r io.Reader) string

func GetSHA256

func GetSHA256(r io.Reader) string

func GetSHA256Base16

func GetSHA256Base16(r io.Reader) string

func NewDecompressionReader

func NewDecompressionReader(r io.Reader) (io.Reader, error)

NewDecompressionReader will return a new reader transparently doing decompression of GZip, BZip2, XZ, and ZStd.

Types

type CustomHTTPClient

type CustomHTTPClient struct {
	WARCWriter chan *RecordBatch
	WaitGroup  *WaitGroupWithCount

	ErrChan chan *Error
	http.Client
	TempDir                string
	WARCWriterDoneChannels []chan bool

	MaxReadBeforeTruncate int
	TLSHandshakeTimeout   time.Duration

	FullOnDisk bool
	// contains filtered or unexported fields
}

func NewWARCWritingHTTPClient

func NewWARCWritingHTTPClient(HTTPClientSettings HTTPClientSettings) (httpClient *CustomHTTPClient, err error)

func (*CustomHTTPClient) Close

func (c *CustomHTTPClient) Close() error

func (*CustomHTTPClient) WriteMetadataRecord

func (c *CustomHTTPClient) WriteMetadataRecord(WARCTargetURI, contentType, payload string)

type DedupeOptions

type DedupeOptions struct {
	CDXURL        string
	CDXCookie     string
	SizeThreshold int
	LocalDedupe   bool
	CDXDedupe     bool
}

type Error

type Error struct {
	Err  error
	Func string
}

type HTTPClientSettings

type HTTPClientSettings struct {
	RotatorSettings       *RotatorSettings
	Proxy                 string
	TempDir               string
	SkipHTTPStatusCodes   []int
	DedupeOptions         DedupeOptions
	DialTimeout           time.Duration
	ResponseHeaderTimeout time.Duration
	TLSHandshakeTimeout   time.Duration
	MaxReadBeforeTruncate int
	TCPTimeout            time.Duration
	DecompressBody        bool
	FollowRedirects       bool
	FullOnDisk            bool
	VerifyCerts           bool
	RandomLocalIP         bool
}

type Header

type Header map[string]string

Header provides information about the WARC record. It stores WARC record field names and their values. Since WARC field names are case-insensitive, the Header methods are case-insensitive as well.

func NewHeader

func NewHeader() Header

NewHeader creates a new WARC header.

func (Header) Del

func (h Header) Del(key string)

Del deletes the value associated with key.

func (Header) Get

func (h Header) Get(key string) string

Get returns the value associated with the given key. If there is no value associated with the key, Get returns "".

func (Header) Set

func (h Header) Set(key, value string)

Set sets the header field associated with key to value.

type ReadSeekCloser

type ReadSeekCloser interface {
	io.Reader
	io.Seeker
	ReaderAt
	io.Closer
	FileName() string
}

ReadSeekCloser is an io.Reader + ReaderAt + io.Seeker + io.Closer + Stat

type ReadWriteSeekCloser

type ReadWriteSeekCloser interface {
	ReadSeekCloser
	io.Writer
}

ReadWriteSeekCloser is an io.Writer + io.Reader + io.Seeker + io.Closer.

func NewSpooledTempFile

func NewSpooledTempFile(filePrefix string, tempDir string, fullOnDisk bool) ReadWriteSeekCloser

NewSpooledTempFile returns an ReadWriteSeekCloser, with some important constraints: you can Write into it, but whenever you call Read or Seek on it, Write is forbidden, will return an error.

type Reader

type Reader struct {
	// contains filtered or unexported fields
}

Reader store the bufio.Reader and gzip.Reader for a WARC file

func NewReader

func NewReader(reader io.ReadCloser) (*Reader, error)

NewReader returns a new WARC reader

func (*Reader) ReadRecord

func (r *Reader) ReadRecord() (*Record, bool, error)

ReadRecord reads the next record from the opened WARC file returns:

• Record: if an error occurred, record **may be** nil. if eol is true, record **must be** nil.
• bool (eol): if true, we readed all records successfully.
• error: error

type ReaderAt

type ReaderAt interface {
	ReadAt(p []byte, off int64) (n int, err error)
}

ReaderAt is the interface for ReadAt - read at position, without moving pointer.

type Record

type Record struct {
	Header  Header
	Content ReadWriteSeekCloser
	Version string // WARC/1.0, WARC/1.1 ...
}

Record represents a WARC record.

func NewRecord

func NewRecord(tempDir string, fullOnDisk bool) *Record

NewRecord creates a new WARC record.

type RecordBatch

type RecordBatch struct {
	Done        chan bool
	CaptureTime string
	Records     []*Record
}

RecordBatch is a structure that contains a bunch of records to be written at the same time, and a common capture timestamp

func NewRecordBatch

func NewRecordBatch() *RecordBatch

NewRecordBatch creates a record batch, it also initialize the capture time

type RotatorSettings

type RotatorSettings struct {
	// Content of the warcinfo record that will be written
	// to all WARC files
	WarcinfoContent Header
	// Prefix used for WARC filenames, WARC 1.1 specifications
	// recommend to name files this way:
	// Prefix-Timestamp-Serial-Crawlhost.warc.gz
	Prefix string
	// Compression algorithm to use
	Compression string
	// Path to a ZSTD compression dictionary to embed (and use) in .warc.zst files
	CompressionDictionary string
	// Directory where the created WARC files will be stored,
	// default will be the current directory
	OutputDirectory string
	// WarcSize is in Megabytes
	WarcSize float64
	// WARCWriterPoolSize defines the number of parallel WARC writers
	WARCWriterPoolSize int
}

RotatorSettings is used to store the settings needed by recordWriter to write WARC files

func NewRotatorSettings

func NewRotatorSettings() *RotatorSettings

NewRotatorSettings creates a RotatorSettings structure and initialize it with default values

func (*RotatorSettings) NewWARCRotator

func (s *RotatorSettings) NewWARCRotator() (recordWriterChan chan *RecordBatch, doneChannels []chan bool, err error)

NewWARCRotator creates and return a channel that can be used to communicate records to be written to WARC files to the recordWriter function running in a goroutine

type WaitGroupWithCount

type WaitGroupWithCount struct {
	sync.WaitGroup
	// contains filtered or unexported fields
}

func (*WaitGroupWithCount) Add

func (wg *WaitGroupWithCount) Add(delta int)

func (*WaitGroupWithCount) Done

func (wg *WaitGroupWithCount) Done()

func (*WaitGroupWithCount) Size

func (wg *WaitGroupWithCount) Size() int

type Writer

type Writer struct {
	GZIPWriter   *gzip.Writer
	ZSTDWriter   *zstd.Encoder
	FileWriter   *bufio.Writer
	FileName     string
	Compression  string
	ParallelGZIP bool
}

Writer writes WARC records to WARC files.

func NewWriter

func NewWriter(writer io.Writer, fileName string, compression string, contentLengthHeader string, newFileCreation bool, dictionary []byte) (*Writer, error)

NewWriter creates a new WARC writer.

func (*Writer) CloseCompressedWriter

func (w *Writer) CloseCompressedWriter()

func (*Writer) WriteInfoRecord

func (w *Writer) WriteInfoRecord(payload map[string]string) (recordID string, err error)

WriteInfoRecord method can be used to write informations record to the WARC file

func (*Writer) WriteRecord

func (w *Writer) WriteRecord(r *Record) (recordID string, err error)

WriteRecord writes a record to the underlying WARC file. A record consists of a version string, the record header followed by a record content block and two newlines:

Version CLRF
Header-Key: Header-Value CLRF
CLRF
Content
CLRF
CLRF