warc

README

warc

WARNING: This project is still a WIP. It is NOT ready to be used in any project.

Introduction

warc provides methods for reading and writing WARC files in Go. This module is based on nlevitt's WARC module.

Install

go get github.com/CorentinB/warc

License

warc is released under CC0 license. You can find a copy of the CC0 License in the LICENSE file.

Documentation

Index

• Constants
• Variables
• func GenerateWarcFileName(prefix string, compression string, serial int) (fileName string)
• func GetSHA1(r io.Reader) string
• type CustomHTTPClient
  • func NewWARCWritingHTTPClient(HTTPClientSettings HTTPClientSettings) (httpClient *CustomHTTPClient, errChan chan error, err error)
  • func (c *CustomHTTPClient) Close() error
• type DedupeOptions
• type HTTPClientSettings
• type Header
  • func NewHeader() Header
  • func (h Header) Del(key string)
  • func (h Header) Get(key string) string
  • func (h Header) Set(key, value string)
• type ReadSeekCloser
• type ReadWriteSeekCloser
  • func NewSpooledTempFile(filePrefix string, tempDir string, fullOnDisk bool) ReadWriteSeekCloser
• type Reader
  • func NewReader(reader io.Reader) (*Reader, error)
  • func (r *Reader) Close()
  • func (r *Reader) ReadRecord() (*Record, error)
• type ReaderAt
• type Record
  • func NewRecord(tempDir string, fullOnDisk bool) *Record
• type RecordBatch
  • func NewRecordBatch() *RecordBatch
• type RotatorSettings
  • func NewRotatorSettings() *RotatorSettings
  • func (s *RotatorSettings) NewWARCRotator() (recordWriterChan chan *RecordBatch, doneChannels []chan bool, err error)
• type WaitGroupWithCount
  • func (wg *WaitGroupWithCount) Add(delta int)
  • func (wg *WaitGroupWithCount) Done()
  • func (wg *WaitGroupWithCount) Size() int
• type Writer
  • func NewWriter(writer io.Writer, fileName string, compression string, ...) (*Writer, error)
  • func (w *Writer) CloseCompressedWriter()
  • func (w *Writer) WriteInfoRecord(payload map[string]string) (recordID string, err error)
  • func (w *Writer) WriteRecord(r *Record) (recordID string, err error)

Constants

const MaxInMemorySize = 1000000

MaxInMemorySize is the max number of bytes (currently 1MB) to hold in memory before starting to write to disk

Variables

var (

	// Create a counter to keep track of the number of bytes written to WARC files
	DataTotal *ratecounter.Counter
)

Functions

func GenerateWarcFileName

func GenerateWarcFileName(prefix string, compression string, serial int) (fileName string)

GenerateWarcFileName generate a WARC file name following recommendations of the specs: Prefix-Timestamp-Serial-Crawlhost.warc.gz

func GetSHA1

func GetSHA1(r io.Reader) string

Types

type CustomHTTPClient

type CustomHTTPClient struct {
	http.Client
	WARCWriter             chan *RecordBatch
	WARCWriterDoneChannels []chan bool
	WaitGroup              *WaitGroupWithCount

	TempDir               string
	FullOnDisk            bool
	MaxReadBeforeTruncate int
	DataTotal             *ratecounter.Counter
	// contains filtered or unexported fields
}

func NewWARCWritingHTTPClient

func NewWARCWritingHTTPClient(HTTPClientSettings HTTPClientSettings) (httpClient *CustomHTTPClient, errChan chan error, err error)

func (*CustomHTTPClient) Close

func (c *CustomHTTPClient) Close() error

type DedupeOptions

type DedupeOptions struct {
	LocalDedupe   bool
	CDXDedupe     bool
	CDXURL        string
	SizeThreshold int
}

type HTTPClientSettings

type HTTPClientSettings struct {
	RotatorSettings       *RotatorSettings
	DedupeOptions         DedupeOptions
	Proxy                 string
	DecompressBody        bool
	SkipHTTPStatusCodes   []int
	VerifyCerts           bool
	TempDir               string
	FullOnDisk            bool
	MaxReadBeforeTruncate int
}

type Header

type Header map[string]string

Header provides information about the WARC record. It stores WARC record field names and their values. Since WARC field names are case-insensitive, the Header methods are case-insensitive as well.

func NewHeader

func NewHeader() Header

NewHeader creates a new WARC header.

func (Header) Del

func (h Header) Del(key string)

Del deletes the value associated with key.

func (Header) Get

func (h Header) Get(key string) string

Get returns the value associated with the given key. If there is no value associated with the key, Get returns "".

func (Header) Set

func (h Header) Set(key, value string)

Set sets the header field associated with key to value.

type ReadSeekCloser

type ReadSeekCloser interface {
	io.Reader
	io.Seeker
	ReaderAt
	io.Closer
	FileName() string
}

ReadSeekCloser is an io.Reader + ReaderAt + io.Seeker + io.Closer + Stat

type ReadWriteSeekCloser

type ReadWriteSeekCloser interface {
	ReadSeekCloser
	io.Writer
}

ReadWriteSeekCloser is an io.Writer + io.Reader + io.Seeker + io.Closer.

func NewSpooledTempFile

func NewSpooledTempFile(filePrefix string, tempDir string, fullOnDisk bool) ReadWriteSeekCloser

NewSpooledTempFile returns an ReadWriteSeekCloser, with some important constraints: you can Write into it, but whenever you call Read or Seek on it, Write is forbidden, will return an error.

type Reader

type Reader struct {
	// contains filtered or unexported fields
}

Reader store the bufio.Reader and gzip.Reader for a WARC file

func NewReader

func NewReader(reader io.Reader) (*Reader, error)

NewReader returns a new WARC reader

func (*Reader) Close

func (r *Reader) Close()

Close closes the reader.

func (*Reader) ReadRecord

func (r *Reader) ReadRecord() (*Record, error)

ReadRecord reads the next record from the opened WARC file

type ReaderAt

type ReaderAt interface {
	ReadAt(p []byte, off int64) (n int, err error)
}

ReaderAt is the interface for ReadAt - read at position, without moving pointer.

type Record

type Record struct {
	Header  Header
	Content ReadWriteSeekCloser
}

Record represents a WARC record.

func NewRecord

func NewRecord(tempDir string, fullOnDisk bool) *Record

NewRecord creates a new WARC record.

type RecordBatch

type RecordBatch struct {
	Records     []*Record
	Done        chan bool
	CaptureTime string
}

RecordBatch is a structure that contains a bunch of records to be written at the same time, and a common capture timestamp

func NewRecordBatch

func NewRecordBatch() *RecordBatch

NewRecordBatch creates a record batch, it also initialize the capture time

type RotatorSettings

type RotatorSettings struct {
	// Content of the warcinfo record that will be written
	// to all WARC files
	WarcinfoContent Header
	// Prefix used for WARC filenames, WARC 1.1 specifications
	// recommend to name files this way:
	// Prefix-Timestamp-Serial-Crawlhost.warc.gz
	Prefix string
	// Compression algorithm to use
	Compression string
	// WarcSize is in MegaBytes
	WarcSize float64
	// Directory where the created WARC files will be stored,
	// default will be the current directory
	OutputDirectory string
	// WARCWriterPoolSize defines the number of parallel WARC writers
	WARCWriterPoolSize int
}

RotatorSettings is used to store the settings needed by recordWriter to write WARC files

func NewRotatorSettings

func NewRotatorSettings() *RotatorSettings

NewRotatorSettings creates a RotatorSettings structure and initialize it with default values

func (*RotatorSettings) NewWARCRotator

func (s *RotatorSettings) NewWARCRotator() (recordWriterChan chan *RecordBatch, doneChannels []chan bool, err error)

NewWARCRotator creates and return a channel that can be used to communicate records to be written to WARC files to the recordWriter function running in a goroutine

type WaitGroupWithCount

type WaitGroupWithCount struct {
	sync.WaitGroup
	// contains filtered or unexported fields
}

func (*WaitGroupWithCount) Add

func (wg *WaitGroupWithCount) Add(delta int)

func (*WaitGroupWithCount) Done

func (wg *WaitGroupWithCount) Done()

func (*WaitGroupWithCount) Size

func (wg *WaitGroupWithCount) Size() int

type Writer

type Writer struct {
	FileName     string
	Compression  string
	GZIPWriter   *gzip.Writer
	PGZIPWriter  *pgzip.Writer
	ZSTDWriter   *zstd.Encoder
	FileWriter   *bufio.Writer
	ParallelGZIP bool
}

Writer writes WARC records to WARC files.

func NewWriter

func NewWriter(writer io.Writer, fileName string, compression string, contentLengthHeader string) (*Writer, error)

NewWriter creates a new WARC writer.

func (*Writer) CloseCompressedWriter

func (w *Writer) CloseCompressedWriter()

func (*Writer) WriteInfoRecord

func (w *Writer) WriteInfoRecord(payload map[string]string) (recordID string, err error)

WriteInfoRecord method can be used to write informations record to the WARC file

func (*Writer) WriteRecord

func (w *Writer) WriteRecord(r *Record) (recordID string, err error)

WriteRecord writes a record to the underlying WARC file. A record consists of a version string, the record header followed by a record content block and two newlines:

Version CLRF
Header-Key: Header-Value CLRF
CLRF
Content
CLRF
CLRF