warc

package module

Go to main page

Versions in this module

v0

v0.8.74

Mar 12, 2025

v0.8.73

Mar 3, 2025

v0.8.72

Feb 28, 2025

Changes in this version

type RecordBatch

+ FeedbackChan chan struct{}

v0.8.71

Feb 21, 2025

Changes in this version

type HTTPClientSettings

+ DNSCacheSize int

v0.8.70

Feb 18, 2025

v0.8.69

Feb 16, 2025

v0.8.68

Jan 31, 2025

v0.8.67

Jan 30, 2025

v0.8.66

Jan 30, 2025

v0.8.65

Jan 27, 2025

v0.8.64

Jan 24, 2025

v0.8.63

Jan 21, 2025

v0.8.62

Jan 17, 2025

Changes in this version

type CustomHTTPClient

+ MaxRAMUsageFraction float64

type HTTPClientSettings

+ MaxRAMUsageFraction float64

v0.8.61

Jan 17, 2025

v0.8.60

Jan 13, 2025

Changes in this version

type ReadSeekCloser

+ Len func() int

v0.8.59

Jan 13, 2025

v0.8.58

Jan 10, 2025

v0.8.57

Dec 6, 2024

v0.8.56

Dec 6, 2024

Changes in this version

+ func GetNextIP(availableIPs *availableIPs) net.IP

v0.8.55

Dec 6, 2024

v0.8.54

Nov 22, 2024

v0.8.53

Oct 4, 2024

v0.8.52

Sep 28, 2024

v0.8.51

Sep 27, 2024

v0.8.50

Sep 26, 2024

Changes in this version

type CustomHTTPClient

+ func (c *CustomHTTPClient) WriteRecord(WARCTargetURI, WARCType, contentType, payload string)

type HTTPClientSettings

+ DNSRecordsTTL time.Duration

+ DNSResolutionTimeout time.Duration

+ DNSServer string

+ DNSServers []string

+ IPv6AnyIP bool

v0.8.49

Sep 24, 2024

Changes in this version

type HTTPClientSettings

+ DisableIPv4 bool

+ DisableIPv6 bool

v0.8.48

Sep 13, 2024

Changes in this version

type CustomHTTPClient

+ TLSHandshakeTimeout time.Duration

type RotatorSettings

+ CompressionDictionary string

v0.8.47

Sep 10, 2024

v0.8.46

Sep 10, 2024

Changes in this version

type HTTPClientSettings

+ DialTimeout time.Duration

+ ResponseHeaderTimeout time.Duration

v0.8.45

Sep 9, 2024

v0.8.44

Sep 5, 2024

v0.8.43

Aug 22, 2024

v0.8.42

Aug 14, 2024

v0.8.41

Aug 12, 2024

Changes in this version

+ func NewDecompressionReader(r io.Reader) (io.Reader, error)

type Record

+ Version string

v0.8.40

Jul 11, 2024

Changes in this version

+ var LocalDedupeTotal *ratecounter.Counter

+ var RemoteDedupeTotal *ratecounter.Counter

v0.8.39

May 8, 2024

Changes in this version

+ var CDXHTTPClient = http.Client

v0.8.38

Apr 25, 2024

Changes in this version

+ var IPv4 *availableIPs

+ var IPv6 *availableIPs

type HTTPClientSettings

+ RandomLocalIP bool

v0.8.37

Apr 5, 2024

Changes in this version

+ func GetSHA256(r io.Reader) string

+ func GetSHA256Base16(r io.Reader) string

v0.8.36

Feb 21, 2024

Changes in this version

type CustomHTTPClient

+ func (c *CustomHTTPClient) WriteMetadataRecord(WARCTargetURI, contentType, payload string)

v0.8.35

Feb 7, 2024

Changes in this version

type DedupeOptions

+ CDXCookie string

v0.8.34

Sep 14, 2023

v0.8.33

Aug 23, 2023

v0.8.32

Jun 23, 2023

v0.8.31

Jun 16, 2023

Changes in this version

type HTTPClientSettings

+ TCPTimeout time.Duration

+ TLSHandshakeTimeout time.Duration

v0.8.30

Jun 8, 2023

Changes in this version

type HTTPClientSettings

+ FollowRedirects bool

v0.8.29

May 17, 2023

Changes in this version

type CustomHTTPClient

+ ErrChan chan *Error

+ type Error struct

+ Err error

+ Func string

v0.8.28

Apr 4, 2023

v0.8.27

Apr 4, 2023

Changes in this version

type DedupeOptions

+ SizeThreshold int

v0.8.26

Mar 1, 2023

Changes in this version

+ func GenerateWarcFileName(prefix string, compression string, serial int) (fileName string)

v0.8.25

Nov 23, 2022

v0.8.24

Oct 26, 2022

Changes in this version

+ var DataTotal *ratecounter.Counter

type CustomHTTPClient

+ DataTotal *ratecounter.Counter

+ MaxReadBeforeTruncate int

type HTTPClientSettings

+ MaxReadBeforeTruncate int

v0.8.23

Sep 8, 2022

v0.8.22

Jul 25, 2022

v0.8.21

Jul 13, 2022

v0.8.20

Jul 4, 2022

Changes in this version

type CustomHTTPClient

+ WARCWriterDoneChannels []chan bool

type RotatorSettings

+ WARCWriterPoolSize int

type Writer

+ PGZIPWriter *pgzip.Writer

+ ParallelGZIP bool

+ func (w *Writer) CloseCompressedWriter()

v0.8.19

Jul 3, 2022

v0.8.18

Jul 3, 2022

Changes in this version

+ type WaitGroupWithCount struct

+ func (wg *WaitGroupWithCount) Add(delta int)

+ func (wg *WaitGroupWithCount) Done()

+ func (wg *WaitGroupWithCount) Size() int

v0.8.17

Jul 1, 2022

v0.8.16

Jul 1, 2022

v0.8.15

Jul 1, 2022

v0.8.14

Jul 1, 2022

Changes in this version

type CustomHTTPClient

+ FullOnDisk bool

+ TempDir string

+ type HTTPClientSettings struct

+ DecompressBody bool

+ DedupeOptions DedupeOptions

+ FullOnDisk bool

+ Proxy string

+ RotatorSettings *RotatorSettings

+ SkipHTTPStatusCodes []int

+ TempDir string

+ VerifyCerts bool

v0.8.13

Jun 30, 2022

v0.8.12

Jun 30, 2022

v0.8.11

Jun 30, 2022

v0.8.10

Jun 29, 2022

v0.8.9

Jun 28, 2022

Changes in this version

+ var MaxInMemorySize = 2097152

+ type ReadSeekCloser interface

+ FileName func() string

+ type ReadWriteSeekCloser interface

+ func NewSpooledTempFile(filePrefix string) ReadWriteSeekCloser

+ type ReaderAt interface

+ ReadAt func(p []byte, off int64) (n int, err error)

v0.8.8

Jun 27, 2022

v0.8.7

Jun 21, 2022

v0.8.6

Jun 20, 2022

v0.8.5

Jun 19, 2022

v0.8.4

May 19, 2022

Changes in this version

type CustomHTTPClient

+ WARCTempDir string

v0.8.3

May 4, 2022

v0.8.2

Apr 21, 2022

v0.8.1

Apr 7, 2022

v0.8.0

Apr 7, 2022

Changes in this version

+ type DedupeOptions struct

+ CDXDedupe bool

+ CDXURL string

+ LocalDedupe bool

v0.7.0

Mar 26, 2022

Changes in this version

+ func GetSHA1FromReader(r io.Reader) string

+ type CustomHTTPClient struct

+ WARCWriter chan *RecordBatch

+ WARCWriterFinish chan bool

+ WaitGroup *sync.WaitGroup

+ func NewWARCWritingHTTPClient(rotatorSettings *RotatorSettings, proxy string, decompressBody bool) (httpClient *CustomHTTPClient, err error)

+ func (c *CustomHTTPClient) Close() error

v0.6.0

Mar 19, 2022

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL