rac

package
Version: v0.2.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Dec 19, 2019 License: Apache-2.0 Imports: 6 Imported by: 0

Documentation

Overview

Package rac provides access to RAC (Random Access Compression) files.

RAC is just a container format, and this package ("rac") is a relatively low-level. Users will typically want to process a particular compression format wrapped in RAC, such as (RAC + Zlib). For that, look at e.g. the sibling "raczlib" package.

The RAC specification is at https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md

Example (IndexLocationAtEnd)

Example_indexLocationAtEnd demonstrates using the low level "rac" package to encode a RAC+Zlib formatted file with IndexLocationAtEnd.

The sibling "raczlib" package provides a higher level API that is easier to use.

See the RAC specification for an explanation of the file format.

package main

import (
	"bytes"
	"compress/zlib"
	"encoding/hex"
	"fmt"
	"hash/adler32"
	"io"
	"log"

	"github.com/google/wuffs/lib/rac"
)

func main() {
	// Manually construct a zlib encoding of "More!\n", one that uses a literal
	// block (that's easy to see in a hex dump) instead of a more compressible
	// Huffman block.
	const src = "More!\n"
	hasher := adler32.New()
	hasher.Write([]byte(src))
	enc := []byte{ // See RFCs 1950 and 1951 for details.
		0x78,       // Deflate compression method; 32KiB window size.
		0x9C,       // Default encoding algorithm; FCHECK bits.
		0x01,       // Literal block (final).
		0x06, 0x00, // Literal length.
		0xF9, 0xFF, // Inverse of the literal length.
	}
	enc = append(enc, src...) // Literal bytes.
	enc = hasher.Sum(enc)     // Adler-32 hash.

	// Check that we've constructed a valid zlib-formatted encoding, by
	// checking that decoding enc produces src.
	{
		b := &bytes.Buffer{}
		r, err := zlib.NewReader(bytes.NewReader(enc))
		if err != nil {
			log.Fatalf("NewReader: %v", err)
		}
		if _, err := io.Copy(b, r); err != nil {
			log.Fatalf("Copy: %v", err)
		}
		if got := b.String(); got != src {
			log.Fatalf("zlib check: got %q, want %q", got, src)
		}
	}

	buf := &bytes.Buffer{}
	w := &rac.ChunkWriter{
		Writer: buf,
	}
	if err := w.AddChunk(uint64(len(src)), rac.CodecZlib, enc, 0, 0); err != nil {
		log.Fatalf("AddChunk: %v", err)
	}
	if err := w.Close(); err != nil {
		log.Fatalf("Close: %v", err)
	}

	fmt.Printf("RAC file:\n%s", hex.Dump(buf.Bytes()))

}
Output:

RAC file:
00000000  72 c3 63 00 78 9c 01 06  00 f9 ff 4d 6f 72 65 21  |r.c.x......More!|
00000010  0a 07 42 01 bf 72 c3 63  01 65 a9 00 ff 06 00 00  |..B..r.c.e......|
00000020  00 00 00 00 01 04 00 00  00 00 00 01 ff 35 00 00  |.............5..|
00000030  00 00 00 01 01                                    |.....|
Example (IndexLocationAtStart)

Example_indexLocationAtStart demonstrates using the low level "rac" package to encode and then decode a RAC+Zlib formatted file with IndexLocationAtStart.

The sibling "raczlib" package provides a higher level API that is easier to use.

See the RAC specification for an explanation of the file format.

package main

import (
	"bytes"
	"compress/zlib"
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"hash/crc32"
	"io"
	"log"
	"os"

	"github.com/google/wuffs/lib/rac"
)

func main() {
	buf := &bytes.Buffer{}
	w := &rac.ChunkWriter{
		Writer:        buf,
		IndexLocation: rac.IndexLocationAtStart,
		TempFile:      &bytes.Buffer{},
	}

	dict := []byte(" sheep.\n")
	if len(dict) >= (1 << 30) {
		log.Fatal("len(dict) is too large")
	}
	encodedDict := []byte{
		uint8(len(dict) >> 0),
		uint8(len(dict) >> 8),
		uint8(len(dict) >> 16),
		uint8(len(dict) >> 24),
	}
	encodedDict = append(encodedDict, dict...)
	checksum := crc32.ChecksumIEEE(dict)
	encodedDict = append(encodedDict,
		uint8(checksum>>0),
		uint8(checksum>>8),
		uint8(checksum>>16),
		uint8(checksum>>24),
	)
	fmt.Printf("Encoded dictionary resource:\n%s\n", hex.Dump(encodedDict))

	dictResource, err := w.AddResource(encodedDict)
	if err != nil {
		log.Fatalf("AddResource: %v", err)
	}

	chunks := []string{
		"One sheep.\n",
		"Two sheep.\n",
		"Three sheep.\n",
	}

	for i, chunk := range chunks {
		b := &bytes.Buffer{}
		if z, err := zlib.NewWriterLevelDict(b, zlib.BestCompression, dict); err != nil {
			log.Fatalf("NewWriterLevelDict: %v", err)
		} else if _, err := z.Write([]byte(chunk)); err != nil {
			log.Fatalf("Write: %v", err)
		} else if err := z.Close(); err != nil {
			log.Fatalf("Close: %v", err)
		}
		encodedChunk := b.Bytes()

		if err := w.AddChunk(uint64(len(chunk)), rac.CodecZlib, encodedChunk, dictResource, 0); err != nil {
			log.Fatalf("AddChunk: %v", err)
		}

		fmt.Printf("Encoded chunk #%d:\n%s\n", i, hex.Dump(encodedChunk))
	}

	if err := w.Close(); err != nil {
		log.Fatalf("Close: %v", err)
	}

	encoded := buf.Bytes()
	fmt.Printf("RAC file:\n%s\n", hex.Dump(encoded))

	// Decode the encoded bytes (the RAC-formatted bytes) to recover the
	// original "One sheep.\nTwo sheep\.Three sheep.\n" source.

	fmt.Printf("Decoded:\n")
	r := &rac.ChunkReader{
		ReadSeeker:     bytes.NewReader(encoded),
		CompressedSize: int64(len(encoded)),
	}
	zr := io.ReadCloser(nil)
	for {
		chunk, err := r.NextChunk()
		if err == io.EOF {
			break
		} else if err != nil {
			log.Fatalf("NextChunk: %v", err)
		}
		if chunk.Codec != rac.CodecZlib {
			log.Fatalf("unexpected chunk codec")
		}
		fmt.Printf("[%2d, %2d): ", chunk.DRange[0], chunk.DRange[1])

		// Parse the RAC+Zlib secondary data. For details, see
		// https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md#rac--zlib
		dict := []byte(nil)
		if secondary := encoded[chunk.CSecondary[0]:chunk.CSecondary[1]]; len(secondary) > 0 {
			if len(secondary) < 8 {
				log.Fatalf("invalid secondary data")
			}
			dictLen := int(binary.LittleEndian.Uint32(secondary))
			secondary = secondary[4:]
			if (dictLen >= (1 << 30)) || ((dictLen + 4) > len(secondary)) {
				log.Fatalf("invalid secondary data")
			}
			checksum := binary.LittleEndian.Uint32(secondary[dictLen:])
			dict = secondary[:dictLen]
			if checksum != crc32.ChecksumIEEE(dict) {
				log.Fatalf("invalid checksum")
			}
		}

		// Decompress the Zlib-encoded primary payload.
		primary := encoded[chunk.CPrimary[0]:chunk.CPrimary[1]]
		if zr == nil {
			if zr, err = zlib.NewReaderDict(bytes.NewReader(primary), dict); err != nil {
				log.Fatalf("zlib.NewReader: %v", err)
			}
		} else if err := zr.(zlib.Resetter).Reset(bytes.NewReader(primary), dict); err != nil {
			log.Fatalf("zlib.Reader.Reset: %v", err)
		}
		if n, err := io.Copy(os.Stdout, zr); err != nil {
			log.Fatalf("io.Copy: %v", err)
		} else if n != chunk.DRange.Size() {
			log.Fatalf("inconsistent DRange size")
		}
		if err := zr.Close(); err != nil {
			log.Fatalf("zlib.Reader.Close: %v", err)
		}
	}

	// Note that these exact bytes depends on the zlib encoder's algorithm, but
	// there is more than one valid zlib encoding of any given input. This
	// "compare to golden output" test is admittedly brittle, as the standard
	// library's zlib package's output isn't necessarily stable across Go
	// releases.

}
Output:

Encoded dictionary resource:
00000000  08 00 00 00 20 73 68 65  65 70 2e 0a d0 8d 7a 47  |.... sheep....zG|

Encoded chunk #0:
00000000  78 f9 0b e0 02 6e f2 cf  4b 85 31 01 01 00 00 ff  |x....n..K.1.....|
00000010  ff 17 21 03 90                                    |..!..|

Encoded chunk #1:
00000000  78 f9 0b e0 02 6e 0a 29  cf 87 31 01 01 00 00 ff  |x....n.)..1.....|
00000010  ff 18 0c 03 a8                                    |.....|

Encoded chunk #2:
00000000  78 f9 0b e0 02 6e 0a c9  28 4a 4d 85 71 00 01 00  |x....n..(JM.q...|
00000010  00 ff ff 21 6e 04 66                              |...!n.f|

RAC file:
00000000  72 c3 63 04 37 39 00 ff  00 00 00 00 00 00 00 ff  |r.c.79..........|
00000010  0b 00 00 00 00 00 00 ff  16 00 00 00 00 00 00 ff  |................|
00000020  23 00 00 00 00 00 00 01  50 00 00 00 00 00 01 ff  |#.......P.......|
00000030  60 00 00 00 00 00 01 00  75 00 00 00 00 00 01 00  |`.......u.......|
00000040  8a 00 00 00 00 00 01 00  a1 00 00 00 00 00 01 04  |................|
00000050  08 00 00 00 20 73 68 65  65 70 2e 0a d0 8d 7a 47  |.... sheep....zG|
00000060  78 f9 0b e0 02 6e f2 cf  4b 85 31 01 01 00 00 ff  |x....n..K.1.....|
00000070  ff 17 21 03 90 78 f9 0b  e0 02 6e 0a 29 cf 87 31  |..!..x....n.)..1|
00000080  01 01 00 00 ff ff 18 0c  03 a8 78 f9 0b e0 02 6e  |..........x....n|
00000090  0a c9 28 4a 4d 85 71 00  01 00 00 ff ff 21 6e 04  |..(JM.q......!n.|
000000a0  66                                                |f|

Decoded:
[ 0, 11): One sheep.
[11, 22): Two sheep.
[22, 35): Three sheep.

Index

Examples

Constants

View Source
const (
	CodecZeroes    = Codec(0x00 << 56)
	CodecZlib      = Codec(0x01 << 56)
	CodecLZ4       = Codec(0x02 << 56)
	CodecZstandard = Codec(0x03 << 56)

	CodecInvalid = Codec((1 << 64) - 1)
)
View Source
const (
	IndexLocationAtEnd   = IndexLocation(0)
	IndexLocationAtStart = IndexLocation(1)
)
View Source
const (
	// MaxSize is the maximum RAC file size (in both CSpace and DSpace).
	MaxSize = (1 << 48) - 1
)
View Source
const NoResourceUsed int = -1

NoResourceUsed is returned by CodecWriter.Compress to indicate that no secondary or tertiary resource was used in the compression.

Variables

View Source
var (
	ErrCodecWriterDoesNotSupportCChunkSize = errors.New("rac: CodecWriter does not support CChunkSize")
)

Functions

This section is empty.

Types

type Chunk

type Chunk struct {
	DRange     Range
	CPrimary   Range
	CSecondary Range
	CTertiary  Range
	STag       uint8
	TTag       uint8
	Codec      Codec
}

Chunk is a compressed chunk returned by a ChunkReader.

See the RAC specification for further discussion.

type ChunkReader

type ChunkReader struct {
	// ReadSeeker is where the RAC-encoded data is read from.
	//
	// It may optionally implement io.ReaderAt, in which case its ReadAt method
	// will be preferred and its Read and Seek methods will never be called.
	// The ReadAt method is safe to use concurrently, so that multiple
	// rac.Reader's can concurrently use the same source provided that the
	// source (this field, nominally an io.ReadSeeker) implements io.ReaderAt.
	//
	// In particular, if the source is a bytes.Reader or an os.File, multiple
	// rac.ChunkReader's can work in parallel, which can speed up decoding.
	//
	// This type itself only implements io.ReadSeeker, not io.ReaderAt, as it
	// is not safe for concurrent use.
	//
	// Nil is an invalid value.
	ReadSeeker io.ReadSeeker

	// CompressedSize is the size of the RAC file in CSpace.
	//
	// Zero is an invalid value. The smallest valid RAC file is 32 bytes long.
	CompressedSize int64
	// contains filtered or unexported fields
}

ChunkReader parses a RAC file.

Do not modify its exported fields after calling any of its methods.

func (*ChunkReader) DecompressedSize

func (r *ChunkReader) DecompressedSize() (int64, error)

DecompressedSize returns the total size of the decompressed data.

func (*ChunkReader) NextChunk

func (r *ChunkReader) NextChunk() (Chunk, error)

NextChunk returns the next independently compressed chunk, or io.EOF if there are no more chunks.

Empty chunks (those that contain no decompressed data, only metadata) are skipped.

func (*ChunkReader) SeekToChunkContaining

func (r *ChunkReader) SeekToChunkContaining(dSpaceOffset int64) error

SeekToChunkContaining sets up NextChunk to return the chunk containing dSpaceOffset. That chunk does not necessarily start at dSpaceOffset.

It is an error to seek to a negative value.

type ChunkWriter

type ChunkWriter struct {
	// Writer is where the RAC-encoded data is written to.
	//
	// Nil is an invalid value.
	Writer io.Writer

	// IndexLocation is whether the index is at the start or end of the RAC
	// file.
	//
	// See the RAC specification for further discussion.
	//
	// The zero value of this field is IndexLocationAtEnd: a one pass encoding.
	IndexLocation IndexLocation

	// TempFile is a temporary file to use for a two pass encoding. The field
	// name says "file" but it need not be a real file, in the operating system
	// sense.
	//
	// For IndexLocationAtEnd, this must be nil. For IndexLocationAtStart, this
	// must be non-nil.
	//
	// It does not need to implement io.Seeker, if it supports separate read
	// and write positions (e.g. it is a bytes.Buffer). If it does implement
	// io.Seeker (e.g. it is an os.File), its current position will be noted
	// (via SeekCurrent), then it will be written to (via the
	// ChunkWriter.AddXxx methods), then its position will be reset (via
	// SeekSet), then it will be read from (via the ChunkWriter.Close method).
	//
	// The ChunkWriter does not call TempFile.Close even if that method exists
	// (e.g. the TempFile is an os.File). In that case, closing the temporary
	// file (and deleting it) after the ChunkWriter is closed is the
	// responsibility of the ChunkWriter caller, not the ChunkWriter itself.
	TempFile io.ReadWriter

	// CPageSize guides padding the output to minimize the number of pages that
	// each chunk occupies (in what the RAC spec calls CSpace).
	//
	// It must either be zero (which means no padding is inserted) or a power
	// of 2, and no larger than MaxSize.
	//
	// For example, suppose that CSpace is partitioned into 1024-byte pages,
	// that 1000 bytes have already been written to the output, and the next
	// chunk is 1500 bytes long.
	//
	//   - With no padding (i.e. with CPageSize set to 0), this chunk will
	//     occupy the half-open range [1000, 2500) in CSpace, which straddles
	//     three 1024-byte pages: the page [0, 1024), the page [1024, 2048) and
	//     the page [2048, 3072). Call those pages p0, p1 and p2.
	//
	//   - With padding (i.e. with CPageSize set to 1024), 24 bytes of zeroes
	//     are first inserted so that this chunk occupies the half-open range
	//     [1024, 2524) in CSpace, which straddles only two pages (p1 and p2).
	//
	// This concept is similar, but not identical, to alignment. Even with a
	// non-zero CPageSize, chunk start offsets are not necessarily aligned to
	// page boundaries. For example, suppose that the chunk size was only 1040
	// bytes, not 1500 bytes. No padding will be inserted, as both [1000, 2040)
	// and [1024, 2064) straddle two pages: either pages p0 and p1, or pages p1
	// and p2.
	//
	// Nonetheless, if all chunks (or all but the final chunk) have a size
	// equal to (or just under) a multiple of the page size, then in practice,
	// each chunk's starting offset will be aligned to a page boundary.
	CPageSize uint64
	// contains filtered or unexported fields
}

ChunkWriter provides a relatively simple way to write a RAC file - one that is created starting from nothing, as opposed to incrementally modifying an existing RAC file.

Other packages may provide a more flexible (and more complicated) way to write or append to RAC files, but that is out of scope of this package.

Do not modify its exported fields after calling any of its methods.

func (*ChunkWriter) AddChunk

func (w *ChunkWriter) AddChunk(
	dRangeSize uint64, codec Codec, primary []byte,
	secondary OptResource, tertiary OptResource) error

AddChunk adds a chunk of compressed data - the (primary, secondary, tertiary) tuple - to the RAC file. Decompressing that chunk should produce dRangeSize bytes, although the ChunkWriter does not attempt to verify that.

The OptResource arguments may be zero, meaning that no resource is used. In that case, the corresponding STag or TTag (see the RAC specification for further discussion) will be 0xFF.

The caller may modify primary's contents after this method returns.

func (*ChunkWriter) AddResource

func (w *ChunkWriter) AddResource(resource []byte) (OptResource, error)

AddResource adds a shared resource to the RAC file. It returns a non-zero OptResource that identifies the resource's bytes. Future calls to AddChunk can pass these identifiers to indicate that decompressing a chunk depends on up to two shared resources.

The caller may modify resource's contents after this method returns.

func (*ChunkWriter) Close

func (w *ChunkWriter) Close() error

Close writes the RAC index to w.Writer and marks that w accepts no further method calls.

For a one pass encoding, no further action is taken. For a two pass encoding (i.e. IndexLocationAtStart), it then copies w.TempFile to w.Writer. Either way, if this method returns nil error, the entirety of what was written to w.Writer constitutes a valid RAC file.

Closing the underlying w.Writer, w.TempFile or both is the responsibility of the ChunkWriter caller, not the ChunkWriter itself.

It is not necessary to call Close, e.g. if an earlier AddXxx call returned a non-nil error. Unlike an os.File, failing to call ChunkWriter.Close will not leak resources such as file descriptors.

type Codec

type Codec uint64

Codec is the compression codec for the RAC file.

For leaf nodes, there are two categories of valid Codecs: Short and Long. A Short Codec's uint64 value's high 2 bits and low 56 bits must be zero. A Long Codec's uint64 value's high 8 bits must be one and then 7 zeroes. Symbolically, Short and Long match:

  • 0b00??????_00000000_00000000_00000000_00000000_00000000_00000000_00000000
  • 0b10000000_????????_????????_????????_????????_????????_????????_????????

In terms of the RAC file format, a Short Codec fits in a single byte: the Codec Byte in the middle of a branch node. A Long Codec uses that Codec Byte to locate 7 other bytes, a location which would otherwise form a "CPtr|CLen" value. When converting from the 7 bytes in the wire format to this Go type (a uint64 value), they are read little-endian: the "CPtr" bytes are the low 6 bytes, the "CLen" is the second-highest byte and the highest byte is hard-coded to 0x80.

For example, the 7 bytes "mdo2\x00\x00\x00" would correspond to a Codec value of 0x8000_0000_326F_646D.

The Mix Bit is not part of the uint64 representation. Neither is a Long Codec's 'c64' index. This package's exported API deals with leaf nodes. Branch nodes' wire formats are internal implementation details.

See the RAC specification for further discussion.

func (Codec) Valid

func (c Codec) Valid() bool

Valid returns whether c matches the Short or Long pattern.

type CodecReader

type CodecReader interface {
	// Close tells the CodecReader that no further calls will be made.
	Close() error

	// Accepts returns whether this CodecReader can decode a Codec.
	Accepts(c Codec) bool

	// Clone duplicates this. The clone and the original can run concurrently.
	Clone() CodecReader

	// MakeDecompressor returns the Codec-specific io.Reader for a chunk.
	//
	// The returned io.Reader may optionally implement the io.Closer interface,
	// in which case this Reader will call Close when has finished the chunk.
	MakeDecompressor(racFile io.ReadSeeker, c Chunk) (io.Reader, error)
}

CodecReader specializes a Reader to decode a specific compression codec.

Instances are not required to support concurrent use.

type CodecWriter

type CodecWriter interface {
	// Close tells the CodecWriter that no further calls will be made.
	Close() error

	// Clone duplicates this. The clone and the original can run concurrently.
	Clone() CodecWriter

	// Compress returns the codec-specific compressed form of the concatenation
	// of p and q.
	//
	// The source bytes are conceptually one continuous data stream, but is
	// presented in two slices to avoid unnecessary copying in the code that
	// calls Compress. One or both of p and q may be an empty slice. A
	// Compress(p, q, etc) is equivalent to, but often more efficient than:
	//   combined := []byte(nil)
	//   combined = append(combined, p...)
	//   combined = append(combined, q...)
	//   Compress(combined, nil, etc)
	//
	// Returning a secondaryResource or tertiaryResource within the range ((0
	// <= i) && (i < len(resourcesData))) indicates that resourcesData[i] was
	// used in the compression. A value outside of that range means that no
	// resource was used in the secondary and/or tertiary slot. The
	// NoResourceUsed constant (-1) is always outside that range.
	Compress(p []byte, q []byte, resourcesData [][]byte) (
		codec Codec, compressed []byte, secondaryResource int, tertiaryResource int, retErr error)

	// CanCut returns whether the CodecWriter supports the optional Cut method.
	CanCut() bool

	// Cut modifies encoded's contents such that encoded[:encodedLen] is valid
	// codec-compressed data, assuming that encoded starts off containing valid
	// codec-compressed data.
	//
	// If a nil error is returned, then encodedLen <= maxEncodedLen will hold.
	//
	// Decompressing that modified, shorter byte slice produces a prefix (of
	// length decodedLen) of the decompression of the original, longer byte
	// slice.
	Cut(codec Codec, encoded []byte, maxEncodedLen int) (
		encodedLen int, decodedLen int, retErr error)

	// WrapResource returns the Codec-specific encoding of the raw resource.
	// For example, if raw is the bytes of a shared LZ77-style dictionary,
	// WrapResource may prepend or append length and checksum data.
	WrapResource(raw []byte) ([]byte, error)
}

CodecWriter specializes a Writer to encode a specific compression codec.

Instances are not required to support concurrent use.

type IndexLocation

type IndexLocation uint8

IndexLocation is whether the index is at the start or end of the RAC file.

See the RAC specification for further discussion.

type OptResource

type OptResource uint32

OptResource is an option type, optionally holding a ChunkWriter-specific identifier for a shared resource.

Zero means that the option is not taken: no shared resource is used.

type Range

type Range [2]int64

Range is the half-open range [low, high). It is invalid for low to be greater than high.

func (Range) Empty

func (r Range) Empty() bool

func (Range) Intersect

func (r Range) Intersect(s Range) Range

func (Range) Size

func (r Range) Size() int64

type Reader

type Reader struct {
	// ReadSeeker is where the RAC-encoded data is read from.
	//
	// It may optionally implement io.ReaderAt, in which case its ReadAt method
	// will be preferred and its Read and Seek methods will never be called.
	// The ReadAt method is safe to use concurrently, so that multiple
	// rac.Reader's can concurrently use the same source provided that the
	// source (this field, nominally an io.ReadSeeker) implements io.ReaderAt.
	//
	// In particular, if the source is a bytes.Reader or an os.File, multiple
	// rac.Reader's can work in parallel, which can speed up decoding.
	//
	// This type itself only implements io.ReadSeeker, not io.ReaderAt, as it
	// is not safe for concurrent use.
	//
	// Nil is an invalid value.
	ReadSeeker io.ReadSeeker

	// CompressedSize is the size of the RAC file in CSpace.
	//
	// Zero is an invalid value. The smallest valid RAC file is 32 bytes long.
	CompressedSize int64

	// CodecReaders are the compression codecs that this Reader can decompress.
	//
	// For example, use a raczlib.CodecReader from the sibilng "raczlib"
	// package.
	CodecReaders []CodecReader

	// Concurrency is how many worker goroutines are used to decode RAC chunks.
	// Bigger values often lead to faster throughput, up to a
	// hardware-dependent point, but also larger memory requirements.
	//
	// If positive, then the ReadSeeker must also be an io.ReaderAt.
	//
	// Non-positive values (including zero) mean a non-concurrent
	// (single-goroutine) reader.
	Concurrency int
	// contains filtered or unexported fields
}

Reader reads a RAC file.

Do not modify its exported fields after calling any of its methods.

Reader implements the io.ReadSeeker and io.Closer interfaces.

func (*Reader) Close

func (r *Reader) Close() error

Close implements io.Closer.

Calling Close will call Close on all of r's CodecReaders.

r.ReadSeeker will not be accessed after Close returns. If r.Concurrency is non-zero, this may involve waiting for various goroutines to shut down, which may take some time. If the caller does not care about waiting until it is safe to close or otherwise release the r.ReadSeeker's resources, call CloseWithoutWaiting instead.

It is not safe to call Close from a separate goroutine while another method call like Read or Seek is in progress.

func (*Reader) CloseWithoutWaiting

func (r *Reader) CloseWithoutWaiting() error

CloseWithoutWaiting is like Close but does not wait until it is safe to close or otherwise release the r.ReadSeeker's resources.

func (*Reader) Read

func (r *Reader) Read(p []byte) (int, error)

Read implements io.Reader.

func (*Reader) Seek

func (r *Reader) Seek(offset int64, whence int) (int64, error)

Seek implements io.Seeker.

func (*Reader) SeekRange

func (r *Reader) SeekRange(low int64, high int64) error

SeekRange restricts r to the half-open range [low, high).

It is more efficient than but conceptually equivalent to calling Seek(low, io.SeekStart) and wrapping r in an io.LimitedReader whose N is (high - low).

Multiple SeekRange calls apply the most recent high limit, not the minimum of the high limits.

Any Seek call, such as Seek(0, io.SeekCurrent), will remove the high limit.

It returns an error if low > high.

type Writer

type Writer struct {
	// Writer is where the RAC-encoded data is written to.
	//
	// Nil is an invalid value.
	Writer io.Writer

	// CodecWriter is the compression codec that this Writer can compress to.
	//
	// For example, use a raczlib.CodecWriter from the sibilng "raczlib"
	// package.
	//
	// Nil is an invalid value.
	CodecWriter CodecWriter

	// IndexLocation is whether the index is at the start or end of the RAC
	// file.
	//
	// See the RAC specification for further discussion.
	//
	// The zero value of this field is IndexLocationAtEnd: a one pass encoding.
	IndexLocation IndexLocation

	// TempFile is a temporary file to use for a two pass encoding. The field
	// name says "file" but it need not be a real file, in the operating system
	// sense.
	//
	// For IndexLocationAtEnd, this must be nil. For IndexLocationAtStart, this
	// must be non-nil.
	//
	// It does not need to implement io.Seeker, if it supports separate read
	// and write positions (e.g. it is a bytes.Buffer). If it does implement
	// io.Seeker (e.g. it is an os.File), its current position will be noted
	// (via SeekCurrent), then it will be written to (via the rac.Writer.Write
	// method), then its position will be reset (via SeekSet), then it will be
	// read from (via the rac.Writer.Close method).
	//
	// The rac.Writer does not call TempFile.Close even if that method exists
	// (e.g. the TempFile is an os.File). In that case, closing the temporary
	// file (and deleting it) after the rac.Writer is closed is the
	// responsibility of the rac.Writer caller, not the rac.Writer itself.
	TempFile io.ReadWriter

	// CPageSize guides padding the output to minimize the number of pages that
	// each chunk occupies (in what the RAC spec calls CSpace).
	//
	// It must either be zero (which means no padding is inserted) or a power
	// of 2, and no larger than MaxSize.
	//
	// For example, suppose that CSpace is partitioned into 1024-byte pages,
	// that 1000 bytes have already been written to the output, and the next
	// chunk is 1500 bytes long.
	//
	//   - With no padding (i.e. with CPageSize set to 0), this chunk will
	//     occupy the half-open range [1000, 2500) in CSpace, which straddles
	//     three 1024-byte pages: the page [0, 1024), the page [1024, 2048) and
	//     the page [2048, 3072). Call those pages p0, p1 and p2.
	//
	//   - With padding (i.e. with CPageSize set to 1024), 24 bytes of zeroes
	//     are first inserted so that this chunk occupies the half-open range
	//     [1024, 2524) in CSpace, which straddles only two pages (p1 and p2).
	//
	// This concept is similar, but not identical, to alignment. Even with a
	// non-zero CPageSize, chunk start offsets are not necessarily aligned to
	// page boundaries. For example, suppose that the chunk size was only 1040
	// bytes, not 1500 bytes. No padding will be inserted, as both [1000, 2040)
	// and [1024, 2064) straddle two pages: either pages p0 and p1, or pages p1
	// and p2.
	//
	// Nonetheless, if all chunks (or all but the final chunk) have a size
	// equal to (or just under) a multiple of the page size, then in practice,
	// each chunk's starting offset will be aligned to a page boundary.
	CPageSize uint64

	// CChunkSize is the compressed size (i.e. size in CSpace) of each chunk.
	// The final chunk might be smaller than this.
	//
	// This field is ignored if DChunkSize is non-zero.
	CChunkSize uint64

	// DChunkSize is the uncompressed size (i.e. size in DSpace) of each chunk.
	// The final chunk might be smaller than this.
	//
	// If both CChunkSize and DChunkSize are non-zero, DChunkSize takes
	// precedence and CChunkSize is ignored.
	//
	// If both CChunkSize and DChunkSize are zero, a default DChunkSize value
	// will be used.
	DChunkSize uint64

	// ResourcesData is a list of resources that can be shared across otherwise
	// independently compressed chunks.
	//
	// The exact semantics for a resource depends on the codec. For example,
	// the Zlib codec interprets them as shared LZ77-style dictionaries.
	//
	// One way to generate shared dictionaries from sub-slices of a single
	// source file is the command line tool at
	// https://github.com/google/brotli/blob/master/research/dictionary_generator.cc
	ResourcesData [][]byte
	// contains filtered or unexported fields
}

Writer provides a relatively simple way to write a RAC file - one that is created starting from nothing, as opposed to incrementally modifying an existing RAC file.

Other packages may provide a more flexible (and more complicated) way to write or append to RAC files, but that is out of scope of this package.

Do not modify its exported fields after calling any of its methods.

Writer implements the io.WriteCloser interface.

func (*Writer) Close

func (w *Writer) Close() error

Close writes the RAC index to w.Writer and marks that w accepts no further method calls.

Calling Close will call Close on w's CodecWriter.

For a one pass encoding, no further action is taken. For a two pass encoding (i.e. IndexLocationAtStart), it then copies w.TempFile to w.Writer. Either way, if this method returns nil error, the entirety of what was written to w.Writer constitutes a valid RAC file.

Closing the underlying w.Writer, w.TempFile or both is the responsibility of the rac.Writer caller, not the rac.Writer itself.

func (*Writer) Write

func (w *Writer) Write(p []byte) (int, error)

Write implements io.Writer.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL