Documentation
¶
Overview ¶
Package rac provides access to RAC (Random Access Compression) files.
RAC is just a container format, and this package ("rac") is a relatively low-level. Users will typically want to process a particular compression format wrapped in RAC, such as (RAC + Zlib). For that, look at e.g. the sibling "raczlib" package.
The RAC specification is at https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md
Example (IndexLocationAtEnd) ¶
Example_indexLocationAtEnd demonstrates using the low level "rac" package to encode a RAC+Zlib formatted file with IndexLocationAtEnd.
The sibling "raczlib" package provides a higher level API that is easier to use.
See the RAC specification for an explanation of the file format.
package main
import (
"bytes"
"compress/zlib"
"encoding/hex"
"fmt"
"hash/adler32"
"io"
"log"
"github.com/google/wuffs/lib/rac"
)
func main() {
// Manually construct a zlib encoding of "More!\n", one that uses a literal
// block (that's easy to see in a hex dump) instead of a more compressible
// Huffman block.
const src = "More!\n"
hasher := adler32.New()
hasher.Write([]byte(src))
enc := []byte{ // See RFCs 1950 and 1951 for details.
0x78, // Deflate compression method; 32KiB window size.
0x9C, // Default encoding algorithm; FCHECK bits.
0x01, // Literal block (final).
0x06, 0x00, // Literal length.
0xF9, 0xFF, // Inverse of the literal length.
}
enc = append(enc, src...) // Literal bytes.
enc = hasher.Sum(enc) // Adler-32 hash.
// Check that we've constructed a valid zlib-formatted encoding, by
// checking that decoding enc produces src.
{
b := &bytes.Buffer{}
r, err := zlib.NewReader(bytes.NewReader(enc))
if err != nil {
log.Fatalf("NewReader: %v", err)
}
if _, err := io.Copy(b, r); err != nil {
log.Fatalf("Copy: %v", err)
}
if got := b.String(); got != src {
log.Fatalf("zlib check: got %q, want %q", got, src)
}
}
buf := &bytes.Buffer{}
w := &rac.ChunkWriter{
Writer: buf,
}
if err := w.AddChunk(uint64(len(src)), rac.CodecZlib, enc, 0, 0); err != nil {
log.Fatalf("AddChunk: %v", err)
}
if err := w.Close(); err != nil {
log.Fatalf("Close: %v", err)
}
fmt.Printf("RAC file:\n%s", hex.Dump(buf.Bytes()))
}
Output: RAC file: 00000000 72 c3 63 00 78 9c 01 06 00 f9 ff 4d 6f 72 65 21 |r.c.x......More!| 00000010 0a 07 42 01 bf 72 c3 63 01 65 a9 00 ff 06 00 00 |..B..r.c.e......| 00000020 00 00 00 00 01 04 00 00 00 00 00 01 ff 35 00 00 |.............5..| 00000030 00 00 00 01 01 |.....|
Example (IndexLocationAtStart) ¶
Example_indexLocationAtStart demonstrates using the low level "rac" package to encode and then decode a RAC+Zlib formatted file with IndexLocationAtStart.
The sibling "raczlib" package provides a higher level API that is easier to use.
See the RAC specification for an explanation of the file format.
package main
import (
"bytes"
"compress/zlib"
"encoding/binary"
"encoding/hex"
"fmt"
"hash/crc32"
"io"
"log"
"os"
"github.com/google/wuffs/lib/rac"
)
func main() {
buf := &bytes.Buffer{}
w := &rac.ChunkWriter{
Writer: buf,
IndexLocation: rac.IndexLocationAtStart,
TempFile: &bytes.Buffer{},
}
dict := []byte(" sheep.\n")
if len(dict) >= (1 << 30) {
log.Fatal("len(dict) is too large")
}
encodedDict := []byte{
uint8(len(dict) >> 0),
uint8(len(dict) >> 8),
uint8(len(dict) >> 16),
uint8(len(dict) >> 24),
}
encodedDict = append(encodedDict, dict...)
checksum := crc32.ChecksumIEEE(dict)
encodedDict = append(encodedDict,
uint8(checksum>>0),
uint8(checksum>>8),
uint8(checksum>>16),
uint8(checksum>>24),
)
fmt.Printf("Encoded dictionary resource:\n%s\n", hex.Dump(encodedDict))
dictResource, err := w.AddResource(encodedDict)
if err != nil {
log.Fatalf("AddResource: %v", err)
}
chunks := []string{
"One sheep.\n",
"Two sheep.\n",
"Three sheep.\n",
}
for i, chunk := range chunks {
b := &bytes.Buffer{}
if z, err := zlib.NewWriterLevelDict(b, zlib.BestCompression, dict); err != nil {
log.Fatalf("NewWriterLevelDict: %v", err)
} else if _, err := z.Write([]byte(chunk)); err != nil {
log.Fatalf("Write: %v", err)
} else if err := z.Close(); err != nil {
log.Fatalf("Close: %v", err)
}
encodedChunk := b.Bytes()
if err := w.AddChunk(uint64(len(chunk)), rac.CodecZlib, encodedChunk, dictResource, 0); err != nil {
log.Fatalf("AddChunk: %v", err)
}
fmt.Printf("Encoded chunk #%d:\n%s\n", i, hex.Dump(encodedChunk))
}
if err := w.Close(); err != nil {
log.Fatalf("Close: %v", err)
}
encoded := buf.Bytes()
fmt.Printf("RAC file:\n%s\n", hex.Dump(encoded))
// Decode the encoded bytes (the RAC-formatted bytes) to recover the
// original "One sheep.\nTwo sheep\.Three sheep.\n" source.
fmt.Printf("Decoded:\n")
r := &rac.ChunkReader{
ReadSeeker: bytes.NewReader(encoded),
CompressedSize: int64(len(encoded)),
}
zr := io.ReadCloser(nil)
for {
chunk, err := r.NextChunk()
if err == io.EOF {
break
} else if err != nil {
log.Fatalf("NextChunk: %v", err)
}
if chunk.Codec != rac.CodecZlib {
log.Fatalf("unexpected chunk codec")
}
fmt.Printf("[%2d, %2d): ", chunk.DRange[0], chunk.DRange[1])
// Parse the RAC+Zlib secondary data. For details, see
// https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md#rac--zlib
dict := []byte(nil)
if secondary := encoded[chunk.CSecondary[0]:chunk.CSecondary[1]]; len(secondary) > 0 {
if len(secondary) < 8 {
log.Fatalf("invalid secondary data")
}
dictLen := int(binary.LittleEndian.Uint32(secondary))
secondary = secondary[4:]
if (dictLen >= (1 << 30)) || ((dictLen + 4) > len(secondary)) {
log.Fatalf("invalid secondary data")
}
checksum := binary.LittleEndian.Uint32(secondary[dictLen:])
dict = secondary[:dictLen]
if checksum != crc32.ChecksumIEEE(dict) {
log.Fatalf("invalid checksum")
}
}
// Decompress the Zlib-encoded primary payload.
primary := encoded[chunk.CPrimary[0]:chunk.CPrimary[1]]
if zr == nil {
if zr, err = zlib.NewReaderDict(bytes.NewReader(primary), dict); err != nil {
log.Fatalf("zlib.NewReader: %v", err)
}
} else if err := zr.(zlib.Resetter).Reset(bytes.NewReader(primary), dict); err != nil {
log.Fatalf("zlib.Reader.Reset: %v", err)
}
if n, err := io.Copy(os.Stdout, zr); err != nil {
log.Fatalf("io.Copy: %v", err)
} else if n != chunk.DRange.Size() {
log.Fatalf("inconsistent DRange size")
}
if err := zr.Close(); err != nil {
log.Fatalf("zlib.Reader.Close: %v", err)
}
}
// Note that these exact bytes depends on the zlib encoder's algorithm, but
// there is more than one valid zlib encoding of any given input. This
// "compare to golden output" test is admittedly brittle, as the standard
// library's zlib package's output isn't necessarily stable across Go
// releases.
}
Output: Encoded dictionary resource: 00000000 08 00 00 00 20 73 68 65 65 70 2e 0a d0 8d 7a 47 |.... sheep....zG| Encoded chunk #0: 00000000 78 f9 0b e0 02 6e f2 cf 4b 85 31 01 01 00 00 ff |x....n..K.1.....| 00000010 ff 17 21 03 90 |..!..| Encoded chunk #1: 00000000 78 f9 0b e0 02 6e 0a 29 cf 87 31 01 01 00 00 ff |x....n.)..1.....| 00000010 ff 18 0c 03 a8 |.....| Encoded chunk #2: 00000000 78 f9 0b e0 02 6e 0a c9 28 4a 4d 85 71 00 01 00 |x....n..(JM.q...| 00000010 00 ff ff 21 6e 04 66 |...!n.f| RAC file: 00000000 72 c3 63 04 37 39 00 ff 00 00 00 00 00 00 00 ff |r.c.79..........| 00000010 0b 00 00 00 00 00 00 ff 16 00 00 00 00 00 00 ff |................| 00000020 23 00 00 00 00 00 00 01 50 00 00 00 00 00 01 ff |#.......P.......| 00000030 60 00 00 00 00 00 01 00 75 00 00 00 00 00 01 00 |`.......u.......| 00000040 8a 00 00 00 00 00 01 00 a1 00 00 00 00 00 01 04 |................| 00000050 08 00 00 00 20 73 68 65 65 70 2e 0a d0 8d 7a 47 |.... sheep....zG| 00000060 78 f9 0b e0 02 6e f2 cf 4b 85 31 01 01 00 00 ff |x....n..K.1.....| 00000070 ff 17 21 03 90 78 f9 0b e0 02 6e 0a 29 cf 87 31 |..!..x....n.)..1| 00000080 01 01 00 00 ff ff 18 0c 03 a8 78 f9 0b e0 02 6e |..........x....n| 00000090 0a c9 28 4a 4d 85 71 00 01 00 00 ff ff 21 6e 04 |..(JM.q......!n.| 000000a0 66 |f| Decoded: [ 0, 11): One sheep. [11, 22): Two sheep. [22, 35): Three sheep.
Index ¶
Examples ¶
Constants ¶
const ( CodecZeroes = Codec(0x00 << 56) CodecZlib = Codec(0x01 << 56) CodecLZ4 = Codec(0x02 << 56) CodecZstandard = Codec(0x03 << 56) CodecInvalid = Codec((1 << 64) - 1) )
const ( IndexLocationAtEnd = IndexLocation(0) IndexLocationAtStart = IndexLocation(1) )
const (
// MaxSize is the maximum RAC file size (in both CSpace and DSpace).
MaxSize = (1 << 48) - 1
)
const NoResourceUsed int = -1
NoResourceUsed is returned by CodecWriter.Compress to indicate that no secondary or tertiary resource was used in the compression.
Variables ¶
var (
ErrCodecWriterDoesNotSupportCChunkSize = errors.New("rac: CodecWriter does not support CChunkSize")
)
Functions ¶
This section is empty.
Types ¶
type Chunk ¶
type Chunk struct {
DRange Range
CPrimary Range
CSecondary Range
CTertiary Range
STag uint8
TTag uint8
Codec Codec
}
Chunk is a compressed chunk returned by a ChunkReader.
See the RAC specification for further discussion.
type ChunkReader ¶
type ChunkReader struct {
// ReadSeeker is where the RAC-encoded data is read from.
//
// It may optionally implement io.ReaderAt, in which case its ReadAt method
// will be preferred and its Read and Seek methods will never be called.
// The ReadAt method is safe to use concurrently, so that multiple
// rac.Reader's can concurrently use the same source provided that the
// source (this field, nominally an io.ReadSeeker) implements io.ReaderAt.
//
// In particular, if the source is a bytes.Reader or an os.File, multiple
// rac.ChunkReader's can work in parallel, which can speed up decoding.
//
// This type itself only implements io.ReadSeeker, not io.ReaderAt, as it
// is not safe for concurrent use.
//
// Nil is an invalid value.
ReadSeeker io.ReadSeeker
// CompressedSize is the size of the RAC file in CSpace.
//
// Zero is an invalid value. The smallest valid RAC file is 32 bytes long.
CompressedSize int64
// contains filtered or unexported fields
}
ChunkReader parses a RAC file.
Do not modify its exported fields after calling any of its methods.
func (*ChunkReader) DecompressedSize ¶
func (r *ChunkReader) DecompressedSize() (int64, error)
DecompressedSize returns the total size of the decompressed data.
func (*ChunkReader) NextChunk ¶
func (r *ChunkReader) NextChunk() (Chunk, error)
NextChunk returns the next independently compressed chunk, or io.EOF if there are no more chunks.
Empty chunks (those that contain no decompressed data, only metadata) are skipped.
func (*ChunkReader) SeekToChunkContaining ¶
func (r *ChunkReader) SeekToChunkContaining(dSpaceOffset int64) error
SeekToChunkContaining sets up NextChunk to return the chunk containing dSpaceOffset. That chunk does not necessarily start at dSpaceOffset.
It is an error to seek to a negative value.
type ChunkWriter ¶
type ChunkWriter struct {
// Writer is where the RAC-encoded data is written to.
//
// Nil is an invalid value.
Writer io.Writer
// IndexLocation is whether the index is at the start or end of the RAC
// file.
//
// See the RAC specification for further discussion.
//
// The zero value of this field is IndexLocationAtEnd: a one pass encoding.
IndexLocation IndexLocation
// TempFile is a temporary file to use for a two pass encoding. The field
// name says "file" but it need not be a real file, in the operating system
// sense.
//
// For IndexLocationAtEnd, this must be nil. For IndexLocationAtStart, this
// must be non-nil.
//
// It does not need to implement io.Seeker, if it supports separate read
// and write positions (e.g. it is a bytes.Buffer). If it does implement
// io.Seeker (e.g. it is an os.File), its current position will be noted
// (via SeekCurrent), then it will be written to (via the
// ChunkWriter.AddXxx methods), then its position will be reset (via
// SeekSet), then it will be read from (via the ChunkWriter.Close method).
//
// The ChunkWriter does not call TempFile.Close even if that method exists
// (e.g. the TempFile is an os.File). In that case, closing the temporary
// file (and deleting it) after the ChunkWriter is closed is the
// responsibility of the ChunkWriter caller, not the ChunkWriter itself.
TempFile io.ReadWriter
// CPageSize guides padding the output to minimize the number of pages that
// each chunk occupies (in what the RAC spec calls CSpace).
//
// It must either be zero (which means no padding is inserted) or a power
// of 2, and no larger than MaxSize.
//
// For example, suppose that CSpace is partitioned into 1024-byte pages,
// that 1000 bytes have already been written to the output, and the next
// chunk is 1500 bytes long.
//
// - With no padding (i.e. with CPageSize set to 0), this chunk will
// occupy the half-open range [1000, 2500) in CSpace, which straddles
// three 1024-byte pages: the page [0, 1024), the page [1024, 2048) and
// the page [2048, 3072). Call those pages p0, p1 and p2.
//
// - With padding (i.e. with CPageSize set to 1024), 24 bytes of zeroes
// are first inserted so that this chunk occupies the half-open range
// [1024, 2524) in CSpace, which straddles only two pages (p1 and p2).
//
// This concept is similar, but not identical, to alignment. Even with a
// non-zero CPageSize, chunk start offsets are not necessarily aligned to
// page boundaries. For example, suppose that the chunk size was only 1040
// bytes, not 1500 bytes. No padding will be inserted, as both [1000, 2040)
// and [1024, 2064) straddle two pages: either pages p0 and p1, or pages p1
// and p2.
//
// Nonetheless, if all chunks (or all but the final chunk) have a size
// equal to (or just under) a multiple of the page size, then in practice,
// each chunk's starting offset will be aligned to a page boundary.
CPageSize uint64
// contains filtered or unexported fields
}
ChunkWriter provides a relatively simple way to write a RAC file - one that is created starting from nothing, as opposed to incrementally modifying an existing RAC file.
Other packages may provide a more flexible (and more complicated) way to write or append to RAC files, but that is out of scope of this package.
Do not modify its exported fields after calling any of its methods.
func (*ChunkWriter) AddChunk ¶
func (w *ChunkWriter) AddChunk( dRangeSize uint64, codec Codec, primary []byte, secondary OptResource, tertiary OptResource) error
AddChunk adds a chunk of compressed data - the (primary, secondary, tertiary) tuple - to the RAC file. Decompressing that chunk should produce dRangeSize bytes, although the ChunkWriter does not attempt to verify that.
The OptResource arguments may be zero, meaning that no resource is used. In that case, the corresponding STag or TTag (see the RAC specification for further discussion) will be 0xFF.
The caller may modify primary's contents after this method returns.
func (*ChunkWriter) AddResource ¶
func (w *ChunkWriter) AddResource(resource []byte) (OptResource, error)
AddResource adds a shared resource to the RAC file. It returns a non-zero OptResource that identifies the resource's bytes. Future calls to AddChunk can pass these identifiers to indicate that decompressing a chunk depends on up to two shared resources.
The caller may modify resource's contents after this method returns.
func (*ChunkWriter) Close ¶
func (w *ChunkWriter) Close() error
Close writes the RAC index to w.Writer and marks that w accepts no further method calls.
For a one pass encoding, no further action is taken. For a two pass encoding (i.e. IndexLocationAtStart), it then copies w.TempFile to w.Writer. Either way, if this method returns nil error, the entirety of what was written to w.Writer constitutes a valid RAC file.
Closing the underlying w.Writer, w.TempFile or both is the responsibility of the ChunkWriter caller, not the ChunkWriter itself.
It is not necessary to call Close, e.g. if an earlier AddXxx call returned a non-nil error. Unlike an os.File, failing to call ChunkWriter.Close will not leak resources such as file descriptors.
type Codec ¶
type Codec uint64
Codec is the compression codec for the RAC file.
For leaf nodes, there are two categories of valid Codecs: Short and Long. A Short Codec's uint64 value's high 2 bits and low 56 bits must be zero. A Long Codec's uint64 value's high 8 bits must be one and then 7 zeroes. Symbolically, Short and Long match:
- 0b00??????_00000000_00000000_00000000_00000000_00000000_00000000_00000000
- 0b10000000_????????_????????_????????_????????_????????_????????_????????
In terms of the RAC file format, a Short Codec fits in a single byte: the Codec Byte in the middle of a branch node. A Long Codec uses that Codec Byte to locate 7 other bytes, a location which would otherwise form a "CPtr|CLen" value. When converting from the 7 bytes in the wire format to this Go type (a uint64 value), they are read little-endian: the "CPtr" bytes are the low 6 bytes, the "CLen" is the second-highest byte and the highest byte is hard-coded to 0x80.
For example, the 7 bytes "mdo2\x00\x00\x00" would correspond to a Codec value of 0x8000_0000_326F_646D.
The Mix Bit is not part of the uint64 representation. Neither is a Long Codec's 'c64' index. This package's exported API deals with leaf nodes. Branch nodes' wire formats are internal implementation details.
See the RAC specification for further discussion.
type CodecReader ¶
type CodecReader interface {
// Close tells the CodecReader that no further calls will be made.
Close() error
// Accepts returns whether this CodecReader can decode a Codec.
Accepts(c Codec) bool
// Clone duplicates this. The clone and the original can run concurrently.
Clone() CodecReader
// MakeDecompressor returns the Codec-specific io.Reader for a chunk.
//
// The returned io.Reader may optionally implement the io.Closer interface,
// in which case this Reader will call Close when has finished the chunk.
MakeDecompressor(racFile io.ReadSeeker, c Chunk) (io.Reader, error)
}
CodecReader specializes a Reader to decode a specific compression codec.
Instances are not required to support concurrent use.
type CodecWriter ¶
type CodecWriter interface {
// Close tells the CodecWriter that no further calls will be made.
Close() error
// Clone duplicates this. The clone and the original can run concurrently.
Clone() CodecWriter
// Compress returns the codec-specific compressed form of the concatenation
// of p and q.
//
// The source bytes are conceptually one continuous data stream, but is
// presented in two slices to avoid unnecessary copying in the code that
// calls Compress. One or both of p and q may be an empty slice. A
// Compress(p, q, etc) is equivalent to, but often more efficient than:
// combined := []byte(nil)
// combined = append(combined, p...)
// combined = append(combined, q...)
// Compress(combined, nil, etc)
//
// Returning a secondaryResource or tertiaryResource within the range ((0
// <= i) && (i < len(resourcesData))) indicates that resourcesData[i] was
// used in the compression. A value outside of that range means that no
// resource was used in the secondary and/or tertiary slot. The
// NoResourceUsed constant (-1) is always outside that range.
Compress(p []byte, q []byte, resourcesData [][]byte) (
codec Codec, compressed []byte, secondaryResource int, tertiaryResource int, retErr error)
// CanCut returns whether the CodecWriter supports the optional Cut method.
CanCut() bool
// Cut modifies encoded's contents such that encoded[:encodedLen] is valid
// codec-compressed data, assuming that encoded starts off containing valid
// codec-compressed data.
//
// If a nil error is returned, then encodedLen <= maxEncodedLen will hold.
//
// Decompressing that modified, shorter byte slice produces a prefix (of
// length decodedLen) of the decompression of the original, longer byte
// slice.
Cut(codec Codec, encoded []byte, maxEncodedLen int) (
encodedLen int, decodedLen int, retErr error)
// WrapResource returns the Codec-specific encoding of the raw resource.
// For example, if raw is the bytes of a shared LZ77-style dictionary,
// WrapResource may prepend or append length and checksum data.
WrapResource(raw []byte) ([]byte, error)
}
CodecWriter specializes a Writer to encode a specific compression codec.
Instances are not required to support concurrent use.
type IndexLocation ¶
type IndexLocation uint8
IndexLocation is whether the index is at the start or end of the RAC file.
See the RAC specification for further discussion.
type OptResource ¶
type OptResource uint32
OptResource is an option type, optionally holding a ChunkWriter-specific identifier for a shared resource.
Zero means that the option is not taken: no shared resource is used.
type Range ¶
type Range [2]int64
Range is the half-open range [low, high). It is invalid for low to be greater than high.
type Reader ¶
type Reader struct {
// ReadSeeker is where the RAC-encoded data is read from.
//
// It may optionally implement io.ReaderAt, in which case its ReadAt method
// will be preferred and its Read and Seek methods will never be called.
// The ReadAt method is safe to use concurrently, so that multiple
// rac.Reader's can concurrently use the same source provided that the
// source (this field, nominally an io.ReadSeeker) implements io.ReaderAt.
//
// In particular, if the source is a bytes.Reader or an os.File, multiple
// rac.Reader's can work in parallel, which can speed up decoding.
//
// This type itself only implements io.ReadSeeker, not io.ReaderAt, as it
// is not safe for concurrent use.
//
// Nil is an invalid value.
ReadSeeker io.ReadSeeker
// CompressedSize is the size of the RAC file in CSpace.
//
// Zero is an invalid value. The smallest valid RAC file is 32 bytes long.
CompressedSize int64
// CodecReaders are the compression codecs that this Reader can decompress.
//
// For example, use a raczlib.CodecReader from the sibilng "raczlib"
// package.
CodecReaders []CodecReader
// Concurrency is how many worker goroutines are used to decode RAC chunks.
// Bigger values often lead to faster throughput, up to a
// hardware-dependent point, but also larger memory requirements.
//
// If positive, then the ReadSeeker must also be an io.ReaderAt.
//
// Non-positive values (including zero) mean a non-concurrent
// (single-goroutine) reader.
Concurrency int
// contains filtered or unexported fields
}
Reader reads a RAC file.
Do not modify its exported fields after calling any of its methods.
Reader implements the io.ReadSeeker and io.Closer interfaces.
func (*Reader) Close ¶
Close implements io.Closer.
Calling Close will call Close on all of r's CodecReaders.
r.ReadSeeker will not be accessed after Close returns. If r.Concurrency is non-zero, this may involve waiting for various goroutines to shut down, which may take some time. If the caller does not care about waiting until it is safe to close or otherwise release the r.ReadSeeker's resources, call CloseWithoutWaiting instead.
It is not safe to call Close from a separate goroutine while another method call like Read or Seek is in progress.
func (*Reader) CloseWithoutWaiting ¶
CloseWithoutWaiting is like Close but does not wait until it is safe to close or otherwise release the r.ReadSeeker's resources.
func (*Reader) SeekRange ¶
SeekRange restricts r to the half-open range [low, high).
It is more efficient than but conceptually equivalent to calling Seek(low, io.SeekStart) and wrapping r in an io.LimitedReader whose N is (high - low).
Multiple SeekRange calls apply the most recent high limit, not the minimum of the high limits.
Any Seek call, such as Seek(0, io.SeekCurrent), will remove the high limit.
It returns an error if low > high.
type Writer ¶
type Writer struct {
// Writer is where the RAC-encoded data is written to.
//
// Nil is an invalid value.
Writer io.Writer
// CodecWriter is the compression codec that this Writer can compress to.
//
// For example, use a raczlib.CodecWriter from the sibilng "raczlib"
// package.
//
// Nil is an invalid value.
CodecWriter CodecWriter
// IndexLocation is whether the index is at the start or end of the RAC
// file.
//
// See the RAC specification for further discussion.
//
// The zero value of this field is IndexLocationAtEnd: a one pass encoding.
IndexLocation IndexLocation
// TempFile is a temporary file to use for a two pass encoding. The field
// name says "file" but it need not be a real file, in the operating system
// sense.
//
// For IndexLocationAtEnd, this must be nil. For IndexLocationAtStart, this
// must be non-nil.
//
// It does not need to implement io.Seeker, if it supports separate read
// and write positions (e.g. it is a bytes.Buffer). If it does implement
// io.Seeker (e.g. it is an os.File), its current position will be noted
// (via SeekCurrent), then it will be written to (via the rac.Writer.Write
// method), then its position will be reset (via SeekSet), then it will be
// read from (via the rac.Writer.Close method).
//
// The rac.Writer does not call TempFile.Close even if that method exists
// (e.g. the TempFile is an os.File). In that case, closing the temporary
// file (and deleting it) after the rac.Writer is closed is the
// responsibility of the rac.Writer caller, not the rac.Writer itself.
TempFile io.ReadWriter
// CPageSize guides padding the output to minimize the number of pages that
// each chunk occupies (in what the RAC spec calls CSpace).
//
// It must either be zero (which means no padding is inserted) or a power
// of 2, and no larger than MaxSize.
//
// For example, suppose that CSpace is partitioned into 1024-byte pages,
// that 1000 bytes have already been written to the output, and the next
// chunk is 1500 bytes long.
//
// - With no padding (i.e. with CPageSize set to 0), this chunk will
// occupy the half-open range [1000, 2500) in CSpace, which straddles
// three 1024-byte pages: the page [0, 1024), the page [1024, 2048) and
// the page [2048, 3072). Call those pages p0, p1 and p2.
//
// - With padding (i.e. with CPageSize set to 1024), 24 bytes of zeroes
// are first inserted so that this chunk occupies the half-open range
// [1024, 2524) in CSpace, which straddles only two pages (p1 and p2).
//
// This concept is similar, but not identical, to alignment. Even with a
// non-zero CPageSize, chunk start offsets are not necessarily aligned to
// page boundaries. For example, suppose that the chunk size was only 1040
// bytes, not 1500 bytes. No padding will be inserted, as both [1000, 2040)
// and [1024, 2064) straddle two pages: either pages p0 and p1, or pages p1
// and p2.
//
// Nonetheless, if all chunks (or all but the final chunk) have a size
// equal to (or just under) a multiple of the page size, then in practice,
// each chunk's starting offset will be aligned to a page boundary.
CPageSize uint64
// CChunkSize is the compressed size (i.e. size in CSpace) of each chunk.
// The final chunk might be smaller than this.
//
// This field is ignored if DChunkSize is non-zero.
CChunkSize uint64
// DChunkSize is the uncompressed size (i.e. size in DSpace) of each chunk.
// The final chunk might be smaller than this.
//
// If both CChunkSize and DChunkSize are non-zero, DChunkSize takes
// precedence and CChunkSize is ignored.
//
// If both CChunkSize and DChunkSize are zero, a default DChunkSize value
// will be used.
DChunkSize uint64
// ResourcesData is a list of resources that can be shared across otherwise
// independently compressed chunks.
//
// The exact semantics for a resource depends on the codec. For example,
// the Zlib codec interprets them as shared LZ77-style dictionaries.
//
// One way to generate shared dictionaries from sub-slices of a single
// source file is the command line tool at
// https://github.com/google/brotli/blob/master/research/dictionary_generator.cc
ResourcesData [][]byte
// contains filtered or unexported fields
}
Writer provides a relatively simple way to write a RAC file - one that is created starting from nothing, as opposed to incrementally modifying an existing RAC file.
Other packages may provide a more flexible (and more complicated) way to write or append to RAC files, but that is out of scope of this package.
Do not modify its exported fields after calling any of its methods.
Writer implements the io.WriteCloser interface.
func (*Writer) Close ¶
Close writes the RAC index to w.Writer and marks that w accepts no further method calls.
Calling Close will call Close on w's CodecWriter.
For a one pass encoding, no further action is taken. For a two pass encoding (i.e. IndexLocationAtStart), it then copies w.TempFile to w.Writer. Either way, if this method returns nil error, the entirety of what was written to w.Writer constitutes a valid RAC file.
Closing the underlying w.Writer, w.TempFile or both is the responsibility of the rac.Writer caller, not the rac.Writer itself.
Source Files