compression

Documentation

Overview

Package compression provides data compression and decompression utilities for ZapFS storage. It supports multiple algorithms (LZ4, ZSTD, Snappy) with a unified interface.

Index

• Variables
• func Compress(algo Algorithm, data []byte) ([]byte, error)
• func CompressReader(algo Algorithm, r io.Reader) (io.ReadCloser, error)
• func CompressWriter(algo Algorithm, w io.Writer) (io.WriteCloser, error)
• func CompressionRatio(originalSize, compressedSize int) float64
• func Decompress(algo Algorithm, data []byte) ([]byte, error)
• func DecompressReader(algo Algorithm, r io.Reader) (io.ReadCloser, error)
• func RecordCompression(algo Algorithm, originalSize, compressedSize int, skipped bool)
• func RecordDecompression(algo Algorithm, compressedSize, originalSize int)
• type Algorithm
  • func CompressIfBeneficial(algo Algorithm, data []byte) ([]byte, Algorithm, error)
  • func ParseAlgorithm(s string) Algorithm
  • func (a Algorithm) IsValid() bool
  • func (a Algorithm) String() string

Variables

var (
	// CompressionRatioHist tracks compression ratios (original_size / compressed_size)
	CompressionRatioHist = promauto.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "zapfs",
			Subsystem: "compression",
			Name:      "ratio",
			Help:      "Compression ratio (original_size / compressed_size)",
			Buckets:   []float64{1.0, 1.25, 1.5, 2.0, 3.0, 4.0, 5.0, 10.0},
		},
		[]string{"algorithm"},
	)

	// CompressionDuration tracks time spent compressing/decompressing
	CompressionDuration = promauto.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "zapfs",
			Subsystem: "compression",
			Name:      "duration_seconds",
			Help:      "Time spent compressing/decompressing data",
			Buckets:   prometheus.DefBuckets,
		},
		[]string{"algorithm", "operation"},
	)

	// CompressionBytesIn tracks original bytes before compression
	CompressionBytesIn = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "zapfs",
			Subsystem: "compression",
			Name:      "bytes_in_total",
			Help:      "Total bytes before compression (original size)",
		},
		[]string{"algorithm"},
	)

	// CompressionBytesOut tracks compressed bytes after compression
	CompressionBytesOut = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "zapfs",
			Subsystem: "compression",
			Name:      "bytes_out_total",
			Help:      "Total bytes after compression (compressed size)",
		},
		[]string{"algorithm"},
	)

	// CompressionSkipped tracks chunks where compression was skipped
	CompressionSkipped = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "zapfs",
			Subsystem: "compression",
			Name:      "skipped_total",
			Help:      "Chunks where compression was skipped (no space savings)",
		},
		[]string{"algorithm"},
	)

	// DecompressionBytesIn tracks compressed bytes read for decompression
	DecompressionBytesIn = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "zapfs",
			Subsystem: "decompression",
			Name:      "bytes_in_total",
			Help:      "Total compressed bytes read for decompression",
		},
		[]string{"algorithm"},
	)

	// DecompressionBytesOut tracks decompressed bytes output
	DecompressionBytesOut = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "zapfs",
			Subsystem: "decompression",
			Name:      "bytes_out_total",
			Help:      "Total bytes after decompression (original size)",
		},
		[]string{"algorithm"},
	)
)

Functions

func Compress

func Compress(algo Algorithm, data []byte) ([]byte, error)

Compress compresses data using the specified algorithm. Returns the original data unchanged if algo is None or empty.

func CompressReader

func CompressReader(algo Algorithm, r io.Reader) (io.ReadCloser, error)

CompressReader wraps a reader to compress data as it's read. The returned ReadCloser must be closed when done.

func CompressWriter

func CompressWriter(algo Algorithm, w io.Writer) (io.WriteCloser, error)

CompressWriter wraps a writer to compress data as it's written. The returned WriteCloser must be closed when done to flush remaining data.

func CompressionRatio

func CompressionRatio(originalSize, compressedSize int) float64

CompressionRatio calculates the compression ratio (original / compressed). Returns 1.0 if compressed size is zero or larger than original.

func Decompress

func Decompress(algo Algorithm, data []byte) ([]byte, error)

Decompress decompresses data using the specified algorithm. Returns the original data unchanged if algo is None or empty.

func DecompressReader

func DecompressReader(algo Algorithm, r io.Reader) (io.ReadCloser, error)

DecompressReader wraps a reader to decompress data as it's read. The returned ReadCloser must be closed when done.

func RecordCompression

func RecordCompression(algo Algorithm, originalSize, compressedSize int, skipped bool)

RecordCompression records metrics for a compression operation

func RecordDecompression

func RecordDecompression(algo Algorithm, compressedSize, originalSize int)

RecordDecompression records metrics for a decompression operation

Types

type Algorithm

type Algorithm string

Algorithm represents a compression algorithm

const (
	// None indicates no compression
	None Algorithm = "none"
	// LZ4 uses the LZ4 compression algorithm (fast, moderate ratio)
	LZ4 Algorithm = "lz4"
	// ZSTD uses the Zstandard compression algorithm (balanced speed/ratio)
	ZSTD Algorithm = "zstd"
	// S2 uses klauspost's S2 compression (faster than Snappy, better ratio)
	S2 Algorithm = "s2"
)

func CompressIfBeneficial

func CompressIfBeneficial(algo Algorithm, data []byte) ([]byte, Algorithm, error)

CompressIfBeneficial compresses data and returns the compressed version only if it's smaller than the original. Otherwise returns the original data and None algorithm.

func ParseAlgorithm

func ParseAlgorithm(s string) Algorithm

ParseAlgorithm parses a string into an Algorithm. Returns None for empty or unrecognized strings.

func (Algorithm) IsValid

func (a Algorithm) IsValid() bool

IsValid returns true if the algorithm is recognized

func (Algorithm) String

func (a Algorithm) String() string

String returns the string representation of the algorithm