qatzip

package
v1.1.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 7, 2024 License: BSD-3-Clause Imports: 12 Imported by: 0

Documentation

Overview

Package qatzip implements Go bindings for the Intel® Quick Assist Technology compression library

Example (WriterReader) 
package main

import (
	"bytes"
	"io"
	"log"
	"os"

	"github.com/intel/qatgo/qatzip"
)

func main() {
	// This is an example of compressing and decompressing using QATgo
	// NewWriter/NewReader provides io.Reader and io.Writer implementations compatible with Go standard library
	buf := new(bytes.Buffer) // contains compressed data
	var str = "this string will be compressed and then decompressed"

	zw := qatzip.NewWriter(buf) // start a compression session with buf as output

	err := zw.Apply(
		qatzip.AlgorithmOption(qatzip.DEFLATE),         // set DEFLATE algorithm (lz4 and zstd are supported as well)
		qatzip.DeflateFmtOption(qatzip.DeflateGzipExt), // set format to gzip extended header (for DEFLATE algorithm only)
		qatzip.CompressionLevelOption(6),               // set compression level 6
	)
	if err != nil {
		log.Fatal("error: could not apply compression options:", err)
	}

	// compress input string and output to buf
	if _, err = zw.Write([]byte(str)); err != nil {
		log.Fatal("error: could not compress input:", err)
	}

	// end compression session: flush remaining input buffer and release resources
	if err = zw.Close(); err != nil {
		log.Fatal("error: could not end compression session:", err)
	}

	zr, err := qatzip.NewReader(buf) // start a decompression session with buf as input
	if err != nil {
		log.Fatal("error: could not start decompression session:", err)
	}

	err = zr.Apply(
		qatzip.AlgorithmOption(qatzip.DEFLATE),         // enable DEFLATE
		qatzip.DeflateFmtOption(qatzip.DeflateGzipExt), // enable gzip extended header
	)
	if err != nil {
		log.Fatal("error: could not apply decompression options:", err)
	}

	// decompress input stream and write to stdout
	if _, err = io.Copy(os.Stdout, zr); err != nil {
		log.Fatal("error: could not decompress input:", err)
	}

	// end decompression session: release accelerator resources
	if err = zr.Close(); err != nil {
		log.Fatal("error: could not end decompression session:", err)
	}

}

Output:

this string will be compressed and then decompressed
Example (WriterReaderDirect) 
package main

import (
	"bytes"
	"fmt"
	"log"
	"strings"

	"github.com/intel/qatgo/qatzip"
)

func main() {
	// This is an example of compressing and decompressing using QATgo's direct methods
	// Direct methods provide direct access to low-level QAT APIs without any high-level QATgo buffer management
	// The methods only supports byte slices for I/O
	// The decompressed data buffer in this example is too small requiring the buffer to be resized

	var str = strings.Repeat("string repeats..", 64)
	cbuf := make([]byte, len(str)+512) // contains compressed data
	dbuf := make([]byte, 512)          // contains decompressed data (will need resizing)

	q, err := qatzip.NewQzBinding() // instantiate direct session object
	if err != nil {
		log.Fatal("error: could not instantiate direct compression session:", err)
	}

	err = q.Apply(
		qatzip.AlgorithmOption(qatzip.DEFLATE),         // enable DEFLATE (lz4 and zstd are supported as well)
		qatzip.DeflateFmtOption(qatzip.DeflateGzipExt), // enable gzip extended header (for DEFLATE algorithm only)
		qatzip.DirOption(qatzip.Compress),              // set direction to compress
		qatzip.CompressionLevelOption(6),               // set compression level 6
	)
	if err != nil {
		log.Fatal("error: could not apply compression options:", err)
	}

	// start new compression session
	if err = q.StartSession(); err != nil {
		log.Fatal("error: could not start compression session:", err)
	}

	// Only one buffer is being compressed, so mark this as the last buffer
	q.SetLast(true)

	// compress input string and output to cbuf, nc is bytes output during compression
	_, nc, err := q.Compress([]byte(str), cbuf)
	if err != nil {
		log.Fatal("error: could not compress input data:", err)
	}

	// end compression session and release resources
	if err = q.Close(); err != nil {
		log.Fatal("error: could not end input session:", err)
	}

	q, err = qatzip.NewQzBinding() // instantiate a new direct session object
	if err != nil {
		log.Fatal("error: could not instatiate direct decompression session:", err)
	}

	err = q.Apply(
		qatzip.AlgorithmOption(qatzip.DEFLATE),         // enable DEFLATE (must match compress)
		qatzip.DeflateFmtOption(qatzip.DeflateGzipExt), // enable gzip extended header
		qatzip.DirOption(qatzip.Decompress),            // set direction to decompress
	)
	if err != nil {
		log.Fatal("error: could not apply compression options:", err)
	}

	// start new compression session
	if err = q.StartSession(); err != nil {
		log.Fatal("error: could not start direct decompression session:", err)
	}

	// decompress input stream and write to stdout, nd = bytes output during decompress
	// in this example the size of the output buffer is too small and will need to be grown
	var nd int
	for {
		if _, nd, err = q.Decompress(cbuf[:nc], dbuf); err != nil {
			// if output buffer is not long enough grow output buffer
			// ErrBuffer means there is insufficient buffer space for output
			if err == qatzip.ErrBuffer {
				dbuf = make([]byte, len(dbuf)*2) // double buffer size and try again
				continue
			}

			log.Fatal("error: could not decompress input data:", err)
		}
		// decompression was successful
		break
	}

	// end decompression session and release accelerator resources
	if err = q.Close(); err != nil {
		log.Fatal("error: could not end decompression session:", err)
	}

	// verify that the decompressed output matches the original input
	if bytes.Equal(dbuf[:nd], []byte(str)) {
		fmt.Println("strings match")
	} else {
		fmt.Println("strings do not match")
	}

}

Output:

strings match

Index

Examples

Constants

View Source 
const (
	DEFLATE_ID uint8 = C.QZ_DEFLATE
	LZ4_ID     uint8 = C.QZ_LZ4
)
View Source 
const (
	DefaultCompression        = 1
	MinBufferLength           = 128 * 1024
	DefaultMaxBufferLength    = 2 * 1024 * 1024 * 1024
	DefaultBufferLength       = 128 * 1024 * 1024
	DefaultBufferGrowth       = 128 * 1024 * 1024
	DefaultBounceBufferLength = 512
	MinBounceBufferLength     = 512
)

Variables

View Source 
var (
	QatErrHdr                  = string("QATzip error: ")
	ErrParams                  = errors.New(QatErrHdr + "invalid parameter in function call")
	ErrFail                    = errors.New(QatErrHdr + "unspecified error")
	ErrBuffer                  = errors.New(QatErrHdr + "insufficient buffer error")
	ErrData                    = errors.New(QatErrHdr + "input data was corrupted")
	ErrTimeout                 = errors.New(QatErrHdr + "operation timed out")
	ErrIntegrity               = errors.New(QatErrHdr + "integrity checked failed")
	ErrNoHw                    = errors.New(QatErrHdr + "using SW: No QAT HW detected")
	ErrNoMDrv                  = errors.New(QatErrHdr + "using SW: No memory driver detected")
	ErrNoInstAttached          = errors.New(QatErrHdr + "using SW: Could not attach to an instance")
	ErrLowMem                  = errors.New(QatErrHdr + "using SW: Not enough pinned memory")
	ErrLowDestMem              = errors.New(QatErrHdr + "using SW: Not enough pinned memory for dest buffer")
	ErrUnsupportedFmt          = errors.New(QatErrHdr + "using SW: QAT device does not support data format")
	ErrNone                    = errors.New(QatErrHdr + "device uninitialized")
	ErrNoSwHw                  = errors.New(QatErrHdr + "not using SW: No QAT HW detected")
	ErrNoSwMDrv                = errors.New(QatErrHdr + "not using SW: No memory driver detected")
	ErrNoSwNoInst              = errors.New(QatErrHdr + "not using SW: Could not attach to instance")
	ErrNoSwLowMem              = errors.New(QatErrHdr + "not using SW: not enough pinned memory")
	ErrNoSwAvail               = errors.New(QatErrHdr + "session may require software, but no software is available")
	ErrNoSwUnsupportedFmt      = errors.New(QatErrHdr + "not using SW: QAT device does not support data format")
	ErrPostProcess             = errors.New(QatErrHdr + "using post process: post process callback encountered an error")
	ErrMetaDataOverflow        = errors.New(QatErrHdr + "insufficent memory allocated for metadata")
	ErrOutOfRange              = errors.New(QatErrHdr + "metadata block_num specified is out of range")
	ErrNotSupported            = errors.New(QatErrHdr + "request not supported")
	ErrParamOutputBufLength    = errors.New(QatErrHdr + "invalid size for output buffer length")
	ErrParamInputBufLength     = errors.New(QatErrHdr + "invalid size for input buffer length")
	ErrParamBufferGrowth       = errors.New(QatErrHdr + "invalid size for buffer Growth")
	ErrParamBounceBufferLength = errors.New(QatErrHdr + "invalid size for bounce buffer length")
	ErrParamAlgorithm          = errors.New(QatErrHdr + "invalid algorithm type")
	ErrParamDirection          = errors.New(QatErrHdr + "invalid direction")
	ErrParamDataFmtDeflate     = errors.New(QatErrHdr + "invalid deflate format type")
	ErrParamHuffmanHdr         = errors.New(QatErrHdr + "invalid huffman header type")
	ErrParamPollingMode        = errors.New(QatErrHdr + "invalid polling mode")
	ErrParamCompressionLevel   = errors.New(QatErrHdr + "invalid compression level")
	ErrWriterClosed            = errors.New(QatErrHdr + "cannot write to closed writer")
	ErrEmptyBuffer             = errors.New(QatErrHdr + "empty buffer")
	ErrNoMem                   = errors.New(QatErrHdr + "out of memory")
	ErrInputBufferMode         = errors.New(QatErrHdr + "invalid input buffer mode")
	ErrApplyPostInit           = errors.New(QatErrHdr + "cannot apply options after Reset() or I/O")
	ErrApplyInvalidType        = errors.New(QatErrHdr + "option appied to incorrect type")
	ErrMaxBufferGrowth         = errors.New(QatErrHdr + "cannot grow buffer large enough")
	ErrParamMaxBufferLength    = errors.New(QatErrHdr + "invalid size for max buffer length")
)

Functions

func Error 

func Error(errorCode int) (err error)

Types

type Algorithm 

type Algorithm int
const (
	DEFLATE Algorithm = iota
	LZ4
	ZSTD
)

type DebugLevel 

type DebugLevel int
const (
	None DebugLevel = iota
	Low
	Med
	High
	Debug
)

type DeflateFmt 

type DeflateFmt int
const (
	/* QzDataFormat_T */
	Deflate48 DeflateFmt = iota
	DeflateGzip
	DeflateGzipExt
	DeflateRaw
)

The following enums must exactly match the equivalent Enum type in QATzip.h

type Direction 

type Direction int
const (
	/* QzDirection_T */
	Compress Direction = iota
	Decompress
	Both
)

type HuffmanHdr 

type HuffmanHdr int
const (
	/* QzHuffmanHdr_T */
	Dynamic HuffmanHdr = iota
	Static
)

type InputBufferMode 

type InputBufferMode int
const (
	Reserve InputBufferMode = iota // Reserve a portion of the input buffer for last
	Bounce                         // Bounce all buffers
	Last                           // Force last=true for all buffers
	NoLast                         // Force last=false for all buffers
)

type Option 

type Option func(a applier) error

func AlgorithmOption 

func AlgorithmOption(alg Algorithm) Option

Algorithm setting [DEFLATE, lz4, zstd]

func BounceBufferLengthOption 

func BounceBufferLengthOption(len int) Option

Length of the bounce buffer (Writer)

func BufferGrowthOption 

func BufferGrowthOption(len int) Option

If output buffer is too small (see QZ_BUF_ERROR) increase size of output buffer by len and retry (Reader/Writer)

func CompressionLevelOption 

func CompressionLevelOption(level int) Option

Compression level setting (Writer)

func DebugLevelOption 

func DebugLevelOption(level DebugLevel) Option

Debug level option [None, Low, Med, High, Debug]

func DeflateFmtOption 

func DeflateFmtOption(fmt DeflateFmt) Option

DEFLATE format setting [raw, gzip, gzip extended]

func DirOption 

func DirOption(dir Direction) Option

I/O direction setting [compress, decompress, both]

func HuffmanHdrOption 

func HuffmanHdrOption(hdrType HuffmanHdr) Option

Dynamic or static Huffman headers setting

func HwBufSizeOption 

func HwBufSizeOption(size int) Option

Hardware buffer size [must be a power of 2KB]

func InputBufLengthOption 

func InputBufLengthOption(len int) Option

Input buffer length (Reader)

func InputBufferModeOption 

func InputBufferModeOption(mode InputBufferMode) Option

Input buffer mode setting (Writer) [Reserve, Bounce, Last, NoLast]

func MaxBufferLengthOption added in v1.1.0 

func MaxBufferLengthOption(n int) Option

Max size an output buffer can grow to in bytes

func MaxForksOption 

func MaxForksOption(max int) Option

Maximum forks permitted in the current thread

func OutputBufLengthOption 

func OutputBufLengthOption(len int) Option

Output buffer size (Reader, Writer)

func PollingModeOption 

func PollingModeOption(mode PollingMode) Option

Polling mode setting [periodical, busy]

func ReqCountThresholdOption 

func ReqCountThresholdOption(n int) Option

Threshold for how many buffer requests QATzip can make on a single thread

func SensitiveOption 

func SensitiveOption(enable bool) Option

Sensitve mode option

func StreamBufSizeOption 

func StreamBufSizeOption(size int) Option

Stream buffer size between [1KB .. 2MB - 5KB]

func SwBackupOption 

func SwBackupOption(enable bool) Option

Software fallback option

func SwSwitchThresholdOption 

func SwSwitchThresholdOption(size int) Option

If input size is less than threshold size switch to SW

func WaitCountThresholdOption 

func WaitCountThresholdOption(n int) Option

Retry count for QATzip initialization if failure occurs

type Perf 

type Perf struct {
	ReadTimeNS   uint64 // time (ns) spent reading from r
	WriteTimeNS  uint64 // time (ns) spent writing to w
	BytesIn      uint64 // bytes sent to QATzip
	BytesOut     uint64 // bytes received from QATzip
	EngineTimeNS uint64 // time (ns) spent in QATzip
	CopyTimeNS   uint64 // time (ns) spent copying buffers + reallocation
}

Performance counters

type PollingMode 

type PollingMode int
const (
	/* QzPollingMode_T */
	Periodical PollingMode = iota
	Busy
)

type QzBinding 

type QzBinding struct {
	// contains filtered or unexported fields
}

Manages QATzip session state

func NewQzBinding 

func NewQzBinding() (q *QzBinding, err error)

Create QATzip session state

func (*QzBinding) Apply 

func (q *QzBinding) Apply(options ...Option) (err error)

Apply options to QATzip session state

func (*QzBinding) Close 

func (q *QzBinding) Close() (err error)

End QATzip session

func (*QzBinding) Compress 

func (q *QzBinding) Compress(in []byte, out []byte) (c int, p int, err error)

QATzip compress (in = input buffer, out = output buffer, c = consumed, p = produced)

func (*QzBinding) CompressCRC 

func (q *QzBinding) CompressCRC(in []byte, out []byte, crc *uint64) (c int, p int, err error)

QATzip compress with CRC (in = input buffer, out = output buffer, crc = crc value, c = consumed, p = produced)

func (*QzBinding) Decompress 

func (q *QzBinding) Decompress(in []byte, out []byte) (c int, p int, err error)

QATzip decompress (in = input buffer, out = output buffer, c = consumed, p = produced)

func (*QzBinding) SetLast 

func (q *QzBinding) SetLast(enable bool)

Set last flag for QATzip compress

func (*QzBinding) StartSession 

func (q *QzBinding) StartSession() (err error)

Start QATzip session

type Reader 

type Reader struct {
	// contains filtered or unexported fields
}

Reader implements an io.Reader. When read from, it decompresses content from r.

func NewReader 

func NewReader(r io.Reader) (*Reader, error)

NewReader creates a new Reader with input io.Reader r

func (*Reader) Apply 

func (z *Reader) Apply(options ...Option) (err error)

Apply options to Reader

func (*Reader) Close 

func (z *Reader) Close() error

Close closes Reader

func (*Reader) GetPerf 

func (z *Reader) GetPerf() Perf

Get performance counters from Reader

func (*Reader) Read 

func (z *Reader) Read(p []byte) (n int, err error)

Read() reads compressed data from io.Reader r and outputs decompressed data to p.

func (*Reader) Reset 

func (z *Reader) Reset(r io.Reader) error

Reset discards current state, loads applied options, and restarts session

type Writer 

type Writer struct {
	// contains filtered or unexported fields
}

Writer implements an io.Writer. When written to, it sends compressed content to w.

func NewWriter 

func NewWriter(w io.Writer) *Writer

NewWriter creates a new Writer with output io.Writer w

func NewWriterLevel 

func NewWriterLevel(w io.Writer, level int) (z *Writer, err error)

NewWriterLevel creates a new Writer with an additional compression level setting

func (*Writer) Apply 

func (z *Writer) Apply(options ...Option) (err error)

Apply options to Writer

func (*Writer) Close 

func (z *Writer) Close() (err error)

Close closes Writer and flushes remaining data

func (*Writer) GetPerf 

func (z *Writer) GetPerf() Perf

Get performance counters from Writer

func (*Writer) Reset 

func (z *Writer) Reset(w io.Writer) (err error)

Reset discards current state, loads applied options, and restarts session

func (*Writer) Write 

func (z *Writer) Write(p []byte) (n int, err error)

Write() inputs data from p and writes compressed output data io.Writer w

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.