compress

Documentation

Overview

Package compress contains some useful tools to compress/decompress data or files

Index

• func AddFileToZip(zipWriter *zip.Writer, filename, basedir string) error
• func GzCompress(in io.Reader, out io.Writer) error
• func GzDecompress(in io.Reader, out io.Writer, opts ...GzDecompressOption) error
• func Unzip(src string, dest string, opts ...UnzipOption) (filenames []string, err error)
• func ZipFiles(output string, files []string) (err error)
• type Compressor
• type GzDecompressOption
  • func WithGzDecompressMaxBytes(bytes int64) GzDecompressOption
• type Gzip
  • func NewGZip(writer io.Writer, opts ...Option) (*Gzip, error)
  • func (c *Gzip) Flush() (err error)
  • func (c *Gzip) Write(d []byte) (int, error)
  • func (c *Gzip) WriteFooter() (err error)
  • func (c *Gzip) WriteString(d string) (int, error)
• type Option
  • func WithBufSizeByte(n int) Option
  • func WithLevel(n int) Option
  • func WithPGzipBlockSize(bytes int) Option
  • func WithPGzipNBlocks(nBlock int) Option
• type PGZip
  • func NewPGZip(writer io.Writer, opts ...Option) (*PGZip, error)
  • func (c *PGZip) Flush() (err error)
  • func (c *PGZip) Write(d []byte) (int, error)
  • func (c *PGZip) WriteFooter() (err error)
  • func (c *PGZip) WriteString(d string) (int, error)
• type UnzipOption
  • func UnzipWithCopyChunkBytes(bytes int64) UnzipOption
  • func UnzipWithMaxBytes(bytes int64) UnzipOption

Examples

• NewGZip
• PGZip

Functions

func AddFileToZip

func AddFileToZip(zipWriter *zip.Writer, filename, basedir string) error

AddFileToZip add file tp zip.Writer

https://golangcode.com/create-zip-files-in-go/

func GzCompress

func GzCompress(in io.Reader, out io.Writer) error

GzCompress compress data by gzip

func GzDecompress

func GzDecompress(in io.Reader, out io.Writer, opts ...GzDecompressOption) error

GzDecompress decompress data by gzip

be careful about the decompression bomb, see https://en.wikipedia.org/wiki/Zip_bomb, default maxBytes is 1GB, it's better to set this value to avoid decompression bomb.

func Unzip

func Unzip(src string, dest string, opts ...UnzipOption) (filenames []string, err error)

Unzip will decompress a zip archive, moving all files and folders within the zip file (parameter 1) to an output directory (parameter 2).

inspired by https://golangcode.com/unzip-files-in-go/

Args:

• src: is the source zip file.
• dest: is the destination directory.
• (opt) UnzipWithMaxBytes: decompressed bytes will not exceed this limit, default/0 is unlimit. it's better to set this value to avoid decompression bomb.
• (opt) UnzipWithCopyChunkBytes: copy chunk by chunk from src to dst

Returns:

• filenames: all filenames in zip file

func ZipFiles

func ZipFiles(output string, files []string) (err error)

ZipFiles compresses one or many files into a single zip archive file.

Args:

• output: is the output zip file's name.
• files: is a list of files to add to the zip. files can be directory.

https://golangcode.com/create-zip-files-in-go/

Types

type Compressor

type Compressor interface {
	Write([]byte) (int, error)
	WriteString(string) (int, error)
	// write footer and flust to lower writer
	Flush() error
	// write footer without flush
	WriteFooter() error
}

Compressor interface of compressor

type GzDecompressOption

type GzDecompressOption func(*gzDecompressOption) error

GzDecompressOption optional arguments for GzDecompress

func WithGzDecompressMaxBytes

func WithGzDecompressMaxBytes(bytes int64) GzDecompressOption

WithGzDecompressMaxBytes decompressed bytes will not exceed this limit,

default is 1GB, it's better to set this value to avoid decompression bomb. set 0 to unlimit.

type Gzip

type Gzip struct {
	// contains filtered or unexported fields
}

GZCompressor compress by gz with buf

func NewGZip

func NewGZip(writer io.Writer, opts ...Option) (*Gzip, error)

NewGZip create new GZCompressor

Example

originText := testCompressraw
writer := &bytes.Buffer{}

var err error
// writer
c, err := NewGZip(
	writer,
	WithLevel(defaultGzipLevel),         // default
	WithBufSizeByte(defaultBufSizeByte), // default
)
if err != nil {
	log.Shared.Error("new compressor", zap.Error(err))
	return
}
if _, err = c.WriteString(originText); err != nil {
	log.Shared.Error("write string to compressor", zap.Error(err))
	return
}
if err = c.Flush(); err != nil {
	log.Shared.Error("flush compressor", zap.Error(err))
	return
}

// reader
var gz *gzip.Reader
if gz, err = gzip.NewReader(writer); err != nil {
	log.Shared.Error("new compressor", zap.Error(err))
	return
}

var bs []byte
if bs, err = io.ReadAll(gz); err != nil {
	log.Shared.Error("read from compressor", zap.Error(err))
	return
}

got := string(bs)
if got != originText {
	log.Shared.Error("extract compressed text invalidate",
		zap.String("got", got),
		zap.ByteString("expect", bs))
	return
}

func (*Gzip) Flush

func (c *Gzip) Flush() (err error)

Flush flush buffer bytes into bottom writer with gz meta footer

func (*Gzip) Write

func (c *Gzip) Write(d []byte) (int, error)

Write write bytes via compressor

func (*Gzip) WriteFooter

func (c *Gzip) WriteFooter() (err error)

WriteFooter write gz footer

func (*Gzip) WriteString

func (c *Gzip) WriteString(d string) (int, error)

WriteString write string via compressor

type Option

type Option func(*option) error

CompressOptFunc options for compressor

func WithBufSizeByte

func WithBufSizeByte(n int) Option

WithBufSizeByte set compressor buf size

func WithLevel

func WithLevel(n int) Option

WithLevel set compressor compress level

func WithPGzipBlockSize

func WithPGzipBlockSize(bytes int) Option

WithPGzipBlockSize set compressor blocks

func WithPGzipNBlocks

func WithPGzipNBlocks(nBlock int) Option

WithPGzipNBlocks set compressor blocks

type PGZip

type PGZip struct {
	// contains filtered or unexported fields
}

PGZip parallel gzip compressor

call `NewPGZip` to create new PGZip

Example

originText := testCompressraw
writer := &bytes.Buffer{}

var err error
// writer
c, err := NewPGZip(writer)
if err != nil {
	log.Shared.Error("new compressor", zap.Error(err))
	return
}
if _, err = c.WriteString(originText); err != nil {
	log.Shared.Error("write string to compressor", zap.Error(err))
	return
}
if err = c.Flush(); err != nil {
	log.Shared.Error("flush compressor", zap.Error(err))
	return
}

// reader
var gz *gzip.Reader
if gz, err = gzip.NewReader(writer); err != nil {
	log.Shared.Error("new compressor", zap.Error(err))
	return
}

var bs []byte
if bs, err = io.ReadAll(gz); err != nil {
	log.Shared.Error("read from compressor", zap.Error(err))
	return
}

got := string(bs)
if got != originText {
	log.Shared.Error("extract compressed text invalidate",
		zap.String("got", got),
		zap.ByteString("expect", bs))
	return
}

func NewPGZip

func NewPGZip(writer io.Writer, opts ...Option) (*PGZip, error)

NewPGZip create new PGZCompressor

func (*PGZip) Flush

func (c *PGZip) Flush() (err error)

Flush flush buffer bytes into bottom writer with gz meta footer

func (*PGZip) Write

func (c *PGZip) Write(d []byte) (int, error)

Write write bytes via compressor

func (*PGZip) WriteFooter

func (c *PGZip) WriteFooter() (err error)

WriteFooter write gz footer

func (*PGZip) WriteString

func (c *PGZip) WriteString(d string) (int, error)

WriteString write string via compressor

type UnzipOption

type UnzipOption func(*unzipOption) error

UnzipOption optional arguments for UnZip

func UnzipWithCopyChunkBytes

func UnzipWithCopyChunkBytes(bytes int64) UnzipOption

UnzipWithCopyChunkBytes copy chunk by chunk from src to dst

func UnzipWithMaxBytes

func UnzipWithMaxBytes(bytes int64) UnzipOption

UnzipWithMaxBytes decompressed bytes will not exceed this limit, default/0 is unlimit.