encoding

Documentation

Index

• Constants
• func BOMBytesFor(charset string) []byte
• func BOMSize(charset string) int
• func Canonical(name string) (string, bool)
• func Count() int
• func Decode(data []byte, charset string) (string, error)
• func DetectionCandidates() []string
• func Encode(content string, enc encoding.Encoding, charset string) ([]byte, error)
• func FormatCandidates(ranked []Candidate) string
• func Get(name string) (encoding.Encoding, bool)
• func IsUTF8(name string) bool
• func SameCharset(a, b string) bool
• func SetDetectionCandidates(names []string) error
• type Candidate
  • func CandidatesFromFile(path string, mode string) ([]Candidate, error)
  • func CandidatesFromSample(data []byte) []Candidate
  • func SupportedAlternatives(ranked []Candidate, exclude string) []Candidate
• type DetectionResult
  • func Detect(data []byte) DetectionResult
  • func DetectBOM(data []byte) (DetectionResult, bool)
  • func DetectFromFile(path string, mode string) (DetectionResult, error)
  • func DetectSample(data []byte) (DetectionResult, bool)
  • func (d DetectionResult) Conclusive() bool
• type EncodingListItem
  • func ListEncodings() []EncodingListItem
• type UnsupportedError
  • func (e *UnsupportedError) Error() string
• type UnsupportedRune

Constants

const (
	ChunkSize               = 128 * 1024 // 128KB chunks for detection
	SmallFileThreshold      = 128 * 1024 // Files smaller than this are read entirely
	HighConfidenceThreshold = 80         // Confidence level to stop sampling early
	MinConfidenceThreshold  = 50         // Minimum confidence to trust detection

)

Functions

func BOMBytesFor

func BOMBytesFor(charset string) []byte

BOMBytesFor returns the BOM bytes for charset, or nil if unsupported.

func BOMSize

func BOMSize(charset string) int

BOMSize returns the byte length of a BOM for the given charset, or 0 if unknown.

func Canonical

func Canonical(name string) (string, bool)

Canonical resolves a name or alias to the canonical encoding name.

func Count

func Count() int

Count returns the number of supported encodings.

func Decode

func Decode(data []byte, charset string) (string, error)

Decode converts data in charset to UTF-8. A BOM decodes as content; strip it first.

func DetectionCandidates

func DetectionCandidates() []string

DetectionCandidates returns the pinned set, nil when detection is unrestricted.

func Encode

func Encode(content string, enc encoding.Encoding, charset string) ([]byte, error)

Encode converts UTF-8 content to charset; the error is an *UnsupportedError.

func FormatCandidates

func FormatCandidates(ranked []Candidate) string

FormatCandidates renders a ranked list as "windows-1251 (62%), koi8-r (55%)".

func Get

func Get(name string) (encoding.Encoding, bool)

func IsUTF8

func IsUTF8(name string) bool

func SameCharset

func SameCharset(a, b string) bool

SameCharset compares through the registry, so "ascii" and "utf-8" are one answer.

func SetDetectionCandidates

func SetDetectionCandidates(names []string) error

SetDetectionCandidates restricts detection to these encodings; an empty list clears the restriction.

Types

type Candidate

type Candidate struct {
	Charset    string
	Confidence int
	Supported  bool // in the registry, so usable as an encoding parameter
}

Candidate is one guess at what the bytes are, with the detector's confidence.

func CandidatesFromFile

func CandidatesFromFile(path string, mode string) ([]Candidate, error)

CandidatesFromFile ranks over the bytes the detector saw: all in "full" mode, samples otherwise.

func CandidatesFromSample

func CandidatesFromSample(data []byte) []Candidate

CandidatesFromSample ranks over the samples DetectSample reads: one bounded pass.

func SupportedAlternatives

func SupportedAlternatives(ranked []Candidate, exclude string) []Candidate

SupportedAlternatives keeps what a caller could retry with: in the registry, and not `exclude`.

type DetectionResult

type DetectionResult struct {
	Charset    string
	Confidence int
	HasBOM     bool
}

func Detect

func Detect(data []byte) DetectionResult

func DetectBOM

func DetectBOM(data []byte) (DetectionResult, bool)

DetectBOM checks for Unicode BOMs; UTF-32 goes before UTF-16 (shared prefixes).

func DetectFromFile

func DetectFromFile(path string, mode string) (DetectionResult, error)

DetectFromFile detects encoding via streaming I/O; modes: sample (~384KB), chunked, full.

func DetectSample

func DetectSample(data []byte) (DetectionResult, bool)

DetectSample samples beginning, middle and end; reports whether to trust it. TODO: make private once grep and convert_encoding stream instead of buffering.

func (DetectionResult) Conclusive

func (d DetectionResult) Conclusive() bool

Conclusive reports whether the result settles which encoding to use. "ascii" never does: it fits every encoding here.

type EncodingListItem

type EncodingListItem struct {
	Name        string   `json:"name"`
	DisplayName string   `json:"displayName"`
	Aliases     []string `json:"aliases"`
	Description string   `json:"description"`
}

func ListEncodings

func ListEncodings() []EncodingListItem

type UnsupportedError

type UnsupportedError struct {
	Charset string
	Runes   []UnsupportedRune // capped at maxReportedRunes
	Total   int               // may exceed len(Runes)
}

UnsupportedError says which characters charset cannot represent, and where.

func (*UnsupportedError) Error

func (e *UnsupportedError) Error() string

type UnsupportedRune

type UnsupportedRune struct {
	Char   string `json:"char"`
	Code   string `json:"code"` // U+XXXX
	Line   int    `json:"line"`
	Column int    `json:"column"`
}

UnsupportedRune locates one character a target encoding cannot represent.