encoding

Documentation

Overview

package encoding contains all the encoding functions

Index

• Constants
• Variables
• func CharsetsMatch(charsetFound, charsetWanted string) bool
• func Decode(contentBytes []byte) (string, string, error)
• func DecodeBytes(contentBytes []byte) (string, string, error)
• func Detect(contentBytes []byte) (string, float64, string)
• func DetectByBOM(contentBytes []byte) string
• func IsBinary(rawFileContent []byte) bool
• func IsBinaryFile(rawFileContent []byte) bool
• func IsStrictBinary(rawFileContent []byte) bool
• func IsUTF16BE(b []byte) float64
• func IsUTF16LE(b []byte) float64
• func IsUTF32BE(b []byte) float64
• func IsUTF32LE(b []byte) float64
• type UnrecogizedEncodingError
  • func (e *UnrecogizedEncodingError) Error() string

Constants

const (
	// HitMissRatioForUTF1632Checks defines the ratio of hits to misses that must
	// be met for the IsUTF* functions to return true. For example, the number 1.1
	// requires the function find 10% more hits than misses. This number was chosen
	// empiricallly, but may need to be adjusted as we gather more test results.
	HitMissRatioForUTF1632Checks = 1.01

	// HitMissRatioIfBOMFound is returned if a file begins with a BOM
	// (Byte Order Mark).
	HitMissRatioIfBOMFound = 1023.0

	// MinConfidenceForUTF1632Checks defines the minimum confidence factor to use
	// our own check for if a file is UTF16/UTF32 encoded. 100 represents
	// a 100% confidence level, which is the highest confidence returned by our
	// chardet library.
	// testdata/wpt/resources/utf-32-big-endian-nobom.html is an 85.0.
	MinConfidenceForUTF1632Checks = 90.0

	// BinaryData contains the string to return if the data contains binary
	// data (data that is not decodable, by any of the decoders that golang provides).
	BinaryData = "binary"

	// UnknownEncoding is returned if the encoding could not be determined.
	UnknownEncoding = "unknown"

	// See https://spec.editorconfig.org/#supported-pairs
	// CharsetUnset defines the value allowing for file encoding.
	CharsetUnset = "unset"
	// CharsetLatin1 defines the value for the ISO-8859-1 file encoding.
	CharsetLatin1 = "latin1"
	// CharsetUTF8 defines the value for the UTF-8 encoding, without a
	// Byte Order Mark (BOM).
	CharsetUTF8 = "utf-8"
	// CharsetUTF8BOM defines the value for the UTF-8 encoding, with a
	// Byte Order Mark (BOM).
	CharsetUTF8BOM = "utf-8-bom"
	// CharsetUTF16BE defines the value for the UTF-16BE (Big Endian) encoding
	// with, or without, a Byte Order Mark (BOM).
	CharsetUTF16BE = "utf-16be"
	// CharsetUTF16LE defines the value for the UTF-16LE (Lil Endian) encoding
	// with, or without, a Byte Order Mark (BOM).
	CharsetUTF16LE = "utf-16le"

	// CharsetUTF32BE defines the value for the UTF-32BE (Big Endian) encoding
	// with, or without, a Byte Order Mark (BOM).
	CharsetUTF32BE = "utf-32be"
	// CharsetUTF32LE defines the value for the UTF-32LE (Lil Endian) encoding
	// with, or without, a Byte Order Mark (BOM).
	CharsetUTF32LE = "utf-32le"
)

Variables

var (
	// https://en.wikipedia.org/wiki/Byte_order_mark
	UTF32LEBOM   = []byte{0xFF, 0xFE, 0x00, 0x00}
	UTF32BEBOM   = []byte{0x00, 0x00, 0xFE, 0xFF}
	UCS43412BOM  = []byte{0xFE, 0xFF, 0x00, 0x00}
	UCS42143BOM  = []byte{0x00, 0x00, 0xFF, 0xFE}
	UTFEBCDICBOM = []byte{0xDD, 0x73, 0x66, 0x73}
	GB18030BOM   = []byte{0x84, 0x31, 0x95, 0x33}
	UTF8BOM      = []byte{0xEF, 0xBB, 0xBF}
	UTF7BOM      = []byte{0x74, 0x64, 0x4C}
	UTF1BOM      = []byte{0xF7, 0x64, 0x4C}
	SCSUBOM      = []byte{0x0E, 0xFE, 0xFF}
	BOCU1BOM     = []byte{0xFB, 0xEE, 0x28}
	UTF16LEBOM   = []byte{0xFF, 0xFE}
	UTF16BEBOM   = []byte{0xFE, 0xFF}
)

var (
	// ValidCharsets contains an array of the allowable values for the charset key
	// in an .editorconfig file.
	ValidCharsets = []string{
		CharsetLatin1,
		CharsetUTF8,
		CharsetUTF8BOM,
		CharsetUTF16BE,
		CharsetUTF16LE,
	}
)

Functions

func CharsetsMatch

func CharsetsMatch(charsetFound, charsetWanted string) bool

CharsetsMatch checks the charset found is effectively a match for the desired charset. For now, all non-UTF8/16 encodings will match `latin1` (at least until we can improve our success rate on encoding detection). Per https://github.com/editorconfig-checker/editorconfig-checker/pull/457#issuecomment-2779587476

func Decode

func Decode(contentBytes []byte) (string, string, error)

Decode attempts to determine a file's character encoding. If successful, it returns the content as a UTF-8 encoded string, and the name of the encoding. If not, it returns the content as a string, the encoding name, and an error.

func DecodeBytes

func DecodeBytes(contentBytes []byte) (string, string, error)

DecodeBytes is deprecated and may be removed in the future. Use Decode instead.

func Detect

func Detect(contentBytes []byte) (string, float64, string)

Detect returns the character encoding, a confidence level, and the language.

func DetectByBOM

func DetectByBOM(contentBytes []byte) string

DetectByBOM detects the file's encoding solely by BOM (byte order mark).

func IsBinary

func IsBinary(rawFileContent []byte) bool

IsBinary returns true if the bytes contain \x00-\x08,\x0b,\x0e-\x1f .

func IsBinaryFile

func IsBinaryFile(rawFileContent []byte) bool

IsBinaryFile is deprecated and may be removed in the future. Use IsBinary instead.

func IsStrictBinary

func IsStrictBinary(rawFileContent []byte) bool

IsStrictBinary is like IsBinary but does not flag files that only contain ESC (0x1b), SO (0x0e), or SI (0x0f) as their C0 control characters. These are used by ISO-2022 family encodings (ISO-2022-JP, ISO-2022-KR, etc.) and should not cause a file to be treated as binary.

func IsUTF16BE

func IsUTF16BE(b []byte) float64

IsUTF16BE returns a hit/miss ratio to identify if the file is UTF16BE encoded.

func IsUTF16LE

func IsUTF16LE(b []byte) float64

IsUTF16LE returns a hit/miss ratio to identify if the file is UTF16LE encoded.

func IsUTF32BE

func IsUTF32BE(b []byte) float64

IsUTF32BE returns a hit/miss ratio to identify if the file is UTF32BE encoded.

func IsUTF32LE

func IsUTF32LE(b []byte) float64

IsUTF32LE returns hit/miss ratio to identify if the file is UTF32LE encoded.

Types

type UnrecogizedEncodingError

type UnrecogizedEncodingError struct {
	// contains filtered or unexported fields
}

UnrecogizedEncodingError is returned if the encountered a character set we don't have a decoder for.

func (*UnrecogizedEncodingError) Error

func (e *UnrecogizedEncodingError) Error() string