charmap

README

Golang library for fast conversion between 8-bit charsets and UTF-8

This little library allows to convert text from various 8-bit encodings to UTF-8. And to make it fast. The code's using translation tables to convert straight from an 8-bit encoding to UTF-8 without recoding to slice of runes or other redundant steps.

Supported charsets

• CP852 (aka IBM852)
• CP855 (aka IBM855)
• CP866 (aka IBM866)
• CP1250 (aka windows-1250)
• CP1251 (aka windows-1251)
• CP1252 (aka windows-1252)
• ISO-8859-5
• KOI8-R
• KOI8-U

Extensibility

The translation tables are generated with a Python script and thus the list of supported charsets can easily be extended. Just add necessary charset to the list in gen_tables.py and run: ./gen_tables.py > tables.go

Examples

import "github.com/aglyzov/charmap"

var koi8_r = []byte("\xf4\xc5\xd3\xd4\xcf\xd7\xc1\xd1 \xd3\xd4\xd2\xcf\xcb\xc1")
var utf8_1 = charmap.ANY_to_UTF8(koi8_r, "KOI8-R")

// utf8_1 == []byte("Тестовая строка")

var cp1251 = []byte("\xd2\xe5\xf1\xf2\xee\xe2\xe0\xff \xf1\xf2\xf0\xee\xea\xe0")
var utf8_2 = charmap.CP1251_to_UTF8(cp1251)

// utf8_2 == utf8_1

var ibm866 = []byte("\x92\xa5\xe1\xe2\xae\xa2\xa0\xef \xe1\xe2\xe0\xae\xaa\xa0")
var utf8_3 = charmap.ToUTF8(charmap.CP866_UTF8_TABLE, ibm866)

// utf8_3 == utf8_2

Documentation

Index

• Variables
• func ANY_to_UTF8(src []byte, charset string) ([]byte, error)
• func CP1250_to_UTF8(src []byte) []byte
• func CP1251_to_UTF8(src []byte) []byte
• func CP1252_to_UTF8(src []byte) []byte
• func CP852_to_UTF8(src []byte) []byte
• func CP855_to_UTF8(src []byte) []byte
• func CP866_to_UTF8(src []byte) []byte
• func ISO88595_to_UTF8(src []byte) []byte
• func KOI8R_to_UTF8(src []byte) []byte
• func KOI8U_to_UTF8(src []byte) []byte
• func ToUTF8(table *DecodeTable, src []byte) []byte
• type DecodeTable

Variables

var CP1250_UTF8_TABLE = DecodeTable{}/* 256 elements not displayed */

var CP1251_UTF8_TABLE = DecodeTable{}/* 256 elements not displayed */

var CP1252_UTF8_TABLE = DecodeTable{}/* 256 elements not displayed */

var CP852_UTF8_TABLE = DecodeTable{}/* 256 elements not displayed */

var CP855_UTF8_TABLE = DecodeTable{}/* 256 elements not displayed */

var CP866_UTF8_TABLE = DecodeTable{}/* 256 elements not displayed */

var ISO88595_UTF8_TABLE = DecodeTable{}/* 256 elements not displayed */

var KOI8R_UTF8_TABLE = DecodeTable{}/* 256 elements not displayed */

var KOI8U_UTF8_TABLE = DecodeTable{}/* 256 elements not displayed */

var KnownCharsets = map[string](*DecodeTable){
	"CP1250":       &CP1250_UTF8_TABLE,
	"cp1250":       &CP1250_UTF8_TABLE,
	"WINDOWS-1250": &CP1250_UTF8_TABLE,
	"windows-1250": &CP1250_UTF8_TABLE,

	"CP1251":       &CP1251_UTF8_TABLE,
	"cp1251":       &CP1251_UTF8_TABLE,
	"WINDOWS-1251": &CP1251_UTF8_TABLE,
	"windows-1251": &CP1251_UTF8_TABLE,

	"CP1252":       &CP1252_UTF8_TABLE,
	"cp1252":       &CP1252_UTF8_TABLE,
	"WINDOWS-1252": &CP1252_UTF8_TABLE,
	"windows-1252": &CP1252_UTF8_TABLE,

	"CP852":  &CP852_UTF8_TABLE,
	"cp852":  &CP852_UTF8_TABLE,
	"IBM852": &CP852_UTF8_TABLE,
	"ibm852": &CP852_UTF8_TABLE,

	"CP855":  &CP855_UTF8_TABLE,
	"cp855":  &CP855_UTF8_TABLE,
	"IBM855": &CP855_UTF8_TABLE,
	"ibm855": &CP855_UTF8_TABLE,

	"CP866":  &CP866_UTF8_TABLE,
	"cp866":  &CP866_UTF8_TABLE,
	"IBM866": &CP866_UTF8_TABLE,
	"ibm866": &CP866_UTF8_TABLE,

	"KOI8-R": &KOI8R_UTF8_TABLE,
	"KOI8R":  &KOI8R_UTF8_TABLE,
	"koi8-r": &KOI8R_UTF8_TABLE,
	"koi8r":  &KOI8R_UTF8_TABLE,

	"KOI8-U": &KOI8U_UTF8_TABLE,
	"KOI8U":  &KOI8U_UTF8_TABLE,
	"koi8-u": &KOI8U_UTF8_TABLE,
	"koi8u":  &KOI8U_UTF8_TABLE,

	"ISO-8859-5": &ISO88595_UTF8_TABLE,
	"ISO8859-5":  &ISO88595_UTF8_TABLE,
	"iso-8859-5": &ISO88595_UTF8_TABLE,
	"iso8859-5":  &ISO88595_UTF8_TABLE,
}

Functions

func ANY_to_UTF8

func ANY_to_UTF8(src []byte, charset string) ([]byte, error)

func CP1250_to_UTF8

func CP1250_to_UTF8(src []byte) []byte

func CP1251_to_UTF8

func CP1251_to_UTF8(src []byte) []byte

func CP1252_to_UTF8

func CP1252_to_UTF8(src []byte) []byte

func CP852_to_UTF8

func CP852_to_UTF8(src []byte) []byte

func CP855_to_UTF8

func CP855_to_UTF8(src []byte) []byte

func CP866_to_UTF8

func CP866_to_UTF8(src []byte) []byte

func ISO88595_to_UTF8

func ISO88595_to_UTF8(src []byte) []byte

func KOI8R_to_UTF8

func KOI8R_to_UTF8(src []byte) []byte

func KOI8U_to_UTF8

func KOI8U_to_UTF8(src []byte) []byte

func ToUTF8

func ToUTF8(table *DecodeTable, src []byte) []byte

Types

type DecodeTable

type DecodeTable [256]uint32