utf8s

README

go-utf8s

Package utf8s provides tools for working with Unicode encoded as UTF-8, for the Go programming language.

Documention

Online documentation, which includes examples, can be found at: http://godoc.org/github.com/reiver/go-utf8s

Example

var reader io.Reader

// ...

r, n, err := utf8s.ReadRune(reader)

var writer io.Writer

// ...

var r rune

// ...

n, err := utf8s.WriteRune(w, r)

var reader io.Reader

// ...

runeReader := utf8s.NewRuneReader(reader)

// ...

r, n, err := runeReader.ReadRune()

var reader io.Reader

// ...

runeScanner := utf8s.NewRuneScanner(reader)

// ...

r, n, err := runeScanner.ReadRune()

// ...

err = runeScanner.UnreadRune()

Documentation

Overview

Package utf8s provides tools for working with Unicode encoded as UTF-8.

Example

var reader io.Reader

// ...

r, n, err := utf8s.ReadRune(reader)

Example

var writer io.Writer

// ...

var r rune

// ...

n, err := utf8s.WriteRune(w, r)

Example

var reader io.Reader

// ...

runeReader := utf8s.NewRuneReader(reader)

// ..

r, n, err := runeReader.ReadRune()

Example

var reader io.Reader

// ...

runeScanner := utf8s.NewRuneScanner(reader)

// ...

r, n, err := runeScanner.ReadRune()

// ...

err = runeScanner.UnreadRune()

Example

var r rune

// ...

s := utfs.FormatBinary(r)

fmt.Printf("%q encoded in UTF-8 as binary is %s \n", r, s)

UTF-8

UTF-8 is a variable length encoding of Unicode. An encoding of a single Unicode code point can be from 1 to 4 bytes longs.

Some examples of UTF-8 encoding of Unicode code points are:

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃                    UTF-8 encoding                     ┃       ┃            ┃         ┃                            ┃                                  ┃
┣━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┫       ┃            ┃         ┃                            ┃                                  ┃
┃    byte 1   ┋    byte 2   ┋    byte 3   ┋    byte 4   ┃ value ┃ code point ┃ decimal ┃           binary           ┃               name               ┃
┡━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
│ 0b0,1000001 ┊             ┊             ┊             │   A   │     U+0041 │      65 │      0b0000,0000,0100,0001 │ LATIN CAPITAL LETTER A           │
├─────────────┼─────────────┼─────────────┼─────────────┼───────┼────────────┼─────────┼────────────────────────────┼──────────────────────────────────┤
│ 0b0,1110010 ┊             ┊             ┊             │   r   │     U+0072 │     114 │      0b0000,0000,0111,0010 │ LATIN SMALL LETTER R             │
├─────────────┼─────────────┼─────────────┼─────────────┼───────┼────────────┼─────────┼────────────────────────────┼──────────────────────────────────┤
│ 0b110,00010 ┊ 0b10,100001 ┊             ┊             │   ¡   │     U+00A1 │     161 │      0b0000,0000,1010,0001 │ INVERTED EXCLAMATION MARK        │
├─────────────┼─────────────┼─────────────┼─────────────┼───────┼────────────┼─────────┼────────────────────────────┼──────────────────────────────────┤
│ 0b110,11011 ┊ 0b10,110101 ┊             ┊             │   ۵   │     U+06F5 │    1781 │      0b0000,0110,1111,0101 │ EXTENDED ARABIC-INDIC DIGIT FIVE │
├─────────────┼─────────────┼─────────────┼─────────────┼───────┼────────────┼─────────┼────────────────────────────┼──────────────────────────────────┤
│ 0b1110,0010 ┊ 0b10,000000 ┊ 0b10,110001 ┊             │   ‱   │     U+2031 │    8241 │      0b0010,0000,0011,0001 │ PER TEN THOUSAND SIGN            │
├─────────────┼─────────────┼─────────────┼─────────────┼───────┼────────────┼─────────┼────────────────────────────┼──────────────────────────────────┤
│ 0b1110,0010 ┊ 0b10,001001 ┊ 0b10,100001 ┊             │   ≡   │     U+2261 │    8801 │      0b0010,0010,0110,0001 │ IDENTICAL TO                     │
├─────────────┼─────────────┼─────────────┼─────────────┼───────┼────────────┼─────────┼────────────────────────────┼──────────────────────────────────┤
│ 0b11110,000 ┊ 0b10,010000 ┊ 0b10,001111 ┊ 0b10,010101 │   𐏕   │ U+000103D5 │   66517 │ 0b0001,0000,0011,1101,0101 │ OLD PERSIAN NUMBER HUNDRED       │
├─────────────┼─────────────┼─────────────┼─────────────┼───────┼────────────┼─────────┼────────────────────────────┼──────────────────────────────────┤
│ 0b11110,000 ┊ 0b10,011111 ┊ 0b10,011001 ┊ 0b10,000010 │   🙂   │ U+0001F642 │  128578 │ 0b0001,1111,0110,0100,0010 │ SLIGHTLY SMILING FACE            │
└─────────────┴─────────────┴─────────────┴─────────────┴───────┴────────────┴─────────┴────────────────────────────┴──────────────────────────────────┘

UTF-8 Versus ASCII

UTF-8 was (partially) designed to be backwards compatible with 7-bit ASCII.

Thus, all 7-bit ASCII is valid UTF-8.

UTF-8 Encoding

Since, at least as of 2003, Unicode fit into 21 bits, UTF-8 was designed to support at most 21 bits of information.

This is done as described in the following table:

┏━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━┓
┃ # of bytes ┃ # bits for code point ┃ 1st code point ┃  last code point ┃  byte 1  ┃  byte 2  ┃  byte 3  ┃  byte 4  ┃
┡━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━┩
│     1      │            7          │    U+000000    │     U+00007F     │ 0xxxxxxx │          │          │          │
├────────────┼───────────────────────┼────────────────┼──────────────────┼──────────┼──────────┼──────────┼──────────┤
│     2      │           11          │    U+000080    │     U+0007FF     │ 110xxxxx │ 10xxxxxx │          │          │
├────────────┼───────────────────────┼────────────────┼──────────────────┼──────────┼──────────┼──────────┼──────────┤
│     3      │           16          │    U+000800    │     U+00FFFF     │ 1110xxxx │ 10xxxxxx │ 10xxxxxx │          │
├────────────┼───────────────────────┼────────────────┼──────────────────┼──────────┼──────────┼──────────┼──────────┤
│     4      │           21          │    U+010000    │     U+10FFFF     │ 11110xxx │ 10xxxxxx │ 10xxxxxx │ 10xxxxxx │
└────────────┴───────────────────────┴────────────────┴──────────────────┴──────────┴──────────┴──────────┴──────────┘

Index

• Constants
• func FormatBinary(r rune) string
• func Len(r rune) int
• func ReadRune(reader io.Reader) (rune, int, error)
• func WriteRune(writer io.Writer, r rune) (int, error)
• type InvalidUTF8Complainer
• type NilReaderComplainer
• type NilWriterComplainer
• type RuneReader
  • func NewRuneReader(reader io.Reader) *RuneReader
  • func (receiver *RuneReader) ReadRune() (rune, int, error)
• type RuneScanner
  • func NewRuneScanner(reader io.Reader) *RuneScanner
  • func (receiver *RuneScanner) ReadRune() (rune, int, error)
  • func (receiver *RuneScanner) UnreadRune() error

Constants

const (
	RuneError = utf8.RuneError
)

Functions

func FormatBinary

func FormatBinary(r rune) string

FormatBinary returns a representation of a rune as a sequence of bytes, given in binary format.

Example

utf8s.FormatBinary('۵')

// Outputs:
// <<0b11011011 ; 0b10110101>>

func Len

func Len(r rune) int

Len returns the number of bytes in a UTF-8 encoding of this Unicode code point.

Example

length := utf8s.Len('A')

// length == 1

Example

length := utf8s.Len('r')

// length == 1

Example

length := utf8s.Len('¡')

// length == 2

Example

length := utf8s.Len('۵')

// length == 2

func ReadRune

func ReadRune(reader io.Reader) (rune, int, error)

ReadRune reads a single UTF-8 encoded Unicode character from an io.Reader, and returns the Unicode character (as a Go rune) and the number of bytes read.

Note that a single UTF-8 encoded Unicode character could be more than one byte.

For example, the Unicode "≡" (IDENTICAL TO) character gets encoded using 3 bytes under UTF-8.

func WriteRune

func WriteRune(writer io.Writer, r rune) (int, error)

WriteRune writes a single UTF-8 encoded Unicode character and returns the number of bytes written.

Types

type InvalidUTF8Complainer

type InvalidUTF8Complainer interface {
	error
	InvalidUTF8Complainer()
}

type NilReaderComplainer

type NilReaderComplainer interface {
	error
	NilReaderComplainer()
}

type NilWriterComplainer

type NilWriterComplainer interface {
	error
	NilWriterComplainer()
}

type RuneReader

type RuneReader struct {
	// contains filtered or unexported fields
}

A utf8s.RuneReader implements the io.RuneReader interface by reading from an io.Reader.

func NewRuneReader

func NewRuneReader(reader io.Reader) *RuneReader

func (*RuneReader) ReadRune

func (receiver *RuneReader) ReadRune() (rune, int, error)

type RuneScanner

type RuneScanner struct {
	// contains filtered or unexported fields
}

A utf8s.RuneScanner implements the io.RuneScanner interface by reading from an io.Reader.

func NewRuneScanner

func NewRuneScanner(reader io.Reader) *RuneScanner

func (*RuneScanner) ReadRune

func (receiver *RuneScanner) ReadRune() (rune, int, error)

func (*RuneScanner) UnreadRune

func (receiver *RuneScanner) UnreadRune() error