textkit

Apr 20, 2026

Changes in this version

type TokenType

+ const Comment

type Tokeniser

+ KeepComments bool

Sep 6, 2023

Changes in this version

type Token

+ func (t *Token) String() string

Sep 5, 2023

Changes in this version

type TokenType

+ const EndIndent

+ func (t TokenType) String() string

type Tokeniser

+ KeepEndIndents bool

Jul 6, 2023

Changes in this version

+ type Location struct

+ Column int

+ File string

+ Line int

+ func (loc Location) String() string

type Token

+ Loc Location

Jan 15, 2023

Changes in this version

type TokenType

+ const EOL

type Tokeniser

+ KeepEOLs bool

Feb 17, 2021

Changes in this version

+ type LexicalEntry struct

+ Lemma string

+ Tag string

+ type MorphologicalLexicon struct

+ func NewMorphologicalLexicon(filename string) (*MorphologicalLexicon, error)

+ func (lex *MorphologicalLexicon) AddEntry(form, lemma, tag string)

+ func (lex *MorphologicalLexicon) Analyse(form string) []*LexicalEntry

+ type Token struct

+ Column int

+ Form []rune

+ Line int

+ Tag string

+ Type TokenType

+ type TokenType int

+ const EOF

+ const Number

+ const String

+ const Symbol

+ const Word

+ type Tokeniser struct

+ CommentPrefix string

+ IdentChars string

+ StringRune rune

+ func (t *Tokeniser) Tokenise(text string) []*Token