sentences

package module

Go to main page

Versions in this module

v1

v1.1.2

Feb 6, 2023

Changes in this version

type DefaultWordTokenizer

+ func (p *DefaultWordTokenizer) IsCoordinatePartOne(t *Token) bool

+ func (p *DefaultWordTokenizer) IsCoordinatePartTwo(t *Token) bool

type TokenExistential

+ IsCoordinatePartOne func(*Token) bool

+ IsCoordinatePartTwo func(*Token) bool

v1.1.1

Dec 21, 2022

v1.1.0

Dec 21, 2022

Changes in this version

type DefaultWordTokenizer

+ func (p *DefaultWordTokenizer) HasUnreliableEndChars(t *Token) bool

+ func (p *DefaultWordTokenizer) IsListNumber(t *Token) bool

type TokenExistential

+ HasUnreliableEndChars func(*Token) bool

+ IsListNumber func(*Token) bool

v1.0.12

Jun 19, 2022

v1.0.11

Jun 18, 2022

v1.0.10

Jun 18, 2022

v1.0.9

Jun 18, 2021

v1.0.8

Jun 18, 2021

v1.0.7

May 26, 2021

Changes in this version

+ func IsCjkPunct(r rune) bool

v1.0.6

Mar 6, 2017

v1.0.5

Mar 5, 2017

v1.0.4

Mar 4, 2017

v1.0.3

Mar 4, 2017

v1.0.2

Apr 15, 2016

v1.0.1

Feb 27, 2016

Changes in this version

+ type AnnotateTokens interface

+ Annotate func([]*Token) []*Token

+ func NewAnnotations(s *Storage, p PunctStrings, word WordTokenizer) []AnnotateTokens

+ type DefaultPunctStrings struct

+ func NewPunctStrings() *DefaultPunctStrings

+ func (p *DefaultPunctStrings) HasSentencePunct(text string) bool

+ func (p *DefaultPunctStrings) NonPunct() string

+ func (p *DefaultPunctStrings) Punctuation() string

+ type DefaultSentenceTokenizer struct

+ Annotations []AnnotateTokens

+ func NewSentenceTokenizer(s *Storage) *DefaultSentenceTokenizer

+ func NewTokenizer(s *Storage, word WordTokenizer, lang PunctStrings) *DefaultSentenceTokenizer

+ func (s *DefaultSentenceTokenizer) AnnotateTokens(tokens []*Token, annotate ...AnnotateTokens) []*Token

+ func (s *DefaultSentenceTokenizer) AnnotatedTokens(text string) []*Token

+ func (s *DefaultSentenceTokenizer) SentencePositions(text string) []int

+ func (s *DefaultSentenceTokenizer) Tokenize(text string) []*Sentence

+ type DefaultTokenGrouper struct

+ func (p *DefaultTokenGrouper) Group(tokens []*Token) [][2]*Token

+ type DefaultWordTokenizer struct

+ func NewWordTokenizer(p PunctStrings) *DefaultWordTokenizer

+ func (p *DefaultWordTokenizer) FirstLower(t *Token) bool

+ func (p *DefaultWordTokenizer) FirstUpper(t *Token) bool

+ func (p *DefaultWordTokenizer) HasPeriodFinal(t *Token) bool

+ func (p *DefaultWordTokenizer) HasSentEndChars(t *Token) bool

+ func (p *DefaultWordTokenizer) IsAlpha(t *Token) bool

+ func (p *DefaultWordTokenizer) IsEllipsis(t *Token) bool

+ func (p *DefaultWordTokenizer) IsInitial(t *Token) bool

+ func (p *DefaultWordTokenizer) IsNonPunct(t *Token) bool

+ func (p *DefaultWordTokenizer) IsNumber(t *Token) bool

+ func (p *DefaultWordTokenizer) Tokenize(text string, onlyPeriodContext bool) []*Token

+ func (p *DefaultWordTokenizer) Type(t *Token) string

+ func (p *DefaultWordTokenizer) TypeNoPeriod(t *Token) string

+ func (p *DefaultWordTokenizer) TypeNoSentPeriod(t *Token) string

+ type Ortho interface

+ Heuristic func(*Token) int

+ type OrthoContext struct

+ func (o *OrthoContext) Heuristic(token *Token) int

+ type PunctStrings interface

+ HasSentencePunct func(string) bool

+ NonPunct func() string

+ Punctuation func() string

+ type Sentence struct

+ End int

+ Start int

+ Text string

+ func (s Sentence) String() string

+ type SentenceTokenizer interface

+ AnnotateTokens func([]*Token, ...AnnotateTokens) []*Token

+ Tokenize func(string) []*Sentence

+ type SetString map[string]int

+ func (ss SetString) Add(str string)

+ func (ss SetString) Array() []string

+ func (ss SetString) Has(str string) bool

+ func (ss SetString) Remove(str string)

+ type Storage struct

+ AbbrevTypes SetString

+ Collocations SetString

+ OrthoContext SetString

+ SentStarters SetString

+ func LoadTraining(data []byte) (*Storage, error)

+ func NewStorage() *Storage

+ func (p *Storage) IsAbbr(tokens ...string) bool

+ type Token struct

+ Abbr bool

+ LineStart bool

+ ParaStart bool

+ Position int

+ SentBreak bool

+ Tok string

+ func NewToken(token string) *Token

+ func (p *Token) String() string

+ type TokenBasedAnnotation struct

+ func (a *TokenBasedAnnotation) Annotate(tokens []*Token) []*Token

+ type TokenExistential interface

+ HasPeriodFinal func(*Token) bool

+ HasSentEndChars func(*Token) bool

+ IsAlpha func(*Token) bool

+ IsEllipsis func(*Token) bool

+ IsInitial func(*Token) bool

+ IsNonPunct func(*Token) bool

+ IsNumber func(*Token) bool

+ type TokenFirst interface

+ FirstLower func(*Token) bool

+ FirstUpper func(*Token) bool

+ type TokenGrouper interface

+ Group func([]*Token) [][2]*Token

+ type TokenParser interface

+ type TokenType interface

+ Type func(*Token) string

+ TypeNoPeriod func(*Token) string

+ TypeNoSentPeriod func(*Token) string

+ type TypeBasedAnnotation struct

+ func NewTypeBasedAnnotation(s *Storage, p PunctStrings, e TokenExistential) *TypeBasedAnnotation

+ func (a *TypeBasedAnnotation) Annotate(tokens []*Token) []*Token

+ type WordTokenizer interface

+ Tokenize func(string, bool) []*Token

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL