nlp

Documentation

Index

• func BayesWeights() map[lang.Language]*bayes.NaiveBayes
• func TagTokens(ts []token.Token, d *dict.Dictionary, nb *bayes.NaiveBayes, thr float64)
• func Train(td TrainingData, d *dict.Dictionary) *bayes.NaiveBayes
• type BayesF
  • func (b BayesF) Name() bayes.FeatureName
  • func (b BayesF) Value() bayes.FeatureValue
• type FeatureSet
  • func NewFeatureSet(ts []token.Token) FeatureSet
  • func (fs *FeatureSet) Flatten() []bayes.Featurer
• type FileName
• type Label
  • func (l Label) String() string
• type NameData
• type NamesPositions
• type TextData
• type TrainingData
  • func NewTrainingData(path string) TrainingData
• type TrainingLanguageData
  • func NewTrainingLanguageData(dir string) TrainingLanguageData

Functions

func BayesWeights

func BayesWeights() map[lang.Language]*bayes.NaiveBayes

func TagTokens

func TagTokens(ts []token.Token, d *dict.Dictionary, nb *bayes.NaiveBayes,
	thr float64)

func Train

func Train(td TrainingData, d *dict.Dictionary) *bayes.NaiveBayes

Train performs the training process

Types

type BayesF

type BayesF struct {
	// contains filtered or unexported fields
}

BayesF implements bayes.Featurer

func (BayesF) Name

func (b BayesF) Name() bayes.FeatureName

Name is required by bayes.Featurer

func (BayesF) Value

func (b BayesF) Value() bayes.FeatureValue

Value is required by bayes.Featurer

type FeatureSet

type FeatureSet struct {
	Uninomial []BayesF
	Species   []BayesF
	InfraSp   []BayesF
}

FeatureSet splits features into Uninomial, Species, Ifraspecies groups

func NewFeatureSet

func NewFeatureSet(ts []token.Token) FeatureSet

BayesFeatures creates slices of features for a token that might represent genus or other uninomial

func (*FeatureSet) Flatten

func (fs *FeatureSet) Flatten() []bayes.Featurer

type FileName

type FileName string

type Label

type Label int

const (
	NotName Label = iota
	Name
)

func (Label) String

func (l Label) String() string

type NameData

type NameData struct {
	Name  string `json:"name"`
	Start int    `json:"start"`
	End   int    `json:"end"`
}

type NamesPositions

type NamesPositions []NameData

type TextData

type TextData struct {
	Text []rune
	NamesPositions
}

type TrainingData

type TrainingData map[FileName]*TextData

func NewTrainingData

func NewTrainingData(path string) TrainingData

NewTrainingData assembles text and name occurance information from several files that contain no names at all, or are botanical and zoological research papers that do contain names.

type TrainingLanguageData

type TrainingLanguageData map[lang.Language]TrainingData

TrainingLanguageData associates a Language with training data

func NewTrainingLanguageData

func NewTrainingLanguageData(dir string) TrainingLanguageData

LoadTrainingData loads TrainingData from a file.