nlp

package
v0.0.0-...-0dfd57a Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 27, 2020 License: MIT Imports: 17 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	START = "-START-"
	END   = "-END-"
)

Variables

View Source 
var DefaultStopWords = []string{
	"a",
	"about",
	"an",
	"are",
	"as",
	"at",
	"be",
	"by",
	"for",
	"from",
	"how",
	"i",
	"in",
	"is",
	"it",
	"of",
	"on",
	"or",
	"that",
	"the",
	"this",
	"to",
	"was",
	"what",
	"when",
	"where",
	"who",
	"will",
	"with",
}
View Source 
var DefaultTags = map[string]string{
	"N": "common noun",
	"O": "pronoun",
	"^": "proper noun",
	"S": "nominal + possessive",
	"Z": "proper noun + possessive",

	"V": "verb",
	"A": "adjective",
	"R": "adverb",
	"!": "interjection",

	"D": "determiner",
	"P": "preposition",
	"&": "conjunction",
	"T": "verb particle",
	"X": "predeterminer",

	"#": "topic",
	"@": "recipient",
	"U": "url",
	"E": "emoticon",

	"$": "numeral",
	",": "punctuation",
	"G": "abbreviations",
	"L": "nominal + verbal",
	"M": "proper noun + verbal",
	"Y": "X + verbal",
}

Functions

func AnalyzeAffirmativeSentiment 

func AnalyzeAffirmativeSentiment(tokens []*Token) (string, float64)

func AnalyzeSentenceFunction 

func AnalyzeSentenceFunction(tokens []*Token) string

func JoinTokens 

func JoinTokens(ts []*Token) string

Types

type ActionPredictor 

type ActionPredictor struct {
	Perceptron *mlearning.Perceptron
}

func NewActionPredictor 

func NewActionPredictor() *ActionPredictor

func (*ActionPredictor) Predict 

func (p *ActionPredictor) Predict(m *Meaning) (string, float64)

func (*ActionPredictor) PredictAll 

func (p *ActionPredictor) PredictAll(m *Meaning) []mlearning.Prediction

func (*ActionPredictor) ReinforceMeaning 

func (p *ActionPredictor) ReinforceMeaning(m *Meaning, action string)

func (*ActionPredictor) Train 

func (p *ActionPredictor) Train(iterations int, dataset DataSet, tokenizer *Tokenizer)

type Data 

type Data struct {
	Action   string
	Sentence string
	Vars     []*Var
}

func NewDataFromTokens 

func NewDataFromTokens(tok []*Token, vars []*Var, action string) Data

func ReadData 

func ReadData(data string) (Data, error)

func (Data) CleanedSentence 

func (d Data) CleanedSentence(placeholder string) string

type DataSet 

type DataSet []Data

func ReadDataSet 

func ReadDataSet(r io.Reader) (DataSet, error)

type Meaning 

type Meaning struct {
	Subject    string `json:"subject,omitempty"`
	Predicate  string `json:"predicate,omitempty"`
	Object     string `json:"object,omitempty"`
	ObjectType string `json:"object_type,omitempty"`

	Tokens []*Token `json:"-"`
	Vars   []*Var   `json:"vars"`
}

func (Meaning) Features 

func (m Meaning) Features() []mlearning.Feature

type MeaningParser 

type MeaningParser struct {
}

func NewMeaningParser 

func NewMeaningParser() *MeaningParser

func (*MeaningParser) ParseDeclarative 

func (p *MeaningParser) ParseDeclarative(tokens []*Token) (*Meaning, error)

func (*MeaningParser) ParseImperative 

func (p *MeaningParser) ParseImperative(tokens []*Token) (*Meaning, error)

func (*MeaningParser) ParseInterrogative 

func (p *MeaningParser) ParseInterrogative(tokens []*Token) (*Meaning, error)

type MessageSchema 

type MessageSchema struct {
	Action string
	Fields map[string]string
}

func (*MessageSchema) Apply 

func (r *MessageSchema) Apply(vars []*Var) sarif.Message

type MessageSchemaStore 

type MessageSchemaStore struct {
	Messages map[string]*MessageSchema
}

func NewMessageSchemaStore 

func NewMessageSchemaStore() *MessageSchemaStore

func (*MessageSchemaStore) Add 

func (s *MessageSchemaStore) Add(schema *MessageSchema)

func (*MessageSchemaStore) AddDataSet 

func (s *MessageSchemaStore) AddDataSet(set DataSet)

func (*MessageSchemaStore) AddMessage 

func (s *MessageSchemaStore) AddMessage(msg *sarif.Message)

func (*MessageSchemaStore) Get 

func (s *MessageSchemaStore) Get(action string) *MessageSchema

func (*MessageSchemaStore) Set 

func (s *MessageSchemaStore) Set(schema *MessageSchema)

type Model 

type Model struct {
	Rules    []string
	Schemata []*MessageSchema
	*mlearning.Model
}

type ParseError 

type ParseError struct {
	Line int
	Err  error
}

func (ParseError) Error 

func (e ParseError) Error() string

type ParseResult 

type ParseResult struct {
	Text    string            `json:"text"`
	Intents []*natural.Intent `json:"intents"`

	Meaning *Meaning `json:"meaning"`
	Tokens  []*Token `json:"tokens"`
}

func (ParseResult) String 

func (r ParseResult) String() string

type Parser 

type Parser struct {
	// contains filtered or unexported fields
}

func NewParser 

func NewParser() *Parser

func (*Parser) InventMessageForMeaning 

func (p *Parser) InventMessageForMeaning(action string, m *Meaning) (sarif.Message, error)

func (*Parser) LearnMessage 

func (p *Parser) LearnMessage(msg *sarif.Message)

func (*Parser) LoadModel 

func (p *Parser) LoadModel(path string) error

func (*Parser) Parse 

func (p *Parser) Parse(ctx *natural.Context) (*ParseResult, error)

func (*Parser) ReinforceSentence 

func (p *Parser) ReinforceSentence(text string, action string) error

func (*Parser) ResolvePronouns 

func (p *Parser) ResolvePronouns(ts []*Token, ctx *natural.Context)

func (*Parser) SaveModel 

func (p *Parser) SaveModel(path string) error

func (*Parser) TrainModel 

func (p *Parser) TrainModel() error

type PosTagger 

type PosTagger struct {
	Perceptron *mlearning.Perceptron
}

func NewPosTagger 

func NewPosTagger() *PosTagger

func (*PosTagger) Predict 

func (p *PosTagger) Predict(s Sentence)

func (*PosTagger) Test 

func (p *PosTagger) Test(sentences []Sentence)

func (*PosTagger) Train 

func (p *PosTagger) Train(iterations int, sentences []Sentence)

type Sentence 

type Sentence []*Token

func LoadCoNLL 

func LoadCoNLL(r io.Reader) ([]Sentence, error)

type Token 

type Token struct {
	Value string              `json:"value,omitempty"`
	Lemma string              `json:"lemma,omitempty"`
	Tags  map[string]struct{} `json:"tags,omitempty"`
}

func (Token) Is 

func (t Token) Is(tag string) bool

func (*Token) Tag 

func (t *Token) Tag(tag string)

type Tokenizer 

type Tokenizer struct {
	SplitQuoted bool
	StopWords   []string
}

func NewTokenizer 

func NewTokenizer() *Tokenizer

func (*Tokenizer) Tokenize 

func (t *Tokenizer) Tokenize(s string) []*Token

type Var 

type Var struct {
	Name   string  `json:"name"`
	Type   string  `json:"type,omitempty"`
	Value  string  `json:"value"`
	Weight float64 `json:"weight"`
}

func (Var) String 

func (v Var) String() string

type VarPredictor 

type VarPredictor struct {
	Tokenizer  *Tokenizer
	Perceptron *mlearning.Perceptron
}

func NewVarPredictor 

func NewVarPredictor(tok *Tokenizer) *VarPredictor

func (*VarPredictor) Predict 

func (p *VarPredictor) Predict(s string, action string, pos int) (string, float64)

func (*VarPredictor) PredictTokens 

func (p *VarPredictor) PredictTokens(tok []*Token, action string) []*Var

func (*VarPredictor) Test 

func (p *VarPredictor) Test(dataset DataSet)

func (*VarPredictor) Train 

func (p *VarPredictor) Train(iterations int, dataset DataSet, tok *Tokenizer)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.