shield

README

Shield is a bayesian text classifier with flexible tokenizer and backend store support

Currently implemented:

• Redis backend
• English tokenizer

Example

package main

import (
  "github.com/eaigner/shield"
)

func main() {
  sh := shield.New(
    shield.NewEnglishTokenizer(),
    shield.NewRedisStore("127.0.0.1:6379", "", 0),
  )

  sh.Learn("good", "sunshine drugs love sex lobster sloth")
  sh.Learn("bad", "fear death horror government zombie god")

  c, _ := sh.Classify("sloths are so cute i love them")
  if c != "good" {
    panic(c)
  }

  c, _ = sh.Classify("i fear god and love the government")
  if c != "bad" {
    panic(c)
  }
}

Documentation

Index

• type RedisStore
  • func (rs *RedisStore) AddClass(class string) (err error)
  • func (rs *RedisStore) ClassWordCounts(class string, words []string) (mc map[string]int64, err error)
  • func (rs *RedisStore) Classes() (a []string, err error)
  • func (rs *RedisStore) IncrementClassWordCounts(m map[string]map[string]int64) (err error)
  • func (rs *RedisStore) Reset() (err error)
  • func (rs *RedisStore) TotalClassWordCounts() (m map[string]int64, err error)
• type Set
• type Shield
  • func New(t Tokenizer, s Store) Shield
• type Store
  • func NewRedisStore(addr, password string, logger *log.Logger, prefix string) Store
• type Tokenizer
  • func NewEnglishTokenizer() Tokenizer

Types

type RedisStore

type RedisStore struct {
	// contains filtered or unexported fields
}

func (*RedisStore) AddClass

func (rs *RedisStore) AddClass(class string) (err error)

func (*RedisStore) ClassWordCounts

func (rs *RedisStore) ClassWordCounts(class string, words []string) (mc map[string]int64, err error)

func (*RedisStore) Classes

func (rs *RedisStore) Classes() (a []string, err error)

func (*RedisStore) IncrementClassWordCounts

func (rs *RedisStore) IncrementClassWordCounts(m map[string]map[string]int64) (err error)

func (*RedisStore) Reset

func (rs *RedisStore) Reset() (err error)

func (*RedisStore) TotalClassWordCounts

func (rs *RedisStore) TotalClassWordCounts() (m map[string]int64, err error)

type Set

type Set struct {
	Class string
	Text  string
}

type Shield

type Shield interface {
	// Learn learns a single document
	Learn(class, text string) (err error)

	// BulkLearn learns many documents at once
	BulkLearn(sets []Set) (err error)

	// Forget forgets the document in the specified class
	Forget(class, text string) (err error)

	// Score returns the scores for each class normalized from 0 to 1
	Score(text string) (scores map[string]float64, err error)

	// Classify returns the class with the highest score
	Classify(text string) (c string, err error)

	// Reset clears the storage
	Reset() error
}

func New

func New(t Tokenizer, s Store) Shield

type Store

type Store interface {
	Classes() ([]string, error)
	AddClass(class string) error
	ClassWordCounts(class string, words []string) (mc map[string]int64, err error)
	IncrementClassWordCounts(m map[string]map[string]int64) error
	TotalClassWordCounts() (map[string]int64, error)
	Reset() error
}

func NewRedisStore

func NewRedisStore(addr, password string, logger *log.Logger, prefix string) Store

type Tokenizer

type Tokenizer interface {
	Tokenize(text string) (words map[string]int64)
}

func NewEnglishTokenizer

func NewEnglishTokenizer() Tokenizer