shield

README

Shield is a bayesian text classifier with flexible tokenizer and backend store support

Currently implemented:

• Redis backend
• LevelDB backend
• English tokenizer
• Russian tokenizer
• Russian porterstemmer

Example (REDIS)

package main

import (
  "github.com/eaigner/shield"
)

func main() {
  sh := shield.New(
    shield.NewEnglishTokenizer(),
    shield.NewRedisStore("127.0.0.1:6379", "", 0),
  )

  sh.Learn("good", "sunshine drugs love sex lobster sloth")
  sh.Learn("bad", "fear death horror government zombie god")

  c, _ := sh.Classify("sloths are so cute i love them")
  if c != "good" {
    panic(c)
  }

  c, _ = sh.Classify("i fear god and love the government")
  if c != "bad" {
    panic(c)
  }
}

Example (LevelDB)

package main

import (
  "github.com/eaigner/shield"
)

func main() {
  sh := shield.New(
    shield.NewEnglishTokenizer(),
    shield.NewLevelDBStore("./db"),
  )

  sh.Learn("good", "sunshine drugs love sex lobster sloth")
  sh.Learn("bad", "fear death horror government zombie god")

  c, _ := sh.Classify("sloths are so cute i love them")
  if c != "good" {
    panic(c)
  }

  c, _ = sh.Classify("i fear god and love the government")
  if c != "bad" {
    panic(c)
  }
}

Documentation

Index

• type LevelDBStore
  • func (ls *LevelDBStore) AddClass(class string) (err error)
  • func (ls *LevelDBStore) ClassWordCounts(class string, words []string) (mc map[string]int64, err error)
  • func (ls *LevelDBStore) Classes() (a []string, err error)
  • func (ls *LevelDBStore) Close() error
  • func (ls *LevelDBStore) IncrementClassWordCounts(m map[string]map[string]int64) (err error)
  • func (ls *LevelDBStore) Reset() (err error)
  • func (ls *LevelDBStore) TotalClassWordCounts() (m map[string]int64, err error)
• type RedisStore
  • func (rs *RedisStore) AddClass(class string) (err error)
  • func (rs *RedisStore) ClassWordCounts(class string, words []string) (mc map[string]int64, err error)
  • func (rs *RedisStore) Classes() (a []string, err error)
  • func (rs *RedisStore) Close() error
  • func (rs *RedisStore) IncrementClassWordCounts(m map[string]map[string]int64) (err error)
  • func (rs *RedisStore) Reset() (err error)
  • func (rs *RedisStore) TotalClassWordCounts() (m map[string]int64, err error)
• type Set
• type Shield
  • func New(t Tokenizer, s Store) Shield
• type Store
  • func NewLevelDBStore(pathToDB string) Store
  • func NewRedisStore(addr, password, db string, logger *log.Logger, prefix string) Store
• type Tokenizer
  • func NewEnglishTokenizer() Tokenizer
  • func NewRussianTokenizer() Tokenizer

Types

type LevelDBStore

type LevelDBStore struct {
	// contains filtered or unexported fields
}

LevelDBStore store

func (*LevelDBStore) AddClass

func (ls *LevelDBStore) AddClass(class string) (err error)

AddClass - add class

func (*LevelDBStore) ClassWordCounts

func (ls *LevelDBStore) ClassWordCounts(class string, words []string) (mc map[string]int64, err error)

ClassWordCounts - count words by class

func (*LevelDBStore) Classes

func (ls *LevelDBStore) Classes() (a []string, err error)

Classes - list classes

func (*LevelDBStore) Close

func (ls *LevelDBStore) Close() error

Close - close connection

func (*LevelDBStore) IncrementClassWordCounts

func (ls *LevelDBStore) IncrementClassWordCounts(m map[string]map[string]int64) (err error)

IncrementClassWordCounts - increment vals

func (*LevelDBStore) Reset

func (ls *LevelDBStore) Reset() (err error)

Reset - clean store

func (*LevelDBStore) TotalClassWordCounts

func (ls *LevelDBStore) TotalClassWordCounts() (m map[string]int64, err error)

TotalClassWordCounts - count words in classes

type RedisStore

type RedisStore struct {
	// contains filtered or unexported fields
}

RedisStore - store

func (*RedisStore) AddClass

func (rs *RedisStore) AddClass(class string) (err error)

func (*RedisStore) ClassWordCounts

func (rs *RedisStore) ClassWordCounts(class string, words []string) (mc map[string]int64, err error)

func (*RedisStore) Classes

func (rs *RedisStore) Classes() (a []string, err error)

func (*RedisStore) Close

func (rs *RedisStore) Close() error

func (*RedisStore) IncrementClassWordCounts

func (rs *RedisStore) IncrementClassWordCounts(m map[string]map[string]int64) (err error)

func (*RedisStore) Reset

func (rs *RedisStore) Reset() (err error)

func (*RedisStore) TotalClassWordCounts

func (rs *RedisStore) TotalClassWordCounts() (m map[string]int64, err error)

type Set

type Set struct {
	Class string
	Text  string
}

Set struct

type Shield

type Shield interface {
	// Learn learns a single document
	Learn(class, text string) (err error)

	// BulkLearn learns many documents at once
	BulkLearn(sets []Set) (err error)

	// Forget forgets the document in the specified class
	Forget(class, text string) (err error)

	// Score returns the scores for each class normalized from 0 to 1
	Score(text string) (scores map[string]float64, err error)

	// Classify returns the class with the highest score
	Classify(text string) (c string, err error)

	// Reset clears the storage
	Reset() error

	// Destroy
	Destroy()
}

Shield interface

func New

func New(t Tokenizer, s Store) Shield

New - new Shield

type Store

type Store interface {
	Classes() ([]string, error)
	AddClass(class string) error
	ClassWordCounts(class string, words []string) (mc map[string]int64, err error)
	IncrementClassWordCounts(m map[string]map[string]int64) error
	TotalClassWordCounts() (map[string]int64, error)
	Reset() error
	Close() error
}

Store interface

func NewLevelDBStore

func NewLevelDBStore(pathToDB string) Store

NewLevelDBStore - new redis store

func NewRedisStore

func NewRedisStore(addr, password, db string, logger *log.Logger, prefix string) Store

NewRedisStore - new redis store

type Tokenizer

type Tokenizer interface {
	Tokenize(text string) (words map[string]int64)
}

Tokenizer interface

func NewEnglishTokenizer

func NewEnglishTokenizer() Tokenizer

func NewRussianTokenizer

func NewRussianTokenizer() Tokenizer

NewRussianTokenizer - new Russian tokenizer