sensitive

README

Sensitive

敏感词查找,验证,过滤和替换

FindAll, Validate, Filter and Replace words.

新增分支Aho-Corasick以支持AC自动机

用法

package main

import (
	"fmt"
	"github.com/importcjj/sensitive"
)

func main() {
	filter := sensitive.New()
	filter.LoadWordDict("path/to/dict")
	// Do something
}

AddWord

添加敏感词

filter.AddWord("垃圾")

Replace

把词语中的字符替换成指定的字符，这里的字符指的是rune字符，比如*就是'*'。

filter.Replace("这篇文章真的好垃圾", '*')
// output => 这篇文章真的好**

Filter

直接移除词语

filter.Filter("这篇文章真的好垃圾啊")
// output => 这篇文章真的好啊

FindIn

查找并返回第一个敏感词，如果没有则返回false

filter.FindIn("这篇文章真的好垃圾")
// output => true, 垃圾

Validate

验证内容是否ok，如果含有敏感词，则返回false和第一个敏感词。

filter.Validate("这篇文章真的好垃圾")
// output => false, 垃圾

FindAll

查找内容中的全部敏感词，以数组返回。

filter.FindAll("这篇文章真的好垃圾")
// output => [垃圾]

LoadNetWordDict

加载网络词库。

filter.LoadNetWordDict("https://raw.githubusercontent.com/importcjj/sensitive/master/dict/dict.txt")

UpdateNoisePattern

设置噪音模式，排除噪音字符。

// failed
filter.FindIn("这篇文章真的好垃x圾")      // false
filter.UpdateNoisePattern(`x`)
// success
filter.FindIn("这篇文章真的好垃x圾")      // true, 垃圾
filter.Validate("这篇文章真的好垃x圾")    // False, 垃圾

Documentation

Index

• type Filter
  • func New() *Filter
  • func (filter *Filter) AddWord(words ...string)
  • func (filter *Filter) DelWord(words ...string)
  • func (filter *Filter) Filter(text string) string
  • func (filter *Filter) FindAll(text string) []string
  • func (filter *Filter) FindIn(text string) (bool, string)
  • func (filter *Filter) Load(rd io.Reader) error
  • func (filter *Filter) LoadNetWordDict(url string) error
  • func (filter *Filter) LoadWordDict(path string) error
  • func (filter *Filter) RemoveNoise(text string) string
  • func (filter *Filter) Replace(text string, repl rune) string
  • func (filter *Filter) UpdateNoisePattern(pattern string)
  • func (filter *Filter) Validate(text string) (bool, string)
  • func (filter *Filter) ValidateWithWildcard(text string, wildcard rune) (bool, string)
• type Node
  • func NewNode(character rune) *Node
  • func NewRootNode(character rune) *Node
  • func (node *Node) IsLeafNode() bool
  • func (node *Node) IsPathEnd() bool
  • func (node *Node) IsRootNode() bool
  • func (node *Node) SoftDel()
• type Trie
  • func NewTrie() *Trie
  • func (tree *Trie) Add(words ...string)
  • func (tree *Trie) Del(words ...string)
  • func (tree *Trie) Filter(text string) string
  • func (tree *Trie) FindAll(text string) []string
  • func (tree *Trie) FindIn(text string) (bool, string)
  • func (tree *Trie) Replace(text string, character rune) string
  • func (tree *Trie) Validate(text string) (bool, string)
  • func (tree *Trie) ValidateWithWildcard(text string, wildcard rune) (bool, string)

Types

type Filter

type Filter struct {
	// contains filtered or unexported fields
}

Filter 敏感词过滤器

func New

func New() *Filter

New 返回一个敏感词过滤器

func (*Filter) AddWord

func (filter *Filter) AddWord(words ...string)

AddWord 添加敏感词

func (*Filter) DelWord

func (filter *Filter) DelWord(words ...string)

DelWord 删除敏感词

func (*Filter) Filter

func (filter *Filter) Filter(text string) string

Filter 过滤敏感词

func (*Filter) FindAll

func (filter *Filter) FindAll(text string) []string

FindAll 找到所有匹配词

func (*Filter) FindIn

func (filter *Filter) FindIn(text string) (bool, string)

FindIn 检测敏感词

func (*Filter) Load

func (filter *Filter) Load(rd io.Reader) error

Load common method to add words

func (*Filter) LoadNetWordDict

func (filter *Filter) LoadNetWordDict(url string) error

LoadNetWordDict 加载网络敏感词字典

func (*Filter) LoadWordDict

func (filter *Filter) LoadWordDict(path string) error

LoadWordDict 加载敏感词字典

func (*Filter) RemoveNoise

func (filter *Filter) RemoveNoise(text string) string

RemoveNoise 去除空格等噪音

func (*Filter) Replace

func (filter *Filter) Replace(text string, repl rune) string

Replace 和谐敏感词

func (*Filter) UpdateNoisePattern

func (filter *Filter) UpdateNoisePattern(pattern string)

UpdateNoisePattern 更新去噪模式

func (*Filter) Validate

func (filter *Filter) Validate(text string) (bool, string)

Validate 检测字符串是否合法

func (*Filter) ValidateWithWildcard

func (filter *Filter) ValidateWithWildcard(text string, wildcard rune) (bool, string)

type Node

type Node struct {
	Character rune
	Children  map[rune]*Node
	// contains filtered or unexported fields
}

Node Trie树上的一个节点.

func NewNode

func NewNode(character rune) *Node

NewNode 新建子节点

func NewRootNode

func NewRootNode(character rune) *Node

NewRootNode 新建根节点

func (*Node) IsLeafNode

func (node *Node) IsLeafNode() bool

IsLeafNode 判断是否叶子节点

func (*Node) IsPathEnd

func (node *Node) IsPathEnd() bool

IsPathEnd 判断是否为某个路径的结束

func (*Node) IsRootNode

func (node *Node) IsRootNode() bool

IsRootNode 判断是否为根节点

func (*Node) SoftDel

func (node *Node) SoftDel()

SoftDel 置软删除状态

type Trie

type Trie struct {
	Root *Node
}

Trie 短语组成的Trie树.

func NewTrie

func NewTrie() *Trie

NewTrie 新建一棵Trie

func (*Trie) Add

func (tree *Trie) Add(words ...string)

Add 添加若干个词

func (*Trie) Del

func (tree *Trie) Del(words ...string)

func (*Trie) Filter

func (tree *Trie) Filter(text string) string

Filter 直接过滤掉字符串中的敏感词

func (*Trie) FindAll

func (tree *Trie) FindAll(text string) []string

FindAll 找有所有包含在词库中的词

func (*Trie) FindIn

func (tree *Trie) FindIn(text string) (bool, string)

FindIn 判断text中是否含有词库中的词

func (*Trie) Replace

func (tree *Trie) Replace(text string, character rune) string

Replace 词语替换

func (*Trie) Validate

func (tree *Trie) Validate(text string) (bool, string)

Validate 验证字符串是否合法，如不合法则返回false和检测到 的第一个敏感词

func (*Trie) ValidateWithWildcard

func (tree *Trie) ValidateWithWildcard(text string, wildcard rune) (bool, string)