gojieba

package module
v0.0.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 24, 2026 License: MIT Imports: 12 Imported by: 0

README

GoJieba

Test Author Tag Performance License GoDoc Coverage Status Go Report Card Awesome

GoJieba是"结巴"中文分词的Golang语言版本。

简介

  • 支持多种分词方式，包括: 最大概率模式, HMM新词发现模式, 搜索引擎模式, 全模式
  • 核心算法底层由C++实现，性能高效。
  • 字典路径可配置，NewJieba(...string), NewExtractor(...string) 可变形参，当参数为空时使用默认词典(推荐方式)
  • C++ 依赖 CppJieba 已直接打包在 deps/ 目录中，无需任何 git submodule 初始化，go get / go mod vendor 开箱即用。

用法

go get github.com/fangzio/gojieba

gojieba 依赖 cgo 和 C++。如果是交叉编译，请确保目标平台的 C/C++ 交叉工具链已经安装，并显式开启 cgo；CGO_ENABLED=0 的纯 Go 交叉编译方式不适用于这个库。

gojieba requires cgo and C++; cross-compilation needs CGO_ENABLED=1 plus a target C/C++ toolchain via CC/CXX.

例如，交叉编译到 Linux amd64 时，需要类似下面这样提供目标平台工具链:

CGO_ENABLED=1 \
CC=x86_64-linux-gnu-gcc \
CXX=x86_64-linux-gnu-g++ \
GOOS=linux \
GOARCH=amd64 \
go build

分词示例

package main

import (
	"fmt"
	"strings"

	"github.com/fangzio/gojieba"
)

func main() {
	var s string
	var words []string
	use_hmm := true
	x := gojieba.NewJieba()
	defer x.Free()

	s = "我来到北京清华大学"
	words = x.CutAll(s)
	fmt.Println(s)
	fmt.Println("全模式:", strings.Join(words, "/"))

	words = x.Cut(s, use_hmm)
	fmt.Println(s)
	fmt.Println("精确模式:", strings.Join(words, "/"))
	s = "比特币"
	words = x.Cut(s, use_hmm)
	fmt.Println(s)
	fmt.Println("精确模式:", strings.Join(words, "/"))

	x.AddWord("比特币")
	// `AddWordEx` 支持指定词语的权重，作为 `AddWord` 权重太低加词失败的补充。
	// `tag` 参数可以为空字符串，也可以指定词性。
	// x.AddWordEx("比特币", 100000, "")
	// 允许词典词条包含 ASCII 空格；同时保留 Tab、换行、中文逗号和句号作为边界。
	// x.ResetSeparators("\\t\\n，。")
	s = "比特币"
	words = x.Cut(s, use_hmm)
	fmt.Println(s)
	fmt.Println("添加词典后,精确模式:", strings.Join(words, "/"))

	s = "他来到了网易杭研大厦"
	words = x.Cut(s, use_hmm)
	fmt.Println(s)
	fmt.Println("新词识别:", strings.Join(words, "/"))

	s = "小明硕士毕业于中国科学院计算所，后在日本京都大学深造"
	words = x.CutForSearch(s, use_hmm)
	fmt.Println(s)
	fmt.Println("搜索引擎模式:", strings.Join(words, "/"))

	s = "长春市长春药店"
	words = x.Tag(s)
	fmt.Println(s)
	fmt.Println("词性标注:", strings.Join(words, ","))

	s = "区块链"
	words = x.Tag(s)
	fmt.Println(s)
	fmt.Println("词性标注:", strings.Join(words, ","))

	s = "长江大桥"
	words = x.CutForSearch(s, !use_hmm)
	fmt.Println(s)
	fmt.Println("搜索引擎模式:", strings.Join(words, "/"))

	wordinfos := x.Tokenize(s, gojieba.SearchMode, !use_hmm)
	fmt.Println(s)
	fmt.Println("Tokenize:(搜索引擎模式)", wordinfos)

	wordinfos = x.Tokenize(s, gojieba.DefaultMode, !use_hmm)
	fmt.Println(s)
	fmt.Println("Tokenize:(默认模式)", wordinfos)

	keywords := x.ExtractWithWeight(s, 5)
	fmt.Println("Extract:", keywords)
}

我来到北京清华大学
全模式: 我/来到/北京/清华/清华大学/华大/大学
我来到北京清华大学
精确模式: 我/来到/北京/清华大学
比特币
精确模式: 比特/币
比特币
添加词典后,精确模式: 比特币
他来到了网易杭研大厦
新词识别: 他/来到/了/网易/杭研/大厦
小明硕士毕业于中国科学院计算所，后在日本京都大学深造
搜索引擎模式: 小明/硕士/毕业/于/中国/科学/学院/科学院/中国科学院/计算/计算所/，/后/在/日本/京都/大学/日本京都大学/深造
长春市长春药店
词性标注: 长春市/ns,长春/ns,药店/n
区块链
词性标注: 区块链/nz
长江大桥
搜索引擎模式: 长江/大桥/长江大桥
长江大桥
Tokenize: [{长江 0 6} {大桥 6 12} {长江大桥 0 12}]

See Details in gojieba-demo See example in jieba_test, extractor_test

Benchmark

Jieba中文分词系列性能评测

Unittest

go test ./...

Benchmark

go test -bench "Jieba" -test.benchtime 10s
go test -bench "Extractor" -test.benchtime 10s

Contributors

Code Contributors

This project exists thanks to all the people who contribute.

Documentation

Index

Examples

Constants

View Source 
const TOTAL_DICT_PATH_NUMBER = 5

Variables

View Source 
var (
	DICT_DIR        string
	DICT_PATH       string
	HMM_PATH        string
	USER_DICT_PATH  string
	IDF_PATH        string
	STOP_WORDS_PATH string
)

Functions

func Trim 

func Trim()

Types

type Jieba 

type Jieba struct {
	// contains filtered or unexported fields
}
Example 
var s string
var words []string
use_hmm := true
// equals with x := NewJieba(DICT_PATH, HMM_PATH, USER_DICT_PATH)
x := NewJieba()
defer x.Free()

s = "我来到北京清华大学"
words = x.CutAll(s)
fmt.Println(s)
fmt.Println("全模式:", strings.Join(words, "/"))

words = x.Cut(s, use_hmm)
fmt.Println(s)
fmt.Println("精确模式:", strings.Join(words, "/"))

s = "比特币"
words = x.Cut(s, use_hmm)
fmt.Println(s)
fmt.Println("精确模式:", strings.Join(words, "/"))

x.AddWord("比特币")
s = "比特币"
words = x.Cut(s, use_hmm)
fmt.Println(s)
fmt.Println("添加词典后,精确模式:", strings.Join(words, "/"))

x.AddWord("这是一个很长的关键字")
s = "这是一个很长的关键字"
words = x.Extract(s, 3)
fmt.Println(s)
fmt.Println("添加词典后,Extract:", strings.Join(words, "/"))

x.RemoveWord("这是一个很长的关键字")
s = "这是一个很长的关键字"
words = x.Extract(s, 3)
fmt.Println(s)
fmt.Println("从词典删除后,Extract:", strings.Join(words, "/"))

s = "他来到了网易杭研大厦"
words = x.Cut(s, use_hmm)
fmt.Println(s)
fmt.Println("新词识别:", strings.Join(words, "/"))

s = "小明硕士毕业于中国科学院计算所，后在日本京都大学深造"
words = x.CutForSearch(s, use_hmm)
fmt.Println(s)
fmt.Println("搜索引擎模式:", strings.Join(words, "/"))

s = "长春市长春药店"
words = x.Tag(s)
fmt.Println(s)
fmt.Println("词性标注:", strings.Join(words, ","))

s = "区块链"
words = x.Tag(s)
fmt.Println(s)
fmt.Println("词性标注:", strings.Join(words, ","))

s = "长江大桥"
words = x.CutForSearch(s, !use_hmm)
fmt.Println(s)
fmt.Println("搜索引擎模式:", strings.Join(words, "/"))

wordinfos := x.Tokenize(s, SearchMode, !use_hmm)
fmt.Println(s)
fmt.Println("Tokenize:", wordinfos)

Output:
我来到北京清华大学
全模式: 我/来到/北京/清华/清华大学/华大/大学
我来到北京清华大学
精确模式: 我/来到/北京/清华大学
比特币
精确模式: 比特/币
比特币
添加词典后,精确模式: 比特币
这是一个很长的关键字
添加词典后,Extract: 这是一个很长的关键字
这是一个很长的关键字
从词典删除后,Extract: 关键字/很长/这是
他来到了网易杭研大厦
新词识别: 他/来到/了/网易/杭研/大厦
小明硕士毕业于中国科学院计算所，后在日本京都大学深造
搜索引擎模式: 小明/硕士/毕业/于/中国/科学/学院/科学院/中国科学院/计算/计算所/，/后/在/日本/京都/大学/日本京都大学/深造
长春市长春药店
词性标注: 长春市/ns,长春/ns,药店/n
区块链
词性标注: 区块链/nz
长江大桥
搜索引擎模式: 长江/大桥/长江大桥
长江大桥
Tokenize: [{长江 0 6} {大桥 6 12} {长江大桥 0 12}]

func NewJieba 

func NewJieba(paths ...string) *Jieba

func (*Jieba) AddWord 

func (x *Jieba) AddWord(s string)

func (*Jieba) AddWordEx 

func (x *Jieba) AddWordEx(s string, freq int, tag string)

func (*Jieba) Cut 

func (x *Jieba) Cut(s string, hmm bool) []string

func (*Jieba) CutAll 

func (x *Jieba) CutAll(s string) []string

func (*Jieba) CutForSearch 

func (x *Jieba) CutForSearch(s string, hmm bool) []string

func (*Jieba) Extract 

func (x *Jieba) Extract(s string, topk int) []string
Example 
x := NewJieba()
defer x.Free()

s := "我是拖拉机学院手扶拖拉机专业的。不用多久，我就会升职加薪，当上CEO，走上人生巅峰。"
words := x.Extract(s, 5)
fmt.Println(s)
fmt.Println("关键词抽取:", strings.Join(words, "/"))
word_weights := x.ExtractWithWeight(s, 5)
fmt.Println("关键词抽取:", word_weights)

x.AddWord("人生巅峰")
words = x.Extract(s, 5)
fmt.Println("AddWord后关键词抽取:", strings.Join(words, "/"))

x.RemoveWord("人生巅峰")
words = x.Extract(s, 5)
fmt.Println("RemoveWord后关键词抽取:", strings.Join(words, "/"))

Output:
我是拖拉机学院手扶拖拉机专业的。不用多久，我就会升职加薪，当上CEO，走上人生巅峰。
关键词抽取: CEO/升职/加薪/手扶拖拉机/巅峰
关键词抽取: [{CEO 11.739204307083542} {升职 10.8561552143} {加薪 10.642581114} {手扶拖拉机 10.0088573539} {巅峰 9.49395840471}]
AddWord后关键词抽取: CEO/人生巅峰/升职/加薪/手扶拖拉机
RemoveWord后关键词抽取: CEO/升职/加薪/手扶拖拉机/巅峰

func (*Jieba) ExtractWithWeight 

func (x *Jieba) ExtractWithWeight(s string, topk int) []WordWeight

func (*Jieba) ExtractWithWeightAndTempWords added in v0.0.2 

func (x *Jieba) ExtractWithWeightAndTempWords(s string, topk int, temporary []TemporaryWord) []WordWeight

ExtractWithWeightAndTempWords applies temporary dictionary words only for this extraction call. The underlying trie is restored before the function returns, including when a temporary word overrides an existing word. Callers must still serialize access to the Jieba instance.

func (*Jieba) Free 

func (x *Jieba) Free()

func (*Jieba) FreeWithTrim deprecated 

func (x *Jieba) FreeWithTrim()

Deprecated: Use Free() instead. Free() now calls Trim() automatically.

func (*Jieba) RemoveWord 

func (x *Jieba) RemoveWord(s string)

func (*Jieba) ResetSeparators 

func (x *Jieba) ResetSeparators(separators string)

ResetSeparators configures the characters that force segmentation boundaries. It applies to Cut, CutForSearch, Tag and keyword extraction. For example, pass "\\t\\n，。" to permit a user dictionary word to contain ASCII spaces while keeping tabs, newlines, Chinese commas and full stops as boundaries.

func (*Jieba) SuggestFrequency added in v0.0.2 

func (x *Jieba) SuggestFrequency(word string) int

SuggestFrequency returns the minimum trie frequency that makes word compete with its current HMM-disabled segmentation, following jieba.suggest_freq's probability calculation. Business callers may round it up further.

func (*Jieba) Tag 

func (x *Jieba) Tag(s string) []string

func (*Jieba) Tokenize 

func (x *Jieba) Tokenize(s string, mode TokenizeMode, hmm bool) []Word

func (*Jieba) WithTrim deprecated 

func (x *Jieba) WithTrim() *Jieba

Deprecated: WithTrim is no longer necessary; Free() now calls Trim() automatically on Linux. Calling this method is a no-op.

type TemporaryWord added in v0.0.2 

type TemporaryWord struct {
	Word string
	Freq int
	Tag  string
}

type TokenizeMode 

type TokenizeMode int
const (
	DefaultMode TokenizeMode = iota
	SearchMode
)

type Word 

type Word struct {
	Str   string
	Start int
	End   int
}

type WordWeight 

type WordWeight struct {
	Word   string
	Weight float64
}

Source Files

View all Source files

Directories

Path Synopsis
deps
cppjieba/dict
Package dict contains the bundled cppjieba dictionaries.
 Package dict contains the bundled cppjieba dictionaries.
cppjieba/dict/pos_dict
Package pos_dict contains the bundled POS dictionaries.
 Package pos_dict contains the bundled POS dictionaries.
cppjieba/include/cppjieba
Package cppjieba contains the bundled C++ headers.
 Package cppjieba contains the bundled C++ headers.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.