ortgenai

README

基于 purego 实现的无 CGO 纯 Go 项目，通过 purego 直接绑定并调用 onnxruntime-genai 原生库接口，无需依赖 CGO 编译环境， 即可实现 ONNX 模型的加载与推理计算，最新版本基于 onnxruntime-genai 0.12.0 的头文件实现。

安装

下载 onnxruntime-genai 0.12 动态链接库，安装 onnxruntime-genai_purego 库，版本对应关系：

onnxruntime-genai	onnxruntime-genai_purego
0.12	v0.12.0

# 下载最新版本
go get -u github.com/getcharzp/onnxruntime-genai_purego

# 针对 onnxruntime-genai 0.12 下载特定版本 purego
go get -u github.com/getcharzp/onnxruntime-genai_purego@v0.12

快速开始

• 初始化引擎

engine, err := ortgenai.NewEngine("onnxruntime-genai.dll")

• 生成文本

text, err := engine.Generate("model_path", "prompt", ortgenai.GenerateOptions{
    MaxTokens:   4096,
    Temperature: 1,
    TopP:        1.0,
    DoSample:    true,
})

• 流式生成文本

err := engine.GenerateStream("model_path", "prompt", ortgenai.GenerateOptions{
    MaxTokens:   4096,
    Temperature: 1,
    TopP:        1.0,
    DoSample:    true,
}, func(text string, _ int32) error {
    fmt.Print(text)
    return nil
})

支持的模型

• Phi-3 Microsoft 推出的小型语言模型

Documentation

Index

• Variables
• type Engine
  • func NewEngine(libPath string) (*Engine, error)
  • func (e *Engine) Generate(modelDir, prompt string, opts GenerateOptions) (string, error)
  • func (e *Engine) GenerateStream(modelDir, prompt string, opts GenerateOptions, ...) error
  • func (e *Engine) NewModel(modelDir string) (*Model, error)
• type GenerateOptions
• type Generator
  • func (g *Generator) AppendTokenSequences(sequences *Sequences) error
  • func (g *Generator) Close()
  • func (g *Generator) GenerateNextToken() error
  • func (g *Generator) GetNextTokens() ([]int32, error)
  • func (g *Generator) IsDone() bool
• type GeneratorParams
  • func (p *GeneratorParams) Close()
  • func (p *GeneratorParams) SetSearchBool(name string, value bool) error
  • func (p *GeneratorParams) SetSearchNumber(name string, value float64) error
• type Model
  • func (m *Model) Close()
  • func (m *Model) NewGenerator(params *GeneratorParams) (*Generator, error)
  • func (m *Model) NewGeneratorParams() (*GeneratorParams, error)
  • func (m *Model) NewTokenizer() (*Tokenizer, error)
• type OgaGenerator
• type OgaGeneratorParams
• type OgaModel
• type OgaResult
• type OgaSequences
• type OgaTokenizer
• type OgaTokenizerStream
• type Sequences
  • func (s *Sequences) Close()
• type Tokenizer
  • func (t *Tokenizer) Close()
  • func (t *Tokenizer) Encode(text string, sequences *Sequences) error
  • func (t *Tokenizer) GetEosTokenIds() ([]int32, error)
  • func (t *Tokenizer) NewSequences() (*Sequences, error)
  • func (t *Tokenizer) NewTokenizerStream() (*TokenizerStream, error)
• type TokenizerStream
  • func (s *TokenizerStream) Close()
  • func (s *TokenizerStream) DecodeToken(token int32) (string, error)

Variables

var ErrStopGeneration = errors.New("stream generation stopped")

Types

type Engine

type Engine struct {
	// contains filtered or unexported fields
}

Engine 推理引擎

func NewEngine

func NewEngine(libPath string) (*Engine, error)

NewEngine 初始化引擎

func (*Engine) Generate

func (e *Engine) Generate(modelDir, prompt string, opts GenerateOptions) (string, error)

Generate 生成文本

Params:

modelDir: 模型目录
prompt: 提示词
opts: 生成参数

func (*Engine) GenerateStream

func (e *Engine) GenerateStream(modelDir, prompt string, opts GenerateOptions, onToken func(text string, token int32) error) error

GenerateStream 流式生成文本

Params:

modelDir: 模型目录
prompt: 提示词
opts: 生成参数
onToken: 生成回调函数

func (*Engine) NewModel

func (e *Engine) NewModel(modelDir string) (*Model, error)

NewModel 初始化模型

type GenerateOptions

type GenerateOptions struct {
	MaxTokens   int     // maximum generated length
	Temperature float64 // sampling temperature
	TopP        float64
	DoSample    bool
	StopOnEos   bool // set true to stop on eos token
}

type Generator

type Generator struct {
	// contains filtered or unexported fields
}

func (*Generator) AppendTokenSequences

func (g *Generator) AppendTokenSequences(sequences *Sequences) error

AppendTokenSequences append token sequences.

func (*Generator) Close

func (g *Generator) Close()

Close release generator resources.

func (*Generator) GenerateNextToken

func (g *Generator) GenerateNextToken() error

GenerateNextToken generate next token.

func (*Generator) GetNextTokens

func (g *Generator) GetNextTokens() ([]int32, error)

GetNextTokens returns next tokens.

func (*Generator) IsDone

func (g *Generator) IsDone() bool

IsDone check if generation is done.

type GeneratorParams

type GeneratorParams struct {
	// contains filtered or unexported fields
}

func (*GeneratorParams) Close

func (p *GeneratorParams) Close()

Close release generator params resources.

func (*GeneratorParams) SetSearchBool

func (p *GeneratorParams) SetSearchBool(name string, value bool) error

SetSearchBool set bool search parameters.

func (*GeneratorParams) SetSearchNumber

func (p *GeneratorParams) SetSearchNumber(name string, value float64) error

SetSearchNumber set number search parameters.

type Model

type Model struct {
	// contains filtered or unexported fields
}

func (*Model) Close

func (m *Model) Close()

Close 释放模型资源

func (*Model) NewGenerator

func (m *Model) NewGenerator(params *GeneratorParams) (*Generator, error)

NewGenerator 创建生成器

func (*Model) NewGeneratorParams

func (m *Model) NewGeneratorParams() (*GeneratorParams, error)

NewGeneratorParams 创建生成器参数

func (*Model) NewTokenizer

func (m *Model) NewTokenizer() (*Tokenizer, error)

NewTokenizer 创建分词器

type OgaGenerator

type OgaGenerator uintptr

type OgaGeneratorParams

type OgaGeneratorParams uintptr

type OgaModel

type OgaModel uintptr

type OgaResult

type OgaResult uintptr

type OgaSequences

type OgaSequences uintptr

type OgaTokenizer

type OgaTokenizer uintptr

type OgaTokenizerStream

type OgaTokenizerStream uintptr

type Sequences

type Sequences struct {
	// contains filtered or unexported fields
}

func (*Sequences) Close

func (s *Sequences) Close()

Close 释放序列资源

type Tokenizer

type Tokenizer struct {
	// contains filtered or unexported fields
}

func (*Tokenizer) Close

func (t *Tokenizer) Close()

Close 释放分词器资源

func (*Tokenizer) Encode

func (t *Tokenizer) Encode(text string, sequences *Sequences) error

Encode 分词器编码

func (*Tokenizer) GetEosTokenIds

func (t *Tokenizer) GetEosTokenIds() ([]int32, error)

GetEosTokenIds returns EOS token ids.

func (*Tokenizer) NewSequences

func (t *Tokenizer) NewSequences() (*Sequences, error)

NewSequences 创建序列

func (*Tokenizer) NewTokenizerStream

func (t *Tokenizer) NewTokenizerStream() (*TokenizerStream, error)

NewTokenizerStream 创建分词器流

type TokenizerStream

type TokenizerStream struct {
	// contains filtered or unexported fields
}

func (*TokenizerStream) Close

func (s *TokenizerStream) Close()

Close 释放分词器流资源

func (*TokenizerStream) DecodeToken

func (s *TokenizerStream) DecodeToken(token int32) (string, error)

DecodeToken 分词器流解码