gpt2

README

go-ggml-transformers.cpp

ggml golang bindings to run transformers.

The go-llama.cpp bindings are high level, as such most of the work is kept into the C/C++ code to avoid any extra computational cost, be more performant and lastly ease out maintenance, while keeping the usage as simple as possible.

Check out this and this write-ups which summarize the impact of a low-level interface which calls C functions from Go.

If you are looking for an high-level OpenAI compatible API, check out here.

Usage

Note: This repository uses git submodules

Clone the repository locally:

git clone --recurse-submodules https://github.com/go-skynet/go-ggml-transformers.cpp

To build the bindings locally, run:

cd go-ggml-transformers.cpp
make libtransformers.a

Now you can run the example with:

LIBRARY_PATH=$PWD C_INCLUDE_PATH=$PWD go run ./examples -m "/model/path/here" -t 14

Enjoy!

The documentation is available here and the full example code is here.

License

MIT

Documentation

Index

• type Dolly
  • func NewDolly(model string) (*Dolly, error)
  • func (l *Dolly) Free()
  • func (l *Dolly) Predict(text string, opts ...PredictOption) (string, error)
• type Falcon
  • func NewFalcon(model string) (*Falcon, error)
  • func (l *Falcon) Free()
  • func (l *Falcon) Predict(text string, opts ...PredictOption) (string, error)
• type GPT2
  • func New(model string) (*GPT2, error)
  • func (l *GPT2) Free()
  • func (l *GPT2) Predict(text string, opts ...PredictOption) (string, error)
• type GPTJ
  • func NewGPTJ(model string) (*GPTJ, error)
  • func (l *GPTJ) Free()
  • func (l *GPTJ) Predict(text string, opts ...PredictOption) (string, error)
• type GPTNeoX
  • func NewGPTNeoX(model string) (*GPTNeoX, error)
  • func (l *GPTNeoX) Free()
  • func (l *GPTNeoX) Predict(text string, opts ...PredictOption) (string, error)
• type MPT
  • func NewMPT(model string) (*MPT, error)
  • func (l *MPT) Free()
  • func (l *MPT) Predict(text string, opts ...PredictOption) (string, error)
• type PredictOption
  • func SetBatch(size int) PredictOption
  • func SetSeed(seed int) PredictOption
  • func SetTemperature(temp float64) PredictOption
  • func SetThreads(threads int) PredictOption
  • func SetTokens(tokens int) PredictOption
  • func SetTopK(topk int) PredictOption
  • func SetTopP(topp float64) PredictOption
• type PredictOptions
  • func NewPredictOptions(opts ...PredictOption) PredictOptions
• type Replit
  • func NewReplit(model string) (*Replit, error)
  • func (l *Replit) Free()
  • func (l *Replit) Predict(text string, opts ...PredictOption) (string, error)
• type Starcoder
  • func NewStarcoder(model string) (*Starcoder, error)
  • func (l *Starcoder) Free()
  • func (l *Starcoder) Predict(text string, opts ...PredictOption) (string, error)

Types

type Dolly

type Dolly struct {
	// contains filtered or unexported fields
}

func NewDolly

func NewDolly(model string) (*Dolly, error)

func (*Dolly) Free

func (l *Dolly) Free()

func (*Dolly) Predict

func (l *Dolly) Predict(text string, opts ...PredictOption) (string, error)

type Falcon

type Falcon struct {
	// contains filtered or unexported fields
}

func NewFalcon

func NewFalcon(model string) (*Falcon, error)

func (*Falcon) Free

func (l *Falcon) Free()

func (*Falcon) Predict

func (l *Falcon) Predict(text string, opts ...PredictOption) (string, error)

type GPT2

type GPT2 struct {
	// contains filtered or unexported fields
}

func New

func New(model string) (*GPT2, error)

func (*GPT2) Free

func (l *GPT2) Free()

func (*GPT2) Predict

func (l *GPT2) Predict(text string, opts ...PredictOption) (string, error)

type GPTJ

type GPTJ struct {
	// contains filtered or unexported fields
}

func NewGPTJ

func NewGPTJ(model string) (*GPTJ, error)

func (*GPTJ) Free

func (l *GPTJ) Free()

func (*GPTJ) Predict

func (l *GPTJ) Predict(text string, opts ...PredictOption) (string, error)

type GPTNeoX

type GPTNeoX struct {
	// contains filtered or unexported fields
}

func NewGPTNeoX

func NewGPTNeoX(model string) (*GPTNeoX, error)

func (*GPTNeoX) Free

func (l *GPTNeoX) Free()

func (*GPTNeoX) Predict

func (l *GPTNeoX) Predict(text string, opts ...PredictOption) (string, error)

type MPT

type MPT struct {
	// contains filtered or unexported fields
}

func NewMPT

func NewMPT(model string) (*MPT, error)

func (*MPT) Free

func (l *MPT) Free()

func (*MPT) Predict

func (l *MPT) Predict(text string, opts ...PredictOption) (string, error)

type PredictOption

type PredictOption func(p *PredictOptions)

func SetBatch

func SetBatch(size int) PredictOption

SetBatch sets the batch size.

func SetSeed

func SetSeed(seed int) PredictOption

SetSeed sets the random seed for sampling text generation.

func SetTemperature

func SetTemperature(temp float64) PredictOption

SetTemperature sets the temperature value for text generation.

func SetThreads

func SetThreads(threads int) PredictOption

SetThreads sets the number of threads to use for text generation.

func SetTokens

func SetTokens(tokens int) PredictOption

SetTokens sets the number of tokens to generate.

func SetTopK

func SetTopK(topk int) PredictOption

SetTopK sets the value for top-K sampling.

func SetTopP

func SetTopP(topp float64) PredictOption

SetTopP sets the value for nucleus sampling.

type PredictOptions

type PredictOptions struct {
	Seed, Threads, Tokens, TopK, Batch int
	TopP, Temperature                  float64
}

var DefaultOptions PredictOptions = PredictOptions{
	Seed:        -1,
	Threads:     4,
	Tokens:      200,
	TopK:        40,
	TopP:        0.90,
	Temperature: 0.96,
	Batch:       9,
}

func NewPredictOptions

func NewPredictOptions(opts ...PredictOption) PredictOptions

Create a new PredictOptions object with the given options.

type Replit

type Replit struct {
	// contains filtered or unexported fields
}

func NewReplit

func NewReplit(model string) (*Replit, error)

func (*Replit) Free

func (l *Replit) Free()

func (*Replit) Predict

func (l *Replit) Predict(text string, opts ...PredictOption) (string, error)

type Starcoder

type Starcoder struct {
	// contains filtered or unexported fields
}

func NewStarcoder

func NewStarcoder(model string) (*Starcoder, error)

func (*Starcoder) Free

func (l *Starcoder) Free()

func (*Starcoder) Predict

func (l *Starcoder) Predict(text string, opts ...PredictOption) (string, error)