Documentation
¶
Overview ¶
Package gemini implements a client for Google's Gemini API.
Not to be confused with Google's Vertex AI.
It is described at https://ai.google.dev/api/?lang=rest but the doc is weirdly organized.
Index ¶
- type Blob
- type ChatRequest
- type ChatResponse
- type ChatStreamChunkResponse
- type Client
- func (c *Client) Chat(ctx context.Context, msgs genai.Messages, opts genai.Validatable) (genai.ChatResult, error)
- func (c *Client) ChatRaw(ctx context.Context, in *ChatRequest, out *ChatResponse) error
- func (c *Client) ChatStream(ctx context.Context, msgs genai.Messages, opts genai.Validatable, ...) error
- func (c *Client) ChatStreamRaw(ctx context.Context, in *ChatRequest, out chan<- ChatStreamChunkResponse) error
- func (c *Client) ListModels(ctx context.Context) ([]genai.Model, error)
- type Content
- type FileData
- type FunctionDeclaration
- type ModalityTokenCount
- type Model
- type Part
- type SafetySetting
- type Schema
- type StructValue
- type Tool
- type ToolConfig
- type Value
Examples ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type ChatRequest ¶
type ChatRequest struct { Contents []Content `json:"contents"` Tools []Tool `json:"tools,omitzero"` ToolConfig ToolConfig `json:"toolConfig,omitzero"` SafetySettings []SafetySetting `json:"safetySettings,omitzero"` SystemInstruction Content `json:"systemInstruction,omitzero"` // https://ai.google.dev/api/generate-content?hl=en#v1beta.GenerationConfig GenerationConfig struct { StopSequences []string `json:"stopSequences,omitzero"` ResponseMimeType string `json:"responseMimeType,omitzero"` ResponseSchema Schema `json:"responseSchema,omitzero"` ResponseModalities []string `json:"responseModalities,omitzero"` CandidateCount int64 `json:"candidateCount,omitzero"` MaxOutputTokens int64 `json:"maxOutputTokens,omitzero"` Temperature float64 `json:"temperature,omitzero"` // [0, 2] TopP float64 `json:"topP,omitzero"` TopK int64 `json:"topK,omitzero"` Seed int64 `json:"seed,omitzero"` PresencePenalty float64 `json:"presencePenalty,omitzero"` FrequencyPenalty float64 `json:"frequencyPenalty,omitzero"` ResponseLogprobs bool `json:"responseLogprobs,omitzero"` Logprobs int64 `json:"logProbs,omitzero"` EnableEnhancedCivicAnswers bool `json:"enableEnhancedCivicAnswers,omitzero"` // https://ai.google.dev/api/generate-content?hl=en#SpeechConfig SpeechConfig struct { // https://ai.google.dev/api/generate-content?hl=en#VoiceConfig VoiceConfig struct { // https://ai.google.dev/api/generate-content?hl=en#PrebuiltVoiceConfig PrebuiltVoiceConfig struct { VoiceName string `json:"voiceName,omitzero"` } `json:"prebuiltVoiceConfig,omitzero"` } `json:"voiceConfig,omitzero"` } `json:"speechConfig,omitzero"` MediaResolution string `json:"mediaResolution,omitzero"` } `json:"generationConfig,omitzero"` CachedContent string `json:"cachedContent,omitzero"` }
https://ai.google.dev/api/generate-content?hl=en#text_gen_text_only_prompt-SHELL
func (*ChatRequest) Init ¶
func (c *ChatRequest) Init(msgs genai.Messages, opts genai.Validatable) error
Init initializes the provider specific completion request with the generic completion request.
type ChatResponse ¶
type ChatResponse struct { // https://ai.google.dev/api/generate-content?hl=en#v1beta.Candidate Candidates []struct { Content Content `json:"content"` // https://ai.google.dev/api/generate-content?hl=en#FinishReason FinishReason string `json:"finishReason"` // "STOP" (uppercase) // https://ai.google.dev/api/generate-content?hl=en#v1beta.SafetyRating SafetyRatings []struct { // https://ai.google.dev/api/generate-content?hl=en#v1beta.HarmCategory Category string `json:"category"` // https://ai.google.dev/api/generate-content?hl=en#HarmProbability Probability string `json:"probability"` Blocked bool `json:"blocked"` } `json:"safetyRatings"` // https://ai.google.dev/api/generate-content?hl=en#v1beta.CitationMetadata CitationMetadata struct { // https://ai.google.dev/api/generate-content?hl=en#CitationSource CitationSources []struct { StartIndex int64 `json:"startIndex"` EndIndex int64 `json:"endIndex"` URI string `json:"uri"` License string `json:"license"` } `json:"citaionSources"` } `json:"citationMetadata"` TokenCount int64 `json:"tokenCount"` // https://ai.google.dev/api/generate-content?hl=en#GroundingAttribution GroundingAttributions []struct { SourceID string `json:"sourceId"` Countent Content `json:"countent"` } `json:"groundingAttributions"` // https://ai.google.dev/api/generate-content?hl=en#GroundingMetadata GroundingMetadata struct { // https://ai.google.dev/api/generate-content?hl=en#GroundingChunk GroundingChuncks []struct { // https://ai.google.dev/api/generate-content?hl=en#Web Web struct { URI string `json:"uri"` Title string `json:"title"` } `json:"web"` } `json:"groundingChuncks"` // https://ai.google.dev/api/generate-content?hl=en#GroundingSupport GroundingSupports []struct { GroundingChunkIndices []int64 `json:"groundingChunkIndices"` ConfidenceScores []float64 `json:"confidenceScores"` // https://ai.google.dev/api/generate-content?hl=en#Segment Segment struct { PartIndex int64 `json:"partIndex"` StartIndex int64 `json:"startIndex"` EndIndex int64 `json:"endIndex"` Text string `json:"text"` } `json:"segment"` } `json:"groundingSupports"` WebSearchQueries []string `json:"webSearchQueries"` // https://ai.google.dev/api/generate-content?hl=en#SearchEntryPoint SearchEntryPoint struct { RenderedContent string `json:"renderedContent"` SDKBlob []byte `json:"sdkBlob"` // JSON encoded list of (search term,search url) results } `json:"searchEntryPoint"` // https://ai.google.dev/api/generate-content?hl=en#RetrievalMetadata RetrievalMetadata struct { GoogleSearchDynamicRetrievalScore float64 `json:"googleSearchDynamicRetrievalScore"` } `json:"retrievalMetadata"` } `json:"groundingMetadata"` AvgLogprobs float64 `json:"avgLogprobs"` LogprobsResult any `json:"logprobsResult"` Index int64 `json:"index"` } `json:"candidates"` PromptFeedback any `json:"promptFeedback,omitzero"` // https://ai.google.dev/api/generate-content?hl=en#UsageMetadata UsageMetadata struct { PromptTokenCount int64 `json:"promptTokenCount"` CachedContentTokenCount int64 `json:"cachedContentTokenCount"` CandidatesTokenCount int64 `json:"candidatesTokenCount"` ToolUsePromptTokenCount int64 `json:"toolUsePromptTokenCount"` ThoughtsTokenCount int64 `json:"thoughtsTokenCount"` TotalTokenCount int64 `json:"totalTokenCount"` PromptTokensDetails []ModalityTokenCount `json:"promptTokensDetails"` CacheTokensDetails []ModalityTokenCount `json:"cacheTokensDetails"` CandidatesTokensDetails []ModalityTokenCount `json:"candidatesTokensDetails"` ToolUsePromptTokensDetails []ModalityTokenCount `json:"toolUsePromptTokensDetails"` } `json:"usageMetadata"` ModelVersion string `json:"modelVersion"` }
https://ai.google.dev/api/generate-content?hl=en#v1beta.GenerateContentResponse
func (*ChatResponse) ToResult ¶
func (c *ChatResponse) ToResult() (genai.ChatResult, error)
type ChatStreamChunkResponse ¶
type ChatStreamChunkResponse struct { Candidates []struct { Content Content `json:"content"` FinishReason string `json:"finishReason"` // STOP } `json:"candidates"` UsageMetadata struct { CandidatesTokenCount int64 `json:"candidatesTokenCount"` PromptTokenCount int64 `json:"promptTokenCount"` TotalTokenCount int64 `json:"totalTokenCount"` PromptTokensDetails []ModalityTokenCount `json:"promptTokensDetails"` CandidatesTokensDetails []ModalityTokenCount `json:"candidatesTokensDetails"` } `json:"usageMetadata"` ModelVersion string `json:"modelVersion"` }
type Client ¶
type Client struct { // Client is exported for testing replay purposes. Client httpjson.Client // contains filtered or unexported fields }
Client implements the REST JSON based API.
func New ¶
New creates a new client to talk to Google's Gemini platform API.
If apiKey is not provided, it tries to load it from the GEMINI_API_KEY environment variable. If none is found, it returns an error. Get your API key at https://ai.google.dev/gemini-api/docs/getting-started If no model is provided, only functions that do not require a model, like ListModels, will work. To use multiple models, create multiple clients. Use one of the model from https://ai.google.dev/gemini-api/docs/models/gemini
See https://ai.google.dev/gemini-api/docs/file-prompting-strategies?hl=en for good ideas on how to prompt with images.
https://ai.google.dev/gemini-api/docs/pricing
Using large files requires a pinned model with caching support.
Supported mime types for images: https://ai.google.dev/gemini-api/docs/vision?hl=en&lang=rest#prompting-images - image/png - image/jpeg - image/webp - image/heic - image/heif
Supported mime types for videos: https://ai.google.dev/gemini-api/docs/vision?hl=en&lang=rest#technical-details-video - video/mp4 - video/mpeg - video/mov - video/avi - video/x-flv - video/mpg - video/webm - video/wmv - video/3gpp
Supported mime types for audio: https://ai.google.dev/gemini-api/docs/audio?hl=en&lang=rest#supported-formats - audio/wav - audio/mp3 - audio/aiff - audio/aac - audio/ogg - audio/flac
Supported mime types for documents: https://ai.google.dev/gemini-api/docs/document-processing?hl=en&lang=rest#technical-details - application/pdf - application/x-javascript, text/javascript - application/x-python, text/x-python - text/plain - text/html - text/css - text/md - text/csv - text/xml - text/rtf
func (*Client) Chat ¶
func (c *Client) Chat(ctx context.Context, msgs genai.Messages, opts genai.Validatable) (genai.ChatResult, error)
Example (Audio) ¶
package main import ( "context" "fmt" "log" "os" "strings" _ "embed" "github.com/maruel/genai" "github.com/maruel/genai/gemini" ) func main() { c, err := gemini.New("", "gemini-2.0-flash-lite") if err != nil { log.Fatal(err) } f, err := os.Open("testdata/mystery_word.opus") if err != nil { log.Fatal(err) } defer f.Close() msgs := genai.Messages{ { Role: genai.User, Contents: []genai.Content{ {Text: "What is the word said? Reply with only the word."}, {Document: f}, }, }, } opts := genai.ChatOptions{ Seed: 1, Temperature: 0.01, MaxTokens: 50, } resp, err := c.Chat(context.Background(), msgs, &opts) if err != nil { log.Fatal(err) } log.Printf("Raw response: %#v", resp) if len(resp.Contents) != 1 { log.Fatal("Unexpected response") } fmt.Printf("Heard: %v\n", strings.TrimRight(strings.ToLower(resp.Contents[0].Text), ".")) // This would Output: Heard: orange }
Output:
Example (PDF) ¶
package main import ( "context" "fmt" "log" "os" "strings" _ "embed" "github.com/maruel/genai" "github.com/maruel/genai/gemini" ) func main() { // Using small model for testing. // See https://ai.google.dev/gemini-api/docs/models/gemini?hl=en c, err := gemini.New("", "gemini-2.0-flash-lite") if err != nil { log.Fatal(err) } f, err := os.Open("testdata/hidden_word.pdf") if err != nil { log.Fatal(err) } defer f.Close() msgs := genai.Messages{ { Role: genai.User, Contents: []genai.Content{ {Text: "What is the word? Reply with only the word."}, {Document: f}, }, }, } opts := genai.ChatOptions{ Seed: 1, Temperature: 0.01, MaxTokens: 50, } resp, err := c.Chat(context.Background(), msgs, &opts) if err != nil { log.Fatal(err) } log.Printf("Raw response: %#v", resp) if len(resp.Contents) != 1 { log.Fatal("Unexpected response") } fmt.Printf("Hidden word in PDF: %v\n", strings.ToLower(resp.Contents[0].Text)) // This would Output: Hidden word in PDF: orange }
Output:
Example (Tool_use) ¶
package main import ( "context" "fmt" "log" "os" "strings" _ "embed" "github.com/maruel/genai" "github.com/maruel/genai/gemini" ) func main() { c, err := gemini.New("", "gemini-2.0-flash-lite") if err != nil { log.Fatal(err) } f, err := os.Open("testdata/animation.mp4") if err != nil { log.Fatal(err) } defer f.Close() msgs := genai.Messages{ { Role: genai.User, Contents: []genai.Content{ {Text: "What is the word? Call the tool hidden_word to tell me what word you saw."}, {Document: f}, }, }, } var got struct { Word string `json:"word" jsonschema:"enum=Orange,enum=Banana,enum=Apple"` } opts := genai.ChatOptions{ Seed: 1, Temperature: 0.01, MaxTokens: 50, Tools: []genai.ToolDef{ { Name: "hidden_word", Description: "A tool to state what word was seen in the video.", InputsAs: &got, }, }, } resp, err := c.Chat(context.Background(), msgs, &opts) if err != nil { log.Fatal(err) } log.Printf("Raw response: %#v", resp) // Warning: there's a bug where it returns two identical tool calls. To verify. if len(resp.ToolCalls) == 0 || resp.ToolCalls[0].Name != "hidden_word" { log.Fatal("Unexpected response") } if err := resp.ToolCalls[0].Decode(&got); err != nil { log.Fatal(err) } fmt.Printf("Saw: %v\n", strings.ToLower(got.Word)) // This would Output: Saw: banana }
Output:
Example (Vision_and_JSON) ¶
package main import ( "bytes" "context" "fmt" "log" _ "embed" "github.com/maruel/genai" "github.com/maruel/genai/gemini" ) // See the 1kib banana jpg online at // https://github.com/maruel/genai/blob/main/gemini/testdata/banana.jpg // //go:embed testdata/banana.jpg var bananaJpg []byte func main() { // Using small model for testing. // See https://ai.google.dev/gemini-api/docs/models/gemini?hl=en c, err := gemini.New("", "gemini-2.0-flash-lite") if err != nil { log.Fatal(err) } msgs := genai.Messages{ { Role: genai.User, Contents: []genai.Content{ {Text: "Is it a banana? Reply as JSON."}, {Filename: "banana.jpg", Document: bytes.NewReader(bananaJpg)}, }, }, } var got struct { Banana bool `json:"banana"` } opts := genai.ChatOptions{ Seed: 1, Temperature: 0.01, MaxTokens: 50, DecodeAs: &got, } resp, err := c.Chat(context.Background(), msgs, &opts) if err != nil { log.Fatal(err) } log.Printf("Raw response: %#v", resp) if len(resp.Contents) != 1 { log.Fatal("Unexpected response") } if err := resp.Contents[0].Decode(&got); err != nil { log.Fatal(err) } fmt.Printf("Banana: %v\n", got.Banana) // This would Output: Banana: true }
Output:
func (*Client) ChatRaw ¶
func (c *Client) ChatRaw(ctx context.Context, in *ChatRequest, out *ChatResponse) error
func (*Client) ChatStream ¶
func (c *Client) ChatStream(ctx context.Context, msgs genai.Messages, opts genai.Validatable, chunks chan<- genai.MessageFragment) error
Example ¶
package main import ( "context" "fmt" "log" "strings" _ "embed" "github.com/maruel/genai" "github.com/maruel/genai/gemini" ) func main() { c, err := gemini.New("", "gemini-2.0-flash-lite") if err != nil { log.Fatal(err) } ctx := context.Background() msgs := genai.Messages{ genai.NewTextMessage(genai.User, "Say hello. Use only one word."), } opts := genai.ChatOptions{ Seed: 1, Temperature: 0.01, MaxTokens: 50, } chunks := make(chan genai.MessageFragment) end := make(chan genai.Message, 10) go func() { var pendingMsgs genai.Messages defer func() { for _, m := range pendingMsgs { end <- m } close(end) }() for { select { case <-ctx.Done(): return case pkt, ok := <-chunks: if !ok { return } var err2 error if pendingMsgs, err2 = pkt.Accumulate(pendingMsgs); err2 != nil { end <- genai.NewTextMessage(genai.Assistant, fmt.Sprintf("Error: %v", err2)) return } } } }() err = c.ChatStream(ctx, msgs, &opts, chunks) close(chunks) var responses genai.Messages for m := range end { responses = append(responses, m) } log.Printf("Raw responses: %#v", responses) if err != nil { log.Fatal(err) } if len(responses) != 1 { log.Fatal("Unexpected responses") } resp := responses[0] if len(resp.Contents) != 1 { log.Fatal("Unexpected response") } // Normalize some of the variance. Obviously many models will still fail this test. fmt.Printf("Response: %s\n", strings.TrimRight(strings.TrimSpace(strings.ToLower(resp.Contents[0].Text)), ".!")) // This would Output: Response: hello }
Output:
func (*Client) ChatStreamRaw ¶
func (c *Client) ChatStreamRaw(ctx context.Context, in *ChatRequest, out chan<- ChatStreamChunkResponse) error
func (*Client) ListModels ¶
Example ¶
package main import ( "context" "fmt" "os" _ "embed" "github.com/maruel/genai/gemini" ) func main() { // Print something so the example runs. fmt.Println("Got models") c, err := gemini.New("", "") if err != nil { fmt.Fprintf(os.Stderr, "Couldn't connect: %v\n", err) return } models, err := c.ListModels(context.Background()) if err != nil { fmt.Printf("Failed to get models: %v\n", err) return } for _, model := range models { // The list of models will change over time. Print them to stderr so the // test doesn't capture them. fmt.Fprintf(os.Stderr, "- %s\n", model) } }
Output: Got models
type Content ¶
type Content struct { Role string `json:"role,omitzero"` // "user", "model" // Parts can be both content and tool calls. Parts []Part `json:"parts"` }
Content is the equivalent of Message for other providers. https://ai.google.dev/api/caching?hl=en#Content
type FileData ¶
type FunctionDeclaration ¶
type ModalityTokenCount ¶
type ModalityTokenCount struct { // https://ai.google.dev/api/generate-content?hl=en#v1beta.ModalityTokenCount Modality string `json:"modality"` // MODALITY_UNSPECIFIED, TEXT, IMAGE, AUDIO TokenCount int64 `json:"tokenCount"` }
https://ai.google.dev/api/generate-content?hl=en#v1beta.ModalityTokenCount
type Model ¶
type Model struct { Name string `json:"name"` BaseModelID string `json:"baseModelId"` Version string `json:"version"` DisplayName string `json:"displayName"` Description string `json:"description"` InputTokenLimit int64 `json:"inputTokenLimit"` OutputTokenLimit int64 `json:"outputTokenLimit"` SupportedGenerationMethods []string `json:"supportedGenerationMethods"` Temperature float64 `json:"temperature"` MaxTemperature float64 `json:"maxTemperature"` TopP float64 `json:"topP"` TopK int64 `json:"topK"` }
type Part ¶
type Part struct { Text string `json:"text,omitzero"` // Uploaded with /v1beta/cachedContents. Content is deleted after 1 hour. InlineData Blob `json:"inlineData,omitzero"` // https://ai.google.dev/api/caching?hl=en#FunctionCall FunctionCall struct { ID string `json:"id,omitzero"` Name string `json:"name,omitzero"` Args StructValue `json:"args,omitzero"` } `json:"functionCall,omitzero"` // https://ai.google.dev/api/caching?hl=en#FunctionResponse FunctionResponse struct { ID string `json:"id,omitzero"` Name string `json:"name,omitzero"` Response StructValue `json:"response,omitzero"` } `json:"functionResponse,omitzero"` // Uploaded with /upload/v1beta/files. Files are deleted after 2 days. FileData FileData `json:"fileData,omitzero"` // https://ai.google.dev/api/caching?hl=en#ExecutableCode ExecutableCode struct { Language string `json:"language,omitzero"` // Only PYTHON is supported as of March 2025. Code string `json:"code,omitzero"` } `json:"executableCode,omitzero"` // TODO // https://ai.google.dev/api/caching?hl=en#CodeExecutionResult CodeExecutionResult struct { Outcome string `json:"outcome,omitzero"` // One of OUTCOME_UNSPECIFIED, OUTCOME_OK, OUTCOME_FAILED, OUTCOME_DEADLINE_EXCEEDED Output string `json:"output,omitzero"` } `json:"codeExecutionResult,omitzero"` // TODO }
Part is a union that only has one of the field set. Part is the equivalent of Content for other providers.
type SafetySetting ¶
type SafetySetting struct { Category string `json:"category"` // https://ai.google.dev/api/generate-content?hl=en#v1beta.HarmCategory Threshold int64 `json:"threshold"` // https://ai.google.dev/api/generate-content?hl=en#HarmBlockThreshold }
https://ai.google.dev/api/generate-content?hl=en#v1beta.SafetySetting
type Schema ¶
type Schema struct { // https://ai.google.dev/api/caching?hl=en#Type // https://spec.openapis.org/oas/v3.0.3#data-types Type string `json:"type,omitzero"` // TYPE_UNSPECIFIED, STRING, NUMBER, INTEGER, BOOLEAN, ARRAY, OBJECT Format string `json:"format,omitzero"` // NUMBER type: float, double for INTEGER type: int32, int64 for STRING type: enum, date-time Description string `json:"description,omitzero"` // Nullable bool `json:"nullable,omitzero"` // Enum []string `json:"enum,omitzero"` // STRING MaxItems int64 `json:"maxItems,omitzero"` // ARRAY MinItems int64 `json:"minItems,omitzero"` // ARRAY Properties map[string]Schema `json:"properties,omitzero"` // OBJECT Required []string `json:"required,omitzero"` // OBJECT PropertyOrdering []string `json:"propertyOrdering,omitzero"` // OBJECT Items *Schema `json:"items,omitzero"` // ARRAY }
https://ai.google.dev/api/caching?hl=en#Schema
func (*Schema) FromJSONSchema ¶
func (s *Schema) FromJSONSchema(j *jsonschema.Schema)
type Tool ¶
type Tool struct { FunctionDeclarations []FunctionDeclaration `json:"functionDeclarations,omitzero"` // https://ai.google.dev/api/caching?hl=en#GoogleSearchRetrieval GoogleSearchRetrieval struct { // https://ai.google.dev/api/caching?hl=en#DynamicRetrievalConfig DynamicRetrievalConfig struct { // https://ai.google.dev/api/caching?hl=en#Mode Mode string `json:"mode,omitzero"` // MODE_UNSPECIFIED, MODE_DYNAMIC DynamicThreshold float64 `json:"dynamicThreshold,omitzero"` } `json:"dynamicRetrievalConfig,omitzero"` } `json:"googleSearchRetrieval,omitzero"` CodeExecution struct{} `json:"codeExecution,omitzero"` GoogleSearch struct{} `json:"googleSearch,omitzero"` }
type ToolConfig ¶
type ToolConfig struct { // https://ai.google.dev/api/caching?hl=en#FunctionCallingConfig FunctionCallingConfig struct { // https://ai.google.dev/api/caching?hl=en#Mode_1 Mode string `json:"mode,omitzero"` // MODE_UNSPECIFIED, AUTO, ANY, NONE AllowedFunctionNames []string `json:"allowedFunctionNames,omitzero"` } `json:"functionCallingConfig,omitzero"` }
type Value ¶
type Value struct { NullValue int64 `json:"null_value,omitzero"` NumberValue float64 `json:"number_value,omitzero"` StringValue string `json:"string_value,omitzero"` BoolValue bool `json:"bool_value,omitzero"` StructValue StructValue `json:"struct_value,omitzero"` ListValue []Value `json:"list_value,omitzero"` }
https://protobuf.dev/reference/protobuf/google.protobuf/#value