streaming

Documentation

Overview

Package streaming provides a streaming interface for LLMs.

This package implements utilities for handling streaming responses from language models, allowing for real-time processing of generated content. It supports five types of chunks: text chunks for regular content, reasoning chunks for step-by-step thinking processes, tool call chunks for function/tool invocations, none chunks as a placeholder, and done chunks to signal the end of a streaming session.

The streaming interface uses a callback-based approach where each chunk is processed as it arrives, enabling applications to display partial results and interactive responses. This is particularly useful for long-form content generation, reasoning steps visualization, and tool-using agents.

Basic usage involves registering a callback function that handles chunks as they arrive:

streamingFunc := func(ctx context.Context, chunk streaming.Chunk) error {
	switch chunk.Type {
	case streaming.ChunkTypeNone:
		// It's not expected to receive this chunk type
		// but it's not an error to receive it
		return nil
	case streaming.ChunkTypeText:
		// Process text content
		fmt.Print(chunk.Content)
	case streaming.ChunkTypeReasoning:
		// Process reasoning/thinking content
		fmt.Print(chunk.ReasoningContent)
	case streaming.ChunkTypeToolCall:
		// Process tool call
		fmt.Printf("Tool call: %s\n", chunk.ToolCall.String())
	case streaming.ChunkTypeDone:
		// Streaming is done - this signals the end of the stream
		// and can be used to finalize processing
		fmt.Println("Streaming is done")
	}
	return nil
}

The callback function can be passed to LLM implementations that support streaming. When the LLM finishes generating all content, it should call the streaming.CallWithDone function to signal the end of the streaming session, allowing consumers to perform any necessary cleanup or finalization.

Index

• Variables
• func AppendToolCall(src ToolCall, dst *ToolCall)
• func CallWithDone(ctx context.Context, cb Callback) error
• func CallWithReasoning(ctx context.Context, cb Callback, reasoning string) error
• func CallWithText(ctx context.Context, cb Callback, text string) error
• func CallWithToolCall(ctx context.Context, cb Callback, toolCall ToolCall) error
• type Callback
• type Chunk
  • func NewDoneChunk() Chunk
  • func NewReasoningChunk(reasoning string) Chunk
  • func NewTextChunk(text string) Chunk
  • func NewToolCallChunk(toolCall ToolCall) Chunk
  • func (c *Chunk) String() string
• type ChunkType
• type ToolCall
  • func NewToolCall(id, name, arguments string) ToolCall
  • func (t *ToolCall) Parse() (map[string]any, error)
  • func (t *ToolCall) String() string

Variables

var (
	ErrToolCallIDRequired   = errors.New("tool call id is required")
	ErrToolCallNameRequired = errors.New("tool call name is required")
)

Functions

func AppendToolCall

func AppendToolCall(src ToolCall, dst *ToolCall)

func CallWithDone

func CallWithDone(ctx context.Context, cb Callback) error

func CallWithReasoning

func CallWithReasoning(ctx context.Context, cb Callback, reasoning string) error

func CallWithText

func CallWithText(ctx context.Context, cb Callback, text string) error

func CallWithToolCall

func CallWithToolCall(ctx context.Context, cb Callback, toolCall ToolCall) error

Types

type Callback

type Callback func(ctx context.Context, chunk Chunk) error

type Chunk

type Chunk struct {
	Type             ChunkType `json:"type"`
	Content          string    `json:"content"`
	ReasoningContent string    `json:"reasoning_content"`
	ToolCall         ToolCall  `json:"tool_call"`
}

func NewDoneChunk

func NewDoneChunk() Chunk

func NewReasoningChunk

func NewReasoningChunk(reasoning string) Chunk

func NewTextChunk

func NewTextChunk(text string) Chunk

func NewToolCallChunk

func NewToolCallChunk(toolCall ToolCall) Chunk

func (*Chunk) String

func (c *Chunk) String() string

type ChunkType

type ChunkType string

const (
	ChunkTypeNone      ChunkType = ""
	ChunkTypeText      ChunkType = "text"
	ChunkTypeReasoning ChunkType = "reasoning"
	ChunkTypeToolCall  ChunkType = "tool_call"
	ChunkTypeDone      ChunkType = "done"
)

type ToolCall

type ToolCall struct {
	ID        string `json:"id"`
	Name      string `json:"name"`
	Arguments string `json:"arguments"`
}

func NewToolCall

func NewToolCall(id, name, arguments string) ToolCall

func (*ToolCall) Parse

func (t *ToolCall) Parse() (map[string]any, error)

func (*ToolCall) String

func (t *ToolCall) String() string