gotoon

README

GoToon

Token-Optimized Object Notation encoder/decoder for Go with intelligent nested object handling.

TOON is a compact, YAML-like format designed to reduce token usage when sending data to LLMs. This package achieves 40-60% token reduction compared to JSON while maintaining full round-trip fidelity.

Installation

go get github.com/b92c/gotoon

Quick Start

package main

import (
    "fmt"
    "github.com/b92c/gotoon"
)

func main() {
    data := map[string]any{
        "users": []map[string]any{
            {"id": 1, "name": "Alice", "role": map[string]any{"id": "admin", "level": 10}},
            {"id": 2, "name": "Bob", "role": map[string]any{"id": "user", "level": 1}},
        },
    }

    // Encode to TOON
    toon, _ := gotoon.Encode(data)
    fmt.Println(toon)

    // Decode back to map
    original, _ := gotoon.Decode(toon)
    fmt.Printf("%+v\n", original)
}

Output:

users:
  items[2]{id,name,role.id,role.level}:
    1,Alice,admin,10
    2,Bob,user,1

Why TOON?

When building MCP servers or LLM-powered applications, every token counts. JSON's verbosity wastes context window space with repeated keys and structural characters.

JSON (398 bytes):

{"orders":[{"id":"ord_1","status":"shipped","customer":{"id":"cust_1","name":"Alice"},"total":99.99},{"id":"ord_2","status":"pending","customer":{"id":"cust_2","name":"Bob"},"total":149.50}]}

TOON (186 bytes) - 53% smaller:

orders:
  items[2]{id,status,customer.id,customer.name,total}:
    ord_1,shipped,cust_1,Alice,99.99
    ord_2,pending,cust_2,Bob,149.5

Features

Nested Object Flattening

The key differentiator. Arrays containing objects with nested properties are automatically flattened using dot notation:

data := []map[string]any{
    {"id": 1, "author": map[string]any{"name": "Jane", "email": "jane@example.com"}},
    {"id": 2, "author": map[string]any{"name": "John", "email": "john@example.com"}},
}

toon, _ := gotoon.Encode(data)
// items[2]{id,author.name,author.email}:
//   1,Jane,jane@example.com
//   2,John,john@example.com

decoded, _ := gotoon.Decode(toon)
// Returns original nested structure

Multi-Level Nesting

Handles deeply nested structures:

data := []map[string]any{
    {
        "id": 1,
        "product": map[string]any{
            "name": "Widget",
            "category": map[string]any{"id": "cat_1", "name": "Electronics"},
        },
    },
}

toon, _ := gotoon.Encode(data)
// items[1]{id,product.name,product.category.id,product.category.name}:
//   1,Widget,cat_1,Electronics

Type Preservation

All scalar types are preserved through encode/decode:

data := map[string]any{
    "count":   42,
    "price":   19.99,
    "active":  true,
    "deleted": false,
    "notes":   nil,
}

decoded, _ := gotoon.Decode(gotoon.Encode(data))
// Types are preserved: int, float, bool, nil

Special Character Escaping

Commas, colons, and newlines in values are automatically escaped:

data := map[string]any{"message": "Hello, World: How are you?"}
toon, _ := gotoon.Encode(data)
// message: Hello\, World\: How are you?

Configuration

Create a custom encoder/decoder with options:

config := &gotoon.Config{
    // Arrays with fewer items use regular object format instead of tables
    MinRowsForTable: 2,

    // How deep to flatten nested objects (deeper = JSON string)
    MaxFlattenDepth: 3,

    // Escape style for special characters
    EscapeStyle: "backslash",

    // Omit values to save tokens
    Omit: []string{"null", "empty"},

    // Always skip these keys
    OmitKeys: []string{"created_at", "updated_at"},

    // Shorten verbose keys
    KeyAliases: map[string]string{
        "description":     "desc",
        "organization_id": "org_id",
    },

    // Format dates
    DateFormat: "2006-01-02",

    // Truncate long strings (adds ... suffix)
    TruncateStrings: 100,

    // Limit decimal places for floats
    NumberPrecision: 2,
}

encoder := gotoon.NewEncoder(config)
decoder := gotoon.NewDecoder(config)

toon, _ := encoder.Encode(data)
decoded, _ := decoder.Decode(toon)

Token-Saving Options

config := &gotoon.Config{
    // Omit values to save tokens: 'null', 'empty', 'false', or 'all'
    Omit: []string{"null", "empty"},

    // Always skip these keys
    OmitKeys: []string{"created_at", "updated_at"},

    // Shorten verbose keys
    KeyAliases: map[string]string{
        "description":     "desc",
        "organization_id": "org_id",
    },
}

Value Transformation

config := &gotoon.Config{
    // Format dates (time.Time objects and ISO strings)
    DateFormat: "2006-01-02",

    // Truncate long strings (adds ... suffix)
    TruncateStrings: 100,

    // Limit decimal places for floats
    NumberPrecision: 2,
}

Utility Functions

Measure Savings

data := getUsersWithRoles()

diff := gotoon.Diff(data)
// map[string]any{
//     "json_chars":      12500,
//     "toon_chars":      5200,
//     "saved_chars":     7300,
//     "savings_percent": 58.4,
// }

Encode Specific Keys Only

users := getAllUsers()

// Only include id and name, exclude email, password, etc.
toon, _ := gotoon.Only(users, []string{"id", "name"})

Use Cases

MCP Servers

Reduce token usage when returning data from MCP tool calls:

func HandleListUsers() (string, error) {
    users := db.GetUsersWithRoles(100)
    
    return gotoon.Encode(map[string]any{
        "count": len(users),
        "users": users,
    })
}

LLM Context

Pack more data into your context window:

context, _ := gotoon.Encode(map[string]any{
    "conversation":   messages,
    "user_profile":   user,
    "recent_orders":  orders,
})

response := llm.Chat([]Message{
    {Role: "system", Content: "Context:\n" + context},
    {Role: "user", Content: question},
})

API Responses

Optional TOON responses for token-conscious clients:

func HandleGetProducts(w http.ResponseWriter, r *http.Request) {
    data := db.GetProducts()
    
    if r.Header.Get("Accept") == "application/toon" {
        toon, _ := gotoon.Encode(data)
        w.Header().Set("Content-Type", "application/toon")
        w.Write([]byte(toon))
        return
    }
    
    json.NewEncoder(w).Encode(data)
}

Benchmarks

Real-world benchmarks from production applications with 17,000+ records:

Data Type	JSON	TOON	Savings
50 records (nested objects)	13,055 bytes	5,080 bytes	61%
100 records (nested objects)	26,156 bytes	10,185 bytes	61%
500 records (nested objects)	129,662 bytes	49,561 bytes	62%
1,000 records (nested objects)	258,965 bytes	98,629 bytes	62%
100 records (mixed nesting)	43,842 bytes	26,267 bytes	40%
Single object	169 bytes	124 bytes	27%

Token Impact

For a typical paginated API response (50 records):

• JSON: ~3,274 tokens
• TOON: ~1,279 tokens
• Saved: ~2,000 tokens per request

Testing

go test ./...
go test -bench=. -benchmem

Requirements

• Go 1.19+

Credits

Based on Laravel TOON by Mischa Sigtermans

License

MIT

Documentation

Overview

Package gotoon provides a Token-Optimized Object Notation (TOON) encoder and decoder. TOON is designed to reduce token usage when sending data to LLMs while maintaining full round-trip fidelity.

Index

• func Decode(toon string) (map[string]any, error)
• func Diff(data any) map[string]any
• func Encode(data any) (string, error)
• func Only(data any, keys []string) (string, error)
• type ArrayFlattener
  • func NewArrayFlattener(maxDepth int) *ArrayFlattener
  • func (f *ArrayFlattener) Flatten(items []any) *FlattenedData
  • func (f *ArrayFlattener) HasNestedObjects(items []any) bool
• type ArrayUnflattener
  • func NewArrayUnflattener() *ArrayUnflattener
  • func (u *ArrayUnflattener) Unflatten(rows [][]any, columns []string) []map[string]any
• type Config
  • func DefaultConfig() *Config
• type Decoder
  • func NewDecoder(config *Config) *Decoder
  • func (d *Decoder) Decode(toon string) (map[string]any, error)
• type Encoder
  • func NewEncoder(config *Config) *Encoder
  • func (e *Encoder) Encode(data any) (string, error)
• type FlattenedData

Functions

func Decode

func Decode(toon string) (map[string]any, error)

Decode converts a TOON format string to Go data structures using the default decoder.

func Diff

func Diff(data any) map[string]any

Diff estimates token savings between JSON and TOON formats. Returns a map with json_chars, toon_chars, saved_chars, and savings_percent.

func Encode

func Encode(data any) (string, error)

Encode converts data to TOON format using the default encoder.

func Only

func Only(data any, keys []string) (string, error)

Only encodes only specific keys from the data.

Types

type ArrayFlattener

type ArrayFlattener struct {
	// contains filtered or unexported fields
}

ArrayFlattener handles flattening of nested objects in arrays.

func NewArrayFlattener

func NewArrayFlattener(maxDepth int) *ArrayFlattener

NewArrayFlattener creates a new ArrayFlattener with the specified max depth.

func (*ArrayFlattener) Flatten

func (f *ArrayFlattener) Flatten(items []any) *FlattenedData

Flatten converts an array of objects with nested structures into a flat table format.

func (*ArrayFlattener) HasNestedObjects

func (f *ArrayFlattener) HasNestedObjects(items []any) bool

HasNestedObjects checks if any item in the array has nested objects.

type ArrayUnflattener

type ArrayUnflattener struct{}

ArrayUnflattener handles unflattening of flat table data back into nested objects.

func NewArrayUnflattener

func NewArrayUnflattener() *ArrayUnflattener

NewArrayUnflattener creates a new ArrayUnflattener.

func (*ArrayUnflattener) Unflatten

func (u *ArrayUnflattener) Unflatten(rows [][]any, columns []string) []map[string]any

Unflatten converts flat rows back into nested objects.

type Config

type Config struct {
	// MinRowsForTable is the minimum number of items required to use table format.
	// Arrays with fewer items will be encoded as regular YAML-like objects.
	MinRowsForTable int

	// MaxFlattenDepth controls how many levels deep to flatten nested objects.
	// Objects nested deeper than this will be JSON-encoded as strings.
	MaxFlattenDepth int

	// EscapeStyle determines how to escape special characters in string values.
	// Currently only "backslash" is supported.
	EscapeStyle string

	// Omit specifies which value types to omit from output.
	// Supported values: "null", "empty", "false", "all"
	Omit []string

	// OmitKeys specifies keys that should always be omitted from output.
	OmitKeys []string

	// KeyAliases maps long key names to shorter aliases to save tokens.
	KeyAliases map[string]string

	// DateFormat specifies the format for time.Time objects and ISO date strings.
	// Uses Go's time format syntax. When empty, dates are passed through as-is.
	DateFormat string

	// TruncateStrings specifies the maximum length for string values.
	// Strings exceeding this length will be truncated with "...".
	// When 0, strings are not truncated.
	TruncateStrings int

	// NumberPrecision specifies the maximum decimal places for float values.
	// When -1, floats are passed through as-is.
	NumberPrecision int
}

Config holds configuration options for TOON encoding and decoding.

func DefaultConfig

func DefaultConfig() *Config

DefaultConfig returns a Config with sensible defaults.

type Decoder

type Decoder struct {
	// contains filtered or unexported fields
}

Decoder handles decoding TOON format strings to Go data structures.

func NewDecoder

func NewDecoder(config *Config) *Decoder

NewDecoder creates a new Decoder with the given configuration.

func (*Decoder) Decode

func (d *Decoder) Decode(toon string) (map[string]any, error)

Decode converts a TOON format string to Go data structures.

type Encoder

type Encoder struct {
	// contains filtered or unexported fields
}

Encoder handles encoding Go data structures to TOON format.

func NewEncoder

func NewEncoder(config *Config) *Encoder

NewEncoder creates a new Encoder with the given configuration.

func (*Encoder) Encode

func (e *Encoder) Encode(data any) (string, error)

Encode converts data to TOON format string.

type FlattenedData

type FlattenedData struct {
	Columns []string
	Rows    [][]any
}

FlattenedData represents flattened array data with columns and rows.