jsontk

README

jsontk

from Top Gear S29E2

又不是不能用，你就说比标准库快不快吧

Features

data := []byte(`{"tokenize":{"json":true,"into":["parts",1.1]}}`)

// Tokenize
res, _ := Tokenize(data)
for _, tk := range res.store {
    fmt.Printf("%s->%s\n", tk.Type.String(), string(tk.Value))
}
// Iterate
res, _ := Iterate(data, func(typ TokenType, idx, len int) {
    // emits same result as `Tokenize`, except with "idx" and "len" information
})
// Validate
err := Validate(data)

// Iterator
var iter Iterator // can be reused
iter.Reset(data)
...

Correctness

tested with https://github.com/nst/JSONTestSuite using demonstrated code:

• Tokenize API

func walk(tks *jsontk.JSON) {
    var err error
    switch tks.Type() {
    case jsontk.BEGIN_OBJECT:
        for _, k := range tks.Keys() {
            walk(tks.Get(k))
        }
    case jsontk.BEGIN_ARRAY:
        for i := tks.Len(); i > 0; i-- {
            walk(tks.Index(i - 1))
        }
    case jsontk.NUMBER:
        _, err = tks.Float64()
    case jsontk.STRING:
        _, err = tks.String()
    }
    if err != nil {
        os.Exit(1)
    }
}

tks, err := jsontk.Tokenize(b)
//fmt.Println(f)
if err != nil {
    os.Exit(1)
}
walk(tks)

• Iterator API

func walk(iter *jsontk.Iterator) {
    var tk jsontk.Token
    switch iter.Peek() {
    case jsontk.BEGIN_OBJECT:
        iter.NextObject(func(key *jsontk.Token) bool {
            _, ok := key.Unquote()
            if !ok {
                os.Exit(1)
            }
            walk(iter)
            return true
        })
    case jsontk.BEGIN_ARRAY:
        iter.NextArray(func(idx int) bool {
            walk(iter)
            return true
        })
    case jsontk.STRING:
        _, ok := iter.NextToken(&tk).Unquote()
        if !ok {
            os.Exit(1)
        }
    case jsontk.NUMBER:
        _, err := iter.NextToken(&tk).Number().Float64()
        if err != nil {
            os.Exit(1)
        }
    case jsontk.INVALID:
        os.Exit(1)
    default:
        iter.Skip()
    }
}

var iter jsontk.Iterator
iter.Reset(b)
walk(&iter)
if iter.Error != nil {
    os.Exit(1)
}
_, _, l := iter.Next()
if l != 0 {
    os.Exit(1)
}
os.Exit(0)

the results only shows the difference between standard library and jsontk, the succeeded cases are not shown.

Warning / Disclaimer

EXPERIMENTAL

This library is solely designed to extract payload fields in an insecure manner. You might not want to use this library unless you understand what you're doing

Documentation

Index

• Variables
• func Iterate(s []byte, cb func(typ TokenType, idx, len int)) error
• func Patch(data []byte, path string, f func([]byte) []byte) ([]byte, int)
• type Iterator
  • func (iter *Iterator) Next() (TokenType, int, int)
  • func (iter *Iterator) NextArray(cb func(idx int) bool) error
  • func (iter *Iterator) NextObject(cb func(key *Token) bool) error
  • func (iter *Iterator) NextToken(t *Token) *Token
  • func (iter *Iterator) Peek() TokenType
  • func (iter *Iterator) Reset(data []byte)
  • func (iter *Iterator) Select(path string, cb func(iter *Iterator)) error
  • func (iter *Iterator) Skip() (TokenType, int, int)
  • func (iter *Iterator) Validate() error
• type Token
  • func (t *Token) AppendTo(data []byte) []byte
  • func (j *Token) Bool() bool
  • func (j *Token) EqualString(s string) bool
  • func (j *Token) Number() json.Number
  • func (j *Token) String() string
  • func (j *Token) Unquote() (string, bool)
• type TokenType
  • func (t TokenType) String() string

Variables

var (
	ErrPanic              = errors.New("panic occurred")
	ErrUnexpectedSep      = errors.New("invalid separator")
	ErrEarlyEOF           = errors.New("early EOF")
	ErrInterrupt          = errors.New("interrupted by user")
	ErrUnexpectedToken    = errors.New("invalid TokenType")
	ErrInvalidParentheses = errors.New("invalid parentheses")
	ErrStandardViolation  = errors.New("json not compliant to RFC8259") // for some simple validations
	ErrInvalidJsonpath    = errors.New("invalid jsonpath")
)

Functions

func Iterate

func Iterate(s []byte, cb func(typ TokenType, idx, len int)) error

func Patch

func Patch(data []byte, path string, f func([]byte) []byte) ([]byte, int)

Patch API is currently unstable

Types

type Iterator

type Iterator struct {
	Error error
	// contains filtered or unexported fields
}

func (*Iterator) Next

func (iter *Iterator) Next() (TokenType, int, int)

func (*Iterator) NextArray

func (iter *Iterator) NextArray(cb func(idx int) bool) error

func (*Iterator) NextObject

func (iter *Iterator) NextObject(cb func(key *Token) bool) error

NextObject iterates over the next value as an object, assuming that it is one. One MUST be aware that the "key" callback parameter is only valid before next call to ANY method on Iterator

func (*Iterator) NextToken

func (iter *Iterator) NextToken(t *Token) *Token

func (*Iterator) Peek

func (iter *Iterator) Peek() TokenType

func (*Iterator) Reset

func (iter *Iterator) Reset(data []byte)

func (*Iterator) Select

func (iter *Iterator) Select(path string, cb func(iter *Iterator)) error

func (*Iterator) Skip

func (iter *Iterator) Skip() (TokenType, int, int)

func (*Iterator) Validate

func (iter *Iterator) Validate() error

type Token

type Token struct {
	Type  TokenType
	Value []byte
}

func (*Token) AppendTo

func (t *Token) AppendTo(data []byte) []byte

func (*Token) Bool

func (j *Token) Bool() bool

func (*Token) EqualString

func (j *Token) EqualString(s string) bool

func (*Token) Number

func (j *Token) Number() json.Number

func (*Token) String

func (j *Token) String() string

String behaves the same way as [Unquote] but not check for results, returns empty string on invalid results

Warning: the returning string is valid only if underlying input byte slice doesn't change

func (*Token) Unquote

func (j *Token) Unquote() (string, bool)

Unquote unquotes the underlying value as quoted string, and returns if it's successfully unquoted

Warning: the returning string is valid only if underlying input byte slice doesn't change

type TokenType

type TokenType uint8

const (
	INVALID TokenType = iota
	BEGIN_OBJECT
	END_OBJECT
	BEGIN_ARRAY
	END_ARRAY
	KEY
	STRING
	NUMBER
	BOOLEAN
	NULL
)

func (TokenType) String

func (t TokenType) String() string