tokenize

Documentation

Overview

Package tokenize provides SQL tokenization.

Index

• Constants
• func TokenTypeName(typ int) string
• type Token
• type Tokenizer
  • func NewTokenizer(sql string) *Tokenizer
  • func (t *Tokenizer) NextToken() *Token
  • func (t *Tokenizer) PeekToken() *Token
  • func (t *Tokenizer) PushBack(tok *Token)

Constants

const (
	TK_INTEGER = iota + 1
	TK_FLOAT
	TK_STRING
	TK_BLOB
	TK_ID
	TK_VARIABLE
	TK_EOF
	TK_SEMI
	TK_LP
	TK_RP
	TK_COMMA
	TK_DOT
	TK_PLUS
	TK_MINUS
	TK_STAR
	TK_SLASH
	TK_PERCENT
	TK_AMP
	TK_PIPE
	TK_LT
	TK_GT
	TK_LE
	TK_GE
	TK_EQ
	TK_NE
	TK_CONCAT
	TK_LSHIFT
	TK_RSHIFT
	TK_BITNOT
	TK_JSONEXTRACT  // -> (JSON extract as text)
	TK_JSONEXTRACT2 // ->> (JSON extract as value)
)

Token types

const (
	TK_ABORT = 100 + iota
	TK_ACTION
	TK_ADD
	TK_AFTER
	TK_ALL
	TK_ALTER
	TK_ANALYZE
	TK_AND
	TK_AS
	TK_ASC
	TK_ATTACH
	TK_BEFORE
	TK_BEGIN
	TK_BETWEEN
	TK_BY
	TK_CASCADE
	TK_CASE
	TK_CAST
	TK_CHECK
	TK_COLLATE
	TK_COLUMN
	TK_COMMIT
	TK_CONFLICT
	TK_CONSTRAINT
	TK_CREATE
	TK_CROSS
	TK_CURRENT
	TK_DATABASE
	TK_DEFAULT
	TK_DEFERRABLE
	TK_DEFERRED
	TK_DELETE
	TK_DESC
	TK_DETACH
	TK_DISTINCT
	TK_DROP
	TK_EACH
	TK_ELSE
	TK_END
	TK_ESCAPE
	TK_EXCEPT
	TK_EXCLUSIVE
	TK_EXISTS
	TK_EXPLAIN
	TK_FAIL
	TK_FOR
	TK_FOREIGN
	TK_FROM
	TK_FULL
	TK_GLOB
	TK_GROUP
	TK_HAVING
	TK_IF
	TK_IGNORE
	TK_IMMEDIATE
	TK_IN
	TK_INDEX
	TK_INITIALLY
	TK_INNER
	TK_INSERT
	TK_INSTEAD
	TK_INTERSECT
	TK_INTO
	TK_IS
	TK_ISNULL
	TK_JOIN
	TK_KEY
	TK_LAST
	TK_LEFT
	TK_LIKE
	TK_LIMIT
	TK_MATCH
	TK_NATURAL
	TK_NEXT
	TK_NO
	TK_NOT
	TK_NOTNULL
	TK_NULL
	TK_OF
	TK_OFFSET
	TK_ON
	TK_OR
	TK_ORDER
	TK_OUTER
	TK_PLAN
	TK_PRAGMA
	TK_PRIMARY
	TK_QUERY
	TK_RAISE
	TK_RECURSIVE
	TK_REFERENCES
	TK_REGEXP
	TK_REINDEX
	TK_RELEASE
	TK_RENAME
	TK_REPLACE
	TK_RESTRICT
	TK_RIGHT
	TK_ROLLBACK
	TK_ROW
	TK_SAVEPOINT
	TK_SELECT
	TK_SET
	TK_TABLE
	TK_TEMP
	TK_TEMPORARY
	TK_THEN
	TK_TIES
	TK_TO
	TK_TRANSACTION
	TK_TRIGGER
	TK_TRUNCATE
	TK_UNION
	TK_UNIQUE
	TK_UPDATE
	TK_USING
	TK_VACUUM
	TK_VALUES
	TK_VIEW
	TK_VIRTUAL
	TK_WHEN
	TK_WHERE
	TK_WITH
	TK_WITHOUT
	TK_DO
	TK_OVER
)

Exported keyword token constants for use by other packages

const (
	TK_LENGTH = 300 + iota
	TK_SUBSTR
	TK_ABS
	TK_ROUND
	TK_UPPER
	TK_LOWER
	TK_COALESCE
	TK_NULLIF
	TK_TYPEOF
	TK_IFNULL
)

Function token types (for scalar functions) - start at 300 to avoid conflicts

const (
	TK_COUNT = 310 + iota
	TK_SUM
	TK_AVG
	TK_MIN
	TK_MAX
)

Aggregate function token types - start at 310

const TK_KEYWORD_START = 100

Keyword token types start at 100

Functions

func TokenTypeName

func TokenTypeName(typ int) string

TokenTypeName returns the name of a token type.

Types

type Token

type Token struct {
	Type  int
	Text  string
	Value interface{}
	Z     string
	N     int
}

Token represents a SQL token.

type Tokenizer

type Tokenizer struct {
	// contains filtered or unexported fields
}

Tokenizer is an SQL tokenizer.

func NewTokenizer

func NewTokenizer(sql string) *Tokenizer

NewTokenizer creates a new tokenizer for the given SQL string.

func (*Tokenizer) NextToken

func (t *Tokenizer) NextToken() *Token

NextToken returns the next token from the input.

func (*Tokenizer) PeekToken

func (t *Tokenizer) PeekToken() *Token

PeekToken returns the next token without consuming it.

func (*Tokenizer) PushBack

func (t *Tokenizer) PushBack(tok *Token)

PushBack returns the last read token to the tokenizer.