ast

package
v0.8.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 23, 2018 License: Apache-2.0 Imports: 15 Imported by: 59

Documentation

Overview

Package ast is an interface for Abstract Syntax Tree parser

Index

Constants

View Source 
const (
	TokenTypeWhitespace       = 0
	TokenTypeWord             = 1
	TokenTypeQuote            = 2
	TokenTypeBacktickQuote    = 3
	TokenTypeReserved         = 4
	TokenTypeReservedToplevel = 5
	TokenTypeReservedNewline  = 6
	TokenTypeBoundary         = 7
	TokenTypeComment          = 8
	TokenTypeBlockComment     = 9
	TokenTypeNumber           = 10
	TokenTypeError            = 11
	TokenTypeVariable         = 12
)

TokenType

Variables

View Source 
var RewriteRules = []Rule{
	{
		Name:        "dml2select",
		Description: "将数据库更新请求转换为只读查询请求，便于执行EXPLAIN",
		Original:    "DELETE FROM film WHERE length > 100",
		Suggest:     "select * from film where length > 100",
		Func:        (*Rewrite).RewriteDML2Select,
	},
	{
		Name:        "star2columns",
		Description: "为SELECT *补全表的列信息",
		Original:    "SELECT * FROM film",
		Suggest:     "select film.film_id, film.title from film",
		Func:        (*Rewrite).RewriteStar2Columns,
	},
	{
		Name:        "insertcolumns",
		Description: "为INSERT补全表的列信息",
		Original:    "insert into film values(1,2,3,4,5)",
		Suggest:     "insert into film(film_id, title, description, release_year, language_id) values (1, 2, 3, 4, 5)",
		Func:        (*Rewrite).RewriteInsertColumns,
	},
	{
		Name:        "having",
		Description: "将查询的HAVING子句改写为WHERE中的查询条件",
		Original:    "SELECT state, COUNT(*) FROM Drivers GROUP BY state HAVING state IN ('GA', 'TX') ORDER BY state",
		Suggest:     "select state, COUNT(*) from Drivers where state in ('GA', 'TX') group by state order by state asc",
		Func:        (*Rewrite).RewriteHaving,
	},
	{
		Name:        "orderbynull",
		Description: "如果GROUP BY语句不指定ORDER BY条件会导致无谓的排序产生，如果不需要排序建议添加ORDER BY NULL",
		Original:    "SELECT sum(col1) FROM tbl GROUP BY col",
		Suggest:     "select sum(col1) from tbl group by col order by null",
		Func:        (*Rewrite).RewriteAddOrderByNull,
	},
	{
		Name:        "unionall",
		Description: "可以接受重复的时间，使用UNION ALL替代UNION以提高查询效率",
		Original:    "select country_id from city union select country_id from country",
		Suggest:     "select country_id from city union all select country_id from country",
		Func:        (*Rewrite).RewriteUnionAll,
	},
	{
		Name:        "or2in",
		Description: "将同一列不同条件的OR查询转写为IN查询",
		Original:    "select country_id from city where col1 = 1 or (col2 = 1 or col2 = 2 ) or col1 = 3;",
		Suggest:     "select country_id from city where (col2 in (1, 2)) or col1 in (1, 3);",
		Func:        (*Rewrite).RewriteOr2In,
	},
	{
		Name:        "innull",
		Description: "如果IN条件中可能有NULL值而又想匹配NULL值时，建议添加OR col IS NULL",
		Original:    "暂不支持",
		Suggest:     "暂不支持",
		Func:        (*Rewrite).RewriteInNull,
	},

	{
		Name:        "or2union",
		Description: "将不同列的OR查询转为UNION查询，建议结合unionall重写策略一起使用",
		Original:    "暂不支持",
		Suggest:     "暂不支持",
		Func:        (*Rewrite).RewriteOr2Union,
	},
	{
		Name:        "dmlorderby",
		Description: "删除DML更新操作中无意义的ORDER BY",
		Original:    "DELETE FROM tbl WHERE col1=1 ORDER BY col",
		Suggest:     "delete from tbl where col1 = 1",
		Func:        (*Rewrite).RewriteRemoveDMLOrderBy,
	},

	{
		Name:        "sub2join",
		Description: "将子查询转换为JOIN查询",
		Original:    "暂不支持",
		Suggest:     "暂不支持",
		Func:        (*Rewrite).RewriteSubQuery2Join,
	},
	{
		Name:        "join2sub",
		Description: "将JOIN查询转换为子查询",
		Original:    "暂不支持",
		Suggest:     "暂不支持",
		Func:        (*Rewrite).RewriteJoin2SubQuery,
	},
	{
		Name:        "distinctstar",
		Description: "DISTINCT *对有主键的表没有意义，可以将DISTINCT删掉",
		Original:    "SELECT DISTINCT * FROM film;",
		Suggest:     "SELECT * FROM film",
		Func:        (*Rewrite).RewriteDistinctStar,
	},
	{
		Name:        "standard",
		Description: "SQL标准化，如：关键字转换为小写",
		Original:    "SELECT sum(col1) FROM tbl GROUP BY 1;",
		Suggest:     "select sum(col1) from tbl group by 1",
		Func:        (*Rewrite).RewriteStandard,
	},
	{
		Name:        "mergealter",
		Description: "合并同一张表的多条ALTER语句",
		Original:    "ALTER TABLE t2 DROP COLUMN c;ALTER TABLE t2 DROP COLUMN d;",
		Suggest:     "ALTER TABLE t2 DROP COLUMN c, DROP COLUMN d;",
	},
	{
		Name:        "alwaystrue",
		Description: "删除无用的恒真判断条件",
		Original:    "SELECT count(col) FROM tbl where 'a'= 'a' or ('b' = 'b' and a = 'b');",
		Suggest:     "select count(col) from tbl where (a = 'b');",
		Func:        (*Rewrite).RewriteAlwaysTrue,
	},
	{
		Name:        "countstar",
		Description: "不建议使用COUNT(col)或COUNT(常量)，建议改写为COUNT(*)",
		Original:    "SELECT count(col) FROM tbl GROUP BY 1;",
		Suggest:     "SELECT count(*) FROM tbl GROUP BY 1;",
		Func:        (*Rewrite).RewriteCountStar,
	},
	{
		Name:        "innodb",
		Description: "建表时建议使用InnoDB引擎，非InnoDB引擎表自动转InnoDB",
		Original:    "CREATE TABLE t1(id bigint(20) NOT NULL AUTO_INCREMENT);",
		Suggest:     "create table t1 (\n\tid bigint(20) not null auto_increment\n) ENGINE=InnoDB;",
		Func:        (*Rewrite).RewriteInnoDB,
	},
	{
		Name:        "autoincrement",
		Description: "将autoincrement初始化为1",
		Original:    "CREATE TABLE t1(id bigint(20) NOT NULL AUTO_INCREMENT) ENGINE=InnoDB AUTO_INCREMENT=123802;",
		Suggest:     "create table t1(id bigint(20) not null auto_increment) ENGINE=InnoDB auto_increment=1;",
		Func:        (*Rewrite).RewriteAutoIncrement,
	},
	{
		Name:        "intwidth",
		Description: "整型数据类型修改默认显示宽度",
		Original:    "create table t1 (id int(20) not null auto_increment) ENGINE=InnoDB;",
		Suggest:     "create table t1 (id int(10) not null auto_increment) ENGINE=InnoDB;",
		Func:        (*Rewrite).RewriteIntWidth,
	},
	{
		Name:        "truncate",
		Description: "不带WHERE条件的DELETE操作建议修改为TRUNCATE",
		Original:    "DELETE FROM tbl",
		Suggest:     "truncate table tbl",
		Func:        (*Rewrite).RewriteTruncate,
	},
	{
		Name:        "rmparenthesis",
		Description: "去除没有意义的括号",
		Original:    "select col from table where (col = 1);",
		Suggest:     "select col from table where col = 1;",
		Func:        (*Rewrite).RewriteRmParenthesis,
	},

	{
		Name:        "delimiter",
		Description: "补全DELIMITER",
		Original:    "use sakila",
		Suggest:     "use sakila;",
		Func:        (*Rewrite).RewriteDelimiter,
	},
}

RewriteRules SQL重写规则，注意这个规则是有序的，先后顺序不能乱

View Source 
var TokenString = map[int]string{}/* 216 elements not displayed */

TokenString sqlparser tokens

Functions

func AlterAffectTable 

func AlterAffectTable(stmt sqlparser.Statement) string

AlterAffectTable 获取ALTER影响的库表名，返回：`db`.`table`

func Compress 

func Compress(sql string) string

Compress compress sql this method is inspired by eversql.com

func FindAllCols 

func FindAllCols(node sqlparser.SQLNode, targets ...string) []*common.Column

FindAllCols 获取AST中某个节点下所有的columns

func FindAllCondition 

func FindAllCondition(node sqlparser.SQLNode) []interface{}

FindAllCondition 获取AST中所有的condition条件

func FindColumn 

func FindColumn(node sqlparser.SQLNode) []*common.Column

FindColumn 从传入的node中获取所有可能加索引的的column信息

func FindEQColsInJoinCond 

func FindEQColsInJoinCond(node sqlparser.SQLNode) []*common.Column

FindEQColsInJoinCond 获取 join condition 中应转为whereEQ条件的列

func FindEQColsInWhere 

func FindEQColsInWhere(node sqlparser.SQLNode) []*common.Column

FindEQColsInWhere 获取等值条件信息 将所有值得加索引的condition条件信息进行过滤

func FindGroupByCols 

func FindGroupByCols(node sqlparser.SQLNode) []*common.Column

FindGroupByCols 获取groupBy中可能需要加索引的列信息

func FindINEQColsInJoinCond 

func FindINEQColsInJoinCond(node sqlparser.SQLNode) []*common.Column

FindINEQColsInJoinCond 获取 join condition 中应转为whereINEQ条件的列

func FindINEQColsInWhere 

func FindINEQColsInWhere(node sqlparser.SQLNode) []*common.Column

FindINEQColsInWhere 获取非等值条件中可能需要加索引的列 将所有值得加索引的condition条件信息进行过滤 TODO: 将where条件中隐含的join条件合并到join condition中

func FindJoinCols 

func FindJoinCols(node sqlparser.SQLNode) [][]*common.Column

FindJoinCols 获取 join condition 中使用到的列（必须是 `列 operator 列` 的情况。 如果列对应的值或是function，则应该移到where condition中） 某些where条件隐含在Join条件中（INNER JOIN）

func FindJoinTable 

func FindJoinTable(node sqlparser.SQLNode, meta common.Meta) common.Meta

FindJoinTable 获取 Join 中需要添加索引的表 join 优化添加索引分为三种类型：1. inner join, 2. left join, 3.right join 针对三种优化类型，需要三种不同的索引添加方案: 1. inner join 需要对 join 左右的表添加索引 2. left join 由于左表为全表扫描，需要对右表的关联列添加索引。 3. right join 与 left join 相反，需要对左表的关联列添加索引。 以上添加索引的策略前提为join的表为实体表而非临时表。

func FindOrderByCols 

func FindOrderByCols(node sqlparser.SQLNode) []*common.Column

FindOrderByCols 为索引优化获取orderBy中可能添加索引的列信息

func FindSubquery 

func FindSubquery(depth int, node sqlparser.SQLNode, queries ...string) []string

FindSubquery 拆分subquery，获取最深层的subquery 为索引优化获取subquery中包含的列信息

func FindWhereEQ 

func FindWhereEQ(node sqlparser.SQLNode) []*common.Column

FindWhereEQ 找到Where中的等值条件

func FindWhereINEQ 

func FindWhereINEQ(node sqlparser.SQLNode) []*common.Column

FindWhereINEQ 找到Where条件中的非等值条件

func GetMeta 

func GetMeta(stmt sqlparser.Statement, meta common.Meta) common.Meta

GetMeta 获取元数据信息，构建到db->table层级。 从 SQL 或 Statement 中获取表信息，并返回。当 meta 不为 nil 时，返回值会将新老 meta 合并去重

func GetSubqueryDepth 

func GetSubqueryDepth(node sqlparser.SQLNode) int

GetSubqueryDepth 获取一条SQL的嵌套深度

func GetTableFromExprs 

func GetTableFromExprs(exprs sqlparser.TableExprs, metas ...common.Meta) common.Meta

GetTableFromExprs 从sqlparser.Exprs中获取所有的库表

func IsMysqlKeyword 

func IsMysqlKeyword(name string) bool

IsMysqlKeyword 判断是否是关键字

func LeftNewLines 

func LeftNewLines(buf []byte) int

LeftNewLines cal left new lines in space

func ListRewriteRules 

func ListRewriteRules(rules []Rule)

ListRewriteRules 打印SQL重写规则

func MergeAlterTables 

func MergeAlterTables(sqls ...string) map[string]string

MergeAlterTables mergealter: 将同一张表的多条ALTER语句合成一条ALTER语句 @input: sql, alter string @output: [[db.]table]sql, 如果找不到DB，key为表名；如果找得到DB，key为db.table

func MysqlEscapeString 

func MysqlEscapeString(source string) (string, error)

MysqlEscapeString mysql_real_escape_string https://github.com/liule/golang_escape

func NewLines 

func NewLines(buf []byte) int

NewLines cal all new lines

func Pretty 

func Pretty(sql string, method string) (output string)

Pretty 格式化输出SQL

func PrintPrettyStmtNode 

func PrintPrettyStmtNode(sql, charset, collation string)

PrintPrettyStmtNode 打印TiParse语法树

func RewriteRuleMatch 

func RewriteRuleMatch(name string) bool

RewriteRuleMatch 检查重写规则是否生效

func SplitStatement 

func SplitStatement(buf []byte, delimiter []byte) (string, []byte)

SplitStatement SQL切分

func TiParse 

func TiParse(sql, charset, collation string) ([]ast.StmtNode, error)

TiParse TiDB 语法解析

Types

type NodeItem 

type NodeItem struct {
	ID    int               // NodeItem在List中的编号，与顺序有关
	Prev  *NodeItem         // 前一个节点
	Self  sqlparser.SQLNode // 自身指向的AST Node
	Next  *NodeItem         // 后一个节点
	Array *NodeList         // 指针指向所在的链表，用于快速跳转node
}

NodeItem 链表节点

type NodeList 

type NodeList struct {
	Length  int
	Head    *NodeItem
	NodeMap map[int]*NodeItem
}

NodeList 链表结构体

func NewNodeList 

func NewNodeList(statement sqlparser.Statement) *NodeList

NewNodeList 从抽象语法树中构造一个链表

func (*NodeList) Add 

func (l *NodeList) Add(node sqlparser.SQLNode) *NodeItem

Add 将会把一个sqlparser.SQLNode添加到节点中

func (*NodeList) First 

func (l *NodeList) First() *NodeItem

First 返回链表头结点

func (*NodeList) Last 

func (l *NodeList) Last() *NodeItem

Last 返回链表末尾节点

func (*NodeList) Remove 

func (l *NodeList) Remove(node *NodeItem) error

Remove 从链表中移除一个节点

type Rewrite 

type Rewrite struct {
	SQL     string
	NewSQL  string
	Stmt    sqlparser.Statement
	Columns common.TableColumns
}

Rewrite 用于重写SQL

func NewRewrite 

func NewRewrite(sql string) *Rewrite

NewRewrite 返回一个*Rewrite对象，如果SQL无法被正常解析，将错误输出到日志中，返回一个nil

func (*Rewrite) Rewrite 

func (rw *Rewrite) Rewrite() *Rewrite

Rewrite 入口函数

func (*Rewrite) RewriteAddOrderByNull 

func (rw *Rewrite) RewriteAddOrderByNull() *Rewrite

RewriteAddOrderByNull orderbynull: 对应CLA.008，GROUP BY无排序要求时添加ORDER BY NULL

func (*Rewrite) RewriteAlwaysTrue 

func (rw *Rewrite) RewriteAlwaysTrue() (reWriter *Rewrite)

RewriteAlwaysTrue alwaystrue: 删除恒真条件

func (*Rewrite) RewriteAutoIncrement 

func (rw *Rewrite) RewriteAutoIncrement() *Rewrite

RewriteAutoIncrement autoincrement: 将auto_increment设置为1

func (*Rewrite) RewriteCountStar 

func (rw *Rewrite) RewriteCountStar() *Rewrite

RewriteCountStar countstar: 将COUNT(col)改写为COUNT(*) COUNT(DISTINCT col)不能替换为COUNT(*)

func (*Rewrite) RewriteDML2Select 

func (rw *Rewrite) RewriteDML2Select() *Rewrite

RewriteDML2Select dml2select: DML转成SELECT，兼容低版本的EXPLAIN

func (*Rewrite) RewriteDelimiter 

func (rw *Rewrite) RewriteDelimiter() *Rewrite

RewriteDelimiter delimiter: 补分号，可以指定不同的DELIMITER

func (*Rewrite) RewriteDistinctStar 

func (rw *Rewrite) RewriteDistinctStar() *Rewrite

RewriteDistinctStar distinctstar: 对应DIS.003，将多余的`DISTINCT *`删除

func (*Rewrite) RewriteGroupByConst 

func (rw *Rewrite) RewriteGroupByConst() *Rewrite

RewriteGroupByConst 对应CLA.004，将GROUP BY CONST替换为列名 TODO:

func (*Rewrite) RewriteHaving 

func (rw *Rewrite) RewriteHaving() *Rewrite

RewriteHaving having: 对应CLA.013，使用WHERE过滤条件替代HAVING

func (*Rewrite) RewriteInNull 

func (rw *Rewrite) RewriteInNull() *Rewrite

RewriteInNull innull: TODO: 对应ARG.004

func (*Rewrite) RewriteInnoDB 

func (rw *Rewrite) RewriteInnoDB() *Rewrite

RewriteInnoDB innodb: 为未指定Engine的表默认添加InnoDB引擎，将其他存储引擎转为InnoDB

func (*Rewrite) RewriteInsertColumns 

func (rw *Rewrite) RewriteInsertColumns() *Rewrite

RewriteInsertColumns insertcolumns: 对应COL.002，INSERT补全列名

func (*Rewrite) RewriteIntWidth 

func (rw *Rewrite) RewriteIntWidth() *Rewrite

RewriteIntWidth intwidth: int类型转为int(10)，bigint类型转为bigint(20)

func (*Rewrite) RewriteJoin2SubQuery 

func (rw *Rewrite) RewriteJoin2SubQuery() *Rewrite

RewriteJoin2SubQuery join2sub: TODO: https://mariadb.com/kb/en/library/subqueries-and-joins/

func (*Rewrite) RewriteOr2In 

func (rw *Rewrite) RewriteOr2In() *Rewrite

RewriteOr2In or2in: 同一列的OR过滤条件使用IN()替代，如果值有相等的会进行合并

func (*Rewrite) RewriteOr2Union 

func (rw *Rewrite) RewriteOr2Union() *Rewrite

RewriteOr2Union or2union: 将OR查询转写为UNION ALL TODO: 暂无对应HeuristicRules https://sqlperformance.com/2014/09/sql-plan/rewriting-queries-improve-performance

func (*Rewrite) RewriteRemoveDMLOrderBy 

func (rw *Rewrite) RewriteRemoveDMLOrderBy() *Rewrite

RewriteRemoveDMLOrderBy dmlorderby: 对应RES.004，删除无LIMIT条件时UPDATE, DELETE中包含的ORDER BY

func (*Rewrite) RewriteRmParenthesis 

func (rw *Rewrite) RewriteRmParenthesis() *Rewrite

RewriteRmParenthesis rmparenthesis: 去除无意义的括号

func (*Rewrite) RewriteStandard 

func (rw *Rewrite) RewriteStandard() *Rewrite

RewriteStandard standard: 使用vitess提供的String功能将抽象语法树转写回SQL，注意：这可能转写失败。

func (*Rewrite) RewriteStar2Columns 

func (rw *Rewrite) RewriteStar2Columns() *Rewrite

RewriteStar2Columns star2columns: 对应COL.001，SELECT补全*指代的列名

func (*Rewrite) RewriteSubQuery2Join 

func (rw *Rewrite) RewriteSubQuery2Join() *Rewrite

RewriteSubQuery2Join 将subquery转写成join

func (*Rewrite) RewriteTruncate 

func (rw *Rewrite) RewriteTruncate() *Rewrite

RewriteTruncate truncate: DELETE全表修改为TRUNCATE TABLE

func (*Rewrite) RewriteUnionAll 

func (rw *Rewrite) RewriteUnionAll() *Rewrite

RewriteUnionAll unionall: 不介意重复数据的情况下使用union all替换union

type Rule 

type Rule struct {
	Name        string                  `json:"Name"`
	Description string                  `json:"Description"`
	Original    string                  `json:"Original"` // 错误示范。为空或"暂不支持"不会出现在list-rewrite-rules中
	Suggest     string                  `json:"Suggest"`  // 正确示范。
	Func        func(*Rewrite) *Rewrite `json:"-"`        // 如果不定义Func需要多条SQL联动改写
}

Rule SQL重写规则

type TiVisitor 

type TiVisitor struct {
	EnterFunc func(node ast.Node) bool
	LeaveFunc func(node ast.Node) bool
}

TiVisitor TODO

func (*TiVisitor) Enter 

func (visitor *TiVisitor) Enter(n ast.Node) (node ast.Node, skip bool)

Enter TODO

func (*TiVisitor) Leave 

func (visitor *TiVisitor) Leave(n ast.Node) (node ast.Node, ok bool)

Leave TODO

type Token 

type Token struct {
	Type int
	Val  string
	// contains filtered or unexported fields
}

Token 基本定义

func Tokenize 

func Tokenize(sql string) []Token

Tokenize 序列化token

func Tokenizer 

func Tokenizer(sql string) []Token

Tokenizer 用于初始化token

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.