normalizer

Documentation

Index

• Variables

Variables

var AnnotationRules = On(Any).Self(
	On(Not(pyast.Module)).Error(errors.New("root must be uast.Module")),
	On(pyast.Module).Roles(uast.File, uast.Module).Descendants(

		On(pyast.BinOp).Roles(uast.Expression, uast.Binary).Children(
			On(HasInternalRole("op")).Roles(uast.Expression, uast.Binary, uast.Operator),
			On(HasInternalRole("left")).Roles(uast.Expression, uast.Binary, uast.Left),
			On(HasInternalRole("right")).Roles(uast.Expression, uast.Binary, uast.Right),
		),

		On(pyast.Eq).Roles(uast.Binary, uast.Operator, uast.Equal, uast.Relational),
		On(pyast.NotEq).Roles(uast.Binary, uast.Operator, uast.Equal, uast.Not, uast.Relational),
		On(pyast.Lt).Roles(uast.Binary, uast.Operator, uast.LessThan, uast.Relational),
		On(pyast.LtE).Roles(uast.Binary, uast.Operator, uast.LessThanOrEqual, uast.Relational),
		On(pyast.Gt).Roles(uast.Binary, uast.Operator, uast.GreaterThan, uast.Relational),
		On(pyast.GtE).Roles(uast.Binary, uast.Operator, uast.GreaterThanOrEqual, uast.Relational),
		On(pyast.Is).Roles(uast.Binary, uast.Operator, uast.Identical, uast.Relational),
		On(pyast.IsNot).Roles(uast.Binary, uast.Operator, uast.Identical, uast.Not, uast.Relational),
		On(pyast.In).Roles(uast.Binary, uast.Operator, uast.Contains, uast.Relational),
		On(pyast.NotIn).Roles(uast.Binary, uast.Operator, uast.Contains, uast.Not, uast.Relational),

		On(pyast.Add).Roles(uast.Binary, uast.Operator, uast.Add, uast.Arithmetic),
		On(pyast.Sub).Roles(uast.Binary, uast.Operator, uast.Substract, uast.Arithmetic),
		On(pyast.Mult).Roles(uast.Binary, uast.Operator, uast.Multiply, uast.Arithmetic),
		On(pyast.Div).Roles(uast.Binary, uast.Operator, uast.Divide, uast.Arithmetic),
		On(pyast.Mod).Roles(uast.Binary, uast.Operator, uast.Modulo, uast.Arithmetic),
		On(pyast.FloorDiv).Roles(uast.Binary, uast.Operator, uast.Divide, uast.Arithmetic, uast.Incomplete),
		On(pyast.Pow).Roles(uast.Binary, uast.Operator, uast.Arithmetic, uast.Incomplete),
		On(pyast.MatMult).Roles(uast.Binary, uast.Operator, uast.Multiply, uast.Arithmetic, uast.Incomplete),

		On(pyast.LShift).Roles(uast.Binary, uast.Operator, uast.Bitwise, uast.LeftShift),
		On(pyast.RShift).Roles(uast.Binary, uast.Operator, uast.Bitwise, uast.RightShift),
		On(pyast.BitOr).Roles(uast.Binary, uast.Operator, uast.Bitwise, uast.Or),
		On(pyast.BitXor).Roles(uast.Binary, uast.Operator, uast.Bitwise, uast.Xor),
		On(pyast.BitAnd).Roles(uast.Binary, uast.Operator, uast.Bitwise, uast.And),

		On(pyast.And).Roles(uast.Binary, uast.Operator, uast.Boolean, uast.And),
		On(pyast.Or).Roles(uast.Binary, uast.Operator, uast.Boolean, uast.Or),
		On(pyast.Not).Roles(uast.Binary, uast.Operator, uast.Boolean, uast.Not),
		On(pyast.UnaryOp).Roles(uast.Binary, uast.Operator, uast.Unary, uast.Expression),

		On(pyast.Invert).Roles(uast.Operator, uast.Unary, uast.Bitwise, uast.Not),
		On(pyast.UAdd).Roles(uast.Operator, uast.Unary, uast.Positive),
		On(pyast.USub).Roles(uast.Operator, uast.Unary, uast.Negative),

		On(pyast.Str).Roles(uast.Literal, uast.String, uast.Expression, uast.Primitive),
		On(pyast.StringLiteral).Roles(uast.Literal, uast.String, uast.Expression, uast.Primitive),
		On(pyast.Bytes).Roles(uast.Literal, uast.ByteString, uast.Expression, uast.Primitive),
		On(pyast.Num).Roles(uast.Literal, uast.Number, uast.Expression, uast.Primitive).Children(
			On(HasInternalRole("n")).Roles(uast.Literal, uast.Number, uast.Expression),
		),
		On(pyast.BoolLiteral).Roles(uast.Literal, uast.Boolean, uast.Expression, uast.Primitive),

		On(pyast.BoolOp).Roles(uast.Expression, uast.Boolean, uast.Incomplete),
		On(pyast.JoinedStr).Roles(uast.Literal, uast.String, uast.Expression, uast.Primitive).Children(
			On(pyast.FormattedValue).Roles(uast.Expression, uast.Incomplete),
		),
		On(pyast.NoneLiteral).Roles(uast.Literal, uast.Null, uast.Expression, uast.Primitive),
		On(pyast.Set).Roles(uast.Literal, uast.Set, uast.Expression, uast.Primitive),
		On(pyast.List).Roles(uast.Literal, uast.List, uast.Expression, uast.Primitive),
		On(pyast.Dict).Roles(uast.Literal, uast.Map, uast.Expression, uast.Primitive).Children(
			On(HasInternalRole("keys")).Roles(uast.Map, uast.Key),
			On(HasInternalRole("values")).Roles(uast.Map, uast.Value),
		),
		On(pyast.Tuple).Roles(uast.Literal, uast.Tuple, uast.Expression, uast.Primitive),

		On(pyast.FunctionDef).Roles(uast.Function, uast.Declaration, uast.Name, uast.Identifier).Children(argumentsAnn),
		On(pyast.AsyncFunctionDef).Roles(uast.Function, uast.Declaration, uast.Name, uast.Identifier, uast.Incomplete).Children(argumentsAnn),
		On(pyast.FuncDecorators).Roles(uast.Function, uast.Declaration, uast.Call, uast.Incomplete),
		On(pyast.FuncDefBody).Roles(uast.Function, uast.Declaration, uast.Body),

		On(pyast.ArgumentDefaults).Roles(uast.Function, uast.Declaration, uast.Argument, uast.Value, uast.Incomplete),
		On(pyast.AsyncFuncDecorators).Roles(uast.Function, uast.Declaration, uast.Call, uast.Incomplete),
		On(pyast.AsyncFuncDefBody).Roles(uast.Function, uast.Declaration, uast.Body),

		On(pyast.Lambda).Roles(uast.Function, uast.Declaration, uast.Expression, uast.Anonymous).Children(
			On(pyast.LambdaBody).Roles(uast.Function, uast.Declaration, uast.Body),
			argumentsAnn,
		),

		On(pyast.Call).Roles(uast.Function, uast.Call, uast.Expression).Children(
			On(HasInternalRole("args")).Roles(uast.Function, uast.Call, uast.Positional, uast.Argument, uast.Name),
			On(HasInternalRole("keywords")).Roles(uast.Function, uast.Call, uast.Argument, uast.Name).Children(
				On(HasInternalRole("value")).Roles(uast.Argument, uast.Value),
			),
			On(HasInternalRole("func")).Self(
				On(pyast.Name).Roles(uast.Call, uast.Callee),
				On(pyast.Attribute).Roles(uast.Call, uast.Callee).Children(
					On(HasInternalRole("value")).Roles(uast.Call, uast.Receiver),
				)),
		),

		On(pyast.Assign).Roles(uast.Binary, uast.Assignment, uast.Expression).Children(
			On(HasInternalRole("targets")).Roles(uast.Left),
			On(HasInternalRole("value")).Roles(uast.Right),
		),

		On(pyast.AugAssign).Roles(uast.Operator, uast.Binary, uast.Assignment, uast.Statement).Children(
			On(HasInternalRole("op")).Roles(uast.Operator, uast.Binary),
			On(HasInternalRole("target")).Roles(uast.Left),
			On(HasInternalRole("value")).Roles(uast.Right),
		),

		On(pyast.Attribute).Roles(uast.Identifier, uast.Expression).Self(
			On(HasChild(pyast.Name)).Children(
				On(pyast.Name).Roles(uast.Qualified),
			),
			On(HasInternalRole("value")).Roles(uast.Qualified),
		),

		On(pyast.Expression).Roles(uast.Expression),
		On(pyast.Expr).Roles(uast.Expression),
		On(pyast.Name).Roles(uast.Identifier, uast.Expression),

		On(pyast.SameLineNoops).Roles(uast.Comment),
		On(pyast.PreviousNoops).Roles(uast.Noop).Children(
			On(HasInternalRole("lines")).Self(
				On(HasProperty("noop_line", "\n")).Roles(uast.Noop, uast.Whitespace),
				On(Not(HasProperty("noop_line", "\n"))).Roles(uast.Noop, uast.Comment),
			),
		),
		On(pyast.RemainderNoops).Roles(uast.Noop).Children(
			On(HasInternalRole("lines")).Self(
				On(HasProperty("noop_line", "\n")).Roles(uast.Noop, uast.Whitespace),
				On(Not(HasProperty("noop_line", "\n"))).Roles(uast.Noop, uast.Comment),
			),
		),

		On(pyast.Constant).Roles(uast.Identifier, uast.Expression),
		On(pyast.Try).Roles(uast.Try, uast.Statement).Children(
			On(pyast.TryBody).Roles(uast.Try, uast.Body),
			On(pyast.TryFinalBody).Roles(uast.Try, uast.Finally),
			On(pyast.TryHandlers).Roles(uast.Try, uast.Catch),
			On(pyast.TryElse).Roles(uast.Try, uast.Body, uast.Else),
		),
		On(pyast.TryExcept).Roles(uast.Try, uast.Catch, uast.Statement),
		On(pyast.ExceptHandler).Roles(uast.Try, uast.Catch, uast.Statement),
		On(pyast.ExceptHandlerName).Roles(uast.Try, uast.Catch, uast.Identifier),
		On(pyast.TryFinally).Roles(uast.Try, uast.Finally, uast.Statement),
		On(pyast.Raise).Roles(uast.Throw, uast.Statement),

		On(pyast.With).Roles(uast.Block, uast.Scope, uast.Statement),
		On(pyast.WithBody).Roles(uast.Block, uast.Scope, uast.Expression, uast.Incomplete),
		On(pyast.WithItems).Roles(uast.Identifier, uast.Expression, uast.Incomplete),
		On(pyast.AsyncWith).Roles(uast.Block, uast.Scope, uast.Statement, uast.Incomplete),
		On(pyast.Withitem).Roles(uast.Identifier, uast.Incomplete),
		On(pyast.Return).Roles(uast.Return, uast.Statement),
		On(pyast.Break).Roles(uast.Break, uast.Statement),
		On(pyast.Continue).Roles(uast.Continue, uast.Statement),

		On(pyast.Compare).Roles(uast.Expression, uast.Binary).Children(

			On(pyast.CompareOps).Roles(uast.Expression),

			On(HasInternalRole("left")).Roles(uast.Expression, uast.Left),

			On(pyast.CompareComparators).Roles(uast.Expression, uast.Right),
		),
		On(pyast.If).Roles(uast.If, uast.Statement).Children(
			On(pyast.IfBody).Roles(uast.If, uast.Body, uast.Then),
			On(HasInternalRole("test")).Roles(uast.If, uast.Condition),
			On(pyast.IfElse).Roles(uast.If, uast.Body, uast.Else),
		),
		On(pyast.IfExp).Roles(uast.If, uast.Expression).Children(

			On(HasInternalRole("body")).Roles(uast.If, uast.Body, uast.Then),
			On(HasInternalRole("test")).Roles(uast.If, uast.Condition),
			On(HasInternalRole("orelse")).Roles(uast.If, uast.Body, uast.Else),
		),
		On(pyast.Import).Roles(uast.Import, uast.Declaration, uast.Statement),

		On(pyast.Alias).Roles(uast.Import, uast.Pathname, uast.Identifier),

		On(pyast.ImportFromModule).Roles(uast.Import, uast.Pathname, uast.Identifier),

		On(pyast.AliasAsName).Roles(uast.Import, uast.Alias, uast.Identifier),
		On(pyast.ImportFrom).Roles(uast.Import, uast.Declaration, uast.Statement),
		On(pyast.ClassDef).Roles(uast.Type, uast.Declaration, uast.Identifier, uast.Statement).Children(
			On(pyast.ClassDefDecorators).Roles(uast.Type, uast.Call, uast.Incomplete),
			On(pyast.ClassDefBody).Roles(uast.Type, uast.Declaration, uast.Body),
			On(pyast.ClassDefBases).Roles(uast.Type, uast.Declaration, uast.Base),
			On(pyast.ClassDefKeywords).Roles(uast.Incomplete).Children(
				On(pyast.Keyword).Roles(uast.Identifier, uast.Incomplete),
			),
		),

		On(pyast.For).Roles(uast.For, uast.Iterator, uast.Statement).Children(
			On(pyast.ForBody).Roles(uast.For, uast.Body),
			On(HasInternalRole("iter")).Roles(uast.For, uast.Expression),
			On(HasInternalRole("target")).Roles(uast.For, uast.Update),
			On(pyast.ForElse).Roles(uast.For, uast.Body, uast.Else),
		),
		On(pyast.AsyncFor).Roles(uast.For, uast.Iterator, uast.Statement, uast.Incomplete).Children(
			On(pyast.AsyncForBody).Roles(uast.For, uast.Body),
			On(HasInternalRole("iter")).Roles(uast.For, uast.Expression),
			On(HasInternalRole("target")).Roles(uast.For, uast.Update),
			On(pyast.AsyncForElse).Roles(uast.For, uast.Body, uast.Else),
		),
		On(pyast.While).Roles(uast.While, uast.Statement).Children(
			On(pyast.WhileBody).Roles(uast.While, uast.Body),
			On(HasInternalRole("test")).Roles(uast.While, uast.Condition),
			On(pyast.WhileElse).Roles(uast.While, uast.Body, uast.Else),
		),
		On(pyast.Pass).Roles(uast.Noop, uast.Statement),
		On(pyast.Assert).Roles(uast.Assert, uast.Statement),

		On(pyast.Exec).Roles(uast.Function, uast.Call, uast.Expression).Children(
			On(HasInternalRole("body")).Roles(uast.Call, uast.Argument, uast.Positional),
			On(HasInternalRole("globals")).Roles(uast.Call, uast.Argument, uast.Positional),
			On(HasInternalRole("locals")).Roles(uast.Call, uast.Argument, uast.Positional),
		),

		On(pyast.Print).Roles(uast.Function, uast.Call, uast.Callee, uast.Identifier, uast.Expression).Children(
			On(HasInternalRole("dest")).Roles(uast.Call, uast.Argument, uast.Positional),
			On(HasInternalRole("nl")).Roles(uast.Call, uast.Argument, uast.Positional),
			On(HasInternalRole("values")).Roles(uast.Call, uast.Argument, uast.Positional).Children(
				On(Any).Roles(uast.Call, uast.Argument, uast.Positional),
			),
		),

		On(pyast.AnnAssign).Roles(uast.Operator, uast.Binary, uast.Assignment),
		On(HasInternalRole("annotation")).Roles(uast.Annotation),
		On(HasInternalRole("returns")).Roles(uast.Annotation),

		On(pyast.Ellipsis).Roles(uast.Identifier, uast.Incomplete),

		On(pyast.ListComp).Roles(uast.List, uast.For, uast.Expression),
		On(pyast.DictComp).Roles(uast.Map, uast.For, uast.Expression),
		On(pyast.SetComp).Roles(uast.Set, uast.For, uast.Expression),
		On(pyast.Comprehension).Roles(uast.For, uast.Iterator, uast.Expression, uast.Incomplete).Children(
			On(HasInternalRole("iter")).Roles(uast.For, uast.Update, uast.Statement),
			On(HasInternalRole("target")).Roles(uast.For, uast.Expression),

			On(pyast.Compare).Roles(uast.If, uast.Condition),
		),

		On(pyast.Delete).Roles(uast.Statement, uast.Incomplete),
		On(pyast.Await).Roles(uast.Statement, uast.Incomplete),
		On(pyast.Global).Roles(uast.Statement, uast.Visibility, uast.World, uast.Incomplete),
		On(pyast.Nonlocal).Roles(uast.Statement, uast.Visibility, uast.Module, uast.Incomplete),

		On(pyast.Yield).Roles(uast.Return, uast.Statement, uast.Incomplete),
		On(pyast.YieldFrom).Roles(uast.Return, uast.Statement, uast.Incomplete),
		On(pyast.Yield).Roles(uast.Literal, uast.List, uast.Expression, uast.Incomplete),

		On(pyast.Subscript).Roles(uast.Expression, uast.Incomplete),
		On(pyast.Index).Roles(uast.Expression, uast.Incomplete),
		On(pyast.Slice).Roles(uast.Expression, uast.Incomplete),
		On(pyast.ExtSlice).Roles(uast.Expression, uast.Incomplete),
	))

AnnotationRules describes how a UAST should be annotated with `uast.Role`.

https://godoc.org/gopkg.in/bblfsh/sdk.v1/uast/ann

var ToNode = &uast.ObjectToNode{
	InternalTypeKey: "ast_type",
	LineKey:         "lineno",
	EndLineKey:      "end_lineno",
	ColumnKey:       "col_offset",
	EndColumnKey:    "end_col_offset",

	TokenKeys: map[string]bool{
		"name": true,

		"id":           true,
		"attr":         true,
		"arg":          true,
		"LiteralValue": true,
		"s":            true,
		"n":            true,
		"noop_line":    true,
	},
	SyntheticTokens: map[string]string{
		"Add":       "+",
		"Assert":    "assert",
		"AugAssign": "?=",
		"BitAnd":    "&",
		"BitOr":     "|",
		"BitXor":    "^",
		"Break":     "break",
		"Continue":  "continue",
		"Delete":    "delete",
		"Div":       "/",
		"Ellipsis":  "...",
		"Eq":        "==",
		"For":       "for",
		"FloorDiv":  "//",
		"Global":    "global",
		"Gt":        ">",
		"GtE":       ">=",
		"If":        "if",
		"In":        "in",
		"Invert":    "~",
		"Is":        "is",
		"IsNot":     "not is",
		"LShift":    "<<",
		"Lt":        "<",
		"LtE":       "<=",
		"Mod":       "%%",
		"Mult":      "*",
		"Nonlocal":  "nonlocal",
		"Not":       "!",
		"NotEq":     "!=",
		"NotIn":     "not in",
		"Pass":      "pass",
		"Pow":       "**",
		"Print":     "print",
		"Raise":     "raise",
		"Return":    "return",
		"RShift":    ">>",
		"Sub":       "-",
		"UAdd":      "+",
		"USub":      "-",
		"While":     "while",
		"With":      "with",
		"Yield":     "yield",
	},
	PromoteAllPropertyLists: false,
	PromotedPropertyLists: map[string]map[string]bool{
		"If":       {"body": true, "orelse": true},
		"For":      {"body": true, "orelse": true},
		"AsyncFor": {"body": true, "orelse": true},
		"While":    {"body": true, "orelse": true},
		"Compare":  {"comparators": true, "ops": true},

		"With":        {"body": true, "items": true},
		"FunctionDef": {"body": true, "decorator_list": true},
		"Lambda":      {"body": true},
		"arguments":   {"defaults": true},
		"Try":         {"body": true, "orelse": true, "finalbody": true},
		"Raise":       {"args": true},
		"ClassDef":    {"body": true, "bases": true, "decorator_list": true, "keywords": true},
	},
	PromotedPropertyStrings: map[string]map[string]bool{
		"alias":         {"asname": true},
		"ImportFrom":    {"module": true},
		"ExceptHandler": {"name": true},
	},
}

ToNode is an instance of `uast.ObjectToNode`, defining how to transform an into a UAST (`uast.Node`).

https://godoc.org/gopkg.in/bblfsh/sdk.v1/uast#ObjectToNode

var Transformers = []transformer.Tranformer{
	annotatter.NewAnnotatter(AnnotationRules),
	positioner.NewFillOffsetFromLineCol(),
}

Transformers is the of list `transformer.Transfomer` to apply to a UAST, to learn more about the Transformers and the available ones take a look to: https://godoc.org/gopkg.in/bblfsh/sdk.v1/uast/transformers