s3select

Documentation

Index

• Constants
• Variables
• func Execute(writer io.Writer, f format.Select) error
• func New(reader io.Reader, size int64, req ObjectSelectRequest) (s3s format.Select, err error)
• type CSVFileHeaderInfo
• type CSVQuoteFields
• type JSONType
• type ObjectSelectRequest
• type ParseSelectTokens
• type QueryExpressionType
• type Row
• type SelectCompressionType
• type SelectFuncs
  • func ParseSelect(f format.Select) ([]string, string, int64, interface{}, []string, SelectFuncs, error)

Constants

const (
	CSVFileHeaderInfoNone   CSVFileHeaderInfo = "NONE"
	CSVFileHeaderInfoIgnore                   = "IGNORE"
	CSVFileHeaderInfoUse                      = "USE"
)

Constants for file header info.

const (
	SelectCompressionNONE SelectCompressionType = "NONE"
	SelectCompressionGZIP                       = "GZIP"
	SelectCompressionBZIP                       = "BZIP2"
)

Constants for compression types under select API.

const (
	MaxCharsPerRecord = 1000000
)

The maximum character per record is set to be 1 MB.

const MaxExpressionLength = 256 * 1024

MaxExpressionLength - 256KiB

Variables

var ErrAmbiguousFieldName = errors.New("Field name matches to multiple fields in the file. Check the SQL expression and the file, and try again")

ErrAmbiguousFieldName is an error that occurs if the column name which is not case sensitive, is not descriptive enough to retrieve a singular column.

var ErrBusy = errors.New("The service is unavailable. Please retry")

ErrBusy is an error if the service is too busy.

var ErrCastFailed = errors.New("Attempt to convert from one data type to another using CAST failed in the SQL expression")

ErrCastFailed occurs if the attempt to convert data types in the cast is not done correctly.

var ErrColumnTooLong = errors.New("The length of a column in the result is greater than maxCharsPerColumn of 1 MB")

ErrColumnTooLong is an error if your query results in a column that is greater than the max amount of characters per column of 1mb

var ErrEvaluatorBindingDoesNotExist = errors.New("A column name or a path provided does not exist in the SQL expression")

ErrEvaluatorBindingDoesNotExist is an error that occurs if a column name or path provided in the expression does not exist.

var ErrEvaluatorInvalidArguments = errors.New("Incorrect number of arguments in the function call in the SQL expression")

ErrEvaluatorInvalidArguments is an error that occurs if there are not the correct number of arguments in a functional call to a SQL expression.

var ErrEvaluatorInvalidTimestampFormatPattern = errors.New("Time stamp format pattern requires additional fields in the SQL expression")

ErrEvaluatorInvalidTimestampFormatPattern is an error that occurs if the Time Stamp Format needs more additional fields to be filled.

var ErrEvaluatorInvalidTimestampFormatPatternSymbol = errors.New("Time stamp format pattern contains an invalid symbol in the SQL expression")

ErrEvaluatorInvalidTimestampFormatPatternSymbol is an error that occurs if the time stamp format pattern has an invalid symbol within the SQL expression.

var ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing = errors.New("Time stamp format pattern contains a valid format symbol that cannot be applied to time stamp parsing in the SQL expression")

ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing is an error that occurs if the format of the time stamp can not be parsed.

var ErrEvaluatorInvalidTimestampFormatPatternToken = errors.New("Time stamp format pattern contains an invalid token in the SQL expression")

ErrEvaluatorInvalidTimestampFormatPatternToken is an error that occurs if there is an invalid token in the time stamp format within the SQL expression.

var ErrEvaluatorTimestampFormatPatternDuplicateFields = errors.New("Time stamp format pattern contains multiple format specifiers representing the time stamp field in the SQL expression")

ErrEvaluatorTimestampFormatPatternDuplicateFields is an error that occurs if the time stamp format pattern contains multiple format specifications which can not be clearly resolved.

var ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch = errors.New("Time stamp format pattern contains a 12-hour hour of day format symbol but doesn't also contain an AM/PM field, or it contains a 24-hour hour of day format specifier and contains an AM/PM field in the SQL expression")

ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch is an error that occurs if the time stamp format pattern contains a 12 hour day of format but does not have an AM/PM field.

var ErrEvaluatorUnterminatedTimestampFormatPatternToken = errors.New("Time stamp format pattern contains unterminated token in the SQL expression")

ErrEvaluatorUnterminatedTimestampFormatPatternToken is an error that occurs if there is an unterminated token in the SQL expression for time stamp format.

var ErrExpressionTooLong = errors.New("The SQL expression is too long: The maximum byte-length for the SQL expression is 256 KB")

ErrExpressionTooLong is an error if your SQL expression too long for processing.

var ErrExternalEvalException = errors.New("The query cannot be evaluated. Check the file and try again")

ErrExternalEvalException is an error that arises if the query can not be evaluated.

var ErrIllegalSQLFunctionArgument = errors.New("Illegal argument was used in the SQL function")

ErrIllegalSQLFunctionArgument is an error if you provide an illegal argument in the SQL function.

var ErrIncorrectSQLFunctionArgumentType = errors.New("Incorrect type of arguments in function call in the SQL expression")

ErrIncorrectSQLFunctionArgumentType is an error that occurs if the wrong argument is provided to a SQL function.

var ErrIntegerOverflow = errors.New("Int overflow or underflow in the SQL expression")

ErrIntegerOverflow is an error that occurs if there is an IntegerOverflow or IntegerUnderFlow in the SQL expression.

var ErrInvalidCast = errors.New("Attempt to convert from one data type to another using CAST failed in the SQL expression")

ErrInvalidCast is an error that occurs if the attempt to convert data types failed and was done in an improper fashion.

var ErrInvalidCompressionFormat = errors.New("The file is not in a supported compression format. Only GZIP is supported at this time")

ErrInvalidCompressionFormat is an error if an unsupported compression type is utilized with the select object query.

var ErrInvalidDataType = errors.New("The SQL expression contains an invalid data type")

ErrInvalidDataType is an error that occurs if the SQL expression contains an invalid data type.

var ErrInvalidFileHeaderInfo = errors.New("The FileHeaderInfo is invalid. Only NONE, USE, and IGNORE are supported")

ErrInvalidFileHeaderInfo is an error if the argument provided to the FileHeader Argument is incorrect.

var ErrInvalidJSONType = errors.New("The JsonType is invalid. Only DOCUMENT and LINES are supported at this time")

ErrInvalidJSONType is an error if the json format provided as an argument is invalid.

var ErrInvalidKeyPath = errors.New("Key path in the SQL expression is invalid")

ErrInvalidKeyPath is an error if you provide a key in the SQL expression that is invalid.

var ErrInvalidQuoteFields = errors.New("The QuoteFields is invalid. Only ALWAYS and ASNEEDED are supported")

ErrInvalidQuoteFields is an error if the arguments provided to the QuoteFields options are not valid.

var ErrInvalidRequestParameter = errors.New("The value of a parameter in Request element is invalid. Check the service API documentation and try again")

ErrInvalidRequestParameter is an error if the value of a parameter in the request element is not valid.

var ErrInvalidTableAlias = errors.New("The SQL expression contains an invalid table alias")

ErrInvalidTableAlias is an error that arises if the table alias provided in the SQL expression is invalid.

var ErrInvalidTextEncoding = errors.New("Invalid encoding type. Only UTF-8 encoding is supported at this time")

ErrInvalidTextEncoding is an error if the text encoding is not valid.

var ErrLexerInvalidChar = errors.New("The SQL expression contains an invalid character")

ErrLexerInvalidChar is an error that occurs if the SQL expression contains an invalid character.

var ErrLexerInvalidIONLiteral = errors.New("The SQL expression contains an invalid operator")

ErrLexerInvalidIONLiteral is an error that occurs if an invalid operator is used

var ErrLexerInvalidLiteral = errors.New("The SQL expression contains an invalid literal")

ErrLexerInvalidLiteral is an error that occurs if an invalid literal is used.

var ErrLexerInvalidOperator = errors.New("The SQL expression contains an invalid operator")

ErrLexerInvalidOperator is an error that occurs if an invalid operator is used.

var ErrLikeInvalidInputs = errors.New("Invalid argument given to the LIKE clause in the SQL expression")

ErrLikeInvalidInputs is an error that occurs if invalid inputs are provided to the argument LIKE Clause.

var ErrMissingHeaders = errors.New("Some headers in the query are missing from the file. Check the file and try again")

ErrMissingHeaders is an error if some of the headers that are requested in the Select Query are not present in the file.

var ErrMissingRequiredParameter = errors.New("The Request entity is missing a required parameter. Check the service documentation and try again")

ErrMissingRequiredParameter is an error that arises if a required argument is omitted from the Request.

var ErrMultipleDataSourcesUnsupported = errors.New("Multiple data sources are not supported")

ErrMultipleDataSourcesUnsupported is an error that arises if multiple data sources are provided.

var ErrObjectSerializationConflict = errors.New("The Request entity can only contain one of CSV or JSON. Check the service documentation and try again")

ErrObjectSerializationConflict is an error that arises if an unsupported output seralization is provided.

var ErrOverMaxColumn = errors.New("The number of columns in the result is greater than maxColumnNumber of 1 MB")

ErrOverMaxColumn is an error if the number of columns from the resulting query is greater than 1Mb.

var ErrOverMaxRecordSize = errors.New("The length of a record in the result is greater than maxCharsPerRecord of 1 MB")

ErrOverMaxRecordSize is an error if the length of a record in the result is greater than 1 Mb.

var ErrParseAsteriskIsNotAloneInSelectList = errors.New("Other expressions are not allowed in the SELECT list when '*' is used without dot notation in the SQL expression")

ErrParseAsteriskIsNotAloneInSelectList is an error that occurs if in addition to an asterix, more column names are provided as arguments in the SQL expression.

var ErrParseCannotMixSqbAndWildcardInSelectList = errors.New("Cannot mix [] and * in the same expression in a SELECT list in SQL expression")

ErrParseCannotMixSqbAndWildcardInSelectList is an error that occurs if list indexing and an asterix are mixed in the SQL expression.

var ErrParseCastArity = errors.New("The SQL expression CAST has incorrect arity")

ErrParseCastArity is an error that occurs because the CAST has incorrect arity.

var ErrParseEmptySelect = errors.New("The SQL expression contains an empty SELECT")

ErrParseEmptySelect is an error that occurs because the SQL expression contains an empty Select

var ErrParseExpected2TokenTypes = errors.New("Did not find the expected token in the SQL expression")

ErrParseExpected2TokenTypes is an error that occurs if 2 token types are not found.

var ErrParseExpectedArgumentDelimiter = errors.New("Did not find the expected argument delimiter in the SQL expression")

ErrParseExpectedArgumentDelimiter is an error that occurs if the argument delimiter for the SQL expression is not provided.

var ErrParseExpectedDatePart = errors.New("Did not find the expected date part in the SQL expression")

ErrParseExpectedDatePart is an error that occurs if the date part is not found in the SQL expression.

var ErrParseExpectedExpression = errors.New("Did not find the expected SQL expression")

ErrParseExpectedExpression is an error that occurs if the SQL expression is not found.

var ErrParseExpectedIdentForAlias = errors.New("Did not find the expected identifier for the alias in the SQL expression")

ErrParseExpectedIdentForAlias is an error that occurs if expected identifier for alias is not in the SQL expression.

var ErrParseExpectedIdentForAt = errors.New("Did not find the expected identifier for AT name in the SQL expression")

ErrParseExpectedIdentForAt is an error that occurs if after "AT" an Alias identifier is not provided.

var ErrParseExpectedIdentForGroupName = errors.New("GROUP is not supported in the SQL expression")

ErrParseExpectedIdentForGroupName is an error that occurs because Group is not supported in the SQL expression.

var ErrParseExpectedKeyword = errors.New("Did not find the expected keyword in the SQL expression")

ErrParseExpectedKeyword is an error that occurs if the expected keyword was not found in the expression.

var ErrParseExpectedLeftParenAfterCast = errors.New("Did not find the expected left parenthesis after CAST in the SQL expression")

ErrParseExpectedLeftParenAfterCast is an error that occurs if the left parenthesis is missing after a cast in the SQL expression.

var ErrParseExpectedLeftParenBuiltinFunctionCall = errors.New("Did not find the expected left parenthesis in the SQL expression")

ErrParseExpectedLeftParenBuiltinFunctionCall is an error that occurs if the left parenthesis is not found in the SQL expression function call.

var ErrParseExpectedLeftParenValueConstructor = errors.New("Did not find expected the left parenthesis in the SQL expression")

ErrParseExpectedLeftParenValueConstructor is an error that occurs if the left parenthesis is not found in the SQL expression.

var ErrParseExpectedMember = errors.New("The SQL expression contains an unsupported use of MEMBER")

ErrParseExpectedMember is an error that occurs if there is an unsupported use of member in the SQL expression.

var ErrParseExpectedNumber = errors.New("Did not find the expected number in the SQL expression")

ErrParseExpectedNumber is an error that occurs if a number is expected but not found in the expression.

var ErrParseExpectedRightParenBuiltinFunctionCall = errors.New("Did not find the expected right parenthesis character in the SQL expression")

ErrParseExpectedRightParenBuiltinFunctionCall is an error that occurs if a right parenthesis is missing.

var ErrParseExpectedTokenType = errors.New("Did not find the expected token in the SQL expression")

ErrParseExpectedTokenType is an error that occurs if the expected token is not found in the SQL expression.

var ErrParseExpectedTypeName = errors.New("Did not find the expected type name in the SQL expression")

ErrParseExpectedTypeName is an error that occurs if a type name is expected but not found.

var ErrParseExpectedWhenClause = errors.New("Did not find the expected WHEN clause in the SQL expression. CASE is not supported")

ErrParseExpectedWhenClause is an error that occurs if a When clause is expected but not found.

var ErrParseInvalidContextForWildcardInSelectList = errors.New("Invalid use of * in SELECT list in the SQL expression")

ErrParseInvalidContextForWildcardInSelectList is an error that occurs if the asterix is used improperly within the SQL expression.

var ErrParseInvalidTypeParam = errors.New("The SQL expression contains an invalid parameter value")

ErrParseInvalidTypeParam is an error that occurs because there is an invalid parameter value.

var ErrParseMalformedJoin = errors.New("JOIN is not supported in the SQL expression")

ErrParseMalformedJoin is an error that occurs if a "join" operation is attempted in the SQL expression as this is not supported.

var ErrParseMissingIdentAfterAt = errors.New("Did not find the expected identifier after the @ symbol in the SQL expression")

ErrParseMissingIdentAfterAt is an error that occurs if the wrong symbol follows the "@" symbol in the SQL expression.

var ErrParseNonUnaryAgregateFunctionCall = errors.New("Only one argument is supported for aggregate functions in the SQL expression")

ErrParseNonUnaryAgregateFunctionCall is an error that occurs if more than one argument is provided as an argument for aggregation functions.

var ErrParseSelectMissingFrom = errors.New("The SQL expression contains a missing FROM after SELECT list")

ErrParseSelectMissingFrom is an error that occurs because there is a missing From after the Select List.

var ErrParseUnexpectedKeyword = errors.New("The SQL expression contains an unexpected keyword")

ErrParseUnexpectedKeyword is an error that occurs if the SQL expression contains an unexpected keyword.

var ErrParseUnexpectedOperator = errors.New("The SQL expression contains an unexpected operator")

ErrParseUnexpectedOperator is an error that occurs if the SQL expression contains an unexpected operator.

var ErrParseUnexpectedTerm = errors.New("The SQL expression contains an unexpected term")

ErrParseUnexpectedTerm is an error that occurs if the SQL expression contains an unexpected term.

var ErrParseUnexpectedToken = errors.New("The SQL expression contains an unexpected token")

ErrParseUnexpectedToken is an error that occurs if the SQL expression contains an unexpected token.

var ErrParseUnknownOperator = errors.New("The SQL expression contains an invalid operator")

ErrParseUnknownOperator is an error that occurs if there is an invalid operator present in the SQL expression.

var ErrParseUnsupportedAlias = errors.New("The SQL expression contains an unsupported use of ALIAS")

ErrParseUnsupportedAlias is an error that occurs if there is an unsupported use of Alias.

var ErrParseUnsupportedCallWithStar = errors.New("Only COUNT with (*) as a parameter is supported in the SQL expression")

ErrParseUnsupportedCallWithStar is an error that occurs if COUNT is used with an argument other than "*".

var ErrParseUnsupportedCase = errors.New("The SQL expression contains an unsupported use of CASE")

ErrParseUnsupportedCase is an error that occurs if there is an unsupported use of case.

var ErrParseUnsupportedCaseClause = errors.New("The SQL expression contains an unsupported use of CASE")

ErrParseUnsupportedCaseClause is an error that occurs if there is an unsupported use of case.

var ErrParseUnsupportedLiteralsGroupBy = errors.New("The SQL expression contains an unsupported use of GROUP BY")

ErrParseUnsupportedLiteralsGroupBy is an error that occurs if the SQL expression has an unsupported use of Group By.

var ErrParseUnsupportedSelect = errors.New("The SQL expression contains an unsupported use of SELECT")

ErrParseUnsupportedSelect is an error that occurs if there is an unsupported use of Select.

var ErrParseUnsupportedSyntax = errors.New("The SQL expression contains unsupported syntax")

ErrParseUnsupportedSyntax is an error that occurs if there is an UnsupportedSyntax in the SQL expression.

var ErrParseUnsupportedToken = errors.New("The SQL expression contains an unsupported token")

ErrParseUnsupportedToken is an error that occurs if the SQL expression contains an unsupported token.

var ErrUnauthorizedAccess = errors.New("You are not authorized to perform this operation")

ErrUnauthorizedAccess is an error if you lack the appropriate credentials to access the object.

var ErrUnrecognizedFormatException = errors.New("Encountered an invalid record type")

ErrUnrecognizedFormatException is an error that arises if there is an invalid record type.

var ErrUnsupportedRangeHeader = errors.New("Range header is not supported for this operation")

ErrUnsupportedRangeHeader is an error that occurs if a range header is provided.

var ErrUnsupportedSQLOperation = errors.New("Encountered an unsupported SQL operation")

ErrUnsupportedSQLOperation is an error that arises if an unsupported SQL operation is used.

var ErrUnsupportedSQLStructure = errors.New("Encountered an unsupported SQL structure. Check the SQL Reference")

ErrUnsupportedSQLStructure is an error that occurs if an unsupported SQL structure is used.

var ErrUnsupportedStorageClass = errors.New("Encountered an invalid storage class. Only STANDARD, STANDARD_IA, and ONEZONE_IA storage classes are supported at this time")

ErrUnsupportedStorageClass is an error that occurs if an invalid storace class is present.

var ErrUnsupportedSyntax = errors.New("Encountered invalid syntax")

ErrUnsupportedSyntax is an error that occurs if invalid syntax is present in the query.

var ErrValueParseFailure = errors.New("Time stamp parse failure in the SQL expression")

ErrValueParseFailure is an error that occurs if the Time Stamp is not parsed correctly in the SQL expression.

Functions

func Execute

func Execute(writer io.Writer, f format.Select) error

Execute is the function where all the blocking occurs, It writes to the HTTP response writer in a streaming fashion so that the client can actively use the results before the query is finally finished executing. The

func New

func New(reader io.Reader, size int64, req ObjectSelectRequest) (s3s format.Select, err error)

New - initialize new select format

Types

type CSVFileHeaderInfo

type CSVFileHeaderInfo string

CSVFileHeaderInfo -Can be either USE IGNORE OR NONE, defines what to do with the first row

type CSVQuoteFields

type CSVQuoteFields string

CSVQuoteFields - Can be either Always or AsNeeded

const (
	CSVQuoteFieldsAlways   CSVQuoteFields = "Always"
	CSVQuoteFieldsAsNeeded                = "AsNeeded"
)

Constants for csv quote styles.

type JSONType

type JSONType string

JSONType determines json input serialization type.

const (
	JSONTypeDocument JSONType = "DOCUMENT"
	JSONLinesType             = "LINES"
)

Constants for JSONTypes.

type ObjectSelectRequest

type ObjectSelectRequest struct {
	XMLName            xml.Name `xml:"SelectObjectContentRequest" json:"-"`
	Expression         string
	ExpressionType     QueryExpressionType
	InputSerialization struct {
		CompressionType SelectCompressionType
		Parquet         *struct{}
		CSV             *struct {
			FileHeaderInfo       CSVFileHeaderInfo
			RecordDelimiter      string
			FieldDelimiter       string
			QuoteCharacter       string
			QuoteEscapeCharacter string
			Comments             string
		}
		JSON *struct {
			Type JSONType
		}
	}
	OutputSerialization struct {
		CSV *struct {
			QuoteFields          CSVQuoteFields
			RecordDelimiter      string
			FieldDelimiter       string
			QuoteCharacter       string
			QuoteEscapeCharacter string
		}
		JSON *struct {
			RecordDelimiter string
		}
	}
	RequestProgress struct {
		Enabled bool
	}
}

ObjectSelectRequest - represents the input select body

type ParseSelectTokens

type ParseSelectTokens struct {
	// contains filtered or unexported fields
}

ParseSelectTokens tokenizes the select query into required Columns, Alias, limit value where clause, aggregate functions, myFunctions, error.

type QueryExpressionType

type QueryExpressionType string

QueryExpressionType - Currently can only be SQL

const (
	QueryExpressionTypeSQL QueryExpressionType = "SQL"
)

Constants for expression type.

type Row

type Row struct {
	// contains filtered or unexported fields
}

Row is a Struct for keeping track of key aspects of a row.

type SelectCompressionType

type SelectCompressionType string

SelectCompressionType - ONLY GZIP is supported

type SelectFuncs

type SelectFuncs struct {
	// contains filtered or unexported fields
}

SelectFuncs contains the relevant values from the parser for S3 Select Functions

func ParseSelect

func ParseSelect(f format.Select) ([]string, string, int64, interface{}, []string, SelectFuncs, error)

ParseSelect parses the SELECT expression, and effectively tokenizes it into its separate parts. It returns the requested column names,alias,limit of records, and the where clause.