Documentation
¶
Overview ¶
Package properties provides Kusto REST properties that will need to be serialized and sent to Kusto based upon the type of ingestion we are doing.
Index ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type Additional ¶
type Additional struct {
// AuthContext is the authorization string that we get from resources.Manager.AuthContext().
AuthContext string `json:"authorizationContext,omitempty"`
// IngestionMapping is a json string that maps the data being imported to the table's columns.
// See: https://docs.microsoft.com/en-us/azure/kusto/management/data-ingestion/
IngestionMapping string `json:"ingestionMapping,omitempty"`
// IngestionMappingRef is a string representing a mapping reference that has been uploaded to the server
// via a Mgmt() call. See: https://docs.microsoft.com/en-us/azure/kusto/management/create-ingestion-mapping-command
IngestionMappingRef string `json:"ingestionMappingReference,omitempty"`
// IngestionMappingType is what the mapping reference is encoded in: csv, json, avro, ...
IngestionMappingType DataFormat `json:"ingestionMappingType,omitempty"`
// ValidationPolicy is a JSON encoded string that tells our ingestion action what policies we want on the
// data being ingested and what to do when that is violated.
ValidationPolicy string `json:"validationPolicy,omitempty"`
Format DataFormat `json:"format,omitempty"`
// Tags is a list of tags to associated with the ingested data.
Tags []string `json:"tags,omitempty"`
// IngestIfNotExists is a string value that, if specified, prevents ingestion from succeeding if the table already
// has data tagged with an ingest-by: tag with the same value. This ensures idempotent data ingestion.
IngestIfNotExists string `json:"ingestIfNotExists,omitempty"`
}
Additional is additional properites.
func (Additional) MarshalJSON ¶
func (a Additional) MarshalJSON() ([]byte, error)
MarshalJSON implements json.Marshaller. This is for use only by the SDK and may be removed at any time.
type All ¶
type All struct {
// Ingestion is a set of properties that are used across all ingestion methods.
Ingestion Ingestion
// Source provides options that are used are used when doing an ingestion on a filesystem.
Source SourceOptions
}
All holds the complete set of properties that might be used.
type CompressionType ¶
type CompressionType int8
CompressionType is a file's compression type.
const ( // CTUnknown indicates that that the compression type was unset. CTUnknown CompressionType = 0 // CTNone indicates that the file was not compressed. CTNone CompressionType = 1 // GZIP indicates that the file is GZIP compressed. GZIP CompressionType = 2 // ZIP indicates that the file is ZIP compressed. ZIP CompressionType = 3 )
func (CompressionType) MarshalJSON ¶
func (c CompressionType) MarshalJSON() ([]byte, error)
MarshalJSON implements json.Marshaler.MarshalJSON.
func (CompressionType) String ¶
func (c CompressionType) String() string
String implements fmt.Stringer.
type DataFormat ¶
type DataFormat int
DataFormat indicates what type of encoding format was used for source data. Note: This is very similar to ingest.DataFormat, except this supports more formats. We are not using a shared list, because this list is used only internally and is for the data itself, not the mapping reference. Structure prevents packages such as filesystem from importing ingest, because ingest imports filesystem. We would end up with recursive imports. More info here: https://docs.microsoft.com/en-us/azure/kusto/management/data-ingestion/
const ( // DFUnknown indicates the EncodingType is not set. DFUnknown DataFormat = 0 // AVRO indicates the source is encoded in Apache Avro format. AVRO DataFormat = 1 // ApacheAVRO indicates the source is encoded in Apache avro2json format. ApacheAVRO DataFormat = 2 // CSV indicates the source is encoded in comma seperated values. CSV DataFormat = 3 // JSON indicates the source is encoded as one or more lines, each containing a record in Javscript Object Notation. JSON DataFormat = 4 // MultiJSON indicates the source is encoded in JSON-Array of individual records in Javscript Object Notation. MultiJSON DataFormat = 5 // ORC indicates the source is encoded in Apache Optimized Row Columnar format. ORC DataFormat = 6 // Parquet indicates the source is encoded in Apache Parquet format. Parquet DataFormat = 7 // PSV is pipe "|" separated values. PSV DataFormat = 8 // Raw is a text file that has only a single string value. Raw DataFormat = 9 // SCSV is a file containing semicolon ";" separated values. SCSV DataFormat = 10 // SOHSV is a file containing SOH-separated values(ASCII codepont 1). SOHSV DataFormat = 11 // SStream indicats the source is encoded as a Microsoft Cosmos Structured Streams format SStream DataFormat = 12 // TSV is a file containing tab seperated values ("\t"). TSV DataFormat = 13 // TSVE is a file containing escaped-tab seperated values ("\t"). TSVE DataFormat = 14 // TXT is a text file with lines deliminated by "\n". TXT DataFormat = 15 // W3CLogFile indicates the source is encoded using W3C Extended Log File format. W3CLogFile DataFormat = 16 )
func DataFormatDiscovery ¶ added in v0.2.0
func DataFormatDiscovery(fName string) DataFormat
DataFormatDiscovery looks at the file name and tries to discern what the file format is.
func (DataFormat) CamelCase ¶
func (d DataFormat) CamelCase() string
CamelCase returns the CamelCase version. This is for internal use, do not use. This can be removed in future versions.
func (DataFormat) IsValidMappingKind ¶ added in v0.2.0
func (d DataFormat) IsValidMappingKind() bool
IsValidMappingKind returns true if a dataformat can be used as a MappingKind.
func (DataFormat) MarshalJSON ¶
func (d DataFormat) MarshalJSON() ([]byte, error)
MarshalJSON implements json.Marshaler.MarshalJSON.
func (DataFormat) RequiresMapping ¶ added in v0.2.0
func (d DataFormat) RequiresMapping() bool
RequiresMapping indicates whether a data format must be provided with valid mapping file information.
type Ingestion ¶
type Ingestion struct {
// ID is the unqique UUID for this upload.
ID uuid.UUID `json:"Id"`
// BlobPath is the URI representing the blob.
BlobPath string
// DatabaseName is the name of the Kusto database the data will ingest into.
DatabaseName string
// TableName is the name of the Kusto table the the data will ingest into.
TableName string
// RawDataSize is the size of the file on the filesystem, if it was provided.
RawDataSize int64 `json:",omitempty"`
// RetainBlobOnSuccess indicates if the source blob should be retained or deleted.
RetainBlobOnSuccess bool `json:",omitempty"`
// Daniel:
// FlushImmediately ... I know what flushing means, but in terms of here, do we not return until the Kusto
// table is updated, does this mean we do.... This is really a duplicate comment on the options in ingest.go
FlushImmediately bool
// Daniel:
// IgnoreSizeLimit
IgnoreSizeLimit bool `json:",omitempty"`
// ReportLevel defines which if any ingestion states are reported.
ReportLevel IngestionReportLevel `json:",omitempty"`
// ReportMethod defines which mechanisms are used to report the ingestion status.
ReportMethod IngestionReportMethod `json:",omitempty"`
// SourceMessageCreationTime is when we created the blob.
SourceMessageCreationTime time.Time `json:",omitempty"`
// Additional (properties) is a set of extra properties added to the ingestion command.
Additional Additional `json:"AdditionalProperties"`
// TableEntryRef points to the staus table entry used to report the status of this ingestion.
TableEntryRef StatusTableDescription `json:"IngestionStatusInTable,omitempty"`
}
Ingestion is a JSON serializable set of options that must be provided to the service.
func (Ingestion) MarshalJSONString ¶
MarshalJSONString will marshal Ingestion into a base64 encoded string.
type IngestionReportLevel ¶ added in v0.3.0
type IngestionReportLevel int
IngestionReportLevel defines which ingestion statuses are reported by the DM.
const ( // FailuresOnly tells to the DM to report the ingestion sytatus of failed ingestions only. FailuresOnly IngestionReportLevel = 0 // None tells to the DM not to report ingestion status. None IngestionReportLevel = 1 // FailureAndSuccess tells to the DM to report ingestion status for failed and successfull ingestions. FailureAndSuccess IngestionReportLevel = 2 )
type IngestionReportMethod ¶ added in v0.3.0
type IngestionReportMethod int
IngestionReportMthod defines where the DM reports ingestion statuses to.
const ( // ReportStatusToQueue tells the DM to report ingestion status to the a queue. ReportStatusToQueue IngestionReportMethod = 0 // ReportStatusToTable tells the DM to report ingestion status to the a table. ReportStatusToTable IngestionReportMethod = 1 // ReportStatusToQueueAndTable tells the DM to report ingestion status to both queues and tables. ReportStatusToQueueAndTable IngestionReportMethod = 2 // ReportStatusToAzureMonitoring tells the DM to report ingestion status to azure monitor. ReportStatusToAzureMonitoring IngestionReportMethod = 3 )
type SourceOptions ¶
type SourceOptions struct {
// ID allows someone to set the UUID for upload themselves. We aren't providing this option at this time, but here
// when we do.
ID uuid.UUID
// DeleteLocalSource indicates to delete the local file after it has been consumed.
DeleteLocalSource bool
}
SourceOptions are options that the user provides about the source file that is going to be uploaded.
type StatusTableDescription ¶ added in v0.3.0
type StatusTableDescription struct {
// TableConnectionString is a secret-free connection string to the status table.
TableConnectionString string `json:",omitempty"`
// PartitionKey is the partition key of the table entry.
PartitionKey string `json:",omitempty"`
// RowKey is the row key of the table entry.
RowKey string `json:",omitempty"`
}
StatusTableDescription is a reference to the table status entry used for this ingestion command.
Source Files