dump

package
v0.9.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 22, 2024 License: Apache-2.0 Imports: 3 Imported by: 1

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func CreateCloudTableDumpTask 

func CreateCloudTableDumpTask(client *golangsdk.ServiceClient, opts CreateCloudTableDumpTaskOpts) error

func CreateDLIDumpTask 

func CreateDLIDumpTask(client *golangsdk.ServiceClient, opts CreateDLIDumpTaskOpts) error

func CreateDWSDumpTask 

func CreateDWSDumpTask(client *golangsdk.ServiceClient, opts CreateDWSDumpTaskOpts) error

func CreateMRSDumpTask 

func CreateMRSDumpTask(client *golangsdk.ServiceClient, opts CreateMRSDumpTaskOpts) error

func CreateOBSDumpTask 

func CreateOBSDumpTask(client *golangsdk.ServiceClient, opts CreateOBSDumpTaskOpts) error

func DeleteTransferTask 

func DeleteTransferTask(client *golangsdk.ServiceClient, opts DeleteTransferTaskOpts) (err error)

func TransferTaskAction 

func TransferTaskAction(client *golangsdk.ServiceClient, opts TransferTaskActionOpts) error

Types

type BatchTransferTask 

type BatchTransferTask struct {
	// Dump task ID.
	Id string `json:"id"`
}

type CloudTableDestinationDescriptorOpts 

type CloudTableDestinationDescriptorOpts struct {
	// Name of the dump task.
	// The task name consists of letters, digits, hyphens (-), and underscores (_).
	// It must be a string of 1 to 64 characters.
	TaskName string `json:"task_name"`
	// Name of the agency created on IAM.
	// DIS uses an agency to access your specified resources.
	// The parameters for creating an agency are as follows:
	// - Agency Type: Cloud service
	// - Cloud Service: DIS
	// - Validity Period: unlimited
	// - Scope: Global service,
	// 	 Project: OBS. Select the Tenant Administrator role for the global service project.
	// If agencies have been created, you can obtain available agencies from the agency list by using the "Listing Agencies " API.
	// This parameter cannot be left blank and the parameter value cannot exceed 64 characters.
	// If there are dump tasks on the console, the system displays a message indicating that an agency will be automatically created.
	// The name of the automatically created agency is dis_admin_agency.
	// Maximum: 64
	AgencyName string `json:"agency_name"`
	// User-defined interval at which data is imported from the current DIS stream into OBS.
	// If no data is pushed to the DIS stream during the current interval, no dump file package will be generated.
	// Value range: 30-900
	// Default value: 300
	// Unit: second
	// Minimum: 30
	// Maximum: 900
	// Default: 300
	DeliverTimeInterval string `json:"deliver_time_interval"`
	// Offset.
	// LATEST: Maximum offset indicating that the latest data will be extracted.
	// TRIM_HORIZON: Minimum offset indicating that the earliest data will be extracted.
	// Default value: LATEST
	// Default: LATEST
	// Enumeration values:
	// LATEST
	// TRIM_HORIZON
	ConsumerStrategy string `json:"consumer_strategy,omitempty"`
	// Name of the CloudTable cluster to which data will be dumped.
	// If you choose to dump data to OpenTSDB, OpenTSDB must be enabled for the cluster.
	CloudTableClusterName string `json:"cloudtable_cluster_name"`
	// ID of the CloudTable cluster to which data will be dumped.
	// If you choose to dump data to OpenTSDB, OpenTSDB must be enabled for the cluster.
	CloudTableClusterID string `json:"cloudtable_cluster_id"`
	// HBase table name of the CloudTable cluster to which data will be dumped.
	// The parameter is mandatory when data is dumped to the CloudTable HBase.
	CloudTableTableName string `json:"cloudtable_table_name,omitempty"`
	// Schema configuration of the CloudTable HBase data.
	// You can set either this parameter or opentsdb_schema, but this parameter is mandatory when data will be dumped to HBase.
	// After this parameter is set, the JSON data in the stream can be converted to another format and then be imported to the CloudTable HBase.
	CloudtableSchema CloudtableSchema `json:"cloudtable_schema,omitempty"`
	// Schema configuration of the CloudTable OpenTSDB data.
	// You can set either this parameter or opentsdb_schema, but this parameter is mandatory when data will be dumped to OpenTSDB.
	// After this parameter is set, the JSON data in the stream can be converted to another format and then be imported to the CloudTable OpenTSDB.
	OpenTSDBSchema OpenTSDBSchema `json:"opentsdb_schema,omitempty"`
	// Delimiter used to separate the user data that generates HBase row keys.
	// Value range: , . | ; \ - _ and ~
	// Default value: .
	CloudtableRowKeyDelimiter string `json:"cloudtable_row_key_delimiter,omitempty"`
	// Name of the OBS bucket used to back up data that failed to be dumped to CloudTable.
	OBSBackupBucketPath string `json:"obs_backup_bucket_path,omitempty"`
	// Self-defined directory created in the OBS bucket and used to back up data that failed to be dumped to CloudTable.
	// Directory levels are separated by slashes (/) and cannot start with slashes.
	// Value range: a string of letters, digits, and underscores (_)
	// The maximum length is 50 characters.
	// This parameter is left empty by default.
	BackupFilePrefix string `json:"backup_file_prefix,omitempty"`
	// Time duration for DIS to retry if data fails to be dumped to CloudTable.
	// If this threshold is exceeded, the data that fails to be dumped will be backed up to the OBS bucket/ backup_file_prefix / cloudtable_error or OBS bucket/ backup_file_prefix / opentsdb_error directory.
	// Value range: 0-7,200
	// Unit: second
	// Default value: 1,800
	RetryDuration string `json:"retry_duration,omitempty"`
}

type CloudtableSchema 

type CloudtableSchema struct {
	// HBase rowkey schema used by the CloudTable cluster to convert JSON data into HBase rowkeys.
	// Value range: 1-64
	RowKey []RowKey `json:"row_key"`
	// HBase column schema used by the CloudTable cluster to convert JSON data into HBase columns.
	// Value range: 1 to 4,096
	Columns []Column `json:"columns"`
}

type Column 

type Column struct {
	// Name of the HBase column family to which data will be dumped.
	ColumnFamilyName string `json:"column_family_name"`
	// Name of the HBase column to which data will be dumped.
	// Value range: a string of 1 to 32 characters, consisting of only letters, digits, and underscores (_)
	ColumnName string `json:"column_name"`
	// JSON attribute name, which is used to generate HBase column values for JSON data in the DIS stream.
	Value string `json:"value"`
	// JSON attribute type of JSON data in the DIS stream.
	// Value range:
	// - Bigint
	// - Double
	// - Boolean
	// - Timestamp
	// - String
	// - Decimal
	// Enumeration values:
	// Bigint
	// Double
	// Boolean
	// Timestamp
	// String
	// Decimal
	Type string `json:"type"`
}

type CreateCloudTableDumpTaskOpts 

type CreateCloudTableDumpTaskOpts struct {
	// Name of the stream.
	// Maximum: 60
	StreamName string
	// Dump destination.
	// Possible values:
	// - OBS: Data is dumped to OBS.
	// - MRS: Data is dumped to MRS.
	// - DLI: Data is dumped to DLI.
	// - CLOUDTABLE: Data is dumped to CloudTable.
	// - DWS: Data is dumped to DWS.
	// Default: NOWHERE
	// Enumeration values:
	// MRS
	DestinationType string `json:"destination_type"`
	// Parameter list of the CloudTable to which data in the DIS stream will be dumped.
	CloudTableDestinationDescriptor CloudTableDestinationDescriptorOpts `json:"cloudtable_destination_descriptor,omitempty"`
}

type CreateDLIDumpTaskOpts 

type CreateDLIDumpTaskOpts struct {
	// Name of the stream.
	// Maximum: 60
	StreamName string
	// Dump destination.
	// Possible values:
	// - OBS: Data is dumped to OBS.
	// - MRS: Data is dumped to MRS.
	// - DLI: Data is dumped to DLI.
	// - CLOUDTABLE: Data is dumped to CloudTable.
	// - DWS: Data is dumped to DWS.
	// Default: NOWHERE
	// Enumeration values:
	// DLI
	DestinationType string `json:"destination_type"`
	// Parameter list of the DLI to which data in the DIS stream will be dumped.
	DLIDestinationDescriptor DLIDestinationDescriptorOpts `json:"mrs_destination_descriptor,omitempty"`
}

type CreateDWSDumpTaskOpts 

type CreateDWSDumpTaskOpts struct {
	// Name of the stream.
	// Maximum: 60
	StreamName string
	// Dump destination.
	// Possible values:
	// - OBS: Data is dumped to OBS.
	// - MRS: Data is dumped to MRS.
	// - DLI: Data is dumped to DLI.
	// - CLOUDTABLE: Data is dumped to CloudTable.
	// - DWS: Data is dumped to DWS.
	// Default: NOWHERE
	// Enumeration values:
	// DWS
	DestinationType string `json:"destination_type"`
	// Parameter list of the DWS to which data in the DIS stream will be dumped.
	DWSDestinationDescriptor DWSDestinationDescriptorOpts `json:"dws_destination_descriptor,omitempty"`
}

type CreateMRSDumpTaskOpts 

type CreateMRSDumpTaskOpts struct {
	// Name of the stream.
	// Maximum: 60
	StreamName string
	// Dump destination.
	// Possible values:
	// - OBS: Data is dumped to OBS.
	// - MRS: Data is dumped to MRS.
	// - DLI: Data is dumped to DLI.
	// - CLOUDTABLE: Data is dumped to CloudTable.
	// - DWS: Data is dumped to DWS.
	// Default: NOWHERE
	// Enumeration values:
	// MRS
	DestinationType string `json:"destination_type"`
	// Parameter list of the MRS to which data in the DIS stream will be dumped.
	MRSDestinationDescriptor MRSDestinationDescriptorOpts `json:"mrs_destination_descriptor,omitempty"`
}

type CreateOBSDumpTaskOpts 

type CreateOBSDumpTaskOpts struct {
	// Name of the stream.
	// Maximum: 60
	StreamName string
	// Dump destination.
	// Possible values:
	// - OBS: Data is dumped to OBS.
	// Default: NOWHERE
	// Enumeration values:
	// OBS
	DestinationType string `json:"destination_type"`
	// Parameter list of OBS to which data in the DIS stream will be dumped.
	OBSDestinationDescriptor OBSDestinationDescriptorOpts `json:"obs_destination_descriptor,omitempty"`
}

type DLIDestinationDescriptorOpts 

type DLIDestinationDescriptorOpts struct {
	// Name of the dump task.
	// The task name consists of letters, digits, hyphens (-), and underscores (_).
	// It must be a string of 1 to 64 characters.
	TaskName string `json:"task_name"`
	// Name of the agency created on IAM.
	// DIS uses an agency to access your specified resources.
	// The parameters for creating an agency are as follows:
	// - Agency Type: Cloud service
	// - Cloud Service: DIS
	// - Validity Period: unlimited
	// - Scope: Global service,
	// 	 Project: OBS. Select the Tenant Administrator role for the global service project.
	// If agencies have been created, you can obtain available agencies from the agency list by using the "Listing Agencies " API.
	// This parameter cannot be left blank and the parameter value cannot exceed 64 characters.
	// If there are dump tasks on the console, the system displays a message indicating that an agency will be automatically created.
	// The name of the automatically created agency is dis_admin_agency.
	// Maximum: 64
	AgencyName string `json:"agency_name"`
	// User-defined interval at which data is imported from the current DIS stream into OBS.
	// If no data is pushed to the DIS stream during the current interval, no dump file package will be generated.
	// Value range: 30-900
	// Default value: 300
	// Unit: second
	// Minimum: 30
	// Maximum: 900
	// Default: 300
	DeliverTimeInterval string `json:"deliver_time_interval"`
	// Offset.
	// LATEST: Maximum offset indicating that the latest data will be extracted.
	// TRIM_HORIZON: Minimum offset indicating that the earliest data will be extracted.
	// Default value: LATEST
	// Default: LATEST
	// Enumeration values:
	// LATEST
	// TRIM_HORIZON
	ConsumerStrategy string `json:"consumer_strategy,omitempty"`
	// Name of the DLI database to which data in the DIS stream will be dumped.
	DLIDatabaseName string `json:"dli_database_name"`
	// Name of the DLI table to which data in the DIS stream will be dumped.
	// Note:
	// Only tables whose data location is DLI are supported, and you must have the permission to insert data into the tables.
	DLITableName string `json:"dli_table_name"`
	// Name of the OBS bucket used to temporarily store data in the DIS stream.
	OBSBucketPath string `json:"obs_bucket_path"`
	// Self-defined directory created in the OBS bucket and used to temporarily store data in the DIS stream.
	// Directory levels are separated by slashes (/) and cannot start with slashes.
	// The value can contain a maximum of 50 characters, including letters, digits, underscores (_), and slashes (/).
	// This parameter is left empty by default.
	FilePrefix string `json:"file_prefix,omitempty"`
	// Time duration for DIS to retry if data fails to be dumped to DLI.
	// If the retry time exceeds the value of this parameter, the data that fails to be dumped is backed up to the OBS bucket_path/file_prefix/mrs_error directory
	// Value range: 0-7,200
	// Unit: second
	// Default value: 1,800
	// If this parameter is set to 0, DIS does not retry when the dump fails.
	RetryDuration string `json:"retry_duration,omitempty"`
}

type DWSDestinationDescriptorOpts 

type DWSDestinationDescriptorOpts struct {
	// Name of the dump task.
	// The task name consists of letters, digits, hyphens (-), and underscores (_).
	// It must be a string of 1 to 64 characters.
	TaskName string `json:"task_name"`
	// Name of the agency created on IAM.
	// DIS uses an agency to access your specified resources.
	// The parameters for creating an agency are as follows:
	// - Agency Type: Cloud service
	// - Cloud Service: DIS
	// - Validity Period: unlimited
	// - Scope: Global service,
	// 	 Project: OBS. Select the Tenant Administrator role for the global service project.
	// If agencies have been created, you can obtain available agencies from the agency list by using the "Listing Agencies " API.
	// This parameter cannot be left blank and the parameter value cannot exceed 64 characters.
	// If there are dump tasks on the console, the system displays a message indicating that an agency will be automatically created.
	// The name of the automatically created agency is dis_admin_agency.
	// Maximum: 64
	AgencyName string `json:"agency_name"`
	// User-defined interval at which data is imported from the current DIS stream into OBS.
	// If no data is pushed to the DIS stream during the current interval, no dump file package will be generated.
	// Value range: 30-900
	// Default value: 300
	// Unit: second
	// Minimum: 30
	// Maximum: 900
	// Default: 300
	DeliverTimeInterval string `json:"deliver_time_interval"`
	// Offset.
	// LATEST: Maximum offset indicating that the latest data will be extracted.
	// TRIM_HORIZON: Minimum offset indicating that the earliest data will be extracted.
	// Default value: LATEST
	// Default: LATEST
	// Enumeration values:
	// LATEST
	// TRIM_HORIZON
	ConsumerStrategy string `json:"consumer_strategy,omitempty"`
	// Name of the DWS cluster that stores the data in the stream.
	DWSClusterName string `json:"dws_cluster_name"`
	// ID of the DWS cluster to which will be dumped.
	DWSClusterID string `json:"dws_cluster_id"`
	// Name of the DWS database that stores the data in the stream.
	DWSDatabaseName string `json:"dws_database_name"`
	// Schema of the DWS database to which data will be dumped.
	DWSSchema string `json:"dws_schema"`
	// Name of the DWS table that stores the data in the stream.
	DWSTableName string `json:"dws_table_name"`
	// Delimiter used to separate the columns in the DWS tables.
	// The value can be a comma (,), semicolon (;), or vertical bar (|).
	DWSDelimiter string `json:"dws_delimiter"`
	// Username of the DWS database to which data will be dumped.
	UserName string `json:"user_name"`
	// Password of the DWS database to which data will be dumped.
	UserPassword string `json:"user_password"`
	// Key created in Key Management Service (KMS) and used to encrypt the password of the DWS database.
	KMSUserKeyName string `json:"kms_user_key_name"`
	// ID of the key created in KMS and used to encrypt the password of the DWS database.
	KMSUserKeyID string `json:"kms_user_key_id"`
	// Name of the OBS bucket used to temporarily store data in the DIS stream.
	OBSBucketPath string `json:"obs_bucket_path"`
	// User-defined directory created in the OBS bucket and used to temporarily store data in the DIS stream.
	// Directory levels are separated by slashes (/) and cannot start with slashes.
	// The value can contain a maximum of 50 characters, including letters, digits, underscores (_), and slashes (/).
	// This parameter is left empty by default.
	FilePrefix string `json:"file_prefix,omitempty"`
	// Duration when you can constantly retry dumping data to DWS after the dump fails.
	// If the dump time exceeds the value of this parameter, the data that fails to be dumped to DWS will be backed up to the OBS bucket_path/file_prefix/dws_error directory.
	// Value range: 0-7,200
	// Unit: second
	// Default value: 1,800
	RetryDuration string `json:"retry_duration,omitempty"`
	// Column to be dumped to the DWS table.
	// If the value is null or empty, all columns are dumped by default.
	// For example, c1,c2 indicates that columns c1 and c2 in the schema are dumped to DWS.
	// This parameter is left blank by default.
	DWSTableColumns string `json:"dws_table_columns,omitempty"`
	// DWS fault tolerance option (used to specify various parameters of foreign table data).
	Options Options `json:"options,omitempty"`
}

type DeleteTransferTaskOpts 

type DeleteTransferTaskOpts struct {
	// Name of the stream.
	StreamName string
	// Name of the dump task to be deleted.
	TaskName string
}

type GetTransferTaskOpts 

type GetTransferTaskOpts struct {
	// Name of the stream.
	StreamName string
	// Name of the dump task to be deleted.
	TaskName string
}

type GetTransferTaskResponse 

type GetTransferTaskResponse struct {
	// Name of the stream to which the dump task belongs.
	StreamName string `json:"stream_name,omitempty"`
	// Name of the dump task.
	TaskName string `json:"task_name,omitempty"`
	// Id of the dump task
	TaskId string `json:"task_id,omitempty"`
	// Dump task status.
	// Possible values:
	// - ERROR: An error occurs.
	// - STARTING: The dump task is being started.
	// - PAUSED: The dump task has been stopped.
	// - RUNNING: The dump task is running.
	// - DELETE: The dump task has been deleted.
	// - ABNORMAL: The dump task is abnormal.
	// Enumeration values:
	//  ERROR
	//  STARTING
	//  PAUSED
	//  RUNNING
	//  DELETE
	//  ABNORMAL
	State string `json:"state,omitempty"`
	// Dump destination.
	// Possible values:
	// - OBS: Data is dumped to OBS.
	// - MRS: Data is dumped to MRS.
	// - DLI: Data is dumped to DLI.
	// - CLOUDTABLE: Data is dumped to CloudTable.
	// - DWS: Data is dumped to DWS.
	// Enumeration values:
	//  OBS
	//  MRS
	//  DLI
	//  CLOUDTABLE
	//  DWS
	DestinationType string `json:"destination_type,omitempty"`
	// Time when the dump task is created.
	CreatedAt *int64 `json:"create_time,omitempty"`
	// Latest dump time of the dump task.
	LastTransferTimestamp *int64 `json:"last_transfer_timestamp,omitempty"`
	// List of partition dump details.
	Partitions []PartitionResult `json:"partitions,omitempty"`
	// Parameter list of OBS to which data in the DIS stream will be dumped.
	OBSDestinationDescription OBSDestinationDescriptorOpts `json:"obs_destination_description,omitempty"`
	// Parameter list of the DWS to which data in the DIS stream will be dumped.
	DWSDestinationDescription DWSDestinationDescriptorOpts `json:"dws_destination_description,omitempty"`
	// Parameter list of the MRS to which data in the DIS stream will be dumped.
	MRSDestinationDescription MRSDestinationDescriptorOpts `json:"mrs_destination_description,omitempty"`
	// Parameter list of the DLI to which data in the DIS stream will be dumped.
	DLIDestinationDescription DLIDestinationDescriptorOpts `json:"dli_destination_description,omitempty"`
	// Parameter list of the CloudTable to which data in the DIS stream will be dumped.
	CloudTableDestinationDescription CloudTableDestinationDescriptorOpts `json:"cloud_table_destination_description,omitempty"`
}

func GetTransferTask 

func GetTransferTask(client *golangsdk.ServiceClient, opts GetTransferTaskOpts) (*GetTransferTaskResponse, error)

type ListTransferTasksResponse 

type ListTransferTasksResponse struct {
	// Total number of dump tasks.
	TotalNumber *int `json:"total_number,omitempty"`
	// List of dump tasks.
	Tasks []TransferTask `json:"tasks,omitempty"`
}

func ListTransferTasks 

func ListTransferTasks(client *golangsdk.ServiceClient, streamName string) (*ListTransferTasksResponse, error)

type MRSDestinationDescriptorOpts 

type MRSDestinationDescriptorOpts struct {
	// Name of the dump task.
	// The task name consists of letters, digits, hyphens (-), and underscores (_).
	// It must be a string of 1 to 64 characters.
	TaskName string `json:"task_name"`
	// Name of the agency created on IAM.
	// DIS uses an agency to access your specified resources.
	// The parameters for creating an agency are as follows:
	// - Agency Type: Cloud service
	// - Cloud Service: DIS
	// - Validity Period: unlimited
	// - Scope: Global service,
	// 	 Project: OBS. Select the Tenant Administrator role for the global service project.
	// If agencies have been created, you can obtain available agencies from the agency list by using the "Listing Agencies " API.
	// This parameter cannot be left blank and the parameter value cannot exceed 64 characters.
	// If there are dump tasks on the console, the system displays a message indicating that an agency will be automatically created.
	// The name of the automatically created agency is dis_admin_agency.
	// Maximum: 64
	AgencyName string `json:"agency_name"`
	// User-defined interval at which data is imported from the current DIS stream into OBS.
	// If no data is pushed to the DIS stream during the current interval, no dump file package will be generated.
	// Value range: 30-900
	// Default value: 300
	// Unit: second
	// Minimum: 30
	// Maximum: 900
	// Default: 300
	DeliverTimeInterval string `json:"deliver_time_interval"`
	// Offset.
	// LATEST: Maximum offset indicating that the latest data will be extracted.
	// TRIM_HORIZON: Minimum offset indicating that the earliest data will be extracted.
	// Default value: LATEST
	// Default: LATEST
	// Enumeration values:
	// LATEST
	// TRIM_HORIZON
	ConsumerStrategy string `json:"consumer_strategy,omitempty"`
	// Name of the MRS cluster that stores the data in the stream.
	// Note:
	// Only MRS clusters with non-Kerberos authentication are supported
	MRSClusterName string `json:"mrs_cluster_name"`
	// ID of the MRS cluster to which data in the DIS stream will be dumped.
	MRSClusterID string `json:"mrs_cluster_id"`
	// Hadoop Distributed File System (HDFS) path of the MRS cluster to which data in the DIS stream will be dumped.
	MRSHdfsPatch string `json:"mrs_hdfs_patch"`
	// Self-defined directory created in the OBS bucket and used to temporarily store data in the DIS stream.
	// Directory levels are separated by slashes (/) and cannot start with slashes.
	// The value can contain a maximum of 50 characters, including letters, digits, underscores (_), and slashes (/).
	// This parameter is left empty by default.
	FilePrefix string `json:"file_prefix,omitempty"`
	// Directory to store files that will be dumped to the chosen MRS cluster.
	// Different directory levels are separated by slash (/).
	// Value range:
	// a string of 0 to 50 characters This parameter is left empty by default.
	HDFSPrefixFolder string `json:"hdfs_prefix_folder,omitempty"`
	// Name of the OBS bucket used to temporarily store data in the DIS stream.
	OBSBucketPath string `json:"obs_bucket_path"`
	// Time duration for DIS to retry if data fails to be dumped.
	// If the retry time exceeds the value of this parameter, the data that fails to be dumped is backed up to the OBS bucket_path/file_prefix/mrs_error directory
	// Value range: 0-7,200
	// Unit: second
	// Default value: 1,800
	// If this parameter is set to 0, DIS does not retry when the dump fails.
	RetryDuration string `json:"retry_duration,omitempty"`
}

type OBSDestinationDescriptorOpts 

type OBSDestinationDescriptorOpts struct {
	// Name of the dump task.
	// The task name consists of letters, digits, hyphens (-), and underscores (_).
	// It must be a string of 1 to 64 characters.
	TaskName string `json:"task_name"`
	// Name of the agency created on IAM.
	// DIS uses an agency to access your specified resources.
	// The parameters for creating an agency are as follows:
	// - Agency Type: Cloud service
	// - Cloud Service: DIS
	// - Validity Period: unlimited
	// - Scope: Global service,
	//	 Project: OBS. Select the Tenant Administrator role for the global service project.
	// If agencies have been created, you can obtain available agencies from the agency list by using the "Listing Agencies " API.
	// This parameter cannot be left blank and the parameter value cannot exceed 64 characters.
	// If there are dump tasks on the console, the system displays a message indicating that an agency will be automatically created.
	// The name of the automatically created agency is dis_admin_agency.
	// Maximum: 6
	AgencyName string `json:"agency_name"`
	// User-defined interval at which data is imported from the current DIS stream into OBS.
	// If no data is pushed to the DIS stream during the current interval, no dump file package will be generated.
	// Value range: 30-900
	// Default value: 300
	// Unit: second
	// Minimum: 30
	// Maximum: 900
	// Default: 300
	DeliverTimeInterval *int `json:"deliver_time_interval"`
	// Offset.
	// LATEST: Maximum offset indicating that the latest data will be extracted.
	// TRIM_HORIZON: Minimum offset indicating that the earliest data will be extracted.
	// Default value: LATEST
	// Default: LATEST
	// Enumeration values:
	// LATEST
	// TRIM_HORIZON
	ConsumerStrategy string `json:"consumer_strategy,omitempty"`
	// Directory to store files that will be dumped to OBS.
	// Different directory levels are separated by slashes (/) and cannot start with slashes.
	// The value can contain a maximum of 50 characters, including letters, digits, underscores (_), and slashes (/).
	// This parameter is left empty by default.
	// Maximum: 50
	FilePrefix string `json:"file_prefix,omitempty"`
	// Directory structure of the object file written into OBS.
	// The directory structure is in the format of yyyy/MM/dd/HH/mm (time at which the dump task was created).
	// - N/A: Leave this parameter empty, indicating that the date and time directory is not used.
	// - yyyy: year
	// - yyyy/MM: year/month
	// - yyyy/MM/dd: year/month/day
	// - yyyy/MM/dd/HH: year/month/day/hour
	// - yyyy/MM/dd/HH/mm: year/month/day/hour/minute
	// Example: in 2017/11/10/14/49, the directory structure is 2017 > 11 > 10 > 14 > 49. 2017 indicates the outermost folder.
	// Default value: empty.
	// Note:
	// After data is successfully dumped, the directory structure is obs_bucket_path/file_prefix/partition_format.
	// Enumeration values:
	// yyyy
	// yyyy/MM
	// yyyy/MM/dd
	// yyyy/MM/dd/HH
	// yyyy/MM/dd/HH/mm
	PartitionFormat string `json:"partition_format,omitempty"`
	// Name of the OBS bucket used to store data from the DIS stream.
	OBSBucketPath string `json:"obs_bucket_path,omitempty"`
	// Dump file format.
	// Possible values:
	// - Text (default)
	// - Parquet
	// - CarbonData
	// Note:
	// You can select Parquet or CarbonData only when Source Data Type is set to JSON an Dump Destination is set to OBS.
	// Default: text
	// Enumeration values:
	// text
	// parquet
	// carbon
	DestinationFileType string `json:"destination_file_type,omitempty"`
	// Dump time directory generated based on the timestamp of the source data and the configured partition_format.
	// Directory structure of the object file written into OBS.
	// The directory structure is in the format of yyyy/MM/dd/HH/mm.
	ProcessingSchema ProcessingSchema `json:"processing_schema,omitempty"`
	// Delimiter for the dump file which is used to separate the user data that is written into the dump file
	// Value range:
	// - Comma (,), which is the default value
	// - Semicolon (;)
	// - Vertical bar (|)
	// - Newline character (\n)
	// Default: \n
	RecordDelimiter string `json:"record_delimiter,omitempty"`
}

type OpenTSDBMetric 

type OpenTSDBMetric struct {
	// When type is set to Constant, the value of metric is the value of Value.
	// When value is set to String, the value of metric is the value of the JSON attribute of the user data in the stream.
	// Enumeration values:
	// Constant
	// String
	Type string `json:"type"`
	// Constant value or JSON attribute name of the user data in the stream.
	// This value is 1 to 32 characters long.
	// Only letters, digits, and periods (.) are allowed.
	Value string `json:"value"`
}

type OpenTSDBSchema 

type OpenTSDBSchema struct {
	// Schema configuration of the OpenTSDB data metric in the CloudTable cluster.
	// After this parameter is set, the JSON data in the stream can be converted to the metric of the OpenTSDB data.
	Metric []OpenTSDBMetric `json:"metric"`
	// Schema configuration of the OpenTSDB data timestamp in the CloudTable cluster.
	// After this parameter is set, the JSON data in the stream can be converted to the timestamp of the OpenTSDB data.
	Timestamp OpenTSDBTimestamp `json:"timestamp"`
	// Schema configuration of the OpenTSDB data value in the CloudTable cluster.
	// After this parameter is set, the JSON data in the stream can be converted to the value of the OpenTSDB data.
	Value OpenTSDBValue `json:"value"`
	// Schema configuration of the OpenTSDB data tags in the CloudTable cluster.
	// After this parameter is set, the JSON data in the stream can be converted to the tags of the OpenTSDB data.
	Tags []OpenTSDBTags `json:"tags"`
}

type OpenTSDBTags 

type OpenTSDBTags struct {
	// Tag name of the OpenTSDB data that stores the data in the stream.
	// Value range: a string of 1 to 32 characters, consisting of only letters, digits, and underscores (_)
	Name string `json:"name"`
	// Type name of the JSON attribute of the user data in the stream.
	// Value range:
	// - Bigint
	// - Double
	// - Boolean
	// - Timestamp
	// - String
	// - Decimal
	Type string `json:"type"`
	// Constant value or JSON attribute name of the user data in the stream.
	// Value range: a string of 1 to 32 characters, consisting of only letters, digits, and underscores (_)
	Value string `json:"value"`
}

type OpenTSDBTimestamp 

type OpenTSDBTimestamp struct {
	// When type is set to Timestamp, the value type of the JSON attribute of the user data in the stream is Timestamp, and the timestamp of OpenTSDB can be generated without converting the data format.
	// When type is set to String, the value type of the JSON attribute of the user data in the stream is Date, and the timestamp of OpenTSDB can be generated only after the data format is converted.
	Type string `json:"type"`
	// JSON attribute name of the user data in the stream.
	// Value range: a string of 1 to 32 characters, consisting of only letters, digits, and underscores (_)
	Value string `json:"value"`
	// This parameter is mandatory when type is set to String.
	// When the value type of the JSON attribute of the user data in the stream is Date, format is required to convert the data format to generate the timestamp of OpenTSDB.
	// Value range:
	// - yyyy/MM/dd HH:mm:ss
	// - MM/dd/yyyy HH:mm:ss
	// - dd/MM/yyyy HH:mm:ss
	// - yyyy-MM-dd HH:mm:ss
	// - MM-dd-yyyy HH:mm:ss
	// - dd-MM-yyyy HH:mm:ss
	// Enumeration values:
	// yyyy/MM/dd HH:mm:ss
	// MM/dd/yyyy HH:mm:ss
	// dd/MM/yyyy HH:mm:ss
	// yyyy-MM-dd HH:mm:ss
	// MM-dd-yyyy HH:mm:ss
	// dd-MM-yyyy HH:mm:ss
	Format string `json:"format"`
}

type OpenTSDBValue 

type OpenTSDBValue struct {
	// Dump destination.
	// Possible values:
	// Value range:
	// - Bigint
	// - Double
	// - Boolean
	// - Timestamp
	// - String
	// - Decimal
	Type string `json:"type"`
	// Constant value or JSON attribute name of the user data in the stream.
	// Value range: a string of 1 to 32 characters, consisting of only letters, digits, and underscores (_)
	Value string `json:"value"`
}

type Options 

type Options struct {
	// Specifies whether to set the field to Null or enable an error message to be displayed in the error table when the last field in a row of the data source file is missing during database import.
	// Value range:
	// - true/on
	// - false/off
	// Default value: false/off
	// Enumeration values:
	// true/on
	// false/off
	FillMissingFields string `json:"fill_missing_fields,omitempty"`
	// Specifies whether to ignore excessive columns when the number of columns in a source data file exceeds that defined in the foreign table.
	// This parameter is used only during data import.
	// Value range:
	// - true/on
	// - false/off
	// Default value: false/off
	// Enumeration values:
	// true/on
	// false/off
	IgnoreExtraData string `json:"ignore_extra_data,omitempty"`
	// Specifies whether to tolerate invalid characters during data import.
	// Specifies whether to convert invalid characters based on the conversion rule and import them to the database, or to report an error and stop the import.
	// Value range:
	// - true/on
	// - false/off
	// Default value: false/off
	// Enumeration values:
	// true/on
	// false/off
	CompatibleIllegalChars string `json:"compatible_illegal_chars,omitempty"`
	// Maximum number of data format errors allowed during the data import.
	// If the number of data format errors does not reach the maximum, the data import is successful.
	// Value range:
	// - integer
	// - unlimited
	// Default value: 0,
	// indicating that error information is returned immediately
	RejectLimit string `json:"reject_limit,omitempty"`
	// Name of the error table that records data format errors.
	// After the parallel import is complete, you can query the error information table to obtain the detailed error information.
	ErrorTableName string `json:"error_table_name,omitempty"`
}

type PartitionResult 

type PartitionResult struct {
	// Current status of the partition.
	// Possible values:
	// - CREATING: The stream is being created.
	// - ACTIVE: The stream is available.
	// - DELETED: The stream is being deleted.
	// - EXPIRED: The stream has expired.
	// Enumeration values:
	//  CREATING
	//  ACTIVE
	//  DELETED
	//  EXPIRED
	Status string `json:"status,omitempty"`
	// Unique identifier of the partition.
	PartitionId string `json:"partition_id,omitempty"`
	// Possible value range of the hash key used by the partition.
	HashRange string `json:"hash_range,omitempty"`
	// Sequence number range of the partition.
	SequenceNumberRange string `json:"sequence_number_range,omitempty"`
	// Parent partition.
	ParentPartitions string `json:"parent_partitions,omitempty"`
}

type ProcessingSchema 

type ProcessingSchema struct {
	// Attribute name of the source data timestamp.
	TimestampName string `json:"timestamp_name"`
	// Type of the source data timestamp.
	// - String
	// - Timestamp: 13-bit timestamp of the long type
	TimestampType string `json:"timestamp_type"`
	// OBS directory generated based on the timestamp format.
	// This parameter is mandatory when the timestamp type of the source data is String.
	// Value range:
	// - yyyy/MM/dd HH:mm:ss
	// - MM/dd/yyyy HH:mm:ss
	// - dd/MM/yyyy HH:mm:ss
	// - yyyy-MM-dd HH:mm:ss
	// - MM-dd-yyyy HH:mm:ss
	// - dd-MM-yyyy HH:mm:ss
	// Enumeration values:
	// yyyy/MM/dd HH:mm:ss
	// MM/dd/yyyy HH:mm:ss
	// dd/MM/yyyy HH:mm:ss
	// yyyy-MM-dd HH:mm:ss
	// MM-dd-yyyy HH:mm:ss
	// dd-MM-yyyy HH:mm:ss
	TimestampFormat string `json:"timestamp_format,omitempty"`
}

type RowKey 

type RowKey struct {
	// JSON attribute name, which is used to generate HBase rowkeys for JSON data in the DIS stream.
	Value string `json:"value"`
	// JSON attribute type of JSON data in the DIS stream.
	// Value range:
	// - Bigint
	// - Double
	// - Boolean
	// - Timestamp
	// - String
	// - Decimal
	// Enumeration values:
	// Bigint
	// Double
	// Boolean
	// Timestamp
	// String
	// Decimal
	Type string `json:"type"`
}

type TransferTask 

type TransferTask struct {
	// Name of the dump task.
	TaskName string `json:"task_name,omitempty"`
	// Id of the dump task
	TaskId string `json:"task_id,omitempty"`
	// Dump task status. Possible values:
	// ERROR: An error occurs.
	// STARTING: The dump task is being started.
	// PAUSED: The dump task has been stopped.
	// RUNNING: The dump task is running.
	// DELETE: The dump task has been deleted.
	// ABNORMAL: The dump task is abnormal.
	// Enumeration values:
	// ERROR
	// STARTING
	// PAUSED
	// RUNNING
	// DELETE
	// ABNORMAL
	State string `json:"state,omitempty"`
	// Dump destination. Possible values:
	// OBS: Data is dumped to OBS.
	// Enumeration values:
	// OBS
	DestinationType string `json:"destination_type,omitempty"`
	// Time when the dump task is created.
	CreatedAt *int64 `json:"create_time,omitempty"`
	// Latest dump time of the dump task.
	LastTransferTimestamp *int64 `json:"last_transfer_timestamp,omitempty"`
}

type TransferTaskActionOpts 

type TransferTaskActionOpts struct {
	// Name of the stream to be queried.
	// Maximum: 60
	StreamName string `json:"stream_name"`
	// Dump task operation.
	// Currently, only the following operation is supported:
	// - stop: The dump task is stopped.
	// Enumeration values:
	//  stop
	// - start: The dump task is started.
	// Enumeration values:
	//  start
	Action string `json:"action"`
	// List of dump tasks to be paused.
	Tasks []BatchTransferTask `json:"tasks"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.