bucketclient

package module
v0.0.0-...-c389e54 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 20, 2025 License: Apache-2.0 Imports: 12 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	MetricsNamespace = "s3_metadata_bucketclient"
)

Variables

View Source 
var MetricsSummaryDefaultObjectives = map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001, 1.0: 0.0001}

Functions

func CompareVersionsListingMarkers 

func CompareVersionsListingMarkers(keyMarker1 string, versionIdMarker1 string,
	keyMarker2 string, versionIdMarker2 string) int

CompareVersionsListingMarkers is a helper function that returns -1, 0, or 1 if the pair keyMarker1/versionIdMarker1 is lexicographically, respectively strictly lower, equal, or strictly higher than the pair keyMarker2/versionIdMarker2.

func CreateBucketMakeIdempotent 

func CreateBucketMakeIdempotent(options *createBucketOptionSet)

func ErrorMalformedResponse 

func ErrorMalformedResponse(apiMethod string, httpMethod string, endpoint string, resource string,
	err error) error

func RequestIdempotent 

func RequestIdempotent(ros *requestOptionSet)

Types

type AdminGetSessionLogResponse 

type AdminGetSessionLogResponse struct {
	Info SessionLogInfo     `json:"info"`
	Log  []SessionLogRecord `json:"log"`
}

type AppendToLogEntry 

type AppendToLogEntry struct {
	Type     string    `json:"type"`
	Key      string    `json:"key,omitempty"`
	Value    string    `json:"value,omitempty"`
	Overhead *Overhead `json:"overhead,omitempty"`
}

type BucketAccessMode 

type BucketAccessMode string
const (
	BucketAccessModeReadWrite BucketAccessMode = "read-write"
	BucketAccessModeReadOnly  BucketAccessMode = "read-only"
)

type BucketClient 

type BucketClient struct {
	*http.Client
	Endpoint string
	Metrics  *BucketClientMetrics
}

func New 

func New(bucketdEndpoint string) *BucketClient

New creates a new BucketClient instance, with the provided endpoint (e.g. "localhost:9000") and the default HTTP client.

func NewWithHTTPClient 

func NewWithHTTPClient(bucketdEndpoint string, httpClient *http.Client) *BucketClient

NewWithHTTPClient creates a new BucketClient instance, with the provided endpoint (e.g. "localhost:9000") using the provided http.Client instance.

func (*BucketClient) AdminBucketRefreshCache 

func (client *BucketClient) AdminBucketRefreshCache(ctx context.Context, bucketName string,
	opts ...RequestOption) error

AdminBucketRefreshCache refreshes the bucketd cache of metastore entries for the given bucket. Useful after switching the raft session of a bucket.

func (*BucketClient) AdminGetAllSessionsInfo 

func (client *BucketClient) AdminGetAllSessionsInfo(ctx context.Context,
	opts ...RequestOption) ([]SessionInfo, error)

AdminGetAllSessionsInfo returns raft session info for all Metadata raft sessions available on the S3C deployment.

func (*BucketClient) AdminGetBucketAccessMode 

func (client *BucketClient) AdminGetBucketAccessMode(ctx context.Context,
	bucketName string, opts ...RequestOption) (BucketAccessMode, error)

AdminGetBucketAccessMode returns the access mode of the given bucket: - "read-write" when the bucket is accessible for reading and writing (default) - "read-only" when the bucket is only accessible for read operations. Returns "" and an error if the bucket doesn't exist, or if a request error occurs.

func (*BucketClient) AdminGetBucketDBBackend 

func (client *BucketClient) AdminGetBucketDBBackend(ctx context.Context, bucketName string,
	opts ...RequestOption) (string, error)

AdminGetBucketDBBackend returns the database backend type of the given bucket. Currently, it returns either "leveldb" or "rocksdb". Returns an empty string and an error if the bucket doesn't exist, or if a request error occurs.

func (*BucketClient) AdminGetBucketSessionID 

func (client *BucketClient) AdminGetBucketSessionID(ctx context.Context, bucketName string,
	opts ...RequestOption) (int, error)

AdminGetBucketSessionID returns the raft session ID of the given bucket. Returns 0 and an error if the bucket doesn't exist, or if a request error occurs.

func (*BucketClient) AdminGetSessionDBBackend 

func (client *BucketClient) AdminGetSessionDBBackend(ctx context.Context, sessionId int,
	opts ...RequestOption) (string, error)

AdminGetSessionDBBackend returns the database backend type of the given RAFT session. Currently, it returns either "leveldb" or "rocksdb". Returns an empty string and an error if the RAFT session doesn't exist, or if a request error occurs.

func (*BucketClient) AdminGetSessionInfo 

func (client *BucketClient) AdminGetSessionInfo(ctx context.Context,
	sessionId int, opts ...RequestOption) (*SessionInfo, error)

AdminGetSessionInfo returns raft session info for the given raft session ID. Returns nil and an error if the raft session doesn't exist, or if a request error occurs.

func (*BucketClient) AdminGetSessionLeader 

func (client *BucketClient) AdminGetSessionLeader(ctx context.Context, sessionId int,
	opts ...RequestOption) (*MemberInfo, error)

Returns nil and an error if the raft session doesn't exist, if bucketd is not connected to the leader, or if a request error occurs.

func (*BucketClient) AdminGetSessionLog 

func (client *BucketClient) AdminGetSessionLog(ctx context.Context,
	sessionId int, beginSeq int64, nRecords int, targetLeader bool,
	opts ...RequestOption) (*AdminGetSessionLogResponse, error)

AdminGetSessionLog returns a range of Raft oplog from the given raft session. sessionId is the raft session ID beginSeq is the first raft sequence number to fetch nRecords is the maximum number of records to fetch starting from beginSeq if targetLeader is true, it fetches the oplog from the leader, otherwise fetches from one of the followers

func (*BucketClient) AdminSetBucketAccessMode 

func (client *BucketClient) AdminSetBucketAccessMode(ctx context.Context,
	bucketName string, accessMode BucketAccessMode, opts ...RequestOption) error

AdminSetBucketAccessMode sets the access mode of the given bucket:

  • "read-write" to restore the default read/write access
  • "read-only" to set the bucket in read-only mode and refuse write operations with a 503 ServiceUnavailable error.

Returns an error if the bucket doesn't exist, or if a request error occurs.

func (*BucketClient) AppendToLog 

func (client *BucketClient) AppendToLog(ctx context.Context,
	bucketName string, batch []AppendToLogEntry, sessionId *string, opts ...RequestOption) error

func (*BucketClient) CreateBucket 

func (client *BucketClient) CreateBucket(ctx context.Context,
	bucketName string, bucketAttributes []byte, opts ...CreateBucketOption) error

CreateBucket creates a bucket in metadata. bucketAttributes is a JSON blob of bucket attributes opts is a set of options: 

	CreateBucketSessionIdOption forces the session ID where the bucket to be
	    created will land

	CreateBucketMakeIdempotent makes the request return a success if a bucket
	    with the same UID already exists (otherwise returns 409 Conflict, as
	    if the option is not passed)

     CreateBucketRequestUIDsOption attaches existing UIDs to the CreateBucket request

func (*BucketClient) CreateMetastoreEntry 

func (client *BucketClient) CreateMetastoreEntry(ctx context.Context, bucketName string,
	metastoreEntry MetastoreEntry, opts ...RequestOption) error

CreateMetastoreEntry creates or updates a metastore entry for the given bucket

func (*BucketClient) DeleteBucket 

func (client *BucketClient) DeleteBucket(ctx context.Context, bucketName string,
	opts ...RequestOption) error

DeleteBucket deletes a bucket entry from metadata.

func (*BucketClient) DeleteMetastoreEntry 

func (client *BucketClient) DeleteMetastoreEntry(ctx context.Context, bucketName string,
	opts ...RequestOption) error

DeleteMetastoreEntry deletes the metastore entry for the given bucket

func (*BucketClient) EnableMetrics 

func (client *BucketClient) EnableMetrics(registerer prometheus.Registerer)

EnableMetrics enables Prometheus metrics gathering for the provided client and registers them in the provided registerer.

Metrics implemented:

  • `s3_metadata_bucketclient_requests_total`: Number of requests processed (counter)
  • `s3_metadata_bucketclient_request_duration_seconds`: Time elapsed processing requests to bucketd, in seconds (summary)
  • `s3_metadata_bucketclient_request_bytes_sent_total`: Number of request body bytes sent to bucketd (counter)
  • `s3_metadata_bucketclient_response_bytes_received_total`: Number of response body bytes received from bucketd (counter)

Metrics have the following labels attached:

  • `endpoint`: bucketd endpoint such as `http://localhost:9000`
  • `method`: HTTP method
  • `action`: name of the API action, such as `CreateBucket`. Admin actions are prefixed with `Admin`.
  • `code`: HTTP status code returned, or "0" for generic network or protocol errors

func (*BucketClient) GetBucketAttributes 

func (client *BucketClient) GetBucketAttributes(ctx context.Context, bucketName string,
	opts ...RequestOption) ([]byte, error)

GetBucketAttributes retrieves the JSON blob containing the bucket attributes attached to a bucket.

func (*BucketClient) GetMetastoreEntry 

func (client *BucketClient) GetMetastoreEntry(ctx context.Context, bucketName string,
	opts ...RequestOption) (MetastoreEntry, error)

GetMetastoreEntry retrieves and parses a metastore entry for the given bucket

func (*BucketClient) ListBasic 

func (client *BucketClient) ListBasic(ctx context.Context,
	bucketName string, opts ...ListBasicOption) (*ListBasicResponse, error)

func (*BucketClient) ListObjectVersions 

func (client *BucketClient) ListObjectVersions(ctx context.Context,
	bucketName string, opts ...ListObjectVersionsOption) (*ListObjectVersionsResponse, error)

func (*BucketClient) PostBatch 

func (client *BucketClient) PostBatch(ctx context.Context,
	bucketName string, batch []PostBatchEntry, opts ...RequestOption) error

func (*BucketClient) PutBucketAttributes 

func (client *BucketClient) PutBucketAttributes(ctx context.Context, bucketName string,
	bucketAttributes []byte, opts ...RequestOption) error

PutBucketAttributes updates the bucket attributes with a new JSON blob.

func (*BucketClient) Request 

func (client *BucketClient) Request(ctx context.Context,
	apiMethod string, httpMethod string, resource string, opts ...RequestOption) ([]byte, error)

type BucketClientError 

type BucketClientError struct {
	ApiMethod  string
	HttpMethod string
	Endpoint   string
	Resource   string
	StatusCode int
	ErrorType  string
	Err        error
}

func (*BucketClientError) Error 

func (e *BucketClientError) Error() string

func (*BucketClientError) Unwrap 

func (e *BucketClientError) Unwrap() error

type BucketClientMetrics 

type BucketClientMetrics struct {
	RequestsTotal              *prometheus.CounterVec
	RequestDurationSeconds     *prometheus.SummaryVec
	RequestBytesSentTotal      *prometheus.CounterVec
	ResponseBytesReceivedTotal *prometheus.CounterVec
}

type CreateBucketOption 

type CreateBucketOption func(*createBucketOptionSet)

func CreateBucketRequestUIDsOption 

func CreateBucketRequestUIDsOption(uids string) CreateBucketOption

func CreateBucketSessionIdOption 

func CreateBucketSessionIdOption(sessionId int) CreateBucketOption

type DBMethodType 

type DBMethodType int
const (
	DBMethodCreate        DBMethodType = 0
	DBMethodDelete        DBMethodType = 1
	DBMethodGet           DBMethodType = 2
	DBMethodPut           DBMethodType = 3
	DBMethodList          DBMethodType = 4
	DBMethodDel           DBMethodType = 5
	DBMethodGetAttributes DBMethodType = 6
	DBMethodPutAttributes DBMethodType = 7
	DBMethodBatch         DBMethodType = 8
	DBMethodNoop          DBMethodType = 9
)

type ListBasicEntry 

type ListBasicEntry struct {
	Key   string `json:"key"`
	Value string `json:"value"`
}

type ListBasicOption 

type ListBasicOption func(*listBasicOptionSet) error

func ListBasicGTEOption 

func ListBasicGTEOption(gte string) ListBasicOption

ListBasicGTEOption only lists keys greater or equal to the given argument

func ListBasicGTOption 

func ListBasicGTOption(gt string) ListBasicOption

ListBasicGTOption only lists keys greater than the given argument

func ListBasicLTEOption 

func ListBasicLTEOption(lte string) ListBasicOption

ListBasicLTEOption only lists keys less or equal to the given argument

func ListBasicLTOption 

func ListBasicLTOption(lt string) ListBasicOption

ListBasicLTOption only lists keys less than the given argument

func ListBasicMaxKeysOption 

func ListBasicMaxKeysOption(maxKeys int) ListBasicOption

ListBasicMaxKeysOption limits the number of returned keys (default and maximum is 10000).

func ListBasicNoKeysOption 

func ListBasicNoKeysOption() ListBasicOption

ListBasicNoKeysOption declares that keys are not needed in the result entries and may be returned empty.

Note: keys may still be returned until ARSN-438 is fixed.

func ListBasicNoValuesOption 

func ListBasicNoValuesOption() ListBasicOption

ListBasicNoValuesOption declares that values are not needed in the result entries and may be returned empty.

Note: values may still be returned until ARSN-438 is fixed.

func ListBasicRequestUIDsOption 

func ListBasicRequestUIDsOption(uids string) ListBasicOption

ListBasicRequestUIDsOption attaches existing request UIDs to the ListBasic request

type ListBasicResponse 

type ListBasicResponse []ListBasicEntry

type ListObjectVersionsEntry 

type ListObjectVersionsEntry struct {
	Key       string `json:"key"`
	VersionId string `json:"versionId"`
	Value     string `json:"value"`
}

type ListObjectVersionsOption 

type ListObjectVersionsOption func(*listObjectVersionsOptionSet) error

func ListObjectVersionsLastMarkerOption 

func ListObjectVersionsLastMarkerOption(lastKeyMarker string, lastVersionIdMarker string) ListObjectVersionsOption

ListObjectVersionsLastMarkerOption option makes the listing behave as if the bucket contains no object which key/versionId is strictly higher than the pair "lastKeyMarker/lastVersionIdMarker".

Note: this option is not implemented natively by bucketd, hence the Go client may truncate the result and adjust the "IsTruncated" field accordingly, before returning the truncated response to the client.

func ListObjectVersionsMarkerOption 

func ListObjectVersionsMarkerOption(keyMarker string, versionIdMarker string) ListObjectVersionsOption

func ListObjectVersionsMaxKeysOption 

func ListObjectVersionsMaxKeysOption(maxKeys int) ListObjectVersionsOption

func ListObjectVersionsRequestUIDsOption 

func ListObjectVersionsRequestUIDsOption(uids string) ListObjectVersionsOption

ListObjectVersionsRequestUIDsOption attaches existing request UIDs to the ListObjectVersions request

type ListObjectVersionsResponse 

type ListObjectVersionsResponse struct {
	Versions            []ListObjectVersionsEntry
	CommonPrefixes      []string
	IsTruncated         bool
	NextKeyMarker       string `json:",omitempty"`
	NextVersionIdMarker string `json:",omitempty"`
}

type MemberInfo 

type MemberInfo struct {
	ID          int    `json:"id"`
	Name        string `json:"name"`
	DisplayName string `json:"display_name"`
	Host        string `json:"host"`
	Port        int    `json:"port"`
	AdminPort   int    `json:"adminPort"`
	MDClusterId string `json:"mdClusterId"`
}

type MetastoreEntry 

type MetastoreEntry struct {
	Name          string `json:"name"`
	Attributes    string `json:"attributes"`
	Creating      bool   `json:"creating"`
	Deleting      bool   `json:"deleting"`
	ID            string `json:"id"`
	RaftSessionID int    `json:"raftSessionID"`
	Version       int    `json:"version"`
	RaftSession   string `json:"raftSession"`
}

type Overhead 

type Overhead struct {
	InternalOp bool `json:"internalOp,omitempty"`
}

type PostBatchEntry 

type PostBatchEntry struct {
	Key      string    `json:"key"`
	Value    string    `json:"value,omitempty"`
	Overhead *Overhead `json:"overhead,omitempty"`
	Type     string    `json:"type,omitempty"`
}

type RequestOption 

type RequestOption func(*requestOptionSet)

func RequestBodyContentTypeOption 

func RequestBodyContentTypeOption(contentType string) RequestOption

func RequestBodyOption 

func RequestBodyOption(body []byte) RequestOption

func RequestUIDsOption 

func RequestUIDsOption(uids string) RequestOption

type SessionInfo 

type SessionInfo struct {
	ID                int          `json:"id"`
	RaftMembers       []MemberInfo `json:"raftMembers"`
	ConnectedToLeader bool         `json:"connectedToLeader"`
}

type SessionLogEntry 

type SessionLogEntry struct {
	Key   string `json:"key,omitempty"`
	Value string `json:"value,omitempty"`
	Type  string `json:"type,omitempty"`
}

type SessionLogInfo 

type SessionLogInfo struct {
	Start int64 `json:"start"`
	CSeq  int64 `json:"cseq"`
	Prune int64 `json:"prune"`
}

type SessionLogRecord 

type SessionLogRecord struct {
	Bucket    string            `json:"db"`
	DBMethod  DBMethodType      `json:"method"`
	Timestamp string            `json:"timestamp,omitempty"`
	Entries   []SessionLogEntry `json:"entries"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.