Documentation
¶
Index ¶
- func GetCompressionExtension(filename string) string
- type AzBlobStorage
- func (a *AzBlobStorage) Close() error
- func (a *AzBlobStorage) Download(filename string) (io.ReadCloser, error)
- func (a *AzBlobStorage) List(prefix string, recursive bool) ([]string, error)
- func (a *AzBlobStorage) Upload(filename string, reader io.Reader, compressFormat string, compressLevel int, ...) error
- type FTPStorage
- type FileStorage
- type GCSStorage
- type RemoteStorage
- type S3LogAdapter
- type S3Storage
- type SFTPStorage
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func GetCompressionExtension ¶
GetCompressionExtension extracts known compression extensions (.gz, .zstd) from a filename.
Types ¶
type AzBlobStorage ¶
type AzBlobStorage struct {
// contains filtered or unexported fields
}
func NewAzBlobStorage ¶
func NewAzBlobStorage(accountName, accountKey, containerName, endpoint string, debug bool) (*AzBlobStorage, error)
NewAzBlobStorage creates a new Azure Blob Storage client.
func (*AzBlobStorage) Close ¶
func (a *AzBlobStorage) Close() error
Close closes the Azure Blob Storage connection.
func (*AzBlobStorage) Download ¶
func (a *AzBlobStorage) Download(filename string) (io.ReadCloser, error)
Download retrieves a blob from Azure Blob Storage. If noClientDecompression is true, the raw blob stream is returned. Otherwise, decompressStream is used based on the filename's extension.
func (*AzBlobStorage) List ¶
func (a *AzBlobStorage) List(prefix string, recursive bool) ([]string, error)
List returns a list of blob names in the Azure container matching the prefix.
func (*AzBlobStorage) Upload ¶
func (a *AzBlobStorage) Upload(filename string, reader io.Reader, compressFormat string, compressLevel int, contentEncoding string) error
Upload uploads data to Azure Blob Storage. If contentEncoding is provided, it's assumed data is pre-compressed, and BlobHTTPHeaders.ContentEncoding is set. Otherwise, compressFormat and compressLevel are used for client-side compression.
type FTPStorage ¶
type FTPStorage struct {
// contains filtered or unexported fields
}
FTPStorage implements RemoteStorage for FTP servers.
func NewFTPStorage ¶
func NewFTPStorage(host, user, password string, debug bool) (*FTPStorage, error)
NewFTPStorage creates a new FTPStorage instance.
func (*FTPStorage) Close ¶
func (f *FTPStorage) Close() error
func (*FTPStorage) Download ¶
func (f *FTPStorage) Download(filename string) (io.ReadCloser, error)
type FileStorage ¶
type FileStorage struct {
// contains filtered or unexported fields
}
FileStorage implements RemoteStorage for local filesystem
func NewFileStorage ¶
func NewFileStorage(basePath string, debug bool) (*FileStorage, error)
NewFileStorage creates a new FileStorage instance
func (*FileStorage) Close ¶
func (f *FileStorage) Close() error
Close is a no-op for local file storage
func (*FileStorage) Download ¶
func (f *FileStorage) Download(fileName string) (io.ReadCloser, error)
Download reads data from a local file. If noClientDecompression is true, data is returned as is. Otherwise, decompressStream is used.
func (*FileStorage) List ¶
func (f *FileStorage) List(prefix string, recursive bool) ([]string, error)
List returns files matching the prefix in the base path
func (*FileStorage) Upload ¶
func (f *FileStorage) Upload(filename string, reader io.Reader, compressFormat string, compressLevel int, contentEncoding string) error
Upload writes data to a local file. If contentEncoding is provided, it's assumed data is pre-compressed. Otherwise, compressFormat and compressLevel are used.
type GCSStorage ¶
type GCSStorage struct {
// contains filtered or unexported fields
}
func NewGCSStorage ¶
func NewGCSStorage(bucketName, endpoint, credentialsFile string, debug bool) (*GCSStorage, error)
NewGCSStorage creates a new Google Cloud Storage client.
func (*GCSStorage) Close ¶
func (g *GCSStorage) Close() error
Close closes the underlying GCS client.
func (*GCSStorage) Download ¶
func (g *GCSStorage) Download(filename string) (io.ReadCloser, error)
Download retrieves an object from GCS. If noClientDecompression is true, the raw object stream is returned. Otherwise, decompressStream is used based on the filename's extension and object's ContentEncoding.
func (*GCSStorage) List ¶
func (g *GCSStorage) List(prefix string, recursive bool) ([]string, error)
List returns a list of object names in the GCS bucket matching the prefix.
func (*GCSStorage) Upload ¶
func (g *GCSStorage) Upload(filename string, reader io.Reader, compressFormat string, compressLevel int, contentEncoding string) error
Upload uploads data to GCS. If contentEncoding is provided, it's assumed data is pre-compressed, and GCS object's ContentEncoding metadata is set. Otherwise, compressFormat and compressLevel are used for client-side compression.
type RemoteStorage ¶
type RemoteStorage interface {
// Upload uploads data from the reader to the specified filename.
// If contentEncoding is provided (e.g., "gzip", "zstd"), the data is assumed to be
// already compressed in that format, and the corresponding extension (.gz, .zstd)
// will be appended to the filename. compressFormat and compressLevel are ignored in this case.
// Otherwise, if contentEncoding is empty, the data will be compressed using
// compressFormat and compressLevel before uploading, and the appropriate extension
// will be appended. If compressFormat is empty or "none", no compression is applied.
Upload(filename string, reader io.Reader, compressFormat string, compressLevel int, contentEncoding string) error
// Download retrieves the content of the specified filename.
// Implementations should automatically handle decompression based on common
// extensions (.gz, .zstd)
Download(filename string) (io.ReadCloser, error)
// List returns a list of filenames in the storage backend matching the prefix.
// The returned filenames might include compression extensions.
// If recursive is true, it will list all files under the prefix recursively.
// The prefix should be treated as a directory path when recursive=true.
List(prefix string, recursive bool) ([]string, error)
// Close terminates the connection to the storage backend, if applicable.
Close() error
}
RemoteStorage defines the interface for interacting with different storage backends.
type S3LogAdapter ¶
type S3LogAdapter struct {
}
func (S3LogAdapter) Logf ¶
func (apapter S3LogAdapter) Logf(severity awsV2Logging.Classification, msg string, args ...interface{})
type S3Storage ¶
type S3Storage struct {
// contains filtered or unexported fields
}
func NewS3Storage ¶
func (*S3Storage) Download ¶
func (s *S3Storage) Download(filename string) (io.ReadCloser, error)
Download retrieves an object from S3. If noClientDecompression is true, the raw object stream is returned. Otherwise, decompressStream is used based on the filename's extension.
func (*S3Storage) Upload ¶
func (s *S3Storage) Upload(filename string, reader io.Reader, compressFormat string, compressLevel int, contentEncoding string) error
Upload uploads data to S3. If contentEncoding is provided, it's assumed data is pre-compressed and ContentEncoding header is set. Otherwise, compressFormat and compressLevel are used for client-side compression.
type SFTPStorage ¶
type SFTPStorage struct {
// contains filtered or unexported fields
}
func NewSFTPStorage ¶
func NewSFTPStorage(host, user, password string, debug bool) (*SFTPStorage, error)
NewSFTPStorage creates a new SFTP storage client.
func (*SFTPStorage) Close ¶
func (s *SFTPStorage) Close() error
Close closes the SFTP client and the underlying SSH connection.
func (*SFTPStorage) Download ¶
func (s *SFTPStorage) Download(filename string) (io.ReadCloser, error)
Download retrieves a file from SFTP.
func (*SFTPStorage) List ¶
func (s *SFTPStorage) List(prefix string, recursive bool) ([]string, error)
List returns a list of filenames in the SFTP server matching the prefix. If recursive is true, it will list all files under the prefix recursively.
func (*SFTPStorage) Upload ¶
func (s *SFTPStorage) Upload(filename string, reader io.Reader, compressFormat string, compressLevel int, contentEncoding string) error
Upload uploads data via SFTP. If contentEncoding is provided, it's assumed data is pre-compressed. Otherwise, compressFormat and compressLevel are used for client-side compression.
Source Files