storage

Documentation

Index

• Constants
• type FileSystemStorage
  • func CreateFileSystemStorage(root string) (*FileSystemStorage, error)
  • func (s *FileSystemStorage) CopyFile(src, dst string) error
  • func (s *FileSystemStorage) CreateDir(path string) error
  • func (s *FileSystemStorage) CreateFile(path string, contents []byte) error
  • func (s *FileSystemStorage) CreateFileStreamed(path string, length int64, contents io.ReadCloser) error
  • func (s *FileSystemStorage) DeleteDir(path string, recursive bool) error
  • func (s *FileSystemStorage) DeleteFile(path string) error
  • func (s *FileSystemStorage) Exists(path string) (bool, string, error)
  • func (s *FileSystemStorage) GetFile(path string) (io.ReadCloser, error)
  • func (s *FileSystemStorage) ListDir(path string, recursive bool) ([]string, error)
  • func (s *FileSystemStorage) MergeFiles(dst string, paths []string, smallFiles bool) error
  • func (s *FileSystemStorage) MoveFile(src, dst string) error
• type MinioObjectStorage
  • func CreateMinioObjectStorage(config config.StorageS3Config) (*MinioObjectStorage, error)
  • func (s *MinioObjectStorage) CopyFile(src, dst string) error
  • func (s *MinioObjectStorage) CreateDir(path string) error
  • func (s *MinioObjectStorage) CreateFile(path string, contents []byte) error
  • func (s *MinioObjectStorage) CreateFileStreamed(path string, length int64, contents io.ReadCloser) error
  • func (s *MinioObjectStorage) DeleteDir(path string, recursive bool) error
  • func (s *MinioObjectStorage) DeleteFile(path string) error
  • func (s *MinioObjectStorage) Exists(path string) (bool, string, error)
  • func (s *MinioObjectStorage) GetFile(path string) (io.ReadCloser, error)
  • func (s *MinioObjectStorage) ListDir(path string, recursive bool) ([]string, error)
  • func (s *MinioObjectStorage) MergeFiles(dst string, paths []string, smallFiles bool) error
  • func (s *MinioObjectStorage) MoveFile(src, dst string) error
• type Storage

Constants

const MinioNotExistsError = "The specified key does not exist."

Types

type FileSystemStorage

type FileSystemStorage struct {
	Storage
	// contains filtered or unexported fields
}

FileSystemStorage

Implementation of the Storage interface for filesystems

func CreateFileSystemStorage

func CreateFileSystemStorage(root string) (*FileSystemStorage, error)

CreateFileSystemStorage

Creates a new FileSystemStorage including creating the configured root path
if it doesn't already exist.

func (*FileSystemStorage) CopyFile

func (s *FileSystemStorage) CopyFile(src, dst string) error

CopyFile

    Copies a file within the configured bucket.

Args:
       - src (string): The path of the file to copy.
       - dst (string): The new path of the file.

func (*FileSystemStorage) CreateDir

func (s *FileSystemStorage) CreateDir(path string) error

CreateDir

    Creates a new directory in the configured bucket.

Args:
       - path (string): The path of the directory to create.

func (*FileSystemStorage) CreateFile

func (s *FileSystemStorage) CreateFile(path string, contents []byte) error

CreateFile

Creates a new file in the configured bucket.

Args:
   - path (string): The path of the file to create.
   - contents ([]byte): The contents of the file.

func (*FileSystemStorage) CreateFileStreamed

func (s *FileSystemStorage) CreateFileStreamed(path string, length int64, contents io.ReadCloser) error

CreateFileStreamed

  Creates a new file in the configured bucket reading from an io.ReadCloser.

Args:
      - path (string): The path of the file to create.
	  - length (int64): The size in bytes of the contents.
      - contents (io.ReadCloser): The contents of the file.

func (*FileSystemStorage) DeleteDir

func (s *FileSystemStorage) DeleteDir(path string, recursive bool) error

DeleteDir

    Deletes a directory in the configured bucket.

Args:
       - path (string): The path of the directory to delete.
	   - recursive (bool): Whether to delete all subdirectories within the passed directory

func (*FileSystemStorage) DeleteFile

func (s *FileSystemStorage) DeleteFile(path string) error

DeleteFile

    Deletes a file from the configured bucket.

Args:
       - path (string): The path of the file to delete.

func (*FileSystemStorage) Exists

func (s *FileSystemStorage) Exists(path string) (bool, string, error)

Exists

   Checks whether the path exists in the configured bucket
   and returns what type of path it is (file, directory, symlink, etc.).

Args:
    - path (string): The path of the file to check.

Returns:
    - (bool): Whether the path exists or not.
    - (string): Path type

func (*FileSystemStorage) GetFile

func (s *FileSystemStorage) GetFile(path string) (io.ReadCloser, error)

GetFile

		Returns a file from the configured bucket.
     Returns nil if the file does not exist.

		Args:
	       - path (string): The path of the file to retrieve.

	 Returns:
	       - (io.ReadCloser): The contents of the file.

func (*FileSystemStorage) ListDir

func (s *FileSystemStorage) ListDir(path string, recursive bool) ([]string, error)

ListDir

	       Lists the contents of a directory in the configured bucket.

	   Args:
	        - path (string): The path of the directory to list.
			- recursive (bool): Whether to list the directory recursively.
	   Returns:
         - []string: The list of files in the directory.

func (*FileSystemStorage) MergeFiles

func (s *FileSystemStorage) MergeFiles(dst string, paths []string, smallFiles bool) error

MergeFiles

    Merges multiple files within the configured bucket.

Args:
       - dst (string): The path of the merged file in the configured bucket.
       - paths ([]string): The paths of the files to merge in order of merge.
       - smallFiles (bool): This parameter is a no-op in this implementation and only used
                            for compatibility with the Storage interface

func (*FileSystemStorage) MoveFile

func (s *FileSystemStorage) MoveFile(src, dst string) error

MoveFile

    Moves a file within the configured bucket.

Args:
       - src (string): The path of the file to move.
       - dst (string): The new path of the file.

type MinioObjectStorage

type MinioObjectStorage struct {
	Storage
	// contains filtered or unexported fields
}

MinioObjectStorage

Implementation of the Storage interface for Minio and all S3 compliant
object storage systems.

func CreateMinioObjectStorage

func CreateMinioObjectStorage(config config.StorageS3Config) (*MinioObjectStorage, error)

CreateMinioObjectStorage

Creates a new MinioObjectStorage including initialization of the Minio client
and creating the configured bucket if it doesn't already exist.

func (*MinioObjectStorage) CopyFile

func (s *MinioObjectStorage) CopyFile(src, dst string) error

CopyFile

    Copies a file within the configured bucket.

Args:
       - src (string): The path of the file to copy.
       - dst (string): The new path of the file.

func (*MinioObjectStorage) CreateDir

func (s *MinioObjectStorage) CreateDir(path string) error

CreateDir

    Creates a new directory in the configured bucket.

	NOTE:
	Since object storage doesn't have the concept of directories
	only prefixes, this function is a no-op. Creating a "directory"
	in object storage is as simple as creating a key (file) at the
	desired directory and the entire prefix (subdirectories) will
	be automatically created.
Args:
       - path (string): The path of the directory to create.

func (*MinioObjectStorage) CreateFile

func (s *MinioObjectStorage) CreateFile(path string, contents []byte) error

CreateFile

Creates a new file in the configured bucket.

Args:
   - path (string): The path of the file to create.
   - contents ([]byte): The contents of the file.

func (*MinioObjectStorage) CreateFileStreamed

func (s *MinioObjectStorage) CreateFileStreamed(path string, length int64, contents io.ReadCloser) error

CreateFileStreamed

  Creates a new file in the configured bucket reading from an io.ReadCloser.

Args:
      - path (string): The path of the file to create.
	  - length (int64): The size in bytes of the contents.
      - contents (io.ReadCloser): The contents of the file.

func (*MinioObjectStorage) DeleteDir

func (s *MinioObjectStorage) DeleteDir(path string, recursive bool) error

DeleteDir

    Deletes a directory in the configured bucket.

Args:
       - path (string): The path of the directory to delete.
	   - recursive (bool): Whether to delete all subdirectories within the passed directory

func (*MinioObjectStorage) DeleteFile

func (s *MinioObjectStorage) DeleteFile(path string) error

DeleteFile

    Deletes a file from the configured bucket.

Args:
       - path (string): The path of the file to delete.

func (*MinioObjectStorage) Exists

func (s *MinioObjectStorage) Exists(path string) (bool, string, error)

Exists

	   Checks whether the path exists in the configured bucket
	   and returns what type of path it is (file, directory, symlink, etc.).

	   NOTE:
	   Since object storage doesn't have a concept of directories this
    function will only check if a key (file) exists. If a valid
	   prefix (directory) is passed the function will return false.
	Args:
	    - path (string): The path of the file to check.

	Returns:
	    - (bool): Whether the path exists or not.
	    - (string): Path type

func (*MinioObjectStorage) GetFile

func (s *MinioObjectStorage) GetFile(path string) (io.ReadCloser, error)

GetFile

		Returns a file from the configured bucket.
     Returns nil if the file does not exist.

		Args:
	       - path (string): The path of the file to retrieve.

	 Returns:
	       - (io.ReadCloser): The contents of the file.

func (*MinioObjectStorage) ListDir

func (s *MinioObjectStorage) ListDir(path string, recursive bool) ([]string, error)

ListDir

	       Lists the contents of a directory in the configured bucket.

	   Args:
	        - path (string): The path of the directory to list.
	   		- recursive (bool): Whether to list the directory recursively.
	   Returns:
         - []string: The list of files in the directory.

func (*MinioObjectStorage) MergeFiles

func (s *MinioObjectStorage) MergeFiles(dst string, paths []string, smallFiles bool) error

MergeFiles

	   Merges multiple files within the configured bucket.

    NOTE:
    The S3 api does not permit the server-side composition (merging) of
    files smaller than 5MB excluding the final file. This function has
    the necessary logic to manually merge the files in the case that
    any of the files (excluding the final) are less than 5MB. However,
    manually merging files requires that the files be downloaded to the
    local file system, merged into single file locally, and re-uploaded
    as the final merged file.

    IT IS THE RESPONSIBILITY OF THE CALLER TO ENSURE SUFFICIENT SPACE AND
	   BANDWIDTH ARE AVAILABLE TO PERFORM A LOCAL MERGE OF THE FILES

	   Args:
           - dst (string): The path of the merged file in the configured bucket.
           - paths ([]string): The paths of the files to merge in order of merge.
           - smallFiles (bool): Whether to manually merge small files if they cannot be merged on the server (see note above)

func (*MinioObjectStorage) MoveFile

func (s *MinioObjectStorage) MoveFile(src, dst string) error

MoveFile

    Moves a file within the configured bucket.

Args:
       - src (string): The path of the file to move.
       - dst (string): The new path of the file.

type Storage

type Storage interface {
	// GetFile
	//
	//		Returns a file from the configured bucket.
	//      Returns nil if the file does not exist.
	//
	//		Args:
	//	       - path (string): The path of the file to retrieve.
	//
	//	 Returns:
	//	       - (io.ReadCloser): The contents of the file.
	GetFile(path string) (io.ReadCloser, error)

	// CreateFile
	//
	//	Creates a new file in the configured bucket.
	//
	//	Args:
	//	   - path (string): The path of the file to create.
	//	   - contents ([]byte): The contents of the file.
	CreateFile(path string, contents []byte) error

	// CreateFileStreamed
	//
	//	  Creates a new file in the configured bucket reading from an io.ReadCloser.
	//
	//	Args:
	//	      - path (string): The path of the file to create.
	//		  - length (int64): The size in bytes of the contents.
	//	      - contents (io.ReadCloser): The contents of the file.
	CreateFileStreamed(path string, length int64, contents io.ReadCloser) error

	// DeleteFile
	//
	//	    Deletes a file from the configured bucket.
	//
	//	Args:
	//	       - path (string): The path of the file to delete.
	DeleteFile(path string) error

	// MoveFile
	//
	//	    Moves a file within the configured bucket.
	//
	//	Args:
	//	       - src (string): The path of the file to move.
	//	       - dst (string): The new path of the file.
	MoveFile(src, dst string) error

	// CopyFile
	//
	//        Copies a file within the configured bucket.
	//
	//    Args:
	//           - src (string): The path of the file to copy.
	//           - dst (string): The new path of the file.
	CopyFile(src, dst string) error

	// MergeFiles
	//
	//        Merges multiple files within the configured bucket.
	//
	//    Args:
	//           - dst (string): The path of the merged file in the configured bucket.
	//           - paths ([]string): The paths of the files to merge in order of merge.
	//           - smallFiles (bool): Used for handling local merges of small files in some object stores
	//                                check the header doc for this function on each implementation to
	//                                determine if this parameter is used and what the implications of its
	//                                use are
	MergeFiles(dst string, paths []string, smallFiles bool) error

	// Exists
	//
	//	   Checks whether the path exists in the configured bucket
	//	   and returns what type of path it is (file, directory, symlink, etc.).
	//
	//	Args:
	//	    - path (string): The path of the file to check.
	//
	//	Returns:
	//	    - (bool): Whether the path exists or not.
	//	    - (string): Path type
	Exists(path string) (bool, string, error)

	// CreateDir
	//
	//	    Creates a new directory in the configured bucket.
	//
	//	Args:
	//	       - path (string): The path of the directory to create.
	CreateDir(path string) error

	// ListDir
	//
	//	       Lists the contents of a directory in the configured bucket.
	//
	//	   Args:
	//	        - path (string): The path of the directory to list.
	//			- recursive (bool): Whether to list the directory recursively.
	//	   Returns:
	//          - []string: The list of files in the directory.
	ListDir(path string, recursive bool) ([]string, error)

	// DeleteDir
	//
	//	    Deletes a directory in the configured bucket.
	//
	//	Args:
	//	       - path (string): The path of the directory to delete.
	//		   - recursive (bool): Whether to delete all subdirectories within the passed directory
	DeleteDir(path string, recursive bool) error
}

Storage Interface for accessing remote object storage systems and local file systems