internal

package
v0.40.8 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 22, 2024 License: Apache-2.0 Imports: 59 Imported by: 0

Documentation

Index

Constants

View Source
const (
	N_INODES  = fuseops.InodeID(1 << 32)
	N_HANDLES = fuseops.HandleID(1 << 32)

	STEAL_INODE_BACKOFF = 200 * time.Millisecond

	UNKNOWN_OWNER = 0
)
View Source
const (
	SRC_NODE_ID_METADATA_KEY = "src-node-id"
	DST_NODE_ID_METADATA_KEY = "dst-node-id"
)
View Source
const (
	FALLOC_FL_KEEP_SIZE      = uint32(0x01)
	FALLOC_FL_PUNCH_HOLE     = uint32(0x02)
	FALLOC_FL_COLLAPSE_RANGE = uint32(0x08)
	FALLOC_FL_ZERO_RANGE     = uint32(0x10)
	FALLOC_FL_INSERT_RANGE   = uint32(0x20)
)
View Source
const (
	ST_CACHED   int32 = 0
	ST_DEAD     int32 = 1
	ST_CREATED  int32 = 2
	ST_MODIFIED int32 = 3
	ST_DELETED  int32 = 4
)
View Source
const (
	XATTR_CREATE  = unix.XATTR_CREATE
	XATTR_REPLACE = unix.XATTR_REPLACE
	ENOATTR       = unix.ENODATA
)
View Source
const ADL1_REQUEST_ID = "X-Ms-Request-Id"
View Source
const ADL2_CLIENT_REQUEST_ID = "X-Ms-Client-Request-Id"
View Source
const ADL2_REQUEST_ID = "X-Ms-Request-Id"
View Source
const AzureBlobMetaDataHeaderPrefix = "x-ms-meta-"
View Source
const AzureDirBlobMetadataKey = "hdi_isfolder"
View Source
const AzuriteEndpoint = "http://127.0.0.1:8080/devstoreaccount1/"
View Source
const CGROUP_FOLDER_PREFIX = "/sys/fs/cgroup/memory"
View Source
const CGROUP_PATH = "/proc/self/cgroup"
View Source
const INIT_ERR_BLOB = "mount.err"
View Source
const IOV_MAX = 1024

On Linux and MacOS, IOV_MAX = 1024

View Source
const MAX_BUF = 2 * 1024 * 1024
View Source
const MAX_FLUSH_PRIORITY = 3
View Source
const MEM_LIMIT_FILE_SUFFIX = "/memory.limit_in_bytes"
View Source
const MEM_USAGE_FILE_SUFFIX = "/memory.usage_in_bytes"
View Source
const OUTSTAGE_TIMEOUT = 10 * time.Second
View Source
const READY_OWNER_BACKOFF = 100 * time.Millisecond
View Source
const READ_BUF_SIZE = 128 * 1024
View Source
const STAT_PRINT_INTERVAL = 1 * time.Second

Variables

View Source
var ErrBufferIsLoading = errors.New("tried to read from a loading buffer")
View Source
var ErrBufferIsMissing = errors.New("tried to read from a missing buffer")
View Source
var SmallActionsGate = make(chan int, 100)
View Source
var TIME_MAX = time.Unix(1<<63-62135596801, 999999999)

Functions

func Dup

func Dup(value []byte) []byte

func IsADLv1Endpoint

func IsADLv1Endpoint(endpoint string) bool

func IsADLv2Endpoint

func IsADLv2Endpoint(endpoint string) bool

func LogClientInterceptor added in v0.36.0

func LogClientInterceptor(ctx context.Context, method string, req, resp interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error

func LogServerInterceptor added in v0.36.0

func LogServerInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error)

func MaxInt

func MaxInt(a, b int) int

func MaxInt64

func MaxInt64(a, b int64) int64

func MaxUInt32

func MaxUInt32(a, b uint32) uint32

func MaxUInt64

func MaxUInt64(a, b uint64) uint64

func MinInt

func MinInt(a, b int) int

func MinInt64

func MinInt64(a, b int64) int64

func MinUInt32

func MinUInt32(a, b uint32) uint32

func MinUInt64

func MinUInt64(a, b uint64) uint64

func MountCluster added in v0.36.0

func MountCluster(
	ctx context.Context,
	bucketName string,
	flags *cfg.FlagStorage,
) (*Goofys, MountedFS, error)

func MountFuse added in v0.36.0

func MountFuse(
	ctx context.Context,
	bucketName string,
	flags *cfg.FlagStorage) (fs *Goofys, mfs MountedFS, err error)

Mount the file system based on the supplied arguments, returning a fuse.MountedFileSystem that can be joined to wait for unmounting.

func NilInt64 added in v0.37.1

func NilInt64(v *int64) int64

func NilStr

func NilStr(v *string) string

func NilUInt32 added in v0.38.5

func NilUInt32(v *uint32) uint32

func PBool

func PBool(v bool) *bool

func PInt32

func PInt32(v int32) *int32

func PInt64

func PInt64(v int64) *int64

func PMetadata

func PMetadata(m map[string]string) map[string]*string

func PString

func PString(v string) *string

func PTime

func PTime(v time.Time) *time.Time

func PUInt32

func PUInt32(v uint32) *uint32

func PUInt64

func PUInt64(v uint64) *uint64

func ReadBackoff added in v0.40.0

func ReadBackoff(flags *cfg.FlagStorage, try func(attempt int) error) (err error)

FIXME: Add similar write backoff (now it's handled by file/dir code)

func SignV2

func SignV2(req *request.Request)

Sign requests with signature version 2.

Will sign the requests with the service config's Credentials object Signing is skipped if the credentials is the credentials.AnonymousCredentials object.

func TryUnmount

func TryUnmount(mountPoint string) (err error)

Types

type ADL2Error

type ADL2Error struct {
	adl2.DataLakeStorageError
}

func (ADL2Error) Error

func (e ADL2Error) Error() string

type ADLv1

type ADLv1 struct {
	// contains filtered or unexported fields
}

func NewADLv1

func NewADLv1(bucket string, flags *cfg.FlagStorage, config *cfg.ADLv1Config) (*ADLv1, error)

func (*ADLv1) Bucket

func (b *ADLv1) Bucket() string

func (*ADLv1) Capabilities

func (b *ADLv1) Capabilities() *Capabilities

func (*ADLv1) CopyBlob

func (b *ADLv1) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error)

func (*ADLv1) Delegate

func (b *ADLv1) Delegate() interface{}

func (*ADLv1) DeleteBlob

func (b *ADLv1) DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error)

func (*ADLv1) DeleteBlobs

func (b *ADLv1) DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error)

func (*ADLv1) GetBlob

func (b *ADLv1) GetBlob(param *GetBlobInput) (*GetBlobOutput, error)

func (*ADLv1) HeadBlob

func (b *ADLv1) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error)

func (*ADLv1) Init

func (b *ADLv1) Init(key string) error

func (*ADLv1) ListBlobs

func (b *ADLv1) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error)

func (*ADLv1) MakeBucket

func (b *ADLv1) MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error)

func (*ADLv1) MultipartBlobAbort

func (b *ADLv1) MultipartBlobAbort(param *MultipartBlobCommitInput) (*MultipartBlobAbortOutput, error)

func (*ADLv1) MultipartBlobAdd

func (b *ADLv1) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error)

func (*ADLv1) MultipartBlobBegin

func (b *ADLv1) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error)

func (*ADLv1) MultipartBlobCommit

func (b *ADLv1) MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error)

func (*ADLv1) MultipartBlobCopy

func (b *ADLv1) MultipartBlobCopy(param *MultipartBlobCopyInput) (*MultipartBlobCopyOutput, error)

func (*ADLv1) MultipartExpire

func (b *ADLv1) MultipartExpire(param *MultipartExpireInput) (*MultipartExpireOutput, error)

func (*ADLv1) PatchBlob added in v0.39.0

func (s *ADLv1) PatchBlob(param *PatchBlobInput) (*PatchBlobOutput, error)

func (*ADLv1) PutBlob

func (b *ADLv1) PutBlob(param *PutBlobInput) (*PutBlobOutput, error)

func (*ADLv1) RemoveBucket

func (b *ADLv1) RemoveBucket(param *RemoveBucketInput) (*RemoveBucketOutput, error)

func (*ADLv1) RenameBlob

func (b *ADLv1) RenameBlob(param *RenameBlobInput) (*RenameBlobOutput, error)

type ADLv1Err

type ADLv1Err struct {
	RemoteException struct {
		Exception     string
		Message       string
		JavaClassName string
	}
	// contains filtered or unexported fields
}

func (ADLv1Err) Error

func (err ADLv1Err) Error() string

type ADLv1MultipartBlobCommitInput

type ADLv1MultipartBlobCommitInput struct {
	Size uint64
}

type ADLv2

type ADLv2 struct {
	// contains filtered or unexported fields
}

func NewADLv2

func NewADLv2(bucket string, flags *cfg.FlagStorage, config *cfg.ADLv2Config) (*ADLv2, error)

func (*ADLv2) Bucket

func (b *ADLv2) Bucket() string

func (*ADLv2) Capabilities

func (b *ADLv2) Capabilities() *Capabilities

func (*ADLv2) CopyBlob

func (b *ADLv2) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error)

func (*ADLv2) Delegate

func (b *ADLv2) Delegate() interface{}

func (*ADLv2) DeleteBlob

func (b *ADLv2) DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error)

func (*ADLv2) DeleteBlobs

func (b *ADLv2) DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error)

func (*ADLv2) GetBlob

func (b *ADLv2) GetBlob(param *GetBlobInput) (*GetBlobOutput, error)

func (*ADLv2) HeadBlob

func (b *ADLv2) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error)

func (*ADLv2) Init

func (b *ADLv2) Init(key string) (err error)

func (*ADLv2) ListBlobs

func (b *ADLv2) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error)

func (*ADLv2) MakeBucket

func (b *ADLv2) MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error)

func (*ADLv2) MultipartBlobAbort

func (b *ADLv2) MultipartBlobAbort(param *MultipartBlobCommitInput) (*MultipartBlobAbortOutput, error)

func (*ADLv2) MultipartBlobAdd

func (b *ADLv2) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error)

func (*ADLv2) MultipartBlobBegin

func (b *ADLv2) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error)

adlv2 doesn't have atomic multipart upload, instead we will hold a lease, replace the object, then release the lease

func (*ADLv2) MultipartBlobCommit

func (b *ADLv2) MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error)

func (*ADLv2) MultipartBlobCopy

func (b *ADLv2) MultipartBlobCopy(param *MultipartBlobCopyInput) (*MultipartBlobCopyOutput, error)

func (*ADLv2) MultipartExpire

func (b *ADLv2) MultipartExpire(param *MultipartExpireInput) (*MultipartExpireOutput, error)

func (*ADLv2) PatchBlob added in v0.39.0

func (s *ADLv2) PatchBlob(param *PatchBlobInput) (*PatchBlobOutput, error)

func (*ADLv2) PutBlob

func (b *ADLv2) PutBlob(param *PutBlobInput) (*PutBlobOutput, error)

func (*ADLv2) RemoveBucket

func (b *ADLv2) RemoveBucket(param *RemoveBucketInput) (*RemoveBucketOutput, error)

func (*ADLv2) RenameBlob

func (b *ADLv2) RenameBlob(param *RenameBlobInput) (*RenameBlobOutput, error)

type ADLv2MultipartBlobCommitInput

type ADLv2MultipartBlobCommitInput struct {
	Size           uint64
	ContentType    string
	RenewLeaseStop chan bool
}

type AZBlob

type AZBlob struct {
	// contains filtered or unexported fields
}

func NewAZBlob

func NewAZBlob(container string, config *cfg.AZBlobConfig) (*AZBlob, error)

func (*AZBlob) Bucket

func (b *AZBlob) Bucket() string

func (*AZBlob) Capabilities

func (b *AZBlob) Capabilities() *Capabilities

func (*AZBlob) CopyBlob

func (b *AZBlob) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error)

func (*AZBlob) Delegate

func (b *AZBlob) Delegate() interface{}

func (*AZBlob) DeleteBlob

func (b *AZBlob) DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error)

func (*AZBlob) DeleteBlobs

func (b *AZBlob) DeleteBlobs(param *DeleteBlobsInput) (ret *DeleteBlobsOutput, deleteError error)

func (*AZBlob) GetBlob

func (b *AZBlob) GetBlob(param *GetBlobInput) (*GetBlobOutput, error)

func (*AZBlob) HeadBlob

func (b *AZBlob) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error)

func (*AZBlob) Init

func (b *AZBlob) Init(key string) error

func (*AZBlob) ListBlobs

func (b *AZBlob) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error)

func (*AZBlob) MakeBucket

func (b *AZBlob) MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error)

func (*AZBlob) MultipartBlobAbort

func (b *AZBlob) MultipartBlobAbort(param *MultipartBlobCommitInput) (*MultipartBlobAbortOutput, error)

func (*AZBlob) MultipartBlobAdd

func (b *AZBlob) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error)

func (*AZBlob) MultipartBlobBegin

func (b *AZBlob) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error)

func (*AZBlob) MultipartBlobCommit

func (b *AZBlob) MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error)

func (*AZBlob) MultipartBlobCopy

func (b *AZBlob) MultipartBlobCopy(param *MultipartBlobCopyInput) (*MultipartBlobCopyOutput, error)

func (*AZBlob) MultipartExpire

func (b *AZBlob) MultipartExpire(param *MultipartExpireInput) (*MultipartExpireOutput, error)

func (*AZBlob) PatchBlob added in v0.39.0

func (s *AZBlob) PatchBlob(param *PatchBlobInput) (*PatchBlobOutput, error)

func (*AZBlob) PutBlob

func (b *AZBlob) PutBlob(param *PutBlobInput) (*PutBlobOutput, error)

func (*AZBlob) RemoveBucket

func (b *AZBlob) RemoveBucket(param *RemoveBucketInput) (*RemoveBucketOutput, error)

func (*AZBlob) RenameBlob

func (b *AZBlob) RenameBlob(param *RenameBlobInput) (*RenameBlobOutput, error)

type BlobItemOutput

type BlobItemOutput struct {
	Key          *string
	ETag         *string
	LastModified *time.Time
	Size         uint64
	StorageClass *string
	// may be nil in list responses for backends that don't return metadata in listings
	Metadata map[string]*string
}

func (BlobItemOutput) String

func (b BlobItemOutput) String() string

type BlobPrefixOutput

type BlobPrefixOutput struct {
	Prefix *string
}

func (BlobPrefixOutput) String

func (b BlobPrefixOutput) String() string

type BucketSpec

type BucketSpec struct {
	Scheme string
	Bucket string
	Prefix string
}

func ParseBucketSpec

func ParseBucketSpec(bucket string) (spec BucketSpec, err error)

type BufferList added in v0.40.0

type BufferList struct {
	// contains filtered or unexported fields
}

func (*BufferList) Add added in v0.40.0

func (l *BufferList) Add(offset uint64, data []byte, state BufferState, copyData bool) (allocated int64)

func (*BufferList) AddLoading added in v0.40.0

func (l *BufferList) AddLoading(offset, size uint64)

func (*BufferList) AddLoadingFromDisk added in v0.40.0

func (l *BufferList) AddLoadingFromDisk(offset, size uint64) (readRanges []Range)

func (*BufferList) AnyDirty added in v0.40.0

func (l *BufferList) AnyDirty() (dirty bool)

func (*BufferList) AnyFlushed added in v0.40.0

func (l *BufferList) AnyFlushed(offset, size uint64) (flushed bool)

func (*BufferList) Ascend added in v0.40.0

func (l *BufferList) Ascend(offset uint64, iter func(end uint64, b *FileBuffer) (cont bool, changed bool))

func (*BufferList) Count added in v0.40.4

func (l *BufferList) Count() int

func (*BufferList) DebugCheckHoles added in v0.40.0

func (l *BufferList) DebugCheckHoles(s string)

func (*BufferList) Dump added in v0.40.0

func (l *BufferList) Dump(offset, size uint64) string

Left here for the ease of debugging

func (*BufferList) EvictFromMemory added in v0.40.0

func (l *BufferList) EvictFromMemory(buf *FileBuffer) (allocated int64, deleted bool)

func (*BufferList) Get added in v0.40.0

func (l *BufferList) Get(end uint64) *FileBuffer

func (*BufferList) GetData added in v0.40.0

func (l *BufferList) GetData(offset, size uint64, returnIds bool) (data [][]byte, ids map[uint64]bool, err error)

func (*BufferList) GetHoles added in v0.40.0

func (l *BufferList) GetHoles(offset, size uint64) (holes []Range, loading bool, flushCleared bool)

func (*BufferList) IterateDirtyParts added in v0.40.0

func (l *BufferList) IterateDirtyParts(cb func(partNum uint64) bool)

func (*BufferList) RemoveLoading added in v0.40.0

func (l *BufferList) RemoveLoading(offset, size uint64)

func (*BufferList) RemoveRange added in v0.40.0

func (l *BufferList) RemoveRange(removeOffset, removeSize uint64, filter func(b *FileBuffer) bool) (allocated int64)

Remove buffers in range (offset..size)

func (*BufferList) ReviveFromDisk added in v0.40.0

func (l *BufferList) ReviveFromDisk(offset uint64, data []byte)

func (*BufferList) Select added in v0.40.0

func (l *BufferList) Select(start, end uint64, cb func(buf *FileBuffer) (good bool)) (bufs []*FileBuffer)

func (*BufferList) SetFlushedClean added in v0.40.0

func (l *BufferList) SetFlushedClean()

func (*BufferList) SetState added in v0.40.0

func (l *BufferList) SetState(offset, size uint64, ids map[uint64]bool, state BufferState)

func (*BufferList) SplitAt added in v0.40.0

func (l *BufferList) SplitAt(offset uint64)

func (*BufferList) ZeroRange added in v0.40.0

func (l *BufferList) ZeroRange(offset, size uint64) (zeroed bool, allocated int64)

type BufferListHelpers added in v0.40.0

type BufferListHelpers interface {
	PartNum(uint64) uint64
	QueueCleanBuffer(*FileBuffer)
	UnqueueCleanBuffer(*FileBuffer)
}

type BufferOrZero

type BufferOrZero struct {
	// contains filtered or unexported fields
}

type BufferPointer

type BufferPointer struct {
	// contains filtered or unexported fields
}

Several FileBuffers may be slices of the same array, but we want to track memory usage, so we have to refcount them... O_o

type BufferPool

type BufferPool struct {
	FreeSomeCleanBuffers func(size int64) (int64, bool)
	// contains filtered or unexported fields
}

BufferPool tracks memory used by cache buffers

func NewBufferPool

func NewBufferPool(limit int64, gcInterval uint64) *BufferPool

func (*BufferPool) Use

func (pool *BufferPool) Use(size int64, ignoreMemoryLimit bool) (err error)

func (*BufferPool) UseUnlocked

func (pool *BufferPool) UseUnlocked(size int64, ignoreMemoryLimit bool) error

type BufferQueue added in v0.40.0

type BufferQueue struct {
	// contains filtered or unexported fields
}

func (*BufferQueue) Add added in v0.40.0

func (l *BufferQueue) Add(inode *Inode, b *FileBuffer)

func (*BufferQueue) Delete added in v0.40.0

func (l *BufferQueue) Delete(b *FileBuffer)

func (*BufferQueue) NextClean added in v0.40.0

func (l *BufferQueue) NextClean(minQueueId uint64) (inode *Inode, end, nextQueueId uint64)

type BufferState added in v0.40.0

type BufferState int16
const (
	// Buffer is clean
	BUF_CLEAN BufferState = 1
	// Buffer is modified locally
	BUF_DIRTY BufferState = 2
	// Buffer is flushed to the server as a full part, but multipart upload is not finalized yet
	BUF_FLUSHED_FULL BufferState = 3
	// Buffer is flushed to the server as an undersized part
	// (and multipart upload is not finalized yet)
	BUF_FLUSHED_CUT BufferState = 4
	// Buffer is flushed to the server and then removed from memory
	// (which is only possible for BUF_FLUSHED_FULL buffers)
	// (and multipart upload is not finalized yet)
	BUF_FL_CLEARED BufferState = 5
)

Yes I know this is against Go style. But it's easier to grep and distinguish visually so fuck off. :-)

type Capabilities

type Capabilities struct {
	MaxMultipartSize uint64
	// indicates that the blob store has native support for directories
	DirBlob bool
	Name    string
}

type ClusterFs added in v0.36.0

type ClusterFs struct {
	Flags  *cfg.FlagStorage
	Conns  *ConnPool
	Goofys *Goofys
	// contains filtered or unexported fields
}

func (*ClusterFs) Join added in v0.36.0

func (fs *ClusterFs) Join(ctx context.Context) error

func (*ClusterFs) StatPrinter added in v0.36.0

func (fs *ClusterFs) StatPrinter()

func (*ClusterFs) Unmount added in v0.36.0

func (fs *ClusterFs) Unmount() error

type ClusterFsFuse added in v0.36.0

type ClusterFsFuse struct {
	fuseutil.NotImplementedFileSystem
	*ClusterFs
}

func (*ClusterFsFuse) CreateFile added in v0.36.0

func (fs *ClusterFsFuse) CreateFile(ctx context.Context, op *fuseops.CreateFileOp) (err error)
func (fs *ClusterFsFuse) CreateSymlink(ctx context.Context, op *fuseops.CreateSymlinkOp) (err error)

func (*ClusterFsFuse) ForgetInode added in v0.36.0

func (fs *ClusterFsFuse) ForgetInode(ctx context.Context, op *fuseops.ForgetInodeOp) (err error)

func (*ClusterFsFuse) GetInodeAttributes added in v0.36.0

func (fs *ClusterFsFuse) GetInodeAttributes(ctx context.Context, op *fuseops.GetInodeAttributesOp) (err error)

func (*ClusterFsFuse) LookUpInode added in v0.36.0

func (fs *ClusterFsFuse) LookUpInode(ctx context.Context, op *fuseops.LookUpInodeOp) (err error)

func (*ClusterFsFuse) MkDir added in v0.36.0

func (fs *ClusterFsFuse) MkDir(ctx context.Context, op *fuseops.MkDirOp) (err error)

func (*ClusterFsFuse) OpenDir added in v0.36.0

func (fs *ClusterFsFuse) OpenDir(ctx context.Context, op *fuseops.OpenDirOp) (err error)

func (*ClusterFsFuse) OpenFile added in v0.36.0

func (fs *ClusterFsFuse) OpenFile(ctx context.Context, op *fuseops.OpenFileOp) (err error)

func (*ClusterFsFuse) ReadDir added in v0.36.0

func (fs *ClusterFsFuse) ReadDir(ctx context.Context, op *fuseops.ReadDirOp) (err error)

func (*ClusterFsFuse) ReadFile added in v0.36.0

func (fs *ClusterFsFuse) ReadFile(ctx context.Context, op *fuseops.ReadFileOp) (err error)
func (fs *ClusterFsFuse) ReadSymlink(ctx context.Context, op *fuseops.ReadSymlinkOp) (err error)

func (*ClusterFsFuse) ReleaseDirHandle added in v0.36.0

func (fs *ClusterFsFuse) ReleaseDirHandle(ctx context.Context, op *fuseops.ReleaseDirHandleOp) (err error)

func (*ClusterFsFuse) ReleaseFileHandle added in v0.36.0

func (fs *ClusterFsFuse) ReleaseFileHandle(ctx context.Context, op *fuseops.ReleaseFileHandleOp) (err error)

func (*ClusterFsFuse) RmDir added in v0.36.0

func (fs *ClusterFsFuse) RmDir(ctx context.Context, op *fuseops.RmDirOp) (err error)

func (*ClusterFsFuse) SetInodeAttributes added in v0.36.0

func (fs *ClusterFsFuse) SetInodeAttributes(ctx context.Context, op *fuseops.SetInodeAttributesOp) (err error)

func (*ClusterFsFuse) StatFS added in v0.36.0

func (fs *ClusterFsFuse) StatFS(ctx context.Context, op *fuseops.StatFSOp) error
func (fs *ClusterFsFuse) Unlink(ctx context.Context, op *fuseops.UnlinkOp) (err error)

func (*ClusterFsFuse) WriteFile added in v0.36.0

func (fs *ClusterFsFuse) WriteFile(ctx context.Context, op *fuseops.WriteFileOp) (err error)

type ClusterFsGrpc added in v0.36.0

type ClusterFsGrpc struct {
	pb.UnimplementedFsGrpcServer
	*ClusterFs
}

func (*ClusterFsGrpc) CreateFile added in v0.36.0

func (*ClusterFsGrpc) ForgetInode added in v0.36.0

func (*ClusterFsGrpc) ForgetInode2 added in v0.36.0

func (*ClusterFsGrpc) GetInodeAttributes added in v0.36.0

func (*ClusterFsGrpc) LookUpInode added in v0.36.0

func (*ClusterFsGrpc) LookUpInode2 added in v0.36.0

func (*ClusterFsGrpc) MkDir added in v0.36.0

func (fs *ClusterFsGrpc) MkDir(ctx context.Context, req *pb.MkDirRequest) (*pb.MkDirResponse, error)

func (*ClusterFsGrpc) OpenDir added in v0.36.0

func (*ClusterFsGrpc) OpenFile added in v0.36.0

func (*ClusterFsGrpc) ReadDir added in v0.36.0

func (*ClusterFsGrpc) ReadFile added in v0.36.0

func (*ClusterFsGrpc) ReleaseDirHandle added in v0.36.0

func (*ClusterFsGrpc) ReleaseFileHandle added in v0.36.0

func (*ClusterFsGrpc) RmDir added in v0.36.0

func (fs *ClusterFsGrpc) RmDir(ctx context.Context, req *pb.RmDirRequest) (*pb.RmDirResponse, error)

func (*ClusterFsGrpc) SetInodeAttributes added in v0.36.0

func (*ClusterFsGrpc) TryStealInodeOwnership added in v0.36.0

func (fs *ClusterFsGrpc) Unlink(ctx context.Context, req *pb.UnlinkRequest) (*pb.UnlinkResponse, error)

func (*ClusterFsGrpc) WriteFile added in v0.36.0

type ConnPool added in v0.36.0

type ConnPool struct {
	// contains filtered or unexported fields
}

func NewConnPool added in v0.36.0

func NewConnPool(flags *cfg.FlagStorage) *ConnPool

func (*ConnPool) Broad added in v0.36.0

func (conns *ConnPool) Broad(
	makeRequst Request,
) (errs map[NodeId]error)

func (*ConnPool) BroadConfigurable added in v0.36.0

func (conns *ConnPool) BroadConfigurable(
	makeRequst Request,
	unmountOnError bool,
) (errs map[NodeId]error)

func (*ConnPool) Unary added in v0.36.0

func (conns *ConnPool) Unary(
	nodeId NodeId,
	makeRequst Request,
) (err error)

func (*ConnPool) UnaryConfiguarble added in v0.36.0

func (conns *ConnPool) UnaryConfiguarble(
	nodeId NodeId,
	makeRequst Request,
	unmountOnError bool,
) (err error)

type CopyBlobInput

type CopyBlobInput struct {
	Source      string
	Destination string

	Size         *uint64
	ETag         *string            // if non-nil, do conditional copy
	Metadata     map[string]*string // if nil, copy from Source
	StorageClass *string            // if nil, copy from Source
}

type CopyBlobOutput

type CopyBlobOutput struct {
	RequestId string
}

type Delegator

type Delegator interface {
	Delegate() interface{}
}

type DeleteBlobInput

type DeleteBlobInput struct {
	Key string
}

type DeleteBlobOutput

type DeleteBlobOutput struct {
	RequestId string
}

type DeleteBlobsInput

type DeleteBlobsInput struct {
	Items []string
}

type DeleteBlobsOutput

type DeleteBlobsOutput struct {
	RequestId string
}

type DirHandle

type DirHandle struct {
	// contains filtered or unexported fields
}

func NewDirHandle

func NewDirHandle(inode *Inode) (dh *DirHandle)

func (*DirHandle) CloseDir

func (dh *DirHandle) CloseDir() error

func (*DirHandle) Next added in v0.37.0

func (dh *DirHandle) Next(name string)

LOCKS_REQUIRED(dh.mu)

func (*DirHandle) ReadDir

func (dh *DirHandle) ReadDir() (inode *Inode, err error)

LOCKS_REQUIRED(dh.mu) LOCKS_EXCLUDED(dh.inode.mu) LOCKS_EXCLUDED(dh.inode.fs)

func (*DirHandle) Seek added in v0.36.0

func (dh *DirHandle) Seek(newOffset fuseops.DirOffset)

LOCKS_REQUIRED(dh.mu)

type DirInodeData

type DirInodeData struct {
	DirTime     time.Time
	ImplicitDir bool

	ModifiedChildren int64

	Children        []*Inode
	DeletedChildren map[string]*Inode
	Gaps            []*SlurpGap
	// contains filtered or unexported fields
}

type FDQueue added in v0.40.0

type FDQueue struct {
	// contains filtered or unexported fields
}

func NewFDQueue added in v0.40.0

func NewFDQueue(maxCount int) *FDQueue

func (*FDQueue) CloseExtra added in v0.40.0

func (l *FDQueue) CloseExtra()

Close unneeded cache FDs

func (*FDQueue) DeleteFD added in v0.40.0

func (l *FDQueue) DeleteFD(inode *Inode)

func (*FDQueue) InsertFD added in v0.40.0

func (l *FDQueue) InsertFD(inode *Inode)

func (*FDQueue) NextFD added in v0.40.0

func (l *FDQueue) NextFD(minQueueID uint64) (inode *Inode, nextQueueId uint64)

func (*FDQueue) UseFD added in v0.40.0

func (l *FDQueue) UseFD(inode *Inode)

type FileBuffer

type FileBuffer struct {
	// contains filtered or unexported fields
}

func (*FileBuffer) Append added in v0.40.0

func (buf *FileBuffer) Append(data []byte) int64

type FileHandle

type FileHandle struct {
	// contains filtered or unexported fields
}

func NewFileHandle

func NewFileHandle(inode *Inode) *FileHandle

NewFileHandle returns a new file handle for the given `inode`

func (*FileHandle) ReadFile

func (fh *FileHandle) ReadFile(sOffset int64, sLen int64) (data [][]byte, bytesRead int, err error)

func (*FileHandle) Release

func (fh *FileHandle) Release()

func (*FileHandle) WriteFile

func (fh *FileHandle) WriteFile(offset int64, data []byte, copyData bool) (err error)

type FuseMfsWrapper added in v0.36.0

type FuseMfsWrapper struct {
	*fuse.MountedFileSystem
	// contains filtered or unexported fields
}

func (*FuseMfsWrapper) Unmount added in v0.36.0

func (m *FuseMfsWrapper) Unmount() error

type GCPCredResponse added in v0.34.2

type GCPCredResponse struct {
	AccessToken string `json:"access_token"`
	TokenType   string `json:"token_type"`
	ExpiresIn   int    `json:"expires_in"`
}

type GCS3

type GCS3 struct {
	*S3Backend
	// contains filtered or unexported fields
}

GCS variant of S3

func NewGCS3

func NewGCS3(bucket string, flags *cfg.FlagStorage, config *cfg.S3Config) (*GCS3, error)

func (*GCS3) Delegate

func (s *GCS3) Delegate() interface{}

func (*GCS3) DeleteBlobs

func (s *GCS3) DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error)

func (*GCS3) ListBlobs

func (s *GCS3) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error)

func (*GCS3) MultipartBlobCopy

func (s *GCS3) MultipartBlobCopy(param *MultipartBlobCopyInput) (*MultipartBlobCopyOutput, error)

FIXME GCS doesn't have UploadPartCopy, so optimized modification flushing doesn't work You can either reupload the whole object or use some other way of making multipart objects For example, Composite Objects are even better than multipart uploads but intermediate objects should be filtered out from List responses so they don't appear as separate files then

func (*GCS3) PatchBlob added in v0.39.0

func (s *GCS3) PatchBlob(param *PatchBlobInput) (*PatchBlobOutput, error)

type GetBlobInput

type GetBlobInput struct {
	Key     string
	Start   uint64
	Count   uint64
	IfMatch *string
}

type GetBlobOutput

type GetBlobOutput struct {
	HeadBlobOutput

	Body io.ReadCloser

	RequestId string
}

type Goofys

type Goofys struct {
	NotifyCallback func(notifications []interface{})
	// contains filtered or unexported fields
}

func NewClusterGoofys added in v0.36.0

func NewClusterGoofys(ctx context.Context, bucket string, flags *cfg.FlagStorage, conns *ConnPool) (*Goofys, error)

func NewGoofys

func NewGoofys(ctx context.Context, bucketName string, flags *cfg.FlagStorage) (*Goofys, error)

func (*Goofys) AddDirHandle added in v0.37.0

func (fs *Goofys) AddDirHandle(dh *DirHandle) fuseops.HandleID

func (*Goofys) AddFileHandle added in v0.37.0

func (fs *Goofys) AddFileHandle(fh *FileHandle) fuseops.HandleID

func (*Goofys) EvictEntry added in v0.37.0

func (fs *Goofys) EvictEntry(id fuseops.InodeID) bool

func (*Goofys) FDCloser added in v0.26.0

func (fs *Goofys) FDCloser()

Close unneeded cache FDs

func (*Goofys) Flusher

func (fs *Goofys) Flusher()

Flusher goroutine. Overall algorithm:

  1. File opened => reads and writes just populate cache
  2. File closed => flush it Created or fully overwritten => Less than 5 MB => upload in a single part More than 5 MB => upload using multipart Updated => CURRENTLY: Less than 5 MB => upload in a single part More than 5 MB => update using multipart copy Also we can't update less than 5 MB because it's the minimal part size
  3. Fsync triggered => intermediate full flush (same algorithm)
  4. Dirty memory limit reached => without on-disk cache we have to flush the whole object. With on-disk cache we can unload some dirty buffers to disk.

func (*Goofys) FreeSomeCleanBuffers

func (fs *Goofys) FreeSomeCleanBuffers(origSize int64) (int64, bool)

Try to reclaim some clean buffers

func (*Goofys) LookupParent added in v0.36.0

func (fs *Goofys) LookupParent(path string) (parent *Inode, child string, err error)

func (*Goofys) LookupPath added in v0.36.0

func (fs *Goofys) LookupPath(path string) (inode *Inode, err error)

func (*Goofys) MetaEvictor added in v0.37.0

func (fs *Goofys) MetaEvictor()

func (*Goofys) Mount

func (fs *Goofys) Mount(mount *Mount)

func (*Goofys) MountAll

func (fs *Goofys) MountAll(mounts []*Mount)

func (*Goofys) RefreshInodeCache added in v0.36.0

func (fs *Goofys) RefreshInodeCache(inode *Inode) error

func (*Goofys) ScheduleRetryFlush added in v0.28.0

func (fs *Goofys) ScheduleRetryFlush()

func (*Goofys) Shutdown added in v0.38.4

func (fs *Goofys) Shutdown()

func (*Goofys) SigUsr1

func (fs *Goofys) SigUsr1()

func (*Goofys) StatPrinter added in v0.34.2

func (fs *Goofys) StatPrinter()

func (*Goofys) SyncFS

func (fs *Goofys) SyncFS(parent *Inode) (err error)

func (*Goofys) Unmount

func (fs *Goofys) Unmount(mountPoint string)

func (*Goofys) WakeupFlusher

func (fs *Goofys) WakeupFlusher()

func (*Goofys) WakeupFlusherAndWait added in v0.35.1

func (fs *Goofys) WakeupFlusherAndWait(wait bool)

type GoofysFuse added in v0.36.0

type GoofysFuse struct {
	fuseutil.NotImplementedFileSystem
	*Goofys
	// contains filtered or unexported fields
}

func NewGoofysFuse added in v0.36.0

func NewGoofysFuse(fs *Goofys) *GoofysFuse

func (*GoofysFuse) CreateFile added in v0.36.0

func (fs *GoofysFuse) CreateFile(
	ctx context.Context,
	op *fuseops.CreateFileOp) (err error)
func (fs *GoofysFuse) CreateSymlink(ctx context.Context,
	op *fuseops.CreateSymlinkOp) (err error)

func (*GoofysFuse) Fallocate added in v0.36.0

func (fs *GoofysFuse) Fallocate(
	ctx context.Context,
	op *fuseops.FallocateOp) (err error)

func (*GoofysFuse) FlushFile added in v0.36.0

func (fs *GoofysFuse) FlushFile(
	ctx context.Context,
	op *fuseops.FlushFileOp) (err error)

func (*GoofysFuse) ForgetInode added in v0.36.0

func (fs *GoofysFuse) ForgetInode(
	ctx context.Context,
	op *fuseops.ForgetInodeOp) (err error)

func (*GoofysFuse) GetInodeAttributes added in v0.36.0

func (fs *GoofysFuse) GetInodeAttributes(
	ctx context.Context,
	op *fuseops.GetInodeAttributesOp) (err error)

func (*GoofysFuse) GetXattr added in v0.36.0

func (fs *GoofysFuse) GetXattr(ctx context.Context,
	op *fuseops.GetXattrOp) (err error)

func (*GoofysFuse) ListXattr added in v0.36.0

func (fs *GoofysFuse) ListXattr(ctx context.Context,
	op *fuseops.ListXattrOp) (err error)

func (*GoofysFuse) LookUpInode added in v0.36.0

func (fs *GoofysFuse) LookUpInode(
	ctx context.Context,
	op *fuseops.LookUpInodeOp) (err error)

func (*GoofysFuse) MkDir added in v0.36.0

func (fs *GoofysFuse) MkDir(
	ctx context.Context,
	op *fuseops.MkDirOp) (err error)

func (*GoofysFuse) MkNode added in v0.36.0

func (fs *GoofysFuse) MkNode(
	ctx context.Context,
	op *fuseops.MkNodeOp) (err error)

MkNode is required for NFS even with regular files because kernel nfsd uses vfs_create() -> fuse_create() -> fuse_mknod() and then separate fuse_open() for file creation instead of fuse_create_open()

func (*GoofysFuse) OpenDir added in v0.36.0

func (fs *GoofysFuse) OpenDir(
	ctx context.Context,
	op *fuseops.OpenDirOp) (err error)

func (*GoofysFuse) OpenFile added in v0.36.0

func (fs *GoofysFuse) OpenFile(
	ctx context.Context,
	op *fuseops.OpenFileOp) (err error)

func (*GoofysFuse) ReadDir added in v0.36.0

func (fs *GoofysFuse) ReadDir(
	ctx context.Context,
	op *fuseops.ReadDirOp) (err error)

func (*GoofysFuse) ReadFile added in v0.36.0

func (fs *GoofysFuse) ReadFile(
	ctx context.Context,
	op *fuseops.ReadFileOp) (err error)
func (fs *GoofysFuse) ReadSymlink(ctx context.Context,
	op *fuseops.ReadSymlinkOp) (err error)

func (*GoofysFuse) ReleaseDirHandle added in v0.36.0

func (fs *GoofysFuse) ReleaseDirHandle(
	ctx context.Context,
	op *fuseops.ReleaseDirHandleOp) (err error)

func (*GoofysFuse) ReleaseFileHandle added in v0.36.0

func (fs *GoofysFuse) ReleaseFileHandle(
	ctx context.Context,
	op *fuseops.ReleaseFileHandleOp) (err error)

func (*GoofysFuse) RemoveXattr added in v0.36.0

func (fs *GoofysFuse) RemoveXattr(ctx context.Context,
	op *fuseops.RemoveXattrOp) (err error)

func (*GoofysFuse) Rename added in v0.36.0

func (fs *GoofysFuse) Rename(
	ctx context.Context,
	op *fuseops.RenameOp) (err error)

rename("from", "to") causes the kernel to send lookup of "from" and "to" prior to sending rename to us

func (*GoofysFuse) RmDir added in v0.36.0

func (fs *GoofysFuse) RmDir(
	ctx context.Context,
	op *fuseops.RmDirOp) (err error)

func (*GoofysFuse) SetConnection added in v0.36.0

func (fs *GoofysFuse) SetConnection(conn *fuse.Connection)

func (*GoofysFuse) SetInodeAttributes added in v0.36.0

func (fs *GoofysFuse) SetInodeAttributes(
	ctx context.Context,
	op *fuseops.SetInodeAttributesOp) (err error)

func (*GoofysFuse) SetXattr added in v0.36.0

func (fs *GoofysFuse) SetXattr(ctx context.Context,
	op *fuseops.SetXattrOp) (err error)

func (*GoofysFuse) StatFS added in v0.36.0

func (fs *GoofysFuse) StatFS(
	ctx context.Context,
	op *fuseops.StatFSOp) (err error)

func (*GoofysFuse) SyncFile added in v0.36.0

func (fs *GoofysFuse) SyncFile(
	ctx context.Context,
	op *fuseops.SyncFileOp) (err error)
func (fs *GoofysFuse) Unlink(
	ctx context.Context,
	op *fuseops.UnlinkOp) (err error)

func (*GoofysFuse) WriteFile added in v0.36.0

func (fs *GoofysFuse) WriteFile(
	ctx context.Context,
	op *fuseops.WriteFileOp) (err error)

type GrpcServer added in v0.36.0

type GrpcServer struct {
	*grpc.Server
	// contains filtered or unexported fields
}

func NewGrpcServer added in v0.36.0

func NewGrpcServer(flags *cfg.FlagStorage) *GrpcServer

func (*GrpcServer) Start added in v0.36.0

func (srv *GrpcServer) Start() error

type HeadBlobInput

type HeadBlobInput struct {
	Key string
}

type HeadBlobOutput

type HeadBlobOutput struct {
	BlobItemOutput

	ContentType *string
	IsDirBlob   bool

	RequestId string
}

func RetryHeadBlob added in v0.40.0

func RetryHeadBlob(flags *cfg.FlagStorage, cloud StorageBackend, req *HeadBlobInput) (resp *HeadBlobOutput, err error)

type IMDSv1Response added in v0.34.2

type IMDSv1Response struct {
	Code       string
	Token      string
	Expiration time.Time
}

type Inode

type Inode struct {
	Id   fuseops.InodeID
	Name string

	Attributes InodeAttributes
	// It is generally safe to read `AttrTime` without locking because if some other
	// operation is modifying `AttrTime`, in most cases the reader is okay with working with
	// stale data. But Time is a struct and modifying it is not atomic. However
	// in practice (until the year 2157) we should be okay because
	// - Almost all uses of AttrTime will be about comparisons (AttrTime < x, AttrTime > x)
	// - Time object will have Time::monotonic bit set (until the year 2157) => the time
	//   comparision just compares Time::ext field
	// Ref: https://github.com/golang/go/blob/e42ae65a8507/src/time/time.go#L12:L56
	AttrTime   time.Time
	ExpireTime time.Time

	// We are not very consistent about enforcing locks for `Parent` because, the
	// parent field very very rarely changes and it is generally fine to operate on
	// stale parent information
	Parent *Inode

	// cached/buffered data
	CacheState int32

	DiskFDQueueID uint64
	DiskCacheFD   *os.File
	OnDisk        bool

	IsFlushing int
	// contains filtered or unexported fields
}

func NewInode

func NewInode(fs *Goofys, parent *Inode, name string) (inode *Inode)

func (*Inode) ChangeOwnerLock added in v0.36.0

func (inode *Inode) ChangeOwnerLock()

func (*Inode) ChangeOwnerUnlock added in v0.36.0

func (inode *Inode) ChangeOwnerUnlock()

func (*Inode) CheckLoadRange

func (inode *Inode) CheckLoadRange(offset, size, readAheadSize uint64, ignoreMemoryLimit bool) (bool, error)

func (*Inode) Create

func (parent *Inode) Create(name string) (*Inode, *FileHandle, error)

func (*Inode) CreateOrOpen added in v0.37.1

func (parent *Inode) CreateOrOpen(name string, open bool) (inode *Inode, fh *FileHandle, err error)
func (parent *Inode) CreateSymlink(
	name string, target string) (inode *Inode, err error)

func (*Inode) DeRef

func (inode *Inode) DeRef(n int64) (stale bool)

LOCKS_REQUIRED(inode.mu) LOCKS_EXCLUDED(fs.mu)

func (*Inode) DowngradeToKeepOwnerLock added in v0.36.0

func (inode *Inode) DowngradeToKeepOwnerLock()

Only for inode in StateLock

func (*Inode) FullName

func (inode *Inode) FullName() string

func (*Inode) GetAttributes

func (inode *Inode) GetAttributes() *fuseops.InodeAttributes

LOCKS_EXCLUDED(inode.mu)

func (*Inode) GetXattr

func (inode *Inode) GetXattr(name string) ([]byte, error)

func (*Inode) InflateAttributes

func (inode *Inode) InflateAttributes() (attr fuseops.InodeAttributes)

func (*Inode) IsRangeLocked

func (inode *Inode) IsRangeLocked(offset uint64, size uint64, onlyFlushing bool) bool

func (*Inode) KeepOwnerLock added in v0.36.0

func (inode *Inode) KeepOwnerLock()

func (*Inode) KeepOwnerUnlock added in v0.36.0

func (inode *Inode) KeepOwnerUnlock()

func (*Inode) ListXattr

func (inode *Inode) ListXattr() ([]string, error)

func (*Inode) LoadRange

func (inode *Inode) LoadRange(offset, size uint64, readAheadSize uint64, ignoreMemoryLimit bool) (miss bool, err error)

Load some inode data into memory Must be called with inode.mu taken Loaded range should be guarded against eviction by adding it into inode.readRanges

func (*Inode) LockRange

func (inode *Inode) LockRange(offset uint64, size uint64, flushing bool)

LockRange/UnlockRange could be moved into buffer_list.go, but they still have to be stored separately from buffers and can't be a refcount - otherwise an overwrite would reset the reference count and break locking

func (*Inode) LookUp

func (parent *Inode) LookUp(name string, doSlurp bool) (*Inode, error)

func (*Inode) LookUpCached added in v0.36.0

func (parent *Inode) LookUpCached(name string) (inode *Inode, err error)

func (*Inode) LookUpInodeMaybeDir

func (parent *Inode) LookUpInodeMaybeDir(name string) (*BlobItemOutput, error)

func (*Inode) MkDir

func (parent *Inode) MkDir(
	name string) (inode *Inode, err error)

func (*Inode) OpenCacheFD added in v0.26.0

func (inode *Inode) OpenCacheFD() error

func (*Inode) OpenDir

func (inode *Inode) OpenDir() (dh *DirHandle)

func (*Inode) OpenFile

func (inode *Inode) OpenFile() (fh *FileHandle, err error)

func (*Inode) PartNum added in v0.40.0

func (inode *Inode) PartNum(offset uint64) uint64

For BufferListHelpers

func (*Inode) QueueCleanBuffer added in v0.40.0

func (inode *Inode) QueueCleanBuffer(buf *FileBuffer)

For BufferListHelpers

func (inode *Inode) ReadSymlink() (target string, err error)

func (*Inode) Ref

func (inode *Inode) Ref()

func (*Inode) RemoveXattr

func (inode *Inode) RemoveXattr(name string) error

func (*Inode) Rename

func (parent *Inode) Rename(from string, newParent *Inode, to string) (err error)

semantic of rename: rename("any", "not_exists") = ok rename("file1", "file2") = ok rename("empty_dir1", "empty_dir2") = ok rename("nonempty_dir1", "empty_dir2") = ok rename("nonempty_dir1", "nonempty_dir2") = ENOTEMPTY rename("file", "dir") = EISDIR rename("dir", "file") = ENOTDIR LOCKS_EXCLUDED(parent.mu) LOCKS_EXCLUDED(newParent.mu)

func (*Inode) ResetForUnmount

func (inode *Inode) ResetForUnmount()

ResetForUnmount resets the Inode as part of unmounting a storage backend mounted at the given inode. ACQUIRES_LOCK(inode.mu)

func (*Inode) ResizeUnlocked

func (inode *Inode) ResizeUnlocked(newSize uint64, finalizeFlushed bool)

func (*Inode) RmDir

func (parent *Inode) RmDir(name string) (err error)

func (*Inode) SendDelete

func (inode *Inode) SendDelete()

func (*Inode) SendMkDir

func (dir *Inode) SendMkDir()

func (*Inode) SetAttrTime added in v0.37.0

func (inode *Inode) SetAttrTime(tm time.Time)

LOCKS_REQUIRED(inode.mu) LOCKS_EXCLUDED(inode.fs.mu)

func (*Inode) SetAttributes added in v0.36.0

func (inode *Inode) SetAttributes(size *uint64, mode *os.FileMode,
	mtime *time.Time, uid *uint32, gid *uint32) (err error)

func (*Inode) SetCacheState

func (inode *Inode) SetCacheState(state int32)

LOCKS_REQUIRED(inode.mu)

func (*Inode) SetExpireLocked added in v0.37.3

func (inode *Inode) SetExpireLocked(tm time.Time)

LOCKS_EXCLUDED(inode.mu) LOCKS_EXCLUDED(inode.fs.mu)

func (*Inode) SetExpireTime added in v0.37.3

func (inode *Inode) SetExpireTime(tm time.Time)

LOCKS_REQUIRED(inode.mu) LOCKS_EXCLUDED(inode.fs.mu)

func (*Inode) SetFromBlobItem

func (inode *Inode) SetFromBlobItem(item *BlobItemOutput)

LOCKS_EXCLUDED(inode.mu)

func (*Inode) SetXattr

func (inode *Inode) SetXattr(name string, value []byte, flags uint32) error

func (*Inode) StateLock added in v0.36.0

func (inode *Inode) StateLock()

func (*Inode) StateUnlock added in v0.36.0

func (inode *Inode) StateUnlock()

func (*Inode) SyncFile

func (inode *Inode) SyncFile() (err error)

func (*Inode) ToDir

func (inode *Inode) ToDir()

func (*Inode) TryFlush

func (inode *Inode) TryFlush(priority int) bool
func (parent *Inode) Unlink(name string) (err error)

func (*Inode) UnlockRange

func (inode *Inode) UnlockRange(offset uint64, size uint64, flushing bool)

func (*Inode) UnqueueCleanBuffer added in v0.40.0

func (inode *Inode) UnqueueCleanBuffer(buf *FileBuffer)

For BufferListHelpers

func (*Inode) UpgradeToStateLock added in v0.36.0

func (inode *Inode) UpgradeToStateLock()

Only for inode in KeepOwnerLock

type InodeAttributes

type InodeAttributes struct {
	Size  uint64
	Mtime time.Time
	Ctime time.Time
	Uid   uint32
	Gid   uint32
	Rdev  uint32
	Mode  os.FileMode
}

type InodeQueue added in v0.40.0

type InodeQueue struct {
	// contains filtered or unexported fields
}

func (*InodeQueue) Add added in v0.40.0

func (l *InodeQueue) Add(inodeID uint64) (queueID uint64)

func (*InodeQueue) Delete added in v0.40.0

func (l *InodeQueue) Delete(queueID uint64)

func (*InodeQueue) Next added in v0.40.0

func (l *InodeQueue) Next(minQueueID uint64) (inodeID, nextQueueID uint64)

func (*InodeQueue) Size added in v0.40.0

func (l *InodeQueue) Size() int

type Joinable added in v0.36.0

type Joinable interface {
	Join(ctx context.Context) error
}

type ListBlobsInput

type ListBlobsInput struct {
	Prefix            *string
	Delimiter         *string
	MaxKeys           *uint32
	StartAfter        *string // XXX: not supported by Azure
	ContinuationToken *string
}

type ListBlobsOutput

type ListBlobsOutput struct {
	Prefixes              []BlobPrefixOutput
	Items                 []BlobItemOutput
	NextContinuationToken *string
	IsTruncated           bool

	RequestId string
}

func RetryListBlobs added in v0.40.0

func RetryListBlobs(flags *cfg.FlagStorage, cloud StorageBackend, req *ListBlobsInput) (resp *ListBlobsOutput, err error)

type MPUPart

type MPUPart struct {
	Num    uint32
	Offset uint64
	Size   uint64
	ETag   string
}

type MakeBucketInput

type MakeBucketInput struct {
}

type MakeBucketOutput

type MakeBucketOutput struct {
	RequestId string
}

type Mount

type Mount struct {
	// contains filtered or unexported fields
}

type MountedFS added in v0.36.0

type MountedFS interface {
	Join(ctx context.Context) error
	Unmount() error
}

type MultiReader

type MultiReader struct {
	// contains filtered or unexported fields
}

func NewMultiReader

func NewMultiReader() *MultiReader

func (*MultiReader) AddBuffer

func (r *MultiReader) AddBuffer(buf []byte)

func (*MultiReader) AddZero

func (r *MultiReader) AddZero(size uint64)

func (*MultiReader) Len

func (r *MultiReader) Len() uint64

func (*MultiReader) Read

func (r *MultiReader) Read(buf []byte) (n int, err error)

func (*MultiReader) Seek

func (r *MultiReader) Seek(offset int64, whence int) (newOffset int64, err error)

type MultipartBlobAbortOutput

type MultipartBlobAbortOutput struct {
	RequestId string
}

type MultipartBlobAddInput

type MultipartBlobAddInput struct {
	Commit     *MultipartBlobCommitInput
	PartNumber uint32

	Body io.ReadSeeker

	Size   uint64 // GCS wants to know part size
	Offset uint64 // ADLv2 needs to know offset
}

type MultipartBlobAddOutput

type MultipartBlobAddOutput struct {
	RequestId string
	PartId    *string
}

type MultipartBlobBeginInput

type MultipartBlobBeginInput struct {
	Key         string
	Metadata    map[string]*string
	ContentType *string
}

type MultipartBlobCommitInput

type MultipartBlobCommitInput struct {
	Key *string

	Metadata map[string]*string
	UploadId *string
	Parts    []*string
	NumParts uint32
	// contains filtered or unexported fields
}

type MultipartBlobCommitOutput

type MultipartBlobCommitOutput struct {
	ETag         *string
	LastModified *time.Time
	StorageClass *string

	RequestId string
}

type MultipartBlobCopyInput

type MultipartBlobCopyInput struct {
	Commit     *MultipartBlobCommitInput
	PartNumber uint32
	CopySource string
	Offset     uint64
	Size       uint64
}

type MultipartBlobCopyOutput

type MultipartBlobCopyOutput struct {
	RequestId string
	PartId    *string
}

type MultipartExpireInput

type MultipartExpireInput struct {
}

type MultipartExpireOutput

type MultipartExpireOutput struct {
	RequestId string
}

type NodeId added in v0.36.0

type NodeId uint64

type OpStats added in v0.34.2

type OpStats struct {
	// contains filtered or unexported fields
}

type PatchBlobInput added in v0.39.0

type PatchBlobInput struct {
	Key            string
	Offset         uint64
	Size           uint64
	AppendPartSize int64

	Body io.ReadSeeker
}

type PatchBlobOutput added in v0.39.0

type PatchBlobOutput struct {
	ETag         *string
	LastModified *time.Time

	RequestId string
}

type Peer added in v0.36.0

type Peer struct {
	// contains filtered or unexported fields
}

type PutBlobInput

type PutBlobInput struct {
	Key         string
	Metadata    map[string]*string
	ContentType *string
	DirBlob     bool

	Body io.ReadSeeker
	Size *uint64
}

type PutBlobOutput

type PutBlobOutput struct {
	ETag         *string
	LastModified *time.Time
	StorageClass *string

	RequestId string
}

type QueuedBuffer added in v0.40.0

type QueuedBuffer struct {
	// contains filtered or unexported fields
}

type Range added in v0.40.0

type Range struct {
	Start, End uint64
}

type ReadRange

type ReadRange struct {
	Offset   uint64
	Size     uint64
	Flushing bool
}

type ReadSeekerCloser

type ReadSeekerCloser struct {
	io.ReadSeeker
}

func (*ReadSeekerCloser) Close

func (r *ReadSeekerCloser) Close() error

type Recovery added in v0.36.0

type Recovery struct {
	pb.UnimplementedRecoveryServer
	Flags *cfg.FlagStorage
}

func (*Recovery) Unmount added in v0.36.0

func (rec *Recovery) Unmount(ctx context.Context, req *pb.UnmountRequest) (*pb.UnmountResponse, error)

type RemoveBucketInput

type RemoveBucketInput struct {
}

type RemoveBucketOutput

type RemoveBucketOutput struct {
	RequestId string
}

type RenameBlobInput

type RenameBlobInput struct {
	Source      string
	Destination string
}

type RenameBlobOutput

type RenameBlobOutput struct {
	RequestId string
}

type Request added in v0.36.0

type Request func(ctx context.Context, conn *grpc.ClientConn) error

type S3Backend

type S3Backend struct {
	*s3.S3
	// contains filtered or unexported fields
}

func NewS3

func NewS3(bucket string, flags *cfg.FlagStorage, config *cfg.S3Config) (*S3Backend, error)

func (*S3Backend) Bucket

func (s *S3Backend) Bucket() string

func (*S3Backend) Capabilities

func (s *S3Backend) Capabilities() *Capabilities

func (*S3Backend) CopyBlob

func (s *S3Backend) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error)

func (*S3Backend) Delegate

func (s *S3Backend) Delegate() interface{}

func (*S3Backend) DeleteBlob

func (s *S3Backend) DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error)

func (*S3Backend) DeleteBlobs

func (s *S3Backend) DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error)

func (*S3Backend) GetBlob

func (s *S3Backend) GetBlob(param *GetBlobInput) (*GetBlobOutput, error)

func (*S3Backend) HeadBlob

func (s *S3Backend) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error)

func (*S3Backend) Init

func (s *S3Backend) Init(key string) error

func (*S3Backend) ListBlobs

func (s *S3Backend) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error)

func (*S3Backend) ListObjectsV2

func (s *S3Backend) ListObjectsV2(params *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, string, error)

func (*S3Backend) MakeBucket

func (s *S3Backend) MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error)

func (*S3Backend) MultipartBlobAbort

func (s *S3Backend) MultipartBlobAbort(param *MultipartBlobCommitInput) (*MultipartBlobAbortOutput, error)

func (*S3Backend) MultipartBlobAdd

func (s *S3Backend) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error)

func (*S3Backend) MultipartBlobBegin

func (s *S3Backend) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error)

func (*S3Backend) MultipartBlobCommit

func (s *S3Backend) MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error)

func (*S3Backend) MultipartBlobCopy

func (s *S3Backend) MultipartBlobCopy(param *MultipartBlobCopyInput) (*MultipartBlobCopyOutput, error)

func (*S3Backend) MultipartExpire

func (s *S3Backend) MultipartExpire(param *MultipartExpireInput) (*MultipartExpireOutput, error)

func (*S3Backend) PatchBlob added in v0.39.0

func (s *S3Backend) PatchBlob(param *PatchBlobInput) (*PatchBlobOutput, error)

func (*S3Backend) PutBlob

func (s *S3Backend) PutBlob(param *PutBlobInput) (*PutBlobOutput, error)

func (*S3Backend) RefreshIAM

func (s *S3Backend) RefreshIAM()

func (*S3Backend) RemoveBucket

func (s *S3Backend) RemoveBucket(param *RemoveBucketInput) (*RemoveBucketOutput, error)

func (*S3Backend) RenameBlob

func (s *S3Backend) RenameBlob(param *RenameBlobInput) (*RenameBlobOutput, error)

func (*S3Backend) TryIAM

func (s *S3Backend) TryIAM() (err error)

type SlurpGap added in v0.34.0

type SlurpGap struct {
	// contains filtered or unexported fields
}

type Stat added in v0.36.0

type Stat struct {
	// contains filtered or unexported fields
}

type StorageBackend

type StorageBackend interface {
	Init(key string) error
	Capabilities() *Capabilities
	// typically this would return bucket/prefix
	Bucket() string
	HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error)
	ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error)
	DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error)
	DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error)
	RenameBlob(param *RenameBlobInput) (*RenameBlobOutput, error)
	CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error)
	GetBlob(param *GetBlobInput) (*GetBlobOutput, error)
	PutBlob(param *PutBlobInput) (*PutBlobOutput, error)
	PatchBlob(param *PatchBlobInput) (*PatchBlobOutput, error)
	MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error)
	MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error)
	MultipartBlobCopy(param *MultipartBlobCopyInput) (*MultipartBlobCopyOutput, error)
	MultipartBlobAbort(param *MultipartBlobCommitInput) (*MultipartBlobAbortOutput, error)
	MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error)
	MultipartExpire(param *MultipartExpireInput) (*MultipartExpireOutput, error)
	RemoveBucket(param *RemoveBucketInput) (*RemoveBucketOutput, error)
	MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error)
	Delegate() interface{}
}

/ Implementations of all the functions here are expected to be / concurrency-safe, except for / / Init() is called exactly once before any other functions are / called. / / Capabilities()/Bucket() are expected to be const

func NewBackend

func NewBackend(bucket string, flags *cfg.FlagStorage) (cloud StorageBackend, err error)

type StorageBackendInitError

type StorageBackendInitError struct {
	// contains filtered or unexported fields
}

func (StorageBackendInitError) Bucket

func (s StorageBackendInitError) Bucket() string

func (StorageBackendInitError) Capabilities

func (e StorageBackendInitError) Capabilities() *Capabilities

func (StorageBackendInitError) CopyBlob

func (StorageBackendInitError) Delegate

func (e StorageBackendInitError) Delegate() interface{}

func (StorageBackendInitError) DeleteBlob

func (StorageBackendInitError) DeleteBlobs

func (StorageBackendInitError) GetBlob

func (StorageBackendInitError) HeadBlob

func (StorageBackendInitError) Init

func (e StorageBackendInitError) Init(key string) error

func (StorageBackendInitError) ListBlobs

func (StorageBackendInitError) MakeBucket

func (StorageBackendInitError) MultipartBlobAbort

func (StorageBackendInitError) MultipartBlobAdd

func (StorageBackendInitError) MultipartBlobBegin

func (StorageBackendInitError) MultipartBlobCommit

func (StorageBackendInitError) MultipartBlobCopy

func (StorageBackendInitError) MultipartExpire

func (StorageBackendInitError) PatchBlob added in v0.39.0

func (StorageBackendInitError) PutBlob

func (StorageBackendInitError) RemoveBucket

func (StorageBackendInitError) RenameBlob

type StorageBackendInitWrapper

type StorageBackendInitWrapper struct {
	StorageBackend
	// contains filtered or unexported fields
}

func (*StorageBackendInitWrapper) Bucket

func (s *StorageBackendInitWrapper) Bucket() string

func (*StorageBackendInitWrapper) Capabilities

func (s *StorageBackendInitWrapper) Capabilities() *Capabilities

func (*StorageBackendInitWrapper) CopyBlob

func (*StorageBackendInitWrapper) DeleteBlob

func (*StorageBackendInitWrapper) DeleteBlobs

func (*StorageBackendInitWrapper) GetBlob

func (*StorageBackendInitWrapper) HeadBlob

func (*StorageBackendInitWrapper) Init

func (s *StorageBackendInitWrapper) Init(key string) error

func (*StorageBackendInitWrapper) ListBlobs

func (*StorageBackendInitWrapper) MakeBucket

func (*StorageBackendInitWrapper) MultipartBlobAbort

func (*StorageBackendInitWrapper) MultipartBlobAdd

func (*StorageBackendInitWrapper) MultipartBlobBegin

func (*StorageBackendInitWrapper) MultipartBlobCommit

func (*StorageBackendInitWrapper) MultipartBlobCopy

func (*StorageBackendInitWrapper) MultipartExpire

func (*StorageBackendInitWrapper) PatchBlob added in v0.39.0

func (*StorageBackendInitWrapper) PutBlob

func (*StorageBackendInitWrapper) RemoveBucket

func (*StorageBackendInitWrapper) RenameBlob

Directories

Path Synopsis
System permissions-related code.
System permissions-related code.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL