fs

package
v1.3.22 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 25, 2024 License: MIT Imports: 34 Imported by: 2

Documentation

Overview

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2021-2023, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Index

Constants

View Source 
const (
	ObjectType   = "ob"
	WorkfileType = "wk"
	ECSliceType  = "ec"
	ECMetaType   = "mt"
)
View Source 
const (
	SieMpathIDMismatch = (1 + iota) * 10
	SieTargetIDMismatch
	SieNotEqVMD
	SieMetaCorrupted
	SieFsDiffers
	SieMpathNotFound
)
View Source 
const (
	// prefixes for workfiles created by various services
	WorkfileRemote       = "remote"         // getting object from neighbor target when rebalancing
	WorkfileColdget      = "cold"           // object GET: coldget
	WorkfilePut          = "put"            // object PUT
	WorkfileCopy         = "copy"           // copy object
	WorkfileAppend       = "append"         // APPEND to object (as file)
	WorkfileAppendToArch = "append-to-arch" // APPEND to existing archive
	WorkfileCreateArch   = "create-arch"    // CREATE multi-object archive
)
View Source 
const (
	FlagBeingDisabled uint64 = 1 << iota
	FlagBeingDetached
)

enum Mountpath.Flags

View Source 
const FlagWaitingDD = FlagBeingDisabled | FlagBeingDetached

Variables

View Source 
var CSM *contentSpecMgr

Functions

func AllMpathBcks 

func AllMpathBcks(opts *WalkOpts) (bcks []cmn.Bck, err error)

func CapStatusGetWhat 

func CapStatusGetWhat() (fsInfo apc.CapacityInfo)

func Clblk added in v1.3.18 

func Clblk()

`ios` delegations

func CleanPathErr added in v1.3.18 

func CleanPathErr(err error)

TODO: define fs.PathErr to return "ais://nnn/shard-99.tar not found" instead of the current " no such file or directory"

func ComputeDiskSize added in v1.3.21 

func ComputeDiskSize()

total disk size

func CountPersisted 

func CountPersisted(fname string) (cnt int)

func CreateBucket 

func CreateBucket(bck *cmn.Bck, nilbmd bool) (errs []error)

func Decommission 

func Decommission(mdOnly bool)

func DestroyBucket 

func DestroyBucket(op string, bck *cmn.Bck, bid uint64) (err error)

NOTE: caller must make sure to evict LOM cache

func DirectOpen 

func DirectOpen(path string, flag int, perm os.FileMode) (*os.File, error)

DirectOpen opens a file with direct disk access (with OS caching disabled).

func FillDiskStats 

func FillDiskStats(m ios.AllDiskStats)

func Get 

func Get() (MPI, MPI)

returns both available and disabled mountpaths (compare with GetAvail)

func GetAllMpathUtils 

func GetAllMpathUtils() (utils *ios.MpathUtil)

func GetDiskSize added in v1.3.21 

func GetDiskSize() uint64

func GetMpathUtil 

func GetMpathUtil(mpath string) int64

func GetXattr 

func GetXattr(fqn, attrName string) ([]byte, error)

GetXattr gets xattr by name - see also the buffered version below

func GetXattrBuf 

func GetXattrBuf(fqn, attrName string, buf []byte) (b []byte, err error)

GetXattr gets xattr by name via provided buffer

func IsDirEmpty 

func IsDirEmpty(dir string) (names []string, empty bool, err error)

func LcacheIdx 

func LcacheIdx(digest uint64) int

func LoadNodeID 

func LoadNodeID(mpaths cos.StrSet) (mDaeID string, err error)

traverses all mountpaths to load and validate node ID

func MarkerExists 

func MarkerExists(marker string) bool

func MountpathsToLists 

func MountpathsToLists() (mpl *apc.MountpathList)

func New 

func New(num int, allowSharedDisksAndNoDisks bool)

create a new singleton

func NumAvail 

func NumAvail() int

func OnDiskSize added in v1.3.21 

func OnDiskSize(bck *cmn.Bck, prefix string) (size uint64)

bucket and bucket+prefix on-disk sizing

func PersistMarker 

func PersistMarker(marker string) (fatalErr, writeErr error)

func PersistOnMpaths 

func PersistOnMpaths(fname, backupName string, meta jsp.Opts, atMost int, b []byte, sgl *memsys.SGL) (cnt, availCnt int)

PersistOnMpaths persists `what` on mountpaths under "mountpath.Path/path" filename. It does it on maximum `atMost` mountPaths. If `atMost == 0`, it does it on every mountpath. If `backupPath != ""`, it removes files from `backupPath` and moves files from `path` to `backupPath`. Returns how many times it has successfully stored a file.

func PutMPI 

func PutMPI(available, disabled MPI)

func RemoveAll added in v1.3.18 

func RemoveAll(dir string) (err error)

retrying ENOTEMPTY - "directory not empty" race vs. new writes

func RemoveMarker 

func RemoveMarker(marker string) (err error)

func RenameBucketDirs 

func RenameBucketDirs(bckFrom, bckTo *cmn.Bck) (err error)

func SetXattr 

func SetXattr(fqn, attrName string, data []byte) (err error)

SetXattr sets xattr name = value

func TestDisableValidation 

func TestDisableValidation()

TestDisableValidation disables fsid checking and allows mountpaths without disks (testing-only)

func TestNew 

func TestNew(iostater ios.IOS)

used only in tests

func ValidateNCopies 

func ValidateNCopies(tname string, copies int) (err error)

func Walk 

func Walk(opts *WalkOpts) error

func WalkBck 

func WalkBck(opts *WalkBckOpts) error

func WalkDir 

func WalkDir(dir string, ucb func(string, DirEntry) error) error

NOTE: using Go filepath.WalkDir pros: lexical deterministic order; cons: reads the entire directory

Types

type CDF added in v1.3.16 

type CDF struct {
	Capacity
	Disks []string `json:"disks"` // owned disks (ios.FsDisks map => slice)
	FS    string   `json:"fs"`    // cos.Fs + cos.FsID
}

Capacity, Disks, Filesystem (CDF) (not to be confused with Cumulative Distribution Function)

type CapStatus 

type CapStatus struct {
	// config
	HighWM int64
	OOS    int64
	// metrics
	TotalUsed  uint64 // bytes
	TotalAvail uint64 // bytes
	PctAvg     int32  // average used (%)
	PctMax     int32  // max used (%)
	PctMin     int32  // max used (%)
}

func Cap added in v1.3.16 

func Cap() (cs CapStatus)

func CapPeriodic 

func CapPeriodic(now int64, config *cmn.Config, tcdf *TargetCDF) (cs CapStatus, updated bool, err, errCap error)

called only and exclusively by `stats.Trunner` providing `config.Periodic.StatsTime` tick

func CapRefresh added in v1.3.16 

func CapRefresh(config *cmn.Config, tcdf *TargetCDF) (cs CapStatus, err, errCap error)

func (*CapStatus) Err 

func (cs *CapStatus) Err() (err error)

note: conditioning on max, not avg

func (*CapStatus) IsNil 

func (cs *CapStatus) IsNil() bool

func (*CapStatus) IsOOS added in v1.3.21 

func (cs *CapStatus) IsOOS() bool

func (*CapStatus) String 

func (cs *CapStatus) String() (s string)

type Capacity 

type Capacity struct {
	Used    uint64 `json:"used,string"`  // bytes
	Avail   uint64 `json:"avail,string"` // ditto
	PctUsed int32  `json:"pct_used"`     // %% used (redundant ok)
}

type ContentInfo 

type ContentInfo struct {
	Dir  string // original directory
	Base string // original basename
	Type string // content type
	Old  bool   // true if old (subj. to space cleanup)
}

type ContentResolver 

type ContentResolver interface {
	// When set to true, services like rebalance have permission to move
	// content for example to another target because it is misplaced (HRW).
	PermToMove() bool
	// When set to true, services like LRU have permission to evict/delete content
	PermToEvict() bool
	// When set to true, content can be checksumed, shown or processed in other ways.
	PermToProcess() bool

	// Generates unique base name for original one. This function may add
	// additional information to the base name.
	// prefix - user-defined marker
	GenUniqueFQN(base, prefix string) (ufqn string)
	// Parses generated unique fqn to the original one.
	ParseUniqueFQN(base string) (orig string, old, ok bool)
}

type DirEntry 

type DirEntry interface {
	IsDir() bool
}

type ECMetaContentResolver 

type ECMetaContentResolver struct{}

func (*ECMetaContentResolver) GenUniqueFQN 

func (*ECMetaContentResolver) GenUniqueFQN(base, _ string) string

func (*ECMetaContentResolver) ParseUniqueFQN 

func (*ECMetaContentResolver) ParseUniqueFQN(base string) (orig string, old, ok bool)

func (*ECMetaContentResolver) PermToEvict 

func (*ECMetaContentResolver) PermToEvict() bool

func (*ECMetaContentResolver) PermToMove 

func (*ECMetaContentResolver) PermToMove() bool

func (*ECMetaContentResolver) PermToProcess 

func (*ECMetaContentResolver) PermToProcess() bool

type ECSliceContentResolver 

type ECSliceContentResolver struct{}

func (*ECSliceContentResolver) GenUniqueFQN 

func (*ECSliceContentResolver) GenUniqueFQN(base, _ string) string

func (*ECSliceContentResolver) ParseUniqueFQN 

func (*ECSliceContentResolver) ParseUniqueFQN(base string) (orig string, old, ok bool)

func (*ECSliceContentResolver) PermToEvict 

func (*ECSliceContentResolver) PermToEvict() bool

func (*ECSliceContentResolver) PermToMove 

func (*ECSliceContentResolver) PermToMove() bool

func (*ECSliceContentResolver) PermToProcess 

func (*ECSliceContentResolver) PermToProcess() bool

type ErrMountpathNoDisks 

type ErrMountpathNoDisks struct {
	Mi *Mountpath
}

func (*ErrMountpathNoDisks) Error 

func (e *ErrMountpathNoDisks) Error() string

type ErrStorageIntegrity 

type ErrStorageIntegrity struct {
	Msg  string
	Code int // Sie* enum above
}

func (*ErrStorageIntegrity) Error 

func (sie *ErrStorageIntegrity) Error() string

type MPI 

type MPI map[string]*Mountpath

func GetAvail 

func GetAvail() MPI

func (MPI) String 

func (mpi MPI) String() string

type MountedFS 

type MountedFS struct {
	// contains filtered or unexported fields
}

MountedFS holds all mountpaths for the target.

type Mountpath added in v1.3.16 

type Mountpath struct {
	Path   string   // clean path
	cos.FS          // underlying filesystem
	Disks  []string // owned disks (ios.FsDisks map => slice)

	PathDigest uint64 // (HRW logic)
	// contains filtered or unexported fields
}

func Add 

func Add(mpath, tid string) (mi *Mountpath, err error)

(used only in _unit_ tests - compare with AddMpath below)

func AddMpath 

func AddMpath(mpath, tid string, cb func(), force bool) (mi *Mountpath, err error)

Add adds new mountpath to the target's `avail` TODO: extend `force=true` to disregard "filesystem sharing"

func BeginDD 

func BeginDD(action string, flags uint64, mpath string) (mi *Mountpath, numAvail int, noResil bool, err error)

begin (disable | detach) transaction: CoW-mark the corresponding mountpath

func Disable 

func Disable(mpath string, cb ...func()) (disabledMpath *Mountpath, err error)

Disables a mountpath, i.e., removes it from usage but keeps in the volume (for possible future re-enablement). If successful, returns the disabled mountpath. Otherwise, returns nil (also in the case if the mountpath was already disabled).

func Enable 

func Enable(mpath string) (enabledMpath *Mountpath, err error)

(unit tests only - compare with EnableMpath below)

func EnableMpath 

func EnableMpath(mpath, tid string, cb func()) (enabledMpath *Mountpath, err error)

Enable enables previously disabled mountpath. enabled is set to true if mountpath has been moved from disabled to available and exists is set to true if such mountpath even exists.

func FQN2Mpath 

func FQN2Mpath(fqn string) (found *Mountpath, relativePath string, err error)

FQN2Mpath matches FQN to mountpath and returns the mountpath and the relative path.

func Hrw added in v1.3.21 

func Hrw(uname string) (mi *Mountpath, digest uint64, err error)

func NewMountpath 

func NewMountpath(mpath string) (mi *Mountpath, err error)

func Path2Mpath 

func Path2Mpath(path string) (found *Mountpath, err error)

Path2Mpath takes in any file path (e.g., ../../a/b/c) and returns the matching `mi`, if exists

func Remove 

func Remove(mpath string, cb ...func()) (*Mountpath, error)

Remove removes mountpaths from the target's mountpaths. It searches for the mountpath in `available` and, if not found, in `disabled`.

func (*Mountpath) AddDisabled added in v1.3.16 

func (mi *Mountpath) AddDisabled(disabled MPI)

func (*Mountpath) AddEnabled added in v1.3.16 

func (mi *Mountpath) AddEnabled(tid string, avail MPI, config *cmn.Config) (err error)

func (*Mountpath) CheckDisks added in v1.3.16 

func (mi *Mountpath) CheckDisks() (err error)

func (*Mountpath) ClearDD added in v1.3.16 

func (mi *Mountpath) ClearDD()

func (*Mountpath) ClearMDs added in v1.3.16 

func (mi *Mountpath) ClearMDs(inclBMD bool) (rerr error)

func (*Mountpath) CreateMissingBckDirs added in v1.3.16 

func (mi *Mountpath) CreateMissingBckDirs(bck *cmn.Bck) (err error)

func (*Mountpath) DeletedRoot added in v1.3.16 

func (mi *Mountpath) DeletedRoot() string

func (*Mountpath) IsAnySet added in v1.3.16 

func (mi *Mountpath) IsAnySet(flags uint64) bool

func (*Mountpath) IsIdle added in v1.3.16 

func (mi *Mountpath) IsIdle(config *cmn.Config) bool

func (*Mountpath) LomCache added in v1.3.16 

func (mi *Mountpath) LomCache(idx int) *sync.Map

func (*Mountpath) MakePathBck added in v1.3.16 

func (mi *Mountpath) MakePathBck(bck *cmn.Bck) string

func (*Mountpath) MakePathCT added in v1.3.16 

func (mi *Mountpath) MakePathCT(bck *cmn.Bck, contentType string) string

func (*Mountpath) MakePathFQN added in v1.3.16 

func (mi *Mountpath) MakePathFQN(bck *cmn.Bck, contentType, objName string) string

func (*Mountpath) MoveToDeleted added in v1.3.16 

func (mi *Mountpath) MoveToDeleted(dir string) (err error)

MoveToDeleted removes directory in steps: 1. Synchronously gets temporary directory name 2. Synchronously renames old folder to temporary directory

func (*Mountpath) RemoveDeleted added in v1.3.16 

func (mi *Mountpath) RemoveDeleted(who string) (rerr error)

func (*Mountpath) SetDaemonIDXattr added in v1.3.16 

func (mi *Mountpath) SetDaemonIDXattr(tid string) error

func (*Mountpath) String added in v1.3.16 

func (mi *Mountpath) String() string

func (*Mountpath) TempDir added in v1.3.16 

func (mi *Mountpath) TempDir(dir string) string

type ObjectContentResolver 

type ObjectContentResolver struct{}

func (*ObjectContentResolver) GenUniqueFQN 

func (*ObjectContentResolver) GenUniqueFQN(base, _ string) string

func (*ObjectContentResolver) ParseUniqueFQN 

func (*ObjectContentResolver) ParseUniqueFQN(base string) (orig string, old, ok bool)

func (*ObjectContentResolver) PermToEvict 

func (*ObjectContentResolver) PermToEvict() bool

func (*ObjectContentResolver) PermToMove 

func (*ObjectContentResolver) PermToMove() bool

func (*ObjectContentResolver) PermToProcess 

func (*ObjectContentResolver) PermToProcess() bool

type ParsedFQN 

type ParsedFQN struct {
	Mountpath   *Mountpath
	ContentType string
	Bck         cmn.Bck
	ObjName     string
	Digest      uint64
}

func ParseFQN 

func ParseFQN(fqn string) (parsed ParsedFQN, err error)

ParseFQN splits a provided FQN (created by `MakePathFQN`) or reports an error.

type PartsFQN 

type PartsFQN interface {
	ObjectName() string
	Bucket() *cmn.Bck
	Mountpath() *Mountpath
	CacheIdx() int
}

type TargetCDF added in v1.3.16 

type TargetCDF struct {
	Mountpaths map[string]*CDF // mpath => [Capacity, Disks, FS (CDF)]
	PctMax     int32           `json:"pct_max"` // max used (%)
	PctAvg     int32           `json:"pct_avg"` // avg used (%)
	PctMin     int32           `json:"pct_min"` // min used (%)
	CsErr      string          `json:"cs_err"`  // OOS or high-wm error message
}

Target (cumulative) CDF

type WalkBckOpts 

type WalkBckOpts struct {
	ValidateCallback walkFunc // should return filepath.SkipDir to skip directory without an error
	WalkOpts
}

type WalkOpts 

type WalkOpts struct {
	Mi       *Mountpath
	Callback walkFunc
	Bck      cmn.Bck
	Dir      string
	CTs      []string
	Sorted   bool
}

type WorkfileContentResolver 

type WorkfileContentResolver struct{}

func (*WorkfileContentResolver) GenUniqueFQN 

func (*WorkfileContentResolver) GenUniqueFQN(base, prefix string) string

func (*WorkfileContentResolver) ParseUniqueFQN 

func (*WorkfileContentResolver) ParseUniqueFQN(base string) (orig string, old, ok bool)

func (*WorkfileContentResolver) PermToEvict 

func (*WorkfileContentResolver) PermToEvict() bool

func (*WorkfileContentResolver) PermToMove 

func (*WorkfileContentResolver) PermToMove() bool

func (*WorkfileContentResolver) PermToProcess 

func (*WorkfileContentResolver) PermToProcess() bool

Source Files

View all Source files

Directories

Path Synopsis
Package health provides a basic mountpath health monitor.
 Package health provides a basic mountpath health monitor.
Package mpather provides per-mountpath concepts.
 Package mpather provides per-mountpath concepts.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.