paths

package
v1.27.0-rc1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 30, 2024 License: Apache-2.0, MIT Imports: 46 Imported by: 13

Documentation

Index

Constants

View Source 
const LockTimeOut = 300 * time.Second

timeout after which we consider a lock to be stale

View Source 
const MetaFile = "sectorstore.json"
View Source 
const NoMinerFilter = abi.ActorID(0)
View Source 
const URLSeparator = ","

Variables

View Source 
var CopyBuf = 1 << 20
View Source 
var FetchTempSubdir = "fetching"
View Source 
var HeartbeatInterval = 10 * time.Second
View Source 
var LocalReaderTimeout = 5 * time.Second

LocalReaderTimeout is the timeout for keeping local reader files open without any read activity.

View Source 
var MaxDiskUsageDuration = time.Second
View Source 
var SkippedHeartbeatThresh = HeartbeatInterval * 5
View Source 
var StatTimeout = 5 * time.Second

Functions

func DoubleCallWrap added in v1.27.0 

func DoubleCallWrap(f func()) func()

DoubleCallWrap wraps a function to make sure it's not called twice

func FetchWithTemp added in v1.17.2 

func FetchWithTemp(ctx context.Context, urls []string, dest string, header http.Header) (string, error)

FetchWithTemp fetches data into a temp 'fetching' directory, then moves the file to destination The set of URLs must refer to the same object, if one fails, another one will be tried.

func FileTypeFromString added in v1.17.2 

func FileTypeFromString(t string) (storiface.SectorFileType, error)

func MinerFilter added in v1.27.0 

func MinerFilter(allowMiners, denyMiners []string, miner abi.ActorID) (bool, string, error)

func Move added in v1.25.0 

func Move(from, to string) error

Types

type BasicLocalStorage added in v1.25.2 

type BasicLocalStorage struct {
	PathToJSON string
}

func (*BasicLocalStorage) DiskUsage added in v1.25.2 

func (ls *BasicLocalStorage) DiskUsage(path string) (int64, error)

func (*BasicLocalStorage) GetStorage added in v1.25.2 

func (ls *BasicLocalStorage) GetStorage() (storiface.StorageConfig, error)

func (*BasicLocalStorage) SetStorage added in v1.25.2 

func (ls *BasicLocalStorage) SetStorage(f func(*storiface.StorageConfig)) error

func (*BasicLocalStorage) Stat added in v1.25.2 

func (ls *BasicLocalStorage) Stat(path string) (fsutil.FsStat, error)

type DBIndex added in v1.25.2 

type DBIndex struct {
	// contains filtered or unexported fields
}

func NewDBIndex added in v1.25.2 

func NewDBIndex(al *alerting.Alerting, db *harmonydb.DB) *DBIndex

func (*DBIndex) StorageAttach added in v1.25.2 

func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo, st fsutil.FsStat) error

func (*DBIndex) StorageBestAlloc added in v1.25.2 

func (dbi *DBIndex) StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType, miner abi.ActorID) ([]storiface.StorageInfo, error)

func (*DBIndex) StorageDeclareSector added in v1.25.2 

func (dbi *DBIndex) StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error

func (*DBIndex) StorageDetach added in v1.25.2 

func (dbi *DBIndex) StorageDetach(ctx context.Context, id storiface.ID, url string) error

func (*DBIndex) StorageDropSector added in v1.25.2 

func (dbi *DBIndex) StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error

func (*DBIndex) StorageFindSector added in v1.25.2 

func (dbi *DBIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]storiface.SectorStorageInfo, error)

func (*DBIndex) StorageGetLocks added in v1.25.2 

func (dbi *DBIndex) StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error)

func (*DBIndex) StorageInfo added in v1.25.2 

func (dbi *DBIndex) StorageInfo(ctx context.Context, id storiface.ID) (storiface.StorageInfo, error)

func (*DBIndex) StorageList added in v1.25.2 

func (dbi *DBIndex) StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error)

func (*DBIndex) StorageLock added in v1.25.2 

func (dbi *DBIndex) StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error

func (*DBIndex) StorageReportHealth added in v1.25.2 

func (dbi *DBIndex) StorageReportHealth(ctx context.Context, id storiface.ID, report storiface.HealthReport) error

func (*DBIndex) StorageTryLock added in v1.25.2 

func (dbi *DBIndex) StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)

type DefaultPartialFileHandler 

type DefaultPartialFileHandler struct{}

DefaultPartialFileHandler is the default implementation of the PartialFileHandler interface. This is probably the only implementation we'll ever use because the purpose of the interface to is to mock out partial file related functionality during testing.

func (*DefaultPartialFileHandler) Close 

func (d *DefaultPartialFileHandler) Close(pf *partialfile.PartialFile) error

Close closes the partial file

func (*DefaultPartialFileHandler) HasAllocated 

func (d *DefaultPartialFileHandler) HasAllocated(pf *partialfile.PartialFile, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error)

func (*DefaultPartialFileHandler) OpenPartialFile 

func (d *DefaultPartialFileHandler) OpenPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*partialfile.PartialFile, error)

func (*DefaultPartialFileHandler) Reader 

func (d *DefaultPartialFileHandler) Reader(pf *partialfile.PartialFile, offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) (io.Reader, error)

type FetchHandler 

type FetchHandler struct {
	Local     Store
	PfHandler PartialFileHandler
}

func (*FetchHandler) ServeHTTP 

func (handler *FetchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)

type Local 

type Local struct {
	// contains filtered or unexported fields
}

func NewLocal 

func NewLocal(ctx context.Context, ls LocalStorage, index SectorIndex, urls []string) (*Local, error)

func (*Local) AcquireSector 

func (st *Local) AcquireSector(ctx context.Context, sid storiface.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, pathType storiface.PathType, op storiface.AcquireMode, opts ...storiface.AcquireOption) (storiface.SectorPaths, storiface.SectorPaths, error)

func (*Local) ClosePath added in v1.17.1 

func (st *Local) ClosePath(ctx context.Context, id storiface.ID) error

func (*Local) FsStat 

func (st *Local) FsStat(ctx context.Context, id storiface.ID) (fsutil.FsStat, error)

func (*Local) GeneratePoRepVanillaProof added in v1.27.0 

func (st *Local) GeneratePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error)

func (*Local) GenerateSingleVanillaProof 

func (st *Local) GenerateSingleVanillaProof(ctx context.Context, minerID abi.ActorID, si storiface.PostSectorChallenge, ppt abi.RegisteredPoStProof) ([]byte, error)

func (*Local) Local 

func (st *Local) Local(ctx context.Context) ([]storiface.StoragePath, error)

func (*Local) MoveStorage 

func (st *Local) MoveStorage(ctx context.Context, s storiface.SectorRef, types storiface.SectorFileType, opts ...storiface.AcquireOption) error

func (*Local) OpenPath 

func (st *Local) OpenPath(ctx context.Context, p string) error

func (*Local) Redeclare 

func (st *Local) Redeclare(ctx context.Context, filterId *storiface.ID, dropMissingDecls bool) error

func (*Local) Remove 

func (st *Local) Remove(ctx context.Context, sid abi.SectorID, typ storiface.SectorFileType, force bool, keepIn []storiface.ID) error

func (*Local) RemoveCopies 

func (st *Local) RemoveCopies(ctx context.Context, sid abi.SectorID, typ storiface.SectorFileType) error

func (*Local) Reserve 

func (st *Local) Reserve(ctx context.Context, sid storiface.SectorRef, ft storiface.SectorFileType, storageIDs storiface.SectorPaths, overheadTab map[storiface.SectorFileType]int) (release func(), err error)

type LocalStorage 

type LocalStorage interface {
	GetStorage() (storiface.StorageConfig, error)
	SetStorage(func(*storiface.StorageConfig)) error

	Stat(path string) (fsutil.FsStat, error)

	// returns real disk usage for a file/directory
	// os.ErrNotExit when file doesn't exist
	DiskUsage(path string) (int64, error)
}

type MemIndex added in v1.25.2 

type MemIndex struct {
	// contains filtered or unexported fields
}

MemIndex represents an in-memory index of storage sectors and storage entries.

func NewMemIndex added in v1.25.2 

func NewMemIndex(al *alerting.Alerting) *MemIndex

func (*MemIndex) FindSector added in v1.25.2 

func (i *MemIndex) FindSector(id abi.SectorID, typ storiface.SectorFileType) ([]storiface.ID, error)

func (*MemIndex) StorageAttach added in v1.25.2 

func (i *MemIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo, st fsutil.FsStat) error

func (*MemIndex) StorageBestAlloc added in v1.25.2 

func (i *MemIndex) StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType, miner abi.ActorID) ([]storiface.StorageInfo, error)

StorageBestAlloc selects the best available storage options for allocating a sector file. It takes into account the allocation type (sealing or storage), sector size, and path type (sealing or storage).

The method first estimates the required space for the allocation based on the sector size and path type. It then iterates through all available storage options and filters out those that cannot be used for the given path type. It also filters out storage options that do not have enough available space or have not received heartbeats within a certain threshold.

The remaining storage options are sorted based on their available space and weight, with higher availability and weight being prioritized. The method then returns the information of the selected storage options.

If no suitable storage options are found, it returns an error indicating that no good path is available.

Parameters: - ctx: the context.Context object for cancellation and timeouts - allocate: the type of allocation (sealing or storage) - ssize: the size of the sector file - pathType: the path type (sealing or storage)

Returns: - []storiface.StorageInfo: the information of the selected storage options - error: an error indicating any issues encountered during the process

func (*MemIndex) StorageDeclareSector added in v1.25.2 

func (i *MemIndex) StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error

func (*MemIndex) StorageDetach added in v1.25.2 

func (i *MemIndex) StorageDetach(ctx context.Context, id storiface.ID, url string) error

func (*MemIndex) StorageDropSector added in v1.25.2 

func (i *MemIndex) StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error

func (*MemIndex) StorageFindSector added in v1.25.2 

func (i *MemIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]storiface.SectorStorageInfo, error)

func (MemIndex) StorageGetLocks added in v1.25.2 

func (i MemIndex) StorageGetLocks(context.Context) (storiface.SectorLocks, error)

func (*MemIndex) StorageInfo added in v1.25.2 

func (i *MemIndex) StorageInfo(ctx context.Context, id storiface.ID) (storiface.StorageInfo, error)

StorageInfo retrieves the storage information for a given storage ID.

The method first acquires a read lock on the MemIndex to ensure thread-safety. It then checks if the storage ID exists in the stores map. If not, it returns an error indicating that the sector store was not found.

Finally, it returns the storage information of the selected storage.

Parameters: - ctx: the context.Context object for cancellation and timeouts - id: the ID of the storage to retrieve information for

Returns: - storiface.StorageInfo: the storage information of the selected storage ID - error: an error indicating any issues encountered during the process

func (*MemIndex) StorageList added in v1.25.2 

func (i *MemIndex) StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error)

func (MemIndex) StorageLock added in v1.25.2 

func (i MemIndex) StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error

func (*MemIndex) StorageReportHealth added in v1.25.2 

func (i *MemIndex) StorageReportHealth(ctx context.Context, id storiface.ID, report storiface.HealthReport) error

func (MemIndex) StorageTryLock added in v1.25.2 

func (i MemIndex) StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)

type PartialFileHandler 

type PartialFileHandler interface {
	// OpenPartialFile opens and returns a partial file at the given path and also verifies it has the given
	// size
	OpenPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*partialfile.PartialFile, error)

	// HasAllocated returns true if the given partial file has an unsealed piece starting at the given offset with the given size.
	// returns false otherwise.
	HasAllocated(pf *partialfile.PartialFile, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error)

	// Reader returns a file from which we can read the unsealed piece in the partial file.
	Reader(pf *partialfile.PartialFile, offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) (io.Reader, error)

	// Close closes the partial file
	Close(pf *partialfile.PartialFile) error
}

PartialFileHandler helps mock out the partial file functionality during testing.

type PoRepVanillaParams added in v1.27.0 

type PoRepVanillaParams struct {
	Sector   storiface.SectorRef
	Sealed   cid.Cid
	Unsealed cid.Cid
	Ticket   abi.SealRandomness
	Seed     abi.InteractiveSealRandomness
}

type Remote 

type Remote struct {
	// contains filtered or unexported fields
}

func NewRemote 

func NewRemote(local Store, index SectorIndex, auth http.Header, fetchLimit int, pfHandler PartialFileHandler) *Remote

func (*Remote) AcquireSector 

func (r *Remote) AcquireSector(ctx context.Context, s storiface.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, pathType storiface.PathType, op storiface.AcquireMode, opts ...storiface.AcquireOption) (storiface.SectorPaths, storiface.SectorPaths, error)

func (*Remote) CheckIsUnsealed 

func (r *Remote) CheckIsUnsealed(ctx context.Context, s storiface.SectorRef, offset, size abi.PaddedPieceSize) (bool, error)

CheckIsUnsealed checks if we have an unsealed piece at the given offset in an already unsealed sector file for the given piece either locally or on any of the workers. Returns true if we have the unsealed piece, false otherwise.

func (*Remote) FsStat 

func (r *Remote) FsStat(ctx context.Context, id storiface.ID) (fsutil.FsStat, error)

func (*Remote) GeneratePoRepVanillaProof added in v1.27.0 

func (r *Remote) GeneratePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error)

func (*Remote) GenerateSingleVanillaProof 

func (r *Remote) GenerateSingleVanillaProof(ctx context.Context, minerID abi.ActorID, sinfo storiface.PostSectorChallenge, ppt abi.RegisteredPoStProof) ([]byte, error)

func (*Remote) MoveStorage 

func (r *Remote) MoveStorage(ctx context.Context, s storiface.SectorRef, types storiface.SectorFileType, opts ...storiface.AcquireOption) error

func (*Remote) Reader 

func (r *Remote) Reader(ctx context.Context, s storiface.SectorRef, offset, size abi.PaddedPieceSize) (func(startOffsetAligned, endOffsetAligned storiface.PaddedByteIndex) (io.ReadCloser, error), error)

Reader returns a reader for an unsealed piece at the given offset in the given sector. If the Miner has the unsealed piece locally, it will return a reader that reads from the local copy. If the Miner does NOT have the unsealed piece locally, it will query all workers that have the unsealed sector file to know if they have the unsealed piece and will then read the unsealed piece data from a worker that has it.

Returns a nil reader if : 1. no worker(local worker included) has an unsealed file for the given sector OR 2. no worker(local worker included) has the unsealed piece in their unsealed sector file. Will return a nil reader and a nil error in such a case.

func (*Remote) ReaderSeq added in v1.27.0 

func (r *Remote) ReaderSeq(ctx context.Context, s storiface.SectorRef, ft storiface.SectorFileType) (io.ReadCloser, error)

ReaderSeq creates a simple sequential reader for a file. Does not work for file types which are a directory (e.g. FTCache).

func (*Remote) Remove 

func (r *Remote) Remove(ctx context.Context, sid abi.SectorID, typ storiface.SectorFileType, force bool, keepIn []storiface.ID) error

func (*Remote) RemoveCopies 

func (r *Remote) RemoveCopies(ctx context.Context, s abi.SectorID, typ storiface.SectorFileType) error

func (*Remote) Reserve 

func (r *Remote) Reserve(ctx context.Context, sid storiface.SectorRef, ft storiface.SectorFileType, storageIDs storiface.SectorPaths, overheadTab map[storiface.SectorFileType]int) (func(), error)

func (*Remote) StatUrl added in v1.27.0 

func (r *Remote) StatUrl(ctx context.Context, urlStr string, id storiface.ID) (fsutil.FsStat, error)

type SectorIndex 

type SectorIndex interface {
	StorageAttach(context.Context, storiface.StorageInfo, fsutil.FsStat) error
	StorageDetach(ctx context.Context, id storiface.ID, url string) error
	StorageInfo(context.Context, storiface.ID) (storiface.StorageInfo, error)
	StorageReportHealth(context.Context, storiface.ID, storiface.HealthReport) error

	StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error
	StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error
	StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]storiface.SectorStorageInfo, error)

	StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType, miner abi.ActorID) ([]storiface.StorageInfo, error)

	// atomically acquire locks on all sector file types. close ctx to unlock
	StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error
	StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)
	StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error)

	StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error)
}

type SingleVanillaParams 

type SingleVanillaParams struct {
	Miner     abi.ActorID
	Sector    storiface.PostSectorChallenge
	ProofType abi.RegisteredPoStProof
}

type Store 

type Store interface {
	AcquireSector(ctx context.Context, s storiface.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, sealing storiface.PathType, op storiface.AcquireMode, opts ...storiface.AcquireOption) (paths storiface.SectorPaths, stores storiface.SectorPaths, err error)
	Remove(ctx context.Context, s abi.SectorID, types storiface.SectorFileType, force bool, keepIn []storiface.ID) error

	// like remove, but doesn't remove the primary sector copy, nor the last
	// non-primary copy if there no primary copies
	RemoveCopies(ctx context.Context, s abi.SectorID, types storiface.SectorFileType) error

	// move sectors into storage
	MoveStorage(ctx context.Context, s storiface.SectorRef, types storiface.SectorFileType, opts ...storiface.AcquireOption) error

	FsStat(ctx context.Context, id storiface.ID) (fsutil.FsStat, error)

	Reserve(ctx context.Context, sid storiface.SectorRef, ft storiface.SectorFileType, storageIDs storiface.SectorPaths, overheadTab map[storiface.SectorFileType]int) (func(), error)

	GenerateSingleVanillaProof(ctx context.Context, minerID abi.ActorID, si storiface.PostSectorChallenge, ppt abi.RegisteredPoStProof) ([]byte, error)
	GeneratePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error)
}

type URLs 

type URLs []string

Source Files

View all Source files

Directories

Path Synopsis
Package mocks is a generated GoMock package.
 Package mocks is a generated GoMock package.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.