paths

package
v1.26.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 20, 2025 License: Apache-2.0, MIT Imports: 56 Imported by: 0

Documentation

Index

Constants

View Source
const BatchMetaFile = "batch.json" // supraseal
View Source
const CommitPhase1OutputFileSupra = "commit-phase1-output"
View Source
const LockTimeOut = 300 * time.Second

timeout after which we consider a lock to be stale

View Source
const MetaFile = "sectorstore.json"
View Source
const MinFreeStoragePercentage = float64(0)
View Source
const NoMinerFilter = abi.ActorID(0)
View Source
const SnapVproofFile = "snap-vproof.json"
View Source
const StashDirName = "stash"
View Source
const URLSeparator = ","

Variables

View Source
var (
	// Measures
	GenerateSingleVanillaProofCalls    = stats.Int64(pre+"generate_single_vanilla_proof_calls", "Number of calls to GenerateSingleVanillaProof", stats.UnitDimensionless)
	GenerateSingleVanillaProofErrors   = stats.Int64(pre+"generate_single_vanilla_proof_errors", "Number of errors in GenerateSingleVanillaProof", stats.UnitDimensionless)
	GenerateSingleVanillaProofDuration = stats.Int64(pre+"generate_single_vanilla_proof_duration_seconds", "Duration of GenerateSingleVanillaProof in seconds", stats.UnitMilliseconds)
)
View Source
var CopyBuf = 1 << 20
View Source
var FetchTempSubdir = "fetching"
View Source
var HeartbeatInterval = 10 * time.Second
View Source
var LocalReaderTimeout = 5 * time.Second

LocalReaderTimeout is the timeout for keeping local reader files open without any read activity.

View Source
var MaxDiskUsageDuration = time.Second
View Source
var ReservationCtxLock = contextlock.NewContextLock()

used to guard allocation decisions between assignment and reservation

View Source
var SkippedHeartbeatThresh = HeartbeatInterval * 5
View Source
var SlowPoStCheckThreshold = 45 * time.Second

time abow which a warn log will be emitted for slow PoSt reads

View Source
var SpaceUseKey = spaceUseCtxKey{}
View Source
var StatTimeout = 5 * time.Second

Functions

func DoubleCallWrap

func DoubleCallWrap(f func()) func()

DoubleCallWrap wraps a function to make sure it's not called twice

func FetchWithTemp

func FetchWithTemp(ctx context.Context, urls []string, dest string, header http.Header) (string, error)

FetchWithTemp fetches data into a temp 'fetching' directory, then moves the file to destination The set of URLs must refer to the same object, if one fails, another one will be tried.

func FileTypeFromString

func FileTypeFromString(t string) (storiface.SectorFileType, error)

func MaxCapacity added in v1.24.3

func MaxCapacity(st fsutil.FsStat) int64

func MinerFilter

func MinerFilter(allowMiners, denyMiners []string, miner abi.ActorID) (bool, string, error)

func Move

func Move(from, to string) error

func ParentsForProof added in v1.23.1

func ParentsForProof(spt abi.RegisteredSealProof) (string, error)

func StorageFromFile added in v1.23.1

func StorageFromFile(path string, def *storiface.StorageConfig) (*storiface.StorageConfig, error)

func StorageFromReader added in v1.23.1

func StorageFromReader(reader io.Reader) (*storiface.StorageConfig, error)

func WriteStorageFile added in v1.23.1

func WriteStorageFile(filePath string, config storiface.StorageConfig) error

Types

type BasicLocalStorage

type BasicLocalStorage struct {
	PathToJSON string
}

func (*BasicLocalStorage) DiskUsage

func (ls *BasicLocalStorage) DiskUsage(path string) (int64, error)

func (*BasicLocalStorage) GetStorage

func (ls *BasicLocalStorage) GetStorage() (storiface.StorageConfig, error)

func (*BasicLocalStorage) SetStorage

func (ls *BasicLocalStorage) SetStorage(f func(*storiface.StorageConfig)) error

func (*BasicLocalStorage) Stat

func (ls *BasicLocalStorage) Stat(path string) (fsutil.FsStat, error)

type BatchMeta added in v1.23.1

type BatchMeta struct {
	SupraSeal     bool
	BlockOffset   uint64
	NumInPipeline int

	BatchSectors int
}

type DBIndex

type DBIndex struct {
	// contains filtered or unexported fields
}

func (*DBIndex) BatchStorageDeclareSectors added in v1.23.1

func (dbi *DBIndex) BatchStorageDeclareSectors(ctx context.Context, declarations []SectorDeclaration) error

func (*DBIndex) StorageAttach

func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo, st fsutil.FsStat) error

func (*DBIndex) StorageBestAlloc

func (dbi *DBIndex) StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType, miner abi.ActorID) ([]storiface.StorageInfo, error)

func (*DBIndex) StorageDeclareSector

func (dbi *DBIndex) StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error

func (*DBIndex) StorageDetach

func (dbi *DBIndex) StorageDetach(ctx context.Context, id storiface.ID, url string) error

func (*DBIndex) StorageDropSector

func (dbi *DBIndex) StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error

func (*DBIndex) StorageFindSector

func (dbi *DBIndex) StorageFindSector(ctx context.Context, s abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]storiface.SectorStorageInfo, error)

func (*DBIndex) StorageGetLocks

func (dbi *DBIndex) StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error)

func (*DBIndex) StorageInfo

func (dbi *DBIndex) StorageInfo(ctx context.Context, id storiface.ID) (storiface.StorageInfo, error)

func (*DBIndex) StorageList

func (dbi *DBIndex) StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error)

func (*DBIndex) StorageLock

func (dbi *DBIndex) StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error

func (*DBIndex) StorageReportHealth

func (dbi *DBIndex) StorageReportHealth(ctx context.Context, id storiface.ID, report storiface.HealthReport) error

func (*DBIndex) StorageTryLock

func (dbi *DBIndex) StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)

type DefaultPartialFileHandler

type DefaultPartialFileHandler struct{}

DefaultPartialFileHandler is the default implementation of the PartialFileHandler interface. This is probably the only implementation we'll ever use because the purpose of the interface to is to mock out partial file related functionality during testing.

func (*DefaultPartialFileHandler) Close

Close closes the partial file

func (*DefaultPartialFileHandler) HasAllocated

func (*DefaultPartialFileHandler) OpenPartialFile

func (d *DefaultPartialFileHandler) OpenPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*partialfile.PartialFile, error)

func (*DefaultPartialFileHandler) Reader

type FetchHandler

type FetchHandler struct {
	Local interface {
		Store
		StashStore
	}
	PfHandler PartialFileHandler
}

func (*FetchHandler) ServeHTTP

func (handler *FetchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)

type Local

type Local struct {
	// contains filtered or unexported fields
}

func NewLocal

func NewLocal(ctx context.Context, ls LocalStorage, index SectorIndex, url string) (*Local, error)

func (*Local) ClosePath

func (st *Local) ClosePath(ctx context.Context, id storiface.ID) error

func (*Local) FsStat

func (st *Local) FsStat(ctx context.Context, id storiface.ID) (fsutil.FsStat, error)

func (*Local) GeneratePoRepVanillaProof

func (st *Local) GeneratePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error)

func (*Local) GenerateSingleVanillaProof

func (st *Local) GenerateSingleVanillaProof(ctx context.Context, minerID abi.ActorID, si storiface.PostSectorChallenge, ppt abi.RegisteredPoStProof) ([]byte, error)

func (*Local) Local

func (st *Local) Local(ctx context.Context) ([]storiface.StoragePath, error)

func (*Local) MoveStorage

func (st *Local) MoveStorage(ctx context.Context, s storiface.SectorRef, types storiface.SectorFileType, opts ...storiface.AcquireOption) error

func (*Local) OpenPath

func (st *Local) OpenPath(ctx context.Context, p string) error

func (*Local) ReadSnapVanillaProof added in v1.23.0

func (st *Local) ReadSnapVanillaProof(ctx context.Context, sr storiface.SectorRef) ([]byte, error)

func (*Local) Redeclare

func (st *Local) Redeclare(ctx context.Context, filterId *storiface.ID, dropMissingDecls bool) error

func (*Local) Remove

func (st *Local) Remove(ctx context.Context, sid abi.SectorID, typ storiface.SectorFileType, force bool, keepIn []storiface.ID) error

func (*Local) RemoveCopies

func (st *Local) RemoveCopies(ctx context.Context, sid abi.SectorID, typ storiface.SectorFileType) error

func (*Local) Reserve

func (st *Local) Reserve(ctx context.Context, sid storiface.SectorRef, ft storiface.SectorFileType,
	storageIDs storiface.SectorPaths, overheadTab map[storiface.SectorFileType]int, minFreePercentage float64) (userRelease func(), err error)

func (*Local) ServeAndRemove added in v1.25.1

func (st *Local) ServeAndRemove(ctx context.Context, id uuid.UUID) (io.ReadCloser, error)

func (*Local) StashCreate added in v1.25.1

func (st *Local) StashCreate(ctx context.Context, maxSize int64, writeFunc func(f *os.File) error) (uuid.UUID, error)

func (*Local) StashRemove added in v1.25.1

func (st *Local) StashRemove(ctx context.Context, id uuid.UUID) error

func (*Local) StashURL added in v1.25.1

func (st *Local) StashURL(id uuid.UUID) (url.URL, error)

type LocalStorage

type LocalStorage interface {
	GetStorage() (storiface.StorageConfig, error)
	SetStorage(func(*storiface.StorageConfig)) error

	Stat(path string) (fsutil.FsStat, error)

	// returns real disk usage for a file/directory
	// os.ErrNotExit when file doesn't exist
	DiskUsage(path string) (int64, error)
}

type PartialFileHandler

type PartialFileHandler interface {
	// OpenPartialFile opens and returns a partial file at the given path and also verifies it has the given
	// size
	OpenPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*partialfile.PartialFile, error)

	// HasAllocated returns true if the given partial file has an unsealed piece starting at the given offset with the given size.
	// returns false otherwise.
	HasAllocated(pf *partialfile.PartialFile, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error)

	// Reader returns a file from which we can read the unsealed piece in the partial file.
	Reader(pf *partialfile.PartialFile, offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) (io.Reader, error)

	// Close closes the partial file
	Close(pf *partialfile.PartialFile) error
}

PartialFileHandler helps mock out the partial file functionality during testing.

type PoRepVanillaParams

type PoRepVanillaParams struct {
	Sector   storiface.SectorRef
	Sealed   cid.Cid
	Unsealed cid.Cid
	Ticket   abi.SealRandomness
	Seed     abi.InteractiveSealRandomness
}

type Remote

type Remote struct {
	// contains filtered or unexported fields
}

func NewRemote

func NewRemote(local Store, index SectorIndex, auth http.Header, fetchLimit int, pfHandler PartialFileHandler) *Remote

func (*Remote) CheckIsUnsealed

func (r *Remote) CheckIsUnsealed(ctx context.Context, s storiface.SectorRef, offset, size abi.PaddedPieceSize) (bool, error)

CheckIsUnsealed checks if we have an unsealed piece at the given offset in an already unsealed sector file for the given piece either locally or on any of the workers. Returns true if we have the unsealed piece, false otherwise.

func (*Remote) FsStat

func (r *Remote) FsStat(ctx context.Context, id storiface.ID) (fsutil.FsStat, error)

func (*Remote) GeneratePoRepVanillaProof

func (r *Remote) GeneratePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error)

func (*Remote) GenerateSingleVanillaProof

func (r *Remote) GenerateSingleVanillaProof(ctx context.Context, minerID abi.ActorID, sinfo storiface.PostSectorChallenge, ppt abi.RegisteredPoStProof) ([]byte, error)

func (*Remote) MoveStorage

func (*Remote) ReadMinCacheInto added in v1.23.0

func (r *Remote) ReadMinCacheInto(ctx context.Context, s storiface.SectorRef, ft storiface.SectorFileType, w io.Writer) error

ReadMinCacheInto reads finalized-like (few MiB) cache files into the target dir

func (*Remote) ReadRemote added in v1.25.1

func (r *Remote) ReadRemote(ctx context.Context, url string, offset, size int64) (io.ReadCloser, error)

func (*Remote) ReadSnapVanillaProof added in v1.23.0

func (r *Remote) ReadSnapVanillaProof(ctx context.Context, sr storiface.SectorRef) ([]byte, error)

func (*Remote) Reader

func (r *Remote) Reader(ctx context.Context, s storiface.SectorRef, offset, size abi.PaddedPieceSize) (func(startOffsetAligned, endOffsetAligned storiface.PaddedByteIndex) (io.ReadCloser, error), error)

Reader returns a reader for an unsealed piece at the given offset in the given sector. If the Miner has the unsealed piece locally, it will return a reader that reads from the local copy. If the Miner does NOT have the unsealed piece locally, it will query all workers that have the unsealed sector file to know if they have the unsealed piece and will then read the unsealed piece data from a worker that has it.

Returns a nil reader if : 1. no worker(local worker included) has an unsealed file for the given sector OR 2. no worker(local worker included) has the unsealed piece in their unsealed sector file. Will return a nil reader and a nil error in such a case.

func (*Remote) ReaderPiece added in v1.25.1

func (r *Remote) ReaderPiece(ctx context.Context, s storiface.SectorRef, ft storiface.SectorFileType, offset, size int64) (func(startOffset, endOffset int64) (io.ReadCloser, error), error)

ReaderPiece returns a reader for a piece at the given offset in the given file. NOTE: This method could also be used in place of Reader, but that one also handles PartialFile logic specfic to unsealed sectors.

Hopefully when PartialFile is removed, this method will be used instead of Reader.

func (*Remote) ReaderSeq

ReaderSeq creates a simple sequential reader for a file. Does not work for file types which are a directory (e.g. FTCache).

func (*Remote) Remove

func (r *Remote) Remove(ctx context.Context, sid abi.SectorID, typ storiface.SectorFileType, force bool, keepIn []storiface.ID) error

func (*Remote) RemoveCopies

func (r *Remote) RemoveCopies(ctx context.Context, s abi.SectorID, typ storiface.SectorFileType) error

func (*Remote) Reserve

func (r *Remote) Reserve(ctx context.Context, sid storiface.SectorRef, ft storiface.SectorFileType, storageIDs storiface.SectorPaths, overheadTab map[storiface.SectorFileType]int, minFreePercentage float64) (func(), error)

func (*Remote) StatUrl

func (r *Remote) StatUrl(ctx context.Context, urlStr string, id storiface.ID) (fsutil.FsStat, error)

type SectorDeclaration added in v1.23.1

type SectorDeclaration struct {
	StorageID storiface.ID
	SectorID  abi.SectorID
	FileType  storiface.SectorFileType
	Primary   bool
}

SectorDeclaration represents a single sector declaration

type SectorIndex

type SectorIndex interface {
	StorageAttach(context.Context, storiface.StorageInfo, fsutil.FsStat) error
	StorageDetach(ctx context.Context, id storiface.ID, url string) error
	StorageInfo(context.Context, storiface.ID) (storiface.StorageInfo, error)
	StorageReportHealth(context.Context, storiface.ID, storiface.HealthReport) error

	StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error
	StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error
	StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]storiface.SectorStorageInfo, error)
	BatchStorageDeclareSectors(ctx context.Context, declarations []SectorDeclaration) error

	StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType, miner abi.ActorID) ([]storiface.StorageInfo, error)

	// atomically acquire locks on all sector file types. close ctx to unlock
	StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error
	StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)
	StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error)

	StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error)
}

type SingleVanillaParams

type SingleVanillaParams struct {
	Miner     abi.ActorID
	Sector    storiface.PostSectorChallenge
	ProofType abi.RegisteredPoStProof
}

type SnapVanillaParams added in v1.23.0

type SnapVanillaParams struct {
	Sector storiface.SectorRef
}

type SpaceUseFunc added in v1.23.1

type SpaceUseFunc func(ft storiface.SectorFileType, ssize abi.SectorSize) (uint64, error)

type StashStore added in v1.25.1

type StashStore interface {
	// StashCreate creates a new stash file with the specified maximum size.
	// It selects a sealing path with the most available space and creates a file
	// named [uuid].tmp in the stash directory.
	//
	// The provided writeFunc is called with an *os.File pointing to the newly
	// created stash file, allowing the caller to write data into it.
	//
	// Parameters:
	//  - ctx: Context for cancellation and timeout.
	//  - maxSize: The maximum size of the stash file in bytes.
	//  - writeFunc: A function that writes data to the stash file.
	//
	// Returns:
	//  - uuid.UUID: A unique identifier for the created stash.
	//  - error: An error if the stash could not be created.
	StashCreate(ctx context.Context, maxSize int64, writeFunc func(f *os.File) error) (uuid.UUID, error)

	// ServeAndRemove serves the stash file identified by the given UUID as an io.ReadCloser.
	// Once the stash has been fully read (i.e., the last byte has been read),
	// the stash file is automatically removed.
	// If the read is incomplete (e.g., due to an error or premature closure),
	// the stash file remains on disk.
	//
	// Parameters:
	//  - ctx: Context for cancellation and timeout.
	//  - id: The UUID of the stash to serve.
	//
	// Returns:
	//  - io.ReadCloser: A reader for the stash file.
	//  - error: An error if the stash could not be served.
	ServeAndRemove(ctx context.Context, id uuid.UUID) (io.ReadCloser, error)

	// StashRemove removes the stash file identified by the given UUID.
	//
	// Parameters:
	//  - ctx: Context for cancellation and timeout.
	//  - id: The UUID of the stash to remove.
	//
	// Returns:
	//  - error: An error if the stash could not be removed.
	StashRemove(ctx context.Context, id uuid.UUID) error

	// StashURL generates a URL for accessing the stash identified by the given UUID.
	//
	// Parameters:
	//  - id: The UUID of the stash.
	//
	// Returns:
	//  - url.URL: The URL where the stash can be accessed.
	//  - error: An error if the URL could not be generated.
	StashURL(id uuid.UUID) (url.URL, error)
}

StashStore provides methods for managing stashes within storage paths. Stashes are temporary files located in the "stash/" subdirectory of sealing paths. They are removed on startup and are not indexed. Stashes are used to store arbitrary data and can be served or removed as needed.

type Store

type Store interface {
	AcquireSector(ctx context.Context, s storiface.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, sealing storiface.PathType, op storiface.AcquireMode, opts ...storiface.AcquireOption) (paths storiface.SectorPaths, stores storiface.SectorPaths, err error)
	Remove(ctx context.Context, s abi.SectorID, types storiface.SectorFileType, force bool, keepIn []storiface.ID) error

	// like remove, but doesn't remove the primary sector copy, nor the last
	// non-primary copy if there no primary copies
	RemoveCopies(ctx context.Context, s abi.SectorID, types storiface.SectorFileType) error

	// move sectors into storage
	MoveStorage(ctx context.Context, s storiface.SectorRef, types storiface.SectorFileType, opts ...storiface.AcquireOption) error

	FsStat(ctx context.Context, id storiface.ID) (fsutil.FsStat, error)

	Reserve(ctx context.Context, sid storiface.SectorRef, ft storiface.SectorFileType, storageIDs storiface.SectorPaths, overheadTab map[storiface.SectorFileType]int, minFreePercentage float64) (func(), error)

	GenerateSingleVanillaProof(ctx context.Context, minerID abi.ActorID, si storiface.PostSectorChallenge, ppt abi.RegisteredPoStProof) ([]byte, error)
	GeneratePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error)
	ReadSnapVanillaProof(ctx context.Context, sr storiface.SectorRef) ([]byte, error)
}

type URLs

type URLs []string

func UrlsFromString added in v1.23.1

func UrlsFromString(in string) URLs

Directories

Path Synopsis
Package mocks is a generated GoMock package.
Package mocks is a generated GoMock package.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL