Documentation
¶
Index ¶
- func CreateChunkDiscardingFileMerkleTree(ctx context.Context, parameters *FileCreationParameters, ...) (...)
- func CreateDirectoryMerkleTree[TDirectory, TFile model_core.ReferenceMetadata](ctx context.Context, concurrency *semaphore.Weighted, group *errgroup.Group, ...) error
- func CreateFileMerkleTree[T model_core.ReferenceMetadata](ctx context.Context, parameters *FileCreationParameters, f io.Reader, ...) (model_core.PatchedMessage[*model_filesystem_pb.FileContents, T], error)
- func DirectoryGetContents[TReference any](ctx context.Context, ...) (model_core.Message[*model_filesystem_pb.DirectoryContents, TReference], error)
- func DirectoryGetLeaves[TReference any](ctx context.Context, ...) (model_core.Message[*model_filesystem_pb.Leaves, TReference], error)
- func FileContentsEntryToProto[TReference object.BasicReference](entry *FileContentsEntry[TReference]) ...
- func GetDirectoryWithContents[TMetadata model_core.ReferenceMetadata](createdDirectory *CreatedDirectory[TMetadata], ...) *model_filesystem_pb.Directory
- func NewCapturedDirectoryWalker(directoryParameters *DirectoryAccessParameters, ...) dag.ObjectContentsWalker
- func NewCapturedFileWalker(fileParameters *FileCreationParameters, r filesystem.FileReader, ...) dag.ObjectContentsWalker
- func NewDirectoryClusterObjectParser[TReference any]() ...
- func NewFileContentsListObjectParser[TReference object.BasicReference]() parser.ObjectParser[TReference, FileContentsList[TReference]]
- type CapturableDirectory
- type CapturableFile
- type CapturedDirectory
- type CreatedDirectory
- type Directory
- type DirectoryAccessParameters
- func (p *DirectoryAccessParameters) DecodeDirectory(contents *object.Contents, decodingParameters []byte) (*model_filesystem_pb.DirectoryContents, error)
- func (p *DirectoryAccessParameters) DecodeLeaves(contents *object.Contents, decodingParameters []byte) (*model_filesystem_pb.Leaves, error)
- func (p *DirectoryAccessParameters) GetEncoder() encoding.BinaryEncoder
- type DirectoryCluster
- type DirectoryComponentWalker
- func (cw *DirectoryComponentWalker[TReference]) GetCurrentDirectoriesStack() ([]model_core.Message[*model_filesystem_pb.DirectoryContents, TReference], ...)
- func (cw *DirectoryComponentWalker[TReference]) GetCurrentFileProperties() model_core.Message[*model_filesystem_pb.FileProperties, TReference]
- func (cw *DirectoryComponentWalker[TReference]) OnDirectory(name path.Component) (path.GotDirectoryOrSymlink, error)
- func (cw *DirectoryComponentWalker[TReference]) OnTerminal(name path.Component) (*path.GotSymlink, error)
- func (cw *DirectoryComponentWalker[TReference]) OnUp() (path.ComponentWalker, error)
- type DirectoryCreationParameters
- type DirectoryMerkleTreeCapturer
- type EscapementCountingScopeWalker
- func (sw *EscapementCountingScopeWalker) GetLevels() *wrapperspb.UInt32Value
- func (sw *EscapementCountingScopeWalker) OnAbsolute() (path.ComponentWalker, error)
- func (sw *EscapementCountingScopeWalker) OnDriveLetter(drive rune) (path.ComponentWalker, error)
- func (sw *EscapementCountingScopeWalker) OnRelative() (path.ComponentWalker, error)
- type FileAccessParameters
- func (p *FileAccessParameters) DecodeFileContentsList(contents *object.Contents, decodingParameters []byte) ([]*model_filesystem_pb.FileContents, error)
- func (p *FileAccessParameters) GetChunkEncoder() encoding.BinaryEncoder
- func (p *FileAccessParameters) GetFileContentsListEncoder() encoding.BinaryEncoder
- type FileContentsEntry
- type FileContentsIterator
- type FileContentsList
- type FileCreationParameters
- type FileMerkleTreeCapturer
- type FileMerkleTreeCapturerForTesting
- type FileReader
- func (fr *FileReader[TReference]) FileOpenRead(ctx context.Context, fileContents FileContentsEntry[TReference], ...) *SequentialFileReader[TReference]
- func (fr *FileReader[TReference]) FileOpenReadAt(ctx context.Context, fileContents FileContentsEntry[TReference]) io.ReaderAt
- func (fr *FileReader[TReference]) FileReadAll(ctx context.Context, fileContents FileContentsEntry[TReference], ...) ([]byte, error)
- func (fr *FileReader[TReference]) FileReadAt(ctx context.Context, fileContents FileContentsEntry[TReference], p []byte, ...) (int, error)
- func (fr *FileReader[TReference]) GetDecodingParametersSizeBytes(isFileContentsList bool) int
- type SectionWriter
- type SequentialFileReader
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func CreateChunkDiscardingFileMerkleTree ¶
func CreateChunkDiscardingFileMerkleTree(ctx context.Context, parameters *FileCreationParameters, f filesystem.FileReader) (model_core.PatchedMessage[*model_filesystem_pb.FileContents, dag.ObjectContentsWalker], error)
CreateChunkDiscardingFileMerkleTree is a helper function for creating a Merkle tree of a file and immediately constructing an ObjectContentsWalker for it. This function takes ownership of the file that is provided. Its contents may be re-read when the ObjectContentsWalker is accessed, and it will be released when no more ObjectContentsWalkers for the file exist.
func CreateDirectoryMerkleTree ¶
func CreateDirectoryMerkleTree[TDirectory, TFile model_core.ReferenceMetadata]( ctx context.Context, concurrency *semaphore.Weighted, group *errgroup.Group, directoryParameters *DirectoryCreationParameters, directory CapturableDirectory[TDirectory, TFile], capturer DirectoryMerkleTreeCapturer[TDirectory, TFile], out *CreatedDirectory[TDirectory], ) error
CreateDirectoryMerkleTree creates a Merkle tree that corresponds to the contents of a given directory. Upon success, a Directory message corresponding with the root directory is returned.
Computation of the Merkle trees of the individual files can be done in parallel. These processes may terminate asynchronously, meaning that group.Wait() needs to be called to ensure that the capturer is invoked for all objects, and the output message is set.
func CreateFileMerkleTree ¶
func CreateFileMerkleTree[T model_core.ReferenceMetadata](ctx context.Context, parameters *FileCreationParameters, f io.Reader, capturer FileMerkleTreeCapturer[T]) (model_core.PatchedMessage[*model_filesystem_pb.FileContents, T], error)
CreateFileMerkleTree creates a Merkle tree structure that corresponds to the contents of a single file. If a file is small, it stores all of its contents in a single object. If a file is large, it creates a B-tree.
Chunking of large files is performed using the MaxCDC algorithm. The resulting B-tree is a Prolly tree. This ensures that minor changes to a file also result in minor changes to the resulting Merkle tree.
func DirectoryGetContents ¶
func DirectoryGetContents[TReference any]( ctx context.Context, reader model_parser.ParsedObjectReader[model_core.Decodable[TReference], model_core.Message[*model_filesystem_pb.DirectoryContents, TReference]], directory model_core.Message[*model_filesystem_pb.Directory, TReference], ) (model_core.Message[*model_filesystem_pb.DirectoryContents, TReference], error)
DirectoryGetContents is a helper function for obtaining the contents of a directory.
func DirectoryGetLeaves ¶
func DirectoryGetLeaves[TReference any]( ctx context.Context, reader model_parser.ParsedObjectReader[model_core.Decodable[TReference], model_core.Message[*model_filesystem_pb.Leaves, TReference]], directory model_core.Message[*model_filesystem_pb.DirectoryContents, TReference], ) (model_core.Message[*model_filesystem_pb.Leaves, TReference], error)
DirectoryGetLeaves is a helper function for obtaining the leaves contained in a directory.
func FileContentsEntryToProto ¶
func FileContentsEntryToProto[TReference object.BasicReference]( entry *FileContentsEntry[TReference], ) model_core.PatchedMessage[*model_filesystem_pb.FileContents, dag.ObjectContentsWalker]
FileContentsEntryToProto converts a FileContentsEntry back to a Protobuf message.
TODO: Should this function take a model_core.ExistingObjectCapturer?
func GetDirectoryWithContents ¶
func GetDirectoryWithContents[TMetadata model_core.ReferenceMetadata]( createdDirectory *CreatedDirectory[TMetadata], externalObject *model_core.Decodable[model_core.CreatedObject[TMetadata]], patcher *model_core.ReferenceMessagePatcher[TMetadata], objectCapturer model_core.CreatedObjectCapturer[TMetadata], ) *model_filesystem_pb.Directory
GetDirectoryWithContents creates a Directory message that either contains or references a DirectoryContents message. This function is typically called as part of inlinedtree.Build(), as that function makes the decision whether there is enough space to inline the contents of a directory.
func NewCapturedDirectoryWalker ¶
func NewCapturedDirectoryWalker(directoryParameters *DirectoryAccessParameters, fileParameters *FileCreationParameters, rootDirectory CapturedDirectory, rootObject *model_core.CreatedObjectTree, decodingParameters []byte) dag.ObjectContentsWalker
NewCapturedDirectoryWalker returns an implementation of ObjectContentsWalker that is capable of walking over a hierarchy over Directory and Leaves objects that were created using CreateDirectoryMerkleTree and captured using FileDiscardingDirectoryMerkleTreeCapturer. This makes it possible to upload such directory hierarchies to a storage server.
These Merkle trees do not contain any file contents, but it is permitted for the storage server to request them. If that happens, we must reobtain them from the underlying file system. This is why the caller must provide a handle to the root directory on which the provided Merkle tree is based.
func NewCapturedFileWalker ¶
func NewCapturedFileWalker(fileParameters *FileCreationParameters, r filesystem.FileReader, fileReference object.LocalReference, fileSizeBytes uint64, fileObject *model_core.CreatedObjectTree, decodingParameters []byte) dag.ObjectContentsWalker
func NewDirectoryClusterObjectParser ¶
func NewDirectoryClusterObjectParser[TReference any]() model_parser.ObjectParser[TReference, model_core.Message[DirectoryCluster, TReference]]
NewDirectoryClusterObjectParser creates an ObjectParser that is capable of parsing directory objects, and exposing them in the form of a DirectoryCluster.
func NewFileContentsListObjectParser ¶
func NewFileContentsListObjectParser[TReference object.BasicReference]() parser.ObjectParser[TReference, FileContentsList[TReference]]
NewFileContentsListObjectParser creates an ObjectParser that is capable of parsing FileContentsList messages, turning them into a list of entries that can be processed by FileContentsIterator.
Types ¶
type CapturableDirectory ¶
type CapturableDirectory[TDirectory, TFile model_core.ReferenceMetadata] interface { // Identical to filesystem.Directory. Close() error ReadDir() ([]filesystem.FileInfo, error) Readlink(name path.Component) (path.Parser, error) // Enter a directory, so that it may be traversed. The // implementation has the possibility to return an existing // Directory message. This can be of use when computing a Merkle // tree that is based on an existing directory structure that is // altered slightly. EnterCapturableDirectory(name path.Component) (*CreatedDirectory[TDirectory], CapturableDirectory[TDirectory, TFile], error) // Open a file, so that a Merkle tree of its contents can be // computed. The actual Merkle tree computation is performed by // calling CapturableFile.CreateFileMerkleTree(). That way file // Merkle tree computation can happen in parallel. OpenForFileMerkleTreeCreation(name path.Component) (CapturableFile[TFile], error) }
CapturableDirectory is an interface for a directory that can be traversed by CreateFileMerkleTree().
type CapturableFile ¶
type CapturableFile[TFile model_core.ReferenceMetadata] interface { CreateFileMerkleTree(ctx context.Context) (model_core.PatchedMessage[*model_filesystem_pb.FileContents, TFile], error) Discard() }
CapturableFile is called into by CreateFileMerkleTree() to obtain a Merkle tree for a given file. Either one of these methods will be called exactly once.
func NewSimpleCapturableFile ¶
func NewSimpleCapturableFile[TFile model_core.ReferenceMetadata](fileContents model_core.PatchedMessage[*model_filesystem_pb.FileContents, TFile]) CapturableFile[TFile]
type CapturedDirectory ¶
type CapturedDirectory interface { Close() error EnterCapturedDirectory(name path.Component) (CapturedDirectory, error) OpenRead(name path.Component) (filesystem.FileReader, error) }
CapturedDirectory is called into by CapturedDirectoryWalker to traverse a directory hierarchy and read file contents.
type CreatedDirectory ¶
type CreatedDirectory[TDirectory model_core.ReferenceMetadata] struct { Message model_core.PatchedMessage[*model_filesystem_pb.DirectoryContents, TDirectory] MaximumSymlinkEscapementLevels *wrapperspb.UInt32Value }
CreatedDirectory contains the message of the root directory of the directory hierarchy for which a Merkle tree was created. It also contains all of the metadata that needs to be available when creating a DirectoryReference message for referring to the root directory.
func NewCreatedDirectoryBare ¶
func NewCreatedDirectoryBare[TDirectory model_core.ReferenceMetadata](message model_core.PatchedMessage[*model_filesystem_pb.DirectoryContents, TDirectory]) (*CreatedDirectory[TDirectory], error)
NewCreatedDirectoryBare creates a CreatedDirectory, recomputing MaximumSymlinkEscapementLevels from the provided Directory message.
This function can be used to embed an existing directory in a Merkle tree for which no DirectoryReference exists (e.g., because it was not the root directory of a directory cluster). In cases where a DirectoryReference is available, this function should not be used.
func (*CreatedDirectory[TDirectory]) ToDirectoryReference ¶
func (cd *CreatedDirectory[TDirectory]) ToDirectoryReference(reference *model_core_pb.DecodableReference) *model_filesystem_pb.DirectoryReference
ToDirectoryReference creates a DirectoryReference message that corresponds to the current directory.
type Directory ¶
type Directory struct { Directory *model_filesystem_pb.DirectoryContents // Indices at which child directories with inline contents are // accessible within the same cluster. If the child directory // has external contents, the index is set to -1. ChildDirectoryIndices []int }
Directory contained in a DirectoryCluster.
type DirectoryAccessParameters ¶
type DirectoryAccessParameters struct {
// contains filtered or unexported fields
}
DirectoryAccessParameters contains parameters that were used when creating Merkle trees of directories that should also be applied when attempting to access its contents afterwards. Parameters include whether files were compressed or encrypted.
func NewDirectoryAccessParametersFromProto ¶
func NewDirectoryAccessParametersFromProto(m *model_filesystem_pb.DirectoryAccessParameters, referenceFormat object.ReferenceFormat) (*DirectoryAccessParameters, error)
NewDirectoryAccessParametersFromProto creates an instance of DirectoryAccessParameters that matches the configuration stored in a Protobuf message. This, for example, permits a server to access files that were uploaded by a client.
func (*DirectoryAccessParameters) DecodeDirectory ¶
func (p *DirectoryAccessParameters) DecodeDirectory(contents *object.Contents, decodingParameters []byte) (*model_filesystem_pb.DirectoryContents, error)
func (*DirectoryAccessParameters) DecodeLeaves ¶
func (p *DirectoryAccessParameters) DecodeLeaves(contents *object.Contents, decodingParameters []byte) (*model_filesystem_pb.Leaves, error)
func (*DirectoryAccessParameters) GetEncoder ¶
func (p *DirectoryAccessParameters) GetEncoder() encoding.BinaryEncoder
type DirectoryCluster ¶
type DirectoryCluster []Directory
DirectoryCluster is a list of all Directory messages that are contained in a single object in storage. Directories are stored in topological order, meaning that the root directory is located at index zero.
The advantage of a DirectoryCluster over a plain Directory message is that it's possible to refer to individual directories using an index.
type DirectoryComponentWalker ¶
type DirectoryComponentWalker[TReference object.BasicReference] struct { // contains filtered or unexported fields }
func NewDirectoryComponentWalker ¶
func NewDirectoryComponentWalker[TReference object.BasicReference]( ctx context.Context, directoryContentsReader model_parser.ParsedObjectReader[model_core.Decodable[TReference], model_core.Message[*model_filesystem_pb.DirectoryContents, TReference]], leavesReader model_parser.ParsedObjectReader[model_core.Decodable[TReference], model_core.Message[*model_filesystem_pb.Leaves, TReference]], onUpHandler func() (path.ComponentWalker, error), currentDirectoryReference model_core.Message[*model_core_pb.DecodableReference, TReference], directoriesStack []model_core.Message[*model_filesystem_pb.DirectoryContents, TReference], ) *DirectoryComponentWalker[TReference]
func (*DirectoryComponentWalker[TReference]) GetCurrentDirectoriesStack ¶
func (cw *DirectoryComponentWalker[TReference]) GetCurrentDirectoriesStack() ([]model_core.Message[*model_filesystem_pb.DirectoryContents, TReference], error)
GetCurrentDirectoriesStack returns the stack of directories that the component walker currently uses to perform resolution. This list can, for example, be copied if the directory walker needs to be cloned.
func (*DirectoryComponentWalker[TReference]) GetCurrentFileProperties ¶
func (cw *DirectoryComponentWalker[TReference]) GetCurrentFileProperties() model_core.Message[*model_filesystem_pb.FileProperties, TReference]
func (*DirectoryComponentWalker[TReference]) OnDirectory ¶
func (cw *DirectoryComponentWalker[TReference]) OnDirectory(name path.Component) (path.GotDirectoryOrSymlink, error)
func (*DirectoryComponentWalker[TReference]) OnTerminal ¶
func (cw *DirectoryComponentWalker[TReference]) OnTerminal(name path.Component) (*path.GotSymlink, error)
func (*DirectoryComponentWalker[TReference]) OnUp ¶
func (cw *DirectoryComponentWalker[TReference]) OnUp() (path.ComponentWalker, error)
type DirectoryCreationParameters ¶
type DirectoryCreationParameters struct { *DirectoryAccessParameters // contains filtered or unexported fields }
func NewDirectoryCreationParametersFromProto ¶
func NewDirectoryCreationParametersFromProto(m *model_filesystem_pb.DirectoryCreationParameters, referenceFormat object.ReferenceFormat) (*DirectoryCreationParameters, error)
type DirectoryMerkleTreeCapturer ¶
type DirectoryMerkleTreeCapturer[TDirectory, TFile any] interface { CaptureFileNode(TFile) TDirectory CaptureDirectory(createdObject model_core.CreatedObject[TDirectory]) TDirectory CaptureLeaves(createdObject model_core.CreatedObject[TDirectory]) TDirectory }
var FileDiscardingDirectoryMerkleTreeCapturer DirectoryMerkleTreeCapturer[model_core.CreatedObjectTree, model_core.NoopReferenceMetadata] = fileDiscardingDirectoryMerkleTreeCapturer{}
FileDiscardingDirectoryMerkleTreeCapturer is an instance of DirectoryMerkleTreeCapturer that keeps any Directory and Leaves objects, but discards FileContents list and file chunk objects.
Discarding the contents of files is typically the right approach for uploading directory structures with changes to only a small number of files. The Merkle trees of files can be recomputed if it turns out they still need to be uploaded.
func NewSimpleDirectoryMerkleTreeCapturer ¶
func NewSimpleDirectoryMerkleTreeCapturer[TMetadata any](capturer model_core.CreatedObjectCapturer[TMetadata]) DirectoryMerkleTreeCapturer[TMetadata, TMetadata]
NewSimpleDirectoryMerkleTreeCapturer creates a DirectoryMerkleTreeCapturer that assumes that directories and leaves need to be captured the same way, and that file metadata uses the same type as directory metadata.
type EscapementCountingScopeWalker ¶
type EscapementCountingScopeWalker struct {
// contains filtered or unexported fields
}
EscapementCountingScopeWalker determines the number of levels a given path escapes the current directory. This is achieved by counting the number of leading ".." components.
If the path is absolute or if ".." components are placed after named components (e.g., "a/../b"), it determines the number of levels is not bounded.
func NewEscapementCountingScopeWalker ¶
func NewEscapementCountingScopeWalker() *EscapementCountingScopeWalker
NewEscapementCountingScopeWalker creates an EscapementCountingScopeWalker that is in the initial path parsing state (i.e., corresponding with path ".").
func (*EscapementCountingScopeWalker) GetLevels ¶
func (sw *EscapementCountingScopeWalker) GetLevels() *wrapperspb.UInt32Value
GetLevels returns the number of levels the parsed path escapes the current directory. If nil is returned, the escapement is not bounded.
func (*EscapementCountingScopeWalker) OnAbsolute ¶
func (sw *EscapementCountingScopeWalker) OnAbsolute() (path.ComponentWalker, error)
func (*EscapementCountingScopeWalker) OnDriveLetter ¶
func (sw *EscapementCountingScopeWalker) OnDriveLetter(drive rune) (path.ComponentWalker, error)
func (*EscapementCountingScopeWalker) OnRelative ¶
func (sw *EscapementCountingScopeWalker) OnRelative() (path.ComponentWalker, error)
type FileAccessParameters ¶
type FileAccessParameters struct {
// contains filtered or unexported fields
}
FileAccessParameters contains parameters that were used when creating Merkle trees of files that should also be applied when attempting to access its contents afterwards. Parameters include whether files were compressed or encrypted.
func NewFileAccessParametersFromProto ¶
func NewFileAccessParametersFromProto(m *model_filesystem_pb.FileAccessParameters, referenceFormat object.ReferenceFormat) (*FileAccessParameters, error)
NewFileAccessParametersFromProto creates an instance of FileAccessParameters that matches the configuration stored in a Protobuf message. This, for example, permits a server to access files that were uploaded by a client.
func (*FileAccessParameters) DecodeFileContentsList ¶
func (p *FileAccessParameters) DecodeFileContentsList(contents *object.Contents, decodingParameters []byte) ([]*model_filesystem_pb.FileContents, error)
DecodeFileContentsList extracts the FileContents list that is stored in an object backed by storage.
TODO: Maybe we should simply throw out this method? It doesn't provide a lot of value.
func (*FileAccessParameters) GetChunkEncoder ¶
func (p *FileAccessParameters) GetChunkEncoder() encoding.BinaryEncoder
func (*FileAccessParameters) GetFileContentsListEncoder ¶
func (p *FileAccessParameters) GetFileContentsListEncoder() encoding.BinaryEncoder
type FileContentsEntry ¶
type FileContentsEntry[TReference any] struct { EndBytes uint64 Reference model_core.Decodable[TReference] }
FileContentsEntry contains the properties of a part of a concatenated file. Note that the Reference field is only set when EndBytes is non-zero.
func NewFileContentsEntryFromProto ¶
func NewFileContentsEntryFromProto[TReference object.BasicReference](fileContents model_core.Message[*model_filesystem_pb.FileContents, TReference]) (FileContentsEntry[TReference], error)
NewFileContentsEntryFromProto constructs a FileContentsEntry based on the contents of a single FileContents Protobuf message, refering to the file as a whole.
type FileContentsIterator ¶
type FileContentsIterator[TReference any] struct { // contains filtered or unexported fields }
FileContentsIterator is a helper type for iterating over the chunks of a concatenated file sequentially.
func NewFileContentsIterator ¶
func NewFileContentsIterator[TReference object.BasicReference](root FileContentsEntry[TReference], initialOffsetBytes uint64) FileContentsIterator[TReference]
NewFileContentsIterator creates a FileContentsIterator that starts iteration at the provided offset within the file. It is the caller's responsibility to ensure the provided offset is less than the size of the file.
func (*FileContentsIterator[TReference]) GetCurrentPart ¶
func (i *FileContentsIterator[TReference]) GetCurrentPart() (reference model_core.Decodable[TReference], offsetBytes, sizeBytes uint64)
GetCurrentPart returns the reference of the part of the file that contain the data corresponding with the current offset. It also returns the offset within the part from which data should be read, and the expected total size of the part.
It is the caller's responsibility to track whether iteration has reached the end of the file. Once the end of the file has been reached, GetCurrentPart() may no longer be called.
func (*FileContentsIterator[TReference]) PushFileContentsList ¶
func (i *FileContentsIterator[TReference]) PushFileContentsList(list FileContentsList[TReference]) error
PushFileContentsList can be invoked after GetCurrentPart() to signal that the current part does not refer to a chunk of data, but another FileContentsList. After calling this method, another call to GetCurrentPart() can be made to retry resolution of the part within the provided FileContentsList.
func (*FileContentsIterator[TReference]) ToNextPart ¶
func (i *FileContentsIterator[TReference]) ToNextPart()
ToNextPart can be invoked after GetCurrentPart() to signal that the current part refers to a chunk of data. The next call to GetCurrentPart() will return the reference of the part that is stored after the current one.
type FileContentsList ¶
type FileContentsList[TReference any] []FileContentsEntry[TReference]
FileContentsList contains the properties of parts of a concatenated file. Parts are stored in the order in which they should be concatenated, with EndBytes increasing.
type FileCreationParameters ¶
type FileCreationParameters struct { *FileAccessParameters // contains filtered or unexported fields }
func NewFileCreationParametersFromProto ¶
func NewFileCreationParametersFromProto(m *model_filesystem_pb.FileCreationParameters, referenceFormat object.ReferenceFormat) (*FileCreationParameters, error)
func (*FileCreationParameters) EncodeChunk ¶
func (p *FileCreationParameters) EncodeChunk(data []byte) (model_core.Decodable[*object.Contents], error)
type FileMerkleTreeCapturer ¶
type FileMerkleTreeCapturer[TMetadata any] interface { CaptureChunk(contents *object.Contents) TMetadata CaptureFileContentsList(createdObject model_core.CreatedObject[TMetadata]) TMetadata }
FileMerkleTreeCapturer is provided by callers of CreateFileMerkleTree to provide logic for how the resulting Merkle tree of the file should be captured.
A no-op implementation can be used by the caller to simply compute a reference of the file. An implementation that actually captures the provided contents can be used to prepare a Merkle tree for uploading.
The methods below return metadata. The metadata for the root object will be returned by CreateFileMerkleTree.
var ChunkDiscardingFileMerkleTreeCapturer FileMerkleTreeCapturer[model_core.CreatedObjectTree] = chunkDiscardingFileMerkleTreeCapturer{}
ChunkDiscardingFileMerkleTreeCapturer is an implementation of FileMerkleTreeCapturer that only preserves the FileContents messages of the Merkle tree. This can be of use when incrementally replicating the contents of a file. In those cases it's wasteful to store the full contents of a file in memory.
func NewSimpleFileMerkleTreeCapturer ¶
func NewSimpleFileMerkleTreeCapturer[TMetadata any](capturer model_core.CreatedObjectCapturer[TMetadata]) FileMerkleTreeCapturer[TMetadata]
NewSimpleFileMerkleTreeCapturer creates a FileMerkleTreeCapturer that assumes that chunks and file contents lists need to be captured the same way.
type FileMerkleTreeCapturerForTesting ¶
type FileMerkleTreeCapturerForTesting FileMerkleTreeCapturer[model_core.ReferenceMetadata]
type FileReader ¶
type FileReader[TReference object.BasicReference] struct { // contains filtered or unexported fields }
func NewFileReader ¶
func NewFileReader[TReference object.BasicReference]( fileContentsListReader model_parser.ParsedObjectReader[model_core.Decodable[TReference], FileContentsList[TReference]], fileChunkReader model_parser.ParsedObjectReader[model_core.Decodable[TReference], []byte], ) *FileReader[TReference]
func (*FileReader[TReference]) FileOpenRead ¶
func (fr *FileReader[TReference]) FileOpenRead(ctx context.Context, fileContents FileContentsEntry[TReference], offsetBytes uint64) *SequentialFileReader[TReference]
func (*FileReader[TReference]) FileOpenReadAt ¶
func (fr *FileReader[TReference]) FileOpenReadAt(ctx context.Context, fileContents FileContentsEntry[TReference]) io.ReaderAt
func (*FileReader[TReference]) FileReadAll ¶
func (fr *FileReader[TReference]) FileReadAll(ctx context.Context, fileContents FileContentsEntry[TReference], maximumSizeBytes uint64) ([]byte, error)
func (*FileReader[TReference]) FileReadAt ¶
func (fr *FileReader[TReference]) FileReadAt(ctx context.Context, fileContents FileContentsEntry[TReference], p []byte, offsetBytes uint64) (int, error)
func (*FileReader[TReference]) GetDecodingParametersSizeBytes ¶
func (fr *FileReader[TReference]) GetDecodingParametersSizeBytes(isFileContentsList bool) int
type SectionWriter ¶
type SectionWriter struct {
// contains filtered or unexported fields
}
SectionWriter provides an implementation of io.Writer on top of io.WriterAt. It is similar to io.SectionReader, but then for writes.
func NewSectionWriter ¶
func NewSectionWriter(w io.WriterAt) *SectionWriter
func (*SectionWriter) GetOffsetBytes ¶
func (w *SectionWriter) GetOffsetBytes() int64
func (*SectionWriter) WriteString ¶
func (w *SectionWriter) WriteString(s string) (int, error)
type SequentialFileReader ¶
type SequentialFileReader[TReference object.BasicReference] struct { // contains filtered or unexported fields }
func (*SequentialFileReader[TReference]) Read ¶
func (r *SequentialFileReader[TReference]) Read(p []byte) (int, error)
func (*SequentialFileReader[TReference]) ReadByte ¶
func (r *SequentialFileReader[TReference]) ReadByte() (byte, error)
Source Files
¶
- captured_directory_walker.go
- create_directory_merkle_tree.go
- create_file_merkle_tree.go
- directory_access_parameters.go
- directory_cluster_object_parser.go
- directory_component_walker.go
- directory_creation_parameters.go
- directory_merkle_tree_capturer.go
- escapement_counting_scope_walker.go
- file_access_parameters.go
- file_contents_iterator.go
- file_contents_list_object_parser.go
- file_creation_parameters.go
- file_merkle_tree_capturer.go
- file_reader.go
- section_writer.go