Documentation
¶
Overview ¶
Package model gathers the main structs and interfaces used in sync library.
Index ¶
- Constants
- func Ignores(endpoint Endpoint, name string) bool
- func IsConfigError(e error) bool
- func IsFolderHiddenFile(name string) bool
- func IsIgnoredFile(path string, ignores ...glob.Glob) (ignored bool)
- func NewConfigError(err error) error
- func NodeRequiresChecksum(node *tree.Node) bool
- func Retry(f func() error, seconds ...time.Duration) error
- func RetryWithCtx(ctx context.Context, f func(retry int) error, seconds ...time.Duration) error
- func StringContentToETag(uuid string) string
- func ZapEndpoint(key string, e Endpoint) zapcore.Field
- type AuthRequester
- type BulkLoader
- type CachedBranchProvider
- type ChecksumProvider
- type Command
- type ConfigError
- type Conflicter
- type DataSyncSource
- type DataSyncTarget
- type DirectionType
- type Endpoint
- type EndpointInfo
- type EndpointOptions
- type EndpointRootStat
- type EndpointStatus
- type ErrorType
- type EventInfo
- type EventType
- type HashStoreReader
- type LockBranchProvider
- type LockEvent
- type LockEventType
- type MetadataProvider
- type MetadataReceiver
- type MultiStater
- type Networking
- type PathSyncSource
- type PathSyncTarget
- type ProcessingStatus
- func (p *ProcessingStatus) AtomicProgress() bool
- func (p *ProcessingStatus) EndpointURI() string
- func (p *ProcessingStatus) Error() error
- func (p *ProcessingStatus) IsError() bool
- func (p *ProcessingStatus) MarshalJSON() ([]byte, error)
- func (p *ProcessingStatus) Node() *tree.Node
- func (p *ProcessingStatus) Progress() float32
- func (p *ProcessingStatus) SetEndpoint(uri string) *ProcessingStatus
- func (p *ProcessingStatus) SetError(e error) *ProcessingStatus
- func (p *ProcessingStatus) SetNode(node *tree.Node) *ProcessingStatus
- func (p *ProcessingStatus) SetProgress(pg float32, atomic ...bool) *ProcessingStatus
- func (p *ProcessingStatus) Status() int
- func (p *ProcessingStatus) String() string
- func (p *ProcessingStatus) Type() StatusScope
- func (p *ProcessingStatus) UnmarshalJSON(data []byte) error
- type ProcessorEvent
- type SessionProvider
- type SnapshotFactory
- type SnapshotUpdater
- type Snapshoter
- type Stater
- type Status
- type StatusProvider
- type StatusScope
- type SyncCmd
- type TaskStatus
- type UuidFoldersRefresher
- type UuidProvider
- type UuidReceiver
- type Versioner
- type WalkNodesFunc
- type WatchConnectionInfo
- type WatchObject
- func (w *WatchObject) Close()
- func (w *WatchObject) ConnectionInfos() chan WatchConnectionInfo
- func (w WatchObject) Done() <-chan bool
- func (w *WatchObject) Errors() chan error
- func (w *WatchObject) Events() chan EventInfo
- func (w WatchObject) NextError() error
- func (w WatchObject) NextEvent() EventInfo
Constants ¶
const ( // Use unique path separator everywhere InternalPathSeparator = "/" GlobSeparator = '/' // Minio default Etag when a new file is detected DefaultEtag = "00000000000000000000000000000000-1" DirectionLeft DirectionType = 1 DirectionRight DirectionType = 2 DirectionBi DirectionType = 3 Interrupt SyncCmd = iota Pause Resume )
const ( StatusScopeTask = iota StatusScopeProcessing )
Variables ¶
This section is empty.
Functions ¶
func IsConfigError ¶
func IsFolderHiddenFile ¶
IsFolderHiddenFile checks if file is .pydio
func NewConfigError ¶
func NodeRequiresChecksum ¶
func RetryWithCtx ¶
RetryWithCtx does like Retry with an additionnal cancellable context
func StringContentToETag ¶
Types ¶
type AuthRequester ¶
type AuthRequester interface {
RequiresAuth() bool
}
type BulkLoader ¶
type BulkLoader interface {
BulkLoadNodes(ctx context.Context, nodes map[string]string) (map[string]interface{}, error)
}
A BulkLoader can stream calls to ReadNode - Better use CachedBranchProvider
type CachedBranchProvider ¶
type CachedBranchProvider interface {
Endpoint
GetCachedBranches(ctx context.Context, roots ...string) PathSyncSource
}
A CachedBranchProvider can quickly load a full branch recursively in memory and expose it as a PathSyncSource
type ChecksumProvider ¶
ChecksumProvider is able to compute a checksum for a given node (typically an Etag)
type ConfigError ¶
type ConfigError struct {
// contains filtered or unexported fields
}
func (*ConfigError) HasConfigError ¶
func (c *ConfigError) HasConfigError() bool
type Conflicter ¶
type Conflicter interface {
Conflict() (bool, interface{})
}
type DataSyncSource ¶
type DataSyncSource interface {
PathSyncSource
// GetReaderOn provides a ReadCloser for reading content of a node located at a given path
GetReaderOn(path string) (out io.ReadCloser, err error)
}
DataSyncSource provides a way to read the actual content of the nodes
func AsDataSyncSource ¶
func AsDataSyncSource(endpoint Endpoint) (DataSyncSource, bool)
AsDataSyncSource tries to cast an Endpoint to a DataSyncSource
type DataSyncTarget ¶
type DataSyncTarget interface {
PathSyncTarget
// GetWriterOn provides a writeCloser for writing content to a given path.
GetWriterOn(cancel context.Context, path string, targetSize int64) (out io.WriteCloser, writeDone chan bool, writeErr chan error, err error)
}
DataSyncTarget provides a way to write some actual content to the nodes
func AsDataSyncTarget ¶
func AsDataSyncTarget(endpoint Endpoint) (DataSyncTarget, bool)
AsDataSyncTarget tries to cast an Endpoint to a DataSyncTarget
type DirectionType ¶
type DirectionType int
type Endpoint ¶
type Endpoint interface {
// LoadNode loads a given node by its path from this endpoint
LoadNode(ctx context.Context, path string, extendedStats ...bool) (node *tree.Node, err error)
// GetEndpointInfo returns static informations about this endpoint
GetEndpointInfo() EndpointInfo
}
Endpoint is the most basic interface for representing an endpoint for synchronization. It is just able to return some info and to load a node
type EndpointInfo ¶
type EndpointInfo struct {
URI string
RequiresNormalization bool
RequiresFoldersRescan bool
IsAsynchronous bool
EchoTime time.Duration
Ignores []string
}
EndpointInfo provides static info about a given Endpoint (returned by GetEndpointInfo method)
type EndpointOptions ¶
EndpointOptions is used to configure an Endpoint at creation time
type EndpointRootStat ¶
type EndpointRootStat struct {
HasChildrenInfo bool
HasSizeInfo bool
Size int64
Folders int64
Files int64
PgSize int64 `json:"-"`
PgChildren int64 `json:"-"`
PgFolders int64 `json:"-"`
PgFiles int64 `json:"-"`
LastPg float64 `json:"-"`
}
EndpointRootStat gives information about the size/files/folders of an endpoint
func (*EndpointRootStat) Children ¶
func (e *EndpointRootStat) Children() int64
Children returns the sum of Folders and Files
func (*EndpointRootStat) IsEmpty ¶
func (e *EndpointRootStat) IsEmpty() bool
IsEmpty checks if the given stat seems empty. Call to IsKnown should be done before otherwise it can be a false negative
func (*EndpointRootStat) IsKnown ¶
func (e *EndpointRootStat) IsKnown() bool
IsKnown returns true if either HasSizeInfo or HasChildrenInfo is set
type EndpointStatus ¶
type EndpointStatus struct {
EndpointInfo
WatchConnection WatchConnectionInfo
Stats *EndpointRootStat
}
type EventInfo ¶
type EventInfo struct {
Time string
Size int64
Etag string
Folder bool
Path string
Source PathSyncSource `json:"-"`
Type EventType
Host string
Port string
UserAgent string
OperationId string
ScanEvent bool
ScanSourceNode *tree.Node
Metadata map[string]string
MoveSource *tree.Node
MoveTarget *tree.Node
}
EventInfo contains the information of the event that occurred and the source IP:PORT of the client which triggerred the event.
func NodeToEventInfo ¶
type EventType ¶
type EventType string
EventType represents the type of the event occurred.
const ( // EventCreate notifies when a new object is created EventCreate EventType = "ObjectCreated" // EventRename notifies when a new object is renamed EventRename EventType = "ObjectRenamed" // EventRemove notifies when a new object is deleted EventRemove EventType = "ObjectRemoved" // EventAccessed notifies when an object is accessed. EventAccessed EventType = "ObjectAccessed" // EventAccessedRead notifies when an object is accessed (specifically read). EventAccessedRead EventType = "ObjectAccessed:Read" // EventAccessedStat notifies when an object is accessed (specifically stat). EventAccessedStat EventType = "ObjectAccessed:Stat" // EventOther notifies any other events EventOther EventType = "ObjectOther" // EventOther notifies any other events EventSureMove EventType = "NodeMoved" )
type HashStoreReader ¶
type HashStoreReader interface {
// SetRefHashStore passes a reference to a loaded snapshot
SetRefHashStore(source PathSyncSource)
}
HashStoreReader can maintain a reference to a snapshot to quickly find hashes for nodes directly from the snapshot if they have not been modified
type LockBranchProvider ¶
type LockBranchProvider interface {
// Lock set a lock on a branch, with a preset sessionUUID and an expiration time
LockBranch(ctx context.Context, node *tree.Node, sessionUUID string, expireAfter time.Duration) error
// Unlock removes lock manually from this branch, if it was not expired already
UnlockBranch(ctx context.Context, sessionUUID string) error
}
LockBranchProvider can set/remove a lock on a branch, with automatic expiration
type LockEvent ¶
type LockEvent struct {
Type LockEventType
Source PathSyncSource
Path string
OperationId string
}
LockEvent is a simple struct for signaling lock/unlock operation
type LockEventType ¶
type LockEventType int
const ( LockEventLock LockEventType = iota LockEventUnlock )
type MetadataProvider ¶
type MetadataProvider interface {
// ProvidesMetadataNamespaces returns a list of patterns to check on provider nodes MetaStore.
ProvidesMetadataNamespaces() ([]glob.Glob, bool)
}
MetadataProvider declares metadata namespaces that may be mapped to target metadata
type MetadataReceiver ¶
type MetadataReceiver interface {
// CreateMetadata add a metadata to the node
CreateMetadata(ctx context.Context, node *tree.Node, namespace string, jsonValue string) error
// UpdateMetadata updates an existing metadata value
UpdateMetadata(ctx context.Context, node *tree.Node, namespace string, jsonValue string) error
// DeleteMetadata deletes a metadata by namespace
DeleteMetadata(ctx context.Context, node *tree.Node, namespace string) error
}
MetadataReceiver implements methods for updating nodes metadata
type MultiStater ¶
func NewMultiStater ¶
func NewMultiStater() MultiStater
func (MultiStater) Stats ¶
func (m MultiStater) Stats() map[string]interface{}
func (MultiStater) String ¶
func (m MultiStater) String() string
type Networking ¶
type Networking interface {
NetworkError() error
}
type PathSyncSource ¶
type PathSyncSource interface {
Endpoint
// Walk walks the nodes with a callback
Walk(walknFc WalkNodesFunc, root string, recursive bool) (err error)
// Watch setup an event watcher on the nodes
Watch(recursivePath string) (*WatchObject, error)
}
PathSyncSource is a type of endpoint that can be used as a source of tree.Nodes for synchronization. It can browse and watch the nodes, but not get the nodes actual content (see DataSyncSource).
func AsPathSyncSource ¶
func AsPathSyncSource(endpoint Endpoint) (PathSyncSource, bool)
AsPathSyncSource tries to cast an Endpoint to a PathSyncSource
type PathSyncTarget ¶
type PathSyncTarget interface {
Endpoint
// CreateNode is used to create a node in the tree
CreateNode(ctx context.Context, node *tree.Node, updateIfExists bool) (err error)
// DeleteNode is used to remove a node (and all its children) from the tree
DeleteNode(ctx context.Context, path string) (err error)
// MoveNode is used to move a node (and all its children) from one place to another in the tree.
MoveNode(ctx context.Context, oldPath string, newPath string) (err error)
}
PathSyncTarget is a type of endpoint that can be used as a target for synchronization, typically an Index. It can be updated with a tree of nodes, but cannot store actual data (see DataSyncTarget)
func AsPathSyncTarget ¶
func AsPathSyncTarget(endpoint Endpoint) (PathSyncTarget, bool)
AsPathSyncTarget tries to cast an Endpoint to a PathSyncTarget
type ProcessingStatus ¶
type ProcessingStatus struct {
// contains filtered or unexported fields
}
func NewProcessingStatus ¶
func NewProcessingStatus(info string) *ProcessingStatus
func (*ProcessingStatus) AtomicProgress ¶
func (p *ProcessingStatus) AtomicProgress() bool
func (*ProcessingStatus) EndpointURI ¶
func (p *ProcessingStatus) EndpointURI() string
func (*ProcessingStatus) Error ¶
func (p *ProcessingStatus) Error() error
func (*ProcessingStatus) IsError ¶
func (p *ProcessingStatus) IsError() bool
func (*ProcessingStatus) MarshalJSON ¶
func (p *ProcessingStatus) MarshalJSON() ([]byte, error)
MarshalJSON implements custom JSON marshalling
func (*ProcessingStatus) Node ¶
func (p *ProcessingStatus) Node() *tree.Node
func (*ProcessingStatus) Progress ¶
func (p *ProcessingStatus) Progress() float32
func (*ProcessingStatus) SetEndpoint ¶
func (p *ProcessingStatus) SetEndpoint(uri string) *ProcessingStatus
func (*ProcessingStatus) SetError ¶
func (p *ProcessingStatus) SetError(e error) *ProcessingStatus
func (*ProcessingStatus) SetNode ¶
func (p *ProcessingStatus) SetNode(node *tree.Node) *ProcessingStatus
func (*ProcessingStatus) SetProgress ¶
func (p *ProcessingStatus) SetProgress(pg float32, atomic ...bool) *ProcessingStatus
func (*ProcessingStatus) Status ¶
func (p *ProcessingStatus) Status() int
func (*ProcessingStatus) String ¶
func (p *ProcessingStatus) String() string
func (*ProcessingStatus) Type ¶
func (p *ProcessingStatus) Type() StatusScope
func (*ProcessingStatus) UnmarshalJSON ¶
func (p *ProcessingStatus) UnmarshalJSON(data []byte) error
UnmarshalJSON implements custom JSON unmarshalling
type ProcessorEvent ¶
type ProcessorEvent struct {
Type string
Data interface{}
}
type SessionProvider ¶
type SessionProvider interface {
// StartSession opens a new indexation session and returns it
StartSession(ctx context.Context, rootNode *tree.Node, silent bool) (*tree.IndexationSession, error)
// FlushSession calls the Flush method on the underlying service without closing the session yet
FlushSession(ctx context.Context, sessionUuid string) error
// FinishSession closes the indexation session
FinishSession(ctx context.Context, sessionUuid string) error
}
SessionProvider has internal mechanism to start/flush/finish an IndexationSession
func AsSessionProvider ¶
func AsSessionProvider(endpoint Endpoint) (SessionProvider, bool)
AsSessionProvider tries to cast an Endpoint to a SessionProvider
type SnapshotFactory ¶
type SnapshotFactory interface {
// Load creates the snapshot and return it
Load(source PathSyncSource) (Snapshoter, error)
Close(ctx context.Context) error
Reset(ctx context.Context) error
}
SnapshotFactory provides dependency injection for creating snapshots using a specific persistence layer.
type SnapshotUpdater ¶
type SnapshotUpdater interface {
// SetUpdateSnapshot stores internal reference to a Snapshot
SetUpdateSnapshot(PathSyncTarget)
// PatchUpdateSnaptshot applies a patch of operations to the internal snapshot
PatchUpdateSnapshot(ctx context.Context, patch interface{})
}
SnapshotUpdater is an endpoint that can embed a reference to a snapshot and update it afterward
type Snapshoter ¶
type Snapshoter interface {
PathSyncSource
// IsEmpty indicates whether this snapshot is properly initialized or not (e.g. underlying DB not found)
IsEmpty() bool
// Capture walks the source and stores all paths inside snapshot. The paths parameter allows to filter the branches to capture.
Capture(ctx context.Context, source PathSyncSource, paths ...string) error
}
Snapshoter is an extended version of PathSyncSource that can capture another source at once.
type StatusProvider ¶
type StatusProvider interface {
// SetupChannels register channels for listening to status and done infos
SetupChannels(status chan Status, done chan interface{}, cmd *Command)
// Status notify of a new ProcessStatus
Status(s Status)
// Done notify the patch is processed, can send any useful info to the associated channel
Done(info interface{})
}
StatusProvider can register channels to send status/done events during processing
type StatusScope ¶
type StatusScope int
type TaskStatus ¶
type TaskStatus int
const ( TaskStatusIdle TaskStatus = iota TaskStatusPaused TaskStatusDisabled TaskStatusProcessing TaskStatusError TaskStatusRestarting TaskStatusStopping TaskStatusRemoved )
type UuidFoldersRefresher ¶
type UuidFoldersRefresher interface {
// ExistingFolders lists all folders with their UUID
ExistingFolders(ctx context.Context) (map[string][]*tree.Node, error)
// UpdateFolderUuid refreshes a given folder UUID and return it.
UpdateFolderUuid(ctx context.Context, node *tree.Node) (*tree.Node, error)
}
UuidFoldersRefresher provides tools to detect UUID duplicates and update them if necessary
type UuidProvider ¶
type UuidProvider interface {
// LoadNodeByUuid loads a node by UUID.
LoadNodeByUuid(ctx context.Context, uuid string) (node *tree.Node, err error)
}
UuidProvider declares an endpoint to be able to load a node by its unique UUID
type UuidReceiver ¶
type UuidReceiver interface {
// UpdateNodeUuid refresh node UUID and returns the new node
UpdateNodeUuid(ctx context.Context, node *tree.Node) (*tree.Node, error)
}
UuidReceiver is able to update an existing node UUID
type WatchConnectionInfo ¶
type WatchConnectionInfo int
const ( WatchConnected WatchConnectionInfo = iota WatchDisconnected WatchActive WatchIdle WatchStats )
type WatchObject ¶
type WatchObject struct {
// eventInfo will be put on this chan
EventInfoChan chan EventInfo
// errors will be put on this chan
ErrorChan chan error
// will stop the watcher goroutines
DoneChan chan bool
// Provides info about the internal watcher connection
// Can be nill if the internal watcher does not support such notifications
ConnectionInfo chan WatchConnectionInfo
}
func (*WatchObject) Close ¶
func (w *WatchObject) Close()
Close the watcher, will stop all goroutines
func (*WatchObject) ConnectionInfos ¶
func (w *WatchObject) ConnectionInfos() chan WatchConnectionInfo
func (WatchObject) Done ¶
func (w WatchObject) Done() <-chan bool
Done returns a channel that unblocks when Close has been called
func (*WatchObject) Errors ¶
func (w *WatchObject) Errors() chan error
Errors returns the chan receiving errors
func (*WatchObject) Events ¶
func (w *WatchObject) Events() chan EventInfo
Events returns the chan receiving events
func (WatchObject) NextError ¶
func (w WatchObject) NextError() error
NextError pops the next error from the ErrorChan
func (WatchObject) NextEvent ¶
func (w WatchObject) NextEvent() EventInfo
NextEvent pops the next event from the EventInfoChan