db

type ClientFS struct {
	*blockfmt.DirFS
	// URL is the URL returned by DirFS.Start.
	URL string
}

type Config struct {
	// Algo is the compression algorithm
	// used for producing output data blocks.
	// (See [blockfmt.CompressorByName].)
	// If Algo is the empty string, Config
	// uses DefaultAlgo instead.
	Algo string
	// Align is the alignment of new
	// blocks to be produced in objects
	// inserted into the index.
	Align int
	// RangeMultiple is the multiple of Align
	// at which we write out metadata.
	// If RangeMultiple is zero, it defaults to
	// DefaultRangeMultiple
	RangeMultiple int
	// MinMergeSize is the base merge
	// size of objects. If MinMergeSize is zero,
	// then DefaultMinMerge is used.
	MinMergeSize int
	// TargetMergeSize is the target size of
	// files that are compacted into larger packfiles.
	// If TargetMergeSize is zero, then DefaultTargetMergeSize is used.
	TargetMergeSize int
	// MinInputBytesPerCPU, if non-zero, determines the minimum
	// number of input bytes necessary to cause the conversion
	// process to decide to use an additional CPU core.
	// For example, if MinInputBytesPerCPU is 512kB, then 3MB of input
	// data would use 6 CPU cores (provided GOMAXPROCS is at least this high).
	// See blockfmt.MinInputBytesPerCPU
	MinInputBytesPerCPU int64
	// Force forces a full index rebuild
	// even when the input appears to be up-to-date.
	Force bool
	// Fallback determines the format for
	// objects when the object format is not
	// obvious from the file extension.
	Fallback func(name string) blockfmt.RowFormat
	// MaxScanObjects is the maximum number
	// of objects to be committed in a single Scan operation.
	// If MaxScanObjects is less than or equal to zero,
	// it is ignored and no limit is applied.
	MaxScanObjects int
	// MaxScanBytes is the maximum number
	// of bytes to ingest in a single Scan or Sync operation
	// (not including merging).
	// If MaxScanBytes is less than or equal to zero,
	// it is ignored and no limit is applied.
	MaxScanBytes int64
	// MaxScanTime is the maximum amount of time
	// to spend listing objects before deciding
	// to bail out of a scan.
	MaxScanTime time.Duration

	// NewIndexScan, if true, enables scanning
	// for newly-created index objects.
	NewIndexScan bool

	// MaxInlineBytes is the maximum number
	// of (decompressed) data bytes for which
	// we should store references directly in
	// blockfmt.Index.Inline.
	// If this value is zero, then DefaultMaxInlineBytes
	// is used instead.
	MaxInlineBytes int64
	// TargetRefSize is the target size for stored
	// indirect references. If this value is zero,
	// a reasonable default is used.
	TargetRefSize int64

	// GCMaxDelay is the longest amount of time that
	// a gc cycle will spend blocking a batch insert operation.
	GCMaxDelay time.Duration
	// GCMinimumAge is the minimum time that
	// a packed file should be left around after
	// it has been dereferenced.
	// See blockfmt.Index.ToDelete.Expiry for
	// how this value is used.
	GCMinimumAge time.Duration

	// InputMinimumAge is the mininum time
	// that an input file leaf should be left
	// around after it is no longer referenced.
	// See blockfmt.Index.ToDelete.Expiry
	InputMinimumAge time.Duration

	// Logf, if non-nil, will be where
	// the builder will log build actions
	// as it is executing. Logf must be
	// safe to call from multiple goroutines
	// simultaneously.
	Logf func(f string, args ...interface{})

	Verbose bool
}

type ContextFS interface {
	fs.FS
	// WithContext returns a copy of the file
	// system configured with the given context.
	WithContext(ctx context.Context) fs.FS
}

type Definition struct {
	// Inputs is the list of inputs that comprise the table.
	Inputs []Input `json:"input,omitempty"`
	// Partitions specifies synthetic fields that
	// are generated from components of the input
	// URI and used to partition table data.
	Partitions []Partition `json:"partitions,omitempty"`
	// Retention is the expiration policy for data.
	// Data older than the expiration window will
	// be periodically purged from the backing
	// store during table updates.
	Retention *RetentionPolicy `json:"retention_policy,omitempty"`
	// Features is a list of feature flags that
	// can be used to turn on features for beta-testing.
	Features []string `json:"beta_features,omitempty"`
	// SkipBackfill, if true, will cause this table
	// to skip scanning the source bucket(s) for matching
	// objects when the first objects are inserted into the table.
	SkipBackfill bool `json:"skip_backfill,omitempty"`
}

type DirFS struct {
	*blockfmt.DirFS
	// contains filtered or unexported fields
}

type GCConfig struct {
	// MinimumAge, if non-zero, specifies
	// the minimum age for any objects removed
	// during a garbage-collection pass.
	// Note that objects are only candidates
	// for garbage collection if they are older
	// than the current index *and* not pointed to
	// by the current index, so the MinimumAge requirement
	// is only necessary if it is possible for GC and ingest
	// to run simultaneously. In that case, MinimumAge should be
	// set to some duration longer than any possible ingest cycle.
	MinimumAge      time.Duration
	InputMinimumAge time.Duration

	// MaxDelay is the maximum amount of time
	// that a GC will spend blocking batch inserts.
	// If MaxDelay is less than or equal to zero,
	// then the amount of time spent GC'ing is unlimited.
	MaxDelay time.Duration

	// Logf, if non-nil, is a callback used for logging
	// detailed information regarding GC decisions.
	Logf func(f string, args ...interface{})

	// Precise determines if GC is performed
	// by only deleting objects that have been
	// explicitly marked for deletion.
	Precise bool
}

type Input struct {
	// Pattern is the glob pattern that
	// specifies which files are fed into
	// the table. Patterns should be URIs
	// where the URI scheme (i.e. s3://, file://, etc.)
	// indicates where the data ought to come from.
	Pattern string `json:"pattern"`
	// Format is the format of the files in pattern.
	// If Format is the empty string, then the format
	// will be inferred from the file extension.
	Format string `json:"format,omitempty"`
	// Hints, if non-nil, is the hints associated
	// with the input data. The hints may perform
	// type-based coercion of certain paths, and may additionally
	// eliminate some of the data as it is parsed.
	// Hints data is format-specific.
	Hints json.RawMessage `json:"hints,omitempty"`
}

type InputFS interface {
	blockfmt.InputFS
}

type OutputFS interface {
	blockfmt.UploadFS
}

type Partition struct {
	// Field is the name of the partition field. If
	// this field conflicts with a field in the
	// input data, the partition field will
	// override it.
	Field string `json:"field"`
	// Type is the type of the partition field.
	// If this is "", this defaults to "string".
	Type string `json:"type,omitempty"`
	// Value is a template string that is used to
	// produce the value for the partition field.
	// The template may reference parts of the
	// input URI specified in the input pattern.
	// If this is "", the field name is used to
	// determine the input URI part that will be
	// used to determine the value.
	Value string `json:"value,omitempty"`
}

type Queue interface {
	// Close should be called when the
	// runner has finished processing items
	// from the queue (usually due to receiving
	// and external signal to stop processing events).
	// Close should only be called after all events
	// returned by Next have been processed via calls
	// to Finalize.
	io.Closer
	// Next should return the next item
	// in the queue. If the provided pause
	// duration is non-negative, then Next
	// should block for up to the provided duration
	// to produce a new value. If pause is negative,
	// then Next should block until it can return
	// a non-nil QueueItem value or an EOF error.
	// Next should return (nil, io.EOF) is the queue
	// has been closed and no further processing should
	// be performed.
	//
	// As an optimization, the returned QueueItem
	// can implement fs.File, which will obviate
	// the need for the caller to perform additional
	// I/O to produce a file handle associated with
	// the QueueItem.
	Next(pause time.Duration) (QueueItem, error)
	// Finalize is called to return the final status
	// of a QueueItem that was previously returned by
	// ReadInputs. If status is anything other than
	// StatusOK, then the Queue should arrange for the
	// item to be read by a future call to ReadInputs.
	//
	// Finalize may panic if Queue.Close has been called.
	Finalize(item QueueItem, status QueueStatus)
}

type QueueItem interface {
	// Path should return the full path
	// of the file, including the fs prefix.
	Path() string
	// ETag should return the ETag of
	// the file.
	ETag() string
	// Size should return the size of
	// the file at Path in bytes.
	Size() int64
	// EventTime should return the time
	// at which the queue item was inserted
	// into the queue. EventTime is used to
	// compute statistics about total queue delays.
	EventTime() time.Time
}

type QueueRunner struct {
	Owner Tenant
	// Conf is the configuration
	// used for building tables.
	Conf Config

	// Logf is used to log errors encountered
	// while processing entries from a queue.
	// Logf may be nil.
	Logf func(f string, args ...interface{})

	// Open is a hook that can be used to override
	// how queue items are opened.
	// The default behavior is to use ifs.Open(item.Path())
	Open func(ifs InputFS, item QueueItem) (fs.File, error)

	// TableRefresh is the interval at which
	// table definitions are refreshed.
	// If TableRefresh is less than or equal
	// to zero, then tables are refreshed every minute.
	TableRefresh time.Duration

	// BatchSize is the maximum number of bytes
	// that the QueueRunner will attempt to read
	// from a Queue in Run before comitting the
	// returned items. Batches may be smaller than
	// BatchSize due to the expiration of BatchInterval
	// or due to receiving an error from the queue
	// after batching a non-zero number of items.
	// (The size of a batch is computed by summing
	// the QueueItem.Size values from each QueueItem
	// returned from Next.)
	//
	// If BatchSize is less than or equal to zero,
	// then DefaultBatchSize is used instead.
	//
	// See also: BatchInterval
	BatchSize int64

	// BatchInterval is the maximum amount of
	// time the queue should wait for successive
	// calls to Queue.Next to accumulate BatchSize items.
	//
	// See also: BatchSize
	BatchInterval time.Duration

	// IOErrDelay determines how long queue processing
	// will pause if it encounters an I/O error from
	// the backing filesystem.
	IOErrDelay time.Duration
}

type QueueStatus int32

type RemoveFS interface {
	fs.FS
	Remove(name string) error
}

type RetentionPolicy struct {
	// Field is the path expression for the field
	// used to determine the age of a record for
	// the purpose of the data retention policy.
	//
	// Currently only timestamp fields are
	// supported.
	Field string `json:"field,omitempty"`
	// ValidFor is the validity window relative to
	// now.
	//
	// This is a string with a format like
	// "<n>y<n>m<n>d" where "<n>" is a number and
	// any component can be omitted.
	//
	// For example: "6m", "1000d", "1y6m15d"
	ValidFor date.Duration `json:"valid_for"`
}

type S3FS struct {
	blockfmt.S3FS
}

type S3Resolver struct {
	// DeriveKey is the callback used to
	// derive a key for a particular bucket.
	DeriveKey func(bucket string) (*aws.SigningKey, error)
	// Client, if non-nil, sets the default
	// client used by returned s3.BucketFS objects.
	Client *http.Client
	Ctx    context.Context
}

type Tenant interface {
	// ID should return the unique ID of the tenant.
	ID() string

	// Key should return the key used
	// for verifying the integrity of database objects.
	Key() *blockfmt.Key

	// Root should return the root of the
	// storage for database objects.
	// The returned FS should implement
	// UploadFS if it supports writing.
	Root() (InputFS, error)

	// Split should trim the prefix off of pattern
	// that specifies the source filesystem and return
	// the result as an InputFS and the trailing glob
	// pattern that can be applied to the input to yield
	// the results.
	Split(pattern string) (InputFS, string, error)
}

type TenantConfig struct {
	// MaxScanBytes is the maximum number of bytes
	// allowed to be scanned for each query. If
	// this is 0, there is no limit.
	MaxScanBytes uint64
}

type TenantConfigurable interface {
	Tenant

	// Config returns the configuration options
	// for this tenant. This may return nil to
	// indicate all defaults should be used.
	Config() *TenantConfig
}