gpfile

package
v4.1.14 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 11, 2025 License: GPL-2.0 Imports: 18 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// FileSuffix denotes the suffix used for the raw data stored
	FileSuffix = ".gpf"

	// ModeRead denotes read access
	ModeRead = os.O_RDONLY

	// ModeWrite denotes write access (no O_APPEND to allow for Seek())
	ModeWrite = os.O_CREATE | os.O_WRONLY
)
View Source 
const (

	// EpochDay is one day in seconds
	EpochDay int64 = 86400
)

Variables

View Source 
var (

	// ErrExceedsEncodingSize covers edge case scenarios where a block might (theoretically)
	// contain data that exceeds the encoding width of 32-bit
	ErrExceedsEncodingSize = errors.New("data size exceeds maximum bit width for encoding")

	// ErrInputSizeTooSmall is thrown if the input size for desiralization is too small to be valid
	ErrInputSizeTooSmall = errors.New("input size too small to be a GPDir metadata header")

	// ErrDirNotOpen denotes that a GPDir is not (yet) open or has been closed
	ErrDirNotOpen = errors.New("GPDir not open, call Open() first")

	// ErrInvalidDirName denotes that the provided name for the GPDir is invalid
	ErrInvalidDirName = errors.New("invalid GPDir path / name")
)

Functions

func DirTimestamp 

func DirTimestamp(timestamp int64) int64

DirTimestamp returns timestamp rounded down to the nearest directory time frame (usually a day)

func ExtractTimestampMetadataSuffix 

func ExtractTimestampMetadataSuffix(filename string) (timestamp int64, metadataSuffix string, err error)

SplitTimestampMetadataSuffix is a convenience function that performs timestamp (and potentially metadata prefix) extraction for the GPDir path / directory name

Types

type GPDir 

type GPDir struct {
	*Metadata
	// contains filtered or unexported fields
}

GPDir denotes a timestamped goDB directory (usually a daily set of blocks)

func NewDirReader 

func NewDirReader(basePath string, timestamp int64, metadataSuffix string, options ...Option) *GPDir

NewDir instantiates a new directory (doesn't yet do anything except for potentially reading / decoding a subset of the metadata from a provided string suffix)

func NewDirWriter 

func NewDirWriter(basePath string, timestamp int64, options ...Option) *GPDir

NewDirWriter instantiates a new directory for writing

func (*GPDir) Close 

func (d *GPDir) Close() error

Close closes all underlying open GPFiles and cleans up resources

func (*GPDir) Column 

func (d *GPDir) Column(colIdx types.ColumnIndex) (*GPFile, error)

Column returns the underlying GPFile for a specified column (lazy-access)

func (*GPDir) IsOpen 

func (d *GPDir) IsOpen() bool

IsOpen returns if the GPFile instance is currently opened

func (*GPDir) Marshal 

func (d *GPDir) Marshal(w concurrency.ReadWriteSeekCloser) error

Marshal marshals and writes the metadata of the GPDir instance into serialized metadata set

func (*GPDir) MetadataPath 

func (d *GPDir) MetadataPath() string

MetadataPath returns the full path of the GPDir metadata file

func (*GPDir) NBlocks 

func (d *GPDir) NBlocks() int

NBlocks returns the number of blocks in this GPDir

func (*GPDir) NumIPv4EntriesAtIndex 

func (d *GPDir) NumIPv4EntriesAtIndex(blockIdx int) uint64

NumIPv4EntriesAtIndex returns the number of IPv4 entries for a given block index

func (*GPDir) NumIPv6EntriesAtIndex 

func (d *GPDir) NumIPv6EntriesAtIndex(blockIdx int) uint64

NumIPv6EntriesAtIndex returns the number of IPv6 entries for a given block index

func (*GPDir) Open 

func (d *GPDir) Open(options ...Option) error

Open accesses the metadata and prepares the GPDir for reading / writing

func (*GPDir) Path 

func (d *GPDir) Path() string

Path returns the path of the GPDir (up to the timestamp and including a potential metadata suffix)

func (*GPDir) ReadBlockAtIndex 

func (d *GPDir) ReadBlockAtIndex(colIdx types.ColumnIndex, blockIdx int) ([]byte, error)

ReadBlockAtIndex returns the block for a specified block index from the underlying GPFile

func (*GPDir) SetMemPool 

func (d *GPDir) SetMemPool(pool concurrency.MemPoolGCable)

SetMemPool sets a memory pool (used to access the underlying GPFiles in full-read mode)

func (*GPDir) TimeRange 

func (d *GPDir) TimeRange() (first int64, last int64)

TimeRange returns the first and last timestamp covered by this GPDir

func (*GPDir) Unmarshal 

func (d *GPDir) Unmarshal(r concurrency.ReadWriteSeekCloser) error

Unmarshal reads and unmarshals a serialized metadata set into the GPDir instance

func (*GPDir) WriteBlocks 

func (d *GPDir) WriteBlocks(timestamp int64, blockTraffic TrafficMetadata, counters types.Counters, dbData [types.ColIdxCount][]byte) error

WriteBlocks writes a set of blocks to the underlying GPFiles and updates the metadata

type GPFile 

type GPFile struct {
	// contains filtered or unexported fields
}

GPFile implements the binary data file used to store goProbe's flows

func New 

func New(filename string, header *storage.BlockHeader, accessMode int, options ...Option) (*GPFile, error)

New returns a new GPFile object to read and write goProbe flow data

func (*GPFile) Blocks 

func (g *GPFile) Blocks() (*storage.BlockHeader, error)

Blocks return the list of available blocks (and its metadata)

func (*GPFile) Close 

func (g *GPFile) Close() error

Close closes the file

func (*GPFile) DefaultEncoder 

func (g *GPFile) DefaultEncoder() encoder.Encoder

DefaultEncoder exposes the default encoding of the GPF file

func (*GPFile) Filename 

func (g *GPFile) Filename() string

Filename exposes the location of the GPF file

func (*GPFile) RawFile 

func (g *GPFile) RawFile() concurrency.ReadWriteSeekCloser

RawFile returns the raw underlying file as a concurrency.ReadWriteSeekCloser

func (*GPFile) ReadBlock 

func (g *GPFile) ReadBlock(timestamp int64) ([]byte, error)

ReadBlock searches if a block for a given timestamp exists and returns in its data

func (*GPFile) ReadBlockAtIndex 

func (g *GPFile) ReadBlockAtIndex(idx int) ([]byte, error)

ReadBlockAtIndex returns the data of the indexed block

type Metadata 

type Metadata struct {
	BlockMetadata [types.ColIdxCount]*storage.BlockHeader
	BlockTraffic  []TrafficMetadata

	Stats
	Version uint64
}

Metadata denotes a serializable set of metadata (both globally and per-block)

func (*Metadata) MarshalString 

func (m *Metadata) MarshalString() string

MarshalString marshals (partial) metadata information into a (compressed) string representation

func (*Metadata) UnmarshalString 

func (m *Metadata) UnmarshalString(input string) error

UnmarshalString deserializes a string representation of (partial) metadata into an existing metadata structure

type Option 

type Option func(any)

Option defines optional arguments to gpfile

func WithEncoder 

func WithEncoder(e encoder.Encoder) Option

WithEncoder allows to set the compression implementation

func WithEncoderTypeLevel 

func WithEncoderTypeLevel(t encoders.Type, l int) Option

WithEncoderTypeLevel allows to set the compression type and level

func WithPermissions 

func WithPermissions(permissions fs.FileMode) Option

WithPermissions sets a non-default set of permissions / file mode for the file

func WithReadAll 

func WithReadAll(pool concurrency.MemPoolGCable) Option

WithReadAll triggers a full read of the underlying file from disk upon first read access to minimize I/O load. Seeking is handled by replacing the underlying file with a seekable in-memory structure (c.f. readWriteSeekCloser interface)

type Stats 

type Stats struct {
	Counts  types.Counters  `json:"counts"`
	Traffic TrafficMetadata `json:"traffic"`
}

Stats denotes statistics for a GPDir instance

func (Stats) Add 

func (s Stats) Add(s2 Stats) Stats

Add computes the sum of all counters and traffic metadata for the stats

func (Stats) Sub 

func (s Stats) Sub(s2 Stats) Stats

Sub computes the sum of all counters and traffic metadata for the stats

type TrafficMetadata 

type TrafficMetadata struct {
	NumV4Entries uint64 `json:"num_v4_entries"`
	NumV6Entries uint64 `json:"num_v6_entries"`
	NumDrops     uint64 `json:"num_drops"`
}

TrafficMetadata denotes a serializable set of metadata information about traffic stats

func (TrafficMetadata) Add 

func (t TrafficMetadata) Add(t2 TrafficMetadata) TrafficMetadata

Add computes the sum of two sets of TrafficMetadata

func (TrafficMetadata) NumFlows 

func (t TrafficMetadata) NumFlows() uint64

NumFlows returns the total number of flows

func (TrafficMetadata) Sub 

func (t TrafficMetadata) Sub(t2 TrafficMetadata) TrafficMetadata

Sub computes the difference of two sets of TrafficMetadata

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.