metrics

package
v0.3.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 25, 2025 License: Apache-2.0 Imports: 7 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

Functions

func ApplyMonitorDefArgs

func ApplyMonitorDefArgs(
	qb *querybuilder.QueryBuilder,
	args *MonitorArgs,
	partitioning *MonitorPartitioning,
	segmentLengthLimit int64,
) *querybuilder.QueryBuilder

func ConvertToMetricType

func ConvertToMetricType[T MetricType](val interface{}) (T, error)

func CountStar added in v0.1.9

func CountStar(metricId MetricId) NumericExpr

func CustomNumericMetricsCols added in v0.1.4

func CustomNumericMetricsCols(sql NumericExpr, metricId MetricId) []Expr

func GetMetricValueFromNumber

func GetMetricValueFromNumber[T MetricType, V int64 | float64](
	metricId MetricId,
	value V,
) (T, error)

func HasMetricValWithoutPrediction

func HasMetricValWithoutPrediction[T MetricType](metricVals []*MetricVal[T]) bool

func IsTimeType

func IsTimeType[T any]() bool

func MetricIdDescription

func MetricIdDescription(id MetricId) string

func NumericMetricsCols

func NumericMetricsCols(field string, opts ...MetricConfOption) []Expr

func NumericMetricsValuesCols added in v0.1.9

func NumericMetricsValuesCols(field string, opts ...MetricConfOption) []Expr

func ProfileColumns added in v0.1.9

func ProfileColumns(
	tableFqn *TableFqnExpr,
	columnsToProfile []*ColumnToProfile,
	args *MonitorArgs,
	partition *Partition,
	limit int64,
	segmentLengthLimit int64,
) (*querybuilder.QueryBuilder, error)

func SegmentsListQuery added in v0.1.4

func SegmentsListQuery(
	tableFqn *TableFqnExpr,
	args *MonitorArgs,
	partition *Partition,
	rowsLimit int64,
	segmentLengthLimit int64,
) (*querybuilder.QueryBuilder, error)

func TableLastLoadedAtMetricsCols

func TableLastLoadedAtMetricsCols(timeCol TimeExpr) []Expr

func TableVolumeMetricsCols

func TableVolumeMetricsCols() []Expr

func TextMetricsCols

func TextMetricsCols(field string, opts ...MetricConfOption) []Expr

func TextMetricsLengthCols added in v0.1.9

func TextMetricsLengthCols(field string, opts ...MetricConfOption) []Expr

func TextMetricsValuesCols added in v0.1.9

func TextMetricsValuesCols(field string, opts ...MetricConfOption) []Expr

func TimeMetricsCols

func TimeMetricsCols(field string, opts ...MetricConfOption) []Expr

func TimeMetricsValuesCols added in v0.1.9

func TimeMetricsValuesCols(field string, opts ...MetricConfOption) []Expr

func UnknownMetricsValuesCols added in v0.2.6

func UnknownMetricsValuesCols(field string, opts ...MetricConfOption) []Expr

Types

type ColumnProfile added in v0.1.9

type ColumnProfile int
const (
	ColumnProfileUnknown ColumnProfile = iota
	ColumnProfileString  ColumnProfile = iota
	ColumnProfileNumeric ColumnProfile = iota
	ColumnProfileTime    ColumnProfile = iota
)

type ColumnToProfile added in v0.1.9

type ColumnToProfile struct {
	Column        string
	ColumnProfile ColumnProfile
}

type CountExpr added in v0.1.9

type CountExpr struct {
	MetricConf
	Expresion Expr
	MetricId  MetricId
}

func (CountExpr) IsNumericExpr added in v0.1.9

func (m CountExpr) IsNumericExpr()

func (CountExpr) OutColumnAlias added in v0.1.9

func (m CountExpr) OutColumnAlias() TextExpr

func (CountExpr) ToSql added in v0.1.9

func (m CountExpr) ToSql(dialect Dialect) (string, error)

type CustomNumericMetricExpr added in v0.1.4

type CustomNumericMetricExpr struct {
	MetricConf
	MetricId MetricId
	Sql      NumericExpr
}

func CustomNumericMetric added in v0.1.4

func CustomNumericMetric(sql NumericExpr, metricId MetricId) *CustomNumericMetricExpr

func (*CustomNumericMetricExpr) OutColumnAlias added in v0.1.4

func (m *CustomNumericMetricExpr) OutColumnAlias() TextExpr

func (*CustomNumericMetricExpr) ToSql added in v0.1.4

func (m *CustomNumericMetricExpr) ToSql(dialect Dialect) (string, error)

type FieldMetrics

type FieldMetrics struct {
	Field   string
	Metrics []MetricId
}

type MetricConf added in v0.1.9

type MetricConf struct {
	AliasPrefix string
}

func DefaultMetricConf added in v0.1.9

func DefaultMetricConf() *MetricConf

func (*MetricConf) PrefixedAliasForMetric added in v0.1.9

func (c *MetricConf) PrefixedAliasForMetric(metricId MetricId) TextExpr

type MetricConfOption added in v0.1.9

type MetricConfOption func(*MetricConf)

func WithPrefixForColumn added in v0.1.9

func WithPrefixForColumn(prefix string) MetricConfOption

type MetricCustomNumeric

type MetricCustomNumeric struct {
	Segment     string    `ch:"segment"        bigquery:"segment"        db:"segment"        json:"segment"`
	TimeSegment time.Time `ch:"time_segment"   bigquery:"time_segment"   db:"time_segment"   json:"time_segment"`
	Numeral     float64   `ch:"custom_numeric" bigquery:"custom_numeric" db:"custom_numeric" json:"custom_numeric"`
}

func (*MetricCustomNumeric) GetIdentity

func (volume *MetricCustomNumeric) GetIdentity() MetricIdentity

func (*MetricCustomNumeric) ToDefault

func (volume *MetricCustomNumeric) ToDefault(timeSegment time.Time, segment string)

func (*MetricCustomNumeric) WithPartition

func (volume *MetricCustomNumeric) WithPartition(timeSegment time.Time, segment string)

type MetricFieldDistribution

type MetricFieldDistribution struct {
	Field       string    `ch:"field"        bigquery:"field"        db:"field"        json:"field"`
	Segment     string    `ch:"segment"      bigquery:"segment"      db:"segment"      json:"segment"`
	TimeSegment time.Time `ch:"time_segment" bigquery:"time_segment" db:"time_segment" json:"time_segment"`

	Counts []int64  `ch:"counts" bigquery:"counts" db:"counts" json:"counts"`
	Labels []string `ch:"labels" bigquery:"labels" db:"labels" json:"labels"`
}

func (*MetricFieldDistribution) GetIdentity

func (metric *MetricFieldDistribution) GetIdentity() MetricIdentity

func (*MetricFieldDistribution) ToDefault

func (metric *MetricFieldDistribution) ToDefault(timeSegment time.Time, segment string)

type MetricFreshness

type MetricFreshness struct {
	Segment     string    `ch:"segment"      bigquery:"segment"      db:"segment"      json:"segment"`
	TimeSegment time.Time `ch:"time_segment" bigquery:"time_segment" db:"time_segment" json:"time_segment"`
	At          time.Time `ch:"at"           bigquery:"at"           db:"at"           json:"at"`
}

func NewMetricFreshness

func NewMetricFreshness(at time.Time) *MetricFreshness

func NewMetricFreshnessWithSegments

func NewMetricFreshnessWithSegments(at time.Time, timeSegment time.Time, segment string) *MetricFreshness

func (*MetricFreshness) GetIdentity

func (metric *MetricFreshness) GetIdentity() MetricIdentity

func (*MetricFreshness) ToDefault

func (metric *MetricFreshness) ToDefault(timeSegment time.Time, segment string)

type MetricId

type MetricId string
const (
	METRIC_FRESHNESS           MetricId = "updated_at"
	METRIC_VOLUME              MetricId = "row_count"
	METRIC_ROW_GROWTH          MetricId = "row_growth"
	METRIC_LAST_LOADED_AT      MetricId = "last_loaded_at"
	METRIC_NUM_ROWS            MetricId = "num_rows"
	METRIC_CUSTOM_NUMERIC      MetricId = "custom_numeric"
	METRIC_NUM_UNIQUE          MetricId = "num_unique"
	METRIC_NUM_NOT_NULL        MetricId = "num_not_null"
	METRIC_NUM_NULL            MetricId = "num_null"
	METRIC_NUM_EMPTY           MetricId = "num_empty"
	METRIC_MEAN                MetricId = "mean"
	METRIC_MIN                 MetricId = "min"
	METRIC_MAX                 MetricId = "max"
	METRIC_MEDIAN              MetricId = "median"
	METRIC_STDDEV              MetricId = "stddev"
	METRIC_DELAY               MetricId = "delay"
	METRIC_VOLUME_CHANGE_DELAY MetricId = "volume_change_delay"
	METRIC_SIZE_BYTES          MetricId = "size_bytes"
	METRIC_PCT_UNIQUE          MetricId = "pct_unique"
	METRIC_PCT_NULL            MetricId = "pct_null"
	METRIC_PCT_EMPTY           MetricId = "pct_empty"
	METRIC_MIN_LENGTH          MetricId = "min_length"
	METRIC_MAX_LENGTH          MetricId = "max_length"
	METRIC_MEAN_LENGTH         MetricId = "mean_length"
)

func (*MetricId) Scan

func (m *MetricId) Scan(value interface{}) error

Implement the sql.Scanner interface

type MetricIdentity

type MetricIdentity struct {
	Time    time.Time
	Segment string
}

func NewMetricIdentity

func NewMetricIdentity(time time.Time, segment string) MetricIdentity

func (MetricIdentity) ToKey

func (identity MetricIdentity) ToKey() string

type MetricLastLoadedAt

type MetricLastLoadedAt struct {
	Segment      string    `ch:"segment"        bigquery:"segment"        db:"segment"        json:"segment"`
	TimeSegment  time.Time `ch:"time_segment"   bigquery:"time_segment"   db:"time_segment"   json:"time_segment"`
	LastLoadedAt time.Time `ch:"last_loaded_at" bigquery:"last_loaded_at" db:"last_loaded_at" json:"last_loaded_at"`
}

func NewMetricLastLoadedAt

func NewMetricLastLoadedAt(lastLoadedAt time.Time) *MetricLastLoadedAt

func NewMetricLastLoadedAtWithSegments

func NewMetricLastLoadedAtWithSegments(updatedAt time.Time, timeSegment time.Time, segment string) *MetricLastLoadedAt

func (*MetricLastLoadedAt) GetIdentity

func (metric *MetricLastLoadedAt) GetIdentity() MetricIdentity

func (*MetricLastLoadedAt) ToDefault

func (metric *MetricLastLoadedAt) ToDefault(timeSegment time.Time, segment string)

func (*MetricLastLoadedAt) WithPartition

func (metric *MetricLastLoadedAt) WithPartition(timeSegment time.Time, segment string)

type MetricNumericFieldStats

type MetricNumericFieldStats struct {
	Field       string    `ch:"field"        bigquery:"field"        db:"field"        json:"field"`
	Segment     string    `ch:"segment"      bigquery:"segment"      db:"segment"      json:"segment"`
	TimeSegment time.Time `ch:"time_segment" bigquery:"time_segment" db:"time_segment" json:"time_segment"`

	NumTotal   int64 `ch:"num_rows"     bigquery:"num_rows"     db:"num_rows"     json:"num_rows"`
	NumUnique  int64 `ch:"num_unique"   bigquery:"num_unique"   db:"num_unique"   json:"num_unique"`
	NumNotNull int64 `ch:"num_not_null" bigquery:"num_not_null" db:"num_not_null" json:"num_not_null"`
	NumEmpty   int64 `ch:"num_empty"    bigquery:"num_empty"    db:"num_empty"    json:"num_empty"`

	PctUnique  float64
	PctNotNull float64
	PctEmpty   float64

	Min    *float64 `ch:"min"    bigquery:"min"    db:"min"    json:"min"`
	Max    *float64 `ch:"max"    bigquery:"max"    db:"max"    json:"max"`
	Mean   *float64 `ch:"mean"   bigquery:"mean"   db:"mean"   json:"mean"`
	Median *float64 `ch:"median" bigquery:"median" db:"median" json:"median"`
	Stddev *float64 `ch:"stddev" bigquery:"stddev" db:"stddev" json:"stddev"`
}

func (*MetricNumericFieldStats) GetIdentity

func (stats *MetricNumericFieldStats) GetIdentity() MetricIdentity

func (*MetricNumericFieldStats) ToDefault

func (stats *MetricNumericFieldStats) ToDefault(timeSegment time.Time, segment string)

func (*MetricNumericFieldStats) WithPartition

func (stats *MetricNumericFieldStats) WithPartition(timeSegment time.Time, segment string)

type MetricResponseI

type MetricResponseI interface {
	GetIdentity() MetricIdentity
	//Initializes the metric to default values
	ToDefault(timeSegment time.Time, segment string)
	//Keeps metric value but sets segment and time segment
	WithPartition(timeSegment time.Time, segment string)
}

type MetricTableStats

type MetricTableStats struct {
	Segment      string     `ch:"segment"        bigquery:"segment"        db:"segment"        json:"segment"`
	TimeSegment  time.Time  `ch:"time_segment"   bigquery:"time_segment"   db:"time_segment"   json:"time_segment"`
	LastLoadedAt *time.Time `ch:"last_loaded_at" bigquery:"last_loaded_at" db:"last_loaded_at" json:"last_loaded_at"`
	NumRows      *int64     `ch:"num_rows"       bigquery:"num_rows"       db:"num_rows"       json:"num_rows"`
	SizeBytes    *int64     `ch:"size_bytes"     bigquery:"size_bytes"     db:"size_bytes"     json:"size_bytes"`
}

func NewMetricTableStats

func NewMetricTableStats(lastLoadedAt *time.Time, numRows *int64, sizeBytes *int64) *MetricTableStats

func NewMetricTableStatsWithSegment

func NewMetricTableStatsWithSegment(lastLoadedAt *time.Time, numRows int64, timeSegment time.Time, segment string) *MetricTableStats

func (*MetricTableStats) GetIdentity

func (stats *MetricTableStats) GetIdentity() MetricIdentity

func (*MetricTableStats) ToDefault

func (stats *MetricTableStats) ToDefault(timeSegment time.Time, segment string)

type MetricTextFieldStats

type MetricTextFieldStats struct {
	Field       string    `ch:"field"        bigquery:"field"        db:"field"        json:"field"`
	Segment     string    `ch:"segment"      bigquery:"segment"      db:"segment"      json:"segment"`
	TimeSegment time.Time `ch:"time_segment" bigquery:"time_segment" db:"time_segment" json:"time_segment"`

	NumTotal   int64 `ch:"num_rows"     bigquery:"num_rows"     db:"num_rows"     json:"num_rows"`
	NumUnique  int64 `ch:"num_unique"   bigquery:"num_unique"   db:"num_unique"   json:"num_unique"`
	NumNotNull int64 `ch:"num_not_null" bigquery:"num_not_null" db:"num_not_null" json:"num_not_null"`
	NumEmpty   int64 `ch:"num_empty"    bigquery:"num_empty"    db:"num_empty"    json:"num_empty"`

	PctUnique  float64
	PctNotNull float64
	PctEmpty   float64
}

func (*MetricTextFieldStats) GetIdentity

func (stats *MetricTextFieldStats) GetIdentity() MetricIdentity

func (*MetricTextFieldStats) ToDefault

func (stats *MetricTextFieldStats) ToDefault(timeSegment time.Time, segment string)

func (*MetricTextFieldStats) WithPartition

func (stats *MetricTextFieldStats) WithPartition(timeSegment time.Time, segment string)

type MetricTimeFieldStats

type MetricTimeFieldStats struct {
	Field       string    `ch:"field"        bigquery:"field"        db:"field"        json:"field"`
	Segment     string    `ch:"segment"      bigquery:"segment"      db:"segment"      json:"segment"`
	TimeSegment time.Time `ch:"time_segment" bigquery:"time_segment" db:"time_segment" json:"time_segment"`

	NumTotal   int64      `ch:"num_rows"     bigquery:"num_rows"     db:"num_rows"     json:"num_rows"`
	NumUnique  int64      `ch:"num_unique"   bigquery:"num_unique"   db:"num_unique"   json:"num_unique"`
	NumNotNull int64      `ch:"num_not_null" bigquery:"num_not_null" db:"num_not_null" json:"num_not_null"`
	Min        *time.Time `ch:"min"          bigquery:"min"          db:"min"          json:"min"`
	Max        *time.Time `ch:"max"          bigquery:"max"          db:"max"          json:"max"`

	PctUnique  float64
	PctNotNull float64
	PctEmpty   float64
}

func (*MetricTimeFieldStats) GetIdentity

func (stats *MetricTimeFieldStats) GetIdentity() MetricIdentity

func (*MetricTimeFieldStats) ToDefault

func (stats *MetricTimeFieldStats) ToDefault(timeSegment time.Time, segment string)

func (*MetricTimeFieldStats) WithPartition

func (stats *MetricTimeFieldStats) WithPartition(timeSegment time.Time, segment string)

type MetricType

type MetricType interface {
	int64 | float64 | time.Time
}

type MetricVal

type MetricVal[T MetricType] struct {
	MonitorPath          string
	MonitoredAssetPath   string
	Segment              string
	Field                string
	MetricId             MetricId
	At                   time.Time
	IngestedAt           time.Time
	Value                T
	MetricsVersion       int32
	IsAnomaly            bool
	IsCorrection         bool
	HasPredictionItem    bool
	IsShiftedTimeSegment bool
}

func ExtractGrowthsFromMetricsSeries

func ExtractGrowthsFromMetricsSeries[T int64 | float64](
	volumeSeries []*MetricVal[T],
	loadSeries []*MetricVal[time.Time],
) []*MetricVal[T]

func FilterMetricVals

func FilterMetricVals[T MetricType](metricVals []*MetricVal[T], path, segment string, from, to time.Time) []*MetricVal[T]

func GetMetricValsBetween

func GetMetricValsBetween[T MetricType](metricVals []*MetricVal[T], from, to time.Time) []*MetricVal[T]

func GetMetricValsIngestedFrom

func GetMetricValsIngestedFrom[T MetricType](metricVals []*MetricVal[T], from time.Time) []*MetricVal[T]

func (*MetricVal[T]) ToFloat

func (m *MetricVal[T]) ToFloat() float64

type MetricVolume

type MetricVolume struct {
	Segment     string    `ch:"segment"      bigquery:"segment"      db:"segment"      json:"segment"`
	TimeSegment time.Time `ch:"time_segment" bigquery:"time_segment" db:"time_segment" json:"time_segment"`
	NumRows     int64     `ch:"num_rows"     bigquery:"num_rows"     db:"num_rows"     json:"num_rows"`
}

func NewMetricVolume

func NewMetricVolume(numRows int64) *MetricVolume

func NewMetricVolumeWithSegments

func NewMetricVolumeWithSegments(numRows int64, timeSegment time.Time, segment string) *MetricVolume

func (*MetricVolume) GetIdentity

func (volume *MetricVolume) GetIdentity() MetricIdentity

func (*MetricVolume) ToDefault

func (volume *MetricVolume) ToDefault(timeSegment time.Time, segment string)

func (*MetricVolume) WithPartition

func (volume *MetricVolume) WithPartition(timeSegment time.Time, segment string)

type MonitorArgs

type MonitorArgs struct {
	Conditions   []CondExpr
	Segmentation []*Segmentation
}

type MonitorPartitioning

type MonitorPartitioning struct {
	Field             string
	Interval          time.Duration
	ScheduleTimeShift time.Duration
}

type NumericMetricExpr

type NumericMetricExpr struct {
	MetricConf
	MetricId MetricId
	Column   NumericExpr
}

func NumericMetric

func NumericMetric(col NumericExpr, metricId MetricId) *NumericMetricExpr

func (*NumericMetricExpr) OutColumnAlias added in v0.1.4

func (m *NumericMetricExpr) OutColumnAlias() TextExpr

func (*NumericMetricExpr) ToSql

func (m *NumericMetricExpr) ToSql(dialect Dialect) (string, error)

type Partition added in v0.1.4

type Partition struct {
	Field string
	From  time.Time
	To    time.Time
}

type Segmentation

type Segmentation struct {
	Expression Expr
	Rule       SegmentationRule
}

type SegmentationRule

type SegmentationRule interface {
	// contains filtered or unexported methods
}

type SegmentationRuleAcceptList

type SegmentationRuleAcceptList struct {
	Values []string
}

func AcceptSegments added in v0.1.11

func AcceptSegments(values ...string) *SegmentationRuleAcceptList

type SegmentationRuleAll

type SegmentationRuleAll struct{}

func AllSegments added in v0.1.11

func AllSegments() *SegmentationRuleAll

type SegmentationRuleExcludeList

type SegmentationRuleExcludeList struct {
	Values []string
}

func ExcludeSegments added in v0.1.11

func ExcludeSegments(values ...string) *SegmentationRuleExcludeList

type StaleMetricsError

type StaleMetricsError struct {
	At time.Time
}

func NewStaleMetricsError

func NewStaleMetricsError(at time.Time) *StaleMetricsError

func (*StaleMetricsError) Error

func (e *StaleMetricsError) Error() string

type TableMetricExpr

type TableMetricExpr struct {
	MetricConf
	MetricId MetricId
}

func TableMetric

func TableMetric(metricId MetricId) *TableMetricExpr

func (*TableMetricExpr) OutColumnAlias added in v0.1.4

func (m *TableMetricExpr) OutColumnAlias() TextExpr

func (*TableMetricExpr) ToSql

func (m *TableMetricExpr) ToSql(dialect Dialect) (string, error)

type TextMetricExpr

type TextMetricExpr struct {
	MetricConf
	MetricId MetricId
	Column   TextExpr
}

func TextMetric

func TextMetric(col TextExpr, metricId MetricId) *TextMetricExpr

func (*TextMetricExpr) As

func (m *TextMetricExpr) As(alias string) *AsExpr

func (*TextMetricExpr) OutColumnAlias added in v0.1.4

func (m *TextMetricExpr) OutColumnAlias() TextExpr

func (*TextMetricExpr) ToSql

func (m *TextMetricExpr) ToSql(dialect Dialect) (string, error)

type TimeMetricExpr

type TimeMetricExpr struct {
	MetricConf
	MetricId MetricId
	TimeExpr TimeExpr
}

func TimeMetric

func TimeMetric(col TimeExpr, metricId MetricId) *TimeMetricExpr

func (*TimeMetricExpr) OutColumnAlias added in v0.1.4

func (m *TimeMetricExpr) OutColumnAlias() TextExpr

func (*TimeMetricExpr) ToSql

func (m *TimeMetricExpr) ToSql(dialect Dialect) (string, error)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL