cache

type Cache interface {
	PodCache
	ModelCache
	MetricCache
	RequestTracker
	ProfileCache
	types.OutputPredictorProvider
	types.RouterProvider
}

type CacheError struct {
	// contains filtered or unexported fields
}

type CachedLoadProvider struct {
	// contains filtered or unexported fields
}

type CappedLoadProvider interface {
	LoadProvider

	// Cap returns the capacity of the pod in terms of the metrics
	Cap() float64
}

type ColdPredictionStrategy int

type Error interface {
	// ErrorType returns the type of the error.
	ErrorType() error
}

type InitOptions struct {
	// EnableKVSync configures whether to start the ZMQ KV event sync
	EnableKVSync bool

	// RedisClient is required for KVSync and other features. Can be nil.
	RedisClient *redis.Client

	// ModelRouterProvider is needed only by the gateway. Can be nil.
	ModelRouterProvider ModelRouterProviderFunc
}

type KVEventManager struct {
	// contains filtered or unexported fields
}

type LoadProvider interface {
	// GetUtilization reads utilization of the pod in terms of the metrics
	GetUtilization(ctx *types.RoutingContext, pod *v1.Pod) (float64, error)

	// GetConsumption reads load consumption of the request on specified pod)
	// If ErrorSLOFailureRequest is returned, the request is considered as failed to meet server capacity,
	// so SLO is expected to violate and can be early rejected.
	GetConsumption(ctx *types.RoutingContext, pod *v1.Pod) (float64, error)
}

type MetricCache interface {
	// GetMetricValueByPod gets metric value for a pod
	// Parameters:
	//   podName: Name of the pod
	//   podNamespace: Namespace of the pod
	//   metricName: Name of the metric
	// Returns:
	//   metrics.MetricValue: Retrieved metric value
	//   error: Error information if operation fails
	GetMetricValueByPod(podName, podNamespace, metricName string) (metrics.MetricValue, error)

	// GetMetricValueByPodModel gets metric value for pod-model pair
	// Parameters:
	//   ctx: Routing context
	//   podName: Name of the pod
	//   podNamespace: Namespace of the pod
	//   modelName: Name of the model
	//   metricName: Name of the metric
	// Returns:
	//   metrics.MetricValue: Retrieved metric value
	//   error: Error information if operation fails
	GetMetricValueByPodModel(podName, podNamespace, modelName string, metricName string) (metrics.MetricValue, error)

	// AddSubscriber adds a metric subscriber
	// Parameters:
	//   subscriber: Metric subscriber implementation
	AddSubscriber(subscriber metrics.MetricSubscriber)
}

type MetricNotFoundError struct {
	*CacheError
	PodName    string
	MetricName string
}

type MissingProfileError struct {
	*CacheError
	ProfileKey string
}

type Model struct {
	// Pods is a CustomizedRegistry that stores *v1.Pod objects.
	// The internal map uses `namespace/name` as the key and `*v1.Pod` as the value.
	// This allows efficient lookups and caching of Pod objects by their unique identifier.
	// By outputing utils.PodArray on Array() call, the PodArray support efficient filtering by
	// GPU (by deployment-like names, see utils.DeploymentNameFromPod() in utils/pod.go).
	Pods *utils.CustomizedRegistry[*v1.Pod, *utils.PodArray]
	// OutputPredictor predicts the number of output tokens per request by history requests.
	// The prediction is essential to SLO-aware features.
	OutputPredictor types.OutputPredictor
	// QueueRouter maintains a local request queue, enabling flexible request reordering.
	QueueRouter types.QueueRouter
	// contains filtered or unexported fields
}

type ModelCache interface {
	// HasModel checks existence of a model
	// Parameters:
	//   modelName: Name of the model
	// Returns:
	//   bool: True if model exists, false otherwise
	HasModel(modelName string) bool

	// ListModels gets all model names
	// Returns:
	//   []string: List of model names
	ListModels() []string

	// ListModelsByPod gets models associated with a pod
	// Parameters:
	//   podName: Name of the pod
	//   podNamespace: Namespace of the pod
	// Returns:
	//   map[string]struct{}: Set of model names
	//   error: Error information if operation fails
	ListModelsByPod(podName, podNamespace string) ([]string, error)
}

type ModelGPUProfile struct {
	Deployment string      `json:"gpu"`     // k8s deployment that specified model and GPU information.
	Cost       float64     `json:"cost"`    // Dollar cost of the unit time GPU computing.
	Tputs      [][]float64 `json:"tputs"`   // Max RPS per correspondent index.
	Indexes    [][]float64 `json:"indexes"` // [output tokens, input tokens]
	Created    float64     `json:"created"` // Profile generation timestamp in Unix format.
	E2E        [][]float64 `json:"e2e"`     // Mean E2E latency per correspondent RPS.
	TTFT       [][]float64 `json:"ttft"`    // Mean TTFT per correspondent RPS.
	TPOT       [][]float64 `json:"tpot"`    // Mean TPOT.
	SLOs       ModelSLOs   `json:"slos"`    // SLOs used for specified model and GPU.
}

type ModelRouterProviderFunc func(modelName string) (types.QueueRouter, error)

type ModelSLOs struct {
	Percentile int     `json:"percentile"` // Percentile applied to SLO metric(s).
	TPUT       float64 `json:"tput"`       // Request Throughput: RPS
	TT         float64 `json:"tt"`         // Token Throughput
	E2E        float64 `json:"e2e"`        // End-to-end latency
	TTFT       float64 `json:"ttft"`       // Time to first token
	TPAT       float64 `json:"tpat"`       // Time per all tokens (suggests normalized E2E latency for different workloads)
	TPOT       float64 `json:"tpot"`       // Time per output tokens
}

type PendingLoadProvider struct {
	*CachedLoadProvider
}

type Pod struct {
	*v1.Pod
	Models       *utils.Registry[string]                    // Model/adapter names that the pod is running
	Metrics      utils.SyncMap[string, metrics.MetricValue] // Pod metrics (metric_name -> value)
	ModelMetrics utils.SyncMap[string, metrics.MetricValue] // Pod-model metrics (model_name/metric_name -> value)
	// contains filtered or unexported fields
}

type PodCache interface {
	// GetPod retrieves a Pod object by name
	// Parameters:
	//   podName: Name of the pod
	//   podNamespace: Namespace of the pod
	// Returns:
	//   *v1.Pod: Found pod object
	//   error: Error information if operation fails
	GetPod(podName, podNamespace string) (*v1.Pod, error)

	// ListPodsByModel gets pods associated with a model
	// Parameters:
	//   modelName: Name of the model
	// Returns:
	//   map[string]*v1.Pod: Pod objects matching the criteria
	//   error: Error information if operation fails
	ListPodsByModel(modelName string) (types.PodList, error)
}

type ProfileCache interface {
	// GetModelProfileByPod gets model profile for a pod
	// Parameters:
	//   pod: Pod object
	//   modelName: Name of the model
	GetModelProfileByPod(pod *v1.Pod, modelName string) (*ModelGPUProfile, error)

	// GetModelProfileByDeploymentName gets model profile for a deployment
	// Parameters:
	//   deploymentName: Name of the deployment
	//   modelName: Name of the model
	GetModelProfileByDeploymentName(deploymentName string, modelName string) (*ModelGPUProfile, error)
}

type RequestTrace struct {
	// contains filtered or unexported fields
}

type RequestTraceMetaKey int

type RequestTracker interface {
	// AddRequestCount tracks the start of a request after routing.
	// To support realtime statistics update and access, AddRequestCount can be called multiple times for a request.
	// As the result, implementation should ensure thread-safe access to the counterm and idempotency.
	//
	// Parameters:
	//   ctx: Routing context
	//   requestID: Unique request identifier
	//   modelName: Name of the model
	// Returns:
	//   int64: Trace term identifier
	AddRequestCount(ctx *types.RoutingContext, requestID string, modelName string) (traceTerm int64)

	// DoneRequestCount tracks the completion of a request without usage information like inputTokens and outputTokens.
	// Only one DoneRequestXXX should be called for a request. Idemptency is not required.
	//
	// Parameters:
	//   requestID: Unique request identifier
	//   modelName: Name of the model
	//   traceTerm: Trace term identifier
	DoneRequestCount(ctx *types.RoutingContext, requestID string, modelName string, traceTerm int64)

	// DoneRequestTrace tracks the completion of a request with usage information like inputTokens and outputTokens.
	// Only one DoneRequestXXX should be called for a request. Idemptency is not required.
	//
	// Parameters:
	//   ctx: Routing context
	//   requestID: Unique request identifier
	//   modelName: Name of the model
	//   inputTokens: Number of input tokens
	//   outputTokens: Number of output tokens
	//   traceTerm: Trace term identifier
	DoneRequestTrace(ctx *types.RoutingContext, requestID string, modelName string, inputTokens, outputTokens, traceTerm int64)
}

type SimpleOutputPredictor struct {
	// contains filtered or unexported fields
}

type Store struct {
	// contains filtered or unexported fields
}