phughk/ottyl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Metrics rethink: counters as the primitive, histogram()/gauge() derived (ticket 022)

Browse Source 
Every metric declaration is now explicitly documented and modeled as
a counter: a match is an occurrence/increment, optionally labeled.
The bucket representation (BucketValue { values: Vec<(Sequence,
Timestamp, MetricPrimitive)> }) already was this; what it lacked was
any way to capture a *value* distinct from the occurrence-count-of-1
marker. Adds RegexToMetricPrimitiveMatcher::role (ParamRole::Label,
the default, or ParamRole::Value — at most one per metric, enforced
at declare time) so a metric can designate one param's captured
number as its per-event value instead of a label.

promql.rs/engine.rs add two new PromQL-evaluable forms built on top
of that value, both consumed as regular PromqlSeriesDto series like
sum()/avg()/rate():
- histogram(metric_name[width]): buckets matching events' values into
  fixed-width buckets, one series per bucket labeled `le`.
- gauge(metric_name): the most-recent value per label combination in
  the queried range.

Existing raw counter queries (metrics with only Label-role params,
sum/avg/rate) are unchanged. Profile files saved before this ticket
still load (role defaults to Label via #[serde(default)]). Metrics.tsx
gains a label/value role picker per param, and the Docs panel explains
the model and the two derived query forms.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XFUNZwCvz3kcod3352ZR2o
This commit is contained in:
Hugh Kaznowski
2026-09-05 15:18:25 +01:00
co-authored by Claude Sonnet 5
parent 1d0f324c43
commit 0c57ed2953
7 changed files with 388 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/engine.rs
+229 -15
View File
@@ -1,4 +1,4 @@
use crate::data_value::{DataValue, MetricPrimitiveKind, RegexToMetricPrimitiveMatcher};
use crate::data_value::{DataValue, MetricPrimitiveKind, ParamRole, RegexToMetricPrimitiveMatcher};
use crate::dto::{
BucketValueDto, LogEntryDto, LogsParams, MetricDeclarationDto, ParamDeclarationDto,
PromqlSeriesDto, StatsDto, TimelineBucketDto, TimelineParams,
@@ -231,6 +231,13 @@ fn declare_metric(
return Err(errors.join("; "));
}
let value_params = request.params.iter().filter(|matcher| matcher.role == ParamRole::Value).count();
if value_params > 1 {
return Err(format!(
"a metric may declare at most one Value-role param, found {value_params}"
));
}
backfill_metric(index, &request.name, &re, &request.params);
index.insert(
IndexKey::MetricDeclaration { name: request.name },
@@ -263,17 +270,22 @@ fn parse_metric_primitive(raw: &str, kind: MetricPrimitiveKind) -> Option<Metric
}
/// Runs one metric's regex against a single log line and, on a match, appends a
/// bucket value entry.
/// bucket value entry: this is the "counter increment" (ticket 022) — a match is
/// an occurrence at a point in time, optionally labeled.
///
/// Design decision: `Bucket.labels` is the only structured place a `Bucket` key has
/// for a metric's declared params, so v1 folds ALL of a metric's params into
/// `labels` rather than splitting them into "label" vs "value" params. The stored
/// value is therefore a fixed presence marker (`MetricPrimitive::Number(Number::Int(1))`)
/// — a bucket's `values` is the timeseries of *when that label combination occurred*
/// (a count/occurrence series), not a record of the captured values themselves. If a
/// param's capture group doesn't match, or its captured text fails to parse as its
/// declared `kind`, the whole match is skipped for this line rather than storing a
/// partially-labeled bucket.
/// `Bucket.labels` holds every `Label`-role param (the default role — see
/// [`ParamRole`]); at most one param may instead be `Value`-role (enforced at
/// declare time), and its parsed primitive becomes the pushed event's value.
/// With no `Value`-role param, the pushed value is a fixed presence marker
/// (`MetricPrimitive::Number(Number::Int(1))`), matching the original v1
/// behavior where a bucket's `values` was purely the timeseries of *when that
/// label combination occurred* — histogram()/gauge() derived queries
/// (`crate::promql::AggOp`) read this same `value`, so a `Value`-role param is
/// what makes those meaningful (bucketing/reading an occurrence-count-of-1
/// series degenerates to "everything in one bucket"/"always 1"). If a param's
/// capture group doesn't match, or its captured text fails to parse as its
/// declared `kind`, the whole match is skipped for this line rather than
/// storing a partially-labeled bucket.
fn apply_metric_to_log(
index: &mut BTreeMap<IndexKey, DataValue>,
metric_name: &str,
@@ -288,14 +300,22 @@ fn apply_metric_to_log(
};
let mut labels = BTreeMap::new();
let mut value = None;
for matcher in params {
let Some(raw) = captures.name(&matcher.capture_group) else {
return;
};
let Some(value) = parse_metric_primitive(raw.as_str(), matcher.kind) else {
let Some(parsed) = parse_metric_primitive(raw.as_str(), matcher.kind) else {
return;
};
labels.insert(matcher.param_name.clone(), value);
match matcher.role {
ParamRole::Label => {
labels.insert(matcher.param_name.clone(), parsed);
}
ParamRole::Value => {
value = Some(parsed);
}
}
}
let key = IndexKey::Bucket { metric_name: metric_name.to_string(), labels };
@@ -303,7 +323,7 @@ fn apply_metric_to_log(
.entry(key)
.or_insert_with(|| DataValue::BucketValue { values: Vec::new() });
if let DataValue::BucketValue { values } = entry {
values.push((sequence, timestamp, MetricPrimitive::Number(Number::Int(1))));
values.push((sequence, timestamp, value.unwrap_or(MetricPrimitive::Number(Number::Int(1)))));
}
}
@@ -980,6 +1000,51 @@ fn evaluate_promql(
})
.collect()
}
Some(AggOp::Gauge) => {
// Ticket 022: collapse each label combination's counter events
// down to their most-recent value in the queried time range —
// "value at time T" derived from the last matching event, rather
// than a distinct stored gauge kind.
series_points
.into_iter()
.filter_map(|(labels, points)| {
let last = *points.last()?;
Some(PromqlSeriesDto { labels, points: vec![last] })
})
.collect()
}
Some(AggOp::Histogram) => {
// Validated by `promql::parse_query`: `histogram()` never parses
// without one.
let bucket_width =
parsed.bucket_width.expect("histogram() always carries a bucket width");
let last_ts = series_points
.values()
.flat_map(|points| points.iter().map(|(ts, _)| *ts))
.max()
.unwrap_or(0);
// Ticket 022: bucket every matching counter event's value
// (across all label combinations — histogram collapses the
// label dimension the same way sum()/avg() do) into fixed-width
// buckets, one output series per bucket labeled `le` (Prometheus's
// "less-than-or-equal" convention) holding a single
// `(last_ts, count)` point.
let mut counts: BTreeMap<i64, u64> = BTreeMap::new();
for points in series_points.values() {
for (_, value) in points {
let bucket_index = (value / bucket_width).floor() as i64;
*counts.entry(bucket_index).or_insert(0) += 1;
}
}
counts
.into_iter()
.map(|(bucket_index, count)| {
let upper_bound = (bucket_index + 1) as f64 * bucket_width;
let labels = BTreeMap::from([("le".to_string(), upper_bound.to_string())]);
PromqlSeriesDto { labels, points: vec![(last_ts, count as f64)] }
})
.collect()
}
}
}
@@ -1103,16 +1168,32 @@ mod tests {
name: &str,
pattern: &str,
params: Vec<(&str, &str, MetricPrimitiveKind)>,
) -> DeclareMetricRequest {
metric_request_with_roles(
name,
pattern,
params
.into_iter()
.map(|(param_name, capture_group, kind)| (param_name, capture_group, kind, ParamRole::Label))
.collect(),
)
}
fn metric_request_with_roles(
name: &str,
pattern: &str,
params: Vec<(&str, &str, MetricPrimitiveKind, ParamRole)>,
) -> DeclareMetricRequest {
DeclareMetricRequest {
name: name.to_string(),
message_match_regex: pattern.to_string(),
params: params
.into_iter()
.map(|(param_name, capture_group, kind)| RegexToMetricPrimitiveMatcher {
.map(|(param_name, capture_group, kind, role)| RegexToMetricPrimitiveMatcher {
param_name: param_name.to_string(),
capture_group: capture_group.to_string(),
kind,
role,
})
.collect(),
}
@@ -1439,6 +1520,139 @@ mod tests {
assert_eq!(series[0].points, vec![(5, 1.0)]);
}
#[test]
fn declares_metric_with_value_role_param() {
let mut index = BTreeMap::new();
let result = declare_metric(
&mut index,
metric_request_with_roles(
"durations",
r"request path=(?P<path>\S+) duration=(?P<duration>\d+)",
vec![
("path", "path", MetricPrimitiveKind::String, ParamRole::Label),
("duration", "duration", MetricPrimitiveKind::Number, ParamRole::Value),
],
),
);
assert!(result.is_ok());
}
#[test]
fn rejects_metric_with_multiple_value_role_params() {
let mut index = BTreeMap::new();
let result = declare_metric(
&mut index,
metric_request_with_roles(
"bad",
r"a=(?P<a>\d+) b=(?P<b>\d+)",
vec![
("a", "a", MetricPrimitiveKind::Number, ParamRole::Value),
("b", "b", MetricPrimitiveKind::Number, ParamRole::Value),
],
),
);
assert!(result.is_err());
assert!(result.unwrap_err().contains("at most one Value-role param"));
}
#[test]
fn value_role_param_becomes_the_bucket_value_not_a_label() {
let mut index = BTreeMap::new();
declare_metric(
&mut index,
metric_request_with_roles(
"durations",
r"request path=(?P<path>\S+) duration=(?P<duration>\d+)",
vec![
("path", "path", MetricPrimitiveKind::String, ParamRole::Label),
("duration", "duration", MetricPrimitiveKind::Number, ParamRole::Value),
],
),
)
.unwrap();
ingest_log(&mut index, 0, "request path=/a duration=5");
let values: Vec<_> = bucket_values(&index, "durations", None).collect();
assert_eq!(values.len(), 1);
let (labels, (_, _, value)) = &values[0];
assert!(!labels.contains_key("duration"));
assert_eq!(labels.get("path").map(|v| v.to_string()), Some("/a".to_string()));
assert_eq!(*value, MetricPrimitive::Number(Number::Int(5)));
}
fn declare_durations_metric(index: &mut BTreeMap<IndexKey, DataValue>) {
declare_metric(
index,
metric_request_with_roles(
"durations",
r"request path=(?P<path>\S+) duration=(?P<duration>\d+)",
vec![
("path", "path", MetricPrimitiveKind::String, ParamRole::Label),
("duration", "duration", MetricPrimitiveKind::Number, ParamRole::Value),
],
),
)
.unwrap();
}
#[test]
fn gauge_returns_most_recent_value_per_label_combination() {
let mut index = BTreeMap::new();
declare_durations_metric(&mut index);
ingest_log(&mut index, 0, "request path=/a duration=5");
ingest_log(&mut index, 1, "request path=/a duration=9");
ingest_log(&mut index, 2, "request path=/b duration=42");
let parsed = promql::parse_query("gauge(durations)").unwrap();
let mut series = evaluate_promql(&index, &parsed, None, None);
series.sort_by(|a, b| a.labels.get("path").cmp(&b.labels.get("path")));
assert_eq!(series.len(), 2);
assert_eq!(series[0].labels.get("path").map(String::as_str), Some("/a"));
assert_eq!(series[0].points, vec![(1, 9.0)]);
assert_eq!(series[1].labels.get("path").map(String::as_str), Some("/b"));
assert_eq!(series[1].points, vec![(2, 42.0)]);
}
#[test]
fn gauge_respects_time_range_bounds() {
let mut index = BTreeMap::new();
declare_durations_metric(&mut index);
ingest_log(&mut index, 0, "request path=/a duration=5");
ingest_log(&mut index, 5, "request path=/a duration=9");
ingest_log(&mut index, 10, "request path=/a duration=42");
let parsed = promql::parse_query("gauge(durations)").unwrap();
let series = evaluate_promql(&index, &parsed, None, Some(9));
assert_eq!(series.len(), 1);
assert_eq!(series[0].points, vec![(5, 9.0)]);
}
#[test]
fn histogram_buckets_captured_values_by_fixed_width() {
let mut index = BTreeMap::new();
declare_durations_metric(&mut index);
for (seq, duration) in [(0, 2), (1, 4), (2, 12), (3, 15), (4, 21)] {
ingest_log(&mut index, seq, &format!("request path=/a duration={duration}"));
}
let parsed = promql::parse_query("histogram(durations[10])").unwrap();
let mut series = evaluate_promql(&index, &parsed, None, None);
series.sort_by(|a, b| a.labels.get("le").cmp(&b.labels.get("le")));
// durations 2,4 -> bucket [0,10) le=10 (count 2); 12,15 -> [10,20) le=20 (count 2);
// 21 -> [20,30) le=30 (count 1).
assert_eq!(series.len(), 3);
assert_eq!(series[0].labels.get("le").map(String::as_str), Some("10"));
assert_eq!(series[0].points[0].1, 2.0);
assert_eq!(series[1].labels.get("le").map(String::as_str), Some("20"));
assert_eq!(series[1].points[0].1, 2.0);
assert_eq!(series[2].labels.get("le").map(String::as_str), Some("30"));
assert_eq!(series[2].points[0].1, 1.0);
}
#[test]
fn handle_query_promql_rejects_malformed_query() {
let index = BTreeMap::new();