Unify PromQL rate() and alert Threshold/Ratio windowing (ticket 040)

Extract the trailing-window rate/count implementation into a new src/rate.rs,
shared by PromQL's rate() (engine::evaluate_promql, via a thin i64<->Timestamp
adapter) and alert Threshold/Ratio conditions (alerts::evaluate_condition, via
the new incremental RateWindow), replacing alerts' bespoke VecDeque+prune loop.
Both now agree on what "windowed rate" means for the same data, verified by a
new regression test (engine::tests::dashboard_rate_and_alert_threshold_agree_on_same_windowed_data).

Behavior change: alert Threshold/Ratio windows are now backfilled once, on
first evaluation, from already-ingested logs within their own trailing window
(bounded cost, not a full history rescan) - a rule created mid-session, or a
process that replays existing log content (e.g. --file tailing, which rereads
the whole file from the start on every process start), no longer starts with
a silently empty window. A bare restart with no replayed history (e.g. stdin
ingestion) still starts empty, since there is genuinely no log history to
backfill from in that case - alert state itself remains in-memory only.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XFUNZwCvz3kcod3352ZR2o
This commit is contained in:
Hugh Kaznowski
2026-09-05 22:34:13 +01:00
co-authored by Claude Sonnet 5
parent 85fb145be7
commit ef60dd267b
4 changed files with 328 additions and 57 deletions
+90 -43
View File
@@ -1,9 +1,16 @@
//! Alert rules: definitions, trigger evaluation, and ntfy delivery. //! Alert rules: definitions, trigger evaluation, and ntfy delivery.
//! //!
//! Alert queries are restricted to regex/sequence/timestamp/json[field] — a metric[param] comparison would require a full bucket rescan per line, breaking the proportional-cost guarantee. json[field] is fine here since, like regex/sequence/timestamp, it only inspects the line's own text/fields, no index lookup. //! Alert queries are restricted to regex/sequence/timestamp/json[field] — a metric[param] comparison would require a full bucket rescan per line, breaking the proportional-cost guarantee. json[field] is fine here since, like regex/sequence/timestamp, it only inspects the line's own text/fields, no index lookup.
//!
//! `Threshold`/`Ratio` windows are maintained via `crate::rate::RateWindow` — the same windowed-rate
//! implementation PromQL's `rate()` uses (`crate::engine::evaluate_promql`) — rather than a
//! bespoke `VecDeque`+prune loop, so the two agree on what "windowed rate" means (ticket 040). A
//! window is also backfilled once, on first evaluation, from already-ingested logs within its own
//! trailing window (see `backfill_window`): a rule created mid-session (or a process that replays
//! existing log content, e.g. `--file` tailing) no longer starts with a silently empty window.
use std::collections::hash_map::DefaultHasher; use std::collections::hash_map::DefaultHasher;
use std::collections::{HashMap, HashSet, VecDeque}; use std::collections::{BTreeMap, HashMap, HashSet};
use std::fmt; use std::fmt;
use std::hash::{Hash, Hasher}; use std::hash::{Hash, Hasher};
use std::io; use std::io;
@@ -13,9 +20,12 @@ use std::time::{SystemTime, UNIX_EPOCH};
use regex::Regex; use regex::Regex;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use crate::engine::{apply_compare_op, eval_json_comparison}; use crate::data_value::DataValue;
use crate::engine::{self, apply_compare_op, eval_json_comparison};
use crate::filter_datetime; use crate::filter_datetime;
use crate::filter_parser::{self, CompareOp, FilterExpr, FilterValue}; use crate::filter_parser::{self, CompareOp, FilterExpr, FilterValue};
use crate::index_key::IndexKey;
use crate::rate::RateWindow;
use crate::settings::Settings; use crate::settings::Settings;
use crate::timestamp::Timestamp; use crate::timestamp::Timestamp;
@@ -135,24 +145,43 @@ impl Settings {
} }
} }
/// Per-rule in-memory evaluation state; lives only for the process's lifetime — not persisted, not backfilled against already-ingested logs. /// Per-rule in-memory evaluation state; lives only for the process's lifetime — not persisted
/// across a restart with no replayed log history. Each `RateWindow` is backfilled once (see
/// `backfill_window`) from whatever matching history is already in the index at first evaluation,
/// so — unlike before ticket 040 — a rule doesn't start blind to matches ingested before it was
/// declared, or before this evaluation loop last started, as long as that history is still in the
/// in-memory log index (e.g. `--file` tailing replays existing file content from the start on
/// every process start).
#[derive(Debug, Default)] #[derive(Debug, Default)]
pub struct AlertRuntimeState { pub struct AlertRuntimeState {
threshold_matches: VecDeque<i64>, threshold_window: RateWindow,
ratio_a_matches: VecDeque<i64>, threshold_backfilled: bool,
ratio_b_matches: VecDeque<i64>, ratio_a_window: RateWindow,
ratio_a_backfilled: bool,
ratio_b_window: RateWindow,
ratio_b_backfilled: bool,
seen_values: HashSet<String>, seen_values: HashSet<String>,
} }
/// `timestamps` is push_back-ordered, so once the front entry is within the window, every entry behind it is too. /// Builds a `RateWindow` seeded from already-ingested logs matching `query`, bounded to
fn prune_window(timestamps: &mut VecDeque<i64>, now_ms: i64, window_ms: i64) { /// `window_ms` before `now` (not the full log history) so the cost stays proportional to the
while let Some(&front) = timestamps.front() { /// window rather than everything ever ingested. `before_sequence` excludes the line currently
if now_ms - front > window_ms { /// being evaluated, which the caller pushes separately if it matches.
timestamps.pop_front(); fn backfill_window(
} else { index: &BTreeMap<IndexKey, DataValue>,
break; query: &str,
} before_sequence: u64,
} now: Timestamp,
window_ms: i64,
) -> RateWindow {
let since_ms = now.as_millis() - window_ms;
let matching_timestamps =
engine::logs_since(index, before_sequence, since_ms, now.as_millis()).filter_map(
|(sequence, timestamp, text)| {
(eval_alert_query(query, sequence, timestamp, text) == Ok(true)).then_some(timestamp)
},
);
RateWindow::backfill(matching_timestamps, now, window_ms)
} }
/// Evaluates a FilterBar query against a single line's own fields only (metric[param] comparisons aren't supported — see module doc; json[field] is supported since it's also line-local). /// Evaluates a FilterBar query against a single line's own fields only (metric[param] comparisons aren't supported — see module doc; json[field] is supported since it's also line-local).
@@ -205,6 +234,7 @@ fn eval_alert_expr(expr: &FilterExpr, sequence: u64, timestamp: Timestamp, text:
pub fn evaluate_alerts( pub fn evaluate_alerts(
rules: &[AlertRule], rules: &[AlertRule],
states: &mut HashMap<String, AlertRuntimeState>, states: &mut HashMap<String, AlertRuntimeState>,
index: &BTreeMap<IndexKey, DataValue>,
sequence: u64, sequence: u64,
timestamp: Timestamp, timestamp: Timestamp,
text: &str, text: &str,
@@ -215,7 +245,9 @@ pub fn evaluate_alerts(
continue; continue;
} }
let state = states.entry(rule.id.clone()).or_default(); let state = states.entry(rule.id.clone()).or_default();
if let Some(message) = evaluate_condition(&rule.condition, state, sequence, timestamp, text, &rule.name) { if let Some(message) =
evaluate_condition(&rule.condition, state, index, sequence, timestamp, text, &rule.name)
{
fired.push((rule.clone(), message)); fired.push((rule.clone(), message));
} }
} }
@@ -225,6 +257,7 @@ pub fn evaluate_alerts(
fn evaluate_condition( fn evaluate_condition(
condition: &AlertCondition, condition: &AlertCondition,
state: &mut AlertRuntimeState, state: &mut AlertRuntimeState,
index: &BTreeMap<IndexKey, DataValue>,
sequence: u64, sequence: u64,
timestamp: Timestamp, timestamp: Timestamp,
text: &str, text: &str,
@@ -240,12 +273,14 @@ fn evaluate_condition(
} }
}, },
AlertCondition::Threshold { query, op, threshold, window_ms } => { AlertCondition::Threshold { query, op, threshold, window_ms } => {
if !state.threshold_backfilled {
state.threshold_window = backfill_window(index, query, sequence, timestamp, *window_ms);
state.threshold_backfilled = true;
}
match eval_alert_query(query, sequence, timestamp, text) { match eval_alert_query(query, sequence, timestamp, text) {
Ok(true) => { Ok(true) => {
let now_ms = timestamp.as_millis(); state.threshold_window.push(timestamp, *window_ms);
state.threshold_matches.push_back(now_ms); let count = state.threshold_window.count() as f64;
prune_window(&mut state.threshold_matches, now_ms, *window_ms);
let count = state.threshold_matches.len() as f64;
apply_compare_op(count.total_cmp(threshold), *op).then(|| { apply_compare_op(count.total_cmp(threshold), *op).then(|| {
format!("{rule_name}: match count {count} {op:?} threshold {threshold} (window {window_ms}ms)") format!("{rule_name}: match count {count} {op:?} threshold {threshold} (window {window_ms}ms)")
}) })
@@ -285,17 +320,22 @@ fn evaluate_condition(
if !matched_a && !matched_b { if !matched_a && !matched_b {
return None; return None;
} }
let now_ms = timestamp.as_millis(); if !state.ratio_a_backfilled {
state.ratio_a_window = backfill_window(index, query_a, sequence, timestamp, *window_ms);
state.ratio_a_backfilled = true;
}
if !state.ratio_b_backfilled {
state.ratio_b_window = backfill_window(index, query_b, sequence, timestamp, *window_ms);
state.ratio_b_backfilled = true;
}
if matched_a { if matched_a {
state.ratio_a_matches.push_back(now_ms); state.ratio_a_window.push(timestamp, *window_ms);
prune_window(&mut state.ratio_a_matches, now_ms, *window_ms);
} }
if matched_b { if matched_b {
state.ratio_b_matches.push_back(now_ms); state.ratio_b_window.push(timestamp, *window_ms);
prune_window(&mut state.ratio_b_matches, now_ms, *window_ms);
} }
let count_a = state.ratio_a_matches.len() as f64; let count_a = state.ratio_a_window.count() as f64;
let count_b = state.ratio_b_matches.len() as f64; let count_b = state.ratio_b_window.count() as f64;
if count_b == 0.0 { if count_b == 0.0 {
// Zero denominator: skip rather than treat the ratio as infinite. // Zero denominator: skip rather than treat the ratio as infinite.
return None; return None;
@@ -417,12 +457,13 @@ mod tests {
fn immediate_fires_on_matching_line_only() { fn immediate_fires_on_matching_line_only() {
let rules = vec![immediate_rule(r#"regex("error")"#)]; let rules = vec![immediate_rule(r#"regex("error")"#)];
let mut states = HashMap::new(); let mut states = HashMap::new();
let index: BTreeMap<IndexKey, DataValue> = BTreeMap::new();
let (seq, ts, text) = line("all good", 0, 0); let (seq, ts, text) = line("all good", 0, 0);
assert!(evaluate_alerts(&rules, &mut states, seq, ts, text).is_empty()); assert!(evaluate_alerts(&rules, &mut states, &index, seq, ts, text).is_empty());
let (seq, ts, text) = line("an error happened", 1, 1); let (seq, ts, text) = line("an error happened", 1, 1);
let fired = evaluate_alerts(&rules, &mut states, seq, ts, text); let fired = evaluate_alerts(&rules, &mut states, &index, seq, ts, text);
assert_eq!(fired.len(), 1); assert_eq!(fired.len(), 1);
assert!(fired[0].1.contains("an error happened")); assert!(fired[0].1.contains("an error happened"));
} }
@@ -432,8 +473,9 @@ mod tests {
let mut rule = immediate_rule(r#"regex("error")"#); let mut rule = immediate_rule(r#"regex("error")"#);
rule.enabled = false; rule.enabled = false;
let mut states = HashMap::new(); let mut states = HashMap::new();
let index: BTreeMap<IndexKey, DataValue> = BTreeMap::new();
let (seq, ts, text) = line("error!", 0, 0); let (seq, ts, text) = line("error!", 0, 0);
assert!(evaluate_alerts(&[rule], &mut states, seq, ts, text).is_empty()); assert!(evaluate_alerts(&[rule], &mut states, &index, seq, ts, text).is_empty());
} }
#[test] #[test]
@@ -450,14 +492,15 @@ mod tests {
}, },
}; };
let mut states = HashMap::new(); let mut states = HashMap::new();
let index: BTreeMap<IndexKey, DataValue> = BTreeMap::new();
for (seq, ms) in [(0u64, 0i64), (1, 1000)] { for (seq, ms) in [(0u64, 0i64), (1, 1000)] {
let fired = evaluate_alerts(&[rule.clone()], &mut states, seq, Timestamp::from_millis(ms), "error"); let fired = evaluate_alerts(&[rule.clone()], &mut states, &index, seq, Timestamp::from_millis(ms), "error");
assert!(fired.is_empty(), "should not fire before threshold is crossed"); assert!(fired.is_empty(), "should not fire before threshold is crossed");
} }
let fired = evaluate_alerts(&[rule.clone()], &mut states, 2, Timestamp::from_millis(2000), "error"); let fired = evaluate_alerts(&[rule.clone()], &mut states, &index, 2, Timestamp::from_millis(2000), "error");
assert_eq!(fired.len(), 1); assert_eq!(fired.len(), 1);
// Stays quiet for a non-matching line even though the window is full. // Stays quiet for a non-matching line even though the window is full.
let fired = evaluate_alerts(&[rule], &mut states, 3, Timestamp::from_millis(2100), "fine"); let fired = evaluate_alerts(&[rule], &mut states, &index, 3, Timestamp::from_millis(2100), "fine");
assert!(fired.is_empty()); assert!(fired.is_empty());
} }
@@ -475,9 +518,10 @@ mod tests {
}, },
}; };
let mut states = HashMap::new(); let mut states = HashMap::new();
assert!(evaluate_alerts(&[rule.clone()], &mut states, 0, Timestamp::from_millis(0), "error").is_empty()); let index: BTreeMap<IndexKey, DataValue> = BTreeMap::new();
assert!(evaluate_alerts(&[rule.clone()], &mut states, &index, 0, Timestamp::from_millis(0), "error").is_empty());
// Second match arrives well after the first has aged out of the window. // Second match arrives well after the first has aged out of the window.
assert!(evaluate_alerts(&[rule], &mut states, 1, Timestamp::from_millis(5_000), "error").is_empty()); assert!(evaluate_alerts(&[rule], &mut states, &index, 1, Timestamp::from_millis(5_000), "error").is_empty());
} }
#[test] #[test]
@@ -492,11 +536,12 @@ mod tests {
}, },
}; };
let mut states = HashMap::new(); let mut states = HashMap::new();
let fired = evaluate_alerts(&[rule.clone()], &mut states, 0, Timestamp::from_millis(0), "agent=curl"); let index: BTreeMap<IndexKey, DataValue> = BTreeMap::new();
let fired = evaluate_alerts(&[rule.clone()], &mut states, &index, 0, Timestamp::from_millis(0), "agent=curl");
assert_eq!(fired.len(), 1); assert_eq!(fired.len(), 1);
let fired = evaluate_alerts(&[rule.clone()], &mut states, 1, Timestamp::from_millis(1), "agent=curl"); let fired = evaluate_alerts(&[rule.clone()], &mut states, &index, 1, Timestamp::from_millis(1), "agent=curl");
assert!(fired.is_empty()); assert!(fired.is_empty());
let fired = evaluate_alerts(&[rule], &mut states, 2, Timestamp::from_millis(2), "agent=wget"); let fired = evaluate_alerts(&[rule], &mut states, &index, 2, Timestamp::from_millis(2), "agent=wget");
assert_eq!(fired.len(), 1); assert_eq!(fired.len(), 1);
} }
@@ -515,16 +560,17 @@ mod tests {
}, },
}; };
let mut states = HashMap::new(); let mut states = HashMap::new();
let index: BTreeMap<IndexKey, DataValue> = BTreeMap::new();
// Only an "error" so far; no "request" yet, so denominator is 0 -> skip. // Only an "error" so far; no "request" yet, so denominator is 0 -> skip.
assert!(evaluate_alerts(&[rule.clone()], &mut states, 0, Timestamp::from_millis(0), "error").is_empty()); assert!(evaluate_alerts(&[rule.clone()], &mut states, &index, 0, Timestamp::from_millis(0), "error").is_empty());
// One request: ratio 1/1 = 1.0 >= 0.5 -> fires. // One request: ratio 1/1 = 1.0 >= 0.5 -> fires.
let fired = evaluate_alerts(&[rule.clone()], &mut states, 1, Timestamp::from_millis(1), "request"); let fired = evaluate_alerts(&[rule.clone()], &mut states, &index, 1, Timestamp::from_millis(1), "request");
assert_eq!(fired.len(), 1); assert_eq!(fired.len(), 1);
// A second request: ratio 1/2 = 0.5 still >= 0.5 -> fires again. // A second request: ratio 1/2 = 0.5 still >= 0.5 -> fires again.
let fired = evaluate_alerts(&[rule.clone()], &mut states, 2, Timestamp::from_millis(2), "request"); let fired = evaluate_alerts(&[rule.clone()], &mut states, &index, 2, Timestamp::from_millis(2), "request");
assert_eq!(fired.len(), 1); assert_eq!(fired.len(), 1);
// A third request with no more errors: ratio drops to 1/3 -> no longer fires. // A third request with no more errors: ratio drops to 1/3 -> no longer fires.
let fired = evaluate_alerts(&[rule], &mut states, 3, Timestamp::from_millis(3), "request"); let fired = evaluate_alerts(&[rule], &mut states, &index, 3, Timestamp::from_millis(3), "request");
assert!(fired.is_empty()); assert!(fired.is_empty());
} }
@@ -532,8 +578,9 @@ mod tests {
fn metric_comparison_in_alert_query_is_rejected_not_panicking() { fn metric_comparison_in_alert_query_is_rejected_not_panicking() {
let rule = immediate_rule("some_metric[param]==5"); let rule = immediate_rule("some_metric[param]==5");
let mut states = HashMap::new(); let mut states = HashMap::new();
let index: BTreeMap<IndexKey, DataValue> = BTreeMap::new();
// Should log a warning and simply not fire, never panic. // Should log a warning and simply not fire, never panic.
let fired = evaluate_alerts(&[rule], &mut states, 0, Timestamp::from_millis(0), "anything"); let fired = evaluate_alerts(&[rule], &mut states, &index, 0, Timestamp::from_millis(0), "anything");
assert!(fired.is_empty()); assert!(fired.is_empty());
} }
+94 -14
View File
@@ -9,6 +9,7 @@ use crate::filter_parser::{self, CompareOp, FilterExpr, FilterValue};
use crate::index_key::IndexKey; use crate::index_key::IndexKey;
use crate::metric_primitive::{MetricPrimitive, Number}; use crate::metric_primitive::{MetricPrimitive, Number};
use crate::promql::{self, AggOp, LabelMatcher, PromqlQuery}; use crate::promql::{self, AggOp, LabelMatcher, PromqlQuery};
use crate::rate;
use crate::settings::SettingsHandle; use crate::settings::SettingsHandle;
use crate::timestamp::Timestamp; use crate::timestamp::Timestamp;
use regex::Regex; use regex::Regex;
@@ -161,7 +162,8 @@ pub async fn run(mut rx: mpsc::Receiver<EngineMessage>, settings: SettingsHandle
let settings = settings.lock().unwrap(); let settings = settings.lock().unwrap();
(settings.alert_rules.clone(), settings.ntfy_topic.clone(), settings.ntfy_server.clone()) (settings.alert_rules.clone(), settings.ntfy_topic.clone(), settings.ntfy_server.clone())
}; };
let fired = alerts::evaluate_alerts(&rules, &mut alert_states, sequence, timestamp, &text); let fired =
alerts::evaluate_alerts(&rules, &mut alert_states, &index, sequence, timestamp, &text);
for (rule, message) in fired { for (rule, message) in fired {
tokio::spawn(alerts::deliver_for_rule(rule.name, message, ntfy_topic.clone(), ntfy_server.clone())); tokio::spawn(alerts::deliver_for_rule(rule.name, message, ntfy_topic.clone(), ntfy_server.clone()));
} }
@@ -962,6 +964,24 @@ fn logs_range(index: &BTreeMap<IndexKey, DataValue>) -> impl Iterator<Item = (u6
}) })
} }
/// Every currently-ingested log with `sequence < before_sequence` and timestamp in
/// `(since_ms, now_ms]`, in ascending order. `pub(crate)` so `crate::alerts` can use it to
/// backfill a freshly-created `Threshold`/`Ratio` window from already-ingested history — bounded
/// to the alert's own window, not the full log history, so the cost stays proportional to the
/// window rather than the whole ingest history (ticket 040).
pub(crate) fn logs_since(
index: &BTreeMap<IndexKey, DataValue>,
before_sequence: u64,
since_ms: i64,
now_ms: i64,
) -> impl Iterator<Item = (u64, Timestamp, &str)> {
logs_range(index).filter(move |(sequence, timestamp, _)| {
*sequence < before_sequence
&& timestamp.as_millis() > since_ms
&& timestamp.as_millis() <= now_ms
})
}
fn handle_query( fn handle_query(
index: &BTreeMap<IndexKey, DataValue>, index: &BTreeMap<IndexKey, DataValue>,
query: EngineQuery, query: EngineQuery,
@@ -1159,20 +1179,17 @@ fn project_labels(
.collect() .collect()
} }
/// Turns one series' occurrence points into a windowed rate: at each of the series' own timestamps, /// Thin adapter over the single shared windowed-rate implementation (`crate::rate`, ticket 040):
/// counts how many points fall in the trailing `(ts - range_ms, ts]` window and divides by the window /// PromQL's series here are plain millisecond `i64` timestamps rather than `Timestamp`, so this
/// width in seconds. Points are occurrence markers rather than cumulative counters, so this reads as /// just converts at the boundary and delegates — the actual window-scan logic lives only in
/// "occurrences per second over the trailing window", evaluated at every point instead of just the last. /// `crate::rate::windowed_rate_series`, also used (via `crate::rate::RateWindow`) by alert
/// `Threshold`/`Ratio` conditions in `crate::alerts`, so both agree on what "windowed rate" means.
fn windowed_rate_series(points: &[(i64, f64)], range_ms: i64) -> Vec<(i64, f64)> { fn windowed_rate_series(points: &[(i64, f64)], range_ms: i64) -> Vec<(i64, f64)> {
let range_seconds = range_ms as f64 / 1000.0; let points: Vec<(Timestamp, f64)> =
points points.iter().map(|(ms, value)| (Timestamp::from_millis(*ms), *value)).collect();
.iter() rate::windowed_rate_series(&points, range_ms)
.map(|(ts, _)| { .into_iter()
let window_start = ts - range_ms; .map(|(ts, value)| (ts.as_millis(), value))
let count_in_window =
points.iter().filter(|(t, _)| *t > window_start && *t <= *ts).count();
(*ts, count_in_window as f64 / range_seconds)
})
.collect() .collect()
} }
@@ -2058,6 +2075,69 @@ mod tests {
assert_eq!(series[1].points, vec![(0, 0.5)]); assert_eq!(series[1].points, vec![(0, 0.5)]);
} }
/// Ticket 040 acceptance criterion: a dashboard `rate()` chart and an alert `Threshold`
/// condition over the same metric/window agree on the computed rate for the same historical
/// data, since both now go through the single shared implementation in `crate::rate` rather
/// than each scanning/windowing independently.
#[test]
fn dashboard_rate_and_alert_threshold_agree_on_same_windowed_data() {
let mut index = BTreeMap::new();
declare_requests_metric(&mut index);
// Five matching lines 500ms apart; `ingest_log`'s `sequence` doubles as the millisecond
// timestamp (see its definition above), so these are also the log timestamps.
let lines: Vec<(u64, &str)> = vec![
(0, "request path=/a"),
(500, "request path=/a"),
(1000, "request path=/a"),
(1500, "request path=/a"),
(2000, "request path=/a"),
];
for (ms, text) in &lines {
ingest_log(&mut index, *ms, text);
}
// Dashboard side: rate() over a 2s window, read at the series' own last timestamp.
let parsed = promql::parse_query("rate(requests[2s])").unwrap();
let series = evaluate_promql(&index, &parsed, None, None);
assert_eq!(series.len(), 1);
let (_, dashboard_rate) = *series[0].points.last().unwrap();
// Alert side: a Threshold condition over the same query/window, replayed over the same
// lines in the same order. threshold=0/op=Ge always fires, so every fired message's
// reported match count is available to compare against the dashboard's rate.
let rule = alerts::AlertRule {
id: "r1".to_string(),
name: "spike".to_string(),
enabled: true,
condition: alerts::AlertCondition::Threshold {
query: r#"regex("request path=/a")"#.to_string(),
op: CompareOp::Ge,
threshold: 0.0,
window_ms: 2_000,
},
};
let mut alert_states = HashMap::new();
let mut last_message = String::new();
for (ms, text) in &lines {
let timestamp = Timestamp::from_millis(*ms as i64);
let fired = alerts::evaluate_alerts(&[rule.clone()], &mut alert_states, &index, *ms, timestamp, text);
assert_eq!(fired.len(), 1);
last_message = fired[0].1.clone();
}
let alert_count: f64 = last_message
.split("match count ")
.nth(1)
.and_then(|rest| rest.split_whitespace().next())
.and_then(|token| token.parse().ok())
.expect("fired message should report a match count");
let window_seconds = 2.0;
assert_eq!(dashboard_rate, 2.0);
assert_eq!(alert_count, 4.0);
assert_eq!(alert_count, dashboard_rate * window_seconds);
}
#[test] #[test]
fn sum_of_rate_without_by_clause_still_collapses_to_one_series() { fn sum_of_rate_without_by_clause_still_collapses_to_one_series() {
let mut index = BTreeMap::new(); let mut index = BTreeMap::new();
+1
View File
@@ -10,6 +10,7 @@ mod ingest;
mod metric_primitive; mod metric_primitive;
mod profile; mod profile;
mod promql; mod promql;
mod rate;
mod settings; mod settings;
mod timestamp; mod timestamp;
+143
View File
@@ -0,0 +1,143 @@
//! Ticket 040: the single windowed-rate/count implementation in the codebase, shared by
//! PromQL's `rate()` (`crate::engine::evaluate_promql`, via the `windowed_rate_series` adapter
//! there) and alert `Threshold`/`Ratio` conditions (`crate::alerts`). Previously each computed
//! its own trailing window independently — PromQL scanning indexed bucket data, alerts scanning
//! raw log-line matches with a hand-rolled `VecDeque` — and could disagree for equivalent-looking
//! rules simply because the window boundary was implemented twice.
//!
//! `count_in_window` is the one place that decides what "falls in the trailing window" means;
//! everything else in this module is built on top of it, either by batch-rescanning a full point
//! series (`rate_at`/`windowed_rate_series`, fine for PromQL's historical/one-shot evaluation) or
//! incrementally (`RateWindow`, for alerts, which see one point at a time and need cost
//! proportional to the window rather than a full rescan per ingested line).
use std::collections::VecDeque;
use crate::timestamp::Timestamp;
/// How many `points` fall in the trailing `(now - window_ms, now]` window. Points are occurrence
/// markers rather than a continuous reading, so only their timestamps matter — the boundary is
/// exclusive on the old end and inclusive of `now` itself, matching the original
/// `windowed_rate_series` behavior this replaces.
fn count_in_window(points: &[(Timestamp, f64)], now: Timestamp, window_ms: i64) -> usize {
let now_ms = now.as_millis();
let window_start = now_ms - window_ms;
points
.iter()
.filter(|(ts, _)| {
let ms = ts.as_millis();
ms > window_start && ms <= now_ms
})
.count()
}
/// Trailing-window occurrence rate (points per second) ending at `now`: how many `points` fall in
/// `(now - window_ms, now]`, divided by the window width in seconds.
pub fn rate_at(points: &[(Timestamp, f64)], now: Timestamp, window_ms: i64) -> f64 {
let window_seconds = window_ms as f64 / 1000.0;
count_in_window(points, now, window_ms) as f64 / window_seconds
}
/// Evaluates `rate_at` at every one of `points`' own timestamps, turning an occurrence series
/// into a windowed rate series — used by PromQL's `rate()` so it plots as a real line rather than
/// one trailing point.
pub fn windowed_rate_series(points: &[(Timestamp, f64)], window_ms: i64) -> Vec<(Timestamp, f64)> {
points.iter().map(|(ts, _)| (*ts, rate_at(points, *ts, window_ms))).collect()
}
/// Incrementally maintained trailing window: for a caller that sees points arrive one at a time
/// (alert `Threshold`/`Ratio` conditions, evaluated per ingested line) and needs cost proportional
/// to the window rather than `count_in_window`'s full rescan per call. Assumes points are pushed
/// in non-decreasing timestamp order, true for ingested log lines.
#[derive(Debug, Default, Clone)]
pub struct RateWindow {
timestamps: VecDeque<Timestamp>,
}
impl RateWindow {
/// Builds a window pre-populated from already-seen history (e.g. currently-ingested logs
/// matching an alert's query), so a freshly-created window starts with whatever of that
/// history already falls in the trailing window instead of empty.
pub fn backfill(timestamps: impl IntoIterator<Item = Timestamp>, now: Timestamp, window_ms: i64) -> Self {
let mut window = Self::default();
for ts in timestamps {
window.push(ts, window_ms);
}
window.prune(now, window_ms);
window
}
/// Records one occurrence at `timestamp` and drops anything that has aged out of the
/// resulting trailing window.
pub fn push(&mut self, timestamp: Timestamp, window_ms: i64) {
self.timestamps.push_back(timestamp);
self.prune(timestamp, window_ms);
}
/// Same boundary as `count_in_window`: drops entries at or before `now - window_ms`.
fn prune(&mut self, now: Timestamp, window_ms: i64) {
let window_start = now.as_millis() - window_ms;
while let Some(&front) = self.timestamps.front() {
if front.as_millis() <= window_start {
self.timestamps.pop_front();
} else {
break;
}
}
}
pub fn count(&self) -> usize {
self.timestamps.len()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn rate_at_matches_hand_computed_window() {
let points = vec![
(Timestamp::from_millis(0), 1.0),
(Timestamp::from_millis(1000), 1.0),
(Timestamp::from_millis(2000), 1.0),
];
// (1000, 2000]: only the point at 2000.
assert_eq!(rate_at(&points, Timestamp::from_millis(2000), 1000), 1.0);
// (0, 2000]: points at 1000 and 2000 -> 2 points / 2s.
assert_eq!(rate_at(&points, Timestamp::from_millis(2000), 2000), 1.0);
}
#[test]
fn windowed_rate_series_matches_rate_at_at_each_point() {
let points = vec![
(Timestamp::from_millis(0), 1.0),
(Timestamp::from_millis(500), 1.0),
(Timestamp::from_millis(1000), 1.0),
];
let series = windowed_rate_series(&points, 1000);
for (ts, rate) in &series {
assert_eq!(*rate, rate_at(&points, *ts, 1000));
}
}
#[test]
fn rate_window_incremental_count_matches_batch_count_in_window() {
let mut window = RateWindow::default();
let millis = [0, 200, 400, 1200];
for ms in millis {
window.push(Timestamp::from_millis(ms), 1000);
}
let points: Vec<(Timestamp, f64)> =
millis.iter().map(|ms| (Timestamp::from_millis(*ms), 1.0)).collect();
assert_eq!(window.count(), count_in_window(&points, Timestamp::from_millis(1200), 1000));
}
#[test]
fn rate_window_backfill_seeds_from_history_within_the_window() {
let history = [0, 500, 2000, 2500].map(Timestamp::from_millis);
// now=2500ms, window=1000ms -> window (1500, 2500]: only 2000 and 2500 qualify.
let window = RateWindow::backfill(history, Timestamp::from_millis(2500), 1000);
assert_eq!(window.count(), 2);
}
}