phughk/ottyl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Unify PromQL rate() and alert Threshold/Ratio windowing (ticket 040)

Browse Source 
Extract the trailing-window rate/count implementation into a new src/rate.rs,
shared by PromQL's rate() (engine::evaluate_promql, via a thin i64<->Timestamp
adapter) and alert Threshold/Ratio conditions (alerts::evaluate_condition, via
the new incremental RateWindow), replacing alerts' bespoke VecDeque+prune loop.
Both now agree on what "windowed rate" means for the same data, verified by a
new regression test (engine::tests::dashboard_rate_and_alert_threshold_agree_on_same_windowed_data).

Behavior change: alert Threshold/Ratio windows are now backfilled once, on
first evaluation, from already-ingested logs within their own trailing window
(bounded cost, not a full history rescan) - a rule created mid-session, or a
process that replays existing log content (e.g. --file tailing, which rereads
the whole file from the start on every process start), no longer starts with
a silently empty window. A bare restart with no replayed history (e.g. stdin
ingestion) still starts empty, since there is genuinely no log history to
backfill from in that case - alert state itself remains in-memory only.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XFUNZwCvz3kcod3352ZR2o
This commit is contained in:
Hugh Kaznowski
2026-09-05 22:34:13 +01:00
co-authored by Claude Sonnet 5
parent 85fb145be7
commit ef60dd267b
4 changed files with 328 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/alerts.rs
+90 -43
View File
@@ -1,9 +1,16 @@
//! Alert rules: definitions, trigger evaluation, and ntfy delivery.
//!
//! Alert queries are restricted to regex/sequence/timestamp/json[field] — a metric[param] comparison would require a full bucket rescan per line, breaking the proportional-cost guarantee. json[field] is fine here since, like regex/sequence/timestamp, it only inspects the line's own text/fields, no index lookup.
//!
//! `Threshold`/`Ratio` windows are maintained via `crate::rate::RateWindow` — the same windowed-rate
//! implementation PromQL's `rate()` uses (`crate::engine::evaluate_promql`) — rather than a
//! bespoke `VecDeque`+prune loop, so the two agree on what "windowed rate" means (ticket 040). A
//! window is also backfilled once, on first evaluation, from already-ingested logs within its own
//! trailing window (see `backfill_window`): a rule created mid-session (or a process that replays
//! existing log content, e.g. `--file` tailing) no longer starts with a silently empty window.
use std::collections::hash_map::DefaultHasher;
use std::collections::{HashMap, HashSet, VecDeque};
use std::collections::{BTreeMap, HashMap, HashSet};
use std::fmt;
use std::hash::{Hash, Hasher};
use std::io;
@@ -13,9 +20,12 @@ use std::time::{SystemTime, UNIX_EPOCH};
use regex::Regex;
use serde::{Deserialize, Serialize};
use crate::engine::{apply_compare_op, eval_json_comparison};
use crate::data_value::DataValue;
use crate::engine::{self, apply_compare_op, eval_json_comparison};
use crate::filter_datetime;
use crate::filter_parser::{self, CompareOp, FilterExpr, FilterValue};
use crate::index_key::IndexKey;
use crate::rate::RateWindow;
use crate::settings::Settings;
use crate::timestamp::Timestamp;
@@ -135,24 +145,43 @@ impl Settings {
}
}
/// Per-rule in-memory evaluation state; lives only for the process's lifetime — not persisted, not backfilled against already-ingested logs.
/// Per-rule in-memory evaluation state; lives only for the process's lifetime — not persisted
/// across a restart with no replayed log history. Each `RateWindow` is backfilled once (see
/// `backfill_window`) from whatever matching history is already in the index at first evaluation,
/// so — unlike before ticket 040 — a rule doesn't start blind to matches ingested before it was
/// declared, or before this evaluation loop last started, as long as that history is still in the
/// in-memory log index (e.g. `--file` tailing replays existing file content from the start on
/// every process start).
#[derive(Debug, Default)]
pub struct AlertRuntimeState {
threshold_matches: VecDeque<i64>,
ratio_a_matches: VecDeque<i64>,
ratio_b_matches: VecDeque<i64>,
threshold_window: RateWindow,
threshold_backfilled: bool,
ratio_a_window: RateWindow,
ratio_a_backfilled: bool,
ratio_b_window: RateWindow,
ratio_b_backfilled: bool,
seen_values: HashSet<String>,
}
/// `timestamps` is push_back-ordered, so once the front entry is within the window, every entry behind it is too.
fn prune_window(timestamps: &mut VecDeque<i64>, now_ms: i64, window_ms: i64) {
while let Some(&front) = timestamps.front() {
if now_ms - front > window_ms {
timestamps.pop_front();
} else {
break;
}
}
/// Builds a `RateWindow` seeded from already-ingested logs matching `query`, bounded to
/// `window_ms` before `now` (not the full log history) so the cost stays proportional to the
/// window rather than everything ever ingested. `before_sequence` excludes the line currently
/// being evaluated, which the caller pushes separately if it matches.
fn backfill_window(
index: &BTreeMap<IndexKey, DataValue>,
query: &str,
before_sequence: u64,
now: Timestamp,
window_ms: i64,
) -> RateWindow {
let since_ms = now.as_millis() - window_ms;
let matching_timestamps =
engine::logs_since(index, before_sequence, since_ms, now.as_millis()).filter_map(
|(sequence, timestamp, text)| {
(eval_alert_query(query, sequence, timestamp, text) == Ok(true)).then_some(timestamp)
},
);
RateWindow::backfill(matching_timestamps, now, window_ms)
}
/// Evaluates a FilterBar query against a single line's own fields only (metric[param] comparisons aren't supported — see module doc; json[field] is supported since it's also line-local).
@@ -205,6 +234,7 @@ fn eval_alert_expr(expr: &FilterExpr, sequence: u64, timestamp: Timestamp, text:
pub fn evaluate_alerts(
rules: &[AlertRule],
states: &mut HashMap<String, AlertRuntimeState>,
index: &BTreeMap<IndexKey, DataValue>,
sequence: u64,
timestamp: Timestamp,
text: &str,
@@ -215,7 +245,9 @@ pub fn evaluate_alerts(
continue;
}
let state = states.entry(rule.id.clone()).or_default();
if let Some(message) = evaluate_condition(&rule.condition, state, sequence, timestamp, text, &rule.name) {
if let Some(message) =
evaluate_condition(&rule.condition, state, index, sequence, timestamp, text, &rule.name)
{
fired.push((rule.clone(), message));
}
}
@@ -225,6 +257,7 @@ pub fn evaluate_alerts(
fn evaluate_condition(
condition: &AlertCondition,
state: &mut AlertRuntimeState,
index: &BTreeMap<IndexKey, DataValue>,
sequence: u64,
timestamp: Timestamp,
text: &str,
@@ -240,12 +273,14 @@ fn evaluate_condition(
}
},
AlertCondition::Threshold { query, op, threshold, window_ms } => {
if !state.threshold_backfilled {
state.threshold_window = backfill_window(index, query, sequence, timestamp, *window_ms);
state.threshold_backfilled = true;
}
match eval_alert_query(query, sequence, timestamp, text) {
Ok(true) => {
let now_ms = timestamp.as_millis();
state.threshold_matches.push_back(now_ms);
prune_window(&mut state.threshold_matches, now_ms, *window_ms);
let count = state.threshold_matches.len() as f64;
state.threshold_window.push(timestamp, *window_ms);
let count = state.threshold_window.count() as f64;
apply_compare_op(count.total_cmp(threshold), *op).then(|| {
format!("{rule_name}: match count {count} {op:?} threshold {threshold} (window {window_ms}ms)")
})
@@ -285,17 +320,22 @@ fn evaluate_condition(
if !matched_a && !matched_b {
return None;
}
let now_ms = timestamp.as_millis();
if !state.ratio_a_backfilled {
state.ratio_a_window = backfill_window(index, query_a, sequence, timestamp, *window_ms);
state.ratio_a_backfilled = true;
}
if !state.ratio_b_backfilled {
state.ratio_b_window = backfill_window(index, query_b, sequence, timestamp, *window_ms);
state.ratio_b_backfilled = true;
}
if matched_a {
state.ratio_a_matches.push_back(now_ms);
prune_window(&mut state.ratio_a_matches, now_ms, *window_ms);
state.ratio_a_window.push(timestamp, *window_ms);
}
if matched_b {
state.ratio_b_matches.push_back(now_ms);
prune_window(&mut state.ratio_b_matches, now_ms, *window_ms);
state.ratio_b_window.push(timestamp, *window_ms);
}
let count_a = state.ratio_a_matches.len() as f64;
let count_b = state.ratio_b_matches.len() as f64;
let count_a = state.ratio_a_window.count() as f64;
let count_b = state.ratio_b_window.count() as f64;
if count_b == 0.0 {
// Zero denominator: skip rather than treat the ratio as infinite.
return None;
@@ -417,12 +457,13 @@ mod tests {
fn immediate_fires_on_matching_line_only() {
let rules = vec![immediate_rule(r#"regex("error")"#)];
let mut states = HashMap::new();
let index: BTreeMap<IndexKey, DataValue> = BTreeMap::new();
let (seq, ts, text) = line("all good", 0, 0);
assert!(evaluate_alerts(&rules, &mut states, seq, ts, text).is_empty());
assert!(evaluate_alerts(&rules, &mut states, &index, seq, ts, text).is_empty());
let (seq, ts, text) = line("an error happened", 1, 1);
let fired = evaluate_alerts(&rules, &mut states, seq, ts, text);
let fired = evaluate_alerts(&rules, &mut states, &index, seq, ts, text);
assert_eq!(fired.len(), 1);
assert!(fired[0].1.contains("an error happened"));
}
@@ -432,8 +473,9 @@ mod tests {
let mut rule = immediate_rule(r#"regex("error")"#);
rule.enabled = false;
let mut states = HashMap::new();
let index: BTreeMap<IndexKey, DataValue> = BTreeMap::new();
let (seq, ts, text) = line("error!", 0, 0);
assert!(evaluate_alerts(&[rule], &mut states, seq, ts, text).is_empty());
assert!(evaluate_alerts(&[rule], &mut states, &index, seq, ts, text).is_empty());
}
#[test]
@@ -450,14 +492,15 @@ mod tests {
},
};
let mut states = HashMap::new();
let index: BTreeMap<IndexKey, DataValue> = BTreeMap::new();
for (seq, ms) in [(0u64, 0i64), (1, 1000)] {
let fired = evaluate_alerts(&[rule.clone()], &mut states, seq, Timestamp::from_millis(ms), "error");
let fired = evaluate_alerts(&[rule.clone()], &mut states, &index, seq, Timestamp::from_millis(ms), "error");
assert!(fired.is_empty(), "should not fire before threshold is crossed");
}
let fired = evaluate_alerts(&[rule.clone()], &mut states, 2, Timestamp::from_millis(2000), "error");
let fired = evaluate_alerts(&[rule.clone()], &mut states, &index, 2, Timestamp::from_millis(2000), "error");
assert_eq!(fired.len(), 1);
// Stays quiet for a non-matching line even though the window is full.
let fired = evaluate_alerts(&[rule], &mut states, 3, Timestamp::from_millis(2100), "fine");
let fired = evaluate_alerts(&[rule], &mut states, &index, 3, Timestamp::from_millis(2100), "fine");
assert!(fired.is_empty());
}
@@ -475,9 +518,10 @@ mod tests {
},
};
let mut states = HashMap::new();
assert!(evaluate_alerts(&[rule.clone()], &mut states, 0, Timestamp::from_millis(0), "error").is_empty());
let index: BTreeMap<IndexKey, DataValue> = BTreeMap::new();
assert!(evaluate_alerts(&[rule.clone()], &mut states, &index, 0, Timestamp::from_millis(0), "error").is_empty());
// Second match arrives well after the first has aged out of the window.
assert!(evaluate_alerts(&[rule], &mut states, 1, Timestamp::from_millis(5_000), "error").is_empty());
assert!(evaluate_alerts(&[rule], &mut states, &index, 1, Timestamp::from_millis(5_000), "error").is_empty());
}
#[test]
@@ -492,11 +536,12 @@ mod tests {
},
};
let mut states = HashMap::new();
let fired = evaluate_alerts(&[rule.clone()], &mut states, 0, Timestamp::from_millis(0), "agent=curl");
let index: BTreeMap<IndexKey, DataValue> = BTreeMap::new();
let fired = evaluate_alerts(&[rule.clone()], &mut states, &index, 0, Timestamp::from_millis(0), "agent=curl");
assert_eq!(fired.len(), 1);
let fired = evaluate_alerts(&[rule.clone()], &mut states, 1, Timestamp::from_millis(1), "agent=curl");
let fired = evaluate_alerts(&[rule.clone()], &mut states, &index, 1, Timestamp::from_millis(1), "agent=curl");
assert!(fired.is_empty());
let fired = evaluate_alerts(&[rule], &mut states, 2, Timestamp::from_millis(2), "agent=wget");
let fired = evaluate_alerts(&[rule], &mut states, &index, 2, Timestamp::from_millis(2), "agent=wget");
assert_eq!(fired.len(), 1);
}
@@ -515,16 +560,17 @@ mod tests {
},
};
let mut states = HashMap::new();
let index: BTreeMap<IndexKey, DataValue> = BTreeMap::new();
// Only an "error" so far; no "request" yet, so denominator is 0 -> skip.
assert!(evaluate_alerts(&[rule.clone()], &mut states, 0, Timestamp::from_millis(0), "error").is_empty());
assert!(evaluate_alerts(&[rule.clone()], &mut states, &index, 0, Timestamp::from_millis(0), "error").is_empty());
// One request: ratio 1/1 = 1.0 >= 0.5 -> fires.
let fired = evaluate_alerts(&[rule.clone()], &mut states, 1, Timestamp::from_millis(1), "request");
let fired = evaluate_alerts(&[rule.clone()], &mut states, &index, 1, Timestamp::from_millis(1), "request");
assert_eq!(fired.len(), 1);
// A second request: ratio 1/2 = 0.5 still >= 0.5 -> fires again.
let fired = evaluate_alerts(&[rule.clone()], &mut states, 2, Timestamp::from_millis(2), "request");
let fired = evaluate_alerts(&[rule.clone()], &mut states, &index, 2, Timestamp::from_millis(2), "request");
assert_eq!(fired.len(), 1);
// A third request with no more errors: ratio drops to 1/3 -> no longer fires.
let fired = evaluate_alerts(&[rule], &mut states, 3, Timestamp::from_millis(3), "request");
let fired = evaluate_alerts(&[rule], &mut states, &index, 3, Timestamp::from_millis(3), "request");
assert!(fired.is_empty());
}
@@ -532,8 +578,9 @@ mod tests {
fn metric_comparison_in_alert_query_is_rejected_not_panicking() {
let rule = immediate_rule("some_metric[param]==5");
let mut states = HashMap::new();
let index: BTreeMap<IndexKey, DataValue> = BTreeMap::new();
// Should log a warning and simply not fire, never panic.
let fired = evaluate_alerts(&[rule], &mut states, 0, Timestamp::from_millis(0), "anything");
let fired = evaluate_alerts(&[rule], &mut states, &index, 0, Timestamp::from_millis(0), "anything");
assert!(fired.is_empty());
}