From 529f79e9da5f53435919142ce399e8961dea9eb6 Mon Sep 17 00:00:00 2001 From: Hugh Kaznowski Date: Sun, 6 Sep 2026 17:03:19 +0100 Subject: [PATCH] Convert engine.rs index-manipulating functions into EngineSnapshot methods Free functions that took the engine's index/sort-helper-index state as a parameter now live on EngineSnapshot (or, for a handful of clean single-receiver cases, on MetricPrimitive/CompareOp/CompiledFilter/ EngineQuery). Route handlers in api.rs and alert evaluation in alerts.rs are updated to call the new methods; alerts.rs now threads an &EngineSnapshot through instead of a raw &BTreeMap so the per-line alert backfill path doesn't clone the whole index just to call a method. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01A1ZJ3SrayzLzVndhVq9BmS --- src/alerts.rs | 48 +- src/api.rs | 6 +- src/engine.rs | 2474 ++++++++++++++++++++++++------------------------ src/profile.rs | 2 +- 4 files changed, 1241 insertions(+), 1289 deletions(-) diff --git a/src/alerts.rs b/src/alerts.rs index e9f3f19..1c80398 100644 --- a/src/alerts.rs +++ b/src/alerts.rs @@ -1,5 +1,5 @@ use std::collections::hash_map::DefaultHasher; -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{HashMap, HashSet}; use std::fmt; use std::hash::{Hash, Hasher}; use std::io; @@ -9,11 +9,9 @@ use std::time::{SystemTime, UNIX_EPOCH}; use regex::Regex; use serde::{Deserialize, Serialize}; -use crate::data_value::DataValue; -use crate::engine::{self, apply_compare_op, eval_json_comparison}; +use crate::engine::{self, eval_json_comparison}; use crate::filter_datetime; use crate::filter_parser::{self, CompareOp, FilterExpr, FilterValue}; -use crate::index_key::IndexKey; use crate::rate::RateWindow; use crate::settings::Settings; use crate::timestamp::Timestamp; @@ -142,7 +140,7 @@ pub struct AlertRuntimeState { } fn backfill_window( - index: &BTreeMap, + snapshot: &engine::EngineSnapshot, query: &str, before_sequence: u64, now: Timestamp, @@ -150,7 +148,7 @@ fn backfill_window( ) -> RateWindow { let since_ms = now.as_millis() - window_ms; let matching_timestamps = - engine::logs_since(index, before_sequence, since_ms, now.as_millis()).filter_map( + snapshot.logs_since(before_sequence, since_ms, now.as_millis()).filter_map( |(sequence, timestamp, text)| { (eval_alert_query(query, sequence, timestamp, text) == Ok(true)).then_some(timestamp) }, @@ -176,7 +174,7 @@ fn eval_alert_expr(expr: &FilterExpr, sequence: u64, timestamp: Timestamp, text: .parse::() .map_err(|_| format!("'ottyl_seq' comparisons require a numeric value, got {raw:?}"))?, }; - Ok(apply_compare_op((sequence as f64).total_cmp(&target), *op)) + Ok(op.matches((sequence as f64).total_cmp(&target))) } FilterExpr::Comparison { metric, param: None, op, value } if metric == "ottyl_ts" => { let target_ms = match value { @@ -185,14 +183,14 @@ fn eval_alert_expr(expr: &FilterExpr, sequence: u64, timestamp: Timestamp, text: .map_err(|err| err.to_string())? .as_millis(), }; - Ok(apply_compare_op(timestamp.as_millis().cmp(&target_ms), *op)) + Ok(op.matches(timestamp.as_millis().cmp(&target_ms))) } FilterExpr::Comparison { metric, param: None, op, value } if metric == "ottyl_fmt" => { let target = match value { FilterValue::Number(n) => n.to_string(), FilterValue::Raw(raw) => raw.clone(), }; - Ok(apply_compare_op(crate::logfmt::detect_format(text).cmp(target.as_str()), *op)) + Ok(op.matches(crate::logfmt::detect_format(text).cmp(target.as_str()))) } FilterExpr::Comparison { metric, .. } => Err(format!( "alert queries only support regex()/ottyl_seq/ottyl_ts/ottyl_fmt (not metric comparisons like \ @@ -213,7 +211,7 @@ fn eval_alert_expr(expr: &FilterExpr, sequence: u64, timestamp: Timestamp, text: pub fn evaluate_alerts( rules: &[AlertRule], states: &mut HashMap, - index: &BTreeMap, + snapshot: &engine::EngineSnapshot, sequence: u64, timestamp: Timestamp, text: &str, @@ -225,7 +223,7 @@ pub fn evaluate_alerts( } let state = states.entry(rule.id.clone()).or_default(); if let Some(message) = - evaluate_condition(&rule.condition, state, index, sequence, timestamp, text, &rule.name) + evaluate_condition(&rule.condition, state, snapshot, sequence, timestamp, text, &rule.name) { fired.push((rule.clone(), message)); } @@ -236,7 +234,7 @@ pub fn evaluate_alerts( fn evaluate_condition( condition: &AlertCondition, state: &mut AlertRuntimeState, - index: &BTreeMap, + snapshot: &engine::EngineSnapshot, sequence: u64, timestamp: Timestamp, text: &str, @@ -253,14 +251,14 @@ fn evaluate_condition( }, AlertCondition::Threshold { query, op, threshold, window_ms } => { if !state.threshold_backfilled { - state.threshold_window = backfill_window(index, query, sequence, timestamp, *window_ms); + state.threshold_window = backfill_window(snapshot, query, sequence, timestamp, *window_ms); state.threshold_backfilled = true; } match eval_alert_query(query, sequence, timestamp, text) { Ok(true) => { state.threshold_window.push(timestamp, *window_ms); let count = state.threshold_window.count() as f64; - apply_compare_op(count.total_cmp(threshold), *op).then(|| { + op.matches(count.total_cmp(threshold)).then(|| { format!("{rule_name}: match count {count} {op:?} threshold {threshold} (window {window_ms}ms)") }) } @@ -300,11 +298,11 @@ fn evaluate_condition( return None; } if !state.ratio_a_backfilled { - state.ratio_a_window = backfill_window(index, query_a, sequence, timestamp, *window_ms); + state.ratio_a_window = backfill_window(snapshot, query_a, sequence, timestamp, *window_ms); state.ratio_a_backfilled = true; } if !state.ratio_b_backfilled { - state.ratio_b_window = backfill_window(index, query_b, sequence, timestamp, *window_ms); + state.ratio_b_window = backfill_window(snapshot, query_b, sequence, timestamp, *window_ms); state.ratio_b_backfilled = true; } if matched_a { @@ -319,7 +317,7 @@ fn evaluate_condition( return None; } let observed = count_a / count_b; - apply_compare_op(observed.total_cmp(ratio), *op).then(|| { + op.matches(observed.total_cmp(ratio)).then(|| { format!( "{rule_name}: ratio {observed:.3} ({count_a}/{count_b}) {op:?} {ratio} (window {window_ms}ms)" ) @@ -431,7 +429,7 @@ mod tests { fn immediate_fires_on_matching_line_only() { let rules = vec![immediate_rule(r#"regex("error")"#)]; let mut states = HashMap::new(); - let index: BTreeMap = BTreeMap::new(); + let index = engine::EngineSnapshot::default(); let (seq, ts, text) = line("all good", 0, 0); assert!(evaluate_alerts(&rules, &mut states, &index, seq, ts, text).is_empty()); @@ -447,7 +445,7 @@ mod tests { let mut rule = immediate_rule(r#"regex("error")"#); rule.enabled = false; let mut states = HashMap::new(); - let index: BTreeMap = BTreeMap::new(); + let index = engine::EngineSnapshot::default(); let (seq, ts, text) = line("error!", 0, 0); assert!(evaluate_alerts(&[rule], &mut states, &index, seq, ts, text).is_empty()); } @@ -466,7 +464,7 @@ mod tests { }, }; let mut states = HashMap::new(); - let index: BTreeMap = BTreeMap::new(); + let index = engine::EngineSnapshot::default(); for (seq, ms) in [(0u64, 0i64), (1, 1000)] { let fired = evaluate_alerts(&[rule.clone()], &mut states, &index, seq, Timestamp::from_millis(ms), "error"); assert!(fired.is_empty(), "should not fire before threshold is crossed"); @@ -491,7 +489,7 @@ mod tests { }, }; let mut states = HashMap::new(); - let index: BTreeMap = BTreeMap::new(); + let index = engine::EngineSnapshot::default(); assert!(evaluate_alerts(&[rule.clone()], &mut states, &index, 0, Timestamp::from_millis(0), "error").is_empty()); assert!(evaluate_alerts(&[rule], &mut states, &index, 1, Timestamp::from_millis(5_000), "error").is_empty()); } @@ -508,7 +506,7 @@ mod tests { }, }; let mut states = HashMap::new(); - let index: BTreeMap = BTreeMap::new(); + let index = engine::EngineSnapshot::default(); let fired = evaluate_alerts(&[rule.clone()], &mut states, &index, 0, Timestamp::from_millis(0), "agent=curl"); assert_eq!(fired.len(), 1); let fired = evaluate_alerts(&[rule.clone()], &mut states, &index, 1, Timestamp::from_millis(1), "agent=curl"); @@ -532,7 +530,7 @@ mod tests { }, }; let mut states = HashMap::new(); - let index: BTreeMap = BTreeMap::new(); + let index = engine::EngineSnapshot::default(); assert!(evaluate_alerts(&[rule.clone()], &mut states, &index, 0, Timestamp::from_millis(0), "error").is_empty()); let fired = evaluate_alerts(&[rule.clone()], &mut states, &index, 1, Timestamp::from_millis(1), "request"); assert_eq!(fired.len(), 1); @@ -546,7 +544,7 @@ mod tests { fn ottyl_fmt_alert_query_matches_detected_format() { let rule = immediate_rule("ottyl_fmt==json"); let mut states = HashMap::new(); - let index: BTreeMap = BTreeMap::new(); + let index = engine::EngineSnapshot::default(); let fired = evaluate_alerts( &[rule.clone()], @@ -566,7 +564,7 @@ mod tests { fn metric_comparison_in_alert_query_is_rejected_not_panicking() { let rule = immediate_rule("some_metric[param]==5"); let mut states = HashMap::new(); - let index: BTreeMap = BTreeMap::new(); + let index = engine::EngineSnapshot::default(); let fired = evaluate_alerts(&[rule], &mut states, &index, 0, Timestamp::from_millis(0), "anything"); assert!(fired.is_empty()); } diff --git a/src/api.rs b/src/api.rs index 45c61bd..144be96 100644 --- a/src/api.rs +++ b/src/api.rs @@ -217,9 +217,9 @@ fn mime_for_extension(extension: &str) -> &'static str { async fn ask(engine: &EngineHandle, query: EngineQuery) -> Result { let read_handle = engine.reader_factory.handle(); - if let Some(sort_by) = crate::engine::query_sort_by(&query) { + if let Some(sort_by) = query.sort_by() { let needs_build = match read_handle.enter() { - Some(guard) => !crate::engine::has_sort_helper_index(&guard, sort_by), + Some(guard) => !guard.has_sort_helper_index(sort_by), None => return Err(StatusCode::SERVICE_UNAVAILABLE), }; if needs_build { @@ -233,7 +233,7 @@ async fn ask(engine: &EngineHandle, query: EngineQuery) -> Result { max_logs = new_max; write.append(EngineOp(Box::new(move |snap: &mut EngineSnapshot| { - evict_excess_logs(&mut snap.index, &mut snap.log_count, new_max, &mut snap.approx_bytes_total); + snap.evict_excess_logs(new_max); }))); write.publish(); } EngineMessage::DeclareParam { request, respond_to } => { let result = write .enter() - .map(|guard| validate_param_declaration(&guard.index, &request)) + .map(|guard| guard.validate_param_declaration(&request)) .unwrap_or_else(|| Err("engine is shutting down".to_string())); let succeeded = result.is_ok(); let name = request.name.clone(); @@ -287,7 +281,7 @@ async fn run( write.append(EngineOp(Box::new(move |snap: &mut EngineSnapshot| { snap.sort_helper_indexes.remove(&name); if succeeded { - insert_param_declaration(&mut snap.index, request_for_op.clone()); + snap.insert_param_declaration(request_for_op.clone()); } }))); write.publish(); @@ -306,12 +300,14 @@ async fn run( let _ = respond_to.send(existed); } EngineMessage::ClearParams { respond_to } => { - let mut scratch = write.enter().map(|guard| guard.index.clone()).unwrap_or_default(); - let removed = clear_all_param_declarations(&mut scratch); - let (reextracted, logs_considered) = reextract_params_from_retained_logs(&mut scratch); + let mut scratch = EngineSnapshot::from_index( + write.enter().map(|guard| guard.index.clone()).unwrap_or_default(), + ); + let removed = scratch.clear_all_param_declarations(); + let (reextracted, logs_considered) = scratch.reextract_params_from_retained_logs(); write.append(EngineOp(Box::new(move |snap: &mut EngineSnapshot| { - clear_all_param_declarations(&mut snap.index); - reextract_params_from_retained_logs(&mut snap.index); + snap.clear_all_param_declarations(); + snap.reextract_params_from_retained_logs(); snap.sort_helper_indexes.clear(); }))); write.publish(); @@ -320,13 +316,13 @@ async fn run( EngineMessage::DeclareMetric { request, respond_to } => { let result = write .enter() - .map(|guard| validate_metric_request(&guard.index, &request).map(|_| ())) + .map(|guard| guard.validate_metric_request(&request).map(|_| ())) .unwrap_or_else(|| Err("engine is shutting down".to_string())); let succeeded = result.is_ok(); let request_for_op = request.clone(); write.append(EngineOp(Box::new(move |snap: &mut EngineSnapshot| { if succeeded { - let _ = declare_metric(&mut snap.index, request_for_op.clone()); + let _ = snap.declare_metric(request_for_op.clone()); } }))); write.publish(); @@ -335,14 +331,14 @@ async fn run( EngineMessage::UpdateMetric { name, request, respond_to } => { let result = write .enter() - .map(|guard| validate_update_metric(&guard.index, &name, &request).map(|_| ())) + .map(|guard| guard.validate_update_metric(&name, &request).map(|_| ())) .unwrap_or_else(|| Err("engine is shutting down".to_string())); let succeeded = result.is_ok(); let existing_name = name.clone(); let request_for_op = request.clone(); write.append(EngineOp(Box::new(move |snap: &mut EngineSnapshot| { if succeeded { - let _ = update_metric(&mut snap.index, &existing_name, request_for_op.clone()); + let _ = snap.update_metric(&existing_name, request_for_op.clone()); } }))); write.publish(); @@ -354,14 +350,14 @@ async fn run( .is_some_and(|guard| guard.index.contains_key(&IndexKey::MetricDeclaration { name: name.clone() })); let name_for_op = name.clone(); write.append(EngineOp(Box::new(move |snap: &mut EngineSnapshot| { - remove_metric_declaration(&mut snap.index, &name_for_op); + snap.remove_metric_declaration(&name_for_op); }))); write.publish(); let _ = respond_to.send(existed); } EngineMessage::EnsureSortHelperIndex { param_name, respond_to } => { write.append(EngineOp(Box::new(move |snap: &mut EngineSnapshot| { - ensure_sort_helper_index_built(&mut snap.sort_helper_indexes, &snap.index, ¶m_name); + snap.ensure_sort_helper_index_built(¶m_name); }))); write.publish(); let _ = respond_to.send(()); @@ -373,60 +369,56 @@ async fn run( } } -fn validate_param_declaration( - index: &BTreeMap, - request: &DeclareParamRequest, -) -> Result<(), String> { - if OTTYL_RESERVED_PARAM_NAMES.contains(&request.name.as_str()) { - return Err(format!( - "'{}' is a reserved name, present on every log line without declaring anything; \ - it can't be redeclared as a param", - request.name - )); - } - if let Some(DataValue::ParamDeclaration(existing)) = - index.get(&IndexKey::ParamDeclaration { name: request.name.clone() }) - { - if existing.auto_derived { +impl EngineSnapshot { + fn validate_param_declaration(&self, request: &DeclareParamRequest) -> Result<(), String> { + if OTTYL_RESERVED_PARAM_NAMES.contains(&request.name.as_str()) { return Err(format!( - "param '{}' was auto-detected from the log stream and can't be edited", + "'{}' is a reserved name, present on every log line without declaring anything; \ + it can't be redeclared as a param", request.name )); } + if let Some(DataValue::ParamDeclaration(existing)) = + self.index.get(&IndexKey::ParamDeclaration { name: request.name.clone() }) + { + if existing.auto_derived { + return Err(format!( + "param '{}' was auto-detected from the log stream and can't be edited", + request.name + )); + } + } + let re = Regex::new(&request.message_match_regex) + .map_err(|err| format!("invalid regex: {err}"))?; + if request.elapsed_since_last && request.kind != MetricPrimitiveKind::Duration { + return Err("elapsed_since_last params must declare kind 'duration'".to_string()); + } + let uses_json_field = request.json_field_path.as_deref().is_some_and(|path| !path.is_empty()); + if request.kind != MetricPrimitiveKind::Boolean + && !request.elapsed_since_last + && !uses_json_field + && !re.capture_names().flatten().any(|name| name == request.capture_group) + { + return Err(format!( + "regex has no capture group named '{}'", + request.capture_group + )); + } + Ok(()) } - let re = Regex::new(&request.message_match_regex) - .map_err(|err| format!("invalid regex: {err}"))?; - if request.elapsed_since_last && request.kind != MetricPrimitiveKind::Duration { - return Err("elapsed_since_last params must declare kind 'duration'".to_string()); - } - let uses_json_field = request.json_field_path.as_deref().is_some_and(|path| !path.is_empty()); - if request.kind != MetricPrimitiveKind::Boolean - && !request.elapsed_since_last - && !uses_json_field - && !re.capture_names().flatten().any(|name| name == request.capture_group) - { - return Err(format!( - "regex has no capture group named '{}'", - request.capture_group - )); - } - Ok(()) -} -fn insert_param_declaration(index: &mut BTreeMap, request: DeclareParamRequest) { - index.insert( - IndexKey::ParamDeclaration { name: request.name.clone() }, - DataValue::ParamDeclaration(ParamDeclaration { auto_derived: false, ..request }), - ); + fn insert_param_declaration(&mut self, request: DeclareParamRequest) { + self.index.insert( + IndexKey::ParamDeclaration { name: request.name.clone() }, + DataValue::ParamDeclaration(ParamDeclaration { auto_derived: false, ..request }), + ); + } } #[cfg(test)] -fn declare_param( - index: &mut BTreeMap, - request: DeclareParamRequest, -) -> Result<(), String> { - validate_param_declaration(index, &request)?; - insert_param_declaration(index, request); +fn declare_param(snapshot: &mut EngineSnapshot, request: DeclareParamRequest) -> Result<(), String> { + snapshot.validate_param_declaration(&request)?; + snapshot.insert_param_declaration(request); Ok(()) } @@ -458,10 +450,6 @@ fn build_auto_param_pattern(key: &str, group: &str, quoted: bool) -> String { -fn auto_derived_param_count(index: &BTreeMap) -> usize { - param_declarations_range(index).filter(|declaration| declaration.6).count() -} - struct AutoParamCandidate { name: String, value: String, @@ -470,62 +458,68 @@ struct AutoParamCandidate { json_field_path: Option, } -fn auto_declare_params(index: &mut BTreeMap, text: &str) { - let mut total = auto_derived_param_count(index); - if total >= MAX_AUTO_DECLARED_PARAMS { - return; +impl EngineSnapshot { + fn auto_derived_param_count(&self) -> usize { + self.param_declarations_range().filter(|declaration| declaration.6).count() } - let candidates: Vec = if crate::logfmt::detect_format(text) == "json" { - crate::data_value::flatten_json_object(text) - .into_iter() - .map(|(path, value)| AutoParamCandidate { - name: path.clone(), - value, - message_match_regex: String::new(), - capture_group: String::new(), - json_field_path: Some(path), - }) - .collect() - } else { - crate::logfmt::extract_pairs(text) - .into_iter() - .map(|(key, value, quoted)| { - let group = sanitize_group_name(key); - AutoParamCandidate { - name: key.to_string(), - value: value.to_string(), - message_match_regex: build_auto_param_pattern(key, &group, quoted), - capture_group: group, - json_field_path: None, - } - }) - .collect() - }; - let mut new_this_line = 0; - for candidate in candidates { - if new_this_line >= MAX_NEW_AUTO_PARAMS_PER_LINE || total >= MAX_AUTO_DECLARED_PARAMS { - break; + fn auto_declare_params(&mut self, text: &str) { + let mut total = self.auto_derived_param_count(); + if total >= MAX_AUTO_DECLARED_PARAMS { + return; } - let index_key = IndexKey::ParamDeclaration { name: candidate.name.clone() }; - if index.contains_key(&index_key) { - continue; + let candidates: Vec = if crate::logfmt::detect_format(text) == "json" { + crate::data_value::flatten_json_object(text) + .into_iter() + .map(|(path, value)| AutoParamCandidate { + name: path.clone(), + value, + message_match_regex: String::new(), + capture_group: String::new(), + json_field_path: Some(path), + }) + .collect() + } else { + crate::logfmt::extract_pairs(text) + .into_iter() + .map(|(key, value, quoted)| { + let group = sanitize_group_name(key); + AutoParamCandidate { + name: key.to_string(), + value: value.to_string(), + message_match_regex: build_auto_param_pattern(key, &group, quoted), + capture_group: group, + json_field_path: None, + } + }) + .collect() + }; + + let mut new_this_line = 0; + for candidate in candidates { + if new_this_line >= MAX_NEW_AUTO_PARAMS_PER_LINE || total >= MAX_AUTO_DECLARED_PARAMS { + break; + } + let index_key = IndexKey::ParamDeclaration { name: candidate.name.clone() }; + if self.index.contains_key(&index_key) { + continue; + } + self.index.insert( + index_key, + DataValue::ParamDeclaration(ParamDeclaration { + name: candidate.name, + message_match_regex: candidate.message_match_regex, + capture_group: candidate.capture_group, + json_field_path: candidate.json_field_path, + kind: infer_kind(&candidate.value), + elapsed_since_last: false, + auto_derived: true, + log_stream_gated: false, + }), + ); + new_this_line += 1; + total += 1; } - index.insert( - index_key, - DataValue::ParamDeclaration(ParamDeclaration { - name: candidate.name, - message_match_regex: candidate.message_match_regex, - capture_group: candidate.capture_group, - json_field_path: candidate.json_field_path, - kind: infer_kind(&candidate.value), - elapsed_since_last: false, - auto_derived: true, - log_stream_gated: false, - }), - ); - new_this_line += 1; - total += 1; } } @@ -599,147 +593,140 @@ fn auto_declare_log_stream_params( } } -fn clear_all_param_declarations(index: &mut BTreeMap) -> usize { - let names: Vec = - param_declarations_range(index).map(|(name, ..)| name.to_string()).collect(); - let removed = names.len(); - for name in names { - index.remove(&IndexKey::ParamDeclaration { name }); - } - removed -} - -fn reextract_params_from_retained_logs(index: &mut BTreeMap) -> (usize, usize) { - let texts: Vec = logs_range(index).map(|(_, _, text)| text.to_string()).collect(); - let logs_considered = texts.len(); - for text in &texts { - auto_declare_params(index, text); - } - let reextracted = param_declarations_range(index).count(); - (reextracted, logs_considered) -} - -fn validate_metric_request( - index: &BTreeMap, - request: &DeclareMetricRequest, -) -> Result { - let re = Regex::new(&request.message_match_regex) - .map_err(|err| format!("invalid regex: {err}"))?; - - let declared = |name: &str| index.contains_key(&IndexKey::ParamDeclaration { name: name.to_string() }); - - let mut errors: Vec = request - .label_params - .iter() - .filter(|name| !declared(name)) - .map(|name| format!("label param '{name}' references an undeclared param")) - .collect(); - if let Some(value_param) = &request.value_param { - if !declared(value_param) { - errors.push(format!("value param '{value_param}' references an undeclared param")); +impl EngineSnapshot { + fn clear_all_param_declarations(&mut self) -> usize { + let names: Vec = + self.param_declarations_range().map(|(name, ..)| name.to_string()).collect(); + let removed = names.len(); + for name in names { + self.index.remove(&IndexKey::ParamDeclaration { name }); } - } - if !errors.is_empty() { - return Err(errors.join("; ")); + removed } - Ok(re) -} - -fn declare_metric( - index: &mut BTreeMap, - request: DeclareMetricRequest, -) -> Result<(), String> { - let re = validate_metric_request(index, &request)?; - let resolved_params = resolved_param_declarations(index); - - backfill_metric( - index, - &request.name, - &re, - &request.label_params, - request.value_param.as_deref(), - &resolved_params, - request.matched_labels.as_ref(), - request.unmatched_labels.as_ref(), - ); - index.insert( - IndexKey::MetricDeclaration { name: request.name }, - DataValue::MetricDeclaration { - message_match_regex: request.message_match_regex, - label_params: request.label_params, - value_param: request.value_param, - matched_labels: request.matched_labels, - unmatched_labels: request.unmatched_labels, - }, - ); - Ok(()) -} - -fn validate_update_metric( - index: &BTreeMap, - existing_name: &str, - request: &DeclareMetricRequest, -) -> Result { - if !index.contains_key(&IndexKey::MetricDeclaration { name: existing_name.to_string() }) { - return Err(format!("no metric declaration named '{existing_name}'")); + fn reextract_params_from_retained_logs(&mut self) -> (usize, usize) { + let texts: Vec = self.logs_range().map(|(_, _, text)| text.to_string()).collect(); + let logs_considered = texts.len(); + for text in &texts { + self.auto_declare_params(text); + } + let reextracted = self.param_declarations_range().count(); + (reextracted, logs_considered) } - if request.name != existing_name - && index.contains_key(&IndexKey::MetricDeclaration { name: request.name.clone() }) - { - return Err(format!("a metric named '{}' already exists", request.name)); - } - validate_metric_request(index, request) } -fn update_metric( - index: &mut BTreeMap, - existing_name: &str, - request: DeclareMetricRequest, -) -> Result<(), String> { - let re = validate_update_metric(index, existing_name, &request)?; - let resolved_params = resolved_param_declarations(index); +impl EngineSnapshot { + fn validate_metric_request(&self, request: &DeclareMetricRequest) -> Result { + let re = Regex::new(&request.message_match_regex) + .map_err(|err| format!("invalid regex: {err}"))?; - clear_metric_data(index, existing_name); - if request.name != existing_name { - index.remove(&IndexKey::MetricDeclaration { name: existing_name.to_string() }); + let declared = + |name: &str| self.index.contains_key(&IndexKey::ParamDeclaration { name: name.to_string() }); + + let mut errors: Vec = request + .label_params + .iter() + .filter(|name| !declared(name)) + .map(|name| format!("label param '{name}' references an undeclared param")) + .collect(); + if let Some(value_param) = &request.value_param { + if !declared(value_param) { + errors.push(format!("value param '{value_param}' references an undeclared param")); + } + } + if !errors.is_empty() { + return Err(errors.join("; ")); + } + + Ok(re) } - backfill_metric( - index, - &request.name, - &re, - &request.label_params, - request.value_param.as_deref(), - &resolved_params, - request.matched_labels.as_ref(), - request.unmatched_labels.as_ref(), - ); - index.insert( - IndexKey::MetricDeclaration { name: request.name.clone() }, - DataValue::MetricDeclaration { - message_match_regex: request.message_match_regex, - label_params: request.label_params, - value_param: request.value_param, - matched_labels: request.matched_labels, - unmatched_labels: request.unmatched_labels, - }, - ); - Ok(()) -} + fn declare_metric(&mut self, request: DeclareMetricRequest) -> Result<(), String> { + let re = self.validate_metric_request(&request)?; + let resolved_params = self.resolved_param_declarations(); -fn clear_metric_data(index: &mut BTreeMap, metric_name: &str) { - index.retain(|key, _| match key { - IndexKey::Bucket { metric_name: name, .. } => name != metric_name, - IndexKey::LastMetricEvent { metric_name: name, .. } => name != metric_name, - _ => true, - }); -} + self.backfill_metric( + &request.name, + &re, + &request.label_params, + request.value_param.as_deref(), + &resolved_params, + request.matched_labels.as_ref(), + request.unmatched_labels.as_ref(), + ); + self.index.insert( + IndexKey::MetricDeclaration { name: request.name }, + DataValue::MetricDeclaration { + message_match_regex: request.message_match_regex, + label_params: request.label_params, + value_param: request.value_param, + matched_labels: request.matched_labels, + unmatched_labels: request.unmatched_labels, + }, + ); + Ok(()) + } -fn remove_metric_declaration(index: &mut BTreeMap, name: &str) -> bool { - let existed = index.remove(&IndexKey::MetricDeclaration { name: name.to_string() }).is_some(); - clear_metric_data(index, name); - existed + fn validate_update_metric( + &self, + existing_name: &str, + request: &DeclareMetricRequest, + ) -> Result { + if !self.index.contains_key(&IndexKey::MetricDeclaration { name: existing_name.to_string() }) { + return Err(format!("no metric declaration named '{existing_name}'")); + } + if request.name != existing_name + && self.index.contains_key(&IndexKey::MetricDeclaration { name: request.name.clone() }) + { + return Err(format!("a metric named '{}' already exists", request.name)); + } + self.validate_metric_request(request) + } + + fn update_metric(&mut self, existing_name: &str, request: DeclareMetricRequest) -> Result<(), String> { + let re = self.validate_update_metric(existing_name, &request)?; + let resolved_params = self.resolved_param_declarations(); + + self.clear_metric_data(existing_name); + if request.name != existing_name { + self.index.remove(&IndexKey::MetricDeclaration { name: existing_name.to_string() }); + } + + self.backfill_metric( + &request.name, + &re, + &request.label_params, + request.value_param.as_deref(), + &resolved_params, + request.matched_labels.as_ref(), + request.unmatched_labels.as_ref(), + ); + self.index.insert( + IndexKey::MetricDeclaration { name: request.name.clone() }, + DataValue::MetricDeclaration { + message_match_regex: request.message_match_regex, + label_params: request.label_params, + value_param: request.value_param, + matched_labels: request.matched_labels, + unmatched_labels: request.unmatched_labels, + }, + ); + Ok(()) + } + + fn clear_metric_data(&mut self, metric_name: &str) { + self.index.retain(|key, _| match key { + IndexKey::Bucket { metric_name: name, .. } => name != metric_name, + IndexKey::LastMetricEvent { metric_name: name, .. } => name != metric_name, + _ => true, + }); + } + + fn remove_metric_declaration(&mut self, name: &str) -> bool { + let existed = self.index.remove(&IndexKey::MetricDeclaration { name: name.to_string() }).is_some(); + self.clear_metric_data(name); + existed + } } fn parse_metric_primitive(raw: &str, kind: MetricPrimitiveKind) -> Option { @@ -770,24 +757,26 @@ struct ResolvedParam { log_stream_gated: bool, } -fn resolved_param_declarations(index: &BTreeMap) -> HashMap { - param_declarations_range(index) - .filter_map(|(name, message_match_regex, capture_group, json_field_path, kind, elapsed_since_last, _, log_stream_gated)| { - Regex::new(message_match_regex).ok().map(|regex| { - ( - name.to_string(), - ResolvedParam { - regex, - capture_group: capture_group.to_string(), - json_field_path: json_field_path.map(|path| path.to_string()), - kind, - elapsed_since_last, - log_stream_gated, - }, - ) +impl EngineSnapshot { + fn resolved_param_declarations(&self) -> HashMap { + self.param_declarations_range() + .filter_map(|(name, message_match_regex, capture_group, json_field_path, kind, elapsed_since_last, _, log_stream_gated)| { + Regex::new(message_match_regex).ok().map(|regex| { + ( + name.to_string(), + ResolvedParam { + regex, + capture_group: capture_group.to_string(), + json_field_path: json_field_path.map(|path| path.to_string()), + kind, + elapsed_since_last, + log_stream_gated, + }, + ) + }) }) - }) - .collect() + .collect() + } } fn extract_param_raw( @@ -806,203 +795,201 @@ fn extract_param_raw( } } -fn resolve_param_value( - index: &mut BTreeMap, - metric_name: &str, - resolved: &ResolvedParam, - labels_so_far: &BTreeMap, - text: &str, - timestamp: Timestamp, -) -> Option { - if resolved.elapsed_since_last { - let last_event_key = IndexKey::LastMetricEvent { - metric_name: metric_name.to_string(), - labels: labels_so_far.clone(), - }; - let previous_timestamp = match index.get(&last_event_key) { - Some(DataValue::LastEventTimestamp { timestamp: previous }) => Some(*previous), - _ => None, - }; - index.insert(last_event_key, DataValue::LastEventTimestamp { timestamp }); - return previous_timestamp - .map(|previous| MetricPrimitive::Duration(timestamp.as_millis() - previous.as_millis())); - } - - let raw = extract_param_raw( - text, - &resolved.regex, - &resolved.capture_group, - resolved.json_field_path.as_deref(), - resolved.log_stream_gated, - )?; - parse_metric_primitive(&raw, resolved.kind) -} - -#[allow(clippy::too_many_arguments)] -fn apply_metric_to_log( - index: &mut BTreeMap, - metric_name: &str, - regex: &Regex, - label_params: &[String], - value_param: Option<&str>, - resolved_params: &HashMap, - matched_labels: Option<&BTreeMap>, - unmatched_labels: Option<&BTreeMap>, - sequence: u64, - timestamp: Timestamp, - text: &str, -) { - if regex.captures(text).is_none() { - if let Some(unmatched_labels) = unmatched_labels { - let labels = unmatched_labels - .iter() - .map(|(name, value)| (name.clone(), MetricPrimitive::String(value.clone()))) - .collect(); - push_bucket_value(index, metric_name, labels, sequence, timestamp, MetricPrimitive::Number(Number::Int(1))); +impl EngineSnapshot { + fn resolve_param_value( + &mut self, + metric_name: &str, + resolved: &ResolvedParam, + labels_so_far: &BTreeMap, + text: &str, + timestamp: Timestamp, + ) -> Option { + if resolved.elapsed_since_last { + let last_event_key = IndexKey::LastMetricEvent { + metric_name: metric_name.to_string(), + labels: labels_so_far.clone(), + }; + let previous_timestamp = match self.index.get(&last_event_key) { + Some(DataValue::LastEventTimestamp { timestamp: previous }) => Some(*previous), + _ => None, + }; + self.index.insert(last_event_key, DataValue::LastEventTimestamp { timestamp }); + return previous_timestamp + .map(|previous| MetricPrimitive::Duration(timestamp.as_millis() - previous.as_millis())); } - return; + + let raw = extract_param_raw( + text, + &resolved.regex, + &resolved.capture_group, + resolved.json_field_path.as_deref(), + resolved.log_stream_gated, + )?; + parse_metric_primitive(&raw, resolved.kind) } - let mut labels = BTreeMap::new(); - for param_name in label_params { - let Some(resolved) = resolved_params.get(param_name) else { return }; - let Some(parsed) = resolve_param_value(index, metric_name, resolved, &labels, text, timestamp) else { + #[allow(clippy::too_many_arguments)] + fn apply_metric_to_log( + &mut self, + metric_name: &str, + regex: &Regex, + label_params: &[String], + value_param: Option<&str>, + resolved_params: &HashMap, + matched_labels: Option<&BTreeMap>, + unmatched_labels: Option<&BTreeMap>, + sequence: u64, + timestamp: Timestamp, + text: &str, + ) { + if regex.captures(text).is_none() { + if let Some(unmatched_labels) = unmatched_labels { + let labels = unmatched_labels + .iter() + .map(|(name, value)| (name.clone(), MetricPrimitive::String(value.clone()))) + .collect(); + self.push_bucket_value(metric_name, labels, sequence, timestamp, MetricPrimitive::Number(Number::Int(1))); + } return; - }; - labels.insert(param_name.clone(), parsed); - } - - if let Some(matched_labels) = matched_labels { - for (name, value) in matched_labels { - labels.insert(name.clone(), MetricPrimitive::String(value.clone())); } - } - let value = match value_param { - Some(value_param) => { - let Some(resolved) = resolved_params.get(value_param) else { return }; - match resolve_param_value(index, metric_name, resolved, &labels, text, timestamp) { - Some(parsed) => parsed, - None => return, + let mut labels = BTreeMap::new(); + for param_name in label_params { + let Some(resolved) = resolved_params.get(param_name) else { return }; + let Some(parsed) = self.resolve_param_value(metric_name, resolved, &labels, text, timestamp) else { + return; + }; + labels.insert(param_name.clone(), parsed); + } + + if let Some(matched_labels) = matched_labels { + for (name, value) in matched_labels { + labels.insert(name.clone(), MetricPrimitive::String(value.clone())); } } - None => MetricPrimitive::Number(Number::Int(1)), - }; - push_bucket_value(index, metric_name, labels, sequence, timestamp, value); -} + let value = match value_param { + Some(value_param) => { + let Some(resolved) = resolved_params.get(value_param) else { return }; + match self.resolve_param_value(metric_name, resolved, &labels, text, timestamp) { + Some(parsed) => parsed, + None => return, + } + } + None => MetricPrimitive::Number(Number::Int(1)), + }; -fn push_bucket_value( - index: &mut BTreeMap, - metric_name: &str, - labels: BTreeMap, - sequence: u64, - timestamp: Timestamp, - value: MetricPrimitive, -) { - let key = IndexKey::Bucket { metric_name: metric_name.to_string(), labels }; - let entry = index - .entry(key) - .or_insert_with(|| DataValue::BucketValue { values: Vec::new() }); - if let DataValue::BucketValue { values } = entry { - values.push((sequence, timestamp, value)); + self.push_bucket_value(metric_name, labels, sequence, timestamp, value); } -} -#[allow(clippy::too_many_arguments)] -fn backfill_metric( - index: &mut BTreeMap, - metric_name: &str, - regex: &Regex, - label_params: &[String], - value_param: Option<&str>, - resolved_params: &HashMap, - matched_labels: Option<&BTreeMap>, - unmatched_labels: Option<&BTreeMap>, -) { - let lines: Vec<(u64, Timestamp, String)> = logs_range(index) - .map(|(sequence, timestamp, text)| (sequence, timestamp, text.to_string())) - .collect(); - for (sequence, timestamp, text) in lines { - apply_metric_to_log( - index, - metric_name, - regex, - label_params, - value_param, - resolved_params, - matched_labels, - unmatched_labels, - sequence, - timestamp, - &text, - ); + fn push_bucket_value( + &mut self, + metric_name: &str, + labels: BTreeMap, + sequence: u64, + timestamp: Timestamp, + value: MetricPrimitive, + ) { + let key = IndexKey::Bucket { metric_name: metric_name.to_string(), labels }; + let entry = self + .index + .entry(key) + .or_insert_with(|| DataValue::BucketValue { values: Vec::new() }); + if let DataValue::BucketValue { values } = entry { + values.push((sequence, timestamp, value)); + } } -} -fn apply_all_metrics( - index: &mut BTreeMap, - sequence: u64, - timestamp: Timestamp, - text: &str, -) { - type SeededMetric = ( - String, - String, - Vec, - Option, - Option>, - Option>, - ); - let metrics: Vec = metric_declarations_range(index) - .map(|(name, message_match_regex, label_params, value_param, matched_labels, unmatched_labels)| { - ( - name.to_string(), - message_match_regex.to_string(), - label_params.to_vec(), - value_param.map(|s| s.to_string()), - matched_labels.cloned(), - unmatched_labels.cloned(), - ) - }) - .collect(); - let resolved_params = resolved_param_declarations(index); - for (name, message_match_regex, label_params, value_param, matched_labels, unmatched_labels) in metrics { - if let Ok(re) = Regex::new(&message_match_regex) { - apply_metric_to_log( - index, - &name, - &re, - &label_params, - value_param.as_deref(), - &resolved_params, - matched_labels.as_ref(), - unmatched_labels.as_ref(), + #[allow(clippy::too_many_arguments)] + fn backfill_metric( + &mut self, + metric_name: &str, + regex: &Regex, + label_params: &[String], + value_param: Option<&str>, + resolved_params: &HashMap, + matched_labels: Option<&BTreeMap>, + unmatched_labels: Option<&BTreeMap>, + ) { + let lines: Vec<(u64, Timestamp, String)> = self + .logs_range() + .map(|(sequence, timestamp, text)| (sequence, timestamp, text.to_string())) + .collect(); + for (sequence, timestamp, text) in lines { + self.apply_metric_to_log( + metric_name, + regex, + label_params, + value_param, + resolved_params, + matched_labels, + unmatched_labels, sequence, timestamp, - text, + &text, ); } } -} -fn bucket_values<'a>( - index: &'a BTreeMap, - metric_name: &'a str, - labels_filter: Option<&'a BTreeMap>, -) -> impl Iterator, &'a (u64, Timestamp, MetricPrimitive))> + 'a -{ - index - .range(IndexKey::MetricBucketSplit..IndexKey::End) - .filter_map(move |(key, value)| match (key, value) { - (IndexKey::Bucket { metric_name: name, labels }, DataValue::BucketValue { values }) - if name == metric_name && labels_filter.is_none_or(|filter| filter == labels) => - { - Some(values.iter().map(move |entry| (labels, entry))) + fn apply_all_metrics(&mut self, sequence: u64, timestamp: Timestamp, text: &str) { + type SeededMetric = ( + String, + String, + Vec, + Option, + Option>, + Option>, + ); + let metrics: Vec = self + .metric_declarations_range() + .map(|(name, message_match_regex, label_params, value_param, matched_labels, unmatched_labels)| { + ( + name.to_string(), + message_match_regex.to_string(), + label_params.to_vec(), + value_param.map(|s| s.to_string()), + matched_labels.cloned(), + unmatched_labels.cloned(), + ) + }) + .collect(); + let resolved_params = self.resolved_param_declarations(); + for (name, message_match_regex, label_params, value_param, matched_labels, unmatched_labels) in metrics { + if let Ok(re) = Regex::new(&message_match_regex) { + self.apply_metric_to_log( + &name, + &re, + &label_params, + value_param.as_deref(), + &resolved_params, + matched_labels.as_ref(), + unmatched_labels.as_ref(), + sequence, + timestamp, + text, + ); } - _ => None, - }) - .flatten() + } + } + + fn bucket_values<'a>( + &'a self, + metric_name: &'a str, + labels_filter: Option<&'a BTreeMap>, + ) -> impl Iterator, &'a (u64, Timestamp, MetricPrimitive))> + 'a + { + self.index + .range(IndexKey::MetricBucketSplit..IndexKey::End) + .filter_map(move |(key, value)| match (key, value) { + (IndexKey::Bucket { metric_name: name, labels }, DataValue::BucketValue { values }) + if name == metric_name && labels_filter.is_none_or(|filter| filter == labels) => + { + Some(values.iter().map(move |entry| (labels, entry))) + } + _ => None, + }) + .flatten() + } } enum CompiledFilter { @@ -1016,165 +1003,154 @@ enum CompiledFilter { Or(Box, Box), } -fn compile_filter_query( - index: &BTreeMap, - filter: &str, - now: Timestamp, -) -> Result { - let expr = filter_parser::parse(filter).map_err(|err| err.to_string())?; - compile_filter(index, &expr, now) -} - -fn compile_filter( - index: &BTreeMap, - expr: &FilterExpr, - now: Timestamp, -) -> Result { - match expr { - FilterExpr::Regex(pattern) => Regex::new(pattern) - .map(CompiledFilter::Regex) - .map_err(|err| format!("invalid regex in filter: {err}")), - FilterExpr::Comparison { metric, param, op, value } => match metric.as_str() { - "ottyl_seq" => compile_ottyl_seq_comparison(*op, value), - "ottyl_ts" => compile_ottyl_ts_comparison(*op, value, now), - "ottyl_fmt" => Ok(compile_ottyl_fmt_comparison(*op, value)), - other => compile_metric_comparison(index, other, param.as_deref(), *op, value, now), - }, - FilterExpr::JsonComparison { field, op, value } => Ok(CompiledFilter::Json { - field: field.clone(), - op: *op, - value: value.clone(), - }), - FilterExpr::And(lhs, rhs) => Ok(CompiledFilter::And( - Box::new(compile_filter(index, lhs, now)?), - Box::new(compile_filter(index, rhs, now)?), - )), - FilterExpr::Or(lhs, rhs) => Ok(CompiledFilter::Or( - Box::new(compile_filter(index, lhs, now)?), - Box::new(compile_filter(index, rhs, now)?), - )), +impl EngineSnapshot { + fn compile_filter_query(&self, filter: &str, now: Timestamp) -> Result { + let expr = filter_parser::parse(filter).map_err(|err| err.to_string())?; + self.compile_filter(&expr, now) } -} -fn compile_ottyl_seq_comparison(op: CompareOp, value: &FilterValue) -> Result { - let target = match value { - FilterValue::Number(n) => *n, - FilterValue::Raw(raw) => raw.parse::().map_err(|_| { - format!("'ottyl_seq' comparisons require a numeric value, got {raw:?}") - })?, - }; - Ok(CompiledFilter::Sequence { op, target }) -} + fn compile_filter(&self, expr: &FilterExpr, now: Timestamp) -> Result { + match expr { + FilterExpr::Regex(pattern) => Regex::new(pattern) + .map(CompiledFilter::Regex) + .map_err(|err| format!("invalid regex in filter: {err}")), + FilterExpr::Comparison { metric, param, op, value } => match metric.as_str() { + "ottyl_seq" => CompiledFilter::from_seq_comparison(*op, value), + "ottyl_ts" => CompiledFilter::from_ts_comparison(*op, value, now), + "ottyl_fmt" => Ok(CompiledFilter::from_fmt_comparison(*op, value)), + other => self.compile_metric_comparison(other, param.as_deref(), *op, value, now), + }, + FilterExpr::JsonComparison { field, op, value } => Ok(CompiledFilter::Json { + field: field.clone(), + op: *op, + value: value.clone(), + }), + FilterExpr::And(lhs, rhs) => Ok(CompiledFilter::And( + Box::new(self.compile_filter(lhs, now)?), + Box::new(self.compile_filter(rhs, now)?), + )), + FilterExpr::Or(lhs, rhs) => Ok(CompiledFilter::Or( + Box::new(self.compile_filter(lhs, now)?), + Box::new(self.compile_filter(rhs, now)?), + )), + } + } -fn compile_ottyl_ts_comparison( - op: CompareOp, - value: &FilterValue, - now: Timestamp, -) -> Result { - let target_ms = match value { - FilterValue::Number(n) => *n as i64, - FilterValue::Raw(raw) => filter_datetime::parse_datetime(raw, now) - .map_err(|err| err.to_string())? - .as_millis(), - }; - Ok(CompiledFilter::Timestamp { op, target_ms }) -} + fn compile_metric_comparison( + &self, + metric: &str, + param: Option<&str>, + op: CompareOp, + value: &FilterValue, + now: Timestamp, + ) -> Result { + let param = param.ok_or_else(|| { + format!( + "metric comparison on '{metric}' requires a [param] (e.g. '{metric}[some_param]{}...'); \ + v1 buckets only carry per-label occurrence data, not a single metric value", + match op { + CompareOp::Lt => "<", + CompareOp::Gt => ">", + CompareOp::Le => "<=", + CompareOp::Ge => ">=", + CompareOp::Eq => "==", + CompareOp::Ne => "!=", + } + ) + })?; -fn compile_ottyl_fmt_comparison(op: CompareOp, value: &FilterValue) -> CompiledFilter { - let target = match value { - FilterValue::Number(n) => n.to_string(), - FilterValue::Raw(raw) => raw.clone(), - }; - CompiledFilter::Fmt { op, target } -} - -fn compile_metric_comparison( - index: &BTreeMap, - metric: &str, - param: Option<&str>, - op: CompareOp, - value: &FilterValue, - now: Timestamp, -) -> Result { - let param = param.ok_or_else(|| { - format!( - "metric comparison on '{metric}' requires a [param] (e.g. '{metric}[some_param]{}...'); \ - v1 buckets only carry per-label occurrence data, not a single metric value", - match op { - CompareOp::Lt => "<", - CompareOp::Gt => ">", - CompareOp::Le => "<=", - CompareOp::Ge => ">=", - CompareOp::Eq => "==", - CompareOp::Ne => "!=", + if let Some(DataValue::MetricDeclaration { label_params, .. }) = + self.index.get(&IndexKey::MetricDeclaration { name: metric.to_string() }) + { + if !label_params.iter().any(|p| p == param) { + return Err(format!( + "param '{param}' is not a label param of metric '{metric}'; declared label params: [{}]", + label_params.join(", ") + )); } - ) - })?; - - if let Some(DataValue::MetricDeclaration { label_params, .. }) = - index.get(&IndexKey::MetricDeclaration { name: metric.to_string() }) - { - if !label_params.iter().any(|p| p == param) { - return Err(format!( - "param '{param}' is not a label param of metric '{metric}'; declared label params: [{}]", - label_params.join(", ") - )); } - } - let mut matching = HashSet::new(); - for (labels, (sequence, _, _)) in bucket_values(index, metric, None) { - let Some(primitive) = labels.get(param) else { - continue; - }; - if compare_metric_primitive(primitive, op, value, now)? { - matching.insert(*sequence); + let mut matching = HashSet::new(); + for (labels, (sequence, _, _)) in self.bucket_values(metric, None) { + let Some(primitive) = labels.get(param) else { + continue; + }; + if primitive.compare(op, value, now)? { + matching.insert(*sequence); + } } + Ok(CompiledFilter::MetricMatch(matching)) } - Ok(CompiledFilter::MetricMatch(matching)) } -fn compare_metric_primitive( - primitive: &MetricPrimitive, - op: CompareOp, - value: &FilterValue, - now: Timestamp, -) -> Result { - let ordering = match (primitive, value) { - (MetricPrimitive::Number(n), FilterValue::Number(v)) => n.as_f64().partial_cmp(v), - (MetricPrimitive::Number(n), FilterValue::Raw(raw)) => { - let v: f64 = raw - .parse() - .map_err(|_| format!("expected a numeric value, got {raw:?}"))?; - n.as_f64().partial_cmp(&v) - } - (MetricPrimitive::Timestamp(ts), FilterValue::Number(v)) => { - ts.as_millis().partial_cmp(&(*v as i64)) - } - (MetricPrimitive::Timestamp(ts), FilterValue::Raw(raw)) => { - let target = filter_datetime::parse_datetime(raw, now).map_err(|err| err.to_string())?; - ts.as_millis().partial_cmp(&target.as_millis()) - } - (MetricPrimitive::String(s), FilterValue::Raw(raw)) => s.as_str().partial_cmp(raw.as_str()), - (MetricPrimitive::String(s), FilterValue::Number(n)) => { - s.as_str().partial_cmp(n.to_string().as_str()) - } - (MetricPrimitive::Duration(ms), FilterValue::Number(v)) => (*ms as f64).partial_cmp(v), - (MetricPrimitive::Duration(ms), FilterValue::Raw(raw)) => { - let v: f64 = raw - .parse() - .map_err(|_| format!("expected a numeric value, got {raw:?}"))?; - (*ms as f64).partial_cmp(&v) - } - (MetricPrimitive::Boolean(b), FilterValue::Raw(raw)) => { - let v: bool = raw - .parse() - .map_err(|_| format!("expected true or false, got {raw:?}"))?; - b.partial_cmp(&v) - } - (MetricPrimitive::Boolean(_), FilterValue::Number(_)) => None, - }; - Ok(ordering.is_some_and(|ordering| apply_compare_op(ordering, op))) +impl CompiledFilter { + fn from_seq_comparison(op: CompareOp, value: &FilterValue) -> Result { + let target = match value { + FilterValue::Number(n) => *n, + FilterValue::Raw(raw) => raw.parse::().map_err(|_| { + format!("'ottyl_seq' comparisons require a numeric value, got {raw:?}") + })?, + }; + Ok(CompiledFilter::Sequence { op, target }) + } + + fn from_ts_comparison(op: CompareOp, value: &FilterValue, now: Timestamp) -> Result { + let target_ms = match value { + FilterValue::Number(n) => *n as i64, + FilterValue::Raw(raw) => filter_datetime::parse_datetime(raw, now) + .map_err(|err| err.to_string())? + .as_millis(), + }; + Ok(CompiledFilter::Timestamp { op, target_ms }) + } + + fn from_fmt_comparison(op: CompareOp, value: &FilterValue) -> Self { + let target = match value { + FilterValue::Number(n) => n.to_string(), + FilterValue::Raw(raw) => raw.clone(), + }; + CompiledFilter::Fmt { op, target } + } +} + +impl MetricPrimitive { + fn compare(&self, op: CompareOp, value: &FilterValue, now: Timestamp) -> Result { + let ordering = match (self, value) { + (MetricPrimitive::Number(n), FilterValue::Number(v)) => n.as_f64().partial_cmp(v), + (MetricPrimitive::Number(n), FilterValue::Raw(raw)) => { + let v: f64 = raw + .parse() + .map_err(|_| format!("expected a numeric value, got {raw:?}"))?; + n.as_f64().partial_cmp(&v) + } + (MetricPrimitive::Timestamp(ts), FilterValue::Number(v)) => { + ts.as_millis().partial_cmp(&(*v as i64)) + } + (MetricPrimitive::Timestamp(ts), FilterValue::Raw(raw)) => { + let target = filter_datetime::parse_datetime(raw, now).map_err(|err| err.to_string())?; + ts.as_millis().partial_cmp(&target.as_millis()) + } + (MetricPrimitive::String(s), FilterValue::Raw(raw)) => s.as_str().partial_cmp(raw.as_str()), + (MetricPrimitive::String(s), FilterValue::Number(n)) => { + s.as_str().partial_cmp(n.to_string().as_str()) + } + (MetricPrimitive::Duration(ms), FilterValue::Number(v)) => (*ms as f64).partial_cmp(v), + (MetricPrimitive::Duration(ms), FilterValue::Raw(raw)) => { + let v: f64 = raw + .parse() + .map_err(|_| format!("expected a numeric value, got {raw:?}"))?; + (*ms as f64).partial_cmp(&v) + } + (MetricPrimitive::Boolean(b), FilterValue::Raw(raw)) => { + let v: bool = raw + .parse() + .map_err(|_| format!("expected true or false, got {raw:?}"))?; + b.partial_cmp(&v) + } + (MetricPrimitive::Boolean(_), FilterValue::Number(_)) => None, + }; + Ok(ordering.is_some_and(|ordering| op.matches(ordering))) + } } pub(crate) fn eval_json_comparison(text: &str, field: &str, op: CompareOp, value: &FilterValue) -> bool { @@ -1195,7 +1171,7 @@ fn compare_json_value(field_value: &serde_json::Value, op: CompareOp, value: &Fi FilterValue::Raw(raw) => raw.parse::().ok(), }; match (n.as_f64(), target) { - (Some(f), Some(v)) => f.partial_cmp(&v).is_some_and(|ordering| apply_compare_op(ordering, op)), + (Some(f), Some(v)) => f.partial_cmp(&v).is_some_and(|ordering| op.matches(ordering)), _ => false, } } @@ -1206,7 +1182,7 @@ fn compare_json_value(field_value: &serde_json::Value, op: CompareOp, value: &Fi }; s.as_str() .partial_cmp(target.as_str()) - .is_some_and(|ordering| apply_compare_op(ordering, op)) + .is_some_and(|ordering| op.matches(ordering)) } serde_json::Value::Bool(b) => { let FilterValue::Raw(raw) = value else { @@ -1225,106 +1201,112 @@ fn compare_json_value(field_value: &serde_json::Value, op: CompareOp, value: &Fi } } -pub(crate) fn apply_compare_op(ordering: Ordering, op: CompareOp) -> bool { - match op { - CompareOp::Lt => ordering == Ordering::Less, - CompareOp::Gt => ordering == Ordering::Greater, - CompareOp::Le => ordering != Ordering::Greater, - CompareOp::Ge => ordering != Ordering::Less, - CompareOp::Eq => ordering == Ordering::Equal, - CompareOp::Ne => ordering != Ordering::Equal, - } -} - -fn eval_compiled(compiled: &CompiledFilter, sequence: u64, timestamp: Timestamp, text: &str) -> bool { - match compiled { - CompiledFilter::Regex(re) => re.is_match(text), - CompiledFilter::Sequence { op, target } => { - apply_compare_op((sequence as f64).total_cmp(target), *op) - } - CompiledFilter::Timestamp { op, target_ms } => { - apply_compare_op(timestamp.as_millis().cmp(target_ms), *op) - } - CompiledFilter::Fmt { op, target } => { - apply_compare_op(crate::logfmt::detect_format(text).cmp(target.as_str()), *op) - } - CompiledFilter::MetricMatch(sequences) => sequences.contains(&sequence), - CompiledFilter::Json { field, op, value } => eval_json_comparison(text, field, *op, value), - CompiledFilter::And(lhs, rhs) => { - eval_compiled(lhs, sequence, timestamp, text) && eval_compiled(rhs, sequence, timestamp, text) - } - CompiledFilter::Or(lhs, rhs) => { - eval_compiled(lhs, sequence, timestamp, text) || eval_compiled(rhs, sequence, timestamp, text) +impl CompareOp { + pub(crate) fn matches(&self, ordering: Ordering) -> bool { + match self { + CompareOp::Lt => ordering == Ordering::Less, + CompareOp::Gt => ordering == Ordering::Greater, + CompareOp::Le => ordering != Ordering::Greater, + CompareOp::Ge => ordering != Ordering::Less, + CompareOp::Eq => ordering == Ordering::Equal, + CompareOp::Ne => ordering != Ordering::Equal, } } } -fn metric_declarations_range( - index: &BTreeMap, -) -> impl Iterator< - Item = ( - &str, - &str, - &[String], - Option<&str>, - Option<&BTreeMap>, - Option<&BTreeMap>, - ), -> { - index - .range(IndexKey::ParamMetricSplit..IndexKey::MetricBucketSplit) - .filter_map(|(key, value)| match (key, value) { - ( - IndexKey::MetricDeclaration { name }, - DataValue::MetricDeclaration { - message_match_regex, - label_params, - value_param, - matched_labels, - unmatched_labels, - }, - ) => Some(( - name.as_str(), - message_match_regex.as_str(), - label_params.as_slice(), - value_param.as_deref(), - matched_labels.as_ref(), - unmatched_labels.as_ref(), - )), - _ => None, - }) +impl CompiledFilter { + fn eval(&self, sequence: u64, timestamp: Timestamp, text: &str) -> bool { + match self { + CompiledFilter::Regex(re) => re.is_match(text), + CompiledFilter::Sequence { op, target } => { + op.matches((sequence as f64).total_cmp(target)) + } + CompiledFilter::Timestamp { op, target_ms } => { + op.matches(timestamp.as_millis().cmp(target_ms)) + } + CompiledFilter::Fmt { op, target } => { + op.matches(crate::logfmt::detect_format(text).cmp(target.as_str())) + } + CompiledFilter::MetricMatch(sequences) => sequences.contains(&sequence), + CompiledFilter::Json { field, op, value } => eval_json_comparison(text, field, *op, value), + CompiledFilter::And(lhs, rhs) => { + lhs.eval(sequence, timestamp, text) && rhs.eval(sequence, timestamp, text) + } + CompiledFilter::Or(lhs, rhs) => { + lhs.eval(sequence, timestamp, text) || rhs.eval(sequence, timestamp, text) + } + } + } } -fn param_declarations_range( - index: &BTreeMap, -) -> impl Iterator, MetricPrimitiveKind, bool, bool, bool)> { - index - .range(IndexKey::LogParamSplit..IndexKey::ParamMetricSplit) - .filter_map(|(key, value)| match (key, value) { - ( - IndexKey::ParamDeclaration { name }, - DataValue::ParamDeclaration(ParamDeclaration { - name: _, - message_match_regex, - capture_group, - json_field_path, - kind, - elapsed_since_last, - auto_derived, - log_stream_gated, - }), - ) => Some(( - name.as_str(), - message_match_regex.as_str(), - capture_group.as_str(), - json_field_path.as_deref(), - *kind, - *elapsed_since_last, - *auto_derived, - *log_stream_gated, - )), - _ => None, - }) +impl EngineSnapshot { + fn metric_declarations_range( + &self, + ) -> impl Iterator< + Item = ( + &str, + &str, + &[String], + Option<&str>, + Option<&BTreeMap>, + Option<&BTreeMap>, + ), + > { + self.index + .range(IndexKey::ParamMetricSplit..IndexKey::MetricBucketSplit) + .filter_map(|(key, value)| match (key, value) { + ( + IndexKey::MetricDeclaration { name }, + DataValue::MetricDeclaration { + message_match_regex, + label_params, + value_param, + matched_labels, + unmatched_labels, + }, + ) => Some(( + name.as_str(), + message_match_regex.as_str(), + label_params.as_slice(), + value_param.as_deref(), + matched_labels.as_ref(), + unmatched_labels.as_ref(), + )), + _ => None, + }) + } + + fn param_declarations_range( + &self, + ) -> impl Iterator, MetricPrimitiveKind, bool, bool, bool)> { + self.index + .range(IndexKey::LogParamSplit..IndexKey::ParamMetricSplit) + .filter_map(|(key, value)| match (key, value) { + ( + IndexKey::ParamDeclaration { name }, + DataValue::ParamDeclaration(ParamDeclaration { + name: _, + message_match_regex, + capture_group, + json_field_path, + kind, + elapsed_since_last, + auto_derived, + log_stream_gated, + }), + ) => Some(( + name.as_str(), + message_match_regex.as_str(), + capture_group.as_str(), + json_field_path.as_deref(), + *kind, + *elapsed_since_last, + *auto_derived, + *log_stream_gated, + )), + _ => None, + }) + } } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] @@ -1363,71 +1345,68 @@ fn derive_single_param( parse_metric_primitive(&raw, *kind) } -fn build_sort_helper_index(index: &BTreeMap, sort_by: &str) -> BTreeMap { - let compiled_params = compiled_param_declarations(index); - logs_range(index) - .map(|(sequence, timestamp, text)| { +impl EngineSnapshot { + fn build_sort_helper_index(&self, sort_by: &str) -> BTreeMap { + let compiled_params = self.compiled_param_declarations(); + self.logs_range() + .map(|(sequence, timestamp, text)| { + let key = match derive_single_param(text, &compiled_params, sort_by) { + Some(value) => SortHelperKey::WithParam(value, sequence), + None => SortHelperKey::WithoutParam(sequence), + }; + (key, timestamp) + }) + .collect() + } + + fn update_sort_helper_indexes_on_ingest(&mut self, sequence: u64, timestamp: Timestamp, text: &str) { + if self.sort_helper_indexes.is_empty() { + return; + } + let compiled_params = self.compiled_param_declarations(); + for (sort_by, entry) in self.sort_helper_indexes.iter_mut() { let key = match derive_single_param(text, &compiled_params, sort_by) { Some(value) => SortHelperKey::WithParam(value, sequence), None => SortHelperKey::WithoutParam(sequence), }; - (key, timestamp) - }) - .collect() -} - -fn update_sort_helper_indexes_on_ingest( - sort_helper_indexes: &mut HashMap, - index: &BTreeMap, - sequence: u64, - timestamp: Timestamp, - text: &str, -) { - if sort_helper_indexes.is_empty() { - return; + entry.index.insert(key, timestamp); + } } - let compiled_params = compiled_param_declarations(index); - for (sort_by, entry) in sort_helper_indexes.iter_mut() { - let key = match derive_single_param(text, &compiled_params, sort_by) { - Some(value) => SortHelperKey::WithParam(value, sequence), - None => SortHelperKey::WithoutParam(sequence), - }; - entry.index.insert(key, timestamp); + + fn purge_stale_sort_helper_indexes(&mut self) { + self.sort_helper_indexes + .retain(|_, entry| entry.last_accessed.elapsed() < SORT_HELPER_INDEX_PURGE_AFTER); } -} -fn purge_stale_sort_helper_indexes(sort_helper_indexes: &mut HashMap) { - sort_helper_indexes.retain(|_, entry| entry.last_accessed.elapsed() < SORT_HELPER_INDEX_PURGE_AFTER); -} + fn ensure_sort_helper_index_built(&mut self, sort_by: &str) { + if !self.sort_helper_indexes.contains_key(sort_by) { + let built = self.build_sort_helper_index(sort_by); + self.sort_helper_indexes.insert( + sort_by.to_string(), + SortHelperIndexEntry { index: built, last_accessed: std::time::Instant::now() }, + ); + } + let entry = self.sort_helper_indexes.get_mut(sort_by).expect("just inserted above if missing"); + entry.last_accessed = std::time::Instant::now(); + } -fn ensure_sort_helper_index_built( - sort_helper_indexes: &mut HashMap, - index: &BTreeMap, - sort_by: &str, -) { - let entry = sort_helper_indexes.entry(sort_by.to_string()).or_insert_with(|| SortHelperIndexEntry { - index: build_sort_helper_index(index, sort_by), - last_accessed: std::time::Instant::now(), - }); - entry.last_accessed = std::time::Instant::now(); -} - -fn compiled_param_declarations(index: &BTreeMap) -> Vec { - param_declarations_range(index) - .filter(|(_, _, _, _, _, elapsed_since_last, _, _)| !elapsed_since_last) - .filter_map(|(name, message_match_regex, capture_group, json_field_path, kind, _, _, log_stream_gated)| { - Regex::new(message_match_regex).ok().map(|re| { - ( - name.to_string(), - re, - capture_group.to_string(), - json_field_path.map(|path| path.to_string()), - kind, - log_stream_gated, - ) + fn compiled_param_declarations(&self) -> Vec { + self.param_declarations_range() + .filter(|(_, _, _, _, _, elapsed_since_last, _, _)| !elapsed_since_last) + .filter_map(|(name, message_match_regex, capture_group, json_field_path, kind, _, _, log_stream_gated)| { + Regex::new(message_match_regex).ok().map(|re| { + ( + name.to_string(), + re, + capture_group.to_string(), + json_field_path.map(|path| path.to_string()), + kind, + log_stream_gated, + ) + }) }) - }) - .collect() + .collect() + } } fn derive_log_params(text: &str, declarations: &[CompiledParamDeclaration]) -> BTreeMap { @@ -1495,51 +1474,53 @@ fn ingest_lag_high_watermark_ms(samples: &[(Timestamp, i64)], now: Timestamp) -> .unwrap_or(0) } -fn evict_excess_logs( - index: &mut BTreeMap, - log_count: &mut usize, - max_logs: Option, - approx_bytes_total: &mut u64, -) { - let Some(max_logs) = max_logs else { return }; - while *log_count > max_logs { - let Some(oldest_key) = index - .range(IndexKey::Start..IndexKey::LogParamSplit) - .next() - .map(|(key, _)| key.clone()) - else { - break; - }; - if let Some(DataValue::LogLiteral { text }) = index.get(&oldest_key) { - *approx_bytes_total = approx_bytes_total.saturating_sub(approx_log_bytes(text)); - } - index.remove(&oldest_key); - *log_count -= 1; +impl EngineSnapshot { + pub(crate) fn from_index(index: BTreeMap) -> Self { + EngineSnapshot { index, ..Default::default() } } -} -fn logs_range(index: &BTreeMap) -> impl Iterator { - index.range(IndexKey::Start..IndexKey::LogParamSplit).filter_map(|(key, value)| { - match (key, value) { - (IndexKey::IngestedLog { sequence, timestamp }, DataValue::LogLiteral { text }) => { - Some((*sequence, *timestamp, text.as_str())) + fn evict_excess_logs(&mut self, max_logs: Option) { + let Some(max_logs) = max_logs else { return }; + while self.log_count > max_logs { + let Some(oldest_key) = self + .index + .range(IndexKey::Start..IndexKey::LogParamSplit) + .next() + .map(|(key, _)| key.clone()) + else { + break; + }; + if let Some(DataValue::LogLiteral { text }) = self.index.get(&oldest_key) { + self.approx_bytes_total = self.approx_bytes_total.saturating_sub(approx_log_bytes(text)); } - _ => None, + self.index.remove(&oldest_key); + self.log_count -= 1; } - }) -} + } -pub(crate) fn logs_since( - index: &BTreeMap, - before_sequence: u64, - since_ms: i64, - now_ms: i64, -) -> impl Iterator { - logs_range(index).filter(move |(sequence, timestamp, _)| { - *sequence < before_sequence - && timestamp.as_millis() > since_ms - && timestamp.as_millis() <= now_ms - }) + fn logs_range(&self) -> impl Iterator { + self.index.range(IndexKey::Start..IndexKey::LogParamSplit).filter_map(|(key, value)| { + match (key, value) { + (IndexKey::IngestedLog { sequence, timestamp }, DataValue::LogLiteral { text }) => { + Some((*sequence, *timestamp, text.as_str())) + } + _ => None, + } + }) + } + + pub(crate) fn logs_since( + &self, + before_sequence: u64, + since_ms: i64, + now_ms: i64, + ) -> impl Iterator { + self.logs_range().filter(move |(sequence, timestamp, _)| { + *sequence < before_sequence + && timestamp.as_millis() > since_ms + && timestamp.as_millis() <= now_ms + }) + } } enum SortHelperAccess<'a> { @@ -1558,7 +1539,15 @@ impl<'a> SortHelperAccess<'a> { match self { #[cfg(test)] SortHelperAccess::Mutable(map) => { - ensure_sort_helper_index_built(map, index, sort_by); + if !map.contains_key(sort_by) { + let built = EngineSnapshot::from_index(index.clone()).build_sort_helper_index(sort_by); + map.insert( + sort_by.to_string(), + SortHelperIndexEntry { index: built, last_accessed: std::time::Instant::now() }, + ); + } + let entry = map.get_mut(sort_by).expect("just inserted above if missing"); + entry.last_accessed = std::time::Instant::now(); map.get(sort_by).map(|entry| &entry.index) } SortHelperAccess::Immutable(map) => map.get(sort_by).map(|entry| &entry.index), @@ -1568,66 +1557,60 @@ impl<'a> SortHelperAccess<'a> { #[cfg(test)] fn handle_query( - index: &BTreeMap, + snapshot: &EngineSnapshot, query: EngineQuery, - approx_bytes_total: u64, - ingest_lag_high_watermark_ms: i64, + _approx_bytes_total: u64, + _ingest_lag_high_watermark_ms: i64, sort_helper_indexes: &mut HashMap, ) -> EngineResponse { - handle_query_inner( - index, - query, - approx_bytes_total, - ingest_lag_high_watermark_ms, - SortHelperAccess::Mutable(sort_helper_indexes), - ) + handle_query_inner(snapshot, query, SortHelperAccess::Mutable(sort_helper_indexes)) } -pub(crate) fn handle_query_ro(snapshot: &EngineSnapshot, query: EngineQuery) -> EngineResponse { - handle_query_inner( - &snapshot.index, - query, - snapshot.approx_bytes_total, - snapshot.ingest_lag_high_watermark_ms, - SortHelperAccess::Immutable(&snapshot.sort_helper_indexes), - ) -} +impl EngineSnapshot { + pub(crate) fn handle_query_ro(&self, query: EngineQuery) -> EngineResponse { + handle_query_inner(self, query, SortHelperAccess::Immutable(&self.sort_helper_indexes)) + } -pub(crate) fn query_sort_by(query: &EngineQuery) -> Option<&str> { - match query { - EngineQuery::Logs(params) => params.sort_by.as_deref(), - _ => None, + pub(crate) fn has_sort_helper_index(&self, sort_by: &str) -> bool { + self.sort_helper_indexes.contains_key(sort_by) } } -pub(crate) fn has_sort_helper_index(snapshot: &EngineSnapshot, sort_by: &str) -> bool { - snapshot.sort_helper_indexes.contains_key(sort_by) +impl EngineQuery { + pub(crate) fn sort_by(&self) -> Option<&str> { + match self { + EngineQuery::Logs(params) => params.sort_by.as_deref(), + _ => None, + } + } } fn handle_query_inner( - index: &BTreeMap, + snapshot: &EngineSnapshot, query: EngineQuery, - approx_bytes_total: u64, - ingest_lag_high_watermark_ms: i64, sort_helper_access: SortHelperAccess<'_>, ) -> EngineResponse { + let index = &snapshot.index; + let approx_bytes_total = snapshot.approx_bytes_total; + let ingest_lag_high_watermark_ms = snapshot.ingest_lag_high_watermark_ms; match query { EngineQuery::Logs(params) => { let compiled_filter = match params.filter.as_deref() { - Some(filter) => match compile_filter_query(index, filter, Timestamp::now()) { + Some(filter) => match snapshot.compile_filter_query(filter, Timestamp::now()) { Ok(compiled) => Some(compiled), Err(err) => return EngineResponse::Error(err), }, None => None, }; if let Some(sort_by) = params.sort_by.as_deref() { - if !param_declarations_range(index).any(|(name, ..)| name == sort_by) { + if !snapshot.param_declarations_range().any(|(name, ..)| name == sort_by) { return EngineResponse::Error(format!("unknown param '{sort_by}' for sort_by")); } } let limit = params.limit.unwrap_or(500).min(10_000); - let compiled_params = compiled_param_declarations(index); - let filtered = logs_range(index) + let compiled_params = snapshot.compiled_param_declarations(); + let filtered = snapshot + .logs_range() .filter(|(sequence, timestamp, _)| { params.since_sequence.is_none_or(|since| *sequence > since) && params.start_ms.is_none_or(|start| timestamp.as_millis() >= start) @@ -1636,7 +1619,7 @@ fn handle_query_inner( .filter(|(sequence, timestamp, text)| { compiled_filter .as_ref() - .is_none_or(|filter| eval_compiled(filter, *sequence, *timestamp, text)) + .is_none_or(|filter| filter.eval(*sequence, *timestamp, text)) }); let entries: Vec = match params.sort_by.as_deref() { @@ -1680,7 +1663,7 @@ fn handle_query_inner( }; if !compiled_filter .as_ref() - .is_none_or(|filter| eval_compiled(filter, sequence, *timestamp, text)) + .is_none_or(|filter| filter.eval(sequence, *timestamp, text)) { continue; } @@ -1702,7 +1685,7 @@ fn handle_query_inner( } EngineQuery::Timeline(params) => { let compiled_filter = match params.filter.as_deref() { - Some(filter) => match compile_filter_query(index, filter, Timestamp::now()) { + Some(filter) => match snapshot.compile_filter_query(filter, Timestamp::now()) { Ok(compiled) => Some(compiled), Err(err) => return EngineResponse::Error(err), }, @@ -1710,13 +1693,13 @@ fn handle_query_inner( }; let bucket_ms = params.bucket_ms.unwrap_or(DEFAULT_TIMELINE_BUCKET_MS).max(1); let mut buckets: BTreeMap = BTreeMap::new(); - for (sequence, timestamp, text) in logs_range(index).filter(|(_, timestamp, _)| { + for (sequence, timestamp, text) in snapshot.logs_range().filter(|(_, timestamp, _)| { params.start_ms.is_none_or(|start| timestamp.as_millis() >= start) && params.end_ms.is_none_or(|end| timestamp.as_millis() <= end) }) { if compiled_filter .as_ref() - .is_some_and(|filter| !eval_compiled(filter, sequence, timestamp, text)) + .is_some_and(|filter| !filter.eval(sequence, timestamp, text)) { continue; } @@ -1735,7 +1718,7 @@ fn handle_query_inner( let mut last_sequence = None; let mut first_timestamp_ms = None; let mut last_timestamp_ms = None; - for (sequence, timestamp, _) in logs_range(index) { + for (sequence, timestamp, _) in snapshot.logs_range() { log_count += 1; first_sequence.get_or_insert(sequence); last_sequence = Some(sequence); @@ -1753,7 +1736,7 @@ fn handle_query_inner( }) } EngineQuery::ParamDeclarations => { - let declarations = param_declarations_range(index) + let declarations = snapshot.param_declarations_range() .map(|(name, message_match_regex, capture_group, json_field_path, kind, elapsed_since_last, auto_derived, _)| { ParamDeclarationDto { name: name.to_string(), @@ -1770,7 +1753,7 @@ fn handle_query_inner( EngineResponse::ParamDeclarations(declarations) } EngineQuery::MetricDeclarations => { - let declarations = metric_declarations_range(index) + let declarations = snapshot.metric_declarations_range() .map(|(name, message_match_regex, label_params, value_param, matched_labels, unmatched_labels)| { MetricDeclarationDto { name: name.to_string(), @@ -1793,7 +1776,7 @@ fn handle_query_inner( limit, } => { let limit = limit.unwrap_or(500).min(10_000); - let values: Vec = bucket_values(index, &metric_name, None) + let values: Vec = snapshot.bucket_values(&metric_name, None) .filter(|(labels, _)| { labels_filter.as_ref().is_none_or(|filter| { filter.iter().all(|(name, expected)| { @@ -1820,7 +1803,7 @@ fn handle_query_inner( } EngineQuery::Promql { query, start_ms, end_ms } => { let response = promql::parse_query(&query) - .and_then(|parsed| evaluate_promql(index, &parsed, start_ms, end_ms)); + .and_then(|parsed| snapshot.evaluate_promql(&parsed, start_ms, end_ms)); EngineResponse::Promql(response) } } @@ -1937,24 +1920,23 @@ fn merge_series_locf(sub_series: &[Vec<(i64, f64)>], is_avg: bool) -> Vec<(i64, .collect() } -fn declared_value_kind( - index: &BTreeMap, - metric_name: &str, -) -> Option { - match index.get(&IndexKey::MetricDeclaration { name: metric_name.to_string() }) { - Some(DataValue::MetricDeclaration { value_param, .. }) => Some( - value_param - .as_ref() - .and_then(|name| { - index.get(&IndexKey::ParamDeclaration { name: name.clone() }) - }) - .map(|value| match value { - DataValue::ParamDeclaration(param) => param.kind, - _ => MetricPrimitiveKind::Number, - }) - .unwrap_or(MetricPrimitiveKind::Number), - ), - _ => None, +impl EngineSnapshot { + fn declared_value_kind(&self, metric_name: &str) -> Option { + match self.index.get(&IndexKey::MetricDeclaration { name: metric_name.to_string() }) { + Some(DataValue::MetricDeclaration { value_param, .. }) => Some( + value_param + .as_ref() + .and_then(|name| { + self.index.get(&IndexKey::ParamDeclaration { name: name.clone() }) + }) + .map(|value| match value { + DataValue::ParamDeclaration(param) => param.kind, + _ => MetricPrimitiveKind::Number, + }) + .unwrap_or(MetricPrimitiveKind::Number), + ), + _ => None, + } } } @@ -1967,72 +1949,73 @@ fn ottyl_reserved_numeric_value_fn(metric_name: &str) -> Option } } -fn evaluate_promql( - index: &BTreeMap, - parsed: &PromqlQuery, - start_ms: Option, - end_ms: Option, -) -> Result, String> { - if parsed.metric_name == "ottyl_fmt" { - return Err( - "'ottyl_fmt' is a categorical value (json/logfmt/none), not a number, so it can't be \ - used in a dashboard PromQL query; filter on it in the search bar instead, e.g. \ - ottyl_fmt==json" - .to_string(), - ); - } +impl EngineSnapshot { + fn evaluate_promql( + &self, + parsed: &PromqlQuery, + start_ms: Option, + end_ms: Option, + ) -> Result, String> { + if parsed.metric_name == "ottyl_fmt" { + return Err( + "'ottyl_fmt' is a categorical value (json/logfmt/none), not a number, so it can't be \ + used in a dashboard PromQL query; filter on it in the search bar instead, e.g. \ + ottyl_fmt==json" + .to_string(), + ); + } - if ottyl_reserved_numeric_value_fn(&parsed.metric_name).is_none() - && !index.contains_key(&IndexKey::MetricDeclaration { name: parsed.metric_name.clone() }) - && bucket_values(index, &parsed.metric_name, None).next().is_none() - { - return Err(format!( - "unknown metric \"{}\": no declaration or recorded data for it (was it deleted?)", - parsed.metric_name - )); - } - if matches!(parsed.agg, Some(AggOp::Sum) | Some(AggOp::Avg)) - && !parsed.inner_rate - && declared_value_kind(index, &parsed.metric_name) == Some(MetricPrimitiveKind::String) - { - return Err(format!( - "cannot sum/avg a string-typed metric \"{}\"; use count() or a numeric-typed metric", - parsed.metric_name - )); - } + if ottyl_reserved_numeric_value_fn(&parsed.metric_name).is_none() + && !self.index.contains_key(&IndexKey::MetricDeclaration { name: parsed.metric_name.clone() }) + && self.bucket_values(&parsed.metric_name, None).next().is_none() + { + return Err(format!( + "unknown metric \"{}\": no declaration or recorded data for it (was it deleted?)", + parsed.metric_name + )); + } + if matches!(parsed.agg, Some(AggOp::Sum) | Some(AggOp::Avg)) + && !parsed.inner_rate + && self.declared_value_kind(&parsed.metric_name) == Some(MetricPrimitiveKind::String) + { + return Err(format!( + "cannot sum/avg a string-typed metric \"{}\"; use count() or a numeric-typed metric", + parsed.metric_name + )); + } - let mut series_points: BTreeMap, Vec<(i64, f64)>> = BTreeMap::new(); - if let Some(reserved_value) = ottyl_reserved_numeric_value_fn(&parsed.metric_name) { - let no_labels: BTreeMap = BTreeMap::new(); - if matches_label_matchers(&no_labels, &parsed.matchers) { - for (sequence, timestamp, _text) in logs_range(index) { - let timestamp_ms = timestamp.as_millis(); - if start_ms.is_none_or(|start| timestamp_ms >= start) - && end_ms.is_none_or(|end| timestamp_ms <= end) - { - series_points - .entry(BTreeMap::new()) - .or_default() - .push((timestamp_ms, reserved_value(sequence, timestamp_ms))); + let mut series_points: BTreeMap, Vec<(i64, f64)>> = BTreeMap::new(); + if let Some(reserved_value) = ottyl_reserved_numeric_value_fn(&parsed.metric_name) { + let no_labels: BTreeMap = BTreeMap::new(); + if matches_label_matchers(&no_labels, &parsed.matchers) { + for (sequence, timestamp, _text) in self.logs_range() { + let timestamp_ms = timestamp.as_millis(); + if start_ms.is_none_or(|start| timestamp_ms >= start) + && end_ms.is_none_or(|end| timestamp_ms <= end) + { + series_points + .entry(BTreeMap::new()) + .or_default() + .push((timestamp_ms, reserved_value(sequence, timestamp_ms))); + } } } + } else { + for (labels, (_, timestamp, value)) in self.bucket_values(&parsed.metric_name, None) + .filter(|(labels, _)| matches_label_matchers(labels, &parsed.matchers)) + .filter(|(_, (_, timestamp, _))| { + start_ms.is_none_or(|start| timestamp.as_millis() >= start) + && end_ms.is_none_or(|end| timestamp.as_millis() <= end) + }) + { + let display_labels: BTreeMap = + labels.iter().map(|(name, value)| (name.clone(), value.to_string())).collect(); + series_points + .entry(display_labels) + .or_default() + .push((timestamp.as_millis(), coerce_to_f64(value))); + } } - } else { - for (labels, (_, timestamp, value)) in bucket_values(index, &parsed.metric_name, None) - .filter(|(labels, _)| matches_label_matchers(labels, &parsed.matchers)) - .filter(|(_, (_, timestamp, _))| { - start_ms.is_none_or(|start| timestamp.as_millis() >= start) - && end_ms.is_none_or(|end| timestamp.as_millis() <= end) - }) - { - let display_labels: BTreeMap = - labels.iter().map(|(name, value)| (name.clone(), value.to_string())).collect(); - series_points - .entry(display_labels) - .or_default() - .push((timestamp.as_millis(), coerce_to_f64(value))); - } - } for points in series_points.values_mut() { points.sort_by_key(|(timestamp_ms, _)| *timestamp_ms); } @@ -2136,7 +2119,8 @@ fn evaluate_promql( }) .collect() } - }) + }) + } } #[cfg(test)] @@ -2158,11 +2142,11 @@ mod tests { #[test] fn declares_valid_param() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); let result = declare_param(&mut index, request("level", r"level=(?P\w+)", "level")); assert!(result.is_ok()); - let declarations: Vec<_> = param_declarations_range(&index).collect(); + let declarations: Vec<_> = index.param_declarations_range().collect(); assert_eq!(declarations.len(), 1); assert_eq!(declarations[0].0, "level"); assert_eq!(declarations[0].1, r"level=(?P\w+)"); @@ -2171,17 +2155,17 @@ mod tests { #[test] fn auto_declare_params_discovers_new_keys_with_inferred_kinds_and_flags_them_auto() { - let mut index = BTreeMap::new(); - auto_declare_params(&mut index, r#"ts=1788648105773 msg="entity_moved" x1=-4 y1=34"#); + let mut index = EngineSnapshot::default(); + index.auto_declare_params(r#"ts=1788648105773 msg="entity_moved" x1=-4 y1=34"#); let declarations: HashMap<&str, _> = - param_declarations_range(&index).map(|d| (d.0, d)).collect(); + index.param_declarations_range().map(|d| (d.0, d)).collect(); assert_eq!(declarations.len(), 4); assert_eq!(declarations["x1"].4, MetricPrimitiveKind::Number); assert!(declarations["x1"].6, "expected x1 to be flagged auto_derived"); assert_eq!(declarations["msg"].4, MetricPrimitiveKind::String); - let compiled = compiled_param_declarations(&index); + let compiled = index.compiled_param_declarations(); let params = derive_log_params(r#"ts=1788648105773 msg="entity_moved" x1=-4 y1=34"#, &compiled); assert_eq!(params.get("x1"), Some(&MetricPrimitive::Number(Number::Int(-4)))); assert_eq!(params.get("msg"), Some(&MetricPrimitive::String("entity_moved".to_string()))); @@ -2189,18 +2173,18 @@ mod tests { #[test] fn auto_declare_params_discovers_json_leaf_fields_via_json_field_path() { - let mut index = BTreeMap::new(); - auto_declare_params(&mut index, r#"{"level":"error","code":500,"request":{"path":"/x"}}"#); + let mut index = EngineSnapshot::default(); + index.auto_declare_params(r#"{"level":"error","code":500,"request":{"path":"/x"}}"#); let declarations: HashMap<&str, _> = - param_declarations_range(&index).map(|d| (d.0, d)).collect(); + index.param_declarations_range().map(|d| (d.0, d)).collect(); assert_eq!(declarations.len(), 3); assert_eq!(declarations["code"].4, MetricPrimitiveKind::Number); assert_eq!(declarations["code"].3, Some("code")); assert!(declarations["level"].6, "expected level to be flagged auto_derived"); assert_eq!(declarations["request.path"].3, Some("request.path")); - let compiled = compiled_param_declarations(&index); + let compiled = index.compiled_param_declarations(); let params = derive_log_params(r#"{"level":"error","code":500,"request":{"path":"/x"}}"#, &compiled); assert_eq!(params.get("level"), Some(&MetricPrimitive::String("error".to_string()))); @@ -2210,12 +2194,12 @@ mod tests { #[test] fn auto_declare_params_never_overwrites_an_existing_declaration() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_param(&mut index, request("x1", r#"x1="(?P[^"]+)""#, "x1")).unwrap(); - auto_declare_params(&mut index, "x1=-4"); + index.auto_declare_params("x1=-4"); - let declarations: Vec<_> = param_declarations_range(&index).collect(); + let declarations: Vec<_> = index.param_declarations_range().collect(); assert_eq!(declarations.len(), 1); assert_eq!(declarations[0].1, r#"x1="(?P[^"]+)""#, "manual declaration must survive untouched"); assert!(!declarations[0].6, "a manual declaration must never be flagged auto_derived"); @@ -2223,34 +2207,34 @@ mod tests { #[test] fn declare_param_rejects_editing_an_auto_derived_param() { - let mut index = BTreeMap::new(); - auto_declare_params(&mut index, "x1=-4"); + let mut index = EngineSnapshot::default(); + index.auto_declare_params("x1=-4"); let result = declare_param(&mut index, request("x1", r#"x1="(?P[^"]+)""#, "x1")); let err = result.unwrap_err(); assert!(err.contains("auto-detected"), "unexpected error message: {err}"); - let declarations: Vec<_> = param_declarations_range(&index).collect(); + let declarations: Vec<_> = index.param_declarations_range().collect(); assert_eq!(declarations.len(), 1); assert!(declarations[0].6, "the original auto-derived declaration must be untouched"); } const LOG_STREAM_FIXTURE: &str = include_str!("fixtures/log_stream_sample.txt"); - fn ingest_log_stream_fixture(index: &mut BTreeMap) { + fn ingest_log_stream_fixture(index: &mut EngineSnapshot) { let mut header = None; for line in LOG_STREAM_FIXTURE.lines() { - auto_declare_log_stream_params(index, &mut header, line); - auto_declare_params(index, line); + auto_declare_log_stream_params(&mut index.index, &mut header, line); + index.auto_declare_params(line); } } #[test] fn ingesting_log_stream_fixture_does_not_blow_up_declared_param_count() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); ingest_log_stream_fixture(&mut index); - let declared = param_declarations_range(&index).count(); + let declared = index.param_declarations_range().count(); assert!( declared <= MAX_AUTO_DECLARED_PARAMS, "expected at most {MAX_AUTO_DECLARED_PARAMS} declared params, got {declared}" @@ -2260,25 +2244,25 @@ mod tests { #[test] fn base64_padding_fragments_in_log_stream_fixture_are_never_auto_declared() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); ingest_log_stream_fixture(&mut index); - let names: HashSet<&str> = param_declarations_range(&index).map(|d| d.0).collect(); + let names: HashSet<&str> = index.param_declarations_range().map(|d| d.0).collect(); assert!(!names.contains("rKTEtG7JmnThhoKPbZiVrA")); assert!(!names.contains("gEdA8N4g")); } #[test] fn log_stream_shaped_lines_expose_header_named_columns_as_queryable_params() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); ingest_log_stream_fixture(&mut index); - let declared: HashSet<&str> = param_declarations_range(&index).map(|d| d.0).collect(); + let declared: HashSet<&str> = index.param_declarations_range().map(|d| d.0).collect(); for name in ["Timestamp", "Thread", "Type", "Activity", "PID", "TTL", "message"] { assert!(declared.contains(name), "expected {name} to be declared, got {declared:?}"); } - let compiled = compiled_param_declarations(&index); + let compiled = index.compiled_param_declarations(); let data_line = LOG_STREAM_FIXTURE.lines().nth(1).unwrap(); let params = derive_log_params(data_line, &compiled); assert_eq!(params.get("Type"), Some(&MetricPrimitive::String("Default".to_string()))); @@ -2294,9 +2278,9 @@ mod tests { #[test] fn log_stream_columns_are_not_contaminated_by_non_columnar_lines() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); ingest_log_stream_fixture(&mut index); - let compiled = compiled_param_declarations(&index); + let compiled = index.compiled_param_declarations(); let continuation_line = "so_gencnt: 4042707 so_state: 0x0"; let params = derive_log_params(continuation_line, &compiled); @@ -2313,14 +2297,14 @@ mod tests { #[test] fn logs_query_sorted_by_a_log_stream_column_actually_reorders_real_log_stream_output() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); let mut header = None; for (i, line) in LOG_STREAM_FIXTURE.lines().enumerate() { let sequence = i as u64; let timestamp = Timestamp::from_millis(sequence as i64); - auto_declare_log_stream_params(&mut index, &mut header, line); - auto_declare_params(&mut index, line); - index.insert( + auto_declare_log_stream_params(&mut index.index, &mut header, line); + index.auto_declare_params(line); + index.index.insert( IndexKey::IngestedLog { sequence, timestamp }, DataValue::LogLiteral { text: line.to_string() }, ); @@ -2351,11 +2335,11 @@ mod tests { #[test] fn non_log_stream_lines_do_not_get_any_columns_declared_without_a_cached_header() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); let mut header = None; - auto_declare_log_stream_params(&mut index, &mut header, r#"ts=1788648105773 msg="entity_moved" x1=-4"#); + auto_declare_log_stream_params(&mut index.index, &mut header, r#"ts=1788648105773 msg="entity_moved" x1=-4"#); - assert!(param_declarations_range(&index).next().is_none()); + assert!(index.param_declarations_range().next().is_none()); assert!(header.is_none()); } @@ -2363,13 +2347,13 @@ mod tests { #[ignore] fn bench_ingest_fixture_throughput() { let repeats = 50; - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); let start = std::time::Instant::now(); for _ in 0..repeats { let mut header = None; for line in LOG_STREAM_FIXTURE.lines() { - auto_declare_log_stream_params(&mut index, &mut header, line); - auto_declare_params(&mut index, line); + auto_declare_log_stream_params(&mut index.index, &mut header, line); + index.auto_declare_params(line); } } let elapsed = start.elapsed(); @@ -2377,42 +2361,42 @@ mod tests { eprintln!( "ingested {total_lines} lines in {elapsed:?} ({:.0} lines/sec), {} params declared", total_lines as f64 / elapsed.as_secs_f64(), - param_declarations_range(&index).count(), + index.param_declarations_range().count(), ); } #[test] fn rejects_invalid_regex() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); let result = declare_param(&mut index, request("level", "level=(", "level")); assert!(result.is_err()); - assert!(index.is_empty()); + assert!(index.index.is_empty()); } #[test] fn rejects_unknown_capture_group() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); let result = declare_param( &mut index, request("level", r"level=(?P\w+)", "missing"), ); assert!(result.is_err()); - assert!(index.is_empty()); + assert!(index.index.is_empty()); } #[test] fn declares_valid_elapsed_since_last_param_without_capture_group() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_elapsed_since_last_param(&mut index, "gap"); - let declarations: Vec<_> = param_declarations_range(&index).collect(); + let declarations: Vec<_> = index.param_declarations_range().collect(); assert_eq!(declarations.len(), 1); assert!(declarations[0].5, "expected elapsed_since_last to be true"); } #[test] fn rejects_elapsed_since_last_param_with_non_duration_kind() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); let result = declare_param( &mut index, DeclareParamRequest { @@ -2427,12 +2411,12 @@ mod tests { }, ); assert!(result.is_err()); - assert!(index.is_empty()); + assert!(index.index.is_empty()); } #[test] fn declares_valid_param_from_json_field_path() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); let result = declare_param( &mut index, DeclareParamRequest { @@ -2448,14 +2432,14 @@ mod tests { ); assert!(result.is_ok()); - let declarations: Vec<_> = param_declarations_range(&index).collect(); + let declarations: Vec<_> = index.param_declarations_range().collect(); assert_eq!(declarations.len(), 1); assert_eq!(declarations[0].3, Some("foo.bar")); } #[test] fn derives_log_param_from_json_field_path() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_param( &mut index, DeclareParamRequest { @@ -2471,7 +2455,7 @@ mod tests { ) .unwrap(); - let declarations = compiled_param_declarations(&index); + let declarations = index.compiled_param_declarations(); let params = derive_log_params(r#"{"foo":{"bar":42}}"#, &declarations); assert_eq!(params.get("bar"), Some(&MetricPrimitive::Number(Number::Int(42)))); @@ -2481,50 +2465,46 @@ mod tests { #[test] fn evict_excess_logs_drops_oldest_sequences_first() { - let mut index = BTreeMap::new(); - let mut log_count = 0; - let mut approx_bytes_total = 0u64; + let mut index = EngineSnapshot::default(); for sequence in 0..5u64 { let text = format!("line {sequence}"); - approx_bytes_total += approx_log_bytes(&text); - index.insert( + index.approx_bytes_total += approx_log_bytes(&text); + index.index.insert( IndexKey::IngestedLog { sequence, timestamp: Timestamp::from_millis(sequence as i64) }, DataValue::LogLiteral { text }, ); - log_count += 1; + index.log_count += 1; } - evict_excess_logs(&mut index, &mut log_count, Some(3), &mut approx_bytes_total); + index.evict_excess_logs(Some(3)); - assert_eq!(log_count, 3); - let remaining: Vec<_> = logs_range(&index).map(|(seq, _, _)| seq).collect(); + assert_eq!(index.log_count, 3); + let remaining: Vec<_> = index.logs_range().map(|(seq, _, _)| seq).collect(); assert_eq!(remaining, vec![2, 3, 4]); let expected_bytes: u64 = remaining.iter().map(|seq| approx_log_bytes(&format!("line {seq}"))).sum(); - assert_eq!(approx_bytes_total, expected_bytes); + assert_eq!(index.approx_bytes_total, expected_bytes); } #[test] fn evict_excess_logs_is_noop_when_unlimited_or_under_cap() { - let mut index = BTreeMap::new(); - let mut log_count = 0; - let mut approx_bytes_total = 0u64; + let mut index = EngineSnapshot::default(); for sequence in 0..3u64 { - approx_bytes_total += approx_log_bytes("x"); - index.insert( + index.approx_bytes_total += approx_log_bytes("x"); + index.index.insert( IndexKey::IngestedLog { sequence, timestamp: Timestamp::from_millis(sequence as i64) }, DataValue::LogLiteral { text: "x".to_string() }, ); - log_count += 1; + index.log_count += 1; } - evict_excess_logs(&mut index, &mut log_count, None, &mut approx_bytes_total); - assert_eq!(log_count, 3); - assert_eq!(approx_bytes_total, 3); + index.evict_excess_logs(None); + assert_eq!(index.log_count, 3); + assert_eq!(index.approx_bytes_total, 3); - evict_excess_logs(&mut index, &mut log_count, Some(10), &mut approx_bytes_total); - assert_eq!(log_count, 3); - assert_eq!(approx_bytes_total, 3); + index.evict_excess_logs(Some(10)); + assert_eq!(index.log_count, 3); + assert_eq!(index.approx_bytes_total, 3); } @@ -2576,8 +2556,8 @@ mod tests { assert_eq!(ingest_lag_high_watermark_ms(&samples, expired), 5); } - fn insert_ingested_log(index: &mut BTreeMap, sequence: u64, text: &str) { - index.insert( + fn insert_ingested_log(index: &mut EngineSnapshot, sequence: u64, text: &str) { + index.index.insert( IndexKey::IngestedLog { sequence, timestamp: Timestamp::from_millis(sequence as i64) }, DataValue::LogLiteral { text: text.to_string() }, ); @@ -2585,35 +2565,35 @@ mod tests { #[test] fn clear_all_param_declarations_removes_manual_and_auto_derived() { - let mut index = BTreeMap::new(); - auto_declare_params(&mut index, "x1=-4"); + let mut index = EngineSnapshot::default(); + index.auto_declare_params("x1=-4"); declare_param(&mut index, request("level", r"level=(?P\w+)", "level")).unwrap(); - assert_eq!(param_declarations_range(&index).count(), 2); + assert_eq!(index.param_declarations_range().count(), 2); - let removed = clear_all_param_declarations(&mut index); + let removed = index.clear_all_param_declarations(); assert_eq!(removed, 2); - assert_eq!(param_declarations_range(&index).count(), 0); + assert_eq!(index.param_declarations_range().count(), 0); } #[test] fn reextract_params_from_retained_logs_rebuilds_from_ingested_text_without_new_ingest() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); insert_ingested_log(&mut index, 0, r#"level=info msg="started""#); insert_ingested_log(&mut index, 1, "x1=-4 y1=34"); - auto_declare_params(&mut index, r#"level=info msg="started""#); - auto_declare_params(&mut index, "x1=-4 y1=34"); - assert_eq!(param_declarations_range(&index).count(), 4); + index.auto_declare_params(r#"level=info msg="started""#); + index.auto_declare_params("x1=-4 y1=34"); + assert_eq!(index.param_declarations_range().count(), 4); - clear_all_param_declarations(&mut index); - assert_eq!(param_declarations_range(&index).count(), 0); + index.clear_all_param_declarations(); + assert_eq!(index.param_declarations_range().count(), 0); - let (reextracted, logs_considered) = reextract_params_from_retained_logs(&mut index); + let (reextracted, logs_considered) = index.reextract_params_from_retained_logs(); assert_eq!(logs_considered, 2); assert_eq!(reextracted, 4); let declarations: HashMap<&str, _> = - param_declarations_range(&index).map(|d| (d.0, d)).collect(); + index.param_declarations_range().map(|d| (d.0, d)).collect(); assert!(declarations.contains_key("level")); assert!(declarations.contains_key("msg")); assert!(declarations.contains_key("x1")); @@ -2623,30 +2603,30 @@ mod tests { #[test] fn reextract_params_from_retained_logs_only_covers_lines_still_in_the_index() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); insert_ingested_log(&mut index, 1, "x1=-4"); - auto_declare_params(&mut index, "evicted_only_key=1"); - auto_declare_params(&mut index, "x1=-4"); - assert_eq!(param_declarations_range(&index).count(), 2); + index.auto_declare_params("evicted_only_key=1"); + index.auto_declare_params("x1=-4"); + assert_eq!(index.param_declarations_range().count(), 2); - clear_all_param_declarations(&mut index); - let (reextracted, logs_considered) = reextract_params_from_retained_logs(&mut index); + index.clear_all_param_declarations(); + let (reextracted, logs_considered) = index.reextract_params_from_retained_logs(); assert_eq!(logs_considered, 1, "only the retained log line should be re-scanned"); assert_eq!(reextracted, 1, "the evicted line's key can't come back"); - let declarations: Vec<_> = param_declarations_range(&index).collect(); + let declarations: Vec<_> = index.param_declarations_range().collect(); assert_eq!(declarations[0].0, "x1"); } #[test] fn declarations_stay_within_their_range() { - let mut index = BTreeMap::new(); - index.insert( + let mut index = EngineSnapshot::default(); + index.index.insert( IndexKey::IngestedLog { sequence: 0, timestamp: Timestamp::from_millis(0) }, DataValue::LogLiteral { text: "hello".to_string() }, ); declare_param(&mut index, request("level", r"level=(?P\w+)", "level")).unwrap(); - index.insert( + index.index.insert( IndexKey::MetricDeclaration { name: "requests".to_string() }, DataValue::MetricDeclaration { message_match_regex: "x".to_string(), @@ -2657,25 +2637,25 @@ mod tests { }, ); - let declarations: Vec<_> = param_declarations_range(&index).collect(); + let declarations: Vec<_> = index.param_declarations_range().collect(); assert_eq!(declarations.len(), 1); assert_eq!(declarations[0].0, "level"); - for key in index.range(IndexKey::LogParamSplit..IndexKey::ParamMetricSplit).map(|(k, _)| k) { + for key in index.index.range(IndexKey::LogParamSplit..IndexKey::ParamMetricSplit).map(|(k, _)| k) { assert!(matches!(key, IndexKey::ParamDeclaration { .. })); } } #[test] fn removes_declaration() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_param(&mut index, request("level", r"level=(?P\w+)", "level")).unwrap(); - assert!(index.remove(&IndexKey::ParamDeclaration { name: "level".to_string() }).is_some()); - assert!(param_declarations_range(&index).next().is_none()); + assert!(index.index.remove(&IndexKey::ParamDeclaration { name: "level".to_string() }).is_some()); + assert!(index.param_declarations_range().next().is_none()); } fn declare_test_param( - index: &mut BTreeMap, + index: &mut EngineSnapshot, name: &str, pattern: &str, capture_group: &str, @@ -2697,7 +2677,7 @@ mod tests { .unwrap(); } - fn declare_elapsed_since_last_param(index: &mut BTreeMap, name: &str) { + fn declare_elapsed_since_last_param(index: &mut EngineSnapshot, name: &str) { declare_param( index, DeclareParamRequest { @@ -2736,15 +2716,15 @@ mod tests { #[test] fn declares_valid_metric() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); let pattern = r"request path=(?P\S+) duration=(?P\d+)"; declare_test_param(&mut index, "path", pattern, "path", MetricPrimitiveKind::String); declare_test_param(&mut index, "duration", pattern, "duration", MetricPrimitiveKind::Number); - let result = declare_metric(&mut index, metric_request("requests", pattern, vec!["path", "duration"])); + let result = index.declare_metric(metric_request("requests", pattern, vec!["path", "duration"])); assert!(result.is_ok()); - let declarations: Vec<_> = metric_declarations_range(&index).collect(); + let declarations: Vec<_> = index.metric_declarations_range().collect(); assert_eq!(declarations.len(), 1); assert_eq!(declarations[0].0, "requests"); assert_eq!(declarations[0].1, pattern); @@ -2753,110 +2733,106 @@ mod tests { #[test] fn rejects_invalid_metric_regex() { - let mut index = BTreeMap::new(); - let result = declare_metric(&mut index, metric_request("requests", "path=(", vec![])); + let mut index = EngineSnapshot::default(); + let result = index.declare_metric(metric_request("requests", "path=(", vec![])); assert!(result.is_err()); - assert!(index.is_empty()); + assert!(index.index.is_empty()); } #[test] fn rejects_metric_referencing_undeclared_param() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); let pattern = r"request path=(?P\S+)"; declare_test_param(&mut index, "path", pattern, "path", MetricPrimitiveKind::String); - let result = declare_metric(&mut index, metric_request("requests", pattern, vec!["path", "duration"])); + let result = index.declare_metric(metric_request("requests", pattern, vec!["path", "duration"])); assert!(result.is_err()); assert!(result.unwrap_err().contains("duration")); - assert!(!index.contains_key(&IndexKey::MetricDeclaration { name: "requests".to_string() })); + assert!(!index.index.contains_key(&IndexKey::MetricDeclaration { name: "requests".to_string() })); } #[test] fn metric_declarations_stay_within_their_range() { - let mut index = BTreeMap::new(); - index.insert( + let mut index = EngineSnapshot::default(); + index.index.insert( IndexKey::IngestedLog { sequence: 0, timestamp: Timestamp::from_millis(0) }, DataValue::LogLiteral { text: "hello".to_string() }, ); declare_param(&mut index, request("level", r"level=(?P\w+)", "level")).unwrap(); let pattern = r"duration=(?P\d+)"; declare_test_param(&mut index, "duration", pattern, "duration", MetricPrimitiveKind::Number); - declare_metric(&mut index, metric_request("requests", pattern, vec!["duration"])).unwrap(); + index.declare_metric(metric_request("requests", pattern, vec!["duration"])).unwrap(); - let declarations: Vec<_> = metric_declarations_range(&index).collect(); + let declarations: Vec<_> = index.metric_declarations_range().collect(); assert_eq!(declarations.len(), 1); assert_eq!(declarations[0].0, "requests"); - for key in index.range(IndexKey::ParamMetricSplit..IndexKey::MetricBucketSplit).map(|(k, _)| k) { + for key in index.index.range(IndexKey::ParamMetricSplit..IndexKey::MetricBucketSplit).map(|(k, _)| k) { assert!(matches!(key, IndexKey::MetricDeclaration { .. })); } } #[test] fn removes_metric_declaration() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); let pattern = r"duration=(?P\d+)"; declare_test_param(&mut index, "duration", pattern, "duration", MetricPrimitiveKind::Number); - declare_metric(&mut index, metric_request("requests", pattern, vec!["duration"])).unwrap(); - assert!(index.remove(&IndexKey::MetricDeclaration { name: "requests".to_string() }).is_some()); - assert!(metric_declarations_range(&index).next().is_none()); + index.declare_metric(metric_request("requests", pattern, vec!["duration"])).unwrap(); + assert!(index.index.remove(&IndexKey::MetricDeclaration { name: "requests".to_string() }).is_some()); + assert!(index.metric_declarations_range().next().is_none()); } #[test] fn remove_metric_declaration_reports_whether_it_existed() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_requests_metric(&mut index); - assert!(remove_metric_declaration(&mut index, "requests")); - assert!(!remove_metric_declaration(&mut index, "requests")); - assert!(!remove_metric_declaration(&mut index, "never-declared")); + assert!(index.remove_metric_declaration("requests")); + assert!(!index.remove_metric_declaration("requests")); + assert!(!index.remove_metric_declaration("never-declared")); } #[test] fn remove_metric_declaration_purges_bucket_history_so_promql_reports_unknown_metric() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_requests_metric(&mut index); ingest_log(&mut index, 0, "request path=/a"); let parsed = promql::parse_query("requests").unwrap(); - assert!(evaluate_promql(&index, &parsed, None, None).unwrap().len() == 1); + assert!(index.evaluate_promql(&parsed, None, None).unwrap().len() == 1); - assert!(remove_metric_declaration(&mut index, "requests")); - assert!(bucket_values(&index, "requests", None).next().is_none()); + assert!(index.remove_metric_declaration("requests")); + assert!(index.bucket_values("requests", None).next().is_none()); - let err = evaluate_promql(&index, &parsed, None, None).unwrap_err(); + let err = index.evaluate_promql(&parsed, None, None).unwrap_err(); assert!(err.contains("unknown metric"), "unexpected error message: {err}"); } #[test] fn undeclared_metric_with_bucket_data_is_still_queryable_via_promql() { - let mut index = BTreeMap::new(); - push_bucket_value( - &mut index, - "scraped_metric", + let mut index = EngineSnapshot::default(); + index.push_bucket_value("scraped_metric", BTreeMap::new(), 0, Timestamp::from_millis(0), MetricPrimitive::Number(Number::Float(1.0)), ); let parsed = promql::parse_query("scraped_metric").unwrap(); - let series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); assert_eq!(series.len(), 1); } #[test] fn promql_rejects_metric_with_no_declaration_and_no_data() { - let index = BTreeMap::new(); + let index = EngineSnapshot::default(); let parsed = promql::parse_query("never_declared").unwrap(); - let err = evaluate_promql(&index, &parsed, None, None).unwrap_err(); + let err = index.evaluate_promql(&parsed, None, None).unwrap_err(); assert!(err.contains("unknown metric"), "unexpected error message: {err}"); } #[test] fn update_metric_rejects_when_no_declaration_exists_yet() { - let mut index = BTreeMap::new(); - let result = update_metric( - &mut index, - "requests", + let mut index = EngineSnapshot::default(); + let result = index.update_metric("requests", metric_request("requests", r"request path=(?P\S+)", vec![]), ); assert!(result.is_err()); @@ -2865,12 +2841,10 @@ mod tests { #[test] fn update_metric_reuses_declare_metric_validation() { - let mut index = BTreeMap::new(); - declare_metric(&mut index, metric_request("requests", r"request path=(?P\S+)", vec![])).unwrap(); + let mut index = EngineSnapshot::default(); + index.declare_metric(metric_request("requests", r"request path=(?P\S+)", vec![])).unwrap(); - let result = update_metric( - &mut index, - "requests", + let result = index.update_metric("requests", metric_request("requests", r"request path=(?P\S+)", vec!["duration"]), ); assert!(result.is_err()); @@ -2879,16 +2853,16 @@ mod tests { #[test] fn update_metric_replaces_declaration_in_place_without_duplicating() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); let pattern = r"request path=(?P\S+) duration=(?P\d+)"; declare_test_param(&mut index, "path", pattern, "path", MetricPrimitiveKind::String); - declare_metric(&mut index, metric_request("requests", r"request path=(?P\S+)", vec!["path"])).unwrap(); + index.declare_metric(metric_request("requests", r"request path=(?P\S+)", vec!["path"])).unwrap(); declare_test_param(&mut index, "duration", pattern, "duration", MetricPrimitiveKind::Number); - let result = update_metric(&mut index, "requests", metric_request("requests", pattern, vec!["path", "duration"])); + let result = index.update_metric("requests", metric_request("requests", pattern, vec!["path", "duration"])); assert!(result.is_ok()); - let declarations: Vec<_> = metric_declarations_range(&index).collect(); + let declarations: Vec<_> = index.metric_declarations_range().collect(); assert_eq!(declarations.len(), 1); assert_eq!(declarations[0].0, "requests"); assert_eq!(declarations[0].2.len(), 2); @@ -2896,68 +2870,64 @@ mod tests { #[test] fn update_metric_rebackfills_bucket_data_from_the_new_declaration() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); ingest_log(&mut index, 0, "request path=/a"); ingest_log(&mut index, 1, "request path=/b"); declare_test_param(&mut index, "path", r"request path=(?P\S+)", "path", MetricPrimitiveKind::String); - declare_metric(&mut index, metric_request("requests", r"request path=(?P\S+)", vec!["path"])).unwrap(); - assert_eq!(bucket_values(&index, "requests", None).count(), 2); + index.declare_metric(metric_request("requests", r"request path=(?P\S+)", vec!["path"])).unwrap(); + assert_eq!(index.bucket_values("requests", None).count(), 2); - update_metric(&mut index, "requests", metric_request("requests", r"request path=(?P/a)", vec!["path"])).unwrap(); - assert_eq!(bucket_values(&index, "requests", None).count(), 1); + index.update_metric("requests", metric_request("requests", r"request path=(?P/a)", vec!["path"])).unwrap(); + assert_eq!(index.bucket_values("requests", None).count(), 1); } #[test] fn update_metric_can_rename_and_rejects_a_rename_onto_an_existing_name() { - let mut index = BTreeMap::new(); - declare_metric(&mut index, metric_request("requests", r"request path=(?P\S+)", vec![])).unwrap(); - declare_metric(&mut index, metric_request("errors", r"error code=(?P\d+)", vec![])).unwrap(); + let mut index = EngineSnapshot::default(); + index.declare_metric(metric_request("requests", r"request path=(?P\S+)", vec![])).unwrap(); + index.declare_metric(metric_request("errors", r"error code=(?P\d+)", vec![])).unwrap(); - let result = update_metric( - &mut index, - "requests", + let result = index.update_metric("requests", metric_request("errors", r"request path=(?P\S+)", vec![]), ); assert!(result.is_err()); - update_metric( - &mut index, - "requests", + index.update_metric("requests", metric_request("http_requests", r"request path=(?P\S+)", vec![]), ) .unwrap(); - let names: Vec<_> = metric_declarations_range(&index).map(|(name, ..)| name.to_string()).collect(); + let names: Vec<_> = index.metric_declarations_range().map(|(name, ..)| name.to_string()).collect(); assert!(names.contains(&"http_requests".to_string())); assert!(!names.contains(&"requests".to_string())); } - fn ingest_log(index: &mut BTreeMap, sequence: u64, text: &str) { + fn ingest_log(index: &mut EngineSnapshot, sequence: u64, text: &str) { let timestamp = Timestamp::from_millis(sequence as i64); - index.insert( + index.index.insert( IndexKey::IngestedLog { sequence, timestamp }, DataValue::LogLiteral { text: text.to_string() }, ); - apply_all_metrics(index, sequence, timestamp, text); + index.apply_all_metrics(sequence, timestamp, text); } #[test] fn declaring_metric_backfills_matching_logs() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); ingest_log(&mut index, 0, "request path=/a duration=1"); ingest_log(&mut index, 1, "request path=/b duration=2"); ingest_log(&mut index, 2, "unrelated line"); let pattern = r"request path=(?P\S+) duration=(?P\d+)"; declare_test_param(&mut index, "path", pattern, "path", MetricPrimitiveKind::String); - declare_metric(&mut index, metric_request("requests", pattern, vec!["path"])).unwrap(); + index.declare_metric(metric_request("requests", pattern, vec!["path"])).unwrap(); - let values: Vec<_> = bucket_values(&index, "requests", None).collect(); + let values: Vec<_> = index.bucket_values("requests", None).collect(); assert_eq!(values.len(), 2); } #[test] fn metric_param_extracts_from_json_field_path() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); ingest_log(&mut index, 0, r#"{"request":{"path":"/a"}}"#); ingest_log(&mut index, 1, "not json"); ingest_log(&mut index, 2, r#"{"request":{}}"#); @@ -2976,9 +2946,9 @@ mod tests { }, ) .unwrap(); - declare_metric(&mut index, metric_request("requests", r".", vec!["path"])).unwrap(); + index.declare_metric(metric_request("requests", r".", vec!["path"])).unwrap(); - let values: Vec<_> = bucket_values(&index, "requests", None).collect(); + let values: Vec<_> = index.bucket_values("requests", None).collect(); assert_eq!(values.len(), 1); assert_eq!( values[0].0.get("path"), @@ -2988,27 +2958,28 @@ mod tests { #[test] fn ingesting_after_declaration_updates_buckets_live() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_test_param(&mut index, "path", r"request path=(?P\S+)", "path", MetricPrimitiveKind::String); - declare_metric(&mut index, metric_request("requests", r"request path=(?P\S+)", vec!["path"])).unwrap(); + index.declare_metric(metric_request("requests", r"request path=(?P\S+)", vec!["path"])).unwrap(); ingest_log(&mut index, 0, "request path=/a"); - let values: Vec<_> = bucket_values(&index, "requests", None).collect(); + let values: Vec<_> = index.bucket_values("requests", None).collect(); assert_eq!(values.len(), 1); assert_eq!(values[0].1.0, 0); } #[test] fn non_matching_log_leaves_bucket_range_untouched() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_test_param(&mut index, "path", r"request path=(?P\S+)", "path", MetricPrimitiveKind::String); - declare_metric(&mut index, metric_request("requests", r"request path=(?P\S+)", vec!["path"])).unwrap(); + index.declare_metric(metric_request("requests", r"request path=(?P\S+)", vec!["path"])).unwrap(); ingest_log(&mut index, 0, "totally unrelated line"); assert!( index + .index .range(IndexKey::MetricBucketSplit..IndexKey::End) .find(|(key, _)| matches!(key, IndexKey::Bucket { .. })) .is_none() @@ -3017,15 +2988,15 @@ mod tests { #[test] fn bucket_values_preserve_insertion_order() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_test_param(&mut index, "path", r"request path=(?P\S+)", "path", MetricPrimitiveKind::String); - declare_metric(&mut index, metric_request("requests", r"request path=(?P\S+)", vec!["path"])).unwrap(); + index.declare_metric(metric_request("requests", r"request path=(?P\S+)", vec!["path"])).unwrap(); ingest_log(&mut index, 5, "request path=/a"); ingest_log(&mut index, 2, "request path=/a"); ingest_log(&mut index, 9, "request path=/a"); - let sequences: Vec = bucket_values(&index, "requests", None) + let sequences: Vec = index.bucket_values("requests", None) .map(|(_, (sequence, _, _))| *sequence) .collect(); assert_eq!(sequences, vec![5, 2, 9]); @@ -3033,14 +3004,15 @@ mod tests { #[test] fn differing_label_combinations_produce_distinct_buckets() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_test_param(&mut index, "path", r"request path=(?P\S+)", "path", MetricPrimitiveKind::String); - declare_metric(&mut index, metric_request("requests", r"request path=(?P\S+)", vec!["path"])).unwrap(); + index.declare_metric(metric_request("requests", r"request path=(?P\S+)", vec!["path"])).unwrap(); ingest_log(&mut index, 0, "request path=/a"); ingest_log(&mut index, 1, "request path=/b"); let bucket_keys: Vec<_> = index + .index .range(IndexKey::MetricBucketSplit..IndexKey::End) .filter(|(key, _)| matches!(key, IndexKey::Bucket { .. })) .collect(); @@ -3055,34 +3027,34 @@ mod tests { #[test] fn single_log_line_can_match_multiple_metrics() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); let pattern = r"request path=(?P\S+) duration=(?P\d+)"; declare_test_param(&mut index, "path", pattern, "path", MetricPrimitiveKind::String); declare_test_param(&mut index, "duration", pattern, "duration", MetricPrimitiveKind::Number); - declare_metric(&mut index, metric_request("requests", pattern, vec!["path"])).unwrap(); - declare_metric(&mut index, metric_request("durations", pattern, vec!["duration"])).unwrap(); + index.declare_metric(metric_request("requests", pattern, vec!["path"])).unwrap(); + index.declare_metric(metric_request("durations", pattern, vec!["duration"])).unwrap(); ingest_log(&mut index, 0, "request path=/a duration=5"); - assert_eq!(bucket_values(&index, "requests", None).count(), 1); - assert_eq!(bucket_values(&index, "durations", None).count(), 1); + assert_eq!(index.bucket_values("requests", None).count(), 1); + assert_eq!(index.bucket_values("durations", None).count(), 1); } - fn declare_requests_metric(index: &mut BTreeMap) { + fn declare_requests_metric(index: &mut EngineSnapshot) { declare_test_param(index, "path", r"request path=(?P\S+)", "path", MetricPrimitiveKind::String); - declare_metric(index, metric_request("requests", r"request path=(?P\S+)", vec!["path"])).unwrap(); + index.declare_metric(metric_request("requests", r"request path=(?P\S+)", vec!["path"])).unwrap(); } #[test] fn promql_query_selects_matching_label_combination() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_requests_metric(&mut index); ingest_log(&mut index, 0, "request path=/a"); ingest_log(&mut index, 1, "request path=/b"); ingest_log(&mut index, 2, "request path=/a"); let parsed = promql::parse_query(r#"requests{path="/a"}"#).unwrap(); - let series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); assert_eq!(series.len(), 1); assert_eq!(series[0].labels.get("path").map(String::as_str), Some("/a")); @@ -3091,24 +3063,24 @@ mod tests { #[test] fn promql_query_with_no_matching_labels_returns_no_series() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_requests_metric(&mut index); ingest_log(&mut index, 0, "request path=/a"); let parsed = promql::parse_query(r#"requests{path="/nope"}"#).unwrap(); - let series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); assert!(series.is_empty()); } #[test] fn promql_sum_aggregates_across_label_combinations() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_requests_metric(&mut index); ingest_log(&mut index, 0, "request path=/a"); ingest_log(&mut index, 0, "request path=/b"); let parsed = promql::parse_query("sum(requests)").unwrap(); - let series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); assert_eq!(series.len(), 1); assert!(series[0].labels.is_empty()); @@ -3117,14 +3089,14 @@ mod tests { #[test] fn promql_sum_by_preserves_the_named_label_dimension() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_requests_metric(&mut index); ingest_log(&mut index, 0, "request path=/a"); ingest_log(&mut index, 0, "request path=/b"); ingest_log(&mut index, 1, "request path=/a"); let parsed = promql::parse_query("sum(requests) by (path)").unwrap(); - let mut series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let mut series = index.evaluate_promql(&parsed, None, None).unwrap(); series.sort_by(|a, b| a.labels.get("path").cmp(&b.labels.get("path"))); assert_eq!(series.len(), 2); @@ -3134,7 +3106,7 @@ mod tests { assert_eq!(series[1].points, vec![(0, 1.0)]); } - fn declare_string_valued_metric(index: &mut BTreeMap) { + fn declare_string_valued_metric(index: &mut EngineSnapshot) { declare_test_param( index, "status", @@ -3142,9 +3114,7 @@ mod tests { "status", MetricPrimitiveKind::String, ); - declare_metric( - index, - metric_request_with_value( + index.declare_metric(metric_request_with_value( "statuses", r"request status=(?P\S+)", vec![], @@ -3154,7 +3124,7 @@ mod tests { .unwrap(); } - fn declare_requests_by_host_metric(index: &mut BTreeMap) { + fn declare_requests_by_host_metric(index: &mut EngineSnapshot) { declare_test_param( index, "path", @@ -3169,9 +3139,7 @@ mod tests { "host", MetricPrimitiveKind::String, ); - declare_metric( - index, - metric_request( + index.declare_metric(metric_request( "requests", r"request path=(?P\S+) host=(?P\S+)", vec!["path", "host"], @@ -3182,12 +3150,12 @@ mod tests { #[test] fn sum_over_string_kind_metric_is_rejected_with_an_actionable_error() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_string_valued_metric(&mut index); ingest_log(&mut index, 0, "request status=ok"); let parsed = promql::parse_query("sum(statuses)").unwrap(); - let err = evaluate_promql(&index, &parsed, None, None).unwrap_err(); + let err = index.evaluate_promql(&parsed, None, None).unwrap_err(); assert!(err.contains("statuses"), "error should name the offending metric: {err}"); assert!(err.to_lowercase().contains("string"), "error should explain why: {err}"); @@ -3195,25 +3163,25 @@ mod tests { #[test] fn avg_over_string_kind_metric_is_rejected_with_an_actionable_error() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_string_valued_metric(&mut index); ingest_log(&mut index, 0, "request status=ok"); let parsed = promql::parse_query("avg(statuses)").unwrap(); - let err = evaluate_promql(&index, &parsed, None, None).unwrap_err(); + let err = index.evaluate_promql(&parsed, None, None).unwrap_err(); assert!(err.contains("statuses")); } #[test] fn count_over_string_kind_metric_counts_occurrences_by_name_not_by_accident() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_string_valued_metric(&mut index); ingest_log(&mut index, 0, "request status=ok"); ingest_log(&mut index, 1, "request status=error"); let parsed = promql::parse_query("count(statuses)").unwrap(); - let series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); assert_eq!(series.len(), 1); assert!(series[0].labels.is_empty()); @@ -3222,13 +3190,13 @@ mod tests { #[test] fn sum_over_numeric_kind_metric_is_unaffected_by_the_string_kind_guard() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_requests_metric(&mut index); ingest_log(&mut index, 0, "request path=/a"); ingest_log(&mut index, 0, "request path=/b"); let parsed = promql::parse_query("sum(requests)").unwrap(); - let series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); assert_eq!(series.len(), 1); assert_eq!(series[0].points, vec![(0, 2.0)]); @@ -3236,39 +3204,39 @@ mod tests { #[test] fn sum_of_rate_over_string_kind_metric_is_still_allowed_since_rate_only_counts_occurrences() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_string_valued_metric(&mut index); ingest_log(&mut index, 0, "request status=ok"); ingest_log(&mut index, 1000, "request status=error"); let parsed = promql::parse_query("sum(rate(statuses[2s]))").unwrap(); - let series = evaluate_promql(&index, &parsed, None, None); + let series = index.evaluate_promql(&parsed, None, None); assert!(series.is_ok(), "sum(rate(...)) never reads the coerced value, so it should not be rejected: {series:?}"); } #[test] fn bare_query_over_string_kind_metric_is_still_allowed() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_string_valued_metric(&mut index); ingest_log(&mut index, 0, "request status=ok"); let parsed = promql::parse_query("statuses").unwrap(); - let series = evaluate_promql(&index, &parsed, None, None); + let series = index.evaluate_promql(&parsed, None, None); assert!(series.is_ok()); } #[test] fn sum_by_merges_staggered_raw_timestamps_across_label_groups_instead_of_going_sparse() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_requests_by_host_metric(&mut index); ingest_log(&mut index, 0, "request path=/a host=web1"); ingest_log(&mut index, 1, "request path=/a host=web2"); ingest_log(&mut index, 2, "request path=/a host=web3"); let parsed = promql::parse_query("sum(requests) by (path)").unwrap(); - let series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); assert_eq!(series.len(), 1); assert_eq!(series[0].labels.get("path").map(String::as_str), Some("/a")); @@ -3283,13 +3251,13 @@ mod tests { #[test] fn avg_by_merges_staggered_raw_timestamps_across_label_groups() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_requests_by_host_metric(&mut index); ingest_log(&mut index, 0, "request path=/a host=web1"); ingest_log(&mut index, 1, "request path=/a host=web2"); let parsed = promql::parse_query("avg(requests) by (path)").unwrap(); - let series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); assert_eq!(series.len(), 1); assert_eq!(series[0].labels.get("path").map(String::as_str), Some("/a")); @@ -3298,13 +3266,13 @@ mod tests { #[test] fn sum_by_still_gives_a_single_label_group_its_own_line_at_full_resolution() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_requests_metric(&mut index); ingest_log(&mut index, 0, "request path=/a"); ingest_log(&mut index, 1, "request path=/a"); let parsed = promql::parse_query("sum(requests) by (path)").unwrap(); - let series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); assert_eq!(series.len(), 1); assert_eq!(series[0].points, vec![(0, 1.0), (1, 1.0)]); @@ -3312,13 +3280,13 @@ mod tests { #[test] fn bare_rate_produces_a_windowed_point_per_timestamp_not_a_single_point() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_requests_metric(&mut index); ingest_log(&mut index, 0, "request path=/a"); ingest_log(&mut index, 1000, "request path=/a"); let parsed = promql::parse_query("rate(requests[2s])").unwrap(); - let series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); assert_eq!(series.len(), 1); assert_eq!(series[0].points, vec![(0, 0.5), (1000, 1.0)]); @@ -3326,14 +3294,14 @@ mod tests { #[test] fn sum_of_rate_by_label_gives_each_label_value_its_own_line() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_requests_metric(&mut index); ingest_log(&mut index, 0, "request path=/a"); ingest_log(&mut index, 1000, "request path=/a"); ingest_log(&mut index, 0, "request path=/b"); let parsed = promql::parse_query("sum(rate(requests[2s])) by (path)").unwrap(); - let mut series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let mut series = index.evaluate_promql(&parsed, None, None).unwrap(); series.sort_by(|a, b| a.labels.get("path").cmp(&b.labels.get("path"))); assert_eq!(series.len(), 2); @@ -3345,13 +3313,13 @@ mod tests { #[test] fn rate_series_gets_a_trailing_zero_point_once_occurrences_stop_early_in_a_wider_range() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_requests_metric(&mut index); ingest_log(&mut index, 0, "request path=/a"); ingest_log(&mut index, 1, "request path=/a"); let parsed = promql::parse_query("rate(requests[2s])").unwrap(); - let series = evaluate_promql(&index, &parsed, None, Some(100_000)).unwrap(); + let series = index.evaluate_promql(&parsed, None, Some(100_000)).unwrap(); assert_eq!(series.len(), 1); assert_eq!( @@ -3363,26 +3331,26 @@ mod tests { #[test] fn rate_series_gets_a_leading_zero_point_when_occurrences_start_late_in_a_wider_range() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_requests_metric(&mut index); ingest_log(&mut index, 50_000, "request path=/a"); let parsed = promql::parse_query("rate(requests[2s])").unwrap(); - let series = evaluate_promql(&index, &parsed, Some(0), None).unwrap(); + let series = index.evaluate_promql(&parsed, Some(0), None).unwrap(); assert_eq!(series[0].points, vec![(0, 0.0), (50_000, 0.5)]); } #[test] fn sum_by_rate_gets_boundary_points_per_group_across_a_wider_range() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_requests_metric(&mut index); ingest_log(&mut index, 0, "request path=/a"); ingest_log(&mut index, 1, "request path=/a"); ingest_log(&mut index, 0, "request path=/b"); let parsed = promql::parse_query("sum(rate(requests[2s])) by (path)").unwrap(); - let mut series = evaluate_promql(&index, &parsed, None, Some(100_000)).unwrap(); + let mut series = index.evaluate_promql(&parsed, None, Some(100_000)).unwrap(); series.sort_by(|a, b| a.labels.get("path").cmp(&b.labels.get("path"))); assert_eq!(series.len(), 2); @@ -3392,20 +3360,20 @@ mod tests { #[test] fn rate_series_skips_boundary_points_already_covered_by_real_data() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_requests_metric(&mut index); ingest_log(&mut index, 0, "request path=/a"); ingest_log(&mut index, 1000, "request path=/a"); let parsed = promql::parse_query("rate(requests[2s])").unwrap(); - let series = evaluate_promql(&index, &parsed, Some(0), Some(1000)).unwrap(); + let series = index.evaluate_promql(&parsed, Some(0), Some(1000)).unwrap(); assert_eq!(series[0].points, vec![(0, 0.5), (1000, 1.0)]); } #[test] fn dashboard_rate_and_alert_threshold_agree_on_same_windowed_data() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_requests_metric(&mut index); let lines: Vec<(u64, &str)> = vec![ @@ -3420,7 +3388,7 @@ mod tests { } let parsed = promql::parse_query("rate(requests[2s])").unwrap(); - let series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); assert_eq!(series.len(), 1); let (_, dashboard_rate) = *series[0].points.last().unwrap(); @@ -3458,13 +3426,13 @@ mod tests { #[test] fn sum_of_rate_without_by_clause_still_collapses_to_one_series() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_requests_metric(&mut index); ingest_log(&mut index, 0, "request path=/a"); ingest_log(&mut index, 0, "request path=/b"); let parsed = promql::parse_query("sum(rate(requests[2s]))").unwrap(); - let series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); assert_eq!(series.len(), 1); assert!(series[0].labels.is_empty()); @@ -3473,14 +3441,14 @@ mod tests { #[test] fn promql_range_bounds_filter_points_via_start_end_ms() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_requests_metric(&mut index); ingest_log(&mut index, 0, "request path=/a"); ingest_log(&mut index, 5, "request path=/a"); ingest_log(&mut index, 10, "request path=/a"); let parsed = promql::parse_query("requests").unwrap(); - let series = evaluate_promql(&index, &parsed, Some(1), Some(9)).unwrap(); + let series = index.evaluate_promql(&parsed, Some(1), Some(9)).unwrap(); assert_eq!(series.len(), 1); assert_eq!(series[0].points, vec![(5, 1.0)]); @@ -3488,13 +3456,11 @@ mod tests { #[test] fn declares_metric_with_value_param() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); let pattern = r"request path=(?P\S+) duration=(?P\d+)"; declare_test_param(&mut index, "path", pattern, "path", MetricPrimitiveKind::String); declare_test_param(&mut index, "duration", pattern, "duration", MetricPrimitiveKind::Number); - let result = declare_metric( - &mut index, - metric_request_with_value("durations", pattern, vec!["path"], Some("duration")), + let result = index.declare_metric(metric_request_with_value("durations", pattern, vec!["path"], Some("duration")), ); assert!(result.is_ok()); } @@ -3502,19 +3468,17 @@ mod tests { #[test] fn value_param_becomes_the_bucket_value_not_a_label() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); let pattern = r"request path=(?P\S+) duration=(?P\d+)"; declare_test_param(&mut index, "path", pattern, "path", MetricPrimitiveKind::String); declare_test_param(&mut index, "duration", pattern, "duration", MetricPrimitiveKind::Number); - declare_metric( - &mut index, - metric_request_with_value("durations", pattern, vec!["path"], Some("duration")), + index.declare_metric(metric_request_with_value("durations", pattern, vec!["path"], Some("duration")), ) .unwrap(); ingest_log(&mut index, 0, "request path=/a duration=5"); - let values: Vec<_> = bucket_values(&index, "durations", None).collect(); + let values: Vec<_> = index.bucket_values("durations", None).collect(); assert_eq!(values.len(), 1); let (labels, (_, _, value)) = &values[0]; assert!(!labels.contains_key("duration")); @@ -3522,27 +3486,25 @@ mod tests { assert_eq!(*value, MetricPrimitive::Number(Number::Int(5))); } - fn declare_durations_metric(index: &mut BTreeMap) { + fn declare_durations_metric(index: &mut EngineSnapshot) { let pattern = r"request path=(?P\S+) duration=(?P\d+)"; declare_test_param(index, "path", pattern, "path", MetricPrimitiveKind::String); declare_test_param(index, "duration", pattern, "duration", MetricPrimitiveKind::Number); - declare_metric( - index, - metric_request_with_value("durations", pattern, vec!["path"], Some("duration")), + index.declare_metric(metric_request_with_value("durations", pattern, vec!["path"], Some("duration")), ) .unwrap(); } #[test] fn gauge_returns_most_recent_value_per_label_combination() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_durations_metric(&mut index); ingest_log(&mut index, 0, "request path=/a duration=5"); ingest_log(&mut index, 1, "request path=/a duration=9"); ingest_log(&mut index, 2, "request path=/b duration=42"); let parsed = promql::parse_query("gauge(durations)").unwrap(); - let mut series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let mut series = index.evaluate_promql(&parsed, None, None).unwrap(); series.sort_by(|a, b| a.labels.get("path").cmp(&b.labels.get("path"))); assert_eq!(series.len(), 2); @@ -3554,14 +3516,14 @@ mod tests { #[test] fn gauge_respects_time_range_bounds() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_durations_metric(&mut index); ingest_log(&mut index, 0, "request path=/a duration=5"); ingest_log(&mut index, 5, "request path=/a duration=9"); ingest_log(&mut index, 10, "request path=/a duration=42"); let parsed = promql::parse_query("gauge(durations)").unwrap(); - let series = evaluate_promql(&index, &parsed, None, Some(9)).unwrap(); + let series = index.evaluate_promql(&parsed, None, Some(9)).unwrap(); assert_eq!(series.len(), 1); assert_eq!(series[0].points, vec![(5, 9.0)]); @@ -3569,14 +3531,14 @@ mod tests { #[test] fn histogram_buckets_captured_values_by_fixed_width() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_durations_metric(&mut index); for (seq, duration) in [(0, 2), (1, 4), (2, 12), (3, 15), (4, 21)] { ingest_log(&mut index, seq, &format!("request path=/a duration={duration}")); } let parsed = promql::parse_query("histogram(durations[10])").unwrap(); - let mut series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let mut series = index.evaluate_promql(&parsed, None, None).unwrap(); series.sort_by(|a, b| a.labels.get("le").cmp(&b.labels.get("le"))); assert_eq!(series.len(), 3); @@ -3590,7 +3552,7 @@ mod tests { #[test] fn histogram_with_by_clause_buckets_each_label_group_separately() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_durations_metric(&mut index); for (seq, path, duration) in [ (0, "/a", 2), @@ -3602,7 +3564,7 @@ mod tests { } let parsed = promql::parse_query("histogram(durations[10]) by (path)").unwrap(); - let mut series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let mut series = index.evaluate_promql(&parsed, None, None).unwrap(); series.sort_by(|a, b| { a.labels.get("path").cmp(&b.labels.get("path")).then(a.labels.get("le").cmp(&b.labels.get("le"))) }); @@ -3622,23 +3584,23 @@ mod tests { assert_eq!(series[2].points[0].1, 1.0); } - fn declare_login_duration_metric(index: &mut BTreeMap) { + fn declare_login_duration_metric(index: &mut EngineSnapshot) { let pattern = r"user=(?P\S+) logged in"; declare_test_param(index, "user", pattern, "user", MetricPrimitiveKind::String); declare_elapsed_since_last_param(index, "gap"); - declare_metric(index, metric_request_with_value("login_gap", pattern, vec!["user"], Some("gap"))).unwrap(); + index.declare_metric(metric_request_with_value("login_gap", pattern, vec!["user"], Some("gap"))).unwrap(); } #[test] fn duration_value_param_computes_elapsed_time_since_previous_same_labels_event() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_login_duration_metric(&mut index); ingest_log(&mut index, 0, "user=alice logged in"); ingest_log(&mut index, 1, "user=bob logged in"); ingest_log(&mut index, 2, "user=alice logged in"); - let values: Vec<_> = bucket_values(&index, "login_gap", None).collect(); + let values: Vec<_> = index.bucket_values("login_gap", None).collect(); assert_eq!(values.len(), 1); let (labels, (sequence, timestamp, value)) = &values[0]; assert_eq!(labels.get("user").map(|v| v.to_string()), Some("alice".to_string())); @@ -3649,14 +3611,14 @@ mod tests { #[test] fn duration_diffs_against_the_immediately_preceding_event_not_the_first() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_login_duration_metric(&mut index); ingest_log(&mut index, 0, "user=alice logged in"); ingest_log(&mut index, 3, "user=alice logged in"); ingest_log(&mut index, 9, "user=alice logged in"); - let values: Vec<_> = bucket_values(&index, "login_gap", None).collect(); + let values: Vec<_> = index.bucket_values("login_gap", None).collect(); assert_eq!(values.len(), 2); assert_eq!(values[0].1.2, MetricPrimitive::Duration(3)); assert_eq!(values[1].1.2, MetricPrimitive::Duration(6)); @@ -3664,22 +3626,20 @@ mod tests { #[test] fn duration_values_are_queryable_via_promql_like_any_other_metric() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_login_duration_metric(&mut index); ingest_log(&mut index, 0, "user=alice logged in"); ingest_log(&mut index, 10, "user=alice logged in"); let parsed = promql::parse_query("login_gap").unwrap(); - let series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); assert_eq!(series.len(), 1); assert_eq!(series[0].points, vec![(10, 10.0)]); } - fn declare_error_category_metric(index: &mut BTreeMap) { - declare_metric( - index, - DeclareMetricRequest { + fn declare_error_category_metric(index: &mut EngineSnapshot) { + index.declare_metric(DeclareMetricRequest { name: "log_category".to_string(), message_match_regex: r"(?i)error".to_string(), label_params: vec![], @@ -3693,7 +3653,7 @@ mod tests { #[test] fn unmatched_labels_registers_occurrence_for_non_matching_lines() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_error_category_metric(&mut index); ingest_log(&mut index, 0, "everything is fine"); @@ -3701,25 +3661,25 @@ mod tests { ingest_log(&mut index, 2, "still fine"); let error_labels = BTreeMap::from([("category".to_string(), MetricPrimitive::String("error".to_string()))]); - let error_values: Vec<_> = bucket_values(&index, "log_category", Some(&error_labels)).collect(); + let error_values: Vec<_> = index.bucket_values("log_category", Some(&error_labels)).collect(); assert_eq!(error_values.len(), 1); assert_eq!(error_values[0].1.0, 1); let other_labels = BTreeMap::from([("category".to_string(), MetricPrimitive::String("other".to_string()))]); - let other_values: Vec<_> = bucket_values(&index, "log_category", Some(&other_labels)).collect(); + let other_values: Vec<_> = index.bucket_values("log_category", Some(&other_labels)).collect(); assert_eq!(other_values.len(), 2); } #[test] fn unmatched_labels_produces_two_series_from_one_bare_promql_query() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_error_category_metric(&mut index); ingest_log(&mut index, 0, "an ERROR occurred"); ingest_log(&mut index, 1, "all good"); ingest_log(&mut index, 2, "all good again"); let parsed = promql::parse_query("log_category").unwrap(); - let mut series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let mut series = index.evaluate_promql(&parsed, None, None).unwrap(); series.sort_by(|a, b| a.labels.get("category").cmp(&b.labels.get("category"))); assert_eq!(series.len(), 2); @@ -3731,15 +3691,15 @@ mod tests { #[test] fn no_unmatched_labels_means_non_matching_lines_are_untouched() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_requests_metric(&mut index); ingest_log(&mut index, 0, "not a request line at all"); - assert_eq!(bucket_values(&index, "requests", None).count(), 0); + assert_eq!(index.bucket_values("requests", None).count(), 0); } #[test] fn handle_query_promql_rejects_malformed_query() { - let index = BTreeMap::new(); + let index = EngineSnapshot::default(); let response = handle_query( &index, EngineQuery::Promql { @@ -3759,7 +3719,7 @@ mod tests { #[test] fn handle_query_promql_returns_series_for_valid_query() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_requests_metric(&mut index); ingest_log(&mut index, 0, "request path=/a"); @@ -3785,25 +3745,25 @@ mod tests { #[test] fn promql_count_over_ottyl_seq_gives_one_point_per_ingested_log_with_no_metric_declaration() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); ingest_log(&mut index, 0, "line one"); ingest_log(&mut index, 1, "line two"); ingest_log(&mut index, 2, "line three"); let parsed = promql::parse_query("count(ottyl_seq)").unwrap(); - let series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); assert_eq!(series.len(), 1); assert_eq!(series[0].points, vec![(0, 1.0), (1, 1.0), (2, 1.0)]); } #[test] fn promql_bare_ottyl_seq_returns_the_actual_sequence_numbers() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); ingest_log(&mut index, 0, "line zero"); ingest_log(&mut index, 5, "line five"); let parsed = promql::parse_query("ottyl_seq").unwrap(); - let mut series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let mut series = index.evaluate_promql(&parsed, None, None).unwrap(); assert_eq!(series.len(), 1); series[0].points.sort_by_key(|(ts, _)| *ts); assert_eq!(series[0].points, vec![(0, 0.0), (5, 5.0)]); @@ -3811,53 +3771,53 @@ mod tests { #[test] fn promql_bare_ottyl_ts_returns_ingest_timestamps() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); ingest_log(&mut index, 0, "a"); ingest_log(&mut index, 1, "b"); let parsed = promql::parse_query("ottyl_ts").unwrap(); - let series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); assert_eq!(series.len(), 1); assert_eq!(series[0].points, vec![(0, 0.0), (1, 1.0)]); } #[test] fn promql_rate_over_ottyl_ts_produces_a_windowed_series_not_an_empty_one() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); for sequence in 0..5u64 { ingest_log(&mut index, sequence, "line"); } let parsed = promql::parse_query("rate(ottyl_ts[5s])").unwrap(); - let series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); assert_eq!(series.len(), 1); assert!(!series[0].points.is_empty(), "rate(ottyl_ts[...]) must not silently return no data"); } #[test] fn promql_ottyl_seq_with_label_matcher_matches_nothing_since_it_has_no_labels() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); ingest_log(&mut index, 0, "line"); let parsed = promql::parse_query(r#"ottyl_seq{path="/a"}"#).unwrap(); - let series = evaluate_promql(&index, &parsed, None, None).unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); assert!(series.is_empty()); } #[test] fn promql_over_ottyl_fmt_is_explicitly_rejected_not_silently_empty() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); ingest_log(&mut index, 0, r#"{"level":"error"}"#); let parsed = promql::parse_query("count(ottyl_fmt)").unwrap(); - let err = evaluate_promql(&index, &parsed, None, None).unwrap_err(); + let err = index.evaluate_promql(&parsed, None, None).unwrap_err(); assert!(err.contains("ottyl_fmt"), "error should name the rejected metric: {err}"); assert!(err.contains("categorical") || err.contains("filter"), "error should explain why: {err}"); } #[test] fn handle_query_promql_returns_series_for_ottyl_seq_end_to_end() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); ingest_log(&mut index, 0, "a"); ingest_log(&mut index, 1, "b"); @@ -3878,7 +3838,7 @@ mod tests { #[test] fn declare_param_rejects_reserved_ottyl_names() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); for name in OTTYL_RESERVED_PARAM_NAMES { let err = declare_param(&mut index, request(name, "x", "x")).unwrap_err(); assert!(err.contains("reserved"), "expected a 'reserved' error for '{name}', got: {err}"); @@ -3921,7 +3881,7 @@ mod tests { #[test] fn logs_query_response_includes_ottyl_fmt_per_entry() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); ingest_log(&mut index, 0, r#"{"level":"error"}"#); ingest_log(&mut index, 1, "level=info msg=hello"); ingest_log(&mut index, 2, "just plain prose"); @@ -3939,7 +3899,7 @@ mod tests { #[test] fn logs_query_with_regex_filter_returns_only_matching_lines() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); ingest_log(&mut index, 0, "everything is fine"); ingest_log(&mut index, 1, "an error occurred"); ingest_log(&mut index, 2, "still fine"); @@ -3956,7 +3916,7 @@ mod tests { #[test] fn logs_query_with_ottyl_seq_comparison_filters_by_sequence() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); for i in 0..5u64 { ingest_log(&mut index, i, &format!("line {i}")); } @@ -3972,10 +3932,10 @@ mod tests { #[test] fn logs_query_with_ottyl_ts_comparison_filters_by_timestamp() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); for i in 0..5u64 { let timestamp = Timestamp::from_millis(i as i64 * 1000); - index.insert( + index.index.insert( IndexKey::IngestedLog { sequence: i, timestamp }, DataValue::LogLiteral { text: format!("line {i}") }, ); @@ -3993,7 +3953,7 @@ mod tests { #[test] fn logs_query_sort_by_orders_lines_by_declared_param_value() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_test_param( &mut index, "priority", @@ -4016,7 +3976,7 @@ mod tests { #[test] fn logs_query_sort_by_missing_param_fallback_orders_by_sequence_and_appends() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_test_param( &mut index, "priority", @@ -4040,7 +4000,7 @@ mod tests { #[test] fn logs_query_sort_by_missing_param_drop_excludes_them() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_test_param( &mut index, "priority", @@ -4064,7 +4024,7 @@ mod tests { #[test] fn logs_query_sort_by_unknown_param_returns_error() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); ingest_log(&mut index, 0, "hello"); match handle_query(&index, logs_query_sorted("does_not_exist", None), 0, 0, &mut HashMap::new()) { @@ -4075,7 +4035,7 @@ mod tests { #[test] fn logs_query_without_sort_by_keeps_default_ottyl_ts_order() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_test_param( &mut index, "priority", @@ -4100,7 +4060,7 @@ mod tests { #[test] fn sort_helper_index_is_built_lazily_and_reused_across_queries() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_test_param( &mut index, "priority", @@ -4126,21 +4086,14 @@ mod tests { assert_eq!(sort_helper_indexes["priority"].index.len(), 3); ingest_log(&mut index, 3, "priority=0 boom"); - update_sort_helper_indexes_on_ingest( - &mut sort_helper_indexes, - &index, - 3, - Timestamp::from_millis(3), - "priority=0 boom", - ); + std::mem::swap(&mut index.sort_helper_indexes, &mut sort_helper_indexes); + index.update_sort_helper_indexes_on_ingest(3, Timestamp::from_millis(3), "priority=0 boom"); + std::mem::swap(&mut index.sort_helper_indexes, &mut sort_helper_indexes); + ingest_log(&mut index, 4, "no priority here"); - update_sort_helper_indexes_on_ingest( - &mut sort_helper_indexes, - &index, - 4, - Timestamp::from_millis(4), - "no priority here", - ); + std::mem::swap(&mut index.sort_helper_indexes, &mut sort_helper_indexes); + index.update_sort_helper_indexes_on_ingest(4, Timestamp::from_millis(4), "no priority here"); + std::mem::swap(&mut index.sort_helper_indexes, &mut sort_helper_indexes); assert_eq!(sort_helper_indexes["priority"].index.len(), 5); match handle_query(&index, logs_query_sorted("priority", None), 0, 0, &mut sort_helper_indexes) { @@ -4154,10 +4107,9 @@ mod tests { #[test] fn sort_helper_index_incremental_update_is_a_noop_when_no_index_is_live() { - let mut sort_helper_indexes: HashMap = HashMap::new(); - let index = BTreeMap::new(); - update_sort_helper_indexes_on_ingest(&mut sort_helper_indexes, &index, 0, Timestamp::from_millis(0), "hello"); - assert!(sort_helper_indexes.is_empty()); + let mut index = EngineSnapshot::default(); + index.update_sort_helper_indexes_on_ingest(0, Timestamp::from_millis(0), "hello"); + assert!(index.sort_helper_indexes.is_empty()); } @@ -4177,7 +4129,9 @@ mod tests { SortHelperIndexEntry { index: BTreeMap::new(), last_accessed: std::time::Instant::now() }, ); - purge_stale_sort_helper_indexes(&mut sort_helper_indexes); + let mut snapshot = EngineSnapshot { sort_helper_indexes, ..Default::default() }; + snapshot.purge_stale_sort_helper_indexes(); + let sort_helper_indexes = snapshot.sort_helper_indexes; assert!(!sort_helper_indexes.contains_key("stale")); assert!(sort_helper_indexes.contains_key("fresh")); @@ -4185,7 +4139,7 @@ mod tests { #[test] fn sorted_query_transparently_rebuilds_a_purged_helper_index() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_test_param( &mut index, "priority", @@ -4215,7 +4169,7 @@ mod tests { #[test] fn logs_query_with_ottyl_fmt_comparison_filters_by_detected_format_for_mixed_format_lines() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); ingest_log(&mut index, 0, r#"{"level":"error","code":500}"#); ingest_log(&mut index, 1, r#"ts=1788648105773 msg="entity_moved" x1=-4"#); ingest_log(&mut index, 2, "just some plain prose here"); @@ -4248,7 +4202,7 @@ mod tests { #[test] fn logs_query_with_json_field_comparison_filters_by_json_field() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); ingest_log(&mut index, 0, r#"{"level":"error","code":500}"#); ingest_log(&mut index, 1, r#"{"level":"info","code":200}"#); ingest_log(&mut index, 2, "not json at all"); @@ -4272,10 +4226,10 @@ mod tests { #[test] fn logs_query_combines_regex_and_metric_comparison_with_and() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); let pattern = r"level=(?P\w+) duration=(?P\d+)"; declare_test_param(&mut index, "level", pattern, "level", MetricPrimitiveKind::String); - declare_metric(&mut index, metric_request("requests", pattern, vec!["level"])).unwrap(); + index.declare_metric(metric_request("requests", pattern, vec!["level"])).unwrap(); ingest_log(&mut index, 0, "level=error duration=1 boom"); ingest_log(&mut index, 1, "level=info duration=2 boom"); @@ -4312,7 +4266,7 @@ mod tests { #[test] fn logs_query_includes_declared_params_on_matching_lines_only() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); declare_param(&mut index, request("level", r"level=(?P\w+)", "level")).unwrap(); ingest_log(&mut index, 0, "level=info hello"); ingest_log(&mut index, 1, "no match here"); @@ -4331,7 +4285,7 @@ mod tests { #[test] fn boolean_param_declares_without_capture_group_and_is_always_present() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); let mut req = request("IsError", r"(?i)error", ""); req.kind = MetricPrimitiveKind::Boolean; assert!(declare_param(&mut index, req).is_ok()); @@ -4350,7 +4304,7 @@ mod tests { #[test] fn timeline_query_applies_filter_before_bucketing() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); ingest_log(&mut index, 0, "everything is fine"); ingest_log(&mut index, 1, "an error occurred"); ingest_log(&mut index, 2, "another error here"); @@ -4366,7 +4320,7 @@ mod tests { #[test] fn malformed_filter_string_surfaces_as_engine_error() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); ingest_log(&mut index, 0, "hello"); match handle_query(&index, logs_query(Some("@@@ not a valid filter @@@")), 0, 0, &mut HashMap::new()) { @@ -4382,10 +4336,10 @@ mod tests { #[test] fn metric_comparison_without_param_is_a_compile_error() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); let pattern = r"level=(?P\w+)"; declare_test_param(&mut index, "level", pattern, "level", MetricPrimitiveKind::String); - declare_metric(&mut index, metric_request("requests", pattern, vec!["level"])).unwrap(); + index.declare_metric(metric_request("requests", pattern, vec!["level"])).unwrap(); ingest_log(&mut index, 0, "level=error"); match handle_query(&index, logs_query(Some("requests==5")), 0, 0, &mut HashMap::new()) { @@ -4396,11 +4350,11 @@ mod tests { #[test] fn metric_comparison_with_undeclared_param_is_rejected_naming_real_label_params() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); let pattern = r"level=(?P\w+) duration=(?P\d+)"; declare_test_param(&mut index, "level", pattern, "level", MetricPrimitiveKind::String); declare_test_param(&mut index, "duration", pattern, "duration", MetricPrimitiveKind::Number); - declare_metric(&mut index, metric_request("requests", pattern, vec!["level", "duration"])).unwrap(); + index.declare_metric(metric_request("requests", pattern, vec!["level", "duration"])).unwrap(); ingest_log(&mut index, 0, "level=error duration=5"); match handle_query(&index, logs_query(Some("requests[nonexistent_param]==\"x\"")), 0, 0, &mut HashMap::new()) { @@ -4424,11 +4378,11 @@ mod tests { #[test] fn metric_comparison_with_valid_label_param_still_matches() { - let mut index = BTreeMap::new(); + let mut index = EngineSnapshot::default(); let pattern = r"level=(?P\w+) duration=(?P\d+)"; declare_test_param(&mut index, "level", pattern, "level", MetricPrimitiveKind::String); declare_test_param(&mut index, "duration", pattern, "duration", MetricPrimitiveKind::Number); - declare_metric(&mut index, metric_request("requests", pattern, vec!["level", "duration"])).unwrap(); + index.declare_metric(metric_request("requests", pattern, vec!["level", "duration"])).unwrap(); ingest_log(&mut index, 0, "level=error duration=5"); ingest_log(&mut index, 1, "level=info duration=2"); diff --git a/src/profile.rs b/src/profile.rs index cffe0af..d2b4dab 100644 --- a/src/profile.rs +++ b/src/profile.rs @@ -700,7 +700,7 @@ mod tests { async fn ask(engine: &EngineHandle, query: EngineQuery) -> EngineResponse { let read_handle = engine.reader_factory.handle(); let guard = read_handle.enter().expect("engine unavailable"); - crate::engine::handle_query_ro(&guard, query) + guard.handle_query_ro(query) } #[test]