diff --git a/src/api.rs b/src/api.rs index 454b85d..ac0c825 100644 --- a/src/api.rs +++ b/src/api.rs @@ -266,6 +266,8 @@ async fn declare_metric( name: request.name, message_match_regex: request.message_match_regex, params: request.params, + matched_labels: request.matched_labels, + unmatched_labels: request.unmatched_labels, }; (StatusCode::CREATED, Json(dto)).into_response() } diff --git a/src/data_value.rs b/src/data_value.rs index ca7ffc5..4fa157b 100644 --- a/src/data_value.rs +++ b/src/data_value.rs @@ -1,6 +1,7 @@ use crate::metric_primitive::MetricPrimitive; use crate::timestamp::Timestamp; use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; /// A single parsed metric primitive extracted from a regex capture group. #[allow(dead_code)] @@ -64,6 +65,10 @@ pub enum DataValue { MetricDeclaration { message_match_regex: String, params: Vec, + /// See `crate::engine::DeclareMetricRequest::matched_labels` (ticket 025). + matched_labels: Option>, + /// See `crate::engine::DeclareMetricRequest::unmatched_labels` (ticket 025). + unmatched_labels: Option>, }, /// Paired with `IndexKey::Bucket`. BucketValue { diff --git a/src/dto.rs b/src/dto.rs index 706c4ae..d4d5386 100644 --- a/src/dto.rs +++ b/src/dto.rs @@ -32,6 +32,10 @@ pub struct MetricDeclarationDto { pub name: String, pub message_match_regex: String, pub params: Vec, + /// See `crate::engine::DeclareMetricRequest::matched_labels` (ticket 025). + pub matched_labels: Option>, + /// See `crate::engine::DeclareMetricRequest::unmatched_labels` (ticket 025). + pub unmatched_labels: Option>, } #[derive(Debug, Clone, Serialize)] diff --git a/src/engine.rs b/src/engine.rs index 652ce70..43a35e8 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -46,6 +46,31 @@ pub struct DeclareMetricRequest { pub name: String, pub message_match_regex: String, pub params: Vec, + /// Ticket 025: fixed labels added on top of whatever `params` captured + /// when the line *does* match `message_match_regex`. Together with + /// `unmatched_labels` below, this is what lets a single metric+query + /// produce a "matched X" vs "didn't match X" pair of series (e.g. the + /// seeded default dashboard's error/non-error split) with clean, + /// consistent label values, rather than whatever case/substring the + /// regex happened to capture (a case-insensitive `error` match could + /// capture `"ERROR"`, `"Error"`, etc., fragmenting what should be one + /// series into several). `#[serde(default)]` so existing declarations + /// without this field are unaffected. + #[serde(default)] + pub matched_labels: Option>, + /// When a line does *not* match `message_match_regex`, and this is + /// `Some`, the metric still registers one occurrence — labeled with + /// these fixed values rather than anything captured (there's nothing to + /// capture on a non-match) — instead of the line being invisible to this + /// metric. This is the other half of the "matched X vs didn't match X" + /// pair described above: plain regex alternation can't express the + /// "didn't match" side without lookaround (unsupported by the `regex` + /// crate), since an alternation branch that's a catch-all always wins at + /// the search's leftmost position, so it can never tell "contains X + /// later in the line" apart from "X is absent entirely". `#[serde(default)]` + /// so existing declarations without this field keep matching only. + #[serde(default)] + pub unmatched_labels: Option>, } #[allow(dead_code)] @@ -238,12 +263,21 @@ fn declare_metric( )); } - backfill_metric(index, &request.name, &re, &request.params); + backfill_metric( + index, + &request.name, + &re, + &request.params, + request.matched_labels.as_ref(), + request.unmatched_labels.as_ref(), + ); index.insert( IndexKey::MetricDeclaration { name: request.name }, DataValue::MetricDeclaration { message_match_regex: request.message_match_regex, params: request.params, + matched_labels: request.matched_labels, + unmatched_labels: request.unmatched_labels, }, ); Ok(()) @@ -304,16 +338,33 @@ fn parse_metric_primitive(raw: &str, kind: MetricPrimitiveKind) -> Option, metric_name: &str, regex: &Regex, params: &[RegexToMetricPrimitiveMatcher], + matched_labels: Option<&BTreeMap>, + unmatched_labels: Option<&BTreeMap>, sequence: u64, timestamp: Timestamp, text: &str, ) { let Some(captures) = regex.captures(text) else { + if let Some(unmatched_labels) = unmatched_labels { + let labels = unmatched_labels + .iter() + .map(|(name, value)| (name.clone(), MetricPrimitive::String(value.clone()))) + .collect(); + push_bucket_value(index, metric_name, labels, sequence, timestamp, MetricPrimitive::Number(Number::Int(1))); + } return; }; @@ -340,6 +391,11 @@ fn apply_metric_to_log( } } } + if let Some(matched_labels) = matched_labels { + for (name, value) in matched_labels { + labels.insert(name.clone(), MetricPrimitive::String(value.clone())); + } + } let value = if is_duration { // Bookkeeping key is separate from the `Bucket` key below: a @@ -365,8 +421,18 @@ fn apply_metric_to_log( value.unwrap_or(MetricPrimitive::Number(Number::Int(1))) }; - let key = IndexKey::Bucket { metric_name: metric_name.to_string(), labels }; + push_bucket_value(index, metric_name, labels, sequence, timestamp, value); +} +fn push_bucket_value( + index: &mut BTreeMap, + metric_name: &str, + labels: BTreeMap, + sequence: u64, + timestamp: Timestamp, + value: MetricPrimitive, +) { + let key = IndexKey::Bucket { metric_name: metric_name.to_string(), labels }; let entry = index .entry(key) .or_insert_with(|| DataValue::BucketValue { values: Vec::new() }); @@ -383,12 +449,24 @@ fn backfill_metric( metric_name: &str, regex: &Regex, params: &[RegexToMetricPrimitiveMatcher], + matched_labels: Option<&BTreeMap>, + unmatched_labels: Option<&BTreeMap>, ) { let lines: Vec<(u64, Timestamp, String)> = logs_range(index) .map(|(sequence, timestamp, text)| (sequence, timestamp, text.to_string())) .collect(); for (sequence, timestamp, text) in lines { - apply_metric_to_log(index, metric_name, regex, params, sequence, timestamp, &text); + apply_metric_to_log( + index, + metric_name, + regex, + params, + matched_labels, + unmatched_labels, + sequence, + timestamp, + &text, + ); } } @@ -401,15 +479,37 @@ fn apply_all_metrics( timestamp: Timestamp, text: &str, ) { - let metrics: Vec<(String, String, Vec)> = - metric_declarations_range(index) - .map(|(name, message_match_regex, params)| { - (name.to_string(), message_match_regex.to_string(), params.to_vec()) - }) - .collect(); - for (name, message_match_regex, params) in metrics { + type SeededMetric = ( + String, + String, + Vec, + Option>, + Option>, + ); + let metrics: Vec = metric_declarations_range(index) + .map(|(name, message_match_regex, params, matched_labels, unmatched_labels)| { + ( + name.to_string(), + message_match_regex.to_string(), + params.to_vec(), + matched_labels.cloned(), + unmatched_labels.cloned(), + ) + }) + .collect(); + for (name, message_match_regex, params, matched_labels, unmatched_labels) in metrics { if let Ok(re) = Regex::new(&message_match_regex) { - apply_metric_to_log(index, &name, &re, ¶ms, sequence, timestamp, text); + apply_metric_to_log( + index, + &name, + &re, + ¶ms, + matched_labels.as_ref(), + unmatched_labels.as_ref(), + sequence, + timestamp, + text, + ); } } } @@ -675,14 +775,28 @@ fn eval_compiled(compiled: &CompiledFilter, sequence: u64, timestamp: Timestamp, fn metric_declarations_range( index: &BTreeMap, -) -> impl Iterator { +) -> impl Iterator< + Item = ( + &str, + &str, + &[RegexToMetricPrimitiveMatcher], + Option<&BTreeMap>, + Option<&BTreeMap>, + ), +> { index .range(IndexKey::ParamMetricSplit..IndexKey::MetricBucketSplit) .filter_map(|(key, value)| match (key, value) { ( IndexKey::MetricDeclaration { name }, - DataValue::MetricDeclaration { message_match_regex, params }, - ) => Some((name.as_str(), message_match_regex.as_str(), params.as_slice())), + DataValue::MetricDeclaration { message_match_regex, params, matched_labels, unmatched_labels }, + ) => Some(( + name.as_str(), + message_match_regex.as_str(), + params.as_slice(), + matched_labels.as_ref(), + unmatched_labels.as_ref(), + )), _ => None, }) } @@ -885,10 +999,12 @@ fn handle_query(index: &BTreeMap, query: EngineQuery) -> En } EngineQuery::MetricDeclarations => { let declarations = metric_declarations_range(index) - .map(|(name, message_match_regex, params)| MetricDeclarationDto { + .map(|(name, message_match_regex, params, matched_labels, unmatched_labels)| MetricDeclarationDto { name: name.to_string(), message_match_regex: message_match_regex.to_string(), params: params.to_vec(), + matched_labels: matched_labels.cloned(), + unmatched_labels: unmatched_labels.cloned(), }) .collect(); EngineResponse::MetricDeclarations(declarations) @@ -1200,6 +1316,8 @@ mod tests { DataValue::MetricDeclaration { message_match_regex: "x".to_string(), params: vec![], + matched_labels: None, + unmatched_labels: None, }, ); @@ -1243,6 +1361,8 @@ mod tests { DeclareMetricRequest { name: name.to_string(), message_match_regex: pattern.to_string(), + matched_labels: None, + unmatched_labels: None, params: params .into_iter() .map(|(param_name, capture_group, kind, role)| RegexToMetricPrimitiveMatcher { @@ -1775,6 +1895,70 @@ mod tests { assert_eq!(series[0].points, vec![(10, 10.0)]); } + // Fixed matched_labels/unmatched_labels (rather than a captured "category" + // param) so a case-insensitive match like "ERROR" vs "Error" still + // collapses into one clean "error" series instead of fragmenting by + // whatever case the source text happened to use. + fn declare_error_category_metric(index: &mut BTreeMap) { + declare_metric( + index, + DeclareMetricRequest { + name: "log_category".to_string(), + message_match_regex: r"(?i)error".to_string(), + params: vec![], + matched_labels: Some(BTreeMap::from([("category".to_string(), "error".to_string())])), + unmatched_labels: Some(BTreeMap::from([("category".to_string(), "other".to_string())])), + }, + ) + .unwrap(); + } + + #[test] + fn unmatched_labels_registers_occurrence_for_non_matching_lines() { + let mut index = BTreeMap::new(); + declare_error_category_metric(&mut index); + + ingest_log(&mut index, 0, "everything is fine"); + ingest_log(&mut index, 1, "an ERROR occurred"); + ingest_log(&mut index, 2, "still fine"); + + let error_labels = BTreeMap::from([("category".to_string(), MetricPrimitive::String("error".to_string()))]); + let error_values: Vec<_> = bucket_values(&index, "log_category", Some(&error_labels)).collect(); + assert_eq!(error_values.len(), 1); + assert_eq!(error_values[0].1.0, 1); + + let other_labels = BTreeMap::from([("category".to_string(), MetricPrimitive::String("other".to_string()))]); + let other_values: Vec<_> = bucket_values(&index, "log_category", Some(&other_labels)).collect(); + assert_eq!(other_values.len(), 2); + } + + #[test] + fn unmatched_labels_produces_two_series_from_one_bare_promql_query() { + let mut index = BTreeMap::new(); + declare_error_category_metric(&mut index); + ingest_log(&mut index, 0, "an ERROR occurred"); + ingest_log(&mut index, 1, "all good"); + ingest_log(&mut index, 2, "all good again"); + + let parsed = promql::parse_query("log_category").unwrap(); + let mut series = evaluate_promql(&index, &parsed, None, None); + series.sort_by(|a, b| a.labels.get("category").cmp(&b.labels.get("category"))); + + assert_eq!(series.len(), 2); + assert_eq!(series[0].labels.get("category").map(String::as_str), Some("error")); + assert_eq!(series[0].points.len(), 1); + assert_eq!(series[1].labels.get("category").map(String::as_str), Some("other")); + assert_eq!(series[1].points.len(), 2); + } + + #[test] + fn no_unmatched_labels_means_non_matching_lines_are_untouched() { + let mut index = BTreeMap::new(); + declare_requests_metric(&mut index); + ingest_log(&mut index, 0, "not a request line at all"); + assert_eq!(bucket_values(&index, "requests", None).count(), 0); + } + #[test] fn handle_query_promql_rejects_malformed_query() { let index = BTreeMap::new(); diff --git a/src/profile.rs b/src/profile.rs index e89bb4c..5084ae6 100644 --- a/src/profile.rs +++ b/src/profile.rs @@ -31,6 +31,7 @@ #![allow(dead_code)] use std::collections::hash_map::DefaultHasher; +use std::collections::BTreeMap; use std::fmt; use std::fs; use std::hash::{Hash, Hasher}; @@ -135,6 +136,16 @@ pub struct MetricDeclarationRecord { pub name: String, pub message_match_regex: String, pub params: Vec, + /// See `crate::engine::DeclareMetricRequest::matched_labels` (ticket + /// 025). `#[serde(default)]` so profiles saved before that ticket still + /// load. + #[serde(default)] + pub matched_labels: Option>, + /// See `crate::engine::DeclareMetricRequest::unmatched_labels` (ticket + /// 025). `#[serde(default)]` so profiles saved before that ticket still + /// load. + #[serde(default)] + pub unmatched_labels: Option>, } impl From<&DeclareMetricRequest> for MetricDeclarationRecord { @@ -152,6 +163,8 @@ impl From<&DeclareMetricRequest> for MetricDeclarationRecord { role: matcher.role, }) .collect(), + matched_labels: request.matched_labels.clone(), + unmatched_labels: request.unmatched_labels.clone(), } } } @@ -161,6 +174,8 @@ impl From<&MetricDeclarationRecord> for DeclareMetricRequest { DeclareMetricRequest { name: record.name.clone(), message_match_regex: record.message_match_regex.clone(), + matched_labels: record.matched_labels.clone(), + unmatched_labels: record.unmatched_labels.clone(), params: record .params .iter() @@ -227,6 +242,54 @@ impl Dashboard { } } +/// Ticket 025: name of the metric seeded into a brand-new `Default` profile, +/// used to drive its starter dashboard's two panels. +const SEEDED_METRIC_NAME: &str = "log_category"; +const SEEDED_METRIC_LABEL: &str = "category"; + +/// A counter (ticket 022) that classifies every ingested line into exactly +/// two buckets — `category="error"` or `category="other"` — without +/// capturing any text: `matched_labels`/`unmatched_labels` (ticket 025) apply +/// those fixed values directly, rather than a param capturing the literal +/// matched substring, so a case-insensitive match like `"ERROR"` vs `"Error"` +/// collapses into one clean series instead of fragmenting by source casing. +/// A plain regex alternation can't express the "didn't match" half of this +/// without lookaround, which the `regex` crate doesn't support (see +/// `crate::engine::DeclareMetricRequest::unmatched_labels`) — hence going +/// through the metric-declaration mechanism instead of the PromQL surface. +fn seeded_error_split_metric() -> MetricDeclarationRecord { + MetricDeclarationRecord { + name: SEEDED_METRIC_NAME.to_string(), + message_match_regex: r"(?i)error".to_string(), + params: Vec::new(), + matched_labels: Some(BTreeMap::from([(SEEDED_METRIC_LABEL.to_string(), "error".to_string())])), + unmatched_labels: Some(BTreeMap::from([(SEEDED_METRIC_LABEL.to_string(), "other".to_string())])), + } +} + +/// The starter dashboard seeded into a brand-new `Default` profile (ticket +/// 025): a line chart and a pie chart, both querying the bare +/// `SEEDED_METRIC_NAME` counter so each renders both the "error" and "other" +/// series/slices from a single PromQL query, same as any user-created panel. +fn seeded_default_dashboard() -> Dashboard { + let mut dashboard = Dashboard::new("Overview".to_string()); + dashboard.panels = vec![ + Panel { + id: generate_id("error-volume-line"), + title: "Error vs. non-error volume".to_string(), + promql: SEEDED_METRIC_NAME.to_string(), + chart_type: ChartType::Line, + }, + Panel { + id: generate_id("error-volume-pie"), + title: "Error vs. non-error ratio".to_string(), + promql: SEEDED_METRIC_NAME.to_string(), + chart_type: ChartType::Pie, + }, + ]; + dashboard +} + /// A named bundle of parsing/indexing rules for a particular CLI app: which /// params and metrics to derive from its log output, plus any saved /// dashboards. @@ -494,12 +557,18 @@ impl ProfileStore { Ok(()) } - /// Guarantees a `Default` profile exists in this store (creating an empty - /// one if not), that it's tracked in `settings.profiles.names`, and that - /// `settings.profiles.active_profile` falls back to it when nothing else - /// is active — so the app always has at least one usable profile, even - /// on a first run with no settings file yet. Persists both the store and - /// `settings` if either changed. + /// Guarantees a `Default` profile exists in this store, that it's tracked + /// in `settings.profiles.names`, and that `settings.profiles.active_profile` + /// falls back to it when nothing else is active — so the app always has + /// at least one usable profile, even on a first run with no settings file + /// yet. Persists both the store and `settings` if either changed. + /// + /// A newly-created `Default` profile (not one that already existed) is + /// seeded with a starter dashboard (ticket 025): a line chart and a pie + /// chart of "error" vs. non-"error" log volume, both querying + /// `seeded_error_split_metric`'s counter — editable/deletable afterward + /// like any other dashboard/metric, and never re-seeded or duplicated on + /// a later call once the profile exists. /// /// Called once on startup (see `main.rs`) before anything else touches /// profiles. @@ -510,7 +579,10 @@ impl ProfileStore { settings_path: &Path, ) -> Result<(), ProfileError> { if self.get(DEFAULT_PROFILE_NAME).is_none() { - self.profiles.push(Profile::new(DEFAULT_PROFILE_NAME.to_string())); + let mut profile = Profile::new(DEFAULT_PROFILE_NAME.to_string()); + profile.metrics.push(seeded_error_split_metric()); + profile.dashboards.push(seeded_default_dashboard()); + self.profiles.push(profile); self.save(store_path)?; } if !settings.profiles.names.iter().any(|n| n == DEFAULT_PROFILE_NAME) { @@ -714,6 +786,8 @@ mod tests { DeclareMetricRequest { name: name.to_string(), message_match_regex: pattern.to_string(), + matched_labels: None, + unmatched_labels: None, params: vec![RegexToMetricPrimitiveMatcher { param_name: param_name.to_string(), capture_group: capture_group.to_string(), @@ -806,6 +880,89 @@ mod tests { assert_eq!(settings.profiles.active_profile, Some("web".to_string())); } + #[test] + fn ensure_default_seeds_starter_dashboard_only_on_creation() { + let dir = tempfile::tempdir().unwrap(); + let store_path = dir.path().join("profiles.json"); + let settings_path = dir.path().join("settings.json"); + let mut settings = Settings::default(); + + let mut store = ProfileStore::load(&store_path); + store.ensure_default(&store_path, &mut settings, &settings_path).unwrap(); + + let default_profile = store.get(DEFAULT_PROFILE_NAME).unwrap(); + assert_eq!(default_profile.metrics.len(), 1); + assert_eq!(default_profile.metrics[0].name, SEEDED_METRIC_NAME); + assert_eq!( + default_profile.metrics[0].matched_labels, + Some(BTreeMap::from([(SEEDED_METRIC_LABEL.to_string(), "error".to_string())])) + ); + assert_eq!( + default_profile.metrics[0].unmatched_labels, + Some(BTreeMap::from([(SEEDED_METRIC_LABEL.to_string(), "other".to_string())])) + ); + assert_eq!(default_profile.dashboards.len(), 1); + let dashboard = &default_profile.dashboards[0]; + assert_eq!(dashboard.panels.len(), 2); + assert!(dashboard.panels.iter().all(|p| p.promql == SEEDED_METRIC_NAME)); + assert!(dashboard.panels.iter().any(|p| p.chart_type == ChartType::Line)); + assert!(dashboard.panels.iter().any(|p| p.chart_type == ChartType::Pie)); + + // Reload (simulating a restart) and ensure_default again: must not + // re-seed or duplicate the dashboard/metric. + let mut store = ProfileStore::load(&store_path); + store.ensure_default(&store_path, &mut settings, &settings_path).unwrap(); + let default_profile = store.get(DEFAULT_PROFILE_NAME).unwrap(); + assert_eq!(default_profile.metrics.len(), 1); + assert_eq!(default_profile.dashboards.len(), 1); + } + + #[tokio::test] + async fn seeded_metric_renders_real_error_vs_other_data() { + use crate::engine::{self, EngineMessage, EngineQuery, EngineResponse, IngestedLine}; + use crate::timestamp::Timestamp; + use tokio::sync::mpsc; + + let dir = tempfile::tempdir().unwrap(); + let store_path = dir.path().join("profiles.json"); + let settings_path = dir.path().join("settings.json"); + let mut settings = Settings::default(); + let mut store = ProfileStore::load(&store_path); + store.ensure_default(&store_path, &mut settings, &settings_path).unwrap(); + let default_profile = store.get(DEFAULT_PROFILE_NAME).unwrap().clone(); + + let (engine_tx, engine_rx) = mpsc::channel(128); + tokio::spawn(engine::run(engine_rx)); + load_into_engine(&engine_tx, &default_profile).await; + + for (sequence, text) in [(0u64, "all good"), (1, "an ERROR happened"), (2, "fine again")] { + engine_tx + .send(EngineMessage::Ingest(IngestedLine { + sequence, + timestamp: Timestamp::from_millis(sequence as i64), + text: text.to_string(), + })) + .await + .unwrap(); + } + + let response = ask( + &engine_tx, + EngineQuery::Promql { query: SEEDED_METRIC_NAME.to_string(), start_ms: None, end_ms: None }, + ) + .await; + let EngineResponse::Promql(Ok(mut series)) = response else { + panic!("expected a successful Promql response"); + }; + series.sort_by(|a, b| a.labels.get(SEEDED_METRIC_LABEL).cmp(&b.labels.get(SEEDED_METRIC_LABEL))); + + assert_eq!(series.len(), 2); + assert_eq!(series[0].labels.get(SEEDED_METRIC_LABEL).map(String::as_str), Some("error")); + assert_eq!(series[0].points.len(), 1); + assert_eq!(series[1].labels.get(SEEDED_METRIC_LABEL).map(String::as_str), Some("other")); + assert_eq!(series[1].points.len(), 2); + } + #[test] fn default_profile_cannot_be_deleted() { let dir = tempfile::tempdir().unwrap(); diff --git a/webapp/src/lib/api.ts b/webapp/src/lib/api.ts index 3ae7997..a627fb4 100644 --- a/webapp/src/lib/api.ts +++ b/webapp/src/lib/api.ts @@ -61,6 +61,9 @@ export interface MetricDeclaration { name: string message_match_regex: string params: MetricParamMatcher[] + /** Fixed labels applied on a match/non-match, on top of any captured params (ticket 025). */ + matched_labels?: Record | null + unmatched_labels?: Record | null } export interface BucketValue {