diff --git a/Makefile b/Makefile index 91f3f0c..fca08df 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,8 @@ test-fe: lint: lint-be lint-fe lint-be: - cargo clippy --all-targets + cargo fmt -- --check + cargo clippy --all-targets -- -D warnings lint-fe: cd webapp && npm run lint diff --git a/src/api/dto.rs b/src/api/dto.rs index 9831293..f318324 100644 --- a/src/api/dto.rs +++ b/src/api/dto.rs @@ -1,8 +1,8 @@ +use crate::config::Settings; +use crate::config::profile::{Panel, Profile}; use crate::engine::alerts::AlertCondition; use crate::engine::data_value::MetricPrimitiveKind; use crate::engine::metric_primitive::MetricPrimitive; -use crate::config::Settings; -use crate::config::profile::{Panel, Profile}; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; diff --git a/src/api/mcp_server.rs b/src/api/mcp_server.rs index eac1eae..f66552e 100644 --- a/src/api/mcp_server.rs +++ b/src/api/mcp_server.rs @@ -381,7 +381,9 @@ impl OttylMcp { // ---- logs / timeline / stats ---- - #[tool(description = "Count or read log lines matching a FilterBar query against the running ottyl instance.")] + #[tool( + description = "Count or read log lines matching a FilterBar query against the running ottyl instance." + )] async fn query_logs( &self, Parameters(params): Parameters, @@ -414,7 +416,9 @@ impl OttylMcp { } } - #[tool(description = "Read the log-count timeline (bucketed histogram) matching an optional FilterBar query — the same data backing the Log Timeline chart.")] + #[tool( + description = "Read the log-count timeline (bucketed histogram) matching an optional FilterBar query — the same data backing the Log Timeline chart." + )] async fn get_timeline( &self, Parameters(params): Parameters, @@ -435,14 +439,18 @@ impl OttylMcp { json_result(self.get_json("/api/timeline", &query).await?) } - #[tool(description = "Read overall ingest stats: log count, sequence/timestamp bounds, ingest lag.")] + #[tool( + description = "Read overall ingest stats: log count, sequence/timestamp bounds, ingest lag." + )] async fn get_stats(&self) -> Result { json_result(self.get_json("/api/stats", &[]).await?) } // ---- params ---- - #[tool(description = "List all currently declared params (name, regex, kind, etc.), including auto-derived ones.")] + #[tool( + description = "List all currently declared params (name, regex, kind, etc.), including auto-derived ones." + )] async fn list_params(&self) -> Result { json_result(self.get_json("/api/params", &[]).await?) } @@ -452,13 +460,22 @@ impl OttylMcp { &self, Parameters(params): Parameters, ) -> Result { - let body = serde_json::to_value(¶ms).map_err(|err| mcp_error("failed to encode params", err))?; - json_result(self.send_json(reqwest::Method::POST, "/api/params", Some(&body)).await?) + let body = serde_json::to_value(¶ms) + .map_err(|err| mcp_error("failed to encode params", err))?; + json_result( + self.send_json(reqwest::Method::POST, "/api/params", Some(&body)) + .await?, + ) } - #[tool(description = "Clear all declared params and re-derive auto params from retained log lines.")] + #[tool( + description = "Clear all declared params and re-derive auto params from retained log lines." + )] async fn clear_params(&self) -> Result { - json_result(self.send_json(reqwest::Method::DELETE, "/api/params", None).await?) + json_result( + self.send_json(reqwest::Method::DELETE, "/api/params", None) + .await?, + ) } // ---- metrics ---- @@ -468,13 +485,19 @@ impl OttylMcp { json_result(self.get_json("/api/metrics", &[]).await?) } - #[tool(description = "Declare a metric: named event counts/values bucketed by label, derived from matching log lines.")] + #[tool( + description = "Declare a metric: named event counts/values bucketed by label, derived from matching log lines." + )] async fn declare_metric( &self, Parameters(params): Parameters, ) -> Result { - let body = serde_json::to_value(¶ms).map_err(|err| mcp_error("failed to encode params", err))?; - json_result(self.send_json(reqwest::Method::POST, "/api/metrics", Some(&body)).await?) + let body = serde_json::to_value(¶ms) + .map_err(|err| mcp_error("failed to encode params", err))?; + json_result( + self.send_json(reqwest::Method::POST, "/api/metrics", Some(&body)) + .await?, + ) } #[tool(description = "Update an existing metric declaration in place by name.")] @@ -509,7 +532,9 @@ impl OttylMcp { ) } - #[tool(description = "Read a metric's raw bucket values (labels, sequence, timestamp, value), optionally filtered by label and time range.")] + #[tool( + description = "Read a metric's raw bucket values (labels, sequence, timestamp, value), optionally filtered by label and time range." + )] async fn get_metric_bucket_values( &self, Parameters(params): Parameters, @@ -531,19 +556,27 @@ impl OttylMcp { query.push(("limit", limit.to_string())); } json_result( - self.get_json(&format!("/api/metrics/{}", urlencoding_path(¶ms.name)), &query) - .await?, + self.get_json( + &format!("/api/metrics/{}", urlencoding_path(¶ms.name)), + &query, + ) + .await?, ) } // ---- promql ---- - #[tool(description = "Evaluate a PromQL-style query directly and get back raw series data (labels + timestamped points) — the same evaluation dashboards use to render charts.")] + #[tool( + description = "Evaluate a PromQL-style query directly and get back raw series data (labels + timestamped points) — the same evaluation dashboards use to render charts." + )] async fn run_promql_query( &self, Parameters(params): Parameters, ) -> Result { - json_result(self.run_promql(¶ms.promql, params.start_ms, params.end_ms).await?) + json_result( + self.run_promql(¶ms.promql, params.start_ms, params.end_ms) + .await?, + ) } // ---- dashboards / panels ---- @@ -556,7 +589,10 @@ impl OttylMcp { let profile = self.resolve_profile(params.profile).await?; json_result( self.get_json( - &format!("/api/settings/profiles/{}/dashboards", urlencoding_path(&profile)), + &format!( + "/api/settings/profiles/{}/dashboards", + urlencoding_path(&profile) + ), &[], ) .await?, @@ -572,14 +608,19 @@ impl OttylMcp { json_result( self.send_json( reqwest::Method::POST, - &format!("/api/settings/profiles/{}/dashboards", urlencoding_path(&profile)), + &format!( + "/api/settings/profiles/{}/dashboards", + urlencoding_path(&profile) + ), Some(&serde_json::json!({ "name": params.name })), ) .await?, ) } - #[tool(description = "Read a dashboard's current panels (definitions only — use render_panel or render_dashboard for the data a panel would chart).")] + #[tool( + description = "Read a dashboard's current panels (definitions only — use render_panel or render_dashboard for the data a panel would chart)." + )] async fn get_dashboard( &self, Parameters(params): Parameters, @@ -631,12 +672,18 @@ impl OttylMcp { .get_mut("panels") .and_then(Value::as_array_mut) .ok_or_else(|| mcp_error("unexpected dashboard shape", "missing panels array"))?; - match panels.iter().position(|panel| panel.get("id") == new_panel.get("id")) { + match panels + .iter() + .position(|panel| panel.get("id") == new_panel.get("id")) + { Some(index) => panels[index] = new_panel, None => panels.push(new_panel), } - json_result(self.save_dashboard(&profile, ¶ms.dashboard_id, dashboard).await?) + json_result( + self.save_dashboard(&profile, ¶ms.dashboard_id, dashboard) + .await?, + ) } #[tool(description = "Remove a panel from a dashboard by panel_id.")] @@ -650,8 +697,13 @@ impl OttylMcp { .get_mut("panels") .and_then(Value::as_array_mut) .ok_or_else(|| mcp_error("unexpected dashboard shape", "missing panels array"))?; - panels.retain(|panel| panel.get("id").and_then(Value::as_str) != Some(params.panel_id.as_str())); - json_result(self.save_dashboard(&profile, ¶ms.dashboard_id, dashboard).await?) + panels.retain(|panel| { + panel.get("id").and_then(Value::as_str) != Some(params.panel_id.as_str()) + }); + json_result( + self.save_dashboard(&profile, ¶ms.dashboard_id, dashboard) + .await?, + ) } async fn save_dashboard( @@ -672,7 +724,9 @@ impl OttylMcp { .await } - #[tool(description = "Render one panel: resolve its promql from the dashboard definition, evaluate it, and return the chart's underlying series data (labels + timestamped points) as JSON — this is how an agent should 'see' a chart without a browser.")] + #[tool( + description = "Render one panel: resolve its promql from the dashboard definition, evaluate it, and return the chart's underlying series data (labels + timestamped points) as JSON — this is how an agent should 'see' a chart without a browser." + )] async fn render_panel( &self, Parameters(params): Parameters, @@ -682,20 +736,30 @@ impl OttylMcp { let panel = dashboard .get("panels") .and_then(Value::as_array) - .and_then(|panels| panels.iter().find(|panel| panel.get("id").and_then(Value::as_str) == Some(params.panel_id.as_str()))) - .ok_or_else(|| McpError::invalid_params("no panel with that panel_id in the dashboard", None))?; + .and_then(|panels| { + panels.iter().find(|panel| { + panel.get("id").and_then(Value::as_str) == Some(params.panel_id.as_str()) + }) + }) + .ok_or_else(|| { + McpError::invalid_params("no panel with that panel_id in the dashboard", None) + })?; let promql = panel .get("promql") .and_then(Value::as_str) .ok_or_else(|| mcp_error("unexpected panel shape", "missing promql"))?; - let series = self.run_promql(promql, params.start_ms, params.end_ms).await?; + let series = self + .run_promql(promql, params.start_ms, params.end_ms) + .await?; json_result(serde_json::json!({ "panel": panel, "series": series, })) } - #[tool(description = "Render every panel in a dashboard: evaluate each panel's promql and return all their chart data as JSON, so an agent can reason about the whole dashboard without a browser.")] + #[tool( + description = "Render every panel in a dashboard: evaluate each panel's promql and return all their chart data as JSON, so an agent can reason about the whole dashboard without a browser." + )] async fn render_dashboard( &self, Parameters(params): Parameters, @@ -709,8 +773,13 @@ impl OttylMcp { .unwrap_or_default(); let mut rendered = Vec::with_capacity(panels.len()); for panel in panels { - let promql = panel.get("promql").and_then(Value::as_str).unwrap_or_default(); - let series = self.run_promql(promql, params.start_ms, params.end_ms).await?; + let promql = panel + .get("promql") + .and_then(Value::as_str) + .unwrap_or_default(); + let series = self + .run_promql(promql, params.start_ms, params.end_ms) + .await?; rendered.push(serde_json::json!({ "panel": panel, "series": series })); } json_result(serde_json::json!({ @@ -721,18 +790,26 @@ impl OttylMcp { // ---- settings / theming ---- - #[tool(description = "Read the running instance's global settings (theme, active profile, max_logs, ntfy config).")] + #[tool( + description = "Read the running instance's global settings (theme, active profile, max_logs, ntfy config)." + )] async fn get_settings(&self) -> Result { json_result(self.get_json("/api/settings", &[]).await?) } - #[tool(description = "Update global settings. Only fields provided are changed; pass null for an Option field to clear it.")] + #[tool( + description = "Update global settings. Only fields provided are changed; pass null for an Option field to clear it." + )] async fn update_settings( &self, Parameters(params): Parameters, ) -> Result { - let body = serde_json::to_value(¶ms).map_err(|err| mcp_error("failed to encode params", err))?; - json_result(self.send_json(reqwest::Method::PATCH, "/api/settings", Some(&body)).await?) + let body = serde_json::to_value(¶ms) + .map_err(|err| mcp_error("failed to encode params", err))?; + json_result( + self.send_json(reqwest::Method::PATCH, "/api/settings", Some(&body)) + .await?, + ) } #[tool(description = "List the names of the themes embedded in this build (see /api/themes).")] @@ -779,7 +856,10 @@ impl OttylMcp { "enabled": params.enabled, "condition": params.condition, }); - json_result(self.send_json(reqwest::Method::POST, "/api/settings/alerts", Some(&body)).await?) + json_result( + self.send_json(reqwest::Method::POST, "/api/settings/alerts", Some(&body)) + .await?, + ) } #[tool(description = "Update an existing alert rule by id.")] @@ -817,21 +897,30 @@ impl OttylMcp { ) } - #[tool(description = "Send a test ntfy notification to verify alert delivery is configured correctly.")] + #[tool( + description = "Send a test ntfy notification to verify alert delivery is configured correctly." + )] async fn test_alert_notification( &self, Parameters(params): Parameters, ) -> Result { - let body = serde_json::to_value(¶ms).map_err(|err| mcp_error("failed to encode params", err))?; + let body = serde_json::to_value(¶ms) + .map_err(|err| mcp_error("failed to encode params", err))?; json_result( - self.send_json(reqwest::Method::POST, "/api/settings/alerts/test", Some(&body)) - .await?, + self.send_json( + reqwest::Method::POST, + "/api/settings/alerts/test", + Some(&body), + ) + .await?, ) } // ---- profiles ---- - #[tool(description = "List all saved profiles with summary counts (params, metrics, dashboards).")] + #[tool( + description = "List all saved profiles with summary counts (params, metrics, dashboards)." + )] async fn list_profiles(&self) -> Result { json_result(self.get_json("/api/settings/profiles", &[]).await?) } @@ -841,8 +930,12 @@ impl OttylMcp { &self, Parameters(params): Parameters, ) -> Result { - let body = serde_json::to_value(¶ms).map_err(|err| mcp_error("failed to encode params", err))?; - json_result(self.send_json(reqwest::Method::POST, "/api/settings/profiles", Some(&body)).await?) + let body = serde_json::to_value(¶ms) + .map_err(|err| mcp_error("failed to encode params", err))?; + json_result( + self.send_json(reqwest::Method::POST, "/api/settings/profiles", Some(&body)) + .await?, + ) } #[tool(description = "Delete a profile by name.")] @@ -860,7 +953,9 @@ impl OttylMcp { ) } - #[tool(description = "Activate a profile: swaps in its param/metric declarations and dashboards as the running instance's live state.")] + #[tool( + description = "Activate a profile: swaps in its param/metric declarations and dashboards as the running instance's live state." + )] async fn activate_profile( &self, Parameters(params): Parameters, @@ -868,7 +963,10 @@ impl OttylMcp { json_result( self.send_json( reqwest::Method::POST, - &format!("/api/settings/profiles/{}/activate", urlencoding_path(¶ms.name)), + &format!( + "/api/settings/profiles/{}/activate", + urlencoding_path(¶ms.name) + ), None, ) .await?, diff --git a/src/api/mod.rs b/src/api/mod.rs index 8c331ef..d203dd1 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -7,23 +7,23 @@ use std::sync::Arc; use crate::api::dto::{ BucketValuesParams, CreateAlertRuleRequest, CreateDashboardRequest, CreateProfileRequest, - ErrorDto, MetricDeclarationDto, - ParamDeclarationDto, LogsParams, ProfileSummaryDto, PromqlParams, SaveDashboardRequest, - SettingsDto, TestAlertNotificationRequest, TimelineParams, UpdateAlertRuleRequest, UpdateSettingsRequest, + ErrorDto, LogsParams, MetricDeclarationDto, ParamDeclarationDto, ProfileSummaryDto, + PromqlParams, SaveDashboardRequest, SettingsDto, TestAlertNotificationRequest, TimelineParams, + UpdateAlertRuleRequest, UpdateSettingsRequest, }; +use crate::config::SettingsHandle; +use crate::config::profile::{Dashboard, ParamDeclarationRecord, Profile, ProfileError}; +use crate::engine::alerts::{AlertError, deliver_ntfy}; use crate::engine::data_value::MetricPrimitiveKind; use crate::engine::{ DeclareMetricRequest, DeclareParamRequest, EngineHandle, EngineMessage, EngineQuery, EngineResponse, }; -use crate::engine::alerts::{deliver_ntfy, AlertError}; -use crate::config::profile::{Dashboard, ParamDeclarationRecord, Profile, ProfileError}; -use crate::config::SettingsHandle; +use axum::Router; use axum::extract::{Path as AxumPath, Query, State}; -use axum::http::{header, StatusCode}; +use axum::http::{StatusCode, header}; use axum::response::{IntoResponse, Json, Response}; use axum::routing::{delete, get, post, put}; -use axum::Router; use fs_embed::Dir as EmbedDir; use tokio::sync::oneshot; @@ -49,7 +49,9 @@ impl ApiState { let Some(profile_name) = settings.active_profile.clone() else { return; }; - if let Err(err) = settings.record_param_declaration(&self.settings_path, &profile_name, request) { + if let Err(err) = + settings.record_param_declaration(&self.settings_path, &profile_name, request) + { eprintln!( "warning: failed to persist param declaration '{}' into profile '{profile_name}': {err}", request.name @@ -79,7 +81,10 @@ impl ApiState { }; let existing_params = { let settings = self.settings.lock().unwrap(); - settings.profile(&profile_name).map(|p| p.params.clone()).unwrap_or_default() + settings + .profile(&profile_name) + .map(|p| p.params.clone()) + .unwrap_or_default() }; let promoted = promoted_param_records(&self.engine, &existing_params, request).await; let mut settings = self.settings.lock().unwrap(); @@ -96,7 +101,11 @@ impl ApiState { } } - async fn persist_metric_declaration_update(&self, old_name: &str, request: &DeclareMetricRequest) { + async fn persist_metric_declaration_update( + &self, + old_name: &str, + request: &DeclareMetricRequest, + ) { let profile_name = { let settings = self.settings.lock().unwrap(); let Some(profile_name) = settings.active_profile.clone() else { @@ -106,7 +115,10 @@ impl ApiState { }; let existing_params = { let settings = self.settings.lock().unwrap(); - settings.profile(&profile_name).map(|p| p.params.clone()).unwrap_or_default() + settings + .profile(&profile_name) + .map(|p| p.params.clone()) + .unwrap_or_default() }; let promoted = promoted_param_records(&self.engine, &existing_params, request).await; let mut settings = self.settings.lock().unwrap(); @@ -129,7 +141,9 @@ impl ApiState { let Some(profile_name) = settings.active_profile.clone() else { return; }; - if let Err(err) = settings.remove_metric_declaration(&self.settings_path, &profile_name, name) { + if let Err(err) = + settings.remove_metric_declaration(&self.settings_path, &profile_name, name) + { eprintln!( "warning: failed to remove metric declaration '{name}' from profile '{profile_name}': {err}" ); @@ -149,39 +163,58 @@ pub fn router(engine: EngineHandle, settings: SettingsHandle, settings_path: Pat .route("/api/logs", get(get_logs)) .route("/api/timeline", get(get_timeline)) .route("/api/stats", get(get_stats)) - .route("/api/params", get(list_params).post(declare_param).delete(clear_params)) + .route( + "/api/params", + get(list_params).post(declare_param).delete(clear_params), + ) .route("/api/metrics", get(list_metrics).post(declare_metric)) .route( "/api/metrics/{name}", - get(get_metric_bucket_values).put(update_metric).delete(delete_metric), + get(get_metric_bucket_values) + .put(update_metric) + .delete(delete_metric), ) .route("/api/query", get(run_promql_query)) .route("/api/settings", get(get_settings).patch(update_settings)) .route("/api/themes", get(list_themes)) .route("/api/themes/{name}", get(get_theme_content)) - .route("/api/settings/alerts", get(list_alert_rules).post(create_alert_rule)) + .route( + "/api/settings/alerts", + get(list_alert_rules).post(create_alert_rule), + ) .route( "/api/settings/alerts/{id}", put(update_alert_rule).delete(delete_alert_rule), ) .route("/api/settings/alerts/test", post(test_alert_notification)) - .route("/api/settings/profiles", get(list_profiles).post(create_profile)) + .route( + "/api/settings/profiles", + get(list_profiles).post(create_profile), + ) .route("/api/settings/profiles/{name}", delete(delete_profile)) - .route("/api/settings/profiles/{name}/activate", post(activate_profile)) + .route( + "/api/settings/profiles/{name}/activate", + post(activate_profile), + ) .route( "/api/settings/profiles/{name}/dashboards", get(list_dashboards).post(create_dashboard), ) .route( "/api/settings/profiles/{name}/dashboards/{dashboard_id}", - get(get_dashboard).put(save_dashboard).delete(delete_dashboard), + get(get_dashboard) + .put(save_dashboard) + .delete(delete_dashboard), ) .with_state(state) .fallback(serve_webapp) } async fn get_skill_md() -> impl IntoResponse { - ([(header::CONTENT_TYPE, "text/markdown; charset=utf-8")], SKILL_MD) + ( + [(header::CONTENT_TYPE, "text/markdown; charset=utf-8")], + SKILL_MD, + ) } async fn serve_webapp(uri: axum::http::Uri) -> Response { @@ -225,10 +258,15 @@ impl EngineHandle { if needs_build { let (respond_to, response) = oneshot::channel(); self.tx - .send(EngineMessage::EnsureSortHelperIndex { param_name: sort_by.to_string(), respond_to }) + .send(EngineMessage::EnsureSortHelperIndex { + param_name: sort_by.to_string(), + respond_to, + }) + .await + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + response .await .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; - response.await.map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; } } let guard = read_handle.enter().ok_or(StatusCode::SERVICE_UNAVAILABLE)?; @@ -363,7 +401,10 @@ async fn declare_param( if state .engine .tx - .send(EngineMessage::DeclareParam { request: body, respond_to }) + .send(EngineMessage::DeclareParam { + request: body, + respond_to, + }) .await .is_err() { @@ -391,7 +432,13 @@ async fn declare_param( async fn clear_params(State(state): State) -> impl IntoResponse { let (respond_to, response) = oneshot::channel(); - if state.engine.tx.send(EngineMessage::ClearParams { respond_to }).await.is_err() { + if state + .engine + .tx + .send(EngineMessage::ClearParams { respond_to }) + .await + .is_err() + { return StatusCode::SERVICE_UNAVAILABLE.into_response(); } match response.await { @@ -420,7 +467,10 @@ async fn declare_metric( if state .engine .tx - .send(EngineMessage::DeclareMetric { request: body, respond_to }) + .send(EngineMessage::DeclareMetric { + request: body, + respond_to, + }) .await .is_err() { @@ -454,7 +504,11 @@ async fn update_metric( if state .engine .tx - .send(EngineMessage::UpdateMetric { name: name.clone(), request: body, respond_to }) + .send(EngineMessage::UpdateMetric { + name: name.clone(), + request: body, + respond_to, + }) .await .is_err() { @@ -462,7 +516,9 @@ async fn update_metric( } match response.await { Ok(Ok(())) => { - state.persist_metric_declaration_update(&name, &request).await; + state + .persist_metric_declaration_update(&name, &request) + .await; let dto = MetricDeclarationDto { name: request.name, message_match_regex: request.message_match_regex, @@ -486,7 +542,10 @@ async fn delete_metric( if state .engine .tx - .send(EngineMessage::RemoveMetricDeclaration { name: name.clone(), respond_to }) + .send(EngineMessage::RemoveMetricDeclaration { + name: name.clone(), + respond_to, + }) .await .is_err() { @@ -552,7 +611,10 @@ async fn run_promql_query( async fn list_themes() -> impl IntoResponse { let mut names: Vec = THEMES .walk() - .filter_map(|file| file.file_name().map(|name| name.trim_end_matches(".css").to_string())) + .filter_map(|file| { + file.file_name() + .map(|name| name.trim_end_matches(".css").to_string()) + }) .collect(); names.sort(); Json(names).into_response() @@ -561,7 +623,11 @@ async fn list_themes() -> impl IntoResponse { async fn get_theme_content(AxumPath(name): AxumPath) -> impl IntoResponse { // Themes are just filenames within a flat embedded dir, not filesystem paths, so reject // anything that isn't a plain name to rule out path traversal. - if name.is_empty() || !name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') { + if name.is_empty() + || !name + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') + { return StatusCode::NOT_FOUND.into_response(); } match THEMES.get_file(&format!("{name}.css")) { @@ -577,7 +643,9 @@ impl AlertError { fn into_response(self) -> axum::response::Response { match self { AlertError::NotFound => (StatusCode::NOT_FOUND, self.to_string()).into_response(), - AlertError::Io(_) => (StatusCode::INTERNAL_SERVER_ERROR, self.to_string()).into_response(), + AlertError::Io(_) => { + (StatusCode::INTERNAL_SERVER_ERROR, self.to_string()).into_response() + } } } } @@ -592,7 +660,12 @@ async fn create_alert_rule( Json(body): Json, ) -> impl IntoResponse { let mut settings = state.settings.lock().unwrap(); - match settings.create_alert_rule(&state.settings_path, body.name, body.enabled, body.condition) { + match settings.create_alert_rule( + &state.settings_path, + body.name, + body.enabled, + body.condition, + ) { Ok(rule) => (StatusCode::CREATED, Json(rule)).into_response(), Err(err) => err.into_response(), } @@ -604,13 +677,22 @@ async fn update_alert_rule( Json(body): Json, ) -> impl IntoResponse { let mut settings = state.settings.lock().unwrap(); - match settings.update_alert_rule(&state.settings_path, &id, body.name, body.enabled, body.condition) { + match settings.update_alert_rule( + &state.settings_path, + &id, + body.name, + body.enabled, + body.condition, + ) { Ok(rule) => Json(rule).into_response(), Err(err) => err.into_response(), } } -async fn delete_alert_rule(State(state): State, AxumPath(id): AxumPath) -> impl IntoResponse { +async fn delete_alert_rule( + State(state): State, + AxumPath(id): AxumPath, +) -> impl IntoResponse { let mut settings = state.settings.lock().unwrap(); match settings.remove_alert_rule(&state.settings_path, &id) { Ok(()) => StatusCode::NO_CONTENT.into_response(), @@ -618,7 +700,9 @@ async fn delete_alert_rule(State(state): State, AxumPath(id): AxumPath } } -async fn test_alert_notification(Json(body): Json) -> impl IntoResponse { +async fn test_alert_notification( + Json(body): Json, +) -> impl IntoResponse { let server = body.server.unwrap_or_else(|| "https://ntfy.sh".to_string()); match deliver_ntfy(&server, &body.topic, "Test notification from ottyl").await { Ok(()) => StatusCode::NO_CONTENT.into_response(), @@ -652,7 +736,9 @@ async fn update_settings( } match settings.save(&state.settings_path) { Ok(()) => (), - Err(err) => return (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(), + Err(err) => { + return (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(); + } } (SettingsDto::from(&*settings), settings.max_logs) }; @@ -671,16 +757,26 @@ impl ProfileError { fn into_response(self) -> axum::response::Response { match self { ProfileError::NotFound => (StatusCode::NOT_FOUND, self.to_string()).into_response(), - ProfileError::DuplicateName => (StatusCode::BAD_REQUEST, self.to_string()).into_response(), - ProfileError::CannotDeleteDefault => (StatusCode::BAD_REQUEST, self.to_string()).into_response(), - ProfileError::Io(_) => (StatusCode::INTERNAL_SERVER_ERROR, self.to_string()).into_response(), + ProfileError::DuplicateName => { + (StatusCode::BAD_REQUEST, self.to_string()).into_response() + } + ProfileError::CannotDeleteDefault => { + (StatusCode::BAD_REQUEST, self.to_string()).into_response() + } + ProfileError::Io(_) => { + (StatusCode::INTERNAL_SERVER_ERROR, self.to_string()).into_response() + } } } } async fn list_profiles(State(state): State) -> impl IntoResponse { let settings = state.settings.lock().unwrap(); - let summaries: Vec = settings.profiles.iter().map(ProfileSummaryDto::from).collect(); + let summaries: Vec = settings + .profiles + .iter() + .map(ProfileSummaryDto::from) + .collect(); Json(summaries).into_response() } @@ -691,7 +787,9 @@ async fn create_profile( let mut settings = state.settings.lock().unwrap(); match settings.create_profile(&state.settings_path, body.name.clone()) { Ok(()) => { - let profile = settings.profile(&body.name).expect("profile was just created by create_profile"); + let profile = settings + .profile(&body.name) + .expect("profile was just created by create_profile"); (StatusCode::CREATED, Json(ProfileSummaryDto::from(profile))).into_response() } Err(err) => err.into_response(), @@ -717,7 +815,10 @@ async fn reload_engine_declarations(engine: &EngineHandle, profile: &Profile) { let (respond_to, response) = oneshot::channel(); if engine .tx - .send(EngineMessage::RemoveParamDeclaration { name: param.name, respond_to }) + .send(EngineMessage::RemoveParamDeclaration { + name: param.name, + respond_to, + }) .await .is_ok() { @@ -732,7 +833,10 @@ async fn reload_engine_declarations(engine: &EngineHandle, profile: &Profile) { let (respond_to, response) = oneshot::channel(); if engine .tx - .send(EngineMessage::RemoveMetricDeclaration { name: metric.name, respond_to }) + .send(EngineMessage::RemoveMetricDeclaration { + name: metric.name, + respond_to, + }) .await .is_ok() { @@ -835,13 +939,13 @@ async fn delete_dashboard( #[cfg(test)] mod tests { use super::*; - use crate::engine::{self, IngestedLine}; use crate::config::Settings; use crate::engine::timestamp::Timestamp; + use crate::engine::{self, IngestedLine}; use axum::body::Body; use axum::http::Request; use http_body_util::BodyExt; - use serde_json::{json, Value}; + use serde_json::{Value, json}; use std::sync::Mutex; use tower::ServiceExt; @@ -875,7 +979,11 @@ mod tests { } async fn get_json(app: &Router, uri: &str) -> (StatusCode, Value) { - let request = Request::builder().method("GET").uri(uri).body(Body::empty()).unwrap(); + let request = Request::builder() + .method("GET") + .uri(uri) + .body(Body::empty()) + .unwrap(); let response = app.clone().oneshot(request).await.unwrap(); let status = response.status(); let bytes = response.into_body().collect().await.unwrap().to_bytes(); @@ -982,11 +1090,22 @@ mod tests { let settings_path = dir.path().join("settings.json"); { let mut settings = settings_handle.lock().unwrap(); - settings.create_profile(&settings_path, "web".to_string()).unwrap(); - settings.set_active_profile(&settings_path, Some("web".to_string())).unwrap(); + settings + .create_profile(&settings_path, "web".to_string()) + .unwrap(); + settings + .set_active_profile(&settings_path, Some("web".to_string())) + .unwrap(); } - declare_test_param(&app, "path", r"request path=(?P\S+)", "path", "string").await; + declare_test_param( + &app, + "path", + r"request path=(?P\S+)", + "path", + "string", + ) + .await; ingest_lines(&engine, 0, &["request path=/a", "level=info x=1"]).await; let (_, before) = get_json(&app, "/api/params").await; @@ -1008,8 +1127,12 @@ mod tests { assert_eq!(result["logs_considered"], 2); let (_, after) = get_json(&app, "/api/params").await; - let after_declared: Vec<_> = - after.as_array().unwrap().iter().filter(|p| p["reserved"] != true).collect(); + let after_declared: Vec<_> = after + .as_array() + .unwrap() + .iter() + .filter(|p| p["reserved"] != true) + .collect(); assert_eq!(after_declared.len(), 3); assert!(after_declared.iter().all(|p| p["auto_derived"] == true)); @@ -1037,7 +1160,13 @@ mod tests { assert!(body["error"].as_str().unwrap().contains("invalid regex")); } - async fn declare_test_param(app: &Router, name: &str, pattern: &str, capture_group: &str, kind: &str) { + async fn declare_test_param( + app: &Router, + name: &str, + pattern: &str, + capture_group: &str, + kind: &str, + ) { let (status, _body) = send_json( app, "POST", @@ -1056,7 +1185,14 @@ mod tests { #[tokio::test] async fn declares_metric_and_shows_up_in_list() { let (app, _engine, _settings, _dir) = test_app(); - declare_test_param(&app, "path", r"request path=(?P\S+)", "path", "string").await; + declare_test_param( + &app, + "path", + r"request path=(?P\S+)", + "path", + "string", + ) + .await; let (status, body) = send_json( &app, "POST", @@ -1099,8 +1235,22 @@ mod tests { #[tokio::test] async fn updates_metric_declaration_in_place_rather_than_duplicating() { let (app, _engine, _settings, _dir) = test_app(); - declare_test_param(&app, "path", r"request path=(?P\S+)", "path", "string").await; - declare_test_param(&app, "duration", r"duration=(?P\d+)", "duration", "number").await; + declare_test_param( + &app, + "path", + r"request path=(?P\S+)", + "path", + "string", + ) + .await; + declare_test_param( + &app, + "duration", + r"duration=(?P\d+)", + "duration", + "number", + ) + .await; send_json( &app, "POST", @@ -1220,8 +1370,12 @@ mod tests { let settings_path = dir.path().join("settings.json"); { let mut settings = settings_handle.lock().unwrap(); - settings.create_profile(&settings_path, "web".to_string()).unwrap(); - settings.set_active_profile(&settings_path, Some("web".to_string())).unwrap(); + settings + .create_profile(&settings_path, "web".to_string()) + .unwrap(); + settings + .set_active_profile(&settings_path, Some("web".to_string())) + .unwrap(); } send_json( @@ -1235,19 +1389,39 @@ mod tests { }), ) .await; - assert_eq!(Settings::load(&settings_path).profile("web").unwrap().metrics.len(), 1); + assert_eq!( + Settings::load(&settings_path) + .profile("web") + .unwrap() + .metrics + .len(), + 1 + ); let status = delete_request(&app, "/api/metrics/requests").await; assert_eq!(status, StatusCode::NO_CONTENT); - assert!(Settings::load(&settings_path).profile("web").unwrap().metrics.is_empty()); + assert!( + Settings::load(&settings_path) + .profile("web") + .unwrap() + .metrics + .is_empty() + ); } #[tokio::test] async fn promql_query_against_deleted_metric_fails_with_readable_error() { let (app, engine, _settings, _dir) = test_app(); - declare_test_param(&app, "path", r"request path=(?P\S+)", "path", "string").await; + declare_test_param( + &app, + "path", + r"request path=(?P\S+)", + "path", + "string", + ) + .await; send_json( &app, "POST", @@ -1288,7 +1462,14 @@ mod tests { async fn queries_bucket_values_with_label_filter_and_range() { let (app, engine, _settings, _dir) = test_app(); - declare_test_param(&app, "path", r"request path=(?P\S+)", "path", "string").await; + declare_test_param( + &app, + "path", + r"request path=(?P\S+)", + "path", + "string", + ) + .await; send_json( &app, "POST", @@ -1341,7 +1522,14 @@ mod tests { async fn promql_query_returns_series_for_seeded_fixture() { let (app, engine, _settings, _dir) = test_app(); - declare_test_param(&app, "path", r"request path=(?P\S+)", "path", "string").await; + declare_test_param( + &app, + "path", + r"request path=(?P\S+)", + "path", + "string", + ) + .await; send_json( &app, "POST", @@ -1398,8 +1586,12 @@ mod tests { let settings_path = dir.path().join("settings.json"); { let mut settings = settings_handle.lock().unwrap(); - settings.create_profile(&settings_path, "web".to_string()).unwrap(); - settings.set_active_profile(&settings_path, Some("web".to_string())).unwrap(); + settings + .create_profile(&settings_path, "web".to_string()) + .unwrap(); + settings + .set_active_profile(&settings_path, Some("web".to_string())) + .unwrap(); } let (status, _body) = send_json( @@ -1429,8 +1621,12 @@ mod tests { let settings_path = dir.path().join("settings.json"); { let mut settings = settings_handle.lock().unwrap(); - settings.create_profile(&settings_path, "web".to_string()).unwrap(); - settings.set_active_profile(&settings_path, Some("web".to_string())).unwrap(); + settings + .create_profile(&settings_path, "web".to_string()) + .unwrap(); + settings + .set_active_profile(&settings_path, Some("web".to_string())) + .unwrap(); } ingest_lines(&engine, 0, &["dx=1 moved"]).await; @@ -1459,7 +1655,10 @@ mod tests { let fresh_engine_tx = crate::engine::spawn(fresh_settings_handle, 16); profile.load_into_engine(&fresh_engine_tx).await; - let response = fresh_engine_tx.ask(EngineQuery::MetricDeclarations).await.unwrap(); + let response = fresh_engine_tx + .ask(EngineQuery::MetricDeclarations) + .await + .unwrap(); let EngineResponse::MetricDeclarations(metrics) = response else { panic!("unexpected response"); }; @@ -1468,12 +1667,20 @@ mod tests { } async fn delete_request(app: &Router, uri: &str) -> StatusCode { - let request = Request::builder().method("DELETE").uri(uri).body(Body::empty()).unwrap(); + let request = Request::builder() + .method("DELETE") + .uri(uri) + .body(Body::empty()) + .unwrap(); app.clone().oneshot(request).await.unwrap().status() } async fn get_status(app: &Router, uri: &str) -> StatusCode { - let request = Request::builder().method("GET").uri(uri).body(Body::empty()).unwrap(); + let request = Request::builder() + .method("GET") + .uri(uri) + .body(Body::empty()) + .unwrap(); app.clone().oneshot(request).await.unwrap().status() } @@ -1481,7 +1688,13 @@ mod tests { async fn dashboard_crud_round_trips_through_http() { let (app, _engine, _settings, _dir) = test_app(); - let (status, _body) = send_json(&app, "POST", "/api/settings/profiles", json!({ "name": "web" })).await; + let (status, _body) = send_json( + &app, + "POST", + "/api/settings/profiles", + json!({ "name": "web" }), + ) + .await; assert_eq!(status, StatusCode::CREATED); let (status, body) = send_json( @@ -1518,12 +1731,19 @@ mod tests { assert_eq!(panels.len(), 1); assert_eq!(panels[0]["chart_type"], "line"); - let (status, body) = - get_json(&app, &format!("/api/settings/profiles/web/dashboards/{dashboard_id}")).await; + let (status, body) = get_json( + &app, + &format!("/api/settings/profiles/web/dashboards/{dashboard_id}"), + ) + .await; assert_eq!(status, StatusCode::OK); assert_eq!(body["panels"].as_array().unwrap().len(), 1); - let status = delete_request(&app, &format!("/api/settings/profiles/web/dashboards/{dashboard_id}")).await; + let status = delete_request( + &app, + &format!("/api/settings/profiles/web/dashboards/{dashboard_id}"), + ) + .await; assert_eq!(status, StatusCode::NO_CONTENT); let (status, body) = get_json(&app, "/api/settings/profiles/web/dashboards").await; assert_eq!(status, StatusCode::OK); @@ -1536,7 +1756,13 @@ mod tests { let status = get_status(&app, "/api/settings/profiles/ghost/dashboards").await; assert_eq!(status, StatusCode::NOT_FOUND); - let (status, _body) = send_json(&app, "POST", "/api/settings/profiles", json!({ "name": "web" })).await; + let (status, _body) = send_json( + &app, + "POST", + "/api/settings/profiles", + json!({ "name": "web" }), + ) + .await; assert_eq!(status, StatusCode::CREATED); let status = get_status(&app, "/api/settings/profiles/web/dashboards/does-not-exist").await; assert_eq!(status, StatusCode::NOT_FOUND); @@ -1546,7 +1772,13 @@ mod tests { async fn saved_dashboard_persists_across_simulated_restart() { let (app, _engine, _settings_handle, dir) = test_app(); - let (status, _body) = send_json(&app, "POST", "/api/settings/profiles", json!({ "name": "web" })).await; + let (status, _body) = send_json( + &app, + "POST", + "/api/settings/profiles", + json!({ "name": "web" }), + ) + .await; assert_eq!(status, StatusCode::CREATED); let (status, body) = send_json( &app, @@ -1605,7 +1837,8 @@ mod tests { assert_eq!(status, StatusCode::OK); assert_eq!(body.as_array().unwrap().len(), 3); - let (status, body) = send_json(&app, "PATCH", "/api/settings", json!({ "max_logs": 2 })).await; + let (status, body) = + send_json(&app, "PATCH", "/api/settings", json!({ "max_logs": 2 })).await; assert_eq!(status, StatusCode::OK); assert_eq!(body["max_logs"], 2); @@ -1644,7 +1877,8 @@ mod tests { let (_, body) = get_json(&app, "/api/logs").await; assert_eq!(body.as_array().unwrap().len(), 1); - let (status, body) = send_json(&app, "PATCH", "/api/settings", json!({ "max_logs": null })).await; + let (status, body) = + send_json(&app, "PATCH", "/api/settings", json!({ "max_logs": null })).await; assert_eq!(status, StatusCode::OK); assert_eq!(body["max_logs"], Value::Null); @@ -1657,7 +1891,14 @@ mod tests { async fn get_logs_sort_by_orders_by_param_and_toggles_missing_param_handling() { let (app, engine, _settings, _dir) = test_app(); - declare_test_param(&app, "priority", r"priority=(?P\d+)", "priority", "number").await; + declare_test_param( + &app, + "priority", + r"priority=(?P\d+)", + "priority", + "number", + ) + .await; ingest_lines( &engine, @@ -1673,17 +1914,36 @@ mod tests { let (status, body) = get_json(&app, "/api/logs").await; assert_eq!(status, StatusCode::OK); - let sequences: Vec = body.as_array().unwrap().iter().map(|e| e["sequence"].as_u64().unwrap()).collect(); + let sequences: Vec = body + .as_array() + .unwrap() + .iter() + .map(|e| e["sequence"].as_u64().unwrap()) + .collect(); assert_eq!(sequences, vec![0, 1, 2, 3]); let (status, body) = get_json(&app, "/api/logs?sort_by=priority").await; assert_eq!(status, StatusCode::OK); - let sequences: Vec = body.as_array().unwrap().iter().map(|e| e["sequence"].as_u64().unwrap()).collect(); + let sequences: Vec = body + .as_array() + .unwrap() + .iter() + .map(|e| e["sequence"].as_u64().unwrap()) + .collect(); assert_eq!(sequences, vec![2, 0, 1, 3]); - let (status, body) = get_json(&app, "/api/logs?sort_by=priority&sort_missing_fallback=false").await; + let (status, body) = get_json( + &app, + "/api/logs?sort_by=priority&sort_missing_fallback=false", + ) + .await; assert_eq!(status, StatusCode::OK); - let sequences: Vec = body.as_array().unwrap().iter().map(|e| e["sequence"].as_u64().unwrap()).collect(); + let sequences: Vec = body + .as_array() + .unwrap() + .iter() + .map(|e| e["sequence"].as_u64().unwrap()) + .collect(); assert_eq!(sequences, vec![2, 0]); } diff --git a/src/cli/mod.rs b/src/cli/mod.rs index eb5612b..9c5341a 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -86,15 +86,16 @@ pub async fn run() { register_session_and_log(&settings_path, &args, bound_port); - axum::serve(listener, app) - .await - .expect("server error"); + axum::serve(listener, app).await.expect("server error"); } async fn spawn_engine(settings: &Arc>) -> crate::engine::EngineHandle { let engine_tx = crate::engine::spawn(settings.clone(), crate::consts::ENGINE_CHANNEL_CAPACITY); let max_logs = settings.lock().unwrap().max_logs; - let _ = engine_tx.tx.send(crate::engine::EngineMessage::SetMaxLogs(max_logs)).await; + let _ = engine_tx + .tx + .send(crate::engine::EngineMessage::SetMaxLogs(max_logs)) + .await; engine_tx } @@ -142,7 +143,10 @@ fn spawn_ingest(args: &Args, engine_tx: &crate::engine::EngineHandle) { } if let Some(metrics_url) = args.metrics_url.clone() { - tokio::spawn(crate::engine::ingest::run_metrics_poll(engine_tx.clone(), metrics_url)); + tokio::spawn(crate::engine::ingest::run_metrics_poll( + engine_tx.clone(), + metrics_url, + )); } } @@ -198,7 +202,10 @@ fn print_sessions(settings_path: &std::path::Path) { return; } for session in sessions { - println!("{}\tport={}\tpid={}", session.name, session.port, session.pid); + println!( + "{}\tport={}\tpid={}", + session.name, session.port, session.pid + ); } } diff --git a/src/cli/sessions.rs b/src/cli/sessions.rs index 657b232..10788f3 100644 --- a/src/cli/sessions.rs +++ b/src/cli/sessions.rs @@ -131,7 +131,11 @@ mod tests { let sessions_dir_path = sessions_dir(&settings_path); fs::create_dir_all(&sessions_dir_path).unwrap(); - let stale = SessionInfo { name: "foo".to_string(), port: 1111, pid: 999_999 }; + let stale = SessionInfo { + name: "foo".to_string(), + port: 1111, + pid: 999_999, + }; fs::write( session_file_path(&sessions_dir_path, "foo"), serde_json::to_string(&stale).unwrap(), diff --git a/src/config/mod.rs b/src/config/mod.rs index 74c0313..6c07ec0 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -8,8 +8,8 @@ use std::sync::{Arc, Mutex}; use directories::ProjectDirs; use serde::{Deserialize, Serialize}; -use crate::engine::alerts::AlertRule; use crate::config::profile::Profile; +use crate::engine::alerts::AlertRule; pub type SettingsHandle = Arc>; @@ -70,13 +70,13 @@ impl From for Settings { ProfilesOnDisk::Inline(profiles) => (profiles, None), ProfilesOnDisk::Legacy(pointer) => (Vec::new(), pointer.active_profile), }; - let theme = on_disk.theme.unwrap_or_else(|| { - match on_disk.color_scheme.as_deref() { + let theme = on_disk + .theme + .unwrap_or_else(|| match on_disk.color_scheme.as_deref() { Some("grey") => "dark".to_string(), Some(scheme) => scheme.to_string(), None => default_theme(), - } - }); + }); Settings { theme, profiles, @@ -185,7 +185,8 @@ impl Settings { .map(|parent| parent.join("profiles")) .unwrap_or_else(|| PathBuf::from("profiles")) .join("profiles.json"); - let Some(profiles) = crate::config::profile::Profile::load_legacy_store(&legacy_path) else { + let Some(profiles) = crate::config::profile::Profile::load_legacy_store(&legacy_path) + else { return; }; self.profiles = profiles; @@ -271,7 +272,9 @@ mod tests { theme: "dark".to_string(), ..Settings::default() }; - settings.profiles.push(crate::config::profile::Profile::new("default".to_string())); + settings + .profiles + .push(crate::config::profile::Profile::new("default".to_string())); settings.active_profile = Some("default".to_string()); settings.alert_rules.push(crate::engine::alerts::AlertRule { id: "rule1".to_string(), diff --git a/src/config/profile.rs b/src/config/profile.rs index fd40450..5d75254 100644 --- a/src/config/profile.rs +++ b/src/config/profile.rs @@ -1,7 +1,7 @@ #![allow(dead_code)] -use std::collections::hash_map::DefaultHasher; use std::collections::BTreeMap; +use std::collections::hash_map::DefaultHasher; use std::fmt; use std::fs; use std::hash::{Hash, Hasher}; @@ -11,10 +11,10 @@ use std::time::{SystemTime, UNIX_EPOCH}; use serde::{Deserialize, Serialize}; +use crate::config::Settings; use crate::consts::{DEFAULT_PROFILE_NAME, SEEDED_METRIC_LABEL, SEEDED_METRIC_NAME}; use crate::engine::data_value::MetricPrimitiveKind; use crate::engine::{DeclareMetricRequest, DeclareParamRequest, EngineHandle, EngineMessage}; -use crate::config::Settings; use tokio::sync::oneshot; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -167,8 +167,11 @@ impl Profile { let mut label_params = Vec::new(); let mut value_param = None; for matcher in &legacy { - let param_name = - self.resolve_or_create_legacy_param(&metric_name, &message_match_regex, matcher); + let param_name = self.resolve_or_create_legacy_param( + &metric_name, + &message_match_regex, + matcher, + ); match matcher.role { LegacyParamRole::Label => label_params.push(param_name), LegacyParamRole::Value => value_param = Some(param_name), @@ -187,14 +190,13 @@ impl Profile { message_match_regex: &str, matcher: &LegacyMetricParamRecord, ) -> String { - if let Some(existing) = self.params.iter().find(|p| p.name == matcher.param_name) { - if existing.message_match_regex == message_match_regex - && existing.capture_group == matcher.capture_group - && existing.json_field_path == matcher.json_field_path - && existing.kind == matcher.kind - { - return existing.name.clone(); - } + if let Some(existing) = self.params.iter().find(|p| p.name == matcher.param_name) + && existing.message_match_regex == message_match_regex + && existing.capture_group == matcher.capture_group + && existing.json_field_path == matcher.json_field_path + && existing.kind == matcher.kind + { + return existing.name.clone(); } let implicit_name = format!("{metric_name}.{}", matcher.param_name); @@ -213,8 +215,11 @@ impl Profile { } fn migrate_duration_value_params(&mut self) -> bool { - let value_param_names: std::collections::HashSet<&str> = - self.metrics.iter().filter_map(|metric| metric.value_param.as_deref()).collect(); + let value_param_names: std::collections::HashSet<&str> = self + .metrics + .iter() + .filter_map(|metric| metric.value_param.as_deref()) + .collect(); let mut migrated = false; for param in &mut self.params { if !param.elapsed_since_last @@ -267,7 +272,11 @@ fn generate_id(seed: &str) -> String { impl Dashboard { pub fn new(name: String) -> Dashboard { let id = generate_id(&name); - Dashboard { id, name, panels: Vec::new() } + Dashboard { + id, + name, + panels: Vec::new(), + } } } @@ -278,8 +287,14 @@ impl MetricDeclarationRecord { message_match_regex: r"(?i)error".to_string(), label_params: Vec::new(), value_param: None, - matched_labels: Some(BTreeMap::from([(SEEDED_METRIC_LABEL.to_string(), "error".to_string())])), - unmatched_labels: Some(BTreeMap::from([(SEEDED_METRIC_LABEL.to_string(), "other".to_string())])), + matched_labels: Some(BTreeMap::from([( + SEEDED_METRIC_LABEL.to_string(), + "error".to_string(), + )])), + unmatched_labels: Some(BTreeMap::from([( + SEEDED_METRIC_LABEL.to_string(), + "other".to_string(), + )])), legacy_params: Vec::new(), } } @@ -336,11 +351,17 @@ impl Profile { let name = request.name.clone(); if engine .tx - .send(EngineMessage::DeclareParam { request, respond_to }) + .send(EngineMessage::DeclareParam { + request, + respond_to, + }) .await .is_err() { - eprintln!("warning: engine unavailable while loading profile '{}'", self.name); + eprintln!( + "warning: engine unavailable while loading profile '{}'", + self.name + ); return; } match response.await { @@ -362,11 +383,17 @@ impl Profile { let name = request.name.clone(); if engine .tx - .send(EngineMessage::DeclareMetric { request, respond_to }) + .send(EngineMessage::DeclareMetric { + request, + respond_to, + }) .await .is_err() { - eprintln!("warning: engine unavailable while loading profile '{}'", self.name); + eprintln!( + "warning: engine unavailable while loading profile '{}'", + self.name + ); return; } match response.await { @@ -444,7 +471,11 @@ impl Settings { self.profiles.iter().find(|p| p.name == name) } - pub fn create_profile(&mut self, settings_path: &Path, name: String) -> Result<(), ProfileError> { + pub fn create_profile( + &mut self, + settings_path: &Path, + name: String, + ) -> Result<(), ProfileError> { if self.profile(&name).is_some() { return Err(ProfileError::DuplicateName); } @@ -463,7 +494,11 @@ impl Settings { if name == DEFAULT_PROFILE_NAME { return Err(ProfileError::CannotDeleteDefault); } - let index = self.profiles.iter().position(|p| p.name == name).ok_or(ProfileError::NotFound)?; + let index = self + .profiles + .iter() + .position(|p| p.name == name) + .ok_or(ProfileError::NotFound)?; self.profiles.remove(index); if self.active_profile.as_deref() == Some(name) { self.active_profile = None; @@ -475,7 +510,9 @@ impl Settings { pub fn ensure_default_profile(&mut self, settings_path: &Path) -> Result<(), ProfileError> { if self.profile(DEFAULT_PROFILE_NAME).is_none() { let mut profile = Profile::new(DEFAULT_PROFILE_NAME.to_string()); - profile.metrics.push(MetricDeclarationRecord::seeded_error_split()); + profile + .metrics + .push(MetricDeclarationRecord::seeded_error_split()); profile.dashboards.push(Dashboard::seeded_default()); self.profiles.push(profile); } @@ -491,10 +528,10 @@ impl Settings { settings_path: &Path, name: Option, ) -> Result<(), ProfileError> { - if let Some(name) = &name { - if self.profile(name).is_none() { - return Err(ProfileError::NotFound); - } + if let Some(name) = &name + && self.profile(name).is_none() + { + return Err(ProfileError::NotFound); } self.active_profile = name; self.save(settings_path)?; @@ -678,10 +715,10 @@ impl Settings { #[cfg(test)] mod tests { use super::*; - use crate::engine::data_value::MetricPrimitiveKind; use crate::api::dto::LogsParams; - use crate::engine::{EngineMessage, EngineQuery, EngineResponse, IngestedLine}; + use crate::engine::data_value::MetricPrimitiveKind; use crate::engine::timestamp::Timestamp; + use crate::engine::{EngineMessage, EngineQuery, EngineResponse, IngestedLine}; fn param_request(name: &str, pattern: &str, capture_group: &str) -> DeclareParamRequest { DeclareParamRequest { @@ -719,8 +756,12 @@ mod tests { let settings_path = dir.path().join("settings.json"); let mut settings = Settings::default(); - settings.create_profile(&settings_path, "web".to_string()).unwrap(); - settings.create_profile(&settings_path, "worker".to_string()).unwrap(); + settings + .create_profile(&settings_path, "web".to_string()) + .unwrap(); + settings + .create_profile(&settings_path, "worker".to_string()) + .unwrap(); assert_eq!(settings.profiles.len(), 2); assert!(settings.profile("web").is_some()); @@ -750,13 +791,27 @@ mod tests { settings.ensure_default_profile(&settings_path).unwrap(); assert!(settings.profile(DEFAULT_PROFILE_NAME).is_some()); - assert_eq!(settings.active_profile, Some(DEFAULT_PROFILE_NAME.to_string())); + assert_eq!( + settings.active_profile, + Some(DEFAULT_PROFILE_NAME.to_string()) + ); - settings.create_profile(&settings_path, "web".to_string()).unwrap(); - settings.set_active_profile(&settings_path, Some("web".to_string())).unwrap(); + settings + .create_profile(&settings_path, "web".to_string()) + .unwrap(); + settings + .set_active_profile(&settings_path, Some("web".to_string())) + .unwrap(); settings.ensure_default_profile(&settings_path).unwrap(); - assert_eq!(settings.profiles.iter().filter(|p| p.name == DEFAULT_PROFILE_NAME).count(), 1); + assert_eq!( + settings + .profiles + .iter() + .filter(|p| p.name == DEFAULT_PROFILE_NAME) + .count(), + 1 + ); assert_eq!(settings.active_profile, Some("web".to_string())); } @@ -773,20 +828,33 @@ mod tests { assert_eq!(default_profile.metrics[0].name, SEEDED_METRIC_NAME); assert_eq!( default_profile.metrics[0].matched_labels, - Some(BTreeMap::from([(SEEDED_METRIC_LABEL.to_string(), "error".to_string())])) + Some(BTreeMap::from([( + SEEDED_METRIC_LABEL.to_string(), + "error".to_string() + )])) ); assert_eq!( default_profile.metrics[0].unmatched_labels, - Some(BTreeMap::from([(SEEDED_METRIC_LABEL.to_string(), "other".to_string())])) + Some(BTreeMap::from([( + SEEDED_METRIC_LABEL.to_string(), + "other".to_string() + )])) ); assert_eq!(default_profile.dashboards.len(), 1); let dashboard = &default_profile.dashboards[0]; assert_eq!(dashboard.panels.len(), 2); - assert!(dashboard - .panels - .iter() - .any(|p| p.chart_type == ChartType::Line && p.promql.contains(SEEDED_METRIC_NAME))); - assert!(dashboard.panels.iter().any(|p| p.chart_type == ChartType::Pie && p.promql == SEEDED_METRIC_NAME)); + assert!( + dashboard + .panels + .iter() + .any(|p| p.chart_type == ChartType::Line && p.promql.contains(SEEDED_METRIC_NAME)) + ); + assert!( + dashboard + .panels + .iter() + .any(|p| p.chart_type == ChartType::Pie && p.promql == SEEDED_METRIC_NAME) + ); let mut reloaded = Settings::load(&settings_path); reloaded.ensure_default_profile(&settings_path).unwrap(); @@ -797,8 +865,8 @@ mod tests { #[tokio::test] async fn seeded_metric_renders_real_error_vs_other_data() { - use crate::engine::{self, EngineMessage, EngineQuery, EngineResponse, IngestedLine}; use crate::engine::timestamp::Timestamp; + use crate::engine::{self, EngineMessage, EngineQuery, EngineResponse, IngestedLine}; let dir = tempfile::tempdir().unwrap(); let settings_path = dir.path().join("settings.json"); @@ -810,7 +878,11 @@ mod tests { let engine_tx = engine::spawn(settings_handle, 128); default_profile.load_into_engine(&engine_tx).await; - for (sequence, text) in [(0u64, "all good"), (1, "an ERROR happened"), (2, "fine again")] { + for (sequence, text) in [ + (0u64, "all good"), + (1, "an ERROR happened"), + (2, "fine again"), + ] { engine_tx .tx .send(EngineMessage::Ingest(IngestedLine { @@ -825,18 +897,38 @@ mod tests { let response = ask( &engine_tx, - EngineQuery::Promql { query: SEEDED_METRIC_NAME.to_string(), start_ms: None, end_ms: None }, + EngineQuery::Promql { + query: SEEDED_METRIC_NAME.to_string(), + start_ms: None, + end_ms: None, + }, ) .await; let EngineResponse::Promql(Ok(mut series)) = response else { panic!("expected a successful Promql response"); }; - series.sort_by(|a, b| a.labels.get(SEEDED_METRIC_LABEL).cmp(&b.labels.get(SEEDED_METRIC_LABEL))); + series.sort_by(|a, b| { + a.labels + .get(SEEDED_METRIC_LABEL) + .cmp(&b.labels.get(SEEDED_METRIC_LABEL)) + }); assert_eq!(series.len(), 2); - assert_eq!(series[0].labels.get(SEEDED_METRIC_LABEL).map(String::as_str), Some("error")); + assert_eq!( + series[0] + .labels + .get(SEEDED_METRIC_LABEL) + .map(String::as_str), + Some("error") + ); assert_eq!(series[0].points.len(), 1); - assert_eq!(series[1].labels.get(SEEDED_METRIC_LABEL).map(String::as_str), Some("other")); + assert_eq!( + series[1] + .labels + .get(SEEDED_METRIC_LABEL) + .map(String::as_str), + Some("other") + ); assert_eq!(series[1].points.len(), 2); } @@ -859,19 +951,31 @@ mod tests { let settings_path = dir.path().join("settings.json"); let mut settings = Settings::default(); - settings.create_profile(&settings_path, "web".to_string()).unwrap(); + settings + .create_profile(&settings_path, "web".to_string()) + .unwrap(); let param_req = param_request("level", r"level=(?P\w+)", "level"); - settings.record_param_declaration(&settings_path, "web", ¶m_req).unwrap(); - settings.create_dashboard(&settings_path, "web", "Overview".to_string()).unwrap(); - settings.set_active_profile(&settings_path, Some("web".to_string())).unwrap(); + settings + .record_param_declaration(&settings_path, "web", ¶m_req) + .unwrap(); + settings + .create_dashboard(&settings_path, "web", "Overview".to_string()) + .unwrap(); + settings + .set_active_profile(&settings_path, Some("web".to_string())) + .unwrap(); - settings.create_profile(&settings_path, "web-copy".to_string()).unwrap(); + settings + .create_profile(&settings_path, "web-copy".to_string()) + .unwrap(); let copy = settings.profile("web-copy").unwrap().clone(); assert_eq!(copy.params, settings.profile("web").unwrap().params); assert_eq!(copy.dashboards, settings.profile("web").unwrap().dashboards); settings.set_active_profile(&settings_path, None).unwrap(); - settings.create_profile(&settings_path, "blank".to_string()).unwrap(); + settings + .create_profile(&settings_path, "blank".to_string()) + .unwrap(); let blank = settings.profile("blank").unwrap(); assert!(blank.params.is_empty()); assert!(blank.dashboards.is_empty()); @@ -883,13 +987,21 @@ mod tests { let settings_path = dir.path().join("settings.json"); let mut settings = Settings::default(); - settings.create_profile(&settings_path, "web".to_string()).unwrap(); - settings.create_profile(&settings_path, "worker".to_string()).unwrap(); + settings + .create_profile(&settings_path, "web".to_string()) + .unwrap(); + settings + .create_profile(&settings_path, "worker".to_string()) + .unwrap(); - settings.set_active_profile(&settings_path, Some("web".to_string())).unwrap(); + settings + .set_active_profile(&settings_path, Some("web".to_string())) + .unwrap(); assert_eq!(settings.active_profile, Some("web".to_string())); - settings.set_active_profile(&settings_path, Some("worker".to_string())).unwrap(); + settings + .set_active_profile(&settings_path, Some("worker".to_string())) + .unwrap(); let reloaded = Settings::load(&settings_path); assert_eq!(reloaded.active_profile, Some("worker".to_string())); @@ -907,13 +1019,19 @@ mod tests { let settings_path = dir.path().join("settings.json"); let mut settings = Settings::default(); - settings.create_profile(&settings_path, "web".to_string()).unwrap(); + settings + .create_profile(&settings_path, "web".to_string()) + .unwrap(); let param_req = param_request("level", r"level=(?P\w+)", "level"); - settings.record_param_declaration(&settings_path, "web", ¶m_req).unwrap(); + settings + .record_param_declaration(&settings_path, "web", ¶m_req) + .unwrap(); let metric_req = metric_request("requests", r"duration=(?P\d+)", "duration"); - settings.record_metric_declaration(&settings_path, "web", &metric_req, &[]).unwrap(); + settings + .record_metric_declaration(&settings_path, "web", &metric_req, &[]) + .unwrap(); let profile = settings.profile("web").unwrap(); assert_eq!(profile.params.len(), 1); @@ -922,7 +1040,9 @@ mod tests { assert_eq!(profile.metrics[0].name, "requests"); let param_req_v2 = param_request("level", r"lvl=(?P\w+)", "level"); - settings.record_param_declaration(&settings_path, "web", ¶m_req_v2).unwrap(); + settings + .record_param_declaration(&settings_path, "web", ¶m_req_v2) + .unwrap(); let profile = settings.profile("web").unwrap(); assert_eq!(profile.params.len(), 1); assert_eq!(profile.params[0].message_match_regex, r"lvl=(?P\w+)"); @@ -939,15 +1059,23 @@ mod tests { let settings_path = dir.path().join("settings.json"); let mut settings = Settings::default(); - settings.create_profile(&settings_path, "web".to_string()).unwrap(); + settings + .create_profile(&settings_path, "web".to_string()) + .unwrap(); let metric_req = metric_request("requests", r"duration=(?P\d+)", "duration"); - settings.record_metric_declaration(&settings_path, "web", &metric_req, &[]).unwrap(); + settings + .record_metric_declaration(&settings_path, "web", &metric_req, &[]) + .unwrap(); assert_eq!(settings.profile("web").unwrap().metrics.len(), 1); - settings.remove_metric_declaration(&settings_path, "web", "requests").unwrap(); + settings + .remove_metric_declaration(&settings_path, "web", "requests") + .unwrap(); assert!(settings.profile("web").unwrap().metrics.is_empty()); - settings.remove_metric_declaration(&settings_path, "web", "never-declared").unwrap(); + settings + .remove_metric_declaration(&settings_path, "web", "never-declared") + .unwrap(); let reloaded = Settings::load(&settings_path); assert!(reloaded.profile("web").unwrap().metrics.is_empty()); @@ -959,7 +1087,9 @@ mod tests { let settings_path = dir.path().join("settings.json"); let mut settings = Settings::default(); - settings.create_profile(&settings_path, "web".to_string()).unwrap(); + settings + .create_profile(&settings_path, "web".to_string()) + .unwrap(); settings .record_param_declaration( &settings_path, @@ -971,18 +1101,28 @@ mod tests { .record_param_declaration( &settings_path, "web", - ¶m_request("duration", r"request duration=(?P\d+)", "duration"), + ¶m_request( + "duration", + r"request duration=(?P\d+)", + "duration", + ), ) .unwrap(); settings .record_metric_declaration( &settings_path, "web", - &metric_request("requests", r"request duration=(?P\d+)", "duration"), + &metric_request( + "requests", + r"request duration=(?P\d+)", + "duration", + ), &[], ) .unwrap(); - settings.set_active_profile(&settings_path, Some("web".to_string())).unwrap(); + settings + .set_active_profile(&settings_path, Some("web".to_string())) + .unwrap(); drop(settings); let settings = Settings::load(&settings_path); @@ -1020,7 +1160,10 @@ mod tests { }; assert_eq!(metrics.len(), 1); assert_eq!(metrics[0].name, "requests"); - assert_eq!(metrics[0].message_match_regex, r"request duration=(?P\d+)"); + assert_eq!( + metrics[0].message_match_regex, + r"request duration=(?P\d+)" + ); assert_eq!(metrics[0].label_params, vec!["duration".to_string()]); let logs = match ask( @@ -1050,9 +1193,13 @@ mod tests { let settings_path = dir.path().join("settings.json"); let mut settings = Settings::default(); - settings.create_profile(&settings_path, "web".to_string()).unwrap(); + settings + .create_profile(&settings_path, "web".to_string()) + .unwrap(); - let dashboard = settings.create_dashboard(&settings_path, "web", "Overview".to_string()).unwrap(); + let dashboard = settings + .create_dashboard(&settings_path, "web", "Overview".to_string()) + .unwrap(); assert_eq!(dashboard.name, "Overview"); assert!(dashboard.panels.is_empty()); @@ -1077,7 +1224,9 @@ mod tests { assert_eq!(dashboards[0].panels.len(), 1); assert_eq!(dashboards[0].panels[0].promql, r#"requests{path="/a"}"#); - reloaded.delete_dashboard(&settings_path, "web", &dashboard.id).unwrap(); + reloaded + .delete_dashboard(&settings_path, "web", &dashboard.id) + .unwrap(); assert!(reloaded.list_dashboards("web").unwrap().is_empty()); let reloaded_again = Settings::load(&settings_path); assert!(reloaded_again.list_dashboards("web").unwrap().is_empty()); @@ -1125,16 +1274,39 @@ mod tests { let profile = settings.profile("legacy").unwrap(); assert_eq!(profile.params.len(), 2); - let path_param = profile.params.iter().find(|p| p.name == "requests.path").unwrap(); - assert_eq!(path_param.message_match_regex, profile.metrics[0].message_match_regex); + let path_param = profile + .params + .iter() + .find(|p| p.name == "requests.path") + .unwrap(); + assert_eq!( + path_param.message_match_regex, + profile.metrics[0].message_match_regex + ); assert_eq!(path_param.capture_group, "path"); - assert!(matches!(path_param.kind, ProfileMetricPrimitiveKind::String)); - let duration_param = profile.params.iter().find(|p| p.name == "requests.duration").unwrap(); - assert!(matches!(duration_param.kind, ProfileMetricPrimitiveKind::Number)); + assert!(matches!( + path_param.kind, + ProfileMetricPrimitiveKind::String + )); + let duration_param = profile + .params + .iter() + .find(|p| p.name == "requests.duration") + .unwrap(); + assert!(matches!( + duration_param.kind, + ProfileMetricPrimitiveKind::Number + )); assert_eq!(profile.metrics.len(), 1); - assert_eq!(profile.metrics[0].label_params, vec!["requests.path".to_string()]); - assert_eq!(profile.metrics[0].value_param, Some("requests.duration".to_string())); + assert_eq!( + profile.metrics[0].label_params, + vec!["requests.path".to_string()] + ); + assert_eq!( + profile.metrics[0].value_param, + Some("requests.duration".to_string()) + ); let reloaded_raw = fs::read_to_string(&settings_path).unwrap(); assert!(!reloaded_raw.contains("param_name")); @@ -1192,10 +1364,20 @@ mod tests { let settings = Settings::load(&settings_path); let profile = settings.profile("legacy").unwrap(); - let gap_param = profile.params.iter().find(|p| p.name == "login_gap.gap").unwrap(); - assert!(matches!(gap_param.kind, ProfileMetricPrimitiveKind::Duration)); + let gap_param = profile + .params + .iter() + .find(|p| p.name == "login_gap.gap") + .unwrap(); + assert!(matches!( + gap_param.kind, + ProfileMetricPrimitiveKind::Duration + )); assert!(gap_param.elapsed_since_last); - assert_eq!(profile.metrics[0].value_param, Some("login_gap.gap".to_string())); + assert_eq!( + profile.metrics[0].value_param, + Some("login_gap.gap".to_string()) + ); } #[test] @@ -1229,7 +1411,14 @@ mod tests { let gap_param = profile.params.iter().find(|p| p.name == "gap").unwrap(); assert!(gap_param.elapsed_since_last); - assert!(!profile.params.iter().find(|p| p.name == "user").unwrap().elapsed_since_last); + assert!( + !profile + .params + .iter() + .find(|p| p.name == "user") + .unwrap() + .elapsed_since_last + ); } #[test] @@ -1271,7 +1460,9 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let settings_path = dir.path().join("settings.json"); let mut settings = Settings::default(); - settings.create_profile(&settings_path, "web".to_string()).unwrap(); + settings + .create_profile(&settings_path, "web".to_string()) + .unwrap(); settings .record_param_declaration( &settings_path, @@ -1280,7 +1471,12 @@ mod tests { ) .unwrap(); settings - .record_metric_declaration(&settings_path, "web", &metric_request("requests", r"path=(?P\S+)", "path"), &[]) + .record_metric_declaration( + &settings_path, + "web", + &metric_request("requests", r"path=(?P\S+)", "path"), + &[], + ) .unwrap(); let before = settings.profile("web").unwrap().clone(); @@ -1295,8 +1491,12 @@ mod tests { let settings_path = dir.path().join("settings.json"); let mut settings = Settings::default(); - settings.create_profile(&settings_path, "web".to_string()).unwrap(); - settings.set_active_profile(&settings_path, Some("web".to_string())).unwrap(); + settings + .create_profile(&settings_path, "web".to_string()) + .unwrap(); + settings + .set_active_profile(&settings_path, Some("web".to_string())) + .unwrap(); let settings_handle = std::sync::Arc::new(std::sync::Mutex::new(settings.clone())); let engine_tx = crate::engine::spawn(settings_handle, 16); @@ -1314,7 +1514,10 @@ mod tests { let (respond_to, response) = oneshot::channel(); engine_tx .tx - .send(EngineMessage::DeclareMetric { request: metric_req.clone(), respond_to }) + .send(EngineMessage::DeclareMetric { + request: metric_req.clone(), + respond_to, + }) .await .unwrap(); response.await.unwrap().unwrap(); @@ -1334,7 +1537,9 @@ mod tests { kind: dx.kind.into(), elapsed_since_last: dx.elapsed_since_last, }; - settings.record_metric_declaration(&settings_path, "web", &metric_req, &[promoted]).unwrap(); + settings + .record_metric_declaration(&settings_path, "web", &metric_req, &[promoted]) + .unwrap(); drop(settings); let reloaded_settings = Settings::load(&settings_path); diff --git a/src/engine/alerts.rs b/src/engine/alerts.rs index 0f75359..48c6d25 100644 --- a/src/engine/alerts.rs +++ b/src/engine/alerts.rs @@ -9,12 +9,12 @@ use std::time::{SystemTime, UNIX_EPOCH}; use regex::Regex; use serde::{Deserialize, Serialize}; -use crate::engine::{self, eval_json_comparison}; +use crate::config::Settings; use crate::engine::filter_datetime; use crate::engine::filter_parser::{self, CompareOp, FilterExpr, FilterValue}; use crate::engine::rate::RateWindow; -use crate::config::Settings; use crate::engine::timestamp::Timestamp; +use crate::engine::{self, eval_json_comparison}; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct AlertRule { @@ -27,7 +27,9 @@ pub struct AlertRule { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "lowercase")] pub enum AlertCondition { - Immediate { query: String }, + Immediate { + query: String, + }, Threshold { query: String, op: CompareOp, @@ -89,7 +91,12 @@ impl Settings { enabled: bool, condition: AlertCondition, ) -> Result { - let rule = AlertRule { id: generate_alert_rule_id(&name), name, enabled, condition }; + let rule = AlertRule { + id: generate_alert_rule_id(&name), + name, + enabled, + condition, + }; self.alert_rules.push(rule.clone()); self.save(settings_path)?; Ok(rule) @@ -147,36 +154,56 @@ fn backfill_window( window_ms: i64, ) -> RateWindow { let since_ms = now.as_millis() - window_ms; - let matching_timestamps = - snapshot.logs_since(before_sequence, since_ms, now.as_millis()).filter_map( - |(sequence, timestamp, text)| { - (eval_alert_query(query, sequence, timestamp, text) == Ok(true)).then_some(timestamp) - }, - ); + let matching_timestamps = snapshot + .logs_since(before_sequence, since_ms, now.as_millis()) + .filter_map(|(sequence, timestamp, text)| { + (eval_alert_query(query, sequence, timestamp, text) == Ok(true)).then_some(timestamp) + }); RateWindow::backfill(matching_timestamps, now, window_ms) } -fn eval_alert_query(query: &str, sequence: u64, timestamp: Timestamp, text: &str) -> Result { +fn eval_alert_query( + query: &str, + sequence: u64, + timestamp: Timestamp, + text: &str, +) -> Result { let expr = filter_parser::parse(query).map_err(|err| err.to_string())?; eval_alert_expr(&expr, sequence, timestamp, text) } -fn eval_alert_expr(expr: &FilterExpr, sequence: u64, timestamp: Timestamp, text: &str) -> Result { +fn eval_alert_expr( + expr: &FilterExpr, + sequence: u64, + timestamp: Timestamp, + text: &str, +) -> Result { match expr { FilterExpr::Regex(pattern) => { - let re = Regex::new(pattern).map_err(|err| format!("invalid regex in alert query: {err}"))?; + let re = Regex::new(pattern) + .map_err(|err| format!("invalid regex in alert query: {err}"))?; Ok(re.is_match(text)) } - FilterExpr::Comparison { metric, param: None, op, value } if metric == "ottyl_seq" => { + FilterExpr::Comparison { + metric, + param: None, + op, + value, + } if metric == "ottyl_seq" => { let target = match value { FilterValue::Number(n) => *n, - FilterValue::Raw(raw) => raw - .parse::() - .map_err(|_| format!("'ottyl_seq' comparisons require a numeric value, got {raw:?}"))?, + FilterValue::Raw(raw) => raw.parse::().map_err(|_| { + format!("'ottyl_seq' comparisons require a numeric value, got {raw:?}") + })?, }; Ok(op.matches((sequence as f64).total_cmp(&target))) } - FilterExpr::Comparison { metric, param: None, op, value } if metric == "ottyl_ts" => { + FilterExpr::Comparison { + metric, + param: None, + op, + value, + } if metric == "ottyl_ts" => { let target_ms = match value { FilterValue::Number(n) => *n as i64, FilterValue::Raw(raw) => filter_datetime::parse_datetime(raw, Timestamp::now()) @@ -185,7 +212,12 @@ fn eval_alert_expr(expr: &FilterExpr, sequence: u64, timestamp: Timestamp, text: }; Ok(op.matches(timestamp.as_millis().cmp(&target_ms))) } - FilterExpr::Comparison { metric, param: None, op, value } if metric == "ottyl_fmt" => { + FilterExpr::Comparison { + metric, + param: None, + op, + value, + } if metric == "ottyl_fmt" => { let target = match value { FilterValue::Number(n) => n.to_string(), FilterValue::Raw(raw) => raw.clone(), @@ -199,12 +231,10 @@ fn eval_alert_expr(expr: &FilterExpr, sequence: u64, timestamp: Timestamp, text: FilterExpr::JsonComparison { field, op, value } => { Ok(eval_json_comparison(text, field, *op, value)) } - FilterExpr::And(lhs, rhs) => { - Ok(eval_alert_expr(lhs, sequence, timestamp, text)? && eval_alert_expr(rhs, sequence, timestamp, text)?) - } - FilterExpr::Or(lhs, rhs) => { - Ok(eval_alert_expr(lhs, sequence, timestamp, text)? || eval_alert_expr(rhs, sequence, timestamp, text)?) - } + FilterExpr::And(lhs, rhs) => Ok(eval_alert_expr(lhs, sequence, timestamp, text)? + && eval_alert_expr(rhs, sequence, timestamp, text)?), + FilterExpr::Or(lhs, rhs) => Ok(eval_alert_expr(lhs, sequence, timestamp, text)? + || eval_alert_expr(rhs, sequence, timestamp, text)?), } } @@ -222,9 +252,15 @@ pub fn evaluate_alerts( continue; } let state = states.entry(rule.id.clone()).or_default(); - if let Some(message) = - evaluate_condition(&rule.condition, state, snapshot, sequence, timestamp, text, &rule.name) - { + if let Some(message) = evaluate_condition( + &rule.condition, + state, + snapshot, + sequence, + timestamp, + text, + &rule.name, + ) { fired.push((rule.clone(), message)); } } @@ -241,17 +277,25 @@ fn evaluate_condition( rule_name: &str, ) -> Option { match condition { - AlertCondition::Immediate { query } => match eval_alert_query(query, sequence, timestamp, text) { - Ok(true) => Some(format!("{rule_name}: matched line: {text}")), - Ok(false) => None, - Err(err) => { - eprintln!("warning: alert '{rule_name}' query error: {err}"); - None + AlertCondition::Immediate { query } => { + match eval_alert_query(query, sequence, timestamp, text) { + Ok(true) => Some(format!("{rule_name}: matched line: {text}")), + Ok(false) => None, + Err(err) => { + eprintln!("warning: alert '{rule_name}' query error: {err}"); + None + } } - }, - AlertCondition::Threshold { query, op, threshold, window_ms } => { + } + AlertCondition::Threshold { + query, + op, + threshold, + window_ms, + } => { if !state.threshold_backfilled { - state.threshold_window = backfill_window(snapshot, query, sequence, timestamp, *window_ms); + state.threshold_window = + backfill_window(snapshot, query, sequence, timestamp, *window_ms); state.threshold_backfilled = true; } match eval_alert_query(query, sequence, timestamp, text) { @@ -269,7 +313,10 @@ fn evaluate_condition( } } } - AlertCondition::NewValue { field_regex, capture_group } => { + AlertCondition::NewValue { + field_regex, + capture_group, + } => { let re = match Regex::new(field_regex) { Ok(re) => re, Err(err) => { @@ -284,7 +331,13 @@ fn evaluate_condition( .insert(value.clone()) .then(|| format!("{rule_name}: new value seen: {value} (line: {text})")) } - AlertCondition::Ratio { query_a, query_b, op, ratio, window_ms } => { + AlertCondition::Ratio { + query_a, + query_b, + op, + ratio, + window_ms, + } => { let matched_a = eval_alert_query(query_a, sequence, timestamp, text); let matched_b = eval_alert_query(query_b, sequence, timestamp, text); let (matched_a, matched_b) = match (matched_a, matched_b) { @@ -298,11 +351,13 @@ fn evaluate_condition( return None; } if !state.ratio_a_backfilled { - state.ratio_a_window = backfill_window(snapshot, query_a, sequence, timestamp, *window_ms); + state.ratio_a_window = + backfill_window(snapshot, query_a, sequence, timestamp, *window_ms); state.ratio_a_backfilled = true; } if !state.ratio_b_backfilled { - state.ratio_b_window = backfill_window(snapshot, query_b, sequence, timestamp, *window_ms); + state.ratio_b_window = + backfill_window(snapshot, query_b, sequence, timestamp, *window_ms); state.ratio_b_backfilled = true; } if matched_a { @@ -326,7 +381,12 @@ fn evaluate_condition( } } -pub async fn deliver_for_rule(rule_name: String, message: String, topic: Option, server: Option) { +pub async fn deliver_for_rule( + rule_name: String, + message: String, + topic: Option, + server: Option, +) { let Some(topic) = topic else { return; }; @@ -364,7 +424,9 @@ mod tests { id: "r1".to_string(), name: "test rule".to_string(), enabled: true, - condition: AlertCondition::Immediate { query: query.to_string() }, + condition: AlertCondition::Immediate { + query: query.to_string(), + }, } } @@ -379,7 +441,9 @@ mod tests { &settings_path, "spike".to_string(), true, - AlertCondition::Immediate { query: r#"regex("error")"#.to_string() }, + AlertCondition::Immediate { + query: r#"regex("error")"#.to_string(), + }, ) .unwrap(); assert_eq!(settings.alert_rules.len(), 1); @@ -393,14 +457,18 @@ mod tests { &created.id, "renamed".to_string(), false, - AlertCondition::Immediate { query: r#"regex("warn")"#.to_string() }, + AlertCondition::Immediate { + query: r#"regex("warn")"#.to_string(), + }, ) .unwrap(); assert_eq!(updated.id, created.id); assert_eq!(updated.name, "renamed"); assert!(!updated.enabled); - settings.remove_alert_rule(&settings_path, &created.id).unwrap(); + settings + .remove_alert_rule(&settings_path, &created.id) + .unwrap(); assert!(settings.alert_rules.is_empty()); let reloaded = Settings::load(&settings_path); assert!(reloaded.alert_rules.is_empty()); @@ -417,7 +485,9 @@ mod tests { "does-not-exist", "x".to_string(), true, - AlertCondition::Immediate { query: r#"regex("x")"#.to_string() }, + AlertCondition::Immediate { + query: r#"regex("x")"#.to_string(), + }, ); assert!(matches!(result, Err(AlertError::NotFound))); @@ -466,12 +536,36 @@ mod tests { let mut states = HashMap::new(); let index = engine::EngineSnapshot::default(); for (seq, ms) in [(0u64, 0i64), (1, 1000)] { - let fired = evaluate_alerts(&[rule.clone()], &mut states, &index, seq, Timestamp::from_millis(ms), "error"); - assert!(fired.is_empty(), "should not fire before threshold is crossed"); + let fired = evaluate_alerts( + std::slice::from_ref(&rule), + &mut states, + &index, + seq, + Timestamp::from_millis(ms), + "error", + ); + assert!( + fired.is_empty(), + "should not fire before threshold is crossed" + ); } - let fired = evaluate_alerts(&[rule.clone()], &mut states, &index, 2, Timestamp::from_millis(2000), "error"); + let fired = evaluate_alerts( + std::slice::from_ref(&rule), + &mut states, + &index, + 2, + Timestamp::from_millis(2000), + "error", + ); assert_eq!(fired.len(), 1); - let fired = evaluate_alerts(&[rule], &mut states, &index, 3, Timestamp::from_millis(2100), "fine"); + let fired = evaluate_alerts( + &[rule], + &mut states, + &index, + 3, + Timestamp::from_millis(2100), + "fine", + ); assert!(fired.is_empty()); } @@ -490,8 +584,28 @@ mod tests { }; let mut states = HashMap::new(); let index = engine::EngineSnapshot::default(); - assert!(evaluate_alerts(&[rule.clone()], &mut states, &index, 0, Timestamp::from_millis(0), "error").is_empty()); - assert!(evaluate_alerts(&[rule], &mut states, &index, 1, Timestamp::from_millis(5_000), "error").is_empty()); + assert!( + evaluate_alerts( + std::slice::from_ref(&rule), + &mut states, + &index, + 0, + Timestamp::from_millis(0), + "error" + ) + .is_empty() + ); + assert!( + evaluate_alerts( + &[rule], + &mut states, + &index, + 1, + Timestamp::from_millis(5_000), + "error" + ) + .is_empty() + ); } #[test] @@ -507,11 +621,32 @@ mod tests { }; let mut states = HashMap::new(); let index = engine::EngineSnapshot::default(); - let fired = evaluate_alerts(&[rule.clone()], &mut states, &index, 0, Timestamp::from_millis(0), "agent=curl"); + let fired = evaluate_alerts( + std::slice::from_ref(&rule), + &mut states, + &index, + 0, + Timestamp::from_millis(0), + "agent=curl", + ); assert_eq!(fired.len(), 1); - let fired = evaluate_alerts(&[rule.clone()], &mut states, &index, 1, Timestamp::from_millis(1), "agent=curl"); + let fired = evaluate_alerts( + std::slice::from_ref(&rule), + &mut states, + &index, + 1, + Timestamp::from_millis(1), + "agent=curl", + ); assert!(fired.is_empty()); - let fired = evaluate_alerts(&[rule], &mut states, &index, 2, Timestamp::from_millis(2), "agent=wget"); + let fired = evaluate_alerts( + &[rule], + &mut states, + &index, + 2, + Timestamp::from_millis(2), + "agent=wget", + ); assert_eq!(fired.len(), 1); } @@ -531,12 +666,43 @@ mod tests { }; let mut states = HashMap::new(); let index = engine::EngineSnapshot::default(); - assert!(evaluate_alerts(&[rule.clone()], &mut states, &index, 0, Timestamp::from_millis(0), "error").is_empty()); - let fired = evaluate_alerts(&[rule.clone()], &mut states, &index, 1, Timestamp::from_millis(1), "request"); + assert!( + evaluate_alerts( + std::slice::from_ref(&rule), + &mut states, + &index, + 0, + Timestamp::from_millis(0), + "error" + ) + .is_empty() + ); + let fired = evaluate_alerts( + std::slice::from_ref(&rule), + &mut states, + &index, + 1, + Timestamp::from_millis(1), + "request", + ); assert_eq!(fired.len(), 1); - let fired = evaluate_alerts(&[rule.clone()], &mut states, &index, 2, Timestamp::from_millis(2), "request"); + let fired = evaluate_alerts( + std::slice::from_ref(&rule), + &mut states, + &index, + 2, + Timestamp::from_millis(2), + "request", + ); assert_eq!(fired.len(), 1); - let fired = evaluate_alerts(&[rule], &mut states, &index, 3, Timestamp::from_millis(3), "request"); + let fired = evaluate_alerts( + &[rule], + &mut states, + &index, + 3, + Timestamp::from_millis(3), + "request", + ); assert!(fired.is_empty()); } @@ -547,7 +713,7 @@ mod tests { let index = engine::EngineSnapshot::default(); let fired = evaluate_alerts( - &[rule.clone()], + std::slice::from_ref(&rule), &mut states, &index, 0, @@ -556,7 +722,14 @@ mod tests { ); assert_eq!(fired.len(), 1); - let fired = evaluate_alerts(&[rule], &mut states, &index, 1, Timestamp::from_millis(1), "plain text"); + let fired = evaluate_alerts( + &[rule], + &mut states, + &index, + 1, + Timestamp::from_millis(1), + "plain text", + ); assert!(fired.is_empty()); } @@ -565,15 +738,22 @@ mod tests { let rule = immediate_rule("some_metric[param]==5"); let mut states = HashMap::new(); let index = engine::EngineSnapshot::default(); - let fired = evaluate_alerts(&[rule], &mut states, &index, 0, Timestamp::from_millis(0), "anything"); + let fired = evaluate_alerts( + &[rule], + &mut states, + &index, + 0, + Timestamp::from_millis(0), + "anything", + ); assert!(fired.is_empty()); } #[tokio::test] async fn deliver_ntfy_posts_message_to_server_topic_path() { + use axum::Router; use axum::extract::State; use axum::routing::post; - use axum::Router; use std::sync::{Arc, Mutex}; use tokio::net::TcpListener; @@ -590,7 +770,9 @@ mod tests { } let captured = Captured::default(); - let app = Router::new().route("/{topic}", post(capture)).with_state(captured.clone()); + let app = Router::new() + .route("/{topic}", post(capture)) + .with_state(captured.clone()); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); tokio::spawn(async move { @@ -598,7 +780,9 @@ mod tests { }); let server = format!("http://{addr}"); - deliver_ntfy(&server, "alerts", "hello world").await.unwrap(); + deliver_ntfy(&server, "alerts", "hello world") + .await + .unwrap(); let (topic, body) = captured.0.lock().unwrap().clone().unwrap(); assert_eq!(topic, "alerts"); diff --git a/src/engine/data_value.rs b/src/engine/data_value.rs index 3af2af9..539812c 100644 --- a/src/engine/data_value.rs +++ b/src/engine/data_value.rs @@ -39,7 +39,8 @@ pub fn extract_json_field(text: &str, field_path: &str) -> Option { } pub fn flatten_json_object(text: &str) -> Vec<(String, String)> { - let Ok(serde_json::Value::Object(root)) = serde_json::from_str::(text) else { + let Ok(serde_json::Value::Object(root)) = serde_json::from_str::(text) + else { return Vec::new(); }; let mut results = Vec::new(); @@ -53,7 +54,11 @@ fn flatten_json_object_into( results: &mut Vec<(String, String)>, ) { for (key, value) in object { - let path = if prefix.is_empty() { key.clone() } else { format!("{prefix}.{key}") }; + let path = if prefix.is_empty() { + key.clone() + } else { + format!("{prefix}.{key}") + }; match value { serde_json::Value::String(s) => results.push((path, s.clone())), serde_json::Value::Number(n) => results.push((path, n.to_string())), @@ -78,7 +83,9 @@ pub enum MetricPrimitiveKind { #[allow(dead_code)] #[derive(Debug, Clone, Serialize)] pub enum DataValue { - LogLiteral { text: String }, + LogLiteral { + text: String, + }, ParamDeclaration(ParamDeclaration), MetricDeclaration { message_match_regex: String, diff --git a/src/engine/filter_datetime.rs b/src/engine/filter_datetime.rs index 0236a8c..904496a 100644 --- a/src/engine/filter_datetime.rs +++ b/src/engine/filter_datetime.rs @@ -58,10 +58,7 @@ fn try_relative(input: &str, now: Timestamp) -> Option Option Option> { +fn try_time_of_day(input: &str, now: Timestamp) -> Option> { if !input.contains(':') { return None; } @@ -129,9 +123,14 @@ fn try_time_of_day( }; let now_millis = now.as_millis(); - let now_dt = Utc.timestamp_millis_opt(now_millis).single().unwrap_or_else(|| { - Utc.timestamp_millis_opt(0).single().expect("epoch is valid") - }); + let now_dt = Utc + .timestamp_millis_opt(now_millis) + .single() + .unwrap_or_else(|| { + Utc.timestamp_millis_opt(0) + .single() + .expect("epoch is valid") + }); let today = now_dt.date_naive(); let dt = Utc.from_utc_datetime(&today.and_time(time)); Some(Ok(timestamp_from_datetime(dt))) diff --git a/src/engine/filter_parser.rs b/src/engine/filter_parser.rs index d9734b4..7a47e26 100644 --- a/src/engine/filter_parser.rs +++ b/src/engine/filter_parser.rs @@ -88,9 +88,7 @@ pub fn parse(input: &str) -> Result { fn build_or_expr(pair: Pair) -> Result { debug_assert_eq!(pair.as_rule(), Rule::or_expr); - let mut and_exprs = pair - .into_inner() - .filter(|p| p.as_rule() != Rule::or_kw); + let mut and_exprs = pair.into_inner().filter(|p| p.as_rule() != Rule::or_kw); let first = and_exprs .next() .ok_or_else(|| FilterParseError::new("malformed 'or' expression"))?; @@ -104,9 +102,7 @@ fn build_or_expr(pair: Pair) -> Result { fn build_and_expr(pair: Pair) -> Result { debug_assert_eq!(pair.as_rule(), Rule::and_expr); - let mut terms = pair - .into_inner() - .filter(|p| p.as_rule() != Rule::and_kw); + let mut terms = pair.into_inner().filter(|p| p.as_rule() != Rule::and_kw); let first = terms .next() .ok_or_else(|| FilterParseError::new("malformed 'and' expression"))?; @@ -182,9 +178,9 @@ fn build_comparison(pair: Pair) -> Result { fn build_json_comparison(pair: Pair) -> Result { let mut inner = pair.into_inner(); - let field_pair = inner - .next() - .ok_or_else(|| FilterParseError::new("malformed json[...] comparison: missing field name"))?; + let field_pair = inner.next().ok_or_else(|| { + FilterParseError::new("malformed json[...] comparison: missing field name") + })?; let field = field_pair.as_str().to_string(); let op_pair = inner @@ -355,8 +351,8 @@ mod tests { #[test] fn parses_and_or_with_and_binding_tighter() { - let expr = parse(r#"regex("error") and ottyl_seq<5 or ottyl_ts>12:00"#) - .expect("should parse"); + let expr = + parse(r#"regex("error") and ottyl_seq<5 or ottyl_ts>12:00"#).expect("should parse"); let expected = FilterExpr::Or( Box::new(FilterExpr::And( @@ -381,8 +377,8 @@ mod tests { #[test] fn parenthesized_expression_overrides_precedence() { - let expr = parse(r#"regex("error") and (ottyl_seq<5 or ottyl_ts>12:00)"#) - .expect("should parse"); + let expr = + parse(r#"regex("error") and (ottyl_seq<5 or ottyl_ts>12:00)"#).expect("should parse"); let expected = FilterExpr::And( Box::new(FilterExpr::Regex("error".to_string())), diff --git a/src/engine/handle.rs b/src/engine/handle.rs index 0ea552e..3070835 100644 --- a/src/engine/handle.rs +++ b/src/engine/handle.rs @@ -110,7 +110,11 @@ fn new_channel_and_handle( let reader = IndexReader { factory: read.factory(), }; - (EngineHandle { tx, reader }, rx, IndexWriter { handle: write }) + ( + EngineHandle { tx, reader }, + rx, + IndexWriter { handle: write }, + ) } pub fn spawn(settings: SettingsHandle, channel_capacity: usize) -> EngineHandle { diff --git a/src/engine/ingest.rs b/src/engine/ingest.rs index c01edd7..4da4459 100644 --- a/src/engine/ingest.rs +++ b/src/engine/ingest.rs @@ -2,8 +2,8 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use crate::consts::{METRICS_POLL_INTERVAL, STDIN_POLL_INTERVAL as POLL_INTERVAL}; -use crate::engine::{EngineHandle, EngineMessage, IngestedLine, MetricSample}; use crate::engine::timestamp::Timestamp; +use crate::engine::{EngineHandle, EngineMessage, IngestedLine, MetricSample}; use tokio::fs::File; use tokio::io::{AsyncBufReadExt, BufReader}; @@ -67,15 +67,12 @@ pub async fn run_file(engine: EngineHandle, path: PathBuf) { let mut chunk = String::new(); match reader.read_line(&mut chunk).await { Ok(0) => { - match tokio::fs::metadata(&path).await { - Ok(meta) => { - let new_id = file_id(&meta); - if new_id != id || meta.len() < offset { - continue 'reopen; - } - id = new_id; + if let Ok(meta) = tokio::fs::metadata(&path).await { + let new_id = file_id(&meta); + if new_id != id || meta.len() < offset { + continue 'reopen; } - Err(_) => {} + id = new_id; } tokio::time::sleep(POLL_INTERVAL).await; } @@ -83,9 +80,7 @@ pub async fn run_file(engine: EngineHandle, path: PathBuf) { offset += n as u64; pending.push_str(&chunk); if pending.ends_with('\n') { - let text = pending - .trim_end_matches(['\n', '\r']) - .to_string(); + let text = pending.trim_end_matches(['\n', '\r']).to_string(); pending.clear(); let line = IngestedLine { @@ -128,7 +123,12 @@ pub async fn run_metrics_poll(engine: EngineHandle, url: String) { Ok(body) => { let now = Timestamp::now(); for sample in parse_openmetrics(&body, now) { - if engine.tx.send(EngineMessage::IngestMetricSample(sample)).await.is_err() { + if engine + .tx + .send(EngineMessage::IngestMetricSample(sample)) + .await + .is_err() + { return; } } @@ -140,7 +140,9 @@ pub async fn run_metrics_poll(engine: EngineHandle, url: String) { } fn parse_openmetrics(body: &str, now: Timestamp) -> Vec { - body.lines().filter_map(|line| parse_openmetrics_line(line, now)).collect() + body.lines() + .filter_map(|line| parse_openmetrics_line(line, now)) + .collect() } fn parse_openmetrics_line(line: &str, now: Timestamp) -> Option { @@ -161,7 +163,10 @@ fn parse_openmetrics_line(line: &str, now: Timestamp) -> Option { let (metric_name, labels) = match name_and_labels.find('{') { Some(open) => { let close = name_and_labels.rfind('}')?; - (name_and_labels[..open].to_string(), parse_openmetrics_labels(&name_and_labels[open + 1..close])) + ( + name_and_labels[..open].to_string(), + parse_openmetrics_labels(&name_and_labels[open + 1..close]), + ) } None => (name_and_labels.to_string(), BTreeMap::new()), }; @@ -169,13 +174,23 @@ fn parse_openmetrics_line(line: &str, now: Timestamp) -> Option { return None; } - Some(MetricSample { metric_name, labels, value, timestamp }) + Some(MetricSample { + metric_name, + labels, + value, + timestamp, + }) } fn parse_openmetrics_labels(raw: &str) -> BTreeMap { raw.split(',') .filter_map(|pair| pair.split_once('=')) - .map(|(name, value)| (name.trim().to_string(), value.trim().trim_matches('"').to_string())) + .map(|(name, value)| { + ( + name.trim().to_string(), + value.trim().trim_matches('"').to_string(), + ) + }) .collect() } @@ -197,10 +212,8 @@ mod openmetrics_tests { #[test] fn parses_labels_and_explicit_timestamp() { let now = Timestamp::from_millis(1000); - let samples = parse_openmetrics( - r#"http_requests_total{method="GET",path="/a"} 7 5000"#, - now, - ); + let samples = + parse_openmetrics(r#"http_requests_total{method="GET",path="/a"} 7 5000"#, now); assert_eq!(samples.len(), 1); let sample = &samples[0]; assert_eq!(sample.metric_name, "http_requests_total"); @@ -223,9 +236,9 @@ mod openmetrics_tests { #[cfg(test)] mod tests { use super::*; - use std::time::Duration; - use std::io::Write; use crate::engine::test_handle; + use std::io::Write; + use std::time::Duration; use tokio::sync::mpsc; async fn recv_text(rx: &mut mpsc::Receiver) -> String { @@ -250,7 +263,10 @@ mod tests { assert_eq!(recv_text(&mut rx).await, "first"); - let mut f = std::fs::OpenOptions::new().append(true).open(&path).unwrap(); + let mut f = std::fs::OpenOptions::new() + .append(true) + .open(&path) + .unwrap(); writeln!(f, "second").unwrap(); assert_eq!(recv_text(&mut rx).await, "second"); } diff --git a/src/engine/logfmt.rs b/src/engine/logfmt.rs index ca77876..3e681e3 100644 --- a/src/engine/logfmt.rs +++ b/src/engine/logfmt.rs @@ -1,9 +1,9 @@ +use nom::IResult; use nom::branch::alt; use nom::bytes::complete::{escaped, tag, take_until, take_while1}; use nom::character::complete::{char, none_of, one_of}; use nom::error::{Error, ErrorKind}; use nom::sequence::delimited; -use nom::IResult; use regex::Regex; use std::sync::LazyLock; @@ -16,7 +16,10 @@ fn key(input: &str) -> IResult<&str, &str> { fn quoted_value(input: &str) -> IResult<&str, &str> { delimited( char('"'), - alt((escaped(none_of("\\\""), '\\', one_of("\"\\nt")), nom::combinator::success(""))), + alt(( + escaped(none_of("\\\""), '\\', one_of("\"\\nt")), + nom::combinator::success(""), + )), char('"'), )(input) } @@ -85,7 +88,8 @@ fn is_separator_run(run: &str) -> bool { run.len() >= 2 || run.contains('\t') } -static WHITESPACE_RUN: LazyLock = LazyLock::new(|| Regex::new(r"[ \t]+").expect("static regex must compile")); +static WHITESPACE_RUN: LazyLock = + LazyLock::new(|| Regex::new(r"[ \t]+").expect("static regex must compile")); fn split_columns_with_offsets(line: &str, max_columns: Option) -> Vec<(usize, &str)> { let mut columns = Vec::new(); @@ -107,7 +111,10 @@ fn split_columns_with_offsets(line: &str, max_columns: Option) -> Vec<(us } fn split_columns(line: &str, max_columns: Option) -> Vec<&str> { - split_columns_with_offsets(line, max_columns).into_iter().map(|(_, s)| s).collect() + split_columns_with_offsets(line, max_columns) + .into_iter() + .map(|(_, s)| s) + .collect() } static COLUMN_NAME: LazyLock = @@ -151,13 +158,21 @@ impl LogStreamHeader { } let leading_ws = slice.len() - slice.trim_start().len(); let start = line[..byte_start + leading_ws].chars().count(); - columns.push(LogStreamColumn { name: trimmed.to_string(), start }); + columns.push(LogStreamColumn { + name: trimmed.to_string(), + start, + }); } if columns.len() < 2 || !columns.iter().all(|c| COLUMN_NAME.is_match(&c.name)) { return None; } - let tab_delimited = WHITESPACE_RUN.find_iter(line).any(|m| m.as_str().contains('\t')); - Some(LogStreamHeader { columns, tab_delimited }) + let tab_delimited = WHITESPACE_RUN + .find_iter(line) + .any(|m| m.as_str().contains('\t')); + Some(LogStreamHeader { + columns, + tab_delimited, + }) } pub fn split_row<'a>(&self, line: &'a str) -> Vec<&'a str> { @@ -210,7 +225,9 @@ impl LogStreamHeader { return pattern; } let start = self.columns.last().map(|c| c.start).unwrap_or(0); - format!("^.{{{start}}}{COLUMN_VALUE_PATTERN}{COLUMN_SEPARATOR_PATTERN}(?P<{capture_group}>.*)$") + format!( + "^.{{{start}}}{COLUMN_VALUE_PATTERN}{COLUMN_SEPARATOR_PATTERN}(?P<{capture_group}>.*)$" + ) } } @@ -240,7 +257,9 @@ fn py_log_prefix(input: &str) -> IResult<&str, (&str, &str, &str, &str)> { let (input, level) = take_while1(|c: char| c.is_ascii_uppercase())(input)?; let (input, _) = char(' ')(input)?; let (input, logger) = - take_while1(|c: char| c.is_ascii_alphanumeric() || c == '_' || c == '-' || c == '.')(input)?; + take_while1(|c: char| c.is_ascii_alphanumeric() || c == '_' || c == '-' || c == '.')( + input, + )?; let (input, _) = char(' ')(input)?; Ok((input, (date, time, level, logger))) } @@ -323,7 +342,10 @@ mod tests { #[test] fn detects_key_value_lines_as_logfmt() { - assert_eq!(detect_format(r#"ts=1788648105773 msg="entity_moved" x1=-4"#), "logfmt"); + assert_eq!( + detect_format(r#"ts=1788648105773 msg="entity_moved" x1=-4"#), + "logfmt" + ); } #[test] @@ -371,7 +393,10 @@ mod tests { fn detects_log_stream_header_columns_generically() { let header_line = LOG_STREAM_FIXTURE.lines().next().unwrap(); let header = LogStreamHeader::detect(header_line).expect("fixture header must be detected"); - assert_eq!(column_names(&header), vec!["Timestamp", "Thread", "Type", "Activity", "PID", "TTL"]); + assert_eq!( + column_names(&header), + vec!["Timestamp", "Thread", "Type", "Activity", "PID", "TTL"] + ); assert!(!header.tab_delimited); } @@ -400,7 +425,10 @@ mod tests { assert_eq!(row[3], "0x0"); assert_eq!(row[4], "606"); assert_eq!(row[5], "0"); - assert_eq!(row[6], "WindowServer: (HSTouchHIDService) something happened"); + assert_eq!( + row[6], + "WindowServer: (HSTouchHIDService) something happened" + ); } #[test] @@ -426,32 +454,57 @@ mod tests { let line = "2026-09-06 01:48:54.518117+0100 0x4dfed2 Default 0x0 606 0 WindowServer: (HSTouchHIDService) something happened"; let type_re = Regex::new(&header.column_pattern(2, "Type")).unwrap(); - assert_eq!(type_re.captures(line).unwrap().name("Type").unwrap().as_str(), "Default"); + assert_eq!( + type_re + .captures(line) + .unwrap() + .name("Type") + .unwrap() + .as_str(), + "Default" + ); let pid_re = Regex::new(&header.column_pattern(4, "PID")).unwrap(); - assert_eq!(pid_re.captures(line).unwrap().name("PID").unwrap().as_str(), "606"); + assert_eq!( + pid_re.captures(line).unwrap().name("PID").unwrap().as_str(), + "606" + ); let message_re = Regex::new(&header.message_pattern("message")).unwrap(); assert_eq!( - message_re.captures(line).unwrap().name("message").unwrap().as_str(), + message_re + .captures(line) + .unwrap() + .name("message") + .unwrap() + .as_str(), "WindowServer: (HSTouchHIDService) something happened" ); let timestamp_re = Regex::new(&header.column_pattern(0, "Timestamp")).unwrap(); assert_eq!( - timestamp_re.captures(line).unwrap().name("Timestamp").unwrap().as_str(), + timestamp_re + .captures(line) + .unwrap() + .name("Timestamp") + .unwrap() + .as_str(), "2026-09-06 01:48:54.518117+0100" ); } #[test] - fn is_log_stream_line_rejects_non_columnar_lines_that_a_fixed_offset_column_regex_would_still_match() { + fn is_log_stream_line_rejects_non_columnar_lines_that_a_fixed_offset_column_regex_would_still_match() + { let header_line = LOG_STREAM_FIXTURE.lines().next().unwrap(); let header = LogStreamHeader::detect(header_line).unwrap(); let timestamp_re = Regex::new(&header.column_pattern(0, "Timestamp")).unwrap(); let continuation_line = "so_gencnt: 4042707 so_state: 0x0"; - assert!(timestamp_re.captures(continuation_line).is_some(), "sanity: regex alone still matches garbage"); + assert!( + timestamp_re.captures(continuation_line).is_some(), + "sanity: regex alone still matches garbage" + ); assert!(!is_log_stream_line(continuation_line)); assert!(!is_log_stream_line(header_line)); assert!(!is_log_stream_line("}")); @@ -473,7 +526,10 @@ mod tests { #[test] fn detects_go_log_shaped_lines() { for line in GO_LOG_FIXTURE.lines() { - assert!(is_go_log_line(line), "expected go-log line to match: {line}"); + assert!( + is_go_log_line(line), + "expected go-log line to match: {line}" + ); assert_eq!(detect_format(line), "go-log"); } } @@ -481,7 +537,10 @@ mod tests { #[test] fn detects_py_log_shaped_lines() { for line in PY_LOG_FIXTURE.lines() { - assert!(is_py_log_line(line), "expected py-log line to match: {line}"); + assert!( + is_py_log_line(line), + "expected py-log line to match: {line}" + ); assert_eq!(detect_format(line), "py-log"); } } @@ -514,7 +573,8 @@ mod tests { #[test] fn py_log_prefix_extracts_date_time_level_and_logger() { let (rest, (date, time, level, logger)) = - py_log_prefix("2026-09-06 10:35:55,217 INFO delete-py-log event=eventC labelC=1").unwrap(); + py_log_prefix("2026-09-06 10:35:55,217 INFO delete-py-log event=eventC labelC=1") + .unwrap(); assert_eq!(date, "2026-09-06"); assert_eq!(time, "10:35:55,217"); assert_eq!(level, "INFO"); diff --git a/src/engine/message.rs b/src/engine/message.rs index 78ca3c7..d8cceed 100644 --- a/src/engine/message.rs +++ b/src/engine/message.rs @@ -4,7 +4,9 @@ use crate::api::dto::{ BucketValueDto, LogEntryDto, LogsParams, MetricDeclarationDto, ParamDeclarationDto, PromqlSeriesDto, StatsDto, TimelineBucketDto, TimelineParams, }; -use crate::engine::{ClearParamsResult, DeclareMetricRequest, DeclareParamRequest, IngestedLine, MetricSample}; +use crate::engine::{ + ClearParamsResult, DeclareMetricRequest, DeclareParamRequest, IngestedLine, MetricSample, +}; use std::collections::BTreeMap; use tokio::sync::oneshot; @@ -23,7 +25,11 @@ pub enum EngineQuery { since_sequence: Option, limit: Option, }, - Promql { query: String, start_ms: Option, end_ms: Option }, + Promql { + query: String, + start_ms: Option, + end_ms: Option, + }, } #[allow(dead_code)] diff --git a/src/engine/mod.rs b/src/engine/mod.rs index 7825885..e8b13c5 100644 --- a/src/engine/mod.rs +++ b/src/engine/mod.rs @@ -16,11 +16,11 @@ pub mod timestamp; #[cfg(test)] mod tests; -pub use handle::{spawn, EngineHandle}; #[cfg(test)] pub(crate) use handle::test_handle; +pub use handle::{EngineHandle, spawn}; pub use message::{EngineMessage, EngineQuery, EngineResponse}; -pub(crate) use snapshot::{eval_json_comparison, EngineSnapshot}; +pub(crate) use snapshot::{EngineSnapshot, eval_json_comparison}; use crate::engine::data_value::ParamDeclaration; use crate::engine::timestamp::Timestamp; diff --git a/src/engine/promql.rs b/src/engine/promql.rs index 6091619..029e613 100644 --- a/src/engine/promql.rs +++ b/src/engine/promql.rs @@ -38,7 +38,10 @@ impl PromqlQuery { let query = parser.parse_query()?; parser.skip_whitespace(); if !parser.is_at_end() { - return Err(format!("unexpected trailing input: '{}'", parser.remaining())); + return Err(format!( + "unexpected trailing input: '{}'", + parser.remaining() + )); } if query.agg == Some(AggOp::Rate) && query.range.is_none() { return Err("rate() requires a range selector, e.g. rate(my_metric[5m])".to_string()); @@ -113,7 +116,9 @@ impl<'a> Parser<'a> { fn parse_quoted_string(&mut self) -> Result { self.expect_char('"')?; let rest = self.remaining(); - let end = rest.find('"').ok_or("unterminated string literal (missing closing '\"')")?; + let end = rest + .find('"') + .ok_or("unterminated string literal (missing closing '\"')")?; let value = &rest[..end]; if value.contains('\\') { return Err("escape sequences in label values are not supported".to_string()); @@ -185,7 +190,9 @@ impl<'a> Parser<'a> { return Ok(Some(RangeSelector::BareWidth(digits as f64))); } - let unit = self.peek().ok_or("range selector is missing a duration unit or ']'")?; + let unit = self + .peek() + .ok_or("range selector is missing a duration unit or ']'")?; let seconds_per_unit: i64 = match unit { 's' => 1, 'm' => 60, @@ -272,7 +279,11 @@ impl<'a> Parser<'a> { _ => None, }; self.skip_whitespace(); - let prefix_by_labels = if agg.is_some() { self.parse_by_clause()? } else { Vec::new() }; + let prefix_by_labels = if agg.is_some() { + self.parse_by_clause()? + } else { + Vec::new() + }; self.skip_whitespace(); if agg.is_some() && self.peek() == Some('(') { self.expect_char('(')?; @@ -314,8 +325,11 @@ impl<'a> Parser<'a> { .to_string(), ); } - inner.by_labels = - if prefix_by_labels.is_empty() { trailing_by_labels } else { prefix_by_labels }; + inner.by_labels = if prefix_by_labels.is_empty() { + trailing_by_labels + } else { + prefix_by_labels + }; Ok(inner) } else { self.pos = checkpoint; @@ -343,7 +357,10 @@ mod tests { assert_eq!(query.metric_name, "requests"); assert_eq!( query.matchers, - vec![LabelMatcher { name: "path".to_string(), value: "/a".to_string() }] + vec![LabelMatcher { + name: "path".to_string(), + value: "/a".to_string() + }] ); } @@ -472,7 +489,13 @@ mod tests { fn parses_gauge_with_no_range_selector() { let query = PromqlQuery::parse(r#"gauge(connections{host="a"})"#).unwrap(); assert_eq!(query.agg, Some(AggOp::Gauge)); - assert_eq!(query.matchers, vec![LabelMatcher { name: "host".to_string(), value: "a".to_string() }]); + assert_eq!( + query.matchers, + vec![LabelMatcher { + name: "host".to_string(), + value: "a".to_string() + }] + ); } #[test] @@ -497,7 +520,13 @@ mod tests { let query = PromqlQuery::parse(r#"avg(rate(requests{path="/a"}[5m]))"#).unwrap(); assert_eq!(query.agg, Some(AggOp::Avg)); assert!(query.inner_rate); - assert_eq!(query.matchers, vec![LabelMatcher { name: "path".to_string(), value: "/a".to_string() }]); + assert_eq!( + query.matchers, + vec![LabelMatcher { + name: "path".to_string(), + value: "/a".to_string() + }] + ); } #[test] @@ -518,7 +547,10 @@ mod tests { fn parses_sum_of_rate_with_by_clause_multiple_labels() { let query = PromqlQuery::parse("sum(rate(requests[1m])) by (path, method)").unwrap(); assert!(query.inner_rate); - assert_eq!(query.by_labels, vec!["path".to_string(), "method".to_string()]); + assert_eq!( + query.by_labels, + vec!["path".to_string(), "method".to_string()] + ); } #[test] @@ -548,7 +580,10 @@ mod tests { #[test] fn parses_sum_with_prefix_by_clause_multiple_labels() { let query = PromqlQuery::parse("sum by (path, method) (requests)").unwrap(); - assert_eq!(query.by_labels, vec!["path".to_string(), "method".to_string()]); + assert_eq!( + query.by_labels, + vec!["path".to_string(), "method".to_string()] + ); } #[test] diff --git a/src/engine/rate.rs b/src/engine/rate.rs index 8f4c0a5..ece74ef 100644 --- a/src/engine/rate.rs +++ b/src/engine/rate.rs @@ -20,7 +20,10 @@ pub fn rate_at(points: &[(Timestamp, f64)], now: Timestamp, window_ms: i64) -> f } pub fn windowed_rate_series(points: &[(Timestamp, f64)], window_ms: i64) -> Vec<(Timestamp, f64)> { - points.iter().map(|(ts, _)| (*ts, rate_at(points, *ts, window_ms))).collect() + points + .iter() + .map(|(ts, _)| (*ts, rate_at(points, *ts, window_ms))) + .collect() } #[derive(Debug, Default, Clone)] @@ -29,7 +32,11 @@ pub struct RateWindow { } impl RateWindow { - pub fn backfill(timestamps: impl IntoIterator, now: Timestamp, window_ms: i64) -> Self { + pub fn backfill( + timestamps: impl IntoIterator, + now: Timestamp, + window_ms: i64, + ) -> Self { let mut window = Self::default(); for ts in timestamps { window.push(ts, window_ms); @@ -94,9 +101,14 @@ mod tests { for ms in millis { window.push(Timestamp::from_millis(ms), 1000); } - let points: Vec<(Timestamp, f64)> = - millis.iter().map(|ms| (Timestamp::from_millis(*ms), 1.0)).collect(); - assert_eq!(window.count(), count_in_window(&points, Timestamp::from_millis(1200), 1000)); + let points: Vec<(Timestamp, f64)> = millis + .iter() + .map(|ms| (Timestamp::from_millis(*ms), 1.0)) + .collect(); + assert_eq!( + window.count(), + count_in_window(&points, Timestamp::from_millis(1200), 1000) + ); } #[test] diff --git a/src/engine/snapshot/filter.rs b/src/engine/snapshot/filter.rs index d40ac04..50a4500 100644 --- a/src/engine/snapshot/filter.rs +++ b/src/engine/snapshot/filter.rs @@ -12,11 +12,24 @@ use std::collections::HashSet; pub(super) enum CompiledFilter { Regex(Regex), - Sequence { op: CompareOp, target: f64 }, - Timestamp { op: CompareOp, target_ms: i64 }, - Fmt { op: CompareOp, target: String }, + Sequence { + op: CompareOp, + target: f64, + }, + Timestamp { + op: CompareOp, + target_ms: i64, + }, + Fmt { + op: CompareOp, + target: String, + }, MetricMatch(HashSet), - Json { field: String, op: CompareOp, value: FilterValue }, + Json { + field: String, + op: CompareOp, + value: FilterValue, + }, And(Box, Box), Or(Box, Box), } @@ -32,7 +45,11 @@ impl CompiledFilter { Ok(CompiledFilter::Sequence { op, target }) } - pub(super) fn from_ts_comparison(op: CompareOp, value: &FilterValue, now: Timestamp) -> Result { + pub(super) fn from_ts_comparison( + op: CompareOp, + value: &FilterValue, + now: Timestamp, + ) -> Result { let target_ms = match value { FilterValue::Number(n) => *n as i64, FilterValue::Raw(raw) => filter_datetime::parse_datetime(raw, now) @@ -52,7 +69,12 @@ impl CompiledFilter { } impl MetricPrimitive { - pub(super) fn compare(&self, op: CompareOp, value: &FilterValue, now: Timestamp) -> Result { + pub(super) fn compare( + &self, + op: CompareOp, + value: &FilterValue, + now: Timestamp, + ) -> Result { let ordering = match (self, value) { (MetricPrimitive::Number(n), FilterValue::Number(v)) => n.as_f64().partial_cmp(v), (MetricPrimitive::Number(n), FilterValue::Raw(raw)) => { @@ -65,10 +87,13 @@ impl MetricPrimitive { ts.as_millis().partial_cmp(&(*v as i64)) } (MetricPrimitive::Timestamp(ts), FilterValue::Raw(raw)) => { - let target = filter_datetime::parse_datetime(raw, now).map_err(|err| err.to_string())?; + let target = + filter_datetime::parse_datetime(raw, now).map_err(|err| err.to_string())?; ts.as_millis().partial_cmp(&target.as_millis()) } - (MetricPrimitive::String(s), FilterValue::Raw(raw)) => s.as_str().partial_cmp(raw.as_str()), + (MetricPrimitive::String(s), FilterValue::Raw(raw)) => { + s.as_str().partial_cmp(raw.as_str()) + } (MetricPrimitive::String(s), FilterValue::Number(n)) => { s.as_str().partial_cmp(n.to_string().as_str()) } @@ -91,7 +116,12 @@ impl MetricPrimitive { } } -pub(crate) fn eval_json_comparison(text: &str, field: &str, op: CompareOp, value: &FilterValue) -> bool { +pub(crate) fn eval_json_comparison( + text: &str, + field: &str, + op: CompareOp, + value: &FilterValue, +) -> bool { let Ok(parsed) = serde_json::from_str::(text) else { return false; }; @@ -136,7 +166,9 @@ fn compare_json_value(field_value: &serde_json::Value, op: CompareOp, value: &Fi Err(_) => false, } } - serde_json::Value::Null | serde_json::Value::Array(_) | serde_json::Value::Object(_) => false, + serde_json::Value::Null | serde_json::Value::Array(_) | serde_json::Value::Object(_) => { + false + } } } @@ -167,7 +199,9 @@ impl CompiledFilter { op.matches(crate::engine::logfmt::detect_format(text).cmp(target.as_str())) } CompiledFilter::MetricMatch(sequences) => sequences.contains(&sequence), - CompiledFilter::Json { field, op, value } => eval_json_comparison(text, field, *op, value), + CompiledFilter::Json { field, op, value } => { + eval_json_comparison(text, field, *op, value) + } CompiledFilter::And(lhs, rhs) => { lhs.eval(sequence, timestamp, text) && rhs.eval(sequence, timestamp, text) } diff --git a/src/engine/snapshot/metrics.rs b/src/engine/snapshot/metrics.rs index 63b712d..0ccfd8d 100644 --- a/src/engine/snapshot/metrics.rs +++ b/src/engine/snapshot/metrics.rs @@ -6,7 +6,10 @@ use crate::engine::data_value::MetricPrimitiveKind; use crate::engine::metric_primitive::{MetricPrimitive, Number}; use crate::engine::timestamp::Timestamp; -pub(super) fn parse_metric_primitive(raw: &str, kind: MetricPrimitiveKind) -> Option { +pub(super) fn parse_metric_primitive( + raw: &str, + kind: MetricPrimitiveKind, +) -> Option { match kind { MetricPrimitiveKind::String => Some(MetricPrimitive::String(raw.to_string())), MetricPrimitiveKind::Number => raw @@ -24,4 +27,3 @@ pub(super) fn parse_metric_primitive(raw: &str, kind: MetricPrimitiveKind) -> Op MetricPrimitiveKind::Boolean => raw.parse::().ok().map(MetricPrimitive::Boolean), } } - diff --git a/src/engine/snapshot/mod.rs b/src/engine/snapshot/mod.rs index e9b4edc..0b2f21e 100644 --- a/src/engine/snapshot/mod.rs +++ b/src/engine/snapshot/mod.rs @@ -18,7 +18,6 @@ use crate::consts::{ MAX_AUTO_DECLARED_PARAMS, MAX_NEW_AUTO_PARAMS_PER_LINE, OTTYL_RESERVED_PARAM_NAMES, SORT_HELPER_INDEX_PURGE_AFTER, }; -use crate::flavour::LogLiteralHelper; use crate::engine::data_value::{DataValue, MetricPrimitiveKind, ParamDeclaration}; use crate::engine::filter_parser::{self, CompareOp, FilterExpr, FilterValue}; use crate::engine::index_key::IndexKey; @@ -27,17 +26,39 @@ use crate::engine::metric_primitive::{MetricPrimitive, Number}; use crate::engine::promql::{AggOp, PromqlQuery}; use crate::engine::timestamp::Timestamp; use crate::engine::{DeclareMetricRequest, DeclareParamRequest}; +use crate::flavour::LogLiteralHelper; use filter::CompiledFilter; use metrics::parse_metric_primitive; -use params::{build_auto_param_pattern, extract_param_raw, infer_kind, sanitize_group_name, AutoParamCandidate, ResolvedParam}; +use params::{ + AutoParamCandidate, ResolvedParam, build_auto_param_pattern, extract_param_raw, infer_kind, + sanitize_group_name, +}; use query::{ coerce_to_f64, handle_query_inner, matches_label_matchers, merge_series_locf, merge_series_windowed, ottyl_reserved_numeric_value_fn, project_labels, windowed_rate_series, }; use regex::Regex; -use sort::{derive_single_param, CompiledParamDeclaration, SortHelperAccess, SortHelperIndexEntry, SortHelperKey}; +use sort::{ + CompiledParamDeclaration, SortHelperAccess, SortHelperIndexEntry, SortHelperKey, + derive_single_param, +}; use std::collections::{BTreeMap, HashMap, HashSet}; +/// One row of a metric declaration: name, message-match regex, label param names, value param +/// name, and the label-name-to-value-param and static-label maps. +type MetricDeclarationRow<'a> = ( + &'a str, + &'a str, + &'a [String], + Option<&'a str>, + Option<&'a BTreeMap>, + Option<&'a BTreeMap>, +); + +/// Per-label-group point series, keyed by the group's label set, before merging into one +/// `PromqlSeriesDto` per group. +type GroupedPointSeries = BTreeMap, Vec>>; + #[derive(Debug, Clone, Default)] pub(crate) struct EngineSnapshot { pub(crate) index: BTreeMap, @@ -58,16 +79,7 @@ pub(crate) struct EngineSnapshot { impl EngineSnapshot { pub(crate) fn metric_declarations_range( &self, - ) -> impl Iterator< - Item = ( - &str, - &str, - &[String], - Option<&str>, - Option<&BTreeMap>, - Option<&BTreeMap>, - ), - > { + ) -> impl Iterator> { self.index .range(IndexKey::ParamMetricSplit..IndexKey::MetricBucketSplit) .filter_map(|(key, value)| match (key, value) { @@ -94,7 +106,18 @@ impl EngineSnapshot { pub(crate) fn param_declarations_range( &self, - ) -> impl Iterator, MetricPrimitiveKind, bool, bool, bool)> { + ) -> impl Iterator< + Item = ( + &str, + &str, + &str, + Option<&str>, + MetricPrimitiveKind, + bool, + bool, + bool, + ), + > { self.index .range(IndexKey::LogParamSplit..IndexKey::ParamMetricSplit) .filter_map(|(key, value)| match (key, value) { @@ -125,7 +148,10 @@ impl EngineSnapshot { } pub(crate) fn from_index(index: BTreeMap) -> Self { - let mut snapshot = EngineSnapshot { index, ..Default::default() }; + let mut snapshot = EngineSnapshot { + index, + ..Default::default() + }; snapshot.rebuild_regex_caches(); snapshot } @@ -138,13 +164,17 @@ impl EngineSnapshot { self.param_regex_cache = self .param_declarations_range() .filter_map(|(name, message_match_regex, ..)| { - regex::Regex::new(message_match_regex).ok().map(|re| (name.to_string(), re)) + regex::Regex::new(message_match_regex) + .ok() + .map(|re| (name.to_string(), re)) }) .collect(); self.metric_regex_cache = self .metric_declarations_range() .filter_map(|(name, message_match_regex, ..)| { - regex::Regex::new(message_match_regex).ok().map(|re| (name.to_string(), re)) + regex::Regex::new(message_match_regex) + .ok() + .map(|re| (name.to_string(), re)) }) .collect(); } @@ -161,7 +191,9 @@ impl EngineSnapshot { break; }; if let Some(DataValue::LogLiteral { text }) = self.index.get(&oldest_key) { - self.approx_bytes_total = self.approx_bytes_total.saturating_sub(text.approx_log_bytes()); + self.approx_bytes_total = self + .approx_bytes_total + .saturating_sub(text.approx_log_bytes()); } self.index.remove(&oldest_key); self.log_count -= 1; @@ -182,29 +214,39 @@ impl EngineSnapshot { sequence_range: impl std::ops::RangeBounds, ) -> impl Iterator { let lower = match sequence_range.start_bound() { - std::ops::Bound::Included(&sequence) => { - IndexKey::IngestedLog { sequence, timestamp: Timestamp::MIN } - } - std::ops::Bound::Excluded(&sequence) => { - IndexKey::IngestedLog { sequence: sequence.saturating_add(1), timestamp: Timestamp::MIN } - } + std::ops::Bound::Included(&sequence) => IndexKey::IngestedLog { + sequence, + timestamp: Timestamp::MIN, + }, + std::ops::Bound::Excluded(&sequence) => IndexKey::IngestedLog { + sequence: sequence.saturating_add(1), + timestamp: Timestamp::MIN, + }, std::ops::Bound::Unbounded => IndexKey::Start, }; let upper = match sequence_range.end_bound() { - std::ops::Bound::Included(&sequence) => { - IndexKey::IngestedLog { sequence: sequence.saturating_add(1), timestamp: Timestamp::MIN } - } - std::ops::Bound::Excluded(&sequence) => { - IndexKey::IngestedLog { sequence, timestamp: Timestamp::MIN } - } + std::ops::Bound::Included(&sequence) => IndexKey::IngestedLog { + sequence: sequence.saturating_add(1), + timestamp: Timestamp::MIN, + }, + std::ops::Bound::Excluded(&sequence) => IndexKey::IngestedLog { + sequence, + timestamp: Timestamp::MIN, + }, std::ops::Bound::Unbounded => IndexKey::LogParamSplit, }; - self.index.range(lower..upper).filter_map(|(key, value)| match (key, value) { - (IndexKey::IngestedLog { sequence, timestamp }, DataValue::LogLiteral { text }) => { - Some((*sequence, *timestamp, text.as_str())) - } - _ => None, - }) + self.index + .range(lower..upper) + .filter_map(|(key, value)| match (key, value) { + ( + IndexKey::IngestedLog { + sequence, + timestamp, + }, + DataValue::LogLiteral { text }, + ) => Some((*sequence, *timestamp, text.as_str())), + _ => None, + }) } pub(crate) fn logs_since( @@ -213,15 +255,20 @@ impl EngineSnapshot { since_ms: i64, now_ms: i64, ) -> impl Iterator { - self.logs_in_sequence_range(..before_sequence).filter(move |(_, timestamp, _)| { - timestamp.as_millis() > since_ms && timestamp.as_millis() <= now_ms - }) + self.logs_in_sequence_range(..before_sequence) + .filter(move |(_, timestamp, _)| { + timestamp.as_millis() > since_ms && timestamp.as_millis() <= now_ms + }) } } // --- filter: compiling search-bar filter expressions --- impl EngineSnapshot { - pub(in crate::engine::snapshot) fn compile_filter_query(&self, filter: &str, now: Timestamp) -> Result { + pub(in crate::engine::snapshot) fn compile_filter_query( + &self, + filter: &str, + now: Timestamp, + ) -> Result { let expr = filter_parser::parse(filter).map_err(|err| err.to_string())?; self.compile_filter(&expr, now) } @@ -231,7 +278,12 @@ impl EngineSnapshot { FilterExpr::Regex(pattern) => Regex::new(pattern) .map(CompiledFilter::Regex) .map_err(|err| format!("invalid regex in filter: {err}")), - FilterExpr::Comparison { metric, param, op, value } => match metric.as_str() { + FilterExpr::Comparison { + metric, + param, + op, + value, + } => match metric.as_str() { "ottyl_seq" => CompiledFilter::from_seq_comparison(*op, value), "ottyl_ts" => CompiledFilter::from_ts_comparison(*op, value, now), "ottyl_fmt" => Ok(CompiledFilter::from_fmt_comparison(*op, value)), @@ -277,14 +329,15 @@ impl EngineSnapshot { })?; if let Some(DataValue::MetricDeclaration { label_params, .. }) = - self.index.get(&IndexKey::MetricDeclaration { name: metric.to_string() }) + self.index.get(&IndexKey::MetricDeclaration { + name: metric.to_string(), + }) + && !label_params.iter().any(|p| p == param) { - if !label_params.iter().any(|p| p == param) { - return Err(format!( - "param '{param}' is not a label param of metric '{metric}'; declared label params: [{}]", - label_params.join(", ") - )); - } + return Err(format!( + "param '{param}' is not a label param of metric '{metric}'; declared label params: [{}]", + label_params.join(", ") + )); } let mut matching = HashSet::new(); @@ -302,12 +355,18 @@ impl EngineSnapshot { // --- metrics: declaring, applying, and querying metric buckets --- impl EngineSnapshot { - pub(crate) fn validate_metric_request(&self, request: &DeclareMetricRequest) -> Result { + pub(crate) fn validate_metric_request( + &self, + request: &DeclareMetricRequest, + ) -> Result { let re = Regex::new(&request.message_match_regex) .map_err(|err| format!("invalid regex: {err}"))?; - let declared = - |name: &str| self.index.contains_key(&IndexKey::ParamDeclaration { name: name.to_string() }); + let declared = |name: &str| { + self.index.contains_key(&IndexKey::ParamDeclaration { + name: name.to_string(), + }) + }; let mut errors: Vec = request .label_params @@ -315,10 +374,12 @@ impl EngineSnapshot { .filter(|name| !declared(name)) .map(|name| format!("label param '{name}' references an undeclared param")) .collect(); - if let Some(value_param) = &request.value_param { - if !declared(value_param) { - errors.push(format!("value param '{value_param}' references an undeclared param")); - } + if let Some(value_param) = &request.value_param + && !declared(value_param) + { + errors.push(format!( + "value param '{value_param}' references an undeclared param" + )); } if !errors.is_empty() { return Err(errors.join("; ")); @@ -329,7 +390,8 @@ impl EngineSnapshot { pub(crate) fn declare_metric(&mut self, request: DeclareMetricRequest) -> Result<(), String> { let re = self.validate_metric_request(&request)?; - self.metric_regex_cache.insert(request.name.clone(), re.clone()); + self.metric_regex_cache + .insert(request.name.clone(), re.clone()); self.resolve_backfill_and_store_metric(request, &re); Ok(()) } @@ -349,7 +411,9 @@ impl EngineSnapshot { request.unmatched_labels.as_ref(), ); self.index.insert( - IndexKey::MetricDeclaration { name: request.name.clone() }, + IndexKey::MetricDeclaration { + name: request.name.clone(), + }, DataValue::MetricDeclaration { message_match_regex: request.message_match_regex, label_params: request.label_params, @@ -365,26 +429,37 @@ impl EngineSnapshot { existing_name: &str, request: &DeclareMetricRequest, ) -> Result { - if !self.index.contains_key(&IndexKey::MetricDeclaration { name: existing_name.to_string() }) { + if !self.index.contains_key(&IndexKey::MetricDeclaration { + name: existing_name.to_string(), + }) { return Err(format!("no metric declaration named '{existing_name}'")); } if request.name != existing_name - && self.index.contains_key(&IndexKey::MetricDeclaration { name: request.name.clone() }) + && self.index.contains_key(&IndexKey::MetricDeclaration { + name: request.name.clone(), + }) { return Err(format!("a metric named '{}' already exists", request.name)); } self.validate_metric_request(request) } - pub(crate) fn update_metric(&mut self, existing_name: &str, request: DeclareMetricRequest) -> Result<(), String> { + pub(crate) fn update_metric( + &mut self, + existing_name: &str, + request: DeclareMetricRequest, + ) -> Result<(), String> { let re = self.validate_update_metric(existing_name, &request)?; self.clear_metric_data(existing_name); if request.name != existing_name { - self.index.remove(&IndexKey::MetricDeclaration { name: existing_name.to_string() }); + self.index.remove(&IndexKey::MetricDeclaration { + name: existing_name.to_string(), + }); self.metric_regex_cache.remove(existing_name); } - self.metric_regex_cache.insert(request.name.clone(), re.clone()); + self.metric_regex_cache + .insert(request.name.clone(), re.clone()); self.resolve_backfill_and_store_metric(request, &re); Ok(()) @@ -392,14 +467,23 @@ impl EngineSnapshot { fn clear_metric_data(&mut self, metric_name: &str) { self.index.retain(|key, _| match key { - IndexKey::Bucket { metric_name: name, .. } => name != metric_name, - IndexKey::LastMetricEvent { metric_name: name, .. } => name != metric_name, + IndexKey::Bucket { + metric_name: name, .. + } => name != metric_name, + IndexKey::LastMetricEvent { + metric_name: name, .. + } => name != metric_name, _ => true, }); } pub(crate) fn remove_metric_declaration(&mut self, name: &str) -> bool { - let existed = self.index.remove(&IndexKey::MetricDeclaration { name: name.to_string() }).is_some(); + let existed = self + .index + .remove(&IndexKey::MetricDeclaration { + name: name.to_string(), + }) + .is_some(); self.metric_regex_cache.remove(name); self.clear_metric_data(name); existed @@ -425,15 +509,25 @@ impl EngineSnapshot { .iter() .map(|(name, value)| (name.clone(), MetricPrimitive::String(value.clone()))) .collect(); - self.push_bucket_value(metric_name, labels, sequence, timestamp, MetricPrimitive::Number(Number::Int(1))); + self.push_bucket_value( + metric_name, + labels, + sequence, + timestamp, + MetricPrimitive::Number(Number::Int(1)), + ); } return; } let mut labels = BTreeMap::new(); for param_name in label_params { - let Some(resolved) = resolved_params.get(param_name) else { return }; - let Some(parsed) = self.resolve_param_value(metric_name, resolved, &labels, text, timestamp) else { + let Some(resolved) = resolved_params.get(param_name) else { + return; + }; + let Some(parsed) = + self.resolve_param_value(metric_name, resolved, &labels, text, timestamp) + else { return; }; labels.insert(param_name.clone(), parsed); @@ -447,7 +541,9 @@ impl EngineSnapshot { let value = match value_param { Some(value_param) => { - let Some(resolved) = resolved_params.get(value_param) else { return }; + let Some(resolved) = resolved_params.get(value_param) else { + return; + }; match self.resolve_param_value(metric_name, resolved, &labels, text, timestamp) { Some(parsed) => parsed, None => return, @@ -467,7 +563,10 @@ impl EngineSnapshot { timestamp: Timestamp, value: MetricPrimitive, ) { - let key = IndexKey::Bucket { metric_name: metric_name.to_string(), labels }; + let key = IndexKey::Bucket { + metric_name: metric_name.to_string(), + labels, + }; let entry = self .index .entry(key) @@ -518,15 +617,17 @@ impl EngineSnapshot { ); let metrics: Vec = self .metric_declarations_range() - .map(|(name, _, label_params, value_param, matched_labels, unmatched_labels)| { - ( - name.to_string(), - label_params.to_vec(), - value_param.map(|s| s.to_string()), - matched_labels.cloned(), - unmatched_labels.cloned(), - ) - }) + .map( + |(name, _, label_params, value_param, matched_labels, unmatched_labels)| { + ( + name.to_string(), + label_params.to_vec(), + value_param.map(|s| s.to_string()), + matched_labels.cloned(), + unmatched_labels.cloned(), + ) + }, + ) .collect(); let resolved_params = self.resolved_param_declarations(); for (name, label_params, value_param, matched_labels, unmatched_labels) in metrics { @@ -551,14 +652,21 @@ impl EngineSnapshot { &'a self, metric_name: &'a str, labels_filter: Option<&'a BTreeMap>, - ) -> impl Iterator, &'a (u64, Timestamp, MetricPrimitive))> + 'a - { + ) -> impl Iterator< + Item = ( + &'a BTreeMap, + &'a (u64, Timestamp, MetricPrimitive), + ), + > + 'a { // IndexKey::Bucket sorts by (metric_name, labels), so bound the range to just this // metric's buckets instead of ranging MetricBucketSplit..End over every declared // metric. '\u{10FFFF}' is the maximum Unicode scalar value, so `metric_name` followed // by it sorts after every real key for this metric (any labels) and before any key // for a lexicographically greater metric name. - let lower = IndexKey::Bucket { metric_name: metric_name.to_string(), labels: BTreeMap::new() }; + let lower = IndexKey::Bucket { + metric_name: metric_name.to_string(), + labels: BTreeMap::new(), + }; let upper = IndexKey::Bucket { metric_name: format!("{metric_name}\u{10FFFF}"), labels: BTreeMap::new(), @@ -566,9 +674,13 @@ impl EngineSnapshot { self.index .range(lower..upper) .filter_map(move |(key, value)| match (key, value) { - (IndexKey::Bucket { metric_name: name, labels }, DataValue::BucketValue { values }) - if name == metric_name && labels_filter.is_none_or(|filter| filter == labels) => - { + ( + IndexKey::Bucket { + metric_name: name, + labels, + }, + DataValue::BucketValue { values }, + ) if name == metric_name && labels_filter.is_none_or(|filter| filter == labels) => { Some(values.iter().map(move |entry| (labels, entry))) } _ => None, @@ -577,12 +689,15 @@ impl EngineSnapshot { } pub(crate) fn declared_value_kind(&self, metric_name: &str) -> Option { - match self.index.get(&IndexKey::MetricDeclaration { name: metric_name.to_string() }) { + match self.index.get(&IndexKey::MetricDeclaration { + name: metric_name.to_string(), + }) { Some(DataValue::MetricDeclaration { value_param, .. }) => Some( value_param .as_ref() .and_then(|name| { - self.index.get(&IndexKey::ParamDeclaration { name: name.clone() }) + self.index + .get(&IndexKey::ParamDeclaration { name: name.clone() }) }) .map(|value| match value { DataValue::ParamDeclaration(param) => param.kind, @@ -597,7 +712,10 @@ impl EngineSnapshot { // --- params: declaring and resolving param values --- impl EngineSnapshot { - pub(crate) fn validate_param_declaration(&self, request: &DeclareParamRequest) -> Result<(), String> { + pub(crate) fn validate_param_declaration( + &self, + request: &DeclareParamRequest, + ) -> Result<(), String> { if OTTYL_RESERVED_PARAM_NAMES.contains(&request.name.as_str()) { return Err(format!( "'{}' is a reserved name, present on every log line without declaring anything; \ @@ -606,25 +724,32 @@ impl EngineSnapshot { )); } if let Some(DataValue::ParamDeclaration(existing)) = - self.index.get(&IndexKey::ParamDeclaration { name: request.name.clone() }) + self.index.get(&IndexKey::ParamDeclaration { + name: request.name.clone(), + }) + && existing.auto_derived { - if existing.auto_derived { - return Err(format!( - "param '{}' was auto-detected from the log stream and can't be edited", - request.name - )); - } + return Err(format!( + "param '{}' was auto-detected from the log stream and can't be edited", + request.name + )); } let re = Regex::new(&request.message_match_regex) .map_err(|err| format!("invalid regex: {err}"))?; if request.elapsed_since_last && request.kind != MetricPrimitiveKind::Duration { return Err("elapsed_since_last params must declare kind 'duration'".to_string()); } - let uses_json_field = request.json_field_path.as_deref().is_some_and(|path| !path.is_empty()); + let uses_json_field = request + .json_field_path + .as_deref() + .is_some_and(|path| !path.is_empty()); if request.kind != MetricPrimitiveKind::Boolean && !request.elapsed_since_last && !uses_json_field - && !re.capture_names().flatten().any(|name| name == request.capture_group) + && !re + .capture_names() + .flatten() + .any(|name| name == request.capture_group) { return Err(format!( "regex has no capture group named '{}'", @@ -639,13 +764,20 @@ impl EngineSnapshot { self.param_regex_cache.insert(request.name.clone(), re); } self.index.insert( - IndexKey::ParamDeclaration { name: request.name.clone() }, - DataValue::ParamDeclaration(ParamDeclaration { auto_derived: false, ..request }), + IndexKey::ParamDeclaration { + name: request.name.clone(), + }, + DataValue::ParamDeclaration(ParamDeclaration { + auto_derived: false, + ..request + }), ); } fn auto_derived_param_count(&self) -> usize { - self.param_declarations_range().filter(|declaration| declaration.6).count() + self.param_declarations_range() + .filter(|declaration| declaration.6) + .count() } pub(crate) fn auto_declare_params(&mut self, text: &str) { @@ -653,32 +785,33 @@ impl EngineSnapshot { if total >= MAX_AUTO_DECLARED_PARAMS { return; } - let candidates: Vec = if crate::engine::logfmt::detect_format(text) == "json" { - crate::engine::data_value::flatten_json_object(text) - .into_iter() - .map(|(path, value)| AutoParamCandidate { - name: path.clone(), - value, - message_match_regex: String::new(), - capture_group: String::new(), - json_field_path: Some(path), - }) - .collect() - } else { - crate::engine::logfmt::extract_pairs(text) - .into_iter() - .map(|(key, value, quoted)| { - let group = sanitize_group_name(key); - AutoParamCandidate { - name: key.to_string(), - value: value.to_string(), - message_match_regex: build_auto_param_pattern(key, &group, quoted), - capture_group: group, - json_field_path: None, - } - }) - .collect() - }; + let candidates: Vec = + if crate::engine::logfmt::detect_format(text) == "json" { + crate::engine::data_value::flatten_json_object(text) + .into_iter() + .map(|(path, value)| AutoParamCandidate { + name: path.clone(), + value, + message_match_regex: String::new(), + capture_group: String::new(), + json_field_path: Some(path), + }) + .collect() + } else { + crate::engine::logfmt::extract_pairs(text) + .into_iter() + .map(|(key, value, quoted)| { + let group = sanitize_group_name(key); + AutoParamCandidate { + name: key.to_string(), + value: value.to_string(), + message_match_regex: build_auto_param_pattern(key, &group, quoted), + capture_group: group, + json_field_path: None, + } + }) + .collect() + }; let mut new_this_line = 0; for candidate in candidates { @@ -694,7 +827,9 @@ impl EngineSnapshot { /// Inserts `candidate` as an auto-derived `ParamDeclaration` unless a param with that name /// is already declared, bumping `*total` and returning whether it inserted. fn try_insert_auto_param(&mut self, candidate: AutoParamCandidate, total: &mut usize) -> bool { - let index_key = IndexKey::ParamDeclaration { name: candidate.name.clone() }; + let index_key = IndexKey::ParamDeclaration { + name: candidate.name.clone(), + }; if self.index.contains_key(&index_key) { return false; } @@ -730,7 +865,9 @@ impl EngineSnapshot { for (i, column) in header.columns.iter().enumerate() { let Some(value) = row.get(i) else { continue }; - let index_key = IndexKey::ParamDeclaration { name: column.name.clone() }; + let index_key = IndexKey::ParamDeclaration { + name: column.name.clone(), + }; if self.index.contains_key(&index_key) { continue; } @@ -753,8 +890,13 @@ impl EngineSnapshot { ); } - if let Some(message) = row.get(header.columns.len()).filter(|m| !m.trim().is_empty()) { - let index_key = IndexKey::ParamDeclaration { name: "message".to_string() }; + if let Some(message) = row + .get(header.columns.len()) + .filter(|m| !m.trim().is_empty()) + { + let index_key = IndexKey::ParamDeclaration { + name: "message".to_string(), + }; if !self.index.contains_key(&index_key) { let message_match_regex = header.message_pattern("message"); if let Ok(re) = Regex::new(&message_match_regex) { @@ -778,18 +920,24 @@ impl EngineSnapshot { } pub(crate) fn clear_all_param_declarations(&mut self) -> usize { - let names: Vec = - self.param_declarations_range().map(|(name, ..)| name.to_string()).collect(); + let names: Vec = self + .param_declarations_range() + .map(|(name, ..)| name.to_string()) + .collect(); let removed = names.len(); for name in names { - self.index.remove(&IndexKey::ParamDeclaration { name: name.clone() }); + self.index + .remove(&IndexKey::ParamDeclaration { name: name.clone() }); self.param_regex_cache.remove(&name); } removed } pub(crate) fn reextract_params_from_retained_logs(&mut self) -> (usize, usize) { - let texts: Vec = self.logs_range().map(|(_, _, text)| text.to_string()).collect(); + let texts: Vec = self + .logs_range() + .map(|(_, _, text)| text.to_string()) + .collect(); let logs_considered = texts.len(); for text in &texts { self.auto_declare_params(text); @@ -798,23 +946,36 @@ impl EngineSnapshot { (reextracted, logs_considered) } - pub(in crate::engine::snapshot) fn resolved_param_declarations(&self) -> HashMap { + pub(in crate::engine::snapshot) fn resolved_param_declarations( + &self, + ) -> HashMap { self.param_declarations_range() - .filter_map(|(name, _, capture_group, json_field_path, kind, elapsed_since_last, _, log_stream_gated)| { - self.param_regex_cache.get(name).map(|regex| { - ( - name.to_string(), - ResolvedParam { - regex: regex.clone(), - capture_group: capture_group.to_string(), - json_field_path: json_field_path.map(|path| path.to_string()), - kind, - elapsed_since_last, - log_stream_gated, - }, - ) - }) - }) + .filter_map( + |( + name, + _, + capture_group, + json_field_path, + kind, + elapsed_since_last, + _, + log_stream_gated, + )| { + self.param_regex_cache.get(name).map(|regex| { + ( + name.to_string(), + ResolvedParam { + regex: regex.clone(), + capture_group: capture_group.to_string(), + json_field_path: json_field_path.map(|path| path.to_string()), + kind, + elapsed_since_last, + log_stream_gated, + }, + ) + }) + }, + ) .collect() } @@ -832,12 +993,16 @@ impl EngineSnapshot { labels: labels_so_far.clone(), }; let previous_timestamp = match self.index.get(&last_event_key) { - Some(DataValue::LastEventTimestamp { timestamp: previous }) => Some(*previous), + Some(DataValue::LastEventTimestamp { + timestamp: previous, + }) => Some(*previous), _ => None, }; - self.index.insert(last_event_key, DataValue::LastEventTimestamp { timestamp }); - return previous_timestamp - .map(|previous| MetricPrimitive::Duration(timestamp.as_millis() - previous.as_millis())); + self.index + .insert(last_event_key, DataValue::LastEventTimestamp { timestamp }); + return previous_timestamp.map(|previous| { + MetricPrimitive::Duration(timestamp.as_millis() - previous.as_millis()) + }); } let raw = extract_param_raw( @@ -854,7 +1019,11 @@ impl EngineSnapshot { // --- query: EngineQuery/PromQL evaluation --- impl EngineSnapshot { pub(crate) fn handle_query_ro(&self, query: EngineQuery) -> EngineResponse { - handle_query_inner(self, query, SortHelperAccess::Immutable(&self.sort_helper_indexes)) + handle_query_inner( + self, + query, + SortHelperAccess::Immutable(&self.sort_helper_indexes), + ) } pub(crate) fn has_sort_helper_index(&self, sort_by: &str) -> bool { @@ -877,8 +1046,13 @@ impl EngineSnapshot { } if ottyl_reserved_numeric_value_fn(&parsed.metric_name).is_none() - && !self.index.contains_key(&IndexKey::MetricDeclaration { name: parsed.metric_name.clone() }) - && self.bucket_values(&parsed.metric_name, None).next().is_none() + && !self.index.contains_key(&IndexKey::MetricDeclaration { + name: parsed.metric_name.clone(), + }) + && self + .bucket_values(&parsed.metric_name, None) + .next() + .is_none() { return Err(format!( "unknown metric \"{}\": no declaration or recorded data for it (was it deleted?)", @@ -895,7 +1069,8 @@ impl EngineSnapshot { )); } - let mut series_points: BTreeMap, Vec<(i64, f64)>> = BTreeMap::new(); + let mut series_points: BTreeMap, Vec<(i64, f64)>> = + BTreeMap::new(); if let Some(reserved_value) = ottyl_reserved_numeric_value_fn(&parsed.metric_name) { let no_labels: BTreeMap = BTreeMap::new(); if matches_label_matchers(&no_labels, &parsed.matchers) { @@ -912,15 +1087,18 @@ impl EngineSnapshot { } } } else { - for (labels, (_, timestamp, value)) in self.bucket_values(&parsed.metric_name, None) + for (labels, (_, timestamp, value)) in self + .bucket_values(&parsed.metric_name, None) .filter(|(labels, _)| matches_label_matchers(labels, &parsed.matchers)) .filter(|(_, (_, timestamp, _))| { start_ms.is_none_or(|start| timestamp.as_millis() >= start) && end_ms.is_none_or(|end| timestamp.as_millis() <= end) }) { - let display_labels: BTreeMap = - labels.iter().map(|(name, value)| (name.clone(), value.to_string())).collect(); + let display_labels: BTreeMap = labels + .iter() + .map(|(name, value)| (name.clone(), value.to_string())) + .collect(); series_points .entry(display_labels) .or_default() @@ -937,10 +1115,12 @@ impl EngineSnapshot { .map(|(labels, points)| PromqlSeriesDto { labels, points }) .collect(), Some(AggOp::Count) => { - let mut groups: BTreeMap, Vec>> = BTreeMap::new(); + let mut groups: GroupedPointSeries = BTreeMap::new(); for (labels, points) in series_points { - let count_points: Vec<(i64, f64)> = - points.iter().map(|(timestamp_ms, _)| (*timestamp_ms, 1.0)).collect(); + let count_points: Vec<(i64, f64)> = points + .iter() + .map(|(timestamp_ms, _)| (*timestamp_ms, 1.0)) + .collect(); let group_key = project_labels(&labels, &parsed.by_labels); groups.entry(group_key).or_default().push(count_points); } @@ -948,17 +1128,23 @@ impl EngineSnapshot { .into_iter() .map(|(group_labels, sub_series)| { let points = merge_series_windowed(&sub_series, false); - PromqlSeriesDto { labels: group_labels, points } + PromqlSeriesDto { + labels: group_labels, + points, + } }) .collect() } Some(AggOp::Sum) | Some(AggOp::Avg) => { let is_avg = parsed.agg == Some(AggOp::Avg); - let rate_range_ms = parsed - .inner_rate - .then(|| parsed.range.expect("inner rate() always carries a range").as_millis() as i64); + let rate_range_ms = parsed.inner_rate.then(|| { + parsed + .range + .expect("inner rate() always carries a range") + .as_millis() as i64 + }); - let mut groups: BTreeMap, Vec>> = BTreeMap::new(); + let mut groups: GroupedPointSeries = BTreeMap::new(); for (labels, points) in series_points { let points = match rate_range_ms { Some(range_ms) => windowed_rate_series(&points, range_ms, start_ms, end_ms), @@ -976,12 +1162,18 @@ impl EngineSnapshot { } else { merge_series_windowed(&sub_series, is_avg) }; - PromqlSeriesDto { labels: group_labels, points } + PromqlSeriesDto { + labels: group_labels, + points, + } }) .collect() } Some(AggOp::Rate) => { - let range_ms = parsed.range.expect("rate() always carries a range").as_millis() as i64; + let range_ms = parsed + .range + .expect("rate() always carries a range") + .as_millis() as i64; series_points .into_iter() .map(|(labels, points)| PromqlSeriesDto { @@ -990,24 +1182,27 @@ impl EngineSnapshot { }) .collect() } - Some(AggOp::Gauge) => { - series_points - .into_iter() - .filter_map(|(labels, points)| { - let last = *points.last()?; - Some(PromqlSeriesDto { labels, points: vec![last] }) + Some(AggOp::Gauge) => series_points + .into_iter() + .filter_map(|(labels, points)| { + let last = *points.last()?; + Some(PromqlSeriesDto { + labels, + points: vec![last], }) - .collect() - } + }) + .collect(), Some(AggOp::Histogram) => { - let bucket_width = - parsed.bucket_width.expect("histogram() always carries a bucket width"); + let bucket_width = parsed + .bucket_width + .expect("histogram() always carries a bucket width"); let last_ts = series_points .values() .flat_map(|points| points.iter().map(|(ts, _)| *ts)) .max() .unwrap_or(0); - let mut groups: BTreeMap, Vec<&(i64, f64)>> = BTreeMap::new(); + let mut groups: BTreeMap, Vec<&(i64, f64)>> = + BTreeMap::new(); for (labels, points) in &series_points { let group_key = project_labels(labels, &parsed.by_labels); groups.entry(group_key).or_default().extend(points.iter()); @@ -1025,7 +1220,10 @@ impl EngineSnapshot { let upper_bound = (bucket_index + 1) as f64 * bucket_width; let mut labels = group_labels.clone(); labels.insert("le".to_string(), upper_bound.to_string()); - PromqlSeriesDto { labels, points: vec![(last_ts, count as f64)] } + PromqlSeriesDto { + labels, + points: vec![(last_ts, count as f64)], + } }) }) .collect() @@ -1036,7 +1234,10 @@ impl EngineSnapshot { // --- sort: sort-helper index maintenance --- impl EngineSnapshot { - pub(super) fn build_sort_helper_index(&self, sort_by: &str) -> BTreeMap { + pub(super) fn build_sort_helper_index( + &self, + sort_by: &str, + ) -> BTreeMap { let compiled_params = self.compiled_param_declarations(); self.logs_range() .map(|(sequence, timestamp, text)| { @@ -1049,7 +1250,12 @@ impl EngineSnapshot { .collect() } - pub(crate) fn update_sort_helper_indexes_on_ingest(&mut self, sequence: u64, timestamp: Timestamp, text: &str) { + pub(crate) fn update_sort_helper_indexes_on_ingest( + &mut self, + sequence: u64, + timestamp: Timestamp, + text: &str, + ) { if self.sort_helper_indexes.is_empty() { return; } @@ -1073,28 +1279,36 @@ impl EngineSnapshot { let built = self.build_sort_helper_index(sort_by); self.sort_helper_indexes.insert( sort_by.to_string(), - SortHelperIndexEntry { index: built, last_accessed: std::time::Instant::now() }, + SortHelperIndexEntry { + index: built, + last_accessed: std::time::Instant::now(), + }, ); } - let entry = self.sort_helper_indexes.get_mut(sort_by).expect("just inserted above if missing"); + let entry = self + .sort_helper_indexes + .get_mut(sort_by) + .expect("just inserted above if missing"); entry.last_accessed = std::time::Instant::now(); } pub(crate) fn compiled_param_declarations(&self) -> Vec { self.param_declarations_range() .filter(|(_, _, _, _, _, elapsed_since_last, _, _)| !elapsed_since_last) - .filter_map(|(name, _, capture_group, json_field_path, kind, _, _, log_stream_gated)| { - self.param_regex_cache.get(name).map(|re| { - ( - name.to_string(), - re.clone(), - capture_group.to_string(), - json_field_path.map(|path| path.to_string()), - kind, - log_stream_gated, - ) - }) - }) + .filter_map( + |(name, _, capture_group, json_field_path, kind, _, _, log_stream_gated)| { + self.param_regex_cache.get(name).map(|re| { + ( + name.to_string(), + re.clone(), + capture_group.to_string(), + json_field_path.map(|path| path.to_string()), + kind, + log_stream_gated, + ) + }) + }, + ) .collect() } } diff --git a/src/engine/snapshot/params.rs b/src/engine/snapshot/params.rs index c92b380..9adc9b9 100644 --- a/src/engine/snapshot/params.rs +++ b/src/engine/snapshot/params.rs @@ -1,23 +1,34 @@ //! Param declaration: validating and storing explicit declarations, and the auto-detection //! logic that derives declarations from logfmt/json/log-stream-shaped lines as they're ingested. +#[cfg(test)] +use crate::engine::DeclareParamRequest; use crate::engine::data_value::MetricPrimitiveKind; #[cfg(test)] use crate::engine::snapshot::EngineSnapshot; -#[cfg(test)] -use crate::engine::DeclareParamRequest; use regex::Regex; #[cfg(test)] -pub(crate) fn declare_param(snapshot: &mut EngineSnapshot, request: DeclareParamRequest) -> Result<(), String> { +pub(crate) fn declare_param( + snapshot: &mut EngineSnapshot, + request: DeclareParamRequest, +) -> Result<(), String> { snapshot.validate_param_declaration(&request)?; snapshot.insert_param_declaration(request); Ok(()) } pub(crate) fn sanitize_group_name(key: &str) -> String { - let mut group: String = - key.chars().map(|c| if c.is_ascii_alphanumeric() || c == '_' { c } else { '_' }).collect(); + let mut group: String = key + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == '_' { + c + } else { + '_' + } + }) + .collect(); if group.chars().next().is_none_or(|c| c.is_ascii_digit()) { group.insert(0, '_'); } @@ -41,7 +52,6 @@ pub(crate) fn build_auto_param_pattern(key: &str, group: &str, quoted: bool) -> } } - pub(crate) struct AutoParamCandidate { pub(crate) name: String, pub(crate) value: String, @@ -94,6 +104,9 @@ pub(crate) fn extract_param_raw( } match json_field_path.filter(|path| !path.is_empty()) { Some(field_path) => crate::engine::data_value::extract_json_field(text, field_path), - None => regex.captures(text)?.name(capture_group).map(|m| m.as_str().to_string()), + None => regex + .captures(text)? + .name(capture_group) + .map(|m| m.as_str().to_string()), } } diff --git a/src/engine/snapshot/query.rs b/src/engine/snapshot/query.rs index 08d0d45..757dcd9 100644 --- a/src/engine/snapshot/query.rs +++ b/src/engine/snapshot/query.rs @@ -2,8 +2,8 @@ //! stats, declaration listing, bucket-value listing, and PromQL evaluation. use crate::api::dto::{ - BucketValueDto, LogEntryDto, MetricDeclarationDto, ParamDeclarationDto, - StatsDto, TimelineBucketDto, + BucketValueDto, LogEntryDto, MetricDeclarationDto, ParamDeclarationDto, StatsDto, + TimelineBucketDto, }; use crate::consts::{DEFAULT_TIMELINE_BUCKET_MS, MERGE_WINDOW_MS}; use crate::engine::data_value::DataValue; @@ -12,17 +12,17 @@ use crate::engine::message::{EngineQuery, EngineResponse}; use crate::engine::metric_primitive::{MetricPrimitive, Number}; use crate::engine::promql::{LabelMatcher, PromqlQuery}; use crate::engine::rate; +use crate::engine::snapshot::EngineSnapshot; use crate::engine::snapshot::filter::CompiledFilter; #[cfg(test)] use crate::engine::snapshot::sort::SortHelperIndexEntry; use crate::engine::snapshot::sort::{ - log_entry_params, CompiledParamDeclaration, SortHelperAccess, SortHelperKey, + CompiledParamDeclaration, SortHelperAccess, SortHelperKey, log_entry_params, }; -use crate::engine::snapshot::EngineSnapshot; use crate::engine::timestamp::Timestamp; +use std::collections::BTreeMap; #[cfg(test)] use std::collections::HashMap; -use std::collections::BTreeMap; #[cfg(test)] pub(crate) fn handle_query( @@ -30,7 +30,11 @@ pub(crate) fn handle_query( query: EngineQuery, sort_helper_indexes: &mut HashMap, ) -> EngineResponse { - handle_query_inner(snapshot, query, SortHelperAccess::Mutable(sort_helper_indexes)) + handle_query_inner( + snapshot, + query, + SortHelperAccess::Mutable(sort_helper_indexes), + ) } /// Compiles `filter` against `snapshot`, or returns `Ok(None)` when there's no filter string @@ -41,7 +45,9 @@ fn compile_optional_filter( filter: Option<&str>, now: Timestamp, ) -> Result, String> { - filter.map(|filter| snapshot.compile_filter_query(filter, now)).transpose() + filter + .map(|filter| snapshot.compile_filter_query(filter, now)) + .transpose() } /// Walks the sort-helper index for `sort_by` — rather than `snapshot.logs_range()`, which is @@ -67,7 +73,9 @@ fn logs_sorted_by_param( let index = &snapshot.index; static EMPTY_HELPER: std::sync::LazyLock> = std::sync::LazyLock::new(BTreeMap::new); - let helper_index = sort_helper_access.resolve(index, sort_by).unwrap_or(&EMPTY_HELPER); + let helper_index = sort_helper_access + .resolve(index, sort_by) + .unwrap_or(&EMPTY_HELPER); let mut entries = Vec::new(); for (helper_key, timestamp) in helper_index.iter() { @@ -86,9 +94,10 @@ fn logs_sorted_by_param( { continue; } - let Some(DataValue::LogLiteral { text }) = - index.get(&IndexKey::IngestedLog { sequence, timestamp: *timestamp }) - else { + let Some(DataValue::LogLiteral { text }) = index.get(&IndexKey::IngestedLog { + sequence, + timestamp: *timestamp, + }) else { continue; }; if !compiled_filter.is_none_or(|filter| filter.eval(sequence, *timestamp, text)) { @@ -117,14 +126,17 @@ pub(super) fn handle_query_inner( match query { EngineQuery::Logs(params) => { let compiled_filter = - match compile_optional_filter(snapshot, params.filter.as_deref(), Timestamp::now()) { + match compile_optional_filter(snapshot, params.filter.as_deref(), Timestamp::now()) + { Ok(compiled) => compiled, Err(err) => return EngineResponse::Error(err), }; - if let Some(sort_by) = params.sort_by.as_deref() { - if !snapshot.param_declarations_range().any(|(name, ..)| name == sort_by) { - return EngineResponse::Error(format!("unknown param '{sort_by}' for sort_by")); - } + if let Some(sort_by) = params.sort_by.as_deref() + && !snapshot + .param_declarations_range() + .any(|(name, ..)| name == sort_by) + { + return EngineResponse::Error(format!("unknown param '{sort_by}' for sort_by")); } let limit = params.limit.unwrap_or(500).min(10_000); let compiled_params = snapshot.compiled_param_declarations(); @@ -135,7 +147,9 @@ pub(super) fn handle_query_inner( let filtered = snapshot .logs_in_sequence_range((since_bound, std::ops::Bound::Unbounded)) .filter(|(_, timestamp, _)| { - params.start_ms.is_none_or(|start| timestamp.as_millis() >= start) + params + .start_ms + .is_none_or(|start| timestamp.as_millis() >= start) && params.end_ms.is_none_or(|end| timestamp.as_millis() <= end) }) .filter(|(sequence, timestamp, text)| { @@ -172,14 +186,20 @@ pub(super) fn handle_query_inner( } EngineQuery::Timeline(params) => { let compiled_filter = - match compile_optional_filter(snapshot, params.filter.as_deref(), Timestamp::now()) { + match compile_optional_filter(snapshot, params.filter.as_deref(), Timestamp::now()) + { Ok(compiled) => compiled, Err(err) => return EngineResponse::Error(err), }; - let bucket_ms = params.bucket_ms.unwrap_or(DEFAULT_TIMELINE_BUCKET_MS).max(1); + let bucket_ms = params + .bucket_ms + .unwrap_or(DEFAULT_TIMELINE_BUCKET_MS) + .max(1); let mut buckets: BTreeMap = BTreeMap::new(); for (sequence, timestamp, text) in snapshot.logs_range().filter(|(_, timestamp, _)| { - params.start_ms.is_none_or(|start| timestamp.as_millis() >= start) + params + .start_ms + .is_none_or(|start| timestamp.as_millis() >= start) && params.end_ms.is_none_or(|end| timestamp.as_millis() <= end) }) { if compiled_filter @@ -221,34 +241,56 @@ pub(super) fn handle_query_inner( }) } EngineQuery::ParamDeclarations => { - let declarations = snapshot.param_declarations_range() - .map(|(name, message_match_regex, capture_group, json_field_path, kind, elapsed_since_last, auto_derived, _)| { - ParamDeclarationDto { - name: name.to_string(), - message_match_regex: message_match_regex.to_string(), - capture_group: capture_group.to_string(), - json_field_path: json_field_path.map(|path| path.to_string()), + let declarations = snapshot + .param_declarations_range() + .map( + |( + name, + message_match_regex, + capture_group, + json_field_path, kind, elapsed_since_last, auto_derived, - reserved: false, - } - }) + _, + )| { + ParamDeclarationDto { + name: name.to_string(), + message_match_regex: message_match_regex.to_string(), + capture_group: capture_group.to_string(), + json_field_path: json_field_path.map(|path| path.to_string()), + kind, + elapsed_since_last, + auto_derived, + reserved: false, + } + }, + ) .collect(); EngineResponse::ParamDeclarations(declarations) } EngineQuery::MetricDeclarations => { - let declarations = snapshot.metric_declarations_range() - .map(|(name, message_match_regex, label_params, value_param, matched_labels, unmatched_labels)| { - MetricDeclarationDto { - name: name.to_string(), - message_match_regex: message_match_regex.to_string(), - label_params: label_params.to_vec(), - value_param: value_param.map(|s| s.to_string()), - matched_labels: matched_labels.cloned(), - unmatched_labels: unmatched_labels.cloned(), - } - }) + let declarations = snapshot + .metric_declarations_range() + .map( + |( + name, + message_match_regex, + label_params, + value_param, + matched_labels, + unmatched_labels, + )| { + MetricDeclarationDto { + name: name.to_string(), + message_match_regex: message_match_regex.to_string(), + label_params: label_params.to_vec(), + value_param: value_param.map(|s| s.to_string()), + matched_labels: matched_labels.cloned(), + unmatched_labels: unmatched_labels.cloned(), + } + }, + ) .collect(); EngineResponse::MetricDeclarations(declarations) } @@ -261,7 +303,8 @@ pub(super) fn handle_query_inner( limit, } => { let limit = limit.unwrap_or(500).min(10_000); - let values: Vec = snapshot.bucket_values(&metric_name, None) + let values: Vec = snapshot + .bucket_values(&metric_name, None) .filter(|(labels, _)| { labels_filter.as_ref().is_none_or(|filter| { filter.iter().all(|(name, expected)| { @@ -286,7 +329,11 @@ pub(super) fn handle_query_inner( .collect(); EngineResponse::BucketValues(values) } - EngineQuery::Promql { query, start_ms, end_ms } => { + EngineQuery::Promql { + query, + start_ms, + end_ms, + } => { let response = PromqlQuery::parse(&query) .and_then(|parsed| snapshot.evaluate_promql(&parsed, start_ms, end_ms)); EngineResponse::Promql(response) @@ -318,7 +365,9 @@ pub(crate) fn matches_label_matchers( matchers: &[LabelMatcher], ) -> bool { matchers.iter().all(|matcher| { - labels.get(&matcher.name).is_some_and(|value| value.to_string() == matcher.value) + labels + .get(&matcher.name) + .is_some_and(|value| value.to_string() == matcher.value) }) } @@ -338,29 +387,34 @@ pub(crate) fn windowed_rate_series( start_ms: Option, end_ms: Option, ) -> Vec<(i64, f64)> { - let ts_points: Vec<(Timestamp, f64)> = - points.iter().map(|(ms, value)| (Timestamp::from_millis(*ms), *value)).collect(); + let ts_points: Vec<(Timestamp, f64)> = points + .iter() + .map(|(ms, value)| (Timestamp::from_millis(*ms), *value)) + .collect(); let mut series: Vec<(i64, f64)> = rate::windowed_rate_series(&ts_points, range_ms) .into_iter() .map(|(ts, value)| (ts.as_millis(), value)) .collect(); - if let Some(end) = end_ms { - if series.last().is_none_or(|(ts, _)| *ts < end) { - let value = rate::rate_at(&ts_points, Timestamp::from_millis(end), range_ms); - series.push((end, value)); - } + if let Some(end) = end_ms + && series.last().is_none_or(|(ts, _)| *ts < end) + { + let value = rate::rate_at(&ts_points, Timestamp::from_millis(end), range_ms); + series.push((end, value)); } - if let Some(start) = start_ms { - if series.first().is_none_or(|(ts, _)| *ts > start) { - let value = rate::rate_at(&ts_points, Timestamp::from_millis(start), range_ms); - series.insert(0, (start, value)); - } + if let Some(start) = start_ms + && series.first().is_none_or(|(ts, _)| *ts > start) + { + let value = rate::rate_at(&ts_points, Timestamp::from_millis(start), range_ms); + series.insert(0, (start, value)); } series } -pub(crate) fn merge_series_windowed(sub_series: &[Vec<(i64, f64)>], is_avg: bool) -> Vec<(i64, f64)> { +pub(crate) fn merge_series_windowed( + sub_series: &[Vec<(i64, f64)>], + is_avg: bool, +) -> Vec<(i64, f64)> { if sub_series.len() <= 1 { return sub_series.first().cloned().unwrap_or_default(); } @@ -375,7 +429,11 @@ pub(crate) fn merge_series_windowed(sub_series: &[Vec<(i64, f64)>], is_avg: bool .into_iter() .map(|(timestamp_ms, values)| { let sum: f64 = values.iter().sum(); - let value = if is_avg { sum / values.len() as f64 } else { sum }; + let value = if is_avg { + sum / values.len() as f64 + } else { + sum + }; (timestamp_ms, value) }) .collect() @@ -398,7 +456,11 @@ pub(crate) fn merge_series_locf(sub_series: &[Vec<(i64, f64)>], is_avg: bool) -> } } let sum: f64 = last_values.iter().sum(); - let value = if is_avg { sum / last_values.len() as f64 } else { sum }; + let value = if is_avg { + sum / last_values.len() as f64 + } else { + sum + }; (ts, value) }) .collect() diff --git a/src/engine/snapshot/sort.rs b/src/engine/snapshot/sort.rs index b80eb24..bf3d90c 100644 --- a/src/engine/snapshot/sort.rs +++ b/src/engine/snapshot/sort.rs @@ -4,12 +4,12 @@ //! kept incrementally up to date on ingest, rather than rebuilt per query. use crate::engine::data_value::DataValue; -use crate::engine::index_key::IndexKey; use crate::engine::data_value::MetricPrimitiveKind; +use crate::engine::index_key::IndexKey; use crate::engine::metric_primitive::MetricPrimitive; -use crate::engine::snapshot::metrics::parse_metric_primitive; #[cfg(test)] use crate::engine::snapshot::EngineSnapshot; +use crate::engine::snapshot::metrics::parse_metric_primitive; use crate::engine::timestamp::Timestamp; use regex::Regex; use std::collections::{BTreeMap, HashMap}; @@ -26,7 +26,14 @@ pub(crate) struct SortHelperIndexEntry { pub(crate) last_accessed: std::time::Instant, } -pub(super) type CompiledParamDeclaration = (String, Regex, String, Option, MetricPrimitiveKind, bool); +pub(super) type CompiledParamDeclaration = ( + String, + Regex, + String, + Option, + MetricPrimitiveKind, + bool, +); pub(crate) fn derive_single_param( text: &str, @@ -49,7 +56,10 @@ pub(crate) fn derive_single_param( parse_metric_primitive(&raw, *kind) } -pub(crate) fn derive_log_params(text: &str, declarations: &[CompiledParamDeclaration]) -> BTreeMap { +pub(crate) fn derive_log_params( + text: &str, + declarations: &[CompiledParamDeclaration], +) -> BTreeMap { let mut params = BTreeMap::new(); for (name, regex, capture_group, json_field_path, kind, log_stream_gated) in declarations { if *log_stream_gated && !crate::engine::logfmt::is_log_stream_line(text) { @@ -63,10 +73,12 @@ pub(crate) fn derive_log_params(text: &str, declarations: &[CompiledParamDeclara continue; }; let raw: String = match json_field_path.as_deref().filter(|path| !path.is_empty()) { - Some(field_path) => match crate::engine::data_value::extract_json_field(text, field_path) { - Some(value) => value, - None => continue, - }, + Some(field_path) => { + match crate::engine::data_value::extract_json_field(text, field_path) { + Some(value) => value, + None => continue, + } + } None => match captures.name(capture_group) { Some(m) => m.as_str().to_string(), None => continue, @@ -80,9 +92,15 @@ pub(crate) fn derive_log_params(text: &str, declarations: &[CompiledParamDeclara params } -pub(super) fn log_entry_params(text: &str, declarations: &[CompiledParamDeclaration]) -> BTreeMap { +pub(super) fn log_entry_params( + text: &str, + declarations: &[CompiledParamDeclaration], +) -> BTreeMap { let mut params = derive_log_params(text, declarations); - params.insert("ottyl_fmt".to_string(), MetricPrimitive::String(crate::engine::logfmt::detect_format(text).to_string())); + params.insert( + "ottyl_fmt".to_string(), + MetricPrimitive::String(crate::engine::logfmt::detect_format(text).to_string()), + ); params } @@ -103,13 +121,19 @@ impl<'a> SortHelperAccess<'a> { #[cfg(test)] SortHelperAccess::Mutable(map) => { if !map.contains_key(sort_by) { - let built = EngineSnapshot::from_index(index.clone()).build_sort_helper_index(sort_by); + let built = + EngineSnapshot::from_index(index.clone()).build_sort_helper_index(sort_by); map.insert( sort_by.to_string(), - SortHelperIndexEntry { index: built, last_accessed: std::time::Instant::now() }, + SortHelperIndexEntry { + index: built, + last_accessed: std::time::Instant::now(), + }, ); } - let entry = map.get_mut(sort_by).expect("just inserted above if missing"); + let entry = map + .get_mut(sort_by) + .expect("just inserted above if missing"); entry.last_accessed = std::time::Instant::now(); map.get(sort_by).map(|entry| &entry.index) } diff --git a/src/engine/tests.rs b/src/engine/tests.rs index c804fd5..e9bb548 100644 --- a/src/engine/tests.rs +++ b/src/engine/tests.rs @@ -3,6 +3,7 @@ //! exercise several submodules' worth of behavior at once (ingest -> auto-declare -> query). #![allow(unused_imports)] +use super::*; use crate::api::dto::*; use crate::consts::*; use crate::engine::data_value::*; @@ -21,2328 +22,3026 @@ use crate::engine::snapshot::*; use crate::engine::timestamp::*; use crate::flavour::LogLiteralHelper; use std::collections::{BTreeMap, HashMap, HashSet}; - use super::*; - fn request(name: &str, pattern: &str, capture_group: &str) -> DeclareParamRequest { - DeclareParamRequest { - name: name.to_string(), - message_match_regex: pattern.to_string(), - capture_group: capture_group.to_string(), - json_field_path: None, - kind: MetricPrimitiveKind::String, - elapsed_since_last: false, - auto_derived: false, - log_stream_gated: false, - } +fn request(name: &str, pattern: &str, capture_group: &str) -> DeclareParamRequest { + DeclareParamRequest { + name: name.to_string(), + message_match_regex: pattern.to_string(), + capture_group: capture_group.to_string(), + json_field_path: None, + kind: MetricPrimitiveKind::String, + elapsed_since_last: false, + auto_derived: false, + log_stream_gated: false, } +} - #[test] - fn declares_valid_param() { - let mut index = EngineSnapshot::default(); - let result = declare_param(&mut index, request("level", r"level=(?P\w+)", "level")); - assert!(result.is_ok()); +#[test] +fn declares_valid_param() { + let mut index = EngineSnapshot::default(); + let result = declare_param( + &mut index, + request("level", r"level=(?P\w+)", "level"), + ); + assert!(result.is_ok()); - let declarations: Vec<_> = index.param_declarations_range().collect(); - assert_eq!(declarations.len(), 1); - assert_eq!(declarations[0].0, "level"); - assert_eq!(declarations[0].1, r"level=(?P\w+)"); - assert_eq!(declarations[0].2, "level"); + let declarations: Vec<_> = index.param_declarations_range().collect(); + assert_eq!(declarations.len(), 1); + assert_eq!(declarations[0].0, "level"); + assert_eq!(declarations[0].1, r"level=(?P\w+)"); + assert_eq!(declarations[0].2, "level"); +} + +#[test] +fn auto_declare_params_discovers_new_keys_with_inferred_kinds_and_flags_them_auto() { + let mut index = EngineSnapshot::default(); + index.auto_declare_params(r#"ts=1788648105773 msg="entity_moved" x1=-4 y1=34"#); + + let declarations: HashMap<&str, _> = + index.param_declarations_range().map(|d| (d.0, d)).collect(); + assert_eq!(declarations.len(), 4); + assert_eq!(declarations["x1"].4, MetricPrimitiveKind::Number); + assert!( + declarations["x1"].6, + "expected x1 to be flagged auto_derived" + ); + assert_eq!(declarations["msg"].4, MetricPrimitiveKind::String); + + let compiled = index.compiled_param_declarations(); + let params = derive_log_params( + r#"ts=1788648105773 msg="entity_moved" x1=-4 y1=34"#, + &compiled, + ); + assert_eq!( + params.get("x1"), + Some(&MetricPrimitive::Number(Number::Int(-4))) + ); + assert_eq!( + params.get("msg"), + Some(&MetricPrimitive::String("entity_moved".to_string())) + ); +} + +#[test] +fn auto_declare_params_discovers_json_leaf_fields_via_json_field_path() { + let mut index = EngineSnapshot::default(); + index.auto_declare_params(r#"{"level":"error","code":500,"request":{"path":"/x"}}"#); + + let declarations: HashMap<&str, _> = + index.param_declarations_range().map(|d| (d.0, d)).collect(); + assert_eq!(declarations.len(), 3); + assert_eq!(declarations["code"].4, MetricPrimitiveKind::Number); + assert_eq!(declarations["code"].3, Some("code")); + assert!( + declarations["level"].6, + "expected level to be flagged auto_derived" + ); + assert_eq!(declarations["request.path"].3, Some("request.path")); + + let compiled = index.compiled_param_declarations(); + let params = derive_log_params( + r#"{"level":"error","code":500,"request":{"path":"/x"}}"#, + &compiled, + ); + assert_eq!( + params.get("level"), + Some(&MetricPrimitive::String("error".to_string())) + ); + assert_eq!( + params.get("code"), + Some(&MetricPrimitive::Number(Number::Int(500))) + ); + assert_eq!( + params.get("request.path"), + Some(&MetricPrimitive::String("/x".to_string())) + ); +} + +#[test] +fn auto_declare_params_never_overwrites_an_existing_declaration() { + let mut index = EngineSnapshot::default(); + declare_param(&mut index, request("x1", r#"x1="(?P[^"]+)""#, "x1")).unwrap(); + + index.auto_declare_params("x1=-4"); + + let declarations: Vec<_> = index.param_declarations_range().collect(); + assert_eq!(declarations.len(), 1); + assert_eq!( + declarations[0].1, r#"x1="(?P[^"]+)""#, + "manual declaration must survive untouched" + ); + assert!( + !declarations[0].6, + "a manual declaration must never be flagged auto_derived" + ); +} + +#[test] +fn declare_param_rejects_editing_an_auto_derived_param() { + let mut index = EngineSnapshot::default(); + index.auto_declare_params("x1=-4"); + + let result = declare_param(&mut index, request("x1", r#"x1="(?P[^"]+)""#, "x1")); + + let err = result.unwrap_err(); + assert!( + err.contains("auto-detected"), + "unexpected error message: {err}" + ); + let declarations: Vec<_> = index.param_declarations_range().collect(); + assert_eq!(declarations.len(), 1); + assert!( + declarations[0].6, + "the original auto-derived declaration must be untouched" + ); +} + +const LOG_STREAM_FIXTURE: &str = include_str!("../fixtures/log_stream_sample.txt"); + +fn ingest_log_stream_fixture(index: &mut EngineSnapshot) { + let mut header = None; + for line in LOG_STREAM_FIXTURE.lines() { + auto_declare_log_stream_params(index, &mut header, line); + index.auto_declare_params(line); } +} - #[test] - fn auto_declare_params_discovers_new_keys_with_inferred_kinds_and_flags_them_auto() { - let mut index = EngineSnapshot::default(); - index.auto_declare_params(r#"ts=1788648105773 msg="entity_moved" x1=-4 y1=34"#); +#[test] +fn ingesting_log_stream_fixture_does_not_blow_up_declared_param_count() { + let mut index = EngineSnapshot::default(); + ingest_log_stream_fixture(&mut index); - let declarations: HashMap<&str, _> = - index.param_declarations_range().map(|d| (d.0, d)).collect(); - assert_eq!(declarations.len(), 4); - assert_eq!(declarations["x1"].4, MetricPrimitiveKind::Number); - assert!(declarations["x1"].6, "expected x1 to be flagged auto_derived"); - assert_eq!(declarations["msg"].4, MetricPrimitiveKind::String); + let declared = index.param_declarations_range().count(); + assert!( + declared <= MAX_AUTO_DECLARED_PARAMS, + "expected at most {MAX_AUTO_DECLARED_PARAMS} declared params, got {declared}" + ); + assert!( + declared < 100, + "declared param count ({declared}) should stay well under the old ~220" + ); +} - let compiled = index.compiled_param_declarations(); - let params = derive_log_params(r#"ts=1788648105773 msg="entity_moved" x1=-4 y1=34"#, &compiled); - assert_eq!(params.get("x1"), Some(&MetricPrimitive::Number(Number::Int(-4)))); - assert_eq!(params.get("msg"), Some(&MetricPrimitive::String("entity_moved".to_string()))); - } +#[test] +fn base64_padding_fragments_in_log_stream_fixture_are_never_auto_declared() { + let mut index = EngineSnapshot::default(); + ingest_log_stream_fixture(&mut index); - #[test] - fn auto_declare_params_discovers_json_leaf_fields_via_json_field_path() { - let mut index = EngineSnapshot::default(); - index.auto_declare_params(r#"{"level":"error","code":500,"request":{"path":"/x"}}"#); + let names: HashSet<&str> = index.param_declarations_range().map(|d| d.0).collect(); + assert!(!names.contains("rKTEtG7JmnThhoKPbZiVrA")); + assert!(!names.contains("gEdA8N4g")); +} - let declarations: HashMap<&str, _> = - index.param_declarations_range().map(|d| (d.0, d)).collect(); - assert_eq!(declarations.len(), 3); - assert_eq!(declarations["code"].4, MetricPrimitiveKind::Number); - assert_eq!(declarations["code"].3, Some("code")); - assert!(declarations["level"].6, "expected level to be flagged auto_derived"); - assert_eq!(declarations["request.path"].3, Some("request.path")); +#[test] +fn log_stream_shaped_lines_expose_header_named_columns_as_queryable_params() { + let mut index = EngineSnapshot::default(); + ingest_log_stream_fixture(&mut index); - let compiled = index.compiled_param_declarations(); - let params = - derive_log_params(r#"{"level":"error","code":500,"request":{"path":"/x"}}"#, &compiled); - assert_eq!(params.get("level"), Some(&MetricPrimitive::String("error".to_string()))); - assert_eq!(params.get("code"), Some(&MetricPrimitive::Number(Number::Int(500)))); - assert_eq!(params.get("request.path"), Some(&MetricPrimitive::String("/x".to_string()))); - } - - #[test] - fn auto_declare_params_never_overwrites_an_existing_declaration() { - let mut index = EngineSnapshot::default(); - declare_param(&mut index, request("x1", r#"x1="(?P[^"]+)""#, "x1")).unwrap(); - - index.auto_declare_params("x1=-4"); - - let declarations: Vec<_> = index.param_declarations_range().collect(); - assert_eq!(declarations.len(), 1); - assert_eq!(declarations[0].1, r#"x1="(?P[^"]+)""#, "manual declaration must survive untouched"); - assert!(!declarations[0].6, "a manual declaration must never be flagged auto_derived"); - } - - #[test] - fn declare_param_rejects_editing_an_auto_derived_param() { - let mut index = EngineSnapshot::default(); - index.auto_declare_params("x1=-4"); - - let result = declare_param(&mut index, request("x1", r#"x1="(?P[^"]+)""#, "x1")); - - let err = result.unwrap_err(); - assert!(err.contains("auto-detected"), "unexpected error message: {err}"); - let declarations: Vec<_> = index.param_declarations_range().collect(); - assert_eq!(declarations.len(), 1); - assert!(declarations[0].6, "the original auto-derived declaration must be untouched"); - } - - const LOG_STREAM_FIXTURE: &str = include_str!("../fixtures/log_stream_sample.txt"); - - fn ingest_log_stream_fixture(index: &mut EngineSnapshot) { - let mut header = None; - for line in LOG_STREAM_FIXTURE.lines() { - auto_declare_log_stream_params(index, &mut header, line); - index.auto_declare_params(line); - } - } - - #[test] - fn ingesting_log_stream_fixture_does_not_blow_up_declared_param_count() { - let mut index = EngineSnapshot::default(); - ingest_log_stream_fixture(&mut index); - - let declared = index.param_declarations_range().count(); + let declared: HashSet<&str> = index.param_declarations_range().map(|d| d.0).collect(); + for name in [ + "Timestamp", + "Thread", + "Type", + "Activity", + "PID", + "TTL", + "message", + ] { assert!( - declared <= MAX_AUTO_DECLARED_PARAMS, - "expected at most {MAX_AUTO_DECLARED_PARAMS} declared params, got {declared}" + declared.contains(name), + "expected {name} to be declared, got {declared:?}" ); - assert!(declared < 100, "declared param count ({declared}) should stay well under the old ~220"); } - #[test] - fn base64_padding_fragments_in_log_stream_fixture_are_never_auto_declared() { - let mut index = EngineSnapshot::default(); - ingest_log_stream_fixture(&mut index); - - let names: HashSet<&str> = index.param_declarations_range().map(|d| d.0).collect(); - assert!(!names.contains("rKTEtG7JmnThhoKPbZiVrA")); - assert!(!names.contains("gEdA8N4g")); - } - - #[test] - fn log_stream_shaped_lines_expose_header_named_columns_as_queryable_params() { - let mut index = EngineSnapshot::default(); - ingest_log_stream_fixture(&mut index); - - let declared: HashSet<&str> = index.param_declarations_range().map(|d| d.0).collect(); - for name in ["Timestamp", "Thread", "Type", "Activity", "PID", "TTL", "message"] { - assert!(declared.contains(name), "expected {name} to be declared, got {declared:?}"); - } - - let compiled = index.compiled_param_declarations(); - let data_line = LOG_STREAM_FIXTURE.lines().nth(1).unwrap(); - let params = derive_log_params(data_line, &compiled); - assert_eq!(params.get("Type"), Some(&MetricPrimitive::String("Default".to_string()))); - assert_eq!(params.get("PID"), Some(&MetricPrimitive::Number(Number::Int(1343)))); - assert_eq!(params.get("TTL"), Some(&MetricPrimitive::Number(Number::Int(7)))); - assert_eq!( + let compiled = index.compiled_param_declarations(); + let data_line = LOG_STREAM_FIXTURE.lines().nth(1).unwrap(); + let params = derive_log_params(data_line, &compiled); + assert_eq!( + params.get("Type"), + Some(&MetricPrimitive::String("Default".to_string())) + ); + assert_eq!( + params.get("PID"), + Some(&MetricPrimitive::Number(Number::Int(1343))) + ); + assert_eq!( + params.get("TTL"), + Some(&MetricPrimitive::Number(Number::Int(7))) + ); + assert_eq!( params.get("message"), Some(&MetricPrimitive::String( "kitty: (SkyLight) [com.apple.SkyLight:default] SLSGetDisplaysWithOpenGLDisplayMask, 3614: Display 0x00000001: Mode 54, (0 x 0) [1728 x 1117], Depth 32 BPP, active".to_string() )) ); +} + +#[test] +fn log_stream_columns_are_not_contaminated_by_non_columnar_lines() { + let mut index = EngineSnapshot::default(); + ingest_log_stream_fixture(&mut index); + let compiled = index.compiled_param_declarations(); + + let continuation_line = "so_gencnt: 4042707 so_state: 0x0"; + let params = derive_log_params(continuation_line, &compiled); + assert!( + !params.contains_key("Timestamp"), + "continuation line got a Timestamp: {params:?}" + ); + assert!( + !params.contains_key("Activity"), + "continuation line got an Activity: {params:?}" + ); + assert!( + !params.contains_key("PID"), + "continuation line got a PID: {params:?}" + ); + + let header_line = LOG_STREAM_FIXTURE.lines().next().unwrap(); + let header_params = derive_log_params(header_line, &compiled); + assert!( + !header_params.contains_key("Timestamp"), + "header line got a Timestamp: {header_params:?}" + ); + assert!( + !header_params.contains_key("Activity"), + "header line got an Activity: {header_params:?}" + ); + assert!( + !header_params.contains_key("Thread"), + "header line got a Thread: {header_params:?}" + ); +} + +#[test] +fn logs_query_sorted_by_a_log_stream_column_actually_reorders_real_log_stream_output() { + let mut index = EngineSnapshot::default(); + let mut header = None; + for (i, line) in LOG_STREAM_FIXTURE.lines().enumerate() { + let sequence = i as u64; + let timestamp = Timestamp::from_millis(sequence as i64); + auto_declare_log_stream_params(&mut index, &mut header, line); + index.auto_declare_params(line); + index.index.insert( + IndexKey::IngestedLog { + sequence, + timestamp, + }, + DataValue::LogLiteral { + text: line.to_string(), + }, + ); } - #[test] - fn log_stream_columns_are_not_contaminated_by_non_columnar_lines() { - let mut index = EngineSnapshot::default(); - ingest_log_stream_fixture(&mut index); - let compiled = index.compiled_param_declarations(); + let entries = match handle_query( + &index, + logs_query_sorted("PID", Some(false)), + &mut HashMap::new(), + ) { + EngineResponse::Logs(entries) => entries, + _ => panic!("expected Logs response"), + }; + assert!( + !entries.is_empty(), + "expected at least one line with a PID value" + ); - let continuation_line = "so_gencnt: 4042707 so_state: 0x0"; - let params = derive_log_params(continuation_line, &compiled); - assert!(!params.contains_key("Timestamp"), "continuation line got a Timestamp: {params:?}"); - assert!(!params.contains_key("Activity"), "continuation line got an Activity: {params:?}"); - assert!(!params.contains_key("PID"), "continuation line got a PID: {params:?}"); + let pids: Vec = entries + .iter() + .map(|entry| match entry.params.get("PID") { + Some(MetricPrimitive::Number(Number::Int(pid))) => *pid, + other => panic!("expected every returned entry to have a numeric PID, got {other:?}"), + }) + .collect(); + let mut sorted_pids = pids.clone(); + sorted_pids.sort_unstable(); + assert_eq!( + pids, sorted_pids, + "entries were not actually reordered by PID" + ); - let header_line = LOG_STREAM_FIXTURE.lines().next().unwrap(); - let header_params = derive_log_params(header_line, &compiled); - assert!(!header_params.contains_key("Timestamp"), "header line got a Timestamp: {header_params:?}"); - assert!(!header_params.contains_key("Activity"), "header line got an Activity: {header_params:?}"); - assert!(!header_params.contains_key("Thread"), "header line got a Thread: {header_params:?}"); - } + let original_order: Vec = entries.iter().map(|e| e.sequence).collect(); + let mut sorted_by_sequence = original_order.clone(); + sorted_by_sequence.sort_unstable(); + assert_ne!( + original_order, sorted_by_sequence, + "PID sort produced the same order as plain ingest order" + ); +} - #[test] - fn logs_query_sorted_by_a_log_stream_column_actually_reorders_real_log_stream_output() { - let mut index = EngineSnapshot::default(); +#[test] +fn non_log_stream_lines_do_not_get_any_columns_declared_without_a_cached_header() { + let mut index = EngineSnapshot::default(); + let mut header = None; + auto_declare_log_stream_params( + &mut index, + &mut header, + r#"ts=1788648105773 msg="entity_moved" x1=-4"#, + ); + + assert!(index.param_declarations_range().next().is_none()); + assert!(header.is_none()); +} + +#[test] +#[ignore] +fn bench_ingest_fixture_throughput() { + let repeats = 50; + let mut index = EngineSnapshot::default(); + let start = std::time::Instant::now(); + for _ in 0..repeats { let mut header = None; - for (i, line) in LOG_STREAM_FIXTURE.lines().enumerate() { - let sequence = i as u64; - let timestamp = Timestamp::from_millis(sequence as i64); + for line in LOG_STREAM_FIXTURE.lines() { auto_declare_log_stream_params(&mut index, &mut header, line); index.auto_declare_params(line); - index.index.insert( - IndexKey::IngestedLog { sequence, timestamp }, - DataValue::LogLiteral { text: line.to_string() }, - ); } - - let entries = match handle_query(&index, logs_query_sorted("PID", Some(false)), &mut HashMap::new()) { - EngineResponse::Logs(entries) => entries, - _ => panic!("expected Logs response"), - }; - assert!(!entries.is_empty(), "expected at least one line with a PID value"); - - let pids: Vec = entries - .iter() - .map(|entry| match entry.params.get("PID") { - Some(MetricPrimitive::Number(Number::Int(pid))) => *pid, - other => panic!("expected every returned entry to have a numeric PID, got {other:?}"), - }) - .collect(); - let mut sorted_pids = pids.clone(); - sorted_pids.sort_unstable(); - assert_eq!(pids, sorted_pids, "entries were not actually reordered by PID"); - - let original_order: Vec = entries.iter().map(|e| e.sequence).collect(); - let mut sorted_by_sequence = original_order.clone(); - sorted_by_sequence.sort_unstable(); - assert_ne!(original_order, sorted_by_sequence, "PID sort produced the same order as plain ingest order"); } + let elapsed = start.elapsed(); + let total_lines = repeats * LOG_STREAM_FIXTURE.lines().count(); + eprintln!( + "ingested {total_lines} lines in {elapsed:?} ({:.0} lines/sec), {} params declared", + total_lines as f64 / elapsed.as_secs_f64(), + index.param_declarations_range().count(), + ); +} - #[test] - fn non_log_stream_lines_do_not_get_any_columns_declared_without_a_cached_header() { - let mut index = EngineSnapshot::default(); - let mut header = None; - auto_declare_log_stream_params(&mut index, &mut header, r#"ts=1788648105773 msg="entity_moved" x1=-4"#); +#[test] +fn rejects_invalid_regex() { + let mut index = EngineSnapshot::default(); + let result = declare_param(&mut index, request("level", "level=(", "level")); + assert!(result.is_err()); + assert!(index.index.is_empty()); +} - assert!(index.param_declarations_range().next().is_none()); - assert!(header.is_none()); - } +#[test] +fn rejects_unknown_capture_group() { + let mut index = EngineSnapshot::default(); + let result = declare_param( + &mut index, + request("level", r"level=(?P\w+)", "missing"), + ); + assert!(result.is_err()); + assert!(index.index.is_empty()); +} - #[test] - #[ignore] - fn bench_ingest_fixture_throughput() { - let repeats = 50; - let mut index = EngineSnapshot::default(); - let start = std::time::Instant::now(); - for _ in 0..repeats { - let mut header = None; - for line in LOG_STREAM_FIXTURE.lines() { - auto_declare_log_stream_params(&mut index, &mut header, line); - index.auto_declare_params(line); - } - } - let elapsed = start.elapsed(); - let total_lines = repeats * LOG_STREAM_FIXTURE.lines().count(); - eprintln!( - "ingested {total_lines} lines in {elapsed:?} ({:.0} lines/sec), {} params declared", - total_lines as f64 / elapsed.as_secs_f64(), - index.param_declarations_range().count(), - ); - } +#[test] +fn declares_valid_elapsed_since_last_param_without_capture_group() { + let mut index = EngineSnapshot::default(); + declare_elapsed_since_last_param(&mut index, "gap"); - #[test] - fn rejects_invalid_regex() { - let mut index = EngineSnapshot::default(); - let result = declare_param(&mut index, request("level", "level=(", "level")); - assert!(result.is_err()); - assert!(index.index.is_empty()); - } + let declarations: Vec<_> = index.param_declarations_range().collect(); + assert_eq!(declarations.len(), 1); + assert!(declarations[0].5, "expected elapsed_since_last to be true"); +} - #[test] - fn rejects_unknown_capture_group() { - let mut index = EngineSnapshot::default(); - let result = declare_param( - &mut index, - request("level", r"level=(?P\w+)", "missing"), - ); - assert!(result.is_err()); - assert!(index.index.is_empty()); - } +#[test] +fn rejects_elapsed_since_last_param_with_non_duration_kind() { + let mut index = EngineSnapshot::default(); + let result = declare_param( + &mut index, + DeclareParamRequest { + name: "gap".to_string(), + message_match_regex: ".".to_string(), + capture_group: String::new(), + json_field_path: None, + kind: MetricPrimitiveKind::String, + elapsed_since_last: true, + auto_derived: false, + log_stream_gated: false, + }, + ); + assert!(result.is_err()); + assert!(index.index.is_empty()); +} - #[test] - fn declares_valid_elapsed_since_last_param_without_capture_group() { - let mut index = EngineSnapshot::default(); - declare_elapsed_since_last_param(&mut index, "gap"); +#[test] +fn declares_valid_param_from_json_field_path() { + let mut index = EngineSnapshot::default(); + let result = declare_param( + &mut index, + DeclareParamRequest { + name: "level".to_string(), + message_match_regex: r".".to_string(), + capture_group: String::new(), + json_field_path: Some("foo.bar".to_string()), + kind: MetricPrimitiveKind::String, + elapsed_since_last: false, + auto_derived: false, + log_stream_gated: false, + }, + ); + assert!(result.is_ok()); - let declarations: Vec<_> = index.param_declarations_range().collect(); - assert_eq!(declarations.len(), 1); - assert!(declarations[0].5, "expected elapsed_since_last to be true"); - } + let declarations: Vec<_> = index.param_declarations_range().collect(); + assert_eq!(declarations.len(), 1); + assert_eq!(declarations[0].3, Some("foo.bar")); +} - #[test] - fn rejects_elapsed_since_last_param_with_non_duration_kind() { - let mut index = EngineSnapshot::default(); - let result = declare_param( - &mut index, - DeclareParamRequest { - name: "gap".to_string(), - message_match_regex: ".".to_string(), - capture_group: String::new(), - json_field_path: None, - kind: MetricPrimitiveKind::String, - elapsed_since_last: true, - auto_derived: false, - log_stream_gated: false, - }, - ); - assert!(result.is_err()); - assert!(index.index.is_empty()); - } +#[test] +fn derives_log_param_from_json_field_path() { + let mut index = EngineSnapshot::default(); + declare_param( + &mut index, + DeclareParamRequest { + name: "bar".to_string(), + message_match_regex: r".".to_string(), + capture_group: String::new(), + json_field_path: Some("foo.bar".to_string()), + kind: MetricPrimitiveKind::Number, + elapsed_since_last: false, + auto_derived: false, + log_stream_gated: false, + }, + ) + .unwrap(); - #[test] - fn declares_valid_param_from_json_field_path() { - let mut index = EngineSnapshot::default(); - let result = declare_param( - &mut index, - DeclareParamRequest { - name: "level".to_string(), - message_match_regex: r".".to_string(), - capture_group: String::new(), - json_field_path: Some("foo.bar".to_string()), - kind: MetricPrimitiveKind::String, - elapsed_since_last: false, - auto_derived: false, - log_stream_gated: false, - }, - ); - assert!(result.is_ok()); + let declarations = index.compiled_param_declarations(); + let params = derive_log_params(r#"{"foo":{"bar":42}}"#, &declarations); + assert_eq!( + params.get("bar"), + Some(&MetricPrimitive::Number(Number::Int(42))) + ); - let declarations: Vec<_> = index.param_declarations_range().collect(); - assert_eq!(declarations.len(), 1); - assert_eq!(declarations[0].3, Some("foo.bar")); - } + assert!(derive_log_params("not json", &declarations).is_empty()); + assert!(derive_log_params(r#"{"foo":{}}"#, &declarations).is_empty()); +} - #[test] - fn derives_log_param_from_json_field_path() { - let mut index = EngineSnapshot::default(); - declare_param( - &mut index, - DeclareParamRequest { - name: "bar".to_string(), - message_match_regex: r".".to_string(), - capture_group: String::new(), - json_field_path: Some("foo.bar".to_string()), - kind: MetricPrimitiveKind::Number, - elapsed_since_last: false, - auto_derived: false, - log_stream_gated: false, - }, - ) - .unwrap(); - - let declarations = index.compiled_param_declarations(); - let params = derive_log_params(r#"{"foo":{"bar":42}}"#, &declarations); - assert_eq!(params.get("bar"), Some(&MetricPrimitive::Number(Number::Int(42)))); - - assert!(derive_log_params("not json", &declarations).is_empty()); - assert!(derive_log_params(r#"{"foo":{}}"#, &declarations).is_empty()); - } - - #[test] - fn evict_excess_logs_drops_oldest_sequences_first() { - let mut index = EngineSnapshot::default(); - for sequence in 0..5u64 { - let text = format!("line {sequence}"); - index.approx_bytes_total += text.approx_log_bytes(); - index.index.insert( - IndexKey::IngestedLog { sequence, timestamp: Timestamp::from_millis(sequence as i64) }, - DataValue::LogLiteral { text }, - ); - index.log_count += 1; - } - - index.evict_excess_logs(Some(3)); - - assert_eq!(index.log_count, 3); - let remaining: Vec<_> = index.logs_range().map(|(seq, _, _)| seq).collect(); - assert_eq!(remaining, vec![2, 3, 4]); - let expected_bytes: u64 = - remaining.iter().map(|seq| format!("line {seq}").approx_log_bytes()).sum(); - assert_eq!(index.approx_bytes_total, expected_bytes); - } - - #[test] - fn logs_in_sequence_range_seeks_instead_of_scanning_from_start() { - let mut index = EngineSnapshot::default(); - for sequence in 0..1000u64 { - index.index.insert( - IndexKey::IngestedLog { sequence, timestamp: Timestamp::from_millis(sequence as i64) }, - DataValue::LogLiteral { text: format!("line {sequence}") }, - ); - } - - // An excluded lower bound (as `since_sequence` polling uses) must start exactly past - // the bound, not include it or anything before it. - let since_990: Vec<_> = - index.logs_in_sequence_range(990..).map(|(seq, _, _)| seq).collect(); - assert_eq!(since_990, vec![990, 991, 992, 993, 994, 995, 996, 997, 998, 999]); - - let after_990: Vec<_> = index - .logs_in_sequence_range((std::ops::Bound::Excluded(990u64), std::ops::Bound::Unbounded)) - .map(|(seq, _, _)| seq) - .collect(); - assert_eq!(after_990, vec![991, 992, 993, 994, 995, 996, 997, 998, 999]); - - // An exclusive upper bound (as `logs_since`'s `before_sequence` uses) must stop - // strictly before it. - let before_5: Vec<_> = index.logs_in_sequence_range(..5u64).map(|(seq, _, _)| seq).collect(); - assert_eq!(before_5, vec![0, 1, 2, 3, 4]); - } - - #[test] - fn evict_excess_logs_is_noop_when_unlimited_or_under_cap() { - let mut index = EngineSnapshot::default(); - for sequence in 0..3u64 { - index.approx_bytes_total += "x".approx_log_bytes(); - index.index.insert( - IndexKey::IngestedLog { sequence, timestamp: Timestamp::from_millis(sequence as i64) }, - DataValue::LogLiteral { text: "x".to_string() }, - ); - index.log_count += 1; - } - - index.evict_excess_logs(None); - assert_eq!(index.log_count, 3); - assert_eq!(index.approx_bytes_total, 3); - - index.evict_excess_logs(Some(10)); - assert_eq!(index.log_count, 3); - assert_eq!(index.approx_bytes_total, 3); - } - - - #[test] - fn ingest_lag_watermark_reports_the_highest_sample_seen() { - let mut samples = Vec::new(); - let t0 = Timestamp::from_millis(0); - record_ingest_lag(&mut samples, t0, 10); - record_ingest_lag(&mut samples, t0, 50); - record_ingest_lag(&mut samples, t0, 20); - assert_eq!(ingest_lag_high_watermark_ms(&samples, t0), 50); - } - - #[test] - fn ingest_lag_watermark_only_replaces_the_current_minimum_once_at_capacity() { - let mut samples = Vec::new(); - let t0 = Timestamp::from_millis(0); - for lag in [10, 20, 30, 40, 50] { - record_ingest_lag(&mut samples, t0, lag); - } - assert_eq!(samples.len(), 5); - assert_eq!(ingest_lag_high_watermark_ms(&samples, t0), 50); - - record_ingest_lag(&mut samples, t0, 5); - assert_eq!(samples.len(), 5); - assert_eq!(ingest_lag_high_watermark_ms(&samples, t0), 50); - - record_ingest_lag(&mut samples, t0, 25); - assert_eq!(samples.len(), 5); - assert!(samples.iter().all(|(_, lag)| *lag != 10), "the old minimum (10) should have been replaced"); - assert_eq!(ingest_lag_high_watermark_ms(&samples, t0), 50); - } - - #[test] - fn ingest_lag_watermark_decays_once_samples_age_out_of_the_window() { - let mut samples = Vec::new(); - let t0 = Timestamp::from_millis(0); - record_ingest_lag(&mut samples, t0, 500); - assert_eq!(ingest_lag_high_watermark_ms(&samples, t0), 500); - - let almost_expired = Timestamp::from_millis(INGEST_LAG_WATERMARK_WINDOW_MS - 1); - assert_eq!(ingest_lag_high_watermark_ms(&samples, almost_expired), 500); - - let expired = Timestamp::from_millis(INGEST_LAG_WATERMARK_WINDOW_MS); - assert_eq!(ingest_lag_high_watermark_ms(&samples, expired), 0); - - record_ingest_lag(&mut samples, expired, 5); - assert_eq!(samples.len(), 1); - assert_eq!(ingest_lag_high_watermark_ms(&samples, expired), 5); - } - - fn insert_ingested_log(index: &mut EngineSnapshot, sequence: u64, text: &str) { +#[test] +fn evict_excess_logs_drops_oldest_sequences_first() { + let mut index = EngineSnapshot::default(); + for sequence in 0..5u64 { + let text = format!("line {sequence}"); + index.approx_bytes_total += text.approx_log_bytes(); index.index.insert( - IndexKey::IngestedLog { sequence, timestamp: Timestamp::from_millis(sequence as i64) }, - DataValue::LogLiteral { text: text.to_string() }, + IndexKey::IngestedLog { + sequence, + timestamp: Timestamp::from_millis(sequence as i64), + }, + DataValue::LogLiteral { text }, ); + index.log_count += 1; } - #[test] - fn clear_all_param_declarations_removes_manual_and_auto_derived() { - let mut index = EngineSnapshot::default(); - index.auto_declare_params("x1=-4"); - declare_param(&mut index, request("level", r"level=(?P\w+)", "level")).unwrap(); - assert_eq!(index.param_declarations_range().count(), 2); + index.evict_excess_logs(Some(3)); - let removed = index.clear_all_param_declarations(); + assert_eq!(index.log_count, 3); + let remaining: Vec<_> = index.logs_range().map(|(seq, _, _)| seq).collect(); + assert_eq!(remaining, vec![2, 3, 4]); + let expected_bytes: u64 = remaining + .iter() + .map(|seq| format!("line {seq}").approx_log_bytes()) + .sum(); + assert_eq!(index.approx_bytes_total, expected_bytes); +} - assert_eq!(removed, 2); - assert_eq!(index.param_declarations_range().count(), 0); - } - - #[test] - fn reextract_params_from_retained_logs_rebuilds_from_ingested_text_without_new_ingest() { - let mut index = EngineSnapshot::default(); - insert_ingested_log(&mut index, 0, r#"level=info msg="started""#); - insert_ingested_log(&mut index, 1, "x1=-4 y1=34"); - index.auto_declare_params(r#"level=info msg="started""#); - index.auto_declare_params("x1=-4 y1=34"); - assert_eq!(index.param_declarations_range().count(), 4); - - index.clear_all_param_declarations(); - assert_eq!(index.param_declarations_range().count(), 0); - - let (reextracted, logs_considered) = index.reextract_params_from_retained_logs(); - - assert_eq!(logs_considered, 2); - assert_eq!(reextracted, 4); - let declarations: HashMap<&str, _> = - index.param_declarations_range().map(|d| (d.0, d)).collect(); - assert!(declarations.contains_key("level")); - assert!(declarations.contains_key("msg")); - assert!(declarations.contains_key("x1")); - assert!(declarations.contains_key("y1")); - assert!(declarations.values().all(|d| d.6), "re-extracted params must be flagged auto_derived"); - } - - #[test] - fn reextract_params_from_retained_logs_only_covers_lines_still_in_the_index() { - let mut index = EngineSnapshot::default(); - insert_ingested_log(&mut index, 1, "x1=-4"); - index.auto_declare_params("evicted_only_key=1"); - index.auto_declare_params("x1=-4"); - assert_eq!(index.param_declarations_range().count(), 2); - - index.clear_all_param_declarations(); - let (reextracted, logs_considered) = index.reextract_params_from_retained_logs(); - - assert_eq!(logs_considered, 1, "only the retained log line should be re-scanned"); - assert_eq!(reextracted, 1, "the evicted line's key can't come back"); - let declarations: Vec<_> = index.param_declarations_range().collect(); - assert_eq!(declarations[0].0, "x1"); - } - - #[test] - fn declarations_stay_within_their_range() { - let mut index = EngineSnapshot::default(); +#[test] +fn logs_in_sequence_range_seeks_instead_of_scanning_from_start() { + let mut index = EngineSnapshot::default(); + for sequence in 0..1000u64 { index.index.insert( - IndexKey::IngestedLog { sequence: 0, timestamp: Timestamp::from_millis(0) }, - DataValue::LogLiteral { text: "hello".to_string() }, + IndexKey::IngestedLog { + sequence, + timestamp: Timestamp::from_millis(sequence as i64), + }, + DataValue::LogLiteral { + text: format!("line {sequence}"), + }, ); - declare_param(&mut index, request("level", r"level=(?P\w+)", "level")).unwrap(); + } + + // An excluded lower bound (as `since_sequence` polling uses) must start exactly past + // the bound, not include it or anything before it. + let since_990: Vec<_> = index + .logs_in_sequence_range(990..) + .map(|(seq, _, _)| seq) + .collect(); + assert_eq!( + since_990, + vec![990, 991, 992, 993, 994, 995, 996, 997, 998, 999] + ); + + let after_990: Vec<_> = index + .logs_in_sequence_range(( + std::ops::Bound::Excluded(990u64), + std::ops::Bound::Unbounded, + )) + .map(|(seq, _, _)| seq) + .collect(); + assert_eq!(after_990, vec![991, 992, 993, 994, 995, 996, 997, 998, 999]); + + // An exclusive upper bound (as `logs_since`'s `before_sequence` uses) must stop + // strictly before it. + let before_5: Vec<_> = index + .logs_in_sequence_range(..5u64) + .map(|(seq, _, _)| seq) + .collect(); + assert_eq!(before_5, vec![0, 1, 2, 3, 4]); +} + +#[test] +fn evict_excess_logs_is_noop_when_unlimited_or_under_cap() { + let mut index = EngineSnapshot::default(); + for sequence in 0..3u64 { + index.approx_bytes_total += "x".approx_log_bytes(); index.index.insert( - IndexKey::MetricDeclaration { name: "requests".to_string() }, - DataValue::MetricDeclaration { - message_match_regex: "x".to_string(), - label_params: vec![], - value_param: None, - matched_labels: None, - unmatched_labels: None, + IndexKey::IngestedLog { + sequence, + timestamp: Timestamp::from_millis(sequence as i64), + }, + DataValue::LogLiteral { + text: "x".to_string(), }, ); - - let declarations: Vec<_> = index.param_declarations_range().collect(); - assert_eq!(declarations.len(), 1); - assert_eq!(declarations[0].0, "level"); - - for key in index.index.range(IndexKey::LogParamSplit..IndexKey::ParamMetricSplit).map(|(k, _)| k) { - assert!(matches!(key, IndexKey::ParamDeclaration { .. })); - } + index.log_count += 1; } - #[test] - fn removes_declaration() { - let mut index = EngineSnapshot::default(); - declare_param(&mut index, request("level", r"level=(?P\w+)", "level")).unwrap(); - assert!(index.index.remove(&IndexKey::ParamDeclaration { name: "level".to_string() }).is_some()); - assert!(index.param_declarations_range().next().is_none()); - } + index.evict_excess_logs(None); + assert_eq!(index.log_count, 3); + assert_eq!(index.approx_bytes_total, 3); - fn declare_test_param( - index: &mut EngineSnapshot, - name: &str, - pattern: &str, - capture_group: &str, - kind: MetricPrimitiveKind, - ) { - declare_param( - index, - DeclareParamRequest { - name: name.to_string(), - message_match_regex: pattern.to_string(), - capture_group: capture_group.to_string(), - json_field_path: None, - kind, - elapsed_since_last: false, - auto_derived: false, - log_stream_gated: false, - }, - ) - .unwrap(); - } + index.evict_excess_logs(Some(10)); + assert_eq!(index.log_count, 3); + assert_eq!(index.approx_bytes_total, 3); +} - fn declare_elapsed_since_last_param(index: &mut EngineSnapshot, name: &str) { - declare_param( - index, - DeclareParamRequest { - name: name.to_string(), - message_match_regex: ".".to_string(), - capture_group: String::new(), - json_field_path: None, - kind: MetricPrimitiveKind::Duration, - elapsed_since_last: true, - auto_derived: false, - log_stream_gated: false, - }, - ) - .unwrap(); - } +#[test] +fn ingest_lag_watermark_reports_the_highest_sample_seen() { + let mut samples = Vec::new(); + let t0 = Timestamp::from_millis(0); + record_ingest_lag(&mut samples, t0, 10); + record_ingest_lag(&mut samples, t0, 50); + record_ingest_lag(&mut samples, t0, 20); + assert_eq!(ingest_lag_high_watermark_ms(&samples, t0), 50); +} - fn metric_request(name: &str, pattern: &str, label_params: Vec<&str>) -> DeclareMetricRequest { - metric_request_with_value(name, pattern, label_params, None) +#[test] +fn ingest_lag_watermark_only_replaces_the_current_minimum_once_at_capacity() { + let mut samples = Vec::new(); + let t0 = Timestamp::from_millis(0); + for lag in [10, 20, 30, 40, 50] { + record_ingest_lag(&mut samples, t0, lag); } + assert_eq!(samples.len(), 5); + assert_eq!(ingest_lag_high_watermark_ms(&samples, t0), 50); - fn metric_request_with_value( - name: &str, - pattern: &str, - label_params: Vec<&str>, - value_param: Option<&str>, - ) -> DeclareMetricRequest { - DeclareMetricRequest { - name: name.to_string(), - message_match_regex: pattern.to_string(), - label_params: label_params.into_iter().map(str::to_string).collect(), - value_param: value_param.map(str::to_string), + record_ingest_lag(&mut samples, t0, 5); + assert_eq!(samples.len(), 5); + assert_eq!(ingest_lag_high_watermark_ms(&samples, t0), 50); + + record_ingest_lag(&mut samples, t0, 25); + assert_eq!(samples.len(), 5); + assert!( + samples.iter().all(|(_, lag)| *lag != 10), + "the old minimum (10) should have been replaced" + ); + assert_eq!(ingest_lag_high_watermark_ms(&samples, t0), 50); +} + +#[test] +fn ingest_lag_watermark_decays_once_samples_age_out_of_the_window() { + let mut samples = Vec::new(); + let t0 = Timestamp::from_millis(0); + record_ingest_lag(&mut samples, t0, 500); + assert_eq!(ingest_lag_high_watermark_ms(&samples, t0), 500); + + let almost_expired = Timestamp::from_millis(INGEST_LAG_WATERMARK_WINDOW_MS - 1); + assert_eq!(ingest_lag_high_watermark_ms(&samples, almost_expired), 500); + + let expired = Timestamp::from_millis(INGEST_LAG_WATERMARK_WINDOW_MS); + assert_eq!(ingest_lag_high_watermark_ms(&samples, expired), 0); + + record_ingest_lag(&mut samples, expired, 5); + assert_eq!(samples.len(), 1); + assert_eq!(ingest_lag_high_watermark_ms(&samples, expired), 5); +} + +fn insert_ingested_log(index: &mut EngineSnapshot, sequence: u64, text: &str) { + index.index.insert( + IndexKey::IngestedLog { + sequence, + timestamp: Timestamp::from_millis(sequence as i64), + }, + DataValue::LogLiteral { + text: text.to_string(), + }, + ); +} + +#[test] +fn clear_all_param_declarations_removes_manual_and_auto_derived() { + let mut index = EngineSnapshot::default(); + index.auto_declare_params("x1=-4"); + declare_param( + &mut index, + request("level", r"level=(?P\w+)", "level"), + ) + .unwrap(); + assert_eq!(index.param_declarations_range().count(), 2); + + let removed = index.clear_all_param_declarations(); + + assert_eq!(removed, 2); + assert_eq!(index.param_declarations_range().count(), 0); +} + +#[test] +fn reextract_params_from_retained_logs_rebuilds_from_ingested_text_without_new_ingest() { + let mut index = EngineSnapshot::default(); + insert_ingested_log(&mut index, 0, r#"level=info msg="started""#); + insert_ingested_log(&mut index, 1, "x1=-4 y1=34"); + index.auto_declare_params(r#"level=info msg="started""#); + index.auto_declare_params("x1=-4 y1=34"); + assert_eq!(index.param_declarations_range().count(), 4); + + index.clear_all_param_declarations(); + assert_eq!(index.param_declarations_range().count(), 0); + + let (reextracted, logs_considered) = index.reextract_params_from_retained_logs(); + + assert_eq!(logs_considered, 2); + assert_eq!(reextracted, 4); + let declarations: HashMap<&str, _> = + index.param_declarations_range().map(|d| (d.0, d)).collect(); + assert!(declarations.contains_key("level")); + assert!(declarations.contains_key("msg")); + assert!(declarations.contains_key("x1")); + assert!(declarations.contains_key("y1")); + assert!( + declarations.values().all(|d| d.6), + "re-extracted params must be flagged auto_derived" + ); +} + +#[test] +fn reextract_params_from_retained_logs_only_covers_lines_still_in_the_index() { + let mut index = EngineSnapshot::default(); + insert_ingested_log(&mut index, 1, "x1=-4"); + index.auto_declare_params("evicted_only_key=1"); + index.auto_declare_params("x1=-4"); + assert_eq!(index.param_declarations_range().count(), 2); + + index.clear_all_param_declarations(); + let (reextracted, logs_considered) = index.reextract_params_from_retained_logs(); + + assert_eq!( + logs_considered, 1, + "only the retained log line should be re-scanned" + ); + assert_eq!(reextracted, 1, "the evicted line's key can't come back"); + let declarations: Vec<_> = index.param_declarations_range().collect(); + assert_eq!(declarations[0].0, "x1"); +} + +#[test] +fn declarations_stay_within_their_range() { + let mut index = EngineSnapshot::default(); + index.index.insert( + IndexKey::IngestedLog { + sequence: 0, + timestamp: Timestamp::from_millis(0), + }, + DataValue::LogLiteral { + text: "hello".to_string(), + }, + ); + declare_param( + &mut index, + request("level", r"level=(?P\w+)", "level"), + ) + .unwrap(); + index.index.insert( + IndexKey::MetricDeclaration { + name: "requests".to_string(), + }, + DataValue::MetricDeclaration { + message_match_regex: "x".to_string(), + label_params: vec![], + value_param: None, matched_labels: None, unmatched_labels: None, - } + }, + ); + + let declarations: Vec<_> = index.param_declarations_range().collect(); + assert_eq!(declarations.len(), 1); + assert_eq!(declarations[0].0, "level"); + + for key in index + .index + .range(IndexKey::LogParamSplit..IndexKey::ParamMetricSplit) + .map(|(k, _)| k) + { + assert!(matches!(key, IndexKey::ParamDeclaration { .. })); } +} - #[test] - fn declares_valid_metric() { - let mut index = EngineSnapshot::default(); - let pattern = r"request path=(?P\S+) duration=(?P\d+)"; - declare_test_param(&mut index, "path", pattern, "path", MetricPrimitiveKind::String); - declare_test_param(&mut index, "duration", pattern, "duration", MetricPrimitiveKind::Number); +#[test] +fn removes_declaration() { + let mut index = EngineSnapshot::default(); + declare_param( + &mut index, + request("level", r"level=(?P\w+)", "level"), + ) + .unwrap(); + assert!( + index + .index + .remove(&IndexKey::ParamDeclaration { + name: "level".to_string() + }) + .is_some() + ); + assert!(index.param_declarations_range().next().is_none()); +} - let result = index.declare_metric(metric_request("requests", pattern, vec!["path", "duration"])); - assert!(result.is_ok()); +fn declare_test_param( + index: &mut EngineSnapshot, + name: &str, + pattern: &str, + capture_group: &str, + kind: MetricPrimitiveKind, +) { + declare_param( + index, + DeclareParamRequest { + name: name.to_string(), + message_match_regex: pattern.to_string(), + capture_group: capture_group.to_string(), + json_field_path: None, + kind, + elapsed_since_last: false, + auto_derived: false, + log_stream_gated: false, + }, + ) + .unwrap(); +} - let declarations: Vec<_> = index.metric_declarations_range().collect(); - assert_eq!(declarations.len(), 1); - assert_eq!(declarations[0].0, "requests"); - assert_eq!(declarations[0].1, pattern); - assert_eq!(declarations[0].2.len(), 2); +fn declare_elapsed_since_last_param(index: &mut EngineSnapshot, name: &str) { + declare_param( + index, + DeclareParamRequest { + name: name.to_string(), + message_match_regex: ".".to_string(), + capture_group: String::new(), + json_field_path: None, + kind: MetricPrimitiveKind::Duration, + elapsed_since_last: true, + auto_derived: false, + log_stream_gated: false, + }, + ) + .unwrap(); +} + +fn metric_request(name: &str, pattern: &str, label_params: Vec<&str>) -> DeclareMetricRequest { + metric_request_with_value(name, pattern, label_params, None) +} + +fn metric_request_with_value( + name: &str, + pattern: &str, + label_params: Vec<&str>, + value_param: Option<&str>, +) -> DeclareMetricRequest { + DeclareMetricRequest { + name: name.to_string(), + message_match_regex: pattern.to_string(), + label_params: label_params.into_iter().map(str::to_string).collect(), + value_param: value_param.map(str::to_string), + matched_labels: None, + unmatched_labels: None, } +} - #[test] - fn rejects_invalid_metric_regex() { - let mut index = EngineSnapshot::default(); - let result = index.declare_metric(metric_request("requests", "path=(", vec![])); - assert!(result.is_err()); - assert!(index.index.is_empty()); +#[test] +fn declares_valid_metric() { + let mut index = EngineSnapshot::default(); + let pattern = r"request path=(?P\S+) duration=(?P\d+)"; + declare_test_param( + &mut index, + "path", + pattern, + "path", + MetricPrimitiveKind::String, + ); + declare_test_param( + &mut index, + "duration", + pattern, + "duration", + MetricPrimitiveKind::Number, + ); + + let result = index.declare_metric(metric_request( + "requests", + pattern, + vec!["path", "duration"], + )); + assert!(result.is_ok()); + + let declarations: Vec<_> = index.metric_declarations_range().collect(); + assert_eq!(declarations.len(), 1); + assert_eq!(declarations[0].0, "requests"); + assert_eq!(declarations[0].1, pattern); + assert_eq!(declarations[0].2.len(), 2); +} + +#[test] +fn rejects_invalid_metric_regex() { + let mut index = EngineSnapshot::default(); + let result = index.declare_metric(metric_request("requests", "path=(", vec![])); + assert!(result.is_err()); + assert!(index.index.is_empty()); +} + +#[test] +fn rejects_metric_referencing_undeclared_param() { + let mut index = EngineSnapshot::default(); + let pattern = r"request path=(?P\S+)"; + declare_test_param( + &mut index, + "path", + pattern, + "path", + MetricPrimitiveKind::String, + ); + + let result = index.declare_metric(metric_request( + "requests", + pattern, + vec!["path", "duration"], + )); + assert!(result.is_err()); + assert!(result.unwrap_err().contains("duration")); + assert!(!index.index.contains_key(&IndexKey::MetricDeclaration { + name: "requests".to_string() + })); +} + +#[test] +fn metric_declarations_stay_within_their_range() { + let mut index = EngineSnapshot::default(); + index.index.insert( + IndexKey::IngestedLog { + sequence: 0, + timestamp: Timestamp::from_millis(0), + }, + DataValue::LogLiteral { + text: "hello".to_string(), + }, + ); + declare_param( + &mut index, + request("level", r"level=(?P\w+)", "level"), + ) + .unwrap(); + let pattern = r"duration=(?P\d+)"; + declare_test_param( + &mut index, + "duration", + pattern, + "duration", + MetricPrimitiveKind::Number, + ); + index + .declare_metric(metric_request("requests", pattern, vec!["duration"])) + .unwrap(); + + let declarations: Vec<_> = index.metric_declarations_range().collect(); + assert_eq!(declarations.len(), 1); + assert_eq!(declarations[0].0, "requests"); + + for key in index + .index + .range(IndexKey::ParamMetricSplit..IndexKey::MetricBucketSplit) + .map(|(k, _)| k) + { + assert!(matches!(key, IndexKey::MetricDeclaration { .. })); } +} - #[test] - fn rejects_metric_referencing_undeclared_param() { - let mut index = EngineSnapshot::default(); - let pattern = r"request path=(?P\S+)"; - declare_test_param(&mut index, "path", pattern, "path", MetricPrimitiveKind::String); +#[test] +fn removes_metric_declaration() { + let mut index = EngineSnapshot::default(); + let pattern = r"duration=(?P\d+)"; + declare_test_param( + &mut index, + "duration", + pattern, + "duration", + MetricPrimitiveKind::Number, + ); + index + .declare_metric(metric_request("requests", pattern, vec!["duration"])) + .unwrap(); + assert!( + index + .index + .remove(&IndexKey::MetricDeclaration { + name: "requests".to_string() + }) + .is_some() + ); + assert!(index.metric_declarations_range().next().is_none()); +} - let result = index.declare_metric(metric_request("requests", pattern, vec!["path", "duration"])); - assert!(result.is_err()); - assert!(result.unwrap_err().contains("duration")); - assert!(!index.index.contains_key(&IndexKey::MetricDeclaration { name: "requests".to_string() })); - } +#[test] +fn remove_metric_declaration_reports_whether_it_existed() { + let mut index = EngineSnapshot::default(); + declare_requests_metric(&mut index); + assert!(index.remove_metric_declaration("requests")); + assert!(!index.remove_metric_declaration("requests")); + assert!(!index.remove_metric_declaration("never-declared")); +} - #[test] - fn metric_declarations_stay_within_their_range() { - let mut index = EngineSnapshot::default(); - index.index.insert( - IndexKey::IngestedLog { sequence: 0, timestamp: Timestamp::from_millis(0) }, - DataValue::LogLiteral { text: "hello".to_string() }, - ); - declare_param(&mut index, request("level", r"level=(?P\w+)", "level")).unwrap(); - let pattern = r"duration=(?P\d+)"; - declare_test_param(&mut index, "duration", pattern, "duration", MetricPrimitiveKind::Number); - index.declare_metric(metric_request("requests", pattern, vec!["duration"])).unwrap(); +#[test] +fn remove_metric_declaration_purges_bucket_history_so_promql_reports_unknown_metric() { + let mut index = EngineSnapshot::default(); + declare_requests_metric(&mut index); + ingest_log(&mut index, 0, "request path=/a"); - let declarations: Vec<_> = index.metric_declarations_range().collect(); - assert_eq!(declarations.len(), 1); - assert_eq!(declarations[0].0, "requests"); + let parsed = PromqlQuery::parse("requests").unwrap(); + assert!(index.evaluate_promql(&parsed, None, None).unwrap().len() == 1); - for key in index.index.range(IndexKey::ParamMetricSplit..IndexKey::MetricBucketSplit).map(|(k, _)| k) { - assert!(matches!(key, IndexKey::MetricDeclaration { .. })); - } - } + assert!(index.remove_metric_declaration("requests")); + assert!(index.bucket_values("requests", None).next().is_none()); - #[test] - fn removes_metric_declaration() { - let mut index = EngineSnapshot::default(); - let pattern = r"duration=(?P\d+)"; - declare_test_param(&mut index, "duration", pattern, "duration", MetricPrimitiveKind::Number); - index.declare_metric(metric_request("requests", pattern, vec!["duration"])).unwrap(); - assert!(index.index.remove(&IndexKey::MetricDeclaration { name: "requests".to_string() }).is_some()); - assert!(index.metric_declarations_range().next().is_none()); - } + let err = index.evaluate_promql(&parsed, None, None).unwrap_err(); + assert!( + err.contains("unknown metric"), + "unexpected error message: {err}" + ); +} - #[test] - fn remove_metric_declaration_reports_whether_it_existed() { - let mut index = EngineSnapshot::default(); - declare_requests_metric(&mut index); - assert!(index.remove_metric_declaration("requests")); - assert!(!index.remove_metric_declaration("requests")); - assert!(!index.remove_metric_declaration("never-declared")); - } +#[test] +fn undeclared_metric_with_bucket_data_is_still_queryable_via_promql() { + let mut index = EngineSnapshot::default(); + index.push_bucket_value( + "scraped_metric", + BTreeMap::new(), + 0, + Timestamp::from_millis(0), + MetricPrimitive::Number(Number::Float(1.0)), + ); + let parsed = PromqlQuery::parse("scraped_metric").unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); + assert_eq!(series.len(), 1); +} - #[test] - fn remove_metric_declaration_purges_bucket_history_so_promql_reports_unknown_metric() { - let mut index = EngineSnapshot::default(); - declare_requests_metric(&mut index); - ingest_log(&mut index, 0, "request path=/a"); +#[test] +fn promql_rejects_metric_with_no_declaration_and_no_data() { + let index = EngineSnapshot::default(); + let parsed = PromqlQuery::parse("never_declared").unwrap(); + let err = index.evaluate_promql(&parsed, None, None).unwrap_err(); + assert!( + err.contains("unknown metric"), + "unexpected error message: {err}" + ); +} - let parsed = PromqlQuery::parse("requests").unwrap(); - assert!(index.evaluate_promql(&parsed, None, None).unwrap().len() == 1); +#[test] +fn update_metric_rejects_when_no_declaration_exists_yet() { + let mut index = EngineSnapshot::default(); + let result = index.update_metric( + "requests", + metric_request("requests", r"request path=(?P\S+)", vec![]), + ); + assert!(result.is_err()); + assert!(result.unwrap_err().contains("requests")); +} - assert!(index.remove_metric_declaration("requests")); - assert!(index.bucket_values("requests", None).next().is_none()); +#[test] +fn update_metric_reuses_declare_metric_validation() { + let mut index = EngineSnapshot::default(); + index + .declare_metric(metric_request( + "requests", + r"request path=(?P\S+)", + vec![], + )) + .unwrap(); - let err = index.evaluate_promql(&parsed, None, None).unwrap_err(); - assert!(err.contains("unknown metric"), "unexpected error message: {err}"); - } + let result = index.update_metric( + "requests", + metric_request("requests", r"request path=(?P\S+)", vec!["duration"]), + ); + assert!(result.is_err()); + assert!(result.unwrap_err().contains("duration")); +} - #[test] - fn undeclared_metric_with_bucket_data_is_still_queryable_via_promql() { - let mut index = EngineSnapshot::default(); - index.push_bucket_value("scraped_metric", - BTreeMap::new(), - 0, - Timestamp::from_millis(0), - MetricPrimitive::Number(Number::Float(1.0)), - ); - let parsed = PromqlQuery::parse("scraped_metric").unwrap(); - let series = index.evaluate_promql(&parsed, None, None).unwrap(); - assert_eq!(series.len(), 1); - } +#[test] +fn update_metric_replaces_declaration_in_place_without_duplicating() { + let mut index = EngineSnapshot::default(); + let pattern = r"request path=(?P\S+) duration=(?P\d+)"; + declare_test_param( + &mut index, + "path", + pattern, + "path", + MetricPrimitiveKind::String, + ); + index + .declare_metric(metric_request( + "requests", + r"request path=(?P\S+)", + vec!["path"], + )) + .unwrap(); - #[test] - fn promql_rejects_metric_with_no_declaration_and_no_data() { - let index = EngineSnapshot::default(); - let parsed = PromqlQuery::parse("never_declared").unwrap(); - let err = index.evaluate_promql(&parsed, None, None).unwrap_err(); - assert!(err.contains("unknown metric"), "unexpected error message: {err}"); - } + declare_test_param( + &mut index, + "duration", + pattern, + "duration", + MetricPrimitiveKind::Number, + ); + let result = index.update_metric( + "requests", + metric_request("requests", pattern, vec!["path", "duration"]), + ); + assert!(result.is_ok()); - #[test] - fn update_metric_rejects_when_no_declaration_exists_yet() { - let mut index = EngineSnapshot::default(); - let result = index.update_metric("requests", - metric_request("requests", r"request path=(?P\S+)", vec![]), - ); - assert!(result.is_err()); - assert!(result.unwrap_err().contains("requests")); - } + let declarations: Vec<_> = index.metric_declarations_range().collect(); + assert_eq!(declarations.len(), 1); + assert_eq!(declarations[0].0, "requests"); + assert_eq!(declarations[0].2.len(), 2); +} - #[test] - fn update_metric_reuses_declare_metric_validation() { - let mut index = EngineSnapshot::default(); - index.declare_metric(metric_request("requests", r"request path=(?P\S+)", vec![])).unwrap(); +#[test] +fn update_metric_rebackfills_bucket_data_from_the_new_declaration() { + let mut index = EngineSnapshot::default(); + ingest_log(&mut index, 0, "request path=/a"); + ingest_log(&mut index, 1, "request path=/b"); + declare_test_param( + &mut index, + "path", + r"request path=(?P\S+)", + "path", + MetricPrimitiveKind::String, + ); + index + .declare_metric(metric_request( + "requests", + r"request path=(?P\S+)", + vec!["path"], + )) + .unwrap(); + assert_eq!(index.bucket_values("requests", None).count(), 2); - let result = index.update_metric("requests", - metric_request("requests", r"request path=(?P\S+)", vec!["duration"]), - ); - assert!(result.is_err()); - assert!(result.unwrap_err().contains("duration")); - } + index + .update_metric( + "requests", + metric_request("requests", r"request path=(?P/a)", vec!["path"]), + ) + .unwrap(); + assert_eq!(index.bucket_values("requests", None).count(), 1); +} - #[test] - fn update_metric_replaces_declaration_in_place_without_duplicating() { - let mut index = EngineSnapshot::default(); - let pattern = r"request path=(?P\S+) duration=(?P\d+)"; - declare_test_param(&mut index, "path", pattern, "path", MetricPrimitiveKind::String); - index.declare_metric(metric_request("requests", r"request path=(?P\S+)", vec!["path"])).unwrap(); +#[test] +fn update_metric_can_rename_and_rejects_a_rename_onto_an_existing_name() { + let mut index = EngineSnapshot::default(); + index + .declare_metric(metric_request( + "requests", + r"request path=(?P\S+)", + vec![], + )) + .unwrap(); + index + .declare_metric(metric_request( + "errors", + r"error code=(?P\d+)", + vec![], + )) + .unwrap(); - declare_test_param(&mut index, "duration", pattern, "duration", MetricPrimitiveKind::Number); - let result = index.update_metric("requests", metric_request("requests", pattern, vec!["path", "duration"])); - assert!(result.is_ok()); + let result = index.update_metric( + "requests", + metric_request("errors", r"request path=(?P\S+)", vec![]), + ); + assert!(result.is_err()); - let declarations: Vec<_> = index.metric_declarations_range().collect(); - assert_eq!(declarations.len(), 1); - assert_eq!(declarations[0].0, "requests"); - assert_eq!(declarations[0].2.len(), 2); - } - - #[test] - fn update_metric_rebackfills_bucket_data_from_the_new_declaration() { - let mut index = EngineSnapshot::default(); - ingest_log(&mut index, 0, "request path=/a"); - ingest_log(&mut index, 1, "request path=/b"); - declare_test_param(&mut index, "path", r"request path=(?P\S+)", "path", MetricPrimitiveKind::String); - index.declare_metric(metric_request("requests", r"request path=(?P\S+)", vec!["path"])).unwrap(); - assert_eq!(index.bucket_values("requests", None).count(), 2); - - index.update_metric("requests", metric_request("requests", r"request path=(?P/a)", vec!["path"])).unwrap(); - assert_eq!(index.bucket_values("requests", None).count(), 1); - } - - #[test] - fn update_metric_can_rename_and_rejects_a_rename_onto_an_existing_name() { - let mut index = EngineSnapshot::default(); - index.declare_metric(metric_request("requests", r"request path=(?P\S+)", vec![])).unwrap(); - index.declare_metric(metric_request("errors", r"error code=(?P\d+)", vec![])).unwrap(); - - let result = index.update_metric("requests", - metric_request("errors", r"request path=(?P\S+)", vec![]), - ); - assert!(result.is_err()); - - index.update_metric("requests", + index + .update_metric( + "requests", metric_request("http_requests", r"request path=(?P\S+)", vec![]), ) .unwrap(); - let names: Vec<_> = index.metric_declarations_range().map(|(name, ..)| name.to_string()).collect(); - assert!(names.contains(&"http_requests".to_string())); - assert!(!names.contains(&"requests".to_string())); - } + let names: Vec<_> = index + .metric_declarations_range() + .map(|(name, ..)| name.to_string()) + .collect(); + assert!(names.contains(&"http_requests".to_string())); + assert!(!names.contains(&"requests".to_string())); +} - fn ingest_log(index: &mut EngineSnapshot, sequence: u64, text: &str) { - let timestamp = Timestamp::from_millis(sequence as i64); - index.index.insert( - IndexKey::IngestedLog { sequence, timestamp }, - DataValue::LogLiteral { text: text.to_string() }, - ); - index.apply_all_metrics(sequence, timestamp, text); - } +fn ingest_log(index: &mut EngineSnapshot, sequence: u64, text: &str) { + let timestamp = Timestamp::from_millis(sequence as i64); + index.index.insert( + IndexKey::IngestedLog { + sequence, + timestamp, + }, + DataValue::LogLiteral { + text: text.to_string(), + }, + ); + index.apply_all_metrics(sequence, timestamp, text); +} - #[test] - fn declaring_metric_backfills_matching_logs() { - let mut index = EngineSnapshot::default(); - ingest_log(&mut index, 0, "request path=/a duration=1"); - ingest_log(&mut index, 1, "request path=/b duration=2"); - ingest_log(&mut index, 2, "unrelated line"); +#[test] +fn declaring_metric_backfills_matching_logs() { + let mut index = EngineSnapshot::default(); + ingest_log(&mut index, 0, "request path=/a duration=1"); + ingest_log(&mut index, 1, "request path=/b duration=2"); + ingest_log(&mut index, 2, "unrelated line"); - let pattern = r"request path=(?P\S+) duration=(?P\d+)"; - declare_test_param(&mut index, "path", pattern, "path", MetricPrimitiveKind::String); - index.declare_metric(metric_request("requests", pattern, vec!["path"])).unwrap(); - - let values: Vec<_> = index.bucket_values("requests", None).collect(); - assert_eq!(values.len(), 2); - } - - #[test] - fn metric_param_extracts_from_json_field_path() { - let mut index = EngineSnapshot::default(); - ingest_log(&mut index, 0, r#"{"request":{"path":"/a"}}"#); - ingest_log(&mut index, 1, "not json"); - ingest_log(&mut index, 2, r#"{"request":{}}"#); - - declare_param( - &mut index, - DeclareParamRequest { - name: "path".to_string(), - message_match_regex: r".".to_string(), - capture_group: String::new(), - json_field_path: Some("request.path".to_string()), - kind: MetricPrimitiveKind::String, - elapsed_since_last: false, - auto_derived: false, - log_stream_gated: false, - }, - ) + let pattern = r"request path=(?P\S+) duration=(?P\d+)"; + declare_test_param( + &mut index, + "path", + pattern, + "path", + MetricPrimitiveKind::String, + ); + index + .declare_metric(metric_request("requests", pattern, vec!["path"])) .unwrap(); - index.declare_metric(metric_request("requests", r".", vec!["path"])).unwrap(); - let values: Vec<_> = index.bucket_values("requests", None).collect(); - assert_eq!(values.len(), 1); - assert_eq!( - values[0].0.get("path"), - Some(&MetricPrimitive::String("/a".to_string())) - ); - } + let values: Vec<_> = index.bucket_values("requests", None).collect(); + assert_eq!(values.len(), 2); +} - #[test] - fn ingesting_after_declaration_updates_buckets_live() { - let mut index = EngineSnapshot::default(); - declare_test_param(&mut index, "path", r"request path=(?P\S+)", "path", MetricPrimitiveKind::String); - index.declare_metric(metric_request("requests", r"request path=(?P\S+)", vec!["path"])).unwrap(); +#[test] +fn metric_param_extracts_from_json_field_path() { + let mut index = EngineSnapshot::default(); + ingest_log(&mut index, 0, r#"{"request":{"path":"/a"}}"#); + ingest_log(&mut index, 1, "not json"); + ingest_log(&mut index, 2, r#"{"request":{}}"#); - ingest_log(&mut index, 0, "request path=/a"); + declare_param( + &mut index, + DeclareParamRequest { + name: "path".to_string(), + message_match_regex: r".".to_string(), + capture_group: String::new(), + json_field_path: Some("request.path".to_string()), + kind: MetricPrimitiveKind::String, + elapsed_since_last: false, + auto_derived: false, + log_stream_gated: false, + }, + ) + .unwrap(); + index + .declare_metric(metric_request("requests", r".", vec!["path"])) + .unwrap(); - let values: Vec<_> = index.bucket_values("requests", None).collect(); - assert_eq!(values.len(), 1); - assert_eq!(values[0].1.0, 0); - } + let values: Vec<_> = index.bucket_values("requests", None).collect(); + assert_eq!(values.len(), 1); + assert_eq!( + values[0].0.get("path"), + Some(&MetricPrimitive::String("/a".to_string())) + ); +} - #[test] - fn non_matching_log_leaves_bucket_range_untouched() { - let mut index = EngineSnapshot::default(); - declare_test_param(&mut index, "path", r"request path=(?P\S+)", "path", MetricPrimitiveKind::String); - index.declare_metric(metric_request("requests", r"request path=(?P\S+)", vec!["path"])).unwrap(); +#[test] +fn ingesting_after_declaration_updates_buckets_live() { + let mut index = EngineSnapshot::default(); + declare_test_param( + &mut index, + "path", + r"request path=(?P\S+)", + "path", + MetricPrimitiveKind::String, + ); + index + .declare_metric(metric_request( + "requests", + r"request path=(?P\S+)", + vec!["path"], + )) + .unwrap(); - ingest_log(&mut index, 0, "totally unrelated line"); + ingest_log(&mut index, 0, "request path=/a"); - assert!( - index - .index - .range(IndexKey::MetricBucketSplit..IndexKey::End) - .find(|(key, _)| matches!(key, IndexKey::Bucket { .. })) - .is_none() - ); - } + let values: Vec<_> = index.bucket_values("requests", None).collect(); + assert_eq!(values.len(), 1); + assert_eq!(values[0].1.0, 0); +} - #[test] - fn bucket_values_preserve_insertion_order() { - let mut index = EngineSnapshot::default(); - declare_test_param(&mut index, "path", r"request path=(?P\S+)", "path", MetricPrimitiveKind::String); - index.declare_metric(metric_request("requests", r"request path=(?P\S+)", vec!["path"])).unwrap(); +#[test] +fn non_matching_log_leaves_bucket_range_untouched() { + let mut index = EngineSnapshot::default(); + declare_test_param( + &mut index, + "path", + r"request path=(?P\S+)", + "path", + MetricPrimitiveKind::String, + ); + index + .declare_metric(metric_request( + "requests", + r"request path=(?P\S+)", + vec!["path"], + )) + .unwrap(); - ingest_log(&mut index, 5, "request path=/a"); - ingest_log(&mut index, 2, "request path=/a"); - ingest_log(&mut index, 9, "request path=/a"); + ingest_log(&mut index, 0, "totally unrelated line"); - let sequences: Vec = index.bucket_values("requests", None) - .map(|(_, (sequence, _, _))| *sequence) - .collect(); - assert_eq!(sequences, vec![5, 2, 9]); - } - - #[test] - fn differing_label_combinations_produce_distinct_buckets() { - let mut index = EngineSnapshot::default(); - declare_test_param(&mut index, "path", r"request path=(?P\S+)", "path", MetricPrimitiveKind::String); - index.declare_metric(metric_request("requests", r"request path=(?P\S+)", vec!["path"])).unwrap(); - - ingest_log(&mut index, 0, "request path=/a"); - ingest_log(&mut index, 1, "request path=/b"); - - let bucket_keys: Vec<_> = index + assert!( + index .index .range(IndexKey::MetricBucketSplit..IndexKey::End) - .filter(|(key, _)| matches!(key, IndexKey::Bucket { .. })) - .collect(); - assert_eq!(bucket_keys.len(), 2); - for (_, value) in &bucket_keys { - match value { - DataValue::BucketValue { values } => assert_eq!(values.len(), 1), - _ => panic!("expected BucketValue"), - } + .find(|(key, _)| matches!(key, IndexKey::Bucket { .. })) + .is_none() + ); +} + +#[test] +fn bucket_values_preserve_insertion_order() { + let mut index = EngineSnapshot::default(); + declare_test_param( + &mut index, + "path", + r"request path=(?P\S+)", + "path", + MetricPrimitiveKind::String, + ); + index + .declare_metric(metric_request( + "requests", + r"request path=(?P\S+)", + vec!["path"], + )) + .unwrap(); + + ingest_log(&mut index, 5, "request path=/a"); + ingest_log(&mut index, 2, "request path=/a"); + ingest_log(&mut index, 9, "request path=/a"); + + let sequences: Vec = index + .bucket_values("requests", None) + .map(|(_, (sequence, _, _))| *sequence) + .collect(); + assert_eq!(sequences, vec![5, 2, 9]); +} + +#[test] +fn differing_label_combinations_produce_distinct_buckets() { + let mut index = EngineSnapshot::default(); + declare_test_param( + &mut index, + "path", + r"request path=(?P\S+)", + "path", + MetricPrimitiveKind::String, + ); + index + .declare_metric(metric_request( + "requests", + r"request path=(?P\S+)", + vec!["path"], + )) + .unwrap(); + + ingest_log(&mut index, 0, "request path=/a"); + ingest_log(&mut index, 1, "request path=/b"); + + let bucket_keys: Vec<_> = index + .index + .range(IndexKey::MetricBucketSplit..IndexKey::End) + .filter(|(key, _)| matches!(key, IndexKey::Bucket { .. })) + .collect(); + assert_eq!(bucket_keys.len(), 2); + for (_, value) in &bucket_keys { + match value { + DataValue::BucketValue { values } => assert_eq!(values.len(), 1), + _ => panic!("expected BucketValue"), } } +} - #[test] - fn single_log_line_can_match_multiple_metrics() { - let mut index = EngineSnapshot::default(); - let pattern = r"request path=(?P\S+) duration=(?P\d+)"; - declare_test_param(&mut index, "path", pattern, "path", MetricPrimitiveKind::String); - declare_test_param(&mut index, "duration", pattern, "duration", MetricPrimitiveKind::Number); - index.declare_metric(metric_request("requests", pattern, vec!["path"])).unwrap(); - index.declare_metric(metric_request("durations", pattern, vec!["duration"])).unwrap(); +#[test] +fn single_log_line_can_match_multiple_metrics() { + let mut index = EngineSnapshot::default(); + let pattern = r"request path=(?P\S+) duration=(?P\d+)"; + declare_test_param( + &mut index, + "path", + pattern, + "path", + MetricPrimitiveKind::String, + ); + declare_test_param( + &mut index, + "duration", + pattern, + "duration", + MetricPrimitiveKind::Number, + ); + index + .declare_metric(metric_request("requests", pattern, vec!["path"])) + .unwrap(); + index + .declare_metric(metric_request("durations", pattern, vec!["duration"])) + .unwrap(); - ingest_log(&mut index, 0, "request path=/a duration=5"); + ingest_log(&mut index, 0, "request path=/a duration=5"); - assert_eq!(index.bucket_values("requests", None).count(), 1); - assert_eq!(index.bucket_values("durations", None).count(), 1); - } + assert_eq!(index.bucket_values("requests", None).count(), 1); + assert_eq!(index.bucket_values("durations", None).count(), 1); +} - fn declare_requests_metric(index: &mut EngineSnapshot) { - declare_test_param(index, "path", r"request path=(?P\S+)", "path", MetricPrimitiveKind::String); - index.declare_metric(metric_request("requests", r"request path=(?P\S+)", vec!["path"])).unwrap(); - } +fn declare_requests_metric(index: &mut EngineSnapshot) { + declare_test_param( + index, + "path", + r"request path=(?P\S+)", + "path", + MetricPrimitiveKind::String, + ); + index + .declare_metric(metric_request( + "requests", + r"request path=(?P\S+)", + vec!["path"], + )) + .unwrap(); +} - #[test] - fn promql_query_selects_matching_label_combination() { - let mut index = EngineSnapshot::default(); - declare_requests_metric(&mut index); - ingest_log(&mut index, 0, "request path=/a"); - ingest_log(&mut index, 1, "request path=/b"); - ingest_log(&mut index, 2, "request path=/a"); +#[test] +fn promql_query_selects_matching_label_combination() { + let mut index = EngineSnapshot::default(); + declare_requests_metric(&mut index); + ingest_log(&mut index, 0, "request path=/a"); + ingest_log(&mut index, 1, "request path=/b"); + ingest_log(&mut index, 2, "request path=/a"); - let parsed = PromqlQuery::parse(r#"requests{path="/a"}"#).unwrap(); - let series = index.evaluate_promql(&parsed, None, None).unwrap(); + let parsed = PromqlQuery::parse(r#"requests{path="/a"}"#).unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); - assert_eq!(series.len(), 1); - assert_eq!(series[0].labels.get("path").map(String::as_str), Some("/a")); - assert_eq!(series[0].points, vec![(0, 1.0), (2, 1.0)]); - } + assert_eq!(series.len(), 1); + assert_eq!(series[0].labels.get("path").map(String::as_str), Some("/a")); + assert_eq!(series[0].points, vec![(0, 1.0), (2, 1.0)]); +} - #[test] - fn promql_query_with_no_matching_labels_returns_no_series() { - let mut index = EngineSnapshot::default(); - declare_requests_metric(&mut index); - ingest_log(&mut index, 0, "request path=/a"); +#[test] +fn promql_query_with_no_matching_labels_returns_no_series() { + let mut index = EngineSnapshot::default(); + declare_requests_metric(&mut index); + ingest_log(&mut index, 0, "request path=/a"); - let parsed = PromqlQuery::parse(r#"requests{path="/nope"}"#).unwrap(); - let series = index.evaluate_promql(&parsed, None, None).unwrap(); - assert!(series.is_empty()); - } + let parsed = PromqlQuery::parse(r#"requests{path="/nope"}"#).unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); + assert!(series.is_empty()); +} - #[test] - fn promql_sum_aggregates_across_label_combinations() { - let mut index = EngineSnapshot::default(); - declare_requests_metric(&mut index); - ingest_log(&mut index, 0, "request path=/a"); - ingest_log(&mut index, 0, "request path=/b"); +#[test] +fn promql_sum_aggregates_across_label_combinations() { + let mut index = EngineSnapshot::default(); + declare_requests_metric(&mut index); + ingest_log(&mut index, 0, "request path=/a"); + ingest_log(&mut index, 0, "request path=/b"); - let parsed = PromqlQuery::parse("sum(requests)").unwrap(); - let series = index.evaluate_promql(&parsed, None, None).unwrap(); + let parsed = PromqlQuery::parse("sum(requests)").unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); - assert_eq!(series.len(), 1); - assert!(series[0].labels.is_empty()); - assert_eq!(series[0].points, vec![(0, 2.0)]); - } + assert_eq!(series.len(), 1); + assert!(series[0].labels.is_empty()); + assert_eq!(series[0].points, vec![(0, 2.0)]); +} - #[test] - fn promql_sum_by_preserves_the_named_label_dimension() { - let mut index = EngineSnapshot::default(); - declare_requests_metric(&mut index); - ingest_log(&mut index, 0, "request path=/a"); - ingest_log(&mut index, 0, "request path=/b"); - ingest_log(&mut index, 1, "request path=/a"); +#[test] +fn promql_sum_by_preserves_the_named_label_dimension() { + let mut index = EngineSnapshot::default(); + declare_requests_metric(&mut index); + ingest_log(&mut index, 0, "request path=/a"); + ingest_log(&mut index, 0, "request path=/b"); + ingest_log(&mut index, 1, "request path=/a"); - let parsed = PromqlQuery::parse("sum(requests) by (path)").unwrap(); - let mut series = index.evaluate_promql(&parsed, None, None).unwrap(); - series.sort_by(|a, b| a.labels.get("path").cmp(&b.labels.get("path"))); + let parsed = PromqlQuery::parse("sum(requests) by (path)").unwrap(); + let mut series = index.evaluate_promql(&parsed, None, None).unwrap(); + series.sort_by(|a, b| a.labels.get("path").cmp(&b.labels.get("path"))); - assert_eq!(series.len(), 2); - assert_eq!(series[0].labels.get("path").map(String::as_str), Some("/a")); - assert_eq!(series[0].points, vec![(0, 1.0), (1, 1.0)]); - assert_eq!(series[1].labels.get("path").map(String::as_str), Some("/b")); - assert_eq!(series[1].points, vec![(0, 1.0)]); - } + assert_eq!(series.len(), 2); + assert_eq!(series[0].labels.get("path").map(String::as_str), Some("/a")); + assert_eq!(series[0].points, vec![(0, 1.0), (1, 1.0)]); + assert_eq!(series[1].labels.get("path").map(String::as_str), Some("/b")); + assert_eq!(series[1].points, vec![(0, 1.0)]); +} - fn declare_string_valued_metric(index: &mut EngineSnapshot) { - declare_test_param( - index, - "status", +fn declare_string_valued_metric(index: &mut EngineSnapshot) { + declare_test_param( + index, + "status", + r"request status=(?P\S+)", + "status", + MetricPrimitiveKind::String, + ); + index + .declare_metric(metric_request_with_value( + "statuses", r"request status=(?P\S+)", - "status", - MetricPrimitiveKind::String, - ); - index.declare_metric(metric_request_with_value( - "statuses", - r"request status=(?P\S+)", - vec![], - Some("status"), - ), - ) + vec![], + Some("status"), + )) .unwrap(); - } +} - fn declare_requests_by_host_metric(index: &mut EngineSnapshot) { - declare_test_param( - index, - "path", +fn declare_requests_by_host_metric(index: &mut EngineSnapshot) { + declare_test_param( + index, + "path", + r"request path=(?P\S+) host=(?P\S+)", + "path", + MetricPrimitiveKind::String, + ); + declare_test_param( + index, + "host", + r"request path=(?P\S+) host=(?P\S+)", + "host", + MetricPrimitiveKind::String, + ); + index + .declare_metric(metric_request( + "requests", r"request path=(?P\S+) host=(?P\S+)", - "path", - MetricPrimitiveKind::String, - ); - declare_test_param( - index, - "host", - r"request path=(?P\S+) host=(?P\S+)", - "host", - MetricPrimitiveKind::String, - ); - index.declare_metric(metric_request( - "requests", - r"request path=(?P\S+) host=(?P\S+)", - vec!["path", "host"], - ), - ) + vec!["path", "host"], + )) .unwrap(); +} + +#[test] +fn sum_over_string_kind_metric_is_rejected_with_an_actionable_error() { + let mut index = EngineSnapshot::default(); + declare_string_valued_metric(&mut index); + ingest_log(&mut index, 0, "request status=ok"); + + let parsed = PromqlQuery::parse("sum(statuses)").unwrap(); + let err = index.evaluate_promql(&parsed, None, None).unwrap_err(); + + assert!( + err.contains("statuses"), + "error should name the offending metric: {err}" + ); + assert!( + err.to_lowercase().contains("string"), + "error should explain why: {err}" + ); +} + +#[test] +fn avg_over_string_kind_metric_is_rejected_with_an_actionable_error() { + let mut index = EngineSnapshot::default(); + declare_string_valued_metric(&mut index); + ingest_log(&mut index, 0, "request status=ok"); + + let parsed = PromqlQuery::parse("avg(statuses)").unwrap(); + let err = index.evaluate_promql(&parsed, None, None).unwrap_err(); + + assert!(err.contains("statuses")); +} + +#[test] +fn count_over_string_kind_metric_counts_occurrences_by_name_not_by_accident() { + let mut index = EngineSnapshot::default(); + declare_string_valued_metric(&mut index); + ingest_log(&mut index, 0, "request status=ok"); + ingest_log(&mut index, 1, "request status=error"); + + let parsed = PromqlQuery::parse("count(statuses)").unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); + + assert_eq!(series.len(), 1); + assert!(series[0].labels.is_empty()); + assert_eq!(series[0].points, vec![(0, 1.0), (1, 1.0)]); +} + +#[test] +fn sum_over_numeric_kind_metric_is_unaffected_by_the_string_kind_guard() { + let mut index = EngineSnapshot::default(); + declare_requests_metric(&mut index); + ingest_log(&mut index, 0, "request path=/a"); + ingest_log(&mut index, 0, "request path=/b"); + + let parsed = PromqlQuery::parse("sum(requests)").unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); + + assert_eq!(series.len(), 1); + assert_eq!(series[0].points, vec![(0, 2.0)]); +} + +#[test] +fn sum_of_rate_over_string_kind_metric_is_still_allowed_since_rate_only_counts_occurrences() { + let mut index = EngineSnapshot::default(); + declare_string_valued_metric(&mut index); + ingest_log(&mut index, 0, "request status=ok"); + ingest_log(&mut index, 1000, "request status=error"); + + let parsed = PromqlQuery::parse("sum(rate(statuses[2s]))").unwrap(); + let series = index.evaluate_promql(&parsed, None, None); + + assert!( + series.is_ok(), + "sum(rate(...)) never reads the coerced value, so it should not be rejected: {series:?}" + ); +} + +#[test] +fn bare_query_over_string_kind_metric_is_still_allowed() { + let mut index = EngineSnapshot::default(); + declare_string_valued_metric(&mut index); + ingest_log(&mut index, 0, "request status=ok"); + + let parsed = PromqlQuery::parse("statuses").unwrap(); + let series = index.evaluate_promql(&parsed, None, None); + + assert!(series.is_ok()); +} + +#[test] +fn sum_by_merges_staggered_raw_timestamps_across_label_groups_instead_of_going_sparse() { + let mut index = EngineSnapshot::default(); + declare_requests_by_host_metric(&mut index); + ingest_log(&mut index, 0, "request path=/a host=web1"); + ingest_log(&mut index, 1, "request path=/a host=web2"); + ingest_log(&mut index, 2, "request path=/a host=web3"); + + let parsed = PromqlQuery::parse("sum(requests) by (path)").unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); + + assert_eq!(series.len(), 1); + assert_eq!(series[0].labels.get("path").map(String::as_str), Some("/a")); + let total: f64 = series[0].points.iter().map(|(_, v)| v).sum(); + assert_eq!( + total, 3.0, + "expected all three staggered occurrences to sum to 3.0, got {:?}", + series[0].points + ); + assert!( + series[0].points.len() < 3, + "expected staggered timestamps to be bucketed together rather than staying sparse/near-empty, got {:?}", + series[0].points + ); +} + +#[test] +fn avg_by_merges_staggered_raw_timestamps_across_label_groups() { + let mut index = EngineSnapshot::default(); + declare_requests_by_host_metric(&mut index); + ingest_log(&mut index, 0, "request path=/a host=web1"); + ingest_log(&mut index, 1, "request path=/a host=web2"); + + let parsed = PromqlQuery::parse("avg(requests) by (path)").unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); + + assert_eq!(series.len(), 1); + assert_eq!(series[0].labels.get("path").map(String::as_str), Some("/a")); + assert_eq!(series[0].points, vec![(0, 1.0)]); +} + +#[test] +fn sum_by_still_gives_a_single_label_group_its_own_line_at_full_resolution() { + let mut index = EngineSnapshot::default(); + declare_requests_metric(&mut index); + ingest_log(&mut index, 0, "request path=/a"); + ingest_log(&mut index, 1, "request path=/a"); + + let parsed = PromqlQuery::parse("sum(requests) by (path)").unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); + + assert_eq!(series.len(), 1); + assert_eq!(series[0].points, vec![(0, 1.0), (1, 1.0)]); +} + +#[test] +fn bare_rate_produces_a_windowed_point_per_timestamp_not_a_single_point() { + let mut index = EngineSnapshot::default(); + declare_requests_metric(&mut index); + ingest_log(&mut index, 0, "request path=/a"); + ingest_log(&mut index, 1000, "request path=/a"); + + let parsed = PromqlQuery::parse("rate(requests[2s])").unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); + + assert_eq!(series.len(), 1); + assert_eq!(series[0].points, vec![(0, 0.5), (1000, 1.0)]); +} + +#[test] +fn sum_of_rate_by_label_gives_each_label_value_its_own_line() { + let mut index = EngineSnapshot::default(); + declare_requests_metric(&mut index); + ingest_log(&mut index, 0, "request path=/a"); + ingest_log(&mut index, 1000, "request path=/a"); + ingest_log(&mut index, 0, "request path=/b"); + + let parsed = PromqlQuery::parse("sum(rate(requests[2s])) by (path)").unwrap(); + let mut series = index.evaluate_promql(&parsed, None, None).unwrap(); + series.sort_by(|a, b| a.labels.get("path").cmp(&b.labels.get("path"))); + + assert_eq!(series.len(), 2); + assert_eq!(series[0].labels.get("path").map(String::as_str), Some("/a")); + assert_eq!(series[0].points, vec![(0, 0.5), (1000, 1.0)]); + assert_eq!(series[1].labels.get("path").map(String::as_str), Some("/b")); + assert_eq!(series[1].points, vec![(0, 0.5)]); +} + +#[test] +fn rate_series_gets_a_trailing_zero_point_once_occurrences_stop_early_in_a_wider_range() { + let mut index = EngineSnapshot::default(); + declare_requests_metric(&mut index); + ingest_log(&mut index, 0, "request path=/a"); + ingest_log(&mut index, 1, "request path=/a"); + + let parsed = PromqlQuery::parse("rate(requests[2s])").unwrap(); + let series = index.evaluate_promql(&parsed, None, Some(100_000)).unwrap(); + + assert_eq!(series.len(), 1); + assert_eq!( + series[0].points, + vec![(0, 0.5), (1, 1.0), (100_000, 0.0)], + "the window (2s) is long since aged out by end_ms, so the appended boundary point must read 0" + ); +} + +#[test] +fn rate_series_gets_a_leading_zero_point_when_occurrences_start_late_in_a_wider_range() { + let mut index = EngineSnapshot::default(); + declare_requests_metric(&mut index); + ingest_log(&mut index, 50_000, "request path=/a"); + + let parsed = PromqlQuery::parse("rate(requests[2s])").unwrap(); + let series = index.evaluate_promql(&parsed, Some(0), None).unwrap(); + + assert_eq!(series[0].points, vec![(0, 0.0), (50_000, 0.5)]); +} + +#[test] +fn sum_by_rate_gets_boundary_points_per_group_across_a_wider_range() { + let mut index = EngineSnapshot::default(); + declare_requests_metric(&mut index); + ingest_log(&mut index, 0, "request path=/a"); + ingest_log(&mut index, 1, "request path=/a"); + ingest_log(&mut index, 0, "request path=/b"); + + let parsed = PromqlQuery::parse("sum(rate(requests[2s])) by (path)").unwrap(); + let mut series = index.evaluate_promql(&parsed, None, Some(100_000)).unwrap(); + series.sort_by(|a, b| a.labels.get("path").cmp(&b.labels.get("path"))); + + assert_eq!(series.len(), 2); + assert_eq!(series[0].points, vec![(0, 0.5), (1, 1.0), (100_000, 0.0)]); + assert_eq!(series[1].points, vec![(0, 0.5), (100_000, 0.0)]); +} + +#[test] +fn rate_series_skips_boundary_points_already_covered_by_real_data() { + let mut index = EngineSnapshot::default(); + declare_requests_metric(&mut index); + ingest_log(&mut index, 0, "request path=/a"); + ingest_log(&mut index, 1000, "request path=/a"); + + let parsed = PromqlQuery::parse("rate(requests[2s])").unwrap(); + let series = index.evaluate_promql(&parsed, Some(0), Some(1000)).unwrap(); + + assert_eq!(series[0].points, vec![(0, 0.5), (1000, 1.0)]); +} + +#[test] +fn dashboard_rate_and_alert_threshold_agree_on_same_windowed_data() { + let mut index = EngineSnapshot::default(); + declare_requests_metric(&mut index); + + let lines: Vec<(u64, &str)> = vec![ + (0, "request path=/a"), + (500, "request path=/a"), + (1000, "request path=/a"), + (1500, "request path=/a"), + (2000, "request path=/a"), + ]; + for (ms, text) in &lines { + ingest_log(&mut index, *ms, text); } - #[test] - fn sum_over_string_kind_metric_is_rejected_with_an_actionable_error() { - let mut index = EngineSnapshot::default(); - declare_string_valued_metric(&mut index); - ingest_log(&mut index, 0, "request status=ok"); + let parsed = PromqlQuery::parse("rate(requests[2s])").unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); + assert_eq!(series.len(), 1); + let (_, dashboard_rate) = *series[0].points.last().unwrap(); - let parsed = PromqlQuery::parse("sum(statuses)").unwrap(); - let err = index.evaluate_promql(&parsed, None, None).unwrap_err(); - - assert!(err.contains("statuses"), "error should name the offending metric: {err}"); - assert!(err.to_lowercase().contains("string"), "error should explain why: {err}"); - } - - #[test] - fn avg_over_string_kind_metric_is_rejected_with_an_actionable_error() { - let mut index = EngineSnapshot::default(); - declare_string_valued_metric(&mut index); - ingest_log(&mut index, 0, "request status=ok"); - - let parsed = PromqlQuery::parse("avg(statuses)").unwrap(); - let err = index.evaluate_promql(&parsed, None, None).unwrap_err(); - - assert!(err.contains("statuses")); - } - - #[test] - fn count_over_string_kind_metric_counts_occurrences_by_name_not_by_accident() { - let mut index = EngineSnapshot::default(); - declare_string_valued_metric(&mut index); - ingest_log(&mut index, 0, "request status=ok"); - ingest_log(&mut index, 1, "request status=error"); - - let parsed = PromqlQuery::parse("count(statuses)").unwrap(); - let series = index.evaluate_promql(&parsed, None, None).unwrap(); - - assert_eq!(series.len(), 1); - assert!(series[0].labels.is_empty()); - assert_eq!(series[0].points, vec![(0, 1.0), (1, 1.0)]); - } - - #[test] - fn sum_over_numeric_kind_metric_is_unaffected_by_the_string_kind_guard() { - let mut index = EngineSnapshot::default(); - declare_requests_metric(&mut index); - ingest_log(&mut index, 0, "request path=/a"); - ingest_log(&mut index, 0, "request path=/b"); - - let parsed = PromqlQuery::parse("sum(requests)").unwrap(); - let series = index.evaluate_promql(&parsed, None, None).unwrap(); - - assert_eq!(series.len(), 1); - assert_eq!(series[0].points, vec![(0, 2.0)]); - } - - #[test] - fn sum_of_rate_over_string_kind_metric_is_still_allowed_since_rate_only_counts_occurrences() { - let mut index = EngineSnapshot::default(); - declare_string_valued_metric(&mut index); - ingest_log(&mut index, 0, "request status=ok"); - ingest_log(&mut index, 1000, "request status=error"); - - let parsed = PromqlQuery::parse("sum(rate(statuses[2s]))").unwrap(); - let series = index.evaluate_promql(&parsed, None, None); - - assert!(series.is_ok(), "sum(rate(...)) never reads the coerced value, so it should not be rejected: {series:?}"); - } - - #[test] - fn bare_query_over_string_kind_metric_is_still_allowed() { - let mut index = EngineSnapshot::default(); - declare_string_valued_metric(&mut index); - ingest_log(&mut index, 0, "request status=ok"); - - let parsed = PromqlQuery::parse("statuses").unwrap(); - let series = index.evaluate_promql(&parsed, None, None); - - assert!(series.is_ok()); - } - - #[test] - fn sum_by_merges_staggered_raw_timestamps_across_label_groups_instead_of_going_sparse() { - let mut index = EngineSnapshot::default(); - declare_requests_by_host_metric(&mut index); - ingest_log(&mut index, 0, "request path=/a host=web1"); - ingest_log(&mut index, 1, "request path=/a host=web2"); - ingest_log(&mut index, 2, "request path=/a host=web3"); - - let parsed = PromqlQuery::parse("sum(requests) by (path)").unwrap(); - let series = index.evaluate_promql(&parsed, None, None).unwrap(); - - assert_eq!(series.len(), 1); - assert_eq!(series[0].labels.get("path").map(String::as_str), Some("/a")); - let total: f64 = series[0].points.iter().map(|(_, v)| v).sum(); - assert_eq!(total, 3.0, "expected all three staggered occurrences to sum to 3.0, got {:?}", series[0].points); - assert!( - series[0].points.len() < 3, - "expected staggered timestamps to be bucketed together rather than staying sparse/near-empty, got {:?}", - series[0].points + let rule = alerts::AlertRule { + id: "r1".to_string(), + name: "spike".to_string(), + enabled: true, + condition: alerts::AlertCondition::Threshold { + query: r#"regex("request path=/a")"#.to_string(), + op: CompareOp::Ge, + threshold: 0.0, + window_ms: 2_000, + }, + }; + let mut alert_states = HashMap::new(); + let mut last_message = String::new(); + for (ms, text) in &lines { + let timestamp = Timestamp::from_millis(*ms as i64); + let fired = alerts::evaluate_alerts( + std::slice::from_ref(&rule), + &mut alert_states, + &index, + *ms, + timestamp, + text, ); + assert_eq!(fired.len(), 1); + last_message = fired[0].1.clone(); } + let alert_count: f64 = last_message + .split("match count ") + .nth(1) + .and_then(|rest| rest.split_whitespace().next()) + .and_then(|token| token.parse().ok()) + .expect("fired message should report a match count"); - #[test] - fn avg_by_merges_staggered_raw_timestamps_across_label_groups() { - let mut index = EngineSnapshot::default(); - declare_requests_by_host_metric(&mut index); - ingest_log(&mut index, 0, "request path=/a host=web1"); - ingest_log(&mut index, 1, "request path=/a host=web2"); + let window_seconds = 2.0; + assert_eq!(dashboard_rate, 2.0); + assert_eq!(alert_count, 4.0); + assert_eq!(alert_count, dashboard_rate * window_seconds); +} - let parsed = PromqlQuery::parse("avg(requests) by (path)").unwrap(); - let series = index.evaluate_promql(&parsed, None, None).unwrap(); +#[test] +fn sum_of_rate_without_by_clause_still_collapses_to_one_series() { + let mut index = EngineSnapshot::default(); + declare_requests_metric(&mut index); + ingest_log(&mut index, 0, "request path=/a"); + ingest_log(&mut index, 0, "request path=/b"); - assert_eq!(series.len(), 1); - assert_eq!(series[0].labels.get("path").map(String::as_str), Some("/a")); - assert_eq!(series[0].points, vec![(0, 1.0)]); - } + let parsed = PromqlQuery::parse("sum(rate(requests[2s]))").unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); - #[test] - fn sum_by_still_gives_a_single_label_group_its_own_line_at_full_resolution() { - let mut index = EngineSnapshot::default(); - declare_requests_metric(&mut index); - ingest_log(&mut index, 0, "request path=/a"); - ingest_log(&mut index, 1, "request path=/a"); + assert_eq!(series.len(), 1); + assert!(series[0].labels.is_empty()); + assert_eq!(series[0].points, vec![(0, 1.0)]); +} - let parsed = PromqlQuery::parse("sum(requests) by (path)").unwrap(); - let series = index.evaluate_promql(&parsed, None, None).unwrap(); +#[test] +fn promql_range_bounds_filter_points_via_start_end_ms() { + let mut index = EngineSnapshot::default(); + declare_requests_metric(&mut index); + ingest_log(&mut index, 0, "request path=/a"); + ingest_log(&mut index, 5, "request path=/a"); + ingest_log(&mut index, 10, "request path=/a"); - assert_eq!(series.len(), 1); - assert_eq!(series[0].points, vec![(0, 1.0), (1, 1.0)]); - } + let parsed = PromqlQuery::parse("requests").unwrap(); + let series = index.evaluate_promql(&parsed, Some(1), Some(9)).unwrap(); - #[test] - fn bare_rate_produces_a_windowed_point_per_timestamp_not_a_single_point() { - let mut index = EngineSnapshot::default(); - declare_requests_metric(&mut index); - ingest_log(&mut index, 0, "request path=/a"); - ingest_log(&mut index, 1000, "request path=/a"); + assert_eq!(series.len(), 1); + assert_eq!(series[0].points, vec![(5, 1.0)]); +} - let parsed = PromqlQuery::parse("rate(requests[2s])").unwrap(); - let series = index.evaluate_promql(&parsed, None, None).unwrap(); +#[test] +fn declares_metric_with_value_param() { + let mut index = EngineSnapshot::default(); + let pattern = r"request path=(?P\S+) duration=(?P\d+)"; + declare_test_param( + &mut index, + "path", + pattern, + "path", + MetricPrimitiveKind::String, + ); + declare_test_param( + &mut index, + "duration", + pattern, + "duration", + MetricPrimitiveKind::Number, + ); + let result = index.declare_metric(metric_request_with_value( + "durations", + pattern, + vec!["path"], + Some("duration"), + )); + assert!(result.is_ok()); +} - assert_eq!(series.len(), 1); - assert_eq!(series[0].points, vec![(0, 0.5), (1000, 1.0)]); - } - - #[test] - fn sum_of_rate_by_label_gives_each_label_value_its_own_line() { - let mut index = EngineSnapshot::default(); - declare_requests_metric(&mut index); - ingest_log(&mut index, 0, "request path=/a"); - ingest_log(&mut index, 1000, "request path=/a"); - ingest_log(&mut index, 0, "request path=/b"); - - let parsed = PromqlQuery::parse("sum(rate(requests[2s])) by (path)").unwrap(); - let mut series = index.evaluate_promql(&parsed, None, None).unwrap(); - series.sort_by(|a, b| a.labels.get("path").cmp(&b.labels.get("path"))); - - assert_eq!(series.len(), 2); - assert_eq!(series[0].labels.get("path").map(String::as_str), Some("/a")); - assert_eq!(series[0].points, vec![(0, 0.5), (1000, 1.0)]); - assert_eq!(series[1].labels.get("path").map(String::as_str), Some("/b")); - assert_eq!(series[1].points, vec![(0, 0.5)]); - } - - #[test] - fn rate_series_gets_a_trailing_zero_point_once_occurrences_stop_early_in_a_wider_range() { - let mut index = EngineSnapshot::default(); - declare_requests_metric(&mut index); - ingest_log(&mut index, 0, "request path=/a"); - ingest_log(&mut index, 1, "request path=/a"); - - let parsed = PromqlQuery::parse("rate(requests[2s])").unwrap(); - let series = index.evaluate_promql(&parsed, None, Some(100_000)).unwrap(); - - assert_eq!(series.len(), 1); - assert_eq!( - series[0].points, - vec![(0, 0.5), (1, 1.0), (100_000, 0.0)], - "the window (2s) is long since aged out by end_ms, so the appended boundary point must read 0" - ); - } - - #[test] - fn rate_series_gets_a_leading_zero_point_when_occurrences_start_late_in_a_wider_range() { - let mut index = EngineSnapshot::default(); - declare_requests_metric(&mut index); - ingest_log(&mut index, 50_000, "request path=/a"); - - let parsed = PromqlQuery::parse("rate(requests[2s])").unwrap(); - let series = index.evaluate_promql(&parsed, Some(0), None).unwrap(); - - assert_eq!(series[0].points, vec![(0, 0.0), (50_000, 0.5)]); - } - - #[test] - fn sum_by_rate_gets_boundary_points_per_group_across_a_wider_range() { - let mut index = EngineSnapshot::default(); - declare_requests_metric(&mut index); - ingest_log(&mut index, 0, "request path=/a"); - ingest_log(&mut index, 1, "request path=/a"); - ingest_log(&mut index, 0, "request path=/b"); - - let parsed = PromqlQuery::parse("sum(rate(requests[2s])) by (path)").unwrap(); - let mut series = index.evaluate_promql(&parsed, None, Some(100_000)).unwrap(); - series.sort_by(|a, b| a.labels.get("path").cmp(&b.labels.get("path"))); - - assert_eq!(series.len(), 2); - assert_eq!(series[0].points, vec![(0, 0.5), (1, 1.0), (100_000, 0.0)]); - assert_eq!(series[1].points, vec![(0, 0.5), (100_000, 0.0)]); - } - - #[test] - fn rate_series_skips_boundary_points_already_covered_by_real_data() { - let mut index = EngineSnapshot::default(); - declare_requests_metric(&mut index); - ingest_log(&mut index, 0, "request path=/a"); - ingest_log(&mut index, 1000, "request path=/a"); - - let parsed = PromqlQuery::parse("rate(requests[2s])").unwrap(); - let series = index.evaluate_promql(&parsed, Some(0), Some(1000)).unwrap(); - - assert_eq!(series[0].points, vec![(0, 0.5), (1000, 1.0)]); - } - - #[test] - fn dashboard_rate_and_alert_threshold_agree_on_same_windowed_data() { - let mut index = EngineSnapshot::default(); - declare_requests_metric(&mut index); - - let lines: Vec<(u64, &str)> = vec![ - (0, "request path=/a"), - (500, "request path=/a"), - (1000, "request path=/a"), - (1500, "request path=/a"), - (2000, "request path=/a"), - ]; - for (ms, text) in &lines { - ingest_log(&mut index, *ms, text); - } - - let parsed = PromqlQuery::parse("rate(requests[2s])").unwrap(); - let series = index.evaluate_promql(&parsed, None, None).unwrap(); - assert_eq!(series.len(), 1); - let (_, dashboard_rate) = *series[0].points.last().unwrap(); - - let rule = alerts::AlertRule { - id: "r1".to_string(), - name: "spike".to_string(), - enabled: true, - condition: alerts::AlertCondition::Threshold { - query: r#"regex("request path=/a")"#.to_string(), - op: CompareOp::Ge, - threshold: 0.0, - window_ms: 2_000, - }, - }; - let mut alert_states = HashMap::new(); - let mut last_message = String::new(); - for (ms, text) in &lines { - let timestamp = Timestamp::from_millis(*ms as i64); - let fired = alerts::evaluate_alerts(&[rule.clone()], &mut alert_states, &index, *ms, timestamp, text); - assert_eq!(fired.len(), 1); - last_message = fired[0].1.clone(); - } - let alert_count: f64 = last_message - .split("match count ") - .nth(1) - .and_then(|rest| rest.split_whitespace().next()) - .and_then(|token| token.parse().ok()) - .expect("fired message should report a match count"); - - let window_seconds = 2.0; - assert_eq!(dashboard_rate, 2.0); - assert_eq!(alert_count, 4.0); - assert_eq!(alert_count, dashboard_rate * window_seconds); - } - - #[test] - fn sum_of_rate_without_by_clause_still_collapses_to_one_series() { - let mut index = EngineSnapshot::default(); - declare_requests_metric(&mut index); - ingest_log(&mut index, 0, "request path=/a"); - ingest_log(&mut index, 0, "request path=/b"); - - let parsed = PromqlQuery::parse("sum(rate(requests[2s]))").unwrap(); - let series = index.evaluate_promql(&parsed, None, None).unwrap(); - - assert_eq!(series.len(), 1); - assert!(series[0].labels.is_empty()); - assert_eq!(series[0].points, vec![(0, 1.0)]); - } - - #[test] - fn promql_range_bounds_filter_points_via_start_end_ms() { - let mut index = EngineSnapshot::default(); - declare_requests_metric(&mut index); - ingest_log(&mut index, 0, "request path=/a"); - ingest_log(&mut index, 5, "request path=/a"); - ingest_log(&mut index, 10, "request path=/a"); - - let parsed = PromqlQuery::parse("requests").unwrap(); - let series = index.evaluate_promql(&parsed, Some(1), Some(9)).unwrap(); - - assert_eq!(series.len(), 1); - assert_eq!(series[0].points, vec![(5, 1.0)]); - } - - #[test] - fn declares_metric_with_value_param() { - let mut index = EngineSnapshot::default(); - let pattern = r"request path=(?P\S+) duration=(?P\d+)"; - declare_test_param(&mut index, "path", pattern, "path", MetricPrimitiveKind::String); - declare_test_param(&mut index, "duration", pattern, "duration", MetricPrimitiveKind::Number); - let result = index.declare_metric(metric_request_with_value("durations", pattern, vec!["path"], Some("duration")), - ); - assert!(result.is_ok()); - } - - - #[test] - fn value_param_becomes_the_bucket_value_not_a_label() { - let mut index = EngineSnapshot::default(); - let pattern = r"request path=(?P\S+) duration=(?P\d+)"; - declare_test_param(&mut index, "path", pattern, "path", MetricPrimitiveKind::String); - declare_test_param(&mut index, "duration", pattern, "duration", MetricPrimitiveKind::Number); - index.declare_metric(metric_request_with_value("durations", pattern, vec!["path"], Some("duration")), - ) +#[test] +fn value_param_becomes_the_bucket_value_not_a_label() { + let mut index = EngineSnapshot::default(); + let pattern = r"request path=(?P\S+) duration=(?P\d+)"; + declare_test_param( + &mut index, + "path", + pattern, + "path", + MetricPrimitiveKind::String, + ); + declare_test_param( + &mut index, + "duration", + pattern, + "duration", + MetricPrimitiveKind::Number, + ); + index + .declare_metric(metric_request_with_value( + "durations", + pattern, + vec!["path"], + Some("duration"), + )) .unwrap(); - ingest_log(&mut index, 0, "request path=/a duration=5"); + ingest_log(&mut index, 0, "request path=/a duration=5"); - let values: Vec<_> = index.bucket_values("durations", None).collect(); - assert_eq!(values.len(), 1); - let (labels, (_, _, value)) = &values[0]; - assert!(!labels.contains_key("duration")); - assert_eq!(labels.get("path").map(|v| v.to_string()), Some("/a".to_string())); - assert_eq!(*value, MetricPrimitive::Number(Number::Int(5))); - } + let values: Vec<_> = index.bucket_values("durations", None).collect(); + assert_eq!(values.len(), 1); + let (labels, (_, _, value)) = &values[0]; + assert!(!labels.contains_key("duration")); + assert_eq!( + labels.get("path").map(|v| v.to_string()), + Some("/a".to_string()) + ); + assert_eq!(*value, MetricPrimitive::Number(Number::Int(5))); +} - #[test] - fn bucket_values_seeks_by_metric_name_and_ignores_lexicographic_neighbors() { - let mut index = EngineSnapshot::default(); - declare_test_param( +#[test] +fn bucket_values_seeks_by_metric_name_and_ignores_lexicographic_neighbors() { + let mut index = EngineSnapshot::default(); + declare_test_param( + &mut index, + "value", + r"value=(?P\d+)", + "value", + MetricPrimitiveKind::Number, + ); + // "aaa" and "aab" are adjacent lexicographically, so this exercises the exact + // boundary the metric-name seek relies on: each metric's own message_match_regex + // gates it to only its own line, so if the seek pulled in the wrong metric's bucket + // range, the value asserted below would be wrong. + index + .declare_metric(metric_request_with_value( + "aaa", + r"^aaa value=\d+$", + vec![], + Some("value"), + )) + .unwrap(); + index + .declare_metric(metric_request_with_value( + "aab", + r"^aab value=\d+$", + vec![], + Some("value"), + )) + .unwrap(); + + ingest_log(&mut index, 0, "aaa value=1"); + ingest_log(&mut index, 1, "aab value=2"); + + let aaa_values: Vec<_> = index + .bucket_values("aaa", None) + .map(|(_, (_, _, value))| value.clone()) + .collect(); + assert_eq!(aaa_values, vec![MetricPrimitive::Number(Number::Int(1))]); + + let aab_values: Vec<_> = index + .bucket_values("aab", None) + .map(|(_, (_, _, value))| value.clone()) + .collect(); + assert_eq!(aab_values, vec![MetricPrimitive::Number(Number::Int(2))]); +} + +fn declare_durations_metric(index: &mut EngineSnapshot) { + let pattern = r"request path=(?P\S+) duration=(?P\d+)"; + declare_test_param(index, "path", pattern, "path", MetricPrimitiveKind::String); + declare_test_param( + index, + "duration", + pattern, + "duration", + MetricPrimitiveKind::Number, + ); + index + .declare_metric(metric_request_with_value( + "durations", + pattern, + vec!["path"], + Some("duration"), + )) + .unwrap(); +} + +#[test] +fn gauge_returns_most_recent_value_per_label_combination() { + let mut index = EngineSnapshot::default(); + declare_durations_metric(&mut index); + ingest_log(&mut index, 0, "request path=/a duration=5"); + ingest_log(&mut index, 1, "request path=/a duration=9"); + ingest_log(&mut index, 2, "request path=/b duration=42"); + + let parsed = PromqlQuery::parse("gauge(durations)").unwrap(); + let mut series = index.evaluate_promql(&parsed, None, None).unwrap(); + series.sort_by(|a, b| a.labels.get("path").cmp(&b.labels.get("path"))); + + assert_eq!(series.len(), 2); + assert_eq!(series[0].labels.get("path").map(String::as_str), Some("/a")); + assert_eq!(series[0].points, vec![(1, 9.0)]); + assert_eq!(series[1].labels.get("path").map(String::as_str), Some("/b")); + assert_eq!(series[1].points, vec![(2, 42.0)]); +} + +#[test] +fn gauge_respects_time_range_bounds() { + let mut index = EngineSnapshot::default(); + declare_durations_metric(&mut index); + ingest_log(&mut index, 0, "request path=/a duration=5"); + ingest_log(&mut index, 5, "request path=/a duration=9"); + ingest_log(&mut index, 10, "request path=/a duration=42"); + + let parsed = PromqlQuery::parse("gauge(durations)").unwrap(); + let series = index.evaluate_promql(&parsed, None, Some(9)).unwrap(); + + assert_eq!(series.len(), 1); + assert_eq!(series[0].points, vec![(5, 9.0)]); +} + +#[test] +fn histogram_buckets_captured_values_by_fixed_width() { + let mut index = EngineSnapshot::default(); + declare_durations_metric(&mut index); + for (seq, duration) in [(0, 2), (1, 4), (2, 12), (3, 15), (4, 21)] { + ingest_log( &mut index, - "value", - r"value=(?P\d+)", - "value", - MetricPrimitiveKind::Number, + seq, + &format!("request path=/a duration={duration}"), ); - // "aaa" and "aab" are adjacent lexicographically, so this exercises the exact - // boundary the metric-name seek relies on: each metric's own message_match_regex - // gates it to only its own line, so if the seek pulled in the wrong metric's bucket - // range, the value asserted below would be wrong. - index - .declare_metric(metric_request_with_value("aaa", r"^aaa value=\d+$", vec![], Some("value"))) - .unwrap(); - index - .declare_metric(metric_request_with_value("aab", r"^aab value=\d+$", vec![], Some("value"))) - .unwrap(); - - ingest_log(&mut index, 0, "aaa value=1"); - ingest_log(&mut index, 1, "aab value=2"); - - let aaa_values: Vec<_> = index - .bucket_values("aaa", None) - .map(|(_, (_, _, value))| value.clone()) - .collect(); - assert_eq!(aaa_values, vec![MetricPrimitive::Number(Number::Int(1))]); - - let aab_values: Vec<_> = index - .bucket_values("aab", None) - .map(|(_, (_, _, value))| value.clone()) - .collect(); - assert_eq!(aab_values, vec![MetricPrimitive::Number(Number::Int(2))]); } - fn declare_durations_metric(index: &mut EngineSnapshot) { - let pattern = r"request path=(?P\S+) duration=(?P\d+)"; - declare_test_param(index, "path", pattern, "path", MetricPrimitiveKind::String); - declare_test_param(index, "duration", pattern, "duration", MetricPrimitiveKind::Number); - index.declare_metric(metric_request_with_value("durations", pattern, vec!["path"], Some("duration")), - ) + let parsed = PromqlQuery::parse("histogram(durations[10])").unwrap(); + let mut series = index.evaluate_promql(&parsed, None, None).unwrap(); + series.sort_by(|a, b| a.labels.get("le").cmp(&b.labels.get("le"))); + + assert_eq!(series.len(), 3); + assert_eq!(series[0].labels.get("le").map(String::as_str), Some("10")); + assert_eq!(series[0].points[0].1, 2.0); + assert_eq!(series[1].labels.get("le").map(String::as_str), Some("20")); + assert_eq!(series[1].points[0].1, 2.0); + assert_eq!(series[2].labels.get("le").map(String::as_str), Some("30")); + assert_eq!(series[2].points[0].1, 1.0); +} + +#[test] +fn histogram_with_by_clause_buckets_each_label_group_separately() { + let mut index = EngineSnapshot::default(); + declare_durations_metric(&mut index); + for (seq, path, duration) in [(0, "/a", 2), (1, "/a", 4), (2, "/a", 12), (3, "/b", 21)] { + ingest_log( + &mut index, + seq, + &format!("request path={path} duration={duration}"), + ); + } + + let parsed = PromqlQuery::parse("histogram(durations[10]) by (path)").unwrap(); + let mut series = index.evaluate_promql(&parsed, None, None).unwrap(); + series.sort_by(|a, b| { + a.labels + .get("path") + .cmp(&b.labels.get("path")) + .then(a.labels.get("le").cmp(&b.labels.get("le"))) + }); + + assert_eq!(series.len(), 3); + + assert_eq!(series[0].labels.get("path").map(String::as_str), Some("/a")); + assert_eq!(series[0].labels.get("le").map(String::as_str), Some("10")); + assert_eq!(series[0].points[0].1, 2.0); + + assert_eq!(series[1].labels.get("path").map(String::as_str), Some("/a")); + assert_eq!(series[1].labels.get("le").map(String::as_str), Some("20")); + assert_eq!(series[1].points[0].1, 1.0); + + assert_eq!(series[2].labels.get("path").map(String::as_str), Some("/b")); + assert_eq!(series[2].labels.get("le").map(String::as_str), Some("30")); + assert_eq!(series[2].points[0].1, 1.0); +} + +fn declare_login_duration_metric(index: &mut EngineSnapshot) { + let pattern = r"user=(?P\S+) logged in"; + declare_test_param(index, "user", pattern, "user", MetricPrimitiveKind::String); + declare_elapsed_since_last_param(index, "gap"); + index + .declare_metric(metric_request_with_value( + "login_gap", + pattern, + vec!["user"], + Some("gap"), + )) .unwrap(); - } +} - #[test] - fn gauge_returns_most_recent_value_per_label_combination() { - let mut index = EngineSnapshot::default(); - declare_durations_metric(&mut index); - ingest_log(&mut index, 0, "request path=/a duration=5"); - ingest_log(&mut index, 1, "request path=/a duration=9"); - ingest_log(&mut index, 2, "request path=/b duration=42"); +#[test] +fn duration_value_param_computes_elapsed_time_since_previous_same_labels_event() { + let mut index = EngineSnapshot::default(); + declare_login_duration_metric(&mut index); - let parsed = PromqlQuery::parse("gauge(durations)").unwrap(); - let mut series = index.evaluate_promql(&parsed, None, None).unwrap(); - series.sort_by(|a, b| a.labels.get("path").cmp(&b.labels.get("path"))); + ingest_log(&mut index, 0, "user=alice logged in"); + ingest_log(&mut index, 1, "user=bob logged in"); + ingest_log(&mut index, 2, "user=alice logged in"); - assert_eq!(series.len(), 2); - assert_eq!(series[0].labels.get("path").map(String::as_str), Some("/a")); - assert_eq!(series[0].points, vec![(1, 9.0)]); - assert_eq!(series[1].labels.get("path").map(String::as_str), Some("/b")); - assert_eq!(series[1].points, vec![(2, 42.0)]); - } + let values: Vec<_> = index.bucket_values("login_gap", None).collect(); + assert_eq!(values.len(), 1); + let (labels, (sequence, timestamp, value)) = &values[0]; + assert_eq!( + labels.get("user").map(|v| v.to_string()), + Some("alice".to_string()) + ); + assert_eq!(*sequence, 2); + assert_eq!(timestamp.as_millis(), 2); + assert_eq!(*value, MetricPrimitive::Duration(2)); +} - #[test] - fn gauge_respects_time_range_bounds() { - let mut index = EngineSnapshot::default(); - declare_durations_metric(&mut index); - ingest_log(&mut index, 0, "request path=/a duration=5"); - ingest_log(&mut index, 5, "request path=/a duration=9"); - ingest_log(&mut index, 10, "request path=/a duration=42"); +#[test] +fn duration_diffs_against_the_immediately_preceding_event_not_the_first() { + let mut index = EngineSnapshot::default(); + declare_login_duration_metric(&mut index); - let parsed = PromqlQuery::parse("gauge(durations)").unwrap(); - let series = index.evaluate_promql(&parsed, None, Some(9)).unwrap(); + ingest_log(&mut index, 0, "user=alice logged in"); + ingest_log(&mut index, 3, "user=alice logged in"); + ingest_log(&mut index, 9, "user=alice logged in"); - assert_eq!(series.len(), 1); - assert_eq!(series[0].points, vec![(5, 9.0)]); - } + let values: Vec<_> = index.bucket_values("login_gap", None).collect(); + assert_eq!(values.len(), 2); + assert_eq!(values[0].1.2, MetricPrimitive::Duration(3)); + assert_eq!(values[1].1.2, MetricPrimitive::Duration(6)); +} - #[test] - fn histogram_buckets_captured_values_by_fixed_width() { - let mut index = EngineSnapshot::default(); - declare_durations_metric(&mut index); - for (seq, duration) in [(0, 2), (1, 4), (2, 12), (3, 15), (4, 21)] { - ingest_log(&mut index, seq, &format!("request path=/a duration={duration}")); - } +#[test] +fn duration_values_are_queryable_via_promql_like_any_other_metric() { + let mut index = EngineSnapshot::default(); + declare_login_duration_metric(&mut index); + ingest_log(&mut index, 0, "user=alice logged in"); + ingest_log(&mut index, 10, "user=alice logged in"); - let parsed = PromqlQuery::parse("histogram(durations[10])").unwrap(); - let mut series = index.evaluate_promql(&parsed, None, None).unwrap(); - series.sort_by(|a, b| a.labels.get("le").cmp(&b.labels.get("le"))); + let parsed = PromqlQuery::parse("login_gap").unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); - assert_eq!(series.len(), 3); - assert_eq!(series[0].labels.get("le").map(String::as_str), Some("10")); - assert_eq!(series[0].points[0].1, 2.0); - assert_eq!(series[1].labels.get("le").map(String::as_str), Some("20")); - assert_eq!(series[1].points[0].1, 2.0); - assert_eq!(series[2].labels.get("le").map(String::as_str), Some("30")); - assert_eq!(series[2].points[0].1, 1.0); - } + assert_eq!(series.len(), 1); + assert_eq!(series[0].points, vec![(10, 10.0)]); +} - #[test] - fn histogram_with_by_clause_buckets_each_label_group_separately() { - let mut index = EngineSnapshot::default(); - declare_durations_metric(&mut index); - for (seq, path, duration) in [ - (0, "/a", 2), - (1, "/a", 4), - (2, "/a", 12), - (3, "/b", 21), - ] { - ingest_log(&mut index, seq, &format!("request path={path} duration={duration}")); - } - - let parsed = PromqlQuery::parse("histogram(durations[10]) by (path)").unwrap(); - let mut series = index.evaluate_promql(&parsed, None, None).unwrap(); - series.sort_by(|a, b| { - a.labels.get("path").cmp(&b.labels.get("path")).then(a.labels.get("le").cmp(&b.labels.get("le"))) - }); - - assert_eq!(series.len(), 3); - - assert_eq!(series[0].labels.get("path").map(String::as_str), Some("/a")); - assert_eq!(series[0].labels.get("le").map(String::as_str), Some("10")); - assert_eq!(series[0].points[0].1, 2.0); - - assert_eq!(series[1].labels.get("path").map(String::as_str), Some("/a")); - assert_eq!(series[1].labels.get("le").map(String::as_str), Some("20")); - assert_eq!(series[1].points[0].1, 1.0); - - assert_eq!(series[2].labels.get("path").map(String::as_str), Some("/b")); - assert_eq!(series[2].labels.get("le").map(String::as_str), Some("30")); - assert_eq!(series[2].points[0].1, 1.0); - } - - fn declare_login_duration_metric(index: &mut EngineSnapshot) { - let pattern = r"user=(?P\S+) logged in"; - declare_test_param(index, "user", pattern, "user", MetricPrimitiveKind::String); - declare_elapsed_since_last_param(index, "gap"); - index.declare_metric(metric_request_with_value("login_gap", pattern, vec!["user"], Some("gap"))).unwrap(); - } - - #[test] - fn duration_value_param_computes_elapsed_time_since_previous_same_labels_event() { - let mut index = EngineSnapshot::default(); - declare_login_duration_metric(&mut index); - - ingest_log(&mut index, 0, "user=alice logged in"); - ingest_log(&mut index, 1, "user=bob logged in"); - ingest_log(&mut index, 2, "user=alice logged in"); - - let values: Vec<_> = index.bucket_values("login_gap", None).collect(); - assert_eq!(values.len(), 1); - let (labels, (sequence, timestamp, value)) = &values[0]; - assert_eq!(labels.get("user").map(|v| v.to_string()), Some("alice".to_string())); - assert_eq!(*sequence, 2); - assert_eq!(timestamp.as_millis(), 2); - assert_eq!(*value, MetricPrimitive::Duration(2)); - } - - #[test] - fn duration_diffs_against_the_immediately_preceding_event_not_the_first() { - let mut index = EngineSnapshot::default(); - declare_login_duration_metric(&mut index); - - ingest_log(&mut index, 0, "user=alice logged in"); - ingest_log(&mut index, 3, "user=alice logged in"); - ingest_log(&mut index, 9, "user=alice logged in"); - - let values: Vec<_> = index.bucket_values("login_gap", None).collect(); - assert_eq!(values.len(), 2); - assert_eq!(values[0].1.2, MetricPrimitive::Duration(3)); - assert_eq!(values[1].1.2, MetricPrimitive::Duration(6)); - } - - #[test] - fn duration_values_are_queryable_via_promql_like_any_other_metric() { - let mut index = EngineSnapshot::default(); - declare_login_duration_metric(&mut index); - ingest_log(&mut index, 0, "user=alice logged in"); - ingest_log(&mut index, 10, "user=alice logged in"); - - let parsed = PromqlQuery::parse("login_gap").unwrap(); - let series = index.evaluate_promql(&parsed, None, None).unwrap(); - - assert_eq!(series.len(), 1); - assert_eq!(series[0].points, vec![(10, 10.0)]); - } - - fn declare_error_category_metric(index: &mut EngineSnapshot) { - index.declare_metric(DeclareMetricRequest { - name: "log_category".to_string(), - message_match_regex: r"(?i)error".to_string(), - label_params: vec![], - value_param: None, - matched_labels: Some(BTreeMap::from([("category".to_string(), "error".to_string())])), - unmatched_labels: Some(BTreeMap::from([("category".to_string(), "other".to_string())])), - }, - ) +fn declare_error_category_metric(index: &mut EngineSnapshot) { + index + .declare_metric(DeclareMetricRequest { + name: "log_category".to_string(), + message_match_regex: r"(?i)error".to_string(), + label_params: vec![], + value_param: None, + matched_labels: Some(BTreeMap::from([( + "category".to_string(), + "error".to_string(), + )])), + unmatched_labels: Some(BTreeMap::from([( + "category".to_string(), + "other".to_string(), + )])), + }) .unwrap(); - } +} - #[test] - fn unmatched_labels_registers_occurrence_for_non_matching_lines() { - let mut index = EngineSnapshot::default(); - declare_error_category_metric(&mut index); +#[test] +fn unmatched_labels_registers_occurrence_for_non_matching_lines() { + let mut index = EngineSnapshot::default(); + declare_error_category_metric(&mut index); - ingest_log(&mut index, 0, "everything is fine"); - ingest_log(&mut index, 1, "an ERROR occurred"); - ingest_log(&mut index, 2, "still fine"); + ingest_log(&mut index, 0, "everything is fine"); + ingest_log(&mut index, 1, "an ERROR occurred"); + ingest_log(&mut index, 2, "still fine"); - let error_labels = BTreeMap::from([("category".to_string(), MetricPrimitive::String("error".to_string()))]); - let error_values: Vec<_> = index.bucket_values("log_category", Some(&error_labels)).collect(); - assert_eq!(error_values.len(), 1); - assert_eq!(error_values[0].1.0, 1); + let error_labels = BTreeMap::from([( + "category".to_string(), + MetricPrimitive::String("error".to_string()), + )]); + let error_values: Vec<_> = index + .bucket_values("log_category", Some(&error_labels)) + .collect(); + assert_eq!(error_values.len(), 1); + assert_eq!(error_values[0].1.0, 1); - let other_labels = BTreeMap::from([("category".to_string(), MetricPrimitive::String("other".to_string()))]); - let other_values: Vec<_> = index.bucket_values("log_category", Some(&other_labels)).collect(); - assert_eq!(other_values.len(), 2); - } + let other_labels = BTreeMap::from([( + "category".to_string(), + MetricPrimitive::String("other".to_string()), + )]); + let other_values: Vec<_> = index + .bucket_values("log_category", Some(&other_labels)) + .collect(); + assert_eq!(other_values.len(), 2); +} - #[test] - fn unmatched_labels_produces_two_series_from_one_bare_promql_query() { - let mut index = EngineSnapshot::default(); - declare_error_category_metric(&mut index); - ingest_log(&mut index, 0, "an ERROR occurred"); - ingest_log(&mut index, 1, "all good"); - ingest_log(&mut index, 2, "all good again"); +#[test] +fn unmatched_labels_produces_two_series_from_one_bare_promql_query() { + let mut index = EngineSnapshot::default(); + declare_error_category_metric(&mut index); + ingest_log(&mut index, 0, "an ERROR occurred"); + ingest_log(&mut index, 1, "all good"); + ingest_log(&mut index, 2, "all good again"); - let parsed = PromqlQuery::parse("log_category").unwrap(); - let mut series = index.evaluate_promql(&parsed, None, None).unwrap(); - series.sort_by(|a, b| a.labels.get("category").cmp(&b.labels.get("category"))); + let parsed = PromqlQuery::parse("log_category").unwrap(); + let mut series = index.evaluate_promql(&parsed, None, None).unwrap(); + series.sort_by(|a, b| a.labels.get("category").cmp(&b.labels.get("category"))); - assert_eq!(series.len(), 2); - assert_eq!(series[0].labels.get("category").map(String::as_str), Some("error")); - assert_eq!(series[0].points.len(), 1); - assert_eq!(series[1].labels.get("category").map(String::as_str), Some("other")); - assert_eq!(series[1].points.len(), 2); - } + assert_eq!(series.len(), 2); + assert_eq!( + series[0].labels.get("category").map(String::as_str), + Some("error") + ); + assert_eq!(series[0].points.len(), 1); + assert_eq!( + series[1].labels.get("category").map(String::as_str), + Some("other") + ); + assert_eq!(series[1].points.len(), 2); +} - #[test] - fn no_unmatched_labels_means_non_matching_lines_are_untouched() { - let mut index = EngineSnapshot::default(); - declare_requests_metric(&mut index); - ingest_log(&mut index, 0, "not a request line at all"); - assert_eq!(index.bucket_values("requests", None).count(), 0); - } +#[test] +fn no_unmatched_labels_means_non_matching_lines_are_untouched() { + let mut index = EngineSnapshot::default(); + declare_requests_metric(&mut index); + ingest_log(&mut index, 0, "not a request line at all"); + assert_eq!(index.bucket_values("requests", None).count(), 0); +} - #[test] - fn handle_query_promql_rejects_malformed_query() { - let index = EngineSnapshot::default(); - let response = handle_query( - &index, - EngineQuery::Promql { - query: "not a valid promql query(".to_string(), - start_ms: None, - end_ms: None, - }, - &mut HashMap::new(), - ); - match response { - EngineResponse::Promql(Err(message)) => assert!(!message.is_empty()), - _ => panic!("expected EngineResponse::Promql(Err(_))"), - } - } - - #[test] - fn handle_query_promql_returns_series_for_valid_query() { - let mut index = EngineSnapshot::default(); - declare_requests_metric(&mut index); - ingest_log(&mut index, 0, "request path=/a"); - - let response = handle_query( - &index, - EngineQuery::Promql { - query: r#"requests{path="/a"}"#.to_string(), - start_ms: None, - end_ms: None, - }, - &mut HashMap::new(), - ); - match response { - EngineResponse::Promql(Ok(series)) => { - assert_eq!(series.len(), 1); - assert_eq!(series[0].points, vec![(0, 1.0)]); - } - _ => panic!("expected EngineResponse::Promql(Ok(_))"), - } - } - - #[test] - fn promql_count_over_ottyl_seq_gives_one_point_per_ingested_log_with_no_metric_declaration() { - let mut index = EngineSnapshot::default(); - ingest_log(&mut index, 0, "line one"); - ingest_log(&mut index, 1, "line two"); - ingest_log(&mut index, 2, "line three"); - - let parsed = PromqlQuery::parse("count(ottyl_seq)").unwrap(); - let series = index.evaluate_promql(&parsed, None, None).unwrap(); - assert_eq!(series.len(), 1); - assert_eq!(series[0].points, vec![(0, 1.0), (1, 1.0), (2, 1.0)]); - } - - #[test] - fn promql_bare_ottyl_seq_returns_the_actual_sequence_numbers() { - let mut index = EngineSnapshot::default(); - ingest_log(&mut index, 0, "line zero"); - ingest_log(&mut index, 5, "line five"); - - let parsed = PromqlQuery::parse("ottyl_seq").unwrap(); - let mut series = index.evaluate_promql(&parsed, None, None).unwrap(); - assert_eq!(series.len(), 1); - series[0].points.sort_by_key(|(ts, _)| *ts); - assert_eq!(series[0].points, vec![(0, 0.0), (5, 5.0)]); - } - - #[test] - fn promql_bare_ottyl_ts_returns_ingest_timestamps() { - let mut index = EngineSnapshot::default(); - ingest_log(&mut index, 0, "a"); - ingest_log(&mut index, 1, "b"); - - let parsed = PromqlQuery::parse("ottyl_ts").unwrap(); - let series = index.evaluate_promql(&parsed, None, None).unwrap(); - assert_eq!(series.len(), 1); - assert_eq!(series[0].points, vec![(0, 0.0), (1, 1.0)]); - } - - #[test] - fn promql_rate_over_ottyl_ts_produces_a_windowed_series_not_an_empty_one() { - let mut index = EngineSnapshot::default(); - for sequence in 0..5u64 { - ingest_log(&mut index, sequence, "line"); - } - - let parsed = PromqlQuery::parse("rate(ottyl_ts[5s])").unwrap(); - let series = index.evaluate_promql(&parsed, None, None).unwrap(); - assert_eq!(series.len(), 1); - assert!(!series[0].points.is_empty(), "rate(ottyl_ts[...]) must not silently return no data"); - } - - #[test] - fn promql_ottyl_seq_with_label_matcher_matches_nothing_since_it_has_no_labels() { - let mut index = EngineSnapshot::default(); - ingest_log(&mut index, 0, "line"); - - let parsed = PromqlQuery::parse(r#"ottyl_seq{path="/a"}"#).unwrap(); - let series = index.evaluate_promql(&parsed, None, None).unwrap(); - assert!(series.is_empty()); - } - - #[test] - fn promql_over_ottyl_fmt_is_explicitly_rejected_not_silently_empty() { - let mut index = EngineSnapshot::default(); - ingest_log(&mut index, 0, r#"{"level":"error"}"#); - - let parsed = PromqlQuery::parse("count(ottyl_fmt)").unwrap(); - let err = index.evaluate_promql(&parsed, None, None).unwrap_err(); - assert!(err.contains("ottyl_fmt"), "error should name the rejected metric: {err}"); - assert!(err.contains("categorical") || err.contains("filter"), "error should explain why: {err}"); - } - - #[test] - fn handle_query_promql_returns_series_for_ottyl_seq_end_to_end() { - let mut index = EngineSnapshot::default(); - ingest_log(&mut index, 0, "a"); - ingest_log(&mut index, 1, "b"); - - let response = handle_query( - &index, - EngineQuery::Promql { query: "count(ottyl_seq)".to_string(), start_ms: None, end_ms: None }, - &mut HashMap::new(), - ); - match response { - EngineResponse::Promql(Ok(series)) => { - assert_eq!(series[0].points, vec![(0, 1.0), (1, 1.0)]); - } - _ => panic!("expected EngineResponse::Promql(Ok(_))"), - } - } - - #[test] - fn declare_param_rejects_reserved_ottyl_names() { - let mut index = EngineSnapshot::default(); - for name in OTTYL_RESERVED_PARAM_NAMES { - let err = declare_param(&mut index, request(name, "x", "x")).unwrap_err(); - assert!(err.contains("reserved"), "expected a 'reserved' error for '{name}', got: {err}"); - } - } - - - fn logs_query(filter: Option<&str>) -> EngineQuery { - EngineQuery::Logs(LogsParams { - since_sequence: None, +#[test] +fn handle_query_promql_rejects_malformed_query() { + let index = EngineSnapshot::default(); + let response = handle_query( + &index, + EngineQuery::Promql { + query: "not a valid promql query(".to_string(), start_ms: None, end_ms: None, - limit: None, - filter: filter.map(str::to_string), - sort_by: None, - sort_missing_fallback: None, - }) + }, + &mut HashMap::new(), + ); + match response { + EngineResponse::Promql(Err(message)) => assert!(!message.is_empty()), + _ => panic!("expected EngineResponse::Promql(Err(_))"), } +} - fn logs_query_sorted(sort_by: &str, sort_missing_fallback: Option) -> EngineQuery { - EngineQuery::Logs(LogsParams { - since_sequence: None, +#[test] +fn handle_query_promql_returns_series_for_valid_query() { + let mut index = EngineSnapshot::default(); + declare_requests_metric(&mut index); + ingest_log(&mut index, 0, "request path=/a"); + + let response = handle_query( + &index, + EngineQuery::Promql { + query: r#"requests{path="/a"}"#.to_string(), start_ms: None, end_ms: None, - limit: None, - filter: None, - sort_by: Some(sort_by.to_string()), - sort_missing_fallback, - }) + }, + &mut HashMap::new(), + ); + match response { + EngineResponse::Promql(Ok(series)) => { + assert_eq!(series.len(), 1); + assert_eq!(series[0].points, vec![(0, 1.0)]); + } + _ => panic!("expected EngineResponse::Promql(Ok(_))"), + } +} + +#[test] +fn promql_count_over_ottyl_seq_gives_one_point_per_ingested_log_with_no_metric_declaration() { + let mut index = EngineSnapshot::default(); + ingest_log(&mut index, 0, "line one"); + ingest_log(&mut index, 1, "line two"); + ingest_log(&mut index, 2, "line three"); + + let parsed = PromqlQuery::parse("count(ottyl_seq)").unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); + assert_eq!(series.len(), 1); + assert_eq!(series[0].points, vec![(0, 1.0), (1, 1.0), (2, 1.0)]); +} + +#[test] +fn promql_bare_ottyl_seq_returns_the_actual_sequence_numbers() { + let mut index = EngineSnapshot::default(); + ingest_log(&mut index, 0, "line zero"); + ingest_log(&mut index, 5, "line five"); + + let parsed = PromqlQuery::parse("ottyl_seq").unwrap(); + let mut series = index.evaluate_promql(&parsed, None, None).unwrap(); + assert_eq!(series.len(), 1); + series[0].points.sort_by_key(|(ts, _)| *ts); + assert_eq!(series[0].points, vec![(0, 0.0), (5, 5.0)]); +} + +#[test] +fn promql_bare_ottyl_ts_returns_ingest_timestamps() { + let mut index = EngineSnapshot::default(); + ingest_log(&mut index, 0, "a"); + ingest_log(&mut index, 1, "b"); + + let parsed = PromqlQuery::parse("ottyl_ts").unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); + assert_eq!(series.len(), 1); + assert_eq!(series[0].points, vec![(0, 0.0), (1, 1.0)]); +} + +#[test] +fn promql_rate_over_ottyl_ts_produces_a_windowed_series_not_an_empty_one() { + let mut index = EngineSnapshot::default(); + for sequence in 0..5u64 { + ingest_log(&mut index, sequence, "line"); } - fn timeline_query(filter: Option<&str>, bucket_ms: i64) -> EngineQuery { - EngineQuery::Timeline(TimelineParams { + let parsed = PromqlQuery::parse("rate(ottyl_ts[5s])").unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); + assert_eq!(series.len(), 1); + assert!( + !series[0].points.is_empty(), + "rate(ottyl_ts[...]) must not silently return no data" + ); +} + +#[test] +fn promql_ottyl_seq_with_label_matcher_matches_nothing_since_it_has_no_labels() { + let mut index = EngineSnapshot::default(); + ingest_log(&mut index, 0, "line"); + + let parsed = PromqlQuery::parse(r#"ottyl_seq{path="/a"}"#).unwrap(); + let series = index.evaluate_promql(&parsed, None, None).unwrap(); + assert!(series.is_empty()); +} + +#[test] +fn promql_over_ottyl_fmt_is_explicitly_rejected_not_silently_empty() { + let mut index = EngineSnapshot::default(); + ingest_log(&mut index, 0, r#"{"level":"error"}"#); + + let parsed = PromqlQuery::parse("count(ottyl_fmt)").unwrap(); + let err = index.evaluate_promql(&parsed, None, None).unwrap_err(); + assert!( + err.contains("ottyl_fmt"), + "error should name the rejected metric: {err}" + ); + assert!( + err.contains("categorical") || err.contains("filter"), + "error should explain why: {err}" + ); +} + +#[test] +fn handle_query_promql_returns_series_for_ottyl_seq_end_to_end() { + let mut index = EngineSnapshot::default(); + ingest_log(&mut index, 0, "a"); + ingest_log(&mut index, 1, "b"); + + let response = handle_query( + &index, + EngineQuery::Promql { + query: "count(ottyl_seq)".to_string(), start_ms: None, end_ms: None, - bucket_ms: Some(bucket_ms), - filter: filter.map(str::to_string), - }) - } - - #[test] - fn logs_query_response_includes_ottyl_fmt_per_entry() { - let mut index = EngineSnapshot::default(); - ingest_log(&mut index, 0, r#"{"level":"error"}"#); - ingest_log(&mut index, 1, "level=info msg=hello"); - ingest_log(&mut index, 2, "just plain prose"); - - match handle_query(&index, logs_query(None), &mut HashMap::new()) { - EngineResponse::Logs(entries) => { - let formats: Vec> = entries.iter().map(|e| e.params.get("ottyl_fmt")).collect(); - assert_eq!(formats[0], Some(&MetricPrimitive::String("json".to_string()))); - assert_eq!(formats[1], Some(&MetricPrimitive::String("logfmt".to_string()))); - assert_eq!(formats[2], Some(&MetricPrimitive::String("none".to_string()))); - } - _ => panic!("expected Logs response"), + }, + &mut HashMap::new(), + ); + match response { + EngineResponse::Promql(Ok(series)) => { + assert_eq!(series[0].points, vec![(0, 1.0), (1, 1.0)]); } + _ => panic!("expected EngineResponse::Promql(Ok(_))"), } +} - #[test] - fn logs_query_with_regex_filter_returns_only_matching_lines() { - let mut index = EngineSnapshot::default(); - ingest_log(&mut index, 0, "everything is fine"); - ingest_log(&mut index, 1, "an error occurred"); - ingest_log(&mut index, 2, "still fine"); - ingest_log(&mut index, 3, "another error here"); - - match handle_query(&index, logs_query(Some(r#"regex("error")"#)), &mut HashMap::new()) { - EngineResponse::Logs(entries) => { - let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); - assert_eq!(sequences, vec![1, 3]); - } - _ => panic!("expected Logs response"), - } +#[test] +fn declare_param_rejects_reserved_ottyl_names() { + let mut index = EngineSnapshot::default(); + for name in OTTYL_RESERVED_PARAM_NAMES { + let err = declare_param(&mut index, request(name, "x", "x")).unwrap_err(); + assert!( + err.contains("reserved"), + "expected a 'reserved' error for '{name}', got: {err}" + ); } +} - #[test] - fn logs_query_with_ottyl_seq_comparison_filters_by_sequence() { - let mut index = EngineSnapshot::default(); - for i in 0..5u64 { - ingest_log(&mut index, i, &format!("line {i}")); - } +fn logs_query(filter: Option<&str>) -> EngineQuery { + EngineQuery::Logs(LogsParams { + since_sequence: None, + start_ms: None, + end_ms: None, + limit: None, + filter: filter.map(str::to_string), + sort_by: None, + sort_missing_fallback: None, + }) +} - match handle_query(&index, logs_query(Some("ottyl_seq<2")), &mut HashMap::new()) { - EngineResponse::Logs(entries) => { - let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); - assert_eq!(sequences, vec![0, 1]); - } - _ => panic!("expected Logs response"), - } - } +fn logs_query_sorted(sort_by: &str, sort_missing_fallback: Option) -> EngineQuery { + EngineQuery::Logs(LogsParams { + since_sequence: None, + start_ms: None, + end_ms: None, + limit: None, + filter: None, + sort_by: Some(sort_by.to_string()), + sort_missing_fallback, + }) +} - #[test] - fn logs_query_with_ottyl_ts_comparison_filters_by_timestamp() { - let mut index = EngineSnapshot::default(); - for i in 0..5u64 { - let timestamp = Timestamp::from_millis(i as i64 * 1000); - index.index.insert( - IndexKey::IngestedLog { sequence: i, timestamp }, - DataValue::LogLiteral { text: format!("line {i}") }, +fn timeline_query(filter: Option<&str>, bucket_ms: i64) -> EngineQuery { + EngineQuery::Timeline(TimelineParams { + start_ms: None, + end_ms: None, + bucket_ms: Some(bucket_ms), + filter: filter.map(str::to_string), + }) +} + +#[test] +fn logs_query_response_includes_ottyl_fmt_per_entry() { + let mut index = EngineSnapshot::default(); + ingest_log(&mut index, 0, r#"{"level":"error"}"#); + ingest_log(&mut index, 1, "level=info msg=hello"); + ingest_log(&mut index, 2, "just plain prose"); + + match handle_query(&index, logs_query(None), &mut HashMap::new()) { + EngineResponse::Logs(entries) => { + let formats: Vec> = + entries.iter().map(|e| e.params.get("ottyl_fmt")).collect(); + assert_eq!( + formats[0], + Some(&MetricPrimitive::String("json".to_string())) + ); + assert_eq!( + formats[1], + Some(&MetricPrimitive::String("logfmt".to_string())) + ); + assert_eq!( + formats[2], + Some(&MetricPrimitive::String("none".to_string())) ); } + _ => panic!("expected Logs response"), + } +} - match handle_query(&index, logs_query(Some("ottyl_ts>2000")), &mut HashMap::new()) { - EngineResponse::Logs(entries) => { - let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); - assert_eq!(sequences, vec![3, 4]); - } - _ => panic!("expected Logs response"), +#[test] +fn logs_query_with_regex_filter_returns_only_matching_lines() { + let mut index = EngineSnapshot::default(); + ingest_log(&mut index, 0, "everything is fine"); + ingest_log(&mut index, 1, "an error occurred"); + ingest_log(&mut index, 2, "still fine"); + ingest_log(&mut index, 3, "another error here"); + + match handle_query( + &index, + logs_query(Some(r#"regex("error")"#)), + &mut HashMap::new(), + ) { + EngineResponse::Logs(entries) => { + let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); + assert_eq!(sequences, vec![1, 3]); } + _ => panic!("expected Logs response"), + } +} + +#[test] +fn logs_query_with_ottyl_seq_comparison_filters_by_sequence() { + let mut index = EngineSnapshot::default(); + for i in 0..5u64 { + ingest_log(&mut index, i, &format!("line {i}")); } - - #[test] - fn logs_query_sort_by_orders_lines_by_declared_param_value() { - let mut index = EngineSnapshot::default(); - declare_test_param( - &mut index, - "priority", - r"priority=(?P\d+)", - "priority", - MetricPrimitiveKind::Number, - ); - ingest_log(&mut index, 0, "priority=3 boom"); - ingest_log(&mut index, 1, "priority=1 boom"); - ingest_log(&mut index, 2, "priority=2 boom"); - - match handle_query(&index, logs_query_sorted("priority", None), &mut HashMap::new()) { - EngineResponse::Logs(entries) => { - let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); - assert_eq!(sequences, vec![1, 2, 0]); - } - _ => panic!("expected Logs response"), + match handle_query(&index, logs_query(Some("ottyl_seq<2")), &mut HashMap::new()) { + EngineResponse::Logs(entries) => { + let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); + assert_eq!(sequences, vec![0, 1]); } + _ => panic!("expected Logs response"), } +} - #[test] - fn logs_query_sort_by_missing_param_fallback_orders_by_sequence_and_appends() { - let mut index = EngineSnapshot::default(); - declare_test_param( - &mut index, - "priority", - r"priority=(?P\d+)", - "priority", - MetricPrimitiveKind::Number, - ); - ingest_log(&mut index, 0, "priority=3 boom"); - ingest_log(&mut index, 1, "no param here"); - ingest_log(&mut index, 2, "priority=1 boom"); - ingest_log(&mut index, 3, "also missing"); - - match handle_query(&index, logs_query_sorted("priority", Some(true)), &mut HashMap::new()) { - EngineResponse::Logs(entries) => { - let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); - assert_eq!(sequences, vec![2, 0, 1, 3]); - } - _ => panic!("expected Logs response"), - } - } - - #[test] - fn logs_query_sort_by_missing_param_drop_excludes_them() { - let mut index = EngineSnapshot::default(); - declare_test_param( - &mut index, - "priority", - r"priority=(?P\d+)", - "priority", - MetricPrimitiveKind::Number, - ); - ingest_log(&mut index, 0, "priority=3 boom"); - ingest_log(&mut index, 1, "no param here"); - ingest_log(&mut index, 2, "priority=1 boom"); - ingest_log(&mut index, 3, "also missing"); - - match handle_query(&index, logs_query_sorted("priority", Some(false)), &mut HashMap::new()) { - EngineResponse::Logs(entries) => { - let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); - assert_eq!(sequences, vec![2, 0]); - } - _ => panic!("expected Logs response"), - } - } - - #[test] - fn logs_query_sort_by_unknown_param_returns_error() { - let mut index = EngineSnapshot::default(); - ingest_log(&mut index, 0, "hello"); - - match handle_query(&index, logs_query_sorted("does_not_exist", None), &mut HashMap::new()) { - EngineResponse::Error(err) => assert!(err.contains("does_not_exist")), - _ => panic!("expected EngineResponse::Error"), - } - } - - #[test] - fn logs_query_without_sort_by_keeps_default_ottyl_ts_order() { - let mut index = EngineSnapshot::default(); - declare_test_param( - &mut index, - "priority", - r"priority=(?P\d+)", - "priority", - MetricPrimitiveKind::Number, - ); - ingest_log(&mut index, 0, "priority=3 boom"); - ingest_log(&mut index, 1, "no param here"); - ingest_log(&mut index, 2, "priority=1 boom"); - ingest_log(&mut index, 3, "also missing"); - - match handle_query(&index, logs_query(None), &mut HashMap::new()) { - EngineResponse::Logs(entries) => { - let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); - assert_eq!(sequences, vec![0, 1, 2, 3]); - } - _ => panic!("expected Logs response"), - } - } - - - #[test] - fn sort_helper_index_is_built_lazily_and_reused_across_queries() { - let mut index = EngineSnapshot::default(); - declare_test_param( - &mut index, - "priority", - r"priority=(?P\d+)", - "priority", - MetricPrimitiveKind::Number, - ); - ingest_log(&mut index, 0, "priority=3 boom"); - ingest_log(&mut index, 1, "priority=1 boom"); - ingest_log(&mut index, 2, "priority=2 boom"); - - let mut sort_helper_indexes = HashMap::new(); - assert!(!sort_helper_indexes.contains_key("priority")); - - match handle_query(&index, logs_query_sorted("priority", None), &mut sort_helper_indexes) { - EngineResponse::Logs(entries) => { - let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); - assert_eq!(sequences, vec![1, 2, 0]); - } - _ => panic!("expected Logs response"), - } - assert!(sort_helper_indexes.contains_key("priority")); - assert_eq!(sort_helper_indexes["priority"].index.len(), 3); - - ingest_log(&mut index, 3, "priority=0 boom"); - std::mem::swap(&mut index.sort_helper_indexes, &mut sort_helper_indexes); - index.update_sort_helper_indexes_on_ingest(3, Timestamp::from_millis(3), "priority=0 boom"); - std::mem::swap(&mut index.sort_helper_indexes, &mut sort_helper_indexes); - - ingest_log(&mut index, 4, "no priority here"); - std::mem::swap(&mut index.sort_helper_indexes, &mut sort_helper_indexes); - index.update_sort_helper_indexes_on_ingest(4, Timestamp::from_millis(4), "no priority here"); - std::mem::swap(&mut index.sort_helper_indexes, &mut sort_helper_indexes); - assert_eq!(sort_helper_indexes["priority"].index.len(), 5); - - match handle_query(&index, logs_query_sorted("priority", None), &mut sort_helper_indexes) { - EngineResponse::Logs(entries) => { - let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); - assert_eq!(sequences, vec![3, 1, 2, 0, 4]); - } - _ => panic!("expected Logs response"), - } - } - - #[test] - fn sort_helper_index_incremental_update_is_a_noop_when_no_index_is_live() { - let mut index = EngineSnapshot::default(); - index.update_sort_helper_indexes_on_ingest(0, Timestamp::from_millis(0), "hello"); - assert!(index.sort_helper_indexes.is_empty()); - } - - - #[test] - fn purge_stale_sort_helper_indexes_drops_only_entries_past_the_purge_duration() { - let mut sort_helper_indexes: HashMap = HashMap::new(); - sort_helper_indexes.insert( - "stale".to_string(), - SortHelperIndexEntry { - index: BTreeMap::new(), - last_accessed: std::time::Instant::now() - SORT_HELPER_INDEX_PURGE_AFTER - - std::time::Duration::from_secs(1), +#[test] +fn logs_query_with_ottyl_ts_comparison_filters_by_timestamp() { + let mut index = EngineSnapshot::default(); + for i in 0..5u64 { + let timestamp = Timestamp::from_millis(i as i64 * 1000); + index.index.insert( + IndexKey::IngestedLog { + sequence: i, + timestamp, + }, + DataValue::LogLiteral { + text: format!("line {i}"), }, ); - sort_helper_indexes.insert( - "fresh".to_string(), - SortHelperIndexEntry { index: BTreeMap::new(), last_accessed: std::time::Instant::now() }, - ); - - let mut snapshot = EngineSnapshot { sort_helper_indexes, ..Default::default() }; - snapshot.purge_stale_sort_helper_indexes(); - let sort_helper_indexes = snapshot.sort_helper_indexes; - - assert!(!sort_helper_indexes.contains_key("stale")); - assert!(sort_helper_indexes.contains_key("fresh")); } - #[test] - fn sorted_query_transparently_rebuilds_a_purged_helper_index() { - let mut index = EngineSnapshot::default(); - declare_test_param( - &mut index, - "priority", - r"priority=(?P\d+)", - "priority", - MetricPrimitiveKind::Number, - ); - ingest_log(&mut index, 0, "priority=3 boom"); - ingest_log(&mut index, 1, "priority=1 boom"); - ingest_log(&mut index, 2, "priority=2 boom"); - - let mut sort_helper_indexes = HashMap::new(); - let _ = handle_query(&index, logs_query_sorted("priority", None), &mut sort_helper_indexes); - assert!(sort_helper_indexes.contains_key("priority")); - - sort_helper_indexes.remove("priority"); - - match handle_query(&index, logs_query_sorted("priority", None), &mut sort_helper_indexes) { - EngineResponse::Logs(entries) => { - let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); - assert_eq!(sequences, vec![1, 2, 0]); - } - _ => panic!("expected Logs response"), + match handle_query( + &index, + logs_query(Some("ottyl_ts>2000")), + &mut HashMap::new(), + ) { + EngineResponse::Logs(entries) => { + let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); + assert_eq!(sequences, vec![3, 4]); } - assert!(sort_helper_indexes.contains_key("priority")); + _ => panic!("expected Logs response"), + } +} + +#[test] +fn logs_query_sort_by_orders_lines_by_declared_param_value() { + let mut index = EngineSnapshot::default(); + declare_test_param( + &mut index, + "priority", + r"priority=(?P\d+)", + "priority", + MetricPrimitiveKind::Number, + ); + ingest_log(&mut index, 0, "priority=3 boom"); + ingest_log(&mut index, 1, "priority=1 boom"); + ingest_log(&mut index, 2, "priority=2 boom"); + + match handle_query( + &index, + logs_query_sorted("priority", None), + &mut HashMap::new(), + ) { + EngineResponse::Logs(entries) => { + let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); + assert_eq!(sequences, vec![1, 2, 0]); + } + _ => panic!("expected Logs response"), + } +} + +#[test] +fn logs_query_sort_by_missing_param_fallback_orders_by_sequence_and_appends() { + let mut index = EngineSnapshot::default(); + declare_test_param( + &mut index, + "priority", + r"priority=(?P\d+)", + "priority", + MetricPrimitiveKind::Number, + ); + ingest_log(&mut index, 0, "priority=3 boom"); + ingest_log(&mut index, 1, "no param here"); + ingest_log(&mut index, 2, "priority=1 boom"); + ingest_log(&mut index, 3, "also missing"); + + match handle_query( + &index, + logs_query_sorted("priority", Some(true)), + &mut HashMap::new(), + ) { + EngineResponse::Logs(entries) => { + let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); + assert_eq!(sequences, vec![2, 0, 1, 3]); + } + _ => panic!("expected Logs response"), + } +} + +#[test] +fn logs_query_sort_by_missing_param_drop_excludes_them() { + let mut index = EngineSnapshot::default(); + declare_test_param( + &mut index, + "priority", + r"priority=(?P\d+)", + "priority", + MetricPrimitiveKind::Number, + ); + ingest_log(&mut index, 0, "priority=3 boom"); + ingest_log(&mut index, 1, "no param here"); + ingest_log(&mut index, 2, "priority=1 boom"); + ingest_log(&mut index, 3, "also missing"); + + match handle_query( + &index, + logs_query_sorted("priority", Some(false)), + &mut HashMap::new(), + ) { + EngineResponse::Logs(entries) => { + let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); + assert_eq!(sequences, vec![2, 0]); + } + _ => panic!("expected Logs response"), + } +} + +#[test] +fn logs_query_sort_by_unknown_param_returns_error() { + let mut index = EngineSnapshot::default(); + ingest_log(&mut index, 0, "hello"); + + match handle_query( + &index, + logs_query_sorted("does_not_exist", None), + &mut HashMap::new(), + ) { + EngineResponse::Error(err) => assert!(err.contains("does_not_exist")), + _ => panic!("expected EngineResponse::Error"), + } +} + +#[test] +fn logs_query_without_sort_by_keeps_default_ottyl_ts_order() { + let mut index = EngineSnapshot::default(); + declare_test_param( + &mut index, + "priority", + r"priority=(?P\d+)", + "priority", + MetricPrimitiveKind::Number, + ); + ingest_log(&mut index, 0, "priority=3 boom"); + ingest_log(&mut index, 1, "no param here"); + ingest_log(&mut index, 2, "priority=1 boom"); + ingest_log(&mut index, 3, "also missing"); + + match handle_query(&index, logs_query(None), &mut HashMap::new()) { + EngineResponse::Logs(entries) => { + let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); + assert_eq!(sequences, vec![0, 1, 2, 3]); + } + _ => panic!("expected Logs response"), + } +} + +#[test] +fn sort_helper_index_is_built_lazily_and_reused_across_queries() { + let mut index = EngineSnapshot::default(); + declare_test_param( + &mut index, + "priority", + r"priority=(?P\d+)", + "priority", + MetricPrimitiveKind::Number, + ); + ingest_log(&mut index, 0, "priority=3 boom"); + ingest_log(&mut index, 1, "priority=1 boom"); + ingest_log(&mut index, 2, "priority=2 boom"); + + let mut sort_helper_indexes = HashMap::new(); + assert!(!sort_helper_indexes.contains_key("priority")); + + match handle_query( + &index, + logs_query_sorted("priority", None), + &mut sort_helper_indexes, + ) { + EngineResponse::Logs(entries) => { + let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); + assert_eq!(sequences, vec![1, 2, 0]); + } + _ => panic!("expected Logs response"), + } + assert!(sort_helper_indexes.contains_key("priority")); + assert_eq!(sort_helper_indexes["priority"].index.len(), 3); + + ingest_log(&mut index, 3, "priority=0 boom"); + std::mem::swap(&mut index.sort_helper_indexes, &mut sort_helper_indexes); + index.update_sort_helper_indexes_on_ingest(3, Timestamp::from_millis(3), "priority=0 boom"); + std::mem::swap(&mut index.sort_helper_indexes, &mut sort_helper_indexes); + + ingest_log(&mut index, 4, "no priority here"); + std::mem::swap(&mut index.sort_helper_indexes, &mut sort_helper_indexes); + index.update_sort_helper_indexes_on_ingest(4, Timestamp::from_millis(4), "no priority here"); + std::mem::swap(&mut index.sort_helper_indexes, &mut sort_helper_indexes); + assert_eq!(sort_helper_indexes["priority"].index.len(), 5); + + match handle_query( + &index, + logs_query_sorted("priority", None), + &mut sort_helper_indexes, + ) { + EngineResponse::Logs(entries) => { + let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); + assert_eq!(sequences, vec![3, 1, 2, 0, 4]); + } + _ => panic!("expected Logs response"), + } +} + +#[test] +fn sort_helper_index_incremental_update_is_a_noop_when_no_index_is_live() { + let mut index = EngineSnapshot::default(); + index.update_sort_helper_indexes_on_ingest(0, Timestamp::from_millis(0), "hello"); + assert!(index.sort_helper_indexes.is_empty()); +} + +#[test] +fn purge_stale_sort_helper_indexes_drops_only_entries_past_the_purge_duration() { + let mut sort_helper_indexes: HashMap = HashMap::new(); + sort_helper_indexes.insert( + "stale".to_string(), + SortHelperIndexEntry { + index: BTreeMap::new(), + last_accessed: std::time::Instant::now() + - SORT_HELPER_INDEX_PURGE_AFTER + - std::time::Duration::from_secs(1), + }, + ); + sort_helper_indexes.insert( + "fresh".to_string(), + SortHelperIndexEntry { + index: BTreeMap::new(), + last_accessed: std::time::Instant::now(), + }, + ); + + let mut snapshot = EngineSnapshot { + sort_helper_indexes, + ..Default::default() + }; + snapshot.purge_stale_sort_helper_indexes(); + let sort_helper_indexes = snapshot.sort_helper_indexes; + + assert!(!sort_helper_indexes.contains_key("stale")); + assert!(sort_helper_indexes.contains_key("fresh")); +} + +#[test] +fn sorted_query_transparently_rebuilds_a_purged_helper_index() { + let mut index = EngineSnapshot::default(); + declare_test_param( + &mut index, + "priority", + r"priority=(?P\d+)", + "priority", + MetricPrimitiveKind::Number, + ); + ingest_log(&mut index, 0, "priority=3 boom"); + ingest_log(&mut index, 1, "priority=1 boom"); + ingest_log(&mut index, 2, "priority=2 boom"); + + let mut sort_helper_indexes = HashMap::new(); + let _ = handle_query( + &index, + logs_query_sorted("priority", None), + &mut sort_helper_indexes, + ); + assert!(sort_helper_indexes.contains_key("priority")); + + sort_helper_indexes.remove("priority"); + + match handle_query( + &index, + logs_query_sorted("priority", None), + &mut sort_helper_indexes, + ) { + EngineResponse::Logs(entries) => { + let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); + assert_eq!(sequences, vec![1, 2, 0]); + } + _ => panic!("expected Logs response"), + } + assert!(sort_helper_indexes.contains_key("priority")); +} + +#[test] +fn logs_query_with_ottyl_fmt_comparison_filters_by_detected_format_for_mixed_format_lines() { + let mut index = EngineSnapshot::default(); + ingest_log(&mut index, 0, r#"{"level":"error","code":500}"#); + ingest_log( + &mut index, + 1, + r#"ts=1788648105773 msg="entity_moved" x1=-4"#, + ); + ingest_log(&mut index, 2, "just some plain prose here"); + ingest_log(&mut index, 3, r#"{"level":"info","code":200}"#); + + match handle_query( + &index, + logs_query(Some("ottyl_fmt==json")), + &mut HashMap::new(), + ) { + EngineResponse::Logs(entries) => { + let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); + assert_eq!(sequences, vec![0, 3]); + } + _ => panic!("expected Logs response"), } - #[test] - fn logs_query_with_ottyl_fmt_comparison_filters_by_detected_format_for_mixed_format_lines() { - let mut index = EngineSnapshot::default(); - ingest_log(&mut index, 0, r#"{"level":"error","code":500}"#); - ingest_log(&mut index, 1, r#"ts=1788648105773 msg="entity_moved" x1=-4"#); - ingest_log(&mut index, 2, "just some plain prose here"); - ingest_log(&mut index, 3, r#"{"level":"info","code":200}"#); - - match handle_query(&index, logs_query(Some("ottyl_fmt==json")), &mut HashMap::new()) { - EngineResponse::Logs(entries) => { - let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); - assert_eq!(sequences, vec![0, 3]); - } - _ => panic!("expected Logs response"), - } - - match handle_query(&index, logs_query(Some("ottyl_fmt==logfmt")), &mut HashMap::new()) { - EngineResponse::Logs(entries) => { - let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); - assert_eq!(sequences, vec![1]); - } - _ => panic!("expected Logs response"), - } - - match handle_query(&index, logs_query(Some("ottyl_fmt==none")), &mut HashMap::new()) { - EngineResponse::Logs(entries) => { - let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); - assert_eq!(sequences, vec![2]); - } - _ => panic!("expected Logs response"), + match handle_query( + &index, + logs_query(Some("ottyl_fmt==logfmt")), + &mut HashMap::new(), + ) { + EngineResponse::Logs(entries) => { + let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); + assert_eq!(sequences, vec![1]); } + _ => panic!("expected Logs response"), } - #[test] - fn logs_query_with_json_field_comparison_filters_by_json_field() { - let mut index = EngineSnapshot::default(); - ingest_log(&mut index, 0, r#"{"level":"error","code":500}"#); - ingest_log(&mut index, 1, r#"{"level":"info","code":200}"#); - ingest_log(&mut index, 2, "not json at all"); - - match handle_query(&index, logs_query(Some(r#"json[level]==error"#)), &mut HashMap::new()) { - EngineResponse::Logs(entries) => { - let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); - assert_eq!(sequences, vec![0]); - } - _ => panic!("expected Logs response"), + match handle_query( + &index, + logs_query(Some("ottyl_fmt==none")), + &mut HashMap::new(), + ) { + EngineResponse::Logs(entries) => { + let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); + assert_eq!(sequences, vec![2]); } + _ => panic!("expected Logs response"), + } +} - match handle_query(&index, logs_query(Some("json[code]>=300")), &mut HashMap::new()) { - EngineResponse::Logs(entries) => { - let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); - assert_eq!(sequences, vec![0]); - } - _ => panic!("expected Logs response"), +#[test] +fn logs_query_with_json_field_comparison_filters_by_json_field() { + let mut index = EngineSnapshot::default(); + ingest_log(&mut index, 0, r#"{"level":"error","code":500}"#); + ingest_log(&mut index, 1, r#"{"level":"info","code":200}"#); + ingest_log(&mut index, 2, "not json at all"); + + match handle_query( + &index, + logs_query(Some(r#"json[level]==error"#)), + &mut HashMap::new(), + ) { + EngineResponse::Logs(entries) => { + let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); + assert_eq!(sequences, vec![0]); } + _ => panic!("expected Logs response"), } - #[test] - fn logs_query_combines_regex_and_metric_comparison_with_and() { - let mut index = EngineSnapshot::default(); - let pattern = r"level=(?P\w+) duration=(?P\d+)"; - declare_test_param(&mut index, "level", pattern, "level", MetricPrimitiveKind::String); - index.declare_metric(metric_request("requests", pattern, vec!["level"])).unwrap(); - - ingest_log(&mut index, 0, "level=error duration=1 boom"); - ingest_log(&mut index, 1, "level=info duration=2 boom"); - ingest_log(&mut index, 2, "level=error duration=3 all good"); - - match handle_query( - &index, - logs_query(Some(r#"regex("boom") and requests[level]==error"#)), - &mut HashMap::new(), - ) { - EngineResponse::Logs(entries) => { - let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); - assert_eq!(sequences, vec![0]); - } - _ => panic!("expected Logs response"), + match handle_query( + &index, + logs_query(Some("json[code]>=300")), + &mut HashMap::new(), + ) { + EngineResponse::Logs(entries) => { + let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); + assert_eq!(sequences, vec![0]); } + _ => panic!("expected Logs response"), + } +} - match handle_query( - &index, - logs_query(Some(r#"regex("boom") or requests[level]==error"#)), - &mut HashMap::new(), - ) { - EngineResponse::Logs(entries) => { - let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); - assert_eq!(sequences, vec![0, 1, 2]); - } - _ => panic!("expected Logs response"), +#[test] +fn logs_query_combines_regex_and_metric_comparison_with_and() { + let mut index = EngineSnapshot::default(); + let pattern = r"level=(?P\w+) duration=(?P\d+)"; + declare_test_param( + &mut index, + "level", + pattern, + "level", + MetricPrimitiveKind::String, + ); + index + .declare_metric(metric_request("requests", pattern, vec!["level"])) + .unwrap(); + + ingest_log(&mut index, 0, "level=error duration=1 boom"); + ingest_log(&mut index, 1, "level=info duration=2 boom"); + ingest_log(&mut index, 2, "level=error duration=3 all good"); + + match handle_query( + &index, + logs_query(Some(r#"regex("boom") and requests[level]==error"#)), + &mut HashMap::new(), + ) { + EngineResponse::Logs(entries) => { + let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); + assert_eq!(sequences, vec![0]); } + _ => panic!("expected Logs response"), } - #[test] - fn logs_query_includes_declared_params_on_matching_lines_only() { - let mut index = EngineSnapshot::default(); - declare_param(&mut index, request("level", r"level=(?P\w+)", "level")).unwrap(); - ingest_log(&mut index, 0, "level=info hello"); - ingest_log(&mut index, 1, "no match here"); - - let EngineResponse::Logs(entries) = handle_query(&index, logs_query(None), &mut HashMap::new()) else { - panic!("expected Logs response"); - }; - - assert_eq!(entries.len(), 2); - assert_eq!( - entries[0].params.get("level"), - Some(&MetricPrimitive::String("info".to_string())) - ); - assert!(entries[1].params.get("level").is_none()); - } - - #[test] - fn boolean_param_declares_without_capture_group_and_is_always_present() { - let mut index = EngineSnapshot::default(); - let mut req = request("IsError", r"(?i)error", ""); - req.kind = MetricPrimitiveKind::Boolean; - assert!(declare_param(&mut index, req).is_ok()); - - ingest_log(&mut index, 0, "an ERROR occurred"); - ingest_log(&mut index, 1, "everything is fine"); - - let EngineResponse::Logs(entries) = handle_query(&index, logs_query(None), &mut HashMap::new()) else { - panic!("expected Logs response"); - }; - - assert_eq!(entries.len(), 2); - assert_eq!(entries[0].params.get("IsError"), Some(&MetricPrimitive::Boolean(true))); - assert_eq!(entries[1].params.get("IsError"), Some(&MetricPrimitive::Boolean(false))); - } - - #[test] - fn timeline_query_applies_filter_before_bucketing() { - let mut index = EngineSnapshot::default(); - ingest_log(&mut index, 0, "everything is fine"); - ingest_log(&mut index, 1, "an error occurred"); - ingest_log(&mut index, 2, "another error here"); - - match handle_query(&index, timeline_query(Some(r#"regex("error")"#), 1_000_000), &mut HashMap::new()) { - EngineResponse::Timeline(buckets) => { - let total: u64 = buckets.iter().map(|b| b.count).sum(); - assert_eq!(total, 2); - } - _ => panic!("expected Timeline response"), + match handle_query( + &index, + logs_query(Some(r#"regex("boom") or requests[level]==error"#)), + &mut HashMap::new(), + ) { + EngineResponse::Logs(entries) => { + let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); + assert_eq!(sequences, vec![0, 1, 2]); } + _ => panic!("expected Logs response"), + } +} + +#[test] +fn logs_query_includes_declared_params_on_matching_lines_only() { + let mut index = EngineSnapshot::default(); + declare_param( + &mut index, + request("level", r"level=(?P\w+)", "level"), + ) + .unwrap(); + ingest_log(&mut index, 0, "level=info hello"); + ingest_log(&mut index, 1, "no match here"); + + let EngineResponse::Logs(entries) = handle_query(&index, logs_query(None), &mut HashMap::new()) + else { + panic!("expected Logs response"); + }; + + assert_eq!(entries.len(), 2); + assert_eq!( + entries[0].params.get("level"), + Some(&MetricPrimitive::String("info".to_string())) + ); + assert!(!entries[1].params.contains_key("level")); +} + +#[test] +fn boolean_param_declares_without_capture_group_and_is_always_present() { + let mut index = EngineSnapshot::default(); + let mut req = request("IsError", r"(?i)error", ""); + req.kind = MetricPrimitiveKind::Boolean; + assert!(declare_param(&mut index, req).is_ok()); + + ingest_log(&mut index, 0, "an ERROR occurred"); + ingest_log(&mut index, 1, "everything is fine"); + + let EngineResponse::Logs(entries) = handle_query(&index, logs_query(None), &mut HashMap::new()) + else { + panic!("expected Logs response"); + }; + + assert_eq!(entries.len(), 2); + assert_eq!( + entries[0].params.get("IsError"), + Some(&MetricPrimitive::Boolean(true)) + ); + assert_eq!( + entries[1].params.get("IsError"), + Some(&MetricPrimitive::Boolean(false)) + ); +} + +#[test] +fn timeline_query_applies_filter_before_bucketing() { + let mut index = EngineSnapshot::default(); + ingest_log(&mut index, 0, "everything is fine"); + ingest_log(&mut index, 1, "an error occurred"); + ingest_log(&mut index, 2, "another error here"); + + match handle_query( + &index, + timeline_query(Some(r#"regex("error")"#), 1_000_000), + &mut HashMap::new(), + ) { + EngineResponse::Timeline(buckets) => { + let total: u64 = buckets.iter().map(|b| b.count).sum(); + assert_eq!(total, 2); + } + _ => panic!("expected Timeline response"), + } +} + +#[test] +fn malformed_filter_string_surfaces_as_engine_error() { + let mut index = EngineSnapshot::default(); + ingest_log(&mut index, 0, "hello"); + + match handle_query( + &index, + logs_query(Some("@@@ not a valid filter @@@")), + &mut HashMap::new(), + ) { + EngineResponse::Error(message) => assert!(!message.is_empty()), + _ => panic!("expected an Error response for a malformed filter string"), } - #[test] - fn malformed_filter_string_surfaces_as_engine_error() { - let mut index = EngineSnapshot::default(); - ingest_log(&mut index, 0, "hello"); - - match handle_query(&index, logs_query(Some("@@@ not a valid filter @@@")), &mut HashMap::new()) { - EngineResponse::Error(message) => assert!(!message.is_empty()), - _ => panic!("expected an Error response for a malformed filter string"), - } - - match handle_query(&index, timeline_query(Some("@@@ not a valid filter @@@"), 1000), &mut HashMap::new()) { - EngineResponse::Error(message) => assert!(!message.is_empty()), - _ => panic!("expected an Error response for a malformed filter string"), - } + match handle_query( + &index, + timeline_query(Some("@@@ not a valid filter @@@"), 1000), + &mut HashMap::new(), + ) { + EngineResponse::Error(message) => assert!(!message.is_empty()), + _ => panic!("expected an Error response for a malformed filter string"), } +} - #[test] - fn metric_comparison_without_param_is_a_compile_error() { - let mut index = EngineSnapshot::default(); - let pattern = r"level=(?P\w+)"; - declare_test_param(&mut index, "level", pattern, "level", MetricPrimitiveKind::String); - index.declare_metric(metric_request("requests", pattern, vec!["level"])).unwrap(); - ingest_log(&mut index, 0, "level=error"); +#[test] +fn metric_comparison_without_param_is_a_compile_error() { + let mut index = EngineSnapshot::default(); + let pattern = r"level=(?P\w+)"; + declare_test_param( + &mut index, + "level", + pattern, + "level", + MetricPrimitiveKind::String, + ); + index + .declare_metric(metric_request("requests", pattern, vec!["level"])) + .unwrap(); + ingest_log(&mut index, 0, "level=error"); - match handle_query(&index, logs_query(Some("requests==5")), &mut HashMap::new()) { - EngineResponse::Error(message) => assert!(message.contains("param")), - _ => panic!("expected an Error response for a metric comparison without [param]"), - } + match handle_query(&index, logs_query(Some("requests==5")), &mut HashMap::new()) { + EngineResponse::Error(message) => assert!(message.contains("param")), + _ => panic!("expected an Error response for a metric comparison without [param]"), } +} - #[test] - fn metric_comparison_with_undeclared_param_is_rejected_naming_real_label_params() { - let mut index = EngineSnapshot::default(); - let pattern = r"level=(?P\w+) duration=(?P\d+)"; - declare_test_param(&mut index, "level", pattern, "level", MetricPrimitiveKind::String); - declare_test_param(&mut index, "duration", pattern, "duration", MetricPrimitiveKind::Number); - index.declare_metric(metric_request("requests", pattern, vec!["level", "duration"])).unwrap(); - ingest_log(&mut index, 0, "level=error duration=5"); +#[test] +fn metric_comparison_with_undeclared_param_is_rejected_naming_real_label_params() { + let mut index = EngineSnapshot::default(); + let pattern = r"level=(?P\w+) duration=(?P\d+)"; + declare_test_param( + &mut index, + "level", + pattern, + "level", + MetricPrimitiveKind::String, + ); + declare_test_param( + &mut index, + "duration", + pattern, + "duration", + MetricPrimitiveKind::Number, + ); + index + .declare_metric(metric_request( + "requests", + pattern, + vec!["level", "duration"], + )) + .unwrap(); + ingest_log(&mut index, 0, "level=error duration=5"); - match handle_query(&index, logs_query(Some("requests[nonexistent_param]==\"x\"")), &mut HashMap::new()) { - EngineResponse::Error(message) => { - assert!( - message.contains("nonexistent_param"), - "error should name the offending param, got: {message}" - ); - assert!( - message.contains("requests"), - "error should name the metric, got: {message}" - ); - assert!( - message.contains("level") && message.contains("duration"), - "error should list the metric's real declared label params, got: {message}" - ); - } - _ => panic!("expected an Error response for an undeclared param"), + match handle_query( + &index, + logs_query(Some("requests[nonexistent_param]==\"x\"")), + &mut HashMap::new(), + ) { + EngineResponse::Error(message) => { + assert!( + message.contains("nonexistent_param"), + "error should name the offending param, got: {message}" + ); + assert!( + message.contains("requests"), + "error should name the metric, got: {message}" + ); + assert!( + message.contains("level") && message.contains("duration"), + "error should list the metric's real declared label params, got: {message}" + ); } + _ => panic!("expected an Error response for an undeclared param"), } +} - #[test] - fn metric_comparison_with_valid_label_param_still_matches() { - let mut index = EngineSnapshot::default(); - let pattern = r"level=(?P\w+) duration=(?P\d+)"; - declare_test_param(&mut index, "level", pattern, "level", MetricPrimitiveKind::String); - declare_test_param(&mut index, "duration", pattern, "duration", MetricPrimitiveKind::Number); - index.declare_metric(metric_request("requests", pattern, vec!["level", "duration"])).unwrap(); +#[test] +fn metric_comparison_with_valid_label_param_still_matches() { + let mut index = EngineSnapshot::default(); + let pattern = r"level=(?P\w+) duration=(?P\d+)"; + declare_test_param( + &mut index, + "level", + pattern, + "level", + MetricPrimitiveKind::String, + ); + declare_test_param( + &mut index, + "duration", + pattern, + "duration", + MetricPrimitiveKind::Number, + ); + index + .declare_metric(metric_request( + "requests", + pattern, + vec!["level", "duration"], + )) + .unwrap(); - ingest_log(&mut index, 0, "level=error duration=5"); - ingest_log(&mut index, 1, "level=info duration=2"); + ingest_log(&mut index, 0, "level=error duration=5"); + ingest_log(&mut index, 1, "level=info duration=2"); - match handle_query(&index, logs_query(Some("requests[level]==error")), &mut HashMap::new()) { - EngineResponse::Logs(entries) => { - let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); - assert_eq!(sequences, vec![0]); - } - _ => panic!("expected Logs response"), + match handle_query( + &index, + logs_query(Some("requests[level]==error")), + &mut HashMap::new(), + ) { + EngineResponse::Logs(entries) => { + let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); + assert_eq!(sequences, vec![0]); } + _ => panic!("expected Logs response"), } +} diff --git a/tickets/094-adopt-rustfmt.md b/tickets/094-adopt-rustfmt.md deleted file mode 100644 index 60f0b5b..0000000 --- a/tickets/094-adopt-rustfmt.md +++ /dev/null @@ -1,34 +0,0 @@ -# 094 — Adopt rustfmt and format the codebase - -## Summary - -The Rust backend has no `rustfmt.toml` and isn't currently formatted -to rustfmt's defaults: `cargo fmt -- --check` exits non-zero, reporting -diffs across 24 files (352 diff hunks total). - -## Where - -- No `rustfmt.toml` / `.rustfmt.toml` exists at the repo root. -- `cargo fmt -- --check` reports diffs in `src/api/{dto,mcp_server,mod}.rs`, - `src/cli/{mod,sessions}.rs`, `src/config/{mod,profile}.rs`, and most - of `src/engine/**` (`alerts.rs`, `data_value.rs`, `filter_datetime.rs`, - `filter_parser.rs`, `ingest.rs`, `logfmt.rs`, `message.rs`, `mod.rs`, - `promql.rs`, `rate.rs`, `snapshot/{filter,metrics,mod,params,query,sort}.rs`, - `tests.rs`). Heaviest: `src/engine/snapshot/mod.rs` (~65 hunks), - `src/config/profile.rs` (~38), `src/api/mod.rs` (~35), - `src/engine/alerts.rs` (~28). -- `Makefile:26` (`lint-be`) currently only runs `cargo clippy - --all-targets` — no `cargo fmt --check` step. - -## Scope - -- Add a `rustfmt.toml` if any defaults should be overridden (otherwise - an empty/absent file is fine and rustfmt defaults apply — decide - whether to keep the file empty for documentation purposes or omit - it). -- Run `cargo fmt` across the whole `src/` tree as a single, - non-functional formatting commit (don't mix with logic changes, to - keep the diff reviewable and `git blame`-friendly). -- Add a `cargo fmt --check` step to `make lint` (or a new `make - fmt-check` wired into CI) so future changes can't drift out of - format again. diff --git a/tickets/095-enforce-clippy-clean.md b/tickets/095-enforce-clippy-clean.md deleted file mode 100644 index 82b3374..0000000 --- a/tickets/095-enforce-clippy-clean.md +++ /dev/null @@ -1,41 +0,0 @@ -# 095 — Resolve clippy warnings and make lint enforce a clean run - -## Summary - -`make lint`'s backend target runs `cargo clippy --all-targets` with no -`-D warnings` (or equivalent), so clippy warnings accumulate silently -instead of failing the lint step. There are currently 23 unique -warnings across 5 lint categories. - -## Where - -- `Makefile:26` (`lint-be:`) — `cargo clippy --all-targets`, no denial - flag. -- `cargo clippy --all-targets` currently reports, by lint: - - `clippy::cloned_ref_to_slice_refs` — 10 occurrences, mostly in - `src/engine/alerts.rs` test code (e.g. around lines 469, 472, 493, - 534). - - `clippy::collapsible_if` — 8 occurrences, concentrated in - `src/engine/snapshot/mod.rs` (lines 608, 940, 961) and - `src/engine/alerts.rs` (around lines 510, 512). - - `clippy::type_complexity` — 3 occurrences. - - `clippy::single_match` — 1 occurrence. - - `clippy::unnecessary_get_then_check` — 1 occurrence, - `src/engine/tests.rs:2234` - (`entries[1].params.get("level").is_none()`). - -## Scope - -- Fix all 23 current warnings (mechanical for most: `from_ref` for the - slice-ref clone cases, collapsing nested `if`s, `is_none()` → - `!contains_key`/`get().is_none()` equivalent simplification per - clippy's suggestion, breaking up the complex types clippy flags or - adding a type alias, converting the single-arm `match` to `if let`). -- Once clean, change `Makefile:26` to deny warnings (e.g. `cargo - clippy --all-targets -- -D warnings`) so the target fails on any - future regression instead of just printing them. -- Don't add blanket `#[allow(...)]`s to silence these — fix the - underlying code per clippy's suggestion, matching the existing - `#[allow(dead_code)]` uses elsewhere in the codebase which are for - genuinely-unused-for-now items, not a precedent for suppressing - active lints.