diff --git a/src/alerts.rs b/src/alerts.rs index 6c4d1de..5340c21 100644 --- a/src/alerts.rs +++ b/src/alerts.rs @@ -1,6 +1,6 @@ //! Alert rules: definitions, trigger evaluation, and ntfy delivery. //! -//! Alert queries are restricted to regex/sequence/timestamp — a metric[param] comparison would require a full bucket rescan per line, breaking the proportional-cost guarantee. +//! Alert queries are restricted to regex/sequence/timestamp/json[field] — a metric[param] comparison would require a full bucket rescan per line, breaking the proportional-cost guarantee. json[field] is fine here since, like regex/sequence/timestamp, it only inspects the line's own text/fields, no index lookup. use std::collections::hash_map::DefaultHasher; use std::collections::{HashMap, HashSet, VecDeque}; @@ -13,7 +13,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use regex::Regex; use serde::{Deserialize, Serialize}; -use crate::engine::apply_compare_op; +use crate::engine::{apply_compare_op, eval_json_comparison}; use crate::filter_datetime; use crate::filter_parser::{self, CompareOp, FilterExpr, FilterValue}; use crate::settings::Settings; @@ -155,7 +155,7 @@ fn prune_window(timestamps: &mut VecDeque, now_ms: i64, window_ms: i64) { } } -/// Evaluates a FilterBar query against a single line's own fields only (metric[param] comparisons aren't supported — see module doc). +/// Evaluates a FilterBar query against a single line's own fields only (metric[param] comparisons aren't supported — see module doc; json[field] is supported since it's also line-local). fn eval_alert_query(query: &str, sequence: u64, timestamp: Timestamp, text: &str) -> Result { let expr = filter_parser::parse(query).map_err(|err| err.to_string())?; eval_alert_expr(&expr, sequence, timestamp, text) @@ -189,6 +189,9 @@ fn eval_alert_expr(expr: &FilterExpr, sequence: u64, timestamp: Timestamp, text: "alert queries only support regex()/sequence/timestamp (not metric comparisons like \ '{metric}[param]'): metric-based alert conditions aren't supported yet" )), + FilterExpr::JsonComparison { field, op, value } => { + Ok(eval_json_comparison(text, field, *op, value)) + } FilterExpr::And(lhs, rhs) => { Ok(eval_alert_expr(lhs, sequence, timestamp, text)? && eval_alert_expr(rhs, sequence, timestamp, text)?) } diff --git a/src/engine.rs b/src/engine.rs index d9b11ee..3fe98e9 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -482,6 +482,10 @@ enum CompiledFilter { Sequence { op: CompareOp, target: f64 }, Timestamp { op: CompareOp, target_ms: i64 }, MetricMatch(HashSet), + /// `json[field] value` — unlike `MetricMatch`, this can't be pre-resolved against the + /// index (the JSON lives in the log line itself, not a declared metric/param), so it's + /// evaluated per line in `eval_compiled` by parsing `text` as JSON. + Json { field: String, op: CompareOp, value: FilterValue }, And(Box, Box), Or(Box, Box), } @@ -510,6 +514,11 @@ fn compile_filter( "timestamp" => compile_timestamp_comparison(*op, value, now), other => compile_metric_comparison(index, other, param.as_deref(), *op, value, now), }, + FilterExpr::JsonComparison { field, op, value } => Ok(CompiledFilter::Json { + field: field.clone(), + op: *op, + value: value.clone(), + }), FilterExpr::And(lhs, rhs) => Ok(CompiledFilter::And( Box::new(compile_filter(index, lhs, now)?), Box::new(compile_filter(index, rhs, now)?), @@ -620,6 +629,59 @@ fn compare_metric_primitive( Ok(ordering.is_some_and(|ordering| apply_compare_op(ordering, op))) } +/// `json[field] value`: the log line is parsed as JSON fresh on every call (no per-line +/// cache), since this doesn't go through the index-backed pre-resolution `MetricMatch` uses. +/// Any line that isn't a JSON object, or lacks `field`, simply doesn't match — no error. +pub(crate) fn eval_json_comparison(text: &str, field: &str, op: CompareOp, value: &FilterValue) -> bool { + let Ok(parsed) = serde_json::from_str::(text) else { + return false; + }; + let Some(field_value) = parsed.as_object().and_then(|obj| obj.get(field)) else { + return false; + }; + compare_json_value(field_value, op, value) +} + +/// Mirrors `compare_metric_primitive`'s "mismatched kinds never match" behavior. Only `Eq`/`Ne` +/// are meaningful for booleans, and `null`/array/object fields aren't scalar-comparable at all. +fn compare_json_value(field_value: &serde_json::Value, op: CompareOp, value: &FilterValue) -> bool { + match field_value { + serde_json::Value::Number(n) => { + let target = match value { + FilterValue::Number(v) => Some(*v), + FilterValue::Raw(raw) => raw.parse::().ok(), + }; + match (n.as_f64(), target) { + (Some(f), Some(v)) => f.partial_cmp(&v).is_some_and(|ordering| apply_compare_op(ordering, op)), + _ => false, + } + } + serde_json::Value::String(s) => { + let target = match value { + FilterValue::Raw(raw) => raw.clone(), + FilterValue::Number(n) => n.to_string(), + }; + s.as_str() + .partial_cmp(target.as_str()) + .is_some_and(|ordering| apply_compare_op(ordering, op)) + } + serde_json::Value::Bool(b) => { + let FilterValue::Raw(raw) = value else { + return false; + }; + match raw.parse::() { + Ok(v) => match op { + CompareOp::Eq => *b == v, + CompareOp::Ne => *b != v, + _ => false, + }, + Err(_) => false, + } + } + serde_json::Value::Null | serde_json::Value::Array(_) | serde_json::Value::Object(_) => false, + } +} + /// `pub(crate)` so `crate::alerts` can reuse it for `Threshold`/`Ratio` comparisons instead of duplicating the match. pub(crate) fn apply_compare_op(ordering: Ordering, op: CompareOp) -> bool { match op { @@ -643,6 +705,7 @@ fn eval_compiled(compiled: &CompiledFilter, sequence: u64, timestamp: Timestamp, apply_compare_op(timestamp.as_millis().cmp(target_ms), *op) } CompiledFilter::MetricMatch(sequences) => sequences.contains(&sequence), + CompiledFilter::Json { field, op, value } => eval_json_comparison(text, field, *op, value), CompiledFilter::And(lhs, rhs) => { eval_compiled(lhs, sequence, timestamp, text) && eval_compiled(rhs, sequence, timestamp, text) } @@ -1915,6 +1978,30 @@ mod tests { } } + #[test] + fn logs_query_with_json_field_comparison_filters_by_json_field() { + let mut index = BTreeMap::new(); + ingest_log(&mut index, 0, r#"{"level":"error","code":500}"#); + ingest_log(&mut index, 1, r#"{"level":"info","code":200}"#); + ingest_log(&mut index, 2, "not json at all"); + + match handle_query(&index, logs_query(Some(r#"json[level]==error"#)), 0) { + EngineResponse::Logs(entries) => { + let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); + assert_eq!(sequences, vec![0]); + } + _ => panic!("expected Logs response"), + } + + match handle_query(&index, logs_query(Some("json[code]>=300")), 0) { + EngineResponse::Logs(entries) => { + let sequences: Vec = entries.iter().map(|e| e.sequence).collect(); + assert_eq!(sequences, vec![0]); + } + _ => panic!("expected Logs response"), + } + } + #[test] fn logs_query_combines_regex_and_metric_comparison_with_and() { let mut index = BTreeMap::new(); diff --git a/src/filter_parser.pest b/src/filter_parser.pest index 1c7f59a..3ccaeba 100644 --- a/src/filter_parser.pest +++ b/src/filter_parser.pest @@ -3,6 +3,7 @@ // Supports: // - regex("...") raw-text regex search over log lines // - metric_name[param] value metric comparisons +// - json[field] value JSON field comparisons (line parsed as JSON) // - sequence value reserved metric name (no [param]) // - timestamp value reserved metric name (no [param]) // - `and` / `or` boolean combinators, with `and` binding tighter than `or` @@ -31,7 +32,12 @@ string_literal = ${ "\"" ~ inner_str ~ "\"" } inner_str = @{ (escaped_char | !("\"" | "\\") ~ ANY)* } escaped_char = @{ "\\" ~ ANY } -comparison = { metric ~ ("[" ~ param ~ "]")? ~ op ~ value } +comparison = _{ json_comparison | metric_comparison } + +// `json[field] value` must be tried before `metric_comparison` since "json" is +// otherwise a syntactically valid (if never declared) metric name. +json_comparison = { "json" ~ "[" ~ param ~ "]" ~ op ~ value } +metric_comparison = { metric ~ ("[" ~ param ~ "]")? ~ op ~ value } metric = @{ ident } param = @{ ident } diff --git a/src/filter_parser.rs b/src/filter_parser.rs index 7289bad..efdc6f3 100644 --- a/src/filter_parser.rs +++ b/src/filter_parser.rs @@ -23,6 +23,13 @@ pub enum FilterExpr { op: CompareOp, value: FilterValue, }, + /// `json[field] value` — a comparison against a field of the line's parsed JSON. + /// Lines that aren't valid JSON (or lack the field) simply don't match. + JsonComparison { + field: String, + op: CompareOp, + value: FilterValue, + }, And(Box, Box), Or(Box, Box), } @@ -130,7 +137,8 @@ fn build_and_expr(pair: Pair) -> Result { fn build_term(pair: Pair) -> Result { match pair.as_rule() { Rule::regex_expr => build_regex(pair), - Rule::comparison => build_comparison(pair), + Rule::metric_comparison => build_comparison(pair), + Rule::json_comparison => build_json_comparison(pair), Rule::or_expr => build_or_expr(pair), other => Err(FilterParseError::new(format!( "unexpected grammar element: {other:?}" @@ -187,6 +195,27 @@ fn build_comparison(pair: Pair) -> Result { }) } +fn build_json_comparison(pair: Pair) -> Result { + let mut inner = pair.into_inner(); + + let field_pair = inner + .next() + .ok_or_else(|| FilterParseError::new("malformed json[...] comparison: missing field name"))?; + let field = field_pair.as_str().to_string(); + + let op_pair = inner + .next() + .ok_or_else(|| FilterParseError::new("malformed json[...] comparison: missing operator"))?; + let op = parse_op(op_pair.as_str())?; + + let value_pair = inner + .next() + .ok_or_else(|| FilterParseError::new("malformed json[...] comparison: missing value"))?; + let value = parse_value(value_pair.as_str()); + + Ok(FilterExpr::JsonComparison { field, op, value }) +} + fn parse_op(s: &str) -> Result { match s { "<" => Ok(CompareOp::Lt), @@ -282,6 +311,51 @@ mod tests { ); } + #[test] + fn parses_json_field_comparison() { + let expr = parse("json[level]==error").expect("should parse"); + assert_eq!( + expr, + FilterExpr::JsonComparison { + field: "level".to_string(), + op: CompareOp::Eq, + value: FilterValue::Raw("error".to_string()), + } + ); + } + + #[test] + fn parses_json_field_numeric_comparison() { + let expr = parse("json[status]>=500").expect("should parse"); + assert_eq!( + expr, + FilterExpr::JsonComparison { + field: "status".to_string(), + op: CompareOp::Ge, + value: FilterValue::Number(500.0), + } + ); + } + + #[test] + fn json_field_composes_with_and_or() { + let expr = parse(r#"json[level]==error and sequence<5"#).expect("should parse"); + let expected = FilterExpr::And( + Box::new(FilterExpr::JsonComparison { + field: "level".to_string(), + op: CompareOp::Eq, + value: FilterValue::Raw("error".to_string()), + }), + Box::new(FilterExpr::Comparison { + metric: "sequence".to_string(), + param: None, + op: CompareOp::Lt, + value: FilterValue::Number(5.0), + }), + ); + assert_eq!(expr, expected); + } + #[test] fn parses_and_or_with_and_binding_tighter() { let expr = parse(r#"regex("error") and sequence<5 or timestamp>12:00"#) diff --git a/tickets/016-filter-by-json-field.md b/tickets/016-filter-by-json-field.md deleted file mode 100644 index fc9be16..0000000 --- a/tickets/016-filter-by-json-field.md +++ /dev/null @@ -1,11 +0,0 @@ -# Support filtering/querying by JSON field in the filter bar - -Follow-up to the JSON log ingestion design decision (repowise decision fdb47034ad954e3ea0b898fdc2750c32). - -Extend the filter query language with a JSON field accessor (e.g. `json[field] value`), comparable to the existing `metric[param] value` comparison, so JSON fields can be filtered without first declaring a metric/param for them. - -## Acceptance criteria -- `filter_parser.pest` gains a `json[field]` (or similar) accessor usable anywhere `metric[param]` is today, for comparisons. -- The engine evaluates `json[field]` against a line's parsed JSON (when it parses as an object), consistent with how declared params/metrics are evaluated today. -- Lines that aren't valid JSON simply don't match any `json[field]` comparison (no error). -- Existing filter syntax (metric/param comparisons, regex(), sequence, timestamp, and/or) is unaffected.