From 202b1dcf9ffd8ca1fa7f26f95a324588d45bcbf8 Mon Sep 17 00:00:00 2001 From: Hugh Kaznowski Date: Sat, 5 Sep 2026 12:45:31 +0100 Subject: [PATCH] Add FilterBar query language parser (pest grammar) Introduces a standalone src/filter_parser.rs module (no dependency on engine/index types) that parses the FilterBar query language into a FilterExpr AST using a pest grammar (src/filter_parser.pest). Supports regex("...") searches, metric_name[param] value comparisons, reserved sequence/timestamp metrics, and and/or combinators with correct precedence (and binds tighter than or) and parenthesized sub-expressions. Evaluation against real log data is left to a later integration ticket. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01XFUNZwCvz3kcod3352ZR2o --- Cargo.lock | 67 +++++++- Cargo.toml | 2 + src/filter_parser.pest | 42 +++++ src/filter_parser.rs | 380 +++++++++++++++++++++++++++++++++++++++++ src/main.rs | 1 + 5 files changed, 489 insertions(+), 3 deletions(-) create mode 100644 src/filter_parser.pest create mode 100644 src/filter_parser.rs diff --git a/Cargo.lock b/Cargo.lock index 7fb22c4..9dca73d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -159,7 +159,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 3.0.5", ] [[package]] @@ -414,6 +414,8 @@ version = "0.1.0" dependencies = [ "axum", "clap", + "pest", + "pest_derive", "serde", "serde_json", "tokio", @@ -449,6 +451,48 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pest" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a07a60cc7a4d00c91f95c685609d1d2f79050e6804b70ebedd7650f0b839bcf" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3a83744a5c8455b8b3e0dc5031362780a347c878bdd11584d1a8984228cc88d" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0cd3451aa3de60d4b9a1e736885e4dea6b31617598026f12256ad566d63304a" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pest_meta" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04d3a0849e241d7dfce834c83b1c5edc8622009e8dd51a12ba1927c32f05496" +dependencies = [ + "pest", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -521,7 +565,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.5", ] [[package]] @@ -598,6 +642,17 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "3.0.5" @@ -640,7 +695,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.5", ] [[package]] @@ -729,6 +784,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + [[package]] name = "unicase" version = "2.9.0" diff --git a/Cargo.toml b/Cargo.toml index 24df485..86c5bce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,3 +10,5 @@ tower-http = { version = "0.6", features = ["fs", "cors"] } serde = { version = "1", features = ["derive"] } serde_json = "1" clap = { version = "4", features = ["derive"] } +pest = "2" +pest_derive = "2" diff --git a/src/filter_parser.pest b/src/filter_parser.pest new file mode 100644 index 0000000..1c7f59a --- /dev/null +++ b/src/filter_parser.pest @@ -0,0 +1,42 @@ +// Grammar for the FilterBar query language. +// +// Supports: +// - regex("...") raw-text regex search over log lines +// - metric_name[param] value metric comparisons +// - sequence value reserved metric name (no [param]) +// - timestamp value reserved metric name (no [param]) +// - `and` / `or` boolean combinators, with `and` binding tighter than `or` +// - parenthesized sub-expressions + +WHITESPACE = _{ " " | "\t" | "\n" | "\r" } + +file = { SOI ~ or_expr ~ EOI } + +or_expr = { and_expr ~ (or_kw ~ and_expr)* } +and_expr = { term ~ (and_kw ~ term)* } + +term = _{ regex_expr | comparison | "(" ~ or_expr ~ ")" } + +// These must be atomic (`@`): a non-atomic rule would let pest's implicit +// whitespace insertion slip in between the keyword and its own boundary +// check below, which breaks the lookahead (it would see the first +// non-whitespace character *after* skipping the separating space, instead +// of the character immediately following the keyword). +and_kw = @{ "and" ~ !(ASCII_ALPHANUMERIC | "_") } +or_kw = @{ "or" ~ !(ASCII_ALPHANUMERIC | "_") } + +regex_expr = { "regex" ~ "(" ~ string_literal ~ ")" } + +string_literal = ${ "\"" ~ inner_str ~ "\"" } +inner_str = @{ (escaped_char | !("\"" | "\\") ~ ANY)* } +escaped_char = @{ "\\" ~ ANY } + +comparison = { metric ~ ("[" ~ param ~ "]")? ~ op ~ value } + +metric = @{ ident } +param = @{ ident } +ident = @{ (ASCII_ALPHANUMERIC | "_")+ } + +op = { "<=" | ">=" | "==" | "!=" | "<" | ">" } + +value = @{ (!(WHITESPACE | ")") ~ ANY)+ } diff --git a/src/filter_parser.rs b/src/filter_parser.rs new file mode 100644 index 0000000..b37c800 --- /dev/null +++ b/src/filter_parser.rs @@ -0,0 +1,380 @@ +//! FilterBar query language: grammar, AST and parser. +//! +//! This module is intentionally standalone and has no dependency on the +//! engine/index types (`data_value`, `index_key`, `engine`, ...). It only +//! knows how to turn a filter string into an [`FilterExpr`] AST. Evaluating +//! that AST against real log data, and resolving [`FilterValue::Raw`] tokens +//! into concrete datetimes, is left to later integration tickets. +//! +//! Grammar (see `src/filter_parser.pest`): +//! +//! - `regex("...")` — raw-text regex search over log lines. +//! - `metric_name[param] value` — metric comparisons, with `` one +//! of `<`, `>`, `<=`, `>=`, `==`, `!=`. +//! - `sequence value` and `timestamp value` — reserved metric +//! names usable the same way, without a `[param]`. +//! - `and` / `or` boolean combinators, with `and` binding tighter than `or`. +//! - Parenthesized sub-expressions. +//! +//! Nothing in `main.rs` calls into this module yet — it is wired up by a +//! later integration ticket — so the compiler's dead-code lint is silenced +//! for the whole module rather than for each individual item. +#![allow(dead_code)] + +use pest::Parser; +use pest::iterators::Pair; +use pest_derive::Parser as PestParser; + +#[derive(PestParser)] +#[grammar = "filter_parser.pest"] +struct FilterGrammar; + +/// The parsed AST for a FilterBar query. +#[derive(Debug, Clone, PartialEq)] +pub enum FilterExpr { + /// `regex("...")` — raw-text regex search over log lines. + Regex(String), + /// `metric[param] value` — a single metric comparison. + Comparison { + metric: String, + param: Option, + op: CompareOp, + value: FilterValue, + }, + And(Box, Box), + Or(Box, Box), +} + +/// Comparison operators supported by metric comparisons. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CompareOp { + Lt, + Gt, + Le, + Ge, + Eq, + Ne, +} + +/// The right-hand side of a comparison. Values that parse as a number +/// become [`FilterValue::Number`]; everything else (e.g. datetime-like +/// tokens such as `12:00`) is kept as an opaque [`FilterValue::Raw`] string +/// to be resolved by a later ticket. +#[derive(Debug, Clone, PartialEq)] +pub enum FilterValue { + Number(f64), + Raw(String), +} + +/// A located, human-readable parse error. +#[derive(Debug, Clone, PartialEq)] +pub struct FilterParseError { + message: String, +} + +impl FilterParseError { + fn new(message: impl Into) -> Self { + FilterParseError { + message: message.into(), + } + } +} + +impl std::fmt::Display for FilterParseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.message) + } +} + +impl std::error::Error for FilterParseError {} + +impl From> for FilterParseError { + fn from(err: pest::error::Error) -> Self { + // pest's own Display already renders a located, human-readable + // message (line/column + a pointer into the offending input), so we + // use it as the basis and add a short prefix for context. + FilterParseError::new(format!("failed to parse filter expression: {err}")) + } +} + +/// Parse a FilterBar query string into a [`FilterExpr`] AST. +pub fn parse(input: &str) -> Result { + let mut pairs = FilterGrammar::parse(Rule::file, input)?; + + let file = pairs + .next() + .ok_or_else(|| FilterParseError::new("empty filter expression"))?; + + let or_expr = file + .into_inner() + .find(|p| p.as_rule() == Rule::or_expr) + .ok_or_else(|| FilterParseError::new("malformed filter expression: missing body"))?; + + build_or_expr(or_expr) +} + +fn build_or_expr(pair: Pair) -> Result { + debug_assert_eq!(pair.as_rule(), Rule::or_expr); + // `or_kw` is atomic (see the grammar file) so it is *not* silent: it + // shows up as a sibling pair between the `and_expr` operands. Skip it + // rather than assuming a fixed alternating structure. + let mut and_exprs = pair + .into_inner() + .filter(|p| p.as_rule() != Rule::or_kw); + let first = and_exprs + .next() + .ok_or_else(|| FilterParseError::new("malformed 'or' expression"))?; + let mut expr = build_and_expr(first)?; + for next in and_exprs { + let rhs = build_and_expr(next)?; + expr = FilterExpr::Or(Box::new(expr), Box::new(rhs)); + } + Ok(expr) +} + +fn build_and_expr(pair: Pair) -> Result { + debug_assert_eq!(pair.as_rule(), Rule::and_expr); + // Same reasoning as above for `and_kw`. + let mut terms = pair + .into_inner() + .filter(|p| p.as_rule() != Rule::and_kw); + let first = terms + .next() + .ok_or_else(|| FilterParseError::new("malformed 'and' expression"))?; + let mut expr = build_term(first)?; + for next in terms { + let rhs = build_term(next)?; + expr = FilterExpr::And(Box::new(expr), Box::new(rhs)); + } + Ok(expr) +} + +fn build_term(pair: Pair) -> Result { + match pair.as_rule() { + Rule::regex_expr => build_regex(pair), + Rule::comparison => build_comparison(pair), + Rule::or_expr => build_or_expr(pair), + other => Err(FilterParseError::new(format!( + "unexpected grammar element: {other:?}" + ))), + } +} + +fn build_regex(pair: Pair) -> Result { + let string_literal = pair + .into_inner() + .find(|p| p.as_rule() == Rule::string_literal) + .ok_or_else(|| FilterParseError::new("malformed regex(...) expression"))?; + let inner_str = string_literal + .into_inner() + .find(|p| p.as_rule() == Rule::inner_str) + .ok_or_else(|| FilterParseError::new("malformed string literal in regex(...)"))?; + Ok(FilterExpr::Regex(unescape(inner_str.as_str()))) +} + +fn build_comparison(pair: Pair) -> Result { + let mut inner = pair.into_inner(); + + let metric_pair = inner + .next() + .ok_or_else(|| FilterParseError::new("malformed comparison: missing metric name"))?; + let metric = metric_pair.as_str().to_string(); + + let mut next = inner + .next() + .ok_or_else(|| FilterParseError::new("malformed comparison: missing operator"))?; + + let param = if next.as_rule() == Rule::param { + let p = next.as_str().to_string(); + next = inner + .next() + .ok_or_else(|| FilterParseError::new("malformed comparison: missing operator"))?; + Some(p) + } else { + None + }; + + let op = parse_op(next.as_str())?; + + let value_pair = inner + .next() + .ok_or_else(|| FilterParseError::new("malformed comparison: missing value"))?; + let value = parse_value(value_pair.as_str()); + + Ok(FilterExpr::Comparison { + metric, + param, + op, + value, + }) +} + +fn parse_op(s: &str) -> Result { + match s { + "<" => Ok(CompareOp::Lt), + ">" => Ok(CompareOp::Gt), + "<=" => Ok(CompareOp::Le), + ">=" => Ok(CompareOp::Ge), + "==" => Ok(CompareOp::Eq), + "!=" => Ok(CompareOp::Ne), + other => Err(FilterParseError::new(format!( + "unknown comparison operator: {other}" + ))), + } +} + +fn parse_value(s: &str) -> FilterValue { + match s.parse::() { + Ok(n) => FilterValue::Number(n), + Err(_) => FilterValue::Raw(s.to_string()), + } +} + +/// Unescape a regex string literal's inner contents (`\"` -> `"`, `\\` -> `\`, +/// any other `\x` -> `x`). +fn unescape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars(); + while let Some(c) = chars.next() { + if c == '\\' { + if let Some(next) = chars.next() { + out.push(next); + } + } else { + out.push(c); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_bare_regex() { + let expr = parse(r#"regex("error")"#).expect("should parse"); + assert_eq!(expr, FilterExpr::Regex("error".to_string())); + } + + #[test] + fn parses_regex_with_escaped_quotes() { + let expr = parse(r#"regex("say \"hi\"")"#).expect("should parse"); + assert_eq!(expr, FilterExpr::Regex(r#"say "hi""#.to_string())); + } + + #[test] + fn parses_single_metric_comparison() { + let expr = parse("metric_name[param]<5").expect("should parse"); + assert_eq!( + expr, + FilterExpr::Comparison { + metric: "metric_name".to_string(), + param: Some("param".to_string()), + op: CompareOp::Lt, + value: FilterValue::Number(5.0), + } + ); + } + + #[test] + fn parses_reserved_metric_sequence() { + let expr = parse("sequence<5").expect("should parse"); + assert_eq!( + expr, + FilterExpr::Comparison { + metric: "sequence".to_string(), + param: None, + op: CompareOp::Lt, + value: FilterValue::Number(5.0), + } + ); + } + + #[test] + fn parses_reserved_metric_timestamp_as_raw_value() { + let expr = parse("timestamp>12:00").expect("should parse"); + assert_eq!( + expr, + FilterExpr::Comparison { + metric: "timestamp".to_string(), + param: None, + op: CompareOp::Gt, + value: FilterValue::Raw("12:00".to_string()), + } + ); + } + + #[test] + fn parses_and_or_with_and_binding_tighter() { + let expr = parse(r#"regex("error") and sequence<5 or timestamp>12:00"#) + .expect("should parse"); + + let expected = FilterExpr::Or( + Box::new(FilterExpr::And( + Box::new(FilterExpr::Regex("error".to_string())), + Box::new(FilterExpr::Comparison { + metric: "sequence".to_string(), + param: None, + op: CompareOp::Lt, + value: FilterValue::Number(5.0), + }), + )), + Box::new(FilterExpr::Comparison { + metric: "timestamp".to_string(), + param: None, + op: CompareOp::Gt, + value: FilterValue::Raw("12:00".to_string()), + }), + ); + + assert_eq!(expr, expected); + } + + #[test] + fn parenthesized_expression_overrides_precedence() { + let expr = parse(r#"regex("error") and (sequence<5 or timestamp>12:00)"#) + .expect("should parse"); + + let expected = FilterExpr::And( + Box::new(FilterExpr::Regex("error".to_string())), + Box::new(FilterExpr::Or( + Box::new(FilterExpr::Comparison { + metric: "sequence".to_string(), + param: None, + op: CompareOp::Lt, + value: FilterValue::Number(5.0), + }), + Box::new(FilterExpr::Comparison { + metric: "timestamp".to_string(), + param: None, + op: CompareOp::Gt, + value: FilterValue::Raw("12:00".to_string()), + }), + )), + ); + + assert_eq!(expr, expected); + } + + #[test] + fn malformed_input_produces_located_error_not_panic() { + let result = parse("regex(\"unterminated and sequence<"); + assert!(result.is_err()); + let err = result.unwrap_err(); + // Ensure the error message is human-readable and carries location + // info (pest errors always mention line/column context). + let msg = err.to_string(); + assert!(!msg.is_empty()); + } + + #[test] + fn empty_input_is_an_error_not_a_panic() { + assert!(parse("").is_err()); + } + + #[test] + fn garbage_input_is_an_error_not_a_panic() { + assert!(parse("@@@ not a valid filter @@@").is_err()); + } +} diff --git a/src/main.rs b/src/main.rs index 4b50470..d0f7bce 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,6 +2,7 @@ mod api; mod data_value; mod dto; mod engine; +mod filter_parser; mod index_key; mod ingest; mod metric_primitive;