Add FilterBar query language parser (pest grammar)
Introduces a standalone src/filter_parser.rs module (no dependency on
engine/index types) that parses the FilterBar query language into a
FilterExpr AST using a pest grammar (src/filter_parser.pest). Supports
regex("...") searches, metric_name[param] <op> value comparisons,
reserved sequence/timestamp metrics, and and/or combinators with
correct precedence (and binds tighter than or) and parenthesized
sub-expressions. Evaluation against real log data is left to a later
integration ticket.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XFUNZwCvz3kcod3352ZR2o
This commit is contained in:
Hugh Kaznowski
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
1fb564d06b
commit
202b1dcf9f
@@ -0,0 +1,42 @@
|
||||
// Grammar for the FilterBar query language.
|
||||
//
|
||||
// Supports:
|
||||
// - regex("...") raw-text regex search over log lines
|
||||
// - metric_name[param] <op> value metric comparisons
|
||||
// - sequence <op> value reserved metric name (no [param])
|
||||
// - timestamp <op> value reserved metric name (no [param])
|
||||
// - `and` / `or` boolean combinators, with `and` binding tighter than `or`
|
||||
// - parenthesized sub-expressions
|
||||
|
||||
WHITESPACE = _{ " " | "\t" | "\n" | "\r" }
|
||||
|
||||
file = { SOI ~ or_expr ~ EOI }
|
||||
|
||||
or_expr = { and_expr ~ (or_kw ~ and_expr)* }
|
||||
and_expr = { term ~ (and_kw ~ term)* }
|
||||
|
||||
term = _{ regex_expr | comparison | "(" ~ or_expr ~ ")" }
|
||||
|
||||
// These must be atomic (`@`): a non-atomic rule would let pest's implicit
|
||||
// whitespace insertion slip in between the keyword and its own boundary
|
||||
// check below, which breaks the lookahead (it would see the first
|
||||
// non-whitespace character *after* skipping the separating space, instead
|
||||
// of the character immediately following the keyword).
|
||||
and_kw = @{ "and" ~ !(ASCII_ALPHANUMERIC | "_") }
|
||||
or_kw = @{ "or" ~ !(ASCII_ALPHANUMERIC | "_") }
|
||||
|
||||
regex_expr = { "regex" ~ "(" ~ string_literal ~ ")" }
|
||||
|
||||
string_literal = ${ "\"" ~ inner_str ~ "\"" }
|
||||
inner_str = @{ (escaped_char | !("\"" | "\\") ~ ANY)* }
|
||||
escaped_char = @{ "\\" ~ ANY }
|
||||
|
||||
comparison = { metric ~ ("[" ~ param ~ "]")? ~ op ~ value }
|
||||
|
||||
metric = @{ ident }
|
||||
param = @{ ident }
|
||||
ident = @{ (ASCII_ALPHANUMERIC | "_")+ }
|
||||
|
||||
op = { "<=" | ">=" | "==" | "!=" | "<" | ">" }
|
||||
|
||||
value = @{ (!(WHITESPACE | ")") ~ ANY)+ }
|
||||
@@ -0,0 +1,380 @@
|
||||
//! FilterBar query language: grammar, AST and parser.
|
||||
//!
|
||||
//! This module is intentionally standalone and has no dependency on the
|
||||
//! engine/index types (`data_value`, `index_key`, `engine`, ...). It only
|
||||
//! knows how to turn a filter string into an [`FilterExpr`] AST. Evaluating
|
||||
//! that AST against real log data, and resolving [`FilterValue::Raw`] tokens
|
||||
//! into concrete datetimes, is left to later integration tickets.
|
||||
//!
|
||||
//! Grammar (see `src/filter_parser.pest`):
|
||||
//!
|
||||
//! - `regex("...")` — raw-text regex search over log lines.
|
||||
//! - `metric_name[param] <op> value` — metric comparisons, with `<op>` one
|
||||
//! of `<`, `>`, `<=`, `>=`, `==`, `!=`.
|
||||
//! - `sequence <op> value` and `timestamp <op> value` — reserved metric
|
||||
//! names usable the same way, without a `[param]`.
|
||||
//! - `and` / `or` boolean combinators, with `and` binding tighter than `or`.
|
||||
//! - Parenthesized sub-expressions.
|
||||
//!
|
||||
//! Nothing in `main.rs` calls into this module yet — it is wired up by a
|
||||
//! later integration ticket — so the compiler's dead-code lint is silenced
|
||||
//! for the whole module rather than for each individual item.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use pest::Parser;
|
||||
use pest::iterators::Pair;
|
||||
use pest_derive::Parser as PestParser;
|
||||
|
||||
#[derive(PestParser)]
|
||||
#[grammar = "filter_parser.pest"]
|
||||
struct FilterGrammar;
|
||||
|
||||
/// The parsed AST for a FilterBar query.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum FilterExpr {
|
||||
/// `regex("...")` — raw-text regex search over log lines.
|
||||
Regex(String),
|
||||
/// `metric[param] <op> value` — a single metric comparison.
|
||||
Comparison {
|
||||
metric: String,
|
||||
param: Option<String>,
|
||||
op: CompareOp,
|
||||
value: FilterValue,
|
||||
},
|
||||
And(Box<FilterExpr>, Box<FilterExpr>),
|
||||
Or(Box<FilterExpr>, Box<FilterExpr>),
|
||||
}
|
||||
|
||||
/// Comparison operators supported by metric comparisons.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum CompareOp {
|
||||
Lt,
|
||||
Gt,
|
||||
Le,
|
||||
Ge,
|
||||
Eq,
|
||||
Ne,
|
||||
}
|
||||
|
||||
/// The right-hand side of a comparison. Values that parse as a number
|
||||
/// become [`FilterValue::Number`]; everything else (e.g. datetime-like
|
||||
/// tokens such as `12:00`) is kept as an opaque [`FilterValue::Raw`] string
|
||||
/// to be resolved by a later ticket.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum FilterValue {
|
||||
Number(f64),
|
||||
Raw(String),
|
||||
}
|
||||
|
||||
/// A located, human-readable parse error.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct FilterParseError {
|
||||
message: String,
|
||||
}
|
||||
|
||||
impl FilterParseError {
|
||||
fn new(message: impl Into<String>) -> Self {
|
||||
FilterParseError {
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for FilterParseError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}", self.message)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for FilterParseError {}
|
||||
|
||||
impl From<pest::error::Error<Rule>> for FilterParseError {
|
||||
fn from(err: pest::error::Error<Rule>) -> Self {
|
||||
// pest's own Display already renders a located, human-readable
|
||||
// message (line/column + a pointer into the offending input), so we
|
||||
// use it as the basis and add a short prefix for context.
|
||||
FilterParseError::new(format!("failed to parse filter expression: {err}"))
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a FilterBar query string into a [`FilterExpr`] AST.
|
||||
pub fn parse(input: &str) -> Result<FilterExpr, FilterParseError> {
|
||||
let mut pairs = FilterGrammar::parse(Rule::file, input)?;
|
||||
|
||||
let file = pairs
|
||||
.next()
|
||||
.ok_or_else(|| FilterParseError::new("empty filter expression"))?;
|
||||
|
||||
let or_expr = file
|
||||
.into_inner()
|
||||
.find(|p| p.as_rule() == Rule::or_expr)
|
||||
.ok_or_else(|| FilterParseError::new("malformed filter expression: missing body"))?;
|
||||
|
||||
build_or_expr(or_expr)
|
||||
}
|
||||
|
||||
fn build_or_expr(pair: Pair<Rule>) -> Result<FilterExpr, FilterParseError> {
|
||||
debug_assert_eq!(pair.as_rule(), Rule::or_expr);
|
||||
// `or_kw` is atomic (see the grammar file) so it is *not* silent: it
|
||||
// shows up as a sibling pair between the `and_expr` operands. Skip it
|
||||
// rather than assuming a fixed alternating structure.
|
||||
let mut and_exprs = pair
|
||||
.into_inner()
|
||||
.filter(|p| p.as_rule() != Rule::or_kw);
|
||||
let first = and_exprs
|
||||
.next()
|
||||
.ok_or_else(|| FilterParseError::new("malformed 'or' expression"))?;
|
||||
let mut expr = build_and_expr(first)?;
|
||||
for next in and_exprs {
|
||||
let rhs = build_and_expr(next)?;
|
||||
expr = FilterExpr::Or(Box::new(expr), Box::new(rhs));
|
||||
}
|
||||
Ok(expr)
|
||||
}
|
||||
|
||||
fn build_and_expr(pair: Pair<Rule>) -> Result<FilterExpr, FilterParseError> {
|
||||
debug_assert_eq!(pair.as_rule(), Rule::and_expr);
|
||||
// Same reasoning as above for `and_kw`.
|
||||
let mut terms = pair
|
||||
.into_inner()
|
||||
.filter(|p| p.as_rule() != Rule::and_kw);
|
||||
let first = terms
|
||||
.next()
|
||||
.ok_or_else(|| FilterParseError::new("malformed 'and' expression"))?;
|
||||
let mut expr = build_term(first)?;
|
||||
for next in terms {
|
||||
let rhs = build_term(next)?;
|
||||
expr = FilterExpr::And(Box::new(expr), Box::new(rhs));
|
||||
}
|
||||
Ok(expr)
|
||||
}
|
||||
|
||||
fn build_term(pair: Pair<Rule>) -> Result<FilterExpr, FilterParseError> {
|
||||
match pair.as_rule() {
|
||||
Rule::regex_expr => build_regex(pair),
|
||||
Rule::comparison => build_comparison(pair),
|
||||
Rule::or_expr => build_or_expr(pair),
|
||||
other => Err(FilterParseError::new(format!(
|
||||
"unexpected grammar element: {other:?}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
fn build_regex(pair: Pair<Rule>) -> Result<FilterExpr, FilterParseError> {
|
||||
let string_literal = pair
|
||||
.into_inner()
|
||||
.find(|p| p.as_rule() == Rule::string_literal)
|
||||
.ok_or_else(|| FilterParseError::new("malformed regex(...) expression"))?;
|
||||
let inner_str = string_literal
|
||||
.into_inner()
|
||||
.find(|p| p.as_rule() == Rule::inner_str)
|
||||
.ok_or_else(|| FilterParseError::new("malformed string literal in regex(...)"))?;
|
||||
Ok(FilterExpr::Regex(unescape(inner_str.as_str())))
|
||||
}
|
||||
|
||||
fn build_comparison(pair: Pair<Rule>) -> Result<FilterExpr, FilterParseError> {
|
||||
let mut inner = pair.into_inner();
|
||||
|
||||
let metric_pair = inner
|
||||
.next()
|
||||
.ok_or_else(|| FilterParseError::new("malformed comparison: missing metric name"))?;
|
||||
let metric = metric_pair.as_str().to_string();
|
||||
|
||||
let mut next = inner
|
||||
.next()
|
||||
.ok_or_else(|| FilterParseError::new("malformed comparison: missing operator"))?;
|
||||
|
||||
let param = if next.as_rule() == Rule::param {
|
||||
let p = next.as_str().to_string();
|
||||
next = inner
|
||||
.next()
|
||||
.ok_or_else(|| FilterParseError::new("malformed comparison: missing operator"))?;
|
||||
Some(p)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let op = parse_op(next.as_str())?;
|
||||
|
||||
let value_pair = inner
|
||||
.next()
|
||||
.ok_or_else(|| FilterParseError::new("malformed comparison: missing value"))?;
|
||||
let value = parse_value(value_pair.as_str());
|
||||
|
||||
Ok(FilterExpr::Comparison {
|
||||
metric,
|
||||
param,
|
||||
op,
|
||||
value,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_op(s: &str) -> Result<CompareOp, FilterParseError> {
|
||||
match s {
|
||||
"<" => Ok(CompareOp::Lt),
|
||||
">" => Ok(CompareOp::Gt),
|
||||
"<=" => Ok(CompareOp::Le),
|
||||
">=" => Ok(CompareOp::Ge),
|
||||
"==" => Ok(CompareOp::Eq),
|
||||
"!=" => Ok(CompareOp::Ne),
|
||||
other => Err(FilterParseError::new(format!(
|
||||
"unknown comparison operator: {other}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_value(s: &str) -> FilterValue {
|
||||
match s.parse::<f64>() {
|
||||
Ok(n) => FilterValue::Number(n),
|
||||
Err(_) => FilterValue::Raw(s.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Unescape a regex string literal's inner contents (`\"` -> `"`, `\\` -> `\`,
|
||||
/// any other `\x` -> `x`).
|
||||
fn unescape(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
let mut chars = s.chars();
|
||||
while let Some(c) = chars.next() {
|
||||
if c == '\\' {
|
||||
if let Some(next) = chars.next() {
|
||||
out.push(next);
|
||||
}
|
||||
} else {
|
||||
out.push(c);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_bare_regex() {
|
||||
let expr = parse(r#"regex("error")"#).expect("should parse");
|
||||
assert_eq!(expr, FilterExpr::Regex("error".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_regex_with_escaped_quotes() {
|
||||
let expr = parse(r#"regex("say \"hi\"")"#).expect("should parse");
|
||||
assert_eq!(expr, FilterExpr::Regex(r#"say "hi""#.to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_single_metric_comparison() {
|
||||
let expr = parse("metric_name[param]<5").expect("should parse");
|
||||
assert_eq!(
|
||||
expr,
|
||||
FilterExpr::Comparison {
|
||||
metric: "metric_name".to_string(),
|
||||
param: Some("param".to_string()),
|
||||
op: CompareOp::Lt,
|
||||
value: FilterValue::Number(5.0),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_reserved_metric_sequence() {
|
||||
let expr = parse("sequence<5").expect("should parse");
|
||||
assert_eq!(
|
||||
expr,
|
||||
FilterExpr::Comparison {
|
||||
metric: "sequence".to_string(),
|
||||
param: None,
|
||||
op: CompareOp::Lt,
|
||||
value: FilterValue::Number(5.0),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_reserved_metric_timestamp_as_raw_value() {
|
||||
let expr = parse("timestamp>12:00").expect("should parse");
|
||||
assert_eq!(
|
||||
expr,
|
||||
FilterExpr::Comparison {
|
||||
metric: "timestamp".to_string(),
|
||||
param: None,
|
||||
op: CompareOp::Gt,
|
||||
value: FilterValue::Raw("12:00".to_string()),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_and_or_with_and_binding_tighter() {
|
||||
let expr = parse(r#"regex("error") and sequence<5 or timestamp>12:00"#)
|
||||
.expect("should parse");
|
||||
|
||||
let expected = FilterExpr::Or(
|
||||
Box::new(FilterExpr::And(
|
||||
Box::new(FilterExpr::Regex("error".to_string())),
|
||||
Box::new(FilterExpr::Comparison {
|
||||
metric: "sequence".to_string(),
|
||||
param: None,
|
||||
op: CompareOp::Lt,
|
||||
value: FilterValue::Number(5.0),
|
||||
}),
|
||||
)),
|
||||
Box::new(FilterExpr::Comparison {
|
||||
metric: "timestamp".to_string(),
|
||||
param: None,
|
||||
op: CompareOp::Gt,
|
||||
value: FilterValue::Raw("12:00".to_string()),
|
||||
}),
|
||||
);
|
||||
|
||||
assert_eq!(expr, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parenthesized_expression_overrides_precedence() {
|
||||
let expr = parse(r#"regex("error") and (sequence<5 or timestamp>12:00)"#)
|
||||
.expect("should parse");
|
||||
|
||||
let expected = FilterExpr::And(
|
||||
Box::new(FilterExpr::Regex("error".to_string())),
|
||||
Box::new(FilterExpr::Or(
|
||||
Box::new(FilterExpr::Comparison {
|
||||
metric: "sequence".to_string(),
|
||||
param: None,
|
||||
op: CompareOp::Lt,
|
||||
value: FilterValue::Number(5.0),
|
||||
}),
|
||||
Box::new(FilterExpr::Comparison {
|
||||
metric: "timestamp".to_string(),
|
||||
param: None,
|
||||
op: CompareOp::Gt,
|
||||
value: FilterValue::Raw("12:00".to_string()),
|
||||
}),
|
||||
)),
|
||||
);
|
||||
|
||||
assert_eq!(expr, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_input_produces_located_error_not_panic() {
|
||||
let result = parse("regex(\"unterminated and sequence<");
|
||||
assert!(result.is_err());
|
||||
let err = result.unwrap_err();
|
||||
// Ensure the error message is human-readable and carries location
|
||||
// info (pest errors always mention line/column context).
|
||||
let msg = err.to_string();
|
||||
assert!(!msg.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_input_is_an_error_not_a_panic() {
|
||||
assert!(parse("").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn garbage_input_is_an_error_not_a_panic() {
|
||||
assert!(parse("@@@ not a valid filter @@@").is_err());
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ mod api;
|
||||
mod data_value;
|
||||
mod dto;
|
||||
mod engine;
|
||||
mod filter_parser;
|
||||
mod index_key;
|
||||
mod ingest;
|
||||
mod metric_primitive;
|
||||
|
||||
Reference in New Issue
Block a user