phughk/ottyl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge ticket 044: ottyl_fmt reserved param + ottyl_seq/ottyl_ts rename

Browse Source
This commit is contained in:
Hugh Kaznowski
2026-09-06 01:50:39 +01:00
parent c84276937a 534dc52e05
commit a7147f47a0
7 changed files with 179 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/alerts.rs
+32 -5
View File
@@ -1,6 +1,6 @@
//! Alert rules: definitions, trigger evaluation, and ntfy delivery.
//!
//! Alert queries are restricted to regex/sequence/timestamp/json[field] — a metric[param] comparison would require a full bucket rescan per line, breaking the proportional-cost guarantee. json[field] is fine here since, like regex/sequence/timestamp, it only inspects the line's own text/fields, no index lookup.
//! Alert queries are restricted to regex/ottyl_seq/ottyl_ts/ottyl_fmt/json[field] — a metric[param] comparison would require a full bucket rescan per line, breaking the proportional-cost guarantee. json[field] is fine here since, like regex/ottyl_seq/ottyl_ts/ottyl_fmt, it only inspects the line's own text/fields, no index lookup.
//!
//! `Threshold`/`Ratio` windows are maintained via `crate::rate::RateWindow` — the same windowed-rate
//! implementation PromQL's `rate()` uses (`crate::engine::evaluate_promql`) — rather than a
@@ -196,16 +196,16 @@ fn eval_alert_expr(expr: &FilterExpr, sequence: u64, timestamp: Timestamp, text:
let re = Regex::new(pattern).map_err(|err| format!("invalid regex in alert query: {err}"))?;
Ok(re.is_match(text))
}
FilterExpr::Comparison { metric, param: None, op, value } if metric == "sequence" => {
FilterExpr::Comparison { metric, param: None, op, value } if metric == "ottyl_seq" => {
let target = match value {
FilterValue::Number(n) => *n,
FilterValue::Raw(raw) => raw
.parse::<f64>()
.map_err(|_| format!("'sequence' comparisons require a numeric value, got {raw:?}"))?,
.map_err(|_| format!("'ottyl_seq' comparisons require a numeric value, got {raw:?}"))?,
};
Ok(apply_compare_op((sequence as f64).total_cmp(&target), *op))
}
FilterExpr::Comparison { metric, param: None, op, value } if metric == "timestamp" => {
FilterExpr::Comparison { metric, param: None, op, value } if metric == "ottyl_ts" => {
let target_ms = match value {
FilterValue::Number(n) => *n as i64,
FilterValue::Raw(raw) => filter_datetime::parse_datetime(raw, Timestamp::now())
@@ -214,8 +214,15 @@ fn eval_alert_expr(expr: &FilterExpr, sequence: u64, timestamp: Timestamp, text:
};
Ok(apply_compare_op(timestamp.as_millis().cmp(&target_ms), *op))
}
FilterExpr::Comparison { metric, param: None, op, value } if metric == "ottyl_fmt" => {
let target = match value {
FilterValue::Number(n) => n.to_string(),
FilterValue::Raw(raw) => raw.clone(),
};
Ok(apply_compare_op(crate::logfmt::detect_format(text).cmp(target.as_str()), *op))
}
FilterExpr::Comparison { metric, .. } => Err(format!(
"alert queries only support regex()/sequence/timestamp (not metric comparisons like \
"alert queries only support regex()/ottyl_seq/ottyl_ts/ottyl_fmt (not metric comparisons like \
'{metric}[param]'): metric-based alert conditions aren't supported yet"
)),
FilterExpr::JsonComparison { field, op, value } => {
@@ -574,6 +581,26 @@ mod tests {
assert!(fired.is_empty());
}
#[test]
fn ottyl_fmt_alert_query_matches_detected_format() {
let rule = immediate_rule("ottyl_fmt==json");
let mut states = HashMap::new();
let index: BTreeMap<IndexKey, DataValue> = BTreeMap::new();
let fired = evaluate_alerts(
&[rule.clone()],
&mut states,
&index,
0,
Timestamp::from_millis(0),
r#"{"level":"error"}"#,
);
assert_eq!(fired.len(), 1);
let fired = evaluate_alerts(&[rule], &mut states, &index, 1, Timestamp::from_millis(1), "plain text");
assert!(fired.is_empty());
}
#[test]
fn metric_comparison_in_alert_query_is_rejected_not_panicking() {
let rule = immediate_rule("some_metric[param]==5");
src/engine.rs
+64 -11
View File
@@ -738,6 +738,9 @@ enum CompiledFilter {
Regex(Regex),
Sequence { op: CompareOp, target: f64 },
Timestamp { op: CompareOp, target_ms: i64 },
/// `ottyl_fmt <op> value` — like `Json`, this is line-local and re-detected fresh per line in
/// `eval_compiled` (`crate::logfmt::detect_format`) rather than pre-resolved against the index.
Fmt { op: CompareOp, target: String },
MetricMatch(HashSet<u64>),
/// `json[field] <op> value` — unlike `MetricMatch`, this can't be pre-resolved against the
/// index (the JSON lives in the log line itself, not a declared metric/param), so it's
@@ -767,8 +770,9 @@ fn compile_filter(
.map(CompiledFilter::Regex)
.map_err(|err| format!("invalid regex in filter: {err}")),
FilterExpr::Comparison { metric, param, op, value } => match metric.as_str() {
"sequence" => compile_sequence_comparison(*op, value),
"timestamp" => compile_timestamp_comparison(*op, value, now),
"ottyl_seq" => compile_ottyl_seq_comparison(*op, value),
"ottyl_ts" => compile_ottyl_ts_comparison(*op, value, now),
"ottyl_fmt" => Ok(compile_ottyl_fmt_comparison(*op, value)),
other => compile_metric_comparison(index, other, param.as_deref(), *op, value, now),
},
FilterExpr::JsonComparison { field, op, value } => Ok(CompiledFilter::Json {
@@ -787,19 +791,19 @@ fn compile_filter(
}
}
/// A `Raw` value is accepted defensively (parsed as a number) even though the grammar shouldn't produce one for `sequence`.
fn compile_sequence_comparison(op: CompareOp, value: &FilterValue) -> Result<CompiledFilter, String> {
/// A `Raw` value is accepted defensively (parsed as a number) even though the grammar shouldn't produce one for `ottyl_seq`.
fn compile_ottyl_seq_comparison(op: CompareOp, value: &FilterValue) -> Result<CompiledFilter, String> {
let target = match value {
FilterValue::Number(n) => *n,
FilterValue::Raw(raw) => raw.parse::<f64>().map_err(|_| {
format!("'sequence' comparisons require a numeric value, got {raw:?}")
format!("'ottyl_seq' comparisons require a numeric value, got {raw:?}")
})?,
};
Ok(CompiledFilter::Sequence { op, target })
}
/// A `Number` is treated as raw milliseconds; a `Raw` token is resolved via `filter_datetime::parse_datetime` against `now`.
fn compile_timestamp_comparison(
fn compile_ottyl_ts_comparison(
op: CompareOp,
value: &FilterValue,
now: Timestamp,
@@ -813,7 +817,19 @@ fn compile_timestamp_comparison(
Ok(CompiledFilter::Timestamp { op, target_ms })
}
/// `metric[param] <op> value` for any metric name other than `sequence`/`timestamp` — see [`CompiledFilter`]'s doc comment.
/// A `Number` target is stringified since `ottyl_fmt`'s value ("json"/"logfmt"/"none",
/// `crate::logfmt::detect_format`) is always text; any op is accepted since the strings compare
/// lexicographically fine for `==`/`!=` (the only ops that make real sense here), same as
/// `compare_metric_primitive`'s `String` arm.
fn compile_ottyl_fmt_comparison(op: CompareOp, value: &FilterValue) -> CompiledFilter {
let target = match value {
FilterValue::Number(n) => n.to_string(),
FilterValue::Raw(raw) => raw.clone(),
};
CompiledFilter::Fmt { op, target }
}
/// `metric[param] <op> value` for any metric name other than `ottyl_seq`/`ottyl_ts`/`ottyl_fmt` — see [`CompiledFilter`]'s doc comment.
fn compile_metric_comparison(
index: &BTreeMap<IndexKey, DataValue>,
metric: &str,
@@ -985,6 +1001,9 @@ fn eval_compiled(compiled: &CompiledFilter, sequence: u64, timestamp: Timestamp,
CompiledFilter::Timestamp { op, target_ms } => {
apply_compare_op(timestamp.as_millis().cmp(target_ms), *op)
}
CompiledFilter::Fmt { op, target } => {
apply_compare_op(crate::logfmt::detect_format(text).cmp(target.as_str()), *op)
}
CompiledFilter::MetricMatch(sequences) => sequences.contains(&sequence),
CompiledFilter::Json { field, op, value } => eval_json_comparison(text, field, *op, value),
CompiledFilter::And(lhs, rhs) => {
@@ -2983,13 +3002,13 @@ mod tests {
}
#[test]
fn logs_query_with_sequence_comparison_filters_by_sequence() {
fn logs_query_with_ottyl_seq_comparison_filters_by_sequence() {
let mut index = BTreeMap::new();
for i in 0..5u64 {
ingest_log(&mut index, i, &format!("line {i}"));
}
match handle_query(&index, logs_query(Some("sequence<2")), 0) {
match handle_query(&index, logs_query(Some("ottyl_seq<2")), 0) {
EngineResponse::Logs(entries) => {
let sequences: Vec<u64> = entries.iter().map(|e| e.sequence).collect();
assert_eq!(sequences, vec![0, 1]);
@@ -2999,7 +3018,7 @@ mod tests {
}
#[test]
fn logs_query_with_timestamp_comparison_filters_by_timestamp() {
fn logs_query_with_ottyl_ts_comparison_filters_by_timestamp() {
let mut index = BTreeMap::new();
for i in 0..5u64 {
let timestamp = Timestamp::from_millis(i as i64 * 1000);
@@ -3009,7 +3028,7 @@ mod tests {
);
}
match handle_query(&index, logs_query(Some("timestamp>2000")), 0) {
match handle_query(&index, logs_query(Some("ottyl_ts>2000")), 0) {
EngineResponse::Logs(entries) => {
let sequences: Vec<u64> = entries.iter().map(|e| e.sequence).collect();
assert_eq!(sequences, vec![3, 4]);
@@ -3018,6 +3037,40 @@ mod tests {
}
}
#[test]
fn logs_query_with_ottyl_fmt_comparison_filters_by_detected_format_for_mixed_format_lines() {
let mut index = BTreeMap::new();
// Seeded fixture mixing all three detectable formats on purpose (ticket 044).
ingest_log(&mut index, 0, r#"{"level":"error","code":500}"#);
ingest_log(&mut index, 1, r#"ts=1788648105773 msg="entity_moved" x1=-4"#);
ingest_log(&mut index, 2, "just some plain prose here");
ingest_log(&mut index, 3, r#"{"level":"info","code":200}"#);
match handle_query(&index, logs_query(Some("ottyl_fmt==json")), 0) {
EngineResponse::Logs(entries) => {
let sequences: Vec<u64> = entries.iter().map(|e| e.sequence).collect();
assert_eq!(sequences, vec![0, 3]);
}
_ => panic!("expected Logs response"),
}
match handle_query(&index, logs_query(Some("ottyl_fmt==logfmt")), 0) {
EngineResponse::Logs(entries) => {
let sequences: Vec<u64> = entries.iter().map(|e| e.sequence).collect();
assert_eq!(sequences, vec![1]);
}
_ => panic!("expected Logs response"),
}
match handle_query(&index, logs_query(Some("ottyl_fmt==none")), 0) {
EngineResponse::Logs(entries) => {
let sequences: Vec<u64> = entries.iter().map(|e| e.sequence).collect();
assert_eq!(sequences, vec![2]);
}
_ => panic!("expected Logs response"),
}
}
#[test]
fn logs_query_with_json_field_comparison_filters_by_json_field() {
let mut index = BTreeMap::new();
src/filter_parser.pest
+3 -2
View File
@@ -4,8 +4,9 @@
// - regex("...") raw-text regex search over log lines
// - metric_name[param] <op> value metric comparisons
// - json[field] <op> value JSON field comparisons (line parsed as JSON)
// - sequence <op> value reserved metric name (no [param])
// - timestamp <op> value reserved metric name (no [param])
// - ottyl_seq <op> value reserved metric name (no [param])
// - ottyl_ts <op> value reserved metric name (no [param])
// - ottyl_fmt <op> value reserved metric name (no [param])
// - `and` / `or` boolean combinators, with `and` binding tighter than `or`
// - parenthesized sub-expressions
src/filter_parser.rs
+28 -14
View File
@@ -284,12 +284,12 @@ mod tests {
}
#[test]
fn parses_reserved_metric_sequence() {
let expr = parse("sequence<5").expect("should parse");
fn parses_reserved_metric_ottyl_seq() {
let expr = parse("ottyl_seq<5").expect("should parse");
assert_eq!(
expr,
FilterExpr::Comparison {
metric: "sequence".to_string(),
metric: "ottyl_seq".to_string(),
param: None,
op: CompareOp::Lt,
value: FilterValue::Number(5.0),
@@ -298,12 +298,12 @@ mod tests {
}
#[test]
fn parses_reserved_metric_timestamp_as_raw_value() {
let expr = parse("timestamp>12:00").expect("should parse");
fn parses_reserved_metric_ottyl_ts_as_raw_value() {
let expr = parse("ottyl_ts>12:00").expect("should parse");
assert_eq!(
expr,
FilterExpr::Comparison {
metric: "timestamp".to_string(),
metric: "ottyl_ts".to_string(),
param: None,
op: CompareOp::Gt,
value: FilterValue::Raw("12:00".to_string()),
@@ -311,6 +311,20 @@ mod tests {
);
}
#[test]
fn parses_reserved_metric_ottyl_fmt() {
let expr = parse("ottyl_fmt==json").expect("should parse");
assert_eq!(
expr,
FilterExpr::Comparison {
metric: "ottyl_fmt".to_string(),
param: None,
op: CompareOp::Eq,
value: FilterValue::Raw("json".to_string()),
}
);
}
#[test]
fn parses_json_field_comparison() {
let expr = parse("json[level]==error").expect("should parse");
@@ -339,7 +353,7 @@ mod tests {
#[test]
fn json_field_composes_with_and_or() {
let expr = parse(r#"json[level]==error and sequence<5"#).expect("should parse");
let expr = parse(r#"json[level]==error and ottyl_seq<5"#).expect("should parse");
let expected = FilterExpr::And(
Box::new(FilterExpr::JsonComparison {
field: "level".to_string(),
@@ -347,7 +361,7 @@ mod tests {
value: FilterValue::Raw("error".to_string()),
}),
Box::new(FilterExpr::Comparison {
metric: "sequence".to_string(),
metric: "ottyl_seq".to_string(),
param: None,
op: CompareOp::Lt,
value: FilterValue::Number(5.0),
@@ -358,21 +372,21 @@ mod tests {
#[test]
fn parses_and_or_with_and_binding_tighter() {
let expr = parse(r#"regex("error") and sequence<5 or timestamp>12:00"#)
let expr = parse(r#"regex("error") and ottyl_seq<5 or ottyl_ts>12:00"#)
.expect("should parse");
let expected = FilterExpr::Or(
Box::new(FilterExpr::And(
Box::new(FilterExpr::Regex("error".to_string())),
Box::new(FilterExpr::Comparison {
metric: "sequence".to_string(),
metric: "ottyl_seq".to_string(),
param: None,
op: CompareOp::Lt,
value: FilterValue::Number(5.0),
}),
)),
Box::new(FilterExpr::Comparison {
metric: "timestamp".to_string(),
metric: "ottyl_ts".to_string(),
param: None,
op: CompareOp::Gt,
value: FilterValue::Raw("12:00".to_string()),
@@ -384,20 +398,20 @@ mod tests {
#[test]
fn parenthesized_expression_overrides_precedence() {
let expr = parse(r#"regex("error") and (sequence<5 or timestamp>12:00)"#)
let expr = parse(r#"regex("error") and (ottyl_seq<5 or ottyl_ts>12:00)"#)
.expect("should parse");
let expected = FilterExpr::And(
Box::new(FilterExpr::Regex("error".to_string())),
Box::new(FilterExpr::Or(
Box::new(FilterExpr::Comparison {
metric: "sequence".to_string(),
metric: "ottyl_seq".to_string(),
param: None,
op: CompareOp::Lt,
value: FilterValue::Number(5.0),
}),
Box::new(FilterExpr::Comparison {
metric: "timestamp".to_string(),
metric: "ottyl_ts".to_string(),
param: None,
op: CompareOp::Gt,
value: FilterValue::Raw("12:00".to_string()),
src/logfmt.rs
+38
View File
@@ -61,6 +61,22 @@ pub fn extract_pairs(line: &str) -> Vec<(&str, &str, bool)> {
results
}
/// The value of the `ottyl_fmt` reserved param (`crate::engine::compile_filter`/`eval_alert_expr`):
/// which format a line was recognized as when parsed. JSON is checked first since a JSON object can
/// incidentally contain `key=value`-shaped substrings that `extract_pairs` would otherwise pick up;
/// only a top-level object counts (arrays/scalars aren't what `crate::engine::eval_json_comparison`
/// treats as JSON either). Anything else falls back to logfmt-style pair extraction, and a line with
/// neither is `"none"`.
pub fn detect_format(text: &str) -> &'static str {
if serde_json::from_str::<serde_json::Value>(text).is_ok_and(|value| value.is_object()) {
"json"
} else if !extract_pairs(text).is_empty() {
"logfmt"
} else {
"none"
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -108,4 +124,26 @@ mod tests {
let pairs = extract_pairs(r#"msg="he said \"hi\"""#);
assert_eq!(pairs, vec![("msg", r#"he said \"hi\""#, true)]);
}
#[test]
fn detects_json_object_lines_as_json() {
assert_eq!(detect_format(r#"{"level":"error","code":500}"#), "json");
}
#[test]
fn detects_key_value_lines_as_logfmt() {
assert_eq!(detect_format(r#"ts=1788648105773 msg="entity_moved" x1=-4"#), "logfmt");
}
#[test]
fn detects_plain_prose_as_none() {
assert_eq!(detect_format("just some plain prose here"), "none");
}
#[test]
fn a_bare_json_array_or_scalar_is_not_treated_as_json() {
// Not an object, so falls through to logfmt/none like `eval_json_comparison` would treat it.
assert_eq!(detect_format("[1,2,3]"), "none");
assert_eq!(detect_format("42"), "none");
}
}
webapp/src/components/Docs.tsx
+13 -8
View File
@@ -87,19 +87,23 @@ const SECTIONS: DocSection[] = [
<Code>{'regex("error")'}</Code> — lines containing "error".
</li>
<li>
<Code>{'regex("error") and sequence<5'}</Code> — lines containing "error" with sequence under 5.
<Code>{'regex("error") and ottyl_seq<5'}</Code> — lines containing "error" with sequence under 5.
</li>
<li>
<Code>{'requests[duration]>100'}</Code> — the <Code>requests</Code> metric's captured{' '}
<Code>duration</Code> param over 100.
</li>
<li>
<Code>{'timestamp>-5m'}</Code> — lines from the last 5 minutes.
<Code>{'ottyl_ts>-5m'}</Code> — lines from the last 5 minutes.
</li>
<li>
<Code>{'ottyl_fmt==json'}</Code> — lines auto-detected as JSON.
</li>
</ul>
<p>
<Code>sequence</Code> and <Code>timestamp</Code> are reserved metric names present on every log line, so
they're always filterable without declaring anything.
<Code>ottyl_seq</Code>, <Code>ottyl_ts</Code>, and <Code>ottyl_fmt</Code> are reserved metric names present
on every log line, so they're always filterable without declaring anything. <Code>ottyl_fmt</Code> holds the
auto-detected format of the line's text — <Code>json</Code>, <Code>logfmt</Code>, or <Code>none</Code>.
</p>
<p className="font-semibold text-text">Date/time formats</p>
<p>Timestamp comparisons accept three forms:</p>
@@ -121,7 +125,7 @@ const SECTIONS: DocSection[] = [
<ul className="list-disc pl-5">
<li>
Only lines mentioning "timeout" in the last 15 minutes:{' '}
<Code>{'regex("timeout") and timestamp>-15m'}</Code>
<Code>{'regex("timeout") and ottyl_ts>-15m'}</Code>
</li>
<li>
Requests slower than 200ms, once a <Code>requests</Code> metric has a <Code>value</Code>-role{' '}
@@ -134,7 +138,7 @@ const SECTIONS: DocSection[] = [
<Code>{'requests[path]==/checkout and regex("5[0-9]{2}")'}</Code>
</li>
<li>
Everything before noon on a specific date: <Code>{'timestamp<2024-06-15T12:00:00Z'}</Code>
Everything before noon on a specific date: <Code>{'ottyl_ts<2024-06-15T12:00:00Z'}</Code>
</li>
</ul>
</div>
@@ -367,8 +371,9 @@ const SECTIONS: DocSection[] = [
</ul>
<p>
Query-based conditions (Immediate/Threshold/Ratio) reuse the filter bar's syntax, restricted to{' '}
<Code>regex(...)</Code>, <Code>sequence</Code>, <Code>timestamp</Code>, and <Code>and</Code>/<Code>or</Code>
. A <Code>metric_name[param]</Code> comparison isn't supported in an alert query.
<Code>regex(...)</Code>, <Code>ottyl_seq</Code>, <Code>ottyl_ts</Code>, <Code>ottyl_fmt</Code>, and{' '}
<Code>and</Code>/<Code>or</Code>. A <Code>metric_name[param]</Code> comparison isn't supported in an alert
query.
</p>
<p>
A window accepts a plain number of milliseconds or a humantime-style duration: <Code>30s</Code>,{' '}
webapp/src/components/FilterBar.tsx
+1 -1
View File
@@ -35,7 +35,7 @@ export function FilterBar({
{collapseToggle}
<input
className={queryInputClassName}
placeholder='regex("error") and sequence<5 or timestamp>12:00'
placeholder='regex("error") and ottyl_seq<5 or ottyl_ts>12:00'
value={draft}
onChange={(e) => setDraft(e.target.value)}
/>