Add nom parsers for go and python

This commit is contained in:
Hugh Kaznowski
2026-09-06 10:45:55 +01:00
parent 8f71f2d056
commit 4158c63086
3 changed files with 129 additions and 7 deletions
+3
View File
@@ -0,0 +1,3 @@
{"time":"2026-09-06T10:35:34.499921+01:00","level":"WARN","msg":"eventC","labelC":1}
{"time":"2026-09-06T10:35:35.000837+01:00","level":"ERROR","msg":"eventB","labelC":2,"labelA":3}
{"time":"2026-09-06T10:35:36.504236+01:00","level":"INFO","msg":"eventB","labelB":"three"}
+3
View File
@@ -0,0 +1,3 @@
2026-09-06 10:35:55,217 INFO delete-py-log event=eventC labelC=1
2026-09-06 10:35:55,627 INFO delete-py-log event=eventB labelB=one
2026-09-06 10:35:56,442 INFO delete-py-log event=eventB labelA=1
+123 -7
View File
@@ -4,7 +4,7 @@
//! with an arbitrary non-logfmt prefix/suffix (e.g. `ts=... msg="..." x1=-4 y1=34`). //! with an arbitrary non-logfmt prefix/suffix (e.g. `ts=... msg="..." x1=-4 y1=34`).
use nom::branch::alt; use nom::branch::alt;
use nom::bytes::complete::{escaped, take_while1}; use nom::bytes::complete::{escaped, tag, take_until, take_while1};
use nom::character::complete::{char, none_of, one_of}; use nom::character::complete::{char, none_of, one_of};
use nom::error::{Error, ErrorKind}; use nom::error::{Error, ErrorKind};
use nom::sequence::delimited; use nom::sequence::delimited;
@@ -112,16 +112,77 @@ pub const LOG_STREAM_PROCESS_PATTERN: &str =
pub const LOG_STREAM_PID_PATTERN: &str = pub const LOG_STREAM_PID_PATTERN: &str =
r"^\d{4}-\d{2}-\d{2}\s+\S+\s+0x[0-9a-fA-F]+\s+\S+\s+0x[0-9a-fA-F]+\s+(?P<pid>\d+)\s+\d+\s+[^\s:]+:\s"; r"^\d{4}-\d{2}-\d{2}\s+\S+\s+0x[0-9a-fA-F]+\s+\S+\s+0x[0-9a-fA-F]+\s+(?P<pid>\d+)\s+\d+\s+[^\s:]+:\s";
/// Go `log/slog` `NewJSONHandler` output (the `go-log` label): a single-line JSON object whose first
/// three fields are always `time`, `level`, `msg` in that fixed order (slog's `JSONHandler` writes the
/// built-in attrs before any caller-supplied ones), followed by zero or more arbitrary attrs and a
/// closing `}`. Parsed with nom rather than a full JSON decode since only that fixed prefix shape
/// needs verifying; the variable tail is deliberately not parsed field-by-field the way
/// `crate::engine::auto_declare_params` will separately pick up JSON leaf fields from the line.
/// Checked ahead of the generic JSON object check in `detect_format` since it's the more specific of
/// the two (see that function's doc comment for the general ordering rationale).
fn go_log_prefix(input: &str) -> IResult<&str, (&str, &str, &str)> {
let (input, _) = tag("{\"time\":\"")(input)?;
let (input, time) = take_until("\"")(input)?;
let (input, _) = tag("\",\"level\":\"")(input)?;
let (input, level) = take_until("\"")(input)?;
let (input, _) = tag("\",\"msg\":\"")(input)?;
let (input, msg) = take_until("\"")(input)?;
let (input, _) = char('"')(input)?;
Ok((input, (time, level, msg)))
}
pub fn is_go_log_line(text: &str) -> bool {
match go_log_prefix(text) {
Ok((rest, _)) => rest.trim_end().ends_with('}'),
Err(_) => false,
}
}
/// Python `logging` module output in the shape `"%(asctime)s %(levelname)s %(name)s %(message)s"`
/// (the `py-log` label): a `YYYY-MM-DD HH:MM:SS,mmm` timestamp, an uppercase level word, a logger
/// name, then a message tail that (for `delete-py-log`, and any caller using `logger.info("event=%s
/// %s=%s", ...)`-style calls) is itself logfmt `key=value` pairs. Only the fixed prefix is parsed with
/// nom; the tail is handed to `extract_pairs` both to confirm the shape (a py-log line with no pairs
/// at all wouldn't be distinguishable from plain prose after this prefix) and because that's the same
/// extraction `crate::engine::auto_declare_params` needs anyway. Checked ahead of the generic logfmt
/// check in `detect_format` since it's the more specific of the two.
fn py_log_prefix(input: &str) -> IResult<&str, (&str, &str, &str, &str)> {
let (input, date) = take_while1(|c: char| c.is_ascii_digit() || c == '-')(input)?;
let (input, _) = char(' ')(input)?;
let (input, time) = take_while1(|c: char| c.is_ascii_digit() || c == ':' || c == ',')(input)?;
let (input, _) = char(' ')(input)?;
let (input, level) = take_while1(|c: char| c.is_ascii_uppercase())(input)?;
let (input, _) = char(' ')(input)?;
let (input, logger) =
take_while1(|c: char| c.is_ascii_alphanumeric() || c == '_' || c == '-' || c == '.')(input)?;
let (input, _) = char(' ')(input)?;
Ok((input, (date, time, level, logger)))
}
pub fn is_py_log_line(text: &str) -> bool {
match py_log_prefix(text) {
Ok((rest, _)) => !extract_pairs(rest).is_empty(),
Err(_) => false,
}
}
/// The value of the `ottyl_fmt` reserved param (`crate::engine::compile_filter`/`eval_alert_expr`): /// The value of the `ottyl_fmt` reserved param (`crate::engine::compile_filter`/`eval_alert_expr`):
/// which format a line was recognized as when parsed. `log stream`'s fixed-column shape is checked /// which format a line was recognized as when parsed. Checked most-specific-first: `log stream`'s
/// first (see `LOG_STREAM_RE`'s doc comment for why it's the most specific of the four). JSON is /// fixed-column shape (see `LOG_STREAM_RE`'s doc comment for why it's the most specific), then the two
/// checked next since a JSON object can incidentally contain `key=value`-shaped substrings that /// named-tool shapes `go-log` (a `time`/`level`/`msg`-prefixed JSON object) and `py-log` (a
/// `extract_pairs` would otherwise pick up; only a top-level object counts (arrays/scalars aren't what /// timestamp/level/logger prefix over logfmt pairs) — each is a strict subset of the generic JSON/logfmt
/// `crate::engine::eval_json_comparison` treats as JSON either). Anything else falls back to /// case checked right after it, so they'd otherwise never be reached. JSON is checked next since a JSON
/// logfmt-style pair extraction, and a line with none of the above is `"none"`. /// object can incidentally contain `key=value`-shaped substrings that `extract_pairs` would otherwise
/// pick up; only a top-level object counts (arrays/scalars aren't what `crate::engine::eval_json_comparison`
/// treats as JSON either). Anything else falls back to logfmt-style pair extraction, and a line with
/// none of the above is `"none"`.
pub fn detect_format(text: &str) -> &'static str { pub fn detect_format(text: &str) -> &'static str {
if is_log_stream_line(text) { if is_log_stream_line(text) {
"log_stream" "log_stream"
} else if is_go_log_line(text) {
"go-log"
} else if is_py_log_line(text) {
"py-log"
} else if serde_json::from_str::<serde_json::Value>(text).is_ok_and(|value| value.is_object()) { } else if serde_json::from_str::<serde_json::Value>(text).is_ok_and(|value| value.is_object()) {
"json" "json"
} else if !extract_pairs(text).is_empty() { } else if !extract_pairs(text).is_empty() {
@@ -250,4 +311,59 @@ mod tests {
// The embedded key=value pairs are still there for auto_declare_params to see. // The embedded key=value pairs are still there for auto_declare_params to see.
assert!(!extract_pairs(line).is_empty()); assert!(!extract_pairs(line).is_empty());
} }
const GO_LOG_FIXTURE: &str = include_str!("fixtures/go_log_sample.txt");
const PY_LOG_FIXTURE: &str = include_str!("fixtures/py_log_sample.txt");
#[test]
fn detects_go_log_shaped_lines() {
for line in GO_LOG_FIXTURE.lines() {
assert!(is_go_log_line(line), "expected go-log line to match: {line}");
assert_eq!(detect_format(line), "go-log");
}
}
#[test]
fn detects_py_log_shaped_lines() {
for line in PY_LOG_FIXTURE.lines() {
assert!(is_py_log_line(line), "expected py-log line to match: {line}");
assert_eq!(detect_format(line), "py-log");
}
}
#[test]
fn a_generic_json_object_is_not_detected_as_go_log() {
let line = r#"{"level":"error","code":500}"#;
assert!(!is_go_log_line(line));
assert_eq!(detect_format(line), "json");
}
#[test]
fn a_generic_logfmt_line_is_not_detected_as_py_log() {
let line = r#"ts=1788648105773 msg="entity_moved" x1=-4"#;
assert!(!is_py_log_line(line));
assert_eq!(detect_format(line), "logfmt");
}
#[test]
fn go_log_prefix_extracts_time_level_and_msg() {
let (rest, (time, level, msg)) =
go_log_prefix(r#"{"time":"2026-09-06T10:35:34.499921+01:00","level":"WARN","msg":"eventC","labelC":1}"#)
.unwrap();
assert_eq!(time, "2026-09-06T10:35:34.499921+01:00");
assert_eq!(level, "WARN");
assert_eq!(msg, "eventC");
assert_eq!(rest, r#","labelC":1}"#);
}
#[test]
fn py_log_prefix_extracts_date_time_level_and_logger() {
let (rest, (date, time, level, logger)) =
py_log_prefix("2026-09-06 10:35:55,217 INFO delete-py-log event=eventC labelC=1").unwrap();
assert_eq!(date, "2026-09-06");
assert_eq!(time, "10:35:55,217");
assert_eq!(level, "INFO");
assert_eq!(logger, "delete-py-log");
assert_eq!(rest, "event=eventC labelC=1");
}
} }