Add nom parsers for go and python
This commit is contained in:
@@ -0,0 +1,3 @@
|
|||||||
|
{"time":"2026-09-06T10:35:34.499921+01:00","level":"WARN","msg":"eventC","labelC":1}
|
||||||
|
{"time":"2026-09-06T10:35:35.000837+01:00","level":"ERROR","msg":"eventB","labelC":2,"labelA":3}
|
||||||
|
{"time":"2026-09-06T10:35:36.504236+01:00","level":"INFO","msg":"eventB","labelB":"three"}
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
2026-09-06 10:35:55,217 INFO delete-py-log event=eventC labelC=1
|
||||||
|
2026-09-06 10:35:55,627 INFO delete-py-log event=eventB labelB=one
|
||||||
|
2026-09-06 10:35:56,442 INFO delete-py-log event=eventB labelA=1
|
||||||
+123
-7
@@ -4,7 +4,7 @@
|
|||||||
//! with an arbitrary non-logfmt prefix/suffix (e.g. `ts=... msg="..." x1=-4 y1=34`).
|
//! with an arbitrary non-logfmt prefix/suffix (e.g. `ts=... msg="..." x1=-4 y1=34`).
|
||||||
|
|
||||||
use nom::branch::alt;
|
use nom::branch::alt;
|
||||||
use nom::bytes::complete::{escaped, take_while1};
|
use nom::bytes::complete::{escaped, tag, take_until, take_while1};
|
||||||
use nom::character::complete::{char, none_of, one_of};
|
use nom::character::complete::{char, none_of, one_of};
|
||||||
use nom::error::{Error, ErrorKind};
|
use nom::error::{Error, ErrorKind};
|
||||||
use nom::sequence::delimited;
|
use nom::sequence::delimited;
|
||||||
@@ -112,16 +112,77 @@ pub const LOG_STREAM_PROCESS_PATTERN: &str =
|
|||||||
pub const LOG_STREAM_PID_PATTERN: &str =
|
pub const LOG_STREAM_PID_PATTERN: &str =
|
||||||
r"^\d{4}-\d{2}-\d{2}\s+\S+\s+0x[0-9a-fA-F]+\s+\S+\s+0x[0-9a-fA-F]+\s+(?P<pid>\d+)\s+\d+\s+[^\s:]+:\s";
|
r"^\d{4}-\d{2}-\d{2}\s+\S+\s+0x[0-9a-fA-F]+\s+\S+\s+0x[0-9a-fA-F]+\s+(?P<pid>\d+)\s+\d+\s+[^\s:]+:\s";
|
||||||
|
|
||||||
|
/// Go `log/slog` `NewJSONHandler` output (the `go-log` label): a single-line JSON object whose first
|
||||||
|
/// three fields are always `time`, `level`, `msg` in that fixed order (slog's `JSONHandler` writes the
|
||||||
|
/// built-in attrs before any caller-supplied ones), followed by zero or more arbitrary attrs and a
|
||||||
|
/// closing `}`. Parsed with nom rather than a full JSON decode since only that fixed prefix shape
|
||||||
|
/// needs verifying; the variable tail is deliberately not parsed field-by-field the way
|
||||||
|
/// `crate::engine::auto_declare_params` will separately pick up JSON leaf fields from the line.
|
||||||
|
/// Checked ahead of the generic JSON object check in `detect_format` since it's the more specific of
|
||||||
|
/// the two (see that function's doc comment for the general ordering rationale).
|
||||||
|
fn go_log_prefix(input: &str) -> IResult<&str, (&str, &str, &str)> {
|
||||||
|
let (input, _) = tag("{\"time\":\"")(input)?;
|
||||||
|
let (input, time) = take_until("\"")(input)?;
|
||||||
|
let (input, _) = tag("\",\"level\":\"")(input)?;
|
||||||
|
let (input, level) = take_until("\"")(input)?;
|
||||||
|
let (input, _) = tag("\",\"msg\":\"")(input)?;
|
||||||
|
let (input, msg) = take_until("\"")(input)?;
|
||||||
|
let (input, _) = char('"')(input)?;
|
||||||
|
Ok((input, (time, level, msg)))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_go_log_line(text: &str) -> bool {
|
||||||
|
match go_log_prefix(text) {
|
||||||
|
Ok((rest, _)) => rest.trim_end().ends_with('}'),
|
||||||
|
Err(_) => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Python `logging` module output in the shape `"%(asctime)s %(levelname)s %(name)s %(message)s"`
|
||||||
|
/// (the `py-log` label): a `YYYY-MM-DD HH:MM:SS,mmm` timestamp, an uppercase level word, a logger
|
||||||
|
/// name, then a message tail that (for `delete-py-log`, and any caller using `logger.info("event=%s
|
||||||
|
/// %s=%s", ...)`-style calls) is itself logfmt `key=value` pairs. Only the fixed prefix is parsed with
|
||||||
|
/// nom; the tail is handed to `extract_pairs` both to confirm the shape (a py-log line with no pairs
|
||||||
|
/// at all wouldn't be distinguishable from plain prose after this prefix) and because that's the same
|
||||||
|
/// extraction `crate::engine::auto_declare_params` needs anyway. Checked ahead of the generic logfmt
|
||||||
|
/// check in `detect_format` since it's the more specific of the two.
|
||||||
|
fn py_log_prefix(input: &str) -> IResult<&str, (&str, &str, &str, &str)> {
|
||||||
|
let (input, date) = take_while1(|c: char| c.is_ascii_digit() || c == '-')(input)?;
|
||||||
|
let (input, _) = char(' ')(input)?;
|
||||||
|
let (input, time) = take_while1(|c: char| c.is_ascii_digit() || c == ':' || c == ',')(input)?;
|
||||||
|
let (input, _) = char(' ')(input)?;
|
||||||
|
let (input, level) = take_while1(|c: char| c.is_ascii_uppercase())(input)?;
|
||||||
|
let (input, _) = char(' ')(input)?;
|
||||||
|
let (input, logger) =
|
||||||
|
take_while1(|c: char| c.is_ascii_alphanumeric() || c == '_' || c == '-' || c == '.')(input)?;
|
||||||
|
let (input, _) = char(' ')(input)?;
|
||||||
|
Ok((input, (date, time, level, logger)))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_py_log_line(text: &str) -> bool {
|
||||||
|
match py_log_prefix(text) {
|
||||||
|
Ok((rest, _)) => !extract_pairs(rest).is_empty(),
|
||||||
|
Err(_) => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// The value of the `ottyl_fmt` reserved param (`crate::engine::compile_filter`/`eval_alert_expr`):
|
/// The value of the `ottyl_fmt` reserved param (`crate::engine::compile_filter`/`eval_alert_expr`):
|
||||||
/// which format a line was recognized as when parsed. `log stream`'s fixed-column shape is checked
|
/// which format a line was recognized as when parsed. Checked most-specific-first: `log stream`'s
|
||||||
/// first (see `LOG_STREAM_RE`'s doc comment for why it's the most specific of the four). JSON is
|
/// fixed-column shape (see `LOG_STREAM_RE`'s doc comment for why it's the most specific), then the two
|
||||||
/// checked next since a JSON object can incidentally contain `key=value`-shaped substrings that
|
/// named-tool shapes `go-log` (a `time`/`level`/`msg`-prefixed JSON object) and `py-log` (a
|
||||||
/// `extract_pairs` would otherwise pick up; only a top-level object counts (arrays/scalars aren't what
|
/// timestamp/level/logger prefix over logfmt pairs) — each is a strict subset of the generic JSON/logfmt
|
||||||
/// `crate::engine::eval_json_comparison` treats as JSON either). Anything else falls back to
|
/// case checked right after it, so they'd otherwise never be reached. JSON is checked next since a JSON
|
||||||
/// logfmt-style pair extraction, and a line with none of the above is `"none"`.
|
/// object can incidentally contain `key=value`-shaped substrings that `extract_pairs` would otherwise
|
||||||
|
/// pick up; only a top-level object counts (arrays/scalars aren't what `crate::engine::eval_json_comparison`
|
||||||
|
/// treats as JSON either). Anything else falls back to logfmt-style pair extraction, and a line with
|
||||||
|
/// none of the above is `"none"`.
|
||||||
pub fn detect_format(text: &str) -> &'static str {
|
pub fn detect_format(text: &str) -> &'static str {
|
||||||
if is_log_stream_line(text) {
|
if is_log_stream_line(text) {
|
||||||
"log_stream"
|
"log_stream"
|
||||||
|
} else if is_go_log_line(text) {
|
||||||
|
"go-log"
|
||||||
|
} else if is_py_log_line(text) {
|
||||||
|
"py-log"
|
||||||
} else if serde_json::from_str::<serde_json::Value>(text).is_ok_and(|value| value.is_object()) {
|
} else if serde_json::from_str::<serde_json::Value>(text).is_ok_and(|value| value.is_object()) {
|
||||||
"json"
|
"json"
|
||||||
} else if !extract_pairs(text).is_empty() {
|
} else if !extract_pairs(text).is_empty() {
|
||||||
@@ -250,4 +311,59 @@ mod tests {
|
|||||||
// The embedded key=value pairs are still there for auto_declare_params to see.
|
// The embedded key=value pairs are still there for auto_declare_params to see.
|
||||||
assert!(!extract_pairs(line).is_empty());
|
assert!(!extract_pairs(line).is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const GO_LOG_FIXTURE: &str = include_str!("fixtures/go_log_sample.txt");
|
||||||
|
const PY_LOG_FIXTURE: &str = include_str!("fixtures/py_log_sample.txt");
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn detects_go_log_shaped_lines() {
|
||||||
|
for line in GO_LOG_FIXTURE.lines() {
|
||||||
|
assert!(is_go_log_line(line), "expected go-log line to match: {line}");
|
||||||
|
assert_eq!(detect_format(line), "go-log");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn detects_py_log_shaped_lines() {
|
||||||
|
for line in PY_LOG_FIXTURE.lines() {
|
||||||
|
assert!(is_py_log_line(line), "expected py-log line to match: {line}");
|
||||||
|
assert_eq!(detect_format(line), "py-log");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_generic_json_object_is_not_detected_as_go_log() {
|
||||||
|
let line = r#"{"level":"error","code":500}"#;
|
||||||
|
assert!(!is_go_log_line(line));
|
||||||
|
assert_eq!(detect_format(line), "json");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_generic_logfmt_line_is_not_detected_as_py_log() {
|
||||||
|
let line = r#"ts=1788648105773 msg="entity_moved" x1=-4"#;
|
||||||
|
assert!(!is_py_log_line(line));
|
||||||
|
assert_eq!(detect_format(line), "logfmt");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn go_log_prefix_extracts_time_level_and_msg() {
|
||||||
|
let (rest, (time, level, msg)) =
|
||||||
|
go_log_prefix(r#"{"time":"2026-09-06T10:35:34.499921+01:00","level":"WARN","msg":"eventC","labelC":1}"#)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(time, "2026-09-06T10:35:34.499921+01:00");
|
||||||
|
assert_eq!(level, "WARN");
|
||||||
|
assert_eq!(msg, "eventC");
|
||||||
|
assert_eq!(rest, r#","labelC":1}"#);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn py_log_prefix_extracts_date_time_level_and_logger() {
|
||||||
|
let (rest, (date, time, level, logger)) =
|
||||||
|
py_log_prefix("2026-09-06 10:35:55,217 INFO delete-py-log event=eventC labelC=1").unwrap();
|
||||||
|
assert_eq!(date, "2026-09-06");
|
||||||
|
assert_eq!(time, "10:35:55,217");
|
||||||
|
assert_eq!(level, "INFO");
|
||||||
|
assert_eq!(logger, "delete-py-log");
|
||||||
|
assert_eq!(rest, "event=eventC labelC=1");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user