use nom::IResult; use nom::branch::alt; use nom::bytes::complete::{escaped, tag, take_until, take_while1}; use nom::character::complete::{char, none_of, one_of}; use nom::error::{Error, ErrorKind}; use nom::sequence::delimited; use regex::Regex; use std::sync::LazyLock; use crate::consts::{COLUMN_SEPARATOR_PATTERN, COLUMN_VALUE_PATTERN}; fn key(input: &str) -> IResult<&str, &str> { take_while1(|c: char| c.is_ascii_alphanumeric() || c == '_' || c == '.' || c == '-')(input) } fn quoted_value(input: &str) -> IResult<&str, &str> { delimited( char('"'), alt(( escaped(none_of("\\\""), '\\', one_of("\"\\nt")), nom::combinator::success(""), )), char('"'), )(input) } fn bareword_value(input: &str) -> IResult<&str, &str> { take_while1(|c: char| !c.is_whitespace())(input) } fn looks_like_base64_padding_artifact(value: &str) -> bool { value.starts_with('=') } fn pair(input: &str) -> IResult<&str, (&str, &str, bool)> { let (input, k) = key(input)?; let (input, _) = char('=')(input)?; if let Ok((rest, v)) = quoted_value(input) { Ok((rest, (k, v, true))) } else { let (rest, v) = bareword_value(input)?; if looks_like_base64_padding_artifact(v) { return Err(nom::Err::Error(Error::new(input, ErrorKind::Verify))); } Ok((rest, (k, v, false))) } } pub fn extract_pairs(line: &str) -> Vec<(&str, &str, bool)> { let mut results = Vec::new(); let mut rest = line; while !rest.is_empty() { match pair(rest) { Ok((remaining, found)) => { results.push(found); rest = remaining; } Err(_) => { let mut chars = rest.chars(); chars.next(); rest = chars.as_str(); } } } results } static LOG_STREAM_RE: LazyLock = LazyLock::new(|| { Regex::new( r"^\d{4}-\d{2}-\d{2}\s+\S+\s+0x[0-9a-fA-F]+\s+\S+\s+0x[0-9a-fA-F]+\s+\d+\s+\d+\s+[^\s:]+:\s", ) .expect("static log_stream regex must compile") }); pub fn is_log_stream_line(line: &str) -> bool { LOG_STREAM_RE.is_match(line) } fn bounded_column_value_pattern(width: usize) -> String { if width <= 1 { r"\S".to_string() } else { format!(r"\S(?:.{{0,{}}}\S)?", width - 2) } } fn is_separator_run(run: &str) -> bool { run.len() >= 2 || run.contains('\t') } static WHITESPACE_RUN: LazyLock = LazyLock::new(|| Regex::new(r"[ \t]+").expect("static regex must compile")); fn split_columns_with_offsets(line: &str, max_columns: Option) -> Vec<(usize, &str)> { let mut columns = Vec::new(); let mut start = 0usize; let mut consumed = 0usize; for m in WHITESPACE_RUN.find_iter(line) { if max_columns.is_some_and(|max| consumed + 1 >= max) { break; } let run = &line[m.start()..m.end()]; if is_separator_run(run) { columns.push((start, &line[start..m.start()])); start = m.end(); consumed += 1; } } columns.push((start, &line[start..])); columns } fn split_columns(line: &str, max_columns: Option) -> Vec<&str> { split_columns_with_offsets(line, max_columns) .into_iter() .map(|(_, s)| s) .collect() } static COLUMN_NAME: LazyLock = LazyLock::new(|| Regex::new(r"^[A-Za-z][A-Za-z0-9_]*$").expect("static regex must compile")); #[derive(Clone)] pub struct LogStreamColumn { pub name: String, start: usize, } #[derive(Clone)] pub struct LogStreamHeader { pub columns: Vec, tab_delimited: bool, } fn char_slice(s: &str, start: usize, end: Option) -> &str { if end.is_some_and(|end| end <= start) { return ""; } let mut indices = s.char_indices(); let byte_start = indices.nth(start).map(|(i, _)| i).unwrap_or(s.len()); let byte_end = match end { Some(end) => s.char_indices().nth(end).map(|(i, _)| i).unwrap_or(s.len()), None => s.len(), }; &s[byte_start..byte_end.max(byte_start)] } impl LogStreamHeader { pub fn detect(line: &str) -> Option { if is_log_stream_line(line) { return None; } let mut columns = Vec::new(); for (byte_start, slice) in split_columns_with_offsets(line, None) { let trimmed = slice.trim(); if trimmed.is_empty() { continue; } let leading_ws = slice.len() - slice.trim_start().len(); let start = line[..byte_start + leading_ws].chars().count(); columns.push(LogStreamColumn { name: trimmed.to_string(), start, }); } if columns.len() < 2 || !columns.iter().all(|c| COLUMN_NAME.is_match(&c.name)) { return None; } let tab_delimited = WHITESPACE_RUN .find_iter(line) .any(|m| m.as_str().contains('\t')); Some(LogStreamHeader { columns, tab_delimited, }) } pub fn split_row<'a>(&self, line: &'a str) -> Vec<&'a str> { if self.tab_delimited { return split_columns(line, Some(self.columns.len() + 1)); } let mut result = Vec::with_capacity(self.columns.len() + 1); for (i, column) in self.columns.iter().enumerate() { match self.columns.get(i + 1) { Some(next) => result.push(char_slice(line, column.start, Some(next.start)).trim()), None => { let tail = char_slice(line, column.start, None); let pieces = split_columns(tail, Some(2)); result.push(pieces[0].trim()); if let Some(message) = pieces.get(1) { result.push(message); } } } } result } pub fn column_pattern(&self, index: usize, capture_group: &str) -> String { if self.tab_delimited { let mut pattern = String::from("^"); for _ in 0..index { pattern.push_str(COLUMN_VALUE_PATTERN); pattern.push_str(COLUMN_SEPARATOR_PATTERN); } pattern.push_str(&format!("(?P<{capture_group}>{COLUMN_VALUE_PATTERN})")); return pattern; } let start = self.columns[index].start; let value_pattern = match self.columns.get(index + 1) { Some(next) => bounded_column_value_pattern(next.start.saturating_sub(start)), None => COLUMN_VALUE_PATTERN.to_string(), }; format!("^.{{{start}}}(?P<{capture_group}>{value_pattern})") } pub fn message_pattern(&self, capture_group: &str) -> String { if self.tab_delimited { let mut pattern = String::from("^"); for _ in 0..self.columns.len() { pattern.push_str(COLUMN_VALUE_PATTERN); pattern.push_str(COLUMN_SEPARATOR_PATTERN); } pattern.push_str(&format!("(?P<{capture_group}>.*)$")); return pattern; } let start = self.columns.last().map(|c| c.start).unwrap_or(0); format!( "^.{{{start}}}{COLUMN_VALUE_PATTERN}{COLUMN_SEPARATOR_PATTERN}(?P<{capture_group}>.*)$" ) } } fn go_log_prefix(input: &str) -> IResult<&str, (&str, &str, &str)> { let (input, _) = tag("{\"time\":\"")(input)?; let (input, time) = take_until("\"")(input)?; let (input, _) = tag("\",\"level\":\"")(input)?; let (input, level) = take_until("\"")(input)?; let (input, _) = tag("\",\"msg\":\"")(input)?; let (input, msg) = take_until("\"")(input)?; let (input, _) = char('"')(input)?; Ok((input, (time, level, msg))) } pub fn is_go_log_line(text: &str) -> bool { match go_log_prefix(text) { Ok((rest, _)) => rest.trim_end().ends_with('}'), Err(_) => false, } } fn py_log_prefix(input: &str) -> IResult<&str, (&str, &str, &str, &str)> { let (input, date) = take_while1(|c: char| c.is_ascii_digit() || c == '-')(input)?; let (input, _) = char(' ')(input)?; let (input, time) = take_while1(|c: char| c.is_ascii_digit() || c == ':' || c == ',')(input)?; let (input, _) = char(' ')(input)?; let (input, level) = take_while1(|c: char| c.is_ascii_uppercase())(input)?; let (input, _) = char(' ')(input)?; let (input, logger) = take_while1(|c: char| c.is_ascii_alphanumeric() || c == '_' || c == '-' || c == '.')( input, )?; let (input, _) = char(' ')(input)?; Ok((input, (date, time, level, logger))) } pub fn is_py_log_line(text: &str) -> bool { match py_log_prefix(text) { Ok((rest, _)) => !extract_pairs(rest).is_empty(), Err(_) => false, } } pub fn detect_format(text: &str) -> &'static str { if is_log_stream_line(text) { "log_stream" } else if is_go_log_line(text) { "go-log" } else if is_py_log_line(text) { "py-log" } else if serde_json::from_str::(text).is_ok_and(|value| value.is_object()) { "json" } else if !extract_pairs(text).is_empty() { "logfmt" } else { "none" } } #[cfg(test)] mod tests { use super::*; #[test] fn extracts_quoted_value() { let pairs = extract_pairs(r#"msg="entity_moved""#); assert_eq!(pairs, vec![("msg", "entity_moved", true)]); } #[test] fn extracts_unquoted_bareword_value() { let pairs = extract_pairs("x1=-4"); assert_eq!(pairs, vec![("x1", "-4", false)]); } #[test] fn extracts_mixed_quoted_and_unquoted_pairs_in_one_line() { let pairs = extract_pairs(r#"ts=1788648105773 msg="entity_moved" x1=-4 y1=34"#); assert_eq!( pairs, vec![ ("ts", "1788648105773", false), ("msg", "entity_moved", true), ("x1", "-4", false), ("y1", "34", false), ] ); } #[test] fn returns_empty_for_lines_with_no_pairs() { assert!(extract_pairs("just some plain prose here").is_empty()); assert!(extract_pairs(r#"{"foo": {"bar": 42}}"#).is_empty()); } #[test] fn does_not_panic_on_empty_value_at_end_of_line() { assert_eq!(extract_pairs("key="), Vec::<(&str, &str, bool)>::new()); assert_eq!(extract_pairs(r#"key="""#), vec![("key", "", true)]); } #[test] fn keeps_escaped_quote_inside_quoted_value() { let pairs = extract_pairs(r#"msg="he said \"hi\"""#); assert_eq!(pairs, vec![("msg", r#"he said \"hi\""#, true)]); } #[test] fn detects_json_object_lines_as_json() { assert_eq!(detect_format(r#"{"level":"error","code":500}"#), "json"); } #[test] fn detects_key_value_lines_as_logfmt() { assert_eq!( detect_format(r#"ts=1788648105773 msg="entity_moved" x1=-4"#), "logfmt" ); } #[test] fn detects_plain_prose_as_none() { assert_eq!(detect_format("just some plain prose here"), "none"); } #[test] fn a_bare_json_array_or_scalar_is_not_treated_as_json() { assert_eq!(detect_format("[1,2,3]"), "none"); assert_eq!(detect_format("42"), "none"); } #[test] fn rejects_base64_padding_fragment_as_a_pair() { let pairs = extract_pairs( "qname: , qtype: A, client pid: 12260 (firefox)", ); assert!( pairs.iter().all(|(k, _, _)| *k != "rKTEtG7JmnThhoKPbZiVrA"), "base64 padding fragment must not be extracted as a pair: {pairs:?}" ); } const LOG_STREAM_FIXTURE: &str = include_str!("../fixtures/log_stream_sample.txt"); #[test] fn detects_log_stream_shaped_lines() { let data_line = LOG_STREAM_FIXTURE.lines().nth(1).unwrap(); assert_eq!(detect_format(data_line), "log_stream"); assert!(is_log_stream_line(data_line)); } #[test] fn header_row_is_not_detected_as_log_stream() { let header = LOG_STREAM_FIXTURE.lines().next().unwrap(); assert!(!is_log_stream_line(header)); } fn column_names(header: &LogStreamHeader) -> Vec<&str> { header.columns.iter().map(|c| c.name.as_str()).collect() } #[test] fn detects_log_stream_header_columns_generically() { let header_line = LOG_STREAM_FIXTURE.lines().next().unwrap(); let header = LogStreamHeader::detect(header_line).expect("fixture header must be detected"); assert_eq!( column_names(&header), vec!["Timestamp", "Thread", "Type", "Activity", "PID", "TTL"] ); assert!(!header.tab_delimited); } #[test] fn a_log_stream_data_line_is_never_detected_as_its_own_header() { let data_line = LOG_STREAM_FIXTURE.lines().nth(1).unwrap(); assert!(LogStreamHeader::detect(data_line).is_none()); } #[test] fn plain_prose_is_not_detected_as_a_log_stream_header() { assert!(LogStreamHeader::detect("just some plain prose here").is_none()); assert!(LogStreamHeader::detect("ts=1788648105773 msg=\"entity_moved\"").is_none()); } #[test] fn splits_log_stream_row_into_named_columns_plus_trailing_message() { let header_line = LOG_STREAM_FIXTURE.lines().next().unwrap(); let header = LogStreamHeader::detect(header_line).unwrap(); let data_line = "2026-09-06 01:48:54.518117+0100 0x4dfed2 Default 0x0 606 0 WindowServer: (HSTouchHIDService) something happened"; let row = header.split_row(data_line); assert_eq!(row.len(), header.columns.len() + 1); assert_eq!(row[0], "2026-09-06 01:48:54.518117+0100"); assert_eq!(row[1], "0x4dfed2"); assert_eq!(row[2], "Default"); assert_eq!(row[3], "0x0"); assert_eq!(row[4], "606"); assert_eq!(row[5], "0"); assert_eq!( row[6], "WindowServer: (HSTouchHIDService) something happened" ); } #[test] fn a_single_tab_separates_columns_exactly_like_a_run_of_spaces() { let header = LogStreamHeader::detect("Timestamp\tThread\tType").unwrap(); assert_eq!(column_names(&header), vec!["Timestamp", "Thread", "Type"]); assert!(header.tab_delimited); let spaced = header.split_row("2026-09-06 01:48:54\t0x4dfed2\t\tDefault trailing message"); let tabbed = header.split_row("2026-09-06 01:48:54\t0x4dfed2\tDefault trailing message"); assert_eq!(spaced[0], "2026-09-06 01:48:54"); assert_eq!(spaced[1], "0x4dfed2"); assert_eq!(spaced[2], "Default trailing message"); assert_eq!(tabbed[0], "2026-09-06 01:48:54"); assert_eq!(tabbed[1], "0x4dfed2"); assert_eq!(tabbed[2], "Default trailing message"); } #[test] fn column_and_message_patterns_capture_the_same_fields_split_log_stream_row_finds() { let header_line = LOG_STREAM_FIXTURE.lines().next().unwrap(); let header = LogStreamHeader::detect(header_line).unwrap(); let line = "2026-09-06 01:48:54.518117+0100 0x4dfed2 Default 0x0 606 0 WindowServer: (HSTouchHIDService) something happened"; let type_re = Regex::new(&header.column_pattern(2, "Type")).unwrap(); assert_eq!( type_re .captures(line) .unwrap() .name("Type") .unwrap() .as_str(), "Default" ); let pid_re = Regex::new(&header.column_pattern(4, "PID")).unwrap(); assert_eq!( pid_re.captures(line).unwrap().name("PID").unwrap().as_str(), "606" ); let message_re = Regex::new(&header.message_pattern("message")).unwrap(); assert_eq!( message_re .captures(line) .unwrap() .name("message") .unwrap() .as_str(), "WindowServer: (HSTouchHIDService) something happened" ); let timestamp_re = Regex::new(&header.column_pattern(0, "Timestamp")).unwrap(); assert_eq!( timestamp_re .captures(line) .unwrap() .name("Timestamp") .unwrap() .as_str(), "2026-09-06 01:48:54.518117+0100" ); } #[test] fn is_log_stream_line_rejects_non_columnar_lines_that_a_fixed_offset_column_regex_would_still_match() { let header_line = LOG_STREAM_FIXTURE.lines().next().unwrap(); let header = LogStreamHeader::detect(header_line).unwrap(); let timestamp_re = Regex::new(&header.column_pattern(0, "Timestamp")).unwrap(); let continuation_line = "so_gencnt: 4042707 so_state: 0x0"; assert!( timestamp_re.captures(continuation_line).is_some(), "sanity: regex alone still matches garbage" ); assert!(!is_log_stream_line(continuation_line)); assert!(!is_log_stream_line(header_line)); assert!(!is_log_stream_line("}")); } #[test] fn a_wifi_stat_dump_line_is_still_recognized_as_log_stream_despite_embedded_key_value_pairs() { let line = LOG_STREAM_FIXTURE .lines() .find(|line| line.contains("txRTSFrm=")) .expect("fixture must contain the WiFi stat dump line"); assert_eq!(detect_format(line), "log_stream"); assert!(!extract_pairs(line).is_empty()); } const GO_LOG_FIXTURE: &str = include_str!("../fixtures/go_log_sample.txt"); const PY_LOG_FIXTURE: &str = include_str!("../fixtures/py_log_sample.txt"); #[test] fn detects_go_log_shaped_lines() { for line in GO_LOG_FIXTURE.lines() { assert!( is_go_log_line(line), "expected go-log line to match: {line}" ); assert_eq!(detect_format(line), "go-log"); } } #[test] fn detects_py_log_shaped_lines() { for line in PY_LOG_FIXTURE.lines() { assert!( is_py_log_line(line), "expected py-log line to match: {line}" ); assert_eq!(detect_format(line), "py-log"); } } #[test] fn a_generic_json_object_is_not_detected_as_go_log() { let line = r#"{"level":"error","code":500}"#; assert!(!is_go_log_line(line)); assert_eq!(detect_format(line), "json"); } #[test] fn a_generic_logfmt_line_is_not_detected_as_py_log() { let line = r#"ts=1788648105773 msg="entity_moved" x1=-4"#; assert!(!is_py_log_line(line)); assert_eq!(detect_format(line), "logfmt"); } #[test] fn go_log_prefix_extracts_time_level_and_msg() { let (rest, (time, level, msg)) = go_log_prefix(r#"{"time":"2026-09-06T10:35:34.499921+01:00","level":"WARN","msg":"eventC","labelC":1}"#) .unwrap(); assert_eq!(time, "2026-09-06T10:35:34.499921+01:00"); assert_eq!(level, "WARN"); assert_eq!(msg, "eventC"); assert_eq!(rest, r#","labelC":1}"#); } #[test] fn py_log_prefix_extracts_date_time_level_and_logger() { let (rest, (date, time, level, logger)) = py_log_prefix("2026-09-06 10:35:55,217 INFO delete-py-log event=eventC labelC=1") .unwrap(); assert_eq!(date, "2026-09-06"); assert_eq!(time, "10:35:55,217"); assert_eq!(level, "INFO"); assert_eq!(logger, "delete-py-log"); assert_eq!(rest, "event=eventC labelC=1"); } }