Compare commits

..
3 Commits
Author SHA1 Message Date
Hugh Kaznowski 4158c63086 Add nom parsers for go and python 2026-09-06 10:45:55 +01:00
Hugh Kaznowski 8f71f2d056 Icons, docs formats 2026-09-06 10:36:26 +01:00
Hugh Kaznowski 17efe222df Add icon 2026-09-06 10:16:11 +01:00
9 changed files with 358 additions and 25 deletions
+34
View File
@@ -67,6 +67,40 @@ pub fn extract_json_field(text: &str, field_path: &str) -> Option<String> {
}
}
/// Flattens a JSON object into dot-separated field paths mapped to each leaf scalar's raw text form
/// (string/number/bool), for [`crate::engine::auto_declare_params`] when a line is JSON
/// (`crate::logfmt::detect_format`) rather than logfmt — the JSON equivalent of
/// `crate::logfmt::extract_pairs`'s `key=value` scan. Nested objects recurse, dot-joining the path
/// (`"foo": {"bar": 1}` yields `"foo.bar"`); arrays and null are skipped since there's no single leaf
/// scalar to declare a param against, matching `extract_json_field`'s own restriction to scalars.
/// Returns an empty vec for anything that isn't a JSON object, same as `extract_pairs` returns empty
/// for a line with no `key=value` pairs.
pub fn flatten_json_object(text: &str) -> Vec<(String, String)> {
let Ok(serde_json::Value::Object(root)) = serde_json::from_str::<serde_json::Value>(text) else {
return Vec::new();
};
let mut results = Vec::new();
flatten_json_object_into(&root, "", &mut results);
results
}
fn flatten_json_object_into(
object: &serde_json::Map<String, serde_json::Value>,
prefix: &str,
results: &mut Vec<(String, String)>,
) {
for (key, value) in object {
let path = if prefix.is_empty() { key.clone() } else { format!("{prefix}.{key}") };
match value {
serde_json::Value::String(s) => results.push((path, s.clone())),
serde_json::Value::Number(n) => results.push((path, n.to_string())),
serde_json::Value::Bool(b) => results.push((path, b.to_string())),
serde_json::Value::Object(nested) => flatten_json_object_into(nested, &path, results),
serde_json::Value::Null | serde_json::Value::Array(_) => {}
}
}
}
/// `rename_all = "lowercase"` matches other `kind`-style enums, so JSON bodies use "string"/"number"/"timestamp".
#[allow(dead_code)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+84 -21
View File
@@ -356,37 +356,79 @@ fn auto_derived_param_count(index: &BTreeMap<IndexKey, DataValue>) -> usize {
param_declarations_range(index).filter(|declaration| declaration.6).count()
}
/// Scans `text` for `key=value`/`key="value"` pairs (`crate::logfmt::extract_pairs`) and declares a
/// standalone param for any key not already declared — auto or manual. A manual declaration for a
/// name always wins and is never touched here; this only fills in names nobody has declared yet.
/// Run on every ingested line (`EngineMessage::Ingest`), same as `apply_all_metrics`, but note param
/// *values* are still recomputed from raw text at read time (`derive_log_params`), not stored here —
/// so a key discovered on line N also back-fills the column for lines before N on the next read.
/// Bounded by `MAX_AUTO_DECLARED_PARAMS`/`MAX_NEW_AUTO_PARAMS_PER_LINE` (see their doc comments) so a
/// single burst of unusual lines can't permanently inflate the declared-param set (ticket 050).
/// One param worth auto-declaring, found on an ingested line by either extraction path
/// `auto_declare_params` supports: a logfmt `key=value` pair (name, and a regex/capture-group built
/// from it) or a JSON leaf field (name being its dot-separated path, extracted via `json_field_path`
/// instead of a regex). Lets `auto_declare_params` share one insert-and-budget loop across both.
struct AutoParamCandidate {
name: String,
value: String,
message_match_regex: String,
capture_group: String,
json_field_path: Option<String>,
}
/// Scans `text` for auto-derivable params and declares a standalone param for any name not already
/// declared — auto or manual. A manual declaration for a name always wins and is never touched here;
/// this only fills in names nobody has declared yet. JSON lines (`crate::logfmt::detect_format`) are
/// scanned via `crate::data_value::flatten_json_object` (dot-path leaf fields, extracted by
/// `json_field_path` rather than regex, since a `key=value` scan finds nothing useful in JSON's
/// `"key":value` shape); anything else falls back to the original `key=value` pair scan
/// (`crate::logfmt::extract_pairs`). Run on every ingested line (`EngineMessage::Ingest`), same as
/// `apply_all_metrics`, but note param *values* are still recomputed from raw text at read time
/// (`derive_log_params`), not stored here — so a key discovered on line N also back-fills the column
/// for lines before N on the next read. Bounded by `MAX_AUTO_DECLARED_PARAMS`/
/// `MAX_NEW_AUTO_PARAMS_PER_LINE` (see their doc comments) so a single burst of unusual lines can't
/// permanently inflate the declared-param set (ticket 050).
fn auto_declare_params(index: &mut BTreeMap<IndexKey, DataValue>, text: &str) {
let mut total = auto_derived_param_count(index);
if total >= MAX_AUTO_DECLARED_PARAMS {
return;
}
let mut new_this_line = 0;
for (key, value, quoted) in crate::logfmt::extract_pairs(text) {
if new_this_line >= MAX_NEW_AUTO_PARAMS_PER_LINE || total >= MAX_AUTO_DECLARED_PARAMS {
break;
}
let index_key = IndexKey::ParamDeclaration { name: key.to_string() };
if index.contains_key(&index_key) {
continue;
}
let candidates: Vec<AutoParamCandidate> = if crate::logfmt::detect_format(text) == "json" {
crate::data_value::flatten_json_object(text)
.into_iter()
.map(|(path, value)| AutoParamCandidate {
name: path.clone(),
value,
message_match_regex: String::new(),
capture_group: String::new(),
json_field_path: Some(path),
})
.collect()
} else {
crate::logfmt::extract_pairs(text)
.into_iter()
.map(|(key, value, quoted)| {
let group = sanitize_group_name(key);
index.insert(
index_key,
DataValue::ParamDeclaration(ParamDeclaration {
AutoParamCandidate {
name: key.to_string(),
value: value.to_string(),
message_match_regex: build_auto_param_pattern(key, &group, quoted),
capture_group: group,
json_field_path: None,
kind: infer_kind(value),
}
})
.collect()
};
let mut new_this_line = 0;
for candidate in candidates {
if new_this_line >= MAX_NEW_AUTO_PARAMS_PER_LINE || total >= MAX_AUTO_DECLARED_PARAMS {
break;
}
let index_key = IndexKey::ParamDeclaration { name: candidate.name.clone() };
if index.contains_key(&index_key) {
continue;
}
index.insert(
index_key,
DataValue::ParamDeclaration(ParamDeclaration {
name: candidate.name,
message_match_regex: candidate.message_match_regex,
capture_group: candidate.capture_group,
json_field_path: candidate.json_field_path,
kind: infer_kind(&candidate.value),
elapsed_since_last: false,
auto_derived: true,
}),
@@ -2012,6 +2054,27 @@ mod tests {
assert_eq!(params.get("msg"), Some(&MetricPrimitive::String("entity_moved".to_string())));
}
#[test]
fn auto_declare_params_discovers_json_leaf_fields_via_json_field_path() {
let mut index = BTreeMap::new();
auto_declare_params(&mut index, r#"{"level":"error","code":500,"request":{"path":"/x"}}"#);
let declarations: HashMap<&str, _> =
param_declarations_range(&index).map(|d| (d.0, d)).collect();
assert_eq!(declarations.len(), 3);
assert_eq!(declarations["code"].4, MetricPrimitiveKind::Number);
assert_eq!(declarations["code"].3, Some("code"));
assert!(declarations["level"].6, "expected level to be flagged auto_derived");
assert_eq!(declarations["request.path"].3, Some("request.path"));
let compiled = compiled_param_declarations(&index);
let params =
derive_log_params(r#"{"level":"error","code":500,"request":{"path":"/x"}}"#, &compiled);
assert_eq!(params.get("level"), Some(&MetricPrimitive::String("error".to_string())));
assert_eq!(params.get("code"), Some(&MetricPrimitive::Number(Number::Int(500))));
assert_eq!(params.get("request.path"), Some(&MetricPrimitive::String("/x".to_string())));
}
#[test]
fn auto_declare_params_never_overwrites_an_existing_declaration() {
let mut index = BTreeMap::new();
+3
View File
@@ -0,0 +1,3 @@
{"time":"2026-09-06T10:35:34.499921+01:00","level":"WARN","msg":"eventC","labelC":1}
{"time":"2026-09-06T10:35:35.000837+01:00","level":"ERROR","msg":"eventB","labelC":2,"labelA":3}
{"time":"2026-09-06T10:35:36.504236+01:00","level":"INFO","msg":"eventB","labelB":"three"}
+3
View File
@@ -0,0 +1,3 @@
2026-09-06 10:35:55,217 INFO delete-py-log event=eventC labelC=1
2026-09-06 10:35:55,627 INFO delete-py-log event=eventB labelB=one
2026-09-06 10:35:56,442 INFO delete-py-log event=eventB labelA=1
+123 -7
View File
@@ -4,7 +4,7 @@
//! with an arbitrary non-logfmt prefix/suffix (e.g. `ts=... msg="..." x1=-4 y1=34`).
use nom::branch::alt;
use nom::bytes::complete::{escaped, take_while1};
use nom::bytes::complete::{escaped, tag, take_until, take_while1};
use nom::character::complete::{char, none_of, one_of};
use nom::error::{Error, ErrorKind};
use nom::sequence::delimited;
@@ -112,16 +112,77 @@ pub const LOG_STREAM_PROCESS_PATTERN: &str =
pub const LOG_STREAM_PID_PATTERN: &str =
r"^\d{4}-\d{2}-\d{2}\s+\S+\s+0x[0-9a-fA-F]+\s+\S+\s+0x[0-9a-fA-F]+\s+(?P<pid>\d+)\s+\d+\s+[^\s:]+:\s";
/// Go `log/slog` `NewJSONHandler` output (the `go-log` label): a single-line JSON object whose first
/// three fields are always `time`, `level`, `msg` in that fixed order (slog's `JSONHandler` writes the
/// built-in attrs before any caller-supplied ones), followed by zero or more arbitrary attrs and a
/// closing `}`. Parsed with nom rather than a full JSON decode since only that fixed prefix shape
/// needs verifying; the variable tail is deliberately not parsed field-by-field the way
/// `crate::engine::auto_declare_params` will separately pick up JSON leaf fields from the line.
/// Checked ahead of the generic JSON object check in `detect_format` since it's the more specific of
/// the two (see that function's doc comment for the general ordering rationale).
fn go_log_prefix(input: &str) -> IResult<&str, (&str, &str, &str)> {
let (input, _) = tag("{\"time\":\"")(input)?;
let (input, time) = take_until("\"")(input)?;
let (input, _) = tag("\",\"level\":\"")(input)?;
let (input, level) = take_until("\"")(input)?;
let (input, _) = tag("\",\"msg\":\"")(input)?;
let (input, msg) = take_until("\"")(input)?;
let (input, _) = char('"')(input)?;
Ok((input, (time, level, msg)))
}
pub fn is_go_log_line(text: &str) -> bool {
match go_log_prefix(text) {
Ok((rest, _)) => rest.trim_end().ends_with('}'),
Err(_) => false,
}
}
/// Python `logging` module output in the shape `"%(asctime)s %(levelname)s %(name)s %(message)s"`
/// (the `py-log` label): a `YYYY-MM-DD HH:MM:SS,mmm` timestamp, an uppercase level word, a logger
/// name, then a message tail that (for `delete-py-log`, and any caller using `logger.info("event=%s
/// %s=%s", ...)`-style calls) is itself logfmt `key=value` pairs. Only the fixed prefix is parsed with
/// nom; the tail is handed to `extract_pairs` both to confirm the shape (a py-log line with no pairs
/// at all wouldn't be distinguishable from plain prose after this prefix) and because that's the same
/// extraction `crate::engine::auto_declare_params` needs anyway. Checked ahead of the generic logfmt
/// check in `detect_format` since it's the more specific of the two.
fn py_log_prefix(input: &str) -> IResult<&str, (&str, &str, &str, &str)> {
let (input, date) = take_while1(|c: char| c.is_ascii_digit() || c == '-')(input)?;
let (input, _) = char(' ')(input)?;
let (input, time) = take_while1(|c: char| c.is_ascii_digit() || c == ':' || c == ',')(input)?;
let (input, _) = char(' ')(input)?;
let (input, level) = take_while1(|c: char| c.is_ascii_uppercase())(input)?;
let (input, _) = char(' ')(input)?;
let (input, logger) =
take_while1(|c: char| c.is_ascii_alphanumeric() || c == '_' || c == '-' || c == '.')(input)?;
let (input, _) = char(' ')(input)?;
Ok((input, (date, time, level, logger)))
}
pub fn is_py_log_line(text: &str) -> bool {
match py_log_prefix(text) {
Ok((rest, _)) => !extract_pairs(rest).is_empty(),
Err(_) => false,
}
}
/// The value of the `ottyl_fmt` reserved param (`crate::engine::compile_filter`/`eval_alert_expr`):
/// which format a line was recognized as when parsed. `log stream`'s fixed-column shape is checked
/// first (see `LOG_STREAM_RE`'s doc comment for why it's the most specific of the four). JSON is
/// checked next since a JSON object can incidentally contain `key=value`-shaped substrings that
/// `extract_pairs` would otherwise pick up; only a top-level object counts (arrays/scalars aren't what
/// `crate::engine::eval_json_comparison` treats as JSON either). Anything else falls back to
/// logfmt-style pair extraction, and a line with none of the above is `"none"`.
/// which format a line was recognized as when parsed. Checked most-specific-first: `log stream`'s
/// fixed-column shape (see `LOG_STREAM_RE`'s doc comment for why it's the most specific), then the two
/// named-tool shapes `go-log` (a `time`/`level`/`msg`-prefixed JSON object) and `py-log` (a
/// timestamp/level/logger prefix over logfmt pairs) — each is a strict subset of the generic JSON/logfmt
/// case checked right after it, so they'd otherwise never be reached. JSON is checked next since a JSON
/// object can incidentally contain `key=value`-shaped substrings that `extract_pairs` would otherwise
/// pick up; only a top-level object counts (arrays/scalars aren't what `crate::engine::eval_json_comparison`
/// treats as JSON either). Anything else falls back to logfmt-style pair extraction, and a line with
/// none of the above is `"none"`.
pub fn detect_format(text: &str) -> &'static str {
if is_log_stream_line(text) {
"log_stream"
} else if is_go_log_line(text) {
"go-log"
} else if is_py_log_line(text) {
"py-log"
} else if serde_json::from_str::<serde_json::Value>(text).is_ok_and(|value| value.is_object()) {
"json"
} else if !extract_pairs(text).is_empty() {
@@ -250,4 +311,59 @@ mod tests {
// The embedded key=value pairs are still there for auto_declare_params to see.
assert!(!extract_pairs(line).is_empty());
}
const GO_LOG_FIXTURE: &str = include_str!("fixtures/go_log_sample.txt");
const PY_LOG_FIXTURE: &str = include_str!("fixtures/py_log_sample.txt");
#[test]
fn detects_go_log_shaped_lines() {
for line in GO_LOG_FIXTURE.lines() {
assert!(is_go_log_line(line), "expected go-log line to match: {line}");
assert_eq!(detect_format(line), "go-log");
}
}
#[test]
fn detects_py_log_shaped_lines() {
for line in PY_LOG_FIXTURE.lines() {
assert!(is_py_log_line(line), "expected py-log line to match: {line}");
assert_eq!(detect_format(line), "py-log");
}
}
#[test]
fn a_generic_json_object_is_not_detected_as_go_log() {
let line = r#"{"level":"error","code":500}"#;
assert!(!is_go_log_line(line));
assert_eq!(detect_format(line), "json");
}
#[test]
fn a_generic_logfmt_line_is_not_detected_as_py_log() {
let line = r#"ts=1788648105773 msg="entity_moved" x1=-4"#;
assert!(!is_py_log_line(line));
assert_eq!(detect_format(line), "logfmt");
}
#[test]
fn go_log_prefix_extracts_time_level_and_msg() {
let (rest, (time, level, msg)) =
go_log_prefix(r#"{"time":"2026-09-06T10:35:34.499921+01:00","level":"WARN","msg":"eventC","labelC":1}"#)
.unwrap();
assert_eq!(time, "2026-09-06T10:35:34.499921+01:00");
assert_eq!(level, "WARN");
assert_eq!(msg, "eventC");
assert_eq!(rest, r#","labelC":1}"#);
}
#[test]
fn py_log_prefix_extracts_date_time_level_and_logger() {
let (rest, (date, time, level, logger)) =
py_log_prefix("2026-09-06 10:35:55,217 INFO delete-py-log event=eventC labelC=1").unwrap();
assert_eq!(date, "2026-09-06");
assert_eq!(time, "10:35:55,217");
assert_eq!(level, "INFO");
assert_eq!(logger, "delete-py-log");
assert_eq!(rest, "event=eventC labelC=1");
}
}
File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 9.3 KiB

After

Width:  |  Height:  |  Size: 669 B

+26
View File
@@ -0,0 +1,26 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">
<defs>
<style>
.cls-1 {
fill: #fff;
stroke-width: 9px;
}
.cls-1, .cls-2 {
stroke: #000;
stroke-miterlimit: 10;
}
.cls-2 {
fill: none;
stroke-width: 8px;
stroke-linecap: round;
stroke-linejoin: round;
}
</style>
</defs>
<rect class="cls-1" x="10" y="10" width="80" height="80" rx="18.79" ry="18.79"/>
<polyline class="cls-2" points="25,33 42,50 25,67"/>
<line class="cls-2" x1="46" y1="66" x2="75" y2="37"/>
</svg>

After

Width:  |  Height:  |  Size: 669 B

+59
View File
@@ -150,6 +150,65 @@ const SECTIONS: DocSection[] = [
</div>
),
},
{
id: 'auto-formats',
title: 'Auto-detected formats',
body: (
<div className="flex flex-col gap-2">
<p>
Every ingested line is classified into one of four formats, exposed as the reserved{' '}
<Code>ottyl_fmt</Code> param (see <em>Filtering</em> above). The format also decides what — if anything —
gets auto-declared as a param without you doing anything:
</p>
<ul className="list-disc pl-5">
<li>
<span className="font-semibold text-text">
<Code>log_stream</Code>
</span>{' '}
— macOS <Code>{'log stream'}</Code> output's fixed-column shape (timestamp, thread, type, activity id,
pid, ttl, then <Code>process: message</Code>). The first such line seen auto-declares{' '}
<Code>log_type</Code>, <Code>process</Code>, and <Code>pid</Code> as params, parsed straight out of those
columns.
</li>
<li>
<span className="font-semibold text-text">
<Code>json</Code>
</span>{' '}
— the line parses as a JSON object (a bare array or scalar doesn't count). Every leaf scalar field
(string/number/bool) auto-declares a param named after its dot-separated path, extracted by JSON field
path rather than regex — a nested field like <Code>{'{"request":{"path":"/x"}}'}</Code> becomes a param
named <Code>request.path</Code>. Nested objects are walked into automatically; arrays and <Code>null</Code>{' '}
fields are skipped since there's no single scalar value to declare a param against.
</li>
<li>
<span className="font-semibold text-text">
<Code>logfmt</Code>
</span>{' '}
— the line contains at least one <Code>key=value</Code>/<Code>{'key="value"'}</Code> pair anywhere in its
text (it doesn't need to be logfmt end-to-end — a free-text prefix or suffix around the pairs is fine).
Every distinct key auto-declares a param of the same name, type-inferred as <Code>number</Code> if its
first-seen value parses as one, else <Code>string</Code>.
</li>
<li>
<span className="font-semibold text-text">
<Code>none</Code>
</span>{' '}
— none of the above; plain prose with no structure ottyl can key off of. Nothing is auto-declared, though
the line is still fully searchable via <Code>regex(...)</Code> in the filter bar.
</li>
</ul>
<p>
Auto-declared params are flagged <Code>auto_derived</Code> in <Code>GET /api/params</Code> and show up as
log-table columns immediately, same as a manually declared param — but they can't be edited or removed
individually, and they're never persisted to a profile (so they're rediscovered fresh each run rather than
traveling with a profile like manual declarations do). A manual param declaration for a name always takes
precedence and is never overwritten by auto-detection. To keep one unusual line from flooding the params
list, auto-detection is capped at 64 auto-derived params total and 8 new ones per line — once either cap is
hit, further would-be auto-declarations from new keys/fields are simply skipped.
</p>
</div>
),
},
{
id: 'parameters',
title: 'Parameters',
+5 -1
View File
@@ -6,6 +6,7 @@ import { useDocsNav, type DocsTarget } from '../lib/docsNav'
import { DocsPanel } from './Docs'
import { HelpIcon } from './HelpIcon'
import { BUILTIN_THEMES, downloadThemeFile, generateThemeCss, parseThemeCss } from '../lib/themes'
import ottylIcon from '../assets/ottyl-icon.svg'
type PanelName = 'settings' | 'docs' | null
@@ -67,7 +68,10 @@ export function NavMenu({ stats }: { stats: Stats | null }) {
<div className="flex h-full">
<div className="flex h-full w-48 shrink-0 flex-col justify-between border-r border-border p-3">
<div>
<div className="mb-4 text-lg font-semibold text-text">ottyl</div>
<div className="mb-4 flex items-center gap-2 text-lg font-semibold text-text">
<img src={ottylIcon} alt="" className="h-5 w-5" />
ottyl
</div>
<PresetsDropdown panel={panel} />
<nav className="mt-2 flex flex-col gap-1 text-sm text-text-muted">
<button