diff --git a/src/data_value.rs b/src/data_value.rs index 62dab2f..cd5c89c 100644 --- a/src/data_value.rs +++ b/src/data_value.rs @@ -67,6 +67,40 @@ pub fn extract_json_field(text: &str, field_path: &str) -> Option { } } +/// Flattens a JSON object into dot-separated field paths mapped to each leaf scalar's raw text form +/// (string/number/bool), for [`crate::engine::auto_declare_params`] when a line is JSON +/// (`crate::logfmt::detect_format`) rather than logfmt — the JSON equivalent of +/// `crate::logfmt::extract_pairs`'s `key=value` scan. Nested objects recurse, dot-joining the path +/// (`"foo": {"bar": 1}` yields `"foo.bar"`); arrays and null are skipped since there's no single leaf +/// scalar to declare a param against, matching `extract_json_field`'s own restriction to scalars. +/// Returns an empty vec for anything that isn't a JSON object, same as `extract_pairs` returns empty +/// for a line with no `key=value` pairs. +pub fn flatten_json_object(text: &str) -> Vec<(String, String)> { + let Ok(serde_json::Value::Object(root)) = serde_json::from_str::(text) else { + return Vec::new(); + }; + let mut results = Vec::new(); + flatten_json_object_into(&root, "", &mut results); + results +} + +fn flatten_json_object_into( + object: &serde_json::Map, + prefix: &str, + results: &mut Vec<(String, String)>, +) { + for (key, value) in object { + let path = if prefix.is_empty() { key.clone() } else { format!("{prefix}.{key}") }; + match value { + serde_json::Value::String(s) => results.push((path, s.clone())), + serde_json::Value::Number(n) => results.push((path, n.to_string())), + serde_json::Value::Bool(b) => results.push((path, b.to_string())), + serde_json::Value::Object(nested) => flatten_json_object_into(nested, &path, results), + serde_json::Value::Null | serde_json::Value::Array(_) => {} + } + } +} + /// `rename_all = "lowercase"` matches other `kind`-style enums, so JSON bodies use "string"/"number"/"timestamp". #[allow(dead_code)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] diff --git a/src/engine.rs b/src/engine.rs index 003b7f1..8992f1d 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -356,37 +356,79 @@ fn auto_derived_param_count(index: &BTreeMap) -> usize { param_declarations_range(index).filter(|declaration| declaration.6).count() } -/// Scans `text` for `key=value`/`key="value"` pairs (`crate::logfmt::extract_pairs`) and declares a -/// standalone param for any key not already declared — auto or manual. A manual declaration for a -/// name always wins and is never touched here; this only fills in names nobody has declared yet. -/// Run on every ingested line (`EngineMessage::Ingest`), same as `apply_all_metrics`, but note param -/// *values* are still recomputed from raw text at read time (`derive_log_params`), not stored here — -/// so a key discovered on line N also back-fills the column for lines before N on the next read. -/// Bounded by `MAX_AUTO_DECLARED_PARAMS`/`MAX_NEW_AUTO_PARAMS_PER_LINE` (see their doc comments) so a -/// single burst of unusual lines can't permanently inflate the declared-param set (ticket 050). +/// One param worth auto-declaring, found on an ingested line by either extraction path +/// `auto_declare_params` supports: a logfmt `key=value` pair (name, and a regex/capture-group built +/// from it) or a JSON leaf field (name being its dot-separated path, extracted via `json_field_path` +/// instead of a regex). Lets `auto_declare_params` share one insert-and-budget loop across both. +struct AutoParamCandidate { + name: String, + value: String, + message_match_regex: String, + capture_group: String, + json_field_path: Option, +} + +/// Scans `text` for auto-derivable params and declares a standalone param for any name not already +/// declared — auto or manual. A manual declaration for a name always wins and is never touched here; +/// this only fills in names nobody has declared yet. JSON lines (`crate::logfmt::detect_format`) are +/// scanned via `crate::data_value::flatten_json_object` (dot-path leaf fields, extracted by +/// `json_field_path` rather than regex, since a `key=value` scan finds nothing useful in JSON's +/// `"key":value` shape); anything else falls back to the original `key=value` pair scan +/// (`crate::logfmt::extract_pairs`). Run on every ingested line (`EngineMessage::Ingest`), same as +/// `apply_all_metrics`, but note param *values* are still recomputed from raw text at read time +/// (`derive_log_params`), not stored here — so a key discovered on line N also back-fills the column +/// for lines before N on the next read. Bounded by `MAX_AUTO_DECLARED_PARAMS`/ +/// `MAX_NEW_AUTO_PARAMS_PER_LINE` (see their doc comments) so a single burst of unusual lines can't +/// permanently inflate the declared-param set (ticket 050). fn auto_declare_params(index: &mut BTreeMap, text: &str) { let mut total = auto_derived_param_count(index); if total >= MAX_AUTO_DECLARED_PARAMS { return; } + let candidates: Vec = if crate::logfmt::detect_format(text) == "json" { + crate::data_value::flatten_json_object(text) + .into_iter() + .map(|(path, value)| AutoParamCandidate { + name: path.clone(), + value, + message_match_regex: String::new(), + capture_group: String::new(), + json_field_path: Some(path), + }) + .collect() + } else { + crate::logfmt::extract_pairs(text) + .into_iter() + .map(|(key, value, quoted)| { + let group = sanitize_group_name(key); + AutoParamCandidate { + name: key.to_string(), + value: value.to_string(), + message_match_regex: build_auto_param_pattern(key, &group, quoted), + capture_group: group, + json_field_path: None, + } + }) + .collect() + }; + let mut new_this_line = 0; - for (key, value, quoted) in crate::logfmt::extract_pairs(text) { + for candidate in candidates { if new_this_line >= MAX_NEW_AUTO_PARAMS_PER_LINE || total >= MAX_AUTO_DECLARED_PARAMS { break; } - let index_key = IndexKey::ParamDeclaration { name: key.to_string() }; + let index_key = IndexKey::ParamDeclaration { name: candidate.name.clone() }; if index.contains_key(&index_key) { continue; } - let group = sanitize_group_name(key); index.insert( index_key, DataValue::ParamDeclaration(ParamDeclaration { - name: key.to_string(), - message_match_regex: build_auto_param_pattern(key, &group, quoted), - capture_group: group, - json_field_path: None, - kind: infer_kind(value), + name: candidate.name, + message_match_regex: candidate.message_match_regex, + capture_group: candidate.capture_group, + json_field_path: candidate.json_field_path, + kind: infer_kind(&candidate.value), elapsed_since_last: false, auto_derived: true, }), @@ -2012,6 +2054,27 @@ mod tests { assert_eq!(params.get("msg"), Some(&MetricPrimitive::String("entity_moved".to_string()))); } + #[test] + fn auto_declare_params_discovers_json_leaf_fields_via_json_field_path() { + let mut index = BTreeMap::new(); + auto_declare_params(&mut index, r#"{"level":"error","code":500,"request":{"path":"/x"}}"#); + + let declarations: HashMap<&str, _> = + param_declarations_range(&index).map(|d| (d.0, d)).collect(); + assert_eq!(declarations.len(), 3); + assert_eq!(declarations["code"].4, MetricPrimitiveKind::Number); + assert_eq!(declarations["code"].3, Some("code")); + assert!(declarations["level"].6, "expected level to be flagged auto_derived"); + assert_eq!(declarations["request.path"].3, Some("request.path")); + + let compiled = compiled_param_declarations(&index); + let params = + derive_log_params(r#"{"level":"error","code":500,"request":{"path":"/x"}}"#, &compiled); + assert_eq!(params.get("level"), Some(&MetricPrimitive::String("error".to_string()))); + assert_eq!(params.get("code"), Some(&MetricPrimitive::Number(Number::Int(500)))); + assert_eq!(params.get("request.path"), Some(&MetricPrimitive::String("/x".to_string()))); + } + #[test] fn auto_declare_params_never_overwrites_an_existing_declaration() { let mut index = BTreeMap::new(); diff --git a/webapp/public/favicon.svg b/webapp/public/favicon.svg index 6893eb1..13e87d2 100644 --- a/webapp/public/favicon.svg +++ b/webapp/public/favicon.svg @@ -1 +1,26 @@ - \ No newline at end of file + + + + + + + + + diff --git a/ottyl-icon.svg b/webapp/src/assets/ottyl-icon.svg similarity index 100% rename from ottyl-icon.svg rename to webapp/src/assets/ottyl-icon.svg diff --git a/webapp/src/components/Docs.tsx b/webapp/src/components/Docs.tsx index cf1fd95..41fbdc0 100644 --- a/webapp/src/components/Docs.tsx +++ b/webapp/src/components/Docs.tsx @@ -150,6 +150,65 @@ const SECTIONS: DocSection[] = [ ), }, + { + id: 'auto-formats', + title: 'Auto-detected formats', + body: ( +
+

+ Every ingested line is classified into one of four formats, exposed as the reserved{' '} + ottyl_fmt param (see Filtering above). The format also decides what — if anything — + gets auto-declared as a param without you doing anything: +

+
    +
  • + + log_stream + {' '} + — macOS {'log stream'} output's fixed-column shape (timestamp, thread, type, activity id, + pid, ttl, then process: message). The first such line seen auto-declares{' '} + log_type, process, and pid as params, parsed straight out of those + columns. +
    • +
  • + + json + {' '} + — the line parses as a JSON object (a bare array or scalar doesn't count). Every leaf scalar field + (string/number/bool) auto-declares a param named after its dot-separated path, extracted by JSON field + path rather than regex — a nested field like {'{"request":{"path":"/x"}}'} becomes a param + named request.path. Nested objects are walked into automatically; arrays and null{' '} + fields are skipped since there's no single scalar value to declare a param against. +
    • +
  • + + logfmt + {' '} + — the line contains at least one key=value/{'key="value"'} pair anywhere in its + text (it doesn't need to be logfmt end-to-end — a free-text prefix or suffix around the pairs is fine). + Every distinct key auto-declares a param of the same name, type-inferred as number if its + first-seen value parses as one, else string. +
    • +
  • + + none + {' '} + — none of the above; plain prose with no structure ottyl can key off of. Nothing is auto-declared, though + the line is still fully searchable via regex(...) in the filter bar. +
    • +
+

+ Auto-declared params are flagged auto_derived in GET /api/params and show up as + log-table columns immediately, same as a manually declared param — but they can't be edited or removed + individually, and they're never persisted to a profile (so they're rediscovered fresh each run rather than + traveling with a profile like manual declarations do). A manual param declaration for a name always takes + precedence and is never overwritten by auto-detection. To keep one unusual line from flooding the params + list, auto-detection is capped at 64 auto-derived params total and 8 new ones per line — once either cap is + hit, further would-be auto-declarations from new keys/fields are simply skipped. +

+
+ ), + }, { id: 'parameters', title: 'Parameters', diff --git a/webapp/src/components/NavMenu.tsx b/webapp/src/components/NavMenu.tsx index c7a829c..64d4ae0 100644 --- a/webapp/src/components/NavMenu.tsx +++ b/webapp/src/components/NavMenu.tsx @@ -6,6 +6,7 @@ import { useDocsNav, type DocsTarget } from '../lib/docsNav' import { DocsPanel } from './Docs' import { HelpIcon } from './HelpIcon' import { BUILTIN_THEMES, downloadThemeFile, generateThemeCss, parseThemeCss } from '../lib/themes' +import ottylIcon from '../assets/ottyl-icon.svg' type PanelName = 'settings' | 'docs' | null @@ -67,7 +68,10 @@ export function NavMenu({ stats }: { stats: Stats | null }) {
-
ottyl
+
+ + ottyl +