Auto derived format

src/engine.rs

@@ -152,6 +152,7 @@ pub async fn run(mut rx: mpsc::Receiver<EngineMessage>, settings: SettingsHandle
                     DataValue::LogLiteral { text: text.clone() },
                 );
                 log_count += 1;
+                auto_declare_params(&mut index, &text);
                 apply_all_metrics(&mut index, sequence, timestamp, &text);
                 evict_excess_logs(&mut index, &mut log_count, max_logs, &mut approx_bytes_total);
 
@@ -219,6 +220,16 @@ fn declare_param(
     index: &mut BTreeMap<IndexKey, DataValue>,
     request: DeclareParamRequest,
 ) -> Result<(), String> {
+    if let Some(DataValue::ParamDeclaration(existing)) =
+        index.get(&IndexKey::ParamDeclaration { name: request.name.clone() })
+    {
+        if existing.auto_derived {
+            return Err(format!(
+                "param '{}' was auto-detected from the log stream and can't be edited",
+                request.name
+            ));
+        }
+    }
     // An `elapsed_since_last` param is never text-extracted at all, so `message_match_regex` is
     // irrelevant to it — but the field always compiles as a regex regardless of extractor (an empty
     // pattern is valid and matches trivially), so it's harmless to still require it to parse.
@@ -243,11 +254,78 @@ fn declare_param(
 
     index.insert(
         IndexKey::ParamDeclaration { name: request.name.clone() },
-        DataValue::ParamDeclaration(request),
+        // `auto_derived` is never trusted from the request — only `auto_declare_params` may set it.
+        DataValue::ParamDeclaration(ParamDeclaration { auto_derived: false, ..request }),
     );
     Ok(())
 }
 
+/// Turns an arbitrary log-line key into a valid regex named-capture-group identifier (ASCII
+/// alphanumeric/underscore, not starting with a digit) — the param's own `name` (shown to the user,
+/// used as the index key) keeps the raw key text; only the capture group embedded in its generated
+/// regex needs sanitizing. Two different keys sanitizing to the same group name is harmless since
+/// each param's regex/capture-group pair is private to that one param.
+fn sanitize_group_name(key: &str) -> String {
+    let mut group: String =
+        key.chars().map(|c| if c.is_ascii_alphanumeric() || c == '_' { c } else { '_' }).collect();
+    if group.chars().next().is_none_or(|c| c.is_ascii_digit()) {
+        group.insert(0, '_');
+    }
+    group
+}
+
+/// Conservative kind inference for an auto-derived param's first sample: only `Number` is guessed
+/// (int, falling back to float), everything else stays `String` — `Boolean`/`Timestamp`/`Duration`
+/// carry extraction semantics beyond "parses as", so guessing them wrong would silently drop values
+/// that don't fit (see `parse_metric_primitive`).
+fn infer_kind(sample: &str) -> MetricPrimitiveKind {
+    if sample.parse::<i64>().is_ok() || sample.parse::<f64>().is_ok() {
+        MetricPrimitiveKind::Number
+    } else {
+        MetricPrimitiveKind::String
+    }
+}
+
+/// Builds the regex an auto-derived param uses to re-extract its value from any line on read
+/// (`derive_log_params`) — quoted keys always look for a `"..."` value, unquoted keys a
+/// whitespace-delimited bareword, matching how `crate::logfmt::extract_pairs` found the sample.
+fn build_auto_param_pattern(key: &str, group: &str, quoted: bool) -> String {
+    let escaped_key = regex::escape(key);
+    if quoted {
+        format!(r#"{escaped_key}="(?P<{group}>[^"]*)""#)
+    } else {
+        format!(r"{escaped_key}=(?P<{group}>\S+)")
+    }
+}
+
+/// Scans `text` for `key=value`/`key="value"` pairs (`crate::logfmt::extract_pairs`) and declares a
+/// standalone param for any key not already declared — auto or manual. A manual declaration for a
+/// name always wins and is never touched here; this only fills in names nobody has declared yet.
+/// Run on every ingested line (`EngineMessage::Ingest`), same as `apply_all_metrics`, but note param
+/// *values* are still recomputed from raw text at read time (`derive_log_params`), not stored here —
+/// so a key discovered on line N also back-fills the column for lines before N on the next read.
+fn auto_declare_params(index: &mut BTreeMap<IndexKey, DataValue>, text: &str) {
+    for (key, value, quoted) in crate::logfmt::extract_pairs(text) {
+        let index_key = IndexKey::ParamDeclaration { name: key.to_string() };
+        if index.contains_key(&index_key) {
+            continue;
+        }
+        let group = sanitize_group_name(key);
+        index.insert(
+            index_key,
+            DataValue::ParamDeclaration(ParamDeclaration {
+                name: key.to_string(),
+                message_match_regex: build_auto_param_pattern(key, &group, quoted),
+                capture_group: group,
+                json_field_path: None,
+                kind: infer_kind(value),
+                elapsed_since_last: false,
+                auto_derived: true,
+            }),
+        );
+    }
+}
+
 /// Validates the metric's own regex plus that every name in `label_params`/`value_param` refers to
 /// an already-declared standalone param (not just the first bad one, so a rejection reports every
 /// problem at once). Shared by `declare_metric` and `update_metric` so both reject a bad
@@ -407,7 +485,7 @@ struct ResolvedParam {
 
 fn resolved_param_declarations(index: &BTreeMap<IndexKey, DataValue>) -> HashMap<String, ResolvedParam> {
     param_declarations_range(index)
-        .filter_map(|(name, message_match_regex, capture_group, json_field_path, kind, elapsed_since_last)| {
+        .filter_map(|(name, message_match_regex, capture_group, json_field_path, kind, elapsed_since_last, _)| {
             Regex::new(message_match_regex).ok().map(|regex| {
                 (
                     name.to_string(),
@@ -956,19 +1034,20 @@ fn metric_declarations_range(
 
 fn param_declarations_range(
     index: &BTreeMap<IndexKey, DataValue>,
-) -> impl Iterator<Item = (&str, &str, &str, Option<&str>, MetricPrimitiveKind, bool)> {
+) -> impl Iterator<Item = (&str, &str, &str, Option<&str>, MetricPrimitiveKind, bool, bool)> {
     index
         .range(IndexKey::LogParamSplit..IndexKey::ParamMetricSplit)
         .filter_map(|(key, value)| match (key, value) {
             (
                 IndexKey::ParamDeclaration { name },
                 DataValue::ParamDeclaration(ParamDeclaration {
+                    name: _,
                     message_match_regex,
                     capture_group,
                     json_field_path,
                     kind,
                     elapsed_since_last,
-                    ..
+                    auto_derived,
                 }),
             ) => Some((
                 name.as_str(),
@@ -977,6 +1056,7 @@ fn param_declarations_range(
                 json_field_path.as_deref(),
                 *kind,
                 *elapsed_since_last,
+                *auto_derived,
             )),
             _ => None,
         })
@@ -991,8 +1071,8 @@ fn compiled_param_declarations(
     index: &BTreeMap<IndexKey, DataValue>,
 ) -> Vec<(String, Regex, String, Option<String>, MetricPrimitiveKind)> {
     param_declarations_range(index)
-        .filter(|(.., elapsed_since_last)| !elapsed_since_last)
-        .filter_map(|(name, message_match_regex, capture_group, json_field_path, kind, _)| {
+        .filter(|(_, _, _, _, _, elapsed_since_last, _)| !elapsed_since_last)
+        .filter_map(|(name, message_match_regex, capture_group, json_field_path, kind, _, _)| {
             Regex::new(message_match_regex).ok().map(|re| {
                 (
                     name.to_string(),
@@ -1198,7 +1278,7 @@ fn handle_query(
         }
         EngineQuery::ParamDeclarations => {
             let declarations = param_declarations_range(index)
-                .map(|(name, message_match_regex, capture_group, json_field_path, kind, elapsed_since_last)| {
+                .map(|(name, message_match_regex, capture_group, json_field_path, kind, elapsed_since_last, auto_derived)| {
                     ParamDeclarationDto {
                         name: name.to_string(),
                         message_match_regex: message_match_regex.to_string(),
@@ -1206,6 +1286,7 @@ fn handle_query(
                         json_field_path: json_field_path.map(|path| path.to_string()),
                         kind,
                         elapsed_since_last,
+                        auto_derived,
                     }
                 })
                 .collect();
@@ -1578,6 +1659,7 @@ mod tests {
             json_field_path: None,
             kind: MetricPrimitiveKind::String,
             elapsed_since_last: false,
+            auto_derived: false,
         }
     }
 
@@ -1594,6 +1676,51 @@ mod tests {
         assert_eq!(declarations[0].2, "level");
     }
 
+    #[test]
+    fn auto_declare_params_discovers_new_keys_with_inferred_kinds_and_flags_them_auto() {
+        let mut index = BTreeMap::new();
+        auto_declare_params(&mut index, r#"ts=1788648105773 msg="entity_moved" x1=-4 y1=34"#);
+
+        let declarations: HashMap<&str, _> =
+            param_declarations_range(&index).map(|d| (d.0, d)).collect();
+        assert_eq!(declarations.len(), 4);
+        assert_eq!(declarations["x1"].4, MetricPrimitiveKind::Number);
+        assert!(declarations["x1"].6, "expected x1 to be flagged auto_derived");
+        assert_eq!(declarations["msg"].4, MetricPrimitiveKind::String);
+
+        let compiled = compiled_param_declarations(&index);
+        let params = derive_log_params(r#"ts=1788648105773 msg="entity_moved" x1=-4 y1=34"#, &compiled);
+        assert_eq!(params.get("x1"), Some(&MetricPrimitive::Number(Number::Int(-4))));
+        assert_eq!(params.get("msg"), Some(&MetricPrimitive::String("entity_moved".to_string())));
+    }
+
+    #[test]
+    fn auto_declare_params_never_overwrites_an_existing_declaration() {
+        let mut index = BTreeMap::new();
+        declare_param(&mut index, request("x1", r#"x1="(?P<x1>[^"]+)""#, "x1")).unwrap();
+
+        auto_declare_params(&mut index, "x1=-4");
+
+        let declarations: Vec<_> = param_declarations_range(&index).collect();
+        assert_eq!(declarations.len(), 1);
+        assert_eq!(declarations[0].1, r#"x1="(?P<x1>[^"]+)""#, "manual declaration must survive untouched");
+        assert!(!declarations[0].6, "a manual declaration must never be flagged auto_derived");
+    }
+
+    #[test]
+    fn declare_param_rejects_editing_an_auto_derived_param() {
+        let mut index = BTreeMap::new();
+        auto_declare_params(&mut index, "x1=-4");
+
+        let result = declare_param(&mut index, request("x1", r#"x1="(?P<x1>[^"]+)""#, "x1"));
+
+        let err = result.unwrap_err();
+        assert!(err.contains("auto-detected"), "unexpected error message: {err}");
+        let declarations: Vec<_> = param_declarations_range(&index).collect();
+        assert_eq!(declarations.len(), 1);
+        assert!(declarations[0].6, "the original auto-derived declaration must be untouched");
+    }
+
     #[test]
     fn rejects_invalid_regex() {
         let mut index = BTreeMap::new();
@@ -1635,6 +1762,7 @@ mod tests {
                 json_field_path: None,
                 kind: MetricPrimitiveKind::String,
                 elapsed_since_last: true,
+                auto_derived: false,
             },
         );
         assert!(result.is_err());
@@ -1655,6 +1783,7 @@ mod tests {
                 json_field_path: Some("foo.bar".to_string()),
                 kind: MetricPrimitiveKind::String,
                 elapsed_since_last: false,
+                auto_derived: false,
             },
         );
         assert!(result.is_ok());
@@ -1676,6 +1805,7 @@ mod tests {
                 json_field_path: Some("foo.bar".to_string()),
                 kind: MetricPrimitiveKind::Number,
                 elapsed_since_last: false,
+                auto_derived: false,
             },
         )
         .unwrap();
@@ -1792,6 +1922,7 @@ mod tests {
                 json_field_path: None,
                 kind,
                 elapsed_since_last: false,
+                auto_derived: false,
             },
         )
         .unwrap();
@@ -1811,6 +1942,7 @@ mod tests {
                 json_field_path: None,
                 kind: MetricPrimitiveKind::Duration,
                 elapsed_since_last: true,
+                auto_derived: false,
             },
         )
         .unwrap();
@@ -2032,6 +2164,7 @@ mod tests {
                 json_field_path: Some("request.path".to_string()),
                 kind: MetricPrimitiveKind::String,
                 elapsed_since_last: false,
+                auto_derived: false,
             },
         )
         .unwrap();