Add JSON field path extraction for params/metrics (ticket 015)

RegexToMetricPrimitiveMatcher and the standalone param declaration
(DeclareParamRequest/DataValue::ParamDeclaration) gain an optional
json_field_path, an alternative to capture_group that reads a
dot-separated field path (e.g. "foo.bar") out of lines that parse as
JSON objects. It's additive and serde-defaulted, so existing
regex-based params/metrics are unaffected. Params.tsx now offers a
"JSON field" extraction option when the message match regex looks
like a JSON object.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XFUNZwCvz3kcod3352ZR2o

src/api.rs

@@ -223,6 +223,7 @@ async fn declare_param(
                 name: request.name,
                 message_match_regex: request.message_match_regex,
                 capture_group: request.capture_group,
+                json_field_path: request.json_field_path,
                 kind: request.kind,
             };
             (StatusCode::CREATED, Json(dto)).into_response()

src/data_value.rs

@@ -8,12 +8,44 @@ use std::collections::BTreeMap;
 pub struct RegexToMetricPrimitiveMatcher {
     pub param_name: String,
     pub capture_group: String,
+    /// Alternative extraction source to `capture_group`: a dot-separated JSON field path (e.g.
+    /// `foo.bar`), read from lines that parse as a JSON object instead of via the regex capture
+    /// group. When present and non-empty, this takes precedence over `capture_group`.
+    /// `#[serde(default)]` so profile files/requests saved before this field existed still load
+    /// as plain regex-capture-group matchers.
+    #[serde(default)]
+    pub json_field_path: Option<String>,
     pub kind: MetricPrimitiveKind,
     /// Whether this param is a bucket label or its captured value (at most one Value-role param per metric); `#[serde(default)]` so profile files saved before this field existed still load as `Label`.
     #[serde(default)]
     pub role: ParamRole,
 }
 
+/// Reads a value out of a JSON object by dot-separated field path (e.g. `"foo.bar"`), for
+/// [`RegexToMetricPrimitiveMatcher::json_field_path`] and the sibling `json_field_path` on a
+/// standalone param declaration. Returns `None` when `text` doesn't parse as a JSON object, the
+/// path doesn't resolve to a value, or the resolved value isn't a scalar (string/number/bool) —
+/// there's no sensible flat text form of a nested object/array to feed into
+/// `parse_metric_primitive`.
+pub fn extract_json_field(text: &str, field_path: &str) -> Option<String> {
+    let root: serde_json::Value = serde_json::from_str(text).ok()?;
+    let mut current = root.as_object()?;
+    let mut segments = field_path.split('.').peekable();
+    loop {
+        let segment = segments.next()?;
+        let next_value = current.get(segment)?;
+        if segments.peek().is_none() {
+            return match next_value {
+                serde_json::Value::String(s) => Some(s.clone()),
+                serde_json::Value::Number(n) => Some(n.to_string()),
+                serde_json::Value::Bool(b) => Some(b.to_string()),
+                _ => None,
+            };
+        }
+        current = next_value.as_object()?;
+    }
+}
+
 /// See [`RegexToMetricPrimitiveMatcher::role`].
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
 #[serde(rename_all = "lowercase")]
@@ -45,6 +77,8 @@ pub enum DataValue {
     ParamDeclaration {
         message_match_regex: String,
         capture_group: String,
+        /// See `RegexToMetricPrimitiveMatcher::json_field_path`.
+        json_field_path: Option<String>,
         kind: MetricPrimitiveKind,
     },
     /// Paired with `IndexKey::MetricDeclaration`.

src/dto.rs

@@ -20,6 +20,8 @@ pub struct ParamDeclarationDto {
     pub name: String,
     pub message_match_regex: String,
     pub capture_group: String,
+    /// See `crate::data_value::RegexToMetricPrimitiveMatcher::json_field_path`.
+    pub json_field_path: Option<String>,
     pub kind: MetricPrimitiveKind,
 }
 

src/engine.rs

@@ -38,6 +38,10 @@ pub struct DeclareParamRequest {
     pub name: String,
     pub message_match_regex: String,
     pub capture_group: String,
+    /// See `RegexToMetricPrimitiveMatcher::json_field_path`; `#[serde(default)]` so existing
+    /// regex-only callers are unaffected.
+    #[serde(default)]
+    pub json_field_path: Option<String>,
     pub kind: MetricPrimitiveKind,
 }
 
@@ -207,7 +211,8 @@ fn declare_param(
 ) -> Result<(), String> {
     let re = Regex::new(&request.message_match_regex)
         .map_err(|err| format!("invalid regex: {err}"))?;
-    if !re.capture_names().flatten().any(|name| name == request.capture_group) {
+    let uses_json_field = request.json_field_path.as_deref().is_some_and(|path| !path.is_empty());
+    if !uses_json_field && !re.capture_names().flatten().any(|name| name == request.capture_group) {
         return Err(format!(
             "regex has no capture group named '{}'",
             request.capture_group
@@ -219,6 +224,7 @@ fn declare_param(
         DataValue::ParamDeclaration {
             message_match_regex: request.message_match_regex,
             capture_group: request.capture_group,
+            json_field_path: request.json_field_path,
             kind: request.kind,
         },
     );
@@ -237,7 +243,9 @@ fn declare_metric(
         .params
         .iter()
         .filter(|matcher| {
-            !re.capture_names().flatten().any(|name| name == matcher.capture_group)
+            let uses_json_field =
+                matcher.json_field_path.as_deref().is_some_and(|path| !path.is_empty());
+            !uses_json_field && !re.capture_names().flatten().any(|name| name == matcher.capture_group)
         })
         .map(|matcher| {
             format!(
@@ -324,14 +332,21 @@ fn apply_metric_to_log(
     let mut value = None;
     let mut is_duration = false;
     for matcher in params {
-        let Some(raw) = captures.name(&matcher.capture_group) else {
-            return;
+        let raw: String = match matcher.json_field_path.as_deref().filter(|path| !path.is_empty()) {
+            Some(field_path) => match crate::data_value::extract_json_field(text, field_path) {
+                Some(value) => value,
+                None => return,
+            },
+            None => match captures.name(&matcher.capture_group) {
+                Some(m) => m.as_str().to_string(),
+                None => return,
+            },
         };
         if matcher.role == ParamRole::Value && matcher.kind == MetricPrimitiveKind::Duration {
             is_duration = true;
             continue;
         }
-        let Some(parsed) = parse_metric_primitive(raw.as_str(), matcher.kind) else {
+        let Some(parsed) = parse_metric_primitive(&raw, matcher.kind) else {
             return;
         };
         match matcher.role {
@@ -682,14 +697,20 @@ fn metric_declarations_range(
 
 fn param_declarations_range(
     index: &BTreeMap<IndexKey, DataValue>,
-) -> impl Iterator<Item = (&str, &str, &str, MetricPrimitiveKind)> {
+) -> impl Iterator<Item = (&str, &str, &str, Option<&str>, MetricPrimitiveKind)> {
     index
         .range(IndexKey::LogParamSplit..IndexKey::ParamMetricSplit)
         .filter_map(|(key, value)| match (key, value) {
             (
                 IndexKey::ParamDeclaration { name },
-                DataValue::ParamDeclaration { message_match_regex, capture_group, kind },
-            ) => Some((name.as_str(), message_match_regex.as_str(), capture_group.as_str(), *kind)),
+                DataValue::ParamDeclaration { message_match_regex, capture_group, json_field_path, kind },
+            ) => Some((
+                name.as_str(),
+                message_match_regex.as_str(),
+                capture_group.as_str(),
+                json_field_path.as_deref(),
+                *kind,
+            )),
             _ => None,
         })
 }
@@ -697,30 +718,44 @@ fn param_declarations_range(
 /// A param whose stored regex fails to compile (shouldn't happen — `declare_param` validates it) is skipped rather than panicking.
 fn compiled_param_declarations(
     index: &BTreeMap<IndexKey, DataValue>,
-) -> Vec<(String, Regex, String, MetricPrimitiveKind)> {
+) -> Vec<(String, Regex, String, Option<String>, MetricPrimitiveKind)> {
     param_declarations_range(index)
-        .filter_map(|(name, message_match_regex, capture_group, kind)| {
-            Regex::new(message_match_regex)
-                .ok()
-                .map(|re| (name.to_string(), re, capture_group.to_string(), kind))
+        .filter_map(|(name, message_match_regex, capture_group, json_field_path, kind)| {
+            Regex::new(message_match_regex).ok().map(|re| {
+                (
+                    name.to_string(),
+                    re,
+                    capture_group.to_string(),
+                    json_field_path.map(|path| path.to_string()),
+                    kind,
+                )
+            })
         })
         .collect()
 }
 
-/// A param is omitted (not included as null/empty) when its regex/capture group doesn't match or the captured text fails to parse — see `LogEntryDto::params`.
+/// A param is omitted (not included as null/empty) when its regex/capture group (or JSON field
+/// path) doesn't match, or the extracted text fails to parse — see `LogEntryDto::params`.
 fn derive_log_params(
     text: &str,
-    declarations: &[(String, Regex, String, MetricPrimitiveKind)],
+    declarations: &[(String, Regex, String, Option<String>, MetricPrimitiveKind)],
 ) -> BTreeMap<String, MetricPrimitive> {
     let mut params = BTreeMap::new();
-    for (name, regex, capture_group, kind) in declarations {
+    for (name, regex, capture_group, json_field_path, kind) in declarations {
         let Some(captures) = regex.captures(text) else {
             continue;
         };
-        let Some(raw) = captures.name(capture_group) else {
-            continue;
+        let raw: String = match json_field_path.as_deref().filter(|path| !path.is_empty()) {
+            Some(field_path) => match crate::data_value::extract_json_field(text, field_path) {
+                Some(value) => value,
+                None => continue,
+            },
+            None => match captures.name(capture_group) {
+                Some(m) => m.as_str().to_string(),
+                None => continue,
+            },
         };
-        let Some(value) = parse_metric_primitive(raw.as_str(), *kind) else {
+        let Some(value) = parse_metric_primitive(&raw, *kind) else {
             continue;
         };
         params.insert(name.clone(), value);
@@ -866,10 +901,11 @@ fn handle_query(
         }
         EngineQuery::ParamDeclarations => {
             let declarations = param_declarations_range(index)
-                .map(|(name, message_match_regex, capture_group, kind)| ParamDeclarationDto {
+                .map(|(name, message_match_regex, capture_group, json_field_path, kind)| ParamDeclarationDto {
                     name: name.to_string(),
                     message_match_regex: message_match_regex.to_string(),
                     capture_group: capture_group.to_string(),
+                    json_field_path: json_field_path.map(|path| path.to_string()),
                     kind,
                 })
                 .collect();
@@ -1066,6 +1102,7 @@ mod tests {
             name: name.to_string(),
             message_match_regex: pattern.to_string(),
             capture_group: capture_group.to_string(),
+            json_field_path: None,
             kind: MetricPrimitiveKind::String,
         }
     }
@@ -1102,6 +1139,52 @@ mod tests {
         assert!(index.is_empty());
     }
 
+    #[test]
+    fn declares_valid_param_from_json_field_path() {
+        let mut index = BTreeMap::new();
+        let result = declare_param(
+            &mut index,
+            DeclareParamRequest {
+                name: "level".to_string(),
+                // No named capture group needed for JSON-field extraction, just something that
+                // matches the lines this param applies to.
+                message_match_regex: r".".to_string(),
+                capture_group: String::new(),
+                json_field_path: Some("foo.bar".to_string()),
+                kind: MetricPrimitiveKind::String,
+            },
+        );
+        assert!(result.is_ok());
+
+        let declarations: Vec<_> = param_declarations_range(&index).collect();
+        assert_eq!(declarations.len(), 1);
+        assert_eq!(declarations[0].3, Some("foo.bar"));
+    }
+
+    #[test]
+    fn derives_log_param_from_json_field_path() {
+        let mut index = BTreeMap::new();
+        declare_param(
+            &mut index,
+            DeclareParamRequest {
+                name: "bar".to_string(),
+                message_match_regex: r".".to_string(),
+                capture_group: String::new(),
+                json_field_path: Some("foo.bar".to_string()),
+                kind: MetricPrimitiveKind::Number,
+            },
+        )
+        .unwrap();
+
+        let declarations = compiled_param_declarations(&index);
+        let params = derive_log_params(r#"{"foo":{"bar":42}}"#, &declarations);
+        assert_eq!(params.get("bar"), Some(&MetricPrimitive::Number(Number::Int(42))));
+
+        // A line that doesn't parse as JSON, or is missing the field, yields no param.
+        assert!(derive_log_params("not json", &declarations).is_empty());
+        assert!(derive_log_params(r#"{"foo":{}}"#, &declarations).is_empty());
+    }
+
     #[test]
     fn evict_excess_logs_drops_oldest_sequences_first() {
         let mut index = BTreeMap::new();
@@ -1215,6 +1298,7 @@ mod tests {
                 .map(|(param_name, capture_group, kind, role)| RegexToMetricPrimitiveMatcher {
                     param_name: param_name.to_string(),
                     capture_group: capture_group.to_string(),
+                    json_field_path: None,
                     kind,
                     role,
                 })
@@ -1349,6 +1433,41 @@ mod tests {
         assert_eq!(values.len(), 2);
     }
 
+    #[test]
+    fn metric_param_extracts_from_json_field_path() {
+        let mut index = BTreeMap::new();
+        ingest_log(&mut index, 0, r#"{"request":{"path":"/a"}}"#);
+        // Doesn't parse as JSON at all — should be skipped, not panic.
+        ingest_log(&mut index, 1, "not json");
+        // Parses as JSON but is missing the field — also skipped.
+        ingest_log(&mut index, 2, r#"{"request":{}}"#);
+
+        declare_metric(
+            &mut index,
+            DeclareMetricRequest {
+                name: "requests".to_string(),
+                message_match_regex: r".".to_string(),
+                matched_labels: None,
+                unmatched_labels: None,
+                params: vec![RegexToMetricPrimitiveMatcher {
+                    param_name: "path".to_string(),
+                    capture_group: String::new(),
+                    json_field_path: Some("request.path".to_string()),
+                    kind: MetricPrimitiveKind::String,
+                    role: ParamRole::Label,
+                }],
+            },
+        )
+        .unwrap();
+
+        let values: Vec<_> = bucket_values(&index, "requests", None).collect();
+        assert_eq!(values.len(), 1);
+        assert_eq!(
+            values[0].0.get("path"),
+            Some(&MetricPrimitive::String("/a".to_string()))
+        );
+    }
+
     #[test]
     fn ingesting_after_declaration_updates_buckets_live() {
         let mut index = BTreeMap::new();

src/profile.rs

@@ -60,6 +60,10 @@ pub struct ParamDeclarationRecord {
     pub name: String,
     pub message_match_regex: String,
     pub capture_group: String,
+    /// See `crate::data_value::RegexToMetricPrimitiveMatcher::json_field_path`; `#[serde(default)]`
+    /// so profiles saved before this field existed still load as plain regex-capture-group params.
+    #[serde(default)]
+    pub json_field_path: Option<String>,
     pub kind: ProfileMetricPrimitiveKind,
 }
 
@@ -69,6 +73,7 @@ impl From<&DeclareParamRequest> for ParamDeclarationRecord {
             name: request.name.clone(),
             message_match_regex: request.message_match_regex.clone(),
             capture_group: request.capture_group.clone(),
+            json_field_path: request.json_field_path.clone(),
             kind: request.kind.into(),
         }
     }
@@ -80,6 +85,7 @@ impl From<&ParamDeclarationRecord> for DeclareParamRequest {
             name: record.name.clone(),
             message_match_regex: record.message_match_regex.clone(),
             capture_group: record.capture_group.clone(),
+            json_field_path: record.json_field_path.clone(),
             kind: record.kind.into(),
         }
     }
@@ -90,6 +96,10 @@ impl From<&ParamDeclarationRecord> for DeclareParamRequest {
 pub struct MetricParamRecord {
     pub param_name: String,
     pub capture_group: String,
+    /// See `crate::data_value::RegexToMetricPrimitiveMatcher::json_field_path`; `#[serde(default)]`
+    /// so profiles saved before this field existed still load as plain regex-capture-group matchers.
+    #[serde(default)]
+    pub json_field_path: Option<String>,
     pub kind: ProfileMetricPrimitiveKind,
     /// See `ParamRole`; `#[serde(default)]` so profiles saved before this field existed still load, defaulting to `Label`.
     #[serde(default)]
@@ -121,6 +131,7 @@ impl From<&DeclareMetricRequest> for MetricDeclarationRecord {
                 .map(|matcher| MetricParamRecord {
                     param_name: matcher.param_name.clone(),
                     capture_group: matcher.capture_group.clone(),
+                    json_field_path: matcher.json_field_path.clone(),
                     kind: matcher.kind.into(),
                     role: matcher.role,
                 })
@@ -144,6 +155,7 @@ impl From<&MetricDeclarationRecord> for DeclareMetricRequest {
                 .map(|param| RegexToMetricPrimitiveMatcher {
                     param_name: param.param_name.clone(),
                     capture_group: param.capture_group.clone(),
+                    json_field_path: param.json_field_path.clone(),
                     kind: param.kind.into(),
                     role: param.role,
                 })
@@ -626,6 +638,7 @@ mod tests {
             name: name.to_string(),
             message_match_regex: pattern.to_string(),
             capture_group: capture_group.to_string(),
+            json_field_path: None,
             kind: MetricPrimitiveKind::String,
         }
     }
@@ -639,6 +652,7 @@ mod tests {
             params: vec![RegexToMetricPrimitiveMatcher {
                 param_name: param_name.to_string(),
                 capture_group: capture_group.to_string(),
+                json_field_path: None,
                 kind: MetricPrimitiveKind::Number,
                 role: ParamRole::Label,
             }],