Add JSON field path extraction for params/metrics (ticket 015)

RegexToMetricPrimitiveMatcher and the standalone param declaration
(DeclareParamRequest/DataValue::ParamDeclaration) gain an optional
json_field_path, an alternative to capture_group that reads a
dot-separated field path (e.g. "foo.bar") out of lines that parse as
JSON objects. It's additive and serde-defaulted, so existing
regex-based params/metrics are unaffected. Params.tsx now offers a
"JSON field" extraction option when the message match regex looks
like a JSON object.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XFUNZwCvz3kcod3352ZR2o
This commit is contained in:
Hugh Kaznowski
2026-09-05 19:38:39 +01:00
co-authored by Claude Sonnet 5
parent e6e5bc8f50
commit b3e0272544
8 changed files with 287 additions and 54 deletions
+1
View File
@@ -223,6 +223,7 @@ async fn declare_param(
name: request.name,
message_match_regex: request.message_match_regex,
capture_group: request.capture_group,
json_field_path: request.json_field_path,
kind: request.kind,
};
(StatusCode::CREATED, Json(dto)).into_response()
+34
View File
@@ -8,12 +8,44 @@ use std::collections::BTreeMap;
pub struct RegexToMetricPrimitiveMatcher {
pub param_name: String,
pub capture_group: String,
/// Alternative extraction source to `capture_group`: a dot-separated JSON field path (e.g.
/// `foo.bar`), read from lines that parse as a JSON object instead of via the regex capture
/// group. When present and non-empty, this takes precedence over `capture_group`.
/// `#[serde(default)]` so profile files/requests saved before this field existed still load
/// as plain regex-capture-group matchers.
#[serde(default)]
pub json_field_path: Option<String>,
pub kind: MetricPrimitiveKind,
/// Whether this param is a bucket label or its captured value (at most one Value-role param per metric); `#[serde(default)]` so profile files saved before this field existed still load as `Label`.
#[serde(default)]
pub role: ParamRole,
}
/// Reads a value out of a JSON object by dot-separated field path (e.g. `"foo.bar"`), for
/// [`RegexToMetricPrimitiveMatcher::json_field_path`] and the sibling `json_field_path` on a
/// standalone param declaration. Returns `None` when `text` doesn't parse as a JSON object, the
/// path doesn't resolve to a value, or the resolved value isn't a scalar (string/number/bool) —
/// there's no sensible flat text form of a nested object/array to feed into
/// `parse_metric_primitive`.
pub fn extract_json_field(text: &str, field_path: &str) -> Option<String> {
let root: serde_json::Value = serde_json::from_str(text).ok()?;
let mut current = root.as_object()?;
let mut segments = field_path.split('.').peekable();
loop {
let segment = segments.next()?;
let next_value = current.get(segment)?;
if segments.peek().is_none() {
return match next_value {
serde_json::Value::String(s) => Some(s.clone()),
serde_json::Value::Number(n) => Some(n.to_string()),
serde_json::Value::Bool(b) => Some(b.to_string()),
_ => None,
};
}
current = next_value.as_object()?;
}
}
/// See [`RegexToMetricPrimitiveMatcher::role`].
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "lowercase")]
@@ -45,6 +77,8 @@ pub enum DataValue {
ParamDeclaration {
message_match_regex: String,
capture_group: String,
/// See `RegexToMetricPrimitiveMatcher::json_field_path`.
json_field_path: Option<String>,
kind: MetricPrimitiveKind,
},
/// Paired with `IndexKey::MetricDeclaration`.
+2
View File
@@ -20,6 +20,8 @@ pub struct ParamDeclarationDto {
pub name: String,
pub message_match_regex: String,
pub capture_group: String,
/// See `crate::data_value::RegexToMetricPrimitiveMatcher::json_field_path`.
pub json_field_path: Option<String>,
pub kind: MetricPrimitiveKind,
}
+139 -20
View File
@@ -38,6 +38,10 @@ pub struct DeclareParamRequest {
pub name: String,
pub message_match_regex: String,
pub capture_group: String,
/// See `RegexToMetricPrimitiveMatcher::json_field_path`; `#[serde(default)]` so existing
/// regex-only callers are unaffected.
#[serde(default)]
pub json_field_path: Option<String>,
pub kind: MetricPrimitiveKind,
}
@@ -207,7 +211,8 @@ fn declare_param(
) -> Result<(), String> {
let re = Regex::new(&request.message_match_regex)
.map_err(|err| format!("invalid regex: {err}"))?;
if !re.capture_names().flatten().any(|name| name == request.capture_group) {
let uses_json_field = request.json_field_path.as_deref().is_some_and(|path| !path.is_empty());
if !uses_json_field && !re.capture_names().flatten().any(|name| name == request.capture_group) {
return Err(format!(
"regex has no capture group named '{}'",
request.capture_group
@@ -219,6 +224,7 @@ fn declare_param(
DataValue::ParamDeclaration {
message_match_regex: request.message_match_regex,
capture_group: request.capture_group,
json_field_path: request.json_field_path,
kind: request.kind,
},
);
@@ -237,7 +243,9 @@ fn declare_metric(
.params
.iter()
.filter(|matcher| {
!re.capture_names().flatten().any(|name| name == matcher.capture_group)
let uses_json_field =
matcher.json_field_path.as_deref().is_some_and(|path| !path.is_empty());
!uses_json_field && !re.capture_names().flatten().any(|name| name == matcher.capture_group)
})
.map(|matcher| {
format!(
@@ -324,14 +332,21 @@ fn apply_metric_to_log(
let mut value = None;
let mut is_duration = false;
for matcher in params {
let Some(raw) = captures.name(&matcher.capture_group) else {
return;
let raw: String = match matcher.json_field_path.as_deref().filter(|path| !path.is_empty()) {
Some(field_path) => match crate::data_value::extract_json_field(text, field_path) {
Some(value) => value,
None => return,
},
None => match captures.name(&matcher.capture_group) {
Some(m) => m.as_str().to_string(),
None => return,
},
};
if matcher.role == ParamRole::Value && matcher.kind == MetricPrimitiveKind::Duration {
is_duration = true;
continue;
}
let Some(parsed) = parse_metric_primitive(raw.as_str(), matcher.kind) else {
let Some(parsed) = parse_metric_primitive(&raw, matcher.kind) else {
return;
};
match matcher.role {
@@ -682,14 +697,20 @@ fn metric_declarations_range(
fn param_declarations_range(
index: &BTreeMap<IndexKey, DataValue>,
) -> impl Iterator<Item = (&str, &str, &str, MetricPrimitiveKind)> {
) -> impl Iterator<Item = (&str, &str, &str, Option<&str>, MetricPrimitiveKind)> {
index
.range(IndexKey::LogParamSplit..IndexKey::ParamMetricSplit)
.filter_map(|(key, value)| match (key, value) {
(
IndexKey::ParamDeclaration { name },
DataValue::ParamDeclaration { message_match_regex, capture_group, kind },
) => Some((name.as_str(), message_match_regex.as_str(), capture_group.as_str(), *kind)),
DataValue::ParamDeclaration { message_match_regex, capture_group, json_field_path, kind },
) => Some((
name.as_str(),
message_match_regex.as_str(),
capture_group.as_str(),
json_field_path.as_deref(),
*kind,
)),
_ => None,
})
}
@@ -697,30 +718,44 @@ fn param_declarations_range(
/// A param whose stored regex fails to compile (shouldn't happen — `declare_param` validates it) is skipped rather than panicking.
fn compiled_param_declarations(
index: &BTreeMap<IndexKey, DataValue>,
) -> Vec<(String, Regex, String, MetricPrimitiveKind)> {
) -> Vec<(String, Regex, String, Option<String>, MetricPrimitiveKind)> {
param_declarations_range(index)
.filter_map(|(name, message_match_regex, capture_group, kind)| {
Regex::new(message_match_regex)
.ok()
.map(|re| (name.to_string(), re, capture_group.to_string(), kind))
.filter_map(|(name, message_match_regex, capture_group, json_field_path, kind)| {
Regex::new(message_match_regex).ok().map(|re| {
(
name.to_string(),
re,
capture_group.to_string(),
json_field_path.map(|path| path.to_string()),
kind,
)
})
})
.collect()
}
/// A param is omitted (not included as null/empty) when its regex/capture group doesn't match or the captured text fails to parse — see `LogEntryDto::params`.
/// A param is omitted (not included as null/empty) when its regex/capture group (or JSON field
/// path) doesn't match, or the extracted text fails to parse — see `LogEntryDto::params`.
fn derive_log_params(
text: &str,
declarations: &[(String, Regex, String, MetricPrimitiveKind)],
declarations: &[(String, Regex, String, Option<String>, MetricPrimitiveKind)],
) -> BTreeMap<String, MetricPrimitive> {
let mut params = BTreeMap::new();
for (name, regex, capture_group, kind) in declarations {
for (name, regex, capture_group, json_field_path, kind) in declarations {
let Some(captures) = regex.captures(text) else {
continue;
};
let Some(raw) = captures.name(capture_group) else {
continue;
let raw: String = match json_field_path.as_deref().filter(|path| !path.is_empty()) {
Some(field_path) => match crate::data_value::extract_json_field(text, field_path) {
Some(value) => value,
None => continue,
},
None => match captures.name(capture_group) {
Some(m) => m.as_str().to_string(),
None => continue,
},
};
let Some(value) = parse_metric_primitive(raw.as_str(), *kind) else {
let Some(value) = parse_metric_primitive(&raw, *kind) else {
continue;
};
params.insert(name.clone(), value);
@@ -866,10 +901,11 @@ fn handle_query(
}
EngineQuery::ParamDeclarations => {
let declarations = param_declarations_range(index)
.map(|(name, message_match_regex, capture_group, kind)| ParamDeclarationDto {
.map(|(name, message_match_regex, capture_group, json_field_path, kind)| ParamDeclarationDto {
name: name.to_string(),
message_match_regex: message_match_regex.to_string(),
capture_group: capture_group.to_string(),
json_field_path: json_field_path.map(|path| path.to_string()),
kind,
})
.collect();
@@ -1066,6 +1102,7 @@ mod tests {
name: name.to_string(),
message_match_regex: pattern.to_string(),
capture_group: capture_group.to_string(),
json_field_path: None,
kind: MetricPrimitiveKind::String,
}
}
@@ -1102,6 +1139,52 @@ mod tests {
assert!(index.is_empty());
}
#[test]
fn declares_valid_param_from_json_field_path() {
let mut index = BTreeMap::new();
let result = declare_param(
&mut index,
DeclareParamRequest {
name: "level".to_string(),
// No named capture group needed for JSON-field extraction, just something that
// matches the lines this param applies to.
message_match_regex: r".".to_string(),
capture_group: String::new(),
json_field_path: Some("foo.bar".to_string()),
kind: MetricPrimitiveKind::String,
},
);
assert!(result.is_ok());
let declarations: Vec<_> = param_declarations_range(&index).collect();
assert_eq!(declarations.len(), 1);
assert_eq!(declarations[0].3, Some("foo.bar"));
}
#[test]
fn derives_log_param_from_json_field_path() {
let mut index = BTreeMap::new();
declare_param(
&mut index,
DeclareParamRequest {
name: "bar".to_string(),
message_match_regex: r".".to_string(),
capture_group: String::new(),
json_field_path: Some("foo.bar".to_string()),
kind: MetricPrimitiveKind::Number,
},
)
.unwrap();
let declarations = compiled_param_declarations(&index);
let params = derive_log_params(r#"{"foo":{"bar":42}}"#, &declarations);
assert_eq!(params.get("bar"), Some(&MetricPrimitive::Number(Number::Int(42))));
// A line that doesn't parse as JSON, or is missing the field, yields no param.
assert!(derive_log_params("not json", &declarations).is_empty());
assert!(derive_log_params(r#"{"foo":{}}"#, &declarations).is_empty());
}
#[test]
fn evict_excess_logs_drops_oldest_sequences_first() {
let mut index = BTreeMap::new();
@@ -1215,6 +1298,7 @@ mod tests {
.map(|(param_name, capture_group, kind, role)| RegexToMetricPrimitiveMatcher {
param_name: param_name.to_string(),
capture_group: capture_group.to_string(),
json_field_path: None,
kind,
role,
})
@@ -1349,6 +1433,41 @@ mod tests {
assert_eq!(values.len(), 2);
}
#[test]
fn metric_param_extracts_from_json_field_path() {
let mut index = BTreeMap::new();
ingest_log(&mut index, 0, r#"{"request":{"path":"/a"}}"#);
// Doesn't parse as JSON at all — should be skipped, not panic.
ingest_log(&mut index, 1, "not json");
// Parses as JSON but is missing the field — also skipped.
ingest_log(&mut index, 2, r#"{"request":{}}"#);
declare_metric(
&mut index,
DeclareMetricRequest {
name: "requests".to_string(),
message_match_regex: r".".to_string(),
matched_labels: None,
unmatched_labels: None,
params: vec![RegexToMetricPrimitiveMatcher {
param_name: "path".to_string(),
capture_group: String::new(),
json_field_path: Some("request.path".to_string()),
kind: MetricPrimitiveKind::String,
role: ParamRole::Label,
}],
},
)
.unwrap();
let values: Vec<_> = bucket_values(&index, "requests", None).collect();
assert_eq!(values.len(), 1);
assert_eq!(
values[0].0.get("path"),
Some(&MetricPrimitive::String("/a".to_string()))
);
}
#[test]
fn ingesting_after_declaration_updates_buckets_live() {
let mut index = BTreeMap::new();
+14
View File
@@ -60,6 +60,10 @@ pub struct ParamDeclarationRecord {
pub name: String,
pub message_match_regex: String,
pub capture_group: String,
/// See `crate::data_value::RegexToMetricPrimitiveMatcher::json_field_path`; `#[serde(default)]`
/// so profiles saved before this field existed still load as plain regex-capture-group params.
#[serde(default)]
pub json_field_path: Option<String>,
pub kind: ProfileMetricPrimitiveKind,
}
@@ -69,6 +73,7 @@ impl From<&DeclareParamRequest> for ParamDeclarationRecord {
name: request.name.clone(),
message_match_regex: request.message_match_regex.clone(),
capture_group: request.capture_group.clone(),
json_field_path: request.json_field_path.clone(),
kind: request.kind.into(),
}
}
@@ -80,6 +85,7 @@ impl From<&ParamDeclarationRecord> for DeclareParamRequest {
name: record.name.clone(),
message_match_regex: record.message_match_regex.clone(),
capture_group: record.capture_group.clone(),
json_field_path: record.json_field_path.clone(),
kind: record.kind.into(),
}
}
@@ -90,6 +96,10 @@ impl From<&ParamDeclarationRecord> for DeclareParamRequest {
pub struct MetricParamRecord {
pub param_name: String,
pub capture_group: String,
/// See `crate::data_value::RegexToMetricPrimitiveMatcher::json_field_path`; `#[serde(default)]`
/// so profiles saved before this field existed still load as plain regex-capture-group matchers.
#[serde(default)]
pub json_field_path: Option<String>,
pub kind: ProfileMetricPrimitiveKind,
/// See `ParamRole`; `#[serde(default)]` so profiles saved before this field existed still load, defaulting to `Label`.
#[serde(default)]
@@ -121,6 +131,7 @@ impl From<&DeclareMetricRequest> for MetricDeclarationRecord {
.map(|matcher| MetricParamRecord {
param_name: matcher.param_name.clone(),
capture_group: matcher.capture_group.clone(),
json_field_path: matcher.json_field_path.clone(),
kind: matcher.kind.into(),
role: matcher.role,
})
@@ -144,6 +155,7 @@ impl From<&MetricDeclarationRecord> for DeclareMetricRequest {
.map(|param| RegexToMetricPrimitiveMatcher {
param_name: param.param_name.clone(),
capture_group: param.capture_group.clone(),
json_field_path: param.json_field_path.clone(),
kind: param.kind.into(),
role: param.role,
})
@@ -626,6 +638,7 @@ mod tests {
name: name.to_string(),
message_match_regex: pattern.to_string(),
capture_group: capture_group.to_string(),
json_field_path: None,
kind: MetricPrimitiveKind::String,
}
}
@@ -639,6 +652,7 @@ mod tests {
params: vec![RegexToMetricPrimitiveMatcher {
param_name: param_name.to_string(),
capture_group: capture_group.to_string(),
json_field_path: None,
kind: MetricPrimitiveKind::Number,
role: ParamRole::Label,
}],
@@ -1,10 +0,0 @@
# Allow params/metrics to be declared from a JSON field path
Follow-up to the JSON log ingestion design decision (repowise decision fdb47034ad954e3ea0b898fdc2750c32).
Extend param/metric declaration so a param can be extracted from a JSON field path (e.g. `foo.bar`) on lines that parse as JSON objects, as an alternative to the existing regex-capture-group extraction.
## Acceptance criteria
- `RegexToMetricPrimitiveMatcher` (src/data_value.rs) or a sibling type supports a JSON-field-path extraction source alongside the existing regex-capture-group source, reusing `ParamRole`/`MetricPrimitiveKind`.
- The params declaration UI (Params.tsx) lets a user choose "JSON field" as an alternative to "regex capture group" when declaring a param, for lines detected as JSON.
- Existing regex-based params/metrics are unaffected.
+72 -3
View File
@@ -1,28 +1,59 @@
import { useState } from 'react'
import { useMemo, useState } from 'react'
import { ApiError, declareParam, type ParamDeclaration } from '../lib/api'
import { CollapseToggle, PopOutButton, sectionHeaderClassName, sectionHeaderLeftClassName, useSectionCollapsed } from './CollapsibleSection'
import { HelpIcon } from './HelpIcon'
const EMPTY_KIND: ParamDeclaration['kind'] = 'string'
type ExtractionSource = 'regex' | 'json_field'
/// A JSON *object* (not an array/scalar) parses cleanly and pretty-prints with braces — the same
/// shape a JSON field path can be read from, so this is the "line looks like JSON" gate that
/// unlocks the "JSON field" extraction option.
function looksLikeJsonObject(text: string): boolean {
const trimmed = text.trim()
if (!trimmed.startsWith('{')) return false
try {
const parsed = JSON.parse(trimmed)
return typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)
} catch {
return false
}
}
export function Params({ params, onDeclared }: { params: ParamDeclaration[]; onDeclared: () => void }) {
const [name, setName] = useState('')
const [regex, setRegex] = useState('')
const [captureGroup, setCaptureGroup] = useState('')
const [jsonFieldPath, setJsonFieldPath] = useState('')
const [source, setSource] = useState<ExtractionSource>('regex')
const [kind, setKind] = useState<ParamDeclaration['kind']>(EMPTY_KIND)
const [submitError, setSubmitError] = useState<string | null>(null)
const [submitting, setSubmitting] = useState(false)
const [collapsed, setCollapsed] = useSectionCollapsed('params')
// The message match regex doubles as a sample line here so a pasted JSON log line unlocks the
// "JSON field" extraction option — regex-based params are unaffected either way.
const detectedJson = useMemo(() => looksLikeJsonObject(regex), [regex])
const effectiveSource: ExtractionSource = detectedJson ? source : 'regex'
async function handleSubmit(e: React.FormEvent) {
e.preventDefault()
setSubmitError(null)
setSubmitting(true)
try {
await declareParam({ name, message_match_regex: regex, capture_group: captureGroup, kind })
await declareParam({
name,
message_match_regex: regex,
capture_group: effectiveSource === 'json_field' ? '' : captureGroup,
json_field_path: effectiveSource === 'json_field' ? jsonFieldPath : null,
kind,
})
setName('')
setRegex('')
setCaptureGroup('')
setJsonFieldPath('')
setSource('regex')
setKind(EMPTY_KIND)
onDeclared()
} catch (err) {
@@ -66,6 +97,43 @@ export function Params({ params, onDeclared }: { params: ParamDeclaration[]; onD
required
/>
</div>
{detectedJson && (
<div>
<label className="mb-1 block text-xs text-text-subtle">Extraction</label>
<select
className="w-full rounded border border-border-soft bg-surface-1 px-2 py-1 text-xs text-text focus:border-border-strong focus:outline-none"
value={source}
onChange={(e) => setSource(e.target.value as ExtractionSource)}
>
<option value="regex">Regex capture group</option>
<option value="json_field">JSON field</option>
</select>
</div>
)}
{effectiveSource === 'json_field' ? (
<div className="flex items-end gap-1">
<div className="min-w-0 flex-1">
<label className="mb-1 block text-xs text-text-subtle">JSON field path</label>
<input
className="w-full min-w-0 rounded border border-border-soft bg-surface-1 px-1.5 py-1 font-mono text-xs text-text placeholder:text-text-subtle focus:border-border-strong focus:outline-none"
value={jsonFieldPath}
onChange={(e) => setJsonFieldPath(e.target.value)}
placeholder="foo.bar"
required
/>
</div>
<select
className="rounded border border-border-soft bg-surface-1 px-1 py-1 text-xs text-text focus:border-border-strong focus:outline-none"
value={kind}
onChange={(e) => setKind(e.target.value as ParamDeclaration['kind'])}
>
<option value="string">string</option>
<option value="number">number</option>
<option value="timestamp">timestamp</option>
<option value="duration">duration</option>
</select>
</div>
) : (
<div className="flex items-end gap-1">
<div className="min-w-0 flex-1">
<label className="mb-1 block text-xs text-text-subtle">Capture group</label>
@@ -88,6 +156,7 @@ export function Params({ params, onDeclared }: { params: ParamDeclaration[]; onD
<option value="duration">duration</option>
</select>
</div>
)}
{submitError && <div className="rounded border border-danger-border bg-danger-bg/50 px-2 py-1 text-xs text-danger-text">{submitError}</div>}
<button
type="submit"
@@ -108,7 +177,7 @@ export function Params({ params, onDeclared }: { params: ParamDeclaration[]; onD
className="rounded border border-border-soft px-2 py-1 text-xs text-text-secondary"
title={p.message_match_regex}
>
{p.name} ({p.kind}, group {p.capture_group})
{p.name} ({p.kind}, {p.json_field_path ? `json field ${p.json_field_path}` : `group ${p.capture_group}`})
</span>
))}
</div>
+4
View File
@@ -25,6 +25,8 @@ export interface ParamDeclaration {
name: string
message_match_regex: string
capture_group: string
/** Alternative to `capture_group`: a dot-separated JSON field path (e.g. "foo.bar"), read from lines that parse as a JSON object. When set (non-empty), takes precedence over `capture_group`. */
json_field_path?: string | null
kind: 'string' | 'number' | 'timestamp' | 'duration'
}
@@ -48,6 +50,8 @@ export type ParamRole = 'label' | 'value'
export interface MetricParamMatcher {
param_name: string
capture_group: string
/** See `ParamDeclaration.json_field_path`. */
json_field_path?: string | null
kind: 'string' | 'number' | 'timestamp' | 'duration'
/** Defaults to 'label' server-side when omitted. */
role?: ParamRole