diff --git a/src/api.rs b/src/api.rs index 7601118..6d1980d 100644 --- a/src/api.rs +++ b/src/api.rs @@ -223,6 +223,7 @@ async fn declare_param( name: request.name, message_match_regex: request.message_match_regex, capture_group: request.capture_group, + json_field_path: request.json_field_path, kind: request.kind, }; (StatusCode::CREATED, Json(dto)).into_response() diff --git a/src/data_value.rs b/src/data_value.rs index 731ecfb..94c4ee8 100644 --- a/src/data_value.rs +++ b/src/data_value.rs @@ -8,12 +8,44 @@ use std::collections::BTreeMap; pub struct RegexToMetricPrimitiveMatcher { pub param_name: String, pub capture_group: String, + /// Alternative extraction source to `capture_group`: a dot-separated JSON field path (e.g. + /// `foo.bar`), read from lines that parse as a JSON object instead of via the regex capture + /// group. When present and non-empty, this takes precedence over `capture_group`. + /// `#[serde(default)]` so profile files/requests saved before this field existed still load + /// as plain regex-capture-group matchers. + #[serde(default)] + pub json_field_path: Option, pub kind: MetricPrimitiveKind, /// Whether this param is a bucket label or its captured value (at most one Value-role param per metric); `#[serde(default)]` so profile files saved before this field existed still load as `Label`. #[serde(default)] pub role: ParamRole, } +/// Reads a value out of a JSON object by dot-separated field path (e.g. `"foo.bar"`), for +/// [`RegexToMetricPrimitiveMatcher::json_field_path`] and the sibling `json_field_path` on a +/// standalone param declaration. Returns `None` when `text` doesn't parse as a JSON object, the +/// path doesn't resolve to a value, or the resolved value isn't a scalar (string/number/bool) — +/// there's no sensible flat text form of a nested object/array to feed into +/// `parse_metric_primitive`. +pub fn extract_json_field(text: &str, field_path: &str) -> Option { + let root: serde_json::Value = serde_json::from_str(text).ok()?; + let mut current = root.as_object()?; + let mut segments = field_path.split('.').peekable(); + loop { + let segment = segments.next()?; + let next_value = current.get(segment)?; + if segments.peek().is_none() { + return match next_value { + serde_json::Value::String(s) => Some(s.clone()), + serde_json::Value::Number(n) => Some(n.to_string()), + serde_json::Value::Bool(b) => Some(b.to_string()), + _ => None, + }; + } + current = next_value.as_object()?; + } +} + /// See [`RegexToMetricPrimitiveMatcher::role`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] #[serde(rename_all = "lowercase")] @@ -45,6 +77,8 @@ pub enum DataValue { ParamDeclaration { message_match_regex: String, capture_group: String, + /// See `RegexToMetricPrimitiveMatcher::json_field_path`. + json_field_path: Option, kind: MetricPrimitiveKind, }, /// Paired with `IndexKey::MetricDeclaration`. diff --git a/src/dto.rs b/src/dto.rs index 05171af..478d169 100644 --- a/src/dto.rs +++ b/src/dto.rs @@ -20,6 +20,8 @@ pub struct ParamDeclarationDto { pub name: String, pub message_match_regex: String, pub capture_group: String, + /// See `crate::data_value::RegexToMetricPrimitiveMatcher::json_field_path`. + pub json_field_path: Option, pub kind: MetricPrimitiveKind, } diff --git a/src/engine.rs b/src/engine.rs index d9b11ee..61abdf3 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -38,6 +38,10 @@ pub struct DeclareParamRequest { pub name: String, pub message_match_regex: String, pub capture_group: String, + /// See `RegexToMetricPrimitiveMatcher::json_field_path`; `#[serde(default)]` so existing + /// regex-only callers are unaffected. + #[serde(default)] + pub json_field_path: Option, pub kind: MetricPrimitiveKind, } @@ -207,7 +211,8 @@ fn declare_param( ) -> Result<(), String> { let re = Regex::new(&request.message_match_regex) .map_err(|err| format!("invalid regex: {err}"))?; - if !re.capture_names().flatten().any(|name| name == request.capture_group) { + let uses_json_field = request.json_field_path.as_deref().is_some_and(|path| !path.is_empty()); + if !uses_json_field && !re.capture_names().flatten().any(|name| name == request.capture_group) { return Err(format!( "regex has no capture group named '{}'", request.capture_group @@ -219,6 +224,7 @@ fn declare_param( DataValue::ParamDeclaration { message_match_regex: request.message_match_regex, capture_group: request.capture_group, + json_field_path: request.json_field_path, kind: request.kind, }, ); @@ -237,7 +243,9 @@ fn declare_metric( .params .iter() .filter(|matcher| { - !re.capture_names().flatten().any(|name| name == matcher.capture_group) + let uses_json_field = + matcher.json_field_path.as_deref().is_some_and(|path| !path.is_empty()); + !uses_json_field && !re.capture_names().flatten().any(|name| name == matcher.capture_group) }) .map(|matcher| { format!( @@ -324,14 +332,21 @@ fn apply_metric_to_log( let mut value = None; let mut is_duration = false; for matcher in params { - let Some(raw) = captures.name(&matcher.capture_group) else { - return; + let raw: String = match matcher.json_field_path.as_deref().filter(|path| !path.is_empty()) { + Some(field_path) => match crate::data_value::extract_json_field(text, field_path) { + Some(value) => value, + None => return, + }, + None => match captures.name(&matcher.capture_group) { + Some(m) => m.as_str().to_string(), + None => return, + }, }; if matcher.role == ParamRole::Value && matcher.kind == MetricPrimitiveKind::Duration { is_duration = true; continue; } - let Some(parsed) = parse_metric_primitive(raw.as_str(), matcher.kind) else { + let Some(parsed) = parse_metric_primitive(&raw, matcher.kind) else { return; }; match matcher.role { @@ -682,14 +697,20 @@ fn metric_declarations_range( fn param_declarations_range( index: &BTreeMap, -) -> impl Iterator { +) -> impl Iterator, MetricPrimitiveKind)> { index .range(IndexKey::LogParamSplit..IndexKey::ParamMetricSplit) .filter_map(|(key, value)| match (key, value) { ( IndexKey::ParamDeclaration { name }, - DataValue::ParamDeclaration { message_match_regex, capture_group, kind }, - ) => Some((name.as_str(), message_match_regex.as_str(), capture_group.as_str(), *kind)), + DataValue::ParamDeclaration { message_match_regex, capture_group, json_field_path, kind }, + ) => Some(( + name.as_str(), + message_match_regex.as_str(), + capture_group.as_str(), + json_field_path.as_deref(), + *kind, + )), _ => None, }) } @@ -697,30 +718,44 @@ fn param_declarations_range( /// A param whose stored regex fails to compile (shouldn't happen — `declare_param` validates it) is skipped rather than panicking. fn compiled_param_declarations( index: &BTreeMap, -) -> Vec<(String, Regex, String, MetricPrimitiveKind)> { +) -> Vec<(String, Regex, String, Option, MetricPrimitiveKind)> { param_declarations_range(index) - .filter_map(|(name, message_match_regex, capture_group, kind)| { - Regex::new(message_match_regex) - .ok() - .map(|re| (name.to_string(), re, capture_group.to_string(), kind)) + .filter_map(|(name, message_match_regex, capture_group, json_field_path, kind)| { + Regex::new(message_match_regex).ok().map(|re| { + ( + name.to_string(), + re, + capture_group.to_string(), + json_field_path.map(|path| path.to_string()), + kind, + ) + }) }) .collect() } -/// A param is omitted (not included as null/empty) when its regex/capture group doesn't match or the captured text fails to parse — see `LogEntryDto::params`. +/// A param is omitted (not included as null/empty) when its regex/capture group (or JSON field +/// path) doesn't match, or the extracted text fails to parse — see `LogEntryDto::params`. fn derive_log_params( text: &str, - declarations: &[(String, Regex, String, MetricPrimitiveKind)], + declarations: &[(String, Regex, String, Option, MetricPrimitiveKind)], ) -> BTreeMap { let mut params = BTreeMap::new(); - for (name, regex, capture_group, kind) in declarations { + for (name, regex, capture_group, json_field_path, kind) in declarations { let Some(captures) = regex.captures(text) else { continue; }; - let Some(raw) = captures.name(capture_group) else { - continue; + let raw: String = match json_field_path.as_deref().filter(|path| !path.is_empty()) { + Some(field_path) => match crate::data_value::extract_json_field(text, field_path) { + Some(value) => value, + None => continue, + }, + None => match captures.name(capture_group) { + Some(m) => m.as_str().to_string(), + None => continue, + }, }; - let Some(value) = parse_metric_primitive(raw.as_str(), *kind) else { + let Some(value) = parse_metric_primitive(&raw, *kind) else { continue; }; params.insert(name.clone(), value); @@ -866,10 +901,11 @@ fn handle_query( } EngineQuery::ParamDeclarations => { let declarations = param_declarations_range(index) - .map(|(name, message_match_regex, capture_group, kind)| ParamDeclarationDto { + .map(|(name, message_match_regex, capture_group, json_field_path, kind)| ParamDeclarationDto { name: name.to_string(), message_match_regex: message_match_regex.to_string(), capture_group: capture_group.to_string(), + json_field_path: json_field_path.map(|path| path.to_string()), kind, }) .collect(); @@ -1066,6 +1102,7 @@ mod tests { name: name.to_string(), message_match_regex: pattern.to_string(), capture_group: capture_group.to_string(), + json_field_path: None, kind: MetricPrimitiveKind::String, } } @@ -1102,6 +1139,52 @@ mod tests { assert!(index.is_empty()); } + #[test] + fn declares_valid_param_from_json_field_path() { + let mut index = BTreeMap::new(); + let result = declare_param( + &mut index, + DeclareParamRequest { + name: "level".to_string(), + // No named capture group needed for JSON-field extraction, just something that + // matches the lines this param applies to. + message_match_regex: r".".to_string(), + capture_group: String::new(), + json_field_path: Some("foo.bar".to_string()), + kind: MetricPrimitiveKind::String, + }, + ); + assert!(result.is_ok()); + + let declarations: Vec<_> = param_declarations_range(&index).collect(); + assert_eq!(declarations.len(), 1); + assert_eq!(declarations[0].3, Some("foo.bar")); + } + + #[test] + fn derives_log_param_from_json_field_path() { + let mut index = BTreeMap::new(); + declare_param( + &mut index, + DeclareParamRequest { + name: "bar".to_string(), + message_match_regex: r".".to_string(), + capture_group: String::new(), + json_field_path: Some("foo.bar".to_string()), + kind: MetricPrimitiveKind::Number, + }, + ) + .unwrap(); + + let declarations = compiled_param_declarations(&index); + let params = derive_log_params(r#"{"foo":{"bar":42}}"#, &declarations); + assert_eq!(params.get("bar"), Some(&MetricPrimitive::Number(Number::Int(42)))); + + // A line that doesn't parse as JSON, or is missing the field, yields no param. + assert!(derive_log_params("not json", &declarations).is_empty()); + assert!(derive_log_params(r#"{"foo":{}}"#, &declarations).is_empty()); + } + #[test] fn evict_excess_logs_drops_oldest_sequences_first() { let mut index = BTreeMap::new(); @@ -1215,6 +1298,7 @@ mod tests { .map(|(param_name, capture_group, kind, role)| RegexToMetricPrimitiveMatcher { param_name: param_name.to_string(), capture_group: capture_group.to_string(), + json_field_path: None, kind, role, }) @@ -1349,6 +1433,41 @@ mod tests { assert_eq!(values.len(), 2); } + #[test] + fn metric_param_extracts_from_json_field_path() { + let mut index = BTreeMap::new(); + ingest_log(&mut index, 0, r#"{"request":{"path":"/a"}}"#); + // Doesn't parse as JSON at all — should be skipped, not panic. + ingest_log(&mut index, 1, "not json"); + // Parses as JSON but is missing the field — also skipped. + ingest_log(&mut index, 2, r#"{"request":{}}"#); + + declare_metric( + &mut index, + DeclareMetricRequest { + name: "requests".to_string(), + message_match_regex: r".".to_string(), + matched_labels: None, + unmatched_labels: None, + params: vec![RegexToMetricPrimitiveMatcher { + param_name: "path".to_string(), + capture_group: String::new(), + json_field_path: Some("request.path".to_string()), + kind: MetricPrimitiveKind::String, + role: ParamRole::Label, + }], + }, + ) + .unwrap(); + + let values: Vec<_> = bucket_values(&index, "requests", None).collect(); + assert_eq!(values.len(), 1); + assert_eq!( + values[0].0.get("path"), + Some(&MetricPrimitive::String("/a".to_string())) + ); + } + #[test] fn ingesting_after_declaration_updates_buckets_live() { let mut index = BTreeMap::new(); diff --git a/src/profile.rs b/src/profile.rs index f8fb82c..d636822 100644 --- a/src/profile.rs +++ b/src/profile.rs @@ -60,6 +60,10 @@ pub struct ParamDeclarationRecord { pub name: String, pub message_match_regex: String, pub capture_group: String, + /// See `crate::data_value::RegexToMetricPrimitiveMatcher::json_field_path`; `#[serde(default)]` + /// so profiles saved before this field existed still load as plain regex-capture-group params. + #[serde(default)] + pub json_field_path: Option, pub kind: ProfileMetricPrimitiveKind, } @@ -69,6 +73,7 @@ impl From<&DeclareParamRequest> for ParamDeclarationRecord { name: request.name.clone(), message_match_regex: request.message_match_regex.clone(), capture_group: request.capture_group.clone(), + json_field_path: request.json_field_path.clone(), kind: request.kind.into(), } } @@ -80,6 +85,7 @@ impl From<&ParamDeclarationRecord> for DeclareParamRequest { name: record.name.clone(), message_match_regex: record.message_match_regex.clone(), capture_group: record.capture_group.clone(), + json_field_path: record.json_field_path.clone(), kind: record.kind.into(), } } @@ -90,6 +96,10 @@ impl From<&ParamDeclarationRecord> for DeclareParamRequest { pub struct MetricParamRecord { pub param_name: String, pub capture_group: String, + /// See `crate::data_value::RegexToMetricPrimitiveMatcher::json_field_path`; `#[serde(default)]` + /// so profiles saved before this field existed still load as plain regex-capture-group matchers. + #[serde(default)] + pub json_field_path: Option, pub kind: ProfileMetricPrimitiveKind, /// See `ParamRole`; `#[serde(default)]` so profiles saved before this field existed still load, defaulting to `Label`. #[serde(default)] @@ -121,6 +131,7 @@ impl From<&DeclareMetricRequest> for MetricDeclarationRecord { .map(|matcher| MetricParamRecord { param_name: matcher.param_name.clone(), capture_group: matcher.capture_group.clone(), + json_field_path: matcher.json_field_path.clone(), kind: matcher.kind.into(), role: matcher.role, }) @@ -144,6 +155,7 @@ impl From<&MetricDeclarationRecord> for DeclareMetricRequest { .map(|param| RegexToMetricPrimitiveMatcher { param_name: param.param_name.clone(), capture_group: param.capture_group.clone(), + json_field_path: param.json_field_path.clone(), kind: param.kind.into(), role: param.role, }) @@ -626,6 +638,7 @@ mod tests { name: name.to_string(), message_match_regex: pattern.to_string(), capture_group: capture_group.to_string(), + json_field_path: None, kind: MetricPrimitiveKind::String, } } @@ -639,6 +652,7 @@ mod tests { params: vec![RegexToMetricPrimitiveMatcher { param_name: param_name.to_string(), capture_group: capture_group.to_string(), + json_field_path: None, kind: MetricPrimitiveKind::Number, role: ParamRole::Label, }], diff --git a/tickets/015-json-field-param-metric-declaration.md b/tickets/015-json-field-param-metric-declaration.md deleted file mode 100644 index d74edbf..0000000 --- a/tickets/015-json-field-param-metric-declaration.md +++ /dev/null @@ -1,10 +0,0 @@ -# Allow params/metrics to be declared from a JSON field path - -Follow-up to the JSON log ingestion design decision (repowise decision fdb47034ad954e3ea0b898fdc2750c32). - -Extend param/metric declaration so a param can be extracted from a JSON field path (e.g. `foo.bar`) on lines that parse as JSON objects, as an alternative to the existing regex-capture-group extraction. - -## Acceptance criteria -- `RegexToMetricPrimitiveMatcher` (src/data_value.rs) or a sibling type supports a JSON-field-path extraction source alongside the existing regex-capture-group source, reusing `ParamRole`/`MetricPrimitiveKind`. -- The params declaration UI (Params.tsx) lets a user choose "JSON field" as an alternative to "regex capture group" when declaring a param, for lines detected as JSON. -- Existing regex-based params/metrics are unaffected. diff --git a/webapp/src/components/Params.tsx b/webapp/src/components/Params.tsx index c7b2c8b..5b2a5d2 100644 --- a/webapp/src/components/Params.tsx +++ b/webapp/src/components/Params.tsx @@ -1,28 +1,59 @@ -import { useState } from 'react' +import { useMemo, useState } from 'react' import { ApiError, declareParam, type ParamDeclaration } from '../lib/api' import { CollapseToggle, PopOutButton, sectionHeaderClassName, sectionHeaderLeftClassName, useSectionCollapsed } from './CollapsibleSection' import { HelpIcon } from './HelpIcon' const EMPTY_KIND: ParamDeclaration['kind'] = 'string' +type ExtractionSource = 'regex' | 'json_field' + +/// A JSON *object* (not an array/scalar) parses cleanly and pretty-prints with braces — the same +/// shape a JSON field path can be read from, so this is the "line looks like JSON" gate that +/// unlocks the "JSON field" extraction option. +function looksLikeJsonObject(text: string): boolean { + const trimmed = text.trim() + if (!trimmed.startsWith('{')) return false + try { + const parsed = JSON.parse(trimmed) + return typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed) + } catch { + return false + } +} + export function Params({ params, onDeclared }: { params: ParamDeclaration[]; onDeclared: () => void }) { const [name, setName] = useState('') const [regex, setRegex] = useState('') const [captureGroup, setCaptureGroup] = useState('') + const [jsonFieldPath, setJsonFieldPath] = useState('') + const [source, setSource] = useState('regex') const [kind, setKind] = useState(EMPTY_KIND) const [submitError, setSubmitError] = useState(null) const [submitting, setSubmitting] = useState(false) const [collapsed, setCollapsed] = useSectionCollapsed('params') + // The message match regex doubles as a sample line here so a pasted JSON log line unlocks the + // "JSON field" extraction option — regex-based params are unaffected either way. + const detectedJson = useMemo(() => looksLikeJsonObject(regex), [regex]) + const effectiveSource: ExtractionSource = detectedJson ? source : 'regex' + async function handleSubmit(e: React.FormEvent) { e.preventDefault() setSubmitError(null) setSubmitting(true) try { - await declareParam({ name, message_match_regex: regex, capture_group: captureGroup, kind }) + await declareParam({ + name, + message_match_regex: regex, + capture_group: effectiveSource === 'json_field' ? '' : captureGroup, + json_field_path: effectiveSource === 'json_field' ? jsonFieldPath : null, + kind, + }) setName('') setRegex('') setCaptureGroup('') + setJsonFieldPath('') + setSource('regex') setKind(EMPTY_KIND) onDeclared() } catch (err) { @@ -66,28 +97,66 @@ export function Params({ params, onDeclared }: { params: ParamDeclaration[]; onD required /> -
-
- - setCaptureGroup(e.target.value)} - placeholder="path" - required - /> + {detectedJson && ( +
+ +
- -
+ )} + {effectiveSource === 'json_field' ? ( +
+
+ + setJsonFieldPath(e.target.value)} + placeholder="foo.bar" + required + /> +
+ +
+ ) : ( +
+
+ + setCaptureGroup(e.target.value)} + placeholder="path" + required + /> +
+ +
+ )} {submitError &&
{submitError}
}
diff --git a/webapp/src/lib/api.ts b/webapp/src/lib/api.ts index 91c32ee..c05bc3f 100644 --- a/webapp/src/lib/api.ts +++ b/webapp/src/lib/api.ts @@ -25,6 +25,8 @@ export interface ParamDeclaration { name: string message_match_regex: string capture_group: string + /** Alternative to `capture_group`: a dot-separated JSON field path (e.g. "foo.bar"), read from lines that parse as a JSON object. When set (non-empty), takes precedence over `capture_group`. */ + json_field_path?: string | null kind: 'string' | 'number' | 'timestamp' | 'duration' } @@ -48,6 +50,8 @@ export type ParamRole = 'label' | 'value' export interface MetricParamMatcher { param_name: string capture_group: string + /** See `ParamDeclaration.json_field_path`. */ + json_field_path?: string | null kind: 'string' | 'number' | 'timestamp' | 'duration' /** Defaults to 'label' server-side when omitted. */ role?: ParamRole