Add JSON field path extraction for params/metrics (ticket 015)
RegexToMetricPrimitiveMatcher and the standalone param declaration (DeclareParamRequest/DataValue::ParamDeclaration) gain an optional json_field_path, an alternative to capture_group that reads a dot-separated field path (e.g. "foo.bar") out of lines that parse as JSON objects. It's additive and serde-defaulted, so existing regex-based params/metrics are unaffected. Params.tsx now offers a "JSON field" extraction option when the message match regex looks like a JSON object. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01XFUNZwCvz3kcod3352ZR2o
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
e6e5bc8f50
commit
b3e0272544
@@ -223,6 +223,7 @@ async fn declare_param(
|
|||||||
name: request.name,
|
name: request.name,
|
||||||
message_match_regex: request.message_match_regex,
|
message_match_regex: request.message_match_regex,
|
||||||
capture_group: request.capture_group,
|
capture_group: request.capture_group,
|
||||||
|
json_field_path: request.json_field_path,
|
||||||
kind: request.kind,
|
kind: request.kind,
|
||||||
};
|
};
|
||||||
(StatusCode::CREATED, Json(dto)).into_response()
|
(StatusCode::CREATED, Json(dto)).into_response()
|
||||||
|
|||||||
@@ -8,12 +8,44 @@ use std::collections::BTreeMap;
|
|||||||
pub struct RegexToMetricPrimitiveMatcher {
|
pub struct RegexToMetricPrimitiveMatcher {
|
||||||
pub param_name: String,
|
pub param_name: String,
|
||||||
pub capture_group: String,
|
pub capture_group: String,
|
||||||
|
/// Alternative extraction source to `capture_group`: a dot-separated JSON field path (e.g.
|
||||||
|
/// `foo.bar`), read from lines that parse as a JSON object instead of via the regex capture
|
||||||
|
/// group. When present and non-empty, this takes precedence over `capture_group`.
|
||||||
|
/// `#[serde(default)]` so profile files/requests saved before this field existed still load
|
||||||
|
/// as plain regex-capture-group matchers.
|
||||||
|
#[serde(default)]
|
||||||
|
pub json_field_path: Option<String>,
|
||||||
pub kind: MetricPrimitiveKind,
|
pub kind: MetricPrimitiveKind,
|
||||||
/// Whether this param is a bucket label or its captured value (at most one Value-role param per metric); `#[serde(default)]` so profile files saved before this field existed still load as `Label`.
|
/// Whether this param is a bucket label or its captured value (at most one Value-role param per metric); `#[serde(default)]` so profile files saved before this field existed still load as `Label`.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub role: ParamRole,
|
pub role: ParamRole,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Reads a value out of a JSON object by dot-separated field path (e.g. `"foo.bar"`), for
|
||||||
|
/// [`RegexToMetricPrimitiveMatcher::json_field_path`] and the sibling `json_field_path` on a
|
||||||
|
/// standalone param declaration. Returns `None` when `text` doesn't parse as a JSON object, the
|
||||||
|
/// path doesn't resolve to a value, or the resolved value isn't a scalar (string/number/bool) —
|
||||||
|
/// there's no sensible flat text form of a nested object/array to feed into
|
||||||
|
/// `parse_metric_primitive`.
|
||||||
|
pub fn extract_json_field(text: &str, field_path: &str) -> Option<String> {
|
||||||
|
let root: serde_json::Value = serde_json::from_str(text).ok()?;
|
||||||
|
let mut current = root.as_object()?;
|
||||||
|
let mut segments = field_path.split('.').peekable();
|
||||||
|
loop {
|
||||||
|
let segment = segments.next()?;
|
||||||
|
let next_value = current.get(segment)?;
|
||||||
|
if segments.peek().is_none() {
|
||||||
|
return match next_value {
|
||||||
|
serde_json::Value::String(s) => Some(s.clone()),
|
||||||
|
serde_json::Value::Number(n) => Some(n.to_string()),
|
||||||
|
serde_json::Value::Bool(b) => Some(b.to_string()),
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
current = next_value.as_object()?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// See [`RegexToMetricPrimitiveMatcher::role`].
|
/// See [`RegexToMetricPrimitiveMatcher::role`].
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||||
#[serde(rename_all = "lowercase")]
|
#[serde(rename_all = "lowercase")]
|
||||||
@@ -45,6 +77,8 @@ pub enum DataValue {
|
|||||||
ParamDeclaration {
|
ParamDeclaration {
|
||||||
message_match_regex: String,
|
message_match_regex: String,
|
||||||
capture_group: String,
|
capture_group: String,
|
||||||
|
/// See `RegexToMetricPrimitiveMatcher::json_field_path`.
|
||||||
|
json_field_path: Option<String>,
|
||||||
kind: MetricPrimitiveKind,
|
kind: MetricPrimitiveKind,
|
||||||
},
|
},
|
||||||
/// Paired with `IndexKey::MetricDeclaration`.
|
/// Paired with `IndexKey::MetricDeclaration`.
|
||||||
|
|||||||
@@ -20,6 +20,8 @@ pub struct ParamDeclarationDto {
|
|||||||
pub name: String,
|
pub name: String,
|
||||||
pub message_match_regex: String,
|
pub message_match_regex: String,
|
||||||
pub capture_group: String,
|
pub capture_group: String,
|
||||||
|
/// See `crate::data_value::RegexToMetricPrimitiveMatcher::json_field_path`.
|
||||||
|
pub json_field_path: Option<String>,
|
||||||
pub kind: MetricPrimitiveKind,
|
pub kind: MetricPrimitiveKind,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+139
-20
@@ -38,6 +38,10 @@ pub struct DeclareParamRequest {
|
|||||||
pub name: String,
|
pub name: String,
|
||||||
pub message_match_regex: String,
|
pub message_match_regex: String,
|
||||||
pub capture_group: String,
|
pub capture_group: String,
|
||||||
|
/// See `RegexToMetricPrimitiveMatcher::json_field_path`; `#[serde(default)]` so existing
|
||||||
|
/// regex-only callers are unaffected.
|
||||||
|
#[serde(default)]
|
||||||
|
pub json_field_path: Option<String>,
|
||||||
pub kind: MetricPrimitiveKind,
|
pub kind: MetricPrimitiveKind,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -207,7 +211,8 @@ fn declare_param(
|
|||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
let re = Regex::new(&request.message_match_regex)
|
let re = Regex::new(&request.message_match_regex)
|
||||||
.map_err(|err| format!("invalid regex: {err}"))?;
|
.map_err(|err| format!("invalid regex: {err}"))?;
|
||||||
if !re.capture_names().flatten().any(|name| name == request.capture_group) {
|
let uses_json_field = request.json_field_path.as_deref().is_some_and(|path| !path.is_empty());
|
||||||
|
if !uses_json_field && !re.capture_names().flatten().any(|name| name == request.capture_group) {
|
||||||
return Err(format!(
|
return Err(format!(
|
||||||
"regex has no capture group named '{}'",
|
"regex has no capture group named '{}'",
|
||||||
request.capture_group
|
request.capture_group
|
||||||
@@ -219,6 +224,7 @@ fn declare_param(
|
|||||||
DataValue::ParamDeclaration {
|
DataValue::ParamDeclaration {
|
||||||
message_match_regex: request.message_match_regex,
|
message_match_regex: request.message_match_regex,
|
||||||
capture_group: request.capture_group,
|
capture_group: request.capture_group,
|
||||||
|
json_field_path: request.json_field_path,
|
||||||
kind: request.kind,
|
kind: request.kind,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
@@ -237,7 +243,9 @@ fn declare_metric(
|
|||||||
.params
|
.params
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|matcher| {
|
.filter(|matcher| {
|
||||||
!re.capture_names().flatten().any(|name| name == matcher.capture_group)
|
let uses_json_field =
|
||||||
|
matcher.json_field_path.as_deref().is_some_and(|path| !path.is_empty());
|
||||||
|
!uses_json_field && !re.capture_names().flatten().any(|name| name == matcher.capture_group)
|
||||||
})
|
})
|
||||||
.map(|matcher| {
|
.map(|matcher| {
|
||||||
format!(
|
format!(
|
||||||
@@ -324,14 +332,21 @@ fn apply_metric_to_log(
|
|||||||
let mut value = None;
|
let mut value = None;
|
||||||
let mut is_duration = false;
|
let mut is_duration = false;
|
||||||
for matcher in params {
|
for matcher in params {
|
||||||
let Some(raw) = captures.name(&matcher.capture_group) else {
|
let raw: String = match matcher.json_field_path.as_deref().filter(|path| !path.is_empty()) {
|
||||||
return;
|
Some(field_path) => match crate::data_value::extract_json_field(text, field_path) {
|
||||||
|
Some(value) => value,
|
||||||
|
None => return,
|
||||||
|
},
|
||||||
|
None => match captures.name(&matcher.capture_group) {
|
||||||
|
Some(m) => m.as_str().to_string(),
|
||||||
|
None => return,
|
||||||
|
},
|
||||||
};
|
};
|
||||||
if matcher.role == ParamRole::Value && matcher.kind == MetricPrimitiveKind::Duration {
|
if matcher.role == ParamRole::Value && matcher.kind == MetricPrimitiveKind::Duration {
|
||||||
is_duration = true;
|
is_duration = true;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let Some(parsed) = parse_metric_primitive(raw.as_str(), matcher.kind) else {
|
let Some(parsed) = parse_metric_primitive(&raw, matcher.kind) else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
match matcher.role {
|
match matcher.role {
|
||||||
@@ -682,14 +697,20 @@ fn metric_declarations_range(
|
|||||||
|
|
||||||
fn param_declarations_range(
|
fn param_declarations_range(
|
||||||
index: &BTreeMap<IndexKey, DataValue>,
|
index: &BTreeMap<IndexKey, DataValue>,
|
||||||
) -> impl Iterator<Item = (&str, &str, &str, MetricPrimitiveKind)> {
|
) -> impl Iterator<Item = (&str, &str, &str, Option<&str>, MetricPrimitiveKind)> {
|
||||||
index
|
index
|
||||||
.range(IndexKey::LogParamSplit..IndexKey::ParamMetricSplit)
|
.range(IndexKey::LogParamSplit..IndexKey::ParamMetricSplit)
|
||||||
.filter_map(|(key, value)| match (key, value) {
|
.filter_map(|(key, value)| match (key, value) {
|
||||||
(
|
(
|
||||||
IndexKey::ParamDeclaration { name },
|
IndexKey::ParamDeclaration { name },
|
||||||
DataValue::ParamDeclaration { message_match_regex, capture_group, kind },
|
DataValue::ParamDeclaration { message_match_regex, capture_group, json_field_path, kind },
|
||||||
) => Some((name.as_str(), message_match_regex.as_str(), capture_group.as_str(), *kind)),
|
) => Some((
|
||||||
|
name.as_str(),
|
||||||
|
message_match_regex.as_str(),
|
||||||
|
capture_group.as_str(),
|
||||||
|
json_field_path.as_deref(),
|
||||||
|
*kind,
|
||||||
|
)),
|
||||||
_ => None,
|
_ => None,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -697,30 +718,44 @@ fn param_declarations_range(
|
|||||||
/// A param whose stored regex fails to compile (shouldn't happen — `declare_param` validates it) is skipped rather than panicking.
|
/// A param whose stored regex fails to compile (shouldn't happen — `declare_param` validates it) is skipped rather than panicking.
|
||||||
fn compiled_param_declarations(
|
fn compiled_param_declarations(
|
||||||
index: &BTreeMap<IndexKey, DataValue>,
|
index: &BTreeMap<IndexKey, DataValue>,
|
||||||
) -> Vec<(String, Regex, String, MetricPrimitiveKind)> {
|
) -> Vec<(String, Regex, String, Option<String>, MetricPrimitiveKind)> {
|
||||||
param_declarations_range(index)
|
param_declarations_range(index)
|
||||||
.filter_map(|(name, message_match_regex, capture_group, kind)| {
|
.filter_map(|(name, message_match_regex, capture_group, json_field_path, kind)| {
|
||||||
Regex::new(message_match_regex)
|
Regex::new(message_match_regex).ok().map(|re| {
|
||||||
.ok()
|
(
|
||||||
.map(|re| (name.to_string(), re, capture_group.to_string(), kind))
|
name.to_string(),
|
||||||
|
re,
|
||||||
|
capture_group.to_string(),
|
||||||
|
json_field_path.map(|path| path.to_string()),
|
||||||
|
kind,
|
||||||
|
)
|
||||||
|
})
|
||||||
})
|
})
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A param is omitted (not included as null/empty) when its regex/capture group doesn't match or the captured text fails to parse — see `LogEntryDto::params`.
|
/// A param is omitted (not included as null/empty) when its regex/capture group (or JSON field
|
||||||
|
/// path) doesn't match, or the extracted text fails to parse — see `LogEntryDto::params`.
|
||||||
fn derive_log_params(
|
fn derive_log_params(
|
||||||
text: &str,
|
text: &str,
|
||||||
declarations: &[(String, Regex, String, MetricPrimitiveKind)],
|
declarations: &[(String, Regex, String, Option<String>, MetricPrimitiveKind)],
|
||||||
) -> BTreeMap<String, MetricPrimitive> {
|
) -> BTreeMap<String, MetricPrimitive> {
|
||||||
let mut params = BTreeMap::new();
|
let mut params = BTreeMap::new();
|
||||||
for (name, regex, capture_group, kind) in declarations {
|
for (name, regex, capture_group, json_field_path, kind) in declarations {
|
||||||
let Some(captures) = regex.captures(text) else {
|
let Some(captures) = regex.captures(text) else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
let Some(raw) = captures.name(capture_group) else {
|
let raw: String = match json_field_path.as_deref().filter(|path| !path.is_empty()) {
|
||||||
continue;
|
Some(field_path) => match crate::data_value::extract_json_field(text, field_path) {
|
||||||
|
Some(value) => value,
|
||||||
|
None => continue,
|
||||||
|
},
|
||||||
|
None => match captures.name(capture_group) {
|
||||||
|
Some(m) => m.as_str().to_string(),
|
||||||
|
None => continue,
|
||||||
|
},
|
||||||
};
|
};
|
||||||
let Some(value) = parse_metric_primitive(raw.as_str(), *kind) else {
|
let Some(value) = parse_metric_primitive(&raw, *kind) else {
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
params.insert(name.clone(), value);
|
params.insert(name.clone(), value);
|
||||||
@@ -866,10 +901,11 @@ fn handle_query(
|
|||||||
}
|
}
|
||||||
EngineQuery::ParamDeclarations => {
|
EngineQuery::ParamDeclarations => {
|
||||||
let declarations = param_declarations_range(index)
|
let declarations = param_declarations_range(index)
|
||||||
.map(|(name, message_match_regex, capture_group, kind)| ParamDeclarationDto {
|
.map(|(name, message_match_regex, capture_group, json_field_path, kind)| ParamDeclarationDto {
|
||||||
name: name.to_string(),
|
name: name.to_string(),
|
||||||
message_match_regex: message_match_regex.to_string(),
|
message_match_regex: message_match_regex.to_string(),
|
||||||
capture_group: capture_group.to_string(),
|
capture_group: capture_group.to_string(),
|
||||||
|
json_field_path: json_field_path.map(|path| path.to_string()),
|
||||||
kind,
|
kind,
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
@@ -1066,6 +1102,7 @@ mod tests {
|
|||||||
name: name.to_string(),
|
name: name.to_string(),
|
||||||
message_match_regex: pattern.to_string(),
|
message_match_regex: pattern.to_string(),
|
||||||
capture_group: capture_group.to_string(),
|
capture_group: capture_group.to_string(),
|
||||||
|
json_field_path: None,
|
||||||
kind: MetricPrimitiveKind::String,
|
kind: MetricPrimitiveKind::String,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1102,6 +1139,52 @@ mod tests {
|
|||||||
assert!(index.is_empty());
|
assert!(index.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn declares_valid_param_from_json_field_path() {
|
||||||
|
let mut index = BTreeMap::new();
|
||||||
|
let result = declare_param(
|
||||||
|
&mut index,
|
||||||
|
DeclareParamRequest {
|
||||||
|
name: "level".to_string(),
|
||||||
|
// No named capture group needed for JSON-field extraction, just something that
|
||||||
|
// matches the lines this param applies to.
|
||||||
|
message_match_regex: r".".to_string(),
|
||||||
|
capture_group: String::new(),
|
||||||
|
json_field_path: Some("foo.bar".to_string()),
|
||||||
|
kind: MetricPrimitiveKind::String,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert!(result.is_ok());
|
||||||
|
|
||||||
|
let declarations: Vec<_> = param_declarations_range(&index).collect();
|
||||||
|
assert_eq!(declarations.len(), 1);
|
||||||
|
assert_eq!(declarations[0].3, Some("foo.bar"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn derives_log_param_from_json_field_path() {
|
||||||
|
let mut index = BTreeMap::new();
|
||||||
|
declare_param(
|
||||||
|
&mut index,
|
||||||
|
DeclareParamRequest {
|
||||||
|
name: "bar".to_string(),
|
||||||
|
message_match_regex: r".".to_string(),
|
||||||
|
capture_group: String::new(),
|
||||||
|
json_field_path: Some("foo.bar".to_string()),
|
||||||
|
kind: MetricPrimitiveKind::Number,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let declarations = compiled_param_declarations(&index);
|
||||||
|
let params = derive_log_params(r#"{"foo":{"bar":42}}"#, &declarations);
|
||||||
|
assert_eq!(params.get("bar"), Some(&MetricPrimitive::Number(Number::Int(42))));
|
||||||
|
|
||||||
|
// A line that doesn't parse as JSON, or is missing the field, yields no param.
|
||||||
|
assert!(derive_log_params("not json", &declarations).is_empty());
|
||||||
|
assert!(derive_log_params(r#"{"foo":{}}"#, &declarations).is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn evict_excess_logs_drops_oldest_sequences_first() {
|
fn evict_excess_logs_drops_oldest_sequences_first() {
|
||||||
let mut index = BTreeMap::new();
|
let mut index = BTreeMap::new();
|
||||||
@@ -1215,6 +1298,7 @@ mod tests {
|
|||||||
.map(|(param_name, capture_group, kind, role)| RegexToMetricPrimitiveMatcher {
|
.map(|(param_name, capture_group, kind, role)| RegexToMetricPrimitiveMatcher {
|
||||||
param_name: param_name.to_string(),
|
param_name: param_name.to_string(),
|
||||||
capture_group: capture_group.to_string(),
|
capture_group: capture_group.to_string(),
|
||||||
|
json_field_path: None,
|
||||||
kind,
|
kind,
|
||||||
role,
|
role,
|
||||||
})
|
})
|
||||||
@@ -1349,6 +1433,41 @@ mod tests {
|
|||||||
assert_eq!(values.len(), 2);
|
assert_eq!(values.len(), 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn metric_param_extracts_from_json_field_path() {
|
||||||
|
let mut index = BTreeMap::new();
|
||||||
|
ingest_log(&mut index, 0, r#"{"request":{"path":"/a"}}"#);
|
||||||
|
// Doesn't parse as JSON at all — should be skipped, not panic.
|
||||||
|
ingest_log(&mut index, 1, "not json");
|
||||||
|
// Parses as JSON but is missing the field — also skipped.
|
||||||
|
ingest_log(&mut index, 2, r#"{"request":{}}"#);
|
||||||
|
|
||||||
|
declare_metric(
|
||||||
|
&mut index,
|
||||||
|
DeclareMetricRequest {
|
||||||
|
name: "requests".to_string(),
|
||||||
|
message_match_regex: r".".to_string(),
|
||||||
|
matched_labels: None,
|
||||||
|
unmatched_labels: None,
|
||||||
|
params: vec![RegexToMetricPrimitiveMatcher {
|
||||||
|
param_name: "path".to_string(),
|
||||||
|
capture_group: String::new(),
|
||||||
|
json_field_path: Some("request.path".to_string()),
|
||||||
|
kind: MetricPrimitiveKind::String,
|
||||||
|
role: ParamRole::Label,
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let values: Vec<_> = bucket_values(&index, "requests", None).collect();
|
||||||
|
assert_eq!(values.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
values[0].0.get("path"),
|
||||||
|
Some(&MetricPrimitive::String("/a".to_string()))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn ingesting_after_declaration_updates_buckets_live() {
|
fn ingesting_after_declaration_updates_buckets_live() {
|
||||||
let mut index = BTreeMap::new();
|
let mut index = BTreeMap::new();
|
||||||
|
|||||||
@@ -60,6 +60,10 @@ pub struct ParamDeclarationRecord {
|
|||||||
pub name: String,
|
pub name: String,
|
||||||
pub message_match_regex: String,
|
pub message_match_regex: String,
|
||||||
pub capture_group: String,
|
pub capture_group: String,
|
||||||
|
/// See `crate::data_value::RegexToMetricPrimitiveMatcher::json_field_path`; `#[serde(default)]`
|
||||||
|
/// so profiles saved before this field existed still load as plain regex-capture-group params.
|
||||||
|
#[serde(default)]
|
||||||
|
pub json_field_path: Option<String>,
|
||||||
pub kind: ProfileMetricPrimitiveKind,
|
pub kind: ProfileMetricPrimitiveKind,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -69,6 +73,7 @@ impl From<&DeclareParamRequest> for ParamDeclarationRecord {
|
|||||||
name: request.name.clone(),
|
name: request.name.clone(),
|
||||||
message_match_regex: request.message_match_regex.clone(),
|
message_match_regex: request.message_match_regex.clone(),
|
||||||
capture_group: request.capture_group.clone(),
|
capture_group: request.capture_group.clone(),
|
||||||
|
json_field_path: request.json_field_path.clone(),
|
||||||
kind: request.kind.into(),
|
kind: request.kind.into(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -80,6 +85,7 @@ impl From<&ParamDeclarationRecord> for DeclareParamRequest {
|
|||||||
name: record.name.clone(),
|
name: record.name.clone(),
|
||||||
message_match_regex: record.message_match_regex.clone(),
|
message_match_regex: record.message_match_regex.clone(),
|
||||||
capture_group: record.capture_group.clone(),
|
capture_group: record.capture_group.clone(),
|
||||||
|
json_field_path: record.json_field_path.clone(),
|
||||||
kind: record.kind.into(),
|
kind: record.kind.into(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -90,6 +96,10 @@ impl From<&ParamDeclarationRecord> for DeclareParamRequest {
|
|||||||
pub struct MetricParamRecord {
|
pub struct MetricParamRecord {
|
||||||
pub param_name: String,
|
pub param_name: String,
|
||||||
pub capture_group: String,
|
pub capture_group: String,
|
||||||
|
/// See `crate::data_value::RegexToMetricPrimitiveMatcher::json_field_path`; `#[serde(default)]`
|
||||||
|
/// so profiles saved before this field existed still load as plain regex-capture-group matchers.
|
||||||
|
#[serde(default)]
|
||||||
|
pub json_field_path: Option<String>,
|
||||||
pub kind: ProfileMetricPrimitiveKind,
|
pub kind: ProfileMetricPrimitiveKind,
|
||||||
/// See `ParamRole`; `#[serde(default)]` so profiles saved before this field existed still load, defaulting to `Label`.
|
/// See `ParamRole`; `#[serde(default)]` so profiles saved before this field existed still load, defaulting to `Label`.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
@@ -121,6 +131,7 @@ impl From<&DeclareMetricRequest> for MetricDeclarationRecord {
|
|||||||
.map(|matcher| MetricParamRecord {
|
.map(|matcher| MetricParamRecord {
|
||||||
param_name: matcher.param_name.clone(),
|
param_name: matcher.param_name.clone(),
|
||||||
capture_group: matcher.capture_group.clone(),
|
capture_group: matcher.capture_group.clone(),
|
||||||
|
json_field_path: matcher.json_field_path.clone(),
|
||||||
kind: matcher.kind.into(),
|
kind: matcher.kind.into(),
|
||||||
role: matcher.role,
|
role: matcher.role,
|
||||||
})
|
})
|
||||||
@@ -144,6 +155,7 @@ impl From<&MetricDeclarationRecord> for DeclareMetricRequest {
|
|||||||
.map(|param| RegexToMetricPrimitiveMatcher {
|
.map(|param| RegexToMetricPrimitiveMatcher {
|
||||||
param_name: param.param_name.clone(),
|
param_name: param.param_name.clone(),
|
||||||
capture_group: param.capture_group.clone(),
|
capture_group: param.capture_group.clone(),
|
||||||
|
json_field_path: param.json_field_path.clone(),
|
||||||
kind: param.kind.into(),
|
kind: param.kind.into(),
|
||||||
role: param.role,
|
role: param.role,
|
||||||
})
|
})
|
||||||
@@ -626,6 +638,7 @@ mod tests {
|
|||||||
name: name.to_string(),
|
name: name.to_string(),
|
||||||
message_match_regex: pattern.to_string(),
|
message_match_regex: pattern.to_string(),
|
||||||
capture_group: capture_group.to_string(),
|
capture_group: capture_group.to_string(),
|
||||||
|
json_field_path: None,
|
||||||
kind: MetricPrimitiveKind::String,
|
kind: MetricPrimitiveKind::String,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -639,6 +652,7 @@ mod tests {
|
|||||||
params: vec![RegexToMetricPrimitiveMatcher {
|
params: vec![RegexToMetricPrimitiveMatcher {
|
||||||
param_name: param_name.to_string(),
|
param_name: param_name.to_string(),
|
||||||
capture_group: capture_group.to_string(),
|
capture_group: capture_group.to_string(),
|
||||||
|
json_field_path: None,
|
||||||
kind: MetricPrimitiveKind::Number,
|
kind: MetricPrimitiveKind::Number,
|
||||||
role: ParamRole::Label,
|
role: ParamRole::Label,
|
||||||
}],
|
}],
|
||||||
|
|||||||
@@ -1,10 +0,0 @@
|
|||||||
# Allow params/metrics to be declared from a JSON field path
|
|
||||||
|
|
||||||
Follow-up to the JSON log ingestion design decision (repowise decision fdb47034ad954e3ea0b898fdc2750c32).
|
|
||||||
|
|
||||||
Extend param/metric declaration so a param can be extracted from a JSON field path (e.g. `foo.bar`) on lines that parse as JSON objects, as an alternative to the existing regex-capture-group extraction.
|
|
||||||
|
|
||||||
## Acceptance criteria
|
|
||||||
- `RegexToMetricPrimitiveMatcher` (src/data_value.rs) or a sibling type supports a JSON-field-path extraction source alongside the existing regex-capture-group source, reusing `ParamRole`/`MetricPrimitiveKind`.
|
|
||||||
- The params declaration UI (Params.tsx) lets a user choose "JSON field" as an alternative to "regex capture group" when declaring a param, for lines detected as JSON.
|
|
||||||
- Existing regex-based params/metrics are unaffected.
|
|
||||||
@@ -1,28 +1,59 @@
|
|||||||
import { useState } from 'react'
|
import { useMemo, useState } from 'react'
|
||||||
import { ApiError, declareParam, type ParamDeclaration } from '../lib/api'
|
import { ApiError, declareParam, type ParamDeclaration } from '../lib/api'
|
||||||
import { CollapseToggle, PopOutButton, sectionHeaderClassName, sectionHeaderLeftClassName, useSectionCollapsed } from './CollapsibleSection'
|
import { CollapseToggle, PopOutButton, sectionHeaderClassName, sectionHeaderLeftClassName, useSectionCollapsed } from './CollapsibleSection'
|
||||||
import { HelpIcon } from './HelpIcon'
|
import { HelpIcon } from './HelpIcon'
|
||||||
|
|
||||||
const EMPTY_KIND: ParamDeclaration['kind'] = 'string'
|
const EMPTY_KIND: ParamDeclaration['kind'] = 'string'
|
||||||
|
|
||||||
|
type ExtractionSource = 'regex' | 'json_field'
|
||||||
|
|
||||||
|
/// A JSON *object* (not an array/scalar) parses cleanly and pretty-prints with braces — the same
|
||||||
|
/// shape a JSON field path can be read from, so this is the "line looks like JSON" gate that
|
||||||
|
/// unlocks the "JSON field" extraction option.
|
||||||
|
function looksLikeJsonObject(text: string): boolean {
|
||||||
|
const trimmed = text.trim()
|
||||||
|
if (!trimmed.startsWith('{')) return false
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(trimmed)
|
||||||
|
return typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)
|
||||||
|
} catch {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export function Params({ params, onDeclared }: { params: ParamDeclaration[]; onDeclared: () => void }) {
|
export function Params({ params, onDeclared }: { params: ParamDeclaration[]; onDeclared: () => void }) {
|
||||||
const [name, setName] = useState('')
|
const [name, setName] = useState('')
|
||||||
const [regex, setRegex] = useState('')
|
const [regex, setRegex] = useState('')
|
||||||
const [captureGroup, setCaptureGroup] = useState('')
|
const [captureGroup, setCaptureGroup] = useState('')
|
||||||
|
const [jsonFieldPath, setJsonFieldPath] = useState('')
|
||||||
|
const [source, setSource] = useState<ExtractionSource>('regex')
|
||||||
const [kind, setKind] = useState<ParamDeclaration['kind']>(EMPTY_KIND)
|
const [kind, setKind] = useState<ParamDeclaration['kind']>(EMPTY_KIND)
|
||||||
const [submitError, setSubmitError] = useState<string | null>(null)
|
const [submitError, setSubmitError] = useState<string | null>(null)
|
||||||
const [submitting, setSubmitting] = useState(false)
|
const [submitting, setSubmitting] = useState(false)
|
||||||
const [collapsed, setCollapsed] = useSectionCollapsed('params')
|
const [collapsed, setCollapsed] = useSectionCollapsed('params')
|
||||||
|
|
||||||
|
// The message match regex doubles as a sample line here so a pasted JSON log line unlocks the
|
||||||
|
// "JSON field" extraction option — regex-based params are unaffected either way.
|
||||||
|
const detectedJson = useMemo(() => looksLikeJsonObject(regex), [regex])
|
||||||
|
const effectiveSource: ExtractionSource = detectedJson ? source : 'regex'
|
||||||
|
|
||||||
async function handleSubmit(e: React.FormEvent) {
|
async function handleSubmit(e: React.FormEvent) {
|
||||||
e.preventDefault()
|
e.preventDefault()
|
||||||
setSubmitError(null)
|
setSubmitError(null)
|
||||||
setSubmitting(true)
|
setSubmitting(true)
|
||||||
try {
|
try {
|
||||||
await declareParam({ name, message_match_regex: regex, capture_group: captureGroup, kind })
|
await declareParam({
|
||||||
|
name,
|
||||||
|
message_match_regex: regex,
|
||||||
|
capture_group: effectiveSource === 'json_field' ? '' : captureGroup,
|
||||||
|
json_field_path: effectiveSource === 'json_field' ? jsonFieldPath : null,
|
||||||
|
kind,
|
||||||
|
})
|
||||||
setName('')
|
setName('')
|
||||||
setRegex('')
|
setRegex('')
|
||||||
setCaptureGroup('')
|
setCaptureGroup('')
|
||||||
|
setJsonFieldPath('')
|
||||||
|
setSource('regex')
|
||||||
setKind(EMPTY_KIND)
|
setKind(EMPTY_KIND)
|
||||||
onDeclared()
|
onDeclared()
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -66,6 +97,43 @@ export function Params({ params, onDeclared }: { params: ParamDeclaration[]; onD
|
|||||||
required
|
required
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
{detectedJson && (
|
||||||
|
<div>
|
||||||
|
<label className="mb-1 block text-xs text-text-subtle">Extraction</label>
|
||||||
|
<select
|
||||||
|
className="w-full rounded border border-border-soft bg-surface-1 px-2 py-1 text-xs text-text focus:border-border-strong focus:outline-none"
|
||||||
|
value={source}
|
||||||
|
onChange={(e) => setSource(e.target.value as ExtractionSource)}
|
||||||
|
>
|
||||||
|
<option value="regex">Regex capture group</option>
|
||||||
|
<option value="json_field">JSON field</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{effectiveSource === 'json_field' ? (
|
||||||
|
<div className="flex items-end gap-1">
|
||||||
|
<div className="min-w-0 flex-1">
|
||||||
|
<label className="mb-1 block text-xs text-text-subtle">JSON field path</label>
|
||||||
|
<input
|
||||||
|
className="w-full min-w-0 rounded border border-border-soft bg-surface-1 px-1.5 py-1 font-mono text-xs text-text placeholder:text-text-subtle focus:border-border-strong focus:outline-none"
|
||||||
|
value={jsonFieldPath}
|
||||||
|
onChange={(e) => setJsonFieldPath(e.target.value)}
|
||||||
|
placeholder="foo.bar"
|
||||||
|
required
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<select
|
||||||
|
className="rounded border border-border-soft bg-surface-1 px-1 py-1 text-xs text-text focus:border-border-strong focus:outline-none"
|
||||||
|
value={kind}
|
||||||
|
onChange={(e) => setKind(e.target.value as ParamDeclaration['kind'])}
|
||||||
|
>
|
||||||
|
<option value="string">string</option>
|
||||||
|
<option value="number">number</option>
|
||||||
|
<option value="timestamp">timestamp</option>
|
||||||
|
<option value="duration">duration</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
<div className="flex items-end gap-1">
|
<div className="flex items-end gap-1">
|
||||||
<div className="min-w-0 flex-1">
|
<div className="min-w-0 flex-1">
|
||||||
<label className="mb-1 block text-xs text-text-subtle">Capture group</label>
|
<label className="mb-1 block text-xs text-text-subtle">Capture group</label>
|
||||||
@@ -88,6 +156,7 @@ export function Params({ params, onDeclared }: { params: ParamDeclaration[]; onD
|
|||||||
<option value="duration">duration</option>
|
<option value="duration">duration</option>
|
||||||
</select>
|
</select>
|
||||||
</div>
|
</div>
|
||||||
|
)}
|
||||||
{submitError && <div className="rounded border border-danger-border bg-danger-bg/50 px-2 py-1 text-xs text-danger-text">{submitError}</div>}
|
{submitError && <div className="rounded border border-danger-border bg-danger-bg/50 px-2 py-1 text-xs text-danger-text">{submitError}</div>}
|
||||||
<button
|
<button
|
||||||
type="submit"
|
type="submit"
|
||||||
@@ -108,7 +177,7 @@ export function Params({ params, onDeclared }: { params: ParamDeclaration[]; onD
|
|||||||
className="rounded border border-border-soft px-2 py-1 text-xs text-text-secondary"
|
className="rounded border border-border-soft px-2 py-1 text-xs text-text-secondary"
|
||||||
title={p.message_match_regex}
|
title={p.message_match_regex}
|
||||||
>
|
>
|
||||||
{p.name} ({p.kind}, group {p.capture_group})
|
{p.name} ({p.kind}, {p.json_field_path ? `json field ${p.json_field_path}` : `group ${p.capture_group}`})
|
||||||
</span>
|
</span>
|
||||||
))}
|
))}
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -25,6 +25,8 @@ export interface ParamDeclaration {
|
|||||||
name: string
|
name: string
|
||||||
message_match_regex: string
|
message_match_regex: string
|
||||||
capture_group: string
|
capture_group: string
|
||||||
|
/** Alternative to `capture_group`: a dot-separated JSON field path (e.g. "foo.bar"), read from lines that parse as a JSON object. When set (non-empty), takes precedence over `capture_group`. */
|
||||||
|
json_field_path?: string | null
|
||||||
kind: 'string' | 'number' | 'timestamp' | 'duration'
|
kind: 'string' | 'number' | 'timestamp' | 'duration'
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -48,6 +50,8 @@ export type ParamRole = 'label' | 'value'
|
|||||||
export interface MetricParamMatcher {
|
export interface MetricParamMatcher {
|
||||||
param_name: string
|
param_name: string
|
||||||
capture_group: string
|
capture_group: string
|
||||||
|
/** See `ParamDeclaration.json_field_path`. */
|
||||||
|
json_field_path?: string | null
|
||||||
kind: 'string' | 'number' | 'timestamp' | 'duration'
|
kind: 'string' | 'number' | 'timestamp' | 'duration'
|
||||||
/** Defaults to 'label' server-side when omitted. */
|
/** Defaults to 'label' server-side when omitted. */
|
||||||
role?: ParamRole
|
role?: ParamRole
|
||||||
|
|||||||
Reference in New Issue
Block a user