phughk/ottyl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge ticket/015-json-field-param-metric-declaration

Browse Source 
Resolves conflicts with ticket 018 (boolean param kind) in
src/engine.rs, webapp/src/lib/api.ts, and Params.tsx: boolean-kind
params and JSON-field-path extraction now coexist as independent
extraction modes alongside regex capture groups.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XFUNZwCvz3kcod3352ZR2o
This commit is contained in:
Hugh Kaznowski
2026-09-05 19:41:36 +01:00
co-authored by Claude Sonnet 5
parent ffb164b24e b3e0272544
commit 47586bf31a
8 changed files with 297 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/engine.rs
+142 -22
View File
@@ -38,6 +38,10 @@ pub struct DeclareParamRequest {
pub name: String,
pub message_match_regex: String,
pub capture_group: String,
/// See `RegexToMetricPrimitiveMatcher::json_field_path`; `#[serde(default)]` so existing
/// regex-only callers are unaffected.
#[serde(default)]
pub json_field_path: Option<String>,
pub kind: MetricPrimitiveKind,
}
@@ -208,7 +212,9 @@ fn declare_param(
let re = Regex::new(&request.message_match_regex)
.map_err(|err| format!("invalid regex: {err}"))?;
// A Boolean-kind param never uses a capture group — its value is just whether the regex matched.
let uses_json_field = request.json_field_path.as_deref().is_some_and(|path| !path.is_empty());
if request.kind != MetricPrimitiveKind::Boolean
&& !uses_json_field
&& !re.capture_names().flatten().any(|name| name == request.capture_group)
{
return Err(format!(
@@ -222,6 +228,7 @@ fn declare_param(
DataValue::ParamDeclaration {
message_match_regex: request.message_match_regex,
capture_group: request.capture_group,
json_field_path: request.json_field_path,
kind: request.kind,
},
);
@@ -240,7 +247,9 @@ fn declare_metric(
.params
.iter()
.filter(|matcher| {
!re.capture_names().flatten().any(|name| name == matcher.capture_group)
let uses_json_field =
matcher.json_field_path.as_deref().is_some_and(|path| !path.is_empty());
!uses_json_field && !re.capture_names().flatten().any(|name| name == matcher.capture_group)
})
.map(|matcher| {
format!(
@@ -331,14 +340,21 @@ fn apply_metric_to_log(
let mut value = None;
let mut is_duration = false;
for matcher in params {
let Some(raw) = captures.name(&matcher.capture_group) else {
return;
let raw: String = match matcher.json_field_path.as_deref().filter(|path| !path.is_empty()) {
Some(field_path) => match crate::data_value::extract_json_field(text, field_path) {
Some(value) => value,
None => return,
},
None => match captures.name(&matcher.capture_group) {
Some(m) => m.as_str().to_string(),
None => return,
},
};
if matcher.role == ParamRole::Value && matcher.kind == MetricPrimitiveKind::Duration {
is_duration = true;
continue;
}
let Some(parsed) = parse_metric_primitive(raw.as_str(), matcher.kind) else {
let Some(parsed) = parse_metric_primitive(&raw, matcher.kind) else {
return;
};
match matcher.role {
@@ -696,14 +712,20 @@ fn metric_declarations_range(
fn param_declarations_range(
index: &BTreeMap<IndexKey, DataValue>,
) -> impl Iterator<Item = (&str, &str, &str, MetricPrimitiveKind)> {
) -> impl Iterator<Item = (&str, &str, &str, Option<&str>, MetricPrimitiveKind)> {
index
.range(IndexKey::LogParamSplit..IndexKey::ParamMetricSplit)
.filter_map(|(key, value)| match (key, value) {
(
IndexKey::ParamDeclaration { name },
DataValue::ParamDeclaration { message_match_regex, capture_group, kind },
) => Some((name.as_str(), message_match_regex.as_str(), capture_group.as_str(), *kind)),
DataValue::ParamDeclaration { message_match_regex, capture_group, json_field_path, kind },
) => Some((
name.as_str(),
message_match_regex.as_str(),
capture_group.as_str(),
json_field_path.as_deref(),
*kind,
)),
_ => None,
})
}
@@ -711,26 +733,33 @@ fn param_declarations_range(
/// A param whose stored regex fails to compile (shouldn't happen — `declare_param` validates it) is skipped rather than panicking.
fn compiled_param_declarations(
index: &BTreeMap<IndexKey, DataValue>,
) -> Vec<(String, Regex, String, MetricPrimitiveKind)> {
) -> Vec<(String, Regex, String, Option<String>, MetricPrimitiveKind)> {
param_declarations_range(index)
.filter_map(|(name, message_match_regex, capture_group, kind)| {
Regex::new(message_match_regex)
.ok()
.map(|re| (name.to_string(), re, capture_group.to_string(), kind))
.filter_map(|(name, message_match_regex, capture_group, json_field_path, kind)| {
Regex::new(message_match_regex).ok().map(|re| {
(
name.to_string(),
re,
capture_group.to_string(),
json_field_path.map(|path| path.to_string()),
kind,
)
})
})
.collect()
}
/// A param is omitted (not included as null/empty) when its regex/capture group doesn't match or the
/// captured text fails to parse — see `LogEntryDto::params`. The one exception is a `Boolean`-kind param,
/// which never uses a capture group and is instead present as `true`/`false` on every line based purely on
/// whether `message_match_regex` matches (case-insensitively or per the regex's own flags).
/// A param is omitted (not included as null/empty) when its regex/capture group (or JSON field
/// path) doesn't match, or the extracted text fails to parse — see `LogEntryDto::params`. The one
/// exception is a `Boolean`-kind param, which never uses a capture group and is instead present as
/// `true`/`false` on every line based purely on whether `message_match_regex` matches
/// (case-insensitively or per the regex's own flags).
fn derive_log_params(
text: &str,
declarations: &[(String, Regex, String, MetricPrimitiveKind)],
declarations: &[(String, Regex, String, Option<String>, MetricPrimitiveKind)],
) -> BTreeMap<String, MetricPrimitive> {
let mut params = BTreeMap::new();
for (name, regex, capture_group, kind) in declarations {
for (name, regex, capture_group, json_field_path, kind) in declarations {
if *kind == MetricPrimitiveKind::Boolean {
params.insert(name.clone(), MetricPrimitive::Boolean(regex.is_match(text)));
continue;
@@ -738,10 +767,17 @@ fn derive_log_params(
let Some(captures) = regex.captures(text) else {
continue;
};
let Some(raw) = captures.name(capture_group) else {
continue;
let raw: String = match json_field_path.as_deref().filter(|path| !path.is_empty()) {
Some(field_path) => match crate::data_value::extract_json_field(text, field_path) {
Some(value) => value,
None => continue,
},
None => match captures.name(capture_group) {
Some(m) => m.as_str().to_string(),
None => continue,
},
};
let Some(value) = parse_metric_primitive(raw.as_str(), *kind) else {
let Some(value) = parse_metric_primitive(&raw, *kind) else {
continue;
};
params.insert(name.clone(), value);
@@ -887,10 +923,11 @@ fn handle_query(
}
EngineQuery::ParamDeclarations => {
let declarations = param_declarations_range(index)
.map(|(name, message_match_regex, capture_group, kind)| ParamDeclarationDto {
.map(|(name, message_match_regex, capture_group, json_field_path, kind)| ParamDeclarationDto {
name: name.to_string(),
message_match_regex: message_match_regex.to_string(),
capture_group: capture_group.to_string(),
json_field_path: json_field_path.map(|path| path.to_string()),
kind,
})
.collect();
@@ -1094,6 +1131,7 @@ mod tests {
name: name.to_string(),
message_match_regex: pattern.to_string(),
capture_group: capture_group.to_string(),
json_field_path: None,
kind: MetricPrimitiveKind::String,
}
}
@@ -1130,6 +1168,52 @@ mod tests {
assert!(index.is_empty());
}
#[test]
fn declares_valid_param_from_json_field_path() {
let mut index = BTreeMap::new();
let result = declare_param(
&mut index,
DeclareParamRequest {
name: "level".to_string(),
// No named capture group needed for JSON-field extraction, just something that
// matches the lines this param applies to.
message_match_regex: r".".to_string(),
capture_group: String::new(),
json_field_path: Some("foo.bar".to_string()),
kind: MetricPrimitiveKind::String,
},
);
assert!(result.is_ok());
let declarations: Vec<_> = param_declarations_range(&index).collect();
assert_eq!(declarations.len(), 1);
assert_eq!(declarations[0].3, Some("foo.bar"));
}
#[test]
fn derives_log_param_from_json_field_path() {
let mut index = BTreeMap::new();
declare_param(
&mut index,
DeclareParamRequest {
name: "bar".to_string(),
message_match_regex: r".".to_string(),
capture_group: String::new(),
json_field_path: Some("foo.bar".to_string()),
kind: MetricPrimitiveKind::Number,
},
)
.unwrap();
let declarations = compiled_param_declarations(&index);
let params = derive_log_params(r#"{"foo":{"bar":42}}"#, &declarations);
assert_eq!(params.get("bar"), Some(&MetricPrimitive::Number(Number::Int(42))));
// A line that doesn't parse as JSON, or is missing the field, yields no param.
assert!(derive_log_params("not json", &declarations).is_empty());
assert!(derive_log_params(r#"{"foo":{}}"#, &declarations).is_empty());
}
#[test]
fn evict_excess_logs_drops_oldest_sequences_first() {
let mut index = BTreeMap::new();
@@ -1243,6 +1327,7 @@ mod tests {
.map(|(param_name, capture_group, kind, role)| RegexToMetricPrimitiveMatcher {
param_name: param_name.to_string(),
capture_group: capture_group.to_string(),
json_field_path: None,
kind,
role,
})
@@ -1377,6 +1462,41 @@ mod tests {
assert_eq!(values.len(), 2);
}
#[test]
fn metric_param_extracts_from_json_field_path() {
let mut index = BTreeMap::new();
ingest_log(&mut index, 0, r#"{"request":{"path":"/a"}}"#);
// Doesn't parse as JSON at all — should be skipped, not panic.
ingest_log(&mut index, 1, "not json");
// Parses as JSON but is missing the field — also skipped.
ingest_log(&mut index, 2, r#"{"request":{}}"#);
declare_metric(
&mut index,
DeclareMetricRequest {
name: "requests".to_string(),
message_match_regex: r".".to_string(),
matched_labels: None,
unmatched_labels: None,
params: vec![RegexToMetricPrimitiveMatcher {
param_name: "path".to_string(),
capture_group: String::new(),
json_field_path: Some("request.path".to_string()),
kind: MetricPrimitiveKind::String,
role: ParamRole::Label,
}],
},
)
.unwrap();
let values: Vec<_> = bucket_values(&index, "requests", None).collect();
assert_eq!(values.len(), 1);
assert_eq!(
values[0].0.get("path"),
Some(&MetricPrimitive::String("/a".to_string()))
);
}
#[test]
fn ingesting_after_declaration_updates_buckets_live() {
let mut index = BTreeMap::new();