Merge branch 'worktree-agent-ab342356b0191d35f'
# Conflicts: # src/engine.rs
This commit is contained in:
@@ -375,6 +375,49 @@ fn param_declarations_range(
|
||||
})
|
||||
}
|
||||
|
||||
/// Compiles every currently-declared param's regex once, so `derive_log_params`
|
||||
/// can be applied to many lines without recompiling each param's regex per
|
||||
/// line. A param whose stored regex somehow fails to compile (shouldn't
|
||||
/// happen — `declare_param` validates it before storing) is skipped rather
|
||||
/// than panicking.
|
||||
fn compiled_param_declarations(
|
||||
index: &BTreeMap<IndexKey, DataValue>,
|
||||
) -> Vec<(String, Regex, String, MetricPrimitiveKind)> {
|
||||
param_declarations_range(index)
|
||||
.filter_map(|(name, message_match_regex, capture_group, kind)| {
|
||||
Regex::new(message_match_regex)
|
||||
.ok()
|
||||
.map(|re| (name.to_string(), re, capture_group.to_string(), kind))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Runs every declared param's regex against one log line's text, returning
|
||||
/// the derived `MetricPrimitive` values keyed by param name. A param is
|
||||
/// omitted from the result (rather than included as null/empty) when its
|
||||
/// regex doesn't match the line, its capture group doesn't match, or the
|
||||
/// captured text fails to parse as the param's declared `kind` — see
|
||||
/// `LogEntryDto::params`.
|
||||
fn derive_log_params(
|
||||
text: &str,
|
||||
declarations: &[(String, Regex, String, MetricPrimitiveKind)],
|
||||
) -> BTreeMap<String, MetricPrimitive> {
|
||||
let mut params = BTreeMap::new();
|
||||
for (name, regex, capture_group, kind) in declarations {
|
||||
let Some(captures) = regex.captures(text) else {
|
||||
continue;
|
||||
};
|
||||
let Some(raw) = captures.name(capture_group) else {
|
||||
continue;
|
||||
};
|
||||
let Some(value) = parse_metric_primitive(raw.as_str(), *kind) else {
|
||||
continue;
|
||||
};
|
||||
params.insert(name.clone(), value);
|
||||
}
|
||||
params
|
||||
}
|
||||
|
||||
fn logs_range(index: &BTreeMap<IndexKey, DataValue>) -> impl Iterator<Item = (u64, Timestamp, &str)> {
|
||||
index.range(IndexKey::Start..IndexKey::LogParamSplit).filter_map(|(key, value)| {
|
||||
match (key, value) {
|
||||
@@ -390,6 +433,17 @@ fn handle_query(index: &BTreeMap<IndexKey, DataValue>, query: EngineQuery) -> En
|
||||
match query {
|
||||
EngineQuery::Logs(params) => {
|
||||
let limit = params.limit.unwrap_or(500).min(10_000);
|
||||
// Compile each currently-declared param's regex once up front, then
|
||||
// apply the compiled set to every *returned* line below (ticket 015).
|
||||
// Perf note: this is a straightforward per-returned-line,
|
||||
// per-declared-param regex scan — O(lines_returned * declared_params)
|
||||
// — rather than precomputing/caching derived params for the whole log
|
||||
// range. That's fine at this codebase's current scale (v1), but if
|
||||
// either the log volume or number of declared params grows a lot,
|
||||
// this should move to incremental derivation at ingest time (mirroring
|
||||
// how `apply_all_metrics` already does for metric buckets) instead of
|
||||
// recomputing on every `/api/logs` query.
|
||||
let compiled_params = compiled_param_declarations(index);
|
||||
let entries: Vec<LogEntryDto> = logs_range(index)
|
||||
.filter(|(sequence, timestamp, _)| {
|
||||
params.since_sequence.is_none_or(|since| *sequence > since)
|
||||
@@ -401,6 +455,7 @@ fn handle_query(index: &BTreeMap<IndexKey, DataValue>, query: EngineQuery) -> En
|
||||
sequence,
|
||||
timestamp_ms: timestamp.as_millis(),
|
||||
text: text.to_string(),
|
||||
params: derive_log_params(text, &compiled_params),
|
||||
})
|
||||
.collect();
|
||||
EngineResponse::Logs(entries)
|
||||
@@ -1085,4 +1140,30 @@ mod tests {
|
||||
_ => panic!("expected EngineResponse::Promql(Ok(_))"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn logs_query_includes_declared_params_on_matching_lines_only() {
|
||||
let mut index = BTreeMap::new();
|
||||
declare_param(&mut index, request("level", r"level=(?P<level>\w+)", "level")).unwrap();
|
||||
ingest_log(&mut index, 0, "level=info hello");
|
||||
ingest_log(&mut index, 1, "no match here");
|
||||
|
||||
let EngineResponse::Logs(entries) =
|
||||
handle_query(&index, EngineQuery::Logs(LogsParams {
|
||||
since_sequence: None,
|
||||
start_ms: None,
|
||||
end_ms: None,
|
||||
limit: None,
|
||||
}))
|
||||
else {
|
||||
panic!("expected Logs response");
|
||||
};
|
||||
|
||||
assert_eq!(entries.len(), 2);
|
||||
assert_eq!(
|
||||
entries[0].params.get("level"),
|
||||
Some(&MetricPrimitive::String("info".to_string()))
|
||||
);
|
||||
assert!(entries[1].params.get("level").is_none());
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user