Trim comments across Rust and webapp source to direct one-liners (ticket 030)
Pass over src/*.rs, webapp/src/**, and the in-app "?" docs prose: condensed multi-paragraph doc comments to single lines that state the non-obvious why, deleted comments that only restated adjacent code, and tightened the Docs.tsx help-panel wording. No logic, structure, or code formatting changed — verified via cargo test (143 passed) and tsc -b && vite build. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01XFUNZwCvz3kcod3352ZR2o
This commit is contained in:
Hugh Kaznowski
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
3cef6dc8ae
commit
00c21c8032
+43
-290
@@ -23,9 +23,7 @@ pub struct IngestedLine {
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
/// Also the request body for `POST /api/params` (ticket 004): `Deserialize` is
|
||||
/// derived directly on this engine type rather than adding a separate DTO,
|
||||
/// since its shape is already exactly what the API needs.
|
||||
/// Also the request body for `POST /api/params` — `Deserialize` is derived directly here rather than adding a separate DTO, since the shape already matches.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct DeclareParamRequest {
|
||||
pub name: String,
|
||||
@@ -34,43 +32,16 @@ pub struct DeclareParamRequest {
|
||||
pub kind: MetricPrimitiveKind,
|
||||
}
|
||||
|
||||
/// v1 mechanism: a metric declaration's params are derived purely from inline named
|
||||
/// capture groups on the metric's own `message_match_regex` (each
|
||||
/// `RegexToMetricPrimitiveMatcher::capture_group` must name a capture group on that
|
||||
/// regex). Referencing an already-declared `ParamDeclaration` (001) by name is not
|
||||
/// supported yet — that's left for a future ticket.
|
||||
///
|
||||
/// Also the request body for `POST /api/metrics` (ticket 004) — see
|
||||
/// `DeclareParamRequest`'s doc comment for why `Deserialize` lives here rather
|
||||
/// than on a separate DTO.
|
||||
/// Params are derived purely from inline named capture groups on this metric's own regex — referencing an already-declared `ParamDeclaration` by name isn't supported yet. Also the request body for `POST /api/metrics` (hence `Deserialize`).
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct DeclareMetricRequest {
|
||||
pub name: String,
|
||||
pub message_match_regex: String,
|
||||
pub params: Vec<RegexToMetricPrimitiveMatcher>,
|
||||
/// Ticket 025: fixed labels added on top of whatever `params` captured
|
||||
/// when the line *does* match `message_match_regex`. Together with
|
||||
/// `unmatched_labels` below, this is what lets a single metric+query
|
||||
/// produce a "matched X" vs "didn't match X" pair of series (e.g. the
|
||||
/// seeded default dashboard's error/non-error split) with clean,
|
||||
/// consistent label values, rather than whatever case/substring the
|
||||
/// regex happened to capture (a case-insensitive `error` match could
|
||||
/// capture `"ERROR"`, `"Error"`, etc., fragmenting what should be one
|
||||
/// series into several). `#[serde(default)]` so existing declarations
|
||||
/// without this field are unaffected.
|
||||
/// Fixed labels layered on top of what `params` captured, so e.g. a case-insensitive match doesn't fragment into "ERROR"/"Error" series; `#[serde(default)]` so existing declarations are unaffected.
|
||||
#[serde(default)]
|
||||
pub matched_labels: Option<BTreeMap<String, String>>,
|
||||
/// When a line does *not* match `message_match_regex`, and this is
|
||||
/// `Some`, the metric still registers one occurrence — labeled with
|
||||
/// these fixed values rather than anything captured (there's nothing to
|
||||
/// capture on a non-match) — instead of the line being invisible to this
|
||||
/// metric. This is the other half of the "matched X vs didn't match X"
|
||||
/// pair described above: plain regex alternation can't express the
|
||||
/// "didn't match" side without lookaround (unsupported by the `regex`
|
||||
/// crate), since an alternation branch that's a catch-all always wins at
|
||||
/// the search's leftmost position, so it can never tell "contains X
|
||||
/// later in the line" apart from "X is absent entirely". `#[serde(default)]`
|
||||
/// so existing declarations without this field keep matching only.
|
||||
/// When a line doesn't match, registers one occurrence labeled with these fixed values instead of being invisible to this metric — plain regex alternation can't express "didn't match" without lookaround; `#[serde(default)]` so existing declarations keep matching-only behavior.
|
||||
#[serde(default)]
|
||||
pub unmatched_labels: Option<BTreeMap<String, String>>,
|
||||
}
|
||||
@@ -82,11 +53,7 @@ pub enum EngineQuery {
|
||||
Stats,
|
||||
ParamDeclarations,
|
||||
MetricDeclarations,
|
||||
/// Bucket values for one metric (ticket 004's `GET /api/metrics/:name`).
|
||||
/// `labels_filter`, when present, keeps only buckets where every listed
|
||||
/// label matches — compared against each label value's `Display` form
|
||||
/// (see `MetricPrimitive`) rather than a typed value, since the filter
|
||||
/// arrives as plain strings off the query string.
|
||||
/// `labels_filter` compares against each label's `Display` form (not a typed value), since the filter arrives as plain strings off the query string.
|
||||
BucketValues {
|
||||
metric_name: String,
|
||||
labels_filter: Option<BTreeMap<String, String>>,
|
||||
@@ -95,12 +62,7 @@ pub enum EngineQuery {
|
||||
since_sequence: Option<u64>,
|
||||
limit: Option<usize>,
|
||||
},
|
||||
/// PromQL query (ticket 013's `GET /api/query`). `query` is parsed here (rather
|
||||
/// than by the API handler) so parsing and evaluation both happen inside the
|
||||
/// single-owner engine loop, matching every other `EngineQuery` variant's
|
||||
/// pattern of evaluating in place against `index` with no copying. A malformed
|
||||
/// query is therefore only discovered once this reaches `handle_query`, hence
|
||||
/// `EngineResponse::Promql` carrying a `Result` rather than a bare `Vec`.
|
||||
/// `query` is parsed inside the engine (not the API handler) so it stays in the single-owner loop with no copying; a malformed query is only discovered here, hence `EngineResponse::Promql` carries a `Result`.
|
||||
Promql { query: String, start_ms: Option<i64>, end_ms: Option<i64> },
|
||||
}
|
||||
|
||||
@@ -123,10 +85,7 @@ pub enum EngineMessage {
|
||||
name: String,
|
||||
respond_to: oneshot::Sender<bool>,
|
||||
},
|
||||
/// Sets the cap on ingested logs kept in memory (ticket 026). `None`
|
||||
/// means unlimited. Sent on startup (from `Settings::max_logs`) and
|
||||
/// whenever the setting changes at runtime, so the cap takes effect on
|
||||
/// the next ingest without a restart.
|
||||
/// `None` means unlimited; sent on startup and whenever the setting changes so the cap applies without a restart.
|
||||
SetMaxLogs(Option<usize>),
|
||||
Query {
|
||||
query: EngineQuery,
|
||||
@@ -142,30 +101,20 @@ pub enum EngineResponse {
|
||||
ParamDeclarations(Vec<ParamDeclarationDto>),
|
||||
MetricDeclarations(Vec<MetricDeclarationDto>),
|
||||
BucketValues(Vec<BucketValueDto>),
|
||||
/// `Err` holds a human-readable message for a malformed query (see
|
||||
/// `crate::promql::parse_query`), which the API layer maps to a 400 response.
|
||||
/// `Err` holds a human-readable message for a malformed query, which the API layer maps to a 400 response.
|
||||
Promql(Result<Vec<PromqlSeriesDto>, String>),
|
||||
/// A query-level error, e.g. a malformed `filter` string on
|
||||
/// `EngineQuery::Logs`/`Timeline` (ticket 007). Kept as a variant on the
|
||||
/// existing response enum, rather than making `handle_query` return a
|
||||
/// `Result`, so every query path still funnels through the same
|
||||
/// `oneshot::Sender<EngineResponse>` — callers (`api.rs`) just add one
|
||||
/// more match arm to turn it into a 4xx.
|
||||
/// Kept as a variant (rather than making `handle_query` return a `Result`) so every query path still funnels through the same `oneshot::Sender<EngineResponse>`.
|
||||
Error(String),
|
||||
}
|
||||
|
||||
pub type EngineHandle = mpsc::Sender<EngineMessage>;
|
||||
|
||||
/// Sole owner of the primary index. Runs single-threaded so the map never needs a
|
||||
/// lock, and does its own response serialization/DTO construction so callers never
|
||||
/// have to clone the index to answer a query.
|
||||
/// Sole owner of the primary index; runs single-threaded so the map never needs a lock, and builds DTOs itself so callers never clone the index.
|
||||
pub async fn run(mut rx: mpsc::Receiver<EngineMessage>, settings: SettingsHandle) {
|
||||
let mut index: BTreeMap<IndexKey, DataValue> = BTreeMap::new();
|
||||
let mut max_logs: Option<usize> = None;
|
||||
let mut log_count: usize = 0;
|
||||
// Ticket 018: per-rule rolling windows / seen-value sets, keyed by rule
|
||||
// id. Lives only as long as this loop (no restart persistence — see
|
||||
// `alerts::AlertCondition::NewValue`'s doc comment).
|
||||
// Per-rule rolling windows/seen-value sets, keyed by rule id; lives only as long as this loop (no restart persistence).
|
||||
let mut alert_states: HashMap<String, AlertRuntimeState> = HashMap::new();
|
||||
|
||||
while let Some(message) = rx.recv().await {
|
||||
@@ -180,9 +129,7 @@ pub async fn run(mut rx: mpsc::Receiver<EngineMessage>, settings: SettingsHandle
|
||||
apply_all_metrics(&mut index, sequence, timestamp, &text);
|
||||
evict_excess_logs(&mut index, &mut log_count, max_logs);
|
||||
|
||||
// Ticket 018: evaluate alert rules against just this line
|
||||
// (not the whole log range), then hand any fired rule off to
|
||||
// ticket 019's delivery, off the hot path via `tokio::spawn`.
|
||||
// Evaluate alert rules against just this line, then hand any fired rule off to delivery via tokio::spawn so it stays off the hot path.
|
||||
let rules = settings.lock().unwrap().alert_rules.clone();
|
||||
let fired = alerts::evaluate_alerts(&rules, &mut alert_states, sequence, timestamp, &text);
|
||||
for (rule, message) in fired {
|
||||
@@ -217,9 +164,7 @@ pub async fn run(mut rx: mpsc::Receiver<EngineMessage>, settings: SettingsHandle
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates the regex and capture group before storing, so a bad declaration is
|
||||
/// rejected up front rather than silently stored (or discovered later at derivation
|
||||
/// time).
|
||||
/// Validates the regex and capture group before storing, so a bad declaration is rejected up front rather than discovered later.
|
||||
fn declare_param(
|
||||
index: &mut BTreeMap<IndexKey, DataValue>,
|
||||
request: DeclareParamRequest,
|
||||
@@ -244,9 +189,7 @@ fn declare_param(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validates the regex and every param matcher's capture-group reference before
|
||||
/// storing. All matchers are checked (not just the first bad one) so a rejection can
|
||||
/// report every problem at once.
|
||||
/// Validates every param matcher's capture-group reference (not just the first bad one) so a rejection reports every problem at once.
|
||||
fn declare_metric(
|
||||
index: &mut BTreeMap<IndexKey, DataValue>,
|
||||
request: DeclareMetricRequest,
|
||||
@@ -298,9 +241,7 @@ fn declare_metric(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Parses a single captured substring into the `MetricPrimitive` its param declares.
|
||||
/// Returns `None` on a parse failure (e.g. a `Number` param whose capture isn't
|
||||
/// numeric), which the caller treats as "this match doesn't apply".
|
||||
/// Returns `None` on a parse failure, which the caller treats as "this match doesn't apply".
|
||||
fn parse_metric_primitive(raw: &str, kind: MetricPrimitiveKind) -> Option<MetricPrimitive> {
|
||||
match kind {
|
||||
MetricPrimitiveKind::String => Some(MetricPrimitive::String(raw.to_string())),
|
||||
@@ -315,52 +256,12 @@ fn parse_metric_primitive(raw: &str, kind: MetricPrimitiveKind) -> Option<Metric
|
||||
.ok()
|
||||
.map(MetricPrimitive::Timestamp)
|
||||
}
|
||||
// A `Value`-role param never reaches this branch (see
|
||||
// `apply_metric_to_log`, which derives its value from elapsed time
|
||||
// instead of parsing captured text). Reached only for a `Label`-role
|
||||
// metric param or a standalone `ParamDeclaration` of kind `Duration`,
|
||||
// where the captured text is parsed as a literal millisecond count.
|
||||
// A Value-role param never reaches this branch (see apply_metric_to_log); reached only for a Label-role param or standalone ParamDeclaration, parsed as a literal millisecond count.
|
||||
MetricPrimitiveKind::Duration => raw.parse::<i64>().ok().map(MetricPrimitive::Duration),
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs one metric's regex against a single log line and, on a match, appends a
|
||||
/// bucket value entry: this is the "counter increment" (ticket 022) — a match is
|
||||
/// an occurrence at a point in time, optionally labeled.
|
||||
///
|
||||
/// `Bucket.labels` holds every `Label`-role param (the default role — see
|
||||
/// [`ParamRole`]); at most one param may instead be `Value`-role (enforced at
|
||||
/// declare time), and its parsed primitive becomes the pushed event's value.
|
||||
/// With no `Value`-role param, the pushed value is a fixed presence marker
|
||||
/// (`MetricPrimitive::Number(Number::Int(1))`), matching the original v1
|
||||
/// behavior where a bucket's `values` was purely the timeseries of *when that
|
||||
/// label combination occurred* — histogram()/gauge() derived queries
|
||||
/// (`crate::promql::AggOp`) read this same `value`, so a `Value`-role param is
|
||||
/// what makes those meaningful (bucketing/reading an occurrence-count-of-1
|
||||
/// series degenerates to "everything in one bucket"/"always 1"). If a param's
|
||||
/// capture group doesn't match, or its captured text fails to parse as its
|
||||
/// declared `kind`, the whole match is skipped for this line rather than
|
||||
/// storing a partially-labeled bucket.
|
||||
///
|
||||
/// A `Value`-role param of kind `Duration` (ticket 023) is a special case: its
|
||||
/// value isn't parsed from captured text at all. It's derived as the elapsed
|
||||
/// time (using this event's ingestion `timestamp`) since the *previous*
|
||||
/// counter event with this exact label set, tracked independently via
|
||||
/// `IndexKey::LastMetricEvent` (not `Bucket.values` itself, since — see
|
||||
/// below — a `Bucket` entry isn't always pushed). On the first occurrence of
|
||||
/// a given label set there is no previous event to diff against, so — per the
|
||||
/// ticket's "omit rather than emit a bogus zero" requirement — no `Bucket`
|
||||
/// entry is pushed for that occurrence at all, rather than a zero/placeholder
|
||||
/// duration; the occurrence is still recorded in `LastMetricEvent` so the
|
||||
/// *next* occurrence of that label set has something to diff against.
|
||||
///
|
||||
/// When `regex` does *not* match and `unmatched_labels` is `Some` (ticket
|
||||
/// 025), a fixed occurrence is registered anyway, labeled with those literal
|
||||
/// values — see `DeclareMetricRequest::unmatched_labels` for why this exists
|
||||
/// instead of a plain regex alternation. When it *does* match, `matched_labels`
|
||||
/// (if `Some`) is layered on top of whatever `params` captured, so the
|
||||
/// "matched" side can use the same clean literal values as the "unmatched"
|
||||
/// side — see `DeclareMetricRequest::matched_labels`.
|
||||
/// At most one param may be `Value`-role (its primitive becomes the event's value, otherwise a fixed `1` marker is used); any param failing to match/parse skips the whole match rather than storing a partial one. A `Value`-role `Duration` param is elapsed time since the previous same-labeled event, so the first occurrence pushes no bucket (nothing to diff against) rather than a bogus zero. When `regex` doesn't match and `unmatched_labels` is `Some`, an occurrence is still registered with those fixed labels.
|
||||
fn apply_metric_to_log(
|
||||
index: &mut BTreeMap<IndexKey, DataValue>,
|
||||
metric_name: &str,
|
||||
@@ -413,12 +314,7 @@ fn apply_metric_to_log(
|
||||
}
|
||||
|
||||
let value = if is_duration {
|
||||
// Bookkeeping key is separate from the `Bucket` key below: a
|
||||
// `Duration` value's first occurrence for this label set pushes no
|
||||
// `Bucket` entry (nothing to diff against), so the "last event"
|
||||
// timestamp can't live inside `Bucket.values` — it has to be tracked
|
||||
// independently so the *next* occurrence still has something to
|
||||
// diff against.
|
||||
// Tracked separately from Bucket: a Duration's first occurrence pushes no Bucket entry, so the last-event timestamp can't live in Bucket.values.
|
||||
let last_event_key =
|
||||
IndexKey::LastMetricEvent { metric_name: metric_name.to_string(), labels: labels.clone() };
|
||||
let previous_timestamp = match index.get(&last_event_key) {
|
||||
@@ -456,9 +352,7 @@ fn push_bucket_value(
|
||||
}
|
||||
}
|
||||
|
||||
/// Applies a newly-declared metric to every already-ingested log, so a metric
|
||||
/// declared after logs have already arrived still gets populated buckets for them
|
||||
/// (backfill), rather than only reflecting logs ingested from that point on.
|
||||
/// Backfills buckets for a newly-declared metric against every already-ingested log, not just future ones.
|
||||
fn backfill_metric(
|
||||
index: &mut BTreeMap<IndexKey, DataValue>,
|
||||
metric_name: &str,
|
||||
@@ -485,9 +379,6 @@ fn backfill_metric(
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs every currently-declared metric's regex against one newly-ingested log line,
|
||||
/// updating whichever buckets match. A line matching zero metrics leaves the bucket
|
||||
/// range untouched.
|
||||
fn apply_all_metrics(
|
||||
index: &mut BTreeMap<IndexKey, DataValue>,
|
||||
sequence: u64,
|
||||
@@ -529,11 +420,7 @@ fn apply_all_metrics(
|
||||
}
|
||||
}
|
||||
|
||||
/// Bucket values for one metric, in insertion order, optionally filtered to a single
|
||||
/// exact label combination. Used directly by tests (exact-match filtering) and, via
|
||||
/// `EngineQuery::BucketValues` in `handle_query` below, by the metrics API (ticket
|
||||
/// 004) — which layers its own textual label-filter and time/sequence-range
|
||||
/// filtering on top rather than using this function's `labels_filter` parameter.
|
||||
/// The metrics API layers its own textual label-filter and range filtering on top rather than using this function's `labels_filter` param (used directly by tests).
|
||||
fn bucket_values<'a>(
|
||||
index: &'a BTreeMap<IndexKey, DataValue>,
|
||||
metric_name: &'a str,
|
||||
@@ -553,42 +440,7 @@ fn bucket_values<'a>(
|
||||
.flatten()
|
||||
}
|
||||
|
||||
/// A [`FilterExpr`] compiled against a *snapshot* of the current index, ready
|
||||
/// to be evaluated per-log-line without touching the index again.
|
||||
///
|
||||
/// ## Design: how metric comparisons resolve against buckets
|
||||
///
|
||||
/// A FilterBar comparison like `my_metric[level]==5` can't be evaluated by
|
||||
/// re-running `my_metric`'s regex against each log line at query time — that
|
||||
/// would duplicate work the engine already did at ingest/backfill time
|
||||
/// (ticket 003), and would require re-parsing the metric's declaration for
|
||||
/// every query. Instead, since ticket 003 already derived, for every metric,
|
||||
/// one `Bucket` per distinct label combination holding the `(sequence,
|
||||
/// timestamp, _)` occurrences that produced it, resolving a metric
|
||||
/// comparison is a **one-time scan of that metric's buckets per query**
|
||||
/// (via [`bucket_values`]), rather than a per-log-line regex re-run:
|
||||
///
|
||||
/// 1. For `metric[param] <op> value`, scan every bucket for `metric` (this
|
||||
/// reuses the same ticket-003 range-scan `bucket_values` already uses for
|
||||
/// `GET /api/metrics/:name`), and keep the buckets whose `labels[param]`
|
||||
/// satisfies `<op> value`.
|
||||
/// 2. Every `(sequence, ..)` occurrence recorded in a kept bucket is a log
|
||||
/// line the comparison matches. Those sequences are collected into a
|
||||
/// `HashSet<u64>` once, up front — [`CompiledFilter::MetricMatch`].
|
||||
/// 3. Evaluating the comparison for a given log line is then just an `O(1)`
|
||||
/// `HashSet::contains(&sequence)` check, reusing the fact that a line's
|
||||
/// bucket membership (which buckets a line matched) was already computed
|
||||
/// when the line was ingested/backfilled, rather than recomputed here.
|
||||
///
|
||||
/// `sequence`/`timestamp` comparisons skip buckets entirely: they compare
|
||||
/// straight against the log's own `IndexKey::IngestedLog` fields, so they're
|
||||
/// compiled into their own lightweight variants ([`CompiledFilter::Sequence`]
|
||||
/// / [`CompiledFilter::Timestamp`]) instead of [`CompiledFilter::MetricMatch`].
|
||||
///
|
||||
/// A metric comparison with no `[param]` isn't meaningful under this model —
|
||||
/// v1's buckets have no single "the metric's value", only a per-label-combo
|
||||
/// occurrence marker (see `apply_metric_to_log`'s doc comment) — so
|
||||
/// `compile_filter` rejects it with a clear error instead of guessing.
|
||||
/// A [`FilterExpr`] compiled against a snapshot of the index, so evaluating per log line touches the index only once: a `metric[param]` comparison is pre-resolved into a `HashSet` of matching sequences ([`CompiledFilter::MetricMatch`]) for O(1) per-line lookup instead of a regex re-run; a comparison with no `[param]` is rejected since v1 buckets have no single "the metric's value", only a per-label-combo occurrence marker.
|
||||
enum CompiledFilter {
|
||||
Regex(Regex),
|
||||
Sequence { op: CompareOp, target: f64 },
|
||||
@@ -598,11 +450,7 @@ enum CompiledFilter {
|
||||
Or(Box<CompiledFilter>, Box<CompiledFilter>),
|
||||
}
|
||||
|
||||
/// Parses and compiles a FilterBar query string against `index`, resolving
|
||||
/// every regex/metric-comparison/datetime-literal up front so evaluating it
|
||||
/// per log line (see [`eval_compiled`]) never touches the index or re-parses
|
||||
/// anything. `now` anchors relative datetime literals (e.g. `-5m`) so every
|
||||
/// comparison in one query is resolved against the same instant.
|
||||
/// Resolves every regex/metric-comparison/datetime-literal up front so per-line evaluation ([`eval_compiled`]) never touches the index; `now` anchors relative literals so every comparison in one query resolves against the same instant.
|
||||
fn compile_filter_query(
|
||||
index: &BTreeMap<IndexKey, DataValue>,
|
||||
filter: &str,
|
||||
@@ -637,12 +485,7 @@ fn compile_filter(
|
||||
}
|
||||
}
|
||||
|
||||
/// `sequence <op> value` — compares straight against the log's own sequence
|
||||
/// number. `value` is normally `FilterValue::Number` (the grammar parses a
|
||||
/// bare integer as a number); a `Raw` value is accepted too as long as it
|
||||
/// still parses as a number (defensive - the grammar shouldn't produce this
|
||||
/// for `sequence`, but nothing stops a caller from writing `sequence<"5"`-ish
|
||||
/// input that the value token still captures as text).
|
||||
/// A `Raw` value is accepted defensively (parsed as a number) even though the grammar shouldn't produce one for `sequence`.
|
||||
fn compile_sequence_comparison(op: CompareOp, value: &FilterValue) -> Result<CompiledFilter, String> {
|
||||
let target = match value {
|
||||
FilterValue::Number(n) => *n,
|
||||
@@ -653,11 +496,7 @@ fn compile_sequence_comparison(op: CompareOp, value: &FilterValue) -> Result<Com
|
||||
Ok(CompiledFilter::Sequence { op, target })
|
||||
}
|
||||
|
||||
/// `timestamp <op> value` — compares straight against the log's own
|
||||
/// timestamp. A `FilterValue::Number` is treated as raw milliseconds (no
|
||||
/// datetime parsing needed); a `FilterValue::Raw` token (e.g. `12:00`,
|
||||
/// `-5m`, `2024-01-15`) is resolved via `filter_datetime::parse_datetime`
|
||||
/// against `now`.
|
||||
/// A `Number` is treated as raw milliseconds; a `Raw` token is resolved via `filter_datetime::parse_datetime` against `now`.
|
||||
fn compile_timestamp_comparison(
|
||||
op: CompareOp,
|
||||
value: &FilterValue,
|
||||
@@ -672,9 +511,7 @@ fn compile_timestamp_comparison(
|
||||
Ok(CompiledFilter::Timestamp { op, target_ms })
|
||||
}
|
||||
|
||||
/// `metric[param] <op> value` for any metric name other than the reserved
|
||||
/// `sequence`/`timestamp` — see [`CompiledFilter`]'s doc comment for the
|
||||
/// overall approach.
|
||||
/// `metric[param] <op> value` for any metric name other than `sequence`/`timestamp` — see [`CompiledFilter`]'s doc comment.
|
||||
fn compile_metric_comparison(
|
||||
index: &BTreeMap<IndexKey, DataValue>,
|
||||
metric: &str,
|
||||
@@ -710,14 +547,7 @@ fn compile_metric_comparison(
|
||||
Ok(CompiledFilter::MetricMatch(matching))
|
||||
}
|
||||
|
||||
/// Compares one bucket label's `MetricPrimitive` against a filter's
|
||||
/// right-hand side. Numbers compare numerically (a `Raw` value is parsed as
|
||||
/// a number); timestamps compare against a resolved datetime (a `Number`
|
||||
/// value is treated as raw milliseconds, a `Raw` value goes through
|
||||
/// `filter_datetime::parse_datetime`); strings compare lexicographically
|
||||
/// against the raw token (or a number's textual form). Mismatched kinds that
|
||||
/// can't be meaningfully compared (e.g. a string label against a bare
|
||||
/// number) simply never match, rather than erroring.
|
||||
/// Mismatched kinds that can't be meaningfully compared (e.g. a string label against a bare number) simply never match, rather than erroring.
|
||||
fn compare_metric_primitive(
|
||||
primitive: &MetricPrimitive,
|
||||
op: CompareOp,
|
||||
@@ -754,8 +584,7 @@ fn compare_metric_primitive(
|
||||
Ok(ordering.is_some_and(|ordering| apply_compare_op(ordering, op)))
|
||||
}
|
||||
|
||||
/// `pub(crate)` so `crate::alerts` can reuse it for `Threshold`/`Ratio`
|
||||
/// comparisons instead of duplicating the match.
|
||||
/// `pub(crate)` so `crate::alerts` can reuse it for `Threshold`/`Ratio` comparisons instead of duplicating the match.
|
||||
pub(crate) fn apply_compare_op(ordering: Ordering, op: CompareOp) -> bool {
|
||||
match op {
|
||||
CompareOp::Lt => ordering == Ordering::Less,
|
||||
@@ -767,10 +596,7 @@ pub(crate) fn apply_compare_op(ordering: Ordering, op: CompareOp) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
/// Evaluates a compiled filter against one log line. Cheap: every regex,
|
||||
/// datetime literal, and bucket scan was already resolved by
|
||||
/// [`compile_filter_query`], so this is just pattern matching, numeric
|
||||
/// comparisons, and `HashSet` lookups — no index access.
|
||||
/// Cheap: everything was already resolved by [`compile_filter_query`], so this is just pattern matching and `HashSet` lookups — no index access.
|
||||
fn eval_compiled(compiled: &CompiledFilter, sequence: u64, timestamp: Timestamp, text: &str) -> bool {
|
||||
match compiled {
|
||||
CompiledFilter::Regex(re) => re.is_match(text),
|
||||
@@ -832,11 +658,7 @@ fn param_declarations_range(
|
||||
})
|
||||
}
|
||||
|
||||
/// Compiles every currently-declared param's regex once, so `derive_log_params`
|
||||
/// can be applied to many lines without recompiling each param's regex per
|
||||
/// line. A param whose stored regex somehow fails to compile (shouldn't
|
||||
/// happen — `declare_param` validates it before storing) is skipped rather
|
||||
/// than panicking.
|
||||
/// A param whose stored regex fails to compile (shouldn't happen — `declare_param` validates it) is skipped rather than panicking.
|
||||
fn compiled_param_declarations(
|
||||
index: &BTreeMap<IndexKey, DataValue>,
|
||||
) -> Vec<(String, Regex, String, MetricPrimitiveKind)> {
|
||||
@@ -849,12 +671,7 @@ fn compiled_param_declarations(
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Runs every declared param's regex against one log line's text, returning
|
||||
/// the derived `MetricPrimitive` values keyed by param name. A param is
|
||||
/// omitted from the result (rather than included as null/empty) when its
|
||||
/// regex doesn't match the line, its capture group doesn't match, or the
|
||||
/// captured text fails to parse as the param's declared `kind` — see
|
||||
/// `LogEntryDto::params`.
|
||||
/// A param is omitted (not included as null/empty) when its regex/capture group doesn't match or the captured text fails to parse — see `LogEntryDto::params`.
|
||||
fn derive_log_params(
|
||||
text: &str,
|
||||
declarations: &[(String, Regex, String, MetricPrimitiveKind)],
|
||||
@@ -875,11 +692,7 @@ fn derive_log_params(
|
||||
params
|
||||
}
|
||||
|
||||
/// Evicts the oldest (lowest-`sequence`) ingested logs until `log_count` is
|
||||
/// back at or under `max_logs`. `sequence` numbering is not renumbered on
|
||||
/// eviction, and any `Bucket` values already derived from an evicted log are
|
||||
/// left in place — buckets are aggregate data, not raw logs, so they're not
|
||||
/// retroactively pruned.
|
||||
/// Bucket values already derived from an evicted log are left in place — buckets are aggregate data, not raw logs, so they're not retroactively pruned.
|
||||
fn evict_excess_logs(
|
||||
index: &mut BTreeMap<IndexKey, DataValue>,
|
||||
log_count: &mut usize,
|
||||
@@ -921,16 +734,7 @@ fn handle_query(index: &BTreeMap<IndexKey, DataValue>, query: EngineQuery) -> En
|
||||
None => None,
|
||||
};
|
||||
let limit = params.limit.unwrap_or(500).min(10_000);
|
||||
// Compile each currently-declared param's regex once up front, then
|
||||
// apply the compiled set to every *returned* line below (ticket 015).
|
||||
// Perf note: this is a straightforward per-returned-line,
|
||||
// per-declared-param regex scan — O(lines_returned * declared_params)
|
||||
// — rather than precomputing/caching derived params for the whole log
|
||||
// range. That's fine at this codebase's current scale (v1), but if
|
||||
// either the log volume or number of declared params grows a lot,
|
||||
// this should move to incremental derivation at ingest time (mirroring
|
||||
// how `apply_all_metrics` already does for metric buckets) instead of
|
||||
// recomputing on every `/api/logs` query.
|
||||
// O(lines_returned * declared_params) per query — fine at v1 scale; if volume/params grow, move to incremental derivation at ingest time like apply_all_metrics does for buckets.
|
||||
let compiled_params = compiled_param_declarations(index);
|
||||
let entries: Vec<LogEntryDto> = logs_range(index)
|
||||
.filter(|(sequence, timestamp, _)| {
|
||||
@@ -1068,18 +872,7 @@ fn handle_query(index: &BTreeMap<IndexKey, DataValue>, query: EngineQuery) -> En
|
||||
}
|
||||
}
|
||||
|
||||
/// Coerces a bucket's stored `MetricPrimitive` into a plottable `f64` for
|
||||
/// `PromqlSeriesDto`:
|
||||
/// - `Number` uses its underlying numeric value.
|
||||
/// - `Timestamp` uses its millisecond value (arbitrary but at least meaningful/
|
||||
/// orderable, since there's no other sensible single number for a point in time).
|
||||
/// - `String` has no numeric reading at all, so it falls back to `1.0` — i.e. it is
|
||||
/// treated the same as the presence/occurrence marker `apply_metric_to_log` already
|
||||
/// stores for every bucket value today (ticket 003), meaning a PromQL series over a
|
||||
/// `String`-valued bucket becomes an occurrence-count series rather than being
|
||||
/// skipped. This keeps every declared metric queryable (matching the ticket's
|
||||
/// "documented fallback" requirement) rather than silently omitting non-numeric
|
||||
/// series.
|
||||
/// Coerces a bucket's `MetricPrimitive` into a plottable `f64`: Number/Timestamp/Duration use their natural numeric value; `String` has no numeric reading, so it falls back to `1.0` (matching the presence/occurrence marker used for every bucket) rather than being skipped, keeping every declared metric queryable.
|
||||
fn coerce_to_f64(value: &MetricPrimitive) -> f64 {
|
||||
match value {
|
||||
MetricPrimitive::Number(n) => match n {
|
||||
@@ -1092,10 +885,7 @@ fn coerce_to_f64(value: &MetricPrimitive) -> f64 {
|
||||
}
|
||||
}
|
||||
|
||||
/// True when every matcher in `matchers` is satisfied by `labels` (compared against
|
||||
/// each label value's `Display` form, same convention as `EngineQuery::BucketValues`'s
|
||||
/// `labels_filter`). Unmatched/extra labels on the bucket are ignored — a matcher list
|
||||
/// only needs to be a subset.
|
||||
/// Compared against each label's `Display` form; extra labels on the bucket are ignored — the matcher list only needs to be a subset.
|
||||
fn matches_label_matchers(
|
||||
labels: &BTreeMap<String, MetricPrimitive>,
|
||||
matchers: &[LabelMatcher],
|
||||
@@ -1105,26 +895,7 @@ fn matches_label_matchers(
|
||||
})
|
||||
}
|
||||
|
||||
/// Evaluates a parsed PromQL query against `index`, producing chart-ready series.
|
||||
///
|
||||
/// Baseline (no aggregation wrapper): one output series per distinct label
|
||||
/// combination that matches `parsed.metric_name` + `parsed.matchers`, with that
|
||||
/// combination's bucket values as `(timestamp_ms, value)` points (see
|
||||
/// `coerce_to_f64`), sorted by timestamp.
|
||||
///
|
||||
/// `sum()`/`avg()`: merges every matching series' points by exact millisecond
|
||||
/// timestamp (summing, or averaging, the values that land on the same timestamp
|
||||
/// across series) into a single output series with empty labels — the aggregation
|
||||
/// collapses the label dimension, same as PromQL's `sum by ()`/`avg by ()` default.
|
||||
///
|
||||
/// `rate()`: requires (and, per `promql::parse_query`, is only ever given) a range
|
||||
/// selector. This is a simplified, single-point-per-series rate rather than PromQL's
|
||||
/// true sliding-window-per-evaluation-step semantics: for each matching series, it
|
||||
/// takes the window `(last_point_ts - range, last_point_ts]` and divides the number
|
||||
/// of points falling in that window by the range's duration in seconds, producing one
|
||||
/// `(last_point_ts, rate)` point per series. Since bucket values are occurrence
|
||||
/// markers rather than cumulative counters (ticket 003), an occurrence-count-per-
|
||||
/// second reading is the meaningful analogue of PromQL's counter-increase rate here.
|
||||
/// Evaluates a parsed PromQL query against `index`, producing chart-ready series: baseline is one series per label combination; `sum()`/`avg()` merge series by timestamp into one label-less series; `rate()` is simplified to a single point over `(last_ts - range, last_ts]` (an occurrence-count-per-second reading, not PromQL's true sliding-window-per-step) since bucket values are occurrence markers, not cumulative counters.
|
||||
fn evaluate_promql(
|
||||
index: &BTreeMap<IndexKey, DataValue>,
|
||||
parsed: &PromqlQuery,
|
||||
@@ -1190,10 +961,7 @@ fn evaluate_promql(
|
||||
.collect()
|
||||
}
|
||||
Some(AggOp::Gauge) => {
|
||||
// Ticket 022: collapse each label combination's counter events
|
||||
// down to their most-recent value in the queried time range —
|
||||
// "value at time T" derived from the last matching event, rather
|
||||
// than a distinct stored gauge kind.
|
||||
// Collapse each label combination's events down to their most-recent value in range — "value at time T", not a distinct stored gauge kind.
|
||||
series_points
|
||||
.into_iter()
|
||||
.filter_map(|(labels, points)| {
|
||||
@@ -1212,12 +980,7 @@ fn evaluate_promql(
|
||||
.flat_map(|points| points.iter().map(|(ts, _)| *ts))
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
// Ticket 022: bucket every matching counter event's value
|
||||
// (across all label combinations — histogram collapses the
|
||||
// label dimension the same way sum()/avg() do) into fixed-width
|
||||
// buckets, one output series per bucket labeled `le` (Prometheus's
|
||||
// "less-than-or-equal" convention) holding a single
|
||||
// `(last_ts, count)` point.
|
||||
// Buckets every value (across all label combinations, collapsing the label dimension like sum()/avg()) into fixed-width buckets, labeled `le` (Prometheus's less-than-or-equal convention).
|
||||
let mut counts: BTreeMap<i64, u64> = BTreeMap::new();
|
||||
for points in series_points.values() {
|
||||
for (_, value) in points {
|
||||
@@ -1667,8 +1430,7 @@ mod tests {
|
||||
|
||||
assert_eq!(series.len(), 1);
|
||||
assert_eq!(series[0].labels.get("path").map(String::as_str), Some("/a"));
|
||||
// Occurrence markers coerce to 1.0 (see `coerce_to_f64`); timestamps come from
|
||||
// `ingest_log`'s `sequence as millis` helper.
|
||||
// Occurrence markers coerce to 1.0 (see coerce_to_f64); timestamps come from ingest_log's sequence-as-millis helper.
|
||||
assert_eq!(series[0].points, vec![(0, 1.0), (2, 1.0)]);
|
||||
}
|
||||
|
||||
@@ -1835,8 +1597,7 @@ mod tests {
|
||||
let mut series = evaluate_promql(&index, &parsed, None, None);
|
||||
series.sort_by(|a, b| a.labels.get("le").cmp(&b.labels.get("le")));
|
||||
|
||||
// durations 2,4 -> bucket [0,10) le=10 (count 2); 12,15 -> [10,20) le=20 (count 2);
|
||||
// 21 -> [20,30) le=30 (count 1).
|
||||
// durations 2,4 -> le=10 (count 2); 12,15 -> le=20 (count 2); 21 -> le=30 (count 1).
|
||||
assert_eq!(series.len(), 3);
|
||||
assert_eq!(series[0].labels.get("le").map(String::as_str), Some("10"));
|
||||
assert_eq!(series[0].points[0].1, 2.0);
|
||||
@@ -1871,9 +1632,7 @@ mod tests {
|
||||
ingest_log(&mut index, 2, "user=alice logged in");
|
||||
|
||||
let values: Vec<_> = bucket_values(&index, "login_gap", None).collect();
|
||||
// First occurrences (alice@0, bob@1) produce no duration value at all —
|
||||
// only alice's second occurrence (sequence 2) has a previous event to
|
||||
// diff against.
|
||||
// First occurrences (alice@0, bob@1) produce no duration value — only alice's second occurrence (seq 2) has something to diff against.
|
||||
assert_eq!(values.len(), 1);
|
||||
let (labels, (sequence, timestamp, value)) = &values[0];
|
||||
assert_eq!(labels.get("user").map(|v| v.to_string()), Some("alice".to_string()));
|
||||
@@ -1912,10 +1671,7 @@ mod tests {
|
||||
assert_eq!(series[0].points, vec![(10, 10.0)]);
|
||||
}
|
||||
|
||||
// Fixed matched_labels/unmatched_labels (rather than a captured "category"
|
||||
// param) so a case-insensitive match like "ERROR" vs "Error" still
|
||||
// collapses into one clean "error" series instead of fragmenting by
|
||||
// whatever case the source text happened to use.
|
||||
// Fixed labels (not a captured param) so a case-insensitive match collapses into one clean "error" series regardless of source-text case.
|
||||
fn declare_error_category_metric(index: &mut BTreeMap<IndexKey, DataValue>) {
|
||||
declare_metric(
|
||||
index,
|
||||
@@ -2016,7 +1772,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
// -- ticket 007: FilterBar query integration -----------------------
|
||||
// -- FilterBar query integration --
|
||||
|
||||
fn logs_query(filter: Option<&str>) -> EngineQuery {
|
||||
EngineQuery::Logs(LogsParams {
|
||||
@@ -2107,10 +1863,7 @@ mod tests {
|
||||
ingest_log(&mut index, 1, "level=info duration=2 boom");
|
||||
ingest_log(&mut index, 2, "level=error duration=3 all good");
|
||||
|
||||
// Only sequence 0 has both the "boom" regex match *and* level=error.
|
||||
// Note: comparison values are bare tokens (see `value` in the
|
||||
// grammar), not quoted string literals like `regex(...)`'s argument
|
||||
// — so `level==error`, not `level=="error"`.
|
||||
// Only sequence 0 has both; note comparison values are bare tokens (level==error), not quoted string literals like regex()'s argument.
|
||||
match handle_query(
|
||||
&index,
|
||||
logs_query(Some(r#"regex("boom") and requests[level]==error"#)),
|
||||
|
||||
Reference in New Issue
Block a user