Make ottyl_seq/ottyl_ts usable in dashboard PromQL queries (ticket 055)
evaluate_promql previously resolved every metric name through bucket_values, which only returns data for a real MetricDeclaration — so a dashboard panel querying ottyl_seq/ottyl_ts/ottyl_fmt (already supported in the filter bar and alert conditions) silently returned an empty series. ottyl_seq/ottyl_ts are now synthesized directly from the ingested-log index instead of requiring a declaration; ottyl_fmt is categorical (json/logfmt/none) with no sensible numeric reading, so it's rejected up front with a clear error pointing back at the filter bar rather than returning nothing. GET /api/params now also lists the three reserved names (marked `reserved: true`, distinct from `auto_derived`) purely for discoverability — merged in at the API layer only, not the engine's internal ParamDeclarations query, so profile-switch removal and metric-param-promotion logic are unaffected. declare_param also now rejects declaring a param under one of these names. Docs.tsx and the Params panel are updated to reflect the new dashboard support and the reserved/non-editable listing. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01XFUNZwCvz3kcod3352ZR2o
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
d17cfe9b83
commit
ff56558b2e
+52
-1
@@ -8,6 +8,7 @@ use crate::dto::{
|
|||||||
ParamDeclarationDto, LogsParams, ProfileSummaryDto, PromqlParams, SaveDashboardRequest,
|
ParamDeclarationDto, LogsParams, ProfileSummaryDto, PromqlParams, SaveDashboardRequest,
|
||||||
SettingsDto, TestAlertNotificationRequest, TimelineParams, UpdateAlertRuleRequest, UpdateSettingsRequest,
|
SettingsDto, TestAlertNotificationRequest, TimelineParams, UpdateAlertRuleRequest, UpdateSettingsRequest,
|
||||||
};
|
};
|
||||||
|
use crate::data_value::MetricPrimitiveKind;
|
||||||
use crate::engine::{
|
use crate::engine::{
|
||||||
DeclareMetricRequest, DeclareParamRequest, EngineHandle, EngineMessage, EngineQuery,
|
DeclareMetricRequest, DeclareParamRequest, EngineHandle, EngineMessage, EngineQuery,
|
||||||
EngineResponse,
|
EngineResponse,
|
||||||
@@ -280,12 +281,61 @@ async fn get_stats(State(state): State<ApiState>) -> impl IntoResponse {
|
|||||||
|
|
||||||
async fn list_params(State(state): State<ApiState>) -> impl IntoResponse {
|
async fn list_params(State(state): State<ApiState>) -> impl IntoResponse {
|
||||||
match ask(&state.engine, EngineQuery::ParamDeclarations).await {
|
match ask(&state.engine, EngineQuery::ParamDeclarations).await {
|
||||||
Ok(EngineResponse::ParamDeclarations(declarations)) => Json(declarations).into_response(),
|
Ok(EngineResponse::ParamDeclarations(mut declarations)) => {
|
||||||
|
// `ottyl_seq`/`ottyl_ts`/`ottyl_fmt` (ticket 055) are computed line-locally rather than
|
||||||
|
// stored as index `ParamDeclaration` entries (see `crate::engine::declare_param`'s
|
||||||
|
// rejection of these names), so they're merged in here purely for discoverability
|
||||||
|
// rather than via `param_declarations_range`. This is deliberately a merge only in this
|
||||||
|
// one HTTP-facing listing, not in `EngineQuery::ParamDeclarations` itself — other
|
||||||
|
// internal callers of that query (promoting a metric's referenced params, replaying
|
||||||
|
// declarations on profile switch) must keep seeing only real, persistable entries.
|
||||||
|
declarations.extend(reserved_param_declaration_dtos());
|
||||||
|
Json(declarations).into_response()
|
||||||
|
}
|
||||||
Ok(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(),
|
Ok(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(),
|
||||||
Err(status) => status.into_response(),
|
Err(status) => status.into_response(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `GET /api/params`-only synthesized entries for the three reserved names
|
||||||
|
/// (`crate::engine::OTTYL_RESERVED_PARAM_NAMES`) — see `list_params`. `message_match_regex`/
|
||||||
|
/// `capture_group` are empty since these are never text-extracted; `reserved: true` marks them as
|
||||||
|
/// non-deletable/non-editable, distinct from `auto_derived`.
|
||||||
|
fn reserved_param_declaration_dtos() -> [ParamDeclarationDto; 3] {
|
||||||
|
[
|
||||||
|
ParamDeclarationDto {
|
||||||
|
name: "ottyl_seq".to_string(),
|
||||||
|
message_match_regex: String::new(),
|
||||||
|
capture_group: String::new(),
|
||||||
|
json_field_path: None,
|
||||||
|
kind: MetricPrimitiveKind::Number,
|
||||||
|
elapsed_since_last: false,
|
||||||
|
auto_derived: false,
|
||||||
|
reserved: true,
|
||||||
|
},
|
||||||
|
ParamDeclarationDto {
|
||||||
|
name: "ottyl_ts".to_string(),
|
||||||
|
message_match_regex: String::new(),
|
||||||
|
capture_group: String::new(),
|
||||||
|
json_field_path: None,
|
||||||
|
kind: MetricPrimitiveKind::Timestamp,
|
||||||
|
elapsed_since_last: false,
|
||||||
|
auto_derived: false,
|
||||||
|
reserved: true,
|
||||||
|
},
|
||||||
|
ParamDeclarationDto {
|
||||||
|
name: "ottyl_fmt".to_string(),
|
||||||
|
message_match_regex: String::new(),
|
||||||
|
capture_group: String::new(),
|
||||||
|
json_field_path: None,
|
||||||
|
kind: MetricPrimitiveKind::String,
|
||||||
|
elapsed_since_last: false,
|
||||||
|
auto_derived: false,
|
||||||
|
reserved: true,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
async fn declare_param(
|
async fn declare_param(
|
||||||
State(state): State<ApiState>,
|
State(state): State<ApiState>,
|
||||||
Json(body): Json<DeclareParamRequest>,
|
Json(body): Json<DeclareParamRequest>,
|
||||||
@@ -313,6 +363,7 @@ async fn declare_param(
|
|||||||
// A request that reached here was accepted by `declare_param`, which always stores
|
// A request that reached here was accepted by `declare_param`, which always stores
|
||||||
// it as non-auto (see `engine::declare_param`).
|
// it as non-auto (see `engine::declare_param`).
|
||||||
auto_derived: false,
|
auto_derived: false,
|
||||||
|
reserved: false,
|
||||||
};
|
};
|
||||||
(StatusCode::CREATED, Json(dto)).into_response()
|
(StatusCode::CREATED, Json(dto)).into_response()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,6 +28,14 @@ pub struct ParamDeclarationDto {
|
|||||||
/// See `crate::data_value::ParamDeclaration::auto_derived`. The frontend uses this to disable
|
/// See `crate::data_value::ParamDeclaration::auto_derived`. The frontend uses this to disable
|
||||||
/// editing — `POST /api/params` also rejects a request naming an auto-derived param.
|
/// editing — `POST /api/params` also rejects a request naming an auto-derived param.
|
||||||
pub auto_derived: bool,
|
pub auto_derived: bool,
|
||||||
|
/// Set only for the built-in `ottyl_seq`/`ottyl_ts`/`ottyl_fmt` names
|
||||||
|
/// (`crate::engine::OTTYL_RESERVED_PARAM_NAMES`), synthesized into the `GET /api/params`
|
||||||
|
/// response by `crate::api::list_params` rather than backed by any index entry — distinct from
|
||||||
|
/// `auto_derived` (which *is* a real, if machine-discovered, `ParamDeclaration`). Always
|
||||||
|
/// `false` for every other entry. There is no removal path for these at all (not just a
|
||||||
|
/// disabled one), so the frontend should treat `reserved` as non-deletable/non-editable.
|
||||||
|
#[serde(default)]
|
||||||
|
pub reserved: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
#[derive(Debug, Clone, Serialize)]
|
||||||
|
|||||||
+162
@@ -220,6 +220,13 @@ fn declare_param(
|
|||||||
index: &mut BTreeMap<IndexKey, DataValue>,
|
index: &mut BTreeMap<IndexKey, DataValue>,
|
||||||
request: DeclareParamRequest,
|
request: DeclareParamRequest,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
|
if OTTYL_RESERVED_PARAM_NAMES.contains(&request.name.as_str()) {
|
||||||
|
return Err(format!(
|
||||||
|
"'{}' is a reserved name, present on every log line without declaring anything; \
|
||||||
|
it can't be redeclared as a param",
|
||||||
|
request.name
|
||||||
|
));
|
||||||
|
}
|
||||||
if let Some(DataValue::ParamDeclaration(existing)) =
|
if let Some(DataValue::ParamDeclaration(existing)) =
|
||||||
index.get(&IndexKey::ParamDeclaration { name: request.name.clone() })
|
index.get(&IndexKey::ParamDeclaration { name: request.name.clone() })
|
||||||
{
|
{
|
||||||
@@ -1348,6 +1355,7 @@ fn handle_query(
|
|||||||
kind,
|
kind,
|
||||||
elapsed_since_last,
|
elapsed_since_last,
|
||||||
auto_derived,
|
auto_derived,
|
||||||
|
reserved: false,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
@@ -1553,6 +1561,24 @@ fn declared_value_kind(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The three reserved param names usable without declaring anything — see `compile_ottyl_seq_comparison`/
|
||||||
|
/// `compile_ottyl_ts_comparison`/`compile_ottyl_fmt_comparison` for the filter-bar equivalent, and
|
||||||
|
/// `webapp/src/components/Docs.tsx` for the user-facing docs. Shared by `evaluate_promql` (dashboard
|
||||||
|
/// panels, this file) and `crate::api::list_params` (`GET /api/params` discoverability).
|
||||||
|
pub const OTTYL_RESERVED_PARAM_NAMES: [&str; 3] = ["ottyl_seq", "ottyl_ts", "ottyl_fmt"];
|
||||||
|
|
||||||
|
/// For `ottyl_seq`/`ottyl_ts` only (not `ottyl_fmt`, which `evaluate_promql` rejects outright before
|
||||||
|
/// this is consulted): returns a `(sequence, timestamp_ms) -> value` mapping used to synthesize a
|
||||||
|
/// virtual PromQL series straight from the ingested-log index, since neither is backed by a
|
||||||
|
/// `MetricDeclaration`/bucket the way every other metric name is.
|
||||||
|
fn ottyl_reserved_numeric_value_fn(metric_name: &str) -> Option<fn(u64, i64) -> f64> {
|
||||||
|
match metric_name {
|
||||||
|
"ottyl_seq" => Some(|sequence, _timestamp_ms| sequence as f64),
|
||||||
|
"ottyl_ts" => Some(|_sequence, timestamp_ms| timestamp_ms as f64),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Evaluates a parsed PromQL query against `index`, producing chart-ready series: baseline is one series per label combination; `sum()`/`avg()` merge series into one series per `by_labels` group (the whole result collapses to a single label-less series when `by_labels` is empty); `rate()` — bare or wrapped in `sum()`/`avg()` — turns each series into a windowed occurrences-per-second reading via `windowed_rate_series`, so it plots as a real line rather than one trailing point; `histogram()` groups by `by_labels` the same way sum()/avg() do, then buckets each group's values independently, so `histogram(m) by (label)` yields one histogram (a family of `le`-labeled series carrying the group's own labels) per label group instead of collapsing all groups together.
|
/// Evaluates a parsed PromQL query against `index`, producing chart-ready series: baseline is one series per label combination; `sum()`/`avg()` merge series into one series per `by_labels` group (the whole result collapses to a single label-less series when `by_labels` is empty); `rate()` — bare or wrapped in `sum()`/`avg()` — turns each series into a windowed occurrences-per-second reading via `windowed_rate_series`, so it plots as a real line rather than one trailing point; `histogram()` groups by `by_labels` the same way sum()/avg() do, then buckets each group's values independently, so `histogram(m) by (label)` yields one histogram (a family of `le`-labeled series carrying the group's own labels) per label group instead of collapsing all groups together.
|
||||||
/// `sum()`/`avg()` over a `String`-kind metric are rejected up front — `coerce_to_f64` has no
|
/// `sum()`/`avg()` over a `String`-kind metric are rejected up front — `coerce_to_f64` has no
|
||||||
/// numeric reading for a string, so silently treating every occurrence as `1.0` would let a sum
|
/// numeric reading for a string, so silently treating every occurrence as `1.0` would let a sum
|
||||||
@@ -1565,6 +1591,20 @@ fn evaluate_promql(
|
|||||||
start_ms: Option<i64>,
|
start_ms: Option<i64>,
|
||||||
end_ms: Option<i64>,
|
end_ms: Option<i64>,
|
||||||
) -> Result<Vec<PromqlSeriesDto>, String> {
|
) -> Result<Vec<PromqlSeriesDto>, String> {
|
||||||
|
// `ottyl_fmt` is categorical ("json"/"logfmt"/"none", `crate::logfmt::detect_format`) with no
|
||||||
|
// sensible numeric reading, unlike `ottyl_seq`/`ottyl_ts` below — rather than silently
|
||||||
|
// returning an empty series (the failure mode ticket 055 calls out), a dashboard query over it
|
||||||
|
// is rejected up front with a message pointing back at the filter/query bar, where it already
|
||||||
|
// works (see `compile_ottyl_fmt_comparison`).
|
||||||
|
if parsed.metric_name == "ottyl_fmt" {
|
||||||
|
return Err(
|
||||||
|
"'ottyl_fmt' is a categorical value (json/logfmt/none), not a number, so it can't be \
|
||||||
|
used in a dashboard PromQL query; filter on it in the search bar instead, e.g. \
|
||||||
|
ottyl_fmt==json"
|
||||||
|
.to_string(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
if matches!(parsed.agg, Some(AggOp::Sum) | Some(AggOp::Avg))
|
if matches!(parsed.agg, Some(AggOp::Sum) | Some(AggOp::Avg))
|
||||||
&& !parsed.inner_rate
|
&& !parsed.inner_rate
|
||||||
&& declared_value_kind(index, &parsed.metric_name) == Some(MetricPrimitiveKind::String)
|
&& declared_value_kind(index, &parsed.metric_name) == Some(MetricPrimitiveKind::String)
|
||||||
@@ -1576,6 +1616,27 @@ fn evaluate_promql(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let mut series_points: BTreeMap<BTreeMap<String, String>, Vec<(i64, f64)>> = BTreeMap::new();
|
let mut series_points: BTreeMap<BTreeMap<String, String>, Vec<(i64, f64)>> = BTreeMap::new();
|
||||||
|
if let Some(reserved_value) = ottyl_reserved_numeric_value_fn(&parsed.metric_name) {
|
||||||
|
// `ottyl_seq`/`ottyl_ts` are computed line-locally (see `compile_ottyl_seq_comparison`/
|
||||||
|
// `compile_ottyl_ts_comparison`) rather than backed by a `MetricDeclaration` and bucket
|
||||||
|
// entries, so they're synthesized here straight from the ingested-log index instead of
|
||||||
|
// going through `bucket_values`. Neither carries any labels, so a non-empty matcher list
|
||||||
|
// simply matches nothing, same as an unmatched label filter would against a real metric.
|
||||||
|
let no_labels: BTreeMap<String, MetricPrimitive> = BTreeMap::new();
|
||||||
|
if matches_label_matchers(&no_labels, &parsed.matchers) {
|
||||||
|
for (sequence, timestamp, _text) in logs_range(index) {
|
||||||
|
let timestamp_ms = timestamp.as_millis();
|
||||||
|
if start_ms.is_none_or(|start| timestamp_ms >= start)
|
||||||
|
&& end_ms.is_none_or(|end| timestamp_ms <= end)
|
||||||
|
{
|
||||||
|
series_points
|
||||||
|
.entry(BTreeMap::new())
|
||||||
|
.or_default()
|
||||||
|
.push((timestamp_ms, reserved_value(sequence, timestamp_ms)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
for (labels, (_, timestamp, value)) in bucket_values(index, &parsed.metric_name, None)
|
for (labels, (_, timestamp, value)) in bucket_values(index, &parsed.metric_name, None)
|
||||||
.filter(|(labels, _)| matches_label_matchers(labels, &parsed.matchers))
|
.filter(|(labels, _)| matches_label_matchers(labels, &parsed.matchers))
|
||||||
.filter(|(_, (_, timestamp, _))| {
|
.filter(|(_, (_, timestamp, _))| {
|
||||||
@@ -1590,6 +1651,7 @@ fn evaluate_promql(
|
|||||||
.or_default()
|
.or_default()
|
||||||
.push((timestamp.as_millis(), coerce_to_f64(value)));
|
.push((timestamp.as_millis(), coerce_to_f64(value)));
|
||||||
}
|
}
|
||||||
|
}
|
||||||
for points in series_points.values_mut() {
|
for points in series_points.values_mut() {
|
||||||
points.sort_by_key(|(timestamp_ms, _)| *timestamp_ms);
|
points.sort_by_key(|(timestamp_ms, _)| *timestamp_ms);
|
||||||
}
|
}
|
||||||
@@ -3005,6 +3067,106 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn promql_count_over_ottyl_seq_gives_one_point_per_ingested_log_with_no_metric_declaration() {
|
||||||
|
let mut index = BTreeMap::new();
|
||||||
|
ingest_log(&mut index, 0, "line one");
|
||||||
|
ingest_log(&mut index, 1, "line two");
|
||||||
|
ingest_log(&mut index, 2, "line three");
|
||||||
|
|
||||||
|
let parsed = promql::parse_query("count(ottyl_seq)").unwrap();
|
||||||
|
let series = evaluate_promql(&index, &parsed, None, None).unwrap();
|
||||||
|
assert_eq!(series.len(), 1);
|
||||||
|
assert_eq!(series[0].points, vec![(0, 1.0), (1, 1.0), (2, 1.0)]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn promql_bare_ottyl_seq_returns_the_actual_sequence_numbers() {
|
||||||
|
let mut index = BTreeMap::new();
|
||||||
|
ingest_log(&mut index, 0, "line zero");
|
||||||
|
ingest_log(&mut index, 5, "line five");
|
||||||
|
|
||||||
|
let parsed = promql::parse_query("ottyl_seq").unwrap();
|
||||||
|
let mut series = evaluate_promql(&index, &parsed, None, None).unwrap();
|
||||||
|
assert_eq!(series.len(), 1);
|
||||||
|
series[0].points.sort_by_key(|(ts, _)| *ts);
|
||||||
|
assert_eq!(series[0].points, vec![(0, 0.0), (5, 5.0)]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn promql_bare_ottyl_ts_returns_ingest_timestamps() {
|
||||||
|
let mut index = BTreeMap::new();
|
||||||
|
ingest_log(&mut index, 0, "a");
|
||||||
|
ingest_log(&mut index, 1, "b");
|
||||||
|
|
||||||
|
let parsed = promql::parse_query("ottyl_ts").unwrap();
|
||||||
|
let series = evaluate_promql(&index, &parsed, None, None).unwrap();
|
||||||
|
assert_eq!(series.len(), 1);
|
||||||
|
assert_eq!(series[0].points, vec![(0, 0.0), (1, 1.0)]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn promql_rate_over_ottyl_ts_produces_a_windowed_series_not_an_empty_one() {
|
||||||
|
let mut index = BTreeMap::new();
|
||||||
|
for sequence in 0..5u64 {
|
||||||
|
ingest_log(&mut index, sequence, "line");
|
||||||
|
}
|
||||||
|
|
||||||
|
let parsed = promql::parse_query("rate(ottyl_ts[5s])").unwrap();
|
||||||
|
let series = evaluate_promql(&index, &parsed, None, None).unwrap();
|
||||||
|
assert_eq!(series.len(), 1);
|
||||||
|
assert!(!series[0].points.is_empty(), "rate(ottyl_ts[...]) must not silently return no data");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn promql_ottyl_seq_with_label_matcher_matches_nothing_since_it_has_no_labels() {
|
||||||
|
let mut index = BTreeMap::new();
|
||||||
|
ingest_log(&mut index, 0, "line");
|
||||||
|
|
||||||
|
let parsed = promql::parse_query(r#"ottyl_seq{path="/a"}"#).unwrap();
|
||||||
|
let series = evaluate_promql(&index, &parsed, None, None).unwrap();
|
||||||
|
assert!(series.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn promql_over_ottyl_fmt_is_explicitly_rejected_not_silently_empty() {
|
||||||
|
let mut index = BTreeMap::new();
|
||||||
|
ingest_log(&mut index, 0, r#"{"level":"error"}"#);
|
||||||
|
|
||||||
|
let parsed = promql::parse_query("count(ottyl_fmt)").unwrap();
|
||||||
|
let err = evaluate_promql(&index, &parsed, None, None).unwrap_err();
|
||||||
|
assert!(err.contains("ottyl_fmt"), "error should name the rejected metric: {err}");
|
||||||
|
assert!(err.contains("categorical") || err.contains("filter"), "error should explain why: {err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn handle_query_promql_returns_series_for_ottyl_seq_end_to_end() {
|
||||||
|
let mut index = BTreeMap::new();
|
||||||
|
ingest_log(&mut index, 0, "a");
|
||||||
|
ingest_log(&mut index, 1, "b");
|
||||||
|
|
||||||
|
let response = handle_query(
|
||||||
|
&index,
|
||||||
|
EngineQuery::Promql { query: "count(ottyl_seq)".to_string(), start_ms: None, end_ms: None },
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
match response {
|
||||||
|
EngineResponse::Promql(Ok(series)) => {
|
||||||
|
assert_eq!(series[0].points, vec![(0, 1.0), (1, 1.0)]);
|
||||||
|
}
|
||||||
|
_ => panic!("expected EngineResponse::Promql(Ok(_))"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn declare_param_rejects_reserved_ottyl_names() {
|
||||||
|
let mut index = BTreeMap::new();
|
||||||
|
for name in OTTYL_RESERVED_PARAM_NAMES {
|
||||||
|
let err = declare_param(&mut index, request(name, "x", "x")).unwrap_err();
|
||||||
|
assert!(err.contains("reserved"), "expected a 'reserved' error for '{name}', got: {err}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// -- FilterBar query integration --
|
// -- FilterBar query integration --
|
||||||
|
|
||||||
fn logs_query(filter: Option<&str>) -> EngineQuery {
|
fn logs_query(filter: Option<&str>) -> EngineQuery {
|
||||||
|
|||||||
@@ -1,77 +0,0 @@
|
|||||||
# 055 — Make `ottyl_*` reserved params usable in dashboard (PromQL) queries, not just filters
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
Three reserved params exist, all hardcoded in the filter compiler:
|
|
||||||
`ottyl_seq` (ingest sequence number), `ottyl_ts` (ingest timestamp,
|
|
||||||
supports relative/absolute values), and `ottyl_fmt` (auto-detected
|
|
||||||
`json`/`logfmt`/`none`) — defined at `src/engine.rs:773-775, 795, 806, 824`,
|
|
||||||
grammar comments at `src/filter_parser.pest:7-9`, docs at
|
|
||||||
`webapp/src/components/Docs.tsx:105-106, 375` ("reserved metric names
|
|
||||||
present on every log line, so they're always filterable without
|
|
||||||
declaring anything"). Alert conditions restrict to the same three plus
|
|
||||||
`regex()`/`json[field]` (`src/alerts.rs:3, 199-225`).
|
|
||||||
|
|
||||||
These already work end-to-end for the filter/query bar syntax used by
|
|
||||||
`GET /api/logs`, `GET /api/timeline`, and alert conditions — that part of
|
|
||||||
"accessible parameters for queries" is true today. What's missing is
|
|
||||||
dashboards: PromQL-style panel queries (`src/promql.rs`,
|
|
||||||
`evaluate_promql` in `src/engine.rs:1562-1601`) have no special case for
|
|
||||||
`ottyl_*` anywhere. A PromQL query resolves `metric_name` via
|
|
||||||
`bucket_values(index, &parsed.metric_name)`
|
|
||||||
(`src/engine.rs`, called from `evaluate_promql`), which only returns data
|
|
||||||
for names backed by a `DataValue::MetricDeclaration` and bucket entries
|
|
||||||
created through `POST /api/metrics`. `ottyl_*` values are computed
|
|
||||||
line-locally at filter-compile time and never written into declared-
|
|
||||||
metric buckets, so `sum(ottyl_seq)` or similar in a dashboard panel would
|
|
||||||
silently return an empty series rather than real data.
|
|
||||||
|
|
||||||
Separately, `ottyl_*` never appear in `GET /api/params`
|
|
||||||
(`src/api.rs:281-287` → `EngineQuery::ParamDeclarations` →
|
|
||||||
`param_declarations_range`, `src/engine.rs:1340-1355`), since that only
|
|
||||||
lists explicit `ParamDeclaration` index entries and `ottyl_*` never
|
|
||||||
create one. `FilterBar.tsx` (`webapp/src/components/FilterBar.tsx:36-41`)
|
|
||||||
has no autocomplete/suggestion list at all today (plain text input), so
|
|
||||||
there's nothing currently excluding them from UI suggestions either —
|
|
||||||
but if param suggestions are ever added, this gap should be kept in mind.
|
|
||||||
|
|
||||||
## Scope
|
|
||||||
|
|
||||||
- Add a special case in `evaluate_promql`/`src/promql.rs` (analogous to
|
|
||||||
the filter compiler's handling at `src/engine.rs:773-775`) that
|
|
||||||
synthesizes a virtual bucket series for `ottyl_seq`/`ottyl_ts`/
|
|
||||||
`ottyl_fmt` metric names from the ingested-log index, instead of
|
|
||||||
requiring a `MetricDeclaration`. Decide and document in the PR how
|
|
||||||
`ottyl_fmt`'s categorical string values map onto a numeric time
|
|
||||||
series (e.g. count-by-value, or reject it from PromQL numeric
|
|
||||||
aggregation with a clear error, since `ottyl_seq`/`ottyl_ts` are
|
|
||||||
naturally numeric but `ottyl_fmt` is not).
|
|
||||||
- Decide whether `ottyl_*` should appear in `GET /api/params` (or a
|
|
||||||
parallel listing) so dashboard panel builders in the webapp can
|
|
||||||
discover them without prior knowledge of the reserved names; if yes,
|
|
||||||
extend `param_declarations_range` or add a small hardcoded merge at the
|
|
||||||
API layer rather than forcing them into the index as real
|
|
||||||
`ParamDeclaration` entries (they aren't user-declared and shouldn't be
|
|
||||||
deletable via ticket 051's/052's params-removal paths).
|
|
||||||
- Update `README.md`/`Docs.tsx` to state whether `ottyl_*` is usable in
|
|
||||||
dashboard queries, once implemented (or explicitly document it as
|
|
||||||
filter-only, if the numeric/categorical mismatch above makes broad
|
|
||||||
PromQL support impractical for `ottyl_fmt`).
|
|
||||||
|
|
||||||
## Acceptance criteria
|
|
||||||
|
|
||||||
- A dashboard panel using a PromQL query over `ottyl_seq` or `ottyl_ts`
|
|
||||||
(e.g. `count(ottyl_seq)` or a rate over `ottyl_ts`) returns real data
|
|
||||||
from ingested logs, verified with a seeded fixture.
|
|
||||||
- The behavior for `ottyl_fmt` in a PromQL query is either implemented
|
|
||||||
(with a stated value-to-series mapping) or explicitly rejected with a
|
|
||||||
clear, documented error — not a silent empty result.
|
|
||||||
- `GET /api/params` behavior for `ottyl_*` is decided and documented;
|
|
||||||
if included, they're clearly marked as reserved/non-deletable rather
|
|
||||||
than indistinguishable from user-declared params.
|
|
||||||
|
|
||||||
## Depends on
|
|
||||||
|
|
||||||
None (independent of the params-removal tickets, but note the
|
|
||||||
non-deletable requirement above if `ottyl_*` end up listed alongside
|
|
||||||
regular params).
|
|
||||||
@@ -105,6 +105,9 @@ const SECTIONS: DocSection[] = [
|
|||||||
<Code>ottyl_seq</Code>, <Code>ottyl_ts</Code>, and <Code>ottyl_fmt</Code> are reserved metric names present
|
<Code>ottyl_seq</Code>, <Code>ottyl_ts</Code>, and <Code>ottyl_fmt</Code> are reserved metric names present
|
||||||
on every log line, so they're always filterable without declaring anything. <Code>ottyl_fmt</Code> holds the
|
on every log line, so they're always filterable without declaring anything. <Code>ottyl_fmt</Code> holds the
|
||||||
auto-detected format of the line's text — <Code>json</Code>, <Code>logfmt</Code>, or <Code>none</Code>.
|
auto-detected format of the line's text — <Code>json</Code>, <Code>logfmt</Code>, or <Code>none</Code>.
|
||||||
|
They also appear (flagged <Code>reserved</Code>) in <Code>GET /api/params</Code> for discoverability, though
|
||||||
|
there's no UI to edit or remove them — see also <em>Dashboards</em> below for using{' '}
|
||||||
|
<Code>ottyl_seq</Code>/<Code>ottyl_ts</Code> in a panel query.
|
||||||
</p>
|
</p>
|
||||||
<p className="font-semibold text-text">Date/time formats</p>
|
<p className="font-semibold text-text">Date/time formats</p>
|
||||||
<p>Timestamp comparisons accept three forms:</p>
|
<p>Timestamp comparisons accept three forms:</p>
|
||||||
@@ -335,6 +338,14 @@ const SECTIONS: DocSection[] = [
|
|||||||
Dashboards are saved per profile, so they travel with it when you switch or copy it — an active profile is
|
Dashboards are saved per profile, so they travel with it when you switch or copy it — an active profile is
|
||||||
required to create or save one; pick or create one from the settings menu first.
|
required to create or save one; pick or create one from the settings menu first.
|
||||||
</p>
|
</p>
|
||||||
|
<p>
|
||||||
|
The reserved <Code>ottyl_seq</Code>/<Code>ottyl_ts</Code> names (see <em>Filtering</em> above) work in panel
|
||||||
|
queries the same as any declared metric — e.g. <Code>count(ottyl_seq)</Code> for log volume, or{' '}
|
||||||
|
<Code>rate(ottyl_ts[1m])</Code> for ingest rate — since every log line carries them without a metric
|
||||||
|
declaration. <Code>ottyl_fmt</Code> is categorical (<Code>json</Code>/<Code>logfmt</Code>/<Code>none</Code>
|
||||||
|
), not a number, so it's rejected in a panel query with an explicit error rather than silently charting
|
||||||
|
nothing — filter on it in the filter bar instead.
|
||||||
|
</p>
|
||||||
</div>
|
</div>
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -49,6 +49,12 @@ export function Params({ params, onDeclared }: { params: ParamDeclaration[]; onD
|
|||||||
// original (and anything referencing it by name) rather than renaming it.
|
// original (and anything referencing it by name) rather than renaming it.
|
||||||
const [editingName, setEditingName] = useState<string | null>(null)
|
const [editingName, setEditingName] = useState<string | null>(null)
|
||||||
|
|
||||||
|
// `reserved` params (`ottyl_seq`/`ottyl_ts`/`ottyl_fmt`) are synthesized server-side purely for
|
||||||
|
// discoverability (see `GET /api/params`) — they aren't real declarations, so they're kept out of
|
||||||
|
// the editable/clickable list and shown separately, non-interactively.
|
||||||
|
const declaredParams = useMemo(() => params.filter((p) => !p.reserved), [params])
|
||||||
|
const reservedParams = useMemo(() => params.filter((p) => p.reserved), [params])
|
||||||
|
|
||||||
const isBoolean = kind === 'boolean'
|
const isBoolean = kind === 'boolean'
|
||||||
const isElapsedSinceLast = source === 'elapsed_since_last'
|
const isElapsedSinceLast = source === 'elapsed_since_last'
|
||||||
// The message match regex doubles as a sample line here so a pasted JSON log line unlocks the
|
// The message match regex doubles as a sample line here so a pasted JSON log line unlocks the
|
||||||
@@ -264,8 +270,10 @@ export function Params({ params, onDeclared }: { params: ParamDeclaration[]; onD
|
|||||||
<div className="min-w-0 flex-1">
|
<div className="min-w-0 flex-1">
|
||||||
<div className="mb-1 text-xs text-text-subtle">Declared params</div>
|
<div className="mb-1 text-xs text-text-subtle">Declared params</div>
|
||||||
<div className="flex flex-wrap gap-1">
|
<div className="flex flex-wrap gap-1">
|
||||||
{params.length === 0 && <span className="text-xs text-text-faint">No params declared yet.</span>}
|
{declaredParams.length === 0 && (
|
||||||
{params.map((p) => (
|
<span className="text-xs text-text-faint">No params declared yet.</span>
|
||||||
|
)}
|
||||||
|
{declaredParams.map((p) => (
|
||||||
<button
|
<button
|
||||||
key={p.name}
|
key={p.name}
|
||||||
type="button"
|
type="button"
|
||||||
@@ -287,6 +295,24 @@ export function Params({ params, onDeclared }: { params: ParamDeclaration[]; onD
|
|||||||
</button>
|
</button>
|
||||||
))}
|
))}
|
||||||
</div>
|
</div>
|
||||||
|
{reservedParams.length > 0 && (
|
||||||
|
<>
|
||||||
|
<div className="mb-1 mt-2 text-xs text-text-subtle">
|
||||||
|
Reserved (built in — present on every line, not editable)
|
||||||
|
</div>
|
||||||
|
<div className="flex flex-wrap gap-1">
|
||||||
|
{reservedParams.map((p) => (
|
||||||
|
<span
|
||||||
|
key={p.name}
|
||||||
|
className="rounded border border-dashed border-border-soft px-2 py-1 text-xs text-text-faint"
|
||||||
|
title={`${p.name} (${p.kind}, reserved)`}
|
||||||
|
>
|
||||||
|
{p.name} ({p.kind})
|
||||||
|
</span>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -32,6 +32,8 @@ export interface ParamDeclaration {
|
|||||||
kind: 'string' | 'number' | 'timestamp' | 'duration' | 'boolean'
|
kind: 'string' | 'number' | 'timestamp' | 'duration' | 'boolean'
|
||||||
/** A third extraction source, distinct from `capture_group`/`json_field_path`: when true, this param's value is never parsed from text — it's computed as elapsed time since the previous event sharing the same label set. Only meaningful paired with `kind: 'duration'`. */
|
/** A third extraction source, distinct from `capture_group`/`json_field_path`: when true, this param's value is never parsed from text — it's computed as elapsed time since the previous event sharing the same label set. Only meaningful paired with `kind: 'duration'`. */
|
||||||
elapsed_since_last?: boolean
|
elapsed_since_last?: boolean
|
||||||
|
/** Set only for the built-in `ottyl_seq`/`ottyl_ts`/`ottyl_fmt` names, synthesized into `GET /api/params` for discoverability — they're usable in the filter bar (and `ottyl_seq`/`ottyl_ts` in dashboard PromQL queries) without ever being declared, and can't be edited or removed. */
|
||||||
|
reserved?: boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface TimelineBucket {
|
export interface TimelineBucket {
|
||||||
|
|||||||
Reference in New Issue
Block a user