Move scattered magic constants into src/consts.rs
Centralizes engine tuning values, poll intervals, logfmt column regex fragments, and app/settings identity strings that were previously defined ad hoc across engine.rs, ingest.rs, logfmt.rs, profile.rs, settings.rs, and main.rs. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01A1ZJ3SrayzLzVndhVq9BmS
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
1c635bc530
commit
284e3562a2
@@ -0,0 +1,54 @@
|
||||
use std::time::Duration;
|
||||
|
||||
// mpsc channel capacity between ingest/API callers and the engine actor task.
|
||||
pub const ENGINE_CHANNEL_CAPACITY: usize = 1024;
|
||||
|
||||
// Hard ceiling on how many params `auto_declare_params` will ever create automatically over the
|
||||
// life of the engine, so one unusual input can't inflate the declared-param set forever.
|
||||
pub const MAX_AUTO_DECLARED_PARAMS: usize = 64;
|
||||
|
||||
// Per-line ceiling on how many *new* params a single ingested line may contribute to the
|
||||
// auto-derived set, so one unusually wide line can't burn the whole per-run budget at once.
|
||||
pub const MAX_NEW_AUTO_PARAMS_PER_LINE: usize = 8;
|
||||
|
||||
// How long a sort helper index may sit unused before it's purged and left to rebuild on demand.
|
||||
pub const SORT_HELPER_INDEX_PURGE_AFTER: Duration = Duration::from_secs(300);
|
||||
|
||||
// Window and capacity for the rolling ingestion-to-engine-processing lag high watermark.
|
||||
pub const INGEST_LAG_WATERMARK_WINDOW_MS: i64 = 5000;
|
||||
pub const INGEST_LAG_WATERMARK_MAX_ENTRIES: usize = 5;
|
||||
|
||||
// Bucket width used to merge multiple sub-series into one combined time series.
|
||||
pub const MERGE_WINDOW_MS: i64 = 1_000;
|
||||
|
||||
// Default timeline query bucket width when the caller doesn't specify one.
|
||||
pub const DEFAULT_TIMELINE_BUCKET_MS: i64 = 1_000;
|
||||
|
||||
// Reserved param names usable without declaring anything (ottyl-synthesized columns).
|
||||
pub const OTTYL_RESERVED_PARAM_NAMES: [&str; 3] = ["ottyl_seq", "ottyl_ts", "ottyl_fmt"];
|
||||
|
||||
// Idle poll cadence while tailing stdin/a file for new log lines.
|
||||
pub const STDIN_POLL_INTERVAL: Duration = Duration::from_millis(200);
|
||||
|
||||
// Poll cadence for a configured Prometheus/OpenMetrics `--metrics-url`.
|
||||
pub const METRICS_POLL_INTERVAL: Duration = Duration::from_secs(15);
|
||||
|
||||
// Column-splitting regex fragments for the fixed-width logfmt column parser.
|
||||
pub const COLUMN_SEPARATOR_PATTERN: &str = r"(?:\t|[ \t]{2,})";
|
||||
pub const COLUMN_VALUE_PATTERN: &str = r"\S+(?: \S+)*";
|
||||
|
||||
// Name of the always-present profile every Settings file is seeded with.
|
||||
pub const DEFAULT_PROFILE_NAME: &str = "Default";
|
||||
|
||||
// Name/label of the metric the Default profile is seeded with.
|
||||
pub const SEEDED_METRIC_NAME: &str = "log_category";
|
||||
pub const SEEDED_METRIC_LABEL: &str = "category";
|
||||
|
||||
// Host the HTTP server binds to; port is user-configurable, host is not.
|
||||
pub const DEFAULT_BIND_HOST: &str = "0.0.0.0";
|
||||
|
||||
// App identity used to resolve the OS-conventional settings directory, and the settings file name.
|
||||
pub const APP_QUALIFIER: &str = "";
|
||||
pub const APP_ORGANIZATION: &str = "";
|
||||
pub const APP_NAME: &str = "ottyl";
|
||||
pub const SETTINGS_FILE_NAME: &str = "settings.json";
|
||||
+6
-8
@@ -1,4 +1,9 @@
|
||||
use crate::alerts::{self, AlertRuntimeState};
|
||||
use crate::consts::{
|
||||
DEFAULT_TIMELINE_BUCKET_MS, INGEST_LAG_WATERMARK_MAX_ENTRIES, INGEST_LAG_WATERMARK_WINDOW_MS,
|
||||
MAX_AUTO_DECLARED_PARAMS, MAX_NEW_AUTO_PARAMS_PER_LINE, MERGE_WINDOW_MS,
|
||||
OTTYL_RESERVED_PARAM_NAMES, SORT_HELPER_INDEX_PURGE_AFTER,
|
||||
};
|
||||
use crate::data_value::{DataValue, MetricPrimitiveKind, ParamDeclaration};
|
||||
use crate::dto::{
|
||||
BucketValueDto, LogEntryDto, LogsParams, MetricDeclarationDto, ParamDeclarationDto,
|
||||
@@ -605,7 +610,6 @@ fn build_auto_param_pattern(key: &str, group: &str, quoted: bool) -> String {
|
||||
/// will never recur meaningfully. Chosen alongside `MAX_NEW_AUTO_PARAMS_PER_LINE` (below) rather than
|
||||
/// on its own: a per-line cap alone still lets an unbounded *variety* of lines each contribute a few
|
||||
/// new keys forever, so this total ceiling is the actual backstop.
|
||||
const MAX_AUTO_DECLARED_PARAMS: usize = 64;
|
||||
|
||||
/// Per-line ceiling on how many *new* params a single ingested line may contribute to the auto-derived
|
||||
/// set. Without this, one unusually wide line (the fixture's WiFi stat dump has 20+ `key=value` pairs
|
||||
@@ -613,7 +617,6 @@ const MAX_AUTO_DECLARED_PARAMS: usize = 64;
|
||||
/// more representative lines ingested later. Pairs beyond the cap are simply never declared (from this
|
||||
/// line or any other, since a later line with fewer pairs would still need to reach them within its own
|
||||
/// per-line budget) — acceptable since they're exactly the low-value, one-off counters ticket 050 flags.
|
||||
const MAX_NEW_AUTO_PARAMS_PER_LINE: usize = 8;
|
||||
|
||||
fn auto_derived_param_count(index: &BTreeMap<IndexKey, DataValue>) -> usize {
|
||||
param_declarations_range(index).filter(|declaration| declaration.6).count()
|
||||
@@ -1662,7 +1665,6 @@ struct SortHelperIndexEntry {
|
||||
/// `Settings` (ticket 061 explicitly scopes that out) — a dropped index is always transparently
|
||||
/// rebuildable on the next query for that sort key, so this is purely a memory-bound knob, not a
|
||||
/// user-visible behavior change.
|
||||
const SORT_HELPER_INDEX_PURGE_AFTER: std::time::Duration = std::time::Duration::from_secs(300);
|
||||
|
||||
/// Derives a single named param's value from `text`, for helper-index maintenance where deriving
|
||||
/// every declared param (`derive_log_params`) per ingested line would be wasted work when only one
|
||||
@@ -1833,11 +1835,9 @@ fn approx_log_bytes(text: &str) -> u64 {
|
||||
|
||||
/// Ticket 073: how long a rolling ingestion-to-engine-processing lag sample stays eligible to be
|
||||
/// the reported high watermark.
|
||||
const INGEST_LAG_WATERMARK_WINDOW_MS: i64 = 5000;
|
||||
/// Ticket 073: at most this many lag samples are held at once — a fresh sample only displaces the
|
||||
/// current *lowest*-lag held sample (never appended past this cap), so the collection always
|
||||
/// reflects the highest few lags seen recently rather than every sample.
|
||||
const INGEST_LAG_WATERMARK_MAX_ENTRIES: usize = 5;
|
||||
|
||||
/// Records one `(engine processing time) - (ottyl_ts)` lag sample into the rolling high-watermark
|
||||
/// collection the wishlist asked for (ticket 073): up to `INGEST_LAG_WATERMARK_MAX_ENTRIES` samples,
|
||||
@@ -2128,7 +2128,7 @@ fn handle_query_inner(
|
||||
},
|
||||
None => None,
|
||||
};
|
||||
let bucket_ms = params.bucket_ms.unwrap_or(1_000).max(1);
|
||||
let bucket_ms = params.bucket_ms.unwrap_or(DEFAULT_TIMELINE_BUCKET_MS).max(1);
|
||||
let mut buckets: BTreeMap<i64, u64> = BTreeMap::new();
|
||||
for (sequence, timestamp, text) in logs_range(index).filter(|(_, timestamp, _)| {
|
||||
params.start_ms.is_none_or(|start| timestamp.as_millis() >= start)
|
||||
@@ -2336,7 +2336,6 @@ fn windowed_rate_series(
|
||||
/// the value axis. Occurrence timestamps across different label groups (e.g. different hosts serving
|
||||
/// the same path) essentially never land on the same millisecond, so summing/averaging only requires
|
||||
/// matching *this* window to combine into "total over time" rather than going sparse (ticket 042).
|
||||
const MERGE_WINDOW_MS: i64 = 1_000;
|
||||
|
||||
/// Merges several series into one combined time series (ticket 042). A single sub-series is returned
|
||||
/// untouched — there's only one contributor, so there's nothing to merge and no reason to lose
|
||||
@@ -2421,7 +2420,6 @@ fn declared_value_kind(
|
||||
/// `compile_ottyl_ts_comparison`/`compile_ottyl_fmt_comparison` for the filter-bar equivalent, and
|
||||
/// `webapp/src/components/Docs.tsx` for the user-facing docs. Shared by `evaluate_promql` (dashboard
|
||||
/// panels, this file) and `crate::api::list_params` (`GET /api/params` discoverability).
|
||||
pub const OTTYL_RESERVED_PARAM_NAMES: [&str; 3] = ["ottyl_seq", "ottyl_ts", "ottyl_fmt"];
|
||||
|
||||
/// For `ottyl_seq`/`ottyl_ts` only (not `ottyl_fmt`, which `evaluate_promql` rejects outright before
|
||||
/// this is consulted): returns a `(sequence, timestamp_ms) -> value` mapping used to synthesize a
|
||||
|
||||
+2
-5
@@ -1,16 +1,12 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::consts::{METRICS_POLL_INTERVAL, STDIN_POLL_INTERVAL as POLL_INTERVAL};
|
||||
use crate::engine::{EngineHandle, EngineMessage, IngestedLine, MetricSample};
|
||||
use crate::timestamp::Timestamp;
|
||||
use tokio::fs::File;
|
||||
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||
|
||||
const POLL_INTERVAL: Duration = Duration::from_millis(200);
|
||||
|
||||
const METRICS_POLL_INTERVAL: Duration = Duration::from_secs(15);
|
||||
|
||||
/// Reads stdin line by line, sequences and timestamps each line, and forwards it
|
||||
/// to the engine task.
|
||||
pub async fn run(engine: EngineHandle) {
|
||||
@@ -237,6 +233,7 @@ mod openmetrics_tests {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::time::Duration;
|
||||
use std::io::Write;
|
||||
use crate::engine::test_handle;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
+2
-7
@@ -12,6 +12,8 @@ use nom::IResult;
|
||||
use regex::Regex;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use crate::consts::{COLUMN_SEPARATOR_PATTERN, COLUMN_VALUE_PATTERN};
|
||||
|
||||
fn key(input: &str) -> IResult<&str, &str> {
|
||||
take_while1(|c: char| c.is_ascii_alphanumeric() || c == '_' || c == '.' || c == '-')(input)
|
||||
}
|
||||
@@ -115,13 +117,6 @@ pub fn is_log_stream_line(line: &str) -> bool {
|
||||
/// this is that rule re-expressed as a regex fragment for the trailing message field's
|
||||
/// `message_match_regex`, which re-derives its value the same way whenever a caller queries by it
|
||||
/// later (declarations don't cache extracted values -- see `crate::engine::extract_param_raw`).
|
||||
const COLUMN_SEPARATOR_PATTERN: &str = r"(?:\t|[ \t]{2,})";
|
||||
|
||||
/// A column's value: one or more non-whitespace tokens joined by single spaces, with no upper bound
|
||||
/// on length. Used for the trailing (unbounded-width) message field and the last named column, where
|
||||
/// there's no next column's start position to bound against.
|
||||
const COLUMN_VALUE_PATTERN: &str = r"\S+(?: \S+)*";
|
||||
|
||||
/// Same idea as `COLUMN_VALUE_PATTERN`, but bounded to a known column `width` (in chars) -- needed for
|
||||
/// every column except the last, since two adjacent `log stream` values can legitimately be separated
|
||||
/// by only a single space when the first one nearly fills its column's on-screen width (e.g. the
|
||||
|
||||
+3
-2
@@ -1,5 +1,6 @@
|
||||
mod alerts;
|
||||
mod api;
|
||||
mod consts;
|
||||
mod data_value;
|
||||
mod dto;
|
||||
mod engine;
|
||||
@@ -106,7 +107,7 @@ async fn main() {
|
||||
|
||||
let settings = Arc::new(Mutex::new(Settings::load(&settings_path)));
|
||||
|
||||
let engine_tx = engine::spawn(settings.clone(), 1024);
|
||||
let engine_tx = engine::spawn(settings.clone(), consts::ENGINE_CHANNEL_CAPACITY);
|
||||
{
|
||||
let max_logs = settings.lock().unwrap().max_logs;
|
||||
let _ = engine_tx.tx.send(engine::EngineMessage::SetMaxLogs(max_logs)).await;
|
||||
@@ -154,7 +155,7 @@ async fn main() {
|
||||
}
|
||||
|
||||
let app = api::router(engine_tx, settings, settings_path.clone());
|
||||
let listener = TcpListener::bind(("0.0.0.0", args.port))
|
||||
let listener = TcpListener::bind((consts::DEFAULT_BIND_HOST, args.port))
|
||||
.await
|
||||
.expect("failed to bind listener");
|
||||
let bound_port = listener
|
||||
|
||||
+1
-7
@@ -16,14 +16,12 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::consts::{DEFAULT_PROFILE_NAME, SEEDED_METRIC_LABEL, SEEDED_METRIC_NAME};
|
||||
use crate::data_value::MetricPrimitiveKind;
|
||||
use crate::engine::{DeclareMetricRequest, DeclareParamRequest, EngineHandle, EngineMessage};
|
||||
use crate::settings::Settings;
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
/// Name of the profile that always exists and can't be deleted (see `Settings::ensure_default_profile`).
|
||||
pub const DEFAULT_PROFILE_NAME: &str = "Default";
|
||||
|
||||
/// Mirrors `data_value::MetricPrimitiveKind` with `Serialize`/`Deserialize` so profiles can round-trip through JSON, since the engine-internal type only derives `Serialize`.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
@@ -333,10 +331,6 @@ impl Dashboard {
|
||||
}
|
||||
}
|
||||
|
||||
/// Name of the metric seeded into a brand-new `Default` profile, used to drive its starter dashboard's two panels.
|
||||
const SEEDED_METRIC_NAME: &str = "log_category";
|
||||
const SEEDED_METRIC_LABEL: &str = "category";
|
||||
|
||||
/// A counter that splits every line into `category="error"`/`"other"` via `matched_labels`/`unmatched_labels` rather than a captured param, so case variants like "ERROR" collapse into one series; the `regex` crate lacks the lookaround needed to express "didn't match" any other way.
|
||||
fn seeded_error_split_metric() -> MetricDeclarationRecord {
|
||||
MetricDeclarationRecord {
|
||||
|
||||
+6
-1
@@ -182,7 +182,12 @@ pub struct Settings {
|
||||
|
||||
/// Computes the default settings file path (`<config_dir>/settings.json`) from OS-conventional config directories; `None` if none could be determined.
|
||||
pub fn default_settings_path() -> Option<PathBuf> {
|
||||
ProjectDirs::from("", "", "ottyl").map(|dirs| dirs.config_dir().join("settings.json"))
|
||||
ProjectDirs::from(
|
||||
crate::consts::APP_QUALIFIER,
|
||||
crate::consts::APP_ORGANIZATION,
|
||||
crate::consts::APP_NAME,
|
||||
)
|
||||
.map(|dirs| dirs.config_dir().join(crate::consts::SETTINGS_FILE_NAME))
|
||||
}
|
||||
|
||||
impl Settings {
|
||||
|
||||
Reference in New Issue
Block a user