From 1d0f324c43a65de60c9901de3846731b6f99482f Mon Sep 17 00:00:00 2001 From: Hugh Kaznowski Date: Sat, 5 Sep 2026 15:12:12 +0100 Subject: [PATCH] Add settings-configurable cap on logs kept in memory (ticket 026) Settings.max_logs (default None = unlimited) is pushed to the engine via a new EngineMessage::SetMaxLogs, enforced on every ingest and whenever the setting changes at runtime (no restart required). Eviction drops the oldest (lowest-sequence) logs; sequence numbering isn't renumbered and buckets already derived from an evicted log are left in place. Exposed via PATCH /api/settings (max_logs: number | null) and a NavMenu numeric field (blank = unlimited). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01XFUNZwCvz3kcod3352ZR2o --- src/api.rs | 109 ++++++++++++++++++++++++++--- src/dto.rs | 25 ++++++- src/engine.rs | 75 ++++++++++++++++++++ src/main.rs | 4 ++ src/settings.rs | 5 ++ webapp/src/components/NavMenu.tsx | 37 +++++++++- webapp/src/lib/api.ts | 12 +++- webapp/src/lib/useSettingsPanel.ts | 10 +++ 8 files changed, 261 insertions(+), 16 deletions(-) diff --git a/src/api.rs b/src/api.rs index 1b38bab..454b85d 100644 --- a/src/api.rs +++ b/src/api.rs @@ -444,21 +444,37 @@ async fn get_settings(State(state): State) -> impl IntoResponse { } /// `PATCH /api/settings` — updates process-wide settings. Body: -/// `{ "color_scheme": "light" | "dark" | "system" }` (omit the field, or -/// send `null`, to leave it unchanged). Persists to `settings_path` and -/// returns the updated settings as `SettingsDto`. +/// `{ "color_scheme": "light" | "dark" | "system", "max_logs": number | +/// null }` (omit a field to leave it unchanged; `max_logs: null` clears the +/// cap back to unlimited). Persists to `settings_path`, pushes a changed +/// `max_logs` to the engine so it takes effect on the next ingest without a +/// restart, and returns the updated settings as `SettingsDto`. async fn update_settings( State(state): State, Json(body): Json, ) -> impl IntoResponse { - let mut settings = state.settings.lock().unwrap(); - if let Some(color_scheme) = body.color_scheme { - settings.color_scheme = color_scheme; - } - match settings.save(&state.settings_path) { - Ok(()) => Json(SettingsDto::from(&*settings)).into_response(), - Err(err) => (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(), + let max_logs_changed = body.max_logs.is_some(); + let new_max_logs = { + let mut settings = state.settings.lock().unwrap(); + if let Some(color_scheme) = body.color_scheme { + settings.color_scheme = color_scheme; + } + if let Some(max_logs) = body.max_logs { + settings.max_logs = max_logs; + } + match settings.save(&state.settings_path) { + Ok(()) => (), + Err(err) => return (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(), + } + (SettingsDto::from(&*settings), settings.max_logs) + }; + if max_logs_changed { + let _ = state + .engine + .send(EngineMessage::SetMaxLogs(new_max_logs.1)) + .await; } + Json(new_max_logs.0).into_response() } /// Maps a `ProfileError` to a response status/message: `NotFound` -> 404, @@ -1126,4 +1142,77 @@ mod tests { assert_eq!(dashboards[0].panels.len(), 1); assert_eq!(dashboards[0].panels[0].promql, "requests"); } + + async fn ingest_lines(engine: &EngineHandle, start_sequence: u64, texts: &[&str]) { + for (offset, text) in texts.iter().enumerate() { + let sequence = start_sequence + offset as u64; + engine + .send(EngineMessage::Ingest(IngestedLine { + sequence, + timestamp: Timestamp::from_millis(sequence as i64 * 1000), + text: text.to_string(), + })) + .await + .unwrap(); + } + } + + #[tokio::test] + async fn setting_max_logs_evicts_oldest_on_next_ingest() { + let (app, engine, _settings, _dir) = test_app(); + + ingest_lines(&engine, 0, &["a", "b", "c"]).await; + let (status, body) = get_json(&app, "/api/logs").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body.as_array().unwrap().len(), 3); + + let (status, body) = send_json(&app, "PATCH", "/api/settings", json!({ "max_logs": 2 })).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["max_logs"], 2); + + // Existing over-cap logs are evicted as soon as the setting change + // reaches the engine, not just on the next ingest. + let (_, body) = get_json(&app, "/api/logs").await; + assert_eq!(body.as_array().unwrap().len(), 2); + + ingest_lines(&engine, 3, &["d"]).await; + let (_, body) = get_json(&app, "/api/logs").await; + let texts: Vec<_> = body + .as_array() + .unwrap() + .iter() + .map(|entry| entry["text"].as_str().unwrap()) + .collect(); + assert_eq!(texts, vec!["c", "d"]); + } + + #[tokio::test] + async fn max_logs_defaults_to_unlimited() { + let (app, engine, _settings, _dir) = test_app(); + + let (_, body) = get_json(&app, "/api/settings").await; + assert_eq!(body["max_logs"], Value::Null); + + ingest_lines(&engine, 0, &["a", "b", "c", "d", "e"]).await; + let (_, body) = get_json(&app, "/api/logs").await; + assert_eq!(body.as_array().unwrap().len(), 5); + } + + #[tokio::test] + async fn max_logs_clears_back_to_unlimited() { + let (app, engine, _settings, _dir) = test_app(); + + send_json(&app, "PATCH", "/api/settings", json!({ "max_logs": 1 })).await; + ingest_lines(&engine, 0, &["a", "b"]).await; + let (_, body) = get_json(&app, "/api/logs").await; + assert_eq!(body.as_array().unwrap().len(), 1); + + let (status, body) = send_json(&app, "PATCH", "/api/settings", json!({ "max_logs": null })).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["max_logs"], Value::Null); + + ingest_lines(&engine, 2, &["c", "d"]).await; + let (_, body) = get_json(&app, "/api/logs").await; + assert_eq!(body.as_array().unwrap().len(), 3); + } } diff --git a/src/dto.rs b/src/dto.rs index 2dbfe0b..706c4ae 100644 --- a/src/dto.rs +++ b/src/dto.rs @@ -194,6 +194,8 @@ impl From<&CssEntry> for CssEntryDto { pub struct SettingsDto { pub color_scheme: ColorScheme, pub active_profile: Option, + /// `None` means unlimited (ticket 026). + pub max_logs: Option, } impl From<&Settings> for SettingsDto { @@ -201,16 +203,33 @@ impl From<&Settings> for SettingsDto { SettingsDto { color_scheme: settings.color_scheme, active_profile: settings.profiles.active_profile.clone(), + max_logs: settings.max_logs, } } } -/// Request body for `PATCH /api/settings`. Only `color_scheme` is -/// updatable for now; `None` leaves the current value untouched (so a -/// partial body doesn't reset other fields). +/// Request body for `PATCH /api/settings`. `color_scheme`: `None` leaves the +/// current value untouched. `max_logs`: distinguishes "field omitted" (leave +/// unchanged, outer `None`) from "field present" (inner `Option` +/// applied verbatim — `null` clears the cap back to unlimited, a number sets +/// it), via `deserialize_double_option`. #[derive(Debug, Deserialize)] pub struct UpdateSettingsRequest { pub color_scheme: Option, + #[serde(default, deserialize_with = "deserialize_double_option")] + pub max_logs: Option>, +} + +/// Distinguishes a missing JSON field (outer `None`, via `#[serde(default)]` +/// short-circuiting before this runs) from a present `null` (`Some(None)`) +/// or present value (`Some(Some(value))`) — plain `Option>` can't +/// tell "absent" and "explicit null" apart on its own. +fn deserialize_double_option<'de, D, T>(deserializer: D) -> Result>, D::Error> +where + D: serde::Deserializer<'de>, + T: Deserialize<'de>, +{ + Option::::deserialize(deserializer).map(Some) } /// Summary of one profile for `GET /api/settings/profiles` (ticket 011): diff --git a/src/engine.rs b/src/engine.rs index 2905530..93b4404 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -96,6 +96,11 @@ pub enum EngineMessage { name: String, respond_to: oneshot::Sender, }, + /// Sets the cap on ingested logs kept in memory (ticket 026). `None` + /// means unlimited. Sent on startup (from `Settings::max_logs`) and + /// whenever the setting changes at runtime, so the cap takes effect on + /// the next ingest without a restart. + SetMaxLogs(Option), Query { query: EngineQuery, respond_to: oneshot::Sender, @@ -129,6 +134,8 @@ pub type EngineHandle = mpsc::Sender; /// have to clone the index to answer a query. pub async fn run(mut rx: mpsc::Receiver) { let mut index: BTreeMap = BTreeMap::new(); + let mut max_logs: Option = None; + let mut log_count: usize = 0; while let Some(message) = rx.recv().await { match message { @@ -138,7 +145,13 @@ pub async fn run(mut rx: mpsc::Receiver) { IndexKey::IngestedLog { sequence, timestamp }, DataValue::LogLiteral { text: text.clone() }, ); + log_count += 1; apply_all_metrics(&mut index, sequence, timestamp, &text); + evict_excess_logs(&mut index, &mut log_count, max_logs); + } + EngineMessage::SetMaxLogs(new_max) => { + max_logs = new_max; + evict_excess_logs(&mut index, &mut log_count, max_logs); } EngineMessage::DeclareParam { request, respond_to } => { let result = declare_param(&mut index, request); @@ -656,6 +669,30 @@ fn derive_log_params( params } +/// Evicts the oldest (lowest-`sequence`) ingested logs until `log_count` is +/// back at or under `max_logs`. `sequence` numbering is not renumbered on +/// eviction, and any `Bucket` values already derived from an evicted log are +/// left in place — buckets are aggregate data, not raw logs, so they're not +/// retroactively pruned. +fn evict_excess_logs( + index: &mut BTreeMap, + log_count: &mut usize, + max_logs: Option, +) { + let Some(max_logs) = max_logs else { return }; + while *log_count > max_logs { + let Some(oldest_key) = index + .range(IndexKey::Start..IndexKey::LogParamSplit) + .next() + .map(|(key, _)| key.clone()) + else { + break; + }; + index.remove(&oldest_key); + *log_count -= 1; + } +} + fn logs_range(index: &BTreeMap) -> impl Iterator { index.range(IndexKey::Start..IndexKey::LogParamSplit).filter_map(|(key, value)| { match (key, value) { @@ -991,6 +1028,44 @@ mod tests { assert!(index.is_empty()); } + #[test] + fn evict_excess_logs_drops_oldest_sequences_first() { + let mut index = BTreeMap::new(); + let mut log_count = 0; + for sequence in 0..5u64 { + index.insert( + IndexKey::IngestedLog { sequence, timestamp: Timestamp::from_millis(sequence as i64) }, + DataValue::LogLiteral { text: format!("line {sequence}") }, + ); + log_count += 1; + } + + evict_excess_logs(&mut index, &mut log_count, Some(3)); + + assert_eq!(log_count, 3); + let remaining: Vec<_> = logs_range(&index).map(|(seq, _, _)| seq).collect(); + assert_eq!(remaining, vec![2, 3, 4]); + } + + #[test] + fn evict_excess_logs_is_noop_when_unlimited_or_under_cap() { + let mut index = BTreeMap::new(); + let mut log_count = 0; + for sequence in 0..3u64 { + index.insert( + IndexKey::IngestedLog { sequence, timestamp: Timestamp::from_millis(sequence as i64) }, + DataValue::LogLiteral { text: "x".to_string() }, + ); + log_count += 1; + } + + evict_excess_logs(&mut index, &mut log_count, None); + assert_eq!(log_count, 3); + + evict_excess_logs(&mut index, &mut log_count, Some(10)); + assert_eq!(log_count, 3); + } + #[test] fn declarations_stay_within_their_range() { let mut index = BTreeMap::new(); diff --git a/src/main.rs b/src/main.rs index 4399c27..d500cff 100644 --- a/src/main.rs +++ b/src/main.rs @@ -58,6 +58,10 @@ async fn main() { let (engine_tx, engine_rx) = mpsc::channel(1024); tokio::spawn(engine::run(engine_rx)); + { + let max_logs = settings.lock().unwrap().max_logs; + let _ = engine_tx.send(engine::EngineMessage::SetMaxLogs(max_logs)).await; + } // Ticket 009: reload the active profile's param/metric declarations into the // engine BEFORE starting ingest below. Ordering matters: declarations must be diff --git a/src/settings.rs b/src/settings.rs index b0c28f6..6f508a6 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -68,6 +68,10 @@ pub struct Settings { pub color_scheme: ColorScheme, pub css_entries: Vec, pub profiles: Profiles, + /// Maximum number of ingested logs kept in memory. `None` (the default) + /// means unlimited — matches today's unbounded retention. + #[serde(default)] + pub max_logs: Option, } impl Default for Settings { @@ -80,6 +84,7 @@ impl Default for Settings { active_profile: None, profiles_dir: "profiles".to_string(), }, + max_logs: None, } } } diff --git a/webapp/src/components/NavMenu.tsx b/webapp/src/components/NavMenu.tsx index ff4ee73..47bc8f8 100644 --- a/webapp/src/components/NavMenu.tsx +++ b/webapp/src/components/NavMenu.tsx @@ -1,4 +1,4 @@ -import { useState } from 'react' +import { useEffect, useState } from 'react' import type { Stats } from '../lib/api' import type { ColorScheme } from '../lib/api' import { useSettingsPanel } from '../lib/useSettingsPanel' @@ -120,6 +120,25 @@ export function NavMenu({ stats }: { stats: Stats | null }) { function SettingsPanel({ panel }: { panel: ReturnType }) { const [filename, setFilename] = useState('') const [content, setContent] = useState('') + const [maxLogsInput, setMaxLogsInput] = useState('') + + useEffect(() => { + setMaxLogsInput(panel.settings?.max_logs != null ? String(panel.settings.max_logs) : '') + }, [panel.settings?.max_logs]) + + function handleMaxLogsBlur() { + const trimmed = maxLogsInput.trim() + if (trimmed === '') { + panel.setMaxLogs(null) + return + } + const parsed = Number(trimmed) + if (!Number.isFinite(parsed) || parsed <= 0) { + setMaxLogsInput(panel.settings?.max_logs != null ? String(panel.settings.max_logs) : '') + return + } + panel.setMaxLogs(Math.floor(parsed)) + } async function handleFileChange(e: React.ChangeEvent) { const file = e.target.files?.[0] @@ -158,6 +177,22 @@ function SettingsPanel({ panel }: { panel: ReturnType } +
+

Log retention

+
+ setMaxLogsInput(e.target.value)} + onBlur={handleMaxLogsBlur} + className="w-28 rounded border border-border-soft bg-surface-1 px-2 py-1 text-xs text-text placeholder:text-text-subtle focus:border-border-strong focus:outline-none" + /> + max logs kept in memory (blank = unlimited) +
+
+

Import CSS

diff --git a/webapp/src/lib/api.ts b/webapp/src/lib/api.ts index 3629880..54fbb6f 100644 --- a/webapp/src/lib/api.ts +++ b/webapp/src/lib/api.ts @@ -90,6 +90,8 @@ export type ColorScheme = export interface Settings { color_scheme: ColorScheme active_profile: string | null + /** Maximum ingested logs kept in memory; `null` means unlimited. */ + max_logs: number | null } export type CssEntryKind = 'imported' | 'linked' @@ -263,8 +265,14 @@ export function fetchSettings(): Promise { return getJson('/api/settings') } -export function updateSettings(update: { colorScheme?: ColorScheme }): Promise { - return sendJson('/api/settings', 'PATCH', { color_scheme: update.colorScheme }) +export function updateSettings(update: { + colorScheme?: ColorScheme + /** Omit to leave unchanged; `null` clears the cap back to unlimited. */ + maxLogs?: number | null +}): Promise { + const body: Record = { color_scheme: update.colorScheme } + if ('maxLogs' in update) body.max_logs = update.maxLogs + return sendJson('/api/settings', 'PATCH', body) } export function fetchCssEntries(): Promise { diff --git a/webapp/src/lib/useSettingsPanel.ts b/webapp/src/lib/useSettingsPanel.ts index 79e3812..f5d26f5 100644 --- a/webapp/src/lib/useSettingsPanel.ts +++ b/webapp/src/lib/useSettingsPanel.ts @@ -123,6 +123,15 @@ export function useSettingsPanel() { [guarded], ) + const setMaxLogs = useCallback( + (maxLogs: number | null) => + guarded(async () => { + const next = await updateSettings({ maxLogs }) + setSettings(next) + })(), + [guarded], + ) + const importCssEntry = useCallback( (filename: string, content: string) => guarded(async () => { @@ -174,6 +183,7 @@ export function useSettingsPanel() { profiles, error, setColorScheme, + setMaxLogs, importCssEntry, removeCssEntry, addProfile,