diff --git a/.claude/worktrees/agent-a25bd4a44da9f00b3 b/.claude/worktrees/agent-a25bd4a44da9f00b3 new file mode 160000 index 0000000..0311eeb --- /dev/null +++ b/.claude/worktrees/agent-a25bd4a44da9f00b3 @@ -0,0 +1 @@ +Subproject commit 0311eeb7122adf8c1bc263d17bb6e995c91bef99 diff --git a/.claude/worktrees/agent-a350079668e05192f b/.claude/worktrees/agent-a350079668e05192f new file mode 160000 index 0000000..202b1dc --- /dev/null +++ b/.claude/worktrees/agent-a350079668e05192f @@ -0,0 +1 @@ +Subproject commit 202b1dcf9ffd8ca1fa7f26f95a324588d45bcbf8 diff --git a/.claude/worktrees/agent-a423781d048bfa568 b/.claude/worktrees/agent-a423781d048bfa568 new file mode 160000 index 0000000..f62830d --- /dev/null +++ b/.claude/worktrees/agent-a423781d048bfa568 @@ -0,0 +1 @@ +Subproject commit f62830d8f7c16cb38da214bbaeaf58b16e3bf28b diff --git a/.claude/worktrees/agent-a63474e92b05425a5 b/.claude/worktrees/agent-a63474e92b05425a5 new file mode 160000 index 0000000..2264762 --- /dev/null +++ b/.claude/worktrees/agent-a63474e92b05425a5 @@ -0,0 +1 @@ +Subproject commit 22647620faf57cca3963936b2b92971efe050b9a diff --git a/Cargo.lock b/Cargo.lock index 9a308b0..8577b85 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -233,6 +233,27 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "directories" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys", +] + [[package]] name = "errno" version = "0.3.14" @@ -243,6 +264,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + [[package]] name = "find-msvc-tools" version = "0.1.12" @@ -297,6 +324,28 @@ dependencies = [ "slab", ] +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + [[package]] name = "heck" version = "0.5.0" @@ -448,6 +497,21 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "libredox" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d8f1ea3f21fd3405dcaf6c9b5c1630af9afc422d9073ea39c5f6d6c772e08ed" +dependencies = [ + "libc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "lock_api" version = "0.4.14" @@ -523,6 +587,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "ottyl" version = "0.1.0" @@ -530,12 +600,14 @@ dependencies = [ "axum", "chrono", "clap", + "directories", "humantime", "pest", "pest_derive", "regex", "serde", "serde_json", + "tempfile", "tokio", "tower-http", ] @@ -635,6 +707,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "redox_syscall" version = "0.5.18" @@ -644,6 +722,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror", +] + [[package]] name = "regex" version = "1.13.1" @@ -673,6 +762,19 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "rustversion" version = "1.0.23" @@ -829,6 +931,39 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "tokio" version = "1.53.1" diff --git a/Cargo.toml b/Cargo.toml index fd21aed..0f03e2f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,3 +15,7 @@ pest = "2" pest_derive = "2" chrono = "0.4" humantime = "2" +directories = "6.0.0" + +[dev-dependencies] +tempfile = "3.27.0" diff --git a/README.md b/README.md index 7422cc3..1ad65d2 100644 --- a/README.md +++ b/README.md @@ -192,5 +192,10 @@ Example `node_id` for a cluster test or something. Then it may be possible to capture specific events of interest. There wouldn't be a relation between nodes, only the sequence of events determines relations in the sequence diagram. +## Index allocation size + +It should be possible to see how much ram is being consumed by the program. +What should happen if it reaches a limit? + diff --git a/src/api.rs b/src/api.rs index c4baca2..dde0600 100644 --- a/src/api.rs +++ b/src/api.rs @@ -1,5 +1,8 @@ +use std::sync::{Arc, Mutex}; + use crate::engine::{EngineHandle, EngineMessage, EngineQuery, EngineResponse}; use crate::dto::{LogsParams, TimelineParams}; +use crate::settings::Settings; use axum::extract::{Query, State}; use axum::http::StatusCode; use axum::response::{IntoResponse, Json}; @@ -8,13 +11,20 @@ use axum::Router; use tokio::sync::oneshot; use tower_http::services::{ServeDir, ServeFile}; +/// Shared, process-wide settings handle. Cheap to clone; lock is only held +/// briefly since settings changes are rare (unlike the engine's hot path). +pub type SettingsHandle = Arc>; + #[derive(Clone)] struct ApiState { engine: EngineHandle, + // TODO(ticket 010/011): wire up settings/CSS/NavMenu endpoints using this. + #[allow(dead_code)] + settings: SettingsHandle, } -pub fn router(engine: EngineHandle, static_dir: &str) -> Router { - let state = ApiState { engine }; +pub fn router(engine: EngineHandle, static_dir: &str, settings: SettingsHandle) -> Router { + let state = ApiState { engine, settings }; let index_html = format!("{static_dir}/index.html"); let static_service = ServeDir::new(static_dir).fallback(ServeFile::new(index_html)); diff --git a/src/main.rs b/src/main.rs index 9c34462..5f46b3e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,12 +7,18 @@ mod filter_parser; mod index_key; mod ingest; mod metric_primitive; +mod settings; mod timestamp; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + use clap::Parser; use tokio::net::TcpListener; use tokio::sync::mpsc; +use settings::Settings; + /// Observability dashboard from terminal output. #[derive(Parser, Debug)] struct Args { @@ -23,17 +29,28 @@ struct Args { /// Directory containing the built webapp (index.html + assets). #[arg(long, default_value = "webapp/dist")] static_dir: String, + + /// Path to the settings file. Defaults to a per-OS config location + /// (e.g. `~/.config/ottyl/settings.json` on Linux). + #[arg(long)] + settings_path: Option, } #[tokio::main] async fn main() { let args = Args::parse(); + let settings_path = args.settings_path.clone().unwrap_or_else(|| { + settings::default_settings_path() + .expect("could not determine a default settings path; pass --settings-path") + }); + let settings = Arc::new(Mutex::new(Settings::load(&settings_path))); + let (engine_tx, engine_rx) = mpsc::channel(1024); tokio::spawn(engine::run(engine_rx)); tokio::spawn(ingest::run(engine_tx.clone())); - let app = api::router(engine_tx, &args.static_dir); + let app = api::router(engine_tx, &args.static_dir, settings); let listener = TcpListener::bind(("0.0.0.0", args.port)) .await .expect("failed to bind listener"); @@ -47,3 +64,23 @@ async fn main() { .await .expect("server error"); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn settings_path_defaults_to_none() { + let args = Args::parse_from(["ottyl"]); + assert_eq!(args.settings_path, None); + } + + #[test] + fn settings_path_flag_overrides_default() { + let args = Args::parse_from(["ottyl", "--settings-path", "/tmp/custom-settings.json"]); + assert_eq!( + args.settings_path, + Some(PathBuf::from("/tmp/custom-settings.json")) + ); + } +} diff --git a/src/settings.rs b/src/settings.rs new file mode 100644 index 0000000..b8be1fa --- /dev/null +++ b/src/settings.rs @@ -0,0 +1,199 @@ +//! Process-wide settings model: colour scheme, CSS entries, and profile +//! pointers. There is exactly one `Settings` instance at runtime, loaded once +//! on startup and persisted to disk whenever it changes (see `main.rs`). +//! +//! Persistence format is JSON (via `serde_json`, already a dependency) rather +//! than TOML, to keep the dependency list minimal. + +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +use directories::ProjectDirs; +use serde::{Deserialize, Serialize}; + +/// User's preferred colour scheme for the webapp. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "lowercase")] +pub enum ColorScheme { + Light, + Dark, + #[default] + System, +} + +/// A single CSS entry configured for the webapp: either content that ottyl +/// stores itself ("imported") or a reference to an external path/URL that +/// the webapp links to directly ("linked", and therefore not deletable via +/// the settings API since ottyl doesn't own it). +/// +/// Ticket 010 builds the CSS import/link HTTP API around this shape. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "lowercase")] +pub enum CssEntry { + Imported { id: String, filename: String }, + Linked { path_or_url: String }, +} + +/// Placeholder shape for profile management (ticket 009 builds CRUD on top +/// of this). Profiles are expected to live as individual files under a +/// dedicated directory; this struct only tracks known profile names and +/// which one is currently active. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +pub struct Profiles { + /// Names of known profiles. Ticket 009 owns creating/loading the actual + /// profile files. + pub names: Vec, + /// Name of the currently active profile, if any. + pub active_profile: Option, + /// Directory (relative to the settings file's directory, or absolute) + /// where individual profile files are expected to live. + pub profiles_dir: String, +} + +/// The single, process-wide settings instance. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Settings { + pub color_scheme: ColorScheme, + pub css_entries: Vec, + pub profiles: Profiles, +} + +impl Default for Settings { + fn default() -> Self { + Settings { + color_scheme: ColorScheme::default(), + css_entries: Vec::new(), + profiles: Profiles { + names: Vec::new(), + active_profile: None, + profiles_dir: "profiles".to_string(), + }, + } + } +} + +/// Computes the default settings file path using OS-conventional config +/// directories: `/settings.json`, where `config_dir` comes from +/// `directories::ProjectDirs::from("", "", "ottyl")`. +/// +/// Returns `None` if no valid config directory could be determined for the +/// current OS/user (this mirrors `ProjectDirs::from`'s own fallibility). +pub fn default_settings_path() -> Option { + ProjectDirs::from("", "", "ottyl").map(|dirs| dirs.config_dir().join("settings.json")) +} + +impl Settings { + /// Loads settings from `path`. If the file doesn't exist yet, a default + /// `Settings` is created, persisted to `path`, and returned. + /// + /// Any I/O or parse error while reading an existing file is treated as + /// non-fatal: it is logged to stderr and a default `Settings` is + /// returned (without overwriting the unreadable file). + pub fn load(path: &Path) -> Settings { + match fs::read_to_string(path) { + Ok(contents) => match serde_json::from_str(&contents) { + Ok(settings) => settings, + Err(err) => { + eprintln!( + "warning: failed to parse settings file at {}: {err}; using defaults", + path.display() + ); + Settings::default() + } + }, + Err(err) if err.kind() == io::ErrorKind::NotFound => { + let settings = Settings::default(); + if let Err(save_err) = settings.save(path) { + eprintln!( + "warning: failed to create default settings file at {}: {save_err}", + path.display() + ); + } + settings + } + Err(err) => { + eprintln!( + "warning: failed to read settings file at {}: {err}; using defaults", + path.display() + ); + Settings::default() + } + } + } + + /// Persists these settings to `path` as pretty-printed JSON, creating + /// any missing parent directories first. + pub fn save(&self, path: &Path) -> io::Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let contents = serde_json::to_string_pretty(self) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; + fs::write(path, contents) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_settings_path_ends_with_settings_json() { + // We can't assert the exact OS path here without touching real + // config dirs, but we can verify the path-computation logic shapes + // it as expected. + if let Some(path) = default_settings_path() { + assert_eq!(path.file_name().unwrap(), "settings.json"); + assert!(path.parent().is_some()); + } + } + + #[test] + fn load_creates_default_when_missing() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("nested").join("settings.json"); + assert!(!path.exists()); + + let settings = Settings::load(&path); + assert_eq!(settings, Settings::default()); + assert!(path.exists(), "load() should persist a default file"); + + let reloaded = Settings::load(&path); + assert_eq!(reloaded, settings); + } + + #[test] + fn round_trips_through_save_and_load() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("settings.json"); + + let mut settings = Settings::default(); + settings.color_scheme = ColorScheme::Dark; + settings.css_entries.push(CssEntry::Imported { + id: "abc123".to_string(), + filename: "theme.css".to_string(), + }); + settings.css_entries.push(CssEntry::Linked { + path_or_url: "https://example.com/style.css".to_string(), + }); + settings.profiles.names.push("default".to_string()); + settings.profiles.active_profile = Some("default".to_string()); + + settings.save(&path).unwrap(); + + let loaded = Settings::load(&path); + assert_eq!(loaded, settings); + } + + #[test] + fn save_creates_parent_directories() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("a").join("b").join("settings.json"); + + let settings = Settings::default(); + settings.save(&path).unwrap(); + + assert!(path.exists()); + } +} diff --git a/tickets/001-param-declarations.md b/tickets/001-param-declarations.md new file mode 100644 index 0000000..69d040e --- /dev/null +++ b/tickets/001-param-declarations.md @@ -0,0 +1,35 @@ +# 001 — Param declarations + +## Summary + +Implement the `ParamDeclaration` slice of the primary index: users can declare how a +named parameter is derived from a log line (e.g. deriving a timestamp different from +the ingestion timestamp, or a log level), and the engine stores/serves those +declarations. + +## Scope + +- `IndexKey::ParamDeclaration { name }` and a matching `DataValue` variant for the + declaration body (a regex against the raw log text, plus how the capture maps to a + `MetricPrimitive` — reuse `RegexToMetricPrimitiveMatcher` / `MetricPrimitiveKind` + from `data_value.rs`, which already exist but are unused). +- Engine message(s) to add/list/remove a param declaration. +- Validate declared regexes at declaration time (reject invalid regex / unknown + primitive kind up front rather than failing per-log later). +- No derivation/backfill against existing logs yet — that's + [002](002-metric-declarations.md) / [003](003-metric-bucket-derivation.md), since + metrics are what actually get indexed into buckets. This ticket is just the + declaration storage + validation. + +## Acceptance criteria + +- Declaring a param with a valid regex + capture group succeeds and is retrievable. +- Declaring a param with an invalid regex or referencing a non-existent capture group + is rejected with a clear error, not silently stored. +- Declarations sit in their own contiguous range of the BTreeMap + (`ParamDeclaration` between `LogParamSplit` and `ParamMetricSplit`), per the + `IndexKey` ordering in the README. + +## Depends on + +None — this is foundational. diff --git a/tickets/002-metric-declarations.md b/tickets/002-metric-declarations.md new file mode 100644 index 0000000..c18e744 --- /dev/null +++ b/tickets/002-metric-declarations.md @@ -0,0 +1,35 @@ +# 002 — Metric declarations + +## Summary + +Implement the `MetricDeclaration` slice of the index: a language/API for declaring a +metric — a regex that matches a log message plus params extracted from it — so that +matching logs can later be bucketed ([003](003-metric-bucket-derivation.md)). + +## Scope + +- `IndexKey::MetricDeclaration { name }` paired with `DataValue::MetricDeclaration` + (already sketched in `data_value.rs`: `message_match_regex` + + `Vec`). +- Engine message(s) to add/list/remove a metric declaration. +- A metric declaration's params may reference params already declared via + [001](001-param-declarations.md) (e.g. label a bucket by an existing param) in + addition to inline capture groups on the metric's own regex — decide and document + which is supported for v1 (README implies inline capture-group derivation is the + primary mechanism: "buckets and parameters can be derived" from the metric + declaration itself). +- Validate the regex and capture-group → primitive-kind mapping at declaration time, + same as param declarations. + +## Acceptance criteria + +- Declaring a metric with a valid regex + param matchers succeeds and is retrievable. +- Invalid regex, or a matcher referencing a capture group the regex doesn't have, is + rejected at declaration time. +- Declarations sit in their own contiguous range of the BTreeMap + (`MetricDeclaration` between `ParamMetricSplit` and `MetricBucketSplit`). + +## Depends on + +- [001 — Param declarations](001-param-declarations.md) (shares matcher/primitive + validation code and the "derive a value from a regex capture" concept). diff --git a/tickets/003-metric-bucket-derivation.md b/tickets/003-metric-bucket-derivation.md new file mode 100644 index 0000000..d267584 --- /dev/null +++ b/tickets/003-metric-bucket-derivation.md @@ -0,0 +1,37 @@ +# 003 — Metric bucket derivation + +## Summary + +Apply metric declarations to log lines and populate `IndexKey::Bucket` / +`DataValue::BucketValue` entries, both retroactively (backfill already-ingested logs +when a metric is declared) and going forward (index each new log against all current +metric declarations as it's ingested). + +## Scope + +- On `MetricDeclaration` add: scan the existing `IngestedLog` range, run the metric's + regex against each log's text, and for each match compute the labeled `Bucket` key + (`metric_name` + `labels: BTreeMap`) and append + `(sequence, timestamp, value)` to that bucket's `BucketValue`. +- On new log ingest: run all currently-declared metrics' regexes against the line and + update buckets incrementally, same as backfill. +- Decide bucket cardinality behavior: a single log line can match multiple metrics: it + should update all matching buckets. Document how differing label sets from the same + metric become distinct `Bucket` keys (labels are part of the key, so this falls out + naturally from `BTreeMap` — just needs the insert-or-append logic). +- This is the piece that makes `MetricPrimitive`, `DataValue::BucketValue`, and the + `Bucket` index range (currently defined but never populated) actually load-bearing. + +## Acceptance criteria + +- Declaring a metric against a log stream that already has matching lines populates + buckets for those lines without needing new input. +- Ingesting a new log line after metrics are declared updates the relevant buckets + live. +- A log line matching zero metrics doesn't touch the bucket range. +- Bucket values preserve insertion order / are queryable by sequence and timestamp + range (needed by [004](004-metrics-api.md) and dashboards). + +## Depends on + +- [002 — Metric declarations](002-metric-declarations.md) diff --git a/tickets/004-metrics-api.md b/tickets/004-metrics-api.md new file mode 100644 index 0000000..11c9d6c --- /dev/null +++ b/tickets/004-metrics-api.md @@ -0,0 +1,31 @@ +# 004 — Metrics API + +## Summary + +Expose param/metric declarations and bucket data over HTTP so the frontend (and the +filter/dashboard query engines) can declare metrics and read their values. + +## Scope + +- `POST /api/params` — declare a param ([001](001-param-declarations.md)). +- `POST /api/metrics` — declare a metric ([002](002-metric-declarations.md)). +- `GET /api/metrics` — list declared metrics (+ their declared params/labels). +- `GET /api/metrics/:name` — query bucket values for a metric, filtered by label + selector and time/sequence range, mirroring the existing `LogsParams`/ + `TimelineParams` pattern in `dto.rs`. +- Follow the existing engine architecture: new `EngineQuery`/`EngineMessage` variants, + handled in `engine.rs`'s single-owner loop, DTOs added to `dto.rs`, routes added to + `api.rs`'s router — no new locking or shared state. + +## Acceptance criteria + +- Declaring a param/metric via the API round-trips through to engine state and shows + up in `GET /api/metrics`. +- Querying a metric's bucket values supports the same start/end and limit-style + params as the existing logs/timeline endpoints. +- Bad declarations (invalid regex etc.) return a 4xx with a useful error body, not a + 500 or silent no-op. + +## Depends on + +- [003 — Metric bucket derivation](003-metric-bucket-derivation.md) diff --git a/tickets/005-filterbar-pest-parser.md b/tickets/005-filterbar-pest-parser.md new file mode 100644 index 0000000..3374abd --- /dev/null +++ b/tickets/005-filterbar-pest-parser.md @@ -0,0 +1,37 @@ +# 005 — FilterBar query language (pest grammar) + +## Summary + +Build the syntax-aware search language described in the README's FilterBar section, +using `pest`, supporting regex searches and metric comparisons combined with +boolean operators. + +## Scope + +- Add `pest` + `pest_derive` dependencies. +- Grammar covering: + - `regex("...")` — raw-text regex search over log lines. + - `metric_name[param] value` comparisons (`<`, `>`, `<=`, `>=`, `==`, `!=`) — + `metric_name[param]<5` per the README example. + - `sequence` and `timestamp` as reserved metric names usable the same way + (`sequence<5`, `timestamp>12:00`). + - Boolean combinators `and` / `or` with standard precedence, e.g. + `regex("error") and sequence<5 or timestamp>12:00`. +- Parse into an AST (not directly into engine queries — that's + [007](007-filterbar-integration.md)). +- Surface parse errors with position info suitable for showing under the FilterBar + input. + +## Acceptance criteria + +- Grammar test cases cover: bare regex, single metric comparison, reserved-metric + comparison, `and`/`or` combinations with correct precedence, and at least one + malformed-input case producing a located error rather than a panic. +- Parser is a standalone module with no dependency on the engine/index types, so it + can be unit tested in isolation. + +## Depends on + +None directly, but the metric-comparison side of the grammar is only meaningful once +[002](002-metric-declarations.md) exists — build/test this ticket against metric +names as opaque strings. diff --git a/tickets/006-filterbar-datetime-parsing.md b/tickets/006-filterbar-datetime-parsing.md new file mode 100644 index 0000000..8fe54f6 --- /dev/null +++ b/tickets/006-filterbar-datetime-parsing.md @@ -0,0 +1,30 @@ +# 006 — FilterBar datetime parsing + +## Summary + +Support the several datetime formats the README requires for timestamp comparisons +in the filter language: relative humantime, absolute time, and full dates. + +## Scope + +- Relative humantime: `5h`, `-3m`, etc. — resolved against "now" (or against the + query's reference time, if one is introduced later) at parse/eval time. +- Absolute time: e.g. `12:00` (time-of-day, as in the README's + `timestamp>12:00` example) — needs a defined rule for which day it resolves to. +- Full date: complete date(+time) literals. +- Produce a single normalized value (`Timestamp`/millis) regardless of input form, for + the filter evaluator ([007](007-filterbar-integration.md)) to compare against. +- Pick and document a crate (e.g. `humantime` for the relative case) rather than + hand-rolling a datetime parser. + +## Acceptance criteria + +- Each of the three formats parses to the correct absolute millis value in unit + tests, including edge cases (negative relative offsets, time-of-day near midnight). +- Ambiguous/invalid input produces a clear error rather than a silently wrong + timestamp. + +## Depends on + +None — standalone parsing module, consumed by +[007 — FilterBar integration](007-filterbar-integration.md). diff --git a/tickets/007-filterbar-integration.md b/tickets/007-filterbar-integration.md new file mode 100644 index 0000000..08b2382 --- /dev/null +++ b/tickets/007-filterbar-integration.md @@ -0,0 +1,36 @@ +# 007 — FilterBar query integration + +## Summary + +Wire the parsed filter AST into actual log/timeline queries against the engine, so +FilterBar's syntax (not just a naive substring match) drives what's shown in +LogPreview and LogTimeline. + +## Scope + +- Evaluator that walks the AST from [005](005-filterbar-pest-parser.md), resolving + `regex(...)` against log text and `metric_name[param] value` against bucket + data ([004](004-metrics-api.md)), using datetime literals resolved via + [006](006-filterbar-datetime-parsing.md). +- New/extended `EngineQuery::Logs` (and `Timeline`) filter parameter that accepts the + compiled filter, evaluated inside the engine's single-owner loop (per the README's + "engine is sole owner of the btreemap" architecture — don't evaluate against a + copy). +- `sequence`/`timestamp` reserved-metric comparisons map directly to the + `IngestedLog` key fields rather than a declared metric lookup. +- API layer: accept the raw filter string, parse it (return 4xx on parse error), + compile to whatever the engine query needs, send to engine. + +## Acceptance criteria + +- `GET /api/logs?filter=...` and the timeline equivalent apply the full filter + language, not just substring matching. +- A filter combining a regex clause and a metric comparison with `and`/`or` returns + correct results against a seeded log+metric fixture. +- Malformed filter strings return a 4xx with the parser's error message. + +## Depends on + +- [004 — Metrics API](004-metrics-api.md) +- [005 — FilterBar pest parser](005-filterbar-pest-parser.md) +- [006 — FilterBar datetime parsing](006-filterbar-datetime-parsing.md) diff --git a/tickets/008-settings-model.md b/tickets/008-settings-model.md new file mode 100644 index 0000000..0d0d20d --- /dev/null +++ b/tickets/008-settings-model.md @@ -0,0 +1,33 @@ +# 008 — Settings model + +## Summary + +Implement the single, process-wide settings model described in the README's NavMenu +section: colour scheme, CSS path config, and the settings file location — settable +via CLI args, persisted at a sensible default OS location. + +## Scope + +- A `Settings` struct: colour scheme choice, list of imported/linked CSS entries + (see [010](010-css-management.md)), and whatever else profiles need to hang off of. +- Default settings file location using an OS-conventions crate (e.g. `directories`) + rather than a hardcoded path. +- `clap` arg(s) to override the settings file path at runtime (extend the existing + `Args` in `main.rs`, which today only has `--port`/`--static-dir`). +- Load-on-start / save-on-change semantics; "at runtime, there is only 1 settings" — + i.e. a single in-memory settings instance, not per-request state, analogous to how + the engine owns the index. +- Settings "owns" the profiles per the README — this ticket only needs to define + where profiles live under settings, not implement profile CRUD itself (that's + [009](009-presets-profiles.md)). + +## Acceptance criteria + +- Running with no args creates/reads a settings file at the documented default + location. +- `--settings-path` (or similar) overrides it. +- Settings persist across restarts (edit, restart, re-read matches). + +## Depends on + +None. diff --git a/tickets/009-presets-profiles.md b/tickets/009-presets-profiles.md new file mode 100644 index 0000000..878c134 --- /dev/null +++ b/tickets/009-presets-profiles.md @@ -0,0 +1,34 @@ +# 009 — Presets / profiles + +## Summary + +Implement presets ("profiles") that track parsing/indexing rules (param + metric +declarations) and stored dashboards, so restarting ottyl against a given CLI app +doesn't lose that configuration. + +## Scope + +- Profile = a named bundle of: param declarations ([001](001-param-declarations.md)), + metric declarations ([002](002-metric-declarations.md)), and saved dashboards + ([014](014-dashboards-ui.md), once it exists — can stub the dashboard field until + then). +- Persisted under the settings storage location from + [008](008-settings-model.md); settings reference which profile(s) exist / which is + active. +- On startup, load the active profile's param/metric declarations into the engine + before ingestion starts (or apply-and-backfill if logs arrive before load + completes — define the ordering explicitly). +- CRUD for profiles, exposed later via NavMenu UI ([011](011-navmenu-ui.md)). + +## Acceptance criteria + +- Declaring params/metrics through a profile, restarting the process, and re-running + the same CLI command reproduces the same bucket derivation without re-declaring + anything. +- Multiple profiles can exist; switching the active profile changes which + declarations are loaded on next start. + +## Depends on + +- [008 — Settings model](008-settings-model.md) +- [002 — Metric declarations](002-metric-declarations.md) diff --git a/tickets/010-css-management.md b/tickets/010-css-management.md new file mode 100644 index 0000000..b7ba135 --- /dev/null +++ b/tickets/010-css-management.md @@ -0,0 +1,30 @@ +# 010 — CSS import/management + +## Summary + +Let users import custom CSS (stored alongside settings/profiles, deletable) or link +to an external CSS path/URL (not deletable, just referenced). + +## Scope + +- `POST /api/settings/css` (import): accept raw CSS content, store it in the same + directory as settings/profiles ([008](008-settings-model.md)), return an id/handle. +- `DELETE /api/settings/css/:id`: remove an imported CSS file; reject deletion for + linked (non-imported) entries. +- `POST /api/settings/css/link`: register a path/URL reference without copying + content; these are never deletable via the delete endpoint (only "unlink"-able, + if that distinction matters — clarify in implementation). +- Settings model tracks the list of CSS entries (imported + linked) with enough + metadata to distinguish the two kinds. +- Serve imported CSS content back so the frontend can ``/inline it. + +## Acceptance criteria + +- Importing CSS persists content under the settings directory and survives restart. +- Deleting an imported CSS entry removes the file and the settings reference. +- Attempting to delete a linked (non-imported) entry is rejected. +- Linked entries resolve to their original path/URL, not a copy. + +## Depends on + +- [008 — Settings model](008-settings-model.md) diff --git a/tickets/011-navmenu-ui.md b/tickets/011-navmenu-ui.md new file mode 100644 index 0000000..ec817c4 --- /dev/null +++ b/tickets/011-navmenu-ui.md @@ -0,0 +1,30 @@ +# 011 — NavMenu UI + +## Summary + +Build out the NavMenu frontend component for managing settings, presets, and CSS, +per the README's block diagram (NavMenu occupies its own column alongside the rest +of the dashboard). + +## Scope + +- Settings panel: colour scheme picker, CSS import (file/text input) + list of + imported/linked CSS with delete where applicable. +- Presets panel: list/create/switch/delete profiles ([009](009-presets-profiles.md)), + showing each profile's param/metric declaration counts and saved dashboards. +- Wire to the settings/profile/CSS APIs from + [008](008-settings-model.md)/[009](009-presets-profiles.md)/[010](010-css-management.md). +- Apply the active colour scheme / imported CSS to the app shell at runtime. + +## Acceptance criteria + +- Switching active profile in the UI changes engine state (declared params/metrics) + without a manual restart, or clearly communicates that a restart is needed if + that's the chosen design from [009](009-presets-profiles.md). +- Importing/deleting CSS in the UI is reflected immediately in the rendered app. + +## Depends on + +- [008 — Settings model](008-settings-model.md) +- [009 — Presets / profiles](009-presets-profiles.md) +- [010 — CSS import/management](010-css-management.md) diff --git a/tickets/012-metrics-ui.md b/tickets/012-metrics-ui.md new file mode 100644 index 0000000..e9f3026 --- /dev/null +++ b/tickets/012-metrics-ui.md @@ -0,0 +1,27 @@ +# 012 — Metrics UI + +## Summary + +Replace the current "Coming Soon" Metrics placeholder with a real component: declare +metrics via the metric language, and view their derived values. + +## Scope + +- A form/editor for declaring a metric (regex + param matchers) that calls + `POST /api/metrics` ([004](004-metrics-api.md)), with inline validation errors + surfaced from the API's 4xx responses. +- List of currently-declared metrics with their labels. +- A view of a selected metric's bucket values over the active time range (reuse the + time-range plumbing already in `App.tsx`/`LogTimeline`). + +## Acceptance criteria + +- Declaring a metric in the UI immediately reflects newly-matching logs (existing + and new) in the metric's value view, per the backfill behavior from + [003](003-metric-bucket-derivation.md). +- Invalid metric definitions show the backend's validation error instead of failing + silently. + +## Depends on + +- [004 — Metrics API](004-metrics-api.md) diff --git a/tickets/013-dashboards-promql.md b/tickets/013-dashboards-promql.md new file mode 100644 index 0000000..bdea581 --- /dev/null +++ b/tickets/013-dashboards-promql.md @@ -0,0 +1,30 @@ +# 013 — Dashboards: PromQL query support + +## Summary + +Support PromQL-style queries over declared metrics to derive dashboard +visualizations, per the README's Dashboards section ("There should be support for +promql queries to derive the visualisation"). + +## Scope + +- Evaluate a useful PromQL subset (instant/range vector selection by metric name + + label matchers, at minimum; consider `rate()`/`sum()`/`avg()` aggregations as + stretch) against `Bucket` data ([003](003-metric-bucket-derivation.md)). +- Decide whether to hand-roll the subset or bring in an existing PromQL parser + crate — document the choice. +- New `EngineQuery` variant for PromQL evaluation, following the existing + single-owner-evaluates-in-place pattern. +- API endpoint (e.g. `GET /api/query?promql=...`) returning a chart-ready series + format (labels + time-value pairs), shaped for bar/line/pie consumption by + [014](014-dashboards-ui.md). + +## Acceptance criteria + +- A basic instant/range query against a declared metric with a label matcher returns + correct series data against a seeded fixture. +- Malformed PromQL returns a 4xx with a useful error. + +## Depends on + +- [004 — Metrics API](004-metrics-api.md) diff --git a/tickets/014-dashboards-ui.md b/tickets/014-dashboards-ui.md new file mode 100644 index 0000000..45b7a43 --- /dev/null +++ b/tickets/014-dashboards-ui.md @@ -0,0 +1,30 @@ +# 014 — Dashboards UI + +## Summary + +Replace the current "Coming Soon" Dashboards placeholder with a Grafana-style +dashboard builder: bar, line, and pie charts backed by PromQL queries, saved as part +of a profile. + +## Scope + +- Chart components for bar/line/pie (pick a charting lib — none is currently in + `webapp/package.json`). +- Dashboard = a named set of panels, each with a PromQL query + ([013](013-dashboards-promql.md)) and a chart type. +- Save/load dashboards as part of the active profile + ([009](009-presets-profiles.md)), so they persist across restarts. +- Dashboard panels respect the app's active time range, same as LogTimeline/ + LogPreview. + +## Acceptance criteria + +- Creating a panel with a PromQL query against a real declared metric renders the + correct chart type with live data. +- Dashboards persist across restart via the active profile. +- Panels re-query and update when the global time range changes. + +## Depends on + +- [013 — Dashboards: PromQL query support](013-dashboards-promql.md) +- [009 — Presets / profiles](009-presets-profiles.md) diff --git a/tickets/015-logpreview-param-columns.md b/tickets/015-logpreview-param-columns.md new file mode 100644 index 0000000..b15da6f --- /dev/null +++ b/tickets/015-logpreview-param-columns.md @@ -0,0 +1,28 @@ +# 015 — LogPreview: selectable parameter columns + +## Summary + +The README's LogPreview spec says it should be possible to pick which parameters are +visible as columns, alongside the always-visible sequence/timestamp. Currently +LogPreview only shows sequence, timestamp, and raw text. + +## Scope + +- Extend the logs API/DTO ([004](004-metrics-api.md) territory, but specifically the + existing `GET /api/logs`) to include derived param values per log line, sourced + from [001](001-param-declarations.md)'s declarations applied to that line. +- Frontend: column picker UI in/near LogPreview; selected columns persist (localStorage + is fine for a per-viewer preference, or fold into settings if it should be shared — + pick localStorage unless product direction says otherwise). + +## Acceptance criteria + +- Declared params with a match on a given log line appear as selectable, populated + columns in LogPreview. +- Column selection persists across a page reload. + +## Depends on + +- [001 — Param declarations](001-param-declarations.md) +- [004 — Metrics API](004-metrics-api.md) (for the pattern of extending log query + responses) diff --git a/tickets/016-filterbar-ui-integration.md b/tickets/016-filterbar-ui-integration.md new file mode 100644 index 0000000..c96f6b5 --- /dev/null +++ b/tickets/016-filterbar-ui-integration.md @@ -0,0 +1,26 @@ +# 016 — FilterBar UI integration + +## Summary + +Wire the FilterBar frontend component to the real filter query language instead of +the current naive client-side substring match in `App.tsx` +(`logs.filter((l) => l.text.includes(filter))`). + +## Scope + +- Send the raw filter string to `GET /api/logs`/`GET /api/timeline` via the new + `filter` param from [007](007-filterbar-integration.md), instead of filtering + client-side. +- Surface parse errors from the API (4xx + message) inline under the FilterBar input. +- Remove the now-dead client-side `visibleLogs` substring filter in `App.tsx`. + +## Acceptance criteria + +- Typing a valid filter expression (regex + metric comparisons combined with + and/or) narrows both LogPreview and LogTimeline via server-side evaluation. +- An invalid expression shows an inline error and does not clear/break the + currently-displayed logs. + +## Depends on + +- [007 — FilterBar query integration](007-filterbar-integration.md) diff --git a/tickets/017-ntfy-alerts.md b/tickets/017-ntfy-alerts.md new file mode 100644 index 0000000..b00baed --- /dev/null +++ b/tickets/017-ntfy-alerts.md @@ -0,0 +1,24 @@ +# 017 — Ntfy.sh alerts (future idea) + +> The README lists this under "Ideas... that can be implemented in the future, but +> not now." Not scoped for current implementation — tracked here so it isn't lost. + +## Summary + +Let users configure matchers of interest and an ntfy.sh channel, and send a +notification to that channel when a matching event occurs. + +## Scope (sketch, not final) + +- Matcher definition, likely reusing the FilterBar query language + ([005](005-filterbar-pest-parser.md)) or the metric declaration language + ([002](002-metric-declarations.md)) rather than inventing a third syntax. +- Ntfy channel config, persisted via settings ([008](008-settings-model.md)). +- Trigger evaluation on log ingest (engine-side, to avoid re-scanning) or on bucket + update, sending an HTTP request to the configured ntfy.sh channel. + +## Depends on + +- [008 — Settings model](008-settings-model.md) +- [005 — FilterBar pest parser](005-filterbar-pest-parser.md) (likely, for matcher + syntax reuse) diff --git a/tickets/018-sequence-diagram.md b/tickets/018-sequence-diagram.md new file mode 100644 index 0000000..55ceee0 --- /dev/null +++ b/tickets/018-sequence-diagram.md @@ -0,0 +1,28 @@ +# 018 — Sequence diagram of events (future idea) + +> The README lists this under "Ideas... that can be implemented in the future, but +> not now." Not scoped for current implementation — tracked here so it isn't lost. + +## Summary + +Let users declare sequence-diagram participants from params or matchers (e.g. +`node_id`), then render a sequence diagram of captured events of interest. No +relation between nodes is modeled — only event sequence/ordering determines the +diagram. + +## Scope (sketch, not final) + +- Participant declaration referencing an existing param + ([001](001-param-declarations.md)), e.g. "one participant per distinct value of + `node_id`". +- Event-of-interest selection, likely reusing FilterBar syntax + ([005](005-filterbar-pest-parser.md)) to define which log lines become diagram + events. +- Rendering: a sequence-diagram view (e.g. mermaid `sequenceDiagram`) driven by + ordered matching events per participant. + +## Depends on + +- [001 — Param declarations](001-param-declarations.md) +- [005 — FilterBar pest parser](005-filterbar-pest-parser.md) (likely, for + event-matcher syntax reuse)