Merge ticket 052: allow clearing all params and re-extracting from ingested logs

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XFUNZwCvz3kcod3352ZR2o

src/engine.rs

@@ -36,6 +36,19 @@ pub struct MetricSample {
 /// Also the request body for `POST /api/params` — `Deserialize` is derived directly here rather than adding a separate DTO, since the shape already matches `ParamDeclaration`.
 pub type DeclareParamRequest = ParamDeclaration;
 
+/// Response to `EngineMessage::ClearParams` (ticket 052) — also the JSON body for `DELETE /api/params`.
+#[derive(Debug, Clone, serde::Serialize)]
+pub struct ClearParamsResult {
+    /// How many param declarations (manual + auto-derived) were removed.
+    pub removed: usize,
+    /// How many params `auto_declare_params` re-derived immediately afterward from retained log text.
+    pub reextracted: usize,
+    /// How many `IndexKey::IngestedLog` entries were still retained (not yet evicted by `max_logs`)
+    /// and so were re-scanned — surfaced so a caller can tell the user re-extraction only covers
+    /// logs still held in memory, not lines evicted before the clear.
+    pub logs_considered: usize,
+}
+
 /// A metric references its params by name (resolved against the engine's standalone
 /// `ParamDeclaration` store in `apply_metric_to_log`) rather than embedding its own
 /// regex/capture-group/kind per param — see ticket 037. Also the request body for
@@ -90,6 +103,16 @@ pub enum EngineMessage {
         name: String,
         respond_to: oneshot::Sender<bool>,
     },
+    /// Ticket 052: wipes the entire standalone-param set (manual *and* auto-derived — unlike
+    /// `reload_engine_declarations`'s profile-switch cleanup, this is an explicit user-driven reset,
+    /// not a "keep engine-discovered facts across profiles" cleanup), then immediately re-runs
+    /// `auto_declare_params` over every `IndexKey::IngestedLog` entry still retained in the index,
+    /// oldest to newest, so declaration-order semantics match a fresh ingest. Lines already evicted
+    /// by `max_logs` can't regain their params — `ClearParamsResult::logs_considered` reports how
+    /// many retained lines were re-scanned, so a caller can surface that bound to the user.
+    ClearParams {
+        respond_to: oneshot::Sender<ClearParamsResult>,
+    },
     DeclareMetric {
         request: DeclareMetricRequest,
         respond_to: oneshot::Sender<Result<(), String>>,
@@ -196,6 +219,11 @@ pub async fn run(mut rx: mpsc::Receiver<EngineMessage>, settings: SettingsHandle
                 let existed = index.remove(&IndexKey::ParamDeclaration { name }).is_some();
                 let _ = respond_to.send(existed);
             }
+            EngineMessage::ClearParams { respond_to } => {
+                let removed = clear_all_param_declarations(&mut index);
+                let (reextracted, logs_considered) = reextract_params_from_retained_logs(&mut index);
+                let _ = respond_to.send(ClearParamsResult { removed, reextracted, logs_considered });
+            }
             EngineMessage::DeclareMetric { request, respond_to } => {
                 let result = declare_metric(&mut index, request);
                 let _ = respond_to.send(result);
@@ -400,6 +428,42 @@ fn auto_declare_log_stream_params(index: &mut BTreeMap<IndexKey, DataValue>, tex
     }
 }
 
+/// Ticket 052: removes every `IndexKey::ParamDeclaration` entry — manual and auto-derived alike.
+/// Returns how many were removed. Unlike `RemoveParamDeclaration`'s single-name removal (used by
+/// `reload_engine_declarations` during a profile switch, which deliberately keeps auto-derived
+/// params since they're engine-discovered facts rather than profile state), this is a full reset:
+/// the caller (`EngineMessage::ClearParams`) immediately re-runs `auto_declare_params` afterward, so
+/// wiping the auto-derived ones too just means they come back fresh instead of stale.
+fn clear_all_param_declarations(index: &mut BTreeMap<IndexKey, DataValue>) -> usize {
+    let names: Vec<String> =
+        param_declarations_range(index).map(|(name, ..)| name.to_string()).collect();
+    let removed = names.len();
+    for name in names {
+        index.remove(&IndexKey::ParamDeclaration { name });
+    }
+    removed
+}
+
+/// Ticket 052: re-runs `auto_declare_params` against every `IndexKey::IngestedLog` entry still held
+/// in the index, oldest to newest (matching normal ingest order, so declaration-order semantics —
+/// e.g. which key "wins" a naming collision — come out the same as if the lines were freshly
+/// ingested). Only ever called right after `clear_all_param_declarations`, so every param produced
+/// here is fresh, not a re-declaration.
+///
+/// Returns `(reextracted, logs_considered)`: how many params came back, and how many retained log
+/// lines were re-scanned. Lines already dropped by `evict_excess_logs` before this ran aren't in the
+/// index anymore and so can't contribute — `logs_considered` is how the caller surfaces that bound
+/// to the user, rather than the re-extraction silently looking complete when it isn't.
+fn reextract_params_from_retained_logs(index: &mut BTreeMap<IndexKey, DataValue>) -> (usize, usize) {
+    let texts: Vec<String> = logs_range(index).map(|(_, _, text)| text.to_string()).collect();
+    let logs_considered = texts.len();
+    for text in &texts {
+        auto_declare_params(index, text);
+    }
+    let reextracted = param_declarations_range(index).count();
+    (reextracted, logs_considered)
+}
+
 /// Validates the metric's own regex plus that every name in `label_params`/`value_param` refers to
 /// an already-declared standalone param (not just the first bad one, so a rejection reports every
 /// problem at once). Shared by `declare_metric` and `update_metric` so both reject a bad
@@ -2116,6 +2180,73 @@ mod tests {
         assert_eq!(approx_bytes_total, 3);
     }
 
+    fn insert_ingested_log(index: &mut BTreeMap<IndexKey, DataValue>, sequence: u64, text: &str) {
+        index.insert(
+            IndexKey::IngestedLog { sequence, timestamp: Timestamp::from_millis(sequence as i64) },
+            DataValue::LogLiteral { text: text.to_string() },
+        );
+    }
+
+    #[test]
+    fn clear_all_param_declarations_removes_manual_and_auto_derived() {
+        let mut index = BTreeMap::new();
+        auto_declare_params(&mut index, "x1=-4");
+        declare_param(&mut index, request("level", r"level=(?P<level>\w+)", "level")).unwrap();
+        assert_eq!(param_declarations_range(&index).count(), 2);
+
+        let removed = clear_all_param_declarations(&mut index);
+
+        assert_eq!(removed, 2);
+        assert_eq!(param_declarations_range(&index).count(), 0);
+    }
+
+    #[test]
+    fn reextract_params_from_retained_logs_rebuilds_from_ingested_text_without_new_ingest() {
+        let mut index = BTreeMap::new();
+        insert_ingested_log(&mut index, 0, r#"level=info msg="started""#);
+        insert_ingested_log(&mut index, 1, "x1=-4 y1=34");
+        auto_declare_params(&mut index, r#"level=info msg="started""#);
+        auto_declare_params(&mut index, "x1=-4 y1=34");
+        assert_eq!(param_declarations_range(&index).count(), 4);
+
+        clear_all_param_declarations(&mut index);
+        assert_eq!(param_declarations_range(&index).count(), 0);
+
+        // No new lines ingested — re-extraction works purely off the `IngestedLog` entries already
+        // retained in the index (ticket 052's core acceptance criterion).
+        let (reextracted, logs_considered) = reextract_params_from_retained_logs(&mut index);
+
+        assert_eq!(logs_considered, 2);
+        assert_eq!(reextracted, 4);
+        let declarations: HashMap<&str, _> =
+            param_declarations_range(&index).map(|d| (d.0, d)).collect();
+        assert!(declarations.contains_key("level"));
+        assert!(declarations.contains_key("msg"));
+        assert!(declarations.contains_key("x1"));
+        assert!(declarations.contains_key("y1"));
+        assert!(declarations.values().all(|d| d.6), "re-extracted params must be flagged auto_derived");
+    }
+
+    #[test]
+    fn reextract_params_from_retained_logs_only_covers_lines_still_in_the_index() {
+        // Simulates a line evicted (by `max_logs`) before the clear+re-extract runs: its key/value
+        // pair is simply gone from the index, same as `evict_excess_logs` leaves things, so
+        // `auto_declare_params` never sees it again.
+        let mut index = BTreeMap::new();
+        insert_ingested_log(&mut index, 1, "x1=-4");
+        auto_declare_params(&mut index, "evicted_only_key=1");
+        auto_declare_params(&mut index, "x1=-4");
+        assert_eq!(param_declarations_range(&index).count(), 2);
+
+        clear_all_param_declarations(&mut index);
+        let (reextracted, logs_considered) = reextract_params_from_retained_logs(&mut index);
+
+        assert_eq!(logs_considered, 1, "only the retained log line should be re-scanned");
+        assert_eq!(reextracted, 1, "the evicted line's key can't come back");
+        let declarations: Vec<_> = param_declarations_range(&index).collect();
+        assert_eq!(declarations[0].0, "x1");
+    }
+
     #[test]
     fn declarations_stay_within_their_range() {
         let mut index = BTreeMap::new();