Improve ingestion performance

src/engine/snapshot/mod.rs

@@ -17,6 +17,7 @@ use crate::engine::index_key::IndexKey;
 use crate::engine::message::{EngineQuery, EngineResponse};
 use crate::engine::timestamp::Timestamp;
 use query::handle_query_inner;
+use regex::Regex;
 use sort::{SortHelperAccess, SortHelperIndexEntry};
 use std::collections::{BTreeMap, HashMap};
 
@@ -24,6 +25,13 @@ use std::collections::{BTreeMap, HashMap};
 pub(crate) struct EngineSnapshot {
     pub(crate) index: BTreeMap<IndexKey, DataValue>,
     pub(crate) sort_helper_indexes: HashMap<String, SortHelperIndexEntry>,
+    /// Compiled regexes for param/metric declarations, keyed by declaration name. Compiling a
+    /// regex is far more expensive than matching with one, and `apply_all_metrics`/
+    /// `update_sort_helper_indexes_on_ingest` run per ingested line, so every insertion or
+    /// removal of a declaration (see `params.rs`/`metrics.rs`) must keep these in sync rather
+    /// than recompiling from the stored pattern string on every line.
+    pub(crate) param_regex_cache: HashMap<String, Regex>,
+    pub(crate) metric_regex_cache: HashMap<String, Regex>,
     pub(crate) log_count: usize,
     pub(crate) approx_bytes_total: u64,
     pub(crate) ingest_lag_high_watermark_ms: i64,
@@ -105,7 +113,28 @@ pub(crate) fn approx_log_bytes(text: &str) -> u64 {
 
 impl EngineSnapshot {
     pub(crate) fn from_index(index: BTreeMap<IndexKey, DataValue>) -> Self {
-        EngineSnapshot { index, ..Default::default() }
+        let mut snapshot = EngineSnapshot { index, ..Default::default() };
+        snapshot.rebuild_regex_caches();
+        snapshot
+    }
+
+    /// Recompiles `param_regex_cache`/`metric_regex_cache` from the declarations currently in
+    /// `index`. Needed whenever a snapshot's declarations are populated by inserting straight
+    /// into `index` rather than through the normal declare/insert methods that keep the caches
+    /// in sync incrementally (e.g. `from_index` building a snapshot from a bare index).
+    pub(crate) fn rebuild_regex_caches(&mut self) {
+        self.param_regex_cache = self
+            .param_declarations_range()
+            .filter_map(|(name, message_match_regex, ..)| {
+                regex::Regex::new(message_match_regex).ok().map(|re| (name.to_string(), re))
+            })
+            .collect();
+        self.metric_regex_cache = self
+            .metric_declarations_range()
+            .filter_map(|(name, message_match_regex, ..)| {
+                regex::Regex::new(message_match_regex).ok().map(|re| (name.to_string(), re))
+            })
+            .collect();
     }
 
     pub(crate) fn evict_excess_logs(&mut self, max_logs: Option<usize>) {