Move alert evaluation to the profile actor (ticket 105)
Alert rules are profile-scoped and read metric buckets, which now live on the profile actor (ticket 104) rather than the primary. alert_states and the per-line evaluate_alerts/deliver_for_rule calls move from the primary's Ingest arm to the profile actor's per-line loop, evaluated against its own snapshot right after that line's apply_all_metrics publishes -- matching the primary's original per-message publish-then- evaluate ordering. The primary's run() no longer needs a SettingsHandle at all now that alerts are gone from its Ingest arm; settings (for alert rules and ntfy config, read fresh per line) moves to engine::profile::spawn instead. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TeyhRfMYjMditek6geDRoQ
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
ce3e741cb0
commit
3751508de8
+9
-41
@@ -3,7 +3,6 @@
|
||||
|
||||
use crate::config::SettingsHandle;
|
||||
use crate::consts::{INGEST_LAG_WATERMARK_MAX_ENTRIES, INGEST_LAG_WATERMARK_WINDOW_MS};
|
||||
use crate::engine::alerts::{self, AlertRuntimeState};
|
||||
use crate::engine::index_key::IndexKey;
|
||||
use crate::engine::message::EngineMessage;
|
||||
use crate::engine::metric_primitive::{MetricPrimitive, Number};
|
||||
@@ -12,7 +11,7 @@ use crate::engine::snapshot::params::log_stream_header_transition;
|
||||
use crate::engine::timestamp::Timestamp;
|
||||
use crate::engine::{ClearParamsResult, IngestedLine};
|
||||
use left_right::Absorb;
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::{mpsc, oneshot, watch};
|
||||
|
||||
@@ -146,13 +145,13 @@ fn new_channel_and_handle(
|
||||
|
||||
pub fn spawn(settings: SettingsHandle, channel_capacity: usize) -> EngineHandle {
|
||||
let (tx, rx, reader, write, version_rx) = new_channel_and_handle(channel_capacity);
|
||||
let profile = crate::engine::profile::spawn(reader.clone(), version_rx);
|
||||
let profile = crate::engine::profile::spawn(reader.clone(), version_rx, settings);
|
||||
let handle = EngineHandle {
|
||||
tx,
|
||||
reader,
|
||||
profile_reader: profile.reader.clone(),
|
||||
};
|
||||
tokio::spawn(run(rx, settings, write, profile));
|
||||
tokio::spawn(run(rx, write, profile));
|
||||
handle
|
||||
}
|
||||
|
||||
@@ -179,25 +178,24 @@ pub(crate) fn spawn_with_profile_observer(
|
||||
channel_capacity: usize,
|
||||
) -> (EngineHandle, tokio::sync::mpsc::UnboundedReceiver<u64>) {
|
||||
let (tx, rx, reader, write, version_rx) = new_channel_and_handle(channel_capacity);
|
||||
let (profile, observed) = crate::engine::profile::spawn_observed(reader.clone(), version_rx);
|
||||
let (profile, observed) =
|
||||
crate::engine::profile::spawn_observed(reader.clone(), version_rx, settings);
|
||||
let handle = EngineHandle {
|
||||
tx,
|
||||
reader,
|
||||
profile_reader: profile.reader.clone(),
|
||||
};
|
||||
tokio::spawn(run(rx, settings, write, profile));
|
||||
tokio::spawn(run(rx, write, profile));
|
||||
(handle, observed)
|
||||
}
|
||||
|
||||
async fn run(
|
||||
mut rx: mpsc::Receiver<EngineMessage>,
|
||||
settings: SettingsHandle,
|
||||
mut write: IndexWriter,
|
||||
profile: crate::engine::profile::ProfileHandle,
|
||||
) {
|
||||
let mut max_logs: Option<usize> = None;
|
||||
let mut metric_sample_sequence: u64 = 0;
|
||||
let mut alert_states: HashMap<String, AlertRuntimeState> = HashMap::new();
|
||||
let mut log_stream_header: Option<crate::engine::logfmt::LogStreamHeader> = None;
|
||||
let mut ingest_lag_samples: Vec<(Timestamp, i64)> = Vec::new();
|
||||
|
||||
@@ -226,48 +224,18 @@ async fn run(
|
||||
log_stream_header.clone()
|
||||
};
|
||||
|
||||
let (rules, ntfy_topic, ntfy_server) = {
|
||||
let settings = settings.lock().unwrap();
|
||||
(
|
||||
settings.alert_rules.clone(),
|
||||
settings.ntfy_topic.clone(),
|
||||
settings.ntfy_server.clone(),
|
||||
)
|
||||
};
|
||||
|
||||
let text_for_op = Arc::clone(&text);
|
||||
write.mutate(move |snap: &mut EngineSnapshot| {
|
||||
// Primary/param extraction only (ticket 101) -- metric computation
|
||||
// (ticket 104) now happens on the profile actor, which wakes on this
|
||||
// publish and reads the newly-visible line itself.
|
||||
// (ticket 104) and alert evaluation (ticket 105) now happen on the
|
||||
// profile actor, which wakes on this publish and reads the
|
||||
// newly-visible line itself.
|
||||
snap.ingest_params(sequence, timestamp, &text_for_op, row_header.as_ref());
|
||||
snap.update_sort_helper_indexes_on_ingest(sequence, timestamp, &text_for_op);
|
||||
snap.purge_stale_sort_helper_indexes();
|
||||
snap.evict_excess_logs(max_logs);
|
||||
snap.ingest_lag_high_watermark_ms = ingest_lag_high_watermark_ms;
|
||||
});
|
||||
|
||||
let fired = write
|
||||
.peek()
|
||||
.map(|guard| {
|
||||
alerts::evaluate_alerts(
|
||||
&rules,
|
||||
&mut alert_states,
|
||||
&guard,
|
||||
sequence,
|
||||
timestamp,
|
||||
&text,
|
||||
)
|
||||
})
|
||||
.unwrap_or_default();
|
||||
for (rule, message) in fired {
|
||||
tokio::spawn(alerts::deliver_for_rule(
|
||||
rule.name,
|
||||
message,
|
||||
ntfy_topic.clone(),
|
||||
ntfy_server.clone(),
|
||||
));
|
||||
}
|
||||
}
|
||||
EngineMessage::IngestMetricSample(sample) => {
|
||||
let labels: BTreeMap<String, MetricPrimitive> = sample
|
||||
|
||||
Reference in New Issue
Block a user