ottyl/src/engine/alerts.rs

use std::collections::hash_map::DefaultHasher;
use std::collections::{HashMap, HashSet};
use std::fmt;
use std::hash::{Hash, Hasher};
use std::io;
use std::path::Path;
use std::time::{SystemTime, UNIX_EPOCH};

use regex::Regex;
use serde::{Deserialize, Serialize};

use crate::config::Settings;
use crate::engine::filter_datetime;
use crate::engine::filter_parser::{self, CompareOp, FilterExpr, FilterValue};
use crate::engine::rate::RateWindow;
use crate::engine::timestamp::Timestamp;
use crate::engine::{self, eval_json_comparison};

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct AlertRule {
    pub id: String,
    pub name: String,
    pub enabled: bool,
    pub condition: AlertCondition,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "lowercase")]
pub enum AlertCondition {
    Immediate {
        query: String,
    },
    Threshold {
        query: String,
        op: CompareOp,
        threshold: f64,
        window_ms: i64,
    },
    NewValue {
        field_regex: String,
        capture_group: String,
    },
    Ratio {
        query_a: String,
        query_b: String,
        op: CompareOp,
        ratio: f64,
        window_ms: i64,
    },
}

#[derive(Debug)]
pub enum AlertError {
    NotFound,
    Io(io::Error),
}

impl fmt::Display for AlertError {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            AlertError::NotFound => write!(f, "no alert rule with that id"),
            AlertError::Io(err) => write!(f, "I/O error: {err}"),
        }
    }
}

impl std::error::Error for AlertError {}

impl From<io::Error> for AlertError {
    fn from(err: io::Error) -> Self {
        AlertError::Io(err)
    }
}

fn generate_alert_rule_id(name: &str) -> String {
    let mut hasher = DefaultHasher::new();
    name.hash(&mut hasher);
    let nanos = SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .map(|d| d.as_nanos())
        .unwrap_or(0);
    nanos.hash(&mut hasher);
    format!("alert-{:016x}", hasher.finish())
}

impl Settings {
    pub fn create_alert_rule(
        &mut self,
        settings_path: &Path,
        name: String,
        enabled: bool,
        condition: AlertCondition,
    ) -> Result<AlertRule, AlertError> {
        let rule = AlertRule {
            id: generate_alert_rule_id(&name),
            name,
            enabled,
            condition,
        };
        self.alert_rules.push(rule.clone());
        self.save(settings_path)?;
        Ok(rule)
    }

    pub fn update_alert_rule(
        &mut self,
        settings_path: &Path,
        id: &str,
        name: String,
        enabled: bool,
        condition: AlertCondition,
    ) -> Result<AlertRule, AlertError> {
        let rule = self
            .alert_rules
            .iter_mut()
            .find(|rule| rule.id == id)
            .ok_or(AlertError::NotFound)?;
        rule.name = name;
        rule.enabled = enabled;
        rule.condition = condition;
        let updated = rule.clone();
        self.save(settings_path)?;
        Ok(updated)
    }

    pub fn remove_alert_rule(&mut self, settings_path: &Path, id: &str) -> Result<(), AlertError> {
        let index = self
            .alert_rules
            .iter()
            .position(|rule| rule.id == id)
            .ok_or(AlertError::NotFound)?;
        self.alert_rules.remove(index);
        self.save(settings_path)?;
        Ok(())
    }
}

#[derive(Debug, Default)]
pub struct AlertRuntimeState {
    threshold_window: RateWindow,
    threshold_backfilled: bool,
    ratio_a_window: RateWindow,
    ratio_a_backfilled: bool,
    ratio_b_window: RateWindow,
    ratio_b_backfilled: bool,
    seen_values: HashSet<String>,
}

fn backfill_window(
    snapshot: &engine::EngineSnapshot,
    query: &str,
    before_sequence: u64,
    now: Timestamp,
    window_ms: i64,
) -> RateWindow {
    let since_ms = now.as_millis() - window_ms;
    let matching_timestamps = snapshot
        .logs_since(before_sequence, since_ms, now.as_millis())
        .filter_map(|(sequence, timestamp, text)| {
            (eval_alert_query(query, sequence, timestamp, text) == Ok(true)).then_some(timestamp)
        });
    RateWindow::backfill(matching_timestamps, now, window_ms)
}

fn eval_alert_query(
    query: &str,
    sequence: u64,
    timestamp: Timestamp,
    text: &str,
) -> Result<bool, String> {
    let expr = filter_parser::parse(query).map_err(|err| err.to_string())?;
    eval_alert_expr(&expr, sequence, timestamp, text)
}

fn eval_alert_expr(
    expr: &FilterExpr,
    sequence: u64,
    timestamp: Timestamp,
    text: &str,
) -> Result<bool, String> {
    match expr {
        FilterExpr::Regex(pattern) => {
            let re = Regex::new(pattern)
                .map_err(|err| format!("invalid regex in alert query: {err}"))?;
            Ok(re.is_match(text))
        }
        FilterExpr::Comparison {
            metric,
            param: None,
            op,
            value,
        } if metric == "ottyl_seq" => {
            let target = match value {
                FilterValue::Number(n) => *n,
                FilterValue::Raw(raw) => raw.parse::<f64>().map_err(|_| {
                    format!("'ottyl_seq' comparisons require a numeric value, got {raw:?}")
                })?,
            };
            Ok(op.matches((sequence as f64).total_cmp(&target)))
        }
        FilterExpr::Comparison {
            metric,
            param: None,
            op,
            value,
        } if metric == "ottyl_ts" => {
            let target_ms = match value {
                FilterValue::Number(n) => *n as i64,
                FilterValue::Raw(raw) => filter_datetime::parse_datetime(raw, Timestamp::now())
                    .map_err(|err| err.to_string())?
                    .as_millis(),
            };
            Ok(op.matches(timestamp.as_millis().cmp(&target_ms)))
        }
        FilterExpr::Comparison {
            metric,
            param: None,
            op,
            value,
        } if metric == "ottyl_fmt" => {
            let target = match value {
                FilterValue::Number(n) => n.to_string(),
                FilterValue::Raw(raw) => raw.clone(),
            };
            Ok(op.matches(crate::engine::logfmt::detect_format(text).cmp(target.as_str())))
        }
        FilterExpr::Comparison { metric, .. } => Err(format!(
            "alert queries only support regex()/ottyl_seq/ottyl_ts/ottyl_fmt (not metric comparisons like \
             '{metric}[param]'): metric-based alert conditions aren't supported yet"
        )),
        FilterExpr::JsonComparison { field, op, value } => {
            Ok(eval_json_comparison(text, field, *op, value))
        }
        FilterExpr::And(lhs, rhs) => Ok(eval_alert_expr(lhs, sequence, timestamp, text)?
            && eval_alert_expr(rhs, sequence, timestamp, text)?),
        FilterExpr::Or(lhs, rhs) => Ok(eval_alert_expr(lhs, sequence, timestamp, text)?
            || eval_alert_expr(rhs, sequence, timestamp, text)?),
    }
}

pub fn evaluate_alerts(
    rules: &[AlertRule],
    states: &mut HashMap<String, AlertRuntimeState>,
    snapshot: &engine::EngineSnapshot,
    sequence: u64,
    timestamp: Timestamp,
    text: &str,
) -> Vec<(AlertRule, String)> {
    let mut fired = Vec::new();
    for rule in rules {
        if !rule.enabled {
            continue;
        }
        let state = states.entry(rule.id.clone()).or_default();
        if let Some(message) = evaluate_condition(
            &rule.condition,
            state,
            snapshot,
            sequence,
            timestamp,
            text,
            &rule.name,
        ) {
            fired.push((rule.clone(), message));
        }
    }
    fired
}

fn evaluate_condition(
    condition: &AlertCondition,
    state: &mut AlertRuntimeState,
    snapshot: &engine::EngineSnapshot,
    sequence: u64,
    timestamp: Timestamp,
    text: &str,
    rule_name: &str,
) -> Option<String> {
    match condition {
        AlertCondition::Immediate { query } => {
            match eval_alert_query(query, sequence, timestamp, text) {
                Ok(true) => Some(format!("{rule_name}: matched line: {text}")),
                Ok(false) => None,
                Err(err) => {
                    eprintln!("warning: alert '{rule_name}' query error: {err}");
                    None
                }
            }
        }
        AlertCondition::Threshold {
            query,
            op,
            threshold,
            window_ms,
        } => {
            if !state.threshold_backfilled {
                state.threshold_window =
                    backfill_window(snapshot, query, sequence, timestamp, *window_ms);
                state.threshold_backfilled = true;
            }
            match eval_alert_query(query, sequence, timestamp, text) {
                Ok(true) => {
                    state.threshold_window.push(timestamp, *window_ms);
                    let count = state.threshold_window.count() as f64;
                    op.matches(count.total_cmp(threshold)).then(|| {
                        format!("{rule_name}: match count {count} {op:?} threshold {threshold} (window {window_ms}ms)")
                    })
                }
                Ok(false) => None,
                Err(err) => {
                    eprintln!("warning: alert '{rule_name}' query error: {err}");
                    None
                }
            }
        }
        AlertCondition::NewValue {
            field_regex,
            capture_group,
        } => {
            let re = match Regex::new(field_regex) {
                Ok(re) => re,
                Err(err) => {
                    eprintln!("warning: alert '{rule_name}' has an invalid field_regex: {err}");
                    return None;
                }
            };
            let captures = re.captures(text)?;
            let value = captures.name(capture_group)?.as_str().to_string();
            state
                .seen_values
                .insert(value.clone())
                .then(|| format!("{rule_name}: new value seen: {value} (line: {text})"))
        }
        AlertCondition::Ratio {
            query_a,
            query_b,
            op,
            ratio,
            window_ms,
        } => {
            let matched_a = eval_alert_query(query_a, sequence, timestamp, text);
            let matched_b = eval_alert_query(query_b, sequence, timestamp, text);
            let (matched_a, matched_b) = match (matched_a, matched_b) {
                (Ok(a), Ok(b)) => (a, b),
                (Err(err), _) | (_, Err(err)) => {
                    eprintln!("warning: alert '{rule_name}' query error: {err}");
                    return None;
                }
            };
            if !matched_a && !matched_b {
                return None;
            }
            if !state.ratio_a_backfilled {
                state.ratio_a_window =
                    backfill_window(snapshot, query_a, sequence, timestamp, *window_ms);
                state.ratio_a_backfilled = true;
            }
            if !state.ratio_b_backfilled {
                state.ratio_b_window =
                    backfill_window(snapshot, query_b, sequence, timestamp, *window_ms);
                state.ratio_b_backfilled = true;
            }
            if matched_a {
                state.ratio_a_window.push(timestamp, *window_ms);
            }
            if matched_b {
                state.ratio_b_window.push(timestamp, *window_ms);
            }
            let count_a = state.ratio_a_window.count() as f64;
            let count_b = state.ratio_b_window.count() as f64;
            if count_b == 0.0 {
                return None;
            }
            let observed = count_a / count_b;
            op.matches(observed.total_cmp(ratio)).then(|| {
                format!(
                    "{rule_name}: ratio {observed:.3} ({count_a}/{count_b}) {op:?} {ratio} (window {window_ms}ms)"
                )
            })
        }
    }
}

pub async fn deliver_for_rule(
    rule_name: String,
    message: String,
    topic: Option<String>,
    server: Option<String>,
) {
    let Some(topic) = topic else {
        return;
    };
    let server = server.unwrap_or_else(|| "https://ntfy.sh".to_string());
    if let Err(err) = deliver_ntfy(&server, &topic, &message).await {
        eprintln!("warning: failed to deliver alert '{rule_name}' via ntfy: {err}");
    }
}

pub async fn deliver_ntfy(server: &str, topic: &str, message: &str) -> Result<(), String> {
    let url = format!("{}/{}", server.trim_end_matches('/'), topic);
    let response = reqwest::Client::new()
        .post(&url)
        .body(message.to_string())
        .send()
        .await
        .map_err(|err| err.to_string())?;
    if response.status().is_success() {
        Ok(())
    } else {
        Err(format!("ntfy responded with status {}", response.status()))
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn line(text: &str, sequence: u64, timestamp_ms: i64) -> (u64, Timestamp, &str) {
        (sequence, Timestamp::from_millis(timestamp_ms), text)
    }

    fn immediate_rule(query: &str) -> AlertRule {
        AlertRule {
            id: "r1".to_string(),
            name: "test rule".to_string(),
            enabled: true,
            condition: AlertCondition::Immediate {
                query: query.to_string(),
            },
        }
    }

    #[test]
    fn alert_rule_crud_round_trips_through_save_and_load() {
        let dir = tempfile::tempdir().unwrap();
        let settings_path = dir.path().join("settings.json");
        let mut settings = Settings::load(&settings_path);

        let created = settings
            .create_alert_rule(
                &settings_path,
                "spike".to_string(),
                true,
                AlertCondition::Immediate {
                    query: r#"regex("error")"#.to_string(),
                },
            )
            .unwrap();
        assert_eq!(settings.alert_rules.len(), 1);

        let reloaded = Settings::load(&settings_path);
        assert_eq!(reloaded.alert_rules, settings.alert_rules);

        let updated = settings
            .update_alert_rule(
                &settings_path,
                &created.id,
                "renamed".to_string(),
                false,
                AlertCondition::Immediate {
                    query: r#"regex("warn")"#.to_string(),
                },
            )
            .unwrap();
        assert_eq!(updated.id, created.id);
        assert_eq!(updated.name, "renamed");
        assert!(!updated.enabled);

        settings
            .remove_alert_rule(&settings_path, &created.id)
            .unwrap();
        assert!(settings.alert_rules.is_empty());
        let reloaded = Settings::load(&settings_path);
        assert!(reloaded.alert_rules.is_empty());
    }

    #[test]
    fn alert_rule_mutations_reject_unknown_id() {
        let dir = tempfile::tempdir().unwrap();
        let settings_path = dir.path().join("settings.json");
        let mut settings = Settings::load(&settings_path);

        let result = settings.update_alert_rule(
            &settings_path,
            "does-not-exist",
            "x".to_string(),
            true,
            AlertCondition::Immediate {
                query: r#"regex("x")"#.to_string(),
            },
        );
        assert!(matches!(result, Err(AlertError::NotFound)));

        let result = settings.remove_alert_rule(&settings_path, "does-not-exist");
        assert!(matches!(result, Err(AlertError::NotFound)));
    }

    #[test]
    fn immediate_fires_on_matching_line_only() {
        let rules = vec![immediate_rule(r#"regex("error")"#)];
        let mut states = HashMap::new();
        let index = engine::EngineSnapshot::default();

        let (seq, ts, text) = line("all good", 0, 0);
        assert!(evaluate_alerts(&rules, &mut states, &index, seq, ts, text).is_empty());

        let (seq, ts, text) = line("an error happened", 1, 1);
        let fired = evaluate_alerts(&rules, &mut states, &index, seq, ts, text);
        assert_eq!(fired.len(), 1);
        assert!(fired[0].1.contains("an error happened"));
    }

    #[test]
    fn disabled_rule_never_fires() {
        let mut rule = immediate_rule(r#"regex("error")"#);
        rule.enabled = false;
        let mut states = HashMap::new();
        let index = engine::EngineSnapshot::default();
        let (seq, ts, text) = line("error!", 0, 0);
        assert!(evaluate_alerts(&[rule], &mut states, &index, seq, ts, text).is_empty());
    }

    #[test]
    fn threshold_fires_once_count_crosses_and_not_before() {
        let rule = AlertRule {
            id: "r1".to_string(),
            name: "spike".to_string(),
            enabled: true,
            condition: AlertCondition::Threshold {
                query: r#"regex("error")"#.to_string(),
                op: CompareOp::Ge,
                threshold: 3.0,
                window_ms: 10_000,
            },
        };
        let mut states = HashMap::new();
        let index = engine::EngineSnapshot::default();
        for (seq, ms) in [(0u64, 0i64), (1, 1000)] {
            let fired = evaluate_alerts(
                std::slice::from_ref(&rule),
                &mut states,
                &index,
                seq,
                Timestamp::from_millis(ms),
                "error",
            );
            assert!(
                fired.is_empty(),
                "should not fire before threshold is crossed"
            );
        }
        let fired = evaluate_alerts(
            std::slice::from_ref(&rule),
            &mut states,
            &index,
            2,
            Timestamp::from_millis(2000),
            "error",
        );
        assert_eq!(fired.len(), 1);
        let fired = evaluate_alerts(
            &[rule],
            &mut states,
            &index,
            3,
            Timestamp::from_millis(2100),
            "fine",
        );
        assert!(fired.is_empty());
    }

    #[test]
    fn threshold_window_expires_old_matches() {
        let rule = AlertRule {
            id: "r1".to_string(),
            name: "spike".to_string(),
            enabled: true,
            condition: AlertCondition::Threshold {
                query: r#"regex("error")"#.to_string(),
                op: CompareOp::Ge,
                threshold: 2.0,
                window_ms: 1_000,
            },
        };
        let mut states = HashMap::new();
        let index = engine::EngineSnapshot::default();
        assert!(
            evaluate_alerts(
                std::slice::from_ref(&rule),
                &mut states,
                &index,
                0,
                Timestamp::from_millis(0),
                "error"
            )
            .is_empty()
        );
        assert!(
            evaluate_alerts(
                &[rule],
                &mut states,
                &index,
                1,
                Timestamp::from_millis(5_000),
                "error"
            )
            .is_empty()
        );
    }

    #[test]
    fn new_value_fires_only_on_first_sighting() {
        let rule = AlertRule {
            id: "r1".to_string(),
            name: "new user agent".to_string(),
            enabled: true,
            condition: AlertCondition::NewValue {
                field_regex: r"agent=(?P<agent>\S+)".to_string(),
                capture_group: "agent".to_string(),
            },
        };
        let mut states = HashMap::new();
        let index = engine::EngineSnapshot::default();
        let fired = evaluate_alerts(
            std::slice::from_ref(&rule),
            &mut states,
            &index,
            0,
            Timestamp::from_millis(0),
            "agent=curl",
        );
        assert_eq!(fired.len(), 1);
        let fired = evaluate_alerts(
            std::slice::from_ref(&rule),
            &mut states,
            &index,
            1,
            Timestamp::from_millis(1),
            "agent=curl",
        );
        assert!(fired.is_empty());
        let fired = evaluate_alerts(
            &[rule],
            &mut states,
            &index,
            2,
            Timestamp::from_millis(2),
            "agent=wget",
        );
        assert_eq!(fired.len(), 1);
    }

    #[test]
    fn ratio_fires_when_crossing_and_skips_zero_denominator() {
        let rule = AlertRule {
            id: "r1".to_string(),
            name: "error ratio".to_string(),
            enabled: true,
            condition: AlertCondition::Ratio {
                query_a: r#"regex("error")"#.to_string(),
                query_b: r#"regex("request")"#.to_string(),
                op: CompareOp::Ge,
                ratio: 0.5,
                window_ms: 10_000,
            },
        };
        let mut states = HashMap::new();
        let index = engine::EngineSnapshot::default();
        assert!(
            evaluate_alerts(
                std::slice::from_ref(&rule),
                &mut states,
                &index,
                0,
                Timestamp::from_millis(0),
                "error"
            )
            .is_empty()
        );
        let fired = evaluate_alerts(
            std::slice::from_ref(&rule),
            &mut states,
            &index,
            1,
            Timestamp::from_millis(1),
            "request",
        );
        assert_eq!(fired.len(), 1);
        let fired = evaluate_alerts(
            std::slice::from_ref(&rule),
            &mut states,
            &index,
            2,
            Timestamp::from_millis(2),
            "request",
        );
        assert_eq!(fired.len(), 1);
        let fired = evaluate_alerts(
            &[rule],
            &mut states,
            &index,
            3,
            Timestamp::from_millis(3),
            "request",
        );
        assert!(fired.is_empty());
    }

    #[test]
    fn ottyl_fmt_alert_query_matches_detected_format() {
        let rule = immediate_rule("ottyl_fmt==json");
        let mut states = HashMap::new();
        let index = engine::EngineSnapshot::default();

        let fired = evaluate_alerts(
            std::slice::from_ref(&rule),
            &mut states,
            &index,
            0,
            Timestamp::from_millis(0),
            r#"{"level":"error"}"#,
        );
        assert_eq!(fired.len(), 1);

        let fired = evaluate_alerts(
            &[rule],
            &mut states,
            &index,
            1,
            Timestamp::from_millis(1),
            "plain text",
        );
        assert!(fired.is_empty());
    }

    #[test]
    fn metric_comparison_in_alert_query_is_rejected_not_panicking() {
        let rule = immediate_rule("some_metric[param]==5");
        let mut states = HashMap::new();
        let index = engine::EngineSnapshot::default();
        let fired = evaluate_alerts(
            &[rule],
            &mut states,
            &index,
            0,
            Timestamp::from_millis(0),
            "anything",
        );
        assert!(fired.is_empty());
    }

    #[tokio::test]
    async fn deliver_ntfy_posts_message_to_server_topic_path() {
        use axum::Router;
        use axum::extract::State;
        use axum::routing::post;
        use std::sync::{Arc, Mutex};
        use tokio::net::TcpListener;

        #[derive(Clone, Default)]
        struct Captured(Arc<Mutex<Option<(String, String)>>>);

        async fn capture(
            State(captured): State<Captured>,
            axum::extract::Path(topic): axum::extract::Path<String>,
            body: String,
        ) -> &'static str {
            *captured.0.lock().unwrap() = Some((topic, body));
            "ok"
        }

        let captured = Captured::default();
        let app = Router::new()
            .route("/{topic}", post(capture))
            .with_state(captured.clone());
        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
        let addr = listener.local_addr().unwrap();
        tokio::spawn(async move {
            axum::serve(listener, app).await.unwrap();
        });

        let server = format!("http://{addr}");
        deliver_ntfy(&server, "alerts", "hello world")
            .await
            .unwrap();

        let (topic, body) = captured.0.lock().unwrap().clone().unwrap();
        assert_eq!(topic, "alerts");
        assert_eq!(body, "hello world");
    }
}