Arc-ify stored log text to remove per-line double clones (ticket 100)
DataValue::LogLiteral and IngestedLine now carry an Arc<str> instead of an owned String, so the two clones the ingest path paid per line (once to move text into the write closure, once to store it) become cheap refcount bumps instead of full string copies. Also cuts the per-string-content cost of left_right's full-EngineSnapshot deep clone on each reconciliation sync. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TeyhRfMYjMditek6geDRoQ
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
5fee6909f2
commit
f43445b280
+4
-4
@@ -1440,7 +1440,7 @@ mod tests {
|
||||
.send(EngineMessage::Ingest(IngestedLine {
|
||||
sequence: 0,
|
||||
timestamp: Timestamp::from_millis(0),
|
||||
text: "request path=/a".to_string(),
|
||||
text: "request path=/a".into(),
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -1494,7 +1494,7 @@ mod tests {
|
||||
.send(EngineMessage::Ingest(IngestedLine {
|
||||
sequence,
|
||||
timestamp: Timestamp::from_millis(sequence as i64 * 1000),
|
||||
text: text.to_string(),
|
||||
text: text.into(),
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -1554,7 +1554,7 @@ mod tests {
|
||||
.send(EngineMessage::Ingest(IngestedLine {
|
||||
sequence,
|
||||
timestamp: Timestamp::from_millis(sequence as i64 * 1000),
|
||||
text: text.to_string(),
|
||||
text: text.into(),
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -1822,7 +1822,7 @@ mod tests {
|
||||
.send(EngineMessage::Ingest(IngestedLine {
|
||||
sequence,
|
||||
timestamp: Timestamp::from_millis(sequence as i64 * 1000),
|
||||
text: text.to_string(),
|
||||
text: (*text).into(),
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
@@ -894,7 +894,7 @@ mod tests {
|
||||
.send(EngineMessage::Ingest(IngestedLine {
|
||||
sequence,
|
||||
timestamp: Timestamp::from_millis(sequence as i64),
|
||||
text: text.to_string(),
|
||||
text: text.into(),
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -1144,7 +1144,7 @@ mod tests {
|
||||
.send(EngineMessage::Ingest(IngestedLine {
|
||||
sequence: 0,
|
||||
timestamp: Timestamp::from_millis(0),
|
||||
text: "level=info request duration=42".to_string(),
|
||||
text: "level=info request duration=42".into(),
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -1327,7 +1327,7 @@ mod tests {
|
||||
.send(EngineMessage::Ingest(IngestedLine {
|
||||
sequence: 0,
|
||||
timestamp: Timestamp::from_millis(0),
|
||||
text: "request path=/a duration=42".to_string(),
|
||||
text: "request path=/a duration=42".into(),
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -1511,7 +1511,7 @@ mod tests {
|
||||
.send(EngineMessage::Ingest(IngestedLine {
|
||||
sequence: 0,
|
||||
timestamp: Timestamp::from_millis(0),
|
||||
text: "dx=1 moved".to_string(),
|
||||
text: "dx=1 moved".into(),
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
@@ -2,6 +2,7 @@ use crate::engine::metric_primitive::MetricPrimitive;
|
||||
use crate::engine::timestamp::Timestamp;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ParamDeclaration {
|
||||
@@ -98,7 +99,7 @@ pub enum MetricSemantics {
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub enum DataValue {
|
||||
LogLiteral {
|
||||
text: String,
|
||||
text: Arc<str>,
|
||||
},
|
||||
ParamDeclaration(ParamDeclaration),
|
||||
MetricDeclaration {
|
||||
|
||||
@@ -15,6 +15,7 @@ use crate::engine::{ClearParamsResult, IngestedLine};
|
||||
use crate::flavour::LogLiteralHelper;
|
||||
use left_right::Absorb;
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
|
||||
struct EngineOp(Box<dyn FnMut(&mut EngineSnapshot) + Send + 'static>);
|
||||
@@ -176,7 +177,7 @@ async fn run(
|
||||
)
|
||||
};
|
||||
|
||||
let text_for_op = text.clone();
|
||||
let text_for_op = Arc::clone(&text);
|
||||
write.mutate(move |snap: &mut EngineSnapshot| {
|
||||
snap.approx_bytes_total += text_for_op.approx_log_bytes();
|
||||
snap.index.insert(
|
||||
@@ -185,7 +186,7 @@ async fn run(
|
||||
timestamp,
|
||||
},
|
||||
DataValue::LogLiteral {
|
||||
text: text_for_op.clone(),
|
||||
text: Arc::clone(&text_for_op),
|
||||
},
|
||||
);
|
||||
snap.log_count += 1;
|
||||
|
||||
@@ -16,7 +16,7 @@ pub async fn run(engine: EngineHandle) {
|
||||
let line = IngestedLine {
|
||||
sequence,
|
||||
timestamp: Timestamp::now(),
|
||||
text,
|
||||
text: text.into(),
|
||||
};
|
||||
sequence += 1;
|
||||
|
||||
@@ -86,7 +86,7 @@ pub async fn run_file(engine: EngineHandle, path: PathBuf) {
|
||||
let line = IngestedLine {
|
||||
sequence,
|
||||
timestamp: Timestamp::now(),
|
||||
text,
|
||||
text: text.into(),
|
||||
};
|
||||
sequence += 1;
|
||||
|
||||
@@ -247,7 +247,7 @@ mod tests {
|
||||
.expect("timed out waiting for ingested line")
|
||||
.expect("engine channel closed")
|
||||
{
|
||||
EngineMessage::Ingest(line) => line.text,
|
||||
EngineMessage::Ingest(line) => line.text.to_string(),
|
||||
_ => panic!("expected Ingest"),
|
||||
}
|
||||
}
|
||||
|
||||
+2
-1
@@ -26,11 +26,12 @@ use crate::engine::data_value::{MetricSemantics, ParamDeclaration};
|
||||
use crate::engine::timestamp::Timestamp;
|
||||
use serde::Deserialize;
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub struct IngestedLine {
|
||||
pub sequence: u64,
|
||||
pub timestamp: Timestamp,
|
||||
pub text: String,
|
||||
pub text: Arc<str>,
|
||||
}
|
||||
|
||||
pub struct MetricSample {
|
||||
|
||||
@@ -249,7 +249,7 @@ impl EngineSnapshot {
|
||||
timestamp,
|
||||
},
|
||||
DataValue::LogLiteral { text },
|
||||
) => Some((*sequence, *timestamp, text.as_str())),
|
||||
) => Some((*sequence, *timestamp, text.as_ref())),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -106,7 +106,7 @@ fn logs_sorted_by_param(
|
||||
entries.push(LogEntryDto {
|
||||
sequence,
|
||||
timestamp_ms: timestamp.as_millis(),
|
||||
text: text.clone(),
|
||||
text: text.to_string(),
|
||||
params: log_entry_params(text, compiled_params),
|
||||
});
|
||||
if entries.len() >= limit {
|
||||
|
||||
+9
-17
@@ -287,9 +287,7 @@ fn logs_query_sorted_by_a_log_stream_column_actually_reorders_real_log_stream_ou
|
||||
sequence,
|
||||
timestamp,
|
||||
},
|
||||
DataValue::LogLiteral {
|
||||
text: line.to_string(),
|
||||
},
|
||||
DataValue::LogLiteral { text: line.into() },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -477,7 +475,7 @@ fn evict_excess_logs_drops_oldest_sequences_first() {
|
||||
sequence,
|
||||
timestamp: Timestamp::from_millis(sequence as i64),
|
||||
},
|
||||
DataValue::LogLiteral { text },
|
||||
DataValue::LogLiteral { text: text.into() },
|
||||
);
|
||||
index.log_count += 1;
|
||||
}
|
||||
@@ -504,7 +502,7 @@ fn logs_in_sequence_range_seeks_instead_of_scanning_from_start() {
|
||||
timestamp: Timestamp::from_millis(sequence as i64),
|
||||
},
|
||||
DataValue::LogLiteral {
|
||||
text: format!("line {sequence}"),
|
||||
text: format!("line {sequence}").into(),
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -548,9 +546,7 @@ fn evict_excess_logs_is_noop_when_unlimited_or_under_cap() {
|
||||
sequence,
|
||||
timestamp: Timestamp::from_millis(sequence as i64),
|
||||
},
|
||||
DataValue::LogLiteral {
|
||||
text: "x".to_string(),
|
||||
},
|
||||
DataValue::LogLiteral { text: "x".into() },
|
||||
);
|
||||
index.log_count += 1;
|
||||
}
|
||||
@@ -621,9 +617,7 @@ fn insert_ingested_log(index: &mut EngineSnapshot, sequence: u64, text: &str) {
|
||||
sequence,
|
||||
timestamp: Timestamp::from_millis(sequence as i64),
|
||||
},
|
||||
DataValue::LogLiteral {
|
||||
text: text.to_string(),
|
||||
},
|
||||
DataValue::LogLiteral { text: text.into() },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -701,7 +695,7 @@ fn declarations_stay_within_their_range() {
|
||||
timestamp: Timestamp::from_millis(0),
|
||||
},
|
||||
DataValue::LogLiteral {
|
||||
text: "hello".to_string(),
|
||||
text: "hello".into(),
|
||||
},
|
||||
);
|
||||
declare_param(
|
||||
@@ -906,7 +900,7 @@ fn metric_declarations_stay_within_their_range() {
|
||||
timestamp: Timestamp::from_millis(0),
|
||||
},
|
||||
DataValue::LogLiteral {
|
||||
text: "hello".to_string(),
|
||||
text: "hello".into(),
|
||||
},
|
||||
);
|
||||
declare_param(
|
||||
@@ -1161,9 +1155,7 @@ fn ingest_log(index: &mut EngineSnapshot, sequence: u64, text: &str) {
|
||||
sequence,
|
||||
timestamp,
|
||||
},
|
||||
DataValue::LogLiteral {
|
||||
text: text.to_string(),
|
||||
},
|
||||
DataValue::LogLiteral { text: text.into() },
|
||||
);
|
||||
index.apply_all_metrics(sequence, timestamp, text);
|
||||
}
|
||||
@@ -2677,7 +2669,7 @@ fn logs_query_with_ottyl_ts_comparison_filters_by_timestamp() {
|
||||
timestamp,
|
||||
},
|
||||
DataValue::LogLiteral {
|
||||
text: format!("line {i}"),
|
||||
text: format!("line {i}").into(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user