Arc-ify stored log text to remove per-line double clones (ticket 100)

DataValue::LogLiteral and IngestedLine now carry an Arc<str> instead
of an owned String, so the two clones the ingest path paid per line
(once to move text into the write closure, once to store it) become
cheap refcount bumps instead of full string copies. Also cuts the
per-string-content cost of left_right's full-EngineSnapshot deep clone
on each reconciliation sync.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TeyhRfMYjMditek6geDRoQ
This commit is contained in:
Hugh Kaznowski
2026-09-07 13:26:03 +01:00
co-authored by Claude Sonnet 5
parent 5fee6909f2
commit f43445b280
11 changed files with 198 additions and 34 deletions
+4 -4
View File
@@ -1440,7 +1440,7 @@ mod tests {
.send(EngineMessage::Ingest(IngestedLine {
sequence: 0,
timestamp: Timestamp::from_millis(0),
text: "request path=/a".to_string(),
text: "request path=/a".into(),
}))
.await
.unwrap();
@@ -1494,7 +1494,7 @@ mod tests {
.send(EngineMessage::Ingest(IngestedLine {
sequence,
timestamp: Timestamp::from_millis(sequence as i64 * 1000),
text: text.to_string(),
text: text.into(),
}))
.await
.unwrap();
@@ -1554,7 +1554,7 @@ mod tests {
.send(EngineMessage::Ingest(IngestedLine {
sequence,
timestamp: Timestamp::from_millis(sequence as i64 * 1000),
text: text.to_string(),
text: text.into(),
}))
.await
.unwrap();
@@ -1822,7 +1822,7 @@ mod tests {
.send(EngineMessage::Ingest(IngestedLine {
sequence,
timestamp: Timestamp::from_millis(sequence as i64 * 1000),
text: text.to_string(),
text: (*text).into(),
}))
.await
.unwrap();
+4 -4
View File
@@ -894,7 +894,7 @@ mod tests {
.send(EngineMessage::Ingest(IngestedLine {
sequence,
timestamp: Timestamp::from_millis(sequence as i64),
text: text.to_string(),
text: text.into(),
}))
.await
.unwrap();
@@ -1144,7 +1144,7 @@ mod tests {
.send(EngineMessage::Ingest(IngestedLine {
sequence: 0,
timestamp: Timestamp::from_millis(0),
text: "level=info request duration=42".to_string(),
text: "level=info request duration=42".into(),
}))
.await
.unwrap();
@@ -1327,7 +1327,7 @@ mod tests {
.send(EngineMessage::Ingest(IngestedLine {
sequence: 0,
timestamp: Timestamp::from_millis(0),
text: "request path=/a duration=42".to_string(),
text: "request path=/a duration=42".into(),
}))
.await
.unwrap();
@@ -1511,7 +1511,7 @@ mod tests {
.send(EngineMessage::Ingest(IngestedLine {
sequence: 0,
timestamp: Timestamp::from_millis(0),
text: "dx=1 moved".to_string(),
text: "dx=1 moved".into(),
}))
.await
.unwrap();
+2 -1
View File
@@ -2,6 +2,7 @@ use crate::engine::metric_primitive::MetricPrimitive;
use crate::engine::timestamp::Timestamp;
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::sync::Arc;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ParamDeclaration {
@@ -98,7 +99,7 @@ pub enum MetricSemantics {
#[derive(Debug, Clone, Serialize)]
pub enum DataValue {
LogLiteral {
text: String,
text: Arc<str>,
},
ParamDeclaration(ParamDeclaration),
MetricDeclaration {
+3 -2
View File
@@ -15,6 +15,7 @@ use crate::engine::{ClearParamsResult, IngestedLine};
use crate::flavour::LogLiteralHelper;
use left_right::Absorb;
use std::collections::{BTreeMap, HashMap};
use std::sync::Arc;
use tokio::sync::{mpsc, oneshot};
struct EngineOp(Box<dyn FnMut(&mut EngineSnapshot) + Send + 'static>);
@@ -176,7 +177,7 @@ async fn run(
)
};
let text_for_op = text.clone();
let text_for_op = Arc::clone(&text);
write.mutate(move |snap: &mut EngineSnapshot| {
snap.approx_bytes_total += text_for_op.approx_log_bytes();
snap.index.insert(
@@ -185,7 +186,7 @@ async fn run(
timestamp,
},
DataValue::LogLiteral {
text: text_for_op.clone(),
text: Arc::clone(&text_for_op),
},
);
snap.log_count += 1;
+3 -3
View File
@@ -16,7 +16,7 @@ pub async fn run(engine: EngineHandle) {
let line = IngestedLine {
sequence,
timestamp: Timestamp::now(),
text,
text: text.into(),
};
sequence += 1;
@@ -86,7 +86,7 @@ pub async fn run_file(engine: EngineHandle, path: PathBuf) {
let line = IngestedLine {
sequence,
timestamp: Timestamp::now(),
text,
text: text.into(),
};
sequence += 1;
@@ -247,7 +247,7 @@ mod tests {
.expect("timed out waiting for ingested line")
.expect("engine channel closed")
{
EngineMessage::Ingest(line) => line.text,
EngineMessage::Ingest(line) => line.text.to_string(),
_ => panic!("expected Ingest"),
}
}
+2 -1
View File
@@ -26,11 +26,12 @@ use crate::engine::data_value::{MetricSemantics, ParamDeclaration};
use crate::engine::timestamp::Timestamp;
use serde::Deserialize;
use std::collections::BTreeMap;
use std::sync::Arc;
pub struct IngestedLine {
pub sequence: u64,
pub timestamp: Timestamp,
pub text: String,
pub text: Arc<str>,
}
pub struct MetricSample {
+1 -1
View File
@@ -249,7 +249,7 @@ impl EngineSnapshot {
timestamp,
},
DataValue::LogLiteral { text },
) => Some((*sequence, *timestamp, text.as_str())),
) => Some((*sequence, *timestamp, text.as_ref())),
_ => None,
})
}
+1 -1
View File
@@ -106,7 +106,7 @@ fn logs_sorted_by_param(
entries.push(LogEntryDto {
sequence,
timestamp_ms: timestamp.as_millis(),
text: text.clone(),
text: text.to_string(),
params: log_entry_params(text, compiled_params),
});
if entries.len() >= limit {
+9 -17
View File
@@ -287,9 +287,7 @@ fn logs_query_sorted_by_a_log_stream_column_actually_reorders_real_log_stream_ou
sequence,
timestamp,
},
DataValue::LogLiteral {
text: line.to_string(),
},
DataValue::LogLiteral { text: line.into() },
);
}
@@ -477,7 +475,7 @@ fn evict_excess_logs_drops_oldest_sequences_first() {
sequence,
timestamp: Timestamp::from_millis(sequence as i64),
},
DataValue::LogLiteral { text },
DataValue::LogLiteral { text: text.into() },
);
index.log_count += 1;
}
@@ -504,7 +502,7 @@ fn logs_in_sequence_range_seeks_instead_of_scanning_from_start() {
timestamp: Timestamp::from_millis(sequence as i64),
},
DataValue::LogLiteral {
text: format!("line {sequence}"),
text: format!("line {sequence}").into(),
},
);
}
@@ -548,9 +546,7 @@ fn evict_excess_logs_is_noop_when_unlimited_or_under_cap() {
sequence,
timestamp: Timestamp::from_millis(sequence as i64),
},
DataValue::LogLiteral {
text: "x".to_string(),
},
DataValue::LogLiteral { text: "x".into() },
);
index.log_count += 1;
}
@@ -621,9 +617,7 @@ fn insert_ingested_log(index: &mut EngineSnapshot, sequence: u64, text: &str) {
sequence,
timestamp: Timestamp::from_millis(sequence as i64),
},
DataValue::LogLiteral {
text: text.to_string(),
},
DataValue::LogLiteral { text: text.into() },
);
}
@@ -701,7 +695,7 @@ fn declarations_stay_within_their_range() {
timestamp: Timestamp::from_millis(0),
},
DataValue::LogLiteral {
text: "hello".to_string(),
text: "hello".into(),
},
);
declare_param(
@@ -906,7 +900,7 @@ fn metric_declarations_stay_within_their_range() {
timestamp: Timestamp::from_millis(0),
},
DataValue::LogLiteral {
text: "hello".to_string(),
text: "hello".into(),
},
);
declare_param(
@@ -1161,9 +1155,7 @@ fn ingest_log(index: &mut EngineSnapshot, sequence: u64, text: &str) {
sequence,
timestamp,
},
DataValue::LogLiteral {
text: text.to_string(),
},
DataValue::LogLiteral { text: text.into() },
);
index.apply_all_metrics(sequence, timestamp, text);
}
@@ -2677,7 +2669,7 @@ fn logs_query_with_ottyl_ts_comparison_filters_by_timestamp() {
timestamp,
},
DataValue::LogLiteral {
text: format!("line {i}"),
text: format!("line {i}").into(),
},
);
}