phughk/ottyl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Resolve tickets 071-074, 076: log-stream contamination, sort, ottyl_fmt column, ingest lag

Browse Source 
- 076: gate auto-declared log-stream column/message params
  (`ParamDeclaration::log_stream_gated`) behind `is_log_stream_line` at
  extraction time, since their fixed-offset regex has no content anchor
  and previously extracted garbage from continuation lines and the
  header row itself (the regex crate doesn't support lookaround, so
  this couldn't be baked into the regex text).
- 074: end-to-end regression test confirms sorting a real `log stream`
  capture by a log-stream column (e.g. PID) now actually reorders
  results, since 076's fix removes the garbage values that were
  corrupting the sort helper index; also add a "Building sort
  index…" indicator in the timeline UI for a slow first-time sort.
- 072: attach the reserved `ottyl_fmt` pseudo-param to each
  `LogEntryDto`'s params so it can be picked as a Log Preview column
  (previously listed as pickable but always blank).
- 073: track a rolling 5s high-watermark of ingestion-to-engine-processing
  lag and surface it in the nav stats popover next to approx_bytes_total.
- 071: group the regex(...) help button back with the filter bar's other
  controls, and restructure the Filtering docs section into scannable
  per-feature subsections (also fixes stale log_type/process/pid mention).

Tickets 070 (CLI scoping) and 075 (API unresponsiveness investigation)
are left in place per their own scope — both are explicitly
conversation/investigation tasks, not implementation tickets.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01A1ZJ3SrayzLzVndhVq9BmS
This commit is contained in:
Hugh Kaznowski
2026-09-06 15:12:07 +01:00
co-authored by Claude Sonnet 5
parent f34dbba6c7
commit 9fc7540a5b
16 changed files with 511 additions and 460 deletions
Download Patch File Download Diff File Expand all files Collapse all files
webapp/src/App.tsx
+45 -16
View File
@@ -69,6 +69,23 @@ function saveLogViewerHeight(px: number) {
}
}
// Ticket 074: a sort helper index's one-time full-scan build (`build_sort_helper_index`) runs
// inline in the engine actor, so a `sort_by` poll can occasionally take much longer than the usual
// sub-poll-interval round trip — without this, that stall looks exactly like "sorting is broken"
// rather than "still indexing". Only flips `setBuilding(true)` if the fetch is still in flight after
// `thresholdMs`, so an ordinary fast poll (helper index already built/incrementally maintained)
// never flashes the indicator.
const SORT_BUILDING_INDICATOR_THRESHOLD_MS = 400
function trackSlowSortedFetch<T>(sortBy: string | null, setBuilding: (building: boolean) => void, promise: Promise<T>): Promise<T> {
if (!sortBy) return promise
const timer = setTimeout(() => setBuilding(true), SORT_BUILDING_INDICATOR_THRESHOLD_MS)
return promise.finally(() => {
clearTimeout(timer)
setBuilding(false)
})
}
/// Thin drag handle between the log viewer and the sections below it. Reports each pointer-move
/// delta via `onResize` (letting the caller clamp/apply it) and fires `onResizeEnd` on release so
/// the final size can be persisted once, not on every pixel of movement.
@@ -286,6 +303,7 @@ function StandaloneLog() {
const [filter, setFilter] = useState('')
const [filterError, setFilterError] = useState<string | null>(null)
const [sortSettings, setSortSettings] = useState<LogSortSettings>(loadLogSortSettings)
const [sortIndexBuilding, setSortIndexBuilding] = useState(false)
function handleSortSettingsChange(next: LogSortSettings) {
setSortSettings(next)
@@ -341,14 +359,18 @@ function StandaloneLog() {
bucketMs: effectiveRange ? computeBucketMs(effectiveRange.endMs - effectiveRange.startMs) : undefined,
filter: filter || undefined,
}),
fetchLogs({
startMs: effectiveRange?.startMs,
endMs: effectiveRange?.endMs,
limit: 1000,
filter: filter || undefined,
sortBy: sortSettings.sortBy ?? undefined,
sortMissingFallback: sortSettings.sortBy ? sortSettings.sortMissingFallback : undefined,
}),
trackSlowSortedFetch(
sortSettings.sortBy,
setSortIndexBuilding,
fetchLogs({
startMs: effectiveRange?.startMs,
endMs: effectiveRange?.endMs,
limit: 1000,
filter: filter || undefined,
sortBy: sortSettings.sortBy ?? undefined,
sortMissingFallback: sortSettings.sortBy ? sortSettings.sortMissingFallback : undefined,
}),
),
// Ticket 058: params can get auto-declared from freshly ingested lines mid-run, not just from a user's own edit.
fetchParamDeclarations(),
])
@@ -393,6 +415,7 @@ function StandaloneLog() {
paramNames={params.map((p) => p.name)}
sortSettings={sortSettings}
onSortSettingsChange={handleSortSettingsChange}
sortIndexBuilding={sortIndexBuilding}
/>
<LogPreview logs={logs} params={params} />
</div>
@@ -423,6 +446,7 @@ function FullApp() {
const [logViewerCollapsed, setLogViewerCollapsed] = useSectionCollapsed('log-viewer')
const [logViewerHeight, setLogViewerHeight] = useState(() => loadLogViewerHeight())
const [sortSettings, setSortSettings] = useState<LogSortSettings>(loadLogSortSettings)
const [sortIndexBuilding, setSortIndexBuilding] = useState(false)
function handleSortSettingsChange(next: LogSortSettings) {
setSortSettings(next)
@@ -497,14 +521,18 @@ function FullApp() {
bucketMs: effectiveRange ? computeBucketMs(effectiveRange.endMs - effectiveRange.startMs) : undefined,
filter: filter || undefined,
}),
fetchLogs({
startMs: effectiveRange?.startMs,
endMs: effectiveRange?.endMs,
limit: 1000,
filter: filter || undefined,
sortBy: sortSettings.sortBy ?? undefined,
sortMissingFallback: sortSettings.sortBy ? sortSettings.sortMissingFallback : undefined,
}),
trackSlowSortedFetch(
sortSettings.sortBy,
setSortIndexBuilding,
fetchLogs({
startMs: effectiveRange?.startMs,
endMs: effectiveRange?.endMs,
limit: 1000,
filter: filter || undefined,
sortBy: sortSettings.sortBy ?? undefined,
sortMissingFallback: sortSettings.sortBy ? sortSettings.sortMissingFallback : undefined,
}),
),
// Ticket 058: params can get auto-declared from freshly ingested lines mid-run, not just from a user's own edit.
fetchParamDeclarations(),
])
@@ -566,6 +594,7 @@ function FullApp() {
paramNames={params.map((p) => p.name)}
sortSettings={sortSettings}
onSortSettingsChange={handleSortSettingsChange}
sortIndexBuilding={sortIndexBuilding}
/>
<LogPreview logs={logs} params={params} />
</div>
webapp/src/components/Docs.tsx
+99 -66
View File
@@ -78,75 +78,108 @@ const SECTIONS: DocSection[] = [
id: 'filtering',
title: 'Filtering',
body: (
<div className="flex flex-col gap-2">
<div className="flex flex-col gap-3">
<p>
The filter bar combines regex search over raw log text with comparisons against declared metrics, joined
by <Code>and</Code>/<Code>or</Code>. Examples:
by <Code>and</Code>/<Code>or</Code>. The server evaluates the filter string; a parse error shows as inline
text under the filter bar.
</p>
<ul className="list-disc pl-5">
<li>
<Code>{'regex("error")'}</Code> — lines containing "error".
</li>
<li>
<Code>{'regex("error") and ottyl_seq<5'}</Code> — lines containing "error" with sequence under 5.
</li>
<li>
<Code>{'requests[duration]>100'}</Code> — the <Code>requests</Code> metric's captured{' '}
<Code>duration</Code> param over 100.
</li>
<li>
<Code>{'ottyl_ts>-5m'}</Code> — lines from the last 5 minutes.
</li>
<li>
<Code>{'ottyl_fmt==json'}</Code> — lines auto-detected as JSON.
</li>
</ul>
<p>
<Code>ottyl_seq</Code>, <Code>ottyl_ts</Code>, and <Code>ottyl_fmt</Code> are reserved metric names present
on every log line, so they're always filterable without declaring anything. <Code>ottyl_fmt</Code> holds the
auto-detected format of the line's text — <Code>log_stream</Code> (macOS <Code>log stream</Code> output),{' '}
<Code>json</Code>, <Code>logfmt</Code>, or <Code>none</Code>. A <Code>log_stream</Code> line also gets{' '}
<Code>log_type</Code>, <Code>process</Code>, and <Code>pid</Code> params derived automatically from its
fixed-column header. All three reserved names also appear (flagged <Code>reserved</Code>) in{' '}
<Code>GET /api/params</Code> for discoverability, though there's no UI to edit or remove them — see also{' '}
<em>Dashboards</em> below for using <Code>ottyl_seq</Code>/<Code>ottyl_ts</Code> in a panel query.
</p>
<p className="font-semibold text-text">Date/time formats</p>
<p>Timestamp comparisons accept three forms:</p>
<ul className="list-disc pl-5">
<li>
relative offsets from now — <Code>5h</Code> (5 hours from now), <Code>-3m</Code> (3 minutes ago)
</li>
<li>
absolute clock times — <Code>12:00</Code>, <Code>09:30:00</Code>
</li>
<li>
full dates — <Code>2026-01-15</Code>, <Code>2026-01-15T09:30:00Z</Code>
</li>
</ul>
<p>
The filter string is evaluated on the server. A parse error shows as inline text under the filter bar.
</p>
<p className="font-semibold text-text">Worked examples</p>
<ul className="list-disc pl-5">
<li>
Only lines mentioning "timeout" in the last 15 minutes:{' '}
<Code>{'regex("timeout") and ottyl_ts>-15m'}</Code>
</li>
<li>
Requests slower than 200ms, once a <Code>requests</Code> metric has a <Code>value</Code>-role{' '}
<Code>duration</Code> param declared: <Code>{'requests[duration]>200'}</Code>
</li>
<li>
5xx responses on one path, given a <Code>requests</Code> metric with a <Code>path</Code> label param —
note that <Code>[param]</Code> comparisons only work against a <em>declared metric's</em> labels, not a
bare param declared on its own, and values are unquoted (no surrounding <Code>"..."</Code>):{' '}
<Code>{'requests[path]==/checkout and regex("5[0-9]{2}")'}</Code>
</li>
<li>
Everything before noon on a specific date: <Code>{'ottyl_ts<2024-06-15T12:00:00Z'}</Code>
</li>
</ul>
<div>
<p className="font-semibold text-text">Regex match</p>
<ul className="list-disc pl-5">
<li>
<Code>{'regex("error")'}</Code> — lines containing "error". See the <Code>@</Code> icon next to the
filter bar for the exact regex syntax supported (Rust's <Code>regex</Code> crate — no lookaround or
backreferences).
</li>
</ul>
</div>
<div>
<p className="font-semibold text-text">Metric/param comparisons</p>
<ul className="list-disc pl-5">
<li>
<Code>{'requests[duration]>100'}</Code> — the <Code>requests</Code> metric's captured{' '}
<Code>duration</Code> param over 100. <Code>[param]</Code> comparisons only work against a{' '}
<em>declared metric's</em> labels, not a bare param declared on its own, and values are unquoted (no
surrounding <Code>"..."</Code>).
</li>
</ul>
</div>
<div>
<p className="font-semibold text-text">Reserved names (always filterable)</p>
<p>
<Code>ottyl_seq</Code>, <Code>ottyl_ts</Code>, and <Code>ottyl_fmt</Code> are present on every log line
without declaring anything, and also appear (flagged <Code>reserved</Code>) in{' '}
<Code>GET /api/params</Code> for discoverability, though there's no UI to edit or remove them.
</p>
<ul className="list-disc pl-5">
<li>
<Code>{'ottyl_seq<5'}</Code> — ingest sequence number under 5.
</li>
<li>
<Code>{'ottyl_ts>-5m'}</Code> — lines from the last 5 minutes (see Date/time formats below).
</li>
<li>
<Code>{'ottyl_fmt==json'}</Code> — the line's auto-detected text format:{' '}
<Code>log_stream</Code> (macOS <Code>log stream</Code> output — its columns are also auto-declared as
their own named params, e.g. <Code>PID</Code>/<Code>Thread</Code>, from that line's own header), <Code>json</Code>,{' '}
<Code>logfmt</Code>, or <Code>none</Code>.
</li>
</ul>
</div>
<div>
<p className="font-semibold text-text">Date/time formats</p>
<p>Timestamp comparisons (e.g. against <Code>ottyl_ts</Code>) accept three forms:</p>
<ul className="list-disc pl-5">
<li>
relative offsets from now — <Code>5h</Code> (5 hours from now), <Code>-3m</Code> (3 minutes ago)
</li>
<li>
absolute clock times — <Code>12:00</Code>, <Code>09:30:00</Code>
</li>
<li>
full dates — <Code>2026-01-15</Code>, <Code>2026-01-15T09:30:00Z</Code>
</li>
</ul>
</div>
<div>
<p className="font-semibold text-text">Combining clauses</p>
<ul className="list-disc pl-5">
<li>
<Code>{'regex("error") and ottyl_seq<5'}</Code> — lines containing "error" with sequence under 5.
</li>
<li>
<Code>{'requests[path]==/checkout and regex("5[0-9]{2}")'}</Code> — 5xx responses on one path, given a{' '}
<Code>requests</Code> metric with a <Code>path</Code> label param.
</li>
</ul>
</div>
<div>
<p className="font-semibold text-text">Worked examples</p>
<ul className="list-disc pl-5">
<li>
Only lines mentioning "timeout" in the last 15 minutes:{' '}
<Code>{'regex("timeout") and ottyl_ts>-15m'}</Code>
</li>
<li>
Requests slower than 200ms, once a <Code>requests</Code> metric has a <Code>value</Code>-role{' '}
<Code>duration</Code> param declared: <Code>{'requests[duration]>200'}</Code>
</li>
<li>
Everything before noon on a specific date: <Code>{'ottyl_ts<2024-06-15T12:00:00Z'}</Code>
</li>
</ul>
<p className="mt-1 text-text-subtle">
See also <em>Dashboards</em> below for using <Code>ottyl_seq</Code>/<Code>ottyl_ts</Code> in a panel
query.
</p>
</div>
</div>
),
},
webapp/src/components/FilterBar.tsx
+5 -4
View File
@@ -41,6 +41,11 @@ export function FilterBar({
<span className="flex items-center gap-1">
<HelpIcon sectionId="filtering" label="Help: filter syntax" />
<span className="text-xs text-text-subtle">Filter syntax</span>
<span className="mx-0.5 h-4 w-px bg-border-soft" aria-hidden="true" />
<RegexHelpIcon label="Help: regex(...) syntax" />
<span className="text-xs text-text-faint" title="Cheat sheet for regex(...) clauses specifically">
regex(...)
</span>
</span>
{collapseToggle}
<input
@@ -49,10 +54,6 @@ export function FilterBar({
value={draft}
onChange={(e) => setDraft(e.target.value)}
/>
<span className="flex items-center gap-1" title="Cheat sheet for regex(...) clauses specifically">
<RegexHelpIcon label="Help: regex(...) syntax" />
<span className="text-xs text-text-faint">regex(...)</span>
</span>
<button
type="submit"
className="rounded border border-border-soft px-3 py-1 text-sm text-text-secondary hover:bg-surface-2"
webapp/src/components/LogTimeline.tsx
+9
View File
@@ -141,6 +141,7 @@ export function LogTimeline({
paramNames,
sortSettings,
onSortSettingsChange,
sortIndexBuilding,
}: {
buckets: TimelineBucket[]
range: TimeRange | null
@@ -151,6 +152,9 @@ export function LogTimeline({
paramNames?: string[]
sortSettings?: LogSortSettings
onSortSettingsChange?: (next: LogSortSettings) => void
/** Ticket 074: set by the caller when a `sort_by` fetch has been in flight long enough that it's
* likely the engine is running `build_sort_helper_index`'s one-time full scan, not just stalled. */
sortIndexBuilding?: boolean
}) {
const containerRef = useRef<HTMLDivElement>(null)
const [dragStartX, setDragStartX] = useState<number | null>(null)
@@ -224,6 +228,11 @@ export function LogTimeline({
{sortSettings && onSortSettingsChange && (
<SortPicker paramNames={paramNames ?? []} settings={sortSettings} onChange={onSortSettingsChange} />
)}
{sortIndexBuilding && (
<span className="text-text-muted" title="First sort by this param scans every ingested line once; later polls reuse the built index.">
Building sort index…
</span>
)}
<div className="flex items-center gap-1.5">
<div className="flex items-center gap-1">
{TIME_RANGE_PRESETS.map((preset) => (
webapp/src/components/NavMenu.tsx
+13
View File
@@ -41,6 +41,16 @@ function schemeLabel(scheme: ColorScheme): string {
return scheme.replace(/_/g, ' ')
}
/** Ticket 073: plain `Xms` under a second (this stat is normally near-zero, so that's the common case), a humantime-style `Xs`/`Xm` above it — mirrors the backend's own humantime-duration inputs (see `Docs.tsx`'s window-parsing note) without pulling in a full duration-formatting library for one stat. */
function formatIngestLag(ms: number): string {
if (ms < 1000) return `${ms}ms`
const totalSeconds = ms / 1000
if (totalSeconds < 60) return `${totalSeconds % 1 === 0 ? totalSeconds : totalSeconds.toFixed(1)}s`
const minutes = Math.floor(totalSeconds / 60)
const seconds = Math.round(totalSeconds % 60)
return seconds === 0 ? `${minutes}m` : `${minutes}m${seconds}s`
}
export function NavMenu({ stats }: { stats: Stats | null }) {
const [openPanel, setOpenPanel] = useState<PanelName>(null)
const panel = useSettingsPanel()
@@ -102,6 +112,9 @@ export function NavMenu({ stats }: { stats: Stats | null }) {
seq {stats.first_sequence ?? 0}–{stats.last_sequence ?? 0}
</div>
<div>{stats.approx_bytes_total.toLocaleString()} bytes</div>
<div title="Highest ingestion-to-engine-processing lag seen in the last 5s">
lag {formatIngestLag(stats.ingest_lag_high_watermark_ms)}
</div>
</>
) : (
<div>connecting…</div>
webapp/src/lib/api.ts
+2
View File
@@ -49,6 +49,8 @@ export interface Stats {
last_timestamp_ms: number | null
/** Ticket 011: running approximate byte size of all currently-ingested logs. */
approx_bytes_total: number
/** Ticket 073: highest ingestion-to-engine-processing lag (ms) seen within the trailing 5s. */
ingest_lag_high_watermark_ms: number
}
export interface MetricDeclaration {