From 31978f3fc3d12223c0c56b4b3e214d87963a8514 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Tue, 15 Sep 2026 16:34:25 +0300 Subject: [PATCH] =?UTF-8?q?fix:=20background-delivery=20ground=20rules=20?= =?UTF-8?q?=E2=80=94=20the=20dock=20is=20the=20source=20of=20truth,=20deli?= =?UTF-8?q?vered=20content=20shows=20the=20moment=20it=20persists?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The owner's three-part test of the context-injection flow surfaced timing/visibility bugs, fixed as standing ground rules for ALL background processes (owner rules, 2026-09-15): 1. THE JOB STARTS AT SEND. triggerGraphContext moves from onFinish to right after the write-ahead persist in handleChatRequest: its only input is the first typed message (the topic), just made durable — so embedding + cosine scan run CONCURRENT with the prober's turn and the dock blob appears the moment the session starts, not after turn one finishes. Extraction stays at onFinish (it needs answers accumulated DURING the turn). 2. DELIVERED CONTENT SHOWS MID-TURN. Steers inserted server-side (claimed at turn start, persisted write-ahead) used to wait for the next turn's completion to appear — the full record sync mid-stream is forbidden (it breaks the SDK's in-place continuation merge), so adoption was deferred to idle. New append-only seam: core mergeAdopted() inserts the record's new message ids in canonical order (in-flight streaming tail kept last) and never rewrites a local message; any other shape bails to the full-sync-at-idle net. Wired at three points, no new standing polls: - turn-start window: three bounded record fetches (400ms/1s/2s) after a send catch the claimed steer while the turn streams; self-terminating, survives the submitted->streaming flip. - jobs-poll settle: mid-turn settles adopt append-only NOW, the owed full sync still runs at idle. - wake-adoption poll: each tick adopts persisted wake content, so steers show at the wake turn's START, not its finish. The dock stays honest with zero gap: jobs hold "running" while queued behind a pending question ("delivering context"), then flip done at claim/delivery — visible from start to delivery, and the turn-start nudge (useActivity().refresh) clears it within ~150ms. 3. THE MARKER IS CLICKABLE (observability). The knowledge-context SystemLine becomes GraphContextMarker: quiet line, popover with the delivered propositions IN DELIVERY ORDER (most similar first — the order the budget filled), each with its folded understanding level; Esc / outside pointerdown close. The part data was already the audit trail — now it's readable at the seam where it landed. Tests: mergeAdopted unit pins (6), marker render pins (3) — 468/468, tsc, eslint, build all green; live-smoked end-to-end: blob at send, marker at t+500ms mid-turn, popover order + levels, instant dock clear. --- components/activity-dock.tsx | 9 ++ components/chat.tsx | 126 +++++++++++++++++---- components/graph-context-marker.tsx | 104 +++++++++++++++++ src/core/ui-message.ts | 46 ++++++++ src/server/chat-handler.ts | 17 ++- tests/merge-adopted.test.ts | 71 ++++++++++++ tests/render-graph-context-marker.test.tsx | 59 ++++++++++ 7 files changed, 404 insertions(+), 28 deletions(-) create mode 100644 components/graph-context-marker.tsx create mode 100644 tests/merge-adopted.test.ts create mode 100644 tests/render-graph-context-marker.test.tsx diff --git a/components/activity-dock.tsx b/components/activity-dock.tsx index cba0669..46b8237 100644 --- a/components/activity-dock.tsx +++ b/components/activity-dock.tsx @@ -110,6 +110,15 @@ export function ActivityProvider({ children }: { children: React.ReactNode }) { return {children}; } +/** The activity feed seam for the rest of the shell: the dock's data + + * its refresh nudge. The chat page nudges on turn start (owner ground + * rules, 2026-09-15): a background process that starts at send — the + * knowledge-context injection — must be visible in the dock the moment + * it begins, without the dock polling faster while idle. */ +export function useActivity(): { feed: ActivityFeed; refresh: () => void } { + return useContext(ActivityContext); +} + // ─── Pure presentation helpers (exported for tests) ────────────────────────── /** One dock blob: the unified view of a job or an extraction run. */ diff --git a/components/chat.tsx b/components/chat.tsx index ec52e83..5a5ea65 100644 --- a/components/chat.tsx +++ b/components/chat.tsx @@ -21,8 +21,9 @@ import { SubagentSteerCard } from "@/components/artifacts/steer-card"; import { Composer } from "@/components/composer"; import { JobsContext } from "@/components/jobs-context"; import { StatsBar } from "@/components/stats-bar"; -import { graphContextOfMessage, type ChatSettings } from "@/src/core"; -import { formatTokens } from "@/components/usage-format"; +import { GraphContextMarker } from "@/components/graph-context-marker"; +import { useActivity } from "@/components/activity-dock"; +import { mergeAdopted, type ChatSettings } from "@/src/core"; import { PlanBlock, PlanPopup, latestTree } from "@/components/plan-block"; import { PlanMarginalia } from "@/components/plan-window"; import { WaypointsIcon } from "lucide-react"; @@ -82,16 +83,6 @@ const useIsoLayoutEffect = typeof window === "undefined" ? useEffect : useLayout /** A wake-pipeline steer message: user-role delivery whose metadata marks it * as a sub-agent result — rendered as an artifact card, never a bubble. */ -/** The knowledge-context card's one-liner: the numbers the observability - * doctrine wants visible at the seam — tokens and proposition count - * (parse-don't-cast; a malformed part still renders a quiet generic line). */ -function graphContextLine(message: { parts: ReadonlyArray<{ type: string; data?: unknown }> }): string { - const data = graphContextOfMessage(message); - return data - ? `Knowledge context — ~${formatTokens(data.tokens)} · ${data.propositions.length} propositions` - : "Knowledge context"; -} - /** The quiet system line (custom-message scheme rendering): centered, muted, * small — the transcript's voice for events that are not speech (wake * scaffolding, the learner's plan verdict). */ @@ -235,6 +226,73 @@ export function Chat({ } }, [chatId, setMessages, sendMessage, status]); + // ── Append-only record adoption (owner ground rules, 2026-09-15) ── + // Content delivered into the record server-side — the knowledge-context + // block, sub-agent artifacts, abort notes — must become visible the + // moment it persists, mid-turn included: it sits between messages, and + // nothing about a running turn should hide it. The full snapshot sync + // mid-stream is forbidden (it breaks the SDK's in-place continuation + // merge — the stale-snapshot lesson), so adoption here is APPEND-ONLY + // (core mergeAdopted): insert the record's new ids in their canonical + // order, never rewrite a local one, keep the in-flight tail last. The + // full syncFromServer stays the net at idle (reorder, plan blocks, turns). + const messagesRef = useRef(messages); + messagesRef.current = messages; + const adoptAppendOnly = useCallback(async () => { + try { + const res = await fetch(`/api/chats/${chatId}`); + if (!res.ok) return; + const record = (await res.json()) as { messages?: TeacherUIMessage[] }; + if (!Array.isArray(record.messages)) return; + const merged = mergeAdopted(messagesRef.current, record.messages); + if (merged) setMessages(merged); + } catch { + /* transient — the idle full sync is the net */ + } + }, [chatId, setMessages]); + + // Turn-start window: the server inserts claimed steers right after our + // outgoing message, BEFORE the stream produces anything — so a bounded, + // self-terminating window of three record fetches (400ms/1s/2s) catches + // the insert and shows the marker during the turn, then stops. No + // standing poll — the window exists only for these seconds. The same + // instant nudges the activity feed: the knowledge-context job starts at + // send (server-side), and the dock must say so immediately. + const { refresh: refreshActivity } = useActivity(); + const turnStartStatusRef = useRef(status); + // The window must survive the submitted→streaming flip (the stream starts + // in well under 400ms), so the timers are guarded by this ref — dropped at + // unmount, left alone otherwise: the window self-terminates after 2s. + const adoptionWindowOpenRef = useRef(false); + const turnStartTimerRefs = useRef[]>([]); + useEffect(() => { + const was = turnStartStatusRef.current; + turnStartStatusRef.current = status; + if (was === "submitted" || status !== "submitted") return; + // A turn just started (status flipped ready → submitted): open the + // bounded adoption window and nudge the dock's feed. + refreshActivity(); + adoptionWindowOpenRef.current = true; + const timers = [400, 1000, 2000].map((ms) => + setTimeout(() => { + if (adoptionWindowOpenRef.current) void adoptAppendOnly(); + }, ms), + ); + // Deliberately no cleanup here: this effect re-runs on every status + // flip, which would cancel the window at submitted→streaming. + timers.forEach((t) => { + turnStartTimerRefs.current.push(t); + }); + }, [status, adoptAppendOnly, refreshActivity]); + useEffect(() => { + const timers = turnStartTimerRefs; + return () => { + adoptionWindowOpenRef.current = false; + timers.current.forEach(clearTimeout); + timers.current = []; + }; + }, []); + // The live plan popup (wave 4): opened by any transcript plan block, the // top-bar plan chip, and the dock band's marginalia. const [planOpen, setPlanOpen] = useState(false); @@ -422,14 +480,18 @@ export function Chat({ const prev = jobStatusRef.current.get(j.id); if (prev === "running" && j.status !== "running") { // The record is only guaranteed complete when no turn is in - // flight — locally or server-side. A coalesced settle fires - // mid-turn by design; adopt the record on the first idle tick. + // flight — locally or server-side (a coalesced settle fires + // mid-turn by design), so the FULL sync waits for idle. + // Ground rule (2026-09-15): delivered content shows + // immediately anyway — adopt append-only NOW; the idle + // sync still settles the record properly. if (localStreamRef.current || data.turnInFlight === true) { pendingSyncRef.current = true; + void adoptAppendOnly(); } else { void syncFromServer(); + } } - } jobStatusRef.current.set(j.id, j.status); } if ( @@ -455,7 +517,7 @@ export function Chat({ cancelled = true; clearInterval(timer); }; - }, [pollMs, jobPartCount, chatId, syncFromServer]); + }, [pollMs, jobPartCount, chatId, syncFromServer, adoptAppendOnly]); // Re-attach after mid-turn navigation (the key fix): poll the record until // the orphaned turn finishes (turnInFlight flips false server-side in @@ -470,10 +532,24 @@ export function Chat({ if (cancelled) return; try { const res = await fetch(`/api/chats/${chatId}`); - if (res.ok && !((await res.json()) as { turnInFlight?: boolean }).turnInFlight) { - setTurnInFlight(false); - void syncFromServer(); - return; + if (res.ok) { + const record = (await res.json()) as { + turnInFlight?: boolean; + messages?: TeacherUIMessage[]; + }; + // Ground rule (2026-09-15): steer content persisted by the + // wake shows NOW, at the wake turn's start — not at its + // finish. Append-only is safe here: no local stream runs + // under a server-side wake turn. + if (Array.isArray(record.messages)) { + const merged = mergeAdopted(messagesRef.current, record.messages); + if (merged) setMessages(merged); + } + if (!record.turnInFlight) { + setTurnInFlight(false); + void syncFromServer(); + return; + } } } catch { /* transient network hiccup — keep polling */ @@ -553,11 +629,11 @@ export function Chat({ ) : m.role === "user" && m.parts.some((p) => p.type === "data-graph-context") ? ( // The prior-knowledge injection's delivery (vision-context-injection.md): - // quiet system line, the numbers only — the block itself was for the - // model (adapter flattening); the part data stays the audit trail. - - {graphContextLine(m)} - + // a quiet, CLICKABLE line (owner observability ask, 2026-09-15) — + // the popover shows which propositions were delivered, in delivery + // order; the block itself was for the model (adapter flattening), + // the part data is the audit trail this reads from. + ) : m.role === "user" && m.parts.some((p) => p.type === "data-probe-handoff") ? ( // The probe→teach handoff wake (custom-message scheme): quiet // system line, never a user bubble — the wake is scaffolding, diff --git a/components/graph-context-marker.tsx b/components/graph-context-marker.tsx new file mode 100644 index 0000000..7d3b833 --- /dev/null +++ b/components/graph-context-marker.tsx @@ -0,0 +1,104 @@ +"use client"; + +import { useEffect, useRef, useState, type ReactNode } from "react"; +import { BookOpenIcon } from "lucide-react"; +import { graphContextOfMessage, type GraphContextData } from "@/src/core"; +import { formatTokens } from "@/components/usage-format"; + +/** + * The knowledge-context delivery marker (vision-context-injection.md): the + * quiet transcript line for the injected prior-knowledge block — CLICKABLE + * (owner observability ask, 2026-09-15): the popover shows exactly which + * propositions were delivered, IN DELIVERY ORDER (most similar first — + * the order the selection filled the budget), each with its folded current + * understanding level. The record part is the audit trail; this makes it + * readable at the seam where it landed. + * + * Same quiet-line discipline as the SystemLine family (centered, muted, + * small); same popover discipline as the stats-detail (close on outside + * pointerdown + Escape, anchored below the marker). + */ +export function GraphContextMarker({ + message, + testId = "graph-context", +}: { + /** The steer message (marker-only user message). */ + message: { parts: ReadonlyArray<{ type: string; data?: unknown }> }; + testId?: string; +}) { + const [open, setOpen] = useState(false); + const data = graphContextOfMessage(message); + const ref = useRef(null); + + useEffect(() => { + if (!open) return; + const onPointerDown = (e: PointerEvent) => { + if (ref.current && !ref.current.contains(e.target as Node)) setOpen(false); + }; + const onKeyDown = (e: KeyboardEvent) => { + if (e.key === "Escape") setOpen(false); + }; + document.addEventListener("pointerdown", onPointerDown); + document.addEventListener("keydown", onKeyDown); + return () => { + document.removeEventListener("pointerdown", onPointerDown); + document.removeEventListener("keydown", onKeyDown); + }; + }, [open]); + + const summary = data + ? `Knowledge context — ~${formatTokens(data.tokens)} · ${data.propositions.length} propositions` + : "Knowledge context"; + + return ( +
+ + {open && data && } +
+ ); +} + +/** The popover body — exported for render tests. Delivery order preserved. */ +export function GraphContextList({ data }: { data: GraphContextData }): ReactNode { + return ( +
+
+ + Prior knowledge delivered + + + ~{formatTokens(data.tokens)} tok of {data.budget} budget + +
+

+ Most similar first — the order the budget filled. The tags are your current + understanding level for each proposition. +

+
    + {data.propositions.map((p, i) => ( +
  1. + {i + 1}. + {p.level && ( + + {p.level} + + )} + {p.statement} +
    2. + ))} +
+
+ ); +} \ No newline at end of file diff --git a/src/core/ui-message.ts b/src/core/ui-message.ts index f2b4258..4c062bf 100644 --- a/src/core/ui-message.ts +++ b/src/core/ui-message.ts @@ -56,3 +56,49 @@ export type TeacherUIDataParts = { }; export type TeacherUIMessage = UIMessage; + +// ─── Append-only record adoption (background-process ground rules) ────────── + +/** + * Merge server-added messages into the client's live list WITHOUT the full + * snapshot swap (owner ground rules, 2026-09-15: content delivered into the + * record — context blocks, sub-agent artifacts, abort notes — must become + * visible the moment it persists, mid-turn included; it sits between messages + * and nothing about a running turn should hide it). + * + * Safe against the stale-snapshot lesson (mid-turn full syncs break the SDK's + * in-place continuation merge): this NEVER rewrites or removes an existing + * message — it returns the RECORD's message order when that order is exactly + * the client's persisted core plus new server-added ids (the server appends + * only: steers are inserted right after the last persisted user message), + * with the client's in-flight streaming assistant message — its last entry, + * not yet persisted — kept at the tail. Any other shape (a missing local id + * = the record was rewritten/reordered) returns null: the caller defers to + * the full sync at idle, the pre-existing safety net. + */ +export function mergeAdopted( + local: ReadonlyArray, + record: ReadonlyArray, +): TeacherUIMessage[] | null { + const recordIds = new Set(record.map((m) => m.id)); + // The in-flight tail: the client's last message may legitimately be absent + // from the record (the SDK's streaming assistant message persists only at + // onFinish). Exactly one such tail is allowed. + let core = local; + let tail: TeacherUIMessage | undefined; + const last = local.at(-1); + if (last && !recordIds.has(last.id)) { + tail = last; + core = local.slice(0, -1); + } + // No rewrite happened if every core message is still in the record — + // otherwise this is not a pure append; bail to the full-sync-at-idle. + if (core.some((m) => !recordIds.has(m.id))) return null; + // Nothing new → nothing to do. + const localIds = new Set(local.map((m) => m.id)); + if (!record.some((m) => !localIds.has(m.id))) return null; + // The record's order is canonical for everything persisted (the server + // appends only; plan-block part attachment changes parts, not order) — + // merged = record + the in-flight tail. + return tail ? [...record, tail] : [...record]; +} diff --git a/src/server/chat-handler.ts b/src/server/chat-handler.ts index b1969c9..ffb545a 100644 --- a/src/server/chat-handler.ts +++ b/src/server/chat-handler.ts @@ -146,6 +146,20 @@ export async function handleChatRequest( console.error(`[chat ${chatId}] write-ahead persist failed:`, err); } + // THE GRAPH READ CHANNEL (vision-context-injection.md): one shot per + // chat, at session start — fired HERE, at turn start, not in onFinish: + // the injection's only input is the first typed message (the topic), + // which the write-ahead persist above just made durable, so it runs + // CONCURRENT with the prober's turn (embedding + cosine scan, no model + // call) and the dock shows the job the moment the session starts. Its + // steer then waits in the wake queue through the turn (waitForIdle) — + // exactly the same hold as a sub-agent result finishing mid-turn. All + // gates (exactly-one-text-message topic, budget, one-shot) live in the + // runner module; for every session past its start this is a no-op. + // Extraction stays in onFinish by contrast: it depends on quiz + // answers accumulated DURING the turn. + triggerGraphContext(graphContext, chatId); + // MONEY CHANNEL (wave 4): one collector per request, fed by the agent's // message_end events. Every model call's usage lands in the chat record's // append-only turns log at onFinish — idempotent per turn id, so a @@ -237,9 +251,6 @@ export async function handleChatRequest( }); } triggerExtraction(stores, extraction, chatId); - // The graph READ channel: one-shot prior-knowledge injection at - // session start — a no-op for every session past its start. - triggerGraphContext(graphContext, chatId); // LAST: a poller that sees not-in-flight must read an already-persisted // record — endTurn only after persistTurn has done its job. endTurn(chatId); diff --git a/tests/merge-adopted.test.ts b/tests/merge-adopted.test.ts new file mode 100644 index 0000000..2488a56 --- /dev/null +++ b/tests/merge-adopted.test.ts @@ -0,0 +1,71 @@ +/** + * mergeAdopted (core/ui-message.ts) — the append-only record-adoption seam + * (owner ground rules, 2026-09-15): server-added steer messages must become + * visible mid-turn without the full snapshot swap that breaks the SDK's + * continuation merge. Pure, no React. + */ + +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { mergeAdopted, type TeacherUIMessage } from "../src/core/index.ts"; + +function userText(id: string, text: string): TeacherUIMessage { + return { id, role: "user", parts: [{ type: "text", text }] }; +} +function steer(id: string): TeacherUIMessage { + return { id, role: "user", parts: [{ type: "data-graph-context", data: { chatId: "c", budget: 1024, tokens: 1, propositions: [] } }] }; +} +function assistant(id: string): TeacherUIMessage { + return { id, role: "assistant", parts: [{ type: "text", text: "reply" }] }; +} + +describe("mergeAdopted — append-only adoption", () => { + it("inserts a server-added steer between the persisted core and the in-flight tail", () => { + const local = [userText("u1", "topic"), userText("u2", "answer"), assistant("a_inflight")]; + const record = [userText("u1", "topic"), userText("u2", "answer"), steer("steer_ctx_c")]; + const merged = mergeAdopted(local, record); + assert.ok(merged); + assert.deepEqual( + merged.map((m) => m.id), + ["u1", "u2", "steer_ctx_c", "a_inflight"], + "marker between the answer and the streaming bubble", + ); + }); + + it("adopts into an idle client (no in-flight tail)", () => { + const local = [userText("u1", "topic")]; + const record = [userText("u1", "topic"), steer("s1"), assistant("a1")]; + const merged = mergeAdopted(local, record); + assert.deepEqual( + merged?.map((m) => m.id), + ["u1", "s1", "a1"], + ); + }); + + it("nothing new → null (no needless re-render)", () => { + const local = [userText("u1", "topic")]; + assert.equal(mergeAdopted(local, [userText("u1", "topic")]), null); + // The tail alone is not 'new' — nothing to adopt. + assert.equal(mergeAdopted([assistant("a_x")], []), null); + }); + + it("a missing core id (record rewrite) → null: defer to the full sync at idle", () => { + // A local message vanished from the record — mid-turn this means a + // rewrite, and the append-only shortcut must not paper over it. + const local = [userText("u1", "a"), userText("u2", "b"), assistant("tail")]; + const record = [userText("u2", "b"), steer("s1")]; + assert.equal(mergeAdopted(local, record), null); + }); + + it("only the LAST local message may be the absent in-flight tail", () => { + // Two absent local messages: not a simple append — bail. + const local = [userText("u1", "a"), userText("u2", "gone"), assistant("tail")]; + const record = [userText("u1", "a"), steer("s1")]; + assert.equal(mergeAdopted(local, record), null); + }); + + it("an empty local list adopts the whole record", () => { + const record = [steer("s1"), assistant("a1")]; + assert.deepEqual(mergeAdopted([], record)?.map((m) => m.id), ["s1", "a1"]); + }); +}); \ No newline at end of file diff --git a/tests/render-graph-context-marker.test.tsx b/tests/render-graph-context-marker.test.tsx new file mode 100644 index 0000000..4125828 --- /dev/null +++ b/tests/render-graph-context-marker.test.tsx @@ -0,0 +1,59 @@ +import { renderToStaticMarkup } from "react-dom/server"; +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { GraphContextList, GraphContextMarker } from "../components/graph-context-marker.tsx"; +import type { GraphContextData, TeacherUIMessage } from "../src/core/index.ts"; + +// The knowledge-context delivery marker (owner observability ask, 2026-09-15): +// a quiet line that opens the delivered propositions, in delivery order, +// each with its folded understanding level. Static markup can't click — the +// closed line and the popover body are pinned separately. + +const DATA: GraphContextData = { + chatId: "c1", + budget: 1024, + tokens: 537, + propositions: [ + { id: "n1", statement: "Water boils at 100°C at standard pressure.", level: "solid" }, + { id: "n2", statement: "Modal realism takes possible worlds as concrete.", level: "shaky" }, + { id: "n3", statement: "An untagged proposition has no folded level.", level: null }, + ], +}; + +function markerMessage(): TeacherUIMessage { + return { id: "steer_ctx_c1", role: "user", parts: [{ type: "data-graph-context", data: DATA }] }; +} + +describe("GraphContextMarker", () => { + it("renders the quiet summary line with token + proposition counts", () => { + const html = renderToStaticMarkup(); + assert.ok(html.includes("Knowledge context — ~537")); + assert.ok(html.includes("3 propositions")); + assert.ok(html.includes('data-testid="graph-context"')); + assert.ok(html.includes(" { + const html = renderToStaticMarkup( + , + ); + assert.ok(html.includes("Knowledge context { + it("lists propositions 1..n with their level tags; null level gets no tag", () => { + const html = renderToStaticMarkup(); + const first = html.indexOf("Water boils"); + const second = html.indexOf("Modal realism"); + const third = html.indexOf("untagged"); + assert.ok(first !== -1 && second !== -1 && third !== -1, "all three statements render"); + assert.ok(first < second && second < third, "delivery order preserved"); + assert.ok(html.includes("solid") && html.includes("shaky"), "level tags render"); + // The null-level row: a bare of text, no level chip before it. + const nullRow = html.slice(html.indexOf("untagged") - 120, html.indexOf("untagged")); + assert.ok(!nullRow.includes(">resisted<") && !nullRow.includes(">accepted<")); + assert.ok(html.includes("537 tok of 1024 budget"), "budget + delivered tokens visible"); + }); +}); \ No newline at end of file