feat: probe/teach agent split — explicit phase markers, position-only plan map
Diagnosis (2026-09-11 user testing, minimal-effort GLM): sessions stuck on 'plan' forever. Two causes: (1) the three-phase view read the stored tree's currentNodeId, which nothing ever writes post author-once (2026-09-03) — a structural dead seam; (2) the teacher stochastically left quiz.nodeId untagged, and done-by-quiz-result semantics would have falsely completed nodes anyway. The redesign (docs/probe-teach-split.md) abolishes derived occupancy in favor of explicit events: - Two agents over one transcript: createProberAgent (quiz without a nodeId field, ask, probe_done) and createTeacherAgent (full toolset, plan_init onward). The chat-handler construction branch keys on a probe_done part in the transcript. MODELS.prober added. - probe_done: server-side seam tool — carries the learner assessment (strands + free-form edge read), refuses while a question is open or the probe is already concluded, and queues a handoff steer through the generalized wake pipeline (steers now dedupe by id, not jobId; job finalization only for job steers). The teacher auto-wakes and authors the plan; the learner sees the assessment via the probe-done card. - phases: probe ends at a concluded probe_done, plan ends at a successful plan_init — strictly sequential, no skipping, no legacy compatibility (data wiped at the pivot per doctrine). - Plan map = position, not completion: derivePlanStatuses reads only tagged quiz CALLS (state/outcome feed nothing); current = latest tagged node with consecutive duplicates collapsed, visited = earlier distinct stays (a revisit stays in both — current wins at render). PlanTree is structure-only; currentNodeId/doneNodeIds and the dead mutation helpers are gone. No fallback current. - Prompts split per agent (prompts/prober/, prompts/teacher/ over the trimmed shared base); graph-context injection removed entirely (shelved, docs/probe-teach-split.md §5); knowledge-graph context no longer reaches either agent. - Plan blocks are full-only snapshots at plan_init (neighborhood machinery retired; the persistent DAG window is the future UI wave); usage capture unchanged. Gates: tsc clean, 294/294 tests (phases/plan-tree/plan-blocks/question- tools rewritten to the new semantics; probe-done + handoff-wake tests added), lint, build. Live GLM smoke PASS: prober concluded with an honest assessment, teacher auto-woke, authored a 10-node plan with the block attached, then tagged checks on slope-instinct → secant derived current/ visited exactly as designed; grading honest both ways (driver off-by-one proved a wrong answer grades false).
This commit is contained in:
amosblomqvist
+67
-203
@@ -1,220 +1,84 @@
|
||||
/**
|
||||
* Phase derivation (W5, vision §2 "two phases") — the probe→teach boundary.
|
||||
* phases — the event-driven phase chain (docs/probe-teach-split.md §4).
|
||||
*
|
||||
* phaseAt is the pure seam-finder: plan_init IS the transition marker, so the
|
||||
* first message containing a tool-plan_init part flips everything from it
|
||||
* onward to "teach". The boundary cases here are the ones the UI leans on:
|
||||
* teach checks collapse behind the reveal button, probe questions auto-open.
|
||||
* probe ends at the first CONCLUDED probe_done (output-available); planning
|
||||
* ends at the first successful plan_init (a tree with nodes). No stored-tree
|
||||
* seam: the phase is a pure fold over the transcript, so it can never stick
|
||||
* (the old currentNodeId-based phaseOfTree read a field nothing ever set).
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import {
|
||||
phaseAt,
|
||||
phaseAt3,
|
||||
phaseOfTree,
|
||||
phaseTimeline,
|
||||
type Phase3Message,
|
||||
type PhaseMessage,
|
||||
} from "../src/core/phases.ts";
|
||||
import assert from "node:assert";
|
||||
import test from "node:test";
|
||||
import type { UIMessage } from "ai";
|
||||
import { phaseAt3, phaseTimeline, type Phase3 } from "../src/core/phases.ts";
|
||||
|
||||
/** A minimal message: just typed parts, exactly the structural slice phaseAt reads. */
|
||||
function msg(...partTypes: string[]): PhaseMessage {
|
||||
return { parts: partTypes.map((type) => ({ type })) };
|
||||
/** Text-only user message. */
|
||||
function user(text: string): UIMessage {
|
||||
return { id: `u_${text}`, role: "user", parts: [{ type: "text", text }] };
|
||||
}
|
||||
|
||||
/** Realistic shape: full tool part objects with state fields (extra fields ignored). */
|
||||
function toolMsg(...parts: { type: string; state: string; toolCallId: string }[]): PhaseMessage {
|
||||
return { parts };
|
||||
}
|
||||
|
||||
describe("phaseAt", () => {
|
||||
it("no messages → probe", () => {
|
||||
assert.equal(phaseAt([]), "probe");
|
||||
});
|
||||
|
||||
it("no plan_init anywhere → probe at every index (pre-plan sessions keep the old behavior)", () => {
|
||||
const messages = [
|
||||
msg("text"),
|
||||
msg("text", "tool-quiz", "text"),
|
||||
toolMsg({ type: "tool-quiz", state: "input-available", toolCallId: "q1" }),
|
||||
msg("text"),
|
||||
];
|
||||
for (let i = 0; i < messages.length; i++) {
|
||||
assert.equal(phaseAt(messages, i), "probe", `index ${i}`);
|
||||
}
|
||||
assert.equal(phaseAt(messages), "probe");
|
||||
});
|
||||
|
||||
it("the boundary: probe before the plan_init message, teach from it onward", () => {
|
||||
const messages = [
|
||||
msg("text"), // probe chatter
|
||||
msg("text", "tool-quiz"), // probe question
|
||||
msg("text", "tool-plan_init", "text"), // ← the seam (divider renders here)
|
||||
msg("text", "tool-quiz"), // a check — teach phase
|
||||
msg("text"),
|
||||
];
|
||||
assert.equal(phaseAt(messages, 0), "probe");
|
||||
assert.equal(phaseAt(messages, 1), "probe");
|
||||
assert.equal(phaseAt(messages, 2), "teach", "the message containing plan_init is already teach");
|
||||
assert.equal(phaseAt(messages, 3), "teach");
|
||||
assert.equal(phaseAt(messages, 4), "teach");
|
||||
assert.equal(phaseAt(messages), "teach", "default = current phase (last message)");
|
||||
});
|
||||
|
||||
it("multiple plan_init parts → the FIRST wins (a later re-init never moves the seam)", () => {
|
||||
const messages = [
|
||||
msg("text"),
|
||||
msg("tool-plan_init"), // first — the seam
|
||||
msg("text"),
|
||||
msg("tool-plan_init"), // a refused re-init attempt: must not matter
|
||||
msg("text"),
|
||||
];
|
||||
assert.equal(phaseAt(messages, 0), "probe");
|
||||
assert.equal(phaseAt(messages, 1), "teach");
|
||||
assert.equal(phaseAt(messages, 2), "teach", "between the two plan_inits: already teach (first wins)");
|
||||
assert.equal(phaseAt(messages, 3), "teach");
|
||||
});
|
||||
|
||||
it("any part state counts — the call itself is the transition, even mid-stream", () => {
|
||||
const messages = [
|
||||
msg("text"),
|
||||
toolMsg({ type: "tool-plan_init", state: "input-streaming", toolCallId: "p1" }),
|
||||
];
|
||||
assert.equal(phaseAt(messages, 1), "teach");
|
||||
});
|
||||
|
||||
it("upToIndex past the end / negative → clamped, never throws", () => {
|
||||
const messages = [msg("tool-plan_init")];
|
||||
assert.equal(phaseAt(messages, 99), "teach");
|
||||
assert.equal(phaseAt(messages, -1), "probe", "empty scan range → probe");
|
||||
assert.equal(phaseAt([]), "probe");
|
||||
});
|
||||
});
|
||||
|
||||
// ─── The three-phase view (picker strips, header chip, transcript dividers) ────
|
||||
|
||||
/** A minimal tree: nodes as opaque ids is all phaseOfTree reads. */
|
||||
function tree(nodes: string[], currentNodeId: string | null): { nodes: unknown[]; currentNodeId: string | null } {
|
||||
return { nodes: nodes.map((id) => ({ id })), currentNodeId };
|
||||
}
|
||||
|
||||
/** A message whose parts are full plan-op tool parts with outputs. */
|
||||
function planMsg(
|
||||
...ops: { tool: string; ok: boolean; tree: ReturnType<typeof tree> }[]
|
||||
): Phase3Message {
|
||||
/** An assistant message carrying a tool part in the given state. */
|
||||
function assistantTool(
|
||||
name: string,
|
||||
state: "input-available" | "output-available" | "output-error",
|
||||
output?: unknown,
|
||||
): UIMessage {
|
||||
return {
|
||||
parts: ops.map((op, i) => ({
|
||||
type: `tool-plan_${op.tool}`,
|
||||
state: "output-available",
|
||||
toolCallId: `p${i}`,
|
||||
output: { ok: op.ok, tree: op.tree },
|
||||
})),
|
||||
id: `a_${name}_${state}_${Math.random().toString(36).slice(2, 6)}`,
|
||||
role: "assistant",
|
||||
parts: [{ type: `tool-${name}`, state, ...(output !== undefined ? { output } : {}) } as never],
|
||||
};
|
||||
}
|
||||
|
||||
describe("phaseOfTree", () => {
|
||||
it("no tree / empty tree → probe", () => {
|
||||
assert.equal(phaseOfTree(undefined), "probe");
|
||||
assert.equal(phaseOfTree(null), "probe");
|
||||
assert.equal(phaseOfTree(tree([], null)), "probe", "emptyTree() (a failed init) reads probe");
|
||||
});
|
||||
const DONE_ASSESSMENT = { assessment: { strands: [{ strand: "s", assessment: "a" }] } };
|
||||
const PLAN_OK = { ok: true, tree: { nodes: [{ id: "n1", title: "N" }], edges: [] } };
|
||||
|
||||
it("tree with no current node → plan", () => {
|
||||
assert.equal(phaseOfTree(tree(["a", "b"], null)), "plan");
|
||||
});
|
||||
test("phaseAt3: probe until a concluded probe_done; plan until a successful plan_init; teach after", () => {
|
||||
const probe = phaseAt3([user("teach me calculus"), assistantTool("quiz", "output-available")]);
|
||||
assert.equal(probe, "probe", "quiz answers alone never end the probe");
|
||||
|
||||
it("occupied node → teach", () => {
|
||||
assert.equal(phaseOfTree(tree(["a", "b"], "a")), "teach");
|
||||
});
|
||||
const concluded = phaseAt3([
|
||||
user("teach me"),
|
||||
assistantTool("probe_done", "output-available", DONE_ASSESSMENT),
|
||||
]);
|
||||
assert.equal(concluded, "plan", "a concluded probe_done ends the probe (planning begins)");
|
||||
|
||||
const plan = phaseAt3([
|
||||
user("teach me"),
|
||||
assistantTool("probe_done", "output-available", DONE_ASSESSMENT),
|
||||
assistantTool("plan_init", "output-available", PLAN_OK),
|
||||
]);
|
||||
assert.equal(plan, "teach", "a successful plan_init ends planning");
|
||||
|
||||
const failedPlan = phaseAt3([
|
||||
user("teach me"),
|
||||
assistantTool("probe_done", "output-available", DONE_ASSESSMENT),
|
||||
assistantTool("plan_init", "output-error", { ok: false, error: "bad mermaid" }),
|
||||
]);
|
||||
assert.equal(failedPlan, "plan", "a failed plan_init does not end planning");
|
||||
});
|
||||
|
||||
describe("phaseAt3", () => {
|
||||
it("no plan parts anywhere → probe at every index", () => {
|
||||
const messages: Phase3Message[] = [msg("text"), msg("tool-quiz"), msg("text")];
|
||||
for (let i = 0; i < messages.length; i++) {
|
||||
assert.equal(phaseAt3(messages, i), "probe", `index ${i}`);
|
||||
}
|
||||
assert.equal(phaseAt3([]), "probe");
|
||||
});
|
||||
|
||||
it("the probe→plan seam: the first plan output with a non-empty tree", () => {
|
||||
const messages: Phase3Message[] = [
|
||||
msg("text"), // probe chatter
|
||||
planMsg({ tool: "plan_init", ok: false, tree: tree([], null) }), // failed init: NOT a seam
|
||||
msg("text"),
|
||||
planMsg({ tool: "plan_init", ok: true, tree: tree(["a", "b"], null) }), // ← the seam
|
||||
msg("text"),
|
||||
];
|
||||
assert.equal(phaseAt3(messages, 0), "probe");
|
||||
assert.equal(phaseAt3(messages, 1), "probe", "a failed init carries an empty tree — no seam");
|
||||
assert.equal(phaseAt3(messages, 2), "probe");
|
||||
assert.equal(phaseAt3(messages, 3), "plan");
|
||||
assert.equal(phaseAt3(messages, 4), "plan");
|
||||
assert.equal(phaseAt3(messages), "plan", "default = current phase");
|
||||
});
|
||||
|
||||
it("the plan→teach seam: the first tree output with an occupied node", () => {
|
||||
const messages: Phase3Message[] = [
|
||||
msg("text"),
|
||||
planMsg({ tool: "plan_init", ok: true, tree: tree(["a", "b"], null) }),
|
||||
msg("text"),
|
||||
planMsg({ tool: "plan_set_current", ok: true, tree: tree(["a", "b"], "a") }), // ← the seam
|
||||
msg("text"),
|
||||
];
|
||||
assert.equal(phaseAt3(messages, 2), "plan");
|
||||
assert.equal(phaseAt3(messages, 3), "teach");
|
||||
assert.equal(phaseAt3(messages, 4), "teach");
|
||||
});
|
||||
|
||||
it("collapsed seams: init and set_current in one message → straight to teach", () => {
|
||||
const messages: Phase3Message[] = [
|
||||
msg("text"),
|
||||
planMsg(
|
||||
{ tool: "plan_init", ok: true, tree: tree(["a"], null) },
|
||||
{ tool: "plan_set_current", ok: true, tree: tree(["a"], "a") },
|
||||
),
|
||||
];
|
||||
assert.equal(phaseAt3(messages, 0), "probe");
|
||||
assert.equal(phaseAt3(messages, 1), "teach");
|
||||
});
|
||||
|
||||
it("monotonic: a mid-teach node removal (current → null) never regresses the phase", () => {
|
||||
const messages: Phase3Message[] = [
|
||||
planMsg({ tool: "plan_init", ok: true, tree: tree(["a", "b"], null) }),
|
||||
planMsg({ tool: "plan_set_current", ok: true, tree: tree(["a", "b"], "a") }),
|
||||
msg("text"),
|
||||
planMsg({ tool: "plan_remove_node", ok: true, tree: tree(["b"], null) }), // edit, not a regression
|
||||
];
|
||||
assert.equal(phaseAt3(messages, 2), "teach");
|
||||
assert.equal(phaseAt3(messages, 3), "teach", "phase only ever advances");
|
||||
});
|
||||
|
||||
it("upToIndex past the end / negative → clamped, never throws", () => {
|
||||
const messages: Phase3Message[] = [planMsg({ tool: "plan_init", ok: true, tree: tree(["a"], null) })];
|
||||
assert.equal(phaseAt3(messages, 99), "plan");
|
||||
assert.equal(phaseAt3(messages, -1), "probe", "empty scan range → probe");
|
||||
});
|
||||
test("phaseAt3: a mere probe_done CALL (input-available, not yet concluded) keeps the probe open", () => {
|
||||
const still = phaseAt3([user("teach me"), assistantTool("probe_done", "input-available")]);
|
||||
assert.equal(still, "probe");
|
||||
});
|
||||
|
||||
describe("phaseTimeline", () => {
|
||||
it("equals phaseAt3 at every index (the linear fold is the same derivation)", () => {
|
||||
const messages: Phase3Message[] = [
|
||||
msg("text"),
|
||||
planMsg({ tool: "plan_init", ok: false, tree: tree([], null) }),
|
||||
planMsg({ tool: "plan_init", ok: true, tree: tree(["a", "b"], null) }),
|
||||
msg("text"),
|
||||
planMsg({ tool: "plan_set_current", ok: true, tree: tree(["a", "b"], "a") }),
|
||||
msg("text"),
|
||||
];
|
||||
const timeline = phaseTimeline(messages);
|
||||
assert.equal(timeline.length, messages.length);
|
||||
for (let i = 0; i < messages.length; i++) {
|
||||
assert.equal(timeline[i], phaseAt3(messages, i), `index ${i}`);
|
||||
}
|
||||
assert.deepEqual(timeline, ["probe", "probe", "plan", "plan", "teach", "teach"]);
|
||||
});
|
||||
|
||||
it("empty transcript → empty timeline", () => {
|
||||
assert.deepEqual(phaseTimeline([]), []);
|
||||
});
|
||||
test("phaseAt3: legacy transcripts without probe_done read as probe (no backwards compat — data wiped)", () => {
|
||||
const legacy = phaseAt3([user("teach me lacan"), assistantTool("plan_init", "output-available", PLAN_OK)]);
|
||||
assert.equal(legacy, "probe", "plan_init alone never flips a phase post-split");
|
||||
});
|
||||
|
||||
test("phaseTimeline: folds messages into phase spans for the dividers", () => {
|
||||
const timeline = phaseTimeline([
|
||||
user("teach me"),
|
||||
assistantTool("quiz", "output-available"),
|
||||
assistantTool("probe_done", "output-available", DONE_ASSESSMENT),
|
||||
assistantTool("plan_init", "output-available", PLAN_OK),
|
||||
user("go on"),
|
||||
]);
|
||||
assert.deepEqual(timeline satisfies Phase3[], ["probe", "probe", "plan", "teach", "teach"]);
|
||||
assert.equal(timeline.length, 5);
|
||||
});
|
||||
|
||||
test("phaseAt3: empty transcript is probe", () => {
|
||||
assert.equal(phaseAt3([]), "probe");
|
||||
});
|
||||
Reference in New Issue
Block a user