fix: deliver Claude teaching prose through an output tool

This commit is contained in:
amosblomqvist
2026-10-26 01:55:57 +02:00
parent fc3673466d
commit 87b04cc86e
8 changed files with 550 additions and 62 deletions
+13 -8
View File
@@ -28,7 +28,7 @@ import type { GraphStore } from "../../core/ports.ts";
import { emit } from "../../core/events.ts";
import { settleGraphLayout } from "../../core/layout-settle.ts";
import { batchRuns, coverageOf, messageViewLines } from "../../core/coverage.ts";
import type { LhMessage } from "../../core/native-messages.ts";
import { failedOutputCallIds, type LhMessage } from "../../core/native-messages.ts";
import { auditorGraphTools } from "../tools/graph.ts";
import { runSubmitAgent, healStringifiedArrays } from "./submit-agent.ts";
@@ -124,16 +124,20 @@ export function transcriptView(record: ChatRecord): string {
}
/** The extraction view under the teacher-prose ruling (owner, 2026-09-28):
* ONLY the teacher agent's text blocks — no learner messages, no
* thinking, no tool calls or results, no quiz blocks — each labeled with
* ONLY learner-facing teacher prose (text or send_to_user) — no learner
* messages, thinking, other tool calls/results, or quiz blocks — labeled with
* its ACTUAL message id (owner, 2026-10-13): the view lines ARE
* core/coverage.ts messageViewLines ("TEACHER [id] text"), the exact
* lines the coverage fingerprints hash, so the extractor can anchor
* every proposition to the passage it came from (the session submit's
* required messageId). The shared markdown engine stays reserved for the
* future user-facing export. */
function proseView(messages: ReadonlyArray<LhMessage>, title?: string): string {
const lines = messages.flatMap((message) => messageViewLines(message, message.id));
function proseView(
messages: ReadonlyArray<LhMessage>,
title?: string,
failedOutputCalls: ReadonlySet<string> = failedOutputCallIds(messages),
): string {
const lines = messages.flatMap((message) => messageViewLines(message, message.id, failedOutputCalls));
const head = title && title.length > 0 ? `# ${title}\n\n` : "";
return head + lines.join("\n\n");
}
@@ -1113,6 +1117,7 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
// stage runs (typed — the trigger skips, routes surface 503).
refuseIfStagesUnready();
const covered = coverageOf(record.messages, graphStore.readCoverage(), chatId);
const failedOutputCalls = failedOutputCallIds(record.messages);
if (covered.uncovered.length === 0) {
console.info(`[extraction ${chatId}] nothing uncovered — no-op`);
return { nodes: 0, chains: 0 };
@@ -1148,14 +1153,14 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
// fingerprints' messageViewLines ("TEACHER [id] text"), one
// shared seam.
const messages = batch.map((slot) => slot.message);
const view = proseView(messages, record.title);
const view = proseView(messages, record.title, failedOutputCalls);
// The batch's teacher-prose ids — the ONLY anchors this
// job's submit may name (validateSessionAnchors at the
// submit seam refuses anything else: ids from other
// batches — even real ones of this chat — other chats,
// and non-prose messages).
const batchIds = new Set(
messages.filter((m) => messageViewLines(m, m.id).length > 0).map((m) => m.id),
messages.filter((m) => messageViewLines(m, m.id, failedOutputCalls).length > 0).map((m) => m.id),
);
setPhase(jobKey, "queued", "chat", undefined, part);
const findings = await withExtractSlot(jobKey, async () => {
@@ -1188,7 +1193,7 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
const batchStats = mergeFindings(chatId, findings, auditResult, { chatStore, graphStore }, {
foldedAt,
coverage: batch
.filter((slot) => messageViewLines(slot.message, slot.message.id).length > 0)
.filter((slot) => slot.tokens > 0)
.map((slot) => ({ chatId, fp: slot.fp, tokens: slot.tokens, foldedAt })),
});
// The layout ledger evolves with the merge (sync — the
+26 -12
View File
@@ -15,10 +15,11 @@ import {
type TeacherToolContext,
} from "./tools.ts";
import { practiceTools, type PracticeToolContext } from "./tools/practice.ts";
import { healToolRecord } from "./tools/heal.ts";
import { healToolRecord, withArgumentHeal } from "./tools/heal.ts";
import { sendToUserTool } from "./tools/send-to-user.ts";
import { quizInputObject, quizInputProblems } from "../core/question-core.ts";
import { QUESTION_TOOLS, type EnabledTools } from "../core/pending.ts";
import { toLlmMessages, nativeInputCause } from "../core/native-messages.ts";
import { toLlmMessages, nativeInputCause, PRACTICE_PROBLEM_TOOL } from "../core/native-messages.ts";
import type { ChatGPTRequestCycle } from "./chatgpt-request-cycle.ts";
import { usageWasAdmitted } from "./subscription-request-cycle.ts";
@@ -145,22 +146,27 @@ export interface AgentBuildOptions {
* belt-and-suspenders HALF for the mixed batch (a question + other calls
* in one message) that terminate-only semantics would let run on — the
* learner cannot answer while a turn is in flight (per-chat 409), so a
* cleanly-posed question MUST always end the turn. maxTurns is gone
* cleanly-posed question MUST always end the turn. Practice's problem pose
* has the same wait-for-the-learner contract, including mixed output/problem
* batches; failed poses must still retry. maxTurns is gone
* (owner ruling 2026-09-25: no maximum-agent-turns net on the teaching
* turn).
*/
function makeQuestionStopper() {
function makeQuestionStopper(kind: AgentKind) {
return (context: {
message: { content: Array<{ type: string; id?: string; name?: string }> };
toolResults: Array<{ toolCallId: string; isError?: boolean }>;
}): boolean => {
const called = new Set(
context.message.content
.filter((c) => c.type === "toolCall" && c.name !== undefined && (QUESTION_TOOLS as readonly string[]).includes(c.name!))
.filter((c) => c.type === "toolCall" && c.name !== undefined && (
(QUESTION_TOOLS as readonly string[]).includes(c.name) ||
(kind === "practice" && c.name === PRACTICE_PROBLEM_TOOL)
))
.map((c) => c.id!),
);
if (called.size === 0) return false;
// Stop only when every question call of this turn resolved cleanly.
// Stop only when every question/problem pose resolved cleanly.
for (const result of context.toolResults) {
if (called.has(result.toolCallId) && result.isError) return false;
}
@@ -174,7 +180,7 @@ function baseAgentOptions(kind: AgentKind, opts: AgentBuildOptions, tools: Agent
// conversation-spec resolution as teaching); the cast keeps AgentKind
// ("practice" is not a ModelRole) honest about that.
const engine = opts.engine ?? piEngineFor(kind as "prober" | "teacher");
const stopper = makeQuestionStopper();
const stopper = makeQuestionStopper(kind);
const systemPrompt = opts.promptSwitches
? composeInstructions(kind, opts.promptSwitches)
: composedInstructions[kind]!;
@@ -190,7 +196,16 @@ function baseAgentOptions(kind: AgentKind, opts: AgentBuildOptions, tools: Agent
systemPrompt,
model: engine.model,
thinkingLevel: engine.level,
tools,
// Conditional registry inclusion, like the graph-tool gates: when
// closed, neither the tool nor its usage instructions reach the model.
// Match Claude across providers (including OpenRouter), not the API
// transport. Explicit test registries remain caller-owned.
tools: [
...tools,
...(opts.tools === undefined && engine.model.id.toLowerCase().includes("claude")
? [withArgumentHeal(sendToUserTool)]
: []),
],
messages: opts.messages ?? [],
},
streamFn: engine.streamFn,
@@ -312,10 +327,9 @@ export function bareQuestionTools(): Record<string, AgentTool<any>> {
* from turn 1 of a kind:"practice" session — no prober→teacher split,
* no plan tools, no question tools. Its ENTIRE registry is
* problem + conclude (tools/practice.ts); the doctrine prompt
* (prompts/practice/) carries the method. The question-stop rule does
* not apply (problem rides terminate: true natively —
* QUESTION_TOOLS does not include it), so baseAgentOptions's stopper is
* a no-op here; it stays wired for uniformity.
* (prompts/practice/) carries the method. A successful problem pose ends
* the run, natively for a problem-only batch and via the shared stop hook
* for a mixed batch (for example, send_to_user + problem).
*/
export function createPracticeAgent(ctx: PracticeToolContext, opts: AgentBuildOptions = {}) {
const tools = opts.tools ?? practiceTools(ctx);
+36
View File
@@ -0,0 +1,36 @@
import { Type } from "typebox";
import type { AgentTool } from "@earendil-works/pi-agent-core";
import { SEND_TO_USER_TOOL } from "../../core/native-messages.ts";
import { serverExecute } from "../tool-execute.ts";
const sendToUserInput = Type.Object({
message: Type.String({
minLength: 1,
pattern: "\\S",
description: "The complete content to display to the learner. Markdown and LaTeX are supported.",
}),
});
/** Claude's documented verbatim mid-turn output channel. Availability is
* model-gated at the shared conversation-agent construction boundary;
* the native call remains in history, and the view renders its message. */
export const sendToUserTool: AgentTool<typeof sendToUserInput> = {
name: SEND_TO_USER_TOOL,
label: SEND_TO_USER_TOOL,
description:
"## REQUIRED OUTPUT CHANNEL — ALWAYS USE `send_to_user`\n\n" +
"**ALL learner-facing messages MUST be sent using `send_to_user`.** This includes explanations, teaching, " +
"feedback, corrections, hints, introductions, progress messages, and final responses. " +
"There is no exception for short messages or the end of a turn.\n\n" +
"Ordinary response text may be converted into summarized thinking instead of reaching the learner. " +
"**Never rely on ordinary response text to communicate with the learner. Thinking is not delivery.**\n\n" +
"Put the complete message in `message`. Markdown and LaTeX are supported. " +
"Do not duplicate it in ordinary response text.\n\n" +
"Question content still belongs in the appropriate question tool; send any accompanying teaching or " +
"commentary through `send_to_user` before calling that tool.\n\n" +
"This tool does not pause or end your turn. Continue with other tools when needed. " +
"After delivering your final message, end the turn without additional ordinary response text.",
parameters: sendToUserInput,
executionMode: "sequential",
execute: serverExecute(SEND_TO_USER_TOOL, sendToUserInput, () => "Message received for display."),
};
+25 -27
View File
@@ -1,9 +1,9 @@
/**
* Extraction coverage — the pure core. THE TEACHER'S PROSE IS THE UNIT
* (owner ruling, 2026-09-28): the extractor sees ONLY the teacher agent's
* text blocks — no learner messages, no thinking, no tool calls or
* results, no quiz blocks. Everything else is INVISIBLE to coverage: it
* neither opens nor joins a run, so answering a quiz or editing a learner
* prose — ordinary text blocks and explicit send_to_user messages. No
* learner messages, thinking, other tool calls/results, or quiz blocks.
* Everything else is INVISIBLE to coverage: it neither opens nor joins a run, so answering a quiz or editing a learner
* message triggers nothing. "Covered" is by construction exactly "what
* the extractor saw": the fingerprint (sha256 of a message's view lines)
* and the extractor's prompt share ONE selection — the teacher prose.
@@ -22,33 +22,27 @@
import { createHash } from "node:crypto";
import type { CoverageEntry } from "./contracts/graph.ts";
import type { LhMessage } from "./native-messages.ts";
import { failedOutputCallIds, userFacingTextOf, type LhMessage } from "./native-messages.ts";
export type { CoverageEntry };
/** The user-facing text of a native message (string or text blocks). */
function textOf(content: unknown): string[] {
if (typeof content === "string") return content.trim() ? [content] : [];
if (!Array.isArray(content)) return [];
return content
.filter((c): c is { type: "text"; text: string } => (c as { type?: string }).type === "text")
.map((c) => c.text)
.filter((t) => t.trim().length > 0);
}
/**
* The extraction-relevant lines of ONE native message — the shared seam
* between the extractor's prompt and messageFingerprint (coverage).
* TEACHER PROSE ONLY (owner, 2026-09-28): an assistant message yields one
* line per text block; everything else (user messages, tool results,
* markers) yields NOTHING and is invisible to coverage. `mid` is the
* message id — part of the hash input, so the same prose in two messages
* line per learner-facing prose block; everything else (user messages,
* tool results, markers) yields NOTHING and is invisible to coverage.
* `mid` is the message id — part of the hash input, so the same prose in two messages
* is two slots (per-message coverage).
*/
export function messageViewLines(message: LhMessage, mid: string): string[] {
export function messageViewLines(
message: LhMessage,
mid: string,
failedOutputCalls: ReadonlySet<string> = failedOutputCallIds([message]),
): string[] {
const lines: string[] = [];
if ((message as { role?: string }).role !== "assistant") return lines;
for (const text of textOf((message as { content: unknown }).content)) {
for (const text of userFacingTextOf((message as { content: unknown }).content, failedOutputCalls)) {
lines.push(`TEACHER [${mid}] ${text.trim()}`);
}
return lines;
@@ -62,9 +56,10 @@ export function messageViewLines(message: LhMessage, mid: string): string[] {
*/
export function teacherProseBlocks(messages: ReadonlyArray<LhMessage>): string[] {
const out: string[] = [];
const failedOutputCalls = failedOutputCallIds(messages);
for (const message of messages) {
if ((message as { role?: string }).role !== "assistant") continue;
for (const text of textOf((message as { content: unknown }).content)) out.push(text.trim());
for (const text of userFacingTextOf((message as { content: unknown }).content, failedOutputCalls)) out.push(text.trim());
}
return out;
}
@@ -79,9 +74,9 @@ export function viewTokens(lines: string[]): number {
* teacher prose, with the message id as the tag). Changes exactly when
* the teacher prose of the message changes — nothing else can re-open it.
*/
export function messageFingerprint(message: LhMessage): string {
export function messageFingerprint(message: LhMessage, failedOutputCalls?: ReadonlySet<string>): string {
const h = createHash("sha256");
h.update(JSON.stringify([(message as { role?: string }).role, messageViewLines(message, message.id)]));
h.update(JSON.stringify([(message as { role?: string }).role, messageViewLines(message, message.id, failedOutputCalls)]));
return h.digest("hex");
}
@@ -95,7 +90,12 @@ export interface CoverageSlot {
/** Fingerprint a whole transcript, in order (used by the trigger, the run,
* and the coverage view — one derivation everywhere). */
export function coverageSlots(messages: ReadonlyArray<LhMessage>): CoverageSlot[] {
return messages.map((message) => ({ message, fp: messageFingerprint(message), tokens: viewTokens(messageViewLines(message, message.id)) }));
const failedOutputCalls = failedOutputCallIds(messages);
return messages.map((message) => ({
message,
fp: messageFingerprint(message, failedOutputCalls),
tokens: viewTokens(messageViewLines(message, message.id, failedOutputCalls)),
}));
}
/** The covered-fingerprint set for one chat, from the ledger. */
@@ -131,16 +131,14 @@ export function coverageOf(
let uncoveredTokens = 0;
for (let i = 0; i < slots.length; i++) {
const slot = slots[i]!;
if (messageViewLines(slot.message, slot.message.id).length === 0) continue; // invisible
if (slot.tokens === 0) continue; // invisible (including failed output calls)
if (covered.has(slot.fp)) coveredTokens += slot.tokens;
else {
uncovered.push(i);
uncoveredTokens += slot.tokens;
}
}
const visibleCount = slots.length - slots.filter(
(s) => messageViewLines(s.message, s.message.id).length === 0,
).length;
const visibleCount = slots.filter((s) => s.tokens > 0).length;
return {
slots,
uncovered,
+49
View File
@@ -98,6 +98,55 @@ export function asAgentMessages(messages: readonly LhMessage[]): AgentMessage[]
return messages as unknown as AgentMessage[];
}
// ─── explicit learner-facing output ─────────────────────────────────────────
export const SEND_TO_USER_TOOL = "send_to_user";
/** Read the output tool's message without changing the native call. Also
* accepts partial arguments while streaming; malformed/empty inputs carry
* no prose. This projection is independent of the currently selected model. */
export function sendToUserText(block: unknown): string | undefined {
if (!block || typeof block !== "object") return undefined;
const call = block as { type?: unknown; name?: unknown; arguments?: unknown };
if (call.type !== "toolCall" || call.name !== SEND_TO_USER_TOOL) return undefined;
if (!call.arguments || typeof call.arguments !== "object") return undefined;
const message = (call.arguments as { message?: unknown }).message;
return typeof message === "string" && message.trim() ? message : undefined;
}
/** Learner-facing prose, in content order. Only the explicit output tool
* joins ordinary text here — thinking and all other tool inputs stay out.
* Shared by coverage/extraction and Markdown; native history stays intact. */
export function userFacingTextOf(content: unknown, failedOutputCalls?: ReadonlySet<string>): string[] {
if (typeof content === "string") return content.trim() ? [content] : [];
if (!Array.isArray(content)) return [];
const out: string[] = [];
for (const block of content) {
if (!block || typeof block !== "object") continue;
if (block.type === "toolCall" && failedOutputCalls?.has(block.id)) continue;
const text = block.type === "text" ? block.text : sendToUserText(block);
if (typeof text === "string" && text.trim()) out.push(text);
}
return out;
}
/** Failed output calls remain retry context, not delivered teaching.
* Truncated calls can have valid-looking partial arguments. Failed/aborted
* generations do not execute their calls at all, so may have no result. */
export function failedOutputCallIds(messages: readonly LhMessage[]): ReadonlySet<string> {
const failed = new Set<string>();
for (const message of messages) {
if (message.role === "toolResult" && message.toolName === SEND_TO_USER_TOOL && message.isError) {
failed.add(message.toolCallId);
} else if (message.role === "assistant" && (message.stopReason === "error" || message.stopReason === "aborted")) {
for (const block of message.content) {
if (block.type === "toolCall" && block.name === SEND_TO_USER_TOOL) failed.add(block.id);
}
}
}
return failed;
}
// ─── id minting ──────────────────────────────────────────────────────────────
/** The app-wide message id shape (same as the UIMessage-era mint). */
+8 -14
View File
@@ -22,22 +22,15 @@
import type { LhMessage } from "./native-messages.ts";
import {
failedOutputCallIds,
PRACTICE_CONCLUDE_TOOL,
PRACTICE_PROBLEM_TOOL,
problemInputForConclude,
SEND_TO_USER_TOOL,
userFacingTextOf,
} from "./native-messages.ts";
import type { ToolResultMessage } from "@earendil-works/pi-ai";
/** The user-facing text blocks of a native message (string or text parts). */
function textOf(content: unknown): string[] {
if (typeof content === "string") return content.trim() ? [content] : [];
if (!Array.isArray(content)) return [];
return content
.filter((c): c is { type: "text"; text: string } => (c as { type?: string }).type === "text")
.map((c) => c.text)
.filter((t) => t.trim().length > 0);
}
/** The practice join context (owner-approved, 2026-10-06 export round):
* practice tool RESULTS are acks — the statement, outcome, and verdict
* all live in the assistant's call arguments, and the model solution
@@ -48,6 +41,7 @@ export interface MarkdownContext {
messages: ReadonlyArray<LhMessage>;
/** toolCallId → the call part's name + arguments. */
calls: Map<string, { name: string; args: Record<string, unknown> | undefined }>;
failedOutputCalls?: ReadonlySet<string>;
}
function markdownContext(messages: ReadonlyArray<LhMessage>): MarkdownContext {
@@ -66,7 +60,7 @@ function markdownContext(messages: ReadonlyArray<LhMessage>): MarkdownContext {
}
}
}
return { messages, calls };
return { messages, calls, failedOutputCalls: failedOutputCallIds(messages) };
}
/** solvedWithHints → "Solved with hints" — the outcome enum reads as prose. */
@@ -172,16 +166,16 @@ export function messageMarkdown(
const parts: string[] = [];
if (role === "user") {
for (const text of textOf((message as { content: unknown }).content)) {
for (const text of userFacingTextOf((message as { content: unknown }).content)) {
parts.push(`### Learner${anchor}\n\n${text.trim()}`);
}
} else if (role === "assistant") {
for (const text of textOf((message as { content: unknown }).content)) {
for (const text of userFacingTextOf((message as { content: unknown }).content, ctx?.failedOutputCalls ?? failedOutputCallIds([message]))) {
parts.push(`### Teacher${anchor}\n\n${text.trim()}`);
}
} else if (role === "toolResult") {
const result = message as ToolResultMessage;
if (result.isError) return null; // retry context, not learning evidence
if (result.isError || result.toolName === SEND_TO_USER_TOOL) return null; // retry context or output ack, not learning evidence
const { heading, body } = toolResultBody(result, ctx);
parts.push(`### ${heading}${anchor}\n\n${body.join("\n\n")}`);
}
+27 -1
View File
@@ -34,7 +34,7 @@
import type { AssistantMessage, ToolResultMessage } from "@earendil-works/pi-ai";
import type { LhMarkerMessage, LhMessage } from "./native-messages.ts";
import { modelSolutionForConclude, PRACTICE_CONCLUDE_TOOL, problemInputForConclude } from "./native-messages.ts";
import { modelSolutionForConclude, PRACTICE_CONCLUDE_TOOL, problemInputForConclude, SEND_TO_USER_TOOL, sendToUserText } from "./native-messages.ts";
import { activeCompactionOf, type CompactionDetails } from "./compaction.ts";
import { QUESTION_TOOLS } from "./pending.ts";
import type { SubagentArtifactData, SubagentResultMeta } from "./contracts/jobs.ts";
@@ -124,6 +124,14 @@ function lastSuccessfulPlanInit(parts: LhViewPart[]): number {
* view messages. Pure: the same input always projects to the same list. */
export function projectView(messages: readonly LhMessage[], live?: LiveTurnState): LhViewMessage[] {
const out: LhViewMessage[] = [];
// Output calls become text rather than tool parts, so read settled
// failures up front: a failed call must retain the normal error path.
const outputErrors = new Map<string, ToolResultMessage>();
for (const message of messages) {
if (message.role === "toolResult" && message.toolName === SEND_TO_USER_TOOL && message.isError) {
outputErrors.set(message.toolCallId, message);
}
}
const lastAssistant = (): LhViewMessage | undefined => {
const last = out[out.length - 1];
@@ -218,6 +226,24 @@ export function projectView(messages: readonly LhMessage[], live?: LiveTurnState
reasoning.push({ type: "reasoning", id: `${m.id}-r${reasoningN++}`, text: block.thinking, state: "done" });
} else if (block.type === "toolCall") {
const part = toolPartOf(block, live, streamingMessage);
if (block.name === SEND_TO_USER_TOOL) {
// Failed/aborted generations never execute their calls;
// the provider error/abort UI already owns that outcome.
if (assistant.stopReason === "error" || assistant.stopReason === "aborted") continue;
const error = outputErrors.get(block.id);
if (error) applyResult(part, error);
if (part.state === "output-error") {
rest.push(part);
} else {
const text = sendToUserText(block);
if (text !== undefined) {
rest.push({ type: "text", id: `${m.id}-t${textN++}`, text, state: "done" });
}
}
// No live tool card or acknowledgement: this channel is
// ordinary prose, including while its arguments stream.
continue;
}
// The conclusion's reveal join (owner round, 2026-10-05): the
// hero of a conclude card is the modelSolution committed at POSE
// time — a record fold, enriched here so the card renders pure.