v0 scaffold: teacher runtime (hermetic pi SDK), data model, ledgers, normalizer
- teacher/system-prompt.md: teach methodology + quiz/ask guidelines folded in - extensions/: quiz.ts + ask-user-question.ts (hermetic copies) - src/teacher/create-session.ts: isolated agent (noTools builtin, own agentDir/sessions/prompt) - src/schema.ts + ledger.ts: append-only ledgers, known-vs-understood data model - src/extract/normalize.ts: session JSONL -> canonical doc (verified on real session) - extractor/merger/derive/what-next: specified stubs
This commit is contained in:
Amos
+78
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* learning-harness CLI
|
||||
*
|
||||
* teach [message] launch an interactive teaching session (full pi TUI)
|
||||
* doctor non-interactive smoke test of the teacher wiring
|
||||
* ingest <file> normalize a session JSONL and print the canonical document
|
||||
* (Stage 1 only — extractor/merger not yet implemented)
|
||||
* graph print graph/ledger stats
|
||||
* what-next (not yet implemented)
|
||||
*/
|
||||
|
||||
import * as fs from "node:fs";
|
||||
import { doctor, runTeachingSession } from "./teacher/create-session.ts";
|
||||
import { normalizeSession } from "./extract/normalize.ts";
|
||||
import { ledgers } from "./ledger.ts";
|
||||
import { deriveAll } from "./derive.ts";
|
||||
import { PATHS } from "../config.ts";
|
||||
|
||||
async function main() {
|
||||
const [command, ...rest] = process.argv.slice(2);
|
||||
|
||||
switch (command) {
|
||||
case "teach": {
|
||||
await runTeachingSession(rest.join(" ") || undefined);
|
||||
break;
|
||||
}
|
||||
case "doctor": {
|
||||
await doctor();
|
||||
break;
|
||||
}
|
||||
case "ingest": {
|
||||
const file = rest[0];
|
||||
if (!file) {
|
||||
console.error("Usage: ingest <session.jsonl>");
|
||||
process.exit(1);
|
||||
}
|
||||
const doc = normalizeSession(file);
|
||||
ledgers.appendSession(doc.session);
|
||||
console.log(`Session: ${doc.session.id}`);
|
||||
console.log(`Topic: ${doc.session.topic}`);
|
||||
console.log(`Entries: ${doc.entries.length}`);
|
||||
const quizzes = doc.entries.filter((e) => e.kind === "quiz");
|
||||
console.log(`Quizzes: ${quizzes.length}`);
|
||||
// Full canonical document for inspection / later pipeline stages.
|
||||
fs.mkdirSync(PATHS.graph, { recursive: true });
|
||||
const out = `${PATHS.graph}/canonical-${doc.session.id.slice(0, 8)}.json`;
|
||||
fs.writeFileSync(out, JSON.stringify(doc, null, "\t"), "utf-8");
|
||||
console.log(`Canonical document written: ${out}`);
|
||||
break;
|
||||
}
|
||||
case "graph": {
|
||||
const nodes = ledgers.nodes();
|
||||
const edges = ledgers.edges();
|
||||
const events = ledgers.events();
|
||||
const sessions = ledgers.sessions();
|
||||
const derived = deriveAll();
|
||||
console.log(`Sessions: ${sessions.length}`);
|
||||
console.log(`Nodes: ${nodes.length} (known: ${derived.filter((d) => d.known).length})`);
|
||||
console.log(`Edges: ${edges.length}`);
|
||||
console.log(`Events: ${events.length}`);
|
||||
break;
|
||||
}
|
||||
case "what-next": {
|
||||
console.error("Not implemented yet — needs extractor + merger first.");
|
||||
process.exit(1);
|
||||
}
|
||||
default: {
|
||||
console.log("Commands: teach [message] | doctor | ingest <file> | graph | what-next");
|
||||
process.exit(command ? 1 : 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,37 @@
|
||||
/**
|
||||
* Derived state — pure code, no LLM.
|
||||
* known (evidence) / understood (structure, aggregation semantics PARKED).
|
||||
* NOT YET IMPLEMENTED beyond the known-set.
|
||||
*/
|
||||
|
||||
import { ledgers } from "./ledger.ts";
|
||||
import { KNOWING_ORDER, type DerivedNodeState, type KnowingLevel } from "./schema.ts";
|
||||
|
||||
/** Rubric → scalar. Deterministic code, never an LLM. */
|
||||
export function knowingScore(level: KnowingLevel): number {
|
||||
return KNOWING_ORDER.indexOf(level) / (KNOWING_ORDER.length - 1);
|
||||
}
|
||||
|
||||
/** Known set: nodes with direct evidence (taught, probe-correct, check-correct...). */
|
||||
export function knownNodeIds(): Set<string> {
|
||||
const evidenced = new Set<string>();
|
||||
for (const event of ledgers.events()) {
|
||||
if (event.kind === "taught" || event.kind === "check-correct" || event.kind === "probe-correct" || event.kind === "recall-at-delay") {
|
||||
evidenced.add(event.node);
|
||||
}
|
||||
}
|
||||
return evidenced;
|
||||
}
|
||||
|
||||
/** Full derived state. Understanding aggregation (weakest-link vs mean vs two-level)
|
||||
* is PARKED pending real extracted data — see docs/architecture.md §7. */
|
||||
export function deriveAll(): DerivedNodeState[] {
|
||||
const known = knownNodeIds();
|
||||
return ledgers.nodes().map((node) => ({
|
||||
node: node.id,
|
||||
known: known.has(node.id),
|
||||
knowing: 0,
|
||||
understood: 0,
|
||||
floating: false,
|
||||
}));
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
/**
|
||||
* Stage 2 — Extract (one LLM pass over the canonical session document).
|
||||
* NOT YET IMPLEMENTED. Interface specified so the shape is fixed:
|
||||
*
|
||||
* Input: CanonicalSessionDocument
|
||||
* Output: ExtractedSession (see schema.ts) — nodes, edges, attributed events,
|
||||
* per-node knowing on the anchored ordinal rubric.
|
||||
*
|
||||
* Constraints:
|
||||
* - One pass for nodes+edges+events+knowing (they are entangled).
|
||||
* - Zero inference about knowledge beyond the evidence: it extracts facts,
|
||||
* edges, and evidence — nothing the session doesn't show.
|
||||
* - Knowing is mapped onto the anchored rubric (KnowingLevel), never a scalar.
|
||||
* - Edges come primarily from the plan's mermaid DAG (already asserted by the
|
||||
* teacher); only prose-articulated Connect-step edges need judgment.
|
||||
*
|
||||
* Followed by an audit pass (extract/audit.ts): a second LLM call verifying the
|
||||
* extraction JSON against the session for hallucination/misattribution.
|
||||
*/
|
||||
|
||||
import type { CanonicalSessionDocument } from "./normalize.ts";
|
||||
import type { ExtractedSession } from "../schema.ts";
|
||||
|
||||
export async function extractSession(doc: CanonicalSessionDocument): Promise<ExtractedSession> {
|
||||
throw new Error("Not implemented — Stage 2 (extractor LLM pass) is the next build step.");
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
/**
|
||||
* Stage 3 — Merge (deterministic code + LLM adjudication for node identity).
|
||||
* NOT YET IMPLEMENTED. Interface specified:
|
||||
*
|
||||
* Input: ExtractedSession (Stage 2 output) + canonical graph
|
||||
* 1. Embed canonical node statements (cached) → kNN candidates per incoming node.
|
||||
* 2. ONE LLM adjudication call per session: each pair →
|
||||
* same → alias + attach events to canonical node
|
||||
* related → new node + proposed cross-session edge (the quiet superpower)
|
||||
* unrelated → new node, no edge
|
||||
* 3. Apply deterministically; log every decision as a SessionMapping
|
||||
* (reversible by design — wrong merges must be one-line fixes).
|
||||
*
|
||||
* Policy: fully automatic, conservative — prefer duplicates over wrong merges.
|
||||
* No domain filter on candidates (cross-domain merging is the point).
|
||||
*/
|
||||
|
||||
import type { ExtractedSession } from "../schema.ts";
|
||||
|
||||
export async function mergeSession(extraction: ExtractedSession): Promise<void> {
|
||||
throw new Error("Not implemented — Stage 3 (merger) comes after the extractor.");
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
/**
|
||||
* Stage 1 of the extraction pipeline — DETERMINISTIC, no LLM.
|
||||
*
|
||||
* pi session JSONL → canonical session document.
|
||||
*
|
||||
* The quiz tool writes full structured details into the session JSONL
|
||||
* (question, options, correct/dontKnow, note, explanation), so quiz events are
|
||||
* read as gold-standard structured data, not parsed from prose.
|
||||
*/
|
||||
|
||||
import * as fs from "node:fs";
|
||||
import type {
|
||||
EventKind,
|
||||
Provenance,
|
||||
SessionRecord,
|
||||
} from "../schema.ts";
|
||||
|
||||
// ─── pi session JSONL entry types (subset we consume) ───────────────────────
|
||||
|
||||
interface PiEntryBase {
|
||||
type: string;
|
||||
id: string;
|
||||
parentId: string | null;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
interface PiSessionHeader {
|
||||
type: "session";
|
||||
version: number;
|
||||
id: string;
|
||||
timestamp: string;
|
||||
cwd: string;
|
||||
}
|
||||
|
||||
interface PiMessageEntry extends PiEntryBase {
|
||||
type: "message";
|
||||
message: {
|
||||
role: "user" | "assistant" | "toolResult" | string;
|
||||
content?: unknown;
|
||||
toolCallId?: string;
|
||||
toolName?: string;
|
||||
details?: unknown;
|
||||
timestamp?: number;
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Canonical session document ──────────────────────────────────────────────
|
||||
|
||||
export interface CanonicalQuizEvent {
|
||||
kind: "quiz";
|
||||
entryId: string;
|
||||
toolCallEntryId: string; // the assistant entry containing the toolCall
|
||||
question: string;
|
||||
context?: string;
|
||||
options: Array<{ index: number; label: string }>;
|
||||
answers: Array<{ index: number; label: string }>;
|
||||
correctIndices: number[];
|
||||
correct: boolean;
|
||||
dontKnow: boolean;
|
||||
note?: string;
|
||||
explanation?: string;
|
||||
}
|
||||
|
||||
export interface CanonicalAskEvent {
|
||||
kind: "ask";
|
||||
entryId: string;
|
||||
toolCallEntryId: string;
|
||||
question: string;
|
||||
answers: Array<{ label: string; value: string }>;
|
||||
}
|
||||
|
||||
export interface CanonicalTextEvent {
|
||||
kind: "user" | "assistant";
|
||||
entryId: string;
|
||||
text: string;
|
||||
}
|
||||
|
||||
export type CanonicalEntry = CanonicalQuizEvent | CanonicalAskEvent | CanonicalTextEvent;
|
||||
|
||||
export interface CanonicalSessionDocument {
|
||||
session: SessionRecord;
|
||||
entries: CanonicalEntry[];
|
||||
}
|
||||
|
||||
// ─── Parsing ─────────────────────────────────────────────────────────────────
|
||||
|
||||
function textFromContent(content: unknown): string {
|
||||
if (typeof content === "string") return content;
|
||||
if (!Array.isArray(content)) return "";
|
||||
return content
|
||||
.filter((b): b is { type: "text"; text: string } => typeof b === "object" && b !== null && (b as { type: string }).type === "text")
|
||||
.map((b) => b.text)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
interface QuizDetails {
|
||||
status: string;
|
||||
question: string;
|
||||
mode: string;
|
||||
answers: Array<{ index: number; label: string }>;
|
||||
correctIndices: number[];
|
||||
options?: Array<{ index: number; label: string }>;
|
||||
correct?: boolean;
|
||||
dontKnow?: boolean;
|
||||
note?: string;
|
||||
explanation?: string;
|
||||
}
|
||||
|
||||
interface AskDetails {
|
||||
status: string;
|
||||
question: string;
|
||||
mode: string;
|
||||
answers?: Array<{ label: string; value: string }>;
|
||||
}
|
||||
|
||||
export function normalizeSession(file: string): CanonicalSessionDocument {
|
||||
const lines = fs.readFileSync(file, "utf-8").split("\n").filter((l) => l.trim().length > 0);
|
||||
const parsed = lines.map((l) => JSON.parse(l));
|
||||
|
||||
const header = parsed[0] as PiSessionHeader;
|
||||
if (header?.type !== "session") throw new Error(`${file}: not a pi session file (missing header)`);
|
||||
|
||||
const entries: CanonicalEntry[] = [];
|
||||
const quizArgsByCallId = new Map<string, { entryId: string; question: string; context?: string }>();
|
||||
|
||||
for (const raw of parsed.slice(1)) {
|
||||
const entry = raw as PiMessageEntry;
|
||||
if (entry.type !== "message") continue;
|
||||
const msg = entry.message;
|
||||
|
||||
if (msg.role === "user") {
|
||||
const text = textFromContent(msg.content).trim();
|
||||
if (text) entries.push({ kind: "user", entryId: entry.id, text });
|
||||
}
|
||||
|
||||
if (msg.role === "assistant" && Array.isArray(msg.content)) {
|
||||
// Collect assistant text (lesson prose — node articulations live here).
|
||||
const text = textFromContent(msg.content).trim();
|
||||
if (text) entries.push({ kind: "assistant", entryId: entry.id, text });
|
||||
// Remember quiz/ask toolCall args (the question) keyed by call id.
|
||||
for (const block of msg.content) {
|
||||
if (typeof block === "object" && block !== null && (block as { type: string }).type === "toolCall") {
|
||||
const call = block as { type: "toolCall"; id: string; name: string; arguments: Record<string, unknown> };
|
||||
if (call.name === "quiz" || call.name === "ask_user_question") {
|
||||
quizArgsByCallId.set(call.id, {
|
||||
entryId: entry.id,
|
||||
question: String(call.arguments.question ?? ""),
|
||||
context: call.arguments.details ? String(call.arguments.details) : undefined,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (msg.role === "toolResult" && (msg.toolName === "quiz" || msg.toolName === "ask_user_question")) {
|
||||
const args = quizArgsByCallId.get(msg.toolCallId ?? "");
|
||||
if (msg.toolName === "quiz") {
|
||||
const d = msg.details as QuizDetails | undefined;
|
||||
if (d && d.status === "answered") {
|
||||
entries.push({
|
||||
kind: "quiz",
|
||||
entryId: entry.id,
|
||||
toolCallEntryId: args?.entryId ?? entry.id,
|
||||
question: d.question ?? args?.question ?? "",
|
||||
context: args?.context,
|
||||
options: d.options ?? [],
|
||||
answers: d.answers ?? [],
|
||||
correctIndices: d.correctIndices ?? [],
|
||||
correct: d.correct ?? false,
|
||||
dontKnow: d.dontKnow ?? false,
|
||||
note: d.note,
|
||||
explanation: d.explanation,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
const d = msg.details as AskDetails | undefined;
|
||||
if (d && d.status === "answered") {
|
||||
entries.push({
|
||||
kind: "ask",
|
||||
entryId: entry.id,
|
||||
toolCallEntryId: args?.entryId ?? entry.id,
|
||||
question: d.question ?? args?.question ?? "",
|
||||
answers: d.answers ?? [],
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const firstUser = entries.find((e): e is CanonicalTextEvent => e.kind === "user");
|
||||
|
||||
return {
|
||||
session: {
|
||||
id: header.id,
|
||||
startedAt: header.timestamp,
|
||||
topic: firstUser ? firstUser.text.slice(0, 120) : "(no user message)",
|
||||
file,
|
||||
},
|
||||
entries,
|
||||
};
|
||||
}
|
||||
|
||||
/** Coarse pre-plan / post-plan classification for quiz events (refined by the extractor LLM). */
|
||||
export function quizPhase(index: number, planIndex: number): "probe" | "check" {
|
||||
return index < planIndex ? "probe" : "check";
|
||||
}
|
||||
|
||||
/** Provisional event kind from a quiz outcome + phase (the extractor refines/attribution). */
|
||||
export function provisionalEventKind(quiz: CanonicalQuizEvent, phase: "probe" | "check"): EventKind {
|
||||
if (quiz.dontKnow) return "dontKnow";
|
||||
if (phase === "probe") return quiz.correct ? "probe-correct" : "probe-miss";
|
||||
return quiz.correct ? "check-correct" : "check-incorrect";
|
||||
}
|
||||
|
||||
export function provenanceOf(sessionId: string, entryId: string): Provenance {
|
||||
return { session: sessionId, entryId };
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* Append-only JSONL ledgers. Same philosophy as the design: the ledgers are
|
||||
* the source of truth; everything else is derived and recomputable.
|
||||
*/
|
||||
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { LEDGER_FILES } from "../config.ts";
|
||||
|
||||
export function newId(prefix: string): string {
|
||||
return `${prefix}_${randomBytes(5).toString("hex")}`;
|
||||
}
|
||||
|
||||
export function append<T>(file: string, record: T): void {
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
fs.appendFileSync(file, JSON.stringify(record) + "\n", "utf-8");
|
||||
}
|
||||
|
||||
export function readAll<T>(file: string): T[] {
|
||||
if (!fs.existsSync(file)) return [];
|
||||
const out: T[] = [];
|
||||
for (const line of fs.readFileSync(file, "utf-8").split("\n")) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.length > 0) out.push(JSON.parse(trimmed) as T);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Typed helpers ───────────────────────────────────────────────────────────────
|
||||
|
||||
import type {
|
||||
GraphEdge,
|
||||
GraphNode,
|
||||
KnowledgeEvent,
|
||||
SessionMapping,
|
||||
SessionRecord,
|
||||
} from "./schema.ts";
|
||||
|
||||
export const ledgers = {
|
||||
appendSession(record: SessionRecord) {
|
||||
// Session records are unique per id — skip if already ingested.
|
||||
if (readAll<SessionRecord>(LEDGER_FILES.sessions).some((s) => s.id === record.id)) return false;
|
||||
append(LEDGER_FILES.sessions, record);
|
||||
return true;
|
||||
},
|
||||
sessions: () => readAll<SessionRecord>(LEDGER_FILES.sessions),
|
||||
|
||||
appendNode(node: GraphNode) {
|
||||
append(LEDGER_FILES.nodes, node);
|
||||
},
|
||||
nodes: () => readAll<GraphNode>(LEDGER_FILES.nodes),
|
||||
|
||||
appendEdge(edge: GraphEdge) {
|
||||
append(LEDGER_FILES.edges, edge);
|
||||
},
|
||||
edges: () => readAll<GraphEdge>(LEDGER_FILES.edges),
|
||||
|
||||
appendEvent(event: KnowledgeEvent) {
|
||||
append(LEDGER_FILES.events, event);
|
||||
},
|
||||
events: () => readAll<KnowledgeEvent>(LEDGER_FILES.events),
|
||||
|
||||
appendMapping(mapping: SessionMapping) {
|
||||
append(LEDGER_FILES.mappings, mapping);
|
||||
},
|
||||
mappings: () => readAll<SessionMapping>(LEDGER_FILES.mappings),
|
||||
};
|
||||
@@ -0,0 +1,18 @@
|
||||
/**
|
||||
* What-next queries: territory / repairs / conversions (+ unlock-value over
|
||||
* the expanded frontier). NOT YET IMPLEMENTED — needs the graph populated first.
|
||||
*
|
||||
* These run as pure traversals over the derived graph:
|
||||
* territory — frontier candidates ranked by work-to-ready (missing
|
||||
* requires-subtree mass), NOT fewest edges.
|
||||
* repairs — weak/shaky/misconceived nodes INSIDE the held region,
|
||||
* ranked by downstream load.
|
||||
* conversions — missing requires-roots that would flip known-but-floating
|
||||
* nodes into understood.
|
||||
* unlock-value — over the lazily expanded frontier, discounted by depth in
|
||||
* generated territory; presented as ranked heuristic, never a number.
|
||||
*/
|
||||
|
||||
export async function whatNext(): Promise<void> {
|
||||
throw new Error("Not implemented — needs a populated graph (extractor + merger first).");
|
||||
}
|
||||
+170
@@ -0,0 +1,170 @@
|
||||
/**
|
||||
* Knowledge graph data model.
|
||||
*
|
||||
* Design invariants (see docs/architecture.md and the design session):
|
||||
* 1. Store the LEDGERS, derive the GRAPH. Events/edges/mappings are append-only
|
||||
* evidence; node states and strengths are always recomputable from them.
|
||||
* 2. Nodes are propositions at ~one-reasoning-step grain.
|
||||
* 3. Edges carry NO scalars — only type (requires/supports) and provenance.
|
||||
* All mass lives in nodes. "Load-bearing" is derived, never stored.
|
||||
* 4. known ≠ understood. Known = evidence property (from the event ledger).
|
||||
* Understood = derived structural property (known + requires-subtree held).
|
||||
* 5. Candidates (frontier-generated nodes) are a separate state class;
|
||||
* promotion only happens when a session touches them or the learner
|
||||
* picks them as a goal — never automatically.
|
||||
* 6. Every event carries session + entryId → jump-to-point provenance.
|
||||
*/
|
||||
|
||||
// ─── Sessions ────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface SessionRecord {
|
||||
/** pi session UUID (from the session JSONL header). */
|
||||
id: string;
|
||||
/** ISO timestamp from the header. */
|
||||
startedAt: string;
|
||||
/** Human topic label (first user message, or /name). */
|
||||
topic: string;
|
||||
/** Absolute path of the session JSONL. */
|
||||
file: string;
|
||||
}
|
||||
|
||||
// ─── Nodes ───────────────────────────────────────────────────────────────────
|
||||
|
||||
export type NodeKind = "truth" | "misconception";
|
||||
export type NodeState = "known" | "not-held" | "candidate";
|
||||
|
||||
export interface GraphNode {
|
||||
/** Opaque stable id, e.g. "n_a1b2c3". */
|
||||
id: string;
|
||||
/** The proposition, one reasoning step of grain. */
|
||||
statement: string;
|
||||
kind: NodeKind;
|
||||
/** Domain tag — metadata for visualization, NEVER a merge barrier. */
|
||||
domain: string;
|
||||
/** candidate = generated by frontier expansion, not yet evidenced. */
|
||||
state: NodeState;
|
||||
createdAt: number;
|
||||
}
|
||||
|
||||
// ─── Edges ───────────────────────────────────────────────────────────────────
|
||||
|
||||
export type EdgeType = "requires" | "supports";
|
||||
|
||||
export interface GraphEdge {
|
||||
id: string;
|
||||
/** The dependency (e.g. "gradient" for gradient descent). */
|
||||
from: string;
|
||||
/** The dependent (e.g. "gradient descent"). */
|
||||
to: string;
|
||||
/**
|
||||
* requires — hard: understanding the target requires holding this.
|
||||
* supports — soft: the target is more anchored with it, holdable without it.
|
||||
*/
|
||||
type: EdgeType;
|
||||
provenance: Provenance;
|
||||
}
|
||||
|
||||
export interface Provenance {
|
||||
session: string;
|
||||
/** Entry id in the session JSONL — jump-to-point. */
|
||||
entryId: string;
|
||||
}
|
||||
|
||||
// ─── Events (append-only ledger — the source of truth) ──────────────────────
|
||||
|
||||
export type EventKind =
|
||||
| "probe-correct" // pre-plan quiz, answered correctly (floor evidence)
|
||||
| "probe-miss" // pre-plan quiz, wrong answer (ceiling evidence)
|
||||
| "dontKnow" // "I don't know" — genuine gap, not a guess
|
||||
| "misconception-signaled" // specific wrong belief identified via distractor/note
|
||||
| "taught" // node established in Phase 3
|
||||
| "check-correct" // post-teach quiz-check, first try
|
||||
| "check-incorrect" // post-teach quiz-check, missed
|
||||
| "recall-at-delay"; // future: spaced repetition
|
||||
|
||||
export interface EventEvidence {
|
||||
quizQuestion: string;
|
||||
/** What the learner selected (labels). */
|
||||
answer: string[];
|
||||
/** Free-text note typed with the answer — diagnostic gold. */
|
||||
note?: string;
|
||||
}
|
||||
|
||||
export interface KnowledgeEvent {
|
||||
id: string;
|
||||
node: string;
|
||||
kind: EventKind;
|
||||
session: string;
|
||||
/** Entry id in the session JSONL → jump to the exact point where this happened. */
|
||||
entryId: string;
|
||||
timestamp: number;
|
||||
evidence?: EventEvidence;
|
||||
}
|
||||
|
||||
// ─── Mappings (merge decisions — reversible by design) ──────────────────────
|
||||
|
||||
export interface SessionMapping {
|
||||
/** Session-local node reference from the extraction JSON. */
|
||||
sessionNodeRef: { session: string; localId: string };
|
||||
canonicalNode: string;
|
||||
decidedBy: "llm-adjudication";
|
||||
confidence: number;
|
||||
/** Mirror of the session-local statement, for auditability. */
|
||||
sessionStatement: string;
|
||||
}
|
||||
|
||||
// ─── Knowing assessment (anchored ordinal rubric — NEVER freehand scalars) ──
|
||||
|
||||
export type KnowingLevel =
|
||||
| "misconception" // actively holds a wrong version
|
||||
| "absent" // dontKnow or wrong with no reasoning
|
||||
| "shakily-known" // correct after a miss, or correct with a hedging note
|
||||
| "known" // first-try correct
|
||||
| "fluent"; // correct AND the note/reasoning shows the right model
|
||||
|
||||
/** Ordinal index — deterministic code maps rubric → number, never an LLM. */
|
||||
export const KNOWING_ORDER: readonly KnowingLevel[] = [
|
||||
"misconception",
|
||||
"absent",
|
||||
"shakily-known",
|
||||
"known",
|
||||
"fluent",
|
||||
];
|
||||
|
||||
// ─── Extraction pipeline (per-session JSON, Stage 2 output) ─────────────────
|
||||
|
||||
export interface ExtractedSession {
|
||||
session: string;
|
||||
nodes: Array<{
|
||||
localId: string;
|
||||
statement: string;
|
||||
kind: NodeKind;
|
||||
domain: string;
|
||||
}>;
|
||||
edges: Array<{
|
||||
from: string; // localId
|
||||
to: string; // localId
|
||||
type: EdgeType;
|
||||
}>;
|
||||
events: Array<{
|
||||
node: string; // localId
|
||||
kind: EventKind;
|
||||
entryId: string;
|
||||
evidence?: EventEvidence;
|
||||
}>;
|
||||
knowing: Array<{
|
||||
node: string; // localId
|
||||
level: KnowingLevel;
|
||||
justification: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
// ─── Derived state (computed by code, never stored as primary data) ─────────
|
||||
|
||||
export interface DerivedNodeState {
|
||||
node: string;
|
||||
known: boolean;
|
||||
knowing: number; // 0..1, from rubric + evidence events
|
||||
understood: number; // 0..1, recursive over requires-subtree (aggregation semantics PARKED)
|
||||
floating: boolean; // known but not understood
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
/**
|
||||
* Teacher session — the teaching runtime of the learning harness.
|
||||
*
|
||||
* pi (@earendil-works/pi-coding-agent) is embedded as a library dependency.
|
||||
* This is NOT the user's global pi: hermetic isolation is enforced by
|
||||
* - agentDir → data/agent/ (settings/auth/skills/extensions; nothing from ~/.pi/agent
|
||||
* loads except auth.json, which is a symlink — credentials only)
|
||||
* - sessions → data/sessions/ (harness-owned session JSONLs)
|
||||
* - systemPrompt → teacher/system-prompt.md (fully replaces pi's coding-agent prompt)
|
||||
* - tools → noTools: "builtin" (read/bash/edit/write off; only extension tools remain)
|
||||
* - extensions→ explicitly: quiz.ts + ask-user-question.ts via additionalExtensionPaths
|
||||
*/
|
||||
|
||||
import path from "node:path";
|
||||
import {
|
||||
createAgentSessionFromServices,
|
||||
createAgentSessionRuntime,
|
||||
createAgentSessionServices,
|
||||
InteractiveMode,
|
||||
initTheme,
|
||||
SessionManager,
|
||||
type CreateAgentSessionRuntimeFactory,
|
||||
} from "@earendil-works/pi-coding-agent";
|
||||
import { PATHS } from "../../config.ts";
|
||||
|
||||
export function buildTeacherRuntimeFactory(): CreateAgentSessionRuntimeFactory {
|
||||
return async ({ cwd, sessionManager, sessionStartEvent }) => {
|
||||
const services = await createAgentSessionServices({
|
||||
cwd,
|
||||
agentDir: PATHS.agentDir,
|
||||
resourceLoaderOptions: {
|
||||
// Only the extensions we explicitly list. No discovery of ~/.pi/agent
|
||||
// or .pi/extensions beyond what agentDir/cwd contain (they contain none).
|
||||
additionalExtensionPaths: [
|
||||
path.join(PATHS.extensions, "quiz.ts"),
|
||||
path.join(PATHS.extensions, "ask-user-question.ts"),
|
||||
],
|
||||
// The teacher system prompt — replaces pi's coding prompt entirely.
|
||||
systemPrompt: PATHS.teacherPrompt,
|
||||
// Hermetic: no skills, no prompt templates, no AGENTS.md context files.
|
||||
// Themes stay enabled for a good TUI.
|
||||
noSkills: true,
|
||||
noPromptTemplates: true,
|
||||
noContextFiles: true,
|
||||
},
|
||||
});
|
||||
return {
|
||||
...(await createAgentSessionFromServices({
|
||||
services,
|
||||
sessionManager,
|
||||
sessionStartEvent,
|
||||
// No read/bash/edit/write. quiz + ask_user_question (extension tools)
|
||||
// remain enabled.
|
||||
noTools: "builtin",
|
||||
})),
|
||||
services,
|
||||
diagnostics: services.diagnostics,
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
/** Launch the interactive teaching session (full pi TUI). */
|
||||
export async function runTeachingSession(initialMessage?: string): Promise<void> {
|
||||
const createRuntime = buildTeacherRuntimeFactory();
|
||||
const sessionManager = SessionManager.create(PATHS.root, PATHS.sessions);
|
||||
|
||||
const runtime = await createAgentSessionRuntime(createRuntime, {
|
||||
cwd: PATHS.root,
|
||||
agentDir: PATHS.agentDir,
|
||||
sessionManager,
|
||||
});
|
||||
|
||||
for (const diagnostic of runtime.diagnostics) {
|
||||
console.error(`[${diagnostic.type}] ${diagnostic.message}`);
|
||||
}
|
||||
if (runtime.diagnostics.some((d) => d.type === "error")) {
|
||||
console.error("Fatal diagnostics — aborting.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
initTheme("dark", true);
|
||||
|
||||
const mode = new InteractiveMode(runtime, {
|
||||
migratedProviders: [],
|
||||
modelFallbackMessage: undefined,
|
||||
initialMessage,
|
||||
initialImages: [],
|
||||
initialMessages: [],
|
||||
});
|
||||
await mode.run();
|
||||
}
|
||||
|
||||
/** Non-interactive smoke test: verify the whole wiring boots correctly. */
|
||||
export async function doctor(): Promise<void> {
|
||||
const sessionManager = SessionManager.inMemory(PATHS.root);
|
||||
const runtime = await buildTeacherRuntimeFactory()({
|
||||
cwd: PATHS.root,
|
||||
agentDir: PATHS.agentDir,
|
||||
sessionManager,
|
||||
sessionStartEvent: { type: "session_start", reason: "startup" },
|
||||
});
|
||||
const { session, services } = runtime;
|
||||
|
||||
for (const diagnostic of runtime.diagnostics) {
|
||||
console.log(`[${diagnostic.type}] ${diagnostic.message}`);
|
||||
}
|
||||
|
||||
const extensions = services.resourceLoader.getExtensions();
|
||||
console.log("\nExtensions loaded:");
|
||||
for (const ext of extensions.extensions) {
|
||||
const tools = Array.from(ext.tools.keys());
|
||||
console.log(` - ${path.basename(ext.path)} tools: ${tools.join(", ") || "(none)"}`);
|
||||
}
|
||||
if (extensions.errors.length > 0) {
|
||||
console.log("Extension errors:");
|
||||
for (const { path: p, error } of extensions.errors) console.log(` ! ${p}: ${error}`);
|
||||
}
|
||||
|
||||
const state = session.agent.state;
|
||||
console.log(`\nTools visible to the agent: ${state.tools.map((t) => t.name).join(", ")}`);
|
||||
console.log(`Model: ${state.model ? `${state.model.provider}/${state.model.id}` : "(none — restore from settings failed)"}`);
|
||||
console.log(`Thinking level: ${state.thinkingLevel}`);
|
||||
|
||||
const prompt = state.systemPrompt;
|
||||
console.log(`\nSystem prompt: ${prompt.length} chars`);
|
||||
console.log("--- first 300 chars ---");
|
||||
console.log(prompt.slice(0, 300));
|
||||
console.log("--- last 200 chars ---");
|
||||
console.log(prompt.slice(-200));
|
||||
|
||||
const expectedTools = ["quiz", "ask_user_question"];
|
||||
const toolNames = state.tools.map((t) => t.name);
|
||||
const missing = expectedTools.filter((t) => !toolNames.includes(t));
|
||||
const leaked = toolNames.filter((t) => ["read", "bash", "edit", "write", "grep", "find", "ls"].includes(t));
|
||||
console.log(`\n${missing.length === 0 ? "✓" : "✗"} expected tools present${missing.length ? ` — MISSING: ${missing.join(", ")}` : ""}`);
|
||||
console.log(`${leaked.length === 0 ? "✓" : "✗"} no coding tools leaked${leaked.length ? ` — LEAKED: ${leaked.join(", ")}` : ""}`);
|
||||
console.log(`${prompt.includes("You are the teacher agent of a learning harness") ? "✓" : "✗"} teacher system prompt active`);
|
||||
console.log(`${prompt.includes("quiz") && prompt.includes("ask_user_question") ? "✓" : "✗"} tool guidelines folded into system prompt`);
|
||||
|
||||
session.dispose();
|
||||
}
|
||||
Reference in New Issue
Block a user