sorokin.oleg.edu/learnharness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Centralized model config (config.ts MODELS) + extraction seam logging

Browse Source 
MODELS in config.ts is now THE single tuning table: model id + OpenRouter
reasoning effort for teacher/researcher/demoBuilder/visualMaker/extractor/
auditor. .env.local is secrets-only (LH_* model vars removed; all sub-agents
previously had NO reasoning setting -> provider-default unbounded thinking,
the extraction token burn). One factory (src/agent/models.ts modelFor) wires
all six call sites; test injection unchanged. EXTRACTION.everyNQuizAnswers
moved to a config literal too. Plus the §5 one-line-per-seam logging in the
extraction pipeline: pipeline start, extractor/auditor start + per-step,
audit kept, merged — previously silent until completion, which made live
runs look like they never fired.
This commit is contained in:
Amos
2026-09-01 01:47:47 +03:00
parent 5dab8d12d4
commit 59fc2ef863
11 changed files with 94 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/agent/subagents/demo-builder.ts
+2 -5
View File
@@ -1,6 +1,6 @@
import { ToolLoopAgent, stepCountIs, tool, type LanguageModel, type ToolSet } from "ai";
import { openrouter } from "@openrouter/ai-sdk-provider";
import { z } from "zod";
import { modelFor } from "../models.ts";
import type { DemoResult } from "../../core/contracts.ts";
import { getHeadless, type DemoRunResult, type HeadlessApi } from "./headless.ts";
@@ -20,9 +20,6 @@ import { getHeadless, type DemoRunResult, type HeadlessApi } from "./headless.ts
* The job wrapper treats html === "" as failure and marks the job error.
*/
const DEFAULT_DEMO_MODEL =
process.env.LH_DEMO_MODEL ?? process.env.LH_MODEL ?? "anthropic/claude-sonnet-4.5";
export interface DemoBuilderInput {
title: string;
brief: string;
@@ -174,7 +171,7 @@ export async function runDemoBuilder(input: DemoBuilderInput, opts: DemoBuilderO
const state: BuilderState = { html: null, testedHtml: null, lastResult: null };
const agent = new ToolLoopAgent({
model: opts.model ?? openrouter(DEFAULT_DEMO_MODEL),
model: opts.model ?? modelFor("demoBuilder"),
instructions: INSTRUCTIONS,
tools: createDemoBuilderTools(state, headless, opts.onActivity),
stopWhen: stepCountIs(opts.maxSteps ?? 24),
src/agent/subagents/extraction.ts
+23 -6
View File
@@ -1,6 +1,6 @@
import { ToolLoopAgent, stepCountIs, tool, type LanguageModel, type UIMessage } from "ai";
import { openrouter } from "@openrouter/ai-sdk-provider";
import { z } from "zod";
import { modelFor } from "../models.ts";
import type { AuditResult, ChatRecord, SessionFindings } from "../../core/contracts.ts";
import { mergeFindings, type EmissionStats, type GraphDeps } from "../../core/graph.ts";
@@ -27,9 +27,6 @@ import { mergeFindings, type EmissionStats, type GraphDeps } from "../../core/gr
* the next trigger re-runs the whole thing.
*/
const DEFAULT_EXTRACTION_MODEL =
process.env.LH_EXTRACT_MODEL ?? process.env.LH_MODEL ?? "anthropic/claude-sonnet-4.5";
// ─── the transcript view (what the LLM sees) ────────────────────────────────
/**
@@ -175,6 +172,12 @@ function createExtractor(model: LanguageModel) {
}),
},
stopWhen: stepCountIs(4),
// One line per step: a background worker's progress must be legible
// from the server log (§5 observability baseline).
onStepFinish: (step) =>
console.info(
`[extraction] extractor step ${step.stepNumber}: ${step.finishReason}, tools: ${step.toolCalls.map((c) => c.toolName).join(", ") || "(none)"}`,
),
});
}
@@ -224,6 +227,10 @@ function createAuditor(model: LanguageModel) {
}),
},
stopWhen: stepCountIs(4),
onStepFinish: (step) =>
console.info(
`[extraction] auditor step ${step.stepNumber}: ${step.finishReason}, tools: ${step.toolCalls.map((c) => c.toolName).join(", ") || "(none)"}`,
),
});
}
@@ -246,14 +253,15 @@ export interface ExtractionPipeline {
export function createExtractionPipeline(deps: ExtractionPipelineDeps): ExtractionPipeline {
const { chatStore, graphStore } = deps;
const extractorAgent = createExtractor(deps.extractor ?? openrouter(DEFAULT_EXTRACTION_MODEL));
const auditorAgent = createAuditor(deps.auditor ?? openrouter(DEFAULT_EXTRACTION_MODEL));
const extractorAgent = createExtractor(deps.extractor ?? modelFor("extractor"));
const auditorAgent = createAuditor(deps.auditor ?? modelFor("auditor"));
const inFlight = new Set<string>();
async function extract(chatId: string): Promise<SessionFindings> {
const record = chatStore.getChat(chatId);
if (!record) throw new Error(`Chat not found: ${chatId}`);
const view = transcriptView(record);
console.info(`[extraction ${chatId}] extractor starting (${view.split("\n").length} transcript lines)`);
const extraction = await extractorAgent.generate({ prompt: `Transcript:\n\n${view}` });
for (const part of extraction.content) {
@@ -268,6 +276,10 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
const record = chatStore.getChat(chatId);
if (!record) throw new Error(`Chat not found: ${chatId}`);
const view = transcriptView(record);
console.info(
`[extraction ${chatId}] auditor starting (${findings.propositions.length} propositions, ${findings.edges.length} edges, ` +
`${Object.keys(findings.attribution.quiz).length} attributions)`,
);
const review = await auditorAgent.generate({
prompt: `Transcript:\n\n${view}\n\nProposed findings:\n\n${JSON.stringify(findings, null, 2)}`,
@@ -286,6 +298,10 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
);
}
const { dropped: _dropped, ...verified } = auditResult;
console.info(
`[extraction ${chatId}] audit kept ${verified.propositions.length}/${findings.propositions.length} propositions, ` +
`${verified.edges.length}/${findings.edges.length} edges`,
);
return verified;
}
@@ -293,6 +309,7 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
if (inFlight.has(chatId)) return { nodes: 0, edges: 0, events: 0 };
inFlight.add(chatId);
try {
console.info(`[extraction ${chatId}] pipeline start`);
const findings = await extract(chatId);
const verified = await audit(chatId, findings);
// The merger re-reads the chat record itself — events are rebuilt
src/agent/subagents/researcher.ts
+2 -5
View File
@@ -1,6 +1,6 @@
import { ToolLoopAgent, stepCountIs, type LanguageModel } from "ai";
import { openrouter } from "@openrouter/ai-sdk-provider";
import { webTools } from "../tools/web.ts";
import { modelFor } from "../models.ts";
/**
* The researcher sub-agent (vision §2.5): a SEPARATE ToolLoopAgent instance
@@ -12,9 +12,6 @@ import { webTools } from "../tools/web.ts";
* The model is injectable so tests run the loop with a MockLanguageModelV4.
*/
const DEFAULT_RESEARCHER_MODEL =
process.env.LH_RESEARCH_MODEL ?? process.env.LH_MODEL ?? "anthropic/claude-sonnet-4.5";
const INSTRUCTIONS = `You are a research specialist. You work for a teacher agent inside a learning
application: given a research question, conduct thorough web research and produce a focused,
well-sourced brief that the teacher can teach from.
@@ -61,7 +58,7 @@ What couldn't be answered. Suggested next steps.`;
export function createResearcherAgent(model?: LanguageModel) {
return new ToolLoopAgent({
model: model ?? openrouter(DEFAULT_RESEARCHER_MODEL),
model: model ?? modelFor("researcher"),
instructions: INSTRUCTIONS,
tools: webTools,
stopWhen: stepCountIs(12),
src/agent/subagents/visual-maker.ts
+2 -5
View File
@@ -1,6 +1,6 @@
import { generateText, tool, type LanguageModel, type ModelMessage, type ToolResultPart } from "ai";
import { openrouter } from "@openrouter/ai-sdk-provider";
import { z } from "zod";
import { modelFor } from "../models.ts";
import type { VisualResult } from "../../core/contracts.ts";
import { getHeadless, type HeadlessApi } from "./headless.ts";
@@ -26,9 +26,6 @@ import { getHeadless, type HeadlessApi } from "./headless.ts";
* Failure semantics mirror the demo-builder: { svg: "", notes: "FAILED: …" }.
*/
const DEFAULT_VISUAL_MODEL =
process.env.LH_VISUAL_MODEL ?? process.env.LH_MODEL ?? "anthropic/claude-sonnet-4.5";
export interface VisualMakerInput {
brief: string;
}
@@ -127,7 +124,7 @@ export async function runVisualMaker(input: VisualMakerInput, opts: VisualMakerO
try {
for (let iteration = 0; iteration < maxIterations; iteration++) {
const result = await generateText({ model: opts.model ?? openrouter(DEFAULT_VISUAL_MODEL), system: INSTRUCTIONS, messages, tools });
const result = await generateText({ model: opts.model ?? modelFor("visualMaker"), system: INSTRUCTIONS, messages, tools });
messages = [...messages, ...result.response.messages];
const toolCalls = result.toolCalls;