sorokin.oleg.edu/learnharness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

practice UI round — top bars + sidebar reopen, one-attempt per-problem submit with instant reveal, score-only grading (owner 2026-09-29)

Browse Source
This commit is contained in:
amosblomqvist
2026-09-29 21:54:32 +03:00
parent ddce919f9b
commit cf0ec4a2f0
14 changed files with 811 additions and 554 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/practice-agents.test.ts
+9 -9
View File
@@ -233,27 +233,27 @@ describe("runGrader", () => {
answer: "Well, if I'm arguing at all, I must own myself…",
};
it("returns the verdict + feedback with its step log", async () => {
it("returns the score + reasoning with its step log", async () => {
const agents = createPracticeAgents({
grader: submitModel("submit_grade", { verdict: "solid", feedback: "The recall happened." }),
grader: submitModel("submit_grade", { score: 9, reasoning: "The recall happened." }),
});
const out = await agents.runGrader(graderInput);
assert.equal(out.verdict, "solid");
assert.equal(out.feedback, "The recall happened.");
assert.equal(out.score, 9);
assert.equal(out.reasoning, "The recall happened.");
assert.ok(out.log.model);
assert.equal(out.log.steps.length, 1);
});
it("schema-rejects an off-union verdict and retries", async () => {
it("schema-rejects an out-of-range score and retries", async () => {
const agents = createPracticeAgents({
grader: retryModel(
"submit_grade",
{ verdict: "excellent", feedback: "…" },
{ verdict: "partial", feedback: "Half the target came to mind." },
{ score: 12, reasoning: "…" },
{ score: 6, reasoning: "Half the target came to mind." },
),
});
const out = await agents.runGrader(graderInput);
assert.equal(out.verdict, "partial");
assert.equal(out.score, 6);
assert.equal(out.log.steps.length, 2);
});
@@ -312,7 +312,7 @@ describe("formatting doctrine pins", () => {
it("GRADER_INSTRUCTIONS carries the formatting paragraph before the OUTPUT section", () => {
assert.ok(
GRADER_INSTRUCTIONS.includes(
"Feedback renders as full Markdown. Mathematics is always LaTeX — inline as \\(…\\), display as \\[…\\] on its own lines (a bare $ is literal currency). Inside math, LaTeX commands for symbols, not unicode stand-ins. Code in fenced blocks with a language tag, identifiers inline as `code`. Lists and tables wherever they carry structure; no headers — feedback is short-form, structure it with paragraphs and bold labels. Name the failed recall and teach the boundary in properly formatted prose, not plain text.",
"The reasoning is plain prose, one or two sentences: which rubric points were met or missed and, when something failed, WHICH proposition failed to come to mind. Mathematics is LaTeX — inline as \\(…\\), display as \\[…\\] (a bare $ is literal currency); no headers, no lists — minimal is the standard.",
),
"the exact grader paragraph",
);
tests/practice-pipeline.test.ts
+18 -14
View File
@@ -152,7 +152,7 @@ function echoGenerator(count = 24, failTargets: string[] = []): PiEngine {
return fauxEngine(Array.from({ length: count }, () => factory));
}
/** A grader whose verdict encodes in the answer text ("solid" → solid). */
/** A grader whose score encodes in the answer text ("solid" → 9, else 6). */
function echoGrader(count: number): PiEngine {
const factory: FauxResponseStep = (context) => {
const answer = lastUserText(context as never).split("THE LEARNER'S ANSWER\n")[1] ?? "";
@@ -161,8 +161,8 @@ function echoGrader(count: number): PiEngine {
fauxToolCall(
"submit_grade",
{
verdict: answer.includes("solid") ? "solid" : "partial",
feedback: "The boundary.",
score: answer.includes("solid") ? 9 : 6,
reasoning: "Which rubric points were met.",
} as ToolCall["arguments"],
{ id: "grade-call" },
),
@@ -521,7 +521,7 @@ describe("createProblemSet (generation)", () => {
const gate = new Promise<void>((r) => (release = r));
// hold both grades at the model seam
const holdSteps: FauxResponseStep[] = Array.from({ length: 2 }, () => () =>
gate.then(() => fauxAssistantMessage([fauxToolCall("submit_grade", { verdict: "solid", feedback: "yes" } as ToolCall["arguments"], { id: "gr" })], { stopReason: "toolUse" })),
gate.then(() => fauxAssistantMessage([fauxToolCall("submit_grade", { score: 9, reasoning: "yes" } as ToolCall["arguments"], { id: "gr" })], { stopReason: "toolUse" })),
);
const f = fixture({
partitioner: partitionModel([{ chains: ["b"] }, { chains: ["c"] }]),
@@ -554,7 +554,7 @@ describe("createProblemSet (generation)", () => {
await sub.done;
assert.equal(practiceStatuses().length, 0);
const graded = f.practice.store.getSet(set.id)!;
assert.ok(graded.problems.every((p) => p.grade?.verdict === "solid"));
assert.ok(graded.problems.every((p) => p.grade?.score === 9));
});
it("abort ONE gen row: that problem dies alone, siblings complete, set reaches ready with one errored", async () => {
@@ -600,7 +600,7 @@ describe("createProblemSet (generation)", () => {
assert.equal(practiceStatuses().length, 0);
});
it("abort ONE grade row (queued): that grade drops (answer kept, re-submittable), sibling grade lands", async () => {
it("abort ONE grade row (queued): that grade drops (answer kept, one attempt — no re-submission), sibling grade lands", async () => {
const f = fixture({
partitioner: partitionModel([{ chains: ["b"] }, { chains: ["c"] }]),
problemGenerator: echoGenerator(2),
@@ -645,14 +645,18 @@ describe("createProblemSet (generation)", () => {
assert.equal(dropped.grade, undefined, "the aborted grade is absent");
assert.equal(dropped.answer, "first", "the answer is kept");
const other = graded.problems.find((p) => p.id !== victimId)!;
assert.equal(other.grade?.verdict, "partial", "the sibling grade landed (echoGrader: no 'solid' in its text)");
assert.equal(other.grade?.score, 6, "the sibling grade landed (echoGrader: no 'solid' in its text)");
assert.equal(practiceStatuses().length, 0);
// re-submittable: the dropped answer grades on a second submit
const resub = await pipeline.submitAnswers(set.id, [{ problemId: victimId, text: "retry answer" }]);
await resub.done;
const retried = f.practice.store.getSet(set.id)!;
assert.equal(retried.problems.find((p) => p.id === victimId)!.grade?.verdict, "partial", "the retry grades (echoGrader verdict)");
// ONE attempt (owner 2026-09-29): the dropped grade's answer
// stands — re-submission is refused, the problem is done.
await assert.rejects(
() => pipeline.submitAnswers(set.id, [{ problemId: victimId, text: "retry answer" }]),
/was already answered/,
);
const after = f.practice.store.getSet(set.id)!;
assert.equal(after.problems.find((p) => p.id === victimId)!.grade, undefined, "still no grade — one attempt");
assert.equal(after.problems.find((p) => p.id === victimId)!.answer, "first");
rh1();
rh2();
@@ -772,8 +776,8 @@ describe("submitAnswers (grading)", () => {
const final = f.practice.store.getSet(set.id)!;
const g1 = final.problems.find((p) => p.id === p1.id)!.grade;
const g2 = final.problems.find((p) => p.id === p2.id)!.grade;
assert.ok(g1 && g1.verdict === "solid", "verdict encodes from the answer text");
assert.ok(g2 && g2.verdict === "partial");
assert.ok(g1 && g1.score === 9, "score encodes from the answer text");
assert.ok(g2 && g2.score === 6);
assert.ok(g1!.gradedAt >= NOW);
assert.equal(practiceStatuses().length, 0, "the grading row clears");
tests/practice-store.test.ts
+13 -9
View File
@@ -79,13 +79,13 @@ describe("the practice store", () => {
rubric: "- names the performative contradiction",
status: "ready",
answer: "Because…",
grade: { verdict: "solid", feedback: "Named it.", gradedAt: "2026-09-29T13:00:00.000Z" },
grade: { score: 9, reasoning: "Named it.", gradedAt: "2026-09-29T13:00:00.000Z" },
});
set.status = "ready";
practiceStore.saveSet(set);
const reread = practiceStore.getSet(set.id);
assert.equal(reread?.status, "ready");
assert.equal(reread?.problems[0]?.grade?.verdict, "solid");
assert.equal(reread?.problems[0]?.grade?.score, 9);
});
it("listSets returns createdAt-descending and skips corrupt/unparseable files", () => {
@@ -157,17 +157,21 @@ describe("isProblemSet", () => {
assert.ok(isProblemSet(minimal));
});
it("grade verdicts are the three-level union", () => {
const withGrade = (verdict: string): unknown => ({
it("grades are integer scores 0–10 with non-empty reasoning (one-attempt redesign)", () => {
const withGrade = (score: unknown, reasoning = "why"): unknown => ({
...init(),
id: "ps_valid",
problems: [
{ id: "pp_1", chains: [], statement: "", solution: "", rubric: "", status: "ready", grade: { verdict, feedback: "", gradedAt: "t" } },
{ id: "pp_1", chains: [], statement: "", solution: "", rubric: "", status: "ready", grade: { score, reasoning, gradedAt: "t" } },
],
});
assert.ok(isProblemSet(withGrade("solid")));
assert.ok(isProblemSet(withGrade("partial")));
assert.ok(isProblemSet(withGrade("missed")));
assert.ok(!isProblemSet(withGrade("excellent")));
assert.ok(isProblemSet(withGrade(0)));
assert.ok(isProblemSet(withGrade(10)));
assert.ok(!isProblemSet(withGrade(11)), "above the range");
assert.ok(!isProblemSet(withGrade(-1)), "below the range");
assert.ok(!isProblemSet(withGrade(7.5)), "fractional scores are refused");
assert.ok(!isProblemSet(withGrade("9")), "string scores are refused");
assert.ok(!isProblemSet(withGrade(9, "")), "empty reasoning is refused");
assert.ok(!isProblemSet({ ...init(), id: "ps_valid", problems: [{ id: "pp_1", chains: [], statement: "", solution: "", rubric: "", status: "ready", grade: { verdict: "solid", feedback: "legacy", gradedAt: "t" } }] }), "legacy verdict grades fail (wipe regime)");
});
});
tests/render-practice.test.tsx
+54 -17
View File
@@ -200,7 +200,7 @@ describe("PracticeListBody — the set list", () => {
it("graded count appears; empty list states it honestly", () => {
const s = set({
problems: [
readyProblem("pp_1", ["n3"], { grade: { verdict: "solid", feedback: "f", gradedAt: "2026-09-29T13:00:00.000Z" } }),
readyProblem("pp_1", ["n3"], { answer: "a", grade: { score: 9, reasoning: "r", gradedAt: "2026-09-29T13:00:00.000Z" } }),
readyProblem("pp_2", ["n1"]),
],
});
@@ -251,9 +251,9 @@ describe("PracticeExamBody — exam states", () => {
drafts: {} as Record<string, string>,
onDraft: () => {},
onSubmit: () => {},
submitting: false,
submitError: null as string | null,
grading: false,
submittingId: null as string | null,
submitErrors: {} as Record<string, string>,
liveGradeIds: new Set<string>(),
};
it("generating: numbered stubs with the placeholder — propositions NEVER surface", () => {
@@ -271,12 +271,19 @@ describe("PracticeExamBody — exam states", () => {
assert.ok(!html.includes("Each person owns themself"));
assert.ok(!html.includes('data-testid="answer-pp_1"'), "no answer box for a stub");
// No submit while generating.
assert.ok(!html.includes('data-testid="practice-submit"'));
assert.ok(!html.includes('data-testid="practice-submit-'));
});
it("the header carries the levels line (named levels, owner 2026-09-29)", () => {
it("the top bar carries the title + sidebar toggle; the body carries the levels line (owner, UI round 2026-09-29)", () => {
const s = set({ problems: [readyProblem("pp_1", ["n3"])] });
const html = renderToStaticMarkup(<PracticeExamBody set={s} {...examBase} />);
// The chat-pattern top bar: drag surface, toggle slot, truncating title.
assert.ok(html.includes("lh-drag"));
assert.ok(html.includes('aria-label="Open sidebar"'));
assert.ok(html.includes('data-testid="practice-exam-title"'));
assert.ok(html.includes("Ethics lesson"));
// The internal scroller (the sidebar stays fixed when this page scrolls).
assert.ok(html.includes('data-testid="practice-exam-scroll"'));
assert.ok(html.includes('data-testid="practice-exam-levels"'));
assert.ok(html.includes("Direct exercise · Adjacent context · Theory"), "the fixture's rung 2 / cue 1 / theory");
});
@@ -286,37 +293,67 @@ describe("PracticeExamBody — exam states", () => {
const gated = renderToStaticMarkup(<PracticeExamBody set={s} {...examBase} />);
assert.ok(gated.includes("self-ownership")); // markdown strong rendered
assert.ok(gated.includes('data-testid="answer-pp_1"'));
assert.ok(gated.includes('data-testid="practice-submit"'));
// PER-PROBLEM submit (owner 2026-09-29): the button lives under the
// box, gated on a draft; there is NO global submit anywhere.
assert.ok(gated.includes('data-testid="practice-submit-pp_1"'));
assert.ok(gated.includes('disabled=""'), "empty drafts — submit disabled");
const filled = renderToStaticMarkup(<PracticeExamBody set={s} {...examBase} drafts={{ pp_1: "because…" }} />);
assert.ok(!filled.includes('data-testid="practice-submit-error"'));
assert.ok(!filled.includes('disabled=""'), "a draft arms the submit");
assert.ok(!filled.includes('data-testid="practice-submit-error-pp_1"'));
});
it("graded: answer read-only, verdict chip + feedback + solution + rubric revealed", () => {
it("submitted + graded: answer read-only, score + reasoning + solution revealed, rubric collapsed", () => {
const s = set({
problems: [
readyProblem("pp_1", ["n3"], {
answer: "because arguing is intentional action",
grade: { verdict: "partial", feedback: "Half the target came to mind.", gradedAt: "2026-09-29T13:00:00.000Z" },
grade: { score: 6, reasoning: "Half the target came to mind.", gradedAt: "2026-09-29T13:00:00.000Z" },
}),
readyProblem("pp_2", ["n1"]),
],
});
const html = renderToStaticMarkup(<PracticeExamBody set={s} {...examBase} />);
// The locked answer.
// The locked answer — one attempt, done.
assert.ok(html.includes("because arguing is intentional action"));
assert.ok(/readOnly/.test(html), "the graded answer is read-only");
// The reveal.
assert.ok(html.includes('data-testid="reveal-pp_1"'));
assert.ok(html.includes("partial"));
assert.ok(html.includes('data-testid="feedback-pp_1"'));
assert.ok(/readOnly/.test(html), "the submitted answer is read-only");
assert.ok(!html.includes('data-testid="practice-submit-pp_1"'), "no re-submission");
// The grade: mono score + brief reasoning.
assert.ok(html.includes('data-testid="grade-pp_1"'));
assert.ok(html.includes("6/10"));
assert.ok(html.includes('data-testid="reasoning-pp_1"'));
assert.ok(html.includes("Half the target came to mind."));
// The instant reveal.
assert.ok(html.includes('data-testid="reveal-pp_1"'));
assert.ok(html.includes('data-testid="solution-pp_1"'));
assert.ok(html.includes("The model solution."));
// The rubric rides a collapsed <details> (owner 2026-09-29).
assert.ok(html.includes("<details"));
assert.ok(!html.includes("<details open"), "collapsed by default");
assert.ok(html.includes('data-testid="rubric-pp_1"'));
assert.ok(html.includes("names self-ownership"));
// Ungraded sibling stays answerable.
// Unanswered sibling stays answerable.
assert.ok(html.includes('data-testid="answer-pp_2"'));
assert.ok(html.includes('data-testid="practice-submit-pp_2"'));
});
it("submitted, grade pending: the reveal shows instantly, grading spins per problem", () => {
const s = set({
problems: [readyProblem("pp_1", ["n3"], { answer: "because arguing is intentional action" })],
});
const live = renderToStaticMarkup(
<PracticeExamBody set={s} {...examBase} liveGradeIds={new Set(["pp_1"])} />,
);
// Instant reveal — the solution needs no grade.
assert.ok(live.includes('data-testid="solution-pp_1"'));
assert.ok(live.includes('data-testid="rubric-pp_1"'));
// The live spinner for exactly this problem.
assert.ok(live.includes('data-testid="grading-pp_1"'));
assert.ok(live.includes("Grading"));
assert.ok(!live.includes('data-testid="grade-pp_1"'), "no score yet");
// No live row (dropped grade): honest, no eternal spinner.
const dropped = renderToStaticMarkup(<PracticeExamBody set={s} {...examBase} />);
assert.ok(dropped.includes('data-testid="grade-interrupted-pp_1"'));
assert.ok(!dropped.includes('data-testid="grading-pp_1"'));
});
it("a failed problem says so; a failed set says so", () => {