fix: retain Google provider dependencies in desktop builds
This commit is contained in:
amosblomqvist
@@ -63,5 +63,7 @@ jobs:
|
||||
NODE_ENV: test
|
||||
ELECTRON_RUN_AS_NODE: "1"
|
||||
- run: npm run build
|
||||
# Check the fresh trace before any static copying/curation.
|
||||
- run: node scripts/smoke-provider-runtime.mjs
|
||||
- run: node scripts/build-server.mjs
|
||||
- run: node scripts/smoke-server.mjs --electron
|
||||
|
||||
@@ -129,6 +129,8 @@ jobs:
|
||||
NODE_ENV: test
|
||||
ELECTRON_RUN_AS_NODE: "1"
|
||||
- run: npm run build
|
||||
# Check the fresh trace before any static copying/curation.
|
||||
- run: node scripts/smoke-provider-runtime.mjs
|
||||
- run: node scripts/build-server.mjs
|
||||
- run: node scripts/smoke-server.mjs --electron
|
||||
- name: Validate macOS release secrets
|
||||
@@ -169,6 +171,25 @@ jobs:
|
||||
- name: Windows/Linux rehearsal (no publish)
|
||||
if: github.event_name == 'workflow_dispatch' && runner.os != 'macOS'
|
||||
run: npx electron-builder ${{ matrix.builder_args }} --publish never
|
||||
- name: Initialize packaged providers offline (host and packaged Electron Node)
|
||||
run: |
|
||||
case "$RUNNER_OS" in
|
||||
macOS)
|
||||
app="dist/mac-arm64/LearnHarness.app/Contents"
|
||||
payload="$app/Resources/server/standalone"
|
||||
runtime="$app/MacOS/LearnHarness"
|
||||
;;
|
||||
Windows)
|
||||
payload="dist/win-unpacked/resources/server/standalone"
|
||||
runtime="dist/win-unpacked/LearnHarness.exe"
|
||||
;;
|
||||
Linux)
|
||||
payload="dist/linux-unpacked/resources/server/standalone"
|
||||
runtime="dist/linux-unpacked/learnharness"
|
||||
;;
|
||||
esac
|
||||
node scripts/smoke-provider-runtime.mjs --standalone "$payload"
|
||||
node scripts/smoke-provider-runtime.mjs --standalone "$payload" --runtime "$runtime"
|
||||
- name: Exercise packaged demo runtime (own Electron, no system Chrome)
|
||||
run: |
|
||||
case "$RUNNER_OS" in
|
||||
|
||||
+16
-4
@@ -17,7 +17,8 @@ const nextConfig: NextConfig = {
|
||||
// into the DMG (the packaged app never reads it: the shell passes
|
||||
// LH_AUTH_FILE → userData). Excluded here; build-server.mjs also REFUSES
|
||||
// to package any standalone still containing it. .env* belt-and-braces
|
||||
// the same class (not observed swept, same seam).
|
||||
// the same class. Next also copies loaded bare .env after tracing;
|
||||
// the build-server refusal remains required even with these excludes.
|
||||
//
|
||||
// ⚠ EXCLUDE PATTERNS ARE LOAD-BEARING IN WAYS THE DOCS DON'T DESCRIBE
|
||||
// (empirically established 2026-10-08): adding dir excludes like
|
||||
@@ -26,8 +27,11 @@ const nextConfig: NextConfig = {
|
||||
// under their dist/ — the traced copy collapsed to a bare package.json
|
||||
// and the packaged server died on ERR_MODULE_NOT_FOUND; openai/
|
||||
// @anthropic-ai — pi-ai's deps — vanished from the trace entirely).
|
||||
// Anchored "./"-prefixed forms behave the same. Only the three entries
|
||||
// below are verified safe (complete externals trace + clean payload).
|
||||
// Anchored "./"-prefixed forms behave the same. **/data/**/* fixes
|
||||
// suffix matching in Next's JavaScript collector, but NOT its native
|
||||
// Turbopack tracer: that still prunes gcp-metadata and its private deps.
|
||||
// Keep the narrow closure workaround below AND the native runtime gate.
|
||||
// Real SDK data/ directories still need their explicit includes too.
|
||||
// The recursive-DMG payload (repo dist/ swept into the bundle, shipping
|
||||
// the previous build inside the next) is instead prevented by the
|
||||
// desktop:dist clean step AND targeted post-trace curation in
|
||||
@@ -37,7 +41,7 @@ const nextConfig: NextConfig = {
|
||||
// stay intact, and the script guards against leaked data/keys.
|
||||
outputFileTracingExcludes: {
|
||||
"*": [
|
||||
"data/**/*",
|
||||
"**/data/**/*",
|
||||
"auth.json",
|
||||
".env*",
|
||||
],
|
||||
@@ -71,6 +75,14 @@ const nextConfig: NextConfig = {
|
||||
"node_modules/@resvg/resvg-js-*/**/*",
|
||||
"node_modules/@earendil-works/pi-ai/dist/providers/data/**/*",
|
||||
"node_modules/@earendil-works/pi-agent-core/dist/**/*.json",
|
||||
// Next 16.3.2 native Turbopack applies unanchored excludes while
|
||||
// traversing modules; **/data/**/* still prunes gcp-metadata and
|
||||
// its private runtime chain. Includes are applied after traversal.
|
||||
// This is the verified GoogleAuth closure, not a general dep copy;
|
||||
// the offline native Pi gate must catch future SDK closure changes.
|
||||
"node_modules/gcp-metadata/**/*",
|
||||
"node_modules/json-bigint/**/*",
|
||||
"node_modules/bignumber.js/**/*",
|
||||
],
|
||||
},
|
||||
// Framework-free core lives in src/ (plain TS, imported by routes and CLI alike).
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@
|
||||
"typecheck": "tsc --noEmit",
|
||||
"desktop:dev": "concurrently -k \"next dev\" \"LH_DEV_URL=http://localhost:3000 electron .\"",
|
||||
"dev:fresh": "bash scripts/dev-fresh.sh",
|
||||
"desktop:dist": "rm -rf dist .next/standalone && next build && node scripts/build-server.mjs && node scripts/smoke-server.mjs && DEVELOPER_DIR=${DEVELOPER_DIR:-/Applications/Xcode.app/Contents/Developer} APPLE_KEYCHAIN_PROFILE=LH_NOTARY APPLE_TEAM_ID=JS6XN22HY6 electron-builder --mac --arm64"
|
||||
"desktop:dist": "rm -rf dist .next/standalone && next build && node scripts/smoke-provider-runtime.mjs && node scripts/build-server.mjs && node scripts/smoke-server.mjs && DEVELOPER_DIR=${DEVELOPER_DIR:-/Applications/Xcode.app/Contents/Developer} APPLE_KEYCHAIN_PROFILE=LH_NOTARY APPLE_TEAM_ID=JS6XN22HY6 electron-builder --mac --arm64"
|
||||
},
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-agent-core": "1.4.0",
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
/**
|
||||
* OFFLINE native Pi Google initialization gate (no generation or provider calls).
|
||||
* Unlike /api/models, this traverses Pi's lazy adapter and constructs GoogleGenAI.
|
||||
* The real onPayload hook throws a unique sentinel BEFORE generateContentStream.
|
||||
* All file modules must resolve inside the payload, including nested dependencies.
|
||||
*
|
||||
* node scripts/smoke-provider-runtime.mjs [--standalone <dir>] [--runtime <Node/Electron>]
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import * as fs from "node:fs";
|
||||
import { createRequire, registerHooks, syncBuiltinESMExports } from "node:module";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { formatProbeError, probeEnv } from "./smoke-demo-runtime.mjs";
|
||||
|
||||
export function verifyProviderRuntime(standalone, runtime = process.execPath) {
|
||||
const home = fs.mkdtempSync(path.join(os.tmpdir(), "lh-provider-runtime-"));
|
||||
try {
|
||||
fs.writeFileSync(path.join(home, "auth.json"), "{}\n", { mode: 0o600 });
|
||||
const result = spawnSync(runtime, [fileURLToPath(import.meta.url), "--probe", path.resolve(standalone)], {
|
||||
cwd: home,
|
||||
env: { ...probeEnv(process.env, home), ELECTRON_RUN_AS_NODE: "1" },
|
||||
encoding: "utf8", timeout: 30_000, maxBuffer: 128 * 1024,
|
||||
});
|
||||
if (result.status !== 0 || result.error) {
|
||||
throw new Error(`provider runtime probe failed (${result.error?.message ?? result.status}):\n${result.stdout ?? ""}${result.stderr ?? ""}`);
|
||||
}
|
||||
assert.match(result.stdout, /native Pi Google initialization passed/, "probe exited without success marker");
|
||||
process.stdout.write(result.stdout);
|
||||
} finally { fs.rmSync(home, { recursive: true, force: true }); }
|
||||
}
|
||||
|
||||
// Dedicated child only: nothing delegates to a live transport, even if the SDK
|
||||
// changes its fetch path. Count attempts as well: a swallowed guard error fails.
|
||||
function closeNetwork() {
|
||||
let attempts = 0;
|
||||
const deny = () => { attempts++; throw new Error("OFFLINE provider gate: network forbidden"); };
|
||||
globalThis.fetch = deny;
|
||||
globalThis.WebSocket = class { constructor() { deny(); } };
|
||||
const require = createRequire(import.meta.url);
|
||||
for (const [name, methods] of [
|
||||
["http", ["request", "get"]], ["https", ["request", "get"]],
|
||||
["net", ["connect", "createConnection"]], ["tls", ["connect"]],
|
||||
["dgram", ["createSocket"]],
|
||||
["dns", ["lookup", "resolve", "resolve4", "resolve6", "resolveAny", "reverse"]],
|
||||
["dns/promises", ["lookup", "resolve", "resolve4", "resolve6", "resolveAny", "reverse"]],
|
||||
]) {
|
||||
const module = require(`node:${name}`);
|
||||
for (const method of methods) module[method] = deny;
|
||||
}
|
||||
require("node:net").Socket.prototype.connect = deny;
|
||||
syncBuiltinESMExports();
|
||||
return () => assert.equal(attempts, 0, "SDK attempted network access");
|
||||
}
|
||||
|
||||
async function probe(standalone) {
|
||||
const noNetwork = closeNetwork();
|
||||
const root = fs.realpathSync(standalone);
|
||||
const inside = (file) => {
|
||||
const relative = path.relative(root, fs.realpathSync(file));
|
||||
assert.ok(relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative),
|
||||
`dependency escaped standalone payload: ${file}`);
|
||||
};
|
||||
const loaded = new Set();
|
||||
const hooks = registerHooks({
|
||||
resolve(specifier, context, nextResolve) {
|
||||
const result = nextResolve(specifier, context);
|
||||
if (result.url.startsWith("file:")) {
|
||||
inside(fileURLToPath(result.url));
|
||||
loaded.add(fileURLToPath(result.url));
|
||||
} else assert.ok(result.url.startsWith("node:"), `unexpected dependency URL: ${result.url}`);
|
||||
return result;
|
||||
},
|
||||
});
|
||||
try {
|
||||
// Pi is an explicit server external. Import its shipped public entry and
|
||||
// factories, never the checkout's module or a direct/fake Google adapter.
|
||||
const entry = pathToFileURL(path.join(root, "node_modules/@earendil-works/pi-ai/dist/index.js"));
|
||||
const { createModels } = await import(entry.href);
|
||||
const models = createModels();
|
||||
for (const [name, factory] of [["google", "googleProvider"], ["opencode", "opencodeProvider"], ["opencode-go", "opencodeGoProvider"]]) {
|
||||
const provider = (await import(new URL(`providers/${name}.js`, entry).href))[factory]();
|
||||
models.setProvider(provider);
|
||||
const model = provider.getModels().find((candidate) => candidate.api === "google-generative-ai");
|
||||
assert.ok(model, `${name}: missing Google-backed catalog model`);
|
||||
const sentinel = `LH_OFFLINE_ON_PAYLOAD_${randomUUID()}`;
|
||||
let reached = 0;
|
||||
const result = await models.streamSimple(model, { messages: [{ role: "user", content: "Offline initialization only", timestamp: 0 }] }, {
|
||||
apiKey: "dummy-offline-provider-gate-not-a-key",
|
||||
onPayload(payload, selected) {
|
||||
assert.equal(selected.api, "google-generative-ai");
|
||||
assert.ok(payload && typeof payload === "object");
|
||||
reached++;
|
||||
throw new Error(sentinel);
|
||||
},
|
||||
}).result();
|
||||
assert.equal(reached, 1, `${name}: did not reach onPayload: ${result.errorMessage}`);
|
||||
assert.equal(result.stopReason, "error");
|
||||
assert.equal(result.errorMessage, sentinel, `${name}: unexpected initialization error`);
|
||||
noNetwork();
|
||||
console.log(`[smoke-provider-runtime] ${name}: native lazy Google adapter reached onPayload; generation stopped`);
|
||||
}
|
||||
// Resolution/evaluation, not folder-exists checks. This also covers the
|
||||
// previously missing transitive edge and permits legitimate nested deps.
|
||||
for (const name of ["@google/genai", "google-auth-library", "gcp-metadata", "json-bigint", "bignumber.js"]) {
|
||||
assert.ok([...loaded].some((file) => file.includes(`${path.sep}node_modules${path.sep}${name.replaceAll("/", path.sep)}${path.sep}`)),
|
||||
`Google runtime chain was not loaded: ${name}`);
|
||||
}
|
||||
console.log(`[smoke-provider-runtime] native Pi Google initialization passed; ${loaded.size} modules confined to ${root}; zero network attempts`);
|
||||
} finally { hooks.deregister(); }
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
if (args[0] === "--probe") return probe(args[1]);
|
||||
const value = (name, fallback) => {
|
||||
const index = args.indexOf(name);
|
||||
if (index < 0) return fallback;
|
||||
assert.ok(args[index + 1] && !args[index + 1].startsWith("--"), `${name} requires a path`);
|
||||
return path.resolve(args[index + 1]);
|
||||
};
|
||||
verifyProviderRuntime(value("--standalone", path.resolve(import.meta.dirname, "../.next/standalone")), value("--runtime", process.execPath));
|
||||
}
|
||||
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
main().catch((error) => { console.error(`[smoke-provider-runtime] FAIL: ${formatProbeError(error)}`); process.exitCode = 1; });
|
||||
}
|
||||
@@ -26,6 +26,7 @@
|
||||
*/
|
||||
import { spawn } from "node:child_process";
|
||||
import { probeEnv, verifyDemoRuntime } from "./smoke-demo-runtime.mjs";
|
||||
import { verifyProviderRuntime } from "./smoke-provider-runtime.mjs";
|
||||
import { createRequire } from "node:module";
|
||||
import * as fs from "node:fs";
|
||||
import * as http from "node:http";
|
||||
@@ -80,6 +81,8 @@ async function main() {
|
||||
// HTTP boot alone never imports the dynamically loaded demo engine.
|
||||
// Fail before packaging if its standalone dependency graph is incomplete.
|
||||
verifyDemoRuntime(STANDALONE, RUNTIME);
|
||||
// /api/models only reads catalogs; initialize the real lazy SDK offline too.
|
||||
verifyProviderRuntime(STANDALONE, RUNTIME);
|
||||
const port = await freePort();
|
||||
const url = `http://127.0.0.1:${port}`;
|
||||
const home = fs.mkdtempSync(path.join(os.tmpdir(), "lh-smoke-"));
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
/** Real installed Pi/Google SDK only; no provider stubs, keys, or generation. */
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import * as fs from "node:fs";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { describe, it } from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const script = fileURLToPath(new URL("../scripts/smoke-provider-runtime.mjs", import.meta.url));
|
||||
const checkout = fileURLToPath(new URL("..", import.meta.url));
|
||||
const run = (payload: string) => spawnSync(process.execPath, [script, "--standalone", payload], { encoding: "utf8", timeout: 40_000 });
|
||||
|
||||
describe("offline native Pi Google initialization gate", () => {
|
||||
it("reaches the sentinel for direct Google and Google-backed OpenCode adapters using the real SDK", () => {
|
||||
// This is an unbundled positive control, NOT proof of standalone completeness.
|
||||
const result = run(checkout);
|
||||
assert.equal(result.status, 0, String(result.stderr));
|
||||
for (const provider of ["google", "opencode", "opencode-go"]) assert.ok(String(result.stdout).includes(`${provider}: native lazy Google adapter reached onPayload`));
|
||||
assert.match(String(result.stdout), /zero network attempts/);
|
||||
});
|
||||
it("rejects a missing Pi entry instead of treating any error as success", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "lh-provider-missing-"));
|
||||
try {
|
||||
const result = run(root);
|
||||
assert.equal(result.status, 1);
|
||||
assert.doesNotMatch(String(result.stdout), /initialization passed/);
|
||||
assert.match(String(result.stderr), /MODULE_NOT_FOUND|ENOENT/);
|
||||
assert.deepEqual(fs.readdirSync(root), [], "gate modified the payload");
|
||||
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
||||
});
|
||||
it("rejects an otherwise complete SDK outside the payload (no host fallback)", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "lh-provider-escape-"));
|
||||
try {
|
||||
fs.symlinkSync(path.join(checkout, "node_modules"), path.join(root, "node_modules"), process.platform === "win32" ? "junction" : "dir");
|
||||
const result = run(root);
|
||||
assert.equal(result.status, 1);
|
||||
assert.match(String(result.stderr), /dependency escaped standalone payload/);
|
||||
assert.doesNotMatch(String(result.stdout), /initialization passed/);
|
||||
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,62 @@
|
||||
/** JS matcher + config-policy regressions; native Turbopack needs the real payload gate. */
|
||||
import assert from "node:assert/strict";
|
||||
import { createRequire } from "node:module";
|
||||
import { describe, it } from "node:test";
|
||||
import nextConfig from "../next.config.ts";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
// Use Next's actual bundled matcher, not a different glob implementation.
|
||||
const picomatch = require("next/dist/compiled/picomatch");
|
||||
const { makeIgnoreFn } = require("next/dist/build/collect-build-traces");
|
||||
const patterns = nextConfig.outputFileTracingExcludes!["*"];
|
||||
|
||||
describe("standalone JavaScript tracing exclusions and native workaround policy", () => {
|
||||
for (const windows of [false, true]) {
|
||||
const root = windows ? "C:\\build\\learnharness" : "/build/learnharness";
|
||||
const file = (relative: string) => root + (windows ? "\\" : "/") + (windows ? relative.replaceAll("/", "\\") : relative);
|
||||
const ignored = picomatch(patterns, { contains: true, dot: true, windows });
|
||||
it(`excludes genuine data directories and secrets (${windows ? "Windows" : "POSIX"})`, () => {
|
||||
for (const relative of ["data/canary.json", "nested/data/private.json", "auth.json", ".env", ".env.local"]) {
|
||||
assert.equal(ignored(file(relative)), true, relative);
|
||||
}
|
||||
// The explicit includes remain necessary: this raw ignore still matches SDK data.
|
||||
assert.equal(ignored(file("node_modules/@earendil-works/pi-ai/dist/providers/data/models.json")), true);
|
||||
assert.ok(nextConfig.outputFileTracingIncludes!["*"]!.includes("node_modules/@earendil-works/pi-ai/dist/providers/data/**/*"));
|
||||
});
|
||||
it(`does not confuse dependency suffixes with data directories (${windows ? "Windows" : "POSIX"})`, () => {
|
||||
for (const relative of ["node_modules/gcp-metadata/build/src/index.js", "node_modules/form-data/lib/form_data.js", "node_modules/@swc/helpers/_/_ts_metadata.js", "node_modules/@swc/helpers/_/_ts_metadata/index.js", "node_modules/somedata/runtime.js"]) {
|
||||
assert.equal(ignored(file(relative)), false, relative);
|
||||
assert.equal(picomatch("data/**/*", { contains: true, dot: true, windows })(file("node_modules/gcp-metadata/build/src/index.js")), true, "old pattern must reproduce the bug");
|
||||
}
|
||||
});
|
||||
}
|
||||
it("uses the installed Next JavaScript early-ignore function on the host platform", () => {
|
||||
const root = process.cwd();
|
||||
const ignore = makeIgnoreFn(root, patterns);
|
||||
assert.equal(ignore(`${root}/data/private.json`), true);
|
||||
assert.equal(ignore(`${root}/node_modules/gcp-metadata/build/src/index.js`), false);
|
||||
});
|
||||
it("preserves all secret exclusions exactly", () => {
|
||||
assert.deepEqual(patterns, ["**/data/**/*", "auth.json", ".env*"]);
|
||||
});
|
||||
it("retains exactly the narrow native-Turbopack closure workaround and existing runtime includes", () => {
|
||||
assert.deepEqual(nextConfig.outputFileTracingIncludes!["*"], [
|
||||
"node_modules/@puppeteer/browsers/lib/browser-data/**/*.js",
|
||||
"node_modules/pdfjs-dist/legacy/**/*",
|
||||
"node_modules/pdfjs-dist/standard_fonts/**/*",
|
||||
"node_modules/@napi-rs/canvas-*/**/*",
|
||||
"node_modules/@resvg/resvg-js-*/**/*",
|
||||
"node_modules/@earendil-works/pi-ai/dist/providers/data/**/*",
|
||||
"node_modules/@earendil-works/pi-agent-core/dist/**/*.json",
|
||||
"node_modules/gcp-metadata/**/*",
|
||||
"node_modules/json-bigint/**/*",
|
||||
"node_modules/bignumber.js/**/*",
|
||||
]);
|
||||
});
|
||||
it("limits the workaround to its three packages, never learner data or unrelated dependencies", () => {
|
||||
const closure = nextConfig.outputFileTracingIncludes!["*"]!.slice(-3);
|
||||
const included = picomatch(closure, { dot: true });
|
||||
for (const file of ["node_modules/gcp-metadata/build/src/index.js", "node_modules/json-bigint/lib/parse.js", "node_modules/bignumber.js/bignumber.js"]) assert.equal(included(file), true, file);
|
||||
for (const file of ["data/private.json", "nested/data/private.json", "auth.json", ".env.local", "node_modules/unrelated/runtime.js", "node_modules/gcp-metadata-lookalike/index.js"]) assert.equal(included(file), false, file);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user