feat(search): Tavily + Brave behind a search-provider port; Google CSE retires
Search port + model slots wave, chunk A (owner design, 2026-09-20):
- core/search.ts: SearchHit + SearchProvider port + SearchStatus —
pure, browser-safe; adapters arrive as injected dependencies.
- server/search.ts: Tavily + Brave adapters (wire shapes VERIFIED
against docs.tavily.com's OpenAPI + Brave's dashboard docs — Tavily
auth is the Bearer header, not the legacy body api_key; Brave is
X-Subscription-Token) + engine resolution per owner decisions: a key
ALONE activates its engine (no engine choice needed), explicit
agent.search.engine pins the preference (both keys: Tavily first),
a pinned engine with a missing key is honestly unavailable — no
silent fallback, no default engine.
- Curated keys catalog: GOOGLE_SEARCH_API_KEY/GOOGLE_CSE_ID retire
(CSE closed to new customers, dies Jan 2027 — stored values become
preserved unknown vars); TAVILY_API_KEY (tvly- shape) + BRAVE_API_KEY
(length guard) join. UI keys-section TOOL_IDS now derives from the
catalog — cannot drift.
- GET /api/search: {available, engine, reason} — the honest status for
the settings gray-out; researcher-disable semantics ride the same
signal (Teacher stream rewires the tool onto the port).
- agent.search settings field (schema + sanitizer + strict parser);
web.ts's WebSearchResult is now the core SearchHit (re-exported).
This commit is contained in:
@@ -1,5 +1,10 @@
|
||||
import { z } from "zod";
|
||||
import { lhTool } from "../lh-tool.ts";
|
||||
// The search-hit shape lives in the core port since the search wave
|
||||
// (2026-09-20) — this module keeps the historical alias so callers don't
|
||||
// churn; the tool's rewiring onto the port is the Teacher stream's.
|
||||
import type { SearchHit as WebSearchResult } from "../../core/search.ts";
|
||||
export type { WebSearchResult };
|
||||
import { Readability } from "@mozilla/readability";
|
||||
import { parseHTML } from "linkedom";
|
||||
import TurndownService from "turndown";
|
||||
@@ -22,12 +27,7 @@ import TurndownService from "turndown";
|
||||
*/
|
||||
|
||||
// ── web_search ───────────────────────────────────────────────────────────────
|
||||
|
||||
export interface WebSearchResult {
|
||||
title: string;
|
||||
url: string;
|
||||
snippet: string;
|
||||
}
|
||||
// (WebSearchResult is the core SearchHit, re-exported above.)
|
||||
|
||||
function searchCredentials(): { apiKey: string; cseId: string } | null {
|
||||
const apiKey = process.env.GOOGLE_SEARCH_API_KEY;
|
||||
|
||||
+17
-20
@@ -37,16 +37,16 @@ export const AUTH_KEYS: readonly AuthKeyInfo[] = [
|
||||
docsUrl: "https://console.deepgram.com/",
|
||||
},
|
||||
{
|
||||
id: "GOOGLE_SEARCH_API_KEY",
|
||||
label: "Google Search API key",
|
||||
description: "Web-search half of the Google pair (with the CSE ID) — powers the researcher sub-agent.",
|
||||
docsUrl: "https://developers.google.com/custom-search/v1/introduction",
|
||||
id: "TAVILY_API_KEY",
|
||||
label: "Tavily",
|
||||
description: "Web-search engine for the researcher sub-agent (AI-native search API). Setting the key activates it — no engine toggle needed.",
|
||||
docsUrl: "https://app.tavily.com/",
|
||||
},
|
||||
{
|
||||
id: "GOOGLE_CSE_ID",
|
||||
label: "Google CSE ID",
|
||||
description: "Programmable Search Engine ID — which engine the search API queries.",
|
||||
docsUrl: "https://programmablesearchengine.google.com/controlpanel",
|
||||
id: "BRAVE_API_KEY",
|
||||
label: "Brave Search",
|
||||
description: "Web-search engine for the researcher sub-agent — the alternative to Tavily. Setting the key activates it.",
|
||||
docsUrl: "https://brave.com/search/api/",
|
||||
},
|
||||
] as const;
|
||||
|
||||
@@ -138,8 +138,8 @@ export type KeyShapeResult = { ok: true } | { ok: false; error: string };
|
||||
* a truncated or wrong-family key can never land in the store). Values are
|
||||
* TRIMMED first (paste hygiene); empty-after-trim always fails.
|
||||
*
|
||||
* CURATED keys carry family rules (Google AIza…, the CSE forms,
|
||||
* Deepgram's length floor — a truncation guard; it publishes no format).
|
||||
* CURATED keys carry family rules (Tavily's tvly-… prefix; Brave and
|
||||
* Deepgram publish no format — length floors + paste hygiene only).
|
||||
* MODEL-PROVIDER keys use genericProviderKeyGuard below instead (families
|
||||
* vary by provider — only paste hygiene applies; owner spec: min length
|
||||
* ~12, no whitespace). OPENROUTER_API_KEY moved to the generic guard with
|
||||
@@ -152,16 +152,13 @@ export function validateKeyShape(id: string, rawValue: string): KeyShapeResult {
|
||||
? { ok: true as const }
|
||||
: { ok: false as const, error: `${id} must be at least ${min} characters with no whitespace` };
|
||||
switch (id) {
|
||||
case "GOOGLE_SEARCH_API_KEY":
|
||||
return value.startsWith("AIza")
|
||||
? nonEmpty(10)
|
||||
: { ok: false, error: `${id} must start with "AIza" (Google's API-key format)` };
|
||||
case "GOOGLE_CSE_ID":
|
||||
return (
|
||||
/^[0-9]{6,}:[0-9A-Za-z_-]+$/.test(value) || /^[0-9A-Za-z]{12,}$/.test(value)
|
||||
)
|
||||
? { ok: true }
|
||||
: { ok: false, error: `${id} must be the Programmable Search Engine ID (digits:digits, or the plain alphanumeric form)` };
|
||||
case "TAVILY_API_KEY":
|
||||
return value.startsWith("tvly-")
|
||||
? nonEmpty(20)
|
||||
: { ok: false, error: `${id} must start with "tvly-" (Tavily's key format)` };
|
||||
case "BRAVE_API_KEY":
|
||||
// No public format — a length floor + paste hygiene only.
|
||||
return nonEmpty(20);
|
||||
case "DEEPGRAM_API_KEY":
|
||||
// No public format — the length floor only catches truncated pastes.
|
||||
return nonEmpty(20);
|
||||
|
||||
@@ -41,6 +41,7 @@ export * from "./fuzzy.ts";
|
||||
// view. Browser-safe — the Wave-4 panel imports these directly.
|
||||
export * from "./settings.ts";
|
||||
export * from "./settings-resolution.ts";
|
||||
export * from "./search.ts"; // search port (search wave, 2026-09-20)
|
||||
// auth-keys.ts (auth-keys wave, 2026-09-19): the curated key catalog +
|
||||
// shape rules + masking — pure, browser-safe (the auth page binds to the
|
||||
// /api/keys routes built on it).
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
/**
|
||||
* The search-provider port (search port + model slots wave, 2026-09-20).
|
||||
*
|
||||
* Search engines are pluggable behind ONE seam: the settings record names a
|
||||
* preferred engine (agent.search.engine — optional), keys live in the auth
|
||||
* store (TAVILY_API_KEY / BRAVE_API_KEY — curated catalog), and the server
|
||||
* composes the live adapter. The Google CSE engine retired with this wave
|
||||
* (closed to new customers; dies Jan 2027) — replaced by Tavily + Brave.
|
||||
*
|
||||
* Owner decisions baked here:
|
||||
* - NO default engine in the factory record: an engine is live iff its
|
||||
* key is present. With a key, the engine auto-activates (the user never
|
||||
* chooses an engine to make search work — setting the key is enough);
|
||||
* agent.search.engine is only an explicit preference.
|
||||
* - Nothing silently falls back: no engine live = search is honestly
|
||||
* unavailable (the tool says so; the researcher sub-agent is disabled
|
||||
* by the agent layer on the same signal — that rewiring is the Teacher
|
||||
* stream's, on this port).
|
||||
*
|
||||
* Pure types + protocol docs only (browser-safe, no node imports): the
|
||||
* adapters are server-side (src/server/search.ts) and arrive as injected
|
||||
* dependencies — the agent-layer tool depends on THIS port only.
|
||||
*/
|
||||
|
||||
/** One search result row — the shape every adapter must produce (the
|
||||
* researcher's tool contract; same fields the old Google adapter emitted). */
|
||||
export interface SearchHit {
|
||||
title: string;
|
||||
url: string;
|
||||
snippet: string;
|
||||
}
|
||||
|
||||
/** The engine ids this wave ships (the catalog: curated keys + adapters). */
|
||||
export type SearchEngineId = "tavily" | "brave";
|
||||
|
||||
/** A live search engine: an id (for status/labels) + one operation. The
|
||||
* adapters own their wire protocol (fetch, timeouts, error mapping) —
|
||||
* callers see only this. `search` throws on failure (adapters map HTTP
|
||||
* errors to clear Error messages); the tool layer catches and reports. */
|
||||
export interface SearchProvider {
|
||||
id: SearchEngineId;
|
||||
search(query: string, count: number): Promise<SearchHit[]>;
|
||||
}
|
||||
|
||||
/** Why search is (un)available — the honest status the UI and the tool
|
||||
* layer explain from. `engine` is the ACTIVE engine when live. */
|
||||
export interface SearchStatus {
|
||||
available: boolean;
|
||||
/** The active engine (null when unavailable). */
|
||||
engine: SearchEngineId | null;
|
||||
/** The one-line reason for the state (status text for the UI). */
|
||||
reason: string;
|
||||
}
|
||||
@@ -24,6 +24,7 @@
|
||||
|
||||
import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
|
||||
import type { ChatSettings } from "./contracts/chat.ts";
|
||||
import type { SearchEngineId } from "./search.ts";
|
||||
import {
|
||||
BACKGROUND_MODEL_ROLES,
|
||||
FACTORY_SETTINGS,
|
||||
@@ -62,6 +63,10 @@ export interface EffectiveSettings {
|
||||
tools: Record<string, boolean>;
|
||||
/** Delta map as stored — absent key → override off. */
|
||||
prompts: Record<string, boolean>;
|
||||
/** The search-engine preference as stored (search wave, 2026-09-20):
|
||||
* passthrough delta — engine resolution (pin → auto → unavailable)
|
||||
* lives in server/search.ts, where keys are visible. */
|
||||
search: { engine?: SearchEngineId };
|
||||
};
|
||||
appearance: EffectiveAppearance;
|
||||
graph: Required<GraphSettings>;
|
||||
@@ -105,6 +110,7 @@ export function resolveSettings(record: UserSettings | null, factory: ModelRoleS
|
||||
models,
|
||||
tools: agent.tools ?? {},
|
||||
prompts: agent.prompts ?? {},
|
||||
search: agent.search ?? {},
|
||||
},
|
||||
appearance: {
|
||||
mode: appearance.mode ?? factoryAppearance.mode,
|
||||
|
||||
@@ -27,6 +27,7 @@
|
||||
|
||||
import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
|
||||
import type { ChatSettings } from "./contracts/chat.ts";
|
||||
import type { SearchEngineId } from "./search.ts";
|
||||
|
||||
/** The schema version this code understands. Bump ONLY for breaking shape
|
||||
* changes (a field renamed/meaning changed); pure additions never bump. */
|
||||
@@ -70,6 +71,13 @@ export interface AgentSettings {
|
||||
/** Tool toggles keyed by tool id (absent key = enabled). User-level
|
||||
* only — no per-chat override (owner decision 2026-09-19). */
|
||||
tools?: Record<string, boolean>;
|
||||
/** Search-engine preference (search wave, 2026-09-20). OPTIONAL by
|
||||
* owner decision: setting a key auto-activates its engine — the user
|
||||
* never chooses an engine to make search work. An explicit choice only
|
||||
* pins the preference (e.g. when BOTH keys are present); the chosen
|
||||
* engine must still have its key, else search is honestly unavailable.
|
||||
* Absent + no key = unavailable (no silent fallback, no default). */
|
||||
search?: { engine?: SearchEngineId };
|
||||
/** System-prompt override SWITCHES keyed by segment path (absent key =
|
||||
* off). Switch-on requires the override file to exist (enforced by the
|
||||
* route); the switch is explicit so users keep a file on disk but run
|
||||
@@ -261,6 +269,13 @@ function sanitizeAgent(raw: Record<string, unknown>): AgentSettings {
|
||||
if (tools) out.tools = tools;
|
||||
const prompts = sanitizeBooleanMap(raw.prompts);
|
||||
if (prompts) out.prompts = prompts;
|
||||
// Search preference: lenient read side (an unknown engine id drops;
|
||||
// the write side's parser is strict — a stale client can't brick the
|
||||
// record, it just loses the pin and auto-detection resumes).
|
||||
if (raw.search !== null && typeof raw.search === "object" && !Array.isArray(raw.search)) {
|
||||
const engine = (raw.search as Record<string, unknown>).engine;
|
||||
if (engine === "tavily" || engine === "brave") out.search = { engine };
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ export { effectivePromptSegments, allSegmentPaths, readPromptOverride } from "..
|
||||
// Importing this module runs the idempotent boot fill-gap — every consumer
|
||||
// keeps its plain process.env reads (that seam is the point).
|
||||
export * from "./keys.ts";
|
||||
export * from "./search.ts";
|
||||
export { authStore } from "../store/auth.ts";
|
||||
export * from "./settings-body.ts";
|
||||
export * from "./turn-liveness.ts";
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
/**
|
||||
* The search engines' server composition (search port + model slots wave,
|
||||
* 2026-09-20): the Tavily + Brave adapters behind the core port
|
||||
* (core/search.ts) + the live-engine resolution the settings UI and the
|
||||
* agent-layer tool both read.
|
||||
*
|
||||
* Owner decisions (2026-09-20) baked in:
|
||||
* - NO default engine: an engine is live iff its key is present (auth
|
||||
* store or env — env wins, keyed via server/keys.ts keyValue).
|
||||
* - Setting a key is ALL the user does — auto-detection: explicit
|
||||
* agent.search.engine pins the preference (when BOTH keys are present
|
||||
* the preference order is Tavily → Brave); without a pin, whichever
|
||||
* engine has a key is the engine.
|
||||
* - An explicit choice does NOT silently fall back to the other engine:
|
||||
* engine pinned but its key missing = honestly unavailable (reason
|
||||
* says so) — no guessing what the user meant.
|
||||
*
|
||||
* The adapters are injected dependencies (fetch-shaped) so the agent-layer
|
||||
* tool depends on the PORT only and tests never touch the network.
|
||||
*/
|
||||
|
||||
import type { SearchEngineId, SearchHit, SearchProvider, SearchStatus } from "../core/search.ts";
|
||||
import { keyValue } from "./keys.ts";
|
||||
import { effectiveSettings } from "./stores.ts";
|
||||
|
||||
/** The wire-callable shape the adapters close over (tests inject a stub). */
|
||||
type FetchLike = (input: string, init: RequestInit) => Promise<Response>;
|
||||
|
||||
/** Tavily: POST https://api.tavily.com/search — Bearer auth in the
|
||||
* Authorization header (the current documented scheme; the legacy
|
||||
* api_key-in-body form is retired), query + max_results (0–20) in the
|
||||
* JSON body. VERIFIED against docs.tavily.com's OpenAPI, 2026-09-20:
|
||||
* response {results: [{title, url, content, score, …}]}, errors
|
||||
* {detail: {error}}. */
|
||||
export function tavilyProvider(apiKey: string, fetchFn: FetchLike = fetch): SearchProvider {
|
||||
return {
|
||||
id: "tavily",
|
||||
async search(query: string, count: number): Promise<SearchHit[]> {
|
||||
const resp = await fetchFn("https://api.tavily.com/search", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({ query, max_results: Math.min(count, 20) }),
|
||||
signal: AbortSignal.timeout(30_000),
|
||||
});
|
||||
if (!resp.ok) {
|
||||
const body = await resp.text();
|
||||
// Their error body is {detail: {error: "…"}} — surface the
|
||||
// message when present, else the raw slice.
|
||||
let detail = body.slice(0, 200);
|
||||
try {
|
||||
const parsed = JSON.parse(body) as { detail?: { error?: string } };
|
||||
if (parsed.detail?.error) detail = parsed.detail.error.slice(0, 200);
|
||||
} catch {
|
||||
/* keep the raw slice */
|
||||
}
|
||||
throw new Error(`Tavily search API ${resp.status}: ${detail}`);
|
||||
}
|
||||
const data = (await resp.json()) as {
|
||||
results?: Array<{ title?: string; url?: string; content?: string }>;
|
||||
};
|
||||
return (data.results ?? [])
|
||||
.filter((r) => typeof r.url === "string" && r.url)
|
||||
.map((r) => ({
|
||||
title: r.title ?? r.url ?? "",
|
||||
url: r.url ?? "",
|
||||
snippet: (r.content ?? "").replace(/\n/g, " ").slice(0, 500),
|
||||
}));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Brave: GET https://api.search.brave.com/res/v1/web/search with the
|
||||
* key in the X-Subscription-Token header. VERIFIED against Brave's
|
||||
* dashboard docs, 2026-09-20: count max 20; response
|
||||
* {web: {results: [{title, url, description, …}]}}. */
|
||||
export function braveProvider(apiKey: string, fetchFn: FetchLike = fetch): SearchProvider {
|
||||
return {
|
||||
id: "brave",
|
||||
async search(query: string, count: number): Promise<SearchHit[]> {
|
||||
const url = new URL("https://api.search.brave.com/res/v1/web/search");
|
||||
url.searchParams.set("q", query);
|
||||
url.searchParams.set("count", String(Math.min(count, 20)));
|
||||
const resp = await fetchFn(url.toString(), {
|
||||
headers: {
|
||||
accept: "application/json",
|
||||
"x-subscription-token": apiKey,
|
||||
},
|
||||
signal: AbortSignal.timeout(30_000),
|
||||
});
|
||||
if (!resp.ok) {
|
||||
const body = await resp.text();
|
||||
throw new Error(`Brave search API ${resp.status}: ${body.slice(0, 200)}`);
|
||||
}
|
||||
const data = (await resp.json()) as {
|
||||
web?: { results?: Array<{ title?: string; url?: string; description?: string }> };
|
||||
};
|
||||
return (data.web?.results ?? [])
|
||||
.filter((r) => typeof r.url === "string" && r.url)
|
||||
.map((r) => ({
|
||||
title: r.title ?? r.url ?? "",
|
||||
url: r.url ?? "",
|
||||
snippet: (r.description ?? "").replace(/\n/g, " ").slice(0, 500),
|
||||
}));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** The engines this wave ships, in PREFERENCE order (both keys present,
|
||||
* no explicit pin → the first live engine wins; owner decision
|
||||
* 2026-09-20: Tavily first — the AI-native search API the wave chose). */
|
||||
const ENGINE_ORDER: ReadonlyArray<{ id: SearchEngineId; envVar: string }> = [
|
||||
{ id: "tavily", envVar: "TAVILY_API_KEY" },
|
||||
{ id: "brave", envVar: "BRAVE_API_KEY" },
|
||||
];
|
||||
|
||||
/** The engine's key (env-or-file; env wins) — null when the engine is
|
||||
* not keyed, which by owner decision means NOT LIVE. */
|
||||
function engineKey(id: SearchEngineId): string | null {
|
||||
const envVar = ENGINE_ORDER.find((e) => e.id === id)?.envVar;
|
||||
const value = envVar ? keyValue(envVar) : undefined;
|
||||
return value ? value : null;
|
||||
}
|
||||
|
||||
/** Resolve the search state PURELY from the given pins (testable): the
|
||||
* explicit engine (if pinned) wins but must be keyed; absent pin → first
|
||||
* keyed engine in preference order. Never throws. */
|
||||
export function resolveSearchEngineFor(
|
||||
pinned: SearchEngineId | undefined,
|
||||
/** Per-engine live keys (absent engine = absent property is fine). */
|
||||
keys: Partial<Record<SearchEngineId, string | undefined>>,
|
||||
): SearchStatus & { key: string | null } {
|
||||
const pinnedKey = pinned ? keys[pinned] ?? null : null;
|
||||
if (pinned) {
|
||||
if (!pinnedKey) {
|
||||
return {
|
||||
available: false,
|
||||
engine: null,
|
||||
reason: `search engine is set to "${pinned}" but its API key is missing — set the ${pinned === "tavily" ? "TAVILY_API_KEY" : "BRAVE_API_KEY"} key`,
|
||||
key: null,
|
||||
};
|
||||
}
|
||||
return { available: true, engine: pinned, reason: `search engine: ${pinned} (chosen)`, key: pinnedKey };
|
||||
}
|
||||
for (const { id } of ENGINE_ORDER) {
|
||||
const key = keys[id] ?? null;
|
||||
if (key) return { available: true, engine: id, reason: `search engine: ${id} (auto)`, key };
|
||||
}
|
||||
return {
|
||||
available: false,
|
||||
engine: null,
|
||||
reason: "no search engine configured — set a Tavily or Brave search API key",
|
||||
key: null,
|
||||
};
|
||||
}
|
||||
|
||||
/** The live search state (the honest availability signal): reads the
|
||||
* settings record's pin + the auth store fresh each call, so a key saved
|
||||
* mid-session activates on the next check (no restart). */
|
||||
export function searchStatus(): SearchStatus {
|
||||
const { available, engine, reason } = resolveSearchEngineFor(
|
||||
effectiveSettings().agent.search?.engine,
|
||||
{
|
||||
tavily: engineKey("tavily") ?? undefined,
|
||||
brave: engineKey("brave") ?? undefined,
|
||||
},
|
||||
);
|
||||
return { available, engine, reason };
|
||||
}
|
||||
|
||||
/** Search available at all? (the settings UI's gray-out signal + the
|
||||
* researcher's enable/disable pin — Teacher stream's rewiring reads this.) */
|
||||
export function searchAvailable(): boolean {
|
||||
return searchStatus().available;
|
||||
}
|
||||
|
||||
/** The LIVE adapter, or null when unavailable. Composed fresh each call —
|
||||
* the tool layer (Teacher stream) calls this once per search, so key
|
||||
* rotations take effect immediately. */
|
||||
export function searchProvider(): SearchProvider | null {
|
||||
const resolved = resolveSearchEngineFor(effectiveSettings().agent.search?.engine, {
|
||||
tavily: engineKey("tavily") ?? undefined,
|
||||
brave: engineKey("brave") ?? undefined,
|
||||
});
|
||||
if (!resolved.available || !resolved.key) return null;
|
||||
return resolved.engine === "tavily" ? tavilyProvider(resolved.key) : braveProvider(resolved.key);
|
||||
}
|
||||
@@ -184,6 +184,16 @@ export function parseAgentSettingsBody(body: unknown): SettingsBodyResult<AgentS
|
||||
const prompts = parseBooleanMap(raw.prompts, "prompts");
|
||||
if (prompts.error) return { ok: false, error: prompts.error };
|
||||
if (prompts.map) settings.prompts = prompts.map;
|
||||
if (raw.search !== undefined) {
|
||||
const search = asRecord(raw.search);
|
||||
if (!search) return { ok: false, error: "search must be an object" };
|
||||
if (search.engine !== undefined) {
|
||||
if (search.engine !== "tavily" && search.engine !== "brave") {
|
||||
return { ok: false, error: 'search.engine must be "tavily" or "brave"' };
|
||||
}
|
||||
settings.search = { engine: search.engine };
|
||||
}
|
||||
}
|
||||
return { ok: true, settings };
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user