From 9649837a633845ffc602ff2661e84e3727840af4 Mon Sep 17 00:00:00 2001 From: Yasha Date: Fri, 19 Jun 2026 09:49:29 +0200 Subject: [PATCH] feat: add WebSearch and WebFetch tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #78. Adds two provider-agnostic agent tools to @harnext/core so the agent can ground answers in current information and read docs/pages on demand, bringing harnext to parity with Claude Code / Codex CLI / Gemini CLI. WebFetch (web_fetch) — packages/core/src/tools/web-fetch.ts - URL + optional extraction prompt; fetches, converts HTML→Markdown, returns readable text. Plain text passes through; binary is described, not dumped. - SSRF protection (packages/core/src/tools/ssrf.ts): http→https upgrade, http/https-only scheme, refuses internal hostnames and private/reserved IPs (incl. cloud metadata 169.254.169.254 and DNS-rebinding via resolved-IP checks), 30s timeout, 10 MB body cap, manual redirects re-validated every hop — same-host followed (≤10), cross-host returned for the agent to re-issue. - Large pages truncated via the existing truncateTail (256 KB), surfaced in the output header. (Small-model summarization deferred per open question #2.) WebSearch (web_search) — packages/core/src/tools/web-search.ts - query + allowed_domains / blocked_domains / count; returns {title,url,snippet}. - Pluggable backends selected from env (Tavily → SearXNG → Serper → Brave), overridable with HARNEXT_SEARCH_BACKEND. No DuckDuckGo/Bing. - No backend configured → clear, actionable setup message (no crash). HTML→Markdown (packages/core/src/tools/html-to-markdown.ts) is implemented dependency-free (no turndown/jsdom) to keep @harnext/core light and tests hermetic — answers open question #4. Wiring: registered in tools/index.ts (codingTools, allTools, createCodingTools, createAllTools) and re-exported from the core barrel; CLI render.ts gives both a distinct badge and arg display. No runtime changes needed. Tests (54): SSRF rejection/redirect/DNS-rebinding, HTML→markdown conversion, truncation, content-type handling, and search backend selection/parsing/filtering. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/cli/src/modes/interactive/render.ts | 6 + packages/core/src/index.ts | 14 + packages/core/src/tools/html-to-markdown.ts | 154 +++++++++ packages/core/src/tools/index.ts | 28 ++ packages/core/src/tools/ssrf.ts | 138 ++++++++ packages/core/src/tools/web-fetch.ts | 281 ++++++++++++++++ packages/core/src/tools/web-search.ts | 330 +++++++++++++++++++ packages/core/tests/html-to-markdown.test.ts | 65 ++++ packages/core/tests/ssrf.test.ts | 97 ++++++ packages/core/tests/web-fetch.test.ts | 136 ++++++++ packages/core/tests/web-search.test.ts | 151 +++++++++ 11 files changed, 1400 insertions(+) create mode 100644 packages/core/src/tools/html-to-markdown.ts create mode 100644 packages/core/src/tools/ssrf.ts create mode 100644 packages/core/src/tools/web-fetch.ts create mode 100644 packages/core/src/tools/web-search.ts create mode 100644 packages/core/tests/html-to-markdown.test.ts create mode 100644 packages/core/tests/ssrf.test.ts create mode 100644 packages/core/tests/web-fetch.test.ts create mode 100644 packages/core/tests/web-search.test.ts diff --git a/packages/cli/src/modes/interactive/render.ts b/packages/cli/src/modes/interactive/render.ts index 1cf4dd2..0362317 100644 --- a/packages/cli/src/modes/interactive/render.ts +++ b/packages/cli/src/modes/interactive/render.ts @@ -201,6 +201,8 @@ function toolColor(name: string): RoleColor { case 'glob': case 'ls': case 'find': + case 'web_fetch': + case 'web_search': return { idx: X.accent, ink: c.accent }; case 'edit': case 'write': @@ -241,6 +243,10 @@ export function toolStart(name: string, args: Record): string { arg = c.green('$ ') + c.bright(cmd); } else if (name === 'read' || name === 'write' || name === 'edit') { arg = c.bright(fitToWidth(String(args.path ?? ''), argBudget)); + } else if (name === 'web_fetch') { + arg = c.bright(truncateOneLine(String(args.url ?? ''), argBudget)); + } else if (name === 'web_search') { + arg = c.bright(truncateOneLine(String(args.query ?? ''), argBudget)); } else { arg = c.dim(truncateOneLine(JSON.stringify(args), argBudget)); } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c86b4b5..2869c47 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -627,6 +627,20 @@ export { type HeartbeatToolDetails, type HeartbeatToolInput, type CreateHeartbeatToolOptions, + webFetchTool, + createWebFetchTool, + type WebFetchToolDetails, + type WebFetchToolInput, + type CreateWebFetchToolOptions, + webSearchTool, + createWebSearchTool, + selectBackend, + type WebSearchToolDetails, + type WebSearchToolInput, + type CreateWebSearchToolOptions, + type SearchResult, + type SearchBackend, + type SearchBackendKind, DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, formatSize, diff --git a/packages/core/src/tools/html-to-markdown.ts b/packages/core/src/tools/html-to-markdown.ts new file mode 100644 index 0000000..0b87139 --- /dev/null +++ b/packages/core/src/tools/html-to-markdown.ts @@ -0,0 +1,154 @@ +/** + * Lightweight, dependency-free HTML→Markdown conversion for the WebFetch tool. + * + * The big harnesses (Claude Code, opencode) use `turndown`, which in Node also + * pulls in a DOM implementation (jsdom). To keep `@harnext/core` dependency-light + * and its tests hermetic, we implement a focused converter that handles the tags + * that matter for agent reading — headings, links, lists, emphasis, code, and + * block structure — and strips everything else. It is intentionally lossy on + * exotic markup; the goal is readable text, not a faithful round-trip. + */ + +const NAMED_ENTITIES: Record = { + amp: '&', + lt: '<', + gt: '>', + quot: '"', + apos: "'", + nbsp: ' ', + copy: '©', + reg: '®', + trade: '™', + hellip: '…', + mdash: '—', + ndash: '–', + lsquo: '‘', + rsquo: '’', + ldquo: '“', + rdquo: '”', + laquo: '«', + raquo: '»', + deg: '°', + middot: '·', + bull: '•', +}; + +/** Decode HTML entities (named + decimal/hex numeric) into plain text. */ +export function decodeEntities(text: string): string { + return text.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*);/g, (match, body: string) => { + if (body[0] === '#') { + const code = + body[1] === 'x' || body[1] === 'X' + ? parseInt(body.slice(2), 16) + : parseInt(body.slice(1), 10); + if (Number.isFinite(code) && code > 0 && code <= 0x10ffff) { + try { + return String.fromCodePoint(code); + } catch { + return match; + } + } + return match; + } + const named = NAMED_ENTITIES[body.toLowerCase()]; + return named ?? match; + }); +} + +/** Pull the contents of the first tag, if any. */ +export function extractTitle(html: string): string | undefined { + const m = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i); + if (!m) return undefined; + const text = decodeEntities(m[1].replace(/\s+/g, ' ')).trim(); + return text.length > 0 ? text : undefined; +} + +/** Remove elements whose contents are never useful as reading material. */ +function stripNonContent(html: string): string { + return html + .replace(/<!--[\s\S]*?-->/g, '') + .replace(/<script\b[\s\S]*?<\/script>/gi, '') + .replace(/<style\b[\s\S]*?<\/style>/gi, '') + .replace(/<noscript\b[\s\S]*?<\/noscript>/gi, '') + .replace(/<template\b[\s\S]*?<\/template>/gi, '') + .replace(/<svg\b[\s\S]*?<\/svg>/gi, '') + .replace(/<head\b[\s\S]*?<\/head>/gi, ''); +} + +/** + * Convert an HTML document or fragment to Markdown-ish plain text. + */ +export function htmlToMarkdown(html: string): string { + let s = stripNonContent(html); + + // Headings. + for (let level = 1; level <= 6; level++) { + const hashes = '#'.repeat(level); + s = s.replace( + new RegExp(`<h${level}[^>]*>([\\s\\S]*?)<\\/h${level}>`, 'gi'), + (_m, inner: string) => `\n\n${hashes} ${collapseInline(inner)}\n\n`, + ); + } + + // Preformatted / code blocks — preserve internal whitespace, fence them. + s = s.replace(/<pre[^>]*>([\s\S]*?)<\/pre>/gi, (_m, inner: string) => { + const code = decodeEntities(stripTags(inner.replace(/<br\s*\/?>(?=)/gi, '\n'))); + return `\n\n\`\`\`\n${code.replace(/\n+$/, '')}\n\`\`\`\n\n`; + }); + + // Inline code. + s = s.replace(/<code[^>]*>([\s\S]*?)<\/code>/gi, (_m, inner: string) => { + return '`' + collapseInline(inner) + '`'; + }); + + // Links — [text](href). + s = s.replace( + /<a\b[^>]*\bhref\s*=\s*(["'])(.*?)\1[^>]*>([\s\S]*?)<\/a>/gi, + (_m, _q, href: string, inner: string) => { + const text = collapseInline(inner); + const url = decodeEntities(href).trim(); + if (!text) return url; + if (!url || url.startsWith('javascript:')) return text; + return `[${text}](${url})`; + }, + ); + + // Emphasis. + s = s.replace(/<(strong|b)\b[^>]*>([\s\S]*?)<\/\1>/gi, (_m, _t, inner: string) => `**${collapseInline(inner)}**`); + s = s.replace(/<(em|i)\b[^>]*>([\s\S]*?)<\/\1>/gi, (_m, _t, inner: string) => `*${collapseInline(inner)}*`); + + // List items. + s = s.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, (_m, inner: string) => `\n- ${collapseInline(inner)}`); + + // Block boundaries → blank lines. + s = s.replace(/<\/(p|div|section|article|header|footer|ul|ol|table|tr|blockquote|h[1-6])\s*>/gi, '\n\n'); + s = s.replace(/<br\s*\/?>(?=)/gi, '\n'); + s = s.replace(/<\/(td|th)\s*>/gi, ' \t '); + + // Drop everything else and decode entities. + s = stripTags(s); + s = decodeEntities(s); + + // Normalize whitespace: trim trailing spaces, collapse 3+ newlines. + s = s + .split('\n') + .map((line) => line.replace(/[ \t]+$/g, '').replace(/^[ \t]+/g, (m) => (m.length > 0 ? '' : m))) + .join('\n') + .replace(/\n{3,}/g, '\n\n') + .replace(/[ \t]{2,}/g, ' ') + .trim(); + + return s; +} + +/** Strip all remaining tags. */ +function stripTags(s: string): string { + return s.replace(/<[^>]+>/g, ''); +} + +/** Reduce an inline fragment to single-line text (tags removed, entities decoded). */ +function collapseInline(inner: string): string { + return decodeEntities(stripTags(inner)) + .replace(/\s+/g, ' ') + .trim(); +} diff --git a/packages/core/src/tools/index.ts b/packages/core/src/tools/index.ts index 8935fe0..9a61ade 100644 --- a/packages/core/src/tools/index.ts +++ b/packages/core/src/tools/index.ts @@ -66,6 +66,24 @@ export { heartbeatTool, createHeartbeatTool, } from './heartbeat-tool.js'; +export { + type WebFetchToolDetails, + type WebFetchToolInput, + type CreateWebFetchToolOptions, + webFetchTool, + createWebFetchTool, +} from './web-fetch.js'; +export { + type WebSearchToolDetails, + type WebSearchToolInput, + type CreateWebSearchToolOptions, + type SearchResult, + type SearchBackend, + type SearchBackendKind, + webSearchTool, + createWebSearchTool, + selectBackend, +} from './web-search.js'; export { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, @@ -86,6 +104,8 @@ import { heartbeatTool, createHeartbeatTool, type CreateHeartbeatToolOptions } f import { memoryTool, createMemoryTool } from './memory.js'; import { readTool, createReadTool } from './read.js'; import { todoTool, createTodoTool } from './todo.js'; +import { webFetchTool, createWebFetchTool } from './web-fetch.js'; +import { webSearchTool, createWebSearchTool } from './web-search.js'; import { writeTool, createWriteTool } from './write.js'; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -100,6 +120,8 @@ export const codingTools: Tool[] = [ exitPlanTool, memoryTool, heartbeatTool, + webFetchTool, + webSearchTool, ]; export const allTools = { @@ -111,6 +133,8 @@ export const allTools = { exit_plan: exitPlanTool, memory: memoryTool, heartbeat: heartbeatTool, + web_fetch: webFetchTool, + web_search: webSearchTool, }; export type ToolName = keyof typeof allTools; @@ -137,6 +161,8 @@ export function createCodingTools(cwd: string, options: CreateCodingToolsOptions createExitPlanTool(), createMemoryTool(cwd), createHeartbeatTool(cwd, options.heartbeat), + createWebFetchTool(), + createWebSearchTool(), ]; // The background-shell companions only exist when a manager is wired in // (i.e. background execution is enabled for this session). @@ -156,5 +182,7 @@ export function createAllTools(cwd: string): Record<ToolName, Tool> { exit_plan: createExitPlanTool(), memory: createMemoryTool(cwd), heartbeat: createHeartbeatTool(cwd), + web_fetch: createWebFetchTool(), + web_search: createWebSearchTool(), }; } diff --git a/packages/core/src/tools/ssrf.ts b/packages/core/src/tools/ssrf.ts new file mode 100644 index 0000000..666619a --- /dev/null +++ b/packages/core/src/tools/ssrf.ts @@ -0,0 +1,138 @@ +import { lookup } from 'node:dns/promises'; +import { isIP } from 'node:net'; + +/** + * SSRF protection for the web tools, modeled on Claude Code's WebFetch safety: + * restrict the scheme to http/https, upgrade http→https, and refuse any URL + * whose host resolves to a private, loopback, link-local, or otherwise + * internal address (including the cloud metadata IP 169.254.169.254). The + * target is re-validated on every redirect hop by the fetch pipeline. + */ +export class SsrfError extends Error { + constructor(message: string) { + super(message); + this.name = 'SsrfError'; + } +} + +/** + * Parse a raw URL, enforce an http/https scheme, and upgrade http→https. + * Throws {@link SsrfError} for unsupported schemes or unparseable input. + */ +export function normalizeUrl(raw: string): URL { + let url: URL; + try { + url = new URL(raw); + } catch { + throw new SsrfError(`Invalid URL: ${raw}`); + } + if (url.protocol === 'http:') { + url.protocol = 'https:'; + } + if (url.protocol !== 'https:') { + throw new SsrfError( + `Unsupported URL scheme "${url.protocol.replace(/:$/, '')}" — only http and https are allowed.`, + ); + } + return url; +} + +/** Hostnames that are always internal regardless of DNS resolution. */ +function isBlockedHostname(hostname: string): boolean { + const host = hostname.toLowerCase().replace(/\.$/, ''); + if (host === 'localhost') return true; + // Reserved / internal TLDs that should never leave the machine. + if (/\.(local|localhost|internal|intranet|home|lan|corp)$/.test(host)) return true; + return false; +} + +/** True for an IPv4 address string in a private / reserved / internal range. */ +export function isPrivateIPv4(ip: string): boolean { + const parts = ip.split('.').map((p) => Number(p)); + if (parts.length !== 4 || parts.some((n) => !Number.isInteger(n) || n < 0 || n > 255)) { + return false; + } + const [a, b] = parts; + if (a === 0) return true; // 0.0.0.0/8 "this host" + if (a === 10) return true; // 10.0.0.0/8 private + if (a === 127) return true; // 127.0.0.0/8 loopback + if (a === 169 && b === 254) return true; // 169.254.0.0/16 link-local (incl. metadata) + if (a === 172 && b >= 16 && b <= 31) return true; // 172.16.0.0/12 private + if (a === 192 && b === 168) return true; // 192.168.0.0/16 private + if (a === 100 && b >= 64 && b <= 127) return true; // 100.64.0.0/10 CGNAT + if (a >= 224) return true; // 224.0.0.0/4 multicast + 240/4 reserved + return false; +} + +/** True for an IPv6 address string in a loopback / ULA / link-local / mapped range. */ +export function isPrivateIPv6(ip: string): boolean { + let addr = ip.toLowerCase(); + // Strip a zone id (e.g. fe80::1%eth0). + const pct = addr.indexOf('%'); + if (pct !== -1) addr = addr.slice(0, pct); + + // IPv4-mapped / -compatible (::ffff:127.0.0.1, ::127.0.0.1) — judge the v4 part. + const v4 = addr.match(/(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/); + if (v4) return isPrivateIPv4(v4[1]); + + if (addr === '::1' || addr === '::') return true; // loopback / unspecified + if (addr.startsWith('fe80') || addr.startsWith('fe9') || addr.startsWith('fea') || addr.startsWith('feb')) + return true; // fe80::/10 link-local + if (/^f[cd]/.test(addr)) return true; // fc00::/7 unique local + if (addr.startsWith('ff')) return true; // ff00::/8 multicast + return false; +} + +/** True if a literal IP address (v4 or v6) is private / internal. */ +export function isPrivateAddress(ip: string): boolean { + const kind = isIP(ip); + if (kind === 4) return isPrivateIPv4(ip); + if (kind === 6) return isPrivateIPv6(ip); + return false; +} + +/** + * Validate that a (already scheme-normalized) URL's host is safe to fetch: + * reject internal hostnames outright, and resolve DNS names to confirm every + * resolved address is public. Throws {@link SsrfError} on any violation. + */ +export async function assertHostAllowed( + url: URL, + resolver: (host: string) => Promise<string[]> = defaultResolver, +): Promise<void> { + const host = url.hostname.replace(/^\[|\]$/g, ''); // unwrap [::1]-style literals + + if (isBlockedHostname(host)) { + throw new SsrfError(`Refusing to fetch internal host "${host}".`); + } + + // Literal IP — no DNS needed. + if (isIP(host)) { + if (isPrivateAddress(host)) { + throw new SsrfError(`Refusing to fetch private/internal address "${host}".`); + } + return; + } + + let addresses: string[]; + try { + addresses = await resolver(host); + } catch (err) { + throw new SsrfError( + `Could not resolve host "${host}": ${err instanceof Error ? err.message : String(err)}`, + ); + } + if (addresses.length === 0) { + throw new SsrfError(`Host "${host}" did not resolve to any address.`); + } + for (const addr of addresses) { + if (isPrivateAddress(addr)) { + throw new SsrfError(`Host "${host}" resolves to a private/internal address (${addr}).`); + } + } +} + +async function defaultResolver(host: string): Promise<string[]> { + const records = await lookup(host, { all: true }); + return records.map((r) => r.address); +} diff --git a/packages/core/src/tools/web-fetch.ts b/packages/core/src/tools/web-fetch.ts new file mode 100644 index 0000000..c2ff7b3 --- /dev/null +++ b/packages/core/src/tools/web-fetch.ts @@ -0,0 +1,281 @@ +import type { AgentTool } from '@earendil-works/pi-agent-core'; +import { Type, type Static } from '@sinclair/typebox'; + +import { VERSION } from '../config.js'; +import { extractTitle, htmlToMarkdown } from './html-to-markdown.js'; +import { assertHostAllowed, normalizeUrl, SsrfError } from './ssrf.js'; +import { DEFAULT_MAX_BYTES, formatSize, truncateTail } from './truncate.js'; + +const DEFAULT_TIMEOUT_MS = 30_000; +const MAX_FETCH_BYTES = 10 * 1024 * 1024; // 10 MB hard cap on the response body +const MAX_REDIRECTS = 10; + +const USER_AGENT = `harnext/${VERSION} (+https://github.com/QualityUnit/harnext)`; + +const webFetchSchema = Type.Object({ + url: Type.String({ description: 'The URL to fetch (http or https; http is upgraded to https).' }), + prompt: Type.Optional( + Type.String({ + description: + 'Optional: what to extract or look for in the page. Recorded with the result to focus your reading.', + }), + ), +}); + +export type WebFetchToolInput = Static<typeof webFetchSchema>; + +export interface WebFetchToolDetails { + url: string; + finalUrl?: string; + status?: number; + contentType?: string; + bytes: number; + truncated: boolean; + title?: string; + error?: string; +} + +export interface CreateWebFetchToolOptions { + /** Overall request timeout in milliseconds (default 30s). */ + timeoutMs?: number; + /** Override the fetch implementation (for tests). */ + fetchImpl?: typeof fetch; + /** Override DNS resolution used by the SSRF guard (for tests). */ + resolver?: (host: string) => Promise<string[]>; +} + +interface FetchOutcome { + finalUrl: string; + status: number; + contentType: string; + body: Buffer; + /** Set when a cross-host redirect was encountered and not followed. */ + crossHostRedirect?: string; +} + +export function createWebFetchTool( + options: CreateWebFetchToolOptions = {}, +): AgentTool<typeof webFetchSchema, WebFetchToolDetails> { + const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const doFetch = options.fetchImpl ?? fetch; + const resolver = options.resolver; + + return { + name: 'web_fetch', + label: 'web_fetch', + description: + 'Fetch a URL and return its readable content. HTML is converted to Markdown; other text is returned as-is. ' + + 'Enforces SSRF protections (private/internal hosts are refused, http is upgraded to https) and re-validates on ' + + 'redirects (same-host redirects are followed; cross-host redirects return the new URL for you to re-issue). ' + + `Large pages are truncated to ${formatSize(DEFAULT_MAX_BYTES)}. Optionally pass a prompt describing what to look for.`, + parameters: webFetchSchema, + async execute(_toolCallId, params) { + const { url: rawUrl, prompt } = params; + let start: URL; + try { + start = normalizeUrl(rawUrl); + } catch (err) { + return errorResult(rawUrl, err); + } + + let outcome: FetchOutcome; + try { + outcome = await fetchWithRedirects(start, { timeoutMs, doFetch, resolver }); + } catch (err) { + return errorResult(rawUrl, err); + } + + if (outcome.crossHostRedirect) { + const text = + `The URL redirected to a different host: ${outcome.crossHostRedirect}\n` + + `Cross-host redirects are not followed automatically. Re-issue web_fetch with that URL if you want to follow it.`; + return { + content: [{ type: 'text', text }], + details: { + url: rawUrl, + finalUrl: outcome.crossHostRedirect, + status: outcome.status, + bytes: 0, + truncated: false, + }, + }; + } + + if (outcome.status >= 400) { + return { + content: [ + { type: 'text', text: `HTTP ${outcome.status} fetching ${outcome.finalUrl}` }, + ], + details: { + url: rawUrl, + finalUrl: outcome.finalUrl, + status: outcome.status, + contentType: outcome.contentType, + bytes: outcome.body.length, + truncated: false, + error: `HTTP ${outcome.status}`, + }, + }; + } + + const { text, title } = renderBody(outcome); + const truncation = truncateTail(text); + const header: string[] = []; + if (prompt) header.push(`Requested: ${prompt}`); + header.push(`URL: ${outcome.finalUrl}`); + if (title) header.push(`Title: ${title}`); + header.push(`Content-Type: ${outcome.contentType || 'unknown'}`); + if (truncation.truncated) { + header.push( + `(truncated to last ${truncation.outputLines} of ${truncation.totalLines} lines, ${formatSize(DEFAULT_MAX_BYTES)} cap)`, + ); + } + + return { + content: [{ type: 'text', text: `${header.join('\n')}\n\n${truncation.content}` }], + details: { + url: rawUrl, + finalUrl: outcome.finalUrl, + status: outcome.status, + contentType: outcome.contentType, + bytes: outcome.body.length, + truncated: truncation.truncated, + title, + }, + }; + }, + }; +} + +/** Decide how to present the response body based on its content type. */ +function renderBody(outcome: FetchOutcome): { + text: string; + title?: string; + rendered: boolean; +} { + const ct = outcome.contentType.toLowerCase(); + const isHtml = ct.includes('text/html') || ct.includes('application/xhtml'); + const isText = + isHtml || + ct.startsWith('text/') || + ct.includes('json') || + ct.includes('xml') || + ct.includes('javascript') || + ct === ''; + + if (!isText) { + return { + text: `[non-text content: ${outcome.contentType || 'unknown'}, ${formatSize(outcome.body.length)}] — not rendered.`, + rendered: false, + }; + } + + const raw = outcome.body.toString('utf-8'); + if (isHtml) { + return { text: htmlToMarkdown(raw), title: extractTitle(raw), rendered: true }; + } + return { text: raw, rendered: false }; +} + +/** + * Fetch a URL, following same-host redirects up to {@link MAX_REDIRECTS} hops and + * re-validating the SSRF guard at every hop. A cross-host redirect stops the loop + * and is reported back rather than followed. + */ +async function fetchWithRedirects( + start: URL, + opts: { + timeoutMs: number; + doFetch: typeof fetch; + resolver?: (host: string) => Promise<string[]>; + }, +): Promise<FetchOutcome> { + let current = start; + + for (let hop = 0; hop <= MAX_REDIRECTS; hop++) { + await assertHostAllowed(current, opts.resolver); + + const res = await opts.doFetch(current.toString(), { + method: 'GET', + redirect: 'manual', + headers: { 'User-Agent': USER_AGENT, Accept: 'text/html,application/xhtml+xml,text/*;q=0.9,*/*;q=0.8' }, + signal: AbortSignal.timeout(opts.timeoutMs), + }); + + // Redirect status with a Location header → validate and decide. + if (res.status >= 300 && res.status < 400) { + const location = res.headers.get('location'); + if (!location) { + // No Location — treat as a terminal response. + return finalize(res, current); + } + let next: URL; + try { + next = normalizeUrl(new URL(location, current).toString()); + } catch (err) { + throw err instanceof SsrfError ? err : new SsrfError(`Invalid redirect target: ${location}`); + } + if (next.host !== current.host) { + return { + finalUrl: current.toString(), + status: res.status, + contentType: '', + body: Buffer.alloc(0), + crossHostRedirect: next.toString(), + }; + } + current = next; + continue; + } + + return finalize(res, current); + } + + throw new SsrfError(`Too many redirects (>${MAX_REDIRECTS}) starting from ${start.toString()}`); +} + +async function finalize(res: Response, current: URL): Promise<FetchOutcome> { + const contentType = res.headers.get('content-type') ?? ''; + const body = await readCapped(res, MAX_FETCH_BYTES); + return { finalUrl: current.toString(), status: res.status, contentType, body }; +} + +/** Read a response body up to a byte cap, stopping early once the cap is hit. */ +async function readCapped(res: Response, cap: number): Promise<Buffer> { + if (!res.body) { + const buf = Buffer.from(await res.arrayBuffer()); + return buf.length > cap ? buf.subarray(0, cap) : buf; + } + const reader = res.body.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + while (total < cap) { + const { done, value } = await reader.read(); + if (done) break; + if (value) { + chunks.push(value); + total += value.byteLength; + } + } + try { + await reader.cancel(); + } catch { + // best-effort: body may already be drained + } + const buf = Buffer.concat(chunks.map((c) => Buffer.from(c))); + return buf.length > cap ? buf.subarray(0, cap) : buf; +} + +function errorResult(url: string, err: unknown): { + content: { type: 'text'; text: string }[]; + details: WebFetchToolDetails; +} { + const message = err instanceof Error ? err.message : String(err); + const prefix = err instanceof SsrfError ? 'Blocked' : 'Error fetching'; + return { + content: [{ type: 'text', text: `${prefix} ${url}: ${message}` }], + details: { url, bytes: 0, truncated: false, error: message }, + }; +} + +export const webFetchTool = createWebFetchTool(); diff --git a/packages/core/src/tools/web-search.ts b/packages/core/src/tools/web-search.ts new file mode 100644 index 0000000..036aecc --- /dev/null +++ b/packages/core/src/tools/web-search.ts @@ -0,0 +1,330 @@ +import type { AgentTool } from '@earendil-works/pi-agent-core'; +import { Type, type Static } from '@sinclair/typebox'; + +const DEFAULT_TIMEOUT_MS = 30_000; +const DEFAULT_COUNT = 5; +const MAX_COUNT = 20; + +const webSearchSchema = Type.Object({ + query: Type.String({ description: 'The search query.' }), + allowed_domains: Type.Optional( + Type.Array(Type.String(), { + description: 'Only include results whose host matches one of these domains.', + }), + ), + blocked_domains: Type.Optional( + Type.Array(Type.String(), { + description: 'Exclude results whose host matches one of these domains.', + }), + ), + count: Type.Optional( + Type.Number({ description: `Number of results to return (default ${DEFAULT_COUNT}, max ${MAX_COUNT}).` }), + ), +}); + +export type WebSearchToolInput = Static<typeof webSearchSchema>; + +export interface SearchResult { + title: string; + url: string; + snippet: string; +} + +export interface WebSearchToolDetails { + query: string; + backend?: string; + resultCount: number; + error?: string; +} + +export type SearchBackendKind = 'tavily' | 'searxng' | 'serper' | 'brave'; + +export interface SearchBackend { + kind: SearchBackendKind; + /** API key (tavily/serper/brave) or instance base URL (searxng). */ + credential: string; +} + +/** + * Pick a search backend from environment variables. harnext is provider-agnostic, + * so rather than leaning on one model provider's hosted search we support several + * pluggable backends, preferring the ones with the least setup. `HARNEXT_SEARCH_BACKEND` + * forces a specific backend; otherwise the first configured one wins. + */ +export function selectBackend(env: NodeJS.ProcessEnv = process.env): SearchBackend | undefined { + const candidates: { kind: SearchBackendKind; credential: string | undefined }[] = [ + { kind: 'tavily', credential: env.TAVILY_API_KEY }, + { kind: 'searxng', credential: env.SEARXNG_URL ?? env.SEARXNG_INSTANCE_URL }, + { kind: 'serper', credential: env.SERPER_API_KEY }, + { kind: 'brave', credential: env.BRAVE_API_KEY ?? env.BRAVE_SEARCH_API_KEY }, + ]; + + const forced = env.HARNEXT_SEARCH_BACKEND?.trim().toLowerCase(); + if (forced) { + const match = candidates.find((c) => c.kind === forced); + if (match?.credential) return { kind: match.kind, credential: match.credential }; + return undefined; + } + + for (const c of candidates) { + if (c.credential) return { kind: c.kind, credential: c.credential }; + } + return undefined; +} + +export const NO_BACKEND_MESSAGE = + 'No web search backend is configured. Set one of these environment variables and retry:\n' + + ' - TAVILY_API_KEY — Tavily (1,000 free searches/mo, no card; https://tavily.com)\n' + + ' - SEARXNG_URL — a SearXNG instance base URL (keyless, self-hostable)\n' + + ' - SERPER_API_KEY — Serper.dev (https://serper.dev)\n' + + ' - BRAVE_API_KEY — Brave Search API (https://brave.com/search/api)\n' + + 'Force a specific backend with HARNEXT_SEARCH_BACKEND=tavily|searxng|serper|brave.'; + +// ── Result parsing (pure; exported for tests) ──────────────────────── + +export function parseTavily(json: unknown): SearchResult[] { + const results = (json as { results?: unknown[] })?.results; + if (!Array.isArray(results)) return []; + return results + .map((r) => { + const o = r as Record<string, unknown>; + return { + title: String(o.title ?? ''), + url: String(o.url ?? ''), + snippet: String(o.content ?? o.snippet ?? ''), + }; + }) + .filter((r) => r.url); +} + +export function parseSearxng(json: unknown): SearchResult[] { + const results = (json as { results?: unknown[] })?.results; + if (!Array.isArray(results)) return []; + return results + .map((r) => { + const o = r as Record<string, unknown>; + return { + title: String(o.title ?? ''), + url: String(o.url ?? ''), + snippet: String(o.content ?? ''), + }; + }) + .filter((r) => r.url); +} + +export function parseSerper(json: unknown): SearchResult[] { + const organic = (json as { organic?: unknown[] })?.organic; + if (!Array.isArray(organic)) return []; + return organic + .map((r) => { + const o = r as Record<string, unknown>; + return { + title: String(o.title ?? ''), + url: String(o.link ?? ''), + snippet: String(o.snippet ?? ''), + }; + }) + .filter((r) => r.url); +} + +export function parseBrave(json: unknown): SearchResult[] { + const results = (json as { web?: { results?: unknown[] } })?.web?.results; + if (!Array.isArray(results)) return []; + return results + .map((r) => { + const o = r as Record<string, unknown>; + return { + title: String(o.title ?? ''), + url: String(o.url ?? ''), + snippet: String(o.description ?? ''), + }; + }) + .filter((r) => r.url); +} + +// ── Domain filtering (pure; exported for tests) ────────────────────── + +function hostMatches(host: string, domain: string): boolean { + const h = host.toLowerCase(); + const d = domain.toLowerCase().replace(/^\*?\.?/, '').replace(/\/+$/, ''); + return h === d || h.endsWith(`.${d}`); +} + +export function filterByDomains( + results: SearchResult[], + allowed?: string[], + blocked?: string[], +): SearchResult[] { + return results.filter((r) => { + let host: string; + try { + host = new URL(r.url).hostname; + } catch { + return false; + } + if (allowed && allowed.length > 0 && !allowed.some((d) => hostMatches(host, d))) return false; + if (blocked && blocked.length > 0 && blocked.some((d) => hostMatches(host, d))) return false; + return true; + }); +} + +/** Format results as a compact, agent-readable text block. */ +export function formatResults(query: string, results: SearchResult[]): string { + if (results.length === 0) return `No results for "${query}".`; + const lines = [`Results for "${query}":`, '']; + results.forEach((r, i) => { + lines.push(`${i + 1}. ${r.title || r.url}`); + lines.push(` ${r.url}`); + if (r.snippet) lines.push(` ${r.snippet.replace(/\s+/g, ' ').trim()}`); + lines.push(''); + }); + return lines.join('\n').trimEnd(); +} + +// ── Backend requests ───────────────────────────────────────────────── + +async function runBackend( + backend: SearchBackend, + query: string, + count: number, + allowed: string[] | undefined, + blocked: string[] | undefined, + doFetch: typeof fetch, + timeoutMs: number, +): Promise<SearchResult[]> { + const signal = AbortSignal.timeout(timeoutMs); + switch (backend.kind) { + case 'tavily': { + const res = await doFetch('https://api.tavily.com/search', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${backend.credential}`, + }, + body: JSON.stringify({ + api_key: backend.credential, + query, + max_results: count, + include_domains: allowed, + exclude_domains: blocked, + }), + signal, + }); + if (!res.ok) throw new Error(`Tavily HTTP ${res.status}: ${await safeText(res)}`); + return parseTavily(await res.json()); + } + case 'searxng': { + const base = backend.credential.replace(/\/+$/, ''); + const u = new URL(`${base}/search`); + u.searchParams.set('q', query); + u.searchParams.set('format', 'json'); + const res = await doFetch(u.toString(), { + headers: { Accept: 'application/json' }, + signal, + }); + if (!res.ok) throw new Error(`SearXNG HTTP ${res.status}: ${await safeText(res)}`); + return parseSearxng(await res.json()); + } + case 'serper': { + const res = await doFetch('https://google.serper.dev/search', { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-API-KEY': backend.credential }, + body: JSON.stringify({ q: query, num: count }), + signal, + }); + if (!res.ok) throw new Error(`Serper HTTP ${res.status}: ${await safeText(res)}`); + return parseSerper(await res.json()); + } + case 'brave': { + const u = new URL('https://api.search.brave.com/res/v1/web/search'); + u.searchParams.set('q', query); + u.searchParams.set('count', String(count)); + const res = await doFetch(u.toString(), { + headers: { Accept: 'application/json', 'X-Subscription-Token': backend.credential }, + signal, + }); + if (!res.ok) throw new Error(`Brave HTTP ${res.status}: ${await safeText(res)}`); + return parseBrave(await res.json()); + } + } +} + +async function safeText(res: Response): Promise<string> { + try { + return (await res.text()).slice(0, 200); + } catch { + return ''; + } +} + +export interface CreateWebSearchToolOptions { + env?: NodeJS.ProcessEnv; + fetchImpl?: typeof fetch; + timeoutMs?: number; +} + +export function createWebSearchTool( + options: CreateWebSearchToolOptions = {}, +): AgentTool<typeof webSearchSchema, WebSearchToolDetails> { + const env = options.env ?? process.env; + const doFetch = options.fetchImpl ?? fetch; + const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + + return { + name: 'web_search', + label: 'web_search', + description: + 'Search the web and return a ranked list of {title, url, snippet} results. Supports allowed_domains / ' + + 'blocked_domains filtering. Requires a configured backend (Tavily, SearXNG, Serper, or Brave via env vars); ' + + 'if none is set, returns instructions for configuring one.', + parameters: webSearchSchema, + async execute(_toolCallId, params) { + const query = params.query.trim(); + if (!query) { + return { + content: [{ type: 'text', text: 'Error: query must not be empty.' }], + details: { query, resultCount: 0, error: 'empty query' }, + }; + } + + const backend = selectBackend(env); + if (!backend) { + return { + content: [{ type: 'text', text: NO_BACKEND_MESSAGE }], + details: { query, resultCount: 0, error: 'no backend configured' }, + }; + } + + const count = Math.min(MAX_COUNT, Math.max(1, params.count ?? DEFAULT_COUNT)); + let results: SearchResult[]; + try { + results = await runBackend( + backend, + query, + count, + params.allowed_domains, + params.blocked_domains, + doFetch, + timeoutMs, + ); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return { + content: [{ type: 'text', text: `Web search failed (${backend.kind}): ${message}` }], + details: { query, backend: backend.kind, resultCount: 0, error: message }, + }; + } + + // Some backends honor include/exclude server-side; apply locally too so + // filtering is uniform across backends. + results = filterByDomains(results, params.allowed_domains, params.blocked_domains).slice(0, count); + + return { + content: [{ type: 'text', text: formatResults(query, results) }], + details: { query, backend: backend.kind, resultCount: results.length }, + }; + }, + }; +} + +export const webSearchTool = createWebSearchTool(); diff --git a/packages/core/tests/html-to-markdown.test.ts b/packages/core/tests/html-to-markdown.test.ts new file mode 100644 index 0000000..c990bfb --- /dev/null +++ b/packages/core/tests/html-to-markdown.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from 'vitest'; + +import { decodeEntities, extractTitle, htmlToMarkdown } from '../src/tools/html-to-markdown.js'; + +describe('decodeEntities', () => { + it('decodes named, decimal, and hex entities', () => { + expect(decodeEntities('a & b < c > d')).toBe('a & b < c > d'); + expect(decodeEntities('AB')).toBe('AB'); + expect(decodeEntities('café')).toBe('café'); // unknown named left as-is + expect(decodeEntities(' ')).toBe(' '); + }); +}); + +describe('extractTitle', () => { + it('pulls the title text', () => { + expect(extractTitle('<html><head><title>Hello & Bye')).toBe( + 'Hello & Bye', + ); + }); + it('returns undefined when absent', () => { + expect(extractTitle('x')).toBeUndefined(); + }); +}); + +describe('htmlToMarkdown', () => { + it('strips scripts and styles entirely', () => { + const html = '

keep

'; + const md = htmlToMarkdown(html); + expect(md).toContain('keep'); + expect(md).not.toContain('alert'); + expect(md).not.toContain('.x{}'); + }); + + it('converts headings', () => { + expect(htmlToMarkdown('

Title

Sub

')).toContain('# Title'); + expect(htmlToMarkdown('

Sub

')).toContain('## Sub'); + }); + + it('converts links to markdown', () => { + expect(htmlToMarkdown('click')).toContain('[click](https://x.com)'); + }); + + it('drops javascript: links but keeps text', () => { + expect(htmlToMarkdown('text')).toBe('text'); + }); + + it('converts emphasis and list items', () => { + expect(htmlToMarkdown('bold')).toBe('**bold**'); + expect(htmlToMarkdown('it')).toBe('*it*'); + const list = htmlToMarkdown('
  • one
  • two
'); + expect(list).toContain('- one'); + expect(list).toContain('- two'); + }); + + it('preserves preformatted code', () => { + const md = htmlToMarkdown('
line1\nline2
'); + expect(md).toContain('```'); + expect(md).toContain('line1\nline2'); + }); + + it('collapses excessive blank lines', () => { + const md = htmlToMarkdown('

a

b

'); + expect(md).not.toMatch(/\n{3,}/); + }); +}); diff --git a/packages/core/tests/ssrf.test.ts b/packages/core/tests/ssrf.test.ts new file mode 100644 index 0000000..04952c0 --- /dev/null +++ b/packages/core/tests/ssrf.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from 'vitest'; + +import { + assertHostAllowed, + isPrivateAddress, + isPrivateIPv4, + isPrivateIPv6, + normalizeUrl, + SsrfError, +} from '../src/tools/ssrf.js'; + +describe('normalizeUrl', () => { + it('upgrades http to https', () => { + expect(normalizeUrl('http://example.com/x').toString()).toBe('https://example.com/x'); + }); + + it('keeps https as-is', () => { + expect(normalizeUrl('https://example.com/').toString()).toBe('https://example.com/'); + }); + + it('rejects non-http(s) schemes', () => { + expect(() => normalizeUrl('ftp://example.com')).toThrow(SsrfError); + expect(() => normalizeUrl('file:///etc/passwd')).toThrow(SsrfError); + }); + + it('rejects unparseable input', () => { + expect(() => normalizeUrl('not a url')).toThrow(SsrfError); + }); +}); + +describe('private address detection', () => { + it('flags private IPv4 ranges', () => { + for (const ip of [ + '127.0.0.1', + '10.1.2.3', + '172.16.0.1', + '172.31.255.255', + '192.168.1.1', + '169.254.169.254', // cloud metadata + '0.0.0.0', + '100.64.0.1', + ]) { + expect(isPrivateIPv4(ip), ip).toBe(true); + expect(isPrivateAddress(ip), ip).toBe(true); + } + }); + + it('allows public IPv4', () => { + for (const ip of ['8.8.8.8', '1.1.1.1', '172.15.0.1', '172.32.0.1', '93.184.216.34']) { + expect(isPrivateIPv4(ip), ip).toBe(false); + } + }); + + it('flags private IPv6 and mapped addresses', () => { + for (const ip of ['::1', 'fe80::1', 'fc00::1', 'fd12:3456::1', '::ffff:127.0.0.1']) { + expect(isPrivateIPv6(ip), ip).toBe(true); + } + }); + + it('allows public IPv6', () => { + expect(isPrivateIPv6('2606:4700:4700::1111')).toBe(false); + }); +}); + +describe('assertHostAllowed', () => { + it('rejects localhost without DNS', async () => { + await expect(assertHostAllowed(new URL('https://localhost/'))).rejects.toThrow(SsrfError); + }); + + it('rejects internal TLDs', async () => { + await expect(assertHostAllowed(new URL('https://db.internal/'))).rejects.toThrow(SsrfError); + }); + + it('rejects literal private IPs without DNS', async () => { + await expect(assertHostAllowed(new URL('https://169.254.169.254/latest/meta-data/'))).rejects.toThrow( + SsrfError, + ); + }); + + it('rejects a public host that resolves to a private IP (DNS rebinding)', async () => { + const resolver = async () => ['10.0.0.5']; + await expect(assertHostAllowed(new URL('https://evil.example.com/'), resolver)).rejects.toThrow( + SsrfError, + ); + }); + + it('allows a public host that resolves to a public IP', async () => { + const resolver = async () => ['93.184.216.34']; + await expect( + assertHostAllowed(new URL('https://example.com/'), resolver), + ).resolves.toBeUndefined(); + }); + + it('allows a literal public IP', async () => { + await expect(assertHostAllowed(new URL('https://8.8.8.8/'))).resolves.toBeUndefined(); + }); +}); diff --git a/packages/core/tests/web-fetch.test.ts b/packages/core/tests/web-fetch.test.ts new file mode 100644 index 0000000..0477e23 --- /dev/null +++ b/packages/core/tests/web-fetch.test.ts @@ -0,0 +1,136 @@ +import { describe, expect, it } from 'vitest'; + +import { createWebFetchTool } from '../src/tools/web-fetch.js'; + +const publicResolver = async () => ['93.184.216.34']; + +/** Build a fetch mock from a url→Response factory map. */ +function mockFetch(routes: Record Response>): typeof fetch { + return (async (input: string | URL) => { + const url = typeof input === 'string' ? input : input.toString(); + const route = routes[url]; + if (!route) throw new Error(`unexpected fetch: ${url}`); + return route(); + }) as unknown as typeof fetch; +} + +function tool(routes: Record Response>) { + return createWebFetchTool({ fetchImpl: mockFetch(routes), resolver: publicResolver }); +} + +describe('web_fetch SSRF', () => { + it('blocks localhost before any fetch', async () => { + const t = createWebFetchTool({ + fetchImpl: (async () => { + throw new Error('should not be called'); + }) as unknown as typeof fetch, + resolver: publicResolver, + }); + const res = await t.execute('id', { url: 'http://localhost:8080/' }); + expect(res.content[0].text).toMatch(/^Blocked/); + expect(res.details.error).toBeDefined(); + }); + + it('blocks the cloud metadata IP', async () => { + const t = createWebFetchTool({ resolver: publicResolver }); + const res = await t.execute('id', { url: 'http://169.254.169.254/latest/meta-data/' }); + expect(res.content[0].text).toMatch(/^Blocked/); + }); + + it('rejects unsupported schemes', async () => { + const t = createWebFetchTool({ resolver: publicResolver }); + const res = await t.execute('id', { url: 'file:///etc/passwd' }); + expect(res.content[0].text).toMatch(/^Blocked/); + }); +}); + +describe('web_fetch content handling', () => { + it('converts HTML to markdown and includes the title', async () => { + const t = tool({ + 'https://example.com/': () => + new Response('Hi

Heading

Body

', { + status: 200, + headers: { 'content-type': 'text/html; charset=utf-8' }, + }), + }); + const res = await t.execute('id', { url: 'https://example.com/' }); + expect(res.content[0].text).toContain('Title: Hi'); + expect(res.content[0].text).toContain('# Heading'); + expect(res.content[0].text).toContain('Body'); + expect(res.details.title).toBe('Hi'); + }); + + it('passes through plain text', async () => { + const t = tool({ + 'https://example.com/robots.txt': () => + new Response('User-agent: *\nDisallow:', { + status: 200, + headers: { 'content-type': 'text/plain' }, + }), + }); + const res = await t.execute('id', { url: 'https://example.com/robots.txt' }); + expect(res.content[0].text).toContain('User-agent: *'); + }); + + it('does not render binary content', async () => { + const t = tool({ + 'https://example.com/img.png': () => + new Response('\x89PNG', { status: 200, headers: { 'content-type': 'image/png' } }), + }); + const res = await t.execute('id', { url: 'https://example.com/img.png' }); + expect(res.content[0].text).toContain('non-text content'); + }); + + it('truncates very large pages', async () => { + const big = '

' + 'x'.repeat(400 * 1024) + '

'; + const t = tool({ + 'https://example.com/big': () => + new Response(big, { status: 200, headers: { 'content-type': 'text/html' } }), + }); + const res = await t.execute('id', { url: 'https://example.com/big' }); + expect(res.details.truncated).toBe(true); + expect(res.content[0].text).toContain('truncated'); + }); + + it('reports HTTP error status', async () => { + const t = tool({ + 'https://example.com/missing': () => new Response('nope', { status: 404 }), + }); + const res = await t.execute('id', { url: 'https://example.com/missing' }); + expect(res.content[0].text).toContain('HTTP 404'); + expect(res.details.status).toBe(404); + }); + + it('echoes the extraction prompt in the header', async () => { + const t = tool({ + 'https://example.com/': () => + new Response('

content

', { status: 200, headers: { 'content-type': 'text/html' } }), + }); + const res = await t.execute('id', { url: 'https://example.com/', prompt: 'find the pricing' }); + expect(res.content[0].text).toContain('Requested: find the pricing'); + }); +}); + +describe('web_fetch redirects', () => { + it('follows same-host redirects', async () => { + const t = tool({ + 'https://example.com/a': () => + new Response(null, { status: 302, headers: { location: 'https://example.com/b' } }), + 'https://example.com/b': () => + new Response('

final

', { status: 200, headers: { 'content-type': 'text/html' } }), + }); + const res = await t.execute('id', { url: 'https://example.com/a' }); + expect(res.content[0].text).toContain('final'); + expect(res.details.finalUrl).toBe('https://example.com/b'); + }); + + it('does not follow cross-host redirects, returns the new URL', async () => { + const t = tool({ + 'https://example.com/a': () => + new Response(null, { status: 302, headers: { location: 'https://other.com/x' } }), + }); + const res = await t.execute('id', { url: 'https://example.com/a' }); + expect(res.content[0].text).toContain('different host'); + expect(res.details.finalUrl).toBe('https://other.com/x'); + }); +}); diff --git a/packages/core/tests/web-search.test.ts b/packages/core/tests/web-search.test.ts new file mode 100644 index 0000000..91c2831 --- /dev/null +++ b/packages/core/tests/web-search.test.ts @@ -0,0 +1,151 @@ +import { describe, expect, it } from 'vitest'; + +import { + createWebSearchTool, + filterByDomains, + formatResults, + NO_BACKEND_MESSAGE, + parseBrave, + parseSearxng, + parseSerper, + parseTavily, + selectBackend, + type SearchResult, +} from '../src/tools/web-search.js'; + +describe('selectBackend', () => { + it('returns undefined when nothing is configured', () => { + expect(selectBackend({})).toBeUndefined(); + }); + + it('prefers tavily, then searxng, then serper, then brave', () => { + expect(selectBackend({ BRAVE_API_KEY: 'b', SERPER_API_KEY: 's', TAVILY_API_KEY: 't' })?.kind).toBe( + 'tavily', + ); + expect(selectBackend({ BRAVE_API_KEY: 'b', SEARXNG_URL: 'http://s' })?.kind).toBe('searxng'); + expect(selectBackend({ BRAVE_API_KEY: 'b', SERPER_API_KEY: 's' })?.kind).toBe('serper'); + expect(selectBackend({ BRAVE_API_KEY: 'b' })?.kind).toBe('brave'); + }); + + it('honors HARNEXT_SEARCH_BACKEND override', () => { + const env = { TAVILY_API_KEY: 't', BRAVE_API_KEY: 'b', HARNEXT_SEARCH_BACKEND: 'brave' }; + expect(selectBackend(env)?.kind).toBe('brave'); + }); + + it('returns undefined when the forced backend is unconfigured', () => { + expect(selectBackend({ TAVILY_API_KEY: 't', HARNEXT_SEARCH_BACKEND: 'brave' })).toBeUndefined(); + }); +}); + +describe('result parsers', () => { + it('parses tavily', () => { + const r = parseTavily({ results: [{ title: 'T', url: 'https://a.com', content: 'snip' }] }); + expect(r).toEqual([{ title: 'T', url: 'https://a.com', snippet: 'snip' }]); + }); + it('parses searxng', () => { + const r = parseSearxng({ results: [{ title: 'T', url: 'https://a.com', content: 'snip' }] }); + expect(r[0].url).toBe('https://a.com'); + }); + it('parses serper organic with link field', () => { + const r = parseSerper({ organic: [{ title: 'T', link: 'https://a.com', snippet: 'snip' }] }); + expect(r[0].url).toBe('https://a.com'); + }); + it('parses brave nested web.results', () => { + const r = parseBrave({ web: { results: [{ title: 'T', url: 'https://a.com', description: 'd' }] } }); + expect(r[0].snippet).toBe('d'); + }); + it('tolerates missing/garbage shapes', () => { + expect(parseTavily({})).toEqual([]); + expect(parseSerper(null)).toEqual([]); + expect(parseBrave({ web: {} })).toEqual([]); + }); +}); + +describe('filterByDomains', () => { + const results: SearchResult[] = [ + { title: 'a', url: 'https://docs.example.com/x', snippet: '' }, + { title: 'b', url: 'https://evil.test/y', snippet: '' }, + { title: 'c', url: 'not-a-url', snippet: '' }, + ]; + + it('keeps only allowed domains (subdomain match)', () => { + const r = filterByDomains(results, ['example.com']); + expect(r.map((x) => x.title)).toEqual(['a']); + }); + + it('removes blocked domains', () => { + const r = filterByDomains(results, undefined, ['evil.test']); + expect(r.map((x) => x.title)).toEqual(['a']); + }); + + it('drops unparseable urls', () => { + const r = filterByDomains(results); + expect(r.map((x) => x.title)).toEqual(['a', 'b']); + }); +}); + +describe('formatResults', () => { + it('formats a numbered list', () => { + const text = formatResults('q', [{ title: 'T', url: 'https://a.com', snippet: 'snip' }]); + expect(text).toContain('1. T'); + expect(text).toContain('https://a.com'); + expect(text).toContain('snip'); + }); + it('handles empty results', () => { + expect(formatResults('q', [])).toContain('No results'); + }); +}); + +describe('web_search tool', () => { + it('returns the setup message when no backend is configured', async () => { + const tool = createWebSearchTool({ env: {} }); + const res = await tool.execute('id', { query: 'hi' }); + expect(res.content[0].text).toBe(NO_BACKEND_MESSAGE); + expect(res.details.error).toBe('no backend configured'); + }); + + it('errors on empty query', async () => { + const tool = createWebSearchTool({ env: { TAVILY_API_KEY: 't' } }); + const res = await tool.execute('id', { query: ' ' }); + expect(res.details.error).toBe('empty query'); + }); + + it('runs the selected backend and formats results', async () => { + const fetchImpl = (async () => + new Response( + JSON.stringify({ results: [{ title: 'Doc', url: 'https://docs.example.com', content: 'hi' }] }), + { status: 200, headers: { 'content-type': 'application/json' } }, + )) as unknown as typeof fetch; + const tool = createWebSearchTool({ env: { TAVILY_API_KEY: 't' }, fetchImpl }); + const res = await tool.execute('id', { query: 'docs' }); + expect(res.details.backend).toBe('tavily'); + expect(res.details.resultCount).toBe(1); + expect(res.content[0].text).toContain('https://docs.example.com'); + }); + + it('applies allowed_domains filtering to results', async () => { + const fetchImpl = (async () => + new Response( + JSON.stringify({ + results: [ + { title: 'Keep', url: 'https://docs.example.com', content: '' }, + { title: 'Drop', url: 'https://other.test', content: '' }, + ], + }), + { status: 200 }, + )) as unknown as typeof fetch; + const tool = createWebSearchTool({ env: { TAVILY_API_KEY: 't' }, fetchImpl }); + const res = await tool.execute('id', { query: 'q', allowed_domains: ['example.com'] }); + expect(res.details.resultCount).toBe(1); + expect(res.content[0].text).toContain('Keep'); + expect(res.content[0].text).not.toContain('Drop'); + }); + + it('reports backend HTTP errors', async () => { + const fetchImpl = (async () => + new Response('nope', { status: 401 })) as unknown as typeof fetch; + const tool = createWebSearchTool({ env: { TAVILY_API_KEY: 't' }, fetchImpl }); + const res = await tool.execute('id', { query: 'q' }); + expect(res.details.error).toContain('401'); + }); +});