feat: add WebSearch and WebFetch tools (#78)

packages/cli/src/modes/interactive/render.ts

@@ -201,6 +201,8 @@ function toolColor(name: string): RoleColor {
     case 'glob':
     case 'ls':
     case 'find':
+    case 'web_fetch':
+    case 'web_search':
       return { idx: X.accent, ink: c.accent };
     case 'edit':
     case 'write':
@@ -241,6 +243,10 @@ export function toolStart(name: string, args: Record<string, unknown>): string {
     arg = c.green('$ ') + c.bright(cmd);
   } else if (name === 'read' || name === 'write' || name === 'edit') {
     arg = c.bright(fitToWidth(String(args.path ?? ''), argBudget));
+  } else if (name === 'web_fetch') {
+    arg = c.bright(truncateOneLine(String(args.url ?? ''), argBudget));
+  } else if (name === 'web_search') {
+    arg = c.bright(truncateOneLine(String(args.query ?? ''), argBudget));
   } else {
     arg = c.dim(truncateOneLine(JSON.stringify(args), argBudget));
   }

packages/core/src/index.ts

@@ -627,6 +627,20 @@ export {
   type HeartbeatToolDetails,
   type HeartbeatToolInput,
   type CreateHeartbeatToolOptions,
+  webFetchTool,
+  createWebFetchTool,
+  type WebFetchToolDetails,
+  type WebFetchToolInput,
+  type CreateWebFetchToolOptions,
+  webSearchTool,
+  createWebSearchTool,
+  selectBackend,
+  type WebSearchToolDetails,
+  type WebSearchToolInput,
+  type CreateWebSearchToolOptions,
+  type SearchResult,
+  type SearchBackend,
+  type SearchBackendKind,
   DEFAULT_MAX_BYTES,
   DEFAULT_MAX_LINES,
   formatSize,

packages/core/src/tools/html-to-markdown.ts

@@ -0,0 +1,154 @@
+/**
+ * Lightweight, dependency-free HTML→Markdown conversion for the WebFetch tool.
+ *
+ * The big harnesses (Claude Code, opencode) use `turndown`, which in Node also
+ * pulls in a DOM implementation (jsdom). To keep `@harnext/core` dependency-light
+ * and its tests hermetic, we implement a focused converter that handles the tags
+ * that matter for agent reading — headings, links, lists, emphasis, code, and
+ * block structure — and strips everything else. It is intentionally lossy on
+ * exotic markup; the goal is readable text, not a faithful round-trip.
+ */
+
+const NAMED_ENTITIES: Record<string, string> = {
+  amp: '&',
+  lt: '<',
+  gt: '>',
+  quot: '"',
+  apos: "'",
+  nbsp: ' ',
+  copy: '©',
+  reg: '®',
+  trade: '™',
+  hellip: '…',
+  mdash: '—',
+  ndash: '–',
+  lsquo: '‘',
+  rsquo: '’',
+  ldquo: '“',
+  rdquo: '”',
+  laquo: '«',
+  raquo: '»',
+  deg: '°',
+  middot: '·',
+  bull: '•',
+};
+
+/** Decode HTML entities (named + decimal/hex numeric) into plain text. */
+export function decodeEntities(text: string): string {
+  return text.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*);/g, (match, body: string) => {
+    if (body[0] === '#') {
+      const code =
+        body[1] === 'x' || body[1] === 'X'
+          ? parseInt(body.slice(2), 16)
+          : parseInt(body.slice(1), 10);
+      if (Number.isFinite(code) && code > 0 && code <= 0x10ffff) {
+        try {
+          return String.fromCodePoint(code);
+        } catch {
+          return match;
+        }
+      }
+      return match;
+    }
+    const named = NAMED_ENTITIES[body.toLowerCase()];
+    return named ?? match;
+  });
+}
+
+/** Pull the contents of the first <title> tag, if any. */
+export function extractTitle(html: string): string | undefined {
+  const m = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+  if (!m) return undefined;
+  const text = decodeEntities(m[1].replace(/\s+/g, ' ')).trim();
+  return text.length > 0 ? text : undefined;
+}
+
+/** Remove elements whose contents are never useful as reading material. */
+function stripNonContent(html: string): string {
+  return html
+    .replace(/<!--[\s\S]*?-->/g, '')
+    .replace(/<script\b[\s\S]*?<\/script>/gi, '')
+    .replace(/<style\b[\s\S]*?<\/style>/gi, '')
+    .replace(/<noscript\b[\s\S]*?<\/noscript>/gi, '')
+    .replace(/<template\b[\s\S]*?<\/template>/gi, '')
+    .replace(/<svg\b[\s\S]*?<\/svg>/gi, '')
+    .replace(/<head\b[\s\S]*?<\/head>/gi, '');
+}
+
+/**
+ * Convert an HTML document or fragment to Markdown-ish plain text.
+ */
+export function htmlToMarkdown(html: string): string {
+  let s = stripNonContent(html);
+
+  // Headings.
+  for (let level = 1; level <= 6; level++) {
+    const hashes = '#'.repeat(level);
+    s = s.replace(
+      new RegExp(`<h${level}[^>]*>([\\s\\S]*?)<\\/h${level}>`, 'gi'),
+      (_m, inner: string) => `\n\n${hashes} ${collapseInline(inner)}\n\n`,
+    );
+  }
+
+  // Preformatted / code blocks — preserve internal whitespace, fence them.
+  s = s.replace(/<pre[^>]*>([\s\S]*?)<\/pre>/gi, (_m, inner: string) => {
+    const code = decodeEntities(stripTags(inner.replace(/<br\s*\/?>(?=)/gi, '\n')));
+    return `\n\n\`\`\`\n${code.replace(/\n+$/, '')}\n\`\`\`\n\n`;
+  });
+
+  // Inline code.
+  s = s.replace(/<code[^>]*>([\s\S]*?)<\/code>/gi, (_m, inner: string) => {
+    return '`' + collapseInline(inner) + '`';
+  });
+
+  // Links — [text](href).
+  s = s.replace(
+    /<a\b[^>]*\bhref\s*=\s*(["'])(.*?)\1[^>]*>([\s\S]*?)<\/a>/gi,
+    (_m, _q, href: string, inner: string) => {
+      const text = collapseInline(inner);
+      const url = decodeEntities(href).trim();
+      if (!text) return url;
+      if (!url || url.startsWith('javascript:')) return text;
+      return `[${text}](${url})`;
+    },
+  );
+
+  // Emphasis.
+  s = s.replace(/<(strong|b)\b[^>]*>([\s\S]*?)<\/\1>/gi, (_m, _t, inner: string) => `**${collapseInline(inner)}**`);
+  s = s.replace(/<(em|i)\b[^>]*>([\s\S]*?)<\/\1>/gi, (_m, _t, inner: string) => `*${collapseInline(inner)}*`);
+
+  // List items.
+  s = s.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, (_m, inner: string) => `\n- ${collapseInline(inner)}`);
+
+  // Block boundaries → blank lines.
+  s = s.replace(/<\/(p|div|section|article|header|footer|ul|ol|table|tr|blockquote|h[1-6])\s*>/gi, '\n\n');
+  s = s.replace(/<br\s*\/?>(?=)/gi, '\n');
+  s = s.replace(/<\/(td|th)\s*>/gi, ' \t ');
+
+  // Drop everything else and decode entities.
+  s = stripTags(s);
+  s = decodeEntities(s);
+
+  // Normalize whitespace: trim trailing spaces, collapse 3+ newlines.
+  s = s
+    .split('\n')
+    .map((line) => line.replace(/[ \t]+$/g, '').replace(/^[ \t]+/g, (m) => (m.length > 0 ? '' : m)))
+    .join('\n')
+    .replace(/\n{3,}/g, '\n\n')
+    .replace(/[ \t]{2,}/g, ' ')
+    .trim();
+
+  return s;
+}
+
+/** Strip all remaining tags. */
+function stripTags(s: string): string {
+  return s.replace(/<[^>]+>/g, '');
+}
+
+/** Reduce an inline fragment to single-line text (tags removed, entities decoded). */
+function collapseInline(inner: string): string {
+  return decodeEntities(stripTags(inner))
+    .replace(/\s+/g, ' ')
+    .trim();
+}

packages/core/src/tools/index.ts

@@ -66,6 +66,24 @@ export {
   heartbeatTool,
   createHeartbeatTool,
 } from './heartbeat-tool.js';
+export {
+  type WebFetchToolDetails,
+  type WebFetchToolInput,
+  type CreateWebFetchToolOptions,
+  webFetchTool,
+  createWebFetchTool,
+} from './web-fetch.js';
+export {
+  type WebSearchToolDetails,
+  type WebSearchToolInput,
+  type CreateWebSearchToolOptions,
+  type SearchResult,
+  type SearchBackend,
+  type SearchBackendKind,
+  webSearchTool,
+  createWebSearchTool,
+  selectBackend,
+} from './web-search.js';
 export {
   DEFAULT_MAX_BYTES,
   DEFAULT_MAX_LINES,
@@ -86,6 +104,8 @@ import { heartbeatTool, createHeartbeatTool, type CreateHeartbeatToolOptions } f
 import { memoryTool, createMemoryTool } from './memory.js';
 import { readTool, createReadTool } from './read.js';
 import { todoTool, createTodoTool } from './todo.js';
+import { webFetchTool, createWebFetchTool } from './web-fetch.js';
+import { webSearchTool, createWebSearchTool } from './web-search.js';
 import { writeTool, createWriteTool } from './write.js';
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -100,6 +120,8 @@ export const codingTools: Tool[] = [
   exitPlanTool,
   memoryTool,
   heartbeatTool,
+  webFetchTool,
+  webSearchTool,
 ];
 
 export const allTools = {
@@ -111,6 +133,8 @@ export const allTools = {
   exit_plan: exitPlanTool,
   memory: memoryTool,
   heartbeat: heartbeatTool,
+  web_fetch: webFetchTool,
+  web_search: webSearchTool,
 };
 
 export type ToolName = keyof typeof allTools;
@@ -137,6 +161,8 @@ export function createCodingTools(cwd: string, options: CreateCodingToolsOptions
     createExitPlanTool(),
     createMemoryTool(cwd),
     createHeartbeatTool(cwd, options.heartbeat),
+    createWebFetchTool(),
+    createWebSearchTool(),
   ];
   // The background-shell companions only exist when a manager is wired in
   // (i.e. background execution is enabled for this session).
@@ -156,5 +182,7 @@ export function createAllTools(cwd: string): Record<ToolName, Tool> {
     exit_plan: createExitPlanTool(),
     memory: createMemoryTool(cwd),
     heartbeat: createHeartbeatTool(cwd),
+    web_fetch: createWebFetchTool(),
+    web_search: createWebSearchTool(),
   };
 }