From d82e13e37e00a9de10f40600ad4a655d07e43c18 Mon Sep 17 00:00:00 2001 From: ryoppippi <1560508+ryoppippi@users.noreply.github.com> Date: Wed, 6 Aug 2025 15:34:40 +0100 Subject: [PATCH 1/9] orama mcp --- .mcp.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.mcp.json b/.mcp.json index 19abcd22..f4ef00cc 100644 --- a/.mcp.json +++ b/.mcp.json @@ -7,6 +7,18 @@ "grep": { "type": "http", "url": "https://mcp.grep.app" + }, + "oramaDocs": { + "type": "stdio", + "command": "bun", + "args": [ + "x", + "sitemcp", + "https://docs.orama.com/", + "--concurrency", + "10" + ], + "env": {} } } } From a18cdac441974024d996ca6881f35fc2a54c177f Mon Sep 17 00:00:00 2001 From: ryoppippi <1560508+ryoppippi@users.noreply.github.com> Date: Wed, 6 Aug 2025 15:35:20 +0100 Subject: [PATCH 2/9] chore(deps): add @orama/orama --- bun.lock | 3 +++ package.json | 1 + 2 files changed, 4 insertions(+) diff --git a/bun.lock b/bun.lock index b5d71ad9..ff47d502 100644 --- a/bun.lock +++ b/bun.lock @@ -4,6 +4,7 @@ "": { "name": "stackone-ai-ts", "dependencies": { + "@orama/orama": "^3.1.11", "json-schema": "^0.4.0", }, "devDependencies": { @@ -113,6 +114,8 @@ "@opentelemetry/api": ["@opentelemetry/api@1.9.0", "", {}, "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg=="], + "@orama/orama": ["@orama/orama@3.1.11", "", {}, "sha512-Szki0cgFiXE5F9RLx2lUyEtJllnuCSQ4B8RLDwIjXkVit6qZjoDAxH+xhJs29MjKLDz0tbPLdKFa6QrQ/qoGGA=="], + "@oxc-project/runtime": ["@oxc-project/runtime@0.72.3", "", {}, "sha512-FtOS+0v7rZcnjXzYTTqv1vu/KDptD1UztFgoZkYBGe/6TcNFm+SP/jQoLvzau1SPir95WgDOBOUm2Gmsm+bQag=="], "@oxc-project/types": ["@oxc-project/types@0.72.3", "", {}, "sha512-CfAC4wrmMkUoISpQkFAIfMVvlPfQV3xg7ZlcqPXPOIMQhdKIId44G8W0mCPgtpWdFFAyJ+SFtiM+9vbyCkoVng=="], diff --git a/package.json b/package.json index dfc8da02..9f5b81d6 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "format": "biome format --write ." }, "dependencies": { + "@orama/orama": "^3.1.11", "json-schema": "^0.4.0" }, "devDependencies": { From da629fff6dbf5ef3a6cfab6070330606323ce2d5 Mon Sep 17 00:00:00 2001 From: ryoppippi <1560508+ryoppippi@users.noreply.github.com> Date: Wed, 6 Aug 2025 17:07:59 +0100 Subject: [PATCH 3/9] base --- src/meta.ts | 9 +++++++++ src/tool.ts | 10 ++++++++++ 2 files changed, 19 insertions(+) create mode 100644 src/meta.ts diff --git a/src/meta.ts b/src/meta.ts new file mode 100644 index 00000000..fcccde89 --- /dev/null +++ b/src/meta.ts @@ -0,0 +1,9 @@ +import { BaseTool } from './tool'; + +export const metaFilterRelevantTools = (): BaseTool => { + return new BaseTool(); +}; + +export const metaExecuteTool = (): BaseTool => { + return new BaseTool(); +}; diff --git a/src/tool.ts b/src/tool.ts index 1672c408..42e9487f 100644 --- a/src/tool.ts +++ b/src/tool.ts @@ -1,5 +1,6 @@ import { type ToolSet, jsonSchema } from 'ai'; import type { ChatCompletionTool } from 'openai/resources/chat/completions'; +import { metaExecuteTool, metaFilterRelevantTools } from './meta'; import { RequestBuilder } from './modules/requestBuilder'; import type { ExecuteConfig, @@ -271,6 +272,15 @@ export class Tools implements Iterable { return new Tools(this.tools.filter(predicate)); } + /** + * return two tools + */ + metaRelevantTools(): Tools { + const baseTools = [metaFilterRelevantTools(), metaExecuteTool()]; + const tools = new Tools(baseTools); + return tools; + } + /** * Iterator implementation */ From 3c1d84060c647411a71a18f7019fa1bd93f3353d Mon Sep 17 00:00:00 2001 From: ryoppippi <1560508+ryoppippi@users.noreply.github.com> Date: Wed, 6 Aug 2025 17:37:49 +0100 Subject: [PATCH 4/9] define tools --- src/meta.ts | 9 ------- src/tool.ts | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 74 insertions(+), 11 deletions(-) delete mode 100644 src/meta.ts diff --git a/src/meta.ts b/src/meta.ts deleted file mode 100644 index fcccde89..00000000 --- a/src/meta.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { BaseTool } from './tool'; - -export const metaFilterRelevantTools = (): BaseTool => { - return new BaseTool(); -}; - -export const metaExecuteTool = (): BaseTool => { - return new BaseTool(); -}; diff --git a/src/tool.ts b/src/tool.ts index 42e9487f..682ceaaa 100644 --- a/src/tool.ts +++ b/src/tool.ts @@ -1,6 +1,5 @@ import { type ToolSet, jsonSchema } from 'ai'; import type { ChatCompletionTool } from 'openai/resources/chat/completions'; -import { metaExecuteTool, metaFilterRelevantTools } from './meta'; import { RequestBuilder } from './modules/requestBuilder'; import type { ExecuteConfig, @@ -276,7 +275,7 @@ export class Tools implements Iterable { * return two tools */ metaRelevantTools(): Tools { - const baseTools = [metaFilterRelevantTools(), metaExecuteTool()]; + const baseTools = [metaFilterRelevantTools(this), metaExecuteTool(this)]; const tools = new Tools(baseTools); return tools; } @@ -319,3 +318,76 @@ export class Tools implements Iterable { this.tools.forEach(callback); } } + +export function metaFilterRelevantTools(tools: Tools): BaseTool { + const name = 'meta_filter_relevant_tools' as const; + const description = + 'find relevant tools. LLm should call this tool before meta_execution' as const; + const parameters = { + type: 'object', + properties: { + query: { + type: 'string', + description: 'Natural language query describing what tools you need', + }, + limit: { + type: 'number', + description: 'Maximum number of tools to return (default: 5)', + default: 5, + }, + minScore: { + type: 'number', + description: 'Minimum relevance score (0-1) for results (default: 0.3)', + default: 0.3, + }, + filterPatterns: { + oneOf: [ + { type: 'string' }, + { + type: 'array', + items: { type: 'string' }, + }, + ], + description: 'Optional glob patterns to filter results (e.g., "hris_*", "!*_delete_*")', + }, + }, + required: ['query'], + } as const satisfies ToolParameters; + + const executeConfig = { + method: 'LOCAL', + url: 'local://get-relevant-tools', + bodyType: 'json', + params: [], + } as const satisfies ExecuteConfig; + + return new BaseTool(name, description, parameters, executeConfig); +} + +export function metaExecuteTool(tools: Tools): BaseTool { + const name = 'meta_execute_tools' as const; + const description = 'Execute a tool based on the provided parameters' as const; + const parameters = { + type: 'object', + properties: { + toolName: { + type: 'string', + description: 'Name of the tool to execute', + }, + params: { + type: 'object', + description: 'Parameters to pass to the tool', + }, + }, + required: ['toolName', 'params'], + } as const satisfies ToolParameters; + + const executeConfig = { + method: 'LOCAL', + url: 'local://execute-tool', + bodyType: 'json', + params: [], + } as const satisfies ExecuteConfig; + + return new BaseTool(name, description, parameters, executeConfig); +} From e354192b733a7acf2d96783e1b25f5b915ba986f Mon Sep 17 00:00:00 2001 From: ryoppippi <1560508+ryoppippi@users.noreply.github.com> Date: Wed, 6 Aug 2025 19:00:41 +0100 Subject: [PATCH 5/9] feat(meta-tools): implement meta tools for dynamic tool discovery and execution - Add metaTools() method to Tools class for getting meta tools - Implement meta_filter_relevant_tools using Orama BM25 search algorithm - Implement meta_execute_tool for dynamic tool execution - Fix async initialization of Orama database - Add MetaToolSearchResult interface for type safety - Include tool configurations in search results for LLM usage - Add comprehensive documentation comments and JSDoc BREAKING CHANGE: metaRelevantTools() renamed to metaTools() and now returns Promise --- src/tool.ts | 187 ++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 167 insertions(+), 20 deletions(-) diff --git a/src/tool.ts b/src/tool.ts index 682ceaaa..cad73696 100644 --- a/src/tool.ts +++ b/src/tool.ts @@ -1,3 +1,4 @@ +import * as orama from '@orama/orama'; import { type ToolSet, jsonSchema } from 'ai'; import type { ChatCompletionTool } from 'openai/resources/chat/completions'; import { RequestBuilder } from './modules/requestBuilder'; @@ -272,10 +273,12 @@ export class Tools implements Iterable { } /** - * return two tools + * Return meta tools for tool discovery and execution + * @beta This feature is in beta and may change in future versions */ - metaRelevantTools(): Tools { - const baseTools = [metaFilterRelevantTools(this), metaExecuteTool(this)]; + async metaTools(): Promise { + const oramaDb = await initializeOramaDb(this.tools); + const baseTools = [metaFilterRelevantTools(oramaDb, this.tools), metaExecuteTool(this)]; const tools = new Tools(baseTools); return tools; } @@ -319,16 +322,72 @@ export class Tools implements Iterable { } } -export function metaFilterRelevantTools(tools: Tools): BaseTool { +/** + * Result from meta_filter_relevant_tools + */ +export interface MetaToolSearchResult { + name: string; + description: string; + parameters: ToolParameters; + score: number; +} + +type OramaDb = Awaited>; + +/** + * Initialize Orama database with BM25 algorithm for tool search + * Using Orama's BM25 scoring algorithm for relevance ranking + * @see https://docs.orama.com/open-source/usage/search/bm25-algorithm/ + */ +async function initializeOramaDb(tools: BaseTool[]): Promise { + // Create Orama database schema with BM25 scoring algorithm + // BM25 provides better relevance ranking for natural language queries + const oramaDb = await orama.create({ + schema: { + name: 'string' as const, + description: 'string' as const, + category: 'string' as const, + tags: 'string[]' as const, + }, + components: { + tokenizer: { + stemming: true, + }, + }, + }); + + // Index all tools + for (const tool of tools) { + // Extract category from tool name (e.g., 'hris_create_employee' -> 'hris') + const parts = tool.name.split('_'); + const category = parts[0]; + + // Extract action type + const actionTypes = ['create', 'update', 'delete', 'get', 'list', 'search']; + const actions = parts.filter((p) => actionTypes.includes(p)); + + await orama.insert(oramaDb, { + name: tool.name, + description: tool.description, + category: category, + tags: [...parts, ...actions], + }); + } + + return oramaDb; +} + +export function metaFilterRelevantTools(oramaDb: OramaDb, allTools: BaseTool[]): BaseTool { const name = 'meta_filter_relevant_tools' as const; const description = - 'find relevant tools. LLm should call this tool before meta_execution' as const; + 'Searches for relevant tools based on a natural language query. This tool should be called first to discover available tools before executing them.' as const; const parameters = { type: 'object', properties: { query: { type: 'string', - description: 'Natural language query describing what tools you need', + description: + 'Natural language query describing what tools you need (e.g., "tools for managing employees", "create time off request")', }, limit: { type: 'number', @@ -340,16 +399,6 @@ export function metaFilterRelevantTools(tools: Tools): BaseTool { description: 'Minimum relevance score (0-1) for results (default: 0.3)', default: 0.3, }, - filterPatterns: { - oneOf: [ - { type: 'string' }, - { - type: 'array', - items: { type: 'string' }, - }, - ], - description: 'Optional glob patterns to filter results (e.g., "hris_*", "!*_delete_*")', - }, }, required: ['query'], } as const satisfies ToolParameters; @@ -361,12 +410,66 @@ export function metaFilterRelevantTools(tools: Tools): BaseTool { params: [], } as const satisfies ExecuteConfig; - return new BaseTool(name, description, parameters, executeConfig); + const tool = new BaseTool(name, description, parameters, executeConfig); + tool.execute = async (inputParams?: JsonDict | string): Promise => { + try { + // Validate params is either undefined, string, or object + if ( + inputParams !== undefined && + typeof inputParams !== 'string' && + typeof inputParams !== 'object' + ) { + throw new StackOneError( + `Invalid parameters type. Expected object or string, got ${typeof inputParams}. Parameters: ${JSON.stringify(inputParams)}` + ); + } + + // Convert string params to object + const params = typeof inputParams === 'string' ? JSON.parse(inputParams) : inputParams || {}; + + // Perform search using Orama + const results = await orama.search(oramaDb, { + term: params.query || '', + limit: params.limit || 5, + } as any); + + // filter results by minimum score + const minScore = params.minScore || 0.3; + const filteredResults = results.hits.filter((hit) => hit.score >= minScore); + + // Map the results to include tool configurations + const toolConfigs = filteredResults + .map((hit) => { + const tool = allTools.find((t) => t.name === hit.document.name); + if (!tool) return null; + + const result: MetaToolSearchResult = { + name: tool.name, + description: tool.description, + parameters: tool.parameters, + score: hit.score, + }; + return result; + }) + .filter(Boolean); + + return { tools: toolConfigs } satisfies JsonDict; + } catch (error) { + if (error instanceof StackOneError) { + throw error; + } + throw new StackOneError( + `Error executing tool: ${error instanceof Error ? error.message : String(error)}` + ); + } + }; + return tool; } export function metaExecuteTool(tools: Tools): BaseTool { - const name = 'meta_execute_tools' as const; - const description = 'Execute a tool based on the provided parameters' as const; + const name = 'meta_execute_tool' as const; + const description = + 'Executes a specific tool by name with the provided parameters. Use this after discovering tools with meta_filter_relevant_tools.' as const; const parameters = { type: 'object', properties: { @@ -389,5 +492,49 @@ export function metaExecuteTool(tools: Tools): BaseTool { params: [], } as const satisfies ExecuteConfig; - return new BaseTool(name, description, parameters, executeConfig); + // Create the tool instance + const tool = new BaseTool(name, description, parameters, executeConfig); + + // Override the execute method to handle tool execution + // receives tool name and parameters and executes the tool + tool.execute = async ( + inputParams?: JsonDict | string, + options?: ExecuteOptions + ): Promise => { + try { + // Validate params is either undefined, string, or object + if ( + inputParams !== undefined && + typeof inputParams !== 'string' && + typeof inputParams !== 'object' + ) { + throw new StackOneError( + `Invalid parameters type. Expected object or string, got ${typeof inputParams}. Parameters: ${JSON.stringify(inputParams)}` + ); + } + + // Convert string params to object + const params = typeof inputParams === 'string' ? JSON.parse(inputParams) : inputParams || {}; + + // Extract tool name and parameters + const { toolName, params: toolParams } = params; + + // Find the tool by name + const toolToExecute = tools.getTool(toolName); + if (!toolToExecute) { + throw new StackOneError(`Tool ${toolName} not found`); + } + + // Execute the tool with the provided parameters + return await toolToExecute.execute(toolParams, options); + } catch (error) { + if (error instanceof StackOneError) { + throw error; + } + throw new StackOneError( + `Error executing tool: ${error instanceof Error ? error.message : String(error)}` + ); + } + }; + return tool; } From b5812f71c79fc8c9ba1eaa81edf703915dc2cdef Mon Sep 17 00:00:00 2001 From: ryoppippi <1560508+ryoppippi@users.noreply.github.com> Date: Wed, 6 Aug 2025 19:15:04 +0100 Subject: [PATCH 6/9] feat: add meta tools for dynamic tool discovery and execution - Add meta_filter_relevant_tools for searching tools using Orama BM25 - Add meta_execute_tool for executing discovered tools dynamically - Filter tool returns full parameter schemas for LLM usage - Use synchronous Orama initialization (not async) --- mocks/handlers.ts | 33 +++ src/tests/meta-tools.spec.ts | 437 +++++++++++++++++++++++++++++++++++ src/tool.ts | 13 +- 3 files changed, 478 insertions(+), 5 deletions(-) create mode 100644 src/tests/meta-tools.spec.ts diff --git a/mocks/handlers.ts b/mocks/handlers.ts index 296fa547..b674122c 100644 --- a/mocks/handlers.ts +++ b/mocks/handlers.ts @@ -39,6 +39,39 @@ export const handlers = [ }); }), + // Meta tools test endpoints + http.post('https://api.example.com/hris/employees', async ({ request }) => { + const body = await request.json(); + return HttpResponse.json(body); + }), + + http.get('https://api.example.com/hris/employees', ({ request }) => { + const url = new URL(request.url); + const limit = url.searchParams.get('limit'); + return HttpResponse.json({ limit: limit ? Number(limit) : undefined }); + }), + + http.post('https://api.example.com/hris/time-off', async ({ request }) => { + const body = await request.json(); + return HttpResponse.json(body); + }), + + http.post('https://api.example.com/ats/candidates', async ({ request }) => { + const body = await request.json(); + return HttpResponse.json(body); + }), + + http.get('https://api.example.com/ats/candidates', ({ request }) => { + const url = new URL(request.url); + const status = url.searchParams.get('status'); + return HttpResponse.json({ status }); + }), + + http.post('https://api.example.com/crm/contacts', async ({ request }) => { + const body = await request.json(); + return HttpResponse.json(body); + }), + // Default handler for unmatched requests http.get('*', () => { return HttpResponse.json({ message: 'Mock endpoint' }); diff --git a/src/tests/meta-tools.spec.ts b/src/tests/meta-tools.spec.ts new file mode 100644 index 00000000..bd3c4033 --- /dev/null +++ b/src/tests/meta-tools.spec.ts @@ -0,0 +1,437 @@ +import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test'; +import { BaseTool, type MetaToolSearchResult, Tools } from '../tool'; +import { ParameterLocation } from '../types'; + +// Create mock tools for testing +const createMockTools = (): BaseTool[] => { + const tools: BaseTool[] = []; + + // HRIS tools + tools.push( + new BaseTool( + 'hris_create_employee', + 'Create a new employee record in the HRIS system', + { + type: 'object', + properties: { + name: { type: 'string', description: 'Employee name' }, + email: { type: 'string', description: 'Employee email' }, + }, + required: ['name', 'email'], + }, + { + method: 'POST', + url: 'https://api.example.com/hris/employees', + bodyType: 'json', + params: [], + } + ) + ); + + tools.push( + new BaseTool( + 'hris_list_employees', + 'List all employees in the HRIS system', + { + type: 'object', + properties: { + limit: { type: 'number', description: 'Number of employees to return' }, + }, + }, + { + method: 'GET', + url: 'https://api.example.com/hris/employees', + bodyType: 'json', + params: [ + { + name: 'limit', + location: ParameterLocation.QUERY, + type: 'number', + }, + ], + } + ) + ); + + tools.push( + new BaseTool( + 'hris_create_time_off', + 'Create a time off request for an employee', + { + type: 'object', + properties: { + employeeId: { type: 'string', description: 'Employee ID' }, + startDate: { type: 'string', description: 'Start date of time off' }, + endDate: { type: 'string', description: 'End date of time off' }, + }, + required: ['employeeId', 'startDate', 'endDate'], + }, + { + method: 'POST', + url: 'https://api.example.com/hris/time-off', + bodyType: 'json', + params: [], + } + ) + ); + + // ATS tools + tools.push( + new BaseTool( + 'ats_create_candidate', + 'Create a new candidate in the ATS', + { + type: 'object', + properties: { + name: { type: 'string', description: 'Candidate name' }, + email: { type: 'string', description: 'Candidate email' }, + }, + required: ['name', 'email'], + }, + { + method: 'POST', + url: 'https://api.example.com/ats/candidates', + bodyType: 'json', + params: [], + } + ) + ); + + tools.push( + new BaseTool( + 'ats_list_candidates', + 'List all candidates in the ATS', + { + type: 'object', + properties: { + status: { type: 'string', description: 'Filter by candidate status' }, + }, + }, + { + method: 'GET', + url: 'https://api.example.com/ats/candidates', + bodyType: 'json', + params: [ + { + name: 'status', + location: ParameterLocation.QUERY, + type: 'string', + }, + ], + } + ) + ); + + // CRM tools + tools.push( + new BaseTool( + 'crm_create_contact', + 'Create a new contact in the CRM', + { + type: 'object', + properties: { + name: { type: 'string', description: 'Contact name' }, + company: { type: 'string', description: 'Company name' }, + }, + required: ['name'], + }, + { + method: 'POST', + url: 'https://api.example.com/crm/contacts', + bodyType: 'json', + params: [], + } + ) + ); + + return tools; +}; + +describe('Meta Tools', () => { + let tools: Tools; + let metaTools: Tools; + + beforeEach(async () => { + const mockTools = createMockTools(); + tools = new Tools(mockTools); + metaTools = await tools.metaTools(); + }); + + afterEach(() => { + mock.restore(); + }); + + describe('metaTools()', () => { + it('should return two meta tools', () => { + expect(metaTools.length).toBe(2); + }); + + it('should include meta_filter_relevant_tools', () => { + const filterTool = metaTools.getTool('meta_filter_relevant_tools'); + expect(filterTool).toBeDefined(); + expect(filterTool?.name).toBe('meta_filter_relevant_tools'); + }); + + it('should include meta_execute_tool', () => { + const executeTool = metaTools.getTool('meta_execute_tool'); + expect(executeTool).toBeDefined(); + expect(executeTool?.name).toBe('meta_execute_tool'); + }); + }); + + describe('meta_filter_relevant_tools', () => { + let filterTool: BaseTool; + + beforeEach(() => { + const tool = metaTools.getTool('meta_filter_relevant_tools'); + if (!tool) throw new Error('meta_filter_relevant_tools not found'); + filterTool = tool; + }); + + it('should find relevant HRIS tools', async () => { + const result = await filterTool.execute({ + query: 'manage employees in HRIS', + limit: 5, + }); + + expect(result.tools).toBeDefined(); + expect(Array.isArray(result.tools)).toBe(true); + + const toolResults = result.tools as MetaToolSearchResult[]; + const toolNames = toolResults.map((t) => t.name); + + // Should find HRIS-related tools + expect(toolNames).toContain('hris_create_employee'); + expect(toolNames).toContain('hris_list_employees'); + }); + + it('should find time off related tools', async () => { + const result = await filterTool.execute({ + query: 'time off request vacation leave', + limit: 3, + }); + + const toolResults = result.tools as MetaToolSearchResult[]; + const toolNames = toolResults.map((t) => t.name); + + expect(toolNames).toContain('hris_create_time_off'); + }); + + it('should respect limit parameter', async () => { + const result = await filterTool.execute({ + query: 'create', + limit: 2, + }); + + const toolResults = result.tools as MetaToolSearchResult[]; + expect(toolResults.length).toBeLessThanOrEqual(2); + }); + + it('should filter by minimum score', async () => { + const result = await filterTool.execute({ + query: 'xyz123 nonexistent', + minScore: 0.8, // High threshold + }); + + const toolResults = result.tools as MetaToolSearchResult[]; + expect(toolResults.length).toBe(0); + }); + + it('should include tool configurations in results', async () => { + const result = await filterTool.execute({ + query: 'create employee', + limit: 1, + }); + + const toolResults = result.tools as MetaToolSearchResult[]; + expect(toolResults.length).toBeGreaterThan(0); + + const firstTool = toolResults[0]; + expect(firstTool).toHaveProperty('name'); + expect(firstTool).toHaveProperty('description'); + expect(firstTool).toHaveProperty('parameters'); + expect(firstTool).toHaveProperty('score'); + expect(typeof firstTool.score).toBe('number'); + }); + + it('should handle empty query', async () => { + const result = await filterTool.execute({ + query: '', + limit: 5, + }); + + expect(result.tools).toBeDefined(); + expect(Array.isArray(result.tools)).toBe(true); + }); + + it('should handle string parameters', async () => { + const result = await filterTool.execute( + JSON.stringify({ + query: 'candidates', + limit: 3, + }) + ); + + const toolResults = result.tools as MetaToolSearchResult[]; + const toolNames = toolResults.map((t) => t.name); + + expect(toolNames).toContain('ats_create_candidate'); + expect(toolNames).toContain('ats_list_candidates'); + }); + }); + + describe('meta_execute_tool', () => { + let executeTool: BaseTool; + + beforeEach(() => { + const tool = metaTools.getTool('meta_execute_tool'); + if (!tool) throw new Error('meta_execute_tool not found'); + executeTool = tool; + }); + + it('should execute a tool by name', async () => { + const result = await executeTool.execute({ + toolName: 'hris_list_employees', + params: { limit: 10 }, + }); + + // The mock tool returns the params + expect(result).toEqual({ limit: 10 }); + }); + + it('should handle tools with required parameters', async () => { + const result = await executeTool.execute({ + toolName: 'hris_create_employee', + params: { + name: 'John Doe', + email: 'john@example.com', + }, + }); + + expect(result).toEqual({ + name: 'John Doe', + email: 'john@example.com', + }); + }); + + it('should throw error for non-existent tool', async () => { + try { + await executeTool.execute({ + toolName: 'nonexistent_tool', + params: {}, + }); + expect(true).toBe(false); // Should not reach here + } catch (error) { + expect((error as Error).message).toContain('Tool nonexistent_tool not found'); + } + }); + + it('should handle string parameters', async () => { + const result = await executeTool.execute( + JSON.stringify({ + toolName: 'crm_create_contact', + params: { + name: 'Jane Smith', + company: 'Acme Corp', + }, + }) + ); + + expect(result).toEqual({ + name: 'Jane Smith', + company: 'Acme Corp', + }); + }); + + it('should pass through execution options', async () => { + const result = await executeTool.execute({ + toolName: 'ats_list_candidates', + params: { status: 'active' }, + }); + + expect(result).toEqual({ status: 'active' }); + }); + }); + + describe('Integration: meta tools workflow', () => { + it('should discover and execute tools in sequence', async () => { + const filterTool = metaTools.getTool('meta_filter_relevant_tools'); + const executeTool = metaTools.getTool('meta_execute_tool'); + if (!filterTool || !executeTool) throw new Error('Meta tools not found'); + + // Step 1: Discover relevant tools + const searchResult = await filterTool.execute({ + query: 'create new employee in HR system', + limit: 3, + }); + + const toolResults = searchResult.tools as MetaToolSearchResult[]; + expect(toolResults.length).toBeGreaterThan(0); + + // Find the create employee tool + const createEmployeeTool = toolResults.find((t) => t.name === 'hris_create_employee'); + expect(createEmployeeTool).toBeDefined(); + + // Step 2: Execute the discovered tool + const executeResult = await executeTool.execute({ + toolName: createEmployeeTool.name, + params: { + name: 'Alice Johnson', + email: 'alice@example.com', + }, + }); + + expect(executeResult).toEqual({ + name: 'Alice Johnson', + email: 'alice@example.com', + }); + }); + }); + + describe('OpenAI format', () => { + it('should convert meta tools to OpenAI format', () => { + const openAITools = metaTools.toOpenAI(); + + expect(openAITools).toHaveLength(2); + + const filterTool = openAITools.find((t) => t.function.name === 'meta_filter_relevant_tools'); + expect(filterTool).toBeDefined(); + expect(filterTool?.function.parameters?.properties).toHaveProperty('query'); + expect(filterTool?.function.parameters?.properties).toHaveProperty('limit'); + expect(filterTool?.function.parameters?.properties).toHaveProperty('minScore'); + + const executeTool = openAITools.find((t) => t.function.name === 'meta_execute_tool'); + expect(executeTool).toBeDefined(); + expect(executeTool?.function.parameters?.properties).toHaveProperty('toolName'); + expect(executeTool?.function.parameters?.properties).toHaveProperty('params'); + }); + }); + + describe('AI SDK format', () => { + it('should convert meta tools to AI SDK format', () => { + const aiSdkTools = metaTools.toAISDK(); + + expect(aiSdkTools).toHaveProperty('meta_filter_relevant_tools'); + expect(aiSdkTools).toHaveProperty('meta_execute_tool'); + + expect(typeof aiSdkTools.meta_filter_relevant_tools.execute).toBe('function'); + expect(typeof aiSdkTools.meta_execute_tool.execute).toBe('function'); + }); + + it('should execute through AI SDK format', async () => { + const aiSdkTools = metaTools.toAISDK(); + + const result = await aiSdkTools.meta_filter_relevant_tools.execute?.( + { query: 'ATS candidates', limit: 2 }, + { toolCallId: 'test-call-1', messages: [] } + ); + if (!result) throw new Error('No result from execute'); + + const toolResults = (result as { tools: MetaToolSearchResult[] }).tools; + expect(Array.isArray(toolResults)).toBe(true); + + const toolNames = toolResults.map((t) => t.name); + expect(toolNames).toContain('ats_create_candidate'); + }); + }); +}); diff --git a/src/tool.ts b/src/tool.ts index cad73696..b8048610 100644 --- a/src/tool.ts +++ b/src/tool.ts @@ -332,17 +332,18 @@ export interface MetaToolSearchResult { score: number; } -type OramaDb = Awaited>; +type OramaDb = ReturnType; /** * Initialize Orama database with BM25 algorithm for tool search * Using Orama's BM25 scoring algorithm for relevance ranking + * @see https://docs.orama.com/open-source/usage/create * @see https://docs.orama.com/open-source/usage/search/bm25-algorithm/ */ async function initializeOramaDb(tools: BaseTool[]): Promise { // Create Orama database schema with BM25 scoring algorithm // BM25 provides better relevance ranking for natural language queries - const oramaDb = await orama.create({ + const oramaDb = orama.create({ schema: { name: 'string' as const, description: 'string' as const, @@ -366,7 +367,7 @@ async function initializeOramaDb(tools: BaseTool[]): Promise { const actionTypes = ['create', 'update', 'delete', 'get', 'list', 'search']; const actions = parts.filter((p) => actionTypes.includes(p)); - await orama.insert(oramaDb, { + orama.insert(oramaDb, { name: tool.name, description: tool.description, category: category, @@ -428,10 +429,11 @@ export function metaFilterRelevantTools(oramaDb: OramaDb, allTools: BaseTool[]): const params = typeof inputParams === 'string' ? JSON.parse(inputParams) : inputParams || {}; // Perform search using Orama + // Type assertion needed due to TypeScript deep instantiation issue with Orama types const results = await orama.search(oramaDb, { term: params.query || '', limit: params.limit || 5, - } as any); + } as Parameters[1]); // filter results by minimum score const minScore = params.minScore || 0.3; @@ -440,7 +442,8 @@ export function metaFilterRelevantTools(oramaDb: OramaDb, allTools: BaseTool[]): // Map the results to include tool configurations const toolConfigs = filteredResults .map((hit) => { - const tool = allTools.find((t) => t.name === hit.document.name); + const doc = hit.document as { name: string }; + const tool = allTools.find((t) => t.name === doc.name); if (!tool) return null; const result: MetaToolSearchResult = { From c8a9d0e0541667cd45c4dcb5cdde0ea2084c412b Mon Sep 17 00:00:00 2001 From: ryoppippi <1560508+ryoppippi@users.noreply.github.com> Date: Wed, 6 Aug 2025 19:15:18 +0100 Subject: [PATCH 7/9] docs: add meta tools usage examples - AI SDK integration example with dynamic tool discovery - OpenAI integration example for agent workflows - Direct usage example without AI frameworks - Dynamic tool router pattern for flexible workflows --- examples/meta-tools.ts | 259 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 259 insertions(+) create mode 100644 examples/meta-tools.ts diff --git a/examples/meta-tools.ts b/examples/meta-tools.ts new file mode 100644 index 00000000..a518d159 --- /dev/null +++ b/examples/meta-tools.ts @@ -0,0 +1,259 @@ +/** + * This example demonstrates how to use meta tools for dynamic tool discovery and execution. + * Meta tools allow AI agents to search for relevant tools based on natural language queries + * and execute them dynamically without hardcoding tool names. + * + * @beta Meta tools are in beta and may change in future versions + */ + +import process from 'node:process'; +import { openai } from '@ai-sdk/openai'; +import { generateText } from 'ai'; +import { StackOneToolSet } from '../src'; +import { ACCOUNT_IDS } from './constants'; + +/** + * Example 1: Using meta tools with AI SDK for dynamic tool discovery + */ +const metaToolsWithAISDK = async (): Promise => { + console.log('🔍 Example 1: Dynamic tool discovery with AI SDK\n'); + + // Initialize StackOne toolset with all available tools + const toolset = new StackOneToolSet(); + const accountId = ACCOUNT_IDS.HRIS; + + // Get all available tools for the account + const allTools = toolset.getStackOneTools('*', accountId); + + // Get meta tools for dynamic discovery and execution + const metaTools = await allTools.metaTools(); + const aiSdkMetaTools = metaTools.toAISDK(); + + // Use meta tools to dynamically find and execute relevant tools + const { text, toolCalls } = await generateText({ + model: openai('gpt-4o-mini'), + tools: aiSdkMetaTools, + prompt: `I need to create a time off request for an employee. + First, find the right tool for this task, then use it to create a time off request + for employee ID "emp_123" from January 15, 2024 to January 19, 2024.`, + maxSteps: 3, // Allow multiple tool calls + }); + + console.log('AI Response:', text); + console.log('\nTool calls made:', toolCalls?.map((call) => call.toolName).join(', ')); +}; + +/** + * Example 2: Using meta tools with OpenAI for HR assistant + */ +const metaToolsWithOpenAI = async (): Promise => { + console.log('\n🤖 Example 2: HR Assistant with OpenAI\n'); + + const { OpenAI } = await import('openai'); + const openaiClient = new OpenAI({ + apiKey: process.env.OPENAI_API_KEY, + }); + + // Initialize StackOne toolset + const toolset = new StackOneToolSet(); + const accountId = ACCOUNT_IDS.HRIS; + + // Get all HRIS tools + const hrisTools = toolset.getStackOneTools('hris_*', accountId); + + // Get meta tools + const metaTools = await hrisTools.metaTools(); + const openAIMetaTools = metaTools.toOpenAI(); + + // Create an HR assistant that can discover and use tools dynamically + const response = await openaiClient.chat.completions.create({ + model: 'gpt-4o-mini', + messages: [ + { + role: 'system', + content: `You are an HR assistant with access to various HR tools. + Use the meta_filter_relevant_tools to find appropriate tools for user requests, + then use meta_execute_tool to execute them.`, + }, + { + role: 'user', + content: + 'Can you help me find tools for managing employee records and then list current employees?', + }, + ], + tools: openAIMetaTools, + tool_choice: 'auto', + }); + + console.log('Assistant response:', response.choices[0].message.content); + + // Handle tool calls if any + if (response.choices[0].message.tool_calls) { + console.log('\nTool calls:'); + for (const toolCall of response.choices[0].message.tool_calls) { + console.log(`- ${toolCall.function.name}: ${toolCall.function.arguments}`); + } + } +}; + +/** + * Example 3: Direct usage of meta tools without AI + */ +const directMetaToolUsage = async (): Promise => { + console.log('\n🛠️ Example 3: Direct meta tool usage\n'); + + // Initialize toolset + const toolset = new StackOneToolSet(); + const accountId = ACCOUNT_IDS.HRIS; + + // Get all available tools + const allTools = toolset.getStackOneTools('*', accountId); + console.log(`Total available tools: ${allTools.length}`); + + // Get meta tools + const metaTools = await allTools.metaTools(); + + // Step 1: Search for relevant tools + const filterTool = metaTools.getTool('meta_filter_relevant_tools'); + if (!filterTool) throw new Error('meta_filter_relevant_tools not found'); + const searchResult = await filterTool.execute({ + query: 'employee management create update list', + limit: 5, + minScore: 0.3, + }); + + console.log('Found relevant tools:'); + const foundTools = searchResult.tools as Array<{ + name: string; + description: string; + score: number; + }>; + for (const tool of foundTools) { + console.log(`- ${tool.name} (score: ${tool.score.toFixed(2)}): ${tool.description}`); + } + + // Step 2: Execute one of the found tools + if (foundTools.length > 0) { + const executeTool = metaTools.getTool('meta_execute_tool'); + if (!executeTool) throw new Error('meta_execute_tool not found'); + const firstTool = foundTools[0]; + + console.log(`\nExecuting ${firstTool.name}...`); + + try { + // Prepare parameters based on the tool's schema + let params: Record = {}; + if (firstTool.name === 'hris_list_employees') { + params = { limit: 5 }; + } else if (firstTool.name === 'hris_create_employee') { + params = { + name: 'John Doe', + email: 'john.doe@example.com', + title: 'Software Engineer', + }; + } + + const result = await executeTool.execute({ + toolName: firstTool.name, + params: params, + }); + + console.log('Execution result:', JSON.stringify(result, null, 2)); + } catch (error) { + console.error('Execution failed:', error); + } + } +}; + +/** + * Example 4: Building a dynamic tool router + */ +const dynamicToolRouter = async (): Promise => { + console.log('\n🔄 Example 4: Dynamic tool router\n'); + + const toolset = new StackOneToolSet(); + const accountId = ACCOUNT_IDS.HRIS; + + // Get tools from multiple categories + const hrisTools = toolset.getStackOneTools('hris_*', accountId); + const atsTools = toolset.getStackOneTools('ats_*', accountId); + + // Combine tools + const combinedTools = new (await import('../src')).Tools([ + ...hrisTools.toArray(), + ...atsTools.toArray(), + ]); + + // Get meta tools for the combined set + const metaTools = await combinedTools.metaTools(); + + // Create a router function that finds and executes tools based on intent + const routeAndExecute = async (intent: string, params: Record = {}) => { + const filterTool = metaTools.getTool('meta_filter_relevant_tools'); + const executeTool = metaTools.getTool('meta_execute_tool'); + if (!filterTool || !executeTool) throw new Error('Meta tools not found'); + + // Find relevant tools + const searchResult = await filterTool.execute({ + query: intent, + limit: 1, + minScore: 0.5, + }); + + const tools = searchResult.tools as Array<{ name: string; score: number }>; + if (tools.length === 0) { + return { error: 'No relevant tools found for the given intent' }; + } + + const selectedTool = tools[0]; + console.log(`Routing to: ${selectedTool.name} (score: ${selectedTool.score.toFixed(2)})`); + + // Execute the selected tool + return await executeTool.execute({ + toolName: selectedTool.name, + params: params, + }); + }; + + // Test the router with different intents + const intents = [ + { intent: 'I want to see all employees', params: { limit: 10 } }, + { + intent: 'Create a new job candidate', + params: { name: 'Jane Smith', email: 'jane@example.com' }, + }, + { intent: 'Find recruitment candidates', params: { status: 'active' } }, + ]; + + for (const { intent, params } of intents) { + console.log(`\nIntent: "${intent}"`); + const result = await routeAndExecute(intent, params); + console.log('Result:', JSON.stringify(result, null, 2)); + } +}; + +// Main execution +const main = async () => { + try { + // Run examples based on environment setup + if (process.env.OPENAI_API_KEY) { + await metaToolsWithAISDK(); + await metaToolsWithOpenAI(); + } else { + console.log('⚠️ OPENAI_API_KEY not found, skipping AI examples\n'); + } + + // These examples work without AI + await directMetaToolUsage(); + await dynamicToolRouter(); + } catch (error) { + console.error('Error running examples:', error); + } +}; + +// Run if this file is executed directly +if (import.meta.main) { + main(); +} + +export { metaToolsWithAISDK, metaToolsWithOpenAI, directMetaToolUsage, dynamicToolRouter }; From 391b15c0620df48909a8d5a31bbedb77901cab7a Mon Sep 17 00:00:00 2001 From: ryoppippi <1560508+ryoppippi@users.noreply.github.com> Date: Wed, 6 Aug 2025 19:15:30 +0100 Subject: [PATCH 8/9] docs: add meta tools documentation to README - Add beta warning for meta tools feature - Explain how meta tools enable dynamic tool discovery - Provide usage examples with AI SDK and OpenAI - Document the workflow and benefits --- README.md | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/README.md b/README.md index c20783c0..ea0c80e9 100644 --- a/README.md +++ b/README.md @@ -224,6 +224,77 @@ const toolsetWithHeaders = new OpenAPIToolSet({ These are some of the features which you can use with the OpenAPIToolSet and StackOneToolSet. +### Meta Tools (Beta) + +Meta tools enable dynamic tool discovery and execution, allowing AI agents to search for relevant tools based on natural language queries without hardcoding tool names. + +> ⚠️ **Beta Feature**: Meta tools are currently in beta and the API may change in future versions. + +#### How Meta Tools Work + +Meta tools provide two core capabilities: +1. **Tool Discovery** (`meta_filter_relevant_tools`): Search for tools using natural language queries +2. **Tool Execution** (`meta_execute_tool`): Execute discovered tools dynamically + +The tool discovery uses Orama's BM25 algorithm for relevance ranking, providing high-quality search results based on tool names, descriptions, and categories. + +#### Basic Usage + +```typescript +import { StackOneToolSet } from "@stackone/ai"; + +const toolset = new StackOneToolSet(); +const tools = toolset.getStackOneTools("*", "account-id"); + +// Get meta tools for dynamic discovery +const metaTools = await tools.metaTools(); + +// Use with OpenAI +const openAITools = metaTools.toOpenAI(); + +// Use with AI SDK +const aiSdkTools = metaTools.toAISDK(); +``` + +#### Example: Dynamic Tool Discovery with AI SDK + +```typescript +import { generateText } from "ai"; +import { openai } from "@ai-sdk/openai"; + +const { text } = await generateText({ + model: openai("gpt-4o-mini"), + tools: aiSdkTools, + prompt: "Find tools for managing employees and create a time off request", + maxSteps: 3, // Allow multiple tool calls +}); +``` + +#### Direct Usage Without AI + +```typescript +// Step 1: Discover relevant tools +const filterTool = metaTools.getTool("meta_filter_relevant_tools"); +const searchResult = await filterTool.execute({ + query: "employee time off vacation", + limit: 5, + minScore: 0.3, // Minimum relevance score (0-1) +}); + +// Step 2: Execute a discovered tool +const executeTool = metaTools.getTool("meta_execute_tool"); +const result = await executeTool.execute({ + toolName: "hris_create_time_off", + params: { + employeeId: "emp_123", + startDate: "2024-01-15", + endDate: "2024-01-19", + }, +}); +``` + +[View full example](examples/meta-tools.ts) + ### Custom Base URL ```typescript From 66503e2f88881a2114879cc13eda3ca432bce442 Mon Sep 17 00:00:00 2001 From: ryoppippi <1560508+ryoppippi@users.noreply.github.com> Date: Thu, 7 Aug 2025 00:29:37 +0100 Subject: [PATCH 9/9] Update src/tool.ts Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com> --- src/tool.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tool.ts b/src/tool.ts index b8048610..a374547c 100644 --- a/src/tool.ts +++ b/src/tool.ts @@ -436,7 +436,7 @@ export function metaFilterRelevantTools(oramaDb: OramaDb, allTools: BaseTool[]): } as Parameters[1]); // filter results by minimum score - const minScore = params.minScore || 0.3; + const minScore = params.minScore ?? 0.3; const filteredResults = results.hits.filter((hit) => hit.score >= minScore); // Map the results to include tool configurations