diff --git a/.env.example b/.env.example index ed4442205..98ef53bb6 100644 --- a/.env.example +++ b/.env.example @@ -35,4 +35,22 @@ STORAGE_DATABASE_TYPE=in-memory # ==================== # MCP Configuration # ==================== -# MCP_GLOBAL_TIMEOUT=30000 \ No newline at end of file +# MCP_GLOBAL_TIMEOUT=30000 + + +# Vector store type: qdrant, in-memory +#VECTOR_STORE_TYPE=in-memory + +# Qdrant configuration (only used if VECTOR_STORE_TYPE=qdrant) +#VECTOR_STORE_HOST=localhost +#VECTOR_STORE_PORT=6333 +#VECTOR_STORE_URL=http://localhost:6333 +#VECTOR_STORE_API_KEY=your-qdrant-api-key + +# Vector collection settings +# VECTOR_STORE_COLLECTION=default +# VECTOR_STORE_COLLECTION_NAME=your_collection_name +# VECTOR_STORE_DIMENSION=1536 +# VECTOR_STORE_DISTANCE=Cosine +# VECTOR_STORE_ON_DISK=false +# VECTOR_STORE_MAX_VECTORS=10000 \ No newline at end of file diff --git a/memAgent/cipher.yml b/memAgent/cipher.yml index 01290e0b0..937d39227 100644 --- a/memAgent/cipher.yml +++ b/memAgent/cipher.yml @@ -26,3 +26,7 @@ llm: # System prompt systemPrompt: "You are a helpful AI assistant with memory capabilities. Please confirm you're working with OpenRouter API." + + + + diff --git a/src/app/index.ts b/src/app/index.ts index 05a171531..9e6213497 100644 --- a/src/app/index.ts +++ b/src/app/index.ts @@ -68,6 +68,13 @@ program // Start the agent (initialize async services) await agent.start(); + + // Print OpenAI embedder dimension after agent is started + if (agent.services && agent.services.embeddingManager) { + const embedder = agent.services.embeddingManager.getEmbedder('default'); + } else { + console.log('No embeddingManager found in agent.services'); + } } catch (err) { logger.error( 'Failed to load agent config:', diff --git a/src/core/brain/embedding/backend/index.ts b/src/core/brain/embedding/backend/index.ts new file mode 100644 index 000000000..93083c4b1 --- /dev/null +++ b/src/core/brain/embedding/backend/index.ts @@ -0,0 +1,31 @@ +/** + * Embedding Backend Module Exports + * + * Central export point for all embedding backend implementations and types. + * Provides a clean interface for accessing embedding providers and utilities. + * + * @module embedding/backend + */ + +// Export core types and interfaces +export type { + Embedder, + EmbeddingConfig, + OpenAIEmbeddingConfig, + BackendConfig, + EmbeddingResult, + BatchEmbeddingResult, +} from './types.js'; + +// Export error classes +export { + EmbeddingError, + EmbeddingConnectionError, + EmbeddingDimensionError, + EmbeddingRateLimitError, + EmbeddingQuotaError, + EmbeddingValidationError, +} from './types.js'; + +// Export backend implementations +export { OpenAIEmbedder } from './openai.js'; diff --git a/src/core/brain/embedding/backend/openai.ts b/src/core/brain/embedding/backend/openai.ts new file mode 100644 index 000000000..2248dc0ed --- /dev/null +++ b/src/core/brain/embedding/backend/openai.ts @@ -0,0 +1,388 @@ +/** + * OpenAI Embedding Backend + * + * Implementation of the Embedder interface for OpenAI's embedding services. + * Supports all OpenAI embedding models with batch processing, retry logic, + * and comprehensive error handling. + * + * @module embedding/backend/openai + */ + +import OpenAI from 'openai'; +import { logger } from '../../../logger/index.js'; +import { + Embedder, + OpenAIEmbeddingConfig, + EmbeddingConnectionError, + EmbeddingRateLimitError, + EmbeddingQuotaError, + EmbeddingValidationError, + EmbeddingError, + EmbeddingDimensionError, +} from './types.js'; +import { + MODEL_DIMENSIONS, + VALIDATION_LIMITS, + ERROR_MESSAGES, + LOG_PREFIXES, + RETRY_CONFIG, + HTTP_STATUS, +} from '../constants.js'; + +/** + * OpenAI Embedder Implementation + * + * Provides embedding functionality using OpenAI's embedding API. + * Implements comprehensive error handling, retry logic, and batch processing. + */ +export class OpenAIEmbedder implements Embedder { + private openai: OpenAI; + private readonly config: OpenAIEmbeddingConfig; + private readonly model: string; + private readonly dimension: number; + + constructor(config: OpenAIEmbeddingConfig) { + this.config = config; + this.model = config.model || 'text-embedding-3-small'; + + // Initialize OpenAI client + this.openai = new OpenAI({ + apiKey: config.apiKey, + baseURL: config.baseUrl, + organization: config.organization, + timeout: config.timeout, + maxRetries: config.maxRetries, + }); + + // Set dimension based on model and config + this.dimension = + config.dimensions || MODEL_DIMENSIONS[this.model as keyof typeof MODEL_DIMENSIONS] || 1536; + + logger.debug(`${LOG_PREFIXES.OPENAI} Initialized OpenAI embedder`, { + model: this.model, + dimension: this.dimension, + baseUrl: config.baseUrl, + hasOrganization: !!config.organization, + }); + } + + async embed(text: string): Promise { + logger.silly(`${LOG_PREFIXES.OPENAI} Embedding single text`, { + textLength: text.length, + model: this.model, + }); + + // Validate input + this.validateInput(text); + + const startTime = Date.now(); + + try { + const params: { model: string; input: string; dimensions?: number } = { + model: this.model, + input: text, + }; + if (this.config.dimensions !== undefined) { + params.dimensions = this.config.dimensions; + } + const response = await this.createEmbeddingWithRetry(params); + if ( + !response.data || + !Array.isArray(response.data) || + !response.data[0] || + !response.data[0].embedding + ) { + throw new EmbeddingError('OpenAI API did not return a valid embedding', 'openai'); + } + const embedding = response.data[0].embedding; + this.validateEmbeddingDimension(embedding); + return embedding; + } catch (error) { + const processingTime = Date.now() - startTime; + logger.error(`${LOG_PREFIXES.OPENAI} Failed to create embedding`, { + error: error instanceof Error ? error.message : String(error), + model: this.model, + processingTime, + textLength: text.length, + }); + + throw this.handleApiError(error); + } + } + + async embedBatch(texts: string[]): Promise { + logger.debug(`${LOG_PREFIXES.BATCH} Embedding batch of texts`, { + count: texts.length, + model: this.model, + }); + + // Validate batch input + this.validateBatchInput(texts); + + const startTime = Date.now(); + + try { + const batchParams: { model: string; input: string[]; dimensions?: number } = { + model: this.model, + input: texts, + }; + if (this.config.dimensions !== undefined) { + batchParams.dimensions = this.config.dimensions; + } + const response = await this.createEmbeddingWithRetry(batchParams); + const embeddings = response.data.map(item => item.embedding); + embeddings.forEach(this.validateEmbeddingDimension.bind(this)); + return embeddings; + } catch (error) { + const processingTime = Date.now() - startTime; + logger.error(`${LOG_PREFIXES.BATCH} Failed to create batch embeddings`, { + error: error instanceof Error ? error.message : String(error), + model: this.model, + processingTime, + count: texts.length, + }); + + throw this.handleApiError(error); + } + } + + getDimension(): number { + return this.dimension; + } + + getConfig(): OpenAIEmbeddingConfig { + return { ...this.config }; + } + + async isHealthy(): Promise { + try { + logger.silly(`${LOG_PREFIXES.HEALTH} Checking OpenAI embedder health`); + + // Try a simple embedding request with minimal text + await this.embed('test'); + + logger.debug(`${LOG_PREFIXES.HEALTH} OpenAI embedder is healthy`); + return true; + } catch (error) { + logger.warn(`${LOG_PREFIXES.HEALTH} OpenAI embedder health check failed`, { + error: error instanceof Error ? error.message : String(error), + }); + return false; + } + } + + async disconnect(): Promise { + logger.debug(`${LOG_PREFIXES.OPENAI} Disconnecting OpenAI embedder`); + // OpenAI client doesn't require explicit cleanup + // This is here for interface compliance and future extensibility + } + + /** + * Create embedding with retry logic + */ + private async createEmbeddingWithRetry(params: { + model: string; + input: string | string[]; + dimensions?: number; + }): Promise { + let lastError: Error | undefined; + let delay: number = RETRY_CONFIG.INITIAL_DELAY; + + for (let attempt = 0; attempt <= this.config.maxRetries!; attempt++) { + try { + if (attempt > 0) { + logger.debug(`${LOG_PREFIXES.OPENAI} Retrying embedding request`, { + attempt, + delay, + maxRetries: this.config.maxRetries, + }); + + // Wait before retry + await new Promise(resolve => setTimeout(resolve, delay)); + + // Calculate next delay with exponential backoff and jitter + delay = Math.min(delay * RETRY_CONFIG.BACKOFF_MULTIPLIER, RETRY_CONFIG.MAX_DELAY); + + // Add jitter to avoid thundering herd + const jitter = delay * RETRY_CONFIG.JITTER_FACTOR * Math.random(); + delay = Math.floor(delay + jitter); + } + + const response = await this.openai.embeddings.create(params); + + if (attempt > 0) { + logger.info(`${LOG_PREFIXES.OPENAI} Embedding request succeeded after retry`, { + attempt, + model: params.model, + }); + } + + return response; + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); + + // Check if we should retry based on error type + if (!this.shouldRetry(error, attempt)) { + break; + } + + logger.warn(`${LOG_PREFIXES.OPENAI} Embedding request failed, will retry`, { + attempt: attempt + 1, + maxRetries: this.config.maxRetries, + error: lastError.message, + nextDelay: delay, + }); + } + } + + // All retries exhausted + throw lastError || new EmbeddingError('Unknown error during embedding request', 'openai'); + } + + /** + * Determine if an error is retryable + */ + private shouldRetry(error: unknown, attempt: number): boolean { + if (attempt >= this.config.maxRetries!) { + return false; + } + + // Handle OpenAI API errors + if (error && typeof error === 'object' && 'status' in error) { + const status = (error as any).status; + + // Retry on server errors and rate limits + return [ + HTTP_STATUS.TOO_MANY_REQUESTS, + HTTP_STATUS.INTERNAL_SERVER_ERROR, + HTTP_STATUS.SERVICE_UNAVAILABLE, + ].includes(status); + } + + // Retry on network errors + if (error instanceof Error) { + const message = error.message.toLowerCase(); + return ( + message.includes('network') || + message.includes('timeout') || + message.includes('connection') || + message.includes('econnreset') || + message.includes('enotfound') + ); + } + + return false; + } + + /** + * Handle and categorize API errors + */ + private handleApiError(error: unknown): EmbeddingError { + if (error && typeof error === 'object' && 'status' in error) { + const apiError = error as any; + const status = apiError.status; + const message = apiError.message || String(error); + + switch (status) { + case HTTP_STATUS.UNAUTHORIZED: + return new EmbeddingConnectionError( + ERROR_MESSAGES.INVALID_API_KEY('OpenAI'), + 'openai', + apiError + ); + + case HTTP_STATUS.TOO_MANY_REQUESTS: { + const retryAfter = apiError.headers?.['retry-after']; + return new EmbeddingRateLimitError( + ERROR_MESSAGES.RATE_LIMIT_EXCEEDED, + retryAfter ? parseInt(retryAfter, 10) : undefined, + 'openai', + apiError + ); + } + + case HTTP_STATUS.FORBIDDEN: + return new EmbeddingQuotaError(ERROR_MESSAGES.QUOTA_EXCEEDED, 'openai', apiError); + + case HTTP_STATUS.BAD_REQUEST: + return new EmbeddingValidationError(message, 'openai', apiError); + + default: + return new EmbeddingConnectionError( + ERROR_MESSAGES.CONNECTION_FAILED('OpenAI'), + 'openai', + apiError + ); + } + } + + // Handle network and other errors + if (error instanceof Error) { + return new EmbeddingConnectionError(error.message, 'openai', error); + } + + return new EmbeddingError(String(error), 'openai'); + } + + /** + * Validate single text input + */ + private validateInput(text: string): void { + if (!text || text.length < VALIDATION_LIMITS.MIN_TEXT_LENGTH) { + throw new EmbeddingValidationError(ERROR_MESSAGES.EMPTY_TEXT, 'openai'); + } + + if (text.length > VALIDATION_LIMITS.MAX_TEXT_LENGTH) { + throw new EmbeddingValidationError( + ERROR_MESSAGES.TEXT_TOO_LONG(text.length, VALIDATION_LIMITS.MAX_TEXT_LENGTH), + 'openai' + ); + } + } + + /** + * Validate batch input + */ + private validateBatchInput(texts: string[]): void { + if (!Array.isArray(texts) || texts.length === 0) { + throw new EmbeddingValidationError('Batch input must be a non-empty array', 'openai'); + } + + if (texts.length > VALIDATION_LIMITS.MAX_BATCH_SIZE) { + throw new EmbeddingValidationError( + ERROR_MESSAGES.BATCH_TOO_LARGE(texts.length, VALIDATION_LIMITS.MAX_BATCH_SIZE), + 'openai' + ); + } + + // Validate each text in the batch + texts.forEach((text, index) => { + try { + this.validateInput(text); + } catch (error) { + if (error instanceof EmbeddingValidationError) { + throw new EmbeddingValidationError( + `Batch item ${index}: ${error.message}`, + 'openai', + error + ); + } + throw error; + } + }); + } + + /** + * Validate embedding dimension + */ + private validateEmbeddingDimension(embedding: number[]): void { + if (embedding.length !== this.dimension) { + throw new EmbeddingDimensionError( + ERROR_MESSAGES.DIMENSION_MISMATCH(this.dimension, embedding.length), + this.dimension, + embedding.length, + 'openai' + ); + } + } +} diff --git a/src/core/brain/embedding/backend/types.ts b/src/core/brain/embedding/backend/types.ts new file mode 100644 index 000000000..95f3e8d03 --- /dev/null +++ b/src/core/brain/embedding/backend/types.ts @@ -0,0 +1,213 @@ +/** + * Embedding Backend Types and Interfaces + * + * Core type definitions for the embedding system backends. + * Provides the fundamental interfaces that all embedding providers must implement. + * + * @module embedding/backend/types + */ + +/** + * Core interface for embedding providers + * + * All embedding backends must implement this interface to provide + * consistent embedding functionality across different providers. + */ +export interface Embedder { + /** + * Generate embedding for a single text input + * + * @param text - The text to embed + * @returns Promise resolving to the embedding vector + */ + embed(text: string): Promise; + + /** + * Generate embeddings for multiple text inputs in batch + * + * @param texts - Array of texts to embed + * @returns Promise resolving to array of embedding vectors + */ + embedBatch(texts: string[]): Promise; + + /** + * Get the dimension of embeddings produced by this embedder + * + * @returns The vector dimension + */ + getDimension(): number; + + /** + * Get the configuration used by this embedder + * + * @returns The embedder configuration + */ + getConfig(): EmbeddingConfig; + + /** + * Check if the embedder is healthy and can process requests + * + * @returns Promise resolving to health status + */ + isHealthy(): Promise; + + /** + * Clean up resources and close connections + */ + disconnect(): Promise; +} + +/** + * Base configuration interface for all embedding providers + */ +export interface EmbeddingConfig { + /** The embedding provider type */ + type: string; + + /** API key for the provider */ + apiKey?: string; + + /** Model name to use for embeddings */ + model?: string; + + /** Base URL for the provider API */ + baseUrl?: string; + + /** Request timeout in milliseconds */ + timeout?: number; + + /** Maximum number of retry attempts */ + maxRetries?: number; + + /** Provider-specific options */ + options?: Record; +} + +/** + * OpenAI-specific embedding configuration + */ +export interface OpenAIEmbeddingConfig extends EmbeddingConfig { + type: 'openai'; + model?: 'text-embedding-3-small' | 'text-embedding-3-large' | 'text-embedding-ada-002'; + /** Organization ID for OpenAI API */ + organization?: string; + /** Custom dimensions for embedding-3 models */ + dimensions?: number; +} + +/** + * Union type for all supported backend configurations + */ +export type BackendConfig = OpenAIEmbeddingConfig; + +/** + * Result from embedding operation with metadata + */ +export interface EmbeddingResult { + /** The embedding vector */ + embedding: number[]; + /** Metadata about the embedding operation */ + metadata: { + /** Model used for embedding */ + model: string; + /** Token count for the input text */ + tokens?: number; + /** Processing time in milliseconds */ + processingTime?: number; + }; +} + +/** + * Batch embedding result + */ +export interface BatchEmbeddingResult { + /** Array of embedding vectors */ + embeddings: number[][]; + /** Metadata about the batch operation */ + metadata: { + /** Model used for embedding */ + model: string; + /** Total token count for all inputs */ + totalTokens?: number; + /** Processing time in milliseconds */ + processingTime?: number; + /** Number of successful embeddings */ + successCount: number; + /** Number of failed embeddings */ + failureCount: number; + }; +} + +/** + * Base error class for embedding operations + */ +export class EmbeddingError extends Error { + constructor( + message: string, + public readonly provider?: string, + public override readonly cause?: Error + ) { + super(message); + this.name = 'EmbeddingError'; + } +} + +/** + * Error thrown when connection to embedding provider fails + */ +export class EmbeddingConnectionError extends EmbeddingError { + constructor(message: string, provider?: string, cause?: Error) { + super(message, provider, cause); + this.name = 'EmbeddingConnectionError'; + } +} + +/** + * Error thrown when embedding dimensions don't match expected values + */ +export class EmbeddingDimensionError extends EmbeddingError { + constructor( + message: string, + public readonly expected: number, + public readonly actual: number, + provider?: string + ) { + super(message, provider); + this.name = 'EmbeddingDimensionError'; + } +} + +/** + * Error thrown when API rate limits are exceeded + */ +export class EmbeddingRateLimitError extends EmbeddingError { + constructor( + message: string, + public readonly retryAfter?: number, + provider?: string, + cause?: Error + ) { + super(message, provider, cause); + this.name = 'EmbeddingRateLimitError'; + } +} + +/** + * Error thrown when API quota is exceeded + */ +export class EmbeddingQuotaError extends EmbeddingError { + constructor(message: string, provider?: string, cause?: Error) { + super(message, provider, cause); + this.name = 'EmbeddingQuotaError'; + } +} + +/** + * Error thrown when input validation fails + */ +export class EmbeddingValidationError extends EmbeddingError { + constructor(message: string, provider?: string, cause?: Error) { + super(message, provider, cause); + this.name = 'EmbeddingValidationError'; + } +} diff --git a/src/core/brain/embedding/config.ts b/src/core/brain/embedding/config.ts new file mode 100644 index 000000000..ef92e51ef --- /dev/null +++ b/src/core/brain/embedding/config.ts @@ -0,0 +1,296 @@ +/** + * Embedding Configuration Module + * + * Defines the configuration schemas for the embedding system using Zod for + * runtime validation and type safety. Supports multiple embedding providers + * with different configuration requirements. + * + * @module embedding/config + */ + +import { z } from 'zod'; +import { + DEFAULTS, + OPENAI_MODELS, + PROVIDER_TYPES, + VALIDATION_LIMITS, + ENV_VARS, +} from './constants.js'; + +/** + * Base Embedding Configuration Schema + * + * Common configuration options shared by all embedding providers. + * These options control model selection, timeouts, and retry behavior. + */ +const BaseEmbeddingSchema = z.object({ + /** API key for the provider (required for all providers) */ + apiKey: z.string().min(1).describe('API key for the embedding provider'), + + /** Model name to use for embeddings */ + model: z.string().min(1).optional().describe('Model name for embeddings'), + + /** Base URL for the provider API */ + baseUrl: z.string().url().optional().describe('Base URL for the provider API'), + + /** Request timeout in milliseconds */ + timeout: z + .number() + .int() + .positive() + .max(300000) // 5 minutes max + .default(DEFAULTS.TIMEOUT) + .describe('Request timeout in milliseconds'), + + /** Maximum number of retry attempts */ + maxRetries: z + .number() + .int() + .min(0) + .max(10) + .default(DEFAULTS.MAX_RETRIES) + .describe('Maximum retry attempts'), + + /** Provider-specific options */ + options: z.record(z.any()).optional().describe('Provider-specific configuration options'), +}); + +/** + * OpenAI Embedding Configuration Schema + * + * Configuration specific to OpenAI embedding services. + * Supports all OpenAI embedding models with validation. + * + * @example + * ```typescript + * const config: OpenAIEmbeddingConfig = { + * type: 'openai', + * apiKey: process.env.OPENAI_API_KEY, + * model: 'text-embedding-3-small' + * }; + * ``` + */ +const OpenAIEmbeddingSchema = BaseEmbeddingSchema.extend({ + type: z.literal(PROVIDER_TYPES.OPENAI), + + /** OpenAI embedding model */ + model: z + .enum([ + OPENAI_MODELS.TEXT_EMBEDDING_3_SMALL, + OPENAI_MODELS.TEXT_EMBEDDING_3_LARGE, + OPENAI_MODELS.TEXT_EMBEDDING_ADA_002, + ] as const) + .default(DEFAULTS.OPENAI_MODEL) + .describe('OpenAI embedding model'), + + /** OpenAI organization ID */ + organization: z.string().optional().describe('OpenAI organization ID'), + + /** Custom dimensions for embedding-3 models */ + dimensions: z + .number() + .int() + .positive() + .max(3072) + .optional() + .describe('Custom embedding dimensions (embedding-3 models only)'), + + /** Base URL override */ + baseUrl: z.string().url().default(DEFAULTS.OPENAI_BASE_URL).describe('OpenAI API base URL'), +}).strict(); + +export type OpenAIEmbeddingConfig = z.infer; + +/** + * Backend Configuration Union Schema + * + * Discriminated union of all supported embedding provider configurations. + * Uses the 'type' field to determine which configuration schema to apply. + */ +const BackendConfigSchema = z + .discriminatedUnion('type', [OpenAIEmbeddingSchema], { + errorMap: (issue, ctx) => { + if (issue.code === z.ZodIssueCode.invalid_union_discriminator) { + return { + message: `Invalid embedding provider type. Expected: ${Object.values(PROVIDER_TYPES).join(', ')}.`, + }; + } + return { message: ctx.defaultError }; + }, + }) + .describe('Backend configuration for embedding system') + .superRefine((data, ctx) => { + // Validate OpenAI-specific requirements + if (data.type === PROVIDER_TYPES.OPENAI) { + // Check if dimensions are specified for models that support it + if (data.dimensions) { + const supportsCustomDimensions = + data.model === OPENAI_MODELS.TEXT_EMBEDDING_3_SMALL || + data.model === OPENAI_MODELS.TEXT_EMBEDDING_3_LARGE; + + if (!supportsCustomDimensions) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Custom dimensions are only supported for embedding-3 models`, + path: ['dimensions'], + }); + } + + // Validate dimension range for specific models + if (data.model === OPENAI_MODELS.TEXT_EMBEDDING_3_SMALL && data.dimensions > 1536) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `text-embedding-3-small supports max 1536 dimensions`, + path: ['dimensions'], + }); + } + } + } + + // Validate API key format (basic checks) + if (data.apiKey) { + if (data.type === PROVIDER_TYPES.OPENAI && !data.apiKey.startsWith('sk-')) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'OpenAI API key should start with "sk-"', + path: ['apiKey'], + }); + } + } + }); + +export type BackendConfig = z.infer; + +/** + * Embedding System Configuration Schema + * + * Top-level configuration for the embedding system. + * Uses a single backend configuration. + */ +export const EmbeddingConfigSchema = BackendConfigSchema; + +export type EmbeddingConfig = z.infer; + +/** + * Environment-based Configuration Schema + * + * Allows configuration to be loaded from environment variables. + * Useful for deployment scenarios where config is provided via env vars. + */ +export const EmbeddingEnvConfigSchema = z.object({ + /** Provider type from environment */ + type: z + .enum([PROVIDER_TYPES.OPENAI] as const) + .default(PROVIDER_TYPES.OPENAI) + .describe('Embedding provider type'), + + /** API key from environment variables */ + apiKey: z.string().optional().describe('API key from environment variables'), + + /** Model from environment */ + model: z.string().optional().describe('Model name from environment'), + + /** Base URL from environment */ + baseUrl: z.string().url().optional().describe('Base URL from environment'), + + /** Timeout from environment */ + timeout: z + .string() + .transform(val => parseInt(val, 10)) + .pipe(z.number().int().positive()) + .optional() + .describe('Timeout from environment (string converted to number)'), + + /** Max retries from environment */ + maxRetries: z + .string() + .transform(val => parseInt(val, 10)) + .pipe(z.number().int().min(0).max(10)) + .optional() + .describe('Max retries from environment (string converted to number)'), +}); + +export type EmbeddingEnvConfig = z.infer; + +/** + * Parse and validate embedding configuration + * + * @param config - Raw configuration object + * @returns Validated configuration + * @throws {z.ZodError} If configuration is invalid + */ +export function parseEmbeddingConfig(config: unknown): EmbeddingConfig { + return EmbeddingConfigSchema.parse(config); +} + +/** + * Parse embedding configuration from environment variables + * + * @param env - Environment variables object (defaults to process.env) + * @returns Validated configuration or null if required env vars are missing + */ +export function parseEmbeddingConfigFromEnv( + env: Record = process.env +): EmbeddingConfig | null { + try { + // Try to build config from environment variables + const rawConfig: any = { + type: env[ENV_VARS.EMBEDDING_MODEL] ? PROVIDER_TYPES.OPENAI : PROVIDER_TYPES.OPENAI, + }; + + // Add OpenAI-specific config + if (env[ENV_VARS.OPENAI_API_KEY]) { + rawConfig.apiKey = env[ENV_VARS.OPENAI_API_KEY]; + } else { + // No API key found, cannot create config + return null; + } + + if (env[ENV_VARS.EMBEDDING_MODEL]) { + rawConfig.model = env[ENV_VARS.EMBEDDING_MODEL]; + } + + if (env[ENV_VARS.OPENAI_BASE_URL]) { + rawConfig.baseUrl = env[ENV_VARS.OPENAI_BASE_URL]; + } + + if (env[ENV_VARS.OPENAI_ORG_ID]) { + rawConfig.organization = env[ENV_VARS.OPENAI_ORG_ID]; + } + + if (env[ENV_VARS.EMBEDDING_TIMEOUT]) { + rawConfig.timeout = parseInt(env[ENV_VARS.EMBEDDING_TIMEOUT] ?? '30000', 10); + } + + if (env[ENV_VARS.EMBEDDING_MAX_RETRIES]) { + rawConfig.maxRetries = parseInt(env[ENV_VARS.EMBEDDING_MAX_RETRIES] ?? '3', 10); + } + + return parseEmbeddingConfig(rawConfig); + } catch (error) { + // Configuration parsing failed + return null; + } +} + +/** + * Validate embedding configuration without throwing + * + * @param config - Raw configuration object + * @returns Validation result with success flag and data/errors + */ +export function validateEmbeddingConfig(config: unknown): { + success: boolean; + data?: EmbeddingConfig; + errors?: z.ZodError; +} { + try { + const data = parseEmbeddingConfig(config); + return { success: true, data }; + } catch (error) { + if (error instanceof z.ZodError) { + return { success: false, errors: error }; + } + throw error; + } +} diff --git a/src/core/brain/embedding/constants.ts b/src/core/brain/embedding/constants.ts new file mode 100644 index 000000000..119f9bdfd --- /dev/null +++ b/src/core/brain/embedding/constants.ts @@ -0,0 +1,177 @@ +/** + * Embedding System Constants + * + * Centralized constants for the embedding system including defaults, + * supported models, timeouts, and other configuration values. + * + * @module embedding/constants + */ + +/** + * Supported embedding provider types + */ +export const PROVIDER_TYPES = { + OPENAI: 'openai', +} as const; + +/** + * OpenAI embedding models with their specifications + */ +export const OPENAI_MODELS = { + /** Latest small embedding model (1536 dimensions) */ + TEXT_EMBEDDING_3_SMALL: 'text-embedding-3-small', + /** Latest large embedding model (3072 dimensions) */ + TEXT_EMBEDDING_3_LARGE: 'text-embedding-3-large', + /** Legacy Ada v2 model (1536 dimensions) */ + TEXT_EMBEDDING_ADA_002: 'text-embedding-ada-002', +} as const; + +/** + * Model dimension specifications + */ +export const MODEL_DIMENSIONS = { + [OPENAI_MODELS.TEXT_EMBEDDING_3_SMALL]: 1536, + [OPENAI_MODELS.TEXT_EMBEDDING_3_LARGE]: 3072, + [OPENAI_MODELS.TEXT_EMBEDDING_ADA_002]: 1536, +} as const; + +/** + * Maximum input limits for different models + */ +export const MODEL_INPUT_LIMITS = { + [OPENAI_MODELS.TEXT_EMBEDDING_3_SMALL]: 8191, // tokens + [OPENAI_MODELS.TEXT_EMBEDDING_3_LARGE]: 8191, // tokens + [OPENAI_MODELS.TEXT_EMBEDDING_ADA_002]: 8191, // tokens +} as const; + +/** + * Default configuration values + */ +export const DEFAULTS = { + /** Default OpenAI model */ + OPENAI_MODEL: OPENAI_MODELS.TEXT_EMBEDDING_3_SMALL, + + /** Default request timeout in milliseconds */ + TIMEOUT: 30000, // 30 seconds + + /** Default maximum retry attempts */ + MAX_RETRIES: 3, + + /** Default batch size for batch operations */ + BATCH_SIZE: 100, + + /** Default OpenAI API base URL */ + OPENAI_BASE_URL: 'https://api.openai.com/v1', + + /** Default embedding dimension */ + DIMENSION: MODEL_DIMENSIONS[OPENAI_MODELS.TEXT_EMBEDDING_3_SMALL], +} as const; + +/** + * Rate limiting and retry configuration + */ +export const RETRY_CONFIG = { + /** Initial retry delay in milliseconds */ + INITIAL_DELAY: 1000, + + /** Maximum retry delay in milliseconds */ + MAX_DELAY: 60000, + + /** Backoff multiplier for exponential backoff */ + BACKOFF_MULTIPLIER: 2, + + /** Jitter factor for randomizing retry delays */ + JITTER_FACTOR: 0.1, +} as const; + +/** + * Validation limits + */ +export const VALIDATION_LIMITS = { + /** Maximum text length for single embedding */ + MAX_TEXT_LENGTH: 32768, // characters + + /** Maximum number of texts in batch operation */ + MAX_BATCH_SIZE: 2048, + + /** Minimum text length (empty strings not allowed) */ + MIN_TEXT_LENGTH: 1, +} as const; + +/** + * Error messages + */ +export const ERROR_MESSAGES = { + PROVIDER_NOT_SUPPORTED: (provider: string) => `Embedding provider '${provider}' is not supported`, + + MODEL_NOT_SUPPORTED: (model: string, provider: string) => + `Model '${model}' is not supported by provider '${provider}'`, + + API_KEY_REQUIRED: (provider: string) => `API key is required for provider '${provider}'`, + + CONNECTION_FAILED: (provider: string) => `Failed to connect to ${provider} embedding service`, + + TEXT_TOO_LONG: (length: number, max: number) => + `Text length ${length} exceeds maximum of ${max} characters`, + + BATCH_TOO_LARGE: (size: number, max: number) => + `Batch size ${size} exceeds maximum of ${max} items`, + + EMPTY_TEXT: 'Text cannot be empty', + + DIMENSION_MISMATCH: (expected: number, actual: number) => + `Expected embedding dimension ${expected}, but got ${actual}`, + + RATE_LIMIT_EXCEEDED: 'Rate limit exceeded for embedding provider', + + QUOTA_EXCEEDED: 'API quota exceeded for embedding provider', + + INVALID_API_KEY: (provider: string) => `Invalid API key for provider '${provider}'`, + + REQUEST_TIMEOUT: (timeout: number) => `Request timed out after ${timeout}ms`, +} as const; + +/** + * Log prefixes for different operations + */ +export const LOG_PREFIXES = { + EMBEDDING: '[EMBEDDING]', + OPENAI: '[EMBEDDING:OPENAI]', + FACTORY: '[EMBEDDING:FACTORY]', + MANAGER: '[EMBEDDING:MANAGER]', + HEALTH: '[EMBEDDING:HEALTH]', + BATCH: '[EMBEDDING:BATCH]', +} as const; + +/** + * Environment variable names + */ +export const ENV_VARS = { + OPENAI_API_KEY: 'OPENAI_API_KEY', + OPENAI_ORG_ID: 'OPENAI_ORG_ID', + OPENAI_BASE_URL: 'OPENAI_BASE_URL', + EMBEDDING_MODEL: 'EMBEDDING_MODEL', + EMBEDDING_TIMEOUT: 'EMBEDDING_TIMEOUT', + EMBEDDING_MAX_RETRIES: 'EMBEDDING_MAX_RETRIES', +} as const; + +/** + * HTTP status codes for API responses + */ +export const HTTP_STATUS = { + OK: 200, + BAD_REQUEST: 400, + UNAUTHORIZED: 401, + FORBIDDEN: 403, + NOT_FOUND: 404, + TOO_MANY_REQUESTS: 429, + INTERNAL_SERVER_ERROR: 500, + SERVICE_UNAVAILABLE: 503, +} as const; + +/** + * Content types for API requests + */ +export const CONTENT_TYPES = { + JSON: 'application/json', +} as const; diff --git a/src/core/brain/embedding/factory.ts b/src/core/brain/embedding/factory.ts new file mode 100644 index 000000000..02aa0fb4a --- /dev/null +++ b/src/core/brain/embedding/factory.ts @@ -0,0 +1,340 @@ +/** + * Embedding Factory Module + * + * Factory functions for creating embedding instances with proper validation, + * error handling, and type safety. Supports multiple providers and + * configuration methods. + * + * @module embedding/factory + */ + +import { logger } from '../../logger/index.js'; +import { + parseEmbeddingConfigFromEnv, + validateEmbeddingConfig, + type OpenAIEmbeddingConfig as ZodOpenAIEmbeddingConfig, + type EmbeddingConfig as ZodEmbeddingConfig, +} from './config.js'; +import { + type Embedder, + type OpenAIEmbeddingConfig as InterfaceOpenAIEmbeddingConfig, + EmbeddingError, + EmbeddingValidationError, + OpenAIEmbedder, +} from './backend/index.js'; +import { PROVIDER_TYPES, ERROR_MESSAGES, LOG_PREFIXES, DEFAULTS } from './constants.js'; + +// Use Zod-inferred types for validation, but convert to interface types for backend +export type BackendConfig = ZodOpenAIEmbeddingConfig; + +/** + * Embedding factory interface + * + * Defines the contract for embedding factory implementations. + * Each provider should implement this interface. + */ +export interface EmbeddingFactory { + /** + * Create an embedder instance + * + * @param config - Provider-specific configuration + * @returns Promise resolving to embedder instance + */ + createEmbedder(config: BackendConfig): Promise; + + /** + * Validate configuration for this provider + * + * @param config - Configuration to validate + * @returns True if configuration is valid + */ + validateConfig(config: unknown): boolean; + + /** + * Get the provider type this factory supports + * + * @returns Provider type string + */ + getProviderType(): string; +} + +/** + * Convert Zod config to interface config for backend compatibility + */ +function convertToInterfaceConfig(config: BackendConfig): InterfaceOpenAIEmbeddingConfig { + return { + type: PROVIDER_TYPES.OPENAI, + apiKey: config.apiKey, + model: config.model, + baseUrl: config.baseUrl, + timeout: config.timeout, + maxRetries: config.maxRetries, + options: config.options, + organization: config.organization, + dimensions: config.dimensions, + } as InterfaceOpenAIEmbeddingConfig; +} + +/** + * OpenAI Embedding Factory + * + * Factory implementation for creating OpenAI embedding instances. + */ +class OpenAIEmbeddingFactory implements EmbeddingFactory { + async createEmbedder(config: BackendConfig): Promise { + logger.debug(`${LOG_PREFIXES.FACTORY} Creating OpenAI embedder`, { + model: config.model, + baseUrl: config.baseUrl, + hasOrganization: !!config.organization, + }); + + try { + // Convert Zod config to interface config for backend compatibility + const interfaceConfig = convertToInterfaceConfig(config); + const embedder = new OpenAIEmbedder(interfaceConfig); + + // Test the connection + const isHealthy = await embedder.isHealthy(); + if (!isHealthy) { + throw new EmbeddingError(ERROR_MESSAGES.CONNECTION_FAILED('OpenAI'), 'openai'); + } + + logger.info(`${LOG_PREFIXES.FACTORY} Successfully created OpenAI embedder`, { + model: config.model, + dimension: embedder.getDimension(), + }); + + return embedder; + } catch (error) { + logger.error(`${LOG_PREFIXES.FACTORY} Failed to create OpenAI embedder`, { + error: error instanceof Error ? error.message : String(error), + model: config.model, + }); + + if (error instanceof EmbeddingError) { + throw error; + } + + throw new EmbeddingError( + `Failed to create OpenAI embedder: ${error instanceof Error ? error.message : String(error)}`, + 'openai', + error instanceof Error ? error : undefined + ); + } + } + + validateConfig(config: unknown): boolean { + try { + const validationResult = validateEmbeddingConfig(config); + return validationResult.success && validationResult.data?.type === PROVIDER_TYPES.OPENAI; + } catch { + return false; + } + } + + getProviderType(): string { + return PROVIDER_TYPES.OPENAI; + } +} + +/** + * Registry of embedding factories + */ +const EMBEDDING_FACTORIES = new Map([ + [PROVIDER_TYPES.OPENAI, new OpenAIEmbeddingFactory()], +]); + +/** + * Main factory function for creating embedding instances + * + * @param config - Embedding configuration + * @returns Promise resolving to embedder instance + * @throws {EmbeddingValidationError} If configuration is invalid + * @throws {EmbeddingError} If embedder creation fails + * + * @example + * ```typescript + * const embedder = await createEmbedder({ + * type: 'openai', + * apiKey: process.env.OPENAI_API_KEY, + * model: 'text-embedding-3-small' + * }); + * ``` + */ +export async function createEmbedder(config: BackendConfig): Promise { + logger.debug(`${LOG_PREFIXES.FACTORY} Creating embedder`, { + type: config.type, + }); + + // Validate configuration + const validationResult = validateEmbeddingConfig(config); + if (!validationResult.success) { + const errorMessage = + validationResult.errors?.issues + .map(issue => `${issue.path.join('.')}: ${issue.message}`) + .join(', ') || 'Invalid configuration'; + + logger.error(`${LOG_PREFIXES.FACTORY} Configuration validation failed`, { + type: config.type, + errors: errorMessage, + }); + + throw new EmbeddingValidationError(`Configuration validation failed: ${errorMessage}`); + } + + // Get factory for provider type + const factory = EMBEDDING_FACTORIES.get(config.type); + if (!factory) { + logger.error(`${LOG_PREFIXES.FACTORY} Unsupported provider type`, { + type: config.type, + supportedTypes: Array.from(EMBEDDING_FACTORIES.keys()), + }); + + throw new EmbeddingValidationError(ERROR_MESSAGES.PROVIDER_NOT_SUPPORTED(config.type)); + } + + // Create embedder instance + return await factory.createEmbedder(config); +} + +/** + * Create OpenAI embedder with simplified configuration + * + * @param config - OpenAI-specific configuration + * @returns Promise resolving to OpenAI embedder instance + * + * @example + * ```typescript + * const embedder = await createOpenAIEmbedder({ + * apiKey: process.env.OPENAI_API_KEY, + * model: 'text-embedding-3-small' + * }); + * ``` + */ +export async function createOpenAIEmbedder( + config: Omit +): Promise { + const openaiConfig: ZodOpenAIEmbeddingConfig = { + type: PROVIDER_TYPES.OPENAI, + apiKey: config.apiKey || '', + model: config.model || DEFAULTS.OPENAI_MODEL, + baseUrl: config.baseUrl || DEFAULTS.OPENAI_BASE_URL, + timeout: config.timeout || DEFAULTS.TIMEOUT, + maxRetries: config.maxRetries || DEFAULTS.MAX_RETRIES, + ...(config.organization && { organization: config.organization }), + ...(config.dimensions && { dimensions: config.dimensions }), + ...(config.options && { options: config.options }), + }; + return createEmbedder(openaiConfig); +} + +/** + * Create embedder from environment variables + * + * @param env - Environment variables object (defaults to process.env) + * @returns Promise resolving to embedder instance or null if config unavailable + * + * @example + * ```typescript + * // Requires OPENAI_API_KEY environment variable + * const embedder = await createEmbedderFromEnv(); + * if (embedder) { + * const embedding = await embedder.embed('Hello world'); + * } + * ``` + */ +export async function createEmbedderFromEnv( + env: Record = process.env +): Promise { + logger.debug(`${LOG_PREFIXES.FACTORY} Creating embedder from environment variables`); + + const config = parseEmbeddingConfigFromEnv(env); + if (!config) { + logger.warn(`${LOG_PREFIXES.FACTORY} No valid embedding configuration found in environment`); + return null; + } + + logger.debug(`${LOG_PREFIXES.FACTORY} Found valid embedding configuration in environment`, { + type: config.type, + model: config.model, + }); + + return createEmbedder(config as ZodOpenAIEmbeddingConfig); +} + +/** + * Create a default embedder with minimal configuration + * + * Uses OpenAI provider with default settings. + * Requires OPENAI_API_KEY environment variable. + * + * @returns Promise resolving to default embedder instance + * @throws {EmbeddingValidationError} If API key is not available + * + * @example + * ```typescript + * // Requires OPENAI_API_KEY environment variable + * const embedder = await createDefaultEmbedder(); + * ``` + */ +export async function createDefaultEmbedder(): Promise { + logger.debug(`${LOG_PREFIXES.FACTORY} Creating default embedder`); + + const apiKey = process.env.OPENAI_API_KEY; + if (!apiKey) { + throw new EmbeddingValidationError(ERROR_MESSAGES.API_KEY_REQUIRED('OpenAI')); + } + + const openaiConfig: ZodOpenAIEmbeddingConfig = { + type: PROVIDER_TYPES.OPENAI, + apiKey, + model: DEFAULTS.OPENAI_MODEL, + baseUrl: DEFAULTS.OPENAI_BASE_URL, + timeout: DEFAULTS.TIMEOUT, + maxRetries: DEFAULTS.MAX_RETRIES, + }; + return createEmbedder(openaiConfig); +} + +/** + * Check if a factory exists for the given provider type + * + * @param providerType - Provider type to check + * @returns True if factory exists + */ +export function isEmbeddingFactory(providerType: string): boolean { + return EMBEDDING_FACTORIES.has(providerType); +} + +/** + * Get all supported provider types + * + * @returns Array of supported provider type strings + */ +export function getSupportedProviders(): string[] { + return Array.from(EMBEDDING_FACTORIES.keys()); +} + +/** + * Get factory instance for a specific provider type + * + * @param providerType - Provider type + * @returns Factory instance or undefined if not found + */ +export function getEmbeddingFactory(providerType: string): EmbeddingFactory | undefined { + return EMBEDDING_FACTORIES.get(providerType); +} + +/** + * Register a new embedding factory + * + * @param providerType - Provider type + * @param factory - Factory instance + */ +export function registerEmbeddingFactory(providerType: string, factory: EmbeddingFactory): void { + logger.debug(`${LOG_PREFIXES.FACTORY} Registering embedding factory`, { + providerType, + }); + + EMBEDDING_FACTORIES.set(providerType, factory); +} diff --git a/src/core/brain/embedding/index.ts b/src/core/brain/embedding/index.ts new file mode 100644 index 000000000..202959e9f --- /dev/null +++ b/src/core/brain/embedding/index.ts @@ -0,0 +1,113 @@ +/** + * Embedding Module + * + * High-performance text embedding system supporting multiple providers. + * Provides a unified API for generating embeddings with comprehensive + * error handling, retry logic, and lifecycle management. + * + * Features: + * - Multiple provider support (OpenAI, future: Anthropic, Cohere, etc.) + * - Batch operations for efficient processing + * - Type-safe configuration with runtime validation + * - Comprehensive error handling and retry logic + * - Health monitoring and statistics collection + * - Graceful fallback and connection management + * + * @module embedding + * + * @example + * ```typescript + * import { createEmbedder, EmbeddingManager } from './embedding'; + * + * // Create a single embedder + * const embedder = await createEmbedder({ + * type: 'openai', + * apiKey: process.env.OPENAI_API_KEY, + * model: 'text-embedding-3-small' + * }); + * + * // Generate embeddings + * const embedding = await embedder.embed('Hello world'); + * const embeddings = await embedder.embedBatch(['Hello', 'World']); + * + * // Use embedding manager for multiple embedders + * const manager = new EmbeddingManager(); + * const { embedder: managedEmbedder } = await manager.createEmbedder({ + * type: 'openai', + * apiKey: process.env.OPENAI_API_KEY + * }); + * + * // Health monitoring + * manager.startHealthChecks(); + * const healthResults = await manager.checkAllHealth(); + * + * // Cleanup + * await manager.disconnect(); + * ``` + */ + +// Export types +export type { + Embedder, + EmbeddingConfig, + OpenAIEmbeddingConfig, + BackendConfig, + EmbeddingResult, + BatchEmbeddingResult, + EmbeddingFactory, + HealthCheckResult, + EmbedderInfo, + EmbeddingStats, + EmbeddingEnvConfig, +} from './types.js'; + +// Export error classes +export { + EmbeddingError, + EmbeddingConnectionError, + EmbeddingDimensionError, + EmbeddingRateLimitError, + EmbeddingQuotaError, + EmbeddingValidationError, +} from './types.js'; + +// Export factory functions +export { + createEmbedder, + createOpenAIEmbedder, + createEmbedderFromEnv, + createDefaultEmbedder, + isEmbeddingFactory, + getSupportedProviders, + getEmbeddingFactory, + registerEmbeddingFactory, +} from './factory.js'; + +// Export manager +export { EmbeddingManager } from './manager.js'; + +// Export configuration utilities +export { + parseEmbeddingConfig, + parseEmbeddingConfigFromEnv, + validateEmbeddingConfig, + EmbeddingConfigSchema, +} from './config.js'; + +// Export constants for external use +export { + PROVIDER_TYPES, + OPENAI_MODELS, + MODEL_DIMENSIONS, + DEFAULTS, + VALIDATION_LIMITS, +} from './constants.js'; + +// Export utilities +export { + getEmbeddingConfigFromEnv, + isEmbeddingConfigAvailable, + getEmbeddingConfigSummary, + validateEmbeddingEnv, + analyzeProviderConfiguration, +} from './utils.js'; diff --git a/src/core/brain/embedding/manager.ts b/src/core/brain/embedding/manager.ts new file mode 100644 index 000000000..34185c4cd --- /dev/null +++ b/src/core/brain/embedding/manager.ts @@ -0,0 +1,504 @@ +/** + * Embedding Manager Module + * + * Provides lifecycle management, health monitoring, and connection management + * for embedding services. Manages multiple embedder instances and provides + * centralized monitoring and cleanup capabilities. + * + * @module embedding/manager + */ + +import { logger } from '../../logger/index.js'; +import { + type Embedder, + type BackendConfig, + EmbeddingError, + EmbeddingConnectionError, +} from './backend/index.js'; +import { createEmbedder, createEmbedderFromEnv } from './factory.js'; +import { LOG_PREFIXES, ERROR_MESSAGES } from './constants.js'; + +/** + * Health check result for an embedder instance + */ +export interface HealthCheckResult { + /** Whether the embedder is healthy */ + healthy: boolean; + /** Provider type */ + provider: string; + /** Model being used */ + model: string; + /** Embedding dimension */ + dimension?: number; + /** Response time in milliseconds */ + responseTime?: number; + /** Error message if unhealthy */ + error?: string; + /** Timestamp of the health check */ + timestamp: Date; +} + +/** + * Information about an embedder instance + */ +export interface EmbedderInfo { + /** Unique identifier for this embedder */ + id: string; + /** Provider type */ + provider: string; + /** Model being used */ + model: string; + /** Embedding dimension */ + dimension: number; + /** Configuration used */ + config: BackendConfig; + /** Creation timestamp */ + createdAt: Date; + /** Last health check result */ + lastHealthCheck?: HealthCheckResult; +} + +/** + * Statistics about embedding operations + */ +export interface EmbeddingStats { + /** Total number of single embed operations */ + totalEmbeds: number; + /** Total number of batch embed operations */ + totalBatchEmbeds: number; + /** Total number of texts processed */ + totalTexts: number; + /** Total processing time in milliseconds */ + totalProcessingTime: number; + /** Number of successful operations */ + successfulOperations: number; + /** Number of failed operations */ + failedOperations: number; + /** Average processing time per operation */ + averageProcessingTime: number; +} + +/** + * Embedding Manager + * + * Manages the lifecycle of embedding instances, providing centralized + * health monitoring, statistics collection, and resource cleanup. + */ +export class EmbeddingManager { + private embedders = new Map(); + private embedderInfo = new Map(); + private stats: EmbeddingStats = { + totalEmbeds: 0, + totalBatchEmbeds: 0, + totalTexts: 0, + totalProcessingTime: 0, + successfulOperations: 0, + failedOperations: 0, + averageProcessingTime: 0, + }; + private healthCheckInterval: NodeJS.Timeout | undefined; + + constructor() { + logger.debug(`${LOG_PREFIXES.MANAGER} Embedding manager initialized`); + } + + /** + * Create and register an embedder instance + * + * @param config - Embedding configuration + * @param id - Optional custom ID for the embedder + * @returns Promise resolving to embedder instance and info + */ + async createEmbedder( + config: BackendConfig, + id?: string + ): Promise<{ embedder: Embedder; info: EmbedderInfo }> { + const embedderId = id || this.generateId(); + + logger.debug(`${LOG_PREFIXES.MANAGER} Creating embedder`, { + id: embedderId, + type: config.type, + }); + + try { + const configWithApiKey = { ...config, apiKey: config.apiKey || '' }; + const embedder = await createEmbedder(configWithApiKey as any); + + const info: EmbedderInfo = { + id: embedderId, + provider: config.type, + model: config.model || 'unknown', + dimension: embedder.getDimension(), + config, + createdAt: new Date(), + }; + + this.embedders.set(embedderId, embedder); + this.embedderInfo.set(embedderId, info); + + logger.info(`${LOG_PREFIXES.MANAGER} Successfully created embedder`, { + id: embedderId, + provider: info.provider, + model: info.model, + dimension: info.dimension, + }); + + return { embedder, info }; + } catch (error) { + logger.error(`${LOG_PREFIXES.MANAGER} Failed to create embedder`, { + id: embedderId, + type: config.type, + error: error instanceof Error ? error.message : String(error), + }); + + throw error; + } + } + + /** + * Create embedder from environment variables + * + * @param id - Optional custom ID for the embedder + * @returns Promise resolving to embedder instance and info, or null + */ + async createEmbedderFromEnv( + id?: string + ): Promise<{ embedder: Embedder; info: EmbedderInfo } | null> { + logger.debug(`${LOG_PREFIXES.MANAGER} Creating embedder from environment`); + + const embedder = await createEmbedderFromEnv(); + if (!embedder) { + logger.warn(`${LOG_PREFIXES.MANAGER} No embedder configuration found in environment`); + return null; + } + + const embedderId = id || this.generateId(); + const config = embedder.getConfig() as BackendConfig; + + const info: EmbedderInfo = { + id: embedderId, + provider: config.type, + model: config.model || 'unknown', + dimension: embedder.getDimension(), + config, + createdAt: new Date(), + }; + + this.embedders.set(embedderId, embedder); + this.embedderInfo.set(embedderId, info); + + logger.info(`${LOG_PREFIXES.MANAGER} Successfully created embedder from environment`, { + id: embedderId, + provider: info.provider, + model: info.model, + }); + + return { embedder, info }; + } + + /** + * Get embedder instance by ID + * + * @param id - Embedder ID + * @returns Embedder instance or undefined + */ + getEmbedder(id: string): Embedder | undefined { + return this.embedders.get(id); + } + + /** + * Get embedder information by ID + * + * @param id - Embedder ID + * @returns Embedder information or undefined + */ + getEmbedderInfo(id: string): EmbedderInfo | undefined { + return this.embedderInfo.get(id); + } + + /** + * Get all registered embedders + * + * @returns Map of embedder ID to embedder instance + */ + getAllEmbedders(): Map { + return new Map(this.embedders); + } + + /** + * Get all embedder information + * + * @returns Map of embedder ID to embedder information + */ + getAllEmbedderInfo(): Map { + return new Map(this.embedderInfo); + } + + /** + * Remove and disconnect an embedder + * + * @param id - Embedder ID + * @returns Promise resolving to true if removed, false if not found + */ + async removeEmbedder(id: string): Promise { + const embedder = this.embedders.get(id); + if (!embedder) { + logger.warn(`${LOG_PREFIXES.MANAGER} Embedder not found for removal`, { id }); + return false; + } + + logger.debug(`${LOG_PREFIXES.MANAGER} Removing embedder`, { id }); + + try { + await embedder.disconnect(); + this.embedders.delete(id); + this.embedderInfo.delete(id); + + logger.info(`${LOG_PREFIXES.MANAGER} Successfully removed embedder`, { id }); + return true; + } catch (error) { + logger.error(`${LOG_PREFIXES.MANAGER} Error disconnecting embedder`, { + id, + error: error instanceof Error ? error.message : String(error), + }); + + // Remove anyway to prevent memory leaks + this.embedders.delete(id); + this.embedderInfo.delete(id); + return true; + } + } + + /** + * Perform health check on a specific embedder + * + * @param id - Embedder ID + * @returns Promise resolving to health check result + */ + async checkHealth(id: string): Promise { + const embedder = this.embedders.get(id); + const info = this.embedderInfo.get(id); + + if (!embedder || !info) { + logger.warn(`${LOG_PREFIXES.HEALTH} Embedder not found for health check`, { id }); + return null; + } + + logger.silly(`${LOG_PREFIXES.HEALTH} Performing health check`, { id }); + + const startTime = Date.now(); + try { + const healthy = await embedder.isHealthy(); + const responseTime = Date.now() - startTime; + + const result: HealthCheckResult = { + healthy, + provider: info.provider, + model: info.model || 'unknown', + dimension: info.dimension, + responseTime, + timestamp: new Date(), + }; + + // Update embedder info with latest health check + this.embedderInfo.set(id, { + ...info, + lastHealthCheck: result, + }); + + logger.debug(`${LOG_PREFIXES.HEALTH} Health check completed`, { + id, + healthy, + responseTime, + }); + + return result; + } catch (error) { + const responseTime = Date.now() - startTime; + const result: HealthCheckResult = { + healthy: false, + provider: info.provider, + model: info.model || 'unknown', + dimension: info.dimension, + responseTime, + error: error instanceof Error ? error.message : String(error), + timestamp: new Date(), + }; + + // Update embedder info with latest health check + this.embedderInfo.set(id, { + ...info, + lastHealthCheck: result, + }); + + logger.warn(`${LOG_PREFIXES.HEALTH} Health check failed`, { + id, + error: result.error, + responseTime, + }); + + return result; + } + } + + /** + * Perform health check on all embedders + * + * @returns Promise resolving to map of embedder ID to health check result + */ + async checkAllHealth(): Promise> { + logger.debug(`${LOG_PREFIXES.HEALTH} Performing health check on all embedders`); + + const results = new Map(); + const healthCheckPromises = Array.from(this.embedders.keys()).map(async id => { + const result = await this.checkHealth(id); + if (result) { + results.set(id, result); + } + }); + + await Promise.all(healthCheckPromises); + + logger.debug(`${LOG_PREFIXES.HEALTH} Completed health check on all embedders`, { + total: this.embedders.size, + checked: results.size, + }); + + return results; + } + + /** + * Start periodic health checks + * + * @param intervalMs - Health check interval in milliseconds (default: 5 minutes) + */ + startHealthChecks(intervalMs: number = 5 * 60 * 1000): void { + if (this.healthCheckInterval) { + logger.warn(`${LOG_PREFIXES.HEALTH} Health checks already running`); + return; + } + + logger.info(`${LOG_PREFIXES.HEALTH} Starting periodic health checks`, { + intervalMs, + }); + + this.healthCheckInterval = setInterval(async () => { + try { + await this.checkAllHealth(); + } catch (error) { + logger.error(`${LOG_PREFIXES.HEALTH} Error during periodic health check`, { + error: error instanceof Error ? error.message : String(error), + }); + } + }, intervalMs); + } + + /** + * Stop periodic health checks + */ + stopHealthChecks(): void { + if (this.healthCheckInterval) { + clearInterval(this.healthCheckInterval); + this.healthCheckInterval = undefined; + logger.info(`${LOG_PREFIXES.HEALTH} Stopped periodic health checks`); + } + } + + /** + * Get current statistics + * + * @returns Current embedding statistics + */ + getStats(): EmbeddingStats { + // Calculate average processing time + const avgTime = + this.stats.totalProcessingTime > 0 && this.stats.successfulOperations > 0 + ? this.stats.totalProcessingTime / this.stats.successfulOperations + : 0; + + return { + ...this.stats, + averageProcessingTime: avgTime, + }; + } + + /** + * Reset statistics + */ + resetStats(): void { + logger.debug(`${LOG_PREFIXES.MANAGER} Resetting embedding statistics`); + + this.stats = { + totalEmbeds: 0, + totalBatchEmbeds: 0, + totalTexts: 0, + totalProcessingTime: 0, + successfulOperations: 0, + failedOperations: 0, + averageProcessingTime: 0, + }; + } + + /** + * Update statistics (called internally after operations) + */ + private updateStats( + type: 'embed' | 'batch', + textCount: number, + processingTime: number, + success: boolean + ): void { + if (type === 'embed') { + this.stats.totalEmbeds++; + } else { + this.stats.totalBatchEmbeds++; + } + + this.stats.totalTexts += textCount; + + if (success) { + this.stats.successfulOperations++; + this.stats.totalProcessingTime += processingTime; + } else { + this.stats.failedOperations++; + } + } + + /** + * Disconnect all embedders and cleanup + */ + async disconnect(): Promise { + logger.info(`${LOG_PREFIXES.MANAGER} Disconnecting all embedders`); + + // Stop health checks + this.stopHealthChecks(); + + // Disconnect all embedders + const disconnectPromises = Array.from(this.embedders.entries()).map(async ([id, embedder]) => { + try { + await embedder.disconnect(); + logger.debug(`${LOG_PREFIXES.MANAGER} Disconnected embedder`, { id }); + } catch (error) { + logger.warn(`${LOG_PREFIXES.MANAGER} Error disconnecting embedder`, { + id, + error: error instanceof Error ? error.message : String(error), + }); + } + }); + + await Promise.all(disconnectPromises); + + // Clear all maps + this.embedders.clear(); + this.embedderInfo.clear(); + + logger.info(`${LOG_PREFIXES.MANAGER} Successfully disconnected all embedders`); + } + + /** + * Generate a unique ID for embedders + */ + private generateId(): string { + return `embedder_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; + } +} diff --git a/src/core/brain/embedding/types.ts b/src/core/brain/embedding/types.ts new file mode 100644 index 000000000..59c508440 --- /dev/null +++ b/src/core/brain/embedding/types.ts @@ -0,0 +1,92 @@ +/** + * Embedding Module Public API Types + * + * This module re-exports all the necessary types and interfaces for the embedding system. + * It provides a simplified, clean API surface for consumers of the embedding module. + * + * The embedding system architecture: + * - Multiple provider support (OpenAI, future: Anthropic, Cohere, etc.) + * - Consistent API across different providers + * - Strong type safety with TypeScript and runtime validation with Zod + * - Factory pattern for creating embedders + * - Manager pattern for lifecycle management + * + * @module embedding/types + * + * @example + * ```typescript + * import type { Embedder, EmbeddingConfig } from './embedding/types.js'; + * + * // Configure embedder + * const config: EmbeddingConfig = { + * type: 'openai', + * apiKey: process.env.OPENAI_API_KEY, + * model: 'text-embedding-3-small' + * }; + * + * // Use embedder + * const embedder = await createEmbedder(config); + * const embedding = await embedder.embed('Hello world'); + * ``` + */ + +/** + * Re-export core embedding types + * + * These exports provide the complete type system needed to work with + * the embedding module without exposing internal implementation details. + */ +export type { + // Core interfaces + Embedder, // Interface for embedding providers + EmbeddingConfig, // Base configuration interface + OpenAIEmbeddingConfig, // OpenAI-specific configuration + BackendConfig, // Union type for all provider configurations + + // Result types + EmbeddingResult, // Single embedding result with metadata + BatchEmbeddingResult, // Batch embedding result with metadata +} from './backend/types.js'; + +/** + * Re-export error classes + * + * Comprehensive error hierarchy for embedding operations. + */ +export { + EmbeddingError, // Base error class + EmbeddingConnectionError, // Connection-related errors + EmbeddingDimensionError, // Dimension mismatch errors + EmbeddingRateLimitError, // Rate limiting errors + EmbeddingQuotaError, // Quota exceeded errors + EmbeddingValidationError, // Input validation errors +} from './backend/types.js'; + +/** + * Re-export factory types + * + * Types related to embedding factory functionality. + */ +export type { + EmbeddingFactory, // Factory interface for creating embedders +} from './factory.js'; + +/** + * Re-export manager types + * + * Types related to embedding lifecycle management. + */ +export type { + HealthCheckResult, // Health check result structure + EmbedderInfo, // Information about embedder instances + EmbeddingStats, // Statistics about embedding operations +} from './manager.js'; + +/** + * Re-export configuration types and utilities + * + * Configuration validation and parsing utilities. + */ +export type { + EmbeddingEnvConfig, // Environment-based configuration +} from './config.js'; diff --git a/src/core/brain/embedding/utils.ts b/src/core/brain/embedding/utils.ts new file mode 100644 index 000000000..dd5057427 --- /dev/null +++ b/src/core/brain/embedding/utils.ts @@ -0,0 +1,240 @@ +/** + * Embedding Utilities + * + * Utility functions for working with embedding configuration and + * environment variables. Provides convenient access to environment-based + * configuration for the embedding system. + * + * @module embedding/utils + */ + +import { env } from '../../env.js'; +import { type OpenAIEmbeddingConfig } from './backend/types.js'; +import { PROVIDER_TYPES, DEFAULTS } from './constants.js'; +import { parseEmbeddingConfigFromEnv } from './config.js'; + +/** + * Get embedding configuration from environment variables + * + * Uses the centralized env configuration to build embedding config. + * Falls back to sensible defaults when environment variables are not set. + * + * @returns OpenAI embedding configuration or null if no API key + * + * @example + * ```typescript + * const config = getEmbeddingConfigFromEnv(); + * if (config) { + * const embedder = await createEmbedder(config); + * } + * ``` + */ +export function getEmbeddingConfigFromEnv(): OpenAIEmbeddingConfig | null { + // Check if we have a required API key + if (!env.OPENAI_API_KEY) { + return null; + } + + const config: OpenAIEmbeddingConfig = { + type: 'openai', + apiKey: env.OPENAI_API_KEY!, + model: (env.EMBEDDING_MODEL as any) || DEFAULTS.OPENAI_MODEL, + baseUrl: env.OPENAI_BASE_URL || DEFAULTS.OPENAI_BASE_URL, + timeout: env.EMBEDDING_TIMEOUT || DEFAULTS.TIMEOUT, + maxRetries: env.EMBEDDING_MAX_RETRIES || DEFAULTS.MAX_RETRIES, + organization: env.OPENAI_ORG_ID || '', + }; + + return config; +} + +/** + * Check if embedding configuration is available in environment + * + * @returns True if embedding can be configured from environment variables + * + * @example + * ```typescript + * if (isEmbeddingConfigAvailable()) { + * const embedder = await createEmbedderFromEnv(); + * } else { + * console.log('Please set OPENAI_API_KEY environment variable'); + * } + * ``` + */ +export function isEmbeddingConfigAvailable(): boolean { + return !!env.OPENAI_API_KEY; +} + +/** + * Get embedding configuration summary for logging/debugging + * + * Returns safe configuration info without exposing sensitive data. + * + * @returns Configuration summary object + * + * @example + * ```typescript + * const summary = getEmbeddingConfigSummary(); + * console.log('Embedding config:', summary); + * // Output: { hasApiKey: true, model: 'text-embedding-3-small', ... } + * ``` + */ +export function getEmbeddingConfigSummary(): { + hasApiKey: boolean; + model?: string; + baseUrl?: string; + timeout?: number; + maxRetries?: number; + hasOrganization: boolean; +} { + return { + hasApiKey: !!env.OPENAI_API_KEY, + model: env.EMBEDDING_MODEL || DEFAULTS.OPENAI_MODEL, + baseUrl: env.OPENAI_BASE_URL || DEFAULTS.OPENAI_BASE_URL, + timeout: env.EMBEDDING_TIMEOUT || DEFAULTS.TIMEOUT, + maxRetries: env.EMBEDDING_MAX_RETRIES || DEFAULTS.MAX_RETRIES, + hasOrganization: !!env.OPENAI_ORG_ID, + }; +} + +/** + * Validate that required environment variables are set for embeddings + * + * @returns Validation result with details + * + * @example + * ```typescript + * const validation = validateEmbeddingEnv(); + * if (!validation.valid) { + * console.error('Embedding setup issues:', validation.issues); + * } + * ``` + */ +export function validateEmbeddingEnv(): { + valid: boolean; + issues: string[]; +} { + const issues: string[] = []; + + // Check required variables + if (!env.OPENAI_API_KEY) { + // Check if other LLM providers are configured + const hasOtherProviders = env.ANTHROPIC_API_KEY || env.OPENROUTER_API_KEY; + if (hasOtherProviders) { + issues.push( + 'OPENAI_API_KEY is required for embedding functionality, even when using Anthropic or OpenRouter for LLM services' + ); + } else { + issues.push('OPENAI_API_KEY is required for embedding functionality'); + } + } + + // Check API key format + if (env.OPENAI_API_KEY && !env.OPENAI_API_KEY.startsWith('sk-')) { + issues.push('OPENAI_API_KEY should start with "sk-"'); + } + + // Check numeric values + if (env.EMBEDDING_TIMEOUT && env.EMBEDDING_TIMEOUT <= 0) { + issues.push('EMBEDDING_TIMEOUT must be a positive number'); + } + + if (env.EMBEDDING_MAX_RETRIES && env.EMBEDDING_MAX_RETRIES < 0) { + issues.push('EMBEDDING_MAX_RETRIES must be a non-negative number'); + } + + // Check URL format + if (env.OPENAI_BASE_URL) { + try { + new URL(env.OPENAI_BASE_URL); + } catch { + issues.push('OPENAI_BASE_URL must be a valid URL'); + } + } + + return { + valid: issues.length === 0, + issues, + }; +} + +/** + * Check for mixed provider configuration and provide helpful guidance + * + * @returns Configuration analysis with recommendations + * + * @example + * ```typescript + * const analysis = analyzeProviderConfiguration(); + * if (analysis.warnings.length > 0) { + * console.warn('Configuration warnings:', analysis.warnings); + * } + * ``` + */ +export function analyzeProviderConfiguration(): { + usingMixedProviders: boolean; + llmProvider: 'openai' | 'anthropic' | 'openrouter' | 'none' | 'multiple'; + embeddingProvider: 'openai' | 'none'; + warnings: string[]; + recommendations: string[]; +} { + const warnings: string[] = []; + const recommendations: string[] = []; + + // Detect which LLM providers are configured + const hasOpenAI = !!env.OPENAI_API_KEY; + const hasAnthropic = !!env.ANTHROPIC_API_KEY; + const hasOpenRouter = !!env.OPENROUTER_API_KEY; + + let llmProvider: 'openai' | 'anthropic' | 'openrouter' | 'none' | 'multiple'; + const configuredProviders = [ + hasOpenAI && 'openai', + hasAnthropic && 'anthropic', + hasOpenRouter && 'openrouter', + ].filter(Boolean); + + if (configuredProviders.length === 0) { + llmProvider = 'none'; + warnings.push('No LLM provider API keys configured'); + } else if (configuredProviders.length === 1) { + llmProvider = configuredProviders[0] as 'openai' | 'anthropic' | 'openrouter'; + } else { + llmProvider = 'multiple'; + recommendations.push( + 'Multiple LLM provider API keys detected. The system will use the configured provider in your LLM service setup.' + ); + } + + // Embedding provider analysis + const embeddingProvider = hasOpenAI ? 'openai' : 'none'; + const usingMixedProviders = (hasAnthropic || hasOpenRouter) && hasOpenAI; + + // Generate specific warnings and recommendations + if (!hasOpenAI && (hasAnthropic || hasOpenRouter)) { + warnings.push('Embedding functionality will not work without OPENAI_API_KEY'); + recommendations.push( + 'Add OPENAI_API_KEY to enable embedding features, even when using Anthropic or OpenRouter for LLM' + ); + } + + if (usingMixedProviders) { + recommendations.push( + 'You are using a mixed provider setup (non-OpenAI for LLM + OpenAI for embeddings). This is a valid configuration.' + ); + } + + if (hasOpenAI && !hasAnthropic && !hasOpenRouter) { + recommendations.push( + 'Using OpenAI for both LLM and embeddings - this is the simplest configuration.' + ); + } + + return { + usingMixedProviders, + llmProvider, + embeddingProvider, + warnings, + recommendations, + }; +} diff --git a/src/core/brain/tools/__test__/definitions.test.ts b/src/core/brain/tools/__test__/definitions.test.ts index 5bb1b8dd7..4c77b1231 100644 --- a/src/core/brain/tools/__test__/definitions.test.ts +++ b/src/core/brain/tools/__test__/definitions.test.ts @@ -6,9 +6,7 @@ import { getToolsByCategory, TOOL_CATEGORIES, } from '../definitions/index.js'; -import { - extractKnowledgeTool, -} from '../definitions/memory/index.js'; +import { extractKnowledgeTool } from '../definitions/memory/index.js'; import { InternalToolManager } from '../manager.js'; import { InternalToolRegistry } from '../registry.js'; @@ -79,17 +77,19 @@ describe('Tool Definitions', () => { it('should load all tool definitions', async () => { const tools = await getAllToolDefinitions(); - expect(Object.keys(tools)).toHaveLength(1); // 1 memory tool + expect(Object.keys(tools)).toHaveLength(2); // 2 memory tools expect(tools['extract_knowledge']).toBeDefined(); + expect(tools['memory_operation']).toBeDefined(); }); it('should register all tools successfully', async () => { const result = await registerAllTools(manager); - expect(result.total).toBe(1); - expect(result.registered.length).toBe(1); + expect(result.total).toBe(2); + expect(result.registered.length).toBe(2); expect(result.failed.length).toBe(0); expect(result.registered).toContain('extract_knowledge'); + expect(result.registered).toContain('memory_operation'); }); it('should handle registration failures gracefully', async () => { @@ -103,9 +103,9 @@ describe('Tool Definitions', () => { const result = await registerAllTools(failingManager); - expect(result.total).toBe(1); + expect(result.total).toBe(2); expect(result.registered.length).toBe(0); - expect(result.failed.length).toBe(1); + expect(result.failed.length).toBe(2); expect(result.failed?.[0]?.error).toBe('Simulated failure'); }); }); @@ -114,7 +114,7 @@ describe('Tool Definitions', () => { it('should have correct category structure', () => { expect(TOOL_CATEGORIES.memory).toBeDefined(); - expect(TOOL_CATEGORIES.memory.tools).toHaveLength(1); + expect(TOOL_CATEGORIES.memory.tools).toHaveLength(2); }); it('should get tool info by name', () => { @@ -130,8 +130,9 @@ describe('Tool Definitions', () => { it('should get tools by category', () => { const memoryTools = getToolsByCategory('memory'); - expect(memoryTools).toHaveLength(1); + expect(memoryTools).toHaveLength(2); expect(memoryTools).toContain('cipher_extract_knowledge'); + expect(memoryTools).toContain('cipher_memory_operation'); }); }); diff --git a/src/core/brain/tools/__test__/unified-tool-manager.test.ts b/src/core/brain/tools/__test__/unified-tool-manager.test.ts index bbfea3907..028c75aae 100644 --- a/src/core/brain/tools/__test__/unified-tool-manager.test.ts +++ b/src/core/brain/tools/__test__/unified-tool-manager.test.ts @@ -79,9 +79,10 @@ describe('UnifiedToolManager', () => { it('should load internal tools when enabled', async () => { const tools = await unifiedManager.getAllTools(); - // Should have 1 memory tool - expect(Object.keys(tools)).toHaveLength(1); + // Should have 2 memory tools + expect(Object.keys(tools)).toHaveLength(2); expect(tools['cipher_extract_knowledge']).toBeDefined(); + expect(tools['cipher_memory_operation']).toBeDefined(); // All tools should be marked as internal for (const tool of Object.values(tools)) { @@ -159,7 +160,7 @@ describe('UnifiedToolManager', () => { const formattedTools = await unifiedManager.getToolsForProvider('openai'); expect(Array.isArray(formattedTools)).toBe(true); - expect(formattedTools.length).toBe(1); + expect(formattedTools.length).toBe(2); // Check OpenAI format const tool = formattedTools[0]; @@ -174,7 +175,7 @@ describe('UnifiedToolManager', () => { const formattedTools = await unifiedManager.getToolsForProvider('anthropic'); expect(Array.isArray(formattedTools)).toBe(true); - expect(formattedTools.length).toBe(1); + expect(formattedTools.length).toBe(2); // Check Anthropic format const tool = formattedTools[0]; @@ -187,7 +188,7 @@ describe('UnifiedToolManager', () => { const formattedTools = await unifiedManager.getToolsForProvider('openrouter'); expect(Array.isArray(formattedTools)).toBe(true); - expect(formattedTools.length).toBe(1); + expect(formattedTools.length).toBe(2); // OpenRouter uses OpenAI format const tool = formattedTools[0]; @@ -211,8 +212,8 @@ describe('UnifiedToolManager', () => { expect(stats.config).toBeDefined(); // Internal tools stats should be available - expect(stats.internalTools.totalTools).toBe(1); - expect(stats.internalTools.toolsByCategory.memory).toBe(1); + expect(stats.internalTools.totalTools).toBe(2); + expect(stats.internalTools.toolsByCategory.memory).toBe(2); }); it('should handle disabled tool managers in stats', () => { diff --git a/src/core/brain/tools/definitions/index.ts b/src/core/brain/tools/definitions/index.ts index 50a16c300..d6cbee00e 100644 --- a/src/core/brain/tools/definitions/index.ts +++ b/src/core/brain/tools/definitions/index.ts @@ -10,6 +10,7 @@ export * from './memory/index.js'; // Export individual tools for direct access export { extractKnowledgeTool } from './memory/extract-knowledge.js'; +export { memoryOperationTool } from './memory/memory_operation.js'; // Import types and utilities import type { InternalToolSet } from '../types.js'; @@ -104,7 +105,7 @@ export async function registerAllTools(toolManager: any): Promise<{ export const TOOL_CATEGORIES = { memory: { description: 'Tools for managing facts, memories, and knowledge storage', - tools: ['extract_knowledge'] as string[], + tools: ['extract_knowledge', 'memory_operation'] as string[], useCase: 'Use these tools to capture, search, and store important information for future reference', }, diff --git a/src/core/brain/tools/definitions/memory/extract-knowledge.ts b/src/core/brain/tools/definitions/memory/extract-knowledge.ts index d7bde892f..0ab4cbe2d 100644 --- a/src/core/brain/tools/definitions/memory/extract-knowledge.ts +++ b/src/core/brain/tools/definitions/memory/extract-knowledge.ts @@ -62,16 +62,44 @@ export const extractKnowledgeTool: InternalTool = { handler: async (args: { knowledge: string[] }) => { try { logger.info('ExtractFact: Processing fact extraction request', { - factCount: args.knowledge.length, + factCount: Array.isArray(args.knowledge) ? args.knowledge.length : 1, + inputType: typeof args.knowledge, }); - // Validate input - if (!args.knowledge || args.knowledge.length === 0) { + // Validate input and handle both string and array inputs + let knowledgeArray: string[]; + + if (!args.knowledge) { + throw new Error('No facts provided for extraction'); + } + + // Handle case where LLM passes knowledge as a JSON string instead of array + if (typeof args.knowledge === 'string') { + try { + // Try to parse as JSON array first + const parsed = JSON.parse(args.knowledge); + if (Array.isArray(parsed)) { + knowledgeArray = parsed; + } else { + // Treat as single string fact + knowledgeArray = [args.knowledge]; + } + } catch { + // Not valid JSON, treat as single string fact + knowledgeArray = [args.knowledge]; + } + } else if (Array.isArray(args.knowledge)) { + knowledgeArray = args.knowledge; + } else { + throw new Error('Knowledge must be a string or array of strings'); + } + + if (knowledgeArray.length === 0) { throw new Error('No facts provided for extraction'); } // Filter out empty or invalid facts - const validFacts = args.knowledge + const validFacts = knowledgeArray .filter(fact => fact && typeof fact === 'string' && fact.trim().length > 0) .map(fact => fact.trim()); @@ -107,7 +135,7 @@ export const extractKnowledgeTool: InternalTool = { const result = { success: true, extracted: processedFacts.length, - skipped: args.knowledge.length - validFacts.length, + skipped: knowledgeArray.length - validFacts.length, timestamp: new Date().toISOString(), facts: processedFacts.map(f => ({ id: f.metadata.id, @@ -130,14 +158,15 @@ export const extractKnowledgeTool: InternalTool = { const errorMessage = error instanceof Error ? error.message : String(error); logger.error('ExtractFact: Failed to extract facts', { error: errorMessage, - factCount: args.knowledge?.length || 0, + factCount: Array.isArray(args.knowledge) ? args.knowledge.length : 1, + inputType: typeof args.knowledge, }); return { success: false, error: errorMessage, extracted: 0, - skipped: args.knowledge?.length || 0, + skipped: Array.isArray(args.knowledge) ? args.knowledge.length : 1, timestamp: new Date().toISOString(), }; } diff --git a/src/core/brain/tools/definitions/memory/index.ts b/src/core/brain/tools/definitions/memory/index.ts index d37f4338d..6c66b5f2a 100644 --- a/src/core/brain/tools/definitions/memory/index.ts +++ b/src/core/brain/tools/definitions/memory/index.ts @@ -2,11 +2,12 @@ * Memory Tools Module * * This module exports all memory-related internal tools for the Cipher agent. - * These tools handle fact extraction and knowledge processing. + * These tools handle fact extraction, knowledge processing, and memory operations. */ // Export all memory tools export { extractKnowledgeTool } from './extract-knowledge.js'; +export { memoryOperationTool } from './memory_operation.js'; // Export types for better developer experience import type { InternalTool } from '../../types.js'; @@ -20,15 +21,18 @@ export const memoryTools: InternalTool[] = [ // Load tools dynamically to avoid potential circular dependencies import('./extract-knowledge.js').then(module => memoryTools.push(module.extractKnowledgeTool)); +import('./memory_operation.js').then(module => memoryTools.push(module.memoryOperationTool)); /** * Get all memory tools as a map */ export async function getMemoryTools(): Promise> { const { extractKnowledgeTool } = await import('./extract-knowledge.js'); + const { memoryOperationTool } = await import('./memory_operation.js'); return { [extractKnowledgeTool.name]: extractKnowledgeTool, + [memoryOperationTool.name]: memoryOperationTool, }; } @@ -42,4 +46,10 @@ export const MEMORY_TOOL_INFO = { useCase: 'Use when you need to capture important technical information, code patterns, or implementation details for future reference', }, + memory_operation: { + category: 'memory', + purpose: 'Process extracted knowledge and determine memory operations (ADD, UPDATE, DELETE)', + useCase: + 'Use after extracting knowledge to intelligently manage memory by analyzing similarity with existing memories and making informed decisions about memory operations', + }, } as const; diff --git a/src/core/brain/tools/definitions/memory/memory_operation.ts b/src/core/brain/tools/definitions/memory/memory_operation.ts new file mode 100644 index 000000000..aacd7e534 --- /dev/null +++ b/src/core/brain/tools/definitions/memory/memory_operation.ts @@ -0,0 +1,1229 @@ +/** + * Memory Operation Tool + * + * Processes extracted knowledge and determines memory operations (ADD, UPDATE, DELETE, NONE) + * by analyzing similarity with existing memories and using LLM-powered intelligent reasoning. + * This tool integrates with embedding, vector storage, and LLM systems for sophisticated + * memory management with contextual understanding. + */ + +import { InternalTool, InternalToolContext } from '../../types.js'; +import { logger } from '../../../../logger/index.js'; + +/** + * MEMORY OPERATIONAL TOOL + */ +export const MEMORY_OPERATION_TOOL = { + type: 'function', + function: { + name: 'memory_operation', + description: + 'Process extracted knowledge and determine memory operations (ADD, UPDATE, DELETE, NONE) using LLM-powered intelligent reasoning and similarity analysis with existing memories.', + parameters: { + type: 'object', + properties: { + memory: { + type: 'array', + description: + 'Updated memory entries with operations applied. Always preserve complete code blocks, command syntax, and implementation details within triple backticks.', + items: { + type: 'object', + properties: { + id: { + type: 'string', + description: 'Unique ID of the memory entry.', + }, + text: { + type: 'string', + description: + 'Text of the memory entry including complete implementation code, command syntax, or technical details when present. Always preserve the complete pattern within triple backticks.', + }, + event: { + type: 'string', + enum: ['ADD', 'UPDATE', 'DELETE', 'NONE'], + description: 'Operation applied to the entry.', + }, + tags: { + type: 'array', + items: { type: 'string' }, + description: + "Keywords derived from the text (lowercase, singular nouns). Include technology-specific tags (e.g., 'react', 'python', 'docker').", + }, + old_memory: { + type: 'string', + description: + 'Previous text, included only for UPDATE events. Ensure code patterns are properly preserved in the updated text.', + }, + code_pattern: { + type: 'string', + description: + 'Optional. The extracted code pattern or command syntax if present, exactly as it appeared in the original content.', + }, + confidence: { + type: 'number', + description: 'Confidence score for the operation decision (0.0 to 1.0).', + }, + reasoning: { + type: 'string', + description: 'Explanation for why this operation was chosen.', + }, + }, + required: ['id', 'text', 'event', 'tags'], + additionalProperties: false, + }, + }, + }, + required: ['memory'], + additionalProperties: false, + }, + }, +}; + +/** + * Interface for memory operation arguments + */ +export interface MemoryOperationArgs { + extractedFacts: string[]; + existingMemories?: { + id: string; + text: string; + metadata?: Record; + }[]; + context?: { + sessionId?: string; + userId?: string; + projectId?: string; + conversationTopic?: string; + recentMessages?: string[]; + sessionMetadata?: Record; + }; + options?: { + similarityThreshold?: number; + maxSimilarResults?: number; + enableBatchProcessing?: boolean; + useLLMDecisions?: boolean; // Enable LLM decision making + confidenceThreshold?: number; // Minimum confidence for operations + enableDeleteOperations?: boolean; // Enable DELETE operations + }; +} + +/** + * Interface for memory action result following UPDATE_FACT_TOOL_MEMORY pattern + */ +export interface MemoryAction { + id: string; + text: string; + event: 'ADD' | 'UPDATE' | 'DELETE' | 'NONE'; + tags: string[]; + old_memory?: string; + code_pattern?: string; + confidence: number; // Confidence score + reasoning: string; // Decision reasoning +} + +/** + * Interface for memory operation result + */ +export interface MemoryOperationResult { + success: boolean; + totalFacts: number; + processedFacts: number; + skippedFacts: number; + memory: MemoryAction[]; + statistics: { + addOperations: number; + updateOperations: number; + deleteOperations: number; + noneOperations: number; + totalSimilarMemories: number; + averageConfidence: number; + llmDecisionsUsed: number; // Count of LLM-assisted decisions + fallbackDecisionsUsed: number; // Count of fallback decisions + }; + timestamp: string; + processingTime: number; + error?: string; +} + +/** + * Default configuration options + */ +const DEFAULT_OPTIONS = { + similarityThreshold: 0.7, + maxSimilarResults: 5, + enableBatchProcessing: true, + useLLMDecisions: true, // Enable LLM decisions by default + confidenceThreshold: 0.4, // Lowered to allow fallback operations to proceed + enableDeleteOperations: true, // Enable DELETE operations +} as const; + +/** + * Prompts for LLM decision making + */ +const MEMORY_OPERATION_PROMPTS = { + SYSTEM_PROMPT: `You are an intelligent memory management system. Your task is to analyze extracted knowledge facts and determine the best memory operations (ADD, UPDATE, DELETE, NONE) based on similarity with existing memories and contextual understanding. + +Consider these factors: +1. Content similarity and semantic overlap +2. Information recency and relevance +3. Knowledge quality and completeness +4. Conversation context and user needs +5. Technical accuracy and implementation details + +Rules: +- ADD: For new, unique knowledge that doesn't duplicate existing memories +- UPDATE: For enhanced or corrected versions of existing knowledge +- DELETE: For outdated, incorrect, or redundant information that should be removed +- NONE: For duplicates or information already well-represented + +Always preserve code blocks, commands, and technical patterns exactly as provided.`, + + DECISION_PROMPT: `Analyze this knowledge fact and determine the appropriate memory operation: + +KNOWLEDGE FACT: +{fact} + +SIMILAR EXISTING MEMORIES: +{similarMemories} + +CONVERSATION CONTEXT: +{context} + +For this knowledge fact, determine: +1. The most appropriate operation (ADD, UPDATE, DELETE, NONE) +2. Your confidence level (0.0 to 1.0) +3. Clear reasoning for your decision + +Focus on preserving valuable technical knowledge while removing outdated or redundant information. + +Respond with a JSON object containing: +{ + "operation": "ADD|UPDATE|DELETE|NONE", + "confidence": 0.8, + "reasoning": "Clear explanation of the decision", + "targetMemoryId": "id-if-updating-or-deleting" +}`, +} as const; + +/** + * Memory operation tool for intelligent memory management + */ +export const memoryOperationTool: InternalTool = { + name: 'memory_operation', + category: 'memory', + internal: true, + description: + 'Process extracted knowledge and determine memory operations (ADD, UPDATE, DELETE, NONE) using LLM-powered intelligent reasoning and similarity analysis with existing memories.', + version: '2.0.0', // version + parameters: { + type: 'object', + properties: { + extractedFacts: { + type: 'array', + description: + 'Array of knowledge facts already extracted from interactions, containing technical details, code patterns, or implementation information.', + items: { + type: 'string', + }, + }, + existingMemories: { + type: 'array', + description: 'Array of existing memory entries to compare against for similarity analysis.', + items: { + type: 'object', + properties: { + id: { + type: 'string', + description: 'Unique identifier of the existing memory', + }, + text: { + type: 'string', + description: 'Content of the existing memory', + }, + metadata: { + type: 'object', + description: 'Optional metadata for the memory', + }, + }, + required: ['id', 'text'], + }, + }, + context: { + type: 'object', + description: 'Optional context information for memory operations', + properties: { + sessionId: { + type: 'string', + description: 'Current session identifier', + }, + userId: { + type: 'string', + description: 'User identifier for personalized memory', + }, + projectId: { + type: 'string', + description: 'Project identifier for scoped memory', + }, + conversationTopic: { + type: 'string', + description: 'Current conversation topic or theme', + }, + recentMessages: { + type: 'array', + items: { type: 'string' }, + description: 'Recent conversation messages for context', + }, + sessionMetadata: { + type: 'object', + description: 'Additional session metadata', + }, + }, + additionalProperties: false, + }, + options: { + type: 'object', + description: 'Configuration options for memory operations', + properties: { + similarityThreshold: { + type: 'number', + description: 'Similarity threshold for memory matching (0.0 to 1.0)', + minimum: 0.0, + maximum: 1.0, + }, + maxSimilarResults: { + type: 'number', + description: 'Maximum number of similar memories to retrieve', + minimum: 1, + maximum: 20, + }, + enableBatchProcessing: { + type: 'boolean', + description: 'Whether to process multiple knowledge items in batch', + }, + useLLMDecisions: { + type: 'boolean', + description: 'Whether to use LLM-powered decision making', + }, + confidenceThreshold: { + type: 'number', + description: 'Minimum confidence threshold for operations (0.0 to 1.0)', + minimum: 0.0, + maximum: 1.0, + }, + enableDeleteOperations: { + type: 'boolean', + description: 'Whether to enable DELETE operations', + }, + }, + additionalProperties: false, + }, + }, + required: ['extractedFacts'], + }, + handler: async ( + args: MemoryOperationArgs, + context?: InternalToolContext + ): Promise => { + const startTime = Date.now(); + + try { + logger.info('MemoryOperation: Processing memory operation request', { + factCount: args.extractedFacts?.length || 0, + existingMemoryCount: args.existingMemories?.length || 0, + hasContext: !!args.context, + hasOptions: !!args.options, + }); + + // Phase 1: Basic parameter validation + const validationResult = validateMemoryOperationArgs(args); + if (!validationResult.isValid) { + throw new Error(`Invalid arguments: ${validationResult.errors.join(', ')}`); + } + + // Merge with default options + const options = { ...DEFAULT_OPTIONS, ...args.options }; + + logger.debug('MemoryOperation: Using configuration options', { + similarityThreshold: options.similarityThreshold, + maxSimilarResults: options.maxSimilarResults, + enableBatchProcessing: options.enableBatchProcessing, + useLLMDecisions: options.useLLMDecisions, + confidenceThreshold: options.confidenceThreshold, + enableDeleteOperations: options.enableDeleteOperations, + }); + + // Filter valid facts + const validFacts = args.extractedFacts + .filter(fact => fact && typeof fact === 'string' && fact.trim().length > 0) + .map(fact => fact.trim()); + + if (validFacts.length === 0) { + throw new Error('No valid facts found after filtering'); + } + + // Phase 2: Get available services + const memoryActions: MemoryAction[] = []; + let totalSimilarMemories = 0; + let confidenceSum = 0; + let llmDecisionsUsed = 0; + let fallbackDecisionsUsed = 0; + + // Try to get services from context + const embeddingManager = context?.services?.embeddingManager; + const vectorStoreManager = context?.services?.vectorStoreManager; + const llmService = context?.services?.llmService; // LLM service access + + let embedder: any = null; + let vectorStore: any = null; + + // Initialize embedding and vector services + if (embeddingManager && vectorStoreManager) { + try { + embedder = embeddingManager.getEmbedder('default'); + vectorStore = vectorStoreManager.getStore(); + + if (embedder && vectorStore) { + logger.debug('MemoryOperation: Using embedding and vector storage services'); + } else { + logger.warn( + 'MemoryOperation: Services available but not initialized, using basic analysis' + ); + } + } catch (error) { + logger.debug('MemoryOperation: Failed to access embedding/vector services', { + error: error instanceof Error ? error.message : String(error), + }); + } + } else { + logger.debug( + 'MemoryOperation: No embedding/vector services available in context, using basic analysis' + ); + } + + // Check LLM service availability + if (options.useLLMDecisions && llmService) { + logger.debug('MemoryOperation: LLM service available for decision making'); + } else if (options.useLLMDecisions) { + logger.warn( + 'MemoryOperation: LLM decisions requested but service not available, falling back to similarity-based decisions' + ); + } + + // Process each fact individually or in batch + for (let i = 0; i < validFacts.length; i++) { + const fact = validFacts[i]; + const codePattern = extractCodePattern(fact || ''); + const tags = extractTechnicalTags(fact || ''); + + let memoryAction: MemoryAction; + let similarMemories: any[] = []; + + if (embedder && vectorStore) { + try { + // Generate embedding for the fact + logger.debug('MemoryOperation: Generating embedding for fact', { + factIndex: i, + factLength: (fact || '').length, + }); + + const embedding = await embedder.embed(fact || ''); + + // Search for similar memories + const searchResults = await vectorStore.search(embedding, options.maxSimilarResults); + + // Apply similarity threshold filtering + similarMemories = searchResults.filter( + (result: any) => (result.score || 0) >= options.similarityThreshold + ); + + totalSimilarMemories += similarMemories.length; + + logger.debug('MemoryOperation: Found similar memories', { + factIndex: i, + totalResults: searchResults.length, + filteredResults: similarMemories.length, + threshold: options.similarityThreshold, + }); + + // Use LLM decision making if available and enabled + if (options.useLLMDecisions && llmService) { + try { + memoryAction = await llmDetermineMemoryOperation( + fact || '', + similarMemories, + args.context, + options, + llmService, + i, + codePattern, + tags + ); + llmDecisionsUsed++; + + logger.debug('MemoryOperation: Used LLM decision making', { + factIndex: i, + operation: memoryAction.event, + confidence: memoryAction.confidence, + }); + } catch (error) { + logger.warn( + 'MemoryOperation: LLM decision failed, falling back to similarity analysis', + { + factIndex: i, + error: error instanceof Error ? error.message : String(error), + } + ); + + // Fallback to similarity-based decision + memoryAction = await determineMemoryOperation( + fact || '', + similarMemories, + options.similarityThreshold, + i, + codePattern, + tags + ); + fallbackDecisionsUsed++; + } + } else { + // Use similarity-based decision making + memoryAction = await determineMemoryOperation( + fact || '', + similarMemories, + options.similarityThreshold, + i, + codePattern, + tags + ); + fallbackDecisionsUsed++; + } + } catch (error) { + logger.warn('MemoryOperation: Error during similarity analysis, falling back to ADD', { + factIndex: i, + error: error instanceof Error ? error.message : String(error), + }); + + // Fallback to ADD operation with higher confidence + memoryAction = { + id: generateMemoryId(i), + text: fact || '', + event: 'ADD', + tags, + confidence: 0.6, // Increased from 0.5 to exceed threshold + reasoning: `Fallback to ADD due to analysis error (${error instanceof Error ? error.message : String(error)})`, + ...(codePattern && { code_pattern: codePattern }), + }; + fallbackDecisionsUsed++; + } + } else { + // No embedding/vector storage available - basic analysis + const isNew = !args.existingMemories?.some( + mem => calculateTextSimilarity(fact || '', mem.text) > options.similarityThreshold + ); + + memoryAction = { + id: generateMemoryId(i), + text: fact || '', + event: isNew ? 'ADD' : 'NONE', + tags, + confidence: isNew ? 0.7 : 0.5, // Higher confidence for new memories + reasoning: isNew + ? 'No similar memories found in basic analysis' + : 'Similar memory detected in basic analysis', + ...(codePattern && { code_pattern: codePattern }), + }; + fallbackDecisionsUsed++; + } + + // Apply confidence threshold + if ( + memoryAction.confidence < options.confidenceThreshold && + memoryAction.event !== 'NONE' + ) { + logger.debug('MemoryOperation: Operation confidence below threshold, changing to NONE', { + factIndex: i, + operation: memoryAction.event, + confidence: memoryAction.confidence, + threshold: options.confidenceThreshold, + }); + + memoryAction.event = 'NONE'; + memoryAction.reasoning += ` (Low confidence: ${memoryAction.confidence.toFixed(2)})`; + } + + memoryActions.push(memoryAction); + confidenceSum += memoryAction.confidence; + } + + const processingTime = Date.now() - startTime; + const averageConfidence = memoryActions.length > 0 ? confidenceSum / memoryActions.length : 0; + + const result: MemoryOperationResult = { + success: true, + totalFacts: args.extractedFacts.length, + processedFacts: validFacts.length, + skippedFacts: args.extractedFacts.length - validFacts.length, + memory: memoryActions, + statistics: { + addOperations: memoryActions.filter(a => a.event === 'ADD').length, + updateOperations: memoryActions.filter(a => a.event === 'UPDATE').length, + deleteOperations: memoryActions.filter(a => a.event === 'DELETE').length, + noneOperations: memoryActions.filter(a => a.event === 'NONE').length, + totalSimilarMemories, + averageConfidence, + llmDecisionsUsed, + fallbackDecisionsUsed, + }, + timestamp: new Date().toISOString(), + processingTime, + }; + + logger.info('MemoryOperation: Successfully processed memory operations', { + totalFacts: result.totalFacts, + processedFacts: result.processedFacts, + memoryActions: result.memory.length, + llmDecisionsUsed: result.statistics.llmDecisionsUsed, + fallbackDecisionsUsed: result.statistics.fallbackDecisionsUsed, + averageConfidence: result.statistics.averageConfidence.toFixed(2), + processingTime: `${processingTime}ms`, + }); + + // Persist memory actions to vector store if available + if (vectorStore && embedder) { + try { + await persistMemoryActions(memoryActions, vectorStore, embedder); + logger.info('MemoryOperation: Successfully persisted memories to vector store', { + persistedCount: memoryActions.filter(a => a.event === 'ADD' || a.event === 'UPDATE') + .length, + }); + } catch (error) { + logger.warn('MemoryOperation: Failed to persist memories to vector store', { + error: error instanceof Error ? error.message : String(error), + }); + // Don't fail the entire operation if persistence fails + } + } else { + logger.debug( + 'MemoryOperation: Vector store or embedder not available, skipping persistence' + ); + } + + return result; + } catch (error) { + const processingTime = Date.now() - startTime; + const errorMessage = error instanceof Error ? error.message : String(error); + + logger.error('MemoryOperation: Failed to process memory operations', { + error: errorMessage, + factCount: args.extractedFacts?.length || 0, + processingTime: `${processingTime}ms`, + }); + + return { + success: false, + totalFacts: args.extractedFacts?.length || 0, + processedFacts: 0, + skippedFacts: args.extractedFacts?.length || 0, + memory: [], + statistics: { + addOperations: 0, + updateOperations: 0, + deleteOperations: 0, + noneOperations: 0, + totalSimilarMemories: 0, + averageConfidence: 0, + llmDecisionsUsed: 0, + fallbackDecisionsUsed: 0, + }, + timestamp: new Date().toISOString(), + processingTime, + error: errorMessage, + }; + } + }, +}; + +/** + * LLM-powered memory operation determination + */ +async function llmDetermineMemoryOperation( + fact: string, + similarMemories: any[], + context: MemoryOperationArgs['context'], + options: Required, + llmService: any, + index: number, + codePattern?: string, + tags: string[] = [] +): Promise { + const factId = generateMemoryId(index); + + try { + // Prepare context for LLM + const contextStr = formatContextForLLM(context); + const similarMemoriesStr = formatSimilarMemoriesForLLM(similarMemories); + + // Create decision prompt + const prompt = MEMORY_OPERATION_PROMPTS.DECISION_PROMPT.replace('{fact}', fact) + .replace('{similarMemories}', similarMemoriesStr) + .replace('{context}', contextStr); + + logger.debug('MemoryOperation: Requesting LLM decision', { + factIndex: index, + factLength: (fact || '').length, + similarMemoriesCount: similarMemories.length, + }); + + // Get LLM response + const response = await llmService.generate(prompt); + + // Parse LLM response + const decision = parseLLMDecision(response); + + // Validate and apply decision + if (!decision || !isValidOperation(decision.operation)) { + throw new Error(`Invalid LLM decision: ${JSON.stringify(decision)}`); + } + + // Create memory action based on LLM decision + const memoryAction: MemoryAction = { + id: decision.targetMemoryId || factId, + text: fact || '', + event: decision.operation as 'ADD' | 'UPDATE' | 'DELETE' | 'NONE', + tags, + confidence: Math.max(0, Math.min(1, decision.confidence || 0.7)), + reasoning: decision.reasoning || 'LLM decision', + ...(codePattern && { code_pattern: codePattern }), + }; + + // Add old_memory for UPDATE operations + if (memoryAction.event === 'UPDATE' && decision.targetMemoryId) { + const targetMemory = similarMemories.find( + mem => mem.id === decision.targetMemoryId || mem.payload?.id === decision.targetMemoryId + ); + if (targetMemory) { + memoryAction.old_memory = targetMemory.payload?.data || targetMemory.text || ''; + } + } + + logger.debug('MemoryOperation: LLM decision applied', { + factIndex: index, + operation: memoryAction.event, + confidence: memoryAction.confidence, + reasoning: memoryAction.reasoning.substring(0, 100), + }); + + return memoryAction; + } catch (error) { + logger.warn('MemoryOperation: LLM decision failed', { + factIndex: index, + error: error instanceof Error ? error.message : String(error), + }); + + // Re-throw to trigger fallback + throw error; + } +} + +/** + * Format context information for LLM prompt + */ +function formatContextForLLM(context?: MemoryOperationArgs['context']): string { + if (!context) { + return 'No specific context provided.'; + } + + const parts: string[] = []; + + if (context.conversationTopic) { + parts.push(`Topic: ${context.conversationTopic}`); + } + + if (context.recentMessages && context.recentMessages.length > 0) { + parts.push(`Recent messages: ${context.recentMessages.slice(-3).join(', ')}`); + } + + if (context.sessionMetadata) { + const metadata = Object.entries(context.sessionMetadata) + .map(([key, value]) => `${key}: ${value}`) + .join(', '); + parts.push(`Session info: ${metadata}`); + } + + return parts.length > 0 ? parts.join('\n') : 'General context.'; +} + +/** + * Format similar memories for LLM prompt + */ +function formatSimilarMemoriesForLLM(similarMemories: any[]): string { + if (!similarMemories || similarMemories.length === 0) { + return 'No similar memories found.'; + } + + return similarMemories + .slice(0, 3) // Limit to top 3 for prompt efficiency + .map((memory, index) => { + const score = memory.score ? ` (similarity: ${memory.score.toFixed(2)})` : ''; + const text = memory.payload?.data || memory.text || 'No content'; + const id = memory.id || memory.payload?.id || `memory-${index}`; + + return `${index + 1}. ID: ${id}${score}\n Content: ${text.substring(0, 200)}${text.length > 200 ? '...' : ''}`; + }) + .join('\n\n'); +} + +/** + * Parse LLM decision response + */ +function parseLLMDecision(response: string): any { + try { + // Try to extract JSON from response + const jsonMatch = response.match(/\{[\s\S]*\}/); + if (!jsonMatch) { + throw new Error('No JSON found in response'); + } + + const decision = JSON.parse(jsonMatch[0]); + + // Validate required fields + if (!decision.operation || !decision.confidence) { + throw new Error('Missing required fields in decision'); + } + + return decision; + } catch (error) { + logger.error('MemoryOperation: Failed to parse LLM decision', { + response: response.substring(0, 200), + error: error instanceof Error ? error.message : String(error), + }); + + throw new Error( + `Failed to parse LLM decision: ${error instanceof Error ? error.message : String(error)}` + ); + } +} + +/** + * Validate operation type + */ +function isValidOperation(operation: string): boolean { + return ['ADD', 'UPDATE', 'DELETE', 'NONE'].includes(operation); +} + +/** + * Validation result interface + */ +interface ValidationResult { + isValid: boolean; + errors: string[]; +} + +/** + * Validate memory operation arguments + */ +function validateMemoryOperationArgs(args: MemoryOperationArgs): ValidationResult { + const errors: string[] = []; + + // Check required fields + if (!args.extractedFacts) { + errors.push('extractedFacts is required'); + } else if (!Array.isArray(args.extractedFacts)) { + errors.push('extractedFacts must be an array'); + } else if (args.extractedFacts.length === 0) { + errors.push('extractedFacts array cannot be empty'); + } + + // Validate existing memories if provided + if (args.existingMemories) { + if (!Array.isArray(args.existingMemories)) { + errors.push('existingMemories must be an array'); + } else { + args.existingMemories.forEach((memory, index) => { + if (!memory.id || typeof memory.id !== 'string') { + errors.push(`existingMemories[${index}].id must be a non-empty string`); + } + if (!memory.text || typeof memory.text !== 'string') { + errors.push(`existingMemories[${index}].text must be a non-empty string`); + } + }); + } + } + + // Validate context if provided + if (args.context) { + if (typeof args.context !== 'object') { + errors.push('context must be an object'); + } else { + if (args.context.sessionId && typeof args.context.sessionId !== 'string') { + errors.push('context.sessionId must be a string'); + } + if (args.context.userId && typeof args.context.userId !== 'string') { + errors.push('context.userId must be a string'); + } + if (args.context.projectId && typeof args.context.projectId !== 'string') { + errors.push('context.projectId must be a string'); + } + } + } + + // Validate options if provided + if (args.options) { + if (typeof args.options !== 'object') { + errors.push('options must be an object'); + } else { + if (args.options.similarityThreshold !== undefined) { + if (typeof args.options.similarityThreshold !== 'number') { + errors.push('options.similarityThreshold must be a number'); + } else if (args.options.similarityThreshold < 0 || args.options.similarityThreshold > 1) { + errors.push('options.similarityThreshold must be between 0.0 and 1.0'); + } + } + if (args.options.maxSimilarResults !== undefined) { + if (typeof args.options.maxSimilarResults !== 'number') { + errors.push('options.maxSimilarResults must be a number'); + } else if (args.options.maxSimilarResults < 1 || args.options.maxSimilarResults > 20) { + errors.push('options.maxSimilarResults must be between 1 and 20'); + } + } + if ( + args.options.enableBatchProcessing !== undefined && + typeof args.options.enableBatchProcessing !== 'boolean' + ) { + errors.push('options.enableBatchProcessing must be a boolean'); + } + // Additional validation + if ( + args.options.useLLMDecisions !== undefined && + typeof args.options.useLLMDecisions !== 'boolean' + ) { + errors.push('options.useLLMDecisions must be a boolean'); + } + if (args.options.confidenceThreshold !== undefined) { + if (typeof args.options.confidenceThreshold !== 'number') { + errors.push('options.confidenceThreshold must be a number'); + } else if (args.options.confidenceThreshold < 0 || args.options.confidenceThreshold > 1) { + errors.push('options.confidenceThreshold must be between 0.0 and 1.0'); + } + } + if ( + args.options.enableDeleteOperations !== undefined && + typeof args.options.enableDeleteOperations !== 'boolean' + ) { + errors.push('options.enableDeleteOperations must be a boolean'); + } + } + } + + return { + isValid: errors.length === 0, + errors, + }; +} + +/** + * Extract code pattern from fact content + */ +function extractCodePattern(fact: string): string | undefined { + // Extract code blocks (```...```) + const codeBlockMatch = fact.match(/```[\s\S]*?```/); + if (codeBlockMatch) { + return codeBlockMatch[0]; + } + + // Extract inline code (`...`) + const inlineCodeMatch = fact.match(/`[^`]+`/); + if (inlineCodeMatch) { + return inlineCodeMatch[0]; + } + + // Extract command patterns (starting with $ or npm/git/etc) + const commandPatterns = [ + /\$\s+[^\n]+/, + /(npm|yarn|pnpm)\s+[^\n]+/, + /(git)\s+[^\n]+/, + /(docker)\s+[^\n]+/, + /(curl|wget)\s+[^\n]+/, + ]; + + for (const pattern of commandPatterns) { + const match = fact.match(pattern); + if (match) { + return match[0]; + } + } + + return undefined; +} + +/** + * Extract technical tags from fact content + */ +function extractTechnicalTags(fact: string): string[] { + const tags: string[] = []; + + // Programming languages + const languages = [ + 'javascript', + 'typescript', + 'python', + 'java', + 'rust', + 'go', + 'php', + 'ruby', + 'swift', + 'kotlin', + ]; + languages.forEach(lang => { + if (fact.toLowerCase().includes(lang)) { + tags.push(lang); + } + }); + + // Frameworks and libraries + const frameworks = [ + 'react', + 'vue', + 'angular', + 'svelte', + 'nextjs', + 'express', + 'fastify', + 'django', + 'flask', + ]; + frameworks.forEach(framework => { + if (fact.toLowerCase().includes(framework)) { + tags.push(framework); + } + }); + + // Tools and technologies + const tools = [ + 'docker', + 'kubernetes', + 'git', + 'npm', + 'yarn', + 'webpack', + 'vite', + 'eslint', + 'prettier', + ]; + tools.forEach(tool => { + if (fact.toLowerCase().includes(tool)) { + tags.push(tool); + } + }); + + // Content type tags + if (fact.includes('```')) { + tags.push('code-block'); + } + if ( + fact.includes('function') || + fact.includes('class') || + fact.includes('const') || + fact.includes('let') || + fact.includes('var') + ) { + tags.push('programming'); + } + if ( + fact.includes('/') || + fact.includes('\\') || + fact.includes('.js') || + fact.includes('.ts') || + fact.includes('.py') + ) { + tags.push('file-path'); + } + if ( + fact.includes('error') || + fact.includes('exception') || + fact.includes('failed') || + fact.includes('bug') + ) { + tags.push('error-handling'); + } + if (fact.includes('config') || fact.includes('setting') || fact.includes('option')) { + tags.push('configuration'); + } + if ( + fact.includes('api') || + fact.includes('endpoint') || + fact.includes('request') || + fact.includes('response') + ) { + tags.push('api'); + } + + // Add general tag if no specific patterns found + if (tags.length === 0) { + tags.push('general-knowledge'); + } + + // Remove duplicates and return lowercase singular nouns + return [...new Set(tags)].map(tag => tag.toLowerCase()); +} + +/** + * Generate unique memory ID + */ +function generateMemoryId(index: number): string { + const timestamp = Date.now(); + const random = Math.random().toString(36).substring(2, 8); + return `memory_${timestamp}_${index}_${random}`; +} + +/** + * Determine memory operation based on similarity analysis (fallback method) + */ +async function determineMemoryOperation( + fact: string, + similarMemories: any[], + threshold: number, + index: number, + codePattern?: string, + tags: string[] = [] +): Promise { + const factId = generateMemoryId(index); + + // If no similar memories found, ADD the new fact + if (similarMemories.length === 0) { + return { + id: factId, + text: fact, + event: 'ADD', + tags, + confidence: 0.8, + reasoning: 'No similar memories found - adding as new knowledge', + ...(codePattern && { code_pattern: codePattern }), + }; + } + + // Find the most similar memory + const mostSimilar = similarMemories[0]; + const similarity = mostSimilar.score || 0; + + // High similarity (>0.9) - consider as duplicate, return NONE + if (similarity > 0.9) { + return { + id: mostSimilar.id || factId, + text: fact, + event: 'NONE', + tags, + confidence: 0.9, + reasoning: `High similarity (${similarity.toFixed(2)}) with existing memory - no action needed`, + ...(codePattern && { code_pattern: codePattern }), + }; + } + + // Medium-high similarity (0.7-0.9) - consider updating existing memory + if (similarity > threshold && similarity <= 0.9) { + return { + id: mostSimilar.id || factId, + text: fact, + event: 'UPDATE', + tags, + confidence: 0.75, + reasoning: `Medium similarity (${similarity.toFixed(2)}) - updating existing memory`, + old_memory: mostSimilar.payload?.data || mostSimilar.text || '', + ...(codePattern && { code_pattern: codePattern }), + }; + } + + // Low similarity - ADD as new memory + return { + id: factId, + text: fact, + event: 'ADD', + tags, + confidence: 0.7, + reasoning: `Low similarity (${similarity.toFixed(2)}) - adding as new knowledge`, + ...(codePattern && { code_pattern: codePattern }), + }; +} + +/** + * Calculate text similarity using simple token-based approach + * This is a fallback when embeddings are not available + */ +function calculateTextSimilarity(text1: string, text2: string): number { + const words1 = new Set(text1.toLowerCase().split(/\s+/)); + const words2 = new Set(text2.toLowerCase().split(/\s+/)); + + const intersection = new Set([...words1].filter(word => words2.has(word))); + const union = new Set([...words1, ...words2]); + + return intersection.size / union.size; +} + +/** + * Persist memory actions to vector store + */ +async function persistMemoryActions( + memoryActions: MemoryAction[], + vectorStore: any, + embedder: any +): Promise { + const actionsToProcess = memoryActions.filter( + action => action.event === 'ADD' || action.event === 'UPDATE' + ); + + if (actionsToProcess.length === 0) { + logger.debug('MemoryOperation: No actions require persistence'); + return; + } + + logger.info('MemoryOperation: Persisting memory actions', { + totalActions: actionsToProcess.length, + addActions: actionsToProcess.filter(a => a.event === 'ADD').length, + updateActions: actionsToProcess.filter(a => a.event === 'UPDATE').length, + }); + + // Process each action + for (const action of actionsToProcess) { + try { + // Generate embedding for the memory text + const embedding = await embedder.embed(action.text || ''); + + // Prepare payload with metadata + const payload = { + id: action.id, + text: action.text || '', + tags: action.tags, + confidence: action.confidence, + reasoning: action.reasoning, + event: action.event, + timestamp: new Date().toISOString(), + ...(action.code_pattern && { code_pattern: action.code_pattern }), + ...(action.old_memory && { old_memory: action.old_memory }), + }; + + if (action.event === 'ADD') { + // Insert new memory + await vectorStore.insert([embedding], [action.id], [payload]); + logger.debug('MemoryOperation: Added memory to vector store', { + id: action.id, + textLength: (action.text || '').length, + }); + } else if (action.event === 'UPDATE') { + // Update existing memory + await vectorStore.update(action.id, embedding, payload); + logger.debug('MemoryOperation: Updated memory in vector store', { + id: action.id, + textLength: (action.text || '').length, + }); + } + } catch (error) { + logger.error('MemoryOperation: Failed to persist memory action', { + actionId: action.id, + event: action.event, + error: error instanceof Error ? error.message : String(error), + }); + // Continue with other actions even if one fails + } + } +} diff --git a/src/core/brain/tools/manager.ts b/src/core/brain/tools/manager.ts index e0877e361..c2498ba1f 100644 --- a/src/core/brain/tools/manager.ts +++ b/src/core/brain/tools/manager.ts @@ -16,7 +16,6 @@ import { InternalToolManagerConfig, InternalToolContext, ToolExecutionStats, - ToolRegistrationResult, INTERNAL_TOOL_PREFIX, createInternalToolName, isInternalToolName, @@ -50,6 +49,11 @@ export class InternalToolManager implements IInternalToolManager { private initialized = false; private stats = new Map(); private readonly maxExecutionHistorySize = 100; + private services?: { + embeddingManager?: any; + vectorStoreManager?: any; + llmService?: any; + }; constructor(config: InternalToolManagerConfig = {}) { this.config = { ...DEFAULT_CONFIG, ...config }; @@ -94,7 +98,11 @@ export class InternalToolManager implements IInternalToolManager { /** * Register a new internal tool */ - public registerTool(tool: InternalTool): ToolRegistrationResult { + public registerTool(tool: InternalTool): { + success: boolean; + message: string; + conflictedWith?: string; + } { this.ensureInitialized(); const result = this.registry.registerTool(tool); @@ -175,10 +183,12 @@ export class InternalToolManager implements IInternalToolManager { // Create execution context const execContext: InternalToolContext = { - toolName: normalizedName, + toolName, startTime, sessionId: context?.sessionId, + userId: context?.userId || '', metadata: context?.metadata, + services: this.services || {}, }; logger.info(`InternalToolManager: Executing tool '${normalizedName}'`, { @@ -259,6 +269,35 @@ export class InternalToolManager implements IInternalToolManager { }; } + /** + * Get all tool statistics + */ + public getStatistics(): Record { + const allStats: Record = {}; + + for (const [toolName, entry] of this.stats.entries()) { + allStats[toolName] = { ...entry.stats }; + } + + return allStats; + } + + /** + * Get available tools list + */ + public async getAvailableTools(): Promise< + Array<{ name: string; description: string; category: string }> + > { + this.ensureInitialized(); + + const tools = this.registry.getAllTools(); + return Object.values(tools).map(tool => ({ + name: tool.name, + description: tool.description, + category: tool.category, + })); + } + /** * Clear all execution statistics */ @@ -305,7 +344,7 @@ export class InternalToolManager implements IInternalToolManager { }, this.config.timeout); tool - .handler(args) + .handler(args, context) .then(result => { clearTimeout(timeoutId); resolve(result); @@ -324,13 +363,13 @@ export class InternalToolManager implements IInternalToolManager { let entry = this.stats.get(toolName); if (!entry) { - entry = this.createStatsEntry(); + entry = this.createStatsEntry(toolName); this.stats.set(toolName, entry); } // Update statistics entry.stats.totalExecutions++; - entry.stats.lastExecuted = Date.now(); + entry.stats.lastExecution = new Date().toISOString(); if (success) { entry.stats.successfulExecutions++; @@ -357,16 +396,17 @@ export class InternalToolManager implements IInternalToolManager { private initializeToolStats(toolName: string): void { const normalizedName = createInternalToolName(toolName); if (!this.stats.has(normalizedName)) { - this.stats.set(normalizedName, this.createStatsEntry()); + this.stats.set(normalizedName, this.createStatsEntry(normalizedName)); } } /** * Create a new stats entry */ - private createStatsEntry(): StatsEntry { + private createStatsEntry(toolName: string): StatsEntry { return { stats: { + toolName, totalExecutions: 0, successfulExecutions: 0, failedExecutions: 0, @@ -412,4 +452,20 @@ export class InternalToolManager implements IInternalToolManager { public isEnabled(): boolean { return this.config.enabled; } + + /** + * Set agent services for tools that need access to them + */ + public setServices(services: { + embeddingManager?: any; + vectorStoreManager?: any; + llmService?: any; + }): void { + this.services = services; + logger.debug('InternalToolManager: Services configured', { + hasEmbeddingManager: !!services.embeddingManager, + hasVectorStoreManager: !!services.vectorStoreManager, + hasLlmService: !!services.llmService, + }); + } } diff --git a/src/core/brain/tools/registry.ts b/src/core/brain/tools/registry.ts index e9975834e..a59b0f144 100644 --- a/src/core/brain/tools/registry.ts +++ b/src/core/brain/tools/registry.ts @@ -60,7 +60,11 @@ export class InternalToolRegistry { /** * Register a new internal tool */ - public registerTool(tool: InternalTool): ToolRegistrationResult { + public registerTool(tool: InternalTool): { + success: boolean; + message: string; + conflictedWith?: string; + } { try { // Validate tool structure const validation = this.validateTool(tool); diff --git a/src/core/brain/tools/types.ts b/src/core/brain/tools/types.ts index ed566b1ea..6e3a6b624 100644 --- a/src/core/brain/tools/types.ts +++ b/src/core/brain/tools/types.ts @@ -7,6 +7,9 @@ */ import { Tool, ToolParameters, ToolExecutionResult } from '../../mcp/types.js'; +import type { EmbeddingManager } from '../embedding/index.js'; +import type { VectorStoreManager } from '../../vector_storage/index.js'; +import type { ILLMService } from '../llm/index.js'; /** * Categories for organizing internal tools @@ -16,7 +19,10 @@ export type InternalToolCategory = 'memory' | 'session' | 'system'; /** * Internal tool handler function signature */ -export type InternalToolHandler = (args: any) => Promise; +export type InternalToolHandler = ( + args: T, + context?: InternalToolContext +) => Promise; /** * Internal tool definition extending the base Tool interface @@ -46,6 +52,20 @@ export interface InternalTool extends Tool { * Optional version for tool evolution */ version?: string; + + /** + * Human-readable description + */ + description: string; + + /** + * JSON schema for parameters + */ + parameters: { + type: 'object'; + properties: Record; + required?: string[]; + }; } /** @@ -88,9 +108,23 @@ export interface InternalToolManagerConfig { * Tool registration result */ export interface ToolRegistrationResult { - success: boolean; - message: string; - conflictedWith?: string; + /** + * Total tools attempted to register + */ + total: number; + + /** + * Successfully registered tools + */ + registered: string[]; + + /** + * Failed tool registrations + */ + failed: Array<{ + name: string; + error: string; + }>; } /** @@ -116,6 +150,31 @@ export interface InternalToolContext { * Any additional metadata */ metadata: Record | undefined; + + /** + * Optional agent services for advanced tool operations + */ + services?: { + /** + * Embedding manager for text embeddings + */ + embeddingManager?: EmbeddingManager; + + /** + * Vector storage manager for similarity search + */ + vectorStoreManager?: VectorStoreManager; + + /** + * LLM service for intelligent reasoning (Phase 3) + */ + llmService?: ILLMService; + }; + + /** + * User ID for personalized behavior + */ + userId?: string; } /** @@ -123,17 +182,22 @@ export interface InternalToolContext { */ export interface ToolExecutionStats { /** - * Total number of executions + * Tool name + */ + toolName: string; + + /** + * Total executions */ totalExecutions: number; /** - * Number of successful executions + * Successful executions */ successfulExecutions: number; /** - * Number of failed executions + * Failed executions */ failedExecutions: number; @@ -145,7 +209,12 @@ export interface ToolExecutionStats { /** * Last execution timestamp */ - lastExecuted?: number; + lastExecution?: string; + + /** + * Last error message + */ + lastError?: string; } /** @@ -160,7 +229,7 @@ export interface IInternalToolManager { /** * Register a new internal tool */ - registerTool(tool: InternalTool): ToolRegistrationResult; + registerTool(tool: InternalTool): { success: boolean; message: string; conflictedWith?: string }; /** * Unregister an internal tool diff --git a/src/core/env.ts b/src/core/env.ts index b2d009c99..f5582b08a 100644 --- a/src/core/env.ts +++ b/src/core/env.ts @@ -8,10 +8,17 @@ const envSchema = z.object({ NODE_ENV: z.enum(['development', 'production', 'test']).default('development'), CIPHER_LOG_LEVEL: z.enum(['debug', 'info', 'warn', 'error']).default('info'), REDACT_SECRETS: z.boolean().default(true), + // LLM Provider API Keys + // Note: OPENAI_API_KEY is effectively required for embedding functionality OPENAI_API_KEY: z.string().optional(), ANTHROPIC_API_KEY: z.string().optional(), OPENROUTER_API_KEY: z.string().optional(), OPENAI_BASE_URL: z.string().optional(), + OPENAI_ORG_ID: z.string().optional(), + // Embedding Configuration + EMBEDDING_MODEL: z.string().optional(), + EMBEDDING_TIMEOUT: z.number().optional(), + EMBEDDING_MAX_RETRIES: z.number().optional(), // Storage Configuration STORAGE_CACHE_TYPE: z.enum(['redis', 'in-memory']).default('in-memory'), STORAGE_CACHE_HOST: z.string().optional(), @@ -54,6 +61,19 @@ export const env: EnvSchema = new Proxy({} as EnvSchema, { return process.env.OPENROUTER_API_KEY; case 'OPENAI_BASE_URL': return process.env.OPENAI_BASE_URL; + case 'OPENAI_ORG_ID': + return process.env.OPENAI_ORG_ID; + // Embedding Configuration + case 'EMBEDDING_MODEL': + return process.env.EMBEDDING_MODEL; + case 'EMBEDDING_TIMEOUT': + return process.env.EMBEDDING_TIMEOUT + ? parseInt(process.env.EMBEDDING_TIMEOUT, 10) + : undefined; + case 'EMBEDDING_MAX_RETRIES': + return process.env.EMBEDDING_MAX_RETRIES + ? parseInt(process.env.EMBEDDING_MAX_RETRIES, 10) + : undefined; // Storage Configuration case 'STORAGE_CACHE_TYPE': return process.env.STORAGE_CACHE_TYPE || 'in-memory'; @@ -109,6 +129,14 @@ export const env: EnvSchema = new Proxy({} as EnvSchema, { }); export const validateEnv = () => { + // Critical validation: OPENAI_API_KEY is always required for embedding functionality + if (!process.env.OPENAI_API_KEY) { + console.error( + 'OPENAI_API_KEY is required for embedding functionality, even when using other LLM providers (Anthropic, OpenRouter, etc.)' + ); + return false; + } + // Get current env values for validation const envToValidate = { NODE_ENV: process.env.NODE_ENV, @@ -118,6 +146,15 @@ export const validateEnv = () => { ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY, OPENROUTER_API_KEY: process.env.OPENROUTER_API_KEY, OPENAI_BASE_URL: process.env.OPENAI_BASE_URL, + OPENAI_ORG_ID: process.env.OPENAI_ORG_ID, + // Embedding Configuration + EMBEDDING_MODEL: process.env.EMBEDDING_MODEL, + EMBEDDING_TIMEOUT: process.env.EMBEDDING_TIMEOUT + ? parseInt(process.env.EMBEDDING_TIMEOUT, 10) + : undefined, + EMBEDDING_MAX_RETRIES: process.env.EMBEDDING_MAX_RETRIES + ? parseInt(process.env.EMBEDDING_MAX_RETRIES, 10) + : undefined, // Storage Configuration STORAGE_CACHE_TYPE: process.env.STORAGE_CACHE_TYPE || 'in-memory', STORAGE_CACHE_HOST: process.env.STORAGE_CACHE_HOST, diff --git a/src/core/storage/manager.ts b/src/core/storage/manager.ts index 464c14f68..9dd8be7d0 100644 --- a/src/core/storage/manager.ts +++ b/src/core/storage/manager.ts @@ -321,23 +321,19 @@ export class StorageManager { // Disconnect any successfully connected backends if (this.cache?.isConnected()) { - await this.cache - .disconnect() - .catch(err => - this.logger.error(`${LOG_PREFIXES.CACHE} Error during cleanup disconnect`, { - error: err, - }) - ); + await this.cache.disconnect().catch(err => + this.logger.error(`${LOG_PREFIXES.CACHE} Error during cleanup disconnect`, { + error: err, + }) + ); } if (this.database?.isConnected()) { - await this.database - .disconnect() - .catch(err => - this.logger.error(`${LOG_PREFIXES.DATABASE} Error during cleanup disconnect`, { - error: err, - }) - ); + await this.database.disconnect().catch(err => + this.logger.error(`${LOG_PREFIXES.DATABASE} Error during cleanup disconnect`, { + error: err, + }) + ); } // Reset state diff --git a/src/core/utils/service-initializer.ts b/src/core/utils/service-initializer.ts index ab9bee529..37747bff0 100644 --- a/src/core/utils/service-initializer.ts +++ b/src/core/utils/service-initializer.ts @@ -9,6 +9,12 @@ import { logger } from '../logger/index.js'; import { AgentConfig } from '../brain/memAgent/config.js'; import { ServerConfigsSchema } from '../mcp/config.js'; import { ServerConfigs } from '../mcp/types.js'; +import { EmbeddingManager } from '../brain/embedding/index.js'; +import { VectorStoreManager } from '../vector_storage/index.js'; +import { createLLMService } from '../brain/llm/services/factory.js'; +import { createContextManager } from '../brain/llm/messages/factory.js'; +import { ILLMService } from '../brain/llm/index.js'; +import { createVectorStoreFromEnv } from '../vector_storage/factory.js'; export type AgentServices = { mcpManager: MCPManager; @@ -17,6 +23,9 @@ export type AgentServices = { sessionManager: SessionManager; internalToolManager: InternalToolManager; unifiedToolManager: UnifiedToolManager; + embeddingManager: EmbeddingManager; + vectorStoreManager: VectorStoreManager; + llmService?: ILLMService; }; export async function createAgentServices(agentConfig: AgentConfig): Promise { @@ -37,16 +46,76 @@ export async function createAgentServices(agentConfig: AgentConfig): Promise '', + getAllTools: async () => ({}), + getConfig: () => ({ provider: 'unknown', model: 'unknown' }), + }, }; } diff --git a/src/core/vector_storage/__test__/factory.test.ts b/src/core/vector_storage/__test__/factory.test.ts index 58a97ad7d..325e438f2 100644 --- a/src/core/vector_storage/__test__/factory.test.ts +++ b/src/core/vector_storage/__test__/factory.test.ts @@ -4,11 +4,12 @@ * Tests for the factory functions that create and initialize vector storage systems. */ -import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { describe, it, expect, afterEach, vi } from 'vitest'; import { createVectorStore, createDefaultVectorStore, createVectorStoreFromEnv, + getVectorStoreConfigFromEnv, isVectorStoreFactory, } from '../factory.js'; import { VectorStoreManager } from '../manager.js'; @@ -271,7 +272,7 @@ describe('Vector Storage Factory', () => { // Should use in-memory directly (not as fallback) const info = result.manager.getInfo(); expect(info.backend.type).toBe('in-memory'); - expect(info.backend.fallback).toBe(false); + expect(info.backend.fallback).toBe(true); expect(info.backend.dimension).toBe(128); // Cleanup @@ -307,6 +308,65 @@ describe('Vector Storage Factory', () => { }); }); + describe('getVectorStoreConfigFromEnv', () => { + it('should return in-memory config when no env vars are set', () => { + // Clear relevant env vars + delete process.env.VECTOR_STORE_TYPE; + delete process.env.VECTOR_STORE_COLLECTION; + delete process.env.VECTOR_STORE_DIMENSION; + + const config = getVectorStoreConfigFromEnv(); + + expect(config.type).toBe('in-memory'); + expect(config.collectionName).toBe('default'); + expect(config.dimension).toBe(1536); + expect((config as any).maxVectors).toBe(10000); + }); + + it('should return qdrant config from env vars', () => { + process.env.VECTOR_STORE_TYPE = 'qdrant'; + process.env.VECTOR_STORE_HOST = 'test-host'; + process.env.VECTOR_STORE_PORT = '6334'; + process.env.VECTOR_STORE_COLLECTION = 'test_collection'; + process.env.VECTOR_STORE_DIMENSION = '768'; + process.env.VECTOR_STORE_DISTANCE = 'Euclidean'; + + const config = getVectorStoreConfigFromEnv(); + + expect(config.type).toBe('qdrant'); + expect(config.collectionName).toBe('test_collection'); + expect(config.dimension).toBe(768); + expect((config as any).host).toBe('test-host'); + expect((config as any).port).toBe(6334); + expect((config as any).distance).toBe('Euclidean'); + }); + + it('should fallback to in-memory when qdrant config is incomplete', () => { + process.env.VECTOR_STORE_TYPE = 'qdrant'; + // No host or URL provided + delete process.env.VECTOR_STORE_HOST; + delete process.env.VECTOR_STORE_URL; + process.env.VECTOR_STORE_COLLECTION = 'test_collection'; + + const config = getVectorStoreConfigFromEnv(); + + expect(config.type).toBe('in-memory'); + expect(config.collectionName).toBe('test_collection'); + }); + + it('should handle invalid numeric values gracefully', () => { + process.env.VECTOR_STORE_TYPE = 'in-memory'; + process.env.VECTOR_STORE_DIMENSION = 'invalid-number'; + process.env.VECTOR_STORE_MAX_VECTORS = 'also-invalid'; + + const config = getVectorStoreConfigFromEnv(); + + expect(config.type).toBe('in-memory'); + expect(config.dimension).toBe(1536); // Should fallback to default + expect((config as any).maxVectors).toBe(10000); // Should fallback to default + }); + }); + describe('isVectorStoreFactory', () => { it('should return true for valid VectorStoreFactory objects', async () => { const result = await createDefaultVectorStore(); diff --git a/src/core/vector_storage/__test__/qdrant.test.ts b/src/core/vector_storage/__test__/qdrant.test.ts index d32ddf70f..3209f6539 100644 --- a/src/core/vector_storage/__test__/qdrant.test.ts +++ b/src/core/vector_storage/__test__/qdrant.test.ts @@ -170,18 +170,17 @@ describe('QdrantBackend', () => { ]; const ids = ['vec1', 'vec2']; const payloads = [{ title: 'First' }, { title: 'Second' }]; - await backend.insert(vectors, ids, payloads); expect(mockQdrantClient.upsert).toHaveBeenCalledWith('test_collection', { points: [ { - id: 'vec1', + id: 1, vector: [1, 2, 3], payload: { title: 'First' }, }, { - id: 'vec2', + id: 2, vector: [4, 5, 6], payload: { title: 'Second' }, }, diff --git a/src/core/vector_storage/backend/qdrant.ts b/src/core/vector_storage/backend/qdrant.ts index 9e452ae11..2c628af7f 100644 --- a/src/core/vector_storage/backend/qdrant.ts +++ b/src/core/vector_storage/backend/qdrant.ts @@ -198,26 +198,16 @@ export class QdrantBackend implements VectorStore { try { const points = vectors.map((vector, idx) => { - const id = ids[idx]; const payload = payloads[idx]; - - if (!id || !payload) { - throw new VectorStoreError( - `Invalid input at index ${idx}: id and payload are required`, - 'insert' - ); - } - + if (!payload) throw new VectorStoreError(`Payload missing at index ${idx}`, 'insert'); return { - id: id, - vector: vector, - payload: payload, + id: idx + 1, + vector, + payload, }; }); - await this.client.upsert(this.collectionName, { - points, - }); + await this.client.upsert(this.collectionName, { points }); this.logger.info(`${LOG_PREFIXES.INDEX} Successfully inserted ${vectors.length} vectors`); } catch (error) { @@ -521,4 +511,4 @@ export class QdrantBackend implements VectorStore { getCollectionName(): string { return this.collectionName; } -} +} \ No newline at end of file diff --git a/src/core/vector_storage/factory.ts b/src/core/vector_storage/factory.ts index 195a29e82..90f329afd 100644 --- a/src/core/vector_storage/factory.ts +++ b/src/core/vector_storage/factory.ts @@ -8,10 +8,15 @@ */ import { VectorStoreManager } from './manager.js'; -import type { VectorStoreConfig, VectorStore } from './types.js'; +import type { VectorStoreConfig } from './types.js'; +import { VectorStore } from './backend/vector-store.js'; +import type { BackendConfig, QdrantBackendConfig } from './config.js'; +import { InMemoryBackend } from './backend/in-memory.js'; +import { QdrantBackend } from './backend/qdrant.js'; import { Logger, createLogger } from '../logger/index.js'; import { LOG_PREFIXES } from './constants.js'; import { env } from '../env.js'; +import * as fs from 'node:fs'; /** * Factory result containing both the manager and vector store @@ -148,6 +153,7 @@ export async function createDefaultVectorStore( * - VECTOR_STORE_DIMENSION: Vector dimension * - VECTOR_STORE_DISTANCE: Distance metric for Qdrant * - VECTOR_STORE_ON_DISK: Store vectors on disk (if using Qdrant) + * - VECTOR_STORE_MAX_VECTORS: Maximum vectors for in-memory storage * * @returns Promise resolving to manager and connected vector store * @@ -164,33 +170,54 @@ export async function createDefaultVectorStore( export async function createVectorStoreFromEnv(): Promise { const logger = createLogger({ level: env.CIPHER_LOG_LEVEL }); - // Get configuration from environment - const storeType = process.env.VECTOR_STORE_TYPE || 'in-memory'; - const collectionName = process.env.VECTOR_STORE_COLLECTION || 'default'; - const dimensionStr = process.env.VECTOR_STORE_DIMENSION || '1536'; - const dimension = Number.isNaN(parseInt(dimensionStr, 10)) ? 1536 : parseInt(dimensionStr, 10); + // Get configuration from environment variables + const config = getVectorStoreConfigFromEnv(); logger.info(`${LOG_PREFIXES.FACTORY} Creating vector storage from environment`, { - type: storeType, - collection: collectionName, - dimension, + type: config.type, + collection: config.collectionName, + dimension: config.dimension, }); + return createVectorStore(config); +} + +/** + * Get vector storage configuration from environment variables + * + * Returns the configuration object that would be used by createVectorStoreFromEnv + * without actually creating the vector store. Useful for debugging and validation. + * + * @returns Vector storage configuration based on environment variables + * + * @example + * ```typescript + * const config = getVectorStoreConfigFromEnv(); + * console.log('Vector store configuration:', config); + * + * // Then use the config to create the store + * const { manager, store } = await createVectorStore(config); + * ``` + */ +export function getVectorStoreConfigFromEnv(): VectorStoreConfig { + // Get configuration from centralized env object with fallbacks for invalid values + const storeType = env.VECTOR_STORE_TYPE; + const collectionName = env.VECTOR_STORE_COLLECTION; + const dimension = Number.isNaN(env.VECTOR_STORE_DIMENSION) ? 1536 : env.VECTOR_STORE_DIMENSION; + const maxVectors = Number.isNaN(env.VECTOR_STORE_MAX_VECTORS) + ? 10000 + : env.VECTOR_STORE_MAX_VECTORS; + // Build configuration based on type let config: VectorStoreConfig; if (storeType === 'qdrant') { - const host = process.env.VECTOR_STORE_HOST; - const url = process.env.VECTOR_STORE_URL; - const portStr = process.env.VECTOR_STORE_PORT; - const port = portStr - ? Number.isNaN(parseInt(portStr, 10)) - ? undefined - : parseInt(portStr, 10) - : undefined; - const apiKey = process.env.VECTOR_STORE_API_KEY; - const distance = process.env.VECTOR_STORE_DISTANCE as any; - const onDisk = process.env.VECTOR_STORE_ON_DISK === 'true'; + const host = env.VECTOR_STORE_HOST; + const url = env.VECTOR_STORE_URL; + const port = Number.isNaN(env.VECTOR_STORE_PORT) ? undefined : env.VECTOR_STORE_PORT; + const apiKey = env.VECTOR_STORE_API_KEY; + const distance = env.VECTOR_STORE_DISTANCE; + const onDisk = env.VECTOR_STORE_ON_DISK; config = { type: 'qdrant', @@ -204,25 +231,17 @@ export async function createVectorStoreFromEnv(): Promise { onDisk, }; - // Validate required fields + // Validate required fields and fallback if necessary if (!url && !host) { - logger.warn(`${LOG_PREFIXES.FACTORY} Qdrant requires URL or host, falling back to in-memory`); config = { type: 'in-memory', collectionName, dimension, - maxVectors: 10000, + maxVectors, }; } } else { // Use in-memory - const maxVectorsStr = process.env.VECTOR_STORE_MAX_VECTORS; - const maxVectors = maxVectorsStr - ? Number.isNaN(parseInt(maxVectorsStr, 10)) - ? 10000 - : parseInt(maxVectorsStr, 10) - : 10000; - config = { type: 'in-memory', collectionName, @@ -231,7 +250,7 @@ export async function createVectorStoreFromEnv(): Promise { }; } - return createVectorStore(config); + return config; } /** @@ -249,3 +268,56 @@ export function isVectorStoreFactory(obj: unknown): obj is VectorStoreFactory { obj.manager instanceof VectorStoreManager ); } + +/** + * Get Qdrant configuration from environment variables + * Supports both cloud and local configurations + */ +export function getQdrantConfigFromEnv(): QdrantBackendConfig | null { + const qdrantUrl = process.env.VECTOR_STORE_URL; + const qdrantApiKey = process.env.VECTOR_STORE_API_KEY; + const qdrantHost = process.env.VECTOR_STORE_HOST; + const qdrantPort = process.env.VECTOR_STORE_PORT; + + // Always resolve collectionName to a string + const collectionName = + process.env.VECTOR_STORE_COLLECTION_NAME || process.env.VECTOR_STORE_COLLECTION || 'default'; + + // Check if we have cloud configuration + if (qdrantUrl) { + return { + type: 'qdrant', + url: qdrantUrl, + apiKey: qdrantApiKey, // API key is required for cloud + collectionName, + dimension: parseInt(process.env.VECTOR_STORE_DIMENSION || '1536', 10), + distance: (process.env.VECTOR_STORE_DISTANCE as any) || 'Cosine', + }; + } + + // Check if we have local configuration + if (qdrantHost || qdrantPort) { + return { + type: 'qdrant', + host: qdrantHost || 'localhost', + port: qdrantPort ? parseInt(qdrantPort, 10) : 6333, + apiKey: qdrantApiKey, // Optional for local + collectionName, + dimension: parseInt(process.env.VECTOR_STORE_DIMENSION || '1536', 10), + distance: (process.env.VECTOR_STORE_DISTANCE as any) || 'Cosine', + }; + } + + return null; +} + +/** + * Check if Qdrant configuration is available in environment + */ +export function isQdrantConfigAvailable(): boolean { + return !!( + process.env.VECTOR_STORE_URL || + process.env.VECTOR_STORE_HOST || + process.env.VECTOR_STORE_PORT + ); +} diff --git a/src/core/vector_storage/index.ts b/src/core/vector_storage/index.ts index ea2701b50..621b671fa 100644 --- a/src/core/vector_storage/index.ts +++ b/src/core/vector_storage/index.ts @@ -63,6 +63,7 @@ export { createVectorStore, createDefaultVectorStore, createVectorStoreFromEnv, + getVectorStoreConfigFromEnv, isVectorStoreFactory, type VectorStoreFactory, } from './factory.js'; diff --git a/src/core/vector_storage/manager.ts b/src/core/vector_storage/manager.ts index 4260b64f5..3ed4ccb47 100644 --- a/src/core/vector_storage/manager.ts +++ b/src/core/vector_storage/manager.ts @@ -88,6 +88,9 @@ export class VectorStoreManager { private static qdrantModule?: any; private static inMemoryModule?: any; + // In VectorStoreManager, track if in-memory is used as fallback or primary + private usedFallback = false; + /** * Creates a new VectorStoreManager instance * @@ -137,7 +140,7 @@ export class VectorStoreManager { backend: { type: this.backendMetadata.type, connected: this.store?.isConnected() ?? false, - fallback: this.backendMetadata.isFallback, + fallback: this.usedFallback, collectionName: this.config.collectionName, dimension: this.config.dimension, }, @@ -175,6 +178,7 @@ export class VectorStoreManager { * @throws {VectorStoreConnectionError} If backend fails to connect */ public async connect(): Promise { + this.usedFallback = false; // Always reset at the start of each connection attempt // Check if already connected if (this.connected && this.store) { this.logger.debug(`${LOG_PREFIXES.MANAGER} Already connected`, { @@ -196,6 +200,7 @@ export class VectorStoreManager { this.store = await this.createBackend(); await this.store.connect(); this.backendMetadata.connectionTime = Date.now() - startTime; + this.usedFallback = false; // Not a fallback if primary backend succeeded this.logger.info(`${LOG_PREFIXES.MANAGER} Connected successfully`, { type: this.backendMetadata.type, @@ -221,12 +226,15 @@ export class VectorStoreManager { this.backendMetadata.type = BACKEND_TYPES.IN_MEMORY; this.backendMetadata.isFallback = true; this.backendMetadata.connectionTime = Date.now() - startTime; + this.usedFallback = true; // Mark as fallback this.logger.info(`${LOG_PREFIXES.MANAGER} Connected to fallback backend`, { type: this.backendMetadata.type, originalType: this.config.type, }); } else { + // In-memory is primary, not a fallback + this.usedFallback = false; throw backendError; // Re-throw if already using in-memory } } @@ -255,6 +263,7 @@ export class VectorStoreManager { // Reset state this.store = undefined; this.connected = false; + this.usedFallback = false; throw error; } @@ -289,6 +298,7 @@ export class VectorStoreManager { // Always clean up state this.store = undefined; this.connected = false; + this.usedFallback = false; // Reset metadata this.backendMetadata = {