From bc8df57ff0731dced940bd49530cc45b9f27acab Mon Sep 17 00:00:00 2001 From: andy Date: Thu, 3 Jul 2025 11:49:54 +0700 Subject: [PATCH 1/5] add memory operation tool --- env.example | 155 +++ src/core/brain/embedding/backend/index.ts | 31 + src/core/brain/embedding/backend/openai.ts | 427 +++++++ src/core/brain/embedding/backend/types.ts | 213 ++++ src/core/brain/embedding/config.ts | 327 +++++ src/core/brain/embedding/constants.ts | 178 +++ src/core/brain/embedding/factory.ts | 357 ++++++ src/core/brain/embedding/index.ts | 113 ++ src/core/brain/embedding/manager.ts | 505 ++++++++ src/core/brain/embedding/types.ts | 92 ++ src/core/brain/embedding/utils.ts | 230 ++++ src/core/brain/index.ts | 2 +- .../brain/tools/definitions/memory/index.ts | 16 +- .../definitions/memory/memory_operation.ts | 1064 +++++++++++++++++ src/core/brain/tools/manager.ts | 64 +- src/core/brain/tools/registry.ts | 2 +- src/core/brain/tools/types.ts | 87 +- src/core/env.ts | 35 + src/core/utils/service-initializer.ts | 89 +- 19 files changed, 3966 insertions(+), 21 deletions(-) create mode 100644 env.example create mode 100644 src/core/brain/embedding/backend/index.ts create mode 100644 src/core/brain/embedding/backend/openai.ts create mode 100644 src/core/brain/embedding/backend/types.ts create mode 100644 src/core/brain/embedding/config.ts create mode 100644 src/core/brain/embedding/constants.ts create mode 100644 src/core/brain/embedding/factory.ts create mode 100644 src/core/brain/embedding/index.ts create mode 100644 src/core/brain/embedding/manager.ts create mode 100644 src/core/brain/embedding/types.ts create mode 100644 src/core/brain/embedding/utils.ts create mode 100644 src/core/brain/tools/definitions/memory/memory_operation.ts diff --git a/env.example b/env.example new file mode 100644 index 000000000..fd38848b4 --- /dev/null +++ b/env.example @@ -0,0 +1,155 @@ +# ============================================================================= +# CIPHER PROJECT ENVIRONMENT CONFIGURATION +# ============================================================================= +# This file contains all available environment variables for the Cipher project. +# Copy this file to .env and configure the values according to your setup. + +# ============================================================================= +# GENERAL CONFIGURATION +# ============================================================================= + +# Environment mode: development, production, or test +NODE_ENV=development + +# Logging level: debug, info, warn, error +CIPHER_LOG_LEVEL=info + +# Whether to redact secrets in logs (true/false) +REDACT_SECRETS=true + +# ============================================================================= +# LLM PROVIDER CONFIGURATION +# ============================================================================= + +# OpenAI Configuration +# Your OpenAI API key (REQUIRED for embedding functionality, even when using other LLM providers) +# This is needed because embeddings currently only support OpenAI +OPENAI_API_KEY=sk-your-openai-api-key + +# OpenAI Organization ID (optional) +OPENAI_ORG_ID=org-your-organization-id + +# OpenAI API Base URL (optional, defaults to https://api.openai.com/v1) +OPENAI_BASE_URL=https://api.openai.com/v1 + +# Anthropic Configuration +# Your Anthropic API key (required for Anthropic/Claude services) +# Note: You still need OPENAI_API_KEY above for embedding functionality +ANTHROPIC_API_KEY=sk-ant-your-anthropic-api-key-here + +# OpenRouter Configuration +# Your OpenRouter API key (required for OpenRouter services) +# Note: You still need OPENAI_API_KEY above for embedding functionality +OPENROUTER_API_KEY=sk-or-your-openrouter-api-key-here + +# ============================================================================= +# EMBEDDING CONFIGURATION +# ============================================================================= +# IMPORTANT: Embeddings currently only support OpenAI. +# OPENAI_API_KEY is REQUIRED above, regardless of which LLM provider you use. + +# Embedding model to use (optional) +# Supported OpenAI models: text-embedding-3-small, text-embedding-3-large, text-embedding-ada-002 +# Default: text-embedding-3-small +EMBEDDING_MODEL=text-embedding-3-small + +# Request timeout for embedding operations in milliseconds (optional) +# Default: 30000 (30 seconds) +EMBEDDING_TIMEOUT=30000 + +# Maximum number of retry attempts for failed embedding requests (optional) +# Default: 3 +EMBEDDING_MAX_RETRIES=3 + +# ============================================================================= +# STORAGE CONFIGURATION +# ============================================================================= + +# Cache Storage Configuration +# Storage type for caching: redis, in-memory +STORAGE_CACHE_TYPE=in-memory + +# Redis configuration (only used if STORAGE_CACHE_TYPE=redis) +STORAGE_CACHE_HOST=localhost +STORAGE_CACHE_PORT=6379 +STORAGE_CACHE_PASSWORD=your-redis-password +STORAGE_CACHE_DATABASE=0 + +# Database Storage Configuration +# Database type: sqlite, in-memory +STORAGE_DATABASE_TYPE=in-memory + +# SQLite configuration (only used if STORAGE_DATABASE_TYPE=sqlite) +STORAGE_DATABASE_PATH=./data/cipher.db +STORAGE_DATABASE_NAME=cipher + +# ============================================================================= +# VECTOR STORAGE CONFIGURATION +# ============================================================================= + +# Vector store type: qdrant, in-memory +VECTOR_STORE_TYPE=in-memory + +# Qdrant configuration (only used if VECTOR_STORE_TYPE=qdrant) +VECTOR_STORE_HOST=localhost +VECTOR_STORE_PORT=6333 +VECTOR_STORE_URL=http://localhost:6333 +VECTOR_STORE_API_KEY=your-qdrant-api-key + +# Vector collection settings +VECTOR_STORE_COLLECTION=default +VECTOR_STORE_DIMENSION=1536 + +# Distance metric: Cosine, Euclidean, Dot, Manhattan +VECTOR_STORE_DISTANCE=Cosine + +# Whether to store vectors on disk (true/false) +VECTOR_STORE_ON_DISK=false + +# Maximum number of vectors for in-memory storage +VECTOR_STORE_MAX_VECTORS=10000 + +# ============================================================================= +# DEVELOPMENT NOTES +# ============================================================================= +# +# Required Variables: +# - OPENAI_API_KEY (ALWAYS required for embedding functionality) +# - At least one LLM provider API key for chat/completion services: +# * OPENAI_API_KEY (can serve both LLM and embedding needs) +# * ANTHROPIC_API_KEY (for Claude, but still need OPENAI_API_KEY for embeddings) +# * OPENROUTER_API_KEY (for OpenRouter, but still need OPENAI_API_KEY for embeddings) +# +# Optional Variables: +# - All other variables have sensible defaults and can be omitted +# +# Common Configurations: +# +# 1. OpenAI for both LLM and embeddings (minimal setup): +# NODE_ENV=development +# CIPHER_LOG_LEVEL=debug +# OPENAI_API_KEY=sk-your-openai-key-here +# +# 2. Anthropic for LLM + OpenAI for embeddings: +# NODE_ENV=development +# OPENAI_API_KEY=sk-your-openai-key-here +# ANTHROPIC_API_KEY=sk-ant-your-anthropic-key-here +# +# 3. Production with Qdrant and Anthropic: +# NODE_ENV=production +# CIPHER_LOG_LEVEL=info +# OPENAI_API_KEY=sk-your-openai-key-here +# ANTHROPIC_API_KEY=sk-ant-your-anthropic-key-here +# VECTOR_STORE_TYPE=qdrant +# VECTOR_STORE_URL=https://your-qdrant-instance.com +# VECTOR_STORE_API_KEY=your-qdrant-key +# +# 4. Development with Redis cache and mixed providers: +# NODE_ENV=development +# OPENAI_API_KEY=sk-your-openai-key-here +# ANTHROPIC_API_KEY=sk-ant-your-anthropic-key-here +# STORAGE_CACHE_TYPE=redis +# STORAGE_CACHE_HOST=localhost +# STORAGE_CACHE_PORT=6379 +# +# ============================================================================= \ No newline at end of file diff --git a/src/core/brain/embedding/backend/index.ts b/src/core/brain/embedding/backend/index.ts new file mode 100644 index 000000000..ca0debf40 --- /dev/null +++ b/src/core/brain/embedding/backend/index.ts @@ -0,0 +1,31 @@ +/** + * Embedding Backend Module Exports + * + * Central export point for all embedding backend implementations and types. + * Provides a clean interface for accessing embedding providers and utilities. + * + * @module embedding/backend + */ + +// Export core types and interfaces +export type { + Embedder, + EmbeddingConfig, + OpenAIEmbeddingConfig, + BackendConfig, + EmbeddingResult, + BatchEmbeddingResult, +} from './types.js'; + +// Export error classes +export { + EmbeddingError, + EmbeddingConnectionError, + EmbeddingDimensionError, + EmbeddingRateLimitError, + EmbeddingQuotaError, + EmbeddingValidationError, +} from './types.js'; + +// Export backend implementations +export { OpenAIEmbedder } from './openai.js'; \ No newline at end of file diff --git a/src/core/brain/embedding/backend/openai.ts b/src/core/brain/embedding/backend/openai.ts new file mode 100644 index 000000000..73edd1132 --- /dev/null +++ b/src/core/brain/embedding/backend/openai.ts @@ -0,0 +1,427 @@ +/** + * OpenAI Embedding Backend + * + * Implementation of the Embedder interface for OpenAI's embedding services. + * Supports all OpenAI embedding models with batch processing, retry logic, + * and comprehensive error handling. + * + * @module embedding/backend/openai + */ + +import OpenAI from 'openai'; +import { logger } from '../../../logger/index.js'; +import { + Embedder, + OpenAIEmbeddingConfig, + EmbeddingConnectionError, + EmbeddingRateLimitError, + EmbeddingQuotaError, + EmbeddingValidationError, + EmbeddingError, + EmbeddingDimensionError +} from './types.js'; +import { + MODEL_DIMENSIONS, + VALIDATION_LIMITS, + ERROR_MESSAGES, + LOG_PREFIXES, + RETRY_CONFIG, + HTTP_STATUS +} from '../constants.js'; + +/** + * OpenAI Embedder Implementation + * + * Provides embedding functionality using OpenAI's embedding API. + * Implements comprehensive error handling, retry logic, and batch processing. + */ +export class OpenAIEmbedder implements Embedder { + private openai: OpenAI; + private readonly config: OpenAIEmbeddingConfig; + private readonly model: string; + private readonly dimension: number; + + constructor(config: OpenAIEmbeddingConfig) { + this.config = config; + this.model = config.model || 'text-embedding-3-small'; + + // Initialize OpenAI client + this.openai = new OpenAI({ + apiKey: config.apiKey, + baseURL: config.baseUrl, + organization: config.organization, + timeout: config.timeout, + maxRetries: config.maxRetries, + }); + + // Set dimension based on model and config + this.dimension = config.dimensions || MODEL_DIMENSIONS[this.model]; + + logger.debug(`${LOG_PREFIXES.OPENAI} Initialized OpenAI embedder`, { + model: this.model, + dimension: this.dimension, + baseUrl: config.baseUrl, + hasOrganization: !!config.organization, + }); + } + + async embed(text: string): Promise { + logger.silly(`${LOG_PREFIXES.OPENAI} Embedding single text`, { + textLength: text.length, + model: this.model, + }); + + // Validate input + this.validateInput(text); + + const startTime = Date.now(); + + try { + const response = await this.createEmbeddingWithRetry({ + model: this.model, + input: text, + dimensions: this.config.dimensions, + }); + + const embedding = response.data[0].embedding; + const processingTime = Date.now() - startTime; + + // Validate embedding dimension + this.validateEmbeddingDimension(embedding); + + logger.debug(`${LOG_PREFIXES.OPENAI} Successfully created embedding`, { + model: this.model, + dimension: embedding.length, + processingTime, + usage: response.usage, + }); + + return embedding; + } catch (error) { + const processingTime = Date.now() - startTime; + logger.error(`${LOG_PREFIXES.OPENAI} Failed to create embedding`, { + error: error instanceof Error ? error.message : String(error), + model: this.model, + processingTime, + textLength: text.length, + }); + + throw this.handleApiError(error); + } + } + + async embedBatch(texts: string[]): Promise { + logger.debug(`${LOG_PREFIXES.BATCH} Embedding batch of texts`, { + count: texts.length, + model: this.model, + }); + + // Validate batch input + this.validateBatchInput(texts); + + const startTime = Date.now(); + + try { + const response = await this.createEmbeddingWithRetry({ + model: this.model, + input: texts, + dimensions: this.config.dimensions, + }); + + const embeddings = response.data.map((item) => item.embedding); + const processingTime = Date.now() - startTime; + + // Validate all embedding dimensions + embeddings.forEach((embedding, index) => { + try { + this.validateEmbeddingDimension(embedding); + } catch (error) { + logger.error(`${LOG_PREFIXES.BATCH} Invalid embedding dimension at index ${index}`, { + index, + dimension: embedding.length, + expected: this.dimension, + }); + throw error; + } + }); + + logger.debug(`${LOG_PREFIXES.BATCH} Successfully created batch embeddings`, { + model: this.model, + count: embeddings.length, + dimension: embeddings[0]?.length, + processingTime, + usage: response.usage, + }); + + return embeddings; + } catch (error) { + const processingTime = Date.now() - startTime; + logger.error(`${LOG_PREFIXES.BATCH} Failed to create batch embeddings`, { + error: error instanceof Error ? error.message : String(error), + model: this.model, + processingTime, + count: texts.length, + }); + + throw this.handleApiError(error); + } + } + + getDimension(): number { + return this.dimension; + } + + getConfig(): OpenAIEmbeddingConfig { + return { ...this.config }; + } + + async isHealthy(): Promise { + try { + logger.silly(`${LOG_PREFIXES.HEALTH} Checking OpenAI embedder health`); + + // Try a simple embedding request with minimal text + await this.embed('test'); + + logger.debug(`${LOG_PREFIXES.HEALTH} OpenAI embedder is healthy`); + return true; + } catch (error) { + logger.warn(`${LOG_PREFIXES.HEALTH} OpenAI embedder health check failed`, { + error: error instanceof Error ? error.message : String(error), + }); + return false; + } + } + + async disconnect(): Promise { + logger.debug(`${LOG_PREFIXES.OPENAI} Disconnecting OpenAI embedder`); + // OpenAI client doesn't require explicit cleanup + // This is here for interface compliance and future extensibility + } + + /** + * Create embedding with retry logic + */ + private async createEmbeddingWithRetry(params: { + model: string; + input: string | string[]; + dimensions?: number; + }): Promise { + let lastError: Error | undefined; + let delay: number = RETRY_CONFIG.INITIAL_DELAY; + + for (let attempt = 0; attempt <= this.config.maxRetries!; attempt++) { + try { + if (attempt > 0) { + logger.debug(`${LOG_PREFIXES.OPENAI} Retrying embedding request`, { + attempt, + delay, + maxRetries: this.config.maxRetries, + }); + + // Wait before retry + await new Promise(resolve => setTimeout(resolve, delay)); + + // Calculate next delay with exponential backoff and jitter + delay = Math.min( + delay * RETRY_CONFIG.BACKOFF_MULTIPLIER, + RETRY_CONFIG.MAX_DELAY + ); + + // Add jitter to avoid thundering herd + const jitter = delay * RETRY_CONFIG.JITTER_FACTOR * Math.random(); + delay = Math.floor(delay + jitter); + } + + const response = await this.openai.embeddings.create(params); + + if (attempt > 0) { + logger.info(`${LOG_PREFIXES.OPENAI} Embedding request succeeded after retry`, { + attempt, + model: params.model, + }); + } + + return response; + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); + + // Check if we should retry based on error type + if (!this.shouldRetry(error, attempt)) { + break; + } + + logger.warn(`${LOG_PREFIXES.OPENAI} Embedding request failed, will retry`, { + attempt: attempt + 1, + maxRetries: this.config.maxRetries, + error: lastError.message, + nextDelay: delay, + }); + } + } + + // All retries exhausted + throw lastError || new EmbeddingError('Unknown error during embedding request', 'openai'); + } + + /** + * Determine if an error is retryable + */ + private shouldRetry(error: unknown, attempt: number): boolean { + if (attempt >= this.config.maxRetries!) { + return false; + } + + // Handle OpenAI API errors + if (error && typeof error === 'object' && 'status' in error) { + const status = (error as any).status; + + // Retry on server errors and rate limits + return [ + HTTP_STATUS.TOO_MANY_REQUESTS, + HTTP_STATUS.INTERNAL_SERVER_ERROR, + HTTP_STATUS.SERVICE_UNAVAILABLE, + ].includes(status); + } + + // Retry on network errors + if (error instanceof Error) { + const message = error.message.toLowerCase(); + return ( + message.includes('network') || + message.includes('timeout') || + message.includes('connection') || + message.includes('econnreset') || + message.includes('enotfound') + ); + } + + return false; + } + + /** + * Handle and categorize API errors + */ + private handleApiError(error: unknown): EmbeddingError { + if (error && typeof error === 'object' && 'status' in error) { + const apiError = error as any; + const status = apiError.status; + const message = apiError.message || String(error); + + switch (status) { + case HTTP_STATUS.UNAUTHORIZED: + return new EmbeddingConnectionError( + ERROR_MESSAGES.INVALID_API_KEY('OpenAI'), + 'openai', + apiError + ); + + case HTTP_STATUS.TOO_MANY_REQUESTS: { + const retryAfter = apiError.headers?.['retry-after']; + return new EmbeddingRateLimitError( + ERROR_MESSAGES.RATE_LIMIT_EXCEEDED, + retryAfter ? parseInt(retryAfter, 10) : undefined, + 'openai', + apiError + ); + } + + case HTTP_STATUS.FORBIDDEN: + return new EmbeddingQuotaError( + ERROR_MESSAGES.QUOTA_EXCEEDED, + 'openai', + apiError + ); + + case HTTP_STATUS.BAD_REQUEST: + return new EmbeddingValidationError( + message, + 'openai', + apiError + ); + + default: + return new EmbeddingConnectionError( + ERROR_MESSAGES.CONNECTION_FAILED('OpenAI'), + 'openai', + apiError + ); + } + } + + // Handle network and other errors + if (error instanceof Error) { + return new EmbeddingConnectionError( + error.message, + 'openai', + error + ); + } + + return new EmbeddingError( + String(error), + 'openai' + ); + } + + /** + * Validate single text input + */ + private validateInput(text: string): void { + if (!text || text.length < VALIDATION_LIMITS.MIN_TEXT_LENGTH) { + throw new EmbeddingValidationError(ERROR_MESSAGES.EMPTY_TEXT, 'openai'); + } + + if (text.length > VALIDATION_LIMITS.MAX_TEXT_LENGTH) { + throw new EmbeddingValidationError( + ERROR_MESSAGES.TEXT_TOO_LONG(text.length, VALIDATION_LIMITS.MAX_TEXT_LENGTH), + 'openai' + ); + } + } + + /** + * Validate batch input + */ + private validateBatchInput(texts: string[]): void { + if (!Array.isArray(texts) || texts.length === 0) { + throw new EmbeddingValidationError('Batch input must be a non-empty array', 'openai'); + } + + if (texts.length > VALIDATION_LIMITS.MAX_BATCH_SIZE) { + throw new EmbeddingValidationError( + ERROR_MESSAGES.BATCH_TOO_LARGE(texts.length, VALIDATION_LIMITS.MAX_BATCH_SIZE), + 'openai' + ); + } + + // Validate each text in the batch + texts.forEach((text, index) => { + try { + this.validateInput(text); + } catch (error) { + if (error instanceof EmbeddingValidationError) { + throw new EmbeddingValidationError( + `Batch item ${index}: ${error.message}`, + 'openai', + error + ); + } + throw error; + } + }); + } + + /** + * Validate embedding dimension + */ + private validateEmbeddingDimension(embedding: number[]): void { + if (embedding.length !== this.dimension) { + throw new EmbeddingDimensionError( + ERROR_MESSAGES.DIMENSION_MISMATCH(this.dimension, embedding.length), + this.dimension, + embedding.length, + 'openai' + ); + } + } +} \ No newline at end of file diff --git a/src/core/brain/embedding/backend/types.ts b/src/core/brain/embedding/backend/types.ts new file mode 100644 index 000000000..29bffb8d7 --- /dev/null +++ b/src/core/brain/embedding/backend/types.ts @@ -0,0 +1,213 @@ +/** + * Embedding Backend Types and Interfaces + * + * Core type definitions for the embedding system backends. + * Provides the fundamental interfaces that all embedding providers must implement. + * + * @module embedding/backend/types + */ + +/** + * Core interface for embedding providers + * + * All embedding backends must implement this interface to provide + * consistent embedding functionality across different providers. + */ +export interface Embedder { + /** + * Generate embedding for a single text input + * + * @param text - The text to embed + * @returns Promise resolving to the embedding vector + */ + embed(text: string): Promise; + + /** + * Generate embeddings for multiple text inputs in batch + * + * @param texts - Array of texts to embed + * @returns Promise resolving to array of embedding vectors + */ + embedBatch(texts: string[]): Promise; + + /** + * Get the dimension of embeddings produced by this embedder + * + * @returns The vector dimension + */ + getDimension(): number; + + /** + * Get the configuration used by this embedder + * + * @returns The embedder configuration + */ + getConfig(): EmbeddingConfig; + + /** + * Check if the embedder is healthy and can process requests + * + * @returns Promise resolving to health status + */ + isHealthy(): Promise; + + /** + * Clean up resources and close connections + */ + disconnect(): Promise; +} + +/** + * Base configuration interface for all embedding providers + */ +export interface EmbeddingConfig { + /** The embedding provider type */ + type: string; + + /** API key for the provider */ + apiKey?: string; + + /** Model name to use for embeddings */ + model?: string; + + /** Base URL for the provider API */ + baseUrl?: string; + + /** Request timeout in milliseconds */ + timeout?: number; + + /** Maximum number of retry attempts */ + maxRetries?: number; + + /** Provider-specific options */ + options?: Record; +} + +/** + * OpenAI-specific embedding configuration + */ +export interface OpenAIEmbeddingConfig extends EmbeddingConfig { + type: 'openai'; + model?: 'text-embedding-3-small' | 'text-embedding-3-large' | 'text-embedding-ada-002'; + /** Organization ID for OpenAI API */ + organization?: string; + /** Custom dimensions for embedding-3 models */ + dimensions?: number; +} + +/** + * Union type for all supported backend configurations + */ +export type BackendConfig = OpenAIEmbeddingConfig; + +/** + * Result from embedding operation with metadata + */ +export interface EmbeddingResult { + /** The embedding vector */ + embedding: number[]; + /** Metadata about the embedding operation */ + metadata: { + /** Model used for embedding */ + model: string; + /** Token count for the input text */ + tokens?: number; + /** Processing time in milliseconds */ + processingTime?: number; + }; +} + +/** + * Batch embedding result + */ +export interface BatchEmbeddingResult { + /** Array of embedding vectors */ + embeddings: number[][]; + /** Metadata about the batch operation */ + metadata: { + /** Model used for embedding */ + model: string; + /** Total token count for all inputs */ + totalTokens?: number; + /** Processing time in milliseconds */ + processingTime?: number; + /** Number of successful embeddings */ + successCount: number; + /** Number of failed embeddings */ + failureCount: number; + }; +} + +/** + * Base error class for embedding operations + */ +export class EmbeddingError extends Error { + constructor( + message: string, + public readonly provider?: string, + public readonly cause?: Error + ) { + super(message); + this.name = 'EmbeddingError'; + } +} + +/** + * Error thrown when connection to embedding provider fails + */ +export class EmbeddingConnectionError extends EmbeddingError { + constructor(message: string, provider?: string, cause?: Error) { + super(message, provider, cause); + this.name = 'EmbeddingConnectionError'; + } +} + +/** + * Error thrown when embedding dimensions don't match expected values + */ +export class EmbeddingDimensionError extends EmbeddingError { + constructor( + message: string, + public readonly expected: number, + public readonly actual: number, + provider?: string + ) { + super(message, provider); + this.name = 'EmbeddingDimensionError'; + } +} + +/** + * Error thrown when API rate limits are exceeded + */ +export class EmbeddingRateLimitError extends EmbeddingError { + constructor( + message: string, + public readonly retryAfter?: number, + provider?: string, + cause?: Error + ) { + super(message, provider, cause); + this.name = 'EmbeddingRateLimitError'; + } +} + +/** + * Error thrown when API quota is exceeded + */ +export class EmbeddingQuotaError extends EmbeddingError { + constructor(message: string, provider?: string, cause?: Error) { + super(message, provider, cause); + this.name = 'EmbeddingQuotaError'; + } +} + +/** + * Error thrown when input validation fails + */ +export class EmbeddingValidationError extends EmbeddingError { + constructor(message: string, provider?: string, cause?: Error) { + super(message, provider, cause); + this.name = 'EmbeddingValidationError'; + } +} \ No newline at end of file diff --git a/src/core/brain/embedding/config.ts b/src/core/brain/embedding/config.ts new file mode 100644 index 000000000..85a23d65a --- /dev/null +++ b/src/core/brain/embedding/config.ts @@ -0,0 +1,327 @@ +/** + * Embedding Configuration Module + * + * Defines the configuration schemas for the embedding system using Zod for + * runtime validation and type safety. Supports multiple embedding providers + * with different configuration requirements. + * + * @module embedding/config + */ + +import { z } from 'zod'; +import { + DEFAULTS, + OPENAI_MODELS, + PROVIDER_TYPES, + VALIDATION_LIMITS, + ENV_VARS +} from './constants.js'; + +/** + * Base Embedding Configuration Schema + * + * Common configuration options shared by all embedding providers. + * These options control model selection, timeouts, and retry behavior. + */ +const BaseEmbeddingSchema = z.object({ + /** API key for the provider (required for all providers) */ + apiKey: z + .string() + .min(1) + .describe('API key for the embedding provider'), + + /** Model name to use for embeddings */ + model: z + .string() + .min(1) + .optional() + .describe('Model name for embeddings'), + + /** Base URL for the provider API */ + baseUrl: z + .string() + .url() + .optional() + .describe('Base URL for the provider API'), + + /** Request timeout in milliseconds */ + timeout: z + .number() + .int() + .positive() + .max(300000) // 5 minutes max + .default(DEFAULTS.TIMEOUT) + .describe('Request timeout in milliseconds'), + + /** Maximum number of retry attempts */ + maxRetries: z + .number() + .int() + .min(0) + .max(10) + .default(DEFAULTS.MAX_RETRIES) + .describe('Maximum retry attempts'), + + /** Provider-specific options */ + options: z + .record(z.any()) + .optional() + .describe('Provider-specific configuration options'), +}); + +/** + * OpenAI Embedding Configuration Schema + * + * Configuration specific to OpenAI embedding services. + * Supports all OpenAI embedding models with validation. + * + * @example + * ```typescript + * const config: OpenAIEmbeddingConfig = { + * type: 'openai', + * apiKey: process.env.OPENAI_API_KEY, + * model: 'text-embedding-3-small' + * }; + * ``` + */ +const OpenAIEmbeddingSchema = BaseEmbeddingSchema.extend({ + type: z.literal(PROVIDER_TYPES.OPENAI), + + /** OpenAI embedding model */ + model: z + .enum([ + OPENAI_MODELS.TEXT_EMBEDDING_3_SMALL, + OPENAI_MODELS.TEXT_EMBEDDING_3_LARGE, + OPENAI_MODELS.TEXT_EMBEDDING_ADA_002, + ] as const) + .default(DEFAULTS.OPENAI_MODEL) + .describe('OpenAI embedding model'), + + /** OpenAI organization ID */ + organization: z + .string() + .optional() + .describe('OpenAI organization ID'), + + /** Custom dimensions for embedding-3 models */ + dimensions: z + .number() + .int() + .positive() + .max(3072) + .optional() + .describe('Custom embedding dimensions (embedding-3 models only)'), + + /** Base URL override */ + baseUrl: z + .string() + .url() + .default(DEFAULTS.OPENAI_BASE_URL) + .describe('OpenAI API base URL'), +}).strict(); + +export type OpenAIEmbeddingConfig = z.infer; + +/** + * Backend Configuration Union Schema + * + * Discriminated union of all supported embedding provider configurations. + * Uses the 'type' field to determine which configuration schema to apply. + */ +const BackendConfigSchema = z + .discriminatedUnion('type', [OpenAIEmbeddingSchema], { + errorMap: (issue, ctx) => { + if (issue.code === z.ZodIssueCode.invalid_union_discriminator) { + return { + message: `Invalid embedding provider type. Expected: ${Object.values(PROVIDER_TYPES).join(', ')}.`, + }; + } + return { message: ctx.defaultError }; + }, + }) + .describe('Backend configuration for embedding system') + .superRefine((data, ctx) => { + // Validate OpenAI-specific requirements + if (data.type === PROVIDER_TYPES.OPENAI) { + // Check if dimensions are specified for models that support it + if (data.dimensions) { + const supportsCustomDimensions = + data.model === OPENAI_MODELS.TEXT_EMBEDDING_3_SMALL || + data.model === OPENAI_MODELS.TEXT_EMBEDDING_3_LARGE; + + if (!supportsCustomDimensions) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Custom dimensions are only supported for embedding-3 models`, + path: ['dimensions'], + }); + } + + // Validate dimension range for specific models + if (data.model === OPENAI_MODELS.TEXT_EMBEDDING_3_SMALL && data.dimensions > 1536) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `text-embedding-3-small supports max 1536 dimensions`, + path: ['dimensions'], + }); + } + } + } + + // Validate API key format (basic checks) + if (data.apiKey) { + if (data.type === PROVIDER_TYPES.OPENAI && !data.apiKey.startsWith('sk-')) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'OpenAI API key should start with "sk-"', + path: ['apiKey'], + }); + } + } + }); + +export type BackendConfig = z.infer; + +/** + * Embedding System Configuration Schema + * + * Top-level configuration for the embedding system. + * Uses a single backend configuration. + */ +export const EmbeddingConfigSchema = BackendConfigSchema; + +export type EmbeddingConfig = z.infer; + +/** + * Environment-based Configuration Schema + * + * Allows configuration to be loaded from environment variables. + * Useful for deployment scenarios where config is provided via env vars. + */ +export const EmbeddingEnvConfigSchema = z.object({ + /** Provider type from environment */ + type: z + .enum([PROVIDER_TYPES.OPENAI] as const) + .default(PROVIDER_TYPES.OPENAI) + .describe('Embedding provider type'), + + /** API key from environment variables */ + apiKey: z + .string() + .optional() + .describe('API key from environment variables'), + + /** Model from environment */ + model: z + .string() + .optional() + .describe('Model name from environment'), + + /** Base URL from environment */ + baseUrl: z + .string() + .url() + .optional() + .describe('Base URL from environment'), + + /** Timeout from environment */ + timeout: z + .string() + .transform((val) => parseInt(val, 10)) + .pipe(z.number().int().positive()) + .optional() + .describe('Timeout from environment (string converted to number)'), + + /** Max retries from environment */ + maxRetries: z + .string() + .transform((val) => parseInt(val, 10)) + .pipe(z.number().int().min(0).max(10)) + .optional() + .describe('Max retries from environment (string converted to number)'), +}); + +export type EmbeddingEnvConfig = z.infer; + +/** + * Parse and validate embedding configuration + * + * @param config - Raw configuration object + * @returns Validated configuration + * @throws {z.ZodError} If configuration is invalid + */ +export function parseEmbeddingConfig(config: unknown): EmbeddingConfig { + return EmbeddingConfigSchema.parse(config); +} + +/** + * Parse embedding configuration from environment variables + * + * @param env - Environment variables object (defaults to process.env) + * @returns Validated configuration or null if required env vars are missing + */ +export function parseEmbeddingConfigFromEnv( + env: Record = process.env +): EmbeddingConfig | null { + try { + // Try to build config from environment variables + const rawConfig: any = { + type: env[ENV_VARS.EMBEDDING_MODEL] ? PROVIDER_TYPES.OPENAI : PROVIDER_TYPES.OPENAI, + }; + + // Add OpenAI-specific config + if (env[ENV_VARS.OPENAI_API_KEY]) { + rawConfig.apiKey = env[ENV_VARS.OPENAI_API_KEY]; + } else { + // No API key found, cannot create config + return null; + } + + if (env[ENV_VARS.EMBEDDING_MODEL]) { + rawConfig.model = env[ENV_VARS.EMBEDDING_MODEL]; + } + + if (env[ENV_VARS.OPENAI_BASE_URL]) { + rawConfig.baseUrl = env[ENV_VARS.OPENAI_BASE_URL]; + } + + if (env[ENV_VARS.OPENAI_ORG_ID]) { + rawConfig.organization = env[ENV_VARS.OPENAI_ORG_ID]; + } + + if (env[ENV_VARS.EMBEDDING_TIMEOUT]) { + rawConfig.timeout = parseInt(env[ENV_VARS.EMBEDDING_TIMEOUT], 10); + } + + if (env[ENV_VARS.EMBEDDING_MAX_RETRIES]) { + rawConfig.maxRetries = parseInt(env[ENV_VARS.EMBEDDING_MAX_RETRIES], 10); + } + + return parseEmbeddingConfig(rawConfig); + } catch (error) { + // Configuration parsing failed + return null; + } +} + +/** + * Validate embedding configuration without throwing + * + * @param config - Raw configuration object + * @returns Validation result with success flag and data/errors + */ +export function validateEmbeddingConfig(config: unknown): { + success: boolean; + data?: EmbeddingConfig; + errors?: z.ZodError; +} { + try { + const data = parseEmbeddingConfig(config); + return { success: true, data }; + } catch (error) { + if (error instanceof z.ZodError) { + return { success: false, errors: error }; + } + throw error; + } +} \ No newline at end of file diff --git a/src/core/brain/embedding/constants.ts b/src/core/brain/embedding/constants.ts new file mode 100644 index 000000000..2069d389b --- /dev/null +++ b/src/core/brain/embedding/constants.ts @@ -0,0 +1,178 @@ +/** + * Embedding System Constants + * + * Centralized constants for the embedding system including defaults, + * supported models, timeouts, and other configuration values. + * + * @module embedding/constants + */ + +/** + * Supported embedding provider types + */ +export const PROVIDER_TYPES = { + OPENAI: 'openai', +} as const; + +/** + * OpenAI embedding models with their specifications + */ +export const OPENAI_MODELS = { + /** Latest small embedding model (1536 dimensions) */ + TEXT_EMBEDDING_3_SMALL: 'text-embedding-3-small', + /** Latest large embedding model (3072 dimensions) */ + TEXT_EMBEDDING_3_LARGE: 'text-embedding-3-large', + /** Legacy Ada v2 model (1536 dimensions) */ + TEXT_EMBEDDING_ADA_002: 'text-embedding-ada-002', +} as const; + +/** + * Model dimension specifications + */ +export const MODEL_DIMENSIONS = { + [OPENAI_MODELS.TEXT_EMBEDDING_3_SMALL]: 1536, + [OPENAI_MODELS.TEXT_EMBEDDING_3_LARGE]: 3072, + [OPENAI_MODELS.TEXT_EMBEDDING_ADA_002]: 1536, +} as const; + +/** + * Maximum input limits for different models + */ +export const MODEL_INPUT_LIMITS = { + [OPENAI_MODELS.TEXT_EMBEDDING_3_SMALL]: 8191, // tokens + [OPENAI_MODELS.TEXT_EMBEDDING_3_LARGE]: 8191, // tokens + [OPENAI_MODELS.TEXT_EMBEDDING_ADA_002]: 8191, // tokens +} as const; + +/** + * Default configuration values + */ +export const DEFAULTS = { + /** Default OpenAI model */ + OPENAI_MODEL: OPENAI_MODELS.TEXT_EMBEDDING_3_SMALL, + + /** Default request timeout in milliseconds */ + TIMEOUT: 30000, // 30 seconds + + /** Default maximum retry attempts */ + MAX_RETRIES: 3, + + /** Default batch size for batch operations */ + BATCH_SIZE: 100, + + /** Default OpenAI API base URL */ + OPENAI_BASE_URL: 'https://api.openai.com/v1', + + /** Default embedding dimension */ + DIMENSION: MODEL_DIMENSIONS[OPENAI_MODELS.TEXT_EMBEDDING_3_SMALL], +} as const; + +/** + * Rate limiting and retry configuration + */ +export const RETRY_CONFIG = { + /** Initial retry delay in milliseconds */ + INITIAL_DELAY: 1000, + + /** Maximum retry delay in milliseconds */ + MAX_DELAY: 60000, + + /** Backoff multiplier for exponential backoff */ + BACKOFF_MULTIPLIER: 2, + + /** Jitter factor for randomizing retry delays */ + JITTER_FACTOR: 0.1, +} as const; + +/** + * Validation limits + */ +export const VALIDATION_LIMITS = { + /** Maximum text length for single embedding */ + MAX_TEXT_LENGTH: 32768, // characters + + /** Maximum number of texts in batch operation */ + MAX_BATCH_SIZE: 2048, + + /** Minimum text length (empty strings not allowed) */ + MIN_TEXT_LENGTH: 1, +} as const; + +/** + * Error messages + */ +export const ERROR_MESSAGES = { + PROVIDER_NOT_SUPPORTED: (provider: string) => + `Embedding provider '${provider}' is not supported`, + + MODEL_NOT_SUPPORTED: (model: string, provider: string) => + `Model '${model}' is not supported by provider '${provider}'`, + + API_KEY_REQUIRED: (provider: string) => `API key is required for provider '${provider}'`, + + CONNECTION_FAILED: (provider: string) => `Failed to connect to ${provider} embedding service`, + + TEXT_TOO_LONG: (length: number, max: number) => + `Text length ${length} exceeds maximum of ${max} characters`, + + BATCH_TOO_LARGE: (size: number, max: number) => + `Batch size ${size} exceeds maximum of ${max} items`, + + EMPTY_TEXT: 'Text cannot be empty', + + DIMENSION_MISMATCH: (expected: number, actual: number) => + `Expected embedding dimension ${expected}, but got ${actual}`, + + RATE_LIMIT_EXCEEDED: 'Rate limit exceeded for embedding provider', + + QUOTA_EXCEEDED: 'API quota exceeded for embedding provider', + + INVALID_API_KEY: (provider: string) => `Invalid API key for provider '${provider}'`, + + REQUEST_TIMEOUT: (timeout: number) => `Request timed out after ${timeout}ms`, +} as const; + +/** + * Log prefixes for different operations + */ +export const LOG_PREFIXES = { + EMBEDDING: '[EMBEDDING]', + OPENAI: '[EMBEDDING:OPENAI]', + FACTORY: '[EMBEDDING:FACTORY]', + MANAGER: '[EMBEDDING:MANAGER]', + HEALTH: '[EMBEDDING:HEALTH]', + BATCH: '[EMBEDDING:BATCH]', +} as const; + +/** + * Environment variable names + */ +export const ENV_VARS = { + OPENAI_API_KEY: 'OPENAI_API_KEY', + OPENAI_ORG_ID: 'OPENAI_ORG_ID', + OPENAI_BASE_URL: 'OPENAI_BASE_URL', + EMBEDDING_MODEL: 'EMBEDDING_MODEL', + EMBEDDING_TIMEOUT: 'EMBEDDING_TIMEOUT', + EMBEDDING_MAX_RETRIES: 'EMBEDDING_MAX_RETRIES', +} as const; + +/** + * HTTP status codes for API responses + */ +export const HTTP_STATUS = { + OK: 200, + BAD_REQUEST: 400, + UNAUTHORIZED: 401, + FORBIDDEN: 403, + NOT_FOUND: 404, + TOO_MANY_REQUESTS: 429, + INTERNAL_SERVER_ERROR: 500, + SERVICE_UNAVAILABLE: 503, +} as const; + +/** + * Content types for API requests + */ +export const CONTENT_TYPES = { + JSON: 'application/json', +} as const; \ No newline at end of file diff --git a/src/core/brain/embedding/factory.ts b/src/core/brain/embedding/factory.ts new file mode 100644 index 000000000..a43b76682 --- /dev/null +++ b/src/core/brain/embedding/factory.ts @@ -0,0 +1,357 @@ +/** + * Embedding Factory Module + * + * Factory functions for creating embedding instances with proper validation, + * error handling, and type safety. Supports multiple providers and + * configuration methods. + * + * @module embedding/factory + */ + +import { logger } from '../../logger/index.js'; +import { + parseEmbeddingConfigFromEnv, + validateEmbeddingConfig, + type OpenAIEmbeddingConfig as ZodOpenAIEmbeddingConfig, + type EmbeddingConfig as ZodEmbeddingConfig +} from './config.js'; +import { + type Embedder, + type OpenAIEmbeddingConfig as InterfaceOpenAIEmbeddingConfig, + EmbeddingError, + EmbeddingValidationError, + OpenAIEmbedder +} from './backend/index.js'; +import { + PROVIDER_TYPES, + ERROR_MESSAGES, + LOG_PREFIXES, + DEFAULTS +} from './constants.js'; + +// Use Zod-inferred types for validation, but convert to interface types for backend +export type BackendConfig = ZodOpenAIEmbeddingConfig; + +/** + * Embedding factory interface + * + * Defines the contract for embedding factory implementations. + * Each provider should implement this interface. + */ +export interface EmbeddingFactory { + /** + * Create an embedder instance + * + * @param config - Provider-specific configuration + * @returns Promise resolving to embedder instance + */ + createEmbedder(config: BackendConfig): Promise; + + /** + * Validate configuration for this provider + * + * @param config - Configuration to validate + * @returns True if configuration is valid + */ + validateConfig(config: unknown): boolean; + + /** + * Get the provider type this factory supports + * + * @returns Provider type string + */ + getProviderType(): string; +} + +/** + * Convert Zod config to interface config for backend compatibility + */ +function convertToInterfaceConfig(config: BackendConfig): InterfaceOpenAIEmbeddingConfig { + return { + type: PROVIDER_TYPES.OPENAI, + apiKey: config.apiKey, + model: config.model, + baseUrl: config.baseUrl, + timeout: config.timeout, + maxRetries: config.maxRetries, + options: config.options, + organization: config.organization, + dimensions: config.dimensions, + } as InterfaceOpenAIEmbeddingConfig; +} + +/** + * OpenAI Embedding Factory + * + * Factory implementation for creating OpenAI embedding instances. + */ +class OpenAIEmbeddingFactory implements EmbeddingFactory { + async createEmbedder(config: BackendConfig): Promise { + logger.debug(`${LOG_PREFIXES.FACTORY} Creating OpenAI embedder`, { + model: config.model, + baseUrl: config.baseUrl, + hasOrganization: !!config.organization, + }); + + try { + // Convert Zod config to interface config for backend compatibility + const interfaceConfig = convertToInterfaceConfig(config); + const embedder = new OpenAIEmbedder(interfaceConfig); + + // Test the connection + const isHealthy = await embedder.isHealthy(); + if (!isHealthy) { + throw new EmbeddingError( + ERROR_MESSAGES.CONNECTION_FAILED('OpenAI'), + 'openai' + ); + } + + logger.info(`${LOG_PREFIXES.FACTORY} Successfully created OpenAI embedder`, { + model: config.model, + dimension: embedder.getDimension(), + }); + + return embedder; + } catch (error) { + logger.error(`${LOG_PREFIXES.FACTORY} Failed to create OpenAI embedder`, { + error: error instanceof Error ? error.message : String(error), + model: config.model, + }); + + if (error instanceof EmbeddingError) { + throw error; + } + + throw new EmbeddingError( + `Failed to create OpenAI embedder: ${error instanceof Error ? error.message : String(error)}`, + 'openai', + error instanceof Error ? error : undefined + ); + } + } + + validateConfig(config: unknown): boolean { + try { + const validationResult = validateEmbeddingConfig(config); + return validationResult.success && + validationResult.data?.type === PROVIDER_TYPES.OPENAI; + } catch { + return false; + } + } + + getProviderType(): string { + return PROVIDER_TYPES.OPENAI; + } +} + +/** + * Registry of embedding factories + */ +const EMBEDDING_FACTORIES = new Map([ + [PROVIDER_TYPES.OPENAI, new OpenAIEmbeddingFactory()], +]); + +/** + * Main factory function for creating embedding instances + * + * @param config - Embedding configuration + * @returns Promise resolving to embedder instance + * @throws {EmbeddingValidationError} If configuration is invalid + * @throws {EmbeddingError} If embedder creation fails + * + * @example + * ```typescript + * const embedder = await createEmbedder({ + * type: 'openai', + * apiKey: process.env.OPENAI_API_KEY, + * model: 'text-embedding-3-small' + * }); + * ``` + */ +export async function createEmbedder(config: BackendConfig): Promise { + logger.debug(`${LOG_PREFIXES.FACTORY} Creating embedder`, { + type: config.type, + }); + + // Validate configuration + const validationResult = validateEmbeddingConfig(config); + if (!validationResult.success) { + const errorMessage = validationResult.errors?.issues + .map(issue => `${issue.path.join('.')}: ${issue.message}`) + .join(', ') || 'Invalid configuration'; + + logger.error(`${LOG_PREFIXES.FACTORY} Configuration validation failed`, { + type: config.type, + errors: errorMessage, + }); + + throw new EmbeddingValidationError( + `Configuration validation failed: ${errorMessage}` + ); + } + + // Get factory for provider type + const factory = EMBEDDING_FACTORIES.get(config.type); + if (!factory) { + logger.error(`${LOG_PREFIXES.FACTORY} Unsupported provider type`, { + type: config.type, + supportedTypes: Array.from(EMBEDDING_FACTORIES.keys()), + }); + + throw new EmbeddingValidationError( + ERROR_MESSAGES.PROVIDER_NOT_SUPPORTED(config.type) + ); + } + + // Create embedder instance + return await factory.createEmbedder(config); +} + +/** + * Create OpenAI embedder with simplified configuration + * + * @param config - OpenAI-specific configuration + * @returns Promise resolving to OpenAI embedder instance + * + * @example + * ```typescript + * const embedder = await createOpenAIEmbedder({ + * apiKey: process.env.OPENAI_API_KEY, + * model: 'text-embedding-3-small' + * }); + * ``` + */ +export async function createOpenAIEmbedder( + config: Omit +): Promise { + const openaiConfig: ZodOpenAIEmbeddingConfig = { + type: PROVIDER_TYPES.OPENAI, + apiKey: config.apiKey || '', + model: config.model || DEFAULTS.OPENAI_MODEL, + baseUrl: config.baseUrl || DEFAULTS.OPENAI_BASE_URL, + timeout: config.timeout || DEFAULTS.TIMEOUT, + maxRetries: config.maxRetries || DEFAULTS.MAX_RETRIES, + ...(config.organization && { organization: config.organization }), + ...(config.dimensions && { dimensions: config.dimensions }), + ...(config.options && { options: config.options }), + }; + return createEmbedder(openaiConfig); +} + +/** + * Create embedder from environment variables + * + * @param env - Environment variables object (defaults to process.env) + * @returns Promise resolving to embedder instance or null if config unavailable + * + * @example + * ```typescript + * // Requires OPENAI_API_KEY environment variable + * const embedder = await createEmbedderFromEnv(); + * if (embedder) { + * const embedding = await embedder.embed('Hello world'); + * } + * ``` + */ +export async function createEmbedderFromEnv( + env: Record = process.env +): Promise { + logger.debug(`${LOG_PREFIXES.FACTORY} Creating embedder from environment variables`); + + const config = parseEmbeddingConfigFromEnv(env); + if (!config) { + logger.warn(`${LOG_PREFIXES.FACTORY} No valid embedding configuration found in environment`); + return null; + } + + logger.debug(`${LOG_PREFIXES.FACTORY} Found valid embedding configuration in environment`, { + type: config.type, + model: config.model, + }); + + return createEmbedder(config as ZodOpenAIEmbeddingConfig); +} + +/** + * Create a default embedder with minimal configuration + * + * Uses OpenAI provider with default settings. + * Requires OPENAI_API_KEY environment variable. + * + * @returns Promise resolving to default embedder instance + * @throws {EmbeddingValidationError} If API key is not available + * + * @example + * ```typescript + * // Requires OPENAI_API_KEY environment variable + * const embedder = await createDefaultEmbedder(); + * ``` + */ +export async function createDefaultEmbedder(): Promise { + logger.debug(`${LOG_PREFIXES.FACTORY} Creating default embedder`); + + const apiKey = process.env.OPENAI_API_KEY; + if (!apiKey) { + throw new EmbeddingValidationError( + ERROR_MESSAGES.API_KEY_REQUIRED('OpenAI') + ); + } + + const openaiConfig: ZodOpenAIEmbeddingConfig = { + type: PROVIDER_TYPES.OPENAI, + apiKey, + model: DEFAULTS.OPENAI_MODEL, + baseUrl: DEFAULTS.OPENAI_BASE_URL, + timeout: DEFAULTS.TIMEOUT, + maxRetries: DEFAULTS.MAX_RETRIES, + }; + return createEmbedder(openaiConfig); +} + +/** + * Check if a factory exists for the given provider type + * + * @param providerType - Provider type to check + * @returns True if factory exists + */ +export function isEmbeddingFactory(providerType: string): boolean { + return EMBEDDING_FACTORIES.has(providerType); +} + +/** + * Get all supported provider types + * + * @returns Array of supported provider type strings + */ +export function getSupportedProviders(): string[] { + return Array.from(EMBEDDING_FACTORIES.keys()); +} + +/** + * Get factory instance for a specific provider type + * + * @param providerType - Provider type + * @returns Factory instance or undefined if not found + */ +export function getEmbeddingFactory(providerType: string): EmbeddingFactory | undefined { + return EMBEDDING_FACTORIES.get(providerType); +} + +/** + * Register a new embedding factory + * + * @param providerType - Provider type + * @param factory - Factory instance + */ +export function registerEmbeddingFactory( + providerType: string, + factory: EmbeddingFactory +): void { + logger.debug(`${LOG_PREFIXES.FACTORY} Registering embedding factory`, { + providerType, + }); + + EMBEDDING_FACTORIES.set(providerType, factory); +} \ No newline at end of file diff --git a/src/core/brain/embedding/index.ts b/src/core/brain/embedding/index.ts new file mode 100644 index 000000000..3ae5eb8d4 --- /dev/null +++ b/src/core/brain/embedding/index.ts @@ -0,0 +1,113 @@ +/** + * Embedding Module + * + * High-performance text embedding system supporting multiple providers. + * Provides a unified API for generating embeddings with comprehensive + * error handling, retry logic, and lifecycle management. + * + * Features: + * - Multiple provider support (OpenAI, future: Anthropic, Cohere, etc.) + * - Batch operations for efficient processing + * - Type-safe configuration with runtime validation + * - Comprehensive error handling and retry logic + * - Health monitoring and statistics collection + * - Graceful fallback and connection management + * + * @module embedding + * + * @example + * ```typescript + * import { createEmbedder, EmbeddingManager } from './embedding'; + * + * // Create a single embedder + * const embedder = await createEmbedder({ + * type: 'openai', + * apiKey: process.env.OPENAI_API_KEY, + * model: 'text-embedding-3-small' + * }); + * + * // Generate embeddings + * const embedding = await embedder.embed('Hello world'); + * const embeddings = await embedder.embedBatch(['Hello', 'World']); + * + * // Use embedding manager for multiple embedders + * const manager = new EmbeddingManager(); + * const { embedder: managedEmbedder } = await manager.createEmbedder({ + * type: 'openai', + * apiKey: process.env.OPENAI_API_KEY + * }); + * + * // Health monitoring + * manager.startHealthChecks(); + * const healthResults = await manager.checkAllHealth(); + * + * // Cleanup + * await manager.disconnect(); + * ``` + */ + +// Export types +export type { + Embedder, + EmbeddingConfig, + OpenAIEmbeddingConfig, + BackendConfig, + EmbeddingResult, + BatchEmbeddingResult, + EmbeddingFactory, + HealthCheckResult, + EmbedderInfo, + EmbeddingStats, + EmbeddingEnvConfig, +} from './types.js'; + +// Export error classes +export { + EmbeddingError, + EmbeddingConnectionError, + EmbeddingDimensionError, + EmbeddingRateLimitError, + EmbeddingQuotaError, + EmbeddingValidationError, +} from './types.js'; + +// Export factory functions +export { + createEmbedder, + createOpenAIEmbedder, + createEmbedderFromEnv, + createDefaultEmbedder, + isEmbeddingFactory, + getSupportedProviders, + getEmbeddingFactory, + registerEmbeddingFactory, +} from './factory.js'; + +// Export manager +export { EmbeddingManager } from './manager.js'; + +// Export configuration utilities +export { + parseEmbeddingConfig, + parseEmbeddingConfigFromEnv, + validateEmbeddingConfig, + EmbeddingConfigSchema, +} from './config.js'; + +// Export constants for external use +export { + PROVIDER_TYPES, + OPENAI_MODELS, + MODEL_DIMENSIONS, + DEFAULTS, + VALIDATION_LIMITS +} from './constants.js'; + +// Export utilities +export { + getEmbeddingConfigFromEnv, + isEmbeddingConfigAvailable, + getEmbeddingConfigSummary, + validateEmbeddingEnv, + analyzeProviderConfiguration, +} from './utils.js'; \ No newline at end of file diff --git a/src/core/brain/embedding/manager.ts b/src/core/brain/embedding/manager.ts new file mode 100644 index 000000000..f42220fb8 --- /dev/null +++ b/src/core/brain/embedding/manager.ts @@ -0,0 +1,505 @@ +/** + * Embedding Manager Module + * + * Provides lifecycle management, health monitoring, and connection management + * for embedding services. Manages multiple embedder instances and provides + * centralized monitoring and cleanup capabilities. + * + * @module embedding/manager + */ + +import { logger } from '../../logger/index.js'; +import { + type Embedder, + type BackendConfig, + EmbeddingError, + EmbeddingConnectionError +} from './backend/index.js'; +import { createEmbedder, createEmbedderFromEnv } from './factory.js'; +import { LOG_PREFIXES, ERROR_MESSAGES } from './constants.js'; + +/** + * Health check result for an embedder instance + */ +export interface HealthCheckResult { + /** Whether the embedder is healthy */ + healthy: boolean; + /** Provider type */ + provider: string; + /** Model being used */ + model?: string; + /** Embedding dimension */ + dimension?: number; + /** Response time in milliseconds */ + responseTime?: number; + /** Error message if unhealthy */ + error?: string; + /** Timestamp of the health check */ + timestamp: Date; +} + +/** + * Information about an embedder instance + */ +export interface EmbedderInfo { + /** Unique identifier for this embedder */ + id: string; + /** Provider type */ + provider: string; + /** Model being used */ + model?: string; + /** Embedding dimension */ + dimension: number; + /** Configuration used */ + config: BackendConfig; + /** Creation timestamp */ + createdAt: Date; + /** Last health check result */ + lastHealthCheck?: HealthCheckResult; +} + +/** + * Statistics about embedding operations + */ +export interface EmbeddingStats { + /** Total number of single embed operations */ + totalEmbeds: number; + /** Total number of batch embed operations */ + totalBatchEmbeds: number; + /** Total number of texts processed */ + totalTexts: number; + /** Total processing time in milliseconds */ + totalProcessingTime: number; + /** Number of successful operations */ + successfulOperations: number; + /** Number of failed operations */ + failedOperations: number; + /** Average processing time per operation */ + averageProcessingTime: number; +} + +/** + * Embedding Manager + * + * Manages the lifecycle of embedding instances, providing centralized + * health monitoring, statistics collection, and resource cleanup. + */ +export class EmbeddingManager { + private embedders = new Map(); + private embedderInfo = new Map(); + private stats: EmbeddingStats = { + totalEmbeds: 0, + totalBatchEmbeds: 0, + totalTexts: 0, + totalProcessingTime: 0, + successfulOperations: 0, + failedOperations: 0, + averageProcessingTime: 0, + }; + private healthCheckInterval?: NodeJS.Timeout; + + constructor() { + logger.debug(`${LOG_PREFIXES.MANAGER} Embedding manager initialized`); + } + + /** + * Create and register an embedder instance + * + * @param config - Embedding configuration + * @param id - Optional custom ID for the embedder + * @returns Promise resolving to embedder instance and info + */ + async createEmbedder( + config: BackendConfig, + id?: string + ): Promise<{ embedder: Embedder; info: EmbedderInfo }> { + const embedderId = id || this.generateId(); + + logger.debug(`${LOG_PREFIXES.MANAGER} Creating embedder`, { + id: embedderId, + type: config.type, + }); + + try { + const embedder = await createEmbedder(config); + + const info: EmbedderInfo = { + id: embedderId, + provider: config.type, + model: config.model, + dimension: embedder.getDimension(), + config, + createdAt: new Date(), + }; + + this.embedders.set(embedderId, embedder); + this.embedderInfo.set(embedderId, info); + + logger.info(`${LOG_PREFIXES.MANAGER} Successfully created embedder`, { + id: embedderId, + provider: info.provider, + model: info.model, + dimension: info.dimension, + }); + + return { embedder, info }; + } catch (error) { + logger.error(`${LOG_PREFIXES.MANAGER} Failed to create embedder`, { + id: embedderId, + type: config.type, + error: error instanceof Error ? error.message : String(error), + }); + + throw error; + } + } + + /** + * Create embedder from environment variables + * + * @param id - Optional custom ID for the embedder + * @returns Promise resolving to embedder instance and info, or null + */ + async createEmbedderFromEnv( + id?: string + ): Promise<{ embedder: Embedder; info: EmbedderInfo } | null> { + logger.debug(`${LOG_PREFIXES.MANAGER} Creating embedder from environment`); + + const embedder = await createEmbedderFromEnv(); + if (!embedder) { + logger.warn(`${LOG_PREFIXES.MANAGER} No embedder configuration found in environment`); + return null; + } + + const embedderId = id || this.generateId(); + const config = embedder.getConfig() as BackendConfig; + + const info: EmbedderInfo = { + id: embedderId, + provider: config.type, + model: config.model, + dimension: embedder.getDimension(), + config, + createdAt: new Date(), + }; + + this.embedders.set(embedderId, embedder); + this.embedderInfo.set(embedderId, info); + + logger.info(`${LOG_PREFIXES.MANAGER} Successfully created embedder from environment`, { + id: embedderId, + provider: info.provider, + model: info.model, + }); + + return { embedder, info }; + } + + /** + * Get embedder instance by ID + * + * @param id - Embedder ID + * @returns Embedder instance or undefined + */ + getEmbedder(id: string): Embedder | undefined { + return this.embedders.get(id); + } + + /** + * Get embedder information by ID + * + * @param id - Embedder ID + * @returns Embedder information or undefined + */ + getEmbedderInfo(id: string): EmbedderInfo | undefined { + return this.embedderInfo.get(id); + } + + /** + * Get all registered embedders + * + * @returns Map of embedder ID to embedder instance + */ + getAllEmbedders(): Map { + return new Map(this.embedders); + } + + /** + * Get all embedder information + * + * @returns Map of embedder ID to embedder information + */ + getAllEmbedderInfo(): Map { + return new Map(this.embedderInfo); + } + + /** + * Remove and disconnect an embedder + * + * @param id - Embedder ID + * @returns Promise resolving to true if removed, false if not found + */ + async removeEmbedder(id: string): Promise { + const embedder = this.embedders.get(id); + if (!embedder) { + logger.warn(`${LOG_PREFIXES.MANAGER} Embedder not found for removal`, { id }); + return false; + } + + logger.debug(`${LOG_PREFIXES.MANAGER} Removing embedder`, { id }); + + try { + await embedder.disconnect(); + this.embedders.delete(id); + this.embedderInfo.delete(id); + + logger.info(`${LOG_PREFIXES.MANAGER} Successfully removed embedder`, { id }); + return true; + } catch (error) { + logger.error(`${LOG_PREFIXES.MANAGER} Error disconnecting embedder`, { + id, + error: error instanceof Error ? error.message : String(error), + }); + + // Remove anyway to prevent memory leaks + this.embedders.delete(id); + this.embedderInfo.delete(id); + return true; + } + } + + /** + * Perform health check on a specific embedder + * + * @param id - Embedder ID + * @returns Promise resolving to health check result + */ + async checkHealth(id: string): Promise { + const embedder = this.embedders.get(id); + const info = this.embedderInfo.get(id); + + if (!embedder || !info) { + logger.warn(`${LOG_PREFIXES.HEALTH} Embedder not found for health check`, { id }); + return null; + } + + logger.silly(`${LOG_PREFIXES.HEALTH} Performing health check`, { id }); + + const startTime = Date.now(); + try { + const healthy = await embedder.isHealthy(); + const responseTime = Date.now() - startTime; + + const result: HealthCheckResult = { + healthy, + provider: info.provider, + model: info.model, + dimension: info.dimension, + responseTime, + timestamp: new Date(), + }; + + // Update embedder info with latest health check + this.embedderInfo.set(id, { + ...info, + lastHealthCheck: result, + }); + + logger.debug(`${LOG_PREFIXES.HEALTH} Health check completed`, { + id, + healthy, + responseTime, + }); + + return result; + } catch (error) { + const responseTime = Date.now() - startTime; + const result: HealthCheckResult = { + healthy: false, + provider: info.provider, + model: info.model, + dimension: info.dimension, + responseTime, + error: error instanceof Error ? error.message : String(error), + timestamp: new Date(), + }; + + // Update embedder info with latest health check + this.embedderInfo.set(id, { + ...info, + lastHealthCheck: result, + }); + + logger.warn(`${LOG_PREFIXES.HEALTH} Health check failed`, { + id, + error: result.error, + responseTime, + }); + + return result; + } + } + + /** + * Perform health check on all embedders + * + * @returns Promise resolving to map of embedder ID to health check result + */ + async checkAllHealth(): Promise> { + logger.debug(`${LOG_PREFIXES.HEALTH} Performing health check on all embedders`); + + const results = new Map(); + const healthCheckPromises = Array.from(this.embedders.keys()).map(async (id) => { + const result = await this.checkHealth(id); + if (result) { + results.set(id, result); + } + }); + + await Promise.all(healthCheckPromises); + + logger.debug(`${LOG_PREFIXES.HEALTH} Completed health check on all embedders`, { + total: this.embedders.size, + checked: results.size, + }); + + return results; + } + + /** + * Start periodic health checks + * + * @param intervalMs - Health check interval in milliseconds (default: 5 minutes) + */ + startHealthChecks(intervalMs: number = 5 * 60 * 1000): void { + if (this.healthCheckInterval) { + logger.warn(`${LOG_PREFIXES.HEALTH} Health checks already running`); + return; + } + + logger.info(`${LOG_PREFIXES.HEALTH} Starting periodic health checks`, { + intervalMs, + }); + + this.healthCheckInterval = setInterval(async () => { + try { + await this.checkAllHealth(); + } catch (error) { + logger.error(`${LOG_PREFIXES.HEALTH} Error during periodic health check`, { + error: error instanceof Error ? error.message : String(error), + }); + } + }, intervalMs); + } + + /** + * Stop periodic health checks + */ + stopHealthChecks(): void { + if (this.healthCheckInterval) { + clearInterval(this.healthCheckInterval); + this.healthCheckInterval = undefined; + logger.info(`${LOG_PREFIXES.HEALTH} Stopped periodic health checks`); + } + } + + /** + * Get current statistics + * + * @returns Current embedding statistics + */ + getStats(): EmbeddingStats { + // Calculate average processing time + const avgTime = this.stats.totalProcessingTime > 0 && + this.stats.successfulOperations > 0 + ? this.stats.totalProcessingTime / this.stats.successfulOperations + : 0; + + return { + ...this.stats, + averageProcessingTime: avgTime, + }; + } + + /** + * Reset statistics + */ + resetStats(): void { + logger.debug(`${LOG_PREFIXES.MANAGER} Resetting embedding statistics`); + + this.stats = { + totalEmbeds: 0, + totalBatchEmbeds: 0, + totalTexts: 0, + totalProcessingTime: 0, + successfulOperations: 0, + failedOperations: 0, + averageProcessingTime: 0, + }; + } + + /** + * Update statistics (called internally after operations) + */ + private updateStats( + type: 'embed' | 'batch', + textCount: number, + processingTime: number, + success: boolean + ): void { + if (type === 'embed') { + this.stats.totalEmbeds++; + } else { + this.stats.totalBatchEmbeds++; + } + + this.stats.totalTexts += textCount; + + if (success) { + this.stats.successfulOperations++; + this.stats.totalProcessingTime += processingTime; + } else { + this.stats.failedOperations++; + } + } + + /** + * Disconnect all embedders and cleanup + */ + async disconnect(): Promise { + logger.info(`${LOG_PREFIXES.MANAGER} Disconnecting all embedders`); + + // Stop health checks + this.stopHealthChecks(); + + // Disconnect all embedders + const disconnectPromises = Array.from(this.embedders.entries()).map( + async ([id, embedder]) => { + try { + await embedder.disconnect(); + logger.debug(`${LOG_PREFIXES.MANAGER} Disconnected embedder`, { id }); + } catch (error) { + logger.warn(`${LOG_PREFIXES.MANAGER} Error disconnecting embedder`, { + id, + error: error instanceof Error ? error.message : String(error), + }); + } + } + ); + + await Promise.all(disconnectPromises); + + // Clear all maps + this.embedders.clear(); + this.embedderInfo.clear(); + + logger.info(`${LOG_PREFIXES.MANAGER} Successfully disconnected all embedders`); + } + + /** + * Generate a unique ID for embedders + */ + private generateId(): string { + return `embedder_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; + } +} \ No newline at end of file diff --git a/src/core/brain/embedding/types.ts b/src/core/brain/embedding/types.ts new file mode 100644 index 000000000..cdecff9d2 --- /dev/null +++ b/src/core/brain/embedding/types.ts @@ -0,0 +1,92 @@ +/** + * Embedding Module Public API Types + * + * This module re-exports all the necessary types and interfaces for the embedding system. + * It provides a simplified, clean API surface for consumers of the embedding module. + * + * The embedding system architecture: + * - Multiple provider support (OpenAI, future: Anthropic, Cohere, etc.) + * - Consistent API across different providers + * - Strong type safety with TypeScript and runtime validation with Zod + * - Factory pattern for creating embedders + * - Manager pattern for lifecycle management + * + * @module embedding/types + * + * @example + * ```typescript + * import type { Embedder, EmbeddingConfig } from './embedding/types.js'; + * + * // Configure embedder + * const config: EmbeddingConfig = { + * type: 'openai', + * apiKey: process.env.OPENAI_API_KEY, + * model: 'text-embedding-3-small' + * }; + * + * // Use embedder + * const embedder = await createEmbedder(config); + * const embedding = await embedder.embed('Hello world'); + * ``` + */ + +/** + * Re-export core embedding types + * + * These exports provide the complete type system needed to work with + * the embedding module without exposing internal implementation details. + */ +export type { + // Core interfaces + Embedder, // Interface for embedding providers + EmbeddingConfig, // Base configuration interface + OpenAIEmbeddingConfig, // OpenAI-specific configuration + BackendConfig, // Union type for all provider configurations + + // Result types + EmbeddingResult, // Single embedding result with metadata + BatchEmbeddingResult, // Batch embedding result with metadata +} from './backend/types.js'; + +/** + * Re-export error classes + * + * Comprehensive error hierarchy for embedding operations. + */ +export { + EmbeddingError, // Base error class + EmbeddingConnectionError, // Connection-related errors + EmbeddingDimensionError, // Dimension mismatch errors + EmbeddingRateLimitError, // Rate limiting errors + EmbeddingQuotaError, // Quota exceeded errors + EmbeddingValidationError, // Input validation errors +} from './backend/types.js'; + +/** + * Re-export factory types + * + * Types related to embedding factory functionality. + */ +export type { + EmbeddingFactory, // Factory interface for creating embedders +} from './factory.js'; + +/** + * Re-export manager types + * + * Types related to embedding lifecycle management. + */ +export type { + HealthCheckResult, // Health check result structure + EmbedderInfo, // Information about embedder instances + EmbeddingStats, // Statistics about embedding operations +} from './manager.js'; + +/** + * Re-export configuration types and utilities + * + * Configuration validation and parsing utilities. + */ +export type { + EmbeddingEnvConfig, // Environment-based configuration +} from './config.js'; \ No newline at end of file diff --git a/src/core/brain/embedding/utils.ts b/src/core/brain/embedding/utils.ts new file mode 100644 index 000000000..deae49aff --- /dev/null +++ b/src/core/brain/embedding/utils.ts @@ -0,0 +1,230 @@ +/** + * Embedding Utilities + * + * Utility functions for working with embedding configuration and + * environment variables. Provides convenient access to environment-based + * configuration for the embedding system. + * + * @module embedding/utils + */ + +import { env } from '../../env.js'; +import { type OpenAIEmbeddingConfig } from './backend/types.js'; +import { PROVIDER_TYPES, DEFAULTS } from './constants.js'; +import { parseEmbeddingConfigFromEnv } from './config.js'; + +/** + * Get embedding configuration from environment variables + * + * Uses the centralized env configuration to build embedding config. + * Falls back to sensible defaults when environment variables are not set. + * + * @returns OpenAI embedding configuration or null if no API key + * + * @example + * ```typescript + * const config = getEmbeddingConfigFromEnv(); + * if (config) { + * const embedder = await createEmbedder(config); + * } + * ``` + */ +export function getEmbeddingConfigFromEnv(): OpenAIEmbeddingConfig | null { + // Check if we have a required API key + if (!env.OPENAI_API_KEY) { + return null; + } + + const config: OpenAIEmbeddingConfig = { + type: PROVIDER_TYPES.OPENAI, + apiKey: env.OPENAI_API_KEY, + model: (env.EMBEDDING_MODEL as any) || DEFAULTS.OPENAI_MODEL, + baseUrl: env.OPENAI_BASE_URL || DEFAULTS.OPENAI_BASE_URL, + timeout: env.EMBEDDING_TIMEOUT || DEFAULTS.TIMEOUT, + maxRetries: env.EMBEDDING_MAX_RETRIES || DEFAULTS.MAX_RETRIES, + organization: env.OPENAI_ORG_ID, + }; + + return config; +} + +/** + * Check if embedding configuration is available in environment + * + * @returns True if embedding can be configured from environment variables + * + * @example + * ```typescript + * if (isEmbeddingConfigAvailable()) { + * const embedder = await createEmbedderFromEnv(); + * } else { + * console.log('Please set OPENAI_API_KEY environment variable'); + * } + * ``` + */ +export function isEmbeddingConfigAvailable(): boolean { + return !!env.OPENAI_API_KEY; +} + +/** + * Get embedding configuration summary for logging/debugging + * + * Returns safe configuration info without exposing sensitive data. + * + * @returns Configuration summary object + * + * @example + * ```typescript + * const summary = getEmbeddingConfigSummary(); + * console.log('Embedding config:', summary); + * // Output: { hasApiKey: true, model: 'text-embedding-3-small', ... } + * ``` + */ +export function getEmbeddingConfigSummary(): { + hasApiKey: boolean; + model?: string; + baseUrl?: string; + timeout?: number; + maxRetries?: number; + hasOrganization: boolean; +} { + return { + hasApiKey: !!env.OPENAI_API_KEY, + model: env.EMBEDDING_MODEL || DEFAULTS.OPENAI_MODEL, + baseUrl: env.OPENAI_BASE_URL || DEFAULTS.OPENAI_BASE_URL, + timeout: env.EMBEDDING_TIMEOUT || DEFAULTS.TIMEOUT, + maxRetries: env.EMBEDDING_MAX_RETRIES || DEFAULTS.MAX_RETRIES, + hasOrganization: !!env.OPENAI_ORG_ID, + }; +} + +/** + * Validate that required environment variables are set for embeddings + * + * @returns Validation result with details + * + * @example + * ```typescript + * const validation = validateEmbeddingEnv(); + * if (!validation.valid) { + * console.error('Embedding setup issues:', validation.issues); + * } + * ``` + */ +export function validateEmbeddingEnv(): { + valid: boolean; + issues: string[]; +} { + const issues: string[] = []; + + // Check required variables + if (!env.OPENAI_API_KEY) { + // Check if other LLM providers are configured + const hasOtherProviders = env.ANTHROPIC_API_KEY || env.OPENROUTER_API_KEY; + if (hasOtherProviders) { + issues.push('OPENAI_API_KEY is required for embedding functionality, even when using Anthropic or OpenRouter for LLM services'); + } else { + issues.push('OPENAI_API_KEY is required for embedding functionality'); + } + } + + // Check API key format + if (env.OPENAI_API_KEY && !env.OPENAI_API_KEY.startsWith('sk-')) { + issues.push('OPENAI_API_KEY should start with "sk-"'); + } + + // Check numeric values + if (env.EMBEDDING_TIMEOUT && env.EMBEDDING_TIMEOUT <= 0) { + issues.push('EMBEDDING_TIMEOUT must be a positive number'); + } + + if (env.EMBEDDING_MAX_RETRIES && env.EMBEDDING_MAX_RETRIES < 0) { + issues.push('EMBEDDING_MAX_RETRIES must be a non-negative number'); + } + + // Check URL format + if (env.OPENAI_BASE_URL) { + try { + new URL(env.OPENAI_BASE_URL); + } catch { + issues.push('OPENAI_BASE_URL must be a valid URL'); + } + } + + return { + valid: issues.length === 0, + issues, + }; +} + +/** + * Check for mixed provider configuration and provide helpful guidance + * + * @returns Configuration analysis with recommendations + * + * @example + * ```typescript + * const analysis = analyzeProviderConfiguration(); + * if (analysis.warnings.length > 0) { + * console.warn('Configuration warnings:', analysis.warnings); + * } + * ``` + */ +export function analyzeProviderConfiguration(): { + usingMixedProviders: boolean; + llmProvider: 'openai' | 'anthropic' | 'openrouter' | 'none' | 'multiple'; + embeddingProvider: 'openai' | 'none'; + warnings: string[]; + recommendations: string[]; +} { + const warnings: string[] = []; + const recommendations: string[] = []; + + // Detect which LLM providers are configured + const hasOpenAI = !!env.OPENAI_API_KEY; + const hasAnthropic = !!env.ANTHROPIC_API_KEY; + const hasOpenRouter = !!env.OPENROUTER_API_KEY; + + let llmProvider: 'openai' | 'anthropic' | 'openrouter' | 'none' | 'multiple'; + const configuredProviders = [ + hasOpenAI && 'openai', + hasAnthropic && 'anthropic', + hasOpenRouter && 'openrouter' + ].filter(Boolean); + + if (configuredProviders.length === 0) { + llmProvider = 'none'; + warnings.push('No LLM provider API keys configured'); + } else if (configuredProviders.length === 1) { + llmProvider = configuredProviders[0] as 'openai' | 'anthropic' | 'openrouter'; + } else { + llmProvider = 'multiple'; + recommendations.push('Multiple LLM provider API keys detected. The system will use the configured provider in your LLM service setup.'); + } + + // Embedding provider analysis + const embeddingProvider = hasOpenAI ? 'openai' : 'none'; + const usingMixedProviders = (hasAnthropic || hasOpenRouter) && hasOpenAI; + + // Generate specific warnings and recommendations + if (!hasOpenAI && (hasAnthropic || hasOpenRouter)) { + warnings.push('Embedding functionality will not work without OPENAI_API_KEY'); + recommendations.push('Add OPENAI_API_KEY to enable embedding features, even when using Anthropic or OpenRouter for LLM'); + } + + if (usingMixedProviders) { + recommendations.push('You are using a mixed provider setup (non-OpenAI for LLM + OpenAI for embeddings). This is a valid configuration.'); + } + + if (hasOpenAI && !hasAnthropic && !hasOpenRouter) { + recommendations.push('Using OpenAI for both LLM and embeddings - this is the simplest configuration.'); + } + + return { + usingMixedProviders, + llmProvider, + embeddingProvider, + warnings, + recommendations, + }; +} \ No newline at end of file diff --git a/src/core/brain/index.ts b/src/core/brain/index.ts index 20b146690..68b232b73 100644 --- a/src/core/brain/index.ts +++ b/src/core/brain/index.ts @@ -1,2 +1,2 @@ export * from './memAgent/index.js'; -export * from './llm/index.js'; +export * from './llm/index.js'; \ No newline at end of file diff --git a/src/core/brain/tools/definitions/memory/index.ts b/src/core/brain/tools/definitions/memory/index.ts index d37f4338d..9c7b8dbc3 100644 --- a/src/core/brain/tools/definitions/memory/index.ts +++ b/src/core/brain/tools/definitions/memory/index.ts @@ -2,11 +2,13 @@ * Memory Tools Module * * This module exports all memory-related internal tools for the Cipher agent. - * These tools handle fact extraction and knowledge processing. + * These tools handle fact extraction, knowledge processing, and memory operations. */ // Export all memory tools export { extractKnowledgeTool } from './extract-knowledge.js'; +// TODO: Re-enable when tests are updated to handle multiple tools +// export { memoryOperationTool } from './memory_operation.js'; // Export types for better developer experience import type { InternalTool } from '../../types.js'; @@ -20,15 +22,21 @@ export const memoryTools: InternalTool[] = [ // Load tools dynamically to avoid potential circular dependencies import('./extract-knowledge.js').then(module => memoryTools.push(module.extractKnowledgeTool)); +// TODO: Re-enable when tests are updated to handle multiple tools +// import('./memory_operation.js').then(module => memoryTools.push(module.memoryOperationTool)); /** * Get all memory tools as a map */ export async function getMemoryTools(): Promise> { const { extractKnowledgeTool } = await import('./extract-knowledge.js'); + // TODO: Re-enable when tests are updated to handle multiple tools + // const { memoryOperationTool } = await import('./memory_operation.js'); return { [extractKnowledgeTool.name]: extractKnowledgeTool, + // TODO: Re-enable when tests are updated to handle multiple tools + // [memoryOperationTool.name]: memoryOperationTool, }; } @@ -42,4 +50,10 @@ export const MEMORY_TOOL_INFO = { useCase: 'Use when you need to capture important technical information, code patterns, or implementation details for future reference', }, + memory_operation: { + category: 'memory', + purpose: 'Process extracted knowledge and determine memory operations (ADD, UPDATE, DELETE)', + useCase: + 'Use after extracting knowledge to intelligently manage memory by analyzing similarity with existing memories and making informed decisions about memory operations', + }, } as const; diff --git a/src/core/brain/tools/definitions/memory/memory_operation.ts b/src/core/brain/tools/definitions/memory/memory_operation.ts new file mode 100644 index 000000000..d94559b90 --- /dev/null +++ b/src/core/brain/tools/definitions/memory/memory_operation.ts @@ -0,0 +1,1064 @@ +/** + * Memory Operation Tool + * + * Processes extracted knowledge and determines memory operations (ADD, UPDATE, DELETE, NONE) + * by analyzing similarity with existing memories and using LLM-powered intelligent reasoning. + * This tool integrates with embedding, vector storage, and LLM systems for sophisticated + * memory management with contextual understanding. + */ + +import { InternalTool, InternalToolContext } from '../../types.js'; +import { logger } from '../../../../logger/index.js'; + +/** + * MEMORY OPERATIONAL TOOL + */ +export const MEMORY_OPERATION_TOOL = { + type: 'function', + function: { + name: 'memory_operation', + description: 'Process extracted knowledge and determine memory operations (ADD, UPDATE, DELETE, NONE) using LLM-powered intelligent reasoning and similarity analysis with existing memories.', + parameters: { + type: 'object', + properties: { + memory: { + type: 'array', + description: 'Updated memory entries with operations applied. Always preserve complete code blocks, command syntax, and implementation details within triple backticks.', + items: { + type: 'object', + properties: { + id: { + type: 'string', + description: 'Unique ID of the memory entry.', + }, + text: { + type: 'string', + description: 'Text of the memory entry including complete implementation code, command syntax, or technical details when present. Always preserve the complete pattern within triple backticks.', + }, + event: { + type: 'string', + enum: ['ADD', 'UPDATE', 'DELETE', 'NONE'], + description: 'Operation applied to the entry.', + }, + tags: { + type: 'array', + items: { type: 'string' }, + description: 'Keywords derived from the text (lowercase, singular nouns). Include technology-specific tags (e.g., \'react\', \'python\', \'docker\').', + }, + old_memory: { + type: 'string', + description: 'Previous text, included only for UPDATE events. Ensure code patterns are properly preserved in the updated text.', + }, + code_pattern: { + type: 'string', + description: 'Optional. The extracted code pattern or command syntax if present, exactly as it appeared in the original content.' + }, + confidence: { + type: 'number', + description: 'Confidence score for the operation decision (0.0 to 1.0).', + }, + reasoning: { + type: 'string', + description: 'Explanation for why this operation was chosen.', + } + }, + required: ['id', 'text', 'event', 'tags'], + additionalProperties: false, + }, + }, + }, + required: ['memory'], + additionalProperties: false, + }, + }, +}; + +/** + * Interface for memory operation arguments + */ +export interface MemoryOperationArgs { + extractedFacts: string[]; + existingMemories?: { + id: string; + text: string; + metadata?: Record; + }[]; + context?: { + sessionId?: string; + userId?: string; + projectId?: string; + conversationTopic?: string; + recentMessages?: string[]; + sessionMetadata?: Record; + }; + options?: { + similarityThreshold?: number; + maxSimilarResults?: number; + enableBatchProcessing?: boolean; + useLLMDecisions?: boolean; // Enable LLM decision making + confidenceThreshold?: number; // Minimum confidence for operations + enableDeleteOperations?: boolean; // Enable DELETE operations + }; +} + +/** + * Interface for memory action result following UPDATE_FACT_TOOL_MEMORY pattern + */ +export interface MemoryAction { + id: string; + text: string; + event: 'ADD' | 'UPDATE' | 'DELETE' | 'NONE'; + tags: string[]; + old_memory?: string; + code_pattern?: string; + confidence: number; // Confidence score + reasoning: string; // Decision reasoning +} + +/** + * Interface for memory operation result + */ +export interface MemoryOperationResult { + success: boolean; + totalFacts: number; + processedFacts: number; + skippedFacts: number; + memory: MemoryAction[]; + statistics: { + addOperations: number; + updateOperations: number; + deleteOperations: number; + noneOperations: number; + totalSimilarMemories: number; + averageConfidence: number; + llmDecisionsUsed: number; // Count of LLM-assisted decisions + fallbackDecisionsUsed: number; // Count of fallback decisions + }; + timestamp: string; + processingTime: number; + error?: string; +} + +/** + * Default configuration options + */ +const DEFAULT_OPTIONS = { + similarityThreshold: 0.7, + maxSimilarResults: 5, + enableBatchProcessing: true, + useLLMDecisions: true, // Enable LLM decisions by default + confidenceThreshold: 0.6, // Minimum confidence threshold + enableDeleteOperations: true, // Enable DELETE operations +} as const; + +/** + * Prompts for LLM decision making + */ +const MEMORY_OPERATION_PROMPTS = { + SYSTEM_PROMPT: `You are an intelligent memory management system. Your task is to analyze extracted knowledge facts and determine the best memory operations (ADD, UPDATE, DELETE, NONE) based on similarity with existing memories and contextual understanding. + +Consider these factors: +1. Content similarity and semantic overlap +2. Information recency and relevance +3. Knowledge quality and completeness +4. Conversation context and user needs +5. Technical accuracy and implementation details + +Rules: +- ADD: For new, unique knowledge that doesn't duplicate existing memories +- UPDATE: For enhanced or corrected versions of existing knowledge +- DELETE: For outdated, incorrect, or redundant information that should be removed +- NONE: For duplicates or information already well-represented + +Always preserve code blocks, commands, and technical patterns exactly as provided.`, + + DECISION_PROMPT: `Analyze this knowledge fact and determine the appropriate memory operation: + +KNOWLEDGE FACT: +{fact} + +SIMILAR EXISTING MEMORIES: +{similarMemories} + +CONVERSATION CONTEXT: +{context} + +For this knowledge fact, determine: +1. The most appropriate operation (ADD, UPDATE, DELETE, NONE) +2. Your confidence level (0.0 to 1.0) +3. Clear reasoning for your decision + +Focus on preserving valuable technical knowledge while removing outdated or redundant information. + +Respond with a JSON object containing: +{ + "operation": "ADD|UPDATE|DELETE|NONE", + "confidence": 0.8, + "reasoning": "Clear explanation of the decision", + "targetMemoryId": "id-if-updating-or-deleting" +}` +} as const; + +/** + * Memory operation tool for intelligent memory management + */ +export const memoryOperationTool: InternalTool = { + name: 'memory_operation', + category: 'memory', + internal: true, + description: + 'Process extracted knowledge and determine memory operations (ADD, UPDATE, DELETE, NONE) using LLM-powered intelligent reasoning and similarity analysis with existing memories.', + version: '2.0.0', // version + parameters: { + type: 'object', + properties: { + extractedFacts: { + type: 'array', + description: + 'Array of knowledge facts already extracted from interactions, containing technical details, code patterns, or implementation information.', + items: { + type: 'string', + }, + }, + existingMemories: { + type: 'array', + description: 'Array of existing memory entries to compare against for similarity analysis.', + items: { + type: 'object', + properties: { + id: { + type: 'string', + description: 'Unique identifier of the existing memory', + }, + text: { + type: 'string', + description: 'Content of the existing memory', + }, + metadata: { + type: 'object', + description: 'Optional metadata for the memory', + }, + }, + required: ['id', 'text'], + }, + }, + context: { + type: 'object', + description: 'Optional context information for memory operations', + properties: { + sessionId: { + type: 'string', + description: 'Current session identifier', + }, + userId: { + type: 'string', + description: 'User identifier for personalized memory', + }, + projectId: { + type: 'string', + description: 'Project identifier for scoped memory', + }, + conversationTopic: { + type: 'string', + description: 'Current conversation topic or theme', + }, + recentMessages: { + type: 'array', + items: { type: 'string' }, + description: 'Recent conversation messages for context', + }, + sessionMetadata: { + type: 'object', + description: 'Additional session metadata', + }, + }, + additionalProperties: false, + }, + options: { + type: 'object', + description: 'Configuration options for memory operations', + properties: { + similarityThreshold: { + type: 'number', + description: 'Similarity threshold for memory matching (0.0 to 1.0)', + minimum: 0.0, + maximum: 1.0, + }, + maxSimilarResults: { + type: 'number', + description: 'Maximum number of similar memories to retrieve', + minimum: 1, + maximum: 20, + }, + enableBatchProcessing: { + type: 'boolean', + description: 'Whether to process multiple knowledge items in batch', + }, + useLLMDecisions: { + type: 'boolean', + description: 'Whether to use LLM-powered decision making', + }, + confidenceThreshold: { + type: 'number', + description: 'Minimum confidence threshold for operations (0.0 to 1.0)', + minimum: 0.0, + maximum: 1.0, + }, + enableDeleteOperations: { + type: 'boolean', + description: 'Whether to enable DELETE operations', + }, + }, + additionalProperties: false, + }, + }, + required: ['extractedFacts'], + }, + handler: async (args: MemoryOperationArgs, context?: InternalToolContext): Promise => { + const startTime = Date.now(); + + try { + logger.info('MemoryOperation: Processing memory operation request', { + factCount: args.extractedFacts?.length || 0, + existingMemoryCount: args.existingMemories?.length || 0, + hasContext: !!args.context, + hasOptions: !!args.options + }); + + // Phase 1: Basic parameter validation + const validationResult = validateMemoryOperationArgs(args); + if (!validationResult.isValid) { + throw new Error(`Invalid arguments: ${validationResult.errors.join(', ')}`); + } + + // Merge with default options + const options = { ...DEFAULT_OPTIONS, ...args.options }; + + logger.debug('MemoryOperation: Using configuration options', { + similarityThreshold: options.similarityThreshold, + maxSimilarResults: options.maxSimilarResults, + enableBatchProcessing: options.enableBatchProcessing, + useLLMDecisions: options.useLLMDecisions, + confidenceThreshold: options.confidenceThreshold, + enableDeleteOperations: options.enableDeleteOperations, + }); + + // Filter valid facts + const validFacts = args.extractedFacts + .filter(fact => fact && typeof fact === 'string' && fact.trim().length > 0) + .map(fact => fact.trim()); + + if (validFacts.length === 0) { + throw new Error('No valid facts found after filtering'); + } + + // Phase 2: Get available services + const memoryActions: MemoryAction[] = []; + let totalSimilarMemories = 0; + let confidenceSum = 0; + let llmDecisionsUsed = 0; + let fallbackDecisionsUsed = 0; + + // Try to get services from context + const embeddingManager = context?.services?.embeddingManager; + const vectorStoreManager = context?.services?.vectorStoreManager; + const llmService = context?.services?.llmService; // LLM service access + + let embedder: any = null; + let vectorStore: any = null; + + // Initialize embedding and vector services + if (embeddingManager && vectorStoreManager) { + try { + embedder = embeddingManager.getEmbedder('default'); + vectorStore = vectorStoreManager.getStore(); + + if (embedder && vectorStore) { + logger.debug('MemoryOperation: Using embedding and vector storage services'); + } else { + logger.warn('MemoryOperation: Services available but not initialized, using basic analysis'); + } + } catch (error) { + logger.debug('MemoryOperation: Failed to access embedding/vector services', { + error: error instanceof Error ? error.message : String(error), + }); + } + } else { + logger.debug('MemoryOperation: No embedding/vector services available in context, using basic analysis'); + } + + // Check LLM service availability + if (options.useLLMDecisions && llmService) { + logger.debug('MemoryOperation: LLM service available for decision making'); + } else if (options.useLLMDecisions) { + logger.warn('MemoryOperation: LLM decisions requested but service not available, falling back to similarity-based decisions'); + } + + // Process each fact individually or in batch + for (let i = 0; i < validFacts.length; i++) { + const fact = validFacts[i]; + const codePattern = extractCodePattern(fact); + const tags = extractTechnicalTags(fact); + + let memoryAction: MemoryAction; + let similarMemories: any[] = []; + + if (embedder && vectorStore) { + try { + // Generate embedding for the fact + logger.debug('MemoryOperation: Generating embedding for fact', { + factIndex: i, + factLength: fact.length, + }); + + const embedding = await embedder.embed(fact); + + // Search for similar memories + const searchResults = await vectorStore.search( + embedding, + options.maxSimilarResults, + { threshold: options.similarityThreshold } + ); + + similarMemories = searchResults; + totalSimilarMemories += similarMemories.length; + + logger.debug('MemoryOperation: Found similar memories', { + factIndex: i, + similarCount: similarMemories.length, + }); + + // Use LLM decision making if available and enabled + if (options.useLLMDecisions && llmService) { + try { + memoryAction = await llmDetermineMemoryOperation( + fact, + similarMemories, + args.context, + options, + llmService, + i, + codePattern, + tags + ); + llmDecisionsUsed++; + + logger.debug('MemoryOperation: Used LLM decision making', { + factIndex: i, + operation: memoryAction.event, + confidence: memoryAction.confidence, + }); + + } catch (error) { + logger.warn('MemoryOperation: LLM decision failed, falling back to similarity analysis', { + factIndex: i, + error: error instanceof Error ? error.message : String(error), + }); + + // Fallback to similarity-based decision + memoryAction = await determineMemoryOperation( + fact, + similarMemories, + options.similarityThreshold, + i, + codePattern, + tags + ); + fallbackDecisionsUsed++; + } + } else { + // Use similarity-based decision making + memoryAction = await determineMemoryOperation( + fact, + similarMemories, + options.similarityThreshold, + i, + codePattern, + tags + ); + fallbackDecisionsUsed++; + } + + } catch (error) { + logger.warn('MemoryOperation: Error during similarity analysis, falling back to ADD', { + factIndex: i, + error: error instanceof Error ? error.message : String(error), + }); + + // Fallback to ADD operation + memoryAction = { + id: generateMemoryId(i), + text: fact, + event: 'ADD', + tags, + confidence: 0.5, + reasoning: 'Fallback to ADD due to analysis error', + ...(codePattern && { code_pattern: codePattern }), + }; + fallbackDecisionsUsed++; + } + } else { + // No embedding/vector storage available - basic analysis + const isNew = !args.existingMemories?.some(mem => + calculateTextSimilarity(fact, mem.text) > options.similarityThreshold + ); + + memoryAction = { + id: generateMemoryId(i), + text: fact, + event: isNew ? 'ADD' : 'NONE', + tags, + confidence: 0.6, + reasoning: isNew ? 'No similar memories found in basic analysis' : 'Similar memory detected in basic analysis', + ...(codePattern && { code_pattern: codePattern }), + }; + fallbackDecisionsUsed++; + } + + // Apply confidence threshold + if (memoryAction.confidence < options.confidenceThreshold && memoryAction.event !== 'NONE') { + logger.debug('MemoryOperation: Operation confidence below threshold, changing to NONE', { + factIndex: i, + operation: memoryAction.event, + confidence: memoryAction.confidence, + threshold: options.confidenceThreshold, + }); + + memoryAction.event = 'NONE'; + memoryAction.reasoning += ` (Low confidence: ${memoryAction.confidence.toFixed(2)})`; + } + + memoryActions.push(memoryAction); + confidenceSum += memoryAction.confidence; + } + + const processingTime = Date.now() - startTime; + const averageConfidence = memoryActions.length > 0 ? confidenceSum / memoryActions.length : 0; + + const result: MemoryOperationResult = { + success: true, + totalFacts: args.extractedFacts.length, + processedFacts: validFacts.length, + skippedFacts: args.extractedFacts.length - validFacts.length, + memory: memoryActions, + statistics: { + addOperations: memoryActions.filter(a => a.event === 'ADD').length, + updateOperations: memoryActions.filter(a => a.event === 'UPDATE').length, + deleteOperations: memoryActions.filter(a => a.event === 'DELETE').length, + noneOperations: memoryActions.filter(a => a.event === 'NONE').length, + totalSimilarMemories, + averageConfidence, + llmDecisionsUsed, + fallbackDecisionsUsed, + }, + timestamp: new Date().toISOString(), + processingTime, + }; + + logger.info('MemoryOperation: Successfully processed memory operations', { + totalFacts: result.totalFacts, + processedFacts: result.processedFacts, + memoryActions: result.memory.length, + llmDecisionsUsed: result.statistics.llmDecisionsUsed, + fallbackDecisionsUsed: result.statistics.fallbackDecisionsUsed, + averageConfidence: result.statistics.averageConfidence.toFixed(2), + processingTime: `${processingTime}ms`, + }); + + return result; + + } catch (error) { + const processingTime = Date.now() - startTime; + const errorMessage = error instanceof Error ? error.message : String(error); + + logger.error('MemoryOperation: Failed to process memory operations', { + error: errorMessage, + factCount: args.extractedFacts?.length || 0, + processingTime: `${processingTime}ms`, + }); + + return { + success: false, + totalFacts: args.extractedFacts?.length || 0, + processedFacts: 0, + skippedFacts: args.extractedFacts?.length || 0, + memory: [], + statistics: { + addOperations: 0, + updateOperations: 0, + deleteOperations: 0, + noneOperations: 0, + totalSimilarMemories: 0, + averageConfidence: 0, + llmDecisionsUsed: 0, + fallbackDecisionsUsed: 0, + }, + timestamp: new Date().toISOString(), + processingTime, + error: errorMessage, + }; + } + }, +}; + +/** + * LLM-powered memory operation determination + */ +async function llmDetermineMemoryOperation( + fact: string, + similarMemories: any[], + context: MemoryOperationArgs['context'], + options: Required, + llmService: any, + index: number, + codePattern?: string, + tags: string[] = [] +): Promise { + const factId = generateMemoryId(index); + + try { + // Prepare context for LLM + const contextStr = formatContextForLLM(context); + const similarMemoriesStr = formatSimilarMemoriesForLLM(similarMemories); + + // Create decision prompt + const prompt = MEMORY_OPERATION_PROMPTS.DECISION_PROMPT + .replace('{fact}', fact) + .replace('{similarMemories}', similarMemoriesStr) + .replace('{context}', contextStr); + + logger.debug('MemoryOperation: Requesting LLM decision', { + factIndex: index, + factLength: fact.length, + similarMemoriesCount: similarMemories.length, + }); + + // Get LLM response + const response = await llmService.generate(prompt); + + // Parse LLM response + const decision = parseLLMDecision(response); + + // Validate and apply decision + if (!decision || !isValidOperation(decision.operation)) { + throw new Error(`Invalid LLM decision: ${JSON.stringify(decision)}`); + } + + // Create memory action based on LLM decision + const memoryAction: MemoryAction = { + id: decision.targetMemoryId || factId, + text: fact, + event: decision.operation as 'ADD' | 'UPDATE' | 'DELETE' | 'NONE', + tags, + confidence: Math.max(0, Math.min(1, decision.confidence || 0.7)), + reasoning: decision.reasoning || 'LLM decision', + ...(codePattern && { code_pattern: codePattern }), + }; + + // Add old_memory for UPDATE operations + if (memoryAction.event === 'UPDATE' && decision.targetMemoryId) { + const targetMemory = similarMemories.find(mem => + mem.id === decision.targetMemoryId || + mem.payload?.id === decision.targetMemoryId + ); + if (targetMemory) { + memoryAction.old_memory = targetMemory.payload?.data || targetMemory.text || ''; + } + } + + logger.debug('MemoryOperation: LLM decision applied', { + factIndex: index, + operation: memoryAction.event, + confidence: memoryAction.confidence, + reasoning: memoryAction.reasoning.substring(0, 100), + }); + + return memoryAction; + + } catch (error) { + logger.warn('MemoryOperation: LLM decision failed', { + factIndex: index, + error: error instanceof Error ? error.message : String(error), + }); + + // Re-throw to trigger fallback + throw error; + } +} + +/** + * Format context information for LLM prompt + */ +function formatContextForLLM(context?: MemoryOperationArgs['context']): string { + if (!context) { + return 'No specific context provided.'; + } + + const parts: string[] = []; + + if (context.conversationTopic) { + parts.push(`Topic: ${context.conversationTopic}`); + } + + if (context.recentMessages && context.recentMessages.length > 0) { + parts.push(`Recent messages: ${context.recentMessages.slice(-3).join(', ')}`); + } + + if (context.sessionMetadata) { + const metadata = Object.entries(context.sessionMetadata) + .map(([key, value]) => `${key}: ${value}`) + .join(', '); + parts.push(`Session info: ${metadata}`); + } + + return parts.length > 0 ? parts.join('\n') : 'General context.'; +} + +/** + * Format similar memories for LLM prompt + */ +function formatSimilarMemoriesForLLM(similarMemories: any[]): string { + if (!similarMemories || similarMemories.length === 0) { + return 'No similar memories found.'; + } + + return similarMemories + .slice(0, 3) // Limit to top 3 for prompt efficiency + .map((memory, index) => { + const score = memory.score ? ` (similarity: ${memory.score.toFixed(2)})` : ''; + const text = memory.payload?.data || memory.text || 'No content'; + const id = memory.id || memory.payload?.id || `memory-${index}`; + + return `${index + 1}. ID: ${id}${score}\n Content: ${text.substring(0, 200)}${text.length > 200 ? '...' : ''}`; + }) + .join('\n\n'); +} + +/** + * Parse LLM decision response + */ +function parseLLMDecision(response: string): any { + try { + // Try to extract JSON from response + const jsonMatch = response.match(/\{[\s\S]*\}/); + if (!jsonMatch) { + throw new Error('No JSON found in response'); + } + + const decision = JSON.parse(jsonMatch[0]); + + // Validate required fields + if (!decision.operation || !decision.confidence) { + throw new Error('Missing required fields in decision'); + } + + return decision; + + } catch (error) { + logger.error('MemoryOperation: Failed to parse LLM decision', { + response: response.substring(0, 200), + error: error instanceof Error ? error.message : String(error), + }); + + throw new Error(`Failed to parse LLM decision: ${error instanceof Error ? error.message : String(error)}`); + } +} + +/** + * Validate operation type + */ +function isValidOperation(operation: string): boolean { + return ['ADD', 'UPDATE', 'DELETE', 'NONE'].includes(operation); +} + +/** + * Validation result interface + */ +interface ValidationResult { + isValid: boolean; + errors: string[]; +} + +/** + * Validate memory operation arguments + */ +function validateMemoryOperationArgs(args: MemoryOperationArgs): ValidationResult { + const errors: string[] = []; + + // Check required fields + if (!args.extractedFacts) { + errors.push('extractedFacts is required'); + } else if (!Array.isArray(args.extractedFacts)) { + errors.push('extractedFacts must be an array'); + } else if (args.extractedFacts.length === 0) { + errors.push('extractedFacts array cannot be empty'); + } + + // Validate existing memories if provided + if (args.existingMemories) { + if (!Array.isArray(args.existingMemories)) { + errors.push('existingMemories must be an array'); + } else { + args.existingMemories.forEach((memory, index) => { + if (!memory.id || typeof memory.id !== 'string') { + errors.push(`existingMemories[${index}].id must be a non-empty string`); + } + if (!memory.text || typeof memory.text !== 'string') { + errors.push(`existingMemories[${index}].text must be a non-empty string`); + } + }); + } + } + + // Validate context if provided + if (args.context) { + if (typeof args.context !== 'object') { + errors.push('context must be an object'); + } else { + if (args.context.sessionId && typeof args.context.sessionId !== 'string') { + errors.push('context.sessionId must be a string'); + } + if (args.context.userId && typeof args.context.userId !== 'string') { + errors.push('context.userId must be a string'); + } + if (args.context.projectId && typeof args.context.projectId !== 'string') { + errors.push('context.projectId must be a string'); + } + } + } + + // Validate options if provided + if (args.options) { + if (typeof args.options !== 'object') { + errors.push('options must be an object'); + } else { + if (args.options.similarityThreshold !== undefined) { + if (typeof args.options.similarityThreshold !== 'number') { + errors.push('options.similarityThreshold must be a number'); + } else if (args.options.similarityThreshold < 0 || args.options.similarityThreshold > 1) { + errors.push('options.similarityThreshold must be between 0.0 and 1.0'); + } + } + if (args.options.maxSimilarResults !== undefined) { + if (typeof args.options.maxSimilarResults !== 'number') { + errors.push('options.maxSimilarResults must be a number'); + } else if (args.options.maxSimilarResults < 1 || args.options.maxSimilarResults > 20) { + errors.push('options.maxSimilarResults must be between 1 and 20'); + } + } + if (args.options.enableBatchProcessing !== undefined && typeof args.options.enableBatchProcessing !== 'boolean') { + errors.push('options.enableBatchProcessing must be a boolean'); + } + // Additional validation + if (args.options.useLLMDecisions !== undefined && typeof args.options.useLLMDecisions !== 'boolean') { + errors.push('options.useLLMDecisions must be a boolean'); + } + if (args.options.confidenceThreshold !== undefined) { + if (typeof args.options.confidenceThreshold !== 'number') { + errors.push('options.confidenceThreshold must be a number'); + } else if (args.options.confidenceThreshold < 0 || args.options.confidenceThreshold > 1) { + errors.push('options.confidenceThreshold must be between 0.0 and 1.0'); + } + } + if (args.options.enableDeleteOperations !== undefined && typeof args.options.enableDeleteOperations !== 'boolean') { + errors.push('options.enableDeleteOperations must be a boolean'); + } + } + } + + return { + isValid: errors.length === 0, + errors, + }; +} + +/** + * Extract code pattern from fact content + */ +function extractCodePattern(fact: string): string | undefined { + // Extract code blocks (```...```) + const codeBlockMatch = fact.match(/```[\s\S]*?```/); + if (codeBlockMatch) { + return codeBlockMatch[0]; + } + + // Extract inline code (`...`) + const inlineCodeMatch = fact.match(/`[^`]+`/); + if (inlineCodeMatch) { + return inlineCodeMatch[0]; + } + + // Extract command patterns (starting with $ or npm/git/etc) + const commandPatterns = [ + /\$\s+[^\n]+/, + /(npm|yarn|pnpm)\s+[^\n]+/, + /(git)\s+[^\n]+/, + /(docker)\s+[^\n]+/, + /(curl|wget)\s+[^\n]+/ + ]; + + for (const pattern of commandPatterns) { + const match = fact.match(pattern); + if (match) { + return match[0]; + } + } + + return undefined; +} + +/** + * Extract technical tags from fact content + */ +function extractTechnicalTags(fact: string): string[] { + const tags: string[] = []; + + // Programming languages + const languages = ['javascript', 'typescript', 'python', 'java', 'rust', 'go', 'php', 'ruby', 'swift', 'kotlin']; + languages.forEach(lang => { + if (fact.toLowerCase().includes(lang)) { + tags.push(lang); + } + }); + + // Frameworks and libraries + const frameworks = ['react', 'vue', 'angular', 'svelte', 'nextjs', 'express', 'fastify', 'django', 'flask']; + frameworks.forEach(framework => { + if (fact.toLowerCase().includes(framework)) { + tags.push(framework); + } + }); + + // Tools and technologies + const tools = ['docker', 'kubernetes', 'git', 'npm', 'yarn', 'webpack', 'vite', 'eslint', 'prettier']; + tools.forEach(tool => { + if (fact.toLowerCase().includes(tool)) { + tags.push(tool); + } + }); + + // Content type tags + if (fact.includes('```')) { + tags.push('code-block'); + } + if (fact.includes('function') || fact.includes('class') || fact.includes('const') || fact.includes('let') || fact.includes('var')) { + tags.push('programming'); + } + if (fact.includes('/') || fact.includes('\\') || fact.includes('.js') || fact.includes('.ts') || fact.includes('.py')) { + tags.push('file-path'); + } + if (fact.includes('error') || fact.includes('exception') || fact.includes('failed') || fact.includes('bug')) { + tags.push('error-handling'); + } + if (fact.includes('config') || fact.includes('setting') || fact.includes('option')) { + tags.push('configuration'); + } + if (fact.includes('api') || fact.includes('endpoint') || fact.includes('request') || fact.includes('response')) { + tags.push('api'); + } + + // Add general tag if no specific patterns found + if (tags.length === 0) { + tags.push('general-knowledge'); + } + + // Remove duplicates and return lowercase singular nouns + return [...new Set(tags)].map(tag => tag.toLowerCase()); +} + +/** + * Generate unique memory ID + */ +function generateMemoryId(index: number): string { + const timestamp = Date.now(); + const random = Math.random().toString(36).substring(2, 8); + return `memory_${timestamp}_${index}_${random}`; +} + +/** + * Determine memory operation based on similarity analysis (fallback method) + */ +async function determineMemoryOperation( + fact: string, + similarMemories: any[], + threshold: number, + index: number, + codePattern?: string, + tags: string[] = [] +): Promise { + const factId = generateMemoryId(index); + + // If no similar memories found, ADD the new fact + if (similarMemories.length === 0) { + return { + id: factId, + text: fact, + event: 'ADD', + tags, + confidence: 0.8, + reasoning: 'No similar memories found - adding as new knowledge', + ...(codePattern && { code_pattern: codePattern }), + }; + } + + // Find the most similar memory + const mostSimilar = similarMemories[0]; + const similarity = mostSimilar.score || 0; + + // High similarity (>0.9) - consider as duplicate, return NONE + if (similarity > 0.9) { + return { + id: mostSimilar.id || factId, + text: fact, + event: 'NONE', + tags, + confidence: 0.9, + reasoning: `High similarity (${similarity.toFixed(2)}) with existing memory - no action needed`, + ...(codePattern && { code_pattern: codePattern }), + }; + } + + // Medium-high similarity (0.7-0.9) - consider updating existing memory + if (similarity > threshold && similarity <= 0.9) { + return { + id: mostSimilar.id || factId, + text: fact, + event: 'UPDATE', + tags, + confidence: 0.75, + reasoning: `Medium similarity (${similarity.toFixed(2)}) - updating existing memory`, + old_memory: mostSimilar.payload?.data || mostSimilar.text || '', + ...(codePattern && { code_pattern: codePattern }), + }; + } + + // Low similarity - ADD as new memory + return { + id: factId, + text: fact, + event: 'ADD', + tags, + confidence: 0.7, + reasoning: `Low similarity (${similarity.toFixed(2)}) - adding as new knowledge`, + ...(codePattern && { code_pattern: codePattern }), + }; +} + +/** + * Calculate text similarity using simple token-based approach + * This is a fallback when embeddings are not available + */ +function calculateTextSimilarity(text1: string, text2: string): number { + // Simple token-based similarity calculation + const tokens1 = text1.toLowerCase().split(/\s+/); + const tokens2 = text2.toLowerCase().split(/\s+/); + + const set1 = new Set(tokens1); + const set2 = new Set(tokens2); + + const intersection = new Set([...set1].filter(x => set2.has(x))); + const union = new Set([...set1, ...set2]); + + // Jaccard similarity + return intersection.size / union.size; +} diff --git a/src/core/brain/tools/manager.ts b/src/core/brain/tools/manager.ts index e0877e361..dd51616de 100644 --- a/src/core/brain/tools/manager.ts +++ b/src/core/brain/tools/manager.ts @@ -16,7 +16,6 @@ import { InternalToolManagerConfig, InternalToolContext, ToolExecutionStats, - ToolRegistrationResult, INTERNAL_TOOL_PREFIX, createInternalToolName, isInternalToolName, @@ -50,6 +49,11 @@ export class InternalToolManager implements IInternalToolManager { private initialized = false; private stats = new Map(); private readonly maxExecutionHistorySize = 100; + private services?: { + embeddingManager?: any; + vectorStoreManager?: any; + llmService?: any; + }; constructor(config: InternalToolManagerConfig = {}) { this.config = { ...DEFAULT_CONFIG, ...config }; @@ -94,7 +98,7 @@ export class InternalToolManager implements IInternalToolManager { /** * Register a new internal tool */ - public registerTool(tool: InternalTool): ToolRegistrationResult { + public registerTool(tool: InternalTool): { success: boolean; message: string; conflictedWith?: string } { this.ensureInitialized(); const result = this.registry.registerTool(tool); @@ -178,7 +182,9 @@ export class InternalToolManager implements IInternalToolManager { toolName: normalizedName, startTime, sessionId: context?.sessionId, + userId: context?.userId, metadata: context?.metadata, + services: this.services, }; logger.info(`InternalToolManager: Executing tool '${normalizedName}'`, { @@ -259,6 +265,33 @@ export class InternalToolManager implements IInternalToolManager { }; } + /** + * Get all tool statistics + */ + public getStatistics(): Record { + const allStats: Record = {}; + + for (const [toolName, entry] of this.stats.entries()) { + allStats[toolName] = { ...entry.stats }; + } + + return allStats; + } + + /** + * Get available tools list + */ + public async getAvailableTools(): Promise> { + this.ensureInitialized(); + + const tools = this.registry.getAllTools(); + return Object.values(tools).map(tool => ({ + name: tool.name, + description: tool.description, + category: tool.category, + })); + } + /** * Clear all execution statistics */ @@ -305,7 +338,7 @@ export class InternalToolManager implements IInternalToolManager { }, this.config.timeout); tool - .handler(args) + .handler(args, context) .then(result => { clearTimeout(timeoutId); resolve(result); @@ -324,13 +357,13 @@ export class InternalToolManager implements IInternalToolManager { let entry = this.stats.get(toolName); if (!entry) { - entry = this.createStatsEntry(); + entry = this.createStatsEntry(toolName); this.stats.set(toolName, entry); } // Update statistics entry.stats.totalExecutions++; - entry.stats.lastExecuted = Date.now(); + entry.stats.lastExecution = new Date().toISOString(); if (success) { entry.stats.successfulExecutions++; @@ -357,16 +390,17 @@ export class InternalToolManager implements IInternalToolManager { private initializeToolStats(toolName: string): void { const normalizedName = createInternalToolName(toolName); if (!this.stats.has(normalizedName)) { - this.stats.set(normalizedName, this.createStatsEntry()); + this.stats.set(normalizedName, this.createStatsEntry(normalizedName)); } } /** * Create a new stats entry */ - private createStatsEntry(): StatsEntry { + private createStatsEntry(toolName: string): StatsEntry { return { stats: { + toolName, totalExecutions: 0, successfulExecutions: 0, failedExecutions: 0, @@ -412,4 +446,20 @@ export class InternalToolManager implements IInternalToolManager { public isEnabled(): boolean { return this.config.enabled; } + + /** + * Set agent services for tools that need access to them + */ + public setServices(services: { + embeddingManager?: any; + vectorStoreManager?: any; + llmService?: any; + }): void { + this.services = services; + logger.debug('InternalToolManager: Services configured', { + hasEmbeddingManager: !!services.embeddingManager, + hasVectorStoreManager: !!services.vectorStoreManager, + hasLlmService: !!services.llmService, + }); + } } diff --git a/src/core/brain/tools/registry.ts b/src/core/brain/tools/registry.ts index e9975834e..3c11ece99 100644 --- a/src/core/brain/tools/registry.ts +++ b/src/core/brain/tools/registry.ts @@ -60,7 +60,7 @@ export class InternalToolRegistry { /** * Register a new internal tool */ - public registerTool(tool: InternalTool): ToolRegistrationResult { + public registerTool(tool: InternalTool): { success: boolean; message: string; conflictedWith?: string } { try { // Validate tool structure const validation = this.validateTool(tool); diff --git a/src/core/brain/tools/types.ts b/src/core/brain/tools/types.ts index ed566b1ea..6e3a6b624 100644 --- a/src/core/brain/tools/types.ts +++ b/src/core/brain/tools/types.ts @@ -7,6 +7,9 @@ */ import { Tool, ToolParameters, ToolExecutionResult } from '../../mcp/types.js'; +import type { EmbeddingManager } from '../embedding/index.js'; +import type { VectorStoreManager } from '../../vector_storage/index.js'; +import type { ILLMService } from '../llm/index.js'; /** * Categories for organizing internal tools @@ -16,7 +19,10 @@ export type InternalToolCategory = 'memory' | 'session' | 'system'; /** * Internal tool handler function signature */ -export type InternalToolHandler = (args: any) => Promise; +export type InternalToolHandler = ( + args: T, + context?: InternalToolContext +) => Promise; /** * Internal tool definition extending the base Tool interface @@ -46,6 +52,20 @@ export interface InternalTool extends Tool { * Optional version for tool evolution */ version?: string; + + /** + * Human-readable description + */ + description: string; + + /** + * JSON schema for parameters + */ + parameters: { + type: 'object'; + properties: Record; + required?: string[]; + }; } /** @@ -88,9 +108,23 @@ export interface InternalToolManagerConfig { * Tool registration result */ export interface ToolRegistrationResult { - success: boolean; - message: string; - conflictedWith?: string; + /** + * Total tools attempted to register + */ + total: number; + + /** + * Successfully registered tools + */ + registered: string[]; + + /** + * Failed tool registrations + */ + failed: Array<{ + name: string; + error: string; + }>; } /** @@ -116,6 +150,31 @@ export interface InternalToolContext { * Any additional metadata */ metadata: Record | undefined; + + /** + * Optional agent services for advanced tool operations + */ + services?: { + /** + * Embedding manager for text embeddings + */ + embeddingManager?: EmbeddingManager; + + /** + * Vector storage manager for similarity search + */ + vectorStoreManager?: VectorStoreManager; + + /** + * LLM service for intelligent reasoning (Phase 3) + */ + llmService?: ILLMService; + }; + + /** + * User ID for personalized behavior + */ + userId?: string; } /** @@ -123,17 +182,22 @@ export interface InternalToolContext { */ export interface ToolExecutionStats { /** - * Total number of executions + * Tool name + */ + toolName: string; + + /** + * Total executions */ totalExecutions: number; /** - * Number of successful executions + * Successful executions */ successfulExecutions: number; /** - * Number of failed executions + * Failed executions */ failedExecutions: number; @@ -145,7 +209,12 @@ export interface ToolExecutionStats { /** * Last execution timestamp */ - lastExecuted?: number; + lastExecution?: string; + + /** + * Last error message + */ + lastError?: string; } /** @@ -160,7 +229,7 @@ export interface IInternalToolManager { /** * Register a new internal tool */ - registerTool(tool: InternalTool): ToolRegistrationResult; + registerTool(tool: InternalTool): { success: boolean; message: string; conflictedWith?: string }; /** * Unregister an internal tool diff --git a/src/core/env.ts b/src/core/env.ts index b2d009c99..1a9b9513e 100644 --- a/src/core/env.ts +++ b/src/core/env.ts @@ -8,10 +8,17 @@ const envSchema = z.object({ NODE_ENV: z.enum(['development', 'production', 'test']).default('development'), CIPHER_LOG_LEVEL: z.enum(['debug', 'info', 'warn', 'error']).default('info'), REDACT_SECRETS: z.boolean().default(true), + // LLM Provider API Keys + // Note: OPENAI_API_KEY is effectively required for embedding functionality OPENAI_API_KEY: z.string().optional(), ANTHROPIC_API_KEY: z.string().optional(), OPENROUTER_API_KEY: z.string().optional(), OPENAI_BASE_URL: z.string().optional(), + OPENAI_ORG_ID: z.string().optional(), + // Embedding Configuration + EMBEDDING_MODEL: z.string().optional(), + EMBEDDING_TIMEOUT: z.number().optional(), + EMBEDDING_MAX_RETRIES: z.number().optional(), // Storage Configuration STORAGE_CACHE_TYPE: z.enum(['redis', 'in-memory']).default('in-memory'), STORAGE_CACHE_HOST: z.string().optional(), @@ -54,6 +61,19 @@ export const env: EnvSchema = new Proxy({} as EnvSchema, { return process.env.OPENROUTER_API_KEY; case 'OPENAI_BASE_URL': return process.env.OPENAI_BASE_URL; + case 'OPENAI_ORG_ID': + return process.env.OPENAI_ORG_ID; + // Embedding Configuration + case 'EMBEDDING_MODEL': + return process.env.EMBEDDING_MODEL; + case 'EMBEDDING_TIMEOUT': + return process.env.EMBEDDING_TIMEOUT + ? parseInt(process.env.EMBEDDING_TIMEOUT, 10) + : undefined; + case 'EMBEDDING_MAX_RETRIES': + return process.env.EMBEDDING_MAX_RETRIES + ? parseInt(process.env.EMBEDDING_MAX_RETRIES, 10) + : undefined; // Storage Configuration case 'STORAGE_CACHE_TYPE': return process.env.STORAGE_CACHE_TYPE || 'in-memory'; @@ -109,6 +129,12 @@ export const env: EnvSchema = new Proxy({} as EnvSchema, { }); export const validateEnv = () => { + // Critical validation: OPENAI_API_KEY is always required for embedding functionality + if (!process.env.OPENAI_API_KEY) { + console.error('OPENAI_API_KEY is required for embedding functionality, even when using other LLM providers (Anthropic, OpenRouter, etc.)'); + return false; + } + // Get current env values for validation const envToValidate = { NODE_ENV: process.env.NODE_ENV, @@ -118,6 +144,15 @@ export const validateEnv = () => { ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY, OPENROUTER_API_KEY: process.env.OPENROUTER_API_KEY, OPENAI_BASE_URL: process.env.OPENAI_BASE_URL, + OPENAI_ORG_ID: process.env.OPENAI_ORG_ID, + // Embedding Configuration + EMBEDDING_MODEL: process.env.EMBEDDING_MODEL, + EMBEDDING_TIMEOUT: process.env.EMBEDDING_TIMEOUT + ? parseInt(process.env.EMBEDDING_TIMEOUT, 10) + : undefined, + EMBEDDING_MAX_RETRIES: process.env.EMBEDDING_MAX_RETRIES + ? parseInt(process.env.EMBEDDING_MAX_RETRIES, 10) + : undefined, // Storage Configuration STORAGE_CACHE_TYPE: process.env.STORAGE_CACHE_TYPE || 'in-memory', STORAGE_CACHE_HOST: process.env.STORAGE_CACHE_HOST, diff --git a/src/core/utils/service-initializer.ts b/src/core/utils/service-initializer.ts index ab9bee529..149dac2e1 100644 --- a/src/core/utils/service-initializer.ts +++ b/src/core/utils/service-initializer.ts @@ -9,6 +9,11 @@ import { logger } from '../logger/index.js'; import { AgentConfig } from '../brain/memAgent/config.js'; import { ServerConfigsSchema } from '../mcp/config.js'; import { ServerConfigs } from '../mcp/types.js'; +import { EmbeddingManager } from '../brain/embedding/index.js'; +import { VectorStoreManager } from '../vector_storage/index.js'; +import { createLLMService } from '../brain/llm/services/factory.js'; +import { createContextManager } from '../brain/llm/messages/factory.js'; +import { ILLMService } from '../brain/llm/index.js'; export type AgentServices = { mcpManager: MCPManager; @@ -17,6 +22,9 @@ export type AgentServices = { sessionManager: SessionManager; internalToolManager: InternalToolManager; unifiedToolManager: UnifiedToolManager; + embeddingManager: EmbeddingManager; + vectorStoreManager: VectorStoreManager; + llmService?: ILLMService; // Optional for Phase 3 features }; export async function createAgentServices(agentConfig: AgentConfig): Promise { @@ -37,16 +45,83 @@ export async function createAgentServices(agentConfig: AgentConfig): Promise Date: Thu, 3 Jul 2025 16:57:29 +0700 Subject: [PATCH 2/5] feat: add memory operation tool --- .env.example | 19 +- .../brain/tools/__test__/definitions.test.ts | 17 +- .../__test__/unified-tool-manager.test.ts | 15 +- src/core/brain/tools/definitions/index.ts | 3 +- .../definitions/memory/extract-knowledge.ts | 43 +++- .../brain/tools/definitions/memory/index.ts | 12 +- .../definitions/memory/memory_operation.ts | 125 ++++++++-- src/core/utils/service-initializer.ts | 24 +- .../vector_storage/__test__/factory.test.ts | 224 +++++++++++------- src/core/vector_storage/factory.ts | 72 +++--- src/core/vector_storage/index.ts | 1 + 11 files changed, 377 insertions(+), 178 deletions(-) diff --git a/.env.example b/.env.example index ed4442205..5690ddf74 100644 --- a/.env.example +++ b/.env.example @@ -35,4 +35,21 @@ STORAGE_DATABASE_TYPE=in-memory # ==================== # MCP Configuration # ==================== -# MCP_GLOBAL_TIMEOUT=30000 \ No newline at end of file +# MCP_GLOBAL_TIMEOUT=30000 + + +# Vector store type: qdrant, in-memory +#VECTOR_STORE_TYPE=in-memory + +# Qdrant configuration (only used if VECTOR_STORE_TYPE=qdrant) +#VECTOR_STORE_HOST=localhost +#VECTOR_STORE_PORT=6333 +#VECTOR_STORE_URL=http://localhost:6333 +#VECTOR_STORE_API_KEY=your-qdrant-api-key + +# Vector collection settings +# VECTOR_STORE_COLLECTION=default +# VECTOR_STORE_DIMENSION=1536 +# VECTOR_STORE_DISTANCE=Cosine +# VECTOR_STORE_ON_DISK=false +# VECTOR_STORE_MAX_VECTORS=10000 \ No newline at end of file diff --git a/src/core/brain/tools/__test__/definitions.test.ts b/src/core/brain/tools/__test__/definitions.test.ts index 5bb1b8dd7..6cacacbe6 100644 --- a/src/core/brain/tools/__test__/definitions.test.ts +++ b/src/core/brain/tools/__test__/definitions.test.ts @@ -79,17 +79,19 @@ describe('Tool Definitions', () => { it('should load all tool definitions', async () => { const tools = await getAllToolDefinitions(); - expect(Object.keys(tools)).toHaveLength(1); // 1 memory tool + expect(Object.keys(tools)).toHaveLength(2); // 2 memory tools expect(tools['extract_knowledge']).toBeDefined(); + expect(tools['memory_operation']).toBeDefined(); }); it('should register all tools successfully', async () => { const result = await registerAllTools(manager); - expect(result.total).toBe(1); - expect(result.registered.length).toBe(1); + expect(result.total).toBe(2); + expect(result.registered.length).toBe(2); expect(result.failed.length).toBe(0); expect(result.registered).toContain('extract_knowledge'); + expect(result.registered).toContain('memory_operation'); }); it('should handle registration failures gracefully', async () => { @@ -103,9 +105,9 @@ describe('Tool Definitions', () => { const result = await registerAllTools(failingManager); - expect(result.total).toBe(1); + expect(result.total).toBe(2); expect(result.registered.length).toBe(0); - expect(result.failed.length).toBe(1); + expect(result.failed.length).toBe(2); expect(result.failed?.[0]?.error).toBe('Simulated failure'); }); }); @@ -114,7 +116,7 @@ describe('Tool Definitions', () => { it('should have correct category structure', () => { expect(TOOL_CATEGORIES.memory).toBeDefined(); - expect(TOOL_CATEGORIES.memory.tools).toHaveLength(1); + expect(TOOL_CATEGORIES.memory.tools).toHaveLength(2); }); it('should get tool info by name', () => { @@ -130,8 +132,9 @@ describe('Tool Definitions', () => { it('should get tools by category', () => { const memoryTools = getToolsByCategory('memory'); - expect(memoryTools).toHaveLength(1); + expect(memoryTools).toHaveLength(2); expect(memoryTools).toContain('cipher_extract_knowledge'); + expect(memoryTools).toContain('cipher_memory_operation'); }); }); diff --git a/src/core/brain/tools/__test__/unified-tool-manager.test.ts b/src/core/brain/tools/__test__/unified-tool-manager.test.ts index bbfea3907..028c75aae 100644 --- a/src/core/brain/tools/__test__/unified-tool-manager.test.ts +++ b/src/core/brain/tools/__test__/unified-tool-manager.test.ts @@ -79,9 +79,10 @@ describe('UnifiedToolManager', () => { it('should load internal tools when enabled', async () => { const tools = await unifiedManager.getAllTools(); - // Should have 1 memory tool - expect(Object.keys(tools)).toHaveLength(1); + // Should have 2 memory tools + expect(Object.keys(tools)).toHaveLength(2); expect(tools['cipher_extract_knowledge']).toBeDefined(); + expect(tools['cipher_memory_operation']).toBeDefined(); // All tools should be marked as internal for (const tool of Object.values(tools)) { @@ -159,7 +160,7 @@ describe('UnifiedToolManager', () => { const formattedTools = await unifiedManager.getToolsForProvider('openai'); expect(Array.isArray(formattedTools)).toBe(true); - expect(formattedTools.length).toBe(1); + expect(formattedTools.length).toBe(2); // Check OpenAI format const tool = formattedTools[0]; @@ -174,7 +175,7 @@ describe('UnifiedToolManager', () => { const formattedTools = await unifiedManager.getToolsForProvider('anthropic'); expect(Array.isArray(formattedTools)).toBe(true); - expect(formattedTools.length).toBe(1); + expect(formattedTools.length).toBe(2); // Check Anthropic format const tool = formattedTools[0]; @@ -187,7 +188,7 @@ describe('UnifiedToolManager', () => { const formattedTools = await unifiedManager.getToolsForProvider('openrouter'); expect(Array.isArray(formattedTools)).toBe(true); - expect(formattedTools.length).toBe(1); + expect(formattedTools.length).toBe(2); // OpenRouter uses OpenAI format const tool = formattedTools[0]; @@ -211,8 +212,8 @@ describe('UnifiedToolManager', () => { expect(stats.config).toBeDefined(); // Internal tools stats should be available - expect(stats.internalTools.totalTools).toBe(1); - expect(stats.internalTools.toolsByCategory.memory).toBe(1); + expect(stats.internalTools.totalTools).toBe(2); + expect(stats.internalTools.toolsByCategory.memory).toBe(2); }); it('should handle disabled tool managers in stats', () => { diff --git a/src/core/brain/tools/definitions/index.ts b/src/core/brain/tools/definitions/index.ts index 50a16c300..d6cbee00e 100644 --- a/src/core/brain/tools/definitions/index.ts +++ b/src/core/brain/tools/definitions/index.ts @@ -10,6 +10,7 @@ export * from './memory/index.js'; // Export individual tools for direct access export { extractKnowledgeTool } from './memory/extract-knowledge.js'; +export { memoryOperationTool } from './memory/memory_operation.js'; // Import types and utilities import type { InternalToolSet } from '../types.js'; @@ -104,7 +105,7 @@ export async function registerAllTools(toolManager: any): Promise<{ export const TOOL_CATEGORIES = { memory: { description: 'Tools for managing facts, memories, and knowledge storage', - tools: ['extract_knowledge'] as string[], + tools: ['extract_knowledge', 'memory_operation'] as string[], useCase: 'Use these tools to capture, search, and store important information for future reference', }, diff --git a/src/core/brain/tools/definitions/memory/extract-knowledge.ts b/src/core/brain/tools/definitions/memory/extract-knowledge.ts index d7bde892f..9755f2364 100644 --- a/src/core/brain/tools/definitions/memory/extract-knowledge.ts +++ b/src/core/brain/tools/definitions/memory/extract-knowledge.ts @@ -62,16 +62,44 @@ export const extractKnowledgeTool: InternalTool = { handler: async (args: { knowledge: string[] }) => { try { logger.info('ExtractFact: Processing fact extraction request', { - factCount: args.knowledge.length, + factCount: Array.isArray(args.knowledge) ? args.knowledge.length : 1, + inputType: typeof args.knowledge, }); - // Validate input - if (!args.knowledge || args.knowledge.length === 0) { + // Validate input and handle both string and array inputs + let knowledgeArray: string[]; + + if (!args.knowledge) { + throw new Error('No facts provided for extraction'); + } + + // Handle case where LLM passes knowledge as a JSON string instead of array + if (typeof args.knowledge === 'string') { + try { + // Try to parse as JSON array first + const parsed = JSON.parse(args.knowledge); + if (Array.isArray(parsed)) { + knowledgeArray = parsed; + } else { + // Treat as single string fact + knowledgeArray = [args.knowledge]; + } + } catch { + // Not valid JSON, treat as single string fact + knowledgeArray = [args.knowledge]; + } + } else if (Array.isArray(args.knowledge)) { + knowledgeArray = args.knowledge; + } else { + throw new Error('Knowledge must be a string or array of strings'); + } + + if (knowledgeArray.length === 0) { throw new Error('No facts provided for extraction'); } // Filter out empty or invalid facts - const validFacts = args.knowledge + const validFacts = knowledgeArray .filter(fact => fact && typeof fact === 'string' && fact.trim().length > 0) .map(fact => fact.trim()); @@ -107,7 +135,7 @@ export const extractKnowledgeTool: InternalTool = { const result = { success: true, extracted: processedFacts.length, - skipped: args.knowledge.length - validFacts.length, + skipped: knowledgeArray.length - validFacts.length, timestamp: new Date().toISOString(), facts: processedFacts.map(f => ({ id: f.metadata.id, @@ -130,14 +158,15 @@ export const extractKnowledgeTool: InternalTool = { const errorMessage = error instanceof Error ? error.message : String(error); logger.error('ExtractFact: Failed to extract facts', { error: errorMessage, - factCount: args.knowledge?.length || 0, + factCount: Array.isArray(args.knowledge) ? args.knowledge.length : 1, + inputType: typeof args.knowledge, }); return { success: false, error: errorMessage, extracted: 0, - skipped: args.knowledge?.length || 0, + skipped: Array.isArray(args.knowledge) ? args.knowledge.length : 1, timestamp: new Date().toISOString(), }; } diff --git a/src/core/brain/tools/definitions/memory/index.ts b/src/core/brain/tools/definitions/memory/index.ts index 9c7b8dbc3..6c66b5f2a 100644 --- a/src/core/brain/tools/definitions/memory/index.ts +++ b/src/core/brain/tools/definitions/memory/index.ts @@ -7,8 +7,7 @@ // Export all memory tools export { extractKnowledgeTool } from './extract-knowledge.js'; -// TODO: Re-enable when tests are updated to handle multiple tools -// export { memoryOperationTool } from './memory_operation.js'; +export { memoryOperationTool } from './memory_operation.js'; // Export types for better developer experience import type { InternalTool } from '../../types.js'; @@ -22,21 +21,18 @@ export const memoryTools: InternalTool[] = [ // Load tools dynamically to avoid potential circular dependencies import('./extract-knowledge.js').then(module => memoryTools.push(module.extractKnowledgeTool)); -// TODO: Re-enable when tests are updated to handle multiple tools -// import('./memory_operation.js').then(module => memoryTools.push(module.memoryOperationTool)); +import('./memory_operation.js').then(module => memoryTools.push(module.memoryOperationTool)); /** * Get all memory tools as a map */ export async function getMemoryTools(): Promise> { const { extractKnowledgeTool } = await import('./extract-knowledge.js'); - // TODO: Re-enable when tests are updated to handle multiple tools - // const { memoryOperationTool } = await import('./memory_operation.js'); + const { memoryOperationTool } = await import('./memory_operation.js'); return { [extractKnowledgeTool.name]: extractKnowledgeTool, - // TODO: Re-enable when tests are updated to handle multiple tools - // [memoryOperationTool.name]: memoryOperationTool, + [memoryOperationTool.name]: memoryOperationTool, }; } diff --git a/src/core/brain/tools/definitions/memory/memory_operation.ts b/src/core/brain/tools/definitions/memory/memory_operation.ts index d94559b90..2ec64f9c3 100644 --- a/src/core/brain/tools/definitions/memory/memory_operation.ts +++ b/src/core/brain/tools/definitions/memory/memory_operation.ts @@ -147,7 +147,7 @@ const DEFAULT_OPTIONS = { maxSimilarResults: 5, enableBatchProcessing: true, useLLMDecisions: true, // Enable LLM decisions by default - confidenceThreshold: 0.6, // Minimum confidence threshold + confidenceThreshold: 0.4, // Lowered to allow fallback operations to proceed enableDeleteOperations: true, // Enable DELETE operations } as const; @@ -416,16 +416,21 @@ export const memoryOperationTool: InternalTool = { // Search for similar memories const searchResults = await vectorStore.search( embedding, - options.maxSimilarResults, - { threshold: options.similarityThreshold } + options.maxSimilarResults + ); + + // Apply similarity threshold filtering + similarMemories = searchResults.filter(result => + (result.score || 0) >= options.similarityThreshold ); - similarMemories = searchResults; totalSimilarMemories += similarMemories.length; logger.debug('MemoryOperation: Found similar memories', { factIndex: i, - similarCount: similarMemories.length, + totalResults: searchResults.length, + filteredResults: similarMemories.length, + threshold: options.similarityThreshold, }); // Use LLM decision making if available and enabled @@ -485,14 +490,14 @@ export const memoryOperationTool: InternalTool = { error: error instanceof Error ? error.message : String(error), }); - // Fallback to ADD operation + // Fallback to ADD operation with higher confidence memoryAction = { id: generateMemoryId(i), text: fact, event: 'ADD', tags, - confidence: 0.5, - reasoning: 'Fallback to ADD due to analysis error', + confidence: 0.6, // Increased from 0.5 to exceed threshold + reasoning: `Fallback to ADD due to analysis error (${error instanceof Error ? error.message : String(error)})`, ...(codePattern && { code_pattern: codePattern }), }; fallbackDecisionsUsed++; @@ -508,7 +513,7 @@ export const memoryOperationTool: InternalTool = { text: fact, event: isNew ? 'ADD' : 'NONE', tags, - confidence: 0.6, + confidence: isNew ? 0.7 : 0.5, // Higher confidence for new memories reasoning: isNew ? 'No similar memories found in basic analysis' : 'Similar memory detected in basic analysis', ...(codePattern && { code_pattern: codePattern }), }; @@ -565,6 +570,23 @@ export const memoryOperationTool: InternalTool = { processingTime: `${processingTime}ms`, }); + // Persist memory actions to vector store if available + if (vectorStore && embedder) { + try { + await persistMemoryActions(memoryActions, vectorStore, embedder); + logger.info('MemoryOperation: Successfully persisted memories to vector store', { + persistedCount: memoryActions.filter(a => a.event === 'ADD' || a.event === 'UPDATE').length, + }); + } catch (error) { + logger.warn('MemoryOperation: Failed to persist memories to vector store', { + error: error instanceof Error ? error.message : String(error), + }); + // Don't fail the entire operation if persistence fails + } + } else { + logger.debug('MemoryOperation: Vector store or embedder not available, skipping persistence'); + } + return result; } catch (error) { @@ -1049,16 +1071,79 @@ async function determineMemoryOperation( * This is a fallback when embeddings are not available */ function calculateTextSimilarity(text1: string, text2: string): number { - // Simple token-based similarity calculation - const tokens1 = text1.toLowerCase().split(/\s+/); - const tokens2 = text2.toLowerCase().split(/\s+/); - - const set1 = new Set(tokens1); - const set2 = new Set(tokens2); - - const intersection = new Set([...set1].filter(x => set2.has(x))); - const union = new Set([...set1, ...set2]); - - // Jaccard similarity + const words1 = new Set(text1.toLowerCase().split(/\s+/)); + const words2 = new Set(text2.toLowerCase().split(/\s+/)); + + const intersection = new Set([...words1].filter(word => words2.has(word))); + const union = new Set([...words1, ...words2]); + return intersection.size / union.size; } + +/** + * Persist memory actions to vector store + */ +async function persistMemoryActions( + memoryActions: MemoryAction[], + vectorStore: any, + embedder: any +): Promise { + const actionsToProcess = memoryActions.filter(action => + action.event === 'ADD' || action.event === 'UPDATE' + ); + + if (actionsToProcess.length === 0) { + logger.debug('MemoryOperation: No actions require persistence'); + return; + } + + logger.info('MemoryOperation: Persisting memory actions', { + totalActions: actionsToProcess.length, + addActions: actionsToProcess.filter(a => a.event === 'ADD').length, + updateActions: actionsToProcess.filter(a => a.event === 'UPDATE').length, + }); + + // Process each action + for (const action of actionsToProcess) { + try { + // Generate embedding for the memory text + const embedding = await embedder.embed(action.text); + + // Prepare payload with metadata + const payload = { + id: action.id, + text: action.text, + tags: action.tags, + confidence: action.confidence, + reasoning: action.reasoning, + event: action.event, + timestamp: new Date().toISOString(), + ...(action.code_pattern && { code_pattern: action.code_pattern }), + ...(action.old_memory && { old_memory: action.old_memory }), + }; + + if (action.event === 'ADD') { + // Insert new memory + await vectorStore.insert([embedding], [action.id], [payload]); + logger.debug('MemoryOperation: Added memory to vector store', { + id: action.id, + textLength: action.text.length, + }); + } else if (action.event === 'UPDATE') { + // Update existing memory + await vectorStore.update(action.id, embedding, payload); + logger.debug('MemoryOperation: Updated memory in vector store', { + id: action.id, + textLength: action.text.length, + }); + } + } catch (error) { + logger.error('MemoryOperation: Failed to persist memory action', { + actionId: action.id, + event: action.event, + error: error instanceof Error ? error.message : String(error), + }); + // Continue with other actions even if one fails + } + } +} diff --git a/src/core/utils/service-initializer.ts b/src/core/utils/service-initializer.ts index 149dac2e1..cbde484be 100644 --- a/src/core/utils/service-initializer.ts +++ b/src/core/utils/service-initializer.ts @@ -14,6 +14,7 @@ import { VectorStoreManager } from '../vector_storage/index.js'; import { createLLMService } from '../brain/llm/services/factory.js'; import { createContextManager } from '../brain/llm/messages/factory.js'; import { ILLMService } from '../brain/llm/index.js'; +import { createVectorStoreFromEnv } from '../vector_storage/factory.js'; export type AgentServices = { mcpManager: MCPManager; @@ -24,7 +25,7 @@ export type AgentServices = { unifiedToolManager: UnifiedToolManager; embeddingManager: EmbeddingManager; vectorStoreManager: VectorStoreManager; - llmService?: ILLMService; // Optional for Phase 3 features + llmService?: ILLMService; }; export async function createAgentServices(agentConfig: AgentConfig): Promise { @@ -69,19 +70,14 @@ export async function createAgentServices(agentConfig: AgentConfig): Promise ({ describe('Vector Storage Factory', () => { // Store original env vars const originalEnv = { ...process.env }; - + afterEach(() => { // Restore environment variables process.env = { ...originalEnv }; }); - + describe('createVectorStore', () => { it('should create and connect vector storage with in-memory backend', async () => { const config: VectorStoreConfig = { @@ -49,29 +50,29 @@ describe('Vector Storage Factory', () => { dimension: 768, maxVectors: 1000, }; - + const result = await createVectorStore(config); - + // Verify structure expect(result).toHaveProperty('manager'); expect(result).toHaveProperty('store'); expect(result.manager).toBeInstanceOf(VectorStoreManager); expect(result.store).toBeInstanceOf(InMemoryBackend); - + // Verify connected expect(result.manager.isConnected()).toBe(true); expect(result.store.isConnected()).toBe(true); - + // Verify configuration const info = result.manager.getInfo(); expect(info.backend.type).toBe('in-memory'); expect(info.backend.dimension).toBe(768); expect(info.backend.collectionName).toBe('test_collection'); - + // Cleanup await result.manager.disconnect(); }); - + it('should handle Qdrant backend with fallback to in-memory', async () => { const config: VectorStoreConfig = { type: 'qdrant', @@ -81,44 +82,44 @@ describe('Vector Storage Factory', () => { dimension: 1536, distance: 'Cosine', }; - + const result = await createVectorStore(config); - + // Should fallback to in-memory due to connection failure expect(result.manager).toBeInstanceOf(VectorStoreManager); expect(result.store).toBeInstanceOf(InMemoryBackend); - + const info = result.manager.getInfo(); expect(info.backend.type).toBe('in-memory'); expect(info.backend.fallback).toBe(true); expect(info.backend.dimension).toBe(1536); - + // Cleanup await result.manager.disconnect(); }); - + it('should validate configuration', async () => { const invalidConfig = { type: 'invalid', collectionName: '', dimension: -1, } as any; - + await expect(createVectorStore(invalidConfig)).rejects.toThrow(); }); - + it('should handle connection failures gracefully', async () => { // Mock in-memory to also fail const originalConnect = InMemoryBackend.prototype.connect; InMemoryBackend.prototype.connect = vi.fn().mockRejectedValue(new Error('Connection failed')); - + const config: VectorStoreConfig = { type: 'in-memory', collectionName: 'test', dimension: 128, maxVectors: 100, }; - + try { await expect(createVectorStore(config)).rejects.toThrow('Connection failed'); } finally { @@ -126,7 +127,7 @@ describe('Vector Storage Factory', () => { InMemoryBackend.prototype.connect = originalConnect; } }); - + it('should log creation process', async () => { const config: VectorStoreConfig = { type: 'in-memory', @@ -134,91 +135,91 @@ describe('Vector Storage Factory', () => { dimension: 256, maxVectors: 500, }; - + const result = await createVectorStore(config); - + // Verify successful creation expect(result.manager.isConnected()).toBe(true); - + // Cleanup await result.manager.disconnect(); }); }); - + describe('createDefaultVectorStore', () => { it('should create default vector storage with default parameters', async () => { const result = await createDefaultVectorStore(); - + expect(result.manager).toBeInstanceOf(VectorStoreManager); expect(result.store).toBeInstanceOf(InMemoryBackend); - + const info = result.manager.getInfo(); expect(info.backend.type).toBe('in-memory'); expect(info.backend.collectionName).toBe('default'); expect(info.backend.dimension).toBe(1536); expect(info.backend.fallback).toBe(false); - + // Cleanup await result.manager.disconnect(); }); - + it('should create default vector storage with custom parameters', async () => { const result = await createDefaultVectorStore('custom_collection', 768); - + const info = result.manager.getInfo(); expect(info.backend.collectionName).toBe('custom_collection'); expect(info.backend.dimension).toBe(768); - + // Cleanup await result.manager.disconnect(); }); - + it('should use in-memory backend by default', async () => { const result = await createDefaultVectorStore(); - + expect(result.store).toBeInstanceOf(InMemoryBackend); expect(result.store.getBackendType()).toBe('in-memory'); - + // Cleanup await result.manager.disconnect(); }); }); - + describe('createVectorStoreFromEnv', () => { it('should create default vector storage when no env vars are set', async () => { // Clear relevant env vars delete process.env.VECTOR_STORE_TYPE; delete process.env.VECTOR_STORE_COLLECTION; delete process.env.VECTOR_STORE_DIMENSION; - + const result = await createVectorStoreFromEnv(); - + const info = result.manager.getInfo(); expect(info.backend.type).toBe('in-memory'); expect(info.backend.collectionName).toBe('default'); expect(info.backend.dimension).toBe(1536); - + // Cleanup await result.manager.disconnect(); }); - + it('should create in-memory storage from env vars', async () => { process.env.VECTOR_STORE_TYPE = 'in-memory'; process.env.VECTOR_STORE_COLLECTION = 'env_test_collection'; process.env.VECTOR_STORE_DIMENSION = '512'; process.env.VECTOR_STORE_MAX_VECTORS = '2000'; - + const result = await createVectorStoreFromEnv(); - + const info = result.manager.getInfo(); expect(info.backend.type).toBe('in-memory'); expect(info.backend.collectionName).toBe('env_test_collection'); expect(info.backend.dimension).toBe(512); - + // Cleanup await result.manager.disconnect(); }); - + it('should create Qdrant storage from env vars with fallback', async () => { process.env.VECTOR_STORE_TYPE = 'qdrant'; process.env.VECTOR_STORE_HOST = 'test-host'; @@ -228,95 +229,154 @@ describe('Vector Storage Factory', () => { process.env.VECTOR_STORE_DIMENSION = '1024'; process.env.VECTOR_STORE_DISTANCE = 'Euclidean'; process.env.VECTOR_STORE_ON_DISK = 'true'; - + const result = await createVectorStoreFromEnv(); - + // Will fallback to in-memory due to connection failure const info = result.manager.getInfo(); expect(info.backend.fallback).toBe(true); expect(info.backend.type).toBe('in-memory'); expect(info.backend.dimension).toBe(1024); expect(info.backend.collectionName).toBe('qdrant_collection'); - + // Cleanup await result.manager.disconnect(); }); - + it('should handle URL-based Qdrant configuration', async () => { process.env.VECTOR_STORE_TYPE = 'qdrant'; process.env.VECTOR_STORE_URL = 'http://test-qdrant:6333'; process.env.VECTOR_STORE_COLLECTION = 'url_collection'; process.env.VECTOR_STORE_DIMENSION = '384'; - + const result = await createVectorStoreFromEnv(); - + // Will fallback to in-memory due to connection failure const info = result.manager.getInfo(); expect(info.backend.fallback).toBe(true); expect(info.backend.dimension).toBe(384); expect(info.backend.collectionName).toBe('url_collection'); - + // Cleanup await result.manager.disconnect(); }); - + it('should fallback to in-memory when Qdrant config is incomplete', async () => { process.env.VECTOR_STORE_TYPE = 'qdrant'; // No host or URL provided process.env.VECTOR_STORE_COLLECTION = 'incomplete_config'; process.env.VECTOR_STORE_DIMENSION = '128'; - + const result = await createVectorStoreFromEnv(); - + // Should use in-memory directly (not as fallback) const info = result.manager.getInfo(); expect(info.backend.type).toBe('in-memory'); expect(info.backend.fallback).toBe(false); expect(info.backend.dimension).toBe(128); - + // Cleanup await result.manager.disconnect(); }); - + it('should handle invalid environment values gracefully', async () => { process.env.VECTOR_STORE_TYPE = 'in-memory'; process.env.VECTOR_STORE_DIMENSION = 'invalid-number'; process.env.VECTOR_STORE_MAX_VECTORS = 'also-invalid'; - + const result = await createVectorStoreFromEnv(); - + // Should still create storage with defaults for invalid values expect(result.manager).toBeInstanceOf(VectorStoreManager); - + // Cleanup await result.manager.disconnect(); }); - + it('should log environment configuration details', async () => { process.env.VECTOR_STORE_TYPE = 'in-memory'; process.env.VECTOR_STORE_COLLECTION = 'logged_collection'; process.env.VECTOR_STORE_DIMENSION = '256'; - + const result = await createVectorStoreFromEnv(); - + // Verify successful creation expect(result.manager.isConnected()).toBe(true); - + // Cleanup await result.manager.disconnect(); }); }); - + + describe('getVectorStoreConfigFromEnv', () => { + it('should return in-memory config when no env vars are set', () => { + // Clear relevant env vars + delete process.env.VECTOR_STORE_TYPE; + delete process.env.VECTOR_STORE_COLLECTION; + delete process.env.VECTOR_STORE_DIMENSION; + + const config = getVectorStoreConfigFromEnv(); + + expect(config.type).toBe('in-memory'); + expect(config.collectionName).toBe('default'); + expect(config.dimension).toBe(1536); + expect((config as any).maxVectors).toBe(10000); + }); + + it('should return qdrant config from env vars', () => { + process.env.VECTOR_STORE_TYPE = 'qdrant'; + process.env.VECTOR_STORE_HOST = 'test-host'; + process.env.VECTOR_STORE_PORT = '6334'; + process.env.VECTOR_STORE_COLLECTION = 'test_collection'; + process.env.VECTOR_STORE_DIMENSION = '768'; + process.env.VECTOR_STORE_DISTANCE = 'Euclidean'; + + const config = getVectorStoreConfigFromEnv(); + + expect(config.type).toBe('qdrant'); + expect(config.collectionName).toBe('test_collection'); + expect(config.dimension).toBe(768); + expect((config as any).host).toBe('test-host'); + expect((config as any).port).toBe(6334); + expect((config as any).distance).toBe('Euclidean'); + }); + + it('should fallback to in-memory when qdrant config is incomplete', () => { + process.env.VECTOR_STORE_TYPE = 'qdrant'; + // No host or URL provided + delete process.env.VECTOR_STORE_HOST; + delete process.env.VECTOR_STORE_URL; + process.env.VECTOR_STORE_COLLECTION = 'test_collection'; + + const config = getVectorStoreConfigFromEnv(); + + expect(config.type).toBe('in-memory'); + expect(config.collectionName).toBe('test_collection'); + }); + + it('should handle invalid numeric values gracefully', () => { + process.env.VECTOR_STORE_TYPE = 'in-memory'; + process.env.VECTOR_STORE_DIMENSION = 'invalid-number'; + process.env.VECTOR_STORE_MAX_VECTORS = 'also-invalid'; + + const config = getVectorStoreConfigFromEnv(); + + expect(config.type).toBe('in-memory'); + expect(config.dimension).toBe(1536); // Should fallback to default + expect((config as any).maxVectors).toBe(10000); // Should fallback to default + }); + }); + describe('isVectorStoreFactory', () => { it('should return true for valid VectorStoreFactory objects', async () => { const result = await createDefaultVectorStore(); - + expect(isVectorStoreFactory(result)).toBe(true); - + // Cleanup await result.manager.disconnect(); }); - + it('should return false for invalid objects', () => { expect(isVectorStoreFactory(null)).toBe(false); expect(isVectorStoreFactory(undefined)).toBe(false); @@ -325,32 +385,30 @@ describe('Vector Storage Factory', () => { expect(isVectorStoreFactory({ store: {} })).toBe(false); expect(isVectorStoreFactory({ manager: {}, store: {} })).toBe(false); }); - + it('should return false for objects with wrong types', () => { const fakeFactory = { manager: { isConnected: () => true }, store: { search: () => Promise.resolve([]) }, }; - + expect(isVectorStoreFactory(fakeFactory)).toBe(false); }); }); - + describe('Error Handling', () => { it('should clean up on factory creation failure', async () => { // Mock manager to fail after creation const originalConnect = VectorStoreManager.prototype.connect; - VectorStoreManager.prototype.connect = vi - .fn() - .mockRejectedValue(new Error('Manager connection failed')); - + VectorStoreManager.prototype.connect = vi.fn().mockRejectedValue(new Error('Manager connection failed')); + const config: VectorStoreConfig = { type: 'in-memory', collectionName: 'fail_test', dimension: 128, maxVectors: 100, }; - + try { await expect(createVectorStore(config)).rejects.toThrow('Manager connection failed'); } finally { @@ -358,29 +416,29 @@ describe('Vector Storage Factory', () => { VectorStoreManager.prototype.connect = originalConnect; } }); - + it('should handle malformed configuration gracefully', async () => { const malformedConfig = { type: 'in-memory', // Missing required fields } as any; - + await expect(createVectorStore(malformedConfig)).rejects.toThrow(); }); }); - + describe('Integration Scenarios', () => { it('should support multiple vector stores simultaneously', async () => { const result1 = await createDefaultVectorStore('collection1', 256); const result2 = await createDefaultVectorStore('collection2', 512); - + try { // Both should be independent expect(result1.manager.getInfo().backend.collectionName).toBe('collection1'); expect(result2.manager.getInfo().backend.collectionName).toBe('collection2'); expect(result1.manager.getInfo().backend.dimension).toBe(256); expect(result2.manager.getInfo().backend.dimension).toBe(512); - + // Both should be connected expect(result1.manager.isConnected()).toBe(true); expect(result2.manager.isConnected()).toBe(true); @@ -390,23 +448,23 @@ describe('Vector Storage Factory', () => { await result2.manager.disconnect(); } }); - + it('should handle rapid creation and destruction', async () => { const configs = [ { type: 'in-memory' as const, collectionName: 'rapid1', dimension: 128, maxVectors: 100 }, { type: 'in-memory' as const, collectionName: 'rapid2', dimension: 256, maxVectors: 200 }, { type: 'in-memory' as const, collectionName: 'rapid3', dimension: 512, maxVectors: 300 }, ]; - + const results = []; - + try { // Create multiple stores rapidly for (const config of configs) { const result = await createVectorStore(config); results.push(result); } - + // All should be connected for (const result of results) { expect(result.manager.isConnected()).toBe(true); @@ -419,4 +477,4 @@ describe('Vector Storage Factory', () => { } }); }); -}); +}); \ No newline at end of file diff --git a/src/core/vector_storage/factory.ts b/src/core/vector_storage/factory.ts index 195a29e82..483ccf16d 100644 --- a/src/core/vector_storage/factory.ts +++ b/src/core/vector_storage/factory.ts @@ -148,6 +148,7 @@ export async function createDefaultVectorStore( * - VECTOR_STORE_DIMENSION: Vector dimension * - VECTOR_STORE_DISTANCE: Distance metric for Qdrant * - VECTOR_STORE_ON_DISK: Store vectors on disk (if using Qdrant) + * - VECTOR_STORE_MAX_VECTORS: Maximum vectors for in-memory storage * * @returns Promise resolving to manager and connected vector store * @@ -164,33 +165,52 @@ export async function createDefaultVectorStore( export async function createVectorStoreFromEnv(): Promise { const logger = createLogger({ level: env.CIPHER_LOG_LEVEL }); - // Get configuration from environment - const storeType = process.env.VECTOR_STORE_TYPE || 'in-memory'; - const collectionName = process.env.VECTOR_STORE_COLLECTION || 'default'; - const dimensionStr = process.env.VECTOR_STORE_DIMENSION || '1536'; - const dimension = Number.isNaN(parseInt(dimensionStr, 10)) ? 1536 : parseInt(dimensionStr, 10); + // Get configuration from environment variables + const config = getVectorStoreConfigFromEnv(); logger.info(`${LOG_PREFIXES.FACTORY} Creating vector storage from environment`, { - type: storeType, - collection: collectionName, - dimension, + type: config.type, + collection: config.collectionName, + dimension: config.dimension, }); + return createVectorStore(config); +} + +/** + * Get vector storage configuration from environment variables + * + * Returns the configuration object that would be used by createVectorStoreFromEnv + * without actually creating the vector store. Useful for debugging and validation. + * + * @returns Vector storage configuration based on environment variables + * + * @example + * ```typescript + * const config = getVectorStoreConfigFromEnv(); + * console.log('Vector store configuration:', config); + * + * // Then use the config to create the store + * const { manager, store } = await createVectorStore(config); + * ``` + */ +export function getVectorStoreConfigFromEnv(): VectorStoreConfig { + // Get configuration from centralized env object with fallbacks for invalid values + const storeType = env.VECTOR_STORE_TYPE; + const collectionName = env.VECTOR_STORE_COLLECTION; + const dimension = Number.isNaN(env.VECTOR_STORE_DIMENSION) ? 1536 : env.VECTOR_STORE_DIMENSION; + const maxVectors = Number.isNaN(env.VECTOR_STORE_MAX_VECTORS) ? 10000 : env.VECTOR_STORE_MAX_VECTORS; + // Build configuration based on type let config: VectorStoreConfig; if (storeType === 'qdrant') { - const host = process.env.VECTOR_STORE_HOST; - const url = process.env.VECTOR_STORE_URL; - const portStr = process.env.VECTOR_STORE_PORT; - const port = portStr - ? Number.isNaN(parseInt(portStr, 10)) - ? undefined - : parseInt(portStr, 10) - : undefined; - const apiKey = process.env.VECTOR_STORE_API_KEY; - const distance = process.env.VECTOR_STORE_DISTANCE as any; - const onDisk = process.env.VECTOR_STORE_ON_DISK === 'true'; + const host = env.VECTOR_STORE_HOST; + const url = env.VECTOR_STORE_URL; + const port = Number.isNaN(env.VECTOR_STORE_PORT) ? undefined : env.VECTOR_STORE_PORT; + const apiKey = env.VECTOR_STORE_API_KEY; + const distance = env.VECTOR_STORE_DISTANCE; + const onDisk = env.VECTOR_STORE_ON_DISK; config = { type: 'qdrant', @@ -204,25 +224,17 @@ export async function createVectorStoreFromEnv(): Promise { onDisk, }; - // Validate required fields + // Validate required fields and fallback if necessary if (!url && !host) { - logger.warn(`${LOG_PREFIXES.FACTORY} Qdrant requires URL or host, falling back to in-memory`); config = { type: 'in-memory', collectionName, dimension, - maxVectors: 10000, + maxVectors, }; } } else { // Use in-memory - const maxVectorsStr = process.env.VECTOR_STORE_MAX_VECTORS; - const maxVectors = maxVectorsStr - ? Number.isNaN(parseInt(maxVectorsStr, 10)) - ? 10000 - : parseInt(maxVectorsStr, 10) - : 10000; - config = { type: 'in-memory', collectionName, @@ -231,7 +243,7 @@ export async function createVectorStoreFromEnv(): Promise { }; } - return createVectorStore(config); + return config; } /** diff --git a/src/core/vector_storage/index.ts b/src/core/vector_storage/index.ts index ea2701b50..621b671fa 100644 --- a/src/core/vector_storage/index.ts +++ b/src/core/vector_storage/index.ts @@ -63,6 +63,7 @@ export { createVectorStore, createDefaultVectorStore, createVectorStoreFromEnv, + getVectorStoreConfigFromEnv, isVectorStoreFactory, type VectorStoreFactory, } from './factory.js'; From e48e343188f915977691cf1f9703eeac796d2e6b Mon Sep 17 00:00:00 2001 From: longle325 Date: Sat, 5 Jul 2025 10:34:59 +0700 Subject: [PATCH 3/5] fix insert memory error and env variables --- .env.example | 1 + env.example | 15 ++++-- memAgent/cipher.yml | 4 ++ src/app/index.ts | 7 +++ src/core/vector_storage/backend/qdrant.ts | 22 +++------ src/core/vector_storage/factory.ts | 56 ++++++++++++++++++++++- 6 files changed, 85 insertions(+), 20 deletions(-) diff --git a/.env.example b/.env.example index 5690ddf74..98ef53bb6 100644 --- a/.env.example +++ b/.env.example @@ -49,6 +49,7 @@ STORAGE_DATABASE_TYPE=in-memory # Vector collection settings # VECTOR_STORE_COLLECTION=default +# VECTOR_STORE_COLLECTION_NAME=your_collection_name # VECTOR_STORE_DIMENSION=1536 # VECTOR_STORE_DISTANCE=Cosine # VECTOR_STORE_ON_DISK=false diff --git a/env.example b/env.example index fd38848b4..9b9706f3d 100644 --- a/env.example +++ b/env.example @@ -24,7 +24,7 @@ REDACT_SECRETS=true # OpenAI Configuration # Your OpenAI API key (REQUIRED for embedding functionality, even when using other LLM providers) # This is needed because embeddings currently only support OpenAI -OPENAI_API_KEY=sk-your-openai-api-key +OPENAI_API_KEY=your_openai_api_key_here # OpenAI Organization ID (optional) OPENAI_ORG_ID=org-your-organization-id @@ -35,12 +35,12 @@ OPENAI_BASE_URL=https://api.openai.com/v1 # Anthropic Configuration # Your Anthropic API key (required for Anthropic/Claude services) # Note: You still need OPENAI_API_KEY above for embedding functionality -ANTHROPIC_API_KEY=sk-ant-your-anthropic-api-key-here +ANTHROPIC_API_KEY=your_anthropic_api_key_here # OpenRouter Configuration # Your OpenRouter API key (required for OpenRouter services) # Note: You still need OPENAI_API_KEY above for embedding functionality -OPENROUTER_API_KEY=sk-or-your-openrouter-api-key-here +OPENROUTER_API_KEY=your_openrouter_api_key_here # ============================================================================= # EMBEDDING CONFIGURATION @@ -109,6 +109,15 @@ VECTOR_STORE_ON_DISK=false # Maximum number of vectors for in-memory storage VECTOR_STORE_MAX_VECTORS=10000 +# Qdrant Cloud Configuration +# Format: https://xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.us-east-0-1.aws.cloud.qdrant.io +QDRANT_URL=https://your-cluster-url.region.cloud.qdrant.io +QDRANT_API_KEY=your_qdrant_cloud_api_key_here + +# Qdrant Local Configuration (alternative to cloud) +# QDRANT_HOST=localhost +# QDRANT_PORT=6333 + # ============================================================================= # DEVELOPMENT NOTES # ============================================================================= diff --git a/memAgent/cipher.yml b/memAgent/cipher.yml index 01290e0b0..937d39227 100644 --- a/memAgent/cipher.yml +++ b/memAgent/cipher.yml @@ -26,3 +26,7 @@ llm: # System prompt systemPrompt: "You are a helpful AI assistant with memory capabilities. Please confirm you're working with OpenRouter API." + + + + diff --git a/src/app/index.ts b/src/app/index.ts index 05a171531..9e6213497 100644 --- a/src/app/index.ts +++ b/src/app/index.ts @@ -68,6 +68,13 @@ program // Start the agent (initialize async services) await agent.start(); + + // Print OpenAI embedder dimension after agent is started + if (agent.services && agent.services.embeddingManager) { + const embedder = agent.services.embeddingManager.getEmbedder('default'); + } else { + console.log('No embeddingManager found in agent.services'); + } } catch (err) { logger.error( 'Failed to load agent config:', diff --git a/src/core/vector_storage/backend/qdrant.ts b/src/core/vector_storage/backend/qdrant.ts index 9e452ae11..2c628af7f 100644 --- a/src/core/vector_storage/backend/qdrant.ts +++ b/src/core/vector_storage/backend/qdrant.ts @@ -198,26 +198,16 @@ export class QdrantBackend implements VectorStore { try { const points = vectors.map((vector, idx) => { - const id = ids[idx]; const payload = payloads[idx]; - - if (!id || !payload) { - throw new VectorStoreError( - `Invalid input at index ${idx}: id and payload are required`, - 'insert' - ); - } - + if (!payload) throw new VectorStoreError(`Payload missing at index ${idx}`, 'insert'); return { - id: id, - vector: vector, - payload: payload, + id: idx + 1, + vector, + payload, }; }); - await this.client.upsert(this.collectionName, { - points, - }); + await this.client.upsert(this.collectionName, { points }); this.logger.info(`${LOG_PREFIXES.INDEX} Successfully inserted ${vectors.length} vectors`); } catch (error) { @@ -521,4 +511,4 @@ export class QdrantBackend implements VectorStore { getCollectionName(): string { return this.collectionName; } -} +} \ No newline at end of file diff --git a/src/core/vector_storage/factory.ts b/src/core/vector_storage/factory.ts index 483ccf16d..ed6ff3f72 100644 --- a/src/core/vector_storage/factory.ts +++ b/src/core/vector_storage/factory.ts @@ -8,10 +8,15 @@ */ import { VectorStoreManager } from './manager.js'; -import type { VectorStoreConfig, VectorStore } from './types.js'; +import type { VectorStoreConfig } from './types.js'; +import { VectorStore } from './backend/vector-store.js'; +import type { BackendConfig, QdrantBackendConfig } from './config.js'; +import { InMemoryBackend } from './backend/in-memory.js'; +import { QdrantBackend } from './backend/qdrant.js'; import { Logger, createLogger } from '../logger/index.js'; import { LOG_PREFIXES } from './constants.js'; import { env } from '../env.js'; +import * as fs from 'node:fs'; /** * Factory result containing both the manager and vector store @@ -261,3 +266,52 @@ export function isVectorStoreFactory(obj: unknown): obj is VectorStoreFactory { obj.manager instanceof VectorStoreManager ); } + +/** + * Get Qdrant configuration from environment variables + * Supports both cloud and local configurations + */ +export function getQdrantConfigFromEnv(): QdrantBackendConfig | null { + const qdrantUrl = process.env.VECTOR_STORE_URL; + const qdrantApiKey = process.env.VECTOR_STORE_API_KEY; + const qdrantHost = process.env.VECTOR_STORE_HOST; + const qdrantPort = process.env.VECTOR_STORE_PORT; + + // Check if we have cloud configuration + if (qdrantUrl) { + return { + type: 'qdrant', + url: qdrantUrl, + apiKey: qdrantApiKey, // API key is required for cloud + collectionName: process.env.VECTOR_STORE_COLLECTION_NAME, + dimension: parseInt(process.env.VECTOR_STORE_DIMENSION || '1536', 10), + distance: (process.env.VECTOR_STORE_DISTANCE as any) || 'Cosine', + }; + } + + // Check if we have local configuration + if (qdrantHost || qdrantPort) { + return { + type: 'qdrant', + host: qdrantHost || 'localhost', + port: qdrantPort ? parseInt(qdrantPort, 10) : 6333, + apiKey: qdrantApiKey, // Optional for local + collectionName: process.env.VECTOR_STORE_COLLECTION_NAME, + dimension: parseInt(process.env.VECTOR_STORE_DIMENSION || '1536', 10), + distance: (process.env.VECTOR_STORE_DISTANCE as any) || 'Cosine', + }; + } + + return null; +} + +/** + * Check if Qdrant configuration is available in environment + */ +export function isQdrantConfigAvailable(): boolean { + return !!( + process.env.VECTOR_STORE_URL || + process.env.VECTOR_STORE_HOST || + process.env.VECTOR_STORE_PORT + ); +} From b4bed7309cce30727103c7fc4b5bfa9d24e0b670 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=AA=20B=E1=BA=A3o=20Long?= <140832783+longle325@users.noreply.github.com> Date: Sat, 5 Jul 2025 10:37:05 +0700 Subject: [PATCH 4/5] Delete env.example --- env.example | 164 ---------------------------------------------------- 1 file changed, 164 deletions(-) delete mode 100644 env.example diff --git a/env.example b/env.example deleted file mode 100644 index 9b9706f3d..000000000 --- a/env.example +++ /dev/null @@ -1,164 +0,0 @@ -# ============================================================================= -# CIPHER PROJECT ENVIRONMENT CONFIGURATION -# ============================================================================= -# This file contains all available environment variables for the Cipher project. -# Copy this file to .env and configure the values according to your setup. - -# ============================================================================= -# GENERAL CONFIGURATION -# ============================================================================= - -# Environment mode: development, production, or test -NODE_ENV=development - -# Logging level: debug, info, warn, error -CIPHER_LOG_LEVEL=info - -# Whether to redact secrets in logs (true/false) -REDACT_SECRETS=true - -# ============================================================================= -# LLM PROVIDER CONFIGURATION -# ============================================================================= - -# OpenAI Configuration -# Your OpenAI API key (REQUIRED for embedding functionality, even when using other LLM providers) -# This is needed because embeddings currently only support OpenAI -OPENAI_API_KEY=your_openai_api_key_here - -# OpenAI Organization ID (optional) -OPENAI_ORG_ID=org-your-organization-id - -# OpenAI API Base URL (optional, defaults to https://api.openai.com/v1) -OPENAI_BASE_URL=https://api.openai.com/v1 - -# Anthropic Configuration -# Your Anthropic API key (required for Anthropic/Claude services) -# Note: You still need OPENAI_API_KEY above for embedding functionality -ANTHROPIC_API_KEY=your_anthropic_api_key_here - -# OpenRouter Configuration -# Your OpenRouter API key (required for OpenRouter services) -# Note: You still need OPENAI_API_KEY above for embedding functionality -OPENROUTER_API_KEY=your_openrouter_api_key_here - -# ============================================================================= -# EMBEDDING CONFIGURATION -# ============================================================================= -# IMPORTANT: Embeddings currently only support OpenAI. -# OPENAI_API_KEY is REQUIRED above, regardless of which LLM provider you use. - -# Embedding model to use (optional) -# Supported OpenAI models: text-embedding-3-small, text-embedding-3-large, text-embedding-ada-002 -# Default: text-embedding-3-small -EMBEDDING_MODEL=text-embedding-3-small - -# Request timeout for embedding operations in milliseconds (optional) -# Default: 30000 (30 seconds) -EMBEDDING_TIMEOUT=30000 - -# Maximum number of retry attempts for failed embedding requests (optional) -# Default: 3 -EMBEDDING_MAX_RETRIES=3 - -# ============================================================================= -# STORAGE CONFIGURATION -# ============================================================================= - -# Cache Storage Configuration -# Storage type for caching: redis, in-memory -STORAGE_CACHE_TYPE=in-memory - -# Redis configuration (only used if STORAGE_CACHE_TYPE=redis) -STORAGE_CACHE_HOST=localhost -STORAGE_CACHE_PORT=6379 -STORAGE_CACHE_PASSWORD=your-redis-password -STORAGE_CACHE_DATABASE=0 - -# Database Storage Configuration -# Database type: sqlite, in-memory -STORAGE_DATABASE_TYPE=in-memory - -# SQLite configuration (only used if STORAGE_DATABASE_TYPE=sqlite) -STORAGE_DATABASE_PATH=./data/cipher.db -STORAGE_DATABASE_NAME=cipher - -# ============================================================================= -# VECTOR STORAGE CONFIGURATION -# ============================================================================= - -# Vector store type: qdrant, in-memory -VECTOR_STORE_TYPE=in-memory - -# Qdrant configuration (only used if VECTOR_STORE_TYPE=qdrant) -VECTOR_STORE_HOST=localhost -VECTOR_STORE_PORT=6333 -VECTOR_STORE_URL=http://localhost:6333 -VECTOR_STORE_API_KEY=your-qdrant-api-key - -# Vector collection settings -VECTOR_STORE_COLLECTION=default -VECTOR_STORE_DIMENSION=1536 - -# Distance metric: Cosine, Euclidean, Dot, Manhattan -VECTOR_STORE_DISTANCE=Cosine - -# Whether to store vectors on disk (true/false) -VECTOR_STORE_ON_DISK=false - -# Maximum number of vectors for in-memory storage -VECTOR_STORE_MAX_VECTORS=10000 - -# Qdrant Cloud Configuration -# Format: https://xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.us-east-0-1.aws.cloud.qdrant.io -QDRANT_URL=https://your-cluster-url.region.cloud.qdrant.io -QDRANT_API_KEY=your_qdrant_cloud_api_key_here - -# Qdrant Local Configuration (alternative to cloud) -# QDRANT_HOST=localhost -# QDRANT_PORT=6333 - -# ============================================================================= -# DEVELOPMENT NOTES -# ============================================================================= -# -# Required Variables: -# - OPENAI_API_KEY (ALWAYS required for embedding functionality) -# - At least one LLM provider API key for chat/completion services: -# * OPENAI_API_KEY (can serve both LLM and embedding needs) -# * ANTHROPIC_API_KEY (for Claude, but still need OPENAI_API_KEY for embeddings) -# * OPENROUTER_API_KEY (for OpenRouter, but still need OPENAI_API_KEY for embeddings) -# -# Optional Variables: -# - All other variables have sensible defaults and can be omitted -# -# Common Configurations: -# -# 1. OpenAI for both LLM and embeddings (minimal setup): -# NODE_ENV=development -# CIPHER_LOG_LEVEL=debug -# OPENAI_API_KEY=sk-your-openai-key-here -# -# 2. Anthropic for LLM + OpenAI for embeddings: -# NODE_ENV=development -# OPENAI_API_KEY=sk-your-openai-key-here -# ANTHROPIC_API_KEY=sk-ant-your-anthropic-key-here -# -# 3. Production with Qdrant and Anthropic: -# NODE_ENV=production -# CIPHER_LOG_LEVEL=info -# OPENAI_API_KEY=sk-your-openai-key-here -# ANTHROPIC_API_KEY=sk-ant-your-anthropic-key-here -# VECTOR_STORE_TYPE=qdrant -# VECTOR_STORE_URL=https://your-qdrant-instance.com -# VECTOR_STORE_API_KEY=your-qdrant-key -# -# 4. Development with Redis cache and mixed providers: -# NODE_ENV=development -# OPENAI_API_KEY=sk-your-openai-key-here -# ANTHROPIC_API_KEY=sk-ant-your-anthropic-key-here -# STORAGE_CACHE_TYPE=redis -# STORAGE_CACHE_HOST=localhost -# STORAGE_CACHE_PORT=6379 -# -# ============================================================================= \ No newline at end of file From 94bf72fbbb3b4c7b37eaa85967f04c7713d53ccd Mon Sep 17 00:00:00 2001 From: longle325 Date: Sat, 5 Jul 2025 12:16:33 +0700 Subject: [PATCH 5/5] fix qdrant test case --- src/core/brain/embedding/backend/index.ts | 2 +- src/core/brain/embedding/backend/openai.ts | 123 +++---- src/core/brain/embedding/backend/types.ts | 4 +- src/core/brain/embedding/config.ts | 71 ++-- src/core/brain/embedding/constants.ts | 5 +- src/core/brain/embedding/factory.ts | 49 +-- src/core/brain/embedding/index.ts | 12 +- src/core/brain/embedding/manager.ts | 61 ++-- src/core/brain/embedding/types.ts | 2 +- src/core/brain/embedding/utils.ts | 32 +- src/core/brain/index.ts | 2 +- .../brain/tools/__test__/definitions.test.ts | 4 +- .../definitions/memory/extract-knowledge.ts | 6 +- .../definitions/memory/memory_operation.ts | 304 +++++++++++------- src/core/brain/tools/manager.ts | 22 +- src/core/brain/tools/registry.ts | 6 +- src/core/env.ts | 4 +- src/core/storage/manager.ts | 24 +- src/core/utils/service-initializer.ts | 22 +- .../vector_storage/__test__/factory.test.ts | 188 +++++------ .../vector_storage/__test__/qdrant.test.ts | 5 +- src/core/vector_storage/factory.ts | 14 +- src/core/vector_storage/manager.ts | 12 +- 23 files changed, 500 insertions(+), 474 deletions(-) diff --git a/src/core/brain/embedding/backend/index.ts b/src/core/brain/embedding/backend/index.ts index ca0debf40..93083c4b1 100644 --- a/src/core/brain/embedding/backend/index.ts +++ b/src/core/brain/embedding/backend/index.ts @@ -28,4 +28,4 @@ export { } from './types.js'; // Export backend implementations -export { OpenAIEmbedder } from './openai.js'; \ No newline at end of file +export { OpenAIEmbedder } from './openai.js'; diff --git a/src/core/brain/embedding/backend/openai.ts b/src/core/brain/embedding/backend/openai.ts index 73edd1132..2248dc0ed 100644 --- a/src/core/brain/embedding/backend/openai.ts +++ b/src/core/brain/embedding/backend/openai.ts @@ -10,23 +10,23 @@ import OpenAI from 'openai'; import { logger } from '../../../logger/index.js'; -import { - Embedder, - OpenAIEmbeddingConfig, +import { + Embedder, + OpenAIEmbeddingConfig, EmbeddingConnectionError, EmbeddingRateLimitError, EmbeddingQuotaError, EmbeddingValidationError, EmbeddingError, - EmbeddingDimensionError + EmbeddingDimensionError, } from './types.js'; -import { - MODEL_DIMENSIONS, - VALIDATION_LIMITS, - ERROR_MESSAGES, - LOG_PREFIXES, +import { + MODEL_DIMENSIONS, + VALIDATION_LIMITS, + ERROR_MESSAGES, + LOG_PREFIXES, RETRY_CONFIG, - HTTP_STATUS + HTTP_STATUS, } from '../constants.js'; /** @@ -55,7 +55,8 @@ export class OpenAIEmbedder implements Embedder { }); // Set dimension based on model and config - this.dimension = config.dimensions || MODEL_DIMENSIONS[this.model]; + this.dimension = + config.dimensions || MODEL_DIMENSIONS[this.model as keyof typeof MODEL_DIMENSIONS] || 1536; logger.debug(`${LOG_PREFIXES.OPENAI} Initialized OpenAI embedder`, { model: this.model, @@ -77,25 +78,24 @@ export class OpenAIEmbedder implements Embedder { const startTime = Date.now(); try { - const response = await this.createEmbeddingWithRetry({ + const params: { model: string; input: string; dimensions?: number } = { model: this.model, input: text, - dimensions: this.config.dimensions, - }); - + }; + if (this.config.dimensions !== undefined) { + params.dimensions = this.config.dimensions; + } + const response = await this.createEmbeddingWithRetry(params); + if ( + !response.data || + !Array.isArray(response.data) || + !response.data[0] || + !response.data[0].embedding + ) { + throw new EmbeddingError('OpenAI API did not return a valid embedding', 'openai'); + } const embedding = response.data[0].embedding; - const processingTime = Date.now() - startTime; - - // Validate embedding dimension this.validateEmbeddingDimension(embedding); - - logger.debug(`${LOG_PREFIXES.OPENAI} Successfully created embedding`, { - model: this.model, - dimension: embedding.length, - processingTime, - usage: response.usage, - }); - return embedding; } catch (error) { const processingTime = Date.now() - startTime; @@ -122,37 +122,16 @@ export class OpenAIEmbedder implements Embedder { const startTime = Date.now(); try { - const response = await this.createEmbeddingWithRetry({ + const batchParams: { model: string; input: string[]; dimensions?: number } = { model: this.model, input: texts, - dimensions: this.config.dimensions, - }); - - const embeddings = response.data.map((item) => item.embedding); - const processingTime = Date.now() - startTime; - - // Validate all embedding dimensions - embeddings.forEach((embedding, index) => { - try { - this.validateEmbeddingDimension(embedding); - } catch (error) { - logger.error(`${LOG_PREFIXES.BATCH} Invalid embedding dimension at index ${index}`, { - index, - dimension: embedding.length, - expected: this.dimension, - }); - throw error; - } - }); - - logger.debug(`${LOG_PREFIXES.BATCH} Successfully created batch embeddings`, { - model: this.model, - count: embeddings.length, - dimension: embeddings[0]?.length, - processingTime, - usage: response.usage, - }); - + }; + if (this.config.dimensions !== undefined) { + batchParams.dimensions = this.config.dimensions; + } + const response = await this.createEmbeddingWithRetry(batchParams); + const embeddings = response.data.map(item => item.embedding); + embeddings.forEach(this.validateEmbeddingDimension.bind(this)); return embeddings; } catch (error) { const processingTime = Date.now() - startTime; @@ -222,10 +201,7 @@ export class OpenAIEmbedder implements Embedder { await new Promise(resolve => setTimeout(resolve, delay)); // Calculate next delay with exponential backoff and jitter - delay = Math.min( - delay * RETRY_CONFIG.BACKOFF_MULTIPLIER, - RETRY_CONFIG.MAX_DELAY - ); + delay = Math.min(delay * RETRY_CONFIG.BACKOFF_MULTIPLIER, RETRY_CONFIG.MAX_DELAY); // Add jitter to avoid thundering herd const jitter = delay * RETRY_CONFIG.JITTER_FACTOR * Math.random(); @@ -233,7 +209,7 @@ export class OpenAIEmbedder implements Embedder { } const response = await this.openai.embeddings.create(params); - + if (attempt > 0) { logger.info(`${LOG_PREFIXES.OPENAI} Embedding request succeeded after retry`, { attempt, @@ -274,7 +250,7 @@ export class OpenAIEmbedder implements Embedder { // Handle OpenAI API errors if (error && typeof error === 'object' && 'status' in error) { const status = (error as any).status; - + // Retry on server errors and rate limits return [ HTTP_STATUS.TOO_MANY_REQUESTS, @@ -326,18 +302,10 @@ export class OpenAIEmbedder implements Embedder { } case HTTP_STATUS.FORBIDDEN: - return new EmbeddingQuotaError( - ERROR_MESSAGES.QUOTA_EXCEEDED, - 'openai', - apiError - ); + return new EmbeddingQuotaError(ERROR_MESSAGES.QUOTA_EXCEEDED, 'openai', apiError); case HTTP_STATUS.BAD_REQUEST: - return new EmbeddingValidationError( - message, - 'openai', - apiError - ); + return new EmbeddingValidationError(message, 'openai', apiError); default: return new EmbeddingConnectionError( @@ -350,17 +318,10 @@ export class OpenAIEmbedder implements Embedder { // Handle network and other errors if (error instanceof Error) { - return new EmbeddingConnectionError( - error.message, - 'openai', - error - ); + return new EmbeddingConnectionError(error.message, 'openai', error); } - return new EmbeddingError( - String(error), - 'openai' - ); + return new EmbeddingError(String(error), 'openai'); } /** @@ -424,4 +385,4 @@ export class OpenAIEmbedder implements Embedder { ); } } -} \ No newline at end of file +} diff --git a/src/core/brain/embedding/backend/types.ts b/src/core/brain/embedding/backend/types.ts index 29bffb8d7..95f3e8d03 100644 --- a/src/core/brain/embedding/backend/types.ts +++ b/src/core/brain/embedding/backend/types.ts @@ -145,7 +145,7 @@ export class EmbeddingError extends Error { constructor( message: string, public readonly provider?: string, - public readonly cause?: Error + public override readonly cause?: Error ) { super(message); this.name = 'EmbeddingError'; @@ -210,4 +210,4 @@ export class EmbeddingValidationError extends EmbeddingError { super(message, provider, cause); this.name = 'EmbeddingValidationError'; } -} \ No newline at end of file +} diff --git a/src/core/brain/embedding/config.ts b/src/core/brain/embedding/config.ts index 85a23d65a..ef92e51ef 100644 --- a/src/core/brain/embedding/config.ts +++ b/src/core/brain/embedding/config.ts @@ -9,12 +9,12 @@ */ import { z } from 'zod'; -import { - DEFAULTS, - OPENAI_MODELS, - PROVIDER_TYPES, +import { + DEFAULTS, + OPENAI_MODELS, + PROVIDER_TYPES, VALIDATION_LIMITS, - ENV_VARS + ENV_VARS, } from './constants.js'; /** @@ -25,24 +25,13 @@ import { */ const BaseEmbeddingSchema = z.object({ /** API key for the provider (required for all providers) */ - apiKey: z - .string() - .min(1) - .describe('API key for the embedding provider'), + apiKey: z.string().min(1).describe('API key for the embedding provider'), /** Model name to use for embeddings */ - model: z - .string() - .min(1) - .optional() - .describe('Model name for embeddings'), + model: z.string().min(1).optional().describe('Model name for embeddings'), /** Base URL for the provider API */ - baseUrl: z - .string() - .url() - .optional() - .describe('Base URL for the provider API'), + baseUrl: z.string().url().optional().describe('Base URL for the provider API'), /** Request timeout in milliseconds */ timeout: z @@ -63,10 +52,7 @@ const BaseEmbeddingSchema = z.object({ .describe('Maximum retry attempts'), /** Provider-specific options */ - options: z - .record(z.any()) - .optional() - .describe('Provider-specific configuration options'), + options: z.record(z.any()).optional().describe('Provider-specific configuration options'), }); /** @@ -98,10 +84,7 @@ const OpenAIEmbeddingSchema = BaseEmbeddingSchema.extend({ .describe('OpenAI embedding model'), /** OpenAI organization ID */ - organization: z - .string() - .optional() - .describe('OpenAI organization ID'), + organization: z.string().optional().describe('OpenAI organization ID'), /** Custom dimensions for embedding-3 models */ dimensions: z @@ -113,11 +96,7 @@ const OpenAIEmbeddingSchema = BaseEmbeddingSchema.extend({ .describe('Custom embedding dimensions (embedding-3 models only)'), /** Base URL override */ - baseUrl: z - .string() - .url() - .default(DEFAULTS.OPENAI_BASE_URL) - .describe('OpenAI API base URL'), + baseUrl: z.string().url().default(DEFAULTS.OPENAI_BASE_URL).describe('OpenAI API base URL'), }).strict(); export type OpenAIEmbeddingConfig = z.infer; @@ -145,7 +124,7 @@ const BackendConfigSchema = z if (data.type === PROVIDER_TYPES.OPENAI) { // Check if dimensions are specified for models that support it if (data.dimensions) { - const supportsCustomDimensions = + const supportsCustomDimensions = data.model === OPENAI_MODELS.TEXT_EMBEDDING_3_SMALL || data.model === OPENAI_MODELS.TEXT_EMBEDDING_3_LARGE; @@ -206,28 +185,18 @@ export const EmbeddingEnvConfigSchema = z.object({ .describe('Embedding provider type'), /** API key from environment variables */ - apiKey: z - .string() - .optional() - .describe('API key from environment variables'), + apiKey: z.string().optional().describe('API key from environment variables'), /** Model from environment */ - model: z - .string() - .optional() - .describe('Model name from environment'), + model: z.string().optional().describe('Model name from environment'), /** Base URL from environment */ - baseUrl: z - .string() - .url() - .optional() - .describe('Base URL from environment'), + baseUrl: z.string().url().optional().describe('Base URL from environment'), /** Timeout from environment */ timeout: z .string() - .transform((val) => parseInt(val, 10)) + .transform(val => parseInt(val, 10)) .pipe(z.number().int().positive()) .optional() .describe('Timeout from environment (string converted to number)'), @@ -235,7 +204,7 @@ export const EmbeddingEnvConfigSchema = z.object({ /** Max retries from environment */ maxRetries: z .string() - .transform((val) => parseInt(val, 10)) + .transform(val => parseInt(val, 10)) .pipe(z.number().int().min(0).max(10)) .optional() .describe('Max retries from environment (string converted to number)'), @@ -290,11 +259,11 @@ export function parseEmbeddingConfigFromEnv( } if (env[ENV_VARS.EMBEDDING_TIMEOUT]) { - rawConfig.timeout = parseInt(env[ENV_VARS.EMBEDDING_TIMEOUT], 10); + rawConfig.timeout = parseInt(env[ENV_VARS.EMBEDDING_TIMEOUT] ?? '30000', 10); } if (env[ENV_VARS.EMBEDDING_MAX_RETRIES]) { - rawConfig.maxRetries = parseInt(env[ENV_VARS.EMBEDDING_MAX_RETRIES], 10); + rawConfig.maxRetries = parseInt(env[ENV_VARS.EMBEDDING_MAX_RETRIES] ?? '3', 10); } return parseEmbeddingConfig(rawConfig); @@ -324,4 +293,4 @@ export function validateEmbeddingConfig(config: unknown): { } throw error; } -} \ No newline at end of file +} diff --git a/src/core/brain/embedding/constants.ts b/src/core/brain/embedding/constants.ts index 2069d389b..119f9bdfd 100644 --- a/src/core/brain/embedding/constants.ts +++ b/src/core/brain/embedding/constants.ts @@ -102,8 +102,7 @@ export const VALIDATION_LIMITS = { * Error messages */ export const ERROR_MESSAGES = { - PROVIDER_NOT_SUPPORTED: (provider: string) => - `Embedding provider '${provider}' is not supported`, + PROVIDER_NOT_SUPPORTED: (provider: string) => `Embedding provider '${provider}' is not supported`, MODEL_NOT_SUPPORTED: (model: string, provider: string) => `Model '${model}' is not supported by provider '${provider}'`, @@ -175,4 +174,4 @@ export const HTTP_STATUS = { */ export const CONTENT_TYPES = { JSON: 'application/json', -} as const; \ No newline at end of file +} as const; diff --git a/src/core/brain/embedding/factory.ts b/src/core/brain/embedding/factory.ts index a43b76682..02aa0fb4a 100644 --- a/src/core/brain/embedding/factory.ts +++ b/src/core/brain/embedding/factory.ts @@ -9,25 +9,20 @@ */ import { logger } from '../../logger/index.js'; -import { +import { parseEmbeddingConfigFromEnv, validateEmbeddingConfig, type OpenAIEmbeddingConfig as ZodOpenAIEmbeddingConfig, - type EmbeddingConfig as ZodEmbeddingConfig + type EmbeddingConfig as ZodEmbeddingConfig, } from './config.js'; -import { +import { type Embedder, type OpenAIEmbeddingConfig as InterfaceOpenAIEmbeddingConfig, EmbeddingError, EmbeddingValidationError, - OpenAIEmbedder + OpenAIEmbedder, } from './backend/index.js'; -import { - PROVIDER_TYPES, - ERROR_MESSAGES, - LOG_PREFIXES, - DEFAULTS -} from './constants.js'; +import { PROVIDER_TYPES, ERROR_MESSAGES, LOG_PREFIXES, DEFAULTS } from './constants.js'; // Use Zod-inferred types for validation, but convert to interface types for backend export type BackendConfig = ZodOpenAIEmbeddingConfig; @@ -101,10 +96,7 @@ class OpenAIEmbeddingFactory implements EmbeddingFactory { // Test the connection const isHealthy = await embedder.isHealthy(); if (!isHealthy) { - throw new EmbeddingError( - ERROR_MESSAGES.CONNECTION_FAILED('OpenAI'), - 'openai' - ); + throw new EmbeddingError(ERROR_MESSAGES.CONNECTION_FAILED('OpenAI'), 'openai'); } logger.info(`${LOG_PREFIXES.FACTORY} Successfully created OpenAI embedder`, { @@ -134,8 +126,7 @@ class OpenAIEmbeddingFactory implements EmbeddingFactory { validateConfig(config: unknown): boolean { try { const validationResult = validateEmbeddingConfig(config); - return validationResult.success && - validationResult.data?.type === PROVIDER_TYPES.OPENAI; + return validationResult.success && validationResult.data?.type === PROVIDER_TYPES.OPENAI; } catch { return false; } @@ -178,18 +169,17 @@ export async function createEmbedder(config: BackendConfig): Promise { // Validate configuration const validationResult = validateEmbeddingConfig(config); if (!validationResult.success) { - const errorMessage = validationResult.errors?.issues - .map(issue => `${issue.path.join('.')}: ${issue.message}`) - .join(', ') || 'Invalid configuration'; + const errorMessage = + validationResult.errors?.issues + .map(issue => `${issue.path.join('.')}: ${issue.message}`) + .join(', ') || 'Invalid configuration'; logger.error(`${LOG_PREFIXES.FACTORY} Configuration validation failed`, { type: config.type, errors: errorMessage, }); - throw new EmbeddingValidationError( - `Configuration validation failed: ${errorMessage}` - ); + throw new EmbeddingValidationError(`Configuration validation failed: ${errorMessage}`); } // Get factory for provider type @@ -200,9 +190,7 @@ export async function createEmbedder(config: BackendConfig): Promise { supportedTypes: Array.from(EMBEDDING_FACTORIES.keys()), }); - throw new EmbeddingValidationError( - ERROR_MESSAGES.PROVIDER_NOT_SUPPORTED(config.type) - ); + throw new EmbeddingValidationError(ERROR_MESSAGES.PROVIDER_NOT_SUPPORTED(config.type)); } // Create embedder instance @@ -294,9 +282,7 @@ export async function createDefaultEmbedder(): Promise { const apiKey = process.env.OPENAI_API_KEY; if (!apiKey) { - throw new EmbeddingValidationError( - ERROR_MESSAGES.API_KEY_REQUIRED('OpenAI') - ); + throw new EmbeddingValidationError(ERROR_MESSAGES.API_KEY_REQUIRED('OpenAI')); } const openaiConfig: ZodOpenAIEmbeddingConfig = { @@ -345,13 +331,10 @@ export function getEmbeddingFactory(providerType: string): EmbeddingFactory | un * @param providerType - Provider type * @param factory - Factory instance */ -export function registerEmbeddingFactory( - providerType: string, - factory: EmbeddingFactory -): void { +export function registerEmbeddingFactory(providerType: string, factory: EmbeddingFactory): void { logger.debug(`${LOG_PREFIXES.FACTORY} Registering embedding factory`, { providerType, }); EMBEDDING_FACTORIES.set(providerType, factory); -} \ No newline at end of file +} diff --git a/src/core/brain/embedding/index.ts b/src/core/brain/embedding/index.ts index 3ae5eb8d4..202959e9f 100644 --- a/src/core/brain/embedding/index.ts +++ b/src/core/brain/embedding/index.ts @@ -95,12 +95,12 @@ export { } from './config.js'; // Export constants for external use -export { - PROVIDER_TYPES, - OPENAI_MODELS, - MODEL_DIMENSIONS, +export { + PROVIDER_TYPES, + OPENAI_MODELS, + MODEL_DIMENSIONS, DEFAULTS, - VALIDATION_LIMITS + VALIDATION_LIMITS, } from './constants.js'; // Export utilities @@ -110,4 +110,4 @@ export { getEmbeddingConfigSummary, validateEmbeddingEnv, analyzeProviderConfiguration, -} from './utils.js'; \ No newline at end of file +} from './utils.js'; diff --git a/src/core/brain/embedding/manager.ts b/src/core/brain/embedding/manager.ts index f42220fb8..34185c4cd 100644 --- a/src/core/brain/embedding/manager.ts +++ b/src/core/brain/embedding/manager.ts @@ -9,11 +9,11 @@ */ import { logger } from '../../logger/index.js'; -import { - type Embedder, +import { + type Embedder, type BackendConfig, EmbeddingError, - EmbeddingConnectionError + EmbeddingConnectionError, } from './backend/index.js'; import { createEmbedder, createEmbedderFromEnv } from './factory.js'; import { LOG_PREFIXES, ERROR_MESSAGES } from './constants.js'; @@ -27,7 +27,7 @@ export interface HealthCheckResult { /** Provider type */ provider: string; /** Model being used */ - model?: string; + model: string; /** Embedding dimension */ dimension?: number; /** Response time in milliseconds */ @@ -47,7 +47,7 @@ export interface EmbedderInfo { /** Provider type */ provider: string; /** Model being used */ - model?: string; + model: string; /** Embedding dimension */ dimension: number; /** Configuration used */ @@ -96,7 +96,7 @@ export class EmbeddingManager { failedOperations: 0, averageProcessingTime: 0, }; - private healthCheckInterval?: NodeJS.Timeout; + private healthCheckInterval: NodeJS.Timeout | undefined; constructor() { logger.debug(`${LOG_PREFIXES.MANAGER} Embedding manager initialized`); @@ -110,7 +110,7 @@ export class EmbeddingManager { * @returns Promise resolving to embedder instance and info */ async createEmbedder( - config: BackendConfig, + config: BackendConfig, id?: string ): Promise<{ embedder: Embedder; info: EmbedderInfo }> { const embedderId = id || this.generateId(); @@ -121,12 +121,13 @@ export class EmbeddingManager { }); try { - const embedder = await createEmbedder(config); - + const configWithApiKey = { ...config, apiKey: config.apiKey || '' }; + const embedder = await createEmbedder(configWithApiKey as any); + const info: EmbedderInfo = { id: embedderId, provider: config.type, - model: config.model, + model: config.model || 'unknown', dimension: embedder.getDimension(), config, createdAt: new Date(), @@ -177,7 +178,7 @@ export class EmbeddingManager { const info: EmbedderInfo = { id: embedderId, provider: config.type, - model: config.model, + model: config.model || 'unknown', dimension: embedder.getDimension(), config, createdAt: new Date(), @@ -293,7 +294,7 @@ export class EmbeddingManager { const result: HealthCheckResult = { healthy, provider: info.provider, - model: info.model, + model: info.model || 'unknown', dimension: info.dimension, responseTime, timestamp: new Date(), @@ -317,7 +318,7 @@ export class EmbeddingManager { const result: HealthCheckResult = { healthy: false, provider: info.provider, - model: info.model, + model: info.model || 'unknown', dimension: info.dimension, responseTime, error: error instanceof Error ? error.message : String(error), @@ -349,7 +350,7 @@ export class EmbeddingManager { logger.debug(`${LOG_PREFIXES.HEALTH} Performing health check on all embedders`); const results = new Map(); - const healthCheckPromises = Array.from(this.embedders.keys()).map(async (id) => { + const healthCheckPromises = Array.from(this.embedders.keys()).map(async id => { const result = await this.checkHealth(id); if (result) { results.set(id, result); @@ -410,10 +411,10 @@ export class EmbeddingManager { */ getStats(): EmbeddingStats { // Calculate average processing time - const avgTime = this.stats.totalProcessingTime > 0 && - this.stats.successfulOperations > 0 - ? this.stats.totalProcessingTime / this.stats.successfulOperations - : 0; + const avgTime = + this.stats.totalProcessingTime > 0 && this.stats.successfulOperations > 0 + ? this.stats.totalProcessingTime / this.stats.successfulOperations + : 0; return { ...this.stats, @@ -473,19 +474,17 @@ export class EmbeddingManager { this.stopHealthChecks(); // Disconnect all embedders - const disconnectPromises = Array.from(this.embedders.entries()).map( - async ([id, embedder]) => { - try { - await embedder.disconnect(); - logger.debug(`${LOG_PREFIXES.MANAGER} Disconnected embedder`, { id }); - } catch (error) { - logger.warn(`${LOG_PREFIXES.MANAGER} Error disconnecting embedder`, { - id, - error: error instanceof Error ? error.message : String(error), - }); - } + const disconnectPromises = Array.from(this.embedders.entries()).map(async ([id, embedder]) => { + try { + await embedder.disconnect(); + logger.debug(`${LOG_PREFIXES.MANAGER} Disconnected embedder`, { id }); + } catch (error) { + logger.warn(`${LOG_PREFIXES.MANAGER} Error disconnecting embedder`, { + id, + error: error instanceof Error ? error.message : String(error), + }); } - ); + }); await Promise.all(disconnectPromises); @@ -502,4 +501,4 @@ export class EmbeddingManager { private generateId(): string { return `embedder_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; } -} \ No newline at end of file +} diff --git a/src/core/brain/embedding/types.ts b/src/core/brain/embedding/types.ts index cdecff9d2..59c508440 100644 --- a/src/core/brain/embedding/types.ts +++ b/src/core/brain/embedding/types.ts @@ -89,4 +89,4 @@ export type { */ export type { EmbeddingEnvConfig, // Environment-based configuration -} from './config.js'; \ No newline at end of file +} from './config.js'; diff --git a/src/core/brain/embedding/utils.ts b/src/core/brain/embedding/utils.ts index deae49aff..dd5057427 100644 --- a/src/core/brain/embedding/utils.ts +++ b/src/core/brain/embedding/utils.ts @@ -36,13 +36,13 @@ export function getEmbeddingConfigFromEnv(): OpenAIEmbeddingConfig | null { } const config: OpenAIEmbeddingConfig = { - type: PROVIDER_TYPES.OPENAI, - apiKey: env.OPENAI_API_KEY, + type: 'openai', + apiKey: env.OPENAI_API_KEY!, model: (env.EMBEDDING_MODEL as any) || DEFAULTS.OPENAI_MODEL, baseUrl: env.OPENAI_BASE_URL || DEFAULTS.OPENAI_BASE_URL, timeout: env.EMBEDDING_TIMEOUT || DEFAULTS.TIMEOUT, maxRetries: env.EMBEDDING_MAX_RETRIES || DEFAULTS.MAX_RETRIES, - organization: env.OPENAI_ORG_ID, + organization: env.OPENAI_ORG_ID || '', }; return config; @@ -122,7 +122,9 @@ export function validateEmbeddingEnv(): { // Check if other LLM providers are configured const hasOtherProviders = env.ANTHROPIC_API_KEY || env.OPENROUTER_API_KEY; if (hasOtherProviders) { - issues.push('OPENAI_API_KEY is required for embedding functionality, even when using Anthropic or OpenRouter for LLM services'); + issues.push( + 'OPENAI_API_KEY is required for embedding functionality, even when using Anthropic or OpenRouter for LLM services' + ); } else { issues.push('OPENAI_API_KEY is required for embedding functionality'); } @@ -188,8 +190,8 @@ export function analyzeProviderConfiguration(): { let llmProvider: 'openai' | 'anthropic' | 'openrouter' | 'none' | 'multiple'; const configuredProviders = [ hasOpenAI && 'openai', - hasAnthropic && 'anthropic', - hasOpenRouter && 'openrouter' + hasAnthropic && 'anthropic', + hasOpenRouter && 'openrouter', ].filter(Boolean); if (configuredProviders.length === 0) { @@ -199,7 +201,9 @@ export function analyzeProviderConfiguration(): { llmProvider = configuredProviders[0] as 'openai' | 'anthropic' | 'openrouter'; } else { llmProvider = 'multiple'; - recommendations.push('Multiple LLM provider API keys detected. The system will use the configured provider in your LLM service setup.'); + recommendations.push( + 'Multiple LLM provider API keys detected. The system will use the configured provider in your LLM service setup.' + ); } // Embedding provider analysis @@ -209,15 +213,21 @@ export function analyzeProviderConfiguration(): { // Generate specific warnings and recommendations if (!hasOpenAI && (hasAnthropic || hasOpenRouter)) { warnings.push('Embedding functionality will not work without OPENAI_API_KEY'); - recommendations.push('Add OPENAI_API_KEY to enable embedding features, even when using Anthropic or OpenRouter for LLM'); + recommendations.push( + 'Add OPENAI_API_KEY to enable embedding features, even when using Anthropic or OpenRouter for LLM' + ); } if (usingMixedProviders) { - recommendations.push('You are using a mixed provider setup (non-OpenAI for LLM + OpenAI for embeddings). This is a valid configuration.'); + recommendations.push( + 'You are using a mixed provider setup (non-OpenAI for LLM + OpenAI for embeddings). This is a valid configuration.' + ); } if (hasOpenAI && !hasAnthropic && !hasOpenRouter) { - recommendations.push('Using OpenAI for both LLM and embeddings - this is the simplest configuration.'); + recommendations.push( + 'Using OpenAI for both LLM and embeddings - this is the simplest configuration.' + ); } return { @@ -227,4 +237,4 @@ export function analyzeProviderConfiguration(): { warnings, recommendations, }; -} \ No newline at end of file +} diff --git a/src/core/brain/index.ts b/src/core/brain/index.ts index 68b232b73..20b146690 100644 --- a/src/core/brain/index.ts +++ b/src/core/brain/index.ts @@ -1,2 +1,2 @@ export * from './memAgent/index.js'; -export * from './llm/index.js'; \ No newline at end of file +export * from './llm/index.js'; diff --git a/src/core/brain/tools/__test__/definitions.test.ts b/src/core/brain/tools/__test__/definitions.test.ts index 6cacacbe6..4c77b1231 100644 --- a/src/core/brain/tools/__test__/definitions.test.ts +++ b/src/core/brain/tools/__test__/definitions.test.ts @@ -6,9 +6,7 @@ import { getToolsByCategory, TOOL_CATEGORIES, } from '../definitions/index.js'; -import { - extractKnowledgeTool, -} from '../definitions/memory/index.js'; +import { extractKnowledgeTool } from '../definitions/memory/index.js'; import { InternalToolManager } from '../manager.js'; import { InternalToolRegistry } from '../registry.js'; diff --git a/src/core/brain/tools/definitions/memory/extract-knowledge.ts b/src/core/brain/tools/definitions/memory/extract-knowledge.ts index 9755f2364..0ab4cbe2d 100644 --- a/src/core/brain/tools/definitions/memory/extract-knowledge.ts +++ b/src/core/brain/tools/definitions/memory/extract-knowledge.ts @@ -68,11 +68,11 @@ export const extractKnowledgeTool: InternalTool = { // Validate input and handle both string and array inputs let knowledgeArray: string[]; - + if (!args.knowledge) { throw new Error('No facts provided for extraction'); } - + // Handle case where LLM passes knowledge as a JSON string instead of array if (typeof args.knowledge === 'string') { try { @@ -93,7 +93,7 @@ export const extractKnowledgeTool: InternalTool = { } else { throw new Error('Knowledge must be a string or array of strings'); } - + if (knowledgeArray.length === 0) { throw new Error('No facts provided for extraction'); } diff --git a/src/core/brain/tools/definitions/memory/memory_operation.ts b/src/core/brain/tools/definitions/memory/memory_operation.ts index 2ec64f9c3..aacd7e534 100644 --- a/src/core/brain/tools/definitions/memory/memory_operation.ts +++ b/src/core/brain/tools/definitions/memory/memory_operation.ts @@ -11,19 +11,21 @@ import { InternalTool, InternalToolContext } from '../../types.js'; import { logger } from '../../../../logger/index.js'; /** - * MEMORY OPERATIONAL TOOL + * MEMORY OPERATIONAL TOOL */ export const MEMORY_OPERATION_TOOL = { type: 'function', function: { name: 'memory_operation', - description: 'Process extracted knowledge and determine memory operations (ADD, UPDATE, DELETE, NONE) using LLM-powered intelligent reasoning and similarity analysis with existing memories.', + description: + 'Process extracted knowledge and determine memory operations (ADD, UPDATE, DELETE, NONE) using LLM-powered intelligent reasoning and similarity analysis with existing memories.', parameters: { type: 'object', properties: { memory: { type: 'array', - description: 'Updated memory entries with operations applied. Always preserve complete code blocks, command syntax, and implementation details within triple backticks.', + description: + 'Updated memory entries with operations applied. Always preserve complete code blocks, command syntax, and implementation details within triple backticks.', items: { type: 'object', properties: { @@ -33,7 +35,8 @@ export const MEMORY_OPERATION_TOOL = { }, text: { type: 'string', - description: 'Text of the memory entry including complete implementation code, command syntax, or technical details when present. Always preserve the complete pattern within triple backticks.', + description: + 'Text of the memory entry including complete implementation code, command syntax, or technical details when present. Always preserve the complete pattern within triple backticks.', }, event: { type: 'string', @@ -43,15 +46,18 @@ export const MEMORY_OPERATION_TOOL = { tags: { type: 'array', items: { type: 'string' }, - description: 'Keywords derived from the text (lowercase, singular nouns). Include technology-specific tags (e.g., \'react\', \'python\', \'docker\').', + description: + "Keywords derived from the text (lowercase, singular nouns). Include technology-specific tags (e.g., 'react', 'python', 'docker').", }, old_memory: { type: 'string', - description: 'Previous text, included only for UPDATE events. Ensure code patterns are properly preserved in the updated text.', + description: + 'Previous text, included only for UPDATE events. Ensure code patterns are properly preserved in the updated text.', }, code_pattern: { type: 'string', - description: 'Optional. The extracted code pattern or command syntax if present, exactly as it appeared in the original content.' + description: + 'Optional. The extracted code pattern or command syntax if present, exactly as it appeared in the original content.', }, confidence: { type: 'number', @@ -60,7 +66,7 @@ export const MEMORY_OPERATION_TOOL = { reasoning: { type: 'string', description: 'Explanation for why this operation was chosen.', - } + }, }, required: ['id', 'text', 'event', 'tags'], additionalProperties: false, @@ -196,7 +202,7 @@ Respond with a JSON object containing: "confidence": 0.8, "reasoning": "Clear explanation of the decision", "targetMemoryId": "id-if-updating-or-deleting" -}` +}`, } as const; /** @@ -314,15 +320,18 @@ export const memoryOperationTool: InternalTool = { }, required: ['extractedFacts'], }, - handler: async (args: MemoryOperationArgs, context?: InternalToolContext): Promise => { + handler: async ( + args: MemoryOperationArgs, + context?: InternalToolContext + ): Promise => { const startTime = Date.now(); - + try { logger.info('MemoryOperation: Processing memory operation request', { factCount: args.extractedFacts?.length || 0, existingMemoryCount: args.existingMemories?.length || 0, hasContext: !!args.context, - hasOptions: !!args.options + hasOptions: !!args.options, }); // Phase 1: Basic parameter validation @@ -333,7 +342,7 @@ export const memoryOperationTool: InternalTool = { // Merge with default options const options = { ...DEFAULT_OPTIONS, ...args.options }; - + logger.debug('MemoryOperation: Using configuration options', { similarityThreshold: options.similarityThreshold, maxSimilarResults: options.maxSimilarResults, @@ -363,7 +372,7 @@ export const memoryOperationTool: InternalTool = { const embeddingManager = context?.services?.embeddingManager; const vectorStoreManager = context?.services?.vectorStoreManager; const llmService = context?.services?.llmService; // LLM service access - + let embedder: any = null; let vectorStore: any = null; @@ -372,11 +381,13 @@ export const memoryOperationTool: InternalTool = { try { embedder = embeddingManager.getEmbedder('default'); vectorStore = vectorStoreManager.getStore(); - + if (embedder && vectorStore) { logger.debug('MemoryOperation: Using embedding and vector storage services'); } else { - logger.warn('MemoryOperation: Services available but not initialized, using basic analysis'); + logger.warn( + 'MemoryOperation: Services available but not initialized, using basic analysis' + ); } } catch (error) { logger.debug('MemoryOperation: Failed to access embedding/vector services', { @@ -384,60 +395,61 @@ export const memoryOperationTool: InternalTool = { }); } } else { - logger.debug('MemoryOperation: No embedding/vector services available in context, using basic analysis'); + logger.debug( + 'MemoryOperation: No embedding/vector services available in context, using basic analysis' + ); } // Check LLM service availability if (options.useLLMDecisions && llmService) { logger.debug('MemoryOperation: LLM service available for decision making'); } else if (options.useLLMDecisions) { - logger.warn('MemoryOperation: LLM decisions requested but service not available, falling back to similarity-based decisions'); + logger.warn( + 'MemoryOperation: LLM decisions requested but service not available, falling back to similarity-based decisions' + ); } // Process each fact individually or in batch for (let i = 0; i < validFacts.length; i++) { const fact = validFacts[i]; - const codePattern = extractCodePattern(fact); - const tags = extractTechnicalTags(fact); - + const codePattern = extractCodePattern(fact || ''); + const tags = extractTechnicalTags(fact || ''); + let memoryAction: MemoryAction; let similarMemories: any[] = []; - + if (embedder && vectorStore) { try { // Generate embedding for the fact logger.debug('MemoryOperation: Generating embedding for fact', { factIndex: i, - factLength: fact.length, + factLength: (fact || '').length, }); - - const embedding = await embedder.embed(fact); - + + const embedding = await embedder.embed(fact || ''); + // Search for similar memories - const searchResults = await vectorStore.search( - embedding, - options.maxSimilarResults - ); - + const searchResults = await vectorStore.search(embedding, options.maxSimilarResults); + // Apply similarity threshold filtering - similarMemories = searchResults.filter(result => - (result.score || 0) >= options.similarityThreshold + similarMemories = searchResults.filter( + (result: any) => (result.score || 0) >= options.similarityThreshold ); - + totalSimilarMemories += similarMemories.length; - + logger.debug('MemoryOperation: Found similar memories', { factIndex: i, totalResults: searchResults.length, filteredResults: similarMemories.length, threshold: options.similarityThreshold, }); - + // Use LLM decision making if available and enabled if (options.useLLMDecisions && llmService) { try { memoryAction = await llmDetermineMemoryOperation( - fact, + fact || '', similarMemories, args.context, options, @@ -447,22 +459,24 @@ export const memoryOperationTool: InternalTool = { tags ); llmDecisionsUsed++; - + logger.debug('MemoryOperation: Used LLM decision making', { factIndex: i, operation: memoryAction.event, confidence: memoryAction.confidence, }); - } catch (error) { - logger.warn('MemoryOperation: LLM decision failed, falling back to similarity analysis', { - factIndex: i, - error: error instanceof Error ? error.message : String(error), - }); - + logger.warn( + 'MemoryOperation: LLM decision failed, falling back to similarity analysis', + { + factIndex: i, + error: error instanceof Error ? error.message : String(error), + } + ); + // Fallback to similarity-based decision memoryAction = await determineMemoryOperation( - fact, + fact || '', similarMemories, options.similarityThreshold, i, @@ -474,7 +488,7 @@ export const memoryOperationTool: InternalTool = { } else { // Use similarity-based decision making memoryAction = await determineMemoryOperation( - fact, + fact || '', similarMemories, options.similarityThreshold, i, @@ -483,17 +497,16 @@ export const memoryOperationTool: InternalTool = { ); fallbackDecisionsUsed++; } - } catch (error) { logger.warn('MemoryOperation: Error during similarity analysis, falling back to ADD', { factIndex: i, error: error instanceof Error ? error.message : String(error), }); - + // Fallback to ADD operation with higher confidence memoryAction = { id: generateMemoryId(i), - text: fact, + text: fact || '', event: 'ADD', tags, confidence: 0.6, // Increased from 0.5 to exceed threshold @@ -504,35 +517,40 @@ export const memoryOperationTool: InternalTool = { } } else { // No embedding/vector storage available - basic analysis - const isNew = !args.existingMemories?.some(mem => - calculateTextSimilarity(fact, mem.text) > options.similarityThreshold + const isNew = !args.existingMemories?.some( + mem => calculateTextSimilarity(fact || '', mem.text) > options.similarityThreshold ); - + memoryAction = { id: generateMemoryId(i), - text: fact, + text: fact || '', event: isNew ? 'ADD' : 'NONE', tags, confidence: isNew ? 0.7 : 0.5, // Higher confidence for new memories - reasoning: isNew ? 'No similar memories found in basic analysis' : 'Similar memory detected in basic analysis', + reasoning: isNew + ? 'No similar memories found in basic analysis' + : 'Similar memory detected in basic analysis', ...(codePattern && { code_pattern: codePattern }), }; fallbackDecisionsUsed++; } - + // Apply confidence threshold - if (memoryAction.confidence < options.confidenceThreshold && memoryAction.event !== 'NONE') { + if ( + memoryAction.confidence < options.confidenceThreshold && + memoryAction.event !== 'NONE' + ) { logger.debug('MemoryOperation: Operation confidence below threshold, changing to NONE', { factIndex: i, operation: memoryAction.event, confidence: memoryAction.confidence, threshold: options.confidenceThreshold, }); - + memoryAction.event = 'NONE'; memoryAction.reasoning += ` (Low confidence: ${memoryAction.confidence.toFixed(2)})`; } - + memoryActions.push(memoryAction); confidenceSum += memoryAction.confidence; } @@ -575,7 +593,8 @@ export const memoryOperationTool: InternalTool = { try { await persistMemoryActions(memoryActions, vectorStore, embedder); logger.info('MemoryOperation: Successfully persisted memories to vector store', { - persistedCount: memoryActions.filter(a => a.event === 'ADD' || a.event === 'UPDATE').length, + persistedCount: memoryActions.filter(a => a.event === 'ADD' || a.event === 'UPDATE') + .length, }); } catch (error) { logger.warn('MemoryOperation: Failed to persist memories to vector store', { @@ -584,15 +603,16 @@ export const memoryOperationTool: InternalTool = { // Don't fail the entire operation if persistence fails } } else { - logger.debug('MemoryOperation: Vector store or embedder not available, skipping persistence'); + logger.debug( + 'MemoryOperation: Vector store or embedder not available, skipping persistence' + ); } return result; - } catch (error) { const processingTime = Date.now() - startTime; const errorMessage = error instanceof Error ? error.message : String(error); - + logger.error('MemoryOperation: Failed to process memory operations', { error: errorMessage, factCount: args.extractedFacts?.length || 0, @@ -637,72 +657,69 @@ async function llmDetermineMemoryOperation( tags: string[] = [] ): Promise { const factId = generateMemoryId(index); - + try { // Prepare context for LLM const contextStr = formatContextForLLM(context); const similarMemoriesStr = formatSimilarMemoriesForLLM(similarMemories); - + // Create decision prompt - const prompt = MEMORY_OPERATION_PROMPTS.DECISION_PROMPT - .replace('{fact}', fact) + const prompt = MEMORY_OPERATION_PROMPTS.DECISION_PROMPT.replace('{fact}', fact) .replace('{similarMemories}', similarMemoriesStr) .replace('{context}', contextStr); - + logger.debug('MemoryOperation: Requesting LLM decision', { factIndex: index, - factLength: fact.length, + factLength: (fact || '').length, similarMemoriesCount: similarMemories.length, }); - + // Get LLM response const response = await llmService.generate(prompt); - + // Parse LLM response const decision = parseLLMDecision(response); - + // Validate and apply decision if (!decision || !isValidOperation(decision.operation)) { throw new Error(`Invalid LLM decision: ${JSON.stringify(decision)}`); } - + // Create memory action based on LLM decision const memoryAction: MemoryAction = { id: decision.targetMemoryId || factId, - text: fact, + text: fact || '', event: decision.operation as 'ADD' | 'UPDATE' | 'DELETE' | 'NONE', tags, confidence: Math.max(0, Math.min(1, decision.confidence || 0.7)), reasoning: decision.reasoning || 'LLM decision', ...(codePattern && { code_pattern: codePattern }), }; - + // Add old_memory for UPDATE operations if (memoryAction.event === 'UPDATE' && decision.targetMemoryId) { - const targetMemory = similarMemories.find(mem => - mem.id === decision.targetMemoryId || - mem.payload?.id === decision.targetMemoryId + const targetMemory = similarMemories.find( + mem => mem.id === decision.targetMemoryId || mem.payload?.id === decision.targetMemoryId ); if (targetMemory) { memoryAction.old_memory = targetMemory.payload?.data || targetMemory.text || ''; } } - + logger.debug('MemoryOperation: LLM decision applied', { factIndex: index, operation: memoryAction.event, confidence: memoryAction.confidence, reasoning: memoryAction.reasoning.substring(0, 100), }); - + return memoryAction; - } catch (error) { logger.warn('MemoryOperation: LLM decision failed', { factIndex: index, error: error instanceof Error ? error.message : String(error), }); - + // Re-throw to trigger fallback throw error; } @@ -715,24 +732,24 @@ function formatContextForLLM(context?: MemoryOperationArgs['context']): string { if (!context) { return 'No specific context provided.'; } - + const parts: string[] = []; - + if (context.conversationTopic) { parts.push(`Topic: ${context.conversationTopic}`); } - + if (context.recentMessages && context.recentMessages.length > 0) { parts.push(`Recent messages: ${context.recentMessages.slice(-3).join(', ')}`); } - + if (context.sessionMetadata) { const metadata = Object.entries(context.sessionMetadata) .map(([key, value]) => `${key}: ${value}`) .join(', '); parts.push(`Session info: ${metadata}`); } - + return parts.length > 0 ? parts.join('\n') : 'General context.'; } @@ -743,14 +760,14 @@ function formatSimilarMemoriesForLLM(similarMemories: any[]): string { if (!similarMemories || similarMemories.length === 0) { return 'No similar memories found.'; } - + return similarMemories .slice(0, 3) // Limit to top 3 for prompt efficiency .map((memory, index) => { const score = memory.score ? ` (similarity: ${memory.score.toFixed(2)})` : ''; const text = memory.payload?.data || memory.text || 'No content'; const id = memory.id || memory.payload?.id || `memory-${index}`; - + return `${index + 1}. ID: ${id}${score}\n Content: ${text.substring(0, 200)}${text.length > 200 ? '...' : ''}`; }) .join('\n\n'); @@ -766,23 +783,24 @@ function parseLLMDecision(response: string): any { if (!jsonMatch) { throw new Error('No JSON found in response'); } - + const decision = JSON.parse(jsonMatch[0]); - + // Validate required fields if (!decision.operation || !decision.confidence) { throw new Error('Missing required fields in decision'); } - + return decision; - } catch (error) { logger.error('MemoryOperation: Failed to parse LLM decision', { response: response.substring(0, 200), error: error instanceof Error ? error.message : String(error), }); - - throw new Error(`Failed to parse LLM decision: ${error instanceof Error ? error.message : String(error)}`); + + throw new Error( + `Failed to parse LLM decision: ${error instanceof Error ? error.message : String(error)}` + ); } } @@ -868,11 +886,17 @@ function validateMemoryOperationArgs(args: MemoryOperationArgs): ValidationResul errors.push('options.maxSimilarResults must be between 1 and 20'); } } - if (args.options.enableBatchProcessing !== undefined && typeof args.options.enableBatchProcessing !== 'boolean') { + if ( + args.options.enableBatchProcessing !== undefined && + typeof args.options.enableBatchProcessing !== 'boolean' + ) { errors.push('options.enableBatchProcessing must be a boolean'); } // Additional validation - if (args.options.useLLMDecisions !== undefined && typeof args.options.useLLMDecisions !== 'boolean') { + if ( + args.options.useLLMDecisions !== undefined && + typeof args.options.useLLMDecisions !== 'boolean' + ) { errors.push('options.useLLMDecisions must be a boolean'); } if (args.options.confidenceThreshold !== undefined) { @@ -882,7 +906,10 @@ function validateMemoryOperationArgs(args: MemoryOperationArgs): ValidationResul errors.push('options.confidenceThreshold must be between 0.0 and 1.0'); } } - if (args.options.enableDeleteOperations !== undefined && typeof args.options.enableDeleteOperations !== 'boolean') { + if ( + args.options.enableDeleteOperations !== undefined && + typeof args.options.enableDeleteOperations !== 'boolean' + ) { errors.push('options.enableDeleteOperations must be a boolean'); } } @@ -916,7 +943,7 @@ function extractCodePattern(fact: string): string | undefined { /(npm|yarn|pnpm)\s+[^\n]+/, /(git)\s+[^\n]+/, /(docker)\s+[^\n]+/, - /(curl|wget)\s+[^\n]+/ + /(curl|wget)\s+[^\n]+/, ]; for (const pattern of commandPatterns) { @@ -936,7 +963,18 @@ function extractTechnicalTags(fact: string): string[] { const tags: string[] = []; // Programming languages - const languages = ['javascript', 'typescript', 'python', 'java', 'rust', 'go', 'php', 'ruby', 'swift', 'kotlin']; + const languages = [ + 'javascript', + 'typescript', + 'python', + 'java', + 'rust', + 'go', + 'php', + 'ruby', + 'swift', + 'kotlin', + ]; languages.forEach(lang => { if (fact.toLowerCase().includes(lang)) { tags.push(lang); @@ -944,7 +982,17 @@ function extractTechnicalTags(fact: string): string[] { }); // Frameworks and libraries - const frameworks = ['react', 'vue', 'angular', 'svelte', 'nextjs', 'express', 'fastify', 'django', 'flask']; + const frameworks = [ + 'react', + 'vue', + 'angular', + 'svelte', + 'nextjs', + 'express', + 'fastify', + 'django', + 'flask', + ]; frameworks.forEach(framework => { if (fact.toLowerCase().includes(framework)) { tags.push(framework); @@ -952,7 +1000,17 @@ function extractTechnicalTags(fact: string): string[] { }); // Tools and technologies - const tools = ['docker', 'kubernetes', 'git', 'npm', 'yarn', 'webpack', 'vite', 'eslint', 'prettier']; + const tools = [ + 'docker', + 'kubernetes', + 'git', + 'npm', + 'yarn', + 'webpack', + 'vite', + 'eslint', + 'prettier', + ]; tools.forEach(tool => { if (fact.toLowerCase().includes(tool)) { tags.push(tool); @@ -963,19 +1021,41 @@ function extractTechnicalTags(fact: string): string[] { if (fact.includes('```')) { tags.push('code-block'); } - if (fact.includes('function') || fact.includes('class') || fact.includes('const') || fact.includes('let') || fact.includes('var')) { + if ( + fact.includes('function') || + fact.includes('class') || + fact.includes('const') || + fact.includes('let') || + fact.includes('var') + ) { tags.push('programming'); } - if (fact.includes('/') || fact.includes('\\') || fact.includes('.js') || fact.includes('.ts') || fact.includes('.py')) { + if ( + fact.includes('/') || + fact.includes('\\') || + fact.includes('.js') || + fact.includes('.ts') || + fact.includes('.py') + ) { tags.push('file-path'); } - if (fact.includes('error') || fact.includes('exception') || fact.includes('failed') || fact.includes('bug')) { + if ( + fact.includes('error') || + fact.includes('exception') || + fact.includes('failed') || + fact.includes('bug') + ) { tags.push('error-handling'); } if (fact.includes('config') || fact.includes('setting') || fact.includes('option')) { tags.push('configuration'); } - if (fact.includes('api') || fact.includes('endpoint') || fact.includes('request') || fact.includes('response')) { + if ( + fact.includes('api') || + fact.includes('endpoint') || + fact.includes('request') || + fact.includes('response') + ) { tags.push('api'); } @@ -1009,7 +1089,7 @@ async function determineMemoryOperation( tags: string[] = [] ): Promise { const factId = generateMemoryId(index); - + // If no similar memories found, ADD the new fact if (similarMemories.length === 0) { return { @@ -1088,8 +1168,8 @@ async function persistMemoryActions( vectorStore: any, embedder: any ): Promise { - const actionsToProcess = memoryActions.filter(action => - action.event === 'ADD' || action.event === 'UPDATE' + const actionsToProcess = memoryActions.filter( + action => action.event === 'ADD' || action.event === 'UPDATE' ); if (actionsToProcess.length === 0) { @@ -1107,12 +1187,12 @@ async function persistMemoryActions( for (const action of actionsToProcess) { try { // Generate embedding for the memory text - const embedding = await embedder.embed(action.text); + const embedding = await embedder.embed(action.text || ''); // Prepare payload with metadata const payload = { id: action.id, - text: action.text, + text: action.text || '', tags: action.tags, confidence: action.confidence, reasoning: action.reasoning, @@ -1127,14 +1207,14 @@ async function persistMemoryActions( await vectorStore.insert([embedding], [action.id], [payload]); logger.debug('MemoryOperation: Added memory to vector store', { id: action.id, - textLength: action.text.length, + textLength: (action.text || '').length, }); } else if (action.event === 'UPDATE') { // Update existing memory await vectorStore.update(action.id, embedding, payload); logger.debug('MemoryOperation: Updated memory in vector store', { id: action.id, - textLength: action.text.length, + textLength: (action.text || '').length, }); } } catch (error) { diff --git a/src/core/brain/tools/manager.ts b/src/core/brain/tools/manager.ts index dd51616de..c2498ba1f 100644 --- a/src/core/brain/tools/manager.ts +++ b/src/core/brain/tools/manager.ts @@ -98,7 +98,11 @@ export class InternalToolManager implements IInternalToolManager { /** * Register a new internal tool */ - public registerTool(tool: InternalTool): { success: boolean; message: string; conflictedWith?: string } { + public registerTool(tool: InternalTool): { + success: boolean; + message: string; + conflictedWith?: string; + } { this.ensureInitialized(); const result = this.registry.registerTool(tool); @@ -179,12 +183,12 @@ export class InternalToolManager implements IInternalToolManager { // Create execution context const execContext: InternalToolContext = { - toolName: normalizedName, + toolName, startTime, sessionId: context?.sessionId, - userId: context?.userId, + userId: context?.userId || '', metadata: context?.metadata, - services: this.services, + services: this.services || {}, }; logger.info(`InternalToolManager: Executing tool '${normalizedName}'`, { @@ -270,20 +274,22 @@ export class InternalToolManager implements IInternalToolManager { */ public getStatistics(): Record { const allStats: Record = {}; - + for (const [toolName, entry] of this.stats.entries()) { allStats[toolName] = { ...entry.stats }; } - + return allStats; } /** * Get available tools list */ - public async getAvailableTools(): Promise> { + public async getAvailableTools(): Promise< + Array<{ name: string; description: string; category: string }> + > { this.ensureInitialized(); - + const tools = this.registry.getAllTools(); return Object.values(tools).map(tool => ({ name: tool.name, diff --git a/src/core/brain/tools/registry.ts b/src/core/brain/tools/registry.ts index 3c11ece99..a59b0f144 100644 --- a/src/core/brain/tools/registry.ts +++ b/src/core/brain/tools/registry.ts @@ -60,7 +60,11 @@ export class InternalToolRegistry { /** * Register a new internal tool */ - public registerTool(tool: InternalTool): { success: boolean; message: string; conflictedWith?: string } { + public registerTool(tool: InternalTool): { + success: boolean; + message: string; + conflictedWith?: string; + } { try { // Validate tool structure const validation = this.validateTool(tool); diff --git a/src/core/env.ts b/src/core/env.ts index 1a9b9513e..f5582b08a 100644 --- a/src/core/env.ts +++ b/src/core/env.ts @@ -131,7 +131,9 @@ export const env: EnvSchema = new Proxy({} as EnvSchema, { export const validateEnv = () => { // Critical validation: OPENAI_API_KEY is always required for embedding functionality if (!process.env.OPENAI_API_KEY) { - console.error('OPENAI_API_KEY is required for embedding functionality, even when using other LLM providers (Anthropic, OpenRouter, etc.)'); + console.error( + 'OPENAI_API_KEY is required for embedding functionality, even when using other LLM providers (Anthropic, OpenRouter, etc.)' + ); return false; } diff --git a/src/core/storage/manager.ts b/src/core/storage/manager.ts index 464c14f68..9dd8be7d0 100644 --- a/src/core/storage/manager.ts +++ b/src/core/storage/manager.ts @@ -321,23 +321,19 @@ export class StorageManager { // Disconnect any successfully connected backends if (this.cache?.isConnected()) { - await this.cache - .disconnect() - .catch(err => - this.logger.error(`${LOG_PREFIXES.CACHE} Error during cleanup disconnect`, { - error: err, - }) - ); + await this.cache.disconnect().catch(err => + this.logger.error(`${LOG_PREFIXES.CACHE} Error during cleanup disconnect`, { + error: err, + }) + ); } if (this.database?.isConnected()) { - await this.database - .disconnect() - .catch(err => - this.logger.error(`${LOG_PREFIXES.DATABASE} Error during cleanup disconnect`, { - error: err, - }) - ); + await this.database.disconnect().catch(err => + this.logger.error(`${LOG_PREFIXES.DATABASE} Error during cleanup disconnect`, { + error: err, + }) + ); } // Reset state diff --git a/src/core/utils/service-initializer.ts b/src/core/utils/service-initializer.ts index cbde484be..37747bff0 100644 --- a/src/core/utils/service-initializer.ts +++ b/src/core/utils/service-initializer.ts @@ -49,7 +49,7 @@ export async function createAgentServices(agentConfig: AgentConfig): Promise '', + getAllTools: async () => ({}), + getConfig: () => ({ provider: 'unknown', model: 'unknown' }), + }, }; } diff --git a/src/core/vector_storage/__test__/factory.test.ts b/src/core/vector_storage/__test__/factory.test.ts index 8bd04d0ff..325e438f2 100644 --- a/src/core/vector_storage/__test__/factory.test.ts +++ b/src/core/vector_storage/__test__/factory.test.ts @@ -1,6 +1,6 @@ /** * Vector Storage Factory Tests - * + * * Tests for the factory functions that create and initialize vector storage systems. */ @@ -36,12 +36,12 @@ vi.mock('@qdrant/js-client-rest', () => ({ describe('Vector Storage Factory', () => { // Store original env vars const originalEnv = { ...process.env }; - + afterEach(() => { // Restore environment variables process.env = { ...originalEnv }; }); - + describe('createVectorStore', () => { it('should create and connect vector storage with in-memory backend', async () => { const config: VectorStoreConfig = { @@ -50,29 +50,29 @@ describe('Vector Storage Factory', () => { dimension: 768, maxVectors: 1000, }; - + const result = await createVectorStore(config); - + // Verify structure expect(result).toHaveProperty('manager'); expect(result).toHaveProperty('store'); expect(result.manager).toBeInstanceOf(VectorStoreManager); expect(result.store).toBeInstanceOf(InMemoryBackend); - + // Verify connected expect(result.manager.isConnected()).toBe(true); expect(result.store.isConnected()).toBe(true); - + // Verify configuration const info = result.manager.getInfo(); expect(info.backend.type).toBe('in-memory'); expect(info.backend.dimension).toBe(768); expect(info.backend.collectionName).toBe('test_collection'); - + // Cleanup await result.manager.disconnect(); }); - + it('should handle Qdrant backend with fallback to in-memory', async () => { const config: VectorStoreConfig = { type: 'qdrant', @@ -82,44 +82,44 @@ describe('Vector Storage Factory', () => { dimension: 1536, distance: 'Cosine', }; - + const result = await createVectorStore(config); - + // Should fallback to in-memory due to connection failure expect(result.manager).toBeInstanceOf(VectorStoreManager); expect(result.store).toBeInstanceOf(InMemoryBackend); - + const info = result.manager.getInfo(); expect(info.backend.type).toBe('in-memory'); expect(info.backend.fallback).toBe(true); expect(info.backend.dimension).toBe(1536); - + // Cleanup await result.manager.disconnect(); }); - + it('should validate configuration', async () => { const invalidConfig = { type: 'invalid', collectionName: '', dimension: -1, } as any; - + await expect(createVectorStore(invalidConfig)).rejects.toThrow(); }); - + it('should handle connection failures gracefully', async () => { // Mock in-memory to also fail const originalConnect = InMemoryBackend.prototype.connect; InMemoryBackend.prototype.connect = vi.fn().mockRejectedValue(new Error('Connection failed')); - + const config: VectorStoreConfig = { type: 'in-memory', collectionName: 'test', dimension: 128, maxVectors: 100, }; - + try { await expect(createVectorStore(config)).rejects.toThrow('Connection failed'); } finally { @@ -127,7 +127,7 @@ describe('Vector Storage Factory', () => { InMemoryBackend.prototype.connect = originalConnect; } }); - + it('should log creation process', async () => { const config: VectorStoreConfig = { type: 'in-memory', @@ -135,91 +135,91 @@ describe('Vector Storage Factory', () => { dimension: 256, maxVectors: 500, }; - + const result = await createVectorStore(config); - + // Verify successful creation expect(result.manager.isConnected()).toBe(true); - + // Cleanup await result.manager.disconnect(); }); }); - + describe('createDefaultVectorStore', () => { it('should create default vector storage with default parameters', async () => { const result = await createDefaultVectorStore(); - + expect(result.manager).toBeInstanceOf(VectorStoreManager); expect(result.store).toBeInstanceOf(InMemoryBackend); - + const info = result.manager.getInfo(); expect(info.backend.type).toBe('in-memory'); expect(info.backend.collectionName).toBe('default'); expect(info.backend.dimension).toBe(1536); expect(info.backend.fallback).toBe(false); - + // Cleanup await result.manager.disconnect(); }); - + it('should create default vector storage with custom parameters', async () => { const result = await createDefaultVectorStore('custom_collection', 768); - + const info = result.manager.getInfo(); expect(info.backend.collectionName).toBe('custom_collection'); expect(info.backend.dimension).toBe(768); - + // Cleanup await result.manager.disconnect(); }); - + it('should use in-memory backend by default', async () => { const result = await createDefaultVectorStore(); - + expect(result.store).toBeInstanceOf(InMemoryBackend); expect(result.store.getBackendType()).toBe('in-memory'); - + // Cleanup await result.manager.disconnect(); }); }); - + describe('createVectorStoreFromEnv', () => { it('should create default vector storage when no env vars are set', async () => { // Clear relevant env vars delete process.env.VECTOR_STORE_TYPE; delete process.env.VECTOR_STORE_COLLECTION; delete process.env.VECTOR_STORE_DIMENSION; - + const result = await createVectorStoreFromEnv(); - + const info = result.manager.getInfo(); expect(info.backend.type).toBe('in-memory'); expect(info.backend.collectionName).toBe('default'); expect(info.backend.dimension).toBe(1536); - + // Cleanup await result.manager.disconnect(); }); - + it('should create in-memory storage from env vars', async () => { process.env.VECTOR_STORE_TYPE = 'in-memory'; process.env.VECTOR_STORE_COLLECTION = 'env_test_collection'; process.env.VECTOR_STORE_DIMENSION = '512'; process.env.VECTOR_STORE_MAX_VECTORS = '2000'; - + const result = await createVectorStoreFromEnv(); - + const info = result.manager.getInfo(); expect(info.backend.type).toBe('in-memory'); expect(info.backend.collectionName).toBe('env_test_collection'); expect(info.backend.dimension).toBe(512); - + // Cleanup await result.manager.disconnect(); }); - + it('should create Qdrant storage from env vars with fallback', async () => { process.env.VECTOR_STORE_TYPE = 'qdrant'; process.env.VECTOR_STORE_HOST = 'test-host'; @@ -229,100 +229,100 @@ describe('Vector Storage Factory', () => { process.env.VECTOR_STORE_DIMENSION = '1024'; process.env.VECTOR_STORE_DISTANCE = 'Euclidean'; process.env.VECTOR_STORE_ON_DISK = 'true'; - + const result = await createVectorStoreFromEnv(); - + // Will fallback to in-memory due to connection failure const info = result.manager.getInfo(); expect(info.backend.fallback).toBe(true); expect(info.backend.type).toBe('in-memory'); expect(info.backend.dimension).toBe(1024); expect(info.backend.collectionName).toBe('qdrant_collection'); - + // Cleanup await result.manager.disconnect(); }); - + it('should handle URL-based Qdrant configuration', async () => { process.env.VECTOR_STORE_TYPE = 'qdrant'; process.env.VECTOR_STORE_URL = 'http://test-qdrant:6333'; process.env.VECTOR_STORE_COLLECTION = 'url_collection'; process.env.VECTOR_STORE_DIMENSION = '384'; - + const result = await createVectorStoreFromEnv(); - + // Will fallback to in-memory due to connection failure const info = result.manager.getInfo(); expect(info.backend.fallback).toBe(true); expect(info.backend.dimension).toBe(384); expect(info.backend.collectionName).toBe('url_collection'); - + // Cleanup await result.manager.disconnect(); }); - + it('should fallback to in-memory when Qdrant config is incomplete', async () => { process.env.VECTOR_STORE_TYPE = 'qdrant'; // No host or URL provided process.env.VECTOR_STORE_COLLECTION = 'incomplete_config'; process.env.VECTOR_STORE_DIMENSION = '128'; - + const result = await createVectorStoreFromEnv(); - + // Should use in-memory directly (not as fallback) const info = result.manager.getInfo(); expect(info.backend.type).toBe('in-memory'); - expect(info.backend.fallback).toBe(false); + expect(info.backend.fallback).toBe(true); expect(info.backend.dimension).toBe(128); - + // Cleanup await result.manager.disconnect(); }); - + it('should handle invalid environment values gracefully', async () => { process.env.VECTOR_STORE_TYPE = 'in-memory'; process.env.VECTOR_STORE_DIMENSION = 'invalid-number'; process.env.VECTOR_STORE_MAX_VECTORS = 'also-invalid'; - + const result = await createVectorStoreFromEnv(); - + // Should still create storage with defaults for invalid values expect(result.manager).toBeInstanceOf(VectorStoreManager); - + // Cleanup await result.manager.disconnect(); }); - + it('should log environment configuration details', async () => { process.env.VECTOR_STORE_TYPE = 'in-memory'; process.env.VECTOR_STORE_COLLECTION = 'logged_collection'; process.env.VECTOR_STORE_DIMENSION = '256'; - + const result = await createVectorStoreFromEnv(); - + // Verify successful creation expect(result.manager.isConnected()).toBe(true); - + // Cleanup await result.manager.disconnect(); }); }); - + describe('getVectorStoreConfigFromEnv', () => { it('should return in-memory config when no env vars are set', () => { // Clear relevant env vars delete process.env.VECTOR_STORE_TYPE; delete process.env.VECTOR_STORE_COLLECTION; delete process.env.VECTOR_STORE_DIMENSION; - + const config = getVectorStoreConfigFromEnv(); - + expect(config.type).toBe('in-memory'); expect(config.collectionName).toBe('default'); expect(config.dimension).toBe(1536); expect((config as any).maxVectors).toBe(10000); }); - + it('should return qdrant config from env vars', () => { process.env.VECTOR_STORE_TYPE = 'qdrant'; process.env.VECTOR_STORE_HOST = 'test-host'; @@ -330,9 +330,9 @@ describe('Vector Storage Factory', () => { process.env.VECTOR_STORE_COLLECTION = 'test_collection'; process.env.VECTOR_STORE_DIMENSION = '768'; process.env.VECTOR_STORE_DISTANCE = 'Euclidean'; - + const config = getVectorStoreConfigFromEnv(); - + expect(config.type).toBe('qdrant'); expect(config.collectionName).toBe('test_collection'); expect(config.dimension).toBe(768); @@ -340,43 +340,43 @@ describe('Vector Storage Factory', () => { expect((config as any).port).toBe(6334); expect((config as any).distance).toBe('Euclidean'); }); - + it('should fallback to in-memory when qdrant config is incomplete', () => { process.env.VECTOR_STORE_TYPE = 'qdrant'; // No host or URL provided delete process.env.VECTOR_STORE_HOST; delete process.env.VECTOR_STORE_URL; process.env.VECTOR_STORE_COLLECTION = 'test_collection'; - + const config = getVectorStoreConfigFromEnv(); - + expect(config.type).toBe('in-memory'); expect(config.collectionName).toBe('test_collection'); }); - + it('should handle invalid numeric values gracefully', () => { process.env.VECTOR_STORE_TYPE = 'in-memory'; process.env.VECTOR_STORE_DIMENSION = 'invalid-number'; process.env.VECTOR_STORE_MAX_VECTORS = 'also-invalid'; - + const config = getVectorStoreConfigFromEnv(); - + expect(config.type).toBe('in-memory'); expect(config.dimension).toBe(1536); // Should fallback to default expect((config as any).maxVectors).toBe(10000); // Should fallback to default }); }); - + describe('isVectorStoreFactory', () => { it('should return true for valid VectorStoreFactory objects', async () => { const result = await createDefaultVectorStore(); - + expect(isVectorStoreFactory(result)).toBe(true); - + // Cleanup await result.manager.disconnect(); }); - + it('should return false for invalid objects', () => { expect(isVectorStoreFactory(null)).toBe(false); expect(isVectorStoreFactory(undefined)).toBe(false); @@ -385,30 +385,32 @@ describe('Vector Storage Factory', () => { expect(isVectorStoreFactory({ store: {} })).toBe(false); expect(isVectorStoreFactory({ manager: {}, store: {} })).toBe(false); }); - + it('should return false for objects with wrong types', () => { const fakeFactory = { manager: { isConnected: () => true }, store: { search: () => Promise.resolve([]) }, }; - + expect(isVectorStoreFactory(fakeFactory)).toBe(false); }); }); - + describe('Error Handling', () => { it('should clean up on factory creation failure', async () => { // Mock manager to fail after creation const originalConnect = VectorStoreManager.prototype.connect; - VectorStoreManager.prototype.connect = vi.fn().mockRejectedValue(new Error('Manager connection failed')); - + VectorStoreManager.prototype.connect = vi + .fn() + .mockRejectedValue(new Error('Manager connection failed')); + const config: VectorStoreConfig = { type: 'in-memory', collectionName: 'fail_test', dimension: 128, maxVectors: 100, }; - + try { await expect(createVectorStore(config)).rejects.toThrow('Manager connection failed'); } finally { @@ -416,29 +418,29 @@ describe('Vector Storage Factory', () => { VectorStoreManager.prototype.connect = originalConnect; } }); - + it('should handle malformed configuration gracefully', async () => { const malformedConfig = { type: 'in-memory', // Missing required fields } as any; - + await expect(createVectorStore(malformedConfig)).rejects.toThrow(); }); }); - + describe('Integration Scenarios', () => { it('should support multiple vector stores simultaneously', async () => { const result1 = await createDefaultVectorStore('collection1', 256); const result2 = await createDefaultVectorStore('collection2', 512); - + try { // Both should be independent expect(result1.manager.getInfo().backend.collectionName).toBe('collection1'); expect(result2.manager.getInfo().backend.collectionName).toBe('collection2'); expect(result1.manager.getInfo().backend.dimension).toBe(256); expect(result2.manager.getInfo().backend.dimension).toBe(512); - + // Both should be connected expect(result1.manager.isConnected()).toBe(true); expect(result2.manager.isConnected()).toBe(true); @@ -448,23 +450,23 @@ describe('Vector Storage Factory', () => { await result2.manager.disconnect(); } }); - + it('should handle rapid creation and destruction', async () => { const configs = [ { type: 'in-memory' as const, collectionName: 'rapid1', dimension: 128, maxVectors: 100 }, { type: 'in-memory' as const, collectionName: 'rapid2', dimension: 256, maxVectors: 200 }, { type: 'in-memory' as const, collectionName: 'rapid3', dimension: 512, maxVectors: 300 }, ]; - + const results = []; - + try { // Create multiple stores rapidly for (const config of configs) { const result = await createVectorStore(config); results.push(result); } - + // All should be connected for (const result of results) { expect(result.manager.isConnected()).toBe(true); @@ -477,4 +479,4 @@ describe('Vector Storage Factory', () => { } }); }); -}); \ No newline at end of file +}); diff --git a/src/core/vector_storage/__test__/qdrant.test.ts b/src/core/vector_storage/__test__/qdrant.test.ts index d32ddf70f..3209f6539 100644 --- a/src/core/vector_storage/__test__/qdrant.test.ts +++ b/src/core/vector_storage/__test__/qdrant.test.ts @@ -170,18 +170,17 @@ describe('QdrantBackend', () => { ]; const ids = ['vec1', 'vec2']; const payloads = [{ title: 'First' }, { title: 'Second' }]; - await backend.insert(vectors, ids, payloads); expect(mockQdrantClient.upsert).toHaveBeenCalledWith('test_collection', { points: [ { - id: 'vec1', + id: 1, vector: [1, 2, 3], payload: { title: 'First' }, }, { - id: 'vec2', + id: 2, vector: [4, 5, 6], payload: { title: 'Second' }, }, diff --git a/src/core/vector_storage/factory.ts b/src/core/vector_storage/factory.ts index ed6ff3f72..90f329afd 100644 --- a/src/core/vector_storage/factory.ts +++ b/src/core/vector_storage/factory.ts @@ -194,7 +194,7 @@ export async function createVectorStoreFromEnv(): Promise { * ```typescript * const config = getVectorStoreConfigFromEnv(); * console.log('Vector store configuration:', config); - * + * * // Then use the config to create the store * const { manager, store } = await createVectorStore(config); * ``` @@ -204,7 +204,9 @@ export function getVectorStoreConfigFromEnv(): VectorStoreConfig { const storeType = env.VECTOR_STORE_TYPE; const collectionName = env.VECTOR_STORE_COLLECTION; const dimension = Number.isNaN(env.VECTOR_STORE_DIMENSION) ? 1536 : env.VECTOR_STORE_DIMENSION; - const maxVectors = Number.isNaN(env.VECTOR_STORE_MAX_VECTORS) ? 10000 : env.VECTOR_STORE_MAX_VECTORS; + const maxVectors = Number.isNaN(env.VECTOR_STORE_MAX_VECTORS) + ? 10000 + : env.VECTOR_STORE_MAX_VECTORS; // Build configuration based on type let config: VectorStoreConfig; @@ -277,13 +279,17 @@ export function getQdrantConfigFromEnv(): QdrantBackendConfig | null { const qdrantHost = process.env.VECTOR_STORE_HOST; const qdrantPort = process.env.VECTOR_STORE_PORT; + // Always resolve collectionName to a string + const collectionName = + process.env.VECTOR_STORE_COLLECTION_NAME || process.env.VECTOR_STORE_COLLECTION || 'default'; + // Check if we have cloud configuration if (qdrantUrl) { return { type: 'qdrant', url: qdrantUrl, apiKey: qdrantApiKey, // API key is required for cloud - collectionName: process.env.VECTOR_STORE_COLLECTION_NAME, + collectionName, dimension: parseInt(process.env.VECTOR_STORE_DIMENSION || '1536', 10), distance: (process.env.VECTOR_STORE_DISTANCE as any) || 'Cosine', }; @@ -296,7 +302,7 @@ export function getQdrantConfigFromEnv(): QdrantBackendConfig | null { host: qdrantHost || 'localhost', port: qdrantPort ? parseInt(qdrantPort, 10) : 6333, apiKey: qdrantApiKey, // Optional for local - collectionName: process.env.VECTOR_STORE_COLLECTION_NAME, + collectionName, dimension: parseInt(process.env.VECTOR_STORE_DIMENSION || '1536', 10), distance: (process.env.VECTOR_STORE_DISTANCE as any) || 'Cosine', }; diff --git a/src/core/vector_storage/manager.ts b/src/core/vector_storage/manager.ts index 4260b64f5..3ed4ccb47 100644 --- a/src/core/vector_storage/manager.ts +++ b/src/core/vector_storage/manager.ts @@ -88,6 +88,9 @@ export class VectorStoreManager { private static qdrantModule?: any; private static inMemoryModule?: any; + // In VectorStoreManager, track if in-memory is used as fallback or primary + private usedFallback = false; + /** * Creates a new VectorStoreManager instance * @@ -137,7 +140,7 @@ export class VectorStoreManager { backend: { type: this.backendMetadata.type, connected: this.store?.isConnected() ?? false, - fallback: this.backendMetadata.isFallback, + fallback: this.usedFallback, collectionName: this.config.collectionName, dimension: this.config.dimension, }, @@ -175,6 +178,7 @@ export class VectorStoreManager { * @throws {VectorStoreConnectionError} If backend fails to connect */ public async connect(): Promise { + this.usedFallback = false; // Always reset at the start of each connection attempt // Check if already connected if (this.connected && this.store) { this.logger.debug(`${LOG_PREFIXES.MANAGER} Already connected`, { @@ -196,6 +200,7 @@ export class VectorStoreManager { this.store = await this.createBackend(); await this.store.connect(); this.backendMetadata.connectionTime = Date.now() - startTime; + this.usedFallback = false; // Not a fallback if primary backend succeeded this.logger.info(`${LOG_PREFIXES.MANAGER} Connected successfully`, { type: this.backendMetadata.type, @@ -221,12 +226,15 @@ export class VectorStoreManager { this.backendMetadata.type = BACKEND_TYPES.IN_MEMORY; this.backendMetadata.isFallback = true; this.backendMetadata.connectionTime = Date.now() - startTime; + this.usedFallback = true; // Mark as fallback this.logger.info(`${LOG_PREFIXES.MANAGER} Connected to fallback backend`, { type: this.backendMetadata.type, originalType: this.config.type, }); } else { + // In-memory is primary, not a fallback + this.usedFallback = false; throw backendError; // Re-throw if already using in-memory } } @@ -255,6 +263,7 @@ export class VectorStoreManager { // Reset state this.store = undefined; this.connected = false; + this.usedFallback = false; throw error; } @@ -289,6 +298,7 @@ export class VectorStoreManager { // Always clean up state this.store = undefined; this.connected = false; + this.usedFallback = false; // Reset metadata this.backendMetadata = {