implemented hybrid retreival

Author: vendz

README.md

@@ -39,11 +39,35 @@ nl2sql chat
 - `/help` - Show commands
 - `/schema` - Display schema
 - `/models` - List models
+- `/explain <query>` - Explain similarity scores (e.g., /explain show users)
 - `/history` - View history
 - `/clear` - Clear chat
 - `/model <name>` - Switch AI model
 - `/exit` - Quit
 
+## Improving Model Discovery
+
+The CLI uses semantic search to find relevant tables. Help it work better by:
+
+1. **Add comments to your models:**
+
+```javascript
+   // User accounts and authentication data
+   const User = sequelize.define('User', { ... });
+```
+
+2. **Use descriptive column names:**
+   - ✅ `email`, `phone_number`, `created_at`
+   - ❌ `col1`, `data`, `field_x`
+
+3. **Document relationships clearly:**
+
+```javascript
+User.hasMany(Order, { as: 'purchases', foreignKey: 'customer_id' });
+```
+
+No special configuration needed - the system learns from your schema structure!
+
 ### Example Queries
 
 - Show all users

src/cli/chat.ts

@@ -2,9 +2,11 @@ import chalk from 'chalk';
 import { WatchableSchema } from '../models/analyzer';
 import { VectorSearch } from '../rag/vector-search';
 import { OpenRouterClient, ChatMessage } from '../llm/openrouter';
+import { HybridRetrieval } from '../rag/hybrid-retrieval';
 import * as readline from 'readline';
 
 const vectorSearch = new VectorSearch();
+let hybridRetrieval: HybridRetrieval;
 
 export async function runChat(watchableSchema: WatchableSchema): Promise<void> {
   console.log(
@@ -17,6 +19,7 @@ export async function runChat(watchableSchema: WatchableSchema): Promise<void> {
   console.log();
 
   await vectorSearch.initialize(watchableSchema.models);
+  hybridRetrieval = new HybridRetrieval(vectorSearch, watchableSchema.models);
 
   const apiKey = process.env.OPENROUTER_API_KEY;
   if (!apiKey) {
@@ -100,9 +103,17 @@ export async function runChat(watchableSchema: WatchableSchema): Promise<void> {
       } catch (error) {
         console.log(chalk.red('❌ Failed to reload schema\n'));
       }
-    } else if (userInput.startsWith('/debug ')) {
-      const query = userInput.substring(7).trim();
-      await debugSimilarity(query, vectorSearch);
+    } else if (userInput.startsWith('/explain ')) {
+      const query = userInput.substring(9).trim();
+      const explanation = await hybridRetrieval.explainSelection(query);
+
+      console.log(chalk.yellow('\n🔍 Selection Explanation:'));
+      console.log(chalk.gray('─'.repeat(60)));
+      explanation.slice(0, 10).forEach((item, i) => {
+        console.log(chalk.cyan(`${i + 1}. ${item.model}`));
+        console.log(chalk.gray(`   ${item.reason}`));
+      });
+      console.log(chalk.gray('─'.repeat(60) + '\n'));
     } else if (userInput.startsWith('/model ')) {
       const modelName = userInput.substring(7).trim();
       try {
@@ -118,13 +129,7 @@ export async function runChat(watchableSchema: WatchableSchema): Promise<void> {
       console.log(chalk.red(`❌ Unknown command: ${userInput}`));
       console.log(chalk.gray('Type /help for available commands.\n'));
     } else {
-      await handleQuery(
-        userInput,
-        llm,
-        watchableSchema,
-        chatHistory,
-        vectorSearch
-      );
+      await handleQuery(userInput, llm, watchableSchema, chatHistory);
     }
 
     rl.prompt();
@@ -147,34 +152,36 @@ async function handleQuery(
   query: string,
   llm: OpenRouterClient,
   watchableSchema: WatchableSchema,
-  chatHistory: ChatMessage[],
-  vectorSearch: VectorSearch
+  chatHistory: ChatMessage[]
 ): Promise<void> {
   try {
     process.stdout.write(chalk.blue('🤔 Thinking... '));
 
-    // Find relevant models using vector search
-    const relevantModels = await vectorSearch.findRelevant(query, 5, 0.25);
+    const relevantModels = await hybridRetrieval.findRelevant(query, {
+      topK: 5,
+      threshold: 0.25,
+      includeRelated: true,
+    });
 
     readline.clearLine(process.stdout, 0);
     readline.cursorTo(process.stdout, 0);
 
-    // Show which models were selected
     if (relevantModels.length > 0) {
       console.log(
         chalk.magenta(
           `📊 Using ${relevantModels.length}/${watchableSchema.models.length} relevant table(s):`
         )
       );
-      relevantModels.forEach((m) => console.log(chalk.gray(`   • ${m.name}`)));
+      relevantModels.forEach((m) =>
+        console.log(chalk.gray(`   • ${m.name} (${m.tableName})`))
+      );
       console.log();
     } else {
       console.log(
         chalk.yellow('⚠️  No relevant tables found, using all models\n')
       );
     }
 
-    // Generate SQL with filtered models
     const modelsToUse =
       relevantModels.length > 0 ? relevantModels : watchableSchema.models;
     const response = await llm.generateSQL(query, modelsToUse, chatHistory);
@@ -203,33 +210,6 @@ async function handleQuery(
   }
 }
 
-async function debugSimilarity(
-  query: string,
-  vectorSearch: VectorSearch
-): Promise<void> {
-  try {
-    const scores = await vectorSearch.getScores(query);
-    console.log(
-      chalk.yellow('\n🔍 Similarity scores for:'),
-      chalk.white(query)
-    );
-    console.log(chalk.gray('─'.repeat(60)));
-    scores.slice(0, 10).forEach((s, i) => {
-      const percentage = (s.score * 100).toFixed(1);
-      const bar = '█'.repeat(Math.floor(s.score * 20));
-      console.log(
-        `${i + 1}. ${chalk.cyan(s.name.padEnd(20))} ${bar} ${percentage}%`
-      );
-    });
-    console.log(chalk.gray('─'.repeat(60) + '\n'));
-  } catch (error) {
-    console.log(
-      chalk.red('❌ Error:'),
-      error instanceof Error ? error.message : 'Failed to get scores'
-    );
-  }
-}
-
 function displayHelp(): void {
   console.log(chalk.yellow('📚 Available commands:'));
   console.log(chalk.gray('─'.repeat(60)));
@@ -244,8 +224,8 @@ function displayHelp(): void {
       chalk.gray('- List all available models with details')
   );
   console.log(
-    chalk.cyan('  /debug    ') +
-      chalk.gray('- Debug similarity scores (e.g., /debug show users)')
+    chalk.cyan('  /explain    ') +
+      chalk.gray('- Explain similarity scores (e.g., /explain show users)')
   );
   console.log(chalk.cyan('  /history  ') + chalk.gray('- Show chat history'));
   console.log(

src/models/analyzer.ts

@@ -5,7 +5,6 @@ import * as chokidar from 'chokidar';
 import { parse } from '@babel/parser';
 import traverse, { NodePath } from '@babel/traverse';
 
-// At the top of analyzer.ts (around line 7)
 interface ColumnInfo {
   name: string;
   type?: string;
@@ -23,6 +22,7 @@ interface ColumnInfo {
 export interface ModelInfo {
   name: string;
   tableName: string;
+  description?: string;
   columns: ColumnInfo[];
   associations: Array<{
     type: string;
@@ -324,33 +324,99 @@ function mergeCentralizedAssociations(
   }
 }
 
+/**
+ * Extract ALL forms of documentation (universal approach)
+ */
+function extractModelDescription(
+  content: string,
+  modelName: string
+): string | undefined {
+  const descriptions: string[] = [];
+
+  // 1. JSDoc comments
+  const jsDocPattern = /\/\*\*\s*([\s\S]*?)\s*\*\//g;
+  let jsDocMatch;
+  while ((jsDocMatch = jsDocPattern.exec(content)) !== null) {
+    const comment = jsDocMatch[1]
+      .split('\n')
+      .map((line) => line.replace(/^\s*\*\s?/, '').trim())
+      .filter((line) => line && !line.startsWith('@'))
+      .join(' ');
+    if (comment) descriptions.push(comment);
+  }
+
+  // 2. Single-line comments above model
+  const singleLinePattern = new RegExp(
+    `(?://\\s*(.+?)\\s*\\n)+[\\s\\S]{0,50}(class|const|export)\\s+${modelName}`,
+    'i'
+  );
+  const singleLineMatch = content.match(singleLinePattern);
+  if (singleLineMatch) {
+    descriptions.push(singleLineMatch[1].trim());
+  }
+
+  // 3. Inline comments in define()
+  const defineCommentPattern = /\.define\s*\([^)]*\)\s*\/\/\s*(.+?)$/m;
+  const defineMatch = content.match(defineCommentPattern);
+  if (defineMatch) {
+    descriptions.push(defineMatch[1].trim());
+  }
+
+  // 4. Comments in table definition
+  const tableNamePattern =
+    /tableName:\s*['"`]([^'"`]+)['"`]\s*,?\s*(?:\/\/\s*(.+?)$)?/m;
+  const tableMatch = content.match(tableNamePattern);
+  if (tableMatch && tableMatch[2]) {
+    descriptions.push(tableMatch[2].trim());
+  }
+
+  // 5. Block comments anywhere in file mentioning the table
+  const blockCommentPattern = /\/\*[\s\S]*?\*\//g;
+  let blockMatch;
+  while ((blockMatch = blockCommentPattern.exec(content)) !== null) {
+    const comment = blockMatch[0];
+    if (
+      comment.toLowerCase().includes(modelName.toLowerCase()) ||
+      comment.toLowerCase().includes('table') ||
+      comment.toLowerCase().includes('model')
+    ) {
+      const cleaned = comment
+        .replace(/\/\*|\*\//g, '')
+        .replace(/\*/g, '')
+        .trim();
+      if (cleaned) descriptions.push(cleaned);
+    }
+  }
+
+  return descriptions.length > 0 ? descriptions.join('. ') : undefined;
+}
+
 /**
  * Parse a Sequelize model file to extract model information
  */
 function parseModelFile(content: string, filename: string): ModelInfo | null {
-  // Extract model name from filename
   const modelName = path.basename(filename, path.extname(filename));
 
-  // Try to extract table name
   let tableName = modelName.toLowerCase();
   const tableNameMatch = content.match(/tableName:\s*['"`]([^'"`]+)['"`]/);
   if (tableNameMatch) {
     tableName = tableNameMatch[1];
   }
 
-  // Extract columns from the model definition
+  const description = extractModelDescription(content, modelName);
+
   const columns = parseColumns(content);
 
   if (columns.length === 0) {
     return null;
   }
 
-  // Extract in-file associations
   const associations = parseAssociations(content);
 
   return {
     name: modelName,
     tableName,
+    description,
     columns,
     associations,
   };