Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,11 @@

# ---------------------------------------------------------------------------
# Required: MongoDB connection URI
# If using the bundled docker-compose.yaml, use:
# MONGO_URI=mongodb://admin:changeme@localhost:27017/
# For an external Atlas cluster:
# MONGO_URI=mongodb+srv://user:pass@cluster.mongodb.net/
# If using the bundled docker-compose.yaml, set this to:
# mongodb://<MONGO_USERNAME>:<MONGO_PASSWORD>@localhost:27017/
# where MONGO_USERNAME and MONGO_PASSWORD match the values below.
# For an external Atlas cluster, set this to your Atlas connection string
# (found in the Atlas UI under "Connect > Drivers").
# ---------------------------------------------------------------------------
MONGO_URI=mongodb://localhost:27017/

Expand Down
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
# --- Git worktrees ---
.worktrees/

# --- Environment variables ---
.env
.env.local
Expand Down
4 changes: 3 additions & 1 deletion backend/app/core/embedding.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
import os
from typing import List
from .database_handlers import OllamaEmbeddingFunction

class EmbeddingCreator:
def __init__(self):
ollama_host = os.environ.get("OLLAMA_HOST", "http://localhost:11435")
self.ef = OllamaEmbeddingFunction(
url="http://localhost:11435/api/embeddings",
url=f"{ollama_host}/api/embeddings",
model_name="nomic-embed-text"
)

Expand Down
33 changes: 19 additions & 14 deletions experiments/08_rag_real_world/run_experiment.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,20 +77,25 @@ def _init_client(self):
keys = os.environ.get("OPENAI_API_KEY", "").split(",")
self.api_keys = [k.strip() for k in keys if k.strip()]
if not self.api_keys:
print("WARN: OPENAI_API_KEY not set. Using mock judge.")
self.client = None
else:
from openai import OpenAI
self.client = OpenAI(api_key=self.api_keys[0])
raise ValueError(
"OPENAI_API_KEY environment variable is not set or is empty. "
"Run: export OPENAI_API_KEY=your_key_here\n"
"Without a valid key, judge scores would be fabricated. Aborting."
)
from openai import OpenAI
self.client = OpenAI(api_key=self.api_keys[0])

elif judge_type == "groq":
keys = os.environ.get("GROQ_API_KEY", "").split(",")
self.api_keys = [k.strip() for k in keys if k.strip()]
if not self.api_keys:
self.client = None
else:
from groq import Groq
self.client = Groq(api_key=self.api_keys[0])
raise ValueError(
"GROQ_API_KEY environment variable is not set or is empty. "
"Run: export GROQ_API_KEY=your_key_here\n"
"Without a valid key, judge scores would be fabricated. Aborting."
)
from groq import Groq
self.client = Groq(api_key=self.api_keys[0])

def _rotate_key(self):
"""Rotate to next API key if available."""
Expand All @@ -114,10 +119,7 @@ def score(self, prediction: str, ground_truth: str) -> float:
"""Score prediction against ground truth."""
if not prediction or len(prediction.strip()) < 5:
return 0.0

if not self.client: # Mock mode
return 0.5


prompt = f"""Compare these two root cause descriptions and rate their similarity from 0.0 to 1.0.

Ground Truth: {ground_truth}
Expand All @@ -134,7 +136,10 @@ def score(self, prediction: str, ground_truth: str) -> float:
print(f" Judge attempt {attempt+1}/3 failed: {e}")
time.sleep(2)

return 0.5
# All retries exhausted — return 0.0 and warn rather than silently return 0.5
print(f" WARNING: all scoring attempts failed for judge '{self.judge_name}'. "
f"Returning 0.0 (not a mock). Check API key and connectivity.")
return 0.0

def _call_judge(self, prompt: str) -> Optional[float]:
judge_type = self.judge_config["type"]
Expand Down