diff --git a/alembic/versions/0021_repro_code_experience.py b/alembic/versions/0021_repro_code_experience.py new file mode 100644 index 00000000..1ca99d04 --- /dev/null +++ b/alembic/versions/0021_repro_code_experience.py @@ -0,0 +1,42 @@ +"""repro_code_experience table for CodeMemory persistence + +Revision ID: 0021_repro_code_experience +Revises: 0020_memory_embedding +Create Date: 2026-03-03 + +Issue #162: Persist CodeMemory experience data so it survives process restarts. +Creates the repro_code_experience table with indexes on paper_id and pack_id. +""" +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision = "0021_repro_code_experience" +down_revision = "0020_memory_embedding" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "repro_code_experience", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("pack_id", sa.String(64), nullable=True), + sa.Column("paper_id", sa.String(256), nullable=True), + sa.Column("pattern_type", sa.String(32), nullable=False), + sa.Column("content", sa.Text(), nullable=False, server_default=""), + sa.Column("code_snippet", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_repro_code_experience_paper_id", "repro_code_experience", ["paper_id"]) + op.create_index("ix_repro_code_experience_pack_id", "repro_code_experience", ["pack_id"]) + op.create_index("ix_repro_code_experience_pattern_type", "repro_code_experience", ["pattern_type"]) + + +def downgrade() -> None: + op.drop_index("ix_repro_code_experience_pattern_type", table_name="repro_code_experience") + op.drop_index("ix_repro_code_experience_pack_id", table_name="repro_code_experience") + op.drop_index("ix_repro_code_experience_paper_id", table_name="repro_code_experience") + op.drop_table("repro_code_experience") diff --git a/src/paperbot/infrastructure/stores/models.py b/src/paperbot/infrastructure/stores/models.py index d8740110..f1f94916 100644 --- a/src/paperbot/infrastructure/stores/models.py +++ b/src/paperbot/infrastructure/stores/models.py @@ -1258,3 +1258,27 @@ class ReproContextFeedbackModel(Base): created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True) pack = relationship("ReproContextPackModel", back_populates="feedback_rows") + + +# ============================================================================ +# Issue #162: CodeMemory Persistence +# ============================================================================ + + +class ReproCodeExperienceModel(Base): + """Persisted code generation experience from the Paper2Code pipeline. + + Stores success patterns, failure reasons, and verified structures so that + CodeMemory can pre-load prior experience when regenerating for the same paper. + """ + + __tablename__ = "repro_code_experience" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + pack_id: Mapped[Optional[str]] = mapped_column(String(64), nullable=True, index=True) + paper_id: Mapped[Optional[str]] = mapped_column(String(256), nullable=True, index=True) + # pattern_type: success_pattern | failure_reason | verified_structure + pattern_type: Mapped[str] = mapped_column(String(32), index=True) + content: Mapped[str] = mapped_column(Text, default="") + code_snippet: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True) diff --git a/src/paperbot/infrastructure/stores/repro_experience_store.py b/src/paperbot/infrastructure/stores/repro_experience_store.py new file mode 100644 index 00000000..a0063af1 --- /dev/null +++ b/src/paperbot/infrastructure/stores/repro_experience_store.py @@ -0,0 +1,99 @@ +"""SQLAlchemy store for ReproCodeExperienceModel (issue #162).""" +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional + +from sqlalchemy import select + +from paperbot.infrastructure.stores.models import Base, ReproCodeExperienceModel +from paperbot.infrastructure.stores.sqlalchemy_db import SessionProvider, get_db_url + +_VALID_TYPES = {"success_pattern", "failure_reason", "verified_structure"} + + +class ReproExperienceStore: + """CRUD store for persisted code generation experiences.""" + + def __init__(self, db_url: Optional[str] = None, *, auto_create_schema: bool = True): + self.db_url = db_url or get_db_url() + self._provider = SessionProvider(self.db_url) + if auto_create_schema: + Base.metadata.create_all(self._provider.engine) + + def add( + self, + *, + pattern_type: str, + content: str, + paper_id: Optional[str] = None, + pack_id: Optional[str] = None, + code_snippet: Optional[str] = None, + ) -> ReproCodeExperienceModel: + """Persist one experience record. Returns the saved row.""" + if pattern_type not in _VALID_TYPES: + raise ValueError(f"pattern_type must be one of {_VALID_TYPES}") + now = datetime.now(timezone.utc) + row = ReproCodeExperienceModel( + pack_id=pack_id, + paper_id=paper_id, + pattern_type=pattern_type, + content=(content or "").strip(), + code_snippet=code_snippet, + created_at=now, + ) + with self._provider.session() as session: + session.add(row) + session.commit() + session.refresh(row) + return row + + def get_by_paper_id( + self, + paper_id: str, + *, + pattern_type: Optional[str] = None, + limit: int = 50, + ) -> List[Dict[str, Any]]: + """Retrieve experiences for a specific paper, newest first.""" + with self._provider.session() as session: + stmt = ( + select(ReproCodeExperienceModel) + .where(ReproCodeExperienceModel.paper_id == paper_id) + ) + if pattern_type: + stmt = stmt.where(ReproCodeExperienceModel.pattern_type == pattern_type) + stmt = stmt.order_by(ReproCodeExperienceModel.created_at.desc()).limit(limit) + rows = session.execute(stmt).scalars().all() + return [self._to_dict(r) for r in rows] + + def get_by_pack_id( + self, + pack_id: str, + *, + pattern_type: Optional[str] = None, + limit: int = 50, + ) -> List[Dict[str, Any]]: + """Retrieve experiences for a specific P2C pack, newest first.""" + with self._provider.session() as session: + stmt = ( + select(ReproCodeExperienceModel) + .where(ReproCodeExperienceModel.pack_id == pack_id) + ) + if pattern_type: + stmt = stmt.where(ReproCodeExperienceModel.pattern_type == pattern_type) + stmt = stmt.order_by(ReproCodeExperienceModel.created_at.desc()).limit(limit) + rows = session.execute(stmt).scalars().all() + return [self._to_dict(r) for r in rows] + + @staticmethod + def _to_dict(r: ReproCodeExperienceModel) -> Dict[str, Any]: + return { + "id": r.id, + "pack_id": r.pack_id, + "paper_id": r.paper_id, + "pattern_type": r.pattern_type, + "content": r.content, + "code_snippet": r.code_snippet, + "created_at": r.created_at.isoformat() if r.created_at else None, + } diff --git a/src/paperbot/repro/agents/debugging_agent.py b/src/paperbot/repro/agents/debugging_agent.py index 04df00e7..97ed9ae9 100644 --- a/src/paperbot/repro/agents/debugging_agent.py +++ b/src/paperbot/repro/agents/debugging_agent.py @@ -102,10 +102,11 @@ class DebuggingAgent(BaseAgent): "bs4": "beautifulsoup4", } - def __init__(self, output_dir: Optional[Path] = None, **kwargs): + def __init__(self, output_dir: Optional[Path] = None, experience_store=None, **kwargs): super().__init__(name="DebuggingAgent", **kwargs) self.output_dir = output_dir self.repair_history: List[RepairAttempt] = [] + self._experience_store = experience_store async def execute(self, context: Dict[str, Any]) -> AgentResult: """Execute debugging pipeline.""" @@ -143,6 +144,25 @@ async def execute(self, context: Dict[str, Any]) -> AgentResult: if repair_result.success: self.log(f"Repair successful: {repair_result.fix_applied}") context["last_repair"] = repair_result + + # Persist failure reason + fix for future runs (issue #162) + if self._experience_store: + paper_context = context.get("paper_context") + paper_id = ( + getattr(paper_context, "paper_id", None) + or getattr(paper_context, "arxiv_id", None) + if paper_context else None + ) + try: + self._experience_store.add( + pattern_type="failure_reason", + content=f"[{error_type.value}] fixed: {repair_result.fix_applied}", + paper_id=paper_id, + code_snippet=repair_result.original_error[:1000], + ) + except Exception: # noqa: BLE001 + pass + return AgentResult.success( data={ "repair": repair_result, diff --git a/src/paperbot/repro/memory/code_memory.py b/src/paperbot/repro/memory/code_memory.py index 1edbd0eb..c2a9be95 100644 --- a/src/paperbot/repro/memory/code_memory.py +++ b/src/paperbot/repro/memory/code_memory.py @@ -15,10 +15,13 @@ import re from dataclasses import dataclass, field from pathlib import Path -from typing import Dict, List, Optional, Set, Tuple +from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple from .symbol_index import SymbolIndex, SymbolInfo +if TYPE_CHECKING: + from paperbot.infrastructure.stores.repro_experience_store import ReproExperienceStore + logger = logging.getLogger(__name__) @@ -58,17 +61,25 @@ class CodeMemory: # Approximate tokens per character (conservative estimate) CHARS_PER_TOKEN = 4 - def __init__(self, max_context_tokens: int = 8000): + def __init__( + self, + max_context_tokens: int = 8000, + experience_store: "Optional[ReproExperienceStore]" = None, + ): """ Initialize CodeMemory. Args: max_context_tokens: Maximum tokens for context injection + experience_store: Optional store for persisting/loading experiences """ self.max_context_tokens = max_context_tokens self._files: Dict[str, FileInfo] = {} self._symbol_index = SymbolIndex() self._generation_order: List[str] = [] + self._experience_store: "Optional[ReproExperienceStore]" = experience_store + # Prior experiences loaded from DB for context injection + self._prior_experiences: List[Dict] = [] def add_file(self, path: str, content: str, purpose: str = "") -> None: """ @@ -198,6 +209,19 @@ def get_relevant_context( if len(interfaces) < remaining_chars: context_parts.append(f"\n# === Available Interfaces ===\n{interfaces}") + # 4. Prior experiences from DB (success patterns / verified structures) + if self._prior_experiences and remaining_chars > 200: + exp_lines = [] + for exp in self._prior_experiences[:5]: + ptype = exp.get("pattern_type", "") + content = exp.get("content", "") + if ptype in ("success_pattern", "verified_structure") and content: + exp_lines.append(f" [{ptype}] {content}") + if exp_lines: + prior_ctx = "# === Prior Experience (same paper) ===\n" + "\n".join(exp_lines) + if len(prior_ctx) < remaining_chars: + context_parts.append(prior_ctx) + return "\n\n".join(context_parts) def _predict_dependencies(self, current_file: str) -> List[str]: @@ -325,11 +349,108 @@ def get_dependency_graph(self) -> Dict[str, Set[str]]: """Get the file dependency graph.""" return {path: info.dependencies for path, info in self._files.items()} + # ------------------------------------------------------------------ + # Persistence helpers (issue #162) + # ------------------------------------------------------------------ + + def load_experiences_from_db( + self, + paper_id: str, + *, + pack_id: Optional[str] = None, + limit: int = 20, + ) -> None: + """Pre-load prior experiences for *paper_id* from the DB into memory. + + Loaded records are stored in ``_prior_experiences`` and injected into + ``get_relevant_context()`` so the LLM can see what worked before. + """ + if not self._experience_store or not paper_id: + return + try: + rows = self._experience_store.get_by_paper_id(paper_id, limit=limit) + if pack_id: + pack_rows = self._experience_store.get_by_pack_id(pack_id, limit=limit) + seen_ids = {r["id"] for r in rows} + rows += [r for r in pack_rows if r["id"] not in seen_ids] + self._prior_experiences = rows + logger.debug("Loaded %d prior experiences for paper_id=%s", len(rows), paper_id) + except Exception: # noqa: BLE001 + logger.debug("Failed to load experiences from DB", exc_info=True) + + def record_success_pattern( + self, + *, + paper_id: Optional[str], + pack_id: Optional[str] = None, + filepath: str, + code_snippet: Optional[str] = None, + ) -> None: + """Record that *filepath* was successfully generated (best-effort).""" + if not self._experience_store: + return + try: + self._experience_store.add( + pattern_type="success_pattern", + content=f"Successfully generated {filepath}", + paper_id=paper_id, + pack_id=pack_id, + code_snippet=(code_snippet or "")[:2000] or None, + ) + except Exception: # noqa: BLE001 + logger.debug("Failed to record success_pattern", exc_info=True) + + def record_verified_structure( + self, + *, + paper_id: Optional[str], + pack_id: Optional[str] = None, + description: str, + code_snippet: Optional[str] = None, + ) -> None: + """Record that a code structure passed verification (best-effort).""" + if not self._experience_store: + return + try: + self._experience_store.add( + pattern_type="verified_structure", + content=description, + paper_id=paper_id, + pack_id=pack_id, + code_snippet=(code_snippet or "")[:2000] or None, + ) + except Exception: # noqa: BLE001 + logger.debug("Failed to record verified_structure", exc_info=True) + + def record_failure_reason( + self, + *, + paper_id: Optional[str], + pack_id: Optional[str] = None, + error_type: str, + fix_applied: str, + code_snippet: Optional[str] = None, + ) -> None: + """Record a debugging fix so future runs can avoid the same error (best-effort).""" + if not self._experience_store: + return + try: + self._experience_store.add( + pattern_type="failure_reason", + content=f"[{error_type}] fixed: {fix_applied}", + paper_id=paper_id, + pack_id=pack_id, + code_snippet=(code_snippet or "")[:2000] or None, + ) + except Exception: # noqa: BLE001 + logger.debug("Failed to record failure_reason", exc_info=True) + def clear(self) -> None: """Clear all memory.""" self._files.clear() self._symbol_index = SymbolIndex() self._generation_order.clear() + self._prior_experiences.clear() @property def files(self) -> Dict[str, str]: diff --git a/src/paperbot/repro/nodes/generation_node.py b/src/paperbot/repro/nodes/generation_node.py index 4122be21..7bcc223a 100644 --- a/src/paperbot/repro/nodes/generation_node.py +++ b/src/paperbot/repro/nodes/generation_node.py @@ -85,6 +85,7 @@ def __init__( llm_client=None, max_context_tokens: int = 8000, use_rag: bool = True, + experience_store=None, **kwargs ): super().__init__(node_name="GenerationNode", **kwargs) @@ -92,8 +93,11 @@ def __init__( self.max_context_tokens = max_context_tokens self.use_rag = use_rag - # Initialize Code Memory - self.memory = CodeMemory(max_context_tokens=max_context_tokens) + # Initialize Code Memory (with optional experience store for persistence) + self.memory = CodeMemory( + max_context_tokens=max_context_tokens, + experience_store=experience_store, + ) # Initialize Knowledge Base self.knowledge_base = CodeKnowledgeBase.from_builtin() if use_rag else None @@ -115,6 +119,11 @@ async def _execute(self, input_data: tuple, **kwargs) -> Dict[str, str]: # Clear memory for fresh generation self.memory.clear() + # Pre-load prior experiences for the same paper (issue #162) + paper_id = getattr(paper_context, "paper_id", None) or getattr(paper_context, "arxiv_id", None) + if paper_id: + self.memory.load_experiences_from_db(paper_id) + files = {} # Determine optimal file generation order @@ -143,6 +152,13 @@ async def _execute(self, input_data: tuple, **kwargs) -> Dict[str, str]: self.memory.add_file(filepath, code, purpose=purpose) logger.debug(f"Generated {filepath} ({len(code)} chars)") + # Persist success pattern (issue #162) + self.memory.record_success_pattern( + paper_id=paper_id, + filepath=filepath, + code_snippet=code[:1000], + ) + # Add requirements.txt files["requirements.txt"] = self._generate_requirements(plan) diff --git a/src/paperbot/repro/nodes/verification_node.py b/src/paperbot/repro/nodes/verification_node.py index a864cdf3..ec687176 100644 --- a/src/paperbot/repro/nodes/verification_node.py +++ b/src/paperbot/repro/nodes/verification_node.py @@ -409,12 +409,14 @@ def __init__( timeout: int = 30, max_repair_attempts: int = 3, enable_self_healing: bool = True, + experience_store=None, **kwargs ): super().__init__(node_name="VerificationNode", **kwargs) self.timeout = timeout self.max_repair_attempts = max_repair_attempts self.enable_self_healing = enable_self_healing + self._experience_store = experience_store def _validate_input(self, input_data: Any, **kwargs) -> Optional[str]: """Validate input is a valid directory path or tuple.""" @@ -497,12 +499,30 @@ async def _execute(self, input_data: Any, **kwargs) -> VerificationResult: if result.imports_ok: test_result = self._run_tests(output_dir) result.tests_ok = test_result["passed"] - + # Step 4: Smoke run (optional, no repair) if result.imports_ok: smoke_result = self._smoke_run(output_dir) result.smoke_ok = smoke_result["passed"] - + + # Persist verified structure when all essential checks pass (issue #162) + if result.all_passed and self._experience_store: + paper_context = input_data[1] if isinstance(input_data, tuple) and len(input_data) > 1 else None + paper_id = ( + getattr(paper_context, "paper_id", None) + or getattr(paper_context, "arxiv_id", None) + if paper_context else None + ) + try: + py_files = [f.name for f in output_dir.glob("*.py")] + self._experience_store.add( + pattern_type="verified_structure", + content=f"Verified structure in {output_dir.name}: {', '.join(py_files[:10])}", + paper_id=paper_id, + ) + except Exception: # noqa: BLE001 + pass + return result def _check_syntax(self, output_dir: Path) -> Dict[str, Any]: diff --git a/tests/unit/test_repro_code_experience.py b/tests/unit/test_repro_code_experience.py new file mode 100644 index 00000000..c9b3e247 --- /dev/null +++ b/tests/unit/test_repro_code_experience.py @@ -0,0 +1,172 @@ +"""Unit tests for issue #162: CodeMemory persistence via ReproExperienceStore.""" +from __future__ import annotations + +from unittest.mock import MagicMock, call, patch + +import pytest + +from paperbot.infrastructure.stores.repro_experience_store import ReproExperienceStore +from paperbot.repro.memory.code_memory import CodeMemory + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_store(db_url: str = "sqlite://") -> ReproExperienceStore: + return ReproExperienceStore(db_url=db_url, auto_create_schema=True) + + +def _make_memory(store=None) -> CodeMemory: + return CodeMemory(max_context_tokens=4000, experience_store=store) + + +# --------------------------------------------------------------------------- +# ReproExperienceStore CRUD +# --------------------------------------------------------------------------- + +class TestReproExperienceStore: + def test_add_and_get_by_paper_id(self): + store = _make_store() + store.add( + pattern_type="success_pattern", + content="Generated model.py", + paper_id="arxiv:1234", + code_snippet="class Model: pass", + ) + rows = store.get_by_paper_id("arxiv:1234") + assert len(rows) == 1 + assert rows[0]["pattern_type"] == "success_pattern" + assert rows[0]["content"] == "Generated model.py" + assert rows[0]["paper_id"] == "arxiv:1234" + + def test_add_and_get_by_pack_id(self): + store = _make_store() + store.add( + pattern_type="failure_reason", + content="[syntax] fixed indent", + pack_id="ctxp_abc", + ) + rows = store.get_by_pack_id("ctxp_abc") + assert len(rows) == 1 + assert rows[0]["pack_id"] == "ctxp_abc" + + def test_filter_by_pattern_type(self): + store = _make_store() + store.add(pattern_type="success_pattern", content="ok", paper_id="p1") + store.add(pattern_type="failure_reason", content="fail", paper_id="p1") + rows = store.get_by_paper_id("p1", pattern_type="success_pattern") + assert all(r["pattern_type"] == "success_pattern" for r in rows) + assert len(rows) == 1 + + def test_invalid_pattern_type_raises(self): + store = _make_store() + with pytest.raises(ValueError): + store.add(pattern_type="unknown_type", content="bad") + + def test_returns_empty_for_unknown_paper(self): + store = _make_store() + assert store.get_by_paper_id("does_not_exist") == [] + + def test_limit_respected(self): + store = _make_store() + for i in range(10): + store.add(pattern_type="success_pattern", content=f"file_{i}.py", paper_id="p_limit") + rows = store.get_by_paper_id("p_limit", limit=3) + assert len(rows) == 3 + + +# --------------------------------------------------------------------------- +# CodeMemory persistence methods +# --------------------------------------------------------------------------- + +class TestCodeMemoryRecordMethods: + def test_record_success_pattern_calls_store(self): + mock_store = MagicMock() + mem = _make_memory(store=mock_store) + mem.record_success_pattern(paper_id="p1", filepath="model.py", code_snippet="x=1") + mock_store.add.assert_called_once() + kwargs = mock_store.add.call_args.kwargs + assert kwargs["pattern_type"] == "success_pattern" + assert "model.py" in kwargs["content"] + + def test_record_verified_structure_calls_store(self): + mock_store = MagicMock() + mem = _make_memory(store=mock_store) + mem.record_verified_structure(paper_id="p1", description="syntax+imports passed") + mock_store.add.assert_called_once() + kwargs = mock_store.add.call_args.kwargs + assert kwargs["pattern_type"] == "verified_structure" + + def test_record_failure_reason_calls_store(self): + mock_store = MagicMock() + mem = _make_memory(store=mock_store) + mem.record_failure_reason( + paper_id="p1", error_type="SYNTAX", fix_applied="fixed indent at line 5" + ) + mock_store.add.assert_called_once() + kwargs = mock_store.add.call_args.kwargs + assert kwargs["pattern_type"] == "failure_reason" + assert "SYNTAX" in kwargs["content"] + + def test_no_store_skips_silently(self): + mem = _make_memory(store=None) + # Should not raise + mem.record_success_pattern(paper_id="p1", filepath="x.py") + mem.record_verified_structure(paper_id="p1", description="ok") + mem.record_failure_reason(paper_id="p1", error_type="LOGIC", fix_applied="fixed") + + def test_store_exception_does_not_propagate(self): + mock_store = MagicMock() + mock_store.add.side_effect = RuntimeError("db gone") + mem = _make_memory(store=mock_store) + # Should not raise despite store error + mem.record_success_pattern(paper_id="p1", filepath="x.py") + + +# --------------------------------------------------------------------------- +# CodeMemory.load_experiences_from_db +# --------------------------------------------------------------------------- + +class TestLoadExperiencesFromDb: + def test_loads_and_stores_prior_experiences(self): + mock_store = MagicMock() + mock_store.get_by_paper_id.return_value = [ + {"id": 1, "pattern_type": "success_pattern", "content": "Generated model.py", + "pack_id": None, "paper_id": "p1", "code_snippet": None, "created_at": None}, + ] + mock_store.get_by_pack_id.return_value = [] + mem = _make_memory(store=mock_store) + mem.load_experiences_from_db("p1") + assert len(mem._prior_experiences) == 1 + mock_store.get_by_paper_id.assert_called_once_with("p1", limit=20) + + def test_no_store_skips_silently(self): + mem = _make_memory(store=None) + mem.load_experiences_from_db("p1") + assert mem._prior_experiences == [] + + def test_prior_experiences_injected_into_context(self): + mock_store = MagicMock() + mock_store.get_by_paper_id.return_value = [ + {"id": 1, "pattern_type": "success_pattern", "content": "Generated model.py", + "pack_id": None, "paper_id": "p1", "code_snippet": None, "created_at": None}, + ] + mock_store.get_by_pack_id.return_value = [] + mem = _make_memory(store=mock_store) + mem.load_experiences_from_db("p1") + ctx = mem.get_relevant_context("trainer.py", "training loop") + assert "Prior Experience" in ctx or "success_pattern" in ctx + + def test_clear_resets_prior_experiences(self): + mock_store = MagicMock() + mock_store.get_by_paper_id.return_value = [ + {"id": 1, "pattern_type": "success_pattern", "content": "x", + "pack_id": None, "paper_id": "p1", "code_snippet": None, "created_at": None}, + ] + mock_store.get_by_pack_id.return_value = [] + mem = _make_memory(store=mock_store) + mem.load_experiences_from_db("p1") + assert len(mem._prior_experiences) == 1 + mem.clear() + assert mem._prior_experiences == []