diff --git a/.coderabbit.yaml b/.coderabbit.yaml new file mode 100644 index 00000000..5efbc785 --- /dev/null +++ b/.coderabbit.yaml @@ -0,0 +1,4 @@ +reviews: + auto_review: + enabled: true + drafts: false diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..607888f9 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,7 @@ +# Default owners for automatic review requests. +* @jerry609 + +# Area owners +src/ @jerry609 +web/ @jerry609 +cli/ @jerry609 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000..9a07d8b7 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,40 @@ +name: codeql + +on: + push: + branches: + - master + pull_request: + branches: + - master + schedule: + - cron: "0 3 * * 1" + +permissions: + actions: read + contents: read + security-events: write + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + language: + - python + - javascript-typescript + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + diff --git a/.github/workflows/gemini-review.yml b/.github/workflows/gemini-review.yml new file mode 100644 index 00000000..a3990645 --- /dev/null +++ b/.github/workflows/gemini-review.yml @@ -0,0 +1,88 @@ +name: gemini-review + +on: + pull_request: + types: [opened, reopened, synchronize, ready_for_review] + +permissions: + contents: read + pull-requests: write + issues: write + id-token: write + +concurrency: + group: gemini-review-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + review: + if: ${{ github.event.pull_request.draft == false }} + runs-on: ubuntu-latest + timeout-minutes: 12 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Check GEMINI_API_KEY availability + id: key_check + env: + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + run: | + if [ -z "${GEMINI_API_KEY}" ]; then + echo "enabled=false" >> "$GITHUB_OUTPUT" + echo "::notice::GEMINI_API_KEY is not configured, skipping Gemini auto review." + else + echo "enabled=true" >> "$GITHUB_OUTPUT" + fi + + - name: Run Gemini auto review + if: ${{ steps.key_check.outputs.enabled == 'true' }} + uses: google-github-actions/run-gemini-cli@v0 + env: + GITHUB_TOKEN: ${{ github.token }} + PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + REPOSITORY: ${{ github.repository }} + ADDITIONAL_CONTEXT: "Focus on correctness, regressions, and missing tests." + with: + gemini_api_key: ${{ secrets.GEMINI_API_KEY }} + gemini_cli_version: ${{ vars.GEMINI_CLI_VERSION || 'latest' }} + gemini_model: ${{ vars.GEMINI_MODEL }} + gemini_debug: ${{ fromJSON(vars.GEMINI_DEBUG || 'false') }} + workflow_name: gemini-review + settings: |- + { + "tools": { + "core": [ + "run_shell_command(gh)", + "run_shell_command(cat)", + "run_shell_command(head)", + "run_shell_command(tail)", + "run_shell_command(grep)", + "run_shell_command(sed)", + "run_shell_command(jq)", + "run_shell_command(echo)" + ] + } + } + prompt: | + You are a pull request review assistant. + + Repository: ${REPOSITORY} + Pull request number: ${PULL_REQUEST_NUMBER} + Additional context: ${ADDITIONAL_CONTEXT} + + Do this sequence: + 1) Use gh to fetch PR metadata and diff: + - gh pr view "${PULL_REQUEST_NUMBER}" --repo "${REPOSITORY}" --json title,body,files,additions,deletions,changedFiles + - gh pr diff "${PULL_REQUEST_NUMBER}" --repo "${REPOSITORY}" > /tmp/pr.diff + 2) Review the changes with focus on correctness, behavior regressions, security, and missing tests. + 3) Write a concise markdown review to /tmp/review.md with sections: + - Critical findings + - Medium findings + - Low findings + - Suggested tests + If no issues exist, clearly state: "No blocking findings." and mention residual risks. + 4) Post exactly one PR review comment: + gh pr review "${PULL_REQUEST_NUMBER}" --repo "${REPOSITORY}" --comment --body-file /tmp/review.md + + Keep feedback concrete, actionable, and tied to changed code. diff --git a/AI4S b/AI4S deleted file mode 160000 index 79e9fc7c..00000000 --- a/AI4S +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 79e9fc7c06bed49d95262c16619bba8950c3ab81 diff --git a/CLAUDE.md b/CLAUDE.md index 367887a1..ea087b2e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -7,9 +7,10 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ### Python Backend ```bash -# Install dependencies +# Install dependencies (use dev extras for testing tools) python -m venv .venv && source .venv/bin/activate -pip install -r requirements.txt +pip install -e ".[dev]" # includes pytest-asyncio, respx, aioresponses, etc. +pip install -r requirements.txt # alternative: full deps without dev extras # Run database migrations (required before first run) export PAPERBOT_DB_URL="sqlite:///data/paperbot.db" @@ -18,38 +19,59 @@ alembic upgrade head # Start API server (dev mode with auto-reload) python -m uvicorn src.paperbot.api.main:app --reload --port 8000 -# Code formatting +# Start ARQ worker (Redis-backed task queue for DailyPaper cron, etc.) +arq paperbot.infrastructure.queue.arq_worker.WorkerSettings + +# Code formatting & type checking python -m black . python -m isort . - -# Type checking pyright src/ ``` ### Testing +**Important**: `asyncio_mode = "strict"` in pyproject.toml — every async test must be decorated with `@pytest.mark.asyncio` explicitly. + ```bash # Run all tests pytest -q -# Run specific test suites -pytest tests/unit/ -pytest tests/integration/ -pytest tests/e2e/ +# Run a single test file or test +pytest tests/unit/test_di_container.py +pytest tests/unit/test_di_container.py::test_singleton -v # Run CI offline gates (same tests as GitHub Actions) +# CI uses requirements-ci.txt (lighter, no weasyprint/heavy deps) PYTHONPATH=src pytest -q \ tests/unit/test_scholar_from_config.py \ tests/unit/test_source_registry_modes.py \ + tests/unit/test_arq_worker_settings.py \ + tests/unit/test_jobs_routes_import.py \ + tests/unit/test_dailypaper.py \ + tests/unit/test_paper_judge.py \ + tests/unit/test_memory_module.py \ + tests/unit/test_memory_metric_collector.py \ + tests/unit/test_llm_service.py \ + tests/unit/test_di_container.py \ + tests/unit/test_pipeline.py \ tests/integration/test_eventlog_sqlalchemy.py \ - tests/integration/test_crawler_contract_parsers.py + tests/integration/test_crawler_contract_parsers.py \ + tests/integration/test_arxiv_connector_fixture.py \ + tests/integration/test_reddit_connector_fixture.py \ + tests/integration/test_x_importer_fixture.py \ + tests/e2e/test_api_track_fullstack_offline.py -# Run evaluation smoke tests +# Eval smoke tests (also in CI) python evals/runners/run_scholar_pipeline_smoke.py python evals/runners/run_track_pipeline_smoke.py python evals/runners/run_eventlog_replay_smoke.py + +# Memory evals (also in CI) +PYTHONPATH=src pytest -q evals/memory/test_deletion_compliance.py evals/memory/test_retrieval_hit_rate.py ``` +**Test patterns**: Tests use stub/fake classes (e.g., `_FakeLLMService`) rather than `unittest.mock`. HTTP mocking via `respx` and `aioresponses`. Use `tmp_path` for temp SQLite DBs. Reset singletons in `setup_method` (e.g., `Container._instance = None`). + ### Web Dashboard (Next.js) ```bash @@ -57,16 +79,24 @@ cd web npm install npm run dev # Dev server at http://localhost:3000 npm run build # Production build +npm run start # Production serve npm run lint # ESLint ``` +Stack: Next.js 16 + React 19 + Tailwind CSS **v4** + Zustand (state) + Vercel AI SDK v5. Monaco editor for code, XTerm for terminal, @xyflow/react for DAG visualization. + ### Terminal CLI (Ink/React) +Requires Node.js >= 18. + ```bash cd cli npm install -npm run build -npm start +npm run dev # Hot-reload dev mode (tsx watch) +npm run build # Production build +npm run start # Run CLI +npm run typecheck # tsc --noEmit +npm run lint # ESLint ``` ## Architecture Overview @@ -74,13 +104,21 @@ npm start PaperBot is a multi-agent research workflow framework with three main components: 1. **Python Backend** (`src/paperbot/`) - FastAPI server with SSE streaming, multi-agent orchestration -2. **Web Dashboard** (`web/`) - Next.js 16 + React 19 + Tailwind CSS -3. **Terminal UI** (`cli/`) - Ink/React CLI +2. **Web Dashboard** (`web/`) - Next.js App Router, pages: dashboard, papers, research, scholars, settings, studio, wiki, workflows +3. **Terminal UI** (`cli/`) - Ink/React CLI (installable as `paperbot` global command) -### Core Python Structure +### Core Python Layers ``` src/paperbot/ +├── domain/ # Domain models: Paper, Scholar, Track, Feedback, Enrichment, Harvest +├── application/ # Use cases and orchestration +│ ├── ports/ # Interface definitions (enrichment, event_log, paper_registry, etc.) +│ ├── services/ # Application services (anchor, paper_search, llm_service, etc.) +│ ├── workflows/ # Pipeline orchestrators (dailypaper, scholar_pipeline, harvest_pipeline) +│ │ └── analysis/ # Paper analysis: judge, summarizer, trend_analyzer, relevance_assessor +│ ├── collaboration/ # Cross-agent message schemas +│ └── registries/ # Source/provider registries ├── agents/ # Multi-agent implementations (BaseAgent in base.py) │ ├── research/ # Paper analysis agents │ ├── scholar_tracking/# Scholar monitoring agents @@ -88,34 +126,42 @@ src/paperbot/ │ ├── verification/ # Claim verification │ └── prompts/ # Agent prompt templates ├── api/ # FastAPI server -│ ├── main.py # App setup, CORS, routers +│ ├── main.py # App setup, CORS, router registration │ ├── streaming.py # SSE utilities -│ └── routes/ # Endpoint handlers (track, analyze, gen_code, review, etc.) +│ └── routes/ # Endpoint handlers ├── core/ # Core abstractions -│ ├── collaboration/ # AgentCoordinator, ScoreShareBus, FailFast -│ ├── di/ # Dependency injection container +│ ├── collaboration/ # AgentCoordinator, ScoreShareBus, FailFastEvaluator +│ ├── di/ # Dependency injection container (Container.instance() singleton) │ ├── pipeline/ # Task pipeline framework │ └── report_engine/ # Report generation with Jinja2 templates -├── repro/ # Paper2Code pipeline (ReproAgent) +├── repro/ # Paper2Code / DeepCode Studio pipeline │ ├── orchestrator.py # Multi-stage execution -│ ├── agents/ # Planning/Coding/Debugging agents -│ ├── memory/ # CodeMemory - cross-file context with AST indexing -│ └── rag/ # CodeRAG - pattern retrieval -├── context_engine/ # Research context & track routing +│ ├── agents/ # Planning/Coding/Debugging/Verification agents +│ ├── nodes/ # Pipeline nodes: planning, blueprint, analysis, environment, generation, verification +│ ├── memory/ # CodeMemory (cross-file context) + SymbolIndex (AST indexing) +│ ├── rag/ # CodeRAG - pattern retrieval from knowledge_base +│ └── *_executor.py # Execution backends: docker, e2b ├── infrastructure/ # External services and persistence -│ ├── llm/ # LLM client (OpenAI/Anthropic) +│ ├── llm/providers/ # OpenAI, Anthropic, Ollama providers │ ├── stores/ # SQLAlchemy repositories -│ ├── event_log/ # Event persistence -│ └── api_clients/ # Semantic Scholar, GitHub clients +│ ├── connectors/ # Data sources: arxiv, openalex, reddit, zotero, hf_daily, paperscool, x +│ ├── harvesters/ # Bulk paper harvesters (arxiv, openalex, semantic_scholar) +│ ├── adapters/ # Search adapters (arxiv, s2, openalex, hf, paperscool) +│ ├── crawling/ # HTTP downloader, request layer, parsers +│ ├── event_log/ # Event persistence (SQLAlchemy, memory, composite, logging) +│ ├── api_clients/ # Semantic Scholar, GitHub clients +│ └── queue/ # ARQ worker (Redis-backed async job queue) +├── context_engine/ # Research context & track routing └── workflows/ # Workflow orchestration, scheduler ``` ### Key Architectural Patterns +- **Domain-Driven Design**: `domain/` models → `application/ports/` interfaces → `infrastructure/` implementations - **Multi-Agent Orchestration**: `AgentCoordinator` registers agents, broadcasts tasks, collects results. `ScoreShareBus` enables cross-stage evaluation sharing. `FailFastEvaluator` stops low-quality work early. -- **Paper2Code Pipeline**: Multi-stage (Planning → Coding → Verification → Debugging) with `CodeMemory` for stateful cross-file context and `CodeRAG` for pattern retrieval. -- **Repository Pattern**: Interface definitions in `application/ports/`, implementations in `infrastructure/stores/`. -- **DI Container**: Single `Container.instance()` holds all services (`core/di/`). +- **Paper2Code Pipeline**: Multi-stage (Planning → Blueprint → Environment → Generation → Verification) with `CodeMemory` for stateful cross-file context and `CodeRAG` for pattern retrieval. Executors run code in Docker or E2B sandboxes. +- **DI Container**: Single `Container.instance()` holds all services. Reset with `Container._instance = None` in tests. +- **ARQ Task Queue**: Redis-backed async jobs for DailyPaper cron and background tasks. ### API Endpoints @@ -125,7 +171,8 @@ Key SSE-streaming endpoints: - `POST /api/gen-code` - Paper2Code generation - `POST /api/review` - Deep review simulation - `POST /api/research/*` - Personalized research context & tracks -- `GET/POST /api/runbook/*`, `/api/sandbox/*` - DeepCode Studio workspace & execution +- `GET/POST /api/runbook/*` - DeepCode Studio file management (list/read/write/snapshot/diff/revert) +- `GET/POST /api/sandbox/*` - Studio execution: queue, run logs, resource metrics ## Configuration @@ -142,19 +189,40 @@ GITHUB_TOKEN=... # Runtime PAPERBOT_DB_URL=sqlite:///data/paperbot.db +PAPERBOT_OFFLINE=true # Disable all external network calls +PAPERBOT_MODE=academic # or "production" + +# Redis (for ARQ worker) +PAPERBOT_REDIS_HOST=127.0.0.1 +PAPERBOT_REDIS_PORT=6379 + +# Studio file access (comma-separated allowed dir prefixes beyond /tmp and cwd) +PAPERBOT_RUNBOOK_ALLOW_DIR_PREFIXES=/path/to/projects + +# Execution backend +PAPERBOT_EXECUTOR=docker # or "e2b" or "auto" ``` Configuration files: - `config/config.yaml` - Main app config (models, venues, thresholds) -- `config/settings.py` - Pydantic settings +- `config/settings.py` - Dataclass settings with env var overrides +- `config/validated_settings.py` - Pydantic-validated settings wrapper +- `config/models.py` - Pydantic models: AppConfig, LLMConfig, ReproConfig, PipelineConfig - `config/scholar_subscriptions.yaml` - Tracked scholars ## Code Style -- **Python**: Black (line-length 100), isort (Black profile), pyright for type checking -- **TypeScript**: Follow existing patterns in `web/src/` +- **Python**: Black (line-length 100, target py310), isort (Black profile), pyright (basic mode) +- **TypeScript**: Follow existing patterns in `web/src/` and `cli/src/` - **Commits**: Conventional Commits format (`feat:`, `fix:`, `refactor:`, `docs:`) +## CI + +GitHub Actions (`.github/workflows/ci.yml`) runs on push/PR to `master`: +- Matrix: Python 3.10, 3.11, 3.12 +- Uses `requirements-ci.txt` (not `requirements.txt`) for lighter installs +- Runs: offline UT/IT/E2E gates → eval smoke tests → memory evals + ## Adding New Components ### New Agent diff --git a/alembic/versions/b94c1a2be26e_add_p2c_repro_context_pack_tables.py b/alembic/versions/b94c1a2be26e_add_p2c_repro_context_pack_tables.py new file mode 100644 index 00000000..d94f3a0b --- /dev/null +++ b/alembic/versions/b94c1a2be26e_add_p2c_repro_context_pack_tables.py @@ -0,0 +1,98 @@ +"""add_p2c_repro_context_pack_tables + +Revision ID: b94c1a2be26e +Revises: 4c71b28a2f67 +Create Date: 2026-02-26 16:34:35.636031 + +""" +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + + +revision = 'b94c1a2be26e' +down_revision = '4c71b28a2f67' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + 'repro_context_pack', + sa.Column('id', sa.String(length=64), nullable=False), + sa.Column('user_id', sa.String(length=64), nullable=False, server_default='default'), + sa.Column('project_id', sa.String(length=64), nullable=True), + sa.Column('paper_id', sa.String(length=256), nullable=False), + sa.Column('paper_title', sa.Text(), nullable=True), + sa.Column('version', sa.String(length=16), nullable=False, server_default='v1'), + sa.Column('depth', sa.String(length=16), nullable=False, server_default='standard'), + sa.Column('status', sa.String(length=32), nullable=False, server_default='pending'), + sa.Column('objective', sa.Text(), nullable=True), + sa.Column('pack_json', sa.Text(), nullable=False, server_default='{}'), + sa.Column('confidence_overall', sa.Float(), nullable=False, server_default='0.0'), + sa.Column('warning_count', sa.Integer(), nullable=False, server_default='0'), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('deleted_at', sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint('id'), + ) + op.create_index('ix_repro_context_pack_user_id', 'repro_context_pack', ['user_id']) + op.create_index('ix_repro_context_pack_paper_id', 'repro_context_pack', ['paper_id']) + op.create_index('ix_repro_context_pack_project_id', 'repro_context_pack', ['project_id']) + op.create_index('ix_repro_context_pack_status', 'repro_context_pack', ['status']) + op.create_index('ix_repro_context_pack_confidence_overall', 'repro_context_pack', ['confidence_overall']) + op.create_index('ix_repro_context_pack_created_at', 'repro_context_pack', ['created_at']) + + op.create_table( + 'repro_context_stage_result', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('context_pack_id', sa.String(length=64), nullable=False), + sa.Column('stage_name', sa.String(length=64), nullable=False), + sa.Column('status', sa.String(length=16), nullable=False, server_default='completed'), + sa.Column('result_json', sa.Text(), nullable=True), + sa.Column('confidence', sa.Float(), nullable=False, server_default='0.0'), + sa.Column('duration_ms', sa.Integer(), nullable=False, server_default='0'), + sa.Column('error_message', sa.Text(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['context_pack_id'], ['repro_context_pack.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + ) + op.create_index('ix_repro_context_stage_result_context_pack_id', 'repro_context_stage_result', ['context_pack_id']) + op.create_index('ix_repro_context_stage_result_stage_name', 'repro_context_stage_result', ['stage_name']) + op.create_index('ix_repro_context_stage_result_status', 'repro_context_stage_result', ['status']) + + op.create_table( + 'repro_context_evidence', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('context_pack_id', sa.String(length=64), nullable=False), + sa.Column('evidence_type', sa.String(length=32), nullable=False), + sa.Column('ref', sa.Text(), nullable=False, server_default=''), + sa.Column('supports_json', sa.Text(), nullable=False, server_default='[]'), + sa.Column('confidence', sa.Float(), nullable=False, server_default='0.0'), + sa.ForeignKeyConstraint(['context_pack_id'], ['repro_context_pack.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + ) + op.create_index('ix_repro_context_evidence_context_pack_id', 'repro_context_evidence', ['context_pack_id']) + op.create_index('ix_repro_context_evidence_evidence_type', 'repro_context_evidence', ['evidence_type']) + + op.create_table( + 'repro_context_feedback', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('context_pack_id', sa.String(length=64), nullable=False), + sa.Column('user_id', sa.String(length=64), nullable=False), + sa.Column('rating', sa.Integer(), nullable=True), + sa.Column('comment', sa.Text(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['context_pack_id'], ['repro_context_pack.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + ) + op.create_index('ix_repro_context_feedback_context_pack_id', 'repro_context_feedback', ['context_pack_id']) + op.create_index('ix_repro_context_feedback_user_id', 'repro_context_feedback', ['user_id']) + + +def downgrade() -> None: + op.drop_table('repro_context_feedback') + op.drop_table('repro_context_evidence') + op.drop_table('repro_context_stage_result') + op.drop_table('repro_context_pack') diff --git a/docs/AUTO_REVIEW_SETUP.md b/docs/AUTO_REVIEW_SETUP.md new file mode 100644 index 00000000..486f00d6 --- /dev/null +++ b/docs/AUTO_REVIEW_SETUP.md @@ -0,0 +1,29 @@ +# Auto Review Setup (CodeRabbit + Copilot + Gemini) + +This repository now includes: + +- `.coderabbit.yaml` to enable CodeRabbit auto review. +- `.github/CODEOWNERS` to auto-request code owners on PRs. +- `.github/workflows/gemini-review.yml` to run Gemini review on new/updated PRs. + +## One-time GitHub settings + +1. Install/enable GitHub Apps for this repo: + - CodeRabbit + - Gemini Assistant (or Gemini CLI workflow with `GEMINI_API_KEY`) + - GitHub Copilot code review + +2. Add repository secret: + - `GEMINI_API_KEY` + +3. (Recommended) Enable branch ruleset for PR quality gates: + - Require pull request before merging + - Require at least 1 approval + - Require conversation resolution + - Enable Copilot code review rule + - Enable Code scanning / CI checks as needed + +## Notes + +- Gemini workflow skips automatically when `GEMINI_API_KEY` is missing. +- For fork PRs, secret access is restricted by GitHub; Gemini review may be skipped. diff --git a/docs/P2C_MODULE1_BENCHMARK.md b/docs/P2C_MODULE1_BENCHMARK.md new file mode 100644 index 00000000..66a4bcb7 --- /dev/null +++ b/docs/P2C_MODULE1_BENCHMARK.md @@ -0,0 +1,47 @@ +# P2C Module 1 Benchmark + +## Purpose + +This benchmark tracks extraction quality changes for P2C Module 1 so refactors can be compared with reproducible numbers. + +## Fixture + +- Source: `evals/fixtures/p2c/module1_gold.json` +- Scope: + - architecture hit + - metrics extraction F1 + - hyperparameter extraction F1 + - evidence hit rate for required observation types + - warnings count + +## Run + +```bash +PYTHONPATH=src python scripts/p2c_module1_benchmark.py \ + --fixtures evals/fixtures/p2c/module1_gold.json \ + --output evals/reports/p2c_module1_baseline.json +``` + +Optional CI gate: + +```bash +PYTHONPATH=src python scripts/p2c_module1_benchmark.py \ + --fail-under-metric-f1 0.90 +``` + +## Current Baseline (2026-02-26) + +- metric_f1: `1.0000` +- hyperparam_f1: `1.0000` +- architecture_hit_rate: `1.0000` +- evidence_hit_rate: `0.8889` +- avg_warnings: `1.3333` + +Raw report: +- `evals/reports/p2c_module1_baseline.json` + +## Known Limits + +- Dataset is intentionally small and synthetic; it is best for regression checks, not real-world coverage. +- Scores can be inflated by keyword-heavy fixtures and should be paired with manual spot checks. +- Evidence hit rate is strict and can remain below 1.0 when environment evidence is missing. diff --git a/docs/P2C_MODULE_1_CORE_ENGINE.md b/docs/P2C_MODULE_1_CORE_ENGINE.md new file mode 100644 index 00000000..0eb402be --- /dev/null +++ b/docs/P2C_MODULE_1_CORE_ENGINE.md @@ -0,0 +1,508 @@ +# P2C Module 1: Core Engine — 提取管线与数据模型 + +- 日期：2026-02-22 +- 状态：Draft +- 负责范围：提取编排、数据模型定义、质量门禁 +- 上游依赖：论文源数据（paper metadata + full text） +- 下游消费方：Module 2（API & Storage）、Module 3（Frontend） + +--- + +## 1. 模块职责 + +Core Engine 是 P2C 的计算核心，职责是： + +1. 接收论文原始数据 + 用户/项目上下文，执行多阶段信息提取； +2. 产出结构化的 `ReproContextPack`（唯一输出契约）； +3. 对每个阶段输出执行质量校验，低质量时降级或标记人工确认； +4. 不负责持久化、不负责 HTTP 路由、不负责 UI 渲染。 + +**一句话定位：论文理解与执行计划的编译器。** + +--- + +## 2. 与现有代码的关系 + +本模块 **复用** 以下现有能力，不重复实现： + +| 现有组件 | 路径 | P2C 中的角色 | +|---|---|---| +| `Blueprint` dataclass | `src/paperbot/repro/models.py` | Stage B 输出类型 | +| `EnvironmentSpec` dataclass | `src/paperbot/repro/models.py` | Stage C 输出类型 | +| `ImplementationSpec` dataclass | `src/paperbot/repro/models.py` | Stage D 输出类型 | +| `BlueprintDistillationNode` | `src/paperbot/repro/nodes/blueprint_node.py` | Stage B 执行节点 | +| `EnvironmentInferenceNode` | `src/paperbot/repro/nodes/environment_node.py` | Stage C 执行节点 | +| `AnalysisNode` | `src/paperbot/repro/nodes/analysis_node.py` | Stage D 执行节点 | +| `PaperContext` | `src/paperbot/repro/models.py` | 输入归一化参考 | +| `ContextEngine` | `src/paperbot/context_engine/engine.py` | 用户记忆注入 | + +**新增** 的部分：Stage A（文献蒸馏）、Stage E（任务拆解）、Stage F（成功标准）、编排器、证据链接器、组装器。 + +--- + +## 3. 核心数据模型 + +### 3.1 输入：`GenerateContextRequest` + +```python +@dataclass +class GenerateContextRequest: + """P2C 管线的唯一入口参数。""" + paper_id: str # Semantic Scholar ID / arXiv ID / DOI + user_id: str = "default" + project_id: Optional[str] = None # 关联项目（用于注入项目上下文） + track_id: Optional[int] = None # 关联 track（用于注入个性化记忆） + depth: Literal["fast", "standard", "deep"] = "standard" + # fast: 仅 Stage B+C，跳过 A/E/F + # standard: 全部 Stage A-F + # deep: 全部 Stage + 交叉验证 + 多模型投票 +``` + +### 3.2 输出：`ReproContextPack` + +这是 P2C 的 **唯一输出契约**，Module 2/3 均通过此结构消费数据。 + +```python +@dataclass +class ReproContextPack: + context_pack_id: str # "ctxp_{uuid}" + version: str = "v1" + created_at: str = "" # ISO 8601 + + # ── 论文身份 ── + paper: PaperIdentity = field(default_factory=PaperIdentity) + + # ── 复现目标 ── + objective: str = "" # 一句话复现目标 + + # ── Stage 产出（每个阶段一个字段） ── + literature_digest: Optional[LiteratureDigest] = None # Stage A + blueprint: Optional[Blueprint] = None # Stage B（复用现有） + environment: Optional[EnvironmentSpec] = None # Stage C（复用现有） + implementation_spec: Optional[ImplementationSpec] = None # Stage D（复用现有） + task_roadmap: List[TaskCheckpoint] = field(default_factory=list) # Stage E + success_criteria: List[SuccessCriterion] = field(default_factory=list) # Stage F + + # ── 质量元数据 ── + evidence_links: List[EvidenceLink] = field(default_factory=list) + confidence: ConfidenceScores = field(default_factory=ConfidenceScores) + warnings: List[str] = field(default_factory=list) # 人工确认提示 +``` + +### 3.3 子类型定义 + +```python +@dataclass +class PaperIdentity: + paper_id: str = "" + title: str = "" + year: int = 0 + authors: List[str] = field(default_factory=list) + identifiers: Dict[str, str] = field(default_factory=dict) # doi/arxiv/s2 + +@dataclass +class LiteratureDigest: + """Stage A 输出：文献机制蒸馏。""" + problem_definition: str = "" # 论文解决什么问题 + core_innovation: str = "" # 方法核心创新点 + relation_to_user: str = "" # 与用户项目/兴趣的关联 + key_references: List[str] = field(default_factory=list) # 关键引用论文 ID + +@dataclass +class TaskCheckpoint: + """Stage E 输出：开发路线图单步。""" + id: str # "T1", "T2", ... + title: str + description: str = "" + acceptance_criteria: List[str] = field(default_factory=list) + depends_on: List[str] = field(default_factory=list) # 依赖的 checkpoint id + estimated_difficulty: Literal["low", "medium", "high"] = "medium" + +@dataclass +class SuccessCriterion: + """Stage F 输出：复现成功标准。""" + metric_name: str # e.g. "Top-1 Accuracy" + target_value: str # e.g. ">= 93.0" + dataset_split: str = "test" + aggregation: str = "mean" + source: str = "" # 论文中的出处 (e.g. "Table 2") + tolerance: Optional[str] = None # 允许偏差 (e.g. "±1.0") + +@dataclass +class EvidenceLink: + """证据溯源记录。""" + type: Literal["paper_span", "table", "figure", "code_snippet", "metadata"] + ref: str # e.g. "method_section#L120-L140" + supports: List[str] # 支持的字段名列表 + confidence: float = 0.0 + +@dataclass +class ConfidenceScores: + overall: float = 0.0 + literature: float = 0.0 + blueprint: float = 0.0 + environment: float = 0.0 + spec: float = 0.0 + roadmap: float = 0.0 + metrics: float = 0.0 +``` + +--- + +## 4. 提取管线设计（6 阶段） + +### 总体流程 + +``` +GenerateContextRequest + │ + ▼ +┌─────────────────────────────────────┐ +│ InputNormalizer │ +│ paper metadata + full text + memory│ +└──────────────┬──────────────────────┘ + │ + ┌──────────▼──────────┐ + │ Stage A: Literature │ ← 新增 + │ Distill │ + └──────────┬──────────┘ + │ + ┌──────────▼──────────┐ + │ Stage B: Blueprint │ ← 复用 BlueprintDistillationNode + │ Extract │ + └──────────┬──────────┘ + │ + ┌──────────▼──────────┐ + │ Stage C: Environment│ ← 复用 EnvironmentInferenceNode + │ Infer │ + └──────────┬──────────┘ + │ + ┌──────────▼──────────┐ + │ Stage D: Spec & │ ← 复用 AnalysisNode + │ Hyperparams │ + └──────────┬──────────┘ + │ + ┌──────────▼──────────┐ + │ Stage E: Task │ ← 新增 + │ Roadmap │ + └──────────┬──────────┘ + │ + ┌──────────▼──────────┐ + │ Stage F: Success │ ← 新增 + │ Criteria │ + └──────────┬──────────┘ + │ + ┌──────────▼──────────┐ + │ EvidenceLinker │ + │ + ContextAssembler │ + └──────────┬──────────┘ + │ + ▼ + ReproContextPack +``` + +### 4.1 InputNormalizer + +**职责**：将异构论文来源统一为管线可消费的内部格式。 + +```python +class InputNormalizer: + async def normalize(self, request: GenerateContextRequest) -> NormalizedInput: + """ + 1. 通过 paper_id 从 ResearchStore 或 SemanticScholar 获取 metadata + 2. 获取论文全文（PDF parse / preprint text） + 3. 从 ContextEngine 注入用户/项目记忆 + 4. 输出统一的 NormalizedInput + """ + ... + +@dataclass +class NormalizedInput: + paper: PaperIdentity + abstract: str + method_text: str # 方法章节文本 + full_text: Optional[str] = None # 全文（deep 模式用） + algorithm_blocks: List[str] = field(default_factory=list) + tables: List[Dict[str, Any]] = field(default_factory=list) + user_memory: Optional[Dict] = None # 用户个性化上下文 + project_context: Optional[Dict] = None # 项目上下文 +``` + +### 4.2 Stage A: 文献机制蒸馏 + +**新增实现。** 参考 claude-scholar `literature-reviewer.md` 的 prompt 结构。 + +- 输入：`NormalizedInput.abstract` + `method_text` + `user_memory` +- 输出：`LiteratureDigest` +- 失败降级：跳过，`confidence.literature = 0`，`warnings` 追加提示 + +### 4.3 Stage B: Blueprint 抽取 + +**复用** `BlueprintDistillationNode`，增加包装层对接 P2C 接口。 + +- 输入：`NormalizedInput` → 构建 `PaperContext` +- 输出：`Blueprint`（现有类型） +- 降级：heuristic fallback（已内置于现有节点） + +### 4.4 Stage C: 环境推断 + +**复用** `EnvironmentInferenceNode`。 + +- 输入：`Blueprint.paper_year` + `Blueprint.framework_hints` + 论文代码片段 +- 输出：`EnvironmentSpec`（现有类型） +- 三路推断策略已内置：year mapping → code pattern → LLM inference + +### 4.5 Stage D: 实现规格提取 + +**复用** `AnalysisNode`。 + +- 输入：`NormalizedInput.method_text` + `Blueprint` +- 输出：`ImplementationSpec`（现有类型） +- regex + LLM merge 策略已内置 + +### 4.6 Stage E: 任务拆解（Roadmap） + +**新增实现。** 参考 claude-scholar `dev-planner.md` + `planning-with-files` skill。 + +- 输入：`Blueprint` + `ImplementationSpec` + `LiteratureDigest` +- 输出：`List[TaskCheckpoint]`，带依赖 DAG +- 约束： + - 每个 checkpoint 必须有 `acceptance_criteria` + - 总步数控制在 3-12 步 + - 第一步必须是数据/环境准备 + +### 4.7 Stage F: 成功标准抽取 + +**新增实现。** 参考 claude-scholar `results-analysis` skill。 + +- 输入：`NormalizedInput.tables` + `method_text` +- 输出：`List[SuccessCriterion]` +- 约束：至少产出 1 项可量化指标；无指标时 `confidence.metrics = 0` + +--- + +## 5. 核心组件接口 + +### 5.1 ExtractionOrchestrator（编排器） + +```python +class ExtractionOrchestrator: + """串联 Stage A-F，管理阶段依赖与降级。""" + + def __init__( + self, + skill_loader: SkillLoader, + llm_client: LLMClient, + blueprint_node: BlueprintDistillationNode, + environment_node: EnvironmentInferenceNode, + analysis_node: AnalysisNode, + ): + ... + + async def run( + self, + normalized_input: NormalizedInput, + depth: Literal["fast", "standard", "deep"] = "standard", + on_stage_complete: Optional[Callable[[str, float], None]] = None, + ) -> ReproContextPack: + """ + 执行提取管线。 + + 参数: + normalized_input: 归一化后的论文输入 + depth: 提取深度 + on_stage_complete: 阶段完成回调 (stage_name, progress_pct) + + 返回: + 完整的 ReproContextPack + + depth 行为: + fast → Stage B + C only + standard → Stage A-F + deep → Stage A-F + cross-validation + multi-model voting + """ + ... +``` + +**阶段依赖关系：** + +``` +A (独立) ──┐ +B (独立) ──┼──→ E (依赖 A, B, D) +C (依赖 B) │ +D (依赖 B) ┘ +F (独立) ──────→ ContextAssembler (依赖全部) +``` + +注意：A、B、F 互相独立，可并行执行。 + +### 5.2 SkillLoader（技能模板加载器） + +```python +class SkillLoader: + """从本地 skill/agent markdown 加载 prompt 模板。""" + + def __init__(self, skills_dir: Path): + ... + + def load_skill(self, key: str) -> SkillTemplate: + """加载技能模板。key 示例: 'literature_distill', 'dev_planner'""" + ... + + def render(self, key: str, variables: Dict[str, Any]) -> str: + """渲染带变量的 prompt 模板。""" + ... + +@dataclass +class SkillTemplate: + key: str + instruction: str # prompt body + output_schema: Optional[Dict] = None # 期望的 JSON schema + metadata: Dict[str, str] = field(default_factory=dict) # frontmatter +``` + +### 5.3 EvidenceLinker（证据链接器） + +```python +class EvidenceLinker: + """为 context pack 中的关键字段绑定论文证据。""" + + async def link( + self, + pack: ReproContextPack, + normalized_input: NormalizedInput, + ) -> Tuple[List[EvidenceLink], ConfidenceScores]: + """ + 遍历 pack 中的关键字段，在原文中查找支撑证据。 + 无证据的高风险字段会降低 confidence 并追加 warning。 + """ + ... +``` + +### 5.4 ContextAssembler（组装器） + +```python +class ContextAssembler: + """将各阶段产出组装为 ReproContextPack。""" + + def assemble( + self, + request: GenerateContextRequest, + normalized_input: NormalizedInput, + literature: Optional[LiteratureDigest], + blueprint: Optional[Blueprint], + environment: Optional[EnvironmentSpec], + spec: Optional[ImplementationSpec], + roadmap: List[TaskCheckpoint], + criteria: List[SuccessCriterion], + evidence: List[EvidenceLink], + confidence: ConfidenceScores, + ) -> ReproContextPack: + ... + + def render_markdown(self, pack: ReproContextPack) -> str: + """渲染为人可读的 REPRODUCTION_PLAN.md。""" + ... +``` + +--- + +## 6. 质量门禁 + +### 6.1 阶段级校验 + +每个 Stage 完成后立即校验： + +| 阶段 | 校验规则 | 失败策略 | +|---|---|---| +| A | `problem_definition` 非空 | 跳过，confidence=0 | +| B | `architecture_type` 非 "unknown"，`module_hierarchy` 至少 1 项 | heuristic fallback | +| C | `python_version` 合法，框架版本可解析 | 使用默认值 | +| D | `optimizer` 非空 | 使用论文常见默认值 | +| E | checkpoint 数量 3-12，DAG 无环 | 重试一次后降级为线性列表 | +| F | 至少 1 项 `SuccessCriterion` | 标记 warning，不阻断 | + +### 6.2 Pack 级校验 + +组装完成后： + +- JSON schema 校验必须通过 +- 必填字段（`paper.title`, `blueprint`, `environment`）不能为 None +- `success_criteria` 至少 1 项（否则 warning） +- `evidence_links` 覆盖关键字段 >= 80%（否则降低 `confidence.overall`） +- `confidence.overall < 0.5` 时自动标记 `warnings: ["建议人工审查后再执行"]` + +### 6.3 离线评测集 + +构建 50 篇带人工标注的论文集，评估维度： + +| 维度 | 指标 | +|---|---| +| 架构识别 | `architecture_type` 准确率 | +| 超参数抽取 | precision / recall | +| 指标目标 | target value 精确匹配率 | +| 路线图可执行性 | 人工 1-5 分评分 | +| 证据覆盖 | evidence link 覆盖率 | + +--- + +## 7. 代码组织 + +``` +src/paperbot/p2c/ + __init__.py + models.py # ReproContextPack 及所有子类型 + input_normalizer.py # InputNormalizer + skill_loader.py # SkillLoader + SkillTemplate + orchestrator.py # ExtractionOrchestrator + evidence_linker.py # EvidenceLinker + assembler.py # ContextAssembler + validators.py # 阶段校验 + Pack 校验 + stages/ + __init__.py + literature_distill.py # Stage A (新增) + blueprint_adapter.py # Stage B (包装现有 BlueprintDistillationNode) + environment_adapter.py # Stage C (包装现有 EnvironmentInferenceNode) + spec_adapter.py # Stage D (包装现有 AnalysisNode) + task_roadmap.py # Stage E (新增) + success_criteria.py # Stage F (新增) + skills/ # prompt 模板资产 + literature_distill.md + task_roadmap.md + success_criteria.md +``` + +--- + +## 8. 与 Module 2 / Module 3 的接口契约 + +### 对 Module 2（API & Storage）的契约 + +Module 2 通过以下方式调用 Core Engine： + +```python +# Module 2 调用 Core Engine 的唯一入口 +orchestrator = ExtractionOrchestrator(...) +pack = await orchestrator.run(normalized_input, depth="standard", on_stage_complete=callback) +# pack: ReproContextPack — 唯一输出 +``` + +- `on_stage_complete` 回调用于 SSE 进度推送 +- Core Engine 不做持久化，由 Module 2 负责存储 `ReproContextPack` + +### 对 Module 3（Frontend）的契约 + +Module 3 不直接调用 Core Engine，通过 Module 2 的 API 间接消费 `ReproContextPack` 的 JSON 序列化。 + +`ReproContextPack` 的 JSON schema 即为前后端契约。 + +--- + +## 9. 风险与缓解 + +| 风险 | 缓解措施 | +|---|---| +| Skill prompt 漂移（claude-scholar 上游更新） | 固化 snapshot 到 `p2c/skills/`，版本号管理 | +| 抽取幻觉导致错误执行 | evidence-link 强制 + 低置信度标记人工确认 | +| 上下文过长导致 LLM 退化 | Blueprint 压缩优先，分层注入（必要字段优先） | +| 阶段级联失败 | 每阶段独立降级，不阻断后续阶段 | diff --git a/docs/P2C_MODULE_2_API_STORAGE.md b/docs/P2C_MODULE_2_API_STORAGE.md new file mode 100644 index 00000000..729b588b --- /dev/null +++ b/docs/P2C_MODULE_2_API_STORAGE.md @@ -0,0 +1,524 @@ +# P2C Module 2: API & Storage — 后端接口与持久化 + +- 日期：2026-02-22 +- 状态：Draft +- 负责范围：HTTP 接口、持久化层、外接 Provider Bridge +- 上游依赖：Module 1（Core Engine）产出的 `ReproContextPack` +- 下游消费方：Module 3（Frontend） + +--- + +## 1. 模块职责 + +1. 提供 HTTP API 供前端调用（生成、查询、创建会话）； +2. 持久化 `ReproContextPack` 及阶段中间结果； +3. 管理 SSE 进度推送（利用现有 `streaming.py` 能力）； +4. 可选双写到外接 context provider（OneContext 等）； +5. 不负责提取逻辑（由 Module 1 Core Engine 执行）。 + +--- + +## 2. 与现有代码的关系 + +| 现有组件 | 路径 | 复用方式 | +|---|---|---| +| `APIRouter` 注册 | `src/paperbot/api/main.py` | 新增 router 注册 | +| SSE 工具 | `src/paperbot/api/streaming.py` | 复用 `sse_response()` | +| `SqlAlchemyResearchStore` | `src/paperbot/infrastructure/stores/research_store.py` | 扩展或新增表 | +| `SqlAlchemyMemoryStore` | `src/paperbot/infrastructure/stores/memory_store.py` | 读取用户记忆 | +| Alembic 迁移 | `alembic/` | 新增迁移脚本 | +| `ContextEngine` | `src/paperbot/context_engine/engine.py` | 注入用户上下文 | + +--- + +## 3. API 端点设计 + +### 3.1 生成上下文包 + +``` +POST /api/research/repro/context/generate +``` + +**请求体：** + +```json +{ + "user_id": "default", + "project_id": "proj_001", + "paper_id": "paper_xxx", + "track_id": 30, + "depth": "standard" +} +``` + +| 字段 | 类型 | 必填 | 说明 | +|---|---|---|---| +| `paper_id` | string | 是 | 论文 ID（S2 / arXiv / DOI） | +| `user_id` | string | 否 | 默认 "default" | +| `project_id` | string | 否 | 关联项目 ID | +| `track_id` | int | 否 | 关联 track | +| `depth` | enum | 否 | `"fast"` / `"standard"` / `"deep"`，默认 `"standard"` | + +**响应体（SSE 流）：** + +SSE 模式下推送阶段进度事件，最终推送完成事件： + +``` +event: stage_progress +data: {"stage": "blueprint_extract", "progress": 0.33, "message": "Extracting blueprint..."} + +event: stage_progress +data: {"stage": "environment_infer", "progress": 0.50, "message": "Inferring environment..."} + +... + +event: completed +data: { + "context_pack_id": "ctxp_a1b2c3", + "status": "completed", + "summary": "ResNet-50 复现包，含 6 步路线图，置信度 0.81", + "confidence": {"overall": 0.81, "blueprint": 0.84, "environment": 0.78, "metrics": 0.73}, + "warnings": [], + "next_action": "create_repro_session" +} + +event: error (仅在失败时) +data: {"error": "Blueprint extraction failed after retry", "partial_pack_id": "ctxp_a1b2c3"} +``` + +**非 SSE 回退（同步模式）：** + +请求头 `Accept: application/json` 时返回同步 JSON 响应，结构同 `completed` 事件的 data。 + +### 3.2 获取上下文包详情 + +``` +GET /api/research/repro/context/{context_pack_id} +``` + +**响应体：** 完整的 `ReproContextPack` JSON 序列化。 + +```json +{ + "context_pack_id": "ctxp_a1b2c3", + "version": "v1", + "created_at": "2026-02-22T10:00:00Z", + "paper": { + "paper_id": "...", + "title": "...", + "year": 2026, + "authors": ["..."], + "identifiers": {"doi": "...", "arxiv": "...", "s2": "..."} + }, + "objective": "复现论文的核心方法并验证主要指标", + "literature_digest": { + "problem_definition": "...", + "core_innovation": "...", + "relation_to_user": "...", + "key_references": ["..."] + }, + "blueprint": { "architecture_type": "transformer", "module_hierarchy": {}, "..." : "..." }, + "environment": { "python_version": "3.10", "framework": "pytorch", "..." : "..." }, + "implementation_spec": { "optimizer": "adamw", "learning_rate": 1e-4, "..." : "..." }, + "task_roadmap": [ + {"id": "T1", "title": "数据预处理", "acceptance_criteria": ["..."], "depends_on": []} + ], + "success_criteria": [ + {"metric_name": "Top-1", "target_value": ">= 93.0", "source": "Table 2"} + ], + "evidence_links": [ {"type": "paper_span", "ref": "...", "supports": ["..."], "confidence": 0.9} ], + "confidence": {"overall": 0.81, "literature": 0.75, "blueprint": 0.84, "environment": 0.78, "spec": 0.80, "roadmap": 0.82, "metrics": 0.73}, + "warnings": [] +} +``` + +### 3.3 列出上下文包 + +``` +GET /api/research/repro/context?user_id=default&paper_id=xxx&limit=20&offset=0 +``` + +**查询参数：** + +| 参数 | 类型 | 说明 | +|---|---|---| +| `user_id` | string | 按用户过滤 | +| `paper_id` | string | 按论文过滤 | +| `project_id` | string | 按项目过滤 | +| `limit` | int | 分页大小，默认 20 | +| `offset` | int | 分页偏移 | + +**响应体：** + +```json +{ + "items": [ + { + "context_pack_id": "ctxp_a1b2c3", + "paper_title": "...", + "created_at": "...", + "confidence_overall": 0.81, + "status": "completed", + "warning_count": 0 + } + ], + "total": 5 +} +``` + +### 3.4 由上下文包创建复现会话 + +``` +POST /api/research/repro/context/{context_pack_id}/session +``` + +**请求体：** + +```json +{ + "executor_preference": "auto", + "override_env": null, + "override_roadmap": null +} +``` + +| 字段 | 类型 | 说明 | +|---|---|---| +| `executor_preference` | enum | `"auto"` / `"claude_code"` / `"codex"` / `"local"` | +| `override_env` | object | 可选，覆盖环境配置 | +| `override_roadmap` | array | 可选，用户编辑后的路线图 | + +**响应体：** + +```json +{ + "session_id": "sess_xxx", + "runbook_id": "rb_xxx", + "initial_steps": [ + {"step_id": "S1", "title": "Setup environment", "command": "...", "status": "pending"} + ], + "initial_prompt": "Based on the reproduction context pack, implement..." +} +``` + +> 此接口将 `ReproContextPack` 转换为现有 Studio runbook 格式，复用 `src/paperbot/api/routes/runbook.py` 的 runbook 创建能力。 + +### 3.5 删除上下文包 + +``` +DELETE /api/research/repro/context/{context_pack_id} +``` + +软删除（标记 `deleted_at`），不物理删除。 + +--- + +## 4. 存储设计 + +### 4.1 新增表 + +```sql +-- 主表：上下文包 +CREATE TABLE repro_context_pack ( + id TEXT PRIMARY KEY, -- "ctxp_{uuid}" + user_id TEXT NOT NULL DEFAULT 'default', + project_id TEXT, + paper_id TEXT NOT NULL, + paper_title TEXT, + version TEXT NOT NULL DEFAULT 'v1', + depth TEXT NOT NULL DEFAULT 'standard', + status TEXT NOT NULL DEFAULT 'pending', -- pending/running/completed/failed + objective TEXT, + pack_json TEXT NOT NULL, -- 完整 ReproContextPack JSON + confidence_overall REAL DEFAULT 0.0, + warning_count INTEGER DEFAULT 0, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + deleted_at TIMESTAMP, + + -- 查询索引 + -- CREATE INDEX ix_rcp_user_paper ON repro_context_pack(user_id, paper_id, created_at DESC); + -- CREATE INDEX ix_rcp_project ON repro_context_pack(project_id, created_at DESC); +); + +-- 阶段中间结果（调试用，可选） +CREATE TABLE repro_context_stage_result ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + context_pack_id TEXT NOT NULL REFERENCES repro_context_pack(id), + stage_name TEXT NOT NULL, -- "literature_distill", "blueprint_extract", ... + status TEXT NOT NULL, -- "completed" / "failed" / "skipped" + result_json TEXT, -- 阶段输出 JSON + confidence REAL DEFAULT 0.0, + duration_ms INTEGER DEFAULT 0, + error_message TEXT, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +-- 证据链接（可选，用于审计） +CREATE TABLE repro_context_evidence ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + context_pack_id TEXT NOT NULL REFERENCES repro_context_pack(id), + evidence_type TEXT NOT NULL, -- "paper_span" / "table" / "figure" / ... + ref TEXT NOT NULL, + supports TEXT NOT NULL, -- JSON array of field names + confidence REAL DEFAULT 0.0 +); + +-- 用户反馈（上线后收集） +CREATE TABLE repro_context_feedback ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + context_pack_id TEXT NOT NULL REFERENCES repro_context_pack(id), + user_id TEXT NOT NULL, + rating INTEGER, -- 1-5 + comment TEXT, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); +``` + +### 4.2 SQLAlchemy Model + +```python +# 放在 src/paperbot/infrastructure/stores/models.py 或独立文件 + +class ReproContextPackModel(Base): + __tablename__ = "repro_context_pack" + + id = Column(String, primary_key=True) + user_id = Column(String, nullable=False, default="default") + project_id = Column(String, nullable=True) + paper_id = Column(String, nullable=False) + paper_title = Column(String, nullable=True) + version = Column(String, nullable=False, default="v1") + depth = Column(String, nullable=False, default="standard") + status = Column(String, nullable=False, default="pending") + objective = Column(Text, nullable=True) + pack_json = Column(Text, nullable=False) + confidence_overall = Column(Float, default=0.0) + warning_count = Column(Integer, default=0) + created_at = Column(DateTime, nullable=False, default=func.now()) + updated_at = Column(DateTime, nullable=False, default=func.now(), onupdate=func.now()) + deleted_at = Column(DateTime, nullable=True) +``` + +### 4.3 Repository 接口 + +遵循现有 Repository Pattern（`application/ports/` 定义接口，`infrastructure/stores/` 实现）。 + +```python +# Port 定义 +class ReproContextPackPort(ABC): + @abstractmethod + async def save(self, pack: ReproContextPack, user_id: str, depth: str) -> str: ... + + @abstractmethod + async def get(self, pack_id: str) -> Optional[ReproContextPack]: ... + + @abstractmethod + async def list_by_user(self, user_id: str, paper_id: Optional[str], limit: int, offset: int) -> Tuple[List[ReproContextPackSummary], int]: ... + + @abstractmethod + async def soft_delete(self, pack_id: str) -> bool: ... + + @abstractmethod + async def save_stage_result(self, pack_id: str, stage_name: str, result: dict, confidence: float, duration_ms: int) -> None: ... +``` + +--- + +## 5. Provider Bridge（外接同步） + +### 5.1 职责 + +- 主写入：本地 SQLAlchemy store（必须成功） +- 可选双写：OneContext 或其他外接 provider（失败不阻断） +- 通过 feature flag 控制 + +### 5.2 接口 + +```python +class ContextProviderBridge: + """本地存储 + 可选外接 provider 双写。""" + + def __init__( + self, + local_store: ReproContextPackPort, + external_providers: List[ExternalContextProvider] = [], + ): + ... + + async def save(self, pack: ReproContextPack, user_id: str, depth: str) -> str: + """ + 1. 写入 local_store（必须成功） + 2. 异步双写到 external_providers（失败仅 log warning） + """ + ... + + async def get(self, pack_id: str) -> Optional[ReproContextPack]: + """仅从 local_store 读取。""" + ... + + +class ExternalContextProvider(ABC): + """外接 context provider 抽象。""" + + @abstractmethod + async def sync(self, pack: ReproContextPack) -> bool: ... + + @abstractmethod + async def health_check(self) -> bool: ... + + +class OneContextProvider(ExternalContextProvider): + """OneContext 实现，behind feature flag。""" + + def __init__(self, api_url: str, api_key: str): + ... + + async def sync(self, pack: ReproContextPack) -> bool: + """将 pack 转换为 OneContext 格式并上传。""" + ... +``` + +### 5.3 Feature Flag + +```python +# config/settings.py 或 .env +PAPERBOT_P2C_ONECONTEXT_ENABLED=false +PAPERBOT_P2C_ONECONTEXT_API_URL= +PAPERBOT_P2C_ONECONTEXT_API_KEY= +``` + +--- + +## 6. 路由注册 + +```python +# src/paperbot/api/routes/repro_context.py + +from fastapi import APIRouter +router = APIRouter(prefix="/api/research/repro/context", tags=["P2C"]) + +@router.post("/generate") +async def generate_context_pack(request: GenerateContextPackRequest): ... + +@router.get("/{context_pack_id}") +async def get_context_pack(context_pack_id: str): ... + +@router.get("/") +async def list_context_packs(user_id: str = "default", ...): ... + +@router.post("/{context_pack_id}/session") +async def create_repro_session(context_pack_id: str, request: CreateSessionRequest): ... + +@router.delete("/{context_pack_id}") +async def delete_context_pack(context_pack_id: str): ... +``` + +在 `src/paperbot/api/main.py` 中注册： + +```python +from paperbot.api.routes.repro_context import router as repro_context_router +app.include_router(repro_context_router) +``` + +--- + +## 7. SSE 进度推送实现 + +复用现有 `src/paperbot/api/streaming.py` 中的 SSE 能力。 + +```python +# generate endpoint 内部 +async def generate_context_pack(request: GenerateContextPackRequest): + async def event_generator(): + def on_stage_complete(stage_name: str, progress: float): + # 通过 asyncio.Queue 传递事件 + queue.put_nowait({ + "event": "stage_progress", + "data": {"stage": stage_name, "progress": progress} + }) + + # 启动 Core Engine + pack = await orchestrator.run( + normalized_input, + depth=request.depth, + on_stage_complete=on_stage_complete, + ) + + # 持久化 + pack_id = await bridge.save(pack, request.user_id, request.depth) + + queue.put_nowait({ + "event": "completed", + "data": {"context_pack_id": pack_id, "confidence": pack.confidence} + }) + + return sse_response(event_generator()) +``` + +--- + +## 8. 代码组织 + +``` +src/paperbot/ + api/routes/ + repro_context.py # 新增：P2C API 端点 + application/ports/ + repro_context_port.py # 新增：Repository 接口 + infrastructure/ + stores/ + repro_context_store.py # 新增：SQLAlchemy 实现 + connectors/ + onecontext_connector.py # 新增：OneContext provider（feature flag） + p2c/ + provider_bridge.py # 新增：双写桥接 +``` + +Alembic 迁移： + +```bash +alembic revision --autogenerate -m "add repro_context_pack tables" +alembic upgrade head +``` + +--- + +## 9. 与 Module 1 / Module 3 的接口契约 + +### 调用 Module 1（Core Engine） + +```python +# 在 generate endpoint 中 +from paperbot.p2c.input_normalizer import InputNormalizer +from paperbot.p2c.orchestrator import ExtractionOrchestrator + +normalizer = InputNormalizer(research_store, memory_store, context_engine) +normalized = await normalizer.normalize(request) + +orchestrator = ExtractionOrchestrator(...) +pack = await orchestrator.run(normalized, depth=request.depth, on_stage_complete=callback) +``` + +### 供 Module 3（Frontend）消费 + +前端通过以下 HTTP 接口消费数据： + +| 前端操作 | 调用的 API | 返回数据 | +|---|---|---| +| 点击 "Generate Reproduction Session" | `POST /generate`（SSE） | 阶段进度 + 完成结果 | +| 查看上下文包详情 | `GET /{pack_id}` | 完整 `ReproContextPack` JSON | +| 列出历史包 | `GET /` | 摘要列表 | +| 创建执行会话 | `POST /{pack_id}/session` | session_id + runbook steps | + +--- + +## 10. 风险与缓解 + +| 风险 | 缓解措施 | +|---|---| +| 外接 provider 不可用 | local-first，双写异步，失败仅 log | +| `pack_json` 存储过大 | 单独表存阶段结果，主表 pack_json 为最终输出 | +| SSE 连接中断 | 支持 `GET /{pack_id}` 轮询回退 | +| 并发生成同一论文 | 幂等检查：同一 `(user_id, paper_id, depth)` 10 分钟内复用 | +| 迁移兼容性 | `pack_json` 为 JSON text，schema 变更向后兼容（新增字段带默认值） | diff --git a/docs/P2C_MODULE_3_FRONTEND.md b/docs/P2C_MODULE_3_FRONTEND.md new file mode 100644 index 00000000..d6339cf1 --- /dev/null +++ b/docs/P2C_MODULE_3_FRONTEND.md @@ -0,0 +1,769 @@ +# P2C Module 3: Frontend Integration — Papers Library/Studio UI 对接 + +- 日期：2026-02-22 +- 状态：Draft +- 负责范围：Papers Library 详情页入口、Studio 页面注入、前端状态管理 +- 上游依赖：Module 2（API & Storage）提供的 HTTP 接口 +- 技术栈：Next.js 16 + React 19 + Tailwind CSS + +--- + +## 1. 模块职责 + +1. 复用 Papers Library 论文详情页的入口按钮（不新增入口按钮）； +2. 展示生成进度（SSE 实时 / 轮询回退）； +3. 生成完成后跳转 Studio 并注入 context pack 数据； +4. 在 Studio 中可视化 context pack（Blueprint、Roadmap、Metrics）； +5. 支持用户编辑后再执行（human-in-the-loop）。 + +--- + +## 2. 与现有代码的关系 + +| 现有组件 | 路径 | 复用方式 | +|---|---|---| +| Studio Store | `web/src/lib/store/studio-store.ts` | 扩展 state，注入 context pack | +| RunbookPanel | `web/src/components/studio/RunbookPanel.tsx` | 复用 runbook 渲染能力 | +| FilesPanel | `web/src/components/studio/FilesPanel.tsx` | 展示生成的文件结构 | +| Papers Library 详情页 | `web/src/app/papers/[id]/page.tsx` | 复用现有入口按钮触发 P2C | +| API 调用约定 | `web/src/app/api/` | 新增 proxy route | + +--- + +## 3. 消费的 API 接口 + +全部由 Module 2 提供，前端通过 Next.js API Route 代理调用。 + +| 操作 | 方法 | 端点 | 响应格式 | +|---|---|---|---| +| 生成上下文包 | POST | `/api/research/repro/context/generate` | SSE stream | +| 获取包详情 | GET | `/api/research/repro/context/{pack_id}` | JSON | +| 列出历史包 | GET | `/api/research/repro/context?user_id=...` | JSON | +| 创建执行会话 | POST | `/api/research/repro/context/{pack_id}/session` | JSON | +| 删除包 | DELETE | `/api/research/repro/context/{pack_id}` | JSON | + +--- + +## 4. 数据类型定义（TypeScript） + +```typescript +// web/src/types/p2c.ts + +export interface ReproContextPack { + context_pack_id: string; + version: string; + created_at: string; + + paper: PaperIdentity; + objective: string; + + literature_digest: LiteratureDigest | null; + blueprint: Blueprint | null; + environment: EnvironmentSpec | null; + implementation_spec: ImplementationSpec | null; + task_roadmap: TaskCheckpoint[]; + success_criteria: SuccessCriterion[]; + + evidence_links: EvidenceLink[]; + confidence: ConfidenceScores; + warnings: string[]; +} + +export interface PaperIdentity { + paper_id: string; + title: string; + year: number; + authors: string[]; + identifiers: Record; +} + +export interface LiteratureDigest { + problem_definition: string; + core_innovation: string; + relation_to_user: string; + key_references: string[]; +} + +export interface Blueprint { + architecture_type: string; + module_hierarchy: Record; + data_flow: [string, string][]; + core_algorithms: AlgorithmSpec[]; + loss_functions: string[]; + optimization_strategy: string; + key_hyperparameters: Record; + input_output_spec: Record; + paper_title: string; + paper_year: number; + framework_hints: string[]; + domain: string; +} + +export interface AlgorithmSpec { + name: string; + pseudocode: string; + complexity: string; + inputs: string[]; + outputs: string[]; +} + +export interface EnvironmentSpec { + python_version: string; + pytorch_version: string | null; + tensorflow_version: string | null; + cuda_version: string | null; + base_image: string; + pip_requirements: string[]; +} + +export interface ImplementationSpec { + model_type: string; + optimizer: string; + learning_rate: number; + batch_size: number; + epochs: number; + extra_params: Record; +} + +export interface TaskCheckpoint { + id: string; + title: string; + description: string; + acceptance_criteria: string[]; + depends_on: string[]; + estimated_difficulty: "low" | "medium" | "high"; +} + +export interface SuccessCriterion { + metric_name: string; + target_value: string; + dataset_split: string; + aggregation: string; + source: string; + tolerance: string | null; +} + +export interface EvidenceLink { + type: "paper_span" | "table" | "figure" | "code_snippet" | "metadata"; + ref: string; + supports: string[]; + confidence: number; +} + +export interface ConfidenceScores { + overall: number; + literature: number; + blueprint: number; + environment: number; + spec: number; + roadmap: number; + metrics: number; +} + +// SSE 事件类型 +export interface StageProgressEvent { + stage: string; + progress: number; + message: string; +} + +export interface GenerateCompletedEvent { + context_pack_id: string; + status: "completed"; + summary: string; + confidence: ConfidenceScores; + warnings: string[]; + next_action: "create_repro_session"; +} + +export interface GenerateErrorEvent { + error: string; + partial_pack_id?: string; +} + +// 列表项摘要 +export interface ContextPackSummary { + context_pack_id: string; + paper_title: string; + created_at: string; + confidence_overall: number; + status: string; + warning_count: number; +} +``` + +--- + +## 5. 前端状态管理 + +### 5.1 扩展 Studio Store + +在 `web/src/lib/store/studio-store.ts` 中扩展： + +```typescript +// 新增 state 字段 +interface StudioState { + // ... 现有字段 ... + + // P2C 相关 + contextPack: ReproContextPack | null; + contextPackLoading: boolean; + contextPackError: string | null; + generationProgress: StageProgressEvent[]; +} + +// 新增 actions +interface StudioActions { + // ... 现有 actions ... + + setContextPack: (pack: ReproContextPack | null) => void; + setContextPackLoading: (loading: boolean) => void; + appendGenerationProgress: (event: StageProgressEvent) => void; + clearGenerationProgress: () => void; + injectContextPackToRunbook: () => void; // pack → runbook steps 转换 +} +``` + +### 5.2 独立 Hook: `useContextPackGeneration` + +```typescript +// web/src/hooks/useContextPackGeneration.ts + +export function useContextPackGeneration() { + const [status, setStatus] = useState<"idle" | "generating" | "completed" | "error">("idle"); + const [progress, setProgress] = useState([]); + const [result, setResult] = useState(null); + const [error, setError] = useState(null); + + const generate = useCallback(async (params: { + paperId: string; + userId?: string; + projectId?: string; + trackId?: number; + depth?: "fast" | "standard" | "deep"; + }) => { + setStatus("generating"); + setProgress([]); + setError(null); + + try { + const response = await fetch("/api/research/repro/context/generate", { + method: "POST", + headers: { "Content-Type": "application/json", "Accept": "text/event-stream" }, + body: JSON.stringify({ + paper_id: params.paperId, + user_id: params.userId ?? "default", + project_id: params.projectId, + track_id: params.trackId, + depth: params.depth ?? "standard", + }), + }); + + const reader = response.body?.getReader(); + const decoder = new TextDecoder(); + + // SSE 解析循环 + while (reader) { + const { done, value } = await reader.read(); + if (done) break; + + const text = decoder.decode(value); + // 解析 SSE event/data 行 + for (const event of parseSSEEvents(text)) { + if (event.type === "stage_progress") { + setProgress(prev => [...prev, event.data as StageProgressEvent]); + } else if (event.type === "completed") { + setResult(event.data as GenerateCompletedEvent); + setStatus("completed"); + } else if (event.type === "error") { + setError((event.data as GenerateErrorEvent).error); + setStatus("error"); + } + } + } + } catch (err) { + setError(err instanceof Error ? err.message : "Unknown error"); + setStatus("error"); + } + }, []); + + return { status, progress, result, error, generate }; +} +``` + +--- + +## 6. UI 组件设计 + +### 6.1 Papers Library 详情页入口按钮（复用现有） + +**位置**：Papers Library 论文详情页头部操作区（不新增入口按钮）。 + +``` +┌─────────────────────────────────────────┐ +│ Attention Is All You Need (2017) │ +│ Vaswani et al. │ +│ │ +│ [PDF] [Chat] [Run Reproduction] ← 复用现有按钮 +└─────────────────────────────────────────┘ +``` + +**现有入口位置**：`web/src/app/papers/[id]/page.tsx` + +```typescript +// web/src/app/papers/[id]/page.tsx + +``` + +**约定**：在此按钮上接入 P2C 生成逻辑；不创建新的入口按钮组件。 + +### 6.2 生成进度条 + +``` +┌──────────────────────────────────────────────────────┐ +│ Generating Reproduction Context... │ +│ │ +│ ✅ Literature Distill │ +│ ✅ Blueprint Extract │ +│ 🔄 Environment Infer ━━━━━━━━━━░░░░░ 50% │ +│ ⬜ Spec & Hyperparams │ +│ ⬜ Task Roadmap │ +│ ⬜ Success Criteria │ +│ │ +│ Overall: ━━━━━━━━━━━━━░░░░░░░░░░ 33% │ +└──────────────────────────────────────────────────────┘ +``` + +**组件**：`GenerationProgressBar` + +```typescript +// web/src/components/research/GenerationProgressBar.tsx + +const STAGE_LABELS: Record = { + literature_distill: "Literature Distill", + blueprint_extract: "Blueprint Extract", + environment_infer: "Environment Infer", + spec_extract: "Spec & Hyperparams", + task_roadmap: "Task Roadmap", + success_criteria: "Success Criteria", +}; + +const ALL_STAGES = Object.keys(STAGE_LABELS); + +interface Props { + stages: StageProgressEvent[]; +} + +export function GenerationProgressBar({ stages }: Props) { + const completedStages = new Set( + stages.filter(s => s.progress >= 1.0).map(s => s.stage) + ); + const currentStage = stages.length > 0 ? stages[stages.length - 1] : null; + const overallProgress = stages.length > 0 + ? stages[stages.length - 1].progress + : 0; + + return ( +
+

Generating Reproduction Context...

+ {ALL_STAGES.map(stage => ( +
+ + {completedStages.has(stage) ? "✅" : + currentStage?.stage === stage ? "🔄" : "⬜"} + + + {STAGE_LABELS[stage]} + +
+ ))} + {/* Overall progress bar */} +
+
+
+
+ ); +} +``` + +### 6.3 Studio 页面：ContextPackPanel + +生成完成跳转 Studio 后，展示 context pack 的结构化数据。 + +``` +┌─ Context Pack: Attention Is All You Need ────────────┐ +│ │ +│ Confidence: ████████░░ 81% │ +│ │ +│ ┌─ Objective ─────────────────────────────────────┐ │ +│ │ 复现论文的核心 Transformer 架构并验证翻译指标 │ │ +│ └─────────────────────────────────────────────────┘ │ +│ │ +│ [Blueprint] [Environment] [Roadmap] [Metrics] │ ← Tab 切换 +│ │ +│ ┌─ Blueprint ─────────────────────────────────────┐ │ +│ │ Architecture: transformer │ │ +│ │ Modules: │ │ +│ │ model → [encoder, decoder, attention] │ │ +│ │ Core Algorithms: │ │ +│ │ - Multi-Head Attention (O(n²d)) │ │ +│ │ - Positional Encoding │ │ +│ └─────────────────────────────────────────────────┘ │ +│ │ +│ ⚠️ Warnings: │ +│ │ CUDA version inferred from paper year, verify │ │ +│ │ +│ [Edit & Customize] [Create Repro Session →] │ +└──────────────────────────────────────────────────────┘ +``` + +**组件**：`ContextPackPanel` + +```typescript +// web/src/components/studio/ContextPackPanel.tsx + +interface Props { + pack: ReproContextPack; + onCreateSession: () => void; + onEdit: () => void; +} + +export function ContextPackPanel({ pack, onCreateSession, onEdit }: Props) { + const [activeTab, setActiveTab] = useState< + "blueprint" | "environment" | "roadmap" | "metrics" + >("blueprint"); + + return ( +
+ {/* Header */} +
+
+

{pack.paper.title}

+

{pack.paper.year} · {pack.paper.authors.join(", ")}

+
+ +
+ + {/* Objective */} +
{pack.objective}
+ + {/* Tabs */} +
+ {(["blueprint", "environment", "roadmap", "metrics"] as const).map(tab => ( + + ))} +
+ + {/* Tab Content */} + {activeTab === "blueprint" && pack.blueprint && } + {activeTab === "environment" && pack.environment && } + {activeTab === "roadmap" && } + {activeTab === "metrics" && } + + {/* Warnings */} + {pack.warnings.length > 0 && ( +
+

Warnings:

+
    + {pack.warnings.map((w, i) =>
  • {w}
    • )} +
+
+ )} + + {/* Actions */} +
+ + +
+
+ ); +} +``` + +### 6.4 子组件 + +#### `ConfidenceBadge` + +```typescript +function ConfidenceBadge({ score }: { score: number }) { + const color = score >= 0.8 ? "green" : score >= 0.6 ? "yellow" : "red"; + return ( + + {Math.round(score * 100)}% confidence + + ); +} +``` + +#### `RoadmapView` + +``` +T1: 数据预处理 ✅ acceptance: [可加载训练集] + │ + ▼ +T2: 模型实现 acceptance: [forward 正确] + │ + ▼ +T3: 训练循环 acceptance: [loss 下降] + │ + ▼ +T4: 评估与指标验证 acceptance: [Top-1 >= 93.0] +``` + +```typescript +function RoadmapView({ checkpoints }: { checkpoints: TaskCheckpoint[] }) { + return ( +
+ {checkpoints.map((cp, i) => ( +
+ {/* 连接线 */} +
+
+ {cp.id} +
+ {i < checkpoints.length - 1 &&
} +
+ + {/* 内容 */} +
+

{cp.title}

+ {cp.description &&

{cp.description}

} +
+ + {cp.estimated_difficulty} + + {cp.acceptance_criteria.length > 0 && ( + + Acceptance: {cp.acceptance_criteria.join(", ")} + + )} +
+
+
+ ))} +
+ ); +} +``` + +#### `MetricsView` + +```typescript +function MetricsView({ criteria }: { criteria: SuccessCriterion[] }) { + return ( + + + + + + + + + + + {criteria.map((c, i) => ( + + + + + + + ))} + +
MetricTargetSplitSource
{c.metric_name}{c.target_value}{c.tolerance ? ` (${c.tolerance})` : ""}{c.dataset_split}{c.source}
+ ); +} +``` + +--- + +## 7. 页面路由与数据流 + +### 7.1 完整用户流程 + +``` +Papers Library Detail Page Studio Page +┌────────────────────────┐ ┌──────────────────────────────┐ +│ 论文详情页 │ │ │ +│ │ ① click │ ContextPackPanel │ +│ [Run Reproduction] │ ──────────→ │ ┌────────────────┐ │ +│ （复用现有按钮） │ ② SSE 进度 │ │ Blueprint/Env │ │ +│ │ ←─────────→ │ │ Roadmap/Metrics│ │ +│ │ ③ 完成后跳转 │ └────────────────┘ │ +└────────────────────────┘ ──────────→ │ │ + │ ④ [Create Repro Session] + │ ↓ │ + │ RunbookPanel (现有) │ + │ ┌────────────────┐ │ + │ │ Step 1: Setup │ │ + │ │ Step 2: Impl │ │ + │ │ Step 3: Verify│ │ + │ └────────────────┘ │ + └────────────────────────┘ +``` + +### 7.2 Studio 页面加载逻辑 + +```typescript +// web/src/app/studio/page.tsx (或对应路由) + +export default function StudioPage() { + const searchParams = useSearchParams(); + const contextPackId = searchParams.get("context_pack_id"); + const { contextPack, setContextPack } = useStudioStore(); + + useEffect(() => { + if (contextPackId && !contextPack) { + // 从 API 加载 context pack + fetch(`/api/research/repro/context/${contextPackId}`) + .then(res => res.json()) + .then(setContextPack); + } + }, [contextPackId, contextPack, setContextPack]); + + return ( +
+ {contextPack && ( + + )} + {/* 现有 Studio 内容 */} +
+ ); +} +``` + +### 7.3 创建会话后注入 Runbook + +```typescript +async function handleCreateSession() { + const res = await fetch( + `/api/research/repro/context/${contextPack.context_pack_id}/session`, + { method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ executor_preference: "auto" }) } + ); + const session = await res.json(); + + // 注入到现有 Studio runbook + studioStore.setRunbookId(session.runbook_id); + studioStore.setRunbookSteps(session.initial_steps); +} +``` + +--- + +## 8. Next.js API Route（代理层） + +前端需要一个代理 route 将请求转发到 Python 后端，复用现有 proxy 模式。 + +``` +web/src/app/api/research/repro/context/ + route.ts # GET (list) + POST (generate) + [packId]/ + route.ts # GET (detail) + DELETE + session/ + route.ts # POST (create session) +``` + +```typescript +// web/src/app/api/research/repro/context/route.ts + +const BACKEND_URL = process.env.PAPERBOT_API_URL || "http://localhost:8000"; + +export async function POST(request: NextRequest) { + const body = await request.json(); + const backendRes = await fetch(`${BACKEND_URL}/api/research/repro/context/generate`, { + method: "POST", + headers: { "Content-Type": "application/json", "Accept": "text/event-stream" }, + body: JSON.stringify(body), + }); + + // 透传 SSE stream + return new Response(backendRes.body, { + headers: { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + "Connection": "keep-alive", + }, + }); +} + +export async function GET(request: NextRequest) { + const { searchParams } = new URL(request.url); + const query = searchParams.toString(); + const backendRes = await fetch(`${BACKEND_URL}/api/research/repro/context?${query}`); + const data = await backendRes.json(); + return NextResponse.json(data); +} +``` + +--- + +## 9. 代码组织 + +``` +web/src/ + types/ + p2c.ts # 新增：TypeScript 类型定义 + hooks/ + useContextPackGeneration.ts # 新增：SSE 生成 hook + components/ + research/ + GenerateReproButton.tsx # 新增：入口按钮 + GenerationProgressBar.tsx # 新增：进度条 + studio/ + ContextPackPanel.tsx # 新增：Pack 可视化面板 + BlueprintView.tsx # 新增：Blueprint tab + EnvironmentView.tsx # 新增：Environment tab + RoadmapView.tsx # 新增：Roadmap tab + MetricsView.tsx # 新增：Metrics tab + ConfidenceBadge.tsx # 新增：置信度徽章 + app/ + api/research/repro/context/ + route.ts # 新增：API proxy + [packId]/ + route.ts # 新增：API proxy + session/ + route.ts # 新增：API proxy + lib/store/ + studio-store.ts # 修改：扩展 P2C state +``` + +--- + +## 10. 与 Module 2 的协调事项 + +| 协调点 | 说明 | +|---|---| +| SSE 事件格式 | 前端假设 `event: stage_progress` / `event: completed` / `event: error`，需与后端对齐 | +| `ReproContextPack` JSON 字段 | 前端 TypeScript 类型需与后端 Python dataclass 的 JSON 序列化一致 | +| API 路径 | 统一使用 `/api/research/repro/context/*` 前缀 | +| 错误响应格式 | 统一为 `{ "error": "message", "detail": "..." }` | +| 分页参数 | `limit` + `offset`，返回 `{ items: [...], total: N }` | + +--- + +## 11. 风险与缓解 + +| 风险 | 缓解措施 | +|---|---| +| SSE 连接断开 | `useContextPackGeneration` 内置重连逻辑 + 轮询 fallback | +| 大 pack JSON 导致渲染卡顿 | Tab 懒加载，仅渲染当前 tab 内容 | +| 用户编辑 roadmap 后与后端不同步 | 编辑后的 override 通过 `create_session` 请求提交 | +| 前后端类型漂移 | 考虑后续引入 OpenAPI spec 自动生成 TypeScript 类型 | diff --git a/docs/PAPER_TO_CONTEXT_MODULE_DESIGN.md b/docs/PAPER_TO_CONTEXT_MODULE_DESIGN.md new file mode 100644 index 00000000..962cd995 --- /dev/null +++ b/docs/PAPER_TO_CONTEXT_MODULE_DESIGN.md @@ -0,0 +1,684 @@ +# Paper-to-Context (P2C) 模块设计文档 + +- 日期：2026-02-21 +- 状态：Draft（未提交） +- 目标仓库：PaperBot +- 关联方向：论文采集/收藏闭环、OneContext 上下文层、Claude Scholar skills/agents、Paper2Code 复现执行 + +--- + +## 1. 背景与目标 + +你当前产品方向是： + +1. 每日个性化推荐论文； +2. LLM 解释“为什么值得复现”； +3. 用户在 PaperBot 内直接创建 task/job 去复现； +4. 后续对接 Claude Code / Codex 执行。 + +当前缺口不是“执行器能力”，而是**把论文转换成高质量可执行上下文**的中间层。 +P2C（Paper-to-Context）模块的职责就是： + +- 将论文（以及用户项目上下文）拆解为结构化复现包； +- 在执行前完成“关键信息提取 + 任务拆解 + 约束注入”； +- 向后兼容现有 Paper2Code 管线； +- 为后续 Claude Code / Codex 适配层提供统一输入。 + +一句话：**P2C 是你的“论文理解与执行计划编译器”**。 + +--- + +## 2. 设计原则 + +1. **执行器无关（Executor-Agnostic）** + P2C 产物不绑定 CC/Codex；通过 adapter 层再转成各端会话格式。 + +2. **结构化优先（Structured First）** + 先产出 JSON schema（可校验、可追踪），再渲染 Markdown（人可读）。 + +3. **多阶段提取（Multi-pass Extraction）** + 不做“一次 Prompt 生成全部”；分阶段抽取与校验，失败可降级。 + +4. **证据可追溯（Evidence Traceability）** + 每条关键结论必须带 evidence span/source，避免幻觉扩散。 + +5. **本地优先 + 外接可选（Local-first + Optional External Context）** + 默认写入本地 memory/research store；OneContext 作为可选同步层。 + +--- + +## 3. 参考现状（仓库内） + +### 3.1 论文来源与收集（已具备） + +- Harvest 与检索： + - `src/paperbot/application/workflows/harvest_pipeline.py` + - `src/paperbot/application/services/paper_search_service.py` +- Seed discovery（related/cited/citing/coauthor）： + - `POST /api/research/discovery/seed`（`src/paperbot/api/routes/research.py`） +- Collections / Saved Papers / BibTeX / Zotero： + - `src/paperbot/api/routes/research.py` + +### 3.2 记忆与个性化（已具备） + +- `ContextEngine` + `TrackRouter`： + - `src/paperbot/context_engine/engine.py` + - `src/paperbot/context_engine/track_router.py` +- Memory schema： + - `src/paperbot/memory/schema.py` + +### 3.3 Paper2Code 关键信息提取（已具备核心能力） + +- `PaperContext` / `Blueprint` / `ReproductionPlan` / `ImplementationSpec`： + - `src/paperbot/repro/models.py` +- 关键提取节点： + - Blueprint 蒸馏（LLM + heuristic fallback）：`src/paperbot/repro/nodes/blueprint_node.py` + - Hyperparameter 分析（regex + LLM merge）：`src/paperbot/repro/nodes/analysis_node.py` + - 环境推断（year/code/LLM 三策略）：`src/paperbot/repro/nodes/environment_node.py` +- 多代理编排与修复循环： + - `src/paperbot/repro/orchestrator.py` + +这意味着：**P2C 不需要从 0 到 1，而是把现有提取能力产品化、结构化、可复用化**。 + +--- + +## 4. 外部参考与可借鉴点 + +### 4.1 OneContext（外接上下文层） + +可借鉴能力（以其公开 README/Documentation 为准）： + +- 统一 context 管理（多 session 归属同一 context）； +- 记录 agent trajectory； +- context 分享（链接/Slack）； +- 共享会话导入与恢复； +- CLI 入口与会话操作（`onecontext`/`oc`）。 + +对 PaperBot 的价值： + +- 适合做“**上下文协作与共享层**”，不替代你的推荐/复现领域逻辑； +- 可用于跨设备、跨会话续跑； +- 可作为外接 provider 做双写。 + +### 4.2 Claude Scholar（skills/agents 编排资产） + +可借鉴能力： + +- `agents/literature-reviewer.md`：文献检索与 Zotero 工作流规范； +- `agents/architect.md`：架构分解模板； +- `agents/dev-planner.md`：任务拆解与交付计划； +- `skills/results-analysis/SKILL.md`：实验结果分析与统计报告规范； +- `skills/planning-with-files/SKILL.md`：plan/notes/deliverable 的“文件化工作记忆”。 + +对 PaperBot 的价值： + +- 可作为 P2C 各阶段的 prompt/规则资产来源； +- 不建议直接耦合全部 command/hook 体系，建议抽取“可复用的 skill prompt 片段”。 + +### 4.3 与现有 Paper2Code 的衔接 + +- 你已具备论文 -> Blueprint/Spec 的 extraction 能力； +- P2C 重点是补齐“面向产品上下文包”的 schema、质量门禁与版本化。 + +--- + +## 5. 模块边界与职责 + +### 5.1 模块名称 + +`paper_to_context`（P2C Engine） + +### 5.2 In Scope + +- 论文输入标准化（paper metadata + text + user/project memory）； +- 多阶段提取与校验（Blueprint/Spec/Plan/Metrics）； +- 输出标准化 `ReproContextPack`； +- 写入本地 store，并可选同步到外接 context provider； +- 提供 API 给 Research UI / Studio UI 调用。 + +### 5.3 Out of Scope + +- 直接执行代码（由 ReproAgent/Orchestrator + executor adapter 负责）； +- 替代推荐引擎排序； +- 替代 OneContext 的产品 UI。 + +--- + +## 6. 端到端架构 + +```text +┌───────────────────────────────────────────────────────────────┐ +│ Paper Sources Layer │ +│ Discovery Seed / Saved Papers / Collections / BibTeX/Zotero │ +└───────────────────────────────┬───────────────────────────────┘ + │ + v +┌───────────────────────────────────────────────────────────────┐ +│ P2C Input Normalizer │ +│ normalize paper identity + full text + user/project memory │ +└───────────────────────────────┬───────────────────────────────┘ + │ + v +┌───────────────────────────────────────────────────────────────┐ +│ Skill Orchestration Pipeline │ +│ Stage A: Literature Distill (claude-scholar literature) │ +│ Stage B: Blueprint Extract (Paper2Code blueprint node) │ +│ Stage C: Environment Extract (env node) │ +│ Stage D: Spec/Hyperparams (analysis node) │ +│ Stage E: Dev Plan/Roadmap (planner skill + planning node)│ +│ Stage F: Result Criteria (results-analysis style) │ +└───────────────────────────────┬───────────────────────────────┘ + │ + v +┌───────────────────────────────────────────────────────────────┐ +│ Context Assembly + Validation │ +│ JSON schema validate + evidence links + confidence scoring │ +└───────────────┬───────────────────────────────┬───────────────┘ + │ │ + v v +┌──────────────────────────────┐ ┌────────────────────────────┐ +│ Local Context Store (primary)│ │ External Context Provider │ +│ SQLAlchemyMemory/ResearchStore│ │ OneContext (optional sync) │ +└───────────────┬──────────────┘ └──────────────┬─────────────┘ + │ │ + └───────────────┬─────────────────┘ + v +┌───────────────────────────────────────────────────────────────┐ +│ Studio/Research UI Seed │ +│ Generate Reproduction Session -> CC/Codex Adapter (next step)│ +└───────────────────────────────────────────────────────────────┘ +``` + +--- + +## 7. 核心数据模型（建议） + +### 7.1 `ReproContextPack` + +```json +{ + "context_pack_id": "ctxp_xxx", + "paper": { + "paper_id": "...", + "title": "...", + "year": 2026, + "identifiers": {"doi": "...", "arxiv": "...", "s2": "..."} + }, + "objective": "复现论文的核心方法并验证主要指标", + "blueprint": {"architecture_type": "transformer", "module_hierarchy": {}, "data_flow": []}, + "environment": {"python_version": "3.10", "framework": "pytorch", "cuda": "11.8"}, + "implementation_spec": { + "optimizer": "adamw", + "learning_rate": 1e-4, + "batch_size": 32, + "epochs": 100 + }, + "task_roadmap": [ + {"id": "T1", "title": "数据预处理", "acceptance": ["可加载训练集"], "depends_on": []}, + {"id": "T2", "title": "模型实现", "acceptance": ["forward 正确"], "depends_on": ["T1"]} + ], + "success_criteria": [ + {"metric": "Top-1", "target": ">= 93.0", "source": "paper_table_2"} + ], + "evidence_links": [ + {"type": "paper_span", "ref": "method_section#L120-L140", "supports": ["optimizer", "lr"]} + ], + "confidence": { + "overall": 0.81, + "blueprint": 0.84, + "env": 0.78, + "metrics": 0.73 + }, + "version": "v1", + "created_at": "2026-02-21T00:00:00Z" +} +``` + +### 7.2 存储建议 + +新增实体（可在 `research_store` 侧落地）： + +- `repro_context_pack` +- `repro_context_stage_result` +- `repro_context_evidence` +- `repro_context_feedback` + +关键索引： + +- `(user_id, project_id, paper_id, created_at desc)` +- `(context_pack_id)` +- `(paper_id, version)` + +--- + +## 8. 技能编排设计（结合 claude-scholar + Paper2Code） + +### Stage A: 文献机制抽取（Literature Distill） + +- 输入：paper meta + abstract + method section + related citations +- 参考资产：`agents/literature-reviewer.md` +- 输出： + - 论文问题定义 + - 方法核心创新点 + - 与用户项目的关联点（why now） + +### Stage B: Blueprint 抽取 + +- 复用：`BlueprintDistillationNode` +- 机制：LLM JSON 抽取失败时 fallback heuristic +- 输出：`Blueprint` + +### Stage C: 环境推断 + +- 复用：`EnvironmentInferenceNode` +- 三路推断：year mapping + code pattern + LLM inference +- 输出：`EnvironmentSpec` + +### Stage D: 实现规格提取 + +- 复用：`AnalysisNode` +- 机制：regex hyperparams + LLM hyperparams merge +- 输出：`ImplementationSpec` + `config_yaml` + +### Stage E: 开发任务拆解（Roadmap） + +- 参考资产： + - `agents/dev-planner.md` + - `skills/planning-with-files/SKILL.md` +- 输出： + - checkpoint 列表 + - 依赖关系 DAG + - 每步验收标准 + +### Stage F: 结果与复现成功标准抽取 + +- 参考资产：`skills/results-analysis/SKILL.md` +- 输出： + - 指标定义（metric name / split / aggregation） + - 统计显著性要求（可选） + - 最小可接受复现标准 + +--- + +## 9. 模块内部组件设计 + +### 9.1 `SkillLoader` + +职责： + +- 从本地 skill/agent markdown 读取模板； +- 提取 frontmatter + instruction body； +- 提供可参数化 prompt 片段。 + +接口建议： + +```python +class SkillLoader: + def load_skill(self, key: str) -> SkillTemplate: ... + def render(self, key: str, variables: dict) -> str: ... +``` + +### 9.2 `ExtractionOrchestrator` + +职责： + +- 串联 Stage A-F； +- 阶段失败降级； +- 汇总 stage confidence。 + +接口建议： + +```python +class ExtractionOrchestrator: + async def run(self, request: GenerateContextRequest) -> ReproContextPack: ... +``` + +### 9.3 `EvidenceLinker` + +职责： + +- 为每个关键字段绑定证据来源（文本 span / 表格 / meta）； +- 对“无证据高风险字段”打低置信度并标记人工确认。 + +### 9.4 `ContextAssembler` + +职责： + +- 将 Blueprint/Env/Spec/Roadmap 合成为 `ReproContextPack`； +- 输出 JSON + Markdown（`REPRODUCTION_PLAN.md`）。 + +### 9.5 `ContextProviderBridge` + +职责： + +- local store 为主写； +- OneContext 可选双写； +- 外接 provider 异常不阻断主流程。 + +--- + +## 10. API 设计（MVP） + +### 10.1 生成上下文包 + +`POST /api/research/repro/context/generate` + +请求： + +```json +{ + "user_id": "default", + "project_id": "proj_001", + "paper_id": "paper_xxx", + "track_id": 30, + "depth": "standard", + "executor_preference": "auto" +} +``` + +响应： + +```json +{ + "context_pack_id": "ctxp_xxx", + "status": "completed", + "summary": "...", + "confidence": {"overall": 0.81}, + "next_action": "create_repro_session" +} +``` + +### 10.2 获取上下文包详情 + +`GET /api/research/repro/context/{context_pack_id}` + +返回完整 `ReproContextPack`。 + +### 10.3 由上下文包创建复现会话 + +`POST /api/research/repro/context/{context_pack_id}/session` + +响应： + +```json +{ + "session_id": "sess_xxx", + "runbook_steps": [...], + "initial_prompt": "..." +} +``` + +> 备注：后续 CC/Codex adapter 对接时，可直接消费该接口输出。 + +--- + +## 11. 与现有 UI 的对接方案 + +### 11.1 Research 页面入口 + +在以下页面增加按钮： + +- discovery 卡片 +- collections item +- saved papers item + +按钮：`Generate Reproduction Session` + +动作： + +1. 调 `/api/research/repro/context/generate`； +2. 轮询或 SSE 获取阶段进度； +3. 成功后跳转 `/studio` 并注入 context pack。 + +### 11.2 Studio 页面注入 + +复用 `web/src/lib/store/studio-store.ts` 的 `paperDraft` / task timeline 能力： + +- 将 `objective/blueprint/task_roadmap` 注入 runbook panel； +- 在 `BlueprintPanel` 展示 extraction 产物； +- 允许用户“编辑后再执行”（human-in-the-loop）。 + +--- + +## 12. 质量门禁与评估 + +### 12.1 抽取质量门禁 + +- JSON schema 校验必须通过； +- 必填字段空值率 < 5%； +- `success_criteria` 至少 1 项； +- 关键字段证据覆盖率 >= 80%。 + +### 12.2 离线评测集 + +构建 50 篇带“人工标准答案”的论文集，评估： + +- 架构类型识别准确率； +- 超参数抽取准确率； +- 指标目标抽取准确率； +- 路线图可执行率（人工评分）。 + +### 12.3 线上业务指标 + +- context 生成成功率 +- context -> job 创建转化率 +- job 成功率提升幅度（对比无 P2C） +- 失败任务中“缺少关键信息”占比下降 + +--- + +## 13. 风险与缓解 + +1. **技能提示词漂移（claude-scholar 上游更新）** + - 缓解：skill snapshot 固化到本仓库；版本号管理；差异审查。 + +2. **外接 provider 可用性风险（OneContext）** + - 缓解：local-first；双写异步；失败自动回退。 + +3. **抽取幻觉导致错误执行** + - 缓解：evidence-link 强制 + 低置信度人工确认。 + +4. **上下文过长导致执行退化** + - 缓解：Blueprint 压缩优先；分层注入（必要字段优先）。 + +5. **许可与合规风险（第三方 prompt 资产）** + - 缓解：仅引用 MIT/明确许可资产；保留归因；禁止受限内容直拷。 + +--- + +## 14. 目录与实现建议（代码组织） + +```text +src/paperbot/ + application/services/p2c/ + __init__.py + models.py # ReproContextPack / stage outputs + skill_loader.py # load/parse skill templates + extraction_orchestrator.py + evidence_linker.py + assembler.py + provider_bridge.py # local + optional onecontext + api/routes/ + repro_context.py # new endpoints + infrastructure/connectors/ + onecontext_connector.py # optional, behind feature flag + +web/src/ + app/api/research/repro/context/ + route.ts + components/research/ + GenerateReproSessionButton.tsx + components/studio/ + ContextPackPanel.tsx +``` + +--- + +## 15. 分阶段落地计划（4 周） + +### Week 1: 基础骨架 + +- 定义 `ReproContextPack` schema； +- 打通 `generate/get` API（先 mock stage）； +- UI 加入口按钮与状态反馈。 + +### Week 2: 复用 Paper2Code 提取能力 + +- 集成 Blueprint/Environment/Analysis 节点； +- 产出第一版 context pack； +- 加 schema + evidence 校验。 + +### Week 3: skill 编排增强 + +- 接入 claude-scholar 参考模板（literature/dev-planner/results）； +- 生成 roadmap + success criteria； +- 引入置信度评分。 + +### Week 4: Provider 与运营化 + +- local-first 持久化稳定； +- OneContext 可选双写（feature flag）； +- 指标埋点与 A/B（有无 P2C）上线。 + +--- + +## 16. 最小可用 DoD + +- 可从任一论文卡片触发“Generate Reproduction Session”； +- 生成结构化 `ReproContextPack` 并可在 Studio 可视化； +- 可一键创建复现会话并带 roadmap； +- 失败可回退并提供可读错误原因； +- 对比基线，job 成功率或启动效率有可量化提升。 + +--- + +## 17. 实施分工（建议） + +> 目标：并行推进，减少跨团队阻塞。以下是建议 owner，可按你团队实际调整。 + +### Workstream A：P2C Core（后端 + 算法） + +- **Owner**：Backend/ML（1-2 人） +- **范围**： + - `application/services/p2c/*` 核心 pipeline + - `ReproContextPack` schema 与 stage 结果持久化 + - evidence-link + confidence 评分 +- **里程碑交付**： + - `POST /api/research/repro/context/generate` 可用 + - 50 篇离线评测脚本可跑通 + +### Workstream B：Skill 资产与提示词治理（PromptOps） + +- **Owner**：Prompt/Research（1 人） +- **范围**： + - claude-scholar skill/agent 的可复用片段抽取 + - skill snapshot 版本管理（避免上游漂移） + - 失败降级策略与质量规则 +- **里程碑交付**： + - `skills_manifest.yaml` + - 每个 stage 的 prompt 模板与回归样例 + +### Workstream C：Provider Bridge（OneContext 对接） + +- **Owner**：Backend Platform（1 人） +- **范围**： + - `onecontext_connector.py` + - feature flag / 双写 / 重试 / dead-letter queue + - provider 健康检查与观测指标 +- **里程碑交付**： + - 本地写成功不受外接失败影响（local-first） + - 同步延迟和失败率可观测 + +### Workstream D：Studio/Research UI（前端） + +- **Owner**：Frontend（1-2 人） +- **范围**： + - Research 页触发入口（Generate Reproduction Session） + - Studio `ContextPackPanel` 展示与确认编辑 + - 执行器适配输入可视化（CC/Codex 统一入口） +- **里程碑交付**： + - 用户 3 步内完成 “论文 -> context -> 任务启动” + - 关键链路 e2e 用例稳定 + +### Workstream E：Infra / SRE / 合规 + +- **Owner**：Infra（1 人） +- **范围**： + - 成本与限流策略（LLM 调用预算） + - 数据合规（license、来源归因、删除策略） + - 线上回滚预案（feature flag + canary） +- **里程碑交付**： + - SLA dashboard + - 合规审计 checklist + +--- + +## 18. Issue 拆分建议（可直接建单） + +> 对应“一个 issue 一个可验收产物”，避免大而全任务。 + +1. `P2C-01`：定义 `ReproContextPack` schema + 校验器 + 示例数据 +2. `P2C-02`：实现 Stage A/B（literature + blueprint）并落库 +3. `P2C-03`：实现 Stage C/D（environment + hyperparams）并落库 +4. `P2C-04`：实现 Stage E/F（roadmap + success criteria） +5. `P2C-05`：evidence-link + confidence 评分 +6. `P2C-06`：`/repro/context/generate` + `/repro/context/{id}` API +7. `P2C-07`：OneContext bridge（feature flag + async 双写） +8. `P2C-08`：Research 页面生成入口 + 进度反馈 +9. `P2C-09`：Studio `ContextPackPanel` + 人工确认 UI +10. `P2C-10`：离线评测集 + 线上指标埋点 + A/B 报告 + +每个 issue 验收建议至少包含： + +- API contract（请求/响应示例）； +- 单元测试或 e2e 证据； +- 指标或日志截图（成功率/耗时/失败原因分布）。 + +--- + +## 19. 与现有 Paper Collection Pipeline 的对接细节 + +### 输入拼装（from collection/discovery） + +- 主数据：`paper_id/title/abstract/year/identifiers` +- 扩展数据：discovery graph 邻居（related/cited/citing/coauthor） +- 用户上下文：track、收藏、最近任务失败原因、项目目标 + +### 输出回流（to memory/execution） + +- 写入 `repro_context_pack`（主表）+ `stage_result`（诊断） +- 更新 track 记忆：该论文的复现优先级、风险标签、建议 action +- 触发 Studio 会话创建：将 roadmap 作为默认 task backlog + +### 闭环信号（execution feedback -> recommendation） + +- 执行成功/失败原因写回 `repro_context_feedback` +- 失败类型（环境不匹配/关键参数缺失/数据不可得）用于修正下次推荐 +- 对高价值失败样本触发“二次 context 生成”（补充提取） + +--- + +## 20. 关键参考链接 + +### 外部资料 + +- OneContext 仓库： +- OneContext 使用文档： +- Claude Scholar 仓库： +- Claude Scholar literature-reviewer： +- Claude Scholar architect： +- Claude Scholar dev-planner： +- Claude Scholar planning-with-files： +- Claude Scholar results-analysis： + +### 本仓库实现参考 + +- `src/paperbot/repro/models.py` +- `src/paperbot/repro/nodes/blueprint_node.py` +- `src/paperbot/repro/nodes/analysis_node.py` +- `src/paperbot/repro/nodes/environment_node.py` +- `src/paperbot/repro/orchestrator.py` +- `src/paperbot/api/routes/research.py` +- `src/paperbot/api/routes/gen_code.py` +- `web/src/components/studio/RunbookPanel.tsx` +- `web/src/lib/store/studio-store.ts` diff --git a/evals/fixtures/p2c/module1_gold.json b/evals/fixtures/p2c/module1_gold.json new file mode 100644 index 00000000..54b856af --- /dev/null +++ b/evals/fixtures/p2c/module1_gold.json @@ -0,0 +1,41 @@ +[ + { + "case_id": "transformer_benchmark", + "title": "Transformer Benchmark Study", + "year": 2025, + "abstract": "We introduce a transformer model and report strong accuracy gains.", + "full_text": "Introduction\nA benchmark-oriented study.\nMethod\nOur model uses transformer layers. Learning rate 1e-4, batch size 32, epochs 10.\nResults\nAccuracy reaches 92.1 on test split.\nConclusion\nThe method is effective.\n", + "expected": { + "architecture": "transformer", + "metrics": ["accuracy"], + "hyperparameters": ["learning_rate", "batch_size", "epochs"], + "evidence_required_types": ["metric", "hyperparameter"] + } + }, + { + "case_id": "gnn_auc", + "title": "Graph Neural Ranking", + "year": 2024, + "abstract": "We propose a graph neural ranking model.", + "full_text": "Background\nWe study retrieval ranking.\nMethod\nA graph neural network encoder is trained with learning rate 0.0005 and batch size 64.\nEvaluation\nAUC improves to 0.89.\nConclusion\nThe method generalizes.\n", + "expected": { + "architecture": "gnn", + "metrics": ["auc"], + "hyperparameters": ["learning_rate", "batch_size"], + "evidence_required_types": ["metric", "hyperparameter", "environment"] + } + }, + { + "case_id": "cnn_mrr", + "title": "Convolutional Retrieval Model", + "year": 2023, + "abstract": "A convolutional architecture for retrieval tasks.", + "full_text": "1 Introduction\nTask setup.\n2 Methodology\nA convolution encoder with lr=0.001 and epochs 12.\n3 Results\nMRR reaches 0.71 in our experiments.\n4 Future Work\nAnalyze robustness.\n", + "expected": { + "architecture": "cnn", + "metrics": ["mrr"], + "hyperparameters": ["learning_rate", "epochs"], + "evidence_required_types": ["metric", "hyperparameter"] + } + } +] diff --git a/scripts/p2c_module1_benchmark.py b/scripts/p2c_module1_benchmark.py new file mode 100644 index 00000000..68da1201 --- /dev/null +++ b/scripts/p2c_module1_benchmark.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import argparse +import asyncio +import json +from pathlib import Path + +from paperbot.application.services.p2c import load_benchmark_cases, run_module1_benchmark + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Run Paper2Context Module 1 benchmark.") + parser.add_argument( + "--fixtures", + default="evals/fixtures/p2c/module1_gold.json", + help="Path to benchmark fixture JSON.", + ) + parser.add_argument( + "--output", + default="", + help="Optional output path for benchmark result JSON.", + ) + parser.add_argument( + "--fail-under-metric-f1", + type=float, + default=0.0, + help="Exit with code 1 if summary metric_f1 is below this value.", + ) + return parser + + +async def _run(args: argparse.Namespace) -> int: + cases = load_benchmark_cases(args.fixtures) + result = await run_module1_benchmark(cases) + + payload = json.dumps(result, indent=2, ensure_ascii=False) + print(payload) + + if args.output: + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(payload + "\n", encoding="utf-8") + + metric_f1 = float(result["summary"].get("metric_f1", 0.0)) + if metric_f1 < args.fail_under_metric_f1: + return 1 + return 0 + + +def main() -> int: + parser = build_parser() + args = parser.parse_args() + return asyncio.run(_run(args)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/paperbot/api/main.py b/src/paperbot/api/main.py index fb63d64f..a328194d 100644 --- a/src/paperbot/api/main.py +++ b/src/paperbot/api/main.py @@ -24,6 +24,7 @@ harvest, model_endpoints, studio_chat, + repro_context, ) from paperbot.infrastructure.event_log.logging_event_log import LoggingEventLog from paperbot.infrastructure.event_log.composite_event_log import CompositeEventLog @@ -71,6 +72,7 @@ async def health_check(): app.include_router(harvest.router, prefix="/api", tags=["Harvest"]) app.include_router(model_endpoints.router, prefix="/api", tags=["Model Endpoints"]) app.include_router(studio_chat.router, prefix="/api", tags=["Studio Chat"]) +app.include_router(repro_context.router, prefix="/api/research/repro/context", tags=["P2C"]) @app.on_event("startup") diff --git a/src/paperbot/api/routes/__init__.py b/src/paperbot/api/routes/__init__.py index eef8cb9a..fb3d6a29 100644 --- a/src/paperbot/api/routes/__init__.py +++ b/src/paperbot/api/routes/__init__.py @@ -15,6 +15,7 @@ paperscool, newsletter, model_endpoints, + repro_context, ) __all__ = [ @@ -32,4 +33,5 @@ "paperscool", "newsletter", "model_endpoints", + "repro_context", ] diff --git a/src/paperbot/api/routes/repro_context.py b/src/paperbot/api/routes/repro_context.py new file mode 100644 index 00000000..e9ca48ab --- /dev/null +++ b/src/paperbot/api/routes/repro_context.py @@ -0,0 +1,365 @@ +""" +P2C (Paper-to-Context) API Route + +Endpoints: + POST /api/research/repro/context/generate — generate context pack (SSE) + GET /api/research/repro/context — list packs for a user + GET /api/research/repro/context/{pack_id} — get full pack detail + POST /api/research/repro/context/{pack_id}/session — create repro session from pack + DELETE /api/research/repro/context/{pack_id} — soft-delete pack +""" + +from __future__ import annotations + +import asyncio +import uuid +from dataclasses import asdict as _asdict +from typing import Literal, Optional + +from fastapi import APIRouter, HTTPException +from fastapi.responses import StreamingResponse +from pydantic import BaseModel + +from paperbot.api.streaming import StreamEvent, wrap_generator +from paperbot.application.services.p2c.models import ( + GenerateContextRequest as P2CRequest, + new_context_pack_id, +) +from paperbot.application.services.p2c.orchestrator import ExtractionOrchestrator +from paperbot.infrastructure.stores.repro_context_store import SqlAlchemyReproContextStore +from paperbot.utils.logging_config import LogFiles, Logger, set_trace_id + +router = APIRouter() + +_store = SqlAlchemyReproContextStore() + + +# ------------------------------------------------------------------ # +# Request / Response schemas # +# ------------------------------------------------------------------ # + +class GenerateContextPackRequest(BaseModel): + paper_id: str + user_id: str = "default" + project_id: Optional[str] = None + track_id: Optional[int] = None + depth: Literal["fast", "standard", "deep"] = "standard" + + +class CreateSessionRequest(BaseModel): + executor_preference: Literal["auto", "claude_code", "codex", "local"] = "auto" + override_env: Optional[dict] = None + override_roadmap: Optional[list] = None + + +# ------------------------------------------------------------------ # +# POST /generate (SSE) # +# ------------------------------------------------------------------ # + +async def _generate_stream(request: GenerateContextPackRequest): + """SSE generator for context pack generation via Module 1 ExtractionOrchestrator.""" + pack_id = new_context_pack_id() + Logger.info( + f"[M2] start generate pack_id={pack_id} paper_id={request.paper_id} depth={request.depth}", + file=LogFiles.API, + ) + + # Persist initial "running" record so the frontend can poll status. + await asyncio.to_thread( + _store.save, + pack_id=pack_id, + user_id=request.user_id, + paper_id=request.paper_id, + depth=request.depth, + pack_data={}, + project_id=request.project_id, + confidence_overall=0.0, + warning_count=0, + ) + + yield StreamEvent(type="status", data={"pack_id": pack_id, "status": "running"}) + + # asyncio.Queue bridges the on_stage_complete callback → SSE generator. + queue: asyncio.Queue = asyncio.Queue() + _DONE = object() + _ERROR = object() + + total_stages = 2 if request.depth == "fast" else 6 + stages_done = 0 + last_stage_name: Optional[str] = None + + async def on_stage_complete(stage_name: str, observations: list, warnings: list) -> None: + nonlocal stages_done + nonlocal last_stage_name + stages_done += 1 + last_stage_name = stage_name + confidence = ( + sum(o.confidence for o in observations) / len(observations) + if observations + else 0.0 + ) + observation_summaries = [ + { + "id": o.id, + "type": o.type, + "title": o.title, + "confidence": o.confidence, + } + for o in observations + ] + await asyncio.to_thread( + _store.save_stage_result, + pack_id=pack_id, + stage_name=stage_name, + status="completed", + result_data={"observations": [o.to_full() for o in observations]}, + confidence=confidence, + duration_ms=0, + ) + Logger.info( + f"[M2] stage_complete pack_id={pack_id} stage={stage_name} obs={len(observations)} warnings={len(warnings)}", + file=LogFiles.API, + ) + await queue.put( + StreamEvent( + type="stage_observations", + data={ + "stage": stage_name, + "observations": observation_summaries, + }, + ) + ) + await queue.put( + StreamEvent( + type="progress", + data={ + "stage": stage_name, + "progress": stages_done / total_stages, + "message": f"Completed {stage_name}", + }, + ) + ) + + p2c_request = P2CRequest( + paper_id=request.paper_id, + user_id=request.user_id, + project_id=request.project_id, + track_id=request.track_id, + depth=request.depth, + ) + orchestrator = ExtractionOrchestrator() + result_holder: list = [] + + async def _run() -> None: + try: + pack = await orchestrator.run(p2c_request, on_stage_complete=on_stage_complete) + result_holder.append(pack) + await queue.put(_DONE) + except ValueError as exc: + result_holder.append({"kind": "client", "message": str(exc)}) + await queue.put(_ERROR) + except Exception as exc: # noqa: BLE001 + Logger.error( + f"[M2] generation_failed pack_id={pack_id} error={exc}", + file=LogFiles.ERROR, + ) + result_holder.append({"kind": "server", "message": "Internal error during context pack generation."}) + await queue.put(_ERROR) + + asyncio.create_task(_run()) + + # Drain queue, yielding progress events until the orchestrator finishes. + while True: + item = await queue.get() + if item is _DONE: + break + elif item is _ERROR: + err = result_holder[0] + await asyncio.to_thread(_store.update_status, pack_id, status="failed") + yield StreamEvent(type="error", data=err) + return + else: + yield item # StreamEvent from on_stage_complete + + if last_stage_name: + yield StreamEvent( + type="progress", + data={ + "stage": last_stage_name, + "progress": 1.0, + "message": f"Completed {last_stage_name}", + }, + ) + + # Serialize and persist the final pack. + pack = result_holder[0] + pack_dict = _asdict(pack) + pack_dict["context_pack_id"] = pack_id # align with our DB record + + await asyncio.to_thread( + _store.update_status, + pack_id, + status="completed", + pack_data=pack_dict, + confidence_overall=pack.confidence.overall, + warning_count=len(pack.warnings), + objective=pack.objective, + ) + Logger.info( + f"[M2] generation_completed pack_id={pack_id} observations={len(pack.observations)} warnings={len(pack.warnings)}", + file=LogFiles.API, + ) + + yield StreamEvent( + type="result", + data={ + "context_pack_id": pack_id, + "status": "completed", + "summary": f"Context pack created for {request.paper_id}", + "confidence": _asdict(pack.confidence), + "warnings": pack.warnings, + "next_action": "create_repro_session", + }, + ) + + +@router.post("/generate") +async def generate_context_pack(request: GenerateContextPackRequest): + """Generate a P2C context pack for the given paper. Returns SSE stream.""" + trace_id = set_trace_id() + Logger.info( + f"[M2] generate_request trace_id={trace_id} paper_id={request.paper_id} user_id={request.user_id}", + file=LogFiles.API, + ) + return StreamingResponse( + wrap_generator( + _generate_stream(request), + workflow="p2c_generate", + ), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "Connection": "keep-alive"}, + ) + + +# ------------------------------------------------------------------ # +# GET / (list) # +# ------------------------------------------------------------------ # + +@router.get("") +async def list_context_packs( + user_id: str = "default", + paper_id: Optional[str] = None, + project_id: Optional[str] = None, + limit: int = 20, + offset: int = 0, +): + """List context packs for a user, with optional filters.""" + set_trace_id() + Logger.info( + f"[M2] list_packs user_id={user_id} paper_id={paper_id} project_id={project_id} limit={limit} offset={offset}", + file=LogFiles.API, + ) + items, total = await asyncio.to_thread( + _store.list_by_user, + user_id=user_id, + paper_id=paper_id, + project_id=project_id, + limit=limit, + offset=offset, + ) + return {"items": items, "total": total} + + +# ------------------------------------------------------------------ # +# GET /{pack_id} # +# ------------------------------------------------------------------ # + +@router.get("/{pack_id}") +async def get_context_pack(pack_id: str): + """Return the full context pack detail.""" + set_trace_id() + Logger.info(f"[M2] get_pack pack_id={pack_id}", file=LogFiles.API) + pack = await asyncio.to_thread(_store.get, pack_id) + if pack is None: + Logger.warning(f"[M2] pack_not_found pack_id={pack_id}", file=LogFiles.API) + raise HTTPException(status_code=404, detail="Context pack not found.") + return pack + + +@router.get("/{pack_id}/observation/{observation_id}") +async def get_observation_detail(pack_id: str, observation_id: str): + """Return a single observation detail by ID.""" + set_trace_id() + Logger.info( + f"[M2] get_observation pack_id={pack_id} observation_id={observation_id}", + file=LogFiles.API, + ) + observation = await asyncio.to_thread(_store.get_observation, pack_id, observation_id) + if observation is None: + Logger.warning( + f"[M2] observation_not_found pack_id={pack_id} observation_id={observation_id}", + file=LogFiles.API, + ) + raise HTTPException(status_code=404, detail="Observation not found.") + return observation + + +# ------------------------------------------------------------------ # +# POST /{pack_id}/session # +# ------------------------------------------------------------------ # + +@router.post("/{pack_id}/session") +async def create_repro_session(pack_id: str, request: CreateSessionRequest): + """ + Convert a context pack into a runbook session. + + TODO: integrate with existing runbook creation once Module 1 is wired. + """ + pack = await asyncio.to_thread(_store.get, pack_id) + if pack is None: + Logger.warning(f"[M2] session_pack_not_found pack_id={pack_id}", file=LogFiles.API) + raise HTTPException(status_code=404, detail="Context pack not found.") + Logger.info( + f"[M2] create_session pack_id={pack_id} executor={request.executor_preference}", + file=LogFiles.API, + ) + + session_id = f"sess_{uuid.uuid4().hex[:16]}" + runbook_id = f"rb_{uuid.uuid4().hex[:12]}" + + roadmap = pack.get("task_roadmap", []) + initial_steps = [ + { + "step_id": f"S{i + 1}", + "title": cp.get("title", f"Step {i + 1}"), + "command": "", + "status": "pending", + } + for i, cp in enumerate(roadmap) + ] or [{"step_id": "S1", "title": "Setup environment", "command": "", "status": "pending"}] + + return { + "session_id": session_id, + "runbook_id": runbook_id, + "initial_steps": initial_steps, + "initial_prompt": ( + f"Based on the reproduction context pack for paper {pack.get('paper_id', '')}, " + "implement the code step by step following the roadmap." + ), + } + + +# ------------------------------------------------------------------ # +# DELETE /{pack_id} # +# ------------------------------------------------------------------ # + +@router.delete("/{pack_id}") +async def delete_context_pack(pack_id: str): + """Soft-delete a context pack.""" + set_trace_id() + Logger.info(f"[M2] delete_pack pack_id={pack_id}", file=LogFiles.API) + deleted = await asyncio.to_thread(_store.soft_delete, pack_id) + if not deleted: + Logger.warning(f"[M2] delete_pack_not_found pack_id={pack_id}", file=LogFiles.API) + raise HTTPException(status_code=404, detail="Context pack not found.") + return {"status": "deleted", "context_pack_id": pack_id} diff --git a/src/paperbot/api/routes/runbook.py b/src/paperbot/api/routes/runbook.py index 3750a62f..afe1a0da 100644 --- a/src/paperbot/api/routes/runbook.py +++ b/src/paperbot/api/routes/runbook.py @@ -9,6 +9,7 @@ from __future__ import annotations +import fcntl import hashlib import json import os @@ -30,14 +31,87 @@ Base.metadata.create_all(_provider.engine) -def _allowed_workdir(workdir: Path) -> bool: - allowed_prefixes = [Path(tempfile.gettempdir()).resolve()] +def _runtime_allowed_dirs_file() -> Path: + return Path("data/runbook_allowed_dirs.json") + + +def _load_runtime_allowed_dirs() -> List[Path]: + f = _runtime_allowed_dirs_file() + if not f.exists(): + return [] + try: + data = json.loads(f.read_text(encoding="utf-8")) + if isinstance(data, list): + return [Path(p).resolve() for p in data if isinstance(p, str) and p.strip()] + except (OSError, json.JSONDecodeError, ValueError): + pass + return [] + + +def _save_runtime_allowed_dir(dir_path: Path) -> None: + f = _runtime_allowed_dirs_file() + f.parent.mkdir(parents=True, exist_ok=True) + resolved = dir_path.resolve() + + # Use file lock to prevent concurrent read-modify-write races + lock_path = f.with_suffix(".lock") + with open(lock_path, "w") as lock_fd: + fcntl.flock(lock_fd, fcntl.LOCK_EX) + try: + existing = _load_runtime_allowed_dirs() + if resolved not in existing: + existing.append(resolved) + # Atomic write via temp file + rename + tmp_fd, tmp_path = tempfile.mkstemp( + dir=str(f.parent), suffix=".tmp", prefix=".allowed_dirs_" + ) + try: + with os.fdopen(tmp_fd, "w", encoding="utf-8") as tmp_f: + json.dump( + [str(p) for p in existing], tmp_f, ensure_ascii=False, indent=2 + ) + os.replace(tmp_path, str(f)) + except BaseException: + # Clean up temp file on failure + try: + os.unlink(tmp_path) + except OSError: + pass + raise + finally: + fcntl.flock(lock_fd, fcntl.LOCK_UN) + + +def _allowed_workdir_prefixes() -> List[Path]: + prefixes: List[Path] = [Path(tempfile.gettempdir()).resolve()] + try: + prefixes.append(Path.cwd().resolve()) + except Exception: + pass + extra = os.getenv("PAPERBOT_RUNBOOK_ALLOW_DIR_PREFIXES", "").strip() if extra: for p in extra.split(","): p = p.strip() if p: - allowed_prefixes.append(Path(p).expanduser().resolve()) + prefixes.append(Path(p).expanduser().resolve()) + + prefixes.extend(_load_runtime_allowed_dirs()) + + # Preserve order and deduplicate + unique: List[Path] = [] + seen = set() + for prefix in prefixes: + key = str(prefix) + if key in seen: + continue + seen.add(key) + unique.append(prefix) + return unique + + +def _allowed_workdir(workdir: Path) -> bool: + allowed_prefixes = _allowed_workdir_prefixes() try: resolved = workdir.resolve() @@ -54,6 +128,99 @@ def _allowed_workdir(workdir: Path) -> bool: return False +class PrepareProjectDirRequest(BaseModel): + project_dir: str + create_if_missing: bool = True + + +@router.post("/runbook/project-dir/prepare") +async def prepare_project_dir(body: PrepareProjectDirRequest): + """ + Validate and normalize a project directory for runbook operations. + + If the path is under allowed prefixes and does not exist yet, this endpoint + can create it to make Studio directory selection smoother. + """ + requested = Path(body.project_dir).expanduser() + try: + resolved = requested.resolve(strict=False) + except Exception: + raise HTTPException(status_code=400, detail="invalid project_dir") + + if not _allowed_workdir(resolved): + raise HTTPException(status_code=403, detail="project_dir is not allowed") + + created = False + if resolved.exists() and not resolved.is_dir(): + raise HTTPException(status_code=400, detail="project_dir must be a directory") + if not resolved.exists(): + if not body.create_if_missing: + raise HTTPException(status_code=400, detail="project_dir must be an existing directory") + try: + resolved.mkdir(parents=True, exist_ok=True) + created = True + except Exception as exc: + raise HTTPException(status_code=400, detail=f"failed to create project_dir: {exc}") from exc + + return { + "project_dir": str(resolved), + "created": created, + "allowed_prefixes": [str(p) for p in _allowed_workdir_prefixes()], + } + + +class AddAllowedDirRequest(BaseModel): + directory: str + + +def _runtime_allowlist_mutation_enabled() -> bool: + return os.getenv("PAPERBOT_RUNBOOK_ALLOWLIST_MUTATION", "false").lower() == "true" + + +# Directories that must never be added to the allowlist (too broad / sensitive). +_DENIED_PATHS = frozenset({ + "/", "/bin", "/sbin", "/usr", "/etc", "/var", "/root", "/sys", "/proc", + "/dev", "/boot", "/lib", "/lib64", "/opt", +}) + + +@router.post("/runbook/allowed-dirs") +async def add_allowed_dir(body: AddAllowedDirRequest): + """Add a directory to the runtime-allowed prefixes list.""" + if not _runtime_allowlist_mutation_enabled(): + raise HTTPException( + status_code=403, + detail="runtime allowlist mutation is disabled" + ) + + try: + resolved = Path(body.directory).expanduser().resolve() + except Exception: + raise HTTPException(status_code=400, detail="invalid directory path") + + if str(resolved) in _DENIED_PATHS: + raise HTTPException( + status_code=403, + detail=f"adding '{resolved}' is not allowed — path is too broad or sensitive" + ) + + if not resolved.exists() or not resolved.is_dir(): + raise HTTPException(status_code=400, detail="directory does not exist or is not a directory") + + _save_runtime_allowed_dir(resolved) + return { + "ok": True, + "directory": str(resolved), + "allowed_prefixes": [str(p) for p in _allowed_workdir_prefixes()], + } + + +@router.get("/runbook/allowed-dirs") +async def get_allowed_dirs(): + """Return all currently allowed workspace directory prefixes.""" + return {"prefixes": [str(p) for p in _allowed_workdir_prefixes()]} + + def _resolve_under_root(root: Path, relative_path: str) -> Path: """ Resolve a relative path within a root directory. diff --git a/src/paperbot/application/ports/repro_context_port.py b/src/paperbot/application/ports/repro_context_port.py new file mode 100644 index 00000000..33560c0d --- /dev/null +++ b/src/paperbot/application/ports/repro_context_port.py @@ -0,0 +1,74 @@ +"""ReproContextPort — interface for P2C context pack persistence.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Protocol, Tuple, runtime_checkable + + +@runtime_checkable +class ReproContextPort(Protocol): + """Abstract interface for ReproContextPack read/write operations.""" + + def save( + self, + *, + pack_id: str, + user_id: str, + paper_id: str, + depth: str, + pack_data: Dict[str, Any], + paper_title: Optional[str] = None, + project_id: Optional[str] = None, + objective: Optional[str] = None, + confidence_overall: float = 0.0, + warning_count: int = 0, + ) -> str: + """Persist a new context pack. Returns the pack_id.""" + ... + + def update_status( + self, + pack_id: str, + *, + status: str, + pack_data: Optional[Dict[str, Any]] = None, + confidence_overall: Optional[float] = None, + warning_count: Optional[int] = None, + objective: Optional[str] = None, + ) -> None: + """Update status (and optionally pack_json) of an existing pack.""" + ... + + def get(self, pack_id: str) -> Optional[Dict[str, Any]]: + """Return the full context pack dict, or None if not found / soft-deleted.""" + ... + + def list_by_user( + self, + *, + user_id: str, + paper_id: Optional[str] = None, + project_id: Optional[str] = None, + limit: int = 20, + offset: int = 0, + ) -> Tuple[List[Dict[str, Any]], int]: + """Return (summary_list, total_count) for the given user.""" + ... + + def soft_delete(self, pack_id: str) -> bool: + """Mark a pack as deleted. Returns True if the row was found and updated.""" + ... + + def save_stage_result( + self, + *, + pack_id: str, + stage_name: str, + status: str, + result_data: Optional[Dict[str, Any]] = None, + confidence: float = 0.0, + duration_ms: int = 0, + error_message: Optional[str] = None, + ) -> None: + """Persist an intermediate stage result for debugging / audit.""" + ... diff --git a/src/paperbot/application/services/__init__.py b/src/paperbot/application/services/__init__.py index a319fd2e..b3eac724 100644 --- a/src/paperbot/application/services/__init__.py +++ b/src/paperbot/application/services/__init__.py @@ -1,4 +1,5 @@ from paperbot.application.services.llm_service import LLMService, get_llm_service +from paperbot.application.services.p2c import ExtractionOrchestrator from paperbot.application.services.paper_deduplicator import PaperDeduplicator from paperbot.application.services.query_rewriter import QueryRewriter from paperbot.application.services.venue_recommender import VenueRecommender @@ -7,6 +8,7 @@ "LLMService", "get_llm_service", "PaperDeduplicator", + "ExtractionOrchestrator", "QueryRewriter", "VenueRecommender", ] diff --git a/src/paperbot/application/services/p2c/__init__.py b/src/paperbot/application/services/p2c/__init__.py new file mode 100644 index 00000000..3e8b6973 --- /dev/null +++ b/src/paperbot/application/services/p2c/__init__.py @@ -0,0 +1,63 @@ +from .benchmark import ( + BenchmarkCase, + aggregate_results, + evaluate_case, + load_benchmark_cases, + run_module1_benchmark, +) +from .evidence import EvidenceLinker, calibrate_confidence +from .input_pipeline import ( + ArXivAdapter, + LocalFileAdapter, + PaperInputAdapter, + PaperInputRouter, + PaperSectionExtractor, + PaperTypeClassifier, + SemanticScholarAdapter, +) +from .models import ( + ConfidenceScores, + EvidenceLink, + ExtractionObservation, + GenerateContextRequest, + NormalizedInput, + PaperIdentity, + PaperType, + RawPaperData, + ReproContextPack, + StageName, + StageResult, + TaskCheckpoint, +) +from .orchestrator import ExtractionOrchestrator, OrchestratorConfig + +__all__ = [ + "ArXivAdapter", + "BenchmarkCase", + "LocalFileAdapter", + "PaperInputAdapter", + "PaperInputRouter", + "PaperSectionExtractor", + "PaperTypeClassifier", + "SemanticScholarAdapter", + "aggregate_results", + "evaluate_case", + "load_benchmark_cases", + "run_module1_benchmark", + "EvidenceLinker", + "calibrate_confidence", + "ConfidenceScores", + "EvidenceLink", + "ExtractionObservation", + "GenerateContextRequest", + "NormalizedInput", + "PaperIdentity", + "PaperType", + "RawPaperData", + "ReproContextPack", + "StageName", + "StageResult", + "TaskCheckpoint", + "ExtractionOrchestrator", + "OrchestratorConfig", +] diff --git a/src/paperbot/application/services/p2c/benchmark.py b/src/paperbot/application/services/p2c/benchmark.py new file mode 100644 index 00000000..c7c2ec57 --- /dev/null +++ b/src/paperbot/application/services/p2c/benchmark.py @@ -0,0 +1,155 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence + +from .models import GenerateContextRequest, RawPaperData, ReproContextPack +from .orchestrator import ExtractionOrchestrator + + +@dataclass +class BenchmarkCase: + case_id: str + title: str + abstract: str = "" + full_text: str = "" + year: int = 0 + expected_metrics: List[str] = field(default_factory=list) + expected_hyperparams: List[str] = field(default_factory=list) + expected_architecture: Optional[str] = None + evidence_required_types: List[str] = field(default_factory=list) + + +def _f1_score(expected: Sequence[str], predicted: Sequence[str]) -> Dict[str, float]: + exp = {item.strip().lower() for item in expected if item} + pred = {item.strip().lower() for item in predicted if item} + if not exp and not pred: + return {"precision": 1.0, "recall": 1.0, "f1": 1.0} + if not exp: + return {"precision": 0.0, "recall": 1.0, "f1": 0.0} + if not pred: + return {"precision": 1.0, "recall": 0.0, "f1": 0.0} + + tp = len(exp & pred) + precision = tp / len(pred) + recall = tp / len(exp) + if precision + recall == 0: + f1 = 0.0 + else: + f1 = 2 * precision * recall / (precision + recall) + return {"precision": precision, "recall": recall, "f1": f1} + + +def load_benchmark_cases(path: str | Path) -> List[BenchmarkCase]: + fixture_path = Path(path) + payload = json.loads(fixture_path.read_text(encoding="utf-8")) + cases: List[BenchmarkCase] = [] + for row in payload: + cases.append( + BenchmarkCase( + case_id=str(row.get("case_id") or ""), + title=str(row.get("title") or ""), + abstract=str(row.get("abstract") or ""), + full_text=str(row.get("full_text") or ""), + year=int(row.get("year") or 0), + expected_metrics=list(row.get("expected", {}).get("metrics", []) or []), + expected_hyperparams=list(row.get("expected", {}).get("hyperparameters", []) or []), + expected_architecture=(row.get("expected", {}) or {}).get("architecture"), + evidence_required_types=list( + row.get("expected", {}).get("evidence_required_types", []) or [] + ), + ) + ) + return cases + + +def evaluate_case(case: BenchmarkCase, pack: ReproContextPack) -> Dict[str, Any]: + metric_candidates: List[str] = [] + for obs in pack.get_by_type("metric"): + metric_candidates.extend((obs.structured_data.get("metrics") or [])) + metric_scores = _f1_score(case.expected_metrics, metric_candidates) + + hyperparam_candidates: List[str] = [] + for obs in pack.get_by_type("hyperparameter"): + hyperparam_candidates.extend((obs.structured_data or {}).keys()) + hyperparam_scores = _f1_score(case.expected_hyperparams, hyperparam_candidates) + + architecture = "" + architecture_obs = pack.get_by_type("architecture") + if architecture_obs: + architecture = str(architecture_obs[0].structured_data.get("architecture_type") or "") + + architecture_hit = ( + 1.0 + if not case.expected_architecture + else float(architecture.lower() == str(case.expected_architecture).lower()) + ) + + required = case.evidence_required_types + if required: + evidence_hit_count = 0 + for obs_type in required: + observations = pack.get_by_type(obs_type) + if any(item.evidence for item in observations): + evidence_hit_count += 1 + evidence_hit_rate = evidence_hit_count / len(required) + else: + evidence_hit_rate = 1.0 + + return { + "case_id": case.case_id, + "metric_f1": metric_scores["f1"], + "hyperparam_f1": hyperparam_scores["f1"], + "architecture_hit": architecture_hit, + "evidence_hit_rate": evidence_hit_rate, + "warnings_count": len(pack.warnings), + } + + +def aggregate_results(case_results: Sequence[Dict[str, Any]]) -> Dict[str, float]: + if not case_results: + return { + "metric_f1": 0.0, + "hyperparam_f1": 0.0, + "architecture_hit_rate": 0.0, + "evidence_hit_rate": 0.0, + "avg_warnings": 0.0, + } + + n = float(len(case_results)) + return { + "metric_f1": sum(float(row["metric_f1"]) for row in case_results) / n, + "hyperparam_f1": sum(float(row["hyperparam_f1"]) for row in case_results) / n, + "architecture_hit_rate": sum(float(row["architecture_hit"]) for row in case_results) / n, + "evidence_hit_rate": sum(float(row["evidence_hit_rate"]) for row in case_results) / n, + "avg_warnings": sum(float(row["warnings_count"]) for row in case_results) / n, + } + + +async def run_module1_benchmark( + cases: Sequence[BenchmarkCase], + *, + orchestrator: Optional[ExtractionOrchestrator] = None, +) -> Dict[str, Any]: + orch = orchestrator or ExtractionOrchestrator() + case_results: List[Dict[str, Any]] = [] + for case in cases: + request = GenerateContextRequest(paper_id=f"benchmark:{case.case_id}", depth="standard") + raw_paper = RawPaperData( + paper_id=request.paper_id, + title=case.title, + abstract=case.abstract, + year=case.year, + full_text=case.full_text, + source_adapter="benchmark_fixture", + ) + pack = await orch.run(request, raw_paper=raw_paper) + case_results.append(evaluate_case(case, pack)) + + summary = aggregate_results(case_results) + return { + "cases": case_results, + "summary": summary, + } diff --git a/src/paperbot/application/services/p2c/evidence.py b/src/paperbot/application/services/p2c/evidence.py new file mode 100644 index 00000000..126d25de --- /dev/null +++ b/src/paperbot/application/services/p2c/evidence.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import re +from typing import Iterable, List, Sequence + +from .models import EvidenceLink + + +def _clamp(value: float, lo: float = 0.0, hi: float = 1.0) -> float: + return max(lo, min(hi, float(value))) + + +class EvidenceLinker: + """Utility for binding extracted claims to text spans.""" + + def find_keyword_evidence( + self, + text: str, + *, + keywords: Sequence[str], + section: str, + supports: Sequence[str], + max_links: int = 3, + ) -> List[EvidenceLink]: + evidence: List[EvidenceLink] = [] + if not text: + return evidence + + for keyword in keywords: + pattern = re.compile(rf"\b{re.escape(keyword)}\b", re.IGNORECASE) + for match in pattern.finditer(text): + evidence.append( + EvidenceLink( + type="paper_span", + ref=f"{section}#char:{match.start()}-{match.end()}", + supports=list(supports), + confidence=0.82, + ) + ) + if len(evidence) >= max_links: + return evidence + return evidence + + def from_match( + self, + *, + section: str, + supports: Sequence[str], + start: int, + end: int, + confidence: float = 0.84, + ) -> EvidenceLink: + return EvidenceLink( + type="paper_span", + ref=f"{section}#char:{start}-{end}", + supports=list(supports), + confidence=_clamp(confidence), + ) + + def section_anchor( + self, + text: str, + *, + section: str, + supports: Sequence[str], + max_chars: int = 180, + ) -> List[EvidenceLink]: + cleaned = " ".join(text.split()) + if not cleaned: + return [] + end = min(len(cleaned), max_chars) + return [ + EvidenceLink( + type="paper_span", + ref=f"{section}#char:0-{end}", + supports=list(supports), + confidence=0.75, + ) + ] + + +def calibrate_confidence( + base_confidence: float, + evidence: Iterable[EvidenceLink], + *, + required: bool = False, +) -> float: + links = list(evidence) + base = _clamp(base_confidence) + if not links: + penalty = 0.55 if required else 0.8 + return _clamp(base * penalty) + + quality = sum(_clamp(link.confidence) for link in links) / len(links) + multiplier = 0.75 + (0.25 * quality) + if required and quality < 0.6: + multiplier *= 0.9 + return _clamp(base * multiplier) diff --git a/src/paperbot/application/services/p2c/input_pipeline.py b/src/paperbot/application/services/p2c/input_pipeline.py new file mode 100644 index 00000000..1812923c --- /dev/null +++ b/src/paperbot/application/services/p2c/input_pipeline.py @@ -0,0 +1,527 @@ +from __future__ import annotations + +import os +import re +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Awaitable, Callable, Iterable, List, Optional + +import aiohttp + +from paperbot.domain.paper_identity import normalize_arxiv_id, normalize_doi +from paperbot.infrastructure.api_clients.semantic_scholar import SemanticScholarClient +from paperbot.infrastructure.connectors.arxiv_connector import ArxivConnector + +from paperbot.utils.logging_config import Logger, LogFiles + +from .models import NormalizedInput, PaperIdentity, PaperType, RawPaperData + +_HEX40_RE = re.compile(r"^[0-9a-f]{40}$", re.IGNORECASE) + + +class PaperInputAdapter(ABC): + name: str = "" + + @abstractmethod + def can_handle(self, paper_id: str) -> bool: + raise NotImplementedError + + @abstractmethod + async def fetch(self, paper_id: str) -> RawPaperData: + raise NotImplementedError + + +class SemanticScholarAdapter(PaperInputAdapter): + name = "semantic_scholar" + + _DEFAULT_FIELDS = [ + "paperId", + "title", + "abstract", + "year", + "authors", + "externalIds", + ] + + def __init__( + self, + *, + client: Optional[SemanticScholarClient] = None, + api_key: Optional[str] = None, + ): + self._client = client + self._api_key = api_key or os.getenv("SEMANTIC_SCHOLAR_API_KEY") or os.getenv("S2_API_KEY") + + def can_handle(self, paper_id: str) -> bool: + pid = paper_id.strip() + pid_upper = pid.upper() + return ( + pid.startswith("s2:") + or bool(_HEX40_RE.match(pid)) + or pid_upper.startswith("DOI:") + or pid_upper.startswith("ARXIV:") + or pid_upper.startswith("CORPUSID:") + or normalize_doi(pid) is not None + ) + + @staticmethod + def _normalize_lookup_id(paper_id: str) -> str: + pid = paper_id.strip() + if pid.startswith("s2:"): + return pid.split(":", 1)[1].strip() + + pid_upper = pid.upper() + if pid_upper.startswith(("DOI:", "ARXIV:", "CORPUSID:")): + prefix, value = pid.split(":", 1) + return f"{prefix.upper()}:{value.strip()}" + + doi = normalize_doi(pid) + if doi: + return f"DOI:{doi}" + + arxiv_id = normalize_arxiv_id(pid) + if arxiv_id: + return f"ARXIV:{arxiv_id}" + + return pid + + async def fetch(self, paper_id: str) -> RawPaperData: + lookup_id = self._normalize_lookup_id(paper_id) + client = self._client + owns_client = False + if client is None: + client = SemanticScholarClient(api_key=self._api_key) + owns_client = True + + try: + record = await client.get_paper(lookup_id, fields=self._DEFAULT_FIELDS) + finally: + if owns_client: + await client.close() + + if not record: + raise ValueError(f"Semantic Scholar paper not found: {lookup_id}") + + authors = [ + str(author.get("name")).strip() + for author in (record.get("authors") or []) + if isinstance(author, dict) and author.get("name") + ] + + year = 0 + year_raw = record.get("year") + if year_raw is not None: + try: + year = int(year_raw) + except (TypeError, ValueError): + year = 0 + + external_ids = ( + record.get("externalIds") if isinstance(record.get("externalIds"), dict) else {} + ) + identifiers: dict[str, str] = {} + paper_id_value = str(record.get("paperId") or "").strip() + if paper_id_value: + identifiers["semantic_scholar"] = paper_id_value + + doi = normalize_doi(external_ids.get("DOI")) + if doi: + identifiers["doi"] = doi + + arxiv_id = normalize_arxiv_id(external_ids.get("ArXiv") or external_ids.get("ARXIV")) + if arxiv_id: + identifiers["arxiv"] = arxiv_id + + normalized_paper_id = f"s2:{paper_id_value}" if paper_id_value else lookup_id + return RawPaperData( + paper_id=normalized_paper_id, + title=str(record.get("title") or "").strip(), + abstract=str(record.get("abstract") or "").strip(), + authors=authors, + year=year, + identifiers=identifiers, + source_adapter=self.name, + ) + + +class ArXivAdapter(PaperInputAdapter): + name = "arxiv" + API_URL = "https://export.arxiv.org/api/query" + + def __init__( + self, + *, + connector: Optional[ArxivConnector] = None, + timeout_seconds: float = 30.0, + fetch_atom_xml: Optional[Callable[[str], Awaitable[str]]] = None, + ): + self._connector = connector or ArxivConnector(timeout_s=timeout_seconds) + self._timeout_seconds = timeout_seconds + self._fetch_atom_xml_override = fetch_atom_xml + + def can_handle(self, paper_id: str) -> bool: + return normalize_arxiv_id(paper_id) is not None + + async def _fetch_atom_xml(self, arxiv_id: str) -> str: + if self._fetch_atom_xml_override is not None: + return await self._fetch_atom_xml_override(arxiv_id) + + params = { + "id_list": arxiv_id, + "max_results": 1, + } + timeout = aiohttp.ClientTimeout(total=self._timeout_seconds) + headers = {"User-Agent": "PaperBot/2.0"} + async with aiohttp.ClientSession(timeout=timeout, headers=headers) as session: + async with session.get(self.API_URL, params=params) as response: + if response.status != 200: + body = await response.text() + raise RuntimeError(f"arXiv API error {response.status}: {body[:200]}") + return await response.text() + + async def fetch(self, paper_id: str) -> RawPaperData: + arxiv_id = normalize_arxiv_id(paper_id) + if not arxiv_id: + raise ValueError(f"Invalid arXiv paper id: {paper_id}") + + xml_text = await self._fetch_atom_xml(arxiv_id) + records = self._connector.parse_atom(xml_text) + if not records: + raise ValueError(f"arXiv paper not found: {arxiv_id}") + + record = records[0] + year = 0 + published = str(record.published or "").strip() + if len(published) >= 4: + try: + year = int(published[:4]) + except ValueError: + year = 0 + + normalized_id = normalize_arxiv_id(record.arxiv_id) or arxiv_id + return RawPaperData( + paper_id=f"arxiv:{normalized_id}", + title=str(record.title or "").strip(), + abstract=str(record.summary or "").strip(), + authors=[name.strip() for name in record.authors if str(name).strip()], + year=year, + identifiers={"arxiv": normalized_id}, + source_adapter=self.name, + ) + + +class LocalFileAdapter(PaperInputAdapter): + """Read a local .txt/.md file as paper input. + + Restricted to explicit allowed directories to prevent path-traversal attacks. + """ + + name = "local_file" + + def __init__(self, allowed_dirs: Optional[Iterable[str]] = None) -> None: + self._allowed_dirs: List[Path] = [ + Path(d).resolve() for d in (allowed_dirs or []) + ] + + def _is_allowed(self, path: Path) -> bool: + try: + resolved = path.resolve() + except (OSError, ValueError): + return False + return any( + resolved == d or str(resolved).startswith(str(d) + os.sep) + for d in self._allowed_dirs + ) + + def can_handle(self, paper_id: str) -> bool: + path = Path(paper_id).expanduser() + return self._is_allowed(path) and path.exists() and path.is_file() + + async def fetch(self, paper_id: str) -> RawPaperData: + path = Path(paper_id).expanduser() + if not self._is_allowed(path): + raise ValueError(f"Path not in allowed directories: {paper_id}") + text = "" + if path.suffix.lower() in {".txt", ".md"}: + text = path.read_text(encoding="utf-8", errors="ignore") + + return RawPaperData( + paper_id=str(path), + title=path.stem, + full_text=text or None, + source_adapter=self.name, + ) + + +class InternalPaperStoreAdapter(PaperInputAdapter): + name = "paper_store" + + def __init__(self) -> None: + from paperbot.infrastructure.stores.paper_store import PaperStore + + self._store = PaperStore() + + def can_handle(self, paper_id: str) -> bool: + return paper_id.strip().isdigit() + + async def fetch(self, paper_id: str) -> RawPaperData: + + + paper_db_id = int(paper_id) + row = self._store.get_paper_by_id(paper_db_id) + if row is None: + Logger.error( + f"[M1] paper_store_not_found paper_id={paper_id}", + file=LogFiles.ERROR, + ) + raise ValueError(f"Paper not found in store: {paper_id}") + + identifiers: dict[str, str] = {} + if row.semantic_scholar_id: + identifiers["semantic_scholar"] = row.semantic_scholar_id + if row.arxiv_id: + identifiers["arxiv"] = row.arxiv_id + if row.doi: + identifiers["doi"] = row.doi + if row.openalex_id: + identifiers["openalex"] = row.openalex_id + + Logger.info( + f"[M1] paper_store_fetch paper_id={paper_id} title={row.title}", + file=LogFiles.API, + ) + + return RawPaperData( + paper_id=str(row.id), + title=row.title or "", + abstract=row.abstract or "", + authors=row.get_authors(), + year=int(row.year or 0), + identifiers=identifiers, + source_adapter=self.name, + ) + + +class PaperInputRouter: + def __init__(self, adapters: Optional[Iterable[PaperInputAdapter]] = None): + self.adapters: List[PaperInputAdapter] = list( + adapters + or [ + InternalPaperStoreAdapter(), + ArXivAdapter(), + SemanticScholarAdapter(), + ] + ) + + async def fetch(self, paper_id: str) -> RawPaperData: + + + Logger.info( + f"[M1] input_router_fetch paper_id={paper_id} adapters={[a.name for a in self.adapters]}", + file=LogFiles.API, + ) + for adapter in self.adapters: + if adapter.can_handle(paper_id): + Logger.info( + f"[M1] adapter_selected paper_id={paper_id} adapter={adapter.name}", + file=LogFiles.API, + ) + return await adapter.fetch(paper_id) + Logger.error( + f"[M1] no_adapter_found paper_id={paper_id} adapters={[a.name for a in self.adapters]}", + file=LogFiles.ERROR, + ) + raise ValueError(f"No adapter can handle paper id: {paper_id}") + + +class PaperSectionExtractor: + """Heuristic section extractor for module1 baseline.""" + + _heading_patterns = { + "introduction": ["introduction", "background", "related work"], + "method": ["method", "methods", "approach", "model", "methodology"], + "results": [ + "results", + "experiments", + "experimental setup", + "evaluation", + "analysis", + ], + "discussion": ["discussion", "conclusion", "future work", "limitations"], + } + _heading_prefix_re = re.compile(r"^\s*(?:\d+(?:\.\d+)*|[ivxlcdm]+)[\).\s:-]+", re.IGNORECASE) + + def _normalize_heading_text(self, line: str) -> str: + normalized = line.strip().lower() + normalized = self._heading_prefix_re.sub("", normalized) + normalized = re.sub(r"\s+", " ", normalized).strip(" :.-") + return normalized + + def _is_heading(self, line: str) -> Optional[str]: + original = line.strip() + if not original: + return None + if len(original) > 120: + return None + if original.endswith((".", "?", "!")) and ":" not in original: + return None + if len(original.split()) > 8: + return None + + normalized = self._normalize_heading_text(line) + if not normalized: + return None + for section, aliases in self._heading_patterns.items(): + for alias in aliases: + if ( + normalized == alias + or normalized.startswith(f"{alias} and ") + or normalized.endswith(f" {alias}") + ): + return section + return None + + @staticmethod + def _merge_offset_range( + offsets: dict[str, tuple[int, int]], + *, + section: str, + start: int, + end: int, + ) -> None: + if start >= end: + return + if section in offsets: + old_start, old_end = offsets[section] + offsets[section] = (min(old_start, start), max(old_end, end)) + else: + offsets[section] = (start, end) + + def _split_sections(self, text: str) -> tuple[dict[str, str], dict[str, tuple[int, int]]]: + sections: dict[str, List[str]] = {k: [] for k in self._heading_patterns.keys()} + offsets: dict[str, tuple[int, int]] = {} + current: Optional[str] = None + current_start: Optional[int] = None + cursor = 0 + + for raw_line in text.splitlines(keepends=True): + line = raw_line.strip() + heading = self._is_heading(line) + if heading: + if current and current_start is not None: + self._merge_offset_range( + offsets, + section=current, + start=current_start, + end=cursor, + ) + current = heading + current_start = cursor + len(raw_line) + cursor += len(raw_line) + continue + if current: + sections[current].append(raw_line.rstrip("\n")) + cursor += len(raw_line) + + if current and current_start is not None: + self._merge_offset_range( + offsets, + section=current, + start=current_start, + end=cursor, + ) + + compact = {name: "\n".join(lines).strip() for name, lines in sections.items() if lines} + offsets = {name: span for name, span in offsets.items() if name in compact} + return compact, offsets + + def _segment_noisy_text(self, text: str) -> tuple[dict[str, str], dict[str, tuple[int, int]]]: + lowered = text.lower() + anchors: list[tuple[int, int, str]] = [] + + for section, aliases in self._heading_patterns.items(): + best_match: Optional[tuple[int, int, str]] = None + for alias in aliases: + match = re.search(rf"\b{re.escape(alias)}\b", lowered) + if match: + candidate = (match.start(), match.end(), section) + if best_match is None or candidate[0] < best_match[0]: + best_match = candidate + if best_match: + anchors.append(best_match) + + if len(anchors) < 2: + return {}, {} + + anchors.sort(key=lambda item: item[0]) + sections: dict[str, str] = {} + offsets: dict[str, tuple[int, int]] = {} + for idx, (_, anchor_end, section) in enumerate(anchors): + next_start = anchors[idx + 1][0] if idx + 1 < len(anchors) else len(text) + snippet = text[anchor_end:next_start].strip() + if not snippet: + continue + if section in sections: + sections[section] = f"{sections[section]}\n{snippet}".strip() + old_start, _ = offsets[section] + offsets[section] = (old_start, next_start) + else: + sections[section] = snippet + offsets[section] = (anchor_end, next_start) + return sections, offsets + + async def extract(self, raw: RawPaperData) -> NormalizedInput: + full_text = (raw.full_text or "").strip() + abstract = raw.abstract.strip() + + sections, section_offsets = self._split_sections(full_text) + if not sections and full_text: + sections, section_offsets = self._segment_noisy_text(full_text) + + if not sections and full_text: + # Last fallback: keep a best-effort method slice. + method_hint = re.search( + r"(method|approach|model|methodology)([\s\S]{0,1200})", + full_text, + flags=re.IGNORECASE, + ) + if method_hint: + sections["method"] = method_hint.group(2).strip() + section_offsets["method"] = (method_hint.start(2), method_hint.end(2)) + + identity = PaperIdentity( + paper_id=raw.paper_id, + title=raw.title, + year=raw.year, + authors=list(raw.authors), + identifiers=dict(raw.identifiers), + ) + + return NormalizedInput( + paper=identity, + abstract=abstract, + full_text=full_text, + sections=sections, + section_offsets=section_offsets, + ) + + +class PaperTypeClassifier: + def classify(self, normalized: NormalizedInput) -> PaperType: + text = " ".join( + [ + normalized.paper.title.lower(), + normalized.abstract.lower(), + normalized.sections.get("method", "").lower(), + ] + ) + + if any(token in text for token in ["survey", "systematic review", "overview"]): + return PaperType.SURVEY + if any(token in text for token in ["theorem", "proof", "bound", "convergence"]): + return PaperType.THEORETICAL + if any(token in text for token in ["benchmark", "leaderboard", "sota"]): + return PaperType.BENCHMARK + if any(token in text for token in ["system", "platform", "serving", "deployment"]): + return PaperType.SYSTEM + return PaperType.EXPERIMENTAL diff --git a/src/paperbot/application/services/p2c/models.py b/src/paperbot/application/services/p2c/models.py new file mode 100644 index 00000000..e4b3e86f --- /dev/null +++ b/src/paperbot/application/services/p2c/models.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from enum import Enum +from typing import Any, Dict, List, Literal, Optional, Sequence +from uuid import uuid4 + + +class StageName(str, Enum): + LITERATURE_DISTILL = "literature_distill" + BLUEPRINT_EXTRACT = "blueprint_extract" + ENVIRONMENT_EXTRACT = "environment_extract" + SPEC_EXTRACT = "spec_extract" + ROADMAP_PLANNING = "roadmap_planning" + SUCCESS_CRITERIA = "success_criteria" + + +class PaperType(str, Enum): + EXPERIMENTAL = "experimental" + THEORETICAL = "theoretical" + SURVEY = "survey" + BENCHMARK = "benchmark" + SYSTEM = "system" + + +Depth = Literal["fast", "standard", "deep"] +Source = Literal["manual", "recommendation", "collection"] +Difficulty = Literal["low", "medium", "high"] +EvidenceType = Literal["paper_span", "table", "figure", "code_snippet", "metadata"] + + +@dataclass +class GenerateContextRequest: + """P2C pipeline entry request.""" + + paper_id: str + user_id: str = "default" + project_id: Optional[str] = None + track_id: Optional[int] = None + depth: Depth = "standard" + source: Source = "manual" + + +@dataclass +class PaperIdentity: + paper_id: str = "" + title: str = "" + year: int = 0 + authors: List[str] = field(default_factory=list) + identifiers: Dict[str, str] = field(default_factory=dict) + + +@dataclass +class TaskCheckpoint: + id: str + title: str + description: str = "" + acceptance_criteria: List[str] = field(default_factory=list) + depends_on: List[str] = field(default_factory=list) + estimated_difficulty: Difficulty = "medium" + + +@dataclass +class EvidenceLink: + type: EvidenceType + ref: str + supports: List[str] = field(default_factory=list) + confidence: float = 0.0 + + +@dataclass +class ConfidenceScores: + overall: float = 0.0 + literature: float = 0.0 + blueprint: float = 0.0 + environment: float = 0.0 + spec: float = 0.0 + roadmap: float = 0.0 + metrics: float = 0.0 + + +@dataclass +class ExtractionObservation: + """ + Structured extraction record inspired by observation-style memory models. + """ + + id: str + stage: str + type: str + title: str + narrative: str + structured_data: Dict[str, Any] = field(default_factory=dict) + evidence: List[EvidenceLink] = field(default_factory=list) + confidence: float = 0.0 + concepts: List[str] = field(default_factory=list) + + def to_compact(self, max_tokens: int = 100) -> str: + max_chars = max(1, max_tokens) * 4 + narrative = self.narrative.strip() + if len(narrative) > max_chars: + narrative = narrative[: max_chars - 3].rstrip() + "..." + return f"[{self.type}] {self.title} (conf={self.confidence:.0%}): {narrative}" + + def to_full(self) -> Dict[str, Any]: + return { + "id": self.id, + "stage": self.stage, + "type": self.type, + "title": self.title, + "narrative": self.narrative, + "structured_data": self.structured_data, + "evidence": [asdict(item) for item in self.evidence], + "confidence": self.confidence, + "concepts": self.concepts, + } + + +def new_observation_id() -> str: + return f"obs_{uuid4().hex[:12]}" + + +def new_context_pack_id() -> str: + return f"ctxp_{uuid4().hex[:12]}" + + +@dataclass +class ReproContextPack: + context_pack_id: str + version: str = "v2" + created_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) + paper: PaperIdentity = field(default_factory=PaperIdentity) + paper_type: PaperType = PaperType.EXPERIMENTAL + objective: str = "" + observations: List[ExtractionObservation] = field(default_factory=list) + task_roadmap: List[TaskCheckpoint] = field(default_factory=list) + confidence: ConfidenceScores = field(default_factory=ConfidenceScores) + warnings: List[str] = field(default_factory=list) + + def get_by_stage(self, stage: str) -> List[ExtractionObservation]: + return [obs for obs in self.observations if obs.stage == stage] + + def get_by_type(self, type_name: str) -> List[ExtractionObservation]: + return [obs for obs in self.observations if obs.type == type_name] + + def get_by_concept(self, concept: str) -> List[ExtractionObservation]: + return [obs for obs in self.observations if concept in obs.concepts] + + def to_compact_context(self, max_tokens: int = 2000) -> str: + concept_priority = [ + "core_method", + "architecture", + "hyperparameter", + "environment", + "gotcha", + ] + + lines = [f"# Reproduction Context: {self.paper.title} ({self.paper.year})"] + if self.objective: + lines.append(f"Objective: {self.objective}") + + budget = max(1, max_tokens) * 4 + used_ids: set[str] = set() + + for concept in concept_priority: + candidates = sorted( + self.get_by_concept(concept), + key=lambda item: item.confidence, + reverse=True, + ) + for obs in candidates: + if obs.id in used_ids: + continue + compact = obs.to_compact(max_tokens=80) + if len(compact) > budget: + return "\n".join(lines) + lines.append(compact) + used_ids.add(obs.id) + budget -= len(compact) + + remaining = sorted(self.observations, key=lambda item: item.confidence, reverse=True) + for obs in remaining: + if obs.id in used_ids: + continue + compact = obs.to_compact(max_tokens=60) + if len(compact) > budget: + break + lines.append(compact) + budget -= len(compact) + + return "\n".join(lines) + + def to_execution_prompt( + self, executor: Literal["claude_code", "codex", "local"] = "claude_code" + ) -> str: + prompt = self.to_compact_context(max_tokens=3000) + if self.task_roadmap: + prompt += "\n\n## Roadmap\n" + for step in self.task_roadmap: + acceptance = ( + ", ".join(step.acceptance_criteria) if step.acceptance_criteria else "none" + ) + prompt += f"- {step.id}: {step.title} (acceptance: {acceptance})\n" + + if executor == "codex": + prompt += "\nFocus on concrete file-level steps and verifiable checkpoints.\n" + elif executor == "claude_code": + prompt += "\nExecute step-by-step and surface blockers early.\n" + return prompt + + +@dataclass +class RawPaperData: + paper_id: str + title: str = "" + abstract: str = "" + authors: List[str] = field(default_factory=list) + year: int = 0 + identifiers: Dict[str, str] = field(default_factory=dict) + full_text: Optional[str] = None + source_adapter: str = "" + + +@dataclass +class NormalizedInput: + paper: PaperIdentity + abstract: str = "" + full_text: str = "" + sections: Dict[str, str] = field(default_factory=dict) + section_offsets: Dict[str, tuple[int, int]] = field(default_factory=dict) + + @property + def method_section(self) -> str: + return self.sections.get("method", "") + + +@dataclass +class StageResult: + observations: List[ExtractionObservation] = field(default_factory=list) + roadmap: List[TaskCheckpoint] = field(default_factory=list) + warnings: List[str] = field(default_factory=list) + + +def stage_mean_confidence(items: Sequence[ExtractionObservation]) -> float: + if not items: + return 0.0 + return float(sum(max(0.0, min(1.0, item.confidence)) for item in items) / len(items)) diff --git a/src/paperbot/application/services/p2c/orchestrator.py b/src/paperbot/application/services/p2c/orchestrator.py new file mode 100644 index 00000000..9d68768d --- /dev/null +++ b/src/paperbot/application/services/p2c/orchestrator.py @@ -0,0 +1,308 @@ +from __future__ import annotations + +import inspect +import logging +from dataclasses import dataclass +from typing import Awaitable, Callable, Dict, Iterable, List, Optional, Protocol, TYPE_CHECKING + +from .input_pipeline import PaperInputRouter, PaperSectionExtractor, PaperTypeClassifier + +if TYPE_CHECKING: + from paperbot.application.services.llm_service import LLMService +from .models import ( + ConfidenceScores, + Depth, + GenerateContextRequest, + NormalizedInput, + PaperType, + RawPaperData, + ReproContextPack, + StageName, + StageResult, + new_context_pack_id, + stage_mean_confidence, +) +from .stages import ( + BlueprintExtractStage, + EnvironmentExtractStage, + LiteratureDistillStage, + RoadmapPlanningStage, + SpecExtractStage, + StageInput, + SuccessCriteriaStage, +) + + +class P2CStage(Protocol): + name: str + + async def run(self, data: StageInput) -> StageResult: ... + + +_STAGE_TO_CONFIDENCE_FIELD = { + StageName.LITERATURE_DISTILL.value: "literature", + StageName.BLUEPRINT_EXTRACT.value: "blueprint", + StageName.ENVIRONMENT_EXTRACT.value: "environment", + StageName.SPEC_EXTRACT.value: "spec", + StageName.ROADMAP_PLANNING.value: "roadmap", + StageName.SUCCESS_CRITERIA.value: "metrics", +} + + +StageCompleteCallback = Callable[ + [str, List["ExtractionObservation"], List[str]], + Optional[Awaitable[None]], +] + + +@dataclass +class OrchestratorConfig: + default_depth: Depth = "standard" + + +logger = logging.getLogger(__name__) + + +def _try_get_llm_service() -> Optional["LLMService"]: + """Try to obtain the default LLMService; return None on failure.""" + try: + from paperbot.application.services.llm_service import get_llm_service + + return get_llm_service() + except Exception: + logger.debug("LLMService unavailable, stages will use heuristic fallback") + return None + + +class ExtractionOrchestrator: + def __init__( + self, + stages: Optional[Iterable[P2CStage]] = None, + *, + input_router: Optional[PaperInputRouter] = None, + section_extractor: Optional[PaperSectionExtractor] = None, + paper_type_classifier: Optional[PaperTypeClassifier] = None, + config: Optional[OrchestratorConfig] = None, + llm: Optional["LLMService"] = None, + ): + resolved_llm = llm + if resolved_llm is None: + resolved_llm = _try_get_llm_service() + active_stages = list( + stages + or [ + LiteratureDistillStage(llm=resolved_llm), + BlueprintExtractStage(llm=resolved_llm), + EnvironmentExtractStage(llm=resolved_llm), + SpecExtractStage(llm=resolved_llm), + RoadmapPlanningStage(llm=resolved_llm), + SuccessCriteriaStage(llm=resolved_llm), + ] + ) + self._stages: Dict[str, P2CStage] = {stage.name: stage for stage in active_stages} + self._input_router = input_router or PaperInputRouter() + self._section_extractor = section_extractor or PaperSectionExtractor() + self._paper_type_classifier = paper_type_classifier or PaperTypeClassifier() + self._config = config or OrchestratorConfig() + + @staticmethod + def resolve_stage_sequence(depth: Depth, paper_type: PaperType) -> List[str]: + if depth == "fast": + stage_names = [ + StageName.BLUEPRINT_EXTRACT.value, + StageName.ENVIRONMENT_EXTRACT.value, + ] + else: + stage_names = [ + StageName.LITERATURE_DISTILL.value, + StageName.BLUEPRINT_EXTRACT.value, + StageName.ENVIRONMENT_EXTRACT.value, + StageName.SPEC_EXTRACT.value, + StageName.ROADMAP_PLANNING.value, + StageName.SUCCESS_CRITERIA.value, + ] + + if paper_type == PaperType.THEORETICAL: + return [ + s + for s in stage_names + if s + not in { + StageName.ENVIRONMENT_EXTRACT.value, + StageName.SPEC_EXTRACT.value, + } + ] + + if paper_type == PaperType.SURVEY: + return [StageName.LITERATURE_DISTILL.value, StageName.SUCCESS_CRITERIA.value] + + if paper_type == PaperType.BENCHMARK: + ordered = [ + StageName.LITERATURE_DISTILL.value, + StageName.BLUEPRINT_EXTRACT.value, + StageName.SPEC_EXTRACT.value, + StageName.SUCCESS_CRITERIA.value, + StageName.ROADMAP_PLANNING.value, + ] + return [name for name in ordered if name in stage_names] + + if paper_type == PaperType.SYSTEM: + ordered = [ + StageName.LITERATURE_DISTILL.value, + StageName.ENVIRONMENT_EXTRACT.value, + StageName.BLUEPRINT_EXTRACT.value, + StageName.ROADMAP_PLANNING.value, + StageName.SUCCESS_CRITERIA.value, + ] + return [name for name in ordered if name in stage_names] + + return stage_names + + async def run( + self, + request: GenerateContextRequest, + *, + raw_paper: Optional[RawPaperData] = None, + normalized_input: Optional[NormalizedInput] = None, + on_stage_complete: Optional[StageCompleteCallback] = None, + ) -> ReproContextPack: + from paperbot.utils.logging_config import Logger, LogFiles + + Logger.info( + f"[M1] orchestrator_start paper_id={request.paper_id} depth={request.depth}", + file=LogFiles.API, + ) + depth = request.depth or self._config.default_depth + + if normalized_input is None: + if raw_paper is None: + raw_paper = await self._input_router.fetch(request.paper_id) + normalized_input = await self._section_extractor.extract(raw_paper) + + paper_type = self._paper_type_classifier.classify(normalized_input) + pack = ReproContextPack( + context_pack_id=new_context_pack_id(), + paper=normalized_input.paper, + paper_type=paper_type, + objective=f"Reproduce core claims of {normalized_input.paper.title}.", + ) + + stage_order = self.resolve_stage_sequence(depth, paper_type) + stage_scores: Dict[str, float] = {} + stage_input = StageInput( + title=normalized_input.paper.title, + abstract=normalized_input.abstract, + full_text=normalized_input.full_text, + sections=normalized_input.sections, + ) + + for stage_name in stage_order: + stage = self._stages.get(stage_name) + if stage is None: + pack.warnings.append(f"Stage {stage_name} is not registered.") + continue + + Logger.info( + f"[M1] stage_start paper_id={request.paper_id} stage={stage_name}", + file=LogFiles.API, + ) + result = await stage.run(stage_input) + if result.observations: + pack.observations.extend(result.observations) + stage_scores[stage_name] = stage_mean_confidence(result.observations) + if result.roadmap: + pack.task_roadmap.extend(result.roadmap) + if result.warnings: + pack.warnings.extend(result.warnings) + Logger.info( + f"[M1] stage_done paper_id={request.paper_id} stage={stage_name} obs={len(result.observations)} warnings={len(result.warnings)}", + file=LogFiles.API, + ) + await self._emit_stage_complete( + on_stage_complete, + stage_name=stage_name, + observations=result.observations, + warnings=result.warnings, + ) + + if depth == "deep": + await self._run_deep_verification(stage_order, stage_input, stage_scores, pack) + pack.observations.sort(key=lambda item: item.confidence, reverse=True) + + self._apply_confidence(pack.confidence, stage_scores) + if depth == "deep": + available_stage_scores = [score for score in stage_scores.values() if score > 0] + pack.confidence.overall = ( + float(sum(available_stage_scores) / len(available_stage_scores)) + if available_stage_scores + else 0.0 + ) + elif pack.observations: + pack.confidence.overall = stage_mean_confidence(pack.observations) + Logger.info( + f"[M1] orchestrator_complete paper_id={request.paper_id} observations={len(pack.observations)} warnings={len(pack.warnings)}", + file=LogFiles.API, + ) + return pack + + @staticmethod + def _apply_confidence(confidence: ConfidenceScores, stage_scores: Dict[str, float]) -> None: + for stage_name, score in stage_scores.items(): + field = _STAGE_TO_CONFIDENCE_FIELD.get(stage_name) + if field: + setattr(confidence, field, score) + + async def _run_deep_verification( + self, + stage_order: List[str], + stage_input: StageInput, + stage_scores: Dict[str, float], + pack: ReproContextPack, + ) -> None: + verification_targets = [ + StageName.BLUEPRINT_EXTRACT.value, + StageName.SPEC_EXTRACT.value, + StageName.SUCCESS_CRITERIA.value, + ] + for stage_name in verification_targets: + if stage_name not in stage_order: + continue + stage = self._stages.get(stage_name) + if stage is None: + continue + + verification = await stage.run(stage_input) + baseline = stage_scores.get(stage_name, 0.0) + if not verification.observations: + if baseline > 0: + stage_scores[stage_name] = baseline * 0.85 + pack.warnings.append( + f"Deep verification found no observations for stage {stage_name}; confidence reduced." + ) + continue + + verified_score = stage_mean_confidence(verification.observations) + if baseline <= 0: + stage_scores[stage_name] = verified_score * 0.95 + continue + + if abs(verified_score - baseline) > 0.1: + pack.warnings.append( + f"Deep verification mismatch on stage {stage_name}: baseline={baseline:.2f}, verified={verified_score:.2f}." + ) + + stage_scores[stage_name] = min(baseline, verified_score) * 0.95 + + @staticmethod + async def _emit_stage_complete( + callback: Optional[StageCompleteCallback], + *, + stage_name: str, + observations: List["ExtractionObservation"], + warnings: List[str], + ) -> None: + if callback is None: + return + result = callback(stage_name, observations, warnings) + if inspect.isawaitable(result): + await result diff --git a/src/paperbot/application/services/p2c/prompts.py b/src/paperbot/application/services/p2c/prompts.py new file mode 100644 index 00000000..7b6afce7 --- /dev/null +++ b/src/paperbot/application/services/p2c/prompts.py @@ -0,0 +1,179 @@ +"""LLM prompt templates for P2C extraction stages.""" + +from __future__ import annotations + +from typing import Dict, Tuple + + +def _truncate(text: str, max_chars: int = 6000) -> str: + if len(text) <= max_chars: + return text + return text[:max_chars] + "\n[truncated]" + + +def _paper_context(title: str, abstract: str, sections: Dict[str, str]) -> str: + parts = [f"Paper: {title}", f"Abstract: {abstract}"] + for key in ("method", "introduction", "experiment", "results", "conclusion"): + section_text = sections.get(key, "") + if section_text: + parts.append(f"{key.title()} section: {_truncate(section_text)}") + return "\n\n".join(parts) + + +def literature_distill_prompt( + title: str, abstract: str, sections: Dict[str, str] +) -> Tuple[str, str]: + system = ( + "You are a research paper analysis expert. " + "Extract the core problem definition, proposed method, and key limitations. " + "Return a JSON array of 2-3 observations." + ) + user = f"""{_paper_context(title, abstract, sections)} + +Return a JSON array of 2-3 observations. Each observation must have: +- "type": "method" or "limitation" +- "title": short descriptive title (5-10 words) +- "narrative": 2-3 sentence description +- "concepts": array from ["core_method", "gotcha", "baseline"] +- "confidence": 0.0-1.0 based on how clearly the paper describes this +- "structured_data": dict with relevant key-value pairs (e.g. "problem_definition", "core_approach", "limitation") + +First observation should cover the problem and core idea (type=method, concepts=["core_method"]). +Second observation should cover key limitations or gotchas (type=limitation, concepts=["gotcha"]). + +Return ONLY the JSON array, no other text.""" + return system, user + + +def blueprint_extract_prompt( + title: str, abstract: str, sections: Dict[str, str] +) -> Tuple[str, str]: + system = ( + "You are a research paper analysis expert specializing in system architecture. " + "Extract the system/model architecture from the given paper. Return a JSON array." + ) + user = f"""{_paper_context(title, abstract, sections)} + +Return a JSON array of 2-3 observations about the system architecture. Each observation must have: +- "type": "architecture" or "method" +- "title": short descriptive title (5-10 words) +- "narrative": 2-3 sentence description +- "concepts": array from ["core_method", "architecture", "trade_off", "reproduction_hint", "gotcha"] +- "confidence": 0.0-1.0 based on evidence strength +- "structured_data": dict with relevant key-value pairs (e.g. "architecture_type", "key_modules", "design_pattern") + +Focus on: +1. Architecture pattern and key components (type=architecture, concepts include "architecture", "core_method") +2. Key design trade-offs (type=architecture, concepts include "trade_off") +3. Implementation details important for reproduction (type=method, concepts include "reproduction_hint") + +Return ONLY the JSON array, no other text.""" + return system, user + + +def environment_extract_prompt( + title: str, abstract: str, sections: Dict[str, str] +) -> Tuple[str, str]: + system = ( + "You are a research paper analysis expert specializing in software environments. " + "Identify the runtime environment, programming languages, frameworks, and dependencies. " + "Return a JSON array." + ) + user = f"""{_paper_context(title, abstract, sections)} + +Return a JSON array of 1-2 observations about the runtime environment. Each observation must have: +- "type": "environment" +- "title": short descriptive title (5-10 words) +- "narrative": 2-3 sentence description +- "concepts": array from ["environment", "reproduction_hint", "gotcha"] +- "confidence": 0.0-1.0 (higher if explicitly stated, lower if inferred) +- "structured_data": dict with keys like "language", "framework", "dependencies", "python_version", "hardware", "os" + +Focus on: +1. Runtime stack: programming language, frameworks, key libraries (concepts include "environment", "reproduction_hint") +2. Build/deployment gotchas if any (concepts include "gotcha") + +If the paper doesn't explicitly mention the environment, infer from the domain and methods described. +Set confidence lower (0.5-0.65) for inferred information vs higher (0.75-0.90) for explicitly stated. + +Return ONLY the JSON array, no other text.""" + return system, user + + +def spec_extract_prompt( + title: str, abstract: str, sections: Dict[str, str] +) -> Tuple[str, str]: + system = ( + "You are a research paper analysis expert specializing in experimental details. " + "Extract key hyperparameters, configurations, and experimental settings. " + "Return a JSON array." + ) + user = f"""{_paper_context(title, abstract, sections)} + +Return a JSON array of 1-3 observations about hyperparameters and specs. Each observation must have: +- "type": "hyperparameter" +- "title": short descriptive title (5-10 words) +- "narrative": 2-3 sentence description listing the key parameters +- "concepts": array from ["hyperparameter", "gotcha", "reproduction_hint"] +- "confidence": 0.0-1.0 (higher if values explicitly stated, lower if typical/inferred) +- "structured_data": dict mapping parameter names to their values (e.g. "learning_rate": "1e-4", "batch_size": "32") + +Extract all hyperparameters you can find: learning rate, batch size, epochs, optimizer, regularization, etc. +If values are not explicitly stated, note "not specified" and set lower confidence. + +Return ONLY the JSON array, no other text.""" + return system, user + + +def roadmap_planning_prompt( + title: str, abstract: str, sections: Dict[str, str] +) -> Tuple[str, str]: + system = ( + "You are a research paper reproduction expert. " + "Generate a paper-specific step-by-step reproduction roadmap. " + "Return a JSON array." + ) + user = f"""{_paper_context(title, abstract, sections)} + +Generate a reproduction roadmap with 4-6 steps tailored to THIS specific paper. +Return a JSON array where each step has: +- "id": "T1", "T2", etc. +- "title": concise step title +- "description": what this step involves (1-2 sentences) +- "acceptance_criteria": array of 1-2 verifiable criteria +- "depends_on": array of step IDs this depends on (empty for first step) +- "estimated_difficulty": "low", "medium", or "high" + +Steps should be specific to the paper's method, not generic. +For example, for a SLAM paper: setup camera/sensor pipeline, implement feature extraction, etc. +For a transformer paper: implement attention mechanism, prepare tokenizer, etc. + +Return ONLY the JSON array, no other text.""" + return system, user + + +def success_criteria_prompt( + title: str, abstract: str, sections: Dict[str, str] +) -> Tuple[str, str]: + system = ( + "You are a research paper analysis expert specializing in evaluation metrics. " + "Extract the specific evaluation metrics, target values, datasets, and success criteria. " + "Return a JSON array." + ) + user = f"""{_paper_context(title, abstract, sections)} + +Return a JSON array of 1-3 observations about success criteria and metrics. Each observation must have: +- "type": "metric" +- "title": short descriptive title (5-10 words) +- "narrative": 2-3 sentence description of the metrics and targets +- "concepts": array from ["baseline", "reproduction_hint", "gotcha"] +- "confidence": 0.0-1.0 (higher if explicit numbers given) +- "structured_data": dict with keys like "metrics" (list), "datasets" (list), "target_values" (dict), "evaluation_protocol" + +Focus on: +1. Primary evaluation metrics with target values if available +2. Datasets and splits used for evaluation +3. Comparison baselines mentioned + +Return ONLY the JSON array, no other text.""" + return system, user diff --git a/src/paperbot/application/services/p2c/stages.py b/src/paperbot/application/services/p2c/stages.py new file mode 100644 index 00000000..3b260842 --- /dev/null +++ b/src/paperbot/application/services/p2c/stages.py @@ -0,0 +1,524 @@ +from __future__ import annotations + +import asyncio +import json +import logging +import re +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, TYPE_CHECKING + +from .evidence import EvidenceLinker, calibrate_confidence +from .models import ( + ExtractionObservation, + StageName, + StageResult, + TaskCheckpoint, + new_observation_id, +) +from .prompts import ( + blueprint_extract_prompt, + environment_extract_prompt, + literature_distill_prompt, + roadmap_planning_prompt, + spec_extract_prompt, + success_criteria_prompt, +) + +if TYPE_CHECKING: + from paperbot.application.services.llm_service import LLMService + +logger = logging.getLogger(__name__) +_evidence_linker = EvidenceLinker() + + +@dataclass +class StageInput: + title: str + abstract: str + full_text: str + sections: Dict[str, str] + + +# --------------------------------------------------------------------------- +# JSON parsing helpers +# --------------------------------------------------------------------------- + +def _safe_parse_json_array(raw: str) -> Optional[List[Dict[str, Any]]]: + """Parse a JSON array from LLM output, tolerating markdown fences.""" + text = (raw or "").strip() + if not text: + return None + # Strip markdown code fences + if text.startswith("```"): + lines = text.split("\n") + lines = [l for l in lines if not l.strip().startswith("```")] + text = "\n".join(lines).strip() + try: + obj = json.loads(text) + if isinstance(obj, list): + return obj + return None + except Exception: + pass + # Try to find array brackets + start = text.find("[") + end = text.rfind("]") + if start >= 0 and end > start: + try: + obj = json.loads(text[start : end + 1]) + if isinstance(obj, list): + return obj + except Exception: + pass + return None + + +def _llm_complete_async(llm: "LLMService", system: str, user: str, task_type: str) -> str: + """Call LLMService.complete() — sync method, run in thread for async context.""" + return llm.complete(task_type=task_type, system=system, user=user) + + +def _obs_from_dict( + raw: Dict[str, Any], *, stage: str, default_type: str = "method" +) -> ExtractionObservation: + """Build an ExtractionObservation from an LLM-returned dict.""" + return ExtractionObservation( + id=new_observation_id(), + stage=stage, + type=str(raw.get("type", default_type)), + title=str(raw.get("title", "")), + narrative=str(raw.get("narrative", "")), + structured_data=raw.get("structured_data") or {}, + confidence=float(raw.get("confidence", 0.7)), + concepts=raw.get("concepts") or [], + ) + + +# --------------------------------------------------------------------------- +# Heuristic helpers (kept for fallback) +# --------------------------------------------------------------------------- + +def _first_sentence(text: str, fallback: str = "") -> str: + cleaned = " ".join(text.split()) + if not cleaned: + return fallback + parts = re.split(r"(?<=[.!?])\s+", cleaned) + return (parts[0] if parts else cleaned).strip() or fallback + + +def _detect_architecture(text: str) -> str: + lower = text.lower() + if "transformer" in lower: + return "transformer" + if "diffusion" in lower: + return "diffusion" + if "graph neural" in lower or "gnn" in lower: + return "gnn" + if "convolution" in lower or "cnn" in lower: + return "cnn" + if "lstm" in lower or "rnn" in lower: + return "rnn" + return "unknown" + + +# =================================================================== +# Stage classes — each tries LLM first, falls back to heuristic +# =================================================================== + + +class LiteratureDistillStage: + name = StageName.LITERATURE_DISTILL.value + + def __init__(self, llm: Optional["LLMService"] = None) -> None: + self._llm = llm + + async def run(self, data: StageInput) -> StageResult: + if self._llm is not None: + try: + return await self._run_llm(data) + except Exception: + logger.warning("LiteratureDistillStage LLM failed, falling back to heuristic") + return self._run_heuristic(data) + + async def _run_llm(self, data: StageInput) -> StageResult: + system, user = literature_distill_prompt(data.title, data.abstract, data.sections) + raw = await asyncio.to_thread(_llm_complete_async, self._llm, system, user, "extraction") + items = _safe_parse_json_array(raw) + if not items: + return self._run_heuristic(data) + observations = [_obs_from_dict(item, stage=self.name, default_type="method") for item in items] + return StageResult(observations=observations) + + def _run_heuristic(self, data: StageInput) -> StageResult: + narrative = _first_sentence( + data.abstract, + fallback="No abstract summary detected; review the original manuscript.", + ) + evidence = _evidence_linker.section_anchor( + data.abstract, + section="abstract", + supports=["problem_definition"], + ) + obs = ExtractionObservation( + id=new_observation_id(), + stage=self.name, + type="method", + title="Problem and core idea", + narrative=narrative, + structured_data={ + "problem_definition": narrative, + "paper_title": data.title, + }, + evidence=evidence, + confidence=calibrate_confidence(0.55, evidence), + concepts=["core_method"], + ) + return StageResult(observations=[obs]) + + +class BlueprintExtractStage: + name = StageName.BLUEPRINT_EXTRACT.value + + def __init__(self, llm: Optional["LLMService"] = None) -> None: + self._llm = llm + + async def run(self, data: StageInput) -> StageResult: + if self._llm is not None: + try: + return await self._run_llm(data) + except Exception: + logger.warning("BlueprintExtractStage LLM failed, falling back to heuristic") + return self._run_heuristic(data) + + async def _run_llm(self, data: StageInput) -> StageResult: + system, user = blueprint_extract_prompt(data.title, data.abstract, data.sections) + raw = await asyncio.to_thread(_llm_complete_async, self._llm, system, user, "extraction") + items = _safe_parse_json_array(raw) + if not items: + return self._run_heuristic(data) + observations = [ + _obs_from_dict(item, stage=self.name, default_type="architecture") for item in items + ] + return StageResult(observations=observations) + + def _run_heuristic(self, data: StageInput) -> StageResult: + source_text = f"{data.title}\n{data.abstract}\n{data.sections.get('method', '')}" + architecture = _detect_architecture(source_text) + evidence = ( + _evidence_linker.find_keyword_evidence( + source_text, + keywords=[architecture], + section="method", + supports=["architecture_type"], + max_links=1, + ) + if architecture != "unknown" + else [] + ) + confidence = calibrate_confidence(0.6 if architecture != "unknown" else 0.3, evidence) + obs = ExtractionObservation( + id=new_observation_id(), + stage=self.name, + type="architecture", + title=f"Architecture: {architecture}", + narrative=f"Detected main architecture pattern as {architecture}.", + structured_data={"architecture_type": architecture}, + evidence=evidence, + confidence=confidence, + concepts=["architecture"], + ) + warnings: List[str] = [] + if architecture != "unknown" and not evidence: + warnings.append("Blueprint extraction lacks direct evidence span for architecture.") + return StageResult(observations=[obs], warnings=warnings) + + +class EnvironmentExtractStage: + name = StageName.ENVIRONMENT_EXTRACT.value + + def __init__(self, llm: Optional["LLMService"] = None) -> None: + self._llm = llm + + async def run(self, data: StageInput) -> StageResult: + if self._llm is not None: + try: + return await self._run_llm(data) + except Exception: + logger.warning("EnvironmentExtractStage LLM failed, falling back to heuristic") + return self._run_heuristic(data) + + async def _run_llm(self, data: StageInput) -> StageResult: + system, user = environment_extract_prompt(data.title, data.abstract, data.sections) + raw = await asyncio.to_thread(_llm_complete_async, self._llm, system, user, "extraction") + items = _safe_parse_json_array(raw) + if not items: + return self._run_heuristic(data) + observations = [ + _obs_from_dict(item, stage=self.name, default_type="environment") for item in items + ] + return StageResult(observations=observations) + + def _run_heuristic(self, data: StageInput) -> StageResult: + text = f"{data.abstract}\n{data.full_text}".lower() + framework = ( + "pytorch" + if "pytorch" in text or "torch" in text + else "tensorflow" if "tensorflow" in text else "unknown" + ) + python_version = "3.10" if framework != "unknown" else "3.11" + evidence = ( + _evidence_linker.find_keyword_evidence( + text, + keywords=[framework], + section="full_text", + supports=["framework"], + max_links=1, + ) + if framework != "unknown" + else [] + ) + confidence = calibrate_confidence( + 0.5 if framework != "unknown" else 0.35, evidence, required=True + ) + obs = ExtractionObservation( + id=new_observation_id(), + stage=self.name, + type="environment", + title=f"Runtime env ({framework})", + narrative=f"Inferred runtime stack: python {python_version}, framework {framework}.", + structured_data={ + "python_version": python_version, + "framework": framework, + }, + evidence=evidence, + confidence=confidence, + concepts=["environment"], + ) + warnings: List[str] = [] + if framework == "unknown": + warnings.append("Environment extraction could not find explicit framework evidence.") + return StageResult(observations=[obs], warnings=warnings) + + +class SpecExtractStage: + name = StageName.SPEC_EXTRACT.value + + _regex_map = { + "learning_rate": re.compile( + r"(?:learning rate|lr)\s*[:=]?\s*([0-9.]+e-?\d+|0?\.\d+)", + re.IGNORECASE, + ), + "batch_size": re.compile(r"batch size\s*[:=]?\s*(\d+)", re.IGNORECASE), + "epochs": re.compile(r"(?:epoch|epochs)\s*[:=]?\s*(\d+)", re.IGNORECASE), + } + + def __init__(self, llm: Optional["LLMService"] = None) -> None: + self._llm = llm + + async def run(self, data: StageInput) -> StageResult: + if self._llm is not None: + try: + return await self._run_llm(data) + except Exception: + logger.warning("SpecExtractStage LLM failed, falling back to heuristic") + return self._run_heuristic(data) + + async def _run_llm(self, data: StageInput) -> StageResult: + system, user = spec_extract_prompt(data.title, data.abstract, data.sections) + raw = await asyncio.to_thread(_llm_complete_async, self._llm, system, user, "extraction") + items = _safe_parse_json_array(raw) + if not items: + return self._run_heuristic(data) + observations = [ + _obs_from_dict(item, stage=self.name, default_type="hyperparameter") for item in items + ] + return StageResult(observations=observations) + + def _run_heuristic(self, data: StageInput) -> StageResult: + blob = f"{data.abstract}\n{data.full_text}" + extracted: Dict[str, str] = {} + evidence = [] + for key, pattern in self._regex_map.items(): + match = pattern.search(blob) + if match: + extracted[key] = match.group(1) + evidence.append( + _evidence_linker.from_match( + section="full_text", + supports=[key], + start=match.start(1), + end=match.end(1), + ) + ) + + if not extracted: + return StageResult( + warnings=["Spec extraction found no explicit hyperparameters in text."], + ) + + narrative = ", ".join(f"{k}={v}" for k, v in extracted.items()) + confidence = calibrate_confidence(0.62, evidence, required=True) + obs = ExtractionObservation( + id=new_observation_id(), + stage=self.name, + type="hyperparameter", + title="Key hyperparameters", + narrative=f"Extracted hyperparameters: {narrative}.", + structured_data=extracted, + evidence=evidence, + confidence=confidence, + concepts=["hyperparameter", "gotcha"], + ) + warnings: List[str] = [] + if not evidence: + warnings.append("Spec extraction produced values without evidence spans.") + return StageResult(observations=[obs], warnings=warnings) + + +class RoadmapPlanningStage: + name = StageName.ROADMAP_PLANNING.value + + def __init__(self, llm: Optional["LLMService"] = None) -> None: + self._llm = llm + + async def run(self, data: StageInput) -> StageResult: + if self._llm is not None: + try: + return await self._run_llm(data) + except Exception: + logger.warning("RoadmapPlanningStage LLM failed, falling back to heuristic") + return self._run_heuristic(data) + + async def _run_llm(self, data: StageInput) -> StageResult: + system, user = roadmap_planning_prompt(data.title, data.abstract, data.sections) + raw = await asyncio.to_thread(_llm_complete_async, self._llm, system, user, "reasoning") + items = _safe_parse_json_array(raw) + if not items: + return self._run_heuristic(data) + + roadmap: List[TaskCheckpoint] = [] + for item in items: + roadmap.append( + TaskCheckpoint( + id=str(item.get("id", f"T{len(roadmap) + 1}")), + title=str(item.get("title", "")), + description=str(item.get("description", "")), + acceptance_criteria=item.get("acceptance_criteria") or [], + depends_on=item.get("depends_on") or [], + estimated_difficulty=item.get("estimated_difficulty", "medium"), + ) + ) + + obs = ExtractionObservation( + id=new_observation_id(), + stage=self.name, + type="roadmap", + title="Paper-specific reproduction roadmap", + narrative=f"Generated a {len(roadmap)}-step reproduction roadmap tailored to this paper.", + structured_data={"task_count": len(roadmap)}, + confidence=0.78, + concepts=["reproduction_hint"], + ) + return StageResult(observations=[obs], roadmap=roadmap) + + def _run_heuristic(self, data: StageInput) -> StageResult: + roadmap = [ + TaskCheckpoint( + id="T1", + title="Prepare dataset and loaders", + acceptance_criteria=["Training split loads without runtime errors"], + estimated_difficulty="medium", + ), + TaskCheckpoint( + id="T2", + title="Implement model architecture", + acceptance_criteria=["Forward pass shape checks pass"], + depends_on=["T1"], + estimated_difficulty="high", + ), + TaskCheckpoint( + id="T3", + title="Reproduce training loop", + acceptance_criteria=["Loss decreases in smoke run"], + depends_on=["T2"], + estimated_difficulty="medium", + ), + ] + obs = ExtractionObservation( + id=new_observation_id(), + stage=self.name, + type="roadmap", + title="Initial execution roadmap", + narrative="Generated a three-step baseline roadmap for reproduction.", + structured_data={"task_count": len(roadmap)}, + confidence=0.58, + concepts=["reproduction_hint"], + ) + return StageResult(observations=[obs], roadmap=roadmap) + + +class SuccessCriteriaStage: + name = StageName.SUCCESS_CRITERIA.value + + _metric_re = re.compile( + r"\b(accuracy|f1|bleu|rouge|mrr|ndcg|auc|wer|cer)\b[^\n.]{0,60}", + re.IGNORECASE, + ) + + def __init__(self, llm: Optional["LLMService"] = None) -> None: + self._llm = llm + + async def run(self, data: StageInput) -> StageResult: + if self._llm is not None: + try: + return await self._run_llm(data) + except Exception: + logger.warning("SuccessCriteriaStage LLM failed, falling back to heuristic") + return self._run_heuristic(data) + + async def _run_llm(self, data: StageInput) -> StageResult: + system, user = success_criteria_prompt(data.title, data.abstract, data.sections) + raw = await asyncio.to_thread(_llm_complete_async, self._llm, system, user, "extraction") + items = _safe_parse_json_array(raw) + if not items: + return self._run_heuristic(data) + observations = [ + _obs_from_dict(item, stage=self.name, default_type="metric") for item in items + ] + return StageResult(observations=observations) + + def _run_heuristic(self, data: StageInput) -> StageResult: + blob = f"{data.abstract}\n{data.full_text}" + matches = list(self._metric_re.finditer(blob)) + metrics = sorted({m.group(1).lower() for m in matches}) + evidence = [ + _evidence_linker.from_match( + section="full_text", + supports=["metrics"], + start=match.start(1), + end=match.end(1), + confidence=0.81, + ) + for match in matches[:5] + ] + if not metrics: + metrics = ["accuracy"] + + confidence = calibrate_confidence(0.52, evidence, required=True) + obs = ExtractionObservation( + id=new_observation_id(), + stage=self.name, + type="metric", + title="Success criteria", + narrative=f"Track metrics: {', '.join(metrics)}.", + structured_data={"metrics": metrics}, + evidence=evidence, + confidence=confidence, + concepts=["baseline"], + ) + warnings: List[str] = [] + if not evidence: + warnings.append( + "Success criteria metrics are inferred without explicit metric span evidence." + ) + return StageResult(observations=[obs], warnings=warnings) diff --git a/src/paperbot/infrastructure/stores/models.py b/src/paperbot/infrastructure/stores/models.py index 915d474f..aa0e51b3 100644 --- a/src/paperbot/infrastructure/stores/models.py +++ b/src/paperbot/infrastructure/stores/models.py @@ -1143,3 +1143,115 @@ def get_errors(self) -> dict: def set_errors(self, errors: dict) -> None: self.error_json = json.dumps(errors or {}, ensure_ascii=False) + + +# ============================================================================ +# P2C (Paper-to-Context) Module 2 Models +# ============================================================================ + + +class ReproContextPackModel(Base): + """P2C context pack: stores the full ReproContextPack JSON produced by Core Engine.""" + + __tablename__ = "repro_context_pack" + + id: Mapped[str] = mapped_column(String(64), primary_key=True) # "ctxp_{uuid}" + user_id: Mapped[str] = mapped_column(String(64), nullable=False, default="default", index=True) + project_id: Mapped[Optional[str]] = mapped_column(String(64), nullable=True, index=True) + paper_id: Mapped[str] = mapped_column(String(256), nullable=False, index=True) + paper_title: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + version: Mapped[str] = mapped_column(String(16), nullable=False, default="v1") + depth: Mapped[str] = mapped_column(String(16), nullable=False, default="standard") # fast/standard/deep + status: Mapped[str] = mapped_column( + String(32), nullable=False, default="pending", index=True + ) # pending/running/completed/failed + objective: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + pack_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}") + confidence_overall: Mapped[float] = mapped_column(Float, default=0.0, index=True) + warning_count: Mapped[int] = mapped_column(Integer, default=0) + + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True) + deleted_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True) + + stage_results = relationship( + "ReproContextStageResultModel", back_populates="pack", cascade="all, delete-orphan" + ) + evidence_links = relationship( + "ReproContextEvidenceModel", back_populates="pack", cascade="all, delete-orphan" + ) + feedback_rows = relationship( + "ReproContextFeedbackModel", back_populates="pack", cascade="all, delete-orphan" + ) + + def get_pack(self) -> dict: + try: + return json.loads(self.pack_json or "{}") + except Exception: + return {} + + def set_pack(self, data: dict) -> None: + self.pack_json = json.dumps(data or {}, ensure_ascii=False) + + +class ReproContextStageResultModel(Base): + """Per-stage intermediate result for debugging and audit.""" + + __tablename__ = "repro_context_stage_result" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + context_pack_id: Mapped[str] = mapped_column( + String(64), ForeignKey("repro_context_pack.id", ondelete="CASCADE"), index=True + ) + stage_name: Mapped[str] = mapped_column(String(64), index=True) + status: Mapped[str] = mapped_column(String(16), default="completed", index=True) # completed/failed/skipped + result_json: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + confidence: Mapped[float] = mapped_column(Float, default=0.0) + duration_ms: Mapped[int] = mapped_column(Integer, default=0) + error_message: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True) + + pack = relationship("ReproContextPackModel", back_populates="stage_results") + + +class ReproContextEvidenceModel(Base): + """Evidence links for audit trail — maps extracted fields back to paper spans.""" + + __tablename__ = "repro_context_evidence" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + context_pack_id: Mapped[str] = mapped_column( + String(64), ForeignKey("repro_context_pack.id", ondelete="CASCADE"), index=True + ) + evidence_type: Mapped[str] = mapped_column(String(32), index=True) # paper_span/table/figure/code_snippet/metadata + ref: Mapped[str] = mapped_column(Text, default="") + supports_json: Mapped[str] = mapped_column(Text, default="[]") # JSON array of field names + confidence: Mapped[float] = mapped_column(Float, default=0.0) + + pack = relationship("ReproContextPackModel", back_populates="evidence_links") + + def get_supports(self) -> list: + try: + return json.loads(self.supports_json or "[]") + except Exception: + return [] + + def set_supports(self, values: list) -> None: + self.supports_json = json.dumps(values or [], ensure_ascii=False) + + +class ReproContextFeedbackModel(Base): + """User ratings and comments on a context pack (collected post-launch).""" + + __tablename__ = "repro_context_feedback" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + context_pack_id: Mapped[str] = mapped_column( + String(64), ForeignKey("repro_context_pack.id", ondelete="CASCADE"), index=True + ) + user_id: Mapped[str] = mapped_column(String(64), index=True) + rating: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) # 1-5 + comment: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True) + + pack = relationship("ReproContextPackModel", back_populates="feedback_rows") diff --git a/src/paperbot/infrastructure/stores/repro_context_store.py b/src/paperbot/infrastructure/stores/repro_context_store.py new file mode 100644 index 00000000..99f669fc --- /dev/null +++ b/src/paperbot/infrastructure/stores/repro_context_store.py @@ -0,0 +1,227 @@ +"""SqlAlchemyReproContextStore — P2C context pack persistence.""" + +from __future__ import annotations + +import json +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Tuple + +from sqlalchemy import select, func + +from paperbot.infrastructure.stores.models import ReproContextPackModel, ReproContextStageResultModel +from paperbot.infrastructure.stores.sqlalchemy_db import SessionProvider + + +def _utcnow() -> datetime: + return datetime.now(timezone.utc) + + +class SqlAlchemyReproContextStore: + """SQLAlchemy implementation of ReproContextPort.""" + + def __init__(self, db_url: Optional[str] = None): + self._provider = SessionProvider(db_url) + + # ------------------------------------------------------------------ # + # Write # + # ------------------------------------------------------------------ # + + def save( + self, + *, + pack_id: str, + user_id: str, + paper_id: str, + depth: str, + pack_data: Dict[str, Any], + paper_title: Optional[str] = None, + project_id: Optional[str] = None, + objective: Optional[str] = None, + confidence_overall: float = 0.0, + warning_count: int = 0, + ) -> str: + now = _utcnow() + row = ReproContextPackModel( + id=pack_id, + user_id=user_id, + project_id=project_id, + paper_id=paper_id, + paper_title=paper_title, + depth=depth, + status="running", + objective=objective, + confidence_overall=confidence_overall, + warning_count=warning_count, + created_at=now, + updated_at=now, + ) + row.set_pack(pack_data) + with self._provider.session() as session: + session.add(row) + session.commit() + return pack_id + + def update_status( + self, + pack_id: str, + *, + status: str, + pack_data: Optional[Dict[str, Any]] = None, + confidence_overall: Optional[float] = None, + warning_count: Optional[int] = None, + objective: Optional[str] = None, + ) -> None: + with self._provider.session() as session: + row = session.get(ReproContextPackModel, pack_id) + if row is None: + return + row.status = status + row.updated_at = _utcnow() + if pack_data is not None: + row.set_pack(pack_data) + if confidence_overall is not None: + row.confidence_overall = confidence_overall + if warning_count is not None: + row.warning_count = warning_count + if objective is not None: + row.objective = objective + session.commit() + + def soft_delete(self, pack_id: str) -> bool: + with self._provider.session() as session: + row = session.get(ReproContextPackModel, pack_id) + if row is None or row.deleted_at is not None: + return False + row.deleted_at = _utcnow() + row.updated_at = _utcnow() + session.commit() + return True + + def save_stage_result( + self, + *, + pack_id: str, + stage_name: str, + status: str, + result_data: Optional[Dict[str, Any]] = None, + confidence: float = 0.0, + duration_ms: int = 0, + error_message: Optional[str] = None, + ) -> None: + now = _utcnow() + row = ReproContextStageResultModel( + context_pack_id=pack_id, + stage_name=stage_name, + status=status, + result_json=json.dumps(result_data or {}, ensure_ascii=False), + confidence=confidence, + duration_ms=duration_ms, + error_message=error_message, + created_at=now, + ) + with self._provider.session() as session: + session.add(row) + session.commit() + + # ------------------------------------------------------------------ # + # Read # + # ------------------------------------------------------------------ # + + def get(self, pack_id: str) -> Optional[Dict[str, Any]]: + with self._provider.session() as session: + row = session.get(ReproContextPackModel, pack_id) + if row is None or row.deleted_at is not None: + return None + return self._row_to_full_dict(row) + + def get_observation(self, pack_id: str, observation_id: str) -> Optional[Dict[str, Any]]: + with self._provider.session() as session: + # Check soft-delete first to avoid leaking data from deleted packs + pack_row = session.get(ReproContextPackModel, pack_id) + if pack_row is None or pack_row.deleted_at is not None: + return None + + stage_rows = session.scalars( + select(ReproContextStageResultModel) + .where(ReproContextStageResultModel.context_pack_id == pack_id) + .order_by(ReproContextStageResultModel.created_at.asc()) + ).all() + + for row in stage_rows: + if not row.result_json: + continue + try: + payload = json.loads(row.result_json) + except (json.JSONDecodeError, ValueError): + continue + for obs in payload.get("observations", []) if isinstance(payload, dict) else []: + if isinstance(obs, dict) and obs.get("id") == observation_id: + return obs + + pack = pack_row.get_pack() + for obs in pack.get("observations", []) if isinstance(pack, dict) else []: + if isinstance(obs, dict) and obs.get("id") == observation_id: + return obs + return None + + def list_by_user( + self, + *, + user_id: str, + paper_id: Optional[str] = None, + project_id: Optional[str] = None, + limit: int = 20, + offset: int = 0, + ) -> Tuple[List[Dict[str, Any]], int]: + with self._provider.session() as session: + base = ( + select(ReproContextPackModel) + .where( + ReproContextPackModel.user_id == user_id, + ReproContextPackModel.deleted_at.is_(None), + ) + .order_by(ReproContextPackModel.created_at.desc()) + ) + if paper_id: + base = base.where(ReproContextPackModel.paper_id == paper_id) + if project_id: + base = base.where(ReproContextPackModel.project_id == project_id) + + total = session.scalar( + select(func.count()).select_from(base.subquery()) + ) or 0 + + rows = session.scalars(base.limit(limit).offset(offset)).all() + summaries = [self._row_to_summary_dict(r) for r in rows] + return summaries, total + + # ------------------------------------------------------------------ # + # Helpers # + # ------------------------------------------------------------------ # + + def _row_to_summary_dict(self, row: ReproContextPackModel) -> Dict[str, Any]: + return { + "context_pack_id": row.id, + "paper_id": row.paper_id, + "paper_title": row.paper_title, + "depth": row.depth, + "status": row.status, + "confidence_overall": row.confidence_overall, + "warning_count": row.warning_count, + "created_at": row.created_at.isoformat() if row.created_at else None, + } + + def _row_to_full_dict(self, row: ReproContextPackModel) -> Dict[str, Any]: + base = self._row_to_summary_dict(row) + base.update({ + "user_id": row.user_id, + "project_id": row.project_id, + "version": row.version, + "updated_at": row.updated_at.isoformat() if row.updated_at else None, + }) + # Merge pack contents directly into the top-level dict so the frontend + # can access `response.paper` instead of `response.pack.paper`. + # Pack fields (e.g. objective) take precedence over the summary placeholders. + pack = row.get_pack() + base.update(pack) + return base diff --git a/tests/integration/test_repro_context_routes.py b/tests/integration/test_repro_context_routes.py new file mode 100644 index 00000000..c0e8679f --- /dev/null +++ b/tests/integration/test_repro_context_routes.py @@ -0,0 +1,269 @@ +"""Integration tests for P2C repro_context API endpoints. + +Covers: + - GET /api/research/repro/context (list) + - GET /api/research/repro/context/{pack_id} (get) + - POST /api/research/repro/context/{pack_id}/session (create session) + - DELETE /api/research/repro/context/{pack_id} (soft-delete) + - POST /api/research/repro/context/generate (SSE streaming) +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import List + +import pytest +from fastapi.testclient import TestClient + +from paperbot.api import main as api_main +from paperbot.api.routes import repro_context as repro_context_module +from paperbot.infrastructure.stores.models import Base +from paperbot.infrastructure.stores.repro_context_store import SqlAlchemyReproContextStore + + +# ------------------------------------------------------------------ # +# Helpers # +# ------------------------------------------------------------------ # + + +def _parse_sse_events(text: str) -> List[dict]: + events = [] + for line in text.split("\n"): + if line.startswith("data: "): + payload = line[6:].strip() + if payload == "[DONE]": + continue + try: + events.append(json.loads(payload)) + except Exception: + pass + return events + + +def _make_store(tmp_path: Path) -> SqlAlchemyReproContextStore: + db_url = f"sqlite:///{tmp_path / 'test_repro.db'}" + store = SqlAlchemyReproContextStore(db_url=db_url) + Base.metadata.create_all(store._provider.engine) + return store + + +def _seed_pack(store: SqlAlchemyReproContextStore, pack_id: str = "pack_abc123", **kwargs): + defaults = dict( + pack_id=pack_id, + user_id="test_user", + paper_id="arxiv:2401.00001", + depth="fast", + pack_data={"paper_id": "arxiv:2401.00001", "objective": "Reproduce method X"}, + project_id="proj_1", + confidence_overall=0.85, + warning_count=1, + ) + defaults.update(kwargs) + store.save(**defaults) + # Mark as completed so get() returns pack data + store.update_status( + pack_id, + status="completed", + pack_data=defaults["pack_data"], + confidence_overall=defaults["confidence_overall"], + warning_count=defaults["warning_count"], + ) + return defaults + + +@pytest.fixture() +def client_and_store(tmp_path: Path, monkeypatch): + store = _make_store(tmp_path) + monkeypatch.setattr(repro_context_module, "_store", store) + with TestClient(api_main.app) as client: + yield client, store + + +# ------------------------------------------------------------------ # +# GET /api/research/repro/context (list) # +# ------------------------------------------------------------------ # + + +def test_list_packs_empty(client_and_store): + client, _ = client_and_store + resp = client.get("/api/research/repro/context", params={"user_id": "nobody"}) + assert resp.status_code == 200 + body = resp.json() + assert body["items"] == [] + assert body["total"] == 0 + + +def test_list_packs_returns_seeded(client_and_store): + client, store = client_and_store + _seed_pack(store, pack_id="pack_1", user_id="alice") + _seed_pack(store, pack_id="pack_2", user_id="alice", paper_id="arxiv:2401.99999") + + resp = client.get("/api/research/repro/context", params={"user_id": "alice"}) + assert resp.status_code == 200 + body = resp.json() + assert body["total"] == 2 + assert len(body["items"]) == 2 + + +def test_list_packs_filters_by_paper_id(client_and_store): + client, store = client_and_store + _seed_pack(store, pack_id="pack_a", user_id="bob", paper_id="arxiv:1111.11111") + _seed_pack(store, pack_id="pack_b", user_id="bob", paper_id="arxiv:2222.22222") + + resp = client.get( + "/api/research/repro/context", + params={"user_id": "bob", "paper_id": "arxiv:1111.11111"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["total"] == 1 + + +# ------------------------------------------------------------------ # +# GET /api/research/repro/context/{pack_id} (get detail) # +# ------------------------------------------------------------------ # + + +def test_get_pack_found(client_and_store): + client, store = client_and_store + _seed_pack(store, pack_id="pack_detail") + + resp = client.get("/api/research/repro/context/pack_detail") + assert resp.status_code == 200 + body = resp.json() + assert body["context_pack_id"] == "pack_detail" + + +def test_get_pack_not_found(client_and_store): + client, _ = client_and_store + resp = client.get("/api/research/repro/context/nonexistent") + assert resp.status_code == 404 + + +# ------------------------------------------------------------------ # +# POST /api/research/repro/context/{pack_id}/session # +# ------------------------------------------------------------------ # + + +def test_create_session_success(client_and_store): + client, store = client_and_store + _seed_pack( + store, + pack_id="pack_sess", + pack_data={ + "paper_id": "arxiv:2401.00001", + "task_roadmap": [ + {"title": "Setup env"}, + {"title": "Implement model"}, + ], + }, + ) + + resp = client.post( + "/api/research/repro/context/pack_sess/session", + json={"executor_preference": "auto"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body["session_id"].startswith("sess_") + assert body["runbook_id"].startswith("rb_") + assert len(body["initial_steps"]) == 2 + assert body["initial_steps"][0]["title"] == "Setup env" + assert "initial_prompt" in body + + +def test_create_session_pack_not_found(client_and_store): + client, _ = client_and_store + resp = client.post( + "/api/research/repro/context/nonexistent/session", + json={"executor_preference": "auto"}, + ) + assert resp.status_code == 404 + + +# ------------------------------------------------------------------ # +# DELETE /api/research/repro/context/{pack_id} # +# ------------------------------------------------------------------ # + + +def test_delete_pack_success(client_and_store): + client, store = client_and_store + _seed_pack(store, pack_id="pack_del") + + resp = client.delete("/api/research/repro/context/pack_del") + assert resp.status_code == 200 + assert resp.json()["status"] == "deleted" + + # Should no longer be retrievable + resp2 = client.get("/api/research/repro/context/pack_del") + assert resp2.status_code == 404 + + +def test_delete_pack_not_found(client_and_store): + client, _ = client_and_store + resp = client.delete("/api/research/repro/context/nonexistent") + assert resp.status_code == 404 + + +# ------------------------------------------------------------------ # +# POST /api/research/repro/context/generate (SSE) # +# ------------------------------------------------------------------ # + + +def test_generate_stream_returns_sse(client_and_store, monkeypatch): + """Verify the generate endpoint returns a valid SSE stream. + + We monkeypatch the ExtractionOrchestrator to avoid real LLM calls. + """ + client, store = client_and_store + + from dataclasses import dataclass, field + + @dataclass + class _FakeConfidence: + overall: float = 0.9 + + @dataclass + class _FakeObservation: + id: str = "obs_1" + type: str = "method" + title: str = "Fake observation" + confidence: float = 0.9 + + def to_full(self): + return {"id": self.id, "type": self.type, "title": self.title, "confidence": self.confidence} + + @dataclass + class _FakePack: + paper_id: str = "arxiv:2401.00001" + objective: str = "Reproduce" + observations: list = field(default_factory=lambda: [_FakeObservation()]) + warnings: list = field(default_factory=list) + confidence: _FakeConfidence = field(default_factory=_FakeConfidence) + task_roadmap: list = field(default_factory=list) + + class _FakeOrchestrator: + async def run(self, request, on_stage_complete=None): + if on_stage_complete: + await on_stage_complete("extraction", [_FakeObservation()], []) + return _FakePack() + + monkeypatch.setattr(repro_context_module, "ExtractionOrchestrator", _FakeOrchestrator) + + resp = client.post( + "/api/research/repro/context/generate", + json={"paper_id": "arxiv:2401.00001", "user_id": "test", "depth": "fast"}, + ) + assert resp.status_code == 200 + assert "text/event-stream" in resp.headers.get("content-type", "") + + events = _parse_sse_events(resp.text) + assert len(events) >= 1 + + event_types = [e.get("type") for e in events] + assert "status" in event_types or "result" in event_types + # Final event should be a result + assert events[-1]["type"] == "result" + assert events[-1]["data"]["status"] == "completed" diff --git a/tests/unit/test_p2c_benchmark.py b/tests/unit/test_p2c_benchmark.py new file mode 100644 index 00000000..d38137fc --- /dev/null +++ b/tests/unit/test_p2c_benchmark.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +import json + +import pytest + +from paperbot.application.services.p2c import ( + BenchmarkCase, + ReproContextPack, + aggregate_results, + evaluate_case, + load_benchmark_cases, + run_module1_benchmark, +) +from paperbot.application.services.p2c.models import ( + EvidenceLink, + ExtractionObservation, + PaperIdentity, +) + + +def test_load_benchmark_cases_parses_fixture(tmp_path): + fixture = tmp_path / "fixture.json" + fixture.write_text( + json.dumps( + [ + { + "case_id": "c1", + "title": "Case 1", + "abstract": "A", + "full_text": "B", + "year": 2026, + "expected": { + "architecture": "transformer", + "metrics": ["accuracy"], + "hyperparameters": ["learning_rate"], + "evidence_required_types": ["metric"], + }, + } + ] + ), + encoding="utf-8", + ) + + cases = load_benchmark_cases(fixture) + assert len(cases) == 1 + assert cases[0].case_id == "c1" + assert cases[0].expected_architecture == "transformer" + assert cases[0].expected_metrics == ["accuracy"] + + +def test_evaluate_case_scores_evidence_and_matches(): + case = BenchmarkCase( + case_id="c1", + title="x", + expected_architecture="transformer", + expected_metrics=["accuracy"], + expected_hyperparams=["learning_rate"], + evidence_required_types=["metric", "hyperparameter"], + ) + pack = ReproContextPack( + context_pack_id="ctx", + paper=PaperIdentity(paper_id="p", title="x"), + observations=[ + ExtractionObservation( + id="o1", + stage="blueprint_extract", + type="architecture", + title="arch", + narrative="n", + structured_data={"architecture_type": "transformer"}, + confidence=0.8, + ), + ExtractionObservation( + id="o2", + stage="spec_extract", + type="hyperparameter", + title="h", + narrative="n", + structured_data={"learning_rate": "1e-4"}, + evidence=[ + EvidenceLink( + type="paper_span", + ref="method#char:1-3", + supports=["learning_rate"], + confidence=0.9, + ) + ], + confidence=0.8, + ), + ExtractionObservation( + id="o3", + stage="success_criteria", + type="metric", + title="m", + narrative="n", + structured_data={"metrics": ["accuracy"]}, + evidence=[ + EvidenceLink( + type="paper_span", + ref="results#char:1-3", + supports=["metrics"], + confidence=0.9, + ) + ], + confidence=0.8, + ), + ], + ) + + result = evaluate_case(case, pack) + assert result["metric_f1"] == 1.0 + assert result["hyperparam_f1"] == 1.0 + assert result["architecture_hit"] == 1.0 + assert result["evidence_hit_rate"] == 1.0 + + +@pytest.mark.asyncio +async def test_run_module1_benchmark_returns_summary(): + cases = [ + BenchmarkCase( + case_id="bench1", + title="Transformer Benchmark Study", + abstract="We propose a transformer model.", + full_text=( + "Method\n" + "Learning rate 1e-4, batch size 32.\n" + "Results\n" + "Accuracy reaches 90.0.\n" + ), + expected_architecture="transformer", + expected_metrics=["accuracy"], + expected_hyperparams=["learning_rate", "batch_size"], + evidence_required_types=["metric", "hyperparameter"], + ) + ] + + result = await run_module1_benchmark(cases) + assert "summary" in result + assert result["cases"] + summary = result["summary"] + assert set(summary.keys()) == { + "metric_f1", + "hyperparam_f1", + "architecture_hit_rate", + "evidence_hit_rate", + "avg_warnings", + } + assert summary["architecture_hit_rate"] >= 0.0 + assert aggregate_results(result["cases"]) == summary diff --git a/tests/unit/test_p2c_module1.py b/tests/unit/test_p2c_module1.py new file mode 100644 index 00000000..6e6ef06a --- /dev/null +++ b/tests/unit/test_p2c_module1.py @@ -0,0 +1,420 @@ +from __future__ import annotations + +import pytest + +from paperbot.application.services.p2c import ( + ArXivAdapter, + EvidenceLink, + ExtractionObservation, + ExtractionOrchestrator, + GenerateContextRequest, + NormalizedInput, + PaperIdentity, + PaperInputRouter, + PaperSectionExtractor, + PaperType, + PaperTypeClassifier, + RawPaperData, + ReproContextPack, + SemanticScholarAdapter, + calibrate_confidence, +) +from paperbot.application.services.p2c.stages import ( + SpecExtractStage, + StageInput, + SuccessCriteriaStage, +) + + +def test_repro_context_pack_compact_context_prioritizes_core_method(): + pack = ReproContextPack( + context_pack_id="ctxp_test", + paper=PaperIdentity(paper_id="p1", title="Test Paper", year=2026), + objective="Reproduce key claim", + observations=[ + ExtractionObservation( + id="obs_a", + stage="literature_distill", + type="method", + title="Core method", + narrative="A" * 60, + confidence=0.9, + concepts=["core_method"], + ), + ExtractionObservation( + id="obs_b", + stage="spec_extract", + type="hyperparameter", + title="Hyperparameters", + narrative="B" * 60, + confidence=0.7, + concepts=["hyperparameter"], + ), + ], + ) + + compact = pack.to_compact_context(max_tokens=200) + assert "Core method" in compact + assert compact.index("Core method") < compact.index("Hyperparameters") + + +def test_paper_type_classifier_identifies_theoretical_text(): + classifier = PaperTypeClassifier() + normalized = NormalizedInput( + paper=PaperIdentity(paper_id="p2", title="Convergence Theorem for XYZ"), + abstract="We provide proofs and bounds for optimization stability.", + full_text="", + sections={}, + ) + + assert classifier.classify(normalized) == PaperType.THEORETICAL + + +def test_resolve_stage_sequence_respects_depth_and_type(): + seq_fast = ExtractionOrchestrator.resolve_stage_sequence("fast", PaperType.EXPERIMENTAL) + assert seq_fast == ["blueprint_extract", "environment_extract"] + + seq_theoretical = ExtractionOrchestrator.resolve_stage_sequence( + "standard", PaperType.THEORETICAL + ) + assert "environment_extract" not in seq_theoretical + assert "spec_extract" not in seq_theoretical + + +@pytest.mark.asyncio +async def test_orchestrator_run_builds_context_pack_from_raw_text(): + orchestrator = ExtractionOrchestrator() + request = GenerateContextRequest(paper_id="local:test", depth="standard") + + raw_paper = RawPaperData( + paper_id="local:test", + title="Transformer Benchmark Study", + abstract="We introduce a transformer model and report accuracy improvements.", + year=2025, + full_text=( + "Introduction\n" + "This paper studies benchmark behavior.\n" + "Method\n" + "Our model uses transformer blocks. Learning rate 1e-4, batch size 32, epochs 10.\n" + "Results\n" + "Accuracy reaches 92.1 on test split.\n" + ), + source_adapter="local_file", + ) + + pack = await orchestrator.run(request, raw_paper=raw_paper) + + assert pack.paper.title == "Transformer Benchmark Study" + assert pack.observations + assert any(obs.type == "architecture" for obs in pack.observations) + assert any(obs.type == "metric" for obs in pack.observations) + assert pack.confidence.overall > 0 + + +@pytest.mark.asyncio +async def test_semantic_scholar_adapter_fetch_maps_metadata_from_client(): + class _FakeS2Client: + def __init__(self): + self.calls = [] + + async def get_paper(self, paper_id, fields=None): + self.calls.append((paper_id, fields)) + return { + "paperId": "abcdef123456", + "title": "Attention Is All You Need", + "abstract": "We propose the Transformer architecture.", + "year": 2017, + "authors": [{"name": "Ashish Vaswani"}, {"name": "Noam Shazeer"}], + "externalIds": {"DOI": "10.48550/arXiv.1706.03762", "ArXiv": "1706.03762"}, + } + + async def close(self): + return None + + fake_client = _FakeS2Client() + adapter = SemanticScholarAdapter(client=fake_client) + result = await adapter.fetch("https://doi.org/10.48550/ARXIV.1706.03762") + + assert fake_client.calls + assert fake_client.calls[0][0] == "DOI:10.48550/arxiv.1706.03762" + assert result.paper_id == "s2:abcdef123456" + assert result.title == "Attention Is All You Need" + assert result.authors == ["Ashish Vaswani", "Noam Shazeer"] + assert result.identifiers["semantic_scholar"] == "abcdef123456" + assert result.identifiers["doi"] == "10.48550/arxiv.1706.03762" + assert result.identifiers["arxiv"] == "1706.03762" + + +@pytest.mark.asyncio +async def test_arxiv_adapter_fetch_maps_metadata_from_atom_feed(): + async def _fake_atom_feed(_: str) -> str: + return """ + + + http://arxiv.org/abs/1706.03762v1 + Attention Is All You Need + Transformer based sequence modeling. + 2017-06-12T00:00:00Z + 2017-06-12T00:00:00Z + Ashish Vaswani + Noam Shazeer + + + + +""" + + adapter = ArXivAdapter(fetch_atom_xml=_fake_atom_feed) + result = await adapter.fetch("https://arxiv.org/abs/1706.03762") + + assert result.paper_id == "arxiv:1706.03762v1" + assert result.title == "Attention Is All You Need" + assert result.abstract == "Transformer based sequence modeling." + assert result.year == 2017 + assert result.authors == ["Ashish Vaswani", "Noam Shazeer"] + assert result.identifiers["arxiv"] == "1706.03762v1" + + +@pytest.mark.asyncio +async def test_input_router_dispatches_to_arxiv_and_semantic_scholar_adapters(): + async def _fake_atom_feed(_: str) -> str: + return """ + + + http://arxiv.org/abs/2401.00001v2 + ArXiv Sample + Sample abstract. + 2024-01-01T00:00:00Z + 2024-01-01T00:00:00Z + Author A + + + +""" + + class _FakeS2Client: + async def get_paper(self, paper_id, fields=None): + if paper_id != "hash123": + return {} + return { + "paperId": "hash123", + "title": "S2 Sample", + "abstract": "S2 abstract", + "year": 2025, + "authors": [{"name": "Author B"}], + "externalIds": {}, + } + + async def close(self): + return None + + router = PaperInputRouter( + adapters=[ + ArXivAdapter(fetch_atom_xml=_fake_atom_feed), + SemanticScholarAdapter(client=_FakeS2Client()), + ] + ) + + arxiv_raw = await router.fetch("arXiv:2401.00001") + s2_raw = await router.fetch("s2:hash123") + + assert arxiv_raw.source_adapter == "arxiv" + assert arxiv_raw.title == "ArXiv Sample" + assert s2_raw.source_adapter == "semantic_scholar" + assert s2_raw.title == "S2 Sample" + + +@pytest.mark.asyncio +async def test_orchestrator_stage_callback_emits_stage_payload(): + orchestrator = ExtractionOrchestrator() + request = GenerateContextRequest(paper_id="local:cb", depth="standard") + events: list[tuple[str, int, int]] = [] + + async def _on_stage_complete(stage_name, observations, warnings): + events.append((stage_name, len(observations), len(warnings))) + + raw_paper = RawPaperData( + paper_id="local:cb", + title="Experimental Transformer Study", + abstract="We present a transformer model with improved accuracy.", + year=2026, + full_text=( + "Method\n" + "Transformer blocks with learning rate 1e-4 and batch size 16 for 5 epochs.\n" + "Results\n" + "Accuracy improves to 90.2.\n" + ), + source_adapter="local_file", + ) + + await orchestrator.run(request, raw_paper=raw_paper, on_stage_complete=_on_stage_complete) + + expected_order = ExtractionOrchestrator.resolve_stage_sequence( + "standard", PaperType.EXPERIMENTAL + ) + assert [item[0] for item in events] == expected_order + assert all(obs_count >= 0 for _, obs_count, _ in events) + + +@pytest.mark.asyncio +async def test_deep_mode_uses_stricter_confidence_than_standard(): + orchestrator = ExtractionOrchestrator() + raw_paper = RawPaperData( + paper_id="local:deep", + title="Experimental Transformer Study", + abstract="We present a transformer model with improved accuracy.", + year=2026, + full_text=( + "Method\n" + "Transformer blocks with learning rate 1e-4 and batch size 16 for 5 epochs.\n" + "Results\n" + "Accuracy improves to 90.2.\n" + ), + source_adapter="local_file", + ) + + standard = await orchestrator.run( + GenerateContextRequest(paper_id="local:deep", depth="standard"), + raw_paper=raw_paper, + ) + deep = await orchestrator.run( + GenerateContextRequest(paper_id="local:deep", depth="deep"), + raw_paper=raw_paper, + ) + + assert deep.confidence.overall < standard.confidence.overall + + +def test_calibrate_confidence_penalizes_missing_required_evidence(): + with_evidence = calibrate_confidence( + 0.6, + [ + EvidenceLink( + type="paper_span", + ref="method#char:1-10", + supports=["learning_rate"], + confidence=0.9, + ) + ], + required=True, + ) + without_evidence = calibrate_confidence(0.6, [], required=True) + assert with_evidence > without_evidence + + +@pytest.mark.asyncio +async def test_spec_extract_emits_evidence_links_for_hyperparameters(): + stage = SpecExtractStage() + result = await stage.run( + StageInput( + title="Hyperparam Test", + abstract="", + full_text="Learning rate 1e-4, batch size 32, epochs 8.", + sections={}, + ) + ) + + assert result.observations + obs = result.observations[0] + assert obs.type == "hyperparameter" + assert obs.evidence + assert all(link.supports for link in obs.evidence) + + +@pytest.mark.asyncio +async def test_success_criteria_confidence_is_lower_without_metric_spans(): + stage = SuccessCriteriaStage() + with_metrics = await stage.run( + StageInput( + title="Metric Rich", + abstract="", + full_text="Results show accuracy improves and F1 reaches 0.80.", + sections={}, + ) + ) + without_metrics = await stage.run( + StageInput( + title="Metric Sparse", + abstract="", + full_text="Results are promising according to qualitative feedback.", + sections={}, + ) + ) + + with_conf = with_metrics.observations[0].confidence + without_conf = without_metrics.observations[0].confidence + assert with_conf > without_conf + assert without_metrics.warnings + + +@pytest.mark.asyncio +async def test_section_extractor_handles_numbered_headings_and_offsets(): + extractor = PaperSectionExtractor() + raw = RawPaperData( + paper_id="local:sec1", + title="Sectioned", + full_text=( + "1 Introduction\n" + "Intro text.\n" + "2 Methodology\n" + "We use transformer encoder blocks.\n" + "3 Experiments\n" + "Evaluation on benchmark datasets.\n" + "4 Conclusion\n" + "Final remarks.\n" + ), + ) + + normalized = await extractor.extract(raw) + assert "method" in normalized.sections + assert "results" in normalized.sections + assert "discussion" in normalized.sections + start, end = normalized.section_offsets["method"] + assert start < end + assert "transformer encoder" in normalized.full_text[start:end].lower() + + +@pytest.mark.asyncio +async def test_section_extractor_handles_uppercase_and_colon_headings(): + extractor = PaperSectionExtractor() + raw = RawPaperData( + paper_id="local:sec2", + title="Uppercase headings", + full_text=( + "I. BACKGROUND\n" + "Background details.\n" + "II. METHODS:\n" + "Method details.\n" + "III. EVALUATION\n" + "Evaluation details.\n" + "IV. FUTURE WORK\n" + "Discussion details.\n" + ), + ) + + normalized = await extractor.extract(raw) + assert "introduction" in normalized.sections + assert "method" in normalized.sections + assert "results" in normalized.sections + assert "discussion" in normalized.sections + assert all(section in normalized.section_offsets for section in normalized.sections) + + +@pytest.mark.asyncio +async def test_section_extractor_fallback_segments_noisy_plain_text(): + extractor = PaperSectionExtractor() + raw = RawPaperData( + paper_id="local:sec3", + title="Noisy plain text", + full_text=( + "This paper introduction explains motivation and setup. " + "Our method uses retrieval augmentation with a lightweight transformer. " + "Experiments show strong results on two benchmarks. " + "Conclusion discusses limitations and future work." + ), + ) + + normalized = await extractor.extract(raw) + assert "method" in normalized.sections + assert "results" in normalized.sections + assert normalized.section_offsets.get("method") diff --git a/tests/unit/test_runbook_project_dir_prepare_route.py b/tests/unit/test_runbook_project_dir_prepare_route.py new file mode 100644 index 00000000..e3c46ea3 --- /dev/null +++ b/tests/unit/test_runbook_project_dir_prepare_route.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from pathlib import Path + +from fastapi.testclient import TestClient + +from paperbot.api import main as api_main + + +def test_prepare_project_dir_creates_missing_directory_under_cwd(tmp_path: Path, monkeypatch): + monkeypatch.chdir(tmp_path) + target = tmp_path / "workspace" / "paper-a" + + with TestClient(api_main.app) as client: + resp = client.post( + "/api/runbook/project-dir/prepare", + json={"project_dir": str(target), "create_if_missing": True}, + ) + + assert resp.status_code == 200 + payload = resp.json() + assert payload["project_dir"] == str(target.resolve()) + assert payload["created"] is True + assert target.is_dir() + + +def test_prepare_project_dir_rejects_path_outside_allowed_prefixes(tmp_path: Path, monkeypatch): + monkeypatch.chdir(tmp_path) + disallowed = Path.home().parent / "paperbot-runbook-disallowed" + + with TestClient(api_main.app) as client: + resp = client.post( + "/api/runbook/project-dir/prepare", + json={"project_dir": str(disallowed), "create_if_missing": False}, + ) + + assert resp.status_code == 403 + assert "not allowed" in resp.json()["detail"] + + +def test_prepare_project_dir_rejects_file_path(tmp_path: Path, monkeypatch): + monkeypatch.chdir(tmp_path) + file_path = tmp_path / "not-a-directory.txt" + file_path.write_text("hello", encoding="utf-8") + + with TestClient(api_main.app) as client: + resp = client.post( + "/api/runbook/project-dir/prepare", + json={"project_dir": str(file_path), "create_if_missing": True}, + ) + + assert resp.status_code == 400 + assert "must be a directory" in resp.json()["detail"] diff --git a/web/src/app/api/research/repro/context/[packId]/observation/[observationId]/route.ts b/web/src/app/api/research/repro/context/[packId]/observation/[observationId]/route.ts new file mode 100644 index 00000000..a380e38d --- /dev/null +++ b/web/src/app/api/research/repro/context/[packId]/observation/[observationId]/route.ts @@ -0,0 +1,16 @@ +import { apiBaseUrl, proxyJson } from "../../../../../_base" +import type { NextRequest } from "next/server" + +export const runtime = "nodejs" + +export async function GET( + req: NextRequest, + { params }: { params: Promise<{ packId: string; observationId: string }> } +) { + const { packId, observationId } = await params + return proxyJson( + req, + `${apiBaseUrl()}/api/research/repro/context/${encodeURIComponent(packId)}/observation/${encodeURIComponent(observationId)}`, + "GET" + ) +} diff --git a/web/src/app/api/research/repro/context/[packId]/route.ts b/web/src/app/api/research/repro/context/[packId]/route.ts new file mode 100644 index 00000000..ecf46381 --- /dev/null +++ b/web/src/app/api/research/repro/context/[packId]/route.ts @@ -0,0 +1,13 @@ +import type { NextRequest } from "next/server" + +import { apiBaseUrl, proxyJson } from "../../../_base" + +export async function GET(req: NextRequest, { params }: { params: Promise<{ packId: string }> }) { + const { packId } = await params + return proxyJson(req, `${apiBaseUrl()}/api/research/repro/context/${encodeURIComponent(packId)}`, "GET") +} + +export async function DELETE(req: NextRequest, { params }: { params: Promise<{ packId: string }> }) { + const { packId } = await params + return proxyJson(req, `${apiBaseUrl()}/api/research/repro/context/${encodeURIComponent(packId)}`, "DELETE") +} diff --git a/web/src/app/api/research/repro/context/[packId]/session/route.ts b/web/src/app/api/research/repro/context/[packId]/session/route.ts new file mode 100644 index 00000000..94955e49 --- /dev/null +++ b/web/src/app/api/research/repro/context/[packId]/session/route.ts @@ -0,0 +1,8 @@ +import type { NextRequest } from "next/server" + +import { apiBaseUrl, proxyJson } from "../../../../_base" + +export async function POST(req: NextRequest, { params }: { params: Promise<{ packId: string }> }) { + const { packId } = await params + return proxyJson(req, `${apiBaseUrl()}/api/research/repro/context/${encodeURIComponent(packId)}/session`, "POST") +} diff --git a/web/src/app/api/research/repro/context/route.ts b/web/src/app/api/research/repro/context/route.ts new file mode 100644 index 00000000..c268f631 --- /dev/null +++ b/web/src/app/api/research/repro/context/route.ts @@ -0,0 +1,30 @@ +import { apiBaseUrl, proxyJson } from "../../_base" + +export const runtime = "nodejs" + +export async function GET(req: Request) { + const url = new URL(req.url) + return proxyJson(req, `${apiBaseUrl()}/api/research/repro/context?${url.searchParams.toString()}`, "GET") +} + +export async function POST(req: Request) { + const body = await req.text() + const upstream = await fetch(`${apiBaseUrl()}/api/research/repro/context/generate`, { + method: "POST", + headers: { + "Content-Type": req.headers.get("content-type") || "application/json", + Accept: "text/event-stream", + }, + body, + }) + + const headers = new Headers() + headers.set("Content-Type", upstream.headers.get("content-type") || "text/event-stream") + headers.set("Cache-Control", "no-cache") + headers.set("Connection", "keep-alive") + + return new Response(upstream.body, { + status: upstream.status, + headers, + }) +} diff --git a/web/src/app/api/runbook/allowed-dirs/route.ts b/web/src/app/api/runbook/allowed-dirs/route.ts new file mode 100644 index 00000000..ce48b0fe --- /dev/null +++ b/web/src/app/api/runbook/allowed-dirs/route.ts @@ -0,0 +1,39 @@ +export const runtime = "nodejs" + +function apiBaseUrl() { + return process.env.PAPERBOT_API_BASE_URL || "http://127.0.0.1:8000" +} + +export async function GET() { + const upstream = await fetch(`${apiBaseUrl()}/api/runbook/allowed-dirs`, { + headers: { Accept: "application/json" }, + }) + const text = await upstream.text() + return new Response(text, { + status: upstream.status, + headers: { + "Content-Type": upstream.headers.get("content-type") || "application/json", + "Cache-Control": "no-cache", + }, + }) +} + +export async function POST(req: Request) { + const body = await req.text() + const upstream = await fetch(`${apiBaseUrl()}/api/runbook/allowed-dirs`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body, + }) + const text = await upstream.text() + return new Response(text, { + status: upstream.status, + headers: { + "Content-Type": upstream.headers.get("content-type") || "application/json", + "Cache-Control": "no-cache", + }, + }) +} diff --git a/web/src/app/api/runbook/project-dir/prepare/route.ts b/web/src/app/api/runbook/project-dir/prepare/route.ts new file mode 100644 index 00000000..cebeec42 --- /dev/null +++ b/web/src/app/api/runbook/project-dir/prepare/route.ts @@ -0,0 +1,25 @@ +export const runtime = "nodejs" + +function apiBaseUrl() { + return process.env.PAPERBOT_API_BASE_URL || "http://127.0.0.1:8000" +} + +export async function POST(req: Request) { + const body = await req.text() + const upstream = await fetch(`${apiBaseUrl()}/api/runbook/project-dir/prepare`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body, + }) + const text = await upstream.text() + return new Response(text, { + status: upstream.status, + headers: { + "Content-Type": upstream.headers.get("content-type") || "application/json", + "Cache-Control": "no-cache", + }, + }) +} diff --git a/web/src/app/papers/[id]/page.tsx b/web/src/app/papers/[id]/page.tsx index 4a626dcd..1a98dae7 100644 --- a/web/src/app/papers/[id]/page.tsx +++ b/web/src/app/papers/[id]/page.tsx @@ -34,7 +34,7 @@ export default async function PaperPage({ params }: { params: Promise<{ id: stri Chat diff --git a/web/src/app/studio/page.tsx b/web/src/app/studio/page.tsx index f2dfd2f3..aa2c2bed 100644 --- a/web/src/app/studio/page.tsx +++ b/web/src/app/studio/page.tsx @@ -8,11 +8,43 @@ import { ReproductionLog } from "@/components/studio/ReproductionLog" import { FilesPanel } from "@/components/studio/FilesPanel" import { MCPProvider } from "@/lib/mcp" import { useStudioStore } from "@/lib/store/studio-store" +import { useContextPackGeneration } from "@/hooks/useContextPackGeneration" +import type { ReproContextPack } from "@/lib/types/p2c" import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs" import { ResizablePanelGroup, ResizablePanel, ResizableHandle } from "@/components/ui/resizable" +function isReproContextPack(payload: unknown): payload is ReproContextPack { + if (!payload || typeof payload !== "object") return false + const value = payload as Partial & { + paper?: { paper_id?: unknown } + confidence?: { overall?: unknown } + } + return ( + typeof value.context_pack_id === "string" && + typeof value.version === "string" && + typeof value.created_at === "string" && + typeof value.objective === "string" && + typeof value.paper_type === "string" && + Array.isArray(value.observations) && + Array.isArray(value.task_roadmap) && + Array.isArray(value.warnings) && + typeof value.paper?.paper_id === "string" && + typeof value.confidence?.overall === "number" + ) +} + function StudioContent() { - const { addPaper, selectPaper, loadPapers, papers } = useStudioStore() + const { + addPaper, + selectPaper, + loadPapers, + papers, + setContextPack, + setContextPackLoading, + setContextPackError, + clearGenerationProgress, + } = useStudioStore() + const { generate } = useContextPackGeneration() const searchParams = useSearchParams() const router = useRouter() const hasProcessedParams = useRef(false) @@ -23,16 +55,34 @@ function StudioContent() { // eslint-disable-next-line react-hooks/exhaustive-deps }, []) - // Handle URL params for paper_id (from /papers page "Run Reproduction" button) + const normalizePack = (payload: unknown): ReproContextPack | null => { + if (!payload || typeof payload !== "object") return null + const raw = payload as Record + if (raw.pack && typeof raw.pack === "object") { + const pack = raw.pack as Record + const maybePack: Record = + !pack.context_pack_id && typeof raw.context_pack_id === "string" + ? { ...pack, context_pack_id: raw.context_pack_id } + : pack + return isReproContextPack(maybePack) ? maybePack : null + } + return isReproContextPack(raw) ? raw : null + } + + // Handle URL params from the Papers detail entry point and context pack deep links useEffect(() => { if (hasProcessedParams.current) return const paperId = searchParams.get("paper_id") const title = searchParams.get("title") const abstract = searchParams.get("abstract") + const generateFlag = searchParams.get("generate") === "true" + const contextPackId = searchParams.get("context_pack_id") + console.info("[P2C:M3] studio:params", { paperId, generateFlag, contextPackId }) + + let shouldCleanUrl = false if (paperId) { - hasProcessedParams.current = true const existingPaper = papers.find(p => p.id === paperId) if (existingPaper) { selectPaper(paperId) @@ -42,17 +92,62 @@ function StudioContent() { abstract: abstract || "", }) } - router.replace("/studio", { scroll: false }) + shouldCleanUrl = true } else if (title && abstract) { - hasProcessedParams.current = true addPaper({ title, abstract }) + shouldCleanUrl = true + } + + if (contextPackId) { + shouldCleanUrl = true + setContextPackLoading(true) + setContextPackError(null) + clearGenerationProgress() + fetch(`/api/research/repro/context/${contextPackId}`) + .then((res) => { + if (!res.ok) { + throw new Error(`Failed to load context pack (${res.status})`) + } + return res.json() + }) + .then((payload) => { + const pack = normalizePack(payload) + if (pack) setContextPack(pack) + }) + .catch((err) => setContextPackError(err instanceof Error ? err.message : String(err))) + .finally(() => setContextPackLoading(false)) + } + + if (generateFlag) { + shouldCleanUrl = true + if (paperId) { + console.info("[P2C:M3] studio:trigger_generate", { paperId }) + generate({ paperId }) + } else { + console.warn("[P2C:M3] studio:missing_paper_id") + setContextPackError("Missing paper_id for generation.") + } + } + + if (shouldCleanUrl) { + hasProcessedParams.current = true router.replace("/studio", { scroll: false }) } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [searchParams, papers]) + }, [ + addPaper, + clearGenerationProgress, + generate, + papers, + router, + searchParams, + selectPaper, + setContextPack, + setContextPackError, + setContextPackLoading, + ]) return ( -
+
{/* Top Bar - minimal */}
diff --git a/web/src/components/studio/ContextPackPanel.tsx b/web/src/components/studio/ContextPackPanel.tsx new file mode 100644 index 00000000..358ac8ce --- /dev/null +++ b/web/src/components/studio/ContextPackPanel.tsx @@ -0,0 +1,338 @@ +"use client" + +import { useEffect, useMemo, useState } from "react" +import { AlertTriangle, ChevronDown, ChevronRight } from "lucide-react" +import { Badge } from "@/components/ui/badge" +import { Button } from "@/components/ui/button" +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card" +import type { + ContextPackSession, + ExtractionObservation, + ObservationConcept, + ObservationType, + ReproContextPack, +} from "@/lib/types/p2c" + +const CONFIDENCE_STYLES: Array<{ max: number; className: string }> = [ + { max: 0.6, className: "bg-red-100 text-red-700" }, + { max: 0.8, className: "bg-amber-100 text-amber-700" }, + { max: 1.1, className: "bg-emerald-100 text-emerald-700" }, +] + +interface Props { + pack: ReproContextPack + onSessionCreated?: (session: ContextPackSession) => void +} + +export function ContextPackPanel({ pack, onSessionCreated }: Props) { + const [creating, setCreating] = useState(false) + const [error, setError] = useState(null) + const [expandedId, setExpandedId] = useState(null) + const [filter, setFilter] = useState<{ stage: string | null; type: ObservationType | null; concept: ObservationConcept | null }>({ + stage: null, + type: null, + concept: null, + }) + + const confidenceStyle = CONFIDENCE_STYLES.find((entry) => pack.confidence.overall <= entry.max) + ?.className || "bg-muted text-muted-foreground" + + const stages = useMemo(() => Array.from(new Set(pack.observations.map((obs) => obs.stage))), [pack.observations]) + const types = useMemo(() => Array.from(new Set(pack.observations.map((obs) => obs.type))), [pack.observations]) + const concepts = useMemo( + () => Array.from(new Set(pack.observations.flatMap((obs) => obs.concepts))), + [pack.observations] + ) + + const filteredObservations = useMemo(() => { + return pack.observations.filter((obs) => { + if (filter.stage && obs.stage !== filter.stage) return false + if (filter.type && obs.type !== filter.type) return false + if (filter.concept && !obs.concepts.includes(filter.concept)) return false + return true + }) + }, [pack.observations, filter]) + + const handleCreateSession = async () => { + setCreating(true) + setError(null) + try { + const res = await fetch(`/api/research/repro/context/${pack.context_pack_id}/session`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ executor_preference: "auto" }), + }) + + if (!res.ok) { + const text = await res.text() + throw new Error(text || `Failed to create session (${res.status})`) + } + + const session = (await res.json()) as ContextPackSession + onSessionCreated?.(session) + } catch (err) { + setError(err instanceof Error ? err.message : "Failed to create session") + } finally { + setCreating(false) + } + } + + return ( +
+ + + {pack.paper.title} +
+ {pack.paper.year} + • + {pack.paper.authors.join(", ")} + {pack.paper_type} + Confidence {Math.round(pack.confidence.overall * 100)}% + {pack.observations.length} observations +
+ + +
+

Objective

+

{pack.objective}

+
+ + {pack.warnings.length > 0 && ( +
+
+ + Warnings +
+
    + {pack.warnings.map((warning) => ( +
  • {warning}
    • + ))} +
+
+ )} + + + +
+ setFilter((prev) => ({ ...prev, stage: value }))} + /> + setFilter((prev) => ({ ...prev, type: value as ObservationType | null }))} + /> + setFilter((prev) => ({ ...prev, concept: value as ObservationConcept | null }))} + /> +
+ +
+ {filteredObservations.map((obs) => ( + setExpandedId(expandedId === obs.id ? null : obs.id)} + /> + ))} + {filteredObservations.length === 0 && ( +

No observations match the current filters.

+ )} +
+ + {pack.task_roadmap.length > 0 && ( + + + Roadmap + + + {pack.task_roadmap.map((step) => ( +
+
+ {step.title} + {step.estimated_difficulty} +
+ {step.description &&

{step.description}

} +
+ ))} + + + )} + + {error &&

{error}

} + +
+ +
+
+ ) +} + +function FilterGroup({ + label, + options, + selected, + onSelect, +}: { + label: string + options: string[] + selected: string | null + onSelect: (value: string | null) => void +}) { + return ( +
+ {label}: + + {options.map((option) => ( + + ))} +
+ ) +} + +function ObservationCard({ + packId, + observation, + expanded, + onToggle, +}: { + packId: string + observation: ExtractionObservation + expanded: boolean + onToggle: () => void +}) { + const { detail, loading, error } = useObservationDetail(packId, expanded ? observation.id : null) + const confidence = Math.round(observation.confidence * 100) + const dotClass = + observation.confidence >= 0.8 + ? "bg-emerald-500" + : observation.confidence >= 0.6 + ? "bg-amber-500" + : "bg-red-500" + + return ( + + +
+ {observation.type} + {observation.stage} + + + {confidence}% + +
+

{observation.title}

+

{observation.narrative}

+
+ {observation.concepts.map((concept) => ( + + {concept} + + ))} +
+ + {expanded && ( +
+ {loading &&

Loading details...

} + {error &&

{error}

} + {!loading && detail?.structured_data && Object.keys(detail.structured_data).length > 0 && ( + 
+                {JSON.stringify(detail.structured_data, null, 2)}
+
+ )} + {!loading && detail?.evidence && detail.evidence.length > 0 && ( +
+

Evidence

+ {detail.evidence?.map((ev, idx) => ( +
+ {ev.type} +
+
{ev.ref}
+ {ev.supports?.length ? ( +
supports: {ev.supports.join(", ")}
+ ) : null} +
+
+ ))} +
+ )} + {!loading && !error && (!detail?.structured_data || Object.keys(detail.structured_data).length === 0) && (!detail?.evidence || detail.evidence.length === 0) && ( +

No additional details yet.

+ )} +
+ )} + + + ) +} + +function useObservationDetail(packId: string, observationId: string | null) { + const [detail, setDetail] = useState(null) + const [loading, setLoading] = useState(false) + const [error, setError] = useState(null) + + useEffect(() => { + if (!observationId) { + setDetail(null) + setLoading(false) + setError(null) + return + } + + let cancelled = false + setLoading(true) + setError(null) + + fetch(`/api/research/repro/context/${packId}/observation/${observationId}`) + .then(async (res) => { + if (!res.ok) { + const text = await res.text() + throw new Error(text || `Failed to load observation (${res.status})`) + } + return res.json() as Promise + }) + .then((data) => { + if (!cancelled) setDetail(data) + }) + .catch((err) => { + if (!cancelled) setError(err instanceof Error ? err.message : "Failed to load observation") + }) + .finally(() => { + if (!cancelled) setLoading(false) + }) + + return () => { + cancelled = true + } + }, [packId, observationId]) + + return { detail, loading, error } +} diff --git a/web/src/components/studio/FilesPanel.tsx b/web/src/components/studio/FilesPanel.tsx index 9093e816..5ba70250 100644 --- a/web/src/components/studio/FilesPanel.tsx +++ b/web/src/components/studio/FilesPanel.tsx @@ -25,6 +25,30 @@ type FileIndexResponse = { truncated?: boolean } +type PrepareProjectDirResponse = { + project_dir: string + created?: boolean +} + +async function readErrorDetail(res: Response): Promise { + const cloned = res.clone() + try { + const data = await cloned.json() as { detail?: string } + if (typeof data?.detail === "string" && data.detail.trim()) { + return data.detail + } + } catch { + // Fall back to plain text below. + } + try { + const text = await res.text() + if (text.trim()) return text + } catch { + // ignore + } + return `Request failed (${res.status})` +} + function languageForPath(path: string): string { const lower = path.toLowerCase() if (lower.endsWith(".py")) return "python" @@ -174,6 +198,9 @@ export function FilesPanel() { const [expandedDirs, setExpandedDirs] = useState>(new Set()) const [dirDialogOpen, setDirDialogOpen] = useState(false) const [newDirPath, setNewDirPath] = useState("") + const [dirError, setDirError] = useState(null) + const [indexError, setIndexError] = useState(null) + const [settingDir, setSettingDir] = useState(false) const filteredFiles = useMemo(() => { const q = query.trim().toLowerCase() @@ -186,9 +213,15 @@ export function FilesPanel() { const refreshIndex = async () => { if (!projectDir) return setLoadingIndex(true) + setIndexError(null) try { const res = await fetch(`/api/runbook/files?project_dir=${encodeURIComponent(projectDir)}&recursive=true`) - if (!res.ok) return + if (!res.ok) { + const detail = await readErrorDetail(res) + setFileIndex([]) + setIndexError(detail) + return + } const data = (await res.json()) as FileIndexResponse setFileIndex(data.files || []) const firstLevelDirs = new Set() @@ -208,6 +241,7 @@ export function FilesPanel() { setFileIndex([]) setQuery("") setExpandedDirs(new Set()) + setIndexError(null) if (projectDir) refreshIndex() // eslint-disable-next-line react-hooks/exhaustive-deps }, [projectDir]) @@ -233,15 +267,39 @@ export function FilesPanel() { }) } - const handleSetDirectory = () => { + const handleSetDirectory = async () => { if (!selectedPaperId || !newDirPath.trim()) return - updatePaper(selectedPaperId, { outputDir: newDirPath.trim() }) - setDirDialogOpen(false) - setNewDirPath("") + setSettingDir(true) + setDirError(null) + try { + const res = await fetch("/api/runbook/project-dir/prepare", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + project_dir: newDirPath.trim(), + create_if_missing: true, + }), + }) + if (!res.ok) { + const detail = await readErrorDetail(res) + setDirError(detail) + return + } + const data = (await res.json()) as PrepareProjectDirResponse + updatePaper(selectedPaperId, { outputDir: data.project_dir || newDirPath.trim() }) + setDirDialogOpen(false) + setNewDirPath("") + setDirError(null) + } catch { + setDirError("Failed to validate directory. Please try again.") + } finally { + setSettingDir(false) + } } const openDirDialog = () => { setNewDirPath(projectDir || "") + setDirError(null) setDirDialogOpen(true) } @@ -288,6 +346,9 @@ export function FilesPanel() { disabled={!projectDir} />
+ {indexError && ( +

{indexError}

+ )}
{/* File Tree */} @@ -356,14 +417,17 @@ export function FilesPanel() {

This directory will be used to save generated code and can be accessed by Claude CLI.

+ {dirError && ( +

{dirError}

+ )}
- diff --git a/web/src/components/studio/GenerationProgressPanel.tsx b/web/src/components/studio/GenerationProgressPanel.tsx new file mode 100644 index 00000000..bce72458 --- /dev/null +++ b/web/src/components/studio/GenerationProgressPanel.tsx @@ -0,0 +1,109 @@ +"use client" + +import { CheckCircle2, Circle, Loader2, AlertCircle } from "lucide-react" +import { Badge } from "@/components/ui/badge" +import { Progress } from "@/components/ui/progress" +import type { StageObservationsEvent, StageProgressEvent } from "@/lib/types/p2c" +import { cn } from "@/lib/utils" + +const STAGES: Array<{ key: string; label: string }> = [ + { key: "literature_distill", label: "Literature Distill" }, + { key: "blueprint_extract", label: "Blueprint Extract" }, + { key: "environment_extract", label: "Environment Extract" }, + { key: "spec_extract", label: "Spec & Hyperparams" }, + { key: "roadmap_planning", label: "Roadmap Planning" }, + { key: "success_criteria", label: "Success Criteria" }, +] + +interface Props { + stages: StageProgressEvent[] + liveObservations?: StageObservationsEvent[] + error?: string | null +} + +export function GenerationProgressPanel({ stages, liveObservations = [], error }: Props) { + const lastStage = stages.length > 0 ? stages[stages.length - 1] : null + const currentStage = lastStage?.stage || null + const currentIndex = currentStage + ? STAGES.findIndex((stage) => stage.key === currentStage) + : -1 + const overallProgress = lastStage ? Math.round(lastStage.progress * 100) : 0 + + const stageLabel = (stageKey: string) => + STAGES.find((stage) => stage.key === stageKey)?.label || stageKey + + return ( +
+
+

Generating Reproduction Context...

+ {lastStage?.message && ( +

{lastStage.message}

+ )} +
+ + {error && ( +
+ + {error} +
+ )} + +
+ {STAGES.map((stage, index) => { + const isComplete = currentIndex > index || (currentIndex === index && overallProgress >= 100) + const isActive = currentIndex === index && !isComplete + const Icon = isComplete ? CheckCircle2 : isActive ? Loader2 : Circle + + return ( +
+ + {stage.label} +
+ ) + })} +
+ +
+
+ Overall progress + {overallProgress}% +
+ +
+ +
+
Observation initialization
+ {liveObservations.length === 0 ? ( +

Waiting for observations…

+ ) : ( +
+ {liveObservations.map((stage) => ( +
+
{stageLabel(stage.stage)}
+
+ {stage.observations.map((obs) => ( +
+ {obs.type} + {obs.title} + {Math.round(obs.confidence * 100)}% +
+ ))} + {stage.observations.length === 0 && ( +
No observations yet.
+ )} +
+
+ ))} +
+ )} +
+
+ ) +} diff --git a/web/src/components/studio/ReproductionLog.tsx b/web/src/components/studio/ReproductionLog.tsx index 25f1d6c6..f674ad5e 100644 --- a/web/src/components/studio/ReproductionLog.tsx +++ b/web/src/components/studio/ReproductionLog.tsx @@ -1,6 +1,6 @@ "use client" -import { useMemo, useRef, useState } from "react" +import { useEffect, useMemo, useState } from "react" import { Button } from "@/components/ui/button" import { ScrollArea } from "@/components/ui/scroll-area" import { Textarea } from "@/components/ui/textarea" @@ -10,9 +10,10 @@ import { readSSE } from "@/lib/sse" import { CodeBlock } from "@/components/ai-elements" import { DiffModal } from "./DiffViewer" import { WorkspaceSetupDialog } from "./WorkspaceSetupDialog" +import { ContextPackPanel } from "./ContextPackPanel" +import { GenerationProgressPanel } from "./GenerationProgressPanel" import { cn } from "@/lib/utils" import { - Sparkles, CheckCircle2, AlertCircle, FileText, @@ -27,18 +28,21 @@ import { X, Save, Send, - Paperclip, Code, + Activity, + Package, + MessageSquare, } from "lucide-react" import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" import Editor from "@monaco-editor/react" import { useTheme } from "next-themes" +import type { ContextPackSession } from "@/lib/types/p2c" type StepStatus = "idle" | "running" | "success" | "error" type Mode = "Code" | "Plan" | "Ask" const actionIcons: Record = { - thinking: Sparkles, + thinking: Loader2, file_change: FileCode, function_call: Wrench, error: AlertCircle, @@ -163,6 +167,11 @@ export function ReproductionLog() { activeTaskId, selectedPaperId, lastGenCodeResult, + contextPack, + contextPackLoading, + contextPackError, + generationProgress, + liveObservations, addTask, addAction, updateTaskStatus, @@ -180,7 +189,6 @@ export function ReproductionLog() { ) const [status, setStatus] = useState("idle") - const runIdRef = useRef(null) const [mode, setMode] = useState("Code") const [model, setModel] = useState("claude-sonnet-4-5") const [lastError, setLastError] = useState(null) @@ -188,11 +196,20 @@ export function ReproductionLog() { const [saving, setSaving] = useState(false) const [messageInput, setMessageInput] = useState("") const [showWorkspaceSetup, setShowWorkspaceSetup] = useState(false) - const [pendingAction, setPendingAction] = useState<"generate" | "install" | "chat" | null>(null) + const [pendingAction, setPendingAction] = useState<"chat" | null>(null) + const [viewMode, setViewMode] = useState<"log" | "generating" | "context_pack">("log") + + // Switch to "generating" view when generation starts + useEffect(() => { + if (contextPackLoading) { + setViewMode("generating") + } + }, [contextPackLoading]) + + // Do not auto-switch away from "generating"; keep it open until the user changes tabs. const activeTask = tasks.find(t => t.id === activeTaskId) const projectDir = selectedPaper?.outputDir || lastGenCodeResult?.outputDir || null - const canRun = (selectedPaper?.title.trim().length ?? 0) > 0 && (selectedPaper?.abstract.trim().length ?? 0) > 0 const isBusy = status === "running" const saveActiveFile = async () => { @@ -221,182 +238,12 @@ export function ReproductionLog() { if (selectedPaperId) { updatePaper(selectedPaperId, { outputDir: directory }) } - // Execute the pending action - if (pendingAction === "generate") { - runPaper2CodeWithDir(directory) - } else if (pendingAction === "install") { - runInstallWithDir(directory) - } else if (pendingAction === "chat") { + if (pendingAction === "chat") { runChatWithDir(directory) } setPendingAction(null) } - const runPaper2Code = async () => { - if (!selectedPaper || !canRun || isBusy) return - - // If no project directory, show workspace setup dialog - if (!projectDir) { - setPendingAction("generate") - setShowWorkspaceSetup(true) - return - } - - runPaper2CodeWithDir(projectDir) - } - - const runPaper2CodeWithDir = async (targetDir: string) => { - if (!selectedPaper || !canRun) return - - setStatus("running") - setLastError(null) - - if (selectedPaperId) { - updatePaper(selectedPaperId, { status: 'generating' }) - } - - const taskId = addTask(`Paper2Code — ${selectedPaper.title.slice(0, 40)}${selectedPaper.title.length > 40 ? "…" : ""}`) - addAction(taskId, { type: "thinking", content: "Starting Paper2Code run…" }) - runIdRef.current = null - let outputDir: string | undefined - - try { - const res = await fetch("/api/gen-code", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - title: selectedPaper.title, - abstract: selectedPaper.abstract, - method_section: selectedPaper.methodSection || undefined, - use_orchestrator: true, - use_rag: true, - }), - }) - - if (!res.ok || !res.body) { - throw new Error(`Failed to start run (${res.status})`) - } - - updateTaskStatus(taskId, "running") - - for await (const evt of readSSE(res.body)) { - if (evt?.type === "progress") { - const data = (evt.data ?? {}) as { phase?: string; message?: string; run_id?: string } - if (data.run_id && !runIdRef.current) { - runIdRef.current = data.run_id - } - addAction(taskId, { - type: "thinking", - content: `${data.phase ? `[${data.phase}] ` : ""}${data.message || "Working…"}`, - }) - } else if (evt?.type === "result") { - const result = evt.data as GenCodeResult - outputDir = result?.outputDir - setLastGenCodeResult(result) - addAction(taskId, { type: "complete", content: "Run completed" }) - updateTaskStatus(taskId, "completed") - if (selectedPaperId && outputDir) { - updatePaper(selectedPaperId, { status: 'ready', outputDir }) - } - setStatus("success") - } else if (evt?.type === "error") { - addAction(taskId, { type: "error", content: evt.message || "Run failed" }) - updateTaskStatus(taskId, "error") - if (selectedPaperId) updatePaper(selectedPaperId, { status: 'error' }) - setLastError(evt.message || "Run failed") - setStatus("error") - return - } - } - } catch (e) { - const message = e instanceof Error ? e.message : String(e) - addAction(taskId, { type: "error", content: message }) - updateTaskStatus(taskId, "error") - if (selectedPaperId) updatePaper(selectedPaperId, { status: 'error' }) - setLastError(message) - setStatus("error") - } - } - - const runInstall = async () => { - if (isBusy) return - - // If no project directory, show workspace setup dialog - if (!projectDir) { - if (!selectedPaper) { - setLastError("Select or create a paper first.") - return - } - setPendingAction("install") - setShowWorkspaceSetup(true) - return - } - - runInstallWithDir(projectDir) - } - - const runInstallWithDir = async (targetDir: string) => { - if (!targetDir) return - - setStatus("running") - setLastError(null) - - const taskId = addTask(`Install — ${targetDir.split("/").pop()}`) - addAction(taskId, { type: "thinking", content: "Installing dependencies via Claude..." }) - - try { - // Use Claude chat to run the install command - const res = await fetch("/api/studio/chat", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - message: "Please install the project dependencies. Look for requirements.txt, setup.py, or pyproject.toml and run the appropriate pip install command.", - mode: "Code", - model, - paper: selectedPaper ? { - title: selectedPaper.title, - abstract: selectedPaper.abstract, - method_section: selectedPaper.methodSection, - } : undefined, - project_dir: targetDir, - }), - }) - - if (!res.ok || !res.body) { - throw new Error(`Failed to start install (${res.status})`) - } - - updateTaskStatus(taskId, "running") - - for await (const evt of readSSE(res.body)) { - if (evt?.type === "progress") { - const data = (evt.data ?? {}) as { delta?: string; content?: string; phase?: string; message?: string } - if (data.delta) { - addAction(taskId, { type: "text", content: data.delta }) - } else if (data.message) { - addAction(taskId, { type: "thinking", content: data.message }) - } - } else if (evt?.type === "result") { - addAction(taskId, { type: "complete", content: "Install completed" }) - updateTaskStatus(taskId, "completed") - setStatus("success") - } else if (evt?.type === "error") { - addAction(taskId, { type: "error", content: evt.message || "Install failed" }) - updateTaskStatus(taskId, "error") - setLastError(evt.message || "Install failed") - setStatus("error") - return - } - } - } catch (e) { - const message = e instanceof Error ? e.message : String(e) - addAction(taskId, { type: "error", content: message }) - updateTaskStatus(taskId, "error") - setLastError(message) - setStatus("error") - } - } - const runChatWithDir = async (targetDir: string) => { // Chat with specified directory - called after workspace setup const message = messageInput.trim() @@ -427,6 +274,7 @@ export function ReproductionLog() { const handleSendMessageWithDir = async (message: string, targetDir?: string) => { setStatus("running") setLastError(null) + setViewMode("log") const taskId = addTask(`Chat — ${message.slice(0, 30)}${message.length > 30 ? "…" : ""}`) addAction(taskId, { type: "thinking", content: `[${mode}] Sending to Claude...` }) @@ -484,38 +332,67 @@ export function ReproductionLog() { } } + const handleSessionCreated = (session: ContextPackSession) => { + setViewMode("log") + if (session.initial_prompt) { + setMessageInput(session.initial_prompt) + } + } + return (
- {/* Simplified Action Bar */} -
- - + {/* Tab Navigation */} +
+ {([ + { key: "generating" as const, label: "Progress", icon: Activity }, + { key: "context_pack" as const, label: "Context Pack", icon: Package }, + { key: "log" as const, label: "Chat", icon: MessageSquare }, + ]).map(({ key, label, icon: TabIcon }) => ( + + ))}
{/* Error banner */} - {lastError && ( + {(lastError || contextPackError) && (
- {lastError} + {contextPackError || lastError}
)} {/* Main content area */}
- {activeFileData ? ( + {viewMode === "generating" ? ( + + ) : viewMode === "context_pack" ? ( + contextPack ? ( + + ) : ( +
+ +

No context pack available yet

+
+ ) + ) : activeFileData ? ( /* File Viewer */
@@ -563,19 +440,19 @@ export function ReproductionLog() {
) : ( - /* Timeline */ + /* Chat Timeline */
{!activeTask || activeTask.actions.length === 0 ? (
- +
-

Ready to reproduce

+

Ready to chat

{selectedPaper - ? "Click Generate code to generate code from the selected paper" + ? "Send a message to start working with Claude on this paper" : "Select or create a paper to get started"}

diff --git a/web/src/components/studio/WorkspaceSetupDialog.tsx b/web/src/components/studio/WorkspaceSetupDialog.tsx index 77103625..788fc018 100644 --- a/web/src/components/studio/WorkspaceSetupDialog.tsx +++ b/web/src/components/studio/WorkspaceSetupDialog.tsx @@ -12,7 +12,7 @@ import { DialogHeader, DialogTitle, } from "@/components/ui/dialog" -import { Folder, FolderOpen, Loader2, Circle, CheckCircle2 } from "lucide-react" +import { Folder, FolderOpen, Loader2, Circle, CheckCircle2, ShieldAlert } from "lucide-react" import { StudioPaper } from "@/lib/store/studio-store" import { cn } from "@/lib/utils" @@ -34,6 +34,8 @@ export function WorkspaceSetupDialog({ const [choice, setChoice] = useState<"current" | "custom">("current") const [customDir, setCustomDir] = useState("") const [error, setError] = useState(null) + const [validating, setValidating] = useState(false) + const [pendingAllowDir, setPendingAllowDir] = useState(null) // Fetch current working directory on mount useEffect(() => { @@ -62,13 +64,84 @@ export function WorkspaceSetupDialog({ } }, [open, paper.title]) - const handleConfirm = () => { + const prepareDirectory = async (directory: string): Promise => { + const res = await fetch("/api/runbook/project-dir/prepare", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + project_dir: directory, + create_if_missing: true, + }), + }) + if (!res.ok) { + if (res.status === 403) { + // Extract parent directory to add as allowed prefix + const parts = directory.replace(/\/+$/, "").split("/") + const parentDir = parts.length > 1 ? parts.slice(0, -1).join("/") : directory + setPendingAllowDir(parentDir) + return false + } + const text = await res.text() + try { + const payload = JSON.parse(text) as { detail?: string } + setError(payload.detail || `Directory is not available (${res.status})`) + } catch { + setError(text || `Directory is not available (${res.status})`) + } + return false + } + const data = await res.json() as { project_dir?: string } + onConfirm(data.project_dir || directory) + return true + } + + const handleConfirm = async () => { const directory = choice === "current" ? currentDir : customDir.trim() if (!directory) { setError("Please enter a valid directory path") return } - onConfirm(directory) + setValidating(true) + setError(null) + setPendingAllowDir(null) + try { + await prepareDirectory(directory) + } catch { + setError("Failed to validate directory path") + } finally { + setValidating(false) + } + } + + const handleAllowDir = async () => { + if (!pendingAllowDir) return + setValidating(true) + setError(null) + try { + const res = await fetch("/api/runbook/allowed-dirs", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ directory: pendingAllowDir }), + }) + if (!res.ok) { + const text = await res.text() + try { + const payload = JSON.parse(text) as { detail?: string } + setError(payload.detail || "Failed to add directory to allowed list") + } catch { + setError(text || "Failed to add directory to allowed list") + } + return + } + setPendingAllowDir(null) + // Retry the original prepare call + const directory = choice === "current" ? currentDir : customDir.trim() + await prepareDirectory(directory) + } catch { + setError("Failed to add directory to allowed list") + } finally { + setValidating(false) + } } const selectedDir = choice === "current" ? currentDir : customDir @@ -93,6 +166,26 @@ export function WorkspaceSetupDialog({
) : (
+ {pendingAllowDir && ( +
+
+ +
+ This directory is outside the allowed workspace paths. + Allow access to {pendingAllowDir}? +
+
+
+ + +
+
+ )} + {error && (
{error} @@ -182,8 +275,8 @@ export function WorkspaceSetupDialog({ - diff --git a/web/src/hooks/useContextPackGeneration.ts b/web/src/hooks/useContextPackGeneration.ts new file mode 100644 index 00000000..595a54e7 --- /dev/null +++ b/web/src/hooks/useContextPackGeneration.ts @@ -0,0 +1,161 @@ +"use client" + +import { useCallback, useState } from "react" +import { readSSE } from "@/lib/sse" +import { useStudioStore } from "@/lib/store/studio-store" +import type { + GenerateCompletedEvent, + GenerateErrorEvent, + GenerationStatus, + ReproContextPack, + StageObservationsEvent, + StageProgressEvent, +} from "@/lib/types/p2c" + +function isReproContextPack(payload: unknown): payload is ReproContextPack { + if (!payload || typeof payload !== "object") return false + const value = payload as Partial & { + paper?: { paper_id?: unknown } + confidence?: { overall?: unknown } + } + return ( + typeof value.context_pack_id === "string" && + typeof value.version === "string" && + typeof value.created_at === "string" && + typeof value.objective === "string" && + typeof value.paper_type === "string" && + Array.isArray(value.observations) && + Array.isArray(value.task_roadmap) && + Array.isArray(value.warnings) && + typeof value.paper?.paper_id === "string" && + typeof value.confidence?.overall === "number" + ) +} + +export function useContextPackGeneration() { + const [status, setStatus] = useState("idle") + const [result, setResult] = useState(null) + const MIN_LOADING_MS = 1500 + + const { + setContextPack, + setContextPackLoading, + setContextPackError, + appendGenerationProgress, + appendLiveObservations, + clearGenerationProgress, + clearLiveObservations, + } = useStudioStore() + + const normalizePack = (payload: unknown): ReproContextPack | null => { + if (!payload || typeof payload !== "object") return null + const raw = payload as Record + if (raw.pack && typeof raw.pack === "object") { + const pack = raw.pack as Record + const maybePack: Record = + !pack.context_pack_id && typeof raw.context_pack_id === "string" + ? { ...pack, context_pack_id: raw.context_pack_id } + : pack + return isReproContextPack(maybePack) ? maybePack : null + } + return isReproContextPack(raw) ? raw : null + } + + const generate = useCallback(async (params: { + paperId: string + userId?: string + depth?: "fast" | "standard" | "deep" + }) => { + const startedAt = Date.now() + console.info("[P2C:M3] generate:start", { paperId: params.paperId, depth: params.depth }) + setStatus("generating") + setResult(null) + setContextPack(null) + setContextPackError(null) + clearGenerationProgress() + clearLiveObservations() + setContextPackLoading(true) + + try { + const response = await fetch("/api/research/repro/context", { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "text/event-stream", + }, + body: JSON.stringify({ + paper_id: params.paperId, + user_id: params.userId ?? "default", + depth: params.depth ?? "standard", + }), + }) + + if (!response.ok || !response.body) { + console.error("[P2C:M3] generate:request_failed", { status: response.status }) + throw new Error(`Failed to start generation (${response.status})`) + } + + for await (const evt of readSSE(response.body)) { + if (!evt?.type) continue + + if (evt.type === "progress" || evt.type === "stage_progress") { + console.debug("[P2C:M3] generate:progress", evt.data) + appendGenerationProgress(evt.data as StageProgressEvent) + } else if (evt.type === "stage_observations") { + console.debug("[P2C:M3] generate:observations", evt.data) + appendLiveObservations(evt.data as StageObservationsEvent) + } else if (evt.type === "result" || evt.type === "completed") { + const completed = evt.data as GenerateCompletedEvent + console.info("[P2C:M3] generate:completed", completed) + setResult(completed) + setStatus("completed") + + let packPayload: ReproContextPack | null = null + if (completed?.context_pack_id) { + const packRes = await fetch(`/api/research/repro/context/${completed.context_pack_id}`) + if (!packRes.ok) { + console.error("[P2C:M3] generate:pack_fetch_failed", { status: packRes.status }) + throw new Error(`Failed to load context pack (${packRes.status})`) + } + const payload = await packRes.json() + const pack = normalizePack(payload) + if (pack) packPayload = pack + } + + const elapsed = Date.now() - startedAt + if (elapsed < MIN_LOADING_MS) { + await new Promise((resolve) => setTimeout(resolve, MIN_LOADING_MS - elapsed)) + } + + if (packPayload) setContextPack(packPayload) + setContextPackLoading(false) + return + } else if (evt.type === "error") { + const error = evt.data as GenerateErrorEvent + const message = error?.error || error?.message || evt.message || "Generation failed" + console.error("[P2C:M3] generate:error", { message, data: evt.data }) + setContextPackError(message) + setStatus("error") + setContextPackLoading(false) + return + } + } + } catch (err) { + const message = err instanceof Error ? err.message : "Unknown error" + console.error("[P2C:M3] generate:exception", { message }) + setContextPackError(message) + setStatus("error") + setContextPackLoading(false) + } + }, [ + appendGenerationProgress, + appendLiveObservations, + clearGenerationProgress, + clearLiveObservations, + setContextPack, + setContextPackError, + setContextPackLoading, + ]) + + return { status, result, generate } +} diff --git a/web/src/lib/store/studio-store.ts b/web/src/lib/store/studio-store.ts index 6a3d8245..7143b46f 100644 --- a/web/src/lib/store/studio-store.ts +++ b/web/src/lib/store/studio-store.ts @@ -1,4 +1,5 @@ import { create } from 'zustand' +import type { ReproContextPack, StageObservationsEvent, StageProgressEvent } from '@/lib/types/p2c' // Agent Action Types export type ActionType = 'thinking' | 'file_change' | 'function_call' | 'mcp_call' | 'error' | 'complete' | 'text' @@ -113,6 +114,13 @@ interface StudioState { lastGenCodeResult: GenCodeResult | null workspaceSnapshotId: number | null + // P2C state (scoped to selected paper) + contextPack: ReproContextPack | null + contextPackLoading: boolean + contextPackError: string | null + generationProgress: StageProgressEvent[] + liveObservations: StageObservationsEvent[] + // Paper actions addPaper: (paper: Omit) => string updatePaper: (paperId: string, updates: Partial>) => void @@ -130,6 +138,15 @@ interface StudioState { setPaperDraft: (partial: Partial) => void setLastGenCodeResult: (result: GenCodeResult | null) => void setWorkspaceSnapshotId: (snapshotId: number | null) => void + + // P2C actions + setContextPack: (pack: ReproContextPack | null) => void + setContextPackLoading: (loading: boolean) => void + setContextPackError: (error: string | null) => void + appendGenerationProgress: (event: StageProgressEvent) => void + clearGenerationProgress: () => void + appendLiveObservations: (event: StageObservationsEvent) => void + clearLiveObservations: () => void } export const useStudioStore = create((set, get) => ({ @@ -145,6 +162,13 @@ export const useStudioStore = create((set, get) => ({ lastGenCodeResult: null, workspaceSnapshotId: null, + // P2C state + contextPack: null, + contextPackLoading: false, + contextPackError: null, + generationProgress: [], + liveObservations: [], + // Paper actions addPaper: (paper) => { const id = generateId() @@ -171,6 +195,11 @@ export const useStudioStore = create((set, get) => ({ }, lastGenCodeResult: null, workspaceSnapshotId: null, + contextPack: null, + contextPackLoading: false, + contextPackError: null, + generationProgress: [], + liveObservations: [], } }) return id @@ -220,6 +249,10 @@ export const useStudioStore = create((set, get) => ({ workspaceSnapshotId: null, // Clear active task when switching papers activeTaskId: null, + contextPack: null, + contextPackLoading: false, + contextPackError: null, + generationProgress: [], }) }, @@ -325,4 +358,22 @@ export const useStudioStore = create((set, get) => ({ }, setWorkspaceSnapshotId: (snapshotId) => set({ workspaceSnapshotId: snapshotId }), + + setContextPack: (pack) => set({ contextPack: pack }), + setContextPackLoading: (loading) => set({ contextPackLoading: loading }), + setContextPackError: (error) => set({ contextPackError: error }), + appendGenerationProgress: (event) => set((state) => ({ + generationProgress: [...state.generationProgress, event], + })), + clearGenerationProgress: () => set({ generationProgress: [] }), + appendLiveObservations: (event) => set((state) => { + const existingIndex = state.liveObservations.findIndex(item => item.stage === event.stage) + if (existingIndex === -1) { + return { liveObservations: [...state.liveObservations, event] } + } + const updated = [...state.liveObservations] + updated[existingIndex] = event + return { liveObservations: updated } + }), + clearLiveObservations: () => set({ liveObservations: [] }), })) diff --git a/web/src/lib/types/p2c.ts b/web/src/lib/types/p2c.ts new file mode 100644 index 00000000..8fb6049d --- /dev/null +++ b/web/src/lib/types/p2c.ts @@ -0,0 +1,134 @@ +export type PaperType = "experimental" | "theoretical" | "survey" | "benchmark" | "system" + +export type ObservationType = + | "method" + | "architecture" + | "hyperparameter" + | "metric" + | "environment" + | "limitation" + +export type ObservationConcept = + | "core_method" + | "gotcha" + | "trade_off" + | "limitation" + | "baseline" + | "reproduction_hint" + +export interface EvidenceLink { + type: "paper_span" | "table" | "figure" | "code_snippet" | "metadata" + ref: string + supports: string[] + confidence: number +} + +export interface ExtractionObservation { + id: string + stage: string + type: ObservationType + title: string + narrative: string + confidence: number + concepts: ObservationConcept[] + structured_data?: Record + evidence?: EvidenceLink[] +} + +export interface PaperIdentity { + paper_id: string + title: string + year: number + authors: string[] + identifiers: Record +} + +export interface TaskCheckpoint { + id: string + title: string + description: string + acceptance_criteria: string[] + depends_on: string[] + estimated_difficulty: "low" | "medium" | "high" +} + +export interface ConfidenceScores { + overall: number + literature: number + blueprint: number + environment: number + spec: number + roadmap: number + metrics: number +} + +export interface ReproContextPack { + context_pack_id: string + version: string + created_at: string + paper: PaperIdentity + paper_type: PaperType + objective: string + observations: ExtractionObservation[] + task_roadmap: TaskCheckpoint[] + confidence: ConfidenceScores + warnings: string[] +} + +export interface StageProgressEvent { + stage: string + progress: number + message?: string +} + +export interface StageObservationsEvent { + stage: string + observations: Array<{ + id: string + type: ObservationType + title: string + confidence: number + }> +} + +export interface GenerateCompletedEvent { + context_pack_id: string + status: "completed" + summary: string + confidence: ConfidenceScores + warnings: string[] + next_action: "create_repro_session" +} + +export interface GenerateErrorEvent { + error?: string + message?: string + partial_pack_id?: string +} + +export interface ContextPackSummary { + context_pack_id: string + paper_id?: string + paper_title?: string + depth?: string + status: string + confidence_overall: number + warning_count: number + created_at?: string | null +} + +export interface SessionStep { + step_id: string + title: string + command?: string + status?: string +} + +export interface ContextPackSession { + session_id: string + runbook_id?: string + initial_steps: SessionStep[] + initial_prompt: string +} + +export type GenerationStatus = "idle" | "generating" | "completed" | "error"