Feat/harvest aligned base v2

alembic/versions/0008_paper_repos.py

@@ -0,0 +1,100 @@
+"""paper repos enrichment table
+
+Revision ID: 0008_paper_repos
+Revises: 0007_paper_harvest_tables
+Create Date: 2026-02-11
+
+Adds paper_repos table to persist repository enrichment metadata linked to
+canonical paper registry rows.
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import context, op
+
+revision = "0008_paper_repos"
+down_revision = "0007_paper_harvest_tables"
+branch_labels = None
+depends_on = None
+
+
+def _is_offline() -> bool:
+    try:
+        return bool(context.is_offline_mode())
+    except Exception:
+        return False
+
+
+def _insp():
+    return sa.inspect(op.get_bind())
+
+
+def _has_table(name: str) -> bool:
+    return _insp().has_table(name)
+
+
+def _get_indexes(table: str) -> set[str]:
+    idx = set()
+    for i in _insp().get_indexes(table):
+        idx.add(str(i.get("name") or ""))
+    return idx
+
+
+def _create_index(name: str, table: str, cols: list[str]) -> None:
+    if _is_offline():
+        op.create_index(name, table, cols)
+        return
+    if name in _get_indexes(table):
+        return
+    op.create_index(name, table, cols)
+
+
+def upgrade() -> None:
+    if _is_offline() or not _has_table("paper_repos"):
+        op.create_table(
+            "paper_repos",
+            sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
+            sa.Column("paper_id", sa.Integer(), sa.ForeignKey("papers.id"), nullable=False),
+            sa.Column("repo_url", sa.String(length=512), nullable=False, server_default=""),
+            sa.Column("full_name", sa.String(length=256), nullable=False, server_default=""),
+            sa.Column("description", sa.Text(), nullable=False, server_default=""),
+            sa.Column("stars", sa.Integer(), nullable=False, server_default="0"),
+            sa.Column("forks", sa.Integer(), nullable=False, server_default="0"),
+            sa.Column("open_issues", sa.Integer(), nullable=False, server_default="0"),
+            sa.Column("watchers", sa.Integer(), nullable=False, server_default="0"),
+            sa.Column("language", sa.String(length=64), nullable=False, server_default=""),
+            sa.Column("license", sa.String(length=64), nullable=False, server_default=""),
+            sa.Column("archived", sa.Boolean(), nullable=False, server_default=sa.false()),
+            sa.Column("html_url", sa.String(length=512), nullable=False, server_default=""),
+            sa.Column("topics_json", sa.Text(), nullable=False, server_default="[]"),
+            sa.Column("updated_at_remote", sa.DateTime(timezone=True), nullable=True),
+            sa.Column("pushed_at_remote", sa.DateTime(timezone=True), nullable=True),
+            sa.Column("query", sa.String(length=256), nullable=False, server_default=""),
+            sa.Column(
+                "source",
+                sa.String(length=32),
+                nullable=False,
+                server_default="paperscool_repo_enrich",
+            ),
+            sa.Column("metadata_json", sa.Text(), nullable=False, server_default="{}"),
+            sa.Column("synced_at", sa.DateTime(timezone=True), nullable=False),
+            sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+            sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+            sa.UniqueConstraint("paper_id", "repo_url", name="uq_paper_repos_paper_repo"),
+        )
+
+    _create_index("ix_paper_repos_paper_id", "paper_repos", ["paper_id"])
+    _create_index("ix_paper_repos_repo_url", "paper_repos", ["repo_url"])
+    _create_index("ix_paper_repos_full_name", "paper_repos", ["full_name"])
+    _create_index("ix_paper_repos_stars", "paper_repos", ["stars"])
+    _create_index("ix_paper_repos_archived", "paper_repos", ["archived"])
+    _create_index("ix_paper_repos_query", "paper_repos", ["query"])
+    _create_index("ix_paper_repos_source", "paper_repos", ["source"])
+    _create_index("ix_paper_repos_synced_at", "paper_repos", ["synced_at"])
+    _create_index("ix_paper_repos_updated_at_remote", "paper_repos", ["updated_at_remote"])
+    _create_index("ix_paper_repos_pushed_at_remote", "paper_repos", ["pushed_at_remote"])
+
+
+def downgrade() -> None:
+    op.drop_table("paper_repos")

docs/PROJECT_ISSUE_BACKLOG.md

@@ -2,6 +2,11 @@
 
 This file splits unfinished items in `docs/ROADMAP_TODO.md` into issue-ready cards for GitHub Projects.
 
+## Delivery Rule
+
+- One issue -> one commit (`1 issue = 1 commit`)
+- Commit message include issue id, e.g. `feat: agent runtime contract (#41)`
+
 ## Suggested Labels
 
 - `roadmap`
@@ -265,3 +270,149 @@ Custom fields:
 - `Priority` (P0/P1/P2)
 - `Area` (Backend/Frontend/Data/Workflow)
 - `Size` (S/M/L/XL)
+
+
+## Issue 15 - Agent Browser Source Runner
+
+- Title: `[Feature] Add agent-browser source runner with fallback connectors`
+- Labels: `roadmap`, `phase-4`, `backend`, `integration`, `priority-p1`
+- Source TODO:
+  - `新增 Browser Source Runner`
+  - `失败后 fallback 到 API connector`
+  - `DOM 抽取模板化`
+- Scope:
+  - Integrate `vercel-labs/agent-browser` as optional source collector
+  - Support structured extraction for HF Papers / arXiv / OpenReview
+  - Persist capture traces (steps/screenshots) for debugging
+- Acceptance:
+  - Browser collector can produce normalized paper candidates
+  - Fallback path is observable and does not block workflow
+
+## Issue 16 - Browser-Driven Workflow E2E in CI
+
+- Title: `[Feature] Add browser-agent E2E for workflow streaming UX`
+- Labels: `roadmap`, `phase-4`, `frontend`, `infra`, `priority-p1`
+- Source TODO:
+  - `SSE 增量渲染 E2E`
+  - `关键截图和性能指标`
+  - `CI artifacts`
+- Scope:
+  - Run end-to-end test for Search → DailyPaper → Analyze
+  - Assert DAG restore, non-blank loading state, incremental judge/trend render
+  - Upload run traces and screenshots in CI
+- Acceptance:
+  - Failing UX regressions are detectable in CI
+  - Artifacts are attached for triage
+
+## Issue 17 - Platform Benchmark Monitor Agent
+
+- Title: `[Feature] Add benchmark monitor agent for HF/AlphaXiv parity tracking`
+- Labels: `roadmap`, `phase-4`, `product`, `automation`, `priority-p2`
+- Source TODO:
+  - `对标监测 Agent`
+  - `能力差距报告 docs/benchmark/`
+- Scope:
+  - Periodically crawl public product pages and extract capability signals
+  - Generate versioned markdown reports under `docs/benchmark/`
+- Acceptance:
+  - Weekly benchmark report updates automatically
+  - Capability diff is auditable over time
+
+## Issue 18 - Push Preview Validation Agent
+
+- Title: `[Feature] Add browser-agent push preview validation`
+- Labels: `roadmap`, `phase-4`, `backend`, `qa`, `priority-p2`
+- Source TODO:
+  - `邮件/Slack/钉钉渲染预览`
+  - `多端一致性截图`
+- Scope:
+  - Open rendered previews via browser agent and capture screenshots
+  - Check mandatory sections and formatting constraints
+- Acceptance:
+  - Template breakages are detected before daily dispatch
+
+## Issue 19 - Browser Extension Smoke Automation
+
+- Title: `[Feature] Add browser extension smoke tests on arXiv pages`
+- Labels: `roadmap`, `phase-4`, `frontend`, `qa`, `priority-p2`
+- Source TODO:
+  - `Browser Extension smoke test`
+- Scope:
+  - Validate content script injection and action flows on arXiv paper pages
+  - Cover CTA button, detail sheet, and API roundtrip
+- Acceptance:
+  - Core extension flow is tested in CI/nightly
+
+## Issue 20 - Browser Agent Security & Rate Control
+
+- Title: `[Feature] Harden browser-agent secrets, auditing, and rate control`
+- Labels: `roadmap`, `phase-4`, `security`, `infra`, `priority-p1`
+- Source TODO:
+  - `session 密钥管理`
+  - `Agent 审计日志`
+  - `速率限制/并发隔离`
+- Scope:
+  - Implement secure secret injection for browser sessions
+  - Add auditable logs for domain/action/latency/error
+  - Add source-level concurrency and throttling policies
+- Acceptance:
+  - No plaintext secrets in repo/runtime logs
+  - Browser automation runs are traceable and throttled
+
+## Issue 21 - Agent Inventory and Boundary Map
+
+- Title: `[Chore] Build agent inventory and responsibility boundary map`
+- Labels: `roadmap`, `phase-3`, `architecture`, `priority-p1`
+- Source TODO:
+  - `盘点现有 Agent 入口与责任边界`
+  - `标记必须 Agent vs 普通 service`
+- Scope:
+  - Inventory all agent entrypoints (routes/workflows)
+  - Document MUST-use-agent vs service-only decision rules
+  - Output `docs/agent_inventory.md`
+- Acceptance:
+  - Every current agent entry has owner, input/output, SLA and fallback
+
+## Issue 22 - AgentRuntime Contract and SourceCollector Port
+
+- Title: `[Feature] Introduce unified AgentRuntime and SourceCollector contracts`
+- Labels: `roadmap`, `phase-3`, `backend`, `architecture`, `priority-p1`
+- Source TODO:
+  - `定义 AgentRuntime 接口`
+  - `定义 AgentMessage/AgentResult/AgentError schema`
+  - `定义 SourceCollector 接口`
+- Scope:
+  - Add `core/abstractions/agent_runtime.py`
+  - Add `application/ports/source_collector.py`
+  - Add compatibility adapters for current agent classes
+- Acceptance:
+  - Existing agent flows compile and run through adapter layer
+
+## Issue 23 - Unified Event Envelope for SSE + Trace Pipeline
+
+- Title: `[Feature] Unify SSE/event-log envelope across workflows and agents`
+- Labels: `roadmap`, `phase-3`, `backend`, `frontend`, `observability`, `priority-p1`
+- Source TODO:
+  - `SSE 事件统一`
+  - `trace_id 全链路贯穿`
+  - `前端统一 event parser`
+- Scope:
+  - Define shared event envelope schema
+  - Map workflow events to the schema
+  - Add frontend parser utility and migrate key pages
+- Acceptance:
+  - One parser handles Search/Daily/Analyze streams consistently
+
+## Issue 24 - AgentRuntime Migration Wave 1 (Analyze/Review)
+
+- Title: `[Refactor] Migrate analyze/review routes to AgentRuntime`
+- Labels: `roadmap`, `phase-3`, `backend`, `priority-p1`
+- Source TODO:
+  - `Step 1: analyze + review`
+- Scope:
+  - Route-level migration for `analyze` and `review`
+  - Keep API contract backward-compatible
+  - Add regression tests for lifecycle events
+- Acceptance:
+  - No behavior regressions in analyze/review, with unified runtime events
+