From 5f16ad69fddefb365297a7e21616cc4c264f5bcc Mon Sep 17 00:00:00 2001 From: jerry <1772030600@qq.com> Date: Thu, 12 Feb 2026 20:29:24 +0800 Subject: [PATCH 1/9] feat: add BibTeX/RIS/Markdown export for saved papers Closes #93 Closes #94 --- src/paperbot/api/routes/research.py | 137 +++++++++++++++++++++++++++ tests/unit/test_export.py | 140 ++++++++++++++++++++++++++++ 2 files changed, 277 insertions(+) create mode 100644 tests/unit/test_export.py diff --git a/src/paperbot/api/routes/research.py b/src/paperbot/api/routes/research.py index 9222ef60..4e408a3e 100644 --- a/src/paperbot/api/routes/research.py +++ b/src/paperbot/api/routes/research.py @@ -8,6 +8,7 @@ from typing import Any, Dict, List, Optional, Tuple from fastapi import APIRouter, BackgroundTasks, HTTPException, Query +from fastapi.responses import Response from pydantic import BaseModel, Field from sqlalchemy.exc import IntegrityError @@ -1608,3 +1609,139 @@ async def scholar_trends(req: ScholarTrendsRequest): recent_papers=recent_papers[:10], trend_summary=trend_summary, ) + + +# --------------------------------------------------------------------------- +# Paper export (BibTeX / RIS / Markdown) +# --------------------------------------------------------------------------- + +def _make_citation_key(authors: List[str], year: Optional[int]) -> str: + """first_author_lastname + year, e.g. 'smith2025'.""" + lastname = "unknown" + if authors: + parts = authors[0].strip().split() + if parts: + lastname = re.sub(r"[^a-zA-Z]", "", parts[-1]).lower() or "unknown" + return f"{lastname}{year or 'nd'}" + + +def _dedup_citation_keys(keys: List[str]) -> List[str]: + """Append a/b/c suffixes when keys collide.""" + seen: Dict[str, int] = {} + result: List[str] = [] + for k in keys: + count = seen.get(k, 0) + seen[k] = count + 1 + result.append(k if count == 0 else f"{k}{chr(ord('a') + count)}") + return result + + +def _escape_bibtex(value: str) -> str: + return value.replace("{", "\\{").replace("}", "\\}") + + +def _paper_to_bibtex(paper: Dict[str, Any], key: str) -> str: + entry_type = "article" if paper.get("doi") else "misc" + lines = [f"@{entry_type}{{{key},"] + lines.append(f" title = {{{_escape_bibtex(paper.get('title') or '')}}},") + authors = paper.get("authors") or [] + if authors: + lines.append(f" author = {{{_escape_bibtex(' and '.join(authors))}}},") + if paper.get("year"): + lines.append(f" year = {{{paper['year']}}},") + if paper.get("venue"): + field = "journal" if paper.get("doi") else "booktitle" + lines.append(f" {field} = {{{_escape_bibtex(paper['venue'])}}},") + if paper.get("doi"): + lines.append(f" doi = {{{paper['doi']}}},") + if paper.get("url"): + lines.append(f" url = {{{paper['url']}}},") + if paper.get("arxiv_id"): + lines.append(f" eprint = {{{paper['arxiv_id']}}},") + lines.append(" archiveprefix = {arXiv},") + lines.append("}") + return "\n".join(lines) + + +def _paper_to_ris(paper: Dict[str, Any]) -> str: + venue = (paper.get("venue") or "").lower() + is_conf = any(kw in venue for kw in ("conf", "proc", "sympos", "workshop")) + lines = [f"TY - {'CONF' if is_conf else 'JOUR'}"] + lines.append(f"TI - {paper.get('title') or ''}") + for author in paper.get("authors") or []: + lines.append(f"AU - {author}") + if paper.get("year"): + lines.append(f"PY - {paper['year']}") + if paper.get("venue"): + lines.append(f"JO - {paper['venue']}") + if paper.get("doi"): + lines.append(f"DO - {paper['doi']}") + if paper.get("url"): + lines.append(f"UR - {paper['url']}") + if paper.get("arxiv_id"): + lines.append(f"AN - {paper['arxiv_id']}") + lines.append("ER - ") + return "\n".join(lines) + + +def _paper_to_markdown(paper: Dict[str, Any]) -> str: + authors = ", ".join(paper.get("authors") or ["Unknown"]) + title = paper.get("title") or "Untitled" + year = paper.get("year") or "n.d." + venue = paper.get("venue") or "" + parts = [f"- **{title}**", f" {authors} ({year})"] + if venue: + parts.append(f" *{venue}*") + links: List[str] = [] + if paper.get("url"): + links.append(f"[URL]({paper['url']})") + if paper.get("doi"): + links.append(f"[DOI](https://doi.org/{paper['doi']})") + if paper.get("arxiv_id"): + links.append(f"[arXiv](https://arxiv.org/abs/{paper['arxiv_id']})") + if links: + parts.append(f" {' | '.join(links)}") + return "\n".join(parts) + + +@router.get("/research/papers/export") +def export_papers( + user_id: str = "default", + track_id: Optional[int] = None, + format: str = Query("bibtex", pattern="^(bibtex|ris|markdown)$"), +): + items = _research_store.list_saved_papers( + user_id=user_id, track_id=track_id, limit=1000 + ) + papers = [item["paper"] for item in items if item.get("paper")] + + if not papers: + raise HTTPException(status_code=404, detail="No saved papers found") + + if format == "bibtex": + raw_keys = [_make_citation_key(p.get("authors") or [], p.get("year")) for p in papers] + keys = _dedup_citation_keys(raw_keys) + body = "\n\n".join(_paper_to_bibtex(p, k) for p, k in zip(papers, keys)) + return Response( + content=body, + media_type="application/x-bibtex", + headers={"Content-Disposition": "attachment; filename=papers.bib"}, + ) + + if format == "ris": + body = "\n\n".join(_paper_to_ris(p) for p in papers) + return Response( + content=body, + media_type="application/x-research-info-systems", + headers={"Content-Disposition": "attachment; filename=papers.ris"}, + ) + + # markdown + now_iso = datetime.now(timezone.utc).strftime("%Y-%m-%d") + header = f"---\nexported: {now_iso}\ncount: {len(papers)}\n---\n\n# Saved Papers\n" + body = header + "\n\n".join(_paper_to_markdown(p) for p in papers) + return Response( + content=body, + media_type="text/markdown", + headers={"Content-Disposition": "attachment; filename=papers.md"}, + ) diff --git a/tests/unit/test_export.py b/tests/unit/test_export.py new file mode 100644 index 00000000..06820465 --- /dev/null +++ b/tests/unit/test_export.py @@ -0,0 +1,140 @@ +"""Unit tests for BibTeX / RIS / Markdown paper export.""" + +from __future__ import annotations + +import pytest + +from paperbot.api.routes.research import ( + _dedup_citation_keys, + _escape_bibtex, + _make_citation_key, + _paper_to_bibtex, + _paper_to_markdown, + _paper_to_ris, +) + +# ---- fixtures --------------------------------------------------------------- + +_PAPER_FULL = { + "title": "Attention Is All You Need", + "authors": ["Ashish Vaswani", "Noam Shazeer"], + "year": 2017, + "venue": "NeurIPS", + "doi": "10.5555/3295222.3295349", + "url": "https://papers.nips.cc/paper/7181", + "arxiv_id": "1706.03762", +} + +_PAPER_MINIMAL = { + "title": "Some Draft", + "authors": [], + "year": None, + "venue": None, + "doi": None, + "url": None, + "arxiv_id": None, +} + +_PAPER_CONF = { + "title": "Conf Paper", + "authors": ["Alice Smith"], + "year": 2024, + "venue": "Proceedings of ICML", + "doi": None, + "url": "https://example.com", + "arxiv_id": None, +} + + +# ---- citation key ----------------------------------------------------------- + +class TestCitationKey: + def test_normal(self): + assert _make_citation_key(["John Smith", "Jane Doe"], 2025) == "smith2025" + + def test_single_name(self): + assert _make_citation_key(["Madonna"], 2020) == "madonna2020" + + def test_empty_authors(self): + assert _make_citation_key([], 2023) == "unknown2023" + + def test_no_year(self): + assert _make_citation_key(["Alice"], None) == "alicend" + + +class TestDedupKeys: + def test_no_collision(self): + assert _dedup_citation_keys(["a", "b", "c"]) == ["a", "b", "c"] + + def test_collision(self): + assert _dedup_citation_keys(["smith2025", "smith2025", "doe2025"]) == [ + "smith2025", + "smith2025b", + "doe2025", + ] + + def test_triple_collision(self): + result = _dedup_citation_keys(["x", "x", "x"]) + assert result == ["x", "xb", "xc"] + + +# ---- BibTeX ----------------------------------------------------------------- + +class TestBibtex: + def test_full_paper(self): + bib = _paper_to_bibtex(_PAPER_FULL, "vaswani2017") + assert bib.startswith("@article{vaswani2017,") + assert "title = {Attention Is All You Need}" in bib + assert "author = {Ashish Vaswani and Noam Shazeer}" in bib + assert "doi = {10.5555/3295222.3295349}" in bib + assert "eprint = {1706.03762}" in bib + assert "archiveprefix = {arXiv}" in bib + + def test_misc_without_doi(self): + bib = _paper_to_bibtex(_PAPER_MINIMAL, "unknown_nd") + assert bib.startswith("@misc{unknown_nd,") + assert "doi" not in bib + + def test_escape(self): + assert _escape_bibtex("a {b} c") == "a \\{b\\} c" + + +# ---- RIS -------------------------------------------------------------------- + +class TestRis: + def test_journal_paper(self): + ris = _paper_to_ris(_PAPER_FULL) + assert ris.startswith("TY - JOUR") + assert "TI - Attention Is All You Need" in ris + assert "AU - Ashish Vaswani" in ris + assert "DO - 10.5555/3295222.3295349" in ris + assert "AN - 1706.03762" in ris + assert ris.strip().endswith("ER -") + + def test_conference_paper(self): + ris = _paper_to_ris(_PAPER_CONF) + assert ris.startswith("TY - CONF") + + def test_minimal(self): + ris = _paper_to_ris(_PAPER_MINIMAL) + assert "TY - JOUR" in ris + assert "DO" not in ris + assert "AN" not in ris + + +# ---- Markdown --------------------------------------------------------------- + +class TestMarkdown: + def test_full_paper(self): + md = _paper_to_markdown(_PAPER_FULL) + assert "**Attention Is All You Need**" in md + assert "Ashish Vaswani, Noam Shazeer" in md + assert "(2017)" in md + assert "[DOI]" in md + assert "[arXiv]" in md + + def test_minimal(self): + md = _paper_to_markdown(_PAPER_MINIMAL) + assert "**Some Draft**" in md + assert "(n.d.)" in md + assert "[DOI]" not in md From af9cb798b7255537461131059d6488c9b4617f0c Mon Sep 17 00:00:00 2001 From: jerry <1772030600@qq.com> Date: Thu, 12 Feb 2026 20:29:41 +0800 Subject: [PATCH 2/9] feat: add evidence_quotes to judge output Closes #91 --- src/paperbot/api/routes/paperscool.py | 5 ++ .../workflows/analysis/judge_prompts.py | 5 +- .../workflows/analysis/paper_judge.py | 28 ++++++++ tests/unit/test_paper_judge.py | 71 +++++++++++++++++++ 4 files changed, 108 insertions(+), 1 deletion(-) diff --git a/src/paperbot/api/routes/paperscool.py b/src/paperbot/api/routes/paperscool.py index 57554350..6e12b9ce 100644 --- a/src/paperbot/api/routes/paperscool.py +++ b/src/paperbot/api/routes/paperscool.py @@ -102,6 +102,11 @@ def _count_report_claims_and_evidence(report: Dict[str, Any]) -> tuple[int, int] claims += 1 if item.get("url") or item.get("pdf_url") or item.get("external_url"): evidences += 1 + judge = item.get("judge") + if isinstance(judge, dict): + eq = judge.get("evidence_quotes") + if isinstance(eq, list) and eq: + evidences += len(eq) return claims, evidences diff --git a/src/paperbot/application/workflows/analysis/judge_prompts.py b/src/paperbot/application/workflows/analysis/judge_prompts.py index b22014d9..20ae6051 100644 --- a/src/paperbot/application/workflows/analysis/judge_prompts.py +++ b/src/paperbot/application/workflows/analysis/judge_prompts.py @@ -50,7 +50,10 @@ def build_paper_judge_user_prompt(*, query: str, paper: Dict[str, Any], rubric: f" {dims_json},\n" ' "overall": ,\n' ' "one_line_summary": "",\n' - ' "recommendation": ""\n' + ' "recommendation": "",\n' + ' "evidence_quotes": [\n' + ' {"text": "", "source_url": "", "page_hint": "
"}\n' + " ]\n" "}\n" ) diff --git a/src/paperbot/application/workflows/analysis/paper_judge.py b/src/paperbot/application/workflows/analysis/paper_judge.py index 5fa44b20..513d904c 100644 --- a/src/paperbot/application/workflows/analysis/paper_judge.py +++ b/src/paperbot/application/workflows/analysis/paper_judge.py @@ -39,6 +39,11 @@ class PaperJudgment: recommendation: str judge_model: str = "" judge_cost_tier: int = 0 + evidence_quotes: List[Dict[str, str]] = None # type: ignore[assignment] + + def __post_init__(self) -> None: + if self.evidence_quotes is None: + self.evidence_quotes = [] def to_dict(self) -> Dict[str, Any]: return { @@ -52,6 +57,7 @@ def to_dict(self) -> Dict[str, Any]: "recommendation": self.recommendation, "judge_model": self.judge_model, "judge_cost_tier": int(self.judge_cost_tier), + "evidence_quotes": list(self.evidence_quotes), } @@ -108,6 +114,7 @@ def pick_recommendation(values: Sequence[str]) -> str: payload["overall"] = self._weighted_overall(dim_medians) payload["one_line_summary"] = judgments[0].one_line_summary payload["recommendation"] = pick_recommendation([j.recommendation for j in judgments]) + payload["evidence_quotes"] = judgments[0].evidence_quotes provider_info = { "model_name": judgments[0].judge_model, @@ -191,6 +198,8 @@ def _to_judgment(self, *, payload: Dict[str, Any], provider_info: Dict[str, Any] if not one_line_summary: one_line_summary = self._fallback_summary(dims) + evidence_quotes = self._parse_evidence_quotes(payload.get("evidence_quotes")) + return PaperJudgment( relevance=dims["relevance"], novelty=dims["novelty"], @@ -202,8 +211,27 @@ def _to_judgment(self, *, payload: Dict[str, Any], provider_info: Dict[str, Any] recommendation=recommendation, judge_model=str(provider_info.get("model_name") or ""), judge_cost_tier=int(provider_info.get("cost_tier") or 0), + evidence_quotes=evidence_quotes, ) + @staticmethod + def _parse_evidence_quotes(raw: Any) -> List[Dict[str, str]]: + if not isinstance(raw, list): + return [] + result: List[Dict[str, str]] = [] + for item in raw: + if not isinstance(item, dict): + continue + text = str(item.get("text") or "").strip() + if not text: + continue + result.append({ + "text": text, + "source_url": str(item.get("source_url") or ""), + "page_hint": str(item.get("page_hint") or ""), + }) + return result + def _weighted_overall(self, scores: Dict[str, int]) -> float: weights = self._rubric.weights() total = 0.0 diff --git a/tests/unit/test_paper_judge.py b/tests/unit/test_paper_judge.py index 455835f7..41e5cd23 100644 --- a/tests/unit/test_paper_judge.py +++ b/tests/unit/test_paper_judge.py @@ -33,6 +33,77 @@ def test_paper_judge_single_parses_scores_and_overall(): assert result.overall == 4.2 assert result.recommendation == "must_read" assert result.judge_model == "judge-model" + assert result.evidence_quotes == [] + + +def test_paper_judge_parses_evidence_quotes(): + payload = { + "relevance": {"score": 4, "rationale": "related"}, + "novelty": {"score": 3, "rationale": "ok"}, + "rigor": {"score": 4, "rationale": "solid"}, + "impact": {"score": 3, "rationale": "moderate"}, + "clarity": {"score": 4, "rationale": "clear"}, + "overall": 3.6, + "one_line_summary": "decent paper", + "recommendation": "worth_reading", + "evidence_quotes": [ + {"text": "We propose a novel method", "source_url": "https://arxiv.org/abs/1234", "page_hint": "Section 3"}, + {"text": "Results show 20% improvement", "source_url": "", "page_hint": "Table 2"}, + ], + } + judge = PaperJudge(llm_service=_FakeLLMService(payload)) + result = judge.judge_single(paper={"title": "x", "snippet": "y"}, query="methods") + + assert len(result.evidence_quotes) == 2 + assert result.evidence_quotes[0]["text"] == "We propose a novel method" + assert result.evidence_quotes[0]["source_url"] == "https://arxiv.org/abs/1234" + assert result.evidence_quotes[1]["page_hint"] == "Table 2" + + d = result.to_dict() + assert "evidence_quotes" in d + assert len(d["evidence_quotes"]) == 2 + + +def test_paper_judge_evidence_quotes_defaults_empty_on_missing(): + payload = { + "relevance": {"score": 3, "rationale": "ok"}, + "novelty": {"score": 3, "rationale": "ok"}, + "rigor": {"score": 3, "rationale": "ok"}, + "impact": {"score": 3, "rationale": "ok"}, + "clarity": {"score": 3, "rationale": "ok"}, + "overall": 3.0, + "one_line_summary": "average", + "recommendation": "skim", + } + judge = PaperJudge(llm_service=_FakeLLMService(payload)) + result = judge.judge_single(paper={"title": "x", "snippet": "y"}, query="q") + + assert result.evidence_quotes == [] + assert result.to_dict()["evidence_quotes"] == [] + + +def test_paper_judge_evidence_quotes_skips_invalid_entries(): + payload = { + "relevance": {"score": 3, "rationale": "ok"}, + "novelty": {"score": 3, "rationale": "ok"}, + "rigor": {"score": 3, "rationale": "ok"}, + "impact": {"score": 3, "rationale": "ok"}, + "clarity": {"score": 3, "rationale": "ok"}, + "overall": 3.0, + "one_line_summary": "test", + "recommendation": "skim", + "evidence_quotes": [ + {"text": "valid quote", "source_url": "", "page_hint": ""}, + "not a dict", + {"text": "", "source_url": "url"}, + {"no_text_key": "value"}, + ], + } + judge = PaperJudge(llm_service=_FakeLLMService(payload)) + result = judge.judge_single(paper={"title": "x", "snippet": "y"}, query="q") + + assert len(result.evidence_quotes) == 1 + assert result.evidence_quotes[0]["text"] == "valid quote" def test_paper_judge_calibration_uses_median(): From 10bc5366ebc6cea989d76e78537d6840ef9bc4d6 Mon Sep 17 00:00:00 2001 From: jerry <1772030600@qq.com> Date: Thu, 12 Feb 2026 20:34:04 +0800 Subject: [PATCH 3/9] feat: add evidence display, export buttons, and source selector - PaperCard: collapsible evidence quotes section below judge scores - SavedTab: export dropdown (BibTeX/RIS/Markdown) triggering browser download - TopicWorkflowDashboard: Download .md button for daily paper reports - SearchResults: add papers_cool and hf_daily to SOURCE_OPTIONS Closes #96 --- web/src/components/research/PaperCard.tsx | 45 ++++++++++++++++- web/src/components/research/SavedTab.tsx | 49 +++++++++++++++++-- web/src/components/research/SearchResults.tsx | 2 + .../research/TopicWorkflowDashboard.tsx | 27 ++++++++-- 4 files changed, 114 insertions(+), 9 deletions(-) diff --git a/web/src/components/research/PaperCard.tsx b/web/src/components/research/PaperCard.tsx index 0e84a7cc..9f74ce09 100644 --- a/web/src/components/research/PaperCard.tsx +++ b/web/src/components/research/PaperCard.tsx @@ -1,7 +1,7 @@ "use client" import { useEffect, useState } from "react" -import { Check, ExternalLink, Heart, Loader2, Save, ThumbsDown } from "lucide-react" +import { Check, ChevronDown, ChevronRight, ExternalLink, Heart, Loader2, Save, ThumbsDown } from "lucide-react" import { cn, safeHref } from "@/lib/utils" import { ReasoningBlock, ToolActionsGroup } from "@/components/ai-elements" @@ -21,6 +21,7 @@ export type Paper = { recommendation?: string one_line_summary?: string judge_model?: string + evidence_quotes?: Array<{ text: string; source_url?: string; page_hint?: string }> } is_saved?: boolean } @@ -54,6 +55,7 @@ export function PaperCard({ const [isLiked, setIsLiked] = useState(false) const [isDisliked, setIsDisliked] = useState(false) const [actionLoading, setActionLoading] = useState(null) + const [evidenceOpen, setEvidenceOpen] = useState(false) const authorText = paper.authors?.slice(0, 3).join(", ") || "Unknown authors" const hasMoreAuthors = (paper.authors?.length || 0) > 3 @@ -61,6 +63,7 @@ export function PaperCard({ const judge = paper.latest_judge const judgeOverall = Number(judge?.overall || 0) const judgeRec = String(judge?.recommendation || "").replace(/_/g, " ") + const evidenceQuotes = judge?.evidence_quotes || [] const handleSave = async () => { if (!onSave || isSaved) return @@ -178,6 +181,46 @@ export function PaperCard({ )} + {/* Evidence quotes */} + {judge && judgeOverall > 0 && ( +
+ + {evidenceOpen && ( +
+ {evidenceQuotes.length > 0 ? ( + evidenceQuotes.map((eq, i) => ( +
+

{eq.text}

+
+ {eq.source_url && ( + + source + + )} + {eq.page_hint && p. {eq.page_hint}} +
+
+ )) + ) : ( + No evidence + )} +
+ )} +
+ )} + {/* Action buttons */} items.map(toPaper), [items]) + const handleExport = async (format: "bibtex" | "ris" | "markdown") => { + const qs = new URLSearchParams({ format, user_id: userId }) + if (trackId) qs.set("track_id", String(trackId)) + try { + const res = await fetch(`/api/papers/export?${qs.toString()}`) + if (!res.ok) throw new Error(`${res.status}`) + const blob = await res.blob() + const ext = format === "bibtex" ? "bib" : format === "ris" ? "ris" : "md" + const url = URL.createObjectURL(blob) + const a = document.createElement("a") + a.href = url + a.download = `papers.${ext}` + a.click() + URL.revokeObjectURL(url) + } catch { + setError("Export failed") + } + } + const load = async () => { setLoading(true) setError(null) @@ -90,10 +115,24 @@ export function SavedTab({ userId, trackId }: SavedTabProps) {

Saved papers scoped to current track.

- +
+ + + + + + handleExport("bibtex")}>BibTeX + handleExport("ris")}>RIS + handleExport("markdown")}>Markdown + + + +
{error && ( diff --git a/web/src/components/research/SearchResults.tsx b/web/src/components/research/SearchResults.tsx index d5a408c7..00559c20 100644 --- a/web/src/components/research/SearchResults.tsx +++ b/web/src/components/research/SearchResults.tsx @@ -24,6 +24,8 @@ const SOURCE_OPTIONS: Array<{ value: string; label: string }> = [ { value: "semantic_scholar", label: "S2" }, { value: "arxiv", label: "arXiv" }, { value: "openalex", label: "OpenAlex" }, + { value: "papers_cool", label: "papers.cool" }, + { value: "hf_daily", label: "HF Daily" }, ] function PaperCardSkeleton() { diff --git a/web/src/components/research/TopicWorkflowDashboard.tsx b/web/src/components/research/TopicWorkflowDashboard.tsx index e67a989b..20188a37 100644 --- a/web/src/components/research/TopicWorkflowDashboard.tsx +++ b/web/src/components/research/TopicWorkflowDashboard.tsx @@ -7,6 +7,7 @@ import { BookOpenIcon, ChevronDownIcon, ChevronRightIcon, + DownloadIcon, FilterIcon, Loader2Icon, MailIcon, @@ -1737,9 +1738,29 @@ export default function TopicWorkflowDashboard({ initialQueries }: TopicWorkflow
DailyPaper Report -
- {dailyResult.markdown_path && MD: {dailyResult.markdown_path}} - {dailyResult.json_path && JSON: {dailyResult.json_path}} +
+ {dailyResult.markdown && ( + + )} +
+ {dailyResult.markdown_path && MD: {dailyResult.markdown_path}} + {dailyResult.json_path && JSON: {dailyResult.json_path}} +
From 13d65e7b570aa4ce10aa29ff0e1c4701787e8aa0 Mon Sep 17 00:00:00 2001 From: jerry <1772030600@qq.com> Date: Thu, 12 Feb 2026 20:35:44 +0800 Subject: [PATCH 4/9] feat: add evidence coverage endpoint, feed ranking tests, and connection test - Add GET /api/research/metrics/evidence-coverage endpoint that returns coverage_rate, total_claims, total_with_evidence, and daily trend data - Add unit test verifying saved papers rank higher than skipped papers in list_track_feed - Enhance POST /api/model-endpoints/{id}/test to return latency_ms, success, and error fields - Add unit tests for all new functionality Closes #92 Closes #95 Closes #97 --- src/paperbot/api/routes/model_endpoints.py | 10 ++ src/paperbot/api/routes/research.py | 38 +++++++ tests/unit/test_connection_test_endpoint.py | 56 ++++++++++ tests/unit/test_evidence_coverage_route.py | 48 ++++++++ tests/unit/test_feed_ranking.py | 115 ++++++++++++++++++++ 5 files changed, 267 insertions(+) create mode 100644 tests/unit/test_connection_test_endpoint.py create mode 100644 tests/unit/test_evidence_coverage_route.py create mode 100644 tests/unit/test_feed_ranking.py diff --git a/src/paperbot/api/routes/model_endpoints.py b/src/paperbot/api/routes/model_endpoints.py index 73e15fb6..8b653896 100644 --- a/src/paperbot/api/routes/model_endpoints.py +++ b/src/paperbot/api/routes/model_endpoints.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import time from typing import Any, Dict, List, Optional from fastapi import APIRouter, HTTPException @@ -66,10 +67,13 @@ class EndpointTestRequest(BaseModel): class EndpointTestResponse(BaseModel): ok: bool + success: bool endpoint_id: int provider: Dict[str, Any] api_key_present: bool + latency_ms: int message: str + error: Optional[str] = None class EndpointCapabilitiesResponse(BaseModel): @@ -169,6 +173,7 @@ def test_model_endpoint(endpoint_id: int, req: EndpointTestRequest): os.getenv(api_key_env) ) + t0 = time.monotonic() try: cfg = _build_model_config(endpoint) router = ModelRouter(RouterConfig(models={"__test__": cfg}, fallback_model="__test__")) @@ -187,8 +192,10 @@ def test_model_endpoint(endpoint_id: int, req: EndpointTestRequest): ) message = f"Remote check success: {(pong or '').strip()[:80]}" + latency_ms = int((time.monotonic() - t0) * 1000) return EndpointTestResponse( ok=True, + success=True, endpoint_id=endpoint_id, provider={ "provider_name": info.provider_name, @@ -196,7 +203,10 @@ def test_model_endpoint(endpoint_id: int, req: EndpointTestRequest): "api_base": info.api_base, }, api_key_present=api_key_present, + latency_ms=latency_ms, message=message, + error=None, ) except Exception as exc: + latency_ms = int((time.monotonic() - t0) * 1000) raise HTTPException(status_code=400, detail=f"test failed: {exc}") from exc diff --git a/src/paperbot/api/routes/research.py b/src/paperbot/api/routes/research.py index 4e408a3e..9b0fe53a 100644 --- a/src/paperbot/api/routes/research.py +++ b/src/paperbot/api/routes/research.py @@ -1120,6 +1120,13 @@ class WorkflowMetricsResponse(BaseModel): summary: Dict[str, Any] +class EvidenceCoverageResponse(BaseModel): + coverage_rate: float + total_claims: int + total_with_evidence: int + trend: List[Dict[str, Any]] + + @router.get("/research/metrics/workflows", response_model=WorkflowMetricsResponse) def get_workflow_metrics_summary( days: int = Query(7, ge=1, le=90), @@ -1134,6 +1141,37 @@ def get_workflow_metrics_summary( return WorkflowMetricsResponse(summary=summary) +@router.get("/research/metrics/evidence-coverage", response_model=EvidenceCoverageResponse) +def get_evidence_coverage( + days: int = Query(7, ge=1, le=90), + workflow: Optional[str] = None, + track_id: Optional[int] = None, +): + summary = _get_workflow_metric_store().summarize( + days=days, + workflow=workflow, + track_id=track_id, + ) + totals = summary.get("totals", {}) + total_claims = int(totals.get("claim_count") or 0) + total_with_evidence = int(totals.get("evidence_count") or 0) + coverage_rate = float(totals.get("coverage_rate") or 0.0) + + trend = [] + for day_bucket in summary.get("by_day", []): + trend.append({ + "date": day_bucket.get("date", ""), + "rate": float(day_bucket.get("coverage_rate") or 0.0), + }) + + return EvidenceCoverageResponse( + coverage_rate=coverage_rate, + total_claims=total_claims, + total_with_evidence=total_with_evidence, + trend=trend, + ) + + @router.post("/research/context", response_model=ContextResponse) async def build_context(req: ContextRequest): set_trace_id() # Initialize trace_id for this request diff --git a/tests/unit/test_connection_test_endpoint.py b/tests/unit/test_connection_test_endpoint.py new file mode 100644 index 00000000..98303bf9 --- /dev/null +++ b/tests/unit/test_connection_test_endpoint.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from pathlib import Path + +from fastapi.testclient import TestClient + +from paperbot.api import main as api_main +from paperbot.api.routes import model_endpoints as model_endpoints_route +from paperbot.infrastructure.stores.model_endpoint_store import ModelEndpointStore + + +def test_connection_test_returns_latency(tmp_path: Path, monkeypatch): + db_url = f"sqlite:///{tmp_path / 'conn-test.db'}" + store = ModelEndpointStore(db_url=db_url) + monkeypatch.setattr(model_endpoints_route, "_store", store) + + with TestClient(api_main.app) as client: + created = client.post( + "/api/model-endpoints", + json={ + "name": "TestProvider", + "vendor": "openai_compatible", + "base_url": "https://test.example/v1", + "api_key_env": "TEST_API_KEY", + "api_key": "sk-test-key", + "models": ["test/model"], + "task_types": ["default"], + "enabled": True, + }, + ) + assert created.status_code == 200 + endpoint_id = created.json()["item"]["id"] + + resp = client.post( + f"/api/model-endpoints/{endpoint_id}/test", + json={"remote": False}, + ) + assert resp.status_code == 200 + data = resp.json() + assert data["ok"] is True + assert data["success"] is True + assert "latency_ms" in data + assert isinstance(data["latency_ms"], int) + assert data["latency_ms"] >= 0 + assert data["error"] is None + assert data["endpoint_id"] == endpoint_id + + +def test_connection_test_not_found(tmp_path: Path, monkeypatch): + db_url = f"sqlite:///{tmp_path / 'conn-test-404.db'}" + store = ModelEndpointStore(db_url=db_url) + monkeypatch.setattr(model_endpoints_route, "_store", store) + + with TestClient(api_main.app) as client: + resp = client.post("/api/model-endpoints/9999/test", json={"remote": False}) + assert resp.status_code == 404 diff --git a/tests/unit/test_evidence_coverage_route.py b/tests/unit/test_evidence_coverage_route.py new file mode 100644 index 00000000..fa491362 --- /dev/null +++ b/tests/unit/test_evidence_coverage_route.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from pathlib import Path + +from fastapi.testclient import TestClient + +from paperbot.api import main as api_main +from paperbot.api.routes import research as research_route +from paperbot.infrastructure.stores.workflow_metric_store import WorkflowMetricStore + + +def test_evidence_coverage_endpoint(tmp_path: Path, monkeypatch): + db_url = f"sqlite:///{tmp_path / 'coverage.db'}" + store = WorkflowMetricStore(db_url=db_url) + + store.record_metric( + workflow="research_context", + stage="auto", + status="completed", + track_id=1, + claim_count=20, + evidence_count=15, + elapsed_ms=100, + ) + store.record_metric( + workflow="research_context", + stage="auto", + status="completed", + track_id=1, + claim_count=10, + evidence_count=8, + elapsed_ms=80, + ) + + monkeypatch.setattr(research_route, "_workflow_metric_store", store) + + with TestClient(api_main.app) as client: + resp = client.get("/api/research/metrics/evidence-coverage?days=7") + + assert resp.status_code == 200 + data = resp.json() + assert data["total_claims"] == 30 + assert data["total_with_evidence"] == 23 + assert data["coverage_rate"] > 0 + assert isinstance(data["trend"], list) + assert len(data["trend"]) >= 1 + assert "date" in data["trend"][0] + assert "rate" in data["trend"][0] diff --git a/tests/unit/test_feed_ranking.py b/tests/unit/test_feed_ranking.py new file mode 100644 index 00000000..3a0093d2 --- /dev/null +++ b/tests/unit/test_feed_ranking.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path + +from paperbot.infrastructure.stores.models import ( + Base, + PaperFeedbackModel, + PaperModel, + ResearchTrackModel, +) +from paperbot.infrastructure.stores.research_store import SqlAlchemyResearchStore + + +def _sha256(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _insert_paper(session, *, title: str, keywords: list[str], paper_id: int | None = None) -> int: + now = datetime.now(timezone.utc) + paper = PaperModel( + title=title, + abstract=f"Abstract about {title}", + title_hash=_sha256(title.lower().strip()), + keywords_json=json.dumps(keywords), + fields_of_study_json="[]", + authors_json='["Author A"]', + primary_source="test", + sources_json='["test"]', + citation_count=10, + created_at=now, + updated_at=now, + ) + session.add(paper) + session.flush() + return int(paper.id) + + +def test_saved_papers_rank_higher_than_skipped(tmp_path: Path): + """Saved papers should rank higher than skipped papers in the track feed.""" + db_url = f"sqlite:///{tmp_path / 'feed-ranking.db'}" + store = SqlAlchemyResearchStore(db_url=db_url) + + user_id = "test-user" + track = store.create_track( + user_id=user_id, + name="ML Research", + keywords=["machine learning"], + activate=True, + ) + track_id = int(track["id"]) + + now = datetime.now(timezone.utc) + with store._provider.session() as session: + saved_pid = _insert_paper( + session, + title="Saved Paper on Machine Learning Optimization", + keywords=["machine learning", "optimization"], + ) + skipped_pid = _insert_paper( + session, + title="Skipped Paper on Machine Learning Inference", + keywords=["machine learning", "inference"], + ) + neutral_pid = _insert_paper( + session, + title="Neutral Paper on Machine Learning Training", + keywords=["machine learning", "training"], + ) + + # Add "save" feedback for saved_pid + session.add(PaperFeedbackModel( + user_id=user_id, + track_id=track_id, + paper_id=str(saved_pid), + paper_ref_id=saved_pid, + canonical_paper_id=saved_pid, + action="save", + weight=0.0, + ts=now, + metadata_json="{}", + )) + # Add "skip" feedback for skipped_pid + session.add(PaperFeedbackModel( + user_id=user_id, + track_id=track_id, + paper_id=str(skipped_pid), + paper_ref_id=skipped_pid, + canonical_paper_id=skipped_pid, + action="skip", + weight=0.0, + ts=now, + metadata_json="{}", + )) + session.commit() + + result = store.list_track_feed(user_id=user_id, track_id=track_id, limit=10, offset=0) + items = result["items"] + + assert len(items) >= 2, f"Expected at least 2 items, got {len(items)}" + + scores_by_pid = {} + for item in items: + pid = int(item["paper"]["id"]) + scores_by_pid[pid] = item["feed_score"] + + assert saved_pid in scores_by_pid, "Saved paper should appear in feed" + assert skipped_pid in scores_by_pid, "Skipped paper should appear in feed" + + assert scores_by_pid[saved_pid] > scores_by_pid[skipped_pid], ( + f"Saved paper score ({scores_by_pid[saved_pid]}) should be higher " + f"than skipped paper score ({scores_by_pid[skipped_pid]})" + ) From e38651691bc2ed3266234afcd0afb6e598e486c4 Mon Sep 17 00:00:00 2001 From: jerry <1772030600@qq.com> Date: Thu, 12 Feb 2026 19:48:46 +0800 Subject: [PATCH 5/9] fix: encrypt API keys at rest with Fernet symmetric encryption - Add utils/secret.py with encrypt/decrypt using Fernet (AES-128-CBC + HMAC-SHA256) - Encryption key from PAPERBOT_SECRET_KEY env var, fallback key with warning - model_endpoint_store: encrypt on write, decrypt on read - Graceful migration: non-Fernet values (legacy plaintext) pass through on decrypt - Fix pre-existing test bugs (unquoted f-string path expressions) Closes #77 --- pyproject.toml | 1 + .../stores/model_endpoint_store.py | 6 +- src/paperbot/utils/secret.py | 74 +++++++++++++++++++ tests/unit/test_model_endpoint_store.py | 6 +- 4 files changed, 82 insertions(+), 5 deletions(-) create mode 100644 src/paperbot/utils/secret.py diff --git a/pyproject.toml b/pyproject.toml index 8de004ad..852659a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "requests>=2.28.0", "aiohttp>=3.8.0", "beautifulsoup4>=4.11.0", + "cryptography>=41.0.0", "lxml>=4.9.0", "pydantic>=2.0", "loguru>=0.7.0", diff --git a/src/paperbot/infrastructure/stores/model_endpoint_store.py b/src/paperbot/infrastructure/stores/model_endpoint_store.py index 1d8ac35e..0f369074 100644 --- a/src/paperbot/infrastructure/stores/model_endpoint_store.py +++ b/src/paperbot/infrastructure/stores/model_endpoint_store.py @@ -8,6 +8,8 @@ from paperbot.infrastructure.stores.models import Base, ModelEndpointModel from paperbot.infrastructure.stores.sqlalchemy_db import SessionProvider, get_db_url +from paperbot.utils.secret import decrypt as _decrypt_secret +from paperbot.utils.secret import encrypt as _encrypt_secret _ALLOWED_VENDORS = { "openai", @@ -129,7 +131,7 @@ def upsert_endpoint( if not api_key_text: row.api_key_value = None elif not api_key_text.startswith("***"): - row.api_key_value = api_key_text + row.api_key_value = _encrypt_secret(api_key_text) row.enabled = bool(payload.get("enabled", row.enabled)) row.is_default = bool(payload.get("is_default", row.is_default)) row.set_models(normalized_models) @@ -205,7 +207,7 @@ def activate_endpoint(self, endpoint_id: int) -> Optional[Dict[str, Any]]: def _to_dict(row: ModelEndpointModel, *, include_secrets: bool = False) -> Dict[str, Any]: models = row.get_models() task_types = row.get_task_types() - key_raw = str(row.api_key_value or "").strip() + key_raw = _decrypt_secret(str(row.api_key_value or "").strip()) key_present = bool(key_raw) or bool(os.getenv(row.api_key_env or "")) key_display = key_raw if include_secrets else _mask_secret(key_raw) return { diff --git a/src/paperbot/utils/secret.py b/src/paperbot/utils/secret.py new file mode 100644 index 00000000..aa170f3c --- /dev/null +++ b/src/paperbot/utils/secret.py @@ -0,0 +1,74 @@ +"""Symmetric encryption helpers for secrets at rest. + +Uses Fernet (AES-128-CBC + HMAC-SHA256) from the ``cryptography`` library. +The encryption key is read from the ``PAPERBOT_SECRET_KEY`` environment +variable. When the variable is absent a deterministic fallback key is +derived so that the application still starts (with a logged warning). +""" + +from __future__ import annotations + +import base64 +import hashlib +import os +from typing import Optional + +from loguru import logger + +_ENV_KEY = "PAPERBOT_SECRET_KEY" + +# Lazy-initialised Fernet instance. +_fernet: Optional["Fernet"] = None # type: ignore[name-defined] + + +def _get_fernet() -> "Fernet": + global _fernet + if _fernet is not None: + return _fernet + + from cryptography.fernet import Fernet + + raw = os.getenv(_ENV_KEY, "").strip() + if raw: + # Accept either a raw 32-byte-urlsafe-b64 Fernet key or an + # arbitrary passphrase that we hash into one. + try: + Fernet(raw.encode()) + key = raw.encode() + except Exception: + key = base64.urlsafe_b64encode( + hashlib.sha256(raw.encode()).digest() + ) + else: + logger.warning( + f"{_ENV_KEY} is not set — using deterministic fallback key. " + "Set this variable in production to secure API keys at rest." + ) + key = base64.urlsafe_b64encode( + hashlib.sha256(b"paperbot-default-insecure-key").digest() + ) + + _fernet = Fernet(key) + return _fernet + + +def encrypt(plaintext: str) -> str: + """Encrypt *plaintext* and return a Fernet token string.""" + if not plaintext: + return "" + return _get_fernet().encrypt(plaintext.encode()).decode() + + +def decrypt(token: str) -> str: + """Decrypt a Fernet *token* back to plaintext. + + If *token* is not a valid Fernet token (e.g. a legacy plaintext value) + it is returned as-is so that existing unencrypted data keeps working. + """ + if not token: + return "" + try: + return _get_fernet().decrypt(token.encode()).decode() + except Exception: + # Graceful migration: treat non-Fernet values as plaintext. + return token diff --git a/tests/unit/test_model_endpoint_store.py b/tests/unit/test_model_endpoint_store.py index e7315cef..71f516a7 100644 --- a/tests/unit/test_model_endpoint_store.py +++ b/tests/unit/test_model_endpoint_store.py @@ -6,7 +6,7 @@ def test_activate_endpoint_switches_default(tmp_path: Path): - db_url = f"sqlite:///{tmp_path / store-activate.db}" + db_url = f"sqlite:///{tmp_path / 'store-activate.db'}" store = ModelEndpointStore(db_url=db_url) p1 = store.upsert_endpoint( @@ -42,7 +42,7 @@ def test_activate_endpoint_switches_default(tmp_path: Path): def test_delete_default_reassigns_new_default(tmp_path: Path): - db_url = f"sqlite:///{tmp_path / store-delete.db}" + db_url = f"sqlite:///{tmp_path / 'store-delete.db'}" store = ModelEndpointStore(db_url=db_url) p1 = store.upsert_endpoint( @@ -75,7 +75,7 @@ def test_delete_default_reassigns_new_default(tmp_path: Path): def test_masked_api_key_write_back_keeps_existing_secret(tmp_path: Path): - db_url = f"sqlite:///{tmp_path / store-mask.db}" + db_url = f"sqlite:///{tmp_path / 'store-mask.db'}" store = ModelEndpointStore(db_url=db_url) created = store.upsert_endpoint( From c2dfae453a3b806c5f7e5a2cf7fb64c58fb0acb1 Mon Sep 17 00:00:00 2001 From: jerry <1772030600@qq.com> Date: Thu, 12 Feb 2026 19:50:40 +0800 Subject: [PATCH 6/9] refactor: remove legacy SemanticScholarSearch fallback from ContextEngine - Remove paper_searcher parameter from ContextEngine.__init__ - Remove legacy SemanticScholarSearch import and direct API call path - ContextEngine now exclusively uses PaperSearchService for paper search - Fallback: logs warning and returns empty results if no search_service provided - Remove unused PaperMeta import Closes #78 --- src/paperbot/context_engine/engine.py | 43 ++------------------------- 1 file changed, 3 insertions(+), 40 deletions(-) diff --git a/src/paperbot/context_engine/engine.py b/src/paperbot/context_engine/engine.py index bb67b4fa..a252fbb6 100644 --- a/src/paperbot/context_engine/engine.py +++ b/src/paperbot/context_engine/engine.py @@ -9,7 +9,6 @@ from typing import Any, Dict, List, Optional, Tuple from paperbot.context_engine.track_router import TrackRouter, TrackRouterConfig -from paperbot.domain.paper import PaperMeta from paperbot.infrastructure.stores.memory_store import SqlAlchemyMemoryStore from paperbot.infrastructure.stores.research_store import SqlAlchemyResearchStore from paperbot.utils.logging_config import Logger, LogFiles @@ -360,7 +359,6 @@ def __init__( research_store: Optional[SqlAlchemyResearchStore] = None, memory_store: Optional[SqlAlchemyMemoryStore] = None, paper_store: Optional[Any] = None, - paper_searcher: Optional[Any] = None, search_service: Optional[Any] = None, track_router: Optional[TrackRouter] = None, config: Optional[ContextEngineConfig] = None, @@ -368,7 +366,6 @@ def __init__( self.research_store = research_store or SqlAlchemyResearchStore() self.memory_store = memory_store or SqlAlchemyMemoryStore() self.paper_store = paper_store - self.paper_searcher = paper_searcher self.search_service = search_service self.config = config or ContextEngineConfig() self.track_router = track_router or TrackRouter( @@ -586,46 +583,12 @@ async def build_context_pack( file=LogFiles.HARVEST, ) else: - # Legacy path: direct SemanticScholarSearch - searcher = self.paper_searcher - if searcher is None: - from paperbot.utils.search import SemanticScholarSearch - - searcher = SemanticScholarSearch() - Logger.info("Initialized SemanticScholarSearch", file=LogFiles.HARVEST) - - Logger.info( - f"Searching papers with query='{merged_query}', limit={fetch_limit}", + Logger.warning( + "No search_service provided — skipping paper search. " + "Pass a PaperSearchService instance to ContextEngine.", file=LogFiles.HARVEST, ) - resp = await asyncio.to_thread( - searcher.search_papers, merged_query, fetch_limit - ) - papers_count = len(getattr(resp, "papers", []) or []) - Logger.info(f"Search returned {papers_count} papers", file=LogFiles.HARVEST) - raw = [] - for p in getattr(resp, "papers", []) or []: - authors = [] - for a in getattr(p, "authors", []) or []: - if isinstance(a, dict): - name = str(a.get("name") or "").strip() - if name: - authors.append(name) - raw.append( - PaperMeta( - paper_id=str(getattr(p, "paper_id", "") or ""), - title=str(getattr(p, "title", "") or ""), - abstract=getattr(p, "abstract", None), - year=getattr(p, "year", None), - venue=getattr(p, "venue", None), - citation_count=int(getattr(p, "citation_count", 0) or 0), - authors=authors, - url=getattr(p, "url", None), - fields_of_study=list(getattr(p, "fields_of_study", []) or []), - publication_date=getattr(p, "publication_date", None), - ).to_dict() - ) # Feedback filtering + dedup seen_titles: set[str] = set() From e71153b83af4ca7fb24f800ed5dad7260574271b Mon Sep 17 00:00:00 2001 From: jerry <1772030600@qq.com> Date: Thu, 12 Feb 2026 19:53:21 +0800 Subject: [PATCH 7/9] refactor: wire HarvestPipeline search through PaperSearchService - Replace direct harvester calls (ArxivHarvester, SemanticScholarHarvester, OpenAlexHarvester) with PaperSearchService.search() - PaperSearchService handles dedup and persistence internally, removing the need for separate dedup and store phases - Remove unused imports (HarvestedPaper, HarvestResult, HarvestRunResult, PaperDeduplicator, HarvesterPort, individual harvesters) - Keep pipeline orchestration: keyword expansion, venue recommendation, harvest run records, progress events Closes #79 --- .../application/workflows/harvest_pipeline.py | 166 ++++++------------ 1 file changed, 56 insertions(+), 110 deletions(-) diff --git a/src/paperbot/application/workflows/harvest_pipeline.py b/src/paperbot/application/workflows/harvest_pipeline.py index 983b8adc..fc32c39a 100644 --- a/src/paperbot/application/workflows/harvest_pipeline.py +++ b/src/paperbot/application/workflows/harvest_pipeline.py @@ -15,22 +15,14 @@ from typing import Any, AsyncGenerator, Dict, List, Optional from paperbot.domain.harvest import ( - HarvestedPaper, - HarvestResult, - HarvestRunResult, HarvestSource, ) from paperbot.application.services import ( - PaperDeduplicator, QueryRewriter, VenueRecommender, ) -from paperbot.application.ports.harvester_port import HarvesterPort -from paperbot.infrastructure.harvesters import ( - ArxivHarvester, - SemanticScholarHarvester, - OpenAlexHarvester, -) +from paperbot.application.services.paper_search_service import PaperSearchService +from paperbot.application.workflows.unified_topic_search import make_default_search_service from paperbot.infrastructure.stores.paper_store import PaperStore logger = logging.getLogger(__name__) @@ -84,9 +76,8 @@ class HarvestPipeline: Orchestrates: 1. Query expansion (QueryRewriter) 2. Venue recommendation (VenueRecommender) - 3. Parallel harvesting from multiple sources - 4. In-memory deduplication (PaperDeduplicator) - 5. Batch storage with DB-level dedup (PaperStore) + 3. Unified search via PaperSearchService (dedup + persist included) + 4. Harvest run record management """ def __init__( @@ -94,6 +85,7 @@ def __init__( db_url: Optional[str] = None, *, venue_config_path: Optional[str] = None, + search_service: Optional[PaperSearchService] = None, ): self.db_url = db_url self._venue_config_path = venue_config_path @@ -101,11 +93,8 @@ def __init__( # Services (initialized lazily) self._query_rewriter: Optional[QueryRewriter] = None self._venue_recommender: Optional[VenueRecommender] = None - self._deduplicator: Optional[PaperDeduplicator] = None self._paper_store: Optional[PaperStore] = None - - # Harvesters (initialized per-run) - self._harvesters: Dict[str, HarvesterPort] = {} + self._search_service = search_service @property def query_rewriter(self) -> QueryRewriter: @@ -121,31 +110,17 @@ def venue_recommender(self) -> VenueRecommender: ) return self._venue_recommender - @property - def deduplicator(self) -> PaperDeduplicator: - if self._deduplicator is None: - self._deduplicator = PaperDeduplicator() - return self._deduplicator - @property def paper_store(self) -> PaperStore: if self._paper_store is None: self._paper_store = PaperStore(self.db_url) return self._paper_store - def _get_harvester(self, source: str) -> Optional[HarvesterPort]: - """Get or create harvester for a source.""" - if source not in self._harvesters: - if source == HarvestSource.ARXIV.value: - self._harvesters[source] = ArxivHarvester() - elif source == HarvestSource.SEMANTIC_SCHOLAR.value: - self._harvesters[source] = SemanticScholarHarvester() - elif source == HarvestSource.OPENALEX.value: - self._harvesters[source] = OpenAlexHarvester() - else: - logger.warning(f"Unknown source: {source}") - return None - return self._harvesters[source] + @property + def search_service(self) -> PaperSearchService: + if self._search_service is None: + self._search_service = make_default_search_service(registry=self.paper_store) + return self._search_service @staticmethod def new_run_id() -> str: @@ -213,84 +188,57 @@ async def run( max_results_per_source=config.max_results_per_source, ) - # Phase 4: Harvest from each source in parallel - all_papers: List[HarvestedPaper] = [] - - # Build search query from expanded keywords + # Phase 4: Search via PaperSearchService (handles dedup + persist) search_query = " ".join(expanded_keywords) - # Harvest from each source - for source in sources: - yield HarvestProgress( - phase="Harvesting", - message=f"Fetching from {source}...", - details={"source": source}, - ) - - harvester = self._get_harvester(source) - if harvester is None: - errors[source] = f"Unknown source: {source}" - source_results[source] = {"papers": 0, "error": errors[source]} - continue - - try: - result = await harvester.search( - query=search_query, - max_results=config.max_results_per_source, - year_from=config.year_from, - year_to=config.year_to, - venues=venues, - ) - - all_papers.extend(result.papers) - source_results[source] = { - "papers": result.total_found, - "error": result.error, - } - - if result.error: - errors[source] = result.error - logger.warning(f"Error from {source}: {result.error}") - else: - logger.info(f"Harvested {result.total_found} papers from {source}") - - except Exception as e: - error_msg = str(e) - errors[source] = error_msg - source_results[source] = {"papers": 0, "error": error_msg} - logger.exception(f"Exception harvesting from {source}") - - # Phase 5: Deduplicate yield HarvestProgress( - phase="Deduplicating", - message=f"Removing duplicates from {len(all_papers)} papers...", + phase="Harvesting", + message=f"Searching {len(sources)} sources via PaperSearchService...", + details={"sources": sources}, ) - unique_papers, deduplicated_count = self.deduplicator.deduplicate(all_papers) - logger.info( - f"Deduplication: {len(all_papers)} → {len(unique_papers)} " - f"({deduplicated_count} removed)" - ) - - # Phase 6: Store papers - yield HarvestProgress( - phase="Storing", - message=f"Saving {len(unique_papers)} papers to database...", - ) - - new_count, updated_count = self.paper_store.upsert_papers_batch(unique_papers) - logger.info(f"Stored papers: {new_count} new, {updated_count} updated") - - # Phase 7: Update harvest run record + try: + search_result = await self.search_service.search( + search_query, + sources=sources, + max_results=config.max_results_per_source * len(sources), + year_from=config.year_from, + year_to=config.year_to, + persist=True, + ) + papers_found = search_result.total_raw + papers_new = len(search_result.papers) + deduplicated_count = search_result.duplicates_removed + + for src in sources: + src_papers = [ + p for p in search_result.papers + if src in search_result.provenance.get(p.title_hash or p.title, []) + ] + source_results[src] = {"papers": len(src_papers), "error": None} + + logger.info( + f"PaperSearchService returned {papers_new} unique papers " + f"({deduplicated_count} duplicates removed)" + ) + except Exception as e: + error_msg = str(e) + errors["search_service"] = error_msg + logger.exception(f"PaperSearchService failed: {e}") + papers_found = 0 + papers_new = 0 + deduplicated_count = 0 + + # Phase 5: Update harvest run record status = "success" if errors: - status = "partial" if unique_papers else "failed" + status = "partial" if papers_new else "failed" self.paper_store.update_harvest_run( run_id=run_id, status=status, - papers_found=len(all_papers), - papers_new=new_count, + papers_found=papers_found, + papers_new=papers_new, papers_deduplicated=deduplicated_count, errors=errors if errors else None, ) @@ -303,8 +251,8 @@ async def run( yield HarvestFinalResult( run_id=run_id, status=status, - papers_found=len(all_papers), - papers_new=new_count, + papers_found=papers_found, + papers_new=papers_new, papers_deduplicated=deduplicated_count, source_results=source_results, errors=errors, @@ -356,15 +304,13 @@ async def run_sync( async def close(self) -> None: """Release all resources.""" - # Close harvesters - for harvester in self._harvesters.values(): + if self._search_service: try: - await harvester.close() + await self._search_service.close() except Exception: pass - self._harvesters.clear() + self._search_service = None - # Close paper store if self._paper_store: self._paper_store.close() self._paper_store = None From ea6ad1fbc7b94d700351302e583c34517645360d Mon Sep 17 00:00:00 2001 From: jerry <1772030600@qq.com> Date: Thu, 12 Feb 2026 20:09:39 +0800 Subject: [PATCH 8/9] refactor: remove deprecated search code and fix stale test mocks Delete PapersCoolTopicSearchWorkflow, TopicSearchSourceRegistry, SemanticScholarSearch and their test files. Update all test mocks to patch _run_topic_search / run_unified_topic_search instead of the removed classes. Fix pre-existing SSE event assertions that expected per-paper judge events no longer emitted by EnrichmentPipeline. Closes #80 --- src/paperbot/api/routes/paperscool.py | 11 - .../workflows/paperscool_topic_search.py | 356 ------------ .../workflows/topic_search_sources.py | 221 -------- src/paperbot/utils/search.py | 528 ------------------ tests/test_framework.py | 23 - tests/unit/test_arq_daily_papers.py | 55 +- tests/unit/test_paperscool_cli.py | 70 +-- tests/unit/test_paperscool_route.py | 50 +- tests/unit/test_paperscool_topic_search.py | 115 ---- tests/unit/test_topic_search_sources.py | 75 --- 10 files changed, 92 insertions(+), 1412 deletions(-) delete mode 100644 src/paperbot/application/workflows/paperscool_topic_search.py delete mode 100644 src/paperbot/application/workflows/topic_search_sources.py delete mode 100644 src/paperbot/utils/search.py delete mode 100644 tests/unit/test_paperscool_topic_search.py delete mode 100644 tests/unit/test_topic_search_sources.py diff --git a/src/paperbot/api/routes/paperscool.py b/src/paperbot/api/routes/paperscool.py index 6e12b9ce..808800f2 100644 --- a/src/paperbot/api/routes/paperscool.py +++ b/src/paperbot/api/routes/paperscool.py @@ -119,17 +119,6 @@ async def _run_topic_search( show_per_branch: int, min_score: float, ) -> Dict[str, Any]: - if callable(PapersCoolTopicSearchWorkflow): - workflow = PapersCoolTopicSearchWorkflow() - return workflow.run( - queries=queries, - sources=sources, - branches=branches, - top_k_per_query=top_k_per_query, - show_per_branch=show_per_branch, - min_score=min_score, - ) - return await run_unified_topic_search( queries=queries, sources=sources, diff --git a/src/paperbot/application/workflows/paperscool_topic_search.py b/src/paperbot/application/workflows/paperscool_topic_search.py deleted file mode 100644 index c9d2382e..00000000 --- a/src/paperbot/application/workflows/paperscool_topic_search.py +++ /dev/null @@ -1,356 +0,0 @@ -from __future__ import annotations - -import math -import re -from dataclasses import dataclass -from datetime import datetime, timezone -from typing import Any, Dict, Iterable, List, Optional, Sequence - -from paperbot.application.workflows.topic_search_sources import ( - TopicSearchRecord, - TopicSearchSourceRegistry, - build_default_topic_source_registry, - dedupe_sources, -) - - -_QUERY_ALIASES = { - "icl压缩": "icl compression", - "icl 压缩": "icl compression", - "icl 隐式偏置": "icl implicit bias", - "icl隐式偏置": "icl implicit bias", - "kv cache加速": "kv cache acceleration", - "kv cache 加速": "kv cache acceleration", -} - - -@dataclass(frozen=True) -class QuerySpec: - raw_query: str - normalized_query: str - tokens: List[str] - - -class PapersCoolTopicSearchWorkflow: - def __init__( - self, - source_registry: Optional[TopicSearchSourceRegistry] = None, - ): - self.source_registry = source_registry or build_default_topic_source_registry() - - def normalize_queries(self, queries: Sequence[str]) -> List[QuerySpec]: - specs: List[QuerySpec] = [] - seen = set() - for raw in queries: - raw_query = (raw or "").strip() - if not raw_query: - continue - normalized = _normalize_query(raw_query) - if normalized in seen: - continue - seen.add(normalized) - specs.append( - QuerySpec( - raw_query=raw_query, - normalized_query=normalized, - tokens=_tokenize_query(normalized), - ) - ) - return specs - - def run( - self, - *, - queries: Sequence[str], - branches: Sequence[str] = ("arxiv", "venue"), - sources: Sequence[str] = ("papers_cool",), - top_k_per_query: int = 5, - show_per_branch: int = 25, - min_score: float = 0.0, - ) -> Dict[str, Any]: - query_specs = self.normalize_queries(queries) - source_names = dedupe_sources(sources) - if not source_names: - source_names = ["papers_cool"] - if not query_specs: - return { - "source": "papers.cool", - "fetched_at": datetime.now(timezone.utc).isoformat(), - "sources": source_names, - "queries": [], - "items": [], - "summary": { - "unique_items": 0, - "total_query_hits": 0, - "top_titles": [], - "query_highlights": [], - "source_breakdown": {}, - "source_errors": [], - }, - } - - aggregated_items: List[Dict[str, Any]] = [] - by_url: Dict[str, Dict[str, Any]] = {} - by_title: Dict[str, Dict[str, Any]] = {} - source_errors: List[Dict[str, str]] = [] - - for spec in query_specs: - for source_name in source_names: - source = self.source_registry.create(source_name) - try: - records = source.search( - query=spec.normalized_query, - branches=branches, - show_per_branch=show_per_branch, - ) - except Exception as exc: - # Keep the workflow usable when one external source is throttled/down. - source_errors.append( - { - "source": source_name, - "query": spec.normalized_query, - "error": str(exc), - } - ) - continue - for record in records: - item = self._build_item(record=record, query_spec=spec) - current = self._find_existing_item(item, by_url=by_url, by_title=by_title) - if current is None: - aggregated_items.append(item) - self._index_item(item, by_url=by_url, by_title=by_title) - else: - self._merge_item(current, item) - - # Quality filter: drop papers below minimum relevance score - if min_score > 0: - aggregated_items = [it for it in aggregated_items if it["score"] >= min_score] - - aggregated_items.sort(key=lambda it: it["score"], reverse=True) - - query_views: List[Dict[str, Any]] = [] - for spec in query_specs: - matched = [ - self._serialize_item(item) - for item in aggregated_items - if spec.normalized_query in item["matched_queries"] - ] - query_views.append( - { - "raw_query": spec.raw_query, - "normalized_query": spec.normalized_query, - "tokens": spec.tokens, - "total_hits": len(matched), - "items": matched[: max(int(top_k_per_query), 0)], - } - ) - - summary = self._build_summary( - query_views=query_views, - aggregated_items=aggregated_items, - source_errors=source_errors, - ) - - return { - "source": "papers.cool", - "fetched_at": datetime.now(timezone.utc).isoformat(), - "sources": source_names, - "queries": query_views, - "items": [self._serialize_item(item) for item in aggregated_items], - "summary": summary, - } - - def _build_item(self, *, record: TopicSearchRecord, query_spec: QuerySpec) -> Dict[str, Any]: - matched_keywords = _matched_keywords(record=record, tokens=query_spec.tokens) - score = _score_record( - record=record, token_count=len(query_spec.tokens), matched=matched_keywords - ) - return { - "paper_id": record.source_record_id, - "title": record.title, - "url": record.url, - "external_url": record.external_url, - "pdf_url": record.pdf_url, - "authors": record.authors, - "subject_or_venue": record.subject_or_venue, - "published_at": record.published_at, - "snippet": record.snippet, - "keywords": record.keywords, - "branches": [record.source_branch], - "sources": [record.source], - "pdf_stars": record.pdf_stars, - "kimi_stars": record.kimi_stars, - "matched_keywords": matched_keywords, - "matched_queries": [query_spec.normalized_query], - "score": score, - "alternative_urls": [], - } - - def _find_existing_item( - self, - item: Dict[str, Any], - *, - by_url: Dict[str, Dict[str, Any]], - by_title: Dict[str, Dict[str, Any]], - ) -> Optional[Dict[str, Any]]: - url = (item.get("url") or "").strip() - if url and url in by_url: - return by_url[url] - - title_key = _normalize_title(item.get("title") or "") - if title_key and title_key in by_title: - return by_title[title_key] - return None - - def _index_item( - self, - item: Dict[str, Any], - *, - by_url: Dict[str, Dict[str, Any]], - by_title: Dict[str, Dict[str, Any]], - ) -> None: - url = (item.get("url") or "").strip() - if url: - by_url[url] = item - title_key = _normalize_title(item.get("title") or "") - if title_key: - by_title[title_key] = item - - def _merge_item(self, target: Dict[str, Any], incoming: Dict[str, Any]) -> None: - target["matched_keywords"] = sorted( - set(target["matched_keywords"]).union(incoming["matched_keywords"]) - ) - target["matched_queries"] = sorted( - set(target["matched_queries"]).union(incoming["matched_queries"]) - ) - target["branches"] = sorted(set(target["branches"]).union(incoming["branches"])) - target["sources"] = sorted(set(target["sources"]).union(incoming["sources"])) - target["keywords"] = sorted(set(target["keywords"]).union(incoming["keywords"])) - target["authors"] = sorted(set(target["authors"]).union(incoming["authors"])) - - incoming_url = (incoming.get("url") or "").strip() - target_url = (target.get("url") or "").strip() - if incoming_url and incoming_url != target_url: - alt = set(target.get("alternative_urls") or []) - alt.add(incoming_url) - target["alternative_urls"] = sorted(alt) - - target["pdf_stars"] = max(int(target["pdf_stars"]), int(incoming["pdf_stars"])) - target["kimi_stars"] = max(int(target["kimi_stars"]), int(incoming["kimi_stars"])) - target["score"] = max(float(target["score"]), float(incoming["score"])) - - def _serialize_item(self, item: Dict[str, Any]) -> Dict[str, Any]: - return { - "paper_id": item["paper_id"], - "title": item["title"], - "url": item["url"], - "external_url": item["external_url"], - "pdf_url": item["pdf_url"], - "authors": item["authors"], - "subject_or_venue": item["subject_or_venue"], - "published_at": item["published_at"], - "snippet": item["snippet"], - "keywords": item["keywords"], - "branches": item["branches"], - "sources": item["sources"], - "matched_keywords": item["matched_keywords"], - "matched_queries": item["matched_queries"], - "score": round(float(item["score"]), 4), - "pdf_stars": item["pdf_stars"], - "kimi_stars": item["kimi_stars"], - "alternative_urls": item["alternative_urls"], - } - - def _build_summary( - self, - *, - query_views: Sequence[Dict[str, Any]], - aggregated_items: Sequence[Dict[str, Any]], - source_errors: Sequence[Dict[str, str]], - ) -> Dict[str, Any]: - query_highlights: List[Dict[str, Any]] = [] - total_query_hits = 0 - for query in query_views: - items = query.get("items") or [] - total_query_hits += int(query.get("total_hits") or 0) - top_item = items[0] if items else None - query_highlights.append( - { - "raw_query": query["raw_query"], - "normalized_query": query["normalized_query"], - "hit_count": query.get("total_hits") or 0, - "top_title": top_item.get("title") if top_item else "", - "top_keywords": ( - (top_item.get("matched_keywords") or [])[:5] if top_item else [] - ), - } - ) - - top_titles = [item["title"] for item in list(aggregated_items)[:5]] - source_breakdown: Dict[str, int] = {} - for item in aggregated_items: - for source in item.get("sources") or []: - source_breakdown[source] = source_breakdown.get(source, 0) + 1 - return { - "unique_items": len(aggregated_items), - "total_query_hits": total_query_hits, - "top_titles": top_titles, - "query_highlights": query_highlights, - "source_breakdown": source_breakdown, - "source_errors": list(source_errors), - } - - -def _normalize_query(query: str) -> str: - base = re.sub(r"\s+", " ", query.strip()).lower() - if base in _QUERY_ALIASES: - return _QUERY_ALIASES[base] - return base - - -def _tokenize_query(query: str) -> List[str]: - seen = set() - tokens: List[str] = [] - for token in re.findall(r"[a-z0-9]+", query.lower()): - if token in seen: - continue - seen.add(token) - tokens.append(token) - return tokens - - -def _normalize_title(title: str) -> str: - return "".join(re.findall(r"[a-z0-9]+", title.lower())) - - -def _matched_keywords(*, record: TopicSearchRecord, tokens: Iterable[str]) -> List[str]: - haystack = " ".join([record.title, record.snippet, " ".join(record.keywords)]).lower() - matched = [] - for token in tokens: - if token and token in haystack: - matched.append(token) - return sorted(set(matched)) - - -def _extract_year(record: TopicSearchRecord) -> Optional[int]: - text = " ".join([record.published_at, record.subject_or_venue, record.source_record_id]) - year_match = re.search(r"(20\d{2})", text) - if not year_match: - return None - year = int(year_match.group(1)) - if year < 1990 or year > 2100: - return None - return year - - -def _score_record(*, record: TopicSearchRecord, token_count: int, matched: Sequence[str]) -> float: - hit_count = len(matched) - coverage = hit_count / max(token_count, 1) - popularity = math.log1p(max(record.pdf_stars, 0) + max(record.kimi_stars, 0)) - - freshness = 0.0 - year = _extract_year(record) - if year is not None: - freshness = max(0.0, min((year - 2018) * 0.15, 2.0)) - - return (3.0 * hit_count) + (2.0 * coverage) + popularity + freshness diff --git a/src/paperbot/application/workflows/topic_search_sources.py b/src/paperbot/application/workflows/topic_search_sources.py deleted file mode 100644 index cb6a73e1..00000000 --- a/src/paperbot/application/workflows/topic_search_sources.py +++ /dev/null @@ -1,221 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import Callable, Dict, Iterable, List, Optional, Protocol, Sequence - -from paperbot.infrastructure.connectors.arxiv_connector import ArxivConnector -from paperbot.infrastructure.connectors.hf_daily_papers_connector import HFDailyPapersConnector -from paperbot.infrastructure.connectors.paperscool_connector import PapersCoolConnector - - -@dataclass -class TopicSearchRecord: - source: str - source_record_id: str - title: str - url: str - source_branch: str - external_url: str - pdf_url: str - authors: List[str] - subject_or_venue: str - published_at: str - snippet: str - keywords: List[str] - pdf_stars: int - kimi_stars: int - - -class TopicSearchSource(Protocol): - name: str - - def search( - self, - *, - query: str, - branches: Sequence[str], - show_per_branch: int, - ) -> List[TopicSearchRecord]: ... - - -class TopicSearchSourceRegistry: - def __init__(self) -> None: - self._factories: Dict[str, Callable[[], TopicSearchSource]] = {} - - def register(self, name: str, factory: Callable[[], TopicSearchSource]) -> None: - self._factories[name.strip().lower()] = factory - - def create(self, name: str) -> TopicSearchSource: - key = _normalize_source_name(name) - if key not in self._factories: - raise KeyError(f"Unknown topic search source: {name}") - return self._factories[key]() - - def available(self) -> List[str]: - return sorted(self._factories.keys()) - - -class PapersCoolTopicSource: - name = "papers_cool" - - def __init__(self, connector: Optional[PapersCoolConnector] = None): - self.connector = connector or PapersCoolConnector() - - def search( - self, - *, - query: str, - branches: Sequence[str], - show_per_branch: int, - ) -> List[TopicSearchRecord]: - rows: List[TopicSearchRecord] = [] - for branch in branches: - records = self.connector.search( - branch=branch, - query=query, - highlight=True, - show=show_per_branch, - ) - for record in records: - rows.append( - TopicSearchRecord( - source=self.name, - source_record_id=record.paper_id, - title=record.title, - url=record.url, - source_branch=record.source_branch, - external_url=record.external_url, - pdf_url=record.pdf_url, - authors=record.authors, - subject_or_venue=record.subject_or_venue, - published_at=record.published_at, - snippet=record.snippet, - keywords=record.keywords, - pdf_stars=record.pdf_stars, - kimi_stars=record.kimi_stars, - ) - ) - return rows - - -class ArxivTopicSource: - """arXiv public API as a TopicSearchSource.""" - - name = "arxiv_api" - - def __init__(self, connector: Optional[ArxivConnector] = None): - self.connector = connector or ArxivConnector() - - def search( - self, - *, - query: str, - branches: Sequence[str], - show_per_branch: int, - ) -> List[TopicSearchRecord]: - records = self.connector.search(query=query, max_results=show_per_branch) - return [ - TopicSearchRecord( - source=self.name, - source_record_id=r.arxiv_id, - title=r.title, - url=r.abs_url, - source_branch="arxiv", - external_url=r.abs_url, - pdf_url=r.pdf_url, - authors=r.authors, - subject_or_venue="", - published_at=r.published, - snippet=r.summary[:500] if r.summary else "", - keywords=[], - pdf_stars=0, - kimi_stars=0, - ) - for r in records - ] - - -class HFDailyTopicSource: - """Hugging Face Daily Papers as a TopicSearchSource.""" - - name = "hf_daily" - - def __init__(self, connector: Optional[HFDailyPapersConnector] = None): - self.connector = connector or HFDailyPapersConnector() - - def search( - self, - *, - query: str, - branches: Sequence[str], - show_per_branch: int, - ) -> List[TopicSearchRecord]: - normalized_branches = {(branch or "").strip().lower() for branch in branches} - if normalized_branches and "arxiv" not in normalized_branches: - return [] - - max_pages = max(1, min(10, (int(show_per_branch) + 99) // 100 + 2)) - records = self.connector.search( - query=query, - max_results=show_per_branch, - page_size=100, - max_pages=max_pages, - ) - return [ - TopicSearchRecord( - source=self.name, - source_record_id=record.paper_id, - title=record.title, - url=record.paper_url, - source_branch="arxiv", - external_url=record.external_url, - pdf_url=record.pdf_url, - authors=record.authors, - subject_or_venue="Hugging Face Daily Papers", - published_at=record.submitted_on_daily_at or record.published_at, - snippet=record.summary[:500], - keywords=record.ai_keywords, - pdf_stars=record.upvotes, - kimi_stars=0, - ) - for record in records - ] - - -def build_default_topic_source_registry( - connector: Optional[PapersCoolConnector] = None, -) -> TopicSearchSourceRegistry: - registry = TopicSearchSourceRegistry() - registry.register("papers_cool", lambda: PapersCoolTopicSource(connector=connector)) - registry.register("paperscool", lambda: PapersCoolTopicSource(connector=connector)) - registry.register("arxiv_api", lambda: ArxivTopicSource()) - registry.register("arxivapi", lambda: ArxivTopicSource()) - registry.register("arxiv", lambda: ArxivTopicSource()) - registry.register("hf_daily", lambda: HFDailyTopicSource()) - registry.register("huggingface_daily", lambda: HFDailyTopicSource()) - registry.register("huggingface_papers", lambda: HFDailyTopicSource()) - registry.register("hf", lambda: HFDailyTopicSource()) - return registry - - -def dedupe_sources(sources: Iterable[str]) -> List[str]: - seen = set() - out: List[str] = [] - for source in sources: - key = _normalize_source_name(source) - if not key or key in seen: - continue - seen.add(key) - out.append(key) - return out - - -def _normalize_source_name(name: str | None) -> str: - key = (name or "").strip().lower() - if key in {"paperscool", "papers.cool"}: - return "papers_cool" - if key in {"arxiv", "arxivapi", "arxiv-api"}: - return "arxiv_api" - if key in {"hf", "hf_daily", "huggingface", "huggingface_daily", "huggingface_papers"}: - return "hf_daily" - return key diff --git a/src/paperbot/utils/search.py b/src/paperbot/utils/search.py deleted file mode 100644 index c5649f30..00000000 --- a/src/paperbot/utils/search.py +++ /dev/null @@ -1,528 +0,0 @@ -""" -学术搜索工具集 - -参考: BettaFish/QueryEngine/tools/search.py -适配: PaperBot 学者追踪 - Semantic Scholar API - -专为 AI Agent 设计的学术搜索工具集，提供: -- 学者搜索 -- 论文搜索 -- 引用关系搜索 -- 作者论文搜索 -""" - -import os -from typing import List, Dict, Any, Optional -from dataclasses import dataclass, field -from datetime import datetime -from loguru import logger - -try: - import requests -except ImportError: - raise ImportError("requests 库未安装，请运行 `pip install requests` 进行安装。") - -from paperbot.utils.retry_helper import with_graceful_retry, SEMANTIC_SCHOLAR_RETRY_CONFIG - - -# ===== 数据结构定义 ===== - -@dataclass -class AuthorResult: - """作者搜索结果""" - author_id: str - name: str - affiliations: List[str] = field(default_factory=list) - paper_count: int = 0 - citation_count: int = 0 - h_index: int = 0 - url: Optional[str] = None - - -@dataclass -class PaperResult: - """论文搜索结果""" - paper_id: str - title: str - abstract: Optional[str] = None - year: Optional[int] = None - venue: Optional[str] = None - citation_count: int = 0 - authors: List[Dict[str, str]] = field(default_factory=list) - url: Optional[str] = None - open_access_pdf: Optional[str] = None - fields_of_study: List[str] = field(default_factory=list) - publication_date: Optional[str] = None - - -@dataclass -class SearchResponse: - """搜索响应封装""" - query: str - total: int = 0 - offset: int = 0 - authors: List[AuthorResult] = field(default_factory=list) - papers: List[PaperResult] = field(default_factory=list) - response_time: Optional[float] = None - - -# ===== Semantic Scholar 搜索客户端 ===== - -class SemanticScholarSearch: - """ - Semantic Scholar 学术搜索工具集 - - 每个公共方法都设计为供 AI Agent 独立调用的工具 - """ - - BASE_URL = "https://api.semanticscholar.org/graph/v1" - - # API 字段定义 - AUTHOR_FIELDS = "authorId,name,affiliations,paperCount,citationCount,hIndex,url" - PAPER_FIELDS = "paperId,title,abstract,year,venue,citationCount,authors,url,openAccessPdf,fieldsOfStudy,publicationDate" - - def __init__(self, api_key: Optional[str] = None): - """ - 初始化客户端 - - Args: - api_key: Semantic Scholar API Key (可选，有 key 可提高限流) - """ - self.api_key = api_key or os.getenv("SEMANTIC_SCHOLAR_API_KEY") - self._headers = {"Accept": "application/json"} - if self.api_key: - self._headers["x-api-key"] = self.api_key - - def _parse_author(self, data: Dict[str, Any]) -> AuthorResult: - """解析作者数据""" - affiliations = data.get("affiliations") or [] - if isinstance(affiliations, list): - affiliations = [a.get("name", str(a)) if isinstance(a, dict) else str(a) for a in affiliations] - - return AuthorResult( - author_id=data.get("authorId", ""), - name=data.get("name", ""), - affiliations=affiliations, - paper_count=data.get("paperCount", 0), - citation_count=data.get("citationCount", 0), - h_index=data.get("hIndex", 0), - url=data.get("url"), - ) - - def _parse_paper(self, data: Dict[str, Any]) -> PaperResult: - """解析论文数据""" - authors = [] - for a in data.get("authors") or []: - authors.append({ - "authorId": a.get("authorId", ""), - "name": a.get("name", ""), - }) - - open_access_pdf = None - pdf_info = data.get("openAccessPdf") - if pdf_info and isinstance(pdf_info, dict): - open_access_pdf = pdf_info.get("url") - - return PaperResult( - paper_id=data.get("paperId", ""), - title=data.get("title", ""), - abstract=data.get("abstract"), - year=data.get("year"), - venue=data.get("venue"), - citation_count=data.get("citationCount", 0), - authors=authors, - url=data.get("url"), - open_access_pdf=open_access_pdf, - fields_of_study=data.get("fieldsOfStudy") or [], - publication_date=data.get("publicationDate"), - ) - - @with_graceful_retry(SEMANTIC_SCHOLAR_RETRY_CONFIG, default_return=SearchResponse(query="搜索失败")) - def search_authors(self, query: str, limit: int = 10) -> SearchResponse: - """ - 【工具】搜索学者: 根据姓名搜索学者信息 - - Args: - query: 学者姓名或关键词 - limit: 返回结果数量上限 - - Returns: - 包含学者列表的搜索响应 - """ - logger.info(f"--- TOOL: 搜索学者 (query: {query}) ---") - - start_time = datetime.now() - - url = f"{self.BASE_URL}/author/search" - params = { - "query": query, - "fields": self.AUTHOR_FIELDS, - "limit": min(limit, 100), - } - - try: - response = requests.get(url, headers=self._headers, params=params, timeout=30) - response.raise_for_status() - - data = response.json() - authors = [self._parse_author(a) for a in data.get("data", [])] - - elapsed = (datetime.now() - start_time).total_seconds() - - return SearchResponse( - query=query, - total=data.get("total", len(authors)), - authors=authors, - response_time=elapsed, - ) - except Exception as e: - logger.error(f"搜索学者失败: {e}") - raise - - @with_graceful_retry(SEMANTIC_SCHOLAR_RETRY_CONFIG, default_return=SearchResponse(query="搜索失败")) - def search_papers(self, query: str, limit: int = 20, year_range: Optional[str] = None) -> SearchResponse: - """ - 【工具】搜索论文: 根据关键词搜索论文 - - Args: - query: 搜索关键词 - limit: 返回结果数量上限 - year_range: 年份范围，格式如 "2020-2024" 或 "2023-" - - Returns: - 包含论文列表的搜索响应 - """ - logger.info(f"--- TOOL: 搜索论文 (query: {query}) ---") - - start_time = datetime.now() - - url = f"{self.BASE_URL}/paper/search" - params = { - "query": query, - "fields": self.PAPER_FIELDS, - "limit": min(limit, 100), - } - - if year_range: - params["year"] = year_range - - try: - response = requests.get(url, headers=self._headers, params=params, timeout=30) - response.raise_for_status() - - data = response.json() - papers = [self._parse_paper(p) for p in data.get("data", [])] - - elapsed = (datetime.now() - start_time).total_seconds() - - return SearchResponse( - query=query, - total=data.get("total", len(papers)), - offset=data.get("offset", 0), - papers=papers, - response_time=elapsed, - ) - except Exception as e: - logger.error(f"搜索论文失败: {e}") - raise - - @with_graceful_retry(SEMANTIC_SCHOLAR_RETRY_CONFIG, default_return=None) - def get_author(self, author_id: str) -> Optional[AuthorResult]: - """ - 【工具】获取学者详情: 根据 ID 获取学者详细信息 - - Args: - author_id: Semantic Scholar 作者 ID - - Returns: - 学者详细信息 - """ - logger.info(f"--- TOOL: 获取学者详情 (id: {author_id}) ---") - - url = f"{self.BASE_URL}/author/{author_id}" - params = {"fields": self.AUTHOR_FIELDS} - - try: - response = requests.get(url, headers=self._headers, params=params, timeout=30) - response.raise_for_status() - - data = response.json() - return self._parse_author(data) - except Exception as e: - logger.error(f"获取学者详情失败: {e}") - return None - - @with_graceful_retry(SEMANTIC_SCHOLAR_RETRY_CONFIG, default_return=[]) - def get_author_papers( - self, - author_id: str, - limit: int = 100, - offset: int = 0, - ) -> List[PaperResult]: - """ - 【工具】获取学者论文: 获取指定学者的论文列表 - - Args: - author_id: Semantic Scholar 作者 ID - limit: 返回结果数量上限 - offset: 分页偏移量 - - Returns: - 论文列表 - """ - logger.info(f"--- TOOL: 获取学者论文 (id: {author_id}) ---") - - url = f"{self.BASE_URL}/author/{author_id}/papers" - params = { - "fields": self.PAPER_FIELDS, - "limit": min(limit, 1000), - "offset": offset, - } - - try: - response = requests.get(url, headers=self._headers, params=params, timeout=30) - response.raise_for_status() - - data = response.json() - return [self._parse_paper(p) for p in data.get("data", [])] - except Exception as e: - logger.error(f"获取学者论文失败: {e}") - return [] - - @with_graceful_retry(SEMANTIC_SCHOLAR_RETRY_CONFIG, default_return=None) - def get_paper(self, paper_id: str) -> Optional[PaperResult]: - """ - 【工具】获取论文详情: 根据 ID 获取论文详细信息 - - Args: - paper_id: Semantic Scholar 论文 ID - - Returns: - 论文详细信息 - """ - logger.info(f"--- TOOL: 获取论文详情 (id: {paper_id}) ---") - - url = f"{self.BASE_URL}/paper/{paper_id}" - params = {"fields": self.PAPER_FIELDS} - - try: - response = requests.get(url, headers=self._headers, params=params, timeout=30) - response.raise_for_status() - - data = response.json() - return self._parse_paper(data) - except Exception as e: - logger.error(f"获取论文详情失败: {e}") - return None - - @with_graceful_retry(SEMANTIC_SCHOLAR_RETRY_CONFIG, default_return=[]) - def get_paper_citations(self, paper_id: str, limit: int = 100) -> List[PaperResult]: - """ - 【工具】获取引用论文: 获取引用了指定论文的论文列表 - - Args: - paper_id: Semantic Scholar 论文 ID - limit: 返回结果数量上限 - - Returns: - 引用论文列表 - """ - logger.info(f"--- TOOL: 获取引用论文 (id: {paper_id}) ---") - - url = f"{self.BASE_URL}/paper/{paper_id}/citations" - params = { - "fields": "citingPaper." + self.PAPER_FIELDS, - "limit": min(limit, 1000), - } - - try: - response = requests.get(url, headers=self._headers, params=params, timeout=30) - response.raise_for_status() - - data = response.json() - papers = [] - for item in data.get("data", []): - citing_paper = item.get("citingPaper", {}) - if citing_paper: - papers.append(self._parse_paper(citing_paper)) - return papers - except Exception as e: - logger.error(f"获取引用论文失败: {e}") - return [] - - @with_graceful_retry(SEMANTIC_SCHOLAR_RETRY_CONFIG, default_return=[]) - def get_paper_references(self, paper_id: str, limit: int = 100) -> List[PaperResult]: - """ - 【工具】获取参考文献: 获取指定论文引用的论文列表 - - Args: - paper_id: Semantic Scholar 论文 ID - limit: 返回结果数量上限 - - Returns: - 参考文献列表 - """ - logger.info(f"--- TOOL: 获取参考文献 (id: {paper_id}) ---") - - url = f"{self.BASE_URL}/paper/{paper_id}/references" - params = { - "fields": "citedPaper." + self.PAPER_FIELDS, - "limit": min(limit, 1000), - } - - try: - response = requests.get(url, headers=self._headers, params=params, timeout=30) - response.raise_for_status() - - data = response.json() - papers = [] - for item in data.get("data", []): - cited_paper = item.get("citedPaper", {}) - if cited_paper: - papers.append(self._parse_paper(cited_paper)) - return papers - except Exception as e: - logger.error(f"获取参考文献失败: {e}") - return [] - - def search_security_papers( - self, - query: str, - venues: Optional[List[str]] = None, - limit: int = 20, - ) -> SearchResponse: - """ - 【工具】搜索安全论文: 专门搜索安全领域顶会论文 - - Args: - query: 搜索关键词 - venues: 限定的会议/期刊列表，默认为四大安全顶会 - limit: 返回结果数量上限 - - Returns: - 包含论文列表的搜索响应 - """ - if venues is None: - venues = [ - "IEEE S&P", - "IEEE Symposium on Security and Privacy", - "CCS", - "ACM Conference on Computer and Communications Security", - "USENIX Security", - "NDSS", - "Network and Distributed System Security", - ] - - # 构建包含会议名称的查询 - venue_query = " OR ".join([f'venue:"{v}"' for v in venues]) - enhanced_query = f"({query}) AND ({venue_query})" - - logger.info(f"--- TOOL: 搜索安全论文 (query: {query}) ---") - - return self.search_papers(enhanced_query, limit=limit) - - -# ===== 结果排序器 ===== - -class SearchResultRanker: - """ - 搜索结果排序器 - - 提供多种排序策略用于对搜索结果进行排序 - """ - - @staticmethod - def rank_papers_by_citation(papers: List[PaperResult], descending: bool = True) -> List[PaperResult]: - """按引用数排序""" - return sorted(papers, key=lambda p: p.citation_count, reverse=descending) - - @staticmethod - def rank_papers_by_year(papers: List[PaperResult], descending: bool = True) -> List[PaperResult]: - """按年份排序""" - return sorted(papers, key=lambda p: p.year or 0, reverse=descending) - - @staticmethod - def rank_papers_by_relevance( - papers: List[PaperResult], - query: str, - citation_weight: float = 0.3, - recency_weight: float = 0.3, - title_match_weight: float = 0.4, - ) -> List[PaperResult]: - """ - 综合相关性排序 - - Args: - papers: 论文列表 - query: 原始查询 - citation_weight: 引用数权重 - recency_weight: 时效性权重 - title_match_weight: 标题匹配权重 - """ - current_year = datetime.now().year - query_lower = query.lower() - - def score(paper: PaperResult) -> float: - # 引用分数 (归一化到 0-1) - max_citations = max(p.citation_count for p in papers) if papers else 1 - citation_score = paper.citation_count / max_citations if max_citations > 0 else 0 - - # 时效分数 (最近5年为满分) - year = paper.year or 2000 - recency_score = max(0, min(1, (year - current_year + 5) / 5)) - - # 标题匹配分数 - title_lower = (paper.title or "").lower() - title_score = sum(1 for word in query_lower.split() if word in title_lower) / len(query_lower.split()) - - return ( - citation_weight * citation_score + - recency_weight * recency_score + - title_match_weight * title_score - ) - - return sorted(papers, key=score, reverse=True) - - @staticmethod - def rank_authors_by_influence( - authors: List[AuthorResult], - h_index_weight: float = 0.4, - citation_weight: float = 0.3, - paper_weight: float = 0.3, - ) -> List[AuthorResult]: - """ - 按影响力排序学者 - - Args: - authors: 学者列表 - h_index_weight: H-index 权重 - citation_weight: 引用数权重 - paper_weight: 论文数权重 - """ - if not authors: - return authors - - max_h = max(a.h_index for a in authors) or 1 - max_citations = max(a.citation_count for a in authors) or 1 - max_papers = max(a.paper_count for a in authors) or 1 - - def score(author: AuthorResult) -> float: - return ( - h_index_weight * (author.h_index / max_h) + - citation_weight * (author.citation_count / max_citations) + - paper_weight * (author.paper_count / max_papers) - ) - - return sorted(authors, key=score, reverse=True) - - -__all__ = [ - # 数据类 - "AuthorResult", - "PaperResult", - "SearchResponse", - # 搜索客户端 - "SemanticScholarSearch", - # 排序器 - "SearchResultRanker", -] diff --git a/tests/test_framework.py b/tests/test_framework.py index 25677fcc..06f0ec8e 100644 --- a/tests/test_framework.py +++ b/tests/test_framework.py @@ -277,29 +277,6 @@ def test_create_search_response(self): assert len(response["data"]) > 0 -class TestSearchModule: - """搜索模块测试""" - - def test_semantic_scholar_search_import(self): - """测试搜索模块导入""" - from tools.search import SemanticScholarSearch, SearchResultRanker - - assert SemanticScholarSearch is not None - assert SearchResultRanker is not None - - def test_ranker_by_citation(self, sample_papers): - """测试按引用排序""" - from tools.search import SearchResultRanker, PaperResult - - papers = [ - PaperResult(paper_id=f"p{i}", title=f"Paper {i}", citation_count=i * 10) - for i in range(5) - ] - - ranked = SearchResultRanker.rank_papers_by_citation(papers) - - assert ranked[0].citation_count >= ranked[-1].citation_count - class TestKeywordOptimizer: """关键词优化器测试""" diff --git a/tests/unit/test_arq_daily_papers.py b/tests/unit/test_arq_daily_papers.py index 4f6d4027..cc1dcc94 100644 --- a/tests/unit/test_arq_daily_papers.py +++ b/tests/unit/test_arq_daily_papers.py @@ -28,26 +28,14 @@ def test_build_daily_paper_cron_jobs_enabled(monkeypatch): @pytest.mark.asyncio async def test_daily_papers_job_generates_report_and_feed(tmp_path, monkeypatch): - class _FakeWorkflow: - def run(self, *, queries, sources, branches, top_k_per_query, show_per_branch): - return { - "source": "papers.cool", - "sources": sources, - "queries": [ - { - "raw_query": queries[0], - "normalized_query": "icl compression", - "total_hits": 1, - "items": [ - { - "title": "UniICL", - "url": "https://papers.cool/venue/2025.acl-long.24@ACL", - "score": 9.9, - "matched_queries": ["icl compression"], - } - ], - } - ], + _fake_search_result = { + "source": "papers.cool", + "sources": ["papers_cool"], + "queries": [ + { + "raw_query": "ICL压缩", + "normalized_query": "icl compression", + "total_hits": 1, "items": [ { "title": "UniICL", @@ -56,16 +44,29 @@ def run(self, *, queries, sources, branches, top_k_per_query, show_per_branch): "matched_queries": ["icl compression"], } ], - "summary": { - "unique_items": 1, - "total_query_hits": 1, - }, } - - import paperbot.application.workflows.paperscool_topic_search as topic_mod + ], + "items": [ + { + "title": "UniICL", + "url": "https://papers.cool/venue/2025.acl-long.24@ACL", + "score": 9.9, + "matched_queries": ["icl compression"], + } + ], + "summary": { + "unique_items": 1, + "total_query_hits": 1, + }, + } + + import paperbot.application.workflows.unified_topic_search as uts_mod from paperbot.infrastructure.queue import arq_worker - monkeypatch.setattr(topic_mod, "PapersCoolTopicSearchWorkflow", _FakeWorkflow) + async def _fake_run_unified(**kwargs): + return _fake_search_result + + monkeypatch.setattr(uts_mod, "run_unified_topic_search", _fake_run_unified) result = await arq_worker.daily_papers_job( ctx={}, diff --git a/tests/unit/test_paperscool_cli.py b/tests/unit/test_paperscool_cli.py index 847213d3..26857fb8 100644 --- a/tests/unit/test_paperscool_cli.py +++ b/tests/unit/test_paperscool_cli.py @@ -3,38 +3,40 @@ from paperbot.presentation.cli import main as cli_main -class _FakeWorkflow: - def run(self, *, queries, sources, branches, top_k_per_query, show_per_branch): - return { - "source": "papers.cool", - "fetched_at": "2026-02-09T00:00:00+00:00", - "sources": sources, - "queries": [ - { - "raw_query": queries[0], - "normalized_query": "icl compression", - "tokens": ["icl", "compression"], - "total_hits": 1, - "items": [], - } - ], +_FAKE_SEARCH_RESULT = { + "source": "papers.cool", + "fetched_at": "2026-02-09T00:00:00+00:00", + "sources": ["papers_cool"], + "queries": [ + { + "raw_query": "ICL压缩", + "normalized_query": "icl compression", + "tokens": ["icl", "compression"], + "total_hits": 1, "items": [], - "summary": { - "unique_items": 1, - "total_query_hits": 1, - "top_titles": ["UniICL"], - "source_breakdown": {sources[0]: 1}, - "query_highlights": [ - { - "raw_query": queries[0], - "normalized_query": "icl compression", - "hit_count": 1, - "top_title": "UniICL", - "top_keywords": ["icl", "compression"], - } - ], - }, } + ], + "items": [], + "summary": { + "unique_items": 1, + "total_query_hits": 1, + "top_titles": ["UniICL"], + "source_breakdown": {"papers_cool": 1}, + "query_highlights": [ + { + "raw_query": "ICL压缩", + "normalized_query": "icl compression", + "hit_count": 1, + "top_title": "UniICL", + "top_keywords": ["icl", "compression"], + } + ], + }, +} + + +async def _fake_unified_search(**kwargs): + return _FAKE_SEARCH_RESULT def test_cli_topic_search_parser_flags(): @@ -66,7 +68,7 @@ def test_cli_topic_search_parser_flags(): def test_cli_topic_search_json_output(monkeypatch, capsys): - monkeypatch.setattr(cli_main, "_create_topic_search_workflow", lambda: _FakeWorkflow()) + monkeypatch.setattr(cli_main, "run_unified_topic_search", _fake_unified_search) exit_code = cli_main.run_cli(["topic-search", "-q", "ICL压缩", "--json"]) captured = capsys.readouterr() @@ -78,7 +80,7 @@ def test_cli_topic_search_json_output(monkeypatch, capsys): def test_cli_daily_paper_json_output(monkeypatch, capsys): - monkeypatch.setattr(cli_main, "_create_topic_search_workflow", lambda: _FakeWorkflow()) + monkeypatch.setattr(cli_main, "run_unified_topic_search", _fake_unified_search) exit_code = cli_main.run_cli(["daily-paper", "-q", "ICL压缩", "--json"]) captured = capsys.readouterr() @@ -109,7 +111,7 @@ def test_cli_daily_paper_parser_with_llm_flags(): def test_cli_daily_paper_json_output_with_llm(monkeypatch, capsys): - monkeypatch.setattr(cli_main, "_create_topic_search_workflow", lambda: _FakeWorkflow()) + monkeypatch.setattr(cli_main, "run_unified_topic_search", _fake_unified_search) monkeypatch.setattr( cli_main, "enrich_daily_paper_report", @@ -153,7 +155,7 @@ def test_cli_daily_paper_parser_with_judge_flags(): def test_cli_daily_paper_json_output_with_judge(monkeypatch, capsys): - monkeypatch.setattr(cli_main, "_create_topic_search_workflow", lambda: _FakeWorkflow()) + monkeypatch.setattr(cli_main, "run_unified_topic_search", _fake_unified_search) def _fake_judge(report, max_items_per_query, n_runs, judge_token_budget=0): report = dict(report) diff --git a/tests/unit/test_paperscool_route.py b/tests/unit/test_paperscool_route.py index 32a81218..3601c898 100644 --- a/tests/unit/test_paperscool_route.py +++ b/tests/unit/test_paperscool_route.py @@ -76,8 +76,24 @@ def run(self, *, queries, sources, branches, top_k_per_query, show_per_branch, m } +async def _fake_run_topic_search(*, queries, sources, branches, top_k_per_query, show_per_branch, min_score=0.0): + """Async fake that replaces _run_topic_search for tests.""" + return _FakeWorkflow().run( + queries=queries, sources=sources, branches=branches, + top_k_per_query=top_k_per_query, show_per_branch=show_per_branch, min_score=min_score, + ) + + +async def _fake_run_topic_search_multi(*, queries, sources, branches, top_k_per_query, show_per_branch, min_score=0.0): + """Async fake returning multiple papers for filter testing.""" + return _FakeWorkflowMultiPaper().run( + queries=queries, sources=sources, branches=branches, + top_k_per_query=top_k_per_query, show_per_branch=show_per_branch, min_score=min_score, + ) + + def test_paperscool_search_route_success(monkeypatch): - monkeypatch.setattr(paperscool_route, "PapersCoolTopicSearchWorkflow", _FakeWorkflow) + monkeypatch.setattr(paperscool_route, "_run_topic_search", _fake_run_topic_search) with TestClient(api_main.app) as client: resp = client.post( @@ -106,7 +122,7 @@ def test_paperscool_search_route_requires_queries(): def test_paperscool_daily_route_success(monkeypatch, tmp_path): - monkeypatch.setattr(paperscool_route, "PapersCoolTopicSearchWorkflow", _FakeWorkflow) + monkeypatch.setattr(paperscool_route, "_run_topic_search", _fake_run_topic_search) with TestClient(api_main.app) as client: resp = client.post( @@ -129,7 +145,7 @@ def test_paperscool_daily_route_success(monkeypatch, tmp_path): def test_paperscool_daily_route_with_llm_enrichment(monkeypatch): - monkeypatch.setattr(paperscool_route, "PapersCoolTopicSearchWorkflow", _FakeWorkflow) + monkeypatch.setattr(paperscool_route, "_run_topic_search", _fake_run_topic_search) class _FakeLLMService: def summarize_paper(self, *, title, abstract): @@ -166,7 +182,7 @@ def generate_daily_insight(self, report): def test_paperscool_daily_route_with_judge(monkeypatch): - monkeypatch.setattr(paperscool_route, "PapersCoolTopicSearchWorkflow", _FakeWorkflow) + monkeypatch.setattr(paperscool_route, "_run_topic_search", _fake_run_topic_search) class _FakeJudgment: def to_dict(self): @@ -210,7 +226,6 @@ def judge_with_calibration(self, *, paper, query, n_runs=1): assert resp.status_code == 200 events = _parse_sse_events(resp.text) types = [e.get("type") for e in events] - assert "judge" in types assert "judge_done" in types result_event = next(e for e in events if e.get("type") == "result") assert result_event["data"]["report"]["judge"]["enabled"] is True @@ -337,7 +352,7 @@ def json(self): def test_paperscool_daily_route_persists_judge_scores(monkeypatch): - monkeypatch.setattr(paperscool_route, "PapersCoolTopicSearchWorkflow", _FakeWorkflow) + monkeypatch.setattr(paperscool_route, "_run_topic_search", _fake_run_topic_search) class _FakeJudgment: def to_dict(self): @@ -442,7 +457,7 @@ def run(self, *, queries, sources, branches, top_k_per_query, show_per_branch, m def test_dailypaper_sse_filter_removes_low_papers(monkeypatch): """End-to-end: judge scores papers, filter removes 'skip' and 'skim'.""" - monkeypatch.setattr(paperscool_route, "PapersCoolTopicSearchWorkflow", _FakeWorkflowMultiPaper) + monkeypatch.setattr(paperscool_route, "_run_topic_search", _fake_run_topic_search_multi) # Judge returns different recommendations per paper title class _VaryingJudgment: @@ -497,7 +512,6 @@ def judge_with_calibration(self, *, paper, query, n_runs=1): types = [e.get("type") for e in events] # All expected phases present - assert "judge" in types assert "judge_done" in types assert "filter_done" in types assert "result" in types @@ -522,16 +536,10 @@ def judge_with_calibration(self, *, paper, query, n_runs=1): assert len(final_items) == 1 assert final_items[0]["title"] == "GoodPaper" - # Judge log events should have all 3 papers (complete log) - judge_events = [e for e in events if e.get("type") == "judge"] - assert len(judge_events) == 3 - judge_titles = {e["data"]["title"] for e in judge_events} - assert judge_titles == {"GoodPaper", "MediocreWork", "WeakPaper"} - def test_dailypaper_sse_full_pipeline_llm_judge_filter(monkeypatch): """End-to-end: LLM enrichment + Judge + Filter in one SSE stream.""" - monkeypatch.setattr(paperscool_route, "PapersCoolTopicSearchWorkflow", _FakeWorkflowMultiPaper) + monkeypatch.setattr(paperscool_route, "_run_topic_search", _fake_run_topic_search_multi) class _FakeLLMService: def summarize_paper(self, *, title, abstract): @@ -608,10 +616,8 @@ def judge_with_calibration(self, *, paper, query, n_runs=1): # Full pipeline phases assert "search_done" in types assert "report_built" in types - assert "llm_summary" in types assert "trend" in types assert "llm_done" in types - assert "judge" in types assert "judge_done" in types assert "filter_done" in types assert "result" in types @@ -637,7 +643,7 @@ def judge_with_calibration(self, *, paper, query, n_runs=1): def test_dailypaper_sync_path_no_llm_no_judge(monkeypatch): """When no LLM/Judge, endpoint returns sync JSON (not SSE).""" - monkeypatch.setattr(paperscool_route, "PapersCoolTopicSearchWorkflow", _FakeWorkflow) + monkeypatch.setattr(paperscool_route, "_run_topic_search", _fake_run_topic_search) with TestClient(api_main.app) as client: resp = client.post( @@ -719,7 +725,7 @@ def ingest_repo_enrichment_rows(self, *, rows, source): def test_paperscool_daily_route_enqueues_repo_enrichment(monkeypatch): - monkeypatch.setattr(paperscool_route, "PapersCoolTopicSearchWorkflow", _FakeWorkflow) + monkeypatch.setattr(paperscool_route, "_run_topic_search", _fake_run_topic_search) called = {"count": 0} @@ -743,7 +749,7 @@ def _fake_enqueue(report): def test_paperscool_daily_resume_session(monkeypatch, tmp_path): - monkeypatch.setattr(paperscool_route, "PapersCoolTopicSearchWorkflow", _FakeWorkflow) + monkeypatch.setattr(paperscool_route, "_run_topic_search", _fake_run_topic_search) paperscool_route._pipeline_session_store = PipelineSessionStore( db_url=f"sqlite:///{tmp_path / 'daily-session.db'}" ) @@ -812,7 +818,7 @@ def judge_with_calibration(self, *, paper, query, n_runs=1): def test_paperscool_daily_route_pending_approval_and_queue(monkeypatch, tmp_path): - monkeypatch.setattr(paperscool_route, "PapersCoolTopicSearchWorkflow", _FakeWorkflow) + monkeypatch.setattr(paperscool_route, "_run_topic_search", _fake_run_topic_search) calls = {"ingest": 0} @@ -858,7 +864,7 @@ def _fake_ingest(report): def test_paperscool_daily_approval_decisions(monkeypatch, tmp_path): - monkeypatch.setattr(paperscool_route, "PapersCoolTopicSearchWorkflow", _FakeWorkflow) + monkeypatch.setattr(paperscool_route, "_run_topic_search", _fake_run_topic_search) monkeypatch.setattr( paperscool_route, "_pipeline_session_store", diff --git a/tests/unit/test_paperscool_topic_search.py b/tests/unit/test_paperscool_topic_search.py deleted file mode 100644 index ed999465..00000000 --- a/tests/unit/test_paperscool_topic_search.py +++ /dev/null @@ -1,115 +0,0 @@ -from paperbot.application.workflows.paperscool_topic_search import PapersCoolTopicSearchWorkflow -from paperbot.application.workflows.topic_search_sources import ( - TopicSearchRecord, - TopicSearchSourceRegistry, -) - - -class _FakeSource: - name = "fake_source" - - def search(self, *, query: str, branches, show_per_branch: int): - if query == "icl compression": - rows = [ - TopicSearchRecord( - source=self.name, - source_record_id="2510.00001", - title="UniICL: An Efficient ICL Framework Unifying Compression, Selection, and Generation", - url="https://papers.cool/arxiv/2510.00001", - source_branch="arxiv", - external_url="https://arxiv.org/abs/2510.00001", - pdf_url="https://arxiv.org/pdf/2510.00001", - authors=["Author A"], - subject_or_venue="Artificial Intelligence", - published_at="2025-10-01 00:00:00 UTC", - snippet="ICL compression with efficient selection and generation.", - keywords=["icl", "compression", "llms"], - pdf_stars=10, - kimi_stars=8, - ), - TopicSearchRecord( - source=self.name, - source_record_id="2025.acl-long.24@ACL", - title="UniICL: An Efficient ICL Framework Unifying Compression, Selection, and Generation", - url="https://papers.cool/venue/2025.acl-long.24@ACL", - source_branch="venue", - external_url="https://aclanthology.org/2025.acl-long.24/", - pdf_url="https://aclanthology.org/2025.acl-long.24.pdf", - authors=["Author B"], - subject_or_venue="ACL.2025 - Long Papers", - published_at="", - snippet="ICL compression in venue paper.", - keywords=["icl", "compression", "selection"], - pdf_stars=30, - kimi_stars=30, - ), - ] - return [row for row in rows if row.source_branch in branches] - - if query == "kv cache acceleration": - rows = [ - TopicSearchRecord( - source=self.name, - source_record_id="2412.19442", - title="A Survey on Large Language Model Acceleration based on KV Cache Management", - url="https://papers.cool/arxiv/2412.19442", - source_branch="arxiv", - external_url="https://arxiv.org/abs/2412.19442", - pdf_url="https://arxiv.org/pdf/2412.19442", - authors=["Author C"], - subject_or_venue="Computation and Language", - published_at="2024-12-15 03:21:00 UTC", - snippet="KV cache acceleration survey.", - keywords=["kv", "cache", "acceleration"], - pdf_stars=24, - kimi_stars=27, - ) - ] - return [row for row in rows if row.source_branch in branches] - - return [] - - -class _FakeSourceFactory: - @staticmethod - def build_registry() -> TopicSearchSourceRegistry: - registry = TopicSearchSourceRegistry() - registry.register("fake_source", _FakeSource) - return registry - - -def test_topic_search_normalizes_and_deduplicates_results(): - workflow = PapersCoolTopicSearchWorkflow(source_registry=_FakeSourceFactory.build_registry()) - - result = workflow.run( - queries=["ICL压缩", " kv cache加速 ", "ICL 压缩"], - branches=["arxiv", "venue"], - sources=["fake_source", "fake_source"], - top_k_per_query=3, - ) - - assert result["source"] == "papers.cool" - assert result["sources"] == ["fake_source"] - assert len(result["queries"]) == 2 - assert result["queries"][0]["normalized_query"] == "icl compression" - assert result["queries"][1]["normalized_query"] == "kv cache acceleration" - - # UniICL should be merged across arxiv+venue via title fallback dedupe. - assert len(result["items"]) == 2 - uniicl_item = next(it for it in result["items"] if it["title"].startswith("UniICL")) - assert sorted(uniicl_item["branches"]) == ["arxiv", "venue"] - assert uniicl_item["sources"] == ["fake_source"] - assert "https://papers.cool/venue/2025.acl-long.24@ACL" in uniicl_item["alternative_urls"] - - query_items = {row["normalized_query"]: row["items"] for row in result["queries"]} - assert len(query_items["icl compression"]) == 1 - assert query_items["icl compression"][0]["matched_keywords"] == ["compression", "icl"] - assert len(query_items["kv cache acceleration"]) == 1 - assert query_items["kv cache acceleration"][0]["score"] > 0 - - assert result["summary"]["unique_items"] == 2 - assert result["summary"]["total_query_hits"] == 2 - assert result["summary"]["source_breakdown"] == {"fake_source": 2} - assert len(result["summary"]["top_titles"]) == 2 - highlight = {h["normalized_query"]: h for h in result["summary"]["query_highlights"]} - assert highlight["icl compression"]["top_title"].startswith("UniICL") diff --git a/tests/unit/test_topic_search_sources.py b/tests/unit/test_topic_search_sources.py deleted file mode 100644 index 72ae604c..00000000 --- a/tests/unit/test_topic_search_sources.py +++ /dev/null @@ -1,75 +0,0 @@ -import pytest - -from paperbot.application.workflows.topic_search_sources import ( - HFDailyTopicSource, - TopicSearchSourceRegistry, - dedupe_sources, -) - - -class _DummySource: - name = "dummy" - - def search(self, *, query, branches, show_per_branch): - return [] - - -def test_topic_source_registry_register_create_available(): - registry = TopicSearchSourceRegistry() - registry.register("dummy", _DummySource) - - assert registry.available() == ["dummy"] - source = registry.create("dummy") - assert source.name == "dummy" - - -def test_topic_source_registry_unknown_source(): - registry = TopicSearchSourceRegistry() - with pytest.raises(KeyError): - registry.create("missing") - - -def test_dedupe_sources_normalizes_case_and_blank(): - assert dedupe_sources([" papers_cool ", "PAPERS_COOL", "", "custom"]) == [ - "papers_cool", - "custom", - ] - - -def test_dedupe_sources_normalizes_hf_aliases(): - assert dedupe_sources(["huggingface_daily", "hf", "HF_DAILY"]) == ["hf_daily"] - - -class _FakeHFDailyConnector: - def search(self, *, query, max_results, page_size=100, max_pages=5): - return [ - type( - "Row", - (), - { - "paper_id": "2602.12345", - "title": "KV Cache Compression", - "summary": "Fast KV cache pruning for long context.", - "published_at": "2026-02-10T00:00:00Z", - "submitted_on_daily_at": "2026-02-10T05:00:00Z", - "authors": ["Alice"], - "ai_keywords": ["kv", "cache"], - "upvotes": 9, - "paper_url": "https://huggingface.co/papers/2602.12345", - "external_url": "https://arxiv.org/abs/2602.12345", - "pdf_url": "https://arxiv.org/pdf/2602.12345.pdf", - }, - )() - ] - - -def test_hf_daily_topic_source_respects_branch_filter(): - source = HFDailyTopicSource(connector=_FakeHFDailyConnector()) - - assert source.search(query="kv cache", branches=["venue"], show_per_branch=5) == [] - - rows = source.search(query="kv cache", branches=["arxiv"], show_per_branch=5) - assert len(rows) == 1 - assert rows[0].source == "hf_daily" - assert rows[0].source_branch == "arxiv" - assert rows[0].pdf_stars == 9 From 5a7419ddd85f8181972410caa14ac4530c6e0c93 Mon Sep 17 00:00:00 2001 From: jerry <1772030600@qq.com> Date: Thu, 12 Feb 2026 21:34:34 +0800 Subject: [PATCH 9/9] refactor: redesign dashboard layout and settings with keychain storage Rewrite dashboard with clean 2/3+1/3 grid (StatsBar, ActivityTimeline, LLMUsageChart, SavedPapers, DeadlineRadar), delete 9 deprecated components. Rebuild settings page with ccswitch-style provider cards and Dialog form. Add KeychainStore for macOS Keychain API key storage with DB encryption fallback. Add usage proxy route. --- pyproject.toml | 1 + .../infrastructure/stores/keychain.py | 66 +++ .../stores/model_endpoint_store.py | 21 +- .../infrastructure/stores/paper_store.py | 4 +- .../app/api/model-endpoints/usage/route.ts | 8 + web/src/app/dashboard/page.tsx | 145 ++---- web/src/app/settings/page.tsx | 466 +++++++++--------- web/src/components/dashboard/ActivityFeed.tsx | 29 -- .../components/dashboard/ActivityTimeline.tsx | 47 ++ .../components/dashboard/DeadlineRadar.tsx | 53 +- .../components/dashboard/LLMUsageChart.tsx | 52 +- .../components/dashboard/PipelineStatus.tsx | 47 -- web/src/components/dashboard/QuickActions.tsx | 34 -- web/src/components/dashboard/QuickFilters.tsx | 59 --- web/src/components/dashboard/ReadingQueue.tsx | 38 -- web/src/components/dashboard/SavedPapers.tsx | 37 ++ web/src/components/dashboard/StatsBar.tsx | 29 ++ web/src/components/dashboard/StatsCard.tsx | 31 -- .../dashboard/feed/ConferenceCard.tsx | 50 -- .../dashboard/feed/MilestoneCard.tsx | 40 -- .../dashboard/feed/NewPaperCard.tsx | 91 ---- web/src/lib/api.ts | 101 ++-- web/src/lib/types.ts | 62 +-- 23 files changed, 579 insertions(+), 932 deletions(-) create mode 100644 src/paperbot/infrastructure/stores/keychain.py create mode 100644 web/src/app/api/model-endpoints/usage/route.ts delete mode 100644 web/src/components/dashboard/ActivityFeed.tsx create mode 100644 web/src/components/dashboard/ActivityTimeline.tsx delete mode 100644 web/src/components/dashboard/PipelineStatus.tsx delete mode 100644 web/src/components/dashboard/QuickActions.tsx delete mode 100644 web/src/components/dashboard/QuickFilters.tsx delete mode 100644 web/src/components/dashboard/ReadingQueue.tsx create mode 100644 web/src/components/dashboard/SavedPapers.tsx create mode 100644 web/src/components/dashboard/StatsBar.tsx delete mode 100644 web/src/components/dashboard/StatsCard.tsx delete mode 100644 web/src/components/dashboard/feed/ConferenceCard.tsx delete mode 100644 web/src/components/dashboard/feed/MilestoneCard.tsx delete mode 100644 web/src/components/dashboard/feed/NewPaperCard.tsx diff --git a/pyproject.toml b/pyproject.toml index 852659a1..950ad8f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ "PyYAML>=6.0", "SQLAlchemy>=2.0.0", "alembic>=1.13.0", + "keyring>=25.0.0", ] [project.optional-dependencies] diff --git a/src/paperbot/infrastructure/stores/keychain.py b/src/paperbot/infrastructure/stores/keychain.py new file mode 100644 index 00000000..d24ef840 --- /dev/null +++ b/src/paperbot/infrastructure/stores/keychain.py @@ -0,0 +1,66 @@ +"""Keychain-backed secret storage using the ``keyring`` library. + +On macOS this stores API keys in the system Keychain. On Linux it +delegates to SecretService / KWallet. When ``keyring`` is not +installed or the backend is unavailable the caller should fall back +to the existing DB-level encryption. +""" + +from __future__ import annotations + +from loguru import logger + +SERVICE_NAME = "paperbot" + +_available: bool | None = None + + +def _is_available() -> bool: + global _available + if _available is not None: + return _available + try: + import keyring # noqa: F401 + + _available = True + except ImportError: + logger.warning("keyring library not installed – falling back to DB encryption for API keys") + _available = False + return _available + + +class KeychainStore: + @staticmethod + def store_key(endpoint_name: str, api_key: str) -> bool: + if not _is_available(): + return False + import keyring + + try: + keyring.set_password(SERVICE_NAME, endpoint_name, api_key) + return True + except Exception as exc: + logger.warning(f"keychain store failed for {endpoint_name}: {exc}") + return False + + @staticmethod + def get_key(endpoint_name: str) -> str | None: + if not _is_available(): + return None + import keyring + + try: + return keyring.get_password(SERVICE_NAME, endpoint_name) + except Exception: + return None + + @staticmethod + def delete_key(endpoint_name: str) -> None: + if not _is_available(): + return + import keyring + + try: + keyring.delete_password(SERVICE_NAME, endpoint_name) + except Exception: + pass diff --git a/src/paperbot/infrastructure/stores/model_endpoint_store.py b/src/paperbot/infrastructure/stores/model_endpoint_store.py index 0f369074..a327b24d 100644 --- a/src/paperbot/infrastructure/stores/model_endpoint_store.py +++ b/src/paperbot/infrastructure/stores/model_endpoint_store.py @@ -4,13 +4,17 @@ from datetime import datetime, timezone from typing import Any, Dict, List, Optional +from loguru import logger from sqlalchemy import delete, desc, select +from paperbot.infrastructure.stores.keychain import KeychainStore from paperbot.infrastructure.stores.models import Base, ModelEndpointModel from paperbot.infrastructure.stores.sqlalchemy_db import SessionProvider, get_db_url from paperbot.utils.secret import decrypt as _decrypt_secret from paperbot.utils.secret import encrypt as _encrypt_secret +_KEYCHAIN_MARKER = "__keychain__" + _ALLOWED_VENDORS = { "openai", "openai_compatible", @@ -130,8 +134,12 @@ def upsert_endpoint( api_key_text = str(payload.get("api_key") or "").strip() if not api_key_text: row.api_key_value = None + KeychainStore.delete_key(name) elif not api_key_text.startswith("***"): - row.api_key_value = _encrypt_secret(api_key_text) + if KeychainStore.store_key(name, api_key_text): + row.api_key_value = _KEYCHAIN_MARKER + else: + row.api_key_value = _encrypt_secret(api_key_text) row.enabled = bool(payload.get("enabled", row.enabled)) row.is_default = bool(payload.get("is_default", row.is_default)) row.set_models(normalized_models) @@ -166,6 +174,7 @@ def delete_endpoint(self, endpoint_id: int) -> bool: if row is None: return False deleted_default = bool(row.is_default) + endpoint_name = row.name session.execute( delete(ModelEndpointModel).where(ModelEndpointModel.id == int(endpoint_id)) ) @@ -181,6 +190,7 @@ def delete_endpoint(self, endpoint_id: int) -> bool: session.add(replacement) session.commit() + KeychainStore.delete_key(endpoint_name) return True def activate_endpoint(self, endpoint_id: int) -> Optional[Dict[str, Any]]: @@ -207,7 +217,13 @@ def activate_endpoint(self, endpoint_id: int) -> Optional[Dict[str, Any]]: def _to_dict(row: ModelEndpointModel, *, include_secrets: bool = False) -> Dict[str, Any]: models = row.get_models() task_types = row.get_task_types() - key_raw = _decrypt_secret(str(row.api_key_value or "").strip()) + stored = str(row.api_key_value or "").strip() + if stored == _KEYCHAIN_MARKER: + key_raw = KeychainStore.get_key(row.name) or "" + key_source = "keychain" + else: + key_raw = _decrypt_secret(stored) + key_source = "db" if key_raw else "" key_present = bool(key_raw) or bool(os.getenv(row.api_key_env or "")) key_display = key_raw if include_secrets else _mask_secret(key_raw) return { @@ -222,6 +238,7 @@ def _to_dict(row: ModelEndpointModel, *, include_secrets: bool = False) -> Dict[ "enabled": bool(row.enabled), "is_default": bool(row.is_default), "api_key_present": key_present, + "key_source": key_source, "created_at": row.created_at.isoformat() if row.created_at else None, "updated_at": row.updated_at.isoformat() if row.updated_at else None, } diff --git a/src/paperbot/infrastructure/stores/paper_store.py b/src/paperbot/infrastructure/stores/paper_store.py index 0d056062..244d2f40 100644 --- a/src/paperbot/infrastructure/stores/paper_store.py +++ b/src/paperbot/infrastructure/stores/paper_store.py @@ -803,7 +803,7 @@ def get_user_library( min_ts = datetime.min.replace(tzinfo=timezone.utc) if sort_by == "saved_at": unique_results.sort( - key=lambda x: x[1].ts or min_ts, reverse=(sort_order.lower() == "desc") + key=lambda x: x[1] or min_ts, reverse=(sort_order.lower() == "desc") ) elif sort_by == "title": unique_results.sort( @@ -819,7 +819,7 @@ def get_user_library( ) else: unique_results.sort( - key=lambda x: x[1].ts or min_ts, reverse=(sort_order.lower() == "desc") + key=lambda x: x[1] or min_ts, reverse=(sort_order.lower() == "desc") ) # Get total count before pagination diff --git a/web/src/app/api/model-endpoints/usage/route.ts b/web/src/app/api/model-endpoints/usage/route.ts new file mode 100644 index 00000000..1522edfc --- /dev/null +++ b/web/src/app/api/model-endpoints/usage/route.ts @@ -0,0 +1,8 @@ +export const runtime = "nodejs" + +import { apiBaseUrl, proxyJson } from "../../research/_base" + +export async function GET(req: Request) { + const url = new URL(req.url) + return proxyJson(req, `${apiBaseUrl()}/api/model-endpoints/usage?${url.searchParams.toString()}`, "GET") +} diff --git a/web/src/app/dashboard/page.tsx b/web/src/app/dashboard/page.tsx index a52e4768..c0e81f79 100644 --- a/web/src/app/dashboard/page.tsx +++ b/web/src/app/dashboard/page.tsx @@ -1,36 +1,28 @@ -import { ActivityFeed } from "@/components/dashboard/ActivityFeed" -import { StatsCard } from "@/components/dashboard/StatsCard" -import { PipelineStatus } from "@/components/dashboard/PipelineStatus" -import { ReadingQueue } from "@/components/dashboard/ReadingQueue" +import { StatsBar } from "@/components/dashboard/StatsBar" +import { ActivityTimeline } from "@/components/dashboard/ActivityTimeline" import { LLMUsageChart } from "@/components/dashboard/LLMUsageChart" -import { QuickActions } from "@/components/dashboard/QuickActions" +import { SavedPapers } from "@/components/dashboard/SavedPapers" import { DeadlineRadar } from "@/components/dashboard/DeadlineRadar" -import { Users, FileText, Zap, BookOpen, Search, TrendingUp } from "lucide-react" -import { fetchStats, fetchTrendingTopics, fetchPipelineTasks, fetchReadingQueue, fetchLLMUsage, fetchDeadlineRadar } from "@/lib/api" -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card" -import { Input } from "@/components/ui/input" -import { Button } from "@/components/ui/button" -import { Badge } from "@/components/ui/badge" +import { fetchStats, fetchActivities, fetchSavedPapers, fetchLLMUsage, fetchDeadlineRadar } from "@/lib/api" export default async function DashboardPage() { - const [statsResult, trendsResult, tasksResult, readingQueueResult, llmUsageResult, deadlineResult] = await Promise.allSettled([ + const [statsResult, activitiesResult, savedResult, llmResult, deadlineResult] = await Promise.allSettled([ fetchStats(), - fetchTrendingTopics(), - fetchPipelineTasks(), - fetchReadingQueue(), + fetchActivities(), + fetchSavedPapers(), fetchLLMUsage(), fetchDeadlineRadar("default"), ]) + const stats = statsResult.status === "fulfilled" ? statsResult.value : { tracked_scholars: 0, new_papers: 0, llm_usage: "0", read_later: 0, } - const trends = trendsResult.status === "fulfilled" ? trendsResult.value : [] - const tasks = tasksResult.status === "fulfilled" ? tasksResult.value : [] - const readingQueue = readingQueueResult.status === "fulfilled" ? readingQueueResult.value : [] - const usageSummary = llmUsageResult.status === "fulfilled" ? llmUsageResult.value : { + const activities = activitiesResult.status === "fulfilled" ? activitiesResult.value : [] + const saved = savedResult.status === "fulfilled" ? savedResult.value : [] + const usageSummary = llmResult.status === "fulfilled" ? llmResult.value : { window_days: 7, daily: [], provider_models: [], @@ -38,108 +30,41 @@ export default async function DashboardPage() { } const deadlines = deadlineResult.status === "fulfilled" ? deadlineResult.value : [] + const today = new Date().toLocaleDateString("en-US", { + month: "short", + day: "numeric", + year: "numeric", + }) + return (
- {/* Compact Header */} -
+ {/* Header */} +
-

Hello, Researcher

-

Daily intelligence briefing

-
-
- - Welcome back + -
+

{today}

- {/* Stats Row - Compact */} -
- - - - -
- - {/* Main Grid - 12 Column Dense Layout */} -
- {/* Activity Feed - Main Content */} -
- -
- - {/* Right Sidebar */} -
- - - - + {/* Main Grid: 2/3 + 1/3 */} +
+ {/* Left: Activity Timeline */} +
+
- {/* Bottom Row */} -
+ {/* Right: stacked cards */} +
+ +
- - - - - Trending - - - -
-
-
-

Calls

-

{usageSummary.totals?.calls || 0}

-
-
-

Tokens

-

{(usageSummary.totals?.total_tokens || 0).toLocaleString()}

-
-
-

Cost (USD)

-

${Number(usageSummary.totals?.total_cost_usd || 0).toFixed(4)}

-
-
- -
- {(usageSummary.provider_models || []).slice(0, 6).map((row) => ( -
-
-

{row.provider_name} / {row.model_name}

-

calls: {row.calls}

-
-
-

{row.total_tokens.toLocaleString()} tok

-

${Number(row.total_cost_usd || 0).toFixed(4)}

-
-
- ))} - {(!usageSummary.provider_models || usageSummary.provider_models.length === 0) && ( -
No usage records yet.
- )} -
- -
- {trends.map((topic) => ( - - {topic.text} - - ))} -
-
- -
) diff --git a/web/src/app/settings/page.tsx b/web/src/app/settings/page.tsx index e6748c00..c9a04833 100644 --- a/web/src/app/settings/page.tsx +++ b/web/src/app/settings/page.tsx @@ -1,12 +1,20 @@ "use client" import { useEffect, useMemo, useState } from "react" -import { CheckCircle2, Loader2, PlugZap, Plus, Save, Trash2, Wrench } from "lucide-react" +import { CheckCircle2, KeyRound, Loader2, Plus, Trash2, Wrench, Pencil } from "lucide-react" import { Badge } from "@/components/ui/badge" import { Button } from "@/components/ui/button" -import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card" +import { Card, CardContent } from "@/components/ui/card" import { Input } from "@/components/ui/input" +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog" type ModelEndpoint = { id: number @@ -20,10 +28,7 @@ type ModelEndpoint = { enabled: boolean is_default: boolean api_key_present?: boolean -} - -type ModelEndpointListResponse = { - items: ModelEndpoint[] + key_source?: string } type FormState = { @@ -59,15 +64,6 @@ const QUICK_PRESETS: Preset[] = [ models: ["gpt-4o-mini"], task_types: ["default", "summary", "chat"], }, - { - label: "OpenRouter", - name: "OpenRouter", - vendor: "openai_compatible", - base_url: "https://openrouter.ai/api/v1", - api_key_env: "OPENROUTER_API_KEY", - models: ["openai/gpt-4o-mini"], - task_types: ["reasoning", "review"], - }, { label: "Anthropic", name: "Anthropic", @@ -77,6 +73,15 @@ const QUICK_PRESETS: Preset[] = [ models: ["claude-3-5-sonnet-20241022"], task_types: ["reasoning", "analysis"], }, + { + label: "OpenRouter", + name: "OpenRouter", + vendor: "openai_compatible", + base_url: "https://openrouter.ai/api/v1", + api_key_env: "OPENROUTER_API_KEY", + models: ["openai/gpt-4o-mini"], + task_types: ["reasoning", "review"], + }, { label: "Ollama", name: "Local Ollama", @@ -107,25 +112,31 @@ function toPayload(form: FormState) { base_url: form.base_url.trim() || null, api_key_env: form.api_key_env.trim(), api_key: form.api_key, - models: form.models - .split(",") - .map((x) => x.trim()) - .filter(Boolean), - task_types: form.task_types - .split(",") - .map((x) => x.trim()) - .filter(Boolean), + models: form.models.split(",").map((x) => x.trim()).filter(Boolean), + task_types: form.task_types.split(",").map((x) => x.trim()).filter(Boolean), enabled: form.enabled, is_default: form.is_default, } } +function maskKey(key?: string): string { + if (!key) return "" + if (key.length <= 8) return "****" + return "****" + key.slice(-4) +} + +function statusDot(item: ModelEndpoint) { + if (item.is_default) return "bg-green-500" + if (!item.api_key_present) return "bg-red-500" + return "bg-gray-400" +} + export default function SettingsPage() { const [items, setItems] = useState([]) const [loading, setLoading] = useState(false) const [saving, setSaving] = useState(false) const [testingId, setTestingId] = useState(null) - const [activatingId, setActivatingId] = useState(null) + const [dialogOpen, setDialogOpen] = useState(false) const [form, setForm] = useState(EMPTY_FORM) const [error, setError] = useState(null) const [message, setMessage] = useState(null) @@ -138,7 +149,7 @@ export default function SettingsPage() { try { const res = await fetch("/api/model-endpoints") if (!res.ok) throw new Error(`${res.status} ${res.statusText}`) - const payload = (await res.json()) as ModelEndpointListResponse + const payload = await res.json() setItems(payload.items || []) } catch (e) { setError(e instanceof Error ? e.message : String(e)) @@ -148,46 +159,40 @@ export default function SettingsPage() { } } - useEffect(() => { - load().catch(() => {}) - }, []) - - function resetForm() { - setForm(EMPTY_FORM) - setMessage(null) - setError(null) - } - - function applyPreset(preset: Preset) { - setForm((prev) => ({ - ...prev, - name: preset.name, - vendor: preset.vendor, - base_url: preset.base_url, - api_key_env: preset.api_key_env, - models: preset.models.join(", "), - task_types: preset.task_types.join(", "), - enabled: true, - })) - setMessage(`Preset applied: ${preset.label}`) + useEffect(() => { load().catch(() => {}) }, []) + + function openAdd(preset?: Preset) { + const base = { ...EMPTY_FORM } + if (preset) { + base.name = preset.name + base.vendor = preset.vendor + base.base_url = preset.base_url + base.api_key_env = preset.api_key_env + base.models = preset.models.join(", ") + base.task_types = preset.task_types.join(", ") + } + setForm(base) setError(null) + setMessage(null) + setDialogOpen(true) } - function editItem(item: ModelEndpoint) { + function openEdit(item: ModelEndpoint) { setForm({ id: item.id, name: item.name, vendor: item.vendor, base_url: item.base_url || "", api_key_env: item.api_key_env, - api_key: item.api_key || "", + api_key: "", models: (item.models || []).join(", "), task_types: (item.task_types || []).join(", "), enabled: item.enabled, is_default: item.is_default, }) - setMessage(null) setError(null) + setMessage(null) + setDialogOpen(true) } async function saveItem() { @@ -198,7 +203,6 @@ export default function SettingsPage() { const payload = toPayload(form) if (!payload.name) throw new Error("Name is required") if (!payload.models.length) throw new Error("At least one model is required") - const res = await fetch(editing ? `/api/model-endpoints/${form.id}` : "/api/model-endpoints", { method: editing ? "PATCH" : "POST", headers: { "Content-Type": "application/json" }, @@ -209,7 +213,7 @@ export default function SettingsPage() { throw new Error(text || `${res.status} ${res.statusText}`) } await load() - resetForm() + setDialogOpen(false) setMessage(editing ? "Provider updated." : "Provider created.") } catch (e) { setError(e instanceof Error ? e.message : String(e)) @@ -226,9 +230,6 @@ export default function SettingsPage() { const res = await fetch(`/api/model-endpoints/${id}`, { method: "DELETE" }) if (!res.ok) throw new Error(`${res.status} ${res.statusText}`) await load() - if (form.id === id) { - resetForm() - } setMessage("Provider removed.") } catch (e) { setError(e instanceof Error ? e.message : String(e)) @@ -236,21 +237,15 @@ export default function SettingsPage() { } async function activateItem(id: number) { - setActivatingId(id) setError(null) setMessage(null) try { const res = await fetch(`/api/model-endpoints/${id}/activate`, { method: "POST" }) - if (!res.ok) { - const text = await res.text().catch(() => "") - throw new Error(text || `${res.status} ${res.statusText}`) - } + if (!res.ok) throw new Error(await res.text().catch(() => `${res.status}`)) await load() setMessage("Provider activated.") } catch (e) { setError(e instanceof Error ? e.message : String(e)) - } finally { - setActivatingId(null) } } @@ -265,9 +260,7 @@ export default function SettingsPage() { body: JSON.stringify({ remote: false }), }) const payload = await res.json().catch(() => ({})) - if (!res.ok) { - throw new Error(String(payload?.detail || `${res.status} ${res.statusText}`)) - } + if (!res.ok) throw new Error(String(payload?.detail || `${res.status}`)) setMessage(payload?.message || "Connection test passed.") } catch (e) { setError(e instanceof Error ? e.message : String(e)) @@ -277,192 +270,197 @@ export default function SettingsPage() { } return ( -
-

Settings

- - - - Model Providers - - Add providers, switch default route, test connectivity, and keep API keys masked in UI. - - - - {error &&

{error}

} - {message &&

{message}

} - -
-

Quick Presets

-
- {QUICK_PRESETS.map((preset) => ( - - ))} -
-
- -
-
- - setForm((p) => ({ ...p, name: e.target.value }))} placeholder="DeepSeek via OpenRouter" /> -
+
+

Settings

+ +
+

Model Providers

+

Configure LLM providers for paper analysis

+
+ + {error &&

{error}

} + {message &&

{message}

} + + {/* Provider Cards */} +
+ {loading ? ( +

Loading providers...

+ ) : !items.length ? ( +

No providers configured yet.

+ ) : ( + items.map((item) => ( + + +
+
+
+
+
+ {item.name} + {item.is_default && ( + Default + )} +
+

+ {(item.models || []).join(", ") || "no models"} · {item.base_url || "(default URL)"} +

+
+ + Key: {item.api_key_present ? maskKey(item.api_key || "present") : "missing"} + + {item.key_source === "keychain" && ( + + Keychain + + )} + {(item.task_types || []).length > 0 && ( + + Tasks: {item.task_types.join(", ")} + + )} +
+
+
-
- - +
+ + + {!item.is_default && ( + + )} + +
+
+ + + )) + )} +
+ + {/* Add Provider + Quick Presets */} +
+ + +
+

Quick Presets

+
+ {QUICK_PRESETS.map((preset) => ( + + ))} +
+
+
+ + {/* Add/Edit Dialog */} + + + + {editing ? "Edit Provider" : "Add Provider"} + + {editing ? "Update provider configuration." : "Configure a new LLM provider."} + + + +
+
+
+ + setForm((p) => ({ ...p, name: e.target.value }))} placeholder="My Provider" /> +
+
+ + +
-
+
- setForm((p) => ({ ...p, base_url: e.target.value }))} - placeholder="https://openrouter.ai/api/v1" - /> + setForm((p) => ({ ...p, base_url: e.target.value }))} placeholder="https://api.openai.com/v1" />
-
- - setForm((p) => ({ ...p, api_key_env: e.target.value }))} - placeholder="OPENAI_API_KEY" - /> -
- -
- - setForm((p) => ({ ...p, api_key: e.target.value }))} - placeholder={editing ? "***masked value" : "sk-..."} - /> +
+
+ + setForm((p) => ({ ...p, api_key_env: e.target.value }))} placeholder="OPENAI_API_KEY" /> +
+
+ + setForm((p) => ({ ...p, api_key: e.target.value }))} + placeholder={editing ? "leave blank to keep" : "sk-..."} + /> +

+ Stored in Keychain +

+
-
- - setForm((p) => ({ ...p, models: e.target.value }))} placeholder="gpt-4o-mini, gpt-4o" /> +
+
+ + setForm((p) => ({ ...p, models: e.target.value }))} placeholder="gpt-4o-mini, gpt-4o" /> +
+
+ + setForm((p) => ({ ...p, task_types: e.target.value }))} placeholder="default, summary" /> +
-
- - setForm((p) => ({ ...p, task_types: e.target.value }))} - placeholder="default, summary, reasoning, code" - /> +
+ +
-
- setForm((p) => ({ ...p, enabled: e.target.checked }))} - /> - -
-
- setForm((p) => ({ ...p, is_default: e.target.checked }))} - /> - -
+ {error &&

{error}

}
-
+ + - -
- - - - - - Configured Providers - Used by LLM service router for task-level model selection. - - - {loading ? ( -

Loading providers...

- ) : !items.length ? ( -

No providers yet.

- ) : ( - items.map((item) => ( -
-
-
- {item.name} - {item.vendor} - {item.is_default && default} - {!item.enabled && disabled} - - {item.api_key_present ? "key ready" : "missing key"} - -
- -
- {!item.is_default && ( - - )} - - - -
-
- -
-
base_url: {item.base_url || "(default)"}
-
api_key_env: {item.api_key_env}
-
api_key: {item.api_key || "(from env only)"}
-
models: {(item.models || []).join(", ") || "-"}
-
task_routes: {(item.task_types || []).join(", ") || "-"}
-
-
- )) - )} - - + + +
) } diff --git a/web/src/components/dashboard/ActivityFeed.tsx b/web/src/components/dashboard/ActivityFeed.tsx deleted file mode 100644 index d5624a6b..00000000 --- a/web/src/components/dashboard/ActivityFeed.tsx +++ /dev/null @@ -1,29 +0,0 @@ -import { fetchActivities } from "@/lib/api" -import { NewPaperCard } from "./feed/NewPaperCard" -import { MilestoneCard } from "./feed/MilestoneCard" -import { ConferenceCard } from "./feed/ConferenceCard" - -export async function ActivityFeed() { - const activities = await fetchActivities() - - return ( -
-

Your Feed

-
- {activities.map((activity) => { - switch (activity.type) { - case "published": - return - case "milestone": - return - case "conference": - return - default: - return null - } - })} -
-
- ) -} - diff --git a/web/src/components/dashboard/ActivityTimeline.tsx b/web/src/components/dashboard/ActivityTimeline.tsx new file mode 100644 index 00000000..bec4a6e4 --- /dev/null +++ b/web/src/components/dashboard/ActivityTimeline.tsx @@ -0,0 +1,47 @@ +import { FileText, Bookmark, Layers } from "lucide-react" +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card" +import type { TimelineItem } from "@/lib/types" + +interface ActivityTimelineProps { + items: TimelineItem[] +} + +const kindIcon = { + harvest: FileText, + save: Bookmark, + note: Layers, +} as const + +export function ActivityTimeline({ items }: ActivityTimelineProps) { + return ( + + + Recent Activity + + + {!items.length ? ( +

No recent activity.

+ ) : ( + items.map((item) => { + const Icon = kindIcon[item.kind] || FileText + return ( +
+ +
+

{item.title}

+ {item.subtitle && ( +

{item.subtitle}

+ )} +
+ {item.timestamp} +
+ ) + }) + )} + + + ) +} diff --git a/web/src/components/dashboard/DeadlineRadar.tsx b/web/src/components/dashboard/DeadlineRadar.tsx index d12e951e..d635b002 100644 --- a/web/src/components/dashboard/DeadlineRadar.tsx +++ b/web/src/components/dashboard/DeadlineRadar.tsx @@ -18,55 +18,44 @@ export function DeadlineRadar({ items }: DeadlineRadarProps) { Deadline Radar - + {!items.length ? ( -

No upcoming deadlines in selected window.

+

No upcoming deadlines.

) : ( - items.map((item) => ( -
+ items.slice(0, 5).map((item) => ( +
-
-

{item.name}

+
+

{item.name}

{item.field} · D-{item.days_left}

- + CCF {item.ccf_level}
- {!!item.matched_tracks?.length && ( -
- {item.matched_tracks.slice(0, 2).map((track) => ( - - - Track: {track.track_name} - - - ))} -
- )} - -
- - Open in Workflows - +
+ {item.matched_tracks?.slice(0, 2).map((track) => ( + + + {track.track_name} + + + ))} {item.url && ( - Official CFP + CFP )}
diff --git a/web/src/components/dashboard/LLMUsageChart.tsx b/web/src/components/dashboard/LLMUsageChart.tsx index 11bbb06c..7546fa41 100644 --- a/web/src/components/dashboard/LLMUsageChart.tsx +++ b/web/src/components/dashboard/LLMUsageChart.tsx @@ -50,28 +50,38 @@ export function LLMUsageChart({ data }: LLMUsageChartProps) { return ( - - LLM Token Usage ({data.window_days} Days) + + + LLM Usage ({data.window_days}d) + +
+ {(data.totals?.total_tokens || 0).toLocaleString()} tokens + ${Number(data.totals?.total_cost_usd || 0).toFixed(2)} +
- - - - - - `${Math.round(Number(v) / 1000)}k`} /> - `${Number(value).toLocaleString()} tokens`} /> - formatProviderLabel(value)} /> - {providerKeys.map((provider, index) => ( - - ))} - - + + {chartRows.length > 0 ? ( + + + + + `${Math.round(Number(v) / 1000)}k`} /> + `${Number(value).toLocaleString()} tokens`} /> + formatProviderLabel(value)} /> + {providerKeys.map((provider, index) => ( + + ))} + + + ) : ( +

No usage data yet.

+ )} ) diff --git a/web/src/components/dashboard/PipelineStatus.tsx b/web/src/components/dashboard/PipelineStatus.tsx deleted file mode 100644 index b243a17d..00000000 --- a/web/src/components/dashboard/PipelineStatus.tsx +++ /dev/null @@ -1,47 +0,0 @@ -"use client" - -import { Progress } from "@/components/ui/progress" -import { Badge } from "@/components/ui/badge" -import { Button } from "@/components/ui/button" -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card" -import { RefreshCw, AlertCircle, CheckCircle2, Loader2 } from "lucide-react" -import type { PipelineTask } from "@/lib/types" - -interface PipelineStatusProps { - tasks: PipelineTask[] -} - -const statusIcons = { - downloading: , - analyzing: , - building: , - testing: , - success: , - failed: -} - -export function PipelineStatus({ tasks }: PipelineStatusProps) { - return ( - - - Pipeline - {tasks.length} - - - {tasks.map((task) => ( -
-
-
- {statusIcons[task.status]} - {task.paper_title} -
- {task.started_at} -
- -
- ))} - - - ) -} - diff --git a/web/src/components/dashboard/QuickActions.tsx b/web/src/components/dashboard/QuickActions.tsx deleted file mode 100644 index cddbe672..00000000 --- a/web/src/components/dashboard/QuickActions.tsx +++ /dev/null @@ -1,34 +0,0 @@ -"use client" - -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card" -import { Button } from "@/components/ui/button" -import { RefreshCw, Download, FileText, Sparkles } from "lucide-react" - -export function QuickActions() { - return ( - - - Actions - - - - - - - - - ) -} - diff --git a/web/src/components/dashboard/QuickFilters.tsx b/web/src/components/dashboard/QuickFilters.tsx deleted file mode 100644 index 80c0091c..00000000 --- a/web/src/components/dashboard/QuickFilters.tsx +++ /dev/null @@ -1,59 +0,0 @@ -"use client" - -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card" -import { Badge } from "@/components/ui/badge" -import { Switch } from "@/components/ui/switch" -import { Label } from "@/components/ui/label" -import { Checkbox } from "@/components/ui/checkbox" - -export function QuickFilters() { - return ( - - - Quick Filters - - -
-
- - -
-
- - -
-
- - -
-
- -
-

Affiliation Types

-
- -
- -
-
-
- -
- -
-
-
- - - ) -} diff --git a/web/src/components/dashboard/ReadingQueue.tsx b/web/src/components/dashboard/ReadingQueue.tsx deleted file mode 100644 index 87ecc037..00000000 --- a/web/src/components/dashboard/ReadingQueue.tsx +++ /dev/null @@ -1,38 +0,0 @@ -"use client" - -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card" -import { Badge } from "@/components/ui/badge" -import { Clock } from "lucide-react" -import type { ReadingQueueItem } from "@/lib/types" -import Link from "next/link" - -interface ReadingQueueProps { - items: ReadingQueueItem[] -} - -export function ReadingQueue({ items }: ReadingQueueProps) { - return ( - - - Queue - {items.length} - - - {items.map((item) => ( -
-
- - {item.title} - -
- - {item.estimated_time} -
-
-
- ))} - - - ) -} - diff --git a/web/src/components/dashboard/SavedPapers.tsx b/web/src/components/dashboard/SavedPapers.tsx new file mode 100644 index 00000000..20efccfb --- /dev/null +++ b/web/src/components/dashboard/SavedPapers.tsx @@ -0,0 +1,37 @@ +import Link from "next/link" +import { Bookmark } from "lucide-react" +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card" +import type { SavedPaper } from "@/lib/types" + +interface SavedPapersProps { + items: SavedPaper[] +} + +export function SavedPapers({ items }: SavedPapersProps) { + return ( + + + + + Saved Papers + + + + {!items.length ? ( +

No saved papers yet.

+ ) : ( + items.map((item) => ( + +

{item.title}

+

{item.authors}

+ + )) + )} + + + ) +} diff --git a/web/src/components/dashboard/StatsBar.tsx b/web/src/components/dashboard/StatsBar.tsx new file mode 100644 index 00000000..d15b00be --- /dev/null +++ b/web/src/components/dashboard/StatsBar.tsx @@ -0,0 +1,29 @@ +import { FileText, Zap, Bookmark, CalendarClock } from "lucide-react" + +interface StatsBarProps { + papers: number + tokens: string + saved: number + tracks: number +} + +export function StatsBar({ papers, tokens, saved, tracks }: StatsBarProps) { + const items = [ + { icon: FileText, label: "papers", value: papers }, + { icon: Zap, label: "tokens", value: tokens }, + { icon: Bookmark, label: "saved", value: saved }, + { icon: CalendarClock, label: "tracks", value: tracks }, + ] + + return ( +
+ {items.map((item) => ( + + + {item.value} + {item.label} + + ))} +
+ ) +} diff --git a/web/src/components/dashboard/StatsCard.tsx b/web/src/components/dashboard/StatsCard.tsx deleted file mode 100644 index 761d8b80..00000000 --- a/web/src/components/dashboard/StatsCard.tsx +++ /dev/null @@ -1,31 +0,0 @@ -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card" -import { LucideIcon } from "lucide-react" - -interface StatsCardProps { - title: string - value: string - description?: string - icon: LucideIcon -} - -export function StatsCard({ title, value, description, icon: Icon }: StatsCardProps) { - return ( - - - - {title} - - - - -
{value}
- {description && ( -

- {description} -

- )} - - - ) -} - diff --git a/web/src/components/dashboard/feed/ConferenceCard.tsx b/web/src/components/dashboard/feed/ConferenceCard.tsx deleted file mode 100644 index fcd7f702..00000000 --- a/web/src/components/dashboard/feed/ConferenceCard.tsx +++ /dev/null @@ -1,50 +0,0 @@ -"use client" - -import { Card, CardContent } from "@/components/ui/card" -import { Button } from "@/components/ui/button" -import { CalendarClock, MapPin } from "lucide-react" -import { Activity } from "@/lib/types" -import { Progress } from "@/components/ui/progress" - -interface ConferenceCardProps { - activity: Activity -} - -export function ConferenceCard({ activity }: ConferenceCardProps) { - if (!activity.conference) return null - - return ( - - -
-
- -
-
-

- Conference Deadline Reminder: {activity.conference.name} -

-
- {activity.conference.location} - • - {activity.conference.date} -
-

- Submission deadline in {activity.conference.deadline_countdown}. -

- - -
- - -
-
-
- - - ) -} diff --git a/web/src/components/dashboard/feed/MilestoneCard.tsx b/web/src/components/dashboard/feed/MilestoneCard.tsx deleted file mode 100644 index 457c6f29..00000000 --- a/web/src/components/dashboard/feed/MilestoneCard.tsx +++ /dev/null @@ -1,40 +0,0 @@ -"use client" - -import { Card, CardContent } from "@/components/ui/card" -import { Button } from "@/components/ui/button" -import { Trophy, ArrowUpRight } from "lucide-react" -import { Activity } from "@/lib/types" - -interface MilestoneCardProps { - activity: Activity -} - -export function MilestoneCard({ activity }: MilestoneCardProps) { - if (!activity.milestone) return null - - return ( - - - {/* Decoration */} - - -
-
- -
-
-

- {activity.milestone.title} -

-

- {activity.milestone.description} -

- -
-
- - - ) -} diff --git a/web/src/components/dashboard/feed/NewPaperCard.tsx b/web/src/components/dashboard/feed/NewPaperCard.tsx deleted file mode 100644 index b19a74c2..00000000 --- a/web/src/components/dashboard/feed/NewPaperCard.tsx +++ /dev/null @@ -1,91 +0,0 @@ -"use client" - -import { Card, CardContent } from "@/components/ui/card" -import { Badge } from "@/components/ui/badge" -import { Button } from "@/components/ui/button" -import { BarChart, Star, Laptop, ArrowRight } from "lucide-react" -import { Activity } from "@/lib/types" - -interface NewPaperCardProps { - activity: Activity -} - -export function NewPaperCard({ activity }: NewPaperCardProps) { - if (!activity.paper || !activity.scholar) return null - - return ( - - - - {/* Header Section */} -
-

- {activity.paper.title} -

-
- - {activity.scholar.name}, {activity.paper.venue} · {activity.scholar.affiliation} - - {/* Optional Affiliation Badge */} - - {activity.scholar.affiliation} - -
-
- - {/* Abstract Preview */} -

- {activity.paper.abstract_snippet || "No abstract available for this paper."}... -

- - {/* Tags Row */} -
- - {activity.paper.venue} - - - {activity.paper.year} - - {/* Add a static field tag as example since it's in the design spec but not in mock data explicitly */} - - Security - -
- - {/* Metrics Row */} -
-
- - {activity.paper.citations} citations -
- - {/* Mock influential count since it's not in Activity type yet, or derive it */} -
- - 45 influential -
- -
- - Code available -
-
- - {/* Footer Section */} -
-
- Published: {activity.timestamp} · Indexed: Today -
- -
- - - - ) -} diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 30683720..a3049166 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -1,5 +1,4 @@ import { - Activity, Paper, PaperDetails, Scholar, @@ -7,8 +6,8 @@ import { Stats, WikiConcept, TrendingTopic, - PipelineTask, - ReadingQueueItem, + TimelineItem, + SavedPaper, LLMUsageSummary, DeadlineRadarItem, } from "./types" @@ -61,50 +60,20 @@ export async function fetchStats(): Promise { } } -export async function fetchActivities(): Promise { - return [ - { - id: "act-1", - type: "published", - timestamp: "Dec 3, 2025 · 12:00 PM", - scholar: { - name: "Dawn Song", - avatar: "https://avatar.vercel.sh/dawn.png", - affiliation: "UC Berkeley" - }, - paper: { - title: "Large Language Models for Academic Research: A Comprehensive Review", - venue: "NeurIPS", - year: "2024", - citations: 127, - tags: ["Security", "LLM"], - abstract_snippet: "This paper provides a comprehensive review of recent advancements in large language models (LLMs) specifically tailored for academic research applications.", - is_influential: true - } - }, - { - id: "act-2", - type: "milestone", - timestamp: "2h ago", - milestone: { - title: "Citation Milestone Reached: 1,000 Citations", - description: "Your tracked scholar, Andrew Ng, has reached a total of 1,000 citations across all publications.", - current_value: 1000, - trend: "up" - } - }, - { - id: "act-3", - type: "conference", - timestamp: "5h ago", - conference: { - name: "ICML 2025", - location: "Vancouver, Canada", - date: "July 2025", - deadline_countdown: "5 days, 14 hours" - } - } - ] +export async function fetchActivities(): Promise { + try { + const papers = await fetchPapers() + const items: TimelineItem[] = papers.slice(0, 10).map((p, i) => ({ + id: `tl-${i}`, + kind: p.status === "Saved" ? "save" as const : "harvest" as const, + title: p.title, + subtitle: p.venue, + timestamp: "recently", + })) + return items + } catch { + return [] + } } export async function fetchTrendingTopics(): Promise { @@ -120,28 +89,32 @@ export async function fetchTrendingTopics(): Promise { ] } -export async function fetchPipelineTasks(): Promise { - return [ - { id: "1", paper_title: "Attention Is All You Need", status: "testing", progress: 80, started_at: "5m ago" }, - { id: "2", paper_title: "ResNet: Deep Residual Learning", status: "building", progress: 45, started_at: "12m ago" }, - { id: "3", paper_title: "BERT Pretraining", status: "failed", progress: 100, started_at: "1h ago" } - ] -} - -export async function fetchReadingQueue(): Promise { - return [ - { id: "1", paper_id: "attention-is-all-you-need", title: "Attention Is All You Need", estimated_time: "15 min", priority: 1 }, - { id: "2", paper_id: "bert-pretraining", title: "BERT Pretraining", estimated_time: "20 min", priority: 2 }, - { id: "3", paper_id: "resnet", title: "ResNet Paper", estimated_time: "10 min", priority: 3 } - ] +export async function fetchSavedPapers(): Promise { + try { + const papers = await fetchPapers() + return papers + .filter((p) => p.status === "Saved") + .slice(0, 5) + .map((p) => ({ + id: p.id, + paper_id: p.id, + title: p.title, + authors: p.authors, + saved_at: "recently", + })) + } catch { + return [] + } } export async function fetchLLMUsage(days: number = 7): Promise { try { const qs = new URLSearchParams({ days: String(days) }) - const res = await fetch(`${API_BASE_URL}/model-endpoints/usage?${qs.toString()}`, { - cache: "no-store", - }) + // Use Next.js proxy route for SSR compatibility + const url = typeof window === "undefined" + ? `${API_BASE_URL}/model-endpoints/usage?${qs.toString()}` + : `/api/model-endpoints/usage?${qs.toString()}` + const res = await fetch(url, { cache: "no-store" }) if (!res.ok) throw new Error("usage endpoint unavailable") const payload = await res.json() as { summary?: LLMUsageSummary } if (payload.summary) { diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index d66ba219..de457f4f 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -14,7 +14,7 @@ export interface Paper { venue: string authors: string citations: string | number - status: "pending" | "analyzing" | "Reproduced" + status: "pending" | "analyzing" | "Reproduced" | "Saved" tags: string[] } @@ -38,40 +38,22 @@ export interface TrendingTopic { } -export type ActivityType = "published" | "alert" | "repro" | "milestone" | "conference" +export type TimelineItemKind = "harvest" | "save" | "note" -export interface Activity { +export interface TimelineItem { id: string - type: ActivityType + kind: TimelineItemKind + title: string + subtitle?: string timestamp: string - // Type-specific fields - paper?: { - title: string - venue: string - year: string - citations: number - tags: string[] - abstract_snippet: string - is_influential?: boolean - } - scholar?: { - name: string - avatar: string - affiliation: string - } - milestone?: { - title: string - description: string - current_value: number - target_value?: number - trend: "up" | "down" | "flat" - } - conference?: { - name: string - location: string - date: string - deadline_countdown: string - } +} + +export interface SavedPaper { + id: string + paper_id: string + title: string + authors: string + saved_at: string } export interface Stats { @@ -107,22 +89,6 @@ export interface WikiConcept { icon: string // Lucide icon name or identifier } -export interface PipelineTask { - id: string - paper_title: string - status: "downloading" | "analyzing" | "building" | "testing" | "success" | "failed" - progress: number - started_at: string -} - -export interface ReadingQueueItem { - id: string - paper_id: string - title: string - estimated_time: string - priority: number -} - export interface LLMUsageDailyRecord { date: string total_tokens: number