From 4aead2695cba986a0f54795308a8a858970239d9 Mon Sep 17 00:00:00 2001 From: jerry609 <1772030600@qq.com> Date: Thu, 12 Mar 2026 14:42:31 +0800 Subject: [PATCH] feat(research): close citation graph and obsidian export gaps --- config/config.yaml | 2 + config/settings.py | 13 + src/paperbot/api/main.py | 2 + src/paperbot/api/routes/__init__.py | 2 + src/paperbot/api/routes/obsidian.py | 86 +++++ .../application/ports/vault_exporter_port.py | 2 + .../infrastructure/api_clients/__init__.py | 5 + .../api_clients/citation_graph.py | 318 ++++++++++++++++ .../infrastructure/exporters/__init__.py | 2 + .../exporters/obsidian_exporter.py | 356 ++++++++++++++++-- .../exporters/obsidian_report_exporter.py | 179 +++++++++ .../infrastructure/exporters/obsidian_sync.py | 31 +- src/paperbot/presentation/cli/main.py | 2 + tests/unit/test_citation_graph_client.py | 172 +++++++++ tests/unit/test_obsidian_cli.py | 10 +- tests/unit/test_obsidian_exporter.py | 85 ++++- tests/unit/test_obsidian_report_exporter.py | 72 ++++ tests/unit/test_obsidian_report_routes.py | 96 +++++ tests/unit/test_obsidian_sync.py | 28 +- 19 files changed, 1422 insertions(+), 41 deletions(-) create mode 100644 src/paperbot/api/routes/obsidian.py create mode 100644 src/paperbot/infrastructure/api_clients/citation_graph.py create mode 100644 src/paperbot/infrastructure/exporters/obsidian_report_exporter.py create mode 100644 tests/unit/test_citation_graph_client.py create mode 100644 tests/unit/test_obsidian_report_exporter.py create mode 100644 tests/unit/test_obsidian_report_routes.py diff --git a/config/config.yaml b/config/config.yaml index ff06e2cc..45cee245 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -126,3 +126,5 @@ obsidian: auto_export_on_save: true auto_sync_tracks: true export_limit: 200 + track_moc_filename: "_MOC.md" + group_tracks_in_folders: true diff --git a/config/settings.py b/config/settings.py index 747b0c58..4c23a3a2 100644 --- a/config/settings.py +++ b/config/settings.py @@ -142,6 +142,8 @@ class ObsidianConfig(BaseModel): auto_export_on_save: bool = True auto_sync_tracks: bool = True export_limit: int = 200 + track_moc_filename: str = "_MOC.md" + group_tracks_in_folders: bool = True class CollabHostConfig(BaseModel): @@ -334,6 +336,8 @@ def load_environment_variables(self) -> None: obsidian_auto_export = os.getenv("PAPERBOT_OBSIDIAN_AUTO_EXPORT") obsidian_auto_sync_tracks = os.getenv("PAPERBOT_OBSIDIAN_AUTO_SYNC_TRACKS") obsidian_export_limit = os.getenv("PAPERBOT_OBSIDIAN_EXPORT_LIMIT") + obsidian_track_moc_filename = os.getenv("PAPERBOT_OBSIDIAN_TRACK_MOC_FILENAME") + obsidian_group_tracks = os.getenv("PAPERBOT_OBSIDIAN_GROUP_TRACKS_IN_FOLDERS") if re_enabled is not None: self.report_engine.enabled = re_enabled.lower() in ("1", "true", "yes", "on") if re_api: @@ -393,6 +397,15 @@ def load_environment_variables(self) -> None: "Ignoring invalid PAPERBOT_OBSIDIAN_EXPORT_LIMIT=%r; expected an integer", obsidian_export_limit, ) + if obsidian_track_moc_filename: + self.obsidian.track_moc_filename = obsidian_track_moc_filename + if obsidian_group_tracks is not None: + self.obsidian.group_tracks_in_folders = obsidian_group_tracks.lower() in ( + "1", + "true", + "yes", + "on", + ) # Collab host LLM host_api = os.getenv("PAPERBOT_HOST_API_KEY") diff --git a/src/paperbot/api/main.py b/src/paperbot/api/main.py index 25472b7f..3c3bf350 100644 --- a/src/paperbot/api/main.py +++ b/src/paperbot/api/main.py @@ -24,6 +24,7 @@ research, paperscool, newsletter, + obsidian, harvest, model_endpoints, studio_chat, @@ -80,6 +81,7 @@ async def health_check(): app.include_router(research.router, prefix="/api", tags=["Research"]) app.include_router(paperscool.router, prefix="/api", tags=["PapersCool"]) app.include_router(newsletter.router, prefix="/api", tags=["Newsletter"]) +app.include_router(obsidian.router, prefix="/api", tags=["Obsidian"]) app.include_router(harvest.router, prefix="/api", tags=["Harvest"]) app.include_router(model_endpoints.router, prefix="/api", tags=["Model Endpoints"]) app.include_router(studio_chat.router, prefix="/api", tags=["Studio Chat"]) diff --git a/src/paperbot/api/routes/__init__.py b/src/paperbot/api/routes/__init__.py index 277b2c77..fd3d34eb 100644 --- a/src/paperbot/api/routes/__init__.py +++ b/src/paperbot/api/routes/__init__.py @@ -14,6 +14,7 @@ research, paperscool, newsletter, + obsidian, model_endpoints, repro_context, feed, @@ -36,6 +37,7 @@ "research", "paperscool", "newsletter", + "obsidian", "model_endpoints", "repro_context", "feed", diff --git a/src/paperbot/api/routes/obsidian.py b/src/paperbot/api/routes/obsidian.py new file mode 100644 index 00000000..c188a43f --- /dev/null +++ b/src/paperbot/api/routes/obsidian.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from pathlib import Path +from typing import List, Optional + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel, Field + +from config.settings import create_settings +from paperbot.infrastructure.exporters import ObsidianReportExporter + +router = APIRouter() + + +class ObsidianReportCitationRequest(BaseModel): + title: str = Field(..., min_length=1, max_length=500) + year: Optional[int] = None + authors: List[str] = Field(default_factory=list) + relevant_finding: str = "" + doi: Optional[str] = None + arxiv_id: Optional[str] = None + semantic_scholar_id: Optional[str] = None + id: Optional[str] = None + + +class ObsidianReportSectionRequest(BaseModel): + title: str = Field(..., min_length=1, max_length=200) + content: str = "" + cited_papers: List[ObsidianReportCitationRequest] = Field(default_factory=list) + + +class ObsidianMethodComparisonRequest(BaseModel): + name: str = Field(..., min_length=1, max_length=120) + paper: str = Field(..., min_length=1, max_length=500) + pros: str = "" + cons: str = "" + + +class ObsidianExportReportRequest(BaseModel): + title: str = Field(..., min_length=1, max_length=300) + vault_path: Optional[str] = None + root_dir: Optional[str] = None + track_name: Optional[str] = Field(default=None, max_length=128) + workflow_type: str = "research" + summary: str = "" + key_insight: str = "" + sections: List[ObsidianReportSectionRequest] = Field(default_factory=list) + methods: List[ObsidianMethodComparisonRequest] = Field(default_factory=list) + trends: str = "" + future_directions: str = "" + references: List[ObsidianReportCitationRequest] = Field(default_factory=list) + tags: List[str] = Field(default_factory=list) + + +class ObsidianExportReportResponse(BaseModel): + vault_path: str + root_dir: str + title: str + note_path: str + + +@router.post("/obsidian/export-report", response_model=ObsidianExportReportResponse) +def export_obsidian_report(req: ObsidianExportReportRequest) -> ObsidianExportReportResponse: + settings = create_settings() + obsidian_config = settings.obsidian + + vault_value = str(req.vault_path or obsidian_config.vault_path or "").strip() + if not vault_value: + raise HTTPException( + status_code=400, + detail="vault_path is required. Pass it in the request or configure obsidian.vault_path.", + ) + + exporter = ObsidianReportExporter() + try: + result = exporter.export_report_note( + vault_path=Path(vault_value), + report=req.model_dump(), + root_dir=str(req.root_dir or obsidian_config.root_dir or "PaperBot"), + track_moc_filename=getattr(obsidian_config, "track_moc_filename", "_MOC.md"), + group_tracks_in_folders=getattr(obsidian_config, "group_tracks_in_folders", True), + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + return ObsidianExportReportResponse(**result) diff --git a/src/paperbot/application/ports/vault_exporter_port.py b/src/paperbot/application/ports/vault_exporter_port.py index c2808fe4..0fecfa8e 100644 --- a/src/paperbot/application/ports/vault_exporter_port.py +++ b/src/paperbot/application/ports/vault_exporter_port.py @@ -18,4 +18,6 @@ def export_library_snapshot( track: Optional[Dict[str, Any]] = None, root_dir: str = "PaperBot", paper_template_path: Optional[Path] = None, + track_moc_filename: str = "_MOC.md", + group_tracks_in_folders: bool = True, ) -> Dict[str, Any]: ... diff --git a/src/paperbot/infrastructure/api_clients/__init__.py b/src/paperbot/infrastructure/api_clients/__init__.py index fc7b3e35..ba0905d3 100644 --- a/src/paperbot/infrastructure/api_clients/__init__.py +++ b/src/paperbot/infrastructure/api_clients/__init__.py @@ -3,12 +3,17 @@ """ from .base import APIClient +from .citation_graph import CitationGraph, CitationGraphClient, CitationGraphEdge, CitationGraphNode from .github_client import GitHubRadarClient from .semantic_scholar import SemanticScholarClient from .x_client import XRecentSearchClient __all__ = [ "APIClient", + "CitationGraph", + "CitationGraphClient", + "CitationGraphEdge", + "CitationGraphNode", "GitHubRadarClient", "SemanticScholarClient", "XRecentSearchClient", diff --git a/src/paperbot/infrastructure/api_clients/citation_graph.py b/src/paperbot/infrastructure/api_clients/citation_graph.py new file mode 100644 index 00000000..eea0c887 --- /dev/null +++ b/src/paperbot/infrastructure/api_clients/citation_graph.py @@ -0,0 +1,318 @@ +"""Multi-hop citation graph traversal built on top of Semantic Scholar.""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, field +from typing import Any, Awaitable, Callable, Dict, Iterable, List, Literal, Optional + +from paperbot.infrastructure.api_clients.semantic_scholar import SemanticScholarClient + +TraversalDirection = Literal["references", "citations", "both"] +RelevanceFilter = Callable[[Dict[str, Any]], float] + +_DEFAULT_FIELDS = [ + "title", + "year", + "citationCount", + "authors", + "references", + "citations", +] + + +@dataclass +class CitationGraphNode: + paper_id: str + title: str + year: Optional[int] = None + citation_count: int = 0 + authors: List[str] = field(default_factory=list) + hop: int = 0 + + def to_dict(self) -> Dict[str, Any]: + return { + "paper_id": self.paper_id, + "title": self.title, + "year": self.year, + "citation_count": self.citation_count, + "authors": list(self.authors), + "hop": self.hop, + } + + +@dataclass +class CitationGraphEdge: + source_id: str + target_id: str + relation: Literal["references", "citations"] + hop: int + weight: float = 1.0 + + def to_dict(self) -> Dict[str, Any]: + return { + "source_id": self.source_id, + "target_id": self.target_id, + "relation": self.relation, + "hop": self.hop, + "weight": self.weight, + } + + +@dataclass +class CitationGraph: + seed_paper_id: str + direction: TraversalDirection + nodes: Dict[str, CitationGraphNode] = field(default_factory=dict) + edges: List[CitationGraphEdge] = field(default_factory=list) + + def add_node(self, node: CitationGraphNode) -> None: + existing = self.nodes.get(node.paper_id) + if existing is None: + self.nodes[node.paper_id] = node + return + if node.hop < existing.hop: + existing.hop = node.hop + if not existing.title and node.title: + existing.title = node.title + if existing.year is None and node.year is not None: + existing.year = node.year + if not existing.authors and node.authors: + existing.authors = list(node.authors) + if node.citation_count > existing.citation_count: + existing.citation_count = node.citation_count + + def add_edge(self, edge: CitationGraphEdge) -> None: + for existing in self.edges: + if ( + existing.source_id == edge.source_id + and existing.target_id == edge.target_id + and existing.relation == edge.relation + ): + if edge.hop < existing.hop: + existing.hop = edge.hop + if edge.weight > existing.weight: + existing.weight = edge.weight + return + self.edges.append(edge) + + def to_dict(self) -> Dict[str, Any]: + return { + "seed_paper_id": self.seed_paper_id, + "direction": self.direction, + "nodes": [node.to_dict() for node in sorted(self.nodes.values(), key=lambda item: (item.hop, item.paper_id))], + "edges": [edge.to_dict() for edge in self.edges], + } + + +class CitationGraphClient: + """Traverse references and citations up to a configurable depth.""" + + def __init__( + self, + semantic_scholar_client: Optional[SemanticScholarClient] = None, + *, + api_key: Optional[str] = None, + timeout: int = 30, + request_interval: float = 1.0, + max_concurrency: int = 4, + ) -> None: + self._owns_client = semantic_scholar_client is None + self._client = semantic_scholar_client or SemanticScholarClient( + api_key=api_key, + timeout=timeout, + request_interval=request_interval, + ) + self._semaphore = asyncio.Semaphore(max(1, int(max_concurrency))) + + async def close(self) -> None: + if self._owns_client and hasattr(self._client, "close"): + await self._client.close() + + async def __aenter__(self) -> "CitationGraphClient": + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + await self.close() + + async def traverse( + self, + seed_paper_id: str, + *, + direction: TraversalDirection, + max_hops: int = 2, + max_papers_per_hop: int = 10, + relevance_filter: Optional[RelevanceFilter] = None, + fields: Optional[List[str]] = None, + ) -> CitationGraph: + normalized_seed = str(seed_paper_id or "").strip() + if not normalized_seed: + raise ValueError("seed_paper_id is required") + if direction not in {"references", "citations", "both"}: + raise ValueError("direction must be one of: references, citations, both") + if max_hops < 1: + raise ValueError("max_hops must be >= 1") + if max_papers_per_hop < 1: + raise ValueError("max_papers_per_hop must be >= 1") + + graph = CitationGraph(seed_paper_id=normalized_seed, direction=direction) + visited: set[str] = set() + frontier: Dict[str, int] = {normalized_seed: 0} + request_fields = fields or list(_DEFAULT_FIELDS) + + for hop in range(max_hops): + batch = [(paper_id, frontier_hop) for paper_id, frontier_hop in frontier.items() if paper_id and paper_id not in visited] + if not batch: + break + + tasks = [ + self._fetch_paper(paper_id=paper_id, fields=request_fields) + for paper_id, _ in batch + ] + responses = await asyncio.gather(*tasks) + next_frontier: Dict[str, int] = {} + + for (requested_id, current_hop), payload in zip(batch, responses): + visited.add(requested_id) + paper = payload or {} + if not paper: + continue + + canonical_id = self._paper_id(paper) or requested_id + graph.add_node(self._node_from_paper(paper=paper, paper_id=canonical_id, hop=current_hop)) + + for relation, related in self._related_candidates(paper=paper, direction=direction): + scored = self._score_related(related, relevance_filter=relevance_filter) + for related_paper, weight in scored[:max_papers_per_hop]: + related_id = self._paper_id(related_paper) + if not related_id: + continue + graph.add_node(self._node_from_paper(paper=related_paper, paper_id=related_id, hop=current_hop + 1)) + if relation == "references": + source_id, target_id = canonical_id, related_id + else: + source_id, target_id = related_id, canonical_id + graph.add_edge( + CitationGraphEdge( + source_id=source_id, + target_id=target_id, + relation=relation, + hop=current_hop + 1, + weight=weight, + ) + ) + if related_id not in visited and related_id not in next_frontier: + next_frontier[related_id] = current_hop + 1 + + frontier = next_frontier + + return graph + + async def _fetch_paper(self, *, paper_id: str, fields: List[str]) -> Dict[str, Any]: + async with self._semaphore: + payload = await self._client.get_paper(paper_id, fields=fields) + return payload or {} + + @staticmethod + def _paper_id(paper: Dict[str, Any]) -> str: + for key in ("paperId", "paper_id", "id", "externalId"): + value = str(paper.get(key) or "").strip() + if value: + return value + return "" + + @staticmethod + def _paper_authors(paper: Dict[str, Any]) -> List[str]: + rows: List[str] = [] + for author in paper.get("authors") or []: + if not isinstance(author, dict): + continue + name = str(author.get("name") or author.get("display_name") or "").strip() + if name: + rows.append(name) + return rows + + @classmethod + def _node_from_paper(cls, *, paper: Dict[str, Any], paper_id: str, hop: int) -> CitationGraphNode: + year = paper.get("year") + try: + parsed_year = int(year) if year is not None else None + except (TypeError, ValueError): + parsed_year = None + citation_count = paper.get("citationCount", paper.get("citation_count", 0)) + try: + parsed_citations = int(citation_count or 0) + except (TypeError, ValueError): + parsed_citations = 0 + return CitationGraphNode( + paper_id=paper_id, + title=str(paper.get("title") or paper_id).strip(), + year=parsed_year, + citation_count=parsed_citations, + authors=cls._paper_authors(paper), + hop=hop, + ) + + @classmethod + def _related_candidates( + cls, + *, + paper: Dict[str, Any], + direction: TraversalDirection, + ) -> List[tuple[Literal["references", "citations"], List[Dict[str, Any]]]]: + groups: List[tuple[Literal["references", "citations"], List[Dict[str, Any]]]] = [] + if direction in {"references", "both"}: + groups.append(("references", cls._extract_related(paper.get("references") or [], nested_key="citedPaper"))) + if direction in {"citations", "both"}: + groups.append(("citations", cls._extract_related(paper.get("citations") or [], nested_key="citingPaper"))) + return groups + + @classmethod + def _extract_related( + cls, + entries: Iterable[Any], + *, + nested_key: str, + ) -> List[Dict[str, Any]]: + related: List[Dict[str, Any]] = [] + for entry in entries: + if not isinstance(entry, dict): + continue + nested = entry.get(nested_key) + if isinstance(nested, dict): + payload = dict(nested) + else: + payload = dict(entry) + if cls._paper_id(payload): + related.append(payload) + return related + + @staticmethod + def _score_related( + papers: List[Dict[str, Any]], + *, + relevance_filter: Optional[RelevanceFilter], + ) -> List[tuple[Dict[str, Any], float]]: + deduped: Dict[str, tuple[Dict[str, Any], float]] = {} + for paper in papers: + paper_id = CitationGraphClient._paper_id(paper) + if not paper_id: + continue + try: + score = float(relevance_filter(paper)) if relevance_filter is not None else 1.0 + except Exception: + score = 0.0 + current = deduped.get(paper_id) + if current is None or score > current[1]: + deduped[paper_id] = (paper, score) + + def _sort_key(item: tuple[Dict[str, Any], float]) -> tuple[float, int, str]: + paper, score = item + citation_count = paper.get("citationCount", paper.get("citation_count", 0)) + try: + parsed_citations = int(citation_count or 0) + except (TypeError, ValueError): + parsed_citations = 0 + return (score, parsed_citations, str(paper.get("title") or "")) + + return sorted(deduped.values(), key=_sort_key, reverse=True) diff --git a/src/paperbot/infrastructure/exporters/__init__.py b/src/paperbot/infrastructure/exporters/__init__.py index 4aa276a8..2e83388c 100644 --- a/src/paperbot/infrastructure/exporters/__init__.py +++ b/src/paperbot/infrastructure/exporters/__init__.py @@ -1,10 +1,12 @@ """Filesystem exporters for external knowledge tools.""" from .obsidian_exporter import ObsidianFilesystemExporter +from .obsidian_report_exporter import ObsidianReportExporter from .obsidian_sync import export_track_snapshot, get_obsidian_config, obsidian_auto_export_enabled __all__ = [ "ObsidianFilesystemExporter", + "ObsidianReportExporter", "export_track_snapshot", "get_obsidian_config", "obsidian_auto_export_enabled", diff --git a/src/paperbot/infrastructure/exporters/obsidian_exporter.py b/src/paperbot/infrastructure/exporters/obsidian_exporter.py index bda02d1c..8ac6c858 100644 --- a/src/paperbot/infrastructure/exporters/obsidian_exporter.py +++ b/src/paperbot/infrastructure/exporters/obsidian_exporter.py @@ -32,6 +32,20 @@ - {{ link }} {% endfor %} {% endif %} +{% if reference_links %} + +## References +{% for link in reference_links -%} +- {{ link }} +{% endfor %} +{% endif %} +{% if cited_by_links %} + +## Cited By +{% for link in cited_by_links -%} +- {{ link }} +{% endfor %} +{% endif %} {% if external_links %} ## Links @@ -95,6 +109,8 @@ def export_library_snapshot( track: Optional[Dict[str, Any]] = None, root_dir: str = "PaperBot", paper_template_path: Optional[Path] = None, + track_moc_filename: str = "_MOC.md", + group_tracks_in_folders: bool = True, ) -> Dict[str, Any]: vault_dir = Path(vault_path).expanduser().resolve() if not vault_dir.exists() or not vault_dir.is_dir(): @@ -122,9 +138,13 @@ def export_library_snapshot( track=track, saved_at=item.get("saved_at"), template_path=template_path, + track_moc_filename=track_moc_filename, + group_tracks_in_folders=group_tracks_in_folders, ) ) + self._sync_paper_citation_backlinks(papers_dir=papers_dir, paper_refs=paper_refs) + track_ref: Optional[Dict[str, str]] = None if track is not None: track_ref = self._write_track_note( @@ -132,6 +152,8 @@ def export_library_snapshot( root_dir=root_dir, track=track, paper_refs=paper_refs, + track_moc_filename=track_moc_filename, + group_tracks_in_folders=group_tracks_in_folders, ) moc_path = self._write_moc_note( @@ -158,6 +180,8 @@ def _write_paper_note( track: Optional[Dict[str, Any]], saved_at: Optional[str], template_path: Optional[Path], + track_moc_filename: str, + group_tracks_in_folders: bool, ) -> Dict[str, str]: note_stem = self._paper_note_stem(paper) note_path = papers_dir / f"{note_stem}.md" @@ -169,17 +193,22 @@ def _write_paper_note( label=note_title, ) track_link = ( - self._wikilink( + self._track_wikilink( root_dir=root_dir, - section="Tracks", - note_stem=self._track_note_stem(track), label=str(track.get("name") or "Track"), + track=track, + track_moc_filename=track_moc_filename, + group_tracks_in_folders=group_tracks_in_folders, ) if track is not None else None ) related_links = self._paper_related_links(paper=paper, root_dir=root_dir) related_titles = self._paper_related_titles(paper) + reference_entries = self._paper_reference_entries(paper) + citation_entries = self._paper_citation_entries(paper) + reference_links = self._paper_entry_links(entries=reference_entries, root_dir=root_dir) + cited_by_links = self._paper_entry_links(entries=citation_entries, root_dir=root_dir) frontmatter = _yaml_frontmatter( { @@ -198,6 +227,8 @@ def _write_paper_note( "track": track.get("name") if track else None, "tags": self._paper_tags(paper, track), "related_papers": related_titles, + "cites": reference_links, + "cited_by": cited_by_links, } ) @@ -215,6 +246,8 @@ def _write_paper_note( track_link=track_link, external_links=self._paper_links(paper), related_links=related_links, + reference_links=reference_links, + cited_by_links=cited_by_links, paper=paper, track=track, related_titles=related_titles, @@ -226,6 +259,8 @@ def _write_paper_note( "path": str(note_path), "link": note_link, "stem": note_stem, + "references": reference_entries, + "citations": citation_entries, } def _write_track_note( @@ -235,14 +270,20 @@ def _write_track_note( root_dir: str, track: Dict[str, Any], paper_refs: List[Dict[str, str]], + track_moc_filename: str, + group_tracks_in_folders: bool, ) -> Dict[str, str]: note_stem = self._track_note_stem(track) - note_path = tracks_dir / f"{note_stem}.md" - note_link = self._wikilink( + note_dir = tracks_dir / note_stem if group_tracks_in_folders else tracks_dir + note_dir.mkdir(parents=True, exist_ok=True) + note_filename = track_moc_filename if group_tracks_in_folders else f"{note_stem}.md" + note_path = note_dir / note_filename + note_link = self._track_wikilink( root_dir=root_dir, - section="Tracks", - note_stem=note_stem, label=str(track.get("name") or "Track"), + track=track, + track_moc_filename=track_moc_filename, + group_tracks_in_folders=group_tracks_in_folders, ) frontmatter = _yaml_frontmatter( @@ -251,6 +292,7 @@ def _write_track_note( "track_id": track.get("id"), "user_id": track.get("user_id"), "name": track.get("name"), + "moc_note": note_filename, "paper_count": len(paper_refs), "keywords": list(track.get("keywords") or []), "methods": list(track.get("methods") or []), @@ -270,8 +312,50 @@ def _write_track_note( f"- Methods: {', '.join(track.get('methods') or []) or 'None'}", f"- Venues: {', '.join(track.get('venues') or []) or 'None'}", "", - "## Saved Papers", + "## Research Tasks", ] + tasks = list(track.get("tasks") or []) + if tasks: + for task in tasks: + title = str(task.get("title") or "Untitled task").strip() + status = str(task.get("status") or "todo").strip() + lines.append(f"- [{status}] {title}") + else: + lines.append("- _No tracked tasks._") + + lines.extend([ + "", + "## Milestones", + ]) + milestones = list(track.get("milestones") or []) + if milestones: + for milestone in milestones: + name = str(milestone.get("name") or "Untitled milestone").strip() + status = str(milestone.get("status") or "todo").strip() + lines.append(f"- [{status}] {name}") + else: + lines.append("- _No milestones defined._") + + lines.extend([ + "", + "## Tracked Scholars", + ]) + scholars = list(track.get("scholars") or track.get("tracked_scholars") or []) + if scholars: + for scholar in scholars: + name = str(scholar.get("name") or scholar.get("scholar_name") or "Unknown scholar").strip() + affiliation = str(scholar.get("affiliation") or "").strip() + if affiliation: + lines.append(f"- {name} ({affiliation})") + else: + lines.append(f"- {name}") + else: + lines.append("- _No linked scholars._") + + lines.extend([ + "", + "## Saved Papers", + ]) if paper_refs: lines.extend([f"- {ref['link']}" for ref in paper_refs]) else: @@ -330,6 +414,30 @@ def _paper_note_stem(paper: Dict[str, Any]) -> str: def _track_note_stem(track: Dict[str, Any]) -> str: return _slugify(str(track.get("name") or "track"))[:120] + def _track_wikilink( + self, + *, + root_dir: str, + label: str, + track: Dict[str, Any], + track_moc_filename: str, + group_tracks_in_folders: bool, + ) -> str: + note_stem = self._track_note_stem(track) + if group_tracks_in_folders: + return self._wikilink( + root_dir=root_dir, + section=f"Tracks/{note_stem}", + note_stem=Path(track_moc_filename).stem, + label=label, + ) + return self._wikilink( + root_dir=root_dir, + section="Tracks", + note_stem=note_stem, + label=label, + ) + @staticmethod def _paper_tags(paper: Dict[str, Any], track: Optional[Dict[str, Any]]) -> List[str]: values: List[str] = [] @@ -386,6 +494,8 @@ def _render_paper_note( track_link: Optional[str], external_links: List[str], related_links: List[str], + reference_links: List[str], + cited_by_links: List[str], paper: Dict[str, Any], track: Optional[Dict[str, Any]], related_titles: List[str], @@ -406,12 +516,106 @@ def _render_paper_note( track_link=track_link, external_links=external_links, related_links=related_links, + reference_links=reference_links, + cited_by_links=cited_by_links, paper=paper, track=track, related_titles=related_titles, ) return f"{frontmatter}{body}" + def _sync_paper_citation_backlinks( + self, + *, + papers_dir: Path, + paper_refs: List[Dict[str, Any]], + ) -> None: + for paper_ref in paper_refs: + note_path = Path(str(paper_ref.get("path") or "")).expanduser() + if not note_path: + continue + current_link = str(paper_ref.get("link") or "").strip() + if not current_link: + continue + + for entry in list(paper_ref.get("references") or []): + target_path = papers_dir / f"{self._paper_note_stem(entry)}.md" + if target_path == note_path: + continue + self._update_note_link_index( + note_path=target_path, + frontmatter_key="cited_by", + heading="Cited By", + link=current_link, + ) + + for entry in list(paper_ref.get("citations") or []): + target_path = papers_dir / f"{self._paper_note_stem(entry)}.md" + if target_path == note_path: + continue + self._update_note_link_index( + note_path=target_path, + frontmatter_key="cites", + heading="References", + link=current_link, + ) + + def _update_note_link_index( + self, + *, + note_path: Path, + frontmatter_key: str, + heading: str, + link: str, + ) -> None: + if not note_path.exists() or not note_path.is_file(): + return + + frontmatter, body = self._read_note(note_path) + values = [str(item).strip() for item in list(frontmatter.get(frontmatter_key) or []) if str(item).strip()] + if link not in values: + values.append(link) + frontmatter[frontmatter_key] = values + + updated_body = self._upsert_markdown_section(body=body, heading=heading, links=values) + note_path.write_text( + f"{_yaml_frontmatter(frontmatter)}{updated_body.rstrip()}\n", + encoding="utf-8", + ) + + @staticmethod + def _read_note(note_path: Path) -> tuple[Dict[str, Any], str]: + text = note_path.read_text(encoding="utf-8") + if text.startswith("---\n"): + match = re.match(r"^---\n(.*?)\n---\n?", text, flags=re.DOTALL) + if match: + payload = yaml.safe_load(match.group(1)) or {} + if isinstance(payload, dict): + return payload, text[match.end():] + return {}, text + + @staticmethod + def _upsert_markdown_section( + *, + body: str, + heading: str, + links: List[str], + ) -> str: + section_lines = [f"## {heading}"] + section_lines.extend([f"- {link}" for link in links]) + section_text = "\n".join(section_lines).rstrip() + + trimmed = body.rstrip() + pattern = re.compile( + rf"(?ms)^## {re.escape(heading)}\n.*?(?=^## |\Z)" + ) + if pattern.search(trimmed): + updated = pattern.sub(section_text + "\n", trimmed) + else: + separator = "\n\n" if trimmed else "" + updated = f"{trimmed}{separator}{section_text}\n" + return updated.rstrip() + "\n" + def _paper_related_links(self, *, paper: Dict[str, Any], root_dir: str) -> List[str]: links: List[str] = [] seen: set[str] = set() @@ -431,6 +635,38 @@ def _paper_related_links(self, *, paper: Dict[str, Any], root_dir: str) -> List[ ) return links + def _paper_entry_links( + self, + *, + entries: List[Dict[str, Any]], + root_dir: str, + ) -> List[str]: + links: List[str] = [] + seen: set[str] = set() + for entry in entries: + title = str(entry.get("title") or "").strip() + if not title: + continue + dedupe_key = str( + entry.get("semantic_scholar_id") + or entry.get("doi") + or entry.get("arxiv_id") + or entry.get("id") + or title.casefold() + ).strip() + if not dedupe_key or dedupe_key in seen: + continue + seen.add(dedupe_key) + links.append( + self._wikilink( + root_dir=root_dir, + section="Papers", + note_stem=self._paper_note_stem(entry), + label=title, + ) + ) + return links + def _paper_related_titles(self, paper: Dict[str, Any]) -> List[str]: titles: List[str] = [] seen: set[str] = set() @@ -442,39 +678,91 @@ def _paper_related_titles(self, paper: Dict[str, Any]) -> List[str]: titles.append(title) return titles - @staticmethod - def _paper_related_entries(paper: Dict[str, Any]) -> List[Dict[str, Any]]: + @classmethod + def _paper_reference_entries(cls, paper: Dict[str, Any]) -> List[Dict[str, Any]]: + metadata = paper.get("metadata") if isinstance(paper.get("metadata"), dict) else {} + return cls._paper_relation_entries( + [ + paper.get("references"), + metadata.get("references"), + ], + nested_keys=["citedPaper", "paper"], + ) + + @classmethod + def _paper_citation_entries(cls, paper: Dict[str, Any]) -> List[Dict[str, Any]]: + metadata = paper.get("metadata") if isinstance(paper.get("metadata"), dict) else {} + return cls._paper_relation_entries( + [ + paper.get("citations"), + metadata.get("citations"), + ], + nested_keys=["citingPaper", "paper"], + ) + + @classmethod + def _paper_related_entries(cls, paper: Dict[str, Any]) -> List[Dict[str, Any]]: + metadata = paper.get("metadata") if isinstance(paper.get("metadata"), dict) else {} + return cls._paper_relation_entries( + [ + paper.get("related_papers"), + paper.get("related_titles"), + paper.get("references"), + metadata.get("related_papers"), + metadata.get("references"), + ], + nested_keys=["citedPaper", "paper"], + ) + + @classmethod + def _paper_relation_entries( + cls, + candidates: List[Any], + *, + nested_keys: List[str], + ) -> List[Dict[str, Any]]: related: List[Dict[str, Any]] = [] - candidates = [ - paper.get("related_papers"), - paper.get("related_titles"), - paper.get("references"), - (paper.get("metadata") or {}).get("related_papers") if isinstance(paper.get("metadata"), dict) else None, - (paper.get("metadata") or {}).get("references") if isinstance(paper.get("metadata"), dict) else None, - ] for bucket in candidates: if not isinstance(bucket, list): continue for item in bucket: - if isinstance(item, dict): - title = str(item.get("title") or item.get("name") or "").strip() - if title: - related.append( - { - "title": title, - "year": item.get("year"), - "arxiv_id": item.get("arxiv_id"), - "doi": item.get("doi"), - "semantic_scholar_id": item.get("semantic_scholar_id"), - "id": item.get("id"), - } - ) - else: - title = str(item or "").strip() - if title: - related.append({"title": title}) + normalized = cls._normalize_paper_relation_entry(item, nested_keys=nested_keys) + if normalized is not None: + related.append(normalized) return related + @staticmethod + def _normalize_paper_relation_entry( + item: Any, + *, + nested_keys: List[str], + ) -> Optional[Dict[str, Any]]: + if isinstance(item, str): + title = str(item).strip() + return {"title": title} if title else None + if not isinstance(item, dict): + return None + + payload: Dict[str, Any] = dict(item) + for nested_key in nested_keys: + nested = payload.get(nested_key) + if isinstance(nested, dict): + payload = dict(nested) + break + + title = str(payload.get("title") or payload.get("name") or "").strip() + if not title: + return None + + return { + "title": title, + "year": payload.get("year"), + "arxiv_id": payload.get("arxiv_id"), + "doi": payload.get("doi"), + "semantic_scholar_id": payload.get("semantic_scholar_id") or payload.get("paperId"), + "id": payload.get("id") or payload.get("paper_id") or payload.get("paperId"), + } + @staticmethod def _wikilink( *, diff --git a/src/paperbot/infrastructure/exporters/obsidian_report_exporter.py b/src/paperbot/infrastructure/exporters/obsidian_report_exporter.py new file mode 100644 index 00000000..bfd6b80e --- /dev/null +++ b/src/paperbot/infrastructure/exporters/obsidian_report_exporter.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional + +from .obsidian_exporter import ObsidianFilesystemExporter, _slugify, _yaml_frontmatter + + +def _markdown_table_cell(value: str) -> str: + return str(value or "").replace("\n", " ").replace("|", "\\|").strip() + + +class ObsidianReportExporter: + """Render long-form research reports into Obsidian-friendly markdown notes.""" + + def __init__(self) -> None: + self._note_exporter = ObsidianFilesystemExporter() + + def export_report_note( + self, + *, + vault_path: Path, + report: Dict[str, Any], + root_dir: str = "PaperBot", + track_moc_filename: str = "_MOC.md", + group_tracks_in_folders: bool = True, + ) -> Dict[str, Any]: + vault_dir = Path(vault_path).expanduser().resolve() + if not vault_dir.exists() or not vault_dir.is_dir(): + raise ValueError("vault_path must be an existing directory") + + root_path = vault_dir / root_dir + reports_dir = root_path / "Reports" + reports_dir.mkdir(parents=True, exist_ok=True) + + title = str(report.get("title") or "Untitled Report").strip() or "Untitled Report" + note_stem = _slugify(title) + note_path = reports_dir / f"{note_stem}.md" + track_name = str(report.get("track_name") or "").strip() + tags = self._report_tags(report) + + frontmatter = _yaml_frontmatter( + { + "title": title, + "type": "research-report", + "track": ( + self._track_link( + root_dir=root_dir, + track_name=track_name, + track_moc_filename=track_moc_filename, + group_tracks_in_folders=group_tracks_in_folders, + ) + if track_name + else None + ), + "tags": tags, + "created": datetime.now(timezone.utc).isoformat(), + "agent_workflow": str(report.get("workflow_type") or "research"), + } + ) + + lines = [ + frontmatter, + f"# {title}", + ] + + summary = str(report.get("summary") or "").strip() + if summary: + lines.extend(["", *self._callout("abstract", "研究概述", summary)]) + + key_insight = str(report.get("key_insight") or "").strip() + if key_insight: + lines.extend(["", *self._callout("tip", "核心观点", key_insight)]) + + for section in list(report.get("sections") or []): + section_title = str(section.get("title") or "Untitled Section").strip() or "Untitled Section" + content = str(section.get("content") or "").strip() or "_No section content provided._" + lines.extend(["", f"## {section_title}", content]) + for cited_paper in list(section.get("cited_papers") or []): + lines.extend([ + "", + *self._callout( + "quote", + self._paper_link(root_dir=root_dir, paper=cited_paper), + str(cited_paper.get("relevant_finding") or "").strip() or "Referenced in this section.", + ), + ]) + + methods = list(report.get("methods") or []) + if methods: + lines.extend(["", "## 方法论对比", "", "| 方法 | 论文 | 优势 | 局限 |", "|---|---|---|---|"]) + for method in methods: + paper_title = str(method.get("paper") or "").strip() + paper_link = ( + self._paper_link(root_dir=root_dir, paper={"title": paper_title}) + if paper_title + else "" + ) + lines.append( + "| {name} | {paper} | {pros} | {cons} |".format( + name=_markdown_table_cell(str(method.get("name") or "")), + paper=_markdown_table_cell(paper_link), + pros=_markdown_table_cell(str(method.get("pros") or "")), + cons=_markdown_table_cell(str(method.get("cons") or "")), + ) + ) + + trends = str(report.get("trends") or "").strip() + if trends: + lines.extend(["", *self._callout("info", "趋势分析", trends)]) + + future_directions = str(report.get("future_directions") or "").strip() + if future_directions: + lines.extend(["", "## 未来方向", future_directions]) + + references = list(report.get("references") or []) + lines.extend(["", "## 引用论文"]) + if references: + for reference in references: + authors = ", ".join([str(name).strip() for name in list(reference.get("authors") or []) if str(name).strip()]) + suffix_parts = [part for part in [authors, str(reference.get("year") or "").strip()] if part] + suffix = f" — {', '.join(suffix_parts)}" if suffix_parts else "" + lines.append(f"- {self._paper_link(root_dir=root_dir, paper=reference)}{suffix}") + else: + lines.append("- _No references linked._") + + note_path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") + return { + "vault_path": str(vault_dir), + "root_dir": root_dir, + "title": title, + "note_path": str(note_path), + } + + @staticmethod + def _report_tags(report: Dict[str, Any]) -> List[str]: + tags = ["report"] + for raw_tag in list(report.get("tags") or []): + tag = _slugify(str(raw_tag)) + if tag and tag not in tags: + tags.append(tag) + workflow_tag = _slugify(str(report.get("workflow_type") or "research")) + if workflow_tag and workflow_tag not in tags: + tags.append(workflow_tag) + return tags + + def _track_link( + self, + *, + root_dir: str, + track_name: str, + track_moc_filename: str, + group_tracks_in_folders: bool, + ) -> str: + return self._note_exporter._track_wikilink( + root_dir=root_dir, + label=track_name, + track={"name": track_name}, + track_moc_filename=track_moc_filename, + group_tracks_in_folders=group_tracks_in_folders, + ) + + def _paper_link(self, *, root_dir: str, paper: Dict[str, Any]) -> str: + title = str(paper.get("title") or "Untitled Paper").strip() or "Untitled Paper" + return self._note_exporter._wikilink( + root_dir=root_dir, + section="Papers", + note_stem=self._note_exporter._paper_note_stem(paper), + label=title, + ) + + @staticmethod + def _callout(callout_type: str, title: str, content: str) -> List[str]: + lines = [f"> [!{callout_type}] {title}"] + content_lines = str(content or "").splitlines() or [""] + for line in content_lines: + lines.append(f"> {line}".rstrip()) + return lines diff --git a/src/paperbot/infrastructure/exporters/obsidian_sync.py b/src/paperbot/infrastructure/exporters/obsidian_sync.py index 55d74ef6..4a9afb7b 100644 --- a/src/paperbot/infrastructure/exporters/obsidian_sync.py +++ b/src/paperbot/infrastructure/exporters/obsidian_sync.py @@ -23,6 +23,20 @@ def obsidian_auto_export_enabled(*, for_tracks: bool = False) -> bool: return config.auto_sync_tracks if for_tracks else config.auto_export_on_save +def _list_track_items( + store: SqlAlchemyResearchStore, + *, + user_id: str, + track_id: int, + attr_name: str, +) -> list[dict[str, Any]]: + list_method = getattr(store, attr_name, None) + if not callable(list_method): + return [] + items = list_method(user_id=user_id, track_id=track_id, limit=100) + return list(items) if isinstance(items, list) else [] + + def export_track_snapshot( *, user_id: str, @@ -56,6 +70,19 @@ def export_track_snapshot( track_id=track_id, limit=max(1, int(config.export_limit)), ) + track_payload = dict(track) + track_payload["tasks"] = _list_track_items( + current_store, + user_id=user_id, + track_id=track_id, + attr_name="list_tasks", + ) + track_payload["milestones"] = _list_track_items( + current_store, + user_id=user_id, + track_id=track_id, + attr_name="list_milestones", + ) exporter = ObsidianFilesystemExporter() template_path = ( Path(config.paper_template_path).expanduser() @@ -65,9 +92,11 @@ def export_track_snapshot( result = exporter.export_library_snapshot( vault_path=vault_path, saved_items=saved_items, - track=track, + track=track_payload, root_dir=config.root_dir, paper_template_path=template_path, + track_moc_filename=getattr(config, "track_moc_filename", "_MOC.md"), + group_tracks_in_folders=getattr(config, "group_tracks_in_folders", True), ) Logger.info( f"Exported track {track_id} snapshot to Obsidian vault {vault_path}", diff --git a/src/paperbot/presentation/cli/main.py b/src/paperbot/presentation/cli/main.py index 313c8e2f..56e4b8f9 100644 --- a/src/paperbot/presentation/cli/main.py +++ b/src/paperbot/presentation/cli/main.py @@ -637,6 +637,8 @@ def _run_obsidian_export(parsed: argparse.Namespace) -> int: track=track, root_dir=root_dir, paper_template_path=template_path, + track_moc_filename=getattr(obsidian_config, "track_moc_filename", "_MOC.md"), + group_tracks_in_folders=getattr(obsidian_config, "group_tracks_in_folders", True), ) if parsed.json: diff --git a/tests/unit/test_citation_graph_client.py b/tests/unit/test_citation_graph_client.py new file mode 100644 index 00000000..5495627a --- /dev/null +++ b/tests/unit/test_citation_graph_client.py @@ -0,0 +1,172 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from paperbot.infrastructure.api_clients.citation_graph import CitationGraphClient + + +class _FakeSemanticScholarClient: + def __init__(self, payloads): + self.payloads = payloads + self.max_concurrency = 0 + self.current_concurrency = 0 + + async def get_paper(self, paper_id, fields=None): + self.current_concurrency += 1 + self.max_concurrency = max(self.max_concurrency, self.current_concurrency) + await asyncio.sleep(0) + try: + payload = self.payloads.get(paper_id) + return dict(payload) if isinstance(payload, dict) else payload + finally: + self.current_concurrency -= 1 + + async def close(self): + return None + + +@pytest.mark.asyncio +async def test_traverse_references_builds_directed_graph(): + client = _FakeSemanticScholarClient( + { + "seed": { + "paperId": "seed", + "title": "Seed", + "year": 2025, + "references": [ + {"citedPaper": {"paperId": "r1", "title": "Ref One", "year": 2024, "citationCount": 12}}, + {"citedPaper": {"paperId": "r2", "title": "Ref Two", "year": 2023, "citationCount": 5}}, + ], + "citations": [ + {"citingPaper": {"paperId": "c1", "title": "Ignored Citation", "year": 2026}}, + ], + } + } + ) + + graph_client = CitationGraphClient(semantic_scholar_client=client, max_concurrency=2) + graph = await graph_client.traverse("seed", direction="references", max_hops=1, max_papers_per_hop=10) + + assert set(graph.nodes) == {"seed", "r1", "r2"} + assert {(edge.source_id, edge.target_id, edge.relation) for edge in graph.edges} == { + ("seed", "r1", "references"), + ("seed", "r2", "references"), + } + + +@pytest.mark.asyncio +async def test_traverse_both_deduplicates_cycles_and_preserves_hops(): + client = _FakeSemanticScholarClient( + { + "seed": { + "paperId": "seed", + "title": "Seed", + "references": [{"citedPaper": {"paperId": "a", "title": "Paper A"}}], + "citations": [{"citingPaper": {"paperId": "b", "title": "Paper B"}}], + }, + "a": { + "paperId": "a", + "title": "Paper A", + "references": [{"citedPaper": {"paperId": "seed", "title": "Seed"}}], + "citations": [], + }, + "b": { + "paperId": "b", + "title": "Paper B", + "references": [], + "citations": [{"citingPaper": {"paperId": "seed", "title": "Seed"}}], + }, + } + ) + + graph_client = CitationGraphClient(semantic_scholar_client=client, max_concurrency=4) + graph = await graph_client.traverse("seed", direction="both", max_hops=2, max_papers_per_hop=10) + + assert graph.nodes["seed"].hop == 0 + assert graph.nodes["a"].hop == 1 + assert graph.nodes["b"].hop == 1 + assert len([edge for edge in graph.edges if edge.source_id == "seed" and edge.target_id == "a"]) == 1 + assert len([edge for edge in graph.edges if edge.source_id == "b" and edge.target_id == "seed"]) == 1 + + +@pytest.mark.asyncio +async def test_traverse_applies_relevance_filter_and_limit_per_hop(): + client = _FakeSemanticScholarClient( + { + "seed": { + "paperId": "seed", + "title": "Seed", + "references": [ + {"citedPaper": {"paperId": "keep", "title": "Keep Me", "citationCount": 1}}, + {"citedPaper": {"paperId": "drop", "title": "Drop Me", "citationCount": 99}}, + ], + "citations": [], + } + } + ) + + graph_client = CitationGraphClient(semantic_scholar_client=client) + graph = await graph_client.traverse( + "seed", + direction="references", + max_hops=1, + max_papers_per_hop=1, + relevance_filter=lambda paper: 10.0 if paper.get("title") == "Keep Me" else 0.5, + ) + + assert set(graph.nodes) == {"seed", "keep"} + assert [(edge.source_id, edge.target_id) for edge in graph.edges] == [("seed", "keep")] + + +@pytest.mark.asyncio +async def test_traverse_fetches_same_hop_in_parallel(): + client = _FakeSemanticScholarClient( + { + "seed": { + "paperId": "seed", + "title": "Seed", + "references": [ + {"citedPaper": {"paperId": "a", "title": "Paper A"}}, + {"citedPaper": {"paperId": "b", "title": "Paper B"}}, + ], + "citations": [], + }, + "a": { + "paperId": "a", + "title": "Paper A", + "references": [], + "citations": [], + }, + "b": { + "paperId": "b", + "title": "Paper B", + "references": [], + "citations": [], + }, + } + ) + + graph_client = CitationGraphClient(semantic_scholar_client=client, max_concurrency=4) + await graph_client.traverse("seed", direction="references", max_hops=2, max_papers_per_hop=10) + + assert client.max_concurrency >= 2 + + +@pytest.mark.asyncio +async def test_traverse_rejects_invalid_arguments(): + client = _FakeSemanticScholarClient({}) + graph_client = CitationGraphClient(semantic_scholar_client=client) + + with pytest.raises(ValueError, match="seed_paper_id"): + await graph_client.traverse("", direction="references") + + with pytest.raises(ValueError, match="direction"): + await graph_client.traverse("seed", direction="invalid") # type: ignore[arg-type] + + with pytest.raises(ValueError, match="max_hops"): + await graph_client.traverse("seed", direction="references", max_hops=0) + + with pytest.raises(ValueError, match="max_papers_per_hop"): + await graph_client.traverse("seed", direction="references", max_papers_per_hop=0) diff --git a/tests/unit/test_obsidian_cli.py b/tests/unit/test_obsidian_cli.py index 0679b0ff..b5838fb4 100644 --- a/tests/unit/test_obsidian_cli.py +++ b/tests/unit/test_obsidian_cli.py @@ -73,6 +73,8 @@ def export_library_snapshot( track=None, root_dir="PaperBot", paper_template_path=None, + track_moc_filename="_MOC.md", + group_tracks_in_folders=True, ): assert Path(vault_path) == Path("/tmp/my-vault") assert len(saved_items) == 1 @@ -80,12 +82,14 @@ def export_library_snapshot( assert track["name"] == "ICL Compression" assert root_dir == "PaperBot" assert paper_template_path is None + assert track_moc_filename == "_MOC.md" + assert group_tracks_in_folders is True return { "vault_path": str(vault_path), "root_dir": root_dir, "paper_count": 1, "paper_notes": ["/tmp/my-vault/PaperBot/Papers/2026-uniicl-2601-12345.md"], - "track_note": "/tmp/my-vault/PaperBot/Tracks/icl-compression.md", + "track_note": "/tmp/my-vault/PaperBot/Tracks/icl-compression/_MOC.md", "moc_note": "/tmp/my-vault/PaperBot/MOC.md", } @@ -141,7 +145,7 @@ def test_cli_obsidian_export_json_output(monkeypatch, capsys): assert exit_code == 0 payload = json.loads(captured.out) assert payload["paper_count"] == 1 - assert payload["track_note"].endswith("icl-compression.md") + assert payload["track_note"].endswith("icl-compression/_MOC.md") assert payload["moc_note"].endswith("MOC.md") @@ -161,6 +165,8 @@ def test_cli_obsidian_export_uses_settings_defaults(monkeypatch, capsys): vault_path="/tmp/my-vault", root_dir="PaperBot", paper_template_path=None, + track_moc_filename="_MOC.md", + group_tracks_in_folders=True, ) ), ) diff --git a/tests/unit/test_obsidian_exporter.py b/tests/unit/test_obsidian_exporter.py index 0b9ce4a7..966500ae 100644 --- a/tests/unit/test_obsidian_exporter.py +++ b/tests/unit/test_obsidian_exporter.py @@ -26,6 +26,20 @@ def test_export_library_snapshot_writes_paper_track_and_moc_notes(tmp_path: Path "keywords": ["ICL", "Compression"], "fields_of_study": ["Natural Language Processing"], "url": "https://example.com/uniicl", + "references": [ + { + "paperId": "prior", + "title": "Prior Compression Work", + "year": 2025, + } + ], + "citations": [ + { + "paperId": "future", + "title": "Future Compression Follow-up", + "year": 2027, + } + ], }, } ] @@ -38,6 +52,9 @@ def test_export_library_snapshot_writes_paper_track_and_moc_notes(tmp_path: Path "methods": ["retrieval"], "venues": ["ICLR"], "is_active": True, + "tasks": [{"title": "Benchmark prompt compression", "status": "doing"}], + "milestones": [{"name": "Submit workshop paper", "status": "todo"}], + "scholars": [{"name": "Alice Smith", "affiliation": "PaperBot Lab"}], } result = exporter.export_library_snapshot( @@ -57,18 +74,31 @@ def test_export_library_snapshot_writes_paper_track_and_moc_notes(tmp_path: Path paper_body = paper_note.read_text(encoding="utf-8") assert "paperbot_type: paper" in paper_body assert "# UniICL" in paper_body - assert "[[PaperBot/Tracks/icl-compression|ICL Compression]]" in paper_body + assert "[[PaperBot/Tracks/icl-compression/_MOC|ICL Compression]]" in paper_body assert "[DOI](https://doi.org/10.1000/uniicl)" in paper_body + assert "cites:" in paper_body + assert "cited_by:" in paper_body + assert "## References" in paper_body + assert "[[PaperBot/Papers/2025-prior-compression-work-prior|Prior Compression Work]]" in paper_body + assert "## Cited By" in paper_body + assert "[[PaperBot/Papers/2027-future-compression-follow-up-future|Future Compression Follow-up]]" in paper_body track_body = track_note.read_text(encoding="utf-8") assert "paperbot_type: track" in track_body assert "# ICL Compression" in track_body + assert "## Research Tasks" in track_body + assert "- [doing] Benchmark prompt compression" in track_body + assert "## Milestones" in track_body + assert "- [todo] Submit workshop paper" in track_body + assert "## Tracked Scholars" in track_body + assert "- Alice Smith (PaperBot Lab)" in track_body assert "[[PaperBot/Papers/2026-uniicl-2601-12345|UniICL]]" in track_body moc_body = moc_note.read_text(encoding="utf-8") assert "# PaperBot MOC" in moc_body - assert "[[PaperBot/Tracks/icl-compression|ICL Compression]]" in moc_body + assert "[[PaperBot/Tracks/icl-compression/_MOC|ICL Compression]]" in moc_body assert "[[PaperBot/Papers/2026-uniicl-2601-12345|UniICL]]" in moc_body + assert track_note == vault / "PaperBot" / "Tracks" / "icl-compression" / "_MOC.md" def test_export_library_snapshot_supports_custom_template_and_related_links(tmp_path: Path): @@ -126,10 +156,59 @@ def test_export_library_snapshot_supports_custom_template_and_related_links(tmp_ assert "Contains <script>alert(1)</script> & evidence." in paper_note assert "[[PaperBot/Papers/2025-prompt-compression-survey|Prompt Compression Survey]]" in paper_note assert "[[PaperBot/Papers/context-distillation-for-llms|Context Distillation for LLMs]]" in paper_note - assert "[[PaperBot/Tracks/icl-compression|ICL Compression]]" in paper_note + assert "[[PaperBot/Tracks/icl-compression/_MOC|ICL Compression]]" in paper_note assert paper_note.count("paperbot_type: paper") == 1 +def test_export_library_snapshot_backfills_existing_cited_by_links(tmp_path: Path): + exporter = ObsidianFilesystemExporter() + vault = tmp_path / "vault" + vault.mkdir() + + exporter.export_library_snapshot( + vault_path=vault, + saved_items=[ + { + "paper": { + "id": "prior", + "title": "Prior Compression Work", + "year": 2025, + } + } + ], + ) + + exporter.export_library_snapshot( + vault_path=vault, + saved_items=[ + { + "paper": { + "id": 1, + "title": "UniICL", + "year": 2026, + "references": [ + { + "paperId": "prior", + "title": "Prior Compression Work", + "year": 2025, + } + ], + } + } + ], + ) + + prior_note = ( + vault + / "PaperBot" + / "Papers" + / "2025-prior-compression-work-prior.md" + ).read_text(encoding="utf-8") + assert "cited_by:" in prior_note + assert "## Cited By" in prior_note + assert "[[PaperBot/Papers/2026-uniicl-1|UniICL]]" in prior_note + + def test_export_library_snapshot_requires_existing_vault_directory(tmp_path: Path): exporter = ObsidianFilesystemExporter() missing_vault = tmp_path / "missing-vault" diff --git a/tests/unit/test_obsidian_report_exporter.py b/tests/unit/test_obsidian_report_exporter.py new file mode 100644 index 00000000..b2dce512 --- /dev/null +++ b/tests/unit/test_obsidian_report_exporter.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from pathlib import Path + +from paperbot.infrastructure.exporters import ObsidianReportExporter + + +def test_export_report_note_writes_obsidian_longform_note(tmp_path: Path): + exporter = ObsidianReportExporter() + vault = tmp_path / "vault" + vault.mkdir() + + result = exporter.export_report_note( + vault_path=vault, + root_dir="PaperBot", + report={ + "title": "ICL Compression Landscape", + "track_name": "ICL Compression", + "workflow_type": "research", + "summary": "Compression methods are converging around retrieval-aware token pruning.", + "key_insight": "Retrieval-aware compression preserves accuracy at lower token budgets.", + "sections": [ + { + "title": "Key Findings", + "content": "Prompt compression is shifting from heuristics to trainable selectors.", + "cited_papers": [ + { + "title": "UniICL", + "year": 2026, + "authors": ["Alice Smith"], + "semantic_scholar_id": "S2-UNIICL", + "relevant_finding": "Introduces a unified compression benchmark.", + } + ], + } + ], + "methods": [ + { + "name": "Retrieval Compression", + "paper": "UniICL", + "pros": "High recall", + "cons": "Extra index latency", + } + ], + "trends": "Benchmarks are becoming more retrieval-centric.", + "future_directions": "Evaluate long-context compression under agent workflows.", + "references": [ + { + "title": "UniICL", + "year": 2026, + "authors": ["Alice Smith"], + "semantic_scholar_id": "S2-UNIICL", + } + ], + "tags": ["NLP", "Compression"], + }, + ) + + note_path = Path(result["note_path"]) + body = note_path.read_text(encoding="utf-8") + + assert note_path == vault / "PaperBot" / "Reports" / "icl-compression-landscape.md" + assert "type: research-report" in body + assert "[[PaperBot/Tracks/icl-compression/_MOC|ICL Compression]]" in body + assert "> [!abstract] 研究概述" in body + assert "> [!tip] 核心观点" in body + assert "> [!quote] [[PaperBot/Papers/2026-uniicl-s2-uniicl|UniICL]]" in body + assert "| 方法 | 论文 | 优势 | 局限 |" in body + assert "| Retrieval Compression | [[PaperBot/Papers/uniicl\\|UniICL]] | High recall | Extra index latency |" in body + assert "> [!info] 趋势分析" in body + assert "## 引用论文" in body + assert "- [[PaperBot/Papers/2026-uniicl-s2-uniicl|UniICL]] — Alice Smith, 2026" in body diff --git a/tests/unit/test_obsidian_report_routes.py b/tests/unit/test_obsidian_report_routes.py new file mode 100644 index 00000000..fee5d6c1 --- /dev/null +++ b/tests/unit/test_obsidian_report_routes.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +from fastapi.testclient import TestClient + +from paperbot.api.main import app +from paperbot.api.routes import obsidian as obsidian_route + + +def test_export_obsidian_report_endpoint_uses_settings_default_vault( + monkeypatch, + tmp_path: Path, +): + vault = tmp_path / "vault" + vault.mkdir() + + monkeypatch.setattr( + obsidian_route, + "create_settings", + lambda: SimpleNamespace( + obsidian=SimpleNamespace( + vault_path=str(vault), + root_dir="PaperBot", + track_moc_filename="_MOC.md", + group_tracks_in_folders=True, + ) + ), + ) + + with TestClient(app) as client: + response = client.post( + "/api/obsidian/export-report", + json={ + "title": "Graph Compression Report", + "track_name": "ICL Compression", + "summary": "A concise report for Obsidian export.", + "sections": [ + { + "title": "Findings", + "content": "Compression quality improves with retrieval-aware selection.", + "cited_papers": [ + { + "title": "UniICL", + "year": 2026, + "semantic_scholar_id": "S2-UNIICL", + "relevant_finding": "Defines a unified benchmark.", + } + ], + } + ], + "references": [ + { + "title": "UniICL", + "year": 2026, + "semantic_scholar_id": "S2-UNIICL", + } + ], + }, + ) + + assert response.status_code == 200 + payload = response.json() + note_path = Path(payload["note_path"]) + assert note_path.exists() + assert note_path == vault / "PaperBot" / "Reports" / "graph-compression-report.md" + + body = note_path.read_text(encoding="utf-8") + assert "# Graph Compression Report" in body + assert "> [!abstract] 研究概述" in body + assert "[[PaperBot/Papers/2026-uniicl-s2-uniicl|UniICL]]" in body + + +def test_export_obsidian_report_endpoint_requires_vault_path(monkeypatch): + monkeypatch.setattr( + obsidian_route, + "create_settings", + lambda: SimpleNamespace( + obsidian=SimpleNamespace( + vault_path="", + root_dir="PaperBot", + track_moc_filename="_MOC.md", + group_tracks_in_folders=True, + ) + ), + ) + + with TestClient(app) as client: + response = client.post( + "/api/obsidian/export-report", + json={"title": "Missing Vault"}, + ) + + assert response.status_code == 400 + assert "vault_path is required" in response.json()["detail"] diff --git a/tests/unit/test_obsidian_sync.py b/tests/unit/test_obsidian_sync.py index 4dd1e1d0..8e31f45f 100644 --- a/tests/unit/test_obsidian_sync.py +++ b/tests/unit/test_obsidian_sync.py @@ -21,6 +21,18 @@ def list_saved_papers(self, *, user_id: str, track_id: int, limit: int): assert limit == 25 return [{"paper": {"id": 1, "title": "UniICL"}}] + def list_tasks(self, *, user_id: str, track_id: int, limit: int): + assert user_id == "default" + assert track_id == 7 + assert limit == 100 + return [{"title": "Benchmark prompt compression", "status": "doing"}] + + def list_milestones(self, *, user_id: str, track_id: int, limit: int): + assert user_id == "default" + assert track_id == 7 + assert limit == 100 + return [{"name": "Submit workshop paper", "status": "todo"}] + def close(self) -> None: self.closed = True @@ -39,12 +51,16 @@ def export_library_snapshot( track, root_dir, paper_template_path=None, + track_moc_filename="_MOC.md", + group_tracks_in_folders=True, ): captured["vault_path"] = Path(vault_path) captured["saved_items"] = saved_items captured["track"] = track captured["root_dir"] = root_dir captured["template_path"] = paper_template_path + captured["track_moc_filename"] = track_moc_filename + captured["group_tracks_in_folders"] = group_tracks_in_folders return {"paper_count": len(saved_items)} monkeypatch.setattr( @@ -59,6 +75,8 @@ def export_library_snapshot( export_limit=25, auto_export_on_save=True, auto_sync_tracks=True, + track_moc_filename="_TRACK.md", + group_tracks_in_folders=False, ) ), ) @@ -70,9 +88,17 @@ def export_library_snapshot( assert result == {"paper_count": 1} assert captured["vault_path"] == vault_dir assert captured["root_dir"] == "PaperBot Notes" - assert captured["track"] == {"id": 7, "user_id": "default", "name": "ICL Compression"} + assert captured["track"] == { + "id": 7, + "user_id": "default", + "name": "ICL Compression", + "tasks": [{"title": "Benchmark prompt compression", "status": "doing"}], + "milestones": [{"name": "Submit workshop paper", "status": "todo"}], + } assert captured["saved_items"] == [{"paper": {"id": 1, "title": "UniICL"}}] assert captured["template_path"] == (tmp_path / "paper.md.j2") + assert captured["track_moc_filename"] == "_TRACK.md" + assert captured["group_tracks_in_folders"] is False def test_obsidian_auto_export_enabled_requires_vault_path(monkeypatch):