diff --git a/README.md b/README.md index 13a4c5cf..5ac48159 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ | **文献卡片** | Structured Card（LLM 提取 method/dataset/conclusion/limitations），懒加载 + DB 缓存 | | **导出增强** | BibTeX/RIS/Markdown/CSL-JSON（Zotero 原生导入），Next.js proxy route 修复 | | **写作辅助** | Related Work 草稿生成（基于 saved papers + topic），[AuthorYear] 引用格式，一键复制 | -| **每日推送** | DailyPaper 生成后自动推送摘要到 Email/Slack/钉钉，支持 API 手动触发和 ARQ Cron 定时推送；MinerU 图表提取与 Apprise 多渠道（Telegram/Discord/企业微信/飞书/RSS）待集成 | +| **每日推送** | DailyPaper 生成后自动推送摘要到 Email/Slack/钉钉，支持 API 手动触发和 ARQ Cron 定时推送；已集成 MinerU 图表提取与 Apprise 多渠道（Telegram/Discord/企业微信/飞书/RSS） | | **Model Provider** | 多 LLM 提供商管理（OpenAI/Anthropic/OpenRouter/Ollama），API Key Keychain 安全存储，任务级路由，连接测试 | | **Deadline Radar** | 会议截止日期追踪，CCF 分级过滤，Research Track 关键词匹配 | | **论文发现** | 种子论文扩展（引用/被引/共作者），Discovery Graph 可视化，论文集合（Collections）管理 | @@ -35,7 +35,7 @@ | DailyPaper | ✅ 可用 | `/research/paperscool/daily` | `daily-paper` | 报告生成 + LLM 增强 + Judge + 保存，完整可用 | | LLM-as-Judge | ✅ 可用 | `/research/paperscool/analyze` | `--with-judge` | 5 维评分 + 多轮校准 + 推荐分级 + Token Budget，SSE 增量推送 | | Analyze SSE | ✅ 可用 | `/research/paperscool/analyze` | — | Judge / Trend / Insight 三通道 SSE 流式，前端逐卡片渲染 | -| Push/Notify | 🟡 基本可用 | `/research/paperscool/daily` | `--notify` | Email/Slack/钉钉 已落地；Apprise 多渠道（Telegram/Discord/企业微信/飞书/RSS）+ MinerU 图表提取待集成 | +| Push/Notify | ✅ 可用 | `/research/paperscool/daily` | `--notify` | Email/Slack/钉钉 + Apprise 多渠道（Telegram/Discord/企业微信/飞书/RSS）+ MinerU 图表提取已落地 | | 学者追踪 | 🟡 基本可用 | `/track` | `track` | 多 Agent 管线 + PIS 评分完整；依赖 Semantic Scholar API Key | | 深度评审 | 🟡 基本可用 | `/review` | `review` | 模拟同行评审流程完整；输出质量取决于 LLM 后端配置 | | Paper2Code | 🟡 基本可用 | `/gen-code`（兼容） + `/research/repro/context/*` | `gen-code` | 编排 + RAG + CodeMemory 完整；执行层计划迁移为 AgentSwarm/Codex 专业执行器 | @@ -463,7 +463,7 @@ DB 持久化（统一主数据模型 Paper/Scholar/Event/Run）、任务队列/ ### Phase 6 — 每日推送优化 -MinerU PDF 图表提取（主方法图自动识别）、推送内容增强（一句话总结 + 结构化摘要）、Apprise 多渠道统一推送层（Telegram/Discord/企业微信/飞书/RSS）、HuggingFace Daily Papers API 数据源接入。 +MinerU PDF 图表提取（主方法图自动识别）、推送内容增强（一句话总结 + 结构化摘要）、Apprise 多渠道统一推送层（Telegram/Discord/企业微信/飞书/RSS）、HuggingFace Daily Papers API 数据源接入已完成。 ## 文档索引 diff --git a/env.example b/env.example index 3b5f260f..30a568c0 100644 --- a/env.example +++ b/env.example @@ -28,6 +28,13 @@ OPENAI_BASE_URL= SEMANTIC_SCHOLAR_API_KEY= GITHUB_TOKEN= MINERU_API_KEY= +# MinerU Cloud API v4 async task endpoint +MINERU_API_BASE_URL=https://mineru.net/api/v4 +# MinerU model_version, e.g. vlm / pipeline +MINERU_MODEL_VERSION=vlm +# Max wait for task polling in seconds +MINERU_MAX_WAIT_SECONDS=180 +# MinerU Cloud limits: URL input only, <=200MB, <=600 pages, github/aws URL may timeout # CCS download (optional, ACM access URL) ACM_LIBRARY_URL= diff --git a/src/paperbot/application/workflows/dailypaper.py b/src/paperbot/application/workflows/dailypaper.py index a784a6e7..9816b086 100644 --- a/src/paperbot/application/workflows/dailypaper.py +++ b/src/paperbot/application/workflows/dailypaper.py @@ -16,11 +16,23 @@ SUPPORTED_LLM_FEATURES = ("summary", "trends", "insight", "relevance", "digest_card") +def _is_publishable_figure_url(url: str) -> bool: + u = (url or "").strip().lower() + if not u.startswith(("http://", "https://")): + return False + if u.endswith(".zip") or ".zip#" in u or ".zip?" in u: + return False + return True + + def extract_figures_for_report( report: Dict[str, Any], *, api_key: str = "", max_items: int = 5, + base_url: str = "", + model_version: str = "vlm", + max_wait_seconds: float = 180.0, ) -> Dict[str, Any]: """Optionally extract figures from top papers using MinerU Cloud API. @@ -32,7 +44,14 @@ def extract_figures_for_report( from paperbot.infrastructure.extractors.mineru_client import MineruClient - client = MineruClient(api_key=api_key) + client_kwargs: Dict[str, Any] = { + "api_key": api_key, + "model_version": model_version, + "max_wait_seconds": max_wait_seconds, + } + if (base_url or "").strip(): + client_kwargs["base_url"] = base_url.strip() + client = MineruClient(**client_kwargs) enriched = copy.deepcopy(report) count = 0 @@ -49,7 +68,7 @@ def extract_figures_for_report( item["figures"] = [ {"url": f.url, "caption": f.caption, "page": f.page} for f in figures[:5] ] - if main: + if main and _is_publishable_figure_url(main.url): item["main_figure"] = {"url": main.url, "caption": main.caption} count += 1 diff --git a/src/paperbot/infrastructure/extractors/mineru_client.py b/src/paperbot/infrastructure/extractors/mineru_client.py index 7b80bd9d..15f7aaec 100644 --- a/src/paperbot/infrastructure/extractors/mineru_client.py +++ b/src/paperbot/infrastructure/extractors/mineru_client.py @@ -1,18 +1,29 @@ """MinerU Cloud API client for PDF figure extraction. -Uses the MinerU Cloud API (HTTP) to extract figures from PDFs. -Falls back gracefully when the service is unavailable. +This client uses MinerU v4 async task API: +- POST /extract/task +- GET /extract/task/{task_id} + +API constraints (as documented by MinerU Cloud): +- Remote file URL only (no direct upload) +- File size <= 200MB +- Page count <= 600 pages +- Some overseas hosts (e.g. github/aws) may timeout from MinerU side """ + from __future__ import annotations import hashlib +import io import json import logging import re import time +import zipfile from dataclasses import dataclass from pathlib import Path from typing import Any, Dict, List, Optional +from urllib.parse import urlparse import httpx @@ -20,6 +31,15 @@ _DEFAULT_BASE_URL = "https://mineru.net/api/v4" _DEFAULT_TIMEOUT = 60.0 +_DEFAULT_MODEL_VERSION = "vlm" +_DEFAULT_POLL_INTERVAL_SECONDS = 2.0 +_DEFAULT_MAX_WAIT_SECONDS = 180.0 + +_UNSUPPORTED_HOST_HINTS = ( + "github.com", + "githubusercontent.com", + "amazonaws.com", +) @dataclass @@ -39,10 +59,7 @@ def area(self) -> int: class MineruClient: - """Client for MinerU Cloud API figure extraction. - - Falls back gracefully when API is unavailable or API key is not set. - """ + """Client for MinerU Cloud API figure extraction.""" def __init__( self, @@ -52,54 +69,206 @@ def __init__( timeout: float = _DEFAULT_TIMEOUT, cache_dir: str = "", cache_ttl_seconds: int = 24 * 3600, + model_version: str = _DEFAULT_MODEL_VERSION, + poll_interval_seconds: float = _DEFAULT_POLL_INTERVAL_SECONDS, + max_wait_seconds: float = _DEFAULT_MAX_WAIT_SECONDS, ): self._api_key = api_key self._base_url = base_url.rstrip("/") self._timeout = timeout self._cache_dir = Path(cache_dir).expanduser() if cache_dir else None self._cache_ttl_seconds = max(0, int(cache_ttl_seconds)) + self._model_version = (model_version or _DEFAULT_MODEL_VERSION).strip() or _DEFAULT_MODEL_VERSION + self._poll_interval_seconds = max(0.5, float(poll_interval_seconds)) + self._max_wait_seconds = max(5.0, float(max_wait_seconds)) def extract_figures(self, pdf_url: str) -> List[Figure]: """Extract figures from a PDF URL via MinerU Cloud API. - Returns an empty list if extraction fails or API is unavailable. + Returns an empty list if extraction fails or API key is not set. """ if not self._api_key: logger.debug("MinerU API key not set, skipping figure extraction") return [] - if not pdf_url or not pdf_url.strip(): + normalized_url = (pdf_url or "").strip() + if not normalized_url: return [] - cached = self._load_cached_figures(pdf_url) + cached = self._load_cached_figures(normalized_url) if cached is not None: return cached try: - figures = self._call_extract(pdf_url) - self._store_cached_figures(pdf_url, figures) + figures = self._call_extract(normalized_url) + self._store_cached_figures(normalized_url, figures) return figures except Exception as exc: logger.warning("MinerU figure extraction failed: %s", exc) return [] def _call_extract(self, pdf_url: str) -> List[Figure]: + self._validate_source_url(pdf_url) + headers = { "Authorization": f"Bearer {self._api_key}", "Content-Type": "application/json", } - payload = {"url": pdf_url, "extract_figures": True} with httpx.Client(timeout=self._timeout) as client: - resp = client.post( - f"{self._base_url}/extract", - json=payload, + task_id = self._create_task(client, headers=headers, pdf_url=pdf_url) + detail = self._poll_until_done(client, headers=headers, task_id=task_id) + + # Some deployments may return figures directly in task payload. + parsed = self._parse_figures(detail) + if parsed: + return parsed + + zip_url = str(detail.get("full_zip_url") or "").strip() + if not zip_url: + return [] + return self._extract_figures_from_zip(client, zip_url) + + def _validate_source_url(self, pdf_url: str) -> None: + parsed = urlparse(pdf_url) + if parsed.scheme not in ("http", "https") or not parsed.netloc: + raise ValueError("MinerU source URL must be an absolute http(s) URL") + + host = (parsed.netloc or "").lower() + if any(host == hint or host.endswith(f".{hint}") for hint in _UNSUPPORTED_HOST_HINTS): + raise ValueError( + "MinerU Cloud may timeout on github/aws URLs; provide a publicly " + "accessible non-github/non-aws URL" + ) + + def _create_task(self, client: httpx.Client, *, headers: Dict[str, str], pdf_url: str) -> str: + payload = {"url": pdf_url, "model_version": self._model_version} + body = self._request_json( + client, + method="POST", + url=f"{self._base_url}/extract/task", + headers=headers, + json_payload=payload, + ) + task_id = str((body.get("data") or {}).get("task_id") or "").strip() + if not task_id: + raise RuntimeError(f"invalid MinerU task response: {body}") + return task_id + + def _poll_until_done( + self, + client: httpx.Client, + *, + headers: Dict[str, str], + task_id: str, + ) -> Dict[str, Any]: + deadline = time.time() + self._max_wait_seconds + while time.time() <= deadline: + body = self._request_json( + client, + method="GET", + url=f"{self._base_url}/extract/task/{task_id}", headers=headers, ) - resp.raise_for_status() - data = resp.json() + detail = body.get("data") or {} + state = str(detail.get("state") or "").strip().lower() - return self._parse_figures(data) + if state == "done": + return detail + if state == "failed": + err_msg = str(detail.get("err_msg") or "task failed") + raise RuntimeError(err_msg) + + time.sleep(self._poll_interval_seconds) + + raise TimeoutError(f"MinerU task timed out after {self._max_wait_seconds:.0f}s") + + def _request_json( + self, + client: httpx.Client, + *, + method: str, + url: str, + headers: Dict[str, str], + json_payload: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + response = client.request(method, url, headers=headers, json=json_payload) + response.raise_for_status() + + body = response.json() + if not isinstance(body, dict): + raise RuntimeError("invalid MinerU response payload") + + code = body.get("code") + if code is not None and int(code) != 0: + msg = str(body.get("msg") or "unknown error") + raise RuntimeError(f"MinerU API error code={code}: {msg}") + + return body + + def _extract_figures_from_zip(self, client: httpx.Client, zip_url: str) -> List[Figure]: + """Download MinerU result zip and parse figures from generated markdown.""" + response = client.get(zip_url) + response.raise_for_status() + + with zipfile.ZipFile(io.BytesIO(response.content), "r") as zf: + md_members = [name for name in zf.namelist() if name.lower().endswith(".md")] + if not md_members: + return [] + + # Prefer the shortest markdown path, usually the top-level document markdown. + target_md = sorted(md_members, key=len)[0] + markdown_text = zf.read(target_md).decode("utf-8", errors="ignore") + + return self._parse_figures_from_markdown(markdown_text, zip_url=zip_url) + + def _parse_figures_from_markdown(self, markdown_text: str, *, zip_url: str = "") -> List[Figure]: + """Parse figures from MinerU markdown output. + + Typical pattern: + ![](images/xxx.jpg) + Figure 1: ... + """ + if not markdown_text.strip(): + return [] + + lines = markdown_text.splitlines() + figures: List[Figure] = [] + image_pattern = re.compile(r"!\[[^\]]*\]\(([^)]+)\)") + caption_pattern = re.compile(r"^\s*(?:Figure|Fig\.?)\s*\d+\s*[:.-]", re.IGNORECASE) + + for idx, line in enumerate(lines): + match = image_pattern.search(line) + if not match: + continue + + raw_ref = str(match.group(1) or "").strip() + if not raw_ref: + continue + + caption = "" + for offset in (1, 2, 3): + cursor = idx + offset + if cursor >= len(lines): + break + candidate = (lines[cursor] or "").strip() + if not candidate: + continue + if caption_pattern.match(candidate): + caption = candidate + break + + figure_url = self._resolve_figure_url(raw_ref=raw_ref, zip_url=zip_url) + figures.append(Figure(url=figure_url, caption=caption, index=len(figures))) + + return figures + + def _resolve_figure_url(self, *, raw_ref: str, zip_url: str) -> str: + if raw_ref.startswith(("http://", "https://")): + return raw_ref + if zip_url: + return f"{zip_url}#/{raw_ref}" + return raw_ref def _cache_path(self, pdf_url: str) -> Optional[Path]: if self._cache_dir is None: @@ -111,13 +280,16 @@ def _load_cached_figures(self, pdf_url: str) -> Optional[List[Figure]]: cache_path = self._cache_path(pdf_url) if cache_path is None or not cache_path.exists(): return None + try: raw = json.loads(cache_path.read_text(encoding="utf-8")) if not isinstance(raw, dict): return None + saved_at = float(raw.get("saved_at") or 0) if self._cache_ttl_seconds > 0 and (time.time() - saved_at) > self._cache_ttl_seconds: return None + rows = raw.get("figures") or [] figures: List[Figure] = [] for row in rows: @@ -137,6 +309,7 @@ def _load_cached_figures(self, pdf_url: str) -> Optional[List[Figure]]: return figures except Exception: return None + return None def _store_cached_figures(self, pdf_url: str, figures: List[Figure]) -> None: @@ -204,28 +377,32 @@ def identify_main_figure(self, figures: List[Figure]) -> Optional[Figure]: if not candidates: candidates = figures - # Score each candidate scored: List[tuple[float, Figure]] = [] for fig in candidates: score = 0.0 caption_lower = fig.caption.lower() - # Caption keyword bonus - main_keywords = ["overview", "architecture", "framework", "pipeline", - "main", "proposed", "system", "model", "approach"] + main_keywords = [ + "overview", + "architecture", + "framework", + "pipeline", + "main", + "proposed", + "system", + "model", + "approach", + ] for kw in main_keywords: if kw in caption_lower: score += 10.0 - # Figure 1 / Fig. 1 bonus if re.search(r"(?:figure|fig\.?)\s*1\b", caption_lower): score += 15.0 - # Early page bonus (pages 1-3) if 1 <= fig.page <= 3: score += 5.0 - # Larger figures get slight bonus if fig.area > 0: score += min(fig.area / 100000, 5.0) diff --git a/src/paperbot/infrastructure/queue/arq_worker.py b/src/paperbot/infrastructure/queue/arq_worker.py index 2b40813b..5191561b 100644 --- a/src/paperbot/infrastructure/queue/arq_worker.py +++ b/src/paperbot/infrastructure/queue/arq_worker.py @@ -70,6 +70,16 @@ def _parse_bool_env(name: str, default: bool = False) -> bool: return raw.strip().lower() in ("1", "true", "yes", "y", "on") +def _parse_float_env(name: str, default: float) -> float: + raw = os.getenv(name) + if raw is None: + return float(default) + try: + return float(raw) + except (TypeError, ValueError): + return float(default) + + async def cron_track_subscriptions(ctx) -> Dict[str, Any]: """ Cron entrypoint: enqueue tracking jobs for all subscribed scholars. @@ -203,6 +213,9 @@ async def cron_daily_papers(ctx) -> Dict[str, Any]: judge_token_budget = int(os.getenv("PAPERBOT_DAILYPAPER_JUDGE_TOKEN_BUDGET", "0")) enable_figures = _parse_bool_env("PAPERBOT_DAILYPAPER_ENABLE_FIGURES", False) figures_max_items = int(os.getenv("PAPERBOT_DAILYPAPER_FIGURES_MAX_ITEMS", "5")) + mineru_api_base_url = os.getenv("MINERU_API_BASE_URL", "") + mineru_model_version = os.getenv("MINERU_MODEL_VERSION", "vlm") + mineru_max_wait_seconds = _parse_float_env("MINERU_MAX_WAIT_SECONDS", 180.0) job = await redis.enqueue_job( "daily_papers_job", @@ -222,6 +235,9 @@ async def cron_daily_papers(ctx) -> Dict[str, Any]: judge_token_budget=judge_token_budget, enable_figures=enable_figures, figures_max_items=max(1, int(figures_max_items)), + mineru_api_base_url=mineru_api_base_url, + mineru_model_version=mineru_model_version, + mineru_max_wait_seconds=max(5.0, float(mineru_max_wait_seconds)), notify=notify_enabled, notify_channels=notify_channels, save=True, @@ -234,6 +250,9 @@ async def cron_daily_papers(ctx) -> Dict[str, Any]: "branches": branches, "enable_figures": enable_figures, "figures_max_items": max(1, int(figures_max_items)), + "mineru_api_base_url": mineru_api_base_url, + "mineru_model_version": mineru_model_version, + "mineru_max_wait_seconds": max(5.0, float(mineru_max_wait_seconds)), } elog.append( make_event( @@ -346,6 +365,9 @@ async def daily_papers_job( save: bool = True, enable_figures: bool = False, figures_max_items: int = 5, + mineru_api_base_url: str = "", + mineru_model_version: str = "vlm", + mineru_max_wait_seconds: float = 180.0, ) -> Dict[str, Any]: """ARQ job: generate DailyPaper report and bridge highlights into feed events.""" run_id = new_run_id() @@ -381,6 +403,11 @@ async def daily_papers_job( "judge_token_budget": judge_token_budget, "notify": notify, "notify_channels": notify_channels or [], + "enable_figures": enable_figures, + "figures_max_items": max(1, int(figures_max_items)), + "mineru_api_base_url": mineru_api_base_url, + "mineru_model_version": mineru_model_version, + "mineru_max_wait_seconds": max(5.0, float(mineru_max_wait_seconds)), }, ) ) @@ -437,6 +464,9 @@ async def daily_papers_job( report, api_key=mineru_key, max_items=max(1, int(figures_max_items)), + base_url=mineru_api_base_url, + model_version=mineru_model_version, + max_wait_seconds=max(5.0, float(mineru_max_wait_seconds)), ) try: diff --git a/src/paperbot/presentation/cli/main.py b/src/paperbot/presentation/cli/main.py index b8512ceb..cfeac3e3 100644 --- a/src/paperbot/presentation/cli/main.py +++ b/src/paperbot/presentation/cli/main.py @@ -168,6 +168,22 @@ def create_parser() -> argparse.ArgumentParser: default=None, help="MinerU Cloud API Key（也可通过 MINERU_API_KEY 环境变量设置）", ) + daily_parser.add_argument( + "--mineru-api-base-url", + default=None, + help="MinerU API Base URL（默认 https://mineru.net/api/v4，也可通过 MINERU_API_BASE_URL 设置）", + ) + daily_parser.add_argument( + "--mineru-model-version", + default=None, + help="MinerU model_version（默认 vlm，也可通过 MINERU_MODEL_VERSION 设置）", + ) + daily_parser.add_argument( + "--mineru-max-wait-seconds", + type=float, + default=None, + help="MinerU 任务轮询最长等待秒数（默认 180，也可通过 MINERU_MAX_WAIT_SECONDS 设置）", + ) daily_parser.add_argument( "--figures-max-items", type=int, @@ -398,10 +414,25 @@ def _run_daily_paper(parsed: argparse.Namespace) -> int: if figures_enabled: mineru_key = parsed.mineru_api_key or os.getenv("MINERU_API_KEY", "") if mineru_key: + mineru_base_url = parsed.mineru_api_base_url or os.getenv("MINERU_API_BASE_URL", "") + mineru_model_version = parsed.mineru_model_version or os.getenv( + "MINERU_MODEL_VERSION", "vlm" + ) + try: + mineru_max_wait_seconds = ( + float(parsed.mineru_max_wait_seconds) + if parsed.mineru_max_wait_seconds is not None + else float(os.getenv("MINERU_MAX_WAIT_SECONDS", "180")) + ) + except (TypeError, ValueError): + mineru_max_wait_seconds = 180.0 report = extract_figures_for_report( report, api_key=mineru_key, max_items=max(1, int(parsed.figures_max_items)), + base_url=mineru_base_url, + model_version=mineru_model_version, + max_wait_seconds=max(5.0, mineru_max_wait_seconds), ) else: print("warning: --with-figures requires --mineru-api-key or MINERU_API_KEY env var", diff --git a/tests/unit/test_arq_daily_papers.py b/tests/unit/test_arq_daily_papers.py index 05af7d72..539e3caa 100644 --- a/tests/unit/test_arq_daily_papers.py +++ b/tests/unit/test_arq_daily_papers.py @@ -49,6 +49,9 @@ def append(self, event): monkeypatch.setattr(arq_worker, "_event_log", lambda: _NoopEventLog()) monkeypatch.setenv("PAPERBOT_DAILYPAPER_ENABLE_FIGURES", "true") monkeypatch.setenv("PAPERBOT_DAILYPAPER_FIGURES_MAX_ITEMS", "7") + monkeypatch.setenv("MINERU_API_BASE_URL", "https://mineru.net/api/v4") + monkeypatch.setenv("MINERU_MODEL_VERSION", "vlm") + monkeypatch.setenv("MINERU_MAX_WAIT_SECONDS", "120") redis = _FakeRedis() result = await arq_worker.cron_daily_papers({"redis": redis}) @@ -58,6 +61,9 @@ def append(self, event): assert redis.kwargs["name"] == "daily_papers_job" assert redis.kwargs["enable_figures"] is True assert redis.kwargs["figures_max_items"] == 7 + assert redis.kwargs["mineru_api_base_url"] == "https://mineru.net/api/v4" + assert redis.kwargs["mineru_model_version"] == "vlm" + assert redis.kwargs["mineru_max_wait_seconds"] == 120.0 @pytest.mark.asyncio diff --git a/tests/unit/test_dailypaper.py b/tests/unit/test_dailypaper.py index 3bcb2603..d39e3d45 100644 --- a/tests/unit/test_dailypaper.py +++ b/tests/unit/test_dailypaper.py @@ -1,5 +1,6 @@ from paperbot.application.workflows.dailypaper import ( DailyPaperReporter, + _is_publishable_figure_url, apply_judge_scores_to_report, build_daily_paper_report, enrich_daily_paper_report, @@ -105,6 +106,12 @@ def test_normalize_llm_features_filters_unknown_items(): assert normalize_llm_features(["summary", "foo", "trends", "summary"]) == ["summary", "trends"] +def test_publishable_figure_url_rejects_zip_artifacts(): + assert _is_publishable_figure_url("https://cdn.example.com/fig1.png") + assert not _is_publishable_figure_url("https://cdn.example.com/result.zip") + assert not _is_publishable_figure_url("https://cdn.example.com/result.zip#/images/fig1.jpg") + + def test_apply_judge_scores_to_report(monkeypatch): report = build_daily_paper_report( search_result=_sample_search_result(), title="Judge Daily", top_n=5 diff --git a/tests/unit/test_mineru_client.py b/tests/unit/test_mineru_client.py index d7b83d3d..2cfda29f 100644 --- a/tests/unit/test_mineru_client.py +++ b/tests/unit/test_mineru_client.py @@ -14,6 +14,40 @@ def test_empty_url_returns_empty(): assert result == [] +def test_validate_source_url_requires_http(): + client = MineruClient(api_key="test-key") + try: + client._validate_source_url("file:///tmp/a.pdf") + assert False, "expected ValueError for non-http source" + except ValueError as exc: + assert "http(s)" in str(exc) + + +def test_validate_source_url_rejects_github_and_aws(): + client = MineruClient(api_key="test-key") + for url in ( + "https://github.com/a/b/raw/main/paper.pdf", + "https://raw.githubusercontent.com/a/b/main/paper.pdf", + "https://bucket.s3.amazonaws.com/paper.pdf", + ): + try: + client._validate_source_url(url) + assert False, f"expected ValueError for URL: {url}" + except ValueError as exc: + assert "github/aws" in str(exc) + + +def test_extract_figures_rejects_unsupported_host_early(monkeypatch): + client = MineruClient(api_key="test-key") + + def _should_not_call(*args, **kwargs): + raise AssertionError("network call should not be reached for unsupported URL") + + monkeypatch.setattr(client, "_create_task", _should_not_call) + result = client.extract_figures("https://github.com/a/b/raw/main/paper.pdf") + assert result == [] + + def test_parse_figures_from_api_response(): client = MineruClient(api_key="test-key") data = { @@ -67,10 +101,36 @@ def test_parse_figures_from_images_key(): assert figures[0].page == 2 +def test_parse_figures_from_markdown_zip_refs(): + client = MineruClient(api_key="test-key") + markdown = """ +![](images/fig1.jpg) +Figure 1: System overview + +![](https://cdn.mineru.net/fig2.png) +Fig. 2: Attention map +""".strip() + zip_url = "https://cdn-mineru.example.com/result.zip" + + figures = client._parse_figures_from_markdown(markdown, zip_url=zip_url) + + assert len(figures) == 2 + assert figures[0].url == f"{zip_url}#/images/fig1.jpg" + assert figures[0].caption == "Figure 1: System overview" + assert figures[1].url == "https://cdn.mineru.net/fig2.png" + assert figures[1].caption == "Fig. 2: Attention map" + + def test_identify_main_figure_prefers_figure_1(): figures = [ Figure(url="fig2.png", caption="Figure 2: Results table", page=4, width=400, height=300), - Figure(url="fig1.png", caption="Figure 1: System architecture overview", page=1, width=600, height=400), + Figure( + url="fig1.png", + caption="Figure 1: System architecture overview", + page=1, + width=600, + height=400, + ), Figure(url="fig3.png", caption="Figure 3: Comparison chart", page=6, width=500, height=350), ] client = MineruClient(api_key="test")