From fd7e1c9bb3028261d6f63c2ae876b1ce23dc7166 Mon Sep 17 00:00:00 2001 From: jerry <1772030600@qq.com> Date: Wed, 4 Mar 2026 15:00:42 +0800 Subject: [PATCH 1/2] fix(mineru): support v4 async task extraction flow --- .../application/workflows/dailypaper.py | 11 +- .../extractors/mineru_client.py | 161 +++++++++++++++++- tests/unit/test_dailypaper.py | 7 + tests/unit/test_mineru_client.py | 20 +++ 4 files changed, 196 insertions(+), 3 deletions(-) diff --git a/src/paperbot/application/workflows/dailypaper.py b/src/paperbot/application/workflows/dailypaper.py index a784a6e7..0d415b75 100644 --- a/src/paperbot/application/workflows/dailypaper.py +++ b/src/paperbot/application/workflows/dailypaper.py @@ -16,6 +16,15 @@ SUPPORTED_LLM_FEATURES = ("summary", "trends", "insight", "relevance", "digest_card") +def _is_publishable_figure_url(url: str) -> bool: + u = (url or "").strip().lower() + if not u.startswith(("http://", "https://")): + return False + if u.endswith(".zip") or ".zip#" in u or ".zip?" in u: + return False + return True + + def extract_figures_for_report( report: Dict[str, Any], *, @@ -49,7 +58,7 @@ def extract_figures_for_report( item["figures"] = [ {"url": f.url, "caption": f.caption, "page": f.page} for f in figures[:5] ] - if main: + if main and _is_publishable_figure_url(main.url): item["main_figure"] = {"url": main.url, "caption": main.caption} count += 1 diff --git a/src/paperbot/infrastructure/extractors/mineru_client.py b/src/paperbot/infrastructure/extractors/mineru_client.py index 7b80bd9d..2fce924d 100644 --- a/src/paperbot/infrastructure/extractors/mineru_client.py +++ b/src/paperbot/infrastructure/extractors/mineru_client.py @@ -6,10 +6,12 @@ from __future__ import annotations import hashlib +import io import json import logging import re import time +import zipfile from dataclasses import dataclass from pathlib import Path from typing import Any, Dict, List, Optional @@ -20,6 +22,9 @@ _DEFAULT_BASE_URL = "https://mineru.net/api/v4" _DEFAULT_TIMEOUT = 60.0 +_DEFAULT_MODEL_VERSION = "pipeline" +_DEFAULT_POLL_INTERVAL_SECONDS = 2.0 +_DEFAULT_MAX_WAIT_SECONDS = 180.0 @dataclass @@ -52,12 +57,18 @@ def __init__( timeout: float = _DEFAULT_TIMEOUT, cache_dir: str = "", cache_ttl_seconds: int = 24 * 3600, + model_version: str = _DEFAULT_MODEL_VERSION, + poll_interval_seconds: float = _DEFAULT_POLL_INTERVAL_SECONDS, + max_wait_seconds: float = _DEFAULT_MAX_WAIT_SECONDS, ): self._api_key = api_key self._base_url = base_url.rstrip("/") self._timeout = timeout self._cache_dir = Path(cache_dir).expanduser() if cache_dir else None self._cache_ttl_seconds = max(0, int(cache_ttl_seconds)) + self._model_version = (model_version or _DEFAULT_MODEL_VERSION).strip() or _DEFAULT_MODEL_VERSION + self._poll_interval_seconds = max(0.5, float(poll_interval_seconds)) + self._max_wait_seconds = max(5.0, float(max_wait_seconds)) def extract_figures(self, pdf_url: str) -> List[Figure]: """Extract figures from a PDF URL via MinerU Cloud API. @@ -84,6 +95,72 @@ def extract_figures(self, pdf_url: str) -> List[Figure]: return [] def _call_extract(self, pdf_url: str) -> List[Figure]: + """Call MinerU API and return parsed figures. + + Try official async task flow first, then fall back to legacy `/extract` + for backward compatibility with self-hosted/older deployments. + """ + try: + return self._call_extract_task_flow(pdf_url) + except Exception as task_exc: + logger.info("MinerU task flow failed, trying legacy endpoint: %s", task_exc) + try: + return self._call_extract_legacy(pdf_url) + except Exception: + raise task_exc + + def _call_extract_task_flow(self, pdf_url: str) -> List[Figure]: + headers = { + "Authorization": f"Bearer {self._api_key}", + "Content-Type": "application/json", + } + + with httpx.Client(timeout=self._timeout) as client: + create_resp = client.post( + f"{self._base_url}/extract/task", + json={"url": pdf_url, "model_version": self._model_version}, + headers=headers, + ) + create_resp.raise_for_status() + create_data = create_resp.json() + task_id = str((create_data.get("data") or {}).get("task_id") or "").strip() + if create_data.get("code") != 0 or not task_id: + raise RuntimeError(f"invalid MinerU task response: {create_data}") + + deadline = time.time() + self._max_wait_seconds + detail: Dict[str, Any] = {} + + while time.time() <= deadline: + status_resp = client.get( + f"{self._base_url}/extract/task/{task_id}", + headers=headers, + ) + status_resp.raise_for_status() + status_data = status_resp.json() + detail = status_data.get("data") or {} + state = str(detail.get("state") or "").lower() + + if state == "done": + # Some deployments may return figure arrays directly. + parsed = self._parse_figures(detail) or self._parse_figures(status_data) + if parsed: + return parsed + + # Official v4 commonly returns a downloadable ZIP artifact. + zip_url = str(detail.get("full_zip_url") or "").strip() + if zip_url: + return self._extract_figures_from_zip(zip_url) + return [] + + if state == "failed": + err_msg = str(detail.get("err_msg") or "task failed") + raise RuntimeError(err_msg) + + time.sleep(self._poll_interval_seconds) + + raise TimeoutError(f"MinerU task timed out after {self._max_wait_seconds:.0f}s") + + def _call_extract_legacy(self, pdf_url: str) -> List[Figure]: headers = { "Authorization": f"Bearer {self._api_key}", "Content-Type": "application/json", @@ -101,6 +178,77 @@ def _call_extract(self, pdf_url: str) -> List[Figure]: return self._parse_figures(data) + def _extract_figures_from_zip(self, zip_url: str) -> List[Figure]: + """Download MinerU result zip and parse figures from generated markdown.""" + if not zip_url: + return [] + + with httpx.Client(timeout=self._timeout) as client: + resp = client.get(zip_url) + resp.raise_for_status() + zip_bytes = resp.content + + md_text = "" + with zipfile.ZipFile(io.BytesIO(zip_bytes), "r") as zf: + md_members = [name for name in zf.namelist() if name.lower().endswith(".md")] + if not md_members: + return [] + # Prefer the shortest markdown path, typically the top-level document. + target = sorted(md_members, key=len)[0] + md_text = zf.read(target).decode("utf-8", errors="ignore") + + return self._parse_figures_from_markdown(md_text, zip_url=zip_url) + + def _parse_figures_from_markdown(self, markdown_text: str, *, zip_url: str = "") -> List[Figure]: + """Parse figure references from markdown generated by MinerU. + + Format usually looks like: + ![](images/abc.jpg) + Figure 1: caption text + """ + if not markdown_text.strip(): + return [] + + lines = markdown_text.splitlines() + figures: List[Figure] = [] + image_pattern = re.compile(r"!\[[^\]]*\]\(([^)]+)\)") + caption_pattern = re.compile(r"^\s*(?:Figure|Fig\.?)[\s\d:.\-]+", re.IGNORECASE) + + for i, line in enumerate(lines): + match = image_pattern.search(line) + if not match: + continue + + raw_ref = (match.group(1) or "").strip() + if not raw_ref: + continue + + caption = "" + for j in (i + 1, i + 2): + if j >= len(lines): + break + candidate = (lines[j] or "").strip() + if candidate and caption_pattern.match(candidate): + caption = candidate + break + + if raw_ref.startswith(("http://", "https://")): + figure_url = raw_ref + elif zip_url: + figure_url = f"{zip_url}#/{raw_ref}" + else: + figure_url = raw_ref + + figures.append( + Figure( + url=figure_url, + caption=caption, + index=len(figures), + ) + ) + + return figures + def _cache_path(self, pdf_url: str) -> Optional[Path]: if self._cache_dir is None: return None @@ -211,8 +359,17 @@ def identify_main_figure(self, figures: List[Figure]) -> Optional[Figure]: caption_lower = fig.caption.lower() # Caption keyword bonus - main_keywords = ["overview", "architecture", "framework", "pipeline", - "main", "proposed", "system", "model", "approach"] + main_keywords = [ + "overview", + "architecture", + "framework", + "pipeline", + "main", + "proposed", + "system", + "model", + "approach", + ] for kw in main_keywords: if kw in caption_lower: score += 10.0 diff --git a/tests/unit/test_dailypaper.py b/tests/unit/test_dailypaper.py index 3bcb2603..d39e3d45 100644 --- a/tests/unit/test_dailypaper.py +++ b/tests/unit/test_dailypaper.py @@ -1,5 +1,6 @@ from paperbot.application.workflows.dailypaper import ( DailyPaperReporter, + _is_publishable_figure_url, apply_judge_scores_to_report, build_daily_paper_report, enrich_daily_paper_report, @@ -105,6 +106,12 @@ def test_normalize_llm_features_filters_unknown_items(): assert normalize_llm_features(["summary", "foo", "trends", "summary"]) == ["summary", "trends"] +def test_publishable_figure_url_rejects_zip_artifacts(): + assert _is_publishable_figure_url("https://cdn.example.com/fig1.png") + assert not _is_publishable_figure_url("https://cdn.example.com/result.zip") + assert not _is_publishable_figure_url("https://cdn.example.com/result.zip#/images/fig1.jpg") + + def test_apply_judge_scores_to_report(monkeypatch): report = build_daily_paper_report( search_result=_sample_search_result(), title="Judge Daily", top_n=5 diff --git a/tests/unit/test_mineru_client.py b/tests/unit/test_mineru_client.py index d7b83d3d..b2aeb058 100644 --- a/tests/unit/test_mineru_client.py +++ b/tests/unit/test_mineru_client.py @@ -67,6 +67,26 @@ def test_parse_figures_from_images_key(): assert figures[0].page == 2 +def test_parse_figures_from_markdown_zip_refs(): + client = MineruClient(api_key="test-key") + markdown = """ +![](images/fig1.jpg) +Figure 1: System overview + +![](https://cdn.mineru.net/fig2.png) +Fig. 2: Attention map +""".strip() + zip_url = "https://cdn-mineru.example.com/result.zip" + + figures = client._parse_figures_from_markdown(markdown, zip_url=zip_url) + + assert len(figures) == 2 + assert figures[0].url == f"{zip_url}#/images/fig1.jpg" + assert figures[0].caption == "Figure 1: System overview" + assert figures[1].url == "https://cdn.mineru.net/fig2.png" + assert figures[1].caption == "Fig. 2: Attention map" + + def test_identify_main_figure_prefers_figure_1(): figures = [ Figure(url="fig2.png", caption="Figure 2: Results table", page=4, width=400, height=300), From 70efd76c1dbcda034e78ca3835a38c603113e814 Mon Sep 17 00:00:00 2001 From: jerry <1772030600@qq.com> Date: Wed, 4 Mar 2026 15:21:53 +0800 Subject: [PATCH 2/2] refactor(mineru): standardize on v4 task API for daily push (#179) --- README.md | 6 +- env.example | 7 + .../application/workflows/dailypaper.py | 12 +- .../extractors/mineru_client.py | 270 ++++++++++-------- .../infrastructure/queue/arq_worker.py | 30 ++ src/paperbot/presentation/cli/main.py | 31 ++ tests/unit/test_arq_daily_papers.py | 6 + tests/unit/test_mineru_client.py | 42 ++- 8 files changed, 274 insertions(+), 130 deletions(-) diff --git a/README.md b/README.md index 13a4c5cf..5ac48159 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ | **文献卡片** | Structured Card（LLM 提取 method/dataset/conclusion/limitations），懒加载 + DB 缓存 | | **导出增强** | BibTeX/RIS/Markdown/CSL-JSON（Zotero 原生导入），Next.js proxy route 修复 | | **写作辅助** | Related Work 草稿生成（基于 saved papers + topic），[AuthorYear] 引用格式，一键复制 | -| **每日推送** | DailyPaper 生成后自动推送摘要到 Email/Slack/钉钉，支持 API 手动触发和 ARQ Cron 定时推送；MinerU 图表提取与 Apprise 多渠道（Telegram/Discord/企业微信/飞书/RSS）待集成 | +| **每日推送** | DailyPaper 生成后自动推送摘要到 Email/Slack/钉钉，支持 API 手动触发和 ARQ Cron 定时推送；已集成 MinerU 图表提取与 Apprise 多渠道（Telegram/Discord/企业微信/飞书/RSS） | | **Model Provider** | 多 LLM 提供商管理（OpenAI/Anthropic/OpenRouter/Ollama），API Key Keychain 安全存储，任务级路由，连接测试 | | **Deadline Radar** | 会议截止日期追踪，CCF 分级过滤，Research Track 关键词匹配 | | **论文发现** | 种子论文扩展（引用/被引/共作者），Discovery Graph 可视化，论文集合（Collections）管理 | @@ -35,7 +35,7 @@ | DailyPaper | ✅ 可用 | `/research/paperscool/daily` | `daily-paper` | 报告生成 + LLM 增强 + Judge + 保存，完整可用 | | LLM-as-Judge | ✅ 可用 | `/research/paperscool/analyze` | `--with-judge` | 5 维评分 + 多轮校准 + 推荐分级 + Token Budget，SSE 增量推送 | | Analyze SSE | ✅ 可用 | `/research/paperscool/analyze` | — | Judge / Trend / Insight 三通道 SSE 流式，前端逐卡片渲染 | -| Push/Notify | 🟡 基本可用 | `/research/paperscool/daily` | `--notify` | Email/Slack/钉钉 已落地；Apprise 多渠道（Telegram/Discord/企业微信/飞书/RSS）+ MinerU 图表提取待集成 | +| Push/Notify | ✅ 可用 | `/research/paperscool/daily` | `--notify` | Email/Slack/钉钉 + Apprise 多渠道（Telegram/Discord/企业微信/飞书/RSS）+ MinerU 图表提取已落地 | | 学者追踪 | 🟡 基本可用 | `/track` | `track` | 多 Agent 管线 + PIS 评分完整；依赖 Semantic Scholar API Key | | 深度评审 | 🟡 基本可用 | `/review` | `review` | 模拟同行评审流程完整；输出质量取决于 LLM 后端配置 | | Paper2Code | 🟡 基本可用 | `/gen-code`（兼容） + `/research/repro/context/*` | `gen-code` | 编排 + RAG + CodeMemory 完整；执行层计划迁移为 AgentSwarm/Codex 专业执行器 | @@ -463,7 +463,7 @@ DB 持久化（统一主数据模型 Paper/Scholar/Event/Run）、任务队列/ ### Phase 6 — 每日推送优化 -MinerU PDF 图表提取（主方法图自动识别）、推送内容增强（一句话总结 + 结构化摘要）、Apprise 多渠道统一推送层（Telegram/Discord/企业微信/飞书/RSS）、HuggingFace Daily Papers API 数据源接入。 +MinerU PDF 图表提取（主方法图自动识别）、推送内容增强（一句话总结 + 结构化摘要）、Apprise 多渠道统一推送层（Telegram/Discord/企业微信/飞书/RSS）、HuggingFace Daily Papers API 数据源接入已完成。 ## 文档索引 diff --git a/env.example b/env.example index 3b5f260f..30a568c0 100644 --- a/env.example +++ b/env.example @@ -28,6 +28,13 @@ OPENAI_BASE_URL= SEMANTIC_SCHOLAR_API_KEY= GITHUB_TOKEN= MINERU_API_KEY= +# MinerU Cloud API v4 async task endpoint +MINERU_API_BASE_URL=https://mineru.net/api/v4 +# MinerU model_version, e.g. vlm / pipeline +MINERU_MODEL_VERSION=vlm +# Max wait for task polling in seconds +MINERU_MAX_WAIT_SECONDS=180 +# MinerU Cloud limits: URL input only, <=200MB, <=600 pages, github/aws URL may timeout # CCS download (optional, ACM access URL) ACM_LIBRARY_URL= diff --git a/src/paperbot/application/workflows/dailypaper.py b/src/paperbot/application/workflows/dailypaper.py index 0d415b75..9816b086 100644 --- a/src/paperbot/application/workflows/dailypaper.py +++ b/src/paperbot/application/workflows/dailypaper.py @@ -30,6 +30,9 @@ def extract_figures_for_report( *, api_key: str = "", max_items: int = 5, + base_url: str = "", + model_version: str = "vlm", + max_wait_seconds: float = 180.0, ) -> Dict[str, Any]: """Optionally extract figures from top papers using MinerU Cloud API. @@ -41,7 +44,14 @@ def extract_figures_for_report( from paperbot.infrastructure.extractors.mineru_client import MineruClient - client = MineruClient(api_key=api_key) + client_kwargs: Dict[str, Any] = { + "api_key": api_key, + "model_version": model_version, + "max_wait_seconds": max_wait_seconds, + } + if (base_url or "").strip(): + client_kwargs["base_url"] = base_url.strip() + client = MineruClient(**client_kwargs) enriched = copy.deepcopy(report) count = 0 diff --git a/src/paperbot/infrastructure/extractors/mineru_client.py b/src/paperbot/infrastructure/extractors/mineru_client.py index 2fce924d..15f7aaec 100644 --- a/src/paperbot/infrastructure/extractors/mineru_client.py +++ b/src/paperbot/infrastructure/extractors/mineru_client.py @@ -1,8 +1,16 @@ """MinerU Cloud API client for PDF figure extraction. -Uses the MinerU Cloud API (HTTP) to extract figures from PDFs. -Falls back gracefully when the service is unavailable. +This client uses MinerU v4 async task API: +- POST /extract/task +- GET /extract/task/{task_id} + +API constraints (as documented by MinerU Cloud): +- Remote file URL only (no direct upload) +- File size <= 200MB +- Page count <= 600 pages +- Some overseas hosts (e.g. github/aws) may timeout from MinerU side """ + from __future__ import annotations import hashlib @@ -15,6 +23,7 @@ from dataclasses import dataclass from pathlib import Path from typing import Any, Dict, List, Optional +from urllib.parse import urlparse import httpx @@ -22,10 +31,16 @@ _DEFAULT_BASE_URL = "https://mineru.net/api/v4" _DEFAULT_TIMEOUT = 60.0 -_DEFAULT_MODEL_VERSION = "pipeline" +_DEFAULT_MODEL_VERSION = "vlm" _DEFAULT_POLL_INTERVAL_SECONDS = 2.0 _DEFAULT_MAX_WAIT_SECONDS = 180.0 +_UNSUPPORTED_HOST_HINTS = ( + "github.com", + "githubusercontent.com", + "amazonaws.com", +) + @dataclass class Figure: @@ -44,10 +59,7 @@ def area(self) -> int: class MineruClient: - """Client for MinerU Cloud API figure extraction. - - Falls back gracefully when API is unavailable or API key is not set. - """ + """Client for MinerU Cloud API figure extraction.""" def __init__( self, @@ -73,138 +85,149 @@ def __init__( def extract_figures(self, pdf_url: str) -> List[Figure]: """Extract figures from a PDF URL via MinerU Cloud API. - Returns an empty list if extraction fails or API is unavailable. + Returns an empty list if extraction fails or API key is not set. """ if not self._api_key: logger.debug("MinerU API key not set, skipping figure extraction") return [] - if not pdf_url or not pdf_url.strip(): + normalized_url = (pdf_url or "").strip() + if not normalized_url: return [] - cached = self._load_cached_figures(pdf_url) + cached = self._load_cached_figures(normalized_url) if cached is not None: return cached try: - figures = self._call_extract(pdf_url) - self._store_cached_figures(pdf_url, figures) + figures = self._call_extract(normalized_url) + self._store_cached_figures(normalized_url, figures) return figures except Exception as exc: logger.warning("MinerU figure extraction failed: %s", exc) return [] def _call_extract(self, pdf_url: str) -> List[Figure]: - """Call MinerU API and return parsed figures. + self._validate_source_url(pdf_url) - Try official async task flow first, then fall back to legacy `/extract` - for backward compatibility with self-hosted/older deployments. - """ - try: - return self._call_extract_task_flow(pdf_url) - except Exception as task_exc: - logger.info("MinerU task flow failed, trying legacy endpoint: %s", task_exc) - try: - return self._call_extract_legacy(pdf_url) - except Exception: - raise task_exc - - def _call_extract_task_flow(self, pdf_url: str) -> List[Figure]: headers = { "Authorization": f"Bearer {self._api_key}", "Content-Type": "application/json", } with httpx.Client(timeout=self._timeout) as client: - create_resp = client.post( - f"{self._base_url}/extract/task", - json={"url": pdf_url, "model_version": self._model_version}, - headers=headers, - ) - create_resp.raise_for_status() - create_data = create_resp.json() - task_id = str((create_data.get("data") or {}).get("task_id") or "").strip() - if create_data.get("code") != 0 or not task_id: - raise RuntimeError(f"invalid MinerU task response: {create_data}") - - deadline = time.time() + self._max_wait_seconds - detail: Dict[str, Any] = {} - - while time.time() <= deadline: - status_resp = client.get( - f"{self._base_url}/extract/task/{task_id}", - headers=headers, - ) - status_resp.raise_for_status() - status_data = status_resp.json() - detail = status_data.get("data") or {} - state = str(detail.get("state") or "").lower() - - if state == "done": - # Some deployments may return figure arrays directly. - parsed = self._parse_figures(detail) or self._parse_figures(status_data) - if parsed: - return parsed - - # Official v4 commonly returns a downloadable ZIP artifact. - zip_url = str(detail.get("full_zip_url") or "").strip() - if zip_url: - return self._extract_figures_from_zip(zip_url) - return [] + task_id = self._create_task(client, headers=headers, pdf_url=pdf_url) + detail = self._poll_until_done(client, headers=headers, task_id=task_id) - if state == "failed": - err_msg = str(detail.get("err_msg") or "task failed") - raise RuntimeError(err_msg) + # Some deployments may return figures directly in task payload. + parsed = self._parse_figures(detail) + if parsed: + return parsed - time.sleep(self._poll_interval_seconds) - - raise TimeoutError(f"MinerU task timed out after {self._max_wait_seconds:.0f}s") - - def _call_extract_legacy(self, pdf_url: str) -> List[Figure]: - headers = { - "Authorization": f"Bearer {self._api_key}", - "Content-Type": "application/json", - } - payload = {"url": pdf_url, "extract_figures": True} + zip_url = str(detail.get("full_zip_url") or "").strip() + if not zip_url: + return [] + return self._extract_figures_from_zip(client, zip_url) + + def _validate_source_url(self, pdf_url: str) -> None: + parsed = urlparse(pdf_url) + if parsed.scheme not in ("http", "https") or not parsed.netloc: + raise ValueError("MinerU source URL must be an absolute http(s) URL") + + host = (parsed.netloc or "").lower() + if any(host == hint or host.endswith(f".{hint}") for hint in _UNSUPPORTED_HOST_HINTS): + raise ValueError( + "MinerU Cloud may timeout on github/aws URLs; provide a publicly " + "accessible non-github/non-aws URL" + ) - with httpx.Client(timeout=self._timeout) as client: - resp = client.post( - f"{self._base_url}/extract", - json=payload, + def _create_task(self, client: httpx.Client, *, headers: Dict[str, str], pdf_url: str) -> str: + payload = {"url": pdf_url, "model_version": self._model_version} + body = self._request_json( + client, + method="POST", + url=f"{self._base_url}/extract/task", + headers=headers, + json_payload=payload, + ) + task_id = str((body.get("data") or {}).get("task_id") or "").strip() + if not task_id: + raise RuntimeError(f"invalid MinerU task response: {body}") + return task_id + + def _poll_until_done( + self, + client: httpx.Client, + *, + headers: Dict[str, str], + task_id: str, + ) -> Dict[str, Any]: + deadline = time.time() + self._max_wait_seconds + while time.time() <= deadline: + body = self._request_json( + client, + method="GET", + url=f"{self._base_url}/extract/task/{task_id}", headers=headers, ) - resp.raise_for_status() - data = resp.json() + detail = body.get("data") or {} + state = str(detail.get("state") or "").strip().lower() - return self._parse_figures(data) + if state == "done": + return detail + if state == "failed": + err_msg = str(detail.get("err_msg") or "task failed") + raise RuntimeError(err_msg) - def _extract_figures_from_zip(self, zip_url: str) -> List[Figure]: - """Download MinerU result zip and parse figures from generated markdown.""" - if not zip_url: - return [] + time.sleep(self._poll_interval_seconds) - with httpx.Client(timeout=self._timeout) as client: - resp = client.get(zip_url) - resp.raise_for_status() - zip_bytes = resp.content + raise TimeoutError(f"MinerU task timed out after {self._max_wait_seconds:.0f}s") + + def _request_json( + self, + client: httpx.Client, + *, + method: str, + url: str, + headers: Dict[str, str], + json_payload: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + response = client.request(method, url, headers=headers, json=json_payload) + response.raise_for_status() + + body = response.json() + if not isinstance(body, dict): + raise RuntimeError("invalid MinerU response payload") + + code = body.get("code") + if code is not None and int(code) != 0: + msg = str(body.get("msg") or "unknown error") + raise RuntimeError(f"MinerU API error code={code}: {msg}") + + return body + + def _extract_figures_from_zip(self, client: httpx.Client, zip_url: str) -> List[Figure]: + """Download MinerU result zip and parse figures from generated markdown.""" + response = client.get(zip_url) + response.raise_for_status() - md_text = "" - with zipfile.ZipFile(io.BytesIO(zip_bytes), "r") as zf: + with zipfile.ZipFile(io.BytesIO(response.content), "r") as zf: md_members = [name for name in zf.namelist() if name.lower().endswith(".md")] if not md_members: return [] - # Prefer the shortest markdown path, typically the top-level document. - target = sorted(md_members, key=len)[0] - md_text = zf.read(target).decode("utf-8", errors="ignore") - return self._parse_figures_from_markdown(md_text, zip_url=zip_url) + # Prefer the shortest markdown path, usually the top-level document markdown. + target_md = sorted(md_members, key=len)[0] + markdown_text = zf.read(target_md).decode("utf-8", errors="ignore") + + return self._parse_figures_from_markdown(markdown_text, zip_url=zip_url) def _parse_figures_from_markdown(self, markdown_text: str, *, zip_url: str = "") -> List[Figure]: - """Parse figure references from markdown generated by MinerU. + """Parse figures from MinerU markdown output. - Format usually looks like: - ![](images/abc.jpg) - Figure 1: caption text + Typical pattern: + ![](images/xxx.jpg) + Figure 1: ... """ if not markdown_text.strip(): return [] @@ -212,43 +235,41 @@ def _parse_figures_from_markdown(self, markdown_text: str, *, zip_url: str = "") lines = markdown_text.splitlines() figures: List[Figure] = [] image_pattern = re.compile(r"!\[[^\]]*\]\(([^)]+)\)") - caption_pattern = re.compile(r"^\s*(?:Figure|Fig\.?)[\s\d:.\-]+", re.IGNORECASE) + caption_pattern = re.compile(r"^\s*(?:Figure|Fig\.?)\s*\d+\s*[:.-]", re.IGNORECASE) - for i, line in enumerate(lines): + for idx, line in enumerate(lines): match = image_pattern.search(line) if not match: continue - raw_ref = (match.group(1) or "").strip() + raw_ref = str(match.group(1) or "").strip() if not raw_ref: continue caption = "" - for j in (i + 1, i + 2): - if j >= len(lines): + for offset in (1, 2, 3): + cursor = idx + offset + if cursor >= len(lines): break - candidate = (lines[j] or "").strip() - if candidate and caption_pattern.match(candidate): + candidate = (lines[cursor] or "").strip() + if not candidate: + continue + if caption_pattern.match(candidate): caption = candidate break - if raw_ref.startswith(("http://", "https://")): - figure_url = raw_ref - elif zip_url: - figure_url = f"{zip_url}#/{raw_ref}" - else: - figure_url = raw_ref - - figures.append( - Figure( - url=figure_url, - caption=caption, - index=len(figures), - ) - ) + figure_url = self._resolve_figure_url(raw_ref=raw_ref, zip_url=zip_url) + figures.append(Figure(url=figure_url, caption=caption, index=len(figures))) return figures + def _resolve_figure_url(self, *, raw_ref: str, zip_url: str) -> str: + if raw_ref.startswith(("http://", "https://")): + return raw_ref + if zip_url: + return f"{zip_url}#/{raw_ref}" + return raw_ref + def _cache_path(self, pdf_url: str) -> Optional[Path]: if self._cache_dir is None: return None @@ -259,13 +280,16 @@ def _load_cached_figures(self, pdf_url: str) -> Optional[List[Figure]]: cache_path = self._cache_path(pdf_url) if cache_path is None or not cache_path.exists(): return None + try: raw = json.loads(cache_path.read_text(encoding="utf-8")) if not isinstance(raw, dict): return None + saved_at = float(raw.get("saved_at") or 0) if self._cache_ttl_seconds > 0 and (time.time() - saved_at) > self._cache_ttl_seconds: return None + rows = raw.get("figures") or [] figures: List[Figure] = [] for row in rows: @@ -285,6 +309,7 @@ def _load_cached_figures(self, pdf_url: str) -> Optional[List[Figure]]: return figures except Exception: return None + return None def _store_cached_figures(self, pdf_url: str, figures: List[Figure]) -> None: @@ -352,13 +377,11 @@ def identify_main_figure(self, figures: List[Figure]) -> Optional[Figure]: if not candidates: candidates = figures - # Score each candidate scored: List[tuple[float, Figure]] = [] for fig in candidates: score = 0.0 caption_lower = fig.caption.lower() - # Caption keyword bonus main_keywords = [ "overview", "architecture", @@ -374,15 +397,12 @@ def identify_main_figure(self, figures: List[Figure]) -> Optional[Figure]: if kw in caption_lower: score += 10.0 - # Figure 1 / Fig. 1 bonus if re.search(r"(?:figure|fig\.?)\s*1\b", caption_lower): score += 15.0 - # Early page bonus (pages 1-3) if 1 <= fig.page <= 3: score += 5.0 - # Larger figures get slight bonus if fig.area > 0: score += min(fig.area / 100000, 5.0) diff --git a/src/paperbot/infrastructure/queue/arq_worker.py b/src/paperbot/infrastructure/queue/arq_worker.py index 2b40813b..5191561b 100644 --- a/src/paperbot/infrastructure/queue/arq_worker.py +++ b/src/paperbot/infrastructure/queue/arq_worker.py @@ -70,6 +70,16 @@ def _parse_bool_env(name: str, default: bool = False) -> bool: return raw.strip().lower() in ("1", "true", "yes", "y", "on") +def _parse_float_env(name: str, default: float) -> float: + raw = os.getenv(name) + if raw is None: + return float(default) + try: + return float(raw) + except (TypeError, ValueError): + return float(default) + + async def cron_track_subscriptions(ctx) -> Dict[str, Any]: """ Cron entrypoint: enqueue tracking jobs for all subscribed scholars. @@ -203,6 +213,9 @@ async def cron_daily_papers(ctx) -> Dict[str, Any]: judge_token_budget = int(os.getenv("PAPERBOT_DAILYPAPER_JUDGE_TOKEN_BUDGET", "0")) enable_figures = _parse_bool_env("PAPERBOT_DAILYPAPER_ENABLE_FIGURES", False) figures_max_items = int(os.getenv("PAPERBOT_DAILYPAPER_FIGURES_MAX_ITEMS", "5")) + mineru_api_base_url = os.getenv("MINERU_API_BASE_URL", "") + mineru_model_version = os.getenv("MINERU_MODEL_VERSION", "vlm") + mineru_max_wait_seconds = _parse_float_env("MINERU_MAX_WAIT_SECONDS", 180.0) job = await redis.enqueue_job( "daily_papers_job", @@ -222,6 +235,9 @@ async def cron_daily_papers(ctx) -> Dict[str, Any]: judge_token_budget=judge_token_budget, enable_figures=enable_figures, figures_max_items=max(1, int(figures_max_items)), + mineru_api_base_url=mineru_api_base_url, + mineru_model_version=mineru_model_version, + mineru_max_wait_seconds=max(5.0, float(mineru_max_wait_seconds)), notify=notify_enabled, notify_channels=notify_channels, save=True, @@ -234,6 +250,9 @@ async def cron_daily_papers(ctx) -> Dict[str, Any]: "branches": branches, "enable_figures": enable_figures, "figures_max_items": max(1, int(figures_max_items)), + "mineru_api_base_url": mineru_api_base_url, + "mineru_model_version": mineru_model_version, + "mineru_max_wait_seconds": max(5.0, float(mineru_max_wait_seconds)), } elog.append( make_event( @@ -346,6 +365,9 @@ async def daily_papers_job( save: bool = True, enable_figures: bool = False, figures_max_items: int = 5, + mineru_api_base_url: str = "", + mineru_model_version: str = "vlm", + mineru_max_wait_seconds: float = 180.0, ) -> Dict[str, Any]: """ARQ job: generate DailyPaper report and bridge highlights into feed events.""" run_id = new_run_id() @@ -381,6 +403,11 @@ async def daily_papers_job( "judge_token_budget": judge_token_budget, "notify": notify, "notify_channels": notify_channels or [], + "enable_figures": enable_figures, + "figures_max_items": max(1, int(figures_max_items)), + "mineru_api_base_url": mineru_api_base_url, + "mineru_model_version": mineru_model_version, + "mineru_max_wait_seconds": max(5.0, float(mineru_max_wait_seconds)), }, ) ) @@ -437,6 +464,9 @@ async def daily_papers_job( report, api_key=mineru_key, max_items=max(1, int(figures_max_items)), + base_url=mineru_api_base_url, + model_version=mineru_model_version, + max_wait_seconds=max(5.0, float(mineru_max_wait_seconds)), ) try: diff --git a/src/paperbot/presentation/cli/main.py b/src/paperbot/presentation/cli/main.py index b8512ceb..cfeac3e3 100644 --- a/src/paperbot/presentation/cli/main.py +++ b/src/paperbot/presentation/cli/main.py @@ -168,6 +168,22 @@ def create_parser() -> argparse.ArgumentParser: default=None, help="MinerU Cloud API Key（也可通过 MINERU_API_KEY 环境变量设置）", ) + daily_parser.add_argument( + "--mineru-api-base-url", + default=None, + help="MinerU API Base URL（默认 https://mineru.net/api/v4，也可通过 MINERU_API_BASE_URL 设置）", + ) + daily_parser.add_argument( + "--mineru-model-version", + default=None, + help="MinerU model_version（默认 vlm，也可通过 MINERU_MODEL_VERSION 设置）", + ) + daily_parser.add_argument( + "--mineru-max-wait-seconds", + type=float, + default=None, + help="MinerU 任务轮询最长等待秒数（默认 180，也可通过 MINERU_MAX_WAIT_SECONDS 设置）", + ) daily_parser.add_argument( "--figures-max-items", type=int, @@ -398,10 +414,25 @@ def _run_daily_paper(parsed: argparse.Namespace) -> int: if figures_enabled: mineru_key = parsed.mineru_api_key or os.getenv("MINERU_API_KEY", "") if mineru_key: + mineru_base_url = parsed.mineru_api_base_url or os.getenv("MINERU_API_BASE_URL", "") + mineru_model_version = parsed.mineru_model_version or os.getenv( + "MINERU_MODEL_VERSION", "vlm" + ) + try: + mineru_max_wait_seconds = ( + float(parsed.mineru_max_wait_seconds) + if parsed.mineru_max_wait_seconds is not None + else float(os.getenv("MINERU_MAX_WAIT_SECONDS", "180")) + ) + except (TypeError, ValueError): + mineru_max_wait_seconds = 180.0 report = extract_figures_for_report( report, api_key=mineru_key, max_items=max(1, int(parsed.figures_max_items)), + base_url=mineru_base_url, + model_version=mineru_model_version, + max_wait_seconds=max(5.0, mineru_max_wait_seconds), ) else: print("warning: --with-figures requires --mineru-api-key or MINERU_API_KEY env var", diff --git a/tests/unit/test_arq_daily_papers.py b/tests/unit/test_arq_daily_papers.py index 05af7d72..539e3caa 100644 --- a/tests/unit/test_arq_daily_papers.py +++ b/tests/unit/test_arq_daily_papers.py @@ -49,6 +49,9 @@ def append(self, event): monkeypatch.setattr(arq_worker, "_event_log", lambda: _NoopEventLog()) monkeypatch.setenv("PAPERBOT_DAILYPAPER_ENABLE_FIGURES", "true") monkeypatch.setenv("PAPERBOT_DAILYPAPER_FIGURES_MAX_ITEMS", "7") + monkeypatch.setenv("MINERU_API_BASE_URL", "https://mineru.net/api/v4") + monkeypatch.setenv("MINERU_MODEL_VERSION", "vlm") + monkeypatch.setenv("MINERU_MAX_WAIT_SECONDS", "120") redis = _FakeRedis() result = await arq_worker.cron_daily_papers({"redis": redis}) @@ -58,6 +61,9 @@ def append(self, event): assert redis.kwargs["name"] == "daily_papers_job" assert redis.kwargs["enable_figures"] is True assert redis.kwargs["figures_max_items"] == 7 + assert redis.kwargs["mineru_api_base_url"] == "https://mineru.net/api/v4" + assert redis.kwargs["mineru_model_version"] == "vlm" + assert redis.kwargs["mineru_max_wait_seconds"] == 120.0 @pytest.mark.asyncio diff --git a/tests/unit/test_mineru_client.py b/tests/unit/test_mineru_client.py index b2aeb058..2cfda29f 100644 --- a/tests/unit/test_mineru_client.py +++ b/tests/unit/test_mineru_client.py @@ -14,6 +14,40 @@ def test_empty_url_returns_empty(): assert result == [] +def test_validate_source_url_requires_http(): + client = MineruClient(api_key="test-key") + try: + client._validate_source_url("file:///tmp/a.pdf") + assert False, "expected ValueError for non-http source" + except ValueError as exc: + assert "http(s)" in str(exc) + + +def test_validate_source_url_rejects_github_and_aws(): + client = MineruClient(api_key="test-key") + for url in ( + "https://github.com/a/b/raw/main/paper.pdf", + "https://raw.githubusercontent.com/a/b/main/paper.pdf", + "https://bucket.s3.amazonaws.com/paper.pdf", + ): + try: + client._validate_source_url(url) + assert False, f"expected ValueError for URL: {url}" + except ValueError as exc: + assert "github/aws" in str(exc) + + +def test_extract_figures_rejects_unsupported_host_early(monkeypatch): + client = MineruClient(api_key="test-key") + + def _should_not_call(*args, **kwargs): + raise AssertionError("network call should not be reached for unsupported URL") + + monkeypatch.setattr(client, "_create_task", _should_not_call) + result = client.extract_figures("https://github.com/a/b/raw/main/paper.pdf") + assert result == [] + + def test_parse_figures_from_api_response(): client = MineruClient(api_key="test-key") data = { @@ -90,7 +124,13 @@ def test_parse_figures_from_markdown_zip_refs(): def test_identify_main_figure_prefers_figure_1(): figures = [ Figure(url="fig2.png", caption="Figure 2: Results table", page=4, width=400, height=300), - Figure(url="fig1.png", caption="Figure 1: System architecture overview", page=1, width=600, height=400), + Figure( + url="fig1.png", + caption="Figure 1: System architecture overview", + page=1, + width=600, + height=400, + ), Figure(url="fig3.png", caption="Figure 3: Comparison chart", page=6, width=500, height=350), ] client = MineruClient(api_key="test")