diff --git a/CHANGELOG.md b/CHANGELOG.md index 86dc8f127..a35797e01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## Unreleased + +### Changed +- Replace `html2text` (GPL-3.0) with `markdownify` (MIT) for HTML→Markdown + conversion in URL ingestion. Aligns the `pdf`/`all` extras with the + project's MIT license and removes a copyleft dependency that affected + anyone redistributing or embedding graphify. Fallback regex-strip path + unchanged. + Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) ## 0.4.23 (2026-04-18) diff --git a/graphify/ingest.py b/graphify/ingest.py index 62d8386b7..cf1420ffa 100644 --- a/graphify/ingest.py +++ b/graphify/ingest.py @@ -49,19 +49,21 @@ def _fetch_html(url: str) -> str: def _html_to_markdown(html: str, url: str) -> str: - """Convert HTML to clean markdown. Uses html2text if available, else basic strip.""" + """Convert HTML to clean markdown. Uses markdownify if available, else basic strip.""" + # Drop script/style content (markdownify's `strip` removes tags but keeps inner text). + cleaned = re.sub(r"]*>.*?", "", html, flags=re.DOTALL | re.IGNORECASE) + cleaned = re.sub(r"]*>.*?", "", cleaned, flags=re.DOTALL | re.IGNORECASE) try: - import html2text - h = html2text.HTML2Text() - h.ignore_links = False - h.ignore_images = True - h.body_width = 0 - return h.handle(html) + from markdownify import markdownify as _md + return _md( + cleaned, + heading_style="ATX", + bullets="-", + strip=["img"], + ).strip() except ImportError: - # Fallback: strip tags - text = re.sub(r"]*>.*?", "", html, flags=re.DOTALL | re.IGNORECASE) - text = re.sub(r"]*>.*?", "", text, flags=re.DOTALL | re.IGNORECASE) - text = re.sub(r"<[^>]+>", " ", text) + # Fallback: strip remaining tags + text = re.sub(r"<[^>]+>", " ", cleaned) text = re.sub(r"\s+", " ", text).strip() return text[:8000] diff --git a/graphify/skill-aider.md b/graphify/skill-aider.md index fa7007f3e..90de579f1 100644 --- a/graphify/skill-aider.md +++ b/graphify/skill-aider.md @@ -1118,7 +1118,7 @@ Supported URL types (auto-detected): - arXiv → abstract + metadata saved as `.md` - PDF → downloaded as `.pdf` - Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text +- Any webpage → converted to markdown via markdownify --- diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index 3d84acbd9..824962095 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -1118,7 +1118,7 @@ Supported URL types (auto-detected): - arXiv → abstract + metadata saved as `.md` - PDF → downloaded as `.pdf` - Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text +- Any webpage → converted to markdown via markdownify --- diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index b2e79e2b5..114225247 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -1177,7 +1177,7 @@ Supported URL types (auto-detected): - arXiv → abstract + metadata saved as `.md` - PDF → downloaded as `.pdf` - Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text +- Any webpage → converted to markdown via markdownify --- diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index 397669ae1..a28e221c5 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -1203,7 +1203,7 @@ Supported URL types (auto-detected): - arXiv → abstract + metadata saved as `.md` - PDF → downloaded as `.pdf` - Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text +- Any webpage → converted to markdown via markdownify --- diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index 6cde93503..73e515e09 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -1174,7 +1174,7 @@ Supported URL types (auto-detected): - arXiv → abstract + metadata saved as `.md` - PDF → downloaded as `.pdf` - Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text +- Any webpage → converted to markdown via markdownify --- diff --git a/graphify/skill-kiro.md b/graphify/skill-kiro.md index b3db4435d..1f1d3dd16 100644 --- a/graphify/skill-kiro.md +++ b/graphify/skill-kiro.md @@ -1117,7 +1117,7 @@ Supported URL types (auto-detected): - arXiv → abstract + metadata saved as `.md` - PDF → downloaded as `.pdf` - Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text +- Any webpage → converted to markdown via markdownify --- diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index 32819c80c..ec7985767 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -1227,7 +1227,7 @@ Supported URL types (auto-detected): - arXiv → abstract + metadata saved as `.md` - PDF → downloaded as `.pdf` - Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text +- Any webpage → converted to markdown via markdownify --- diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index 2b5b401e7..ae860166a 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -1141,7 +1141,7 @@ Supported URL types (auto-detected): - arXiv → abstract + metadata saved as `.md` - PDF → downloaded as `.pdf` - Images (.png/.jpg/.webp) → downloaded, vision extracts on next run -- Any webpage → converted to markdown via html2text +- Any webpage → converted to markdown via markdownify --- diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index 9984022e0..68b37357d 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -1167,7 +1167,7 @@ Supported URL types (auto-detected): - arXiv → abstract + metadata saved as `.md` - PDF → downloaded as `.pdf` - Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text +- Any webpage → converted to markdown via markdownify --- diff --git a/graphify/skill.md b/graphify/skill.md index be1e7dba0..d1509cfa1 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -1312,7 +1312,7 @@ Supported URL types (auto-detected): - arXiv → abstract + metadata saved as `.md` - PDF → downloaded as `.pdf` - Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text +- Any webpage → converted to markdown via markdownify --- diff --git a/pyproject.toml b/pyproject.toml index 2c39472d0..200cf5068 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,13 +44,13 @@ Issues = "https://github.com/safishamsi/graphify/issues" [project.optional-dependencies] mcp = ["mcp"] neo4j = ["neo4j"] -pdf = ["pypdf", "html2text"] +pdf = ["pypdf", "markdownify"] watch = ["watchdog"] svg = ["matplotlib"] leiden = ["graspologic; python_version < '3.13'"] office = ["python-docx", "openpyxl"] video = ["faster-whisper", "yt-dlp"] -all = ["mcp", "neo4j", "pypdf", "html2text", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper", "yt-dlp", "matplotlib"] +all = ["mcp", "neo4j", "pypdf", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper", "yt-dlp", "matplotlib"] [project.scripts] graphify = "graphify.__main__:main" diff --git a/tests/test_html_to_markdown.py b/tests/test_html_to_markdown.py new file mode 100644 index 000000000..00e28c90d --- /dev/null +++ b/tests/test_html_to_markdown.py @@ -0,0 +1,187 @@ +"""Tests for graphify.ingest._html_to_markdown and HTML→markdown ingestion path.""" +from __future__ import annotations + +import importlib.util +import sys + +import pytest + +from graphify.ingest import _html_to_markdown, _fetch_webpage + +# markdownify is an optional dependency (declared in the `pdf`/`all` extras). +# Tests that assert markdownify-specific output are gated behind this marker so +# the suite still runs cleanly on a base install (`pip install -e .`). +_HAS_MARKDOWNIFY = importlib.util.find_spec("markdownify") is not None +requires_markdownify = pytest.mark.skipif( + not _HAS_MARKDOWNIFY, + reason="markdownify not installed (optional 'pdf' extra)", +) + + +# --- Direct conversion tests --------------------------------------------------- + +def test_basic_paragraph(): + out = _html_to_markdown("

Hello world

", "http://example.com") + assert "Hello world" in out + assert "

" not in out + + +@requires_markdownify +def test_heading_atx_style(): + out = _html_to_markdown("

Title

", "http://example.com") + assert "# Title" in out + # Confirm we did NOT get setext (===) style + assert "===" not in out + + +@requires_markdownify +def test_links_preserved(): + html = '

See x for more.

' + out = _html_to_markdown(html, "http://example.com") + assert "[x](https://example.com/x)" in out + + +def test_images_dropped(): + html = '

before

cat

after

' + out = _html_to_markdown(html, "http://example.com") + assert "before" in out + assert "after" in out + # Image should not appear as markdown image syntax or as a raw tag + assert "![" not in out + assert ".x{color:red}" + "" + "

visible content

" + ) + out = _html_to_markdown(html, "http://example.com") + assert "visible content" in out + assert "alert" not in out + assert "color:red" not in out + + +@requires_markdownify +def test_bullet_list(): + html = "
  • alpha
  • beta
" + out = _html_to_markdown(html, "http://example.com") + assert "- alpha" in out + assert "- beta" in out + + +@requires_markdownify +def test_no_body_wrapping(): + long_text = "word " * 50 # ~250 chars on one line + html = f"

{long_text.strip()}

" + out = _html_to_markdown(html, "http://example.com") + # The single paragraph should survive as one logical line (no hard wrap at 80 cols). + # Find the line containing 'word' and assert it's not chopped. + longest = max((len(line) for line in out.splitlines() if "word" in line), default=0) + assert longest > 200, f"output appears wrapped: longest 'word' line = {longest}" + + +def test_empty_html(): + out = _html_to_markdown("", "http://example.com") + assert out.strip() == "" + + +def test_malformed_html_no_exception(): + # markdownify / bs4 should be lenient; our regex fallback must also be. + html = "

unclosed nested bold" + out = _html_to_markdown(html, "http://example.com") + assert "unclosed" in out + assert "nested" in out + assert "bold" in out + + +# --- Fallback path ------------------------------------------------------------- + +def test_fallback_when_markdownify_missing(monkeypatch): + """If markdownify cannot be imported, the regex-strip fallback must still + return readable plain text without raising.""" + # Force ImportError for the lazy `from markdownify import markdownify` + monkeypatch.setitem(sys.modules, "markdownify", None) + html = ( + "" + "" + "

Title

body text here

" + ) + out = _html_to_markdown(html, "http://example.com") + assert "Title" in out + assert "body text here" in out + # Stripped noise + assert "alert" not in out + assert "color:red" not in out + # Fallback returns plain text (no markdown headings) + assert "<" not in out + + +def test_fallback_basic_text_extraction(monkeypatch): + """Fallback path must extract readable text from a realistic HTML mix + (heading + list + paragraph), with all tags removed. Runs unconditionally + so base installs (`pip install -e .`) still get end-to-end coverage of + the regex-strip path.""" + monkeypatch.setitem(sys.modules, "markdownify", None) + html = "

T

  • a
  • b

p

" + out = _html_to_markdown(html, "http://example.com") + for token in ("T", "a", "b", "p"): + assert token in out, f"missing {token!r} in fallback output: {out!r}" + assert "<" not in out + + +# --- Integration with _fetch_webpage ------------------------------------------ + +@requires_markdownify +def test_fetch_webpage_uses_converter(monkeypatch): + """End-to-end smoke for the only caller of _html_to_markdown.""" + canned_html = ( + "Example Page" + "

Heading

Paragraph with " + 'a link.

' + ) + monkeypatch.setattr("graphify.ingest._fetch_html", lambda url: canned_html) + + content, filename = _fetch_webpage( + "https://example.com/page", + author=None, + contributor="tester", + ) + + # Frontmatter (string values may be quoted or unquoted depending on impl) + assert "https://example.com/page" in content + assert "type: webpage" in content + assert "Example Page" in content # title + assert "tester" in content # contributor + + # Converted markdown body + assert "# Heading" in content + assert "[a link](https://example.com/link)" in content + + # Filename safe + assert filename.endswith(".md") + assert "/" not in filename + + +# --- Regression guard ---------------------------------------------------------- + +def test_html2text_not_referenced_in_source(): + """Prevent accidental reintroduction of the GPL-3.0 dependency.""" + import pathlib + + repo_root = pathlib.Path(__file__).resolve().parents[1] + offenders = [] + # Only check shippable code + dependency manifest. Skip docs/skill files + # (which may reference html2text in historical context only) and tests. + targets = list((repo_root / "graphify").rglob("*.py")) + targets.append(repo_root / "pyproject.toml") + for path in targets: + text = path.read_text(encoding="utf-8", errors="ignore") + if "html2text" in text: + offenders.append(str(path.relative_to(repo_root))) + assert not offenders, ( + "html2text references found (GPL-3.0, must not return): " + + ", ".join(offenders) + )