From fc74fb11e5961c62f66664fd38986c97f921fa25 Mon Sep 17 00:00:00 2001 From: mrummuka Date: Tue, 28 Apr 2026 12:17:57 +0300 Subject: [PATCH 1/4] =?UTF-8?q?test:=20add=20coverage=20for=20=5Fhtml=5Fto?= =?UTF-8?q?=5Fmarkdown=20HTML=E2=86=92Markdown=20conversion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds 12 tests covering paragraphs, headings (ATX), links, image removal, script/style stripping, bullet lists, line wrapping, empty input, malformed HTML, the regex-strip fallback path, an end-to-end _fetch_webpage smoke test, and a regression guard ensuring the GPL-3.0 'html2text' dependency does not creep back into shipped code or pyproject.toml. Three tests intentionally fail against the current html2text-based implementation; they will turn green when html2text is replaced with markdownify (MIT) in the following commit. --- tests/test_html_to_markdown.py | 159 +++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 tests/test_html_to_markdown.py diff --git a/tests/test_html_to_markdown.py b/tests/test_html_to_markdown.py new file mode 100644 index 000000000..d605bbc61 --- /dev/null +++ b/tests/test_html_to_markdown.py @@ -0,0 +1,159 @@ +"""Tests for graphify.ingest._html_to_markdown and HTML→markdown ingestion path.""" +from __future__ import annotations + +import sys + +import pytest + +from graphify.ingest import _html_to_markdown, _fetch_webpage + + +# --- Direct conversion tests --------------------------------------------------- + +def test_basic_paragraph(): + out = _html_to_markdown("

Hello world

", "http://example.com") + assert "Hello world" in out + assert "

" not in out + + +def test_heading_atx_style(): + out = _html_to_markdown("

Title

", "http://example.com") + assert "# Title" in out + # Confirm we did NOT get setext (===) style + assert "===" not in out + + +def test_links_preserved(): + html = '

See x for more.

' + out = _html_to_markdown(html, "http://example.com") + assert "[x](https://example.com/x)" in out + + +def test_images_dropped(): + html = '

before

cat

after

' + out = _html_to_markdown(html, "http://example.com") + assert "before" in out + assert "after" in out + # Image should not appear as markdown image syntax or as a raw tag + assert "![" not in out + assert ".x{color:red}" + "" + "

visible content

" + ) + out = _html_to_markdown(html, "http://example.com") + assert "visible content" in out + assert "alert" not in out + assert "color:red" not in out + + +def test_bullet_list(): + html = "" + out = _html_to_markdown(html, "http://example.com") + assert "- alpha" in out + assert "- beta" in out + + +def test_no_body_wrapping(): + long_text = "word " * 50 # ~250 chars on one line + html = f"

{long_text.strip()}

" + out = _html_to_markdown(html, "http://example.com") + # The single paragraph should survive as one logical line (no hard wrap at 80 cols). + # Find the line containing 'word' and assert it's not chopped. + longest = max((len(line) for line in out.splitlines() if "word" in line), default=0) + assert longest > 200, f"output appears wrapped: longest 'word' line = {longest}" + + +def test_empty_html(): + out = _html_to_markdown("", "http://example.com") + assert out.strip() == "" + + +def test_malformed_html_no_exception(): + # markdownify / bs4 should be lenient; our regex fallback must also be. + html = "

unclosed nested bold" + out = _html_to_markdown(html, "http://example.com") + assert "unclosed" in out + assert "nested" in out + assert "bold" in out + + +# --- Fallback path ------------------------------------------------------------- + +def test_fallback_when_markdownify_missing(monkeypatch): + """If markdownify cannot be imported, the regex-strip fallback must still + return readable plain text without raising.""" + # Force ImportError for the lazy `from markdownify import markdownify` + monkeypatch.setitem(sys.modules, "markdownify", None) + html = ( + "" + "" + "

Title

body text here

" + ) + out = _html_to_markdown(html, "http://example.com") + assert "Title" in out + assert "body text here" in out + # Stripped noise + assert "alert" not in out + assert "color:red" not in out + # Fallback returns plain text (no markdown headings) + assert "<" not in out + + +# --- Integration with _fetch_webpage ------------------------------------------ + +def test_fetch_webpage_uses_converter(monkeypatch): + """End-to-end smoke for the only caller of _html_to_markdown.""" + canned_html = ( + "Example Page" + "

Heading

Paragraph with " + 'a link.

' + ) + monkeypatch.setattr("graphify.ingest._fetch_html", lambda url: canned_html) + + content, filename = _fetch_webpage( + "https://example.com/page", + author=None, + contributor="tester", + ) + + # Frontmatter (string values may be quoted or unquoted depending on impl) + assert "https://example.com/page" in content + assert "type: webpage" in content + assert "Example Page" in content # title + assert "tester" in content # contributor + + # Converted markdown body + assert "# Heading" in content + assert "[a link](https://example.com/link)" in content + + # Filename safe + assert filename.endswith(".md") + assert "/" not in filename + + +# --- Regression guard ---------------------------------------------------------- + +def test_html2text_not_referenced_in_source(): + """Prevent accidental reintroduction of the GPL-3.0 dependency.""" + import pathlib + + repo_root = pathlib.Path(__file__).resolve().parents[1] + offenders = [] + # Only check shippable code + dependency manifest. Skip docs/skill files + # (which may reference html2text in historical context only) and tests. + targets = list((repo_root / "graphify").rglob("*.py")) + targets.append(repo_root / "pyproject.toml") + for path in targets: + text = path.read_text(encoding="utf-8", errors="ignore") + if "html2text" in text: + offenders.append(str(path.relative_to(repo_root))) + assert not offenders, ( + "html2text references found (GPL-3.0, must not return): " + + ", ".join(offenders) + ) From badb2e957e294d03448eddb03f425e6b6b3616de Mon Sep 17 00:00:00 2001 From: mrummuka Date: Tue, 28 Apr 2026 12:19:53 +0300 Subject: [PATCH 2/4] feat: replace html2text (GPL-3.0) with markdownify (MIT) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Swap the optional HTML→Markdown converter used by URL ingestion from html2text to markdownify. Aligns the 'pdf' and 'all' extras with the project's MIT license and removes a copyleft dependency that affected anyone redistributing or embedding graphify. Behaviour preserved: - Headings rendered ATX style (# Title) - Links kept inline - Images dropped (matches prior ignore_images=True) - No body-width wrapping - Regex-strip fallback path retained when markdownify is unavailable Script/style blocks are now pre-stripped (with content) before conversion: markdownify's strip= removes tags but preserves their inner text, which previously leaked CSS/JS into the output. All 305 tests pass (293 existing + 12 new). --- graphify/ingest.py | 24 +++++++++++++----------- pyproject.toml | 4 ++-- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/graphify/ingest.py b/graphify/ingest.py index 62d8386b7..cf1420ffa 100644 --- a/graphify/ingest.py +++ b/graphify/ingest.py @@ -49,19 +49,21 @@ def _fetch_html(url: str) -> str: def _html_to_markdown(html: str, url: str) -> str: - """Convert HTML to clean markdown. Uses html2text if available, else basic strip.""" + """Convert HTML to clean markdown. Uses markdownify if available, else basic strip.""" + # Drop script/style content (markdownify's `strip` removes tags but keeps inner text). + cleaned = re.sub(r"]*>.*?", "", html, flags=re.DOTALL | re.IGNORECASE) + cleaned = re.sub(r"]*>.*?", "", cleaned, flags=re.DOTALL | re.IGNORECASE) try: - import html2text - h = html2text.HTML2Text() - h.ignore_links = False - h.ignore_images = True - h.body_width = 0 - return h.handle(html) + from markdownify import markdownify as _md + return _md( + cleaned, + heading_style="ATX", + bullets="-", + strip=["img"], + ).strip() except ImportError: - # Fallback: strip tags - text = re.sub(r"]*>.*?", "", html, flags=re.DOTALL | re.IGNORECASE) - text = re.sub(r"]*>.*?", "", text, flags=re.DOTALL | re.IGNORECASE) - text = re.sub(r"<[^>]+>", " ", text) + # Fallback: strip remaining tags + text = re.sub(r"<[^>]+>", " ", cleaned) text = re.sub(r"\s+", " ", text).strip() return text[:8000] diff --git a/pyproject.toml b/pyproject.toml index 2c39472d0..200cf5068 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,13 +44,13 @@ Issues = "https://github.com/safishamsi/graphify/issues" [project.optional-dependencies] mcp = ["mcp"] neo4j = ["neo4j"] -pdf = ["pypdf", "html2text"] +pdf = ["pypdf", "markdownify"] watch = ["watchdog"] svg = ["matplotlib"] leiden = ["graspologic; python_version < '3.13'"] office = ["python-docx", "openpyxl"] video = ["faster-whisper", "yt-dlp"] -all = ["mcp", "neo4j", "pypdf", "html2text", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper", "yt-dlp", "matplotlib"] +all = ["mcp", "neo4j", "pypdf", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper", "yt-dlp", "matplotlib"] [project.scripts] graphify = "graphify.__main__:main" From 370945448b9f508533b17d4ff2b3c0a07f6ec2b3 Mon Sep 17 00:00:00 2001 From: mrummuka Date: Tue, 28 Apr 2026 12:20:58 +0300 Subject: [PATCH 3/4] =?UTF-8?q?docs:=20note=20html2text=E2=86=92markdownif?= =?UTF-8?q?y=20swap=20in=20CHANGELOG=20and=20skill=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CHANGELOG: add Unreleased entry documenting the license-motivated swap - skill.md (both copies): update user-facing description from 'converted to markdown via html2text' to '...via markdownify' --- CHANGELOG.md | 9 +++++++++ graphify/skill-aider.md | 2 +- graphify/skill-claw.md | 2 +- graphify/skill-codex.md | 2 +- graphify/skill-copilot.md | 2 +- graphify/skill-droid.md | 2 +- graphify/skill-kiro.md | 2 +- graphify/skill-opencode.md | 2 +- graphify/skill-trae.md | 2 +- graphify/skill-windows.md | 2 +- graphify/skill.md | 2 +- 11 files changed, 19 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86dc8f127..a35797e01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## Unreleased + +### Changed +- Replace `html2text` (GPL-3.0) with `markdownify` (MIT) for HTML→Markdown + conversion in URL ingestion. Aligns the `pdf`/`all` extras with the + project's MIT license and removes a copyleft dependency that affected + anyone redistributing or embedding graphify. Fallback regex-strip path + unchanged. + Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) ## 0.4.23 (2026-04-18) diff --git a/graphify/skill-aider.md b/graphify/skill-aider.md index fa7007f3e..90de579f1 100644 --- a/graphify/skill-aider.md +++ b/graphify/skill-aider.md @@ -1118,7 +1118,7 @@ Supported URL types (auto-detected): - arXiv → abstract + metadata saved as `.md` - PDF → downloaded as `.pdf` - Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text +- Any webpage → converted to markdown via markdownify --- diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index 3d84acbd9..824962095 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -1118,7 +1118,7 @@ Supported URL types (auto-detected): - arXiv → abstract + metadata saved as `.md` - PDF → downloaded as `.pdf` - Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text +- Any webpage → converted to markdown via markdownify --- diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index b2e79e2b5..114225247 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -1177,7 +1177,7 @@ Supported URL types (auto-detected): - arXiv → abstract + metadata saved as `.md` - PDF → downloaded as `.pdf` - Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text +- Any webpage → converted to markdown via markdownify --- diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index 397669ae1..a28e221c5 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -1203,7 +1203,7 @@ Supported URL types (auto-detected): - arXiv → abstract + metadata saved as `.md` - PDF → downloaded as `.pdf` - Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text +- Any webpage → converted to markdown via markdownify --- diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index 6cde93503..73e515e09 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -1174,7 +1174,7 @@ Supported URL types (auto-detected): - arXiv → abstract + metadata saved as `.md` - PDF → downloaded as `.pdf` - Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text +- Any webpage → converted to markdown via markdownify --- diff --git a/graphify/skill-kiro.md b/graphify/skill-kiro.md index b3db4435d..1f1d3dd16 100644 --- a/graphify/skill-kiro.md +++ b/graphify/skill-kiro.md @@ -1117,7 +1117,7 @@ Supported URL types (auto-detected): - arXiv → abstract + metadata saved as `.md` - PDF → downloaded as `.pdf` - Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text +- Any webpage → converted to markdown via markdownify --- diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index 32819c80c..ec7985767 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -1227,7 +1227,7 @@ Supported URL types (auto-detected): - arXiv → abstract + metadata saved as `.md` - PDF → downloaded as `.pdf` - Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text +- Any webpage → converted to markdown via markdownify --- diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index 2b5b401e7..ae860166a 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -1141,7 +1141,7 @@ Supported URL types (auto-detected): - arXiv → abstract + metadata saved as `.md` - PDF → downloaded as `.pdf` - Images (.png/.jpg/.webp) → downloaded, vision extracts on next run -- Any webpage → converted to markdown via html2text +- Any webpage → converted to markdown via markdownify --- diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index 9984022e0..68b37357d 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -1167,7 +1167,7 @@ Supported URL types (auto-detected): - arXiv → abstract + metadata saved as `.md` - PDF → downloaded as `.pdf` - Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build -- Any webpage → converted to markdown via html2text +- Any webpage → converted to markdown via markdownify --- diff --git a/graphify/skill.md b/graphify/skill.md index be1e7dba0..d1509cfa1 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -1312,7 +1312,7 @@ Supported URL types (auto-detected): - arXiv → abstract + metadata saved as `.md` - PDF → downloaded as `.pdf` - Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run -- Any webpage → converted to markdown via html2text +- Any webpage → converted to markdown via markdownify --- From b48a0a50da2425d7f4a64632fc62bd75463e13a1 Mon Sep 17 00:00:00 2001 From: mrummuka Date: Wed, 29 Apr 2026 15:48:26 +0300 Subject: [PATCH 4/4] test: skip markdownify-specific assertions when extra is not installed Reviewer (PR #586) noted that the new tests asserted markdownify-specific output unconditionally, breaking 'pip install -e .' (no extras) test runs. markdownify is only declared in the 'pdf'/'all' extras; on a base install the converter falls back to regex-strip. - Add a 'requires_markdownify' skip marker (importlib.util.find_spec, no side effects) and decorate the 5 tests that assert markdownify-shape output: ATX headings, link syntax, bullet lists, no-wrap, end-to-end _fetch_webpage rendering. - Keep all fallback/regression tests unconditional. - Add 'test_fallback_basic_text_extraction' so base installs still get end-to-end coverage of the regex-strip path on a realistic HTML mix. Validation: with extras: 13 passed (was 12) without: 8 passed, 5 skipped, 0 failed full suite (with extras): 454 passed full suite (without extras): 449 passed, 5 skipped --- tests/test_html_to_markdown.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_html_to_markdown.py b/tests/test_html_to_markdown.py index d605bbc61..00e28c90d 100644 --- a/tests/test_html_to_markdown.py +++ b/tests/test_html_to_markdown.py @@ -1,12 +1,22 @@ """Tests for graphify.ingest._html_to_markdown and HTML→markdown ingestion path.""" from __future__ import annotations +import importlib.util import sys import pytest from graphify.ingest import _html_to_markdown, _fetch_webpage +# markdownify is an optional dependency (declared in the `pdf`/`all` extras). +# Tests that assert markdownify-specific output are gated behind this marker so +# the suite still runs cleanly on a base install (`pip install -e .`). +_HAS_MARKDOWNIFY = importlib.util.find_spec("markdownify") is not None +requires_markdownify = pytest.mark.skipif( + not _HAS_MARKDOWNIFY, + reason="markdownify not installed (optional 'pdf' extra)", +) + # --- Direct conversion tests --------------------------------------------------- @@ -16,6 +26,7 @@ def test_basic_paragraph(): assert "

" not in out +@requires_markdownify def test_heading_atx_style(): out = _html_to_markdown("

Title

", "http://example.com") assert "# Title" in out @@ -23,6 +34,7 @@ def test_heading_atx_style(): assert "===" not in out +@requires_markdownify def test_links_preserved(): html = '

See x for more.

' out = _html_to_markdown(html, "http://example.com") @@ -52,6 +64,7 @@ def test_script_and_style_stripped(): assert "color:red" not in out +@requires_markdownify def test_bullet_list(): html = "
  • alpha
  • beta
" out = _html_to_markdown(html, "http://example.com") @@ -59,6 +72,7 @@ def test_bullet_list(): assert "- beta" in out +@requires_markdownify def test_no_body_wrapping(): long_text = "word " * 50 # ~250 chars on one line html = f"

{long_text.strip()}

" @@ -105,8 +119,22 @@ def test_fallback_when_markdownify_missing(monkeypatch): assert "<" not in out +def test_fallback_basic_text_extraction(monkeypatch): + """Fallback path must extract readable text from a realistic HTML mix + (heading + list + paragraph), with all tags removed. Runs unconditionally + so base installs (`pip install -e .`) still get end-to-end coverage of + the regex-strip path.""" + monkeypatch.setitem(sys.modules, "markdownify", None) + html = "

T

  • a
  • b

p

" + out = _html_to_markdown(html, "http://example.com") + for token in ("T", "a", "b", "p"): + assert token in out, f"missing {token!r} in fallback output: {out!r}" + assert "<" not in out + + # --- Integration with _fetch_webpage ------------------------------------------ +@requires_markdownify def test_fetch_webpage_uses_converter(monkeypatch): """End-to-end smoke for the only caller of _html_to_markdown.""" canned_html = (