diff --git a/README.md b/README.md index 37be2d1d0..9e6547152 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,7 @@ At the moment, the following optional dependencies are available: * `[az-content-understanding]` Installs dependencies for Azure Content Understanding * `[audio-transcription]` Installs dependencies for audio transcription of wav and mp3 files * `[youtube-transcription]` Installs dependencies for fetching YouTube video transcription +* `[readability]` Installs dependencies for reader mode (article content extraction from web pages) ### Plugins diff --git a/packages/markitdown/pyproject.toml b/packages/markitdown/pyproject.toml index d4c20a402..ef9d971fa 100644 --- a/packages/markitdown/pyproject.toml +++ b/packages/markitdown/pyproject.toml @@ -46,6 +46,7 @@ all = [ "pydub", "SpeechRecognition", "youtube-transcript-api~=1.0.0", + "readability-lxml", "azure-ai-documentintelligence", "azure-ai-contentunderstanding>=1.2.0b1", "azure-identity", @@ -58,6 +59,7 @@ pdf = ["pdfminer.six>=20251230", "pdfplumber>=0.11.9"] outlook = ["olefile"] audio-transcription = ["pydub", "SpeechRecognition"] youtube-transcription = ["youtube-transcript-api"] +readability = ["readability-lxml"] az-doc-intel = ["azure-ai-documentintelligence", "azure-identity"] # >=1.2.0b1 required for to_llm_input() helper used by ContentUnderstandingConverter az-content-understanding = ["azure-ai-contentunderstanding>=1.2.0b1", "azure-identity"] diff --git a/packages/markitdown/src/markitdown/__main__.py b/packages/markitdown/src/markitdown/__main__.py index 56eb89cd8..225b5804e 100644 --- a/packages/markitdown/src/markitdown/__main__.py +++ b/packages/markitdown/src/markitdown/__main__.py @@ -16,8 +16,7 @@ def main(): description="Convert various file formats to markdown.", prog="markitdown", formatter_class=argparse.RawDescriptionHelpFormatter, - usage=dedent( - """ + usage=dedent(""" SYNTAX: markitdown @@ -42,8 +41,7 @@ def main(): OR markitdown example.pdf > example.md - """ - ).strip(), + """).strip(), ) parser.add_argument( @@ -138,6 +136,13 @@ def main(): help="Keep data URIs (like base64-encoded images) in the output. By default, data URIs are truncated.", ) + parser.add_argument( + "-r", + "--reader-mode", + action="store_true", + help="Extract main article content from web pages, stripping navigation, sidebars, and footers. Requires the [readability] extra.", + ) + parser.add_argument("filename", nargs="?") args = parser.parse_args() @@ -249,10 +254,14 @@ def main(): sys.stdin.buffer, stream_info=stream_info, keep_data_uris=args.keep_data_uris, + reader_mode=args.reader_mode, ) else: result = markitdown.convert( - args.filename, stream_info=stream_info, keep_data_uris=args.keep_data_uris + args.filename, + stream_info=stream_info, + keep_data_uris=args.keep_data_uris, + reader_mode=args.reader_mode, ) _handle_output(args, result) diff --git a/packages/markitdown/src/markitdown/converters/_html_converter.py b/packages/markitdown/src/markitdown/converters/_html_converter.py index 029b27f57..b2ca503cf 100644 --- a/packages/markitdown/src/markitdown/converters/_html_converter.py +++ b/packages/markitdown/src/markitdown/converters/_html_converter.py @@ -1,12 +1,20 @@ import io +import sys import warnings from typing import Any, BinaryIO, Optional from bs4 import BeautifulSoup from .._base_converter import DocumentConverter, DocumentConverterResult +from .._exceptions import MissingDependencyException, MISSING_DEPENDENCY_MESSAGE from .._stream_info import StreamInfo from ._markdownify import _CustomMarkdownify +_dependency_exc_info = None +try: + from readability import Document as ReadabilityDocument +except ImportError: + _dependency_exc_info = sys.exc_info() + ACCEPTED_MIME_TYPE_PREFIXES = [ "text/html", "application/xhtml", @@ -45,13 +53,33 @@ def convert( stream_info: StreamInfo, **kwargs: Any, # Options to pass to the converter ) -> DocumentConverterResult: - # Pop our own keyword before forwarding the rest to markdownify. + # Pop our own keywords before forwarding the rest to markdownify. # strict=True raises RecursionError instead of falling back to plain text. strict: bool = kwargs.pop("strict", False) + reader_mode: bool = kwargs.pop("reader_mode", False) # Parse the stream encoding = "utf-8" if stream_info.charset is None else stream_info.charset - soup = BeautifulSoup(file_stream, "html.parser", from_encoding=encoding) + raw_html = file_stream.read() + + title = None + + if reader_mode: + if _dependency_exc_info is not None: + raise MissingDependencyException( + MISSING_DEPENDENCY_MESSAGE.format( + converter="HtmlConverter", + extension="html (reader mode)", + feature="readability", + ) + ) + html_text = raw_html.decode(encoding, errors="replace") + doc = ReadabilityDocument(html_text, url=stream_info.url) + article_html = doc.summary() + title = doc.short_title() + soup = BeautifulSoup(article_html, "html.parser") + else: + soup = BeautifulSoup(raw_html, "html.parser", from_encoding=encoding) # Remove javascript and style blocks for script in soup(["script", "style"]): @@ -85,9 +113,12 @@ def convert( # remove leading and trailing \n webpage_text = webpage_text.strip() + if title is None: + title = None if soup.title is None else soup.title.string + return DocumentConverterResult( markdown=webpage_text, - title=None if soup.title is None else soup.title.string, + title=title, ) def convert_string( diff --git a/packages/markitdown/tests/test_reader_mode.py b/packages/markitdown/tests/test_reader_mode.py new file mode 100644 index 000000000..b3c56f0ad --- /dev/null +++ b/packages/markitdown/tests/test_reader_mode.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 -m pytest +import io +import pytest +from unittest.mock import patch + +from markitdown import MarkItDown, StreamInfo +from markitdown._exceptions import MissingDependencyException + +SAMPLE_HTML = """ + + +Test Article + + + +
+

Main Article Title

+

This is the main article content that should be extracted by reader mode. + It contains several paragraphs of meaningful text that readability should + identify as the primary content of the page.

+

Here is another paragraph with more substantive content about the topic + at hand. Reader mode should preserve this while stripping away the + navigation and sidebar elements.

+

A third paragraph ensures there is enough content for readability to + confidently identify this as the main article body.

+
+ + + +""" + + +class TestReaderMode: + def test_reader_mode_extracts_article_content(self): + """Reader mode should extract main article content and exclude nav/footer.""" + md = MarkItDown() + result = md.convert_stream( + io.BytesIO(SAMPLE_HTML.encode("utf-8")), + stream_info=StreamInfo( + mimetype="text/html", extension=".html", charset="utf-8" + ), + reader_mode=True, + ) + + assert "Main Article Title" in result.markdown + assert "main article content" in result.markdown + + def test_without_reader_mode_includes_everything(self): + """Without reader mode, the full page including nav/footer should appear.""" + md = MarkItDown() + result = md.convert_stream( + io.BytesIO(SAMPLE_HTML.encode("utf-8")), + stream_info=StreamInfo( + mimetype="text/html", extension=".html", charset="utf-8" + ), + reader_mode=False, + ) + + assert "Home" in result.markdown + assert "Privacy Policy" in result.markdown + assert "Main Article Title" in result.markdown + + def test_reader_mode_missing_dependency(self): + """Should raise MissingDependencyException when readability-lxml is not installed.""" + import markitdown.converters._html_converter as html_mod + from markitdown.converters._html_converter import HtmlConverter + + original = html_mod._dependency_exc_info + try: + html_mod._dependency_exc_info = ( + ImportError, + ImportError("no module"), + None, + ) + + converter = HtmlConverter() + with pytest.raises(MissingDependencyException): + converter.convert( + io.BytesIO(SAMPLE_HTML.encode("utf-8")), + StreamInfo( + mimetype="text/html", extension=".html", charset="utf-8" + ), + reader_mode=True, + ) + finally: + html_mod._dependency_exc_info = original + + def test_reader_mode_default_is_false(self): + """Default behavior (no reader_mode flag) should convert the full page.""" + md = MarkItDown() + result = md.convert_stream( + io.BytesIO(SAMPLE_HTML.encode("utf-8")), + stream_info=StreamInfo( + mimetype="text/html", extension=".html", charset="utf-8" + ), + ) + + assert "Home" in result.markdown + assert "Privacy Policy" in result.markdown