feat: add reader mode to extract article content from web pages

README.md

@@ -110,6 +110,7 @@ At the moment, the following optional dependencies are available:
 * `[az-content-understanding]` Installs dependencies for Azure Content Understanding
 * `[audio-transcription]` Installs dependencies for audio transcription of wav and mp3 files
 * `[youtube-transcription]` Installs dependencies for fetching YouTube video transcription
+* `[readability]` Installs dependencies for reader mode (article content extraction from web pages)
 
 ### Plugins
 

packages/markitdown/pyproject.toml

@@ -46,6 +46,7 @@ all = [
   "pydub",
   "SpeechRecognition",
   "youtube-transcript-api~=1.0.0",
+  "readability-lxml",
   "azure-ai-documentintelligence",
   "azure-ai-contentunderstanding>=1.2.0b1",
   "azure-identity",
@@ -58,6 +59,7 @@ pdf = ["pdfminer.six>=20251230", "pdfplumber>=0.11.9"]
 outlook = ["olefile"]
 audio-transcription = ["pydub", "SpeechRecognition"]
 youtube-transcription = ["youtube-transcript-api"]
+readability = ["readability-lxml"]
 az-doc-intel = ["azure-ai-documentintelligence", "azure-identity"]
 # >=1.2.0b1 required for to_llm_input() helper used by ContentUnderstandingConverter
 az-content-understanding = ["azure-ai-contentunderstanding>=1.2.0b1", "azure-identity"]

packages/markitdown/src/markitdown/__main__.py

@@ -16,8 +16,7 @@ def main():
         description="Convert various file formats to markdown.",
         prog="markitdown",
         formatter_class=argparse.RawDescriptionHelpFormatter,
-        usage=dedent(
-            """
+        usage=dedent("""
             SYNTAX:
 
                 markitdown <OPTIONAL: FILENAME>
@@ -42,8 +41,7 @@ def main():
                 OR
 
                 markitdown example.pdf > example.md
-            """
-        ).strip(),
+            """).strip(),
     )
 
     parser.add_argument(
@@ -138,6 +136,13 @@ def main():
         help="Keep data URIs (like base64-encoded images) in the output. By default, data URIs are truncated.",
     )
 
+    parser.add_argument(
+        "-r",
+        "--reader-mode",
+        action="store_true",
+        help="Extract main article content from web pages, stripping navigation, sidebars, and footers. Requires the [readability] extra.",
+    )
+
     parser.add_argument("filename", nargs="?")
     args = parser.parse_args()
 
@@ -249,10 +254,14 @@ def main():
             sys.stdin.buffer,
             stream_info=stream_info,
             keep_data_uris=args.keep_data_uris,
+            reader_mode=args.reader_mode,
         )
     else:
         result = markitdown.convert(
-            args.filename, stream_info=stream_info, keep_data_uris=args.keep_data_uris
+            args.filename,
+            stream_info=stream_info,
+            keep_data_uris=args.keep_data_uris,
+            reader_mode=args.reader_mode,
         )
 
     _handle_output(args, result)

packages/markitdown/src/markitdown/converters/_html_converter.py

@@ -1,12 +1,20 @@
 import io
+import sys
 import warnings
 from typing import Any, BinaryIO, Optional
 from bs4 import BeautifulSoup
 
 from .._base_converter import DocumentConverter, DocumentConverterResult
+from .._exceptions import MissingDependencyException, MISSING_DEPENDENCY_MESSAGE
 from .._stream_info import StreamInfo
 from ._markdownify import _CustomMarkdownify
 
+_dependency_exc_info = None
+try:
+    from readability import Document as ReadabilityDocument
+except ImportError:
+    _dependency_exc_info = sys.exc_info()
+
 ACCEPTED_MIME_TYPE_PREFIXES = [
     "text/html",
     "application/xhtml",
@@ -45,13 +53,33 @@ def convert(
         stream_info: StreamInfo,
         **kwargs: Any,  # Options to pass to the converter
     ) -> DocumentConverterResult:
-        # Pop our own keyword before forwarding the rest to markdownify.
+        # Pop our own keywords before forwarding the rest to markdownify.
         # strict=True raises RecursionError instead of falling back to plain text.
         strict: bool = kwargs.pop("strict", False)
+        reader_mode: bool = kwargs.pop("reader_mode", False)
 
         # Parse the stream
         encoding = "utf-8" if stream_info.charset is None else stream_info.charset
-        soup = BeautifulSoup(file_stream, "html.parser", from_encoding=encoding)
+        raw_html = file_stream.read()
+
+        title = None
+
+        if reader_mode:
+            if _dependency_exc_info is not None:
+                raise MissingDependencyException(
+                    MISSING_DEPENDENCY_MESSAGE.format(
+                        converter="HtmlConverter",
+                        extension="html (reader mode)",
+                        feature="readability",
+                    )
+                )
+            html_text = raw_html.decode(encoding, errors="replace")
+            doc = ReadabilityDocument(html_text, url=stream_info.url)
+            article_html = doc.summary()
+            title = doc.short_title()
+            soup = BeautifulSoup(article_html, "html.parser")
+        else:
+            soup = BeautifulSoup(raw_html, "html.parser", from_encoding=encoding)
 
         # Remove javascript and style blocks
         for script in soup(["script", "style"]):
@@ -85,9 +113,12 @@ def convert(
         # remove leading and trailing \n
         webpage_text = webpage_text.strip()
 
+        if title is None:
+            title = None if soup.title is None else soup.title.string
+
         return DocumentConverterResult(
             markdown=webpage_text,
-            title=None if soup.title is None else soup.title.string,
+            title=title,
         )
 
     def convert_string(

packages/markitdown/tests/test_reader_mode.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3 -m pytest
+import io
+import pytest
+from unittest.mock import patch
+
+from markitdown import MarkItDown, StreamInfo
+from markitdown._exceptions import MissingDependencyException
+
+SAMPLE_HTML = """
+<!DOCTYPE html>
+<html>
+<head><title>Test Article</title></head>
+<body>
+    <nav>
+        <ul>
+            <li><a href="/">Home</a></li>
+            <li><a href="/about">About</a></li>
+            <li><a href="/contact">Contact</a></li>
+        </ul>
+    </nav>
+    <aside class="sidebar">
+        <h3>Related Articles</h3>
+        <ul>
+            <li><a href="/other">Other Article</a></li>
+        </ul>
+    </aside>
+    <article>
+        <h1>Main Article Title</h1>
+        <p>This is the main article content that should be extracted by reader mode.
+        It contains several paragraphs of meaningful text that readability should
+        identify as the primary content of the page.</p>
+        <p>Here is another paragraph with more substantive content about the topic
+        at hand. Reader mode should preserve this while stripping away the
+        navigation and sidebar elements.</p>
+        <p>A third paragraph ensures there is enough content for readability to
+        confidently identify this as the main article body.</p>
+    </article>
+    <footer>
+        <p>Copyright 2024 Test Site</p>
+        <nav>
+            <a href="/privacy">Privacy Policy</a>
+            <a href="/terms">Terms of Service</a>
+        </nav>
+    </footer>
+</body>
+</html>
+"""
+
+
+class TestReaderMode:
+    def test_reader_mode_extracts_article_content(self):
+        """Reader mode should extract main article content and exclude nav/footer."""
+        md = MarkItDown()
+        result = md.convert_stream(
+            io.BytesIO(SAMPLE_HTML.encode("utf-8")),
+            stream_info=StreamInfo(
+                mimetype="text/html", extension=".html", charset="utf-8"
+            ),
+            reader_mode=True,
+        )
+
+        assert "Main Article Title" in result.markdown
+        assert "main article content" in result.markdown
+
+    def test_without_reader_mode_includes_everything(self):
+        """Without reader mode, the full page including nav/footer should appear."""
+        md = MarkItDown()
+        result = md.convert_stream(
+            io.BytesIO(SAMPLE_HTML.encode("utf-8")),
+            stream_info=StreamInfo(
+                mimetype="text/html", extension=".html", charset="utf-8"
+            ),
+            reader_mode=False,
+        )
+
+        assert "Home" in result.markdown
+        assert "Privacy Policy" in result.markdown
+        assert "Main Article Title" in result.markdown
+
+    def test_reader_mode_missing_dependency(self):
+        """Should raise MissingDependencyException when readability-lxml is not installed."""
+        import markitdown.converters._html_converter as html_mod
+        from markitdown.converters._html_converter import HtmlConverter
+
+        original = html_mod._dependency_exc_info
+        try:
+            html_mod._dependency_exc_info = (
+                ImportError,
+                ImportError("no module"),
+                None,
+            )
+
+            converter = HtmlConverter()
+            with pytest.raises(MissingDependencyException):
+                converter.convert(
+                    io.BytesIO(SAMPLE_HTML.encode("utf-8")),
+                    StreamInfo(
+                        mimetype="text/html", extension=".html", charset="utf-8"
+                    ),
+                    reader_mode=True,
+                )
+        finally:
+            html_mod._dependency_exc_info = original
+
+    def test_reader_mode_default_is_false(self):
+        """Default behavior (no reader_mode flag) should convert the full page."""
+        md = MarkItDown()
+        result = md.convert_stream(
+            io.BytesIO(SAMPLE_HTML.encode("utf-8")),
+            stream_info=StreamInfo(
+                mimetype="text/html", extension=".html", charset="utf-8"
+            ),
+        )
+
+        assert "Home" in result.markdown
+        assert "Privacy Policy" in result.markdown