From 927bfaec4f9f9c60fffa99d5798036a2536576d9 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 21 Mar 2023 00:06:58 -0400 Subject: [PATCH 01/11] fix: properly add quotes to markdown --- monty/utils/markdown.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monty/utils/markdown.py b/monty/utils/markdown.py index 858698ea..c729e802 100644 --- a/monty/utils/markdown.py +++ b/monty/utils/markdown.py @@ -166,7 +166,7 @@ def block_code(self, code: str, info: str = None) -> str: def block_quote(self, text: str) -> str: """Quote the provided text.""" if text: - return "> " + "> ".join(text) + "\n" + return "> " + "> ".join(text.splitlines(keepends=True)) + "\n" return "" def block_html(self, html: str) -> str: From 4cb3f6253ef07032a32509c38828b1abb2a0a947 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 21 Mar 2023 00:59:57 -0400 Subject: [PATCH 02/11] feat: show lists in markdown renderer --- monty/constants.py | 1 + monty/exts/info/github_info.py | 2 +- monty/utils/markdown.py | 15 +++++++++++---- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/monty/constants.py b/monty/constants.py index 77fcd71e..0523eaf3 100644 --- a/monty/constants.py +++ b/monty/constants.py @@ -140,6 +140,7 @@ class Emojis: confirmation = "\u2705" decline = "\u274c" + no_choice_light = "\u25fb\ufe0f" x = "\U0001f1fd" o = "\U0001f1f4" diff --git a/monty/exts/info/github_info.py b/monty/exts/info/github_info.py index 13ddff80..5e22d5a7 100644 --- a/monty/exts/info/github_info.py +++ b/monty/exts/info/github_info.py @@ -254,7 +254,7 @@ async def fetch_data( def render_github_markdown(self, body: str, *, context: RenderContext = None, limit: int = 700) -> str: """Render GitHub Flavored Markdown to Discord flavoured markdown.""" - markdown = mistune.create_markdown(escape=False, renderer=DiscordRenderer()) + markdown = mistune.create_markdown(escape=False, renderer=DiscordRenderer(), plugins=["task_lists"]) body = markdown(body) or "" if len(body) > limit: diff --git a/monty/utils/markdown.py b/monty/utils/markdown.py index c729e802..9c3d0473 100644 --- a/monty/utils/markdown.py +++ b/monty/utils/markdown.py @@ -6,6 +6,8 @@ from bs4.element import PageElement, Tag from markdownify import MarkdownConverter +from monty import constants + # taken from version 0.6.1 of markdownify WHITESPACE_RE = re.compile(r"[\r\n\s\t ]+") @@ -187,12 +189,17 @@ def paragraph(self, text: str) -> str: return text + "\n" def list(self, text: str, ordered: bool, level: int, start: Any = None) -> str: - """Do nothing when encountering a list.""" - return "" + """Return the unedited list.""" + return text def list_item(self, text: Any, level: int) -> str: - """Do nothing when encountering a list.""" - return "" + """Show the list, indented to its proper level.""" + return "\u200b " * (level - 1) * 8 + f"- {text}\n" + + def task_list_item(self, text: Any, level: int, checked: bool = False, **attrs) -> str: + """Convert task list options to emoji.""" + emoji = constants.Emojis.confirmation if checked else constants.Emojis.no_choice_light + return self.list_item(emoji + " " + text, level=level) def finalize(self, data: Any) -> str: """Finalize the data.""" From 73f65453f0c0fe57b3b2445e82256f0cdfb47f85 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 21 Mar 2023 01:00:21 -0400 Subject: [PATCH 03/11] fix: show new lines before headings --- monty/utils/markdown.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monty/utils/markdown.py b/monty/utils/markdown.py index 9c3d0473..b16f1a6c 100644 --- a/monty/utils/markdown.py +++ b/monty/utils/markdown.py @@ -130,9 +130,9 @@ def strong(self, text: str) -> str: def heading(self, text: str, level: int) -> str: """Format the heading to be bold if its large enough. Otherwise underline it.""" if level in (1, 2, 3): - return f"**{text}**\n" + return "\n" f"**{text}**\n" else: - return f"__{text}__\n" + return "\n" f"__{text}__\n" def newline(self) -> str: """Return a new line.""" From a80ea2d0df4572c475371f2a6423efaa79a04d52 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 21 Mar 2023 01:41:36 -0400 Subject: [PATCH 04/11] fix: s p a c i n g --- monty/exts/info/github_info.py | 10 +++++++++- monty/utils/markdown.py | 17 ++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/monty/exts/info/github_info.py b/monty/exts/info/github_info.py index 5e22d5a7..5ab05a43 100644 --- a/monty/exts/info/github_info.py +++ b/monty/exts/info/github_info.py @@ -254,7 +254,15 @@ async def fetch_data( def render_github_markdown(self, body: str, *, context: RenderContext = None, limit: int = 700) -> str: """Render GitHub Flavored Markdown to Discord flavoured markdown.""" - markdown = mistune.create_markdown(escape=False, renderer=DiscordRenderer(), plugins=["task_lists"]) + markdown = mistune.create_markdown( + escape=False, + renderer=DiscordRenderer(), + plugins=[ + "strikethrough", + "task_lists", + "url", + ], + ) body = markdown(body) or "" if len(body) > limit: diff --git a/monty/utils/markdown.py b/monty/utils/markdown.py index b16f1a6c..38bf5517 100644 --- a/monty/utils/markdown.py +++ b/monty/utils/markdown.py @@ -140,8 +140,16 @@ def newline(self) -> str: def linebreak(self) -> str: """Return two new lines.""" + return "\n" + + def blank_line(self) -> str: + """Return a blank line.""" return "\n\n" + def softbreak(self) -> str: + """Return a softbreak.""" + return "\n" + def inline_html(self, html: str) -> str: """No op.""" return "" @@ -186,15 +194,18 @@ def codespan(self, text: str) -> str: def paragraph(self, text: str) -> str: """Return a paragraph with a newline postceeding.""" - return text + "\n" + return f"{text}\n" def list(self, text: str, ordered: bool, level: int, start: Any = None) -> str: """Return the unedited list.""" - return text + # todo: figure out how this should actually work + if level != 1: + return text + return text.lstrip("\n") + "\n" def list_item(self, text: Any, level: int) -> str: """Show the list, indented to its proper level.""" - return "\u200b " * (level - 1) * 8 + f"- {text}\n" + return "\n" + "\u200b " * (level - 1) * 8 + f"- {text}" def task_list_item(self, text: Any, level: int, checked: bool = False, **attrs) -> str: """Convert task list options to emoji.""" From 4923c82d81ed9835211cc00f98de3ace4b2e37ef Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 21 Mar 2023 01:43:12 -0400 Subject: [PATCH 05/11] feat: show up to 2700 characters in github previews --- monty/exts/info/github_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monty/exts/info/github_info.py b/monty/exts/info/github_info.py index 5ab05a43..3609a362 100644 --- a/monty/exts/info/github_info.py +++ b/monty/exts/info/github_info.py @@ -252,7 +252,7 @@ async def fetch_data( await self.request_cache.set(cache_key, (None, body), timeout=timedelta(minutes=30).total_seconds()) return body - def render_github_markdown(self, body: str, *, context: RenderContext = None, limit: int = 700) -> str: + def render_github_markdown(self, body: str, *, context: RenderContext = None, limit: int = 2700) -> str: """Render GitHub Flavored Markdown to Discord flavoured markdown.""" markdown = mistune.create_markdown( escape=False, From 7ab71fe86c872e0c257273289def79612bdc27c4 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 21 Mar 2023 01:52:22 -0400 Subject: [PATCH 06/11] feat: show the image alt text if it exists --- monty/utils/markdown.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/monty/utils/markdown.py b/monty/utils/markdown.py index 38bf5517..6efd2535 100644 --- a/monty/utils/markdown.py +++ b/monty/utils/markdown.py @@ -115,9 +115,9 @@ def link(self, link: str, text: Optional[str] = None, title: Optional[str] = Non else: return link - def image(self, src: str, alt: str = "", title: str = None) -> str: + def image(self, src: str, alt: str = None, title: str = None) -> str: """Return a link to the provided image.""" - return self.link(src, text="image", title=title) + return self.link(src, text="!image", title=alt) def emphasis(self, text: str) -> str: """Return italiced text.""" @@ -140,7 +140,7 @@ def newline(self) -> str: def linebreak(self) -> str: """Return two new lines.""" - return "\n" + return "\n\n" def blank_line(self) -> str: """Return a blank line.""" From 3ff1bd5fe6ae43ff6bd336b8f9ec0cfee0719028 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 21 Mar 2023 02:19:50 -0400 Subject: [PATCH 07/11] feat: convert issue tags to links --- monty/exts/info/github_info.py | 7 +++++-- monty/utils/markdown.py | 12 +++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/monty/exts/info/github_info.py b/monty/exts/info/github_info.py index 3609a362..090703ea 100644 --- a/monty/exts/info/github_info.py +++ b/monty/exts/info/github_info.py @@ -254,9 +254,10 @@ async def fetch_data( def render_github_markdown(self, body: str, *, context: RenderContext = None, limit: int = 2700) -> str: """Render GitHub Flavored Markdown to Discord flavoured markdown.""" + url_prefix = context and context.html_url markdown = mistune.create_markdown( escape=False, - renderer=DiscordRenderer(), + renderer=DiscordRenderer(repo=url_prefix), plugins=[ "strikethrough", "task_lists", @@ -662,7 +663,9 @@ def format_embed_expanded_issue( body: Optional[str] = json_data["body"] if body and not body.isspace(): # escape wack stuff from the markdown - embed.description = self.render_github_markdown(body, context=None) + embed.description = self.render_github_markdown( + body, context=RenderContext(user=issue.organisation, repo=issue.repository) + ) if not body or body.isspace(): embed.description = "*No description provided.*" return embed diff --git a/monty/utils/markdown.py b/monty/utils/markdown.py index 6efd2535..5d95ced4 100644 --- a/monty/utils/markdown.py +++ b/monty/utils/markdown.py @@ -98,8 +98,18 @@ def convert_hr(self, el: PageElement, text: str, convert_as_inline: bool) -> str class DiscordRenderer(mistune.renderers.BaseRenderer): """Custom renderer for markdown to discord compatiable markdown.""" + def __init__(self, repo: str = None): + self._repo = (repo or "").rstrip("/") + def text(self, text: str) -> str: - """No op.""" + """Replace GitHub links with their expanded versions.""" + if self._repo: + # todo: expand this to all different varieties of automatic links + # if a repository is provided we replace all snippets with the correct thing + def replacement(match: re.Match[str]) -> str: + return self.link(self._repo + "/issues/" + match[1], text=match[0]) + + return re.sub(r"(?:GH-|#)(\d+)", replacement, text) return text def link(self, link: str, text: Optional[str] = None, title: Optional[str] = None) -> str: From 40c97601fd4db19c9918ef4a4def9443fd14ee99 Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 21 Mar 2023 21:24:36 -0400 Subject: [PATCH 08/11] fix: shuffle new lines a bit --- monty/utils/markdown.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/monty/utils/markdown.py b/monty/utils/markdown.py index 5d95ced4..69c5883c 100644 --- a/monty/utils/markdown.py +++ b/monty/utils/markdown.py @@ -152,14 +152,6 @@ def linebreak(self) -> str: """Return two new lines.""" return "\n\n" - def blank_line(self) -> str: - """Return a blank line.""" - return "\n\n" - - def softbreak(self) -> str: - """Return a softbreak.""" - return "\n" - def inline_html(self, html: str) -> str: """No op.""" return "" @@ -204,14 +196,14 @@ def codespan(self, text: str) -> str: def paragraph(self, text: str) -> str: """Return a paragraph with a newline postceeding.""" - return f"{text}\n" + return f"{text}\n\n" def list(self, text: str, ordered: bool, level: int, start: Any = None) -> str: """Return the unedited list.""" # todo: figure out how this should actually work if level != 1: return text - return text.lstrip("\n") + "\n" + return text.lstrip("\n") + "\n\n" def list_item(self, text: Any, level: int) -> str: """Show the list, indented to its proper level.""" From e6b74249ddde3c70d086bad861025d62df1eedfe Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 21 Mar 2023 21:26:08 -0400 Subject: [PATCH 09/11] fix strip new lines from quotes --- monty/utils/markdown.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monty/utils/markdown.py b/monty/utils/markdown.py index 69c5883c..ccb34986 100644 --- a/monty/utils/markdown.py +++ b/monty/utils/markdown.py @@ -178,7 +178,7 @@ def block_code(self, code: str, info: str = None) -> str: def block_quote(self, text: str) -> str: """Quote the provided text.""" if text: - return "> " + "> ".join(text.splitlines(keepends=True)) + "\n" + return "> " + "> ".join(text.rstrip().splitlines(keepends=True)) + "\n" return "" def block_html(self, html: str) -> str: From 1b8bbcfd8d445620b26dc2054949bb423e7584fa Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 21 Mar 2023 21:37:46 -0400 Subject: [PATCH 10/11] fix: add newlines to codeblocks --- monty/utils/markdown.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monty/utils/markdown.py b/monty/utils/markdown.py index ccb34986..2092f485 100644 --- a/monty/utils/markdown.py +++ b/monty/utils/markdown.py @@ -173,7 +173,7 @@ def block_code(self, code: str, info: str = None) -> str: lang = info.split(None, 1)[0] md += lang md += "\n" - return md + code.replace("`" * 3, "`\u200b" * 3) + "\n```" + return md + code.replace("`" * 3, "`\u200b" * 3) + "\n```\n" def block_quote(self, text: str) -> str: """Quote the provided text.""" From 97481b8eea8a21e3bc1606c615a439a41a625ded Mon Sep 17 00:00:00 2001 From: onerandomusername Date: Tue, 21 Mar 2023 21:42:24 -0400 Subject: [PATCH 11/11] chore: pre-compile regex --- monty/utils/markdown.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/monty/utils/markdown.py b/monty/utils/markdown.py index 2092f485..fb5b830c 100644 --- a/monty/utils/markdown.py +++ b/monty/utils/markdown.py @@ -9,6 +9,11 @@ from monty import constants +__all__ = ( + "remove_codeblocks", + "DocMarkdownConverter", + "DiscordRenderer", +) # taken from version 0.6.1 of markdownify WHITESPACE_RE = re.compile(r"[\r\n\s\t ]+") @@ -18,6 +23,8 @@ re.DOTALL | re.MULTILINE, ) +GH_ISSUE_RE = re.compile(r"(?:GH-|#)(\d+)") + def remove_codeblocks(content: str) -> str: """Remove any codeblock in a message.""" @@ -109,7 +116,7 @@ def text(self, text: str) -> str: def replacement(match: re.Match[str]) -> str: return self.link(self._repo + "/issues/" + match[1], text=match[0]) - return re.sub(r"(?:GH-|#)(\d+)", replacement, text) + return GH_ISSUE_RE.sub(replacement, text) return text def link(self, link: str, text: Optional[str] = None, title: Optional[str] = None) -> str: