From e5ebda8586d39a97d5751200f578f2265dda8ce8 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 18 Jan 2023 14:57:57 -0800 Subject: [PATCH 01/24] Create file structure for rtf tokenizer and initial class --- extract_msg/rtf/__init__.py | 0 extract_msg/rtf/rtf_tokenizer.py | 3 +++ 2 files changed, 3 insertions(+) create mode 100644 extract_msg/rtf/__init__.py create mode 100644 extract_msg/rtf/rtf_tokenizer.py diff --git a/extract_msg/rtf/__init__.py b/extract_msg/rtf/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/extract_msg/rtf/rtf_tokenizer.py b/extract_msg/rtf/rtf_tokenizer.py new file mode 100644 index 00000000..947c69cf --- /dev/null +++ b/extract_msg/rtf/rtf_tokenizer.py @@ -0,0 +1,3 @@ +class RTFTokenizer: + def __init__(data : bytes = None): + pass From 5f2448959862424b44e924595fde3f4771165ead Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sun, 22 Jan 2023 22:21:47 -0800 Subject: [PATCH 02/24] Started work on functions for rtf tokenizer --- extract_msg/rtf/rtf_tokenizer.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/extract_msg/rtf/rtf_tokenizer.py b/extract_msg/rtf/rtf_tokenizer.py index 947c69cf..a9d2a3fc 100644 --- a/extract_msg/rtf/rtf_tokenizer.py +++ b/extract_msg/rtf/rtf_tokenizer.py @@ -1,3 +1,10 @@ class RTFTokenizer: - def __init__(data : bytes = None): + def __init__(self, data : bytes = None): + + # Feed the data to our parser if provided. Feeding will clear all of the + # existing data. + if data: + self.feed(data) + + def feed(self, data : bytes) -> None: pass From 08005c5ea4b7d7fbf58a86b1d1974387c6589fcd Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sat, 28 Jan 2023 17:00:49 -0800 Subject: [PATCH 03/24] More work on tokenizer (will likely throw errors) --- .travis.yml | 2 +- README.rst | 6 +-- extract_msg/{rtf => _rtf}/__init__.py | 0 extract_msg/_rtf/rtf_tokenizer.py | 73 +++++++++++++++++++++++++++ extract_msg/rtf/rtf_tokenizer.py | 10 ---- setup.py | 2 +- temp-changelog-rtf.md | 2 + 7 files changed, 80 insertions(+), 15 deletions(-) rename extract_msg/{rtf => _rtf}/__init__.py (100%) create mode 100644 extract_msg/_rtf/rtf_tokenizer.py delete mode 100644 extract_msg/rtf/rtf_tokenizer.py create mode 100644 temp-changelog-rtf.md diff --git a/.travis.yml b/.travis.yml index 2293c0f9..1a35814b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: python python: - - "3.6" + - "3.8" install: - python setup.py install script: diff --git a/README.rst b/README.rst index 7a653cd2..f935af73 100644 --- a/README.rst +++ b/README.rst @@ -38,7 +38,7 @@ attachments. The script uses Philippe Lagadec's Python module that reads Microsoft OLE2 files (also called Structured Storage, Compound File Binary Format or Compound Document File Format). This is the underlying format of -Outlook's .msg files. This library currently supports Python 3.6 and above. +Outlook's .msg files. This library currently supports Python 3.7 and above. The script was originally built using Peter Fiskerstrand's documentation of the .msg format. Redemption's discussion of the different property types used within @@ -237,8 +237,8 @@ your access to the newest major version of extract-msg. .. |PyPI3| image:: https://img.shields.io/badge/pypi-0.39.0-blue.svg :target: https://pypi.org/project/extract-msg/0.39.0/ -.. |PyPI2| image:: https://img.shields.io/badge/python-3.6+-brightgreen.svg - :target: https://www.python.org/downloads/release/python-367/ +.. |PyPI2| image:: https://img.shields.io/badge/python-3.8+-brightgreen.svg + :target: https://www.python.org/downloads/release/python-3816/ .. _Matthew Walker: https://github.com/mattgwwalker .. _Destiny Peterson (The Elemental of Destruction): https://github.com/TheElementalOfDestruction .. _JP Bourget: https://github.com/punkrokk diff --git a/extract_msg/rtf/__init__.py b/extract_msg/_rtf/__init__.py similarity index 100% rename from extract_msg/rtf/__init__.py rename to extract_msg/_rtf/__init__.py diff --git a/extract_msg/_rtf/rtf_tokenizer.py b/extract_msg/_rtf/rtf_tokenizer.py new file mode 100644 index 00000000..f4de587f --- /dev/null +++ b/extract_msg/_rtf/rtf_tokenizer.py @@ -0,0 +1,73 @@ +import copy +import enum +import io + +from typing import List, NamedTuple + + +class RTFTokenizer: + """ + Class designed to take in RTF bytes and split it into individual tokens with + minimal validation of the contents. + + Tokens can be iterated by iteraating over the instance directly or can be + accessed directly from the `tokens` instance variable. + """ + + def __init__(self, data : bytes = None): + self.tokens : List[Token] = [] + # Feed the data to our parser if provided. Feeding will clear all of the + # existing data. + if data: + self.feed(data) + + def __iter__(self): + return self.tokens.__iter__() + + def feed(self, data : bytes) -> None: + """ + Reads in the bytes and sets the tokens list to the contents after + tokenizing. If tokenizing fails, the current tokens list will not be + changed. + + Direct references to the previous tokens list will only point to the + previous and not to the current one. + + :raises TypeError: The data is not recognized as RTF. + """ + reader = io.BytesIO(data) + # This tokenizer *only* breaks things up. It does *not* care about + # groups and stuff, as that is for a parser to deal with. All we do is + # track the current backslash state and token state. We also simply + # check that the first token is "\rtf1" preceeded by a group start, and + # that is it. + start = reader.read(6) + if start != b'{\\rtf1': + raise TypeError('Data') + + tokens = [Token(b'{'), Token(b'\rtf')] + + + +class TokenType(enum.Enum): + GROUP_START = 0 + GROUP_END = 1 + CONTROL = 2 + SYMBOL = 3 + DESTINATION = 4 + + + +class Token(NamedTuple): + # The raw bytes for the token, used to recreate the document. + raw : bytes + # The type of the token. + type : TokenType + ## The following are optional as they only apply for certain types of tokens. + # The name of the token, if it is a control or destination. + name : Optional[bytes] = None + # The parameter of the token, if it has one. If the token is a `\'hh` token, + # this will be the decimal equivelent of the hex value. + parameter : Optional[int] = None + # The symbol the token represents, if it is a symbol. + symbol : Optional[str] = None diff --git a/extract_msg/rtf/rtf_tokenizer.py b/extract_msg/rtf/rtf_tokenizer.py deleted file mode 100644 index a9d2a3fc..00000000 --- a/extract_msg/rtf/rtf_tokenizer.py +++ /dev/null @@ -1,10 +0,0 @@ -class RTFTokenizer: - def __init__(self, data : bytes = None): - - # Feed the data to our parser if provided. Feeding will clear all of the - # existing data. - if data: - self.feed(data) - - def feed(self, data : bytes) -> None: - pass diff --git a/setup.py b/setup.py index 2c1add09..a4ca2a9d 100644 --- a/setup.py +++ b/setup.py @@ -46,5 +46,5 @@ }, include_package_data=True, install_requires=dependencies, - python_requires='>=3.6', + python_requires='>=3.8', ) diff --git a/temp-changelog-rtf.md b/temp-changelog-rtf.md new file mode 100644 index 00000000..0f805fe0 --- /dev/null +++ b/temp-changelog-rtf.md @@ -0,0 +1,2 @@ +**v??.??.??** +* Updated minimum Python version to 3.8 as 3.6 has reached end of support and 3.7 will reach end of support within the year. From e6efdc8dcf1a0bc2ecb52d51ec58851867f1b185 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 7 Feb 2023 14:06:08 -0800 Subject: [PATCH 04/24] More progress on tokenizer --- extract_msg/_rtf/rtf_tokenizer.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/extract_msg/_rtf/rtf_tokenizer.py b/extract_msg/_rtf/rtf_tokenizer.py index f4de587f..0ca5fb82 100644 --- a/extract_msg/_rtf/rtf_tokenizer.py +++ b/extract_msg/_rtf/rtf_tokenizer.py @@ -43,9 +43,17 @@ def feed(self, data : bytes) -> None: # that is it. start = reader.read(6) if start != b'{\\rtf1': - raise TypeError('Data') + raise TypeError('Data does not start with "{\\rtf1".') - tokens = [Token(b'{'), Token(b'\rtf')] + tokens = [ + Token(b'{', TokenType.GROUP_START), + Token(b'\rtf1', TokenType.CONTROL, b'rtf', 1), + ] + nextChar = reader.read(1) + + # If the next character is a space, ignore it. + if nextChar != ' ': + reader.seek(reader.tell() - 1) From 3d8a300364202fc3f5ec31bb034de10f6c0cd173 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sun, 12 Feb 2023 15:08:57 -0800 Subject: [PATCH 05/24] Progress on one form of RTF Tokenizer. Considering another --- extract_msg/_rtf/rtf_tokenizer.py | 107 ++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/extract_msg/_rtf/rtf_tokenizer.py b/extract_msg/_rtf/rtf_tokenizer.py index 0ca5fb82..a4cc0215 100644 --- a/extract_msg/_rtf/rtf_tokenizer.py +++ b/extract_msg/_rtf/rtf_tokenizer.py @@ -24,6 +24,20 @@ def __init__(self, data : bytes = None): def __iter__(self): return self.tokens.__iter__() + def __handleTag(self, tag : bytes, param : bytes) -> Token: + """ + Handles converting an RTF tag into a Token. + """ + if tag[1] == '*': + # Handle custom destination. We also need to handle bad destination + # data. TODO + pass + elif tag[1] in self.__knownSymbols: + pass + + # TODO + pass + def feed(self, data : bytes) -> None: """ Reads in the bytes and sets the tokens list to the contents after @@ -34,6 +48,7 @@ def feed(self, data : bytes) -> None: previous and not to the current one. :raises TypeError: The data is not recognized as RTF. + :raises ValueError: An issue with basic parsing occured. """ reader = io.BytesIO(data) # This tokenizer *only* breaks things up. It does *not* care about @@ -55,6 +70,97 @@ def feed(self, data : bytes) -> None: if nextChar != ' ': reader.seek(reader.tell() - 1) + # Setup the loop variables. + lastCharacter = b'' + inTag = False + param = b'' + raw = b'' + # Tracking for if we are handling a `\'HH` tag. + isHex = False + + while (currentChar := reader.read()) != b'': + if currentChar == b' ' and inTag: + # End the tag and drop the space. + tokens.append(self.__handleTag(raw, param)) + raw = b'' + param = b'' + inTag = False + elif currentChar in (b'{', b'}'): + # Brackets are second fastest to parse, so do them next. + if inTag: + if raw == b'\\': + # If we only have the backslash, this is a symbol. + tokens.append(self.__handleTag(b'\\' + currentChar, b'')) + raw = b'' + param = b'' + inTag = False + else: + # We already have a currentTag, so we need to push it + # then push a group. + tokens.append(self.__handleTag(raw, param)) + raw = b'' + param = b'' + inTag = False + if currentChar = b'{': + tokens.append(Token(currentChar, TokenType.GROUP_START)) + else: + tokens.append(Token(currentChar, TokenType.GROUP_END)) + else: + if raw: + tokens.append(Token(raw, TokenType.TEXT)) + raw = b'' + if currentChar = b'{': + tokens.append(Token(currentChar, TokenType.GROUP_START)) + else: + tokens.append(Token(currentChar, TokenType.GROUP_END)) + elif currentChar == b'*' and inTag: + if len(raw) == 1: + raw += b'*' + else: + # End the current tag and start new text. + tokens.append(self.__handleTag(raw, param)) + raw = b'*' + param = b'' + inTag = False + elif currentChar == b'\\': + if inTag: + if lastCharacter == b'\\': + # If the current character is a backslash and we are in + # a tag, then it is a backslash symbol and we need to + # push it to the list. + tokens.append(self.__handleTag(raw + b'\\', param)) + raw = b'' + param = b'' + elif lastCharacter == b'*': + # This is a custom destination, but we aren't handling + # that in this section, so just add to the current tag. + raw += currentChar + else: + # We are starting a new tag. + tokens.append(self.__handleTag(raw, param)) + raw = b'\\' + param = b'' + else: + # If we were not already in a tag, check if we have text to + # push, and if we do we push it. After that, start a tag. + if raw: + tokens.append() + raw += b'\\' + inTag = True + elif currentChar == b'\'' and inTag: + # If the current character is an apostrophe and we are in a tag, + # check if it is the first character after the backslash. If it + # is, then we are handling hex character, otherwise we are + + elif currentChar.isalpha(): + pass + + + lastCharacter = currentChar + + # Since we are done, set the tokens list to the new one. + self.tokens = tokens + class TokenType(enum.Enum): @@ -63,6 +169,7 @@ class TokenType(enum.Enum): CONTROL = 2 SYMBOL = 3 DESTINATION = 4 + TEXT = 5 From 5420b28f5fae6e017779716a0d7913180495d64b Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sun, 12 Feb 2023 16:52:03 -0800 Subject: [PATCH 06/24] Completely new RTF tokenizing method (I like this one much more) --- extract_msg/_rtf/rtf_tokenizer.py | 245 +++++++++++++++--------------- 1 file changed, 125 insertions(+), 120 deletions(-) diff --git a/extract_msg/_rtf/rtf_tokenizer.py b/extract_msg/_rtf/rtf_tokenizer.py index a4cc0215..e7054756 100644 --- a/extract_msg/_rtf/rtf_tokenizer.py +++ b/extract_msg/_rtf/rtf_tokenizer.py @@ -2,7 +2,36 @@ import enum import io -from typing import List, NamedTuple +from typing import List, NamedTuple, Tuple + + +class TokenType(enum.Enum): + GROUP_START = 0 + GROUP_END = 1 + CONTROL = 2 + SYMBOL = 3 + TEXT = 4 + DESTINATION = 5 + IGNORABLE_DESTSINATION = 6 + # This one is special, used for handling the binary data. + BINARY = 7 + + + +class Token(NamedTuple): + # The raw bytes for the token, used to recreate the document. + raw : bytes + # The type of the token. + type : TokenType + ## The following are optional as they only apply for certain types of tokens. + # The name of the token, if it is a control or destination. + name : Optional[bytes] = None + # The parameter of the token, if it has one. If the token is a `\'hh` token, + # this will be the decimal equivelent of the hex value. + parameter : Optional[int] = None + # The symbol the token represents, if it is a symbol. + symbol : Optional[str] = None + class RTFTokenizer: @@ -38,6 +67,72 @@ def __handleTag(self, tag : bytes, param : bytes) -> Token: # TODO pass + def __readControl(self, startChar : bytes, reader : io.BytesIO) -> Tuple[Tuple[Token], bytes]: + """ + Attempts to read the next data as a control, returning as many tokens + as necessary. + """ + # First, read the next character, as it decides how to handle + # everything. + nextChar = reader.read(1) + if nextChar == b'': + raise ValueError('Unexpected end of data.') + elif nextChar.isalpha(): + # Most control symbols would return immediately, but there are two + # exceptions. + startChar += nextChar + if nextChar == b'*': + # This is going to be a custom destination. First, validation. + if len(nextChar := reader.read(1)) != 1: + raise ValueError('Unexpected end of data.') + elif nextChar != b'\\': + raise ValueError(f'Bad custom destination (expected a backslash, got {nextChar}).') + + startChar += nextChar + + # Check the the next char is alpha. + if not (nextChar := reader.read(1)).isalpha(): + raise ValueError(f'Expected alpha character for destination, got {nextChar}.') + + pass + + + elif nextChar == b'\'': + # This is a hex character, so immediately read 2 more bytes. + hexChars = reader.read(2) + if len(hexChars) != 2: + raise ValueError('Unexpected end of data.') + try: + param = int(hexChars, 16) + except ValueError: + context = e.__cause__ or e.__context__ + raise ValueError(f'Hex data was not hexidecimal (got {hexChars}).') from context + return (self.__handleTag(startChar + hexChars, param),), reader.read(1) + else: + # If it is a control symbol, immediately return. + return (self.__handleTag(startChar, b''),), reader.read(1) + + else: + # If is an alphabetical character, so start the handling of a tag. + pass + + # Handling \binN is going to be the hardest to do, so just give it + # to it's entire own function. + + def __readText(self, startChar : bytes, reader : io.BytesIO) -> Tuple[Tuple[Token], bytes]: + """ + Attempts to read the next data as text. + """ + # Text is actually the easiest to read, as we just read until end of + # stream or until a special character. However, a few characters are + # simply dropped during reading. + while (nextChar := reader.read(1)) not in (b'{', b'}', b'\\'): + # Certain characters are simply dropped. + if nextChar not in (b'\r', b'\n'): + startChar += nextChar + + return (Token(startChar, TokenType.Text),), nextChar + def feed(self, data : bytes) -> None: """ Reads in the bytes and sets the tokens list to the contents after @@ -67,122 +162,32 @@ def feed(self, data : bytes) -> None: nextChar = reader.read(1) # If the next character is a space, ignore it. - if nextChar != ' ': - reader.seek(reader.tell() - 1) - - # Setup the loop variables. - lastCharacter = b'' - inTag = False - param = b'' - raw = b'' - # Tracking for if we are handling a `\'HH` tag. - isHex = False - - while (currentChar := reader.read()) != b'': - if currentChar == b' ' and inTag: - # End the tag and drop the space. - tokens.append(self.__handleTag(raw, param)) - raw = b'' - param = b'' - inTag = False - elif currentChar in (b'{', b'}'): - # Brackets are second fastest to parse, so do them next. - if inTag: - if raw == b'\\': - # If we only have the backslash, this is a symbol. - tokens.append(self.__handleTag(b'\\' + currentChar, b'')) - raw = b'' - param = b'' - inTag = False - else: - # We already have a currentTag, so we need to push it - # then push a group. - tokens.append(self.__handleTag(raw, param)) - raw = b'' - param = b'' - inTag = False - if currentChar = b'{': - tokens.append(Token(currentChar, TokenType.GROUP_START)) - else: - tokens.append(Token(currentChar, TokenType.GROUP_END)) - else: - if raw: - tokens.append(Token(raw, TokenType.TEXT)) - raw = b'' - if currentChar = b'{': - tokens.append(Token(currentChar, TokenType.GROUP_START)) - else: - tokens.append(Token(currentChar, TokenType.GROUP_END)) - elif currentChar == b'*' and inTag: - if len(raw) == 1: - raw += b'*' - else: - # End the current tag and start new text. - tokens.append(self.__handleTag(raw, param)) - raw = b'*' - param = b'' - inTag = False - elif currentChar == b'\\': - if inTag: - if lastCharacter == b'\\': - # If the current character is a backslash and we are in - # a tag, then it is a backslash symbol and we need to - # push it to the list. - tokens.append(self.__handleTag(raw + b'\\', param)) - raw = b'' - param = b'' - elif lastCharacter == b'*': - # This is a custom destination, but we aren't handling - # that in this section, so just add to the current tag. - raw += currentChar - else: - # We are starting a new tag. - tokens.append(self.__handleTag(raw, param)) - raw = b'\\' - param = b'' - else: - # If we were not already in a tag, check if we have text to - # push, and if we do we push it. After that, start a tag. - if raw: - tokens.append() - raw += b'\\' - inTag = True - elif currentChar == b'\'' and inTag: - # If the current character is an apostrophe and we are in a tag, - # check if it is the first character after the backslash. If it - # is, then we are handling hex character, otherwise we are - - elif currentChar.isalpha(): - pass - - - lastCharacter = currentChar - - # Since we are done, set the tokens list to the new one. - self.tokens = tokens - - - -class TokenType(enum.Enum): - GROUP_START = 0 - GROUP_END = 1 - CONTROL = 2 - SYMBOL = 3 - DESTINATION = 4 - TEXT = 5 - - - -class Token(NamedTuple): - # The raw bytes for the token, used to recreate the document. - raw : bytes - # The type of the token. - type : TokenType - ## The following are optional as they only apply for certain types of tokens. - # The name of the token, if it is a control or destination. - name : Optional[bytes] = None - # The parameter of the token, if it has one. If the token is a `\'hh` token, - # this will be the decimal equivelent of the hex value. - parameter : Optional[int] = None - # The symbol the token represents, if it is a symbol. - symbol : Optional[str] = None + if nextChar == ' ': + nextChar = reader.read() + + newToken = None + + # At every iteration, so long as there is more data, nextChar should be + # set. As such, use it to determine what kind of data to try to read, + # using the delimeter of that type of data to know what to do next. + while nextChar != b'': + # We should hav exactly one character, the start of the next + # section. Use it to determine what to do. + if nextChar == b'\\': + newTokens, nextChar = self.__readTag(nextChar, reader) + elif nextChar == b'{': + # This will always be a group start, which has nothing left to + # read. + nextChar = reader.read() + newTokens = (Token(b'{', TokenType.GROUP_START),) + elif nextChar == b'}': + # This will always be a group end, which has nothing left to + # read. + nextChar = reader.read() + newTokens = (Token(b'}', TokenType.GROUP_END),) + else: + # Otherwise, it's just text. + newTokens, nextChar = self.__readText(nextChar, reader) + tokens.extend(newTokens) + + self.tokens = tokens From 4e9ffd0c130a2a12fbe6a6da9b9fc69074a65379 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sun, 12 Feb 2023 22:29:22 -0800 Subject: [PATCH 07/24] Progress on tokenizer (testing needed) --- extract_msg/_rtf/rtf_tokenizer.py | 90 ++++++++++++++++++++++--------- 1 file changed, 66 insertions(+), 24 deletions(-) diff --git a/extract_msg/_rtf/rtf_tokenizer.py b/extract_msg/_rtf/rtf_tokenizer.py index e7054756..3ae879d0 100644 --- a/extract_msg/_rtf/rtf_tokenizer.py +++ b/extract_msg/_rtf/rtf_tokenizer.py @@ -2,7 +2,7 @@ import enum import io -from typing import List, NamedTuple, Tuple +from typing import List, NamedTuple, Optional, Tuple class TokenType(enum.Enum): @@ -29,8 +29,6 @@ class Token(NamedTuple): # The parameter of the token, if it has one. If the token is a `\'hh` token, # this will be the decimal equivelent of the hex value. parameter : Optional[int] = None - # The symbol the token represents, if it is a symbol. - symbol : Optional[str] = None @@ -53,19 +51,47 @@ def __init__(self, data : bytes = None): def __iter__(self): return self.tokens.__iter__() - def __handleTag(self, tag : bytes, param : bytes) -> Token: + def __finishTag(self, startText : bytes, reader : io.BytesIO) -> Tuple[bytes, Optional[bytes], Optional[int], Bytes]: """ - Handles converting an RTF tag into a Token. + Finishes reading a tag, returning the needed parameters to make it a + token. The return is a 4 tuple of the raw token bytes, the name field, + the parameter field (as an int), and the next character after the tag. """ - if tag[1] == '*': - # Handle custom destination. We also need to handle bad destination - # data. TODO - pass - elif tag[1] in self.__knownSymbols: - pass + # Very simple rules here. Anything other than a letter and we change + # state. If the next character is a hypen, check if the character after + # is a digit, otherwise return. If it is a digit or that previously + # mentioned next character was a digit, read digits until anything else + # is detected, then return. + name = startText[-1:] + param = b'' + + while (nextChar := reader.read(1)) != b'' and nextChar.isalpha(): + # Read until not alpha. + startText += nextChar + name += nextChar + + # Check what the next character is to decide what to do with it. + if nextChar == b'-': + # We do this as a separate check. + nextNext = reader.read() + if nextNext = b'': + raise ValueError('Unexpected end of data.') + elif nextNext.isdigit(): + startText += nextChar + nextChar = nextNext + + if nextChar.isdigit(): + startText += nextChar + param += nextChar + while (nextChar := reader.read(1)) != b'' and nextChar.isdigit(): + startText += nextChar + param += nextChar + + param = int(param) + else: + param = None - # TODO - pass + return startText, name, param, nextChar def __readControl(self, startChar : bytes, reader : io.BytesIO) -> Tuple[Tuple[Token], bytes]: """ @@ -78,6 +104,21 @@ def __readControl(self, startChar : bytes, reader : io.BytesIO) -> Tuple[Tuple[T if nextChar == b'': raise ValueError('Unexpected end of data.') elif nextChar.isalpha(): + # If is an alphabetical character, so start the handling of a tag. + text, name, param, nextChar = self.__finishTag(startChar + nextChar, reader) + # Important, check if the name is "bin". If it is, handle that + # specially before returning. + if name == b'bin': + if nextChar == b'': + raise ValueError('Unexpected end of data.') + binText = nextChar + reader.read(param - 1) + if len(binText) != param: + raise ValueError('Unexpected end of data.') + return (Token(text, TokenType.CONTROL, name, param), Token(binText, TokenType.BINARY)), nextChar + elif name in self.__KNOWN_DESTINATIONS: + return (Token(text, TokenType.DESTINATION, name, param),), nextChar + + else: # Most control symbols would return immediately, but there are two # exceptions. startChar += nextChar @@ -94,7 +135,11 @@ def __readControl(self, startChar : bytes, reader : io.BytesIO) -> Tuple[Tuple[T if not (nextChar := reader.read(1)).isalpha(): raise ValueError(f'Expected alpha character for destination, got {nextChar}.') - pass + startChar += nextChar + + # Call the function to read until a clear end of tag. + text, name, param, nextChar = self.__finishTag(startChar, reader) + return (Token(text, TokenType.IGNORABLE_DESTSINATION, name, param),), nextChar elif nextChar == b'\'': @@ -107,31 +152,28 @@ def __readControl(self, startChar : bytes, reader : io.BytesIO) -> Tuple[Tuple[T except ValueError: context = e.__cause__ or e.__context__ raise ValueError(f'Hex data was not hexidecimal (got {hexChars}).') from context - return (self.__handleTag(startChar + hexChars, param),), reader.read(1) + return (Token(startChar + hexChars, TokenType.SYMBOL, None, param),), reader.read(1) else: # If it is a control symbol, immediately return. return (self.__handleTag(startChar, b''),), reader.read(1) - else: - # If is an alphabetical character, so start the handling of a tag. - pass - - # Handling \binN is going to be the hardest to do, so just give it - # to it's entire own function. - def __readText(self, startChar : bytes, reader : io.BytesIO) -> Tuple[Tuple[Token], bytes]: """ Attempts to read the next data as text. """ + chars = [startChar] # Text is actually the easiest to read, as we just read until end of # stream or until a special character. However, a few characters are # simply dropped during reading. while (nextChar := reader.read(1)) not in (b'{', b'}', b'\\'): # Certain characters are simply dropped. if nextChar not in (b'\r', b'\n'): - startChar += nextChar + chars.append(nextChar) + + # Now, we actually are reading the text as *individual tokens*, so we + # need to - return (Token(startChar, TokenType.Text),), nextChar + return tuple(Token(startChar, TokenType.Text) for x in chars), nextChar def feed(self, data : bytes) -> None: """ From 4ab595ecc380561444c1987be6a67c34ff1dc3b1 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sun, 12 Feb 2023 23:48:13 -0800 Subject: [PATCH 08/24] Update to fix some issues, then tokenizer should be good --- extract_msg/_rtf/rtf_tokenizer.py | 74 ++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 12 deletions(-) diff --git a/extract_msg/_rtf/rtf_tokenizer.py b/extract_msg/_rtf/rtf_tokenizer.py index 3ae879d0..a53ffa57 100644 --- a/extract_msg/_rtf/rtf_tokenizer.py +++ b/extract_msg/_rtf/rtf_tokenizer.py @@ -41,6 +41,46 @@ class RTFTokenizer: accessed directly from the `tokens` instance variable. """ + __KNOWN_DESTINATIONS = ( + b'aftncn', + b'aftnsep', + b'aftnsepc', + b'annotation', + b'author', + b'buptim', + b'category', + b'colortbl', + b'comment', + b'company', + b'creatim', + b'doccomm', + b'dptxbxtext', + b'factoidname', + b'fonttbl', + b'footer', + b'footerf', + b'footerl', + b'footerr', + b'ftncn', + b'ftnsep', + b'ftnsepc', + b'header', + b'headerf', + b'headerl', + b'headerr', + b'hlinkbase', + b'keywords', + b'manager', + b'operator', + b'pict', + b'printim', + b'private', + b'revtim', + b'stylesheet', + b'subject', + b'title', + ) + def __init__(self, data : bytes = None): self.tokens : List[Token] = [] # Feed the data to our parser if provided. Feeding will clear all of the @@ -51,7 +91,7 @@ def __init__(self, data : bytes = None): def __iter__(self): return self.tokens.__iter__() - def __finishTag(self, startText : bytes, reader : io.BytesIO) -> Tuple[bytes, Optional[bytes], Optional[int], Bytes]: + def __finishTag(self, startText : bytes, reader : io.BytesIO) -> Tuple[bytes, Optional[bytes], Optional[int], bytes]: """ Finishes reading a tag, returning the needed parameters to make it a token. The return is a 4 tuple of the raw token bytes, the name field, @@ -73,8 +113,8 @@ def __finishTag(self, startText : bytes, reader : io.BytesIO) -> Tuple[bytes, Op # Check what the next character is to decide what to do with it. if nextChar == b'-': # We do this as a separate check. - nextNext = reader.read() - if nextNext = b'': + nextNext = reader.read(1) + if nextNext == b'': raise ValueError('Unexpected end of data.') elif nextNext.isdigit(): startText += nextChar @@ -91,6 +131,11 @@ def __finishTag(self, startText : bytes, reader : io.BytesIO) -> Tuple[bytes, Op else: param = None + # Finally, check if the next char is a space, and if it is, read one + # more char to replace it. + if nextChar == b' ': + nextChar = reader.read(1) + return startText, name, param, nextChar def __readControl(self, startChar : bytes, reader : io.BytesIO) -> Tuple[Tuple[Token], bytes]: @@ -118,6 +163,7 @@ def __readControl(self, startChar : bytes, reader : io.BytesIO) -> Tuple[Tuple[T elif name in self.__KNOWN_DESTINATIONS: return (Token(text, TokenType.DESTINATION, name, param),), nextChar + return (Token(text, TokenType.CONTROL, name, param),), nextChar else: # Most control symbols would return immediately, but there are two # exceptions. @@ -140,8 +186,6 @@ def __readControl(self, startChar : bytes, reader : io.BytesIO) -> Tuple[Tuple[T # Call the function to read until a clear end of tag. text, name, param, nextChar = self.__finishTag(startChar, reader) return (Token(text, TokenType.IGNORABLE_DESTSINATION, name, param),), nextChar - - elif nextChar == b'\'': # This is a hex character, so immediately read 2 more bytes. hexChars = reader.read(2) @@ -165,7 +209,7 @@ def __readText(self, startChar : bytes, reader : io.BytesIO) -> Tuple[Tuple[Toke # Text is actually the easiest to read, as we just read until end of # stream or until a special character. However, a few characters are # simply dropped during reading. - while (nextChar := reader.read(1)) not in (b'{', b'}', b'\\'): + while (nextChar := reader.read(1)) != b'' and nextChar not in (b'{', b'}', b'\\'): # Certain characters are simply dropped. if nextChar not in (b'\r', b'\n'): chars.append(nextChar) @@ -173,7 +217,7 @@ def __readText(self, startChar : bytes, reader : io.BytesIO) -> Tuple[Tuple[Toke # Now, we actually are reading the text as *individual tokens*, so we # need to - return tuple(Token(startChar, TokenType.Text) for x in chars), nextChar + return tuple(Token(x, TokenType.TEXT) for x in chars), nextChar def feed(self, data : bytes) -> None: """ @@ -205,7 +249,7 @@ def feed(self, data : bytes) -> None: # If the next character is a space, ignore it. if nextChar == ' ': - nextChar = reader.read() + nextChar = reader.read(1) newToken = None @@ -213,23 +257,29 @@ def feed(self, data : bytes) -> None: # set. As such, use it to determine what kind of data to try to read, # using the delimeter of that type of data to know what to do next. while nextChar != b'': - # We should hav exactly one character, the start of the next + # We should have exactly one character, the start of the next # section. Use it to determine what to do. + if nextChar in (b'\r', b'\n'): + # Just read the next character and start the loop over. + nextChar = reader.read(1) + continue + if nextChar == b'\\': - newTokens, nextChar = self.__readTag(nextChar, reader) + newTokens, nextChar = self.__readControl(nextChar, reader) elif nextChar == b'{': # This will always be a group start, which has nothing left to # read. - nextChar = reader.read() + nextChar = reader.read(1) newTokens = (Token(b'{', TokenType.GROUP_START),) elif nextChar == b'}': # This will always be a group end, which has nothing left to # read. - nextChar = reader.read() + nextChar = reader.read(1) newTokens = (Token(b'}', TokenType.GROUP_END),) else: # Otherwise, it's just text. newTokens, nextChar = self.__readText(nextChar, reader) + [print(x) for x in newTokens] tokens.extend(newTokens) self.tokens = tokens From bb51cd7d4f5abdfb99b13e6ef292406864c963f3 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Mon, 13 Feb 2023 14:59:12 -0800 Subject: [PATCH 09/24] Removed debug code --- extract_msg/_rtf/rtf_tokenizer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/extract_msg/_rtf/rtf_tokenizer.py b/extract_msg/_rtf/rtf_tokenizer.py index a53ffa57..03e1d56a 100644 --- a/extract_msg/_rtf/rtf_tokenizer.py +++ b/extract_msg/_rtf/rtf_tokenizer.py @@ -279,7 +279,6 @@ def feed(self, data : bytes) -> None: else: # Otherwise, it's just text. newTokens, nextChar = self.__readText(nextChar, reader) - [print(x) for x in newTokens] tokens.extend(newTokens) self.tokens = tokens From 14e18396c83e04c27e669cce2542072aee57b90e Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 22 Feb 2023 14:20:51 -0800 Subject: [PATCH 10/24] Fix readme --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index f26cf767..d48bdf61 100644 --- a/README.rst +++ b/README.rst @@ -38,7 +38,7 @@ attachments. The script uses Philippe Lagadec's Python module that reads Microsoft OLE2 files (also called Structured Storage, Compound File Binary Format or Compound Document File Format). This is the underlying format of -Outlook's .msg files. This library currently supports Python 3.7 and above. +Outlook's .msg files. This library currently supports Python 3.8 and above. The script was originally built using Peter Fiskerstrand's documentation of the .msg format. Redemption's discussion of the different property types used within From dc71297375145e373b681f1bba85daa8903bd44e Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 22 Feb 2023 21:08:44 -0800 Subject: [PATCH 11/24] Started further RTF classes --- extract_msg/_rtf/rtf_converter.py | 0 extract_msg/_rtf/rtf_parser.py | 22 ++++++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 extract_msg/_rtf/rtf_converter.py create mode 100644 extract_msg/_rtf/rtf_parser.py diff --git a/extract_msg/_rtf/rtf_converter.py b/extract_msg/_rtf/rtf_converter.py new file mode 100644 index 00000000..e69de29b diff --git a/extract_msg/_rtf/rtf_parser.py b/extract_msg/_rtf/rtf_parser.py new file mode 100644 index 00000000..ea60e956 --- /dev/null +++ b/extract_msg/_rtf/rtf_parser.py @@ -0,0 +1,22 @@ +from typing import Optional + +from . import RTFTokenizer + + +class RTFParser(RTFTokenizer): + """ + Extension of the RTFTokenizer class which handles advanced parsing including + grouping, determining if the data is in the correct order, etc. + """ + + def __init__(self, data : Optional[bytes] = None): + super().__init__(data) + + def feed(self, data : bytes) -> None: + # Backup our tokens, since we need to be more careful with them. + oldTokens = self.tokens + + # First feed the data to the superclass. + super().feed(data) + + # TODO From 93dc853bdaba99a8851a00731acb5234b3724a27 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Mon, 27 Feb 2023 21:54:05 -0800 Subject: [PATCH 12/24] Removed accidental double line --- extract_msg/message_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extract_msg/message_base.py b/extract_msg/message_base.py index 841654d4..01e9b162 100644 --- a/extract_msg/message_base.py +++ b/extract_msg/message_base.py @@ -16,7 +16,7 @@ import RTFDE from email.parser import Parser as EmailParser -from typing import Callable, Dict, Optional, Tuple, Union +from typing import Callable, Dict, List, Optional, Tuple, Union from . import constants from .enums import DeencapType, RecipientType @@ -1265,7 +1265,7 @@ def recipientSeparator(self) -> str: return self.__recipientSeparator @property - def recipients(self) -> list: + def recipients(self) -> List[Recipient]: """ Returns a list of all recipients. """ From d76914a9bed1ef5a0b23d3185bdc492f70a14155 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Mon, 27 Feb 2023 21:54:58 -0800 Subject: [PATCH 13/24] See last commit --- extract_msg/message_base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/extract_msg/message_base.py b/extract_msg/message_base.py index 01e9b162..bdc4bdf6 100644 --- a/extract_msg/message_base.py +++ b/extract_msg/message_base.py @@ -393,7 +393,6 @@ def getSavePdfBody(self, **kwargs) -> bytes: # Log the arguments. logger.info(f'Converting to PDF with the following arguments: {processArgs}') - # Get the html body *before* calling Popen. htmlBody = self.getSaveHtmlBody(**kwargs) From f377328b1df975fc5812ee18a1d3398fc2aa4124 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 8 Mar 2023 15:21:13 -0800 Subject: [PATCH 14/24] Started refactoring RTF stuff into functions instead of classes --- extract_msg/_rtf/__init__.py | 7 + extract_msg/_rtf/rtf_converter.py | 0 extract_msg/_rtf/rtf_parser.py | 22 --- extract_msg/_rtf/rtf_tokenizer.py | 284 ------------------------------ extract_msg/_rtf/token.py | 29 +++ extract_msg/_rtf/tokenize_rtf.py | 244 +++++++++++++++++++++++++ 6 files changed, 280 insertions(+), 306 deletions(-) delete mode 100644 extract_msg/_rtf/rtf_converter.py delete mode 100644 extract_msg/_rtf/rtf_parser.py delete mode 100644 extract_msg/_rtf/rtf_tokenizer.py create mode 100644 extract_msg/_rtf/token.py create mode 100644 extract_msg/_rtf/tokenize_rtf.py diff --git a/extract_msg/_rtf/__init__.py b/extract_msg/_rtf/__init__.py index e69de29b..838ca727 100644 --- a/extract_msg/_rtf/__init__.py +++ b/extract_msg/_rtf/__init__.py @@ -0,0 +1,7 @@ +""" +Module that provides access to functions to help manage RTF data. +""" + + +from .token import Token, TokenType +from .tokenize_rtf import tokenizeRTF diff --git a/extract_msg/_rtf/rtf_converter.py b/extract_msg/_rtf/rtf_converter.py deleted file mode 100644 index e69de29b..00000000 diff --git a/extract_msg/_rtf/rtf_parser.py b/extract_msg/_rtf/rtf_parser.py deleted file mode 100644 index ea60e956..00000000 --- a/extract_msg/_rtf/rtf_parser.py +++ /dev/null @@ -1,22 +0,0 @@ -from typing import Optional - -from . import RTFTokenizer - - -class RTFParser(RTFTokenizer): - """ - Extension of the RTFTokenizer class which handles advanced parsing including - grouping, determining if the data is in the correct order, etc. - """ - - def __init__(self, data : Optional[bytes] = None): - super().__init__(data) - - def feed(self, data : bytes) -> None: - # Backup our tokens, since we need to be more careful with them. - oldTokens = self.tokens - - # First feed the data to the superclass. - super().feed(data) - - # TODO diff --git a/extract_msg/_rtf/rtf_tokenizer.py b/extract_msg/_rtf/rtf_tokenizer.py deleted file mode 100644 index 03e1d56a..00000000 --- a/extract_msg/_rtf/rtf_tokenizer.py +++ /dev/null @@ -1,284 +0,0 @@ -import copy -import enum -import io - -from typing import List, NamedTuple, Optional, Tuple - - -class TokenType(enum.Enum): - GROUP_START = 0 - GROUP_END = 1 - CONTROL = 2 - SYMBOL = 3 - TEXT = 4 - DESTINATION = 5 - IGNORABLE_DESTSINATION = 6 - # This one is special, used for handling the binary data. - BINARY = 7 - - - -class Token(NamedTuple): - # The raw bytes for the token, used to recreate the document. - raw : bytes - # The type of the token. - type : TokenType - ## The following are optional as they only apply for certain types of tokens. - # The name of the token, if it is a control or destination. - name : Optional[bytes] = None - # The parameter of the token, if it has one. If the token is a `\'hh` token, - # this will be the decimal equivelent of the hex value. - parameter : Optional[int] = None - - - -class RTFTokenizer: - """ - Class designed to take in RTF bytes and split it into individual tokens with - minimal validation of the contents. - - Tokens can be iterated by iteraating over the instance directly or can be - accessed directly from the `tokens` instance variable. - """ - - __KNOWN_DESTINATIONS = ( - b'aftncn', - b'aftnsep', - b'aftnsepc', - b'annotation', - b'author', - b'buptim', - b'category', - b'colortbl', - b'comment', - b'company', - b'creatim', - b'doccomm', - b'dptxbxtext', - b'factoidname', - b'fonttbl', - b'footer', - b'footerf', - b'footerl', - b'footerr', - b'ftncn', - b'ftnsep', - b'ftnsepc', - b'header', - b'headerf', - b'headerl', - b'headerr', - b'hlinkbase', - b'keywords', - b'manager', - b'operator', - b'pict', - b'printim', - b'private', - b'revtim', - b'stylesheet', - b'subject', - b'title', - ) - - def __init__(self, data : bytes = None): - self.tokens : List[Token] = [] - # Feed the data to our parser if provided. Feeding will clear all of the - # existing data. - if data: - self.feed(data) - - def __iter__(self): - return self.tokens.__iter__() - - def __finishTag(self, startText : bytes, reader : io.BytesIO) -> Tuple[bytes, Optional[bytes], Optional[int], bytes]: - """ - Finishes reading a tag, returning the needed parameters to make it a - token. The return is a 4 tuple of the raw token bytes, the name field, - the parameter field (as an int), and the next character after the tag. - """ - # Very simple rules here. Anything other than a letter and we change - # state. If the next character is a hypen, check if the character after - # is a digit, otherwise return. If it is a digit or that previously - # mentioned next character was a digit, read digits until anything else - # is detected, then return. - name = startText[-1:] - param = b'' - - while (nextChar := reader.read(1)) != b'' and nextChar.isalpha(): - # Read until not alpha. - startText += nextChar - name += nextChar - - # Check what the next character is to decide what to do with it. - if nextChar == b'-': - # We do this as a separate check. - nextNext = reader.read(1) - if nextNext == b'': - raise ValueError('Unexpected end of data.') - elif nextNext.isdigit(): - startText += nextChar - nextChar = nextNext - - if nextChar.isdigit(): - startText += nextChar - param += nextChar - while (nextChar := reader.read(1)) != b'' and nextChar.isdigit(): - startText += nextChar - param += nextChar - - param = int(param) - else: - param = None - - # Finally, check if the next char is a space, and if it is, read one - # more char to replace it. - if nextChar == b' ': - nextChar = reader.read(1) - - return startText, name, param, nextChar - - def __readControl(self, startChar : bytes, reader : io.BytesIO) -> Tuple[Tuple[Token], bytes]: - """ - Attempts to read the next data as a control, returning as many tokens - as necessary. - """ - # First, read the next character, as it decides how to handle - # everything. - nextChar = reader.read(1) - if nextChar == b'': - raise ValueError('Unexpected end of data.') - elif nextChar.isalpha(): - # If is an alphabetical character, so start the handling of a tag. - text, name, param, nextChar = self.__finishTag(startChar + nextChar, reader) - # Important, check if the name is "bin". If it is, handle that - # specially before returning. - if name == b'bin': - if nextChar == b'': - raise ValueError('Unexpected end of data.') - binText = nextChar + reader.read(param - 1) - if len(binText) != param: - raise ValueError('Unexpected end of data.') - return (Token(text, TokenType.CONTROL, name, param), Token(binText, TokenType.BINARY)), nextChar - elif name in self.__KNOWN_DESTINATIONS: - return (Token(text, TokenType.DESTINATION, name, param),), nextChar - - return (Token(text, TokenType.CONTROL, name, param),), nextChar - else: - # Most control symbols would return immediately, but there are two - # exceptions. - startChar += nextChar - if nextChar == b'*': - # This is going to be a custom destination. First, validation. - if len(nextChar := reader.read(1)) != 1: - raise ValueError('Unexpected end of data.') - elif nextChar != b'\\': - raise ValueError(f'Bad custom destination (expected a backslash, got {nextChar}).') - - startChar += nextChar - - # Check the the next char is alpha. - if not (nextChar := reader.read(1)).isalpha(): - raise ValueError(f'Expected alpha character for destination, got {nextChar}.') - - startChar += nextChar - - # Call the function to read until a clear end of tag. - text, name, param, nextChar = self.__finishTag(startChar, reader) - return (Token(text, TokenType.IGNORABLE_DESTSINATION, name, param),), nextChar - elif nextChar == b'\'': - # This is a hex character, so immediately read 2 more bytes. - hexChars = reader.read(2) - if len(hexChars) != 2: - raise ValueError('Unexpected end of data.') - try: - param = int(hexChars, 16) - except ValueError: - context = e.__cause__ or e.__context__ - raise ValueError(f'Hex data was not hexidecimal (got {hexChars}).') from context - return (Token(startChar + hexChars, TokenType.SYMBOL, None, param),), reader.read(1) - else: - # If it is a control symbol, immediately return. - return (self.__handleTag(startChar, b''),), reader.read(1) - - def __readText(self, startChar : bytes, reader : io.BytesIO) -> Tuple[Tuple[Token], bytes]: - """ - Attempts to read the next data as text. - """ - chars = [startChar] - # Text is actually the easiest to read, as we just read until end of - # stream or until a special character. However, a few characters are - # simply dropped during reading. - while (nextChar := reader.read(1)) != b'' and nextChar not in (b'{', b'}', b'\\'): - # Certain characters are simply dropped. - if nextChar not in (b'\r', b'\n'): - chars.append(nextChar) - - # Now, we actually are reading the text as *individual tokens*, so we - # need to - - return tuple(Token(x, TokenType.TEXT) for x in chars), nextChar - - def feed(self, data : bytes) -> None: - """ - Reads in the bytes and sets the tokens list to the contents after - tokenizing. If tokenizing fails, the current tokens list will not be - changed. - - Direct references to the previous tokens list will only point to the - previous and not to the current one. - - :raises TypeError: The data is not recognized as RTF. - :raises ValueError: An issue with basic parsing occured. - """ - reader = io.BytesIO(data) - # This tokenizer *only* breaks things up. It does *not* care about - # groups and stuff, as that is for a parser to deal with. All we do is - # track the current backslash state and token state. We also simply - # check that the first token is "\rtf1" preceeded by a group start, and - # that is it. - start = reader.read(6) - if start != b'{\\rtf1': - raise TypeError('Data does not start with "{\\rtf1".') - - tokens = [ - Token(b'{', TokenType.GROUP_START), - Token(b'\rtf1', TokenType.CONTROL, b'rtf', 1), - ] - nextChar = reader.read(1) - - # If the next character is a space, ignore it. - if nextChar == ' ': - nextChar = reader.read(1) - - newToken = None - - # At every iteration, so long as there is more data, nextChar should be - # set. As such, use it to determine what kind of data to try to read, - # using the delimeter of that type of data to know what to do next. - while nextChar != b'': - # We should have exactly one character, the start of the next - # section. Use it to determine what to do. - if nextChar in (b'\r', b'\n'): - # Just read the next character and start the loop over. - nextChar = reader.read(1) - continue - - if nextChar == b'\\': - newTokens, nextChar = self.__readControl(nextChar, reader) - elif nextChar == b'{': - # This will always be a group start, which has nothing left to - # read. - nextChar = reader.read(1) - newTokens = (Token(b'{', TokenType.GROUP_START),) - elif nextChar == b'}': - # This will always be a group end, which has nothing left to - # read. - nextChar = reader.read(1) - newTokens = (Token(b'}', TokenType.GROUP_END),) - else: - # Otherwise, it's just text. - newTokens, nextChar = self.__readText(nextChar, reader) - tokens.extend(newTokens) - - self.tokens = tokens diff --git a/extract_msg/_rtf/token.py b/extract_msg/_rtf/token.py new file mode 100644 index 00000000..779cb96e --- /dev/null +++ b/extract_msg/_rtf/token.py @@ -0,0 +1,29 @@ +import enum + +from typing import NamedTuple, Optional + + +class TokenType(enum.Enum): + GROUP_START = 0 + GROUP_END = 1 + CONTROL = 2 + SYMBOL = 3 + TEXT = 4 + DESTINATION = 5 + IGNORABLE_DESTSINATION = 6 + # This one is special, used for handling the binary data. + BINARY = 7 + + + +class Token(NamedTuple): + # The raw bytes for the token, used to recreate the document. + raw : bytes + # The type of the token. + type : TokenType + ## The following are optional as they only apply for certain types of tokens. + # The name of the token, if it is a control or destination. + name : Optional[bytes] = None + # The parameter of the token, if it has one. If the token is a `\'hh` token, + # this will be the decimal equivelent of the hex value. + parameter : Optional[int] = None diff --git a/extract_msg/_rtf/tokenize_rtf.py b/extract_msg/_rtf/tokenize_rtf.py new file mode 100644 index 00000000..1d9cddd2 --- /dev/null +++ b/extract_msg/_rtf/tokenize_rtf.py @@ -0,0 +1,244 @@ +import copy +import enum +import io + +from typing import List, NamedTuple, Optional, Tuple + +from .token import Token, TokenType + + +_KNOWN_DESTINATIONS = ( + b'aftncn', + b'aftnsep', + b'aftnsepc', + b'annotation', + b'author', + b'buptim', + b'category', + b'colortbl', + b'comment', + b'company', + b'creatim', + b'doccomm', + b'dptxbxtext', + b'factoidname', + b'fonttbl', + b'footer', + b'footerf', + b'footerl', + b'footerr', + b'ftncn', + b'ftnsep', + b'ftnsepc', + b'header', + b'headerf', + b'headerl', + b'headerr', + b'hlinkbase', + b'keywords', + b'manager', + b'operator', + b'pict', + b'printim', + b'private', + b'revtim', + b'stylesheet', + b'subject', + b'title', +) + + +def _finishTag(startText : bytes, reader : io.BytesIO) -> Tuple[bytes, Optional[bytes], Optional[int], bytes]: + """ + Finishes reading a tag, returning the needed parameters to make it a + token. The return is a 4 tuple of the raw token bytes, the name field, + the parameter field (as an int), and the next character after the tag. + """ + # Very simple rules here. Anything other than a letter and we change + # state. If the next character is a hypen, check if the character after + # is a digit, otherwise return. If it is a digit or that previously + # mentioned next character was a digit, read digits until anything else + # is detected, then return. + name = startText[-1:] + param = b'' + + while (nextChar := reader.read(1)) != b'' and nextChar.isalpha(): + # Read until not alpha. + startText += nextChar + name += nextChar + + # Check what the next character is to decide what to do with it. + if nextChar == b'-': + # We do this as a separate check. + nextNext = reader.read(1) + if nextNext == b'': + raise ValueError('Unexpected end of data.') + elif nextNext.isdigit(): + startText += nextChar + nextChar = nextNext + + if nextChar.isdigit(): + startText += nextChar + param += nextChar + while (nextChar := reader.read(1)) != b'' and nextChar.isdigit(): + startText += nextChar + param += nextChar + + param = int(param) + else: + param = None + + # Finally, check if the next char is a space, and if it is, read one + # more char to replace it. + if nextChar == b' ': + nextChar = reader.read(1) + + return startText, name, param, nextChar + + +def _readControl(startChar : bytes, reader : io.BytesIO) -> Tuple[Tuple[Token], bytes]: + """ + Attempts to read the next data as a control, returning as many tokens + as necessary. + """ + # First, read the next character, as it decides how to handle + # everything. + nextChar = reader.read(1) + if nextChar == b'': + raise ValueError('Unexpected end of data.') + elif nextChar.isalpha(): + # If is an alphabetical character, so start the handling of a tag. + text, name, param, nextChar = _finishTag(startChar + nextChar, reader) + # Important, check if the name is "bin". If it is, handle that + # specially before returning. + if name == b'bin': + if nextChar == b'': + raise ValueError('Unexpected end of data.') + binText = nextChar + reader.read(param - 1) + if len(binText) != param: + raise ValueError('Unexpected end of data.') + return (Token(text, TokenType.CONTROL, name, param), Token(binText, TokenType.BINARY)), nextChar + elif name in _KNOWN_DESTINATIONS: + return (Token(text, TokenType.DESTINATION, name, param),), nextChar + + return (Token(text, TokenType.CONTROL, name, param),), nextChar + else: + # Most control symbols would return immediately, but there are two + # exceptions. + startChar += nextChar + if nextChar == b'*': + # This is going to be a custom destination. First, validation. + if len(nextChar := reader.read(1)) != 1: + raise ValueError('Unexpected end of data.') + elif nextChar != b'\\': + raise ValueError(f'Bad custom destination (expected a backslash, got {nextChar}).') + + startChar += nextChar + + # Check the the next char is alpha. + if not (nextChar := reader.read(1)).isalpha(): + raise ValueError(f'Expected alpha character for destination, got {nextChar}.') + + startChar += nextChar + + # Call the function to read until a clear end of tag. + text, name, param, nextChar = _finishTag(startChar, reader) + return (Token(text, TokenType.IGNORABLE_DESTSINATION, name, param),), nextChar + elif nextChar == b'\'': + # This is a hex character, so immediately read 2 more bytes. + hexChars = reader.read(2) + if len(hexChars) != 2: + raise ValueError('Unexpected end of data.') + try: + param = int(hexChars, 16) + except ValueError: + context = e.__cause__ or e.__context__ + raise ValueError(f'Hex data was not hexidecimal (got {hexChars}).') from context + return (Token(startChar + hexChars, TokenType.SYMBOL, None, param),), reader.read(1) + else: + # If it is a control symbol, immediately return. + return (_handleTag(startChar, b''),), reader.read(1) + + +def _readText(startChar : bytes, reader : io.BytesIO) -> Tuple[Tuple[Token], bytes]: + """ + Attempts to read the next data as text. + """ + chars = [startChar] + # Text is actually the easiest to read, as we just read until end of + # stream or until a special character. However, a few characters are + # simply dropped during reading. + while (nextChar := reader.read(1)) != b'' and nextChar not in (b'{', b'}', b'\\'): + # Certain characters are simply dropped. + if nextChar not in (b'\r', b'\n'): + chars.append(nextChar) + + # Now, we actually are reading the text as *individual tokens*, so we + # need to + + return tuple(Token(x, TokenType.TEXT) for x in chars), nextChar + + +def tokenizeRTF(data : bytes) -> None: + """ + Reads in the bytes and sets the tokens list to the contents after + tokenizing. If tokenizing fails, the current tokens list will not be + changed. + + Direct references to the previous tokens list will only point to the + previous and not to the current one. + + :raises TypeError: The data is not recognized as RTF. + :raises ValueError: An issue with basic parsing occured. + """ + reader = io.BytesIO(data) + # This tokenizer *only* breaks things up. It does *not* care about + # groups and stuff, as that is for a parser to deal with. All we do is + # track the current backslash state and token state. We also simply + # check that the first token is "\rtf1" preceeded by a group start, and + # that is it. + start = reader.read(6) + if start != b'{\\rtf1': + raise TypeError('Data does not start with "{\\rtf1".') + + tokens = [ + Token(b'{', TokenType.GROUP_START), + Token(b'\rtf1', TokenType.CONTROL, b'rtf', 1), + ] + nextChar = reader.read(1) + + # If the next character is a space, ignore it. + if nextChar == ' ': + nextChar = reader.read(1) + + newToken = None + + # At every iteration, so long as there is more data, nextChar should be + # set. As such, use it to determine what kind of data to try to read, + # using the delimeter of that type of data to know what to do next. + while nextChar != b'': + # We should have exactly one character, the start of the next + # section. Use it to determine what to do. + if nextChar in (b'\r', b'\n'): + # Just read the next character and start the loop over. + nextChar = reader.read(1) + continue + + if nextChar == b'\\': + newTokens, nextChar = _readControl(nextChar, reader) + elif nextChar == b'{': + # This will always be a group start, which has nothing left to + # read. + nextChar = reader.read(1) + newTokens = (Token(b'{', TokenType.GROUP_START),) + elif nextChar == b'}': + # This will always be a group end, which has nothing left to + # read. + nextChar = reader.read(1) + newTokens = (Token(b'}', TokenType.GROUP_END),) + else: + # Otherwise, it's just text. + newTokens, nextChar = _readText(nextChar, reader) + tokens.extend(newTokens) + + return tokens From 10c3ef599e9d1a0fd47c320c9399baeb6cff7ef8 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 8 Mar 2023 16:13:59 -0800 Subject: [PATCH 15/24] Started working on additional RTF functions, fixed type in enum --- extract_msg/_rtf/create_doc.py | 21 +++++++++++++++ extract_msg/_rtf/inject_rtf.py | 37 +++++++++++++++++++++++++++ extract_msg/_rtf/token.py | 2 +- extract_msg/_rtf/tokenize_rtf.py | 44 ++++++++++++++++++-------------- 4 files changed, 84 insertions(+), 20 deletions(-) create mode 100644 extract_msg/_rtf/create_doc.py create mode 100644 extract_msg/_rtf/inject_rtf.py diff --git a/extract_msg/_rtf/create_doc.py b/extract_msg/_rtf/create_doc.py new file mode 100644 index 00000000..f580a8b1 --- /dev/null +++ b/extract_msg/_rtf/create_doc.py @@ -0,0 +1,21 @@ +from .token import Token, TokenType + + +def createDocument(tokens : List[Tokens]) -> bytes: + """ + Combines the tokenized data into bytes and returns the document. + """ + document = b'' + + # Recombining follows a few very basic rules that are based solely on the + # token type. Since every token has the raw bytes, this is pretty easy. In + # fact, control words are the only place where we put a space, as a space + # anywhere else would be literal, and omitting a space could cause issues on + # some control words. + for token in tokens: + if token.type in (TokenType.CONTROL, TokenType.DESTINATION, TokenType.IGNORABLE_DESTINATION): + document += token.raw + b' ' + else: + document += token.raw + + return document diff --git a/extract_msg/_rtf/inject_rtf.py b/extract_msg/_rtf/inject_rtf.py new file mode 100644 index 00000000..e21c28d3 --- /dev/null +++ b/extract_msg/_rtf/inject_rtf.py @@ -0,0 +1,37 @@ +from .token import Token, TokenType +from .tokenize_rtf import tokenizeRTF + +from typing import List, Optional, Union + + +def injectStartRTF(document : bytes, injectTokens : Union[bytes, List[Token]]) -> List[Token]: + """ + Injects the specified tokens into the document, returning a new copy of the + document as a list of Tokens. Injects the data just before the first + rendered character. + + :param document: The bytes representing the RTF document. + :param injectTokens: The tokens to inject into the document. Can either be + a list of Tokens or bytes to be tokenized. + + :raises TypeError: The data is not recognized as RTF. + :raises ValueError: An issue with basic parsing occured. + """ + return injectStartRTFTokenized(tokenizeRTF(document), injectTokens) + + +def injectStartRTFTokenized(document : List[Token], injectTokens : Union[bytes, List[Token]]) -> List[Token]: + """ + Like :function injectStartRTF:, injects the specified tokens into the + document, returning a reference to the document, except that it accepts a + document in the form of a list of tokens. Injects the data just before the + first rendered character. + + :param document: The list of tokens representing the RTF document. Will only + be modified if the function is successful. + :param injectTokens: The tokens to inject into the document. Can either be + a list of Tokens or bytes to be tokenized. + + :raises TypeError: The data is not recognized as RTF. + :raises ValueError: An issue with basic parsing occured. + """ diff --git a/extract_msg/_rtf/token.py b/extract_msg/_rtf/token.py index 779cb96e..0f9a6b61 100644 --- a/extract_msg/_rtf/token.py +++ b/extract_msg/_rtf/token.py @@ -10,7 +10,7 @@ class TokenType(enum.Enum): SYMBOL = 3 TEXT = 4 DESTINATION = 5 - IGNORABLE_DESTSINATION = 6 + IGNORABLE_DESTINATION = 6 # This one is special, used for handling the binary data. BINARY = 7 diff --git a/extract_msg/_rtf/tokenize_rtf.py b/extract_msg/_rtf/tokenize_rtf.py index 1d9cddd2..cbbc87a1 100644 --- a/extract_msg/_rtf/tokenize_rtf.py +++ b/extract_msg/_rtf/tokenize_rtf.py @@ -143,7 +143,7 @@ def _readControl(startChar : bytes, reader : io.BytesIO) -> Tuple[Tuple[Token], # Call the function to read until a clear end of tag. text, name, param, nextChar = _finishTag(startChar, reader) - return (Token(text, TokenType.IGNORABLE_DESTSINATION, name, param),), nextChar + return (Token(text, TokenType.IGNORABLE_DESTINATION, name, param),), nextChar elif nextChar == b'\'': # This is a hex character, so immediately read 2 more bytes. hexChars = reader.read(2) @@ -179,7 +179,7 @@ def _readText(startChar : bytes, reader : io.BytesIO) -> Tuple[Tuple[Token], byt return tuple(Token(x, TokenType.TEXT) for x in chars), nextChar -def tokenizeRTF(data : bytes) -> None: +def tokenizeRTF(data : bytes, validateStart : bool = True) -> None: """ Reads in the bytes and sets the tokens list to the contents after tokenizing. If tokenizing fails, the current tokens list will not be @@ -188,29 +188,35 @@ def tokenizeRTF(data : bytes) -> None: Direct references to the previous tokens list will only point to the previous and not to the current one. + :param validateStart: If False, does not check the first few tags. Useful + when tokenizing a snippet rather than a document. + :raises TypeError: The data is not recognized as RTF. :raises ValueError: An issue with basic parsing occured. """ reader = io.BytesIO(data) - # This tokenizer *only* breaks things up. It does *not* care about - # groups and stuff, as that is for a parser to deal with. All we do is - # track the current backslash state and token state. We also simply - # check that the first token is "\rtf1" preceeded by a group start, and - # that is it. - start = reader.read(6) - if start != b'{\\rtf1': - raise TypeError('Data does not start with "{\\rtf1".') - - tokens = [ - Token(b'{', TokenType.GROUP_START), - Token(b'\rtf1', TokenType.CONTROL, b'rtf', 1), - ] - nextChar = reader.read(1) - - # If the next character is a space, ignore it. - if nextChar == ' ': + if validateStart: + # This tokenizer *only* breaks things up. It does *not* care about + # groups and stuff, as that is for a parser to deal with. All we do is + # track the current backslash state and token state. We also simply + # check that the first token is "\rtf1" preceeded by a group start, and + # that is it. + start = reader.read(6) + if start != b'{\\rtf1': + raise TypeError('Data does not start with "{\\rtf1".') + + tokens = [ + Token(b'{', TokenType.GROUP_START), + Token(b'\rtf1', TokenType.CONTROL, b'rtf', 1), + ] nextChar = reader.read(1) + # If the next character is a space, ignore it. + if nextChar == ' ': + nextChar = reader.read(1) + else: + tokens = [] + newToken = None # At every iteration, so long as there is more data, nextChar should be From 3ec019b870b4606ad03fa8fbb605fd910b67932c Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Mon, 13 Mar 2023 16:14:16 -0700 Subject: [PATCH 16/24] Progress on injection. Fixed issue with tokenizer --- extract_msg/_rtf/inject_rtf.py | 55 ++++++++++++++++++++++++++++++++ extract_msg/_rtf/tokenize_rtf.py | 3 +- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/extract_msg/_rtf/inject_rtf.py b/extract_msg/_rtf/inject_rtf.py index e21c28d3..be26aac8 100644 --- a/extract_msg/_rtf/inject_rtf.py +++ b/extract_msg/_rtf/inject_rtf.py @@ -1,9 +1,44 @@ +import copy + from .token import Token, TokenType from .tokenize_rtf import tokenizeRTF from typing import List, Optional, Union +# A tuple of destinations (including custom ones) used in the header. +_HEADER_DESTINATIONS = ( + 'fonttbl' +) + +# A tuple of control words that are part of the header and that we simply skip. +_HEADER_SKIPPABLE = ( + # Tag used to specify something. + b'fbidis', + # Character set tags. + b'ansi', + b'mac', + b'pc', + b'pca', + b'ansicpg', + # From. + b'fromtext', + b'fromhtml', + # Def font. + b'deff', + b'adeff', + b'stshfdbch', + b'stshfloch', + b'stshfhich', + b'stshfbi', + # Def lang. + b'deflang', + b'deflangfe', + b'adeflang', + +) + + def injectStartRTF(document : bytes, injectTokens : Union[bytes, List[Token]]) -> List[Token]: """ Injects the specified tokens into the document, returning a new copy of the @@ -35,3 +70,23 @@ def injectStartRTFTokenized(document : List[Token], injectTokens : Union[bytes, :raises TypeError: The data is not recognized as RTF. :raises ValueError: An issue with basic parsing occured. """ + # Get to a list of tokens to inject instead of + if isinstance(injectTokens, bytes): + injectTokens = tokenizeRTF(injectTokens, False) + + # Find the location to insert into. THis is annoyingly complicated, and we + # do this by looking for the parts of the header (if they exist) as we go + # token by token. The moment we confirm we are no longer in the header (and + # we are not in a custom destination that we can simply ignore), we use the + # last recorded spot as the insert point. We don't move that recorded spot + # until we know that what we checked was part of the header. + + currentLocation = 0 + + # First confirm the first two tokens are what we expect. + if len(document < 3): + raise ValueError('RTF documents cannot be less than 3 tokens.') + if document[0].type != TokenType.GROUP_START or : + + + # We have verified the minimal amount. Now, diff --git a/extract_msg/_rtf/tokenize_rtf.py b/extract_msg/_rtf/tokenize_rtf.py index cbbc87a1..6e14b951 100644 --- a/extract_msg/_rtf/tokenize_rtf.py +++ b/extract_msg/_rtf/tokenize_rtf.py @@ -245,6 +245,7 @@ def tokenizeRTF(data : bytes, validateStart : bool = True) -> None: else: # Otherwise, it's just text. newTokens, nextChar = _readText(nextChar, reader) + print(newTokens) tokens.extend(newTokens) - return tokens + return tokens From 273984287c2017f55a34a67b07f992a4fa141560 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Mon, 13 Mar 2023 16:15:34 -0700 Subject: [PATCH 17/24] Removed debug line --- extract_msg/_rtf/tokenize_rtf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extract_msg/_rtf/tokenize_rtf.py b/extract_msg/_rtf/tokenize_rtf.py index 6e14b951..a4424382 100644 --- a/extract_msg/_rtf/tokenize_rtf.py +++ b/extract_msg/_rtf/tokenize_rtf.py @@ -245,7 +245,7 @@ def tokenizeRTF(data : bytes, validateStart : bool = True) -> None: else: # Otherwise, it's just text. newTokens, nextChar = _readText(nextChar, reader) - print(newTokens) + tokens.extend(newTokens) return tokens From 8a06e1aad2ae00f5f9015950946f3d5aa7b98ef3 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 14 Mar 2023 13:14:12 -0700 Subject: [PATCH 18/24] Progress on RTF injection --- extract_msg/_rtf/inject_rtf.py | 41 +++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/extract_msg/_rtf/inject_rtf.py b/extract_msg/_rtf/inject_rtf.py index be26aac8..c3eafc49 100644 --- a/extract_msg/_rtf/inject_rtf.py +++ b/extract_msg/_rtf/inject_rtf.py @@ -3,12 +3,14 @@ from .token import Token, TokenType from .tokenize_rtf import tokenizeRTF -from typing import List, Optional, Union +from typing import List, Iterable, Optional, Union -# A tuple of destinations (including custom ones) used in the header. +# A tuple of destinations used in the header. All ignorable ones are skipped +# anyways, so we don't need to list those here. _HEADER_DESTINATIONS = ( - 'fonttbl' + b'fonttbl', + b'', ) # A tuple of control words that are part of the header and that we simply skip. @@ -39,6 +41,23 @@ ) +def _listInsertMult(dest : List, source : Iterable, index : int = -1): + """ + Inserts into :param dest: all the items in :param source: at the index + specified. :param dest: can be any mutable sequence with :method insert:, + :method __len__:, and :method extend:. + + If :param index: is not specified, the default position is the end of the + list. This is also where things will be inserted if index is greater than or + equal to the size of the list. + """ + if index == -1 or index >= len(dest): + dest.extend(source) + else: + for offset, item in enumerate(source): + dest.insert(index + offset, item) + + def injectStartRTF(document : bytes, injectTokens : Union[bytes, List[Token]]) -> List[Token]: """ Injects the specified tokens into the document, returning a new copy of the @@ -55,7 +74,7 @@ def injectStartRTF(document : bytes, injectTokens : Union[bytes, List[Token]]) - return injectStartRTFTokenized(tokenizeRTF(document), injectTokens) -def injectStartRTFTokenized(document : List[Token], injectTokens : Union[bytes, List[Token]]) -> List[Token]: +def injectStartRTFTokenized(document : List[Token], injectTokens : Union[bytes, Iterable[Token]]) -> List[Token]: """ Like :function injectStartRTF:, injects the specified tokens into the document, returning a reference to the document, except that it accepts a @@ -86,7 +105,17 @@ def injectStartRTFTokenized(document : List[Token], injectTokens : Union[bytes, # First confirm the first two tokens are what we expect. if len(document < 3): raise ValueError('RTF documents cannot be less than 3 tokens.') - if document[0].type != TokenType.GROUP_START or : + if document[0].type != TokenType.GROUP_START or document[1].raw != b'\\rtf1': + raise TypeError('RTF document *must* start with "{\\rtf1".') + + # Confirm that all start groups have an end group somewhere. + if sum(x.type == TokenType.GROUP_START for x in document) != sum(x.type == TokenType.GROUP_END for x in document): + raise ValueError('Number of group opens did not match number of group closes.') + # If the length is exactly 3, insert right before the end and return. + if len(document) == 3: + _listInsertMult(document, injectTokens, 2) + return document - # We have verified the minimal amount. Now, + # We have verified the minimal amount. Now, iterate through the rest to find + # the injection point. From f96d7e6474f9a9baa42e2431eb8909e15ecdf8ac Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 14 Mar 2023 14:32:01 -0700 Subject: [PATCH 19/24] Finsihed rtf injection (check needed) --- extract_msg/_rtf/inject_rtf.py | 49 ++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/extract_msg/_rtf/inject_rtf.py b/extract_msg/_rtf/inject_rtf.py index c3eafc49..2fe078ac 100644 --- a/extract_msg/_rtf/inject_rtf.py +++ b/extract_msg/_rtf/inject_rtf.py @@ -10,7 +10,8 @@ # anyways, so we don't need to list those here. _HEADER_DESTINATIONS = ( b'fonttbl', - b'', + b'colortbl', + b'stylesheet', ) # A tuple of control words that are part of the header and that we simply skip. @@ -37,7 +38,6 @@ b'deflang', b'deflangfe', b'adeflang', - ) @@ -119,3 +119,48 @@ def injectStartRTFTokenized(document : List[Token], injectTokens : Union[bytes, # We have verified the minimal amount. Now, iterate through the rest to find # the injection point. + currentInsertPos = 2 + # Current number of open groups. + groupCount = 1 + # Set to True when looking for if the group is a destination. + checkingDest = False + + for item in document[2:]: + if groupCount == 1: + if item.tokenType is TokenType.GROUP_END: + break + elif item.tokenType is TokenType.GROUP_START: + groupCount += 1 + checkingDest = True + elif item.tokenType is TokenType.CONTROL and item.name in _HEADER_SKIPPABLE: + # If the control is one we know about in the header, skip it. + currentInsertPos += 1 + else: + # Anything else means we are out of the header. + break + elif checkingDest: + if item.tokenType is TokenType.DESTINATION: + # If it is *not* a header destination, just break, otherwise add + # 2 to the insert location and skip the destination. + if item.name in _HEADER_DESTINATIONS: + currentInsertPos += 2 + else: + break + elif item.tokenType is TokenType.IGNORABLE_DESTINATION: + # Add 2 to insert location and skip. + currentInsertPos += 2 + else: + # If it is not an ignorible destination, we are now out of the + # header, so break. + break + checkingDest = False + else: + # Skip the current token, keeping track of groups. + if item.TokenType is TokenType.GROUP_START: + groupCount += 1 + if item.TokenType is TokenType.GROUP_END: + groupCount -= 1 + currentInsertPos += 1 + + _listInsertMult(document, injectTokens, currentInsertPos) + return document From 438f40238b0d8dcddfc7a472ba5cf20c96efbe2e Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 14 Mar 2023 16:19:11 -0700 Subject: [PATCH 20/24] Fix bugs --- extract_msg/_rtf/__init__.py | 2 ++ extract_msg/_rtf/create_doc.py | 5 ++++- extract_msg/_rtf/inject_rtf.py | 14 +++++++------- extract_msg/_rtf/tokenize_rtf.py | 3 ++- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/extract_msg/_rtf/__init__.py b/extract_msg/_rtf/__init__.py index 838ca727..9f05a429 100644 --- a/extract_msg/_rtf/__init__.py +++ b/extract_msg/_rtf/__init__.py @@ -3,5 +3,7 @@ """ +from .create_doc import createDocument +from .inject_rtf import injectStartRTF, injectStartRTFTokenized from .token import Token, TokenType from .tokenize_rtf import tokenizeRTF diff --git a/extract_msg/_rtf/create_doc.py b/extract_msg/_rtf/create_doc.py index f580a8b1..27641a10 100644 --- a/extract_msg/_rtf/create_doc.py +++ b/extract_msg/_rtf/create_doc.py @@ -1,7 +1,10 @@ +from typing import Iterable + from .token import Token, TokenType -def createDocument(tokens : List[Tokens]) -> bytes: + +def createDocument(tokens : Iterable[Token]) -> bytes: """ Combines the tokenized data into bytes and returns the document. """ diff --git a/extract_msg/_rtf/inject_rtf.py b/extract_msg/_rtf/inject_rtf.py index 2fe078ac..23fe85a4 100644 --- a/extract_msg/_rtf/inject_rtf.py +++ b/extract_msg/_rtf/inject_rtf.py @@ -103,9 +103,9 @@ def injectStartRTFTokenized(document : List[Token], injectTokens : Union[bytes, currentLocation = 0 # First confirm the first two tokens are what we expect. - if len(document < 3): + if len(document) < 3: raise ValueError('RTF documents cannot be less than 3 tokens.') - if document[0].type != TokenType.GROUP_START or document[1].raw != b'\\rtf1': + if document[0].type is not TokenType.GROUP_START or document[1].raw != b'\\rtf1': raise TypeError('RTF document *must* start with "{\\rtf1".') # Confirm that all start groups have an end group somewhere. @@ -127,26 +127,26 @@ def injectStartRTFTokenized(document : List[Token], injectTokens : Union[bytes, for item in document[2:]: if groupCount == 1: - if item.tokenType is TokenType.GROUP_END: + if item.type is TokenType.GROUP_END: break - elif item.tokenType is TokenType.GROUP_START: + elif item.type is TokenType.GROUP_START: groupCount += 1 checkingDest = True - elif item.tokenType is TokenType.CONTROL and item.name in _HEADER_SKIPPABLE: + elif item.type is TokenType.CONTROL and item.name in _HEADER_SKIPPABLE: # If the control is one we know about in the header, skip it. currentInsertPos += 1 else: # Anything else means we are out of the header. break elif checkingDest: - if item.tokenType is TokenType.DESTINATION: + if item.type is TokenType.DESTINATION: # If it is *not* a header destination, just break, otherwise add # 2 to the insert location and skip the destination. if item.name in _HEADER_DESTINATIONS: currentInsertPos += 2 else: break - elif item.tokenType is TokenType.IGNORABLE_DESTINATION: + elif item.type is TokenType.IGNORABLE_DESTINATION: # Add 2 to insert location and skip. currentInsertPos += 2 else: diff --git a/extract_msg/_rtf/tokenize_rtf.py b/extract_msg/_rtf/tokenize_rtf.py index a4424382..5c03ba76 100644 --- a/extract_msg/_rtf/tokenize_rtf.py +++ b/extract_msg/_rtf/tokenize_rtf.py @@ -207,7 +207,7 @@ def tokenizeRTF(data : bytes, validateStart : bool = True) -> None: tokens = [ Token(b'{', TokenType.GROUP_START), - Token(b'\rtf1', TokenType.CONTROL, b'rtf', 1), + Token(b'\\rtf1', TokenType.CONTROL, b'rtf', 1), ] nextChar = reader.read(1) @@ -216,6 +216,7 @@ def tokenizeRTF(data : bytes, validateStart : bool = True) -> None: nextChar = reader.read(1) else: tokens = [] + nextChar = reader.read(1) newToken = None From b5064df053da2868e478ee589769966f5334ebaf Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 14 Mar 2023 16:42:59 -0700 Subject: [PATCH 21/24] Testing needed, but #338 should be fixed --- extract_msg/constants.py | 15 ------------ extract_msg/message_base.py | 49 ++++++++++--------------------------- temp-changelog-rtf.md | 2 ++ 3 files changed, 15 insertions(+), 51 deletions(-) diff --git a/extract_msg/constants.py b/extract_msg/constants.py index 697639ba..7a9dc70e 100644 --- a/extract_msg/constants.py +++ b/extract_msg/constants.py @@ -32,21 +32,6 @@ # Regular expression to find the start of the html body in encapsulated RTF. # This is used for one of the pattern types that makes life easy. RE_RTF_ENC_BODY_START_1 = re.compile(br'\{\\\*\\htmltag[0-9]* ?]*>\}') -# Unfortunately, while it would make it easy to find the start of the body in -# terms of the encapsulated HTML, trying to inject directly into this location -# has proven to cause some rendering issues that I'll figure out later. For now -# this is basically the universal start we will try to use. -RE_RTF_BODY_START = re.compile(br'\\lang[0-9]*') -# This is an unrelible one to use as it doesn't have a proper way to verify that -# it will inject in exactly the right place. This is kind of just a "well, let's -# hope this one works" method. -RE_RTF_ENC_BODY_UGLY = re.compile(br']*>[^}]*?\}') -# The following tags are fallbacks that we will try to use, with the higher ones -# having priority. If we can't find any other way, we try these which should -# hopefully always work. -RE_RTF_BODY_FALLBACK_FS = re.compile(br'\\fs[0-9]*[^a-zA-Z]') -RE_RTF_BODY_FALLBACK_F = re.compile(br'\\f[0-9]*[^a-zA-Z]') -RE_RTF_FALLBACK_PLAIN = re.compile(br'\\plain[^a-zA-Z0-9]') # This is used in the workaround for decoding issues in RTFDE. We find `\bin` # sections and try to remove all of them to help with the decoding. RE_BIN = re.compile(br'\\bin([0-9]+) ?') diff --git a/extract_msg/message_base.py b/extract_msg/message_base.py index 841654d4..309129b6 100644 --- a/extract_msg/message_base.py +++ b/extract_msg/message_base.py @@ -19,6 +19,8 @@ from typing import Callable, Dict, Optional, Tuple, Union from . import constants +from ._rtf.create_doc import createDocument +from ._rtf.inject_rtf import injectStartRTF from .enums import DeencapType, RecipientType from .exceptions import ( DataNotFoundError, DeencapMalformedData, DeencapNotEncapsulated, @@ -570,7 +572,7 @@ def replace(bodyMarker): """ Internal function to replace the body tag with itself plus the header. """ - return bodyMarker.group() + injectableHeader.encode('utf-8') + return bodyMarker.group() + injectableHeader # Use the previously defined function to inject the RTF header. We are # trying a few different methods to determine where to place the header. @@ -581,42 +583,17 @@ def replace(bodyMarker): logger.debug('Successfully injected RTF header using first method.') return data - # This second method only applies to encapsulated HTML, so we need to check - # for that first. + # This first method only applies to documents with encapsulated HTML + # that is formatted in a nice way. if isEncapsulatedRtf(self.rtfBody): data = constants.RE_RTF_ENC_BODY_START_1.sub(replace, self.rtfBody, 1) if data != self.rtfBody: - logger.debug('Successfully injected RTF header using second method.') + logger.debug('Successfully injected RTF header using encapsulation method.') return data - # This third method is a lot less reliable, and actually would just - # simply violate the encapuslated html, so for this one we don't even - # try to worry about what the html will think about it. If it injects, - # we swap to basic and then inject again, more worried about it working - # than looking nice inside. - if constants.RE_RTF_ENC_BODY_UGLY.sub(replace, self.rtfBody, 1) != self.rtfBody: - injectableHeader = constants.RTF_PLAIN_INJECTABLE_HEADER - data = constants.RE_RTF_ENC_BODY_UGLY.sub(replace, self.rtfBody, 1) - logger.debug('Successfully injected RTF header using third method.') - return data - - # Severe fallback attempts. - data = constants.RE_RTF_BODY_FALLBACK_FS.sub(replace, self.rtfBody, 1) - if data != self.rtfBody: - logger.debug('Successfully injected RTF header using forth method.') - return data - - data = constants.RE_RTF_BODY_FALLBACK_F.sub(replace, self.rtfBody, 1) - if data != self.rtfBody: - logger.debug('Successfully injected RTF header using fifth method.') - return data - - data = constants.RE_RTF_BODY_FALLBACK_PLAIN.sub(replace, self.rtfBody, 1) - if data != self.rtfBody: - logger.debug('Successfully injected RTF header using sixth method.') - return data - - raise RuntimeError('All injection attempts failed. Please report this to the developer.') + # If the normal encapsulated HTML injection fails or it isn't + # encapsulated, use the internal _rtf module. + return createDocument(injectStartRTF(self.rtfBody, injectableHeader)) def save(self, **kwargs): """ @@ -1306,7 +1283,7 @@ def rtfBody(self) -> Optional[bytes]: return self._rtfBody @property - def rtfEncapInjectableHeader(self) -> str: + def rtfEncapInjectableHeader(self) -> bytes: """ The header that can be formatted and injected into the plain RTF body. """ @@ -1315,10 +1292,10 @@ def rtfEncapInjectableHeader(self) -> str: joinStr = r'{\*\htmltag116
}\htmlrtf \line\htmlrtf0 ' formatter = (lambda name, value : fr'\htmlrtf {{\b\htmlrtf0{{\*\htmltag84 }}{name}: {{\*\htmltag92 }}\htmlrtf \b0\htmlrtf0 {inputToString(rtfSanitizeHtml(value), self.stringEncoding)}\htmlrtf }}\htmlrtf0') - return self.getInjectableHeader(prefix, joinStr, suffix, formatter) + return self.getInjectableHeader(prefix, joinStr, suffix, formatter).encode('utf-8') @property - def rtfPlainInjectableHeader(self) -> str: + def rtfPlainInjectableHeader(self) -> bytes: """ The header that can be formatted and injected into the encapsulated RTF body. @@ -1328,7 +1305,7 @@ def rtfPlainInjectableHeader(self) -> str: joinStr = r'\line' formatter = (lambda name, value : fr'{{\b {name}: \b0 {inputToString(rtfSanitizePlain(value), self.stringEncoding)}}}') - return self.getInjectableHeader(prefix, joinStr, suffix, formatter) + return self.getInjectableHeader(prefix, joinStr, suffix, formatter).encode('utf-8') @property def sender(self) -> Optional[str]: diff --git a/temp-changelog-rtf.md b/temp-changelog-rtf.md index 0f805fe0..108ff6c6 100644 --- a/temp-changelog-rtf.md +++ b/temp-changelog-rtf.md @@ -1,2 +1,4 @@ **v??.??.??** +* [[TeamMsgExtractor #338](https://github.com/TeamMsgExtractor/msg-extractor/issues/338)] Added new code to handle injection of text into the RTF body. For many cases, this will be much more effective as it relies on ensuring that it is in the main group and past the header before injection. It is *not* currently the first choice as it doesn't have proper respect for encapsulated HTML, however it will replace some of the old methods entirely. Solving this issue was done through the use of a few functions and the internal `_rtf` module. This module in it's entirety is considered to be implementation details, and I give no guarantee that it will remain in it's current state even across patch versions. As such, it is not recommended to use it outside of the module. +* Changed `MessageBase.rtfEncapInjectableHeader` and `MessageBase.rtfPlainInjectableHeader` from `str` to `bytes`. They always get encoded anyways, so I don't know why I had them returning as `str`. * Updated minimum Python version to 3.8 as 3.6 has reached end of support and 3.7 will reach end of support within the year. From 292fabdbacd31cb2e012901660434a3a85cb588f Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Fri, 17 Mar 2023 14:54:49 -0700 Subject: [PATCH 22/24] Fix bugs with _rtf module and RTF injection --- extract_msg/_rtf/inject_rtf.py | 4 ++-- extract_msg/_rtf/tokenize_rtf.py | 2 +- extract_msg/constants.py | 2 +- extract_msg/message_base.py | 13 +++---------- 4 files changed, 7 insertions(+), 14 deletions(-) diff --git a/extract_msg/_rtf/inject_rtf.py b/extract_msg/_rtf/inject_rtf.py index 23fe85a4..8d121bfe 100644 --- a/extract_msg/_rtf/inject_rtf.py +++ b/extract_msg/_rtf/inject_rtf.py @@ -156,9 +156,9 @@ def injectStartRTFTokenized(document : List[Token], injectTokens : Union[bytes, checkingDest = False else: # Skip the current token, keeping track of groups. - if item.TokenType is TokenType.GROUP_START: + if item.type is TokenType.GROUP_START: groupCount += 1 - if item.TokenType is TokenType.GROUP_END: + if item.type is TokenType.GROUP_END: groupCount -= 1 currentInsertPos += 1 diff --git a/extract_msg/_rtf/tokenize_rtf.py b/extract_msg/_rtf/tokenize_rtf.py index 5c03ba76..74562802 100644 --- a/extract_msg/_rtf/tokenize_rtf.py +++ b/extract_msg/_rtf/tokenize_rtf.py @@ -157,7 +157,7 @@ def _readControl(startChar : bytes, reader : io.BytesIO) -> Tuple[Tuple[Token], return (Token(startChar + hexChars, TokenType.SYMBOL, None, param),), reader.read(1) else: # If it is a control symbol, immediately return. - return (_handleTag(startChar, b''),), reader.read(1) + return (Token(startChar, TokenType.SYMBOL),), reader.read(1) def _readText(startChar : bytes, reader : io.BytesIO) -> Tuple[Tuple[Token], bytes]: diff --git a/extract_msg/constants.py b/extract_msg/constants.py index 7a9dc70e..7cacdaff 100644 --- a/extract_msg/constants.py +++ b/extract_msg/constants.py @@ -31,7 +31,7 @@ RE_HTML_BODY_START = re.compile(b']*>') # Regular expression to find the start of the html body in encapsulated RTF. # This is used for one of the pattern types that makes life easy. -RE_RTF_ENC_BODY_START_1 = re.compile(br'\{\\\*\\htmltag[0-9]* ?]*>\}') +RE_RTF_ENC_BODY_START = re.compile(br'\{\\\*\\htmltag[0-9]* ?]*>\}') # This is used in the workaround for decoding issues in RTFDE. We find `\bin` # sections and try to remove all of them to help with the decoding. RE_BIN = re.compile(br'\\bin([0-9]+) ?') diff --git a/extract_msg/message_base.py b/extract_msg/message_base.py index 309129b6..a1a9b8ab 100644 --- a/extract_msg/message_base.py +++ b/extract_msg/message_base.py @@ -574,25 +574,18 @@ def replace(bodyMarker): """ return bodyMarker.group() + injectableHeader - # Use the previously defined function to inject the RTF header. We are - # trying a few different methods to determine where to place the header. - data = constants.RE_RTF_BODY_START.sub(replace, self.rtfBody, 1) - # If after any method the data does not match the RTF body, then we have - # succeeded. - if data != self.rtfBody: - logger.debug('Successfully injected RTF header using first method.') - return data - # This first method only applies to documents with encapsulated HTML # that is formatted in a nice way. if isEncapsulatedRtf(self.rtfBody): - data = constants.RE_RTF_ENC_BODY_START_1.sub(replace, self.rtfBody, 1) + data = constants.RE_RTF_ENC_BODY_START.sub(replace, self.rtfBody, 1) if data != self.rtfBody: logger.debug('Successfully injected RTF header using encapsulation method.') return data + logger.debug('RTF has encapsulated HTML, but injection method failed. It is likely dirty. Will use normal RTF injection method.') # If the normal encapsulated HTML injection fails or it isn't # encapsulated, use the internal _rtf module. + logger.debug('Using _rtf module to inject RTF text header.') return createDocument(injectStartRTF(self.rtfBody, injectableHeader)) def save(self, **kwargs): From aec2370ff04f7bf6c3d4584f8e47898a50652f6d Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sat, 18 Mar 2023 15:49:49 -0700 Subject: [PATCH 23/24] Updated parts of module for next-release to keep local branch items off --- CHANGELOG.md | 6 ++++++ README.rst | 15 ++++++++++++--- docs/conf.py | 2 +- extract_msg/__init__.py | 4 ++-- temp-changelog-rtf.md | 4 ---- 5 files changed, 21 insertions(+), 10 deletions(-) delete mode 100644 temp-changelog-rtf.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 73808c81..951e0ab3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +**v0.40.0** +* [[TeamMsgExtractor #338](https://github.com/TeamMsgExtractor/msg-extractor/issues/338)] Added new code to handle injection of text into the RTF body. For many cases, this will be much more effective as it relies on ensuring that it is in the main group and past the header before injection. It is *not* currently the first choice as it doesn't have proper respect for encapsulated HTML, however it will replace some of the old methods entirely. Solving this issue was done through the use of a few functions and the internal `_rtf` module. This module in it's entirety is considered to be implementation details, and I give no guarantee that it will remain in it's current state even across patch versions. As such, it is not recommended to use it outside of the module. +* Changed `MessageBase.rtfEncapInjectableHeader` and `MessageBase.rtfPlainInjectableHeader` from `str` to `bytes`. They always get encoded anyways, so I don't know why I had them returning as `str`. +* Updated minimum Python version to 3.8 as 3.6 has reached end of support and 3.7 will reach end of support within the year. +* Updated information in `README`. + **v0.39.2** * Fixed issues with `AttachmentBase.name` that could cause it to generate wrong. * Added convenience function `MSGFile.exportBytes` which returns the exported version from `MSGFile.export` as bytes instead of writing it to a file or file-like object. diff --git a/README.rst b/README.rst index cbb8a053..b16c827d 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,6 @@ |License: GPL v3| |PyPI3| |PyPI2| -msg-extractor +extract-msg ============= Extracts emails and attachments saved in Microsoft Outlook's .msg files @@ -8,11 +8,18 @@ Extracts emails and attachments saved in Microsoft Outlook's .msg files The python package extract_msg automates the extraction of key email data (from, to, cc, date, subject, body) and the email's attachments. +Documentation can be found in the code, on the `wiki`_, and on the +`read the docs`_ page. + NOTICE ====== 0.29.* is the branch that supports both Python 2 and Python 3. It is now only receiving bug fixes and will not be receiving feature updates. +0.39.* is the last versions that supported Python 3.6 and 3.7. Support for those +was dropped to allow the use of new features from 3.8 and because the life spans +of those versions had ended. + This module has a Discord server for general discussion. You can find it here: `Discord`_ @@ -234,8 +241,8 @@ your access to the newest major version of extract-msg. .. |License: GPL v3| image:: https://img.shields.io/badge/License-GPLv3-blue.svg :target: LICENSE.txt -.. |PyPI3| image:: https://img.shields.io/badge/pypi-0.39.2-blue.svg - :target: https://pypi.org/project/extract-msg/0.39.2/ +.. |PyPI3| image:: https://img.shields.io/badge/pypi-0.40.0-blue.svg + :target: https://pypi.org/project/extract-msg/0.40.0/ .. |PyPI2| image:: https://img.shields.io/badge/python-3.8+-brightgreen.svg :target: https://www.python.org/downloads/release/python-3816/ @@ -252,3 +259,5 @@ your access to the newest major version of extract-msg. .. _Ko-fi: https://ko-fi.com/destructione .. _Patreon: https://www.patreon.com/DestructionE .. _msg-explorer: https://pypi.org/project/msg-explorer/ +.. _wiki: https://github.com/TeamMsgExtractor/msg-extractor/wiki +.. _read the docs: https://msg-extractor.rtfd.io/ diff --git a/docs/conf.py b/docs/conf.py index c1e3f650..377babe9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,7 +12,7 @@ sys.path.insert(0, os.path.abspath("..")) __author__ = 'Destiny Peterson & Matthew Walker' -__version__ = '0.39.2' +__version__ = '0.40.0' __year__ = '2023' diff --git a/extract_msg/__init__.py b/extract_msg/__init__.py index faa77307..d7cd8e70 100644 --- a/extract_msg/__init__.py +++ b/extract_msg/__init__.py @@ -27,8 +27,8 @@ # along with this program. If not, see . __author__ = 'Destiny Peterson & Matthew Walker' -__date__ = '2023-02-26' -__version__ = '0.39.2' +__date__ = '2023-03-18' +__version__ = '0.40.0' import logging diff --git a/temp-changelog-rtf.md b/temp-changelog-rtf.md deleted file mode 100644 index 108ff6c6..00000000 --- a/temp-changelog-rtf.md +++ /dev/null @@ -1,4 +0,0 @@ -**v??.??.??** -* [[TeamMsgExtractor #338](https://github.com/TeamMsgExtractor/msg-extractor/issues/338)] Added new code to handle injection of text into the RTF body. For many cases, this will be much more effective as it relies on ensuring that it is in the main group and past the header before injection. It is *not* currently the first choice as it doesn't have proper respect for encapsulated HTML, however it will replace some of the old methods entirely. Solving this issue was done through the use of a few functions and the internal `_rtf` module. This module in it's entirety is considered to be implementation details, and I give no guarantee that it will remain in it's current state even across patch versions. As such, it is not recommended to use it outside of the module. -* Changed `MessageBase.rtfEncapInjectableHeader` and `MessageBase.rtfPlainInjectableHeader` from `str` to `bytes`. They always get encoded anyways, so I don't know why I had them returning as `str`. -* Updated minimum Python version to 3.8 as 3.6 has reached end of support and 3.7 will reach end of support within the year. From 0998c659e42e32a70b4218e509953cf2a93a2ab7 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sat, 18 Mar 2023 16:05:47 -0700 Subject: [PATCH 24/24] Fix changelog link (transfered changes from one branch to another) --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index b16c827d..db6544c0 100644 --- a/README.rst +++ b/README.rst @@ -26,7 +26,7 @@ This module has a Discord server for general discussion. You can find it here: Changelog --------- -- `Changelog `__ +- `Changelog`_ Usage ----- @@ -261,3 +261,4 @@ your access to the newest major version of extract-msg. .. _msg-explorer: https://pypi.org/project/msg-explorer/ .. _wiki: https://github.com/TeamMsgExtractor/msg-extractor/wiki .. _read the docs: https://msg-extractor.rtfd.io/ +.. _Changelog: https://github.com/TeamMsgExtractor/msg-extractor/blob/master/CHANGELOG.md