diff --git a/CHANGELOG.md b/CHANGELOG.md index 0145e45d..39603a93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +**v0.42.0** +* Added new submodule `custom_attachments`. This submodule provides an extendable way to handle custom attachment types, attachment types whose structure and formatting are not defined in the Microsoft documentation for MSG files. +* Added new property `AttachmentBase.clsid` which returns the listed CLSID value of the data stream/storage of the attachment. +* Changed internal behavior of `MSGFile.attachments`. This should not cause any noticeable changes to the output. + **v0.41.5** * Fixed an issue from version `0.41.3` where the header being present but missing the `From` field would cause an exception. diff --git a/changelog_temp.md b/changelog_temp.md new file mode 100644 index 00000000..bb6f63f4 --- /dev/null +++ b/changelog_temp.md @@ -0,0 +1,7 @@ +Temporary location for the changelog entry to ensure it doesn't conflict. + +**v0.??.??** +* Added new submodule `custom_attachments`. This submodule provides an extendable way to handle custom attachment types, attachment types whose structure and formatting are not defined in the Microsoft documentation for MSG files. +* Added new property `AttachmentBase.clsid` which returns the listed CLSID value of the data stream/storage of the attachment. +* Changed internal behavior of `MSGFile.attachments`. This should not cause any noticeable changes to the output. +* Removed some debug code that was left behind. diff --git a/extract_msg/attachment.py b/extract_msg/attachment.py index 0a387fe3..34cb797c 100644 --- a/extract_msg/attachment.py +++ b/extract_msg/attachment.py @@ -19,6 +19,7 @@ from . import constants from .attachment_base import AttachmentBase +from .custom_attachments import CustomAttachmentHandler, getHandler from .enums import AttachmentType from .exceptions import StandardViolationError from .utils import createZipOpen, inputToString, openMsg, prepareFilename @@ -47,6 +48,7 @@ def __init__(self, msg, dir_): located. """ super().__init__(msg, dir_) + self.__customHandler = None if '37050003' not in self.props: from .prop import createProp @@ -80,9 +82,11 @@ def __init__(self, msg, dir_): self.__data = self._getStream('__substg1.0_37010102') elif self.exists('__substg1.0_3701000D'): if (self.props['37050003'].value & 0x7) != 0x5: - raise NotImplementedError( - 'Current version of extract_msg does not support extraction of containers that are not embedded msg files.') - # TODO add implementation. + self.__type = AttachmentType.CUSTOM + # Check if we have any custom handlers. If not, it will raise + # an error automatically. + self.__customHandler = getHandler(self) + self.__data = self.__customHandler.data else: self.__prefix = msg.prefixList + [dir_, '__substg1.0_3701000D'] self.__type = AttachmentType.MSG @@ -118,8 +122,10 @@ def getFilename(self, **kwargs) -> str: # Check if user wants to save the file under the Content-ID. if kwargs.get('contentId', False): filename = self.cid - # If filename is None at this point, use long filename as first - # preference. + # If we are using a custom handler, prefer it's name. + if self.type is AttachmentType.CUSTOM: + filename = self.__customHandler.name + # If we are here, try to get the filename however else we can. if not filename: filename = self.name # Otherwise just make something up! @@ -213,7 +219,7 @@ def save(self, **kwargs) -> Optional[Union[str, MSGFile]]: fullFilename = customPath / filename - if self.type is AttachmentType.DATA: + if isinstance(self.__data, bytes): if _zip: name, ext = os.path.splitext(filename) nameList = _zip.namelist() @@ -248,7 +254,7 @@ def save(self, **kwargs) -> Optional[Union[str, MSGFile]]: _zip.close() return str(fullFilename) - else: + elif self.__data: if kwargs.get('extractEmbedded', False): with _open(str(fullFilename), mode) as f: self.data.export(f) @@ -268,6 +274,14 @@ def saveEmbededMessage(self, **kwargs) -> None: """ self.data.save(**kwargs) + @property + def customHandler(self) -> Optional[CustomAttachmentHandler]: + """ + The instance of the custom handler associated with this attachment, if + it has one. + """ + return self.__customHandler + @property def data(self) -> Optional[Union[bytes, MSGFile]]: """ diff --git a/extract_msg/attachment_base.py b/extract_msg/attachment_base.py index 62a3938e..1f1996a7 100644 --- a/extract_msg/attachment_base.py +++ b/extract_msg/attachment_base.py @@ -9,7 +9,7 @@ import datetime import logging -from functools import partial +from functools import cached_property, partial from typing import Optional, Tuple, TYPE_CHECKING from .enums import AttachmentType, ErrorBehavior, PropertiesType @@ -262,7 +262,7 @@ def existsTypedProperty(self, id, _type = None) -> bool: def attachmentEncoding(self) -> Optional[bytes]: """ The encoding information about the attachment object. Will return - b'*\x86H\x86\xf7\x14\x03\x0b\x01' if encoded in MacBinary format, + b'*\\x86H\\x86\\xf7\\x14\\x03\\x0b\\x01' if encoded in MacBinary format, otherwise it is unset. """ return self._ensureSet('_attachmentEncoding', '__substg1.0_37020102', False) @@ -287,10 +287,37 @@ def cid(self) -> Optional[str]: contendId = cid + @cached_property + def clsid(self) -> str: + """ + Returns the CLSID for the data stream/storage of the attachment. + """ + # Set some default values. + clsid = '00000000-0000-0000-0000-000000000000' + dataStream = None + + # See if we can find the data stream/storage. + if self.type in (AttachmentType.CUSTOM, AttachmentType.MSG): + dataStream = [self.__dir, '__substg1.0_3701000D'] + elif self.type is AttachmentType.DATA: + dataStream = [self.__dir, '__substg1.0_37010102'] + elif self.type is AttachmentType.UNSUPPORTED: + # Special check for custom attachments. + if self.exists('__substg1.0_3701000D'): + dataStream = [self.__dir, '__substg1.0_3701000D'] + elif self.exists('__substg1.0_37010102'): + dataStream = [self.__dir, '__substg1.0_37010102'] + + # If we found the right item, get the CLSID. + if dataStream: + clsid = self.__msg._getOleEntry(dataStream).clsid or clsid + + return self.__clsid + @property - def dir(self): + def dir(self) -> str: """ - Returns the directory inside the msg file where the attachment is + Returns the directory inside the MSG file where the attachment is located. """ return self.__dir diff --git a/extract_msg/custom_attachments/__init__.py b/extract_msg/custom_attachments/__init__.py new file mode 100644 index 00000000..13e32914 --- /dev/null +++ b/extract_msg/custom_attachments/__init__.py @@ -0,0 +1,79 @@ +from __future__ import annotations + + +""" +Submodule designed to help with saving and using custom attachments. Custom +attachments are those follow standards not defined in the MSG documentation. Use +the function `getHandler` to get an instance of a subclass of +CustomAttachmentHandler. + +CustomAttachmentHandler subclasses will all define the following methods: + injectHtml: A method which takes HTML and inserts the + +It should hopefully be completely unnecessary for your code to know what type of +handler it is using, as the abstract base class should give all of the functions +you would typically want. + +If you would like to add your own handler, simply subclass +CustomAttachmentHandler and add it using the `registerHandler` function. +""" + +__all__ = [ + # Classes. + 'CustomAttachmentHandler', + 'OutlookImageDIB', + + # Functions. + 'getHandler', + 'registerHandler', +] + + +from typing import List, Type, TYPE_CHECKING + +from .custom_handler import CustomAttachmentHandler + + +# Create a way to register handlers. +_knownHandlers : List[CustomAttachmentHandler] = [] + +def registerHandler(handler : Type[CustomAttachmentHandler]) -> None: + """ + Registers the CustomAttachmentHandler subclass as a handler. + + :raises TypeError: The handler was not a subclass of + CustomAttachmentHandler. + """ + # Make sure it is a subclass of CustomAttachmentHandler. + if not isinstance(handler, type): + raise ValueError(':param handler: must be a class, not an instance of a class.') + if not issubclass(handler, CustomAttachmentHandler): + raise ValueError(':param handler: must be a subclass of CustomAttachmentHandler.') + _knownHandlers.append(handler) + + + +# Import built-in handler modules. They will all automatically register their +# respecive handler(s). +from .outlook_image_dib import OutlookImageDIB + + +if TYPE_CHECKING: + from ..attachment import Attachment + + +# Function designed to route to the correct handler. +def getHandler(attachment : Attachment) -> CustomAttachmentHandler: + """ + Takes an attachment and uses it to find the correct handler. Returns an + instance created using the specified attachment. + + :raises NotImplementedError: No handler could be found. + :raises ValueError: A handler was found, but something was wrong with the + attachment data. + """ + for handler in _knownHandlers: + if handler.isCorrectHandler(attachment): + return handler(attachment) + + raise NotImplementedError('No valid handler could be found for the attachment. Contact the developers for help.') diff --git a/extract_msg/custom_attachments/custom_handler.py b/extract_msg/custom_attachments/custom_handler.py new file mode 100644 index 00000000..f6ebe21c --- /dev/null +++ b/extract_msg/custom_attachments/custom_handler.py @@ -0,0 +1,65 @@ +from __future__ import annotations + + +__all__ = [ + 'CustomAttachmentHandler', +] + + +import abc + +from typing import List, Optional, Tuple, TYPE_CHECKING + + +if TYPE_CHECKING: + from ..attachment import Attachment + + +class CustomAttachmentHandler(abc.ABC): + """ + A class designed to help with custom attachments that may require parsing in + special ways that are completely different from one another. + """ + + def __init__(self, attachment : Attachment): + super().__init__() + self.__att = attachment + + @classmethod + @abc.abstractmethod + def isCorrectHandler(cls, attachment : Attachment) -> bool: + """ + Checks if this is the correct handler for the attachment. + """ + + @abc.abstractmethod + def generateRtf(self) -> Optional[bytes]: + """ + Generates the RTF to inject in place of the \objattph tag. + + If this function should do nothing, returns None. + """ + + @property + def attachment(self): + """ + The attachment this handler is associated with. + """ + return self.__att + + @property + @abc.abstractmethod + def data(self) -> bytes: + """ + Gets the data for the attachment. + + If an attachment should do nothing when saving, return None from this + property. + """ + + @property + @abc.abstractmethod + def name(self) -> str: + """ + Returns the name to be used when saving the attachment. + """ diff --git a/extract_msg/custom_attachments/outlook_image_dib.py b/extract_msg/custom_attachments/outlook_image_dib.py new file mode 100644 index 00000000..4ed48a45 --- /dev/null +++ b/extract_msg/custom_attachments/outlook_image_dib.py @@ -0,0 +1,134 @@ +from __future__ import annotations + + +__all__ = [ + 'OutlookImage', +] + + +import base64 +import struct + +from typing import List, Optional, Tuple, TYPE_CHECKING + +from . import registerHandler +from .custom_handler import CustomAttachmentHandler +from .utils import htmlSplitRendered +from ..enums import DVAspect +from ..exceptions import CustomAttachmentError + + +if TYPE_CHECKING: + from ..attachment import Attachment + +_ST_OLE = struct.Struct(' bool: + if attachment.clsid != '00000316-0000-0000-C000-000000000046': + return False + + # Check for the required streams. + if not attachment.exists('__substg1.0_3701000D/CONTENTS'): + return False + if not attachment.exists('__substg1.0_3701000D/\x01Ole'): + return False + if not attachment.exists('__substg1.0_3701000D/\x03MailStream'): + return False + + return True + + def generateRtf(self) -> Optional[bytes]: + """ + Generates the RTF to inject in place of the \objattph tag. + + If this function should do nothing, returns None. + + This function requires PIL or Pillow. If neither are found, raises an + import error. + """ + try: + import PIL.Image + except ImportError: + raise ImportError('PIL or Pillow is required for inserting an Outlook Image into the body.') + + # First, convert the bitmap into a PNG so we can insert it into the + # body. + import io + + # Note, use self.data instead of self.__data to allow support for + # extensions. + with PIL.Image.open(io.BytesIO(self.data)) as img: + out = io.BytesIO() + img.save(out, 'PNG') + + hexData = out.getvalue().hex() + + inject = '{\\*\\shppict\n{\\pict\\picscalex100\\picscaley100' + inject += f'\\picw{img.width}\\pich{img.height}' + inject += f'\\picwgoal{self.__xtwips}\\pichgoal{self.__ytwips}\n' + inject += '\\pngblip ' + hexData + '}}' + + return inject.encode() + + @property + def data(self) -> bytes: + return self.__data + + @property + def name(self) -> str: + return self.attachment.shortFilename + '.bmp' + + + + +registerHandler(OutlookImageDIB) diff --git a/extract_msg/custom_attachments/utils.py b/extract_msg/custom_attachments/utils.py new file mode 100644 index 00000000..18d8507e --- /dev/null +++ b/extract_msg/custom_attachments/utils.py @@ -0,0 +1,207 @@ +""" +Utilities for extract-msg that are more specialized for the custom_attachments +submodule than for the main module. +""" + +__all__ = [ + 'htmlSplitRendered', + 'tokenizeHtml', +] + + +import bs4 + +from typing import List + + +_WHITESPACE_BREAKERS = ( + ' bool: + """ + Helper function to indicate that a tag breaks a chain of whitespace. + """ + for x in _WHITESPACE_BREAKERS: + if token.startswith(x) and len(token) > len(x) and token[len(x)] in ('>', ' ', '/'): + return True + + return False + + +def _isWhitespaceToken(token : str) -> bool: + if token[0] == '<': + for x in _WHITESPACE_TAGS: + if token.startswith(x) and len(token) > len(x) and token[len(x)] in ('>', ' ', '/'): + return True + elif token in (' ', ' ', ' ', ' ', ' '): + return True + else: + return token.isspace() + + return False + + +def htmlSplitRendered(html : bytes) -> List[str]: + """ + Takes html bytes and returns a list of the rendered characters, with data + that is not being rendered being attached to the next rendered character. + """ + # Unfortunately bs4 didn't seem particularly great for tokenizing, so I did + # my own function that works well enough. First, let's tokenize the html. + tokens = tokenizeHtml(bs4.BeautifulSoup(html, features = 'html.parser').decode()) + + # Next, let's break things down further. + breakDown = [] + for token in tokens: + # We tell what we are looking at by checking the first character of the + # token. If it's a <, then it is an HTML tag. If it is a & then it is an + # escape. Otherwise, it is plain text. For both tags and escapes, just + # dump them into the list. + if token[0] in ('<', '&'): + breakDown.append(token) + else: + # If we are looking at plain text, add it by extending the list. + breakDown.extend(token) + + # Now that we have broken things down further, let's go through and join our + # pieces togethered into rendered tokens. Here is were we actually need to + # know what an html tag is. If it's an escape or just a non-whitespace + # character, we can just shove it onto what we currently have. + current = '' + renderedCharacters = [] + lastWhitespace = None + for item in breakDown: + if item[0] == '&': + if _isWhitespaceToken(item): + if lastWhitespace is None: + lastWhitespace == item + else: + if lastWhitespace is not None and lastWhitespace[0] != '<': + current += lastWhitespace + renderedCharacters.append(current) + current = '' + lastWhitespace = None + current += item + renderedCharacters.append(current) + current = '' + elif item[0] == '<': + if _isWhitespaceToken(item): + # If we are here, add it to current, push current, and set this + # tag as the last whitespace. + current += item + renderedCharacters.append(current) + current = '' + lastWhitespace = item + else: + + # Some tags will break whitespace chains. + if _isWhitespaceBreaker(item): + if lastWhitespace is not None and lastWhitespace[0] != '<': + current += lastWhitespace + renderedCharacters.append(current) + current = '' + lastWhitespace = None + + current += item + else: + # Here is where we handle text, which is not particularly fun. + # Basically if it is whitespace and lastWhitespace is not none, we + # set the whitespace. + if _isWhitespaceToken(item): + if lastWhitespace is None: + lastWhitespace = item + else: + if lastWhitespace is not None and lastWhitespace[0] != '<': + current += lastWhitespace + renderedCharacters.append(current) + current = '' + lastWhitespace = None + current += item + renderedCharacters.append(current) + current = '' + + if current: + renderedCharacters.append(current) + + return renderedCharacters + + +def tokenizeHtml(html : str) -> List[str]: + # Setup a few variables for state tracking. + inTag = False + # Used for tracking escapes starting with &. If your escape ends up at 100 + # characters because it is missing the semicolon, we are going to throw an + # error. + inEscape = False + inString = False + # Only used when in string. Last character was a backslash. + isBackslash = False + # Tells which quote type we are in. + isDoubleQuote = False + + tokens = [] + currentToken = '' + + # Finally, let's start breaking things up. Our rules are that if we start a + # quote while in a tag, then we acknowledge it, otherwise it is treated as + # plain text. + for character in html: + # First we need to know our state, as our state determines what how we + # process a character. + if inTag: + currentToken += character + if inString: + if character == '"' and isDoubleQuote: + # If isBackslash then we stay in the quote, otherwise... + isQuote = isBackslash + elif character == "'" and not isDoubleQuote: + # If isBackslash then we stay in the quote, otherwise... + isQuote = isBackslash + elif character == '\\': + isBackslash = not isBackslash + if character != '\\': + isBackslash = False + else: + if character == '>': + inTag = False + tokens.append(currentToken) + currentToken = '' + elif inEscape: + currentToken += character + if len(currentToken) > 99: + raise ValueError('Found escape that was too long (is a ; missing?)') + if character == ';': + tokens.append(currentToken) + currentToken = '' + inEscape = False + elif inString: + # This is an error. We should *never* be in a quote if we are not in + # a tag. + raise ValueError('Found to be inQuote when not in tag.') + else: + # We are currently processing plain text, so let's just handle. + if character == '&': + if currentToken: + tokens.append(currentToken) + currentToken = character + inEscape = True + elif character == '<': + if currentToken: + tokens.append(currentToken) + currentToken = character + inTag = True + inString = False + else: + currentToken += character + + if currentToken: + tokens.append(currentToken) + + return tokens diff --git a/extract_msg/enums.py b/extract_msg/enums.py index f5ace9b5..b51c9c29 100644 --- a/extract_msg/enums.py +++ b/extract_msg/enums.py @@ -131,6 +131,7 @@ class AttachmentType(enum.Enum): BROKEN = 4 UNSUPPORTED = 5 SIGNED_EMBEDDED = 6 + CUSTOM = 7 UNKNOWN = 0xFFFFFFFF @@ -355,6 +356,7 @@ class Color(enum.IntEnum): BLACK = 1 + class ContactAddressIndex(enum.Enum): EMAIL_1 = 0 EMAIL_2 = 1 @@ -415,6 +417,16 @@ class DisplayType(enum.Enum): +class DVAspect(enum.IntEnum): + """ + Part of the extra data for Outlook signatures. Different sources seem to + disagree on the meanings, so I'm sticking to the meanings in the official + Microsoft documentation of the DVASPECT enumeration. + """ + CONTENT = 1 + ICON = 4 + + class ElectronicAddressProperties(enum.Enum): @classmethod def fromBits(cls, value : int) -> Set['ElectronicAddressProperties']: diff --git a/extract_msg/exceptions.py b/extract_msg/exceptions.py index d323746e..2223faca 100644 --- a/extract_msg/exceptions.py +++ b/extract_msg/exceptions.py @@ -46,6 +46,11 @@ class ConversionError(Exception): An error occured during type conversion. """ +class CustomAttachmentError(Exception): + """ + A generic error used for issues handling custom attachments. + """ + class DataNotFoundError(Exception): """ Requested stream type was unavailable. @@ -91,7 +96,7 @@ class StandardViolationError(InvalidFileFormatError): A critical violation of the MSG standards was detected and could not be recovered from. Recoverable violations will result in log messages instead. - Any that could reasonably be skipped, although are likely to still cause + Any that could reasonably be skipped, although are likely to still cause errors down the line, can be suppressed. """ diff --git a/extract_msg/msg.py b/extract_msg/msg.py index 06afbc57..8b7a55ff 100644 --- a/extract_msg/msg.py +++ b/extract_msg/msg.py @@ -667,10 +667,9 @@ def attachments(self) -> List: # Get the attachments. attachmentDirs = [] prefixLen = self.prefixLen - for dir_ in self.listDir(False, True): - if dir_[prefixLen].startswith('__attach') and \ - dir_[prefixLen] not in attachmentDirs: - attachmentDirs.append(dir_[prefixLen]) + for dir_ in self.listDir(False, True, False): + if dir_[0].startswith('__attach') and dir_[0] not in attachmentDirs: + attachmentDirs.append(dir_[0]) self._attachments = []