From 24d2dde3c44b76a5cb648db89d5cf35c44e6dbf3 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 30 Nov 2022 16:12:34 -0800 Subject: [PATCH 01/30] Start the process for outlook signatures --- extract_msg/attachment.py | 17 ++++++++++--- extract_msg/custom_attachments/__init__.py | 19 ++++++++++++++ .../custom_attachments/custom_handler.py | 25 +++++++++++++++++++ .../custom_attachments/outlook_signature.py | 5 ++++ extract_msg/enums.py | 11 ++++++++ 5 files changed, 73 insertions(+), 4 deletions(-) create mode 100644 extract_msg/custom_attachments/__init__.py create mode 100644 extract_msg/custom_attachments/custom_handler.py create mode 100644 extract_msg/custom_attachments/outlook_signature.py diff --git a/extract_msg/attachment.py b/extract_msg/attachment.py index 7a65995e..7588e207 100644 --- a/extract_msg/attachment.py +++ b/extract_msg/attachment.py @@ -5,7 +5,7 @@ import string import zipfile -from typing import Optional, Union +from typing import Any, Dict, Optional, Union from . import constants from .attachment_base import AttachmentBase @@ -32,6 +32,7 @@ def __init__(self, msg, dir_): located. """ super().__init__(msg, dir_) + self.__extraData = None # Get attachment data. if self.exists('__substg1.0_37010102'): @@ -39,9 +40,9 @@ def __init__(self, msg, dir_): self.__data = self._getStream('__substg1.0_37010102') elif self.exists('__substg1.0_3701000D'): if (self.props['37050003'].value & 0x7) != 0x5: - raise NotImplementedError( - 'Current version of extract_msg does not support extraction of containers that are not embedded msg files.') - # TODO add implementation. + # Check if we can recognize it as an Outlook signature. + + raise NotImplementedError('Unrecognized custom attachment format. Support may be possible but is not likely.') else: self.__prefix = msg.prefixList + [dir_, '__substg1.0_3701000D'] self.__type = AttachmentType.MSG @@ -233,6 +234,14 @@ def data(self) -> Optional[Union[bytes, 'MSGFile']]: """ return self.__data + @property + def extraData(self) -> Optional[CustomAttachmentHandler]: + """ + The extra data for a custom attachment type, if any. If the data + returned is a dict, check the "type" key + """ + return self.__extraData + @property def randomFilename(self) -> str: """ diff --git a/extract_msg/custom_attachments/__init__.py b/extract_msg/custom_attachments/__init__.py new file mode 100644 index 00000000..af28c015 --- /dev/null +++ b/extract_msg/custom_attachments/__init__.py @@ -0,0 +1,19 @@ +""" +Submodule designed to help with saving and using custom attachments. Custom +attachments are those follow standards not defined in the MSG documentation. Use +the function `getHandler` to get an instance of a subclass of +CustomAttachmentHandler. + +CustomAttachmentHandler subclasses will all define the following methods: + injectHtml: A method which takes HTML and inserts the +""" + +from custom_handler import CustomAttachmentHandler +from outlook_signature import OutlookSignature + + +# Function designed to route to the correct handler. +def getHandler(attachment : 'Attachment'): + """ + Takes an attachment and uses it to find the correct hanlder. + """ diff --git a/extract_msg/custom_attachments/custom_handler.py b/extract_msg/custom_attachments/custom_handler.py new file mode 100644 index 00000000..196db49a --- /dev/null +++ b/extract_msg/custom_attachments/custom_handler.py @@ -0,0 +1,25 @@ +import abc + + +class CustomAttachmentHandler(abc.ABC): + """ + A class designed to help with custom attachments that may require parsing in + special ways that are completely different from one another. + """ + def __init__(self, attachment : 'Attachment'): + super().__init__() + self.__att = attachment + + @classmethod + @abc.abstractmethod + def isCorrectHandler(cls, attachment : 'Attachment') -> bool: + """ + Checks if this is the correct handler for the attachment. + """ + + @property + @abc.abstractmethod + def data(self): + """ + Gets the data for the attachment. + """ diff --git a/extract_msg/custom_attachments/outlook_signature.py b/extract_msg/custom_attachments/outlook_signature.py new file mode 100644 index 00000000..257ffdd2 --- /dev/null +++ b/extract_msg/custom_attachments/outlook_signature.py @@ -0,0 +1,5 @@ +from .custom_handler import CustomAttachmentHandler + + +class OutlookSignature(CustomAttachmentHandler): + pass diff --git a/extract_msg/enums.py b/extract_msg/enums.py index 3f746da4..aeb6ca1b 100644 --- a/extract_msg/enums.py +++ b/extract_msg/enums.py @@ -316,6 +316,7 @@ class Color(enum.IntEnum): BLACK = 1 + class ContactAddressIndex(enum.Enum): EMAIL_1 = 0 EMAIL_2 = 1 @@ -376,6 +377,16 @@ class DisplayType(enum.Enum): +class DVAspect(enum.IntEnum): + """ + Part of the extra data for Outlook signatures. Different sources seem to + disagree on the meanings, so I'm sticking to the meanings in the official + Microsoft documentation of the DVASPECT enumeration. + """ + CONTENT = 1 + ICON = 4 + + class ElectronicAddressProperties(enum.Enum): @classmethod def fromBits(cls, value : int) -> Set['ElectronicAddressProperties']: From e81222538369bd4e3d70ad7723ed130803c0726f Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 30 Nov 2022 16:40:03 -0800 Subject: [PATCH 02/30] More progress on outlook signatures. Can generate html tag --- extract_msg/attachment.py | 9 +++--- .../custom_attachments/outlook_signature.py | 28 ++++++++++++++++++- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/extract_msg/attachment.py b/extract_msg/attachment.py index 7588e207..3a526959 100644 --- a/extract_msg/attachment.py +++ b/extract_msg/attachment.py @@ -40,9 +40,9 @@ def __init__(self, msg, dir_): self.__data = self._getStream('__substg1.0_37010102') elif self.exists('__substg1.0_3701000D'): if (self.props['37050003'].value & 0x7) != 0x5: - # Check if we can recognize it as an Outlook signature. - - raise NotImplementedError('Unrecognized custom attachment format. Support may be possible but is not likely.') + # Check if we have any custom handlers. If not, it will raise + # an error automatically. + self.__customHandler = getHandler(self) else: self.__prefix = msg.prefixList + [dir_, '__substg1.0_3701000D'] self.__type = AttachmentType.MSG @@ -237,8 +237,7 @@ def data(self) -> Optional[Union[bytes, 'MSGFile']]: @property def extraData(self) -> Optional[CustomAttachmentHandler]: """ - The extra data for a custom attachment type, if any. If the data - returned is a dict, check the "type" key + """ return self.__extraData diff --git a/extract_msg/custom_attachments/outlook_signature.py b/extract_msg/custom_attachments/outlook_signature.py index 257ffdd2..a1e579e4 100644 --- a/extract_msg/custom_attachments/outlook_signature.py +++ b/extract_msg/custom_attachments/outlook_signature.py @@ -1,5 +1,31 @@ +import struct + from .custom_handler import CustomAttachmentHandler +from ..enums import DVAspect + + +_MAILSTREAM_STRUCT = struct.Struct(''.encode('ascii') From 0e23a28936f205265d18df46412c1eccb20d16ef Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 30 Nov 2022 20:09:27 -0800 Subject: [PATCH 03/30] Started parsing for Ole Stream --- .../custom_attachments/outlook_signature.py | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/extract_msg/custom_attachments/outlook_signature.py b/extract_msg/custom_attachments/outlook_signature.py index a1e579e4..c3dc4c8a 100644 --- a/extract_msg/custom_attachments/outlook_signature.py +++ b/extract_msg/custom_attachments/outlook_signature.py @@ -4,7 +4,8 @@ from ..enums import DVAspect -_MAILSTREAM_STRUCT = struct.Struct(''.encode('ascii') From e919f0dc7be573b4286a97dde1e279cd3fc405ec Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 30 Nov 2022 21:47:24 -0800 Subject: [PATCH 04/30] More progress --- changelog_temp.md | 6 +++ extract_msg/attachment.py | 11 +++-- extract_msg/attachment_base.py | 13 ++++-- extract_msg/custom_attachments/__init__.py | 40 +++++++++++++++++-- .../custom_attachments/outlook_signature.py | 35 ++++++++++++++-- extract_msg/enums.py | 1 + extract_msg/msg.py | 8 ++-- 7 files changed, 94 insertions(+), 20 deletions(-) create mode 100644 changelog_temp.md diff --git a/changelog_temp.md b/changelog_temp.md new file mode 100644 index 00000000..b8bbe33a --- /dev/null +++ b/changelog_temp.md @@ -0,0 +1,6 @@ +Temporary location for the changelog entry to ensure it doesn't conflict. + +**v0.??.??** +* Added new submodule `custom_attachments`. This submodule provides an extendable way to handle custom attachment types, attachment types whose structure and formatting are not defined in the Microsoft documentation for MSG files. +* Added new property `AttachmentBase.clsid` which returns the listed CLSID value of the data stream/storage of the attachment. +* Changed internal behavior of `MSGFile.attachments`. This should not cause any noticeable changes to the output. diff --git a/extract_msg/attachment.py b/extract_msg/attachment.py index 3a526959..8ee36b7f 100644 --- a/extract_msg/attachment.py +++ b/extract_msg/attachment.py @@ -9,6 +9,7 @@ from . import constants from .attachment_base import AttachmentBase +from .custom_attachments import CustomAttachmentHandler, getHandler from .enums import AttachmentType from .utils import createZipOpen, inputToString, openMsg, prepareFilename @@ -32,7 +33,7 @@ def __init__(self, msg, dir_): located. """ super().__init__(msg, dir_) - self.__extraData = None + self.__customHandler = None # Get attachment data. if self.exists('__substg1.0_37010102'): @@ -40,6 +41,7 @@ def __init__(self, msg, dir_): self.__data = self._getStream('__substg1.0_37010102') elif self.exists('__substg1.0_3701000D'): if (self.props['37050003'].value & 0x7) != 0x5: + self.__type = AttachmentType.CUSTOM # Check if we have any custom handlers. If not, it will raise # an error automatically. self.__customHandler = getHandler(self) @@ -235,11 +237,12 @@ def data(self) -> Optional[Union[bytes, 'MSGFile']]: return self.__data @property - def extraData(self) -> Optional[CustomAttachmentHandler]: + def customHandler(self) -> Optional[CustomAttachmentHandler]: """ - + The instance of the custom handler associated with this attachment, if + it has one. """ - return self.__extraData + return self.__customHandler @property def randomFilename(self) -> str: diff --git a/extract_msg/attachment_base.py b/extract_msg/attachment_base.py index 2b5a10ee..8be2d0fb 100644 --- a/extract_msg/attachment_base.py +++ b/extract_msg/attachment_base.py @@ -33,7 +33,7 @@ def __init__(self, msg, dir_): self.__dir = dir_ self.__props = Properties(self._getStream('__properties_version1.0'), PropertiesType.ATTACHMENT) self.__namedProperties = NamedProperties(msg.named, self) - + self.__clsid = msg._getOleEntry(dir_).clsid or '' def _ensureSet(self, variable, streamID, stringStream = True, **kwargs): """ @@ -271,9 +271,16 @@ def cid(self) -> Optional[str]: contendId = cid @property - def dir(self): + def clsid(self) -> str: + """ + Returns the CLSID for the data stream/storage of the attachment. + """ + return self.__clsid + + @property + def dir(self) -> str: """ - Returns the directory inside the msg file where the attachment is + Returns the directory inside the MSG file where the attachment is located. """ return self.__dir diff --git a/extract_msg/custom_attachments/__init__.py b/extract_msg/custom_attachments/__init__.py index af28c015..9d6a32f4 100644 --- a/extract_msg/custom_attachments/__init__.py +++ b/extract_msg/custom_attachments/__init__.py @@ -6,14 +6,46 @@ CustomAttachmentHandler subclasses will all define the following methods: injectHtml: A method which takes HTML and inserts the + +It should hopefully be completely unnecessary for your code to know what type of +handler it is using, as the abstract base class should give all of the functions +you would typically want. + +If you would like to add your own handler, simply subclass +CustomAttachmentHandler and add it using the `registerHandler` function. """ -from custom_handler import CustomAttachmentHandler -from outlook_signature import OutlookSignature +from typing import List + +from .custom_handler import CustomAttachmentHandler + + +# Create a way to register handlers. +_knownHandlers : List[CustomAttachmentHandler] = [] +# This line is cheating a little bit, but is more efficient than wrapping it in +# a function. +registerHandler = _knownHandlers.append + + +# Import built-in handler modules. THey will all automatically register their +# respecive handler(s). +from .outlook_signature import OutlookSignature + + # Function designed to route to the correct handler. -def getHandler(attachment : 'Attachment'): +def getHandler(attachment : 'Attachment') -> CustomAttachmentHandler: """ - Takes an attachment and uses it to find the correct hanlder. + Takes an attachment and uses it to find the correct handler. Returns an + instance created using the specified attachment. + + :raises NotImplementedError: No handler could be found. + :raises ValueError: A handler was found, but something was wrong with the + attachment data. """ + for handler in _knownHandlers: + if handler.isCorrectHandler(attachment): + return handler(attachment) + + raise NotImplementedError('No valid handler could be found for the attachment. Contact the developers for help.') diff --git a/extract_msg/custom_attachments/outlook_signature.py b/extract_msg/custom_attachments/outlook_signature.py index c3dc4c8a..bfd8afef 100644 --- a/extract_msg/custom_attachments/outlook_signature.py +++ b/extract_msg/custom_attachments/outlook_signature.py @@ -1,5 +1,6 @@ import struct +from . import registerHandler from .custom_handler import CustomAttachmentHandler from ..enums import DVAspect @@ -25,15 +26,21 @@ def __init__(self, attachment : 'Attachment'): oleStream = attachment._getStream('__substg1.0_3701000D/\x01Ole') if not oleStream: raise ValueError('OLE stream could not be found.') - if len(oleStream) != 20: - raise ValueError('OLE stream is the wrong length.') + + # While I have only seen this stream be one length, it could in theory + # be more than one length. So long as it is *at least* 20 bytes, we + # call it valid. + if len(oleStream) < 20: + raise ValueError('OLE stream is too short.') # Unpack and verify the OLE stream. - vals = _ST_OLE.unpack(oleStream) + vals = _ST_OLE.unpack(oleStream[:20]) # Check the version magic. if vals[0] != 0x20000001: raise ValueError('OLE stream has wrong version magic.') - # TODO. + # Check the reserved bytes. + if vals[3] != 0: + raise ValueError('OLE stream has non-zero reserved int.') # Unpack the mailstream and create the HTML tag. vals = _ST_MAILSTREAM.unpack(stream) @@ -43,3 +50,23 @@ def __init__(self, attachment : 'Attachment'): hwStyle = f'height: {self.__x / 100.0:.2f}mm; width: {self.__y / 100.0:.2f}mm;' imgData = f'data:image;base64,{base64.b64encode(self.__data)}'; self.__htmlTag = f''.encode('ascii') + + @classmethod + def isCorrectHandler(cls, attachment : Attachment) -> bool: + if attachment.clsid != '': + return False + + # Check for the required streams. + if not attachment._exists('__substg1.0_3701000D/CONTENTS'): + return False + if not attachment._exists('__substg1.0_3701000D/\x01Ole'): + return False + if not attachment._exists('__substg1.0_3701000D/\x03MailStream'): + return False + + return True + + + + +registerHandler(OutlookSignature) diff --git a/extract_msg/enums.py b/extract_msg/enums.py index aeb6ca1b..93ea5bb1 100644 --- a/extract_msg/enums.py +++ b/extract_msg/enums.py @@ -113,6 +113,7 @@ class AttachmentType(enum.Enum): SIGNED = 3 BROKEN = 4 UNSUPPORTED = 5 + CUSTOM = 6 UNKNOWN = 0xFFFFFFFF diff --git a/extract_msg/msg.py b/extract_msg/msg.py index 5e096a53..fb7e5222 100644 --- a/extract_msg/msg.py +++ b/extract_msg/msg.py @@ -627,10 +627,9 @@ def attachments(self) -> List: # Get the attachments. attachmentDirs = [] prefixLen = self.prefixLen - for dir_ in self.listDir(False, True): - if dir_[prefixLen].startswith('__attach') and \ - dir_[prefixLen] not in attachmentDirs: - attachmentDirs.append(dir_[prefixLen]) + for dir_ in self.listDir(False, True, False): + if dir_[0].startswith('__attach') and dir_[0] not in attachmentDirs: + attachmentDirs.append(dir_[0]) self._attachments = [] @@ -638,7 +637,6 @@ def attachments(self) -> List: try: self._attachments.append(self.attachmentClass(self, attachmentDir)) except (NotImplementedError, UnrecognizedMSGTypeError) as e: - print("Hello") if self.attachmentErrorBehavior != AttachErrorBehavior.THROW: logger.error(f'Error processing attachment at {attachmentDir}') logger.exception(e) From 660b63b9a77a671923466a2c3ce4b0350c4d35b0 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Thu, 1 Dec 2022 00:19:38 -0800 Subject: [PATCH 05/30] More progress. --- extract_msg/attachment.py | 14 +++++----- extract_msg/attachment_base.py | 26 +++++++++++++++++-- .../custom_attachments/custom_handler.py | 22 +++++++++++++++- .../custom_attachments/outlook_signature.py | 23 +++++++++++----- 4 files changed, 69 insertions(+), 16 deletions(-) diff --git a/extract_msg/attachment.py b/extract_msg/attachment.py index 8ee36b7f..c1d7edd0 100644 --- a/extract_msg/attachment.py +++ b/extract_msg/attachment.py @@ -229,13 +229,6 @@ def saveEmbededMessage(self, **kwargs) -> None: """ self.data.save(**kwargs) - @property - def data(self) -> Optional[Union[bytes, 'MSGFile']]: - """ - Returns the attachment data. - """ - return self.__data - @property def customHandler(self) -> Optional[CustomAttachmentHandler]: """ @@ -244,6 +237,13 @@ def customHandler(self) -> Optional[CustomAttachmentHandler]: """ return self.__customHandler + @property + def data(self) -> Optional[Union[bytes, 'MSGFile']]: + """ + Returns the attachment data. + """ + return self.__data + @property def randomFilename(self) -> str: """ diff --git a/extract_msg/attachment_base.py b/extract_msg/attachment_base.py index 8be2d0fb..bc5d6fee 100644 --- a/extract_msg/attachment_base.py +++ b/extract_msg/attachment_base.py @@ -33,7 +33,6 @@ def __init__(self, msg, dir_): self.__dir = dir_ self.__props = Properties(self._getStream('__properties_version1.0'), PropertiesType.ATTACHMENT) self.__namedProperties = NamedProperties(msg.named, self) - self.__clsid = msg._getOleEntry(dir_).clsid or '' def _ensureSet(self, variable, streamID, stringStream = True, **kwargs): """ @@ -275,7 +274,30 @@ def clsid(self) -> str: """ Returns the CLSID for the data stream/storage of the attachment. """ - return self.__clsid + try: + return self.__clsid + except AttributeError: + # Set some default values. + self.__clsid = '00000000-0000-0000-0000-000000000000' + dataStream = None + + # See if we can find the data stream/storage. + if self.type in (AttachmentType.CUSTOM, AttachmentType.MSG): + dataStream = [self.__dir, '__substg1.0_3701000D'] + elif self.type == AttachmentType.DATA: + dataStream = [self.__dir, '__substg1.0_37010102'] + elif self.type == AttachmentType.UNSUPPORTED: + # Special check for custom attachments. + if self.exists('__substg1.0_3701000D'): + dataStream = [self.__dir, '__substg1.0_3701000D'] + elif self.exists('__substg1.0_37010102'): + dataStream = [self.__dir, '__substg1.0_37010102'] + + # If we found the right item, get the CLSID. + if dataStream: + self.__clsid = self.__msg._getOleEntry(dataStream).clsid or '00000000-0000-0000-0000-000000000000' + + return self.__clsid @property def dir(self) -> str: diff --git a/extract_msg/custom_attachments/custom_handler.py b/extract_msg/custom_attachments/custom_handler.py index 196db49a..19891432 100644 --- a/extract_msg/custom_attachments/custom_handler.py +++ b/extract_msg/custom_attachments/custom_handler.py @@ -17,9 +17,29 @@ def isCorrectHandler(cls, attachment : 'Attachment') -> bool: Checks if this is the correct handler for the attachment. """ + @abc.abstractmethod + def injectHTML(self, html : bytes) -> bytes: + """ + Adds the relevent tag, if any, to the HTML for making prepared HTML. + """ + + @property + def attachment(self): + """ + The attachment this handler is associated with. + """ + return self.__att + @property @abc.abstractmethod - def data(self): + def data(self) -> bytes: """ Gets the data for the attachment. """ + + @property + @abc.abstractmethod + def name(self) -> str: + """ + Returns the name to be used when saving the attachment. + """ diff --git a/extract_msg/custom_attachments/outlook_signature.py b/extract_msg/custom_attachments/outlook_signature.py index bfd8afef..52320e66 100644 --- a/extract_msg/custom_attachments/outlook_signature.py +++ b/extract_msg/custom_attachments/outlook_signature.py @@ -5,7 +5,7 @@ from ..enums import DVAspect -_ST_OLE = struct.Struck(''.encode('ascii') @classmethod - def isCorrectHandler(cls, attachment : Attachment) -> bool: - if attachment.clsid != '': + def isCorrectHandler(cls, attachment : 'Attachment') -> bool: + if attachment.clsid != '00000316-0000-0000-C000-000000000046': return False # Check for the required streams. - if not attachment._exists('__substg1.0_3701000D/CONTENTS'): + if not attachment.exists('__substg1.0_3701000D/CONTENTS'): return False - if not attachment._exists('__substg1.0_3701000D/\x01Ole'): + if not attachment.exists('__substg1.0_3701000D/\x01Ole'): return False - if not attachment._exists('__substg1.0_3701000D/\x03MailStream'): + if not attachment.exists('__substg1.0_3701000D/\x03MailStream'): return False return True + def injectHTML(self, html : bytes) -> bytes: + return html # TODO. + + @property + def data(self) -> bytes: + return self.__data + + @property + def name(self) -> str: + return self.attachment.shortFilename + '.bmp' + From e7357acec77a9fcdddaf3d4a4eb174ac73eea10f Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Thu, 1 Dec 2022 00:28:51 -0800 Subject: [PATCH 06/30] Fix ole magic value --- extract_msg/custom_attachments/outlook_signature.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extract_msg/custom_attachments/outlook_signature.py b/extract_msg/custom_attachments/outlook_signature.py index 52320e66..ec07c08b 100644 --- a/extract_msg/custom_attachments/outlook_signature.py +++ b/extract_msg/custom_attachments/outlook_signature.py @@ -36,7 +36,7 @@ def __init__(self, attachment : 'Attachment'): # Unpack and verify the OLE stream. vals = _ST_OLE.unpack(oleStream[:20]) # Check the version magic. - if vals[0] != 0x20000001: + if vals[0] != 0x2000001: raise ValueError('OLE stream has wrong version magic.') # Check the reserved bytes. if vals[3] != 0: From 63e76ede69e5b06c94086811e73ab259d982ae4e Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Thu, 1 Dec 2022 02:23:33 -0800 Subject: [PATCH 07/30] Fix import and readme --- README.rst | 3 ++- extract_msg/custom_attachments/outlook_signature.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a9fad26a..5d386e8d 100644 --- a/README.rst +++ b/README.rst @@ -19,7 +19,7 @@ This module has a Discord server for general discussion. You can find it here: Changelog --------- -- `Changelog `__ +- `Changelog`_ Usage ----- @@ -251,3 +251,4 @@ your access to the newest major version of extract-msg. .. _Ko-fi: https://ko-fi.com/destructione .. _Patreon: https://www.patreon.com/DestructionE .. _msg-explorer: https://pypi.org/project/msg-explorer/ +.. _Changelog: https://github.com/TeamMsgExtractor/msg-extractor/blob/master/CHANGELOG.md diff --git a/extract_msg/custom_attachments/outlook_signature.py b/extract_msg/custom_attachments/outlook_signature.py index ec07c08b..5e02d309 100644 --- a/extract_msg/custom_attachments/outlook_signature.py +++ b/extract_msg/custom_attachments/outlook_signature.py @@ -1,3 +1,4 @@ +import base64 import struct from . import registerHandler From d748c1fde9e254432cdfaa550d55d909c0114504 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Thu, 1 Dec 2022 02:50:43 -0800 Subject: [PATCH 08/30] Started framework for actually saving signatures --- extract_msg/attachment.py | 7 +++++-- extract_msg/custom_attachments/outlook_signature.py | 2 +- extract_msg/message_base.py | 12 ++++++++++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/extract_msg/attachment.py b/extract_msg/attachment.py index c1d7edd0..8fc6bb32 100644 --- a/extract_msg/attachment.py +++ b/extract_msg/attachment.py @@ -45,6 +45,7 @@ def __init__(self, msg, dir_): # Check if we have any custom handlers. If not, it will raise # an error automatically. self.__customHandler = getHandler(self) + self.__data = self.__customHandler.data else: self.__prefix = msg.prefixList + [dir_, '__substg1.0_3701000D'] self.__type = AttachmentType.MSG @@ -80,8 +81,10 @@ def getFilename(self, **kwargs) -> str: # Check if user wants to save the file under the Content-ID. if kwargs.get('contentId', False): filename = self.cid - # If filename is None at this point, use long filename as first - # preference. + # If we are using a custom handler, prefer it's name. + if self.type == AttachmentType.CUSTOM: + filename = self.__customHandler.name + # If we are here, try to get the filename however else we can. if not filename: filename = self.name # Otherwise just make something up! diff --git a/extract_msg/custom_attachments/outlook_signature.py b/extract_msg/custom_attachments/outlook_signature.py index 5e02d309..fc1380fa 100644 --- a/extract_msg/custom_attachments/outlook_signature.py +++ b/extract_msg/custom_attachments/outlook_signature.py @@ -49,7 +49,7 @@ def __init__(self, attachment : 'Attachment'): self.__x = vals[1] self.__y = vals[2] hwStyle = f'height: {self.__x / 100.0:.2f}mm; width: {self.__y / 100.0:.2f}mm;' - imgData = f'data:image;base64,{base64.b64encode(self.__data)}'; + imgData = f'data:image;base64,{base64.b64encode(self.__data)}' self.__htmlTag = f''.encode('ascii') @classmethod diff --git a/extract_msg/message_base.py b/extract_msg/message_base.py index 8068c856..b12b952b 100644 --- a/extract_msg/message_base.py +++ b/extract_msg/message_base.py @@ -19,7 +19,7 @@ from typing import Callable, Dict, Optional, Tuple, Union from . import constants -from .enums import DeencapType, RecipientType +from .enums import AttachmentType, DeencapType, RecipientType from .exceptions import ( DataNotFoundError, DeencapMalformedData, DeencapNotEncapsulated, IncompatibleOptionsError, WKError @@ -1178,8 +1178,16 @@ def htmlBodyPrepared(self) -> Optional[bytes]: if not self.htmlBody: return self.htmlBody + html = self.htmlBody + + # Iterate through all attachments, and inject all of the custom data + # from them. + for x in self.attachments: + if x.type == AttachmentType.CUSTOM: + html = x.customhandler.injectHTML(html) + # Create the BeautifulSoup instance to use. - soup = bs4.BeautifulSoup(self.htmlBody, 'html.parser') + soup = bs4.BeautifulSoup(html, 'html.parser') # Get a list of image tags to see if we can inject into. If the source # of an image starts with "cid:" that means it is one of the attachments From ceb5ed34aaa74f79ac68a3efa58361ae1cdbb18e Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sat, 3 Dec 2022 00:08:53 -0800 Subject: [PATCH 09/30] More progress --- changelog_temp.md | 1 + extract_msg/attachment.py | 4 ++-- extract_msg/attachment_base.py | 6 +++--- extract_msg/message_base.py | 4 ++-- extract_msg/ole_writer.py | 4 ---- 5 files changed, 8 insertions(+), 11 deletions(-) diff --git a/changelog_temp.md b/changelog_temp.md index b8bbe33a..bb6f63f4 100644 --- a/changelog_temp.md +++ b/changelog_temp.md @@ -4,3 +4,4 @@ Temporary location for the changelog entry to ensure it doesn't conflict. * Added new submodule `custom_attachments`. This submodule provides an extendable way to handle custom attachment types, attachment types whose structure and formatting are not defined in the Microsoft documentation for MSG files. * Added new property `AttachmentBase.clsid` which returns the listed CLSID value of the data stream/storage of the attachment. * Changed internal behavior of `MSGFile.attachments`. This should not cause any noticeable changes to the output. +* Removed some debug code that was left behind. diff --git a/extract_msg/attachment.py b/extract_msg/attachment.py index 8fc6bb32..74673304 100644 --- a/extract_msg/attachment.py +++ b/extract_msg/attachment.py @@ -82,7 +82,7 @@ def getFilename(self, **kwargs) -> str: if kwargs.get('contentId', False): filename = self.cid # If we are using a custom handler, prefer it's name. - if self.type == AttachmentType.CUSTOM: + if self.type is AttachmentType.CUSTOM: filename = self.__customHandler.name # If we are here, try to get the filename however else we can. if not filename: @@ -176,7 +176,7 @@ def save(self, **kwargs) -> Optional[Union[str, 'MSGFile']]: fullFilename = customPath / filename - if self.type is AttachmentType.DATA: + if self.type is AttachmentType.DATA or (self.type is AttachmentType.CUSTOM and isinstance(self.__data, bytes)): if _zip: name, ext = os.path.splitext(filename) nameList = _zip.namelist() diff --git a/extract_msg/attachment_base.py b/extract_msg/attachment_base.py index bc5d6fee..ef7c3bef 100644 --- a/extract_msg/attachment_base.py +++ b/extract_msg/attachment_base.py @@ -284,9 +284,9 @@ def clsid(self) -> str: # See if we can find the data stream/storage. if self.type in (AttachmentType.CUSTOM, AttachmentType.MSG): dataStream = [self.__dir, '__substg1.0_3701000D'] - elif self.type == AttachmentType.DATA: + elif self.type is AttachmentType.DATA: dataStream = [self.__dir, '__substg1.0_37010102'] - elif self.type == AttachmentType.UNSUPPORTED: + elif self.type is AttachmentType.UNSUPPORTED: # Special check for custom attachments. if self.exists('__substg1.0_3701000D'): dataStream = [self.__dir, '__substg1.0_3701000D'] @@ -296,7 +296,7 @@ def clsid(self) -> str: # If we found the right item, get the CLSID. if dataStream: self.__clsid = self.__msg._getOleEntry(dataStream).clsid or '00000000-0000-0000-0000-000000000000' - + return self.__clsid @property diff --git a/extract_msg/message_base.py b/extract_msg/message_base.py index b12b952b..8653ae09 100644 --- a/extract_msg/message_base.py +++ b/extract_msg/message_base.py @@ -1183,8 +1183,8 @@ def htmlBodyPrepared(self) -> Optional[bytes]: # Iterate through all attachments, and inject all of the custom data # from them. for x in self.attachments: - if x.type == AttachmentType.CUSTOM: - html = x.customhandler.injectHTML(html) + if x.type is AttachmentType.CUSTOM: + html = x.customHandler.injectHTML(html) # Create the BeautifulSoup instance to use. soup = bs4.BeautifulSoup(html, 'html.parser') diff --git a/extract_msg/ole_writer.py b/extract_msg/ole_writer.py index 49378ee5..52ea73c0 100644 --- a/extract_msg/ole_writer.py +++ b/extract_msg/ole_writer.py @@ -415,9 +415,6 @@ def addOleEntry(self, path, entry : OleDirectoryEntry, data : Optional[bytes] = _dir = self.__dirEntries while len(pathList) > 1: if pathList[0] not in _dir: - print(pathList[0]) - print(_dir) - print(self.__dirEntries) # If no entry has been provided already for the directory, that # is considered a fatal error. raise ValueError('Path not found.') @@ -568,5 +565,4 @@ def _unClsid(clsid : str) -> bytes: int(clsid[30:32], 16), )) except Exception: - print(clsid) raise From 0961da44e8d15b416cb0e7b9ed40968e06adbc9d Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sat, 3 Dec 2022 15:30:01 -0800 Subject: [PATCH 10/30] Added notes and changed signature for CustomAttachmentHandler.injectHtmt --- .../custom_attachments/custom_handler.py | 11 +++++++++- .../custom_attachments/outlook_signature.py | 20 ++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/extract_msg/custom_attachments/custom_handler.py b/extract_msg/custom_attachments/custom_handler.py index 19891432..c844151a 100644 --- a/extract_msg/custom_attachments/custom_handler.py +++ b/extract_msg/custom_attachments/custom_handler.py @@ -1,5 +1,7 @@ import abc +from typing import List, Optional, Tuple + class CustomAttachmentHandler(abc.ABC): """ @@ -18,9 +20,16 @@ def isCorrectHandler(cls, attachment : 'Attachment') -> bool: """ @abc.abstractmethod - def injectHTML(self, html : bytes) -> bytes: + def injectHTML(self, html : bytes, renderedList : Optional[List[str]] = None) -> Tuple[bytes, Optional[List[str]]]: """ Adds the relevent tag, if any, to the HTML for making prepared HTML. + + If this function should do nothing, returns the two arguments without + modification. + + :param html: The HTML body to inject into (if at all). + :param renderedList: The list to use (if needed) of "rendered + characters" which will be returned and matches the HTML returned. """ @property diff --git a/extract_msg/custom_attachments/outlook_signature.py b/extract_msg/custom_attachments/outlook_signature.py index fc1380fa..bcd9c4f9 100644 --- a/extract_msg/custom_attachments/outlook_signature.py +++ b/extract_msg/custom_attachments/outlook_signature.py @@ -1,6 +1,8 @@ import base64 import struct +from typing import List, Optional, Tuple + from . import registerHandler from .custom_handler import CustomAttachmentHandler from ..enums import DVAspect @@ -67,8 +69,24 @@ def isCorrectHandler(cls, attachment : 'Attachment') -> bool: return True - def injectHTML(self, html : bytes) -> bytes: + def injectHTML(self, html : bytes, renderedList : Optional[List[str]] = None) -> Tuple[bytes, Optional[List[str]]]: return html # TODO. + # Here we want to do the following: + # 1. Decode the data: We need to know the encoding to be able to + # figure out what a "rendered character" is. + # 2. Break the data up into rendered characters. This will likely + # require going one character at a time or using beautiful soup + # to first get the body, head, etc. separated. + # 3. If something isn't going to be rendered, shove that whole + # section as part of the next "rendered character" so we can + # fully recombine the data later. + # 4. Find our position and shove our tag onto the front of the next + # rendered character so it's in the correct position. + # 5. Recombine and re-encode the data. + # 6. Return the new bytes and a list of the rendered characters for + # the next function to call to save time. + # 7. The next time one of these functions is called, it will know to + # generate it's own list if the list evaluates as False. @property def data(self) -> bytes: From f9a20e280cd31d3e31c56ce16c8d50b65416eecf Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Mon, 5 Dec 2022 03:42:24 -0800 Subject: [PATCH 11/30] Start of html tokenizer --- .../custom_attachments/outlook_signature.py | 3 ++- extract_msg/custom_attachments/utils.py | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 extract_msg/custom_attachments/utils.py diff --git a/extract_msg/custom_attachments/outlook_signature.py b/extract_msg/custom_attachments/outlook_signature.py index bcd9c4f9..caa2a401 100644 --- a/extract_msg/custom_attachments/outlook_signature.py +++ b/extract_msg/custom_attachments/outlook_signature.py @@ -70,7 +70,7 @@ def isCorrectHandler(cls, attachment : 'Attachment') -> bool: return True def injectHTML(self, html : bytes, renderedList : Optional[List[str]] = None) -> Tuple[bytes, Optional[List[str]]]: - return html # TODO. + return (html, renderedList) # TODO. # Here we want to do the following: # 1. Decode the data: We need to know the encoding to be able to # figure out what a "rendered character" is. @@ -87,6 +87,7 @@ def injectHTML(self, html : bytes, renderedList : Optional[List[str]] = None) -> # the next function to call to save time. # 7. The next time one of these functions is called, it will know to # generate it's own list if the list evaluates as False. + @property def data(self) -> bytes: diff --git a/extract_msg/custom_attachments/utils.py b/extract_msg/custom_attachments/utils.py new file mode 100644 index 00000000..e2a65606 --- /dev/null +++ b/extract_msg/custom_attachments/utils.py @@ -0,0 +1,25 @@ +""" +Utilities for extract-msg that are more specialized for the custom_attachments +submodule than for the main module. +""" + +import bs4 + + +def tokenizeHtml(html : str) -> List[str]: + # Setup a few variables for state tracking. + inTag = False + inEscape = False + inString = False + # Only used when in string. Last character was a backslash. + isBackslash = False + +def htmlSplitRendered(html : bytes) -> List[str]: + """ + Takes html bytes and returns a list of the rendered characters, with data + that is not being rendered being attached to the next rendered character. + """ + # We use bs4 to convert the bytes to a string as accurately as possible. We + # would also use it for tokenizing, but it doesn't allow for a quick and + # easy way to do that and might just be faster to do ourselves. + tokens = tokenizeHtml(bs4.BeautifulSoup(html).decode()) From 10dc680f6731352e43cccb49d1cf1de81e382456 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Thu, 8 Dec 2022 00:53:46 -0800 Subject: [PATCH 12/30] Progress/finished with HTML tokenizer --- extract_msg/custom_attachments/utils.py | 68 +++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/extract_msg/custom_attachments/utils.py b/extract_msg/custom_attachments/utils.py index e2a65606..adc5d8a1 100644 --- a/extract_msg/custom_attachments/utils.py +++ b/extract_msg/custom_attachments/utils.py @@ -5,14 +5,82 @@ import bs4 +from typing import List + def tokenizeHtml(html : str) -> List[str]: # Setup a few variables for state tracking. inTag = False + # Used for tracking escapes starting with &. If your escape ends up at 100 + # characters because it is missing the semicolon, we are going to throw an + # error. inEscape = False inString = False # Only used when in string. Last character was a backslash. isBackslash = False + # Tells which quote type we are in. + isDoubleQuote = False + + tokens = [] + currentToken = '' + + # Finally, let's start breaking things up. Our rules are that if we start a + # quote while in a tag, then we acknowledge it, otherwise it is treated as + # plain text. + for character in html: + # First we need to know our state, as our state determines what how we + # process a character. + if inTag: + currentToken += character + if inString: + if character == '"' and isDoubleQuote: + # If isBackslash then we stay in the quote, otherwise... + isQuote = isBackslash + elif character == "'" and not isDoubleQuote: + # If isBackslash then we stay in the quote, otherwise... + isQuote = isBackslash + elif character == '\\': + isBackslash = not isBackslash + if character != '\\': + isBackslash = False + else: + if character == '>': + inTag = False + tokens.append(currentToken) + currentToken = '' + elif inEscape: + currentToken += character + if len(currentToken) > 99: + raise ValueError('Found escape that was too long (is a ; missing?)') + if character == ';': + tokens.append(currentToken) + currentToken = '' + inEscape = False + elif inString: + # This is an error. We should *never* be in a quote if we are not in + # a tag. + raise ValueError('Found to be inQuote when not in tag.') + else: + # We are currently processing plain text, so let's just handle. + if character == '&': + if currentToken: + tokens.append(currentToken) + currentToken = character + inEscape = True + elif character == '<': + if currentToken: + tokens.append(currentToken) + currentToken = character + inTag = True + inString = False + else: + currentToken += character + + if currentToken: + tokens.append(currentToken) + + return tokens + def htmlSplitRendered(html : bytes) -> List[str]: """ From c6284e5e304e8f6f2ebc7f525b89152bc66a65a2 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sat, 17 Dec 2022 01:32:45 -0800 Subject: [PATCH 13/30] Significant progress, custom images can be injected into HTML. --- extract_msg/custom_attachments/__init__.py | 2 +- ...{outlook_signature.py => outlook_image.py} | 43 +++--- extract_msg/custom_attachments/utils.py | 130 ++++++++++++++++-- extract_msg/exceptions.py | 17 +-- extract_msg/message_base.py | 4 +- 5 files changed, 147 insertions(+), 49 deletions(-) rename extract_msg/custom_attachments/{outlook_signature.py => outlook_image.py} (68%) diff --git a/extract_msg/custom_attachments/__init__.py b/extract_msg/custom_attachments/__init__.py index 9d6a32f4..1b153c6f 100644 --- a/extract_msg/custom_attachments/__init__.py +++ b/extract_msg/custom_attachments/__init__.py @@ -29,7 +29,7 @@ # Import built-in handler modules. THey will all automatically register their # respecive handler(s). -from .outlook_signature import OutlookSignature +from .outlook_image import OutlookImage diff --git a/extract_msg/custom_attachments/outlook_signature.py b/extract_msg/custom_attachments/outlook_image.py similarity index 68% rename from extract_msg/custom_attachments/outlook_signature.py rename to extract_msg/custom_attachments/outlook_image.py index caa2a401..54428599 100644 --- a/extract_msg/custom_attachments/outlook_signature.py +++ b/extract_msg/custom_attachments/outlook_image.py @@ -5,14 +5,16 @@ from . import registerHandler from .custom_handler import CustomAttachmentHandler +from .utils import htmlSplitRendered from ..enums import DVAspect +from ..exceptions import CustomAttachmentError _ST_OLE = struct.Struct(''.encode('ascii') + imgData = f'data:image;base64,{base64.b64encode(self.__data).decode("ascii")}' + self.__htmlTag = f'' @classmethod def isCorrectHandler(cls, attachment : 'Attachment') -> bool: @@ -70,24 +72,17 @@ def isCorrectHandler(cls, attachment : 'Attachment') -> bool: return True def injectHTML(self, html : bytes, renderedList : Optional[List[str]] = None) -> Tuple[bytes, Optional[List[str]]]: - return (html, renderedList) # TODO. - # Here we want to do the following: - # 1. Decode the data: We need to know the encoding to be able to - # figure out what a "rendered character" is. - # 2. Break the data up into rendered characters. This will likely - # require going one character at a time or using beautiful soup - # to first get the body, head, etc. separated. - # 3. If something isn't going to be rendered, shove that whole - # section as part of the next "rendered character" so we can - # fully recombine the data later. - # 4. Find our position and shove our tag onto the front of the next - # rendered character so it's in the correct position. - # 5. Recombine and re-encode the data. - # 6. Return the new bytes and a list of the rendered characters for - # the next function to call to save time. - # 7. The next time one of these functions is called, it will know to - # generate it's own list if the list evaluates as False. - + if not renderedList: + renderedList = htmlSplitRendered(html) + + rp = self.attachment.renderingPosition + + if rp >= len(renderedList): + raise CustomAttachmentError(f'Rendering position beyond calculated number of rendered characters (expected less than {len(renderedList)}, got {rp}).') + + renderedList[rp] = self.__htmlTag + renderedList[rp] + + return (''.join(renderedList).encode('utf-8'), renderedList) @property def data(self) -> bytes: @@ -100,4 +95,4 @@ def name(self) -> str: -registerHandler(OutlookSignature) +registerHandler(OutlookImage) diff --git a/extract_msg/custom_attachments/utils.py b/extract_msg/custom_attachments/utils.py index adc5d8a1..8a61e3ec 100644 --- a/extract_msg/custom_attachments/utils.py +++ b/extract_msg/custom_attachments/utils.py @@ -8,6 +8,125 @@ from typing import List +_WHITESPACE_BREAKERS = ( + ' bool: + """ + Helper function to indicate that a tag breaks a chain of whitespace. + """ + for x in _WHITESPACE_BREAKERS: + if token.startswith(x) and len(token) > len(x) and token[len(x)] in ('>', ' ', '/'): + return True + + return False + + +def _isWhitespaceToken(token : str) -> bool: + if token[0] == '<': + for x in _WHITESPACE_TAGS: + if token.startswith(x) and len(token) > len(x) and token[len(x)] in ('>', ' ', '/'): + return True + elif token in (' ', ' ', ' ', ' ', ' '): + return True + else: + return token.isspace() + + return False + + +def htmlSplitRendered(html : bytes) -> List[str]: + """ + Takes html bytes and returns a list of the rendered characters, with data + that is not being rendered being attached to the next rendered character. + """ + # Unfortunately bs4 didn't seem particularly great for tokenizing, so I did + # my own function that works well enough. First, let's tokenize the html. + tokens = tokenizeHtml(bs4.BeautifulSoup(html, features = 'html.parser').decode()) + + # Next, let's break things down further. + breakDown = [] + for token in tokens: + # We tell what we are looking at by checking the first character of the + # token. If it's a <, then it is an HTML tag. If it is a & then it is an + # escape. Otherwise, it is plain text. For both tags and escapes, just + # dump them into the list. + if token[0] in ('<', '&'): + breakDown.append(token) + else: + # If we are looking at plain text, add it by extending the list. + breakDown.extend(token) + + # Now that we have broken things down further, let's go through and join our + # pieces togethered into rendered tokens. Here is were we actually need to + # know what an html tag is. If it's an escape or just a non-whitespace + # character, we can just shove it onto what we currently have. + current = '' + renderedCharacters = [] + lastWhitespace = None + for item in breakDown: + if item[0] == '&': + if _isWhitespaceToken(item): + if lastWhitespace is None: + lastWhitespace == item + else: + if lastWhitespace is not None and lastWhitespace[0] != '<': + current += lastWhitespace + renderedCharacters.append(current) + current = '' + lastWhitespace = None + current += item + renderedCharacters.append(current) + current = '' + elif item[0] == '<': + if _isWhitespaceToken(item): + # If we are here, add it to current, push current, and set this + # tag as the last whitespace. + current += item + renderedCharacters.append(current) + current = '' + lastWhitespace = item + else: + + # Some tags will break whitespace chains. + if _isWhitespaceBreaker(item): + if lastWhitespace is not None and lastWhitespace[0] != '<': + current += lastWhitespace + renderedCharacters.append(current) + current = '' + lastWhitespace = None + + current += item + else: + # Here is where we handle text, which is not particularly fun. + # Basically if it is whitespace and lastWhitespace is not none, we + # set the whitespace. + if _isWhitespaceToken(item): + if lastWhitespace is None: + lastWhitespace = item + else: + if lastWhitespace is not None and lastWhitespace[0] != '<': + current += lastWhitespace + renderedCharacters.append(current) + current = '' + lastWhitespace = None + current += item + renderedCharacters.append(current) + current = '' + + if current: + renderedCharacters.append(current) + + return renderedCharacters + + def tokenizeHtml(html : str) -> List[str]: # Setup a few variables for state tracking. inTag = False @@ -80,14 +199,3 @@ def tokenizeHtml(html : str) -> List[str]: tokens.append(currentToken) return tokens - - -def htmlSplitRendered(html : bytes) -> List[str]: - """ - Takes html bytes and returns a list of the rendered characters, with data - that is not being rendered being attached to the next rendered character. - """ - # We use bs4 to convert the bytes to a string as accurately as possible. We - # would also use it for tokenizing, but it doesn't allow for a quick and - # easy way to do that and might just be faster to do ourselves. - tokens = tokenizeHtml(bs4.BeautifulSoup(html).decode()) diff --git a/extract_msg/exceptions.py b/extract_msg/exceptions.py index e835df64..1ce1fc88 100644 --- a/extract_msg/exceptions.py +++ b/extract_msg/exceptions.py @@ -17,19 +17,21 @@ class BadHtmlError(ValueError): """ HTML failed to pass validation. """ - pass class ConversionError(Exception): """ An error occured during type conversion. """ - pass + +class CustomAttachmentError(Exception): + """ + A generic error used for issues handling custom attachments. + """ class DataNotFoundError(Exception): """ Requested stream type was unavailable. """ - pass class DeencapMalformedData(Exception): """ @@ -45,7 +47,6 @@ class ExecutableNotFound(Exception): """ Could not find the specified executable. """ - pass class IncompatibleOptionsError(Exception): """ @@ -56,26 +57,22 @@ class InvalidFileFormatError(OSError): """ An Invalid File Format Error occurred. """ - pass class InvaildPropertyIdError(Exception): """ The provided property ID was invalid. """ - pass class InvalidVersionError(Exception): """ The version specified is invalid. """ - pass class StandardViolationError(Exception): """ A critical violation of the MSG standards was detected and could not be recovered from. Recoverable violations will result in log messages instead. """ - pass class TZError(Exception): """ @@ -91,13 +88,11 @@ class UnknownCodepageError(Exception): """ The codepage provided was not one we know of. """ - pass class UnknownTypeError(Exception): """ The type specified is not one that is recognized. """ - pass class UnsupportedMSGTypeError(NotImplementedError): """ @@ -110,10 +105,8 @@ class UnrecognizedMSGTypeError(TypeError): An exception that is raised when the module cannot determine how to properly open a specific class of msg file. """ - pass class WKError(RuntimeError): """ An error occured while running wkhtmltopdf. """ - pass diff --git a/extract_msg/message_base.py b/extract_msg/message_base.py index d747ee1e..2d998242 100644 --- a/extract_msg/message_base.py +++ b/extract_msg/message_base.py @@ -1180,11 +1180,13 @@ def htmlBodyPrepared(self) -> Optional[bytes]: html = self.htmlBody + renderedCharacters = None + # Iterate through all attachments, and inject all of the custom data # from them. for x in self.attachments: if x.type is AttachmentType.CUSTOM: - html = x.customHandler.injectHTML(html) + html, renderedCharacters = x.customHandler.injectHTML(html, renderedCharacters) # Create the BeautifulSoup instance to use. soup = bs4.BeautifulSoup(html, 'html.parser') From ac753158a6c5a6409aa034026a9a06a1b98443ba Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Fri, 23 Dec 2022 21:55:03 -0800 Subject: [PATCH 14/30] Attempted to fix #318 --- CHANGELOG.md | 3 +++ README.rst | 4 ++-- extract_msg/__init__.py | 4 ++-- extract_msg/attachment.py | 26 ++++++++++++++++++++++++++ extract_msg/properties.py | 9 +++++++++ 5 files changed, 42 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2593b2f1..3113d843 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +**v0.38.5** +* Added code to handle a standards violation (from what I can tell, anyways) caused by the attachment not having an `AttachMethod` property. The code will log a warning, attempt to detect the method, and throw a `StandardViolationError` if it fails. + **v0.38.4** * Fix line in `OleWriter` that was causing exporting to fail. * Fixed some issues with the `README`. diff --git a/README.rst b/README.rst index 80d74b9e..33efaec7 100644 --- a/README.rst +++ b/README.rst @@ -234,8 +234,8 @@ your access to the newest major version of extract-msg. .. |License: GPL v3| image:: https://img.shields.io/badge/License-GPLv3-blue.svg :target: LICENSE.txt -.. |PyPI3| image:: https://img.shields.io/badge/pypi-0.38.4-blue.svg - :target: https://pypi.org/project/extract-msg/0.38.4/ +.. |PyPI3| image:: https://img.shields.io/badge/pypi-0.38.5-blue.svg + :target: https://pypi.org/project/extract-msg/0.38.5/ .. |PyPI2| image:: https://img.shields.io/badge/python-3.6+-brightgreen.svg :target: https://www.python.org/downloads/release/python-367/ diff --git a/extract_msg/__init__.py b/extract_msg/__init__.py index a3fedb44..6e0114c5 100644 --- a/extract_msg/__init__.py +++ b/extract_msg/__init__.py @@ -27,8 +27,8 @@ # along with this program. If not, see . __author__ = 'Destiny Peterson & Matthew Walker' -__date__ = '2022-12-03' -__version__ = '0.38.4' +__date__ = '2022-12-23' +__version__ = '0.38.5' import logging diff --git a/extract_msg/attachment.py b/extract_msg/attachment.py index 7a65995e..8be19f53 100644 --- a/extract_msg/attachment.py +++ b/extract_msg/attachment.py @@ -10,6 +10,7 @@ from . import constants from .attachment_base import AttachmentBase from .enums import AttachmentType +from .exceptions import StandardViolationError from .utils import createZipOpen, inputToString, openMsg, prepareFilename @@ -33,6 +34,31 @@ def __init__(self, msg, dir_): """ super().__init__(msg, dir_) + if '37050003' not in self.props: + from .prop import createProp + + logger.warning('Attahcment method property not found on attachment. Code will attempt to guess the type.') + + # Because this condition is actually kind of a violation of the + # standard, we are just going to do this in a dumb way. Basically we + # are going to try to set the attach method *manually* just so I + # don't have to go and modify the following code. + if self.exists('__substg1.0_37010102'): + # Set it as data and call it a day. + propData = b'\x03\x00\x057\x07\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00' + elif self.exists('__substg1.0_3701000D'): + # If it is a folder and we have properties, call it an MSG file. + if self.exists('__substg1.0_3701000D/__properties_version1.0'): + propData = b'\x03\x00\x057\x07\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00' + else: + # Call if custom attachment data. + propData = b'\x03\x00\x057\x07\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00' + else: + # Can't autodetect it, so throw an error. + raise StandardViolationError('Attachment method missing, and it could not be determined automatically.') + + self.props._propDict['37050003'] = createProp(propData) + # Get attachment data. if self.exists('__substg1.0_37010102'): self.__type = AttachmentType.DATA diff --git a/extract_msg/properties.py b/extract_msg/properties.py index 0f993a72..90bddbc5 100644 --- a/extract_msg/properties.py +++ b/extract_msg/properties.py @@ -191,6 +191,15 @@ def props(self) -> Dict: """ return copy.deepcopy(self.__props) + @property + def _propDict(self) -> Dict: + """ + A direct reference to the underlying property dictionary. Used in one + place in the code, and not recommended to be used if you are not a + developer. Use `Properties.props` instead for a safe reference. + """ + return self.__props + @property def rawData(self) -> bytes: """ From f61ab7ef41079c4dc292b6ab1502b7d5cbd0957b Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sun, 1 Jan 2023 16:39:48 -0800 Subject: [PATCH 15/30] Many fixes, started extending OleWriter a bit --- CHANGELOG.md | 3 +++ extract_msg/__init__.py | 2 +- extract_msg/attachment.py | 4 +++- extract_msg/contact.py | 4 ++++ extract_msg/message_base.py | 4 ++-- extract_msg/msg.py | 11 ++++++----- extract_msg/ole_writer.py | 2 +- 7 files changed, 20 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3113d843..0df8c84f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ **v0.38.5** * Added code to handle a standards violation (from what I can tell, anyways) caused by the attachment not having an `AttachMethod` property. The code will log a warning, attempt to detect the method, and throw a `StandardViolationError` if it fails. +* Added additional functions to the `OleWriter` class to make it a lot more functional. +* Fixed up a few docstrings. +* Fixed a few issues in `MSGFile` regarding the `filename` keyword argument. **v0.38.4** * Fix line in `OleWriter` that was causing exporting to fail. diff --git a/extract_msg/__init__.py b/extract_msg/__init__.py index 6e0114c5..38ac8fd4 100644 --- a/extract_msg/__init__.py +++ b/extract_msg/__init__.py @@ -27,7 +27,7 @@ # along with this program. If not, see . __author__ = 'Destiny Peterson & Matthew Walker' -__date__ = '2022-12-23' +__date__ = '2023-01-01' __version__ = '0.38.5' import logging diff --git a/extract_msg/attachment.py b/extract_msg/attachment.py index 8be19f53..74db001f 100644 --- a/extract_msg/attachment.py +++ b/extract_msg/attachment.py @@ -147,9 +147,11 @@ def save(self, **kwargs) -> Optional[Union[str, 'MSGFile']]: to :param zip:. If :param zip: is an instance, :param customPath: will refer to a location inside the zip file. - :param extractEmbedded: If true, causes the attachment, should it be an + :param extractEmbedded: If True, causes the attachment, should it be an embedded MSG file, to save as a .msg file instead of calling it's save function. + :param skipEmbedded: If True, skips saving this attachment if it is an + embedded MSG file. """ # First check if we are skipping embedded messages and stop # *immediately* if we are. diff --git a/extract_msg/contact.py b/extract_msg/contact.py index 941ae3ff..c835af62 100644 --- a/extract_msg/contact.py +++ b/extract_msg/contact.py @@ -19,6 +19,8 @@ def __init__(self, path, **kwargs): :param path: path to the msg file in the system or is the raw msg file. :param prefix: used for extracting embeded msg files inside the main one. Do not set manually unless you know what you are doing. + :param parentMsg: Used for synchronizing named properties instances. Do + not set this unless you know what you are doing. :param attachmentClass: optional, the class the MSGFile object will use for attachments. You probably should not change this value unless you know what you are doing. @@ -28,6 +30,8 @@ def __init__(self, path, **kwargs): be retrieved. :param filename: optional, the filename to be used by default when saving. + :param attachmentErrorBehavior: Optional, the behavior to use in the + event of an error when parsing the attachments. :param overrideEncoding: optional, an encoding to use instead of the one specified by the msg file. Do not report encoding errors caused by this. diff --git a/extract_msg/message_base.py b/extract_msg/message_base.py index dceb5596..841654d4 100644 --- a/extract_msg/message_base.py +++ b/extract_msg/message_base.py @@ -79,8 +79,8 @@ def __init__(self, path, **kwargs): body is desired. The function should return a string for plain text and bytes for HTML. If any problems occur, the function *must* either return None or raise one of the appropriate functions from - extract_msg.exceptions. All other functions must be handled - internally or they will continue. The original deencapsulation + extract_msg.exceptions. All other exceptions must be handled + internally or they will not be caught. The original deencapsulation method will not run if this is set. """ super().__init__(path, **kwargs) diff --git a/extract_msg/msg.py b/extract_msg/msg.py index 937b155c..6fecb2b0 100644 --- a/extract_msg/msg.py +++ b/extract_msg/msg.py @@ -37,10 +37,9 @@ def __init__(self, path, **kwargs): one. Do not set manually unless you know what you are doing. :param parentMsg: Used for synchronizing named properties instances. Do not set this unless you know what you are doing. - :param attachmentClass: Optional, the class the MSGFile object - will use for attachments. You probably should - not change this value unless you know what you - are doing. + :param attachmentClass: Optional, the class the MSGFile object will use + for attachments. You probably should not change this value unless + you know what you are doing. :param delayAttachments: Optional, delays the initialization of attachments until the user attempts to retrieve them. Allows MSG files with bad attachments to be initialized so the other data can @@ -112,6 +111,8 @@ def __init__(self, path, **kwargs): del kwargsCopy['prefix'] if 'parentMsg' in kwargsCopy: del kwargsCopy['parentMsg'] + if 'filename' in kwargsCopy: + del kwargsCopy['filename'] self.__kwargs = kwargsCopy prefixl = [] @@ -134,7 +135,7 @@ def __init__(self, path, **kwargs): self.__prefix = prefix self.__prefixList = prefixl self.__prefixLen = len(prefixl) - if prefix: + if prefix and not filename: filename = self._getStringStream(prefixl[:-1] + ['__substg1.0_3001'], prefix = False) if filename: self.filename = filename diff --git a/extract_msg/ole_writer.py b/extract_msg/ole_writer.py index 4d49c26f..d4aa6792 100644 --- a/extract_msg/ole_writer.py +++ b/extract_msg/ole_writer.py @@ -494,7 +494,7 @@ def fromMsg(self, msg : 'MSGFile') -> None: for x in gen: self.addOleEntry(x, msg._getOleEntry(x, prefix = False), msg._getStream(x, prefix = False)) - def fromOleFile(self, ole : OleFileIO) -> None: + def fromOleFile(self, ole : OleFileIO, rootPath = []) -> None: """ Copies all the streams from the proided OLE file into this writer. """ From 16ab3ac4d322b3bd3803eda462bbcfa4832ed25c Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Mon, 2 Jan 2023 00:30:08 -0800 Subject: [PATCH 16/30] Few more fixes --- CHANGELOG.md | 1 + extract_msg/__init__.py | 2 +- extract_msg/attachment.py | 4 ++-- extract_msg/message_base.py | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0df8c84f..cb547459 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ * Added additional functions to the `OleWriter` class to make it a lot more functional. * Fixed up a few docstrings. * Fixed a few issues in `MSGFile` regarding the `filename` keyword argument. +* Removed the bool check on the `_zip` variable in the save functions. The other check of `createdZip` *requires* that `_zip` is set, making the check redundant. **v0.38.4** * Fix line in `OleWriter` that was causing exporting to fail. diff --git a/extract_msg/__init__.py b/extract_msg/__init__.py index 38ac8fd4..c75b56bd 100644 --- a/extract_msg/__init__.py +++ b/extract_msg/__init__.py @@ -27,7 +27,7 @@ # along with this program. If not, see . __author__ = 'Destiny Peterson & Matthew Walker' -__date__ = '2023-01-01' +__date__ = '2023-01-02' __version__ = '0.38.5' import logging diff --git a/extract_msg/attachment.py b/extract_msg/attachment.py index 74db001f..7ca69539 100644 --- a/extract_msg/attachment.py +++ b/extract_msg/attachment.py @@ -158,7 +158,7 @@ def save(self, **kwargs) -> Optional[Union[str, 'MSGFile']]: if self.type is AttachmentType.MSG and kwargs.get('skipEmbedded'): return None - # Check if the user has specified a custom filename + # Get the filename to use. filename = self.getFilename(**kwargs) # Someone managed to have a null character here, so let's get rid of that @@ -229,7 +229,7 @@ def save(self, **kwargs) -> Optional[Union[str, 'MSGFile']]: f.write(self.__data) # Close the ZipFile if this function created it. - if _zip and createdZip: + if createdZip: _zip.close() return str(fullFilename) diff --git a/extract_msg/message_base.py b/extract_msg/message_base.py index 841654d4..bf9da425 100644 --- a/extract_msg/message_base.py +++ b/extract_msg/message_base.py @@ -908,7 +908,7 @@ def save(self, **kwargs): raise finally: # Close the ZipFile if this function created it. - if _zip and createdZip: + if createdZip: _zip.close() # Return the instance so that functions can easily be chained. From 7073935825ba8c288dbf4a946dd9a9d17a866aa6 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Mon, 9 Jan 2023 18:08:49 -0800 Subject: [PATCH 17/30] Added new argument to `OleWriter.fromOleFile` for more functionality --- CHANGELOG.md | 2 +- extract_msg/__init__.py | 2 +- extract_msg/ole_writer.py | 50 ++++++++++++++++++++++++++++++++++----- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb547459..f057dceb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,9 @@ **v0.38.5** * Added code to handle a standards violation (from what I can tell, anyways) caused by the attachment not having an `AttachMethod` property. The code will log a warning, attempt to detect the method, and throw a `StandardViolationError` if it fails. -* Added additional functions to the `OleWriter` class to make it a lot more functional. * Fixed up a few docstrings. * Fixed a few issues in `MSGFile` regarding the `filename` keyword argument. * Removed the bool check on the `_zip` variable in the save functions. The other check of `createdZip` *requires* that `_zip` is set, making the check redundant. +* Added new argument `rootPath` to `OleWriter.fromOleFile` for saving a specific directory from an OLE file instead of just copying the entire file. That directory will become the root of the new one. **v0.38.4** * Fix line in `OleWriter` that was causing exporting to fail. diff --git a/extract_msg/__init__.py b/extract_msg/__init__.py index c75b56bd..2502b1ba 100644 --- a/extract_msg/__init__.py +++ b/extract_msg/__init__.py @@ -27,7 +27,7 @@ # along with this program. If not, see . __author__ = 'Destiny Peterson & Matthew Walker' -__date__ = '2023-01-02' +__date__ = '2023-01-09' __version__ = '0.38.5' import logging diff --git a/extract_msg/ole_writer.py b/extract_msg/ole_writer.py index d4aa6792..e48b13ac 100644 --- a/extract_msg/ole_writer.py +++ b/extract_msg/ole_writer.py @@ -497,16 +497,54 @@ def fromMsg(self, msg : 'MSGFile') -> None: def fromOleFile(self, ole : OleFileIO, rootPath = []) -> None: """ Copies all the streams from the proided OLE file into this writer. + + NOTE: This method does *not* handle any special rule that may be + required by a format that uses the compound binary file format as a base + when extracting an embedded directory. For example, MSG files require + modification of an embedded properties stream when extracting an + embedded MSG file. + + :param rootPath: A path (accepted by olefile.OleFileIO) to the directory + to use as the root of the file. If not provided, the file root will + be used. + + :raises OSError: If :param rootPath: does not exist in the file. """ - # Copy the clsid of the root entry. - self.__rootClsid = _unClsid(ole.direntries[0].clsid) + rootPath = inputToMsgPath(rootPath) + + # Check if the root path is simply the top of the file. + if rootPath == []: + # Copy the clsid of the root entry. + self.__rootClsid = _unClsid(ole.direntries[0].clsid) + paths = {tuple(x): (x, ole.direntries[ole._find(x)]) for x in ole.listdir(True, True)} + else: + # If it is not the top of the file, we need to do some filtering. + # First get the CLSID from the entry the path points to. + try: + entry = ole.direntries[ole._find(rootPath)] + self.__rootClsid = _unClsid(entry.clsid) + + except OSError as e: + if str(e) == 'file not found': + # Get the cause/context for the original exception and use + # it for the new exception. This hides the exception from + # OleFileIO. + context = e.__cause__ or e.__context__ + raise OSError('Root path was not found in the OLE file.') from context + else: + raise + + paths = {tuple(x[len(rootPath):]): (x, ole.direntries[ole._find(x)]) + for x in ole.listdir(True, True) if len(x) > len(rootPath)} + - # Copy all of the other entries. - for x in ole.listdir(True, True): - entry = ole.direntries[ole._find(x)] + # Copy all of the other entries. Ensure that directories come before + # their streams by sorting the paths. + for x in sorted(paths.keys()): + fullPath, entry = paths[x] if entry.entry_type == DirectoryEntryType.STREAM: - with ole.openstream(x) as f: + with ole.openstream(fullPath) as f: data = f.read() else: data = None From 4e3e8be5c37351e3f19865313f244ab04a3ee29a Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Mon, 9 Jan 2023 20:45:07 -0800 Subject: [PATCH 18/30] Adjusted code to allow for editing streams by adding them again --- CHANGELOG.md | 1 + extract_msg/ole_writer.py | 67 +++++++++++++++++++++++++++++++-------- 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f057dceb..b60a5bef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * Fixed a few issues in `MSGFile` regarding the `filename` keyword argument. * Removed the bool check on the `_zip` variable in the save functions. The other check of `createdZip` *requires* that `_zip` is set, making the check redundant. * Added new argument `rootPath` to `OleWriter.fromOleFile` for saving a specific directory from an OLE file instead of just copying the entire file. That directory will become the root of the new one. +* Adjusted code for `OleWriter` to generate certain values *only* at save time to make them more dynamic. This allows for existing streams to be properly edited (although has issues with allowing storages to be edited). **v0.38.4** * Fix line in `OleWriter` that was causing exporting to fail. diff --git a/extract_msg/ole_writer.py b/extract_msg/ole_writer.py index e48b13ac..5fff220a 100644 --- a/extract_msg/ole_writer.py +++ b/extract_msg/ole_writer.py @@ -1,3 +1,4 @@ +import copy import io import pathlib import re @@ -79,7 +80,10 @@ class OleWriter: [MS-CFB]. """ def __init__(self, rootClsid : bytes = constants.DEFAULT_CLSID): - self.__rootClsid = rootClsid + self.__rootEntry = _DirectoryEntry() + self.__rootEntry.name = "Root Entry" + self.__rootEntry.type = DirectoryEntryType.ROOT_STORAGE + self.__rootEntry.clsid = rootClsid # The root entry will always exist, so this must be at least 1. self.__dirEntryCount = 1 self.__dirEntries = {} @@ -87,6 +91,43 @@ def __init__(self, rootClsid : bytes = constants.DEFAULT_CLSID): self.__largeEntrySectors = 0 self.__numMinifatSectors = 0 + def __recalculateSectors(self): + """ + Recalculates several of the internal variables used for saving that + specify the number of sectors and where things should go. + """ + self.__dirEntryCount = 0 + self.__numMinifatSectors = 0 + self.__largeEntries.clear() + self.__largeEntrySectors = 0 + + count = 0 + for entry in self.__walkEntries(): + self.__dirEntryCount += 1 + if entry.type == DirectoryEntryType.STREAM: + if len(entry.data) < 4096: + self.__numMinifatSectors += ceilDiv(len(entry.data), 64) + else: + self.__largeEntries.append(entry) + self.__largeEntrySectors += ceilDiv(len(entry.data), 512) + + def __walkEntries(self): + """ + Returns a generator that will walk the entires recursively. Each item + returned by it will be a _DirectoryEntry instance. + """ + toProcess = [self.__dirEntries] + yield self.__rootEntry + + while len(toProcess) > 0: + for name, item in toProcess.pop(0).items(): + if name != '::DirectoryEntry': + if isinstance(item, dict): + yield item['::DirectoryEntry'] + toProcess.append(item) + else: + yield item + @property def __numberOfSectors(self) -> int: """ @@ -100,6 +141,9 @@ def __numberOfSectors(self) -> int: @property def __numMinifat(self) -> int: + """ + The number of FAT sectors needed to store the mini FAT. + """ return ceilDiv(self.__numMinifatSectors, 8) def _getFatSectors(self): @@ -125,13 +169,13 @@ def _treeSort(self, startingSector : int) -> List[_DirectoryEntry]: writing the file, returning a list, in order, of the entries to write. """ # First, create the root entry. - root = _DirectoryEntry() - root.name = "Root Entry" - root.type = DirectoryEntryType.ROOT_STORAGE - root.clsid = self.__rootClsid + root = copy.copy(self.__rootEntry) + # Add the location of the start of the mini stream. root.startingSectorLocation = (startingSector + ceilDiv(self.__dirEntryCount, 4) + ceilDiv(self.__numMinifatSectors, 128)) if self.__numMinifat > 0 else 0xFFFFFFFE root.streamSize = self.__numMinifatSectors * 64 + root.childTreeRoot = None + root.childID = 0xFFFFFFFF entries = [root] toProcess = [(root, self.__dirEntries)] @@ -222,6 +266,8 @@ def _writeBeginning(self, f) -> int: :returns: The current sector number after all the data is written. """ + # Recalculate some things needed for saving. + self.__recalculateSectors() # Since we are going to need these multiple times, get them now. numFat, numDifat, totalSectors = self._getFatSectors() @@ -447,11 +493,6 @@ def addOleEntry(self, path, entry : OleDirectoryEntry, data : Optional[bytes] = # Finally, handle the data. newEntry.data = data or b'' - if len(newEntry.data) < 4096: - self.__numMinifatSectors += ceilDiv(len(newEntry.data), 64) - else: - self.__largeEntries.append(newEntry) - self.__largeEntrySectors += ceilDiv(len(newEntry.data), 512) self.__dirEntryCount += 1 @@ -460,7 +501,7 @@ def fromMsg(self, msg : 'MSGFile') -> None: Copies the streams and stream information necessary from the MSG file. """ # Get the root OLE entry's CLSID. - self.__rootClsid = _unClsid(msg._getOleEntry('/').clsid) + self.__rootEntry.clsid = _unClsid(msg._getOleEntry('/').clsid) # List both storages and directories, but sort them by shortest length # first to prevent errors. @@ -515,14 +556,14 @@ def fromOleFile(self, ole : OleFileIO, rootPath = []) -> None: # Check if the root path is simply the top of the file. if rootPath == []: # Copy the clsid of the root entry. - self.__rootClsid = _unClsid(ole.direntries[0].clsid) + self.__rootEntry.clsid = _unClsid(ole.direntries[0].clsid) paths = {tuple(x): (x, ole.direntries[ole._find(x)]) for x in ole.listdir(True, True)} else: # If it is not the top of the file, we need to do some filtering. # First get the CLSID from the entry the path points to. try: entry = ole.direntries[ole._find(rootPath)] - self.__rootClsid = _unClsid(entry.clsid) + self.__rootEntry.clsid = _unClsid(entry.clsid) except OSError as e: if str(e) == 'file not found': From b0d0db549f84fc1a3764f0802545bd016e319923 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 11 Jan 2023 18:23:57 -0800 Subject: [PATCH 19/30] Added code to allow more tolerance in named properties --- CHANGELOG.md | 3 ++- extract_msg/named.py | 42 ++++++++++++++++++++++++++++-------------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b60a5bef..346ee1d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ **v0.38.5** -* Added code to handle a standards violation (from what I can tell, anyways) caused by the attachment not having an `AttachMethod` property. The code will log a warning, attempt to detect the method, and throw a `StandardViolationError` if it fails. +* [[TeamMsgExtractor #320](https://github.com/TeamMsgExtractor/msg-extractor/issues/320)] Changed the way string named properties are handled to allow for the string stream to have some errors and still be parsed. Warnings about these errors will be logged. +* [[TeamMsgExtractor #318](https://github.com/TeamMsgExtractor/msg-extractor/issues/318)]Added code to handle a standards violation (from what I can tell, anyways) caused by the attachment not having an `AttachMethod` property. The code will log a warning, attempt to detect the method, and throw a `StandardViolationError` if it fails. * Fixed up a few docstrings. * Fixed a few issues in `MSGFile` regarding the `filename` keyword argument. * Removed the bool check on the `_zip` variable in the save functions. The other check of `createdZip` *requires* that `_zip` is set, making the check redundant. diff --git a/extract_msg/named.py b/extract_msg/named.py index b5c3d43f..76612944 100644 --- a/extract_msg/named.py +++ b/extract_msg/named.py @@ -21,14 +21,12 @@ def __init__(self, msg): # Get the basic streams. If all are emtpy, then nothing to do. guidStream = self._getStream('__substg1.0_00020102') or self._getStream('__substg1.0_00020102', False) entryStream = self._getStream('__substg1.0_00030102') or self._getStream('__substg1.0_00030102', False) - namesStream = self._getStream('__substg1.0_00040102') or self._getStream('__substg1.0_00040102', False) self.guidStream = guidStream self.entryStream = entryStream - self.namesStream = namesStream + self.namesStream = namesStream = self._getStream('__substg1.0_00040102') or self._getStream('__substg1.0_00040102', False) # The if else stuff is for protection against None. guidStreamLength = len(guidStream) if guidStream else 0 entryStreamLength = len(entryStream) if entryStream else 0 - namesStreamLength = len(namesStream) if namesStream else 0 self.__propertiesDict = {} self.__properties = [] @@ -51,21 +49,11 @@ def __init__(self, msg): entry['guid'] = guids[entry['guid_index']] entries.append(entry) - # Parse the names stream. - names = self.__names - pos = 0 - while pos < namesStreamLength: - nameLength = constants.STNP_NAM.unpack(namesStream[pos:pos+4])[0] - pos += 4 # Move to the start of the entry. - names[pos - 4] = namesStream[pos:pos+nameLength].decode('utf-16-le') # Names are stored in the dictionary as the position they start at. - pos += roundUp(nameLength, 4) - self.entries = entries self.__guids = guids for entry in entries: - streamID = properHex(0x8000 + entry['pid']) - self.__properties.append(StringNamedProperty(entry, names[entry['id']]) if entry['pkind'] == NamedPropertyType.STRING_NAMED else NumericalNamedProperty(entry)) + self.__properties.append(StringNamedProperty(entry, self.__getName(entry['id'])) if entry['pkind'] == NamedPropertyType.STRING_NAMED else NumericalNamedProperty(entry)) for property in self.__properties: name = property.name if isinstance(property, StringNamedProperty) else property.propertyID @@ -80,6 +68,32 @@ def __iter__(self): def __len__(self) -> int: return self.__propertiesDict.__len__() + def __getName(self, offset : int) -> str: + """ + Parses the offset into the named stream and returns the name found. + """ + # We used to parse names by handing it as an array, as specified by the + # documentation, but this new method allows for a little bit more wiggle + # room in terms of what is accepted by the module. + if offset & 3 != 0: + # If the offset is not a multiple of 4, that is an error, but we are + # reducing it to a warning. + logger.warning(f'Malformed named properties detected due to bad offset ({offset}). Ignoring.') + # Check that offset is in string stream. + if offset > len(self.namesStream): + raise ValueError('Failed to parse named property: offset was not in string stream.') + + # Get the length, in bytes, of the string. + length = constants.STNP_NAM.unpack(self.namesStream[offset:offset + 4])[0] + offset += 4 + + # Make sure the string can be read entirely. If it can't, something was + # corrupt. + if offset + length > len(self.namesStream): + raise ValueError(f'Failed to parse named property: length ({length}) of string overflows the string stream. This is probably due to a bad offset.') + + return self.namesStream[offset:offset + length].decode('utf-16-le') + def _getStream(self, filename, prefix = True) -> Optional[bytes]: return self.__msg._getStream([self.__dir, filename], prefix = prefix) From 0778bd217a8ce526c42923b84a7c0d2282445cf4 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Thu, 12 Jan 2023 07:47:25 -0800 Subject: [PATCH 20/30] Reverted change that I made (I broke code thinking I was fixing it) --- CHANGELOG.md | 1 - extract_msg/attachment.py | 2 +- extract_msg/message_base.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 346ee1d9..d1aea157 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,6 @@ * [[TeamMsgExtractor #318](https://github.com/TeamMsgExtractor/msg-extractor/issues/318)]Added code to handle a standards violation (from what I can tell, anyways) caused by the attachment not having an `AttachMethod` property. The code will log a warning, attempt to detect the method, and throw a `StandardViolationError` if it fails. * Fixed up a few docstrings. * Fixed a few issues in `MSGFile` regarding the `filename` keyword argument. -* Removed the bool check on the `_zip` variable in the save functions. The other check of `createdZip` *requires* that `_zip` is set, making the check redundant. * Added new argument `rootPath` to `OleWriter.fromOleFile` for saving a specific directory from an OLE file instead of just copying the entire file. That directory will become the root of the new one. * Adjusted code for `OleWriter` to generate certain values *only* at save time to make them more dynamic. This allows for existing streams to be properly edited (although has issues with allowing storages to be edited). diff --git a/extract_msg/attachment.py b/extract_msg/attachment.py index 7ca69539..cf02bc82 100644 --- a/extract_msg/attachment.py +++ b/extract_msg/attachment.py @@ -229,7 +229,7 @@ def save(self, **kwargs) -> Optional[Union[str, 'MSGFile']]: f.write(self.__data) # Close the ZipFile if this function created it. - if createdZip: + if _zip and createdZip: _zip.close() return str(fullFilename) diff --git a/extract_msg/message_base.py b/extract_msg/message_base.py index bf9da425..841654d4 100644 --- a/extract_msg/message_base.py +++ b/extract_msg/message_base.py @@ -908,7 +908,7 @@ def save(self, **kwargs): raise finally: # Close the ZipFile if this function created it. - if createdZip: + if _zip and createdZip: _zip.close() # Return the instance so that functions can easily be chained. From c82dc920631e82eb3ed75dfa5caea34ec407bd35 Mon Sep 17 00:00:00 2001 From: Donald Ness Date: Thu, 12 Jan 2023 09:44:07 -0600 Subject: [PATCH 21/30] Fix cases where contact headers have missing values --- extract_msg/contact.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/extract_msg/contact.py b/extract_msg/contact.py index c835af62..f694f178 100644 --- a/extract_msg/contact.py +++ b/extract_msg/contact.py @@ -689,11 +689,13 @@ def headerFormatProperties(self) -> constants.HEADER_FORMAT_TYPE: If you class should not do *any* header injection, return None from this property. """ - def strListToStr(inp : Union[str, List[str]]): + def strListToStr(inp : Union[str, None, List[str]]): """ Small internal function for things that may return a string or list. """ - if isinstance(inp, str): + if inp is None: + return None + elif isinstance(inp, str): return inp else: return ', '.join(inp) @@ -750,7 +752,7 @@ def strListToStr(inp : Union[str, List[str]]): 'Anniversary': self.weddingAnniversary.__format__('%B %d, %Y') if self.weddingAnniversaryLocal else None, 'Spouse/Partner': self.spouseName, 'Profession': self.profession, - 'Children': ', '.join(self.childrensNames), + 'Children': strListToStr(self.childrensNames), 'Hobbies': self.hobbies, 'Assistant': self.assistant, 'Web Page': self.webpageUrl, From c7dc99f01483675f4d9fda236b44dc9b2fa3b2be Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sat, 14 Jan 2023 13:10:56 -0800 Subject: [PATCH 22/30] Minor changes to pull request for contact fixes --- extract_msg/contact.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/extract_msg/contact.py b/extract_msg/contact.py index f694f178..104411d0 100644 --- a/extract_msg/contact.py +++ b/extract_msg/contact.py @@ -689,13 +689,11 @@ def headerFormatProperties(self) -> constants.HEADER_FORMAT_TYPE: If you class should not do *any* header injection, return None from this property. """ - def strListToStr(inp : Union[str, None, List[str]]): + def strListToStr(inp : Optional[Union[str, List[str]]]): """ Small internal function for things that may return a string or list. """ - if inp is None: - return None - elif isinstance(inp, str): + if inp is None or isinstance(inp, str): return inp else: return ', '.join(inp) From 11816cbbbc45cbb64f61452d8bff68a8297db861 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sat, 14 Jan 2023 20:30:36 -0800 Subject: [PATCH 23/30] Finished a large amount of the new functions. Some still need testing --- CHANGELOG.md | 16 +- extract_msg/__init__.py | 4 +- extract_msg/constants.py | 3 + extract_msg/named.py | 2 +- extract_msg/ole_writer.py | 380 +++++++++++++++++++++++++++++++++----- extract_msg/utils.py | 32 +++- 6 files changed, 381 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1aea157..a848e734 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,20 @@ -**v0.38.5** +**v0.39.0** +* [[TeamMsgExtractor #318](https://github.com/TeamMsgExtractor/msg-extractor/issues/318)] Added code to handle a standards violation (from what I can tell, anyways) caused by the attachment not having an `AttachMethod` property. The code will log a warning, attempt to detect the method, and throw a `StandardViolationError` if it fails. * [[TeamMsgExtractor #320](https://github.com/TeamMsgExtractor/msg-extractor/issues/320)] Changed the way string named properties are handled to allow for the string stream to have some errors and still be parsed. Warnings about these errors will be logged. -* [[TeamMsgExtractor #318](https://github.com/TeamMsgExtractor/msg-extractor/issues/318)]Added code to handle a standards violation (from what I can tell, anyways) caused by the attachment not having an `AttachMethod` property. The code will log a warning, attempt to detect the method, and throw a `StandardViolationError` if it fails. +* [[TeamMsgExtractor #326](https://github.com/TeamMsgExtractor/msg-extractor/issues/326)] Fixed a bug that could cause some files to error when exporting. +* Fixed an issue where creation and modification times were not being copied to the new OLE file created by `OleWriter`. * Fixed up a few docstrings. * Fixed a few issues in `MSGFile` regarding the `filename` keyword argument. * Added new argument `rootPath` to `OleWriter.fromOleFile` for saving a specific directory from an OLE file instead of just copying the entire file. That directory will become the root of the new one. * Adjusted code for `OleWriter` to generate certain values *only* at save time to make them more dynamic. This allows for existing streams to be properly edited (although has issues with allowing storages to be edited). +* Added new function `OleWriter.deleteEntry` to remove an entry that was already added. If the entry is a storage, all children will be removed too. +* Added new function `OleWriter.editEntry` to edit an entry that was already added. +* Added new function `OleWriter.addEntry` to add a new entry to the writer without an `OleFileIO` instance. Properties of the entry are instead set using the same keyword arguments as described in `OleWriter.editEntry`. +* Changed `_DirectoryEntry` to `DirectoryEntry` to make the more finalized version public. Access to the originals that the `OleWriter` class creates should never happen, instead copies should be returned to ensure the behavior is as expected. +* Added new function `OleWriter.getEntry` which returns a copy of the `DirectoryEntry` instance for that stream or storage in the writer. Use this function to see the current internal state of an entry. +* Added new function `OleWriter.renameEntry` which allows the user to rename a stream or storage (in place). This only changes it's direct name and not it's location in the new OLE file. +* Added a small amount of path validation to `inputToMsgPath` which is used in a lot of places where user input for a path is accepted. It ensures illegal characters don't exist and that the path segments (each name for a storage or stream) are less than 32 characters. This will be most helpful for `OleWriter`. +* Added *many* internal helper functions to `OleWriter` to make extensions easier and consolidate common code. **v0.38.4** * Fix line in `OleWriter` that was causing exporting to fail. @@ -26,7 +36,7 @@ * Added function `MSGFile.export` which copies all streams and storages from an MSG file into a new file. This can "clone" an MSG file or be used for extracting an MSG file that is embedded inside of another. * Added hidden function to `MSGFile` for getting the `OleDirectoryEntry` for a storage or stream. This is mainly for use by the `OleWriter` class. * Added option `extractEmbedded` to `Attachment.save` (`--extract-embedded` on the command line) which causes embedded MSG files to be extracted instead of running their save methods. -* Fixed minor issues with `utils.inputToMsgPath` (renamed from `utils.inputToMsgpath`). +* Fixed minor issues with `utils.inputToMsgPath` (renamed from `utils.inputToMsgPath`). * Renamed `utils.msgpathToString` to `utils.msgPathToString`. * Made some of the module requirements a little more strict to better version control. I'll be trying to make periodic checks for updates to the dependency packages and make sure that new versions are compatible before changing the allowed versions, while also trying to keep the requirements a bit flexible. diff --git a/extract_msg/__init__.py b/extract_msg/__init__.py index 2502b1ba..c7b19b8e 100644 --- a/extract_msg/__init__.py +++ b/extract_msg/__init__.py @@ -27,8 +27,8 @@ # along with this program. If not, see . __author__ = 'Destiny Peterson & Matthew Walker' -__date__ = '2023-01-09' -__version__ = '0.38.5' +__date__ = '2023-01-14' +__version__ = '0.39.0' import logging diff --git a/extract_msg/constants.py b/extract_msg/constants.py index 9f89cdad..697639ba 100644 --- a/extract_msg/constants.py +++ b/extract_msg/constants.py @@ -50,6 +50,9 @@ # This is used in the workaround for decoding issues in RTFDE. We find `\bin` # sections and try to remove all of them to help with the decoding. RE_BIN = re.compile(br'\\bin([0-9]+) ?') +# Used in the vaildation of OLE paths. Any of these characters in a name make it +# invalid. +RE_INVALID_OLE_PATH = re.compile(r'[:/\\!]') FIXED_LENGTH_PROPS = ( 0x0000, diff --git a/extract_msg/named.py b/extract_msg/named.py index 76612944..be22cb19 100644 --- a/extract_msg/named.py +++ b/extract_msg/named.py @@ -23,7 +23,7 @@ def __init__(self, msg): entryStream = self._getStream('__substg1.0_00030102') or self._getStream('__substg1.0_00030102', False) self.guidStream = guidStream self.entryStream = entryStream - self.namesStream = namesStream = self._getStream('__substg1.0_00040102') or self._getStream('__substg1.0_00040102', False) + self.namesStream = self._getStream('__substg1.0_00040102') or self._getStream('__substg1.0_00040102', False) # The if else stuff is for protection against None. guidStreamLength = len(guidStream) if guidStream else 0 entryStreamLength = len(entryStream) if entryStream else 0 diff --git a/extract_msg/ole_writer.py b/extract_msg/ole_writer.py index 5fff220a..7d78960c 100644 --- a/extract_msg/ole_writer.py +++ b/extract_msg/ole_writer.py @@ -3,23 +3,24 @@ import pathlib import re -from typing import List, Optional, Union +from typing import Dict, List, Optional, Tuple, Union from . import constants from .enums import Color, DirectoryEntryType -from .utils import ceilDiv, inputToMsgPath +from .utils import ceilDiv, dictGetCasedKey, inputToMsgPath from olefile.olefile import OleDirectoryEntry, OleFileIO from red_black_dict_mod import RedBlackTree -class _DirectoryEntry: +class DirectoryEntry: """ - Hidden class, will probably be modified later. + An internal representation of a stream or storage in the OleWriter. + Originals should be inaccessible outside of the class. """ name : str = '' - rightChild : '_DirectoryEntry' = None - leftChild : '_DirectoryEntry' = None - childTreeRoot : '_DirectoryEntry' = None + rightChild : 'DirectoryEntry' = None + leftChild : 'DirectoryEntry' = None + childTreeRoot : 'DirectoryEntry' = None stateBits : int = 0 creationTime : int = 0 modifiedTime : int = 0 @@ -36,7 +37,7 @@ class _DirectoryEntry: startingSectorLocation : int = 0 color : Color = Color.BLACK - clsid : bytes = b'' + clsid : bytes = b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' data : bytes = b'' def __init__(self): @@ -80,7 +81,7 @@ class OleWriter: [MS-CFB]. """ def __init__(self, rootClsid : bytes = constants.DEFAULT_CLSID): - self.__rootEntry = _DirectoryEntry() + self.__rootEntry = DirectoryEntry() self.__rootEntry.name = "Root Entry" self.__rootEntry.type = DirectoryEntryType.ROOT_STORAGE self.__rootEntry.clsid = rootClsid @@ -91,6 +92,127 @@ def __init__(self, rootClsid : bytes = constants.DEFAULT_CLSID): self.__largeEntrySectors = 0 self.__numMinifatSectors = 0 + def __getContainingStorage(self, path : List[str], entryExists : bool = True, create : bool = False) -> Dict: + """ + Finds the storage dict internally where the entry specified by + :param path: would be created. If :param create: is True, missing + storages will be created with default settings. + + :param entryExists: If True, throws an error when the requested entry + does not yet exist. + :param create: If True, creates missing storages with default settings. + + :raises OSError: If :param create: is False and the path could not be + found. Also raised if :param entryExists: is True and the requested + entry does not exist. + :raises ValueError: Tried to access an interal stream or tried to use + both the create option and the entryExists option as True. + + :returns: The storage dict that the entry is in. + """ + # Quick check for incompatability between create and entryExists. + if create and entryExists: + raise ValueError(':param create: and :param entryExists: cannot both be True (an entry cannot exist if it is being created).') + + # Check that the path is not an internal entry. + if '::directoryentry' in map(str.lower, path): + raise ValueError('Found internal name in path.') + + _dir = self.__dirEntries + + for index, name in enumerate(path[:-1]): + # If no entry in the current stream matches the path, raise an + # OSError, *unless* the option to create storages is True. + if name.lower() not in map(str.lower, _dir.keys()): + if create: + self.addEntry(path[:index + 1], storage = True) + else: + raise OSError('Entry not found.') + _dir = _dir[dictGetCasedKey(_dir, name)] + + # If the current item is not a storage and we have more to the path, + # raise an OSError. + if not isinstance(_dir, dict): + raise OSError('Attempted to access children of a stream.') + + if entryExists and path[-1].lower() not in map(str.lower, _dir.keys()): + raise OSError('Entry not found.') + + return _dir + + def __getEntry(self, path : List[str]) -> DirectoryEntry: + """ + Finds and returns an existing DirectoryEntry instance in the writer. + + :raises OSError: If the entry does not exist. + :raises ValueError: If access to an internal item is attempted. + """ + _dir = self.__getContainingStorage(path) + item = _dir[dictGetCasedKey(_dir, path[-1])] + if isinstance(item, dict): + return item['::DirectoryEntry'] + else: + return item + + def __modifyEntry(self, entry : DirectoryEntry, **kwargs): + """ + Edits the DirectoryEntry with the data provided. Common code used for + :method addEntry: and :method editEntry:. + + :raises ValueError: Some part of the data given to modify the various + properties was invalid. See the the listed methods for details. + """ + # Extract the arguments. + data = kwargs.get('data') + clsid = kwargs.get('clsid') + creationTime = kwargs.get('creationTime') + modifiedTime = kwargs.get('modifiedTime') + stateBits = kwargs.get('stateBits') + + # I don't like that I have repeated if statements for checking each of + # the arguments, but I need to make sure nothing changes if something is + # invalid. + if data is not None: + if entry.type is not DirectoryEntryType.STREAM: + raise ValueError('Cannot set the data of a storage object.') + if not isinstance(data, bytes): + raise ValueError('Data must be a bytes instance if set.') + + if clsid is not None: + if not isinstance(clsid, bytes): + raise ValueError('CLSID must be bytes.') + if len(clsid) != 16: + raise ValueError('CLSID must be 16 bytes.') + + if creationTime is not None: + if entry.type is DirectoryEntryType.STREAM: + raise ValueError('Modification of creation time cannot be done on a stream.') + if not isinstance(creationTime, int) or creationTime < 0 or creationTime > 0xFFFFFFFFFFFFFFFF: + raise ValueError('Creation time must be a positive 8 byte int.') + + if modifiedTime is not None: + if entry.type is DirectoryEntryType.STREAM: + raise ValueError('Modification of modified time cannot be done on a stream.') + if not isinstance(modifiedTime, int) or modifiedTime < 0 or modifiedTime > 0xFFFFFFFFFFFFFFFF: + raise ValueError('Modified time must be a positive 8 byte int.') + + if stateBits is not None: + if not isinstance(stateBits, int) or stateBits < 0 or stateBits > 0xFFFFFFFF: + raise ValueError('State bits must be a positive 4 byte int.') + + + # Now that all our checks have passed, let's set our data. + if data is not None: + entry.data = data + if clsid is not None: + entry.clsid = clsid + if creationTime is not None: + entry.creationTime = creationTime + if modifiedTime is not None: + entry.modifiedTime = modifiedTime + if stateBits is not None: + entry.stateBits = stateBits + def __recalculateSectors(self): """ Recalculates several of the internal variables used for saving that @@ -114,7 +236,7 @@ def __recalculateSectors(self): def __walkEntries(self): """ Returns a generator that will walk the entires recursively. Each item - returned by it will be a _DirectoryEntry instance. + returned by it will be a DirectoryEntry instance. """ toProcess = [self.__dirEntries] yield self.__rootEntry @@ -146,7 +268,22 @@ def __numMinifat(self) -> int: """ return ceilDiv(self.__numMinifatSectors, 8) - def _getFatSectors(self): + def _cleanupEntries(self) -> None: + """ + Cleans up the node connections by walking the tree and removing + references that were added during writing. + """ + self.__largeEntries.clear() + for entry in self.__walkEntries(): + entry.id = -1 + entry.leftChild = None + entry.rightChild = None + entry.childTreeRoot = None + entry.leftSiblingID = 0xFFFFFFFF + entry.rightSiblingID = 0xFFFFFFFF + entry.childID = 0xFFFFFFFF + + def _getFatSectors(self) -> Tuple[int, int, int]: """ Returns a tuple containing the number of FAT sectors, the number of DIFAT sectors, and the total number of sectors the saved file will have. @@ -163,7 +300,7 @@ def _getFatSectors(self): return (numFat, numDifat, self.__numberOfSectors + numDifat + numFat) - def _treeSort(self, startingSector : int) -> List[_DirectoryEntry]: + def _treeSort(self, startingSector : int) -> List[DirectoryEntry]: """ Uses red-black trees to sort the internal data in preparation for writing the file, returning a list, in order, of the entries to write. @@ -197,9 +334,9 @@ def _treeSort(self, startingSector : int) -> List[_DirectoryEntry]: # the processing list. if isinstance(val, dict): toProcess.append((val['::DirectoryEntry'], val)) - entries.append(val['::DirectoryEntry']) - else: - entries.append(val) + val = val['::DirectoryEntry'] + + entries.append(val) # Add the data to the tree. tree.add((len(name), name.upper()), val) @@ -213,23 +350,15 @@ def _treeSort(self, startingSector : int) -> List[_DirectoryEntry]: for node in tree.in_order(): item = node.value # Set the color immediately. - if isinstance(item, dict): - item = item['::DirectoryEntry'] item.color = Color.BLACK if node.is_black else Color.RED + if node.left: - val = node.left.value - if isinstance(val, _DirectoryEntry): - item.leftChild = val - else: - item.leftChild = val['::DirectoryEntry'] + item.leftChild = node.left.value else: item.leftChild = None + if node.right: - val = node.right.value - if isinstance(val, _DirectoryEntry): - item.rightChild = val - else: - item.rightChild = val['::DirectoryEntry'] + item.rightChild = node.right.value else: item.rightChild = None @@ -391,7 +520,7 @@ def _writeBeginning(self, f) -> int: # Finally, return the current sector index for use in other places. return numDifat + numFat - def _writeDirectoryEntries(self, f, startingSector : int) -> List[_DirectoryEntry]: + def _writeDirectoryEntries(self, f, startingSector : int) -> List[DirectoryEntry]: """ Writes out all the directory entries. Returns the list generated. """ @@ -403,7 +532,7 @@ def _writeDirectoryEntries(self, f, startingSector : int) -> List[_DirectoryEntr return entries - def _writeDirectoryEntry(self, f, entry : _DirectoryEntry) -> None: + def _writeDirectoryEntry(self, f, entry : DirectoryEntry) -> None: """ Writes the directory entry to the file f. """ @@ -419,7 +548,7 @@ def _writeFinal(self, f) -> None: if len(x.data) & 511: f.write(b'\x00' * (512 - (len(x.data) & 511))) - def _writeMini(self, f, entries : List[_DirectoryEntry]) -> None: + def _writeMini(self, f, entries : List[DirectoryEntry]) -> None: """ Writes the mini FAT followed by the full mini stream. """ @@ -449,43 +578,77 @@ def _writeMini(self, f, entries : List[_DirectoryEntry]) -> None: if self.__numMinifatSectors & 7: f.write((b'\x00' * 64) * (8 - (self.__numMinifatSectors & 7))) + def addEntry(self, path, data : bytes = None, storage : bool = False, **kwargs): + """ + Adds an entry to the OleWriter instance at the path specified, adding + storages with default settings where necessary. If the entry is not a + storage, :param data: *must* be set. + + :param path: The path to add the entry at. Must not contain a path part + that is an already added stream. + :param data: The bytes for a stream. + :param storage: If True, the entry to add is a storage. Otherwise, the + entry is a stream. + :param clsid: The CLSID for the stream/storage. Must a a bytes instance + that is 16 bytes long. + :param creationTime: An 8 byte filetime int. Sets the creation time of + the entry. Not applicable to streams. + :param modifiedTime: An 8 byte filetime int. Sets the modification time + of the entry. Not applicable to streams. + :param stateBits: A 4 byte int. Sets the state bits, user-defined flags, + of the entry. For a stream, this *SHOULD* be unset. + + :raises OSError: A stream was found on the path before the end. + :raises ValueError: Attempts to access an internal item. + """ + path = inputToMsgPath(path) + # First, find the current place in our dict to add the item. + _dir = self.__getContainingStorage(path, False) + # Now, check that the item *is not* already in our dict, as that would + # cause problems. + if path[-1].lower() in map(str.lower, _dir.keys()): + raise OSError('Cannot add an entry that already exists.') + + # Create a new entry with basic data and insert it. + entry = DirectoryEntry() + entry.type = DirectoryEntryType.STORAGE if storage else DirectoryEntryType.STREAM + entry.name = path[-1] + self.__modifyEntry(entry, data = data, **kwargs) + def addOleEntry(self, path, entry : OleDirectoryEntry, data : Optional[bytes] = None) -> None: """ Uses the entry provided to add the data to the writer. - :raises ValueError: Tried to add an entry to a path that has not yet - been added. + :raises OSError: Tried to add an entry to a path that has not yet + been added, tried to add as a child of a stream, or tried to add an + entry where one already exists under the same name. """ - pathList = inputToMsgPath(path) + path = inputToMsgPath(path) # First, find the current place in our dict to add the item. - _dir = self.__dirEntries - while len(pathList) > 1: - if pathList[0] not in _dir: - print(pathList[0]) - print(_dir) - print(self.__dirEntries) - # If no entry has been provided already for the directory, that - # is considered a fatal error. - raise ValueError('Path not found.') - _dir = _dir[pathList[0]] - pathList.pop(0) + _dir = self.__getContainingStorage(path, False) + # Now, check that the item *is not* already in our dict, as that would + # cause problems. + if path[-1].lower() in map(str.lower, _dir.keys()): + raise OSError('Cannot add an entry that already exists.') # Now that we are in the right place, add our data. - newEntry = _DirectoryEntry() + newEntry = DirectoryEntry() if entry.entry_type == DirectoryEntryType.STORAGE: # Handle a storage entry. # First add the dict to our tree of items. - _dir[pathList[0]] = {'::DirectoryEntry': newEntry} + _dir[path[-1]] = {'::DirectoryEntry': newEntry} # Finally, setup the values for the stream. newEntry.name = entry.name newEntry.type = DirectoryEntryType.STORAGE newEntry.clsid = _unClsid(entry.clsid) newEntry.stateBits = entry.dwUserFlags + newEntry.creationTime = entry.createTime + newEntry.modifiedTime = entry.modifyTime else: # Handle a stream entry. # First add the entry to out dict of entries. - _dir[pathList[0]] = newEntry + _dir[path[-1]] = newEntry newEntry.name = entry.name newEntry.type = DirectoryEntryType.STREAM newEntry.clsid = _unClsid(entry.clsid) @@ -496,6 +659,60 @@ def addOleEntry(self, path, entry : OleDirectoryEntry, data : Optional[bytes] = self.__dirEntryCount += 1 + def deleteEntry(self, path) -> None: + """ + Deletes the entry specified by :param path:, including all children. + + :raises OSError: If the entry does not exist or a part of the path that + is not the last was a stream. + :raises ValueError: Attempted to delete an internal data stream. + """ + path = inputToMsgPath(path) + # Get the containing storage for the entry. + _dir = self.__getContainingStorage(path) + + # We need to protect against deliberately trying to break the code, so + # make sure someone cannot simply delete the storage entry. + if path[-1] == '::DirectoryEntry': + raise ValueError('Attempted to delete a storage directory entry directly.') + + # The garbage collector will take care of all the loose items, so just + # remove the entry. Also, once again we deal with the case insensitive + # nature of the path. Even though comparisons are case insensitive, the + # path does remember the case used. + del _dir[dictGetCasedKey(_dir, path[-1])] + + def editEntry(self, path, **kwargs) -> None: + """ + Used to edit values of an entry by setting the specific kwargs. Set a + value to something other than None to set it. + + :param data: The data of a stream. Will error if used for something + other than a stream. + :param clsid: The CLSID for the stream/storage. Must a a bytes instance + that is 16 bytes long. + :param creationTime: An 8 byte filetime int. Sets the creation time of + the entry. Not applicable to streams. + :param modifiedTime: An 8 byte filetime int. Sets the modification time + of the entry. Not applicable to streams. + :param stateBits: A 4 byte int. Sets the state bits, user-defined flags, + of the entry. For a stream, this *SHOULD* be unset. + + + To convert a 32 character hexadecial CLSID into the bytes for this + function, the _unClsid function in the ole_writer submodule can be used. + + :raises OSError: The entry does not exist in the file. + :raises TypeError: Attempted to modify the bytes of a storage. + :raises ValueError: The type of a parameter was wrong, or the data of a + parameter was invalid. + """ + # First, find our entry to edit. + entry = self.__getEntry(path) + + # Send it to be modified using the arguments given. + self.__modifyEntry(entry, **kwargs) + def fromMsg(self, msg : 'MSGFile') -> None: """ Copies the streams and stream information necessary from the MSG file. @@ -592,6 +809,70 @@ def fromOleFile(self, ole : OleFileIO, rootPath = []) -> None: self.addOleEntry(x, entry, data) + def getEntry(self, path) -> DirectoryEntry: + """ + Finds and returns a copy of an existing DirectoryEntry instance in the + writer. Use this method to check the internal status of an entry. + + :raises OSError: If the entry does not exist. + :raises ValueError: If access to an internal item is attempted. + """ + return copy.copy(self.__getEntry(inputToMsgPath(path))) + + def listDir(self, streams = True, storages = False) -> List[List[str]]: + """ + Returns a list of the specified items currently in the writter. + """ + # TODO TODO TODO TODO + + def renameEntry(self, path, newName : str): + """ + Changes the name of an entry, leaving it in it's current position. + + :raises OSError: If the entry does not exist or an entry with the new + name already exists, + :raises ValueError: If access to an internal item is attempted or the + new name provided is invalid. + """ + # First, validate the new name. + if not isinstance(newName, str): + raise ValueError('New name must be a string.') + if constants.RE_INVALID_OLE_PATH.search(newName): + raise ValueError('Invalid character(s) in new name. Must not contain the following characters: \\//!:') + if len(newName) > 31: + raise ValueError('New name must be less than 32 characters.') + + # Get the storage for our entry. Entry *must* exist. + _dir = self.__getContainingStorage(inputToMsgPath(path)) + + # See if an item in the storage already has that new name. + if newName.lower() in map(str.lower, _dir.keys()): + raise OSError('An entry with the new name already exists.') + + # Get the original name. + originalName = dictGetCasedKey(_dir, path[-1]) + + # Get the entry to change. + entry = _dir[originalName] + if isinstance(entry, dict): + dirData = entry + entry = entry['::DirectoryEntry'] + else: + dirData = None + + # Change the name on the entry first. + entry.name = newName + + # Now, we need to remove the item from the current storage and add it + # back with the new name. + del _dir[originalName] + + if dirData is None: + _dir[newName] = entry + else: + _dir[newName] = dirData + + def write(self, path) -> None: """ Writes the data to the path specified. If :param path: has a write @@ -609,12 +890,15 @@ def write(self, path) -> None: # Make sure we close the file after everything, especially if there is # an error. try: - ### First we need to write the header. + # Write each section, transferring data between functions where + # necessary. offset = self._writeBeginning(f) entries = self._writeDirectoryEntries(f, offset) self._writeMini(f, entries) self._writeFinal(f) finally: + self._cleanupEntries() + if opened: f.close() diff --git a/extract_msg/utils.py b/extract_msg/utils.py index 713f6fe9..4ef39e06 100644 --- a/extract_msg/utils.py +++ b/extract_msg/utils.py @@ -26,7 +26,7 @@ import tzlocal from html import escape as htmlEscape -from typing import Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union from . import constants from .enums import AttachmentType @@ -142,6 +142,18 @@ def _open(name, mode, *args, **kwargs): return _open +def dictGetCasedKey(_dict : Dict, key : Any) -> Any: + """ + Retrieves the key from the dictionary with the proper casing using a + caseless key. + """ + try: + return next((x for x in _dict.keys() if x.lower() == key.lower())) + except StopIteration: + # If we couldn't find the key, raise a KeyError. + raise KeyError(key) + + def divide(string, length : int) -> List: """ Divides a string into multiple substrings of equal length. If there is not @@ -482,10 +494,26 @@ def inputToBytes(stringInputVar, encoding) -> bytes: def inputToMsgPath(inp) -> List: """ Converts the input into an msg path. + + :raises ValueError: The path contains an illegal character. """ if isinstance(inp, (list, tuple)): inp = '/'.join(inp) - ret = [x for x in inputToString(inp, 'utf-8').replace('\\', '/').split('/') if x] + + inp = inputToString(inp, 'utf-8') + + # Validate the path is okay. Normally we would check for '/' and '\', but + # we are expecting a string or similar which will use those as path + # separators, so we will ignore that for now. + if ':' in inp or '!' in inp: + raise ValueError('Illegal character ("!" or ":") found in MSG path.') + + ret = [x for x in inp.replace('\\', '/').split('/') if x] + + # One last thing to check: all path segments can be, at most, 31 characters + # (32 if you include the null character), so we should verify that. + if any(len(x) > 31 for x in ret): + raise ValueError('Path segments must not be greater than 31 characters.') return ret From aec0b73a6345dcb2e2485d4bc68d9dd4cba91ee6 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sat, 14 Jan 2023 21:17:25 -0800 Subject: [PATCH 24/30] Fix issues with helper function and addEntry --- extract_msg/ole_writer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extract_msg/ole_writer.py b/extract_msg/ole_writer.py index 7d78960c..1e4a5023 100644 --- a/extract_msg/ole_writer.py +++ b/extract_msg/ole_writer.py @@ -603,7 +603,7 @@ def addEntry(self, path, data : bytes = None, storage : bool = False, **kwargs): """ path = inputToMsgPath(path) # First, find the current place in our dict to add the item. - _dir = self.__getContainingStorage(path, False) + _dir = self.__getContainingStorage(path, False, True) # Now, check that the item *is not* already in our dict, as that would # cause problems. if path[-1].lower() in map(str.lower, _dir.keys()): @@ -614,6 +614,7 @@ def addEntry(self, path, data : bytes = None, storage : bool = False, **kwargs): entry.type = DirectoryEntryType.STORAGE if storage else DirectoryEntryType.STREAM entry.name = path[-1] self.__modifyEntry(entry, data = data, **kwargs) + _dir[path[-1]] = entry def addOleEntry(self, path, entry : OleDirectoryEntry, data : Optional[bytes] = None) -> None: """ From 9edf1bc0fbacd8903a0edce272ce5af4b3e8f4de Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sat, 14 Jan 2023 21:30:19 -0800 Subject: [PATCH 25/30] Fix more issues in new OleWriter code --- extract_msg/ole_writer.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/extract_msg/ole_writer.py b/extract_msg/ole_writer.py index 1e4a5023..8581ba3f 100644 --- a/extract_msg/ole_writer.py +++ b/extract_msg/ole_writer.py @@ -127,7 +127,7 @@ def __getContainingStorage(self, path : List[str], entryExists : bool = True, cr if create: self.addEntry(path[:index + 1], storage = True) else: - raise OSError('Entry not found.') + raise OSError(f'Entry not found: {name}') _dir = _dir[dictGetCasedKey(_dir, name)] # If the current item is not a storage and we have more to the path, @@ -614,7 +614,10 @@ def addEntry(self, path, data : bytes = None, storage : bool = False, **kwargs): entry.type = DirectoryEntryType.STORAGE if storage else DirectoryEntryType.STREAM entry.name = path[-1] self.__modifyEntry(entry, data = data, **kwargs) - _dir[path[-1]] = entry + if storage: + _dir[path[-1]] = {'::DirectoryEntry': entry} + else: + _dir[path[-1]] = entry def addOleEntry(self, path, entry : OleDirectoryEntry, data : Optional[bytes] = None) -> None: """ @@ -709,7 +712,7 @@ def editEntry(self, path, **kwargs) -> None: parameter was invalid. """ # First, find our entry to edit. - entry = self.__getEntry(path) + entry = self.__getEntry(inputToMsgPath(path)) # Send it to be modified using the arguments given. self.__modifyEntry(entry, **kwargs) From fa996ad3c24b31b5f0ccc6c0caa0e1bb8c18d626 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 17 Jan 2023 16:22:54 -0800 Subject: [PATCH 26/30] Added new function which will make OleWriter.listStructure easier. --- CHANGELOG.md | 3 ++- extract_msg/ole_writer.py | 44 +++++++++++++++++++++++++++++---------- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a848e734..9c667a33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,8 +13,9 @@ * Changed `_DirectoryEntry` to `DirectoryEntry` to make the more finalized version public. Access to the originals that the `OleWriter` class creates should never happen, instead copies should be returned to ensure the behavior is as expected. * Added new function `OleWriter.getEntry` which returns a copy of the `DirectoryEntry` instance for that stream or storage in the writer. Use this function to see the current internal state of an entry. * Added new function `OleWriter.renameEntry` which allows the user to rename a stream or storage (in place). This only changes it's direct name and not it's location in the new OLE file. +* Added new function `OleWriter.walk` which is similar to `os.walk` but for walking the structure of the new OLE file. * Added a small amount of path validation to `inputToMsgPath` which is used in a lot of places where user input for a path is accepted. It ensures illegal characters don't exist and that the path segments (each name for a storage or stream) are less than 32 characters. This will be most helpful for `OleWriter`. -* Added *many* internal helper functions to `OleWriter` to make extensions easier and consolidate common code. +* Added *many* internal helper functions to `OleWriter` to make extensions easier and consolidate common code. Many of these involve direct access to internal data which is why they are private. **v0.38.4** * Fix line in `OleWriter` that was causing exporting to fail. diff --git a/extract_msg/ole_writer.py b/extract_msg/ole_writer.py index 8581ba3f..7b1f7306 100644 --- a/extract_msg/ole_writer.py +++ b/extract_msg/ole_writer.py @@ -3,7 +3,7 @@ import pathlib import re -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, Iterator, List, Optional, Tuple, Union from . import constants from .enums import Color, DirectoryEntryType @@ -114,8 +114,10 @@ def __getContainingStorage(self, path : List[str], entryExists : bool = True, cr if create and entryExists: raise ValueError(':param create: and :param entryExists: cannot both be True (an entry cannot exist if it is being created).') - # Check that the path is not an internal entry. - if '::directoryentry' in map(str.lower, path): + # Check that the path is not an internal entry. Given the validation on + # paths that most functions should do because of the call to + # inputToMsgPath, this shouldn't actually be necessary. + if any(x.startswith('::') for x in path): raise ValueError('Found internal name in path.') _dir = self.__dirEntries @@ -136,7 +138,7 @@ def __getContainingStorage(self, path : List[str], entryExists : bool = True, cr raise OSError('Attempted to access children of a stream.') if entryExists and path[-1].lower() not in map(str.lower, _dir.keys()): - raise OSError('Entry not found.') + raise OSError(f'Entry not found: {path[-1]}') return _dir @@ -243,7 +245,7 @@ def __walkEntries(self): while len(toProcess) > 0: for name, item in toProcess.pop(0).items(): - if name != '::DirectoryEntry': + if not name.startswith('::'): if isinstance(item, dict): yield item['::DirectoryEntry'] toProcess.append(item) @@ -675,11 +677,6 @@ def deleteEntry(self, path) -> None: # Get the containing storage for the entry. _dir = self.__getContainingStorage(path) - # We need to protect against deliberately trying to break the code, so - # make sure someone cannot simply delete the storage entry. - if path[-1] == '::DirectoryEntry': - raise ValueError('Attempted to delete a storage directory entry directly.') - # The garbage collector will take care of all the loose items, so just # remove the entry. Also, once again we deal with the case insensitive # nature of the path. Even though comparisons are case insensitive, the @@ -823,7 +820,7 @@ def getEntry(self, path) -> DirectoryEntry: """ return copy.copy(self.__getEntry(inputToMsgPath(path))) - def listDir(self, streams = True, storages = False) -> List[List[str]]: + def listStructure(self, streams = True, storages = False) -> List[List[str]]: """ Returns a list of the specified items currently in the writter. """ @@ -876,6 +873,31 @@ def renameEntry(self, path, newName : str): else: _dir[newName] = dirData + def walk(self) -> Iterator[Tuple[str, List[str], List[str]]]: + """ + Functional equivelent to :function os.walk:, but for going over the file + structure of the OLE file to be written. Unlike :function os.walk:, it + takes no arguments, and the root will be omitted. The exception is the + first return which the directory will be an empty string. + """ + toProcess = [('', self.__dirEntries)] + + # Go through the to process list, removing the last item every time to + # mimic the behavior of os.walk, but also to give results more like + # olefile.OleFileIO.listdir. + while toProcess: + currentDir, dirDict = toProcess.pop() + storages = [] + streams = [] + for name in sorted(dirDict.keys(), key = str.lower): + if not name.startswith('::'): + if isinstance(dirDict[name], dict): + storages.append(name) + toProcess.append((name, dirDict[name])) + else: + streams.append(name) + + yield (currentDir, storages, streams) def write(self, path) -> None: """ From 5980d1316010dd61a4a8574111e731587660f684 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 17 Jan 2023 17:56:46 -0800 Subject: [PATCH 27/30] Version 0.39.0 --- CHANGELOG.md | 1 + extract_msg/__init__.py | 2 +- extract_msg/ole_writer.py | 58 +++++++++++++++++++++++++++++---------- 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c667a33..504335fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ * Added new function `OleWriter.getEntry` which returns a copy of the `DirectoryEntry` instance for that stream or storage in the writer. Use this function to see the current internal state of an entry. * Added new function `OleWriter.renameEntry` which allows the user to rename a stream or storage (in place). This only changes it's direct name and not it's location in the new OLE file. * Added new function `OleWriter.walk` which is similar to `os.walk` but for walking the structure of the new OLE file. +* Added new function `OleWriter.listItems` which is functionally equivalent to `olefile.OleFileIO.listdir` which returns a list of paths to every item. Optionally a user can get the paths just for streams, just for storages, or both. Requesting neither will simply return an empty list. Default is to just return streams. * Added a small amount of path validation to `inputToMsgPath` which is used in a lot of places where user input for a path is accepted. It ensures illegal characters don't exist and that the path segments (each name for a storage or stream) are less than 32 characters. This will be most helpful for `OleWriter`. * Added *many* internal helper functions to `OleWriter` to make extensions easier and consolidate common code. Many of these involve direct access to internal data which is why they are private. diff --git a/extract_msg/__init__.py b/extract_msg/__init__.py index c7b19b8e..fba28a0e 100644 --- a/extract_msg/__init__.py +++ b/extract_msg/__init__.py @@ -27,7 +27,7 @@ # along with this program. If not, see . __author__ = 'Destiny Peterson & Matthew Walker' -__date__ = '2023-01-14' +__date__ = '2023-01-17' __version__ = '0.39.0' import logging diff --git a/extract_msg/ole_writer.py b/extract_msg/ole_writer.py index 7b1f7306..69a9cf44 100644 --- a/extract_msg/ole_writer.py +++ b/extract_msg/ole_writer.py @@ -215,7 +215,7 @@ def __modifyEntry(self, entry : DirectoryEntry, **kwargs): if stateBits is not None: entry.stateBits = stateBits - def __recalculateSectors(self): + def __recalculateSectors(self) -> None: """ Recalculates several of the internal variables used for saving that specify the number of sectors and where things should go. @@ -235,7 +235,7 @@ def __recalculateSectors(self): self.__largeEntries.append(entry) self.__largeEntrySectors += ceilDiv(len(entry.data), 512) - def __walkEntries(self): + def __walkEntries(self) -> Iterator[DirectoryEntry]: """ Returns a generator that will walk the entires recursively. Each item returned by it will be a DirectoryEntry instance. @@ -580,7 +580,7 @@ def _writeMini(self, f, entries : List[DirectoryEntry]) -> None: if self.__numMinifatSectors & 7: f.write((b'\x00' * 64) * (8 - (self.__numMinifatSectors & 7))) - def addEntry(self, path, data : bytes = None, storage : bool = False, **kwargs): + def addEntry(self, path, data : bytes = None, storage : bool = False, **kwargs) -> None: """ Adds an entry to the OleWriter instance at the path specified, adding storages with default settings where necessary. If the entry is not a @@ -820,13 +820,37 @@ def getEntry(self, path) -> DirectoryEntry: """ return copy.copy(self.__getEntry(inputToMsgPath(path))) - def listStructure(self, streams = True, storages = False) -> List[List[str]]: + def listItems(self, streams = True, storages = False) -> List[List[str]]: """ Returns a list of the specified items currently in the writter. - """ - # TODO TODO TODO TODO - def renameEntry(self, path, newName : str): + :param streams: If True, includes the path for each stream in the list. + :param storages: If True, includes the path for each storage in the + list. + """ + # We are actually abusing the walk function a bit here to life much + # easier. The way we do this is to look at the current directory that + # the walk function is giving information about and then deciding what + # parts of it we want to use. Once we have all the paths created, we + # will then sort and return it to give an output similar, if not + # identical, to OleFileIO.listdir. The mentioned method sorts keeping + # case in mind. + if not streams and not storages: + return [] + + paths = [] + for currentDir, stor, stre in self.walk(): + if storages: + for name in stor: + paths.append(currentDir + [name]) + if streams: + for name in stre: + paths.append(currentDir + [name]) + + paths.sort() + return paths + + def renameEntry(self, path, newName : str) -> None: """ Changes the name of an entry, leaving it in it's current position. @@ -873,18 +897,22 @@ def renameEntry(self, path, newName : str): else: _dir[newName] = dirData - def walk(self) -> Iterator[Tuple[str, List[str], List[str]]]: + def walk(self) -> Iterator[Tuple[List[str], List[str], List[str]]]: """ Functional equivelent to :function os.walk:, but for going over the file structure of the OLE file to be written. Unlike :function os.walk:, it - takes no arguments, and the root will be omitted. The exception is the - first return which the directory will be an empty string. + takes no arguments. + + :returns: A tuple of three lists. The first is the path, as a list of + strings, for the directory (or an empty list for the root), the + second is a list of the storages in the current directory, and the + last is a list of the streams. Streams and storages are sorted + caselessly. """ - toProcess = [('', self.__dirEntries)] + toProcess = [([], self.__dirEntries)] - # Go through the to process list, removing the last item every time to - # mimic the behavior of os.walk, but also to give results more like - # olefile.OleFileIO.listdir. + # Go through the toProcess list, removing the last item every time to + # mimic the behavior of os.walk. while toProcess: currentDir, dirDict = toProcess.pop() storages = [] @@ -893,7 +921,7 @@ def walk(self) -> Iterator[Tuple[str, List[str], List[str]]]: if not name.startswith('::'): if isinstance(dirDict[name], dict): storages.append(name) - toProcess.append((name, dirDict[name])) + toProcess.append((currentDir + [name], dirDict[name])) else: streams.append(name) From b16df2619c6b97d3decfb1e4869576fa1cfebdfb Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 17 Jan 2023 18:35:19 -0800 Subject: [PATCH 28/30] Revert "Merge branch 'outlook-signature' into next-release" This reverts commit d089d2e606c842540f55574d1d24b21ce86a349c, reversing changes made to c6284e5e304e8f6f2ebc7f525b89152bc66a65a2. --- CHANGELOG.md | 22 +- README.rst | 4 +- extract_msg/__init__.py | 4 +- extract_msg/attachment.py | 32 +-- extract_msg/constants.py | 3 - extract_msg/contact.py | 10 +- extract_msg/message_base.py | 4 +- extract_msg/msg.py | 11 +- extract_msg/named.py | 42 +-- extract_msg/ole_writer.py | 543 ++++-------------------------------- extract_msg/properties.py | 9 - extract_msg/utils.py | 32 +-- 12 files changed, 94 insertions(+), 622 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 504335fe..2593b2f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,23 +1,3 @@ -**v0.39.0** -* [[TeamMsgExtractor #318](https://github.com/TeamMsgExtractor/msg-extractor/issues/318)] Added code to handle a standards violation (from what I can tell, anyways) caused by the attachment not having an `AttachMethod` property. The code will log a warning, attempt to detect the method, and throw a `StandardViolationError` if it fails. -* [[TeamMsgExtractor #320](https://github.com/TeamMsgExtractor/msg-extractor/issues/320)] Changed the way string named properties are handled to allow for the string stream to have some errors and still be parsed. Warnings about these errors will be logged. -* [[TeamMsgExtractor #326](https://github.com/TeamMsgExtractor/msg-extractor/issues/326)] Fixed a bug that could cause some files to error when exporting. -* Fixed an issue where creation and modification times were not being copied to the new OLE file created by `OleWriter`. -* Fixed up a few docstrings. -* Fixed a few issues in `MSGFile` regarding the `filename` keyword argument. -* Added new argument `rootPath` to `OleWriter.fromOleFile` for saving a specific directory from an OLE file instead of just copying the entire file. That directory will become the root of the new one. -* Adjusted code for `OleWriter` to generate certain values *only* at save time to make them more dynamic. This allows for existing streams to be properly edited (although has issues with allowing storages to be edited). -* Added new function `OleWriter.deleteEntry` to remove an entry that was already added. If the entry is a storage, all children will be removed too. -* Added new function `OleWriter.editEntry` to edit an entry that was already added. -* Added new function `OleWriter.addEntry` to add a new entry to the writer without an `OleFileIO` instance. Properties of the entry are instead set using the same keyword arguments as described in `OleWriter.editEntry`. -* Changed `_DirectoryEntry` to `DirectoryEntry` to make the more finalized version public. Access to the originals that the `OleWriter` class creates should never happen, instead copies should be returned to ensure the behavior is as expected. -* Added new function `OleWriter.getEntry` which returns a copy of the `DirectoryEntry` instance for that stream or storage in the writer. Use this function to see the current internal state of an entry. -* Added new function `OleWriter.renameEntry` which allows the user to rename a stream or storage (in place). This only changes it's direct name and not it's location in the new OLE file. -* Added new function `OleWriter.walk` which is similar to `os.walk` but for walking the structure of the new OLE file. -* Added new function `OleWriter.listItems` which is functionally equivalent to `olefile.OleFileIO.listdir` which returns a list of paths to every item. Optionally a user can get the paths just for streams, just for storages, or both. Requesting neither will simply return an empty list. Default is to just return streams. -* Added a small amount of path validation to `inputToMsgPath` which is used in a lot of places where user input for a path is accepted. It ensures illegal characters don't exist and that the path segments (each name for a storage or stream) are less than 32 characters. This will be most helpful for `OleWriter`. -* Added *many* internal helper functions to `OleWriter` to make extensions easier and consolidate common code. Many of these involve direct access to internal data which is why they are private. - **v0.38.4** * Fix line in `OleWriter` that was causing exporting to fail. * Fixed some issues with the `README`. @@ -38,7 +18,7 @@ * Added function `MSGFile.export` which copies all streams and storages from an MSG file into a new file. This can "clone" an MSG file or be used for extracting an MSG file that is embedded inside of another. * Added hidden function to `MSGFile` for getting the `OleDirectoryEntry` for a storage or stream. This is mainly for use by the `OleWriter` class. * Added option `extractEmbedded` to `Attachment.save` (`--extract-embedded` on the command line) which causes embedded MSG files to be extracted instead of running their save methods. -* Fixed minor issues with `utils.inputToMsgPath` (renamed from `utils.inputToMsgPath`). +* Fixed minor issues with `utils.inputToMsgPath` (renamed from `utils.inputToMsgpath`). * Renamed `utils.msgpathToString` to `utils.msgPathToString`. * Made some of the module requirements a little more strict to better version control. I'll be trying to make periodic checks for updates to the dependency packages and make sure that new versions are compatible before changing the allowed versions, while also trying to keep the requirements a bit flexible. diff --git a/README.rst b/README.rst index 76e73266..68e001ff 100644 --- a/README.rst +++ b/README.rst @@ -234,8 +234,8 @@ your access to the newest major version of extract-msg. .. |License: GPL v3| image:: https://img.shields.io/badge/License-GPLv3-blue.svg :target: LICENSE.txt -.. |PyPI3| image:: https://img.shields.io/badge/pypi-0.38.5-blue.svg - :target: https://pypi.org/project/extract-msg/0.38.5/ +.. |PyPI3| image:: https://img.shields.io/badge/pypi-0.38.4-blue.svg + :target: https://pypi.org/project/extract-msg/0.38.4/ .. |PyPI2| image:: https://img.shields.io/badge/python-3.6+-brightgreen.svg :target: https://www.python.org/downloads/release/python-367/ diff --git a/extract_msg/__init__.py b/extract_msg/__init__.py index fba28a0e..a3fedb44 100644 --- a/extract_msg/__init__.py +++ b/extract_msg/__init__.py @@ -27,8 +27,8 @@ # along with this program. If not, see . __author__ = 'Destiny Peterson & Matthew Walker' -__date__ = '2023-01-17' -__version__ = '0.39.0' +__date__ = '2022-12-03' +__version__ = '0.38.4' import logging diff --git a/extract_msg/attachment.py b/extract_msg/attachment.py index 0655f8e9..74673304 100644 --- a/extract_msg/attachment.py +++ b/extract_msg/attachment.py @@ -11,7 +11,6 @@ from .attachment_base import AttachmentBase from .custom_attachments import CustomAttachmentHandler, getHandler from .enums import AttachmentType -from .exceptions import StandardViolationError from .utils import createZipOpen, inputToString, openMsg, prepareFilename @@ -36,31 +35,6 @@ def __init__(self, msg, dir_): super().__init__(msg, dir_) self.__customHandler = None - if '37050003' not in self.props: - from .prop import createProp - - logger.warning('Attahcment method property not found on attachment. Code will attempt to guess the type.') - - # Because this condition is actually kind of a violation of the - # standard, we are just going to do this in a dumb way. Basically we - # are going to try to set the attach method *manually* just so I - # don't have to go and modify the following code. - if self.exists('__substg1.0_37010102'): - # Set it as data and call it a day. - propData = b'\x03\x00\x057\x07\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00' - elif self.exists('__substg1.0_3701000D'): - # If it is a folder and we have properties, call it an MSG file. - if self.exists('__substg1.0_3701000D/__properties_version1.0'): - propData = b'\x03\x00\x057\x07\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00' - else: - # Call if custom attachment data. - propData = b'\x03\x00\x057\x07\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00' - else: - # Can't autodetect it, so throw an error. - raise StandardViolationError('Attachment method missing, and it could not be determined automatically.') - - self.props._propDict['37050003'] = createProp(propData) - # Get attachment data. if self.exists('__substg1.0_37010102'): self.__type = AttachmentType.DATA @@ -153,18 +127,16 @@ def save(self, **kwargs) -> Optional[Union[str, 'MSGFile']]: to :param zip:. If :param zip: is an instance, :param customPath: will refer to a location inside the zip file. - :param extractEmbedded: If True, causes the attachment, should it be an + :param extractEmbedded: If true, causes the attachment, should it be an embedded MSG file, to save as a .msg file instead of calling it's save function. - :param skipEmbedded: If True, skips saving this attachment if it is an - embedded MSG file. """ # First check if we are skipping embedded messages and stop # *immediately* if we are. if self.type is AttachmentType.MSG and kwargs.get('skipEmbedded'): return None - # Get the filename to use. + # Check if the user has specified a custom filename filename = self.getFilename(**kwargs) # Someone managed to have a null character here, so let's get rid of that diff --git a/extract_msg/constants.py b/extract_msg/constants.py index 697639ba..9f89cdad 100644 --- a/extract_msg/constants.py +++ b/extract_msg/constants.py @@ -50,9 +50,6 @@ # This is used in the workaround for decoding issues in RTFDE. We find `\bin` # sections and try to remove all of them to help with the decoding. RE_BIN = re.compile(br'\\bin([0-9]+) ?') -# Used in the vaildation of OLE paths. Any of these characters in a name make it -# invalid. -RE_INVALID_OLE_PATH = re.compile(r'[:/\\!]') FIXED_LENGTH_PROPS = ( 0x0000, diff --git a/extract_msg/contact.py b/extract_msg/contact.py index 104411d0..941ae3ff 100644 --- a/extract_msg/contact.py +++ b/extract_msg/contact.py @@ -19,8 +19,6 @@ def __init__(self, path, **kwargs): :param path: path to the msg file in the system or is the raw msg file. :param prefix: used for extracting embeded msg files inside the main one. Do not set manually unless you know what you are doing. - :param parentMsg: Used for synchronizing named properties instances. Do - not set this unless you know what you are doing. :param attachmentClass: optional, the class the MSGFile object will use for attachments. You probably should not change this value unless you know what you are doing. @@ -30,8 +28,6 @@ def __init__(self, path, **kwargs): be retrieved. :param filename: optional, the filename to be used by default when saving. - :param attachmentErrorBehavior: Optional, the behavior to use in the - event of an error when parsing the attachments. :param overrideEncoding: optional, an encoding to use instead of the one specified by the msg file. Do not report encoding errors caused by this. @@ -689,11 +685,11 @@ def headerFormatProperties(self) -> constants.HEADER_FORMAT_TYPE: If you class should not do *any* header injection, return None from this property. """ - def strListToStr(inp : Optional[Union[str, List[str]]]): + def strListToStr(inp : Union[str, List[str]]): """ Small internal function for things that may return a string or list. """ - if inp is None or isinstance(inp, str): + if isinstance(inp, str): return inp else: return ', '.join(inp) @@ -750,7 +746,7 @@ def strListToStr(inp : Optional[Union[str, List[str]]]): 'Anniversary': self.weddingAnniversary.__format__('%B %d, %Y') if self.weddingAnniversaryLocal else None, 'Spouse/Partner': self.spouseName, 'Profession': self.profession, - 'Children': strListToStr(self.childrensNames), + 'Children': ', '.join(self.childrensNames), 'Hobbies': self.hobbies, 'Assistant': self.assistant, 'Web Page': self.webpageUrl, diff --git a/extract_msg/message_base.py b/extract_msg/message_base.py index 334e322d..2d998242 100644 --- a/extract_msg/message_base.py +++ b/extract_msg/message_base.py @@ -79,8 +79,8 @@ def __init__(self, path, **kwargs): body is desired. The function should return a string for plain text and bytes for HTML. If any problems occur, the function *must* either return None or raise one of the appropriate functions from - extract_msg.exceptions. All other exceptions must be handled - internally or they will not be caught. The original deencapsulation + extract_msg.exceptions. All other functions must be handled + internally or they will continue. The original deencapsulation method will not run if this is set. """ super().__init__(path, **kwargs) diff --git a/extract_msg/msg.py b/extract_msg/msg.py index 9db13122..b58bfdde 100644 --- a/extract_msg/msg.py +++ b/extract_msg/msg.py @@ -37,9 +37,10 @@ def __init__(self, path, **kwargs): one. Do not set manually unless you know what you are doing. :param parentMsg: Used for synchronizing named properties instances. Do not set this unless you know what you are doing. - :param attachmentClass: Optional, the class the MSGFile object will use - for attachments. You probably should not change this value unless - you know what you are doing. + :param attachmentClass: Optional, the class the MSGFile object + will use for attachments. You probably should + not change this value unless you know what you + are doing. :param delayAttachments: Optional, delays the initialization of attachments until the user attempts to retrieve them. Allows MSG files with bad attachments to be initialized so the other data can @@ -111,8 +112,6 @@ def __init__(self, path, **kwargs): del kwargsCopy['prefix'] if 'parentMsg' in kwargsCopy: del kwargsCopy['parentMsg'] - if 'filename' in kwargsCopy: - del kwargsCopy['filename'] self.__kwargs = kwargsCopy prefixl = [] @@ -135,7 +134,7 @@ def __init__(self, path, **kwargs): self.__prefix = prefix self.__prefixList = prefixl self.__prefixLen = len(prefixl) - if prefix and not filename: + if prefix: filename = self._getStringStream(prefixl[:-1] + ['__substg1.0_3001'], prefix = False) if filename: self.filename = filename diff --git a/extract_msg/named.py b/extract_msg/named.py index be22cb19..b5c3d43f 100644 --- a/extract_msg/named.py +++ b/extract_msg/named.py @@ -21,12 +21,14 @@ def __init__(self, msg): # Get the basic streams. If all are emtpy, then nothing to do. guidStream = self._getStream('__substg1.0_00020102') or self._getStream('__substg1.0_00020102', False) entryStream = self._getStream('__substg1.0_00030102') or self._getStream('__substg1.0_00030102', False) + namesStream = self._getStream('__substg1.0_00040102') or self._getStream('__substg1.0_00040102', False) self.guidStream = guidStream self.entryStream = entryStream - self.namesStream = self._getStream('__substg1.0_00040102') or self._getStream('__substg1.0_00040102', False) + self.namesStream = namesStream # The if else stuff is for protection against None. guidStreamLength = len(guidStream) if guidStream else 0 entryStreamLength = len(entryStream) if entryStream else 0 + namesStreamLength = len(namesStream) if namesStream else 0 self.__propertiesDict = {} self.__properties = [] @@ -49,11 +51,21 @@ def __init__(self, msg): entry['guid'] = guids[entry['guid_index']] entries.append(entry) + # Parse the names stream. + names = self.__names + pos = 0 + while pos < namesStreamLength: + nameLength = constants.STNP_NAM.unpack(namesStream[pos:pos+4])[0] + pos += 4 # Move to the start of the entry. + names[pos - 4] = namesStream[pos:pos+nameLength].decode('utf-16-le') # Names are stored in the dictionary as the position they start at. + pos += roundUp(nameLength, 4) + self.entries = entries self.__guids = guids for entry in entries: - self.__properties.append(StringNamedProperty(entry, self.__getName(entry['id'])) if entry['pkind'] == NamedPropertyType.STRING_NAMED else NumericalNamedProperty(entry)) + streamID = properHex(0x8000 + entry['pid']) + self.__properties.append(StringNamedProperty(entry, names[entry['id']]) if entry['pkind'] == NamedPropertyType.STRING_NAMED else NumericalNamedProperty(entry)) for property in self.__properties: name = property.name if isinstance(property, StringNamedProperty) else property.propertyID @@ -68,32 +80,6 @@ def __iter__(self): def __len__(self) -> int: return self.__propertiesDict.__len__() - def __getName(self, offset : int) -> str: - """ - Parses the offset into the named stream and returns the name found. - """ - # We used to parse names by handing it as an array, as specified by the - # documentation, but this new method allows for a little bit more wiggle - # room in terms of what is accepted by the module. - if offset & 3 != 0: - # If the offset is not a multiple of 4, that is an error, but we are - # reducing it to a warning. - logger.warning(f'Malformed named properties detected due to bad offset ({offset}). Ignoring.') - # Check that offset is in string stream. - if offset > len(self.namesStream): - raise ValueError('Failed to parse named property: offset was not in string stream.') - - # Get the length, in bytes, of the string. - length = constants.STNP_NAM.unpack(self.namesStream[offset:offset + 4])[0] - offset += 4 - - # Make sure the string can be read entirely. If it can't, something was - # corrupt. - if offset + length > len(self.namesStream): - raise ValueError(f'Failed to parse named property: length ({length}) of string overflows the string stream. This is probably due to a bad offset.') - - return self.namesStream[offset:offset + length].decode('utf-16-le') - def _getStream(self, filename, prefix = True) -> Optional[bytes]: return self.__msg._getStream([self.__dir, filename], prefix = prefix) diff --git a/extract_msg/ole_writer.py b/extract_msg/ole_writer.py index 53ce1593..66e586f4 100644 --- a/extract_msg/ole_writer.py +++ b/extract_msg/ole_writer.py @@ -1,26 +1,24 @@ -import copy import io import pathlib import re -from typing import Dict, Iterator, List, Optional, Tuple, Union +from typing import List, Optional, Union from . import constants from .enums import Color, DirectoryEntryType -from .utils import ceilDiv, dictGetCasedKey, inputToMsgPath +from .utils import ceilDiv, inputToMsgPath from olefile.olefile import OleDirectoryEntry, OleFileIO from red_black_dict_mod import RedBlackTree -class DirectoryEntry: +class _DirectoryEntry: """ - An internal representation of a stream or storage in the OleWriter. - Originals should be inaccessible outside of the class. + Hidden class, will probably be modified later. """ name : str = '' - rightChild : 'DirectoryEntry' = None - leftChild : 'DirectoryEntry' = None - childTreeRoot : 'DirectoryEntry' = None + rightChild : '_DirectoryEntry' = None + leftChild : '_DirectoryEntry' = None + childTreeRoot : '_DirectoryEntry' = None stateBits : int = 0 creationTime : int = 0 modifiedTime : int = 0 @@ -37,7 +35,7 @@ class DirectoryEntry: startingSectorLocation : int = 0 color : Color = Color.BLACK - clsid : bytes = b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + clsid : bytes = b'' data : bytes = b'' def __init__(self): @@ -81,10 +79,7 @@ class OleWriter: [MS-CFB]. """ def __init__(self, rootClsid : bytes = constants.DEFAULT_CLSID): - self.__rootEntry = DirectoryEntry() - self.__rootEntry.name = "Root Entry" - self.__rootEntry.type = DirectoryEntryType.ROOT_STORAGE - self.__rootEntry.clsid = rootClsid + self.__rootClsid = rootClsid # The root entry will always exist, so this must be at least 1. self.__dirEntryCount = 1 self.__dirEntries = {} @@ -92,166 +87,6 @@ def __init__(self, rootClsid : bytes = constants.DEFAULT_CLSID): self.__largeEntrySectors = 0 self.__numMinifatSectors = 0 - def __getContainingStorage(self, path : List[str], entryExists : bool = True, create : bool = False) -> Dict: - """ - Finds the storage dict internally where the entry specified by - :param path: would be created. If :param create: is True, missing - storages will be created with default settings. - - :param entryExists: If True, throws an error when the requested entry - does not yet exist. - :param create: If True, creates missing storages with default settings. - - :raises OSError: If :param create: is False and the path could not be - found. Also raised if :param entryExists: is True and the requested - entry does not exist. - :raises ValueError: Tried to access an interal stream or tried to use - both the create option and the entryExists option as True. - - :returns: The storage dict that the entry is in. - """ - # Quick check for incompatability between create and entryExists. - if create and entryExists: - raise ValueError(':param create: and :param entryExists: cannot both be True (an entry cannot exist if it is being created).') - - # Check that the path is not an internal entry. Given the validation on - # paths that most functions should do because of the call to - # inputToMsgPath, this shouldn't actually be necessary. - if any(x.startswith('::') for x in path): - raise ValueError('Found internal name in path.') - - _dir = self.__dirEntries - - for index, name in enumerate(path[:-1]): - # If no entry in the current stream matches the path, raise an - # OSError, *unless* the option to create storages is True. - if name.lower() not in map(str.lower, _dir.keys()): - if create: - self.addEntry(path[:index + 1], storage = True) - else: - raise OSError(f'Entry not found: {name}') - _dir = _dir[dictGetCasedKey(_dir, name)] - - # If the current item is not a storage and we have more to the path, - # raise an OSError. - if not isinstance(_dir, dict): - raise OSError('Attempted to access children of a stream.') - - if entryExists and path[-1].lower() not in map(str.lower, _dir.keys()): - raise OSError(f'Entry not found: {path[-1]}') - - return _dir - - def __getEntry(self, path : List[str]) -> DirectoryEntry: - """ - Finds and returns an existing DirectoryEntry instance in the writer. - - :raises OSError: If the entry does not exist. - :raises ValueError: If access to an internal item is attempted. - """ - _dir = self.__getContainingStorage(path) - item = _dir[dictGetCasedKey(_dir, path[-1])] - if isinstance(item, dict): - return item['::DirectoryEntry'] - else: - return item - - def __modifyEntry(self, entry : DirectoryEntry, **kwargs): - """ - Edits the DirectoryEntry with the data provided. Common code used for - :method addEntry: and :method editEntry:. - - :raises ValueError: Some part of the data given to modify the various - properties was invalid. See the the listed methods for details. - """ - # Extract the arguments. - data = kwargs.get('data') - clsid = kwargs.get('clsid') - creationTime = kwargs.get('creationTime') - modifiedTime = kwargs.get('modifiedTime') - stateBits = kwargs.get('stateBits') - - # I don't like that I have repeated if statements for checking each of - # the arguments, but I need to make sure nothing changes if something is - # invalid. - if data is not None: - if entry.type is not DirectoryEntryType.STREAM: - raise ValueError('Cannot set the data of a storage object.') - if not isinstance(data, bytes): - raise ValueError('Data must be a bytes instance if set.') - - if clsid is not None: - if not isinstance(clsid, bytes): - raise ValueError('CLSID must be bytes.') - if len(clsid) != 16: - raise ValueError('CLSID must be 16 bytes.') - - if creationTime is not None: - if entry.type is DirectoryEntryType.STREAM: - raise ValueError('Modification of creation time cannot be done on a stream.') - if not isinstance(creationTime, int) or creationTime < 0 or creationTime > 0xFFFFFFFFFFFFFFFF: - raise ValueError('Creation time must be a positive 8 byte int.') - - if modifiedTime is not None: - if entry.type is DirectoryEntryType.STREAM: - raise ValueError('Modification of modified time cannot be done on a stream.') - if not isinstance(modifiedTime, int) or modifiedTime < 0 or modifiedTime > 0xFFFFFFFFFFFFFFFF: - raise ValueError('Modified time must be a positive 8 byte int.') - - if stateBits is not None: - if not isinstance(stateBits, int) or stateBits < 0 or stateBits > 0xFFFFFFFF: - raise ValueError('State bits must be a positive 4 byte int.') - - - # Now that all our checks have passed, let's set our data. - if data is not None: - entry.data = data - if clsid is not None: - entry.clsid = clsid - if creationTime is not None: - entry.creationTime = creationTime - if modifiedTime is not None: - entry.modifiedTime = modifiedTime - if stateBits is not None: - entry.stateBits = stateBits - - def __recalculateSectors(self) -> None: - """ - Recalculates several of the internal variables used for saving that - specify the number of sectors and where things should go. - """ - self.__dirEntryCount = 0 - self.__numMinifatSectors = 0 - self.__largeEntries.clear() - self.__largeEntrySectors = 0 - - count = 0 - for entry in self.__walkEntries(): - self.__dirEntryCount += 1 - if entry.type == DirectoryEntryType.STREAM: - if len(entry.data) < 4096: - self.__numMinifatSectors += ceilDiv(len(entry.data), 64) - else: - self.__largeEntries.append(entry) - self.__largeEntrySectors += ceilDiv(len(entry.data), 512) - - def __walkEntries(self) -> Iterator[DirectoryEntry]: - """ - Returns a generator that will walk the entires recursively. Each item - returned by it will be a DirectoryEntry instance. - """ - toProcess = [self.__dirEntries] - yield self.__rootEntry - - while len(toProcess) > 0: - for name, item in toProcess.pop(0).items(): - if not name.startswith('::'): - if isinstance(item, dict): - yield item['::DirectoryEntry'] - toProcess.append(item) - else: - yield item - @property def __numberOfSectors(self) -> int: """ @@ -265,27 +100,9 @@ def __numberOfSectors(self) -> int: @property def __numMinifat(self) -> int: - """ - The number of FAT sectors needed to store the mini FAT. - """ return ceilDiv(self.__numMinifatSectors, 8) - def _cleanupEntries(self) -> None: - """ - Cleans up the node connections by walking the tree and removing - references that were added during writing. - """ - self.__largeEntries.clear() - for entry in self.__walkEntries(): - entry.id = -1 - entry.leftChild = None - entry.rightChild = None - entry.childTreeRoot = None - entry.leftSiblingID = 0xFFFFFFFF - entry.rightSiblingID = 0xFFFFFFFF - entry.childID = 0xFFFFFFFF - - def _getFatSectors(self) -> Tuple[int, int, int]: + def _getFatSectors(self): """ Returns a tuple containing the number of FAT sectors, the number of DIFAT sectors, and the total number of sectors the saved file will have. @@ -302,19 +119,19 @@ def _getFatSectors(self) -> Tuple[int, int, int]: return (numFat, numDifat, self.__numberOfSectors + numDifat + numFat) - def _treeSort(self, startingSector : int) -> List[DirectoryEntry]: + def _treeSort(self, startingSector : int) -> List[_DirectoryEntry]: """ Uses red-black trees to sort the internal data in preparation for writing the file, returning a list, in order, of the entries to write. """ # First, create the root entry. - root = copy.copy(self.__rootEntry) - + root = _DirectoryEntry() + root.name = "Root Entry" + root.type = DirectoryEntryType.ROOT_STORAGE + root.clsid = self.__rootClsid # Add the location of the start of the mini stream. root.startingSectorLocation = (startingSector + ceilDiv(self.__dirEntryCount, 4) + ceilDiv(self.__numMinifatSectors, 128)) if self.__numMinifat > 0 else 0xFFFFFFFE root.streamSize = self.__numMinifatSectors * 64 - root.childTreeRoot = None - root.childID = 0xFFFFFFFF entries = [root] toProcess = [(root, self.__dirEntries)] @@ -336,9 +153,9 @@ def _treeSort(self, startingSector : int) -> List[DirectoryEntry]: # the processing list. if isinstance(val, dict): toProcess.append((val['::DirectoryEntry'], val)) - val = val['::DirectoryEntry'] - - entries.append(val) + entries.append(val['::DirectoryEntry']) + else: + entries.append(val) # Add the data to the tree. tree.add((len(name), name.upper()), val) @@ -352,15 +169,23 @@ def _treeSort(self, startingSector : int) -> List[DirectoryEntry]: for node in tree.in_order(): item = node.value # Set the color immediately. + if isinstance(item, dict): + item = item['::DirectoryEntry'] item.color = Color.BLACK if node.is_black else Color.RED - if node.left: - item.leftChild = node.left.value + val = node.left.value + if isinstance(val, _DirectoryEntry): + item.leftChild = val + else: + item.leftChild = val['::DirectoryEntry'] else: item.leftChild = None - if node.right: - item.rightChild = node.right.value + val = node.right.value + if isinstance(val, _DirectoryEntry): + item.rightChild = val + else: + item.rightChild = val['::DirectoryEntry'] else: item.rightChild = None @@ -397,8 +222,6 @@ def _writeBeginning(self, f) -> int: :returns: The current sector number after all the data is written. """ - # Recalculate some things needed for saving. - self.__recalculateSectors() # Since we are going to need these multiple times, get them now. numFat, numDifat, totalSectors = self._getFatSectors() @@ -522,7 +345,7 @@ def _writeBeginning(self, f) -> int: # Finally, return the current sector index for use in other places. return numDifat + numFat - def _writeDirectoryEntries(self, f, startingSector : int) -> List[DirectoryEntry]: + def _writeDirectoryEntries(self, f, startingSector : int) -> List[_DirectoryEntry]: """ Writes out all the directory entries. Returns the list generated. """ @@ -534,7 +357,7 @@ def _writeDirectoryEntries(self, f, startingSector : int) -> List[DirectoryEntry return entries - def _writeDirectoryEntry(self, f, entry : DirectoryEntry) -> None: + def _writeDirectoryEntry(self, f, entry : _DirectoryEntry) -> None: """ Writes the directory entry to the file f. """ @@ -550,7 +373,7 @@ def _writeFinal(self, f) -> None: if len(x.data) & 511: f.write(b'\x00' * (512 - (len(x.data) & 511))) - def _writeMini(self, f, entries : List[DirectoryEntry]) -> None: + def _writeMini(self, f, entries : List[_DirectoryEntry]) -> None: """ Writes the mini FAT followed by the full mini stream. """ @@ -580,82 +403,40 @@ def _writeMini(self, f, entries : List[DirectoryEntry]) -> None: if self.__numMinifatSectors & 7: f.write((b'\x00' * 64) * (8 - (self.__numMinifatSectors & 7))) - def addEntry(self, path, data : bytes = None, storage : bool = False, **kwargs) -> None: - """ - Adds an entry to the OleWriter instance at the path specified, adding - storages with default settings where necessary. If the entry is not a - storage, :param data: *must* be set. - - :param path: The path to add the entry at. Must not contain a path part - that is an already added stream. - :param data: The bytes for a stream. - :param storage: If True, the entry to add is a storage. Otherwise, the - entry is a stream. - :param clsid: The CLSID for the stream/storage. Must a a bytes instance - that is 16 bytes long. - :param creationTime: An 8 byte filetime int. Sets the creation time of - the entry. Not applicable to streams. - :param modifiedTime: An 8 byte filetime int. Sets the modification time - of the entry. Not applicable to streams. - :param stateBits: A 4 byte int. Sets the state bits, user-defined flags, - of the entry. For a stream, this *SHOULD* be unset. - - :raises OSError: A stream was found on the path before the end. - :raises ValueError: Attempts to access an internal item. - """ - path = inputToMsgPath(path) - # First, find the current place in our dict to add the item. - _dir = self.__getContainingStorage(path, False, True) - # Now, check that the item *is not* already in our dict, as that would - # cause problems. - if path[-1].lower() in map(str.lower, _dir.keys()): - raise OSError('Cannot add an entry that already exists.') - - # Create a new entry with basic data and insert it. - entry = DirectoryEntry() - entry.type = DirectoryEntryType.STORAGE if storage else DirectoryEntryType.STREAM - entry.name = path[-1] - self.__modifyEntry(entry, data = data, **kwargs) - if storage: - _dir[path[-1]] = {'::DirectoryEntry': entry} - else: - _dir[path[-1]] = entry - def addOleEntry(self, path, entry : OleDirectoryEntry, data : Optional[bytes] = None) -> None: """ Uses the entry provided to add the data to the writer. - :raises OSError: Tried to add an entry to a path that has not yet - been added, tried to add as a child of a stream, or tried to add an - entry where one already exists under the same name. + :raises ValueError: Tried to add an entry to a path that has not yet + been added. """ - path = inputToMsgPath(path) + pathList = inputToMsgPath(path) # First, find the current place in our dict to add the item. - - _dir = self.__getContainingStorage(path, False) - # Now, check that the item *is not* already in our dict, as that would - # cause problems. - if path[-1].lower() in map(str.lower, _dir.keys()): - raise OSError('Cannot add an entry that already exists.') + _dir = self.__dirEntries + while len(pathList) > 1: + if pathList[0] not in _dir: + # If no entry has been provided already for the directory, that + # is considered a fatal error. + raise ValueError('Path not found.') + _dir = _dir[pathList[0]] + pathList.pop(0) # Now that we are in the right place, add our data. - newEntry = DirectoryEntry() + newEntry = _DirectoryEntry() if entry.entry_type == DirectoryEntryType.STORAGE: # Handle a storage entry. # First add the dict to our tree of items. - _dir[path[-1]] = {'::DirectoryEntry': newEntry} + _dir[pathList[0]] = {'::DirectoryEntry': newEntry} # Finally, setup the values for the stream. newEntry.name = entry.name newEntry.type = DirectoryEntryType.STORAGE newEntry.clsid = _unClsid(entry.clsid) newEntry.stateBits = entry.dwUserFlags - newEntry.creationTime = entry.createTime - newEntry.modifiedTime = entry.modifyTime else: # Handle a stream entry. # First add the entry to out dict of entries. - _dir[path[-1]] = newEntry + _dir[pathList[0]] = newEntry newEntry.name = entry.name newEntry.type = DirectoryEntryType.STREAM newEntry.clsid = _unClsid(entry.clsid) @@ -663,64 +444,20 @@ def addOleEntry(self, path, entry : OleDirectoryEntry, data : Optional[bytes] = # Finally, handle the data. newEntry.data = data or b'' + if len(newEntry.data) < 4096: + self.__numMinifatSectors += ceilDiv(len(newEntry.data), 64) + else: + self.__largeEntries.append(newEntry) + self.__largeEntrySectors += ceilDiv(len(newEntry.data), 512) self.__dirEntryCount += 1 - def deleteEntry(self, path) -> None: - """ - Deletes the entry specified by :param path:, including all children. - - :raises OSError: If the entry does not exist or a part of the path that - is not the last was a stream. - :raises ValueError: Attempted to delete an internal data stream. - """ - path = inputToMsgPath(path) - # Get the containing storage for the entry. - _dir = self.__getContainingStorage(path) - - # The garbage collector will take care of all the loose items, so just - # remove the entry. Also, once again we deal with the case insensitive - # nature of the path. Even though comparisons are case insensitive, the - # path does remember the case used. - del _dir[dictGetCasedKey(_dir, path[-1])] - - def editEntry(self, path, **kwargs) -> None: - """ - Used to edit values of an entry by setting the specific kwargs. Set a - value to something other than None to set it. - - :param data: The data of a stream. Will error if used for something - other than a stream. - :param clsid: The CLSID for the stream/storage. Must a a bytes instance - that is 16 bytes long. - :param creationTime: An 8 byte filetime int. Sets the creation time of - the entry. Not applicable to streams. - :param modifiedTime: An 8 byte filetime int. Sets the modification time - of the entry. Not applicable to streams. - :param stateBits: A 4 byte int. Sets the state bits, user-defined flags, - of the entry. For a stream, this *SHOULD* be unset. - - - To convert a 32 character hexadecial CLSID into the bytes for this - function, the _unClsid function in the ole_writer submodule can be used. - - :raises OSError: The entry does not exist in the file. - :raises TypeError: Attempted to modify the bytes of a storage. - :raises ValueError: The type of a parameter was wrong, or the data of a - parameter was invalid. - """ - # First, find our entry to edit. - entry = self.__getEntry(inputToMsgPath(path)) - - # Send it to be modified using the arguments given. - self.__modifyEntry(entry, **kwargs) - def fromMsg(self, msg : 'MSGFile') -> None: """ Copies the streams and stream information necessary from the MSG file. """ # Get the root OLE entry's CLSID. - self.__rootEntry.clsid = _unClsid(msg._getOleEntry('/').clsid) + self.__rootClsid = _unClsid(msg._getOleEntry('/').clsid) # List both storages and directories, but sort them by shortest length # first to prevent errors. @@ -754,180 +491,25 @@ def fromMsg(self, msg : 'MSGFile') -> None: for x in gen: self.addOleEntry(x, msg._getOleEntry(x, prefix = False), msg._getStream(x, prefix = False)) - def fromOleFile(self, ole : OleFileIO, rootPath = []) -> None: + def fromOleFile(self, ole : OleFileIO) -> None: """ Copies all the streams from the proided OLE file into this writer. - - NOTE: This method does *not* handle any special rule that may be - required by a format that uses the compound binary file format as a base - when extracting an embedded directory. For example, MSG files require - modification of an embedded properties stream when extracting an - embedded MSG file. - - :param rootPath: A path (accepted by olefile.OleFileIO) to the directory - to use as the root of the file. If not provided, the file root will - be used. - - :raises OSError: If :param rootPath: does not exist in the file. """ - rootPath = inputToMsgPath(rootPath) - - # Check if the root path is simply the top of the file. - if rootPath == []: - # Copy the clsid of the root entry. - self.__rootEntry.clsid = _unClsid(ole.direntries[0].clsid) - paths = {tuple(x): (x, ole.direntries[ole._find(x)]) for x in ole.listdir(True, True)} - else: - # If it is not the top of the file, we need to do some filtering. - # First get the CLSID from the entry the path points to. - try: - entry = ole.direntries[ole._find(rootPath)] - self.__rootEntry.clsid = _unClsid(entry.clsid) - - except OSError as e: - if str(e) == 'file not found': - # Get the cause/context for the original exception and use - # it for the new exception. This hides the exception from - # OleFileIO. - context = e.__cause__ or e.__context__ - raise OSError('Root path was not found in the OLE file.') from context - else: - raise - - paths = {tuple(x[len(rootPath):]): (x, ole.direntries[ole._find(x)]) - for x in ole.listdir(True, True) if len(x) > len(rootPath)} + # Copy the clsid of the root entry. + self.__rootClsid = _unClsid(ole.direntries[0].clsid) - - # Copy all of the other entries. Ensure that directories come before - # their streams by sorting the paths. - for x in sorted(paths.keys()): - fullPath, entry = paths[x] + # Copy all of the other entries. + for x in ole.listdir(True, True): + entry = ole.direntries[ole._find(x)] if entry.entry_type == DirectoryEntryType.STREAM: - with ole.openstream(fullPath) as f: + with ole.openstream(x) as f: data = f.read() else: data = None self.addOleEntry(x, entry, data) - def getEntry(self, path) -> DirectoryEntry: - """ - Finds and returns a copy of an existing DirectoryEntry instance in the - writer. Use this method to check the internal status of an entry. - - :raises OSError: If the entry does not exist. - :raises ValueError: If access to an internal item is attempted. - """ - return copy.copy(self.__getEntry(inputToMsgPath(path))) - - def listItems(self, streams = True, storages = False) -> List[List[str]]: - """ - Returns a list of the specified items currently in the writter. - - :param streams: If True, includes the path for each stream in the list. - :param storages: If True, includes the path for each storage in the - list. - """ - # We are actually abusing the walk function a bit here to life much - # easier. The way we do this is to look at the current directory that - # the walk function is giving information about and then deciding what - # parts of it we want to use. Once we have all the paths created, we - # will then sort and return it to give an output similar, if not - # identical, to OleFileIO.listdir. The mentioned method sorts keeping - # case in mind. - if not streams and not storages: - return [] - - paths = [] - for currentDir, stor, stre in self.walk(): - if storages: - for name in stor: - paths.append(currentDir + [name]) - if streams: - for name in stre: - paths.append(currentDir + [name]) - - paths.sort() - return paths - - def renameEntry(self, path, newName : str) -> None: - """ - Changes the name of an entry, leaving it in it's current position. - - :raises OSError: If the entry does not exist or an entry with the new - name already exists, - :raises ValueError: If access to an internal item is attempted or the - new name provided is invalid. - """ - # First, validate the new name. - if not isinstance(newName, str): - raise ValueError('New name must be a string.') - if constants.RE_INVALID_OLE_PATH.search(newName): - raise ValueError('Invalid character(s) in new name. Must not contain the following characters: \\//!:') - if len(newName) > 31: - raise ValueError('New name must be less than 32 characters.') - - # Get the storage for our entry. Entry *must* exist. - _dir = self.__getContainingStorage(inputToMsgPath(path)) - - # See if an item in the storage already has that new name. - if newName.lower() in map(str.lower, _dir.keys()): - raise OSError('An entry with the new name already exists.') - - # Get the original name. - originalName = dictGetCasedKey(_dir, path[-1]) - - # Get the entry to change. - entry = _dir[originalName] - if isinstance(entry, dict): - dirData = entry - entry = entry['::DirectoryEntry'] - else: - dirData = None - - # Change the name on the entry first. - entry.name = newName - - # Now, we need to remove the item from the current storage and add it - # back with the new name. - del _dir[originalName] - - if dirData is None: - _dir[newName] = entry - else: - _dir[newName] = dirData - - def walk(self) -> Iterator[Tuple[List[str], List[str], List[str]]]: - """ - Functional equivelent to :function os.walk:, but for going over the file - structure of the OLE file to be written. Unlike :function os.walk:, it - takes no arguments. - - :returns: A tuple of three lists. The first is the path, as a list of - strings, for the directory (or an empty list for the root), the - second is a list of the storages in the current directory, and the - last is a list of the streams. Streams and storages are sorted - caselessly. - """ - toProcess = [([], self.__dirEntries)] - - # Go through the toProcess list, removing the last item every time to - # mimic the behavior of os.walk. - while toProcess: - currentDir, dirDict = toProcess.pop() - storages = [] - streams = [] - for name in sorted(dirDict.keys(), key = str.lower): - if not name.startswith('::'): - if isinstance(dirDict[name], dict): - storages.append(name) - toProcess.append((currentDir + [name], dirDict[name])) - else: - streams.append(name) - - yield (currentDir, storages, streams) - def write(self, path) -> None: """ Writes the data to the path specified. If :param path: has a write @@ -945,15 +527,12 @@ def write(self, path) -> None: # Make sure we close the file after everything, especially if there is # an error. try: - # Write each section, transferring data between functions where - # necessary. + ### First we need to write the header. offset = self._writeBeginning(f) entries = self._writeDirectoryEntries(f, offset) self._writeMini(f, entries) self._writeFinal(f) finally: - self._cleanupEntries() - if opened: f.close() diff --git a/extract_msg/properties.py b/extract_msg/properties.py index 90bddbc5..0f993a72 100644 --- a/extract_msg/properties.py +++ b/extract_msg/properties.py @@ -191,15 +191,6 @@ def props(self) -> Dict: """ return copy.deepcopy(self.__props) - @property - def _propDict(self) -> Dict: - """ - A direct reference to the underlying property dictionary. Used in one - place in the code, and not recommended to be used if you are not a - developer. Use `Properties.props` instead for a safe reference. - """ - return self.__props - @property def rawData(self) -> bytes: """ diff --git a/extract_msg/utils.py b/extract_msg/utils.py index 4ef39e06..713f6fe9 100644 --- a/extract_msg/utils.py +++ b/extract_msg/utils.py @@ -26,7 +26,7 @@ import tzlocal from html import escape as htmlEscape -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union from . import constants from .enums import AttachmentType @@ -142,18 +142,6 @@ def _open(name, mode, *args, **kwargs): return _open -def dictGetCasedKey(_dict : Dict, key : Any) -> Any: - """ - Retrieves the key from the dictionary with the proper casing using a - caseless key. - """ - try: - return next((x for x in _dict.keys() if x.lower() == key.lower())) - except StopIteration: - # If we couldn't find the key, raise a KeyError. - raise KeyError(key) - - def divide(string, length : int) -> List: """ Divides a string into multiple substrings of equal length. If there is not @@ -494,26 +482,10 @@ def inputToBytes(stringInputVar, encoding) -> bytes: def inputToMsgPath(inp) -> List: """ Converts the input into an msg path. - - :raises ValueError: The path contains an illegal character. """ if isinstance(inp, (list, tuple)): inp = '/'.join(inp) - - inp = inputToString(inp, 'utf-8') - - # Validate the path is okay. Normally we would check for '/' and '\', but - # we are expecting a string or similar which will use those as path - # separators, so we will ignore that for now. - if ':' in inp or '!' in inp: - raise ValueError('Illegal character ("!" or ":") found in MSG path.') - - ret = [x for x in inp.replace('\\', '/').split('/') if x] - - # One last thing to check: all path segments can be, at most, 31 characters - # (32 if you include the null character), so we should verify that. - if any(len(x) > 31 for x in ret): - raise ValueError('Path segments must not be greater than 31 characters.') + ret = [x for x in inputToString(inp, 'utf-8').replace('\\', '/').split('/') if x] return ret From d0272fd14cb1cde227ad974714e28b4f9106e6f6 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 17 Jan 2023 18:46:31 -0800 Subject: [PATCH 29/30] Fix changes to next-release (gotta love github) --- CHANGELOG.md | 22 +- README.rst | 7 +- changelog_temp.md | 7 - extract_msg/__init__.py | 4 +- extract_msg/attachment.py | 60 +- extract_msg/attachment_base.py | 35 +- extract_msg/constants.py | 3 + extract_msg/contact.py | 10 +- extract_msg/custom_attachments/__init__.py | 51 -- .../custom_attachments/custom_handler.py | 54 -- .../custom_attachments/outlook_image.py | 98 ---- extract_msg/custom_attachments/utils.py | 201 ------- extract_msg/enums.py | 12 - extract_msg/exceptions.py | 17 +- extract_msg/message_base.py | 18 +- extract_msg/msg.py | 19 +- extract_msg/named.py | 42 +- extract_msg/ole_writer.py | 542 ++++++++++++++++-- extract_msg/properties.py | 9 + extract_msg/utils.py | 32 +- 20 files changed, 651 insertions(+), 592 deletions(-) delete mode 100644 changelog_temp.md delete mode 100644 extract_msg/custom_attachments/__init__.py delete mode 100644 extract_msg/custom_attachments/custom_handler.py delete mode 100644 extract_msg/custom_attachments/outlook_image.py delete mode 100644 extract_msg/custom_attachments/utils.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2593b2f1..504335fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,23 @@ +**v0.39.0** +* [[TeamMsgExtractor #318](https://github.com/TeamMsgExtractor/msg-extractor/issues/318)] Added code to handle a standards violation (from what I can tell, anyways) caused by the attachment not having an `AttachMethod` property. The code will log a warning, attempt to detect the method, and throw a `StandardViolationError` if it fails. +* [[TeamMsgExtractor #320](https://github.com/TeamMsgExtractor/msg-extractor/issues/320)] Changed the way string named properties are handled to allow for the string stream to have some errors and still be parsed. Warnings about these errors will be logged. +* [[TeamMsgExtractor #326](https://github.com/TeamMsgExtractor/msg-extractor/issues/326)] Fixed a bug that could cause some files to error when exporting. +* Fixed an issue where creation and modification times were not being copied to the new OLE file created by `OleWriter`. +* Fixed up a few docstrings. +* Fixed a few issues in `MSGFile` regarding the `filename` keyword argument. +* Added new argument `rootPath` to `OleWriter.fromOleFile` for saving a specific directory from an OLE file instead of just copying the entire file. That directory will become the root of the new one. +* Adjusted code for `OleWriter` to generate certain values *only* at save time to make them more dynamic. This allows for existing streams to be properly edited (although has issues with allowing storages to be edited). +* Added new function `OleWriter.deleteEntry` to remove an entry that was already added. If the entry is a storage, all children will be removed too. +* Added new function `OleWriter.editEntry` to edit an entry that was already added. +* Added new function `OleWriter.addEntry` to add a new entry to the writer without an `OleFileIO` instance. Properties of the entry are instead set using the same keyword arguments as described in `OleWriter.editEntry`. +* Changed `_DirectoryEntry` to `DirectoryEntry` to make the more finalized version public. Access to the originals that the `OleWriter` class creates should never happen, instead copies should be returned to ensure the behavior is as expected. +* Added new function `OleWriter.getEntry` which returns a copy of the `DirectoryEntry` instance for that stream or storage in the writer. Use this function to see the current internal state of an entry. +* Added new function `OleWriter.renameEntry` which allows the user to rename a stream or storage (in place). This only changes it's direct name and not it's location in the new OLE file. +* Added new function `OleWriter.walk` which is similar to `os.walk` but for walking the structure of the new OLE file. +* Added new function `OleWriter.listItems` which is functionally equivalent to `olefile.OleFileIO.listdir` which returns a list of paths to every item. Optionally a user can get the paths just for streams, just for storages, or both. Requesting neither will simply return an empty list. Default is to just return streams. +* Added a small amount of path validation to `inputToMsgPath` which is used in a lot of places where user input for a path is accepted. It ensures illegal characters don't exist and that the path segments (each name for a storage or stream) are less than 32 characters. This will be most helpful for `OleWriter`. +* Added *many* internal helper functions to `OleWriter` to make extensions easier and consolidate common code. Many of these involve direct access to internal data which is why they are private. + **v0.38.4** * Fix line in `OleWriter` that was causing exporting to fail. * Fixed some issues with the `README`. @@ -18,7 +38,7 @@ * Added function `MSGFile.export` which copies all streams and storages from an MSG file into a new file. This can "clone" an MSG file or be used for extracting an MSG file that is embedded inside of another. * Added hidden function to `MSGFile` for getting the `OleDirectoryEntry` for a storage or stream. This is mainly for use by the `OleWriter` class. * Added option `extractEmbedded` to `Attachment.save` (`--extract-embedded` on the command line) which causes embedded MSG files to be extracted instead of running their save methods. -* Fixed minor issues with `utils.inputToMsgPath` (renamed from `utils.inputToMsgpath`). +* Fixed minor issues with `utils.inputToMsgPath` (renamed from `utils.inputToMsgPath`). * Renamed `utils.msgpathToString` to `utils.msgPathToString`. * Made some of the module requirements a little more strict to better version control. I'll be trying to make periodic checks for updates to the dependency packages and make sure that new versions are compatible before changing the allowed versions, while also trying to keep the requirements a bit flexible. diff --git a/README.rst b/README.rst index 68e001ff..33efaec7 100644 --- a/README.rst +++ b/README.rst @@ -19,7 +19,7 @@ This module has a Discord server for general discussion. You can find it here: Changelog --------- -- `Changelog`_ +- `Changelog `__ Usage ----- @@ -234,8 +234,8 @@ your access to the newest major version of extract-msg. .. |License: GPL v3| image:: https://img.shields.io/badge/License-GPLv3-blue.svg :target: LICENSE.txt -.. |PyPI3| image:: https://img.shields.io/badge/pypi-0.38.4-blue.svg - :target: https://pypi.org/project/extract-msg/0.38.4/ +.. |PyPI3| image:: https://img.shields.io/badge/pypi-0.38.5-blue.svg + :target: https://pypi.org/project/extract-msg/0.38.5/ .. |PyPI2| image:: https://img.shields.io/badge/python-3.6+-brightgreen.svg :target: https://www.python.org/downloads/release/python-367/ @@ -252,4 +252,3 @@ your access to the newest major version of extract-msg. .. _Ko-fi: https://ko-fi.com/destructione .. _Patreon: https://www.patreon.com/DestructionE .. _msg-explorer: https://pypi.org/project/msg-explorer/ -.. _Changelog: https://github.com/TeamMsgExtractor/msg-extractor/blob/master/CHANGELOG.md diff --git a/changelog_temp.md b/changelog_temp.md deleted file mode 100644 index bb6f63f4..00000000 --- a/changelog_temp.md +++ /dev/null @@ -1,7 +0,0 @@ -Temporary location for the changelog entry to ensure it doesn't conflict. - -**v0.??.??** -* Added new submodule `custom_attachments`. This submodule provides an extendable way to handle custom attachment types, attachment types whose structure and formatting are not defined in the Microsoft documentation for MSG files. -* Added new property `AttachmentBase.clsid` which returns the listed CLSID value of the data stream/storage of the attachment. -* Changed internal behavior of `MSGFile.attachments`. This should not cause any noticeable changes to the output. -* Removed some debug code that was left behind. diff --git a/extract_msg/__init__.py b/extract_msg/__init__.py index a3fedb44..fba28a0e 100644 --- a/extract_msg/__init__.py +++ b/extract_msg/__init__.py @@ -27,8 +27,8 @@ # along with this program. If not, see . __author__ = 'Destiny Peterson & Matthew Walker' -__date__ = '2022-12-03' -__version__ = '0.38.4' +__date__ = '2023-01-17' +__version__ = '0.39.0' import logging diff --git a/extract_msg/attachment.py b/extract_msg/attachment.py index 74673304..cf02bc82 100644 --- a/extract_msg/attachment.py +++ b/extract_msg/attachment.py @@ -5,12 +5,12 @@ import string import zipfile -from typing import Any, Dict, Optional, Union +from typing import Optional, Union from . import constants from .attachment_base import AttachmentBase -from .custom_attachments import CustomAttachmentHandler, getHandler from .enums import AttachmentType +from .exceptions import StandardViolationError from .utils import createZipOpen, inputToString, openMsg, prepareFilename @@ -33,7 +33,31 @@ def __init__(self, msg, dir_): located. """ super().__init__(msg, dir_) - self.__customHandler = None + + if '37050003' not in self.props: + from .prop import createProp + + logger.warning('Attahcment method property not found on attachment. Code will attempt to guess the type.') + + # Because this condition is actually kind of a violation of the + # standard, we are just going to do this in a dumb way. Basically we + # are going to try to set the attach method *manually* just so I + # don't have to go and modify the following code. + if self.exists('__substg1.0_37010102'): + # Set it as data and call it a day. + propData = b'\x03\x00\x057\x07\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00' + elif self.exists('__substg1.0_3701000D'): + # If it is a folder and we have properties, call it an MSG file. + if self.exists('__substg1.0_3701000D/__properties_version1.0'): + propData = b'\x03\x00\x057\x07\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00' + else: + # Call if custom attachment data. + propData = b'\x03\x00\x057\x07\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00' + else: + # Can't autodetect it, so throw an error. + raise StandardViolationError('Attachment method missing, and it could not be determined automatically.') + + self.props._propDict['37050003'] = createProp(propData) # Get attachment data. if self.exists('__substg1.0_37010102'): @@ -41,11 +65,9 @@ def __init__(self, msg, dir_): self.__data = self._getStream('__substg1.0_37010102') elif self.exists('__substg1.0_3701000D'): if (self.props['37050003'].value & 0x7) != 0x5: - self.__type = AttachmentType.CUSTOM - # Check if we have any custom handlers. If not, it will raise - # an error automatically. - self.__customHandler = getHandler(self) - self.__data = self.__customHandler.data + raise NotImplementedError( + 'Current version of extract_msg does not support extraction of containers that are not embedded msg files.') + # TODO add implementation. else: self.__prefix = msg.prefixList + [dir_, '__substg1.0_3701000D'] self.__type = AttachmentType.MSG @@ -81,10 +103,8 @@ def getFilename(self, **kwargs) -> str: # Check if user wants to save the file under the Content-ID. if kwargs.get('contentId', False): filename = self.cid - # If we are using a custom handler, prefer it's name. - if self.type is AttachmentType.CUSTOM: - filename = self.__customHandler.name - # If we are here, try to get the filename however else we can. + # If filename is None at this point, use long filename as first + # preference. if not filename: filename = self.name # Otherwise just make something up! @@ -127,16 +147,18 @@ def save(self, **kwargs) -> Optional[Union[str, 'MSGFile']]: to :param zip:. If :param zip: is an instance, :param customPath: will refer to a location inside the zip file. - :param extractEmbedded: If true, causes the attachment, should it be an + :param extractEmbedded: If True, causes the attachment, should it be an embedded MSG file, to save as a .msg file instead of calling it's save function. + :param skipEmbedded: If True, skips saving this attachment if it is an + embedded MSG file. """ # First check if we are skipping embedded messages and stop # *immediately* if we are. if self.type is AttachmentType.MSG and kwargs.get('skipEmbedded'): return None - # Check if the user has specified a custom filename + # Get the filename to use. filename = self.getFilename(**kwargs) # Someone managed to have a null character here, so let's get rid of that @@ -176,7 +198,7 @@ def save(self, **kwargs) -> Optional[Union[str, 'MSGFile']]: fullFilename = customPath / filename - if self.type is AttachmentType.DATA or (self.type is AttachmentType.CUSTOM and isinstance(self.__data, bytes)): + if self.type is AttachmentType.DATA: if _zip: name, ext = os.path.splitext(filename) nameList = _zip.namelist() @@ -232,14 +254,6 @@ def saveEmbededMessage(self, **kwargs) -> None: """ self.data.save(**kwargs) - @property - def customHandler(self) -> Optional[CustomAttachmentHandler]: - """ - The instance of the custom handler associated with this attachment, if - it has one. - """ - return self.__customHandler - @property def data(self) -> Optional[Union[bytes, 'MSGFile']]: """ diff --git a/extract_msg/attachment_base.py b/extract_msg/attachment_base.py index ef7c3bef..2b5a10ee 100644 --- a/extract_msg/attachment_base.py +++ b/extract_msg/attachment_base.py @@ -34,6 +34,7 @@ def __init__(self, msg, dir_): self.__props = Properties(self._getStream('__properties_version1.0'), PropertiesType.ATTACHMENT) self.__namedProperties = NamedProperties(msg.named, self) + def _ensureSet(self, variable, streamID, stringStream = True, **kwargs): """ Ensures that the variable exists, otherwise will set it using the @@ -270,39 +271,9 @@ def cid(self) -> Optional[str]: contendId = cid @property - def clsid(self) -> str: - """ - Returns the CLSID for the data stream/storage of the attachment. - """ - try: - return self.__clsid - except AttributeError: - # Set some default values. - self.__clsid = '00000000-0000-0000-0000-000000000000' - dataStream = None - - # See if we can find the data stream/storage. - if self.type in (AttachmentType.CUSTOM, AttachmentType.MSG): - dataStream = [self.__dir, '__substg1.0_3701000D'] - elif self.type is AttachmentType.DATA: - dataStream = [self.__dir, '__substg1.0_37010102'] - elif self.type is AttachmentType.UNSUPPORTED: - # Special check for custom attachments. - if self.exists('__substg1.0_3701000D'): - dataStream = [self.__dir, '__substg1.0_3701000D'] - elif self.exists('__substg1.0_37010102'): - dataStream = [self.__dir, '__substg1.0_37010102'] - - # If we found the right item, get the CLSID. - if dataStream: - self.__clsid = self.__msg._getOleEntry(dataStream).clsid or '00000000-0000-0000-0000-000000000000' - - return self.__clsid - - @property - def dir(self) -> str: + def dir(self): """ - Returns the directory inside the MSG file where the attachment is + Returns the directory inside the msg file where the attachment is located. """ return self.__dir diff --git a/extract_msg/constants.py b/extract_msg/constants.py index 9f89cdad..697639ba 100644 --- a/extract_msg/constants.py +++ b/extract_msg/constants.py @@ -50,6 +50,9 @@ # This is used in the workaround for decoding issues in RTFDE. We find `\bin` # sections and try to remove all of them to help with the decoding. RE_BIN = re.compile(br'\\bin([0-9]+) ?') +# Used in the vaildation of OLE paths. Any of these characters in a name make it +# invalid. +RE_INVALID_OLE_PATH = re.compile(r'[:/\\!]') FIXED_LENGTH_PROPS = ( 0x0000, diff --git a/extract_msg/contact.py b/extract_msg/contact.py index 941ae3ff..104411d0 100644 --- a/extract_msg/contact.py +++ b/extract_msg/contact.py @@ -19,6 +19,8 @@ def __init__(self, path, **kwargs): :param path: path to the msg file in the system or is the raw msg file. :param prefix: used for extracting embeded msg files inside the main one. Do not set manually unless you know what you are doing. + :param parentMsg: Used for synchronizing named properties instances. Do + not set this unless you know what you are doing. :param attachmentClass: optional, the class the MSGFile object will use for attachments. You probably should not change this value unless you know what you are doing. @@ -28,6 +30,8 @@ def __init__(self, path, **kwargs): be retrieved. :param filename: optional, the filename to be used by default when saving. + :param attachmentErrorBehavior: Optional, the behavior to use in the + event of an error when parsing the attachments. :param overrideEncoding: optional, an encoding to use instead of the one specified by the msg file. Do not report encoding errors caused by this. @@ -685,11 +689,11 @@ def headerFormatProperties(self) -> constants.HEADER_FORMAT_TYPE: If you class should not do *any* header injection, return None from this property. """ - def strListToStr(inp : Union[str, List[str]]): + def strListToStr(inp : Optional[Union[str, List[str]]]): """ Small internal function for things that may return a string or list. """ - if isinstance(inp, str): + if inp is None or isinstance(inp, str): return inp else: return ', '.join(inp) @@ -746,7 +750,7 @@ def strListToStr(inp : Union[str, List[str]]): 'Anniversary': self.weddingAnniversary.__format__('%B %d, %Y') if self.weddingAnniversaryLocal else None, 'Spouse/Partner': self.spouseName, 'Profession': self.profession, - 'Children': ', '.join(self.childrensNames), + 'Children': strListToStr(self.childrensNames), 'Hobbies': self.hobbies, 'Assistant': self.assistant, 'Web Page': self.webpageUrl, diff --git a/extract_msg/custom_attachments/__init__.py b/extract_msg/custom_attachments/__init__.py deleted file mode 100644 index 1b153c6f..00000000 --- a/extract_msg/custom_attachments/__init__.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -Submodule designed to help with saving and using custom attachments. Custom -attachments are those follow standards not defined in the MSG documentation. Use -the function `getHandler` to get an instance of a subclass of -CustomAttachmentHandler. - -CustomAttachmentHandler subclasses will all define the following methods: - injectHtml: A method which takes HTML and inserts the - -It should hopefully be completely unnecessary for your code to know what type of -handler it is using, as the abstract base class should give all of the functions -you would typically want. - -If you would like to add your own handler, simply subclass -CustomAttachmentHandler and add it using the `registerHandler` function. -""" - -from typing import List - -from .custom_handler import CustomAttachmentHandler - - -# Create a way to register handlers. -_knownHandlers : List[CustomAttachmentHandler] = [] -# This line is cheating a little bit, but is more efficient than wrapping it in -# a function. -registerHandler = _knownHandlers.append - - -# Import built-in handler modules. THey will all automatically register their -# respecive handler(s). -from .outlook_image import OutlookImage - - - - -# Function designed to route to the correct handler. -def getHandler(attachment : 'Attachment') -> CustomAttachmentHandler: - """ - Takes an attachment and uses it to find the correct handler. Returns an - instance created using the specified attachment. - - :raises NotImplementedError: No handler could be found. - :raises ValueError: A handler was found, but something was wrong with the - attachment data. - """ - for handler in _knownHandlers: - if handler.isCorrectHandler(attachment): - return handler(attachment) - - raise NotImplementedError('No valid handler could be found for the attachment. Contact the developers for help.') diff --git a/extract_msg/custom_attachments/custom_handler.py b/extract_msg/custom_attachments/custom_handler.py deleted file mode 100644 index c844151a..00000000 --- a/extract_msg/custom_attachments/custom_handler.py +++ /dev/null @@ -1,54 +0,0 @@ -import abc - -from typing import List, Optional, Tuple - - -class CustomAttachmentHandler(abc.ABC): - """ - A class designed to help with custom attachments that may require parsing in - special ways that are completely different from one another. - """ - def __init__(self, attachment : 'Attachment'): - super().__init__() - self.__att = attachment - - @classmethod - @abc.abstractmethod - def isCorrectHandler(cls, attachment : 'Attachment') -> bool: - """ - Checks if this is the correct handler for the attachment. - """ - - @abc.abstractmethod - def injectHTML(self, html : bytes, renderedList : Optional[List[str]] = None) -> Tuple[bytes, Optional[List[str]]]: - """ - Adds the relevent tag, if any, to the HTML for making prepared HTML. - - If this function should do nothing, returns the two arguments without - modification. - - :param html: The HTML body to inject into (if at all). - :param renderedList: The list to use (if needed) of "rendered - characters" which will be returned and matches the HTML returned. - """ - - @property - def attachment(self): - """ - The attachment this handler is associated with. - """ - return self.__att - - @property - @abc.abstractmethod - def data(self) -> bytes: - """ - Gets the data for the attachment. - """ - - @property - @abc.abstractmethod - def name(self) -> str: - """ - Returns the name to be used when saving the attachment. - """ diff --git a/extract_msg/custom_attachments/outlook_image.py b/extract_msg/custom_attachments/outlook_image.py deleted file mode 100644 index 54428599..00000000 --- a/extract_msg/custom_attachments/outlook_image.py +++ /dev/null @@ -1,98 +0,0 @@ -import base64 -import struct - -from typing import List, Optional, Tuple - -from . import registerHandler -from .custom_handler import CustomAttachmentHandler -from .utils import htmlSplitRendered -from ..enums import DVAspect -from ..exceptions import CustomAttachmentError - - -_ST_OLE = struct.Struct('' - - @classmethod - def isCorrectHandler(cls, attachment : 'Attachment') -> bool: - if attachment.clsid != '00000316-0000-0000-C000-000000000046': - return False - - # Check for the required streams. - if not attachment.exists('__substg1.0_3701000D/CONTENTS'): - return False - if not attachment.exists('__substg1.0_3701000D/\x01Ole'): - return False - if not attachment.exists('__substg1.0_3701000D/\x03MailStream'): - return False - - return True - - def injectHTML(self, html : bytes, renderedList : Optional[List[str]] = None) -> Tuple[bytes, Optional[List[str]]]: - if not renderedList: - renderedList = htmlSplitRendered(html) - - rp = self.attachment.renderingPosition - - if rp >= len(renderedList): - raise CustomAttachmentError(f'Rendering position beyond calculated number of rendered characters (expected less than {len(renderedList)}, got {rp}).') - - renderedList[rp] = self.__htmlTag + renderedList[rp] - - return (''.join(renderedList).encode('utf-8'), renderedList) - - @property - def data(self) -> bytes: - return self.__data - - @property - def name(self) -> str: - return self.attachment.shortFilename + '.bmp' - - - - -registerHandler(OutlookImage) diff --git a/extract_msg/custom_attachments/utils.py b/extract_msg/custom_attachments/utils.py deleted file mode 100644 index 8a61e3ec..00000000 --- a/extract_msg/custom_attachments/utils.py +++ /dev/null @@ -1,201 +0,0 @@ -""" -Utilities for extract-msg that are more specialized for the custom_attachments -submodule than for the main module. -""" - -import bs4 - -from typing import List - - -_WHITESPACE_BREAKERS = ( - ' bool: - """ - Helper function to indicate that a tag breaks a chain of whitespace. - """ - for x in _WHITESPACE_BREAKERS: - if token.startswith(x) and len(token) > len(x) and token[len(x)] in ('>', ' ', '/'): - return True - - return False - - -def _isWhitespaceToken(token : str) -> bool: - if token[0] == '<': - for x in _WHITESPACE_TAGS: - if token.startswith(x) and len(token) > len(x) and token[len(x)] in ('>', ' ', '/'): - return True - elif token in (' ', ' ', ' ', ' ', ' '): - return True - else: - return token.isspace() - - return False - - -def htmlSplitRendered(html : bytes) -> List[str]: - """ - Takes html bytes and returns a list of the rendered characters, with data - that is not being rendered being attached to the next rendered character. - """ - # Unfortunately bs4 didn't seem particularly great for tokenizing, so I did - # my own function that works well enough. First, let's tokenize the html. - tokens = tokenizeHtml(bs4.BeautifulSoup(html, features = 'html.parser').decode()) - - # Next, let's break things down further. - breakDown = [] - for token in tokens: - # We tell what we are looking at by checking the first character of the - # token. If it's a <, then it is an HTML tag. If it is a & then it is an - # escape. Otherwise, it is plain text. For both tags and escapes, just - # dump them into the list. - if token[0] in ('<', '&'): - breakDown.append(token) - else: - # If we are looking at plain text, add it by extending the list. - breakDown.extend(token) - - # Now that we have broken things down further, let's go through and join our - # pieces togethered into rendered tokens. Here is were we actually need to - # know what an html tag is. If it's an escape or just a non-whitespace - # character, we can just shove it onto what we currently have. - current = '' - renderedCharacters = [] - lastWhitespace = None - for item in breakDown: - if item[0] == '&': - if _isWhitespaceToken(item): - if lastWhitespace is None: - lastWhitespace == item - else: - if lastWhitespace is not None and lastWhitespace[0] != '<': - current += lastWhitespace - renderedCharacters.append(current) - current = '' - lastWhitespace = None - current += item - renderedCharacters.append(current) - current = '' - elif item[0] == '<': - if _isWhitespaceToken(item): - # If we are here, add it to current, push current, and set this - # tag as the last whitespace. - current += item - renderedCharacters.append(current) - current = '' - lastWhitespace = item - else: - - # Some tags will break whitespace chains. - if _isWhitespaceBreaker(item): - if lastWhitespace is not None and lastWhitespace[0] != '<': - current += lastWhitespace - renderedCharacters.append(current) - current = '' - lastWhitespace = None - - current += item - else: - # Here is where we handle text, which is not particularly fun. - # Basically if it is whitespace and lastWhitespace is not none, we - # set the whitespace. - if _isWhitespaceToken(item): - if lastWhitespace is None: - lastWhitespace = item - else: - if lastWhitespace is not None and lastWhitespace[0] != '<': - current += lastWhitespace - renderedCharacters.append(current) - current = '' - lastWhitespace = None - current += item - renderedCharacters.append(current) - current = '' - - if current: - renderedCharacters.append(current) - - return renderedCharacters - - -def tokenizeHtml(html : str) -> List[str]: - # Setup a few variables for state tracking. - inTag = False - # Used for tracking escapes starting with &. If your escape ends up at 100 - # characters because it is missing the semicolon, we are going to throw an - # error. - inEscape = False - inString = False - # Only used when in string. Last character was a backslash. - isBackslash = False - # Tells which quote type we are in. - isDoubleQuote = False - - tokens = [] - currentToken = '' - - # Finally, let's start breaking things up. Our rules are that if we start a - # quote while in a tag, then we acknowledge it, otherwise it is treated as - # plain text. - for character in html: - # First we need to know our state, as our state determines what how we - # process a character. - if inTag: - currentToken += character - if inString: - if character == '"' and isDoubleQuote: - # If isBackslash then we stay in the quote, otherwise... - isQuote = isBackslash - elif character == "'" and not isDoubleQuote: - # If isBackslash then we stay in the quote, otherwise... - isQuote = isBackslash - elif character == '\\': - isBackslash = not isBackslash - if character != '\\': - isBackslash = False - else: - if character == '>': - inTag = False - tokens.append(currentToken) - currentToken = '' - elif inEscape: - currentToken += character - if len(currentToken) > 99: - raise ValueError('Found escape that was too long (is a ; missing?)') - if character == ';': - tokens.append(currentToken) - currentToken = '' - inEscape = False - elif inString: - # This is an error. We should *never* be in a quote if we are not in - # a tag. - raise ValueError('Found to be inQuote when not in tag.') - else: - # We are currently processing plain text, so let's just handle. - if character == '&': - if currentToken: - tokens.append(currentToken) - currentToken = character - inEscape = True - elif character == '<': - if currentToken: - tokens.append(currentToken) - currentToken = character - inTag = True - inString = False - else: - currentToken += character - - if currentToken: - tokens.append(currentToken) - - return tokens diff --git a/extract_msg/enums.py b/extract_msg/enums.py index 93ea5bb1..3f746da4 100644 --- a/extract_msg/enums.py +++ b/extract_msg/enums.py @@ -113,7 +113,6 @@ class AttachmentType(enum.Enum): SIGNED = 3 BROKEN = 4 UNSUPPORTED = 5 - CUSTOM = 6 UNKNOWN = 0xFFFFFFFF @@ -317,7 +316,6 @@ class Color(enum.IntEnum): BLACK = 1 - class ContactAddressIndex(enum.Enum): EMAIL_1 = 0 EMAIL_2 = 1 @@ -378,16 +376,6 @@ class DisplayType(enum.Enum): -class DVAspect(enum.IntEnum): - """ - Part of the extra data for Outlook signatures. Different sources seem to - disagree on the meanings, so I'm sticking to the meanings in the official - Microsoft documentation of the DVASPECT enumeration. - """ - CONTENT = 1 - ICON = 4 - - class ElectronicAddressProperties(enum.Enum): @classmethod def fromBits(cls, value : int) -> Set['ElectronicAddressProperties']: diff --git a/extract_msg/exceptions.py b/extract_msg/exceptions.py index 1ce1fc88..e835df64 100644 --- a/extract_msg/exceptions.py +++ b/extract_msg/exceptions.py @@ -17,21 +17,19 @@ class BadHtmlError(ValueError): """ HTML failed to pass validation. """ + pass class ConversionError(Exception): """ An error occured during type conversion. """ - -class CustomAttachmentError(Exception): - """ - A generic error used for issues handling custom attachments. - """ + pass class DataNotFoundError(Exception): """ Requested stream type was unavailable. """ + pass class DeencapMalformedData(Exception): """ @@ -47,6 +45,7 @@ class ExecutableNotFound(Exception): """ Could not find the specified executable. """ + pass class IncompatibleOptionsError(Exception): """ @@ -57,22 +56,26 @@ class InvalidFileFormatError(OSError): """ An Invalid File Format Error occurred. """ + pass class InvaildPropertyIdError(Exception): """ The provided property ID was invalid. """ + pass class InvalidVersionError(Exception): """ The version specified is invalid. """ + pass class StandardViolationError(Exception): """ A critical violation of the MSG standards was detected and could not be recovered from. Recoverable violations will result in log messages instead. """ + pass class TZError(Exception): """ @@ -88,11 +91,13 @@ class UnknownCodepageError(Exception): """ The codepage provided was not one we know of. """ + pass class UnknownTypeError(Exception): """ The type specified is not one that is recognized. """ + pass class UnsupportedMSGTypeError(NotImplementedError): """ @@ -105,8 +110,10 @@ class UnrecognizedMSGTypeError(TypeError): An exception that is raised when the module cannot determine how to properly open a specific class of msg file. """ + pass class WKError(RuntimeError): """ An error occured while running wkhtmltopdf. """ + pass diff --git a/extract_msg/message_base.py b/extract_msg/message_base.py index 2d998242..841654d4 100644 --- a/extract_msg/message_base.py +++ b/extract_msg/message_base.py @@ -19,7 +19,7 @@ from typing import Callable, Dict, Optional, Tuple, Union from . import constants -from .enums import AttachmentType, DeencapType, RecipientType +from .enums import DeencapType, RecipientType from .exceptions import ( DataNotFoundError, DeencapMalformedData, DeencapNotEncapsulated, IncompatibleOptionsError, WKError @@ -79,8 +79,8 @@ def __init__(self, path, **kwargs): body is desired. The function should return a string for plain text and bytes for HTML. If any problems occur, the function *must* either return None or raise one of the appropriate functions from - extract_msg.exceptions. All other functions must be handled - internally or they will continue. The original deencapsulation + extract_msg.exceptions. All other exceptions must be handled + internally or they will not be caught. The original deencapsulation method will not run if this is set. """ super().__init__(path, **kwargs) @@ -1178,18 +1178,8 @@ def htmlBodyPrepared(self) -> Optional[bytes]: if not self.htmlBody: return self.htmlBody - html = self.htmlBody - - renderedCharacters = None - - # Iterate through all attachments, and inject all of the custom data - # from them. - for x in self.attachments: - if x.type is AttachmentType.CUSTOM: - html, renderedCharacters = x.customHandler.injectHTML(html, renderedCharacters) - # Create the BeautifulSoup instance to use. - soup = bs4.BeautifulSoup(html, 'html.parser') + soup = bs4.BeautifulSoup(self.htmlBody, 'html.parser') # Get a list of image tags to see if we can inject into. If the source # of an image starts with "cid:" that means it is one of the attachments diff --git a/extract_msg/msg.py b/extract_msg/msg.py index b58bfdde..6fecb2b0 100644 --- a/extract_msg/msg.py +++ b/extract_msg/msg.py @@ -37,10 +37,9 @@ def __init__(self, path, **kwargs): one. Do not set manually unless you know what you are doing. :param parentMsg: Used for synchronizing named properties instances. Do not set this unless you know what you are doing. - :param attachmentClass: Optional, the class the MSGFile object - will use for attachments. You probably should - not change this value unless you know what you - are doing. + :param attachmentClass: Optional, the class the MSGFile object will use + for attachments. You probably should not change this value unless + you know what you are doing. :param delayAttachments: Optional, delays the initialization of attachments until the user attempts to retrieve them. Allows MSG files with bad attachments to be initialized so the other data can @@ -112,6 +111,8 @@ def __init__(self, path, **kwargs): del kwargsCopy['prefix'] if 'parentMsg' in kwargsCopy: del kwargsCopy['parentMsg'] + if 'filename' in kwargsCopy: + del kwargsCopy['filename'] self.__kwargs = kwargsCopy prefixl = [] @@ -134,7 +135,7 @@ def __init__(self, path, **kwargs): self.__prefix = prefix self.__prefixList = prefixl self.__prefixLen = len(prefixl) - if prefix: + if prefix and not filename: filename = self._getStringStream(prefixl[:-1] + ['__substg1.0_3001'], prefix = False) if filename: self.filename = filename @@ -630,9 +631,10 @@ def attachments(self) -> List: # Get the attachments. attachmentDirs = [] prefixLen = self.prefixLen - for dir_ in self.listDir(False, True, False): - if dir_[0].startswith('__attach') and dir_[0] not in attachmentDirs: - attachmentDirs.append(dir_[0]) + for dir_ in self.listDir(False, True): + if dir_[prefixLen].startswith('__attach') and \ + dir_[prefixLen] not in attachmentDirs: + attachmentDirs.append(dir_[prefixLen]) self._attachments = [] @@ -640,6 +642,7 @@ def attachments(self) -> List: try: self._attachments.append(self.attachmentClass(self, attachmentDir)) except (NotImplementedError, UnrecognizedMSGTypeError) as e: + print("Hello") if self.attachmentErrorBehavior != AttachErrorBehavior.THROW: logger.error(f'Error processing attachment at {attachmentDir}') logger.exception(e) diff --git a/extract_msg/named.py b/extract_msg/named.py index b5c3d43f..be22cb19 100644 --- a/extract_msg/named.py +++ b/extract_msg/named.py @@ -21,14 +21,12 @@ def __init__(self, msg): # Get the basic streams. If all are emtpy, then nothing to do. guidStream = self._getStream('__substg1.0_00020102') or self._getStream('__substg1.0_00020102', False) entryStream = self._getStream('__substg1.0_00030102') or self._getStream('__substg1.0_00030102', False) - namesStream = self._getStream('__substg1.0_00040102') or self._getStream('__substg1.0_00040102', False) self.guidStream = guidStream self.entryStream = entryStream - self.namesStream = namesStream + self.namesStream = self._getStream('__substg1.0_00040102') or self._getStream('__substg1.0_00040102', False) # The if else stuff is for protection against None. guidStreamLength = len(guidStream) if guidStream else 0 entryStreamLength = len(entryStream) if entryStream else 0 - namesStreamLength = len(namesStream) if namesStream else 0 self.__propertiesDict = {} self.__properties = [] @@ -51,21 +49,11 @@ def __init__(self, msg): entry['guid'] = guids[entry['guid_index']] entries.append(entry) - # Parse the names stream. - names = self.__names - pos = 0 - while pos < namesStreamLength: - nameLength = constants.STNP_NAM.unpack(namesStream[pos:pos+4])[0] - pos += 4 # Move to the start of the entry. - names[pos - 4] = namesStream[pos:pos+nameLength].decode('utf-16-le') # Names are stored in the dictionary as the position they start at. - pos += roundUp(nameLength, 4) - self.entries = entries self.__guids = guids for entry in entries: - streamID = properHex(0x8000 + entry['pid']) - self.__properties.append(StringNamedProperty(entry, names[entry['id']]) if entry['pkind'] == NamedPropertyType.STRING_NAMED else NumericalNamedProperty(entry)) + self.__properties.append(StringNamedProperty(entry, self.__getName(entry['id'])) if entry['pkind'] == NamedPropertyType.STRING_NAMED else NumericalNamedProperty(entry)) for property in self.__properties: name = property.name if isinstance(property, StringNamedProperty) else property.propertyID @@ -80,6 +68,32 @@ def __iter__(self): def __len__(self) -> int: return self.__propertiesDict.__len__() + def __getName(self, offset : int) -> str: + """ + Parses the offset into the named stream and returns the name found. + """ + # We used to parse names by handing it as an array, as specified by the + # documentation, but this new method allows for a little bit more wiggle + # room in terms of what is accepted by the module. + if offset & 3 != 0: + # If the offset is not a multiple of 4, that is an error, but we are + # reducing it to a warning. + logger.warning(f'Malformed named properties detected due to bad offset ({offset}). Ignoring.') + # Check that offset is in string stream. + if offset > len(self.namesStream): + raise ValueError('Failed to parse named property: offset was not in string stream.') + + # Get the length, in bytes, of the string. + length = constants.STNP_NAM.unpack(self.namesStream[offset:offset + 4])[0] + offset += 4 + + # Make sure the string can be read entirely. If it can't, something was + # corrupt. + if offset + length > len(self.namesStream): + raise ValueError(f'Failed to parse named property: length ({length}) of string overflows the string stream. This is probably due to a bad offset.') + + return self.namesStream[offset:offset + length].decode('utf-16-le') + def _getStream(self, filename, prefix = True) -> Optional[bytes]: return self.__msg._getStream([self.__dir, filename], prefix = prefix) diff --git a/extract_msg/ole_writer.py b/extract_msg/ole_writer.py index 66e586f4..69a9cf44 100644 --- a/extract_msg/ole_writer.py +++ b/extract_msg/ole_writer.py @@ -1,24 +1,26 @@ +import copy import io import pathlib import re -from typing import List, Optional, Union +from typing import Dict, Iterator, List, Optional, Tuple, Union from . import constants from .enums import Color, DirectoryEntryType -from .utils import ceilDiv, inputToMsgPath +from .utils import ceilDiv, dictGetCasedKey, inputToMsgPath from olefile.olefile import OleDirectoryEntry, OleFileIO from red_black_dict_mod import RedBlackTree -class _DirectoryEntry: +class DirectoryEntry: """ - Hidden class, will probably be modified later. + An internal representation of a stream or storage in the OleWriter. + Originals should be inaccessible outside of the class. """ name : str = '' - rightChild : '_DirectoryEntry' = None - leftChild : '_DirectoryEntry' = None - childTreeRoot : '_DirectoryEntry' = None + rightChild : 'DirectoryEntry' = None + leftChild : 'DirectoryEntry' = None + childTreeRoot : 'DirectoryEntry' = None stateBits : int = 0 creationTime : int = 0 modifiedTime : int = 0 @@ -35,7 +37,7 @@ class _DirectoryEntry: startingSectorLocation : int = 0 color : Color = Color.BLACK - clsid : bytes = b'' + clsid : bytes = b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' data : bytes = b'' def __init__(self): @@ -79,7 +81,10 @@ class OleWriter: [MS-CFB]. """ def __init__(self, rootClsid : bytes = constants.DEFAULT_CLSID): - self.__rootClsid = rootClsid + self.__rootEntry = DirectoryEntry() + self.__rootEntry.name = "Root Entry" + self.__rootEntry.type = DirectoryEntryType.ROOT_STORAGE + self.__rootEntry.clsid = rootClsid # The root entry will always exist, so this must be at least 1. self.__dirEntryCount = 1 self.__dirEntries = {} @@ -87,6 +92,166 @@ def __init__(self, rootClsid : bytes = constants.DEFAULT_CLSID): self.__largeEntrySectors = 0 self.__numMinifatSectors = 0 + def __getContainingStorage(self, path : List[str], entryExists : bool = True, create : bool = False) -> Dict: + """ + Finds the storage dict internally where the entry specified by + :param path: would be created. If :param create: is True, missing + storages will be created with default settings. + + :param entryExists: If True, throws an error when the requested entry + does not yet exist. + :param create: If True, creates missing storages with default settings. + + :raises OSError: If :param create: is False and the path could not be + found. Also raised if :param entryExists: is True and the requested + entry does not exist. + :raises ValueError: Tried to access an interal stream or tried to use + both the create option and the entryExists option as True. + + :returns: The storage dict that the entry is in. + """ + # Quick check for incompatability between create and entryExists. + if create and entryExists: + raise ValueError(':param create: and :param entryExists: cannot both be True (an entry cannot exist if it is being created).') + + # Check that the path is not an internal entry. Given the validation on + # paths that most functions should do because of the call to + # inputToMsgPath, this shouldn't actually be necessary. + if any(x.startswith('::') for x in path): + raise ValueError('Found internal name in path.') + + _dir = self.__dirEntries + + for index, name in enumerate(path[:-1]): + # If no entry in the current stream matches the path, raise an + # OSError, *unless* the option to create storages is True. + if name.lower() not in map(str.lower, _dir.keys()): + if create: + self.addEntry(path[:index + 1], storage = True) + else: + raise OSError(f'Entry not found: {name}') + _dir = _dir[dictGetCasedKey(_dir, name)] + + # If the current item is not a storage and we have more to the path, + # raise an OSError. + if not isinstance(_dir, dict): + raise OSError('Attempted to access children of a stream.') + + if entryExists and path[-1].lower() not in map(str.lower, _dir.keys()): + raise OSError(f'Entry not found: {path[-1]}') + + return _dir + + def __getEntry(self, path : List[str]) -> DirectoryEntry: + """ + Finds and returns an existing DirectoryEntry instance in the writer. + + :raises OSError: If the entry does not exist. + :raises ValueError: If access to an internal item is attempted. + """ + _dir = self.__getContainingStorage(path) + item = _dir[dictGetCasedKey(_dir, path[-1])] + if isinstance(item, dict): + return item['::DirectoryEntry'] + else: + return item + + def __modifyEntry(self, entry : DirectoryEntry, **kwargs): + """ + Edits the DirectoryEntry with the data provided. Common code used for + :method addEntry: and :method editEntry:. + + :raises ValueError: Some part of the data given to modify the various + properties was invalid. See the the listed methods for details. + """ + # Extract the arguments. + data = kwargs.get('data') + clsid = kwargs.get('clsid') + creationTime = kwargs.get('creationTime') + modifiedTime = kwargs.get('modifiedTime') + stateBits = kwargs.get('stateBits') + + # I don't like that I have repeated if statements for checking each of + # the arguments, but I need to make sure nothing changes if something is + # invalid. + if data is not None: + if entry.type is not DirectoryEntryType.STREAM: + raise ValueError('Cannot set the data of a storage object.') + if not isinstance(data, bytes): + raise ValueError('Data must be a bytes instance if set.') + + if clsid is not None: + if not isinstance(clsid, bytes): + raise ValueError('CLSID must be bytes.') + if len(clsid) != 16: + raise ValueError('CLSID must be 16 bytes.') + + if creationTime is not None: + if entry.type is DirectoryEntryType.STREAM: + raise ValueError('Modification of creation time cannot be done on a stream.') + if not isinstance(creationTime, int) or creationTime < 0 or creationTime > 0xFFFFFFFFFFFFFFFF: + raise ValueError('Creation time must be a positive 8 byte int.') + + if modifiedTime is not None: + if entry.type is DirectoryEntryType.STREAM: + raise ValueError('Modification of modified time cannot be done on a stream.') + if not isinstance(modifiedTime, int) or modifiedTime < 0 or modifiedTime > 0xFFFFFFFFFFFFFFFF: + raise ValueError('Modified time must be a positive 8 byte int.') + + if stateBits is not None: + if not isinstance(stateBits, int) or stateBits < 0 or stateBits > 0xFFFFFFFF: + raise ValueError('State bits must be a positive 4 byte int.') + + + # Now that all our checks have passed, let's set our data. + if data is not None: + entry.data = data + if clsid is not None: + entry.clsid = clsid + if creationTime is not None: + entry.creationTime = creationTime + if modifiedTime is not None: + entry.modifiedTime = modifiedTime + if stateBits is not None: + entry.stateBits = stateBits + + def __recalculateSectors(self) -> None: + """ + Recalculates several of the internal variables used for saving that + specify the number of sectors and where things should go. + """ + self.__dirEntryCount = 0 + self.__numMinifatSectors = 0 + self.__largeEntries.clear() + self.__largeEntrySectors = 0 + + count = 0 + for entry in self.__walkEntries(): + self.__dirEntryCount += 1 + if entry.type == DirectoryEntryType.STREAM: + if len(entry.data) < 4096: + self.__numMinifatSectors += ceilDiv(len(entry.data), 64) + else: + self.__largeEntries.append(entry) + self.__largeEntrySectors += ceilDiv(len(entry.data), 512) + + def __walkEntries(self) -> Iterator[DirectoryEntry]: + """ + Returns a generator that will walk the entires recursively. Each item + returned by it will be a DirectoryEntry instance. + """ + toProcess = [self.__dirEntries] + yield self.__rootEntry + + while len(toProcess) > 0: + for name, item in toProcess.pop(0).items(): + if not name.startswith('::'): + if isinstance(item, dict): + yield item['::DirectoryEntry'] + toProcess.append(item) + else: + yield item + @property def __numberOfSectors(self) -> int: """ @@ -100,9 +265,27 @@ def __numberOfSectors(self) -> int: @property def __numMinifat(self) -> int: + """ + The number of FAT sectors needed to store the mini FAT. + """ return ceilDiv(self.__numMinifatSectors, 8) - def _getFatSectors(self): + def _cleanupEntries(self) -> None: + """ + Cleans up the node connections by walking the tree and removing + references that were added during writing. + """ + self.__largeEntries.clear() + for entry in self.__walkEntries(): + entry.id = -1 + entry.leftChild = None + entry.rightChild = None + entry.childTreeRoot = None + entry.leftSiblingID = 0xFFFFFFFF + entry.rightSiblingID = 0xFFFFFFFF + entry.childID = 0xFFFFFFFF + + def _getFatSectors(self) -> Tuple[int, int, int]: """ Returns a tuple containing the number of FAT sectors, the number of DIFAT sectors, and the total number of sectors the saved file will have. @@ -119,19 +302,19 @@ def _getFatSectors(self): return (numFat, numDifat, self.__numberOfSectors + numDifat + numFat) - def _treeSort(self, startingSector : int) -> List[_DirectoryEntry]: + def _treeSort(self, startingSector : int) -> List[DirectoryEntry]: """ Uses red-black trees to sort the internal data in preparation for writing the file, returning a list, in order, of the entries to write. """ # First, create the root entry. - root = _DirectoryEntry() - root.name = "Root Entry" - root.type = DirectoryEntryType.ROOT_STORAGE - root.clsid = self.__rootClsid + root = copy.copy(self.__rootEntry) + # Add the location of the start of the mini stream. root.startingSectorLocation = (startingSector + ceilDiv(self.__dirEntryCount, 4) + ceilDiv(self.__numMinifatSectors, 128)) if self.__numMinifat > 0 else 0xFFFFFFFE root.streamSize = self.__numMinifatSectors * 64 + root.childTreeRoot = None + root.childID = 0xFFFFFFFF entries = [root] toProcess = [(root, self.__dirEntries)] @@ -153,9 +336,9 @@ def _treeSort(self, startingSector : int) -> List[_DirectoryEntry]: # the processing list. if isinstance(val, dict): toProcess.append((val['::DirectoryEntry'], val)) - entries.append(val['::DirectoryEntry']) - else: - entries.append(val) + val = val['::DirectoryEntry'] + + entries.append(val) # Add the data to the tree. tree.add((len(name), name.upper()), val) @@ -169,23 +352,15 @@ def _treeSort(self, startingSector : int) -> List[_DirectoryEntry]: for node in tree.in_order(): item = node.value # Set the color immediately. - if isinstance(item, dict): - item = item['::DirectoryEntry'] item.color = Color.BLACK if node.is_black else Color.RED + if node.left: - val = node.left.value - if isinstance(val, _DirectoryEntry): - item.leftChild = val - else: - item.leftChild = val['::DirectoryEntry'] + item.leftChild = node.left.value else: item.leftChild = None + if node.right: - val = node.right.value - if isinstance(val, _DirectoryEntry): - item.rightChild = val - else: - item.rightChild = val['::DirectoryEntry'] + item.rightChild = node.right.value else: item.rightChild = None @@ -222,6 +397,8 @@ def _writeBeginning(self, f) -> int: :returns: The current sector number after all the data is written. """ + # Recalculate some things needed for saving. + self.__recalculateSectors() # Since we are going to need these multiple times, get them now. numFat, numDifat, totalSectors = self._getFatSectors() @@ -345,7 +522,7 @@ def _writeBeginning(self, f) -> int: # Finally, return the current sector index for use in other places. return numDifat + numFat - def _writeDirectoryEntries(self, f, startingSector : int) -> List[_DirectoryEntry]: + def _writeDirectoryEntries(self, f, startingSector : int) -> List[DirectoryEntry]: """ Writes out all the directory entries. Returns the list generated. """ @@ -357,7 +534,7 @@ def _writeDirectoryEntries(self, f, startingSector : int) -> List[_DirectoryEntr return entries - def _writeDirectoryEntry(self, f, entry : _DirectoryEntry) -> None: + def _writeDirectoryEntry(self, f, entry : DirectoryEntry) -> None: """ Writes the directory entry to the file f. """ @@ -373,7 +550,7 @@ def _writeFinal(self, f) -> None: if len(x.data) & 511: f.write(b'\x00' * (512 - (len(x.data) & 511))) - def _writeMini(self, f, entries : List[_DirectoryEntry]) -> None: + def _writeMini(self, f, entries : List[DirectoryEntry]) -> None: """ Writes the mini FAT followed by the full mini stream. """ @@ -403,40 +580,81 @@ def _writeMini(self, f, entries : List[_DirectoryEntry]) -> None: if self.__numMinifatSectors & 7: f.write((b'\x00' * 64) * (8 - (self.__numMinifatSectors & 7))) + def addEntry(self, path, data : bytes = None, storage : bool = False, **kwargs) -> None: + """ + Adds an entry to the OleWriter instance at the path specified, adding + storages with default settings where necessary. If the entry is not a + storage, :param data: *must* be set. + + :param path: The path to add the entry at. Must not contain a path part + that is an already added stream. + :param data: The bytes for a stream. + :param storage: If True, the entry to add is a storage. Otherwise, the + entry is a stream. + :param clsid: The CLSID for the stream/storage. Must a a bytes instance + that is 16 bytes long. + :param creationTime: An 8 byte filetime int. Sets the creation time of + the entry. Not applicable to streams. + :param modifiedTime: An 8 byte filetime int. Sets the modification time + of the entry. Not applicable to streams. + :param stateBits: A 4 byte int. Sets the state bits, user-defined flags, + of the entry. For a stream, this *SHOULD* be unset. + + :raises OSError: A stream was found on the path before the end. + :raises ValueError: Attempts to access an internal item. + """ + path = inputToMsgPath(path) + # First, find the current place in our dict to add the item. + _dir = self.__getContainingStorage(path, False, True) + # Now, check that the item *is not* already in our dict, as that would + # cause problems. + if path[-1].lower() in map(str.lower, _dir.keys()): + raise OSError('Cannot add an entry that already exists.') + + # Create a new entry with basic data and insert it. + entry = DirectoryEntry() + entry.type = DirectoryEntryType.STORAGE if storage else DirectoryEntryType.STREAM + entry.name = path[-1] + self.__modifyEntry(entry, data = data, **kwargs) + if storage: + _dir[path[-1]] = {'::DirectoryEntry': entry} + else: + _dir[path[-1]] = entry + def addOleEntry(self, path, entry : OleDirectoryEntry, data : Optional[bytes] = None) -> None: """ Uses the entry provided to add the data to the writer. - :raises ValueError: Tried to add an entry to a path that has not yet - been added. + :raises OSError: Tried to add an entry to a path that has not yet + been added, tried to add as a child of a stream, or tried to add an + entry where one already exists under the same name. """ - pathList = inputToMsgPath(path) + path = inputToMsgPath(path) # First, find the current place in our dict to add the item. - _dir = self.__dirEntries - while len(pathList) > 1: - if pathList[0] not in _dir: - # If no entry has been provided already for the directory, that - # is considered a fatal error. - raise ValueError('Path not found.') - _dir = _dir[pathList[0]] - pathList.pop(0) + _dir = self.__getContainingStorage(path, False) + # Now, check that the item *is not* already in our dict, as that would + # cause problems. + if path[-1].lower() in map(str.lower, _dir.keys()): + raise OSError('Cannot add an entry that already exists.') # Now that we are in the right place, add our data. - newEntry = _DirectoryEntry() + newEntry = DirectoryEntry() if entry.entry_type == DirectoryEntryType.STORAGE: # Handle a storage entry. # First add the dict to our tree of items. - _dir[pathList[0]] = {'::DirectoryEntry': newEntry} + _dir[path[-1]] = {'::DirectoryEntry': newEntry} # Finally, setup the values for the stream. newEntry.name = entry.name newEntry.type = DirectoryEntryType.STORAGE newEntry.clsid = _unClsid(entry.clsid) newEntry.stateBits = entry.dwUserFlags + newEntry.creationTime = entry.createTime + newEntry.modifiedTime = entry.modifyTime else: # Handle a stream entry. # First add the entry to out dict of entries. - _dir[pathList[0]] = newEntry + _dir[path[-1]] = newEntry newEntry.name = entry.name newEntry.type = DirectoryEntryType.STREAM newEntry.clsid = _unClsid(entry.clsid) @@ -444,20 +662,64 @@ def addOleEntry(self, path, entry : OleDirectoryEntry, data : Optional[bytes] = # Finally, handle the data. newEntry.data = data or b'' - if len(newEntry.data) < 4096: - self.__numMinifatSectors += ceilDiv(len(newEntry.data), 64) - else: - self.__largeEntries.append(newEntry) - self.__largeEntrySectors += ceilDiv(len(newEntry.data), 512) self.__dirEntryCount += 1 + def deleteEntry(self, path) -> None: + """ + Deletes the entry specified by :param path:, including all children. + + :raises OSError: If the entry does not exist or a part of the path that + is not the last was a stream. + :raises ValueError: Attempted to delete an internal data stream. + """ + path = inputToMsgPath(path) + # Get the containing storage for the entry. + _dir = self.__getContainingStorage(path) + + # The garbage collector will take care of all the loose items, so just + # remove the entry. Also, once again we deal with the case insensitive + # nature of the path. Even though comparisons are case insensitive, the + # path does remember the case used. + del _dir[dictGetCasedKey(_dir, path[-1])] + + def editEntry(self, path, **kwargs) -> None: + """ + Used to edit values of an entry by setting the specific kwargs. Set a + value to something other than None to set it. + + :param data: The data of a stream. Will error if used for something + other than a stream. + :param clsid: The CLSID for the stream/storage. Must a a bytes instance + that is 16 bytes long. + :param creationTime: An 8 byte filetime int. Sets the creation time of + the entry. Not applicable to streams. + :param modifiedTime: An 8 byte filetime int. Sets the modification time + of the entry. Not applicable to streams. + :param stateBits: A 4 byte int. Sets the state bits, user-defined flags, + of the entry. For a stream, this *SHOULD* be unset. + + + To convert a 32 character hexadecial CLSID into the bytes for this + function, the _unClsid function in the ole_writer submodule can be used. + + :raises OSError: The entry does not exist in the file. + :raises TypeError: Attempted to modify the bytes of a storage. + :raises ValueError: The type of a parameter was wrong, or the data of a + parameter was invalid. + """ + # First, find our entry to edit. + entry = self.__getEntry(inputToMsgPath(path)) + + # Send it to be modified using the arguments given. + self.__modifyEntry(entry, **kwargs) + def fromMsg(self, msg : 'MSGFile') -> None: """ Copies the streams and stream information necessary from the MSG file. """ # Get the root OLE entry's CLSID. - self.__rootClsid = _unClsid(msg._getOleEntry('/').clsid) + self.__rootEntry.clsid = _unClsid(msg._getOleEntry('/').clsid) # List both storages and directories, but sort them by shortest length # first to prevent errors. @@ -491,25 +753,180 @@ def fromMsg(self, msg : 'MSGFile') -> None: for x in gen: self.addOleEntry(x, msg._getOleEntry(x, prefix = False), msg._getStream(x, prefix = False)) - def fromOleFile(self, ole : OleFileIO) -> None: + def fromOleFile(self, ole : OleFileIO, rootPath = []) -> None: """ Copies all the streams from the proided OLE file into this writer. + + NOTE: This method does *not* handle any special rule that may be + required by a format that uses the compound binary file format as a base + when extracting an embedded directory. For example, MSG files require + modification of an embedded properties stream when extracting an + embedded MSG file. + + :param rootPath: A path (accepted by olefile.OleFileIO) to the directory + to use as the root of the file. If not provided, the file root will + be used. + + :raises OSError: If :param rootPath: does not exist in the file. """ - # Copy the clsid of the root entry. - self.__rootClsid = _unClsid(ole.direntries[0].clsid) + rootPath = inputToMsgPath(rootPath) + + # Check if the root path is simply the top of the file. + if rootPath == []: + # Copy the clsid of the root entry. + self.__rootEntry.clsid = _unClsid(ole.direntries[0].clsid) + paths = {tuple(x): (x, ole.direntries[ole._find(x)]) for x in ole.listdir(True, True)} + else: + # If it is not the top of the file, we need to do some filtering. + # First get the CLSID from the entry the path points to. + try: + entry = ole.direntries[ole._find(rootPath)] + self.__rootEntry.clsid = _unClsid(entry.clsid) + + except OSError as e: + if str(e) == 'file not found': + # Get the cause/context for the original exception and use + # it for the new exception. This hides the exception from + # OleFileIO. + context = e.__cause__ or e.__context__ + raise OSError('Root path was not found in the OLE file.') from context + else: + raise + + paths = {tuple(x[len(rootPath):]): (x, ole.direntries[ole._find(x)]) + for x in ole.listdir(True, True) if len(x) > len(rootPath)} - # Copy all of the other entries. - for x in ole.listdir(True, True): - entry = ole.direntries[ole._find(x)] + + # Copy all of the other entries. Ensure that directories come before + # their streams by sorting the paths. + for x in sorted(paths.keys()): + fullPath, entry = paths[x] if entry.entry_type == DirectoryEntryType.STREAM: - with ole.openstream(x) as f: + with ole.openstream(fullPath) as f: data = f.read() else: data = None self.addOleEntry(x, entry, data) + def getEntry(self, path) -> DirectoryEntry: + """ + Finds and returns a copy of an existing DirectoryEntry instance in the + writer. Use this method to check the internal status of an entry. + + :raises OSError: If the entry does not exist. + :raises ValueError: If access to an internal item is attempted. + """ + return copy.copy(self.__getEntry(inputToMsgPath(path))) + + def listItems(self, streams = True, storages = False) -> List[List[str]]: + """ + Returns a list of the specified items currently in the writter. + + :param streams: If True, includes the path for each stream in the list. + :param storages: If True, includes the path for each storage in the + list. + """ + # We are actually abusing the walk function a bit here to life much + # easier. The way we do this is to look at the current directory that + # the walk function is giving information about and then deciding what + # parts of it we want to use. Once we have all the paths created, we + # will then sort and return it to give an output similar, if not + # identical, to OleFileIO.listdir. The mentioned method sorts keeping + # case in mind. + if not streams and not storages: + return [] + + paths = [] + for currentDir, stor, stre in self.walk(): + if storages: + for name in stor: + paths.append(currentDir + [name]) + if streams: + for name in stre: + paths.append(currentDir + [name]) + + paths.sort() + return paths + + def renameEntry(self, path, newName : str) -> None: + """ + Changes the name of an entry, leaving it in it's current position. + + :raises OSError: If the entry does not exist or an entry with the new + name already exists, + :raises ValueError: If access to an internal item is attempted or the + new name provided is invalid. + """ + # First, validate the new name. + if not isinstance(newName, str): + raise ValueError('New name must be a string.') + if constants.RE_INVALID_OLE_PATH.search(newName): + raise ValueError('Invalid character(s) in new name. Must not contain the following characters: \\//!:') + if len(newName) > 31: + raise ValueError('New name must be less than 32 characters.') + + # Get the storage for our entry. Entry *must* exist. + _dir = self.__getContainingStorage(inputToMsgPath(path)) + + # See if an item in the storage already has that new name. + if newName.lower() in map(str.lower, _dir.keys()): + raise OSError('An entry with the new name already exists.') + + # Get the original name. + originalName = dictGetCasedKey(_dir, path[-1]) + + # Get the entry to change. + entry = _dir[originalName] + if isinstance(entry, dict): + dirData = entry + entry = entry['::DirectoryEntry'] + else: + dirData = None + + # Change the name on the entry first. + entry.name = newName + + # Now, we need to remove the item from the current storage and add it + # back with the new name. + del _dir[originalName] + + if dirData is None: + _dir[newName] = entry + else: + _dir[newName] = dirData + + def walk(self) -> Iterator[Tuple[List[str], List[str], List[str]]]: + """ + Functional equivelent to :function os.walk:, but for going over the file + structure of the OLE file to be written. Unlike :function os.walk:, it + takes no arguments. + + :returns: A tuple of three lists. The first is the path, as a list of + strings, for the directory (or an empty list for the root), the + second is a list of the storages in the current directory, and the + last is a list of the streams. Streams and storages are sorted + caselessly. + """ + toProcess = [([], self.__dirEntries)] + + # Go through the toProcess list, removing the last item every time to + # mimic the behavior of os.walk. + while toProcess: + currentDir, dirDict = toProcess.pop() + storages = [] + streams = [] + for name in sorted(dirDict.keys(), key = str.lower): + if not name.startswith('::'): + if isinstance(dirDict[name], dict): + storages.append(name) + toProcess.append((currentDir + [name], dirDict[name])) + else: + streams.append(name) + + yield (currentDir, storages, streams) + def write(self, path) -> None: """ Writes the data to the path specified. If :param path: has a write @@ -527,12 +944,15 @@ def write(self, path) -> None: # Make sure we close the file after everything, especially if there is # an error. try: - ### First we need to write the header. + # Write each section, transferring data between functions where + # necessary. offset = self._writeBeginning(f) entries = self._writeDirectoryEntries(f, offset) self._writeMini(f, entries) self._writeFinal(f) finally: + self._cleanupEntries() + if opened: f.close() diff --git a/extract_msg/properties.py b/extract_msg/properties.py index 0f993a72..90bddbc5 100644 --- a/extract_msg/properties.py +++ b/extract_msg/properties.py @@ -191,6 +191,15 @@ def props(self) -> Dict: """ return copy.deepcopy(self.__props) + @property + def _propDict(self) -> Dict: + """ + A direct reference to the underlying property dictionary. Used in one + place in the code, and not recommended to be used if you are not a + developer. Use `Properties.props` instead for a safe reference. + """ + return self.__props + @property def rawData(self) -> bytes: """ diff --git a/extract_msg/utils.py b/extract_msg/utils.py index 713f6fe9..4ef39e06 100644 --- a/extract_msg/utils.py +++ b/extract_msg/utils.py @@ -26,7 +26,7 @@ import tzlocal from html import escape as htmlEscape -from typing import Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union from . import constants from .enums import AttachmentType @@ -142,6 +142,18 @@ def _open(name, mode, *args, **kwargs): return _open +def dictGetCasedKey(_dict : Dict, key : Any) -> Any: + """ + Retrieves the key from the dictionary with the proper casing using a + caseless key. + """ + try: + return next((x for x in _dict.keys() if x.lower() == key.lower())) + except StopIteration: + # If we couldn't find the key, raise a KeyError. + raise KeyError(key) + + def divide(string, length : int) -> List: """ Divides a string into multiple substrings of equal length. If there is not @@ -482,10 +494,26 @@ def inputToBytes(stringInputVar, encoding) -> bytes: def inputToMsgPath(inp) -> List: """ Converts the input into an msg path. + + :raises ValueError: The path contains an illegal character. """ if isinstance(inp, (list, tuple)): inp = '/'.join(inp) - ret = [x for x in inputToString(inp, 'utf-8').replace('\\', '/').split('/') if x] + + inp = inputToString(inp, 'utf-8') + + # Validate the path is okay. Normally we would check for '/' and '\', but + # we are expecting a string or similar which will use those as path + # separators, so we will ignore that for now. + if ':' in inp or '!' in inp: + raise ValueError('Illegal character ("!" or ":") found in MSG path.') + + ret = [x for x in inp.replace('\\', '/').split('/') if x] + + # One last thing to check: all path segments can be, at most, 31 characters + # (32 if you include the null character), so we should verify that. + if any(len(x) > 31 for x in ret): + raise ValueError('Path segments must not be greater than 31 characters.') return ret From 6fc3c017f2b8b795329577f23ce2a93da23a74f5 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 17 Jan 2023 19:22:01 -0800 Subject: [PATCH 30/30] Update README for new version --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 33efaec7..7a653cd2 100644 --- a/README.rst +++ b/README.rst @@ -234,8 +234,8 @@ your access to the newest major version of extract-msg. .. |License: GPL v3| image:: https://img.shields.io/badge/License-GPLv3-blue.svg :target: LICENSE.txt -.. |PyPI3| image:: https://img.shields.io/badge/pypi-0.38.5-blue.svg - :target: https://pypi.org/project/extract-msg/0.38.5/ +.. |PyPI3| image:: https://img.shields.io/badge/pypi-0.39.0-blue.svg + :target: https://pypi.org/project/extract-msg/0.39.0/ .. |PyPI2| image:: https://img.shields.io/badge/python-3.6+-brightgreen.svg :target: https://www.python.org/downloads/release/python-367/