From b784a83dcf3e37c6380e3f0a3db6647777ba4150 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sat, 12 Aug 2023 17:04:17 -0700 Subject: [PATCH 01/68] Fixes to previous version's changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cd45afe..e54b4dea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ * `getMultipleBinary`: Gets a multiple binary property as a list of `bytes` objects. * `getSingleOrMultipleBinary`: A combination of `getStream` and `getMultipleBinary` which prefers a single binary stream. Returns a single `bytes` object or a list of `bytes` objects. * `getMultipleString`: Gets a multiple string property as a list of `str` objects. - * `getSingleOrMultipleString`: A combination of `getStringStream` and `getMultipleString` which prefers a single string stream. Returns a single bytes objecct or a list of bytes objects. + * `getSingleOrMultipleString`: A combination of `getStringStream` and `getMultipleString` which prefers a single string stream. Returns a single bytes object or a list of bytes objects. * `getPropertyVal`: Shortcut for `instance.props.getValue` that allows new behavior to be added by overriding it. * `getNamedProp`: Shortcut for `instance.namedProperties.get((propertyName, guid), default)` that allows new behavior to be added by overriding it. * Removed `Named._getStringStream` and `Named.sExists`. The named properties storage will *always* use regular streams and not string streams. From 0a8a20ce257e7a73b3cdf3db3041518373ea2445 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sat, 12 Aug 2023 22:50:43 -0700 Subject: [PATCH 02/68] Progress on 0.45.1 which extends --- CHANGELOG.md | 6 + extract_msg/enums.py | 7 + extract_msg/msg_classes/journal.py | 136 +++++++++++++++++++ extract_msg/msg_classes/msg.py | 18 ++- extract_msg/structures/__init__.py | 1 + extract_msg/structures/contact_link_entry.py | 23 ++++ extract_msg/structures/entry_id.py | 17 ++- 7 files changed, 206 insertions(+), 2 deletions(-) create mode 100644 extract_msg/msg_classes/journal.py create mode 100644 extract_msg/structures/contact_link_entry.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e54b4dea..4da150d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +**v0.45.1** +* Changed the base class of `EntryID` from no base class to `abc.ABC`. +* Added `position` property to `EntryID` to tell how many bytes were used to create the `EntryID`. +* Added additional properties to `MSGFile`: `contacts` and `contactLinkEntry`. +* Added support for Journal objects. + **v0.45.0** * BREAKING: Changed parsing of string multiple properties to remove the trailing null byte. This *will* cause the output of parsing them to differ. * Updated typing information for some functions and classes. diff --git a/extract_msg/enums.py b/extract_msg/enums.py index 2c237b01..7e8cbef7 100644 --- a/extract_msg/enums.py +++ b/extract_msg/enums.py @@ -34,6 +34,7 @@ 'Importance', 'InsecureFeatures', 'Intelligence', + 'LogFlags', 'MacintoshEncoding', 'MeetingObjectChange', 'MeetingRecipientType', @@ -1235,6 +1236,12 @@ class Intelligence(enum.IntEnum): +class LogFlags(enum.IntFlag): + NO_JOURNAL_ASSOCIATED_ATT = 0x00000000 + HAS_JOUNRAL_ASSOCIATED_ATT = 0x40000000 + + + class MacintoshEncoding(enum.IntEnum): """ The encoding to use for Macintosh-specific data attachments. diff --git a/extract_msg/msg_classes/journal.py b/extract_msg/msg_classes/journal.py new file mode 100644 index 00000000..219156f9 --- /dev/null +++ b/extract_msg/msg_classes/journal.py @@ -0,0 +1,136 @@ +__all__ = [ + 'Journal', +] + + +import datetime +import functools + +from typing import List, Optional + +from ..constants import HEADER_FORMAT_TYPE, ps +from ..enums import LogFlags +from .message_base import MessageBase + + +class Journal(MessageBase): + """ + Class for parsing Journal messages. + """ + + @functools.cached_property + def companies(self) -> Optional[List[str]]: + """ + The start time for the object. + """ + return self.getNamedProp('8539', ps.PSETID_COMMON) + + @functools.cached_property + def logDocumentPosted(self) -> bool: + """ + Indicates whether the document was sent by email of posted to a server + folder during journaling. + """ + return bool(self.getNamedProp('8711', ps.PSETID_LOG)) + + @functools.cached_property + def logDocumentPrinted(self) -> bool: + """ + Indicates whether the document was printed during journaling. + """ + return bool(self.getNamedProp('870E', ps.PSETID_LOG)) + + @functools.cached_property + def logDocumentRouted(self) -> bool: + """ + Indicates whether the document was sent to a routing recipient during + journaling. + """ + return bool(self.getNamedProp('8710', ps.PSETID_LOG)) + + @functools.cached_property + def logDocumentSaved(self) -> bool: + """ + Indicates whether the document was saved during journaling. + """ + return bool(self.getNamedProp('870F', ps.PSETID_LOG)) + + @functools.cached_property + def logDuration(self) -> Optional[int]: + """ + The duration, in minutes, of the activity. + """ + return self.getNamedProp('8707', ps.PSETID_LOG) + + @functools.cached_property + def logEnd(self) -> Optional[datetime.datetime]: + """ + The name of the activity that is being recorded. + """ + return self.getNamedProp('8708', ps.PSETID_LOG) + + @functools.cached_property + def logFlags(self) -> LogFlags: + """ + The name of the activity that is being recorded. + """ + return LogFlags(self.getNamedProp('870C', ps.PSETID_LOG, 0)) + + @functools.cached_property + def logStart(self) -> Optional[datetime.datetime]: + """ + The name of the activity that is being recorded. + """ + return self.getNamedProp('8706', ps.PSETID_LOG) + + @functools.cached_property + def logType(self) -> Optional[str]: + """ + The name of the activity that is being recorded. + """ + return self.getNamedProp('8700', ps.PSETID_LOG) + + @functools.cached_property + def logTypeDesc(self) -> Optional[str]: + """ + The description of the activity that is being recorded. + """ + return self.getNamedProp('8712', ps.PSETID_LOG) + + @property + def headerFormatProperties(self) -> HEADER_FORMAT_TYPE: + """ + Returns a dictionary of properties, in order, to be formatted into the + header. Keys are the names to use in the header while the values are one + of the following: + None: Signifies no data was found for the property and it should be + omitted from the header. + str: A string to be formatted into the header using the string encoding. + Tuple[Union[str, None], bool]: A string should be formatted into the + header. If the bool is True, then place an empty string if the value + is None, otherwise follow the same behavior as regular None. + + Additional note: If the value is an empty string, it will be dropped as + well by default. + + Additionally you can group members of a header together by placing them + in an embedded dictionary. Groups will be spaced out using a second + instance of the join string. If any member of a group is being printed, + it will be spaced apart from the next group/item. + + If you class should not do *any* header injection, return None from this + property. + """ + return { + '-main details-': { + 'From': self.sender, + 'Posted At': self.date, + 'Conversation': self.conversation, + }, + '-subject-': { + 'Subject': self.subject, + }, + '-importance-': { + 'Importance': self.importanceString, + }, + } diff --git a/extract_msg/msg_classes/msg.py b/extract_msg/msg_classes/msg.py index 2dcf5383..99a5f0bc 100644 --- a/extract_msg/msg_classes/msg.py +++ b/extract_msg/msg_classes/msg.py @@ -37,8 +37,8 @@ StandardViolationError ) from ..properties.named import Named, NamedProperties -from ..properties.prop import FixedLengthProp from ..properties.properties_store import PropertiesStore +from ..structures.contact_link_entry import ContactLinkEntry from ..utils import ( divide, hasLen, inputToMsgPath, makeWeakRef, msgPathToString, parseType, verifyPropertyId, verifyType, windowsUnicode @@ -856,6 +856,22 @@ def commonStart(self) -> Optional[datetime.datetime]: """ return self.getNamedProp('8516', constants.ps.PSETID_COMMON) + @functools.cached_property + def contactLinkEntry(self) -> Optional[ContactLinkEntry]: + """ + Returns a class that contains the list of Address Book EntryIDs linked + to this Message object. + """ + return self.getNamedAs('', constants.ps.PSETID_COMMON, ContactLinkEntry) + + @functools.cached_property + def contacts(self) -> Optional[List[str]]: + """ + Contains the display name property of each Address Book EntryID + referenced in the value of the contactLinkEntry property. + """ + return self.getNamedProp('853A', constants.ps.PSETID_COMMON) + @functools.cached_property def currentVersion(self) -> Optional[int]: """ diff --git a/extract_msg/structures/__init__.py b/extract_msg/structures/__init__.py index b831ac4f..92d2f600 100644 --- a/extract_msg/structures/__init__.py +++ b/extract_msg/structures/__init__.py @@ -5,6 +5,7 @@ __all__ = [ '_helpers', + 'contact_link_entry', 'business_card', 'entry_id', 'misc_id', diff --git a/extract_msg/structures/contact_link_entry.py b/extract_msg/structures/contact_link_entry.py new file mode 100644 index 00000000..7898a973 --- /dev/null +++ b/extract_msg/structures/contact_link_entry.py @@ -0,0 +1,23 @@ +__all__ = [ + 'ContactLinkEntry', +] + + +from typing import List + +from ._helpers import BytesReader +from .entry_id import AddressBookEntryID + + +class ContactLinkEntry: + entries : List[AddressBookEntryID] + + def __init__(self, data : bytes): + reader = BytesReader(data) + count = reader.readUnsignedInt() + reader.read(4) + remaining = reader.read() + self.entries = [] + for _ in range(count): + idStruct = AddressBookEntryID(remaining) + remaining = remaining[idStruct.position:] \ No newline at end of file diff --git a/extract_msg/structures/entry_id.py b/extract_msg/structures/entry_id.py index 777a7b27..0522e804 100644 --- a/extract_msg/structures/entry_id.py +++ b/extract_msg/structures/entry_id.py @@ -16,7 +16,9 @@ ] +import abc import logging + from typing import Union from ._helpers import BytesReader @@ -30,7 +32,7 @@ # First we define the main EntryID structure that is the base for the others. -class EntryID: +class EntryID(abc.ABC): """ Base class for all EntryID structures. Use :classmethod autoCreate: to automatically create the correct EntryID structure type from the specified @@ -115,6 +117,14 @@ def longTerm(self) -> bool: """ return self.__flags == b'\x00\x00\x00\x00' + @property + @abc.abstractmethod + def position(self) -> int: + """ + Used to tell the amount of bytes read in this EntryID. Useful for + EntryID data that has been chained together with no separator. + """ + @property def providerUID(self) -> bytes: """ @@ -147,6 +157,11 @@ def __init__(self, data : bytes): self.__type = AddressBookType(reader.readUnsignedInt()) self.__X500DN = reader.readByteString() + self.__position = reader.tell() + + @property + def position(self) -> int: + return self.__position @property def type(self) -> AddressBookType: From 05adade876e0187e2428464a8c8404bd72dc7a43 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sun, 13 Aug 2023 12:15:00 -0700 Subject: [PATCH 03/68] FInished EntryID stuff and added Journal --- CHANGELOG.md | 2 + extract_msg/enums.py | 7 ++ extract_msg/msg_classes/__init__.py | 2 + extract_msg/open_msg.py | 10 ++- extract_msg/structures/entry_id.py | 101 ++++++++++++++++++++++++++-- 5 files changed, 113 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4da150d8..53ac1d6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ * Added `position` property to `EntryID` to tell how many bytes were used to create the `EntryID`. * Added additional properties to `MSGFile`: `contacts` and `contactLinkEntry`. * Added support for Journal objects. +* Changed internal code of `PermanentEntryID` to correctly parse the data. Previously the distinguished name did not actually end at the null character, instead ending at the end of the bytes provided. If there was trailing data, it would be captured inadvertantly. +* Finished definition for `StoreObjectEntryID`. **v0.45.0** * BREAKING: Changed parsing of string multiple properties to remove the trailing null byte. This *will* cause the output of parsing them to differ. diff --git a/extract_msg/enums.py b/extract_msg/enums.py index 7e8cbef7..0807d01e 100644 --- a/extract_msg/enums.py +++ b/extract_msg/enums.py @@ -72,6 +72,7 @@ 'TaskState', 'TaskStatus', 'TZFlag', + 'WrappedType', ] @@ -1796,6 +1797,12 @@ class TZFlag(enum.IntFlag): +class WrappedType(enum.IntEnum): + MESSAGE_STORE = 0x6 + MAILBOX_STORE = 0xC + + + class _EnumDeprecator: """ Special class for handling deprecated enums in a way that shouldn't break diff --git a/extract_msg/msg_classes/__init__.py b/extract_msg/msg_classes/__init__.py index e22dd0b4..4e5d2c2d 100644 --- a/extract_msg/msg_classes/__init__.py +++ b/extract_msg/msg_classes/__init__.py @@ -8,6 +8,7 @@ 'Calendar', 'CalendarBase', 'Contact', + 'Journal', 'MeetingCancellation', 'MeetingException', 'MeetingForwardNotification', @@ -30,6 +31,7 @@ from .calendar_base import CalendarBase from .calendar import Calendar from .contact import Contact +from .journal import Journal from .meeting_cancellation import MeetingCancellation from .meeting_exception import MeetingException from .meeting_forward import MeetingForwardNotification diff --git a/extract_msg/open_msg.py b/extract_msg/open_msg.py index 439fa1a0..7c880f9d 100644 --- a/extract_msg/open_msg.py +++ b/extract_msg/open_msg.py @@ -69,9 +69,10 @@ def openMsg(path, **kwargs) -> MSGFile: :raises UnrecognizedMSGTypeError: if the type is not recognized. """ from .msg_classes import ( - AppointmentMeeting, Contact, MeetingCancellation, MeetingException, - MeetingForwardNotification, MeetingRequest, MeetingResponse, - Message, MSGFile, MessageSigned, Post, StickyNote, Task, TaskRequest + AppointmentMeeting, Contact, Journal, MeetingCancellation, + MeetingException, MeetingForwardNotification, MeetingRequest, + MeetingResponse, Message, MSGFile, MessageSigned, Post, StickyNote, + Task, TaskRequest ) # When the initial MSG file is opened, it should *always* delay attachments @@ -110,6 +111,9 @@ def openMsg(path, **kwargs) -> MSGFile: return MessageSigned(path, **kwargs) else: return Message(path, **kwargs) + elif classType.startswith('ipm.activity'): + msg.close() + return Journal(path, **kwargs) elif classType.startswith('ipm.appointment'): msg.close() return AppointmentMeeting(path, **kwargs) diff --git a/extract_msg/structures/entry_id.py b/extract_msg/structures/entry_id.py index 0522e804..4316a7b4 100644 --- a/extract_msg/structures/entry_id.py +++ b/extract_msg/structures/entry_id.py @@ -19,11 +19,15 @@ import abc import logging -from typing import Union +from typing import Optional, Union from ._helpers import BytesReader from .. import constants -from ..enums import AddressBookType, ContactAddressIndex, DisplayType, EntryIDType, MacintoshEncoding, MessageFormat, MessageType, OORBodyFormat +from ..enums import ( + AddressBookType, ContactAddressIndex, DisplayType, EntryIDType, + MacintoshEncoding, MessageFormat, MessageType, OORBodyFormat, + WrappedType + ) from ..utils import bitwiseAdjustedAnd, bytesToGuid @@ -157,7 +161,7 @@ def __init__(self, data : bytes): self.__type = AddressBookType(reader.readUnsignedInt()) self.__X500DN = reader.readByteString() - self.__position = reader.tell() + self.__position = reader.tell() + 20 @property def position(self) -> int: @@ -203,6 +207,7 @@ def __init__(self, data : bytes): self.__index = ContactAddressIndex(reader.readUnsignedInt()) self.__entryIdCount = reader.readUnsignedInt() self.__entryID = MessageEntryID(reader.read(self.__entryIdCount)) + self.__position = reader.tell() + 20 @property def entryID(self) -> MessageEntryID: @@ -225,6 +230,10 @@ def index(self) -> ContactAddressIndex: """ return self.__index + @property + def position(self) -> int: + return self.__position + class FolderEntryID(EntryID): @@ -265,6 +274,10 @@ def globalCounter(self) -> int: """ return self.__globalCounter + @property + def position(self) -> int: + return self.__SIZE__ + class MessageEntryID(EntryID): @@ -287,7 +300,7 @@ def __init__(self, data : bytes): self.__messageGlobalCounter = constants.st.ST_LE_UI64.unpack(reader.read(6) + b'\x00\x00') reader.assertNull(2, 'Pad bytes were not 0.') # Not sure why Microsoft decided to say "yes, let's do 2 6-byte integers - # followed by 2 pad bits each" instead of just 2 8-byte integers with a + # followed by 2 pad bytes each" instead of just 2 8-byte integers with a # maximum value, but here we are. @property @@ -328,6 +341,10 @@ def messageType(self) -> MessageType: """ return self.__messageType + @property + def position(self) -> int: + return self.__SIZE__ + class NNTPNewsgroupFolderEntryID(EntryID): @@ -342,6 +359,7 @@ def __init__(self, data : bytes): if self.__folderType != 0x000C: raise ValueError(f'Folder type was not 0x000C (got {self.__folderType})') self.__newsgroupName = reader.readAnsiString() + self.__position = reader.tell() + 20 @property def folderType(self) -> int: @@ -357,6 +375,10 @@ def newsgroupName(self) -> str: """ return self.__newsgroupName + @property + def position(self) -> int: + return self.__position + class OneOffRecipient(EntryID): @@ -406,6 +428,8 @@ def __init__(self, data : bytes): self.__addressType = reader.readByteString() self.__emailAddress = reader.readByteString() + self.__position = reader.tell() + 20 + @property def addressType(self) -> Union[str, bytes]: """ @@ -462,6 +486,10 @@ def messageFormat(self) -> MessageFormat: """ return self.__messageFormat + @property + def position(self) -> int: + return self.__position + class PermanentEntryID(EntryID): @@ -471,11 +499,13 @@ class PermanentEntryID(EntryID): def __init__(self, data : bytes): super().__init__(data) - unpacked = constants.st.STPEID.unpack(data[:28]) + reader = BytesReader(data) + unpacked = reader.readStruct(constants.st.STPEID) if unpacked[0] != 0: raise TypeError(f'Not a PermanentEntryID (expected 0, got {unpacked[0]}).') self.__displayTypeString = DisplayType(unpacked[2]) - self.__distinguishedName = data[28:-1].decode('ascii') # Cut off the null character at the end and decode the data as ascii + self.__distinguishedName = reader.readAsciiString() + self.__position = reader.tell() @property def displayTypeString(self) -> DisplayType: @@ -491,6 +521,10 @@ def distinguishedName(self) -> str: """ return self.__distinguishedName + @property + def position(self) -> int: + return self.__position + class PersonalDistributionListEntryID(EntryID): @@ -509,6 +543,7 @@ def __init__(self, data : bytes): raise ValueError(f'Index must be 255 (got {self.__version}).') self.__entryIdCount = reader.readUnsignedInt() self.__entryID = MessageEntryID(reader.read(self.__entryIdCount)) + self.__position = reader.tell() + 20 @property def entryID(self) -> MessageEntryID: @@ -524,6 +559,11 @@ def entryIDCount(self) -> int: """ return self.__entryIdCount + @property + def position(self) -> int: + return self.__position + + class StoreObjectEntryID(EntryID): """ @@ -548,6 +588,18 @@ def __init__(self, data : bytes): if self.__wrappedFlags != 0: raise ValueError(f'Wrapped flags was not set to 0 (got {self.__wrappedFlags}).') + self.__wrappedProviderUID = reader.read(16) + self.__wrappedType = WrappedType(reader.readUnsignedInt()) + # Don't know how this is encoded, just that it is "single-byte + # characters". + self.__serverShortname = reader.readByteString() + if self.__wrappedProviderUID == b'\x1B\x55\xFA\x20\xAA\x66\x11\xCD\x9B\xC8\x00\xAA\x00\x2F\xC4\x5A': + self.__mailboxDN = reader.readAsciiString() + else: + self.__mailboxDN = None + + self.__position = reader.tell() + 20 + @property def dllFileName(self) -> bytes: """ @@ -559,10 +611,42 @@ def dllFileName(self) -> bytes: def flag(self) -> int: return self.__flag + @property + def mailboxDN(self) -> Optional[str]: + """ + A string representing the X500 DN of the mailbox, as specified in + [MS-OXOAB]. THis field is present only for mailbox databases. + """ + return self.__mailboxDN + + @property + def position(self) -> int: + return self.__position + + @property + def serverShortname(self) -> bytes: + """ + A string of single-byte characters indicating the short name or NetBIOS + name of the server. + """ + return self.__serverShortname + @property def version(self) -> int: return self.__version + @property + def wrappedProviderUID(self) -> bytes: + return self.__wrappedProviderUID + + @property + def wrappedType(self) -> WrappedType: + """ + Determined by where the folder is located. + """ + return self.__wrappedType + + class WrappedEntryID(EntryID): """ @@ -584,6 +668,7 @@ def __init__(self, data : bytes): raise ValueError(f'Found wrapped entry id with invalid type (type bits were {bits}).') self.__embeddedIsOneOff = self.__type & 0x80 == 0 + self.__position = 21 + self.__embeddedEntryID.position @property def embeddedEntryID(self) -> EntryID: @@ -599,6 +684,10 @@ def embeddedIsOneOff(self) -> bool: """ return self.__embeddedIsOneOff + @property + def position(self) -> int: + return self.__position + @property def type(self) -> int: """ From b0c5ff9c5b5a5e2b052224751524d2a43573fd2f Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sun, 13 Aug 2023 15:19:51 -0700 Subject: [PATCH 04/68] Add named property id for contactlinkentry --- extract_msg/msg_classes/msg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extract_msg/msg_classes/msg.py b/extract_msg/msg_classes/msg.py index 99a5f0bc..e98e731e 100644 --- a/extract_msg/msg_classes/msg.py +++ b/extract_msg/msg_classes/msg.py @@ -862,7 +862,7 @@ def contactLinkEntry(self) -> Optional[ContactLinkEntry]: Returns a class that contains the list of Address Book EntryIDs linked to this Message object. """ - return self.getNamedAs('', constants.ps.PSETID_COMMON, ContactLinkEntry) + return self.getNamedAs('8585', constants.ps.PSETID_COMMON, ContactLinkEntry) @functools.cached_property def contacts(self) -> Optional[List[str]]: From c1ade4c895d5c89a59cab04584534193c7184838 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sun, 13 Aug 2023 15:30:05 -0700 Subject: [PATCH 05/68] Fix the companies property of Journal --- extract_msg/msg_classes/journal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extract_msg/msg_classes/journal.py b/extract_msg/msg_classes/journal.py index 219156f9..dff569f6 100644 --- a/extract_msg/msg_classes/journal.py +++ b/extract_msg/msg_classes/journal.py @@ -21,7 +21,8 @@ class Journal(MessageBase): @functools.cached_property def companies(self) -> Optional[List[str]]: """ - The start time for the object. + Contains a list of company names, each of which is accociated with a + contact this is precified in the contacts property. """ return self.getNamedProp('8539', ps.PSETID_COMMON) From 14e186f5f77228a969ba21f786a17e4697c916c6 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sun, 13 Aug 2023 16:00:10 -0700 Subject: [PATCH 06/68] Added some notes --- notes/Custom Attachment CLSIDs.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 notes/Custom Attachment CLSIDs.txt diff --git a/notes/Custom Attachment CLSIDs.txt b/notes/Custom Attachment CLSIDs.txt new file mode 100644 index 00000000..1073b406 --- /dev/null +++ b/notes/Custom Attachment CLSIDs.txt @@ -0,0 +1,2 @@ +00020D09-0000-0000-C000-000000000046: Seems to be a link to an outlook object. +00000316-0000-0000-C000-000000000046: Device Independent Bitmap. \ No newline at end of file From 3d82333d377816f262a500d06bcd845aeadb95c3 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sun, 13 Aug 2023 19:05:25 -0700 Subject: [PATCH 07/68] Change datetime formating, Cleanup, journal saving --- CHANGELOG.md | 8 +++- extract_msg/constants/__init__.py | 7 ++++ extract_msg/msg_classes/appointment.py | 4 +- extract_msg/msg_classes/contact.py | 4 +- extract_msg/msg_classes/journal.py | 42 +++++-------------- .../msg_classes/meeting_cancellation.py | 4 +- extract_msg/msg_classes/meeting_forward.py | 4 +- extract_msg/msg_classes/meeting_request.py | 4 +- extract_msg/msg_classes/message_base.py | 14 +++---- extract_msg/msg_classes/msg.py | 21 ++++++++++ extract_msg/msg_classes/post.py | 4 +- extract_msg/msg_classes/task.py | 2 +- extract_msg/properties/properties_store.py | 4 +- extract_msg/utils.py | 40 +++++++----------- 14 files changed, 80 insertions(+), 82 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53ac1d6d..c76e707b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,16 @@ -**v0.45.1** +**v0.46.0** * Changed the base class of `EntryID` from no base class to `abc.ABC`. * Added `position` property to `EntryID` to tell how many bytes were used to create the `EntryID`. * Added additional properties to `MSGFile`: `contacts` and `contactLinkEntry`. * Added support for Journal objects. * Changed internal code of `PermanentEntryID` to correctly parse the data. Previously the distinguished name did not actually end at the null character, instead ending at the end of the bytes provided. If there was trailing data, it would be captured inadvertantly. * Finished definition for `StoreObjectEntryID`. +* Added new kwargs for MSG files: `dateFormat` and `datetimeFormat`. These allow the user to easily override the strings being used for format dates and dates that include a time component, respectively. + * In unifying all the formats into 2 options, you may notice that some will look a bit different starting from this version, as there was an unfortunately large amount of variation. +* Removed `MessageBase.parsedDate`. +* Fixed issues with `MessageBase.date` and related things either being incorrectly documented or doing things that are not specified by the documentation. It was *supposed* to have been changed to use `datetime` objects, but it was still using strings. +* Removed unused function `extract_msg.utils.isEmptyString`. +* Removed unused function `extract_msg.utils.properHex`. **v0.45.0** * BREAKING: Changed parsing of string multiple properties to remove the trailing null byte. This *will* cause the output of parsing them to differ. diff --git a/extract_msg/constants/__init__.py b/extract_msg/constants/__init__.py index 6a908f87..bc695433 100644 --- a/extract_msg/constants/__init__.py +++ b/extract_msg/constants/__init__.py @@ -11,7 +11,9 @@ 'st', # Constants. + 'DATE_FORMAT', 'DEFAULT_CLSID', + 'DT_FORMAT', 'FIXED_LENGTH_PROPS', 'FIXED_LENGTH_PROPS_STRING', 'HEADER_FORMAT', @@ -48,6 +50,11 @@ from ..enums import SaveType +# Constants for formating datetime objects. +DATE_FORMAT = '%d %B, %Y' +DT_FORMAT = '%a, %d %b %Y %H:%M:%S %z' + + # Typing Constants. HEADER_FORMAT_VALUE_TYPE = Union[str, Tuple[Union[str, None], bool], None] # Basically a dict of HEADER_FORMAT_TYPE and dicts containing them. diff --git a/extract_msg/msg_classes/appointment.py b/extract_msg/msg_classes/appointment.py index a770928c..469b1f67 100644 --- a/extract_msg/msg_classes/appointment.py +++ b/extract_msg/msg_classes/appointment.py @@ -141,8 +141,8 @@ def headerFormatProperties(self) -> HEADER_FORMAT_TYPE: 'Location': self.location, }, '-date-': { - 'Start': self.startDate.__format__('%a, %d %b %Y %H:%M %z') if self.startDate else None, - 'End': self.endDate.__format__('%a, %d %b %Y %H:%M %z') if self.endDate else None, + 'Start': self.startDate.__format__(self.datetimeFormat) if self.startDate else None, + 'End': self.endDate.__format__(self.datetimeFormat) if self.endDate else None, }, '-recurrence-': { 'Recurrance': recur, diff --git a/extract_msg/msg_classes/contact.py b/extract_msg/msg_classes/contact.py index 36fa727a..ec8fbf67 100644 --- a/extract_msg/msg_classes/contact.py +++ b/extract_msg/msg_classes/contact.py @@ -709,8 +709,8 @@ def strListToStr(inp : Optional[Union[str, List[str]]]): 'Email3 Display As': self.email3DisplayName, }, '-other-': { - 'Birthday': self.birthday.__format__('%B %d, %Y') if self.birthdayLocal else None, - 'Anniversary': self.weddingAnniversary.__format__('%B %d, %Y') if self.weddingAnniversaryLocal else None, + 'Birthday': self.birthday.__format__(self.dateFormat) if self.birthdayLocal else None, + 'Anniversary': self.weddingAnniversary.__format__(self.dateFormat) if self.weddingAnniversaryLocal else None, 'Spouse/Partner': self.spouseName, 'Profession': self.profession, 'Children': strListToStr(self.childrensNames), diff --git a/extract_msg/msg_classes/journal.py b/extract_msg/msg_classes/journal.py index dff569f6..e01bcf79 100644 --- a/extract_msg/msg_classes/journal.py +++ b/extract_msg/msg_classes/journal.py @@ -11,6 +11,7 @@ from ..constants import HEADER_FORMAT_TYPE, ps from ..enums import LogFlags from .message_base import MessageBase +from ..utils import minutesToDurationStr class Journal(MessageBase): @@ -57,11 +58,11 @@ def logDocumentSaved(self) -> bool: return bool(self.getNamedProp('870F', ps.PSETID_LOG)) @functools.cached_property - def logDuration(self) -> Optional[int]: + def logDuration(self) -> int: """ The duration, in minutes, of the activity. """ - return self.getNamedProp('8707', ps.PSETID_LOG) + return self.getNamedProp('8707', ps.PSETID_LOG, 0) @functools.cached_property def logEnd(self) -> Optional[datetime.datetime]: @@ -100,38 +101,15 @@ def logTypeDesc(self) -> Optional[str]: @property def headerFormatProperties(self) -> HEADER_FORMAT_TYPE: - """ - Returns a dictionary of properties, in order, to be formatted into the - header. Keys are the names to use in the header while the values are one - of the following: - None: Signifies no data was found for the property and it should be - omitted from the header. - str: A string to be formatted into the header using the string encoding. - Tuple[Union[str, None], bool]: A string should be formatted into the - header. If the bool is True, then place an empty string if the value - is None, otherwise follow the same behavior as regular None. - - Additional note: If the value is an empty string, it will be dropped as - well by default. - - Additionally you can group members of a header together by placing them - in an embedded dictionary. Groups will be spaced out using a second - instance of the join string. If any member of a group is being printed, - it will be spaced apart from the next group/item. - - If you class should not do *any* header injection, return None from this - property. - """ return { '-main details-': { - 'From': self.sender, - 'Posted At': self.date, - 'Conversation': self.conversation, - }, - '-subject-': { 'Subject': self.subject, + 'Entry Type': self.logTypeDesc, + 'Company': self.companies[0] if self.companies else None, }, - '-importance-': { - 'Importance': self.importanceString, - }, + '-time-': { + 'Start': self.logStart.__format__(self.datetimeFormat), + 'End': self.logEnd.__format__(self.datetimeFormat), + 'Duration': minutesToDurationStr(self.duration), + } } diff --git a/extract_msg/msg_classes/meeting_cancellation.py b/extract_msg/msg_classes/meeting_cancellation.py index 2d184659..d499cc4b 100644 --- a/extract_msg/msg_classes/meeting_cancellation.py +++ b/extract_msg/msg_classes/meeting_cancellation.py @@ -69,8 +69,8 @@ def headerFormatProperties(self) -> constants.HEADER_FORMAT_TYPE: 'Location': self.location, }, '-date-': { - 'Start': self.startDate.__format__('%a, %d %b %Y %H:%M %z') if self.startDate else None, - 'End': self.endDate.__format__('%a, %d %b %Y %H:%M %z') if self.endDate else None, + 'Start': self.startDate.__format__(self.datetimeFormat) if self.startDate else None, + 'End': self.endDate.__format__(self.datetimeFormat) if self.endDate else None, 'Show Time As': 'Free', }, '-recurrence-': { diff --git a/extract_msg/msg_classes/meeting_forward.py b/extract_msg/msg_classes/meeting_forward.py index 3ab66caf..81080513 100644 --- a/extract_msg/msg_classes/meeting_forward.py +++ b/extract_msg/msg_classes/meeting_forward.py @@ -72,8 +72,8 @@ def headerFormatProperties(self) -> constants.HEADER_FORMAT_TYPE: 'Location': self.location, }, '-date-': { - 'Start': self.startDate.__format__('%a, %d %b %Y %H:%M %z') if self.startDate else None, - 'End': self.endDate.__format__('%a, %d %b %Y %H:%M %z') if self.endDate else None, + 'Start': self.startDate.__format__(self.datetimeFormat) if self.startDate else None, + 'End': self.endDate.__format__(self.datetimeFormat) if self.endDate else None, }, '-recurrence-': { 'Recurrance': recur, diff --git a/extract_msg/msg_classes/meeting_request.py b/extract_msg/msg_classes/meeting_request.py index 3c56a868..c0a7f71b 100644 --- a/extract_msg/msg_classes/meeting_request.py +++ b/extract_msg/msg_classes/meeting_request.py @@ -110,8 +110,8 @@ def headerFormatProperties(self) -> HEADER_FORMAT_TYPE: 'Location': self.location, }, '-date-': { - 'Start': self.startDate.__format__('%a, %d %b %Y %H:%M %z') if self.startDate else None, - 'End': self.endDate.__format__('%a, %d %b %Y %H:%M %z') if self.endDate else None, + 'Start': self.startDate.__format__(self.datetimeFormat) if self.startDate else None, + 'End': self.endDate.__format__(self.datetimeFormat) if self.endDate else None, 'Show Time As': showTime, }, '-recurrence-': { diff --git a/extract_msg/msg_classes/message_base.py b/extract_msg/msg_classes/message_base.py index 1db80fca..093d469f 100644 --- a/extract_msg/msg_classes/message_base.py +++ b/extract_msg/msg_classes/message_base.py @@ -251,7 +251,7 @@ def dump(self) -> None: """ print('Message') print('Subject:', self.subject) - print('Date:', self.date) + print('Date:', self.date.__format__(self.datetimeFormat)) print('Body:') print(self.body) @@ -324,7 +324,7 @@ def getJson(self) -> str: 'cc': inputToString(self.cc, self.stringEncoding), 'bcc': inputToString(self.bcc, self.stringEncoding), 'subject': inputToString(self.subject, self.stringEncoding), - 'date': inputToString(self.date, self.stringEncoding), + 'date': inputToString(self.date.__format__(self.datetimeFormat), self.stringEncoding), 'body': decode_utf7(self.body), }) @@ -1048,11 +1048,11 @@ def header(self) -> email.message.Message: if headerText: header = HeaderParser(policy = policy.default).parsestr(headerText) del header['Date'] - header['Date'] = self.date + header['Date'] = self.date.__format__(self.datetimeFormat) else: logger.info('Header is empty or was not found. Header will be generated from other streams.') header = HeaderParser(policy = policy.default).parsestr('') - header.add_header('Date', self.date) + header.add_header('Date', self.date.__format__(self.datetimeFormat)) header.add_header('From', self.sender) header.add_header('To', self.to) header.add_header('Cc', self.cc) @@ -1110,7 +1110,7 @@ def headerFormatProperties(self) -> constants.HEADER_FORMAT_TYPE: return { '-basic info-': { 'From': self.sender, - 'Sent': self.date, + 'Sent': self.date.__format__(self.datetimeFormat), 'To': self.to, 'Cc': self.cc, 'Bcc': self.bcc, @@ -1232,10 +1232,6 @@ def messageId(self) -> Optional[str]: logger.info('Header found, but "Message-Id" is not included. Will be generated from other streams.') return self.getStringStream('__substg1.0_1035') - @functools.cached_property - def parsedDate(self): - return email.utils.parsedate(self.date) - @functools.cached_property def receivedTime(self) -> Optional[datetime.datetime]: """ diff --git a/extract_msg/msg_classes/msg.py b/extract_msg/msg_classes/msg.py index e98e731e..37f9e989 100644 --- a/extract_msg/msg_classes/msg.py +++ b/extract_msg/msg_classes/msg.py @@ -84,6 +84,9 @@ def __init__(self, path, **kwargs): :param insecureFeatures: Optional, an enum value that specifies if certain insecure features should be enabled. These features should only be used on data that you trust. Uses the InsecureFeatures enum. + :param dateFormat: Optional, the format string to use for dates. + :param datetimeFormat: Optional, the format string to use for dates + that include a time component. :raises InvalidFileFormatError: If the file is not an OleFile or could not be parsed as an MSG file. @@ -116,6 +119,8 @@ def __init__(self, path, **kwargs): self.__attachmentsDelayed = kwargs.get('delayAttachments', False) self.__attachmentsReady = False self.__errorBehavior = ErrorBehavior(kwargs.get('errorBehavior', ErrorBehavior.THROW)) + self.__dateFormat = kwargs.get('dateFormat', constants.DATE_FORMAT) + self.__dtFormat = kwargs.get('datetimeFormat', constants.DT_FORMAT) if overrideEncoding is not None: codecs.lookup(overrideEncoding) @@ -887,6 +892,22 @@ def currentVersionName(self) -> Optional[str]: """ return self.getNamedProp('8554', constants.ps.PSETID_COMMON) + @property + def dateFormat(self) -> str: + """ + The format string to use when converting dates to strings. This is used + for dates with no time component. + """ + return self.__dateFormat + + @property + def datetimeFormat(self) -> str: + """ + The format string to use when converting datetimes to strings. This is + used for dates that have time components. + """ + return self.__dtFormat + @property def errorBehavior(self) -> ErrorBehavior: """ diff --git a/extract_msg/msg_classes/post.py b/extract_msg/msg_classes/post.py index aaf673f8..875fb717 100644 --- a/extract_msg/msg_classes/post.py +++ b/extract_msg/msg_classes/post.py @@ -27,7 +27,7 @@ def getJson(self) -> str: return json.dumps({ 'from': inputToString(self.sender, self.stringEncoding), 'subject': inputToString(self.subject, self.stringEncoding), - 'date': inputToString(self.date, self.stringEncoding), + 'date': inputToString(self.date.__format__(self.datetimeFormat), self.stringEncoding), 'conversation': inputToString(self.conversation, self.stringEncoding), 'body': decode_utf7(self.body), }) @@ -66,7 +66,7 @@ def headerFormatProperties(self) -> constants.HEADER_FORMAT_TYPE: return { '-main details-': { 'From': self.sender, - 'Posted At': self.date, + 'Posted At': self.date.__format__(self.datetimeFormat), 'Conversation': self.conversation, }, '-subject-': { diff --git a/extract_msg/msg_classes/task.py b/extract_msg/msg_classes/task.py index 23b1c2cd..a2c9ea34 100644 --- a/extract_msg/msg_classes/task.py +++ b/extract_msg/msg_classes/task.py @@ -68,7 +68,7 @@ def headerFormatProperties(self) -> constants.HEADER_FORMAT_TYPE: '-status-': { 'Status': status, 'Percent Complete': f'{self.percentComplete*100:.0f}%', - 'Date Completed': self.taskDateCompleted.__format__('%w, %B %d, %Y') if self.taskDateCompleted else None, + 'Date Completed': self.taskDateCompleted.__format__(self.dateFormat) if self.taskDateCompleted else None, }, '-work-': { 'Total Work': f'{self.taskEstimatedEffort or 0} minutes', diff --git a/extract_msg/properties/properties_store.py b/extract_msg/properties/properties_store.py index 5884c897..1372a36d 100644 --- a/extract_msg/properties/properties_store.py +++ b/extract_msg/properties/properties_store.py @@ -239,10 +239,10 @@ def date(self) -> Optional[datetime.datetime]: self.__date = None if '00390040' in self: dateValue = self.get('00390040').value - # A date can by bytes if it fails to initialize, so we check it + # A date can be bytes if it fails to initialize, so we check it # first. if isinstance(dateValue, datetime.datetime): - self.__date = dateValue.__format__('%a, %d %b %Y %H:%M:%S %z') + self.__date = dateValue return self.__date @property diff --git a/extract_msg/utils.py b/extract_msg/utils.py index 857f94dd..6b949fe3 100644 --- a/extract_msg/utils.py +++ b/extract_msg/utils.py @@ -29,7 +29,6 @@ 'inputToMsgPath', 'inputToString', 'isEncapsulatedRtf', - 'isEmptyString', 'makeWeakRef', 'msgPathToString', 'parseType', @@ -595,13 +594,6 @@ def inputToString(bytesInputVar, encoding) -> str: raise ConversionError('Cannot convert to str type.') -def isEmptyString(inp : str) -> bool: - """ - Returns true if the input is None or is an Empty string. - """ - return (inp == '' or inp is None) - - def isEncapsulatedRtf(inp : bytes) -> bool: """ Currently the detection is made to be *extremly* basic, but this will work @@ -621,6 +613,21 @@ def makeWeakRef(obj : Optional[_T]) -> Optional[weakref.ReferenceType[_T]]: except TypeError: return None +def minutesToDurationStr(minutes : int) -> str: + """ + Converts the number of minutes into a duration string. + """ + if minutes == 0: + return '0 hours' + elif minutes == 1: + return '1 minute' + elif minutes < 60: + return f'{minutes} minutes' + elif minutes % 60 == 0: + return f'{minutes // 60} hours' + else: + return f'{minutes // 60} hours {minutes % 60} minutes' + def msgPathToString(inp : Union[str, Iterable[str]]) -> str: """ @@ -776,23 +783,6 @@ def prepareFilename(filename) -> str: return ''.join(i for i in filename if i not in r'\/:*?"<>|' + '\x00').strip() -def properHex(inp, length : int = 0) -> str: - """ - Takes in various input types and converts them into a hex string whose - length will always be even. - """ - a = '' - if isinstance(inp, str): - a = ''.join([hex(ord(inp[x]))[2:].rjust(2, '0') for x in range(len(inp))]) - elif isinstance(inp, bytes): - a = inp.hex() - elif isinstance(inp, int): - a = hex(inp)[2:] - if len(a) % 2 != 0: - a = '0' + a - return a.rjust(length, '0').upper() - - def roundUp(inp : int, mult : int) -> int: """ Rounds :param inp: up to the nearest multiple of :param mult:. From 029bab0be05e1743cdf5ee6c18d32695ac7811ac Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 15 Aug 2023 16:24:49 -0700 Subject: [PATCH 08/68] Small formatting adjustments --- extract_msg/msg_classes/appointment.py | 22 ------------------- extract_msg/msg_classes/contact.py | 22 ------------------- extract_msg/msg_classes/journal.py | 2 +- .../msg_classes/meeting_cancellation.py | 22 ------------------- extract_msg/msg_classes/meeting_forward.py | 22 ------------------- extract_msg/msg_classes/meeting_request.py | 22 ------------------- extract_msg/msg_classes/post.py | 22 ------------------- extract_msg/msg_classes/task.py | 22 ------------------- extract_msg/msg_classes/task_request.py | 22 ------------------- 9 files changed, 1 insertion(+), 177 deletions(-) diff --git a/extract_msg/msg_classes/appointment.py b/extract_msg/msg_classes/appointment.py index 469b1f67..65a81470 100644 --- a/extract_msg/msg_classes/appointment.py +++ b/extract_msg/msg_classes/appointment.py @@ -90,28 +90,6 @@ def fInvited(self) -> bool: @property def headerFormatProperties(self) -> HEADER_FORMAT_TYPE: - """ - Returns a dictionary of properties, in order, to be formatted into the - header. Keys are the names to use in the header while the values are one - of the following: - None: Signifies no data was found for the property and it should be - omitted from the header. - str: A string to be formatted into the header using the string encoding. - Tuple[Union[str, None], bool]: A string should be formatted into the - header. If the bool is True, then place an empty string if the value - is None, otherwise follow the same behavior as regular None. - - Additional note: If the value is an empty string, it will be dropped as - well by default. - - Additionally you can group members of a header together by placing them - in an embedded dictionary. Groups will be spaced out using a second - instance of the join string. If any member of a group is being printed, - it will be spaced apart from the next group/item. - - If you class should not do *any* header injection, return None from this - property. - """ meetingStatusString = { ResponseStatus.NONE: None, ResponseStatus.ORGANIZED: 'Meeting organizer', diff --git a/extract_msg/msg_classes/contact.py b/extract_msg/msg_classes/contact.py index ec8fbf67..89a48664 100644 --- a/extract_msg/msg_classes/contact.py +++ b/extract_msg/msg_classes/contact.py @@ -630,28 +630,6 @@ def hasPicture(self) -> bool: @property def headerFormatProperties(self) -> HEADER_FORMAT_TYPE: - """ - Returns a dictionary of properties, in order, to be formatted into the - header. Keys are the names to use in the header while the values are one - of the following: - None: Signifies no data was found for the property and it should be - omitted from the header. - str: A string to be formatted into the header using the string encoding. - Tuple[Union[str, None], bool]: A string should be formatted into the - header. If the bool is True, then place an empty string if the value - is None, otherwise follow the same behavior as regular None. - - Additional note: If the value is an empty string, it will be dropped as - well by default. - - Additionally you can group members of a header together by placing them - in an embedded dictionary. Groups will be spaced out using a second - instance of the join string. If any member of a group is being printed, - it will be spaced apart from the next group/item. - - If you class should not do *any* header injection, return None from this - property. - """ def strListToStr(inp : Optional[Union[str, List[str]]]): """ Small internal function for things that may return a string or list. diff --git a/extract_msg/msg_classes/journal.py b/extract_msg/msg_classes/journal.py index e01bcf79..475d01c9 100644 --- a/extract_msg/msg_classes/journal.py +++ b/extract_msg/msg_classes/journal.py @@ -111,5 +111,5 @@ def headerFormatProperties(self) -> HEADER_FORMAT_TYPE: 'Start': self.logStart.__format__(self.datetimeFormat), 'End': self.logEnd.__format__(self.datetimeFormat), 'Duration': minutesToDurationStr(self.duration), - } + }, } diff --git a/extract_msg/msg_classes/meeting_cancellation.py b/extract_msg/msg_classes/meeting_cancellation.py index d499cc4b..6929c427 100644 --- a/extract_msg/msg_classes/meeting_cancellation.py +++ b/extract_msg/msg_classes/meeting_cancellation.py @@ -17,28 +17,6 @@ class MeetingCancellation(MeetingRelated): @property def headerFormatProperties(self) -> constants.HEADER_FORMAT_TYPE: - """ - Returns a dictionary of properties, in order, to be formatted into the - header. Keys are the names to use in the header while the values are one - of the following: - None: Signifies no data was found for the property and it should be - omitted from the header. - str: A string to be formatted into the header using the string encoding. - Tuple[Union[str, None], bool]: A string should be formatted into the - header. If the bool is True, then place an empty string if the value - is None, otherwise follow the same behavior as regular None. - - Additional note: If the value is an empty string, it will be dropped as - well by default. - - Additionally you can group members of a header together by placing them - in an embedded dictionary. Groups will be spaced out using a second - instance of the join string. If any member of a group is being printed, - it will be spaced apart from the next group/item. - - If you class should not do *any* header injection, return None from this - property. - """ meetingStatusString = { ResponseStatus.NONE: None, ResponseStatus.ORGANIZED: 'Meeting organizer', diff --git a/extract_msg/msg_classes/meeting_forward.py b/extract_msg/msg_classes/meeting_forward.py index 81080513..bfe17eb6 100644 --- a/extract_msg/msg_classes/meeting_forward.py +++ b/extract_msg/msg_classes/meeting_forward.py @@ -30,28 +30,6 @@ def forwardNotificationRecipients(self) -> Optional[bytes]: @property def headerFormatProperties(self) -> constants.HEADER_FORMAT_TYPE: - """ - Returns a dictionary of properties, in order, to be formatted into the - header. Keys are the names to use in the header while the values are one - of the following: - None: Signifies no data was found for the property and it should be - omitted from the header. - str: A string to be formatted into the header using the string encoding. - Tuple[Union[str, None], bool]: A string should be formatted into the - header. If the bool is True, then place an empty string if the value - is None, otherwise follow the same behavior as regular None. - - Additional note: If the value is an empty string, it will be dropped as - well by default. - - Additionally you can group members of a header together by placing them - in an embedded dictionary. Groups will be spaced out using a second - instance of the join string. If any member of a group is being printed, - it will be spaced apart from the next group/item. - - If you class should not do *any* header injection, return None from this - property. - """ # Get the recurrence string. recur = '(none)' if self.appointmentRecur: diff --git a/extract_msg/msg_classes/meeting_request.py b/extract_msg/msg_classes/meeting_request.py index c0a7f71b..cea88407 100644 --- a/extract_msg/msg_classes/meeting_request.py +++ b/extract_msg/msg_classes/meeting_request.py @@ -57,28 +57,6 @@ def forwardInstance(self) -> bool: @property def headerFormatProperties(self) -> HEADER_FORMAT_TYPE: - """ - Returns a dictionary of properties, in order, to be formatted into the - header. Keys are the names to use in the header while the values are one - of the following: - None: Signifies no data was found for the property and it should be - omitted from the header. - str: A string to be formatted into the header using the string encoding. - Tuple[Union[str, None], bool]: A string should be formatted into the - header. If the bool is True, then place an empty string if the value - is None, otherwise follow the same behavior as regular None. - - Additional note: If the value is an empty string, it will be dropped as - well by default. - - Additionally you can group members of a header together by placing them - in an embedded dictionary. Groups will be spaced out using a second - instance of the join string. If any member of a group is being printed, - it will be spaced apart from the next group/item. - - If you class should not do *any* header injection, return None from this - property. - """ meetingStatusString = { ResponseStatus.NONE: None, ResponseStatus.ORGANIZED: 'Meeting organizer', diff --git a/extract_msg/msg_classes/post.py b/extract_msg/msg_classes/post.py index 875fb717..c2faab5b 100644 --- a/extract_msg/msg_classes/post.py +++ b/extract_msg/msg_classes/post.py @@ -41,28 +41,6 @@ def conversation(self) -> Optional[str]: @property def headerFormatProperties(self) -> constants.HEADER_FORMAT_TYPE: - """ - Returns a dictionary of properties, in order, to be formatted into the - header. Keys are the names to use in the header while the values are one - of the following: - None: Signifies no data was found for the property and it should be - omitted from the header. - str: A string to be formatted into the header using the string encoding. - Tuple[Union[str, None], bool]: A string should be formatted into the - header. If the bool is True, then place an empty string if the value - is None, otherwise follow the same behavior as regular None. - - Additional note: If the value is an empty string, it will be dropped as - well by default. - - Additionally you can group members of a header together by placing them - in an embedded dictionary. Groups will be spaced out using a second - instance of the join string. If any member of a group is being printed, - it will be spaced apart from the next group/item. - - If you class should not do *any* header injection, return None from this - property. - """ return { '-main details-': { 'From': self.sender, diff --git a/extract_msg/msg_classes/task.py b/extract_msg/msg_classes/task.py index a2c9ea34..43ee3c88 100644 --- a/extract_msg/msg_classes/task.py +++ b/extract_msg/msg_classes/task.py @@ -30,28 +30,6 @@ class Task(MessageBase): @property def headerFormatProperties(self) -> constants.HEADER_FORMAT_TYPE: - """ - Returns a dictionary of properties, in order, to be formatted into the - header. Keys are the names to use in the header while the values are one - of the following: - None: Signifies no data was found for the property and it should be - omitted from the header. - str: A string to be formatted into the header using the string encoding. - Tuple[Union[str, None], bool]: A string should be formatted into the - header. If the bool is True, then place an empty string if the value - is None, otherwise follow the same behavior as regular None. - - Additional note: If the value is an empty string, it will be dropped as - well by default. - - Additionally you can group members of a header together by placing them - in an embedded dictionary. Groups will be spaced out using a second - instance of the join string. If any member of a group is being printed, - it will be spaced apart from the next group/item. - - If you class should not do *any* header injection, return None from this - property. - """ status = { TaskStatus.NOT_STARTED: 'Not Started', TaskStatus.IN_PROGRESS: 'In Progress', diff --git a/extract_msg/msg_classes/task_request.py b/extract_msg/msg_classes/task_request.py index 34a83d7a..41051590 100644 --- a/extract_msg/msg_classes/task_request.py +++ b/extract_msg/msg_classes/task_request.py @@ -27,28 +27,6 @@ class TaskRequest(MessageBase): @property def headerFormatProperties(self) -> constants.HEADER_FORMAT_TYPE: - """ - Returns a dictionary of properties, in order, to be formatted into the - header. Keys are the names to use in the header while the values are one - of the following: - None: Signifies no data was found for the property and it should be - omitted from the header. - str: A string to be formatted into the header using the string encoding. - Tuple[Union[str, None], bool]: A string should be formatted into the - header. If the bool is True, then place an empty string if the value - is None, otherwise follow the same behavior as regular None. - - Additional note: If the value is an empty string, it will be dropped as - well by default. - - Additionally you can group members of a header together by placing them - in an embedded dictionary. Groups will be spaced out using a second - instance of the join string. If any member of a group is being printed, - it will be spaced apart from the next group/item. - - If you class should not do *any* header injection, return None from this - property. - """ # So this is rather weird. Looks like TaskRequest does not rely on # headers at all, simply using the body itself for all the data to # print. So I guess we just return None and handle that. From 8925d294231fc92f30e4378b58b0c2fa97618cce Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 16 Aug 2023 18:48:33 -0700 Subject: [PATCH 09/68] Word on stuff for journal special attachments --- CHANGELOG.md | 5 ++ .../custom_att_handler/__init__.py | 2 + .../custom_att_handler/custom_handler.py | 30 ++++++- .../custom_att_handler/jrnl_assoc_att.py | 90 +++++++++++++++++++ extract_msg/structures/entry_id.py | 10 ++- 5 files changed, 132 insertions(+), 5 deletions(-) create mode 100644 extract_msg/attachments/custom_att_handler/jrnl_assoc_att.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c76e707b..d095899c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,11 @@ * Fixed issues with `MessageBase.date` and related things either being incorrectly documented or doing things that are not specified by the documentation. It was *supposed* to have been changed to use `datetime` objects, but it was still using strings. * Removed unused function `extract_msg.utils.isEmptyString`. * Removed unused function `extract_msg.utils.properHex`. +* Added a `getStream` helper function to `CustomAttachmentHandler`. +* Added a `getStreamAs` helper function to `CustomAttachmentHandler`. +* Added new custom attachment handler for journal-associated attachments. +* Changed `EntryID.autoCreate` to return `None` if given `None` or empty bytes. +* Changed `EntryID.autoCreate` to raise a `FeatureNotImplemented` exception if no valid entry ID class is found. **v0.45.0** * BREAKING: Changed parsing of string multiple properties to remove the trailing null byte. This *will* cause the output of parsing them to differ. diff --git a/extract_msg/attachments/custom_att_handler/__init__.py b/extract_msg/attachments/custom_att_handler/__init__.py index b3318f4a..3e4feff3 100644 --- a/extract_msg/attachments/custom_att_handler/__init__.py +++ b/extract_msg/attachments/custom_att_handler/__init__.py @@ -21,6 +21,7 @@ __all__ = [ # Classes. 'CustomAttachmentHandler', + 'JournalAssociatedAttachment', 'OutlookImageDIB', # Functions. @@ -56,6 +57,7 @@ def registerHandler(handler : Type[CustomAttachmentHandler]) -> None: # Import built-in handler modules. They will all automatically register their # respecive handler(s). from .outlook_image_dib import OutlookImageDIB +from .jrnl_assoc_att import JournalAssociatedAttachment if TYPE_CHECKING: diff --git a/extract_msg/attachments/custom_att_handler/custom_handler.py b/extract_msg/attachments/custom_att_handler/custom_handler.py index 69419f85..c2d7c1bc 100644 --- a/extract_msg/attachments/custom_att_handler/custom_handler.py +++ b/extract_msg/attachments/custom_att_handler/custom_handler.py @@ -8,12 +8,16 @@ import abc -from typing import Optional, TYPE_CHECKING +from typing import Callable, Optional, TYPE_CHECKING, TypeVar + +from ...utils import msgPathToString if TYPE_CHECKING: from ..attachment_base import AttachmentBase +_T = TypeVar('_T') + class CustomAttachmentHandler(abc.ABC): """ @@ -25,6 +29,30 @@ def __init__(self, attachment : AttachmentBase): super().__init__() self.__att = attachment + def getStream(self, path) -> Optional[bytes]: + """ + Gets a stream from the custom data directory. + """ + return self.attachment.getStream('__substg1.0_3701000D/' + msgPathToString(path)) + + def getStreamAs(self, streamID, overrideClass : Callable[..., _T]) -> Optional[_T]: + """ + Returns the specified stream, modifying it to the specified class if it + is found. + + :param overrideClass: Class/function to use to morph the data that was + read. The data will be the first argument to the class's __init__ + function or the function itself, if that is what is provided. If + the value is None, this function is not called. If you want it to + be called regardless, you should handle the data directly. + """ + value = self.getStream(streamID) + + if value is not None: + value = overrideClass(value) + + return value + @classmethod @abc.abstractmethod def isCorrectHandler(cls, attachment : AttachmentBase) -> bool: diff --git a/extract_msg/attachments/custom_att_handler/jrnl_assoc_att.py b/extract_msg/attachments/custom_att_handler/jrnl_assoc_att.py new file mode 100644 index 00000000..c2fc4901 --- /dev/null +++ b/extract_msg/attachments/custom_att_handler/jrnl_assoc_att.py @@ -0,0 +1,90 @@ +from __future__ import annotations + + +__all__ = [ + 'JournalAssociatedAttachment', +] + + +from functools import cached_property +from typing import Optional, TYPE_CHECKING + +from . import registerHandler +from .custom_handler import CustomAttachmentHandler +from ...structures.entry_id import EntryID + + +if TYPE_CHECKING: + from ..attachment_base import AttachmentBase + + +class JournalAssociatedAttachment(CustomAttachmentHandler): + def __init__(self, attachment : AttachmentBase): + super().__init__(attachment) + + @classmethod + def isCorrectHandler(cls, attachment : AttachmentBase) -> bool: + # This only applies to journal objects. + if not attachment.msg.classType.lower().startswith('ipm.activity'): + return False + if attachment.clsid != '00020D09-0000-0000-C000-000000000046': + return False + + return True + + @cached_property + def mailMsgAttFld(self) -> Optional[EntryID]: + """ + The EntryID of the folder of the linked Message object. + """ + return EntryID.autoCreate(self.getStream('MailMsgAttFld')) + + @cached_property + def mailMsgAttMdb(self) -> Optional[EntryID]: + """ + The EntryID of the store of the linked Message object. + """ + return EntryID.autoCreate(self.getStream('MailMsgAttMdb')) + + @cached_property + def mailMsgAttMsg(self) -> Optional[EntryID]: + """ + The EntryID linked Message object; required only if the + mailMsgAttSrchKey property is None. + """ + return EntryID.autoCreate(self.getStream('MailMsgAttMsg')) + + @cached_property + def mailMsgAttSrchFld(self) -> Optional[EntryID]: + """ + The object EntryID of the Sent Items special folder of the linked + Message object. + """ + return EntryID.autoCreate(self.getStream('MailMsgAttSrchFld')) + + @cached_property + def mailMsgAttSrchKey(self) -> Optional[bytes]: + """ + The search key for the linked message object; required only if + mailMsgAttMsg is None. + """ + return self.getStream('MailMsgAttSrchKey') + + @cached_property + def metafileBytes(self) -> Optional[bytes]: + """ + The metafile that contains the icon to be used when rendering the + attachment. + + From my understanding, this MUST be set, but we are treating it as + SHOULD be set. + """ + # The documentation specifies clearly that the filename is "IOlePres000" + # HOWEVER my tests revealed that the "I" is actually a "\x02" character. + # This is quite confusing but whatever. We'll just look for both of + # them. + return self.getStream('IOlePres000') or self.getStream('\x02OlePres000') + + + +registerHandler(JournalAssociatedAttachment) \ No newline at end of file diff --git a/extract_msg/structures/entry_id.py b/extract_msg/structures/entry_id.py index 4316a7b4..f8c383a5 100644 --- a/extract_msg/structures/entry_id.py +++ b/extract_msg/structures/entry_id.py @@ -28,6 +28,7 @@ MacintoshEncoding, MessageFormat, MessageType, OORBodyFormat, WrappedType ) +from ..exceptions import FeatureNotImplemented from ..utils import bitwiseAdjustedAnd, bytesToGuid @@ -44,12 +45,15 @@ class EntryID(abc.ABC): """ @classmethod - def autoCreate(cls, data) -> EntryID: + def autoCreate(cls, data) -> Optional[EntryID]: """ Automatically determines the type of EntryID and returns an instance of the correct subclass. If the subclass cannot be determined, will return a plain EntryID instance. """ + if not data: + return None + if len(data) < 20: raise ValueError('Cannot create an EntryID with less than 20 bytes.') providerUID = data[4:20] @@ -87,9 +91,7 @@ def autoCreate(cls, data) -> EntryID: if providerUID == EntryIDType.WRAPPED: return WrappedEntryID(data) - logger.warn(f'UID for EntryID found in database, but no class was specified for it: {providerUID}') - # If all else fails and we do recognize it, just return a plain EntryID. - return cls(data) + raise FeatureNotImplemented(f'UID for EntryID found in database, but no class was specified for it: {providerUID}') def __init__(self, data : bytes): self.__flags = data[:4] From 3d25ff661298a29bdf322622ed31dd260f0afe1c Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 16 Aug 2023 18:52:56 -0700 Subject: [PATCH 10/68] Make sure JournalAssociatedAttachment can be made --- CHANGELOG.md | 1 + .../custom_att_handler/custom_handler.py | 4 ++-- .../custom_att_handler/jrnl_assoc_att.py | 19 +++++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d095899c..8e41ed5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ * Added new custom attachment handler for journal-associated attachments. * Changed `EntryID.autoCreate` to return `None` if given `None` or empty bytes. * Changed `EntryID.autoCreate` to raise a `FeatureNotImplemented` exception if no valid entry ID class is found. +* Fix typing annocations for `CustomAttachmentHandler`. **v0.45.0** * BREAKING: Changed parsing of string multiple properties to remove the trailing null byte. This *will* cause the output of parsing them to differ. diff --git a/extract_msg/attachments/custom_att_handler/custom_handler.py b/extract_msg/attachments/custom_att_handler/custom_handler.py index c2d7c1bc..99300e42 100644 --- a/extract_msg/attachments/custom_att_handler/custom_handler.py +++ b/extract_msg/attachments/custom_att_handler/custom_handler.py @@ -69,7 +69,7 @@ def generateRtf(self) -> Optional[bytes]: """ @property - def attachment(self): + def attachment(self) -> AttachmentBase: """ The attachment this handler is associated with. """ @@ -77,7 +77,7 @@ def attachment(self): @property @abc.abstractmethod - def data(self) -> bytes: + def data(self) -> Optional[bytes]: """ Gets the data for the attachment. diff --git a/extract_msg/attachments/custom_att_handler/jrnl_assoc_att.py b/extract_msg/attachments/custom_att_handler/jrnl_assoc_att.py index c2fc4901..fc2d1b2e 100644 --- a/extract_msg/attachments/custom_att_handler/jrnl_assoc_att.py +++ b/extract_msg/attachments/custom_att_handler/jrnl_assoc_att.py @@ -32,6 +32,15 @@ def isCorrectHandler(cls, attachment : AttachmentBase) -> bool: return True + def generateRtf(self) -> Optional[bytes]: + # TODO + return None + + @property + def data(self) -> None: + # This type of attachment has no direct associated data. + return None + @cached_property def mailMsgAttFld(self) -> Optional[EntryID]: """ @@ -85,6 +94,16 @@ def metafileBytes(self) -> Optional[bytes]: # them. return self.getStream('IOlePres000') or self.getStream('\x02OlePres000') + @property + def name(self) -> None: + # Doesn't save. + return None + + @property + def obj(self) -> None: + # No object to represent this. + return None + registerHandler(JournalAssociatedAttachment) \ No newline at end of file From 6be7c9b4f782fd1a7238be49919bb8f821a023a9 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Thu, 17 Aug 2023 17:48:58 -0700 Subject: [PATCH 11/68] More properties, restore parsedDate --- CHANGELOG.md | 5 +++-- extract_msg/enums.py | 20 ++++++++++++++++++++ extract_msg/msg_classes/calendar_base.py | 7 ------- extract_msg/msg_classes/message_base.py | 22 +++++++++++++++++----- extract_msg/msg_classes/msg.py | 21 ++++++++++++++++++--- 5 files changed, 58 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e41ed5c..7bc8a2ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,14 @@ **v0.46.0** * Changed the base class of `EntryID` from no base class to `abc.ABC`. * Added `position` property to `EntryID` to tell how many bytes were used to create the `EntryID`. -* Added additional properties to `MSGFile`: `contacts` and `contactLinkEntry`. +* Added a number of properties to `MSGFile` from \[MS-OXCMSG\]. +* Moved some properties down to `MessageBase` from it's subclasses. * Added support for Journal objects. * Changed internal code of `PermanentEntryID` to correctly parse the data. Previously the distinguished name did not actually end at the null character, instead ending at the end of the bytes provided. If there was trailing data, it would be captured inadvertantly. * Finished definition for `StoreObjectEntryID`. * Added new kwargs for MSG files: `dateFormat` and `datetimeFormat`. These allow the user to easily override the strings being used for format dates and dates that include a time component, respectively. * In unifying all the formats into 2 options, you may notice that some will look a bit different starting from this version, as there was an unfortunately large amount of variation. -* Removed `MessageBase.parsedDate`. +* Fixed code for `MessageBase.parsedDate` which could have incorrect values. * Fixed issues with `MessageBase.date` and related things either being incorrectly documented or doing things that are not specified by the documentation. It was *supposed* to have been changed to use `datetime` objects, but it was still using strings. * Removed unused function `extract_msg.utils.isEmptyString`. * Removed unused function `extract_msg.utils.properHex`. diff --git a/extract_msg/enums.py b/extract_msg/enums.py index 0807d01e..03c734a8 100644 --- a/extract_msg/enums.py +++ b/extract_msg/enums.py @@ -58,6 +58,7 @@ 'RecurPatternType', 'ResponseStatus', 'ResponseType', + 'RetentionFlags', 'RuleActionType', 'SaveType', 'Sensitivity', @@ -1547,6 +1548,25 @@ class ResponseType(enum.Enum): +class RetentionFlags(enum.IntFlag): + """ + Flags that specify the status of nature of an item's retention tag or + archive tag. + + See the section labeled "PidTagRetentionFlags" of [MS-OXCMSG] for details. + """ + EXPLICIT_TAG = 0x001 + USER_OVERRIDE = 0x002 + AUTO_TAG = 0x004 + PERSONAL_TAG = 0x008 + EXPLICIT_ARCHIVE_TAG = 0x010 + KEEP_IN_PLACE = 0x020 + SYSTEM_DATA = 0x040 + NEEDS_RESCAN = 0x080 + PENDING_RESCAN = 0x100 + + + class RuleActionType(enum.IntEnum): OP_MOVE = 0x01 OP_COPY = 0x02 diff --git a/extract_msg/msg_classes/calendar_base.py b/extract_msg/msg_classes/calendar_base.py index d95bd380..99f4f6dc 100644 --- a/extract_msg/msg_classes/calendar_base.py +++ b/extract_msg/msg_classes/calendar_base.py @@ -432,13 +432,6 @@ def resourceAttendees(self) -> Optional[str]: """ return self.getNamedProp('0008', ps.PSETID_MEETING) - @functools.cached_property - def responseRequested(self) -> bool: - """ - Whether to send Meeting Response objects to the organizer. - """ - return bool(self.getPropertyVal('0063000B')) - @functools.cached_property def responseStatus(self) -> ResponseStatus: """ diff --git a/extract_msg/msg_classes/message_base.py b/extract_msg/msg_classes/message_base.py index 093d469f..326e591a 100644 --- a/extract_msg/msg_classes/message_base.py +++ b/extract_msg/msg_classes/message_base.py @@ -24,7 +24,7 @@ from email import policy from email.message import EmailMessage from email.parser import HeaderParser -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, List, Optional, Tuple, Union from .. import constants from .._rtf.create_doc import createDocument @@ -1014,7 +1014,7 @@ def defaultFolderName(self) -> str: try: return self._defaultFolderName except AttributeError: - d = self.parsedDate + d = self.parsedDate or tuple([0] * 9) dirName = '{0:02d}-{1:02d}-{2:02d}_{3:02d}{4:02d}'.format(*d) if d else 'UnknownDate' dirName += ' ' + (prepareFilename(self.subject) if self.subject else '[No subject]') @@ -1047,12 +1047,10 @@ def header(self) -> email.message.Message: headerText = self.headerText if headerText: header = HeaderParser(policy = policy.default).parsestr(headerText) - del header['Date'] - header['Date'] = self.date.__format__(self.datetimeFormat) else: logger.info('Header is empty or was not found. Header will be generated from other streams.') header = HeaderParser(policy = policy.default).parsestr('') - header.add_header('Date', self.date.__format__(self.datetimeFormat)) + header.add_header('Date', email.utils.format_datetime(self.date)) header.add_header('From', self.sender) header.add_header('To', self.to) header.add_header('Cc', self.cc) @@ -1232,6 +1230,13 @@ def messageId(self) -> Optional[str]: logger.info('Header found, but "Message-Id" is not included. Will be generated from other streams.') return self.getStringStream('__substg1.0_1035') + @functools.cached_property + def parsedDate(self) -> Optional[Tuple[int, int, int, int, int, int, int, int, int]]: + """ + Returns a 9 tuple of the parsed date from the header. + """ + return email.utils.parsedate(self.header['Date']) + @functools.cached_property def receivedTime(self) -> Optional[datetime.datetime]: """ @@ -1264,6 +1269,13 @@ def reportTag(self) -> Optional[ReportTag]: """ return self.getStreamAs('__substg1.0_00310102', ReportTag) + @functools.cached_property + def responseRequested(self) -> bool: + """ + Whether to send Meeting Response objects to the organizer. + """ + return bool(self.getPropertyVal('0063000B')) + @functools.cached_property def rtfBody(self) -> Optional[bytes]: """ diff --git a/extract_msg/msg_classes/msg.py b/extract_msg/msg_classes/msg.py index 37f9e989..39d8488d 100644 --- a/extract_msg/msg_classes/msg.py +++ b/extract_msg/msg_classes/msg.py @@ -30,7 +30,7 @@ from ..encoding import lookupCodePage from ..enums import ( ErrorBehavior, InsecureFeatures, Importance, Priority, PropertiesType, - SaveType, Sensitivity, SideEffect + RetentionFlags, SaveType, Sensitivity, SideEffect ) from ..exceptions import ( ConversionError, InvalidFileFormatError, PrefixError, @@ -1047,6 +1047,22 @@ def props(self) -> PropertiesStore: return PropertiesStore(stream, PropertiesType.MESSAGE if self.prefix == '' else PropertiesType.MESSAGE_EMBED) + @functools.cached_property + def retentionDate(self) -> Optional[datetime.datetime]: + """ + The date, in UTC, after which a Message Object is expired by the server. + If None, the Message object never expires. + """ + return self.getPropertyVal('301C0040') + + @functools.cached_property + def retentionFlags(self) -> Optional[RetentionFlags]: + """ + Flags that specify the status or nature of an item's retention tag or + archive tag. + """ + return self.getPropertyAs('301D0003', RetentionFlags) + @functools.cached_property def sensitivity(self) -> Optional[Sensitivity]: """ @@ -1071,7 +1087,6 @@ def stringEncoding(self): # Let's first check if the encoding will be unicode: if self.areStringsUnicode: self.__stringEncoding = "utf-16-le" - return self.__stringEncoding else: # Well, it's not unicode. Now we have to figure out what it IS. if '3FFD0003' not in self.props: @@ -1083,7 +1098,7 @@ def stringEncoding(self): enc = cast(int, self.getPropertyVal('3FFD0003')) # Now we just need to translate that value. self.__stringEncoding = lookupCodePage(enc) - return self.__stringEncoding + return self.__stringEncoding @property def treePath(self) -> List[weakref.ReferenceType]: From ae6f0e351428152fefea65db4d94c56aab082567 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Thu, 17 Aug 2023 18:09:46 -0700 Subject: [PATCH 12/68] Adjusted more properties to use cached property --- extract_msg/msg_classes/message_base.py | 75 +++++++++++-------------- 1 file changed, 33 insertions(+), 42 deletions(-) diff --git a/extract_msg/msg_classes/message_base.py b/extract_msg/msg_classes/message_base.py index 326e591a..09ca3008 100644 --- a/extract_msg/msg_classes/message_base.py +++ b/extract_msg/msg_classes/message_base.py @@ -20,6 +20,7 @@ import bs4 import compressed_rtf import RTFDE +import RTFDE.exceptions from email import policy from email.message import EmailMessage @@ -966,45 +967,38 @@ def date(self) -> Optional[datetime.datetime]: """ return self.props.date if self.isSent else None - @property + @functools.cached_property def deencapsulatedRtf(self) -> Optional[RTFDE.DeEncapsulator]: """ Returns the instance of the deencapsulated RTF body. If there is no RTF body or the body is not encasulated, returns None. """ - try: - return self._deencapsultor - except AttributeError: - if self.rtfBody: - # If there is an RTF body, we try to deencapsulate it. - body = self.rtfBody - # Sometimes you get MSG files whose RTF body has stuff - # *after* the body, and RTFDE can't handle that. Here is - # how we compensate. - while body and body[-1] != 125: - body = body[:-1] + if self.rtfBody: + # If there is an RTF body, we try to deencapsulate it. + body = self.rtfBody + # Sometimes you get MSG files whose RTF body has stuff + # *after* the body, and RTFDE can't handle that. Here is + # how we compensate. + while body and body[-1] != 125: + body = body[:-1] - try: - self._deencapsultor = RTFDE.DeEncapsulator(body) - self._deencapsultor.deencapsulate() - except RTFDE.exceptions.NotEncapsulatedRtf as e: - logger.debug('RTF body is not encapsulated.') - self._deencapsultor = None - except RTFDE.exceptions.MalformedEncapsulatedRtf as _e: - if ErrorBehavior.RTFDE_MALFORMED not in self.errorBehavior: - raise - logger.info('RTF body contains malformed encapsulated content.') - self._deencapsultor = None - except Exception: - # If we are just ignoring the errors, log it then set to - # None. Otherwise, continue the exception. - if ErrorBehavior.RTFDE_UNKNOWN_ERROR not in self.errorBehavior: - raise - logger.exception('Unhandled error happened while using RTFDE. You have choosen to ignore these errors.') - self._deencapsultor = None - else: - self._deencapsultor = None - return self._deencapsultor + try: + deencapsultor = RTFDE.DeEncapsulator(body) + deencapsultor.deencapsulate() + return deencapsultor + except RTFDE.exceptions.NotEncapsulatedRtf: + logger.debug('RTF body is not encapsulated.') + except RTFDE.exceptions.MalformedEncapsulatedRtf: + if ErrorBehavior.RTFDE_MALFORMED not in self.errorBehavior: + raise + logger.info('RTF body contains malformed encapsulated content.') + except Exception: + # If we are just ignoring the errors, log it then set to + # None. Otherwise, continue the exception. + if ErrorBehavior.RTFDE_UNKNOWN_ERROR not in self.errorBehavior: + raise + logger.exception('Unhandled error happened while using RTFDE. You have choosen to ignore these errors.') + return None @property def defaultFolderName(self) -> str: @@ -1062,20 +1056,17 @@ def header(self) -> email.message.Message: self.__headerInit = True return header - @property + @functools.cached_property def headerDict(self) -> Dict: """ Returns a dictionary of the entries in the header """ + headerDict = dict(self.header._headers) try: - return self._headerDict - except AttributeError: - self._headerDict = dict(self.header._headers) - try: - self._headerDict.pop('Received') - except KeyError: - pass - return self._headerDict + headerDict.pop('Received') + except KeyError: + pass + return headerDict @property def headerFormatProperties(self) -> constants.HEADER_FORMAT_TYPE: From cfafda349b48d709c7ba906e4e0cf6332f155a79 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Thu, 17 Aug 2023 19:42:44 -0700 Subject: [PATCH 13/68] Started fixing json outputs, removed imapclient --- CHANGELOG.md | 3 +++ extract_msg/msg_classes/message_base.py | 27 +++++++++++++------------ extract_msg/msg_classes/post.py | 14 ++++++------- extract_msg/msg_classes/sticky_note.py | 10 +++++++++ requirements.txt | 1 - 5 files changed, 33 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bc8a2ae..17e0ae69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ * Changed `EntryID.autoCreate` to return `None` if given `None` or empty bytes. * Changed `EntryID.autoCreate` to raise a `FeatureNotImplemented` exception if no valid entry ID class is found. * Fix typing annocations for `CustomAttachmentHandler`. +* Removed unneed `imapclient` dependency. +* Changed `getJson` to have values be null if they aren't found rather than an empty string. +* Implemented the `getJson` method correctly for a number of classes. **v0.45.0** * BREAKING: Changed parsing of string multiple properties to remove the trailing null byte. This *will* cause the output of parsing them to differ. diff --git a/extract_msg/msg_classes/message_base.py b/extract_msg/msg_classes/message_base.py index 09ca3008..12ee047c 100644 --- a/extract_msg/msg_classes/message_base.py +++ b/extract_msg/msg_classes/message_base.py @@ -46,8 +46,6 @@ prepareFilename, rtfSanitizeHtml, rtfSanitizePlain, validateHtml ) -from imapclient.imapclient import decode_utf7 - logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) @@ -252,7 +250,8 @@ def dump(self) -> None: """ print('Message') print('Subject:', self.subject) - print('Date:', self.date.__format__(self.datetimeFormat)) + if self.date: + print('Date:', self.date.__format__(self.datetimeFormat)) print('Body:') print(self.body) @@ -271,12 +270,13 @@ def getInjectableHeader(self, prefix : str, joinStr : str, suffix : str, formatt If self.headerFormatProperties is None, immediately returns an empty string. """ - formattedProps = [] allProps = self.headerFormatProperties if allProps is None: return '' + formattedProps = [] + for entry in allProps: isGroup = False entryUsed = False @@ -320,13 +320,13 @@ def getJson(self) -> str: Returns the JSON representation of the Message. """ return json.dumps({ - 'from': inputToString(self.sender, self.stringEncoding), - 'to': inputToString(self.to, self.stringEncoding), - 'cc': inputToString(self.cc, self.stringEncoding), - 'bcc': inputToString(self.bcc, self.stringEncoding), - 'subject': inputToString(self.subject, self.stringEncoding), - 'date': inputToString(self.date.__format__(self.datetimeFormat), self.stringEncoding), - 'body': decode_utf7(self.body), + 'from': self.sender, + 'to': self.to, + 'cc': self.cc, + 'bcc': self.bcc, + 'subject': self.subject, + 'date': self.date.__format__(self.datetimeFormat) if self.date else None, + 'body': self.body, }) def getSaveBody(self, **_) -> bytes: @@ -1044,7 +1044,8 @@ def header(self) -> email.message.Message: else: logger.info('Header is empty or was not found. Header will be generated from other streams.') header = HeaderParser(policy = policy.default).parsestr('') - header.add_header('Date', email.utils.format_datetime(self.date)) + if self.date: + header.add_header('Date', email.utils.format_datetime(self.date)) header.add_header('From', self.sender) header.add_header('To', self.to) header.add_header('Cc', self.cc) @@ -1099,7 +1100,7 @@ def headerFormatProperties(self) -> constants.HEADER_FORMAT_TYPE: return { '-basic info-': { 'From': self.sender, - 'Sent': self.date.__format__(self.datetimeFormat), + 'Sent': self.date.__format__(self.datetimeFormat) if self.date else None, 'To': self.to, 'Cc': self.cc, 'Bcc': self.bcc, diff --git a/extract_msg/msg_classes/post.py b/extract_msg/msg_classes/post.py index c2faab5b..152d8d2b 100644 --- a/extract_msg/msg_classes/post.py +++ b/extract_msg/msg_classes/post.py @@ -12,8 +12,6 @@ from .message_base import MessageBase from ..utils import inputToString -from imapclient.imapclient import decode_utf7 - class Post(MessageBase): """ @@ -25,11 +23,11 @@ def getJson(self) -> str: Returns the JSON representation of the Post. """ return json.dumps({ - 'from': inputToString(self.sender, self.stringEncoding), - 'subject': inputToString(self.subject, self.stringEncoding), - 'date': inputToString(self.date.__format__(self.datetimeFormat), self.stringEncoding), - 'conversation': inputToString(self.conversation, self.stringEncoding), - 'body': decode_utf7(self.body), + 'from': self.sender, + 'subject': self.subject, + 'date': self.date.__format__(self.datetimeFormat) if self.date else None, + 'conversation': self.conversation, + 'body': self.body, }) @functools.cached_property @@ -44,7 +42,7 @@ def headerFormatProperties(self) -> constants.HEADER_FORMAT_TYPE: return { '-main details-': { 'From': self.sender, - 'Posted At': self.date.__format__(self.datetimeFormat), + 'Posted At': self.date.__format__(self.datetimeFormat) if self.date else None, 'Conversation': self.conversation, }, '-subject-': { diff --git a/extract_msg/msg_classes/sticky_note.py b/extract_msg/msg_classes/sticky_note.py index b382228b..a827c0ea 100644 --- a/extract_msg/msg_classes/sticky_note.py +++ b/extract_msg/msg_classes/sticky_note.py @@ -1,4 +1,5 @@ import functools +import json from typing import Optional @@ -13,6 +14,15 @@ class StickyNote(MessageBase): """ A sticky note. """ + def getJson(self) -> str: + return json.dumps({ + 'subject': self.subject, + 'date': self.date.__format__(self.datetimeFormat) if self.date else None, + 'body': self.body, + 'height': self.noteHeight, + 'width': self.noteWidth, + 'color': None if self.noteColor is None else self.noteColor.name, + }) @property def headerFormatProperties(self) -> constants.HEADER_FORMAT_TYPE: diff --git a/requirements.txt b/requirements.txt index def1fce6..26c4face 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ # First level requirements -imapclient>=2.3.0,<3 olefile==0.46 tzlocal>=4.2,<6 compressed_rtf>=1.0.6,<2 From 56dda07a00c85c03a98fc500732e4e1281188a50 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Thu, 17 Aug 2023 19:48:04 -0700 Subject: [PATCH 14/68] Bump version so I don't forget to later --- README.rst | 4 ++-- extract_msg/__init__.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 0e15f44f..22962c0c 100644 --- a/README.rst +++ b/README.rst @@ -259,8 +259,8 @@ your access to the newest major version of extract-msg. .. |License: GPL v3| image:: https://img.shields.io/badge/License-GPLv3-blue.svg :target: LICENSE.txt -.. |PyPI3| image:: https://img.shields.io/badge/pypi-0.45.0-blue.svg - :target: https://pypi.org/project/extract-msg/0.45.0/ +.. |PyPI3| image:: https://img.shields.io/badge/pypi-0.46.0-blue.svg + :target: https://pypi.org/project/extract-msg/0.46.0/ .. |PyPI2| image:: https://img.shields.io/badge/python-3.8+-brightgreen.svg :target: https://www.python.org/downloads/release/python-3816/ diff --git a/extract_msg/__init__.py b/extract_msg/__init__.py index 83251e96..81fc86b7 100644 --- a/extract_msg/__init__.py +++ b/extract_msg/__init__.py @@ -27,12 +27,13 @@ # along with this program. If not, see . __author__ = 'Destiny Peterson & Matthew Walker' -__date__ = '2023-08-12' -__version__ = '0.45.0' +__date__ = '2023-08-17' +__version__ = '0.46.0' __all__ = [ # Modules: 'attachments', + 'constants', 'enums', 'exceptions', 'msg_classes', From d82957bd0ff7314bc5287078a87b34a6d9c5e778 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Fri, 18 Aug 2023 15:26:21 -0700 Subject: [PATCH 15/68] Fix json for sticky note --- extract_msg/msg_classes/sticky_note.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extract_msg/msg_classes/sticky_note.py b/extract_msg/msg_classes/sticky_note.py index a827c0ea..7451eb75 100644 --- a/extract_msg/msg_classes/sticky_note.py +++ b/extract_msg/msg_classes/sticky_note.py @@ -21,7 +21,7 @@ def getJson(self) -> str: 'body': self.body, 'height': self.noteHeight, 'width': self.noteWidth, - 'color': None if self.noteColor is None else self.noteColor.name, + 'color': None if self.noteColor is None else self.noteColor.name.lower(), }) @property From bba5f54098e6fcb2577f5e8dd8c66af746c6e5ea Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Fri, 18 Aug 2023 16:05:06 -0700 Subject: [PATCH 16/68] Adjust typing, add getJson fro AppointmentMeeting --- extract_msg/attachments/attachment_base.py | 8 ++-- .../custom_att_handler/custom_handler.py | 4 +- extract_msg/msg_classes/appointment.py | 38 +++++++++++++++++++ extract_msg/msg_classes/msg.py | 13 +++---- extract_msg/recipient.py | 6 +-- 5 files changed, 53 insertions(+), 16 deletions(-) diff --git a/extract_msg/attachments/attachment_base.py b/extract_msg/attachments/attachment_base.py index 90457cfd..6eb7fa3c 100644 --- a/extract_msg/attachments/attachment_base.py +++ b/extract_msg/attachments/attachment_base.py @@ -291,7 +291,7 @@ def getMultipleString(self, filename) -> Optional[List[str]]: raise ReferenceError('The msg file for this Attachment instance has been garbage collected.') return msg.getMultipleString([self.__dir, msgPathToString(filename)]) - def getNamedAs(self, propertyName : str, guid : str, overrideClass : Callable[..., _T]) -> Optional[_T]: + def getNamedAs(self, propertyName : str, guid : str, overrideClass : Callable[[Any], _T]) -> Optional[_T]: """ Returns the named property, setting the class if specified. @@ -314,7 +314,7 @@ def getNamedProp(self, propertyName : str, guid : str, default : _T = None) -> U """ return self.namedProperties.get((propertyName, guid), default) - def getPropertyAs(self, propertyName, overrideClass : Callable[..., _T]) -> Optional[_T]: + def getPropertyAs(self, propertyName, overrideClass : Callable[[Any], _T]) -> Optional[_T]: """ Returns the property, setting the class if found. @@ -391,7 +391,7 @@ def getStream(self, filename) -> Optional[bytes]: raise ReferenceError('The msg file for this Attachment instance has been garbage collected.') return msg.getStream([self.__dir, msgPathToString(filename)]) - def getStreamAs(self, streamID, overrideClass : Callable[..., _T]) -> Optional[_T]: + def getStreamAs(self, streamID, overrideClass : Callable[[Any], _T]) -> Optional[_T]: """ Returns the specified stream, modifying it to the specified class if it is found. @@ -424,7 +424,7 @@ def getStringStream(self, filename) -> Optional[str]: raise ReferenceError('The msg file for this Attachment instance has been garbage collected.') return msg.getStringStream([self.__dir, msgPathToString(filename)]) - def getStringStreamAs(self, streamID, overrideClass : Callable[..., _T]) -> Optional[_T]: + def getStringStreamAs(self, streamID, overrideClass : Callable[[Any], _T]) -> Optional[_T]: """ Returns the specified string stream, modifying it to the specified class if it is found. diff --git a/extract_msg/attachments/custom_att_handler/custom_handler.py b/extract_msg/attachments/custom_att_handler/custom_handler.py index 99300e42..43e2b1b0 100644 --- a/extract_msg/attachments/custom_att_handler/custom_handler.py +++ b/extract_msg/attachments/custom_att_handler/custom_handler.py @@ -8,7 +8,7 @@ import abc -from typing import Callable, Optional, TYPE_CHECKING, TypeVar +from typing import Any, Callable, Optional, TYPE_CHECKING, TypeVar from ...utils import msgPathToString @@ -35,7 +35,7 @@ def getStream(self, path) -> Optional[bytes]: """ return self.attachment.getStream('__substg1.0_3701000D/' + msgPathToString(path)) - def getStreamAs(self, streamID, overrideClass : Callable[..., _T]) -> Optional[_T]: + def getStreamAs(self, streamID, overrideClass : Callable[[Any], _T]) -> Optional[_T]: """ Returns the specified stream, modifying it to the specified class if it is found. diff --git a/extract_msg/msg_classes/appointment.py b/extract_msg/msg_classes/appointment.py index 65a81470..4b6317c0 100644 --- a/extract_msg/msg_classes/appointment.py +++ b/extract_msg/msg_classes/appointment.py @@ -5,6 +5,7 @@ import datetime import functools +import json from typing import Optional @@ -23,6 +24,43 @@ class AppointmentMeeting(Calendar): object. """ + def getJson(self) -> str: + meetingStatusString = { + ResponseStatus.NONE: None, + ResponseStatus.ORGANIZED: 'Meeting organizer', + ResponseStatus.TENTATIVE: 'Tentatively accepted', + ResponseStatus.ACCEPTED: 'Accepted', + ResponseStatus.DECLINED: 'Declined', + ResponseStatus.NOT_RESPONDED: 'Not yet responded', + }[self.responseStatus] + + # Get the recurrence string. + recur = '(none)' + if self.appointmentRecur: + recur = { + RecurPatternType.DAY: 'Daily', + RecurPatternType.WEEK: 'Weekly', + RecurPatternType.MONTH: 'Monthly', + RecurPatternType.MONTH_NTH: 'Monthly', + RecurPatternType.MONTH_END: 'Monthly', + RecurPatternType.HJ_MONTH: 'Monthly', + RecurPatternType.HJ_MONTH_NTH: 'Monthly', + RecurPatternType.HJ_MONTH_END: 'Monthly', + }[self.appointmentRecur.patternType] + + return json.dumps({ + 'recurrence': recur, + 'recurrencePattern': self.recurrencePattern, + 'body': self.body, + 'meetingStatus': meetingStatusString, + 'organizer': self.organizer, + 'requiredAttendees': self.to, + 'optionalAttendees': self.cc, + 'resources': self.bcc, + 'start': self.startDate.__format__(self.datetimeFormat) if self.endDate else None, + 'end': self.endDate.__format__(self.datetimeFormat) if self.endDate else None, + }) + @functools.cached_property def appointmentCounterProposal(self) -> bool: """ diff --git a/extract_msg/msg_classes/msg.py b/extract_msg/msg_classes/msg.py index 39d8488d..47732461 100644 --- a/extract_msg/msg_classes/msg.py +++ b/extract_msg/msg_classes/msg.py @@ -544,7 +544,7 @@ def getMultipleString(self, filename, prefix : bool = True) -> Optional[List[str ret[index] = item.decode(self.stringEncoding)[:-1] return ret - def getNamedAs(self, propertyName : str, guid : str, overrideClass : Callable[..., _T]) -> Optional[_T]: + def getNamedAs(self, propertyName : str, guid : str, overrideClass : Callable[[Any], _T]) -> Optional[_T]: """ Returns the named property, setting the class if specified. @@ -567,7 +567,7 @@ def getNamedProp(self, propertyName : str, guid : str, default : _T = None) -> U """ return self.namedProperties.get((propertyName, guid), default) - def getPropertyAs(self, propertyName, overrideClass : Callable[..., _T]) -> Optional[_T]: + def getPropertyAs(self, propertyName, overrideClass : Callable[[Any], _T]) -> Optional[_T]: """ Returns the property, setting the class if found. @@ -650,7 +650,7 @@ def getStream(self, filename, prefix : bool = True) -> Optional[bytes]: logger.info(f'Stream "{filename}" was requested but could not be found. Returning `None`.') return None - def getStreamAs(self, streamID, overrideClass : Callable[..., _T]) -> Optional[_T]: + def getStreamAs(self, streamID, overrideClass : Callable[[Any], _T]) -> Optional[_T]: """ Returns the specified stream, modifying it to the specified class if it is found. @@ -689,7 +689,7 @@ def getStringStream(self, filename, prefix : bool = True) -> Optional[str]: tmp = self.getStream(filename + '001E', prefix = False) return None if tmp is None else tmp.decode(self.stringEncoding) - def getStringStreamAs(self, streamID, overrideClass : Callable[..., _T]) -> Optional[_T]: + def getStringStreamAs(self, streamID, overrideClass : Callable[[Any], _T]) -> Optional[_T]: """ Returns the specified string stream, modifying it to the specified class if it is found. @@ -785,9 +785,8 @@ def saveRaw(self, path) -> None: # Save contents of directory. with zfile.open(sysdir + '/' + filename, 'w') as f: data = self.getStream(dir_) - # Specifically check for None. If this is bytes we still want to do this line. - # There was actually this weird issue where for some reason data would be bytes - # but then also simultaneously register as None? + # Specifically check for None. If this is bytes we still + # want to do this line. if data is not None: f.write(data) diff --git a/extract_msg/recipient.py b/extract_msg/recipient.py index 72ee3a97..1050b13d 100644 --- a/extract_msg/recipient.py +++ b/extract_msg/recipient.py @@ -246,7 +246,7 @@ def getMultipleString(self, filename) -> Optional[List[str]]: raise ReferenceError('The msg file for this Recipient instance has been garbage collected.') return msg.getMultipleString([self.__dir, msgPathToString(filename)]) - def getPropertyAs(self, propertyName, overrideClass : Callable[..., _T]) -> Optional[_T]: + def getPropertyAs(self, propertyName, overrideClass : Callable[[Any], _T]) -> Optional[_T]: """ Returns the property, setting the class if found. @@ -323,7 +323,7 @@ def getStream(self, filename) -> Optional[bytes]: raise ReferenceError('The msg file for this Recipient instance has been garbage collected.') return msg.getStream([self.__dir, msgPathToString(filename)]) - def getStreamAs(self, streamID, overrideClass : Callable[..., _T]) -> Optional[_T]: + def getStreamAs(self, streamID, overrideClass : Callable[[Any], _T]) -> Optional[_T]: """ Returns the specified stream, modifying it to the specified class if it is found. @@ -359,7 +359,7 @@ def getStringStream(self, filename) -> Optional[str]: raise ReferenceError('The msg file for this Recipient instance has been garbage collected.') return msg.getStringStream([self.__dir, msgPathToString(filename)]) - def getStringStreamAs(self, streamID, overrideClass : Callable[..., _T]) -> Optional[_T]: + def getStringStreamAs(self, streamID, overrideClass : Callable[[Any], _T]) -> Optional[_T]: """ Returns the specified string stream, modifying it to the specified class if it is found. From 15a31734bea3fe93ca4fd16857a5732fdbcd7d28 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Fri, 18 Aug 2023 16:11:22 -0700 Subject: [PATCH 17/68] Added getJson for Journal --- extract_msg/msg_classes/journal.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/extract_msg/msg_classes/journal.py b/extract_msg/msg_classes/journal.py index 475d01c9..dc9aa10e 100644 --- a/extract_msg/msg_classes/journal.py +++ b/extract_msg/msg_classes/journal.py @@ -3,8 +3,10 @@ ] +import base64 import datetime import functools +import json from typing import List, Optional @@ -19,6 +21,20 @@ class Journal(MessageBase): Class for parsing Journal messages. """ + def getJson(self) -> str: + return json.dumps({ + 'subject': self.subject, + 'entryType': self.logTypeDesc, + 'company': self.companies[0] if self.companies else None, + 'start': self.logStart.__format__(self.datetimeFormat) if self.logStart else None, + 'end': self.logEnd.__format__(self.datetimeFormat) if self.logEnd else None, + 'duration': minutesToDurationStr(self.duration), + 'body': self.body, + # There is a good chance the body property won't exist, so this is a + # backup. + 'rtfBodyB64': base64.b64encode(self.rtfBody) if self.rtfBody else None, + }) + @functools.cached_property def companies(self) -> Optional[List[str]]: """ @@ -108,8 +124,8 @@ def headerFormatProperties(self) -> HEADER_FORMAT_TYPE: 'Company': self.companies[0] if self.companies else None, }, '-time-': { - 'Start': self.logStart.__format__(self.datetimeFormat), - 'End': self.logEnd.__format__(self.datetimeFormat), + 'Start': self.logStart.__format__(self.datetimeFormat) if self.logStart else None, + 'End': self.logEnd.__format__(self.datetimeFormat) if self.logEnd else None, 'Duration': minutesToDurationStr(self.duration), }, } From aacaa78555796752bde5618afa164bc011a1ba24 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Fri, 18 Aug 2023 16:26:08 -0700 Subject: [PATCH 18/68] More getJson implementations --- .../msg_classes/meeting_cancellation.py | 40 ++++++++++++++++++- extract_msg/msg_classes/meeting_forward.py | 40 ++++++++++++++++++- extract_msg/msg_classes/meeting_request.py | 38 ++++++++++++++++++ 3 files changed, 116 insertions(+), 2 deletions(-) diff --git a/extract_msg/msg_classes/meeting_cancellation.py b/extract_msg/msg_classes/meeting_cancellation.py index 6929c427..f1ae6e80 100644 --- a/extract_msg/msg_classes/meeting_cancellation.py +++ b/extract_msg/msg_classes/meeting_cancellation.py @@ -3,6 +3,8 @@ ] +import json + from .. import constants from ..enums import RecurPatternType, ResponseStatus from .meeting_related import MeetingRelated @@ -15,6 +17,43 @@ class MeetingCancellation(MeetingRelated): Class for a Meeting Cancellation object. """ + def getJson(self) -> str: + meetingStatusString = { + ResponseStatus.NONE: None, + ResponseStatus.ORGANIZED: 'Meeting organizer', + ResponseStatus.TENTATIVE: 'Tentatively accepted', + ResponseStatus.ACCEPTED: 'Accepted', + ResponseStatus.DECLINED: 'Declined', + ResponseStatus.NOT_RESPONDED: 'Not yet responded', + }[self.responseStatus] + + # Get the recurrence string. + recur = '(none)' + if self.appointmentRecur: + recur = { + RecurPatternType.DAY: 'Daily', + RecurPatternType.WEEK: 'Weekly', + RecurPatternType.MONTH: 'Monthly', + RecurPatternType.MONTH_NTH: 'Monthly', + RecurPatternType.MONTH_END: 'Monthly', + RecurPatternType.HJ_MONTH: 'Monthly', + RecurPatternType.HJ_MONTH_NTH: 'Monthly', + RecurPatternType.HJ_MONTH_END: 'Monthly', + }[self.appointmentRecur.patternType] + + return json.dumps({ + 'recurrence': recur, + 'recurrencePattern': self.recurrencePattern, + 'body': self.body, + 'meetingStatus': meetingStatusString, + 'organizer': self.organizer, + 'requiredAttendees': self.to, + 'optionalAttendees': self.cc, + 'resources': self.bcc, + 'start': self.startDate.__format__(self.datetimeFormat) if self.endDate else None, + 'end': self.endDate.__format__(self.datetimeFormat) if self.endDate else None, + }) + @property def headerFormatProperties(self) -> constants.HEADER_FORMAT_TYPE: meetingStatusString = { @@ -40,7 +79,6 @@ def headerFormatProperties(self) -> constants.HEADER_FORMAT_TYPE: RecurPatternType.HJ_MONTH_END: 'Monthly', }[self.appointmentRecur.patternType] - return { '-main info-': { 'Subject': self.subject, diff --git a/extract_msg/msg_classes/meeting_forward.py b/extract_msg/msg_classes/meeting_forward.py index bfe17eb6..1a3605fa 100644 --- a/extract_msg/msg_classes/meeting_forward.py +++ b/extract_msg/msg_classes/meeting_forward.py @@ -4,12 +4,13 @@ import functools +import json from typing import Optional from .. import constants from .meeting_related import MeetingRelated -from ..enums import RecurPatternType +from ..enums import RecurPatternType, ResponseStatus class MeetingForwardNotification(MeetingRelated): @@ -17,6 +18,43 @@ class MeetingForwardNotification(MeetingRelated): Class for handling Meeting Forward Notification objects. """ + def getJson(self) -> str: + meetingStatusString = { + ResponseStatus.NONE: None, + ResponseStatus.ORGANIZED: 'Meeting organizer', + ResponseStatus.TENTATIVE: 'Tentatively accepted', + ResponseStatus.ACCEPTED: 'Accepted', + ResponseStatus.DECLINED: 'Declined', + ResponseStatus.NOT_RESPONDED: 'Not yet responded', + }[self.responseStatus] + + # Get the recurrence string. + recur = '(none)' + if self.appointmentRecur: + recur = { + RecurPatternType.DAY: 'Daily', + RecurPatternType.WEEK: 'Weekly', + RecurPatternType.MONTH: 'Monthly', + RecurPatternType.MONTH_NTH: 'Monthly', + RecurPatternType.MONTH_END: 'Monthly', + RecurPatternType.HJ_MONTH: 'Monthly', + RecurPatternType.HJ_MONTH_NTH: 'Monthly', + RecurPatternType.HJ_MONTH_END: 'Monthly', + }[self.appointmentRecur.patternType] + + return json.dumps({ + 'recurrence': recur, + 'recurrencePattern': self.recurrencePattern, + 'body': self.body, + 'meetingStatus': meetingStatusString, + 'organizer': self.organizer, + 'requiredAttendees': self.to, + 'optionalAttendees': self.cc, + 'resources': self.bcc, + 'start': self.startDate.__format__(self.datetimeFormat) if self.endDate else None, + 'end': self.endDate.__format__(self.datetimeFormat) if self.endDate else None, + }) + @functools.cached_property def forwardNotificationRecipients(self) -> Optional[bytes]: """ diff --git a/extract_msg/msg_classes/meeting_request.py b/extract_msg/msg_classes/meeting_request.py index cea88407..23887136 100644 --- a/extract_msg/msg_classes/meeting_request.py +++ b/extract_msg/msg_classes/meeting_request.py @@ -5,6 +5,7 @@ import datetime import functools +import json from typing import Optional @@ -18,6 +19,43 @@ class MeetingRequest(MeetingRelated): Class for handling Meeting Request and Meeting Update objects. """ + def getJson(self) -> str: + meetingStatusString = { + ResponseStatus.NONE: None, + ResponseStatus.ORGANIZED: 'Meeting organizer', + ResponseStatus.TENTATIVE: 'Tentatively accepted', + ResponseStatus.ACCEPTED: 'Accepted', + ResponseStatus.DECLINED: 'Declined', + ResponseStatus.NOT_RESPONDED: 'Not yet responded', + }[self.responseStatus] + + # Get the recurrence string. + recur = '(none)' + if self.appointmentRecur: + recur = { + RecurPatternType.DAY: 'Daily', + RecurPatternType.WEEK: 'Weekly', + RecurPatternType.MONTH: 'Monthly', + RecurPatternType.MONTH_NTH: 'Monthly', + RecurPatternType.MONTH_END: 'Monthly', + RecurPatternType.HJ_MONTH: 'Monthly', + RecurPatternType.HJ_MONTH_NTH: 'Monthly', + RecurPatternType.HJ_MONTH_END: 'Monthly', + }[self.appointmentRecur.patternType] + + return json.dumps({ + 'recurrence': recur, + 'recurrencePattern': self.recurrencePattern, + 'body': self.body, + 'meetingStatus': meetingStatusString, + 'organizer': self.organizer, + 'requiredAttendees': self.to, + 'optionalAttendees': self.cc, + 'resources': self.bcc, + 'start': self.startDate.__format__(self.datetimeFormat) if self.endDate else None, + 'end': self.endDate.__format__(self.datetimeFormat) if self.endDate else None, + }) + @functools.cached_property def appointmentMessageClass(self) -> Optional[str]: """ From f79398013e80a95b3ae57d42b9541d9f790984e5 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Fri, 18 Aug 2023 16:30:46 -0700 Subject: [PATCH 19/68] Task.getJson --- CHANGELOG.md | 1 + extract_msg/msg_classes/task.py | 26 ++++++++++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17e0ae69..dafd1035 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ * Removed unneed `imapclient` dependency. * Changed `getJson` to have values be null if they aren't found rather than an empty string. * Implemented the `getJson` method correctly for a number of classes. +* Changed `Task.percentComplete` to always return a float. **v0.45.0** * BREAKING: Changed parsing of string multiple properties to remove the trailing null byte. This *will* cause the output of parsing them to differ. diff --git a/extract_msg/msg_classes/task.py b/extract_msg/msg_classes/task.py index 43ee3c88..43ed5861 100644 --- a/extract_msg/msg_classes/task.py +++ b/extract_msg/msg_classes/task.py @@ -5,6 +5,7 @@ import datetime import functools +import json import logging from typing import Optional @@ -28,6 +29,27 @@ class Task(MessageBase): Class used for parsing task files. """ + def getJson(self) -> str: + status = { + TaskStatus.NOT_STARTED: 'Not Started', + TaskStatus.IN_PROGRESS: 'In Progress', + TaskStatus.COMPLETE: 'Completed', + TaskStatus.WAITING_ON_OTHER: 'Waiting on someone else', + TaskStatus.DEFERRED: 'Deferred', + None: None, + }[self.taskStatus] + + return json.dumps({ + 'subject': self.subject, + 'status': status, + 'percentComplete': f'{self.percentComplete*100:.0f}%', + 'dateCompleted': self.taskDateCompleted.__format__(self.dateFormat) if self.taskDateCompleted else None, + 'totalWork': f'{self.taskEstimatedEffort or 0} minutes', + 'actualWork': f'{self.taskActualEffort or 0} minutes', + 'owner': self.taskOwner, + 'importance': self.importanceString, + }) + @property def headerFormatProperties(self) -> constants.HEADER_FORMAT_TYPE: status = { @@ -61,12 +83,12 @@ def headerFormatProperties(self) -> constants.HEADER_FORMAT_TYPE: } @functools.cached_property - def percentComplete(self) -> Optional[float]: + def percentComplete(self) -> float: """ Indicates whether a time-flagged Message object is complete. Returns a percentage in decimal form. 1.0 indicates it is complete. """ - return self.getNamedProp('8102', constants.ps.PSETID_TASK) + return self.getNamedProp('8102', constants.ps.PSETID_TASK, 0.0) @functools.cached_property def taskAcceptanceState(self) -> Optional[TaskAcceptance]: From ed97c3993b3bf64270497de613eaa8d0d0b8fba4 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Fri, 18 Aug 2023 16:37:57 -0700 Subject: [PATCH 20/68] Contanct.getJson --- extract_msg/__init__.py | 2 +- extract_msg/msg_classes/contact.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/extract_msg/__init__.py b/extract_msg/__init__.py index 81fc86b7..ea0d3c6c 100644 --- a/extract_msg/__init__.py +++ b/extract_msg/__init__.py @@ -27,7 +27,7 @@ # along with this program. If not, see . __author__ = 'Destiny Peterson & Matthew Walker' -__date__ = '2023-08-17' +__date__ = '2023-08-18' __version__ = '0.46.0' __all__ = [ diff --git a/extract_msg/msg_classes/contact.py b/extract_msg/msg_classes/contact.py index 89a48664..934d417a 100644 --- a/extract_msg/msg_classes/contact.py +++ b/extract_msg/msg_classes/contact.py @@ -3,9 +3,10 @@ ] -import io import datetime import functools +import io +import json from typing import Dict, List, Optional, Set, Tuple, Union @@ -16,8 +17,8 @@ ) from ..exceptions import SecurityError from .message_base import MessageBase -from ..structures.entry_id import EntryID from ..structures.business_card import BusinessCardDisplayDefinition +from ..structures.entry_id import EntryID class Contact(MessageBase): @@ -25,6 +26,11 @@ class Contact(MessageBase): Class used for parsing contacts. """ + def getJson(self) -> str: + # To save a lot of trouble and repetiion, just return a JSON version of + # the header format properties. + return json.dumps(self.headerFormatProperties) + @functools.cached_property def account(self) -> Optional[str]: """ From 8d1a7eede1b1bca156049914f9a2a1116964c81f Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Fri, 18 Aug 2023 16:39:38 -0700 Subject: [PATCH 21/68] Fixed typos in changelog --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dafd1035..2de98447 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,9 @@ * Added a number of properties to `MSGFile` from \[MS-OXCMSG\]. * Moved some properties down to `MessageBase` from it's subclasses. * Added support for Journal objects. -* Changed internal code of `PermanentEntryID` to correctly parse the data. Previously the distinguished name did not actually end at the null character, instead ending at the end of the bytes provided. If there was trailing data, it would be captured inadvertantly. +* Changed internal code of `PermanentEntryID` to correctly parse the data. Previously the distinguished name did not actually end at the null character, instead ending at the end of the bytes provided. If there was trailing data, it would be captured inadvertently. * Finished definition for `StoreObjectEntryID`. -* Added new kwargs for MSG files: `dateFormat` and `datetimeFormat`. These allow the user to easily override the strings being used for format dates and dates that include a time component, respectively. +* Added new keyword arguments for MSG files: `dateFormat` and `datetimeFormat`. These allow the user to easily override the strings being used for format dates and dates that include a time component, respectively. * In unifying all the formats into 2 options, you may notice that some will look a bit different starting from this version, as there was an unfortunately large amount of variation. * Fixed code for `MessageBase.parsedDate` which could have incorrect values. * Fixed issues with `MessageBase.date` and related things either being incorrectly documented or doing things that are not specified by the documentation. It was *supposed* to have been changed to use `datetime` objects, but it was still using strings. @@ -17,8 +17,8 @@ * Added new custom attachment handler for journal-associated attachments. * Changed `EntryID.autoCreate` to return `None` if given `None` or empty bytes. * Changed `EntryID.autoCreate` to raise a `FeatureNotImplemented` exception if no valid entry ID class is found. -* Fix typing annocations for `CustomAttachmentHandler`. -* Removed unneed `imapclient` dependency. +* Fix typing annotations for `CustomAttachmentHandler`. +* Removed unneeded `imapclient` dependency. * Changed `getJson` to have values be null if they aren't found rather than an empty string. * Implemented the `getJson` method correctly for a number of classes. * Changed `Task.percentComplete` to always return a float. From 2fd2eb8f1edd99289df444068e13269936dcfe27 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Fri, 18 Aug 2023 22:14:14 -0700 Subject: [PATCH 22/68] Numerous fixes and typing updates --- CHANGELOG.md | 5 ++ extract_msg/attachments/__init__.py | 4 +- extract_msg/attachments/attachment_base.py | 43 +++++++------ .../custom_att_handler/__init__.py | 8 +-- .../custom_att_handler/custom_handler.py | 7 ++- .../custom_att_handler/jrnl_assoc_att.py | 7 +++ .../custom_att_handler/outlook_image_dib.py | 8 +-- extract_msg/constants/__init__.py | 6 +- extract_msg/constants/st.py | 2 +- extract_msg/enums.py | 8 +-- extract_msg/msg_classes/appointment.py | 2 +- extract_msg/msg_classes/calendar_base.py | 7 ++- extract_msg/msg_classes/message_base.py | 22 +++++-- extract_msg/msg_classes/msg.py | 63 ++++++++++--------- extract_msg/properties/named.py | 11 ++-- extract_msg/recipient.py | 37 +++++------ extract_msg/structures/_helpers.py | 21 ++++--- extract_msg/structures/entry_id.py | 28 ++++----- extract_msg/utils.py | 30 ++++----- 19 files changed, 179 insertions(+), 140 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2de98447..5a52bf52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,11 @@ * Changed `getJson` to have values be null if they aren't found rather than an empty string. * Implemented the `getJson` method correctly for a number of classes. * Changed `Task.percentComplete` to always return a float. +* Changed the `NotImplementedError` for custom attachment handler not being found to `FeatureNotImplemented`. Additionally, changed the error message to specify the CLSID found on the attachment to better enable people to report issues. +* Changed code for `Recipient` and `MessageBase` that makes it rely on `MessageBase.recipientTypeClass` to determine the class to use for the `recipientType` property. Adjusted the typing of `Recipient` to have it reflect the type that will be used. +* Correctly changed the returned value for `ResponseStatus.fromIter` to actually return a List instead of a set. +* Filled out typing information for a significant portion of the module where variables or functions were missing it. +* Corrected a number of minor issues. **v0.45.0** * BREAKING: Changed parsing of string multiple properties to remove the trailing null byte. This *will* cause the output of parsing them to differ. diff --git a/extract_msg/attachments/__init__.py b/extract_msg/attachments/__init__.py index 29cceaf0..81da157d 100644 --- a/extract_msg/attachments/__init__.py +++ b/extract_msg/attachments/__init__.py @@ -97,7 +97,7 @@ def initStandardAttachment(msg : MSGFile, dir_ : str) -> AttachmentBase: propStore._propDict['37050003'] = createProp(propData) - attMethod = propStore['37050003'].value & 7 + attMethod = propStore.getValue('37050003', 0) & 7 if msg.exists([dir_, '__substg1.0_37010102']): return Attachment(msg, dir_, propStore) @@ -139,6 +139,6 @@ def initStandardAttachment(msg : MSGFile, dir_ : str) -> AttachmentBase: except Exception: if ErrorBehavior.ATTACH_BROKEN in msg.errorBehavior: _logger.exception(f'Error processing attachment at {dir_}') - return BrokenAttachment(msg, dir_) + return BrokenAttachment(msg, dir_, propStore) else: raise diff --git a/extract_msg/attachments/attachment_base.py b/extract_msg/attachments/attachment_base.py index 6eb7fa3c..10609314 100644 --- a/extract_msg/attachments/attachment_base.py +++ b/extract_msg/attachments/attachment_base.py @@ -20,14 +20,13 @@ Union ) -from .. import constants +from ..constants import MSG_PATH, SAVE_TYPE from ..enums import AttachmentType from ..properties.named import NamedProperties from ..properties.prop import FixedLengthProp from ..properties.properties_store import PropertiesStore from ..utils import ( - makeWeakRef, msgPathToString, tryGetMimetype, verifyPropertyId, - verifyType + msgPathToString, tryGetMimetype, verifyPropertyId, verifyType ) @@ -53,13 +52,13 @@ def __init__(self, msg : MSGFile, dir_ : str, propStore : PropertiesStore): :param propStore: The PropertiesStore instance for the attachment. If not provided, it will be found automatically. """ - self.__msg = makeWeakRef(msg) + self.__msg = weakref.ref(msg) self.__dir = dir_ self.__props = propStore self.__namedProperties = NamedProperties(msg.named, self) - self.__treePath = msg.treePath + [makeWeakRef(self)] + self.__treePath = msg.treePath + [weakref.ref(self)] - def _getStream(self, filename) -> Optional[bytes]: + def _getStream(self, filename : MSG_PATH) -> Optional[bytes]: """ Gets a binary representation of the requested filename. @@ -73,7 +72,7 @@ def _getStream(self, filename) -> Optional[bytes]: warnings.warn(':method _getStream: has been deprecated and moved to the public api. Use :method getStream: instead (remove the underscore).', DeprecationWarning) return self.getStream(filename) - def _getStringStream(self, filename) -> Optional[str]: + def _getStringStream(self, filename : MSG_PATH) -> Optional[str]: """ Gets a string representation of the requested filename. Checks for both ASCII and Unicode representations and returns @@ -156,7 +155,7 @@ def _getTypedProperty(self, propertyID, _type = None) -> Tuple[bool, Optional[ob return True, ret - def _getTypedStream(self, filename, _type = None): + def _getTypedStream(self, filename : MSG_PATH, _type = None): """ Gets the contents of the specified stream as the type that it is supposed to be. @@ -226,7 +225,7 @@ def _handleFnc(self, _zip, filename, customPath, kwargs) -> pathlib.Path: return fullFilename - def exists(self, filename) -> bool: + def exists(self, filename : MSG_PATH) -> bool: """ Checks if stream exists inside the attachment folder. @@ -237,7 +236,7 @@ def exists(self, filename) -> bool: raise ReferenceError('The msg file for this Attachment instance has been garbage collected.') return msg.exists([self.__dir, msgPathToString(filename)]) - def sExists(self, filename) -> bool: + def sExists(self, filename : MSG_PATH) -> bool: """ Checks if the string stream exists inside the attachment folder. @@ -248,7 +247,7 @@ def sExists(self, filename) -> bool: raise ReferenceError('The msg file for this Attachment instance has been garbage collected.') return msg.sExists([self.__dir, filename]) - def existsTypedProperty(self, id, _type = None) -> bool: + def existsTypedProperty(self, id : Union[int, str], _type = None) -> bool: """ Determines if the stream with the provided id exists. The return of this function is 2 values, the first being a boolean for if anything was @@ -261,7 +260,7 @@ def existsTypedProperty(self, id, _type = None) -> bool: raise ReferenceError('The msg file for this Attachment instance has been garbage collected.') return msg.existsTypedProperty(id, self.__dir, _type, True, self.__props) - def getMultipleBinary(self, filename) -> Optional[List[bytes]]: + def getMultipleBinary(self, filename : MSG_PATH) -> Optional[List[bytes]]: """ Gets a multiple binary property as a list of bytes objects. @@ -276,7 +275,7 @@ def getMultipleBinary(self, filename) -> Optional[List[bytes]]: raise ReferenceError('The msg file for this Attachment instance has been garbage collected.') return msg.getMultipleBinary([self.__dir, msgPathToString(filename)]) - def getMultipleString(self, filename) -> Optional[List[str]]: + def getMultipleString(self, filename : MSG_PATH) -> Optional[List[str]]: """ Gets a multiple string property as a list of str objects. @@ -314,7 +313,7 @@ def getNamedProp(self, propertyName : str, guid : str, default : _T = None) -> U """ return self.namedProperties.get((propertyName, guid), default) - def getPropertyAs(self, propertyName, overrideClass : Callable[[Any], _T]) -> Optional[_T]: + def getPropertyAs(self, propertyName : Union[int, str], overrideClass : Callable[[Any], _T]) -> Optional[_T]: """ Returns the property, setting the class if found. @@ -331,7 +330,7 @@ def getPropertyAs(self, propertyName, overrideClass : Callable[[Any], _T]) -> Op return value - def getPropertyVal(self, name, default : _T = None) -> Union[Any, _T]: + def getPropertyVal(self, name : Union[int, str], default : _T = None) -> Union[Any, _T]: """ instance.props.getValue(name, default) @@ -339,7 +338,7 @@ def getPropertyVal(self, name, default : _T = None) -> Union[Any, _T]: """ return self.props.getValue(name, default) - def getSingleOrMultipleBinary(self, filename) -> Optional[Union[List[bytes], bytes]]: + def getSingleOrMultipleBinary(self, filename : MSG_PATH) -> Optional[Union[List[bytes], bytes]]: """ A combination of :method getStringStream: and :method getMultipleString:. @@ -358,7 +357,7 @@ def getSingleOrMultipleBinary(self, filename) -> Optional[Union[List[bytes], byt raise ReferenceError('The msg file for this Attachment instance has been garbage collected.') return msg.getSingleOrMultipleBinary([self.__dir, msgPathToString(filename)]) - def getSingleOrMultipleString(self, filename) -> Optional[Union[List[str], str]]: + def getSingleOrMultipleString(self, filename : MSG_PATH) -> Optional[Union[List[str], str]]: """ A combination of :method getStringStream: and :method getMultipleString:. @@ -377,7 +376,7 @@ def getSingleOrMultipleString(self, filename) -> Optional[Union[List[str], str]] raise ReferenceError('The msg file for this Attachment instance has been garbage collected.') return msg.getSingleOrMultipleString([self.__dir, msgPathToString(filename)]) - def getStream(self, filename) -> Optional[bytes]: + def getStream(self, filename : MSG_PATH) -> Optional[bytes]: """ Gets a binary representation of the requested filename. @@ -391,7 +390,7 @@ def getStream(self, filename) -> Optional[bytes]: raise ReferenceError('The msg file for this Attachment instance has been garbage collected.') return msg.getStream([self.__dir, msgPathToString(filename)]) - def getStreamAs(self, streamID, overrideClass : Callable[[Any], _T]) -> Optional[_T]: + def getStreamAs(self, streamID : MSG_PATH, overrideClass : Callable[[Any], _T]) -> Optional[_T]: """ Returns the specified stream, modifying it to the specified class if it is found. @@ -409,7 +408,7 @@ def getStreamAs(self, streamID, overrideClass : Callable[[Any], _T]) -> Optional return value - def getStringStream(self, filename) -> Optional[str]: + def getStringStream(self, filename : MSG_PATH) -> Optional[str]: """ Gets a string representation of the requested filename. Checks for both ASCII and Unicode representations and returns @@ -424,7 +423,7 @@ def getStringStream(self, filename) -> Optional[str]: raise ReferenceError('The msg file for this Attachment instance has been garbage collected.') return msg.getStringStream([self.__dir, msgPathToString(filename)]) - def getStringStreamAs(self, streamID, overrideClass : Callable[[Any], _T]) -> Optional[_T]: + def getStringStreamAs(self, streamID : MSG_PATH, overrideClass : Callable[[Any], _T]) -> Optional[_T]: """ Returns the specified string stream, modifying it to the specified class if it is found. @@ -455,7 +454,7 @@ def getFilename(self, **kwargs) -> str: """ @abc.abstractmethod - def save(self, **kwargs) -> constants.SAVE_TYPE: + def save(self, **kwargs) -> SAVE_TYPE: """ Saves the attachment data. diff --git a/extract_msg/attachments/custom_att_handler/__init__.py b/extract_msg/attachments/custom_att_handler/__init__.py index 3e4feff3..b53d7c25 100644 --- a/extract_msg/attachments/custom_att_handler/__init__.py +++ b/extract_msg/attachments/custom_att_handler/__init__.py @@ -33,10 +33,11 @@ from typing import List, Type, TYPE_CHECKING from .custom_handler import CustomAttachmentHandler +from ...exceptions import FeatureNotImplemented # Create a way to register handlers. -_knownHandlers : List[CustomAttachmentHandler] = [] +_knownHandlers : List[Type[CustomAttachmentHandler]] = [] def registerHandler(handler : Type[CustomAttachmentHandler]) -> None: """ @@ -48,12 +49,11 @@ def registerHandler(handler : Type[CustomAttachmentHandler]) -> None: # Make sure it is a subclass of CustomAttachmentHandler. if not isinstance(handler, type): raise ValueError(':param handler: must be a class, not an instance of a class.') - if not issubclass(handler, CustomAttachmentHandler): + if not issubclass(handler, CustomAttachmentHandler): # pyright: ignore raise ValueError(':param handler: must be a subclass of CustomAttachmentHandler.') _knownHandlers.append(handler) - # Import built-in handler modules. They will all automatically register their # respecive handler(s). from .outlook_image_dib import OutlookImageDIB @@ -78,4 +78,4 @@ def getHandler(attachment : AttachmentBase) -> CustomAttachmentHandler: if handler.isCorrectHandler(attachment): return handler(attachment) - raise NotImplementedError('No valid handler could be found for the attachment. Contact the developers for help.') + raise FeatureNotImplemented(f'No valid handler could be found for the attachment. Contact the developers for help. If the CLSID is not all zeros, include it in the title or message. (CLSID: {attachment.clsid})') diff --git a/extract_msg/attachments/custom_att_handler/custom_handler.py b/extract_msg/attachments/custom_att_handler/custom_handler.py index 43e2b1b0..d1aa601f 100644 --- a/extract_msg/attachments/custom_att_handler/custom_handler.py +++ b/extract_msg/attachments/custom_att_handler/custom_handler.py @@ -10,6 +10,7 @@ from typing import Any, Callable, Optional, TYPE_CHECKING, TypeVar +from ...constants import MSG_PATH from ...utils import msgPathToString @@ -29,13 +30,13 @@ def __init__(self, attachment : AttachmentBase): super().__init__() self.__att = attachment - def getStream(self, path) -> Optional[bytes]: + def getStream(self, path : MSG_PATH) -> Optional[bytes]: """ Gets a stream from the custom data directory. """ return self.attachment.getStream('__substg1.0_3701000D/' + msgPathToString(path)) - def getStreamAs(self, streamID, overrideClass : Callable[[Any], _T]) -> Optional[_T]: + def getStreamAs(self, streamID : MSG_PATH, overrideClass : Callable[[Any], _T]) -> Optional[_T]: """ Returns the specified stream, modifying it to the specified class if it is found. @@ -63,7 +64,7 @@ def isCorrectHandler(cls, attachment : AttachmentBase) -> bool: @abc.abstractmethod def generateRtf(self) -> Optional[bytes]: """ - Generates the RTF to inject in place of the \objattph tag. + Generates the RTF to inject in place of the \\objattph tag. If this function should do nothing, returns None. """ diff --git a/extract_msg/attachments/custom_att_handler/jrnl_assoc_att.py b/extract_msg/attachments/custom_att_handler/jrnl_assoc_att.py index fc2d1b2e..c736f6f3 100644 --- a/extract_msg/attachments/custom_att_handler/jrnl_assoc_att.py +++ b/extract_msg/attachments/custom_att_handler/jrnl_assoc_att.py @@ -21,10 +21,17 @@ class JournalAssociatedAttachment(CustomAttachmentHandler): def __init__(self, attachment : AttachmentBase): super().__init__(attachment) + stream = attachment.getStream('__substg1.0_3701000D/\x03MailStream') + if not stream: + raise ValueError('MailStream could not be found.') + if len(stream) != 12: + raise ValueError('MailStream is the wrong length.') @classmethod def isCorrectHandler(cls, attachment : AttachmentBase) -> bool: # This only applies to journal objects. + if not attachment.msg.classType: + return False if not attachment.msg.classType.lower().startswith('ipm.activity'): return False if attachment.clsid != '00020D09-0000-0000-C000-000000000046': diff --git a/extract_msg/attachments/custom_att_handler/outlook_image_dib.py b/extract_msg/attachments/custom_att_handler/outlook_image_dib.py index 37c2c57f..12fa9e6d 100644 --- a/extract_msg/attachments/custom_att_handler/outlook_image_dib.py +++ b/extract_msg/attachments/custom_att_handler/outlook_image_dib.py @@ -32,17 +32,17 @@ class OutlookImageDIB(CustomAttachmentHandler): def __init__(self, attachment : AttachmentBase): super().__init__(attachment) # First we need to get the mailstream. - stream = attachment.getStream('__substg1.0_3701000D/\x03MailStream') + stream = self.getStream('\x03MailStream') if not stream: raise ValueError('MailStream could not be found.') if len(stream) != 12: raise ValueError('MailStream is the wrong length.') # Next get the bitmap data. - self.__data = attachment.getStream('__substg1.0_3701000D/CONTENTS') + self.__data = self.getStream('CONTENTS') if not self.__data: raise ValueError('Bitmap data could not be read for Outlook signature.') # Get the OLE data. - oleStream = attachment.getStream('__substg1.0_3701000D/\x01Ole') + oleStream = self.getStream('\x01Ole') if not oleStream: raise ValueError('OLE stream could not be found.') @@ -87,7 +87,7 @@ def isCorrectHandler(cls, attachment : AttachmentBase) -> bool: def generateRtf(self) -> Optional[bytes]: """ - Generates the RTF to inject in place of the \objattph tag. + Generates the RTF to inject in place of the \\objattph tag. If this function should do nothing, returns None. diff --git a/extract_msg/constants/__init__.py b/extract_msg/constants/__init__.py index bc695433..34fd6681 100644 --- a/extract_msg/constants/__init__.py +++ b/extract_msg/constants/__init__.py @@ -22,6 +22,7 @@ 'KNOWN_CLASS_TYPES', 'KNOWN_FILE_FLAGS', 'MAINDOC', + 'MSG_PATH', 'MULTIPLE_16_BYTES', 'MULTIPLE_16_BYTES_HEX', 'MULTIPLE_2_BYTES', @@ -44,7 +45,7 @@ import datetime -from typing import Dict, List, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union from . import ps, re, st from ..enums import SaveType @@ -58,8 +59,9 @@ # Typing Constants. HEADER_FORMAT_VALUE_TYPE = Union[str, Tuple[Union[str, None], bool], None] # Basically a dict of HEADER_FORMAT_TYPE and dicts containing them. -HEADER_FORMAT_TYPE = Dict[str, Union[HEADER_FORMAT_VALUE_TYPE, Dict[str, HEADER_FORMAT_VALUE_TYPE]]] +HEADER_FORMAT_TYPE = Optional[Dict[str, Union[HEADER_FORMAT_VALUE_TYPE, Dict[str, HEADER_FORMAT_VALUE_TYPE]]]] SAVE_TYPE = Tuple[SaveType, Union[List[str], str, None]] +MSG_PATH = Union[str, List[str], Tuple[str]] diff --git a/extract_msg/constants/st.py b/extract_msg/constants/st.py index 7de1534e..64d68353 100644 --- a/extract_msg/constants/st.py +++ b/extract_msg/constants/st.py @@ -3,7 +3,7 @@ """ __all__ = [ - 'ST1' + 'ST1', 'ST2', 'ST3', 'STF32', diff --git a/extract_msg/enums.py b/extract_msg/enums.py index 03c734a8..8b500691 100644 --- a/extract_msg/enums.py +++ b/extract_msg/enums.py @@ -79,7 +79,7 @@ import enum -from typing import Dict, Iterable, List, Set, Union +from typing import Dict, Iterable, List, Set, Type, Union class AddressBookType(enum.IntEnum): @@ -1450,7 +1450,7 @@ class RecurDOW(enum.IntEnum): class RecurEndType(enum.IntEnum): @classmethod - def fromInt(cls, value) -> RecurEndType: + def fromInt(cls, value : int) -> RecurEndType: """ Some enum values CAN be created from more than one int, so handle that. """ @@ -1527,7 +1527,7 @@ def fromIter(cls, items : Iterable[int]) -> List[ResponseStatus]: """ Uses the iterable of ints to create a list of this enum. """ - return {cls(x) for x in items} + return [cls(x) for x in items] NONE = 0x00000000 ORGANIZED = 0x00000001 @@ -1828,7 +1828,7 @@ class _EnumDeprecator: Special class for handling deprecated enums in a way that shouldn't break existing code, including code for checking `is` on a member of the enum. """ - def __init__(self, oldClassName : str, newClass : enum.Enum, nameConversion : Dict = {}, valueConversion : Dict = {}): + def __init__(self, oldClassName : str, newClass : Type[enum.Enum], nameConversion : Dict[str, Any] = {}, valueConversion : Dict = {}): """ :param oldClassName: The name to use in the deprecation message. :param newClass: The new enum class to look for the value in. diff --git a/extract_msg/msg_classes/appointment.py b/extract_msg/msg_classes/appointment.py index 4b6317c0..283eeb29 100644 --- a/extract_msg/msg_classes/appointment.py +++ b/extract_msg/msg_classes/appointment.py @@ -184,7 +184,7 @@ def isMeeting(self) -> bool: Attempts to determine if the object is a Meeting. True if meeting, False if appointment. """ - return self.appointmentStateFlags and AppointmentStateFlag.MEETING in self.appointmentStateFlags + return bool(self.appointmentStateFlags) and (AppointmentStateFlag.MEETING in self.appointmentStateFlags) @functools.cached_property def originalStoreEntryID(self) -> Optional[EntryID]: diff --git a/extract_msg/msg_classes/calendar_base.py b/extract_msg/msg_classes/calendar_base.py index 99f4f6dc..a800935d 100644 --- a/extract_msg/msg_classes/calendar_base.py +++ b/extract_msg/msg_classes/calendar_base.py @@ -4,10 +4,11 @@ import datetime +import enum import functools import logging -from typing import List, Optional, Union +from typing import List, Optional, Type, Union from ..constants import ps from ..enums import AppointmentAuxilaryFlag, AppointmentColor, AppointmentStateFlag, BusyStatus, IconIndex, MeetingRecipientType, ResponseStatus @@ -396,6 +397,10 @@ def ownerCriticalChange(self) -> Optional[datetime.datetime]: """ return self.getNamedProp('001A', ps.PSETID_MEETING) + @property + def recipientTypeClass(self) -> Type[enum.IntEnum]: + return MeetingRecipientType + @functools.cached_property def recurrencePattern(self) -> Optional[str]: """ diff --git a/extract_msg/msg_classes/message_base.py b/extract_msg/msg_classes/message_base.py index 12ee047c..2b7b093a 100644 --- a/extract_msg/msg_classes/message_base.py +++ b/extract_msg/msg_classes/message_base.py @@ -7,6 +7,7 @@ import datetime import email.message import email.utils +import enum import functools import html import json @@ -25,7 +26,7 @@ from email import policy from email.message import EmailMessage from email.parser import HeaderParser -from typing import Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, cast, Dict, List, Optional, Tuple, Type, Union from .. import constants from .._rtf.create_doc import createDocument @@ -80,7 +81,7 @@ def __init__(self, path, **kwargs): # if an error occurs. try: self.__headerInit = False - self.__recipientSeparator = kwargs.get('recipientSeparator', ';') + self.__recipientSeparator : str = kwargs.get('recipientSeparator', ';') self.__deencap = kwargs.get('deencapsulationFunc') # Initialize properties in the order that is least likely to cause bugs. # TODO have each function check for initialization of needed data so @@ -116,7 +117,7 @@ def _genRecipient(self, recipientStr : str, recipientType : RecipientType) -> Op value = None # Check header first. if self.headerInit: - value = self.header[recipientStr] + value = cast(Optional[str], self.header[recipientStr]) if value: value = decodeRfc2047(value) value = value.replace(',', self.__recipientSeparator) @@ -1058,11 +1059,11 @@ def header(self) -> email.message.Message: return header @functools.cached_property - def headerDict(self) -> Dict: + def headerDict(self) -> Dict[str, Any]: """ Returns a dictionary of the entries in the header """ - headerDict = dict(self.header._headers) + headerDict = {x: self.header[x] for x in self.header} try: headerDict.pop('Received') except KeyError: @@ -1254,6 +1255,17 @@ def recipients(self) -> List[Recipient]: return [Recipient(recipientDir, self) for recipientDir in recipientDirs] + @property + def recipientTypeClass() -> Type[enum.IntEnum]: + """ + The class to use for a recipient's recipientType property. + + The default is extract_msg.enums.RecipientType. If a subclass + attributes different meanings to the values, you can override this + property to return a valid enum. + """ + return RecipientType + @functools.cached_property def reportTag(self) -> Optional[ReportTag]: """ diff --git a/extract_msg/msg_classes/msg.py b/extract_msg/msg_classes/msg.py index 47732461..f33305b1 100644 --- a/extract_msg/msg_classes/msg.py +++ b/extract_msg/msg_classes/msg.py @@ -24,6 +24,7 @@ ) from .. import constants +from ..constants import DATE_FORMAT, DT_FORMAT, MSG_PATH, ps, SAVE_TYPE from ..attachments import ( AttachmentBase, initStandardAttachment, SignedAttachment ) @@ -104,7 +105,7 @@ def __init__(self, path, **kwargs): """ # Retrieve all the kwargs that we need. self.__inscFeat = kwargs.get('insecureFeatures', InsecureFeatures.NONE) - prefix = kwargs.get('prefix', '') + prefix = cast(str, kwargs.get('prefix', '')) self.__parentMsg = makeWeakRef(cast(MSGFile, kwargs.get('parentMsg'))) self.__treePath = kwargs.get('treePath', []) + [makeWeakRef(self)] # Verify it is a valid class. @@ -119,8 +120,8 @@ def __init__(self, path, **kwargs): self.__attachmentsDelayed = kwargs.get('delayAttachments', False) self.__attachmentsReady = False self.__errorBehavior = ErrorBehavior(kwargs.get('errorBehavior', ErrorBehavior.THROW)) - self.__dateFormat = kwargs.get('dateFormat', constants.DATE_FORMAT) - self.__dtFormat = kwargs.get('datetimeFormat', constants.DT_FORMAT) + self.__dateFormat = kwargs.get('dateFormat', DATE_FORMAT) + self.__dtFormat = kwargs.get('datetimeFormat', DT_FORMAT) if overrideEncoding is not None: codecs.lookup(overrideEncoding) @@ -214,7 +215,7 @@ def __enter__(self) -> MSGFile: def __exit__(self, *_) -> None: self.close() - def _getOleEntry(self, filename, prefix : bool = True) -> olefile.olefile.OleDirectoryEntry: + def _getOleEntry(self, filename : MSG_PATH, prefix : bool = True) -> olefile.olefile.OleDirectoryEntry: """ Finds the directory entry from the olefile for the stream or storage specified. Use '/' to get the root entry. @@ -230,7 +231,7 @@ def _getOleEntry(self, filename, prefix : bool = True) -> olefile.olefile.OleDir return self.__ole.direntries[sid] - def _getStream(self, filename, prefix : bool = True) -> Optional[bytes]: + def _getStream(self, filename : MSG_PATH, prefix : bool = True) -> Optional[bytes]: """ Gets a binary representation of the requested filename. @@ -241,7 +242,7 @@ def _getStream(self, filename, prefix : bool = True) -> Optional[bytes]: warnings.warn(':method _getStream: has been deprecated and moved to the public api. Use :method getStream: instead (remove the underscore).', DeprecationWarning) return self.getStream(filename, prefix) - def _getStringStream(self, filename, prefix : bool = True) -> Optional[str]: + def _getStringStream(self, filename : MSG_PATH, prefix : bool = True) -> Optional[str]: """ Gets a string representation of the requested filename. @@ -317,7 +318,7 @@ def _getTypedProperty(self, propertyID : str, _type = None) -> Tuple[bool, Optio return True, ret - def _getTypedStream(self, filename, prefix : bool = True, _type = None) -> Tuple[bool, Optional[Any]]: + def _getTypedStream(self, filename : MSG_PATH, prefix : bool = True, _type = None) -> Tuple[bool, Optional[Any]]: """ Gets the contents of the specified stream as the type that it is supposed to be. @@ -353,7 +354,7 @@ def _getTypedStream(self, filename, prefix : bool = True, _type = None) -> Tuple elif _type in ('1002', '1003', '1004', '1005', '1007', '1014', '1040', '1048'): try: streams = self.props[x[-8:]].realLength - except Exception: + except (KeyError, AttributeError): logger.error(f'Could not find matching VariableLengthProp for stream {x}') streams = len(contents) // (2 if _type in constants.MULTIPLE_2_BYTES else 4 if _type in constants.MULTIPLE_4_BYTES else 8 if _type in constants.MULTIPLE_8_BYTES else 16) else: @@ -398,21 +399,21 @@ def debug(self) -> None: print('Directory: ' + str(dir_[:-1])) print(f'Contents: {self.getStream(dir_)}') - def exists(self, inp, prefix : bool = True) -> bool: + def exists(self, inp : MSG_PATH, prefix : bool = True) -> bool: """ Checks if :param inp: exists in the msg file. """ inp = self.fixPath(inp, prefix) return self.__ole.exists(inp) - def sExists(self, inp, prefix : bool = True) -> bool: + def sExists(self, inp : MSG_PATH, prefix : bool = True) -> bool: """ Checks if string stream :param inp: exists in the msg file. """ inp = self.fixPath(inp, prefix) return self.exists(inp + '001F') or self.exists(inp + '001E') - def existsTypedProperty(self, _id, location = None, _type = None, prefix = True, propertiesInstance = None): + def existsTypedProperty(self, _id, location = None, _type = None, prefix : bool = True, propertiesInstance : Optional[PropertiesStore] = None) -> Tuple[bool, int]: """ Determines if the stream with the provided id exists in the location specified. If no location is specified, the root directory is searched. @@ -475,7 +476,7 @@ def exportBytes(self) -> bytes: self.export(out) return out.getvalue() - def fixPath(self, inp, prefix : bool = True) -> str: + def fixPath(self, inp : MSG_PATH, prefix : bool = True) -> str: """ Changes paths so that they have the proper prefix (should :param prefix: be True) and are strings rather than lists or tuples. @@ -485,7 +486,7 @@ def fixPath(self, inp, prefix : bool = True) -> str: inp = self.__prefix + inp return inp - def getMultipleBinary(self, filename, prefix : bool = True) -> Optional[List[bytes]]: + def getMultipleBinary(self, filename : MSG_PATH, prefix : bool = True) -> Optional[List[bytes]]: """ Gets a multiple binary property as a list of bytes objects. @@ -513,7 +514,7 @@ def getMultipleBinary(self, filename, prefix : bool = True) -> Optional[List[byt return ret[:index] return ret - def getMultipleString(self, filename, prefix : bool = True) -> Optional[List[str]]: + def getMultipleString(self, filename : MSG_PATH, prefix : bool = True) -> Optional[List[str]]: """ Gets a multiple string property as a list of str objects. @@ -567,7 +568,7 @@ def getNamedProp(self, propertyName : str, guid : str, default : _T = None) -> U """ return self.namedProperties.get((propertyName, guid), default) - def getPropertyAs(self, propertyName, overrideClass : Callable[[Any], _T]) -> Optional[_T]: + def getPropertyAs(self, propertyName : Union[int, str], overrideClass : Callable[[Any], _T]) -> Optional[_T]: """ Returns the property, setting the class if found. @@ -584,7 +585,7 @@ def getPropertyAs(self, propertyName, overrideClass : Callable[[Any], _T]) -> Op return value - def getPropertyVal(self, name, default : _T = None) -> Union[Any, _T]: + def getPropertyVal(self, name : Union[int, str], default : _T = None) -> Union[Any, _T]: """ instance.props.getValue(name, default) @@ -592,7 +593,7 @@ def getPropertyVal(self, name, default : _T = None) -> Union[Any, _T]: """ return self.props.getValue(name, default) - def getSingleOrMultipleBinary(self, filename, prefix : bool = True) -> Optional[Union[List[bytes], bytes]]: + def getSingleOrMultipleBinary(self, filename : MSG_PATH, prefix : bool = True) -> Optional[Union[List[bytes], bytes]]: """ A combination of :method getStringStream: and :method getMultipleString:. @@ -612,7 +613,7 @@ def getSingleOrMultipleBinary(self, filename, prefix : bool = True) -> Optional[ # work. return self.getMultipleBinary(filename, False) - def getSingleOrMultipleString(self, filename, prefix : bool = True) -> Optional[Union[List[str], str]]: + def getSingleOrMultipleString(self, filename : MSG_PATH, prefix : bool = True) -> Optional[Union[List[str], str]]: """ A combination of :method getStringStream: and :method getMultipleString:. @@ -632,7 +633,7 @@ def getSingleOrMultipleString(self, filename, prefix : bool = True) -> Optional[ # work. return self.getMultipleString(filename, False) - def getStream(self, filename, prefix : bool = True) -> Optional[bytes]: + def getStream(self, filename : MSG_PATH, prefix : bool = True) -> Optional[bytes]: """ Gets a binary representation of the requested filename. @@ -668,7 +669,7 @@ def getStreamAs(self, streamID, overrideClass : Callable[[Any], _T]) -> Optional return value - def getStringStream(self, filename, prefix : bool = True) -> Optional[str]: + def getStringStream(self, filename : MSG_PATH, prefix : bool = True) -> Optional[str]: """ Gets a string representation of the requested filename. @@ -741,7 +742,7 @@ def slistDir(self, streams : bool = True, storages : bool = False, includePrefix """ return [msgPathToString(x) for x in self.listDir(streams, storages, includePrefix)] - def save(self, **kwargs) -> constants.SAVE_TYPE: + def save(self, **kwargs) -> SAVE_TYPE: if kwargs.get('skipNotImplemented', False): return (SaveType.NONE, None) @@ -837,7 +838,7 @@ def classified(self) -> bool: Indicates whether the contents of this message are regarded as classified information. """ - return bool(self.getNamedProp('85B5', constants.ps.PSETID_COMMON)) + return bool(self.getNamedProp('85B5', ps.PSETID_COMMON)) @functools.cached_property def classType(self) -> Optional[str]: @@ -851,14 +852,14 @@ def commonEnd(self) -> Optional[datetime.datetime]: """ The end time for the object. """ - return self.getNamedProp('8517', constants.ps.PSETID_COMMON) + return self.getNamedProp('8517', ps.PSETID_COMMON) @functools.cached_property def commonStart(self) -> Optional[datetime.datetime]: """ The start time for the object. """ - return self.getNamedProp('8516', constants.ps.PSETID_COMMON) + return self.getNamedProp('8516', ps.PSETID_COMMON) @functools.cached_property def contactLinkEntry(self) -> Optional[ContactLinkEntry]: @@ -866,7 +867,7 @@ def contactLinkEntry(self) -> Optional[ContactLinkEntry]: Returns a class that contains the list of Address Book EntryIDs linked to this Message object. """ - return self.getNamedAs('8585', constants.ps.PSETID_COMMON, ContactLinkEntry) + return self.getNamedAs('8585', ps.PSETID_COMMON, ContactLinkEntry) @functools.cached_property def contacts(self) -> Optional[List[str]]: @@ -874,7 +875,7 @@ def contacts(self) -> Optional[List[str]]: Contains the display name property of each Address Book EntryID referenced in the value of the contactLinkEntry property. """ - return self.getNamedProp('853A', constants.ps.PSETID_COMMON) + return self.getNamedProp('853A', ps.PSETID_COMMON) @functools.cached_property def currentVersion(self) -> Optional[int]: @@ -882,14 +883,14 @@ def currentVersion(self) -> Optional[int]: Specifies the build number of the client application that sent the message. """ - return self.getNamedProp('8552', constants.ps.PSETID_COMMON) + return self.getNamedProp('8552', ps.PSETID_COMMON) @functools.cached_property def currentVersionName(self) -> Optional[str]: """ Specifies the name of the client application that sent the message. """ - return self.getNamedProp('8554', constants.ps.PSETID_COMMON) + return self.getNamedProp('8554', ps.PSETID_COMMON) @property def dateFormat(self) -> str: @@ -1003,7 +1004,7 @@ def path(self): return self.__path @property - def prefix(self): + def prefix(self) -> str: """ Returns the prefix of the Message instance. Intended for developer use. """ @@ -1075,10 +1076,10 @@ def sideEffects(self) -> Optional[SideEffect]: Controls how a Message object is handled by the client in relation to certain user interface actions by the user, such as deleting a message. """ - return self.getNamedAs('8510', constants.ps.PSETID_COMMON, SideEffect) + return self.getNamedAs('8510', ps.PSETID_COMMON, SideEffect) @property - def stringEncoding(self): + def stringEncoding(self) -> str: try: return self.__stringEncoding except AttributeError: diff --git a/extract_msg/properties/named.py b/extract_msg/properties/named.py index 490e52ff..ddf11b0f 100644 --- a/extract_msg/properties/named.py +++ b/extract_msg/properties/named.py @@ -14,6 +14,7 @@ import copy import logging import pprint +import weakref from typing import ( Any, Dict, Iterable, Iterator, List, Optional, Tuple, TYPE_CHECKING, @@ -22,7 +23,7 @@ from .. import constants from ..enums import NamedPropertyType -from ..utils import bytesToGuid, divide, makeWeakRef, msgPathToString +from ..utils import bytesToGuid, divide, msgPathToString from compressed_rtf.crc32 import crc32 @@ -45,7 +46,7 @@ class Named: __dir = '__nameid_version1.0' def __init__(self, msg : MSGFile): - self.__msg = makeWeakRef(msg) + self.__msg = weakref.ref(msg) # Get the basic streams. If all are emtpy, then nothing to do. guidStream = self.getStream('__substg1.0_00020102') entryStream = self.getStream('__substg1.0_00030102') @@ -129,7 +130,7 @@ def __getName(self, offset : int) -> str: return self.namesStream[offset:offset + length].decode('utf-16-le') - def _getStream(self, filename) -> Optional[bytes]: + def _getStream(self, filename : constants.MSG_PATH) -> Optional[bytes]: """ Gets a binary representation of the requested filename. @@ -143,7 +144,7 @@ def _getStream(self, filename) -> Optional[bytes]: warnings.warn(':method _getStream: has been deprecated and moved to the public api. Use :method getStream: instead (remove the underscore).', DeprecationWarning) return self.getStream(filename) - def exists(self, filename) -> bool: + def exists(self, filename : constants.MSG_PATH) -> bool: """ Checks if stream exists inside the named properties folder. @@ -164,7 +165,7 @@ def get(self, propertyName : Tuple[str, str], default : _T = None) -> Union[Name except KeyError: return default - def getStream(self, filename) -> Optional[bytes]: + def getStream(self, filename : constants.MSG_PATH) -> Optional[bytes]: """ Gets a binary representation of the requested filename. diff --git a/extract_msg/recipient.py b/extract_msg/recipient.py index 1050b13d..2ad40a09 100644 --- a/extract_msg/recipient.py +++ b/extract_msg/recipient.py @@ -6,19 +6,23 @@ ] +import enum import functools import logging +import weakref from typing import ( - Any, Callable, List, Optional, Tuple, TYPE_CHECKING, TypeVar, Union + Any, Callable, Generic, List, Optional, Tuple, TYPE_CHECKING, Type, + TypeVar, Union ) -from .enums import ErrorBehavior, MeetingRecipientType, PropertiesType, RecipientType +from .constants import MSG_PATH +from .enums import ErrorBehavior, PropertiesType from .exceptions import StandardViolationError from .properties.prop import FixedLengthProp from .properties.properties_store import PropertiesStore from .structures.entry_id import PermanentEntryID -from .utils import makeWeakRef, msgPathToString, verifyPropertyId, verifyType +from .utils import msgPathToString, verifyPropertyId, verifyType if TYPE_CHECKING: @@ -28,15 +32,16 @@ logger.addHandler(logging.NullHandler()) _T = TypeVar('_T') +_RT = TypeVar('_RT', bound = enum.IntEnum) -class Recipient: +class Recipient(Generic[_RT]): """ Contains the data of one of the recipients in an MSG file. """ - def __init__(self, _dir, msg : MSGFile): - self.__msg = makeWeakRef(msg) # Allows calls to original msg file. + def __init__(self, _dir : str, msg : MSGFile, recipientTypeClass : Type[_RT]): + self.__msg = weakref.ref(msg) # Allows calls to original msg file. self.__dir = _dir if not self.exists('__properties_version1.0'): if ErrorBehavior.STANDARDS_VIOLATION in msg.errorBehavior: @@ -49,14 +54,10 @@ def __init__(self, _dir, msg : MSGFile): self.__email = self.getStringStream('__substg1.0_3003') self.__name = self.getStringStream('__substg1.0_3001') self.__typeFlags = self.__props.getValue('0C150003', 0) - from .msg_classes.calendar_base import CalendarBase - if isinstance(msg, CalendarBase): - self.__type = MeetingRecipientType(0xF & self.__typeFlags) - else: - self.__type = RecipientType(0xF & self.__typeFlags) + self.__type = recipientTypeClass(0xF & self.__typeFlags) self.__formatted = f'{self.__name} <{self.__email}>' - def _getStream(self, filename) -> Optional[bytes]: + def _getStream(self, filename : MSG_PATH) -> Optional[bytes]: """ Gets a binary representation of the requested filename. @@ -70,7 +71,7 @@ def _getStream(self, filename) -> Optional[bytes]: warnings.warn(':method _getStream: has been deprecated and moved to the public api. Use :method getStream: instead (remove the underscore).', DeprecationWarning) return self.getStream(filename) - def _getStringStream(self, filename) -> Optional[str]: + def _getStringStream(self, filename : MSG_PATH) -> Optional[str]: """ Gets a string representation of the requested filename. @@ -181,7 +182,7 @@ def _getTypedStream(self, filename, _type = None): raise ReferenceError('The msg file for this Recipient instance has been garbage collected.') return msg._getTypedStream(self, [self.__dir, msgPathToString(filename)], True, _type) - def exists(self, filename) -> bool: + def exists(self, filename : MSG_PATH) -> bool: """ Checks if stream exists inside the recipient folder. @@ -309,7 +310,7 @@ def getSingleOrMultipleString(self, filename) -> Optional[Union[List[str], str]] raise ReferenceError('The msg file for this Recipient instance has been garbage collected.') return msg.getSingleOrMultipleString([self.__dir, msgPathToString(filename)]) - def getStream(self, filename) -> Optional[bytes]: + def getStream(self, filename : MSG_PATH) -> Optional[bytes]: """ Gets a binary representation of the requested filename. @@ -341,7 +342,7 @@ def getStreamAs(self, streamID, overrideClass : Callable[[Any], _T]) -> Optional return value - def getStringStream(self, filename) -> Optional[str]: + def getStringStream(self, filename : MSG_PATH) -> Optional[str]: """ Gets a string representation of the requested filename. @@ -359,7 +360,7 @@ def getStringStream(self, filename) -> Optional[str]: raise ReferenceError('The msg file for this Recipient instance has been garbage collected.') return msg.getStringStream([self.__dir, msgPathToString(filename)]) - def getStringStreamAs(self, streamID, overrideClass : Callable[[Any], _T]) -> Optional[_T]: + def getStringStreamAs(self, streamID : MSG_PATH, overrideClass : Callable[[Any], _T]) -> Optional[_T]: """ Returns the specified string stream, modifying it to the specified class if it is found. @@ -455,7 +456,7 @@ def transmittableDisplayName(self) -> Optional[str]: return self.getStringStream('__substg1.0_3A20') @property - def type(self) -> Union[RecipientType, MeetingRecipientType]: + def type(self) -> _RT: """ Returns the recipient type. Type is: * Sender if `type & 0xf == 0` diff --git a/extract_msg/structures/_helpers.py b/extract_msg/structures/_helpers.py index 7e3331cd..a658298c 100644 --- a/extract_msg/structures/_helpers.py +++ b/extract_msg/structures/_helpers.py @@ -10,10 +10,12 @@ import io import struct -from typing import Any, Tuple, Union +from typing import Any, Optional, Tuple, Type, TypeVar, Union from .. import constants +_T = TypeVar('_T') + class BytesReader(io.BytesIO): """ @@ -21,7 +23,7 @@ class BytesReader(io.BytesIO): stream. """ - def __init__(self, *args, littleEndian = True, **kwargs): + def __init__(self, *args, littleEndian : bool = True, **kwargs): super().__init__(*args, **kwargs) self.__le = bool(littleEndian) if self.__le: @@ -47,7 +49,7 @@ def __init__(self, *args, littleEndian = True, **kwargs): self.__float_t = constants.st.ST_BE_F32 self.__double_t = constants.st.ST_BE_F64 - def _readDecodedString(self, encoding, width : int = 1) -> str: + def _readDecodedString(self, encoding : str, width : int = 1) -> str: """ Reads a null terminated string with the specified character width decoded using the specified encoding. If it cannot be read or cannot be @@ -61,7 +63,7 @@ def _readDecodedString(self, encoding, width : int = 1) -> str: self.seek(position) raise - def assertNull(self, length : int, errorMsg : str = None) -> bytes: + def assertNull(self, length : int, errorMsg : Optional[str] = None) -> bytes: """ Reads the number of bytes specified and ensures they are all null. @@ -91,7 +93,7 @@ def assertNull(self, length : int, errorMsg : str = None) -> bytes: return valueRead - def assertRead(self, value : bytes, errorMsg : str = None) -> bytes: + def assertRead(self, value : bytes, errorMsg : Optional[str] = None) -> bytes: """ Reads the number of bytes and compares them to the value provided. If it does not match, throws a value error. @@ -114,7 +116,7 @@ def assertRead(self, value : bytes, errorMsg : str = None) -> bytes: if len(value) == 0: return b'' - if not isinstance(value, bytes): + if not isinstance(value, bytes): # pyright: ignore raise TypeError(':param value: was not bytes.') valueRead = self.tryReadBytes(len(value)) @@ -163,7 +165,6 @@ def readByteString(self, width : int = 1) -> bytes: position = self.tell() string = b'' - endFound = False; null = b'\x00' * width while True: @@ -181,7 +182,7 @@ def readByteString(self, width : int = 1) -> bytes: # Otherwise add the character to our string. string += nextChar - def readClass(self, _class): + def readClass(self, _class : Type[_T]) -> _T: """ Takes anything with a __SIZE__ property and a call function that takes a single bytes argument and returns the result of that function. @@ -190,6 +191,8 @@ def readClass(self, _class): instance of the class created with that amount of bytes. However, there is little reason to truly limit it to only that. """ + if not hasattr(_class, '__SIZE__'): + raise TypeError('Argument to readClass MUST have a __SIZE__ attribute.') value = self.tryReadBytes(_class.__SIZE__) if value: return _class(value) @@ -246,7 +249,7 @@ def readShort(self) -> int: else: raise IOError('Not enough bytes left in buffer.') - def readStruct(self, _struct : Union[struct.Struct, Any]) -> Tuple: + def readStruct(self, _struct : Union[struct.Struct, Any]) -> Tuple[Any, ...]: """ Read enough bytes for a struct and unpack it, returning the tuple of values. diff --git a/extract_msg/structures/entry_id.py b/extract_msg/structures/entry_id.py index f8c383a5..8217e4e6 100644 --- a/extract_msg/structures/entry_id.py +++ b/extract_msg/structures/entry_id.py @@ -45,7 +45,7 @@ class EntryID(abc.ABC): """ @classmethod - def autoCreate(cls, data) -> Optional[EntryID]: + def autoCreate(cls, data : Optional[bytes]) -> Optional[EntryID]: """ Automatically determines the type of EntryID and returns an instance of the correct subclass. If the subclass cannot be determined, will return @@ -202,10 +202,10 @@ class ContactAddressEntryID(EntryID): def __init__(self, data : bytes): super().__init__(data) reader = BytesReader(data[20:]) - if reader.readUnsignedInt() != 3: - raise ValueError(f'Version must be 3 (got {self.__version}).') - if reader.readUnsignedInt() != 5: - raise ValueError(f'Type must be 4 (got {self.__version}).') + if (version := reader.readUnsignedInt()) != 3: + raise ValueError(f'Version must be 3 (got {version}).') + if (type_ := reader.readUnsignedInt()) != 5: + raise ValueError(f'Type must be 4 (got {type_}).') self.__index = ContactAddressIndex(reader.readUnsignedInt()) self.__entryIdCount = reader.readUnsignedInt() self.__entryID = MessageEntryID(reader.read(self.__entryIdCount)) @@ -251,7 +251,7 @@ def __init__(self, data : bytes): self.__folderType = MessageType(reader.readUnsignedShort()) self.__databaseGuid = bytesToGuid(reader.read(16)) # This entry is 6 bytes, so we pull some shenanigans to unpack it. - self.__globalCounter = constants.st.ST_LE_UI64.unpack(reader.read(6) + b'\x00\x00') + self.__globalCounter = constants.st.ST_LE_UI64.unpack(reader.read(6) + b'\x00\x00')[0] reader.assertNull(2, 'Pad bytes were not 0.') @property @@ -295,11 +295,11 @@ def __init__(self, data : bytes): self.__messageType = MessageType(reader.readUnsignedShort()) self.__folderDatabaseGuid = bytesToGuid(reader.read(16)) # This entry is 6 bytes, so we pull some shenanigans to unpack it. - self.__folderGlobalCounter = constants.st.ST_LE_UI64.unpack(reader.read(6) + b'\x00\x00') + self.__folderGlobalCounter = constants.st.ST_LE_UI64.unpack(reader.read(6) + b'\x00\x00')[0] reader.assertNull(2, 'Pad bytes were not 0.') self.__messageDatabaseGuid = bytesToGuid(reader.read(16)) # This entry is 6 bytes, so we pull some shenanigans to unpack it. - self.__messageGlobalCounter = constants.st.ST_LE_UI64.unpack(reader.read(6) + b'\x00\x00') + self.__messageGlobalCounter = constants.st.ST_LE_UI64.unpack(reader.read(6) + b'\x00\x00')[0] reader.assertNull(2, 'Pad bytes were not 0.') # Not sure why Microsoft decided to say "yes, let's do 2 6-byte integers # followed by 2 pad bytes each" instead of just 2 8-byte integers with a @@ -537,12 +537,12 @@ class PersonalDistributionListEntryID(EntryID): def __init__(self, data : bytes): super().__init__(data) reader = BytesReader(data[20:]) - if reader.readUnsignedInt() != 3: - raise ValueError(f'Version must be 3 (got {self.__version}).') - if reader.readUnsignedInt() != 5: - raise ValueError(f'Type must be 5 (got {self.__version}).') - if reader.readUnsignedInt() != 0xFF: - raise ValueError(f'Index must be 255 (got {self.__version}).') + if (arg := reader.readUnsignedInt()) != 3: + raise ValueError(f'Version must be 3 (got {arg}).') + if (arg := reader.readUnsignedInt()) != 5: + raise ValueError(f'Type must be 5 (got {arg}).') + if (arg := reader.readUnsignedInt()) != 0xFF: + raise ValueError(f'Index must be 255 (got {arg}).') self.__entryIdCount = reader.readUnsignedInt() self.__entryID = MessageEntryID(reader.read(self.__entryIdCount)) self.__position = reader.tell() + 20 diff --git a/extract_msg/utils.py b/extract_msg/utils.py index 6b949fe3..15aa10a1 100644 --- a/extract_msg/utils.py +++ b/extract_msg/utils.py @@ -33,7 +33,6 @@ 'msgPathToString', 'parseType', 'prepareFilename', - 'properHex', 'roundUp', 'rtfSanitizeHtml', 'rtfSanitizePlain', @@ -88,12 +87,13 @@ # Allow for nice type checking. if TYPE_CHECKING: from .msg_classes.msg import MSGFile + from .attachments import AttachmentBase logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) logging.addLevelName(5, 'DEVELOPER') -_T = TypeVar("_T") +_T = TypeVar('_T') def addNumToDir(dirName : pathlib.Path) -> Optional[pathlib.Path]: @@ -218,7 +218,7 @@ def decodeRfc2047(encoded : str) -> str: ) -def dictGetCasedKey(_dict : Dict, key : Any) -> Any: +def dictGetCasedKey(_dict : Dict[str, Any], key : str) -> str: """ Retrieves the key from the dictionary with the proper casing using a caseless key. @@ -320,7 +320,7 @@ def fromTimeStamp(stamp : int) -> datetime.datetime: """ try: tz = tzlocal.get_localzone() - except Exception as e: + except Exception: # I know "generalized exception catching is bad" but if *any* exception # happens here that is a subclass of Exception then something has gone # wrong with tzlocal. @@ -512,7 +512,7 @@ def getCommandArgs(args : Sequence[str]) -> argparse.Namespace: return options -def hasLen(obj) -> bool: +def hasLen(obj : Any) -> bool: """ Checks if :param obj: has a __len__ attribute. """ @@ -536,7 +536,7 @@ def htmlSanitize(inp : str) -> str: return inp -def inputToBytes(stringInputVar, encoding : str) -> bytes: +def inputToBytes(stringInputVar : Optional[Union[str, bytes]], encoding : str) -> bytes: """ Converts the input into bytes. @@ -552,7 +552,7 @@ def inputToBytes(stringInputVar, encoding : str) -> bytes: raise ConversionError('Cannot convert to bytes.') -def inputToMsgPath(inp) -> List[str]: +def inputToMsgPath(inp : constants.MSG_PATH) -> List[str]: """ Converts the input into an msg path. @@ -578,7 +578,7 @@ def inputToMsgPath(inp) -> List[str]: return ret -def inputToString(bytesInputVar, encoding) -> str: +def inputToString(bytesInputVar : Optional[Union[str, bytes]], encoding : str) -> str: """ Converts the input into a string. @@ -608,10 +608,11 @@ def makeWeakRef(obj : Optional[_T]) -> Optional[weakref.ReferenceType[_T]]: Attempts to return a weak reference to the object, returning None if not possible. """ - try: - return weakref.ref(obj) - except TypeError: + if obj is None: return None + else: + return weakref.ref(obj) + def minutesToDurationStr(minutes : int) -> str: """ @@ -774,7 +775,7 @@ def parseType(_type : int, stream, encoding, extras): return value -def prepareFilename(filename) -> str: +def prepareFilename(filename : str) -> str: """ Adjusts :param filename: so that it can succesfully be used as an actual file name. @@ -926,7 +927,7 @@ def setupLogging(defaultPath = None, defaultLevel = logging.WARN, logfile = None return True -def tryGetMimetype(att, mimetype : Union[str, None]) -> Union[str, None]: +def tryGetMimetype(att : AttachmentBase, mimetype : Union[str, None]) -> Union[str, None]: """ Uses an optional dependency to try and get the mimetype of an attachment. If the mimetype has already been found, the optional dependency does not exist, @@ -946,7 +947,8 @@ def tryGetMimetype(att, mimetype : Union[str, None]) -> Union[str, None]: try: import magic - return magic.from_buffer(att.data, mime = True) + if isinstance(att.data, (str, bytes)): + return magic.from_buffer(att.data, mime = True) except ImportError: logger.info('Mimetype not found on attachment, and `mime` dependency not installed. Won\'t try to generate.') From 8be7a1f2b62756b904b5208be51e4d2e9027488d Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Fri, 18 Aug 2023 23:02:46 -0700 Subject: [PATCH 23/68] More fixes and typing --- extract_msg/_rtf/inject_rtf.py | 6 +++-- extract_msg/_rtf/tokenize_rtf.py | 6 ++--- extract_msg/attachments/attachment_base.py | 2 +- extract_msg/exceptions.py | 1 - extract_msg/msg_classes/message_base.py | 2 +- extract_msg/msg_classes/msg.py | 15 +++++++----- extract_msg/msg_classes/post.py | 1 - extract_msg/msg_classes/task_request.py | 2 +- extract_msg/ole_writer.py | 24 ++++++++++---------- extract_msg/recipient.py | 18 +++++++-------- extract_msg/structures/__init__.py | 6 +++++ extract_msg/structures/business_card.py | 10 ++++---- extract_msg/structures/misc_id.py | 10 ++++---- extract_msg/structures/recurrence_pattern.py | 5 ++-- extract_msg/structures/system_time.py | 18 +++++++-------- extract_msg/structures/tz_rule.py | 5 +--- extract_msg/utils.py | 2 +- 17 files changed, 69 insertions(+), 64 deletions(-) diff --git a/extract_msg/_rtf/inject_rtf.py b/extract_msg/_rtf/inject_rtf.py index e3781f5e..caef7750 100644 --- a/extract_msg/_rtf/inject_rtf.py +++ b/extract_msg/_rtf/inject_rtf.py @@ -7,9 +7,11 @@ from .token import Token, TokenType from .tokenize_rtf import tokenizeRTF -from typing import List, Iterable, Union +from typing import List, Iterable, TypeVar, Union +_T = TypeVar('_T') + # A tuple of destinations used in the header. All ignorable ones are skipped # anyways, so we don't need to list those here. _HEADER_DESTINATIONS = ( @@ -45,7 +47,7 @@ ) -def _listInsertMult(dest : List, source : Iterable, index : int = -1): +def _listInsertMult(dest : List[_T], source : Iterable[_T], index : int = -1): """ Inserts into :param dest: all the items in :param source: at the index specified. :param dest: can be any mutable sequence with :method insert:, diff --git a/extract_msg/_rtf/tokenize_rtf.py b/extract_msg/_rtf/tokenize_rtf.py index 0df620ab..1d943565 100644 --- a/extract_msg/_rtf/tokenize_rtf.py +++ b/extract_msg/_rtf/tokenize_rtf.py @@ -5,7 +5,7 @@ import io -from typing import Optional, Tuple +from typing import List, Optional, Tuple from .token import Token, TokenType @@ -182,7 +182,7 @@ def _readText(startChar : bytes, reader : io.BytesIO) -> Tuple[Tuple[Token, ...] return tuple(Token(x, TokenType.TEXT) for x in chars), nextChar -def tokenizeRTF(data : bytes, validateStart : bool = True) -> None: +def tokenizeRTF(data : bytes, validateStart : bool = True) -> List[Token]: """ Reads in the bytes and sets the tokens list to the contents after tokenizing. If tokenizing fails, the current tokens list will not be @@ -215,7 +215,7 @@ def tokenizeRTF(data : bytes, validateStart : bool = True) -> None: nextChar = reader.read(1) # If the next character is a space, ignore it. - if nextChar == ' ': + if nextChar == b' ': nextChar = reader.read(1) else: tokens = [] diff --git a/extract_msg/attachments/attachment_base.py b/extract_msg/attachments/attachment_base.py index 10609314..cce243ca 100644 --- a/extract_msg/attachments/attachment_base.py +++ b/extract_msg/attachments/attachment_base.py @@ -247,7 +247,7 @@ def sExists(self, filename : MSG_PATH) -> bool: raise ReferenceError('The msg file for this Attachment instance has been garbage collected.') return msg.sExists([self.__dir, filename]) - def existsTypedProperty(self, id : Union[int, str], _type = None) -> bool: + def existsTypedProperty(self, id, _type = None) -> bool: """ Determines if the stream with the provided id exists. The return of this function is 2 values, the first being a boolean for if anything was diff --git a/extract_msg/exceptions.py b/extract_msg/exceptions.py index 17793ac1..cbf95d38 100644 --- a/extract_msg/exceptions.py +++ b/extract_msg/exceptions.py @@ -17,7 +17,6 @@ 'IncompatibleOptionsError', 'InvalidFileFormatError', 'InvaildPropertyIdError', - 'InvalidVersionError', 'StandardViolationError', 'TZError', 'UnknownCodepageError', diff --git a/extract_msg/msg_classes/message_base.py b/extract_msg/msg_classes/message_base.py index 2b7b093a..03075d50 100644 --- a/extract_msg/msg_classes/message_base.py +++ b/extract_msg/msg_classes/message_base.py @@ -1253,7 +1253,7 @@ def recipients(self) -> List[Recipient]: dir_[prefixLen] not in recipientDirs: recipientDirs.append(dir_[prefixLen]) - return [Recipient(recipientDir, self) for recipientDir in recipientDirs] + return [Recipient(recipientDir, self, self.recipientTypeClass) for recipientDir in recipientDirs] @property def recipientTypeClass() -> Type[enum.IntEnum]: diff --git a/extract_msg/msg_classes/msg.py b/extract_msg/msg_classes/msg.py index f33305b1..fb9d5394 100644 --- a/extract_msg/msg_classes/msg.py +++ b/extract_msg/msg_classes/msg.py @@ -129,13 +129,16 @@ def __init__(self, path, **kwargs): self.__stringEncoding = overrideEncoding self.__overrideEncoding = overrideEncoding - self.__listDirRes = {} + self.__listDirRes : Dict[Tuple[bool, bool, bool], List[List[str]]] = {} if self.__parentMsg: # We should be able to directly access the private variables of # another instance with no issue. - self.__ole = self.__parentMsg().__ole - self.__oleOwner = False + if (msg := self.__parentMsg()) is not None: + self.__ole = msg.__ole + self.__oleOwner = False + else: + raise ReferenceError('Parent MSG was garbage collected during init of child msg.') else: # Verify the path at least evaluates to True, as not doing so can # allow an OleFile to be created without a path. @@ -651,7 +654,7 @@ def getStream(self, filename : MSG_PATH, prefix : bool = True) -> Optional[bytes logger.info(f'Stream "{filename}" was requested but could not be found. Returning `None`.') return None - def getStreamAs(self, streamID, overrideClass : Callable[[Any], _T]) -> Optional[_T]: + def getStreamAs(self, streamID : MSG_PATH, overrideClass : Callable[[Any], _T]) -> Optional[_T]: """ Returns the specified stream, modifying it to the specified class if it is found. @@ -690,7 +693,7 @@ def getStringStream(self, filename : MSG_PATH, prefix : bool = True) -> Optional tmp = self.getStream(filename + '001E', prefix = False) return None if tmp is None else tmp.decode(self.stringEncoding) - def getStringStreamAs(self, streamID, overrideClass : Callable[[Any], _T]) -> Optional[_T]: + def getStringStreamAs(self, streamID : MSG_PATH, overrideClass : Callable[[Any], _T]) -> Optional[_T]: """ Returns the specified string stream, modifying it to the specified class if it is found. @@ -953,7 +956,7 @@ def insecureFeatures(self) -> InsecureFeatures: return self.__inscFeat @property - def kwargs(self) -> Dict[str, object]: + def kwargs(self) -> Dict[str, Any]: """ The kwargs used to initialize this message, excluding the prefix. This is used for initializing embedded msg files. diff --git a/extract_msg/msg_classes/post.py b/extract_msg/msg_classes/post.py index 152d8d2b..366d3e23 100644 --- a/extract_msg/msg_classes/post.py +++ b/extract_msg/msg_classes/post.py @@ -10,7 +10,6 @@ from .. import constants from .message_base import MessageBase -from ..utils import inputToString class Post(MessageBase): diff --git a/extract_msg/msg_classes/task_request.py b/extract_msg/msg_classes/task_request.py index 41051590..f445d5c8 100644 --- a/extract_msg/msg_classes/task_request.py +++ b/extract_msg/msg_classes/task_request.py @@ -79,7 +79,7 @@ def taskObject(self) -> Optional[Task]: return cast(Task, task[1]) @functools.cached_property - def taskRequestType(self) -> TaskRequestType: + def taskRequestType(self) -> Optional[TaskRequestType]: """ The type of task request. """ diff --git a/extract_msg/ole_writer.py b/extract_msg/ole_writer.py index 489c106b..b29e05fc 100644 --- a/extract_msg/ole_writer.py +++ b/extract_msg/ole_writer.py @@ -13,6 +13,7 @@ from typing import Dict, Iterator, List, Optional, Tuple, TYPE_CHECKING from . import constants +from .constants import MSG_PATH from .enums import Color, DirectoryEntryType from .utils import ceilDiv, dictGetCasedKey, inputToMsgPath from olefile.olefile import OleDirectoryEntry, OleFileIO @@ -30,9 +31,9 @@ class DirectoryEntry: Originals should be inaccessible outside of the class. """ name : str = '' - rightChild : DirectoryEntry = None - leftChild : DirectoryEntry = None - childTreeRoot : DirectoryEntry = None + rightChild : Optional[DirectoryEntry] = None + leftChild : Optional[DirectoryEntry] = None + childTreeRoot : Optional[DirectoryEntry] = None stateBits : int = 0 creationTime : int = 0 modifiedTime : int = 0 @@ -100,7 +101,7 @@ def __init__(self, rootClsid : bytes = constants.DEFAULT_CLSID): # The root entry will always exist, so this must be at least 1. self.__dirEntryCount = 1 self.__dirEntries = {} - self.__largeEntries = [] + self.__largeEntries : List[DirectoryEntry] = [] self.__largeEntrySectors = 0 self.__numMinifatSectors = 0 @@ -241,7 +242,6 @@ def __recalculateSectors(self) -> None: self.__largeEntries.clear() self.__largeEntrySectors = 0 - count = 0 for entry in self.__walkEntries(): self.__dirEntryCount += 1 if entry.type == DirectoryEntryType.STREAM: @@ -596,7 +596,7 @@ def _writeMini(self, f, entries : List[DirectoryEntry]) -> None: if self.__numMinifatSectors & 7: f.write((b'\x00' * 64) * (8 - (self.__numMinifatSectors & 7))) - def addEntry(self, path, data : bytes = None, storage : bool = False, **kwargs) -> None: + def addEntry(self, path : MSG_PATH, data : Optional[bytes] = None, storage : bool = False, **kwargs) -> None: """ Adds an entry to the OleWriter instance at the path specified, adding storages with default settings where necessary. If the entry is not a @@ -637,7 +637,7 @@ def addEntry(self, path, data : bytes = None, storage : bool = False, **kwargs) else: _dir[path[-1]] = entry - def addOleEntry(self, path, entry : OleDirectoryEntry, data : Optional[bytes] = None) -> None: + def addOleEntry(self, path : MSG_PATH, entry : OleDirectoryEntry, data : Optional[bytes] = None) -> None: """ Uses the entry provided to add the data to the writer. @@ -699,7 +699,7 @@ def deleteEntry(self, path) -> None: # path does remember the case used. del _dir[dictGetCasedKey(_dir, path[-1])] - def editEntry(self, path, **kwargs) -> None: + def editEntry(self, path : MSG_PATH, **kwargs) -> None: """ Used to edit values of an entry by setting the specific kwargs. Set a value to something other than None to set it. @@ -769,7 +769,7 @@ def fromMsg(self, msg : MSGFile) -> None: for x in gen: self.addOleEntry(x, msg._getOleEntry(x, prefix = False), msg.getStream(x, prefix = False)) - def fromOleFile(self, ole : OleFileIO, rootPath = []) -> None: + def fromOleFile(self, ole : OleFileIO, rootPath : MSG_PATH = []) -> None: """ Copies all the streams from the proided OLE file into this writer. @@ -826,7 +826,7 @@ def fromOleFile(self, ole : OleFileIO, rootPath = []) -> None: self.addOleEntry(x, entry, data) - def getEntry(self, path) -> DirectoryEntry: + def getEntry(self, path : MSG_PATH) -> DirectoryEntry: """ Finds and returns a copy of an existing DirectoryEntry instance in the writer. Use this method to check the internal status of an entry. @@ -836,7 +836,7 @@ def getEntry(self, path) -> DirectoryEntry: """ return copy.copy(self.__getEntry(inputToMsgPath(path))) - def listItems(self, streams = True, storages = False) -> List[List[str]]: + def listItems(self, streams : bool = True, storages : bool = False) -> List[List[str]]: """ Returns a list of the specified items currently in the writter. @@ -866,7 +866,7 @@ def listItems(self, streams = True, storages = False) -> List[List[str]]: paths.sort() return paths - def renameEntry(self, path, newName : str) -> None: + def renameEntry(self, path : MSG_PATH, newName : str) -> None: """ Changes the name of an entry, leaving it in it's current position. diff --git a/extract_msg/recipient.py b/extract_msg/recipient.py index 2ad40a09..549633c6 100644 --- a/extract_msg/recipient.py +++ b/extract_msg/recipient.py @@ -157,7 +157,7 @@ def _getTypedProperty(self, propertyID : str, _type = None) -> Tuple[bool, Optio return False, None - def _getTypedStream(self, filename, _type = None): + def _getTypedStream(self, filename : MSG_PATH, _type = None): """ Gets the contents of the specified stream as the type that it is supposed to be. @@ -193,7 +193,7 @@ def exists(self, filename : MSG_PATH) -> bool: raise ReferenceError('The msg file for this Recipient instance has been garbage collected.') return msg.exists([self.__dir, msgPathToString(filename)]) - def sExists(self, filename) -> bool: + def sExists(self, filename : MSG_PATH) -> bool: """ Checks if the string stream exists inside the recipient folder. @@ -217,7 +217,7 @@ def existsTypedProperty(self, id, _type = None) -> bool: raise ReferenceError('The msg file for this Recipient instance has been garbage collected.') return msg.existsTypedProperty(id, self.__dir, _type, True, self.__props) - def getMultipleBinary(self, filename) -> Optional[List[bytes]]: + def getMultipleBinary(self, filename : MSG_PATH) -> Optional[List[bytes]]: """ Gets a multiple binary property as a list of bytes objects. @@ -232,7 +232,7 @@ def getMultipleBinary(self, filename) -> Optional[List[bytes]]: raise ReferenceError('The msg file for this Recipient instance has been garbage collected.') return msg.getMultipleBinary([self.__dir, msgPathToString(filename)]) - def getMultipleString(self, filename) -> Optional[List[str]]: + def getMultipleString(self, filename : MSG_PATH) -> Optional[List[str]]: """ Gets a multiple string property as a list of str objects. @@ -247,7 +247,7 @@ def getMultipleString(self, filename) -> Optional[List[str]]: raise ReferenceError('The msg file for this Recipient instance has been garbage collected.') return msg.getMultipleString([self.__dir, msgPathToString(filename)]) - def getPropertyAs(self, propertyName, overrideClass : Callable[[Any], _T]) -> Optional[_T]: + def getPropertyAs(self, propertyName : Union[int, str], overrideClass : Callable[[Any], _T]) -> Optional[_T]: """ Returns the property, setting the class if found. @@ -264,7 +264,7 @@ def getPropertyAs(self, propertyName, overrideClass : Callable[[Any], _T]) -> Op return value - def getPropertyVal(self, name, default : _T = None) -> Union[Any, _T]: + def getPropertyVal(self, name : Union[int, str], default : _T = None) -> Union[Any, _T]: """ instance.props.getValue(name, default) @@ -272,7 +272,7 @@ def getPropertyVal(self, name, default : _T = None) -> Union[Any, _T]: """ return self.props.getValue(name, default) - def getSingleOrMultipleBinary(self, filename) -> Optional[Union[List[bytes], bytes]]: + def getSingleOrMultipleBinary(self, filename : MSG_PATH) -> Optional[Union[List[bytes], bytes]]: """ A combination of :method getStringStream: and :method getMultipleString:. @@ -291,7 +291,7 @@ def getSingleOrMultipleBinary(self, filename) -> Optional[Union[List[bytes], byt raise ReferenceError('The msg file for this Recipient instance has been garbage collected.') return msg.getSingleOrMultipleBinary([self.__dir, msgPathToString(filename)]) - def getSingleOrMultipleString(self, filename) -> Optional[Union[List[str], str]]: + def getSingleOrMultipleString(self, filename : MSG_PATH) -> Optional[Union[List[str], str]]: """ A combination of :method getStringStream: and :method getMultipleString:. @@ -324,7 +324,7 @@ def getStream(self, filename : MSG_PATH) -> Optional[bytes]: raise ReferenceError('The msg file for this Recipient instance has been garbage collected.') return msg.getStream([self.__dir, msgPathToString(filename)]) - def getStreamAs(self, streamID, overrideClass : Callable[[Any], _T]) -> Optional[_T]: + def getStreamAs(self, streamID : MSG_PATH, overrideClass : Callable[[Any], _T]) -> Optional[_T]: """ Returns the specified stream, modifying it to the specified class if it is found. diff --git a/extract_msg/structures/__init__.py b/extract_msg/structures/__init__.py index 92d2f600..c4f0747e 100644 --- a/extract_msg/structures/__init__.py +++ b/extract_msg/structures/__init__.py @@ -16,3 +16,9 @@ 'time_zone_struct', 'tz_rule', ] + +from . import ( + _helpers, contact_link_entry, business_card, entry_id, misc_id, + recurrence_pattern, report_tag, system_time, time_zone_definition, + time_zone_struct, tz_rule + ) \ No newline at end of file diff --git a/extract_msg/structures/business_card.py b/extract_msg/structures/business_card.py index d6f64294..1b286af2 100644 --- a/extract_msg/structures/business_card.py +++ b/extract_msg/structures/business_card.py @@ -4,7 +4,7 @@ ] -from typing import Tuple +from typing import Optional, Tuple from ._helpers import BytesReader from .. import constants @@ -37,7 +37,7 @@ def __init__(self, data : bytes): bitwiseAdjustedAnd(unpacked[8], 0xFF0000)) self.__imageArea = unpacked[9] self.__extraInfoField = data[17 + 16 * self.__countOfFields:] - self.__fields = tuple(FieldInfo(reader.read(16), self.__extraInfoField) for x in range(self.__countOfFields)) + self.__fields = tuple(FieldInfo(reader.read(16), self.__extraInfoField) for _ in range(self.__countOfFields)) @property def backgroundColor(self) -> Tuple[int, int, int]: @@ -150,7 +150,7 @@ def __init__(self, data : bytes, extraInfo : bytes): bitwiseAdjustedAnd(unpacked[5], 0xFF00), bitwiseAdjustedAnd(unpacked[5], 0xFF0000)) - self.__valueFontColor = (bitwiseAdjustedAnd(unpacked[6], 0xFF), + self.__labelFontColor = (bitwiseAdjustedAnd(unpacked[6], 0xFF), bitwiseAdjustedAnd(unpacked[6], 0xFF00), bitwiseAdjustedAnd(unpacked[6], 0xFF0000)) @@ -182,12 +182,12 @@ def labelOffset(self) -> int: """ An integer that specified the byte offset into the ExtraInfo field of BusinessCardDisplayDefinition. If the text field does not have a label, - must be 0xFFFE + must be 0xFFFE. """ return self.__labelOffset @property - def labelText(self) -> str: + def labelText(self) -> Optional[str]: """ The text of the label, if it exists. """ diff --git a/extract_msg/structures/misc_id.py b/extract_msg/structures/misc_id.py index 20011945..44a8006d 100644 --- a/extract_msg/structures/misc_id.py +++ b/extract_msg/structures/misc_id.py @@ -28,9 +28,9 @@ class FolderID: def __init__(self, data : bytes): self.__rawData = data - self.__replicaID = constants.st.STUI16.unpack(data[:2]) + self.__replicaID = constants.st.ST_DATA_UI16.unpack(data[:2])[0] # This entry is 6 bytes, so we pull some shenanigans to unpack it. - self.__globalCounter = constants.st.STUI64.unpack(data[2:8] + b'\x00\x00') + self.__globalCounter = constants.st.ST_LE_UI64.unpack(data[2:8] + b'\x00\x00')[0] @property def globalCounter(self) -> int: @@ -141,9 +141,9 @@ class MessageID: def __init__(self, data : bytes): self.__rawData = data - self.__replicaID = constants.st.STUI16.unpack(data[:2]) + self.__replicaID = constants.st.ST_LE_UI16.unpack(data[:2])[0] # This entry is 6 bytes, so we pull some shenanigans to unpack it. - self.__globalCounter = constants.st.STUI64.unpack(data[2:8] + b'\x00\x00') + self.__globalCounter = constants.st.ST_LE_UI64.unpack(data[2:8] + b'\x00\x00')[0] @property def globalCounter(self) -> int: @@ -192,7 +192,7 @@ def __init__(self, data : bytes): self.__rawData = data self.__folderID = FolderID(data[1:9]) self.__messageID = MessageID(data[9:17]) - self.__instance = constants.st.STUI32.unpack(data[17:21]) + self.__instance = constants.st.STUI32.unpack(data[17:21])[0] @property def folderID(self) -> FolderID: diff --git a/extract_msg/structures/recurrence_pattern.py b/extract_msg/structures/recurrence_pattern.py index 2a37b0bc..3b526db3 100644 --- a/extract_msg/structures/recurrence_pattern.py +++ b/extract_msg/structures/recurrence_pattern.py @@ -5,7 +5,6 @@ from typing import Any, Tuple -from .. import constants from ..enums import RecurCalendarType, RecurDOW, RecurEndType, RecurFrequency, RecurMonthNthWeek, RecurPatternType, RecurPatternTypeSpecificWeekday from ._helpers import BytesReader @@ -46,9 +45,9 @@ def __init__(self, data : bytes): self.__occurrenceCount = reader.readUnsignedInt() self.__firstDOW = RecurDOW(reader.readUnsignedInt()) deletedInstanceCount = reader.readUnsignedInt() - self.__deletedInstanceDates = tuple(reader.readUnsignedInt() for x in range(deletedInstanceCount)) + self.__deletedInstanceDates = tuple(reader.readUnsignedInt() for _ in range(deletedInstanceCount)) modifiedInstanceCount = reader.readUnsignedInt() - self.__modifiedInstanceDates = tuple(reader.readUnsignedInt() for x in range(modifiedInstanceCount)) + self.__modifiedInstanceDates = tuple(reader.readUnsignedInt() for _ in range(modifiedInstanceCount)) self.__startDate = reader.readUnsignedInt() self.__endDate = reader.readUnsignedInt() diff --git a/extract_msg/structures/system_time.py b/extract_msg/structures/system_time.py index fc9b82f5..b023ca44 100644 --- a/extract_msg/structures/system_time.py +++ b/extract_msg/structures/system_time.py @@ -11,20 +11,20 @@ class SystemTime: A SYSTEMTIME struct, as defined in [MS-DTYP]. """ - year : int = None - month : int = None - dayOfWeek : int = None - day : int = None - hour : int = None - minute : int = None - second : int = None - milliseconds : int = None + year : int = 0 + month : int = 0 + dayOfWeek : int = 0 + day : int = 0 + hour : int = 0 + minute : int = 0 + second : int = 0 + milliseconds : int = 0 def __init__(self, data : bytes): self.unpack(data) def __eq__(self, other) -> bool: - return self.pack() == other.pack() + return isinstance(other, SystemTime) and self.pack() == other.pack() def __ne__(self, other) -> bool: return not self.__eq__(other) diff --git a/extract_msg/structures/tz_rule.py b/extract_msg/structures/tz_rule.py index d5427ac8..9f1601cf 100644 --- a/extract_msg/structures/tz_rule.py +++ b/extract_msg/structures/tz_rule.py @@ -3,9 +3,6 @@ ] -from typing import Set - -from .. import constants from ..enums import TZFlag from ._helpers import BytesReader from .system_time import SystemTime @@ -59,7 +56,7 @@ def daylightDate(self) -> SystemTime: return self.__daylightDate @property - def flags(self) -> Set[TZFlag]: + def flags(self) -> TZFlag: """ The flags for this rule. """ diff --git a/extract_msg/utils.py b/extract_msg/utils.py index 15aa10a1..b579fca1 100644 --- a/extract_msg/utils.py +++ b/extract_msg/utils.py @@ -512,7 +512,7 @@ def getCommandArgs(args : Sequence[str]) -> argparse.Namespace: return options -def hasLen(obj : Any) -> bool: +def hasLen(obj) -> bool: """ Checks if :param obj: has a __len__ attribute. """ From 20b7ee497bb8deb93ce897ad6a73736bd5995ad9 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 23 Aug 2023 11:51:02 -0700 Subject: [PATCH 24/68] Fixed json property of journal being bytes --- extract_msg/__init__.py | 2 +- extract_msg/msg_classes/journal.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extract_msg/__init__.py b/extract_msg/__init__.py index ea0d3c6c..df0a6e39 100644 --- a/extract_msg/__init__.py +++ b/extract_msg/__init__.py @@ -27,7 +27,7 @@ # along with this program. If not, see . __author__ = 'Destiny Peterson & Matthew Walker' -__date__ = '2023-08-18' +__date__ = '2023-08-23' __version__ = '0.46.0' __all__ = [ diff --git a/extract_msg/msg_classes/journal.py b/extract_msg/msg_classes/journal.py index dc9aa10e..e2d8f58d 100644 --- a/extract_msg/msg_classes/journal.py +++ b/extract_msg/msg_classes/journal.py @@ -32,7 +32,7 @@ def getJson(self) -> str: 'body': self.body, # There is a good chance the body property won't exist, so this is a # backup. - 'rtfBodyB64': base64.b64encode(self.rtfBody) if self.rtfBody else None, + 'rtfBodyB64': base64.b64encode(self.rtfBody).decode('ascii') if self.rtfBody else None, }) @functools.cached_property From 0db4ac2096f99e7d2bee714fa5c5e3f2032c0ab7 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 23 Aug 2023 12:08:29 -0700 Subject: [PATCH 25/68] Add structures submodule to public api --- extract_msg/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extract_msg/__init__.py b/extract_msg/__init__.py index df0a6e39..12f3c3db 100644 --- a/extract_msg/__init__.py +++ b/extract_msg/__init__.py @@ -38,6 +38,7 @@ 'exceptions', 'msg_classes', 'properties', + 'structures', # Classes: 'Attachment', @@ -60,7 +61,7 @@ # Ensure these are imported before anything else. from . import constants, enums, exceptions -from . import attachments, msg_classes, properties +from . import attachments, msg_classes, properties, structures from .attachments import Attachment, AttachmentBase, SignedAttachment from .msg_classes import Message, MSGFile from .ole_writer import OleWriter From 072026d0f3fe87dcea13f54b9d357dadd46771de Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 23 Aug 2023 15:18:41 -0700 Subject: [PATCH 26/68] Started work on adding parser for OlePres --- extract_msg/enums.py | 14 ++- extract_msg/structures/ole_pres.py | 141 +++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 extract_msg/structures/ole_pres.py diff --git a/extract_msg/enums.py b/extract_msg/enums.py index 8b500691..45458bef 100644 --- a/extract_msg/enums.py +++ b/extract_msg/enums.py @@ -16,6 +16,7 @@ 'BodyTypes', 'BusyStatus', 'ClientIntentFlag', + 'ClipboardFormat', 'Color', 'ContactAddressIndex', 'ContactLinkState', @@ -79,7 +80,7 @@ import enum -from typing import Dict, Iterable, List, Set, Type, Union +from typing import Any, Dict, Iterable, List, Set, Type, Union class AddressBookType(enum.IntEnum): @@ -389,6 +390,17 @@ class ClientIntentFlag(enum.IntFlag): +class ClipboardFormat(enum.IntEnum): + """ + The standard clipboard formats, as specified in [MS-OLEDS]. + """ + CF_BITMAP = 0x00000002 + CF_METAFILEPICT = 0x00000003 + CF_DIB = 0x00000008 + CF_ENHMETAFILE = 0x0000000E + + + class Color(enum.IntEnum): RED = 0 BLACK = 1 diff --git a/extract_msg/structures/ole_pres.py b/extract_msg/structures/ole_pres.py new file mode 100644 index 00000000..fd3979ee --- /dev/null +++ b/extract_msg/structures/ole_pres.py @@ -0,0 +1,141 @@ +__all__ = [ + 'ClipboardFormatOrAnsiString', + 'OLEPresentationStream', +] + + + +from typing import Optional, Union + +from .. import constants +from ._helpers import BytesReader +from ..enums import ClipboardFormat, DVAspect + + +class ClipboardFormatOrAnsiString: + def __init__(self, reader : Union[bytes, BytesReader]): + if isinstance(reader, bytes): + reader = BytesReader(reader) + + self.__markerOrLength = reader.readUnsignedInt() + if self.__markerOrLength > 0xFFFFFFFD: + self.__ansiString = None + self.__clipboardFormat = ClipboardFormat(reader.readUnsignedInt()) + elif self.__markerOrLength > 0: + self.__ansiString = reader.read(self.__markerOrLength) + self.__clipboardFormat = None + else: + self.__ansiString = None + self.__clipboardFormat = None + + def toBytes(self) -> bytes: + ret = constants.st.ST_LE_UI32.pack(self.markerOrLength) + if self.markerOrLength > 0xFFFFFFFD: + ret += constants.st.ST_LE_UI32.pack(self.clipboardFormat) + elif self.markerOrLength > 0: + ret += self.ansiString + return ret + + @property + def ansiString(self) -> Optional[bytes]: + """ + The null-terminated ANSI string, as bytes, of the name of a registered + clipboard format. Only set if markerOrLength is not 0x00000000, + 0xFFFFFFFE, or 0xFFFFFFFF. + + Setting this will modify the markerOrLength field automatically. + """ + return self.__ansiString + + @ansiString.setter + def setter(self, val : bytes) -> None: + if not val: + raise ValueError('Cannot set :property ansiString: to None or empty bytes. Set :property markerOrLength: to a value ') + + self.__ansiString = val + + @property + def clipboardFormat(self) -> Optional[ClipboardFormat]: + """ + The clipboard format, if any. + + To set this, make sure that :property markerOrLength: is 0xFFFFFFFE or + 0xFFFFFFFF *before* setting. + """ + return self.__clipboardFormat + + @clipboardFormat.setter + def setter(self, val : ClipboardFormat) -> None: + if not val: + raise ValueError('Cannot set clipboard format to None.') + if self.markerOrLength < 0xFFFFFFFE: + raise ValueError('Cannot set the clipboard format while the marker or length is not 0xFFFFFFFE or 0xFFFFFFFF') + self.__clipboardFormat = val + + @property + def markerOrLength(self) -> int: + """ + If set the 0x00000000, then neither the format property nor the + ansiString property will be set. If it is 0xFFFFFFFF or 0xFFFFFFFE, then + the clipboardFormat property will be set. Otherwise, the ansiString + property + will be set. + """ + return self.__markerOrLength + + @markerOrLength.setter + def setter(self, val : int) -> None: + if val < 0: + raise ValueError('markerOrLength must be a positive integer.') + if val > 0xFFFFFFFF: + raise ValueError('markerOrLength must be a 4 byte unsigned integer.') + + if val == 0: + self.__ansiString = None + self.__clipboardFormat = None + elif val > 0xFFFFFFFD: + self.__ansiString = None + self.__clipboardFormat = ClipboardFormat.CF_BITMAP + else: + raise ValueError('Cannot set :property markerOrLength: to a length value. Set :property ansiString: instead.') + self.__markerOrLength = val + + + +class DVTargetDevice: + pass + + + +class OLEPresentationStream: + """ + [MS-OLEDS] OLEPresentationStream. + """ + ansiClipboardFormat : ClipboardFormatOrAnsiString + targetDeviceSize : int + targetDevice : Optional[DVTargetDevice] + + def __init__(self, data : bytes): + reader = BytesReader(data) + self.ansiClipboardFormat = ClipboardFormatOrAnsiString(reader) + + # Validate the structure based on the documentation. + if self.ansiClipboardFormat.markerOrLength == 0: + raise ValueError('Invalid OLEPresentationStream (MarkerOrLength is 0).') + if self.ansiClipboardFormat.clipboardFormat is ClipboardFormat.CF_BITMAP: + raise ValueError('Invalid OLEPresentationStream (Format is CF_BITMAP).') + if 0x201 < self.ansiClipboardFormat.markerOrLength < 0xFFFFFFFE: + raise ValueError('Invalid OLEPresentationStream (ANSI length was more than 0x201).') + + self.targetDeviceSize = reader.readUnsignedInt() + if self.targetDevice < 0x4: + raise ValueError('Invalid OLEPresentationStream (TargetDeviceSize was less than 4).') + if self.targetDevice > 0x4: + # Read the TargetDevice field. + self.targetDevice = DVTargetDevice(reader) + else: + self.targetDevice = None + + self.aspect = DVAspect(reader.readUnsignedInt) + + # TODO \ No newline at end of file From 6945fbef76a02935a75b63b7a3a6e31e60f9faad Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 23 Aug 2023 15:44:24 -0700 Subject: [PATCH 27/68] More progress on olepres --- CHANGELOG.md | 2 ++ extract_msg/enums.py | 15 +++++++++ extract_msg/structures/ole_pres.py | 49 +++++++++++++++++++++++++++--- 3 files changed, 61 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a52bf52..b7bc65bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,8 @@ * Correctly changed the returned value for `ResponseStatus.fromIter` to actually return a List instead of a set. * Filled out typing information for a significant portion of the module where variables or functions were missing it. * Corrected a number of minor issues. +* Extended values for `DVAspect` enum. +* Added new enums to go with parsing for `OLEPresentationStream`. **v0.45.0** * BREAKING: Changed parsing of string multiple properties to remove the trailing null byte. This *will* cause the output of parsing them to differ. diff --git a/extract_msg/enums.py b/extract_msg/enums.py index 45458bef..768f8cff 100644 --- a/extract_msg/enums.py +++ b/extract_msg/enums.py @@ -3,6 +3,7 @@ __all__ = [ 'AddressBookType', + 'ADVF', 'AppointmentAuxilaryFlag', 'AppointmentColor', 'AppointmentStateFlag', @@ -102,6 +103,17 @@ class AddressBookType(enum.IntEnum): +class ADVF(enum.IntEnum): + ADVF_NODATA = 1 + ADVF_PRIMEFIRST = 2 + ADVF_ONLYONCE = 4 + ADVF_DATAONSTOP = 64 + ADVFCACHE_NOHANDLER = 8 + ADVFCACHE_FORVEBUILTIN = 16 + ADVFCACHE_ONSAVE = 32 + + + class AppointmentAuxilaryFlag(enum.IntFlag): """ Describes the auxilary state of the object. @@ -474,7 +486,10 @@ class DVAspect(enum.IntEnum): Microsoft documentation of the DVASPECT enumeration. """ CONTENT = 1 + THUMBNAIL = 2 ICON = 4 + DOCPRINT = 8 + class ElectronicAddressProperties(enum.IntEnum): diff --git a/extract_msg/structures/ole_pres.py b/extract_msg/structures/ole_pres.py index fd3979ee..14aa9c93 100644 --- a/extract_msg/structures/ole_pres.py +++ b/extract_msg/structures/ole_pres.py @@ -1,15 +1,17 @@ +from __future__ import annotations + + __all__ = [ 'ClipboardFormatOrAnsiString', 'OLEPresentationStream', ] - -from typing import Optional, Union +from typing import List, Optional, Union from .. import constants from ._helpers import BytesReader -from ..enums import ClipboardFormat, DVAspect +from ..enums import ADVF, ClipboardFormat, DVAspect class ClipboardFormatOrAnsiString: @@ -103,7 +105,7 @@ def setter(self, val : int) -> None: class DVTargetDevice: - pass + pass # TODO @@ -114,6 +116,15 @@ class OLEPresentationStream: ansiClipboardFormat : ClipboardFormatOrAnsiString targetDeviceSize : int targetDevice : Optional[DVTargetDevice] + aspect : Union[int, DVAspect] + lindex : int + advf : Union[int, ADVF] + width : int + height : int + data : int + reserved2 : Optional[bytes] + tocSignature : int + tocEntries : List[TOCEntry] def __init__(self, data : bytes): reader = BytesReader(data) @@ -136,6 +147,34 @@ def __init__(self, data : bytes): else: self.targetDevice = None - self.aspect = DVAspect(reader.readUnsignedInt) + self.aspect = reader.readUnsignedInt() + self.lindex = reader.readUnsignedInt() + self.advf = reader.readUnsignedInt() + + # Reserved1. + reader.readUnsignedInt() + + self.width = reader.readUnsignedInt() + self.height = reader.readUnsignedInt() + size = reader.readUnsignedInt() + self.data = reader.read(size) + + if self.ansiClipboardFormat.clipboardFormat is ClipboardFormat.CF_METAFILEPICT: + self.reserved2 = reader.read(18) + else: + self.reserved2 = None + + self.tocSignature = reader.readUnsignedInt() + self.tocEntries = [] + if self.tocSignature == 0x494E414E: # b'NANI' in little endian. + for x in range(reader.readUnsignedInt()): + self.tocEntries.append(TOCEntry(reader)) + + + +class TOCEntry: + def __init__(self, reader : Union[bytes, BytesReader]): + if isinstance(reader, bytes): + reader = BytesReader(reader) # TODO \ No newline at end of file From cf0bd87250218d0ae40e8114e510388fb35afe8c Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 29 Aug 2023 20:24:36 -0700 Subject: [PATCH 28/68] More work on OlePres --- CHANGELOG.md | 1 + extract_msg/structures/entry_id.py | 6 +- extract_msg/structures/ole_pres.py | 188 ++++++++++++++++++++++++++++- 3 files changed, 188 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7bc65bc..1048b2b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ * Corrected a number of minor issues. * Extended values for `DVAspect` enum. * Added new enums to go with parsing for `OLEPresentationStream`. +* Changed `NNTPNewsgroupFolderEntryID.newsgroupName` to bytes instead of string since it is ANSI. **v0.45.0** * BREAKING: Changed parsing of string multiple properties to remove the trailing null byte. This *will* cause the output of parsing them to differ. diff --git a/extract_msg/structures/entry_id.py b/extract_msg/structures/entry_id.py index 8217e4e6..549ffe1c 100644 --- a/extract_msg/structures/entry_id.py +++ b/extract_msg/structures/entry_id.py @@ -360,7 +360,7 @@ def __init__(self, data : bytes): self.__folderType = reader.readUnsignedShort() if self.__folderType != 0x000C: raise ValueError(f'Folder type was not 0x000C (got {self.__folderType})') - self.__newsgroupName = reader.readAnsiString() + self.__newsgroupName = reader.readByteString() self.__position = reader.tell() + 20 @property @@ -371,9 +371,9 @@ def folderType(self) -> int: return self.__folderType @property - def newsgroupName(self) -> str: + def newsgroupName(self) -> bytes: """ - The name of the newsgroup. + The name of the newsgroup, as an ANSI string. """ return self.__newsgroupName diff --git a/extract_msg/structures/ole_pres.py b/extract_msg/structures/ole_pres.py index 14aa9c93..7a7ce87d 100644 --- a/extract_msg/structures/ole_pres.py +++ b/extract_msg/structures/ole_pres.py @@ -7,6 +7,8 @@ ] +import struct + from typing import List, Optional, Union from .. import constants @@ -104,8 +106,186 @@ def setter(self, val : int) -> None: +class DevModeA: + """ + A DEVMODEA structure, as specified in [MS-OLEDS]. For the purposes of + parsing from bytes, if something goes wrong this will evaluate to False when + converting to bool. If no data is prodided + """ + __parseStruct = struct.Struct('<32s32s4HI13H14xI4x4I16x') + + def __init__(self, data : Optional[bytes]): + self.__valid = data is None + if self.__valid: + self.__deviceName = b'\x00' * 32 + self.__formName = b'\x00' * 32 + # TODO set all properties to null values. + return + + reader = BytesReader(data) + + try: + data = reader.readStruct(self.__parseStruct) + except IOError: + return + + self.__valid = True + + def __bool__(self) -> bool: + return self.__valid + + + + class DVTargetDevice: - pass # TODO + """ + Specifies information about a device that renders the presentation data. + + The creator of this data structure MUST NOT assume that it will be + understood during processing. + """ + + def __init__(self, data : Optional[bytes]): + self.__driverName = None + self.__deviceName = None + self.__portName = None + self.__extDevMode = None + + if not data: + return + reader = BytesReader(data) + + # We have 4 fields to read, and *technically* they may not all even be + # present, given that this structure can be 4 bytes? Reading all of + # these is also much more complicated than other structures, as they can + # technically overlap. We are just going to be *much* more lenient about + # this structure. + offset1 = offset2 = offset3 = offset4 = -1 + try: + offset1 = reader.readUnsignedShort() + offset2 = reader.readUnsignedShort() + offset3 = reader.readUnsignedShort() + offset4 = reader.readUnsignedShort() + except IOError: + pass + + if offset1 != -1 and offset1 < len(data): + reader.seek(offset1) + try: + self.__driverName = reader.readByteString() + except IOError: + self.__driverName = reader.read() + if not self.__driverName: + self.__driverName = None + + if offset2 != -1 and offset2 < len(data): + reader.seek(offset2) + try: + self.__deviceName = reader.readByteString() + except IOError: + self.__deviceName = reader.read() + if not self.__deviceName: + self.__deviceName = None + + if offset3 != -1 and offset3 < len(data): + reader.seek(offset3) + try: + self.__portName = reader.readByteString() + except IOError: + self.__portName = reader.read() + if not self.__portName: + self.__portName = None + + if offset4 != -1 and offset4 < len(data): + reader.seek(offset4) + try: + devmode = DevModeA(reader.read(56)) + if devmode: + self.__extDevMode = devmode + except IOError: + self.__extDevMode = None + + def toBytes(self) -> Optional[bytes]: + if not (self.driverName or self.deviceName or self.portName or self.extDevMode): + return None + currentPosition = 8 + + offset1 = 8 if self.__driverName else 0 + if offset1: + currentPosition += len(self.__driverName) + 1 + + offset2 = currentPosition if self.__deviceName else 0 + if offset2: + currentPosition += len(self.__deviceName) + 1 + + offset3 = currentPosition if self.__portName else 0 + if offset3: + currentPosition += len(self.__portName) + 1 + + extDevModeBytes = self.__extDevMode.toBytes() if self.__extDevMode else None + offset4 = currentPosition if extDevModeBytes else 0 + + try: + ret = struct.pack(' Optional[bytes]: + """ + Optional ANSI string that contains a hunt on how to display or print + presentation data. + """ + return self.__driverName + + @driverName.setter + def setter(self, data : Optional[bytes]) -> None: + self.__driverName = None if not data else data + + @property + def deviceName(self) -> Optional[bytes]: + """ + Optional ANSI string that contains a hunt on how to display or print + presentation data. + """ + return self.__deviceName + + @deviceName.setter + def setter(self, data : Optional[bytes]) -> None: + self.__deviceName = None if not data else data + + @property + def portName(self) -> Optional[bytes]: + """ + Optional ANSI string that contains any arbitrary value. + """ + return self.__portName + + @portName.setter + def setter(self, data : Optional[bytes]) -> None: + self.__portName = None if not data else data + + @property + def extDevMode(self) -> Optional[DevModeA]: + """ + Optional ANSI string that contains a hunt on how to display or print + presentation data. + """ + return self.__extDevMode + + @extDevMode.setter + def setter(self, data : Optional[DevModeA]) -> None: + self.__extDevMode = None if not data else data @@ -139,11 +319,11 @@ def __init__(self, data : bytes): raise ValueError('Invalid OLEPresentationStream (ANSI length was more than 0x201).') self.targetDeviceSize = reader.readUnsignedInt() - if self.targetDevice < 0x4: + if self.targetDeviceSize < 0x4: raise ValueError('Invalid OLEPresentationStream (TargetDeviceSize was less than 4).') - if self.targetDevice > 0x4: + if self.targetDeviceSize > 0x4: # Read the TargetDevice field. - self.targetDevice = DVTargetDevice(reader) + self.targetDevice = DVTargetDevice(reader.read(self.targetDeviceSize)) else: self.targetDevice = None From fe8b6ec2f386322b65e2e1913190b5369bb768e0 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 30 Aug 2023 12:40:49 -0700 Subject: [PATCH 29/68] Handle python/cpython#85329 when parsing headers --- CHANGELOG.md | 1 + extract_msg/msg_classes/message_base.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1048b2b7..c756093a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ * Extended values for `DVAspect` enum. * Added new enums to go with parsing for `OLEPresentationStream`. * Changed `NNTPNewsgroupFolderEntryID.newsgroupName` to bytes instead of string since it is ANSI. +* Fixed an issue that would cause headers to fail to parse properly if the header text starts with "Microsoft Mail Internet Headers Version 2.0" which is common on some MSG files. This is fixed by stripping that from the beginning before actually parsing the text. This is to circumvent CPython issue #85329, confirmed to still exist in *at least* some of the supported Python versions. **v0.45.0** * BREAKING: Changed parsing of string multiple properties to remove the trailing null byte. This *will* cause the output of parsing them to differ. diff --git a/extract_msg/msg_classes/message_base.py b/extract_msg/msg_classes/message_base.py index 03075d50..03aa4a99 100644 --- a/extract_msg/msg_classes/message_base.py +++ b/extract_msg/msg_classes/message_base.py @@ -1041,6 +1041,9 @@ def header(self) -> email.message.Message: """ headerText = self.headerText if headerText: + # Fix an issue with prefixed headers not parsing correctly. + if headerText.startswith('Microsoft Mail Internet Headers Version 2.0'): + headerText = headerText[43:].lstrip() header = HeaderParser(policy = policy.default).parsestr(headerText) else: logger.info('Header is empty or was not found. Header will be generated from other streams.') From 8372cf7df7412dc6b3af91519c4b67c7f278684a Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 30 Aug 2023 16:30:46 -0700 Subject: [PATCH 30/68] Much more progress (see changelog) --- .gitignore | 7 +- CHANGELOG.md | 30 ++++---- extract_msg/__init__.py | 2 +- extract_msg/attachments/attachment.py | 2 +- extract_msg/attachments/attachment_base.py | 42 ++++++++--- extract_msg/attachments/custom_att.py | 2 +- .../custom_att_handler/__init__.py | 4 +- .../custom_att_handler/custom_handler.py | 8 ++- .../custom_att_handler/jrnl_assoc_att.py | 53 +++++++++----- .../custom_att_handler/outlook_image_dib.py | 7 +- extract_msg/attachments/emb_msg_att.py | 2 +- extract_msg/enums.py | 68 ++++++++++++++++++ extract_msg/exceptions.py | 10 ++- extract_msg/msg_classes/calendar_base.py | 4 +- extract_msg/msg_classes/message_base.py | 11 --- extract_msg/msg_classes/msg.py | 37 ++++++---- extract_msg/recipient.py | 21 +++++- extract_msg/structures/__init__.py | 3 +- extract_msg/structures/odt.py | 72 +++++++++++++++++++ extract_msg/utils.py | 40 ++++++++++- notes/Custom Attachment CLSIDs.txt | 3 +- setup.cfg | 5 +- 22 files changed, 343 insertions(+), 90 deletions(-) create mode 100644 extract_msg/structures/odt.py diff --git a/.gitignore b/.gitignore index 111ab755..8e5f646e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,13 @@ # Mac .DS_Store -# PyCharm crud +# PyCharm stuff /extract_msg/.idea -# build files and folders +# VS Code stuff. +/.vscode/ + +# Build files and folders /build/ *.egg-info/ /dist/ diff --git a/CHANGELOG.md b/CHANGELOG.md index c756093a..da04e19f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ **v0.46.0** +* [[TeamMsgExtractor #95](https://github.com/TeamMsgExtractor/msg-extractor/issues/95)] Adjusted the `overrideEncoding` property of `MSGFile` to allow automatic encoding detection. Simply set the property to the string `"chardet"` and, assuming the `chardet` module is installed, it will analyze a number of the strings to try and form a consensus about the encoding. This will *ignore* the specified encoding. * Changed the base class of `EntryID` from no base class to `abc.ABC`. * Added `position` property to `EntryID` to tell how many bytes were used to create the `EntryID`. * Added a number of properties to `MSGFile` from \[MS-OXCMSG\]. @@ -31,6 +32,11 @@ * Added new enums to go with parsing for `OLEPresentationStream`. * Changed `NNTPNewsgroupFolderEntryID.newsgroupName` to bytes instead of string since it is ANSI. * Fixed an issue that would cause headers to fail to parse properly if the header text starts with "Microsoft Mail Internet Headers Version 2.0" which is common on some MSG files. This is fixed by stripping that from the beginning before actually parsing the text. This is to circumvent CPython issue #85329, confirmed to still exist in *at least* some of the supported Python versions. +* Added `listDir` and `slistDir` as methods to `AttachmentBase`, `Recipient`, and `Named`. These *always* exclude the prefix, returning as if their directory is the root of the object. This allows the named to be directly used for accessing those files. +* Numerous spelling fixes in docstrings, comments, and exceptions. +* Reduced the amount of initialization performed by `MessageBase`. Much of this initialization was there from before a lot of stuff changed to `cached_property` and a number of internal variables were being used. Now all of the relevant variables will be initialized by the way they are accessed. +* Added new exception `DependencyError`. +* Changed the errors for missing optional dependencies from `ImportError` to `DependencyError`. **v0.45.0** * BREAKING: Changed parsing of string multiple properties to remove the trailing null byte. This *will* cause the output of parsing them to differ. @@ -72,7 +78,7 @@ * Fixed a bug that caused `MessageBase.headerInit` to always return `False` after the 0.42.0 update. * Changed `MessageBase.headerInit` to a property. * Fixed `extract_msg.utils.__all__`. -* Minor regoanization within `extract_msg/utils.py`. +* Minor reorganization within `extract_msg/utils.py`. * Minor changes to docstrings. * Minor README updates. * Fix issue with folded header fields decoding incorrectly when given to `extract_msg.utils.decodeRfc2047`. @@ -108,7 +114,7 @@ * Changed internal behavior of `MSGFile.attachments`. This should not cause any noticeable changes to the output. * Refactored code significantly to make it more organized. * Changed the exports from the main module to only include an important subset of the module. For other items, you'll have to import the submodule that it falls under to access it. Submodules export all important pieces, so it will be easier to find. - * This includes having many modules be under entirely new paths. Some of these changes have been done with no deprecation, something I generally try to avoid. This is happening at the same time as the public api is significantly changing, which makes it more acceptable. + * This includes having many modules be under entirely new paths. Some of these changes have been done with no deprecation, something I generally try to avoid. This is happening at the same time as the public API is significantly changing, which makes it more acceptable. * Fixed `__main__` using the wrong enum for error behavior. * Fixed `Named.get` being severely out of date (it's not used anywhere by the module which is why it wasn't noticed). * Fixed `Named.__getitem__` being entirely case-sensitive. @@ -120,7 +126,7 @@ * Removed unused function `getFullClassName`. * Fixes to the HTML body when saving as HTML will no longer require the `preparedHtml`/`--prepared-html` option. * Removed unused exceptions. -* Entirely reoganized the way attachments are initialized, including the class that will be used in various circumstances. Embedded MSG files, custom attachments, and web attachments will all use dedicated classes that are subclasses of `AttachmentBase`. +* Entirely reorganized the way attachments are initialized, including the class that will be used in various circumstances. Embedded MSG files, custom attachments, and web attachments will all use dedicated classes that are subclasses of `AttachmentBase`. * With this change, the way to specify a new `Attachment` class is to override the function used when creating attachments. This can be done by passing `attachmentInit = myFunction` as an option to `openMsg`. This function MUST return an instance of `AttachmentBase`. * Added first implementation of web attachments. Saving is not currently possible, but basic relevant property access is now possible. Saving will not be stopped by this attachment if `skipNotImplemented = True` is passed to the save function. * Changed the option to suppress `RTFDE` errors to fall under the `ErrorBehavior` enum. Usage of the original option will be allowable, but is being marked as deprecated. However, it is still a dedicated option from the command line. @@ -388,7 +394,7 @@ * Fixed an issue that would cause signed attachments to not properly generate. **v0.34.0** -* [[TeamMsgExtractor #102](https://github.com/TeamMsgExtractor/msg-extractor/issues/102)] Added the option to directly save body to pdf. This requires that you either have wkhtmltopdf on your path or that you provide a path directly to it in order to work. Simply pass `pdf = True` to save to turn it on. More details in the doc for the save function. You can also do this from the command line using the `--pdf` option, incompatible with other body types. +* [[TeamMsgExtractor #102](https://github.com/TeamMsgExtractor/msg-extractor/issues/102)] Added the option to directly save body to pdf. This requires that you either have `wkhtmltopdf` on your path or that you provide a path directly to it in order to work. Simply pass `pdf = True` to save to turn it on. More details in the doc for the save function. You can also do this from the command line using the `--pdf` option, incompatible with other body types. * Added `--glob` option for allowing you to provide an msg path that will evaluate wildcards. * Removed per-file output names as they weren't actually functional and currently add too much complexity. If anyone knows a way to handle it directly with `argparse` let me know. * Added `chardet` as a requirement to help work around an error in `RTFDE`. @@ -454,7 +460,7 @@ * Updated docstring for `MessageBase.deencapsulatedRtf`. **v0.30.9** -* Fixed the behavior of `Properties.get` so it actually behaves like a dict (that was the intent of it, but I did it the wrong way for some reason). +* Fixed the behavior of `Properties.get` so it actually behaves like a `dict` (that was the intent of it, but I did it the wrong way for some reason). * Fixed a type that caused an exception when no HTML body could be found nor generated. **v0.30.8** @@ -520,7 +526,7 @@ **v0.29.3** * [[TeamMsgExtractor #226](https://github.com/TeamMsgExtractor/msg-extractor/issues/198)] Fix typo in command parsing that prevented the usage of `allowFallback`. -* Fixed main still manually navigating to a new directory with os.chdir instead of using `customPath`. +* Fixed main still manually navigating to a new directory with `os.chdir` instead of using `customPath`. * Fixed issue in main where the `--html` option was being using for both html *and* rtf. This meant if you wanted rtf it would not have used it, and if you wanted html it would have thrown an error. * Fixed `--out-name` having no effect. * Fixed `--out` having no effect. @@ -597,7 +603,7 @@ **v0.28.6** * [[TeamMsgExtractor #191](https://github.com/TeamMsgExtractor/msg-extractor/issues/191)] This feature was never properly implemented, so it's not officially supported. However, this specific issue should be fixed. This is a temporary patch until I can get around to rewriting the way the module saves files in general. -* Added `venv` to the .gitignore list. +* Added `venv` to the `.gitignore` list. * Added information to the readme. **v0.28.5** @@ -628,7 +634,7 @@ * [[TeamMsgExtractor #87](https://github.com/TeamMsgExtractor/msg-extractor/issues/87)] Added a new system to handle `NotImplementedError` and other exceptions. All msg classes now have an option called `attachmentErrorBehavior` that tells the class what to do if it has an error. The value should be one of three constants: `ATTACHMENT_ERROR_THROW`, `ATTACHMENT_ERROR_NOT_IMPLEMENTED`, or `ATTACHMENT_ERROR_BROKEN`. `ATTACHMENT_ERROR_THROW` tells the class to not catch and exceptions and just let the user handle them. `ATTACHMENT_ERROR_NOT_IMPLEMENTED` tells the class to catch `NotImplementedError` exceptions and put an instance of `UnsupportedAttachment` in place of a regular attachment. `ATTACHMENT_ERROR_BROKEN` tells the class to catch *all* exceptions and either replace the attachment with `UnsupportedAttachment` if it is a `NotImplementedError` or `BrokenAttachment` for all other exceptions. With both of those options, caught exceptions will be logged. * In making the previous point work, much code from `Attachment` has been moved to a new class called `AttachmentBase`. Both `BrokenAttachment` and `UnsupportedAttachment` are subclasses of `AttachmentBase` meaning data can be extracted from their streams in the same way as a functioning attachment. * [[TeamMsgExtractor #162](https://github.com/TeamMsgExtractor/msg-extractor/issues/162)] Pretty sure I actually got it this time. The execution flag should be applied by pip now. -* Fixed typos in some exceptions +* Fixed typos in some exceptions. **v0.27.16** * [[TeamMsgExtractor #177](https://github.com/TeamMsgExtractor/msg-extractor/issues/177)] Fixed incorrect struct being used. It should be the correct one now, but further testing will be required to confirm this. @@ -655,7 +661,7 @@ * [[TeamMsgExtractor #162](https://github.com/TeamMsgExtractor/msg-extractor/issues/162)] Fixed line endings in the wrapper script to be UNIX line endings rather than Windows line endings. Attempted to add the execution flag to the runnable script. **v0.27.9** -* [[TeamMsgExtractor #161](https://github.com/TeamMsgExtractor/msg-extractor/issues/161)] Added commands to the command line that will allow the user to specify that they want the message data to be output to stdout rather than to a file. +* [[TeamMsgExtractor #161](https://github.com/TeamMsgExtractor/msg-extractor/issues/161)] Added commands to the command line that will allow the user to specify that they want the message data to be output to `stdout` rather than to a file. * [[TeamMsgExtractor #162](https://github.com/TeamMsgExtractor/msg-extractor/issues/162)] Added a wrapper for extract_msg that will be installed. * Fixed some of the encoding names to allow them to actually be used in Python. The names they previously held were not aliases that currently exist. * Added more documentation to `constants.CODE_PAGES` to give more information about what it is. As it is a list of the possible encodings an msg file can use, I also specified which ones were supported by Python 3. @@ -673,7 +679,7 @@ **v0.27.5** * Fixed an error in `utils.divide` that would cause it to drop the extra data if there was not enough to create a full division. For example, if you had a string that was 10 characters, and divided by 3, you would only receive a total of 9 characters back. * Added some useful functions that will be used in the future. -* [[TeamMsgExtractor #155](https://github.com/TeamMsgExtractor/msg-extractor/issues/155)] Updated to use new version of tzlocal. +* [[TeamMsgExtractor #155](https://github.com/TeamMsgExtractor/msg-extractor/issues/155)] Updated to use new version of `tzlocal`. * Updated changelog to fit new repository. **v0.27.4** @@ -789,7 +795,7 @@ **v0.22.0** * [[TheElementalOfDestruction #18](https://github.com/TheElementalOfDestruction/msg-extractor/issues/18)] Added `--validate` option. * [[TheElementalOfDestruction #16](https://github.com/TheElementalOfDestruction/msg-extractor/issues/16)] Moved all dev code into its own scripts. Use `--dev` to use from the command line. -* [[TeamMsgExtractor #67](https://github.com/TeamMsgExtractor/msg-extractor/issues/67)] Added compatibility module to enforce Unicode os functions. +* [[TeamMsgExtractor #67](https://github.com/TeamMsgExtractor/msg-extractor/issues/67)] Added compatibility module to enforce Unicode `os` functions. * Added new function to `Message` class: `Message.sExists`. This function checks if a string stream exists. It's input should be formatted identically to that of `Message._getStringStream`. * Added new function to `Message` class: `Message.fix_path`. This function will add the proper prefix to the path (if the `prefix` parameter is true) and adjust the path to be a string rather than a list or tuple. * Added new function to `utils.py`: `get_full_class_name`. This function returns a string containing the module name and the class name of any instance of any class. It is returned in the format of `{module}.{class}`. @@ -812,9 +818,7 @@ **v0.20.8** * Fixed a tab issue and parameter type in `message.py`. - **v0.20.7** - * Separated classes into their own files to make things more manageable. * Placed `__doc__` back inside of `__init__.py`. * Rewrote the `Prop` class to be two different classes that extend from a base class. diff --git a/extract_msg/__init__.py b/extract_msg/__init__.py index 12f3c3db..eedf1935 100644 --- a/extract_msg/__init__.py +++ b/extract_msg/__init__.py @@ -27,7 +27,7 @@ # along with this program. If not, see . __author__ = 'Destiny Peterson & Matthew Walker' -__date__ = '2023-08-23' +__date__ = '2023-08-30' __version__ = '0.46.0' __all__ = [ diff --git a/extract_msg/attachments/attachment.py b/extract_msg/attachments/attachment.py index 2740b708..ee16c357 100644 --- a/extract_msg/attachments/attachment.py +++ b/extract_msg/attachments/attachment.py @@ -53,7 +53,7 @@ def getFilename(self, **kwargs) -> str: :param contentId: Use the contentId, if available. :param customFilename: A custom name to use for the file. - If the filename starts with "UnknownFilename" then there is no guarentee + If the filename starts with "UnknownFilename" then there is no guarantee that the files will have exactly the same filename. """ filename = None diff --git a/extract_msg/attachments/attachment_base.py b/extract_msg/attachments/attachment_base.py index cce243ca..53562e9e 100644 --- a/extract_msg/attachments/attachment_base.py +++ b/extract_msg/attachments/attachment_base.py @@ -42,7 +42,7 @@ class AttachmentBase(abc.ABC): """ - The base class for all Attachments used by the module, if not overriden. + The base class for all standard Attachments used by the module. """ def __init__(self, msg : MSGFile, dir_ : str, propStore : PropertiesStore): @@ -212,7 +212,7 @@ def _handleFnc(self, _zip, filename, customPath, kwargs) -> pathlib.Path: raise FileExistsError(f'Could not create the specified file because it already exists ("{fullFilename}").') else: if not overwriteExisting and fullFilename.exists(): - # Try to split the filename into a name and extention. + # Try to split the filename into a name and extension. name, ext = os.path.splitext(filename) # Try to add a number to it so that we can save without overwriting. for i in range(2, 100): @@ -260,6 +260,18 @@ def existsTypedProperty(self, id, _type = None) -> bool: raise ReferenceError('The msg file for this Attachment instance has been garbage collected.') return msg.existsTypedProperty(id, self.__dir, _type, True, self.__props) + @abc.abstractmethod + def getFilename(self, **kwargs) -> str: + """ + Returns the filename to use for the attachment. + + :param contentId: Use the contentId, if available. + :param customFilename: A custom name to use for the file. + + If the filename starts with "UnknownFilename" then there is no guarantee + that the files will have exactly the same filename. + """ + def getMultipleBinary(self, filename : MSG_PATH) -> Optional[List[bytes]]: """ Gets a multiple binary property as a list of bytes objects. @@ -334,7 +346,7 @@ def getPropertyVal(self, name : Union[int, str], default : _T = None) -> Union[A """ instance.props.getValue(name, default) - Can be overriden to create new behavior. + Can be overridden to create new behavior. """ return self.props.getValue(name, default) @@ -441,17 +453,25 @@ class if it is found. return value - @abc.abstractmethod - def getFilename(self, **kwargs) -> str: + def listDir(self, streams : bool = True, storages : bool = False) -> List[List[str]]: """ - Returns the filename to use for the attachment. + Lists the streams and or storages that exist in the attachment + directory. - :param contentId: Use the contentId, if available. - :param customFilename: A custom name to use for the file. + Returns the paths *excluding* the attachment directory, allowing the + paths to be directly used for accessing a file. + """ + if (msg := self.__msg()) is None: + raise ReferenceError('The msg file for this Attachment instance has been garbage collected.') + return [path[1:] for path in msg.listDir(streams, storages, False) + if len(path) > 1 and path[0] == self.__dir] - If the filename starts with "UnknownFilename" then there is no guarentee - that the files will have exactly the same filename. + def slistDir(self, streams : bool = True, storages : bool = False) -> List[str]: """ + Like listDir, except it returns the paths as strings. + """ + return ['/'.join(path) for path in self.listDir(streams, storages)] + @abc.abstractmethod def save(self, **kwargs) -> SAVE_TYPE: @@ -669,7 +689,7 @@ def props(self) -> PropertiesStore: @functools.cached_property def renderingPosition(self) -> Optional[int]: """ - The offset, in redered characters, to use when rendering the attachment + The offset, in rendered characters, to use when rendering the attachment within the main message text. A value of 0xFFFFFFFF indicates a hidden attachment that is not to be rendered. """ diff --git a/extract_msg/attachments/custom_att.py b/extract_msg/attachments/custom_att.py index 603ff9ce..01b7ba81 100644 --- a/extract_msg/attachments/custom_att.py +++ b/extract_msg/attachments/custom_att.py @@ -48,7 +48,7 @@ def getFilename(self, **kwargs) -> str: :param contentId: Use the contentId, if available. :param customFilename: A custom name to use for the file. - If the filename starts with "UnknownFilename" then there is no guarentee + If the filename starts with "UnknownFilename" then there is no guarantee that the files will have exactly the same filename. """ filename = None diff --git a/extract_msg/attachments/custom_att_handler/__init__.py b/extract_msg/attachments/custom_att_handler/__init__.py index b53d7c25..26a5d9cb 100644 --- a/extract_msg/attachments/custom_att_handler/__init__.py +++ b/extract_msg/attachments/custom_att_handler/__init__.py @@ -21,7 +21,7 @@ __all__ = [ # Classes. 'CustomAttachmentHandler', - 'JournalAssociatedAttachment', + 'LinkedObjectAttachment', 'OutlookImageDIB', # Functions. @@ -57,7 +57,7 @@ def registerHandler(handler : Type[CustomAttachmentHandler]) -> None: # Import built-in handler modules. They will all automatically register their # respecive handler(s). from .outlook_image_dib import OutlookImageDIB -from .jrnl_assoc_att import JournalAssociatedAttachment +from .jrnl_assoc_att import LinkedObjectAttachment if TYPE_CHECKING: diff --git a/extract_msg/attachments/custom_att_handler/custom_handler.py b/extract_msg/attachments/custom_att_handler/custom_handler.py index d1aa601f..7efea836 100644 --- a/extract_msg/attachments/custom_att_handler/custom_handler.py +++ b/extract_msg/attachments/custom_att_handler/custom_handler.py @@ -8,7 +8,7 @@ import abc -from typing import Any, Callable, Optional, TYPE_CHECKING, TypeVar +from typing import Any, Callable, Dict, Optional, TYPE_CHECKING, TypeVar from ...constants import MSG_PATH from ...utils import msgPathToString @@ -30,6 +30,12 @@ def __init__(self, attachment : AttachmentBase): super().__init__() self.__att = attachment + def getPresentationStreams(self) -> Optional[Dict[int, bytes]]: + """ + Returns a dict of all presentation streams, as bytes. + """ + presLinks = [(x[-1][-3:], self.getStream(x[-1])) for x in self.attachment.listDir()] + def getStream(self, path : MSG_PATH) -> Optional[bytes]: """ Gets a stream from the custom data directory. diff --git a/extract_msg/attachments/custom_att_handler/jrnl_assoc_att.py b/extract_msg/attachments/custom_att_handler/jrnl_assoc_att.py index c736f6f3..18d041ef 100644 --- a/extract_msg/attachments/custom_att_handler/jrnl_assoc_att.py +++ b/extract_msg/attachments/custom_att_handler/jrnl_assoc_att.py @@ -2,23 +2,36 @@ __all__ = [ - 'JournalAssociatedAttachment', + 'LinkedObjectAttachment', ] from functools import cached_property -from typing import Optional, TYPE_CHECKING +from typing import List, Optional, TYPE_CHECKING from . import registerHandler from .custom_handler import CustomAttachmentHandler from ...structures.entry_id import EntryID +from ...structures.ole_pres import OLEPresentationStream if TYPE_CHECKING: from ..attachment_base import AttachmentBase -class JournalAssociatedAttachment(CustomAttachmentHandler): +class LinkedObjectAttachment(CustomAttachmentHandler): + """ + A link to an Outlook object. + + Not *positive* I understand what this attachment type is, but this seems to + be the most likely name. Contains presentation data about how to render it + as well as properties with data that link to it. It looks *similar* to what + the documentation for Journal specifies would be it's custom attachment + type, however some small details don't perfectly add up. + + I've also only seen this on Journal objects thus far. + """ + def __init__(self, attachment : AttachmentBase): super().__init__(attachment) stream = attachment.getStream('__substg1.0_3701000D/\x03MailStream') @@ -29,11 +42,6 @@ def __init__(self, attachment : AttachmentBase): @classmethod def isCorrectHandler(cls, attachment : AttachmentBase) -> bool: - # This only applies to journal objects. - if not attachment.msg.classType: - return False - if not attachment.msg.classType.lower().startswith('ipm.activity'): - return False if attachment.clsid != '00020D09-0000-0000-C000-000000000046': return False @@ -87,19 +95,28 @@ def mailMsgAttSrchKey(self) -> Optional[bytes]: return self.getStream('MailMsgAttSrchKey') @cached_property - def metafileBytes(self) -> Optional[bytes]: + def presentationStreams(self) -> Optional[List[OLEPresentationStream]]: """ - The metafile that contains the icon to be used when rendering the - attachment. + The presentation streams, as a list of OLEPresentationStream object. From my understanding, this MUST be set, but we are treating it as - SHOULD be set. + SHOULD be set. It also looks like the correct number to have is exactly + 1, but I can't guarantee that, so this is a list. + """ + if self.presentationStreamsBytes: + return [OLEPresentationStream(x) for x in self.presentationStreams] + else: + return None + + @cached_property + def presentationStreamsBytes(self) -> Optional[List[bytes]]: + """ + The presentation streams, as a list of bytes. + + From my understanding, there should exist EXACTLY 1 of these, but... + this appears to be an undocumented custom attachment that is easy to + make. """ - # The documentation specifies clearly that the filename is "IOlePres000" - # HOWEVER my tests revealed that the "I" is actually a "\x02" character. - # This is quite confusing but whatever. We'll just look for both of - # them. - return self.getStream('IOlePres000') or self.getStream('\x02OlePres000') @property def name(self) -> None: @@ -113,4 +130,4 @@ def obj(self) -> None: -registerHandler(JournalAssociatedAttachment) \ No newline at end of file +registerHandler(LinkedObjectAttachment) \ No newline at end of file diff --git a/extract_msg/attachments/custom_att_handler/outlook_image_dib.py b/extract_msg/attachments/custom_att_handler/outlook_image_dib.py index 12fa9e6d..ae8d2cf7 100644 --- a/extract_msg/attachments/custom_att_handler/outlook_image_dib.py +++ b/extract_msg/attachments/custom_att_handler/outlook_image_dib.py @@ -13,7 +13,7 @@ from . import registerHandler from .custom_handler import CustomAttachmentHandler from ...enums import DVAspect, InsecureFeatures -from ...exceptions import SecurityError +from ...exceptions import DependencyError, SecurityError if TYPE_CHECKING: @@ -91,8 +91,7 @@ def generateRtf(self) -> Optional[bytes]: If this function should do nothing, returns None. - This function requires PIL or Pillow. If neither are found, raises an - import error. + :raises DependencyError: PIL or Pillow could not be found. """ if InsecureFeatures.PIL_IMAGE_PARSING not in self.attachment.msg.insecureFeatures: raise SecurityError('Generating the RTF for a custom attachment requires the insecure feature PIL_IMAGE_PARSING.') @@ -100,7 +99,7 @@ def generateRtf(self) -> Optional[bytes]: try: import PIL.Image except ImportError: - raise ImportError('PIL or Pillow is required for inserting an Outlook Image into the body.') + raise DependencyError('PIL or Pillow is required for inserting an Outlook Image into the body.') # First, convert the bitmap into a PNG so we can insert it into the # body. diff --git a/extract_msg/attachments/emb_msg_att.py b/extract_msg/attachments/emb_msg_att.py index b93c09c4..0bb2418e 100644 --- a/extract_msg/attachments/emb_msg_att.py +++ b/extract_msg/attachments/emb_msg_att.py @@ -44,7 +44,7 @@ def getFilename(self, **kwargs) -> str: :param contentId: Use the contentId, if available. :param customFilename: A custom name to use for the file. - If the filename starts with "UnknownFilename" then there is no guarentee + If the filename starts with "UnknownFilename" then there is no guarantee that the files will have exactly the same filename. """ customFilename = kwargs.get('customFilename') diff --git a/extract_msg/enums.py b/extract_msg/enums.py index 768f8cff..ed99b2db 100644 --- a/extract_msg/enums.py +++ b/extract_msg/enums.py @@ -45,6 +45,9 @@ 'MessageType', 'NamedPropertyType', 'NoteColor', + 'ODTCf', + 'ODTPersist1', + 'ODTPersist2', 'OORBodyFormat', 'PostalAddressID', 'Priority', @@ -1376,6 +1379,71 @@ class NoteColor(enum.IntEnum): +class ODTCf(enum.IntEnum): + """ + Values for the `cf` field of the ODT structure. + """ + UNSPECIFIED = 0x0000 + RICH_TEXT_FORMAT = 0x0001 + TEXT_FORMAT = 0x0002 + METAFILE = 0x0003 + BITMAP = 0x0004 + DEVICE_INDEPENDENT_BITMAP = 0x0005 + HTML_FORMAT = 0x000A + UNICODE_TEXT_FORMAT = 0x0014 + + + +class ODTPersist1(enum.IntFlag): + """ + Flag values for ODTPersist1, in the order they would appear when unpacking a + little endian unsigned short. + """ + NONE = 0b0000000000000000 + RESERVED_1 = 0b0000000000000001 + F_DEF_HANDLER = 0b0000000000000010 + RESERVED_2 = 0b0000000000000100 + RESERVED_3 = 0b0000000000001000 + F_LINK = 0b0000000000010000 + RESERVED_4 = 0b0000000000100000 + F_ICON = 0b0000000001000000 + F_IS_OLE1 = 0b0000000010000000 + F_MANUAL = 0b0000000100000000 + F_RECOMPOSE_ON_RESIZE = 0b0000001000000000 + RESERVED_5 = 0b0000010000000000 + RESERVED_6 = 0b0000100000000000 + F_OCX = 0b0001000000000000 + F_STREAM = 0b0010000000000000 + RESERVED_7 = 0b0100000000000000 + F_VIEW_OBJECT = 0b1000000000000000 + + + +class ODTPersist2(enum.IntFlag): + """ + Flag values for ODTPersist2, in the order they would appear when unpacking a + little endian unsigned short. + """ + NONE = 0b0000000000000000 + F_EMF = 0b0000000000000001 + RESERVED_1 = 0b0000000000000010 + F_QUERIED_EMF = 0b0000000000000100 + F_STORED_AS_EMF = 0b0000000000001000 + RESERVED_2 = 0b0000000000010000 + RESERVED_3 = 0b0000000000100000 + RESERVED_4 = 0b0000000001000000 + RESERVED_5 = 0b0000000010000000 + RESERVED_6 = 0b0000000100000000 + RESERVED_7 = 0b0000001000000000 + RESERVED_8 = 0b0000010000000000 + RESERVED_9 = 0b0000100000000000 + RESERVED_10 = 0b0001000000000000 + RESERVED_11 = 0b0010000000000000 + RESERVED_12 = 0b0100000000000000 + RESERVED_13 = 0b1000000000000000 + + + class OORBodyFormat(enum.IntEnum): """ The body format for One Off Recipients. diff --git a/extract_msg/exceptions.py b/extract_msg/exceptions.py index cbf95d38..8be240e3 100644 --- a/extract_msg/exceptions.py +++ b/extract_msg/exceptions.py @@ -65,7 +65,13 @@ class DeencapNotEncapsulated(ExMsgBaseException): Data to deencapsulate did not contain any encapsulated data. """ -class ExecutableNotFound(ExMsgBaseException): +class DependencyError(ExMsgBaseException): + """ + An optional dependdency could not be found or was unable to be used as + expected. + """ + +class ExecutableNotFound(DependencyError): """ Could not find the specified executable. """ @@ -143,7 +149,7 @@ class UnrecognizedMSGTypeError(ExMsgBaseException): open a specific class of msg file. """ -class WKError(ExMsgBaseException): +class WKError(DependencyError): """ An error occured while running wkhtmltopdf. """ diff --git a/extract_msg/msg_classes/calendar_base.py b/extract_msg/msg_classes/calendar_base.py index a800935d..967e51e9 100644 --- a/extract_msg/msg_classes/calendar_base.py +++ b/extract_msg/msg_classes/calendar_base.py @@ -383,9 +383,9 @@ def organizer(self) -> Optional[str]: @functools.cached_property def ownerAppointmentID(self) -> Optional[int]: """ - A quasi-unique value amond all Calendar objects in a user's mailbox. + A quasi-unique value among all Calendar objects in a user's mailbox. Assists a client or server in finding a Calendar object but is not - guarenteed to be unique amoung all objects. + guaranteed to be unique among all objects. """ return self.getPropertyVal('00620003') diff --git a/extract_msg/msg_classes/message_base.py b/extract_msg/msg_classes/message_base.py index 03aa4a99..fd8ea30a 100644 --- a/extract_msg/msg_classes/message_base.py +++ b/extract_msg/msg_classes/message_base.py @@ -83,17 +83,8 @@ def __init__(self, path, **kwargs): self.__headerInit = False self.__recipientSeparator : str = kwargs.get('recipientSeparator', ';') self.__deencap = kwargs.get('deencapsulationFunc') - # Initialize properties in the order that is least likely to cause bugs. - # TODO have each function check for initialization of needed data so - # these lines will be unnecessary. - self.props self.header - self.recipients - self.to - self.cc - self.sender - self.date # This variable keeps track of what the new line character should be. self._crlf = '\n' try: @@ -101,8 +92,6 @@ def __init__(self, path, **kwargs): except Exception as e: # Prevent an error in the body from preventing opening. logger.exception('Critical error accessing the body. File opened but accessing the body will throw an exception.') - self.named - self.namedProperties except: try: self.close() diff --git a/extract_msg/msg_classes/msg.py b/extract_msg/msg_classes/msg.py index fb9d5394..861d7967 100644 --- a/extract_msg/msg_classes/msg.py +++ b/extract_msg/msg_classes/msg.py @@ -41,8 +41,8 @@ from ..properties.properties_store import PropertiesStore from ..structures.contact_link_entry import ContactLinkEntry from ..utils import ( - divide, hasLen, inputToMsgPath, makeWeakRef, msgPathToString, - parseType, verifyPropertyId, verifyType, windowsUnicode + divide, guessEncoding, hasLen, inputToMsgPath, makeWeakRef, + msgPathToString, parseType, verifyPropertyId, verifyType, windowsUnicode ) @@ -57,6 +57,8 @@ class MSGFile: Parser for .msg files. """ + filename : Optional[str] + def __init__(self, path, **kwargs): """ :param path: path to the msg file in the system or is the raw msg file. @@ -77,8 +79,10 @@ def __init__(self, path, **kwargs): :param errorBehavior: Optional, the behavior to use in the event of certain types of errors. Uses the ErrorBehavior enum. :param overrideEncoding: Optional, an encoding to use instead of the one - specified by the msg file. Do not report encoding errors caused by - this. + specified by the msg file. If the value is "chardet" and you have + the chardet module installed, an attempt will be made to + auto-detect the encoding based on some of the string properties. Do + not report encoding errors caused by this. :param treePath: Internal variable used for giving representation of the path, as a tuple of objects, of the MSGFile. When passing, this is the path to the parent object of this instance. @@ -124,9 +128,16 @@ def __init__(self, path, **kwargs): self.__dtFormat = kwargs.get('datetimeFormat', DT_FORMAT) if overrideEncoding is not None: - codecs.lookup(overrideEncoding) + if overrideEncoding.lower() == 'chardet': + encoding = guessEncoding(self) + if encoding: + self.__overrideEncoding = encoding + else: + logger.warning('Attempted to auto-detect encoding, but no consensus could be formed based on the top-level strings.') + else: + codecs.lookup(overrideEncoding) + self.__stringEncoding = overrideEncoding logger.warning('You have chosen to override the string encoding. Do not report encoding errors caused by this.') - self.__stringEncoding = overrideEncoding self.__overrideEncoding = overrideEncoding self.__listDirRes : Dict[Tuple[bool, bool, bool], List[List[str]]] = {} @@ -361,7 +372,7 @@ def _getTypedStream(self, filename : MSG_PATH, prefix : bool = True, _type = Non logger.error(f'Could not find matching VariableLengthProp for stream {x}') streams = len(contents) // (2 if _type in constants.MULTIPLE_2_BYTES else 4 if _type in constants.MULTIPLE_4_BYTES else 8 if _type in constants.MULTIPLE_8_BYTES else 16) else: - raise NotImplementedError(f'The stream specified is of type {_type}. We don\'t currently understand exactly how this type works. If it is mandatory that you have the contents of this stream, please create an issue labled "NotImplementedError: _getTypedStream {_type}".') + raise NotImplementedError(f'The stream specified is of type {_type}. We don\'t currently understand exactly how this type works. If it is mandatory that you have the contents of this stream, please create an issue labeled "NotImplementedError: _getTypedStream {_type}".') if _type in ('101F', '101E', '1102'): if self.exists(x + '-00000000', False): for y in range(streams): @@ -592,7 +603,7 @@ def getPropertyVal(self, name : Union[int, str], default : _T = None) -> Union[A """ instance.props.getValue(name, default) - Can be overriden to create new behavior. + Can be overridden to create new behavior. """ return self.props.getValue(name, default) @@ -716,7 +727,7 @@ def listDir(self, streams : bool = True, storages : bool = False, includePrefix Replacement for OleFileIO.listdir that runs at the current prefix directory. - :param includePrefix: If false, removed the part of the path that is the + :param includePrefix: If False, removes the part of the path that is the prefix. """ # Get the items from OleFileIO. @@ -797,7 +808,7 @@ def saveRaw(self, path) -> None: @functools.cached_property def areStringsUnicode(self) -> bool: """ - Returns a boolean telling if the strings are unicode encoded. + Returns a boolean telling if the strings are Unicode encoded. """ return (self.getPropertyVal('340D0003', 0) & 0x40000) != 0 @@ -993,7 +1004,7 @@ def namedProperties(self) -> NamedProperties: @property def overrideEncoding(self): """ - Returns None is the encoding has not been overriden, otherwise returns + Returns None is the encoding has not been overridden, otherwise returns the encoding. """ return self.__overrideEncoding @@ -1087,11 +1098,11 @@ def stringEncoding(self) -> str: return self.__stringEncoding except AttributeError: # We need to calculate the encoding. - # Let's first check if the encoding will be unicode: + # Let's first check if the encoding will be Unicode: if self.areStringsUnicode: self.__stringEncoding = "utf-16-le" else: - # Well, it's not unicode. Now we have to figure out what it IS. + # Well, it's not Unicode. Now we have to figure out what it IS. if '3FFD0003' not in self.props: # If this property is not set by the client, we SHOULD set # it to ISO-8859-15, but MAY set it to ISO-8859-1. diff --git a/extract_msg/recipient.py b/extract_msg/recipient.py index 549633c6..e68636b3 100644 --- a/extract_msg/recipient.py +++ b/extract_msg/recipient.py @@ -268,7 +268,7 @@ def getPropertyVal(self, name : Union[int, str], default : _T = None) -> Union[A """ instance.props.getValue(name, default) - Can be overriden to create new behavior. + Can be overridden to create new behavior. """ return self.props.getValue(name, default) @@ -378,6 +378,25 @@ class if it is found. return value + def listDir(self, streams : bool = True, storages : bool = False) -> List[List[str]]: + """ + Lists the streams and or storages that exist in the attachment + directory. + + Returns the paths *excluding* the attachment directory, allowing the + paths to be directly used for accessing a file. + """ + if (msg := self.__msg()) is None: + raise ReferenceError('The msg file for this Recipient instance has been garbage collected.') + return [path[1:] for path in msg.listDir(streams, storages, False) + if len(path) > 1 and path[0] == self.__dir] + + def slistDir(self, streams : bool = True, storages : bool = False) -> List[str]: + """ + Like listDir, except it returns the paths as strings. + """ + return ['/'.join(path) for path in self.listDir(streams, storages)] + @functools.cached_property def account(self) -> Optional[str]: """ diff --git a/extract_msg/structures/__init__.py b/extract_msg/structures/__init__.py index c4f0747e..6b1ebe8f 100644 --- a/extract_msg/structures/__init__.py +++ b/extract_msg/structures/__init__.py @@ -9,6 +9,7 @@ 'business_card', 'entry_id', 'misc_id', + 'odt', 'recurrence_pattern', 'report_tag', 'system_time', @@ -18,7 +19,7 @@ ] from . import ( - _helpers, contact_link_entry, business_card, entry_id, misc_id, + _helpers, contact_link_entry, business_card, entry_id, misc_id, odt, recurrence_pattern, report_tag, system_time, time_zone_definition, time_zone_struct, tz_rule ) \ No newline at end of file diff --git a/extract_msg/structures/odt.py b/extract_msg/structures/odt.py new file mode 100644 index 00000000..bb414645 --- /dev/null +++ b/extract_msg/structures/odt.py @@ -0,0 +1,72 @@ +__all__ = [ + 'Cf', + \ + 'ODTStruct', +] + +import struct +from typing import Optional + +from ..enums import ODTCf, ODTPersist1, ODTPersist2 + + +class ODTStruct: + def __init__(self, data : Optional[bytes]): + if data: + values = struct.unpack('= 6: + self.__persist2 = ODTPersist2(struct.unpack(' bytes: + return struct.pack(' ODTCf: + """ + An enum value that specifies the format this OLE object uses to + transmit data to the host application. + """ + return self.__cf + + @cf.setter + def setter(self, value : ODTCf) -> None: + if not isinstance(value, ODTCf): + raise TypeError(':property cf: MUST be of type ODTCf.') + + self.__cf = value + + @property + def odtPersist1(self) -> ODTPersist1: + """ + Flags the specify information about the OLE object. + """ + return self.__persist1 + + @odtPersist1.setter + def setter(self, value : ODTPersist1) -> None: + if not isinstance(value, ODTPersist1): + raise TypeError(':property odtPersist1: MUST be of type ODTPersist1.') + + self.__persist1 = value + + @property + def odtPersist2(self) -> ODTPersist1: + """ + Flags the specify additional information about the OLE object. + """ + return self.__persist2 + + @odtPersist2.setter + def setter(self, value : ODTPersist2) -> None: + if not isinstance(value, ODTPersist2): + raise TypeError(':property odtPersist2: MUST be of type ODTPersist2.') + + self.__persist2 = value diff --git a/extract_msg/utils.py b/extract_msg/utils.py index b579fca1..7bceb7ff 100644 --- a/extract_msg/utils.py +++ b/extract_msg/utils.py @@ -23,6 +23,7 @@ 'findWk', 'fromTimeStamp', 'getCommandArgs', + 'guessEncoding', 'hasLen', 'htmlSanitize', 'inputToBytes', @@ -72,15 +73,16 @@ from html import escape as htmlEscape from typing import ( - Any, Callable, Dict, Iterable, List, Optional, Sequence, TypeVar, + Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, TypeVar, TYPE_CHECKING, Union ) from . import constants from .enums import AttachmentType from .exceptions import ( - ConversionError, ExecutableNotFound, IncompatibleOptionsError, - InvaildPropertyIdError, TZError, UnknownTypeError + ConversionError, DependencyError, ExecutableNotFound, + IncompatibleOptionsError, InvaildPropertyIdError, TZError, + UnknownTypeError ) @@ -512,6 +514,38 @@ def getCommandArgs(args : Sequence[str]) -> argparse.Namespace: return options + +def guessEncoding(msg : MSGFile) -> Optional[str]: + """ + Analyzes the strings on an MSG file and attempts to form a consensus about the encoding based on the top-level strings. + + Returns None if no consensus could be formed. + + :raises DependencyError: chardet is not installed or could not be used + properly. + """ + try: + import chardet + except ImportError: + raise DependencyError('Cannot guess the encoding of an MSG file if chardet is not installed.') + + data = b'' + for name in (x[0] for x in msg.listDir(True, False, False) if len(x) == 1): + if name.lower().endswith('001f'): + # This is a guarentee. + return 'utf-16-le' + elif name.lower().endswith('001e'): + data += msg.getStream(name) + + try: + if not data or (result := chardet.detect(data))['confidence'] < 0.5: + return None + + return result['encoding'] + except Exception as e: + raise DependencyError(f'Failed to detect encoding: {e}') + + def hasLen(obj) -> bool: """ Checks if :param obj: has a __len__ attribute. diff --git a/notes/Custom Attachment CLSIDs.txt b/notes/Custom Attachment CLSIDs.txt index 1073b406..b03180fa 100644 --- a/notes/Custom Attachment CLSIDs.txt +++ b/notes/Custom Attachment CLSIDs.txt @@ -1,2 +1,3 @@ 00020D09-0000-0000-C000-000000000046: Seems to be a link to an outlook object. -00000316-0000-0000-C000-000000000046: Device Independent Bitmap. \ No newline at end of file +00000316-0000-0000-C000-000000000046: Device Independent Bitmap. +00000300-0000-0000-C000-000000000046: CLSID_StdOleLink \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index dc7d7bb5..92ce9a5e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,10 @@ universal=1 all = extract-msg[mime] extract-msg[image] + extract-msg[encoding] mime = python-magic>=0.4.27,<0.5 image = - Pillow>=9.5.0,<10 \ No newline at end of file + Pillow>=9.5.0,<10 +encoding = + chardet>=3.0.0,<6 # This can probably be unbound. \ No newline at end of file From 41657c9fc94afc0f5cab358889999ad624db0fc8 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 30 Aug 2023 17:06:57 -0700 Subject: [PATCH 31/68] Fixed errors --- CHANGELOG.md | 2 +- extract_msg/msg_classes/msg.py | 31 ++++++++++++++++--------------- extract_msg/properties/named.py | 2 +- extract_msg/utils.py | 2 +- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da04e19f..f207c468 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ **v0.46.0** -* [[TeamMsgExtractor #95](https://github.com/TeamMsgExtractor/msg-extractor/issues/95)] Adjusted the `overrideEncoding` property of `MSGFile` to allow automatic encoding detection. Simply set the property to the string `"chardet"` and, assuming the `chardet` module is installed, it will analyze a number of the strings to try and form a consensus about the encoding. This will *ignore* the specified encoding. +* [[TeamMsgExtractor #95](https://github.com/TeamMsgExtractor/msg-extractor/issues/95)] Adjusted the `overrideEncoding` property of `MSGFile` to allow automatic encoding detection. Simply set the property to the string `"chardet"` and, assuming the `chardet` module is installed, it will analyze a number of the strings to try and form a consensus about the encoding. This will *ignore* the specified encoding *only if* if successfully detects. Otherwise it will log a warning and fall back to the default behavior. * Changed the base class of `EntryID` from no base class to `abc.ABC`. * Added `position` property to `EntryID` to tell how many bytes were used to create the `EntryID`. * Added a number of properties to `MSGFile` from \[MS-OXCMSG\]. diff --git a/extract_msg/msg_classes/msg.py b/extract_msg/msg_classes/msg.py index 861d7967..f4d273a5 100644 --- a/extract_msg/msg_classes/msg.py +++ b/extract_msg/msg_classes/msg.py @@ -127,19 +127,6 @@ def __init__(self, path, **kwargs): self.__dateFormat = kwargs.get('dateFormat', DATE_FORMAT) self.__dtFormat = kwargs.get('datetimeFormat', DT_FORMAT) - if overrideEncoding is not None: - if overrideEncoding.lower() == 'chardet': - encoding = guessEncoding(self) - if encoding: - self.__overrideEncoding = encoding - else: - logger.warning('Attempted to auto-detect encoding, but no consensus could be formed based on the top-level strings.') - else: - codecs.lookup(overrideEncoding) - self.__stringEncoding = overrideEncoding - logger.warning('You have chosen to override the string encoding. Do not report encoding errors caused by this.') - self.__overrideEncoding = overrideEncoding - self.__listDirRes : Dict[Tuple[bool, bool, bool], List[List[str]]] = {} if self.__parentMsg: @@ -171,6 +158,8 @@ def __init__(self, path, **kwargs): # closing. We set it here for error handling. self.__oleOwner = True + self.__open = True + # The rest *must* be in a try-except block to ensure we close the file. try: kwargsCopy = copy.copy(kwargs) @@ -194,6 +183,20 @@ def __init__(self, path, **kwargs): self.__prefix = prefix self.__prefixList = prefixl self.__prefixLen = len(prefixl) + + if overrideEncoding is not None: + logger.warning('You have chosen to override the string encoding. Do not report encoding errors caused by this.') + if overrideEncoding.lower() == 'chardet': + encoding = guessEncoding(self) + if encoding: + self.__stringEncoding = encoding.lower() + else: + logger.warning('Attempted to auto-detect encoding, but no consensus could be formed based on the top-level strings. Defaulting to normal detection methods.') + else: + codecs.lookup(overrideEncoding) + self.__stringEncoding = overrideEncoding + self.__overrideEncoding = overrideEncoding + if prefix and not filename: filename = self.getStringStream(prefixl[:-1] + ['__substg1.0_3001'], prefix = False) if filename: @@ -208,8 +211,6 @@ def __init__(self, path, **kwargs): else: self.filename = None - self.__open = True - # Now, load the attachments if we are not delaying them. if not self.__attachmentsDelayed: self.attachments diff --git a/extract_msg/properties/named.py b/extract_msg/properties/named.py index ddf11b0f..fa5e95b9 100644 --- a/extract_msg/properties/named.py +++ b/extract_msg/properties/named.py @@ -235,7 +235,7 @@ def __init__(self, named : Named, streamSource : Union[MSGFile, AttachmentBase]) property. """ self.__named = named - self.__streamSource = makeWeakRef(streamSource) + self.__streamSource = weakref.ref(streamSource) def __getitem__(self, item): """ diff --git a/extract_msg/utils.py b/extract_msg/utils.py index 7bceb7ff..b6859d22 100644 --- a/extract_msg/utils.py +++ b/extract_msg/utils.py @@ -535,7 +535,7 @@ def guessEncoding(msg : MSGFile) -> Optional[str]: # This is a guarentee. return 'utf-16-le' elif name.lower().endswith('001e'): - data += msg.getStream(name) + data += msg.getStream(name) + b'\n' try: if not data or (result := chardet.detect(data))['confidence'] < 0.5: From c680097f5e52f00c64545ed0caeebb70eea4c185 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sat, 2 Sep 2023 04:06:40 -0700 Subject: [PATCH 32/68] Add more stuff to CustomAttachmentHandler --- .../custom_att_handler/__init__.py | 2 +- .../custom_att_handler/custom_handler.py | 29 ++- .../{jrnl_assoc_att.py => lnk_obj_att.py} | 240 ++++++++---------- extract_msg/utils.py | 4 +- 4 files changed, 133 insertions(+), 142 deletions(-) rename extract_msg/attachments/custom_att_handler/{jrnl_assoc_att.py => lnk_obj_att.py} (75%) diff --git a/extract_msg/attachments/custom_att_handler/__init__.py b/extract_msg/attachments/custom_att_handler/__init__.py index 26a5d9cb..825a8c74 100644 --- a/extract_msg/attachments/custom_att_handler/__init__.py +++ b/extract_msg/attachments/custom_att_handler/__init__.py @@ -57,7 +57,7 @@ def registerHandler(handler : Type[CustomAttachmentHandler]) -> None: # Import built-in handler modules. They will all automatically register their # respecive handler(s). from .outlook_image_dib import OutlookImageDIB -from .jrnl_assoc_att import LinkedObjectAttachment +from .lnk_obj_att import LinkedObjectAttachment if TYPE_CHECKING: diff --git a/extract_msg/attachments/custom_att_handler/custom_handler.py b/extract_msg/attachments/custom_att_handler/custom_handler.py index 7efea836..07124301 100644 --- a/extract_msg/attachments/custom_att_handler/custom_handler.py +++ b/extract_msg/attachments/custom_att_handler/custom_handler.py @@ -7,10 +7,13 @@ import abc +import functools from typing import Any, Callable, Dict, Optional, TYPE_CHECKING, TypeVar from ...constants import MSG_PATH +from ...structures.odt import ODTStruct +from ...structures.ole_pres import OLEPresentationStream from ...utils import msgPathToString @@ -30,12 +33,6 @@ def __init__(self, attachment : AttachmentBase): super().__init__() self.__att = attachment - def getPresentationStreams(self) -> Optional[Dict[int, bytes]]: - """ - Returns a dict of all presentation streams, as bytes. - """ - presLinks = [(x[-1][-3:], self.getStream(x[-1])) for x in self.attachment.listDir()] - def getStream(self, path : MSG_PATH) -> Optional[bytes]: """ Gets a stream from the custom data directory. @@ -107,4 +104,22 @@ def obj(self) -> Optional[object]: If there is no object to represent the custom attachment, including bytes, returns None. - """ \ No newline at end of file + """ + + @property + def objInfo(self) -> Optional[ODTStruct]: + """ + The structure representing the stream "\\x03ObjInfo", if it exists. + """ + self.getStreamAs('\x03ObjInfo', ODTStruct) + + @functools.cached_property + def presentationObjs(self) -> Optional[Dict[int, OLEPresentationStream]]: + """ + Returns a dict of all presentation streams, as bytes. + """ + return { + int(x[1][-3:]): self.getStreamAs(x[-1], OLEPresentationStream) + for x in self.attachment.listDir() + if x[0] == '__substg1.0_3701000D' and x[1].startswith('\x01OlePres') + } \ No newline at end of file diff --git a/extract_msg/attachments/custom_att_handler/jrnl_assoc_att.py b/extract_msg/attachments/custom_att_handler/lnk_obj_att.py similarity index 75% rename from extract_msg/attachments/custom_att_handler/jrnl_assoc_att.py rename to extract_msg/attachments/custom_att_handler/lnk_obj_att.py index 18d041ef..25592aa2 100644 --- a/extract_msg/attachments/custom_att_handler/jrnl_assoc_att.py +++ b/extract_msg/attachments/custom_att_handler/lnk_obj_att.py @@ -1,133 +1,109 @@ -from __future__ import annotations - - -__all__ = [ - 'LinkedObjectAttachment', -] - - -from functools import cached_property -from typing import List, Optional, TYPE_CHECKING - -from . import registerHandler -from .custom_handler import CustomAttachmentHandler -from ...structures.entry_id import EntryID -from ...structures.ole_pres import OLEPresentationStream - - -if TYPE_CHECKING: - from ..attachment_base import AttachmentBase - - -class LinkedObjectAttachment(CustomAttachmentHandler): - """ - A link to an Outlook object. - - Not *positive* I understand what this attachment type is, but this seems to - be the most likely name. Contains presentation data about how to render it - as well as properties with data that link to it. It looks *similar* to what - the documentation for Journal specifies would be it's custom attachment - type, however some small details don't perfectly add up. - - I've also only seen this on Journal objects thus far. - """ - - def __init__(self, attachment : AttachmentBase): - super().__init__(attachment) - stream = attachment.getStream('__substg1.0_3701000D/\x03MailStream') - if not stream: - raise ValueError('MailStream could not be found.') - if len(stream) != 12: - raise ValueError('MailStream is the wrong length.') - - @classmethod - def isCorrectHandler(cls, attachment : AttachmentBase) -> bool: - if attachment.clsid != '00020D09-0000-0000-C000-000000000046': - return False - - return True - - def generateRtf(self) -> Optional[bytes]: - # TODO - return None - - @property - def data(self) -> None: - # This type of attachment has no direct associated data. - return None - - @cached_property - def mailMsgAttFld(self) -> Optional[EntryID]: - """ - The EntryID of the folder of the linked Message object. - """ - return EntryID.autoCreate(self.getStream('MailMsgAttFld')) - - @cached_property - def mailMsgAttMdb(self) -> Optional[EntryID]: - """ - The EntryID of the store of the linked Message object. - """ - return EntryID.autoCreate(self.getStream('MailMsgAttMdb')) - - @cached_property - def mailMsgAttMsg(self) -> Optional[EntryID]: - """ - The EntryID linked Message object; required only if the - mailMsgAttSrchKey property is None. - """ - return EntryID.autoCreate(self.getStream('MailMsgAttMsg')) - - @cached_property - def mailMsgAttSrchFld(self) -> Optional[EntryID]: - """ - The object EntryID of the Sent Items special folder of the linked - Message object. - """ - return EntryID.autoCreate(self.getStream('MailMsgAttSrchFld')) - - @cached_property - def mailMsgAttSrchKey(self) -> Optional[bytes]: - """ - The search key for the linked message object; required only if - mailMsgAttMsg is None. - """ - return self.getStream('MailMsgAttSrchKey') - - @cached_property - def presentationStreams(self) -> Optional[List[OLEPresentationStream]]: - """ - The presentation streams, as a list of OLEPresentationStream object. - - From my understanding, this MUST be set, but we are treating it as - SHOULD be set. It also looks like the correct number to have is exactly - 1, but I can't guarantee that, so this is a list. - """ - if self.presentationStreamsBytes: - return [OLEPresentationStream(x) for x in self.presentationStreams] - else: - return None - - @cached_property - def presentationStreamsBytes(self) -> Optional[List[bytes]]: - """ - The presentation streams, as a list of bytes. - - From my understanding, there should exist EXACTLY 1 of these, but... - this appears to be an undocumented custom attachment that is easy to - make. - """ - - @property - def name(self) -> None: - # Doesn't save. - return None - - @property - def obj(self) -> None: - # No object to represent this. - return None - - - +from __future__ import annotations + + +__all__ = [ + 'LinkedObjectAttachment', +] + + +from functools import cached_property +from typing import List, Optional, TYPE_CHECKING + +from . import registerHandler +from .custom_handler import CustomAttachmentHandler +from ...structures.entry_id import EntryID +from ...structures.ole_pres import OLEPresentationStream + + +if TYPE_CHECKING: + from ..attachment_base import AttachmentBase + + +class LinkedObjectAttachment(CustomAttachmentHandler): + """ + A link to an Outlook object. + + Not *positive* I understand what this attachment type is, but this seems to + be the most likely name. Contains presentation data about how to render it + as well as properties with data that link to it. It looks *similar* to what + the documentation for Journal specifies would be it's custom attachment + type, however some small details don't perfectly add up. + + I've also only seen this on Journal objects thus far. + """ + + def __init__(self, attachment : AttachmentBase): + super().__init__(attachment) + stream = attachment.getStream('__substg1.0_3701000D/\x03MailStream') + if not stream: + raise ValueError('MailStream could not be found.') + if len(stream) != 12: + raise ValueError('MailStream is the wrong length.') + + @classmethod + def isCorrectHandler(cls, attachment : AttachmentBase) -> bool: + if attachment.clsid != '00020D09-0000-0000-C000-000000000046': + return False + + return True + + def generateRtf(self) -> Optional[bytes]: + # TODO + return None + + @property + def data(self) -> None: + # This type of attachment has no direct associated data. + return None + + @cached_property + def mailMsgAttFld(self) -> Optional[EntryID]: + """ + The EntryID of the folder of the linked Message object. + """ + return EntryID.autoCreate(self.getStream('MailMsgAttFld')) + + @cached_property + def mailMsgAttMdb(self) -> Optional[EntryID]: + """ + The EntryID of the store of the linked Message object. + """ + return EntryID.autoCreate(self.getStream('MailMsgAttMdb')) + + @cached_property + def mailMsgAttMsg(self) -> Optional[EntryID]: + """ + The EntryID linked Message object; required only if the + mailMsgAttSrchKey property is None. + """ + return EntryID.autoCreate(self.getStream('MailMsgAttMsg')) + + @cached_property + def mailMsgAttSrchFld(self) -> Optional[EntryID]: + """ + The object EntryID of the Sent Items special folder of the linked + Message object. + """ + return EntryID.autoCreate(self.getStream('MailMsgAttSrchFld')) + + @cached_property + def mailMsgAttSrchKey(self) -> Optional[bytes]: + """ + The search key for the linked message object; required only if + mailMsgAttMsg is None. + """ + return self.getStream('MailMsgAttSrchKey') + + @property + def name(self) -> None: + # Doesn't save. + return None + + @property + def obj(self) -> None: + # No object to represent this. + return None + + + registerHandler(LinkedObjectAttachment) \ No newline at end of file diff --git a/extract_msg/utils.py b/extract_msg/utils.py index b6859d22..8d26b53c 100644 --- a/extract_msg/utils.py +++ b/extract_msg/utils.py @@ -73,7 +73,7 @@ from html import escape as htmlEscape from typing import ( - Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, TypeVar, + Any, Callable, Dict, Iterable, List, Optional, Sequence, TypeVar, TYPE_CHECKING, Union ) @@ -979,7 +979,7 @@ def tryGetMimetype(att : AttachmentBase, mimetype : Union[str, None]) -> Union[s if att.dataType: # Try to import our dependency module to use it. try: - import magic + import magic # pyright: ignore if isinstance(att.data, (str, bytes)): return magic.from_buffer(att.data, mime = True) From ce8dbd939ade1bf64cb094a0a6be4fef5f502c2b Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sat, 2 Sep 2023 05:34:57 -0700 Subject: [PATCH 33/68] Work on more structures for custom attachments --- .../custom_att_handler/custom_handler.py | 12 ++++- extract_msg/structures/__init__.py | 8 +-- extract_msg/structures/mon_stream.py | 49 +++++++++++++++++++ extract_msg/structures/ole_stream_struct.py | 34 +++++++++++++ 4 files changed, 98 insertions(+), 5 deletions(-) create mode 100644 extract_msg/structures/mon_stream.py create mode 100644 extract_msg/structures/ole_stream_struct.py diff --git a/extract_msg/attachments/custom_att_handler/custom_handler.py b/extract_msg/attachments/custom_att_handler/custom_handler.py index 07124301..7903dede 100644 --- a/extract_msg/attachments/custom_att_handler/custom_handler.py +++ b/extract_msg/attachments/custom_att_handler/custom_handler.py @@ -14,6 +14,7 @@ from ...constants import MSG_PATH from ...structures.odt import ODTStruct from ...structures.ole_pres import OLEPresentationStream +from ...structures.ole_stream_struct import OleStreamStruct from ...utils import msgPathToString @@ -106,12 +107,19 @@ def obj(self) -> Optional[object]: bytes, returns None. """ - @property + @functools.cached_property def objInfo(self) -> Optional[ODTStruct]: """ The structure representing the stream "\\x03ObjInfo", if it exists. """ - self.getStreamAs('\x03ObjInfo', ODTStruct) + return self.getStreamAs('\x03ObjInfo', ODTStruct) + + @functools.cached_property + def ole(self) -> Optional[OleStreamStruct]: + """ + The structure representing the stream "\\x01Ole", if it exists. + """ + return self.getStreamAs('\x01Ole', OleStreamStruct) @functools.cached_property def presentationObjs(self) -> Optional[Dict[int, OLEPresentationStream]]: diff --git a/extract_msg/structures/__init__.py b/extract_msg/structures/__init__.py index 6b1ebe8f..1cbd229d 100644 --- a/extract_msg/structures/__init__.py +++ b/extract_msg/structures/__init__.py @@ -10,6 +10,8 @@ 'entry_id', 'misc_id', 'odt', + 'ole_pres', + 'ole_stream_struct', 'recurrence_pattern', 'report_tag', 'system_time', @@ -19,7 +21,7 @@ ] from . import ( - _helpers, contact_link_entry, business_card, entry_id, misc_id, odt, - recurrence_pattern, report_tag, system_time, time_zone_definition, - time_zone_struct, tz_rule + _helpers, contact_link_entry, business_card, entry_id, misc_id, odt, + ole_pres, ole_stream_struct, recurrence_pattern, report_tag, + system_time, time_zone_definition, time_zone_struct, tz_rule ) \ No newline at end of file diff --git a/extract_msg/structures/mon_stream.py b/extract_msg/structures/mon_stream.py new file mode 100644 index 00000000..786f329c --- /dev/null +++ b/extract_msg/structures/mon_stream.py @@ -0,0 +1,49 @@ +__all__ = [ + 'MonikerStream', +] + + +from typing import Optional + + +class MonikerStream: + def __init__(self, data : Optional[bytes]): + if data: + self.__clsid = data[:16] + self.__streamData = data[16:] + else: + self.__clsid = b'\x00' * 16 + self.__streamData = b'' + + def toBytes(self) -> bytes: + pass # TODO + + @property + def clsid(self) -> bytes: + """ + The CLSID, as a stream of 16 bytes, of an implementation specific object + capable of processing the stream data. + """ + return self.__clsid + + @clsid.setter + def setter(self, data : bytes) -> None: + if not isinstance(data, bytes): + raise TypeError('CLSID MUST be bytes.') + if len(data) != 16: + raise ValueError('CLSID MUST be 16 bytes.') + self.__clsid = data + + @property + def streamData(self) -> bytes: + """ + An array of bytes that specifies the reference to the linked object. + """ + return self.__streamData + + @streamData.setter + def setter(self, data : bytes) -> None: + if not isinstance(data, bytes): + raise TypeError('Stream data MUST be bytes.') + self.__streamData = data + diff --git a/extract_msg/structures/ole_stream_struct.py b/extract_msg/structures/ole_stream_struct.py new file mode 100644 index 00000000..fadae9e6 --- /dev/null +++ b/extract_msg/structures/ole_stream_struct.py @@ -0,0 +1,34 @@ +__all__ = [ + 'OleStreamStruct', +] + + +from typing import Optional + +from ._helpers import BytesReader +from .mon_stream import MonikerStream + + +class OleStreamStruct: + def __init__(self, data : Optional[bytes] = None): + self.__reservedMonikerStream = None + if not data: + self.__flags = 0 + self.__linkUpdateOption = 0 + return + reader = BytesReader(data) + # Assert the version. + reader.assertRead(b'\x01\x00\x00\x02', 'Ole stream had invalid version (expected {expected}, got {actual}).') + self.__flags = reader.readUnsignedInt() + self.__linkUpdateOption = reader.readUnsignedInt() + reader.assertNull(4, 'Ole stream reserved was not null (got {actual}).') + rmsSize = reader.readUnsignedInt() + if rmsSize > 0: + self.__reservedMonikerStream = MonikerStream(reader.read(rmsSize)) + + # TODO implement the rest. It's all optional things. + + def toBytes(self) -> bytes: + pass # TODO + + From eba824ace70116d536483ebdf1938cb00052f5a0 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Mon, 4 Sep 2023 05:00:48 -0700 Subject: [PATCH 34/68] Typing and struct adjustments --- CHANGELOG.md | 2 +- extract_msg/constants/__init__.py | 55 ++++++++-------- extract_msg/constants/ps.py | 38 +++++------ extract_msg/constants/re.py | 13 ++-- extract_msg/constants/st.py | 101 +++++++++++++++--------------- extract_msg/properties/prop.py | 20 +++--- extract_msg/utils.py | 8 +-- 7 files changed, 119 insertions(+), 118 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f207c468..16f443cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,7 @@ * Changed the `NotImplementedError` for custom attachment handler not being found to `FeatureNotImplemented`. Additionally, changed the error message to specify the CLSID found on the attachment to better enable people to report issues. * Changed code for `Recipient` and `MessageBase` that makes it rely on `MessageBase.recipientTypeClass` to determine the class to use for the `recipientType` property. Adjusted the typing of `Recipient` to have it reflect the type that will be used. * Correctly changed the returned value for `ResponseStatus.fromIter` to actually return a List instead of a set. -* Filled out typing information for a significant portion of the module where variables or functions were missing it. +* Filled out typing information for a significant portion of the module where variables or functions were missing it. This includes the entirety of the constants submodule. * Corrected a number of minor issues. * Extended values for `DVAspect` enum. * Added new enums to go with parsing for `OLEPresentationStream`. diff --git a/extract_msg/constants/__init__.py b/extract_msg/constants/__init__.py index 34fd6681..38aca5e8 100644 --- a/extract_msg/constants/__init__.py +++ b/extract_msg/constants/__init__.py @@ -43,17 +43,16 @@ ] -import datetime - -from typing import Dict, List, Optional, Tuple, Union +from datetime import datetime +from typing import Dict, Final, List, Optional, Tuple, Union from . import ps, re, st from ..enums import SaveType # Constants for formating datetime objects. -DATE_FORMAT = '%d %B, %Y' -DT_FORMAT = '%a, %d %b %Y %H:%M:%S %z' +DATE_FORMAT : Final[str] = '%d %B, %Y' +DT_FORMAT : Final[str] = '%a, %d %b %Y %H:%M:%S %z' # Typing Constants. @@ -65,7 +64,7 @@ -FIXED_LENGTH_PROPS = ( +FIXED_LENGTH_PROPS : Final[Tuple[int, ...]] = ( 0x0000, 0x0001, 0x0002, @@ -81,7 +80,7 @@ 0x0048, ) -FIXED_LENGTH_PROPS_STRING = ( +FIXED_LENGTH_PROPS_STRING : Final[Tuple[str, ...]] = ( '0000', '0001', '0002', @@ -97,7 +96,7 @@ '0048', ) -VARIABLE_LENGTH_PROPS = ( +VARIABLE_LENGTH_PROPS : Final[Tuple[int, ...]] = ( 0x000D, 0x001E, 0x001F, @@ -119,7 +118,7 @@ 0x1102, ) -VARIABLE_LENGTH_PROPS_STRING = ( +VARIABLE_LENGTH_PROPS_STRING : Final[Tuple[str, ...]] = ( '000D', '001E', '001F', @@ -142,34 +141,34 @@ ) # Multiple type properties that take up 2 bytes. -MULTIPLE_2_BYTES = ( +MULTIPLE_2_BYTES : Final[Tuple[str, ...]] = ( '1002', ) -MULTIPLE_2_BYTES_HEX = ( +MULTIPLE_2_BYTES_HEX : Final[Tuple[int, ...]] = ( 0x1002, ) # Multiple type properties that take up 4 bytes. -MULTIPLE_4_BYTES = ( +MULTIPLE_4_BYTES : Final[Tuple[str, ...]] = ( '1003', '1004', ) -MULTIPLE_4_BYTES_HEX = ( +MULTIPLE_4_BYTES_HEX : Final[Tuple[int, ...]] = ( 0x1003, 0x1004, ) # Multiple type properties that take up 8 bytes. -MULTIPLE_8_BYTES = ( +MULTIPLE_8_BYTES : Final[Tuple[str, ...]] = ( '1005', '1007', '1014', '1040', ) -MULTIPLE_8_BYTES_HEX = ( +MULTIPLE_8_BYTES_HEX : Final[Tuple[int, ...]] = ( 0x1005, 0x1007, 0x1014, @@ -177,17 +176,17 @@ ) # Multiple type properties that take up 16 bytes. -MULTIPLE_16_BYTES = ( +MULTIPLE_16_BYTES : Final[Tuple[str, ...]] = ( '1048', ) -MULTIPLE_16_BYTES_HEX = ( +MULTIPLE_16_BYTES_HEX : Final[Tuple[int, ...]] = ( 0x1048, ) # Used to format the header for saving only the header. -HEADER_FORMAT = """From: {From} +HEADER_FORMAT : Final[str] = """From: {From} To: {To} Cc: {Cc} Bcc: {Bcc} @@ -197,7 +196,7 @@ """ -KNOWN_CLASS_TYPES = ( +KNOWN_CLASS_TYPES : Final[Tuple[str, ...]] = ( 'ipm.activity', 'ipm.appointment', # [MS-OXOCAL] 'ipm.contact', # [MS-OXOCNTC] @@ -221,31 +220,31 @@ # Each item is a tuple of the lowercase class type and the issue number # associated with it. -REFUSED_CLASS_TYPES = ( +REFUSED_CLASS_TYPES : Final[Tuple[Tuple[str, str], ...]] = ( ('ipm.outlook.recall', '235'), ) -PYTPFLOATINGTIME_START = datetime.datetime(1899, 12, 30) -NULL_DATE = datetime.datetime(4500, 8, 31, 23, 59) +PYTPFLOATINGTIME_START : Final[datetime] = datetime(1899, 12, 30) +NULL_DATE : Final[datetime] = datetime(4500, 8, 31, 23, 59) # Constants used for argparse stuff. -KNOWN_FILE_FLAGS = ( +KNOWN_FILE_FLAGS : Final[Tuple[str, ...]] = ( '--out-name', ) -NEEDS_ARG = ( +NEEDS_ARG : Final[Tuple[str, ...]]= ( '--out-name', ) -REPOSITORY_URL = 'https://github.com/TeamMsgExtractor/msg-extractor' -MAINDOC = f"""extract_msg: +REPOSITORY_URL : Final[str] = 'https://github.com/TeamMsgExtractor/msg-extractor' +MAINDOC : Final[str] = f"""extract_msg: \tExtracts emails and attachments saved in Microsoft Outlook's .msg files. {REPOSITORY_URL}""" # Default class ID for the root entry for OleWriter. This should be # referencing Outlook if I understand it correctly. -DEFAULT_CLSID = b'\x0b\r\x02\x00\x00\x00\x00\x00\xc0\x00\x00\x00\x00\x00\x00F' +DEFAULT_CLSID : Final[bytes] = b'\x0b\r\x02\x00\x00\x00\x00\x00\xc0\x00\x00\x00\x00\x00\x00F' -PTYPES = { +PTYPES : Final[Dict[int, str]]= { 0x0000: 'PtypUnspecified', 0x0001: 'PtypNull', 0x0002: 'PtypInteger16', # Signed short. diff --git a/extract_msg/constants/ps.py b/extract_msg/constants/ps.py index eb1bf167..3dff1af0 100644 --- a/extract_msg/constants/ps.py +++ b/extract_msg/constants/ps.py @@ -23,22 +23,24 @@ 'PS_PUBLIC_STRINGS', ] +from typing import Final -PS_MAPI = '{00020328-0000-0000-C000-000000000046}' -PS_PUBLIC_STRINGS = '{00020329-0000-0000-C000-000000000046}' -PSETID_COMMON = '{00062008-0000-0000-C000-000000000046}' -PSETID_ADDRESS = '{00062004-0000-0000-C000-000000000046}' -PS_INTERNET_HEADERS = '{00020386-0000-0000-C000-000000000046}' -PSETID_APPOINTMENT = '{00062002-0000-0000-C000-000000000046}' -PSETID_MEETING = '{6ED8DA90-450B-101B-98DA-00AA003F1305}' -PSETID_LOG = '{0006200A-0000-0000-C000-000000000046}' -PSETID_MESSAGING = '{41F28F13-83F4-4114-A584-EEDB5A6B0BFF}' -PSETID_NOTE = '{0006200E-0000-0000-C000-000000000046}' -PSETID_POSTRSS = '{00062041-0000-0000-C000-000000000046}' -PSETID_TASK = '{00062003-0000-0000-C000-000000000046}' -PSETID_UNIFIEDMESSAGING = '{4442858E-A9E3-4E80-B900-317A210CC15B}' -PSETID_AIRSYNC = '{71035549-0739-4DCB-9163-00F0580DBBDF}' -PSETID_SHARING = '{00062040-0000-0000-C000-000000000046}' -PSETID_XMLEXTRACTEDENTITIES = '{23239608-685D-4732-9C55-4C95CB4E8E33}' -PSETID_ATTACHMENT = '{96357F7F-59E1-47D0-99A7-46515C183B54}' -PSETID_CALENDAR_ASSISTANT = '{11000E07-B51B-40D6-AF21-CAA85EDAB1D0}' \ No newline at end of file + +PS_MAPI : Final[str] = '{00020328-0000-0000-C000-000000000046}' +PS_PUBLIC_STRINGS : Final[str] = '{00020329-0000-0000-C000-000000000046}' +PSETID_COMMON : Final[str] = '{00062008-0000-0000-C000-000000000046}' +PSETID_ADDRESS : Final[str] = '{00062004-0000-0000-C000-000000000046}' +PS_INTERNET_HEADERS : Final[str] = '{00020386-0000-0000-C000-000000000046}' +PSETID_APPOINTMENT : Final[str] = '{00062002-0000-0000-C000-000000000046}' +PSETID_MEETING : Final[str] = '{6ED8DA90-450B-101B-98DA-00AA003F1305}' +PSETID_LOG : Final[str] = '{0006200A-0000-0000-C000-000000000046}' +PSETID_MESSAGING : Final[str] = '{41F28F13-83F4-4114-A584-EEDB5A6B0BFF}' +PSETID_NOTE : Final[str] = '{0006200E-0000-0000-C000-000000000046}' +PSETID_POSTRSS : Final[str] = '{00062041-0000-0000-C000-000000000046}' +PSETID_TASK : Final[str] = '{00062003-0000-0000-C000-000000000046}' +PSETID_UNIFIEDMESSAGING : Final[str] = '{4442858E-A9E3-4E80-B900-317A210CC15B}' +PSETID_AIRSYNC : Final[str] = '{71035549-0739-4DCB-9163-00F0580DBBDF}' +PSETID_SHARING : Final[str] = '{00062040-0000-0000-C000-000000000046}' +PSETID_XMLEXTRACTEDENTITIES : Final[str] = '{23239608-685D-4732-9C55-4C95CB4E8E33}' +PSETID_ATTACHMENT : Final[str] = '{96357F7F-59E1-47D0-99A7-46515C183B54}' +PSETID_CALENDAR_ASSISTANT : Final[str] = '{11000E07-B51B-40D6-AF21-CAA85EDAB1D0}' \ No newline at end of file diff --git a/extract_msg/constants/re.py b/extract_msg/constants/re.py index 3300c1b9..deadde3a 100644 --- a/extract_msg/constants/re.py +++ b/extract_msg/constants/re.py @@ -2,6 +2,7 @@ Regular expression constants. """ + __all__ = [ 'HTML_BODY_START', 'HTML_SAN_SPACE', @@ -13,17 +14,19 @@ import re +from typing import Final + # Characters that are invalid in a filename. -INVALID_FILENAME_CHARS = re.compile(r'[\\/:*?"<>|]') +INVALID_FILENAME_CHARS : Final[re.Pattern[str]] = re.compile(r'[\\/:*?"<>|]') # Regular expression to find sections of spaces for htmlSanitize. -HTML_SAN_SPACE = re.compile(' +') +HTML_SAN_SPACE : Final[re.Pattern[str]] = re.compile(' +') # Regular expression to find the start of the html body. -HTML_BODY_START = re.compile(b']*>') +HTML_BODY_START : Final[re.Pattern[bytes]] = re.compile(b']*>') # Regular expression to find the start of the html body in encapsulated RTF. # This is used for one of the pattern types that makes life easy. -RTF_ENC_BODY_START = re.compile(br'\{\\\*\\htmltag[0-9]* ?]*>\}') +RTF_ENC_BODY_START : Final[re.Pattern[bytes]] = re.compile(br'\{\\\*\\htmltag[0-9]* ?]*>\}') # Used in the vaildation of OLE paths. Any of these characters in a name make it # invalid. -INVALID_OLE_PATH = re.compile(r'[:/\\!]') +INVALID_OLE_PATH : Final[re.Pattern[str]] = re.compile(r'[:/\\!]') diff --git a/extract_msg/constants/st.py b/extract_msg/constants/st.py index 64d68353..137d951f 100644 --- a/extract_msg/constants/st.py +++ b/extract_msg/constants/st.py @@ -5,7 +5,6 @@ __all__ = [ 'ST1', 'ST2', - 'ST3', 'STF32', 'STF64', 'STFIX', @@ -37,8 +36,6 @@ 'ST_BE_UI8', 'ST_CF_DIR_ENTRY', 'ST_DATA_UI16', - 'ST_DATA_UI32', - 'ST_DATA_UI8', 'ST_GUID', 'ST_LE_F32', 'ST_LE_F64', @@ -57,70 +54,70 @@ import struct +from typing import Final + # Define pre-compiled structs to make unpacking slightly faster. # General structs. -ST1 = struct.Struct('<8x4I') -ST2 = struct.Struct('b') -ST_BE_I16 = struct.Struct('>h') -ST_BE_I32 = struct.Struct('>i') -ST_BE_I64 = struct.Struct('>q') -ST_BE_UI8 = struct.Struct('>B') -ST_BE_UI16 = struct.Struct('>H') -ST_BE_UI32 = struct.Struct('>I') -ST_BE_UI64 = struct.Struct('>Q') -ST_BE_F32 = struct.Struct('>f') -ST_BE_F64 = struct.Struct('>d') \ No newline at end of file +ST_LE_I8 : Final[struct.Struct] = STI8 +ST_LE_I16 : Final[struct.Struct] = STMI16 +ST_LE_I32 : Final[struct.Struct] = STMI32 +ST_LE_I64 : Final[struct.Struct] = STMI64 +ST_LE_UI8 : Final[struct.Struct] = struct.Struct('b') +ST_BE_I16 : Final[struct.Struct] = struct.Struct('>h') +ST_BE_I32 : Final[struct.Struct] = struct.Struct('>i') +ST_BE_I64 : Final[struct.Struct] = struct.Struct('>q') +ST_BE_UI8 : Final[struct.Struct] = struct.Struct('>B') +ST_BE_UI16 : Final[struct.Struct] = struct.Struct('>H') +ST_BE_UI32 : Final[struct.Struct] = struct.Struct('>I') +ST_BE_UI64 : Final[struct.Struct] = struct.Struct('>Q') +ST_BE_F32: Final[struct.Struct] = struct.Struct('>f') +ST_BE_F64 : Final[struct.Struct] = struct.Struct('>d') \ No newline at end of file diff --git a/extract_msg/properties/prop.py b/extract_msg/properties/prop.py index 663a1ed3..f09ef64d 100644 --- a/extract_msg/properties/prop.py +++ b/extract_msg/properties/prop.py @@ -131,20 +131,20 @@ def parseType(self, _type : int, stream : bytes) -> Any: logger.warning('Property type is PtypNull, but is not equal to 0.') value = None elif _type == 0x0002: # PtypInteger16 - value = constants.st.STI16.unpack(value)[0] + value = constants.st.ST_LE_I16.unpack(value)[0] elif _type == 0x0003: # PtypInteger32 - value = constants.st.STI32.unpack(value)[0] + value = constants.st.ST_LE_I32.unpack(value)[0] elif _type == 0x0004: # PtypFloating32 - value = constants.st.STF32.unpack(value)[0] + value = constants.st.ST_LE_F32.unpack(value)[0] elif _type == 0x0005: # PtypFloating64 - value = constants.st.STF64.unpack(value)[0] + value = constants.st.ST_LE_F64.unpack(value)[0] elif _type == 0x0006: # PtypCurrency - value = (constants.st.STI64.unpack(value))[0] / 10000.0 + value = (constants.st.ST_LE_I64.unpack(value))[0] / 10000.0 elif _type == 0x0007: # PtypFloatingTime - value = constants.st.STF64.unpack(value)[0] + value = constants.st.ST_LE_F64.unpack(value)[0] return constants.PYTPFLOATINGTIME_START + datetime.timedelta(days = value) elif _type == 0x000A: # PtypErrorCode - value = constants.st.STI32.unpack(value)[0] + value = constants.st.ST_LE_I32.unpack(value)[0] try: value = ErrorCodeType(value) except ValueError: @@ -157,11 +157,11 @@ def parseType(self, _type : int, stream : bytes) -> Any: except ValueError: pass elif _type == 0x000B: # PtypBoolean - value = constants.st.ST3.unpack(value)[0] == 1 + value = constants.st.ST_LE_UI64.unpack(value)[0] == 1 elif _type == 0x0014: # PtypInteger64 - value = constants.st.STI64.unpack(value)[0] + value = constants.st.ST_LE_I64.unpack(value)[0] elif _type == 0x0040: # PtypTime - rawTime = constants.st.ST3.unpack(value)[0] + rawTime = constants.st.ST_LE_UI64.unpack(value)[0] try: value = filetimeToDatetime(rawTime) except ValueError as e: diff --git a/extract_msg/utils.py b/extract_msg/utils.py index 8d26b53c..639c4257 100644 --- a/extract_msg/utils.py +++ b/extract_msg/utils.py @@ -728,7 +728,7 @@ def parseType(_type : int, stream, encoding, extras): pass return value elif _type == 0x000B: # PtypBoolean - return constants.st.ST3.unpack(value)[0] == 1 + return constants.st.ST_LE_UI64.unpack(value)[0] == 1 elif _type == 0x000D: # PtypObject/PtypEmbeddedTable # TODO parsing for this. # Wait, that's the extension for an attachment folder, so parsing this @@ -742,12 +742,12 @@ def parseType(_type : int, stream, encoding, extras): elif _type == 0x001F: # PtypString return value.decode('utf-16-le') elif _type == 0x0040: # PtypTime - rawTime = constants.st.ST3.unpack(value)[0] + rawTime = constants.st.ST_LE_UI64.unpack(value)[0] return filetimeToDatetime(rawTime) elif _type == 0x0048: # PtypGuid return bytesToGuid(value) elif _type == 0x00FB: # PtypServerId - count = constants.st.STUI16.unpack(value[:2]) + count = constants.st.ST_LE_UI16.unpack(value[:2]) # If the first byte is a 1 then it uses the ServerID structure. if value[3] == 1: from .structures.misc_id import ServerID @@ -801,7 +801,7 @@ def parseType(_type : int, stream, encoding, extras): if _type == 0x1014: # PtypMultipleInteger64 return tuple(constants.st.STMI64.unpack(x)[0] for x in extras) if _type == 0x1040: # PtypMultipleTime - return tuple(filetimeToUtc(constants.st.ST3.unpack(x)[0]) for x in extras) + return tuple(filetimeToUtc(constants.st.ST_LE_UI64.unpack(x)[0]) for x in extras) if _type == 0x1048: # PtypMultipleGuid return tuple(bytesToGuid(x) for x in extras) else: From 974eadf83b91638d086b25613cd8d837bfd96c85 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Mon, 4 Sep 2023 05:01:44 -0700 Subject: [PATCH 35/68] typing and tobytes --- extract_msg/structures/ole_stream_struct.py | 30 ++++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/extract_msg/structures/ole_stream_struct.py b/extract_msg/structures/ole_stream_struct.py index fadae9e6..6ff8eaee 100644 --- a/extract_msg/structures/ole_stream_struct.py +++ b/extract_msg/structures/ole_stream_struct.py @@ -3,15 +3,17 @@ ] -from typing import Optional +from typing import final, Optional +from ..constants import st from ._helpers import BytesReader from .mon_stream import MonikerStream +@final class OleStreamStruct: def __init__(self, data : Optional[bytes] = None): - self.__reservedMonikerStream = None + self.__rms = None if not data: self.__flags = 0 self.__linkUpdateOption = 0 @@ -24,11 +26,31 @@ def __init__(self, data : Optional[bytes] = None): reader.assertNull(4, 'Ole stream reserved was not null (got {actual}).') rmsSize = reader.readUnsignedInt() if rmsSize > 0: - self.__reservedMonikerStream = MonikerStream(reader.read(rmsSize)) + self.__rms = MonikerStream(reader.read(rmsSize)) # TODO implement the rest. It's all optional things. def toBytes(self) -> bytes: - pass # TODO + ret = b'\x01\x00\x00\x02' + ret += st.ST_LE_UI32.pack(self.__flags) + ret += st.ST_LE_UI32.pack(self.__linkUpdateOption) + ret += b'\x00\x00\x00\x00' + rmsBytes = b'' if self.__rms is None else self.__rms.toBytes() + ret += st.ST_LE_UI32.pack(len(rmsBytes)) + rmsBytes + # TODO finish this with the optional properties. + + return ret + + @property + def reservedMonikerStream(self) -> Optional[MonikerStream]: + """ + + """ + return self.__rms + + @reservedMonikerStream.setter + def setter(self, data : Optional[MonikerStream]) -> None: + if data is not None and not isinstance(data, MonikerStream): + raise TypeError('Reserved moniker stream must be a MonikerStream instance or None.') From 799646677e4315e2d96ab57cb01f788f3bcee466 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Mon, 4 Sep 2023 05:36:38 -0700 Subject: [PATCH 36/68] Finsih setter for OleStreamStruct --- extract_msg/structures/ole_stream_struct.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extract_msg/structures/ole_stream_struct.py b/extract_msg/structures/ole_stream_struct.py index 6ff8eaee..3ef48d7e 100644 --- a/extract_msg/structures/ole_stream_struct.py +++ b/extract_msg/structures/ole_stream_struct.py @@ -53,4 +53,6 @@ def setter(self, data : Optional[MonikerStream]) -> None: if data is not None and not isinstance(data, MonikerStream): raise TypeError('Reserved moniker stream must be a MonikerStream instance or None.') + self.__rms = data + From 95f972d83e10afd7a8a9525f21e67bfd55a6eb03 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Mon, 4 Sep 2023 07:00:08 -0700 Subject: [PATCH 37/68] More adjustments for writing --- CHANGELOG.md | 2 + extract_msg/enums.py | 8 ++++ extract_msg/properties/prop.py | 39 +++---------------- extract_msg/properties/properties_store.py | 10 ++--- extract_msg/structures/business_card.py | 22 ++++------- extract_msg/structures/entry_id.py | 10 ++--- extract_msg/structures/misc_id.py | 40 ++++++-------------- extract_msg/structures/mon_stream.py | 5 ++- extract_msg/structures/odt.py | 5 +-- extract_msg/structures/recurrence_pattern.py | 10 ++--- 10 files changed, 49 insertions(+), 102 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16f443cd..821845ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,8 @@ * Reduced the amount of initialization performed by `MessageBase`. Much of this initialization was there from before a lot of stuff changed to `cached_property` and a number of internal variables were being used. Now all of the relevant variables will be initialized by the way they are accessed. * Added new exception `DependencyError`. * Changed the errors for missing optional dependencies from `ImportError` to `DependencyError`. +* Removed all instances of the `rawData` property in favor of the `toBytes` method. For now, many of these will simply return the raw data used, specifically those that are still unmodifiable. ANy whose properties have the ability to be modified will have properly implemented versions. These classes also allow `None` to be passed as the value for their data, which will be the default if no arguments have been passed to the constructor. If no arguments or `None` is given as the data, it will create a new instance with default values. This is all in an effort to move towards the ability to create new MSG files and the `MSGWriter` class. All `toBytes` methods will either exclusively return `bytes` or will return `None` to specify that the structure isn't valid to convert to bytes. Structures that may be invalid will be annotated as `Optional[bytes]` for the return type. +* Removed the individual `PropBase` flag properties and changed the main `flags` property to return an enum containing the flags. **v0.45.0** * BREAKING: Changed parsing of string multiple properties to remove the trailing null byte. This *will* cause the output of parsing them to differ. diff --git a/extract_msg/enums.py b/extract_msg/enums.py index ed99b2db..b0ded000 100644 --- a/extract_msg/enums.py +++ b/extract_msg/enums.py @@ -52,6 +52,7 @@ 'PostalAddressID', 'Priority', 'PropertiesType', + 'PropertyFlags', 'RecipientRowFlagType', 'RecipientType', 'RecurCalendarType', @@ -1484,6 +1485,13 @@ class PropertiesType(enum.IntEnum): +class PropertyFlags(enum.IntFlag): + MANDATORY = 0b001 + READABLE = 0b010 + WRITABLE = 0b100 + + + class RecipientRowFlagType(enum.IntEnum): NOTYPE = 0x0 X500DN = 0x1 diff --git a/extract_msg/properties/prop.py b/extract_msg/properties/prop.py index f09ef64d..002e11bd 100644 --- a/extract_msg/properties/prop.py +++ b/extract_msg/properties/prop.py @@ -19,7 +19,7 @@ from typing import Any from .. import constants -from ..enums import ErrorCode, ErrorCodeType +from ..enums import ErrorCode, ErrorCodeType, PropertyFlags from ..utils import filetimeToDatetime @@ -46,34 +46,14 @@ class PropBase(abc.ABC): def __init__(self, data : bytes): self.__rawData = data self.__name = data[3::-1].hex().upper() - self.__type, self.__flags = constants.st.ST2.unpack(data) - self.__fm = self.__flags & 1 == 1 - self.__fr = self.__flags & 2 == 2 - self.__fw = self.__flags & 4 == 4 + self.__type, flags = constants.st.ST2.unpack(data) + self.__flags = PropertyFlags(flags) - @property - def flagMandatory(self) -> bool: - """ - Boolean, is the "mandatory" flag set? - """ - return self.__fm - - @property - def flagReadable(self) -> bool: - """ - Boolean, is the "readable" flag set? - """ - return self.__fr - - @property - def flagWritable(self) -> bool: - """ - Boolean, is the "writable" flag set? - """ - return self.__fw + def toBytes(self) -> bytes: + return self.__rawData @property - def flags(self) -> int: + def flags(self) -> PropertyFlags: """ Integer that contains property flags. """ @@ -86,13 +66,6 @@ def name(self) -> str: """ return self.__name - @property - def rawData(self) -> bytes: - """ - The raw bytes used to create this object. - """ - return self.__rawData - @property def type(self) -> int: """ diff --git a/extract_msg/properties/properties_store.py b/extract_msg/properties/properties_store.py index 1372a36d..7bfa2651 100644 --- a/extract_msg/properties/properties_store.py +++ b/extract_msg/properties/properties_store.py @@ -210,6 +210,9 @@ def pprintKeys(self) -> None: """ pprint.pprint(sorted(tuple(self.__props.keys()))) + def toBytes(self) -> bytes: + return self.__rawData + def values(self) -> Iterable[PropBase]: return self.__props.values() @@ -292,13 +295,6 @@ def _propDict(self) -> Dict[str, PropBase]: """ return self.__props - @property - def rawData(self) -> bytes: - """ - The raw bytes used to create this object. - """ - return self.__rawData - @property def recipientCount(self) -> int: """ diff --git a/extract_msg/structures/business_card.py b/extract_msg/structures/business_card.py index 1b286af2..5dc6d0f1 100644 --- a/extract_msg/structures/business_card.py +++ b/extract_msg/structures/business_card.py @@ -39,6 +39,9 @@ def __init__(self, data : bytes): self.__extraInfoField = data[17 + 16 * self.__countOfFields:] self.__fields = tuple(FieldInfo(reader.read(16), self.__extraInfoField) for _ in range(self.__countOfFields)) + def toBytes(self) -> bytes: + return self.__rawData + @property def backgroundColor(self) -> Tuple[int, int, int]: """ @@ -120,13 +123,6 @@ def minorVersion(self) -> int: """ return self.__minorVersion - @property - def rawData(self) -> bytes: - """ - The bytes used to generate this instance. - """ - return self.__rawData - @property def templateID(self) -> BCTemplateID: """ @@ -138,7 +134,7 @@ def templateID(self) -> BCTemplateID: class FieldInfo: def __init__(self, data : bytes, extraInfo : bytes): - self.__raw = data + self.__rawData = data unpacked = constants.st.ST_BC_FIELD_INFO.unpack(data) self.__textPropertyID = unpacked[0] self.__textFormat = BCTextFormat(unpacked[1]) @@ -153,6 +149,9 @@ def __init__(self, data : bytes, extraInfo : bytes): self.__labelFontColor = (bitwiseAdjustedAnd(unpacked[6], 0xFF), bitwiseAdjustedAnd(unpacked[6], 0xFF00), bitwiseAdjustedAnd(unpacked[6], 0xFF0000)) + + def toBytes(self) -> bytes: + return self.__rawData @property def fontSize(self) -> int: @@ -193,13 +192,6 @@ def labelText(self) -> Optional[str]: """ return self.__labelText - @property - def rawData(self) -> bytes: - """ - The bytes used to generate this instance. - """ - return self.__raw - @property def textFormat(self) -> BCTextFormat: """ diff --git a/extract_msg/structures/entry_id.py b/extract_msg/structures/entry_id.py index 549ffe1c..4c8c3247 100644 --- a/extract_msg/structures/entry_id.py +++ b/extract_msg/structures/entry_id.py @@ -98,6 +98,9 @@ def __init__(self, data : bytes): self.__providerUID = data[4:20] self.__rawData = data + def toBytes(self) -> bytes: + return self.__rawData + @property def flags(self) -> bytes: """ @@ -138,13 +141,6 @@ def providerUID(self) -> bytes: """ return self.__providerUID - @property - def rawData(self) -> bytes: - """ - The raw bytes used in this Entry ID. - """ - return self.__rawData - # Now for the specific types. diff --git a/extract_msg/structures/misc_id.py b/extract_msg/structures/misc_id.py index 44a8006d..38df625d 100644 --- a/extract_msg/structures/misc_id.py +++ b/extract_msg/structures/misc_id.py @@ -32,6 +32,9 @@ def __init__(self, data : bytes): # This entry is 6 bytes, so we pull some shenanigans to unpack it. self.__globalCounter = constants.st.ST_LE_UI64.unpack(data[2:8] + b'\x00\x00')[0] + def toBytes(self) -> bytes: + return self.__rawData + @property def globalCounter(self) -> int: """ @@ -39,13 +42,6 @@ def globalCounter(self) -> int: """ return self.__globalCounter - @property - def rawData(self) -> bytes: - """ - The raw bytes used to create this object. - """ - return self.__rawData - @property def replicaID(self) -> int: """ @@ -76,6 +72,9 @@ def __init__(self, data : bytes): size = reader.readUnsignedInt() self.__data = reader.read(size) + def toBytes(self) -> bytes: + return self.__rawData + @property def byteArrayID(self) -> bytes: """ @@ -115,13 +114,6 @@ def month(self) -> int: """ return self.__month - @property - def rawData(self) -> bytes: - """ - The raw bytes used to create this object. - """ - return self.__rawData - @property def year(self) -> int: """ @@ -145,6 +137,9 @@ def __init__(self, data : bytes): # This entry is 6 bytes, so we pull some shenanigans to unpack it. self.__globalCounter = constants.st.ST_LE_UI64.unpack(data[2:8] + b'\x00\x00')[0] + def toBytes(self) -> bytes: + return self.__rawData + @property def globalCounter(self) -> int: """ @@ -159,13 +154,6 @@ def isFolder(self) -> bool: """ return self.__globalCounter == 0 and self.__replicaID == 0 - @property - def rawData(self) -> bytes: - """ - The raw bytes used to create this object. - """ - return self.__rawData - @property def replicaID(self) -> int: """ @@ -194,6 +182,9 @@ def __init__(self, data : bytes): self.__messageID = MessageID(data[9:17]) self.__instance = constants.st.STUI32.unpack(data[17:21])[0] + def toBytes(self) -> bytes: + return self.__rawData + @property def folderID(self) -> FolderID: """ @@ -217,10 +208,3 @@ def messageID(self) -> MessageID: properties will be 0. """ return self.__messageID - - @property - def rawData(self) -> bytes: - """ - The raw data used to create this object. - """ - return self.__rawData diff --git a/extract_msg/structures/mon_stream.py b/extract_msg/structures/mon_stream.py index 786f329c..33150ec0 100644 --- a/extract_msg/structures/mon_stream.py +++ b/extract_msg/structures/mon_stream.py @@ -3,9 +3,10 @@ ] -from typing import Optional +from typing import final, Optional +@final class MonikerStream: def __init__(self, data : Optional[bytes]): if data: @@ -16,7 +17,7 @@ def __init__(self, data : Optional[bytes]): self.__streamData = b'' def toBytes(self) -> bytes: - pass # TODO + return self.__clsid + self.__streamData @property def clsid(self) -> bytes: diff --git a/extract_msg/structures/odt.py b/extract_msg/structures/odt.py index bb414645..04c0b8f5 100644 --- a/extract_msg/structures/odt.py +++ b/extract_msg/structures/odt.py @@ -1,15 +1,14 @@ __all__ = [ - 'Cf', - \ 'ODTStruct', ] import struct -from typing import Optional +from typing import final, Optional from ..enums import ODTCf, ODTPersist1, ODTPersist2 +@final class ODTStruct: def __init__(self, data : Optional[bytes]): if data: diff --git a/extract_msg/structures/recurrence_pattern.py b/extract_msg/structures/recurrence_pattern.py index 3b526db3..b33e8b34 100644 --- a/extract_msg/structures/recurrence_pattern.py +++ b/extract_msg/structures/recurrence_pattern.py @@ -51,6 +51,9 @@ def __init__(self, data : bytes): self.__startDate = reader.readUnsignedInt() self.__endDate = reader.readUnsignedInt() + def toBytes(self) -> bytes: + return self.__rawData + @property def calendarType(self) -> RecurCalendarType: """ @@ -160,13 +163,6 @@ def period(self) -> int: """ return self.__period - @property - def rawData(self) -> bytes: - """ - The raw bytes used to create this object. - """ - return self.__rawData - @property def readerVersion(self) -> int: return self.__readerVersion From 857ab09bdfc82db51db271d52e4ee65c0f0ea78b Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 5 Sep 2023 19:34:15 -0700 Subject: [PATCH 38/68] More MSG writing progress --- CHANGELOG.md | 5 +- extract_msg/structures/report_tag.py | 10 +-- extract_msg/structures/system_time.py | 7 +- .../structures/time_zone_definition.py | 67 +++++++++++++++---- extract_msg/structures/time_zone_struct.py | 35 ++++++---- extract_msg/structures/tz_rule.py | 49 ++++++++++---- 6 files changed, 124 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 821845ea..a43ab9de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,8 +37,11 @@ * Reduced the amount of initialization performed by `MessageBase`. Much of this initialization was there from before a lot of stuff changed to `cached_property` and a number of internal variables were being used. Now all of the relevant variables will be initialized by the way they are accessed. * Added new exception `DependencyError`. * Changed the errors for missing optional dependencies from `ImportError` to `DependencyError`. -* Removed all instances of the `rawData` property in favor of the `toBytes` method. For now, many of these will simply return the raw data used, specifically those that are still unmodifiable. ANy whose properties have the ability to be modified will have properly implemented versions. These classes also allow `None` to be passed as the value for their data, which will be the default if no arguments have been passed to the constructor. If no arguments or `None` is given as the data, it will create a new instance with default values. This is all in an effort to move towards the ability to create new MSG files and the `MSGWriter` class. All `toBytes` methods will either exclusively return `bytes` or will return `None` to specify that the structure isn't valid to convert to bytes. Structures that may be invalid will be annotated as `Optional[bytes]` for the return type. +* Removed all instances of the `rawData` property in favor of the `toBytes` method. For now, many of these will simply return the raw data used, specifically those that are still unmodifiable. Any whose properties have the ability to be modified will have properly implemented versions. These classes also allow `None` to be passed as the value for their data, which will be the default if no arguments have been passed to the constructor. If no arguments or `None` is given as the data, it will create a new instance with default values. This is all in an effort to move towards the ability to create new MSG files and the `MSGWriter` class. All `toBytes` methods will either exclusively return `bytes` or will return `None` to specify that the structure isn't valid to convert to bytes. Structures that may be invalid will be annotated as `Optional[bytes]` for the return type. * Removed the individual `PropBase` flag properties and changed the main `flags` property to return an enum containing the flags. +* Changed various data structs to allow modification and creation of new instances for writing to an MSG file. +* changed `TZRule` to use unsigned values where applicable. +* Changed `TZRule` to require the 14 null bytes (I noticed there is a note about outlook violating that standard and will look into it). **v0.45.0** * BREAKING: Changed parsing of string multiple properties to remove the trailing null byte. This *will* cause the output of parsing them to differ. diff --git a/extract_msg/structures/report_tag.py b/extract_msg/structures/report_tag.py index ad8e3d5f..648b743e 100644 --- a/extract_msg/structures/report_tag.py +++ b/extract_msg/structures/report_tag.py @@ -57,6 +57,9 @@ def __init__(self, data : bytes): else: self.__ansiText = None + def toBytes(self) -> bytes: + return self.__rawData + @property def ansiText(self) -> Optional[bytes]: """ @@ -92,13 +95,6 @@ def messageSearchKey(self) -> Optional[bytes]: """ return self.__messageSearchKey - @property - def rawData(self) -> bytes: - """ - The raw bytes used to create this object. - """ - return self.__rawData - @property def searchFolderEntryID(self) -> Optional[EntryID]: """ diff --git a/extract_msg/structures/system_time.py b/extract_msg/structures/system_time.py index b023ca44..806de47d 100644 --- a/extract_msg/structures/system_time.py +++ b/extract_msg/structures/system_time.py @@ -3,6 +3,8 @@ ] +from typing import Optional + from .. import constants @@ -20,7 +22,8 @@ class SystemTime: second : int = 0 milliseconds : int = 0 - def __init__(self, data : bytes): + def __init__(self, data : Optional[bytes] = None): + data = data or (b'\x00' * 1) self.unpack(data) def __eq__(self, other) -> bool: @@ -29,7 +32,7 @@ def __eq__(self, other) -> bool: def __ne__(self, other) -> bool: return not self.__eq__(other) - def pack(self) -> bytes: + def toBytes(self) -> bytes: """ Packs the current data into bytes. """ diff --git a/extract_msg/structures/time_zone_definition.py b/extract_msg/structures/time_zone_definition.py index d4f3d77a..8df261d0 100644 --- a/extract_msg/structures/time_zone_definition.py +++ b/extract_msg/structures/time_zone_definition.py @@ -3,8 +3,9 @@ ] -from typing import Tuple +from typing import List, Optional +from ..constants import st from ._helpers import BytesReader from .tz_rule import TZRule @@ -14,8 +15,13 @@ class TimeZoneDefinition: Structure for PidLidAppointmentTimeZoneDefinitionRecur from [MS-OXOCAL]. """ - def __init__(self, data : bytes): - self.__rawData = data + def __init__(self, data : Optional[bytes] = None): + if not data: + self.__majorVersion = 2 + self.__minorVersion = 1 + self.__keyName = '' + self.__rules = [TZRule()] + return reader = BytesReader(data) self.__majorVersion = reader.readUnsignedByte() self.__minorVersion = reader.readUnsignedByte() @@ -24,7 +30,25 @@ def __init__(self, data : bytes): cchKeyName = reader.readUnsignedShort() self.__keyName = reader.read(2 * cchKeyName).decode('utf-16-le') cRules = reader.readUnsignedShort() - self.__rules = tuple(reader.readClass(TZRule) for x in range(cRules)) + if len(cRules) < 1 or len(cRules) > 1024: + raise ValueError('Value for cRules was out of range.') + self.__rules = [reader.readClass(TZRule) for _ in range(cRules)] + + def toBytes(self) -> bytes: + # Validate some of the data. + if len(self.__rules) < 1: + raise ValueError('Cannot pack a TimeZoneDefinition with no rules.') + if len(self.__rules) > 1024: + raise ValueError('TimeZoneDefintion can only have up to 1024 rules.') + + ret = bytes((self.__majorVersion, self.__minorVersion)) + ret += st.ST_LE_UI16.pack(6 + 2 * len(self.__keyName)) + ret += b'\x02\x00' + ret += st.ST_LE_UI16.pack(2* len(self.__keyName)) + ret += st.ST_LE_UI16.pack(len(self.__rules)) + ret += b''.join(x.toBytes() for x in self.__rules) + + return ret @property def keyName(self) -> str: @@ -34,12 +58,29 @@ def keyName(self) -> str: """ return self.__keyName + @keyName.setter + def setter(self, value : str) -> None: + value = str(value) + if len(value) > 260: + raise ValueError('Key name must be a string less than 261 characters.') + + self.__keyName = value + @property def majorVersion(self) -> int: """ The major version. """ return self.__majorVersion + + @majorVersion.setter + def setter(self, value : int) -> None: + if value > 255: + raise ValueError('Major version cannot be greater than 255') + if value < 0: + raise ValueError('Major version must be positive.') + + self.__minorVersion = value @property def minorVersion(self) -> int: @@ -47,16 +88,18 @@ def minorVersion(self) -> int: The minor version. """ return self.__minorVersion + + @minorVersion.setter + def setter(self, value : int) -> None: + if value > 255: + raise ValueError('Minor version cannot be greater than 255') + if value < 0: + raise ValueError('Minor version must be positive.') + + self.__minorVersion = value @property - def rawData(self) -> bytes: - """ - The raw bytes used to create this object. - """ - return self.__rawData - - @property - def rules(self) -> Tuple[TZRule, ...]: + def rules(self) -> List[TZRule]: """ A tuple of TZRule structures that specifies a time zone. """ diff --git a/extract_msg/structures/time_zone_struct.py b/extract_msg/structures/time_zone_struct.py index f3d97d1a..0e911963 100644 --- a/extract_msg/structures/time_zone_struct.py +++ b/extract_msg/structures/time_zone_struct.py @@ -3,6 +3,8 @@ ] +from typing import Optional + from .. import constants from .system_time import SystemTime @@ -12,17 +14,31 @@ class TimeZoneStruct: A TimeZoneStruct, as specified in [MS-OXOCAL]. """ - def __init__(self, data : bytes): - self.__rawData = data + def __init__(self, data : Optional[bytes] = None): + if not data: + self.__bias = 0 + self.__standardBias = 0 + self.__daylightBias = 0 + self.__standardDate = SystemTime() + self.__daylightDate = SystemTime() + + return unpacked = constants.st.ST_TZ.unpack(data) self.__bias = unpacked[0] self.__standardBias = unpacked[1] self.__daylightBias = unpacked[2] - self.__standardYear = unpacked[3] self.__standardDate = SystemTime(unpacked[4]) - self.__daylightYear = unpacked[5] self.__daylightDate = SystemTime(unpacked[6]) + def toBytes(self) -> bytes: + return constants.st.ST_TZ.pack(self.__bias, + self.__standardBias, + self.__daylightBias, + self.standardYear, + self.__standardDate.toBytes(), + self.daylightYear, + self.__daylightDate.toBytes()) + @property def bias(self) -> int: """ @@ -52,14 +68,7 @@ def daylightYear(self) -> int: """ The value of the daylightDate field's year. """ - return self.__daylightYear - - @property - def rawData(self) -> bytes: - """ - The raw bytes used to create this object. - """ - return self.__rawData + return self.__daylightDate.year @property def standardBias(self) -> int: @@ -87,4 +96,4 @@ def standardYear(self) -> int: """ The value of the standardDate field's year. """ - return self.__standardYear + return self.__standardDate.year diff --git a/extract_msg/structures/tz_rule.py b/extract_msg/structures/tz_rule.py index 9f1601cf..923b973f 100644 --- a/extract_msg/structures/tz_rule.py +++ b/extract_msg/structures/tz_rule.py @@ -3,34 +3,62 @@ ] +from struct import Struct +from typing import Final, final, Optional + from ..enums import TZFlag from ._helpers import BytesReader from .system_time import SystemTime +@final class TZRule: """ A TZRule structure, as defined in [MS-OXOCAL]. """ __SIZE__ : int = 66 + __struct : Final[Struct] = Struct('4B2H14x3i16s16s') + + def __init__(self, data : Optional[bytes] = None): + if not data: + self.__majorVersion = 2 + self.__minorVersion = 1 + self.__flags = TZFlag(0) + self.__year = 0 + self.__bias = 0 + self.__standardBias = 0 + self.__daylightBias = 0 + self.__standardDate = SystemTime() + self.__daylightDate = SystemTime() + return - def __init__(self, data : bytes): - self.__rawData = data reader = BytesReader(data) - self.__majorVersion = reader.readByte() - self.__minorVersion = reader.readByte() + self.__majorVersion = reader.readUnsignedByte() + self.__minorVersion = reader.readUnsignedByte() reader.assertRead(b'\x3E\x00') self.__flags = TZFlag(reader.readUnsignedShort()) - self.__year = reader.readShort() - # We *should* be doing this, but Outlook is violating the standard so... - #reader.assertNull(14) + self.__year = reader.readUnsignedShort() + reader.assertNull(14) self.__bias = reader.readInt() self.__standardBias = reader.readInt() self.__daylightBias = reader.readInt() self.__standardDate = SystemTime(reader.read(16)) self.__daylightDate = SystemTime(reader.read(16)) + def toBytes(self) -> bytes: + return self.__struct.pack(self.__majorVersion, + self.__minorVersion, + 62, + 0, + self.__flags, + self.__year, + self.__bias, + self.__standardBias, + self.__daylightBias, + self.__standardDate.toBytes(), + self.__daylightDate.toBytes()) + @property def bias(self) -> int: """ @@ -76,13 +104,6 @@ def minorVersion(self) -> int: """ return self.__minorVersion - @property - def rawData(self) -> bytes: - """ - The raw bytes used to create this object. - """ - return self.__rawData - @property def standardBias(self) -> int: """ From cd6f5c724d8bb5365cc12d66698e4ea315cb2d5c Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 5 Sep 2023 23:22:27 -0700 Subject: [PATCH 39/68] Writing, typing, and remove deprecated items --- CHANGELOG.md | 2 + extract_msg/attachments/attachment_base.py | 47 +----- .../custom_att_handler/custom_handler.py | 6 +- extract_msg/constants/__init__.py | 10 +- extract_msg/msg_classes/msg.py | 47 ++---- extract_msg/properties/named.py | 16 +- extract_msg/properties/properties_store.py | 2 +- extract_msg/recipient.py | 140 +--------------- extract_msg/structures/mon_stream.py | 13 +- extract_msg/structures/odt.py | 10 +- extract_msg/structures/ole_pres.py | 28 ++-- extract_msg/structures/ole_stream_struct.py | 12 +- extract_msg/structures/system_time.py | 153 ++++++++++++++---- .../structures/time_zone_definition.py | 18 +-- extract_msg/utils.py | 6 - 15 files changed, 212 insertions(+), 298 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a43ab9de..ecfcb3fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ **v0.46.0** * [[TeamMsgExtractor #95](https://github.com/TeamMsgExtractor/msg-extractor/issues/95)] Adjusted the `overrideEncoding` property of `MSGFile` to allow automatic encoding detection. Simply set the property to the string `"chardet"` and, assuming the `chardet` module is installed, it will analyze a number of the strings to try and form a consensus about the encoding. This will *ignore* the specified encoding *only if* if successfully detects. Otherwise it will log a warning and fall back to the default behavior. +* Removed methods deprecated in `v0.45.0`. * Changed the base class of `EntryID` from no base class to `abc.ABC`. * Added `position` property to `EntryID` to tell how many bytes were used to create the `EntryID`. * Added a number of properties to `MSGFile` from \[MS-OXCMSG\]. @@ -42,6 +43,7 @@ * Changed various data structs to allow modification and creation of new instances for writing to an MSG file. * changed `TZRule` to use unsigned values where applicable. * Changed `TZRule` to require the 14 null bytes (I noticed there is a note about outlook violating that standard and will look into it). +* Removed unneeded function `windowsUnicode`. **v0.45.0** * BREAKING: Changed parsing of string multiple properties to remove the trailing null byte. This *will* cause the output of parsing them to differ. diff --git a/extract_msg/attachments/attachment_base.py b/extract_msg/attachments/attachment_base.py index 53562e9e..e08b1774 100644 --- a/extract_msg/attachments/attachment_base.py +++ b/extract_msg/attachments/attachment_base.py @@ -16,14 +16,12 @@ from functools import cached_property from typing import ( - Any, Callable, List, Optional, Tuple, Type, TYPE_CHECKING, TypeVar, - Union + Any, List, Optional, Tuple, Type, TYPE_CHECKING, TypeVar, Union ) -from ..constants import MSG_PATH, SAVE_TYPE +from ..constants import MSG_PATH, OVERRIDE_CLASS, SAVE_TYPE from ..enums import AttachmentType from ..properties.named import NamedProperties -from ..properties.prop import FixedLengthProp from ..properties.properties_store import PropertiesStore from ..utils import ( msgPathToString, tryGetMimetype, verifyPropertyId, verifyType @@ -58,35 +56,6 @@ def __init__(self, msg : MSGFile, dir_ : str, propStore : PropertiesStore): self.__namedProperties = NamedProperties(msg.named, self) self.__treePath = msg.treePath + [weakref.ref(self)] - def _getStream(self, filename : MSG_PATH) -> Optional[bytes]: - """ - Gets a binary representation of the requested filename. - - This should ALWAYS return a bytes object if it was found, otherwise - returns None. - - :raises ReferenceError: The associated MSGFile instance has been garbage - collected. - """ - import warnings - warnings.warn(':method _getStream: has been deprecated and moved to the public api. Use :method getStream: instead (remove the underscore).', DeprecationWarning) - return self.getStream(filename) - - def _getStringStream(self, filename : MSG_PATH) -> Optional[str]: - """ - Gets a string representation of the requested filename. - Checks for both ASCII and Unicode representations and returns - a value if possible. If there are both ASCII and Unicode - versions, then :param prefer: specifies which will be - returned. - - :raises ReferenceError: The associated MSGFile instance has been garbage - collected. - """ - import warnings - warnings.warn(':method _getStringStream: has been deprecated and moved to the public api. Use :method getStringStream: instead (remove the underscore).', DeprecationWarning) - return self.getStringStream(filename) - def _getTypedAs(self, _id : str, overrideClass = None, preserveNone : bool = True): """ Like the other get as functions, but designed for when something @@ -245,7 +214,7 @@ def sExists(self, filename : MSG_PATH) -> bool: """ if (msg := self.__msg()) is None: raise ReferenceError('The msg file for this Attachment instance has been garbage collected.') - return msg.sExists([self.__dir, filename]) + return msg.sExists([self.__dir, msgPathToString(filename)]) def existsTypedProperty(self, id, _type = None) -> bool: """ @@ -302,7 +271,7 @@ def getMultipleString(self, filename : MSG_PATH) -> Optional[List[str]]: raise ReferenceError('The msg file for this Attachment instance has been garbage collected.') return msg.getMultipleString([self.__dir, msgPathToString(filename)]) - def getNamedAs(self, propertyName : str, guid : str, overrideClass : Callable[[Any], _T]) -> Optional[_T]: + def getNamedAs(self, propertyName : str, guid : str, overrideClass : OVERRIDE_CLASS[_T]) -> Optional[_T]: """ Returns the named property, setting the class if specified. @@ -325,7 +294,7 @@ def getNamedProp(self, propertyName : str, guid : str, default : _T = None) -> U """ return self.namedProperties.get((propertyName, guid), default) - def getPropertyAs(self, propertyName : Union[int, str], overrideClass : Callable[[Any], _T]) -> Optional[_T]: + def getPropertyAs(self, propertyName : Union[int, str], overrideClass : OVERRIDE_CLASS[_T]) -> Optional[_T]: """ Returns the property, setting the class if found. @@ -402,7 +371,7 @@ def getStream(self, filename : MSG_PATH) -> Optional[bytes]: raise ReferenceError('The msg file for this Attachment instance has been garbage collected.') return msg.getStream([self.__dir, msgPathToString(filename)]) - def getStreamAs(self, streamID : MSG_PATH, overrideClass : Callable[[Any], _T]) -> Optional[_T]: + def getStreamAs(self, streamID : MSG_PATH, overrideClass : OVERRIDE_CLASS[_T]) -> Optional[_T]: """ Returns the specified stream, modifying it to the specified class if it is found. @@ -435,7 +404,7 @@ def getStringStream(self, filename : MSG_PATH) -> Optional[str]: raise ReferenceError('The msg file for this Attachment instance has been garbage collected.') return msg.getStringStream([self.__dir, msgPathToString(filename)]) - def getStringStreamAs(self, streamID : MSG_PATH, overrideClass : Callable[[Any], _T]) -> Optional[_T]: + def getStringStreamAs(self, streamID : MSG_PATH, overrideClass : OVERRIDE_CLASS[_T]) -> Optional[_T]: """ Returns the specified string stream, modifying it to the specified class if it is found. @@ -703,7 +672,7 @@ def shortFilename(self) -> Optional[str]: return self.getStringStream('__substg1.0_3704') @property - def treePath(self) -> List[weakref.ReferenceType]: + def treePath(self) -> List[weakref.ReferenceType[Any]]: """ A path, as a tuple of instances, needed to get to this instance through the MSGFile-Attachment tree. diff --git a/extract_msg/attachments/custom_att_handler/custom_handler.py b/extract_msg/attachments/custom_att_handler/custom_handler.py index 7903dede..571b8c03 100644 --- a/extract_msg/attachments/custom_att_handler/custom_handler.py +++ b/extract_msg/attachments/custom_att_handler/custom_handler.py @@ -9,9 +9,9 @@ import abc import functools -from typing import Any, Callable, Dict, Optional, TYPE_CHECKING, TypeVar +from typing import Dict, Optional, TYPE_CHECKING, TypeVar -from ...constants import MSG_PATH +from ...constants import MSG_PATH, OVERRIDE_CLASS from ...structures.odt import ODTStruct from ...structures.ole_pres import OLEPresentationStream from ...structures.ole_stream_struct import OleStreamStruct @@ -40,7 +40,7 @@ def getStream(self, path : MSG_PATH) -> Optional[bytes]: """ return self.attachment.getStream('__substg1.0_3701000D/' + msgPathToString(path)) - def getStreamAs(self, streamID : MSG_PATH, overrideClass : Callable[[Any], _T]) -> Optional[_T]: + def getStreamAs(self, streamID : MSG_PATH, overrideClass : OVERRIDE_CLASS[_T]) -> Optional[_T]: """ Returns the specified stream, modifying it to the specified class if it is found. diff --git a/extract_msg/constants/__init__.py b/extract_msg/constants/__init__.py index 38aca5e8..af89bffc 100644 --- a/extract_msg/constants/__init__.py +++ b/extract_msg/constants/__init__.py @@ -44,12 +44,17 @@ from datetime import datetime -from typing import Dict, Final, List, Optional, Tuple, Union +from typing import ( + Any, Callable, Dict, Final, List, Optional, Tuple, Type, TypeVar, Union + ) from . import ps, re, st from ..enums import SaveType +_T = TypeVar('_T') + + # Constants for formating datetime objects. DATE_FORMAT : Final[str] = '%d %B, %Y' DT_FORMAT : Final[str] = '%a, %d %b %Y %H:%M:%S %z' @@ -61,7 +66,8 @@ HEADER_FORMAT_TYPE = Optional[Dict[str, Union[HEADER_FORMAT_VALUE_TYPE, Dict[str, HEADER_FORMAT_VALUE_TYPE]]]] SAVE_TYPE = Tuple[SaveType, Union[List[str], str, None]] MSG_PATH = Union[str, List[str], Tuple[str]] - +# Type used for the getXAs methods' overrideClass argument. +OVERRIDE_CLASS = Union[Type[_T], Callable[[Any], _T]] FIXED_LENGTH_PROPS : Final[Tuple[int, ...]] = ( diff --git a/extract_msg/msg_classes/msg.py b/extract_msg/msg_classes/msg.py index f4d273a5..c2da4f97 100644 --- a/extract_msg/msg_classes/msg.py +++ b/extract_msg/msg_classes/msg.py @@ -24,7 +24,9 @@ ) from .. import constants -from ..constants import DATE_FORMAT, DT_FORMAT, MSG_PATH, ps, SAVE_TYPE +from ..constants import ( + DATE_FORMAT, DT_FORMAT, MSG_PATH, OVERRIDE_CLASS, ps, SAVE_TYPE + ) from ..attachments import ( AttachmentBase, initStandardAttachment, SignedAttachment ) @@ -246,32 +248,6 @@ def _getOleEntry(self, filename : MSG_PATH, prefix : bool = True) -> olefile.ole return self.__ole.direntries[sid] - def _getStream(self, filename : MSG_PATH, prefix : bool = True) -> Optional[bytes]: - """ - Gets a binary representation of the requested filename. - - This should ALWAYS return a bytes object if it was found, otherwise - returns None. - """ - import warnings - warnings.warn(':method _getStream: has been deprecated and moved to the public api. Use :method getStream: instead (remove the underscore).', DeprecationWarning) - return self.getStream(filename, prefix) - - def _getStringStream(self, filename : MSG_PATH, prefix : bool = True) -> Optional[str]: - """ - Gets a string representation of the requested filename. - - Rather than the full filename, you should only feed this function the - filename sans the type. So if the full name is "__substg1.0_001A001F", - the filename this function should receive should be "__substg1.0_001A". - - This should ALWAYS return a string if it was found, otherwise returns - None. - """ - import warnings - warnings.warn(':method _getStringStream: has been deprecated and moved to the public api. Use :method getStringStream: instead (remove the underscore).', DeprecationWarning) - return self.getStringStream(filename, prefix) - def _getTypedAs(self, _id : str, overrideClass = None, preserveNone : bool = True): """ Like the other get as functions, but designed for when something @@ -560,7 +536,7 @@ def getMultipleString(self, filename : MSG_PATH, prefix : bool = True) -> Option ret[index] = item.decode(self.stringEncoding)[:-1] return ret - def getNamedAs(self, propertyName : str, guid : str, overrideClass : Callable[[Any], _T]) -> Optional[_T]: + def getNamedAs(self, propertyName : str, guid : str, overrideClass : OVERRIDE_CLASS[_T]) -> Optional[_T]: """ Returns the named property, setting the class if specified. @@ -583,7 +559,7 @@ def getNamedProp(self, propertyName : str, guid : str, default : _T = None) -> U """ return self.namedProperties.get((propertyName, guid), default) - def getPropertyAs(self, propertyName : Union[int, str], overrideClass : Callable[[Any], _T]) -> Optional[_T]: + def getPropertyAs(self, propertyName : Union[int, str], overrideClass : OVERRIDE_CLASS[_T]) -> Optional[_T]: """ Returns the property, setting the class if found. @@ -666,7 +642,7 @@ def getStream(self, filename : MSG_PATH, prefix : bool = True) -> Optional[bytes logger.info(f'Stream "{filename}" was requested but could not be found. Returning `None`.') return None - def getStreamAs(self, streamID : MSG_PATH, overrideClass : Callable[[Any], _T]) -> Optional[_T]: + def getStreamAs(self, streamID : MSG_PATH, overrideClass : OVERRIDE_CLASS[_T]) -> Optional[_T]: """ Returns the specified stream, modifying it to the specified class if it is found. @@ -700,12 +676,13 @@ def getStringStream(self, filename : MSG_PATH, prefix : bool = True) -> Optional """ filename = self.fixPath(filename, prefix) if self.areStringsUnicode: - return windowsUnicode(self.getStream(filename + '001F', prefix = False)) + tmp = self.getStream(filename + '001F', prefix = False) else: tmp = self.getStream(filename + '001E', prefix = False) - return None if tmp is None else tmp.decode(self.stringEncoding) - def getStringStreamAs(self, streamID : MSG_PATH, overrideClass : Callable[[Any], _T]) -> Optional[_T]: + return None if tmp is None else tmp.decode(self.stringEncoding) + + def getStringStreamAs(self, streamID : MSG_PATH, overrideClass : OVERRIDE_CLASS[_T]) -> Optional[_T]: """ Returns the specified string stream, modifying it to the specified class if it is found. @@ -1003,7 +980,7 @@ def namedProperties(self) -> NamedProperties: return NamedProperties(self.named, self) @property - def overrideEncoding(self): + def overrideEncoding(self) -> Optional[str]: """ Returns None is the encoding has not been overridden, otherwise returns the encoding. @@ -1116,7 +1093,7 @@ def stringEncoding(self) -> str: return self.__stringEncoding @property - def treePath(self) -> List[weakref.ReferenceType]: + def treePath(self) -> List[weakref.ReferenceType[Any]]: """ A path, as a list of weak reference to the instances needed to get to this instance through the MSGFile-Attachment tree. These are weak diff --git a/extract_msg/properties/named.py b/extract_msg/properties/named.py index fa5e95b9..00a75f1d 100644 --- a/extract_msg/properties/named.py +++ b/extract_msg/properties/named.py @@ -52,7 +52,7 @@ def __init__(self, msg : MSGFile): entryStream = self.getStream('__substg1.0_00030102') self.guidStream = guidStream self.entryStream = entryStream - self.namesStream = self.getStream('__substg1.0_00040102') + self.namesStream = self.getStream('__substg1.0_00040102') or b'' self.__propertiesDict : Dict[Tuple[str, str], NamedPropertyBase] = {} self.__properties : List[NamedPropertyBase] = [] @@ -130,20 +130,6 @@ def __getName(self, offset : int) -> str: return self.namesStream[offset:offset + length].decode('utf-16-le') - def _getStream(self, filename : constants.MSG_PATH) -> Optional[bytes]: - """ - Gets a binary representation of the requested filename. - - This should ALWAYS return a bytes object if it was found, otherwise - returns None. - - :raises ReferenceError: The associated MSGFile instance has been garbage - collected. - """ - import warnings - warnings.warn(':method _getStream: has been deprecated and moved to the public api. Use :method getStream: instead (remove the underscore).', DeprecationWarning) - return self.getStream(filename) - def exists(self, filename : constants.MSG_PATH) -> bool: """ Checks if stream exists inside the named properties folder. diff --git a/extract_msg/properties/properties_store.py b/extract_msg/properties/properties_store.py index 7bfa2651..cd45d55b 100644 --- a/extract_msg/properties/properties_store.py +++ b/extract_msg/properties/properties_store.py @@ -29,7 +29,7 @@ class PropertiesStore: Parser for msg properties files. """ - def __init__(self, data : Optional[bytes], _type : Optional[PropertiesType] = None, skip : Optional[int] = None): + def __init__(self, data : Optional[bytes] = b'', _type : Optional[PropertiesType] = None, skip : Optional[int] = None): if not data: # If data comes back false, make sure is is empty bytes. data = b'' diff --git a/extract_msg/recipient.py b/extract_msg/recipient.py index e68636b3..aa6c6016 100644 --- a/extract_msg/recipient.py +++ b/extract_msg/recipient.py @@ -12,17 +12,16 @@ import weakref from typing import ( - Any, Callable, Generic, List, Optional, Tuple, TYPE_CHECKING, Type, - TypeVar, Union + Any, Callable, Generic, List, Optional, TYPE_CHECKING, Type, TypeVar, + Union ) -from .constants import MSG_PATH +from .constants import MSG_PATH, OVERRIDE_CLASS from .enums import ErrorBehavior, PropertiesType from .exceptions import StandardViolationError -from .properties.prop import FixedLengthProp from .properties.properties_store import PropertiesStore from .structures.entry_id import PermanentEntryID -from .utils import msgPathToString, verifyPropertyId, verifyType +from .utils import msgPathToString if TYPE_CHECKING: @@ -57,131 +56,6 @@ def __init__(self, _dir : str, msg : MSGFile, recipientTypeClass : Type[_RT]): self.__type = recipientTypeClass(0xF & self.__typeFlags) self.__formatted = f'{self.__name} <{self.__email}>' - def _getStream(self, filename : MSG_PATH) -> Optional[bytes]: - """ - Gets a binary representation of the requested filename. - - This should ALWAYS return a bytes object if it was found, otherwise - returns None. - - :raises ReferenceError: The associated MSGFile instance has been garbage - collected. - """ - import warnings - warnings.warn(':method _getStream: has been deprecated and moved to the public api. Use :method getStream: instead (remove the underscore).', DeprecationWarning) - return self.getStream(filename) - - def _getStringStream(self, filename : MSG_PATH) -> Optional[str]: - """ - Gets a string representation of the requested filename. - - Rather than the full filename, you should only feed this function the - filename sans the type. So if the full name is "__substg1.0_001A001F", - the filename this function should receive should be "__substg1.0_001A". - - This should ALWAYS return a string if it was found, otherwise returns - None. - - :raises ReferenceError: The associated MSGFile instance has been garbage - collected. - """ - import warnings - warnings.warn(':method _getStringStream: has been deprecated and moved to the public api. Use :method getStringStream: instead (remove the underscore).', DeprecationWarning) - return self.getStringStream(filename) - - def _getTypedAs(self, _id : str, overrideClass = None, preserveNone : bool = True): - """ - Like the other get as functions, but designed for when something - could be multiple types (where only one will be present). This way you - have no need to set the type, it will be handled for you. - - :param overrideClass: Class/function to use to morph the data that was - read. The data will be the first argument to the class's __init__ - function or the function itself, if that is what is provided. By - default, this will be completely ignored if the value was not found. - :param preserveNone: If true (default), causes the function to ignore - :param overrideClass: when the value could not be found (is None). - If this is changed to False, then the value will be used regardless. - """ - value = self._getTypedData(_id) - # Check if we should be overriding the data type for this instance. - if overrideClass is not None: - if value is not None or not preserveNone: - value = overrideClass(value) - - return value - - def _getTypedData(self, _id, _type = None): - """ - Gets the data for the specified id as the type that it is supposed to - be. :param id: MUST be a 4 digit hexadecimal string. - - If you know for sure what type the data is before hand, you can specify - it as being one of the strings in the constant FIXED_LENGTH_PROPS_STRING - or VARIABLE_LENGTH_PROPS_STRING. - - :raises ReferenceError: The associated MSGFile instance has been garbage - collected. - """ - verifyPropertyId(id) - _id = _id.upper() - found, result = self._getTypedStream('__substg1.0_' + _id, _type) - if found: - return result - else: - found, result = self._getTypedProperty(_id, _type) - return result if found else None - - def _getTypedProperty(self, propertyID : str, _type = None) -> Tuple[bool, Optional[object]]: - """ - Gets the property with the specified id as the type that it is supposed - to be. :param id: MUST be a 4 digit hexadecimal string. - - If you know for sure what type the property is before hand, you can - specify it as being one of the strings in the constant - FIXED_LENGTH_PROPS_STRING or VARIABLE_LENGTH_PROPS_STRING. - """ - verifyPropertyId(propertyID) - if _type: - verifyType(_type) - prop = self.props.get(propertyID + _type) - if isinstance(prop, FixedLengthProp): - return True, prop.value - else: - return False, None - else: - props = self.props.getProperties(propertyID) - for prop in props: - if isinstance(prop, FixedLengthProp): - return True, prop.value - - return False, None - - def _getTypedStream(self, filename : MSG_PATH, _type = None): - """ - Gets the contents of the specified stream as the type that it is - supposed to be. - - Rather than the full filename, you should only feed this function the - filename sans the type. So if the full name is "__substg1.0_001A001F", - the filename this function should receive should be "__substg1.0_001A". - - If you know for sure what type the stream is before hand, you can - specify it as being one of the strings in the constant - FIXED_LENGTH_PROPS_STRING or VARIABLE_LENGTH_PROPS_STRING. - - If you have not specified the type, the type this function returns in - many cases cannot be predicted. As such, when using this function it is - best for you to check the type that it returns. If the function returns - None, that means it could not find the stream specified. - - :raises ReferenceError: The associated MSGFile instance has been garbage - collected. - """ - if (msg := self.__msg()) is None: - raise ReferenceError('The msg file for this Recipient instance has been garbage collected.') - return msg._getTypedStream(self, [self.__dir, msgPathToString(filename)], True, _type) - def exists(self, filename : MSG_PATH) -> bool: """ Checks if stream exists inside the recipient folder. @@ -247,7 +121,7 @@ def getMultipleString(self, filename : MSG_PATH) -> Optional[List[str]]: raise ReferenceError('The msg file for this Recipient instance has been garbage collected.') return msg.getMultipleString([self.__dir, msgPathToString(filename)]) - def getPropertyAs(self, propertyName : Union[int, str], overrideClass : Callable[[Any], _T]) -> Optional[_T]: + def getPropertyAs(self, propertyName : Union[int, str], overrideClass : OVERRIDE_CLASS[_T]) -> Optional[_T]: """ Returns the property, setting the class if found. @@ -324,7 +198,7 @@ def getStream(self, filename : MSG_PATH) -> Optional[bytes]: raise ReferenceError('The msg file for this Recipient instance has been garbage collected.') return msg.getStream([self.__dir, msgPathToString(filename)]) - def getStreamAs(self, streamID : MSG_PATH, overrideClass : Callable[[Any], _T]) -> Optional[_T]: + def getStreamAs(self, streamID : MSG_PATH, overrideClass : OVERRIDE_CLASS[_T]) -> Optional[_T]: """ Returns the specified stream, modifying it to the specified class if it is found. @@ -360,7 +234,7 @@ def getStringStream(self, filename : MSG_PATH) -> Optional[str]: raise ReferenceError('The msg file for this Recipient instance has been garbage collected.') return msg.getStringStream([self.__dir, msgPathToString(filename)]) - def getStringStreamAs(self, streamID : MSG_PATH, overrideClass : Callable[[Any], _T]) -> Optional[_T]: + def getStringStreamAs(self, streamID : MSG_PATH, overrideClass : OVERRIDE_CLASS[_T]) -> Optional[_T]: """ Returns the specified string stream, modifying it to the specified class if it is found. diff --git a/extract_msg/structures/mon_stream.py b/extract_msg/structures/mon_stream.py index 33150ec0..33d1f531 100644 --- a/extract_msg/structures/mon_stream.py +++ b/extract_msg/structures/mon_stream.py @@ -1,3 +1,4 @@ +# pyright: ignore[reportUnnecessaryIsInstance] __all__ = [ 'MonikerStream', ] @@ -8,7 +9,7 @@ @final class MonikerStream: - def __init__(self, data : Optional[bytes]): + def __init__(self, data : Optional[bytes] = None): if data: self.__clsid = data[:16] self.__streamData = data[16:] @@ -26,9 +27,9 @@ def clsid(self) -> bytes: capable of processing the stream data. """ return self.__clsid - + @clsid.setter - def setter(self, data : bytes) -> None: + def _(self, data : bytes) -> None: if not isinstance(data, bytes): raise TypeError('CLSID MUST be bytes.') if len(data) != 16: @@ -41,10 +42,10 @@ def streamData(self) -> bytes: An array of bytes that specifies the reference to the linked object. """ return self.__streamData - + @streamData.setter - def setter(self, data : bytes) -> None: + def _(self, data : bytes) -> None: if not isinstance(data, bytes): raise TypeError('Stream data MUST be bytes.') self.__streamData = data - + diff --git a/extract_msg/structures/odt.py b/extract_msg/structures/odt.py index 04c0b8f5..8b115d83 100644 --- a/extract_msg/structures/odt.py +++ b/extract_msg/structures/odt.py @@ -10,7 +10,7 @@ @final class ODTStruct: - def __init__(self, data : Optional[bytes]): + def __init__(self, data : Optional[bytes] = None): if data: values = struct.unpack(' ODTCf: return self.__cf @cf.setter - def setter(self, value : ODTCf) -> None: + def _(self, value : ODTCf) -> None: if not isinstance(value, ODTCf): raise TypeError(':property cf: MUST be of type ODTCf.') @@ -50,21 +50,21 @@ def odtPersist1(self) -> ODTPersist1: return self.__persist1 @odtPersist1.setter - def setter(self, value : ODTPersist1) -> None: + def _(self, value : ODTPersist1) -> None: if not isinstance(value, ODTPersist1): raise TypeError(':property odtPersist1: MUST be of type ODTPersist1.') self.__persist1 = value @property - def odtPersist2(self) -> ODTPersist1: + def odtPersist2(self) -> ODTPersist2: """ Flags the specify additional information about the OLE object. """ return self.__persist2 @odtPersist2.setter - def setter(self, value : ODTPersist2) -> None: + def _(self, value : ODTPersist2) -> None: if not isinstance(value, ODTPersist2): raise TypeError(':property odtPersist2: MUST be of type ODTPersist2.') diff --git a/extract_msg/structures/ole_pres.py b/extract_msg/structures/ole_pres.py index 7a7ce87d..8535711c 100644 --- a/extract_msg/structures/ole_pres.py +++ b/extract_msg/structures/ole_pres.py @@ -37,7 +37,7 @@ def toBytes(self) -> bytes: if self.markerOrLength > 0xFFFFFFFD: ret += constants.st.ST_LE_UI32.pack(self.clipboardFormat) elif self.markerOrLength > 0: - ret += self.ansiString + ret += self.__ansiString return ret @property @@ -52,7 +52,7 @@ def ansiString(self) -> Optional[bytes]: return self.__ansiString @ansiString.setter - def setter(self, val : bytes) -> None: + def _(self, val : bytes) -> None: if not val: raise ValueError('Cannot set :property ansiString: to None or empty bytes. Set :property markerOrLength: to a value ') @@ -69,7 +69,7 @@ def clipboardFormat(self) -> Optional[ClipboardFormat]: return self.__clipboardFormat @clipboardFormat.setter - def setter(self, val : ClipboardFormat) -> None: + def _(self, val : ClipboardFormat) -> None: if not val: raise ValueError('Cannot set clipboard format to None.') if self.markerOrLength < 0xFFFFFFFE: @@ -88,7 +88,7 @@ def markerOrLength(self) -> int: return self.__markerOrLength @markerOrLength.setter - def setter(self, val : int) -> None: + def _(self, val : int) -> None: if val < 0: raise ValueError('markerOrLength must be a positive integer.') if val > 0xFFFFFFFF: @@ -125,7 +125,8 @@ def __init__(self, data : Optional[bytes]): reader = BytesReader(data) try: - data = reader.readStruct(self.__parseStruct) + items = reader.readStruct(self.__parseStruct) + #TODO except IOError: return @@ -134,6 +135,9 @@ def __init__(self, data : Optional[bytes]): def __bool__(self) -> bool: return self.__valid + def toBytes(self) -> bytes: + pass # TODO + @@ -230,7 +234,7 @@ def toBytes(self) -> Optional[bytes]: except struct.error: raise ValueError('DVTargetDevice structure contains too much data.') if self.__driverName: - ret += self.__driverName + 'b\x00' + ret += self.__driverName + b'\x00' if self.__deviceName: ret += self.__deviceName + b'\x00' if self.__portName: @@ -249,7 +253,7 @@ def driverName(self) -> Optional[bytes]: return self.__driverName @driverName.setter - def setter(self, data : Optional[bytes]) -> None: + def _(self, data : Optional[bytes]) -> None: self.__driverName = None if not data else data @property @@ -261,7 +265,7 @@ def deviceName(self) -> Optional[bytes]: return self.__deviceName @deviceName.setter - def setter(self, data : Optional[bytes]) -> None: + def _(self, data : Optional[bytes]) -> None: self.__deviceName = None if not data else data @property @@ -272,7 +276,7 @@ def portName(self) -> Optional[bytes]: return self.__portName @portName.setter - def setter(self, data : Optional[bytes]) -> None: + def _(self, data : Optional[bytes]) -> None: self.__portName = None if not data else data @property @@ -284,7 +288,7 @@ def extDevMode(self) -> Optional[DevModeA]: return self.__extDevMode @extDevMode.setter - def setter(self, data : Optional[DevModeA]) -> None: + def _(self, data : Optional[DevModeA]) -> None: self.__extDevMode = None if not data else data @@ -301,7 +305,7 @@ class OLEPresentationStream: advf : Union[int, ADVF] width : int height : int - data : int + data : bytes reserved2 : Optional[bytes] tocSignature : int tocEntries : List[TOCEntry] @@ -347,7 +351,7 @@ def __init__(self, data : bytes): self.tocSignature = reader.readUnsignedInt() self.tocEntries = [] if self.tocSignature == 0x494E414E: # b'NANI' in little endian. - for x in range(reader.readUnsignedInt()): + for _ in range(reader.readUnsignedInt()): self.tocEntries.append(TOCEntry(reader)) diff --git a/extract_msg/structures/ole_stream_struct.py b/extract_msg/structures/ole_stream_struct.py index 3ef48d7e..2042676f 100644 --- a/extract_msg/structures/ole_stream_struct.py +++ b/extract_msg/structures/ole_stream_struct.py @@ -28,7 +28,9 @@ def __init__(self, data : Optional[bytes] = None): if rmsSize > 0: self.__rms = MonikerStream(reader.read(rmsSize)) - # TODO implement the rest. It's all optional things. + if self.__flags & 1: + # Only check this stuff if this is not for an embedded object. + pass # TODO def toBytes(self) -> bytes: ret = b'\x01\x00\x00\x02' @@ -44,15 +46,15 @@ def toBytes(self) -> bytes: @property def reservedMonikerStream(self) -> Optional[MonikerStream]: """ - + A MonikerStream structure that can contain any arbitrary value. """ return self.__rms @reservedMonikerStream.setter - def setter(self, data : Optional[MonikerStream]) -> None: + def _(self, data : Optional[MonikerStream]) -> None: if data is not None and not isinstance(data, MonikerStream): raise TypeError('Reserved moniker stream must be a MonikerStream instance or None.') - + self.__rms = data - + diff --git a/extract_msg/structures/system_time.py b/extract_msg/structures/system_time.py index 806de47d..cb22b74e 100644 --- a/extract_msg/structures/system_time.py +++ b/extract_msg/structures/system_time.py @@ -3,7 +3,7 @@ ] -from typing import Optional +from typing import Any, Optional from .. import constants @@ -12,45 +12,144 @@ class SystemTime: """ A SYSTEMTIME struct, as defined in [MS-DTYP]. """ - - year : int = 0 - month : int = 0 - dayOfWeek : int = 0 - day : int = 0 - hour : int = 0 - minute : int = 0 - second : int = 0 - milliseconds : int = 0 - def __init__(self, data : Optional[bytes] = None): - data = data or (b'\x00' * 1) + data = data or (b'\x00' * 16) self.unpack(data) - def __eq__(self, other) -> bool: - return isinstance(other, SystemTime) and self.pack() == other.pack() + def __eq__(self, other : Any) -> bool: + return isinstance(other, SystemTime) and self.toBytes() == other.toBytes() - def __ne__(self, other) -> bool: + def __ne__(self, other : Any) -> bool: return not self.__eq__(other) def toBytes(self) -> bytes: """ Packs the current data into bytes. """ - return constants.st.ST_SYSTEMTIME.pack(self.year, self.month, - self.dayOfWeek, self.day, self.hour, - self.minute, self.second, - self.milliseconds) + return constants.st.ST_SYSTEMTIME.pack(self.__year, + self.__month, + self.__dayOfWeek, + self.__day, + self.__hour, + self.__minute, + self.__second, + self.__milliseconds) def unpack(self, data : bytes) -> None: """ Fills out the fields of this instance by unpacking the bytes. """ unpacked = constants.st.ST_SYSTEMTIME.unpack(data) - self.year = unpacked[0] - self.month = unpacked[1] - self.dayOfWeek = unpacked[2] - self.day = unpacked[3] - self.hour = unpacked[4] - self.minute = unpacked[5] - self.second = unpacked[6] - self.milliseconds = unpacked[7] + self.__year = unpacked[0] + self.__month = unpacked[1] + self.__dayOfWeek = unpacked[2] + self.__day = unpacked[3] + self.__hour = unpacked[4] + self.__minute = unpacked[5] + self.__second = unpacked[6] + self.__milliseconds = unpacked[7] + + @property + def day(self) -> int: + return self.__day + + @day.setter + def _(self, value : int) -> None: + if value < 0: + raise ValueError('Day must be positive.') + if value > 0xFFFF: + raise ValueError('Day must be less than 65535.') + + self.__day = value + + @property + def dayOfWeek(self) -> int: + return self.__dayOfWeek + + @dayOfWeek.setter + def _(self, value : int) -> None: + if value < 0: + raise ValueError('Day of week must be positive.') + if value > 0xFFFF: + raise ValueError('Day of week must be less than 65535.') + + self.__dayOfWeek = value + + @property + def hour(self) -> int: + return self.__hour + + @hour.setter + def _(self, value : int) -> None: + if value < 0: + raise ValueError('Hour must be positive.') + if value > 0xFFFF: + raise ValueError('Hour must be less than 65535.') + + self.__hour = value + + @property + def milliseconds(self) -> int: + return self.__milliseconds + + @milliseconds.setter + def _(self, value : int) -> None: + if value < 0: + raise ValueError('Milliseconds must be positive.') + if value > 0xFFFF: + raise ValueError('Milliseconds must be less than 65535.') + + self.__milliseconds = value + + @property + def minute(self) -> int: + return self.__minute + + @minute.setter + def _(self, value : int) -> None: + if value < 0: + raise ValueError('Minute must be positive.') + if value > 0xFFFF: + raise ValueError('Minute must be less than 65535.') + + self.__minute = value + + @property + def month(self) -> int: + return self.__month + + @month.setter + def _(self, value : int) -> None: + if value < 0: + raise ValueError('Month must be positive.') + if value > 0xFFFF: + raise ValueError('Month must be less than 65535.') + + self.__month = value + + @property + def second(self) -> int: + return self.__second + + @second.setter + def _(self, value : int) -> None: + if value < 0: + raise ValueError('Second must be positive.') + if value > 0xFFFF: + raise ValueError('Second must be less than 65535.') + + self.__second = value + + @property + def year(self) -> int: + return self.__year + + @year.setter + def _(self, value : int) -> None: + if value < 0: + raise ValueError('Year must be positive.') + if value > 0xFFFF: + raise ValueError('Year must be less than 65535.') + + self.__year = value + diff --git a/extract_msg/structures/time_zone_definition.py b/extract_msg/structures/time_zone_definition.py index 8df261d0..c06f8a1a 100644 --- a/extract_msg/structures/time_zone_definition.py +++ b/extract_msg/structures/time_zone_definition.py @@ -30,7 +30,7 @@ def __init__(self, data : Optional[bytes] = None): cchKeyName = reader.readUnsignedShort() self.__keyName = reader.read(2 * cchKeyName).decode('utf-16-le') cRules = reader.readUnsignedShort() - if len(cRules) < 1 or len(cRules) > 1024: + if cRules < 1 or cRules > 1024: raise ValueError('Value for cRules was out of range.') self.__rules = [reader.readClass(TZRule) for _ in range(cRules)] @@ -59,11 +59,11 @@ def keyName(self) -> str: return self.__keyName @keyName.setter - def setter(self, value : str) -> None: + def _(self, value : str) -> None: value = str(value) if len(value) > 260: raise ValueError('Key name must be a string less than 261 characters.') - + self.__keyName = value @property @@ -72,14 +72,14 @@ def majorVersion(self) -> int: The major version. """ return self.__majorVersion - + @majorVersion.setter - def setter(self, value : int) -> None: + def _(self, value : int) -> None: if value > 255: raise ValueError('Major version cannot be greater than 255') if value < 0: raise ValueError('Major version must be positive.') - + self.__minorVersion = value @property @@ -88,14 +88,14 @@ def minorVersion(self) -> int: The minor version. """ return self.__minorVersion - + @minorVersion.setter - def setter(self, value : int) -> None: + def _(self, value : int) -> None: if value > 255: raise ValueError('Minor version cannot be greater than 255') if value < 0: raise ValueError('Minor version must be positive.') - + self.__minorVersion = value @property diff --git a/extract_msg/utils.py b/extract_msg/utils.py index 639c4257..4e170825 100644 --- a/extract_msg/utils.py +++ b/extract_msg/utils.py @@ -45,7 +45,6 @@ 'validateHtml', 'verifyPropertyId', 'verifyType', - 'windowsUnicode', ] @@ -1231,8 +1230,3 @@ def verifyType(_type) -> None: if _type is not None: if (_type not in constants.VARIABLE_LENGTH_PROPS_STRING) and (_type not in constants.FIXED_LENGTH_PROPS_STRING): raise UnknownTypeError(f'Unknown type {_type}.') - - -def windowsUnicode(string) -> Optional[str]: - return str(string, 'utf-16-le') if string is not None else None - From d0bc11df219eb2359a3f88e21279a7b4a46b28b9 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sat, 9 Sep 2023 16:11:19 -0700 Subject: [PATCH 40/68] Typing, fix bug from #eba824a --- CHANGELOG.md | 1 + extract_msg/msg_classes/msg.py | 18 +++++++++--------- extract_msg/properties/named.py | 12 ++++++------ extract_msg/properties/prop.py | 18 +++++++++--------- extract_msg/utils.py | 12 ++++++------ 5 files changed, 31 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecfcb3fb..9bf4f1fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ * changed `TZRule` to use unsigned values where applicable. * Changed `TZRule` to require the 14 null bytes (I noticed there is a note about outlook violating that standard and will look into it). * Removed unneeded function `windowsUnicode`. +* Moved `FixedLengthProperty.parseType` to the private API. This was not intended for external use anyways, so leaving it as public API didn't make sense. **v0.45.0** * BREAKING: Changed parsing of string multiple properties to remove the trailing null byte. This *will* cause the output of parsing them to differ. diff --git a/extract_msg/msg_classes/msg.py b/extract_msg/msg_classes/msg.py index c2da4f97..fe897837 100644 --- a/extract_msg/msg_classes/msg.py +++ b/extract_msg/msg_classes/msg.py @@ -44,7 +44,7 @@ from ..structures.contact_link_entry import ContactLinkEntry from ..utils import ( divide, guessEncoding, hasLen, inputToMsgPath, makeWeakRef, - msgPathToString, parseType, verifyPropertyId, verifyType, windowsUnicode + msgPathToString, parseType, verifyPropertyId, verifyType ) @@ -110,20 +110,20 @@ def __init__(self, path, **kwargs): specific exceptions was raised. """ # Retrieve all the kwargs that we need. - self.__inscFeat = kwargs.get('insecureFeatures', InsecureFeatures.NONE) - prefix = cast(str, kwargs.get('prefix', '')) + self.__inscFeat : InsecureFeatures = kwargs.get('insecureFeatures', InsecureFeatures.NONE) + prefix : str = cast(str, kwargs.get('prefix', '')) self.__parentMsg = makeWeakRef(cast(MSGFile, kwargs.get('parentMsg'))) - self.__treePath = kwargs.get('treePath', []) + [makeWeakRef(self)] + self.__treePath = kwargs.get('treePath', []) + [weakref.ref(self)] # Verify it is a valid class. if self.__parentMsg and not isinstance(self.__parentMsg(), MSGFile): raise TypeError(':param parentMsg: must be an instance of MSGFile or a subclass.') - filename = kwargs.get('filename', None) - overrideEncoding = kwargs.get('overrideEncoding', None) + filename = kwargs.get('filename') + overrideEncoding = kwargs.get('overrideEncoding') # WARNING DO NOT MANUALLY MODIFY PREFIX. Let the program set it. self.__path = path self.__initAttachmentFunc = kwargs.get('initAttachment', initStandardAttachment) - self.__attachmentsDelayed = kwargs.get('delayAttachments', False) + self.__attachmentsDelayed = bool(kwargs.get('delayAttachments', False)) self.__attachmentsReady = False self.__errorBehavior = ErrorBehavior(kwargs.get('errorBehavior', ErrorBehavior.THROW)) self.__dateFormat = kwargs.get('dateFormat', DATE_FORMAT) @@ -309,7 +309,7 @@ def _getTypedProperty(self, propertyID : str, _type = None) -> Tuple[bool, Optio return True, ret - def _getTypedStream(self, filename : MSG_PATH, prefix : bool = True, _type = None) -> Tuple[bool, Optional[Any]]: + def _getTypedStream(self, filename : MSG_PATH, prefix : bool = True, _type : Optional[str] = None) -> Tuple[bool, Optional[Any]]: """ Gets the contents of the specified stream as the type that it is supposed to be. @@ -335,7 +335,7 @@ def _getTypedStream(self, filename : MSG_PATH, prefix : bool = True, _type = Non continue if len(contents) == 0: return True, None # We found the file, but it was empty. - extras = [] + extras : List[bytes]= [] _type = x[-4:] if x[-4] == '1': # It's a multiple if _type in ('101F', '101E'): diff --git a/extract_msg/properties/named.py b/extract_msg/properties/named.py index 00a75f1d..7b921131 100644 --- a/extract_msg/properties/named.py +++ b/extract_msg/properties/named.py @@ -48,8 +48,8 @@ class Named: def __init__(self, msg : MSGFile): self.__msg = weakref.ref(msg) # Get the basic streams. If all are emtpy, then nothing to do. - guidStream = self.getStream('__substg1.0_00020102') - entryStream = self.getStream('__substg1.0_00030102') + guidStream = self.getStream('__substg1.0_00020102') or b'' + entryStream = self.getStream('__substg1.0_00030102') or b'' self.guidStream = guidStream self.entryStream = entryStream self.namesStream = self.getStream('__substg1.0_00040102') or b'' @@ -223,7 +223,7 @@ def __init__(self, named : Named, streamSource : Union[MSGFile, AttachmentBase]) self.__named = named self.__streamSource = weakref.ref(streamSource) - def __getitem__(self, item): + def __getitem__(self, item : Union[Tuple[str, str], NamedPropertyBase]): """ Get a named property using the [] operator. Item must be a named property instance or a tuple with 2 items: the name and the GUID string. @@ -238,7 +238,7 @@ def __getitem__(self, item): else: return source._getTypedData(self.__named[item].propertyStreamID) - def get(self, item, default : _T = None) -> Union[Any, _T]: + def get(self, item : Union[Tuple[str, str], NamedPropertyBase], default : _T = None) -> Union[Any, _T]: """ Get a named property, returning the value of :param default: if not found. Item must be a tuple with 2 items: the name and the GUID string. @@ -254,7 +254,7 @@ def get(self, item, default : _T = None) -> Union[Any, _T]: class NamedPropertyBase(abc.ABC): - def __init__(self, entry : Dict): + def __init__(self, entry : Dict[str, Any]): self.__entry = entry self.__guidIndex = entry['guid_index'] self.__namedPropertyID = entry['pid'] @@ -290,7 +290,7 @@ def propertyStreamID(self) -> str: return self.__propertyStreamID @property - def rawEntry(self) -> Dict: + def rawEntry(self) -> Dict[str, Any]: return copy.deepcopy(self.__entry) @property diff --git a/extract_msg/properties/prop.py b/extract_msg/properties/prop.py index 002e11bd..651fd772 100644 --- a/extract_msg/properties/prop.py +++ b/extract_msg/properties/prop.py @@ -3,7 +3,7 @@ __all__ = [ # Classes: - 'FixedLengthProp' + 'FixedLengthProp', 'PropBase', 'VariableLengthProp', @@ -84,9 +84,9 @@ class FixedLengthProp(PropBase): def __init__(self, data : bytes): super().__init__(data) - self.__value = self.parseType(self.type, constants.st.STFIX.unpack(data)[0]) + self.__value = self._parseType(self.type, constants.st.STFIX.unpack(data)[0], data) - def parseType(self, _type : int, stream : bytes) -> Any: + def _parseType(self, _type : int, stream : bytes, raw : bytes) -> Any: """ Converts the data in :param stream: to a much more accurate type, specified by :param _type:, if possible. @@ -104,11 +104,11 @@ def parseType(self, _type : int, stream : bytes) -> Any: logger.warning('Property type is PtypNull, but is not equal to 0.') value = None elif _type == 0x0002: # PtypInteger16 - value = constants.st.ST_LE_I16.unpack(value)[0] + value = constants.st.ST_LE_I16.unpack(value[:3])[0] elif _type == 0x0003: # PtypInteger32 - value = constants.st.ST_LE_I32.unpack(value)[0] + value = constants.st.ST_LE_I32.unpack(value[:4])[0] elif _type == 0x0004: # PtypFloating32 - value = constants.st.ST_LE_F32.unpack(value)[0] + value = constants.st.ST_LE_F32.unpack(value[:4])[0] elif _type == 0x0005: # PtypFloating64 value = constants.st.ST_LE_F64.unpack(value)[0] elif _type == 0x0006: # PtypCurrency @@ -117,7 +117,7 @@ def parseType(self, _type : int, stream : bytes) -> Any: value = constants.st.ST_LE_F64.unpack(value)[0] return constants.PYTPFLOATINGTIME_START + datetime.timedelta(days = value) elif _type == 0x000A: # PtypErrorCode - value = constants.st.ST_LE_I32.unpack(value)[0] + value = constants.st.ST_LE_UI32.unpack(value[:4])[0] try: value = ErrorCodeType(value) except ValueError: @@ -137,8 +137,8 @@ def parseType(self, _type : int, stream : bytes) -> Any: rawTime = constants.st.ST_LE_UI64.unpack(value)[0] try: value = filetimeToDatetime(rawTime) - except ValueError as e: - logger.exception(self.rawData) + except ValueError: + logger.exception(raw) elif _type == 0x0048: # PtypGuid # TODO parsing for this pass diff --git a/extract_msg/utils.py b/extract_msg/utils.py index 4e170825..83a816e7 100644 --- a/extract_msg/utils.py +++ b/extract_msg/utils.py @@ -72,8 +72,8 @@ from html import escape as htmlEscape from typing import ( - Any, Callable, Dict, Iterable, List, Optional, Sequence, TypeVar, - TYPE_CHECKING, Union + Any, AnyStr, Callable, Dict, Iterable, List, Optional, Sequence, + TypeVar, TYPE_CHECKING, Union ) from . import constants @@ -231,7 +231,7 @@ def dictGetCasedKey(_dict : Dict[str, Any], key : str) -> str: raise KeyError(key) -def divide(string, length : int) -> List: +def divide(string : AnyStr, length : int) -> List[AnyStr]: """ Divides a string into multiple substrings of equal length. If there is not enough for the last substring to be equal, it will simply use the rest of @@ -674,7 +674,7 @@ def msgPathToString(inp : Union[str, Iterable[str]]) -> str: return inp -def parseType(_type : int, stream, encoding, extras): +def parseType(_type : int, stream : Union[int, bytes], encoding : str, extras : Sequence[bytes]): """ Converts the data in :param stream: to a much more accurate type, specified by :param _type:. @@ -746,7 +746,7 @@ def parseType(_type : int, stream, encoding, extras): elif _type == 0x0048: # PtypGuid return bytesToGuid(value) elif _type == 0x00FB: # PtypServerId - count = constants.st.ST_LE_UI16.unpack(value[:2]) + count = constants.st.ST_LE_UI16.unpack(value[:2])[0] # If the first byte is a 1 then it uses the ServerID structure. if value[3] == 1: from .structures.misc_id import ServerID @@ -1221,7 +1221,7 @@ def verifyPropertyId(id : str) -> None: raise InvaildPropertyIdError('ID was not a 4 digit hexadecimal string') -def verifyType(_type) -> None: +def verifyType(_type : Optional[str]) -> None: """ Verifies that the type is valid. Raises an exception if it is not. From 6933a8f4225ee04902595048a9f18c44c05527e2 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Thu, 14 Sep 2023 17:34:20 -0700 Subject: [PATCH 41/68] Fix bugs and more todo work --- extract_msg/msg_classes/message_base.py | 2 +- extract_msg/structures/ole_pres.py | 29 +++++++++++++++++++------ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/extract_msg/msg_classes/message_base.py b/extract_msg/msg_classes/message_base.py index fd8ea30a..7300390c 100644 --- a/extract_msg/msg_classes/message_base.py +++ b/extract_msg/msg_classes/message_base.py @@ -1248,7 +1248,7 @@ def recipients(self) -> List[Recipient]: return [Recipient(recipientDir, self, self.recipientTypeClass) for recipientDir in recipientDirs] @property - def recipientTypeClass() -> Type[enum.IntEnum]: + def recipientTypeClass(self) -> Type[enum.IntEnum]: """ The class to use for a recipient's recipientType property. diff --git a/extract_msg/structures/ole_pres.py b/extract_msg/structures/ole_pres.py index 8535711c..88b2be42 100644 --- a/extract_msg/structures/ole_pres.py +++ b/extract_msg/structures/ole_pres.py @@ -110,7 +110,8 @@ class DevModeA: """ A DEVMODEA structure, as specified in [MS-OLEDS]. For the purposes of parsing from bytes, if something goes wrong this will evaluate to False when - converting to bool. If no data is prodided + converting to bool. If no data is prodided, the fields are set to default + values. """ __parseStruct = struct.Struct('<32s32s4HI13H14xI4x4I16x') @@ -126,7 +127,10 @@ def __init__(self, data : Optional[bytes]): try: items = reader.readStruct(self.__parseStruct) - #TODO + self.__specVersion = items[0] + self.__driverVersion = items[1] + self.__size = items[2] + # TODO except IOError: return @@ -215,15 +219,15 @@ def toBytes(self) -> Optional[bytes]: currentPosition = 8 offset1 = 8 if self.__driverName else 0 - if offset1: + if self.__driverName: currentPosition += len(self.__driverName) + 1 offset2 = currentPosition if self.__deviceName else 0 - if offset2: + if self.__deviceName: currentPosition += len(self.__deviceName) + 1 offset3 = currentPosition if self.__portName else 0 - if offset3: + if self.__portName: currentPosition += len(self.__portName) + 1 extDevModeBytes = self.__extDevMode.toBytes() if self.__extDevMode else None @@ -336,7 +340,7 @@ def __init__(self, data : bytes): self.advf = reader.readUnsignedInt() # Reserved1. - reader.readUnsignedInt() + reader.read(4) self.width = reader.readUnsignedInt() self.height = reader.readUnsignedInt() @@ -360,5 +364,16 @@ class TOCEntry: def __init__(self, reader : Union[bytes, BytesReader]): if isinstance(reader, bytes): reader = BytesReader(reader) - + self.__clipFormat = ClipboardFormatOrAnsiString(reader) + targetDeviceSize = reader.readUnsignedInt() + self.__aspect = reader.readUnsignedInt() + self.__lindex = reader.readUnsignedInt() + self.__tymed = reader.readUnsignedInt() + reader.read(4) + self.__advf = reader.readUnsignedInt() + reader.read(4) + if targetDeviceSize == 0: + self.__targetDevice = None + else: + self.__targetDevice = DVTargetDevice(reader.read(targetDeviceSize)) # TODO \ No newline at end of file From 639015601407ab680f032d8f984e9e5212f09fad Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Thu, 14 Sep 2023 17:58:33 -0700 Subject: [PATCH 42/68] Update config for pyright --- pyrightconfig.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyrightconfig.json b/pyrightconfig.json index 31845b85..34dfc84f 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -1,4 +1,7 @@ { "pythonVersion": "3.8", - "pythonPlatform": "All" + "pythonPlatform": "All", + "reportUnnecessaryIsInstance": "information", + "reportConstantRedefinition": "error", + "reportDeprecated": "warning" } \ No newline at end of file From 8db612425c2bd41d73e8e26d77e24788b523263a Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Thu, 14 Sep 2023 18:01:58 -0700 Subject: [PATCH 43/68] Adjustments to typing --- extract_msg/structures/ole_pres.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/extract_msg/structures/ole_pres.py b/extract_msg/structures/ole_pres.py index 88b2be42..0c4eead4 100644 --- a/extract_msg/structures/ole_pres.py +++ b/extract_msg/structures/ole_pres.py @@ -9,7 +9,7 @@ import struct -from typing import List, Optional, Union +from typing import List, Final, Optional, Union from .. import constants from ._helpers import BytesReader @@ -113,7 +113,7 @@ class DevModeA: converting to bool. If no data is prodided, the fields are set to default values. """ - __parseStruct = struct.Struct('<32s32s4HI13H14xI4x4I16x') + PARSE_STRUCT : Final[struct.Struct] = struct.Struct('<32s32s4HI13H14xI4x4I16x') def __init__(self, data : Optional[bytes]): self.__valid = data is None @@ -126,7 +126,8 @@ def __init__(self, data : Optional[bytes]): reader = BytesReader(data) try: - items = reader.readStruct(self.__parseStruct) + items = reader.readStruct(self.PARSE_STRUCT) + # Double check these are the right indexes. self.__specVersion = items[0] self.__driverVersion = items[1] self.__size = items[2] From b6fafa9ae328ca9590ba04cddfbf7dc8a3c70acc Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Mon, 18 Sep 2023 19:14:59 -0700 Subject: [PATCH 44/68] More work on DevModeA --- extract_msg/enums.py | 28 ++++++++++++++++++++++++++++ extract_msg/structures/ole_pres.py | 28 +++++++++++++++++++++++----- 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/extract_msg/enums.py b/extract_msg/enums.py index b0ded000..a97426ed 100644 --- a/extract_msg/enums.py +++ b/extract_msg/enums.py @@ -24,6 +24,7 @@ 'DeencapType', 'DirectoryEntryType', 'DisplayType', + 'DMPaperSize', 'DVAspect', 'ElectronicAddressProperties', 'EntryIDType', @@ -483,6 +484,33 @@ class DisplayType(enum.IntEnum): +class DMPaperSize: + DMPAPER_LETTER = 0x0001 + DMPAPER_LEGAL = 0x0005 + DMPAPER_A3 = 0x0008 + DMPAPER_A4 = 0x0009 + DMPAPER_A4SMALL = 0x000A + DMPAPER_A5 = 0x000B + DMPAPER_B4 = 0x000C + DMPAPER_B5 = 0x000D + DMPAPER_10X14 = 0x0010 + DMPAPER_11X17 = 0x0011 + DMPAPER_CSHEET = 0x0018 + DMPAPER_DBL_JAPANESE_POSTCARD = 0x0045 + DMPAPER_A6 = 0x0046 + DMPAPER_A3_ROTATED = 0x004C + DMPAPER_A4_ROTATED = 0x004D + DMPAPER_A5_ROTATED = 0x004E + DMPAPER_B4_JIS_ROTATED = 0x004F + DMPAPER_B5_JIS_ROTATED = 0x0050 + DMPAPER_A6_ROTATED = 0x0053 + DMPAPER_B6_JIS = 0x0058 + DMPAPER_B6_JIS_ROTATED = 0x0059 + DMPAPER_12X11 = 0x005A + # TODO + + + class DVAspect(enum.IntEnum): """ Part of the extra data for Outlook signatures. Different sources seem to diff --git a/extract_msg/structures/ole_pres.py b/extract_msg/structures/ole_pres.py index 0c4eead4..22ab54c3 100644 --- a/extract_msg/structures/ole_pres.py +++ b/extract_msg/structures/ole_pres.py @@ -7,6 +7,8 @@ ] +import enum +import logging import struct from typing import List, Final, Optional, Union @@ -16,6 +18,10 @@ from ..enums import ADVF, ClipboardFormat, DVAspect +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) + + class ClipboardFormatOrAnsiString: def __init__(self, reader : Union[bytes, BytesReader]): if isinstance(reader, bytes): @@ -113,7 +119,7 @@ class DevModeA: converting to bool. If no data is prodided, the fields are set to default values. """ - PARSE_STRUCT : Final[struct.Struct] = struct.Struct('<32s32s4HI13H14xI4x4I16x') + PARSE_STRUCT : Final[struct.Struct] = struct.Struct('<32s32s4HI13h14xI4x4I16x') def __init__(self, data : Optional[bytes]): self.__valid = data is None @@ -127,10 +133,17 @@ def __init__(self, data : Optional[bytes]): try: items = reader.readStruct(self.PARSE_STRUCT) - # Double check these are the right indexes. - self.__specVersion = items[0] - self.__driverVersion = items[1] - self.__size = items[2] + self.__deviceName = items[0] + self.__formName = items[1] + self.__specVersion = items[2] + self.__driverVersion = items[3] + if items[4] != self.PARSE_STRUCT.size: + logger.warn(f'Unexpected `size` field for DevModeA detected ({items[4]})') + self.__diverExtra = items[5] + self.__fields = _DevModeFields(items[6]) + # TODO fields specifies if we should read the field or ignore it. + self.__orientation = items[7] + self.__paperSize = items[8] # TODO except IOError: return @@ -145,6 +158,11 @@ def toBytes(self) -> bytes: +class _DevModeFields(enum.IntFlag): + DM_NUP = 0b00000000000000000000000000000000 + DM_SCALE = 0b00000000000000000000000000000000 + DM_ICMINTENT = 0b00000000000000000000000000000000 + # TODO class DVTargetDevice: """ From 3631e2b1979e3bd4838ab2f3e025ec6afa5f2fdf Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 19 Sep 2023 20:41:17 -0700 Subject: [PATCH 45/68] Add ignore comment in testing --- extract_msg_tests/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extract_msg_tests/constants.py b/extract_msg_tests/constants.py index fb93224a..53063fa3 100644 --- a/extract_msg_tests/constants.py +++ b/extract_msg_tests/constants.py @@ -14,4 +14,4 @@ if bool(userTestDir := os.environ.get('EXTRACT_MSG_TEST_DIR')): userTestDir = Path(userTestDir) if userTestDir.exists(): - USER_TEST_DIR = userTestDir + USER_TEST_DIR = userTestDir # type: ignore From 7e54b85ad3b63530232d879449c2933781951cc8 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Mon, 25 Sep 2023 20:24:07 -0700 Subject: [PATCH 46/68] Progress on DevModeA --- extract_msg/structures/ole_pres.py | 122 ++++++++++++++++++++++++++--- 1 file changed, 111 insertions(+), 11 deletions(-) diff --git a/extract_msg/structures/ole_pres.py b/extract_msg/structures/ole_pres.py index 22ab54c3..e3e6f956 100644 --- a/extract_msg/structures/ole_pres.py +++ b/extract_msg/structures/ole_pres.py @@ -121,12 +121,36 @@ class DevModeA: """ PARSE_STRUCT : Final[struct.Struct] = struct.Struct('<32s32s4HI13h14xI4x4I16x') - def __init__(self, data : Optional[bytes]): + def __init__(self, data : Optional[bytes] = None): self.__valid = data is None + + # Set default values for fields that may not be initialized. + self.__orientation = 0 + self.__paperSize = 0 + self.__paperLength = 0 + self.__paperWidth = 0 + self.__scale = 0 + self.__copies = 0 + self.__defaultSource = 0 + self.__printQuality = 0 + self.__color = 0 + self.__duplex = 0 + self.__yResolution = 0 + self.__ttOption = 0 + self.__collate = 0 + self.__nup = 0 + self.__icmMethod = 0 + self.__icmIntent = 0 + self.__mediaType = 0 + self.__ditherType = 0 + if self.__valid: self.__deviceName = b'\x00' * 32 self.__formName = b'\x00' * 32 - # TODO set all properties to null values. + self.__specVersion = 0 + self.__driverVersion = 0 + self.__driverExtra = 0 + self.__fields = _DevModeFields[0] return reader = BytesReader(data) @@ -139,12 +163,45 @@ def __init__(self, data : Optional[bytes]): self.__driverVersion = items[3] if items[4] != self.PARSE_STRUCT.size: logger.warn(f'Unexpected `size` field for DevModeA detected ({items[4]})') - self.__diverExtra = items[5] + self.__driverExtra = items[5] self.__fields = _DevModeFields(items[6]) # TODO fields specifies if we should read the field or ignore it. - self.__orientation = items[7] - self.__paperSize = items[8] - # TODO + if _DevModeFields.DM_ORIENTATION in self.__fields: + self.__orientation = items[7] + if _DevModeFields.DM_PAPERSIZE in self.__fields: + self.__paperSize = items[8] + if _DevModeFields.DM_PAPERLENGTH in self.__fields: + self.__paperLength = items[9] + if _DevModeFields.DM_PAPERWIDTH in self.__fields: + self.__paperWidth = items[10] + if _DevModeFields.DM_SCALE in self.__fields: + self.__scale = items[11] + if _DevModeFields.DM_COPIES in self.__fields: + self.__copies = items[12] + if _DevModeFields.DM_DEFAULTSOURCE in self.__fields: + self.__defaultSource = items[13] + if _DevModeFields.DM_PRINTQUALITY in self.__fields: + self.__printQuality = items[14] + if _DevModeFields.DM_COLOR in self.__fields: + self.__color = items[15] + if _DevModeFields.DM_DUPLEX in self.__fields: + self.__duplex = items[16] + if _DevModeFields.DM_YRESOLUTION in self.__fields: + self.__yResolution = items[17] + if _DevModeFields.DM_TTOPTION in self.__fields: + self.__ttOption = items[18] + if _DevModeFields.DM_COLLATE in self.__fields: + self.__collate = items[19] + if _DevModeFields.DM_NUP in self.__fields: + self.__nup = items[20] + if _DevModeFields.DM_ICMMETHOD in self.__fields: + self.__icmMethod = items[21] + if _DevModeFields.DM_ICMINTENT in self.__fields: + self.__icmIntent = items[22] + if _DevModeFields.DM_MEDIATYPE in self.__fields: + self.__mediaType = items[23] + if _DevModeFields.DM_DITHERTYPE in self.__fields: + self.__ditherType = items[24] except IOError: return @@ -154,15 +211,58 @@ def __bool__(self) -> bool: return self.__valid def toBytes(self) -> bytes: - pass # TODO + return self.PARSE_STRUCT.pack( + self.__deviceName, + self.__formName, + self.__specVersion, + self.__driverVersion, + self.PARSE_STRUCT.size, + self.__driverExtra, + self.__fields, + self.__orientation, + self.__paperSize, + self.__paperLength, + self.__paperWidth, + self.__scale, + self.__copies, + self.__defaultSource, + self.__printQuality, + self.__color, + self.__duplex, + self.__yResolution, + self.__ttOption, + self.__collate, + self.__nup, + self.__icmMethod, + self.__icmIntent, + self.__mediaType, + self.__ditherType, + ) class _DevModeFields(enum.IntFlag): - DM_NUP = 0b00000000000000000000000000000000 - DM_SCALE = 0b00000000000000000000000000000000 - DM_ICMINTENT = 0b00000000000000000000000000000000 - # TODO + DM_NUP = 0b00000000000000000000000000000010 + DM_SCALE = 0b00000000000000000000000000001000 + DM_PAPERWIDTH = 0b00000000000000000000000000010000 + DM_PAPERLENGTH = 0b00000000000000000000000000100000 + DM_PAPERSIZE = 0b00000000000000000000000001000000 + DM_ORIENTATION = 0b00000000000000000000000010000000 + DM_COLLATE = 0b00000000000000000000000100000000 + DM_TTOPTION = 0b00000000000000000000001000000000 + DM_YRESOLUTION = 0b00000000000000000000010000000000 + DM_DUPLEX = 0b00000000000000000000100000000000 + DM_COLOR = 0b00000000000000000001000000000000 + DM_PRINTQUALITY = 0b00000000000000000010000000000000 + DM_DEFAULTSOURCE = 0b00000000000000000100000000000000 + DM_COPIES = 0b00000000000000001000000000000000 + DM_ICMMETHOD = 0b00000000000000010000000000000000 + DM_FORMNAME = 0b00000000100000000000000000000000 + DM_DITHERTYPE = 0b00100000000000000000000000000000 + DM_MEDIATYPE = 0b01000000000000000000000000000000 + DM_ICMINTENT = 0b10000000000000000000000000000000 + + class DVTargetDevice: """ From ebd6327577902409841173b6b2366ab5c50fdb1b Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 27 Sep 2023 18:09:14 -0700 Subject: [PATCH 47/68] Minor patch to deal with #387. Needs better --- extract_msg/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extract_msg/utils.py b/extract_msg/utils.py index 83a816e7..c1858e7d 100644 --- a/extract_msg/utils.py +++ b/extract_msg/utils.py @@ -214,7 +214,7 @@ def decodeRfc2047(encoded : str) -> str: # decode_header header will return a string instead of bytes for the first # object if the input is not encoded, something that is frustrating. return ''.join( - x[0].decode(x[1] or 'ascii') if isinstance(x[0], bytes) else x[0] + x[0].decode(x[1] or 'raw-unicode-escape') if isinstance(x[0], bytes) else x[0] for x in email.header.decode_header(encoded) ) From 9368cc261608c735c6078ef0acf4795158a0d371 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sat, 30 Sep 2023 09:51:22 -0700 Subject: [PATCH 48/68] Better fix (might change) for #387 --- CHANGELOG.md | 2 ++ extract_msg/__init__.py | 2 +- extract_msg/msg_classes/message_base.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bf4f1fe..4aeca1ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ **v0.46.0** * [[TeamMsgExtractor #95](https://github.com/TeamMsgExtractor/msg-extractor/issues/95)] Adjusted the `overrideEncoding` property of `MSGFile` to allow automatic encoding detection. Simply set the property to the string `"chardet"` and, assuming the `chardet` module is installed, it will analyze a number of the strings to try and form a consensus about the encoding. This will *ignore* the specified encoding *only if* if successfully detects. Otherwise it will log a warning and fall back to the default behavior. +* [[TeamMsgExtractor #387](https://github.com/TeamMsgExtractor/msg-extractor/issues/387)] Changed `extract_msg.utils.decodeRfc2047` to not throw decoding errors if the content given is not ASCII. +* [[TeamMsgExtractor #387](https://github.com/TeamMsgExtractor/msg-extractor/issues/387)] Changed header parsing policy to `email.policy.compat32` to prevent partial parsing of quoted header fields. * Removed methods deprecated in `v0.45.0`. * Changed the base class of `EntryID` from no base class to `abc.ABC`. * Added `position` property to `EntryID` to tell how many bytes were used to create the `EntryID`. diff --git a/extract_msg/__init__.py b/extract_msg/__init__.py index eedf1935..4720b5d7 100644 --- a/extract_msg/__init__.py +++ b/extract_msg/__init__.py @@ -27,7 +27,7 @@ # along with this program. If not, see . __author__ = 'Destiny Peterson & Matthew Walker' -__date__ = '2023-08-30' +__date__ = '2023-09-30' __version__ = '0.46.0' __all__ = [ diff --git a/extract_msg/msg_classes/message_base.py b/extract_msg/msg_classes/message_base.py index 7300390c..eee7561f 100644 --- a/extract_msg/msg_classes/message_base.py +++ b/extract_msg/msg_classes/message_base.py @@ -1033,7 +1033,7 @@ def header(self) -> email.message.Message: # Fix an issue with prefixed headers not parsing correctly. if headerText.startswith('Microsoft Mail Internet Headers Version 2.0'): headerText = headerText[43:].lstrip() - header = HeaderParser(policy = policy.default).parsestr(headerText) + header = HeaderParser(policy = policy.compat32).parsestr(headerText) else: logger.info('Header is empty or was not found. Header will be generated from other streams.') header = HeaderParser(policy = policy.default).parsestr('') From 1fb39c455cd29fc3813973293a640779075d97ba Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sat, 30 Sep 2023 11:43:26 -0700 Subject: [PATCH 49/68] Fix TZRule --- CHANGELOG.md | 4 ++-- extract_msg/structures/tz_rule.py | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4aeca1ab..6a0265e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,8 +43,8 @@ * Removed all instances of the `rawData` property in favor of the `toBytes` method. For now, many of these will simply return the raw data used, specifically those that are still unmodifiable. Any whose properties have the ability to be modified will have properly implemented versions. These classes also allow `None` to be passed as the value for their data, which will be the default if no arguments have been passed to the constructor. If no arguments or `None` is given as the data, it will create a new instance with default values. This is all in an effort to move towards the ability to create new MSG files and the `MSGWriter` class. All `toBytes` methods will either exclusively return `bytes` or will return `None` to specify that the structure isn't valid to convert to bytes. Structures that may be invalid will be annotated as `Optional[bytes]` for the return type. * Removed the individual `PropBase` flag properties and changed the main `flags` property to return an enum containing the flags. * Changed various data structs to allow modification and creation of new instances for writing to an MSG file. -* changed `TZRule` to use unsigned values where applicable. -* Changed `TZRule` to require the 14 null bytes (I noticed there is a note about outlook violating that standard and will look into it). +* Changed `TZRule` to use unsigned values where applicable. +* Changed `TZRule` to require the 14 null bytes (I commented it out completely on accident instead of swapping it to a plain read). It now logs a warning about the bytes not being null. * Removed unneeded function `windowsUnicode`. * Moved `FixedLengthProperty.parseType` to the private API. This was not intended for external use anyways, so leaving it as public API didn't make sense. diff --git a/extract_msg/structures/tz_rule.py b/extract_msg/structures/tz_rule.py index 923b973f..8b3dbe18 100644 --- a/extract_msg/structures/tz_rule.py +++ b/extract_msg/structures/tz_rule.py @@ -3,6 +3,8 @@ ] +import logging + from struct import Struct from typing import Final, final, Optional @@ -11,6 +13,10 @@ from .system_time import SystemTime +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) + + @final class TZRule: """ @@ -39,7 +45,10 @@ def __init__(self, data : Optional[bytes] = None): reader.assertRead(b'\x3E\x00') self.__flags = TZFlag(reader.readUnsignedShort()) self.__year = reader.readUnsignedShort() - reader.assertNull(14) + # This *MUST* be null, however I've seen Outlook not follow that. Simply + # log a warning about it even though it's a violation. + if any(b := reader.read(14)): + logger.warning(f'Read TZRule with non-null X section (got {b}).') self.__bias = reader.readInt() self.__standardBias = reader.readInt() self.__daylightBias = reader.readInt() @@ -48,8 +57,8 @@ def __init__(self, data : Optional[bytes] = None): def toBytes(self) -> bytes: return self.__struct.pack(self.__majorVersion, - self.__minorVersion, - 62, + self.__minorVersion, + 62, 0, self.__flags, self.__year, From 8651c8323f4000b92ba4cc0e4381aa19334e9844 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Sun, 1 Oct 2023 16:42:38 -0700 Subject: [PATCH 50/68] More olepres, losts of various fixes --- CHANGELOG.md | 1 + extract_msg/enums.py | 24 + extract_msg/structures/__init__.py | 5 +- extract_msg/structures/cfoas.py | 99 ++++ extract_msg/structures/contact_link_entry.py | 35 +- extract_msg/structures/dev_mode_a.py | 548 +++++++++++++++++++ extract_msg/structures/dv_target_device.py | 162 ++++++ extract_msg/structures/entry_id.py | 2 +- extract_msg/structures/ole_pres.py | 431 +-------------- extract_msg/structures/toc_entry.py | 29 + extract_msg/utils.py | 4 +- extract_msg_tests/constants.py | 6 +- 12 files changed, 908 insertions(+), 438 deletions(-) create mode 100644 extract_msg/structures/cfoas.py create mode 100644 extract_msg/structures/dev_mode_a.py create mode 100644 extract_msg/structures/dv_target_device.py create mode 100644 extract_msg/structures/toc_entry.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a0265e7..0c43c7af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ * Changed `TZRule` to require the 14 null bytes (I commented it out completely on accident instead of swapping it to a plain read). It now logs a warning about the bytes not being null. * Removed unneeded function `windowsUnicode`. * Moved `FixedLengthProperty.parseType` to the private API. This was not intended for external use anyways, so leaving it as public API didn't make sense. +* Fixed check for type in `ContactAddressEntryID` being the wrong value. **v0.45.0** * BREAKING: Changed parsing of string multiple properties to remove the trailing null byte. This *will* cause the output of parsing them to differ. diff --git a/extract_msg/enums.py b/extract_msg/enums.py index a97426ed..9f0f8088 100644 --- a/extract_msg/enums.py +++ b/extract_msg/enums.py @@ -22,6 +22,7 @@ 'ContactAddressIndex', 'ContactLinkState', 'DeencapType', + 'DevModeFields', 'DirectoryEntryType', 'DisplayType', 'DMPaperSize', @@ -460,6 +461,29 @@ class DeencapType(enum.IntEnum): +class DevModeFields(enum.IntFlag): + DM_NUP = 0b00000000000000000000000000000010 + DM_SCALE = 0b00000000000000000000000000001000 + DM_PAPERWIDTH = 0b00000000000000000000000000010000 + DM_PAPERLENGTH = 0b00000000000000000000000000100000 + DM_PAPERSIZE = 0b00000000000000000000000001000000 + DM_ORIENTATION = 0b00000000000000000000000010000000 + DM_COLLATE = 0b00000000000000000000000100000000 + DM_TTOPTION = 0b00000000000000000000001000000000 + DM_YRESOLUTION = 0b00000000000000000000010000000000 + DM_DUPLEX = 0b00000000000000000000100000000000 + DM_COLOR = 0b00000000000000000001000000000000 + DM_PRINTQUALITY = 0b00000000000000000010000000000000 + DM_DEFAULTSOURCE = 0b00000000000000000100000000000000 + DM_COPIES = 0b00000000000000001000000000000000 + DM_ICMMETHOD = 0b00000000000000010000000000000000 + DM_FORMNAME = 0b00000000100000000000000000000000 + DM_DITHERTYPE = 0b00100000000000000000000000000000 + DM_MEDIATYPE = 0b01000000000000000000000000000000 + DM_ICMINTENT = 0b10000000000000000000000000000000 + + + class DirectoryEntryType(enum.IntEnum): UNALLOCATED = 0 UNKNOWN = 0 diff --git a/extract_msg/structures/__init__.py b/extract_msg/structures/__init__.py index 1cbd229d..c6d4ddeb 100644 --- a/extract_msg/structures/__init__.py +++ b/extract_msg/structures/__init__.py @@ -7,6 +7,7 @@ '_helpers', 'contact_link_entry', 'business_card', + 'dev_mode_a', 'entry_id', 'misc_id', 'odt', @@ -21,7 +22,7 @@ ] from . import ( - _helpers, contact_link_entry, business_card, entry_id, misc_id, odt, - ole_pres, ole_stream_struct, recurrence_pattern, report_tag, + _helpers, contact_link_entry, dev_mode_a, business_card, entry_id, misc_id, odt, + ole_pres, ole_stream_struct, recurrence_pattern, report_tag, system_time, time_zone_definition, time_zone_struct, tz_rule ) \ No newline at end of file diff --git a/extract_msg/structures/cfoas.py b/extract_msg/structures/cfoas.py new file mode 100644 index 00000000..ef46737b --- /dev/null +++ b/extract_msg/structures/cfoas.py @@ -0,0 +1,99 @@ +__all__ = [ + 'ClipboardFormatOrAnsiString', +] + + +from typing import Optional, Union + +from .. import constants +from ._helpers import BytesReader +from ..enums import ClipboardFormat + + +class ClipboardFormatOrAnsiString: + def __init__(self, reader : Union[bytes, BytesReader]): + if isinstance(reader, bytes): + reader = BytesReader(reader) + + self.__markerOrLength = reader.readUnsignedInt() + if self.__markerOrLength > 0xFFFFFFFD: + self.__ansiString = None + self.__clipboardFormat = ClipboardFormat(reader.readUnsignedInt()) + elif self.__markerOrLength > 0: + self.__ansiString = reader.read(self.__markerOrLength) + self.__clipboardFormat = None + else: + self.__ansiString = None + self.__clipboardFormat = None + + def toBytes(self) -> bytes: + ret = constants.st.ST_LE_UI32.pack(self.markerOrLength) + if self.markerOrLength > 0xFFFFFFFD: + ret += constants.st.ST_LE_UI32.pack(self.clipboardFormat) + elif self.markerOrLength > 0: + ret += self.__ansiString + return ret + + @property + def ansiString(self) -> Optional[bytes]: + """ + The null-terminated ANSI string, as bytes, of the name of a registered + clipboard format. Only set if markerOrLength is not 0x00000000, + 0xFFFFFFFE, or 0xFFFFFFFF. + + Setting this will modify the markerOrLength field automatically. + """ + return self.__ansiString + + @ansiString.setter + def _(self, val : bytes) -> None: + if not val: + raise ValueError('Cannot set :property ansiString: to None or empty bytes. Set :property markerOrLength: to a value ') + + self.__ansiString = val + + @property + def clipboardFormat(self) -> Optional[ClipboardFormat]: + """ + The clipboard format, if any. + + To set this, make sure that :property markerOrLength: is 0xFFFFFFFE or + 0xFFFFFFFF *before* setting. + """ + return self.__clipboardFormat + + @clipboardFormat.setter + def _(self, val : ClipboardFormat) -> None: + if not val: + raise ValueError('Cannot set clipboard format to None.') + if self.markerOrLength < 0xFFFFFFFE: + raise ValueError('Cannot set the clipboard format while the marker or length is not 0xFFFFFFFE or 0xFFFFFFFF') + self.__clipboardFormat = val + + @property + def markerOrLength(self) -> int: + """ + If set the 0x00000000, then neither the format property nor the + ansiString property will be set. If it is 0xFFFFFFFF or 0xFFFFFFFE, then + the clipboardFormat property will be set. Otherwise, the ansiString + property + will be set. + """ + return self.__markerOrLength + + @markerOrLength.setter + def _(self, val : int) -> None: + if val < 0: + raise ValueError('markerOrLength must be a positive integer.') + if val > 0xFFFFFFFF: + raise ValueError('markerOrLength must be a 4 byte unsigned integer.') + + if val == 0: + self.__ansiString = None + self.__clipboardFormat = None + elif val > 0xFFFFFFFD: + self.__ansiString = None + self.__clipboardFormat = ClipboardFormat.CF_BITMAP + else: + raise ValueError('Cannot set :property markerOrLength: to a length value. Set :property ansiString: instead.') + self.__markerOrLength = val \ No newline at end of file diff --git a/extract_msg/structures/contact_link_entry.py b/extract_msg/structures/contact_link_entry.py index 7898a973..cc4d9c14 100644 --- a/extract_msg/structures/contact_link_entry.py +++ b/extract_msg/structures/contact_link_entry.py @@ -6,18 +6,43 @@ from typing import List from ._helpers import BytesReader -from .entry_id import AddressBookEntryID +from ..constants import st +from .entry_id import EntryID class ContactLinkEntry: - entries : List[AddressBookEntryID] + entries : List[EntryID] def __init__(self, data : bytes): + # My experience with this data almost entirely doesn't match the + # documentation, so I'm just going to do what I see and not what I'm + # told. reader = BytesReader(data) count = reader.readUnsignedInt() + # Ignore this field. reader.read(4) - remaining = reader.read() self.entries = [] for _ in range(count): - idStruct = AddressBookEntryID(remaining) - remaining = remaining[idStruct.position:] \ No newline at end of file + size = reader.readUnsignedInt() + self.entries.append(EntryID.autoCreate(reader.read(size))) + if (size & 3) != 0: + reader.read(4 - (size & 3)) + + def toBytes(self) -> bytes: + ret = st.ST_LE_UI32.pack(len(self.entries)) + + # Need to handle the data before hand. + data = b'' + for entry in self.entries: + entryData = entry.toBytes() + # Size goes before data. + data += st.ST_LE_UI32.pack(edLen := len(entryData)) + data += entryData + # Handle padding. + if edLen & 3: + data += b'\x00' * (4 - edLen) + + ret += st.ST_LE_UI32.pack(len(data)) + ret += data + + return ret diff --git a/extract_msg/structures/dev_mode_a.py b/extract_msg/structures/dev_mode_a.py new file mode 100644 index 00000000..1ec8cf07 --- /dev/null +++ b/extract_msg/structures/dev_mode_a.py @@ -0,0 +1,548 @@ +import logging +import struct + +from typing import Final, Optional + +from ._helpers import BytesReader +from ..enums import DevModeFields + + +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) + + +class DevModeA: + """ + A DEVMODEA structure, as specified in [MS-OLEDS]. For the purposes of + parsing from bytes, if something goes wrong this will evaluate to False when + converting to bool. If no data is prodided, the fields are set to default + values. + """ + PARSE_STRUCT : Final[struct.Struct] = struct.Struct('<32s32s4HI13h14xI4x4I16x') + + def __init__(self, data : Optional[bytes] = None): + self.__valid = data is None + + # Set default values for fields that may not be initialized. + self.__orientation = 0 + self.__paperSize = 0 + self.__paperLength = 0 + self.__paperWidth = 0 + self.__scale = 0 + self.__copies = 0 + self.__defaultSource = 0 + self.__printQuality = 0 + self.__color = 0 + self.__duplex = 0 + self.__yResolution = 0 + self.__ttOption = 0 + self.__collate = 0 + self.__nup = 0 + self.__icmMethod = 0 + self.__icmIntent = 0 + self.__mediaType = 0 + self.__ditherType = 0 + + if self.__valid: + self.__deviceName = b'\x00' * 32 + self.__formName = b'\x00' * 32 + self.__specVersion = 0 + self.__driverVersion = 0 + self.__driverExtra = 0 + self.__fields = DevModeFields[0] + return + + reader = BytesReader(data) + + try: + items = reader.readStruct(self.PARSE_STRUCT) + self.__deviceName = items[0] + self.__formName = items[1] + self.__specVersion = items[2] + self.__driverVersion = items[3] + if items[4] != self.PARSE_STRUCT.size: + logger.warn(f'Unexpected `size` field for DevModeA detected ({items[4]})') + self.__driverExtra = items[5] + self.__fields = DevModeFields(items[6]) + # TODO fields specifies if we should read the field or ignore it. + if DevModeFields.DM_ORIENTATION in self.__fields: + self.__orientation = items[7] + if DevModeFields.DM_PAPERSIZE in self.__fields: + self.__paperSize = items[8] + if DevModeFields.DM_PAPERLENGTH in self.__fields: + self.__paperLength = items[9] + if DevModeFields.DM_PAPERWIDTH in self.__fields: + self.__paperWidth = items[10] + if DevModeFields.DM_SCALE in self.__fields: + self.__scale = items[11] + if DevModeFields.DM_COPIES in self.__fields: + self.__copies = items[12] + if DevModeFields.DM_DEFAULTSOURCE in self.__fields: + self.__defaultSource = items[13] + if DevModeFields.DM_PRINTQUALITY in self.__fields: + self.__printQuality = items[14] + if DevModeFields.DM_COLOR in self.__fields: + self.__color = items[15] + if DevModeFields.DM_DUPLEX in self.__fields: + self.__duplex = items[16] + if DevModeFields.DM_YRESOLUTION in self.__fields: + self.__yResolution = items[17] + if DevModeFields.DM_TTOPTION in self.__fields: + self.__ttOption = items[18] + if DevModeFields.DM_COLLATE in self.__fields: + self.__collate = items[19] + if DevModeFields.DM_NUP in self.__fields: + self.__nup = items[20] + if DevModeFields.DM_ICMMETHOD in self.__fields: + self.__icmMethod = items[21] + if DevModeFields.DM_ICMINTENT in self.__fields: + self.__icmIntent = items[22] + if DevModeFields.DM_MEDIATYPE in self.__fields: + self.__mediaType = items[23] + if DevModeFields.DM_DITHERTYPE in self.__fields: + self.__ditherType = items[24] + except IOError: + return + + self.__valid = True + + def __bool__(self) -> bool: + return self.__valid + + def toBytes(self) -> bytes: + return self.PARSE_STRUCT.pack( + self.__deviceName, + self.__formName, + self.__specVersion, + self.__driverVersion, + self.PARSE_STRUCT.size, + self.__driverExtra, + self.__fields, + self.__orientation, + self.__paperSize, + self.__paperLength, + self.__paperWidth, + self.__scale, + self.__copies, + self.__defaultSource, + self.__printQuality, + self.__color, + self.__duplex, + self.__yResolution, + self.__ttOption, + self.__collate, + self.__nup, + self.__icmMethod, + self.__icmIntent, + self.__mediaType, + self.__ditherType, + ) + + @property + def collate(self) -> int: + return self.__collate + + @collate.setter + def _(self, val : Optional[int]) -> None: + if val is None: + self.__collate = 0 + if DevModeFields.DM_COLLATE in self.__fields: + self.__fields ^= DevModeFields.DM_COLLATE + + if val < -32768: + raise ValueError('collate cannot be less than -32768.') + if val > 32767: + raise ValueError('collate cannot be greater than 32767.') + + self.__fields |= DevModeFields.DM_COLLATE + self.__collate = val + + @property + def color(self) -> int: + return self.__color + + @color.setter + def _(self, val : Optional[int]) -> None: + if val is None: + self.__color = 0 + if DevModeFields.DM_COLOR in self.__fields: + self.__fields ^= DevModeFields.DM_COLOR + + if val < -32768: + raise ValueError('color cannot be less than -32768.') + if val > 32767: + raise ValueError('color cannot be greater than 32767.') + + self.__fields |= DevModeFields.DM_COLOR + self.__color = val + + @property + def copies(self) -> int: + return self.__copies + + @copies.setter + def _(self, val : Optional[int]) -> None: + if val is None: + self.__copies = 0 + if DevModeFields.DM_COPIES in self.__fields: + self.__fields ^= DevModeFields.DM_COPIES + + if val < -32768: + raise ValueError('copies cannot be less than -32768.') + if val > 32767: + raise ValueError('copies cannot be greater than 32767.') + + self.__fields |= DevModeFields.DM_COPIES + self.__copies = val + + @property + def defaultSource(self) -> int: + return self.__defaultSource + + @defaultSource.setter + def _(self, val : Optional[int]) -> None: + if val is None: + self.__defaultSource = 0 + if DevModeFields.DM_DEFAULTSOURCE in self.__fields: + self.__fields ^= DevModeFields.DM_DEFAULTSOURCE + + if val < -32768: + raise ValueError('defaultSource cannot be less than -32768.') + if val > 32767: + raise ValueError('defaultSource cannot be greater than 32767.') + + self.__fields |= DevModeFields.DM_DEFAULTSOURCE + self.__defaultSource = val + + @property + def deviceName(self) -> bytes: + """ + A 32 byte ANSI string. + """ + return self.__deviceName + + @deviceName.setter + def _(self, val : bytes) -> None: + if len(val) != 32: + raise ValueError('deviceName must be exactly 32 bytes.') + + self.__deviceName = val + + @property + def ditherType(self) -> int: + return self.__ditherType + + @ditherType.setter + def _(self, val : Optional[int]) -> None: + if val is None: + self.__ditherType = 0 + if DevModeFields.DM_DITHERTYPE in self.__fields: + self.__fields ^= DevModeFields.DM_DITHERTYPE + + if val < 0: + raise ValueError('ditherType must be positive.') + if val > 4294967295: + raise ValueError('ditherType cannot be greater than 4294967295.') + + self.__fields |= DevModeFields.DM_DITHERTYPE + self.__ditherType = val + + @property + def driverExtra(self) -> int: + return self.__driverExtra + + @driverExtra.setter + def _(self, val : int) -> None: + if val < 0: + raise ValueError('driverExtra must be positive.') + if val > 65535: + raise ValueError('driverExtra cannot be greater than 65535.') + + self.__driverExtra = val + + @property + def driverVersion(self) -> int: + return self.__driverVersion + + @driverVersion.setter + def _(self, val : int) -> None: + if val < 0: + raise ValueError('driverVersion must be positive.') + if val > 65535: + raise ValueError('driverVersion cannot be greater than 65535.') + + self.__driverVersion = val + + @property + def duplex(self) -> int: + return self.__duplex + + @duplex.setter + def _(self, val : Optional[int]) -> None: + if val is None: + self.__duplex = 0 + if DevModeFields.DM_DUPLEX in self.__fields: + self.__fields ^= DevModeFields.DM_DUPLEX + + if val < -32768: + raise ValueError('duplex cannot be less than -32768.') + if val > 32767: + raise ValueError('duplex cannot be greater than 32767.') + + self.__fields |= DevModeFields.DM_DUPLEX + self.__duplex = val + + @property + def formName(self) -> bytes: + """ + A 32 byte ANSI string. + """ + return self.__formName + + @formName.setter + def _(self, val : bytes) -> None: + if len(val) != 32: + raise ValueError('formName must be exactly 32 bytes.') + + self.__formName = val + + @property + def icmIntent(self) -> int: + return self.__icmIntent + + @icmIntent.setter + def _(self, val : Optional[int]) -> None: + if val is None: + self.__icmIntent = 0 + if DevModeFields.DM_ICMINTENT in self.__fields: + self.__fields ^= DevModeFields.DM_ICMINTENT + + if val < 0: + raise ValueError('icmIntent must be positive.') + if val > 4294967295: + raise ValueError('icmIntent cannot be greater than 4294967295.') + + self.__fields |= DevModeFields.DM_ICMINTENT + self.__icmIntent = val + + @property + def icmMethod(self) -> int: + return self.__icmMethod + + @icmMethod.setter + def _(self, val : Optional[int]) -> None: + if val is None: + self.__icmMethod = 0 + if DevModeFields.DM_ICMMETHOD in self.__fields: + self.__fields ^= DevModeFields.DM_ICMMETHOD + + if val < 0: + raise ValueError('icmMethod must be positive.') + if val > 4294967295: + raise ValueError('icmMethod cannot be greater than 4294967295.') + + self.__fields |= DevModeFields.DM_ICMMETHOD + self.__icmMethod = val + + @property + def mediaType(self) -> int: + return self.__mediaType + + @mediaType.setter + def _(self, val : Optional[int]) -> None: + if val is None: + self.__mediaType = 0 + if DevModeFields.DM_MEDIATYPE in self.__fields: + self.__fields ^= DevModeFields.DM_MEDIATYPE + + if val < 0: + raise ValueError('mediaType must be positive.') + if val > 4294967295: + raise ValueError('mediaType cannot be greater than 4294967295.') + + self.__fields |= DevModeFields.DM_MEDIATYPE + self.__mediaType = val + + @property + def nup(self) -> int: + return self.__nup + + @nup.setter + def _(self, val : Optional[int]) -> None: + if val is None: + self.__nup = 0 + if DevModeFields.DM_NUP in self.__fields: + self.__fields ^= DevModeFields.DM_NUP + + if val < 0: + raise ValueError('nup must be positive.') + if val > 4294967295: + raise ValueError('nup cannot be greater than 4294967295.') + + self.__fields |= DevModeFields.DM_NUP + self.__nup = val + + @property + def orientation(self) -> int: + return self.__orientation + + @orientation.setter + def _(self, val : Optional[int]) -> None: + if val is None: + self.__orientation = 0 + if DevModeFields.DM_ORIENTATION in self.__fields: + self.__fields ^= DevModeFields.DM_ORIENTATION + + if val < -32768: + raise ValueError('orientation cannot be less than -32768.') + if val > 32767: + raise ValueError('orientation cannot be greater than 32767.') + + self.__fields |= DevModeFields.DM_ORIENTATION + self.__orientation = val + + @property + def paperLength(self) -> int: + return self.__paperLength + + @paperLength.setter + def _(self, val : Optional[int]) -> None: + if val is None: + self.__paperLength = 0 + if DevModeFields.DM_PAPERLENGTH in self.__fields: + self.__fields ^= DevModeFields.DM_PAPERLENGTH + + if val < -32768: + raise ValueError('paperLength cannot be less than -32768.') + if val > 32767: + raise ValueError('paperLength cannot be greater than 32767.') + + self.__fields |= DevModeFields.DM_PAPERLENGTH + self.__paperLength = val + + @property + def paperSize(self) -> int: + return self.__paperSize + + @paperSize.setter + def _(self, val : Optional[int]) -> None: + if val is None: + self.__paperSize = 0 + if DevModeFields.DM_PAPERSIZE in self.__fields: + self.__fields ^= DevModeFields.DM_PAPERSIZE + + if val < -32768: + raise ValueError('paperSize cannot be less than -32768.') + if val > 32767: + raise ValueError('paperSize cannot be greater than 32767.') + + self.__fields |= DevModeFields.DM_PAPERSIZE + self.__paperSize = val + + @property + def paperWidth(self) -> int: + return self.__paperWidth + + @paperWidth.setter + def _(self, val : Optional[int]) -> None: + if val is None: + self.__paperWidth = 0 + if DevModeFields.DM_PAPERWIDTH in self.__fields: + self.__fields ^= DevModeFields.DM_PAPERWIDTH + + if val < -32768: + raise ValueError('paperWidth cannot be less than -32768.') + if val > 32767: + raise ValueError('paperWidth cannot be greater than 32767.') + + self.__fields |= DevModeFields.DM_PAPERWIDTH + self.__paperWidth = val + + @property + def printQuality(self) -> int: + return self.__printQuality + + @printQuality.setter + def _(self, val : Optional[int]) -> None: + if val is None: + self.__printQuality = 0 + if DevModeFields.DM_PRINTQUALITY in self.__fields: + self.__fields ^= DevModeFields.DM_PRINTQUALITY + + if val < -32768: + raise ValueError('printQuality cannot be less than -32768.') + if val > 32767: + raise ValueError('printQuality cannot be greater than 32767.') + + self.__fields |= DevModeFields.DM_PRINTQUALITY + self.__printQuality = val + + @property + def scale(self) -> int: + return self.__scale + + @scale.setter + def _(self, val : Optional[int]) -> None: + if val is None: + self.__scale = 0 + if DevModeFields.DM_SCALE in self.__fields: + self.__fields ^= DevModeFields.DM_SCALE + + if val < -32768: + raise ValueError('scale cannot be less than -32768.') + if val > 32767: + raise ValueError('scale cannot be greater than 32767.') + + self.__fields |= DevModeFields.DM_SCALE + self.__scale = val + + @property + def specVersion(self) -> int: + return self.__specVersion + + @specVersion.setter + def _(self, val : int) -> None: + if val < 0: + raise ValueError('specVersion must be positive.') + if val > 65535: + raise ValueError('specVersion cannot be greater than 65535.') + + self.__specVersion = val + + @property + def ttOption(self) -> int: + return self.__ttOption + + @ttOption.setter + def _(self, val : Optional[int]) -> None: + if val is None: + self.__ttOption = 0 + if DevModeFields.DM_TTOPTION in self.__fields: + self.__fields ^= DevModeFields.DM_TTOPTION + + if val < -32768: + raise ValueError('ttOption cannot be less than -32768.') + if val > 32767: + raise ValueError('ttOption cannot be greater than 32767.') + + self.__fields |= DevModeFields.DM_TTOPTION + self.__ttOption = val + + @property + def yResolution(self) -> int: + return self.__yResolution + + @yResolution.setter + def _(self, val : Optional[int]) -> None: + if val is None: + self.__yResolution = 0 + if DevModeFields.DM_YRESOLUTION in self.__fields: + self.__fields ^= DevModeFields.DM_YRESOLUTION + + if val < -32768: + raise ValueError('yResolution cannot be less than -32768.') + if val > 32767: + raise ValueError('yResolution cannot be greater than 32767.') + + self.__fields |= DevModeFields.DM_YRESOLUTION + self.__yResolution = val \ No newline at end of file diff --git a/extract_msg/structures/dv_target_device.py b/extract_msg/structures/dv_target_device.py new file mode 100644 index 00000000..adbdf452 --- /dev/null +++ b/extract_msg/structures/dv_target_device.py @@ -0,0 +1,162 @@ +__all__ = [ + 'DVTargetDevice', +] + + +import struct + +from typing import Optional + +from ._helpers import BytesReader +from .dev_mode_a import DevModeA + + +class DVTargetDevice: + """ + Specifies information about a device that renders the presentation data. + + The creator of this data structure MUST NOT assume that it will be + understood during processing. + """ + + def __init__(self, data : Optional[bytes]): + self.__driverName = None + self.__deviceName = None + self.__portName = None + self.__extDevMode = None + + if not data: + return + reader = BytesReader(data) + + # We have 4 fields to read, and *technically* they may not all even be + # present, given that this structure can be 4 bytes? Reading all of + # these is also much more complicated than other structures, as they can + # technically overlap. We are just going to be *much* more lenient about + # this structure. + offset1 = offset2 = offset3 = offset4 = -1 + try: + offset1 = reader.readUnsignedShort() + offset2 = reader.readUnsignedShort() + offset3 = reader.readUnsignedShort() + offset4 = reader.readUnsignedShort() + except IOError: + pass + + if offset1 != -1 and offset1 < len(data): + reader.seek(offset1) + try: + self.__driverName = reader.readByteString() + except IOError: + self.__driverName = reader.read() + if not self.__driverName: + self.__driverName = None + + if offset2 != -1 and offset2 < len(data): + reader.seek(offset2) + try: + self.__deviceName = reader.readByteString() + except IOError: + self.__deviceName = reader.read() + if not self.__deviceName: + self.__deviceName = None + + if offset3 != -1 and offset3 < len(data): + reader.seek(offset3) + try: + self.__portName = reader.readByteString() + except IOError: + self.__portName = reader.read() + if not self.__portName: + self.__portName = None + + if offset4 != -1 and offset4 < len(data): + reader.seek(offset4) + try: + devmode = DevModeA(reader.read(56)) + if devmode: + self.__extDevMode = devmode + except IOError: + self.__extDevMode = None + + def toBytes(self) -> Optional[bytes]: + if not (self.driverName or self.deviceName or self.portName or self.extDevMode): + return None + currentPosition = 8 + + offset1 = 8 if self.__driverName else 0 + if self.__driverName: + currentPosition += len(self.__driverName) + 1 + + offset2 = currentPosition if self.__deviceName else 0 + if self.__deviceName: + currentPosition += len(self.__deviceName) + 1 + + offset3 = currentPosition if self.__portName else 0 + if self.__portName: + currentPosition += len(self.__portName) + 1 + + extDevModeBytes = self.__extDevMode.toBytes() if self.__extDevMode else None + offset4 = currentPosition if extDevModeBytes else 0 + + try: + ret = struct.pack(' Optional[bytes]: + """ + Optional ANSI string that contains a hunt on how to display or print + presentation data. + """ + return self.__driverName + + @driverName.setter + def _(self, data : Optional[bytes]) -> None: + self.__driverName = None if not data else data + + @property + def deviceName(self) -> Optional[bytes]: + """ + Optional ANSI string that contains a hunt on how to display or print + presentation data. + """ + return self.__deviceName + + @deviceName.setter + def _(self, data : Optional[bytes]) -> None: + self.__deviceName = None if not data else data + + @property + def portName(self) -> Optional[bytes]: + """ + Optional ANSI string that contains any arbitrary value. + """ + return self.__portName + + @portName.setter + def _(self, data : Optional[bytes]) -> None: + self.__portName = None if not data else data + + @property + def extDevMode(self) -> Optional[DevModeA]: + """ + Optional ANSI string that contains a hunt on how to display or print + presentation data. + """ + return self.__extDevMode + + @extDevMode.setter + def _(self, data : Optional[DevModeA]) -> None: + self.__extDevMode = None if not data else data \ No newline at end of file diff --git a/extract_msg/structures/entry_id.py b/extract_msg/structures/entry_id.py index 4c8c3247..e4ad6607 100644 --- a/extract_msg/structures/entry_id.py +++ b/extract_msg/structures/entry_id.py @@ -200,7 +200,7 @@ def __init__(self, data : bytes): reader = BytesReader(data[20:]) if (version := reader.readUnsignedInt()) != 3: raise ValueError(f'Version must be 3 (got {version}).') - if (type_ := reader.readUnsignedInt()) != 5: + if (type_ := reader.readUnsignedInt()) != 4: raise ValueError(f'Type must be 4 (got {type_}).') self.__index = ContactAddressIndex(reader.readUnsignedInt()) self.__entryIdCount = reader.readUnsignedInt() diff --git a/extract_msg/structures/ole_pres.py b/extract_msg/structures/ole_pres.py index e3e6f956..8471b28b 100644 --- a/extract_msg/structures/ole_pres.py +++ b/extract_msg/structures/ole_pres.py @@ -2,418 +2,17 @@ __all__ = [ - 'ClipboardFormatOrAnsiString', 'OLEPresentationStream', ] -import enum -import logging -import struct +from typing import List, Optional, Union -from typing import List, Final, Optional, Union - -from .. import constants from ._helpers import BytesReader +from .cfoas import ClipboardFormatOrAnsiString +from .dv_target_device import DVTargetDevice from ..enums import ADVF, ClipboardFormat, DVAspect - - -logger = logging.getLogger(__name__) -logger.addHandler(logging.NullHandler()) - - -class ClipboardFormatOrAnsiString: - def __init__(self, reader : Union[bytes, BytesReader]): - if isinstance(reader, bytes): - reader = BytesReader(reader) - - self.__markerOrLength = reader.readUnsignedInt() - if self.__markerOrLength > 0xFFFFFFFD: - self.__ansiString = None - self.__clipboardFormat = ClipboardFormat(reader.readUnsignedInt()) - elif self.__markerOrLength > 0: - self.__ansiString = reader.read(self.__markerOrLength) - self.__clipboardFormat = None - else: - self.__ansiString = None - self.__clipboardFormat = None - - def toBytes(self) -> bytes: - ret = constants.st.ST_LE_UI32.pack(self.markerOrLength) - if self.markerOrLength > 0xFFFFFFFD: - ret += constants.st.ST_LE_UI32.pack(self.clipboardFormat) - elif self.markerOrLength > 0: - ret += self.__ansiString - return ret - - @property - def ansiString(self) -> Optional[bytes]: - """ - The null-terminated ANSI string, as bytes, of the name of a registered - clipboard format. Only set if markerOrLength is not 0x00000000, - 0xFFFFFFFE, or 0xFFFFFFFF. - - Setting this will modify the markerOrLength field automatically. - """ - return self.__ansiString - - @ansiString.setter - def _(self, val : bytes) -> None: - if not val: - raise ValueError('Cannot set :property ansiString: to None or empty bytes. Set :property markerOrLength: to a value ') - - self.__ansiString = val - - @property - def clipboardFormat(self) -> Optional[ClipboardFormat]: - """ - The clipboard format, if any. - - To set this, make sure that :property markerOrLength: is 0xFFFFFFFE or - 0xFFFFFFFF *before* setting. - """ - return self.__clipboardFormat - - @clipboardFormat.setter - def _(self, val : ClipboardFormat) -> None: - if not val: - raise ValueError('Cannot set clipboard format to None.') - if self.markerOrLength < 0xFFFFFFFE: - raise ValueError('Cannot set the clipboard format while the marker or length is not 0xFFFFFFFE or 0xFFFFFFFF') - self.__clipboardFormat = val - - @property - def markerOrLength(self) -> int: - """ - If set the 0x00000000, then neither the format property nor the - ansiString property will be set. If it is 0xFFFFFFFF or 0xFFFFFFFE, then - the clipboardFormat property will be set. Otherwise, the ansiString - property - will be set. - """ - return self.__markerOrLength - - @markerOrLength.setter - def _(self, val : int) -> None: - if val < 0: - raise ValueError('markerOrLength must be a positive integer.') - if val > 0xFFFFFFFF: - raise ValueError('markerOrLength must be a 4 byte unsigned integer.') - - if val == 0: - self.__ansiString = None - self.__clipboardFormat = None - elif val > 0xFFFFFFFD: - self.__ansiString = None - self.__clipboardFormat = ClipboardFormat.CF_BITMAP - else: - raise ValueError('Cannot set :property markerOrLength: to a length value. Set :property ansiString: instead.') - self.__markerOrLength = val - - - -class DevModeA: - """ - A DEVMODEA structure, as specified in [MS-OLEDS]. For the purposes of - parsing from bytes, if something goes wrong this will evaluate to False when - converting to bool. If no data is prodided, the fields are set to default - values. - """ - PARSE_STRUCT : Final[struct.Struct] = struct.Struct('<32s32s4HI13h14xI4x4I16x') - - def __init__(self, data : Optional[bytes] = None): - self.__valid = data is None - - # Set default values for fields that may not be initialized. - self.__orientation = 0 - self.__paperSize = 0 - self.__paperLength = 0 - self.__paperWidth = 0 - self.__scale = 0 - self.__copies = 0 - self.__defaultSource = 0 - self.__printQuality = 0 - self.__color = 0 - self.__duplex = 0 - self.__yResolution = 0 - self.__ttOption = 0 - self.__collate = 0 - self.__nup = 0 - self.__icmMethod = 0 - self.__icmIntent = 0 - self.__mediaType = 0 - self.__ditherType = 0 - - if self.__valid: - self.__deviceName = b'\x00' * 32 - self.__formName = b'\x00' * 32 - self.__specVersion = 0 - self.__driverVersion = 0 - self.__driverExtra = 0 - self.__fields = _DevModeFields[0] - return - - reader = BytesReader(data) - - try: - items = reader.readStruct(self.PARSE_STRUCT) - self.__deviceName = items[0] - self.__formName = items[1] - self.__specVersion = items[2] - self.__driverVersion = items[3] - if items[4] != self.PARSE_STRUCT.size: - logger.warn(f'Unexpected `size` field for DevModeA detected ({items[4]})') - self.__driverExtra = items[5] - self.__fields = _DevModeFields(items[6]) - # TODO fields specifies if we should read the field or ignore it. - if _DevModeFields.DM_ORIENTATION in self.__fields: - self.__orientation = items[7] - if _DevModeFields.DM_PAPERSIZE in self.__fields: - self.__paperSize = items[8] - if _DevModeFields.DM_PAPERLENGTH in self.__fields: - self.__paperLength = items[9] - if _DevModeFields.DM_PAPERWIDTH in self.__fields: - self.__paperWidth = items[10] - if _DevModeFields.DM_SCALE in self.__fields: - self.__scale = items[11] - if _DevModeFields.DM_COPIES in self.__fields: - self.__copies = items[12] - if _DevModeFields.DM_DEFAULTSOURCE in self.__fields: - self.__defaultSource = items[13] - if _DevModeFields.DM_PRINTQUALITY in self.__fields: - self.__printQuality = items[14] - if _DevModeFields.DM_COLOR in self.__fields: - self.__color = items[15] - if _DevModeFields.DM_DUPLEX in self.__fields: - self.__duplex = items[16] - if _DevModeFields.DM_YRESOLUTION in self.__fields: - self.__yResolution = items[17] - if _DevModeFields.DM_TTOPTION in self.__fields: - self.__ttOption = items[18] - if _DevModeFields.DM_COLLATE in self.__fields: - self.__collate = items[19] - if _DevModeFields.DM_NUP in self.__fields: - self.__nup = items[20] - if _DevModeFields.DM_ICMMETHOD in self.__fields: - self.__icmMethod = items[21] - if _DevModeFields.DM_ICMINTENT in self.__fields: - self.__icmIntent = items[22] - if _DevModeFields.DM_MEDIATYPE in self.__fields: - self.__mediaType = items[23] - if _DevModeFields.DM_DITHERTYPE in self.__fields: - self.__ditherType = items[24] - except IOError: - return - - self.__valid = True - - def __bool__(self) -> bool: - return self.__valid - - def toBytes(self) -> bytes: - return self.PARSE_STRUCT.pack( - self.__deviceName, - self.__formName, - self.__specVersion, - self.__driverVersion, - self.PARSE_STRUCT.size, - self.__driverExtra, - self.__fields, - self.__orientation, - self.__paperSize, - self.__paperLength, - self.__paperWidth, - self.__scale, - self.__copies, - self.__defaultSource, - self.__printQuality, - self.__color, - self.__duplex, - self.__yResolution, - self.__ttOption, - self.__collate, - self.__nup, - self.__icmMethod, - self.__icmIntent, - self.__mediaType, - self.__ditherType, - ) - - - -class _DevModeFields(enum.IntFlag): - DM_NUP = 0b00000000000000000000000000000010 - DM_SCALE = 0b00000000000000000000000000001000 - DM_PAPERWIDTH = 0b00000000000000000000000000010000 - DM_PAPERLENGTH = 0b00000000000000000000000000100000 - DM_PAPERSIZE = 0b00000000000000000000000001000000 - DM_ORIENTATION = 0b00000000000000000000000010000000 - DM_COLLATE = 0b00000000000000000000000100000000 - DM_TTOPTION = 0b00000000000000000000001000000000 - DM_YRESOLUTION = 0b00000000000000000000010000000000 - DM_DUPLEX = 0b00000000000000000000100000000000 - DM_COLOR = 0b00000000000000000001000000000000 - DM_PRINTQUALITY = 0b00000000000000000010000000000000 - DM_DEFAULTSOURCE = 0b00000000000000000100000000000000 - DM_COPIES = 0b00000000000000001000000000000000 - DM_ICMMETHOD = 0b00000000000000010000000000000000 - DM_FORMNAME = 0b00000000100000000000000000000000 - DM_DITHERTYPE = 0b00100000000000000000000000000000 - DM_MEDIATYPE = 0b01000000000000000000000000000000 - DM_ICMINTENT = 0b10000000000000000000000000000000 - - - -class DVTargetDevice: - """ - Specifies information about a device that renders the presentation data. - - The creator of this data structure MUST NOT assume that it will be - understood during processing. - """ - - def __init__(self, data : Optional[bytes]): - self.__driverName = None - self.__deviceName = None - self.__portName = None - self.__extDevMode = None - - if not data: - return - reader = BytesReader(data) - - # We have 4 fields to read, and *technically* they may not all even be - # present, given that this structure can be 4 bytes? Reading all of - # these is also much more complicated than other structures, as they can - # technically overlap. We are just going to be *much* more lenient about - # this structure. - offset1 = offset2 = offset3 = offset4 = -1 - try: - offset1 = reader.readUnsignedShort() - offset2 = reader.readUnsignedShort() - offset3 = reader.readUnsignedShort() - offset4 = reader.readUnsignedShort() - except IOError: - pass - - if offset1 != -1 and offset1 < len(data): - reader.seek(offset1) - try: - self.__driverName = reader.readByteString() - except IOError: - self.__driverName = reader.read() - if not self.__driverName: - self.__driverName = None - - if offset2 != -1 and offset2 < len(data): - reader.seek(offset2) - try: - self.__deviceName = reader.readByteString() - except IOError: - self.__deviceName = reader.read() - if not self.__deviceName: - self.__deviceName = None - - if offset3 != -1 and offset3 < len(data): - reader.seek(offset3) - try: - self.__portName = reader.readByteString() - except IOError: - self.__portName = reader.read() - if not self.__portName: - self.__portName = None - - if offset4 != -1 and offset4 < len(data): - reader.seek(offset4) - try: - devmode = DevModeA(reader.read(56)) - if devmode: - self.__extDevMode = devmode - except IOError: - self.__extDevMode = None - - def toBytes(self) -> Optional[bytes]: - if not (self.driverName or self.deviceName or self.portName or self.extDevMode): - return None - currentPosition = 8 - - offset1 = 8 if self.__driverName else 0 - if self.__driverName: - currentPosition += len(self.__driverName) + 1 - - offset2 = currentPosition if self.__deviceName else 0 - if self.__deviceName: - currentPosition += len(self.__deviceName) + 1 - - offset3 = currentPosition if self.__portName else 0 - if self.__portName: - currentPosition += len(self.__portName) + 1 - - extDevModeBytes = self.__extDevMode.toBytes() if self.__extDevMode else None - offset4 = currentPosition if extDevModeBytes else 0 - - try: - ret = struct.pack(' Optional[bytes]: - """ - Optional ANSI string that contains a hunt on how to display or print - presentation data. - """ - return self.__driverName - - @driverName.setter - def _(self, data : Optional[bytes]) -> None: - self.__driverName = None if not data else data - - @property - def deviceName(self) -> Optional[bytes]: - """ - Optional ANSI string that contains a hunt on how to display or print - presentation data. - """ - return self.__deviceName - - @deviceName.setter - def _(self, data : Optional[bytes]) -> None: - self.__deviceName = None if not data else data - - @property - def portName(self) -> Optional[bytes]: - """ - Optional ANSI string that contains any arbitrary value. - """ - return self.__portName - - @portName.setter - def _(self, data : Optional[bytes]) -> None: - self.__portName = None if not data else data - - @property - def extDevMode(self) -> Optional[DevModeA]: - """ - Optional ANSI string that contains a hunt on how to display or print - presentation data. - """ - return self.__extDevMode - - @extDevMode.setter - def _(self, data : Optional[DevModeA]) -> None: - self.__extDevMode = None if not data else data - +from .toc_entry import TOCEntry class OLEPresentationStream: @@ -475,24 +74,4 @@ def __init__(self, data : bytes): self.tocEntries = [] if self.tocSignature == 0x494E414E: # b'NANI' in little endian. for _ in range(reader.readUnsignedInt()): - self.tocEntries.append(TOCEntry(reader)) - - - -class TOCEntry: - def __init__(self, reader : Union[bytes, BytesReader]): - if isinstance(reader, bytes): - reader = BytesReader(reader) - self.__clipFormat = ClipboardFormatOrAnsiString(reader) - targetDeviceSize = reader.readUnsignedInt() - self.__aspect = reader.readUnsignedInt() - self.__lindex = reader.readUnsignedInt() - self.__tymed = reader.readUnsignedInt() - reader.read(4) - self.__advf = reader.readUnsignedInt() - reader.read(4) - if targetDeviceSize == 0: - self.__targetDevice = None - else: - self.__targetDevice = DVTargetDevice(reader.read(targetDeviceSize)) - # TODO \ No newline at end of file + self.tocEntries.append(TOCEntry(reader)) \ No newline at end of file diff --git a/extract_msg/structures/toc_entry.py b/extract_msg/structures/toc_entry.py new file mode 100644 index 00000000..0c4fc45d --- /dev/null +++ b/extract_msg/structures/toc_entry.py @@ -0,0 +1,29 @@ +__all__ = [ + 'TOCEntry', +] + + +from typing import Optional, Union + +from ._helpers import BytesReader +from .cfoas import ClipboardFormatOrAnsiString +from .dv_target_device import DVTargetDevice + + +class TOCEntry: + def __init__(self, reader : Union[bytes, BytesReader]): + if isinstance(reader, bytes): + reader = BytesReader(reader) + self.__clipFormat = ClipboardFormatOrAnsiString(reader) + targetDeviceSize = reader.readUnsignedInt() + self.__aspect = reader.readUnsignedInt() + self.__lindex = reader.readUnsignedInt() + self.__tymed = reader.readUnsignedInt() + reader.read(4) + self.__advf = reader.readUnsignedInt() + reader.read(4) + if targetDeviceSize == 0: + self.__targetDevice = None + else: + self.__targetDevice = DVTargetDevice(reader.read(targetDeviceSize)) + # TODO \ No newline at end of file diff --git a/extract_msg/utils.py b/extract_msg/utils.py index c1858e7d..5f73e041 100644 --- a/extract_msg/utils.py +++ b/extract_msg/utils.py @@ -192,9 +192,9 @@ def createZipOpen(func) -> Callable: Creates a wrapper for the open function of a ZipFile that will automatically set the current date as the modified time to the current time. """ - def _open(name, mode, *args, **kwargs): + def _open(name, mode = 'r', *args, **kwargs): if mode == 'w': - name = zipfile.ZipInfo(name, datetime.datetime.now().timetuple()) + name = zipfile.ZipInfo(name, datetime.datetime.now().timetuple()[:6]) return func(name, mode, *args, **kwargs) diff --git a/extract_msg_tests/constants.py b/extract_msg_tests/constants.py index 53063fa3..c3a4c412 100644 --- a/extract_msg_tests/constants.py +++ b/extract_msg_tests/constants.py @@ -10,8 +10,10 @@ TEST_FILE_DIR = Path(__file__).parent.parent / 'example-msg-files' -USER_TEST_DIR = None +_utd = None if bool(userTestDir := os.environ.get('EXTRACT_MSG_TEST_DIR')): userTestDir = Path(userTestDir) if userTestDir.exists(): - USER_TEST_DIR = userTestDir # type: ignore + _utd = userTestDir + +USER_TEST_DIR = _utd From cf2e51c679686baac70b8069a8a931cf91f8537a Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 4 Oct 2023 08:19:32 -0700 Subject: [PATCH 51/68] More work on olepres --- extract_msg/structures/ole_pres.py | 275 ++++++++++++++++++++++++---- extract_msg/structures/toc_entry.py | 8 +- 2 files changed, 247 insertions(+), 36 deletions(-) diff --git a/extract_msg/structures/ole_pres.py b/extract_msg/structures/ole_pres.py index 8471b28b..f435fe00 100644 --- a/extract_msg/structures/ole_pres.py +++ b/extract_msg/structures/ole_pres.py @@ -10,6 +10,7 @@ from ._helpers import BytesReader from .cfoas import ClipboardFormatOrAnsiString +from ..constants import st from .dv_target_device import DVTargetDevice from ..enums import ADVF, ClipboardFormat, DVAspect from .toc_entry import TOCEntry @@ -19,59 +20,263 @@ class OLEPresentationStream: """ [MS-OLEDS] OLEPresentationStream. """ - ansiClipboardFormat : ClipboardFormatOrAnsiString - targetDeviceSize : int - targetDevice : Optional[DVTargetDevice] - aspect : Union[int, DVAspect] - lindex : int - advf : Union[int, ADVF] - width : int - height : int - data : bytes - reserved2 : Optional[bytes] - tocSignature : int - tocEntries : List[TOCEntry] + __ansiClipboardFormat : ClipboardFormatOrAnsiString + __targetDevice : Optional[DVTargetDevice] + __aspect : Union[int, DVAspect] + __lindex : int + __advf : Union[int, ADVF] + __width : int + __height : int + __data : bytes + __reserved2 : Optional[bytes] + __tocSignature : int + __tocEntries : List[TOCEntry] def __init__(self, data : bytes): reader = BytesReader(data) - self.ansiClipboardFormat = ClipboardFormatOrAnsiString(reader) + acf = self.__ansiClipboardFormat = ClipboardFormatOrAnsiString(reader) # Validate the structure based on the documentation. - if self.ansiClipboardFormat.markerOrLength == 0: + if acf.markerOrLength == 0: raise ValueError('Invalid OLEPresentationStream (MarkerOrLength is 0).') - if self.ansiClipboardFormat.clipboardFormat is ClipboardFormat.CF_BITMAP: + if acf.clipboardFormat is ClipboardFormat.CF_BITMAP: raise ValueError('Invalid OLEPresentationStream (Format is CF_BITMAP).') - if 0x201 < self.ansiClipboardFormat.markerOrLength < 0xFFFFFFFE: + if 0x201 < acf.markerOrLength < 0xFFFFFFFE: raise ValueError('Invalid OLEPresentationStream (ANSI length was more than 0x201).') - self.targetDeviceSize = reader.readUnsignedInt() - if self.targetDeviceSize < 0x4: + targetDeviceSize = reader.readUnsignedInt() + if targetDeviceSize < 0x4: raise ValueError('Invalid OLEPresentationStream (TargetDeviceSize was less than 4).') - if self.targetDeviceSize > 0x4: + if targetDeviceSize > 0x4: # Read the TargetDevice field. - self.targetDevice = DVTargetDevice(reader.read(self.targetDeviceSize)) + self.__targetDevice = DVTargetDevice(reader.read(targetDeviceSize)) else: - self.targetDevice = None + self.__targetDevice = None - self.aspect = reader.readUnsignedInt() - self.lindex = reader.readUnsignedInt() - self.advf = reader.readUnsignedInt() + self.__aspect = reader.readUnsignedInt() + self.__lindex = reader.readUnsignedInt() + self.__advf = reader.readUnsignedInt() # Reserved1. - reader.read(4) + self.__reserved1 = reader.read(4) - self.width = reader.readUnsignedInt() - self.height = reader.readUnsignedInt() + self.__width = reader.readUnsignedInt() + self.__height = reader.readUnsignedInt() size = reader.readUnsignedInt() - self.data = reader.read(size) + self.__data = reader.read(size) - if self.ansiClipboardFormat.clipboardFormat is ClipboardFormat.CF_METAFILEPICT: - self.reserved2 = reader.read(18) + if acf.clipboardFormat is ClipboardFormat.CF_METAFILEPICT: + self.__reserved2 = reader.read(18) else: - self.reserved2 = None + self.__reserved2 = None - self.tocSignature = reader.readUnsignedInt() - self.tocEntries = [] - if self.tocSignature == 0x494E414E: # b'NANI' in little endian. + self.__tocSignature = reader.readUnsignedInt() + self.__tocEntries = [] + if self.__tocSignature == 0x494E414E: # b'NANI' in little endian. for _ in range(reader.readUnsignedInt()): - self.tocEntries.append(TOCEntry(reader)) \ No newline at end of file + self.__tocEntries.append(TOCEntry(reader)) + + + def toBytes(self) -> bytes: + ret = self.__ansiClipboardFormat.toBytes() + + if self.__targetDevice is None: + ret += b'\x04\x00\x00\x00' + else: + dvData = self.__targetDevice.toBytes() + ret += st.ST_LE_UI32.pack(len(dvData)) + ret += dvData + + ret += st.ST_LE_UI32.pack(self.__aspect) + ret += st.ST_LE_UI32.pack(self.__lindex) + ret += st.ST_LE_UI32.pack(self.__advf) + ret += self.__reserved1 + ret += st.ST_LE_UI32.pack(self.__width) + ret += st.ST_LE_UI32.pack(self.__height) + ret += st.ST_LE_UI32.pack(len(self.__data)) + self.__data + if self.__reserved2: # Shortcut since this property has protection. + ret += self.__reserved2 + + if self.__tocSignature == 0x494E414E: + ret += st.ST_LE_UI32.pack(len(self.__tocEntries)) + for entry in self.__tocEntries: + ret += entry.toBytes() + else: + ret += b'\x00\x00\x00\x00' + + return ret + + @property + def advf(self) -> Union[int, ADVF]: + """ + + """ + return self.__advf + + @advf.setter + def _(self, val : Union[int, ADVF]) -> None: + if not isinstance(val, int): + raise TypeError('advf must be an int.') + if val < 0: + raise ValueError('advf must be positive.') + if val > 4294967295: + raise ValueError('advf cannot be greater than 4294967295.') + + self.__advf = val + + @property + def ansiClipboardFormat(self) -> ClipboardFormatOrAnsiString: + """ + + """ + return self.__ansiClipboardFormat + + @property + def aspect(self) -> Union[int, DVAspect]: + """ + + """ + return self.__aspect + + @aspect.setter + def _(self, val : Union[int, DVAspect]) -> None: + if not isinstance(val, int): + raise TypeError(':property aspect: must be an int.') + if val < 0: + raise ValueError(':property aspect: must be positive.') + if val > 4294967295: + raise ValueError(':property aspect: cannot be greater than 4294967295.') + + self.__aspect = val + + @property + def data(self) -> bytes: + """ + + """ + return self.__data + + @data.setter + def _(self, val : bytes) -> None: + if not isinstance(val, bytes): + raise TypeError(':property data: must be bytes.') + self.__data = val + + @property + def height(self) -> int: + """ + + """ + return self.__height + + @height.setter + def _(self, val : int) -> None: + if val < 0: + raise ValueError(':property height: must be positive.') + if val > 4294967295: + raise ValueError(':property height: cannot be greater than 4294967295.') + + self.__height = val + + @property + def lindex(self) -> int: + """ + + """ + return self.__lindex + + @lindex.setter + def _(self, val : int) -> None: + if val < 0: + raise ValueError(':property lindex: must be positive.') + if val > 4294967295: + raise ValueError(':property lindex: cannot be greater than 4294967295.') + + self.__lindex = val + + @property + def reserved1(self) -> bytes: + """ + + """ + return self.__reserved1 + + @reserved1.setter + def _(self, val : bytes) -> None: + if not isinstance(val, bytes): + raise TypeError(':property reserved1: must by bytes.') + if len(val) != 4: + raise ValueError(':property reserved1: must be exactly 4 bytes.') + + self.__reserved1 = val + + @property + def reserved2(self) -> Optional[bytes]: + """ + + """ + return self.__reserved2 + + @reserved2.setter + def _(self, val : bytes) -> None: + if not isinstance(val, bytes): + raise TypeError(':property reserved2: must by bytes.') + if len(val) != 18: + raise ValueError(':property reserved2: must be exactly 18 bytes.') + + self.__reserved2 = val + + @property + def targetDevice(self) -> Optional[DVTargetDevice]: + """ + + """ + return self.__targetDevice + + @targetDevice.setter + def _(self, val : Optional[DVTargetDevice]) -> None: + if val is not None and not isinstance(val, DVTargetDevice): + raise TypeError(':property targetDevice: must be None or a DVTargetDevice.') + + self.__targetDevice = val + + @property + def tocEntries(self) -> List[TOCEntry]: + """ + + """ + return self.__tocEntries + + @property + def tocSignature(self) -> int: + """ + + """ + return self.__tocSignature + + @tocSignature.setter + def _(self, val : int) -> None: + if val < 0: + raise ValueError('tocSignature must be positive.') + if val > 4294967295: + raise ValueError('tocSignature cannot be greater than 4294967295.') + + self.__tocSignature = val + + @property + def width(self) -> int: + """ + + """ + return self.__width + + @width.setter + def _(self, val : int) -> None: + if val < 0: + raise ValueError('width must be positive.') + if val > 4294967295: + raise ValueError('width cannot be greater than 4294967295.') + + self.__width = val + + diff --git a/extract_msg/structures/toc_entry.py b/extract_msg/structures/toc_entry.py index 0c4fc45d..632a6f6d 100644 --- a/extract_msg/structures/toc_entry.py +++ b/extract_msg/structures/toc_entry.py @@ -26,4 +26,10 @@ def __init__(self, reader : Union[bytes, BytesReader]): self.__targetDevice = None else: self.__targetDevice = DVTargetDevice(reader.read(targetDeviceSize)) - # TODO \ No newline at end of file + # TODO + + def toBytes(self) -> bytes: + ret = self.__clipFormat.toBytes() + # TODO + + return ret \ No newline at end of file From 18fa73cf85a0a474ab6c3c6428ca75b91859de93 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 4 Oct 2023 10:14:48 -0700 Subject: [PATCH 52/68] Significant progress on olepres --- extract_msg/structures/ole_pres.py | 73 +++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 22 deletions(-) diff --git a/extract_msg/structures/ole_pres.py b/extract_msg/structures/ole_pres.py index f435fe00..c87c612a 100644 --- a/extract_msg/structures/ole_pres.py +++ b/extract_msg/structures/ole_pres.py @@ -68,7 +68,7 @@ def __init__(self, data : bytes): if acf.clipboardFormat is ClipboardFormat.CF_METAFILEPICT: self.__reserved2 = reader.read(18) else: - self.__reserved2 = None + self.__reserved2 = b'\x00' * 18. self.__tocSignature = reader.readUnsignedInt() self.__tocEntries = [] @@ -94,7 +94,7 @@ def toBytes(self) -> bytes: ret += st.ST_LE_UI32.pack(self.__width) ret += st.ST_LE_UI32.pack(self.__height) ret += st.ST_LE_UI32.pack(len(self.__data)) + self.__data - if self.__reserved2: # Shortcut since this property has protection. + if self.reserved2: # Shortcut since this property has protection. ret += self.__reserved2 if self.__tocSignature == 0x494E414E: @@ -109,32 +109,31 @@ def toBytes(self) -> bytes: @property def advf(self) -> Union[int, ADVF]: """ - + An implementation specific hint on how to render the presentation data + on screen. May be ignored on processing. """ return self.__advf @advf.setter def _(self, val : Union[int, ADVF]) -> None: if not isinstance(val, int): - raise TypeError('advf must be an int.') + raise TypeError(':property advf: must be an int.') if val < 0: - raise ValueError('advf must be positive.') + raise ValueError(':property advf: must be positive.') if val > 4294967295: - raise ValueError('advf cannot be greater than 4294967295.') + raise ValueError(':property advf: cannot be greater than 4294967295.') self.__advf = val @property def ansiClipboardFormat(self) -> ClipboardFormatOrAnsiString: - """ - - """ return self.__ansiClipboardFormat @property def aspect(self) -> Union[int, DVAspect]: """ - + An implementation specific hint on how to render the presentation data + on screen. May be ignored on processing. """ return self.__aspect @@ -152,7 +151,7 @@ def _(self, val : Union[int, DVAspect]) -> None: @property def data(self) -> bytes: """ - + The presentation data. The form of this data depends on :property clipboardFormat: of :property ansiClipboardFormat:. """ return self.__data @@ -165,12 +164,14 @@ def _(self, val : bytes) -> None: @property def height(self) -> int: """ - + The height, in pixels, of the presentation data. """ return self.__height @height.setter def _(self, val : int) -> None: + if not isinstance(val, int): + raise TypeError(':property height: must be an int.') if val < 0: raise ValueError(':property height: must be positive.') if val > 4294967295: @@ -181,12 +182,14 @@ def _(self, val : int) -> None: @property def lindex(self) -> int: """ - + An implementation specific hint on how to render the presentation data on screen. May be ignored on processing. """ return self.__lindex @lindex.setter def _(self, val : int) -> None: + if not isinstance(val, int): + raise TypeError(':property lindex: must be an int.') if val < 0: raise ValueError(':property lindex: must be positive.') if val > 4294967295: @@ -197,7 +200,8 @@ def _(self, val : int) -> None: @property def reserved1(self) -> bytes: """ - + 4 bytes that can contain any arbitrary data. Must be *exactly* 4 bytes + when setting. """ return self.__reserved1 @@ -213,12 +217,22 @@ def _(self, val : bytes) -> None: @property def reserved2(self) -> Optional[bytes]: """ + Optional additional data that is only set if the clipboard format of + :property ansiClipboardFormat: is CF_METAFILEPICT. + + Getting this will automatically correct the value retrieved based on + the clipboard format, but will *not* modify the underlying data. + Must be *exactly* 18 bytes when setting. """ + if self.__ansiClipboardFormat.clipboardFormat is not ClipboardFormat.CF_METAFILEPICT: + return None return self.__reserved2 @reserved2.setter def _(self, val : bytes) -> None: + if self.__ansiClipboardFormat.clipboardFormat is not ClipboardFormat.CF_METAFILEPICT: + raise ValueError(':property reserved2: cannot be set if the clipboard format (from :property ansiClipboardFormat:) is not CF_METAFILEPICT.') if not isinstance(val, bytes): raise TypeError(':property reserved2: must by bytes.') if len(val) != 18: @@ -228,9 +242,6 @@ def _(self, val : bytes) -> None: @property def targetDevice(self) -> Optional[DVTargetDevice]: - """ - - """ return self.__targetDevice @targetDevice.setter @@ -243,39 +254,57 @@ def _(self, val : Optional[DVTargetDevice]) -> None: @property def tocEntries(self) -> List[TOCEntry]: """ + A list of TOCEntry structures. If :property tocSignature: is not set to + 0x494E414E, accessing this value will clear the list. + :returns: A direct reference to the list, allowing for modification. This class WILL NOT change this reference over the lifetime of the object. """ + if self.__tocSignature != 0x494E414E: + self.__tocEntries.clear() return self.__tocEntries @property def tocSignature(self) -> int: """ + If this field does not contain 0x494E414E, then :property tocEntries: + MUST be empty. Modifications to the list will be lost when it is next + retrieved, meaning changes while this property is not 0x494E414E WILL be + lost. + Setting this to a value other than 0x494E414E will clear the list + immediately. """ return self.__tocSignature @tocSignature.setter def _(self, val : int) -> None: + if not isinstance(val, int): + raise TypeError(':property tocSignature: must be an int.') if val < 0: - raise ValueError('tocSignature must be positive.') + raise ValueError(':property tocSignature: must be positive.') if val > 4294967295: - raise ValueError('tocSignature cannot be greater than 4294967295.') + raise ValueError(':property tocSignature: cannot be greater than 4294967295.') + + if val != 0x494E414E: + self.__tocEntries.clear() self.__tocSignature = val @property def width(self) -> int: """ - + The width, in pixels, of the presentation data. """ return self.__width @width.setter def _(self, val : int) -> None: + if not isinstance(val, int): + raise TypeError(':property width: must be an int.') if val < 0: - raise ValueError('width must be positive.') + raise ValueError(':property width: must be positive.') if val > 4294967295: - raise ValueError('width cannot be greater than 4294967295.') + raise ValueError(':property width: cannot be greater than 4294967295.') self.__width = val From 22e57a614f04f5c05e466da280c68d996a23a7c3 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 4 Oct 2023 16:23:18 -0700 Subject: [PATCH 53/68] More work on OlePres --- extract_msg/structures/cfoas.py | 6 +++--- extract_msg/structures/dev_mode_a.py | 22 +++++++++++++--------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/extract_msg/structures/cfoas.py b/extract_msg/structures/cfoas.py index ef46737b..bfaec488 100644 --- a/extract_msg/structures/cfoas.py +++ b/extract_msg/structures/cfoas.py @@ -48,7 +48,7 @@ def ansiString(self) -> Optional[bytes]: @ansiString.setter def _(self, val : bytes) -> None: if not val: - raise ValueError('Cannot set :property ansiString: to None or empty bytes. Set :property markerOrLength: to a value ') + raise ValueError('Cannot set :property ansiString: to None or empty bytes.') self.__ansiString = val @@ -84,9 +84,9 @@ def markerOrLength(self) -> int: @markerOrLength.setter def _(self, val : int) -> None: if val < 0: - raise ValueError('markerOrLength must be a positive integer.') + raise ValueError(':property markerOrLength: must be a positive integer.') if val > 0xFFFFFFFF: - raise ValueError('markerOrLength must be a 4 byte unsigned integer.') + raise ValueError(':property markerOrLength: must be a 4 byte unsigned integer.') if val == 0: self.__ansiString = None diff --git a/extract_msg/structures/dev_mode_a.py b/extract_msg/structures/dev_mode_a.py index 1ec8cf07..b6ddc3ae 100644 --- a/extract_msg/structures/dev_mode_a.py +++ b/extract_msg/structures/dev_mode_a.py @@ -150,9 +150,9 @@ def _(self, val : Optional[int]) -> None: self.__fields ^= DevModeFields.DM_COLLATE if val < -32768: - raise ValueError('collate cannot be less than -32768.') + raise ValueError(':property collate: cannot be less than -32768.') if val > 32767: - raise ValueError('collate cannot be greater than 32767.') + raise ValueError(':property collate: cannot be greater than 32767.') self.__fields |= DevModeFields.DM_COLLATE self.__collate = val @@ -169,9 +169,9 @@ def _(self, val : Optional[int]) -> None: self.__fields ^= DevModeFields.DM_COLOR if val < -32768: - raise ValueError('color cannot be less than -32768.') + raise ValueError(':property color: cannot be less than -32768.') if val > 32767: - raise ValueError('color cannot be greater than 32767.') + raise ValueError(':property color: cannot be greater than 32767.') self.__fields |= DevModeFields.DM_COLOR self.__color = val @@ -188,9 +188,9 @@ def _(self, val : Optional[int]) -> None: self.__fields ^= DevModeFields.DM_COPIES if val < -32768: - raise ValueError('copies cannot be less than -32768.') + raise ValueError(':property copies: cannot be less than -32768.') if val > 32767: - raise ValueError('copies cannot be greater than 32767.') + raise ValueError(':property copies: cannot be greater than 32767.') self.__fields |= DevModeFields.DM_COPIES self.__copies = val @@ -207,9 +207,9 @@ def _(self, val : Optional[int]) -> None: self.__fields ^= DevModeFields.DM_DEFAULTSOURCE if val < -32768: - raise ValueError('defaultSource cannot be less than -32768.') + raise ValueError(':property defaultSource: cannot be less than -32768.') if val > 32767: - raise ValueError('defaultSource cannot be greater than 32767.') + raise ValueError(':property defaultSource: cannot be greater than 32767.') self.__fields |= DevModeFields.DM_DEFAULTSOURCE self.__defaultSource = val @@ -223,8 +223,10 @@ def deviceName(self) -> bytes: @deviceName.setter def _(self, val : bytes) -> None: + if not isinstance(val, bytes): + raise TypeError(':property deviceName: must be bytes.') if len(val) != 32: - raise ValueError('deviceName must be exactly 32 bytes.') + raise ValueError(':property deviceName: must be exactly 32 bytes.') self.__deviceName = val @@ -238,6 +240,8 @@ def _(self, val : Optional[int]) -> None: self.__ditherType = 0 if DevModeFields.DM_DITHERTYPE in self.__fields: self.__fields ^= DevModeFields.DM_DITHERTYPE + elif not isinstance(val, int): + raise TypeError(':property ditherType: must be an int or None.') if val < 0: raise ValueError('ditherType must be positive.') From 450f51013b992a4877c7e8b716e5ce2c5aa734e9 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 4 Oct 2023 16:49:16 -0700 Subject: [PATCH 54/68] Consistency and more type checking for devmodea --- extract_msg/structures/dev_mode_a.py | 130 +++++++++++++++++++-------- 1 file changed, 95 insertions(+), 35 deletions(-) diff --git a/extract_msg/structures/dev_mode_a.py b/extract_msg/structures/dev_mode_a.py index b6ddc3ae..5d1d898a 100644 --- a/extract_msg/structures/dev_mode_a.py +++ b/extract_msg/structures/dev_mode_a.py @@ -148,6 +148,9 @@ def _(self, val : Optional[int]) -> None: self.__collate = 0 if DevModeFields.DM_COLLATE in self.__fields: self.__fields ^= DevModeFields.DM_COLLATE + return + elif not isinstance(val, int): + raise TypeError(':property collate: must be an int or None.') if val < -32768: raise ValueError(':property collate: cannot be less than -32768.') @@ -167,6 +170,9 @@ def _(self, val : Optional[int]) -> None: self.__color = 0 if DevModeFields.DM_COLOR in self.__fields: self.__fields ^= DevModeFields.DM_COLOR + return + elif not isinstance(val, int): + raise TypeError(':property color: must be an int or None.') if val < -32768: raise ValueError(':property color: cannot be less than -32768.') @@ -186,6 +192,9 @@ def _(self, val : Optional[int]) -> None: self.__copies = 0 if DevModeFields.DM_COPIES in self.__fields: self.__fields ^= DevModeFields.DM_COPIES + return + elif not isinstance(val, int): + raise TypeError(':property copies: must be an int or None.') if val < -32768: raise ValueError(':property copies: cannot be less than -32768.') @@ -205,6 +214,9 @@ def _(self, val : Optional[int]) -> None: self.__defaultSource = 0 if DevModeFields.DM_DEFAULTSOURCE in self.__fields: self.__fields ^= DevModeFields.DM_DEFAULTSOURCE + return + elif not isinstance(val, int): + raise TypeError(':property defaultSource: must be an int or None.') if val < -32768: raise ValueError(':property defaultSource: cannot be less than -32768.') @@ -240,13 +252,14 @@ def _(self, val : Optional[int]) -> None: self.__ditherType = 0 if DevModeFields.DM_DITHERTYPE in self.__fields: self.__fields ^= DevModeFields.DM_DITHERTYPE + return elif not isinstance(val, int): raise TypeError(':property ditherType: must be an int or None.') if val < 0: - raise ValueError('ditherType must be positive.') + raise ValueError(':property ditherType: must be positive.') if val > 4294967295: - raise ValueError('ditherType cannot be greater than 4294967295.') + raise ValueError(':property ditherType: cannot be greater than 4294967295.') self.__fields |= DevModeFields.DM_DITHERTYPE self.__ditherType = val @@ -257,10 +270,12 @@ def driverExtra(self) -> int: @driverExtra.setter def _(self, val : int) -> None: + if not isinstance(val, int): + raise TypeError(':property driverExtra: must be an int.') if val < 0: - raise ValueError('driverExtra must be positive.') + raise ValueError(':property driverExtra: must be positive.') if val > 65535: - raise ValueError('driverExtra cannot be greater than 65535.') + raise ValueError(':property driverExtra: cannot be greater than 65535.') self.__driverExtra = val @@ -270,10 +285,12 @@ def driverVersion(self) -> int: @driverVersion.setter def _(self, val : int) -> None: + if not isinstance(val, int): + raise TypeError(':property driverVersion: must be an int or None.') if val < 0: - raise ValueError('driverVersion must be positive.') + raise ValueError(':property driverVersion: must be positive.') if val > 65535: - raise ValueError('driverVersion cannot be greater than 65535.') + raise ValueError(':property driverVersion: cannot be greater than 65535.') self.__driverVersion = val @@ -287,11 +304,14 @@ def _(self, val : Optional[int]) -> None: self.__duplex = 0 if DevModeFields.DM_DUPLEX in self.__fields: self.__fields ^= DevModeFields.DM_DUPLEX + return + elif not isinstance(val, int): + raise TypeError(':property duplex: must be an int or None.') if val < -32768: - raise ValueError('duplex cannot be less than -32768.') + raise ValueError(':property duplex: cannot be less than -32768.') if val > 32767: - raise ValueError('duplex cannot be greater than 32767.') + raise ValueError(':property duplex: cannot be greater than 32767.') self.__fields |= DevModeFields.DM_DUPLEX self.__duplex = val @@ -305,8 +325,10 @@ def formName(self) -> bytes: @formName.setter def _(self, val : bytes) -> None: + if not isinstance(val, bytes): + raise TypeError(':property formName: must be bytes.') if len(val) != 32: - raise ValueError('formName must be exactly 32 bytes.') + raise ValueError(':property formName: must be exactly 32 bytes.') self.__formName = val @@ -320,11 +342,14 @@ def _(self, val : Optional[int]) -> None: self.__icmIntent = 0 if DevModeFields.DM_ICMINTENT in self.__fields: self.__fields ^= DevModeFields.DM_ICMINTENT + return + elif not isinstance(val, int): + raise TypeError(':property icmIntent: must be an int or None.') if val < 0: - raise ValueError('icmIntent must be positive.') + raise ValueError(':property icmIntent: must be positive.') if val > 4294967295: - raise ValueError('icmIntent cannot be greater than 4294967295.') + raise ValueError(':property icmIntent: cannot be greater than 4294967295.') self.__fields |= DevModeFields.DM_ICMINTENT self.__icmIntent = val @@ -339,11 +364,14 @@ def _(self, val : Optional[int]) -> None: self.__icmMethod = 0 if DevModeFields.DM_ICMMETHOD in self.__fields: self.__fields ^= DevModeFields.DM_ICMMETHOD + return + elif not isinstance(val, int): + raise TypeError(':property icmMethod: must be an int or None.') if val < 0: - raise ValueError('icmMethod must be positive.') + raise ValueError(':property icmMethod: must be positive.') if val > 4294967295: - raise ValueError('icmMethod cannot be greater than 4294967295.') + raise ValueError(':property icmMethod: cannot be greater than 4294967295.') self.__fields |= DevModeFields.DM_ICMMETHOD self.__icmMethod = val @@ -358,11 +386,14 @@ def _(self, val : Optional[int]) -> None: self.__mediaType = 0 if DevModeFields.DM_MEDIATYPE in self.__fields: self.__fields ^= DevModeFields.DM_MEDIATYPE + return + elif not isinstance(val, int): + raise TypeError(':property mediaType: must be an int or None.') if val < 0: - raise ValueError('mediaType must be positive.') + raise ValueError(':property mediaType: must be positive.') if val > 4294967295: - raise ValueError('mediaType cannot be greater than 4294967295.') + raise ValueError(':property mediaType: cannot be greater than 4294967295.') self.__fields |= DevModeFields.DM_MEDIATYPE self.__mediaType = val @@ -377,11 +408,14 @@ def _(self, val : Optional[int]) -> None: self.__nup = 0 if DevModeFields.DM_NUP in self.__fields: self.__fields ^= DevModeFields.DM_NUP + return + elif not isinstance(val, int): + raise TypeError(':property nup: must be an int or None.') if val < 0: - raise ValueError('nup must be positive.') + raise ValueError(':property nup: must be positive.') if val > 4294967295: - raise ValueError('nup cannot be greater than 4294967295.') + raise ValueError(':property nup: cannot be greater than 4294967295.') self.__fields |= DevModeFields.DM_NUP self.__nup = val @@ -396,11 +430,14 @@ def _(self, val : Optional[int]) -> None: self.__orientation = 0 if DevModeFields.DM_ORIENTATION in self.__fields: self.__fields ^= DevModeFields.DM_ORIENTATION + return + elif not isinstance(val, int): + raise TypeError(':property orientation: must be an int or None.') if val < -32768: - raise ValueError('orientation cannot be less than -32768.') + raise ValueError(':property orientation: cannot be less than -32768.') if val > 32767: - raise ValueError('orientation cannot be greater than 32767.') + raise ValueError(':property orientation: cannot be greater than 32767.') self.__fields |= DevModeFields.DM_ORIENTATION self.__orientation = val @@ -415,11 +452,14 @@ def _(self, val : Optional[int]) -> None: self.__paperLength = 0 if DevModeFields.DM_PAPERLENGTH in self.__fields: self.__fields ^= DevModeFields.DM_PAPERLENGTH + return + elif not isinstance(val, int): + raise TypeError(':property paperLength: must be an int or None.') if val < -32768: - raise ValueError('paperLength cannot be less than -32768.') + raise ValueError(':property paperLength: cannot be less than -32768.') if val > 32767: - raise ValueError('paperLength cannot be greater than 32767.') + raise ValueError(':property paperLength: cannot be greater than 32767.') self.__fields |= DevModeFields.DM_PAPERLENGTH self.__paperLength = val @@ -434,11 +474,14 @@ def _(self, val : Optional[int]) -> None: self.__paperSize = 0 if DevModeFields.DM_PAPERSIZE in self.__fields: self.__fields ^= DevModeFields.DM_PAPERSIZE + return + elif not isinstance(val, int): + raise TypeError(':property paperSize: must be an int or None.') if val < -32768: - raise ValueError('paperSize cannot be less than -32768.') + raise ValueError(':property paperSize: cannot be less than -32768.') if val > 32767: - raise ValueError('paperSize cannot be greater than 32767.') + raise ValueError(':property paperSize: cannot be greater than 32767.') self.__fields |= DevModeFields.DM_PAPERSIZE self.__paperSize = val @@ -453,11 +496,14 @@ def _(self, val : Optional[int]) -> None: self.__paperWidth = 0 if DevModeFields.DM_PAPERWIDTH in self.__fields: self.__fields ^= DevModeFields.DM_PAPERWIDTH + return + elif not isinstance(val, int): + raise TypeError(':property paperWidth: must be an int or None.') if val < -32768: - raise ValueError('paperWidth cannot be less than -32768.') + raise ValueError(':property paperWidth: cannot be less than -32768.') if val > 32767: - raise ValueError('paperWidth cannot be greater than 32767.') + raise ValueError(':property paperWidth: cannot be greater than 32767.') self.__fields |= DevModeFields.DM_PAPERWIDTH self.__paperWidth = val @@ -472,11 +518,14 @@ def _(self, val : Optional[int]) -> None: self.__printQuality = 0 if DevModeFields.DM_PRINTQUALITY in self.__fields: self.__fields ^= DevModeFields.DM_PRINTQUALITY + return + elif not isinstance(val, int): + raise TypeError(':property printQuality: must be an int or None.') if val < -32768: - raise ValueError('printQuality cannot be less than -32768.') + raise ValueError(':property printQuality: cannot be less than -32768.') if val > 32767: - raise ValueError('printQuality cannot be greater than 32767.') + raise ValueError(':property printQuality: cannot be greater than 32767.') self.__fields |= DevModeFields.DM_PRINTQUALITY self.__printQuality = val @@ -491,11 +540,14 @@ def _(self, val : Optional[int]) -> None: self.__scale = 0 if DevModeFields.DM_SCALE in self.__fields: self.__fields ^= DevModeFields.DM_SCALE + return + elif not isinstance(val, int): + raise TypeError(':property scale: must be an int or None.') if val < -32768: - raise ValueError('scale cannot be less than -32768.') + raise ValueError(':property scale: cannot be less than -32768.') if val > 32767: - raise ValueError('scale cannot be greater than 32767.') + raise ValueError(':property scale: cannot be greater than 32767.') self.__fields |= DevModeFields.DM_SCALE self.__scale = val @@ -506,10 +558,12 @@ def specVersion(self) -> int: @specVersion.setter def _(self, val : int) -> None: + if not isinstance(val, int): + raise TypeError(':property specVersion: must be an int.') if val < 0: - raise ValueError('specVersion must be positive.') + raise ValueError(':property specVersion: must be positive.') if val > 65535: - raise ValueError('specVersion cannot be greater than 65535.') + raise ValueError(':property specVersion: cannot be greater than 65535.') self.__specVersion = val @@ -523,11 +577,14 @@ def _(self, val : Optional[int]) -> None: self.__ttOption = 0 if DevModeFields.DM_TTOPTION in self.__fields: self.__fields ^= DevModeFields.DM_TTOPTION + return + elif not isinstance(val, int): + raise TypeError(':property ttOption: must be an int or None.') if val < -32768: - raise ValueError('ttOption cannot be less than -32768.') + raise ValueError(':property ttOption: cannot be less than -32768.') if val > 32767: - raise ValueError('ttOption cannot be greater than 32767.') + raise ValueError(':property ttOption: cannot be greater than 32767.') self.__fields |= DevModeFields.DM_TTOPTION self.__ttOption = val @@ -542,11 +599,14 @@ def _(self, val : Optional[int]) -> None: self.__yResolution = 0 if DevModeFields.DM_YRESOLUTION in self.__fields: self.__fields ^= DevModeFields.DM_YRESOLUTION + return + elif not isinstance(val, int): + raise TypeError(':property yResolution: must be an int or None.') if val < -32768: - raise ValueError('yResolution cannot be less than -32768.') + raise ValueError(':property yResolution: cannot be less than -32768.') if val > 32767: - raise ValueError('yResolution cannot be greater than 32767.') + raise ValueError(':property yResolution: cannot be greater than 32767.') self.__fields |= DevModeFields.DM_YRESOLUTION self.__yResolution = val \ No newline at end of file From b8db2b022f3a9802e8f278d8a6171324ef873b05 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 11 Oct 2023 12:17:24 -0700 Subject: [PATCH 55/68] Use compat32 as policy for all headers --- extract_msg/msg_classes/message_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extract_msg/msg_classes/message_base.py b/extract_msg/msg_classes/message_base.py index eee7561f..36917633 100644 --- a/extract_msg/msg_classes/message_base.py +++ b/extract_msg/msg_classes/message_base.py @@ -1036,7 +1036,7 @@ def header(self) -> email.message.Message: header = HeaderParser(policy = policy.compat32).parsestr(headerText) else: logger.info('Header is empty or was not found. Header will be generated from other streams.') - header = HeaderParser(policy = policy.default).parsestr('') + header = HeaderParser(policy = policy.compat32).parsestr('') if self.date: header.add_header('Date', email.utils.format_datetime(self.date)) header.add_header('From', self.sender) From 7b5466ecc823352559b2622f9731a092730ba540 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 25 Oct 2023 09:59:35 -0700 Subject: [PATCH 56/68] Work on tocentry plus export doc update --- extract_msg/msg_classes/msg.py | 4 ++ extract_msg/structures/toc_entry.py | 57 +++++++++++++++++++++++++++-- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/extract_msg/msg_classes/msg.py b/extract_msg/msg_classes/msg.py index fe897837..4d08f1c8 100644 --- a/extract_msg/msg_classes/msg.py +++ b/extract_msg/msg_classes/msg.py @@ -448,6 +448,10 @@ def export(self, path) -> None: and directories will be added to it as if they were at the root, allowing you to save it as it's own MSG file. + This function pulls directly from the source MSG file, so modifications + to the properties of of an MSGFile object (or one of it's subclasses) + will not be reflected in the saved file. + :param path: An IO device with a write method which accepts bytes or a path-like object (including strings and pathlib.Path objects). """ diff --git a/extract_msg/structures/toc_entry.py b/extract_msg/structures/toc_entry.py index 632a6f6d..ec9985cf 100644 --- a/extract_msg/structures/toc_entry.py +++ b/extract_msg/structures/toc_entry.py @@ -7,7 +7,9 @@ from ._helpers import BytesReader from .cfoas import ClipboardFormatOrAnsiString +from ..constants import st from .dv_target_device import DVTargetDevice +from ..enums import ADVF, DVAspect class TOCEntry: @@ -19,7 +21,7 @@ def __init__(self, reader : Union[bytes, BytesReader]): self.__aspect = reader.readUnsignedInt() self.__lindex = reader.readUnsignedInt() self.__tymed = reader.readUnsignedInt() - reader.read(4) + reader.read(12) self.__advf = reader.readUnsignedInt() reader.read(4) if targetDeviceSize == 0: @@ -30,6 +32,55 @@ def __init__(self, reader : Union[bytes, BytesReader]): def toBytes(self) -> bytes: ret = self.__clipFormat.toBytes() - # TODO + td = self.__targetDevice.toBytes() if self.__targetDevice else b'' + ret += st.ST_LE_UI32.pack(len(td)) + ret += st.ST_LE_UI32.pack(self.__aspect) + ret += st.ST_LE_UI32.pack(self.__lindex) + ret += st.ST_LE_UI32.pack(self.__tymed) + #TODO + + return ret + + @property + def advf(self) -> Union[int, ADVF]: + """ + An implementation specific hint on how to render the presentation data + on screen. May be ignored on processing. + """ + return self.__advf + + @advf.setter + def _(self, val : Union[int, ADVF]) -> None: + if not isinstance(val, int): + raise TypeError(':property advf: must be an int.') + if val < 0: + raise ValueError(':property advf: must be positive.') + if val > 4294967295: + raise ValueError(':property advf: cannot be greater than 4294967295.') + + self.__advf = val + + @property + def ansiClipboardFormat(self) -> ClipboardFormatOrAnsiString: + return self.__clipFormat + + @property + def aspect(self) -> Union[int, DVAspect]: + """ + An implementation specific hint on how to render the presentation data + on screen. May be ignored on processing. + """ + return self.__aspect + + @aspect.setter + def _(self, val : Union[int, DVAspect]) -> None: + if not isinstance(val, int): + raise TypeError(':property aspect: must be an int.') + if val < 0: + raise ValueError(':property aspect: must be positive.') + if val > 4294967295: + raise ValueError(':property aspect: cannot be greater than 4294967295.') + + self.__aspect = val - return ret \ No newline at end of file + \ No newline at end of file From d38c80f5c12ddc46d229e405725bf1df679d6e82 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 7 Nov 2023 09:13:19 -0800 Subject: [PATCH 57/68] Change many values to hex, add __bytes__, tocentry --- CHANGELOG.md | 2 + extract_msg/msg_classes/msg.py | 3 + extract_msg/ole_writer.py | 6 +- extract_msg/properties/prop.py | 3 + extract_msg/properties/properties_store.py | 3 + extract_msg/structures/business_card.py | 8 +- extract_msg/structures/cfoas.py | 13 +- extract_msg/structures/contact_link_entry.py | 3 + extract_msg/structures/dev_mode_a.py | 136 +++++++++--------- extract_msg/structures/dv_target_device.py | 8 +- extract_msg/structures/entry_id.py | 3 + extract_msg/structures/misc_id.py | 12 ++ extract_msg/structures/mon_stream.py | 3 + extract_msg/structures/odt.py | 3 + extract_msg/structures/ole_pres.py | 48 ++++--- extract_msg/structures/ole_stream_struct.py | 5 +- extract_msg/structures/recurrence_pattern.py | 3 + extract_msg/structures/report_tag.py | 3 + extract_msg/structures/system_time.py | 21 +-- .../structures/time_zone_definition.py | 5 +- extract_msg/structures/time_zone_struct.py | 7 +- extract_msg/structures/toc_entry.py | 98 +++++++++---- extract_msg/structures/tz_rule.py | 4 +- extract_msg/utils.py | 25 ++-- 24 files changed, 278 insertions(+), 147 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c43c7af..f5fbc3ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ * Added new exception `DependencyError`. * Changed the errors for missing optional dependencies from `ImportError` to `DependencyError`. * Removed all instances of the `rawData` property in favor of the `toBytes` method. For now, many of these will simply return the raw data used, specifically those that are still unmodifiable. Any whose properties have the ability to be modified will have properly implemented versions. These classes also allow `None` to be passed as the value for their data, which will be the default if no arguments have been passed to the constructor. If no arguments or `None` is given as the data, it will create a new instance with default values. This is all in an effort to move towards the ability to create new MSG files and the `MSGWriter` class. All `toBytes` methods will either exclusively return `bytes` or will return `None` to specify that the structure isn't valid to convert to bytes. Structures that may be invalid will be annotated as `Optional[bytes]` for the return type. + * Additionally, these objects will also support the `__bytes__` method. If the object returned is *not* bytes, the method will throw a `TypeError`. * Removed the individual `PropBase` flag properties and changed the main `flags` property to return an enum containing the flags. * Changed various data structs to allow modification and creation of new instances for writing to an MSG file. * Changed `TZRule` to use unsigned values where applicable. @@ -48,6 +49,7 @@ * Removed unneeded function `windowsUnicode`. * Moved `FixedLengthProperty.parseType` to the private API. This was not intended for external use anyways, so leaving it as public API didn't make sense. * Fixed check for type in `ContactAddressEntryID` being the wrong value. +* Modified `inputToBytes` to support objects with the `__bytes__` method. If the method exists *and works* then it will be used as a last resort. **v0.45.0** * BREAKING: Changed parsing of string multiple properties to remove the trailing null byte. This *will* cause the output of parsing them to differ. diff --git a/extract_msg/msg_classes/msg.py b/extract_msg/msg_classes/msg.py index 4d08f1c8..f08b509a 100644 --- a/extract_msg/msg_classes/msg.py +++ b/extract_msg/msg_classes/msg.py @@ -225,6 +225,9 @@ def __init__(self, path, **kwargs): # Raise the exception after trying to close the file. raise + def __bytes__(self) -> bytes: + return self.exportBytes() + def __enter__(self) -> MSGFile: self.__ole.__enter__() return self diff --git a/extract_msg/ole_writer.py b/extract_msg/ole_writer.py index b29e05fc..0606cfe5 100644 --- a/extract_msg/ole_writer.py +++ b/extract_msg/ole_writer.py @@ -53,10 +53,10 @@ class DirectoryEntry: clsid : bytes = b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' data : bytes = b'' - def __init__(self): - pass + def __bytes__(self) -> bytes: + return self.toBytes() - def toBytes(self): + def toBytes(self) -> bytes: """ Converts the entry to bytes to be writen to a file. """ diff --git a/extract_msg/properties/prop.py b/extract_msg/properties/prop.py index 651fd772..e233e098 100644 --- a/extract_msg/properties/prop.py +++ b/extract_msg/properties/prop.py @@ -49,6 +49,9 @@ def __init__(self, data : bytes): self.__type, flags = constants.st.ST2.unpack(data) self.__flags = PropertyFlags(flags) + def __bytes__(self) -> bytes: + return self.toBytes() + def toBytes(self) -> bytes: return self.__rawData diff --git a/extract_msg/properties/properties_store.py b/extract_msg/properties/properties_store.py index cd45d55b..e99bb0f6 100644 --- a/extract_msg/properties/properties_store.py +++ b/extract_msg/properties/properties_store.py @@ -88,6 +88,9 @@ def __init__(self, data : Optional[bytes] = b'', _type : Optional[PropertiesType logger.warning(f'Found stream from divide that was not 16 bytes: {st}. Ignoring.') self.__pl = len(self.__props) + def __bytes__(self) -> bytes: + return self.toBytes() + def __contains__(self, key) -> bool: return self.__props.__contains__(key) diff --git a/extract_msg/structures/business_card.py b/extract_msg/structures/business_card.py index 5dc6d0f1..65c3c2e0 100644 --- a/extract_msg/structures/business_card.py +++ b/extract_msg/structures/business_card.py @@ -39,6 +39,9 @@ def __init__(self, data : bytes): self.__extraInfoField = data[17 + 16 * self.__countOfFields:] self.__fields = tuple(FieldInfo(reader.read(16), self.__extraInfoField) for _ in range(self.__countOfFields)) + def __bytes__(self) -> bytes: + return self.toBytes() + def toBytes(self) -> bytes: return self.__rawData @@ -149,7 +152,10 @@ def __init__(self, data : bytes, extraInfo : bytes): self.__labelFontColor = (bitwiseAdjustedAnd(unpacked[6], 0xFF), bitwiseAdjustedAnd(unpacked[6], 0xFF00), bitwiseAdjustedAnd(unpacked[6], 0xFF0000)) - + + def __bytes__(self) -> bytes: + return self.toBytes() + def toBytes(self) -> bytes: return self.__rawData diff --git a/extract_msg/structures/cfoas.py b/extract_msg/structures/cfoas.py index bfaec488..a8be17c7 100644 --- a/extract_msg/structures/cfoas.py +++ b/extract_msg/structures/cfoas.py @@ -11,7 +11,13 @@ class ClipboardFormatOrAnsiString: - def __init__(self, reader : Union[bytes, BytesReader]): + def __init__(self, reader : Optional[Union[bytes, BytesReader]] = None): + if reader is None: + self.__markerOrLength = 0 + self.__clipboardFormat = None + self.__ansiString = None + return + if isinstance(reader, bytes): reader = BytesReader(reader) @@ -26,6 +32,9 @@ def __init__(self, reader : Union[bytes, BytesReader]): self.__ansiString = None self.__clipboardFormat = None + def __bytes__(self) -> bytes: + return self.toBytes() + def toBytes(self) -> bytes: ret = constants.st.ST_LE_UI32.pack(self.markerOrLength) if self.markerOrLength > 0xFFFFFFFD: @@ -86,7 +95,7 @@ def _(self, val : int) -> None: if val < 0: raise ValueError(':property markerOrLength: must be a positive integer.') if val > 0xFFFFFFFF: - raise ValueError(':property markerOrLength: must be a 4 byte unsigned integer.') + raise ValueError(':property markerOrLength: cannot be greater than 0xFFFFFFFF') if val == 0: self.__ansiString = None diff --git a/extract_msg/structures/contact_link_entry.py b/extract_msg/structures/contact_link_entry.py index cc4d9c14..4be5bd63 100644 --- a/extract_msg/structures/contact_link_entry.py +++ b/extract_msg/structures/contact_link_entry.py @@ -28,6 +28,9 @@ def __init__(self, data : bytes): if (size & 3) != 0: reader.read(4 - (size & 3)) + def __bytes__(self) -> bytes: + return self.toBytes() + def toBytes(self) -> bytes: ret = st.ST_LE_UI32.pack(len(self.entries)) diff --git a/extract_msg/structures/dev_mode_a.py b/extract_msg/structures/dev_mode_a.py index 5d1d898a..08e27fa6 100644 --- a/extract_msg/structures/dev_mode_a.py +++ b/extract_msg/structures/dev_mode_a.py @@ -152,10 +152,10 @@ def _(self, val : Optional[int]) -> None: elif not isinstance(val, int): raise TypeError(':property collate: must be an int or None.') - if val < -32768: - raise ValueError(':property collate: cannot be less than -32768.') - if val > 32767: - raise ValueError(':property collate: cannot be greater than 32767.') + if val < -0x8000: + raise ValueError(':property collate: cannot be less than -0x8000.') + if val > 0x7FFF: + raise ValueError(':property collate: cannot be greater than 0x7FFF.') self.__fields |= DevModeFields.DM_COLLATE self.__collate = val @@ -174,10 +174,10 @@ def _(self, val : Optional[int]) -> None: elif not isinstance(val, int): raise TypeError(':property color: must be an int or None.') - if val < -32768: - raise ValueError(':property color: cannot be less than -32768.') - if val > 32767: - raise ValueError(':property color: cannot be greater than 32767.') + if val < -0x8000: + raise ValueError(':property color: cannot be less than -0x8000.') + if val > 0x7FFF: + raise ValueError(':property color: cannot be greater than 0x7FFF.') self.__fields |= DevModeFields.DM_COLOR self.__color = val @@ -196,10 +196,10 @@ def _(self, val : Optional[int]) -> None: elif not isinstance(val, int): raise TypeError(':property copies: must be an int or None.') - if val < -32768: - raise ValueError(':property copies: cannot be less than -32768.') - if val > 32767: - raise ValueError(':property copies: cannot be greater than 32767.') + if val < -0x8000: + raise ValueError(':property copies: cannot be less than -0x8000.') + if val > 0x7FFF: + raise ValueError(':property copies: cannot be greater than 0x7FFF.') self.__fields |= DevModeFields.DM_COPIES self.__copies = val @@ -218,10 +218,10 @@ def _(self, val : Optional[int]) -> None: elif not isinstance(val, int): raise TypeError(':property defaultSource: must be an int or None.') - if val < -32768: - raise ValueError(':property defaultSource: cannot be less than -32768.') - if val > 32767: - raise ValueError(':property defaultSource: cannot be greater than 32767.') + if val < -0x8000: + raise ValueError(':property defaultSource: cannot be less than -0x8000.') + if val > 0x7FFF: + raise ValueError(':property defaultSource: cannot be greater than 0x7FFF.') self.__fields |= DevModeFields.DM_DEFAULTSOURCE self.__defaultSource = val @@ -258,8 +258,8 @@ def _(self, val : Optional[int]) -> None: if val < 0: raise ValueError(':property ditherType: must be positive.') - if val > 4294967295: - raise ValueError(':property ditherType: cannot be greater than 4294967295.') + if val > 0xFFFFFFFF: + raise ValueError(':property ditherType: cannot be greater than 0xFFFFFFFF.') self.__fields |= DevModeFields.DM_DITHERTYPE self.__ditherType = val @@ -274,8 +274,8 @@ def _(self, val : int) -> None: raise TypeError(':property driverExtra: must be an int.') if val < 0: raise ValueError(':property driverExtra: must be positive.') - if val > 65535: - raise ValueError(':property driverExtra: cannot be greater than 65535.') + if val > 0xFFFF: + raise ValueError(':property driverExtra: cannot be greater than 0xFFFF.') self.__driverExtra = val @@ -289,8 +289,8 @@ def _(self, val : int) -> None: raise TypeError(':property driverVersion: must be an int or None.') if val < 0: raise ValueError(':property driverVersion: must be positive.') - if val > 65535: - raise ValueError(':property driverVersion: cannot be greater than 65535.') + if val > 0xFFFF: + raise ValueError(':property driverVersion: cannot be greater than 0xFFFF.') self.__driverVersion = val @@ -308,10 +308,10 @@ def _(self, val : Optional[int]) -> None: elif not isinstance(val, int): raise TypeError(':property duplex: must be an int or None.') - if val < -32768: - raise ValueError(':property duplex: cannot be less than -32768.') - if val > 32767: - raise ValueError(':property duplex: cannot be greater than 32767.') + if val < -0x8000: + raise ValueError(':property duplex: cannot be less than -0x8000.') + if val > 0x7FFF: + raise ValueError(':property duplex: cannot be greater than 0x7FFF.') self.__fields |= DevModeFields.DM_DUPLEX self.__duplex = val @@ -348,8 +348,8 @@ def _(self, val : Optional[int]) -> None: if val < 0: raise ValueError(':property icmIntent: must be positive.') - if val > 4294967295: - raise ValueError(':property icmIntent: cannot be greater than 4294967295.') + if val > 0xFFFFFFFF: + raise ValueError(':property icmIntent: cannot be greater than 0xFFFFFFFF.') self.__fields |= DevModeFields.DM_ICMINTENT self.__icmIntent = val @@ -370,8 +370,8 @@ def _(self, val : Optional[int]) -> None: if val < 0: raise ValueError(':property icmMethod: must be positive.') - if val > 4294967295: - raise ValueError(':property icmMethod: cannot be greater than 4294967295.') + if val > 0xFFFFFFFF: + raise ValueError(':property icmMethod: cannot be greater than 0xFFFFFFFF.') self.__fields |= DevModeFields.DM_ICMMETHOD self.__icmMethod = val @@ -392,8 +392,8 @@ def _(self, val : Optional[int]) -> None: if val < 0: raise ValueError(':property mediaType: must be positive.') - if val > 4294967295: - raise ValueError(':property mediaType: cannot be greater than 4294967295.') + if val > 0xFFFFFFFF: + raise ValueError(':property mediaType: cannot be greater than 0xFFFFFFFF.') self.__fields |= DevModeFields.DM_MEDIATYPE self.__mediaType = val @@ -414,8 +414,8 @@ def _(self, val : Optional[int]) -> None: if val < 0: raise ValueError(':property nup: must be positive.') - if val > 4294967295: - raise ValueError(':property nup: cannot be greater than 4294967295.') + if val > 0xFFFFFFFF: + raise ValueError(':property nup: cannot be greater than 0xFFFFFFFF.') self.__fields |= DevModeFields.DM_NUP self.__nup = val @@ -434,10 +434,10 @@ def _(self, val : Optional[int]) -> None: elif not isinstance(val, int): raise TypeError(':property orientation: must be an int or None.') - if val < -32768: - raise ValueError(':property orientation: cannot be less than -32768.') - if val > 32767: - raise ValueError(':property orientation: cannot be greater than 32767.') + if val < -0x8000: + raise ValueError(':property orientation: cannot be less than -0x8000.') + if val > 0x7FFF: + raise ValueError(':property orientation: cannot be greater than 0x7FFF.') self.__fields |= DevModeFields.DM_ORIENTATION self.__orientation = val @@ -456,10 +456,10 @@ def _(self, val : Optional[int]) -> None: elif not isinstance(val, int): raise TypeError(':property paperLength: must be an int or None.') - if val < -32768: - raise ValueError(':property paperLength: cannot be less than -32768.') - if val > 32767: - raise ValueError(':property paperLength: cannot be greater than 32767.') + if val < -0x8000: + raise ValueError(':property paperLength: cannot be less than -0x8000.') + if val > 0x7FFF: + raise ValueError(':property paperLength: cannot be greater than 0x7FFF.') self.__fields |= DevModeFields.DM_PAPERLENGTH self.__paperLength = val @@ -478,10 +478,10 @@ def _(self, val : Optional[int]) -> None: elif not isinstance(val, int): raise TypeError(':property paperSize: must be an int or None.') - if val < -32768: - raise ValueError(':property paperSize: cannot be less than -32768.') - if val > 32767: - raise ValueError(':property paperSize: cannot be greater than 32767.') + if val < -0x8000: + raise ValueError(':property paperSize: cannot be less than -0x8000.') + if val > 0x7FFF: + raise ValueError(':property paperSize: cannot be greater than 0x7FFF.') self.__fields |= DevModeFields.DM_PAPERSIZE self.__paperSize = val @@ -500,10 +500,10 @@ def _(self, val : Optional[int]) -> None: elif not isinstance(val, int): raise TypeError(':property paperWidth: must be an int or None.') - if val < -32768: - raise ValueError(':property paperWidth: cannot be less than -32768.') - if val > 32767: - raise ValueError(':property paperWidth: cannot be greater than 32767.') + if val < -0x8000: + raise ValueError(':property paperWidth: cannot be less than -0x8000.') + if val > 0x7FFF: + raise ValueError(':property paperWidth: cannot be greater than 0x7FFF.') self.__fields |= DevModeFields.DM_PAPERWIDTH self.__paperWidth = val @@ -522,10 +522,10 @@ def _(self, val : Optional[int]) -> None: elif not isinstance(val, int): raise TypeError(':property printQuality: must be an int or None.') - if val < -32768: - raise ValueError(':property printQuality: cannot be less than -32768.') - if val > 32767: - raise ValueError(':property printQuality: cannot be greater than 32767.') + if val < -0x8000: + raise ValueError(':property printQuality: cannot be less than -0x8000.') + if val > 0x7FFF: + raise ValueError(':property printQuality: cannot be greater than 0x7FFF.') self.__fields |= DevModeFields.DM_PRINTQUALITY self.__printQuality = val @@ -544,10 +544,10 @@ def _(self, val : Optional[int]) -> None: elif not isinstance(val, int): raise TypeError(':property scale: must be an int or None.') - if val < -32768: - raise ValueError(':property scale: cannot be less than -32768.') - if val > 32767: - raise ValueError(':property scale: cannot be greater than 32767.') + if val < -0x8000: + raise ValueError(':property scale: cannot be less than -0x8000.') + if val > 0x7FFF: + raise ValueError(':property scale: cannot be greater than 0x7FFF.') self.__fields |= DevModeFields.DM_SCALE self.__scale = val @@ -562,8 +562,8 @@ def _(self, val : int) -> None: raise TypeError(':property specVersion: must be an int.') if val < 0: raise ValueError(':property specVersion: must be positive.') - if val > 65535: - raise ValueError(':property specVersion: cannot be greater than 65535.') + if val > 0xFFFF: + raise ValueError(':property specVersion: cannot be greater than 0xFFFF.') self.__specVersion = val @@ -581,10 +581,10 @@ def _(self, val : Optional[int]) -> None: elif not isinstance(val, int): raise TypeError(':property ttOption: must be an int or None.') - if val < -32768: - raise ValueError(':property ttOption: cannot be less than -32768.') - if val > 32767: - raise ValueError(':property ttOption: cannot be greater than 32767.') + if val < -0x8000: + raise ValueError(':property ttOption: cannot be less than -0x8000.') + if val > 0x7FFF: + raise ValueError(':property ttOption: cannot be greater than 0x7FFF.') self.__fields |= DevModeFields.DM_TTOPTION self.__ttOption = val @@ -603,10 +603,10 @@ def _(self, val : Optional[int]) -> None: elif not isinstance(val, int): raise TypeError(':property yResolution: must be an int or None.') - if val < -32768: - raise ValueError(':property yResolution: cannot be less than -32768.') - if val > 32767: - raise ValueError(':property yResolution: cannot be greater than 32767.') + if val < -0x8000: + raise ValueError(':property yResolution: cannot be less than -0x8000.') + if val > 0x7FFF: + raise ValueError(':property yResolution: cannot be greater than 0x7FFF.') self.__fields |= DevModeFields.DM_YRESOLUTION self.__yResolution = val \ No newline at end of file diff --git a/extract_msg/structures/dv_target_device.py b/extract_msg/structures/dv_target_device.py index adbdf452..06109e94 100644 --- a/extract_msg/structures/dv_target_device.py +++ b/extract_msg/structures/dv_target_device.py @@ -79,6 +79,12 @@ def __init__(self, data : Optional[bytes]): except IOError: self.__extDevMode = None + def __bytes__(self) -> bytes: + ret = self.toBytes() + if not isinstance(ret, bytes): + raise TypeError(f'Cannot convert {self.__class__.__name__} instance to bytes.') + return ret + def toBytes(self) -> Optional[bytes]: if not (self.driverName or self.deviceName or self.portName or self.extDevMode): return None @@ -96,7 +102,7 @@ def toBytes(self) -> Optional[bytes]: if self.__portName: currentPosition += len(self.__portName) + 1 - extDevModeBytes = self.__extDevMode.toBytes() if self.__extDevMode else None + extDevModeBytes = bytes(self.__extDevMode) if self.__extDevMode else None offset4 = currentPosition if extDevModeBytes else 0 try: diff --git a/extract_msg/structures/entry_id.py b/extract_msg/structures/entry_id.py index e4ad6607..8bdbaa4a 100644 --- a/extract_msg/structures/entry_id.py +++ b/extract_msg/structures/entry_id.py @@ -98,6 +98,9 @@ def __init__(self, data : bytes): self.__providerUID = data[4:20] self.__rawData = data + def __bytes__(self) -> bytes: + return self.toBytes() + def toBytes(self) -> bytes: return self.__rawData diff --git a/extract_msg/structures/misc_id.py b/extract_msg/structures/misc_id.py index 38df625d..989e0cad 100644 --- a/extract_msg/structures/misc_id.py +++ b/extract_msg/structures/misc_id.py @@ -32,6 +32,9 @@ def __init__(self, data : bytes): # This entry is 6 bytes, so we pull some shenanigans to unpack it. self.__globalCounter = constants.st.ST_LE_UI64.unpack(data[2:8] + b'\x00\x00')[0] + def __bytes__(self) -> bytes: + return self.toBytes() + def toBytes(self) -> bytes: return self.__rawData @@ -72,6 +75,9 @@ def __init__(self, data : bytes): size = reader.readUnsignedInt() self.__data = reader.read(size) + def __bytes__(self) -> bytes: + return self.toBytes() + def toBytes(self) -> bytes: return self.__rawData @@ -137,6 +143,9 @@ def __init__(self, data : bytes): # This entry is 6 bytes, so we pull some shenanigans to unpack it. self.__globalCounter = constants.st.ST_LE_UI64.unpack(data[2:8] + b'\x00\x00')[0] + def __bytes__(self) -> bytes: + return self.toBytes() + def toBytes(self) -> bytes: return self.__rawData @@ -182,6 +191,9 @@ def __init__(self, data : bytes): self.__messageID = MessageID(data[9:17]) self.__instance = constants.st.STUI32.unpack(data[17:21])[0] + def __bytes__(self) -> bytes: + return self.toBytes() + def toBytes(self) -> bytes: return self.__rawData diff --git a/extract_msg/structures/mon_stream.py b/extract_msg/structures/mon_stream.py index 33d1f531..220e82bb 100644 --- a/extract_msg/structures/mon_stream.py +++ b/extract_msg/structures/mon_stream.py @@ -17,6 +17,9 @@ def __init__(self, data : Optional[bytes] = None): self.__clsid = b'\x00' * 16 self.__streamData = b'' + def __bytes__(self) -> bytes: + return self.toBytes() + def toBytes(self) -> bytes: return self.__clsid + self.__streamData diff --git a/extract_msg/structures/odt.py b/extract_msg/structures/odt.py index 8b115d83..6b4b3245 100644 --- a/extract_msg/structures/odt.py +++ b/extract_msg/structures/odt.py @@ -24,6 +24,9 @@ def __init__(self, data : Optional[bytes] = None): self.__persist1 = ODTPersist1.NONE self.__persist2 = ODTPersist2.NONE + def __bytes__(self) -> bytes: + return self.toBytes() + def toBytes(self) -> bytes: return struct.pack(' bytes: + return self.toBytes() def toBytes(self) -> bytes: - ret = self.__ansiClipboardFormat.toBytes() + ret = bytes(self.__ansiClipboardFormat) if self.__targetDevice is None: ret += b'\x04\x00\x00\x00' else: - dvData = self.__targetDevice.toBytes() + dvData = bytes(self.__targetDevice) ret += st.ST_LE_UI32.pack(len(dvData)) ret += dvData @@ -100,7 +102,7 @@ def toBytes(self) -> bytes: if self.__tocSignature == 0x494E414E: ret += st.ST_LE_UI32.pack(len(self.__tocEntries)) for entry in self.__tocEntries: - ret += entry.toBytes() + ret += bytes(entry) else: ret += b'\x00\x00\x00\x00' @@ -109,7 +111,7 @@ def toBytes(self) -> bytes: @property def advf(self) -> Union[int, ADVF]: """ - An implementation specific hint on how to render the presentation data + An implementation specific hint on how to render the presentation data on screen. May be ignored on processing. """ return self.__advf @@ -120,8 +122,8 @@ def _(self, val : Union[int, ADVF]) -> None: raise TypeError(':property advf: must be an int.') if val < 0: raise ValueError(':property advf: must be positive.') - if val > 4294967295: - raise ValueError(':property advf: cannot be greater than 4294967295.') + if val > 0xFFFFFFFF: + raise ValueError(':property advf: cannot be greater than 0xFFFFFFFF.') self.__advf = val @@ -132,7 +134,7 @@ def ansiClipboardFormat(self) -> ClipboardFormatOrAnsiString: @property def aspect(self) -> Union[int, DVAspect]: """ - An implementation specific hint on how to render the presentation data + An implementation specific hint on how to render the presentation data on screen. May be ignored on processing. """ return self.__aspect @@ -143,8 +145,8 @@ def _(self, val : Union[int, DVAspect]) -> None: raise TypeError(':property aspect: must be an int.') if val < 0: raise ValueError(':property aspect: must be positive.') - if val > 4294967295: - raise ValueError(':property aspect: cannot be greater than 4294967295.') + if val > 0xFFFFFFFF: + raise ValueError(':property aspect: cannot be greater than 0xFFFFFFFF.') self.__aspect = val @@ -174,8 +176,8 @@ def _(self, val : int) -> None: raise TypeError(':property height: must be an int.') if val < 0: raise ValueError(':property height: must be positive.') - if val > 4294967295: - raise ValueError(':property height: cannot be greater than 4294967295.') + if val > 0xFFFFFFFF: + raise ValueError(':property height: cannot be greater than 0xFFFFFFFF.') self.__height = val @@ -192,15 +194,15 @@ def _(self, val : int) -> None: raise TypeError(':property lindex: must be an int.') if val < 0: raise ValueError(':property lindex: must be positive.') - if val > 4294967295: - raise ValueError(':property lindex: cannot be greater than 4294967295.') + if val > 0xFFFFFFFF: + raise ValueError(':property lindex: cannot be greater than 0xFFFFFFFF.') self.__lindex = val @property def reserved1(self) -> bytes: """ - 4 bytes that can contain any arbitrary data. Must be *exactly* 4 bytes + 4 bytes that can contain any arbitrary data. Must be *exactly* 4 bytes when setting. """ return self.__reserved1 @@ -217,7 +219,7 @@ def _(self, val : bytes) -> None: @property def reserved2(self) -> Optional[bytes]: """ - Optional additional data that is only set if the clipboard format of + Optional additional data that is only set if the clipboard format of :property ansiClipboardFormat: is CF_METAFILEPICT. Getting this will automatically correct the value retrieved based on @@ -266,12 +268,12 @@ def tocEntries(self) -> List[TOCEntry]: @property def tocSignature(self) -> int: """ - If this field does not contain 0x494E414E, then :property tocEntries: - MUST be empty. Modifications to the list will be lost when it is next + If this field does not contain 0x494E414E, then :property tocEntries: + MUST be empty. Modifications to the list will be lost when it is next retrieved, meaning changes while this property is not 0x494E414E WILL be lost. - Setting this to a value other than 0x494E414E will clear the list + Setting this to a value other than 0x494E414E will clear the list immediately. """ return self.__tocSignature @@ -282,9 +284,9 @@ def _(self, val : int) -> None: raise TypeError(':property tocSignature: must be an int.') if val < 0: raise ValueError(':property tocSignature: must be positive.') - if val > 4294967295: - raise ValueError(':property tocSignature: cannot be greater than 4294967295.') - + if val > 0xFFFFFFFF: + raise ValueError(':property tocSignature: cannot be greater than 0xFFFFFFFF.') + if val != 0x494E414E: self.__tocEntries.clear() @@ -303,8 +305,8 @@ def _(self, val : int) -> None: raise TypeError(':property width: must be an int.') if val < 0: raise ValueError(':property width: must be positive.') - if val > 4294967295: - raise ValueError(':property width: cannot be greater than 4294967295.') + if val > 0xFFFFFFFF: + raise ValueError(':property width: cannot be greater than 0xFFFFFFFF.') self.__width = val diff --git a/extract_msg/structures/ole_stream_struct.py b/extract_msg/structures/ole_stream_struct.py index 2042676f..2ba311b2 100644 --- a/extract_msg/structures/ole_stream_struct.py +++ b/extract_msg/structures/ole_stream_struct.py @@ -32,12 +32,15 @@ def __init__(self, data : Optional[bytes] = None): # Only check this stuff if this is not for an embedded object. pass # TODO + def __bytes__(self) -> bytes: + return self.toBytes() + def toBytes(self) -> bytes: ret = b'\x01\x00\x00\x02' ret += st.ST_LE_UI32.pack(self.__flags) ret += st.ST_LE_UI32.pack(self.__linkUpdateOption) ret += b'\x00\x00\x00\x00' - rmsBytes = b'' if self.__rms is None else self.__rms.toBytes() + rmsBytes = b'' if self.__rms is None else bytes(self.__rms) ret += st.ST_LE_UI32.pack(len(rmsBytes)) + rmsBytes # TODO finish this with the optional properties. diff --git a/extract_msg/structures/recurrence_pattern.py b/extract_msg/structures/recurrence_pattern.py index b33e8b34..21b98aa9 100644 --- a/extract_msg/structures/recurrence_pattern.py +++ b/extract_msg/structures/recurrence_pattern.py @@ -51,6 +51,9 @@ def __init__(self, data : bytes): self.__startDate = reader.readUnsignedInt() self.__endDate = reader.readUnsignedInt() + def __bytes__(self) -> bytes: + return self.toBytes() + def toBytes(self) -> bytes: return self.__rawData diff --git a/extract_msg/structures/report_tag.py b/extract_msg/structures/report_tag.py index 648b743e..c84197bc 100644 --- a/extract_msg/structures/report_tag.py +++ b/extract_msg/structures/report_tag.py @@ -57,6 +57,9 @@ def __init__(self, data : bytes): else: self.__ansiText = None + def __bytes__(self) -> bytes: + return self.toBytes() + def toBytes(self) -> bytes: return self.__rawData diff --git a/extract_msg/structures/system_time.py b/extract_msg/structures/system_time.py index cb22b74e..0c3a10b3 100644 --- a/extract_msg/structures/system_time.py +++ b/extract_msg/structures/system_time.py @@ -17,11 +17,14 @@ def __init__(self, data : Optional[bytes] = None): self.unpack(data) def __eq__(self, other : Any) -> bool: - return isinstance(other, SystemTime) and self.toBytes() == other.toBytes() + return isinstance(other, SystemTime) and bytes(self) == bytes(other) def __ne__(self, other : Any) -> bool: return not self.__eq__(other) + def __bytes__(self) -> bytes: + return self.toBytes() + def toBytes(self) -> bytes: """ Packs the current data into bytes. @@ -58,7 +61,7 @@ def _(self, value : int) -> None: if value < 0: raise ValueError('Day must be positive.') if value > 0xFFFF: - raise ValueError('Day must be less than 65535.') + raise ValueError('Day cannot be greater than 0xFFFF.') self.__day = value @@ -71,7 +74,7 @@ def _(self, value : int) -> None: if value < 0: raise ValueError('Day of week must be positive.') if value > 0xFFFF: - raise ValueError('Day of week must be less than 65535.') + raise ValueError('Day of week cannot be greater than 0xFFFF.') self.__dayOfWeek = value @@ -84,7 +87,7 @@ def _(self, value : int) -> None: if value < 0: raise ValueError('Hour must be positive.') if value > 0xFFFF: - raise ValueError('Hour must be less than 65535.') + raise ValueError('Hour cannot be greater than 0xFFFF.') self.__hour = value @@ -97,7 +100,7 @@ def _(self, value : int) -> None: if value < 0: raise ValueError('Milliseconds must be positive.') if value > 0xFFFF: - raise ValueError('Milliseconds must be less than 65535.') + raise ValueError('Milliseconds cannot be greater than 0xFFFF.') self.__milliseconds = value @@ -110,7 +113,7 @@ def _(self, value : int) -> None: if value < 0: raise ValueError('Minute must be positive.') if value > 0xFFFF: - raise ValueError('Minute must be less than 65535.') + raise ValueError('Minute cannot be greater than 0xFFFF.') self.__minute = value @@ -123,7 +126,7 @@ def _(self, value : int) -> None: if value < 0: raise ValueError('Month must be positive.') if value > 0xFFFF: - raise ValueError('Month must be less than 65535.') + raise ValueError('Month cannot be greater than 0xFFFF.') self.__month = value @@ -136,7 +139,7 @@ def _(self, value : int) -> None: if value < 0: raise ValueError('Second must be positive.') if value > 0xFFFF: - raise ValueError('Second must be less than 65535.') + raise ValueError('Second cannot be greater than 0xFFFF.') self.__second = value @@ -149,7 +152,7 @@ def _(self, value : int) -> None: if value < 0: raise ValueError('Year must be positive.') if value > 0xFFFF: - raise ValueError('Year must be less than 65535.') + raise ValueError('Year cannot be greater than 0xFFFF.') self.__year = value diff --git a/extract_msg/structures/time_zone_definition.py b/extract_msg/structures/time_zone_definition.py index c06f8a1a..0b29a061 100644 --- a/extract_msg/structures/time_zone_definition.py +++ b/extract_msg/structures/time_zone_definition.py @@ -34,6 +34,9 @@ def __init__(self, data : Optional[bytes] = None): raise ValueError('Value for cRules was out of range.') self.__rules = [reader.readClass(TZRule) for _ in range(cRules)] + def __bytes__(self) -> bytes: + return self.toBytes() + def toBytes(self) -> bytes: # Validate some of the data. if len(self.__rules) < 1: @@ -46,7 +49,7 @@ def toBytes(self) -> bytes: ret += b'\x02\x00' ret += st.ST_LE_UI16.pack(2* len(self.__keyName)) ret += st.ST_LE_UI16.pack(len(self.__rules)) - ret += b''.join(x.toBytes() for x in self.__rules) + ret += b''.join(bytes(x) for x in self.__rules) return ret diff --git a/extract_msg/structures/time_zone_struct.py b/extract_msg/structures/time_zone_struct.py index 0e911963..d6adc6be 100644 --- a/extract_msg/structures/time_zone_struct.py +++ b/extract_msg/structures/time_zone_struct.py @@ -30,14 +30,17 @@ def __init__(self, data : Optional[bytes] = None): self.__standardDate = SystemTime(unpacked[4]) self.__daylightDate = SystemTime(unpacked[6]) + def __bytes__(self) -> bytes: + return self.toBytes() + def toBytes(self) -> bytes: return constants.st.ST_TZ.pack(self.__bias, self.__standardBias, self.__daylightBias, self.standardYear, - self.__standardDate.toBytes(), + bytes(self.__standardDate), self.daylightYear, - self.__daylightDate.toBytes()) + bytes(self.__daylightDate)) @property def bias(self) -> int: diff --git a/extract_msg/structures/toc_entry.py b/extract_msg/structures/toc_entry.py index ec9985cf..f44bdba9 100644 --- a/extract_msg/structures/toc_entry.py +++ b/extract_msg/structures/toc_entry.py @@ -13,38 +13,54 @@ class TOCEntry: - def __init__(self, reader : Union[bytes, BytesReader]): + def __init__(self, reader : Optional[Union[bytes, BytesReader]] = None): + if reader is None: + self.__clipFormat = ClipboardFormatOrAnsiString() + self.__aspect = 0 + self.__lindex = 0 + self.__tymed = 0 + self.__advf = 0 + self.__targetDevice = None + return + if isinstance(reader, bytes): reader = BytesReader(reader) - self.__clipFormat = ClipboardFormatOrAnsiString(reader) - targetDeviceSize = reader.readUnsignedInt() - self.__aspect = reader.readUnsignedInt() - self.__lindex = reader.readUnsignedInt() - self.__tymed = reader.readUnsignedInt() - reader.read(12) - self.__advf = reader.readUnsignedInt() - reader.read(4) - if targetDeviceSize == 0: - self.__targetDevice = None - else: - self.__targetDevice = DVTargetDevice(reader.read(targetDeviceSize)) - # TODO + + self.__clipFormat = ClipboardFormatOrAnsiString(reader) + targetDeviceSize = reader.readUnsignedInt() + self.__aspect = reader.readUnsignedInt() + self.__lindex = reader.readUnsignedInt() + self.__tymed = reader.readUnsignedInt() + reader.read(12) + self.__advf = reader.readUnsignedInt() + reader.read(4) + if targetDeviceSize == 0: + self.__targetDevice = None + else: + self.__targetDevice = DVTargetDevice(reader.read(targetDeviceSize)) + + + def __bytes__(self) -> bytes: + return self.toBytes() def toBytes(self) -> bytes: - ret = self.__clipFormat.toBytes() - td = self.__targetDevice.toBytes() if self.__targetDevice else b'' + ret = bytes(self.__clipFormat) + td = bytes(self.__targetDevice) if self.__targetDevice else b'' ret += st.ST_LE_UI32.pack(len(td)) ret += st.ST_LE_UI32.pack(self.__aspect) ret += st.ST_LE_UI32.pack(self.__lindex) ret += st.ST_LE_UI32.pack(self.__tymed) - #TODO + ret += b'\x00' * 12 + ret += st.ST_LE_UI32.pack(self.__advf) + ret += b'\x00' * 4 + ret += td return ret @property def advf(self) -> Union[int, ADVF]: """ - An implementation specific hint on how to render the presentation data + An implementation specific hint on how to render the presentation data on screen. May be ignored on processing. """ return self.__advf @@ -55,19 +71,19 @@ def _(self, val : Union[int, ADVF]) -> None: raise TypeError(':property advf: must be an int.') if val < 0: raise ValueError(':property advf: must be positive.') - if val > 4294967295: - raise ValueError(':property advf: cannot be greater than 4294967295.') + if val > 0xFFFFFFFF: + raise ValueError(':property advf: cannot be greater than 0xFFFFFFFF.') self.__advf = val @property def ansiClipboardFormat(self) -> ClipboardFormatOrAnsiString: return self.__clipFormat - + @property def aspect(self) -> Union[int, DVAspect]: """ - An implementation specific hint on how to render the presentation data + An implementation specific hint on how to render the presentation data on screen. May be ignored on processing. """ return self.__aspect @@ -78,9 +94,41 @@ def _(self, val : Union[int, DVAspect]) -> None: raise TypeError(':property aspect: must be an int.') if val < 0: raise ValueError(':property aspect: must be positive.') - if val > 4294967295: - raise ValueError(':property aspect: cannot be greater than 4294967295.') + if val > 0xFFFFFFFF: + raise ValueError(':property aspect: cannot be greater than 0xFFFFFFFF.') self.__aspect = val - \ No newline at end of file + @property + def lindex(self) -> int: + """ + An implementation specific hint on how to render the presentation data + on screen. May be ignored on processing. + """ + return self.__lindex + + @lindex.setter + def _(self, val : int) -> None: + if not isinstance(val, int): + raise TypeError(':property lindex: must be an int.') + if val < 0: + raise ValueError(':property lindex: must be positive.') + if val > 0xFFFFFFFF: + raise ValueError(':property lindex: cannot be greater than 0xFFFFFFFF.') + + self.__lindex = val + + @property + def tymed(self) -> int: + return self.__tymed + + @tymed.setter + def _(self, val : int) -> None: + if not isinstance(val, int): + raise TypeError(':property lindex: must be an int.') + if val < 0: + raise ValueError(':property lindex: must be positive.') + if val > 0xFFFFFFFF: + raise ValueError(':property lindex: cannot be greater than 0xFFFFFFFF.') + + self.__tymed = val diff --git a/extract_msg/structures/tz_rule.py b/extract_msg/structures/tz_rule.py index 8b3dbe18..bc2e2169 100644 --- a/extract_msg/structures/tz_rule.py +++ b/extract_msg/structures/tz_rule.py @@ -65,8 +65,8 @@ def toBytes(self) -> bytes: self.__bias, self.__standardBias, self.__daylightBias, - self.__standardDate.toBytes(), - self.__daylightDate.toBytes()) + bytes(self.__standardDate), + bytes(self.__daylightDate)) @property def bias(self) -> int: diff --git a/extract_msg/utils.py b/extract_msg/utils.py index 5f73e041..c85ca367 100644 --- a/extract_msg/utils.py +++ b/extract_msg/utils.py @@ -569,20 +569,27 @@ def htmlSanitize(inp : str) -> str: return inp -def inputToBytes(stringInputVar : Optional[Union[str, bytes]], encoding : str) -> bytes: +def inputToBytes(obj : Any, encoding : str) -> bytes: """ Converts the input into bytes. - :raises ConversionError: if the input cannot be converted. + :raises ConversionError: The input cannot be converted. + :raises UnicodeEncodeError: The input was a str but the encoding was not + valid. + :raises TypeError: The input has a __bytes__ method, but it failed. + :raises ValueError: Same as above. """ - if isinstance(stringInputVar, bytes): - return stringInputVar - elif isinstance(stringInputVar, str): - return stringInputVar.encode(encoding) - elif stringInputVar is None: + if isinstance(obj, bytes): + return obj + if isinstance(obj, str): + return obj.encode(encoding) + if obj is None: return b'' - else: - raise ConversionError('Cannot convert to bytes.') + if hasattr(obj, '__bytes__'): + try: + + + raise ConversionError('Cannot convert to bytes.') def inputToMsgPath(inp : constants.MSG_PATH) -> List[str]: From 3ce6040716c3e8a9a9659a06aa3bbd322e567c3c Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 7 Nov 2023 09:21:32 -0800 Subject: [PATCH 58/68] Finish TocEntry --- extract_msg/structures/toc_entry.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/extract_msg/structures/toc_entry.py b/extract_msg/structures/toc_entry.py index f44bdba9..4578643f 100644 --- a/extract_msg/structures/toc_entry.py +++ b/extract_msg/structures/toc_entry.py @@ -20,7 +20,7 @@ def __init__(self, reader : Optional[Union[bytes, BytesReader]] = None): self.__lindex = 0 self.__tymed = 0 self.__advf = 0 - self.__targetDevice = None + self.__targetDevice = DVTargetDevice() return if isinstance(reader, bytes): @@ -33,11 +33,10 @@ def __init__(self, reader : Optional[Union[bytes, BytesReader]] = None): self.__tymed = reader.readUnsignedInt() reader.read(12) self.__advf = reader.readUnsignedInt() - reader.read(4) - if targetDeviceSize == 0: - self.__targetDevice = None - else: - self.__targetDevice = DVTargetDevice(reader.read(targetDeviceSize)) + + # Based off the wording of the documentation, it seems like this can't + # actually be 0 bytes, so this should be fine. + self.__targetDevice = DVTargetDevice(reader.read(targetDeviceSize)) def __bytes__(self) -> bytes: @@ -45,7 +44,7 @@ def __bytes__(self) -> bytes: def toBytes(self) -> bytes: ret = bytes(self.__clipFormat) - td = bytes(self.__targetDevice) if self.__targetDevice else b'' + td = bytes(self.__targetDevice) ret += st.ST_LE_UI32.pack(len(td)) ret += st.ST_LE_UI32.pack(self.__aspect) ret += st.ST_LE_UI32.pack(self.__lindex) @@ -118,6 +117,10 @@ def _(self, val : int) -> None: self.__lindex = val + @property + def targetDevice(self) -> DVTargetDevice: + return self.__targetDevice + @property def tymed(self) -> int: return self.__tymed From 18d647ab3ef248dbba1ec6f8b4e31ad6ceee4e62 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 7 Nov 2023 09:34:10 -0800 Subject: [PATCH 59/68] Finish utils.inputToBytes --- extract_msg/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extract_msg/utils.py b/extract_msg/utils.py index c85ca367..7acdd817 100644 --- a/extract_msg/utils.py +++ b/extract_msg/utils.py @@ -586,8 +586,7 @@ def inputToBytes(obj : Any, encoding : str) -> bytes: if obj is None: return b'' if hasattr(obj, '__bytes__'): - try: - + return bytes(obj) raise ConversionError('Cannot convert to bytes.') From 2350f10d9c625f94524576fdbb88e95b4fadfc58 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 7 Nov 2023 09:48:04 -0800 Subject: [PATCH 60/68] Allow bytes conversion in OleWriter --- CHANGELOG.md | 2 ++ extract_msg/ole_writer.py | 24 ++++++++++++++++-------- extract_msg/utils.py | 4 ++-- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5fbc3ff..2d7a7cf5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,8 @@ * Moved `FixedLengthProperty.parseType` to the private API. This was not intended for external use anyways, so leaving it as public API didn't make sense. * Fixed check for type in `ContactAddressEntryID` being the wrong value. * Modified `inputToBytes` to support objects with the `__bytes__` method. If the method exists *and works* then it will be used as a last resort. +* Modified `OleWriter` to accept objects with a `__bytes__` method for the data to use for an entry. +* Added `__bytes__` method to `MSGFile`. This is equivalent to calling `MSGFile.exportBytes`. **v0.45.0** * BREAKING: Changed parsing of string multiple properties to remove the trailing null byte. This *will* cause the output of parsing them to differ. diff --git a/extract_msg/ole_writer.py b/extract_msg/ole_writer.py index 0606cfe5..efea94a3 100644 --- a/extract_msg/ole_writer.py +++ b/extract_msg/ole_writer.py @@ -10,7 +10,10 @@ import copy import re -from typing import Dict, Iterator, List, Optional, Tuple, TYPE_CHECKING +from typing import ( + Dict, Iterator, List, Optional, SupportsBytes, Tuple, TYPE_CHECKING, + Union + ) from . import constants from .constants import MSG_PATH @@ -195,7 +198,10 @@ def __modifyEntry(self, entry : DirectoryEntry, **kwargs): if entry.type is not DirectoryEntryType.STREAM: raise TypeError('Cannot set the data of a storage object.') if not isinstance(data, bytes): - raise ValueError('Data must be a bytes instance if set.') + try: + data = bytes(data) + except Exception: + raise ValueError('Data must be a bytes instance or convertable to bytes if set.') if clsid is not None: if not isinstance(clsid, bytes): @@ -554,7 +560,7 @@ def _writeDirectoryEntry(self, f, entry : DirectoryEntry) -> None: """ Writes the directory entry to the file f. """ - f.write(entry.toBytes()) + f.write(bytes(entry)) def _writeFinal(self, f) -> None: """ @@ -596,7 +602,7 @@ def _writeMini(self, f, entries : List[DirectoryEntry]) -> None: if self.__numMinifatSectors & 7: f.write((b'\x00' * 64) * (8 - (self.__numMinifatSectors & 7))) - def addEntry(self, path : MSG_PATH, data : Optional[bytes] = None, storage : bool = False, **kwargs) -> None: + def addEntry(self, path : MSG_PATH, data : Optional[Union[bytes, SupportsBytes]] = None, storage : bool = False, **kwargs) -> None: """ Adds an entry to the OleWriter instance at the path specified, adding storages with default settings where necessary. If the entry is not a @@ -604,7 +610,8 @@ def addEntry(self, path : MSG_PATH, data : Optional[bytes] = None, storage : boo :param path: The path to add the entry at. Must not contain a path part that is an already added stream. - :param data: The bytes for a stream. + :param data: The bytes for a stream or an object with the __bytes__ + method. :param storage: If True, the entry to add is a storage. Otherwise, the entry is a stream. :param clsid: The CLSID for the stream/storage. Must a a bytes instance @@ -637,7 +644,7 @@ def addEntry(self, path : MSG_PATH, data : Optional[bytes] = None, storage : boo else: _dir[path[-1]] = entry - def addOleEntry(self, path : MSG_PATH, entry : OleDirectoryEntry, data : Optional[bytes] = None) -> None: + def addOleEntry(self, path : MSG_PATH, entry : OleDirectoryEntry, data : Optional[Union[bytes, SupportsBytes]] = None) -> None: """ Uses the entry provided to add the data to the writer. @@ -677,7 +684,8 @@ def addOleEntry(self, path : MSG_PATH, entry : OleDirectoryEntry, data : Optiona newEntry.stateBits = entry.dwUserFlags # Finally, handle the data. - newEntry.data = data or b'' + data = data or b'' + newEntry.data = bytes(data) self.__dirEntryCount += 1 @@ -705,7 +713,7 @@ def editEntry(self, path : MSG_PATH, **kwargs) -> None: value to something other than None to set it. :param data: The data of a stream. Will error if used for something - other than a stream. + other than a stream. Must be bytes or convertable to bytes. :param clsid: The CLSID for the stream/storage. Must a a bytes instance that is 16 bytes long. :param creationTime: An 8 byte filetime int. Sets the creation time of diff --git a/extract_msg/utils.py b/extract_msg/utils.py index 7acdd817..6d194e3c 100644 --- a/extract_msg/utils.py +++ b/extract_msg/utils.py @@ -73,7 +73,7 @@ from html import escape as htmlEscape from typing import ( Any, AnyStr, Callable, Dict, Iterable, List, Optional, Sequence, - TypeVar, TYPE_CHECKING, Union + SupportsBytes, TypeVar, TYPE_CHECKING, Union ) from . import constants @@ -569,7 +569,7 @@ def htmlSanitize(inp : str) -> str: return inp -def inputToBytes(obj : Any, encoding : str) -> bytes: +def inputToBytes(obj : Union[bytes, None, str, SupportsBytes], encoding : str) -> bytes: """ Converts the input into bytes. From 8b05e439de5d91e6f91c2d7d39f7b5696f2a6ef1 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 7 Nov 2023 09:50:41 -0800 Subject: [PATCH 61/68] Update date in __init__ --- extract_msg/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extract_msg/__init__.py b/extract_msg/__init__.py index 4720b5d7..d59bf8ab 100644 --- a/extract_msg/__init__.py +++ b/extract_msg/__init__.py @@ -27,7 +27,7 @@ # along with this program. If not, see . __author__ = 'Destiny Peterson & Matthew Walker' -__date__ = '2023-09-30' +__date__ = '2023-11-07' __version__ = '0.46.0' __all__ = [ From c6f6a3cc91740c2da06269b06a172e3f1c119972 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 7 Nov 2023 09:55:21 -0800 Subject: [PATCH 62/68] Update extract_msg/structures/__init__.py --- extract_msg/structures/__init__.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/extract_msg/structures/__init__.py b/extract_msg/structures/__init__.py index c6d4ddeb..f52e9633 100644 --- a/extract_msg/structures/__init__.py +++ b/extract_msg/structures/__init__.py @@ -7,9 +7,13 @@ '_helpers', 'contact_link_entry', 'business_card', + 'cfoas', + 'contact_link_entry', 'dev_mode_a', + 'dv_target_device', 'entry_id', 'misc_id', + 'mon_stream', 'odt', 'ole_pres', 'ole_stream_struct', @@ -18,11 +22,13 @@ 'system_time', 'time_zone_definition', 'time_zone_struct', + 'toc_entry', 'tz_rule', ] from . import ( - _helpers, contact_link_entry, dev_mode_a, business_card, entry_id, misc_id, odt, - ole_pres, ole_stream_struct, recurrence_pattern, report_tag, - system_time, time_zone_definition, time_zone_struct, tz_rule + _helpers, business_card, cfoas, contact_link_entry, dev_mode_a, + dv_target_device, entry_id, misc_id, mon_stream, odt, ole_pres, + ole_stream_struct, recurrence_pattern, report_tag, system_time, + time_zone_definition, time_zone_struct, toc_entry, tz_rule ) \ No newline at end of file From 5494fbdc59c072bd7156c7bc21dc9b439f7fd304 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 7 Nov 2023 10:01:32 -0800 Subject: [PATCH 63/68] Add exports, remove unnecesary data, remove debug --- extract_msg/structures/dev_mode_a.py | 5 +++++ extract_msg/structures/mon_stream.py | 1 - extract_msg/structures/ole_pres.py | 11 ----------- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/extract_msg/structures/dev_mode_a.py b/extract_msg/structures/dev_mode_a.py index 08e27fa6..51c2027d 100644 --- a/extract_msg/structures/dev_mode_a.py +++ b/extract_msg/structures/dev_mode_a.py @@ -1,3 +1,8 @@ +__all__ = [ + 'DevModeA', +] + + import logging import struct diff --git a/extract_msg/structures/mon_stream.py b/extract_msg/structures/mon_stream.py index 220e82bb..e5c1253d 100644 --- a/extract_msg/structures/mon_stream.py +++ b/extract_msg/structures/mon_stream.py @@ -1,4 +1,3 @@ -# pyright: ignore[reportUnnecessaryIsInstance] __all__ = [ 'MonikerStream', ] diff --git a/extract_msg/structures/ole_pres.py b/extract_msg/structures/ole_pres.py index cbdeccbd..50c2181d 100644 --- a/extract_msg/structures/ole_pres.py +++ b/extract_msg/structures/ole_pres.py @@ -20,17 +20,6 @@ class OLEPresentationStream: """ [MS-OLEDS] OLEPresentationStream. """ - __ansiClipboardFormat : ClipboardFormatOrAnsiString - __targetDevice : Optional[DVTargetDevice] - __aspect : Union[int, DVAspect] - __lindex : int - __advf : Union[int, ADVF] - __width : int - __height : int - __data : bytes - __reserved2 : Optional[bytes] - __tocSignature : int - __tocEntries : List[TOCEntry] def __init__(self, data : bytes): reader = BytesReader(data) From 295987a1fc1d0f2337044e66d64677d042dc02c7 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Tue, 7 Nov 2023 10:07:22 -0800 Subject: [PATCH 64/68] Fix name for presentation streams being wrong --- extract_msg/attachments/custom_att_handler/custom_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extract_msg/attachments/custom_att_handler/custom_handler.py b/extract_msg/attachments/custom_att_handler/custom_handler.py index 571b8c03..3dcc64f9 100644 --- a/extract_msg/attachments/custom_att_handler/custom_handler.py +++ b/extract_msg/attachments/custom_att_handler/custom_handler.py @@ -129,5 +129,5 @@ def presentationObjs(self) -> Optional[Dict[int, OLEPresentationStream]]: return { int(x[1][-3:]): self.getStreamAs(x[-1], OLEPresentationStream) for x in self.attachment.listDir() - if x[0] == '__substg1.0_3701000D' and x[1].startswith('\x01OlePres') + if x[0] == '__substg1.0_3701000D' and x[1].startswith('\x02OlePres') } \ No newline at end of file From 69f9f2800affa285e180a7cab1bae2c274e69f7f Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 8 Nov 2023 06:09:45 -0800 Subject: [PATCH 65/68] Adjusted doc for export to remove abiguity --- extract_msg/msg_classes/msg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extract_msg/msg_classes/msg.py b/extract_msg/msg_classes/msg.py index f08b509a..9cfb948b 100644 --- a/extract_msg/msg_classes/msg.py +++ b/extract_msg/msg_classes/msg.py @@ -455,8 +455,8 @@ def export(self, path) -> None: to the properties of of an MSGFile object (or one of it's subclasses) will not be reflected in the saved file. - :param path: An IO device with a write method which accepts bytes or a - path-like object (including strings and pathlib.Path objects). + :param path: A path-like object (including strings and pathlib.Path + objects) or an IO device with a write method which accepts bytes. """ from ..ole_writer import OleWriter From 083736e2739b5ad7bc2ff5d47920576839c25f99 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 8 Nov 2023 06:10:56 -0800 Subject: [PATCH 66/68] update changelog --- CHANGELOG.md | 1 + extract_msg/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d7a7cf5..cc79d7ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ * [[TeamMsgExtractor #95](https://github.com/TeamMsgExtractor/msg-extractor/issues/95)] Adjusted the `overrideEncoding` property of `MSGFile` to allow automatic encoding detection. Simply set the property to the string `"chardet"` and, assuming the `chardet` module is installed, it will analyze a number of the strings to try and form a consensus about the encoding. This will *ignore* the specified encoding *only if* if successfully detects. Otherwise it will log a warning and fall back to the default behavior. * [[TeamMsgExtractor #387](https://github.com/TeamMsgExtractor/msg-extractor/issues/387)] Changed `extract_msg.utils.decodeRfc2047` to not throw decoding errors if the content given is not ASCII. * [[TeamMsgExtractor #387](https://github.com/TeamMsgExtractor/msg-extractor/issues/387)] Changed header parsing policy to `email.policy.compat32` to prevent partial parsing of quoted header fields. +* [[TeamMsgExtractor #388](https://github.com/TeamMsgExtractor/msg-extractor/issues/388)] Updated documentation of `MSGFile.export` to specify that updated fields on an `MSGFile` instance (and it's subclasses) will *not* be reflected in the result of the function. Many of the functions do use the newest version of a cached_property, but this is not one of them. * Removed methods deprecated in `v0.45.0`. * Changed the base class of `EntryID` from no base class to `abc.ABC`. * Added `position` property to `EntryID` to tell how many bytes were used to create the `EntryID`. diff --git a/extract_msg/__init__.py b/extract_msg/__init__.py index d59bf8ab..da194a14 100644 --- a/extract_msg/__init__.py +++ b/extract_msg/__init__.py @@ -11,7 +11,7 @@ # --- LICENSE.txt -------------------------------------------------------------- # -# Copyright 2013-2022 Matthew Walker and Destiny Peterson +# Copyright 2013-2023 Matthew Walker and Destiny Peterson # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -27,7 +27,7 @@ # along with this program. If not, see . __author__ = 'Destiny Peterson & Matthew Walker' -__date__ = '2023-11-07' +__date__ = '2023-11-08' __version__ = '0.46.0' __all__ = [ From 318c609d816804345a8a843d9cc8a54c06241715 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 8 Nov 2023 06:17:09 -0800 Subject: [PATCH 67/68] Update sphinx documentation --- docs/conf.py | 2 +- ...act_msg.attachments.custom_att_handler.rst | 8 +++ docs/extract_msg.msg_classes.rst | 8 +++ docs/extract_msg.structures.rst | 72 +++++++++++++++++++ 4 files changed, 89 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 29af9ff5..f786864e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,7 +12,7 @@ sys.path.insert(0, os.path.abspath('..')) __author__ = 'Destiny Peterson & Matthew Walker' -__version__ = '0.45.0' +__version__ = '0.46.0' __year__ = '2023' diff --git a/docs/extract_msg.attachments.custom_att_handler.rst b/docs/extract_msg.attachments.custom_att_handler.rst index dd0f515f..5788b238 100644 --- a/docs/extract_msg.attachments.custom_att_handler.rst +++ b/docs/extract_msg.attachments.custom_att_handler.rst @@ -12,6 +12,14 @@ extract\_msg.attachments.custom\_att\_handler.custom\_handler.py module :undoc-members: :show-inheritance: +extract\_msg.attachments.custom\_att\_handler.lnk\_obj\_att.py module +--------------------------------------------------------------------- + +.. automodule:: extract_msg.attachments.custom_att_handler.lnk_obj_att.py + :members: + :undoc-members: + :show-inheritance: + extract\_msg.attachments.custom\_att\_handler.outlook\_image\_dib.py module --------------------------------------------------------------------------- diff --git a/docs/extract_msg.msg_classes.rst b/docs/extract_msg.msg_classes.rst index 4ac15ca4..cc20b339 100644 --- a/docs/extract_msg.msg_classes.rst +++ b/docs/extract_msg.msg_classes.rst @@ -36,6 +36,14 @@ extract\_msg.msg\_classes.contact.py module :undoc-members: :show-inheritance: +extract\_msg.msg\_classes.journal.py module +------------------------------------------- + +.. automodule:: extract_msg.msg_classes.journal.py + :members: + :undoc-members: + :show-inheritance: + extract\_msg.msg\_classes.meeting\_cancellation.py module --------------------------------------------------------- diff --git a/docs/extract_msg.structures.rst b/docs/extract_msg.structures.rst index 101270e3..213e1928 100644 --- a/docs/extract_msg.structures.rst +++ b/docs/extract_msg.structures.rst @@ -12,6 +12,38 @@ extract\_msg.structures.business\_card.py module :undoc-members: :show-inheritance: +extract\_msg.structures.cfoas.py module +--------------------------------------- + +.. automodule:: extract_msg.structures.cfoas.py + :members: + :undoc-members: + :show-inheritance: + +extract\_msg.structures.contact\_link\_entry.py module +------------------------------------------------------ + +.. automodule:: extract_msg.structures.contact_link_entry.py + :members: + :undoc-members: + :show-inheritance: + +extract\_msg.structures.dev\_mode\_a.py module +---------------------------------------------- + +.. automodule:: extract_msg.structures.dev_mode_a.py + :members: + :undoc-members: + :show-inheritance: + +extract\_msg.structures.dv\_target\_device.py module +---------------------------------------------------- + +.. automodule:: extract_msg.structures.dv_target_device.py + :members: + :undoc-members: + :show-inheritance: + extract\_msg.structures.entry\_id.py module ------------------------------------------- @@ -28,6 +60,38 @@ extract\_msg.structures.misc\_id.py module :undoc-members: :show-inheritance: +extract\_msg.structures.mon\_stream.py module +--------------------------------------------- + +.. automodule:: extract_msg.structures.mon_stream.py + :members: + :undoc-members: + :show-inheritance: + +extract\_msg.structures.odt.py module +------------------------------------- + +.. automodule:: extract_msg.structures.odt.py + :members: + :undoc-members: + :show-inheritance: + +extract\_msg.structures.ole\_pres.py module +------------------------------------------- + +.. automodule:: extract_msg.structures.ole_pres.py + :members: + :undoc-members: + :show-inheritance: + +extract\_msg.structures.ole\_stream\_struct.py module +----------------------------------------------------- + +.. automodule:: extract_msg.structures.ole_stream_struct.py + :members: + :undoc-members: + :show-inheritance: + extract\_msg.structures.recurrence\_pattern.py module ----------------------------------------------------- @@ -68,6 +132,14 @@ extract\_msg.structures.time\_zone\_struct.py module :undoc-members: :show-inheritance: +extract\_msg.structures.toc\_entry.py module +-------------------------------------------- + +.. automodule:: extract_msg.structures.toc_entry.py + :members: + :undoc-members: + :show-inheritance: + extract\_msg.structures.tz\_rule.py module ------------------------------------------ From 7debebfcbf3afde71023816a0c61dafde9fb7620 Mon Sep 17 00:00:00 2001 From: TheElementalOfDestruction Date: Wed, 8 Nov 2023 06:25:06 -0800 Subject: [PATCH 68/68] Add missing `__bytes__` methods --- extract_msg/structures/dev_mode_a.py | 3 +++ extract_msg/structures/tz_rule.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/extract_msg/structures/dev_mode_a.py b/extract_msg/structures/dev_mode_a.py index 51c2027d..4deb3d4d 100644 --- a/extract_msg/structures/dev_mode_a.py +++ b/extract_msg/structures/dev_mode_a.py @@ -114,6 +114,9 @@ def __init__(self, data : Optional[bytes] = None): def __bool__(self) -> bool: return self.__valid + def __bytes__(self) -> bytes: + return self.toBytes() + def toBytes(self) -> bytes: return self.PARSE_STRUCT.pack( self.__deviceName, diff --git a/extract_msg/structures/tz_rule.py b/extract_msg/structures/tz_rule.py index bc2e2169..740793df 100644 --- a/extract_msg/structures/tz_rule.py +++ b/extract_msg/structures/tz_rule.py @@ -55,6 +55,9 @@ def __init__(self, data : Optional[bytes] = None): self.__standardDate = SystemTime(reader.read(16)) self.__daylightDate = SystemTime(reader.read(16)) + def __bytes__(self) -> bytes: + return self.toBytes() + def toBytes(self) -> bytes: return self.__struct.pack(self.__majorVersion, self.__minorVersion,