From 7bae818a14e368f909317fd386dfe4526b643b4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Thu, 29 Apr 2021 16:40:02 -0700 Subject: [PATCH 01/11] Add JSONEncoder and tests --- .../azure-core/azure/core/serialization.py | 103 +++++- sdk/core/azure-core/dev_requirements.txt | 1 + .../azure-core/tests/test_serialization.py | 350 +++++++++++++++++- 3 files changed, 449 insertions(+), 5 deletions(-) diff --git a/sdk/core/azure-core/azure/core/serialization.py b/sdk/core/azure-core/azure/core/serialization.py index c3422efa0c27..41fab5d011a9 100644 --- a/sdk/core/azure-core/azure/core/serialization.py +++ b/sdk/core/azure-core/azure/core/serialization.py @@ -4,16 +4,20 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- +import base64 +import datetime +from json import JSONEncoder __all__ = ["NULL"] + class _Null(object): - """To create a Falsy object - """ + """To create a Falsy object""" + def __bool__(self): return False - __nonzero__ = __bool__ # Python2 compatibility + __nonzero__ = __bool__ # Python2 compatibility NULL = _Null() @@ -21,3 +25,96 @@ def __bool__(self): A falsy sentinel object which is supposed to be used to specify attributes with no data. This gets serialized to `null` on the wire. """ + + +def iso_timedelta(value): + """Represent a timedelta in ISO 8601 format. + + Function from the Tin Can Python project: https://github.com/RusticiSoftware/TinCanPython + """ + + # split seconds to larger units + seconds = value.total_seconds() + minutes, seconds = divmod(seconds, 60) + hours, minutes = divmod(minutes, 60) + days, hours = divmod(hours, 24) + + days, hours, minutes = list(map(int, (days, hours, minutes))) + seconds = round(seconds, 6) + + # build date + date = '' + if days: + date = '%sD' % days + + # build time + time = 'T' + + # hours + bigger_exists = date or hours + if bigger_exists: + time += '{:02}H'.format(hours) + + # minutes + bigger_exists = bigger_exists or minutes + if bigger_exists: + time += '{:02}M'.format(minutes) + + # seconds + if seconds.is_integer(): + seconds = '{:02}'.format(int(seconds)) + else: + # 9 chars long w/leading 0, 6 digits after decimal + seconds = '%09.6f' % seconds + # remove trailing zeros + seconds = seconds.rstrip('0') + + time += '{}S'.format(seconds) + + return 'P' + date + time + + +class UTC(datetime.tzinfo): + """Time Zone info for handling UTC""" + + def utcoffset(self, dt): + """UTF offset for UTC is 0.""" + return datetime.timedelta(0) + + def tzname(self, dt): + """Timestamp representation.""" + return "Z" + + def dst(self, dt): + """No daylight saving for UTC.""" + return datetime.timedelta(hours=1) + + +try: + from datetime import timezone + + TZ_UTC = timezone.utc # type: ignore +except ImportError: + TZ_UTC = UTC() # type: ignore + + +class ComplexEncoder(JSONEncoder): + """A JSON encoder that's capable of serializing datetime objects and bytes.""" + + def default(self, o): # pylint: disable=too-many-return-statements + try: + return super(ComplexEncoder, self).default(o) + except TypeError: + o_type = type(o) + + if o_type is datetime.date or o_type is datetime.time: + return o.isoformat() + if o_type is datetime.datetime: + if not o.tzinfo: # astimezone() fails for naive times in Python 2.7 + return o.replace(tzinfo=TZ_UTC).isoformat() + return o.astimezone(TZ_UTC).isoformat() + if o_type is datetime.timedelta: + return iso_timedelta(o) + if o_type is bytes or o_type is bytearray: + return base64.b64encode(o).decode() + return super(ComplexEncoder, self).default(o) diff --git a/sdk/core/azure-core/dev_requirements.txt b/sdk/core/azure-core/dev_requirements.txt index 6297f8880d39..b88530ecd335 100644 --- a/sdk/core/azure-core/dev_requirements.txt +++ b/sdk/core/azure-core/dev_requirements.txt @@ -1,5 +1,6 @@ trio; python_version >= '3.5' aiohttp>=3.0; python_version >= '3.5' +isodate>=0.6.0 typing_extensions>=3.7.2 opencensus>=0.6.0 opencensus-ext-azure diff --git a/sdk/core/azure-core/tests/test_serialization.py b/sdk/core/azure-core/tests/test_serialization.py index 7ac58850cd91..87c11c5d3de4 100644 --- a/sdk/core/azure-core/tests/test_serialization.py +++ b/sdk/core/azure-core/tests/test_serialization.py @@ -2,10 +2,356 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # ------------------------------------ +import base64 +from datetime import date, timedelta, time, datetime +from enum import Enum +import json + +from azure.core.serialization import ComplexEncoder, NULL +import isodate +import pytest + + +def _expand_value(obj): + try: + try: + return obj.to_dict() + + except AttributeError: + if isinstance(obj, Enum): + return obj.value + elif isinstance(obj, list): + return [_expand_value(item) for item in obj] + elif isinstance(obj, dict): + return _expand_dict(obj) + else: + return _expand_dict(vars(obj)) + + except TypeError: + return obj + + +def _expand_dict(d): + return dict((key, _expand_value(value)) for key, value in d.items()) + + +class SerializerMixin(object): + """Mixin that provides methods for representing a model as a dictionary""" + + def to_dict(self): + return _expand_value(vars(self)) -from azure.core.serialization import NULL def test_NULL_is_falsy(): assert NULL is not False assert bool(NULL) is False - assert NULL is NULL \ No newline at end of file + assert NULL is NULL + +@pytest.fixture +def json_dumps_with_encoder(): + def func(obj): + return json.dumps(obj, cls=ComplexEncoder) + return func + +def test_bytes(json_dumps_with_encoder): + test_bytes = b"mybytes" + result = json.loads(json_dumps_with_encoder(test_bytes)) + try: + assert base64.b64decode(result) == test_bytes # Python 3 + except TypeError: + assert result == test_bytes # Python 2.7 + +def test_byte_array_ascii(json_dumps_with_encoder): + test_byte_array = bytearray("mybytes", "ascii") + result = json.loads(json_dumps_with_encoder(test_byte_array)) + assert base64.b64decode(result) == test_byte_array + +def test_byte_array_utf8(json_dumps_with_encoder): + test_byte_array = bytearray("mybytes", "utf-8") + result = json.loads(json_dumps_with_encoder(test_byte_array)) + assert base64.b64decode(result) == test_byte_array + +def test_byte_array_utf16(json_dumps_with_encoder): + test_byte_array = bytearray("mybytes", "utf-16") + result = json.loads(json_dumps_with_encoder(test_byte_array)) + assert base64.b64decode(result) == test_byte_array + +def test_dictionary_basic(json_dumps_with_encoder): + test_obj = { + "string": "myid", + "number": 42, + "boolean": True, + "list_of_string": [1, 2, 3], + "dictionary_of_number": {"pi": 3.14}, + } + complex_serialized = json_dumps_with_encoder(test_obj) + assert json.dumps(test_obj) == complex_serialized + assert json.loads(complex_serialized) == test_obj + +def test_model_basic(json_dumps_with_encoder): + class BasicModel(SerializerMixin): + def __init__(self): + self.string = "myid" + self.number = 42 + self.boolean = True + self.list_of_ints = [1, 2, 3] + self.dictionary_of_number = {"pi": 3.14} + + expected = BasicModel() + expected_dict = { + "string": "myid", + "number": 42, + "boolean": True, + "list_of_ints": [1, 2, 3], + "dictionary_of_number": {"pi": 3.14}, + } + assert json.loads(json_dumps_with_encoder(expected.to_dict())) == expected_dict + +def test_dictionary_datetime(json_dumps_with_encoder): + test_obj = { + "timedelta": timedelta(1), + "date": date(2021, 5, 12), + "datetime": isodate.parse_datetime('2012-02-24T00:53:52.780Z'), + "time": time(11,12,13), + } + expected = { + "timedelta": "P1DT00H00M00S", + "date": "2021-05-12", + "datetime": '2012-02-24T00:53:52.780000+00:00', + 'time': '11:12:13', + } + assert json.loads(json_dumps_with_encoder(test_obj)) == expected + +def test_model_datetime(json_dumps_with_encoder): + class DatetimeModel(SerializerMixin): + def __init__(self): + self.timedelta = timedelta(1) + self.date = date(2021, 5, 12) + self.datetime = isodate.parse_datetime('2012-02-24T00:53:52.780Z') + self.time = time(11,12,13) + + expected = DatetimeModel() + expected_dict = { + "timedelta": "P1DT00H00M00S", + "date": "2021-05-12", + "datetime": '2012-02-24T00:53:52.780000+00:00', + 'time': '11:12:13', + } + assert json.loads(json_dumps_with_encoder(expected.to_dict())) == expected_dict + +def test_serialize_datetime(json_dumps_with_encoder): + + date_obj = isodate.parse_datetime('2015-01-01T00:00:00') + date_str = json_dumps_with_encoder(date_obj) + + assert date_str == '"2015-01-01T00:00:00+00:00"' + + date_obj = isodate.parse_datetime('1999-12-31T23:59:59-12:00') + date_str = json_dumps_with_encoder(date_obj) + + assert date_str == '"2000-01-01T11:59:59+00:00"' + + date_obj = isodate.parse_datetime("2015-06-01T16:10:08.0121-07:00") + date_str = json_dumps_with_encoder(date_obj) + + assert date_str == '"2015-06-01T23:10:08.012100+00:00"' + + date_obj = datetime.min + date_str = json_dumps_with_encoder(date_obj) + assert date_str == '"0001-01-01T00:00:00+00:00"' + + date_obj = datetime.max + date_str = json_dumps_with_encoder(date_obj) + assert date_str == '"9999-12-31T23:59:59.999999+00:00"' + + date_obj = isodate.parse_datetime('2012-02-24T00:53:52.000001Z') + date_str = json_dumps_with_encoder(date_obj) + assert date_str == '"2012-02-24T00:53:52.000001+00:00"' + + date_obj = isodate.parse_datetime('2012-02-24T00:53:52.780Z') + date_str = json_dumps_with_encoder(date_obj) + assert date_str == '"2012-02-24T00:53:52.780000+00:00"' + +def test_serialize_time(json_dumps_with_encoder): + + time_str = json_dumps_with_encoder(time(11,22,33)) + assert time_str == '"11:22:33"' + + time_str = json_dumps_with_encoder(time(11,22,33,444444)) + assert time_str == '"11:22:33.444444"' + +class BasicEnum(Enum): + val = "Basic" + +class StringEnum(str, Enum): + val = "string" + +class IntEnum(int, Enum): + val = 1 + +class FloatEnum(float, Enum): + val = 1.5 + +def test_dictionary_enum(json_dumps_with_encoder): + test_obj = { + "basic": BasicEnum.val + } + with pytest.raises(TypeError): + json_dumps_with_encoder(test_obj) + + test_obj = { + "basic": BasicEnum.val.value, + "string": StringEnum.val.value, + "int": IntEnum.val.value, + "float": FloatEnum.val.value + } + expected = { + "basic": "Basic", + "string": "string", + "int": 1, + "float": 1.5 + } + serialized = json_dumps_with_encoder(test_obj) + assert json.dumps(test_obj) == serialized + assert json.loads(serialized) == expected + +def test_model_enum(json_dumps_with_encoder): + class BasicEnumModel: + def __init__(self): + self.basic = BasicEnum.val + + with pytest.raises(TypeError): + json_dumps_with_encoder(BasicEnumModel()) + + class EnumModel(SerializerMixin): + def __init__(self): + self.basic = BasicEnum.val.value + self.string = StringEnum.val + self.int = IntEnum.val + self.float = FloatEnum.val + + expected = EnumModel() + expected_dict = { + "basic": "Basic", + "string": "string", + "int": 1, + "float": 1.5 + } + assert json.loads(json_dumps_with_encoder(expected.to_dict())) == expected_dict + +def test_dictionary_none(json_dumps_with_encoder): + assert json_dumps_with_encoder(None) == json.dumps(None) + test_obj = { + "entry": None + } + assert json.loads(json_dumps_with_encoder(test_obj)) == test_obj + +def test_model_none(json_dumps_with_encoder): + class NoneModel(SerializerMixin): + def __init__(self): + self.entry = None + + expected = NoneModel() + expected_dict = {"entry": None} + assert json.loads(json_dumps_with_encoder(expected.to_dict())) == expected_dict + +def test_dictionary_empty_collections(json_dumps_with_encoder): + test_obj = { + "dictionary": {}, + "list": [], + # "tuple": (), json represents tuples as lists, so this won't round-trip with json.loads + # "set": set(), json can't serialize sets. should we? + } + + assert json.dumps(test_obj) == json_dumps_with_encoder(test_obj) + assert json.loads(json_dumps_with_encoder(test_obj)) == test_obj + +def test_model_empty_collections(json_dumps_with_encoder): + class EmptyCollectionsModel(SerializerMixin): + def __init__(self): + self.dictionary = {} + self.list = [] + # self.tuple = (), json represents tuples as lists, so this won't round-trip with json.loads + # self.set = set(), json can't serialize sets. should we? + + expected = EmptyCollectionsModel() + expected_dict = { + "dictionary": {}, + "list": [], + } + assert json.loads(json_dumps_with_encoder(expected.to_dict())) == expected_dict + +def test_model_inheritance(json_dumps_with_encoder): + class ParentModel(SerializerMixin): + def __init__(self): + self.parent = "parent" + + class ChildModel(ParentModel): + def __init__(self): + super(ChildModel, self).__init__() + self.child = "child" + + expected = ChildModel() + expected_dict = { + "parent": "parent", + "child": "child", + } + assert json.loads(json_dumps_with_encoder(expected.to_dict())) == expected_dict + +def test_model_recursion(json_dumps_with_encoder): + class RecursiveModel(SerializerMixin): + def __init__(self): + self.name = "it's me!" + self.list_of_me = None + self.dict_of_me = None + self.dict_of_list_of_me = None + self.list_of_dict_of_me = None + + expected = RecursiveModel() + expected.list_of_me = [RecursiveModel()] + expected.dict_of_me = {"me": RecursiveModel()} + expected.dict_of_list_of_me = {"many mes": [RecursiveModel()]} + expected.list_of_dict_of_me = [{"me": RecursiveModel()}] + expected_dict = { + "name": "it's me!", + "list_of_me": [ + { + "name": "it's me!", + "list_of_me": None, + "dict_of_me": None, + "dict_of_list_of_me": None, + "list_of_dict_of_me": None + } + ], + "dict_of_me": { + "me": { + "name": "it's me!", + "list_of_me": None, + "dict_of_me": None, + "dict_of_list_of_me": None, + "list_of_dict_of_me": None + } + }, + "dict_of_list_of_me": { + "many mes": [ + { + "name": "it's me!", + "list_of_me": None, + "dict_of_me": None, + "dict_of_list_of_me": None, + "list_of_dict_of_me": None + } + ] + }, + "list_of_dict_of_me": [ + {"me": { + "name": "it's me!", + "list_of_me": None, + "dict_of_me": None, + "dict_of_list_of_me": None, + "list_of_dict_of_me": None + } + } + ] + } + assert json.loads(json_dumps_with_encoder(expected.to_dict())) == expected_dict From 59903df07a372179cb3a5e369f3eb8e77b736557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Thu, 8 Jul 2021 10:27:11 -0700 Subject: [PATCH 02/11] Use _FixedOffset, run black --- .../azure-core/azure/core/serialization.py | 40 ++++++------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/sdk/core/azure-core/azure/core/serialization.py b/sdk/core/azure-core/azure/core/serialization.py index 41fab5d011a9..352c4be4c0f3 100644 --- a/sdk/core/azure-core/azure/core/serialization.py +++ b/sdk/core/azure-core/azure/core/serialization.py @@ -8,6 +8,8 @@ import datetime from json import JSONEncoder +from ._utils import _FixedOffset + __all__ = ["NULL"] @@ -43,51 +45,35 @@ def iso_timedelta(value): seconds = round(seconds, 6) # build date - date = '' + date = "" if days: - date = '%sD' % days + date = "%sD" % days # build time - time = 'T' + time = "T" # hours bigger_exists = date or hours if bigger_exists: - time += '{:02}H'.format(hours) + time += "{:02}H".format(hours) # minutes bigger_exists = bigger_exists or minutes if bigger_exists: - time += '{:02}M'.format(minutes) + time += "{:02}M".format(minutes) # seconds if seconds.is_integer(): - seconds = '{:02}'.format(int(seconds)) + seconds = "{:02}".format(int(seconds)) else: # 9 chars long w/leading 0, 6 digits after decimal - seconds = '%09.6f' % seconds + seconds = "%09.6f" % seconds # remove trailing zeros - seconds = seconds.rstrip('0') - - time += '{}S'.format(seconds) - - return 'P' + date + time - - -class UTC(datetime.tzinfo): - """Time Zone info for handling UTC""" - - def utcoffset(self, dt): - """UTF offset for UTC is 0.""" - return datetime.timedelta(0) + seconds = seconds.rstrip("0") - def tzname(self, dt): - """Timestamp representation.""" - return "Z" + time += "{}S".format(seconds) - def dst(self, dt): - """No daylight saving for UTC.""" - return datetime.timedelta(hours=1) + return "P" + date + time try: @@ -95,7 +81,7 @@ def dst(self, dt): TZ_UTC = timezone.utc # type: ignore except ImportError: - TZ_UTC = UTC() # type: ignore + TZ_UTC = _FixedOffset(0) # type: ignore class ComplexEncoder(JSONEncoder): From 6c161e5cc5c0a82dafe470a26f038cffe0aeebe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Mon, 12 Jul 2021 18:48:11 -0700 Subject: [PATCH 03/11] Reference new utils location --- sdk/core/azure-core/azure/core/serialization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/serialization.py b/sdk/core/azure-core/azure/core/serialization.py index 352c4be4c0f3..9d5ea544fb0c 100644 --- a/sdk/core/azure-core/azure/core/serialization.py +++ b/sdk/core/azure-core/azure/core/serialization.py @@ -8,7 +8,7 @@ import datetime from json import JSONEncoder -from ._utils import _FixedOffset +from .utils._utils import _FixedOffset __all__ = ["NULL"] From 833457a49cfed5ac13363e5c5f24574789eb0866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Fri, 16 Jul 2021 15:56:28 -0700 Subject: [PATCH 04/11] Reorganize conditional logic --- .../azure-core/azure/core/serialization.py | 65 +++++++++++-------- 1 file changed, 38 insertions(+), 27 deletions(-) diff --git a/sdk/core/azure-core/azure/core/serialization.py b/sdk/core/azure-core/azure/core/serialization.py index 9d5ea544fb0c..14aa70a0b7f0 100644 --- a/sdk/core/azure-core/azure/core/serialization.py +++ b/sdk/core/azure-core/azure/core/serialization.py @@ -29,13 +29,14 @@ def __bool__(self): """ -def iso_timedelta(value): - """Represent a timedelta in ISO 8601 format. +def timedelta_as_isostr(value): + # type: (datetime.timedelta) -> str + """Converts a datetime.timedelta object into an ISO 8601 formatted string, e.g. 'P4DT12H30M05S' - Function from the Tin Can Python project: https://github.com/RusticiSoftware/TinCanPython + Function adapted from the Tin Can Python project: https://github.com/RusticiSoftware/TinCanPython """ - # split seconds to larger units + # Split seconds to larger units seconds = value.total_seconds() minutes, seconds = divmod(seconds, 60) hours, minutes = divmod(minutes, 60) @@ -44,32 +45,35 @@ def iso_timedelta(value): days, hours, minutes = list(map(int, (days, hours, minutes))) seconds = round(seconds, 6) - # build date + # Build date date = "" if days: date = "%sD" % days - # build time + # Build time time = "T" - # hours + # Hours bigger_exists = date or hours if bigger_exists: time += "{:02}H".format(hours) - # minutes + # Minutes bigger_exists = bigger_exists or minutes if bigger_exists: time += "{:02}M".format(minutes) - # seconds - if seconds.is_integer(): - seconds = "{:02}".format(int(seconds)) - else: - # 9 chars long w/leading 0, 6 digits after decimal - seconds = "%09.6f" % seconds - # remove trailing zeros - seconds = seconds.rstrip("0") + # Seconds + try: + if seconds.is_integer(): + seconds = "{:02}".format(int(seconds)) + else: + # 9 chars long w/ leading 0, 6 digits after decimal + seconds = "%09.6f" % seconds + # Remove trailing zeros + seconds = seconds.rstrip("0") + except AttributeError: # int.is_integer() raises on Python 2.7 + seconds = "{:02}".format(seconds) time += "{}S".format(seconds) @@ -91,16 +95,23 @@ def default(self, o): # pylint: disable=too-many-return-statements try: return super(ComplexEncoder, self).default(o) except TypeError: - o_type = type(o) - - if o_type is datetime.date or o_type is datetime.time: - return o.isoformat() - if o_type is datetime.datetime: - if not o.tzinfo: # astimezone() fails for naive times in Python 2.7 - return o.replace(tzinfo=TZ_UTC).isoformat() - return o.astimezone(TZ_UTC).isoformat() - if o_type is datetime.timedelta: - return iso_timedelta(o) - if o_type is bytes or o_type is bytearray: + if isinstance(o, (bytes, bytearray)): return base64.b64encode(o).decode() + try: + # First try datetime.datetime + if hasattr(o, "year") and hasattr(o, "hour"): + if not o.tzinfo: # astimezone() fails for naive times in Python 2.7 + return o.replace(tzinfo=TZ_UTC).isoformat() + return o.astimezone(TZ_UTC).isoformat() + # Next try datetime.date or datetime.time + else: + return o.isoformat() + except AttributeError: + pass + # Last, try datetime.timedelta + try: + return timedelta_as_isostr(o) + except AttributeError: + # This will be raised when it hits value.total_seconds in the method above + pass return super(ComplexEncoder, self).default(o) From 3d35a008a4ac13648703ddd17f20212f69a129e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Fri, 16 Jul 2021 18:55:12 -0700 Subject: [PATCH 05/11] Update tests --- sdk/core/azure-core/dev_requirements.txt | 1 - .../azure-core/tests/test_serialization.py | 55 +++++++++++++++---- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/sdk/core/azure-core/dev_requirements.txt b/sdk/core/azure-core/dev_requirements.txt index b88530ecd335..6297f8880d39 100644 --- a/sdk/core/azure-core/dev_requirements.txt +++ b/sdk/core/azure-core/dev_requirements.txt @@ -1,6 +1,5 @@ trio; python_version >= '3.5' aiohttp>=3.0; python_version >= '3.5' -isodate>=0.6.0 typing_extensions>=3.7.2 opencensus>=0.6.0 opencensus-ext-azure diff --git a/sdk/core/azure-core/tests/test_serialization.py b/sdk/core/azure-core/tests/test_serialization.py index 87c11c5d3de4..d65357ae3075 100644 --- a/sdk/core/azure-core/tests/test_serialization.py +++ b/sdk/core/azure-core/tests/test_serialization.py @@ -3,12 +3,11 @@ # Licensed under the MIT License. # ------------------------------------ import base64 -from datetime import date, timedelta, time, datetime +from datetime import date, datetime, time, timedelta, tzinfo from enum import Enum import json from azure.core.serialization import ComplexEncoder, NULL -import isodate import pytest @@ -35,6 +34,10 @@ def _expand_dict(d): return dict((key, _expand_value(value)) for key, value in d.items()) +class DatetimeSubclass(datetime): + """datetime.datetime subclass that tests datetimes without a type() of datetime.datetime""" + + class SerializerMixin(object): """Mixin that provides methods for representing a model as a dictionary""" @@ -42,6 +45,32 @@ def to_dict(self): return _expand_value(vars(self)) +class NegativeUtcOffset(tzinfo): + """tzinfo class with UTC offset of -12 hours""" + _offset = timedelta(seconds=-43200) + _dst = timedelta(0) + _name = "-1200" + def utcoffset(self, dt): + return self.__class__._offset + def dst(self, dt): + return self.__class__._dst + def tzname(self, dt): + return self.__class__._name + + +class PositiveUtcOffset(tzinfo): + """tzinfo class with UTC offset of +12 hours""" + _offset = timedelta(seconds=43200) + _dst = timedelta(0) + _name = "+1200" + def utcoffset(self, dt): + return self.__class__._offset + def dst(self, dt): + return self.__class__._dst + def tzname(self, dt): + return self.__class__._name + + def test_NULL_is_falsy(): assert NULL is not False assert bool(NULL) is False @@ -111,7 +140,7 @@ def test_dictionary_datetime(json_dumps_with_encoder): test_obj = { "timedelta": timedelta(1), "date": date(2021, 5, 12), - "datetime": isodate.parse_datetime('2012-02-24T00:53:52.780Z'), + "datetime": datetime.strptime('2012-02-24T00:53:52.780Z', "%Y-%m-%dT%H:%M:%S.%fZ"), "time": time(11,12,13), } expected = { @@ -127,7 +156,7 @@ class DatetimeModel(SerializerMixin): def __init__(self): self.timedelta = timedelta(1) self.date = date(2021, 5, 12) - self.datetime = isodate.parse_datetime('2012-02-24T00:53:52.780Z') + self.datetime = datetime.strptime('2012-02-24T00:53:52.780Z', "%Y-%m-%dT%H:%M:%S.%fZ") self.time = time(11,12,13) expected = DatetimeModel() @@ -141,20 +170,20 @@ def __init__(self): def test_serialize_datetime(json_dumps_with_encoder): - date_obj = isodate.parse_datetime('2015-01-01T00:00:00') + date_obj = datetime.strptime('2015-01-01T00:00:00', "%Y-%m-%dT%H:%M:%S") date_str = json_dumps_with_encoder(date_obj) assert date_str == '"2015-01-01T00:00:00+00:00"' - date_obj = isodate.parse_datetime('1999-12-31T23:59:59-12:00') + date_obj = datetime.strptime('1999-12-31T23:59:59', "%Y-%m-%dT%H:%M:%S").replace(tzinfo=NegativeUtcOffset()) date_str = json_dumps_with_encoder(date_obj) assert date_str == '"2000-01-01T11:59:59+00:00"' - date_obj = isodate.parse_datetime("2015-06-01T16:10:08.0121-07:00") + date_obj = datetime.strptime("2015-06-01T16:10:08.0121", "%Y-%m-%dT%H:%M:%S.%f").replace(tzinfo=PositiveUtcOffset()) date_str = json_dumps_with_encoder(date_obj) - assert date_str == '"2015-06-01T23:10:08.012100+00:00"' + assert date_str == '"2015-06-01T04:10:08.012100+00:00"' date_obj = datetime.min date_str = json_dumps_with_encoder(date_obj) @@ -164,11 +193,17 @@ def test_serialize_datetime(json_dumps_with_encoder): date_str = json_dumps_with_encoder(date_obj) assert date_str == '"9999-12-31T23:59:59.999999+00:00"' - date_obj = isodate.parse_datetime('2012-02-24T00:53:52.000001Z') + date_obj = datetime.strptime('2012-02-24T00:53:52.000001Z', "%Y-%m-%dT%H:%M:%S.%fZ") date_str = json_dumps_with_encoder(date_obj) assert date_str == '"2012-02-24T00:53:52.000001+00:00"' - date_obj = isodate.parse_datetime('2012-02-24T00:53:52.780Z') + date_obj = datetime.strptime('2012-02-24T00:53:52.780Z', "%Y-%m-%dT%H:%M:%S.%fZ") + date_str = json_dumps_with_encoder(date_obj) + assert date_str == '"2012-02-24T00:53:52.780000+00:00"' + +def test_serialize_datetime_subclass(json_dumps_with_encoder): + + date_obj = DatetimeSubclass.strptime('2012-02-24T00:53:52.780Z', "%Y-%m-%dT%H:%M:%S.%fZ") date_str = json_dumps_with_encoder(date_obj) assert date_str == '"2012-02-24T00:53:52.780000+00:00"' From ff843f96b70704aa7b5f41aab8ec82a0ef5956db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Thu, 22 Jul 2021 15:42:44 -0700 Subject: [PATCH 06/11] Pylint and mypy --- .../azure-core/azure/core/serialization.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/sdk/core/azure-core/azure/core/serialization.py b/sdk/core/azure-core/azure/core/serialization.py index 14aa70a0b7f0..6e2a63920b05 100644 --- a/sdk/core/azure-core/azure/core/serialization.py +++ b/sdk/core/azure-core/azure/core/serialization.py @@ -5,11 +5,14 @@ # license information. # -------------------------------------------------------------------------- import base64 -import datetime from json import JSONEncoder +from typing import TYPE_CHECKING from .utils._utils import _FixedOffset +if TYPE_CHECKING: + from datetime import timedelta + __all__ = ["NULL"] @@ -30,7 +33,7 @@ def __bool__(self): def timedelta_as_isostr(value): - # type: (datetime.timedelta) -> str + # type: (timedelta) -> str """Converts a datetime.timedelta object into an ISO 8601 formatted string, e.g. 'P4DT12H30M05S' Function adapted from the Tin Can Python project: https://github.com/RusticiSoftware/TinCanPython @@ -66,16 +69,16 @@ def timedelta_as_isostr(value): # Seconds try: if seconds.is_integer(): - seconds = "{:02}".format(int(seconds)) + seconds_string = "{:02}".format(int(seconds)) else: # 9 chars long w/ leading 0, 6 digits after decimal - seconds = "%09.6f" % seconds + seconds_string = "%09.6f" % seconds # Remove trailing zeros - seconds = seconds.rstrip("0") + seconds_string = seconds_string.rstrip("0") except AttributeError: # int.is_integer() raises on Python 2.7 - seconds = "{:02}".format(seconds) + seconds_string = "{:02}".format(seconds) - time += "{}S".format(seconds) + time += "{}S".format(seconds_string) return "P" + date + time @@ -104,8 +107,7 @@ def default(self, o): # pylint: disable=too-many-return-statements return o.replace(tzinfo=TZ_UTC).isoformat() return o.astimezone(TZ_UTC).isoformat() # Next try datetime.date or datetime.time - else: - return o.isoformat() + return o.isoformat() except AttributeError: pass # Last, try datetime.timedelta From 62e61ea3fc6d0961ff9e82568c3dc667b3dd938d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Mon, 26 Jul 2021 13:44:50 -0700 Subject: [PATCH 07/11] Private _timedelta_as_isostr --- sdk/core/azure-core/azure/core/serialization.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/azure/core/serialization.py b/sdk/core/azure-core/azure/core/serialization.py index 6e2a63920b05..db63a708a01b 100644 --- a/sdk/core/azure-core/azure/core/serialization.py +++ b/sdk/core/azure-core/azure/core/serialization.py @@ -32,7 +32,7 @@ def __bool__(self): """ -def timedelta_as_isostr(value): +def _timedelta_as_isostr(value): # type: (timedelta) -> str """Converts a datetime.timedelta object into an ISO 8601 formatted string, e.g. 'P4DT12H30M05S' @@ -75,7 +75,7 @@ def timedelta_as_isostr(value): seconds_string = "%09.6f" % seconds # Remove trailing zeros seconds_string = seconds_string.rstrip("0") - except AttributeError: # int.is_integer() raises on Python 2.7 + except AttributeError: # int.is_integer() raises seconds_string = "{:02}".format(seconds) time += "{}S".format(seconds_string) @@ -112,7 +112,7 @@ def default(self, o): # pylint: disable=too-many-return-statements pass # Last, try datetime.timedelta try: - return timedelta_as_isostr(o) + return _timedelta_as_isostr(o) except AttributeError: # This will be raised when it hits value.total_seconds in the method above pass From cdac229d99056ae011b43f23a472f5dcef86455b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Fri, 30 Jul 2021 10:48:45 -0700 Subject: [PATCH 08/11] Public encoder, clearer comment --- sdk/core/azure-core/azure/core/serialization.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/serialization.py b/sdk/core/azure-core/azure/core/serialization.py index db63a708a01b..69163678a8a1 100644 --- a/sdk/core/azure-core/azure/core/serialization.py +++ b/sdk/core/azure-core/azure/core/serialization.py @@ -13,7 +13,7 @@ if TYPE_CHECKING: from datetime import timedelta -__all__ = ["NULL"] +__all__ = ["NULL", "ComplexEncoder"] class _Null(object): @@ -103,7 +103,8 @@ def default(self, o): # pylint: disable=too-many-return-statements try: # First try datetime.datetime if hasattr(o, "year") and hasattr(o, "hour"): - if not o.tzinfo: # astimezone() fails for naive times in Python 2.7 + # astimezone() fails for naive times in Python 2.7, so make make sure o is aware (tzinfo is set) + if not o.tzinfo: return o.replace(tzinfo=TZ_UTC).isoformat() return o.astimezone(TZ_UTC).isoformat() # Next try datetime.date or datetime.time From 264fc801d7dfbcf8b7b1f7575d8f2592a94f5a64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Mon, 2 Aug 2021 11:11:30 -0700 Subject: [PATCH 09/11] Rename to AzureJSONEncoder --- sdk/core/azure-core/azure/core/serialization.py | 8 ++++---- sdk/core/azure-core/tests/test_serialization.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/core/azure-core/azure/core/serialization.py b/sdk/core/azure-core/azure/core/serialization.py index 69163678a8a1..a7022aad131b 100644 --- a/sdk/core/azure-core/azure/core/serialization.py +++ b/sdk/core/azure-core/azure/core/serialization.py @@ -13,7 +13,7 @@ if TYPE_CHECKING: from datetime import timedelta -__all__ = ["NULL", "ComplexEncoder"] +__all__ = ["NULL", "AzureJSONEncoder"] class _Null(object): @@ -91,12 +91,12 @@ def _timedelta_as_isostr(value): TZ_UTC = _FixedOffset(0) # type: ignore -class ComplexEncoder(JSONEncoder): +class AzureJSONEncoder(JSONEncoder): """A JSON encoder that's capable of serializing datetime objects and bytes.""" def default(self, o): # pylint: disable=too-many-return-statements try: - return super(ComplexEncoder, self).default(o) + return super(AzureJSONEncoder, self).default(o) except TypeError: if isinstance(o, (bytes, bytearray)): return base64.b64encode(o).decode() @@ -117,4 +117,4 @@ def default(self, o): # pylint: disable=too-many-return-statements except AttributeError: # This will be raised when it hits value.total_seconds in the method above pass - return super(ComplexEncoder, self).default(o) + return super(AzureJSONEncoder, self).default(o) diff --git a/sdk/core/azure-core/tests/test_serialization.py b/sdk/core/azure-core/tests/test_serialization.py index d65357ae3075..61dffec7b54b 100644 --- a/sdk/core/azure-core/tests/test_serialization.py +++ b/sdk/core/azure-core/tests/test_serialization.py @@ -7,7 +7,7 @@ from enum import Enum import json -from azure.core.serialization import ComplexEncoder, NULL +from azure.core.serialization import AzureJSONEncoder, NULL import pytest @@ -79,7 +79,7 @@ def test_NULL_is_falsy(): @pytest.fixture def json_dumps_with_encoder(): def func(obj): - return json.dumps(obj, cls=ComplexEncoder) + return json.dumps(obj, cls=AzureJSONEncoder) return func def test_bytes(json_dumps_with_encoder): From edd36b6ba7bbc023396231db7175dc18bb5eb1c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Mon, 2 Aug 2021 18:37:01 -0700 Subject: [PATCH 10/11] Key Vault model test --- .../azure-core/tests/test_serialization.py | 57 +++++++++++++++++-- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/sdk/core/azure-core/tests/test_serialization.py b/sdk/core/azure-core/tests/test_serialization.py index 61dffec7b54b..ca0cdf45a284 100644 --- a/sdk/core/azure-core/tests/test_serialization.py +++ b/sdk/core/azure-core/tests/test_serialization.py @@ -125,6 +125,7 @@ def __init__(self): self.boolean = True self.list_of_ints = [1, 2, 3] self.dictionary_of_number = {"pi": 3.14} + self.bytes_data = b"data as bytes" expected = BasicModel() expected_dict = { @@ -133,6 +134,7 @@ def __init__(self): "boolean": True, "list_of_ints": [1, 2, 3], "dictionary_of_number": {"pi": 3.14}, + "bytes_data": "ZGF0YSBhcyBieXRlcw==", } assert json.loads(json_dumps_with_encoder(expected.to_dict())) == expected_dict @@ -168,6 +170,57 @@ def __init__(self): } assert json.loads(json_dumps_with_encoder(expected.to_dict())) == expected_dict +def test_model_key_vault(json_dumps_with_encoder): + class Attributes(SerializerMixin): + def __init__(self): + self.enabled = True + self.not_before = datetime.strptime('2012-02-24T00:53:52.780Z', "%Y-%m-%dT%H:%M:%S.%fZ") + self.expires = datetime.strptime('2032-02-24T00:53:52.780Z', "%Y-%m-%dT%H:%M:%S.%fZ") + self.created = datetime.strptime('2020-02-24T00:53:52.780Z', "%Y-%m-%dT%H:%M:%S.%fZ") + self.updated = datetime.strptime('2021-02-24T00:53:52.780Z', "%Y-%m-%dT%H:%M:%S.%fZ") + + class ResourceId(SerializerMixin): + def __init__(self): + self.source_id = "source-id" + self.vault_url = "vault-url" + self.name = "name" + self.version = None + + class Identifier(SerializerMixin): + def __init__(self): + self._resource_id = ResourceId() + + class Properties(SerializerMixin): + def __init__(self): + self._attributes = Attributes() + self._id = "id" + self._vault_id = Identifier() + self._thumbprint = b"thumbprint bytes" + self._tags = None + + expected = Properties() + expected_dict = { + "_attributes": { + "enabled": True, + "not_before": "2012-02-24T00:53:52.780000+00:00", + "expires": "2032-02-24T00:53:52.780000+00:00", + "created": "2020-02-24T00:53:52.780000+00:00", + "updated": "2021-02-24T00:53:52.780000+00:00", + }, + "_id": "id", + "_vault_id": { + "_resource_id": { + "source_id": "source-id", + "vault_url": "vault-url", + "name": "name", + "version": None, + }, + }, + "_thumbprint": "dGh1bWJwcmludCBieXRlcw==", + "_tags": None, + } + assert json.loads(json_dumps_with_encoder(expected.to_dict())) == expected_dict + def test_serialize_datetime(json_dumps_with_encoder): date_obj = datetime.strptime('2015-01-01T00:00:00', "%Y-%m-%dT%H:%M:%S") @@ -294,8 +347,6 @@ def test_dictionary_empty_collections(json_dumps_with_encoder): test_obj = { "dictionary": {}, "list": [], - # "tuple": (), json represents tuples as lists, so this won't round-trip with json.loads - # "set": set(), json can't serialize sets. should we? } assert json.dumps(test_obj) == json_dumps_with_encoder(test_obj) @@ -306,8 +357,6 @@ class EmptyCollectionsModel(SerializerMixin): def __init__(self): self.dictionary = {} self.list = [] - # self.tuple = (), json represents tuples as lists, so this won't round-trip with json.loads - # self.set = set(), json can't serialize sets. should we? expected = EmptyCollectionsModel() expected_dict = { From defb0ee081349754ea45b8ebc5faf3c2db3ad091 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Tue, 3 Aug 2021 18:48:29 -0700 Subject: [PATCH 11/11] 2.7-compatible tests --- sdk/core/azure-core/tests/test_serialization.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/tests/test_serialization.py b/sdk/core/azure-core/tests/test_serialization.py index ca0cdf45a284..e7a4980cdc9f 100644 --- a/sdk/core/azure-core/tests/test_serialization.py +++ b/sdk/core/azure-core/tests/test_serialization.py @@ -6,6 +6,7 @@ from datetime import date, datetime, time, timedelta, tzinfo from enum import Enum import json +import sys from azure.core.serialization import AzureJSONEncoder, NULL import pytest @@ -128,13 +129,14 @@ def __init__(self): self.bytes_data = b"data as bytes" expected = BasicModel() + expected_bytes = "data as bytes" if sys.version_info.major == 2 else "ZGF0YSBhcyBieXRlcw==" expected_dict = { "string": "myid", "number": 42, "boolean": True, "list_of_ints": [1, 2, 3], "dictionary_of_number": {"pi": 3.14}, - "bytes_data": "ZGF0YSBhcyBieXRlcw==", + "bytes_data": expected_bytes, } assert json.loads(json_dumps_with_encoder(expected.to_dict())) == expected_dict @@ -199,6 +201,7 @@ def __init__(self): self._tags = None expected = Properties() + expected_bytes = "thumbprint bytes" if sys.version_info.major == 2 else "dGh1bWJwcmludCBieXRlcw==" expected_dict = { "_attributes": { "enabled": True, @@ -216,7 +219,7 @@ def __init__(self): "version": None, }, }, - "_thumbprint": "dGh1bWJwcmludCBieXRlcw==", + "_thumbprint": expected_bytes, "_tags": None, } assert json.loads(json_dumps_with_encoder(expected.to_dict())) == expected_dict