diff --git a/inbox/sqlalchemy_ext/json_util.py b/inbox/sqlalchemy_ext/json_util.py new file mode 100644 index 000000000..c3afc660b --- /dev/null +++ b/inbox/sqlalchemy_ext/json_util.py @@ -0,0 +1,77 @@ +# Copyright 2009-2015 MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tools for using Python's :mod:`json` module with BSON documents. + +This module provides two helper methods `dumps` and `loads` that wrap the +native :mod:`json` methods and provide explicit datetime.datetime conversion to and from +json. This is a very stripped version of `Mongo Extended JSON +`_'s *Strict* +mode handling just datetime.dateime fields because we don't serialize any +other types that are not handled by JSON. + +The original unstripped version of this module can be found at +https://github.com/mongodb/mongo-python-driver/blob/18328a909545ece6e1cd7e172e28271a59e367d5/bson/json_util.py. +""" + +import calendar +import datetime +import json + +EPOCH_NAIVE = datetime.datetime.utcfromtimestamp(0) + + +def dumps(obj, *args, **kwargs): + """Helper function that wraps :class:`json.dumps`. + + Recursive function that handles all datetime.datetime type. + """ + return json.dumps(_json_convert(obj), *args, **kwargs) + + +def loads(s, *args, **kwargs): + """Helper function that wraps :class:`json.loads`.""" + kwargs["object_hook"] = lambda dct: object_hook(dct) + return json.loads(s, *args, **kwargs) + + +def _json_convert(obj): + """Recursive helper method that converts datetime.datetime type so it can be + converted into json. + """ + if hasattr(obj, "items"): + return dict(((k, _json_convert(v)) for k, v in obj.items())) + elif hasattr(obj, "__iter__") and not isinstance(obj, (str, bytes)): + return list((_json_convert(v) for v in obj)) + try: + return default(obj) + except TypeError: + return obj + + +def object_hook(dct): + if "$date" in dct: + dtm = dct["$date"] + secs = float(dtm) / 1000.0 + return EPOCH_NAIVE + datetime.timedelta(seconds=secs) + return dct + + +def default(obj): + if isinstance(obj, datetime.datetime): + if obj.utcoffset() is not None: + obj = obj - obj.utcoffset() + millis = int(calendar.timegm(obj.timetuple()) * 1000 + obj.microsecond / 1000) + return {"$date": millis} + raise TypeError(f"{obj!r} is not JSON serializable") diff --git a/inbox/sqlalchemy_ext/util.py b/inbox/sqlalchemy_ext/util.py index 60a0a5f73..6307bf5db 100644 --- a/inbox/sqlalchemy_ext/util.py +++ b/inbox/sqlalchemy_ext/util.py @@ -6,12 +6,6 @@ import weakref from typing import Any, MutableMapping, Optional, Tuple -from bson import EPOCH_NAIVE, json_util - -# Monkeypatch to not include tz_info in decoded JSON. -# Kind of a ridiculous solution, but works. -json_util.EPOCH_AWARE = EPOCH_NAIVE - from sqlalchemy import String, Text, event from sqlalchemy.engine import Engine from sqlalchemy.ext.mutable import Mutable @@ -20,6 +14,7 @@ from sqlalchemy.types import BINARY, TypeDecorator from inbox.logging import get_logger +from inbox.sqlalchemy_ext import json_util from inbox.util.encoding import base36decode, base36encode log = get_logger() @@ -169,8 +164,8 @@ def process_result_value( # http://bit.ly/1LbMnqu -# Can simply use this as is because though we use bson.json_util, loads() -# dumps() return standard Python dicts like the json.* equivalents +# Can simply use this as is because though we use inbox.sqlalchemy_ext.json_util, +# loads() dumps() return standard Python dicts like the json.* equivalents # (because these are simply called under the hood) class MutableDict(Mutable, dict): @classmethod diff --git a/migrations/versions/057_consolidate_account_sync_status_columns.py b/migrations/versions/057_consolidate_account_sync_status_columns.py index cbfead593..7311e2acf 100644 --- a/migrations/versions/057_consolidate_account_sync_status_columns.py +++ b/migrations/versions/057_consolidate_account_sync_status_columns.py @@ -12,7 +12,8 @@ import sqlalchemy as sa from alembic import op -from bson import json_util + +from inbox.sqlalchemy_ext import json_util def upgrade(): diff --git a/requirements/prod.txt b/requirements/prod.txt index 15b58c3a2..43beb45f2 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -62,7 +62,6 @@ pycparser==2.21 pygments==2.17.2 pyinstrument==3.2.0 pyinstrument-cext==0.2.2 -pymongo==3.0 # For json_util in bson Pympler==0.9 PyNaCl==1.5.0 python-dateutil==2.8.2