From 1d02d442d7f0dbb334b788d584ea3a9221089134 Mon Sep 17 00:00:00 2001 From: Charles Julian Knight Date: Mon, 3 Aug 2020 18:03:05 -0400 Subject: [PATCH 1/4] set decimal precision --- target_postgres/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/target_postgres/__init__.py b/target_postgres/__init__.py index 1508baf..4bdbcef 100644 --- a/target_postgres/__init__.py +++ b/target_postgres/__init__.py @@ -14,12 +14,17 @@ import pkg_resources from jsonschema import Draft4Validator, FormatChecker +import decimal from decimal import Decimal import singer from target_postgres.db_sync import DbSync logger = singer.get_logger() +# This value comes from the precision definition of +# Postgres numeric/decimal types: +# https://www.postgresql.org/docs/9.1/datatype-numeric.html +decimal.getcontext().prec = 147455 def float_to_decimal(value): '''Walk the given data structure and turn all instances of float into From 2cd83c32f945400e37251e685df4c4e67004d552 Mon Sep 17 00:00:00 2001 From: Charles Julian Knight Date: Mon, 3 Aug 2020 19:22:09 -0400 Subject: [PATCH 2/4] refactor, set precision based on jsonschema --- target_postgres/__init__.py | 39 ++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/target_postgres/__init__.py b/target_postgres/__init__.py index 4bdbcef..62f27e4 100644 --- a/target_postgres/__init__.py +++ b/target_postgres/__init__.py @@ -2,6 +2,7 @@ import argparse import io +import math import os import sys import json @@ -21,11 +22,6 @@ logger = singer.get_logger() -# This value comes from the precision definition of -# Postgres numeric/decimal types: -# https://www.postgresql.org/docs/9.1/datatype-numeric.html -decimal.getcontext().prec = 147455 - def float_to_decimal(value): '''Walk the given data structure and turn all instances of float into double.''' @@ -37,6 +33,38 @@ def float_to_decimal(value): return {k: float_to_decimal(v) for k, v in value.items()} return value +def numeric_schema_with_precision(schema): + if 'type' not in schema: + return False + if isinstance(schema['type'], list): + if 'number' not in schema['type']: + return False + elif schema['type'] != 'number': + return False + if 'multipleOf' in schema: + return True + return 'minimum' in schema or 'maximum' in schema + + +def walk_schema_for_numeric_precision(schema): + if isinstance(schema, list): + for v in schema: + walk_schema_for_numeric_precision(v) + elif isinstance(schema, dict): + if numeric_schema_with_precision(schema): + precision = round(max( + len(Decimal(schema.get('minimum', '0')).as_tuple().digits), + len(Decimal(schema.get('maximum', '0')).as_tuple().digits), + abs(math.log10(schema.get('multipleOf', 1))), + )) + if decimal.getcontext().prec < precision: + logger.debug('Setting decimal precision to {}'.format(precision)) + decimal.getcontext().prec = precision + else: + for v in schema.values(): + walk_schema_for_numeric_precision(v) + + def emit_state(state): if state is not None: line = json.dumps(state) @@ -111,6 +139,7 @@ def persist_lines(config, lines): stream = o['stream'] schemas[stream] = o schema = float_to_decimal(o['schema']) + walk_schema_for_numeric_precision(schema) validators[stream] = Draft4Validator(schema, format_checker=FormatChecker()) if 'key_properties' not in o: raise Exception("key_properties field is required") From f13cfec221722d9bf778966abbb4f3bb3a79c171 Mon Sep 17 00:00:00 2001 From: Charles Julian Knight Date: Mon, 3 Aug 2020 20:09:37 -0400 Subject: [PATCH 3/4] infer exact precision required based on min, max, and multipleOf --- target_postgres/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/target_postgres/__init__.py b/target_postgres/__init__.py index 62f27e4..318e879 100644 --- a/target_postgres/__init__.py +++ b/target_postgres/__init__.py @@ -52,11 +52,14 @@ def walk_schema_for_numeric_precision(schema): walk_schema_for_numeric_precision(v) elif isinstance(schema, dict): if numeric_schema_with_precision(schema): - precision = round(max( - len(Decimal(schema.get('minimum', '0')).as_tuple().digits), - len(Decimal(schema.get('maximum', '0')).as_tuple().digits), - abs(math.log10(schema.get('multipleOf', 1))), - )) + def get_precision(key): + v = math.log10(schema.get(key, 1)) + if v < 0: + return round(math.floor(v)) + return round(math.ceil(v)) + scale = -1 * get_precision('multipleOf') + digits = max(get_precision('minimum'), get_precision('maximum')) + precision = digits + scale if decimal.getcontext().prec < precision: logger.debug('Setting decimal precision to {}'.format(precision)) decimal.getcontext().prec = precision From a25ed64c9b472f17eda4749aed5e361da17ac431 Mon Sep 17 00:00:00 2001 From: Charles Julian Knight Date: Tue, 4 Aug 2020 13:03:11 -0400 Subject: [PATCH 4/4] fix for ValueError: math domain error --- target_postgres/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target_postgres/__init__.py b/target_postgres/__init__.py index 318e879..4074339 100644 --- a/target_postgres/__init__.py +++ b/target_postgres/__init__.py @@ -53,7 +53,7 @@ def walk_schema_for_numeric_precision(schema): elif isinstance(schema, dict): if numeric_schema_with_precision(schema): def get_precision(key): - v = math.log10(schema.get(key, 1)) + v = abs(Decimal(schema.get(key, 1))).log10() if v < 0: return round(math.floor(v)) return round(math.ceil(v))