diff --git a/target_postgres/__init__.py b/target_postgres/__init__.py index 1508baf..318e879 100644 --- a/target_postgres/__init__.py +++ b/target_postgres/__init__.py @@ -2,6 +2,7 @@ import argparse import io +import math import os import sys import json @@ -14,13 +15,13 @@ import pkg_resources from jsonschema import Draft4Validator, FormatChecker +import decimal from decimal import Decimal import singer from target_postgres.db_sync import DbSync logger = singer.get_logger() - def float_to_decimal(value): '''Walk the given data structure and turn all instances of float into double.''' @@ -32,6 +33,41 @@ def float_to_decimal(value): return {k: float_to_decimal(v) for k, v in value.items()} return value +def numeric_schema_with_precision(schema): + if 'type' not in schema: + return False + if isinstance(schema['type'], list): + if 'number' not in schema['type']: + return False + elif schema['type'] != 'number': + return False + if 'multipleOf' in schema: + return True + return 'minimum' in schema or 'maximum' in schema + + +def walk_schema_for_numeric_precision(schema): + if isinstance(schema, list): + for v in schema: + walk_schema_for_numeric_precision(v) + elif isinstance(schema, dict): + if numeric_schema_with_precision(schema): + def get_precision(key): + v = math.log10(schema.get(key, 1)) + if v < 0: + return round(math.floor(v)) + return round(math.ceil(v)) + scale = -1 * get_precision('multipleOf') + digits = max(get_precision('minimum'), get_precision('maximum')) + precision = digits + scale + if decimal.getcontext().prec < precision: + logger.debug('Setting decimal precision to {}'.format(precision)) + decimal.getcontext().prec = precision + else: + for v in schema.values(): + walk_schema_for_numeric_precision(v) + + def emit_state(state): if state is not None: line = json.dumps(state) @@ -106,6 +142,7 @@ def persist_lines(config, lines): stream = o['stream'] schemas[stream] = o schema = float_to_decimal(o['schema']) + walk_schema_for_numeric_precision(schema) validators[stream] = Draft4Validator(schema, format_checker=FormatChecker()) if 'key_properties' not in o: raise Exception("key_properties field is required")