diff --git a/requirements.txt b/requirements.txt index 306999b..ac8147a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,6 @@ invoke==0.10.1 pandas # intentionally leaving pandas version blank, b/c `pip install` generally # fails on Linux - must use the package manager version -psycopg2==2.6.1 requests==2.7.0 gunicorn==19.3.0 cryptography>=1.0 diff --git a/utils.py b/utils.py index dd7f063..11af670 100644 --- a/utils.py +++ b/utils.py @@ -1,8 +1,6 @@ import logging -import multiprocessing import os import tempfile -import threading import pandas as pd import sqlalchemy as sa @@ -87,11 +85,13 @@ def load_table(filename, engine = engine or sa.create_engine(config.SQLA_URI) file = ensure_csv(filename) # Pass data types to iterator to ensure consistent types across chunks - dtypes = pd.read_csv(file.name, nrows=infer_size).dtypes + dtypes = pd.read_csv(file.name, nrows=infer_size, + skipinitialspace=True).dtypes chunks = pd.read_csv(file.name, chunksize=chunk_size, iterator=True, - dtype=dtypes) + dtype=dtypes, + skipinitialspace=True) for idx, chunk in enumerate(chunks): chunk.index += chunk_size * idx sql_engine = pandasSQL_builder(engine)