From 6a80fbda7e2d0972df915348bdaf42c3a57cefcc Mon Sep 17 00:00:00 2001 From: Vraj Mohan Date: Thu, 21 Jul 2016 06:58:53 -0400 Subject: [PATCH 1/3] Remove library that is not needed As the product is database agnostic, there is no need to install psycopg2. --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 306999b..ac8147a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,6 @@ invoke==0.10.1 pandas # intentionally leaving pandas version blank, b/c `pip install` generally # fails on Linux - must use the package manager version -psycopg2==2.6.1 requests==2.7.0 gunicorn==19.3.0 cryptography>=1.0 From d95be730f50b664540ae5baccfd41bdeb427b4e7 Mon Sep 17 00:00:00 2001 From: Vraj Mohan Date: Thu, 21 Jul 2016 07:00:59 -0400 Subject: [PATCH 2/3] Remove unnecessary imports --- utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/utils.py b/utils.py index dd7f063..9945b6b 100644 --- a/utils.py +++ b/utils.py @@ -1,8 +1,6 @@ import logging -import multiprocessing import os import tempfile -import threading import pandas as pd import sqlalchemy as sa From 9a7349d278ad5b66f5fdb51e728b959f5ff3bb0b Mon Sep 17 00:00:00 2001 From: Vraj Mohan Date: Thu, 21 Jul 2016 07:05:05 -0400 Subject: [PATCH 3/3] Skip spaces after delimiter This removes leading spaces in field names and field values. E.g, name, age Tom, 12 will be parsed as: "name", "age" "Tom", 12 --- utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/utils.py b/utils.py index 9945b6b..11af670 100644 --- a/utils.py +++ b/utils.py @@ -85,11 +85,13 @@ def load_table(filename, engine = engine or sa.create_engine(config.SQLA_URI) file = ensure_csv(filename) # Pass data types to iterator to ensure consistent types across chunks - dtypes = pd.read_csv(file.name, nrows=infer_size).dtypes + dtypes = pd.read_csv(file.name, nrows=infer_size, + skipinitialspace=True).dtypes chunks = pd.read_csv(file.name, chunksize=chunk_size, iterator=True, - dtype=dtypes) + dtype=dtypes, + skipinitialspace=True) for idx, chunk in enumerate(chunks): chunk.index += chunk_size * idx sql_engine = pandasSQL_builder(engine)