From 3901f43c6a712a1a3efc340b5b8d8fd0cbe8ee63 Mon Sep 17 00:00:00 2001 From: Simon Wiles Date: Thu, 27 Aug 2020 13:05:59 -0700 Subject: [PATCH 1/5] Insert all columns for every chunk --- sqlite_utils/db.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index a8791c339..75599f6bb 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1074,6 +1074,14 @@ def insert_all( all_columns = list(sorted(all_columns)) if hash_id: all_columns.insert(0, hash_id) + else: + all_columns += [ + column + for record in chunk + for column in record + if column not in all_columns + ] + validate_column_names(all_columns) first = False # values is the list of insert data that is passed to the From 222bbb1375b1b20162caa4c2bcadac7559e1814c Mon Sep 17 00:00:00 2001 From: Simon Wiles Date: Fri, 28 Aug 2020 14:51:19 -0700 Subject: [PATCH 2/5] Update unit test to reflect new behaviour Exactly as suggested by @simonw at #139. --- tests/test_create.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_create.py b/tests/test_create.py index a84eb8dab..bd7c984bd 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -707,13 +707,15 @@ def test_insert_thousands_using_generator(fresh_db): assert 10000 == fresh_db["test"].count -def test_insert_thousands_ignores_extra_columns_after_first_100(fresh_db): +def test_insert_thousands_adds_extra_columns_after_first_100(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/139 fresh_db["test"].insert_all( [{"i": i, "word": "word_{}".format(i)} for i in range(100)] - + [{"i": 101, "extra": "This extra column should cause an exception"}] + + [{"i": 101, "extra": "Should trigger ALTER"}], + alter=True, ) rows = fresh_db.execute_returning_dicts("select * from test where i = 101") - assert [{"i": 101, "word": None}] == rows + assert [{"i": 101, "word": None, "extra": "Should trigger ALTER"}] == rows def test_insert_ignore(fresh_db): From 2ded6545b4be0bc9345436928cd660f210ffd466 Mon Sep 17 00:00:00 2001 From: Simon Wiles Date: Fri, 28 Aug 2020 14:58:01 -0700 Subject: [PATCH 3/5] Rename test --- tests/test_create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_create.py b/tests/test_create.py index bd7c984bd..71dc5494d 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -707,7 +707,7 @@ def test_insert_thousands_using_generator(fresh_db): assert 10000 == fresh_db["test"].count -def test_insert_thousands_adds_extra_columns_after_first_100(fresh_db): +def test_insert_thousands_adds_extra_columns_after_first_100_with_alter(fresh_db): # https://github.com/simonw/sqlite-utils/issues/139 fresh_db["test"].insert_all( [{"i": i, "word": "word_{}".format(i)} for i in range(100)] From eb6527078256853bf0f527d54ae623be31cb818f Mon Sep 17 00:00:00 2001 From: Simon Wiles Date: Fri, 28 Aug 2020 14:58:25 -0700 Subject: [PATCH 4/5] Test that exception is raised --- tests/test_create.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_create.py b/tests/test_create.py index 71dc5494d..fc8edc02d 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -707,6 +707,15 @@ def test_insert_thousands_using_generator(fresh_db): assert 10000 == fresh_db["test"].count +def test_insert_thousands_raises_exception_wtih_extra_columns_after_first_100(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/139 + with pytest.raises(Exception, match="table test has no column named extra"): + fresh_db["test"].insert_all( + [{"i": i, "word": "word_{}".format(i)} for i in range(100)] + + [{"i": 101, "extra": "This extra column should cause an exception"}], + ) + + def test_insert_thousands_adds_extra_columns_after_first_100_with_alter(fresh_db): # https://github.com/simonw/sqlite-utils/issues/139 fresh_db["test"].insert_all( From 1b992c7c1e5855caa87c7939efad287052f446f3 Mon Sep 17 00:00:00 2001 From: Simon Wiles Date: Fri, 28 Aug 2020 15:20:48 -0700 Subject: [PATCH 5/5] Update documentation --- docs/python-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 9697fd3d3..5860397ea 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -410,7 +410,7 @@ Use it like this: "is_good_dog": True, }], pk="id", column_order=("id", "twitter", "name")) -The column types used in the ``CREATE TABLE`` statement are automatically derived from the types of data in that first batch of rows. Any additional or missing columns in subsequent batches will be ignored. +The column types used in the ``CREATE TABLE`` statement are automatically derived from the types of data in that first batch of rows. Any additional columns in subsequent batches will cause a ``sqlite3.OperationalError`` exception to be raised unless the ``alter=True`` argument is supplied, in which case the new columns will be created. The function can accept an iterator or generator of rows and will commit them according to the batch size. The default batch size is 100, but you can specify a different size using the ``batch_size`` parameter: