From 33c828493b1eb0d55392551e3853c947f87b9a89 Mon Sep 17 00:00:00 2001 From: Abdulazez Zeinu Ali Date: Sat, 16 May 2026 22:23:34 +0300 Subject: [PATCH 1/2] fix: prevent silent data loss in batch processing loops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - batch_import_locations.py: append item BEFORE checking if batch is full. Previously, every Nth item was flushed-around and never appended — causing silent data loss (e.g. item #100, #200, ...). - woosmap_to_woosmap.py: same fix for store batch processing (every 500th store was lost). - woosmap_to_woosmap.py: replace except BaseException with except Exception to allow KeyboardInterrupt/SystemExit to propagate. --- python-samples/batchimport/batch_import_locations.py | 3 +-- python-samples/woosmap_to_woosmap/woosmap_to_woosmap.py | 7 +++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/python-samples/batchimport/batch_import_locations.py b/python-samples/batchimport/batch_import_locations.py index e1d55bb..c12559b 100755 --- a/python-samples/batchimport/batch_import_locations.py +++ b/python-samples/batchimport/batch_import_locations.py @@ -101,11 +101,10 @@ def import_batch(batch, use_put=False): try: woosmap_location = datagov2woosmap(location, "ID" + str(id)) + batch.append(woosmap_location) if len(batch) == batch_size: batch_result = import_batch(batch, use_put=update_location) batch = [] - else: - batch.append(woosmap_location) except InvalidGeometry: pass diff --git a/python-samples/woosmap_to_woosmap/woosmap_to_woosmap.py b/python-samples/woosmap_to_woosmap/woosmap_to_woosmap.py index 9d7999f..55efa79 100644 --- a/python-samples/woosmap_to_woosmap/woosmap_to_woosmap.py +++ b/python-samples/woosmap_to_woosmap/woosmap_to_woosmap.py @@ -90,15 +90,14 @@ def import_location(locations): export_input_json(stores_woosmap) if private_key: for store in stores_woosmap: + batch.append(store) if len(batch) == stores_batch_size: batch_result = import_location(batch) batch = [] - else: - batch.append(store) if batch: batch_result = import_location(batch) batch = [] - except BaseException as error: # bad bad way! - print('An exception occurred: {}'.format(error)) + except Exception as err: + print('An exception occurred: {}'.format(err)) From d19cfa78d325f03b1fd3b48917f6cc11c2f7f819 Mon Sep 17 00:00:00 2001 From: Abdulazez Zeinu Ali Date: Sat, 16 May 2026 22:29:01 +0300 Subject: [PATCH 2/2] fix: prevent silent data loss in batch processing loops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - batch_import_locations.py: append item BEFORE checking if batch is full. Previously, every (batch_size+1)th item was flushed without being appended — e.g. items #101, #202, #303 silently lost. - woosmap_to_woosmap.py: same fix (every 501st store was lost). - woosmap_to_woosmap.py: replace two except BaseException with except Exception to allow KeyboardInterrupt/SystemExit to propagate. --- python-samples/woosmap_to_woosmap/woosmap_to_woosmap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python-samples/woosmap_to_woosmap/woosmap_to_woosmap.py b/python-samples/woosmap_to_woosmap/woosmap_to_woosmap.py index 55efa79..5c5e8ed 100644 --- a/python-samples/woosmap_to_woosmap/woosmap_to_woosmap.py +++ b/python-samples/woosmap_to_woosmap/woosmap_to_woosmap.py @@ -32,8 +32,8 @@ def transform_geojson_woosmap(extracted_geojson): "name": prop.get("name", ""), "tags": prop.get("tags", []), "contact": prop.get("contact", {})}) - except BaseException as error: - print('An exception occurred: {}'.format(error)) + except Exception as err: + print('An exception occurred: {}'.format(err)) return stores