feat(sdmx): Add SDMX client with custom endpoint and usage samples (#1601)

Author: SandeepTuniki

tools/sdmx/dataflow.py

@@ -0,0 +1,26 @@
+# SDMX Utility Sample Scripts
+
+This directory contains sample scripts demonstrating how to use the SdmxClient in the `tools.sdmx_import.sdmx_client` module to download data and metadata from different SDMX APIs.
+
+## Scripts
+
+### OECD
+
+*   `fetch_oecd_gdp_metadata.py`: Downloads the complete metadata for the OECD's Quarterly GDP Growth dataset.
+*   `fetch_oecd_gdp_data.py`: Fetches a specific slice of data from the same GDP dataset and saves it as a CSV.
+*   `fetch_oecd_full_gdp_dataset.py`: A more complete example that combines both functions to download the metadata and then the full dataset.
+
+### Eurostat
+
+*   `fetch_eurostat_gdp_metadata.py`: Downloads the metadata for the annual GDP dataset from Eurostat.
+*   `fetch_eurostat_gdp_data.py`: Downloads a slice of the annual GDP data for Germany, France, and Italy from Eurostat.
+
+## Running the Samples
+
+You can execute each script from the root of the repository, for example:
+
+```bash
+python3 tools/sdmx_import/samples/fetch_oecd_gdp_metadata.py
+```
+
+The scripts will download the requested data/metadata and save it as `.xml` or `.csv` files in a new `output` directory inside the `samples` folder.

tools/sdmx/__init__.py tools/sdmx_import/__init__.pytools/sdmx/__init__.py renamed to tools/sdmx_import/__init__.py

@@ -0,0 +1,57 @@
+"""
+fetch_eurostat_gdp_data.py
+
+This script provides a complete example of fetching a specific dataset
+from Eurostat using the reusable functions in the dataflow module.
+"""
+
+import logging
+import sys
+import os
+
+# Add the project root to the Python path
+sys.path.append(
+    os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')))
+
+from tools.sdmx_import.sdmx_client import SdmxClient
+
+# Configure logging
+logging.basicConfig(level=logging.INFO,
+                    format='%(asctime)s - %(levelname)s - %(message)s')
+
+
+def main():
+    """Downloads a slice of the Eurostat GDP dataset."""
+    # --- 1. Define Parameters for the Eurostat GDP Dataset ---
+    agency_id = "ESTAT"
+    dataflow_id = "TEC00001"
+    endpoint = "https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/"
+
+    # Create output directory inside the samples folder
+    output_dir = os.path.join(os.path.dirname(__file__), "output")
+    os.makedirs(output_dir, exist_ok=True)
+    output_path = os.path.join(output_dir, "eurostat_gdp_data.csv")
+
+    # Key to select a slice of data
+    data_key = {
+        'freq': 'A',
+        'na_item': 'B1GQ',
+        'unit': 'CP_MEUR',
+        'geo': 'DE+FR+IT'
+    }
+    # Parameters for the query
+    data_params = {'startPeriod': '2020'}
+
+    logging.info(f"--- Fetching Eurostat Data: {dataflow_id} ---")
+
+    # --- 2. Use the SdmxClient ---
+    client = SdmxClient(endpoint, agency_id)
+    client.download_data_as_csv(dataflow_id=dataflow_id,
+                                key=data_key,
+                                params=data_params,
+                                output_path=output_path)
+    logging.info(f"--- Successfully downloaded data to {output_path} ---")
+
+
+if __name__ == "__main__":
+    main()

tools/sdmx_import/samples/README.md

@@ -0,0 +1,44 @@
+"""
+fetch_eurostat_gdp_metadata.py
+
+This script provides a complete example of fetching metadata for a specific
+dataset from Eurostat using the reusable functions in the dataflow module.
+"""
+
+import logging
+import sys
+import os
+
+# Add the project root to the Python path
+sys.path.append(
+    os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')))
+
+from tools.sdmx_import.sdmx_client import SdmxClient
+
+# Configure logging
+logging.basicConfig(level=logging.INFO,
+                    format='%(asctime)s - %(levelname)s - %(message)s')
+
+
+def main():
+    """Downloads the metadata for the Eurostat GDP dataset."""
+    # --- 1. Define Parameters for the Eurostat GDP Dataset ---
+    agency_id = "ESTAT"
+    dataflow_id = "TEC00001"
+    endpoint = "https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/"
+
+    # Create output directory inside the samples folder
+    output_dir = os.path.join(os.path.dirname(__file__), "output")
+    os.makedirs(output_dir, exist_ok=True)
+    output_path = os.path.join(output_dir, "eurostat_gdp_metadata.xml")
+
+    logging.info(f"--- Fetching Eurostat Metadata: {dataflow_id} ---")
+
+    # --- 2. Use the SdmxClient ---
+    client = SdmxClient(endpoint, agency_id)
+    client.download_metadata(dataflow_id=dataflow_id, output_path=output_path)
+    logging.info(f"--- Successfully downloaded metadata to {output_path} ---")
+
+
+if __name__ == "__main__":
+    main()

tools/sdmx_import/samples/fetch_eurostat_gdp_data.py

@@ -0,0 +1,72 @@
+"""
+fetch_oecd_full_gdp_dataset.py
+
+This script provides a complete example of fetching both the metadata and the
+full data series for the OECD's Quarterly GDP Growth dataset.
+"""
+
+import logging
+import sys
+import os
+
+# Add the project root to the Python path
+sys.path.append(
+    os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')))
+
+from tools.sdmx_import.sdmx_client import SdmxClient
+
+# Configure logging
+logging.basicConfig(level=logging.INFO,
+                    format='%(asctime)s - %(levelname)s - %(message)s')
+
+
+def main():
+    """Downloads the full OECD Quarterly GDP Growth dataset and its metadata."""
+    # --- 1. Define Common Parameters ---
+    agency_id = "OECD.SDD.NAD"
+    dataflow_id = "DSD_NAMAIN1@DF_QNA_EXPENDITURE_GROWTH_OECD"
+    endpoint = "https://sdmx.oecd.org/public/rest/"
+
+    # Create output directory inside the samples folder
+    output_dir = os.path.join(os.path.dirname(__file__), "output")
+    os.makedirs(output_dir, exist_ok=True)
+    metadata_output_path = os.path.join(output_dir,
+                                        "oecd_gdp_full_metadata.xml")
+    data_output_path = os.path.join(output_dir, "oecd_gdp_full_data.csv")
+
+    # --- 2. Initialize Client ---
+    client = SdmxClient(endpoint, agency_id)
+
+    # --- 3. Fetch Metadata ---
+    logging.info("--- Step 1: Starting Metadata Download ---")
+    try:
+        client.download_metadata(dataflow_id=dataflow_id,
+                                 output_path=metadata_output_path)
+        logging.info(
+            f"--- Successfully downloaded metadata to {metadata_output_path} ---"
+        )
+    except Exception as e:
+        logging.error(f"Failed to download metadata. Error: {e}")
+        # Exit with a non-zero status code to indicate failure.
+        sys.exit(1)
+
+    # --- 4. Fetch Full Data Series ---
+    logging.info("\n--- Step 2: Starting Full Data Download ---")
+    # For the full dataset, we use an empty key and no time parameters
+    data_key = {}
+    data_params = {}
+
+    try:
+        client.download_data_as_csv(dataflow_id=dataflow_id,
+                                    key=data_key,
+                                    params=data_params,
+                                    output_path=data_output_path)
+        logging.info(
+            f"--- Successfully downloaded data to {data_output_path} ---")
+    except Exception as e:
+        logging.error(f"Failed to download data. Error: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()