From 026e8cc03f7158dc1d337334683832bd47be83af Mon Sep 17 00:00:00 2001 From: Angelica Lo Duca Date: Fri, 10 Dec 2021 12:06:10 +0100 Subject: [PATCH 01/28] Created Life Expectancy Scenario with Data Ingestion Jobs --- examples/README.md | 19 ++ examples/life-expectancy/README.md | 167 ++++++++++++++++++ examples/life-expectancy/le_utils/__init__.py | 0 examples/life-expectancy/le_utils/utils.py | 36 ++++ ...delete_table_life_expectancy_2010_2015.sql | 1 + .../20_delete_table_life_expectancy_2018.sql | 1 + ...create_table_life_expectancy_2010_2015.sql | 10 ++ .../40_create_table_life_expectancy_2018.sql | 7 + ..._ingest_table_life_expectancy_2010_2015.py | 46 +++++ .../60_ingest_table_life_expectancy_2018.py | 45 +++++ .../life-expectancy/config.ini | 32 ++++ .../life-expectancy/requirements.txt | 4 + 12 files changed, 368 insertions(+) create mode 100644 examples/README.md create mode 100644 examples/life-expectancy/README.md create mode 100644 examples/life-expectancy/le_utils/__init__.py create mode 100644 examples/life-expectancy/le_utils/utils.py create mode 100644 examples/life-expectancy/life-expectancy/10_delete_table_life_expectancy_2010_2015.sql create mode 100644 examples/life-expectancy/life-expectancy/20_delete_table_life_expectancy_2018.sql create mode 100644 examples/life-expectancy/life-expectancy/30_create_table_life_expectancy_2010_2015.sql create mode 100644 examples/life-expectancy/life-expectancy/40_create_table_life_expectancy_2018.sql create mode 100644 examples/life-expectancy/life-expectancy/50_ingest_table_life_expectancy_2010_2015.py create mode 100644 examples/life-expectancy/life-expectancy/60_ingest_table_life_expectancy_2018.py create mode 100644 examples/life-expectancy/life-expectancy/config.ini create mode 100644 examples/life-expectancy/life-expectancy/requirements.txt diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000000..a15d0101b3 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,19 @@ +# Versatile Data Kit Examples + +This section provides a set of examples that explore some of the features provided by Versatile Data Kit. + +## Basic Examples +Tutorials to get started with generic ingestion tasks in VDK: +* [Ingest from CSV](ingest-csv-file-example/) +* [Ingest from database](ingest-from-db-example/) +* [Ingest from REST API](ingest-from-rest-api-example/) +* [Ingest from S3](ingest-from-s3-example/) + +Tutorials to get started with generic processing tasks in VDK: +* [Sqlite Processing](sqlite-processing-example/) + +## Advanced Examples +Tutorials with a complete pipeline from ingestion to final publication: +* [Life Expectancy](life-expectancy/) +* [Online Exhibition](online-exhibition/) +* [Energy](energy/) \ No newline at end of file diff --git a/examples/life-expectancy/README.md b/examples/life-expectancy/README.md new file mode 100644 index 0000000000..c88abadbe5 --- /dev/null +++ b/examples/life-expectancy/README.md @@ -0,0 +1,167 @@ +# Life Expectancy +This example provides the complete code to perform the following tasks: +- Data Ingestion from CSV +- Data Processing +- Data Publication in form of Visual Report/Dashboard + +The directory is organized as follows: +* **life-expectancy** - contains jobs +* **le_utils** - contains some auxiliary_functions. + +## Scenario + +Researchers from the Department of Population History and Social Structure have to conduct a study on the life expectancy of Americans. The objective of the study is to understand over time which American states have the greatest life expectancy. + +## Data Scoures +After extensive research, the researchers found two main datasets, both of which are available as CSV files: + +- [U.S. Life Expectancy at Birth by State and Census Tract - 2010-2015](https://catalog.data.gov/dataset/u-s-life-expectancy-at-birth-by-state-and-census-tract-2010-2015) +- [U.S. State Life Expectancy by Sex, 2018](https://catalog.data.gov/dataset/u-s-state-life-expectancy-by-sex-2018) + +The dataset **U.S. Life Expectancy at Birth by State and Census Tract - 2010-2015** contains 73,121 records relating to the life expectancy of Americans, divided by state and county and relating to the period 2010-2015. For each record, the following information is available: + +- State +- County +- Census Tract Number +- Life Expectancy +- Life Expectancy Range +- Life Expectancy Standard Error + +For some records, some fields are missing. + +The following table shows an excerpt from the dataset: + +| **State** | **County** | **Census Tract Number** | **Life Expectancy** | **Life Expectancy Range** | **Life Expectancy Standard Error** | +| --- | --- | --- | --- | --- | --- | +| **Alabama** | (blank) | | 75.5 | 75.2-77.5 | 0.0328 | +| **Alabama** | Autauga County, AL | 0201.00 | 73.1 | 56.9-75.1 | 2.2348 | +| **Alabama** | Autauga County, AL | 0202.00 | 76.9 | 75.2-77.5 | 3.3453 | +| **Alabama** | Autauga County, AL | 0203.00 | | | | +| **Alabama** | Autauga County, AL | 0204.00 | 75.4 | 75.2-77.5 | 1.0216 | +| **Alabama** | Autauga County, AL | 0205.00 | 79.4 | 77.6-79.5 | 1.1768 | +| **Alabama** | Autauga County, AL | 0206.00 | 73.1 | 56.9-75.1 | 1.5519 | +| **Alabama** | Autauga County, AL | 0207.00 | | | | +| **Alabama** | Autauga County, AL | 0208.01 | 78.3 | 77.6-79.5 | 2.3861 | +| **Alabama** | Autauga County, AL | 0208.02 | 76.9 | 75.2-77.5 | 1.2628 | +| **Alabama** | Autauga County, AL | 0209.00 | 73.9 | 56.9-75.1 | 1.5923 | + +The dataset **U.S. State Life Expectancy by Sex, 2018** contains 156 records relating to the life expectancy of Americans in 2018, divided by state and sex (male, female, total). For each record, the following information is available: + +- State +- Sex +- LEB - Life Expectancy at birth +- SE - Life Expectancy Standard Error +- Quartile - Life Expectancy Range + +The following table shows an excerpt from the dataset: + +| **State** | **Sex** | **LEB** | **SE** | **Quartile** | +| --- | --- | --- | --- | --- | +| **United States** | Total | 78.7 | | \* | +| **West Virginia** | Total | 74.4 | 114 | 74.4 - 77.2 | +| **Mississippi** | Total | 74.6 | 88 | 74.4 - 77.2 | +| **Alabama** | Total | 75.1 | 67 | 74.4 - 77.2 | +| **Kentucky** | Total | 75.3 | 68 | 74.4 - 77.2 | +| **Tennessee** | Total | 75.5 | 57 | 74.4 - 77.2 | +| **Arkansas** | Total | 75.6 | 86 | 74.4 - 77.2 | +| **Oklahoma** | Total | 75.6 | 73 | 74.4 - 77.2 | + +## Requirements +To run this example, you need: +* Versatile Data Kit +* Trino DB +* Versatile Data Kit plugin for Trino + +### Versatile Data Kit +If you have not done so already, you can install Versatile Data Kit and the plugins required for this example by running the following commands from a terminal: + +``` +pip install quickstart-vdk +``` + +Note that Versatile Data Kit requires Python 3.7+. See the [Installation Page](https://github.com/vmware/versatile-data-kit/wiki/Installation#install-sdk "Installation page") for more details. + +### Trino DB +This example also requires Trino DB installed. See the Trino [Official Documentation](https://trino.io/ "Official Documentation") for more details about installation. + +### Versatile Data Kit Plugin for Trino +Since this example requires Trino, you should also install the Versatile Data Kit plugin for Trino: + +``` +pip install vdk-trino +``` + +See the vdk-trino [Documentation Page](https://github.com/vmware/versatile-data-kit/tree/main/projects/vdk-plugins/vdk-trino "Documentation Page") for more details. + +### Other Requirements +This example also requires the following Python libraries, which are included in the `requirement.txt` file: + +``` +inspect +math +numpy +pandas +``` + +## Configuration + +### Trino DB +In this example Trino is running locally, with the following minimal `config.properties` configuration file: + +``` +coordinator=true +node-scheduler.include-coordinator=true +http-server.http.port=8080 +query.max-memory=5GB +query.max-memory-per-node=1GB +query.max-total-memory-per-node=2GB +discovery.uri=http://127.0.0.1:8080 +http-server.https.enabled=false + +``` + +In addition, the Trino DB exploits the MySQL catalog, with the following configuration (file `mysql.properties` located in the catalog folder of the Trino server: + +``` +connector.name=mysql +connection-url=jdbc:mysql://localhost:3306 +connection-user=root +connection-password= +allow-drop-table=true +``` + +More complex configurations can be used too. + +Finally, this example assumes that an empty schema, called `life-expectancy` exists on the MySQL server. + +### Versatile Data Kit +The Life Expectancy Data Job runs with the following configuration (`config.ini`): + +``` +db_default_type=TRINO +ingest_method_default = trino +trino_catalog = mysql +trino_use_ssl = +trino_host = localhost +trino_port = 8080 +trino_user = root +trino_schema = life-expectancy +trino_ssl_verify = +``` + +## Data Ingestion +Data Ingestion uploads in the database the two CSV tables, defined in the Data Source section. For each table, data Ingestion is performed through the following steps: +* delete the existing table (if any) +* create a new table +* ingest table values directly from the CSV file. + +The path to the CSV file is specified as a URL, thus this example requires an active Internet connection to work properly. + +Jobs from 10 to 60 are devoted to Data Ingestion: +* 10 - 30 ingest data in table `life_expectancy_2010_2015` +* 40 - 60 ingest data in table `life_expectancy_2018` + + + + + diff --git a/examples/life-expectancy/le_utils/__init__.py b/examples/life-expectancy/le_utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/life-expectancy/le_utils/utils.py b/examples/life-expectancy/le_utils/utils.py new file mode 100644 index 0000000000..9a3bb58bf9 --- /dev/null +++ b/examples/life-expectancy/le_utils/utils.py @@ -0,0 +1,36 @@ +import math +import numpy as np + +def build_insert_query(row, cols, output_table): + """ + Build an formatted query for insertion. + + Keyword arguments: + -- row (pandas.Series) the list of values to be inserted in the query + -- cols (list) the list of columns name of the SQL table + + Return: + -- query (string) a string containing the insert statement + + """ + values = "" + keys = cols.copy() + for index in range(0, len(row)): + + # manage NaN values, both for strings and numeric values + if (row[index] != row[index]) and (math.isnan(row[index])): + # do not insert NaN values, thus remove it from columns + keys.remove(cols[index]) + + else: + # quote only strings and convert numeric values to strings + if isinstance(row[index], str): + values = values + ", '" + row[index] + "'" + else: + values = values + ", " + str(row[index]) + + # remove the first comma + values = values[1:] + + cols = ', '.join([str(i) for i in keys]) + return f'INSERT INTO {output_table} ({cols}) VALUES ({values})' \ No newline at end of file diff --git a/examples/life-expectancy/life-expectancy/10_delete_table_life_expectancy_2010_2015.sql b/examples/life-expectancy/life-expectancy/10_delete_table_life_expectancy_2010_2015.sql new file mode 100644 index 0000000000..e6520a710f --- /dev/null +++ b/examples/life-expectancy/life-expectancy/10_delete_table_life_expectancy_2010_2015.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS life_expectancy_2010_2015 \ No newline at end of file diff --git a/examples/life-expectancy/life-expectancy/20_delete_table_life_expectancy_2018.sql b/examples/life-expectancy/life-expectancy/20_delete_table_life_expectancy_2018.sql new file mode 100644 index 0000000000..942f550f70 --- /dev/null +++ b/examples/life-expectancy/life-expectancy/20_delete_table_life_expectancy_2018.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS life_expectancy_2018 \ No newline at end of file diff --git a/examples/life-expectancy/life-expectancy/30_create_table_life_expectancy_2010_2015.sql b/examples/life-expectancy/life-expectancy/30_create_table_life_expectancy_2010_2015.sql new file mode 100644 index 0000000000..f4f0ccbfab --- /dev/null +++ b/examples/life-expectancy/life-expectancy/30_create_table_life_expectancy_2010_2015.sql @@ -0,0 +1,10 @@ +CREATE TABLE life_expectancy_2010_2015 ( + State varchar(32), + County varchar(32), + CensusTractNumber varchar(32), + LifeExpectancy decimal(4,2), + LifeExpectancyRange varchar, + LifeExpectancyStandardError decimal(4,2) +) + + diff --git a/examples/life-expectancy/life-expectancy/40_create_table_life_expectancy_2018.sql b/examples/life-expectancy/life-expectancy/40_create_table_life_expectancy_2018.sql new file mode 100644 index 0000000000..1426f0eb1f --- /dev/null +++ b/examples/life-expectancy/life-expectancy/40_create_table_life_expectancy_2018.sql @@ -0,0 +1,7 @@ +CREATE TABLE life_expectancy_2018 ( + State varchar(32), + Sex varchar(32), + LEB decimal(3,1), + SE decimal(3,1), + Quartile varchar(32) +) \ No newline at end of file diff --git a/examples/life-expectancy/life-expectancy/50_ingest_table_life_expectancy_2010_2015.py b/examples/life-expectancy/life-expectancy/50_ingest_table_life_expectancy_2010_2015.py new file mode 100644 index 0000000000..e015a4535a --- /dev/null +++ b/examples/life-expectancy/life-expectancy/50_ingest_table_life_expectancy_2010_2015.py @@ -0,0 +1,46 @@ +import inspect +import logging +import os +import numpy as np +import pandas as pd +import sys + +current_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) +parent_dir = os.path.dirname(current_dir) +sys.path.insert(0, parent_dir) + +from le_utils.utils import build_insert_query + +from vdk.api.job_input import IJobInput + +log = logging.getLogger(__name__) + + + +def run(job_input: IJobInput): + """ + Download datasets required by the scenario and put them in the data directory. + """ + log.info(f"Starting job step {__name__}") + + # url of the U.S. Life Expectancy at Birth by State and Census Tract - 2010-2015 dataset + url = 'http://data.cdc.gov/api/views/5h56-n989/rows.csv' + + dtypes = { + 'State' : str, + 'County' : str, + 'Census Tract Number' : str, + 'Life Expectancy' : np.float64, + 'Life Expectancy Range' : str, + 'Life Expectancy Standard Error' : np.float64 + } + + df = pd.read_csv(url, dtype=dtypes) + + df.columns = df.columns.str.replace(' ', '') + + + for i,row in df.iterrows(): + query = build_insert_query(row, df.columns.tolist(), 'life_expectancy_2010_2015') + + job_input.execute_query(query) diff --git a/examples/life-expectancy/life-expectancy/60_ingest_table_life_expectancy_2018.py b/examples/life-expectancy/life-expectancy/60_ingest_table_life_expectancy_2018.py new file mode 100644 index 0000000000..0e755d7b93 --- /dev/null +++ b/examples/life-expectancy/life-expectancy/60_ingest_table_life_expectancy_2018.py @@ -0,0 +1,45 @@ +import inspect +import logging +import os +import numpy as np +import pandas as pd +import sys + +current_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) +parent_dir = os.path.dirname(current_dir) +sys.path.insert(0, parent_dir) + +from le_utils.utils import build_insert_query + +from vdk.api.job_input import IJobInput + +log = logging.getLogger(__name__) + + + +def run(job_input: IJobInput): + """ + Download datasets required by the scenario and put them in the data directory. + """ + log.info(f"Starting job step {__name__}") + + # url of the U.S. Life Expectancy at Birth by State and Census Tract - 2010-2015 dataset + url = 'http://data.cdc.gov/api/views/a5a8-jsrq/rows.csv' + + dtypes = { + 'State' : str, + 'Sex' : str, + 'LEB' : np.float64, + 'SE' : np.float64, + 'Quartile' : str, + } + + df = pd.read_csv(url, dtype=dtypes, na_values='*') + + log.info(df.head(5)) + + + for i,row in df.iterrows(): + query = build_insert_query(row, df.columns.tolist(), 'life_expectancy_2018') + + job_input.execute_query(query) diff --git a/examples/life-expectancy/life-expectancy/config.ini b/examples/life-expectancy/life-expectancy/config.ini new file mode 100644 index 0000000000..3d9df38abe --- /dev/null +++ b/examples/life-expectancy/life-expectancy/config.ini @@ -0,0 +1,32 @@ +; Supported format: https://docs.python.org/3/library/configparser.html#supported-ini-file-structure + +; This is the only file required to deploy a Data Job. +; Read more to understand what each option means: + +; Information about the owner of the Data Job +[owner] + +; Team is a way to group Data Jobs that belonged to the same team. +team = my-team + +; Configuration related to running data jobs +[job] +; For format see https://en.wikipedia.org/wiki/Cron +; The cron expression is evaluated in UTC time. +; If it is time for a new job run and the previous job run hasn’t finished yet, +; the cron job waits until the previous execution has finished. +schedule_cron = 11 23 5 8 1 + +[vdk] +; Key value pairs of any configuration options that can be passed to vdk. +; For possible options in your vdk installation execute command vdk config-help +db_default_type=TRINO +ingest_method_default = trino + +trino_catalog = mysql +trino_use_ssl = +trino_host = localhost +trino_port = 8080 +trino_user = root +trino_schema = life-expectancy +trino_ssl_verify = diff --git a/examples/life-expectancy/life-expectancy/requirements.txt b/examples/life-expectancy/life-expectancy/requirements.txt new file mode 100644 index 0000000000..659a4163f4 --- /dev/null +++ b/examples/life-expectancy/life-expectancy/requirements.txt @@ -0,0 +1,4 @@ +inspect +math +numpy +pandas From c24d65257ef3c0a780567e019e60f21ab1d5ecf5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 10 Dec 2021 11:11:50 +0000 Subject: [PATCH 02/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/README.md | 2 +- examples/life-expectancy/README.md | 7 +--- examples/life-expectancy/le_utils/__init__.py | 3 ++ examples/life-expectancy/le_utils/utils.py | 15 +++++--- ...delete_table_life_expectancy_2010_2015.sql | 2 +- .../20_delete_table_life_expectancy_2018.sql | 2 +- ...create_table_life_expectancy_2010_2015.sql | 2 -- .../40_create_table_life_expectancy_2018.sql | 2 +- ..._ingest_table_life_expectancy_2010_2015.py | 36 ++++++++++--------- .../60_ingest_table_life_expectancy_2018.py | 30 ++++++++-------- .../life-expectancy/config.ini | 4 +-- 11 files changed, 56 insertions(+), 49 deletions(-) diff --git a/examples/README.md b/examples/README.md index a15d0101b3..e3be0d6a9b 100644 --- a/examples/README.md +++ b/examples/README.md @@ -16,4 +16,4 @@ Tutorials to get started with generic processing tasks in VDK: Tutorials with a complete pipeline from ingestion to final publication: * [Life Expectancy](life-expectancy/) * [Online Exhibition](online-exhibition/) -* [Energy](energy/) \ No newline at end of file +* [Energy](energy/) diff --git a/examples/life-expectancy/README.md b/examples/life-expectancy/README.md index c88abadbe5..ec8a600adb 100644 --- a/examples/life-expectancy/README.md +++ b/examples/life-expectancy/README.md @@ -141,7 +141,7 @@ The Life Expectancy Data Job runs with the following configuration (`config.ini` db_default_type=TRINO ingest_method_default = trino trino_catalog = mysql -trino_use_ssl = +trino_use_ssl = trino_host = localhost trino_port = 8080 trino_user = root @@ -160,8 +160,3 @@ The path to the CSV file is specified as a URL, thus this example requires an ac Jobs from 10 to 60 are devoted to Data Ingestion: * 10 - 30 ingest data in table `life_expectancy_2010_2015` * 40 - 60 ingest data in table `life_expectancy_2018` - - - - - diff --git a/examples/life-expectancy/le_utils/__init__.py b/examples/life-expectancy/le_utils/__init__.py index e69de29bb2..2bb2feb8c9 100644 --- a/examples/life-expectancy/le_utils/__init__.py +++ b/examples/life-expectancy/le_utils/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2021 VMware, Inc. +# SPDX-License-Identifier: Apache-2.0 + diff --git a/examples/life-expectancy/le_utils/utils.py b/examples/life-expectancy/le_utils/utils.py index 9a3bb58bf9..847b813756 100644 --- a/examples/life-expectancy/le_utils/utils.py +++ b/examples/life-expectancy/le_utils/utils.py @@ -1,6 +1,11 @@ +# Copyright 2021 VMware, Inc. +# SPDX-License-Identifier: Apache-2.0 + import math + import numpy as np + def build_insert_query(row, cols, output_table): """ Build an formatted query for insertion. @@ -16,12 +21,12 @@ def build_insert_query(row, cols, output_table): values = "" keys = cols.copy() for index in range(0, len(row)): - + # manage NaN values, both for strings and numeric values if (row[index] != row[index]) and (math.isnan(row[index])): # do not insert NaN values, thus remove it from columns keys.remove(cols[index]) - + else: # quote only strings and convert numeric values to strings if isinstance(row[index], str): @@ -29,8 +34,8 @@ def build_insert_query(row, cols, output_table): else: values = values + ", " + str(row[index]) - # remove the first comma + # remove the first comma values = values[1:] - cols = ', '.join([str(i) for i in keys]) - return f'INSERT INTO {output_table} ({cols}) VALUES ({values})' \ No newline at end of file + cols = ", ".join([str(i) for i in keys]) + return f"INSERT INTO {output_table} ({cols}) VALUES ({values})" diff --git a/examples/life-expectancy/life-expectancy/10_delete_table_life_expectancy_2010_2015.sql b/examples/life-expectancy/life-expectancy/10_delete_table_life_expectancy_2010_2015.sql index e6520a710f..f40394ebf4 100644 --- a/examples/life-expectancy/life-expectancy/10_delete_table_life_expectancy_2010_2015.sql +++ b/examples/life-expectancy/life-expectancy/10_delete_table_life_expectancy_2010_2015.sql @@ -1 +1 @@ -DROP TABLE IF EXISTS life_expectancy_2010_2015 \ No newline at end of file +DROP TABLE IF EXISTS life_expectancy_2010_2015 diff --git a/examples/life-expectancy/life-expectancy/20_delete_table_life_expectancy_2018.sql b/examples/life-expectancy/life-expectancy/20_delete_table_life_expectancy_2018.sql index 942f550f70..885866133a 100644 --- a/examples/life-expectancy/life-expectancy/20_delete_table_life_expectancy_2018.sql +++ b/examples/life-expectancy/life-expectancy/20_delete_table_life_expectancy_2018.sql @@ -1 +1 @@ -DROP TABLE IF EXISTS life_expectancy_2018 \ No newline at end of file +DROP TABLE IF EXISTS life_expectancy_2018 diff --git a/examples/life-expectancy/life-expectancy/30_create_table_life_expectancy_2010_2015.sql b/examples/life-expectancy/life-expectancy/30_create_table_life_expectancy_2010_2015.sql index f4f0ccbfab..ee5ab4746d 100644 --- a/examples/life-expectancy/life-expectancy/30_create_table_life_expectancy_2010_2015.sql +++ b/examples/life-expectancy/life-expectancy/30_create_table_life_expectancy_2010_2015.sql @@ -6,5 +6,3 @@ CREATE TABLE life_expectancy_2010_2015 ( LifeExpectancyRange varchar, LifeExpectancyStandardError decimal(4,2) ) - - diff --git a/examples/life-expectancy/life-expectancy/40_create_table_life_expectancy_2018.sql b/examples/life-expectancy/life-expectancy/40_create_table_life_expectancy_2018.sql index 1426f0eb1f..a9bc0f005e 100644 --- a/examples/life-expectancy/life-expectancy/40_create_table_life_expectancy_2018.sql +++ b/examples/life-expectancy/life-expectancy/40_create_table_life_expectancy_2018.sql @@ -4,4 +4,4 @@ CREATE TABLE life_expectancy_2018 ( LEB decimal(3,1), SE decimal(3,1), Quartile varchar(32) -) \ No newline at end of file +) diff --git a/examples/life-expectancy/life-expectancy/50_ingest_table_life_expectancy_2010_2015.py b/examples/life-expectancy/life-expectancy/50_ingest_table_life_expectancy_2010_2015.py index e015a4535a..f51ed00f57 100644 --- a/examples/life-expectancy/life-expectancy/50_ingest_table_life_expectancy_2010_2015.py +++ b/examples/life-expectancy/life-expectancy/50_ingest_table_life_expectancy_2010_2015.py @@ -1,13 +1,17 @@ +# Copyright 2021 VMware, Inc. +# SPDX-License-Identifier: Apache-2.0 + import inspect import logging import os +import sys + import numpy as np import pandas as pd -import sys current_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) parent_dir = os.path.dirname(current_dir) -sys.path.insert(0, parent_dir) +sys.path.insert(0, parent_dir) from le_utils.utils import build_insert_query @@ -16,7 +20,6 @@ log = logging.getLogger(__name__) - def run(job_input: IJobInput): """ Download datasets required by the scenario and put them in the data directory. @@ -24,23 +27,24 @@ def run(job_input: IJobInput): log.info(f"Starting job step {__name__}") # url of the U.S. Life Expectancy at Birth by State and Census Tract - 2010-2015 dataset - url = 'http://data.cdc.gov/api/views/5h56-n989/rows.csv' + url = "http://data.cdc.gov/api/views/5h56-n989/rows.csv" dtypes = { - 'State' : str, - 'County' : str, - 'Census Tract Number' : str, - 'Life Expectancy' : np.float64, - 'Life Expectancy Range' : str, - 'Life Expectancy Standard Error' : np.float64 + "State": str, + "County": str, + "Census Tract Number": str, + "Life Expectancy": np.float64, + "Life Expectancy Range": str, + "Life Expectancy Standard Error": np.float64, } df = pd.read_csv(url, dtype=dtypes) - - df.columns = df.columns.str.replace(' ', '') - - for i,row in df.iterrows(): - query = build_insert_query(row, df.columns.tolist(), 'life_expectancy_2010_2015') - + df.columns = df.columns.str.replace(" ", "") + + for i, row in df.iterrows(): + query = build_insert_query( + row, df.columns.tolist(), "life_expectancy_2010_2015" + ) + job_input.execute_query(query) diff --git a/examples/life-expectancy/life-expectancy/60_ingest_table_life_expectancy_2018.py b/examples/life-expectancy/life-expectancy/60_ingest_table_life_expectancy_2018.py index 0e755d7b93..233c0af3e4 100644 --- a/examples/life-expectancy/life-expectancy/60_ingest_table_life_expectancy_2018.py +++ b/examples/life-expectancy/life-expectancy/60_ingest_table_life_expectancy_2018.py @@ -1,13 +1,17 @@ +# Copyright 2021 VMware, Inc. +# SPDX-License-Identifier: Apache-2.0 + import inspect import logging import os +import sys + import numpy as np import pandas as pd -import sys current_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) parent_dir = os.path.dirname(current_dir) -sys.path.insert(0, parent_dir) +sys.path.insert(0, parent_dir) from le_utils.utils import build_insert_query @@ -16,7 +20,6 @@ log = logging.getLogger(__name__) - def run(job_input: IJobInput): """ Download datasets required by the scenario and put them in the data directory. @@ -24,22 +27,21 @@ def run(job_input: IJobInput): log.info(f"Starting job step {__name__}") # url of the U.S. Life Expectancy at Birth by State and Census Tract - 2010-2015 dataset - url = 'http://data.cdc.gov/api/views/a5a8-jsrq/rows.csv' + url = "http://data.cdc.gov/api/views/a5a8-jsrq/rows.csv" dtypes = { - 'State' : str, - 'Sex' : str, - 'LEB' : np.float64, - 'SE' : np.float64, - 'Quartile' : str, + "State": str, + "Sex": str, + "LEB": np.float64, + "SE": np.float64, + "Quartile": str, } - df = pd.read_csv(url, dtype=dtypes, na_values='*') + df = pd.read_csv(url, dtype=dtypes, na_values="*") log.info(df.head(5)) - - for i,row in df.iterrows(): - query = build_insert_query(row, df.columns.tolist(), 'life_expectancy_2018') - + for i, row in df.iterrows(): + query = build_insert_query(row, df.columns.tolist(), "life_expectancy_2018") + job_input.execute_query(query) diff --git a/examples/life-expectancy/life-expectancy/config.ini b/examples/life-expectancy/life-expectancy/config.ini index 3d9df38abe..3634d140a6 100644 --- a/examples/life-expectancy/life-expectancy/config.ini +++ b/examples/life-expectancy/life-expectancy/config.ini @@ -24,9 +24,9 @@ db_default_type=TRINO ingest_method_default = trino trino_catalog = mysql -trino_use_ssl = +trino_use_ssl = trino_host = localhost trino_port = 8080 trino_user = root trino_schema = life-expectancy -trino_ssl_verify = +trino_ssl_verify = From 40a359b612a6f63d0d8a97c41c65bbbc5e545d45 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 10 Dec 2021 18:29:56 +0000 Subject: [PATCH 03/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/life-expectancy/le_utils/__init__.py | 1 - examples/life-expectancy/le_utils/utils.py | 1 - .../life-expectancy/50_ingest_table_life_expectancy_2010_2015.py | 1 - .../life-expectancy/60_ingest_table_life_expectancy_2018.py | 1 - 4 files changed, 4 deletions(-) diff --git a/examples/life-expectancy/le_utils/__init__.py b/examples/life-expectancy/le_utils/__init__.py index 2bb2feb8c9..50c007580a 100644 --- a/examples/life-expectancy/le_utils/__init__.py +++ b/examples/life-expectancy/le_utils/__init__.py @@ -1,3 +1,2 @@ # Copyright 2021 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 - diff --git a/examples/life-expectancy/le_utils/utils.py b/examples/life-expectancy/le_utils/utils.py index 847b813756..fab3e79872 100644 --- a/examples/life-expectancy/le_utils/utils.py +++ b/examples/life-expectancy/le_utils/utils.py @@ -1,6 +1,5 @@ # Copyright 2021 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 - import math import numpy as np diff --git a/examples/life-expectancy/life-expectancy/50_ingest_table_life_expectancy_2010_2015.py b/examples/life-expectancy/life-expectancy/50_ingest_table_life_expectancy_2010_2015.py index f51ed00f57..18e5671ed1 100644 --- a/examples/life-expectancy/life-expectancy/50_ingest_table_life_expectancy_2010_2015.py +++ b/examples/life-expectancy/life-expectancy/50_ingest_table_life_expectancy_2010_2015.py @@ -1,6 +1,5 @@ # Copyright 2021 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 - import inspect import logging import os diff --git a/examples/life-expectancy/life-expectancy/60_ingest_table_life_expectancy_2018.py b/examples/life-expectancy/life-expectancy/60_ingest_table_life_expectancy_2018.py index 233c0af3e4..033f974763 100644 --- a/examples/life-expectancy/life-expectancy/60_ingest_table_life_expectancy_2018.py +++ b/examples/life-expectancy/life-expectancy/60_ingest_table_life_expectancy_2018.py @@ -1,6 +1,5 @@ # Copyright 2021 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 - import inspect import logging import os From f5c840ac95d2e3c0d9833ec87b6a8c840eecb87f Mon Sep 17 00:00:00 2001 From: Angelica Lo Duca Date: Fri, 10 Dec 2021 19:55:19 +0100 Subject: [PATCH 04/28] bug fix --- .../50_ingest_table_life_expectancy_2010_2015.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/life-expectancy/life-expectancy/50_ingest_table_life_expectancy_2010_2015.py b/examples/life-expectancy/life-expectancy/50_ingest_table_life_expectancy_2010_2015.py index 18e5671ed1..ee51a160c4 100644 --- a/examples/life-expectancy/life-expectancy/50_ingest_table_life_expectancy_2010_2015.py +++ b/examples/life-expectancy/life-expectancy/50_ingest_table_life_expectancy_2010_2015.py @@ -37,7 +37,7 @@ def run(job_input: IJobInput): "Life Expectancy Standard Error": np.float64, } - df = pd.read_csv(url, dtype=dtypes) + df = pd.read_csv(url, dtype=dtypes).replace("'","''", regex=True) df.columns = df.columns.str.replace(" ", "") From 466bdd40ccb0c695401b07eb6678f271903272f7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 10 Dec 2021 18:57:32 +0000 Subject: [PATCH 05/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../50_ingest_table_life_expectancy_2010_2015.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/life-expectancy/life-expectancy/50_ingest_table_life_expectancy_2010_2015.py b/examples/life-expectancy/life-expectancy/50_ingest_table_life_expectancy_2010_2015.py index ee51a160c4..4a7f6a29b4 100644 --- a/examples/life-expectancy/life-expectancy/50_ingest_table_life_expectancy_2010_2015.py +++ b/examples/life-expectancy/life-expectancy/50_ingest_table_life_expectancy_2010_2015.py @@ -37,7 +37,7 @@ def run(job_input: IJobInput): "Life Expectancy Standard Error": np.float64, } - df = pd.read_csv(url, dtype=dtypes).replace("'","''", regex=True) + df = pd.read_csv(url, dtype=dtypes).replace("'", "''", regex=True) df.columns = df.columns.str.replace(" ", "") From bc52977cbc012a6fb173f0742ee32111d37f3534 Mon Sep 17 00:00:00 2001 From: Angelica Lo Duca Date: Thu, 16 Dec 2021 10:17:13 +0100 Subject: [PATCH 06/28] Reviewed Data Ingestion and Added Part of Data Processing Data ingestion still remains with SQL INSERT (to be fixed) Added Data Cleaning with the creation of two tables. Since Trino does not support the creation of views with MySQL, two new cleaned tables are added. --- examples/life-expectancy/README.md | 55 +++++++++++++++++++ ...elete_table_life_expectancy_2010_2015.sql} | 0 ... 02_delete_table_life_expectancy_2018.sql} | 0 ...reate_table_life_expectancy_2010_2015.sql} | 0 ... 04_create_table_life_expectancy_2018.sql} | 0 ...ingest_table_life_expectancy_2010_2015.py} | 14 +++-- ...> 06_ingest_table_life_expectancy_2018.py} | 0 ...able_cleaned_life_expectancy_2010_2015.sql | 1 + ...able_cleaned_life_expectancy_2010_2015.sql | 9 +++ ...ete_table_cleaned_life_expectancy_2018.sql | 1 + ...ate_table_cleaned_life_expectancy_2018.sql | 9 +++ 11 files changed, 83 insertions(+), 6 deletions(-) rename examples/life-expectancy/life-expectancy/{10_delete_table_life_expectancy_2010_2015.sql => 01_delete_table_life_expectancy_2010_2015.sql} (100%) rename examples/life-expectancy/life-expectancy/{20_delete_table_life_expectancy_2018.sql => 02_delete_table_life_expectancy_2018.sql} (100%) rename examples/life-expectancy/life-expectancy/{30_create_table_life_expectancy_2010_2015.sql => 03_create_table_life_expectancy_2010_2015.sql} (100%) rename examples/life-expectancy/life-expectancy/{40_create_table_life_expectancy_2018.sql => 04_create_table_life_expectancy_2018.sql} (100%) rename examples/life-expectancy/life-expectancy/{50_ingest_table_life_expectancy_2010_2015.py => 05_ingest_table_life_expectancy_2010_2015.py} (73%) rename examples/life-expectancy/life-expectancy/{60_ingest_table_life_expectancy_2018.py => 06_ingest_table_life_expectancy_2018.py} (100%) create mode 100644 examples/life-expectancy/life-expectancy/07_delete_table_cleaned_life_expectancy_2010_2015.sql create mode 100644 examples/life-expectancy/life-expectancy/08_create_table_cleaned_life_expectancy_2010_2015.sql create mode 100644 examples/life-expectancy/life-expectancy/09_delete_table_cleaned_life_expectancy_2018.sql create mode 100644 examples/life-expectancy/life-expectancy/10_create_table_cleaned_life_expectancy_2018.sql diff --git a/examples/life-expectancy/README.md b/examples/life-expectancy/README.md index ec8a600adb..49431463f1 100644 --- a/examples/life-expectancy/README.md +++ b/examples/life-expectancy/README.md @@ -160,3 +160,58 @@ The path to the CSV file is specified as a URL, thus this example requires an ac Jobs from 10 to 60 are devoted to Data Ingestion: * 10 - 30 ingest data in table `life_expectancy_2010_2015` * 40 - 60 ingest data in table `life_expectancy_2018` + +### Clean tables +Table cleaning includes jobs from XX to YY. +Cleaning the `life_expectancy_2010_2015` table includes the following two operations: +* group records by County +* split the column `LifeExpectancyRange` in two decimal columns `MinLifeExpectancyRange` and `MaxLifeExpectancyRange`. + +The output of the cleaning process for the `life_expectancy_2010_2015` table is stored in a new table, called `cleaned_life_expectancy_2010_2015`. + +The following table shows an example of the `cleaned_life_expectancy_2010_2015` table: + +| **State** | **LifeExpectancy** | **MinLifeExpectancyRange** | **MaxLifeExpectancyRange** | **LifeExpectancyStandardError** | +| ------------- | ------------------------- | -------------------------- | -------------------------- | -------------------------------------- | +| **Florida** | 78.38 | 74.10 | 82.15 | 1.94 | +| **Georgia** | 76.62 | 70.38 | 79.17 | 1.82 | +| **Hawaii** | 81.32 | 79.09 | 88.06 | 1.94 | +| **New York** | 80.33 | 77.74 | 85.87 | 1.97 | +| **Kansas** | 78.05 | 73.95 | 81.30 | 1.81 | +| **Louisiana** | 75.37 | 67.82 | 77.87 | 1.83 | +| **Oregon** | 79.09 | 76.45 | 82.41 | 1.76 | +| **Arizona** | 78.37 | 74.97 | 81.47 | 2.00 | +| **Alabama** | 74.81 | 65.64 | 77.27 | 1.72 | +| **Missouri** | 76.96 | 71.37 | 79.92 + +Cleaning the `life_expectancy_2010_2015` table includes the following operations: +* rename column `LEB` to `LifeExpectancy` +* rename column `SE` to `LifeExpectancyStandardError` +* split the column `Quartile` in two decimal columns `MinLifeExpectancyRange` and `MaxLifeExpectancyRange` +* select only rows with `Sex = 'Total'`. + +The following table shows an example of the `cleaned_life_expectancy_2018` table: + +| **State** | **LifeExpectancy** | **MinlifeExpectancyRange** | **MaxLifeExpectancyRange** | **LifeExpectancyStandardError** | +| ------------------ | ------------------ | -------------------------- | -------------------------- | ------------------------------- | +| **West Virginia** | 74.4 | 74.40 | 77.20 | 0.1 | +| **Mississippi** | 74.6 | 74.40 | 77.20 | 0.1 | +| **Alabama** | 75.1 | 74.40 | 77.20 | 0.1 | +| **Kentucky** | 75.3 | 74.40 | 77.20 | 0.1 | +| **Tennessee** | 75.5 | 74.40 | 77.20 | 0.1 | +| **Arkansas** | 75.6 | 74.40 | 77.20 | 0.1 | +| **Oklahoma** | 75.6 | 74.40 | 77.20 | 0.1 | +| **Louisiana** | 75.6 | 74.40 | 77.20 | 0.1 | +| **South Carolina** | 76.5 | 74.40 | 77.20 | 0.1 | +| **Missouri** | 76.6 | 74.40 | 77.20 | 0.1 | + +### Merge the two datasets +* Build a view that shows only the top 10 states with the greatest life expectancy + * Order dataset + * Take only the first 10 states + + + + + + diff --git a/examples/life-expectancy/life-expectancy/10_delete_table_life_expectancy_2010_2015.sql b/examples/life-expectancy/life-expectancy/01_delete_table_life_expectancy_2010_2015.sql similarity index 100% rename from examples/life-expectancy/life-expectancy/10_delete_table_life_expectancy_2010_2015.sql rename to examples/life-expectancy/life-expectancy/01_delete_table_life_expectancy_2010_2015.sql diff --git a/examples/life-expectancy/life-expectancy/20_delete_table_life_expectancy_2018.sql b/examples/life-expectancy/life-expectancy/02_delete_table_life_expectancy_2018.sql similarity index 100% rename from examples/life-expectancy/life-expectancy/20_delete_table_life_expectancy_2018.sql rename to examples/life-expectancy/life-expectancy/02_delete_table_life_expectancy_2018.sql diff --git a/examples/life-expectancy/life-expectancy/30_create_table_life_expectancy_2010_2015.sql b/examples/life-expectancy/life-expectancy/03_create_table_life_expectancy_2010_2015.sql similarity index 100% rename from examples/life-expectancy/life-expectancy/30_create_table_life_expectancy_2010_2015.sql rename to examples/life-expectancy/life-expectancy/03_create_table_life_expectancy_2010_2015.sql diff --git a/examples/life-expectancy/life-expectancy/40_create_table_life_expectancy_2018.sql b/examples/life-expectancy/life-expectancy/04_create_table_life_expectancy_2018.sql similarity index 100% rename from examples/life-expectancy/life-expectancy/40_create_table_life_expectancy_2018.sql rename to examples/life-expectancy/life-expectancy/04_create_table_life_expectancy_2018.sql diff --git a/examples/life-expectancy/life-expectancy/50_ingest_table_life_expectancy_2010_2015.py b/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py similarity index 73% rename from examples/life-expectancy/life-expectancy/50_ingest_table_life_expectancy_2010_2015.py rename to examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py index 4a7f6a29b4..4b8058f7ba 100644 --- a/examples/life-expectancy/life-expectancy/50_ingest_table_life_expectancy_2010_2015.py +++ b/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py @@ -41,9 +41,11 @@ def run(job_input: IJobInput): df.columns = df.columns.str.replace(" ", "") - for i, row in df.iterrows(): - query = build_insert_query( - row, df.columns.tolist(), "life_expectancy_2010_2015" - ) - - job_input.execute_query(query) + #records = [row for index, row in df.iterrows()] + job_input.send_tabular_data_for_ingestion(df.itertuples(index=False), destination_table="life_expectancy_2010_2015", column_names=df.columns, method='trino') + #for i, row in df.iterrows(): + # query = build_insert_query( + # row, df.columns.tolist(), "life_expectancy_2010_2015" + # ) + + # job_input.execute_query(query) diff --git a/examples/life-expectancy/life-expectancy/60_ingest_table_life_expectancy_2018.py b/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py similarity index 100% rename from examples/life-expectancy/life-expectancy/60_ingest_table_life_expectancy_2018.py rename to examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py diff --git a/examples/life-expectancy/life-expectancy/07_delete_table_cleaned_life_expectancy_2010_2015.sql b/examples/life-expectancy/life-expectancy/07_delete_table_cleaned_life_expectancy_2010_2015.sql new file mode 100644 index 0000000000..da39f900d9 --- /dev/null +++ b/examples/life-expectancy/life-expectancy/07_delete_table_cleaned_life_expectancy_2010_2015.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS cleaned_life_expectancy_2010_2015 diff --git a/examples/life-expectancy/life-expectancy/08_create_table_cleaned_life_expectancy_2010_2015.sql b/examples/life-expectancy/life-expectancy/08_create_table_cleaned_life_expectancy_2010_2015.sql new file mode 100644 index 0000000000..a04774827d --- /dev/null +++ b/examples/life-expectancy/life-expectancy/08_create_table_cleaned_life_expectancy_2010_2015.sql @@ -0,0 +1,9 @@ +CREATE TABLE cleaned_life_expectancy_2010_2015 AS +(SELECT State, + LifeExpectancy, + cast(split(life_expectancy_2010_2015.LifeExpectancyRange,'-')[1] AS decimal(4,2)) AS MinLifeExpectancyRange, + cast(split(life_expectancy_2010_2015.LifeExpectancyRange,'-')[2] AS decimal(4,2)) AS MaxLifeExpectancyRange, + LifeExpectancyStandardError +FROM life_expectancy_2010_2015 +WHERE County = '(blank)' +) diff --git a/examples/life-expectancy/life-expectancy/09_delete_table_cleaned_life_expectancy_2018.sql b/examples/life-expectancy/life-expectancy/09_delete_table_cleaned_life_expectancy_2018.sql new file mode 100644 index 0000000000..4d3d5b14ab --- /dev/null +++ b/examples/life-expectancy/life-expectancy/09_delete_table_cleaned_life_expectancy_2018.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS cleaned_life_expectancy_2018 diff --git a/examples/life-expectancy/life-expectancy/10_create_table_cleaned_life_expectancy_2018.sql b/examples/life-expectancy/life-expectancy/10_create_table_cleaned_life_expectancy_2018.sql new file mode 100644 index 0000000000..5d047572bd --- /dev/null +++ b/examples/life-expectancy/life-expectancy/10_create_table_cleaned_life_expectancy_2018.sql @@ -0,0 +1,9 @@ +CREATE TABLE cleaned_life_expectancy_2018 AS +(SELECT State, + LEB AS LifeExpectancy, + cast(split(life_expectancy_2018.Quartile,' - ')[1] AS decimal(4,2)) AS MinLifeExpectancyRange, + cast(split(life_expectancy_2018.Quartile,' - ')[2] AS decimal(4,2)) AS MaxLifeExpectancyRange, + SE AS LifeExpectancyStandardError +FROM life_expectancy_2018 +WHERE Sex = 'Total' and State <> 'United States' +) From 02a699dd28c116988265c4bf9bfa5be845808709 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 Dec 2021 09:18:19 +0000 Subject: [PATCH 07/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/life-expectancy/README.md | 10 ++-------- .../05_ingest_table_life_expectancy_2010_2015.py | 11 ++++++++--- ...create_table_cleaned_life_expectancy_2010_2015.sql | 2 +- .../10_create_table_cleaned_life_expectancy_2018.sql | 2 +- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/examples/life-expectancy/README.md b/examples/life-expectancy/README.md index 49431463f1..bd7bc115af 100644 --- a/examples/life-expectancy/README.md +++ b/examples/life-expectancy/README.md @@ -162,7 +162,7 @@ Jobs from 10 to 60 are devoted to Data Ingestion: * 40 - 60 ingest data in table `life_expectancy_2018` ### Clean tables -Table cleaning includes jobs from XX to YY. +Table cleaning includes jobs from XX to YY. Cleaning the `life_expectancy_2010_2015` table includes the following two operations: * group records by County * split the column `LifeExpectancyRange` in two decimal columns `MinLifeExpectancyRange` and `MaxLifeExpectancyRange`. @@ -182,7 +182,7 @@ The following table shows an example of the `cleaned_life_expectancy_2010_2015` | **Oregon** | 79.09 | 76.45 | 82.41 | 1.76 | | **Arizona** | 78.37 | 74.97 | 81.47 | 2.00 | | **Alabama** | 74.81 | 65.64 | 77.27 | 1.72 | -| **Missouri** | 76.96 | 71.37 | 79.92 +| **Missouri** | 76.96 | 71.37 | 79.92 Cleaning the `life_expectancy_2010_2015` table includes the following operations: * rename column `LEB` to `LifeExpectancy` @@ -209,9 +209,3 @@ The following table shows an example of the `cleaned_life_expectancy_2018` table * Build a view that shows only the top 10 states with the greatest life expectancy * Order dataset * Take only the first 10 states - - - - - - diff --git a/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py b/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py index 4b8058f7ba..4205bf37c0 100644 --- a/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py +++ b/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py @@ -41,9 +41,14 @@ def run(job_input: IJobInput): df.columns = df.columns.str.replace(" ", "") - #records = [row for index, row in df.iterrows()] - job_input.send_tabular_data_for_ingestion(df.itertuples(index=False), destination_table="life_expectancy_2010_2015", column_names=df.columns, method='trino') - #for i, row in df.iterrows(): + # records = [row for index, row in df.iterrows()] + job_input.send_tabular_data_for_ingestion( + df.itertuples(index=False), + destination_table="life_expectancy_2010_2015", + column_names=df.columns, + method="trino", + ) + # for i, row in df.iterrows(): # query = build_insert_query( # row, df.columns.tolist(), "life_expectancy_2010_2015" # ) diff --git a/examples/life-expectancy/life-expectancy/08_create_table_cleaned_life_expectancy_2010_2015.sql b/examples/life-expectancy/life-expectancy/08_create_table_cleaned_life_expectancy_2010_2015.sql index a04774827d..9f2bac9402 100644 --- a/examples/life-expectancy/life-expectancy/08_create_table_cleaned_life_expectancy_2010_2015.sql +++ b/examples/life-expectancy/life-expectancy/08_create_table_cleaned_life_expectancy_2010_2015.sql @@ -1,5 +1,5 @@ CREATE TABLE cleaned_life_expectancy_2010_2015 AS -(SELECT State, +(SELECT State, LifeExpectancy, cast(split(life_expectancy_2010_2015.LifeExpectancyRange,'-')[1] AS decimal(4,2)) AS MinLifeExpectancyRange, cast(split(life_expectancy_2010_2015.LifeExpectancyRange,'-')[2] AS decimal(4,2)) AS MaxLifeExpectancyRange, diff --git a/examples/life-expectancy/life-expectancy/10_create_table_cleaned_life_expectancy_2018.sql b/examples/life-expectancy/life-expectancy/10_create_table_cleaned_life_expectancy_2018.sql index 5d047572bd..1452450a81 100644 --- a/examples/life-expectancy/life-expectancy/10_create_table_cleaned_life_expectancy_2018.sql +++ b/examples/life-expectancy/life-expectancy/10_create_table_cleaned_life_expectancy_2018.sql @@ -1,5 +1,5 @@ CREATE TABLE cleaned_life_expectancy_2018 AS -(SELECT State, +(SELECT State, LEB AS LifeExpectancy, cast(split(life_expectancy_2018.Quartile,' - ')[1] AS decimal(4,2)) AS MinLifeExpectancyRange, cast(split(life_expectancy_2018.Quartile,' - ')[2] AS decimal(4,2)) AS MaxLifeExpectancyRange, From c12164410bf806f59ec44c729f19235ebdf0a80d Mon Sep 17 00:00:00 2001 From: Angelica Lo Duca Date: Thu, 16 Dec 2021 10:21:57 +0100 Subject: [PATCH 08/28] Updated Readme --- examples/life-expectancy/README.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/examples/life-expectancy/README.md b/examples/life-expectancy/README.md index 49431463f1..364b3de231 100644 --- a/examples/life-expectancy/README.md +++ b/examples/life-expectancy/README.md @@ -158,11 +158,19 @@ Data Ingestion uploads in the database the two CSV tables, defined in the Data S The path to the CSV file is specified as a URL, thus this example requires an active Internet connection to work properly. Jobs from 10 to 60 are devoted to Data Ingestion: -* 10 - 30 ingest data in table `life_expectancy_2010_2015` -* 40 - 60 ingest data in table `life_expectancy_2018` +* 01 - 03 ingest data in table `life_expectancy_2010_2015` +* 04 - 06 ingest data in table `life_expectancy_2018` +## Data Processing +Data Processing includes the following tasks: +* Clean tables +* Merge the cleaned datasets + +### Clean tables ### Clean tables -Table cleaning includes jobs from XX to YY. +Table cleaning includes jobs from 07 to 10. + +** Tasks 07 - 08 ** Cleaning the `life_expectancy_2010_2015` table includes the following two operations: * group records by County * split the column `LifeExpectancyRange` in two decimal columns `MinLifeExpectancyRange` and `MaxLifeExpectancyRange`. @@ -184,6 +192,7 @@ The following table shows an example of the `cleaned_life_expectancy_2010_2015` | **Alabama** | 74.81 | 65.64 | 77.27 | 1.72 | | **Missouri** | 76.96 | 71.37 | 79.92 +** Tasks 09 - 10 ** Cleaning the `life_expectancy_2010_2015` table includes the following operations: * rename column `LEB` to `LifeExpectancy` * rename column `SE` to `LifeExpectancyStandardError` From a20159e71c0b9dda48cc4e7787a32faa855c2e57 Mon Sep 17 00:00:00 2001 From: Angelica Lo Duca Date: Thu, 16 Dec 2021 10:24:50 +0100 Subject: [PATCH 09/28] Updated Readme --- examples/life-expectancy/README.md | 224 +++++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 examples/life-expectancy/README.md diff --git a/examples/life-expectancy/README.md b/examples/life-expectancy/README.md new file mode 100644 index 0000000000..401f8ff5ff --- /dev/null +++ b/examples/life-expectancy/README.md @@ -0,0 +1,224 @@ +# Life Expectancy +This example provides the complete code to perform the following tasks: +- Data Ingestion from CSV +- Data Processing +- Data Publication in form of Visual Report/Dashboard + +The directory is organized as follows: +* **life-expectancy** - contains jobs +* **le_utils** - contains some auxiliary_functions. + +## Scenario + +Researchers from the Department of Population History and Social Structure have to conduct a study on the life expectancy of Americans. The objective of the study is to understand over time which American states have the greatest life expectancy. + +## Data Scoures +After extensive research, the researchers found two main datasets, both of which are available as CSV files: + +- [U.S. Life Expectancy at Birth by State and Census Tract - 2010-2015](https://catalog.data.gov/dataset/u-s-life-expectancy-at-birth-by-state-and-census-tract-2010-2015) +- [U.S. State Life Expectancy by Sex, 2018](https://catalog.data.gov/dataset/u-s-state-life-expectancy-by-sex-2018) + +The dataset **U.S. Life Expectancy at Birth by State and Census Tract - 2010-2015** contains 73,121 records relating to the life expectancy of Americans, divided by state and county and relating to the period 2010-2015. For each record, the following information is available: + +- State +- County +- Census Tract Number +- Life Expectancy +- Life Expectancy Range +- Life Expectancy Standard Error + +For some records, some fields are missing. + +The following table shows an excerpt from the dataset: + +| **State** | **County** | **Census Tract Number** | **Life Expectancy** | **Life Expectancy Range** | **Life Expectancy Standard Error** | +| --- | --- | --- | --- | --- | --- | +| **Alabama** | (blank) | | 75.5 | 75.2-77.5 | 0.0328 | +| **Alabama** | Autauga County, AL | 0201.00 | 73.1 | 56.9-75.1 | 2.2348 | +| **Alabama** | Autauga County, AL | 0202.00 | 76.9 | 75.2-77.5 | 3.3453 | +| **Alabama** | Autauga County, AL | 0203.00 | | | | +| **Alabama** | Autauga County, AL | 0204.00 | 75.4 | 75.2-77.5 | 1.0216 | +| **Alabama** | Autauga County, AL | 0205.00 | 79.4 | 77.6-79.5 | 1.1768 | +| **Alabama** | Autauga County, AL | 0206.00 | 73.1 | 56.9-75.1 | 1.5519 | +| **Alabama** | Autauga County, AL | 0207.00 | | | | +| **Alabama** | Autauga County, AL | 0208.01 | 78.3 | 77.6-79.5 | 2.3861 | +| **Alabama** | Autauga County, AL | 0208.02 | 76.9 | 75.2-77.5 | 1.2628 | +| **Alabama** | Autauga County, AL | 0209.00 | 73.9 | 56.9-75.1 | 1.5923 | + +The dataset **U.S. State Life Expectancy by Sex, 2018** contains 156 records relating to the life expectancy of Americans in 2018, divided by state and sex (male, female, total). For each record, the following information is available: + +- State +- Sex +- LEB - Life Expectancy at birth +- SE - Life Expectancy Standard Error +- Quartile - Life Expectancy Range + +The following table shows an excerpt from the dataset: + +| **State** | **Sex** | **LEB** | **SE** | **Quartile** | +| --- | --- | --- | --- | --- | +| **United States** | Total | 78.7 | | \* | +| **West Virginia** | Total | 74.4 | 114 | 74.4 - 77.2 | +| **Mississippi** | Total | 74.6 | 88 | 74.4 - 77.2 | +| **Alabama** | Total | 75.1 | 67 | 74.4 - 77.2 | +| **Kentucky** | Total | 75.3 | 68 | 74.4 - 77.2 | +| **Tennessee** | Total | 75.5 | 57 | 74.4 - 77.2 | +| **Arkansas** | Total | 75.6 | 86 | 74.4 - 77.2 | +| **Oklahoma** | Total | 75.6 | 73 | 74.4 - 77.2 | + +## Requirements +To run this example, you need: +* Versatile Data Kit +* Trino DB +* Versatile Data Kit plugin for Trino + +### Versatile Data Kit +If you have not done so already, you can install Versatile Data Kit and the plugins required for this example by running the following commands from a terminal: + +``` +pip install quickstart-vdk +``` + +Note that Versatile Data Kit requires Python 3.7+. See the [Installation Page](https://github.com/vmware/versatile-data-kit/wiki/Installation#install-sdk "Installation page") for more details. + +### Trino DB +This example also requires Trino DB installed. See the Trino [Official Documentation](https://trino.io/ "Official Documentation") for more details about installation. + +### Versatile Data Kit Plugin for Trino +Since this example requires Trino, you should also install the Versatile Data Kit plugin for Trino: + +``` +pip install vdk-trino +``` + +See the vdk-trino [Documentation Page](https://github.com/vmware/versatile-data-kit/tree/main/projects/vdk-plugins/vdk-trino "Documentation Page") for more details. + +### Other Requirements +This example also requires the following Python libraries, which are included in the `requirement.txt` file: + +``` +inspect +math +numpy +pandas +``` + +## Configuration + +### Trino DB +In this example Trino is running locally, with the following minimal `config.properties` configuration file: + +``` +coordinator=true +node-scheduler.include-coordinator=true +http-server.http.port=8080 +query.max-memory=5GB +query.max-memory-per-node=1GB +query.max-total-memory-per-node=2GB +discovery.uri=http://127.0.0.1:8080 +http-server.https.enabled=false + +``` + +In addition, the Trino DB exploits the MySQL catalog, with the following configuration (file `mysql.properties` located in the catalog folder of the Trino server: + +``` +connector.name=mysql +connection-url=jdbc:mysql://localhost:3306 +connection-user=root +connection-password= +allow-drop-table=true +``` + +More complex configurations can be used too. + +Finally, this example assumes that an empty schema, called `life-expectancy` exists on the MySQL server. + +### Versatile Data Kit +The Life Expectancy Data Job runs with the following configuration (`config.ini`): + +``` +db_default_type=TRINO +ingest_method_default = trino +trino_catalog = mysql +trino_use_ssl = +trino_host = localhost +trino_port = 8080 +trino_user = root +trino_schema = life-expectancy +trino_ssl_verify = +``` + +## Data Ingestion +Data Ingestion uploads in the database the two CSV tables, defined in the Data Source section. For each table, data Ingestion is performed through the following steps: +* delete the existing table (if any) +* create a new table +* ingest table values directly from the CSV file. + +The path to the CSV file is specified as a URL, thus this example requires an active Internet connection to work properly. + +Jobs from 10 to 60 are devoted to Data Ingestion: +* 01 - 03 ingest data in table `life_expectancy_2010_2015` +* 04 - 06 ingest data in table `life_expectancy_2018` + +## Data Processing +Data Processing includes the following tasks: +* Clean tables +* Merge the cleaned datasets + +### Clean tables +### Clean tables +<<<<<<< HEAD +Table cleaning includes jobs from 07 to 10. + +** Tasks 07 - 08 ** +======= +Table cleaning includes jobs from XX to YY. +>>>>>>> 02a699dd28c116988265c4bf9bfa5be845808709 +Cleaning the `life_expectancy_2010_2015` table includes the following two operations: +* group records by County +* split the column `LifeExpectancyRange` in two decimal columns `MinLifeExpectancyRange` and `MaxLifeExpectancyRange`. + +The output of the cleaning process for the `life_expectancy_2010_2015` table is stored in a new table, called `cleaned_life_expectancy_2010_2015`. + +The following table shows an example of the `cleaned_life_expectancy_2010_2015` table: + +| **State** | **LifeExpectancy** | **MinLifeExpectancyRange** | **MaxLifeExpectancyRange** | **LifeExpectancyStandardError** | +| ------------- | ------------------------- | -------------------------- | -------------------------- | -------------------------------------- | +| **Florida** | 78.38 | 74.10 | 82.15 | 1.94 | +| **Georgia** | 76.62 | 70.38 | 79.17 | 1.82 | +| **Hawaii** | 81.32 | 79.09 | 88.06 | 1.94 | +| **New York** | 80.33 | 77.74 | 85.87 | 1.97 | +| **Kansas** | 78.05 | 73.95 | 81.30 | 1.81 | +| **Louisiana** | 75.37 | 67.82 | 77.87 | 1.83 | +| **Oregon** | 79.09 | 76.45 | 82.41 | 1.76 | +| **Arizona** | 78.37 | 74.97 | 81.47 | 2.00 | +| **Alabama** | 74.81 | 65.64 | 77.27 | 1.72 | +| **Missouri** | 76.96 | 71.37 | 79.92 + +** Tasks 09 - 10 ** +Cleaning the `life_expectancy_2010_2015` table includes the following operations: +* rename column `LEB` to `LifeExpectancy` +* rename column `SE` to `LifeExpectancyStandardError` +* split the column `Quartile` in two decimal columns `MinLifeExpectancyRange` and `MaxLifeExpectancyRange` +* select only rows with `Sex = 'Total'`. + +The following table shows an example of the `cleaned_life_expectancy_2018` table: + +| **State** | **LifeExpectancy** | **MinlifeExpectancyRange** | **MaxLifeExpectancyRange** | **LifeExpectancyStandardError** | +| ------------------ | ------------------ | -------------------------- | -------------------------- | ------------------------------- | +| **West Virginia** | 74.4 | 74.40 | 77.20 | 0.1 | +| **Mississippi** | 74.6 | 74.40 | 77.20 | 0.1 | +| **Alabama** | 75.1 | 74.40 | 77.20 | 0.1 | +| **Kentucky** | 75.3 | 74.40 | 77.20 | 0.1 | +| **Tennessee** | 75.5 | 74.40 | 77.20 | 0.1 | +| **Arkansas** | 75.6 | 74.40 | 77.20 | 0.1 | +| **Oklahoma** | 75.6 | 74.40 | 77.20 | 0.1 | +| **Louisiana** | 75.6 | 74.40 | 77.20 | 0.1 | +| **South Carolina** | 76.5 | 74.40 | 77.20 | 0.1 | +| **Missouri** | 76.6 | 74.40 | 77.20 | 0.1 | + +### Merge the two datasets +* Build a view that shows only the top 10 states with the greatest life expectancy + * Order dataset + * Take only the first 10 states From 4c9bff2915d78d23491571357b8160a2509f5b48 Mon Sep 17 00:00:00 2001 From: Angelica Lo Duca Date: Thu, 16 Dec 2021 10:26:04 +0100 Subject: [PATCH 10/28] Updated Readme --- examples/life-expectancy/README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/examples/life-expectancy/README.md b/examples/life-expectancy/README.md index 401f8ff5ff..6a3743ae0b 100644 --- a/examples/life-expectancy/README.md +++ b/examples/life-expectancy/README.md @@ -168,13 +168,9 @@ Data Processing includes the following tasks: ### Clean tables ### Clean tables -<<<<<<< HEAD Table cleaning includes jobs from 07 to 10. ** Tasks 07 - 08 ** -======= -Table cleaning includes jobs from XX to YY. ->>>>>>> 02a699dd28c116988265c4bf9bfa5be845808709 Cleaning the `life_expectancy_2010_2015` table includes the following two operations: * group records by County * split the column `LifeExpectancyRange` in two decimal columns `MinLifeExpectancyRange` and `MaxLifeExpectancyRange`. From b7ec895c4619e1d31d4149d44c597f60555c09d2 Mon Sep 17 00:00:00 2001 From: Angelica Lo Duca Date: Thu, 16 Dec 2021 10:26:59 +0100 Subject: [PATCH 11/28] Update README.md --- examples/life-expectancy/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/life-expectancy/README.md b/examples/life-expectancy/README.md index 6a3743ae0b..7e2c5dea79 100644 --- a/examples/life-expectancy/README.md +++ b/examples/life-expectancy/README.md @@ -170,7 +170,7 @@ Data Processing includes the following tasks: ### Clean tables Table cleaning includes jobs from 07 to 10. -** Tasks 07 - 08 ** +**Tasks 07 - 08** Cleaning the `life_expectancy_2010_2015` table includes the following two operations: * group records by County * split the column `LifeExpectancyRange` in two decimal columns `MinLifeExpectancyRange` and `MaxLifeExpectancyRange`. @@ -192,7 +192,7 @@ The following table shows an example of the `cleaned_life_expectancy_2010_2015` | **Alabama** | 74.81 | 65.64 | 77.27 | 1.72 | | **Missouri** | 76.96 | 71.37 | 79.92 -** Tasks 09 - 10 ** +**Tasks 09 - 10** Cleaning the `life_expectancy_2010_2015` table includes the following operations: * rename column `LEB` to `LifeExpectancy` * rename column `SE` to `LifeExpectancyStandardError` From 938f59401e9b0e6d24e5f5fb0504ceb7133fd3fb Mon Sep 17 00:00:00 2001 From: Angelica Lo Duca Date: Thu, 16 Dec 2021 10:27:30 +0100 Subject: [PATCH 12/28] Updated Readme --- examples/life-expectancy/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/life-expectancy/README.md b/examples/life-expectancy/README.md index 7e2c5dea79..8ad9c9143c 100644 --- a/examples/life-expectancy/README.md +++ b/examples/life-expectancy/README.md @@ -166,7 +166,6 @@ Data Processing includes the following tasks: * Clean tables * Merge the cleaned datasets -### Clean tables ### Clean tables Table cleaning includes jobs from 07 to 10. From 955885e76fa694c39abc0c6522f349367aed1eca Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 Dec 2021 09:29:16 +0000 Subject: [PATCH 13/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/life-expectancy/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/life-expectancy/README.md b/examples/life-expectancy/README.md index 8ad9c9143c..130f31ec83 100644 --- a/examples/life-expectancy/README.md +++ b/examples/life-expectancy/README.md @@ -167,9 +167,9 @@ Data Processing includes the following tasks: * Merge the cleaned datasets ### Clean tables -Table cleaning includes jobs from 07 to 10. +Table cleaning includes jobs from 07 to 10. -**Tasks 07 - 08** +**Tasks 07 - 08** Cleaning the `life_expectancy_2010_2015` table includes the following two operations: * group records by County * split the column `LifeExpectancyRange` in two decimal columns `MinLifeExpectancyRange` and `MaxLifeExpectancyRange`. @@ -191,7 +191,7 @@ The following table shows an example of the `cleaned_life_expectancy_2010_2015` | **Alabama** | 74.81 | 65.64 | 77.27 | 1.72 | | **Missouri** | 76.96 | 71.37 | 79.92 -**Tasks 09 - 10** +**Tasks 09 - 10** Cleaning the `life_expectancy_2010_2015` table includes the following operations: * rename column `LEB` to `LifeExpectancy` * rename column `SE` to `LifeExpectancyStandardError` From 5fbd30dba82ea87cea994c8b2d4c2fadc7ef7446 Mon Sep 17 00:00:00 2001 From: Angelica Lo Duca Date: Thu, 16 Dec 2021 10:36:26 +0100 Subject: [PATCH 14/28] Update README.md --- examples/life-expectancy/README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/examples/life-expectancy/README.md b/examples/life-expectancy/README.md index 8ad9c9143c..5e1f6e0cc3 100644 --- a/examples/life-expectancy/README.md +++ b/examples/life-expectancy/README.md @@ -178,18 +178,18 @@ The output of the cleaning process for the `life_expectancy_2010_2015` table is The following table shows an example of the `cleaned_life_expectancy_2010_2015` table: -| **State** | **LifeExpectancy** | **MinLifeExpectancyRange** | **MaxLifeExpectancyRange** | **LifeExpectancyStandardError** | -| ------------- | ------------------------- | -------------------------- | -------------------------- | -------------------------------------- | -| **Florida** | 78.38 | 74.10 | 82.15 | 1.94 | -| **Georgia** | 76.62 | 70.38 | 79.17 | 1.82 | -| **Hawaii** | 81.32 | 79.09 | 88.06 | 1.94 | -| **New York** | 80.33 | 77.74 | 85.87 | 1.97 | -| **Kansas** | 78.05 | 73.95 | 81.30 | 1.81 | -| **Louisiana** | 75.37 | 67.82 | 77.87 | 1.83 | -| **Oregon** | 79.09 | 76.45 | 82.41 | 1.76 | -| **Arizona** | 78.37 | 74.97 | 81.47 | 2.00 | -| **Alabama** | 74.81 | 65.64 | 77.27 | 1.72 | -| **Missouri** | 76.96 | 71.37 | 79.92 +| **State** | **LifeExpectancy** | **MinlifeExpectancyRange** | **MaxLifeExpectancyRange** | **LifeExpectancyStandardError** | +| ------------------------ | ------------------ | -------------------------- | -------------------------- | ------------------------------- | +| **Alabama** | 75.50 | 75.20 | 77.50 | 0.03 | +| **Alaska** | 78.80 | 77.60 | 79.50 | 0.10 | +| **Arizona** | 79.90 | 79.60 | 81.60 | 0.03 | +| **Arkansas** | 76.00 | 75.20 | 77.50 | 0.04 | +| **California** | 81.30 | 79.60 | 81.60 | 0.01 | +| **Colorado** | 80.50 | 79.60 | 81.60 | 0.03 | +| **Connecticut** | 80.90 | 79.60 | 81.60 | 0.04 | +| **Delaware** | 78.70 | 77.60 | 79.50 | 0.08 | +| **District of Columbia** | 78.50 | 77.60 | 79.50 | 0.10 | +| **Florida** | 80.10 | 79.60 | 81.60 | 0.02 | **Tasks 09 - 10** Cleaning the `life_expectancy_2010_2015` table includes the following operations: From 644a0efe2b575e5a9f577e7499a36594b8ff33c0 Mon Sep 17 00:00:00 2001 From: Angelica Lo Duca Date: Thu, 16 Dec 2021 11:32:31 +0100 Subject: [PATCH 15/28] Update README.md --- examples/life-expectancy/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/life-expectancy/README.md b/examples/life-expectancy/README.md index bc6175a00d..6952300c07 100644 --- a/examples/life-expectancy/README.md +++ b/examples/life-expectancy/README.md @@ -178,7 +178,7 @@ The output of the cleaning process for the `life_expectancy_2010_2015` table is The following table shows an example of the `cleaned_life_expectancy_2010_2015` table: -| **State** | **LifeExpectancy** | **MinlifeExpectancyRange** | **MaxLifeExpectancyRange** | **LifeExpectancyStandardError** | +| **State** | **LifeExpectancy** | **MinLifeExpectancyRange** | **MaxLifeExpectancyRange** | **LifeExpectancyStandardError** | | ------------------------ | ------------------ | -------------------------- | -------------------------- | ------------------------------- | | **Alabama** | 75.50 | 75.20 | 77.50 | 0.03 | | **Alaska** | 78.80 | 77.60 | 79.50 | 0.10 | @@ -200,7 +200,7 @@ Cleaning the `life_expectancy_2010_2015` table includes the following operations The following table shows an example of the `cleaned_life_expectancy_2018` table: -| **State** | **LifeExpectancy** | **MinlifeExpectancyRange** | **MaxLifeExpectancyRange** | **LifeExpectancyStandardError** | +| **State** | **LifeExpectancy** | **MinLifeExpectancyRange** | **MaxLifeExpectancyRange** | **LifeExpectancyStandardError** | | ------------------ | ------------------ | -------------------------- | -------------------------- | ------------------------------- | | **West Virginia** | 74.4 | 74.40 | 77.20 | 0.1 | | **Mississippi** | 74.6 | 74.40 | 77.20 | 0.1 | From 3871fe4ee39227939f90efed8fd86dae7668613a Mon Sep 17 00:00:00 2001 From: Angelica Lo Duca Date: Tue, 21 Dec 2021 12:12:44 +0100 Subject: [PATCH 16/28] Completed Life Expectancy Scenario New tables have been added as well as a dashboard. --- examples/life-expectancy/README.md | 108 ++++++++++++++++-- examples/life-expectancy/dashboard.py | 101 ++++++++++++++++ ..._ingest_table_life_expectancy_2010_2015.py | 26 ++--- .../06_ingest_table_life_expectancy_2018.py | 4 +- .../07_delete_table_us_regions.sql | 1 + .../08_create_table_us_regions.sql | 6 + .../09_ingest_table_us_regions.py | 43 +++++++ .../10_delete_table_us_gdp.sql | 1 + .../11_create_table_us_gdp.sql | 7 ++ .../life-expectancy/12_ingest_table_us_gdp.py | 47 ++++++++ ...ble_cleaned_life_expectancy_2010_2015.sql} | 0 ...ble_cleaned_life_expectancy_2010_2015.sql} | 0 ...te_table_cleaned_life_expectancy_2018.sql} | 0 ...te_table_cleaned_life_expectancy_2018.sql} | 0 ...17_delete_table_merged_life_expectancy.sql | 1 + ...18_create_table_merged_life_expectancy.sql | 24 ++++ 16 files changed, 342 insertions(+), 27 deletions(-) create mode 100644 examples/life-expectancy/dashboard.py create mode 100644 examples/life-expectancy/life-expectancy/07_delete_table_us_regions.sql create mode 100644 examples/life-expectancy/life-expectancy/08_create_table_us_regions.sql create mode 100644 examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py create mode 100644 examples/life-expectancy/life-expectancy/10_delete_table_us_gdp.sql create mode 100644 examples/life-expectancy/life-expectancy/11_create_table_us_gdp.sql create mode 100644 examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py rename examples/life-expectancy/life-expectancy/{07_delete_table_cleaned_life_expectancy_2010_2015.sql => 13_delete_table_cleaned_life_expectancy_2010_2015.sql} (100%) rename examples/life-expectancy/life-expectancy/{08_create_table_cleaned_life_expectancy_2010_2015.sql => 14_create_table_cleaned_life_expectancy_2010_2015.sql} (100%) rename examples/life-expectancy/life-expectancy/{09_delete_table_cleaned_life_expectancy_2018.sql => 15_delete_table_cleaned_life_expectancy_2018.sql} (100%) rename examples/life-expectancy/life-expectancy/{10_create_table_cleaned_life_expectancy_2018.sql => 16_create_table_cleaned_life_expectancy_2018.sql} (100%) create mode 100644 examples/life-expectancy/life-expectancy/17_delete_table_merged_life_expectancy.sql create mode 100644 examples/life-expectancy/life-expectancy/18_create_table_merged_life_expectancy.sql diff --git a/examples/life-expectancy/README.md b/examples/life-expectancy/README.md index 6952300c07..7023958298 100644 --- a/examples/life-expectancy/README.md +++ b/examples/life-expectancy/README.md @@ -18,6 +18,10 @@ After extensive research, the researchers found two main datasets, both of which - [U.S. Life Expectancy at Birth by State and Census Tract - 2010-2015](https://catalog.data.gov/dataset/u-s-life-expectancy-at-birth-by-state-and-census-tract-2010-2015) - [U.S. State Life Expectancy by Sex, 2018](https://catalog.data.gov/dataset/u-s-state-life-expectancy-by-sex-2018) +In addition to the previous datasets, the researchers found the following additional datasets: +- [U.S. Gross Domestic Product by County](https://www.bea.gov/data/gdp/gdp-county-metro-and-other-areas) +- [U.S. Census Bureau Regions and Divisions](https://raw.githubusercontent.com/cphalpert/census-regions/master/us%20census%20bureau%20regions%20and%20divisions.csv) + The dataset **U.S. Life Expectancy at Birth by State and Census Tract - 2010-2015** contains 73,121 records relating to the life expectancy of Americans, divided by state and county and relating to the period 2010-2015. For each record, the following information is available: - State @@ -66,11 +70,54 @@ The following table shows an excerpt from the dataset: | **Arkansas** | Total | 75.6 | 86 | 74.4 - 77.2 | | **Oklahoma** | Total | 75.6 | 73 | 74.4 - 77.2 | +The dataset **U.S. Gross Domestic Product by County** contains 3,163 records, related to the U.S. real Gross Domestic Product, by County, referring to the period years 2017–2020. +The dataset contains many columns. For each record, the following information is extracted: +- County +- 2017 +- 2018 +- 2019 +- 2020 + +The following table shows an excerpt from the dataset: + +| **County** | **2017** | **2018** | **2019** | **2020** | +| ----------------- | ------------ | ------------ | ------------ | ------------ | +| **United States** | 18079084000 | 18606787000 | 19032672000 | 18384687000 | +| **Alabama** | 197566622 | 200800889 | 203383898 | 196906061 | +| **Autauga** | 1587695 | 1602077 | 1540762 | 1520973 | +| **Baldwin** | 6453588 | 6799715 | 7134734 | 6985901 | +| **Barbour** | 721125 | 730518 | 729105 | 687074 | +| **Bibb** | 353234 | 353016 | 380453 | 388330 | +| **Blount** | 920401 | 967135 | 932215 | 881874 | +| **Bullock** | 225304 | 235358 | 247229 | 250385 | +| **Butler** | 584855 | 609244 | 615641 | 574717 | + +The dataset **U.S. Census Bureau Regions and Divisions** contains 51 records related to mapping between each U.S. County and its Region. For each record, the following information is extracted: +- State +- State Code +- Region +- Division + +The following table shows an excerpt from the dataset: + +| **State** | **StateCode** | **Region** | **Division** | +| ------------------------ | ------------- | ---------- | ------------------ | +| **Alaska** | AK | West | Pacific | +| **Alabama** | AL | South | East South Central | +| **Arkansas** | AR | South | West South Central | +| **Arizona** | AZ | West | Mountain | +| **California** | CA | West | Pacific | +| **Colorado** | CO | West | Mountain | +| **Connecticut** | CT | Northeast | New England | +| **District of Columbia** | DC | South | South Atlantic | +| **Delaware** | DE | South | South Atlantic | + ## Requirements To run this example, you need: * Versatile Data Kit * Trino DB * Versatile Data Kit plugin for Trino +* Streamlit ### Versatile Data Kit If you have not done so already, you can install Versatile Data Kit and the plugins required for this example by running the following commands from a terminal: @@ -91,7 +138,17 @@ Since this example requires Trino, you should also install the Versatile Data Ki pip install vdk-trino ``` -See the vdk-trino [Documentation Page](https://github.com/vmware/versatile-data-kit/tree/main/projects/vdk-plugins/vdk-trino "Documentation Page") for more details. +See the vdk-trino [Documentation Page](https://github.com/vmware/versatile-data-kit/tree/main/projects/vdk-plugins/vdk-trino) for more details. + +### Streamlit +The final dashboard/report is built in [streamlit](https://streamlit.io/). +You can install streamlit through the following command: + +``` +pip install streamlit +``` + +See the streamlit [Documentation Page](https://docs.streamlit.io/library/get-started/installation) for more details. ### Other Requirements This example also requires the following Python libraries, which are included in the `requirement.txt` file: @@ -103,6 +160,11 @@ numpy pandas ``` +The following Python libraries are required by the dashboard: +``` +altair +``` + ## Configuration ### Trino DB @@ -157,19 +219,20 @@ Data Ingestion uploads in the database the two CSV tables, defined in the Data S The path to the CSV file is specified as a URL, thus this example requires an active Internet connection to work properly. -Jobs from 10 to 60 are devoted to Data Ingestion: +Jobs from 01 to 12 are devoted to Data Ingestion: * 01 - 03 ingest data in table `life_expectancy_2010_2015` * 04 - 06 ingest data in table `life_expectancy_2018` - +* 07 - 09 ingest data in table `us_regions` +* 10 - 12 ingest data in table `us_gdp` ## Data Processing Data Processing includes the following tasks: * Clean tables -* Merge the cleaned datasets +* Merge the cleaned tables ### Clean tables -Table cleaning includes jobs from 07 to 10. +Table cleaning includes jobs from 13 to 18. -**Tasks 07 - 08** +**Tasks 13 - 14** Cleaning the `life_expectancy_2010_2015` table includes the following two operations: * group records by County * split the column `LifeExpectancyRange` in two decimal columns `MinLifeExpectancyRange` and `MaxLifeExpectancyRange`. @@ -191,8 +254,8 @@ The following table shows an example of the `cleaned_life_expectancy_2010_2015` | **District of Columbia** | 78.50 | 77.60 | 79.50 | 0.10 | | **Florida** | 80.10 | 79.60 | 81.60 | 0.02 | -**Tasks 09 - 10** -Cleaning the `life_expectancy_2010_2015` table includes the following operations: +**Tasks 15 - 16** +Cleaning the `life_expectancy_2018` table includes the following operations: * rename column `LEB` to `LifeExpectancy` * rename column `SE` to `LifeExpectancyStandardError` * split the column `Quartile` in two decimal columns `MinLifeExpectancyRange` and `MaxLifeExpectancyRange` @@ -213,7 +276,28 @@ The following table shows an example of the `cleaned_life_expectancy_2018` table | **South Carolina** | 76.5 | 74.40 | 77.20 | 0.1 | | **Missouri** | 76.6 | 74.40 | 77.20 | 0.1 | -### Merge the two datasets -* Build a view that shows only the top 10 states with the greatest life expectancy - * Order dataset - * Take only the first 10 states +### Merge the cleaned tables +Jobs 17 and 18 are devoted to vertical merging between the two cleaned datasets `cleaned_life_expectancy_2010_2015`, `cleaned_life_expectancy_2018`, `us_gdp` and `us_regions`. Vertical merging means that the second dataset is appended to the first dataset and three new columns, called `Period`, `GDP` and `Region`, are added to the resulting table, named `merged_life_expectancy`. +The `GDP` attribute is set only for records with `Period = '2018'`. For the other records, it is set to `0`, since it is not available. + +The following table shows an example of the `merged_life_expectancy` table: + +| **State** | **LifeExpectancy** | **MinlifeExpectancyRange** | **MaxLifeExpectancyRange** | **Period** | **Region** | **GDP** | +| --------------- | ------------------ | -------------------------- | -------------------------- | ---------- | ---------- | --------- | +| **Oklahoma** | 75.60 | 74.40 | 77.20 | 2018 | South | 197358323 | +| **Connecticut** | 80.90 | 79.60 | 81.60 | 2010-2015 | Northeast | 0 | +| **Michigan** | 78.20 | 77.60 | 79.50 | 2010-2015 | Midwest | 0 | +| **Mississippi** | 74.90 | 56.90 | 75.10 | 2010-2015 | South | 0 | +| **Nebraska** | 79.60 | 79.60 | 81.60 | 2010-2015 | Midwest | 0 | +| **Virginia** | 79.00 | 78.70 | 79.30 | 2018 | South | 477819754 | +| **Alaska** | 78.80 | 77.60 | 79.50 | 2010-2015 | West | 0 | +| **California** | 81.30 | 79.60 | 81.60 | 2010-2015 | West | 0 | +| **Ohio** | 77.60 | 77.60 | 79.50 | 2010-2015 | Midwest | 0 | + +## Data Publication +The `dashboard.py` script contains a dashboard/report on the Life Expectancy. +To run the report, simply run the following command: + +``` +streamlit run dashboard.py +``` diff --git a/examples/life-expectancy/dashboard.py b/examples/life-expectancy/dashboard.py new file mode 100644 index 0000000000..7e2fabfd85 --- /dev/null +++ b/examples/life-expectancy/dashboard.py @@ -0,0 +1,101 @@ +import altair as alt +import numpy as np +import pandas as pd +import streamlit as st +import math +import trino + +# connect to server +conn = trino.dbapi.connect( + host='localhost', + port=8080, + user='root', + catalog='mysql', + http_scheme='HTTP', + verify=True, + schema='life-expectancy', +) + +palette = ['#ff9494', '#ff5252', '#8B0000','#000000'] + +# Design the UI +st.markdown("# Life Expectancy in the US") +st.markdown("""According to the U.S. Small-area Life Expectancy Estimates Project (USALEEP), +in 2018 in the US, Life Expectancy was greater than 76 years old for all the Regions. +**Southern Regions had the lowest life expectancy**, with an average life expectancy of 76.66, +and Northeastern Regions the highest one, with an average life expectancy of 79.52. +A similar behaviour happened in the 2010-2015 period. +""") + +periods = pd.read_sql_query(f"SELECT DISTINCT(Period) FROM merged_life_expectancy", conn) +period = st.selectbox('Select Period', periods['Period'], key='period') +df = pd.read_sql_query(f"SELECT Region, avg(LifeExpectancy) AS AvgLifeExpectancy FROM merged_life_expectancy WHERE Period = '{period}' GROUP BY Region", conn) + +c = alt.Chart(df).mark_bar(size=90).encode( + y=alt.Y('AvgLifeExpectancy:Q', scale=alt.Scale(domain=[0,100]), axis=None), + x=alt.X('Region:N',axis=alt.Axis(labelAngle=0), title='US Regions'), + color=alt.condition( + alt.datum.Region == 'South', # If the year is 1810 this test returns True, + alt.value('#8B0000'), # which sets the bar orange. + alt.value('gray') # And if it's not true it sets the bar steelblue. + ) +).properties( + width = 500, +) + +text = c.mark_text( + align='center', + baseline='middle', + dy=-15 +).encode( + text='AvgLifeExpectancy:Q' +) + +bars = ( + c + text +).properties( + height=300, + title=f"Average Life Expectancy By Region ({period})" +).configure_axis( + grid=False, + domain=False +) +bars + +st.markdown("There is a correlation between the Life Expectancy and the Gross Domestic Product (GDP), as shown in the following graph:") + +df = pd.read_sql_query(f"SELECT State, LifeExpectancy, GDP, Region FROM merged_life_expectancy WHERE Period = '2018'", conn) + +scatter = alt.Chart(df).mark_circle(size=100).encode( + y=alt.Y('GDP:Q', title = 'GDP ($)', scale=alt.Scale( domain=[90e6,3e9])), + x=alt.X('LifeExpectancy:Q', title='Life Expectancy',scale=alt.Scale(domain=[74,81])), + tooltip='State:N', + color=alt.Color('Region:N', scale=alt.Scale(range=palette)) + +).interactive() +st.altair_chart(scatter, use_container_width=True) + +st.markdown("In fact, most of the Southern Regions have the smallest GDP (see the bottom left part of the graph).") + + +st.markdown("## Detailed Report") +st.markdown("The following table shows an overview of the Life Expectancy Dataset.") + +df = pd.read_sql_query("SELECT LifeExpectancy, State, Region, Period FROM merged_life_expectancy ORDER BY LifeExpectancy DESC", conn) +df + +st.markdown("Select the reference Period and the top-N US states to dynamically update the bar chart.") + +n_records_per_year = math.ceil(df.shape[0]/2) +period2 = st.selectbox('Select Period', periods['Period'], key='period2') +n_items = st.selectbox('Select the number of States to show', np.arange(5, n_records_per_year + 5, step=5)) + +df_sorted = df[df['Period'] == period2].sort_values(by='LifeExpectancy', ascending=False).head(n_items) +c = alt.Chart(df_sorted).mark_bar().encode( + x=alt.X('LifeExpectancy:Q', title="Life Expectancy"), + y=alt.Y('State:N', sort='-x'), + color=alt.Color('Region:N', scale=alt.Scale(range=palette)) +) + +st.altair_chart(c, use_container_width=True) + diff --git a/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py b/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py index 4205bf37c0..b3a2d2ad7a 100644 --- a/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py +++ b/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py @@ -21,7 +21,7 @@ def run(job_input: IJobInput): """ - Download datasets required by the scenario and put them in the data directory. + Download datasets required by the scenario and put them in the data lake. """ log.info(f"Starting job step {__name__}") @@ -42,15 +42,15 @@ def run(job_input: IJobInput): df.columns = df.columns.str.replace(" ", "") # records = [row for index, row in df.iterrows()] - job_input.send_tabular_data_for_ingestion( - df.itertuples(index=False), - destination_table="life_expectancy_2010_2015", - column_names=df.columns, - method="trino", - ) - # for i, row in df.iterrows(): - # query = build_insert_query( - # row, df.columns.tolist(), "life_expectancy_2010_2015" - # ) - - # job_input.execute_query(query) + #job_input.send_tabular_data_for_ingestion( + # df.itertuples(index=False), + # destination_table="life_expectancy_2010_2015", + # column_names=df.columns, + # method="trino", + #) + for i, row in df.iterrows(): + query = build_insert_query( + row, df.columns.tolist(), "life_expectancy_2010_2015" + ) + + job_input.execute_query(query) diff --git a/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py b/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py index 033f974763..59e47a8cd5 100644 --- a/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py +++ b/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py @@ -21,11 +21,11 @@ def run(job_input: IJobInput): """ - Download datasets required by the scenario and put them in the data directory. + Download datasets required by the scenario and store in the data lake. """ log.info(f"Starting job step {__name__}") - # url of the U.S. Life Expectancy at Birth by State and Census Tract - 2010-2015 dataset + # url of the U.S. Life Expectancy at Birth by State and Census Tract - 2018 dataset url = "http://data.cdc.gov/api/views/a5a8-jsrq/rows.csv" dtypes = { diff --git a/examples/life-expectancy/life-expectancy/07_delete_table_us_regions.sql b/examples/life-expectancy/life-expectancy/07_delete_table_us_regions.sql new file mode 100644 index 0000000000..efa1b5a19e --- /dev/null +++ b/examples/life-expectancy/life-expectancy/07_delete_table_us_regions.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS us_regions diff --git a/examples/life-expectancy/life-expectancy/08_create_table_us_regions.sql b/examples/life-expectancy/life-expectancy/08_create_table_us_regions.sql new file mode 100644 index 0000000000..bd5cdc23de --- /dev/null +++ b/examples/life-expectancy/life-expectancy/08_create_table_us_regions.sql @@ -0,0 +1,6 @@ +CREATE TABLE us_regions ( + State varchar(32), + StateCode varchar(2), + Region varchar(32), + Division varchar(32) +) diff --git a/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py b/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py new file mode 100644 index 0000000000..a32b677276 --- /dev/null +++ b/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py @@ -0,0 +1,43 @@ +# Copyright 2021 VMware, Inc. +# SPDX-License-Identifier: Apache-2.0 +import inspect +import logging +import os +import sys +import numpy as np +import pandas as pd + + +current_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) +parent_dir = os.path.dirname(current_dir) +sys.path.insert(0, parent_dir) + +from le_utils.utils import build_insert_query + +from vdk.api.job_input import IJobInput + +log = logging.getLogger(__name__) + + +def run(job_input: IJobInput): + """ + Download datasets required by the scenario and put them in the data data lake. + """ + log.info(f"Starting job step {__name__}") + + # url of the U.S. Life Expectancy at Birth by State and Census Tract - 2010-2015 dataset + url = "https://raw.githubusercontent.com/cphalpert/census-regions/master/us%20census%20bureau%20regions%20and%20divisions.csv" + + df = pd.read_csv(url, sep=',') + + df.columns = df.columns.str.replace(" ", "") + + + for i, row in df.iterrows(): + query = build_insert_query( + row, df.columns.tolist(), "us_regions" + ) + + job_input.execute_query(query) + + diff --git a/examples/life-expectancy/life-expectancy/10_delete_table_us_gdp.sql b/examples/life-expectancy/life-expectancy/10_delete_table_us_gdp.sql new file mode 100644 index 0000000000..37b698cc6e --- /dev/null +++ b/examples/life-expectancy/life-expectancy/10_delete_table_us_gdp.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS us_gdp diff --git a/examples/life-expectancy/life-expectancy/11_create_table_us_gdp.sql b/examples/life-expectancy/life-expectancy/11_create_table_us_gdp.sql new file mode 100644 index 0000000000..7d9e2974ec --- /dev/null +++ b/examples/life-expectancy/life-expectancy/11_create_table_us_gdp.sql @@ -0,0 +1,7 @@ +CREATE TABLE us_gdp ( + County varchar(32), + Year2017 bigint, + Year2018 bigint, + Year2019 bigint, + Year2020 bigint +) diff --git a/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py b/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py new file mode 100644 index 0000000000..7daeca8373 --- /dev/null +++ b/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py @@ -0,0 +1,47 @@ +# Copyright 2021 VMware, Inc. +# SPDX-License-Identifier: Apache-2.0 +import inspect +import logging +import os +import sys + +import numpy as np +import pandas as pd + + +current_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) +parent_dir = os.path.dirname(current_dir) +sys.path.insert(0, parent_dir) + +from le_utils.utils import build_insert_query + +from vdk.api.job_input import IJobInput + +log = logging.getLogger(__name__) + + +def run(job_input: IJobInput): + """ + Download datasets required by the scenario and put them in the data lake. + """ + log.info(f"Starting job step {__name__}") + + url = "https://www.bea.gov/sites/default/files/2021-12/lagdp1221.xlsx" + + df = pd.read_excel(url, header=3, na_values="(NA)").replace("'", "''", regex=True) + + # select only interesting rows + df = df[['Unnamed: 0', 2017,2018, 2019, 2020]] + df.rename({'Unnamed: 0' : 'County', 2017 : 'Year2017', 2018 : 'Year2018', 2019: 'Year2019', 2020: 'Year2020'}, axis=1, inplace=True) + df.dropna(axis=0, inplace=True) + + + for i, row in df.iterrows(): + log.info(row) + query = build_insert_query( + row, df.columns.tolist(), "us_gdp" + ) + + job_input.execute_query(query) + + diff --git a/examples/life-expectancy/life-expectancy/07_delete_table_cleaned_life_expectancy_2010_2015.sql b/examples/life-expectancy/life-expectancy/13_delete_table_cleaned_life_expectancy_2010_2015.sql similarity index 100% rename from examples/life-expectancy/life-expectancy/07_delete_table_cleaned_life_expectancy_2010_2015.sql rename to examples/life-expectancy/life-expectancy/13_delete_table_cleaned_life_expectancy_2010_2015.sql diff --git a/examples/life-expectancy/life-expectancy/08_create_table_cleaned_life_expectancy_2010_2015.sql b/examples/life-expectancy/life-expectancy/14_create_table_cleaned_life_expectancy_2010_2015.sql similarity index 100% rename from examples/life-expectancy/life-expectancy/08_create_table_cleaned_life_expectancy_2010_2015.sql rename to examples/life-expectancy/life-expectancy/14_create_table_cleaned_life_expectancy_2010_2015.sql diff --git a/examples/life-expectancy/life-expectancy/09_delete_table_cleaned_life_expectancy_2018.sql b/examples/life-expectancy/life-expectancy/15_delete_table_cleaned_life_expectancy_2018.sql similarity index 100% rename from examples/life-expectancy/life-expectancy/09_delete_table_cleaned_life_expectancy_2018.sql rename to examples/life-expectancy/life-expectancy/15_delete_table_cleaned_life_expectancy_2018.sql diff --git a/examples/life-expectancy/life-expectancy/10_create_table_cleaned_life_expectancy_2018.sql b/examples/life-expectancy/life-expectancy/16_create_table_cleaned_life_expectancy_2018.sql similarity index 100% rename from examples/life-expectancy/life-expectancy/10_create_table_cleaned_life_expectancy_2018.sql rename to examples/life-expectancy/life-expectancy/16_create_table_cleaned_life_expectancy_2018.sql diff --git a/examples/life-expectancy/life-expectancy/17_delete_table_merged_life_expectancy.sql b/examples/life-expectancy/life-expectancy/17_delete_table_merged_life_expectancy.sql new file mode 100644 index 0000000000..13d54bae40 --- /dev/null +++ b/examples/life-expectancy/life-expectancy/17_delete_table_merged_life_expectancy.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS merged_life_expectancy diff --git a/examples/life-expectancy/life-expectancy/18_create_table_merged_life_expectancy.sql b/examples/life-expectancy/life-expectancy/18_create_table_merged_life_expectancy.sql new file mode 100644 index 0000000000..fabb2c5124 --- /dev/null +++ b/examples/life-expectancy/life-expectancy/18_create_table_merged_life_expectancy.sql @@ -0,0 +1,24 @@ +CREATE TABLE merged_life_expectancy AS +(SELECT us_regions.State, + LifeExpectancy, + MinLifeExpectancyRange, + MaxLifeExpectancyRange, + '2010-2015' AS Period, + Region, + 0 AS GDP +FROM + cleaned_life_expectancy_2010_2015 JOIN us_regions ON us_regions.State = cleaned_life_expectancy_2010_2015.State +) +UNION +(SELECT us_regions.State, + LifeExpectancy, + MinLifeExpectancyRange, + MaxLifeExpectancyRange, + '2018' AS Period, + Region, + Year2018 AS GDP +FROM cleaned_life_expectancy_2018 + JOIN us_regions ON us_regions.State = cleaned_life_expectancy_2018.State + INNER JOIN us_gdp ON us_gdp.County = cleaned_life_expectancy_2018.State +WHERE Year2018 > 100000000 +) \ No newline at end of file From 58edbb490b83eccc5a24c378107e3397b05ea845 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Dec 2021 11:13:50 +0000 Subject: [PATCH 17/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/life-expectancy/README.md | 8 +- examples/life-expectancy/dashboard.py | 151 +++++++++++------- ..._ingest_table_life_expectancy_2010_2015.py | 4 +- .../09_ingest_table_us_regions.py | 10 +- .../life-expectancy/12_ingest_table_us_gdp.py | 21 ++- ...18_create_table_merged_life_expectancy.sql | 8 +- 6 files changed, 120 insertions(+), 82 deletions(-) diff --git a/examples/life-expectancy/README.md b/examples/life-expectancy/README.md index 7023958298..5d6bcbb838 100644 --- a/examples/life-expectancy/README.md +++ b/examples/life-expectancy/README.md @@ -70,13 +70,13 @@ The following table shows an excerpt from the dataset: | **Arkansas** | Total | 75.6 | 86 | 74.4 - 77.2 | | **Oklahoma** | Total | 75.6 | 73 | 74.4 - 77.2 | -The dataset **U.S. Gross Domestic Product by County** contains 3,163 records, related to the U.S. real Gross Domestic Product, by County, referring to the period years 2017–2020. +The dataset **U.S. Gross Domestic Product by County** contains 3,163 records, related to the U.S. real Gross Domestic Product, by County, referring to the period years 2017–2020. The dataset contains many columns. For each record, the following information is extracted: - County - 2017 - 2018 - 2019 -- 2020 +- 2020 The following table shows an excerpt from the dataset: @@ -141,7 +141,7 @@ pip install vdk-trino See the vdk-trino [Documentation Page](https://github.com/vmware/versatile-data-kit/tree/main/projects/vdk-plugins/vdk-trino) for more details. ### Streamlit -The final dashboard/report is built in [streamlit](https://streamlit.io/). +The final dashboard/report is built in [streamlit](https://streamlit.io/). You can install streamlit through the following command: ``` @@ -277,7 +277,7 @@ The following table shows an example of the `cleaned_life_expectancy_2018` table | **Missouri** | 76.6 | 74.40 | 77.20 | 0.1 | ### Merge the cleaned tables -Jobs 17 and 18 are devoted to vertical merging between the two cleaned datasets `cleaned_life_expectancy_2010_2015`, `cleaned_life_expectancy_2018`, `us_gdp` and `us_regions`. Vertical merging means that the second dataset is appended to the first dataset and three new columns, called `Period`, `GDP` and `Region`, are added to the resulting table, named `merged_life_expectancy`. +Jobs 17 and 18 are devoted to vertical merging between the two cleaned datasets `cleaned_life_expectancy_2010_2015`, `cleaned_life_expectancy_2018`, `us_gdp` and `us_regions`. Vertical merging means that the second dataset is appended to the first dataset and three new columns, called `Period`, `GDP` and `Region`, are added to the resulting table, named `merged_life_expectancy`. The `GDP` attribute is set only for records with `Period = '2018'`. For the other records, it is set to `0`, since it is not available. The following table shows an example of the `merged_life_expectancy` table: diff --git a/examples/life-expectancy/dashboard.py b/examples/life-expectancy/dashboard.py index 7e2fabfd85..2e85d13a22 100644 --- a/examples/life-expectancy/dashboard.py +++ b/examples/life-expectancy/dashboard.py @@ -1,101 +1,138 @@ +# Copyright 2021 VMware, Inc. +# SPDX-License-Identifier: Apache-2.0 + +import math + import altair as alt import numpy as np import pandas as pd import streamlit as st -import math import trino # connect to server conn = trino.dbapi.connect( - host='localhost', + host="localhost", port=8080, - user='root', - catalog='mysql', - http_scheme='HTTP', + user="root", + catalog="mysql", + http_scheme="HTTP", verify=True, - schema='life-expectancy', + schema="life-expectancy", ) -palette = ['#ff9494', '#ff5252', '#8B0000','#000000'] +palette = ["#ff9494", "#ff5252", "#8B0000", "#000000"] # Design the UI st.markdown("# Life Expectancy in the US") -st.markdown("""According to the U.S. Small-area Life Expectancy Estimates Project (USALEEP), +st.markdown( + """According to the U.S. Small-area Life Expectancy Estimates Project (USALEEP), in 2018 in the US, Life Expectancy was greater than 76 years old for all the Regions. **Southern Regions had the lowest life expectancy**, with an average life expectancy of 76.66, and Northeastern Regions the highest one, with an average life expectancy of 79.52. A similar behaviour happened in the 2010-2015 period. -""") - -periods = pd.read_sql_query(f"SELECT DISTINCT(Period) FROM merged_life_expectancy", conn) -period = st.selectbox('Select Period', periods['Period'], key='period') -df = pd.read_sql_query(f"SELECT Region, avg(LifeExpectancy) AS AvgLifeExpectancy FROM merged_life_expectancy WHERE Period = '{period}' GROUP BY Region", conn) - -c = alt.Chart(df).mark_bar(size=90).encode( - y=alt.Y('AvgLifeExpectancy:Q', scale=alt.Scale(domain=[0,100]), axis=None), - x=alt.X('Region:N',axis=alt.Axis(labelAngle=0), title='US Regions'), - color=alt.condition( - alt.datum.Region == 'South', # If the year is 1810 this test returns True, - alt.value('#8B0000'), # which sets the bar orange. - alt.value('gray') # And if it's not true it sets the bar steelblue. +""" +) + +periods = pd.read_sql_query( + f"SELECT DISTINCT(Period) FROM merged_life_expectancy", conn +) +period = st.selectbox("Select Period", periods["Period"], key="period") +df = pd.read_sql_query( + f"SELECT Region, avg(LifeExpectancy) AS AvgLifeExpectancy FROM merged_life_expectancy WHERE Period = '{period}' GROUP BY Region", + conn, +) + +c = ( + alt.Chart(df) + .mark_bar(size=90) + .encode( + y=alt.Y("AvgLifeExpectancy:Q", scale=alt.Scale(domain=[0, 100]), axis=None), + x=alt.X("Region:N", axis=alt.Axis(labelAngle=0), title="US Regions"), + color=alt.condition( + alt.datum.Region == "South", # If the year is 1810 this test returns True, + alt.value("#8B0000"), # which sets the bar orange. + alt.value("gray"), # And if it's not true it sets the bar steelblue. + ), + ) + .properties( + width=500, ) -).properties( - width = 500, ) -text = c.mark_text( - align='center', - baseline='middle', - dy=-15 -).encode( - text='AvgLifeExpectancy:Q' +text = c.mark_text(align="center", baseline="middle", dy=-15).encode( + text="AvgLifeExpectancy:Q" ) bars = ( - c + text -).properties( - height=300, - title=f"Average Life Expectancy By Region ({period})" -).configure_axis( - grid=False, - domain=False + (c + text) + .properties(height=300, title=f"Average Life Expectancy By Region ({period})") + .configure_axis(grid=False, domain=False) ) bars -st.markdown("There is a correlation between the Life Expectancy and the Gross Domestic Product (GDP), as shown in the following graph:") - -df = pd.read_sql_query(f"SELECT State, LifeExpectancy, GDP, Region FROM merged_life_expectancy WHERE Period = '2018'", conn) +st.markdown( + "There is a correlation between the Life Expectancy and the Gross Domestic Product (GDP), as shown in the following graph:" +) -scatter = alt.Chart(df).mark_circle(size=100).encode( - y=alt.Y('GDP:Q', title = 'GDP ($)', scale=alt.Scale( domain=[90e6,3e9])), - x=alt.X('LifeExpectancy:Q', title='Life Expectancy',scale=alt.Scale(domain=[74,81])), - tooltip='State:N', - color=alt.Color('Region:N', scale=alt.Scale(range=palette)) +df = pd.read_sql_query( + f"SELECT State, LifeExpectancy, GDP, Region FROM merged_life_expectancy WHERE Period = '2018'", + conn, +) -).interactive() +scatter = ( + alt.Chart(df) + .mark_circle(size=100) + .encode( + y=alt.Y("GDP:Q", title="GDP ($)", scale=alt.Scale(domain=[90e6, 3e9])), + x=alt.X( + "LifeExpectancy:Q", + title="Life Expectancy", + scale=alt.Scale(domain=[74, 81]), + ), + tooltip="State:N", + color=alt.Color("Region:N", scale=alt.Scale(range=palette)), + ) + .interactive() +) st.altair_chart(scatter, use_container_width=True) -st.markdown("In fact, most of the Southern Regions have the smallest GDP (see the bottom left part of the graph).") +st.markdown( + "In fact, most of the Southern Regions have the smallest GDP (see the bottom left part of the graph)." +) st.markdown("## Detailed Report") st.markdown("The following table shows an overview of the Life Expectancy Dataset.") -df = pd.read_sql_query("SELECT LifeExpectancy, State, Region, Period FROM merged_life_expectancy ORDER BY LifeExpectancy DESC", conn) +df = pd.read_sql_query( + "SELECT LifeExpectancy, State, Region, Period FROM merged_life_expectancy ORDER BY LifeExpectancy DESC", + conn, +) df -st.markdown("Select the reference Period and the top-N US states to dynamically update the bar chart.") +st.markdown( + "Select the reference Period and the top-N US states to dynamically update the bar chart." +) -n_records_per_year = math.ceil(df.shape[0]/2) -period2 = st.selectbox('Select Period', periods['Period'], key='period2') -n_items = st.selectbox('Select the number of States to show', np.arange(5, n_records_per_year + 5, step=5)) +n_records_per_year = math.ceil(df.shape[0] / 2) +period2 = st.selectbox("Select Period", periods["Period"], key="period2") +n_items = st.selectbox( + "Select the number of States to show", np.arange(5, n_records_per_year + 5, step=5) +) -df_sorted = df[df['Period'] == period2].sort_values(by='LifeExpectancy', ascending=False).head(n_items) -c = alt.Chart(df_sorted).mark_bar().encode( - x=alt.X('LifeExpectancy:Q', title="Life Expectancy"), - y=alt.Y('State:N', sort='-x'), - color=alt.Color('Region:N', scale=alt.Scale(range=palette)) +df_sorted = ( + df[df["Period"] == period2] + .sort_values(by="LifeExpectancy", ascending=False) + .head(n_items) +) +c = ( + alt.Chart(df_sorted) + .mark_bar() + .encode( + x=alt.X("LifeExpectancy:Q", title="Life Expectancy"), + y=alt.Y("State:N", sort="-x"), + color=alt.Color("Region:N", scale=alt.Scale(range=palette)), + ) ) st.altair_chart(c, use_container_width=True) - diff --git a/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py b/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py index b3a2d2ad7a..215ff7479c 100644 --- a/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py +++ b/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py @@ -42,12 +42,12 @@ def run(job_input: IJobInput): df.columns = df.columns.str.replace(" ", "") # records = [row for index, row in df.iterrows()] - #job_input.send_tabular_data_for_ingestion( + # job_input.send_tabular_data_for_ingestion( # df.itertuples(index=False), # destination_table="life_expectancy_2010_2015", # column_names=df.columns, # method="trino", - #) + # ) for i, row in df.iterrows(): query = build_insert_query( row, df.columns.tolist(), "life_expectancy_2010_2015" diff --git a/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py b/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py index a32b677276..f27c2b866c 100644 --- a/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py +++ b/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py @@ -4,6 +4,7 @@ import logging import os import sys + import numpy as np import pandas as pd @@ -28,16 +29,11 @@ def run(job_input: IJobInput): # url of the U.S. Life Expectancy at Birth by State and Census Tract - 2010-2015 dataset url = "https://raw.githubusercontent.com/cphalpert/census-regions/master/us%20census%20bureau%20regions%20and%20divisions.csv" - df = pd.read_csv(url, sep=',') + df = pd.read_csv(url, sep=",") df.columns = df.columns.str.replace(" ", "") - for i, row in df.iterrows(): - query = build_insert_query( - row, df.columns.tolist(), "us_regions" - ) + query = build_insert_query(row, df.columns.tolist(), "us_regions") job_input.execute_query(query) - - diff --git a/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py b/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py index 7daeca8373..364cfe95b2 100644 --- a/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py +++ b/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py @@ -31,17 +31,22 @@ def run(job_input: IJobInput): df = pd.read_excel(url, header=3, na_values="(NA)").replace("'", "''", regex=True) # select only interesting rows - df = df[['Unnamed: 0', 2017,2018, 2019, 2020]] - df.rename({'Unnamed: 0' : 'County', 2017 : 'Year2017', 2018 : 'Year2018', 2019: 'Year2019', 2020: 'Year2020'}, axis=1, inplace=True) + df = df[["Unnamed: 0", 2017, 2018, 2019, 2020]] + df.rename( + { + "Unnamed: 0": "County", + 2017: "Year2017", + 2018: "Year2018", + 2019: "Year2019", + 2020: "Year2020", + }, + axis=1, + inplace=True, + ) df.dropna(axis=0, inplace=True) - for i, row in df.iterrows(): log.info(row) - query = build_insert_query( - row, df.columns.tolist(), "us_gdp" - ) + query = build_insert_query(row, df.columns.tolist(), "us_gdp") job_input.execute_query(query) - - diff --git a/examples/life-expectancy/life-expectancy/18_create_table_merged_life_expectancy.sql b/examples/life-expectancy/life-expectancy/18_create_table_merged_life_expectancy.sql index fabb2c5124..831d8cdec3 100644 --- a/examples/life-expectancy/life-expectancy/18_create_table_merged_life_expectancy.sql +++ b/examples/life-expectancy/life-expectancy/18_create_table_merged_life_expectancy.sql @@ -6,7 +6,7 @@ CREATE TABLE merged_life_expectancy AS '2010-2015' AS Period, Region, 0 AS GDP -FROM +FROM cleaned_life_expectancy_2010_2015 JOIN us_regions ON us_regions.State = cleaned_life_expectancy_2010_2015.State ) UNION @@ -17,8 +17,8 @@ UNION '2018' AS Period, Region, Year2018 AS GDP -FROM cleaned_life_expectancy_2018 - JOIN us_regions ON us_regions.State = cleaned_life_expectancy_2018.State +FROM cleaned_life_expectancy_2018 + JOIN us_regions ON us_regions.State = cleaned_life_expectancy_2018.State INNER JOIN us_gdp ON us_gdp.County = cleaned_life_expectancy_2018.State WHERE Year2018 > 100000000 -) \ No newline at end of file +) From ec6d1a755edbad715510b0544d64a0435415b4ed Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 25 Dec 2021 22:42:23 +0000 Subject: [PATCH 18/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/life-expectancy/dashboard.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/life-expectancy/dashboard.py b/examples/life-expectancy/dashboard.py index 2e85d13a22..99c2f429da 100644 --- a/examples/life-expectancy/dashboard.py +++ b/examples/life-expectancy/dashboard.py @@ -1,6 +1,5 @@ # Copyright 2021 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 - import math import altair as alt From 709bd3c0e9e4f0448495438579161c2be9ee7768 Mon Sep 17 00:00:00 2001 From: Angelica Lo Duca Date: Tue, 4 Jan 2022 16:19:40 +0100 Subject: [PATCH 19/28] Update README.md --- examples/life-expectancy/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/life-expectancy/README.md b/examples/life-expectancy/README.md index 5d6bcbb838..bc27cded55 100644 --- a/examples/life-expectancy/README.md +++ b/examples/life-expectancy/README.md @@ -12,7 +12,7 @@ The directory is organized as follows: Researchers from the Department of Population History and Social Structure have to conduct a study on the life expectancy of Americans. The objective of the study is to understand over time which American states have the greatest life expectancy. -## Data Scoures +## Data Sources After extensive research, the researchers found two main datasets, both of which are available as CSV files: - [U.S. Life Expectancy at Birth by State and Census Tract - 2010-2015](https://catalog.data.gov/dataset/u-s-life-expectancy-at-birth-by-state-and-census-tract-2010-2015) From 342a86631b7c5635825722e78b5e97c8d4dd711a Mon Sep 17 00:00:00 2001 From: Angelica Lo Duca Date: Mon, 31 Jan 2022 16:45:35 +0100 Subject: [PATCH 20/28] Updated varchar and requirements - Changed varchar(32) to varchar in all the table creation files - Added openpyxl to the requirements.txt file --- .../03_create_table_life_expectancy_2010_2015.sql | 6 +++--- .../04_create_table_life_expectancy_2018.sql | 6 +++--- .../08_create_table_us_regions.sql | 6 +++--- .../life-expectancy/11_create_table_us_gdp.sql | 2 +- .../life-expectancy/requirements.txt | 1 + .../cb_2018_us_state_500k.shx | Bin 0 -> 548 bytes 6 files changed, 11 insertions(+), 10 deletions(-) create mode 100644 examples/life-expectancy/resources/cb_2018_us_state_500k/cb_2018_us_state_500k.shx diff --git a/examples/life-expectancy/life-expectancy/03_create_table_life_expectancy_2010_2015.sql b/examples/life-expectancy/life-expectancy/03_create_table_life_expectancy_2010_2015.sql index ee5ab4746d..89cf74f0b5 100644 --- a/examples/life-expectancy/life-expectancy/03_create_table_life_expectancy_2010_2015.sql +++ b/examples/life-expectancy/life-expectancy/03_create_table_life_expectancy_2010_2015.sql @@ -1,7 +1,7 @@ CREATE TABLE life_expectancy_2010_2015 ( - State varchar(32), - County varchar(32), - CensusTractNumber varchar(32), + State varchar, + County varchar, + CensusTractNumber varchar, LifeExpectancy decimal(4,2), LifeExpectancyRange varchar, LifeExpectancyStandardError decimal(4,2) diff --git a/examples/life-expectancy/life-expectancy/04_create_table_life_expectancy_2018.sql b/examples/life-expectancy/life-expectancy/04_create_table_life_expectancy_2018.sql index a9bc0f005e..7958e79fa5 100644 --- a/examples/life-expectancy/life-expectancy/04_create_table_life_expectancy_2018.sql +++ b/examples/life-expectancy/life-expectancy/04_create_table_life_expectancy_2018.sql @@ -1,7 +1,7 @@ CREATE TABLE life_expectancy_2018 ( - State varchar(32), - Sex varchar(32), + State varchar, + Sex varchar, LEB decimal(3,1), SE decimal(3,1), - Quartile varchar(32) + Quartile varchar ) diff --git a/examples/life-expectancy/life-expectancy/08_create_table_us_regions.sql b/examples/life-expectancy/life-expectancy/08_create_table_us_regions.sql index bd5cdc23de..a127b7e708 100644 --- a/examples/life-expectancy/life-expectancy/08_create_table_us_regions.sql +++ b/examples/life-expectancy/life-expectancy/08_create_table_us_regions.sql @@ -1,6 +1,6 @@ CREATE TABLE us_regions ( - State varchar(32), + State varchar, StateCode varchar(2), - Region varchar(32), - Division varchar(32) + Region varchar, + Division varchar ) diff --git a/examples/life-expectancy/life-expectancy/11_create_table_us_gdp.sql b/examples/life-expectancy/life-expectancy/11_create_table_us_gdp.sql index 7d9e2974ec..665ac3e199 100644 --- a/examples/life-expectancy/life-expectancy/11_create_table_us_gdp.sql +++ b/examples/life-expectancy/life-expectancy/11_create_table_us_gdp.sql @@ -1,5 +1,5 @@ CREATE TABLE us_gdp ( - County varchar(32), + County varchar, Year2017 bigint, Year2018 bigint, Year2019 bigint, diff --git a/examples/life-expectancy/life-expectancy/requirements.txt b/examples/life-expectancy/life-expectancy/requirements.txt index 659a4163f4..149f1137b3 100644 --- a/examples/life-expectancy/life-expectancy/requirements.txt +++ b/examples/life-expectancy/life-expectancy/requirements.txt @@ -1,4 +1,5 @@ inspect math numpy +openpyxl pandas diff --git a/examples/life-expectancy/resources/cb_2018_us_state_500k/cb_2018_us_state_500k.shx b/examples/life-expectancy/resources/cb_2018_us_state_500k/cb_2018_us_state_500k.shx new file mode 100644 index 0000000000000000000000000000000000000000..829f123d285ac01c4556ab65fc5755ca646b2757 GIT binary patch literal 548 zcmZvZT}YF06vltAw%4_-Tr(|r-*$EnU|sq6$z0@nscUy47Mv>I$&aoM2lpNSpF9JR4v8o%xKV5q5PE?! z(hP6IRMkzv-4Uv03-HRKy5}BBx|su|XE0R3*8K!xvdnxE{w$*=#^FK{b>$|ku2GXd z5A_9VJ~qG=3wv!AI`i23<0y}>a`5XqBnvnuh4221%D?gI2Sgc zHiI)OK8FM^U4#K}9qAW5j?23U_uu2PrlBK&>x<6ez8`3wJvf#^Yq#K55$(Z4I2%BF zC;g!Sx_eu49R0TVg_GzR9lEyBf2nX{5W^b+bseLn6MPkn%k6N(gYh&7r~YE(dti47 z@1p$GP8i?(nCP_P`z?34S1{EW9L-{Ct Date: Mon, 31 Jan 2022 15:46:43 +0000 Subject: [PATCH 21/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/life-expectancy/life-expectancy/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/life-expectancy/life-expectancy/requirements.txt b/examples/life-expectancy/life-expectancy/requirements.txt index 149f1137b3..8a25428a80 100644 --- a/examples/life-expectancy/life-expectancy/requirements.txt +++ b/examples/life-expectancy/life-expectancy/requirements.txt @@ -1,5 +1,5 @@ inspect math numpy -openpyxl +openpyxl pandas From 47d92b3f6fbced776f241a7a97c3bd7635c04bc8 Mon Sep 17 00:00:00 2001 From: Angelica Lo Duca Date: Mon, 31 Jan 2022 16:55:38 +0100 Subject: [PATCH 22/28] Updated Readme Updated readme with the content of requirements.txt --- examples/life-expectancy/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/life-expectancy/README.md b/examples/life-expectancy/README.md index bc27cded55..bd388ebd60 100644 --- a/examples/life-expectancy/README.md +++ b/examples/life-expectancy/README.md @@ -157,6 +157,7 @@ This example also requires the following Python libraries, which are included in inspect math numpy +openpyxl pandas ``` From b0d14dd3259c727ef81716c819c2f999f0e860b9 Mon Sep 17 00:00:00 2001 From: Angelica Lo Duca Date: Mon, 31 Jan 2022 17:12:04 +0100 Subject: [PATCH 23/28] Moved le_utils within life-expectancy and removed import os block code - Moved le_utils within life-expectancy - removed import os block code from .py codes --- .../05_ingest_table_life_expectancy_2010_2015.py | 4 ---- .../life-expectancy/06_ingest_table_life_expectancy_2018.py | 4 ---- .../life-expectancy/09_ingest_table_us_regions.py | 4 ---- .../life-expectancy/12_ingest_table_us_gdp.py | 5 ----- .../{ => life-expectancy}/le_utils/__init__.py | 0 .../life-expectancy/{ => life-expectancy}/le_utils/utils.py | 0 6 files changed, 17 deletions(-) rename examples/life-expectancy/{ => life-expectancy}/le_utils/__init__.py (100%) rename examples/life-expectancy/{ => life-expectancy}/le_utils/utils.py (100%) diff --git a/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py b/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py index 215ff7479c..0bbaa86a48 100644 --- a/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py +++ b/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py @@ -8,10 +8,6 @@ import numpy as np import pandas as pd -current_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) -parent_dir = os.path.dirname(current_dir) -sys.path.insert(0, parent_dir) - from le_utils.utils import build_insert_query from vdk.api.job_input import IJobInput diff --git a/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py b/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py index 59e47a8cd5..2baef3f471 100644 --- a/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py +++ b/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py @@ -8,10 +8,6 @@ import numpy as np import pandas as pd -current_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) -parent_dir = os.path.dirname(current_dir) -sys.path.insert(0, parent_dir) - from le_utils.utils import build_insert_query from vdk.api.job_input import IJobInput diff --git a/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py b/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py index f27c2b866c..0fa9389dd1 100644 --- a/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py +++ b/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py @@ -9,10 +9,6 @@ import pandas as pd -current_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) -parent_dir = os.path.dirname(current_dir) -sys.path.insert(0, parent_dir) - from le_utils.utils import build_insert_query from vdk.api.job_input import IJobInput diff --git a/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py b/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py index 364cfe95b2..75d99993ca 100644 --- a/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py +++ b/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py @@ -8,11 +8,6 @@ import numpy as np import pandas as pd - -current_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) -parent_dir = os.path.dirname(current_dir) -sys.path.insert(0, parent_dir) - from le_utils.utils import build_insert_query from vdk.api.job_input import IJobInput diff --git a/examples/life-expectancy/le_utils/__init__.py b/examples/life-expectancy/life-expectancy/le_utils/__init__.py similarity index 100% rename from examples/life-expectancy/le_utils/__init__.py rename to examples/life-expectancy/life-expectancy/le_utils/__init__.py diff --git a/examples/life-expectancy/le_utils/utils.py b/examples/life-expectancy/life-expectancy/le_utils/utils.py similarity index 100% rename from examples/life-expectancy/le_utils/utils.py rename to examples/life-expectancy/life-expectancy/le_utils/utils.py From 94ecf19fce1902b3b23caebca93882ef5005a753 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 31 Jan 2022 16:13:13 +0000 Subject: [PATCH 24/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../05_ingest_table_life_expectancy_2010_2015.py | 2 -- .../life-expectancy/06_ingest_table_life_expectancy_2018.py | 2 -- .../life-expectancy/09_ingest_table_us_regions.py | 3 --- .../life-expectancy/life-expectancy/12_ingest_table_us_gdp.py | 2 -- 4 files changed, 9 deletions(-) diff --git a/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py b/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py index 0bbaa86a48..ff58529aee 100644 --- a/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py +++ b/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py @@ -7,9 +7,7 @@ import numpy as np import pandas as pd - from le_utils.utils import build_insert_query - from vdk.api.job_input import IJobInput log = logging.getLogger(__name__) diff --git a/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py b/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py index 2baef3f471..6482d3cc18 100644 --- a/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py +++ b/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py @@ -7,9 +7,7 @@ import numpy as np import pandas as pd - from le_utils.utils import build_insert_query - from vdk.api.job_input import IJobInput log = logging.getLogger(__name__) diff --git a/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py b/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py index 0fa9389dd1..b905c50ce7 100644 --- a/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py +++ b/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py @@ -7,10 +7,7 @@ import numpy as np import pandas as pd - - from le_utils.utils import build_insert_query - from vdk.api.job_input import IJobInput log = logging.getLogger(__name__) diff --git a/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py b/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py index 75d99993ca..259c05860e 100644 --- a/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py +++ b/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py @@ -7,9 +7,7 @@ import numpy as np import pandas as pd - from le_utils.utils import build_insert_query - from vdk.api.job_input import IJobInput log = logging.getLogger(__name__) From 9842a0af39be34f1bb39cbfeed4d95a29821fcdf Mon Sep 17 00:00:00 2001 From: Angelica Lo Duca Date: Tue, 1 Feb 2022 18:21:10 +0100 Subject: [PATCH 25/28] updated .py ingesting jobs Updated ingested jobs with send_tabular_data_for_ingestion() --- ...5_ingest_table_life_expectancy_2010_2015.py | 18 ++++++++++++------ .../06_ingest_table_life_expectancy_2018.py | 11 ++++++----- .../09_ingest_table_us_regions.py | 10 ++++++---- .../life-expectancy/12_ingest_table_us_gdp.py | 11 ++++++----- .../life-expectancy/life-expectancy/config.ini | 5 +++-- 5 files changed, 33 insertions(+), 22 deletions(-) diff --git a/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py b/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py index ff58529aee..fef6bf3e38 100644 --- a/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py +++ b/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py @@ -42,9 +42,15 @@ def run(job_input: IJobInput): # column_names=df.columns, # method="trino", # ) - for i, row in df.iterrows(): - query = build_insert_query( - row, df.columns.tolist(), "life_expectancy_2010_2015" - ) - - job_input.execute_query(query) + #for i, row in df.iterrows(): + # query = build_insert_query( + # row, df.columns.tolist(), "life_expectancy_2010_2015" + # ) + + # job_input.execute_query(query) + + job_input.send_tabular_data_for_ingestion( + df.itertuples(index=False), + destination_table="life_expectancy_2010_2015", + column_names=df.columns.tolist() + ) \ No newline at end of file diff --git a/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py b/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py index 6482d3cc18..1d64399c2d 100644 --- a/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py +++ b/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py @@ -32,9 +32,10 @@ def run(job_input: IJobInput): df = pd.read_csv(url, dtype=dtypes, na_values="*") - log.info(df.head(5)) + job_input.send_tabular_data_for_ingestion( + df.itertuples(index=False), + destination_table="life_expectancy_2018", + column_names=df.columns.tolist() + ) - for i, row in df.iterrows(): - query = build_insert_query(row, df.columns.tolist(), "life_expectancy_2018") - - job_input.execute_query(query) + \ No newline at end of file diff --git a/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py b/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py index b905c50ce7..f355a92f7c 100644 --- a/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py +++ b/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py @@ -26,7 +26,9 @@ def run(job_input: IJobInput): df.columns = df.columns.str.replace(" ", "") - for i, row in df.iterrows(): - query = build_insert_query(row, df.columns.tolist(), "us_regions") - - job_input.execute_query(query) + + job_input.send_tabular_data_for_ingestion( + df.itertuples(index=False), + destination_table="us_regions", + column_names=df.columns.tolist(), + ) \ No newline at end of file diff --git a/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py b/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py index 259c05860e..a742a0981b 100644 --- a/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py +++ b/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py @@ -38,8 +38,9 @@ def run(job_input: IJobInput): ) df.dropna(axis=0, inplace=True) - for i, row in df.iterrows(): - log.info(row) - query = build_insert_query(row, df.columns.tolist(), "us_gdp") - - job_input.execute_query(query) + + job_input.send_tabular_data_for_ingestion( + df.itertuples(index=False), + destination_table="us_gdp", + column_names=df.columns.tolist(), + ) \ No newline at end of file diff --git a/examples/life-expectancy/life-expectancy/config.ini b/examples/life-expectancy/life-expectancy/config.ini index 3634d140a6..0c126b77e4 100644 --- a/examples/life-expectancy/life-expectancy/config.ini +++ b/examples/life-expectancy/life-expectancy/config.ini @@ -24,9 +24,10 @@ db_default_type=TRINO ingest_method_default = trino trino_catalog = mysql -trino_use_ssl = +trino_use_ssl = false trino_host = localhost trino_port = 8080 trino_user = root trino_schema = life-expectancy -trino_ssl_verify = +trino_ssl_verify = false + From df74d969aad77fb361c4dd5a6847c8af88f43947 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 1 Feb 2022 17:22:21 +0000 Subject: [PATCH 26/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../05_ingest_table_life_expectancy_2010_2015.py | 10 +++++----- .../06_ingest_table_life_expectancy_2018.py | 8 +++----- .../life-expectancy/09_ingest_table_us_regions.py | 9 ++++----- .../life-expectancy/12_ingest_table_us_gdp.py | 9 ++++----- examples/life-expectancy/life-expectancy/config.ini | 1 - 5 files changed, 16 insertions(+), 21 deletions(-) diff --git a/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py b/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py index fef6bf3e38..21ec58d2de 100644 --- a/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py +++ b/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py @@ -42,7 +42,7 @@ def run(job_input: IJobInput): # column_names=df.columns, # method="trino", # ) - #for i, row in df.iterrows(): + # for i, row in df.iterrows(): # query = build_insert_query( # row, df.columns.tolist(), "life_expectancy_2010_2015" # ) @@ -50,7 +50,7 @@ def run(job_input: IJobInput): # job_input.execute_query(query) job_input.send_tabular_data_for_ingestion( - df.itertuples(index=False), - destination_table="life_expectancy_2010_2015", - column_names=df.columns.tolist() - ) \ No newline at end of file + df.itertuples(index=False), + destination_table="life_expectancy_2010_2015", + column_names=df.columns.tolist(), + ) diff --git a/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py b/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py index 1d64399c2d..45892b7b99 100644 --- a/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py +++ b/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py @@ -33,9 +33,7 @@ def run(job_input: IJobInput): df = pd.read_csv(url, dtype=dtypes, na_values="*") job_input.send_tabular_data_for_ingestion( - df.itertuples(index=False), - destination_table="life_expectancy_2018", - column_names=df.columns.tolist() + df.itertuples(index=False), + destination_table="life_expectancy_2018", + column_names=df.columns.tolist(), ) - - \ No newline at end of file diff --git a/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py b/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py index f355a92f7c..787260084c 100644 --- a/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py +++ b/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py @@ -26,9 +26,8 @@ def run(job_input: IJobInput): df.columns = df.columns.str.replace(" ", "") - job_input.send_tabular_data_for_ingestion( - df.itertuples(index=False), - destination_table="us_regions", - column_names=df.columns.tolist(), - ) \ No newline at end of file + df.itertuples(index=False), + destination_table="us_regions", + column_names=df.columns.tolist(), + ) diff --git a/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py b/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py index a742a0981b..a8ae40a5c8 100644 --- a/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py +++ b/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py @@ -38,9 +38,8 @@ def run(job_input: IJobInput): ) df.dropna(axis=0, inplace=True) - job_input.send_tabular_data_for_ingestion( - df.itertuples(index=False), - destination_table="us_gdp", - column_names=df.columns.tolist(), - ) \ No newline at end of file + df.itertuples(index=False), + destination_table="us_gdp", + column_names=df.columns.tolist(), + ) diff --git a/examples/life-expectancy/life-expectancy/config.ini b/examples/life-expectancy/life-expectancy/config.ini index 0c126b77e4..6d1c108dd3 100644 --- a/examples/life-expectancy/life-expectancy/config.ini +++ b/examples/life-expectancy/life-expectancy/config.ini @@ -30,4 +30,3 @@ trino_port = 8080 trino_user = root trino_schema = life-expectancy trino_ssl_verify = false - From 35b413a1536c12267a88d47165862ffcda5ab8c8 Mon Sep 17 00:00:00 2001 From: Angelica Lo Duca Date: Thu, 3 Feb 2022 16:54:35 +0100 Subject: [PATCH 27/28] removed le_utils and unused libraries, updated readme - Removed the le_utils folder, because it is not longer necessary - Removed unused libraries from .py scripts - Updated the README - first section Life Expectancy --- examples/life-expectancy/README.md | 3 +- ..._ingest_table_life_expectancy_2010_2015.py | 19 +-------- .../06_ingest_table_life_expectancy_2018.py | 5 +-- .../09_ingest_table_us_regions.py | 6 +-- .../life-expectancy/12_ingest_table_us_gdp.py | 5 --- .../life-expectancy/le_utils/__init__.py | 2 - .../life-expectancy/le_utils/utils.py | 40 ------------------- 7 files changed, 5 insertions(+), 75 deletions(-) delete mode 100644 examples/life-expectancy/life-expectancy/le_utils/__init__.py delete mode 100644 examples/life-expectancy/life-expectancy/le_utils/utils.py diff --git a/examples/life-expectancy/README.md b/examples/life-expectancy/README.md index bd388ebd60..f89477c305 100644 --- a/examples/life-expectancy/README.md +++ b/examples/life-expectancy/README.md @@ -6,7 +6,8 @@ This example provides the complete code to perform the following tasks: The directory is organized as follows: * **life-expectancy** - contains jobs -* **le_utils** - contains some auxiliary_functions. +* **resources** - contains some additional files used by the dashboard +* `dashboard.py` - contains the code to generate the dashboard ## Scenario diff --git a/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py b/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py index 21ec58d2de..521e78c25e 100644 --- a/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py +++ b/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py @@ -2,12 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 import inspect import logging -import os -import sys - import numpy as np import pandas as pd -from le_utils.utils import build_insert_query + from vdk.api.job_input import IJobInput log = logging.getLogger(__name__) @@ -35,20 +32,6 @@ def run(job_input: IJobInput): df.columns = df.columns.str.replace(" ", "") - # records = [row for index, row in df.iterrows()] - # job_input.send_tabular_data_for_ingestion( - # df.itertuples(index=False), - # destination_table="life_expectancy_2010_2015", - # column_names=df.columns, - # method="trino", - # ) - # for i, row in df.iterrows(): - # query = build_insert_query( - # row, df.columns.tolist(), "life_expectancy_2010_2015" - # ) - - # job_input.execute_query(query) - job_input.send_tabular_data_for_ingestion( df.itertuples(index=False), destination_table="life_expectancy_2010_2015", diff --git a/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py b/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py index 45892b7b99..7091a77e96 100644 --- a/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py +++ b/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py @@ -2,12 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 import inspect import logging -import os -import sys - import numpy as np import pandas as pd -from le_utils.utils import build_insert_query + from vdk.api.job_input import IJobInput log = logging.getLogger(__name__) diff --git a/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py b/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py index 787260084c..8262e35505 100644 --- a/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py +++ b/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py @@ -2,12 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 import inspect import logging -import os -import sys - -import numpy as np import pandas as pd -from le_utils.utils import build_insert_query + from vdk.api.job_input import IJobInput log = logging.getLogger(__name__) diff --git a/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py b/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py index a8ae40a5c8..95ced5ebe6 100644 --- a/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py +++ b/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py @@ -2,12 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import inspect import logging -import os -import sys - -import numpy as np import pandas as pd -from le_utils.utils import build_insert_query from vdk.api.job_input import IJobInput log = logging.getLogger(__name__) diff --git a/examples/life-expectancy/life-expectancy/le_utils/__init__.py b/examples/life-expectancy/life-expectancy/le_utils/__init__.py deleted file mode 100644 index 50c007580a..0000000000 --- a/examples/life-expectancy/life-expectancy/le_utils/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright 2021 VMware, Inc. -# SPDX-License-Identifier: Apache-2.0 diff --git a/examples/life-expectancy/life-expectancy/le_utils/utils.py b/examples/life-expectancy/life-expectancy/le_utils/utils.py deleted file mode 100644 index fab3e79872..0000000000 --- a/examples/life-expectancy/life-expectancy/le_utils/utils.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2021 VMware, Inc. -# SPDX-License-Identifier: Apache-2.0 -import math - -import numpy as np - - -def build_insert_query(row, cols, output_table): - """ - Build an formatted query for insertion. - - Keyword arguments: - -- row (pandas.Series) the list of values to be inserted in the query - -- cols (list) the list of columns name of the SQL table - - Return: - -- query (string) a string containing the insert statement - - """ - values = "" - keys = cols.copy() - for index in range(0, len(row)): - - # manage NaN values, both for strings and numeric values - if (row[index] != row[index]) and (math.isnan(row[index])): - # do not insert NaN values, thus remove it from columns - keys.remove(cols[index]) - - else: - # quote only strings and convert numeric values to strings - if isinstance(row[index], str): - values = values + ", '" + row[index] + "'" - else: - values = values + ", " + str(row[index]) - - # remove the first comma - values = values[1:] - - cols = ", ".join([str(i) for i in keys]) - return f"INSERT INTO {output_table} ({cols}) VALUES ({values})" From 0dd0271344f3cb079f146b065d0fe95c26aa449b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 3 Feb 2022 15:55:47 +0000 Subject: [PATCH 28/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../05_ingest_table_life_expectancy_2010_2015.py | 2 +- .../life-expectancy/06_ingest_table_life_expectancy_2018.py | 2 +- .../life-expectancy/09_ingest_table_us_regions.py | 2 +- .../life-expectancy/life-expectancy/12_ingest_table_us_gdp.py | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py b/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py index 521e78c25e..ec2aea78b2 100644 --- a/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py +++ b/examples/life-expectancy/life-expectancy/05_ingest_table_life_expectancy_2010_2015.py @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 import inspect import logging + import numpy as np import pandas as pd - from vdk.api.job_input import IJobInput log = logging.getLogger(__name__) diff --git a/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py b/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py index 7091a77e96..8a95ac039c 100644 --- a/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py +++ b/examples/life-expectancy/life-expectancy/06_ingest_table_life_expectancy_2018.py @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 import inspect import logging + import numpy as np import pandas as pd - from vdk.api.job_input import IJobInput log = logging.getLogger(__name__) diff --git a/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py b/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py index 8262e35505..9f17668713 100644 --- a/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py +++ b/examples/life-expectancy/life-expectancy/09_ingest_table_us_regions.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 import inspect import logging -import pandas as pd +import pandas as pd from vdk.api.job_input import IJobInput log = logging.getLogger(__name__) diff --git a/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py b/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py index 95ced5ebe6..d16b987082 100644 --- a/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py +++ b/examples/life-expectancy/life-expectancy/12_ingest_table_us_gdp.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import inspect import logging + import pandas as pd from vdk.api.job_input import IJobInput