62 refresh kaggle dataset

Dockerfile

@@ -11,7 +11,7 @@ ENV PYTHON_VERSION=${PYTHON_VERSION}
 # install required software and programmes for development environment
 RUN apt-get update
 RUN apt-get install -y apt-utils vim curl wget unzip tree htop adduser
-RUN apt-get install -y imagemagick=8:7.1.1.43+dfsg1-1+deb13u5 libssl-dev=3.5.4-1~deb13u2
+RUN apt-get install -y imagemagick=8:7.1.1.43+dfsg1-1+deb13u5 libssl-dev=3.5.4-1~deb13u2 libpq-dev=17.8-0+deb13u1 linux-libc-dev=6.12.69-1
 
 # set up home environment
 RUN adduser ${user}

config/uv/RandomTelecomPayments.cmd

@@ -1,2 +1,2 @@
 :: call powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"
-call uv add -r requirements.txt --link-mode=copy
+call uv add -r ..\..\requirements.txt --link-mode=copy

generator/cons.py

@@ -14,7 +14,7 @@
 # set data files
 fpath_randomtelecomtransdata = os.path.join(subdir_data,'RandomTelecomPayments.csv')
 fpath_randomtelecomusersdata = os.path.join(subdir_data,'RandomTelecomUsers.parquet')
-fpath_arch_randomtelecomdata = os.path.join(subdir_data, 'arch', 'RandomTelecomPayments.csv')
+fpath_arch_randomtelecomdata = os.path.join(subdir_data, 'arch', 'RandomTelecomPaymentsV0.3.csv')
 fpath_temp_llama_first_names = os.path.join(subdir_data, 'temp', 'llama_first_names_{country}.csv')
 fpath_temp_llama_last_names = os.path.join(subdir_data, 'temp', 'llama_last_names_{country}.csv')
 fpath_temp_llama_email_domains = os.path.join(subdir_data, 'temp', 'llama_email_domains_{country}.csv')

generator/main.py

@@ -11,6 +11,7 @@
 from utilities.input_error_handling import input_error_handling
 from utilities.multiprocess import multiprocess
 from app.gen_random_telecom_data import gen_random_telecom_data
+import qa
 
 def main(input_params_dict: dict):
     """
@@ -56,10 +57,6 @@ def main(input_params_dict: dict):
     # order results by userid and transaction date ascending
     user_data = user_data.sort_values(by = 'uid').reset_index(drop = True)
     trans_data = trans_data.sort_values(by = 'transaction_date').reset_index(drop = True)
-    # end timer
-    t1 = time()
-    total_runtime_seconds = round(t1 - t0, 2)
-    logging.info(f'Total Runtime: {total_runtime_seconds} seconds')
     # print out head and shape of data
     logging.info(f'RandomTeleComUsersData.shape: {user_data.shape}')
     logging.info(f'RandomTeleComTransData.shape: {trans_data.shape}')
@@ -73,6 +70,15 @@ def main(input_params_dict: dict):
     logging.info(f'Writing output trans level random telecoms data to: {cons.fpath_randomtelecomtransdata}')
     user_data.to_parquet(cons.fpath_randomtelecomusersdata, engine='fastparquet')
     trans_data.to_csv(cons.fpath_randomtelecomtransdata, index = False)
+    # initialise QA objects
+    qa.Uids(data=trans_data).run_all()
+    qa.Transactions(data=trans_data).run_all()
+    qa.Cards(data=trans_data).run_all()
+    qa.Ips(data=trans_data).run_all()
+    # end timer
+    t1 = time()
+    total_runtime_seconds = round(t1 - t0, 2)
+    logging.info(f'Total Runtime: {total_runtime_seconds} seconds')
     # return dataframes as dictionary
     return {"user_data": user_data, "trans_data": trans_data}
 

generator/qa/Cards.py

@@ -0,0 +1,51 @@
+import logging
+import seaborn as sns
+
+class Cards():
+
+    def __init__(self, data, show_logs=False, show_plots=False):
+        """
+        """
+        logging.info("Initialising Cards QA")
+        self.data = data
+        self.show_logs = show_logs
+        self.show_plots = show_plots
+
+    def unique_card_types(self):
+        """
+        """
+        nunique_card_types_per_card = self.data.groupby(by=['card_hash'], as_index=False, dropna=False).agg({'card_type':'nunique'})
+        card_type_payment_totals = self.data.groupby(by=['card_type', 'transaction_payment_method'], as_index=False, dropna=False).agg({'transaction_hash':'size', 'transaction_amount':'sum'})
+        # test assertions
+        assert nunique_card_types_per_card['card_type'].max() == 1
+        assert nunique_card_types_per_card['card_type'].min() == 0
+        assert not nunique_card_types_per_card['card_type'].isnull().any()
+        if self.show_plots:
+            # plot distribution
+            sns.histplot(data=nunique_card_types_per_card,x='card_type', bins = 20)
+        if self.show_logs:
+            # show logs
+            logging.info(card_type_payment_totals)
+
+    def unique_country_codes(self):
+        """
+        """
+        nunique_country_codes_per_card = self.data.groupby(by=['card_hash'], as_index=False, dropna=False).agg({'card_country_code':'nunique'})
+        card_country_code_payment_totals = self.data.assign(card_country_code=self.data['card_country_code'].notnull().astype(int)).groupby(by=['card_country_code', 'transaction_payment_method'], as_index=False, dropna=False).agg({'transaction_hash':'size', 'transaction_amount':'sum'})
+        # test assertions
+        assert nunique_country_codes_per_card['card_country_code'].max() == 1
+        assert nunique_country_codes_per_card['card_country_code'].min() == 0
+        assert not nunique_country_codes_per_card['card_country_code'].isnull().any()
+        if self.show_plots:
+            # plot distribution
+            sns.histplot(data=nunique_country_codes_per_card,x='card_country_code', bins = 20)
+        if self.show_logs:
+            # show logs
+            logging.info(card_country_code_payment_totals)
+
+    def run_all(self):
+        """
+        """
+        self.unique_card_types()
+        self.unique_country_codes()
+        logging.info("All Cards QA checks passed.")

generator/qa/Ips.py

@@ -0,0 +1,34 @@
+import logging
+import seaborn as sns
+
+class Ips():
+
+    def __init__(self, data, show_logs=False, show_plots=False):
+        """
+        """
+        logging.info("Initialising Ips QA")
+        self.data = data
+        self.show_logs = show_logs
+        self.show_plots = show_plots
+
+    def unique_country_codes(self):
+        """
+        """
+        nunique_country_codes_per_ip = self.data.groupby(by=['ip_hash'], as_index=False, dropna=False).agg({'ip_country_code':'nunique'})
+        ip_country_codes_totals = self.data.groupby(by=['ip_country_code'], as_index=False, dropna=False).size()
+        # test assertions
+        assert nunique_country_codes_per_ip['ip_country_code'].max() == 1
+        assert nunique_country_codes_per_ip['ip_country_code'].min() == 1
+        assert not nunique_country_codes_per_ip['ip_country_code'].isnull().any()
+        if self.show_plots:
+            # plot distribution
+            sns.histplot(data=nunique_country_codes_per_ip,x='ip_country_code', bins = 20)
+        if self.show_logs:
+            # show logs
+            logging.info(ip_country_codes_totals.to_markdown())
+
+    def run_all(self):
+        """
+        """
+        self.unique_country_codes()
+        logging.info("All Ips QA checks passed.")