add tableprint for latex

Author: Duftata

evaluation/print_table_4_latex.py

@@ -0,0 +1,45 @@
+from evaluation.quantitative import mean_df, pd, mode
+
+
+def print_mark(key, name):
+    if key in name:
+        return '\checkmark'
+    else:
+        return ''
+
+
+if mode == 'end2end':
+    table = pd.DataFrame(columns=['Img', 'Loc', 'Seg', 'Rep', 'Accuracy', 'F1', 'Precision', 'Recall', 'AUROC'])
+    for experiment, row in mean_df.iterrows():
+        table = pd.concat([table, pd.DataFrame({
+            'Img': print_mark('image', experiment),
+            'Loc': print_mark('loc', experiment),
+            'Seg': print_mark('seg', experiment),
+            'Rep': print_mark('clip', experiment),
+            'Accuracy': row['Accuracy'],
+            'F1': row['F1'],
+            'Precision': row['Precision'],
+            'Recall': row['Recall'],
+            'AUROC': row['AUROC']
+        }, index=[0]), ], ignore_index=True)
+    # multiply floats by 100 to get percentage
+    table.iloc[:, 4:] *= 100
+    table = table.round(2)
+
+    print('\n\n')
+    print(table.to_latex(index=False, float_format='%.2f'))
+else:
+    table = pd.DataFrame(columns=['Encoder', 'Accuracy', 'F1', 'AUROC'])
+    for experiment, row in mean_df.iterrows():
+        table = pd.concat([table, pd.DataFrame({
+            'Encoder': experiment,
+            'Accuracy': row['Accuracy'],
+            'F1': row['F1'],
+            'AUROC': row['AUROC']
+        }, index=[0]), ], ignore_index=True)
+    # multiply floats by 100 to get percentage
+    table.iloc[:, 1:] *= 100
+    table = table.round(2)
+
+    print('\n\n')
+    print(table.to_latex(index=False, float_format='%.2f'))

evaluation/quantitative.py

@@ -1,30 +1,31 @@
+from pathlib import Path
+
+import pandas as pd
 import torch
 from torchmetrics import classification, MetricCollection
+
 from dataset.grazpedwri_dataset import GrazPedWriDataset
-import pandas as pd
-from pathlib import Path
-from evaluation.best_shot_accuracy import BestShotAccuracy
 
-mode = ['end2end', 'lin_eval'][0]
+mode = ['end2end', 'lin_eval'][1]
 
 metrics_kwargs = {'num_labels': GrazPedWriDataset.N_CLASSES, 'average': None}
 metrics = MetricCollection({
-    "Acc": classification.MultilabelAccuracy(**metrics_kwargs),
+    "Accuracy": classification.MultilabelAccuracy(**metrics_kwargs),
     "F1": classification.MultilabelF1Score(**metrics_kwargs),
     "Precision": classification.MultilabelPrecision(**metrics_kwargs),
     "Recall": classification.MultilabelRecall(**metrics_kwargs),
-    "AUROC": classification.MultilabelAUROC(**metrics_kwargs),
-    "BestShotAcc": BestShotAccuracy()
+    "AUROC": classification.MultilabelAUROC(**metrics_kwargs)
 })
 pred_dir = Path('evaluation/predictions')
 gt = torch.load(pred_dir / 'ground_truth.pt')
 
-mean_df = pd.DataFrame(columns=['Experiment', 'Acc', 'BestShotAcc', 'F1', 'Precision', 'Recall', 'AUROC'])
-experiment_df = pd.DataFrame(columns=['Experiment', 'Acc', 'F1', 'Precision', 'Recall', 'AUROC', 'AO_Class'])
+mean_df = pd.DataFrame(columns=['Experiment', 'Accuracy', 'F1', 'Precision', 'Recall', 'AUROC'])
+experiment_df = pd.DataFrame(columns=['Experiment', 'Accuracy', 'F1', 'Precision', 'Recall', 'AUROC', 'AO_Class'])
 for experiment in pred_dir.iterdir():
     is_line_eval = experiment.stem.startswith('LE')
     match_mode = (mode == 'lin_eval' and is_line_eval) or (mode == 'end2end' and not is_line_eval)
-    if experiment.stem == 'ground_truth' or experiment.is_dir() or not match_mode:
+    contains_mult_seg = 'mult_seg' in experiment.stem
+    if experiment.stem == 'ground_truth' or experiment.is_dir() or not match_mode or contains_mult_seg:
         continue
 
     pred = torch.load(experiment)
@@ -39,8 +40,7 @@
     performance = metrics(y_hat, y)
     mean_df = pd.concat([mean_df, pd.DataFrame({
         'Experiment': experiment.stem.rsplit('_', 1)[0],
-        'Acc': performance['Acc'].mean().item(),
-        "BestShotAcc": performance['BestShotAcc'].item(),
+        'Accuracy': performance['Accuracy'].mean().item(),
         'F1': performance['F1'].mean().item(),
         'Precision': performance['Precision'].mean().item(),
         'Recall': performance['Recall'].mean().item(),
@@ -49,7 +49,7 @@
 
     experiment_df = pd.concat([experiment_df, pd.DataFrame({
         'Experiment': experiment.stem.rsplit('_', 1)[0],
-        'Acc': performance['Acc'].tolist(),
+        'Accuracy': performance['Accuracy'].tolist(),
         'F1': performance['F1'].tolist(),
         'Precision': performance['Precision'].tolist(),
         'Recall': performance['Recall'].tolist(),

evaluation/statistical_significance_test.py

@@ -10,6 +10,7 @@
 warnings.filterwarnings("ignore")
 
 candidate = "image_frac_loc_bin_seg_clip"
+test_direction = ['greater', 'two-sided'][0]
 significance_level = 0.05
 
 pred_dir = Path('evaluation/predictions')
@@ -22,11 +23,12 @@
 filelist = list(gt.keys())
 gt = torch.stack([gt[file_stem] for file_stem in filelist]).int()
 
-candidate_path = [experiment for experiment in available_experiments if experiment.stem.startswith(candidate)][0]
+candidate_path = [experiment for experiment in available_experiments if experiment.name.rsplit('_', 1)[0] == candidate][0]
 y_pred_canditate = torch.load(candidate_path)
 y_pred_canditate = torch.stack([y_pred_canditate[file_stem] for file_stem in filelist])
 auroc_canditate = metric(y_pred_canditate, gt)
-print(f'Candidate: {candidate_path.stem.rsplit('_', 1)[0]} with AUROC: {auroc_canditate.mean().item()}')
+print(f'Candidate: {candidate_path.name.rsplit('_', 1)[0]} with AUROC: {auroc_canditate.mean().item()}')
+print(f'Test direction: {test_direction}')
 
 df = pd.DataFrame(columns=['Challenger', 'AUROC', 'statistic', 'p-value', f'significant at {significance_level}'])
 for challenger in available_experiments:
@@ -47,5 +49,6 @@
     }, index=[0]), ], ignore_index=True)
 
 df.set_index('Challenger', inplace=True)
-df.sort_values('AUROC', ascending=False, inplace=True)
+df.sort_index(inplace=True)
+df.sort_values('p-value', ascending=False, inplace=True)
 print(df.to_string())