updated embeddings/configs

Author: Lisa Dunlap

clip_utils.py

@@ -177,8 +177,8 @@ def get_ensamble_preds(val_features, probs, zeroshot_weights, dataset_domains=No
     except:
         outputs = probs
     print(outputs.shape)
-    salem_preds = np.argmax(outputs, axis=1)
-    print(salem_preds.shape)
+    lads_preds = np.argmax(outputs, axis=1)
+    print(lads_preds.shape)
     # CLIP ZS
     zeroshot_weights = zeroshot_weights.cuda()
     images = torch.tensor(val_features).cuda()
@@ -196,27 +196,27 @@ def get_ensamble_preds(val_features, probs, zeroshot_weights, dataset_domains=No
         dom_preds  = []
         for i in range(len(ensambled_preds)):
             if soft_dom_label[i] == 0:
-                dom_preds.append(salem_preds[i])
+                dom_preds.append(lads_preds[i])
             else:
                 dom_preds.append(ensambled_preds[i])
         ret_preds = np.array(dom_preds)
     else:
         ret_preds = ensambled_preds
 
-    return salem_preds, zs_preds, ensambled_preds, ret_preds
+    return lads_preds, zs_preds, ensambled_preds, ret_preds
 
-def get_pred_overlap(salem_preds, zs_preds, labels):
+def get_pred_overlap(lads_preds, zs_preds, labels):
     """
-    Get the overlap in correct predictions for salem and zeroshot.
+    Get the overlap in correct predictions for lads and zeroshot.
     """
-    salem_correct = np.where(salem_preds == labels)[0]
+    lads_correct = np.where(lads_preds == labels)[0]
     zs_correct = np.where(zs_preds == labels)[0]
-    print(len(salem_correct), len(zs_correct))
-    print("salem correct ", salem_correct[:10])
+    print(len(lads_correct), len(zs_correct))
+    print("lads correct ", lads_correct[:10])
     print("zs correct ", zs_correct[:10])
-    salem_overlap = [i for i in salem_correct if i in zs_correct]
-    salem_nonverlap = [i for i in salem_correct if not (i in zs_correct)]
-    zs_nonverlap = [i for i in zs_correct if not (i in salem_correct)]
-    num_zs_correct_nonoverlap = len(zs_correct) - len(salem_overlap)
-    num_salem_correct_nonverlap = len(salem_correct) - len(salem_overlap)
-    return num_salem_correct_nonverlap, num_salem_correct_nonverlap/len(labels), num_salem_correct_nonverlap/len(salem_correct)
+    lads_overlap = [i for i in lads_correct if i in zs_correct]
+    lads_nonverlap = [i for i in lads_correct if not (i in zs_correct)]
+    zs_nonverlap = [i for i in zs_correct if not (i in lads_correct)]
+    num_zs_correct_nonoverlap = len(zs_correct) - len(lads_overlap)
+    num_lads_correct_nonverlap = len(lads_correct) - len(lads_overlap)
+    return num_lads_correct_nonverlap, num_lads_correct_nonverlap/len(labels), num_lads_correct_nonverlap/len(lads_correct)

configs/ColoredMNIST/ZS.yaml

@@ -1,16 +1,16 @@
 EXP:
   ADVICE_METHOD: "CLIPZS"
   WANDB_SILENT: False
-  PROJ: "ColoredMNIST_ViT_Final"
+  PROJ: "ColoredMNIST"
   SEED: 0
   TEXT_PROMPTS: ['0','1','2','3','4','5','6','7','8','9']
   TEMPLATES: 'color_mnist_templates'
-  # TEMPLATES: 'no_template'
 
 DATA:
   DATASET: "ColoredMNISTBinary"
   LOAD_CACHED: True
   BATCH_SIZE: 256
+  ROOT: './data'
 
 METHOD:
   MODEL:

configs/ColoredMNIST/lads.yaml

@@ -1,22 +1,20 @@
 EXP:
   ADVICE_METHOD: "ClipMLP"
   WANDB_SILENT: False
-  PROJ: "ColoredMNIST_ViT_Final"
+  PROJ: "ColoredMNIST"
   SEED: 0
+  AUGMENTATION: 'LADSBias'
   TEXT_PROMPTS: [['a photo of a red number {}.'], ['a photo of a blue number {}.']]
-  AUGMENTATION: 'BiasDirectional'
+  NEUTRAL_TEXT_PROMPTS: ['a photo of a red number {}.', 'a photo of a blue number {}.']
   EPOCHS: 200
   CHECKPOINT_VAL: True
   ENSAMBLE: False
 
-  # CLIP_MODEL: 'ViT-H-14'
-  # CLIP_PRETRAINED_DATASET: 'laion2b_s32b_b79k'
-  # IMAGE_FEATURES: 'openclip'
-
 DATA:
   DATASET: "ColoredMNISTBinary"
   LOAD_CACHED: True
   BATCH_SIZE: 256
+  ROOT: './data'
 
 METHOD:
   MODEL:

configs/ColoredMNIST/mlp.yaml

@@ -1,7 +1,7 @@
 EXP:
   ADVICE_METHOD: "ClipMLP"
   WANDB_SILENT: False
-  PROJ: "ColoredMNIST_ViT_Final"
+  PROJ: "ColoredMNIST"
   SEED: 0
   TEXT_PROMPTS: ['a photo of a red number "{}".', 'a photo of a blue number "{}".']
   NEUTRAL_TEXT_PROMPTS: ['a photo of a white number "{}".']
@@ -10,14 +10,11 @@ EXP:
   GENERIC: False
   LOG_NN: True
 
-  # CLIP_MODEL: 'ViT-H-14'
-  # CLIP_PRETRAINED_DATASET: 'laion2b_s32b_b79k'
-  # IMAGE_FEATURES: 'openclip'
-
 DATA:
   DATASET: "ColoredMNISTBinary"
   LOAD_CACHED: True
   BATCH_SIZE: 256
+  ROOT: './data'
 
 METHOD:
   MODEL:

configs/Waterbirds/ZS.yaml

@@ -12,10 +12,4 @@ DATA:
   BATCH_SIZE: 256
   LOAD_CACHED: True
   UPWEIGHT_CLASSES: True
-
-METHOD:
-  MODEL:
-    NUM_LAYERS: 1
-    SEPERATE_CLASSES: True
-    RESUME: True
-  NORMALIZE: False
+  ROOT: /shared/lisabdunlap/vl-attention/data

configs/Waterbirds/lads.yaml

@@ -4,8 +4,8 @@ EXP:
   PROJ: "LADS_Waterbirds_Rebuttal"
   SEED: 0
   TEXT_PROMPTS: [['a photo of a {} on forest.'], ['a photo of a {} on water.']]
-  NEUTRAL_TEXT_PROMPTS: []
-  AUGMENTATION: 'BiasDirectional'
+  NEUTRAL_TEXT_PROMPTS: ['a photo of a {} on forest.', 'a photo of a {} on water.']
+  AUGMENTATION: 'LADSBias'
   EPOCHS: 200
   CHECKPOINT_VAL: True
   ENSAMBLE: False
@@ -15,31 +15,27 @@ DATA:
   DATASET: "Waterbirds"
   LOAD_CACHED: True
   BATCH_SIZE: 256
+  ROOT: /shared/lisabdunlap/vl-attention/data
 
 METHOD:
   MODEL:
     NUM_LAYERS: 1
     DOM_WEIGHT: 1.0
     LR: 0.001
-    WEIGHT_DECAY: 0.05
-    CHECKPOINT: 'checkpoint/mlp_simple.pth'
-    CHECKPOINT_NAME: 'mlp-directional'
+    CHECKPOINT: 'checkpoint/mlp.pth'
+    CHECKPOINT_NAME: "mlp"
     RESUME: False
-  # USE_DOM_GT: True
 
 AUGMENTATION:
   MODEL:
     LR: 0.005
-    WEIGHT_DECAY: 0.05
+    WEIGHT_DECAY: 0.005
     NUM_LAYERS: 1
     HIDDEN_DIM: 512
   EPOCHS: 50
   GENERIC: False
   DOM_LABELS: ["forest", "water"]
-  DOM_SPECIFIC_XE: true
+  DOM_SPECIFIC_XE: True
   ALPHA: 0.75
-  # CLIP_NN_LOSS: True
-  # COMPARE_BEFORE_AUG: True
-  # NN_INCLUDE_SAMPLE: True
-  # DOM_WEIGHT: 1.0
-  # NN_WEIGHT: 1.0
+  COMPARE_BEFORE_AUG: True
+  NN_INCLUDE_SAMPLE: True

configs/Waterbirds/mlp.yaml

@@ -10,14 +10,14 @@ EXP:
 DATA:
   DATASET: "Waterbirds"
   BATCH_SIZE: 256
+  ROOT: /shared/lisabdunlap/vl-attention/data
   LOAD_CACHED: True
 
 METHOD:
   MODEL:
     NUM_LAYERS: 1
     DOM_WEIGHT: 1.0
     LR: 0.001
-    # WEIGHT_DECAY: 0.005
     CHECKPOINT: 'checkpoint/mlp.pth'
     CHECKPOINT_NAME: "mlp"
     RESUME: False

configs/Waterbirds/mlpzs.yaml

@@ -12,6 +12,7 @@ DATA:
   DATASET: "Waterbirds"
   BATCH_SIZE: 256
   LOAD_CACHED: True
+  ROOT: /shared/lisabdunlap/vl-attention/data
 
 METHOD:
   MODEL:

configs/Waterbirds/new_lads.yaml

@@ -23,6 +23,7 @@ DATA:
   UPWEIGHT_DOMAINS: False
   UPWEIGHT_CLASSES: True
   MODEL_DIM: 1024
+  ROOT: '/shared/lisabdunlap'
 
 METHOD:
   MODEL: