improving results

Author: Lisa Dunlap

README.md

@@ -5,6 +5,11 @@ Official Implementation of [LADS (Latent Augmentation using Domain descriptionS)
 
 *WARNING: this is still WIP, please raise an issue if you run into any bugs.*
 
+## TODOs
+[] add e2e method for DA
+[] add in selective augmentation
+[] run 2 layer mlp baselines
+
 ## Getting started
 
 1. Install the dependencies for our code using Conda. You may need to adjust the environment YAML file depending on your setup.

clip_utils.py

@@ -26,67 +26,6 @@
 
 import omegaconf
 
-def get_domain_text_embs(model, cfg, source_text_prompts, target_text_prompts, class_names):
-    """
-    Gets the text embeddings of the prompts describing the source and target domains. 
-    If generic is True, source_text_prompts and target_text_prompts are strings instead of 
-    templates to put the class name in. 
-    """
-    if cfg.AUGMENTATION.GENERIC:
-        text_embeddings = zeroshot_classifier(target_text_prompts, model, normalize=cfg.METHOD.NORMALIZE, model_type=cfg.EXP.IMAGE_FEATURES)
-        text_embeddings = np.transpose(text_embeddings, (1,0))
-        orig_prompts = text_embeddings
-        if len(source_text_prompts) > 0:
-            source_embeddings = zeroshot_classifier(source_text_prompts, model, normalize=cfg.METHOD.NORMALIZE, model_type=cfg.EXP.IMAGE_FEATURES)
-            print("source emb before averaging", source_embeddings.shape)
-            source_embeddings = source_embeddings.mean(dim=0)
-            print("source emb after averaging", source_embeddings.shape)
-            diffs = torch.stack([emb-source_embeddings[0] for emb in text_embeddings])
-            diffs /= text_embeddings.norm(dim=-1, keepdim=True)
-    else:
-        print(target_text_prompts)
-        # print("yo", len(source_text_prompts), len(source_text_prompts[0]))
-        # go on a per class basis
-        templates = target_text_prompts
-        all_texts = []
-        for t in source_text_prompts:
-            texts = [[t.format(c)] for c in class_names]
-            text_emb = zeroshot_classifier(texts, model, normalize=cfg.METHOD.NORMALIZE, model_type=cfg.EXP.IMAGE_FEATURES).T
-            print(texts, "text_emb", text_emb.shape)
-            all_texts.append(text_emb)
-        if type(target_text_prompts[0]) == str:
-            target_text_prompts = [target_text_prompts]
-        print(target_text_prompts)
-        for p in target_text_prompts:
-            print(p)
-            texts = [[t.format(c) for t in p] for c in class_names]
-            text_emb = zeroshot_classifier(texts, model, normalize=cfg.METHOD.NORMALIZE, model_type=cfg.EXP.IMAGE_FEATURES).T
-            all_texts.append(text_emb)
-        # this subtracts the neutral embedding from the domain embeddings and normalizes. 
-        text_pairs = torch.stack(all_texts)
-        print("text pairs", text_pairs.shape)
-        target_embeddings, source_embeddings = text_pairs, []
-        if len(source_text_prompts) > 0:
-            source_embeddings = text_pairs[:len(source_text_prompts)]
-            target_embeddings = text_pairs[len(source_text_prompts):]
-        else:
-            source_embeddings = torch.zeros_like(target_embeddings)
-        #     text_diffs = []
-        #     source_domain = text_pairs[0]
-        #     for target_domain in text_pairs[1:]:
-        #         diff = target_domain - source_domain
-        #         diff /= np.linalg.norm(diff, axis=-1, keepdims=True)
-        #         # diff = np.expand_dims(diff, axis=0)
-        #         text_diffs.append(diff)
-        # else:
-        #     target_embeddings = text_pairs
-        #     text_diffs = text_pairs
-        # diffs = torch.stack(text_diffs).permute(1,0,2) # should be (num_classes, num_domains, emb_size)
-        # print("diffs shape", diffs.shape)
-        # print("source embeddings", source_embeddings.shape)
-        print("target embeddings", target_embeddings.shape)
-    return source_embeddings, target_embeddings
-
 def get_features(dataset, model, device, model_type):
     if model_type != 'clip' and model_type != 'openclip':
         return get_resnet_features(dataset, model, device)

configs/CUB/salemE2EMulti.yaml configs/CUB/lads_e2e.yamlconfigs/CUB/salemE2EMulti.yaml renamed to configs/CUB/lads_e2e.yaml

@@ -3,13 +3,8 @@ EXP:
   WANDB_SILENT: False
   PROJ: "CUB_Painting_E2E"
   SEED: 0
-  # TEXT_PROMPTS: [["clipart"], ["painting"], ["real photo"]]
-  # TEXT_PROMPTS: [['a painting of a bird']]
-  TEXT_PROMPTS: [['a painting of a {} bird.']]
-  # TEXT_PROMPTS: ['an anime drawing of a {} bird.']
+  TEXT_PROMPTS: [['a painting of a {} bird.'], ['art of a {} bird.'], ['a drawing of a {} bird.']]
   NEUTRAL_TEXT_PROMPTS: ['a photo of a {} bird.']
-  # NEUTRAL_TEXT_PROMPTS: [['a photo of a bird']]
-  # AUGMENTATION: 'None'
   EPOCHS: 200
   LOG_HIST: False
   ENSAMBLE: True
@@ -41,4 +36,6 @@ AUGMENTATION:
   DOM_WEIGHT: 1
   ALPHA: 0.5
   GENERIC: False
-  DOM_LABELS: ['painting']
+  DOM_LABELS: ['painting']
+  RANDOMIZE_PROB: 0.0
+  RANDOMIZE: False