Skip to content

Commit 06e7c20

Browse files
david-baekdavid-baek
committed
Add MNIST
1 parent b9fe0c2 commit 06e7c20

File tree

12 files changed

+1224
-49
lines changed

12 files changed

+1224
-49
lines changed

‎.gitignore‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,6 @@ results
44
scratch*.ipynb
55
*/slurm*.out
66

7-
figure
7+
figures
8+
harmonic_archive.zip
9+
data

‎environment.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,4 +107,4 @@ dependencies:
107107
- tqdm==4.67.1
108108
- triton==3.1.0
109109
- tzdata==2024.2
110-
prefix: /home/gridsan/dbaek/.conda/envs/crystal
110+
prefix: /om/user/dbaek/.conda/envs/crystal

‎notebooks/case_study_circle.ipynb‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,18 +50,18 @@
5050
},
5151
{
5252
"cell_type": "code",
53-
"execution_count": 47,
53+
"execution_count": 6,
5454
"metadata": {},
5555
"outputs": [
5656
{
5757
"name": "stderr",
5858
"output_type": "stream",
5959
"text": [
60-
"/tmp/ipykernel_3578131/2344840747.py:40: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
60+
"/tmp/ipykernel_3613932/1967124822.py:42: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
6161
" weights = torch.load(f'../results/case_study_{config_list[i-3][0]}_{config_list[i-3][1]}.pt')\n",
62-
"/tmp/ipykernel_3578131/2344840747.py:40: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
62+
"/tmp/ipykernel_3613932/1967124822.py:42: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
6363
" weights = torch.load(f'../results/case_study_{config_list[i-3][0]}_{config_list[i-3][1]}.pt')\n",
64-
"/tmp/ipykernel_3578131/2344840747.py:40: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
64+
"/tmp/ipykernel_3613932/1967124822.py:42: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
6565
" weights = torch.load(f'../results/case_study_{config_list[i-3][0]}_{config_list[i-3][1]}.pt')\n"
6666
]
6767
},
@@ -82,6 +82,8 @@
8282
"import matplotlib.pyplot as plt\n",
8383
"from sklearn.decomposition import PCA\n",
8484
"from adjustText import adjust_text\n",
85+
"import json\n",
86+
"import torch\n",
8587
"\n",
8688
"plt.rcParams.update({'font.size': 13})\n",
8789
"\n",
@@ -144,6 +146,13 @@
144146
"plt.subplots_adjust(wspace=0.0, hspace=0.3)\n",
145147
"plt.savefig('../figures/circle_case_study.pdf', bbox_inches='tight')"
146148
]
149+
},
150+
{
151+
"cell_type": "code",
152+
"execution_count": null,
153+
"metadata": {},
154+
"outputs": [],
155+
"source": []
147156
}
148157
],
149158
"metadata": {

‎notebooks/final_figures.ipynb‎

Lines changed: 552 additions & 0 deletions
Large diffs are not rendered by default.

‎notebooks/mnist.ipynb‎

Lines changed: 338 additions & 0 deletions
Large diffs are not rendered by default.

‎scripts/HM_equiv.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
2-
#SBATCH -t 16:00:00
2+
#SBATCH -t 23:59:00
33
#SBATCH --gres=gpu:a100:1
44

55
python ../src/run_exp.py --data_id equivalence --model_id H_MLP
6-
6+
python ../src/run_exp.py --data_id circle --model_id H_MLP

‎scripts/HM_lattice.sh‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/bin/bash
2-
#SBATCH -t 16:00:00
2+
#SBATCH -t 23:59:00
33
#SBATCH --gres=gpu:a100:1
44

55
python ../src/run_exp.py --data_id lattice --model_id H_MLP
6-
6+
python ../src/run_exp.py --data_id family_tree --model_id H_MLP
7+
python ../src/run_exp.py --data_id equivalence --model_id H_MLP
8+
python ../src/run_exp.py --data_id circle --model_id H_MLP

‎scripts/M_equiv.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#!/bin/bash
2-
#SBATCH -t 16:00:00
2+
#SBATCH -t 23:59:00
33
#SBATCH --gres=gpu:a100:1
44

55
python ../src/run_exp.py --data_id equivalence --model_id standard_MLP
6+
python ../src/run_exp.py --data_id circle --model_id standard_MLP
67

‎scripts/M_lattice.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,7 @@
33
#SBATCH --gres=gpu:a100:1
44

55
python ../src/run_exp.py --data_id lattice --model_id standard_MLP
6+
python ../src/run_exp.py --data_id family_tree --model_id standard_MLP
7+
python ../src/run_exp.py --data_id equivalence --model_id standard_MLP
8+
python ../src/run_exp.py --data_id circle --model_id standard_MLP
69

‎src/run_exp.py‎

Lines changed: 34 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535

3636
data_size = 1000
3737
train_ratio = 0.8
38-
embd_dim = 10
38+
embd_dim = 16
3939

4040
lr = 0.002
4141
weight_decay = 0.01
@@ -58,7 +58,7 @@
5858
results_root = "../results"
5959

6060
current_datetime = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
61-
results_root = f"{results_root}/{seed}-{data_id}-{model_id}"
61+
results_root = f"{results_root}/{current_datetime}-{seed}-{data_id}-{model_id}"
6262
os.mkdir(results_root)
6363

6464
param_dict_json = {k: v for k, v in param_dict.items() if k != 'device'} # since torch.device is not JSON serializable
@@ -138,42 +138,42 @@
138138
# json.dump(metric_dict, f, indent=4)
139139

140140
# ## Exp3: Metric vs Train Fraction (fixed dataset size)
141-
# print(f"Experiment 3: Metric vs Train Fraction (fixed dataset size)")
142-
# train_ratio_list = np.arange(1, 10) / 10
143-
# data_size = 1000
144-
# for i in tqdm(range(len(train_ratio_list))):
145-
# train_ratio = train_ratio_list[i]
146-
# param_dict = {
147-
# 'seed': seed,
148-
# 'data_id': data_id,
149-
# 'data_size': data_size,
150-
# 'train_ratio': train_ratio,
151-
# 'model_id': model_id,
152-
# 'device': torch.device('cuda' if torch.cuda.is_available() else 'cpu'),
153-
# 'embd_dim': embd_dim,
154-
# 'n_exp': n_exp,
155-
# 'lr': lr,
156-
# 'weight_decay':weight_decay
157-
# }
158-
# print(f"Training model with seed {seed}, data_id {data_id}, model_id {model_id}, n_exp {n_exp}, embd_dim {embd_dim}")
159-
# ret_dic = train_single_model(param_dict)
160-
# model = ret_dic['model']
161-
# dataset = ret_dic['dataset']
141+
print(f"Experiment 3: Metric vs Train Fraction (fixed dataset size)")
142+
train_ratio_list = np.arange(1, 10) / 10
143+
data_size = 1000
144+
for i in tqdm(range(len(train_ratio_list))):
145+
train_ratio = train_ratio_list[i]
146+
param_dict = {
147+
'seed': seed,
148+
'data_id': data_id,
149+
'data_size': data_size,
150+
'train_ratio': train_ratio,
151+
'model_id': model_id,
152+
'device': torch.device('cuda' if torch.cuda.is_available() else 'cpu'),
153+
'embd_dim': embd_dim,
154+
'n_exp': n_exp,
155+
'lr': lr,
156+
'weight_decay':weight_decay
157+
}
158+
print(f"Training model with seed {seed}, data_id {data_id}, model_id {model_id}, n_exp {n_exp}, embd_dim {embd_dim}")
159+
ret_dic = train_single_model(param_dict)
160+
model = ret_dic['model']
161+
dataset = ret_dic['dataset']
162162

163-
# torch.save(model.state_dict(), f"{results_root}/{seed}_{data_id}_{model_id}_{data_size}_{train_ratio}_{n_exp}.pt")
164-
# with open(f"{results_root}/{seed}_{data_id}_{model_id}_{data_size}_{train_ratio}_{n_exp}_train_results.json", "w") as f:
165-
# json.dump(ret_dic["results"], f, indent=4)
163+
torch.save(model.state_dict(), f"{results_root}/{seed}_{data_id}_{model_id}_{data_size}_{train_ratio}_{n_exp}.pt")
164+
with open(f"{results_root}/{seed}_{data_id}_{model_id}_{data_size}_{train_ratio}_{n_exp}_train_results.json", "w") as f:
165+
json.dump(ret_dic["results"], f, indent=4)
166166

167-
# if data_id == "family_tree":
168-
# aux_info["dict_level"] = dataset['dict_level']
167+
if data_id == "family_tree":
168+
aux_info["dict_level"] = dataset['dict_level']
169169

170-
# if hasattr(model.embedding, 'weight'):
171-
# metric_dict = crystal_metric(model.embedding.weight.cpu().detach(), data_id, aux_info)
172-
# else:
173-
# metric_dict = crystal_metric(model.embedding.data.cpu(), data_id, aux_info)
170+
if hasattr(model.embedding, 'weight'):
171+
metric_dict = crystal_metric(model.embedding.weight.cpu().detach(), data_id, aux_info)
172+
else:
173+
metric_dict = crystal_metric(model.embedding.data.cpu(), data_id, aux_info)
174174

175-
# with open(f"{results_root}/{seed}_{data_id}_{model_id}_{data_size}_{train_ratio}_{n_exp}_metric.json", "w") as f:
176-
# json.dump(metric_dict, f, indent=4)
175+
with open(f"{results_root}/{seed}_{data_id}_{model_id}_{data_size}_{train_ratio}_{n_exp}_metric.json", "w") as f:
176+
json.dump(metric_dict, f, indent=4)
177177

178178
## Exp4: Grokking plot: Run with different seeds
179179
print(f"Experiment 4: Train with different seeds")

0 commit comments

Comments
 (0)