fix memory leak issue during puzzletron scoring, #1681

examples/puzzletron/configs/llama-3_1-8B_pruneffn_memory/validate_model_defaults.yaml

@@ -3,7 +3,7 @@ autocast_dtype: torch.bfloat16 # dtype for torch.autocast for validate_model
 block_size: 8192
 bos_rate: 0.5
 data_column: messages
-val_dataset_name: validation
+val_dataset_name: valid
 shuffle_seed: 81436
 seed: 42
 fim_rate: 0

modelopt/torch/puzzletron/tools/validate_puzzle_with_multi_replacements.py

@@ -20,6 +20,7 @@
 
 # mypy: ignore-errors
 
+import gc
 import json
 import warnings
 from functools import partial
@@ -219,6 +220,15 @@ def validate_puzzle_solutions(args: DictConfig) -> None:
             torch.cuda.empty_cache()
             torch.cuda.synchronize()
 
+            # Stitching installs hooks that create reference cycles between the
+            # model and its hook closures, so the per-solution CPU model is not
+            # reclaimed by refcounting alone. Drop references and force a cyclic
+            # GC pass to avoid accumulating one full model per iteration (host
+            # RAM OOM / SIGKILL after ~25 solutions on an 8B model).
+            del stitched_model
+            del model
+            gc.collect()
+
         dist.barrier()