fix: include data information in cache key generation (microsoft#566)

Author: you-n-g

rdagent/utils/env.py

@@ -428,6 +428,18 @@ def cached_run(
         """
         target_folder = Path(RD_AGENT_SETTINGS.pickle_cache_folder_path_str) / f"utils.env.run"
         target_folder.mkdir(parents=True, exist_ok=True)
+
+        # we must add the information of data (beyound code) into the key.
+        # Otherwise, all commands operating on data will become invalue (e.g. rm -r submission.csv)
+        # So we recursively walk in the folder and add the sorted relative filename list as part of the key.
+        data_key = []
+        for path in Path(local_path).rglob("*"):
+            p = str(path.relative_to(Path(local_path)))
+            if p.startswith("__pycache__"):
+                continue
+            data_key.append(p)
+        data_key = sorted(data_key)
+
         key = md5_hash(
             json.dumps(
                 [
@@ -437,6 +449,7 @@ def cached_run(
             )
             + json.dumps({"entry": entry, "running_extra_volume": running_extra_volume})
             + json.dumps({"extra_volumes": self.conf.extra_volumes})
+            + json.dumps(data_key)
         )
         if Path(target_folder / f"{key}.pkl").exists() and Path(target_folder / f"{key}.zip").exists():
             with open(target_folder / f"{key}.pkl", "rb") as f: