fix: Runnable on first complete & Rename method to next_incomplete_component for clarity (microsoft#615)

Author: you-n-g

rdagent/app/data_science/loop.py

@@ -93,18 +93,24 @@ def coding(self, prev_out: dict[str, Any]):
 
     def running(self, prev_out: dict[str, Any]):
         exp: DSExperiment = prev_out["coding"]
-        if self.trace.next_component_required() is None:
+        if exp.is_ready_to_run():
             new_exp = self.runner.develop(exp)
             logger.log_object(new_exp)
             return new_exp
-        else:
-            return exp
+        return exp
 
     def feedback(self, prev_out: dict[str, Any]) -> ExperimentFeedback:
+        """
+        Assumption:
+        - If we come to feedback phase, the previous development steps are successful.
+        """
         exp: DSExperiment = prev_out["running"]
-        if self.trace.next_component_required() is None:
+        if self.trace.next_incomplete_component() is None:
+            # we have alreadly completed components in previous trace. So current loop is focusing on a new proposed idea.
+            # So we need feedback for the proposal.
             feedback = self.summarizer.generate_feedback(exp, self.trace)
         else:
+            # Otherwise, it is on drafting stage, don't need complicated feedbacks.
             feedback = ExperimentFeedback(
                 reason=f"{exp.hypothesis.component} is completed.",
                 decision=True,

rdagent/scenarios/data_science/experiment/experiment.py

@@ -19,3 +19,10 @@ def __init__(self, pending_tasks_list: list, *args, **kwargs) -> None:
         self.experiment_workspace = FBWorkspace()
         self.pending_tasks_list = pending_tasks_list
         self.format_check_result = None
+
+    def is_ready_to_run(self) -> bool:
+        """
+        ready to run does not indicate the experiment is runnable
+        (so it is different from `trace.next_incomplete_component`.)
+        """
+        return self.experiment_workspace is not None and "main.py" in self.experiment_workspace.file_dict

rdagent/scenarios/data_science/proposal/exp_gen.py

@@ -92,7 +92,11 @@ def __init__(self, scen: DataScienceScen, knowledge_base: KnowledgeBase | None =
 
     COMPLETE_ORDER = ("DataLoadSpec", "FeatureEng", "Model", "Ensemble", "Workflow")
 
-    def next_component_required(self) -> COMPONENT | None:
+    def next_incomplete_component(self) -> COMPONENT | None:
+        """
+        NOTE:
+        - A component will be complete until get True decision feedback !!!
+        """
         for c in self.COMPLETE_ORDER:
             if not self.has_compponent(c):
                 return c
@@ -105,27 +109,17 @@ def has_compponent(self, component: COMPONENT) -> bool:
                 return True
         return False
 
-    def sota_experiment(self, last_n: int = -1) -> DSExperiment | None:
+    def sota_experiment(self) -> DSExperiment | None:
         """
-        Access the last experiment result.
-
-        Parameters
-        ----------
-        last_n : int
-            The index from the last experiment result to access.
-            Use -1 for the most recent experiment, -2 for the second most recent, and so on.
-
         Returns
         -------
         Experiment or None
             The experiment result if found, otherwise None.
         """
-        assert last_n < 0
-        for exp, ef in self.hist[::-1]:
-            # the sota exp should be accepted decision and all required components are completed.
-            if ef.decision and self.next_component_required() is None:
-                last_n += 1
-                if last_n == 0:
+        if self.next_incomplete_component() is None:
+            for exp, ef in self.hist[::-1]:
+                # the sota exp should be accepted decision and all required components are completed.
+                if ef.decision:
                     return exp
         return None
 
@@ -252,7 +246,7 @@ def gen(self, trace: DSTrace) -> DSExperiment:
         scenario_desc = trace.scen.get_scenario_all_desc()
         last_successful_exp = trace.last_successful_exp()
 
-        next_missing_component = trace.next_component_required()
+        next_missing_component = trace.next_incomplete_component()
 
         init_component_config = {
             "DataLoadSpec": {"task_cls": DataLoaderTask, "spec_file": None, "component_prompt_key": "data_loader"},