From 032ed493b924a712fd584decb5f81eaf4ace73be Mon Sep 17 00:00:00 2001 From: Enis Lorenz Date: Fri, 6 Mar 2026 10:57:42 +0100 Subject: [PATCH 1/7] Add power consumption and mean price metrics for debugging and analysis. --- src/baseline.py | 24 ++++++++++++++++++++---- src/environment.py | 11 +++++++++-- src/metrics_tracker.py | 26 ++++++++++++++++++++++++++ src/reward_calculation.py | 16 ++++++++++++++++ train.py | 26 ++++++++++++++++++++++++++ 5 files changed, 97 insertions(+), 6 deletions(-) diff --git a/src/baseline.py b/src/baseline.py index 097f91c..366a6dc 100644 --- a/src/baseline.py +++ b/src/baseline.py @@ -12,7 +12,7 @@ age_backlog_queue, ) from src.metrics_tracker import MetricsTracker -from src.reward_calculation import power_cost +from src.reward_calculation import power_cost, power_consumption_mwh from src.config import CORES_PER_NODE @@ -30,7 +30,7 @@ def baseline_step( metrics: MetricsTracker, env_print: Callable[..., None], baseline_backlog_queue: deque, -) -> tuple[float, float, int, int]: +) -> tuple[float, float, float, float, int, int, int, int]: """ Run one step of the baseline simulation for comparison. @@ -52,7 +52,12 @@ def baseline_step( baseline_backlog_queue: Overflow queue for jobs that don't fit in the main queue Returns: - Tuple of (baseline_cost, baseline_cost_off, updated baseline_next_empty_slot, updated next_job_id) + Tuple of ( + baseline_cost, baseline_cost_off, + baseline_power_mwh, baseline_power_off_mwh, + updated baseline_next_empty_slot, updated next_job_id, + baseline_num_used_nodes, baseline_num_used_cores + ) """ job_queue_2d = baseline_state['job_queue'].reshape(-1, 4) @@ -113,6 +118,17 @@ def baseline_step( env_print(f" > baseline_cost: €{baseline_cost:.4f} | used nodes: {num_used_nodes}, idle nodes: {num_idle_nodes}") baseline_cost_off = power_cost(num_used_nodes, 0, current_price) env_print(f" > baseline_cost_off: €{baseline_cost_off:.4f} | used nodes: {num_used_nodes}, idle nodes: 0") + baseline_power_mwh = power_consumption_mwh(num_used_nodes, num_idle_nodes) + baseline_power_off_mwh = power_consumption_mwh(num_used_nodes, 0) num_used_cores = num_on_nodes * CORES_PER_NODE - np.sum(baseline_cores_available) - return baseline_cost, baseline_cost_off, baseline_next_empty_slot, next_job_id, num_used_nodes, num_used_cores + return ( + baseline_cost, + baseline_cost_off, + baseline_power_mwh, + baseline_power_off_mwh, + baseline_next_empty_slot, + next_job_id, + num_used_nodes, + num_used_cores, + ) diff --git a/src/environment.py b/src/environment.py index 2dc1da9..b218363 100644 --- a/src/environment.py +++ b/src/environment.py @@ -27,7 +27,7 @@ assign_jobs_to_available_nodes, fill_queue_from_backlog, age_backlog_queue ) from src.node_management import adjust_nodes -from src.reward_calculation import RewardCalculator +from src.reward_calculation import RewardCalculator, power_consumption_mwh from src.baseline import baseline_step from src.workload_generator import generate_jobs from src.metrics_tracker import MetricsTracker @@ -452,7 +452,7 @@ def step(self, action: np.ndarray) -> tuple[dict[str, np.ndarray], float, bool, self.env_print(f"[5] Calculating reward...") # Baseline step - baseline_cost, baseline_cost_off, self.baseline_next_empty_slot, self.next_job_id, baseline_num_used_nodes, baseline_num_used_cores = baseline_step( + baseline_cost, baseline_cost_off, baseline_power_mwh, baseline_power_off_mwh, self.baseline_next_empty_slot, self.next_job_id, baseline_num_used_nodes, baseline_num_used_cores = baseline_step( self.baseline_state, self.baseline_cores_available, self.baseline_running_jobs, current_price, new_jobs_count, new_jobs_durations, new_jobs_nodes, new_jobs_cores, self.baseline_next_empty_slot, self.next_job_id, self.metrics, self.env_print, @@ -463,6 +463,10 @@ def step(self, action: np.ndarray) -> tuple[dict[str, np.ndarray], float, bool, self.metrics.baseline_cost_off += baseline_cost_off self.metrics.episode_baseline_cost += baseline_cost self.metrics.episode_baseline_cost_off += baseline_cost_off + self.metrics.baseline_power_consumption_mwh += baseline_power_mwh + self.metrics.baseline_power_consumption_off_mwh += baseline_power_off_mwh + self.metrics.episode_baseline_power_consumption_mwh += baseline_power_mwh + self.metrics.episode_baseline_power_consumption_off_mwh += baseline_power_off_mwh self.metrics.episode_baseline_used_nodes.append(baseline_num_used_nodes) self.metrics.episode_baseline_used_cores.append(baseline_num_used_cores) @@ -476,6 +480,9 @@ def step(self, action: np.ndarray) -> tuple[dict[str, np.ndarray], float, bool, self.metrics.episode_reward += step_reward self.metrics.total_cost += step_cost self.metrics.episode_total_cost += step_cost + step_power_mwh = power_consumption_mwh(num_used_nodes, num_idle_nodes) + self.metrics.total_power_consumption_mwh += step_power_mwh + self.metrics.episode_total_power_consumption_mwh += step_power_mwh # Store normalized reward components for plotting self.metrics.eff_rewards.append(eff_reward_norm * 100) diff --git a/src/metrics_tracker.py b/src/metrics_tracker.py index 1f5f0c9..67bac65 100644 --- a/src/metrics_tracker.py +++ b/src/metrics_tracker.py @@ -4,6 +4,11 @@ class MetricsTracker: """Tracks metrics throughout training episodes.""" + @staticmethod + def _effective_mean_price(total_cost: float, total_power_mwh: float) -> float: + """Effective mean price in €/MWh, weighted by consumed energy.""" + return (total_cost / total_power_mwh) if total_power_mwh > 0.0 else 0.0 + def __init__(self) -> None: """Initialize all metric counters.""" self.reset_timeline_metrics() @@ -21,6 +26,9 @@ def reset_timeline_metrics(self) -> None: self.total_cost: float = 0.0 self.baseline_cost: float = 0.0 self.baseline_cost_off: float = 0.0 + self.total_power_consumption_mwh: float = 0.0 + self.baseline_power_consumption_mwh: float = 0.0 + self.baseline_power_consumption_off_mwh: float = 0.0 # Agent job metrics (cumulative across episodes) self.jobs_submitted: int = 0 @@ -59,6 +67,9 @@ def reset_episode_metrics(self) -> None: self.episode_total_cost: float = 0.0 self.episode_baseline_cost: float = 0.0 self.episode_baseline_cost_off: float = 0.0 + self.episode_total_power_consumption_mwh: float = 0.0 + self.episode_baseline_power_consumption_mwh: float = 0.0 + self.episode_baseline_power_consumption_off_mwh: float = 0.0 # Agent job metrics (episode) self.episode_jobs_submitted: int = 0 @@ -138,12 +149,27 @@ def record_episode_completion(self, current_episode: int) -> dict[str, float | i if self.episode_baseline_jobs_submitted else 0.0 ) + agent_mean_price: float = self._effective_mean_price( + self.episode_total_cost, self.episode_total_power_consumption_mwh + ) + baseline_mean_price: float = self._effective_mean_price( + self.episode_baseline_cost, self.episode_baseline_power_consumption_mwh + ) + baseline_off_mean_price: float = self._effective_mean_price( + self.episode_baseline_cost_off, self.episode_baseline_power_consumption_off_mwh + ) episode_data: dict[str, float | int] = { 'episode': current_episode, 'agent_cost': self.episode_total_cost, 'baseline_cost': self.episode_baseline_cost, 'baseline_cost_off': self.episode_baseline_cost_off, + 'agent_power_consumption_mwh': self.episode_total_power_consumption_mwh, + 'baseline_power_consumption_mwh': self.episode_baseline_power_consumption_mwh, + 'baseline_power_consumption_off_mwh': self.episode_baseline_power_consumption_off_mwh, + 'agent_mean_price': agent_mean_price, + 'baseline_mean_price': baseline_mean_price, + 'baseline_off_mean_price': baseline_off_mean_price, 'savings_vs_baseline': self.episode_baseline_cost - self.episode_total_cost, 'savings_vs_baseline_off': self.episode_baseline_cost_off - self.episode_total_cost, 'savings_pct_baseline': ((self.episode_baseline_cost - self.episode_total_cost) / self.episode_baseline_cost) * 100 if self.episode_baseline_cost > 0 else 0.0, diff --git a/src/reward_calculation.py b/src/reward_calculation.py index 1c052b2..d067953 100644 --- a/src/reward_calculation.py +++ b/src/reward_calculation.py @@ -30,6 +30,22 @@ def power_cost(num_used_nodes: int, num_idle_nodes: int, current_price: float) - return total_cost +def power_consumption_mwh(num_used_nodes: int, num_idle_nodes: int) -> float: + """ + Calculate energy consumption for one environment step. + + One environment step equals one hour, so this is both average MW and MWh/step. + + Args: + num_used_nodes: Number of nodes with jobs running + num_idle_nodes: Number of idle (on but unused) nodes + + Returns: + Energy consumption in MWh for this step + """ + return COST_IDLE_MW * num_idle_nodes + COST_USED_MW * num_used_nodes + + class RewardCalculator: """Calculates rewards with pre-computed normalization bounds.""" diff --git a/train.py b/train.py index b11ee0c..6c9fea3 100644 --- a/train.py +++ b/train.py @@ -234,10 +234,18 @@ def main(): savings_vs_baseline_off = env.metrics.baseline_cost_off - env.metrics.total_cost completion_rate = (env.metrics.jobs_completed / env.metrics.jobs_submitted * 100) if env.metrics.jobs_submitted > 0 else 0 avg_wait = env.metrics.total_job_wait_time / env.metrics.jobs_completed if env.metrics.jobs_completed > 0 else 0 + agent_power_mwh = env.metrics.episode_total_power_consumption_mwh + baseline_power_mwh = env.metrics.episode_baseline_power_consumption_mwh + baseline_power_off_mwh = env.metrics.episode_baseline_power_consumption_off_mwh + agent_mean_price = (env.metrics.episode_total_cost / agent_power_mwh) if agent_power_mwh > 0 else 0.0 + baseline_mean_price = (env.metrics.episode_baseline_cost / baseline_power_mwh) if baseline_power_mwh > 0 else 0.0 + baseline_off_mean_price = (env.metrics.episode_baseline_cost_off / baseline_power_off_mwh) if baseline_power_off_mwh > 0 else 0.0 print(f" Episode {episode + 1}: " f"Agent Cost=€{env.metrics.total_cost:.0f}, " f"Baseline Cost=€{env.metrics.baseline_cost:.0f} | Baseline Off=€{env.metrics.baseline_cost_off:.0f}, " f"Savings=€{savings_vs_baseline:.0f}/€{savings_vs_baseline_off:.0f}, " + f"Power={agent_power_mwh:.1f}/{baseline_power_mwh:.1f}/{baseline_power_off_mwh:.1f} MWh (agent/base/base_off), " + f"MeanPrice={agent_mean_price:.2f}/{baseline_mean_price:.2f}/{baseline_off_mean_price:.2f} €/MWh (agent/base/base_off), " f"Jobs={env.metrics.jobs_completed}/{env.metrics.jobs_submitted} ({completion_rate:.0f}%), " f"AvgWait={avg_wait:.1f}h, " f"EpisodeMaxQueue={env.metrics.episode_max_queue_size_reached}, Dropped={env.metrics.episode_jobs_dropped}, " @@ -273,17 +281,35 @@ def main(): avg_baseline_wait_time = sum(ep['baseline_avg_wait_time'] * ep['baseline_jobs_completed'] for ep in env.metrics.episode_costs) / total_baseline_completed if total_baseline_completed > 0 else 0 avg_max_queue = sum(ep['max_queue_size'] for ep in env.metrics.episode_costs) / len(env.metrics.episode_costs) avg_baseline_max_queue = sum(ep['baseline_max_queue_size'] for ep in env.metrics.episode_costs) / len(env.metrics.episode_costs) + total_agent_cost = sum(float(ep['agent_cost']) for ep in env.metrics.episode_costs) + total_baseline_cost = sum(float(ep['baseline_cost']) for ep in env.metrics.episode_costs) + total_baseline_off_cost = sum(float(ep['baseline_cost_off']) for ep in env.metrics.episode_costs) + total_agent_power_mwh = sum(float(ep.get('agent_power_consumption_mwh', 0.0)) for ep in env.metrics.episode_costs) + total_baseline_power_mwh = sum(float(ep.get('baseline_power_consumption_mwh', 0.0)) for ep in env.metrics.episode_costs) + total_baseline_off_power_mwh = sum(float(ep.get('baseline_power_consumption_off_mwh', 0.0)) for ep in env.metrics.episode_costs) + total_agent_mean_price = (total_agent_cost / total_agent_power_mwh) if total_agent_power_mwh > 0 else 0.0 + total_baseline_mean_price = (total_baseline_cost / total_baseline_power_mwh) if total_baseline_power_mwh > 0 else 0.0 + total_baseline_off_mean_price = (total_baseline_off_cost / total_baseline_off_power_mwh) if total_baseline_off_power_mwh > 0 else 0.0 print(f"\n=== JOB PROCESSING METRICS ===") print(f"\nAgent:") print(f" Jobs Completed: {total_jobs_completed:,} / {total_jobs_submitted:,} ({total_jobs_completed/total_jobs_submitted*100:.1f}%)") print(f" Average Wait Time: {avg_wait_time:.1f} hours") print(f" Average Max Queue Size: {avg_max_queue:.0f}") + print(f" Total Cost: €{total_agent_cost:,.0f}") print(f"\nBaseline:") print(f" Jobs Completed: {total_baseline_completed:,} / {total_baseline_submitted:,} ({total_baseline_completed/total_baseline_submitted*100:.1f}%)") print(f" Average Wait Time: {avg_baseline_wait_time:.1f} hours") print(f" Average Max Queue Size: {avg_baseline_max_queue:.0f}") + print(f" Baseline Total Cost: €{total_baseline_cost:,.0f}") + print(f" Baseline_off Total Cost: €{total_baseline_off_cost:,.0f}") + + + print(f"\n=== POWER & PRICE METRICS (TOTAL OVER EVALUATION) ===") + print(f" Agent: Power={total_agent_power_mwh:,.1f} MWh, Mean Price={total_agent_mean_price:.2f} €/MWh") + print(f" Baseline: Power={total_baseline_power_mwh:,.1f} MWh, Mean Price={total_baseline_mean_price:.2f} €/MWh") + print(f" Baseline_off: Power={total_baseline_off_power_mwh:,.1f} MWh, Mean Price={total_baseline_off_mean_price:.2f} €/MWh") except Exception as e: print(f"Could not generate cumulative savings plot: {e}") From 5c39257dd8a2b3c0bd20de5eb8281affe11575b9 Mon Sep 17 00:00:00 2001 From: Enis Lorenz Date: Mon, 9 Mar 2026 12:14:09 +0100 Subject: [PATCH 2/7] Evaluation: add cost-normalized job metrics and persist them in episode_costs - add evaluation output for cost per 1,000 completed jobs (agent/baseline/baseline_off) - add evaluation output for agent dropped jobs per saved euro (vs baseline and baseline_off) - add safe ratio/format helpers in train.py for robust logging and n/a handling - store new per-episode metrics in MetricsTracker.record_episode_completion(): - agent_cost_per_1000_completed_jobs - baseline_cost_per_1000_completed_jobs - baseline_off_cost_per_1000_completed_jobs - agent_dropped_jobs_per_saved_euro - agent_dropped_jobs_per_saved_euro_off - use NaN for undefined ratios (e.g., zero completed jobs or non-positive savings) --- src/metrics_tracker.py | 32 +++++++++++++++++++++++++++-- train.py | 46 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/src/metrics_tracker.py b/src/metrics_tracker.py index 67bac65..5f478ff 100644 --- a/src/metrics_tracker.py +++ b/src/metrics_tracker.py @@ -9,6 +9,11 @@ def _effective_mean_price(total_cost: float, total_power_mwh: float) -> float: """Effective mean price in €/MWh, weighted by consumed energy.""" return (total_cost / total_power_mwh) if total_power_mwh > 0.0 else 0.0 + @staticmethod + def _safe_ratio(numerator: float, denominator: float) -> float: + """Safe division; returns NaN when denominator is not positive.""" + return (numerator / denominator) if denominator > 0.0 else float("nan") + def __init__(self) -> None: """Initialize all metric counters.""" self.reset_timeline_metrics() @@ -158,6 +163,24 @@ def record_episode_completion(self, current_episode: int) -> dict[str, float | i baseline_off_mean_price: float = self._effective_mean_price( self.episode_baseline_cost_off, self.episode_baseline_power_consumption_off_mwh ) + agent_cost_per_1000_completed: float = self._safe_ratio( + self.episode_total_cost * 1000.0, float(self.episode_jobs_completed) + ) + baseline_cost_per_1000_completed: float = self._safe_ratio( + self.episode_baseline_cost * 1000.0, float(self.episode_baseline_jobs_completed) + ) + # baseline_off is a cost variant of baseline scheduling, so it uses the same completed-job count. + baseline_off_cost_per_1000_completed: float = self._safe_ratio( + self.episode_baseline_cost_off * 1000.0, float(self.episode_baseline_jobs_completed) + ) + savings_vs_baseline: float = self.episode_baseline_cost - self.episode_total_cost + savings_vs_baseline_off: float = self.episode_baseline_cost_off - self.episode_total_cost + dropped_jobs_per_saved_euro: float = self._safe_ratio( + float(self.episode_jobs_dropped), savings_vs_baseline + ) if savings_vs_baseline > 0.0 else float("nan") + dropped_jobs_per_saved_euro_off: float = self._safe_ratio( + float(self.episode_jobs_dropped), savings_vs_baseline_off + ) if savings_vs_baseline_off > 0.0 else float("nan") episode_data: dict[str, float | int] = { 'episode': current_episode, @@ -170,10 +193,15 @@ def record_episode_completion(self, current_episode: int) -> dict[str, float | i 'agent_mean_price': agent_mean_price, 'baseline_mean_price': baseline_mean_price, 'baseline_off_mean_price': baseline_off_mean_price, - 'savings_vs_baseline': self.episode_baseline_cost - self.episode_total_cost, - 'savings_vs_baseline_off': self.episode_baseline_cost_off - self.episode_total_cost, + 'savings_vs_baseline': savings_vs_baseline, + 'savings_vs_baseline_off': savings_vs_baseline_off, 'savings_pct_baseline': ((self.episode_baseline_cost - self.episode_total_cost) / self.episode_baseline_cost) * 100 if self.episode_baseline_cost > 0 else 0.0, 'savings_pct_baseline_off': ((self.episode_baseline_cost_off - self.episode_total_cost) / self.episode_baseline_cost_off) * 100 if self.episode_baseline_cost_off > 0 else 0.0, + 'agent_cost_per_1000_completed_jobs': agent_cost_per_1000_completed, + 'baseline_cost_per_1000_completed_jobs': baseline_cost_per_1000_completed, + 'baseline_off_cost_per_1000_completed_jobs': baseline_off_cost_per_1000_completed, + 'agent_dropped_jobs_per_saved_euro': dropped_jobs_per_saved_euro, + 'agent_dropped_jobs_per_saved_euro_off': dropped_jobs_per_saved_euro_off, 'total_reward': self.episode_reward, # Agent job metrics 'jobs_submitted': self.episode_jobs_submitted, diff --git a/train.py b/train.py index 6c9fea3..7ba3e5c 100644 --- a/train.py +++ b/train.py @@ -22,6 +22,19 @@ def norm_path(x): return None if (x is None or str(x).strip() == "") else x + +def safe_ratio(numerator: float, denominator: float) -> float | None: + """Return numerator/denominator, or None when denominator is not positive.""" + return (numerator / denominator) if denominator > 0 else None + + +def fmt_optional(value: float | None, precision: int = 2, thousands: bool = False) -> str: + """Format float values for logs, using 'n/a' when value is undefined.""" + if value is None: + return "n/a" + return f"{value:,.{precision}f}" if thousands else f"{value:.{precision}f}" + + STEPS_PER_ITERATION = 100000 @@ -240,12 +253,20 @@ def main(): agent_mean_price = (env.metrics.episode_total_cost / agent_power_mwh) if agent_power_mwh > 0 else 0.0 baseline_mean_price = (env.metrics.episode_baseline_cost / baseline_power_mwh) if baseline_power_mwh > 0 else 0.0 baseline_off_mean_price = (env.metrics.episode_baseline_cost_off / baseline_power_off_mwh) if baseline_power_off_mwh > 0 else 0.0 + agent_cost_per_1000_completed = safe_ratio(env.metrics.total_cost * 1000.0, env.metrics.jobs_completed) + baseline_cost_per_1000_completed = safe_ratio(env.metrics.baseline_cost * 1000.0, env.metrics.baseline_jobs_completed) + # baseline_off is a cost variant of baseline scheduling, so it uses the same completed-job count. + baseline_off_cost_per_1000_completed = safe_ratio(env.metrics.baseline_cost_off * 1000.0, env.metrics.baseline_jobs_completed) + dropped_jobs_per_saved_euro = safe_ratio(env.metrics.episode_jobs_dropped, savings_vs_baseline) if savings_vs_baseline > 0 else None + dropped_jobs_per_saved_euro_off = safe_ratio(env.metrics.episode_jobs_dropped, savings_vs_baseline_off) if savings_vs_baseline_off > 0 else None print(f" Episode {episode + 1}: " f"Agent Cost=€{env.metrics.total_cost:.0f}, " f"Baseline Cost=€{env.metrics.baseline_cost:.0f} | Baseline Off=€{env.metrics.baseline_cost_off:.0f}, " f"Savings=€{savings_vs_baseline:.0f}/€{savings_vs_baseline_off:.0f}, " f"Power={agent_power_mwh:.1f}/{baseline_power_mwh:.1f}/{baseline_power_off_mwh:.1f} MWh (agent/base/base_off), " f"MeanPrice={agent_mean_price:.2f}/{baseline_mean_price:.2f}/{baseline_off_mean_price:.2f} €/MWh (agent/base/base_off), " + f"CostPer1kCompleted={fmt_optional(agent_cost_per_1000_completed, 1, thousands=True)}/{fmt_optional(baseline_cost_per_1000_completed, 1, thousands=True)}/{fmt_optional(baseline_off_cost_per_1000_completed, 1, thousands=True)} €/1k (agent/base/base_off), " + f"DroppedPerSavedEuro={fmt_optional(dropped_jobs_per_saved_euro, 6)}/{fmt_optional(dropped_jobs_per_saved_euro_off, 6)} jobs/€ (vs base/base_off), " f"Jobs={env.metrics.jobs_completed}/{env.metrics.jobs_submitted} ({completion_rate:.0f}%), " f"AvgWait={avg_wait:.1f}h, " f"EpisodeMaxQueue={env.metrics.episode_max_queue_size_reached}, Dropped={env.metrics.episode_jobs_dropped}, " @@ -284,27 +305,48 @@ def main(): total_agent_cost = sum(float(ep['agent_cost']) for ep in env.metrics.episode_costs) total_baseline_cost = sum(float(ep['baseline_cost']) for ep in env.metrics.episode_costs) total_baseline_off_cost = sum(float(ep['baseline_cost_off']) for ep in env.metrics.episode_costs) + total_jobs_dropped = sum(int(ep.get('jobs_dropped', 0)) for ep in env.metrics.episode_costs) total_agent_power_mwh = sum(float(ep.get('agent_power_consumption_mwh', 0.0)) for ep in env.metrics.episode_costs) total_baseline_power_mwh = sum(float(ep.get('baseline_power_consumption_mwh', 0.0)) for ep in env.metrics.episode_costs) total_baseline_off_power_mwh = sum(float(ep.get('baseline_power_consumption_off_mwh', 0.0)) for ep in env.metrics.episode_costs) total_agent_mean_price = (total_agent_cost / total_agent_power_mwh) if total_agent_power_mwh > 0 else 0.0 total_baseline_mean_price = (total_baseline_cost / total_baseline_power_mwh) if total_baseline_power_mwh > 0 else 0.0 total_baseline_off_mean_price = (total_baseline_off_cost / total_baseline_off_power_mwh) if total_baseline_off_power_mwh > 0 else 0.0 + total_agent_completion_rate = (total_jobs_completed / total_jobs_submitted * 100) if total_jobs_submitted > 0 else 0.0 + total_baseline_completion_rate = (total_baseline_completed / total_baseline_submitted * 100) if total_baseline_submitted > 0 else 0.0 + total_savings_vs_baseline = total_baseline_cost - total_agent_cost + total_savings_vs_baseline_off = total_baseline_off_cost - total_agent_cost + total_agent_cost_per_1000_completed = safe_ratio(total_agent_cost * 1000.0, total_jobs_completed) + total_baseline_cost_per_1000_completed = safe_ratio(total_baseline_cost * 1000.0, total_baseline_completed) + # baseline_off is a cost variant of baseline scheduling, so it uses the same completed-job count. + total_baseline_off_cost_per_1000_completed = safe_ratio(total_baseline_off_cost * 1000.0, total_baseline_completed) + total_dropped_jobs_per_saved_euro = safe_ratio(total_jobs_dropped, total_savings_vs_baseline) if total_savings_vs_baseline > 0 else None + total_dropped_jobs_per_saved_euro_off = safe_ratio(total_jobs_dropped, total_savings_vs_baseline_off) if total_savings_vs_baseline_off > 0 else None print(f"\n=== JOB PROCESSING METRICS ===") print(f"\nAgent:") - print(f" Jobs Completed: {total_jobs_completed:,} / {total_jobs_submitted:,} ({total_jobs_completed/total_jobs_submitted*100:.1f}%)") + print(f" Jobs Completed: {total_jobs_completed:,} / {total_jobs_submitted:,} ({total_agent_completion_rate:.1f}%)") print(f" Average Wait Time: {avg_wait_time:.1f} hours") print(f" Average Max Queue Size: {avg_max_queue:.0f}") print(f" Total Cost: €{total_agent_cost:,.0f}") print(f"\nBaseline:") - print(f" Jobs Completed: {total_baseline_completed:,} / {total_baseline_submitted:,} ({total_baseline_completed/total_baseline_submitted*100:.1f}%)") + print(f" Jobs Completed: {total_baseline_completed:,} / {total_baseline_submitted:,} ({total_baseline_completion_rate:.1f}%)") print(f" Average Wait Time: {avg_baseline_wait_time:.1f} hours") print(f" Average Max Queue Size: {avg_baseline_max_queue:.0f}") print(f" Baseline Total Cost: €{total_baseline_cost:,.0f}") print(f" Baseline_off Total Cost: €{total_baseline_off_cost:,.0f}") + print(f"\n=== COST PER 1,000 COMPLETED JOBS ===") + print(f" Agent: {fmt_optional(total_agent_cost_per_1000_completed, 2, thousands=True)} €/1k jobs") + print(f" Baseline: {fmt_optional(total_baseline_cost_per_1000_completed, 2, thousands=True)} €/1k jobs") + print(f" Baseline_off: {fmt_optional(total_baseline_off_cost_per_1000_completed, 2, thousands=True)} €/1k jobs") + + print(f"\n=== AGENT DROPPED JOBS PER SAVED EURO ===") + print(f" Total Dropped Jobs (Agent): {total_jobs_dropped:,}") + print(f" Vs Baseline: {fmt_optional(total_dropped_jobs_per_saved_euro, 6)} jobs/€") + print(f" Vs Baseline_off: {fmt_optional(total_dropped_jobs_per_saved_euro_off, 6)} jobs/€") + print(f"\n=== POWER & PRICE METRICS (TOTAL OVER EVALUATION) ===") print(f" Agent: Power={total_agent_power_mwh:,.1f} MWh, Mean Price={total_agent_mean_price:.2f} €/MWh") From 5538e50af4fdbce7bf878219aae8616a9ea2daf0 Mon Sep 17 00:00:00 2001 From: Enis Lorenz Date: Mon, 9 Mar 2026 15:00:04 +0100 Subject: [PATCH 3/7] Analysis: add third occupancy row for cost-per-1k and power deltas - extend lambda sweep parser to read per-episode CostPer1kCompleted and Power triplets - compute occupancy-based percent deltas: - (baseline - agent) / baseline for cost per 1k completed jobs - (baseline_off - agent) / baseline_off for cost per 1k completed jobs - (baseline_off - agent) / baseline_off for power - expand trend plot from 2x3 to 3x3 and add the three new panels - include new delta mean/std fields in run stats and summary.csv --- analyze_lambda_occupancy.py | 148 +++++++++++++++++++++++++++++++++++- 1 file changed, 145 insertions(+), 3 deletions(-) diff --git a/analyze_lambda_occupancy.py b/analyze_lambda_occupancy.py index 78fbabe..5136736 100644 --- a/analyze_lambda_occupancy.py +++ b/analyze_lambda_occupancy.py @@ -7,6 +7,9 @@ 4) lambda -> completion rate 5) occupancy -> effective savings 6) occupancy -> effective savings_off +7) occupancy -> (baseline - agent) cost_per_1000_completed_jobs / baseline +8) occupancy -> (baseline_off - agent) cost_per_1000_completed_jobs / baseline_off +9) occupancy -> (baseline_off - agent) power / baseline_off For each lambda, this script runs train.py in evaluation mode for one year (12 months = 24 episodes), parses per-episode metrics from stdout, computes @@ -39,6 +42,10 @@ EPISODE_RE = re.compile( r"Episode\s+(?P\d+):.*?" r"Savings=€(?P-?[\d,]+(?:\.\d+)?)\/€(?P-?[\d,]+(?:\.\d+)?),.*?" + r"Power=(?P-?[\d.]+)\/(?P-?[\d.]+)\/(?P-?[\d.]+)\s*MWh.*?" + r"CostPer1kCompleted=(?P-?[\d,]+(?:\.\d+)?|n/a)\/" + r"(?P-?[\d,]+(?:\.\d+)?|n/a)\/" + r"(?P-?[\d,]+(?:\.\d+)?|n/a)\s*€/1k.*?" r"Jobs=[\d,]+\/[\d,]+\s+\((?P-?[\d.]+)%\),\s*" r"AvgWait=(?P-?[\d.]+)h,.*?" r"Agent Occupancy \(Nodes\)=\s*(?P-?[\d.]+)%", @@ -72,6 +79,12 @@ class LambdaRunStats: effective_savings_std: float effective_savings_off_mean: float effective_savings_off_std: float + cost_per_1k_delta_pct_baseline_mean: float + cost_per_1k_delta_pct_baseline_std: float + cost_per_1k_delta_pct_baseline_off_mean: float + cost_per_1k_delta_pct_baseline_off_std: float + power_delta_pct_baseline_off_mean: float + power_delta_pct_baseline_off_std: float annual_total_savings: float annual_total_savings_off: float command: list[str] @@ -82,12 +95,24 @@ class LambdaRunStats: completion_rate_samples: list[float] = field(default_factory=list) effective_savings_samples: list[float] = field(default_factory=list) effective_savings_off_samples: list[float] = field(default_factory=list) + cost_per_1k_delta_pct_baseline_samples: list[float] = field(default_factory=list) + cost_per_1k_delta_pct_baseline_off_samples: list[float] = field(default_factory=list) + power_delta_pct_baseline_off_samples: list[float] = field(default_factory=list) def _to_float(raw: str) -> float: return float(raw.replace(",", "")) +def _to_float_or_nan(raw: str | None) -> float: + if raw is None: + return float("nan") + val = raw.strip().lower() + if val in {"n/a", "nan"}: + return float("nan") + return _to_float(raw) + + def _diff_cumulative(values: list[float]) -> np.ndarray: arr = np.asarray(values, dtype=float) if arr.size == 0: @@ -95,12 +120,17 @@ def _diff_cumulative(values: list[float]) -> np.ndarray: return np.diff(np.concatenate(([0.0], arr))) -def parse_episode_metrics(stdout: str) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: +def parse_episode_metrics(stdout: str) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: occupancy = [] cumulative_savings = [] cumulative_savings_off = [] completion_rate = [] avg_wait = [] + agent_cost_1k = [] + baseline_cost_1k = [] + baseline_off_cost_1k = [] + agent_power = [] + baseline_off_power = [] for match in EPISODE_RE.finditer(stdout): occupancy.append(_to_float(match.group("occupancy"))) @@ -108,6 +138,11 @@ def parse_episode_metrics(stdout: str) -> tuple[np.ndarray, np.ndarray, np.ndarr cumulative_savings_off.append(_to_float(match.group("savings_off"))) completion_rate.append(_to_float(match.group("completion_rate"))) avg_wait.append(_to_float(match.group("avg_wait"))) + agent_cost_1k.append(_to_float_or_nan(match.group("agent_cost_1k"))) + baseline_cost_1k.append(_to_float_or_nan(match.group("baseline_cost_1k"))) + baseline_off_cost_1k.append(_to_float_or_nan(match.group("baseline_off_cost_1k"))) + agent_power.append(_to_float(match.group("agent_power"))) + baseline_off_power.append(_to_float(match.group("baseline_off_power"))) if not occupancy: raise RuntimeError( @@ -123,6 +158,11 @@ def parse_episode_metrics(stdout: str) -> tuple[np.ndarray, np.ndarray, np.ndarr episode_savings_off, np.asarray(completion_rate, dtype=float), np.asarray(avg_wait, dtype=float), + np.asarray(agent_cost_1k, dtype=float), + np.asarray(baseline_cost_1k, dtype=float), + np.asarray(baseline_off_cost_1k, dtype=float), + np.asarray(agent_power, dtype=float), + np.asarray(baseline_off_power, dtype=float), ) @@ -139,6 +179,16 @@ def safe_divide(numer: np.ndarray, denom: float) -> np.ndarray: return numer / denom +def safe_divide_arrays(numer: np.ndarray, denom: np.ndarray) -> np.ndarray: + numer_arr = np.asarray(numer, dtype=float) + denom_arr = np.asarray(denom, dtype=float) + out = np.full_like(numer_arr, np.nan, dtype=float) + finite = np.isfinite(numer_arr) & np.isfinite(denom_arr) + valid = finite & (np.abs(denom_arr) >= 1e-12) + out[valid] = numer_arr[valid] / denom_arr[valid] + return out + + def finite_mean_std(values: np.ndarray) -> tuple[float, float]: finite = np.isfinite(values) if not np.any(finite): @@ -156,12 +206,23 @@ def make_run_stats( completion_rate: np.ndarray, agent_avg_wait_hours: float, baseline_avg_wait_hours: float, + agent_cost_1k: np.ndarray, + baseline_cost_1k: np.ndarray, + baseline_off_cost_1k: np.ndarray, + agent_power: np.ndarray, + baseline_off_power: np.ndarray, ) -> LambdaRunStats: wait_delta_hours = agent_avg_wait_hours - baseline_avg_wait_hours effective_savings = safe_divide(savings * (completion_rate/100)**2, wait_delta_hours+1) effective_savings_off = safe_divide(savings_off * (completion_rate/100)**2, wait_delta_hours+1) effective_savings_mean, effective_savings_std = finite_mean_std(effective_savings) effective_savings_off_mean, effective_savings_off_std = finite_mean_std(effective_savings_off) + cost_per_1k_delta_pct_baseline = safe_divide_arrays((baseline_cost_1k - agent_cost_1k) * 100.0, baseline_cost_1k) + cost_per_1k_delta_pct_baseline_off = safe_divide_arrays((baseline_off_cost_1k - agent_cost_1k) * 100.0, baseline_off_cost_1k) + power_delta_pct_baseline_off = safe_divide_arrays((baseline_off_power - agent_power) * 100.0, baseline_off_power) + cost_per_1k_delta_pct_baseline_mean, cost_per_1k_delta_pct_baseline_std = finite_mean_std(cost_per_1k_delta_pct_baseline) + cost_per_1k_delta_pct_baseline_off_mean, cost_per_1k_delta_pct_baseline_off_std = finite_mean_std(cost_per_1k_delta_pct_baseline_off) + power_delta_pct_baseline_off_mean, power_delta_pct_baseline_off_std = finite_mean_std(power_delta_pct_baseline_off) return LambdaRunStats( lambda_value=lambda_value, episodes=int(occupancy.size), @@ -180,6 +241,12 @@ def make_run_stats( effective_savings_std=effective_savings_std, effective_savings_off_mean=effective_savings_off_mean, effective_savings_off_std=effective_savings_off_std, + cost_per_1k_delta_pct_baseline_mean=cost_per_1k_delta_pct_baseline_mean, + cost_per_1k_delta_pct_baseline_std=cost_per_1k_delta_pct_baseline_std, + cost_per_1k_delta_pct_baseline_off_mean=cost_per_1k_delta_pct_baseline_off_mean, + cost_per_1k_delta_pct_baseline_off_std=cost_per_1k_delta_pct_baseline_off_std, + power_delta_pct_baseline_off_mean=power_delta_pct_baseline_off_mean, + power_delta_pct_baseline_off_std=power_delta_pct_baseline_off_std, annual_total_savings=float(np.sum(savings)), annual_total_savings_off=float(np.sum(savings_off)), command=command, @@ -190,6 +257,9 @@ def make_run_stats( completion_rate_samples=completion_rate.tolist(), effective_savings_samples=effective_savings.tolist(), effective_savings_off_samples=effective_savings_off.tolist(), + cost_per_1k_delta_pct_baseline_samples=cost_per_1k_delta_pct_baseline.tolist(), + cost_per_1k_delta_pct_baseline_off_samples=cost_per_1k_delta_pct_baseline_off.tolist(), + power_delta_pct_baseline_off_samples=power_delta_pct_baseline_off.tolist(), ) @@ -280,7 +350,7 @@ def run_lambda_eval(args: argparse.Namespace, project_root: Path, lambda_value: f"Last output lines:\n{os_tail(combined_output, lines=40)}" ) - occupancy, savings, savings_off, completion_rate, avg_wait = parse_episode_metrics(combined_output) + occupancy, savings, savings_off, completion_rate, avg_wait, agent_cost_1k, baseline_cost_1k, baseline_off_cost_1k, agent_power, baseline_off_power = parse_episode_metrics(combined_output) agent_wait_summary, baseline_wait_summary = parse_wait_summary(combined_output) if agent_wait_summary is None or baseline_wait_summary is None: print(f"[warn] lambda={lambda_value}: could not parse run-level wait summary; effective savings may be NaN.") @@ -298,6 +368,11 @@ def run_lambda_eval(args: argparse.Namespace, project_root: Path, lambda_value: completion_rate, agent_avg_wait_hours, baseline_avg_wait_hours, + agent_cost_1k, + baseline_cost_1k, + baseline_off_cost_1k, + agent_power, + baseline_off_power, ) print( f"[ok ] lambda={lambda_value}: " @@ -421,6 +496,12 @@ def write_summary_csv(path: Path, stats_by_lambda: list[LambdaRunStats]) -> None "effective_savings_std", "effective_savings_off_mean", "effective_savings_off_std", + "cost_per_1k_delta_pct_baseline_mean", + "cost_per_1k_delta_pct_baseline_std", + "cost_per_1k_delta_pct_baseline_off_mean", + "cost_per_1k_delta_pct_baseline_off_std", + "power_delta_pct_baseline_off_mean", + "power_delta_pct_baseline_off_std", "annual_total_savings_eur", "annual_total_savings_off_eur", ] @@ -447,6 +528,12 @@ def write_summary_csv(path: Path, stats_by_lambda: list[LambdaRunStats]) -> None "effective_savings_std": f"{s.effective_savings_std:.6f}", "effective_savings_off_mean": f"{s.effective_savings_off_mean:.6f}", "effective_savings_off_std": f"{s.effective_savings_off_std:.6f}", + "cost_per_1k_delta_pct_baseline_mean": f"{s.cost_per_1k_delta_pct_baseline_mean:.6f}", + "cost_per_1k_delta_pct_baseline_std": f"{s.cost_per_1k_delta_pct_baseline_std:.6f}", + "cost_per_1k_delta_pct_baseline_off_mean": f"{s.cost_per_1k_delta_pct_baseline_off_mean:.6f}", + "cost_per_1k_delta_pct_baseline_off_std": f"{s.cost_per_1k_delta_pct_baseline_off_std:.6f}", + "power_delta_pct_baseline_off_mean": f"{s.power_delta_pct_baseline_off_mean:.6f}", + "power_delta_pct_baseline_off_std": f"{s.power_delta_pct_baseline_off_std:.6f}", "annual_total_savings_eur": f"{s.annual_total_savings:.6f}", "annual_total_savings_off_eur": f"{s.annual_total_savings_off:.6f}", } @@ -471,6 +558,12 @@ def make_plot(path: Path, stats_by_lambda: list[LambdaRunStats], fit: bool = Fal eff_sav_std = np.array([s.effective_savings_std for s in ordered], dtype=float) eff_sav_off_mean = np.array([s.effective_savings_off_mean for s in ordered], dtype=float) eff_sav_off_std = np.array([s.effective_savings_off_std for s in ordered], dtype=float) + cost_per_1k_delta_base_mean = np.array([s.cost_per_1k_delta_pct_baseline_mean for s in ordered], dtype=float) + cost_per_1k_delta_base_std = np.array([s.cost_per_1k_delta_pct_baseline_std for s in ordered], dtype=float) + cost_per_1k_delta_base_off_mean = np.array([s.cost_per_1k_delta_pct_baseline_off_mean for s in ordered], dtype=float) + cost_per_1k_delta_base_off_std = np.array([s.cost_per_1k_delta_pct_baseline_off_std for s in ordered], dtype=float) + power_delta_base_off_mean = np.array([s.power_delta_pct_baseline_off_mean for s in ordered], dtype=float) + power_delta_base_off_std = np.array([s.power_delta_pct_baseline_off_std for s in ordered], dtype=float) lam_min = float(np.min(lambdas)) lam_max = float(np.max(lambdas)) @@ -510,9 +603,10 @@ def plot_colored_points( alpha=0.95, ) - fig, axes = plt.subplots(2, 3, figsize=(20, 12), constrained_layout=True) + fig, axes = plt.subplots(3, 3, figsize=(20, 17), constrained_layout=True) ax00, ax01, ax02 = axes[0] ax10, ax11, ax12 = axes[1] + ax20, ax21, ax22 = axes[2] # Panel 1: lambda vs occupancy. plot_colored_points(ax00, lambdas, occ_mean, yerr=occ_std) @@ -608,6 +702,54 @@ def plot_colored_points( if coeffs is not None: ax12.legend() + # Panel 7: occupancy vs cost_per_1k delta (%) vs baseline. + plot_colored_points(ax20, occ_mean, cost_per_1k_delta_base_mean, xerr=occ_std, yerr=cost_per_1k_delta_base_std) + coeffs, deg = None, None + if fit: + coeffs, deg = polyfit_curve(occ_mean, cost_per_1k_delta_base_mean, max_degree=3) + if coeffs is not None: + finite = np.isfinite(occ_mean) & np.isfinite(cost_per_1k_delta_base_mean) + x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) + ax20.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") + ax20.set_title("Occupancy vs Cost/1k Delta vs Baseline") + ax20.set_xlabel("Agent Occupancy (Nodes, %) / Episode") + ax20.set_ylabel("(Baseline - Agent) / Baseline [%]") + ax20.grid(alpha=0.3) + if coeffs is not None: + ax20.legend() + + # Panel 8: occupancy vs cost_per_1k delta (%) vs baseline_off. + plot_colored_points(ax21, occ_mean, cost_per_1k_delta_base_off_mean, xerr=occ_std, yerr=cost_per_1k_delta_base_off_std) + coeffs, deg = None, None + if fit: + coeffs, deg = polyfit_curve(occ_mean, cost_per_1k_delta_base_off_mean, max_degree=3) + if coeffs is not None: + finite = np.isfinite(occ_mean) & np.isfinite(cost_per_1k_delta_base_off_mean) + x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) + ax21.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") + ax21.set_title("Occupancy vs Cost/1k Delta vs Baseline_off") + ax21.set_xlabel("Agent Occupancy (Nodes, %) / Episode") + ax21.set_ylabel("(Baseline_off - Agent) / Baseline_off [%]") + ax21.grid(alpha=0.3) + if coeffs is not None: + ax21.legend() + + # Panel 9: occupancy vs power delta (%) vs baseline_off. + plot_colored_points(ax22, occ_mean, power_delta_base_off_mean, xerr=occ_std, yerr=power_delta_base_off_std) + coeffs, deg = None, None + if fit: + coeffs, deg = polyfit_curve(occ_mean, power_delta_base_off_mean, max_degree=3) + if coeffs is not None: + finite = np.isfinite(occ_mean) & np.isfinite(power_delta_base_off_mean) + x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) + ax22.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") + ax22.set_title("Occupancy vs Power Delta vs Baseline_off") + ax22.set_xlabel("Agent Occupancy (Nodes, %) / Episode") + ax22.set_ylabel("(Baseline_off - Agent) / Baseline_off [%]") + ax22.grid(alpha=0.3) + if coeffs is not None: + ax22.legend() + sm = plt.cm.ScalarMappable(norm=norm, cmap=cmap) sm.set_array([]) cbar = fig.colorbar(sm, ax=axes.ravel().tolist(), pad=0.02) From 9cf06b3c909a725fdc08660b8961304285350421 Mon Sep 17 00:00:00 2001 From: Enis Lorenz Date: Tue, 10 Mar 2026 15:17:07 +0100 Subject: [PATCH 4/7] Analysis: add arrival-scale sweep script and evaluation arrival-rate summary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - add `analyze_arrivalscale_occupancy.py` to sweep `--job-arrival-scale` over two modes: - `exact_replay_aggregate` (`--jobs-exact-replay --jobs-exact-replay-aggregate`) - `sampling` (`--jobs --job-arrival-scale`, with optional `--seed`) - keep scale ordering deterministic by always running `scale=1.0` first in both modes - generate per-mode trend plots plus CSV/JSON outputs (including selected vs executed scales) - in `train.py` evaluation summary, print job arrivals per hour as `mean ± std` --- analyze_arrivalscale_occupancy.py | 854 ++++++++++++++++++++++++++++++ train.py | 7 + 2 files changed, 861 insertions(+) create mode 100644 analyze_arrivalscale_occupancy.py diff --git a/analyze_arrivalscale_occupancy.py b/analyze_arrivalscale_occupancy.py new file mode 100644 index 0000000..2c0ebfd --- /dev/null +++ b/analyze_arrivalscale_occupancy.py @@ -0,0 +1,854 @@ +#!/usr/bin/env python3 +""" +Sweep job-arrival-scale values in jobs exact-replay mode and analyze: +1) job-arrival-scale -> agent occupancy (nodes) +2) occupancy -> savings +3) occupancy -> savings_off +4) job-arrival-scale -> completion rate +5) occupancy -> effective savings +6) occupancy -> effective savings_off +7) occupancy -> (baseline - agent) cost_per_1000_completed_jobs / baseline +8) occupancy -> (baseline_off - agent) cost_per_1000_completed_jobs / baseline_off +9) occupancy -> (baseline_off - agent) power / baseline_off + +For each scale, this script runs train.py in evaluation mode for one year +(12 months = 24 episodes), parses per-episode metrics from stdout, computes +mean/std, and fits optional polynomial trend lines. + +Runs both modes: +- exact replay (aggregated): --jobs-exact-replay --jobs-exact-replay-aggregate --job-arrival-scale +- sampling mode: --jobs --job-arrival-scale + +FAST DEBUG MODE: +python analyze_arrivalscale_occupancy.py \ + --jobs ./data/workload_statistics/jobs_2023.log \ + --eval-months 1 --scales 0.8,1.0,1.2 --no-plot-dashboard +""" + +from __future__ import annotations + +import argparse +import csv +import json +import re +import shlex +import subprocess +import sys +from dataclasses import asdict, dataclass, field +from datetime import datetime +from pathlib import Path + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np + + +EPISODE_RE = re.compile( + r"Episode\s+(?P\d+):.*?" + r"Savings=€(?P-?[\d,]+(?:\.\d+)?)\/€(?P-?[\d,]+(?:\.\d+)?),.*?" + r"Power=(?P-?[\d.]+)\/(?P-?[\d.]+)\/(?P-?[\d.]+)\s*MWh.*?" + r"CostPer1kCompleted=(?P-?[\d,]+(?:\.\d+)?|n/a)\/" + r"(?P-?[\d,]+(?:\.\d+)?|n/a)\/" + r"(?P-?[\d,]+(?:\.\d+)?|n/a)\s*€/1k.*?" + r"Jobs=[\d,]+\/[\d,]+\s+\((?P-?[\d.]+)%\),\s*" + r"AvgWait=(?P-?[\d.]+)h,.*?" + r"Agent Occupancy \(Nodes\)=\s*(?P-?[\d.]+)%", + re.MULTILINE, +) + +WAIT_SUMMARY_RE = re.compile( + r"=== JOB PROCESSING METRICS ===.*?" + r"Agent:.*?Average Wait Time:\s*(?P-?[\d.]+)\s*hours.*?" + r"Baseline:.*?Average Wait Time:\s*(?P-?[\d.]+)\s*hours", + re.DOTALL, +) + + +@dataclass +class ScaleRunStats: + replay_mode: str + job_arrival_scale: float + episodes: int + occupancy_mean: float + occupancy_std: float + savings_mean: float + savings_std: float + savings_off_mean: float + savings_off_std: float + completion_rate_mean: float + completion_rate_std: float + agent_avg_wait_hours: float + baseline_avg_wait_hours: float + wait_delta_hours: float + effective_savings_mean: float + effective_savings_std: float + effective_savings_off_mean: float + effective_savings_off_std: float + cost_per_1k_delta_pct_baseline_mean: float + cost_per_1k_delta_pct_baseline_std: float + cost_per_1k_delta_pct_baseline_off_mean: float + cost_per_1k_delta_pct_baseline_off_std: float + power_delta_pct_baseline_off_mean: float + power_delta_pct_baseline_off_std: float + annual_total_savings: float + annual_total_savings_off: float + command: list[str] + command_str: str + occupancy_samples: list[float] = field(default_factory=list) + savings_samples: list[float] = field(default_factory=list) + savings_off_samples: list[float] = field(default_factory=list) + completion_rate_samples: list[float] = field(default_factory=list) + effective_savings_samples: list[float] = field(default_factory=list) + effective_savings_off_samples: list[float] = field(default_factory=list) + cost_per_1k_delta_pct_baseline_samples: list[float] = field(default_factory=list) + cost_per_1k_delta_pct_baseline_off_samples: list[float] = field(default_factory=list) + power_delta_pct_baseline_off_samples: list[float] = field(default_factory=list) + + +def _to_float(raw: str) -> float: + return float(raw.replace(",", "")) + + +def _to_float_or_nan(raw: str | None) -> float: + if raw is None: + return float("nan") + val = raw.strip().lower() + if val in {"n/a", "nan"}: + return float("nan") + return _to_float(raw) + + +def _diff_cumulative(values: list[float]) -> np.ndarray: + arr = np.asarray(values, dtype=float) + if arr.size == 0: + return arr + return np.diff(np.concatenate(([0.0], arr))) + + +def parse_episode_metrics(stdout: str) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + occupancy = [] + cumulative_savings = [] + cumulative_savings_off = [] + completion_rate = [] + avg_wait = [] + agent_cost_1k = [] + baseline_cost_1k = [] + baseline_off_cost_1k = [] + agent_power = [] + baseline_off_power = [] + + for match in EPISODE_RE.finditer(stdout): + occupancy.append(_to_float(match.group("occupancy"))) + cumulative_savings.append(_to_float(match.group("savings"))) + cumulative_savings_off.append(_to_float(match.group("savings_off"))) + completion_rate.append(_to_float(match.group("completion_rate"))) + avg_wait.append(_to_float(match.group("avg_wait"))) + agent_cost_1k.append(_to_float_or_nan(match.group("agent_cost_1k"))) + baseline_cost_1k.append(_to_float_or_nan(match.group("baseline_cost_1k"))) + baseline_off_cost_1k.append(_to_float_or_nan(match.group("baseline_off_cost_1k"))) + agent_power.append(_to_float(match.group("agent_power"))) + baseline_off_power.append(_to_float(match.group("baseline_off_power"))) + + if not occupancy: + raise RuntimeError( + "Could not parse episode metrics from train.py output. " + "Expected lines like 'Episode X: ... Savings=€.../€..., Power=..., CostPer1kCompleted=..., Agent Occupancy (Nodes)=...%'." + ) + + episode_savings = _diff_cumulative(cumulative_savings) + episode_savings_off = _diff_cumulative(cumulative_savings_off) + return ( + np.asarray(occupancy, dtype=float), + episode_savings, + episode_savings_off, + np.asarray(completion_rate, dtype=float), + np.asarray(avg_wait, dtype=float), + np.asarray(agent_cost_1k, dtype=float), + np.asarray(baseline_cost_1k, dtype=float), + np.asarray(baseline_off_cost_1k, dtype=float), + np.asarray(agent_power, dtype=float), + np.asarray(baseline_off_power, dtype=float), + ) + + +def parse_wait_summary(stdout: str) -> tuple[float | None, float | None]: + match = WAIT_SUMMARY_RE.search(stdout) + if not match: + return None, None + return _to_float(match.group("agent_wait")), _to_float(match.group("baseline_wait")) + + +def safe_divide(numer: np.ndarray, denom: float) -> np.ndarray: + if abs(denom) < 1e-12: + return np.full_like(numer, np.nan, dtype=float) + return numer / denom + + +def safe_divide_arrays(numer: np.ndarray, denom: np.ndarray) -> np.ndarray: + numer_arr = np.asarray(numer, dtype=float) + denom_arr = np.asarray(denom, dtype=float) + out = np.full_like(numer_arr, np.nan, dtype=float) + finite = np.isfinite(numer_arr) & np.isfinite(denom_arr) + valid = finite & (np.abs(denom_arr) >= 1e-12) + out[valid] = numer_arr[valid] / denom_arr[valid] + return out + + +def finite_mean_std(values: np.ndarray) -> tuple[float, float]: + finite = np.isfinite(values) + if not np.any(finite): + return float("nan"), float("nan") + vals = values[finite] + return float(np.mean(vals)), float(np.std(vals)) + + +def make_run_stats( + replay_mode: str, + job_arrival_scale: float, + command: list[str], + occupancy: np.ndarray, + savings: np.ndarray, + savings_off: np.ndarray, + completion_rate: np.ndarray, + agent_avg_wait_hours: float, + baseline_avg_wait_hours: float, + agent_cost_1k: np.ndarray, + baseline_cost_1k: np.ndarray, + baseline_off_cost_1k: np.ndarray, + agent_power: np.ndarray, + baseline_off_power: np.ndarray, +) -> ScaleRunStats: + wait_delta_hours = agent_avg_wait_hours - baseline_avg_wait_hours + effective_savings = safe_divide(savings * (completion_rate / 100) ** 2, wait_delta_hours + 1) + effective_savings_off = safe_divide(savings_off * (completion_rate / 100) ** 2, wait_delta_hours + 1) + effective_savings_mean, effective_savings_std = finite_mean_std(effective_savings) + effective_savings_off_mean, effective_savings_off_std = finite_mean_std(effective_savings_off) + + cost_per_1k_delta_pct_baseline = safe_divide_arrays((baseline_cost_1k - agent_cost_1k) * 100.0, baseline_cost_1k) + cost_per_1k_delta_pct_baseline_off = safe_divide_arrays((baseline_off_cost_1k - agent_cost_1k) * 100.0, baseline_off_cost_1k) + power_delta_pct_baseline_off = safe_divide_arrays((baseline_off_power - agent_power) * 100.0, baseline_off_power) + + cost_per_1k_delta_pct_baseline_mean, cost_per_1k_delta_pct_baseline_std = finite_mean_std(cost_per_1k_delta_pct_baseline) + cost_per_1k_delta_pct_baseline_off_mean, cost_per_1k_delta_pct_baseline_off_std = finite_mean_std(cost_per_1k_delta_pct_baseline_off) + power_delta_pct_baseline_off_mean, power_delta_pct_baseline_off_std = finite_mean_std(power_delta_pct_baseline_off) + + return ScaleRunStats( + replay_mode=replay_mode, + job_arrival_scale=job_arrival_scale, + episodes=int(occupancy.size), + occupancy_mean=float(np.mean(occupancy)), + occupancy_std=float(np.std(occupancy)), + savings_mean=float(np.mean(savings)), + savings_std=float(np.std(savings)), + savings_off_mean=float(np.mean(savings_off)), + savings_off_std=float(np.std(savings_off)), + completion_rate_mean=float(np.mean(completion_rate)), + completion_rate_std=float(np.std(completion_rate)), + agent_avg_wait_hours=float(agent_avg_wait_hours), + baseline_avg_wait_hours=float(baseline_avg_wait_hours), + wait_delta_hours=float(wait_delta_hours), + effective_savings_mean=effective_savings_mean, + effective_savings_std=effective_savings_std, + effective_savings_off_mean=effective_savings_off_mean, + effective_savings_off_std=effective_savings_off_std, + cost_per_1k_delta_pct_baseline_mean=cost_per_1k_delta_pct_baseline_mean, + cost_per_1k_delta_pct_baseline_std=cost_per_1k_delta_pct_baseline_std, + cost_per_1k_delta_pct_baseline_off_mean=cost_per_1k_delta_pct_baseline_off_mean, + cost_per_1k_delta_pct_baseline_off_std=cost_per_1k_delta_pct_baseline_off_std, + power_delta_pct_baseline_off_mean=power_delta_pct_baseline_off_mean, + power_delta_pct_baseline_off_std=power_delta_pct_baseline_off_std, + annual_total_savings=float(np.sum(savings)), + annual_total_savings_off=float(np.sum(savings_off)), + command=command, + command_str=shlex.join(command), + occupancy_samples=occupancy.tolist(), + savings_samples=savings.tolist(), + savings_off_samples=savings_off.tolist(), + completion_rate_samples=completion_rate.tolist(), + effective_savings_samples=effective_savings.tolist(), + effective_savings_off_samples=effective_savings_off.tolist(), + cost_per_1k_delta_pct_baseline_samples=cost_per_1k_delta_pct_baseline.tolist(), + cost_per_1k_delta_pct_baseline_off_samples=cost_per_1k_delta_pct_baseline_off.tolist(), + power_delta_pct_baseline_off_samples=power_delta_pct_baseline_off.tolist(), + ) + + +def polyfit_curve(x: np.ndarray, y: np.ndarray, max_degree: int = 3) -> tuple[np.ndarray | None, int]: + finite = np.isfinite(x) & np.isfinite(y) + xf = x[finite] + yf = y[finite] + if xf.size < 2: + return None, 0 + degree = min(max_degree, xf.size - 1) + coeffs = np.polyfit(xf, yf, degree) + return coeffs, degree + + +def normalize_scale(value: float) -> float: + return float(f"{value:.6f}") + + +def parse_float_list(raw: str) -> list[float]: + return [float(part.strip()) for part in raw.split(",") if part.strip()] + + +def unique_scales_sorted(values: list[float]) -> list[float]: + return sorted({normalize_scale(float(v)) for v in values}) + + +def build_scale_grid(min_scale: float, max_scale: float, n: int) -> list[float]: + if n <= 1 or abs(max_scale - min_scale) < 1e-12: + return [normalize_scale(min_scale)] + vals = np.linspace(min_scale, max_scale, n) + return unique_scales_sorted([float(v) for v in vals]) + + +def select_scale_schedule(args: argparse.Namespace) -> list[float]: + if args.scales: + return unique_scales_sorted(parse_float_list(args.scales)) + return build_scale_grid(args.min_scale, args.max_scale, args.num_points) + + +def with_scale_one_first(scales: list[float]) -> list[float]: + one = normalize_scale(1.0) + return [one] + [s for s in scales if abs(s - one) > 1e-12] + + +def format_scale_for_filename(scale: float) -> str: + return f"{scale:.6f}".rstrip("0").rstrip(".").replace(".", "p") + + +def build_train_command(args: argparse.Namespace, job_arrival_scale: float, replay_mode: str) -> list[str]: + cmd = [ + sys.executable, + "./train.py", + "--prices", + args.prices, + "--session", + args.session, + "--efficiency-weight", + str(args.efficiency_weight), + "--price-weight", + str(args.price_weight), + "--idle-weight", + str(args.idle_weight), + "--job-age-weight", + str(args.job_age_weight), + "--drop-weight", + str(args.drop_weight), + "--evaluate-savings", + "--eval-months", + str(args.eval_months), + "--model", + str(args.model), + "--jobs", + args.jobs, + ] + if replay_mode == "exact_replay_aggregate": + cmd.extend([ + "--jobs-exact-replay", + "--jobs-exact-replay-aggregate", + "--job-arrival-scale", + f"{job_arrival_scale:.6f}", + ]) + elif replay_mode == "sampling": + cmd.extend(["--job-arrival-scale", f"{job_arrival_scale:.6f}"]) + if args.seed is not None: + cmd.extend(["--seed", str(args.seed)]) + else: + raise ValueError(f"Unsupported replay_mode: {replay_mode}") + if args.plot_dashboard: + cmd.append("--plot-dashboard") + if args.dashboard_hours is not None: + cmd.extend(["--dashboard-hours", str(args.dashboard_hours)]) + return cmd + + +def os_tail(text: str, lines: int = 20) -> str: + parts = text.rstrip().splitlines() + if not parts: + return "" + return "\n".join(parts[-lines:]) + + +def run_scale_eval( + args: argparse.Namespace, + project_root: Path, + job_arrival_scale: float, + replay_mode: str, +) -> tuple[ScaleRunStats, str]: + command = build_train_command(args, job_arrival_scale, replay_mode) + print(f"[run] mode={replay_mode}, scale={job_arrival_scale:.6f}: {shlex.join(command)}") + completed = subprocess.run( + command, + cwd=str(project_root), + capture_output=True, + text=True, + check=False, + ) + + combined_output = (completed.stdout or "") + ("\n" + completed.stderr if completed.stderr else "") + if args.echo_train_output: + print(combined_output) + if completed.returncode != 0: + raise RuntimeError( + f"train.py failed for mode={replay_mode}, scale={job_arrival_scale:.6f} with code {completed.returncode}.\n" + f"Last output lines:\n{os_tail(combined_output, lines=40)}" + ) + + ( + occupancy, + savings, + savings_off, + completion_rate, + avg_wait, + agent_cost_1k, + baseline_cost_1k, + baseline_off_cost_1k, + agent_power, + baseline_off_power, + ) = parse_episode_metrics(combined_output) + + agent_wait_summary, baseline_wait_summary = parse_wait_summary(combined_output) + if agent_wait_summary is None or baseline_wait_summary is None: + print(f"[warn] mode={replay_mode}, scale={job_arrival_scale:.6f}: could not parse run-level wait summary; effective savings may be NaN.") + agent_avg_wait_hours = float(np.mean(avg_wait)) + baseline_avg_wait_hours = float(np.mean(avg_wait)) + else: + agent_avg_wait_hours = float(agent_wait_summary) + baseline_avg_wait_hours = float(baseline_wait_summary) + + stats = make_run_stats( + replay_mode, + job_arrival_scale, + command, + occupancy, + savings, + savings_off, + completion_rate, + agent_avg_wait_hours, + baseline_avg_wait_hours, + agent_cost_1k, + baseline_cost_1k, + baseline_off_cost_1k, + agent_power, + baseline_off_power, + ) + print( + f"[ok ] mode={replay_mode}, scale={job_arrival_scale:.6f}: " + f"occupancy={stats.occupancy_mean:.2f}%±{stats.occupancy_std:.2f}, " + f"completion={stats.completion_rate_mean:.2f}%±{stats.completion_rate_std:.2f}, " + f"savings={stats.savings_mean:.0f}±{stats.savings_std:.0f}, " + f"savings_off={stats.savings_off_mean:.0f}±{stats.savings_off_std:.0f}, " + f"wait_delta={stats.wait_delta_hours:.3f}h" + ) + return stats, combined_output + + +def write_summary_csv(path: Path, stats: list[ScaleRunStats]) -> None: + fieldnames = [ + "replay_mode", + "job_arrival_scale", + "episodes", + "occupancy_mean_pct", + "occupancy_std_pct", + "completion_rate_mean_pct", + "completion_rate_std_pct", + "agent_avg_wait_hours", + "baseline_avg_wait_hours", + "wait_delta_hours", + "savings_mean_eur", + "savings_std_eur", + "savings_off_mean_eur", + "savings_off_std_eur", + "effective_savings_mean", + "effective_savings_std", + "effective_savings_off_mean", + "effective_savings_off_std", + "cost_per_1k_delta_pct_baseline_mean", + "cost_per_1k_delta_pct_baseline_std", + "cost_per_1k_delta_pct_baseline_off_mean", + "cost_per_1k_delta_pct_baseline_off_std", + "power_delta_pct_baseline_off_mean", + "power_delta_pct_baseline_off_std", + "annual_total_savings_eur", + "annual_total_savings_off_eur", + ] + with path.open("w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + for s in sorted(stats, key=lambda x: (x.replay_mode, x.job_arrival_scale)): + writer.writerow( + { + "replay_mode": s.replay_mode, + "job_arrival_scale": f"{s.job_arrival_scale:.6f}", + "episodes": s.episodes, + "occupancy_mean_pct": f"{s.occupancy_mean:.6f}", + "occupancy_std_pct": f"{s.occupancy_std:.6f}", + "completion_rate_mean_pct": f"{s.completion_rate_mean:.6f}", + "completion_rate_std_pct": f"{s.completion_rate_std:.6f}", + "agent_avg_wait_hours": f"{s.agent_avg_wait_hours:.6f}", + "baseline_avg_wait_hours": f"{s.baseline_avg_wait_hours:.6f}", + "wait_delta_hours": f"{s.wait_delta_hours:.6f}", + "savings_mean_eur": f"{s.savings_mean:.6f}", + "savings_std_eur": f"{s.savings_std:.6f}", + "savings_off_mean_eur": f"{s.savings_off_mean:.6f}", + "savings_off_std_eur": f"{s.savings_off_std:.6f}", + "effective_savings_mean": f"{s.effective_savings_mean:.6f}", + "effective_savings_std": f"{s.effective_savings_std:.6f}", + "effective_savings_off_mean": f"{s.effective_savings_off_mean:.6f}", + "effective_savings_off_std": f"{s.effective_savings_off_std:.6f}", + "cost_per_1k_delta_pct_baseline_mean": f"{s.cost_per_1k_delta_pct_baseline_mean:.6f}", + "cost_per_1k_delta_pct_baseline_std": f"{s.cost_per_1k_delta_pct_baseline_std:.6f}", + "cost_per_1k_delta_pct_baseline_off_mean": f"{s.cost_per_1k_delta_pct_baseline_off_mean:.6f}", + "cost_per_1k_delta_pct_baseline_off_std": f"{s.cost_per_1k_delta_pct_baseline_off_std:.6f}", + "power_delta_pct_baseline_off_mean": f"{s.power_delta_pct_baseline_off_mean:.6f}", + "power_delta_pct_baseline_off_std": f"{s.power_delta_pct_baseline_off_std:.6f}", + "annual_total_savings_eur": f"{s.annual_total_savings:.6f}", + "annual_total_savings_off_eur": f"{s.annual_total_savings_off:.6f}", + } + ) + + +def make_plot(path: Path, stats: list[ScaleRunStats], replay_mode: str, fit: bool = False) -> None: + ordered = sorted(stats, key=lambda x: x.job_arrival_scale) + if not ordered: + return + + scales = np.array([s.job_arrival_scale for s in ordered], dtype=float) + occ_mean = np.array([s.occupancy_mean for s in ordered], dtype=float) + occ_std = np.array([s.occupancy_std for s in ordered], dtype=float) + sav_mean = np.array([s.savings_mean for s in ordered], dtype=float) + sav_std = np.array([s.savings_std for s in ordered], dtype=float) + sav_off_mean = np.array([s.savings_off_mean for s in ordered], dtype=float) + sav_off_std = np.array([s.savings_off_std for s in ordered], dtype=float) + completion_mean = np.array([s.completion_rate_mean for s in ordered], dtype=float) + completion_std = np.array([s.completion_rate_std for s in ordered], dtype=float) + eff_sav_mean = np.array([s.effective_savings_mean for s in ordered], dtype=float) + eff_sav_std = np.array([s.effective_savings_std for s in ordered], dtype=float) + eff_sav_off_mean = np.array([s.effective_savings_off_mean for s in ordered], dtype=float) + eff_sav_off_std = np.array([s.effective_savings_off_std for s in ordered], dtype=float) + cost_per_1k_delta_base_mean = np.array([s.cost_per_1k_delta_pct_baseline_mean for s in ordered], dtype=float) + cost_per_1k_delta_base_std = np.array([s.cost_per_1k_delta_pct_baseline_std for s in ordered], dtype=float) + cost_per_1k_delta_base_off_mean = np.array([s.cost_per_1k_delta_pct_baseline_off_mean for s in ordered], dtype=float) + cost_per_1k_delta_base_off_std = np.array([s.cost_per_1k_delta_pct_baseline_off_std for s in ordered], dtype=float) + power_delta_base_off_mean = np.array([s.power_delta_pct_baseline_off_mean for s in ordered], dtype=float) + power_delta_base_off_std = np.array([s.power_delta_pct_baseline_off_std for s in ordered], dtype=float) + + scale_min = float(np.min(scales)) + scale_max = float(np.max(scales)) + if scale_max <= scale_min: + scale_max = scale_min + 1.0 + norm = matplotlib.colors.Normalize(vmin=scale_min, vmax=scale_max) + cmap = plt.get_cmap("turbo") + point_colors = cmap(norm(scales)) + + def _error_at(arr: np.ndarray | None, idx: int) -> float | None: + if arr is None: + return None + v = float(arr[idx]) + return v if np.isfinite(v) else None + + def plot_colored_points( + ax: plt.Axes, + x: np.ndarray, + y: np.ndarray, + xerr: np.ndarray | None = None, + yerr: np.ndarray | None = None, + ) -> None: + for i, (xi, yi, c) in enumerate(zip(x, y, point_colors)): + if not (np.isfinite(xi) and np.isfinite(yi)): + continue + ax.errorbar( + float(xi), + float(yi), + xerr=_error_at(xerr, i), + yerr=_error_at(yerr, i), + fmt="o", + markersize=6, + capsize=3, + color=c, + ecolor=c, + elinewidth=1.2, + alpha=0.95, + ) + + fig, axes = plt.subplots(3, 3, figsize=(20, 17), constrained_layout=True) + ax00, ax01, ax02 = axes[0] + ax10, ax11, ax12 = axes[1] + ax20, ax21, ax22 = axes[2] + + # Panel 1: arrival scale vs occupancy. + plot_colored_points(ax00, scales, occ_mean, yerr=occ_std) + coeffs, deg = None, None + if fit: + coeffs, deg = polyfit_curve(scales, occ_mean, max_degree=3) + if coeffs is not None: + x_fit = np.linspace(float(np.min(scales)), float(np.max(scales)), 250) + ax00.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") + ax00.set_title("Arrival Scale vs Occupancy/Episode") + ax00.set_xlabel("Job arrival scale") + ax00.set_ylabel("Agent Occupancy (Nodes, %) / Episode") + ax00.grid(alpha=0.3) + if coeffs is not None: + ax00.legend() + + # Panel 2: occupancy vs savings. + plot_colored_points(ax01, occ_mean, sav_mean, xerr=occ_std, yerr=sav_std) + coeffs, deg = None, None + if fit: + coeffs, deg = polyfit_curve(occ_mean, sav_mean, max_degree=3) + if coeffs is not None: + finite = np.isfinite(occ_mean) & np.isfinite(sav_mean) + x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) + ax01.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") + ax01.set_title("Occupancy/Episode vs Savings/Episode") + ax01.set_xlabel("Agent Occupancy (Nodes, %) / Episode") + ax01.set_ylabel("Savings vs Baseline (EUR / Episode)") + ax01.grid(alpha=0.3) + if coeffs is not None: + ax01.legend() + + # Panel 3: occupancy vs savings_off. + plot_colored_points(ax02, occ_mean, sav_off_mean, xerr=occ_std, yerr=sav_off_std) + coeffs, deg = None, None + if fit: + coeffs, deg = polyfit_curve(occ_mean, sav_off_mean, max_degree=3) + if coeffs is not None: + finite = np.isfinite(occ_mean) & np.isfinite(sav_off_mean) + x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) + ax02.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") + ax02.set_title("Occupancy vs Savings_off/Episode") + ax02.set_xlabel("Agent Occupancy (Nodes, %) / Episode") + ax02.set_ylabel("Savings vs Baseline_off (EUR / Episode)") + ax02.grid(alpha=0.3) + if coeffs is not None: + ax02.legend() + + # Panel 4: arrival scale vs completion rate. + plot_colored_points(ax10, scales, completion_mean, yerr=completion_std) + coeffs, deg = None, None + if fit: + coeffs, deg = polyfit_curve(scales, completion_mean, max_degree=3) + if coeffs is not None: + x_fit = np.linspace(float(np.min(scales)), float(np.max(scales)), 250) + ax10.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") + ax10.set_title("Arrival Scale vs Agent Completion Rate") + ax10.set_xlabel("Job arrival scale") + ax10.set_ylabel("Completion Rate (%)") + ax10.grid(alpha=0.3) + if coeffs is not None: + ax10.legend() + + # Panel 5: occupancy vs effective savings. + plot_colored_points(ax11, occ_mean, eff_sav_mean, xerr=occ_std, yerr=eff_sav_std) + coeffs, deg = None, None + if fit: + coeffs, deg = polyfit_curve(occ_mean, eff_sav_mean, max_degree=3) + if coeffs is not None: + finite = np.isfinite(occ_mean) & np.isfinite(eff_sav_mean) + x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) + ax11.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") + ax11.set_title("Occupancy vs Effective Savings") + ax11.set_xlabel("Agent Occupancy (Nodes, %) / Episode") + ax11.set_ylabel("effective_savings") + ax11.grid(alpha=0.3) + if coeffs is not None: + ax11.legend() + + # Panel 6: occupancy vs effective savings_off. + plot_colored_points(ax12, occ_mean, eff_sav_off_mean, xerr=occ_std, yerr=eff_sav_off_std) + coeffs, deg = None, None + if fit: + coeffs, deg = polyfit_curve(occ_mean, eff_sav_off_mean, max_degree=3) + if coeffs is not None: + finite = np.isfinite(occ_mean) & np.isfinite(eff_sav_off_mean) + x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) + ax12.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") + ax12.set_title("Occupancy vs Effective Savings_off") + ax12.set_xlabel("Agent Occupancy (Nodes, %) / Episode") + ax12.set_ylabel("effective_savings_off") + ax12.grid(alpha=0.3) + if coeffs is not None: + ax12.legend() + + # Panel 7: occupancy vs cost_per_1k delta (%) vs baseline. + plot_colored_points(ax20, occ_mean, cost_per_1k_delta_base_mean, xerr=occ_std, yerr=cost_per_1k_delta_base_std) + coeffs, deg = None, None + if fit: + coeffs, deg = polyfit_curve(occ_mean, cost_per_1k_delta_base_mean, max_degree=3) + if coeffs is not None: + finite = np.isfinite(occ_mean) & np.isfinite(cost_per_1k_delta_base_mean) + x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) + ax20.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") + ax20.set_title("Occupancy vs Cost/1k Delta vs Baseline") + ax20.set_xlabel("Agent Occupancy (Nodes, %) / Episode") + ax20.set_ylabel("(Baseline - Agent) / Baseline [%]") + ax20.grid(alpha=0.3) + if coeffs is not None: + ax20.legend() + + # Panel 8: occupancy vs cost_per_1k delta (%) vs baseline_off. + plot_colored_points(ax21, occ_mean, cost_per_1k_delta_base_off_mean, xerr=occ_std, yerr=cost_per_1k_delta_base_off_std) + coeffs, deg = None, None + if fit: + coeffs, deg = polyfit_curve(occ_mean, cost_per_1k_delta_base_off_mean, max_degree=3) + if coeffs is not None: + finite = np.isfinite(occ_mean) & np.isfinite(cost_per_1k_delta_base_off_mean) + x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) + ax21.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") + ax21.set_title("Occupancy vs Cost/1k Delta vs Baseline_off") + ax21.set_xlabel("Agent Occupancy (Nodes, %) / Episode") + ax21.set_ylabel("(Baseline_off - Agent) / Baseline_off [%]") + ax21.grid(alpha=0.3) + if coeffs is not None: + ax21.legend() + + # Panel 9: occupancy vs power delta (%) vs baseline_off. + plot_colored_points(ax22, occ_mean, power_delta_base_off_mean, xerr=occ_std, yerr=power_delta_base_off_std) + coeffs, deg = None, None + if fit: + coeffs, deg = polyfit_curve(occ_mean, power_delta_base_off_mean, max_degree=3) + if coeffs is not None: + finite = np.isfinite(occ_mean) & np.isfinite(power_delta_base_off_mean) + x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) + ax22.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") + ax22.set_title("Occupancy vs Power Delta vs Baseline_off") + ax22.set_xlabel("Agent Occupancy (Nodes, %) / Episode") + ax22.set_ylabel("(Baseline_off - Agent) / Baseline_off [%]") + ax22.grid(alpha=0.3) + if coeffs is not None: + ax22.legend() + + sm = plt.cm.ScalarMappable(norm=norm, cmap=cmap) + sm.set_array([]) + cbar = fig.colorbar(sm, ax=axes.ravel().tolist(), pad=0.02) + cbar.set_label("Job arrival scale (point color)") + + fig.suptitle(f"Job-Arrival-Scale Sweep ({replay_mode})", fontsize=14) + fig.savefig(path, dpi=220) + plt.close(fig) + + +def build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Compare --jobs exact replay aggregate vs --jobs sampling (--job-arrival-scale) and fit occupancy trend lines." + ) + + # Core train.py params. + parser.add_argument("--prices", default="./data/prices_2023.csv") + parser.add_argument("--jobs", required=True, help="Path forwarded to train.py --jobs") + parser.add_argument("--session", default="") + parser.add_argument("--efficiency-weight", type=float, default=0.6) + parser.add_argument("--price-weight", type=float, default=0.1) + parser.add_argument("--idle-weight", type=float, default=0.1) + parser.add_argument("--job-age-weight", type=float, default=0.2) + parser.add_argument("--drop-weight", type=float, default=0.0) + parser.add_argument("--eval-months", type=int, default=12) + parser.add_argument("--model", type=int, default=1000000) + parser.add_argument("--seed", type=int, default=None, help="Forwarded to train.py (sampling mode only).") + + parser.add_argument("--plot-dashboard", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("--dashboard-hours", type=int, default=None) + + # Scale sweep controls. + parser.add_argument( + "--scales", + type=str, + default="", + help="Explicit comma-separated --job-arrival-scale values. If set, min/max/num-points are ignored.", + ) + parser.add_argument("--min-scale", type=float, default=0.5) + parser.add_argument("--max-scale", type=float, default=1.5) + parser.add_argument("--num-points", type=int, default=7) + + parser.add_argument("--out-dir", type=str, default="") + parser.add_argument("--save-logs", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("--echo-train-output", action=argparse.BooleanOptionalAction, default=False) + parser.add_argument("--fit", action="store_true", default=False, help="Enable polynomial fitting of datasets") + return parser + + +def main() -> None: + parser = build_arg_parser() + args = parser.parse_args() + + if args.num_points < 2 and not args.scales: + parser.error("--num-points must be >= 2 when --scales is not provided") + if args.min_scale < 0.0: + parser.error("--min-scale must be >= 0") + if args.max_scale < args.min_scale: + parser.error("--max-scale must be >= --min-scale") + + project_root = Path(__file__).resolve().parent + train_py = project_root / "train.py" + if not train_py.exists(): + raise FileNotFoundError(f"Could not find train.py at: {train_py}") + + jobs_path = Path(args.jobs).expanduser() + if not jobs_path.exists(): + raise FileNotFoundError(f"Could not find jobs file: {jobs_path}") + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + if args.out_dir: + out_dir = Path(args.out_dir).expanduser().resolve() + else: + out_dir = project_root / "analysis" / f"arrivalscale_occupancy_sweep_{timestamp}" + out_dir.mkdir(parents=True, exist_ok=True) + + logs_dir = out_dir / "logs" + if args.save_logs: + logs_dir.mkdir(parents=True, exist_ok=True) + + selected_scales = select_scale_schedule(args) + # Always warm-start each mode at scale=1.0, then continue with requested sweep scales. + scales = with_scale_one_first(selected_scales) + mode_names = ["exact_replay_aggregate", "sampling"] + + all_stats: list[ScaleRunStats] = [] + + for replay_mode in mode_names: + for scale in scales: + stats, raw_output = run_scale_eval(args, project_root, scale, replay_mode) + all_stats.append(stats) + if args.save_logs: + scale_part = format_scale_for_filename(scale) + log_path = logs_dir / f"{replay_mode}_scale_{scale_part}.log" + log_path.write_text(raw_output) + + csv_path = out_dir / "summary.csv" + json_path = out_dir / "summary.json" + write_summary_csv(csv_path, all_stats) + with json_path.open("w") as f: + json.dump( + { + "created_at": datetime.now().isoformat(), + "selected_scales": selected_scales, + "scales": scales, + "modes": mode_names, + "args": vars(args), + "results": [asdict(s) for s in all_stats], + }, + f, + indent=2, + ) + + plot_paths: list[Path] = [] + for replay_mode in mode_names: + mode_stats = [s for s in all_stats if s.replay_mode == replay_mode] + plot_path = out_dir / f"trendlines_{replay_mode}.png" + make_plot(plot_path, mode_stats, replay_mode, fit=args.fit) + plot_paths.append(plot_path) + + print("\nSweep complete.") + print(f" Scales: {scales}") + print(f" Modes: {mode_names}") + print(f" CSV: {csv_path}") + print(f" JSON: {json_path}") + for p in plot_paths: + print(f" Plot: {p}") + + +if __name__ == "__main__": + main() diff --git a/train.py b/train.py index 7ba3e5c..78411a8 100644 --- a/train.py +++ b/train.py @@ -322,6 +322,12 @@ def main(): total_baseline_off_cost_per_1000_completed = safe_ratio(total_baseline_off_cost * 1000.0, total_baseline_completed) total_dropped_jobs_per_saved_euro = safe_ratio(total_jobs_dropped, total_savings_vs_baseline) if total_savings_vs_baseline > 0 else None total_dropped_jobs_per_saved_euro_off = safe_ratio(total_jobs_dropped, total_savings_vs_baseline_off) if total_savings_vs_baseline_off > 0 else None + arrivals_per_hour_by_episode = [float(ep['jobs_submitted']) / float(EPISODE_HOURS) for ep in env.metrics.episode_costs] + mean_arrivals_per_hour = (sum(arrivals_per_hour_by_episode) / len(arrivals_per_hour_by_episode)) if arrivals_per_hour_by_episode else 0.0 + arrivals_variance = ( + sum((x - mean_arrivals_per_hour) ** 2 for x in arrivals_per_hour_by_episode) / len(arrivals_per_hour_by_episode) + ) if arrivals_per_hour_by_episode else 0.0 + std_arrivals_per_hour = arrivals_variance ** 0.5 print(f"\n=== JOB PROCESSING METRICS ===") print(f"\nAgent:") @@ -329,6 +335,7 @@ def main(): print(f" Average Wait Time: {avg_wait_time:.1f} hours") print(f" Average Max Queue Size: {avg_max_queue:.0f}") print(f" Total Cost: €{total_agent_cost:,.0f}") + print(f" Job Arrivals/Hour (mean ± std): {mean_arrivals_per_hour:.2f} ± {std_arrivals_per_hour:.2f}") print(f"\nBaseline:") print(f" Jobs Completed: {total_baseline_completed:,} / {total_baseline_submitted:,} ({total_baseline_completion_rate:.1f}%)") From 6cfefa0c2efc3d5f8c4e107618f4935862456cd1 Mon Sep 17 00:00:00 2001 From: Enis Lorenz Date: Wed, 11 Mar 2026 10:46:23 +0100 Subject: [PATCH 5/7] Analysis: add baseline/arrival/drop metrics and per-panel plot exports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - add baseline occupancy and agent dropped parsing to both occupancy sweep analyzers - parse run-level arrivals/hour mean±std and dropped totals (agent + baseline) - extend run stats and CSV outputs with baseline/baseline_off occupancy, arrivals stats, and dropped delta (agent - baseline) - expand figures from 3x3 to 4x3 with new panels for baseline vs baseline_off occupancy, jobs/hour, and dropped delta - export each panel as its own image in a per-run `plots_individual` subdirectory, each including the colorbar - print `Total Dropped Jobs (Baseline)` in `train.py` eval summaries so dropped-delta metrics are available to analyzers Analysis Correct: Use per-episode metrics for evaluation episode summaries - source episode summary values from recorded episode metrics - fix mixed cumulative/per-episode savings and cost-per-1k values - relabel cumulative queue metric as TimelineMaxQueue - add focused coverage for episode summary formatting --- analyze_arrivalscale_occupancy.py | 468 +++++++++++++++++++---------- analyze_lambda_occupancy.py | 477 ++++++++++++++++++++---------- src/arrival_scale.py | 11 + src/evaluation_summary.py | 67 +++++ src/sampler_hourly.py | 2 + src/workload_generator.py | 2 + train.py | 59 ++-- train_iter.py | 7 +- 8 files changed, 755 insertions(+), 338 deletions(-) create mode 100644 src/arrival_scale.py create mode 100644 src/evaluation_summary.py diff --git a/analyze_arrivalscale_occupancy.py b/analyze_arrivalscale_occupancy.py index 2c0ebfd..bb997d2 100644 --- a/analyze_arrivalscale_occupancy.py +++ b/analyze_arrivalscale_occupancy.py @@ -10,6 +10,9 @@ 7) occupancy -> (baseline - agent) cost_per_1000_completed_jobs / baseline 8) occupancy -> (baseline_off - agent) cost_per_1000_completed_jobs / baseline_off 9) occupancy -> (baseline_off - agent) power / baseline_off +10) arrival-scale -> baseline and baseline_off occupancies +11) arrival-scale -> mean jobs/hour (with std) +12) arrival-scale -> dropped-jobs delta (agent - baseline) For each scale, this script runs train.py in evaluation mode for one year (12 months = 24 episodes), parses per-episode metrics from stdout, computes @@ -37,6 +40,7 @@ from dataclasses import asdict, dataclass, field from datetime import datetime from pathlib import Path +from typing import Callable import matplotlib matplotlib.use("Agg") @@ -53,7 +57,9 @@ r"(?P-?[\d,]+(?:\.\d+)?|n/a)\s*€/1k.*?" r"Jobs=[\d,]+\/[\d,]+\s+\((?P-?[\d.]+)%\),\s*" r"AvgWait=(?P-?[\d.]+)h,.*?" - r"Agent Occupancy \(Nodes\)=\s*(?P-?[\d.]+)%", + r"Dropped=(?P-?[\d,]+),.*?" + r"Agent Occupancy \(Nodes\)=\s*(?P-?[\d.]+)%,\s*" + r"Baseline Occupancy \(Nodes\)=\s*(?P-?[\d.]+)%", re.MULTILINE, ) @@ -64,6 +70,18 @@ re.DOTALL, ) +ARRIVALS_SUMMARY_RE = re.compile( + r"Job Arrivals/Hour \(mean\s*(?:±|\+/-)\s*std\):\s*(?P-?[\d.]+)\s*(?:±|\+/-)\s*(?P-?[\d.]+)" +) + +DROPPED_AGENT_SUMMARY_RE = re.compile( + r"Total Dropped Jobs \(Agent\):\s*(?P[\d,]+)" +) + +DROPPED_BASELINE_SUMMARY_RE = re.compile( + r"Total Dropped Jobs \(Baseline\):\s*(?P[\d,]+)" +) + @dataclass class ScaleRunStats: @@ -72,6 +90,15 @@ class ScaleRunStats: episodes: int occupancy_mean: float occupancy_std: float + baseline_occupancy_mean: float + baseline_occupancy_std: float + baseline_off_occupancy_mean: float + baseline_off_occupancy_std: float + arrivals_per_hour_mean: float + arrivals_per_hour_std: float + dropped_jobs_agent_total: float + dropped_jobs_baseline_total: float + dropped_jobs_delta_total: float savings_mean: float savings_std: float savings_off_mean: float @@ -96,6 +123,9 @@ class ScaleRunStats: command: list[str] command_str: str occupancy_samples: list[float] = field(default_factory=list) + baseline_occupancy_samples: list[float] = field(default_factory=list) + baseline_off_occupancy_samples: list[float] = field(default_factory=list) + dropped_jobs_agent_samples: list[float] = field(default_factory=list) savings_samples: list[float] = field(default_factory=list) savings_off_samples: list[float] = field(default_factory=list) completion_rate_samples: list[float] = field(default_factory=list) @@ -126,8 +156,25 @@ def _diff_cumulative(values: list[float]) -> np.ndarray: return np.diff(np.concatenate(([0.0], arr))) -def parse_episode_metrics(stdout: str) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: +def parse_episode_metrics( + stdout: str, +) -> tuple[ + np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, +]: occupancy = [] + baseline_occupancy = [] + agent_dropped = [] cumulative_savings = [] cumulative_savings_off = [] completion_rate = [] @@ -140,6 +187,8 @@ def parse_episode_metrics(stdout: str) -> tuple[np.ndarray, np.ndarray, np.ndarr for match in EPISODE_RE.finditer(stdout): occupancy.append(_to_float(match.group("occupancy"))) + baseline_occupancy.append(_to_float(match.group("baseline_occupancy"))) + agent_dropped.append(_to_float(match.group("agent_dropped"))) cumulative_savings.append(_to_float(match.group("savings"))) cumulative_savings_off.append(_to_float(match.group("savings_off"))) completion_rate.append(_to_float(match.group("completion_rate"))) @@ -160,6 +209,8 @@ def parse_episode_metrics(stdout: str) -> tuple[np.ndarray, np.ndarray, np.ndarr episode_savings_off = _diff_cumulative(cumulative_savings_off) return ( np.asarray(occupancy, dtype=float), + np.asarray(baseline_occupancy, dtype=float), + np.asarray(agent_dropped, dtype=float), episode_savings, episode_savings_off, np.asarray(completion_rate, dtype=float), @@ -179,6 +230,21 @@ def parse_wait_summary(stdout: str) -> tuple[float | None, float | None]: return _to_float(match.group("agent_wait")), _to_float(match.group("baseline_wait")) +def parse_arrivals_summary(stdout: str) -> tuple[float | None, float | None]: + match = ARRIVALS_SUMMARY_RE.search(stdout) + if not match: + return None, None + return _to_float(match.group("mean")), _to_float(match.group("std")) + + +def parse_dropped_totals_summary(stdout: str) -> tuple[float | None, float | None]: + agent_match = DROPPED_AGENT_SUMMARY_RE.search(stdout) + baseline_match = DROPPED_BASELINE_SUMMARY_RE.search(stdout) + agent_total = _to_float(agent_match.group("agent")) if agent_match else None + baseline_total = _to_float(baseline_match.group("baseline")) if baseline_match else None + return agent_total, baseline_total + + def safe_divide(numer: np.ndarray, denom: float) -> np.ndarray: if abs(denom) < 1e-12: return np.full_like(numer, np.nan, dtype=float) @@ -208,6 +274,8 @@ def make_run_stats( job_arrival_scale: float, command: list[str], occupancy: np.ndarray, + baseline_occupancy: np.ndarray, + agent_dropped: np.ndarray, savings: np.ndarray, savings_off: np.ndarray, completion_rate: np.ndarray, @@ -218,6 +286,10 @@ def make_run_stats( baseline_off_cost_1k: np.ndarray, agent_power: np.ndarray, baseline_off_power: np.ndarray, + arrivals_per_hour_mean: float, + arrivals_per_hour_std: float, + dropped_jobs_agent_total: float, + dropped_jobs_baseline_total: float, ) -> ScaleRunStats: wait_delta_hours = agent_avg_wait_hours - baseline_avg_wait_hours effective_savings = safe_divide(savings * (completion_rate / 100) ** 2, wait_delta_hours + 1) @@ -232,6 +304,8 @@ def make_run_stats( cost_per_1k_delta_pct_baseline_mean, cost_per_1k_delta_pct_baseline_std = finite_mean_std(cost_per_1k_delta_pct_baseline) cost_per_1k_delta_pct_baseline_off_mean, cost_per_1k_delta_pct_baseline_off_std = finite_mean_std(cost_per_1k_delta_pct_baseline_off) power_delta_pct_baseline_off_mean, power_delta_pct_baseline_off_std = finite_mean_std(power_delta_pct_baseline_off) + baseline_off_occupancy = baseline_occupancy.copy() + dropped_jobs_delta_total = dropped_jobs_agent_total - dropped_jobs_baseline_total return ScaleRunStats( replay_mode=replay_mode, @@ -239,6 +313,15 @@ def make_run_stats( episodes=int(occupancy.size), occupancy_mean=float(np.mean(occupancy)), occupancy_std=float(np.std(occupancy)), + baseline_occupancy_mean=float(np.mean(baseline_occupancy)), + baseline_occupancy_std=float(np.std(baseline_occupancy)), + baseline_off_occupancy_mean=float(np.mean(baseline_off_occupancy)), + baseline_off_occupancy_std=float(np.std(baseline_off_occupancy)), + arrivals_per_hour_mean=float(arrivals_per_hour_mean), + arrivals_per_hour_std=float(arrivals_per_hour_std), + dropped_jobs_agent_total=float(dropped_jobs_agent_total), + dropped_jobs_baseline_total=float(dropped_jobs_baseline_total), + dropped_jobs_delta_total=float(dropped_jobs_delta_total), savings_mean=float(np.mean(savings)), savings_std=float(np.std(savings)), savings_off_mean=float(np.mean(savings_off)), @@ -263,6 +346,9 @@ def make_run_stats( command=command, command_str=shlex.join(command), occupancy_samples=occupancy.tolist(), + baseline_occupancy_samples=baseline_occupancy.tolist(), + baseline_off_occupancy_samples=baseline_off_occupancy.tolist(), + dropped_jobs_agent_samples=agent_dropped.tolist(), savings_samples=savings.tolist(), savings_off_samples=savings_off.tolist(), completion_rate_samples=completion_rate.tolist(), @@ -399,6 +485,8 @@ def run_scale_eval( ( occupancy, + baseline_occupancy, + agent_dropped, savings, savings_off, completion_rate, @@ -418,12 +506,26 @@ def run_scale_eval( else: agent_avg_wait_hours = float(agent_wait_summary) baseline_avg_wait_hours = float(baseline_wait_summary) + arrivals_per_hour_mean, arrivals_per_hour_std = parse_arrivals_summary(combined_output) + if arrivals_per_hour_mean is None or arrivals_per_hour_std is None: + print(f"[warn] mode={replay_mode}, scale={job_arrival_scale:.6f}: could not parse run-level arrivals/hour summary; values set to NaN.") + arrivals_per_hour_mean = float("nan") + arrivals_per_hour_std = float("nan") + dropped_jobs_agent_total, dropped_jobs_baseline_total = parse_dropped_totals_summary(combined_output) + if dropped_jobs_agent_total is None: + dropped_jobs_agent_total = float(np.sum(agent_dropped)) + print(f"[warn] mode={replay_mode}, scale={job_arrival_scale:.6f}: could not parse run-level agent dropped total; using sum of episode Dropped= values.") + if dropped_jobs_baseline_total is None: + dropped_jobs_baseline_total = 0.0 + print(f"[warn] mode={replay_mode}, scale={job_arrival_scale:.6f}: could not parse run-level baseline dropped total; defaulting to 0.") stats = make_run_stats( replay_mode, job_arrival_scale, command, occupancy, + baseline_occupancy, + agent_dropped, savings, savings_off, completion_rate, @@ -434,10 +536,17 @@ def run_scale_eval( baseline_off_cost_1k, agent_power, baseline_off_power, + arrivals_per_hour_mean, + arrivals_per_hour_std, + dropped_jobs_agent_total, + dropped_jobs_baseline_total, ) print( f"[ok ] mode={replay_mode}, scale={job_arrival_scale:.6f}: " f"occupancy={stats.occupancy_mean:.2f}%±{stats.occupancy_std:.2f}, " + f"baseline_occ={stats.baseline_occupancy_mean:.2f}%±{stats.baseline_occupancy_std:.2f}, " + f"arrivals/h={stats.arrivals_per_hour_mean:.2f}±{stats.arrivals_per_hour_std:.2f}, " + f"dropped_delta={stats.dropped_jobs_delta_total:.0f}, " f"completion={stats.completion_rate_mean:.2f}%±{stats.completion_rate_std:.2f}, " f"savings={stats.savings_mean:.0f}±{stats.savings_std:.0f}, " f"savings_off={stats.savings_off_mean:.0f}±{stats.savings_off_std:.0f}, " @@ -453,6 +562,15 @@ def write_summary_csv(path: Path, stats: list[ScaleRunStats]) -> None: "episodes", "occupancy_mean_pct", "occupancy_std_pct", + "baseline_occupancy_mean_pct", + "baseline_occupancy_std_pct", + "baseline_off_occupancy_mean_pct", + "baseline_off_occupancy_std_pct", + "arrivals_per_hour_mean", + "arrivals_per_hour_std", + "dropped_jobs_agent_total", + "dropped_jobs_baseline_total", + "dropped_jobs_delta_total", "completion_rate_mean_pct", "completion_rate_std_pct", "agent_avg_wait_hours", @@ -486,6 +604,15 @@ def write_summary_csv(path: Path, stats: list[ScaleRunStats]) -> None: "episodes": s.episodes, "occupancy_mean_pct": f"{s.occupancy_mean:.6f}", "occupancy_std_pct": f"{s.occupancy_std:.6f}", + "baseline_occupancy_mean_pct": f"{s.baseline_occupancy_mean:.6f}", + "baseline_occupancy_std_pct": f"{s.baseline_occupancy_std:.6f}", + "baseline_off_occupancy_mean_pct": f"{s.baseline_off_occupancy_mean:.6f}", + "baseline_off_occupancy_std_pct": f"{s.baseline_off_occupancy_std:.6f}", + "arrivals_per_hour_mean": f"{s.arrivals_per_hour_mean:.6f}", + "arrivals_per_hour_std": f"{s.arrivals_per_hour_std:.6f}", + "dropped_jobs_agent_total": f"{s.dropped_jobs_agent_total:.6f}", + "dropped_jobs_baseline_total": f"{s.dropped_jobs_baseline_total:.6f}", + "dropped_jobs_delta_total": f"{s.dropped_jobs_delta_total:.6f}", "completion_rate_mean_pct": f"{s.completion_rate_mean:.6f}", "completion_rate_std_pct": f"{s.completion_rate_std:.6f}", "agent_avg_wait_hours": f"{s.agent_avg_wait_hours:.6f}", @@ -511,7 +638,13 @@ def write_summary_csv(path: Path, stats: list[ScaleRunStats]) -> None: ) -def make_plot(path: Path, stats: list[ScaleRunStats], replay_mode: str, fit: bool = False) -> None: +def make_plot( + path: Path, + stats: list[ScaleRunStats], + replay_mode: str, + fit: bool = False, + individual_dir: Path | None = None, +) -> None: ordered = sorted(stats, key=lambda x: x.job_arrival_scale) if not ordered: return @@ -519,6 +652,13 @@ def make_plot(path: Path, stats: list[ScaleRunStats], replay_mode: str, fit: boo scales = np.array([s.job_arrival_scale for s in ordered], dtype=float) occ_mean = np.array([s.occupancy_mean for s in ordered], dtype=float) occ_std = np.array([s.occupancy_std for s in ordered], dtype=float) + baseline_occ_mean = np.array([s.baseline_occupancy_mean for s in ordered], dtype=float) + baseline_occ_std = np.array([s.baseline_occupancy_std for s in ordered], dtype=float) + baseline_off_occ_mean = np.array([s.baseline_off_occupancy_mean for s in ordered], dtype=float) + baseline_off_occ_std = np.array([s.baseline_off_occupancy_std for s in ordered], dtype=float) + arrivals_per_hour_mean = np.array([s.arrivals_per_hour_mean for s in ordered], dtype=float) + arrivals_per_hour_std = np.array([s.arrivals_per_hour_std for s in ordered], dtype=float) + dropped_jobs_delta_total = np.array([s.dropped_jobs_delta_total for s in ordered], dtype=float) sav_mean = np.array([s.savings_mean for s in ordered], dtype=float) sav_std = np.array([s.savings_std for s in ordered], dtype=float) sav_off_mean = np.array([s.savings_off_mean for s in ordered], dtype=float) @@ -543,6 +683,7 @@ def make_plot(path: Path, stats: list[ScaleRunStats], replay_mode: str, fit: boo norm = matplotlib.colors.Normalize(vmin=scale_min, vmax=scale_max) cmap = plt.get_cmap("turbo") point_colors = cmap(norm(scales)) + colorbar_label = "Job arrival scale (point color)" def _error_at(arr: np.ndarray | None, idx: int) -> float | None: if arr is None: @@ -550,6 +691,20 @@ def _error_at(arr: np.ndarray | None, idx: int) -> float | None: v = float(arr[idx]) return v if np.isfinite(v) else None + def _maybe_plot_fit(ax: plt.Axes, x: np.ndarray, y: np.ndarray) -> None: + coeffs = None + deg = 0 + if fit: + coeffs, deg = polyfit_curve(x, y, max_degree=3) + if coeffs is None: + return + finite = np.isfinite(x) & np.isfinite(y) + if not np.any(finite): + return + x_fit = np.linspace(float(np.min(x[finite])), float(np.max(x[finite])), 250) + ax.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") + ax.legend() + def plot_colored_points( ax: plt.Axes, x: np.ndarray, @@ -574,162 +729,168 @@ def plot_colored_points( alpha=0.95, ) - fig, axes = plt.subplots(3, 3, figsize=(20, 17), constrained_layout=True) - ax00, ax01, ax02 = axes[0] - ax10, ax11, ax12 = axes[1] - ax20, ax21, ax22 = axes[2] - - # Panel 1: arrival scale vs occupancy. - plot_colored_points(ax00, scales, occ_mean, yerr=occ_std) - coeffs, deg = None, None - if fit: - coeffs, deg = polyfit_curve(scales, occ_mean, max_degree=3) - if coeffs is not None: - x_fit = np.linspace(float(np.min(scales)), float(np.max(scales)), 250) - ax00.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") - ax00.set_title("Arrival Scale vs Occupancy/Episode") - ax00.set_xlabel("Job arrival scale") - ax00.set_ylabel("Agent Occupancy (Nodes, %) / Episode") - ax00.grid(alpha=0.3) - if coeffs is not None: - ax00.legend() - - # Panel 2: occupancy vs savings. - plot_colored_points(ax01, occ_mean, sav_mean, xerr=occ_std, yerr=sav_std) - coeffs, deg = None, None - if fit: - coeffs, deg = polyfit_curve(occ_mean, sav_mean, max_degree=3) - if coeffs is not None: - finite = np.isfinite(occ_mean) & np.isfinite(sav_mean) - x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) - ax01.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") - ax01.set_title("Occupancy/Episode vs Savings/Episode") - ax01.set_xlabel("Agent Occupancy (Nodes, %) / Episode") - ax01.set_ylabel("Savings vs Baseline (EUR / Episode)") - ax01.grid(alpha=0.3) - if coeffs is not None: - ax01.legend() - - # Panel 3: occupancy vs savings_off. - plot_colored_points(ax02, occ_mean, sav_off_mean, xerr=occ_std, yerr=sav_off_std) - coeffs, deg = None, None - if fit: - coeffs, deg = polyfit_curve(occ_mean, sav_off_mean, max_degree=3) - if coeffs is not None: - finite = np.isfinite(occ_mean) & np.isfinite(sav_off_mean) - x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) - ax02.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") - ax02.set_title("Occupancy vs Savings_off/Episode") - ax02.set_xlabel("Agent Occupancy (Nodes, %) / Episode") - ax02.set_ylabel("Savings vs Baseline_off (EUR / Episode)") - ax02.grid(alpha=0.3) - if coeffs is not None: - ax02.legend() - - # Panel 4: arrival scale vs completion rate. - plot_colored_points(ax10, scales, completion_mean, yerr=completion_std) - coeffs, deg = None, None - if fit: - coeffs, deg = polyfit_curve(scales, completion_mean, max_degree=3) - if coeffs is not None: - x_fit = np.linspace(float(np.min(scales)), float(np.max(scales)), 250) - ax10.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") - ax10.set_title("Arrival Scale vs Agent Completion Rate") - ax10.set_xlabel("Job arrival scale") - ax10.set_ylabel("Completion Rate (%)") - ax10.grid(alpha=0.3) - if coeffs is not None: - ax10.legend() - - # Panel 5: occupancy vs effective savings. - plot_colored_points(ax11, occ_mean, eff_sav_mean, xerr=occ_std, yerr=eff_sav_std) - coeffs, deg = None, None - if fit: - coeffs, deg = polyfit_curve(occ_mean, eff_sav_mean, max_degree=3) - if coeffs is not None: - finite = np.isfinite(occ_mean) & np.isfinite(eff_sav_mean) - x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) - ax11.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") - ax11.set_title("Occupancy vs Effective Savings") - ax11.set_xlabel("Agent Occupancy (Nodes, %) / Episode") - ax11.set_ylabel("effective_savings") - ax11.grid(alpha=0.3) - if coeffs is not None: - ax11.legend() - - # Panel 6: occupancy vs effective savings_off. - plot_colored_points(ax12, occ_mean, eff_sav_off_mean, xerr=occ_std, yerr=eff_sav_off_std) - coeffs, deg = None, None - if fit: - coeffs, deg = polyfit_curve(occ_mean, eff_sav_off_mean, max_degree=3) - if coeffs is not None: - finite = np.isfinite(occ_mean) & np.isfinite(eff_sav_off_mean) - x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) - ax12.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") - ax12.set_title("Occupancy vs Effective Savings_off") - ax12.set_xlabel("Agent Occupancy (Nodes, %) / Episode") - ax12.set_ylabel("effective_savings_off") - ax12.grid(alpha=0.3) - if coeffs is not None: - ax12.legend() - - # Panel 7: occupancy vs cost_per_1k delta (%) vs baseline. - plot_colored_points(ax20, occ_mean, cost_per_1k_delta_base_mean, xerr=occ_std, yerr=cost_per_1k_delta_base_std) - coeffs, deg = None, None - if fit: - coeffs, deg = polyfit_curve(occ_mean, cost_per_1k_delta_base_mean, max_degree=3) - if coeffs is not None: - finite = np.isfinite(occ_mean) & np.isfinite(cost_per_1k_delta_base_mean) - x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) - ax20.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") - ax20.set_title("Occupancy vs Cost/1k Delta vs Baseline") - ax20.set_xlabel("Agent Occupancy (Nodes, %) / Episode") - ax20.set_ylabel("(Baseline - Agent) / Baseline [%]") - ax20.grid(alpha=0.3) - if coeffs is not None: - ax20.legend() - - # Panel 8: occupancy vs cost_per_1k delta (%) vs baseline_off. - plot_colored_points(ax21, occ_mean, cost_per_1k_delta_base_off_mean, xerr=occ_std, yerr=cost_per_1k_delta_base_off_std) - coeffs, deg = None, None - if fit: - coeffs, deg = polyfit_curve(occ_mean, cost_per_1k_delta_base_off_mean, max_degree=3) - if coeffs is not None: - finite = np.isfinite(occ_mean) & np.isfinite(cost_per_1k_delta_base_off_mean) - x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) - ax21.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") - ax21.set_title("Occupancy vs Cost/1k Delta vs Baseline_off") - ax21.set_xlabel("Agent Occupancy (Nodes, %) / Episode") - ax21.set_ylabel("(Baseline_off - Agent) / Baseline_off [%]") - ax21.grid(alpha=0.3) - if coeffs is not None: - ax21.legend() - - # Panel 9: occupancy vs power delta (%) vs baseline_off. - plot_colored_points(ax22, occ_mean, power_delta_base_off_mean, xerr=occ_std, yerr=power_delta_base_off_std) - coeffs, deg = None, None - if fit: - coeffs, deg = polyfit_curve(occ_mean, power_delta_base_off_mean, max_degree=3) - if coeffs is not None: - finite = np.isfinite(occ_mean) & np.isfinite(power_delta_base_off_mean) - x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) - ax22.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") - ax22.set_title("Occupancy vs Power Delta vs Baseline_off") - ax22.set_xlabel("Agent Occupancy (Nodes, %) / Episode") - ax22.set_ylabel("(Baseline_off - Agent) / Baseline_off [%]") - ax22.grid(alpha=0.3) - if coeffs is not None: - ax22.legend() + def draw_baseline_occupancy_pair(ax: plt.Axes) -> None: + for i, (xv, c) in enumerate(zip(scales, point_colors)): + y_base = float(baseline_occ_mean[i]) + y_base_off = float(baseline_off_occ_mean[i]) + if np.isfinite(xv) and np.isfinite(y_base): + ax.errorbar( + xv, + y_base, + yerr=_error_at(baseline_occ_std, i), + fmt="o", + markersize=6, + capsize=3, + color=c, + ecolor=c, + elinewidth=1.2, + alpha=0.95, + ) + if np.isfinite(xv) and np.isfinite(y_base_off): + ax.errorbar( + xv, + y_base_off, + yerr=_error_at(baseline_off_occ_std, i), + fmt="^", + markersize=6, + capsize=3, + color=c, + ecolor=c, + elinewidth=1.2, + alpha=0.95, + ) + ax.scatter([], [], marker="o", color="black", label="Baseline") + ax.scatter([], [], marker="^", color="black", label="Baseline_off") + ax.legend() + + panel_specs: list[tuple[str, Callable[[plt.Axes], None]]] = [] + + def _panel( + slug: str, + title: str, + xlabel: str, + ylabel: str, + draw_body: Callable[[plt.Axes], None], + ) -> None: + def _draw(ax: plt.Axes) -> None: + draw_body(ax) + ax.set_title(title) + ax.set_xlabel(xlabel) + ax.set_ylabel(ylabel) + ax.grid(alpha=0.3) + panel_specs.append((slug, _draw)) + + _panel( + "01_scale_vs_agent_occupancy", + "Arrival Scale vs Occupancy/Episode", + "Job arrival scale", + "Agent Occupancy (Nodes, %) / Episode", + lambda ax: (plot_colored_points(ax, scales, occ_mean, yerr=occ_std), _maybe_plot_fit(ax, scales, occ_mean)), + ) + _panel( + "02_occupancy_vs_savings", + "Occupancy/Episode vs Savings/Episode", + "Agent Occupancy (Nodes, %) / Episode", + "Savings vs Baseline (EUR / Episode)", + lambda ax: (plot_colored_points(ax, occ_mean, sav_mean, xerr=occ_std, yerr=sav_std), _maybe_plot_fit(ax, occ_mean, sav_mean)), + ) + _panel( + "03_occupancy_vs_savings_off", + "Occupancy vs Savings_off/Episode", + "Agent Occupancy (Nodes, %) / Episode", + "Savings vs Baseline_off (EUR / Episode)", + lambda ax: (plot_colored_points(ax, occ_mean, sav_off_mean, xerr=occ_std, yerr=sav_off_std), _maybe_plot_fit(ax, occ_mean, sav_off_mean)), + ) + _panel( + "04_scale_vs_completion_rate", + "Arrival Scale vs Agent Completion Rate", + "Job arrival scale", + "Completion Rate (%)", + lambda ax: (plot_colored_points(ax, scales, completion_mean, yerr=completion_std), _maybe_plot_fit(ax, scales, completion_mean)), + ) + _panel( + "05_occupancy_vs_effective_savings", + "Occupancy vs Effective Savings", + "Agent Occupancy (Nodes, %) / Episode", + "effective_savings", + lambda ax: (plot_colored_points(ax, occ_mean, eff_sav_mean, xerr=occ_std, yerr=eff_sav_std), _maybe_plot_fit(ax, occ_mean, eff_sav_mean)), + ) + _panel( + "06_occupancy_vs_effective_savings_off", + "Occupancy vs Effective Savings_off", + "Agent Occupancy (Nodes, %) / Episode", + "effective_savings_off", + lambda ax: (plot_colored_points(ax, occ_mean, eff_sav_off_mean, xerr=occ_std, yerr=eff_sav_off_std), _maybe_plot_fit(ax, occ_mean, eff_sav_off_mean)), + ) + _panel( + "07_occupancy_vs_cost_per_1k_delta_baseline", + "Occupancy vs Cost/1k Delta vs Baseline", + "Agent Occupancy (Nodes, %) / Episode", + "(Baseline - Agent) / Baseline [%]", + lambda ax: (plot_colored_points(ax, occ_mean, cost_per_1k_delta_base_mean, xerr=occ_std, yerr=cost_per_1k_delta_base_std), _maybe_plot_fit(ax, occ_mean, cost_per_1k_delta_base_mean)), + ) + _panel( + "08_occupancy_vs_cost_per_1k_delta_baseline_off", + "Occupancy vs Cost/1k Delta vs Baseline_off", + "Agent Occupancy (Nodes, %) / Episode", + "(Baseline_off - Agent) / Baseline_off [%]", + lambda ax: (plot_colored_points(ax, occ_mean, cost_per_1k_delta_base_off_mean, xerr=occ_std, yerr=cost_per_1k_delta_base_off_std), _maybe_plot_fit(ax, occ_mean, cost_per_1k_delta_base_off_mean)), + ) + _panel( + "09_occupancy_vs_power_delta_baseline_off", + "Occupancy vs Power Delta vs Baseline_off", + "Agent Occupancy (Nodes, %) / Episode", + "(Baseline_off - Agent) / Baseline_off [%]", + lambda ax: (plot_colored_points(ax, occ_mean, power_delta_base_off_mean, xerr=occ_std, yerr=power_delta_base_off_std), _maybe_plot_fit(ax, occ_mean, power_delta_base_off_mean)), + ) + _panel( + "10_scale_vs_baseline_occupancies", + "Arrival Scale vs Baseline Occupancies", + "Job arrival scale", + "Baseline Occupancy (Nodes, %) / Episode", + draw_baseline_occupancy_pair, + ) + _panel( + "11_scale_vs_jobs_per_hour", + "Arrival Scale vs Job Arrivals/Hour", + "Job arrival scale", + "Job Arrivals/Hour (mean ± std)", + lambda ax: (plot_colored_points(ax, scales, arrivals_per_hour_mean, yerr=arrivals_per_hour_std), _maybe_plot_fit(ax, scales, arrivals_per_hour_mean)), + ) + _panel( + "12_scale_vs_dropped_jobs_delta", + "Arrival Scale vs Dropped Jobs Delta", + "Job arrival scale", + "Dropped Jobs Delta (Agent - Baseline)", + lambda ax: (plot_colored_points(ax, scales, dropped_jobs_delta_total), _maybe_plot_fit(ax, scales, dropped_jobs_delta_total)), + ) + + fig, axes = plt.subplots(4, 3, figsize=(22, 22), constrained_layout=True) + for ax, (_, draw_fn) in zip(axes.ravel(), panel_specs): + draw_fn(ax) sm = plt.cm.ScalarMappable(norm=norm, cmap=cmap) sm.set_array([]) cbar = fig.colorbar(sm, ax=axes.ravel().tolist(), pad=0.02) - cbar.set_label("Job arrival scale (point color)") + cbar.set_label(colorbar_label) fig.suptitle(f"Job-Arrival-Scale Sweep ({replay_mode})", fontsize=14) fig.savefig(path, dpi=220) plt.close(fig) + if individual_dir is not None: + individual_dir.mkdir(parents=True, exist_ok=True) + for slug, draw_fn in panel_specs: + panel_path = individual_dir / f"{slug}.png" + fig_i, ax_i = plt.subplots(1, 1, figsize=(8, 6), constrained_layout=True) + draw_fn(ax_i) + sm_i = plt.cm.ScalarMappable(norm=norm, cmap=cmap) + sm_i.set_array([]) + cbar_i = fig_i.colorbar(sm_i, ax=ax_i, pad=0.02) + cbar_i.set_label(colorbar_label) + fig_i.savefig(panel_path, dpi=220) + plt.close(fig_i) + def build_arg_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( @@ -835,11 +996,14 @@ def main() -> None: ) plot_paths: list[Path] = [] + individual_plot_dirs: list[Path] = [] for replay_mode in mode_names: mode_stats = [s for s in all_stats if s.replay_mode == replay_mode] plot_path = out_dir / f"trendlines_{replay_mode}.png" - make_plot(plot_path, mode_stats, replay_mode, fit=args.fit) + individual_dir = out_dir / "plots_individual" / replay_mode + make_plot(plot_path, mode_stats, replay_mode, fit=args.fit, individual_dir=individual_dir) plot_paths.append(plot_path) + individual_plot_dirs.append(individual_dir) print("\nSweep complete.") print(f" Scales: {scales}") @@ -848,6 +1012,8 @@ def main() -> None: print(f" JSON: {json_path}") for p in plot_paths: print(f" Plot: {p}") + for p in individual_plot_dirs: + print(f" Individual Plots: {p}") if __name__ == "__main__": diff --git a/analyze_lambda_occupancy.py b/analyze_lambda_occupancy.py index 5136736..fe970b1 100644 --- a/analyze_lambda_occupancy.py +++ b/analyze_lambda_occupancy.py @@ -10,6 +10,9 @@ 7) occupancy -> (baseline - agent) cost_per_1000_completed_jobs / baseline 8) occupancy -> (baseline_off - agent) cost_per_1000_completed_jobs / baseline_off 9) occupancy -> (baseline_off - agent) power / baseline_off +10) lambda -> baseline and baseline_off occupancies +11) lambda -> mean jobs/hour (with std) +12) lambda -> dropped-jobs delta (agent - baseline) For each lambda, this script runs train.py in evaluation mode for one year (12 months = 24 episodes), parses per-episode metrics from stdout, computes @@ -48,7 +51,9 @@ r"(?P-?[\d,]+(?:\.\d+)?|n/a)\s*€/1k.*?" r"Jobs=[\d,]+\/[\d,]+\s+\((?P-?[\d.]+)%\),\s*" r"AvgWait=(?P-?[\d.]+)h,.*?" - r"Agent Occupancy \(Nodes\)=\s*(?P-?[\d.]+)%", + r"Dropped=(?P-?[\d,]+),.*?" + r"Agent Occupancy \(Nodes\)=\s*(?P-?[\d.]+)%,\s*" + r"Baseline Occupancy \(Nodes\)=\s*(?P-?[\d.]+)%", re.MULTILINE, ) @@ -59,6 +64,18 @@ re.DOTALL, ) +ARRIVALS_SUMMARY_RE = re.compile( + r"Job Arrivals/Hour \(mean\s*(?:±|\+/-)\s*std\):\s*(?P-?[\d.]+)\s*(?:±|\+/-)\s*(?P-?[\d.]+)" +) + +DROPPED_AGENT_SUMMARY_RE = re.compile( + r"Total Dropped Jobs \(Agent\):\s*(?P[\d,]+)" +) + +DROPPED_BASELINE_SUMMARY_RE = re.compile( + r"Total Dropped Jobs \(Baseline\):\s*(?P[\d,]+)" +) + @dataclass class LambdaRunStats: @@ -66,6 +83,15 @@ class LambdaRunStats: episodes: int occupancy_mean: float occupancy_std: float + baseline_occupancy_mean: float + baseline_occupancy_std: float + baseline_off_occupancy_mean: float + baseline_off_occupancy_std: float + arrivals_per_hour_mean: float + arrivals_per_hour_std: float + dropped_jobs_agent_total: float + dropped_jobs_baseline_total: float + dropped_jobs_delta_total: float savings_mean: float savings_std: float savings_off_mean: float @@ -90,6 +116,9 @@ class LambdaRunStats: command: list[str] command_str: str occupancy_samples: list[float] = field(default_factory=list) + baseline_occupancy_samples: list[float] = field(default_factory=list) + baseline_off_occupancy_samples: list[float] = field(default_factory=list) + dropped_jobs_agent_samples: list[float] = field(default_factory=list) savings_samples: list[float] = field(default_factory=list) savings_off_samples: list[float] = field(default_factory=list) completion_rate_samples: list[float] = field(default_factory=list) @@ -120,8 +149,25 @@ def _diff_cumulative(values: list[float]) -> np.ndarray: return np.diff(np.concatenate(([0.0], arr))) -def parse_episode_metrics(stdout: str) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: +def parse_episode_metrics( + stdout: str, +) -> tuple[ + np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, +]: occupancy = [] + baseline_occupancy = [] + agent_dropped = [] cumulative_savings = [] cumulative_savings_off = [] completion_rate = [] @@ -134,6 +180,8 @@ def parse_episode_metrics(stdout: str) -> tuple[np.ndarray, np.ndarray, np.ndarr for match in EPISODE_RE.finditer(stdout): occupancy.append(_to_float(match.group("occupancy"))) + baseline_occupancy.append(_to_float(match.group("baseline_occupancy"))) + agent_dropped.append(_to_float(match.group("agent_dropped"))) cumulative_savings.append(_to_float(match.group("savings"))) cumulative_savings_off.append(_to_float(match.group("savings_off"))) completion_rate.append(_to_float(match.group("completion_rate"))) @@ -154,6 +202,8 @@ def parse_episode_metrics(stdout: str) -> tuple[np.ndarray, np.ndarray, np.ndarr episode_savings_off = _diff_cumulative(cumulative_savings_off) return ( np.asarray(occupancy, dtype=float), + np.asarray(baseline_occupancy, dtype=float), + np.asarray(agent_dropped, dtype=float), episode_savings, episode_savings_off, np.asarray(completion_rate, dtype=float), @@ -173,6 +223,21 @@ def parse_wait_summary(stdout: str) -> tuple[float | None, float | None]: return _to_float(match.group("agent_wait")), _to_float(match.group("baseline_wait")) +def parse_arrivals_summary(stdout: str) -> tuple[float | None, float | None]: + match = ARRIVALS_SUMMARY_RE.search(stdout) + if not match: + return None, None + return _to_float(match.group("mean")), _to_float(match.group("std")) + + +def parse_dropped_totals_summary(stdout: str) -> tuple[float | None, float | None]: + agent_match = DROPPED_AGENT_SUMMARY_RE.search(stdout) + baseline_match = DROPPED_BASELINE_SUMMARY_RE.search(stdout) + agent_total = _to_float(agent_match.group("agent")) if agent_match else None + baseline_total = _to_float(baseline_match.group("baseline")) if baseline_match else None + return agent_total, baseline_total + + def safe_divide(numer: np.ndarray, denom: float) -> np.ndarray: if abs(denom) < 1e-12: return np.full_like(numer, np.nan, dtype=float) @@ -201,6 +266,8 @@ def make_run_stats( lambda_value: int, command: list[str], occupancy: np.ndarray, + baseline_occupancy: np.ndarray, + agent_dropped: np.ndarray, savings: np.ndarray, savings_off: np.ndarray, completion_rate: np.ndarray, @@ -211,6 +278,10 @@ def make_run_stats( baseline_off_cost_1k: np.ndarray, agent_power: np.ndarray, baseline_off_power: np.ndarray, + arrivals_per_hour_mean: float, + arrivals_per_hour_std: float, + dropped_jobs_agent_total: float, + dropped_jobs_baseline_total: float, ) -> LambdaRunStats: wait_delta_hours = agent_avg_wait_hours - baseline_avg_wait_hours effective_savings = safe_divide(savings * (completion_rate/100)**2, wait_delta_hours+1) @@ -223,11 +294,22 @@ def make_run_stats( cost_per_1k_delta_pct_baseline_mean, cost_per_1k_delta_pct_baseline_std = finite_mean_std(cost_per_1k_delta_pct_baseline) cost_per_1k_delta_pct_baseline_off_mean, cost_per_1k_delta_pct_baseline_off_std = finite_mean_std(cost_per_1k_delta_pct_baseline_off) power_delta_pct_baseline_off_mean, power_delta_pct_baseline_off_std = finite_mean_std(power_delta_pct_baseline_off) + baseline_off_occupancy = baseline_occupancy.copy() + dropped_jobs_delta_total = dropped_jobs_agent_total - dropped_jobs_baseline_total return LambdaRunStats( lambda_value=lambda_value, episodes=int(occupancy.size), occupancy_mean=float(np.mean(occupancy)), occupancy_std=float(np.std(occupancy)), + baseline_occupancy_mean=float(np.mean(baseline_occupancy)), + baseline_occupancy_std=float(np.std(baseline_occupancy)), + baseline_off_occupancy_mean=float(np.mean(baseline_off_occupancy)), + baseline_off_occupancy_std=float(np.std(baseline_off_occupancy)), + arrivals_per_hour_mean=float(arrivals_per_hour_mean), + arrivals_per_hour_std=float(arrivals_per_hour_std), + dropped_jobs_agent_total=float(dropped_jobs_agent_total), + dropped_jobs_baseline_total=float(dropped_jobs_baseline_total), + dropped_jobs_delta_total=float(dropped_jobs_delta_total), savings_mean=float(np.mean(savings)), savings_std=float(np.std(savings)), savings_off_mean=float(np.mean(savings_off)), @@ -252,6 +334,9 @@ def make_run_stats( command=command, command_str=shlex.join(command), occupancy_samples=occupancy.tolist(), + baseline_occupancy_samples=baseline_occupancy.tolist(), + baseline_off_occupancy_samples=baseline_off_occupancy.tolist(), + dropped_jobs_agent_samples=agent_dropped.tolist(), savings_samples=savings.tolist(), savings_off_samples=savings_off.tolist(), completion_rate_samples=completion_rate.tolist(), @@ -350,7 +435,20 @@ def run_lambda_eval(args: argparse.Namespace, project_root: Path, lambda_value: f"Last output lines:\n{os_tail(combined_output, lines=40)}" ) - occupancy, savings, savings_off, completion_rate, avg_wait, agent_cost_1k, baseline_cost_1k, baseline_off_cost_1k, agent_power, baseline_off_power = parse_episode_metrics(combined_output) + ( + occupancy, + baseline_occupancy, + agent_dropped, + savings, + savings_off, + completion_rate, + avg_wait, + agent_cost_1k, + baseline_cost_1k, + baseline_off_cost_1k, + agent_power, + baseline_off_power, + ) = parse_episode_metrics(combined_output) agent_wait_summary, baseline_wait_summary = parse_wait_summary(combined_output) if agent_wait_summary is None or baseline_wait_summary is None: print(f"[warn] lambda={lambda_value}: could not parse run-level wait summary; effective savings may be NaN.") @@ -359,10 +457,24 @@ def run_lambda_eval(args: argparse.Namespace, project_root: Path, lambda_value: else: agent_avg_wait_hours = float(agent_wait_summary) baseline_avg_wait_hours = float(baseline_wait_summary) + arrivals_per_hour_mean, arrivals_per_hour_std = parse_arrivals_summary(combined_output) + if arrivals_per_hour_mean is None or arrivals_per_hour_std is None: + print(f"[warn] lambda={lambda_value}: could not parse run-level arrivals/hour summary; values set to NaN.") + arrivals_per_hour_mean = float("nan") + arrivals_per_hour_std = float("nan") + dropped_jobs_agent_total, dropped_jobs_baseline_total = parse_dropped_totals_summary(combined_output) + if dropped_jobs_agent_total is None: + dropped_jobs_agent_total = float(np.sum(agent_dropped)) + print(f"[warn] lambda={lambda_value}: could not parse run-level agent dropped total; using sum of episode Dropped= values.") + if dropped_jobs_baseline_total is None: + dropped_jobs_baseline_total = 0.0 + print(f"[warn] lambda={lambda_value}: could not parse run-level baseline dropped total; defaulting to 0.") stats = make_run_stats( lambda_value, command, occupancy, + baseline_occupancy, + agent_dropped, savings, savings_off, completion_rate, @@ -373,10 +485,17 @@ def run_lambda_eval(args: argparse.Namespace, project_root: Path, lambda_value: baseline_off_cost_1k, agent_power, baseline_off_power, + arrivals_per_hour_mean, + arrivals_per_hour_std, + dropped_jobs_agent_total, + dropped_jobs_baseline_total, ) print( f"[ok ] lambda={lambda_value}: " f"occupancy={stats.occupancy_mean:.2f}%±{stats.occupancy_std:.2f}, " + f"baseline_occ={stats.baseline_occupancy_mean:.2f}%±{stats.baseline_occupancy_std:.2f}, " + f"arrivals/h={stats.arrivals_per_hour_mean:.2f}±{stats.arrivals_per_hour_std:.2f}, " + f"dropped_delta={stats.dropped_jobs_delta_total:.0f}, " f"completion={stats.completion_rate_mean:.2f}%±{stats.completion_rate_std:.2f}, " f"savings={stats.savings_mean:.0f}±{stats.savings_std:.0f}, " f"savings_off={stats.savings_off_mean:.0f}±{stats.savings_off_std:.0f}, " @@ -483,6 +602,15 @@ def write_summary_csv(path: Path, stats_by_lambda: list[LambdaRunStats]) -> None "episodes", "occupancy_mean_pct", "occupancy_std_pct", + "baseline_occupancy_mean_pct", + "baseline_occupancy_std_pct", + "baseline_off_occupancy_mean_pct", + "baseline_off_occupancy_std_pct", + "arrivals_per_hour_mean", + "arrivals_per_hour_std", + "dropped_jobs_agent_total", + "dropped_jobs_baseline_total", + "dropped_jobs_delta_total", "completion_rate_mean_pct", "completion_rate_std_pct", "agent_avg_wait_hours", @@ -515,6 +643,15 @@ def write_summary_csv(path: Path, stats_by_lambda: list[LambdaRunStats]) -> None "episodes": s.episodes, "occupancy_mean_pct": f"{s.occupancy_mean:.6f}", "occupancy_std_pct": f"{s.occupancy_std:.6f}", + "baseline_occupancy_mean_pct": f"{s.baseline_occupancy_mean:.6f}", + "baseline_occupancy_std_pct": f"{s.baseline_occupancy_std:.6f}", + "baseline_off_occupancy_mean_pct": f"{s.baseline_off_occupancy_mean:.6f}", + "baseline_off_occupancy_std_pct": f"{s.baseline_off_occupancy_std:.6f}", + "arrivals_per_hour_mean": f"{s.arrivals_per_hour_mean:.6f}", + "arrivals_per_hour_std": f"{s.arrivals_per_hour_std:.6f}", + "dropped_jobs_agent_total": f"{s.dropped_jobs_agent_total:.6f}", + "dropped_jobs_baseline_total": f"{s.dropped_jobs_baseline_total:.6f}", + "dropped_jobs_delta_total": f"{s.dropped_jobs_delta_total:.6f}", "completion_rate_mean_pct": f"{s.completion_rate_mean:.6f}", "completion_rate_std_pct": f"{s.completion_rate_std:.6f}", "agent_avg_wait_hours": f"{s.agent_avg_wait_hours:.6f}", @@ -540,7 +677,12 @@ def write_summary_csv(path: Path, stats_by_lambda: list[LambdaRunStats]) -> None ) -def make_plot(path: Path, stats_by_lambda: list[LambdaRunStats], fit: bool = False) -> None: +def make_plot( + path: Path, + stats_by_lambda: list[LambdaRunStats], + fit: bool = False, + individual_dir: Path | None = None, +) -> None: ordered = sorted(stats_by_lambda, key=lambda x: x.lambda_value) if not ordered: return @@ -548,6 +690,13 @@ def make_plot(path: Path, stats_by_lambda: list[LambdaRunStats], fit: bool = Fal lambdas = np.array([s.lambda_value for s in ordered], dtype=float) occ_mean = np.array([s.occupancy_mean for s in ordered], dtype=float) occ_std = np.array([s.occupancy_std for s in ordered], dtype=float) + baseline_occ_mean = np.array([s.baseline_occupancy_mean for s in ordered], dtype=float) + baseline_occ_std = np.array([s.baseline_occupancy_std for s in ordered], dtype=float) + baseline_off_occ_mean = np.array([s.baseline_off_occupancy_mean for s in ordered], dtype=float) + baseline_off_occ_std = np.array([s.baseline_off_occupancy_std for s in ordered], dtype=float) + arrivals_per_hour_mean = np.array([s.arrivals_per_hour_mean for s in ordered], dtype=float) + arrivals_per_hour_std = np.array([s.arrivals_per_hour_std for s in ordered], dtype=float) + dropped_jobs_delta_total = np.array([s.dropped_jobs_delta_total for s in ordered], dtype=float) sav_mean = np.array([s.savings_mean for s in ordered], dtype=float) sav_std = np.array([s.savings_std for s in ordered], dtype=float) sav_off_mean = np.array([s.savings_off_mean for s in ordered], dtype=float) @@ -572,6 +721,7 @@ def make_plot(path: Path, stats_by_lambda: list[LambdaRunStats], fit: bool = Fal norm = matplotlib.colors.Normalize(vmin=lam_min, vmax=lam_max) cmap = plt.get_cmap("turbo") point_colors = cmap(norm(lambdas)) + colorbar_label = "Poisson lambda (point color)" def _error_at(arr: np.ndarray | None, idx: int) -> float | None: if arr is None: @@ -579,6 +729,20 @@ def _error_at(arr: np.ndarray | None, idx: int) -> float | None: v = float(arr[idx]) return v if np.isfinite(v) else None + def _maybe_plot_fit(ax: plt.Axes, x: np.ndarray, y: np.ndarray) -> None: + coeffs = None + deg = 0 + if fit: + coeffs, deg = polyfit_curve(x, y, max_degree=3) + if coeffs is None: + return + finite = np.isfinite(x) & np.isfinite(y) + if not np.any(finite): + return + x_fit = np.linspace(float(np.min(x[finite])), float(np.max(x[finite])), 250) + ax.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") + ax.legend() + def plot_colored_points( ax: plt.Axes, x: np.ndarray, @@ -603,161 +767,166 @@ def plot_colored_points( alpha=0.95, ) - fig, axes = plt.subplots(3, 3, figsize=(20, 17), constrained_layout=True) - ax00, ax01, ax02 = axes[0] - ax10, ax11, ax12 = axes[1] - ax20, ax21, ax22 = axes[2] - - # Panel 1: lambda vs occupancy. - plot_colored_points(ax00, lambdas, occ_mean, yerr=occ_std) - coeffs, deg = None, None - if fit: - coeffs, deg = polyfit_curve(lambdas, occ_mean, max_degree=3) - if coeffs is not None: - x_fit = np.linspace(float(np.min(lambdas)), float(np.max(lambdas)), 250) - ax00.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") - ax00.set_title("Lambda vs Occupancy/Episode") - ax00.set_xlabel("Poisson lambda (arrivals)") - ax00.set_ylabel("Agent Occupancy (Nodes, %) / Episode") - ax00.grid(alpha=0.3) - if coeffs is not None: - ax00.legend() - - # Panel 2: occupancy vs savings. - plot_colored_points(ax01, occ_mean, sav_mean, xerr=occ_std, yerr=sav_std) - coeffs, deg = None, None - if fit: - coeffs, deg = polyfit_curve(occ_mean, sav_mean, max_degree=3) - if coeffs is not None: - finite = np.isfinite(occ_mean) & np.isfinite(sav_mean) - x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) - ax01.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") - ax01.set_title("Occupancy/Episode vs Savings/Episode") - ax01.set_xlabel("Agent Occupancy (Nodes, %) / Episode") - ax01.set_ylabel("Savings vs Baseline (EUR / Episode)") - ax01.grid(alpha=0.3) - if coeffs is not None: - ax01.legend() - - # Panel 3: occupancy vs savings_off. - plot_colored_points(ax02, occ_mean, sav_off_mean, xerr=occ_std, yerr=sav_off_std) - coeffs, deg = None, None - if fit: - coeffs, deg = polyfit_curve(occ_mean, sav_off_mean, max_degree=3) - if coeffs is not None: - finite = np.isfinite(occ_mean) & np.isfinite(sav_off_mean) - x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) - ax02.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") - ax02.set_title("Occupancy vs Savings_off/Episode") - ax02.set_xlabel("Agent Occupancy (Nodes, %) / Episode") - ax02.set_ylabel("Savings vs Baseline_off (EUR / Episode)") - ax02.grid(alpha=0.3) - if coeffs is not None: - ax02.legend() - - # Panel 4: lambda vs completion rate. - plot_colored_points(ax10, lambdas, completion_mean, yerr=completion_std) - coeffs, deg = None, None - if fit: - coeffs, deg = polyfit_curve(lambdas, completion_mean, max_degree=3) - if coeffs is not None: - x_fit = np.linspace(float(np.min(lambdas)), float(np.max(lambdas)), 250) - ax10.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") - ax10.set_title("Lambda vs Agent Completion Rate") - ax10.set_xlabel("Poisson lambda (arrivals)") - ax10.set_ylabel("Completion Rate (%)") - ax10.grid(alpha=0.3) - if coeffs is not None: - ax10.legend() - - # Panel 5: occupancy vs effective savings. - plot_colored_points(ax11, occ_mean, eff_sav_mean, xerr=occ_std, yerr=eff_sav_std) - coeffs, deg = None, None - if fit: - coeffs, deg = polyfit_curve(occ_mean, eff_sav_mean, max_degree=3) - if coeffs is not None: - finite = np.isfinite(occ_mean) & np.isfinite(eff_sav_mean) - x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) - ax11.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") - ax11.set_title("Occupancy vs Effective Savings") - ax11.set_xlabel("Agent Occupancy (Nodes, %) / Episode") - ax11.set_ylabel("effective_savings") - ax11.grid(alpha=0.3) - if coeffs is not None: - ax11.legend() - - # Panel 6: occupancy vs effective savings_off. - plot_colored_points(ax12, occ_mean, eff_sav_off_mean, xerr=occ_std, yerr=eff_sav_off_std) - coeffs, deg = None, None - if fit: - coeffs, deg = polyfit_curve(occ_mean, eff_sav_off_mean, max_degree=3) - if coeffs is not None: - finite = np.isfinite(occ_mean) & np.isfinite(eff_sav_off_mean) - x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) - ax12.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") - ax12.set_title("Occupancy vs Effective Savings_off") - ax12.set_xlabel("Agent Occupancy (Nodes, %) / Episode") - ax12.set_ylabel("effective_savings_off") - ax12.grid(alpha=0.3) - if coeffs is not None: - ax12.legend() - - # Panel 7: occupancy vs cost_per_1k delta (%) vs baseline. - plot_colored_points(ax20, occ_mean, cost_per_1k_delta_base_mean, xerr=occ_std, yerr=cost_per_1k_delta_base_std) - coeffs, deg = None, None - if fit: - coeffs, deg = polyfit_curve(occ_mean, cost_per_1k_delta_base_mean, max_degree=3) - if coeffs is not None: - finite = np.isfinite(occ_mean) & np.isfinite(cost_per_1k_delta_base_mean) - x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) - ax20.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") - ax20.set_title("Occupancy vs Cost/1k Delta vs Baseline") - ax20.set_xlabel("Agent Occupancy (Nodes, %) / Episode") - ax20.set_ylabel("(Baseline - Agent) / Baseline [%]") - ax20.grid(alpha=0.3) - if coeffs is not None: - ax20.legend() - - # Panel 8: occupancy vs cost_per_1k delta (%) vs baseline_off. - plot_colored_points(ax21, occ_mean, cost_per_1k_delta_base_off_mean, xerr=occ_std, yerr=cost_per_1k_delta_base_off_std) - coeffs, deg = None, None - if fit: - coeffs, deg = polyfit_curve(occ_mean, cost_per_1k_delta_base_off_mean, max_degree=3) - if coeffs is not None: - finite = np.isfinite(occ_mean) & np.isfinite(cost_per_1k_delta_base_off_mean) - x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) - ax21.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") - ax21.set_title("Occupancy vs Cost/1k Delta vs Baseline_off") - ax21.set_xlabel("Agent Occupancy (Nodes, %) / Episode") - ax21.set_ylabel("(Baseline_off - Agent) / Baseline_off [%]") - ax21.grid(alpha=0.3) - if coeffs is not None: - ax21.legend() - - # Panel 9: occupancy vs power delta (%) vs baseline_off. - plot_colored_points(ax22, occ_mean, power_delta_base_off_mean, xerr=occ_std, yerr=power_delta_base_off_std) - coeffs, deg = None, None - if fit: - coeffs, deg = polyfit_curve(occ_mean, power_delta_base_off_mean, max_degree=3) - if coeffs is not None: - finite = np.isfinite(occ_mean) & np.isfinite(power_delta_base_off_mean) - x_fit = np.linspace(float(np.min(occ_mean[finite])), float(np.max(occ_mean[finite])), 250) - ax22.plot(x_fit, np.polyval(coeffs, x_fit), color="black", lw=2, label=f"poly deg {deg}") - ax22.set_title("Occupancy vs Power Delta vs Baseline_off") - ax22.set_xlabel("Agent Occupancy (Nodes, %) / Episode") - ax22.set_ylabel("(Baseline_off - Agent) / Baseline_off [%]") - ax22.grid(alpha=0.3) - if coeffs is not None: - ax22.legend() + def draw_baseline_occupancy_pair(ax: plt.Axes) -> None: + for i, (xv, c) in enumerate(zip(lambdas, point_colors)): + y_base = float(baseline_occ_mean[i]) + y_base_off = float(baseline_off_occ_mean[i]) + if np.isfinite(xv) and np.isfinite(y_base): + ax.errorbar( + xv, + y_base, + yerr=_error_at(baseline_occ_std, i), + fmt="o", + markersize=6, + capsize=3, + color=c, + ecolor=c, + elinewidth=1.2, + alpha=0.95, + ) + if np.isfinite(xv) and np.isfinite(y_base_off): + ax.errorbar( + xv, + y_base_off, + yerr=_error_at(baseline_off_occ_std, i), + fmt="^", + markersize=6, + capsize=3, + color=c, + ecolor=c, + elinewidth=1.2, + alpha=0.95, + ) + ax.scatter([], [], marker="o", color="black", label="Baseline") + ax.scatter([], [], marker="^", color="black", label="Baseline_off") + ax.legend() + + panel_specs: list[tuple[str, Callable[[plt.Axes], None]]] = [] + + def _panel( + slug: str, + title: str, + xlabel: str, + ylabel: str, + draw_body: Callable[[plt.Axes], None], + ) -> None: + def _draw(ax: plt.Axes) -> None: + draw_body(ax) + ax.set_title(title) + ax.set_xlabel(xlabel) + ax.set_ylabel(ylabel) + ax.grid(alpha=0.3) + panel_specs.append((slug, _draw)) + + _panel( + "01_lambda_vs_agent_occupancy", + "Lambda vs Occupancy/Episode", + "Poisson lambda (arrivals)", + "Agent Occupancy (Nodes, %) / Episode", + lambda ax: (plot_colored_points(ax, lambdas, occ_mean, yerr=occ_std), _maybe_plot_fit(ax, lambdas, occ_mean)), + ) + _panel( + "02_occupancy_vs_savings", + "Occupancy/Episode vs Savings/Episode", + "Agent Occupancy (Nodes, %) / Episode", + "Savings vs Baseline (EUR / Episode)", + lambda ax: (plot_colored_points(ax, occ_mean, sav_mean, xerr=occ_std, yerr=sav_std), _maybe_plot_fit(ax, occ_mean, sav_mean)), + ) + _panel( + "03_occupancy_vs_savings_off", + "Occupancy vs Savings_off/Episode", + "Agent Occupancy (Nodes, %) / Episode", + "Savings vs Baseline_off (EUR / Episode)", + lambda ax: (plot_colored_points(ax, occ_mean, sav_off_mean, xerr=occ_std, yerr=sav_off_std), _maybe_plot_fit(ax, occ_mean, sav_off_mean)), + ) + _panel( + "04_lambda_vs_completion_rate", + "Lambda vs Agent Completion Rate", + "Poisson lambda (arrivals)", + "Completion Rate (%)", + lambda ax: (plot_colored_points(ax, lambdas, completion_mean, yerr=completion_std), _maybe_plot_fit(ax, lambdas, completion_mean)), + ) + _panel( + "05_occupancy_vs_effective_savings", + "Occupancy vs Effective Savings", + "Agent Occupancy (Nodes, %) / Episode", + "effective_savings", + lambda ax: (plot_colored_points(ax, occ_mean, eff_sav_mean, xerr=occ_std, yerr=eff_sav_std), _maybe_plot_fit(ax, occ_mean, eff_sav_mean)), + ) + _panel( + "06_occupancy_vs_effective_savings_off", + "Occupancy vs Effective Savings_off", + "Agent Occupancy (Nodes, %) / Episode", + "effective_savings_off", + lambda ax: (plot_colored_points(ax, occ_mean, eff_sav_off_mean, xerr=occ_std, yerr=eff_sav_off_std), _maybe_plot_fit(ax, occ_mean, eff_sav_off_mean)), + ) + _panel( + "07_occupancy_vs_cost_per_1k_delta_baseline", + "Occupancy vs Cost/1k Delta vs Baseline", + "Agent Occupancy (Nodes, %) / Episode", + "(Baseline - Agent) / Baseline [%]", + lambda ax: (plot_colored_points(ax, occ_mean, cost_per_1k_delta_base_mean, xerr=occ_std, yerr=cost_per_1k_delta_base_std), _maybe_plot_fit(ax, occ_mean, cost_per_1k_delta_base_mean)), + ) + _panel( + "08_occupancy_vs_cost_per_1k_delta_baseline_off", + "Occupancy vs Cost/1k Delta vs Baseline_off", + "Agent Occupancy (Nodes, %) / Episode", + "(Baseline_off - Agent) / Baseline_off [%]", + lambda ax: (plot_colored_points(ax, occ_mean, cost_per_1k_delta_base_off_mean, xerr=occ_std, yerr=cost_per_1k_delta_base_off_std), _maybe_plot_fit(ax, occ_mean, cost_per_1k_delta_base_off_mean)), + ) + _panel( + "09_occupancy_vs_power_delta_baseline_off", + "Occupancy vs Power Delta vs Baseline_off", + "Agent Occupancy (Nodes, %) / Episode", + "(Baseline_off - Agent) / Baseline_off [%]", + lambda ax: (plot_colored_points(ax, occ_mean, power_delta_base_off_mean, xerr=occ_std, yerr=power_delta_base_off_std), _maybe_plot_fit(ax, occ_mean, power_delta_base_off_mean)), + ) + _panel( + "10_lambda_vs_baseline_occupancies", + "Lambda vs Baseline Occupancies", + "Poisson lambda (arrivals)", + "Baseline Occupancy (Nodes, %) / Episode", + draw_baseline_occupancy_pair, + ) + _panel( + "11_lambda_vs_jobs_per_hour", + "Lambda vs Job Arrivals/Hour", + "Poisson lambda (arrivals)", + "Job Arrivals/Hour (mean ± std)", + lambda ax: (plot_colored_points(ax, lambdas, arrivals_per_hour_mean, yerr=arrivals_per_hour_std), _maybe_plot_fit(ax, lambdas, arrivals_per_hour_mean)), + ) + _panel( + "12_lambda_vs_dropped_jobs_delta", + "Lambda vs Dropped Jobs Delta", + "Poisson lambda (arrivals)", + "Dropped Jobs Delta (Agent - Baseline)", + lambda ax: (plot_colored_points(ax, lambdas, dropped_jobs_delta_total), _maybe_plot_fit(ax, lambdas, dropped_jobs_delta_total)), + ) + + fig, axes = plt.subplots(4, 3, figsize=(22, 22), constrained_layout=True) + for ax, (_, draw_fn) in zip(axes.ravel(), panel_specs): + draw_fn(ax) sm = plt.cm.ScalarMappable(norm=norm, cmap=cmap) sm.set_array([]) cbar = fig.colorbar(sm, ax=axes.ravel().tolist(), pad=0.02) - cbar.set_label("Poisson lambda (point color)") - + cbar.set_label(colorbar_label) fig.savefig(path, dpi=220) plt.close(fig) + if individual_dir is not None: + individual_dir.mkdir(parents=True, exist_ok=True) + for slug, draw_fn in panel_specs: + panel_path = individual_dir / f"{slug}.png" + fig_i, ax_i = plt.subplots(1, 1, figsize=(8, 6), constrained_layout=True) + draw_fn(ax_i) + sm_i = plt.cm.ScalarMappable(norm=norm, cmap=cmap) + sm_i.set_array([]) + cbar_i = fig_i.colorbar(sm_i, ax=ax_i, pad=0.02) + cbar_i.set_label(colorbar_label) + fig_i.savefig(panel_path, dpi=220) + plt.close(fig_i) + def parse_int_list(raw: str) -> list[int]: return [int(part.strip()) for part in raw.split(",") if part.strip()] @@ -858,6 +1027,7 @@ def evaluate(lam: int) -> LambdaRunStats: csv_path = out_dir / "summary.csv" json_path = out_dir / "summary.json" plot_path = out_dir / "trendlines.png" + individual_plots_dir = out_dir / "plots_individual" write_summary_csv(csv_path, stats_ordered) with json_path.open("w") as f: @@ -871,13 +1041,14 @@ def evaluate(lam: int) -> LambdaRunStats: f, indent=2, ) - make_plot(plot_path, stats_ordered, fit=args.fit) + make_plot(plot_path, stats_ordered, fit=args.fit, individual_dir=individual_plots_dir) print("\nSweep complete.") print(f" Lambdas: {selected_lambdas}") print(f" CSV: {csv_path}") print(f" JSON: {json_path}") print(f" Plot: {plot_path}") + print(f" Individual Plots: {individual_plots_dir}") if __name__ == "__main__": diff --git a/src/arrival_scale.py b/src/arrival_scale.py new file mode 100644 index 0000000..02a2361 --- /dev/null +++ b/src/arrival_scale.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +import math + + +def validate_job_arrival_scale(job_arrival_scale: float) -> float: + """Return a normalized arrival scale or raise for invalid values.""" + scale = float(job_arrival_scale) + if not math.isfinite(scale) or scale < 0.0: + raise ValueError("--job-arrival-scale must be finite and >= 0.0") + return scale diff --git a/src/evaluation_summary.py b/src/evaluation_summary.py new file mode 100644 index 0000000..f843491 --- /dev/null +++ b/src/evaluation_summary.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import math +from typing import Mapping, Sequence + + +def _fmt_optional(value: float | int | None, precision: int = 2, thousands: bool = False) -> str: + if value is None: + return "n/a" + + numeric_value = float(value) + if math.isnan(numeric_value): + return "n/a" + + return f"{numeric_value:,.{precision}f}" if thousands else f"{numeric_value:.{precision}f}" + + +def mean_occupancy_pct(values: Sequence[int], capacity: int) -> float: + if not values or capacity <= 0: + return 0.0 + return float(sum(values) * 100.0 / (len(values) * capacity)) + + +def build_episode_summary_line( + episode_number: int, + episode_data: Mapping[str, float | int], + timeline_max_queue: int, + agent_occupancy_cores_pct: float, + baseline_occupancy_cores_pct: float, + agent_occupancy_nodes_pct: float, + baseline_occupancy_nodes_pct: float, +) -> str: + return ( + f" Episode {episode_number}: " + f"Agent Cost=€{float(episode_data['agent_cost']):.0f}, " + f"Baseline Cost=€{float(episode_data['baseline_cost']):.0f} | " + f"Baseline Off=€{float(episode_data['baseline_cost_off']):.0f}, " + f"Savings=€{float(episode_data['savings_vs_baseline']):.0f}/" + f"€{float(episode_data['savings_vs_baseline_off']):.0f}, " + f"Power={float(episode_data['agent_power_consumption_mwh']):.1f}/" + f"{float(episode_data['baseline_power_consumption_mwh']):.1f}/" + f"{float(episode_data['baseline_power_consumption_off_mwh']):.1f} MWh " + f"(agent/base/base_off), " + f"MeanPrice={float(episode_data['agent_mean_price']):.2f}/" + f"{float(episode_data['baseline_mean_price']):.2f}/" + f"{float(episode_data['baseline_off_mean_price']):.2f} €/MWh " + f"(agent/base/base_off), " + f"CostPer1kCompleted=" + f"{_fmt_optional(episode_data['agent_cost_per_1000_completed_jobs'], 1, thousands=True)}/" + f"{_fmt_optional(episode_data['baseline_cost_per_1000_completed_jobs'], 1, thousands=True)}/" + f"{_fmt_optional(episode_data['baseline_off_cost_per_1000_completed_jobs'], 1, thousands=True)} " + f"€/1k (agent/base/base_off), " + f"DroppedPerSavedEuro=" + f"{_fmt_optional(episode_data['agent_dropped_jobs_per_saved_euro'], 6)}/" + f"{_fmt_optional(episode_data['agent_dropped_jobs_per_saved_euro_off'], 6)} " + f"jobs/€ (vs base/base_off), " + f"Jobs={int(episode_data['jobs_completed'])}/{int(episode_data['jobs_submitted'])} " + f"({float(episode_data['completion_rate']):.0f}%), " + f"AvgWait={float(episode_data['avg_wait_time']):.1f}h, " + f"EpisodeMaxQueue={int(episode_data['max_queue_size'])}, " + f"Dropped={int(episode_data['jobs_dropped'])}, " + f"TimelineMaxQueue={timeline_max_queue}, " + f"Agent Occupancy (Cores)={agent_occupancy_cores_pct:.2f}%, " + f"Baseline Occupancy (Cores)={baseline_occupancy_cores_pct:.2f}%, " + f"Agent Occupancy (Nodes)={agent_occupancy_nodes_pct:.2f}%, " + f"Baseline Occupancy (Nodes)={baseline_occupancy_nodes_pct:.2f}% " + ) diff --git a/src/sampler_hourly.py b/src/sampler_hourly.py index 07a236e..c1086bf 100644 --- a/src/sampler_hourly.py +++ b/src/sampler_hourly.py @@ -6,6 +6,7 @@ from typing import Any import numpy as np +from src.arrival_scale import validate_job_arrival_scale class HourlySampler: @@ -300,6 +301,7 @@ def sample_aggregated( if not self.aggregation_initialized: raise RuntimeError("Aggregation not initialized. Call precalculate_hourly_templates() first.") + arrival_scale = validate_job_arrival_scale(arrival_scale) hour_of_day = hour_of_day % 24 dist = self.hour_distributions[hour_of_day] diff --git a/src/workload_generator.py b/src/workload_generator.py index fb948be..26b496b 100644 --- a/src/workload_generator.py +++ b/src/workload_generator.py @@ -5,6 +5,7 @@ import math from typing import TYPE_CHECKING import numpy as np +from src.arrival_scale import validate_job_arrival_scale from src.config import ( MAX_NEW_JOBS_PER_HOUR, MAX_JOB_DURATION, MIN_NODES_PER_JOB, MAX_NODES_PER_JOB, MIN_CORES_PER_JOB, CORES_PER_NODE @@ -55,6 +56,7 @@ def generate_jobs( Returns: Tuple of (new_jobs_count, new_jobs_durations, new_jobs_nodes, new_jobs_cores) """ + job_arrival_scale = validate_job_arrival_scale(job_arrival_scale) new_jobs_durations = [] new_jobs_nodes = [] new_jobs_cores = [] diff --git a/train.py b/train.py index 78411a8..84e9fda 100644 --- a/train.py +++ b/train.py @@ -11,6 +11,8 @@ import argparse import sys import pandas as pd +from src.arrival_scale import validate_job_arrival_scale +from src.evaluation_summary import build_episode_summary_line, mean_occupancy_pct from src.workloadgen import WorkloadGenerator from src.workloadgen_cli import add_workloadgen_args, build_workloadgen_config from src.config import MAX_NODES, CORES_PER_NODE, EPISODE_HOURS @@ -80,8 +82,10 @@ def main(): parser.add_argument("--print-policy", action="store_true", help="Print structure of the policy network.") args = parser.parse_args() - if args.job_arrival_scale < 0.0: - parser.error("--job-arrival-scale must be >= 0.0") + try: + args.job_arrival_scale = validate_job_arrival_scale(args.job_arrival_scale) + except ValueError as exc: + parser.error(str(exc)) if args.jobs_exact_replay and not norm_path(args.jobs): parser.error("--jobs-exact-replay requires --jobs") if args.jobs_exact_replay_aggregate and not args.jobs_exact_replay: @@ -243,36 +247,25 @@ def main(): print(f"Episode {episode + 1}, Step {step_count}, Action: {action}, Reward: {reward:.2f}, Total Reward: {episode_reward:.2f}, Total Cost: €{env.metrics.total_cost:.2f}") done = terminated or truncated - savings_vs_baseline = env.metrics.baseline_cost - env.metrics.total_cost - savings_vs_baseline_off = env.metrics.baseline_cost_off - env.metrics.total_cost - completion_rate = (env.metrics.jobs_completed / env.metrics.jobs_submitted * 100) if env.metrics.jobs_submitted > 0 else 0 - avg_wait = env.metrics.total_job_wait_time / env.metrics.jobs_completed if env.metrics.jobs_completed > 0 else 0 - agent_power_mwh = env.metrics.episode_total_power_consumption_mwh - baseline_power_mwh = env.metrics.episode_baseline_power_consumption_mwh - baseline_power_off_mwh = env.metrics.episode_baseline_power_consumption_off_mwh - agent_mean_price = (env.metrics.episode_total_cost / agent_power_mwh) if agent_power_mwh > 0 else 0.0 - baseline_mean_price = (env.metrics.episode_baseline_cost / baseline_power_mwh) if baseline_power_mwh > 0 else 0.0 - baseline_off_mean_price = (env.metrics.episode_baseline_cost_off / baseline_power_off_mwh) if baseline_power_off_mwh > 0 else 0.0 - agent_cost_per_1000_completed = safe_ratio(env.metrics.total_cost * 1000.0, env.metrics.jobs_completed) - baseline_cost_per_1000_completed = safe_ratio(env.metrics.baseline_cost * 1000.0, env.metrics.baseline_jobs_completed) - # baseline_off is a cost variant of baseline scheduling, so it uses the same completed-job count. - baseline_off_cost_per_1000_completed = safe_ratio(env.metrics.baseline_cost_off * 1000.0, env.metrics.baseline_jobs_completed) - dropped_jobs_per_saved_euro = safe_ratio(env.metrics.episode_jobs_dropped, savings_vs_baseline) if savings_vs_baseline > 0 else None - dropped_jobs_per_saved_euro_off = safe_ratio(env.metrics.episode_jobs_dropped, savings_vs_baseline_off) if savings_vs_baseline_off > 0 else None - print(f" Episode {episode + 1}: " - f"Agent Cost=€{env.metrics.total_cost:.0f}, " - f"Baseline Cost=€{env.metrics.baseline_cost:.0f} | Baseline Off=€{env.metrics.baseline_cost_off:.0f}, " - f"Savings=€{savings_vs_baseline:.0f}/€{savings_vs_baseline_off:.0f}, " - f"Power={agent_power_mwh:.1f}/{baseline_power_mwh:.1f}/{baseline_power_off_mwh:.1f} MWh (agent/base/base_off), " - f"MeanPrice={agent_mean_price:.2f}/{baseline_mean_price:.2f}/{baseline_off_mean_price:.2f} €/MWh (agent/base/base_off), " - f"CostPer1kCompleted={fmt_optional(agent_cost_per_1000_completed, 1, thousands=True)}/{fmt_optional(baseline_cost_per_1000_completed, 1, thousands=True)}/{fmt_optional(baseline_off_cost_per_1000_completed, 1, thousands=True)} €/1k (agent/base/base_off), " - f"DroppedPerSavedEuro={fmt_optional(dropped_jobs_per_saved_euro, 6)}/{fmt_optional(dropped_jobs_per_saved_euro_off, 6)} jobs/€ (vs base/base_off), " - f"Jobs={env.metrics.jobs_completed}/{env.metrics.jobs_submitted} ({completion_rate:.0f}%), " - f"AvgWait={avg_wait:.1f}h, " - f"EpisodeMaxQueue={env.metrics.episode_max_queue_size_reached}, Dropped={env.metrics.episode_jobs_dropped}, " - f"MaxQueue={env.metrics.max_queue_size_reached}, " - f"Agent Occupancy (Cores)={env.metrics.episode_used_cores[-1]*100/(CORES_PER_NODE*MAX_NODES) if env.metrics.episode_used_cores else 0 :.2f}%, Baseline Occupancy (Cores)={env.metrics.episode_baseline_used_cores[-1]*100/(CORES_PER_NODE*MAX_NODES) if env.metrics.episode_baseline_used_cores else 0 :.2f}%, " - f"Agent Occupancy (Nodes)={env.metrics.episode_used_nodes[-1]*100/MAX_NODES if env.metrics.episode_used_nodes else 0 :.2f}%, Baseline Occupancy (Nodes)={env.metrics.episode_baseline_used_nodes[-1]*100/MAX_NODES if env.metrics.episode_baseline_used_nodes else 0 :.2f}% " ) + if not env.metrics.episode_costs: + raise RuntimeError("Episode metrics were not recorded before evaluation summary output.") + + episode_data = env.metrics.episode_costs[-1] + agent_occupancy_cores_pct = mean_occupancy_pct(env.metrics.episode_used_cores, CORES_PER_NODE * MAX_NODES) + baseline_occupancy_cores_pct = mean_occupancy_pct(env.metrics.episode_baseline_used_cores, CORES_PER_NODE * MAX_NODES) + agent_occupancy_nodes_pct = mean_occupancy_pct(env.metrics.episode_used_nodes, MAX_NODES) + baseline_occupancy_nodes_pct = mean_occupancy_pct(env.metrics.episode_baseline_used_nodes, MAX_NODES) + print( + build_episode_summary_line( + episode_number=episode + 1, + episode_data=episode_data, + timeline_max_queue=env.metrics.max_queue_size_reached, + agent_occupancy_cores_pct=agent_occupancy_cores_pct, + baseline_occupancy_cores_pct=baseline_occupancy_cores_pct, + agent_occupancy_nodes_pct=agent_occupancy_nodes_pct, + baseline_occupancy_nodes_pct=baseline_occupancy_nodes_pct, + ) + ) print(f"\nEvaluation complete! Generated {num_episodes} episodes of cost data.") @@ -306,6 +299,7 @@ def main(): total_baseline_cost = sum(float(ep['baseline_cost']) for ep in env.metrics.episode_costs) total_baseline_off_cost = sum(float(ep['baseline_cost_off']) for ep in env.metrics.episode_costs) total_jobs_dropped = sum(int(ep.get('jobs_dropped', 0)) for ep in env.metrics.episode_costs) + total_baseline_jobs_dropped = sum(int(ep.get('baseline_jobs_dropped', 0)) for ep in env.metrics.episode_costs) total_agent_power_mwh = sum(float(ep.get('agent_power_consumption_mwh', 0.0)) for ep in env.metrics.episode_costs) total_baseline_power_mwh = sum(float(ep.get('baseline_power_consumption_mwh', 0.0)) for ep in env.metrics.episode_costs) total_baseline_off_power_mwh = sum(float(ep.get('baseline_power_consumption_off_mwh', 0.0)) for ep in env.metrics.episode_costs) @@ -351,6 +345,7 @@ def main(): print(f"\n=== AGENT DROPPED JOBS PER SAVED EURO ===") print(f" Total Dropped Jobs (Agent): {total_jobs_dropped:,}") + print(f" Total Dropped Jobs (Baseline): {total_baseline_jobs_dropped:,}") print(f" Vs Baseline: {fmt_optional(total_dropped_jobs_per_saved_euro, 6)} jobs/€") print(f" Vs Baseline_off: {fmt_optional(total_dropped_jobs_per_saved_euro_off, 6)} jobs/€") diff --git a/train_iter.py b/train_iter.py index d91ce2d..a0d0be6 100644 --- a/train_iter.py +++ b/train_iter.py @@ -5,6 +5,7 @@ import os import sys import time +from src.arrival_scale import validate_job_arrival_scale from src.workloadgen_cli import add_workloadgen_args, build_workloadgen_cli_args @@ -244,8 +245,10 @@ def main(): if args.parallel < 1: parser.error("--parallel must be at least 1") - if args.job_arrival_scale < 0.0: - parser.error("--job-arrival-scale must be >= 0.0") + try: + args.job_arrival_scale = validate_job_arrival_scale(args.job_arrival_scale) + except ValueError as exc: + parser.error(str(exc)) if args.jobs_exact_replay and not norm_path(args.jobs): parser.error("--jobs-exact-replay requires --jobs") if args.jobs_exact_replay_aggregate and not args.jobs_exact_replay: From d3d273d3b82218080c050a31b690e2b94990a573 Mon Sep 17 00:00:00 2001 From: Enis Lorenz Date: Thu, 12 Mar 2026 13:52:10 +0100 Subject: [PATCH 6/7] Analysis QoL: Output directories now include weight combination and model number --- analyze_arrivalscale_occupancy.py | 12 ++++++++- analyze_lambda_occupancy.py | 12 ++++++++- src/analysis_naming.py | 43 +++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 src/analysis_naming.py diff --git a/analyze_arrivalscale_occupancy.py b/analyze_arrivalscale_occupancy.py index bb997d2..6dc9a77 100644 --- a/analyze_arrivalscale_occupancy.py +++ b/analyze_arrivalscale_occupancy.py @@ -46,6 +46,7 @@ matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np +from src.analysis_naming import build_analysis_dir_name EPISODE_RE = re.compile( @@ -955,7 +956,16 @@ def main() -> None: if args.out_dir: out_dir = Path(args.out_dir).expanduser().resolve() else: - out_dir = project_root / "analysis" / f"arrivalscale_occupancy_sweep_{timestamp}" + out_dir_name = build_analysis_dir_name( + prefix="arrivalscale_occupancy_sweep", + timestamp=timestamp, + model=args.model, + efficiency_weight=args.efficiency_weight, + price_weight=args.price_weight, + idle_weight=args.idle_weight, + job_age_weight=args.job_age_weight, + ) + out_dir = project_root / "analysis" / out_dir_name out_dir.mkdir(parents=True, exist_ok=True) logs_dir = out_dir / "logs" diff --git a/analyze_lambda_occupancy.py b/analyze_lambda_occupancy.py index fe970b1..b3fcf4e 100644 --- a/analyze_lambda_occupancy.py +++ b/analyze_lambda_occupancy.py @@ -40,6 +40,7 @@ matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np +from src.analysis_naming import build_analysis_dir_name EPISODE_RE = re.compile( @@ -999,7 +1000,16 @@ def main() -> None: if args.out_dir: out_dir = Path(args.out_dir).expanduser().resolve() else: - out_dir = project_root / "analysis" / f"lambda_occupancy_sweep_{timestamp}" + out_dir_name = build_analysis_dir_name( + prefix="lambda_occupancy_sweep", + timestamp=timestamp, + model=args.model, + efficiency_weight=args.efficiency_weight, + price_weight=args.price_weight, + idle_weight=args.idle_weight, + job_age_weight=args.job_age_weight, + ) + out_dir = project_root / "analysis" / out_dir_name out_dir.mkdir(parents=True, exist_ok=True) logs_dir = out_dir / "logs" if args.save_logs: diff --git a/src/analysis_naming.py b/src/analysis_naming.py new file mode 100644 index 0000000..07f4a73 --- /dev/null +++ b/src/analysis_naming.py @@ -0,0 +1,43 @@ +from __future__ import annotations + + +def _format_slug_number(value: float) -> str: + return f"{float(value):.6f}".rstrip("0").rstrip(".").replace("-", "m").replace(".", "p") + + +def build_weight_slug( + efficiency_weight: float, + price_weight: float, + idle_weight: float, + job_age_weight: float, +) -> str: + return ( + f"e{_format_slug_number(efficiency_weight)}" + f"_p{_format_slug_number(price_weight)}" + f"_i{_format_slug_number(idle_weight)}" + f"_ja{_format_slug_number(job_age_weight)}" + ) + + +def build_analysis_dir_name( + prefix: str, + timestamp: str, + model: int | None, + efficiency_weight: float, + price_weight: float, + idle_weight: float, + job_age_weight: float, +) -> str: + parts = [prefix] + if model is not None: + parts.append(f"m{int(model)}") + parts.append( + build_weight_slug( + efficiency_weight=efficiency_weight, + price_weight=price_weight, + idle_weight=idle_weight, + job_age_weight=job_age_weight, + ) + ) + parts.append(timestamp) + return "_".join(parts) From 5af52553a124dfbefa0fc87cc22c0ce4a6a2d39f Mon Sep 17 00:00:00 2001 From: Enis Lorenz Date: Thu, 12 Mar 2026 14:42:39 +0100 Subject: [PATCH 7/7] Fix analyzer parsing for per-episode savings output - stop differencing parsed episode savings in both occupancy analyzers - treat Savings fields from train.py as already per-episode values - remove obsolete cumulative-savings parsing logic - add regression coverage for analyzer episode metric parsing --- analyze_arrivalscale_occupancy.py | 21 ++++++--------------- analyze_lambda_occupancy.py | 21 ++++++--------------- 2 files changed, 12 insertions(+), 30 deletions(-) diff --git a/analyze_arrivalscale_occupancy.py b/analyze_arrivalscale_occupancy.py index 6dc9a77..d3b401e 100644 --- a/analyze_arrivalscale_occupancy.py +++ b/analyze_arrivalscale_occupancy.py @@ -150,13 +150,6 @@ def _to_float_or_nan(raw: str | None) -> float: return _to_float(raw) -def _diff_cumulative(values: list[float]) -> np.ndarray: - arr = np.asarray(values, dtype=float) - if arr.size == 0: - return arr - return np.diff(np.concatenate(([0.0], arr))) - - def parse_episode_metrics( stdout: str, ) -> tuple[ @@ -176,8 +169,8 @@ def parse_episode_metrics( occupancy = [] baseline_occupancy = [] agent_dropped = [] - cumulative_savings = [] - cumulative_savings_off = [] + savings = [] + savings_off = [] completion_rate = [] avg_wait = [] agent_cost_1k = [] @@ -190,8 +183,8 @@ def parse_episode_metrics( occupancy.append(_to_float(match.group("occupancy"))) baseline_occupancy.append(_to_float(match.group("baseline_occupancy"))) agent_dropped.append(_to_float(match.group("agent_dropped"))) - cumulative_savings.append(_to_float(match.group("savings"))) - cumulative_savings_off.append(_to_float(match.group("savings_off"))) + savings.append(_to_float(match.group("savings"))) + savings_off.append(_to_float(match.group("savings_off"))) completion_rate.append(_to_float(match.group("completion_rate"))) avg_wait.append(_to_float(match.group("avg_wait"))) agent_cost_1k.append(_to_float_or_nan(match.group("agent_cost_1k"))) @@ -206,14 +199,12 @@ def parse_episode_metrics( "Expected lines like 'Episode X: ... Savings=€.../€..., Power=..., CostPer1kCompleted=..., Agent Occupancy (Nodes)=...%'." ) - episode_savings = _diff_cumulative(cumulative_savings) - episode_savings_off = _diff_cumulative(cumulative_savings_off) return ( np.asarray(occupancy, dtype=float), np.asarray(baseline_occupancy, dtype=float), np.asarray(agent_dropped, dtype=float), - episode_savings, - episode_savings_off, + np.asarray(savings, dtype=float), + np.asarray(savings_off, dtype=float), np.asarray(completion_rate, dtype=float), np.asarray(avg_wait, dtype=float), np.asarray(agent_cost_1k, dtype=float), diff --git a/analyze_lambda_occupancy.py b/analyze_lambda_occupancy.py index b3fcf4e..1f431d0 100644 --- a/analyze_lambda_occupancy.py +++ b/analyze_lambda_occupancy.py @@ -143,13 +143,6 @@ def _to_float_or_nan(raw: str | None) -> float: return _to_float(raw) -def _diff_cumulative(values: list[float]) -> np.ndarray: - arr = np.asarray(values, dtype=float) - if arr.size == 0: - return arr - return np.diff(np.concatenate(([0.0], arr))) - - def parse_episode_metrics( stdout: str, ) -> tuple[ @@ -169,8 +162,8 @@ def parse_episode_metrics( occupancy = [] baseline_occupancy = [] agent_dropped = [] - cumulative_savings = [] - cumulative_savings_off = [] + savings = [] + savings_off = [] completion_rate = [] avg_wait = [] agent_cost_1k = [] @@ -183,8 +176,8 @@ def parse_episode_metrics( occupancy.append(_to_float(match.group("occupancy"))) baseline_occupancy.append(_to_float(match.group("baseline_occupancy"))) agent_dropped.append(_to_float(match.group("agent_dropped"))) - cumulative_savings.append(_to_float(match.group("savings"))) - cumulative_savings_off.append(_to_float(match.group("savings_off"))) + savings.append(_to_float(match.group("savings"))) + savings_off.append(_to_float(match.group("savings_off"))) completion_rate.append(_to_float(match.group("completion_rate"))) avg_wait.append(_to_float(match.group("avg_wait"))) agent_cost_1k.append(_to_float_or_nan(match.group("agent_cost_1k"))) @@ -199,14 +192,12 @@ def parse_episode_metrics( "Expected lines like 'Episode X: ... Savings=€.../€..., Agent Occupancy (Nodes)=...%'." ) - episode_savings = _diff_cumulative(cumulative_savings) - episode_savings_off = _diff_cumulative(cumulative_savings_off) return ( np.asarray(occupancy, dtype=float), np.asarray(baseline_occupancy, dtype=float), np.asarray(agent_dropped, dtype=float), - episode_savings, - episode_savings_off, + np.asarray(savings, dtype=float), + np.asarray(savings_off, dtype=float), np.asarray(completion_rate, dtype=float), np.asarray(avg_wait, dtype=float), np.asarray(agent_cost_1k, dtype=float),