diff --git a/sagemaker-serve/src/sagemaker/serve/bedrock_model_builder.py b/sagemaker-serve/src/sagemaker/serve/bedrock_model_builder.py index f3f36ac3ca..902b6dcfd1 100644 --- a/sagemaker-serve/src/sagemaker/serve/bedrock_model_builder.py +++ b/sagemaker-serve/src/sagemaker/serve/bedrock_model_builder.py @@ -364,7 +364,7 @@ def deploy( self._get_bedrock_client(), model_arn ) if existing_deployment: - logger.warning( + logger.info( "Reusing existing custom model %s and deployment %s " "(matched model-source tag). No new resources were created. " "Pass reuse_resources=False to force new resources.", @@ -375,7 +375,7 @@ def deploy( "modelArn": model_arn, "customModelDeploymentArn": existing_deployment, } - logger.warning( + logger.info( "Reusing existing custom model %s (matched model-source tag); " "creating a new deployment on it. Pass reuse_resources=False to " "force a new model.", diff --git a/sagemaker-serve/src/sagemaker/serve/model_builder.py b/sagemaker-serve/src/sagemaker/serve/model_builder.py index f039c53bdc..c125a749f4 100644 --- a/sagemaker-serve/src/sagemaker/serve/model_builder.py +++ b/sagemaker-serve/src/sagemaker/serve/model_builder.py @@ -4061,7 +4061,7 @@ def build( reusable_endpoint = self._find_reusable_endpoint() if reusable_endpoint: self._reused_endpoint_name = reusable_endpoint - logger.warning( + logger.info( "Reusing existing Model %r (matched model-source tag). " "No new Model will be created. Pass reuse_resources=False " "to force a new Model.", @@ -5528,7 +5528,7 @@ def deploy( endpoint_name, reusable_endpoint, ) - logger.warning( + logger.info( "Reusing existing endpoint %r (matched model-source tag and " "deployment configuration). No new resources were created. " "Pass reuse_resources=False to force a new endpoint.", diff --git a/sagemaker-train/src/sagemaker/train/base_trainer.py b/sagemaker-train/src/sagemaker/train/base_trainer.py index c0da67a880..c367fb1154 100644 --- a/sagemaker-train/src/sagemaker/train/base_trainer.py +++ b/sagemaker-train/src/sagemaker/train/base_trainer.py @@ -399,11 +399,20 @@ def show_metrics( ValueError: If no training job has been run yet, no logs/metrics are found, or MLflow is not configured for OSS models. """ - # Validate that we have a training job to get metrics from - if not hasattr(self, '_latest_training_job') or self._latest_training_job is None: - raise ValueError( - "No training job found. Call .train() first, then call .show_metrics() " - "to view training metrics." + # Resolve the job reference. Prefer _latest_training_job (CreateTrainingJob), + # fall back to _latest_job (generic CreateJob API used by MTRL). + resolved_job = getattr(self, '_latest_training_job', None) + if resolved_job is None: + latest_job = getattr(self, '_latest_job', None) + if latest_job is None: + raise ValueError( + "No training job found. Call .train() first, then call .show_metrics() " + "to view training metrics. If training has already completed, set the " + "job name directly via trainer._latest_training_job = '' or " + "trainer._latest_job = ''." + ) + resolved_job = ( + latest_job.job_name if hasattr(latest_job, 'job_name') else str(latest_job) ) # Route based on model type @@ -411,18 +420,19 @@ def show_metrics( is_nova = _is_nova_model(model_name) if model_name else False if is_nova: - return self._show_metrics_cloudwatch(metrics, starting_step, ending_step, start_time, end_time) + return self._show_metrics_cloudwatch(resolved_job, metrics, starting_step, ending_step, start_time, end_time) else: - return self._show_metrics_mlflow(metrics, starting_step, ending_step) + return self._show_metrics_mlflow(resolved_job, metrics, starting_step, ending_step) def _show_metrics_mlflow( self, + resolved_job, metrics: Optional[List[str]] = None, starting_step: Optional[int] = None, ending_step: Optional[int] = None, ) -> None: """Pull and plot training metrics from MLflow for non-Nova models.""" - training_job = self._latest_training_job + training_job = resolved_job # Resolve the TrainingJob object if it's a string if isinstance(training_job, str): @@ -456,6 +466,7 @@ def _show_metrics_mlflow( def _show_metrics_cloudwatch( self, + resolved_job, metrics: Optional[List[str]] = None, starting_step: Optional[int] = None, ending_step: Optional[int] = None, @@ -464,7 +475,7 @@ def _show_metrics_cloudwatch( ) -> Any: """Parse and plot training metrics from CloudWatch logs (Nova models).""" - training_job = self._latest_training_job + training_job = resolved_job if hasattr(training_job, 'training_job_name'): job_id = training_job.training_job_name elif isinstance(training_job, str): @@ -631,10 +642,21 @@ def stream_logs(self, poll: int = 5, start_time: Optional[Any] = None) -> None: Raises: ValueError: If no training job has been run yet. """ - if not hasattr(self, '_latest_training_job') or self._latest_training_job is None: - raise ValueError( - "No training job found. Call .train(wait=False) first, " - "then call .stream_logs() to stream logs in real-time." + # Resolve the job reference. Prefer _latest_training_job (CreateTrainingJob), + # fall back to _latest_job (generic CreateJob API used by MTRL). + resolved_job = getattr(self, '_latest_training_job', None) + if resolved_job is None: + latest_job = getattr(self, '_latest_job', None) + if latest_job is None: + raise ValueError( + "No training job found. Call .train(wait=False) first, " + "then call .stream_logs() to stream logs in real-time. " + "If training has already completed, set the job name directly via " + "trainer._latest_training_job = '' or " + "trainer._latest_job = ''." + ) + resolved_job = ( + latest_job.job_name if hasattr(latest_job, 'job_name') else str(latest_job) ) # Resolve start_time for SMHP jobs @@ -645,7 +667,7 @@ def stream_logs(self, poll: int = 5, start_time: Optional[Any] = None) -> None: else: start_time_ms = int(start_time) - training_job = self._latest_training_job + training_job = resolved_job compute = getattr(self, 'compute', None) if isinstance(compute, HyperPodCompute): diff --git a/sagemaker-train/src/sagemaker/train/common_utils/cloudwatch_metrics.py b/sagemaker-train/src/sagemaker/train/common_utils/cloudwatch_metrics.py index 7a2ade9d43..4bec69559a 100644 --- a/sagemaker-train/src/sagemaker/train/common_utils/cloudwatch_metrics.py +++ b/sagemaker-train/src/sagemaker/train/common_utils/cloudwatch_metrics.py @@ -35,11 +35,13 @@ "SFT": {"training_loss": TRAINING_LOSS_REGEX, "lr": LEARNING_RATE_REGEX}, "CPT": {"training_loss": TRAINING_LOSS_REGEX, "lr": LEARNING_RATE_REGEX}, "RLVR": {"reward_score": SMTJ_RLVR_REWARD_SCORE_REGEX}, + "MTRL": {"reward_score": SMTJ_RLVR_REWARD_SCORE_REGEX}, }, "smhp": { "SFT": {"training_loss": TRAINING_LOSS_REGEX, "lr": LEARNING_RATE_REGEX}, "CPT": {"training_loss": TRAINING_LOSS_REGEX, "lr": LEARNING_RATE_REGEX}, "RLVR": {"reward_score": SMHP_RLVR_REWARD_SCORE_REGEX}, + "MTRL": {"reward_score": SMHP_RLVR_REWARD_SCORE_REGEX}, }, } diff --git a/sagemaker-train/src/sagemaker/train/common_utils/data_utils.py b/sagemaker-train/src/sagemaker/train/common_utils/data_utils.py index f02c35084c..3c417b56e9 100644 --- a/sagemaker-train/src/sagemaker/train/common_utils/data_utils.py +++ b/sagemaker-train/src/sagemaker/train/common_utils/data_utils.py @@ -248,7 +248,7 @@ def is_multimodal_data(dataset: Union[str, "DataSet"]) -> bool: True if multimodal fields detected, False otherwise """ - logger.info(f"Auto-detecting whether dataset is multimodal: {dataset}") + logger.debug(f"Auto-detecting whether dataset is multimodal: {dataset}") if isinstance(dataset, DataSet): data_s3_path = dataset.source diff --git a/sagemaker-train/src/sagemaker/train/cpt_trainer.py b/sagemaker-train/src/sagemaker/train/cpt_trainer.py index d151847005..51a0985a2d 100644 --- a/sagemaker-train/src/sagemaker/train/cpt_trainer.py +++ b/sagemaker-train/src/sagemaker/train/cpt_trainer.py @@ -40,7 +40,6 @@ from sagemaker.core.telemetry.constants import Feature logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) class CPTTrainer(BaseTrainer): diff --git a/sagemaker-train/src/sagemaker/train/dpo_trainer.py b/sagemaker-train/src/sagemaker/train/dpo_trainer.py index 20879e202f..7ef48f361e 100644 --- a/sagemaker-train/src/sagemaker/train/dpo_trainer.py +++ b/sagemaker-train/src/sagemaker/train/dpo_trainer.py @@ -31,7 +31,6 @@ from sagemaker.train.constants import get_sagemaker_hub_name logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) class DPOTrainer(BaseTrainer): diff --git a/sagemaker-train/src/sagemaker/train/multi_turn_rl_trainer.py b/sagemaker-train/src/sagemaker/train/multi_turn_rl_trainer.py index 8794a2a625..3b25820234 100644 --- a/sagemaker-train/src/sagemaker/train/multi_turn_rl_trainer.py +++ b/sagemaker-train/src/sagemaker/train/multi_turn_rl_trainer.py @@ -173,6 +173,8 @@ class MultiTurnRLTrainer(BaseTrainer): and 'job_name_prefix'. If not specified, no notifications are sent. """ + _customization_technique = "MTRL" + def __init__( self, model: Union[str, ModelPackage], diff --git a/sagemaker-train/src/sagemaker/train/sft_trainer.py b/sagemaker-train/src/sagemaker/train/sft_trainer.py index 005cdec054..90ab6f3206 100644 --- a/sagemaker-train/src/sagemaker/train/sft_trainer.py +++ b/sagemaker-train/src/sagemaker/train/sft_trainer.py @@ -41,7 +41,6 @@ from sagemaker.core.training.constants import TrainingPlatform logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) class SFTTrainer(BaseTrainer):