Skip to content

test: add integ tests for show_metrics and model reuse (reuse_resourc… #6158

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mujtaba1747 merged 1 commit into from
Aug 5, 2026
Merged

test: add integ tests for show_metrics and model reuse (reuse_resourc… #6158

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 72 additions & 35 deletions sagemaker-serve/tests/integ/test_model_customization_deployment.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@
logger = logging.getLogger(__name__)

from sagemaker.core.helper.session_helper import Session
from sagemaker.core.resources import TrainingJob, ModelPackage, InferenceComponent, Endpoint
from sagemaker.core.utils.exceptions import FailedStatusError
from sagemaker.serve import ModelBuilder
from sagemaker.serve.bedrock_model_builder import BedrockModelBuilder
from sagemaker.serve.model_reuse import MODEL_SOURCE_TAG_KEY

# This test relies on resources in a specific region
AWS_REGION = "us-west-2"
Expand Down Expand Up @@ -68,7 +73,6 @@ def model_package_arn():
@pytest.fixture
def endpoint_name():
"""Generate unique endpoint name."""
import time
return f"e2e-{int(time.time())}-{random.randint(100, 10000)}"


Expand All @@ -80,7 +84,6 @@ def cleanup_endpoints():

for ep_name in endpoints_to_cleanup:
try:
from sagemaker.core.resources import Endpoint
endpoint = Endpoint.get(endpoint_name=ep_name, region=AWS_REGION)
endpoint.delete()
except Exception:
Expand All @@ -92,9 +95,6 @@ class TestModelCustomizationFromTrainingJob:

def test_build_from_training_job(self, training_job_name, sagemaker_session):
"""Test building model from training job."""
from sagemaker.core.resources import TrainingJob
from sagemaker.serve import ModelBuilder
import time

training_job = TrainingJob.get(training_job_name=training_job_name, region=AWS_REGION)
model_builder = ModelBuilder(model=training_job, sagemaker_session=sagemaker_session)
Expand All @@ -112,11 +112,7 @@ def test_deploy_from_training_job(self, training_job_name, endpoint_name, cleanu
For LORA models, this verifies the two-step deployment:
base IC + adapter IC are both created on the same endpoint.
"""
from sagemaker.core.resources import TrainingJob, InferenceComponent
from sagemaker.serve import ModelBuilder
import time

from sagemaker.core.utils.exceptions import FailedStatusError

training_job = TrainingJob.get(training_job_name=training_job_name, region=AWS_REGION)
model_builder = ModelBuilder(model=training_job, instance_type="ml.g5.4xlarge", sagemaker_session=sagemaker_session)
Expand Down Expand Up @@ -148,7 +144,6 @@ def test_deploy_from_training_job(self, training_job_name, endpoint_name, cleanu
assert endpoint.endpoint_status == "InService"

# Verify model-source tag is present on the endpoint for reuse discovery.
from sagemaker.serve.model_reuse import MODEL_SOURCE_TAG_KEY
sm_client = boto3.client("sagemaker", region_name=AWS_REGION)
endpoint_tags = sm_client.list_tags(ResourceArn=endpoint.endpoint_arn).get("Tags", [])
assert MODEL_SOURCE_TAG_KEY in {t["Key"] for t in endpoint_tags}, (
Expand Down Expand Up @@ -200,22 +195,85 @@ def test_deploy_from_training_job(self, training_job_name, endpoint_name, cleanu

def test_fetch_endpoint_names_for_base_model(self, training_job_name, sagemaker_session):
"""Test fetching endpoint names for base model."""
from sagemaker.core.resources import TrainingJob
from sagemaker.serve import ModelBuilder

training_job = TrainingJob.get(training_job_name=training_job_name, region=AWS_REGION)
model_builder = ModelBuilder(model=training_job, sagemaker_session=sagemaker_session)
endpoint_names = model_builder.fetch_endpoint_names_for_base_model()

assert isinstance(endpoint_names, set)

def test_deploy_reuse_returns_existing_endpoint(self, training_job_name, endpoint_name, cleanup_endpoints, sagemaker_session):
"""deploy(reuse_resources=True) finds and returns an existing tagged endpoint.

Flow:
1. First deploy creates an endpoint with the model-source tag.
2. Second deploy with reuse_resources=True returns the same endpoint.
"""

training_job = TrainingJob.get(training_job_name=training_job_name, region=AWS_REGION)
model_builder = ModelBuilder(model=training_job, instance_type="ml.g5.4xlarge", sagemaker_session=sagemaker_session)
model_builder.accept_eula = True
model_builder.build(model_name=f"test-model-{int(time.time())}-{random.randint(100, 10000)}", region=AWS_REGION)

try:
endpoint = model_builder.deploy(endpoint_name=endpoint_name)
except (FailedStatusError, ClientError) as e:
msg = str(e)
if "InsufficientInstanceCapacity" in msg or "ResourceLimitExceeded" in msg:
cleanup_endpoints.append(endpoint_name)
pytest.xfail(f"Capacity/quota limit: {e}")
raise

cleanup_endpoints.append(endpoint_name)
assert endpoint is not None
assert endpoint.endpoint_status == "InService"

# Second deploy with reuse should find the same endpoint
builder2 = ModelBuilder(model=training_job, instance_type="ml.g5.4xlarge", sagemaker_session=sagemaker_session)
builder2.accept_eula = True
builder2.build(region=AWS_REGION, reuse_resources=True)
endpoint2 = builder2.deploy(reuse_resources=True)

assert endpoint2 is not None
assert endpoint2.endpoint_arn == endpoint.endpoint_arn, (

@mujtaba1747 mujtaba1747 Aug 5, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nice

f"Expected reuse to return {endpoint.endpoint_arn}, got {endpoint2.endpoint_arn}"
)

def test_build_reuse_skips_model_creation(self, training_job_name, sagemaker_session):
"""build(reuse_resources=True) reuses an existing tagged Model."""

training_job = TrainingJob.get(training_job_name=training_job_name, region=AWS_REGION)
# Note: tiny collision risk if parallel CI runs share the same timestamp+random seed
unique_id = f"{int(time.time())}-{random.randint(100, 10000)}"
model_name = f"reuse-build-{unique_id}"

builder1 = ModelBuilder(model=training_job, instance_type="ml.g5.4xlarge", sagemaker_session=sagemaker_session)
builder1.accept_eula = True
model1 = builder1.build(model_name=model_name, region=AWS_REGION, reuse_resources=False)
assert model1 is not None
assert model1.model_arn is not None

# Second build with reuse
builder2 = ModelBuilder(model=training_job, instance_type="ml.g5.4xlarge", sagemaker_session=sagemaker_session)
builder2.accept_eula = True
model2 = builder2.build(region=AWS_REGION, reuse_resources=True)
assert model2 is not None
assert model2.model_arn == model1.model_arn, (
f"Expected build reuse to return {model1.model_arn}, got {model2.model_arn}"
)

# Cleanup: delete the model created during this test
sm_client = boto3.client("sagemaker", region_name=AWS_REGION)
try:
sm_client.delete_model(ModelName=model_name)
except Exception:
pass


class TestModelCustomizationFromModelPackage:

def test_build_from_model_package(self, model_package_arn, sagemaker_session):
"""Test building model from model package."""
from sagemaker.core.resources import ModelPackage
from sagemaker.serve import ModelBuilder

model_package = ModelPackage.get(model_package_name=model_package_arn, region=AWS_REGION)
model_builder = ModelBuilder(model=model_package, sagemaker_session=sagemaker_session)
Expand All @@ -227,9 +285,6 @@ def test_build_from_model_package(self, model_package_arn, sagemaker_session):

def test_deploy_from_model_package(self, model_package_arn, cleanup_endpoints, sagemaker_session):
"""Test deploying model from model package."""
from sagemaker.core.resources import ModelPackage
from sagemaker.serve import ModelBuilder
import time

model_package = ModelPackage.get(model_package_name=model_package_arn, region=AWS_REGION)
endpoint_name = f"e2e-{int(time.time())}-{random.randint(100, 10000)}"
Expand All @@ -249,8 +304,6 @@ class TestInstanceTypeAutoDetection:

def test_instance_type_from_recipe(self, training_job_name, sagemaker_session):
"""Test instance type auto-detection from recipe."""
from sagemaker.core.resources import TrainingJob
from sagemaker.serve import ModelBuilder

training_job = TrainingJob.get(training_job_name=training_job_name, region=AWS_REGION)
model_builder = ModelBuilder(model=training_job, sagemaker_session=sagemaker_session)
Expand All @@ -266,8 +319,6 @@ class TestModelCustomizationDetection:

def test_is_model_customization_training_job(self, training_job_name, sagemaker_session):
"""Test detection from training job."""
from sagemaker.core.resources import TrainingJob
from sagemaker.serve import ModelBuilder

training_job = TrainingJob.get(training_job_name=training_job_name, region=AWS_REGION)
model_builder = ModelBuilder(model=training_job, sagemaker_session=sagemaker_session)
Expand All @@ -276,8 +327,6 @@ def test_is_model_customization_training_job(self, training_job_name, sagemaker_

def test_is_model_customization_model_package(self, model_package_arn, sagemaker_session):
"""Test detection from model package."""
from sagemaker.core.resources import ModelPackage
from sagemaker.serve import ModelBuilder

model_package = ModelPackage.get(model_package_name=model_package_arn, region=AWS_REGION)
model_builder = ModelBuilder(model=model_package, sagemaker_session=sagemaker_session)
Expand All @@ -286,8 +335,6 @@ def test_is_model_customization_model_package(self, model_package_arn, sagemaker

def test_fetch_model_package_arn(self, training_job_name, sagemaker_session):
"""Test fetching model package ARN."""
from sagemaker.core.resources import TrainingJob
from sagemaker.serve import ModelBuilder

training_job = TrainingJob.get(training_job_name=training_job_name, region=AWS_REGION)
model_builder = ModelBuilder(model=training_job, sagemaker_session=sagemaker_session)
Expand All @@ -303,9 +350,6 @@ class TestTrainerIntegration:

def test_sft_trainer_build(self, training_job_name, sagemaker_session):
"""Test building model from SFTTrainer."""
from sagemaker.core.resources import TrainingJob
from sagemaker.train.sft_trainer import SFTTrainer
from sagemaker.serve import ModelBuilder

training_job = TrainingJob.get(
training_job_name=training_job_name, region=AWS_REGION
Expand All @@ -327,9 +371,6 @@ def test_sft_trainer_build(self, training_job_name, sagemaker_session):

def test_dpo_trainer_build(self, training_job_name, sagemaker_session):
"""Test building model from DPOTrainer."""
from sagemaker.core.resources import TrainingJob
from sagemaker.train.dpo_trainer import DPOTrainer
from sagemaker.serve import ModelBuilder
from unittest.mock import patch

training_job = TrainingJob.get(
Expand Down Expand Up @@ -363,9 +404,6 @@ def test_dpo_trainer_build(self, training_job_name, sagemaker_session):
- Improved test assertions to work with new object structures
"""

from sagemaker.core.resources import TrainingJob, ModelPackage
from sagemaker.serve.bedrock_model_builder import BedrockModelBuilder


@pytest.mark.serial
class TestModelCustomizationDeployment:
Expand All @@ -374,7 +412,6 @@ class TestModelCustomizationDeployment:
@pytest.fixture(scope="class")
def setup_config(self, training_job_name):
"""Setup test configuration."""
from sagemaker.core.helper.session_helper import get_execution_role
return {
"training_job_name": training_job_name,
"region": AWS_REGION,
Expand Down
Loading