From fb2461e815470cad789af85a94ca0aa7a23987d4 Mon Sep 17 00:00:00 2001 From: Ajay Karpur Date: Fri, 18 Sep 2020 13:04:37 -0700 Subject: [PATCH 1/6] fix: add missing account IDs for af-south-1 and eu-south-1 --- src/sagemaker/amazon/amazon_estimator.py | 6 ++++ src/sagemaker/debugger.py | 2 ++ src/sagemaker/fw_registry.py | 10 +++++++ src/sagemaker/fw_utils.py | 14 +++++++-- src/sagemaker/model.py | 2 ++ .../model_monitor/model_monitoring.py | 2 ++ tests/integ/__init__.py | 29 +++++++++++++++++-- tests/unit/__init__.py | 2 ++ 8 files changed, 62 insertions(+), 5 deletions(-) diff --git a/src/sagemaker/amazon/amazon_estimator.py b/src/sagemaker/amazon/amazon_estimator.py index 0466ae23f0..ece56e5bf7 100644 --- a/src/sagemaker/amazon/amazon_estimator.py +++ b/src/sagemaker/amazon/amazon_estimator.py @@ -513,6 +513,8 @@ def registry(region_name, algorithm=None): "me-south-1": "249704162688", "cn-north-1": "390948362332", "cn-northwest-1": "387376663083", + "af-south-1": "455444449433", + "eu-south-1": "257386234256", } elif algorithm in ["lda"]: region_to_accounts = { @@ -556,6 +558,8 @@ def registry(region_name, algorithm=None): "me-south-1": "249704162688", "cn-north-1": "390948362332", "cn-northwest-1": "387376663083", + "af-south-1": "455444449433", + "eu-south-1": "257386234256", } elif algorithm in [ "xgboost", @@ -588,6 +592,8 @@ def registry(region_name, algorithm=None): "me-south-1": "249704162688", "cn-north-1": "390948362332", "cn-northwest-1": "387376663083", + "af-south-1": "455444449433", + "eu-south-1": "257386234256", } elif algorithm in ["image-classification-neo", "xgboost-neo"]: region_to_accounts = NEO_IMAGE_ACCOUNT diff --git a/src/sagemaker/debugger.py b/src/sagemaker/debugger.py index 8e040ca890..016a6bf6e6 100644 --- a/src/sagemaker/debugger.py +++ b/src/sagemaker/debugger.py @@ -48,6 +48,8 @@ "ca-central-1": {RULES_ECR_REPO_NAME: "519511493484"}, "cn-north-1": {RULES_ECR_REPO_NAME: "618459771430"}, "cn-northwest-1": {RULES_ECR_REPO_NAME: "658757709296"}, + "af-south-1": {RULES_ECR_REPO_NAME: "314341159256"}, + "eu-south-1": {RULES_ECR_REPO_NAME: "563282790590"}, } diff --git a/src/sagemaker/fw_registry.py b/src/sagemaker/fw_registry.py index f0310c926b..0435a3b873 100644 --- a/src/sagemaker/fw_registry.py +++ b/src/sagemaker/fw_registry.py @@ -130,6 +130,16 @@ "scikit-learn": "451049120500", "xgboost": "451049120500", }, + "af-south-1": { + "sparkml-serving": "510948584623", + "scikit-learn": "510948584623", + "xgboost": "455444449433", + }, + "eu-south-1": { + "sparkml-serving": "978288397137", + "scikit-learn": "978288397137", + "xgboost": "257386234256", + }, } diff --git a/src/sagemaker/fw_utils.py b/src/sagemaker/fw_utils.py index f2b9ff629e..7e936dcd82 100644 --- a/src/sagemaker/fw_utils.py +++ b/src/sagemaker/fw_utils.py @@ -91,8 +91,18 @@ "cn-north-1": "727897471807", "cn-northwest-1": "727897471807", } -OPT_IN_ACCOUNTS_BY_REGION = {"ap-east-1": "057415533634", "me-south-1": "724002660598"} -ASIMOV_OPT_IN_ACCOUNTS_BY_REGION = {"ap-east-1": "871362719292", "me-south-1": "217643126080"} +OPT_IN_ACCOUNTS_BY_REGION = { + "ap-east-1": "057415533634", + "me-south-1": "724002660598", + "af-south-1": "313743910680", + "eu-south-1": "048378556238", +} +ASIMOV_OPT_IN_ACCOUNTS_BY_REGION = { + "ap-east-1": "871362719292", + "me-south-1": "217643126080", + "af-south-1": "626614931356", + "eu-south-1": "692866216735", +} DEFAULT_ACCOUNT = "520713654638" ASIMOV_PROD_ACCOUNT = "763104351884" ASIMOV_DEFAULT_ACCOUNT = ASIMOV_PROD_ACCOUNT diff --git a/src/sagemaker/model.py b/src/sagemaker/model.py index b135f386fc..403f833987 100644 --- a/src/sagemaker/model.py +++ b/src/sagemaker/model.py @@ -50,6 +50,8 @@ "cn-north-1": "472730292857", "cn-northwest-1": "474822919863", "us-gov-west-1": "263933020539", + "af-south-1": "774647643957", + "eu-south-1": "966458181534", } INFERENTIA_INSTANCE_PREFIX = "ml_inf" diff --git a/src/sagemaker/model_monitor/model_monitoring.py b/src/sagemaker/model_monitor/model_monitoring.py index 1dbe582704..bf55fde11d 100644 --- a/src/sagemaker/model_monitor/model_monitoring.py +++ b/src/sagemaker/model_monitor/model_monitoring.py @@ -58,6 +58,8 @@ "cn-north-1": "453000072557", "cn-northwest-1": "453252182341", "us-gov-west-1": "362178532790", + "af-south-1": "875698925577", + "eu-south-1": "933208885752", } STATISTICS_JSON_DEFAULT_FILE_NAME = "statistics.json" diff --git a/tests/integ/__init__.py b/tests/integ/__init__.py index 3ae638d103..8183354da0 100644 --- a/tests/integ/__init__.py +++ b/tests/integ/__init__.py @@ -74,9 +74,32 @@ "us-west-2", ] -NO_LDA_REGIONS = ["eu-west-3", "eu-north-1", "sa-east-1", "ap-east-1", "me-south-1"] -NO_MARKET_PLACE_REGIONS = ["eu-west-3", "eu-north-1", "sa-east-1", "ap-east-1", "me-south-1"] -NO_AUTO_ML_REGIONS = ["sa-east-1", "me-south-1", "ap-east-1", "eu-west-3"] +NO_LDA_REGIONS = [ + "eu-west-3", + "eu-north-1", + "sa-east-1", + "ap-east-1", + "me-south-1", + "af-south-1", + "eu-south-1", +] +NO_MARKET_PLACE_REGIONS = [ + "eu-west-3", + "eu-north-1", + "sa-east-1", + "ap-east-1", + "me-south-1", + "af-south-1", + "eu-south-1", +] +NO_AUTO_ML_REGIONS = [ + "sa-east-1", + "me-south-1", + "ap-east-1", + "eu-west-3", + "af-south-1", + "eu-south-1", +] NO_MODEL_MONITORING_REGIONS = ["me-south-1"] EFS_TEST_ENABLED_REGION = [] diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index a94934a1bc..202658aa2d 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -38,4 +38,6 @@ "cn-north-1", "cn-northwest-1", "us-gov-west-1", + "af-south-1", + "eu-south-1", ] From 80b540e68754170fb1e9ce43288fa6e40b82032f Mon Sep 17 00:00:00 2001 From: Ajay Karpur Date: Fri, 18 Sep 2020 14:01:33 -0700 Subject: [PATCH 2/6] drop number of processes to 384 --- buildspec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildspec.yml b/buildspec.yml index f19c4db7ba..4b8036f702 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -16,7 +16,7 @@ phases: - start_time=`date +%s` - | - execute-command-if-has-matching-changes "env -u AWS_DEFAULT_REGION tox -e py38 -- tests/integ -m \"not local_mode\" -n 512 --reruns 3 --reruns-delay 5 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml" + execute-command-if-has-matching-changes "env -u AWS_DEFAULT_REGION tox -e py38 -- tests/integ -m \"not local_mode\" -n 384 --reruns 3 --reruns-delay 5 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml" - ./ci-scripts/displaytime.sh 'py38 tests/integ' $start_time post_build: From 139c603ec94db07eedce94aec2cfa6508e9a5f0a Mon Sep 17 00:00:00 2001 From: Ajay Karpur Date: Fri, 18 Sep 2020 14:12:12 -0700 Subject: [PATCH 3/6] update buildspec --- buildspec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildspec.yml b/buildspec.yml index 4b8036f702..54bffbe2b4 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -16,7 +16,7 @@ phases: - start_time=`date +%s` - | - execute-command-if-has-matching-changes "env -u AWS_DEFAULT_REGION tox -e py38 -- tests/integ -m \"not local_mode\" -n 384 --reruns 3 --reruns-delay 5 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml" + execute-command-if-has-matching-changes "env -u AWS_DEFAULT_REGION tox -e py38 -- tests/integ -m \"not local_mode and not cron\" -n 384 --reruns 3 --reruns-delay 15 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml" - ./ci-scripts/displaytime.sh 'py38 tests/integ' $start_time post_build: From ed1742725680afc655e8dcd0f9b681cc3ea069c9 Mon Sep 17 00:00:00 2001 From: Ajay Karpur Date: Fri, 18 Sep 2020 16:24:26 -0700 Subject: [PATCH 4/6] fix xgboost accounts --- src/sagemaker/fw_registry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sagemaker/fw_registry.py b/src/sagemaker/fw_registry.py index 0435a3b873..7d54c9574d 100644 --- a/src/sagemaker/fw_registry.py +++ b/src/sagemaker/fw_registry.py @@ -133,12 +133,12 @@ "af-south-1": { "sparkml-serving": "510948584623", "scikit-learn": "510948584623", - "xgboost": "455444449433", + "xgboost": "510948584623", }, "eu-south-1": { "sparkml-serving": "978288397137", "scikit-learn": "978288397137", - "xgboost": "257386234256", + "xgboost": "978288397137", }, } From 40a3f8c96ddbc760537c7d8e01a9775102073aaf Mon Sep 17 00:00:00 2001 From: Ajay Karpur Date: Mon, 21 Sep 2020 12:49:11 -0700 Subject: [PATCH 5/6] temporarily add debugging changes to buildspec --- buildspec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildspec.yml b/buildspec.yml index 54bffbe2b4..8e2ed833be 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -16,7 +16,7 @@ phases: - start_time=`date +%s` - | - execute-command-if-has-matching-changes "env -u AWS_DEFAULT_REGION tox -e py38 -- tests/integ -m \"not local_mode and not cron\" -n 384 --reruns 3 --reruns-delay 15 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml" + execute-command-if-has-matching-changes "env -u AWS_DEFAULT_REGION tox -e py38 -- tests/integ -m \"not local_mode and not cron\" -k "test_airflow_config.py" -n 384 --reruns 3 --reruns-delay 15 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml" - ./ci-scripts/displaytime.sh 'py38 tests/integ' $start_time post_build: From 86602dbd7999c39917451a881e337cea8e1a3111 Mon Sep 17 00:00:00 2001 From: Ajay Karpur Date: Mon, 21 Sep 2020 12:58:44 -0700 Subject: [PATCH 6/6] update debugging in buildspec --- buildspec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildspec.yml b/buildspec.yml index 8e2ed833be..3533dc0042 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -16,7 +16,7 @@ phases: - start_time=`date +%s` - | - execute-command-if-has-matching-changes "env -u AWS_DEFAULT_REGION tox -e py38 -- tests/integ -m \"not local_mode and not cron\" -k "test_airflow_config.py" -n 384 --reruns 3 --reruns-delay 15 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml" + execute-command-if-has-matching-changes "env -u AWS_DEFAULT_REGION tox -e py38 -- tests/integ -m \"not local_mode and not cron\" -k "not byo_airflow_config_uploads_data_source_to_s3_when_inputs_provided" -n 384 --reruns 3 --reruns-delay 15 --durations 50 --boto-config '{\"region_name\": \"us-east-2\"}'" "tests/integ" "tests/scripts" "tests/data" "tests/conftest.py" "tests/__init__.py" "src/*.py" "setup.py" "setup.cfg" "buildspec.yml" - ./ci-scripts/displaytime.sh 'py38 tests/integ' $start_time post_build: