From 26ce2f39a10fa81e78d79db781857eda67cbfdad Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Tue, 28 Nov 2023 13:16:09 -0800 Subject: [PATCH 01/11] Update CI script to run test_mps --- .github/workflows/pull_mps.yml | 33 ++++++++++++++-- .../mps/ci/scripts/gather_test_models_mps.py | 38 +++++++++++++++++-- backends/apple/mps/ci/scripts/test-mps.sh | 31 ++++++++++++++- backends/apple/mps/test/test_mps.py | 2 +- backends/apple/mps/test/test_mps_utils.py | 4 +- 5 files changed, 97 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pull_mps.yml b/.github/workflows/pull_mps.yml index 80002605b0c..90a8782d431 100644 --- a/.github/workflows/pull_mps.yml +++ b/.github/workflows/pull_mps.yml @@ -16,6 +16,7 @@ jobs: runs-on: macos-executorch outputs: models: ${{ steps.gather-models-mps.outputs.models }} + mps_models: ${{ steps.gather-models-mps.outputs.mps_models }} steps: - uses: actions/checkout@v3 with: @@ -34,12 +35,36 @@ jobs: install_pip_dependencies install_executorch PYTHONPATH="${PWD}" python -m backends.apple.mps.ci.scripts.gather_test_models_mps - test-mps-delegate-macos: - name: test-mps-delegate-macos + # test-mps-models-macos: + # name: test-mps-models-macos + # runs-on: macos-executorch + # needs: gather-models-mps + # strategy: + # matrix: ${{ fromJSON(needs.gather-models-mps.outputs.models) }} + # fail-fast: false + # steps: + # - uses: actions/checkout@v3 + # with: + # submodules: 'true' + # - name: Run test ${{ matrix.test }} + # if: always() + # run: | + # WORKSPACE=$(pwd) + # pushd "${WORKSPACE}" + # MODEL_NAME=${{ matrix.model }} + # BUILD_TOOL=${{ matrix.build-tool }} + # # Setup MacOS dependencies as there is no Docker support on MacOS atm + # PYTHON_EXECUTABLE=python bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}" + # PYTHON_EXECUTABLE=python bash backends/apple/mps/install_requirements.sh + # # Build and test ExecuTorch + # PYTHON_EXECUTABLE=python bash backends/apple/mps/ci/scripts/test-mps.sh "${MODEL_NAME}" "${BUILD_TOOL}" + # popd + test-mps-macos: + name: test-mps-macos runs-on: macos-executorch needs: gather-models-mps strategy: - matrix: ${{ fromJSON(needs.gather-models-mps.outputs.models) }} + matrix: ${{ fromJSON(needs.gather-models-mps.outputs.mps_models) }} fail-fast: false steps: - uses: actions/checkout@v3 @@ -56,5 +81,5 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}" PYTHON_EXECUTABLE=python bash backends/apple/mps/install_requirements.sh # Build and test ExecuTorch - PYTHON_EXECUTABLE=python bash backends/apple/mps/ci/scripts/test-mps.sh "${MODEL_NAME}" "${BUILD_TOOL}" + PYTHON_EXECUTABLE=python bash backends/apple/mps/ci/scripts/test-mps.sh "${MODEL_NAME}" "${BUILD_TOOL}" 1 popd diff --git a/backends/apple/mps/ci/scripts/gather_test_models_mps.py b/backends/apple/mps/ci/scripts/gather_test_models_mps.py index 8f448cae703..d76a4a16f71 100644 --- a/backends/apple/mps/ci/scripts/gather_test_models_mps.py +++ b/backends/apple/mps/ci/scripts/gather_test_models_mps.py @@ -9,6 +9,10 @@ from examples.models import MODEL_NAME_TO_MODEL from examples.xnnpack import MODEL_NAME_TO_OPTIONS +import unittest + +MPS_TEST_SUITE_PATH = 'backends/apple/mps/test' +MPS_SUITE = unittest.defaultTestLoader.discover(MPS_TEST_SUITE_PATH) BUILD_TOOLS = { "cmake": {"macos-14"}, @@ -43,6 +47,17 @@ def set_output(name: str, val: Any) -> None: else: print(f"::set-output name={name}::{val}") +def gather_mps_test_list(suite, mps_test_list): + if hasattr(suite, '__iter__'): + for x in suite: + gather_mps_test_list(x, mps_test_list) + else: + mps_test_list.append(suite) + +def gather_mps_tests(suite): + mps_test_list = [] + gather_mps_test_list(suite, mps_test_list) + return mps_test_list def export_models_for_ci() -> None: """ @@ -55,9 +70,6 @@ def export_models_for_ci() -> None: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs models = {"include": []} for name in MODEL_NAME_TO_MODEL.keys(): - delegation_configs = { - name in MODEL_NAME_TO_OPTIONS and MODEL_NAME_TO_OPTIONS[name].delegation, - } for build_tool in BUILD_TOOLS.keys(): if target_os not in BUILD_TOOLS[build_tool]: continue @@ -72,6 +84,26 @@ def export_models_for_ci() -> None: set_output("models", json.dumps(models)) + # Gather the list of all mps tests + mps_models = {"include": []} + mps_test_list = gather_mps_tests(MPS_SUITE) + start_path = ".".join([MPS_TEST_SUITE_PATH.replace("/", "."), "test_mps"]) + for testcase in mps_test_list: + if "test_mps" not in str(testcase.__class__): + continue + for build_tool in BUILD_TOOLS.keys(): + if target_os not in BUILD_TOOLS[build_tool]: + continue + + record = { + "build-tool": build_tool, + "model": testcase._testMethodName, + "runner": DEFAULT_RUNNERS.get(target_os), + } + + mps_models["include"].append(record) + + set_output("mps_models", json.dumps(mps_models)) if __name__ == "__main__": export_models_for_ci() diff --git a/backends/apple/mps/ci/scripts/test-mps.sh b/backends/apple/mps/ci/scripts/test-mps.sh index d68ea0e980d..a826bb74a05 100644 --- a/backends/apple/mps/ci/scripts/test-mps.sh +++ b/backends/apple/mps/ci/scripts/test-mps.sh @@ -21,6 +21,12 @@ if [[ -z "${BUILD_TOOL:-}" ]]; then exit 1 fi +MPS_TESTS=$3 +if [[ -z "${MPS_TESTS:-}" ]]; then + echo "Missing flag specifying which tests to run, exiting..." + exit 1 +fi + which "${PYTHON_EXECUTABLE}" CMAKE_OUTPUT_DIR=cmake-out @@ -62,5 +68,28 @@ test_model_with_mps() { fi } +test_mps_model() { + "${PYTHON_EXECUTABLE}" -m unittest "${MODEL_NAME}" + + TEST_NAME="${MODEL_NAME##*.}" + OUTPUT_MODEL_PATH="${TEST_NAME}.pte" + STR_LEN=${#OUTPUT_MODEL_PATH} + FINAL_OUTPUT_MODEL_PATH=${OUTPUT_MODEL_PATH:5:$STR_LEN-5} + + if [[ "${BUILD_TOOL}" == "cmake" ]]; then + if [[ ! -f ${CMAKE_OUTPUT_DIR}/examples/apple/mps/mps_executor_runner ]]; then + build_cmake_mps_executor_runner + fi + ./${CMAKE_OUTPUT_DIR}/examples/apple/mps/mps_executor_runner --model_path "${FINAL_OUTPUT_MODEL_PATH}" --bundled_program + else + echo "Invalid build tool ${BUILD_TOOL}. Only cmake is supported atm" + exit 1 + fi +} + echo "Testing ${MODEL_NAME} with MPS..." -test_model_with_mps +if [[ "${MPS_TESTS}" == false ]]; then + test_model_with_mps +else + test_mps_model +fi diff --git a/backends/apple/mps/test/test_mps.py b/backends/apple/mps/test/test_mps.py index c980df6cae1..6e6bc065fa5 100644 --- a/backends/apple/mps/test/test_mps.py +++ b/backends/apple/mps/test/test_mps.py @@ -134,7 +134,7 @@ def forward(self, *args): MethodTestSuite( method_name="forward", test_cases=[ - MethodTestCase(inputs=m_inputs, expected_outputs=model(*m_inputs)) + MethodTestCase(inputs=m_inputs, expected_outputs=m(*m_inputs)) ], ) ] diff --git a/backends/apple/mps/test/test_mps_utils.py b/backends/apple/mps/test/test_mps_utils.py index 38c05640335..f95a9dc1a39 100644 --- a/backends/apple/mps/test/test_mps_utils.py +++ b/backends/apple/mps/test/test_mps_utils.py @@ -195,7 +195,7 @@ def forward(self, *args): method_name="forward", test_cases=[ MethodTestCase( - input=sample_inputs, expected_outputs=module(*sample_inputs) + inputs=sample_inputs, expected_outputs=module(*sample_inputs) ) ], ) @@ -208,7 +208,7 @@ def forward(self, *args): bundled_program ) - filename = f"{func_name}.bpte" + filename = f"{func_name}.pte" logging.info(f"Step 5: Saving bundled program to {filename}...") with open(filename, "wb") as file: file.write(bundled_program_buffer) From d48318e248ee32d5a75e01ddb52c11c9371739e2 Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Tue, 28 Nov 2023 13:20:05 -0800 Subject: [PATCH 02/11] Update cmdline --- backends/apple/mps/ci/scripts/gather_test_models_mps.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/backends/apple/mps/ci/scripts/gather_test_models_mps.py b/backends/apple/mps/ci/scripts/gather_test_models_mps.py index d76a4a16f71..9c317e13cbf 100644 --- a/backends/apple/mps/ci/scripts/gather_test_models_mps.py +++ b/backends/apple/mps/ci/scripts/gather_test_models_mps.py @@ -66,6 +66,8 @@ def export_models_for_ci() -> None: args = parse_args() target_os = args.target_os + print(f"target os: {target_os}") + # This is the JSON syntax for configuration matrix used by GitHub # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs models = {"include": []} @@ -95,9 +97,13 @@ def export_models_for_ci() -> None: if target_os not in BUILD_TOOLS[build_tool]: continue + print(f"method: {testcase._testMethodName}") + print(f"{testcase.__class__}") + print(f"{testcase.__class__.__name__}") + cmd = ".".join([start_path, testcase.__class__.__name__, testcase._testMethodName]) record = { "build-tool": build_tool, - "model": testcase._testMethodName, + "model": cmd, "runner": DEFAULT_RUNNERS.get(target_os), } From 61bfeb266b8c8233ab87c3f7353baa4b05fb0066 Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Tue, 28 Nov 2023 13:23:42 -0800 Subject: [PATCH 03/11] Add lint for mps --- .github/workflows/lint_mps.yml | 52 ++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/lint_mps.yml diff --git a/.github/workflows/lint_mps.yml b/.github/workflows/lint_mps.yml new file mode 100644 index 00000000000..a7d511dc89b --- /dev/null +++ b/.github/workflows/lint_mps.yml @@ -0,0 +1,52 @@ +name: Lint MPS + +on: + pull_request: + push: + branches: + - main + - release/* + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }} + cancel-in-progress: true + +jobs: + lintrunner-mps: + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + runner: macos-executorch + fetch-depth: 0 + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + script: | + # The generic Linux job chooses to use base env, not the one setup by the image + CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") + conda activate "${CONDA_ENV}" + + CACHE_DIRECTORY="/tmp/.lintbin" + # Try to recover the cached binaries + if [[ -d "${CACHE_DIRECTORY}" ]]; then + # It's ok to fail this as lintrunner init would download these binaries + # again if they do not exist + cp -r "${CACHE_DIRECTORY}" . || true + fi + + # This has already been cached in the docker image + lintrunner init 2> /dev/null + + RC=0 + # Run lintrunner on all files + if ! lintrunner --force-color --all-files --tee-json=lint.json 2> /dev/null; then + echo "" + echo -e "\e[1m\e[36mYou can reproduce these results locally by using \`lintrunner\`. (If you don't get the same results, run \'lintrunner init\' to update your local linter)\e[0m" + echo -e "\e[1m\e[36mSee https://github.com/pytorch/pytorch/wiki/lintrunner for setup instructions.\e[0m" + RC=1 + fi + + # Use jq to massage the JSON lint output into GitHub Actions workflow commands. + jq --raw-output \ + '"::\(if .severity == "advice" or .severity == "disabled" then "warning" else .severity end) file=\(.path),line=\(.line),col=\(.char),title=\(.code) \(.name)::" + (.description | gsub("\\n"; "%0A"))' \ + lint.json || true + + exit $RC From 1997c13edd02f6b6c479ff9350a8bf85f142425a Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Tue, 28 Nov 2023 13:25:56 -0800 Subject: [PATCH 04/11] Update lint script --- .github/workflows/lint_mps.yml | 119 +++++++++++++++++++++++---------- 1 file changed, 83 insertions(+), 36 deletions(-) diff --git a/.github/workflows/lint_mps.yml b/.github/workflows/lint_mps.yml index a7d511dc89b..795bdefbfa9 100644 --- a/.github/workflows/lint_mps.yml +++ b/.github/workflows/lint_mps.yml @@ -14,39 +14,86 @@ concurrency: jobs: lintrunner-mps: - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - runner: macos-executorch - fetch-depth: 0 - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - script: | - # The generic Linux job chooses to use base env, not the one setup by the image - CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") - conda activate "${CONDA_ENV}" - - CACHE_DIRECTORY="/tmp/.lintbin" - # Try to recover the cached binaries - if [[ -d "${CACHE_DIRECTORY}" ]]; then - # It's ok to fail this as lintrunner init would download these binaries - # again if they do not exist - cp -r "${CACHE_DIRECTORY}" . || true - fi - - # This has already been cached in the docker image - lintrunner init 2> /dev/null - - RC=0 - # Run lintrunner on all files - if ! lintrunner --force-color --all-files --tee-json=lint.json 2> /dev/null; then - echo "" - echo -e "\e[1m\e[36mYou can reproduce these results locally by using \`lintrunner\`. (If you don't get the same results, run \'lintrunner init\' to update your local linter)\e[0m" - echo -e "\e[1m\e[36mSee https://github.com/pytorch/pytorch/wiki/lintrunner for setup instructions.\e[0m" - RC=1 - fi - - # Use jq to massage the JSON lint output into GitHub Actions workflow commands. - jq --raw-output \ - '"::\(if .severity == "advice" or .severity == "disabled" then "warning" else .severity end) file=\(.path),line=\(.line),col=\(.char),title=\(.code) \(.name)::" + (.description | gsub("\\n"; "%0A"))' \ - lint.json || true - - exit $RC + runs-on: macos-executorch + steps: + - uses: actions/checkout@v3 + with: + submodules: 'true' + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + cache: pip + - name: Extract the list of models to test + id: lint-mps + run: | + set -eux + source .ci/scripts/utils.sh + # This is a simple Python script but as it tries to import executorch.examples.models, + # it requires a whole bunch of ExecuTorch dependencies on the Docker image + install_pip_dependencies + install_executorch + CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") + conda activate "${CONDA_ENV}" + + CACHE_DIRECTORY="/tmp/.lintbin" + # Try to recover the cached binaries + if [[ -d "${CACHE_DIRECTORY}" ]]; then + # It's ok to fail this as lintrunner init would download these binaries + # again if they do not exist + cp -r "${CACHE_DIRECTORY}" . || true + fi + + # This has already been cached in the docker image + lintrunner init 2> /dev/null + + RC=0 + # Run lintrunner on all files + if ! lintrunner --force-color --all-files --tee-json=lint.json 2> /dev/null; then + echo "" + echo -e "\e[1m\e[36mYou can reproduce these results locally by using \`lintrunner\`. (If you don't get the same results, run \'lintrunner init\' to update your local linter)\e[0m" + echo -e "\e[1m\e[36mSee https://github.com/pytorch/pytorch/wiki/lintrunner for setup instructions.\e[0m" + RC=1 + fi + + # Use jq to massage the JSON lint output into GitHub Actions workflow commands. + jq --raw-output \ + '"::\(if .severity == "advice" or .severity == "disabled" then "warning" else .severity end) file=\(.path),line=\(.line),col=\(.char),title=\(.code) \(.name)::" + (.description | gsub("\\n"; "%0A"))' \ + lint.json || true + + exit $RC + + # with: + # runner: macos-executorch + # fetch-depth: 0 + # ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + # script: | + # # The generic Linux job chooses to use base env, not the one setup by the image + # CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") + # conda activate "${CONDA_ENV}" + + # CACHE_DIRECTORY="/tmp/.lintbin" + # # Try to recover the cached binaries + # if [[ -d "${CACHE_DIRECTORY}" ]]; then + # # It's ok to fail this as lintrunner init would download these binaries + # # again if they do not exist + # cp -r "${CACHE_DIRECTORY}" . || true + # fi + + # # This has already been cached in the docker image + # lintrunner init 2> /dev/null + + # RC=0 + # # Run lintrunner on all files + # if ! lintrunner --force-color --all-files --tee-json=lint.json 2> /dev/null; then + # echo "" + # echo -e "\e[1m\e[36mYou can reproduce these results locally by using \`lintrunner\`. (If you don't get the same results, run \'lintrunner init\' to update your local linter)\e[0m" + # echo -e "\e[1m\e[36mSee https://github.com/pytorch/pytorch/wiki/lintrunner for setup instructions.\e[0m" + # RC=1 + # fi + + # # Use jq to massage the JSON lint output into GitHub Actions workflow commands. + # jq --raw-output \ + # '"::\(if .severity == "advice" or .severity == "disabled" then "warning" else .severity end) file=\(.path),line=\(.line),col=\(.char),title=\(.code) \(.name)::" + (.description | gsub("\\n"; "%0A"))' \ + # lint.json || true + + # exit $RC From 7d8c84dd3907e6bf067ab946e80ae7431603a393 Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Tue, 28 Nov 2023 13:32:31 -0800 Subject: [PATCH 05/11] Update lint script --- .github/workflows/lint_mps.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint_mps.yml b/.github/workflows/lint_mps.yml index 795bdefbfa9..80c90e28b8a 100644 --- a/.github/workflows/lint_mps.yml +++ b/.github/workflows/lint_mps.yml @@ -32,8 +32,13 @@ jobs: # it requires a whole bunch of ExecuTorch dependencies on the Docker image install_pip_dependencies install_executorch - CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") - conda activate "${CONDA_ENV}" + brew install jq + + source .ci/scripts/utils.sh + + # Install lint depdendencies + pip install lintrunner==0.11.0 + pip install lintrunner-adapters==0.11.0 CACHE_DIRECTORY="/tmp/.lintbin" # Try to recover the cached binaries From 52f995c0ce45448dbb8b98ff8e2220bf64fd8872 Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Tue, 28 Nov 2023 13:56:47 -0800 Subject: [PATCH 06/11] Fix lint --- .github/workflows/pull_mps.yml | 8 ++++---- .../mps/ci/scripts/gather_test_models_mps.py | 20 ++++++++++++------- backends/apple/mps/test/test_mps.py | 4 +--- examples/apple/mps/CMakeLists.txt | 11 +++++++--- 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/.github/workflows/pull_mps.yml b/.github/workflows/pull_mps.yml index 90a8782d431..e282930d09f 100644 --- a/.github/workflows/pull_mps.yml +++ b/.github/workflows/pull_mps.yml @@ -35,8 +35,8 @@ jobs: install_pip_dependencies install_executorch PYTHONPATH="${PWD}" python -m backends.apple.mps.ci.scripts.gather_test_models_mps - # test-mps-models-macos: - # name: test-mps-models-macos + # test-mps-models: + # name: test-mps-models # runs-on: macos-executorch # needs: gather-models-mps # strategy: @@ -59,8 +59,8 @@ jobs: # # Build and test ExecuTorch # PYTHON_EXECUTABLE=python bash backends/apple/mps/ci/scripts/test-mps.sh "${MODEL_NAME}" "${BUILD_TOOL}" # popd - test-mps-macos: - name: test-mps-macos + test-mps: + name: test-mps runs-on: macos-executorch needs: gather-models-mps strategy: diff --git a/backends/apple/mps/ci/scripts/gather_test_models_mps.py b/backends/apple/mps/ci/scripts/gather_test_models_mps.py index 9c317e13cbf..e7cbca84388 100644 --- a/backends/apple/mps/ci/scripts/gather_test_models_mps.py +++ b/backends/apple/mps/ci/scripts/gather_test_models_mps.py @@ -5,13 +5,12 @@ import json import os +import unittest from typing import Any from examples.models import MODEL_NAME_TO_MODEL -from examples.xnnpack import MODEL_NAME_TO_OPTIONS -import unittest -MPS_TEST_SUITE_PATH = 'backends/apple/mps/test' +MPS_TEST_SUITE_PATH = "backends/apple/mps/test" MPS_SUITE = unittest.defaultTestLoader.discover(MPS_TEST_SUITE_PATH) BUILD_TOOLS = { @@ -21,6 +20,7 @@ "macos-14": "macos-executorch", } + def parse_args() -> Any: from argparse import ArgumentParser @@ -47,18 +47,21 @@ def set_output(name: str, val: Any) -> None: else: print(f"::set-output name={name}::{val}") + def gather_mps_test_list(suite, mps_test_list): - if hasattr(suite, '__iter__'): + if hasattr(suite, "__iter__"): for x in suite: - gather_mps_test_list(x, mps_test_list) + gather_mps_test_list(x, mps_test_list) else: mps_test_list.append(suite) + def gather_mps_tests(suite): mps_test_list = [] gather_mps_test_list(suite, mps_test_list) return mps_test_list + def export_models_for_ci() -> None: """ This gathers all the example models that we want to test on GitHub OSS CI @@ -92,7 +95,7 @@ def export_models_for_ci() -> None: start_path = ".".join([MPS_TEST_SUITE_PATH.replace("/", "."), "test_mps"]) for testcase in mps_test_list: if "test_mps" not in str(testcase.__class__): - continue + continue for build_tool in BUILD_TOOLS.keys(): if target_os not in BUILD_TOOLS[build_tool]: continue @@ -100,7 +103,9 @@ def export_models_for_ci() -> None: print(f"method: {testcase._testMethodName}") print(f"{testcase.__class__}") print(f"{testcase.__class__.__name__}") - cmd = ".".join([start_path, testcase.__class__.__name__, testcase._testMethodName]) + cmd = ".".join( + [start_path, testcase.__class__.__name__, testcase._testMethodName] + ) record = { "build-tool": build_tool, "model": cmd, @@ -111,5 +116,6 @@ def export_models_for_ci() -> None: set_output("mps_models", json.dumps(mps_models)) + if __name__ == "__main__": export_models_for_ci() diff --git a/backends/apple/mps/test/test_mps.py b/backends/apple/mps/test/test_mps.py index 6e6bc065fa5..43a69eb3ef7 100644 --- a/backends/apple/mps/test/test_mps.py +++ b/backends/apple/mps/test/test_mps.py @@ -133,9 +133,7 @@ def forward(self, *args): method_test_suites = [ MethodTestSuite( method_name="forward", - test_cases=[ - MethodTestCase(inputs=m_inputs, expected_outputs=m(*m_inputs)) - ], + test_cases=[MethodTestCase(inputs=m_inputs, expected_outputs=m(*m_inputs))], ) ] diff --git a/examples/apple/mps/CMakeLists.txt b/examples/apple/mps/CMakeLists.txt index 6643178b5ed..7a29d266880 100644 --- a/examples/apple/mps/CMakeLists.txt +++ b/examples/apple/mps/CMakeLists.txt @@ -52,7 +52,8 @@ add_custom_command( OUTPUT ${_bundled_program_schema__outputs} COMMAND ${FLATC_EXECUTABLE} --cpp --cpp-std c++11 --gen-mutable --scoped-enums -o - "${_program_schema__include_dir}/executorch/sdk/bundled_program/schema" ${_bundled_program_schema__srcs} + "${_program_schema__include_dir}/executorch/sdk/bundled_program/schema" + ${_bundled_program_schema__srcs} WORKING_DIRECTORY ${EXECUTORCH_ROOT} DEPENDS ${FLATC_EXECUTABLE} ${_bundled_program_schema__srcs} COMMENT "Generating bundled_program headers" @@ -68,8 +69,12 @@ set(mps_executor_runner_libs "-framework Foundation" "-weak_framework MetalPerformanceShaders" "-weak_framework MetalPerformanceShadersGraph" "-weak_framework Metal") -list(TRANSFORM _mps_executor_runner__srcs PREPEND "${EXECUTORCH_ROOT}/") -add_executable(mps_executor_runner ${_mps_executor_runner__srcs} ${_bundled_program_schema__outputs}) +list( + TRANSFORM _mps_executor_runner__srcs PREPEND "${EXECUTORCH_ROOT}/") +add_executable( + mps_executor_runner + ${_mps_executor_runner__srcs} + ${_bundled_program_schema__outputs}) target_include_directories( mps_executor_runner INTERFACE ${CMAKE_BINARY_DIR}/schema/include/ ${EXECUTORCH_ROOT}/third-party/flatbuffers/include) From 0d89593674f770cc18b3598b1290db0da34387bc Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Tue, 28 Nov 2023 14:10:41 -0800 Subject: [PATCH 07/11] Fix lint --- .github/workflows/lint_mps.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint_mps.yml b/.github/workflows/lint_mps.yml index 80c90e28b8a..bce4c893ada 100644 --- a/.github/workflows/lint_mps.yml +++ b/.github/workflows/lint_mps.yml @@ -23,7 +23,7 @@ jobs: with: python-version: '3.10' cache: pip - - name: Extract the list of models to test + - name: Lintrunner id: lint-mps run: | set -eux @@ -53,7 +53,7 @@ jobs: RC=0 # Run lintrunner on all files - if ! lintrunner --force-color --all-files --tee-json=lint.json 2> /dev/null; then + if ! lintrunner --force-color --all-files 2> /dev/null; then echo "" echo -e "\e[1m\e[36mYou can reproduce these results locally by using \`lintrunner\`. (If you don't get the same results, run \'lintrunner init\' to update your local linter)\e[0m" echo -e "\e[1m\e[36mSee https://github.com/pytorch/pytorch/wiki/lintrunner for setup instructions.\e[0m" From 6e907795ad8917fb272e92be81e76bf03b9bbf67 Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Tue, 28 Nov 2023 14:19:59 -0800 Subject: [PATCH 08/11] Fix lint --- .github/workflows/lint_mps.yml | 45 +------------------------------ .github/workflows/pull_mps.yml | 48 +++++++++++++++++----------------- 2 files changed, 25 insertions(+), 68 deletions(-) diff --git a/.github/workflows/lint_mps.yml b/.github/workflows/lint_mps.yml index bce4c893ada..c06464c5dde 100644 --- a/.github/workflows/lint_mps.yml +++ b/.github/workflows/lint_mps.yml @@ -53,52 +53,9 @@ jobs: RC=0 # Run lintrunner on all files - if ! lintrunner --force-color --all-files 2> /dev/null; then + if ! lintrunner --force-color --all-files --tee-json lint.json 2> /dev/null; then echo "" echo -e "\e[1m\e[36mYou can reproduce these results locally by using \`lintrunner\`. (If you don't get the same results, run \'lintrunner init\' to update your local linter)\e[0m" echo -e "\e[1m\e[36mSee https://github.com/pytorch/pytorch/wiki/lintrunner for setup instructions.\e[0m" RC=1 fi - - # Use jq to massage the JSON lint output into GitHub Actions workflow commands. - jq --raw-output \ - '"::\(if .severity == "advice" or .severity == "disabled" then "warning" else .severity end) file=\(.path),line=\(.line),col=\(.char),title=\(.code) \(.name)::" + (.description | gsub("\\n"; "%0A"))' \ - lint.json || true - - exit $RC - - # with: - # runner: macos-executorch - # fetch-depth: 0 - # ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - # script: | - # # The generic Linux job chooses to use base env, not the one setup by the image - # CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") - # conda activate "${CONDA_ENV}" - - # CACHE_DIRECTORY="/tmp/.lintbin" - # # Try to recover the cached binaries - # if [[ -d "${CACHE_DIRECTORY}" ]]; then - # # It's ok to fail this as lintrunner init would download these binaries - # # again if they do not exist - # cp -r "${CACHE_DIRECTORY}" . || true - # fi - - # # This has already been cached in the docker image - # lintrunner init 2> /dev/null - - # RC=0 - # # Run lintrunner on all files - # if ! lintrunner --force-color --all-files --tee-json=lint.json 2> /dev/null; then - # echo "" - # echo -e "\e[1m\e[36mYou can reproduce these results locally by using \`lintrunner\`. (If you don't get the same results, run \'lintrunner init\' to update your local linter)\e[0m" - # echo -e "\e[1m\e[36mSee https://github.com/pytorch/pytorch/wiki/lintrunner for setup instructions.\e[0m" - # RC=1 - # fi - - # # Use jq to massage the JSON lint output into GitHub Actions workflow commands. - # jq --raw-output \ - # '"::\(if .severity == "advice" or .severity == "disabled" then "warning" else .severity end) file=\(.path),line=\(.line),col=\(.char),title=\(.code) \(.name)::" + (.description | gsub("\\n"; "%0A"))' \ - # lint.json || true - - # exit $RC diff --git a/.github/workflows/pull_mps.yml b/.github/workflows/pull_mps.yml index e282930d09f..e3cd4d2b2d6 100644 --- a/.github/workflows/pull_mps.yml +++ b/.github/workflows/pull_mps.yml @@ -35,30 +35,30 @@ jobs: install_pip_dependencies install_executorch PYTHONPATH="${PWD}" python -m backends.apple.mps.ci.scripts.gather_test_models_mps - # test-mps-models: - # name: test-mps-models - # runs-on: macos-executorch - # needs: gather-models-mps - # strategy: - # matrix: ${{ fromJSON(needs.gather-models-mps.outputs.models) }} - # fail-fast: false - # steps: - # - uses: actions/checkout@v3 - # with: - # submodules: 'true' - # - name: Run test ${{ matrix.test }} - # if: always() - # run: | - # WORKSPACE=$(pwd) - # pushd "${WORKSPACE}" - # MODEL_NAME=${{ matrix.model }} - # BUILD_TOOL=${{ matrix.build-tool }} - # # Setup MacOS dependencies as there is no Docker support on MacOS atm - # PYTHON_EXECUTABLE=python bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}" - # PYTHON_EXECUTABLE=python bash backends/apple/mps/install_requirements.sh - # # Build and test ExecuTorch - # PYTHON_EXECUTABLE=python bash backends/apple/mps/ci/scripts/test-mps.sh "${MODEL_NAME}" "${BUILD_TOOL}" - # popd + test-mps-models: + name: test-mps-models + runs-on: macos-executorch + needs: gather-models-mps + strategy: + matrix: ${{ fromJSON(needs.gather-models-mps.outputs.models) }} + fail-fast: false + steps: + - uses: actions/checkout@v3 + with: + submodules: 'true' + - name: Run test ${{ matrix.test }} + if: always() + run: | + WORKSPACE=$(pwd) + pushd "${WORKSPACE}" + MODEL_NAME=${{ matrix.model }} + BUILD_TOOL=${{ matrix.build-tool }} + # Setup MacOS dependencies as there is no Docker support on MacOS atm + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}" + PYTHON_EXECUTABLE=python bash backends/apple/mps/install_requirements.sh + # Build and test ExecuTorch + PYTHON_EXECUTABLE=python bash backends/apple/mps/ci/scripts/test-mps.sh "${MODEL_NAME}" "${BUILD_TOOL}" 0 + popd test-mps: name: test-mps runs-on: macos-executorch From 6861cf517bac9cb93a663d6e65f404e182904173 Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Tue, 28 Nov 2023 14:22:23 -0800 Subject: [PATCH 09/11] Fix lint --- backends/apple/mps/ci/scripts/gather_test_models_mps.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/backends/apple/mps/ci/scripts/gather_test_models_mps.py b/backends/apple/mps/ci/scripts/gather_test_models_mps.py index e7cbca84388..887161bdfb3 100644 --- a/backends/apple/mps/ci/scripts/gather_test_models_mps.py +++ b/backends/apple/mps/ci/scripts/gather_test_models_mps.py @@ -69,8 +69,6 @@ def export_models_for_ci() -> None: args = parse_args() target_os = args.target_os - print(f"target os: {target_os}") - # This is the JSON syntax for configuration matrix used by GitHub # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs models = {"include": []} From 95b1e930d08dd0875c294b6f459f9895354097ee Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Tue, 28 Nov 2023 14:23:28 -0800 Subject: [PATCH 10/11] Fix lint --- backends/apple/mps/ci/scripts/gather_test_models_mps.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/backends/apple/mps/ci/scripts/gather_test_models_mps.py b/backends/apple/mps/ci/scripts/gather_test_models_mps.py index 887161bdfb3..d13c8a16be2 100644 --- a/backends/apple/mps/ci/scripts/gather_test_models_mps.py +++ b/backends/apple/mps/ci/scripts/gather_test_models_mps.py @@ -98,9 +98,6 @@ def export_models_for_ci() -> None: if target_os not in BUILD_TOOLS[build_tool]: continue - print(f"method: {testcase._testMethodName}") - print(f"{testcase.__class__}") - print(f"{testcase.__class__.__name__}") cmd = ".".join( [start_path, testcase.__class__.__name__, testcase._testMethodName] ) From 02bd49dfac8ca6a5e67b45ddb623584db511ee21 Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Tue, 28 Nov 2023 14:31:05 -0800 Subject: [PATCH 11/11] Fix lint --- .github/workflows/pull_mps.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_mps.yml b/.github/workflows/pull_mps.yml index e3cd4d2b2d6..d24265f1175 100644 --- a/.github/workflows/pull_mps.yml +++ b/.github/workflows/pull_mps.yml @@ -57,7 +57,7 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}" PYTHON_EXECUTABLE=python bash backends/apple/mps/install_requirements.sh # Build and test ExecuTorch - PYTHON_EXECUTABLE=python bash backends/apple/mps/ci/scripts/test-mps.sh "${MODEL_NAME}" "${BUILD_TOOL}" 0 + PYTHON_EXECUTABLE=python bash backends/apple/mps/ci/scripts/test-mps.sh "${MODEL_NAME}" "${BUILD_TOOL}" false popd test-mps: name: test-mps @@ -81,5 +81,5 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}" PYTHON_EXECUTABLE=python bash backends/apple/mps/install_requirements.sh # Build and test ExecuTorch - PYTHON_EXECUTABLE=python bash backends/apple/mps/ci/scripts/test-mps.sh "${MODEL_NAME}" "${BUILD_TOOL}" 1 + PYTHON_EXECUTABLE=python bash backends/apple/mps/ci/scripts/test-mps.sh "${MODEL_NAME}" "${BUILD_TOOL}" true popd