From 66712f383c99cb05ce288d4d9219457e2ac81be7 Mon Sep 17 00:00:00 2001 From: George Wu Date: Thu, 9 Jan 2020 10:43:31 -0800 Subject: [PATCH 1/7] don't run cuda tests if building with tensorrt --- tools/ci_build/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 09995aa63de8a..c7aad8e0e2760 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -939,7 +939,7 @@ def main(): else: tensorrt_run_onnx_tests(build_dir, configs, "") - if args.use_cuda: + if args.use_cuda and not args.use_tensorrt: run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'cuda', args.enable_multi_device_test, False, 2) if args.use_ngraph: From ad8e02345c427cc3aa1ff2b17dcad128b61492de Mon Sep 17 00:00:00 2001 From: George Wu Date: Thu, 9 Jan 2020 12:32:45 -0800 Subject: [PATCH 2/7] remove unnecessary build options for win trt ci --- .../github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml index d5401cd13b2e4..79554ee4993a3 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml @@ -4,7 +4,7 @@ jobs: AgentPool : 'Win-GPU-CUDA10' DoDebugBuild: 'true' DoCompliance: 'false' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --enable_pybind --use_dnnl --use_tensorrt --tensorrt_home="C:\local\TensorRT-6.0.1.5" --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --gen_doc' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_wheel --use_tensorrt --tensorrt_home="C:\local\TensorRT-6.0.1.5" --enable_onnx_tests --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda"' JobName: 'Windows_CI_GPU_Dev' DoNugetPack: 'false' NuPackScript : '' From e6452cdb5d26b094542b48eff80ec626055fb6ce Mon Sep 17 00:00:00 2001 From: George Wu Date: Thu, 9 Jan 2020 18:16:52 -0800 Subject: [PATCH 3/7] refactor win gpu tensorrt ci yml --- .../win-gpu-tensorrt-ci-pipeline.yml | 107 +++++++++++++++--- 1 file changed, 92 insertions(+), 15 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml index 79554ee4993a3..a7f7f6a99babb 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml @@ -1,16 +1,93 @@ jobs: -- template: templates/win-ci.yml - parameters: - AgentPool : 'Win-GPU-CUDA10' - DoDebugBuild: 'true' - DoCompliance: 'false' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_wheel --use_tensorrt --tensorrt_home="C:\local\TensorRT-6.0.1.5" --enable_onnx_tests --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda"' - JobName: 'Windows_CI_GPU_Dev' - DoNugetPack: 'false' - NuPackScript : '' - DoTestCoverage: 'false' - BuildArch: 'amd64' - SetVcvars: 'true' - MsbuildArguments: '/m /p:CudaToolkitDir=C:\local\cuda_10.0.130_win10\' - EnvSetupScript: 'setup_env_cuda.bat' - CudaVersion: '10.0' +- job: 'build' + pool: 'Win-GPU-CUDA10' + strategy: + maxParallel: 2 + matrix: + debug: + BuildConfig: 'Debug' + release: + BuildConfig: 'RelWithDebInfo' + variables: + OrtPackageId: 'Microsoft.ML.OnnxRuntime' + MsbuildArguments: '-detailedsummary -maxcpucount -consoleloggerparameters:PerformanceSummary' + OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)' + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true + EnvSetupScript: setup_env.bat + buildArch: x64 + setVcvars: true + timeoutInMinutes: 120 + workspace: + clean: all + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + addToPath: true + architecture: $(buildArch) + + - task: BatchScript@1 + displayName: 'setup env' + inputs: + filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)' + modifyEnvironment: true + workingFolder: '$(Build.BinariesDirectory)' + + - script: | + python -m pip install -q pyopenssl setuptools wheel numpy + workingDirectory: '$(Build.BinariesDirectory)' + displayName: 'Install python modules' + + - task: PythonScript@0 + displayName: 'Generate cmake config' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --build_wheel --use_tensorrt --tensorrt_home="C:\local\TensorRT-6.0.1.5" --enable_onnx_tests --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda"' + workingDirectory: '$(Build.BinariesDirectory)' + + - task: VSBuild@1 + displayName: 'Build' + inputs: + solution: '$(Build.BinariesDirectory)\$(BuildConfig)\onnxruntime.sln' + platform: 'x64' + configuration: $(BuildConfig) + msbuildArgs: $(MsbuildArguments) + msbuildArchitecture: $(buildArch) + maximumCpuCount: true + logProjectEvents: false + workingFolder: '$(Build.BinariesDirectory)\$(BuildConfig)' + createLogFile: true + + - task: PythonScript@0 + displayName: 'Build wheel' + inputs: + scriptPath: '$(Build.SourcesDirectory)\setup.py' + arguments: 'bdist_wheel' + workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)' + + - template: templates/set-test-data-variables-step.yml + + - task: PythonScript@0 + displayName: 'Download test data' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py' + arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) + workingDirectory: $(Build.BinariesDirectory) + + - script: | + mklink /D /J $(Build.BinariesDirectory)\$(BuildConfig)\models $(Build.BinariesDirectory)\models + DIR dist\ /S /B > wheel_filename_file + set /p WHEEL_FILENAME= Date: Thu, 9 Jan 2020 19:24:15 -0800 Subject: [PATCH 4/7] --numpy_version=1.17 --- .../github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml index a7f7f6a99babb..bde57dd4c45b8 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml @@ -42,7 +42,7 @@ jobs: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --build_wheel --use_tensorrt --tensorrt_home="C:\local\TensorRT-6.0.1.5" --enable_onnx_tests --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda"' + arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --numpy_version=1.17 --build_wheel --use_tensorrt --tensorrt_home="C:\local\TensorRT-6.0.1.5" --enable_onnx_tests --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda"' workingDirectory: '$(Build.BinariesDirectory)' - task: VSBuild@1 From d6b9a78c35cdeb9fd230e6a7cd22171c80ecbc00 Mon Sep 17 00:00:00 2001 From: George Wu Date: Thu, 9 Jan 2020 19:25:54 -0800 Subject: [PATCH 5/7] update --- .../github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml index bde57dd4c45b8..b59d281edbf23 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml @@ -81,7 +81,7 @@ jobs: del wheel_filename_file python.exe -m pip install -q --upgrade %WHEEL_FILENAME% set PATH=$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig);%PATH% - python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --test --build_wheel --enable_onnx_tests --use_tensorrt --cuda_version=10.0 --tensorrt_home="C:\local\TensorRT-6.0.1.5" --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" + python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --test --numpy_version=1.17 --build_wheel --enable_onnx_tests --use_tensorrt --cuda_version=10.0 --tensorrt_home="C:\local\TensorRT-6.0.1.5" --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)' displayName: 'Run tests' From eb2ccb702ce5019b15bc56d340782e74299540b9 Mon Sep 17 00:00:00 2001 From: George Wu Date: Thu, 9 Jan 2020 19:43:56 -0800 Subject: [PATCH 6/7] update --- .../github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml index b59d281edbf23..8c5661d450322 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml @@ -42,7 +42,7 @@ jobs: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --numpy_version=1.17 --build_wheel --use_tensorrt --tensorrt_home="C:\local\TensorRT-6.0.1.5" --enable_onnx_tests --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda"' + arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --numpy_version=1.17.4 --build_wheel --use_tensorrt --tensorrt_home="C:\local\TensorRT-6.0.1.5" --enable_onnx_tests --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda"' workingDirectory: '$(Build.BinariesDirectory)' - task: VSBuild@1 @@ -81,7 +81,7 @@ jobs: del wheel_filename_file python.exe -m pip install -q --upgrade %WHEEL_FILENAME% set PATH=$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig);%PATH% - python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --test --numpy_version=1.17 --build_wheel --enable_onnx_tests --use_tensorrt --cuda_version=10.0 --tensorrt_home="C:\local\TensorRT-6.0.1.5" --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" + python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --test --numpy_version=1.17.4 --build_wheel --enable_onnx_tests --use_tensorrt --cuda_version=10.0 --tensorrt_home="C:\local\TensorRT-6.0.1.5" --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)' displayName: 'Run tests' From 5c0f3a53f96ad718537d702bedd7d83ff8175660 Mon Sep 17 00:00:00 2001 From: George Wu Date: Thu, 9 Jan 2020 20:37:28 -0800 Subject: [PATCH 7/7] azcopy and cuda path --- .../win-gpu-tensorrt-ci-pipeline.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml index 8c5661d450322..8c6db895c45ba 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml @@ -33,6 +33,19 @@ jobs: modifyEnvironment: true workingFolder: '$(Build.BinariesDirectory)' + - task: PowerShell@1 + displayName: 'Set CUDA path' + inputs: + scriptName: 'tools/ci_build/github/windows/set_cuda_path.ps1' + arguments: '-CudaMsbuildPath C:\local\cudaMsbuildIntegration-10.0.130-win10 -CudaVersion 10.0' + + - task: PowerShell@2 + displayName: 'Download AzCopy (used for download test data script)' + inputs: + targetType: 'inline' + script: | + Invoke-WebRequest -OutFile $(Build.BinariesDirectory)\azcopy.exe https://onnxruntimetestdata.blob.core.windows.net/models/azcopy.exe + - script: | python -m pip install -q pyopenssl setuptools wheel numpy workingDirectory: '$(Build.BinariesDirectory)'