From 08e8ed2979b3f84fd8beda4d1bd4b893843e5e59 Mon Sep 17 00:00:00 2001 From: Ori Levari Date: Tue, 28 Apr 2020 13:42:57 -0700 Subject: [PATCH 01/15] add test collateral coming from onnxruntime test directory for new api tests --- .../github/azure-pipelines/templates/windowsai-nuget-build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/ci_build/github/azure-pipelines/templates/windowsai-nuget-build.yml b/tools/ci_build/github/azure-pipelines/templates/windowsai-nuget-build.yml index 27401a1ed3f2b..7dbe95f4af8fc 100644 --- a/tools/ci_build/github/azure-pipelines/templates/windowsai-nuget-build.yml +++ b/tools/ci_build/github/azure-pipelines/templates/windowsai-nuget-build.yml @@ -113,6 +113,8 @@ steps: xcopy $(Build.SourcesDirectory)\winml\test\collateral\models\ModelSubdirectory $(Build.ArtifactStagingDirectory)\test_artifact\ModelSubdirectory\ /i copy $(Build.SourcesDirectory)\winml\test\collateral\images\*.png $(Build.ArtifactStagingDirectory)\test_artifact\ copy $(Build.SourcesDirectory)\winml\test\collateral\images\*.jpg $(Build.ArtifactStagingDirectory)\test_artifact\ + copy $(Build.SourcesDirectory)\onnxruntime\test\testdata\sequence_length.onnx $(Build.ArtifactStagingDirectory)\test_artifact\ + copy $(Build.SourcesDirectory)\onnxruntime\test\testdata\sequence_construct.onnx $(Build.ArtifactStagingDirectory)\test_artifact\ displayName: 'Copy WinML test collateral to artifact directory' - task: NuGetToolInstaller@0 displayName: Use Nuget 4.9 From 8abc511ef59351d1dfa339176cebc8a3a6422108 Mon Sep 17 00:00:00 2001 From: Ori Levari Date: Wed, 6 May 2020 13:01:18 -0700 Subject: [PATCH 02/15] temp commit --- winml/test/api/LearningModelSessionAPITest.cpp | 12 ++++++++++-- winml/test/api/LearningModelSessionAPITest.h | 4 ++-- winml/test/common/googleTestMacros.h | 14 ++++++++++++++ winml/test/common/taefTestMacros.h | 18 ++++++++++++++++++ winml/test/common/test.h | 6 ++++++ 5 files changed, 50 insertions(+), 4 deletions(-) diff --git a/winml/test/api/LearningModelSessionAPITest.cpp b/winml/test/api/LearningModelSessionAPITest.cpp index dc277350801fb..5becef72feefd 100644 --- a/winml/test/api/LearningModelSessionAPITest.cpp +++ b/winml/test/api/LearningModelSessionAPITest.cpp @@ -397,8 +397,8 @@ static void CloseSession() }); } -const LearningModelSesssionAPITestsApi& getapi() { - static constexpr LearningModelSesssionAPITestsApi api = +const LearningModelSessionAPITestsApi& getapi() { + static LearningModelSessionAPITestsApi api = { LearningModelSessionAPITestsClassSetup, LearningModelSessionAPITestsGpuMethodSetup, @@ -418,5 +418,13 @@ const LearningModelSesssionAPITestsApi& getapi() { EvaluateSessionAndCloseModel, CloseSession, }; + + if (SKIP_GPU_TESTS) { + api.CreateSessionDeviceDirectX = SkipTest; + api.CreateSessionDeviceDirectXHighPerformance = SkipTest; + api.CreateSessionDeviceDirectXMinimumPower = SkipTest; + api.CreateSessionWithCastToFloat16InModel = SkipTest; + api.DISABLED_CreateSessionWithFloat16InitializersInModel = SkipTest; + } return api; } diff --git a/winml/test/api/LearningModelSessionAPITest.h b/winml/test/api/LearningModelSessionAPITest.h index 02b7ad2d054b9..72112792d29af 100644 --- a/winml/test/api/LearningModelSessionAPITest.h +++ b/winml/test/api/LearningModelSessionAPITest.h @@ -3,7 +3,7 @@ #include "test.h" -struct LearningModelSesssionAPITestsApi { +struct LearningModelSessionAPITestsApi { SetupClass LearningModelSessionAPITestsClassSetup; SetupTest LearningModelSessionAPITestsGpuMethodSetup; SetupTest LearningModelSessionAPITestsGpuSkipEdgeCoreMethodSetup; @@ -22,7 +22,7 @@ struct LearningModelSesssionAPITestsApi { VoidTest EvaluateSessionAndCloseModel; VoidTest CloseSession; }; -const LearningModelSesssionAPITestsApi& getapi(); +const LearningModelSessionAPITestsApi& getapi(); WINML_TEST_CLASS_BEGIN(LearningModelSessionAPITests) WINML_TEST_CLASS_SETUP_CLASS(LearningModelSessionAPITestsClassSetup) diff --git a/winml/test/common/googleTestMacros.h b/winml/test/common/googleTestMacros.h index ca66160d75fd7..921a19cf2012c 100644 --- a/winml/test/common/googleTestMacros.h +++ b/winml/test/common/googleTestMacros.h @@ -67,6 +67,7 @@ #define INSTANTIATE_TEST_SUITE_P INSTANTIATE_TEST_CASE_P #endif + #define WINML_SKIP_TEST(message) \ WINML_SUPRESS_UNREACHABLE_BELOW(GTEST_SKIP() << message) @@ -107,3 +108,16 @@ WINML_SKIP_TEST("Test can't be run in EdgeCore"); \ } \ } while (0) + + +#define RUNTIME_PARAMETER_EXISTS(param) \ + auto no_gpu_tests = RuntimeParameters::Parameters.find(param); \ + no_gpu_tests != RuntimeParameters::Parameters.end() && no_gpu_tests->second != "0" + +#ifndef USE_DML +#define SKIP_GPU_TESTS\ + true +#else +#define SKIP_GPU_TESTS \ + RUNTIME_PARAMETER_EXISTS("noGPUtests") +#endif \ No newline at end of file diff --git a/winml/test/common/taefTestMacros.h b/winml/test/common/taefTestMacros.h index c366592007851..f91fc721beada 100644 --- a/winml/test/common/taefTestMacros.h +++ b/winml/test/common/taefTestMacros.h @@ -84,6 +84,24 @@ using namespace WEX::TestExecution; !SUCCEEDED(RuntimeParameters::TryGetValue(L"noGPUtests", _no_gpu_tests)) || !_no_gpu_tests #endif +#define RUNTIME_PARAMETER_EXISTS(param) \ + bool param_value; \ + SUCCEEDED(RuntimeParameters::TryGetValue(param, param_value)) && param_value; + +#ifndef USE_DML +#define SKIP_GPU_TESTS(isSkipped, skipReason) \ + isSkipped = true + skipReason = "GPU tests disabled because this is a WinML only build (no DML)" +#else +#define SKIP_GPU_TESTS(isSkipped, skipReason) \ + if (RUNTIME_PARAMETER_EXISTS(L"noGPUtests")) {\ + isSkipped = true; \ + skipReason = "This test is disabled by the no_gpu_tests runtime parameter." \ + } +#endif + + + #define SKIP_EDGECORE \ do { \ bool is_edge_core; \ diff --git a/winml/test/common/test.h b/winml/test/common/test.h index e30736b0f7cc1..8d73710b8f0ad 100644 --- a/winml/test/common/test.h +++ b/winml/test/common/test.h @@ -25,3 +25,9 @@ constexpr bool alwaysFalse() { #include "taefTestMacros.h" #endif #endif + +#if defined(BUILD_GOOGLE_TEST) || defined(BUILD_TAEF_TEST) +static void SkipTest() { + WINML_SKIP_TEST("") +} +#endif \ No newline at end of file From 4ae9a0cdfd636c8c0d5dfa71424c56c69fad84b8 Mon Sep 17 00:00:00 2001 From: Ori Levari Date: Wed, 6 May 2020 15:14:19 -0700 Subject: [PATCH 03/15] api tests converted to new way of test skipping and confirmed to work for googletest scenario --- winml/test/api/LearningModelAPITest.cpp | 11 ++++---- winml/test/api/LearningModelAPITest.h | 9 +------ .../test/api/LearningModelBindingAPITest.cpp | 16 +++++++----- winml/test/api/LearningModelBindingAPITest.h | 19 +++++--------- .../test/api/LearningModelSessionAPITest.cpp | 15 +++-------- winml/test/api/LearningModelSessionAPITest.h | 26 +++++-------------- winml/test/api/RawApiTestsGpu.cpp | 17 +++++++----- winml/test/api/RawApiTestsGpu.h | 2 -- 8 files changed, 43 insertions(+), 72 deletions(-) diff --git a/winml/test/api/LearningModelAPITest.cpp b/winml/test/api/LearningModelAPITest.cpp index d9a514e4be581..9e8d8e8ca16a7 100644 --- a/winml/test/api/LearningModelAPITest.cpp +++ b/winml/test/api/LearningModelAPITest.cpp @@ -17,10 +17,6 @@ static void LearningModelAPITestsClassSetup() { init_apartment(); } -static void LearningModelAPITestsGpuMethodSetup() { - GPUTEST; -} - static void CreateModelFromFilePath() { LearningModel learningModel = nullptr; WINML_EXPECT_NO_THROW(APITest::LoadModel(L"squeezenet_modifiedforruntimestests.onnx", learningModel)); @@ -269,10 +265,9 @@ static void CheckMetadataCaseInsensitive() { } const LearningModelApiTestsApi& getapi() { - static constexpr LearningModelApiTestsApi api = + static LearningModelApiTestsApi api = { LearningModelAPITestsClassSetup, - LearningModelAPITestsGpuMethodSetup, CreateModelFromFilePath, CreateModelFileNotFound, CreateModelFromIStorage, @@ -290,5 +285,9 @@ const LearningModelApiTestsApi& getapi() { CloseModelNoNewSessions, CheckMetadataCaseInsensitive }; + + if (SKIP_GPU_TESTS) { + api.CloseModelCheckEval = SkipTest; + } return api; } \ No newline at end of file diff --git a/winml/test/api/LearningModelAPITest.h b/winml/test/api/LearningModelAPITest.h index cd836246201f2..fafc1d9965b53 100644 --- a/winml/test/api/LearningModelAPITest.h +++ b/winml/test/api/LearningModelAPITest.h @@ -5,7 +5,6 @@ struct LearningModelApiTestsApi { SetupClass LearningModelAPITestsClassSetup; - SetupTest LearningModelAPITestsGpuMethodSetup; VoidTest CreateModelFromFilePath; VoidTest CreateModelFileNotFound; VoidTest CreateModelFromIStorage; @@ -42,12 +41,6 @@ WINML_TEST(LearningModelAPITests, EnumerateInputs) WINML_TEST(LearningModelAPITests, EnumerateOutputs) WINML_TEST(LearningModelAPITests, CloseModelCheckMetadata) WINML_TEST(LearningModelAPITests, CloseModelNoNewSessions) +WINML_TEST(LearningModelAPITests, CloseModelCheckEval) WINML_TEST(LearningModelAPITests, CheckMetadataCaseInsensitive) -WINML_TEST_CLASS_END() - -WINML_TEST_CLASS_BEGIN(LearningModelAPITestsGpu) -WINML_TEST_CLASS_SETUP_CLASS(LearningModelAPITestsClassSetup) -WINML_TEST_CLASS_SETUP_METHOD(LearningModelAPITestsGpuMethodSetup) -WINML_TEST_CLASS_BEGIN_TESTS -WINML_TEST(LearningModelAPITestsGpu, CloseModelCheckEval) WINML_TEST_CLASS_END() \ No newline at end of file diff --git a/winml/test/api/LearningModelBindingAPITest.cpp b/winml/test/api/LearningModelBindingAPITest.cpp index b1a345a8c6ca2..79305a652c028 100644 --- a/winml/test/api/LearningModelBindingAPITest.cpp +++ b/winml/test/api/LearningModelBindingAPITest.cpp @@ -20,10 +20,6 @@ static void LearningModelBindingAPITestsClassSetup() { init_apartment(); } -static void LearningModelBindingAPITestsGpuMethodSetup() { - GPUTEST; -} - static void CpuSqueezeNet() { std::string cpuInstance("CPU"); @@ -712,10 +708,9 @@ static void SequenceConstructTensorString() } const LearningModelBindingAPITestsApi& getapi() { - static constexpr LearningModelBindingAPITestsApi api = + static LearningModelBindingAPITestsApi api = { LearningModelBindingAPITestsClassSetup, - LearningModelBindingAPITestsGpuMethodSetup, CpuSqueezeNet, CpuSqueezeNetEmptyOutputs, CpuSqueezeNetUnboundOutputs, @@ -737,5 +732,14 @@ const LearningModelBindingAPITestsApi& getapi() { SequenceLengthTensorFloat, SequenceConstructTensorString }; + + if (SKIP_GPU_TESTS) { + api.GpuSqueezeNet = SkipTest; + api.GpuSqueezeNetEmptyOutputs = SkipTest; + api.GpuSqueezeNetUnboundOutputs = SkipTest; + api.ImageBindingDimensions = SkipTest; + api.VerifyInvalidBindExceptions = SkipTest; + api.BindInvalidInputName = SkipTest; + } return api; } diff --git a/winml/test/api/LearningModelBindingAPITest.h b/winml/test/api/LearningModelBindingAPITest.h index b752c37065e6b..f3ae26dab47da 100644 --- a/winml/test/api/LearningModelBindingAPITest.h +++ b/winml/test/api/LearningModelBindingAPITest.h @@ -5,7 +5,6 @@ struct LearningModelBindingAPITestsApi { SetupClass LearningModelBindingAPITestsClassSetup; - SetupTest LearningModelBindingAPITestsGpuMethodSetup; VoidTest CpuSqueezeNet; VoidTest CpuSqueezeNetEmptyOutputs; VoidTest CpuSqueezeNetUnboundOutputs; @@ -46,16 +45,10 @@ WINML_TEST(LearningModelBindingAPITests, VerifyOutputAfterEvaluateAsyncCalledTwi WINML_TEST(LearningModelBindingAPITests, VerifyOutputAfterImageBindCalledTwice) WINML_TEST(LearningModelBindingAPITests, SequenceLengthTensorFloat) WINML_TEST(LearningModelBindingAPITests, SequenceConstructTensorString) -WINML_TEST_CLASS_END() - -WINML_TEST_CLASS_BEGIN(LearningModelBindingAPITestsGPU) -WINML_TEST_CLASS_SETUP_CLASS(LearningModelBindingAPITestsClassSetup) -WINML_TEST_CLASS_SETUP_METHOD(LearningModelBindingAPITestsGpuMethodSetup) -WINML_TEST_CLASS_BEGIN_TESTS -WINML_TEST(LearningModelBindingAPITestsGPU, GpuSqueezeNet) -WINML_TEST(LearningModelBindingAPITestsGPU, GpuSqueezeNetEmptyOutputs) -WINML_TEST(LearningModelBindingAPITestsGPU, GpuSqueezeNetUnboundOutputs) -WINML_TEST(LearningModelBindingAPITestsGPU, ImageBindingDimensions) -WINML_TEST(LearningModelBindingAPITestsGPU, VerifyInvalidBindExceptions) -WINML_TEST(LearningModelBindingAPITestsGPU, BindInvalidInputName) +WINML_TEST(LearningModelBindingAPITests, GpuSqueezeNet) +WINML_TEST(LearningModelBindingAPITests, GpuSqueezeNetEmptyOutputs) +WINML_TEST(LearningModelBindingAPITests, GpuSqueezeNetUnboundOutputs) +WINML_TEST(LearningModelBindingAPITests, ImageBindingDimensions) +WINML_TEST(LearningModelBindingAPITests, VerifyInvalidBindExceptions) +WINML_TEST(LearningModelBindingAPITests, BindInvalidInputName) WINML_TEST_CLASS_END() \ No newline at end of file diff --git a/winml/test/api/LearningModelSessionAPITest.cpp b/winml/test/api/LearningModelSessionAPITest.cpp index 5becef72feefd..97bef1505093b 100644 --- a/winml/test/api/LearningModelSessionAPITest.cpp +++ b/winml/test/api/LearningModelSessionAPITest.cpp @@ -23,15 +23,6 @@ static void LearningModelSessionAPITestsClassSetup() { init_apartment(); } -static void LearningModelSessionAPITestsGpuMethodSetup() { - GPUTEST; -} - -static void LearningModelSessionAPITestsGpuSkipEdgeCoreMethodSetup() { - LearningModelSessionAPITestsGpuMethodSetup(); - SKIP_EDGECORE; -} - static void CreateSessionDeviceDefault() { LearningModel learningModel = nullptr; @@ -401,8 +392,6 @@ const LearningModelSessionAPITestsApi& getapi() { static LearningModelSessionAPITestsApi api = { LearningModelSessionAPITestsClassSetup, - LearningModelSessionAPITestsGpuMethodSetup, - LearningModelSessionAPITestsGpuSkipEdgeCoreMethodSetup, CreateSessionDeviceDefault, CreateSessionDeviceCpu, CreateSessionWithModelLoadedFromStream, @@ -425,6 +414,10 @@ const LearningModelSessionAPITestsApi& getapi() { api.CreateSessionDeviceDirectXMinimumPower = SkipTest; api.CreateSessionWithCastToFloat16InModel = SkipTest; api.DISABLED_CreateSessionWithFloat16InitializersInModel = SkipTest; + api.AdapterIdAndDevice = SkipTest; + } + if (RUNTIME_PARAMETER_EXISTS("EdgeCore")) { + api.AdapterIdAndDevice = SkipTest; } return api; } diff --git a/winml/test/api/LearningModelSessionAPITest.h b/winml/test/api/LearningModelSessionAPITest.h index 72112792d29af..d8c51c9475a0e 100644 --- a/winml/test/api/LearningModelSessionAPITest.h +++ b/winml/test/api/LearningModelSessionAPITest.h @@ -5,8 +5,6 @@ struct LearningModelSessionAPITestsApi { SetupClass LearningModelSessionAPITestsClassSetup; - SetupTest LearningModelSessionAPITestsGpuMethodSetup; - SetupTest LearningModelSessionAPITestsGpuSkipEdgeCoreMethodSetup; VoidTest CreateSessionDeviceDefault; VoidTest CreateSessionDeviceCpu; VoidTest CreateSessionWithModelLoadedFromStream; @@ -34,22 +32,10 @@ WINML_TEST(LearningModelSessionAPITests,EvaluateFeatures) WINML_TEST(LearningModelSessionAPITests,EvaluateFeaturesAsync) WINML_TEST(LearningModelSessionAPITests,EvaluationProperties) WINML_TEST(LearningModelSessionAPITests,EvaluateSessionAndCloseModel) +WINML_TEST(LearningModelSessionAPITests, CreateSessionDeviceDirectX) +WINML_TEST(LearningModelSessionAPITests, CreateSessionDeviceDirectXHighPerformance) +WINML_TEST(LearningModelSessionAPITests, CreateSessionDeviceDirectXMinimumPower) +WINML_TEST(LearningModelSessionAPITests, CreateSessionWithCastToFloat16InModel) +WINML_TEST(LearningModelSessionAPITests, DISABLED_CreateSessionWithFloat16InitializersInModel) +WINML_TEST(LearningModelSessionAPITests, AdapterIdAndDevice) WINML_TEST_CLASS_END() - -WINML_TEST_CLASS_BEGIN(LearningModelSessionAPITestsGpu) -WINML_TEST_CLASS_SETUP_CLASS(LearningModelSessionAPITestsClassSetup) -WINML_TEST_CLASS_SETUP_METHOD(LearningModelSessionAPITestsGpuMethodSetup) -WINML_TEST_CLASS_BEGIN_TESTS -WINML_TEST(LearningModelSessionAPITestsGpu, CreateSessionDeviceDirectX) -WINML_TEST(LearningModelSessionAPITestsGpu, CreateSessionDeviceDirectXHighPerformance) -WINML_TEST(LearningModelSessionAPITestsGpu, CreateSessionDeviceDirectXMinimumPower) -WINML_TEST(LearningModelSessionAPITestsGpu, CreateSessionWithCastToFloat16InModel) -WINML_TEST(LearningModelSessionAPITestsGpu, DISABLED_CreateSessionWithFloat16InitializersInModel) -WINML_TEST_CLASS_END() - -WINML_TEST_CLASS_BEGIN(LearningModelSessionAPITestsGpuSkipEdgeCore) -WINML_TEST_CLASS_SETUP_CLASS(LearningModelSessionAPITestsClassSetup) -WINML_TEST_CLASS_SETUP_METHOD(LearningModelSessionAPITestsGpuSkipEdgeCoreMethodSetup) -WINML_TEST_CLASS_BEGIN_TESTS -WINML_TEST(LearningModelSessionAPITestsGpuSkipEdgeCore, AdapterIdAndDevice) -WINML_TEST_CLASS_END() \ No newline at end of file diff --git a/winml/test/api/RawApiTestsGpu.cpp b/winml/test/api/RawApiTestsGpu.cpp index 63d9e1e66f37c..fb22affbd278d 100644 --- a/winml/test/api/RawApiTestsGpu.cpp +++ b/winml/test/api/RawApiTestsGpu.cpp @@ -104,10 +104,6 @@ static void RawApiTestsGpuApiTestsClassSetup() { RoInitialize(RO_INIT_TYPE::RO_INIT_SINGLETHREADED); } -static void GpuMethodSetup() { - GPUTEST; -} - static void CreateDirectXDevice() { WINML_EXPECT_NO_THROW(CreateDevice(DeviceType::DirectX)); } @@ -155,9 +151,8 @@ static void EvaluateNoInputCopy() { } const RawApiTestsGpuApi& getapi() { - static constexpr RawApiTestsGpuApi api = { + static RawApiTestsGpuApi api = { RawApiTestsGpuApiTestsClassSetup, - GpuMethodSetup, CreateDirectXDevice, CreateD3D11DeviceDevice, CreateD3D12CommandQueueDevice, @@ -166,5 +161,15 @@ const RawApiTestsGpuApi& getapi() { Evaluate, EvaluateNoInputCopy }; + + if (SKIP_GPU_TESTS) { + api.CreateDirectXDevice = SkipTest; + api.CreateD3D11DeviceDevice = SkipTest; + api.CreateD3D12CommandQueueDevice = SkipTest; + api.CreateDirectXHighPerformanceDevice = SkipTest; + api.CreateDirectXMinPowerDevice = SkipTest; + api.Evaluate = SkipTest; + api.EvaluateNoInputCopy = SkipTest; + } return api; } \ No newline at end of file diff --git a/winml/test/api/RawApiTestsGpu.h b/winml/test/api/RawApiTestsGpu.h index fcf725e397655..c02025325f8ba 100644 --- a/winml/test/api/RawApiTestsGpu.h +++ b/winml/test/api/RawApiTestsGpu.h @@ -6,7 +6,6 @@ struct RawApiTestsGpuApi { SetupClass RawApiTestsGpuApiTestsClassSetup; - SetupTest MethodSetup; VoidTest CreateDirectXDevice; VoidTest CreateD3D11DeviceDevice; VoidTest CreateD3D12CommandQueueDevice; @@ -20,7 +19,6 @@ const RawApiTestsGpuApi& getapi(); WINML_TEST_CLASS_BEGIN(RawApiTestsGpu) WINML_TEST_CLASS_SETUP_CLASS(RawApiTestsGpuApiTestsClassSetup) -WINML_TEST_CLASS_SETUP_METHOD(MethodSetup) WINML_TEST_CLASS_BEGIN_TESTS WINML_TEST(RawApiTestsGpu, CreateDirectXDevice) WINML_TEST(RawApiTestsGpu, CreateD3D11DeviceDevice) From bda39566ca874f7d8120d68c8be4e215b3c9fb53 Mon Sep 17 00:00:00 2001 From: Ori Levari Date: Wed, 6 May 2020 18:08:26 -0700 Subject: [PATCH 04/15] all tests use the proper test filtering mechanisms --- winml/test/adapter/AdapterSessionTest.cpp | 18 +-- winml/test/adapter/AdapterSessionTest.h | 4 +- winml/test/common/googleTestMacros.h | 41 ++---- winml/test/common/taefTestMacros.h | 45 ++----- winml/test/concurrency/ConcurrencyTests.cpp | 20 +-- winml/test/concurrency/ConcurrencyTests.h | 11 +- winml/test/scenario/cppwinrt/CustomOps.cpp | 15 +-- winml/test/scenario/cppwinrt/CustomOps.h | 9 +- .../cppwinrt/scenariotestscppwinrt.cpp | 120 ++++++++++-------- .../scenario/cppwinrt/scenariotestscppwinrt.h | 67 ++++------ 10 files changed, 143 insertions(+), 207 deletions(-) diff --git a/winml/test/adapter/AdapterSessionTest.cpp b/winml/test/adapter/AdapterSessionTest.cpp index 0244902893c02..0099fea3027b0 100644 --- a/winml/test/adapter/AdapterSessionTest.cpp +++ b/winml/test/adapter/AdapterSessionTest.cpp @@ -84,7 +84,6 @@ UniqueOrtSession CreateUniqueOrtSession(const std::wstring& model_path, const Un } void AppendExecutionProvider_DML() { - GPUTEST; const auto session_options = CreateUniqueOrtSessionOptions(); const auto device = CreateD3DDevice(); @@ -107,7 +106,6 @@ void GetExecutionProvider() { } void GetExecutionProvider_DML() { - GPUTEST; const auto session_options = CreateUniqueOrtSessionOptions(); THROW_IF_NOT_OK_MSG(ort_api->DisableMemPattern(session_options.get()), ort_api); const auto device = CreateD3DDevice(); @@ -130,7 +128,6 @@ void RegisterGraphTransformers() { } void RegisterGraphTransformers_DML() { - GPUTEST; const auto session_options = CreateUniqueOrtSessionOptions(); auto session = CreateUniqueOrtSession(session_options); winml_adapter_api->SessionRegisterGraphTransformers(session.get()); @@ -147,7 +144,6 @@ void RegisterCustomRegistry() { } void RegisterCustomRegistry_DML() { - GPUTEST; IMLOperatorRegistry* registry; THROW_IF_NOT_OK_MSG(winml_adapter_api->CreateCustomRegistry(®istry), ort_api); WINML_EXPECT_NOT_EQUAL(nullptr, registry); @@ -250,8 +246,6 @@ void CopyInputAcrossDevices() { } void CopyInputAcrossDevices_DML() { - GPUTEST; - const auto session_options = CreateUniqueOrtSessionOptions(); THROW_IF_NOT_OK_MSG(ort_api->DisableMemPattern(session_options.get()), ort_api); const auto device = CreateD3DDevice(); @@ -287,8 +281,8 @@ void CopyInputAcrossDevices_DML() { } } -const AdapterSessionTestAPi& getapi() { - static constexpr AdapterSessionTestAPi api = +const AdapterSessionTestAPI& getapi() { + static AdapterSessionTestAPI api = { AdapterSessionTestSetup, AdapterSessionTestTeardown, @@ -307,5 +301,13 @@ const AdapterSessionTestAPi& getapi() { CopyInputAcrossDevices, CopyInputAcrossDevices_DML }; + + if (SKIP_GPU_TESTS) { + api.AppendExecutionProvider_DML = SkipTest; + api.GetExecutionProvider_DML = SkipTest; + api.RegisterGraphTransformers_DML = SkipTest; + api.RegisterCustomRegistry_DML = SkipTest; + api.CopyInputAcrossDevices_DML = SkipTest; + } return api; } diff --git a/winml/test/adapter/AdapterSessionTest.h b/winml/test/adapter/AdapterSessionTest.h index 221a98020380c..319e4915d2e50 100644 --- a/winml/test/adapter/AdapterSessionTest.h +++ b/winml/test/adapter/AdapterSessionTest.h @@ -2,7 +2,7 @@ // Licensed under the MIT License. #include "test.h" -struct AdapterSessionTestAPi +struct AdapterSessionTestAPI { SetupClass AdapterSessionTestSetup; TeardownClass AdapterSessionTestTeardown; @@ -21,7 +21,7 @@ struct AdapterSessionTestAPi VoidTest CopyInputAcrossDevices; VoidTest CopyInputAcrossDevices_DML; }; -const AdapterSessionTestAPi& getapi(); +const AdapterSessionTestAPI& getapi(); WINML_TEST_CLASS_BEGIN(AdapterSessionTest) WINML_TEST_CLASS_SETUP_CLASS(AdapterSessionTestSetup) diff --git a/winml/test/common/googleTestMacros.h b/winml/test/common/googleTestMacros.h index 921a19cf2012c..f96addb63725b 100644 --- a/winml/test/common/googleTestMacros.h +++ b/winml/test/common/googleTestMacros.h @@ -85,39 +85,22 @@ #define WINML_EXPECT_HRESULT_FAILED(hresult_expression) EXPECT_HRESULT_FAILED(hresult_expression) #define WINML_EXPECT_THROW_SPECIFIC(statement, exception, condition) EXPECT_THROW_SPECIFIC(statement, exception, condition) -#ifndef USE_DML -#define GPUTEST \ - WINML_SKIP_TEST("GPU tests disabled because this is a WinML only build (no DML)") -#define GPUTEST_ENABLED alwaysFalse() -#else -#define GPUTEST \ - do { \ - if (auto no_gpu_tests = RuntimeParameters::Parameters.find("noGPUtests"); \ - no_gpu_tests != RuntimeParameters::Parameters.end() && no_gpu_tests->second != "0") { \ - WINML_SKIP_TEST("GPU tests disabled"); \ - } \ - } while (0) -#define GPUTEST_ENABLED auto _no_gpu_tests = RuntimeParameters::Parameters.find("noGPUtests"); \ - _no_gpu_tests == RuntimeParameters::Parameters.end() || _no_gpu_tests->second == "0" -#endif - -#define SKIP_EDGECORE \ - do { \ - if (auto is_edge_core = RuntimeParameters::Parameters.find("EdgeCore"); \ - is_edge_core != RuntimeParameters::Parameters.end() && is_edge_core->second != "0") { \ - WINML_SKIP_TEST("Test can't be run in EdgeCore"); \ - } \ - } while (0) - -#define RUNTIME_PARAMETER_EXISTS(param) \ - auto no_gpu_tests = RuntimeParameters::Parameters.find(param); \ +#define RUNTIME_PARAMETER_EXISTS(param) \ + auto no_gpu_tests = RuntimeParameters::Parameters.find(param); \ no_gpu_tests != RuntimeParameters::Parameters.end() && no_gpu_tests->second != "0" #ifndef USE_DML -#define SKIP_GPU_TESTS\ +#define SKIP_GPU_TESTS \ true #else -#define SKIP_GPU_TESTS \ +#define SKIP_GPU_TESTS \ RUNTIME_PARAMETER_EXISTS("noGPUtests") -#endif \ No newline at end of file +#endif + + +#define GPUTEST \ + if (SKIP_GPU_TESTS) { \ + WINML_SKIP_TEST("Gpu tests disabled"); \ + } + diff --git a/winml/test/common/taefTestMacros.h b/winml/test/common/taefTestMacros.h index f91fc721beada..de661f520086f 100644 --- a/winml/test/common/taefTestMacros.h +++ b/winml/test/common/taefTestMacros.h @@ -67,46 +67,19 @@ using namespace WEX::TestExecution; #define WINML_EXPECT_THROW_SPECIFIC(statement, exception, condition) VERIFY_THROWS_SPECIFIC(statement, exception, condition) #define WINML_EXPECT_HRESULT_FAILED(hresult_expression) VERIFY_FAILED(hresult_expression) -#ifndef USE_DML -#define GPUTEST \ - WINML_SKIP_TEST("GPU tests disabled because this is a WinML only build (no DML)") -#define GPUTEST_ENABLED alwaysFalse() -#else -#define GPUTEST \ - do { \ - bool no_gpu_tests; \ - if (SUCCEEDED(RuntimeParameters::TryGetValue(L"noGPUtests", no_gpu_tests)) && no_gpu_tests) { \ - WINML_SKIP_TEST("This test is disabled by the no_gpu_tests runtime parameter."); \ - return; \ - } \ - } while (0) -#define GPUTEST_ENABLED bool _no_gpu_tests; \ - !SUCCEEDED(RuntimeParameters::TryGetValue(L"noGPUtests", _no_gpu_tests)) || !_no_gpu_tests -#endif - #define RUNTIME_PARAMETER_EXISTS(param) \ - bool param_value; \ + bool param_value; \ SUCCEEDED(RuntimeParameters::TryGetValue(param, param_value)) && param_value; #ifndef USE_DML -#define SKIP_GPU_TESTS(isSkipped, skipReason) \ - isSkipped = true - skipReason = "GPU tests disabled because this is a WinML only build (no DML)" +#define SKIP_GPU_TESTS \ + true #else -#define SKIP_GPU_TESTS(isSkipped, skipReason) \ - if (RUNTIME_PARAMETER_EXISTS(L"noGPUtests")) {\ - isSkipped = true; \ - skipReason = "This test is disabled by the no_gpu_tests runtime parameter." \ - } +#define SKIP_GPU_TESTS \ + RUNTIME_PARAMETER_EXISTS(L"noGPUtests") #endif - - -#define SKIP_EDGECORE \ - do { \ - bool is_edge_core; \ - if (SUCCEEDED(RuntimeParameters::TryGetValue(L"EdgeCore", is_edge_core)) && is_edge_core) { \ - WINML_SKIP_TEST("This test is disabled by the EdgeCore runtime parameter."); \ - return; \ - } \ - } while (0) +#define GPUTEST \ + if (SKIP_GPU_TESTS) { \ + WINML_SKIP_TEST("Gpu tests disabled"); \ + } \ No newline at end of file diff --git a/winml/test/concurrency/ConcurrencyTests.cpp b/winml/test/concurrency/ConcurrencyTests.cpp index 623224af0e806..39b5eec17d532 100644 --- a/winml/test/concurrency/ConcurrencyTests.cpp +++ b/winml/test/concurrency/ConcurrencyTests.cpp @@ -25,10 +25,10 @@ void LoadBindEvalSqueezenetRealDataWithValidationConcurrently() { for (const auto& instance : {"1", "2", "3", "4"}) { threads.emplace_back(load_test_model, instance, LearningModelDeviceKind::Cpu); } - if (GPUTEST_ENABLED) { - for (const auto& instance : {"GPU_1", "GPU_2", "GPU_3", "GPU_4"}) { - threads.emplace_back(load_test_model, instance, LearningModelDeviceKind::DirectX); - } + if (SKIP_GPU_TESTS) {} else { + for (const auto& instance : {"GPU_1", "GPU_2", "GPU_3", "GPU_4"}) { + threads.emplace_back(load_test_model, instance, LearningModelDeviceKind::DirectX); + } } for (auto& thread : threads) { @@ -41,10 +41,6 @@ void ConcurrencyTestsClassSetup() { std::srand(static_cast(std::time(nullptr))); } -void ConcurrencyTestsGpuMethodSetup() { - GPUTEST; -} - struct EvaluationUnit { LearningModel model; LearningModelSession session; @@ -330,9 +326,8 @@ void MultiThreadSingleSessionGpu() { } const ConcurrencyTestsApi& getapi() { - static constexpr ConcurrencyTestsApi api = { + static ConcurrencyTestsApi api = { ConcurrencyTestsClassSetup, - ConcurrencyTestsGpuMethodSetup, LoadBindEvalSqueezenetRealDataWithValidationConcurrently, MultiThreadLoadModel, MultiThreadMultiSession, @@ -343,5 +338,10 @@ const ConcurrencyTestsApi& getapi() { EvalAsyncDifferentSessions, EvalAsyncDifferentBindings }; + + if (SKIP_GPU_TESTS) { + api.MultiThreadMultiSessionGpu = SkipTest; + api.MultiThreadSingleSessionGpu = SkipTest; + } return api; } diff --git a/winml/test/concurrency/ConcurrencyTests.h b/winml/test/concurrency/ConcurrencyTests.h index 0fa8e4a45fd2c..045b96295e8c2 100644 --- a/winml/test/concurrency/ConcurrencyTests.h +++ b/winml/test/concurrency/ConcurrencyTests.h @@ -7,7 +7,6 @@ struct ConcurrencyTestsApi { SetupClass ConcurrencyTestsClassSetup; - SetupTest ConcurrencyTestsGpuMethodSetup; VoidTest LoadBindEvalSqueezenetRealDataWithValidationConcurrently; VoidTest MultiThreadLoadModel; VoidTest MultiThreadMultiSession; @@ -30,14 +29,8 @@ WINML_TEST(ConcurrencyTests, MultiThreadSingleSession) WINML_TEST(ConcurrencyTests, EvalAsyncDifferentModels) WINML_TEST(ConcurrencyTests, EvalAsyncDifferentSessions) WINML_TEST(ConcurrencyTests, EvalAsyncDifferentBindings) -WINML_TEST_CLASS_END() - -WINML_TEST_CLASS_BEGIN(ConcurrencyTestsGpu) -WINML_TEST_CLASS_SETUP_CLASS(ConcurrencyTestsClassSetup) -WINML_TEST_CLASS_SETUP_METHOD(ConcurrencyTestsGpuMethodSetup) -WINML_TEST_CLASS_BEGIN_TESTS -WINML_TEST(ConcurrencyTestsGpu, MultiThreadMultiSessionGpu) -WINML_TEST(ConcurrencyTestsGpu, MultiThreadSingleSessionGpu) +WINML_TEST(ConcurrencyTests, MultiThreadMultiSessionGpu) +WINML_TEST(ConcurrencyTests, MultiThreadSingleSessionGpu) WINML_TEST_CLASS_END() // indices for imagenet label diff --git a/winml/test/scenario/cppwinrt/CustomOps.cpp b/winml/test/scenario/cppwinrt/CustomOps.cpp index d7e057b93a36e..f4e273fd739ac 100644 --- a/winml/test/scenario/cppwinrt/CustomOps.cpp +++ b/winml/test/scenario/cppwinrt/CustomOps.cpp @@ -33,11 +33,6 @@ static void CustomOpsScenarioTestsClassSetup() winrt::init_apartment(); } -static void CustomOpsScenarioTestsGpuMethodSetup() -{ - GPUTEST; -} - // Tests that the execution provider correctly fuses operators together when custom ops are involved. static void CustomOperatorFusion() { constexpr const wchar_t* c_modelFilename = L"squeezenet_tensor_input.onnx"; @@ -677,12 +672,16 @@ static void CustomKernelWithCustomSchema() { } const CustomOpsTestsApi& getapi() { - static constexpr CustomOpsTestsApi api = + static CustomOpsTestsApi api = { CustomOpsScenarioTestsClassSetup, - CustomOpsScenarioTestsGpuMethodSetup, CustomOperatorFusion, CustomKernelWithBuiltInSchema, - CustomKernelWithCustomSchema}; + CustomKernelWithCustomSchema + }; + + if (SKIP_GPU_TESTS) { + api.CustomOperatorFusion = SkipTest; + } return api; } \ No newline at end of file diff --git a/winml/test/scenario/cppwinrt/CustomOps.h b/winml/test/scenario/cppwinrt/CustomOps.h index 4659234f0d6d2..a117e2e9cd650 100644 --- a/winml/test/scenario/cppwinrt/CustomOps.h +++ b/winml/test/scenario/cppwinrt/CustomOps.h @@ -5,7 +5,6 @@ struct CustomOpsTestsApi { SetupTest CustomOpsScenarioTestsClassSetup; - SetupTest CustomOpsScenarioTestsGpuMethodSetup; VoidTest CustomOperatorFusion; VoidTest CustomKernelWithBuiltInSchema; VoidTest CustomKernelWithCustomSchema; @@ -17,11 +16,5 @@ WINML_TEST_CLASS_SETUP_CLASS(CustomOpsScenarioTestsClassSetup) WINML_TEST_CLASS_BEGIN_TESTS WINML_TEST(CustomOpsScenarioTests, CustomKernelWithBuiltInSchema) WINML_TEST(CustomOpsScenarioTests, CustomKernelWithCustomSchema) -WINML_TEST_CLASS_END() - -WINML_TEST_CLASS_BEGIN(CustomOpsScenarioGpuTests) -WINML_TEST_CLASS_SETUP_CLASS(CustomOpsScenarioTestsClassSetup) -WINML_TEST_CLASS_SETUP_METHOD(CustomOpsScenarioTestsGpuMethodSetup) -WINML_TEST_CLASS_BEGIN_TESTS -WINML_TEST(CustomOpsScenarioGpuTests, CustomOperatorFusion) +WINML_TEST(CustomOpsScenarioTests, CustomOperatorFusion) WINML_TEST_CLASS_END() \ No newline at end of file diff --git a/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp index 3a49b02bc283a..aef5def20e62d 100644 --- a/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp +++ b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp @@ -50,20 +50,6 @@ static void ScenarioCppWinrtTestsClassSetup() { winrt::init_apartment(); } - -static void ScenarioCppWinrtTestsGpuMethodSetup() { - GPUTEST; -}; - -static void ScenarioCppWinrtTestsSkipEdgeCoreMethodSetup() { - SKIP_EDGECORE; -}; - -static void ScenarioCppWinrtTestsGpuSkipEdgeCoreMethodSetup() { - ScenarioCppWinrtTestsGpuMethodSetup(); - SKIP_EDGECORE; -}; - static void Sample1() { LearningModel model = nullptr; std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx"; @@ -1410,46 +1396,74 @@ static void D2DInterop() { } const ScenarioTestsApi& getapi() { - static constexpr ScenarioTestsApi api = + static ScenarioTestsApi api = { - ScenarioCppWinrtTestsClassSetup, - ScenarioCppWinrtTestsGpuMethodSetup, - ScenarioCppWinrtTestsSkipEdgeCoreMethodSetup, - ScenarioCppWinrtTestsGpuSkipEdgeCoreMethodSetup, - Sample1, - Scenario1LoadBindEvalDefault, - Scenario2LoadModelFromStream, - Scenario5AsyncEval, - Scenario7EvalWithNoBind, - Scenario8SetDeviceSampleDefault, - Scenario8SetDeviceSampleCPU, - Scenario17DevDiagnostics, - Scenario22ImageBindingAsCPUTensor, - QuantizedModels, - EncryptedStream, - Scenario3SoftwareBitmapInputBinding, - Scenario6BindWithProperties, - Scenario8SetDeviceSampleDefaultDirectX, - Scenario8SetDeviceSampleMinPower, - Scenario8SetDeviceSampleMaxPerf, - Scenario8SetDeviceSampleMyCameraDevice, - Scenario8SetDeviceSampleCustomCommandQueue, - Scenario9LoadBindEvalInputTensorGPU, - Scenario13SingleModelOnCPUandGPU, - Scenario11FreeDimensionsTensor, - Scenario11FreeDimensionsImage, - Scenario14RunModelSwapchain, - Scenario20aLoadBindEvalCustomOperatorCPU, - Scenario20bLoadBindEvalReplacementCustomOperatorCPU, - Scenario21RunModel2ChainZ, - Scenario22ImageBindingAsGPUTensor, - MsftQuantizedModels, - SyncVsAsync, - CustomCommandQueueWithFence, - ReuseVideoFrame, - DeviceLostRecovery, - Scenario8SetDeviceSampleD3D11Device, - D2DInterop, + ScenarioCppWinrtTestsClassSetup, + Sample1, + Scenario1LoadBindEvalDefault, + Scenario2LoadModelFromStream, + Scenario5AsyncEval, + Scenario7EvalWithNoBind, + Scenario8SetDeviceSampleDefault, + Scenario8SetDeviceSampleCPU, + Scenario17DevDiagnostics, + Scenario22ImageBindingAsCPUTensor, + QuantizedModels, + EncryptedStream, + Scenario3SoftwareBitmapInputBinding, + Scenario6BindWithProperties, + Scenario8SetDeviceSampleDefaultDirectX, + Scenario8SetDeviceSampleMinPower, + Scenario8SetDeviceSampleMaxPerf, + Scenario8SetDeviceSampleMyCameraDevice, + Scenario8SetDeviceSampleCustomCommandQueue, + Scenario9LoadBindEvalInputTensorGPU, + Scenario13SingleModelOnCPUandGPU, + Scenario11FreeDimensionsTensor, + Scenario11FreeDimensionsImage, + Scenario14RunModelSwapchain, + Scenario20aLoadBindEvalCustomOperatorCPU, + Scenario20bLoadBindEvalReplacementCustomOperatorCPU, + Scenario21RunModel2ChainZ, + Scenario22ImageBindingAsGPUTensor, + MsftQuantizedModels, + SyncVsAsync, + CustomCommandQueueWithFence, + ReuseVideoFrame, + DeviceLostRecovery, + Scenario8SetDeviceSampleD3D11Device, + D2DInterop, }; + + if (SKIP_GPU_TESTS) { + api.Scenario6BindWithProperties = SkipTest; + api.Scenario8SetDeviceSampleDefaultDirectX = SkipTest; + api.Scenario8SetDeviceSampleMinPower = SkipTest; + api.Scenario8SetDeviceSampleMaxPerf = SkipTest; + api.Scenario8SetDeviceSampleCustomCommandQueue = SkipTest; + api.DISABLED_Scenario9LoadBindEvalInputTensorGPU = SkipTest; + api.Scenario13SingleModelOnCPUandGPU = SkipTest; + api.Scenario11FreeDimensionsTensor = SkipTest; + api.Scenario11FreeDimensionsImage = SkipTest; + api.Scenario14RunModelSwapchain = SkipTest; + api.Scenario20aLoadBindEvalCustomOperatorCPU = SkipTest; + api.Scenario20bLoadBindEvalReplacementCustomOperatorCPU = SkipTest; + api.DISABLED_Scenario21RunModel2ChainZ = SkipTest; + api.DISABLED_Scenario22ImageBindingAsGPUTensor = SkipTest; + api.MsftQuantizedModels = SkipTest; + api.DISABLED_SyncVsAsync = SkipTest; + api.DISABLED_CustomCommandQueueWithFence = SkipTest; + api.DISABLED_ReuseVideoFrame = SkipTest; + api.DeviceLostRecovery = SkipTest; + api.Scenario8SetDeviceSampleD3D11Device = SkipTest; + api.D2DInterop = SkipTest; + } + + if (RUNTIME_PARAMETER_EXISTS("EdgeCore")) { + api.Scenario8SetDeviceSampleMyCameraDevice = SkipTest; + api.Scenario8SetDeviceSampleD3D11Device = SkipTest; + api.D2DInterop = SkipTest; + } + return api; } diff --git a/winml/test/scenario/cppwinrt/scenariotestscppwinrt.h b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.h index 9999b4465ca03..4fee0e57472cd 100644 --- a/winml/test/scenario/cppwinrt/scenariotestscppwinrt.h +++ b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.h @@ -5,9 +5,6 @@ struct ScenarioTestsApi { SetupClass ScenarioCppWinrtTestsClassSetup; - SetupTest ScenarioCppWinrtTestsGpuMethodSetup; - SetupTest ScenarioCppWinrtTestsSkipEdgeCoreMethodSetup; - SetupTest ScenarioCppWinrtTestsGpuSkipEdgeCoreMethodSetup; VoidTest Sample1; VoidTest Scenario1LoadBindEvalDefault; VoidTest Scenario2LoadModelFromStream; @@ -59,45 +56,27 @@ WINML_TEST(ScenarioCppWinrtTests, Scenario17DevDiagnostics) WINML_TEST(ScenarioCppWinrtTests, DISABLED_Scenario22ImageBindingAsCPUTensor) WINML_TEST(ScenarioCppWinrtTests, QuantizedModels) WINML_TEST(ScenarioCppWinrtTests, EncryptedStream) +WINML_TEST(ScenarioCppWinrtTests, Scenario3SoftwareBitmapInputBinding) +WINML_TEST(ScenarioCppWinrtTests, Scenario6BindWithProperties) +WINML_TEST(ScenarioCppWinrtTests, Scenario8SetDeviceSampleDefaultDirectX) +WINML_TEST(ScenarioCppWinrtTests, Scenario8SetDeviceSampleMinPower) +WINML_TEST(ScenarioCppWinrtTests, Scenario8SetDeviceSampleMaxPerf) +WINML_TEST(ScenarioCppWinrtTests, Scenario8SetDeviceSampleCustomCommandQueue) +WINML_TEST(ScenarioCppWinrtTests, DISABLED_Scenario9LoadBindEvalInputTensorGPU) +WINML_TEST(ScenarioCppWinrtTests, Scenario13SingleModelOnCPUandGPU) +WINML_TEST(ScenarioCppWinrtTests, Scenario11FreeDimensionsTensor) +WINML_TEST(ScenarioCppWinrtTests, Scenario11FreeDimensionsImage) +WINML_TEST(ScenarioCppWinrtTests, Scenario14RunModelSwapchain) +WINML_TEST(ScenarioCppWinrtTests, Scenario20aLoadBindEvalCustomOperatorCPU) +WINML_TEST(ScenarioCppWinrtTests, Scenario20bLoadBindEvalReplacementCustomOperatorCPU) +WINML_TEST(ScenarioCppWinrtTests, DISABLED_Scenario21RunModel2ChainZ) +WINML_TEST(ScenarioCppWinrtTests, DISABLED_Scenario22ImageBindingAsGPUTensor) +WINML_TEST(ScenarioCppWinrtTests, MsftQuantizedModels) +WINML_TEST(ScenarioCppWinrtTests, DISABLED_SyncVsAsync) +WINML_TEST(ScenarioCppWinrtTests, DISABLED_CustomCommandQueueWithFence) +WINML_TEST(ScenarioCppWinrtTests, DISABLED_ReuseVideoFrame) +WINML_TEST(ScenarioCppWinrtTests, DeviceLostRecovery) +WINML_TEST(ScenarioCppWinrtTests, Scenario8SetDeviceSampleD3D11Device) +WINML_TEST(ScenarioCppWinrtTests, D2DInterop) +WINML_TEST(ScenarioCppWinrtTests, Scenario8SetDeviceSampleMyCameraDevice) WINML_TEST_CLASS_END() - -WINML_TEST_CLASS_BEGIN(ScenarioCppWinrtTestsGpu) -WINML_TEST_CLASS_SETUP_CLASS(ScenarioCppWinrtTestsClassSetup) -WINML_TEST_CLASS_SETUP_METHOD(ScenarioCppWinrtTestsGpuMethodSetup) -WINML_TEST_CLASS_BEGIN_TESTS -WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario3SoftwareBitmapInputBinding) -WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario6BindWithProperties) -WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario8SetDeviceSampleDefaultDirectX) -WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario8SetDeviceSampleMinPower) -WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario8SetDeviceSampleMaxPerf) -WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario8SetDeviceSampleCustomCommandQueue) -WINML_TEST(ScenarioCppWinrtTestsGpu, DISABLED_Scenario9LoadBindEvalInputTensorGPU) -WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario13SingleModelOnCPUandGPU) -WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario11FreeDimensionsTensor) -WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario11FreeDimensionsImage) -WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario14RunModelSwapchain) -WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario20aLoadBindEvalCustomOperatorCPU) -WINML_TEST(ScenarioCppWinrtTestsGpu, Scenario20bLoadBindEvalReplacementCustomOperatorCPU) -WINML_TEST(ScenarioCppWinrtTestsGpu, DISABLED_Scenario21RunModel2ChainZ) -WINML_TEST(ScenarioCppWinrtTestsGpu, DISABLED_Scenario22ImageBindingAsGPUTensor) -WINML_TEST(ScenarioCppWinrtTestsGpu, MsftQuantizedModels) -WINML_TEST(ScenarioCppWinrtTestsGpu, DISABLED_SyncVsAsync) -WINML_TEST(ScenarioCppWinrtTestsGpu, DISABLED_CustomCommandQueueWithFence) -WINML_TEST(ScenarioCppWinrtTestsGpu, DISABLED_ReuseVideoFrame) -WINML_TEST(ScenarioCppWinrtTestsGpu, DeviceLostRecovery) -WINML_TEST_CLASS_END() - -WINML_TEST_CLASS_BEGIN(ScenarioCppWinrtTestsSkipEdgeCore) -WINML_TEST_CLASS_SETUP_CLASS(ScenarioCppWinrtTestsClassSetup) -WINML_TEST_CLASS_SETUP_METHOD(ScenarioCppWinrtTestsSkipEdgeCoreMethodSetup) -WINML_TEST_CLASS_BEGIN_TESTS -WINML_TEST(ScenarioCppWinrtTestsSkipEdgeCore, Scenario8SetDeviceSampleMyCameraDevice) -WINML_TEST_CLASS_END() - -WINML_TEST_CLASS_BEGIN(ScenarioCppWinrtTestsGpuSkipEdgeCore) -WINML_TEST_CLASS_SETUP_CLASS(ScenarioCppWinrtTestsClassSetup) -WINML_TEST_CLASS_SETUP_METHOD(ScenarioCppWinrtTestsGpuSkipEdgeCoreMethodSetup) -WINML_TEST_CLASS_BEGIN_TESTS -WINML_TEST(ScenarioCppWinrtTestsGpuSkipEdgeCore, Scenario8SetDeviceSampleD3D11Device) -WINML_TEST(ScenarioCppWinrtTestsGpuSkipEdgeCore, D2DInterop) -WINML_TEST_CLASS_END() \ No newline at end of file From fa3d31608bed56072c561d5b08632804b6471cf1 Mon Sep 17 00:00:00 2001 From: Ori Levari Date: Wed, 6 May 2020 20:51:51 -0700 Subject: [PATCH 05/15] add videoframe filtering --- winml/test/api/LearningModelAPITest.cpp | 2 +- .../test/api/LearningModelBindingAPITest.cpp | 5 +++- winml/test/scenario/cppwinrt/CustomOps.cpp | 3 +++ .../cppwinrt/scenariotestscppwinrt.cpp | 25 ++++++++++++++++++- 4 files changed, 32 insertions(+), 3 deletions(-) diff --git a/winml/test/api/LearningModelAPITest.cpp b/winml/test/api/LearningModelAPITest.cpp index 9e8d8e8ca16a7..841cc785c9a6a 100644 --- a/winml/test/api/LearningModelAPITest.cpp +++ b/winml/test/api/LearningModelAPITest.cpp @@ -286,7 +286,7 @@ const LearningModelApiTestsApi& getapi() { CheckMetadataCaseInsensitive }; - if (SKIP_GPU_TESTS) { + if (RUNTIME_PARAMETER_EXISTS("noVideoFrameTests")) { api.CloseModelCheckEval = SkipTest; } return api; diff --git a/winml/test/api/LearningModelBindingAPITest.cpp b/winml/test/api/LearningModelBindingAPITest.cpp index 79305a652c028..e5d813f826f49 100644 --- a/winml/test/api/LearningModelBindingAPITest.cpp +++ b/winml/test/api/LearningModelBindingAPITest.cpp @@ -737,9 +737,12 @@ const LearningModelBindingAPITestsApi& getapi() { api.GpuSqueezeNet = SkipTest; api.GpuSqueezeNetEmptyOutputs = SkipTest; api.GpuSqueezeNetUnboundOutputs = SkipTest; + } + if (RUNTIME_PARAMETER_EXISTS("noVideoFrameTests")) { api.ImageBindingDimensions = SkipTest; - api.VerifyInvalidBindExceptions = SkipTest; api.BindInvalidInputName = SkipTest; + api.VerifyOutputAfterImageBindCalledTwice = SkipTest; + api.VerifyInvalidBindExceptions = SkipTest; } return api; } diff --git a/winml/test/scenario/cppwinrt/CustomOps.cpp b/winml/test/scenario/cppwinrt/CustomOps.cpp index f4e273fd739ac..613c7ec91fada 100644 --- a/winml/test/scenario/cppwinrt/CustomOps.cpp +++ b/winml/test/scenario/cppwinrt/CustomOps.cpp @@ -683,5 +683,8 @@ const CustomOpsTestsApi& getapi() { if (SKIP_GPU_TESTS) { api.CustomOperatorFusion = SkipTest; } + if (RUNTIME_PARAMETER_EXISTS("noVideoFrameTests")) { + api.CustomOperatorFusion = SkipTest; + } return api; } \ No newline at end of file diff --git a/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp index aef5def20e62d..a941b51fafc3d 100644 --- a/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp +++ b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp @@ -1465,5 +1465,28 @@ const ScenarioTestsApi& getapi() { api.D2DInterop = SkipTest; } - return api; + if (RUNTIME_PARAMETER_EXISTS("noVideoFrameTests")) { + api.Scenario1LoadBindEvalDefault = SkipTest; + api.Scenario3SoftwareBitmapInputBinding = SkipTest; + api.Scenario5AsyncEval = SkipTest; + api.Scenario6BindWithProperties = SkipTest; + api.Scenario7EvalWithNoBind = SkipTest; + api.DISABLED_Scenario9LoadBindEvalInputTensorGPU = SkipTest; + api.Scenario11FreeDimensionsTensor = SkipTest; + api.Scenario11FreeDimensionsImage = SkipTest; + api.Scenario13SingleModelOnCPUandGPU = SkipTest; + api.Scenario14RunModelSwapchain = SkipTest; + api.Scenario17DevDiagnostics = SkipTest; + api.DISABLED_Scenario21RunModel2ChainZ = SkipTest; + api.DISABLED_Scenario22ImageBindingAsCPUTensor = SkipTest; + api.DISABLED_Scenario22ImageBindingAsGPUTensor = SkipTest; + api.DISABLED_CustomCommandQueueWithFence = SkipTest; + api.DISABLED_ReuseVideoFrame = SkipTest; + api.D2DInterop = SkipTest; + api.DeviceLostRecovery = SkipTest; + api.QuantizedModels = SkipTest; + api.MsftQuantizedModels = SkipTest; + } + + return api; } From b67a1d3310669ca1cb511414dcf2061c8bb6c0ee Mon Sep 17 00:00:00 2001 From: Ori Levari Date: Thu, 7 May 2020 16:08:14 -0700 Subject: [PATCH 06/15] add delayload to libloader due to rawabi apitests calling LoadLibraryW for inbox fallback --- cmake/winml_unittests.cmake | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmake/winml_unittests.cmake b/cmake/winml_unittests.cmake index 29f5bdf9abee9..50bc86b262778 100644 --- a/cmake/winml_unittests.cmake +++ b/cmake/winml_unittests.cmake @@ -70,7 +70,8 @@ function(get_winml_test_scenario_src else() set(winml_test_scenario_src "${winml_test_src_path}/scenario/cppwinrt/scenariotestscppwinrt.h" - "${winml_test_src_path}/scenario/cppwinrt/scenariotestscppwinrt.cpp") + "${winml_test_src_path}/scenario/cppwinrt/scenariotestscppwinrt.cpp" + ) endif() set(${output_winml_test_scenario_src} ${winml_test_scenario_src} PARENT_SCOPE) endfunction() @@ -142,7 +143,7 @@ add_winml_test( SOURCES ${winml_test_api_src} LIBS winml_test_common delayimp.lib ) -target_link_options(winml_test_api PRIVATE /DELAYLOAD:dxgi.dll /DELAYLOAD:d3d12.dll /DELAYLOAD:api-ms-win-core-file-l1-2-2.dll /DELAYLOAD:api-ms-win-core-synch-l1-2-1.dll) +target_link_options(winml_test_api PRIVATE /DELAYLOAD:dxgi.dll /DELAYLOAD:d3d12.dll /DELAYLOAD:api-ms-win-core-file-l1-2-2.dll /DELAYLOAD:api-ms-win-core-synch-l1-2-1.dll /DELAYLOAD:api-ms-win-core-libraryloader-l1-2-1.dll ) if (onnxruntime_USE_DML) target_link_options(winml_test_api PRIVATE /DELAYLOAD:directml.dll) endif() From 3a11e7ff850c9513f62dceefbdd5c358ba4a1b9a Mon Sep 17 00:00:00 2001 From: Ori Levari Date: Thu, 7 May 2020 19:21:15 -0700 Subject: [PATCH 07/15] modifications to get all non videoframe or gpu api and scenario tests running on win 8.1 --- cmake/winml_unittests.cmake | 2 +- winml/test/api/raw/winml_microsoft.h | 2 +- winml/test/api/raw/winml_windows.h | 2 +- winml/test/scenario/cppwinrt/CustomOperatorProvider.h | 2 +- winml/test/scenario/cppwinrt/CustomOps.cpp | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmake/winml_unittests.cmake b/cmake/winml_unittests.cmake index 50bc86b262778..ef6fc954472b8 100644 --- a/cmake/winml_unittests.cmake +++ b/cmake/winml_unittests.cmake @@ -143,7 +143,7 @@ add_winml_test( SOURCES ${winml_test_api_src} LIBS winml_test_common delayimp.lib ) -target_link_options(winml_test_api PRIVATE /DELAYLOAD:dxgi.dll /DELAYLOAD:d3d12.dll /DELAYLOAD:api-ms-win-core-file-l1-2-2.dll /DELAYLOAD:api-ms-win-core-synch-l1-2-1.dll /DELAYLOAD:api-ms-win-core-libraryloader-l1-2-1.dll ) +target_link_options(winml_test_api PRIVATE /DELAYLOAD:d3d11.dll /DELAYLOAD:dxgi.dll /DELAYLOAD:d3d12.dll /DELAYLOAD:api-ms-win-core-file-l1-2-2.dll /DELAYLOAD:api-ms-win-core-synch-l1-2-1.dll) if (onnxruntime_USE_DML) target_link_options(winml_test_api PRIVATE /DELAYLOAD:directml.dll) endif() diff --git a/winml/test/api/raw/winml_microsoft.h b/winml/test/api/raw/winml_microsoft.h index 1f76789bd690a..d61555f7a36d1 100644 --- a/winml/test/api/raw/winml_microsoft.h +++ b/winml/test/api/raw/winml_microsoft.h @@ -172,7 +172,7 @@ inline HRESULT GetActivationFactory( void** factory) noexcept { // Fallback to OS binary if the redistributable is not present! - auto library = LoadLibraryW(MachineLearningDll); + auto library = LoadLibraryExW(MachineLearningDll, nullptr, 0); using DllGetActivationFactory = HRESULT __stdcall(HSTRING, void** factory); auto call = reinterpret_cast(GetProcAddress(library, "DllGetActivationFactory")); diff --git a/winml/test/api/raw/winml_windows.h b/winml/test/api/raw/winml_windows.h index c48f7a20b8440..7891cc92c38a4 100644 --- a/winml/test/api/raw/winml_windows.h +++ b/winml/test/api/raw/winml_windows.h @@ -172,7 +172,7 @@ inline HRESULT GetActivationFactory( void** factory) noexcept { // Fallback to OS binary if the redistributable is not present! - auto library = LoadLibraryW(MachineLearningDll); + auto library = LoadLibraryExW(MachineLearningDll, nullptr, 0); using DllGetActivationFactory = HRESULT __stdcall(HSTRING, void** factory); auto call = reinterpret_cast(GetProcAddress(library, "DllGetActivationFactory")); diff --git a/winml/test/scenario/cppwinrt/CustomOperatorProvider.h b/winml/test/scenario/cppwinrt/CustomOperatorProvider.h index 87c51c8412a18..0156c69e29cda 100644 --- a/winml/test/scenario/cppwinrt/CustomOperatorProvider.h +++ b/winml/test/scenario/cppwinrt/CustomOperatorProvider.h @@ -22,7 +22,7 @@ struct CustomOperatorProvider : auto winml_dll_name = dll.str(); #if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) - m_library = LoadLibraryW(winml_dll_name.c_str()); + m_library = LoadLibraryExW(winml_dll_name.c_str(), nullptr, 0); #elif WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_PC_APP) m_library = LoadPackagedLibrary(winml_dll_name.c_str(), 0 /*Reserved*/); #endif diff --git a/winml/test/scenario/cppwinrt/CustomOps.cpp b/winml/test/scenario/cppwinrt/CustomOps.cpp index 613c7ec91fada..6d3252e2bda28 100644 --- a/winml/test/scenario/cppwinrt/CustomOps.cpp +++ b/winml/test/scenario/cppwinrt/CustomOps.cpp @@ -80,7 +80,7 @@ static void CustomOperatorFusion() { auto winml_dll_name = dll.str(); #if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) - auto m_library = LoadLibraryW(winml_dll_name.c_str()); + auto m_library = LoadLibraryExW(winml_dll_name.c_str(), nullptr, 0); #elif WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_PC_APP) auto m_library = LoadPackagedLibrary(winml_dll_name.c_str(), 0 /*Reserved*/); #endif @@ -170,7 +170,7 @@ struct LocalCustomOperatorProvider : winrt::implements< auto winml_dll_name = dll.str(); #if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) - auto m_library = LoadLibraryW(winml_dll_name.c_str()); + auto m_library = LoadLibraryExW(winml_dll_name.c_str(), nullptr, 0); #elif WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_PC_APP) auto m_library = LoadPackagedLibrary(winml_dll_name.c_str(), 0 /*Reserved*/); #endif From ad57a93f99dfdf175e9bddba3768b990bccc1b5f Mon Sep 17 00:00:00 2001 From: Ori Levari Date: Fri, 8 May 2020 00:00:08 -0700 Subject: [PATCH 08/15] add option for disabling downlevel tests due to missing idxgifactory6 methods --- winml/test/api/LearningModelSessionAPITest.cpp | 4 ++++ winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/winml/test/api/LearningModelSessionAPITest.cpp b/winml/test/api/LearningModelSessionAPITest.cpp index 97bef1505093b..fc38a17e7e247 100644 --- a/winml/test/api/LearningModelSessionAPITest.cpp +++ b/winml/test/api/LearningModelSessionAPITest.cpp @@ -419,5 +419,9 @@ const LearningModelSessionAPITestsApi& getapi() { if (RUNTIME_PARAMETER_EXISTS("EdgeCore")) { api.AdapterIdAndDevice = SkipTest; } + if (RUNTIME_PARAMETER_EXISTS("noIDXGIFactory6Tests")) { + api.CreateSessionDeviceDirectXHighPerformance = SkipTest; + api.CreateSessionDeviceDirectXMinimumPower = SkipTest; + } return api; } diff --git a/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp index a941b51fafc3d..cb62e0c651e01 100644 --- a/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp +++ b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp @@ -1487,6 +1487,10 @@ const ScenarioTestsApi& getapi() { api.QuantizedModels = SkipTest; api.MsftQuantizedModels = SkipTest; } + if (RUNTIME_PARAMETER_EXISTS("noIDXGIFactory6Tests")) { + api.Scenario8SetDeviceSampleMinPower = SkipTest; + api.Scenario8SetDeviceSampleMaxPerf = SkipTest; + } return api; } From a8602912df01e03c8212e1509ccb92c9a163274d Mon Sep 17 00:00:00 2001 From: Ori Levari Date: Fri, 8 May 2020 10:35:01 -0700 Subject: [PATCH 09/15] add adapteridanddevice as dxgifactory6test --- winml/test/api/LearningModelSessionAPITest.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/winml/test/api/LearningModelSessionAPITest.cpp b/winml/test/api/LearningModelSessionAPITest.cpp index fc38a17e7e247..14040861ecd36 100644 --- a/winml/test/api/LearningModelSessionAPITest.cpp +++ b/winml/test/api/LearningModelSessionAPITest.cpp @@ -422,6 +422,7 @@ const LearningModelSessionAPITestsApi& getapi() { if (RUNTIME_PARAMETER_EXISTS("noIDXGIFactory6Tests")) { api.CreateSessionDeviceDirectXHighPerformance = SkipTest; api.CreateSessionDeviceDirectXMinimumPower = SkipTest; + api.AdapterIdAndDevice = SkipTest; } return api; } From 1f90b478f27bf2964e666add385cec8184645454 Mon Sep 17 00:00:00 2001 From: Ori Levari Date: Fri, 8 May 2020 13:51:37 -0700 Subject: [PATCH 10/15] add flag to skip tests below RS5 that use ID3D12Device5 functions. Applies to DeviceLostRecovery test --- winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp index cb62e0c651e01..11a3af8b345d7 100644 --- a/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp +++ b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp @@ -1491,6 +1491,8 @@ const ScenarioTestsApi& getapi() { api.Scenario8SetDeviceSampleMinPower = SkipTest; api.Scenario8SetDeviceSampleMaxPerf = SkipTest; } - - return api; + if (RUNTIME_PARAMETER_EXISTS("noID3D12Device5Tests")) { + api.DeviceLostRecovery = SkipTest; + } + return api; } From bf384bd6fa8159bb5331972eacb0dc75550f7311 Mon Sep 17 00:00:00 2001 From: Ori Levari Date: Mon, 11 May 2020 11:23:16 -0700 Subject: [PATCH 11/15] remove trailing semicolon --- winml/test/common/taefTestMacros.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/winml/test/common/taefTestMacros.h b/winml/test/common/taefTestMacros.h index de661f520086f..ccea659ec5716 100644 --- a/winml/test/common/taefTestMacros.h +++ b/winml/test/common/taefTestMacros.h @@ -69,7 +69,7 @@ using namespace WEX::TestExecution; #define RUNTIME_PARAMETER_EXISTS(param) \ bool param_value; \ - SUCCEEDED(RuntimeParameters::TryGetValue(param, param_value)) && param_value; + SUCCEEDED(RuntimeParameters::TryGetValue(param, param_value)) && param_value #ifndef USE_DML #define SKIP_GPU_TESTS \ From 3d0645a14e04fa9ea7192b667a1ee2b706486f10 Mon Sep 17 00:00:00 2001 From: Ori Levari Date: Mon, 11 May 2020 13:21:15 -0700 Subject: [PATCH 12/15] Convert test filter functions from macros into functions. Standardize parameter string type to wide string between test frameworks. --- winml/test/adapter/AdapterSessionTest.cpp | 2 +- winml/test/api/LearningModelAPITest.cpp | 2 +- .../test/api/LearningModelBindingAPITest.cpp | 4 +- .../test/api/LearningModelSessionAPITest.cpp | 6 +-- winml/test/api/RawApiTestsGpu.cpp | 2 +- winml/test/common/googleTestMacros.h | 49 +++++++++---------- winml/test/common/googletest/main.cpp | 2 +- winml/test/common/taefTestMacros.h | 30 ++++++------ winml/test/concurrency/ConcurrencyTests.cpp | 4 +- winml/test/scenario/cppwinrt/CustomOps.cpp | 4 +- .../cppwinrt/scenariotestscppwinrt.cpp | 10 ++-- 11 files changed, 57 insertions(+), 58 deletions(-) diff --git a/winml/test/adapter/AdapterSessionTest.cpp b/winml/test/adapter/AdapterSessionTest.cpp index 0099fea3027b0..b5864f45ad7c2 100644 --- a/winml/test/adapter/AdapterSessionTest.cpp +++ b/winml/test/adapter/AdapterSessionTest.cpp @@ -302,7 +302,7 @@ const AdapterSessionTestAPI& getapi() { CopyInputAcrossDevices_DML }; - if (SKIP_GPU_TESTS) { + if (SkipGpuTests()) { api.AppendExecutionProvider_DML = SkipTest; api.GetExecutionProvider_DML = SkipTest; api.RegisterGraphTransformers_DML = SkipTest; diff --git a/winml/test/api/LearningModelAPITest.cpp b/winml/test/api/LearningModelAPITest.cpp index 841cc785c9a6a..bd76b7e3d6a0f 100644 --- a/winml/test/api/LearningModelAPITest.cpp +++ b/winml/test/api/LearningModelAPITest.cpp @@ -286,7 +286,7 @@ const LearningModelApiTestsApi& getapi() { CheckMetadataCaseInsensitive }; - if (RUNTIME_PARAMETER_EXISTS("noVideoFrameTests")) { + if (RuntimeParameterExists(L"noVideoFrameTests")) { api.CloseModelCheckEval = SkipTest; } return api; diff --git a/winml/test/api/LearningModelBindingAPITest.cpp b/winml/test/api/LearningModelBindingAPITest.cpp index e5d813f826f49..c5419affb3b19 100644 --- a/winml/test/api/LearningModelBindingAPITest.cpp +++ b/winml/test/api/LearningModelBindingAPITest.cpp @@ -733,12 +733,12 @@ const LearningModelBindingAPITestsApi& getapi() { SequenceConstructTensorString }; - if (SKIP_GPU_TESTS) { + if (SkipGpuTests()) { api.GpuSqueezeNet = SkipTest; api.GpuSqueezeNetEmptyOutputs = SkipTest; api.GpuSqueezeNetUnboundOutputs = SkipTest; } - if (RUNTIME_PARAMETER_EXISTS("noVideoFrameTests")) { + if (RuntimeParameterExists(L"noVideoFrameTests")) { api.ImageBindingDimensions = SkipTest; api.BindInvalidInputName = SkipTest; api.VerifyOutputAfterImageBindCalledTwice = SkipTest; diff --git a/winml/test/api/LearningModelSessionAPITest.cpp b/winml/test/api/LearningModelSessionAPITest.cpp index 14040861ecd36..2132fe9bfefaf 100644 --- a/winml/test/api/LearningModelSessionAPITest.cpp +++ b/winml/test/api/LearningModelSessionAPITest.cpp @@ -408,7 +408,7 @@ const LearningModelSessionAPITestsApi& getapi() { CloseSession, }; - if (SKIP_GPU_TESTS) { + if (SkipGpuTests()) { api.CreateSessionDeviceDirectX = SkipTest; api.CreateSessionDeviceDirectXHighPerformance = SkipTest; api.CreateSessionDeviceDirectXMinimumPower = SkipTest; @@ -416,10 +416,10 @@ const LearningModelSessionAPITestsApi& getapi() { api.DISABLED_CreateSessionWithFloat16InitializersInModel = SkipTest; api.AdapterIdAndDevice = SkipTest; } - if (RUNTIME_PARAMETER_EXISTS("EdgeCore")) { + if (RuntimeParameterExists(L"EdgeCore")) { api.AdapterIdAndDevice = SkipTest; } - if (RUNTIME_PARAMETER_EXISTS("noIDXGIFactory6Tests")) { + if (RuntimeParameterExists(L"noIDXGIFactory6Tests")) { api.CreateSessionDeviceDirectXHighPerformance = SkipTest; api.CreateSessionDeviceDirectXMinimumPower = SkipTest; api.AdapterIdAndDevice = SkipTest; diff --git a/winml/test/api/RawApiTestsGpu.cpp b/winml/test/api/RawApiTestsGpu.cpp index fb22affbd278d..fa35101bb8554 100644 --- a/winml/test/api/RawApiTestsGpu.cpp +++ b/winml/test/api/RawApiTestsGpu.cpp @@ -162,7 +162,7 @@ const RawApiTestsGpuApi& getapi() { EvaluateNoInputCopy }; - if (SKIP_GPU_TESTS) { + if (SkipGpuTests()) { api.CreateDirectXDevice = SkipTest; api.CreateD3D11DeviceDevice = SkipTest; api.CreateD3D12CommandQueueDevice = SkipTest; diff --git a/winml/test/common/googleTestMacros.h b/winml/test/common/googleTestMacros.h index f96addb63725b..74f87c9e7fe54 100644 --- a/winml/test/common/googleTestMacros.h +++ b/winml/test/common/googleTestMacros.h @@ -12,8 +12,8 @@ getapi().test_name(); \ } -#define WINML_TEST_CLASS_BEGIN(test_class_name) \ - namespace { \ +#define WINML_TEST_CLASS_BEGIN(test_class_name) \ + namespace { \ class test_class_name : public ::testing::Test { #define WINML_TEST_CLASS_SETUP_CLASS(setup_class) \ @@ -41,10 +41,8 @@ } #define WINML_TEST_CLASS_BEGIN_TESTS }; - #define WINML_TEST_CLASS_END() } - // For old versions of gtest without GTEST_SKIP, stream the message and return success instead #ifndef GTEST_SKIP #define GTEST_SKIP_(message) \ @@ -52,22 +50,21 @@ #define GTEST_SKIP GTEST_SKIP_("") #endif -#define EXPECT_THROW_SPECIFIC(statement, exception, condition) \ - EXPECT_THROW( \ - try { \ - statement; \ - } catch (const exception& e) { \ - EXPECT_TRUE(condition(e)); \ - throw; \ - } \ - , exception); +#define EXPECT_THROW_SPECIFIC(statement, exception, condition) \ + EXPECT_THROW( \ + try { \ + statement; \ + } catch (const exception& e) { \ + EXPECT_TRUE(condition(e)); \ + throw; \ + }, \ + exception); #ifndef INSTANTIATE_TEST_SUITE_P // Use the old name, removed in newer versions of googletest #define INSTANTIATE_TEST_SUITE_P INSTANTIATE_TEST_CASE_P #endif - #define WINML_SKIP_TEST(message) \ WINML_SUPRESS_UNREACHABLE_BELOW(GTEST_SKIP() << message) @@ -79,28 +76,30 @@ #define WINML_LOG_ERROR(message) \ ADD_FAILURE() << message -#define WINML_LOG_COMMENT(message)\ +#define WINML_LOG_COMMENT(message) \ SCOPED_TRACE(message) #define WINML_EXPECT_HRESULT_SUCCEEDED(hresult_expression) EXPECT_HRESULT_SUCCEEDED(hresult_expression) #define WINML_EXPECT_HRESULT_FAILED(hresult_expression) EXPECT_HRESULT_FAILED(hresult_expression) #define WINML_EXPECT_THROW_SPECIFIC(statement, exception, condition) EXPECT_THROW_SPECIFIC(statement, exception, condition) +static bool RuntimeParameterExists(std::wstring param) +{ + std::string narrowParam = std::wstring_convert>().to_bytes(param); + auto no_gpu_tests = RuntimeParameters::Parameters.find(narrowParam); + return no_gpu_tests != RuntimeParameters::Parameters.end() && no_gpu_tests->second != "0"; +} -#define RUNTIME_PARAMETER_EXISTS(param) \ - auto no_gpu_tests = RuntimeParameters::Parameters.find(param); \ - no_gpu_tests != RuntimeParameters::Parameters.end() && no_gpu_tests->second != "0" - +static bool SkipGpuTests() +{ #ifndef USE_DML -#define SKIP_GPU_TESTS \ - true + return true; #else -#define SKIP_GPU_TESTS \ - RUNTIME_PARAMETER_EXISTS("noGPUtests") + return RuntimeParameterExists(L"noGPUtests"); #endif +} #define GPUTEST \ - if (SKIP_GPU_TESTS) { \ + if (SkipGpuTests()) { \ WINML_SKIP_TEST("Gpu tests disabled"); \ } - diff --git a/winml/test/common/googletest/main.cpp b/winml/test/common/googletest/main.cpp index bf08c10ce3d6d..0a672dcd8ff79 100644 --- a/winml/test/common/googletest/main.cpp +++ b/winml/test/common/googletest/main.cpp @@ -8,7 +8,7 @@ #include "runtimeParameters.h" namespace RuntimeParameters { -std::unordered_map Parameters; +std::unordered_map Parameters; } namespace { diff --git a/winml/test/common/taefTestMacros.h b/winml/test/common/taefTestMacros.h index ccea659ec5716..8391d54499719 100644 --- a/winml/test/common/taefTestMacros.h +++ b/winml/test/common/taefTestMacros.h @@ -13,7 +13,7 @@ using namespace WEX::TestExecution; TEST_CLASS(test_class_name); #define WINML_TEST_CLASS_SETUP_CLASS(setup_class) \ - TEST_CLASS_SETUP(TestClassSetup) { \ + TEST_CLASS_SETUP(TestClassSetup) { \ getapi().setup_class(); \ return true; \ } @@ -25,9 +25,9 @@ using namespace WEX::TestExecution; } #define WINML_TEST_CLASS_SETUP_METHOD(setup_method) \ - TEST_METHOD_SETUP(TestMethodSetup) { \ - getapi().setup_method(); \ - return true; \ + TEST_METHOD_SETUP(TestMethodSetup) { \ + getapi().setup_method(); \ + return true; \ } #define WINML_TEST_CLASS_TEARDOWN_METHOD(teardown_method) \ @@ -51,8 +51,7 @@ using namespace WEX::TestExecution; WINML_SUPRESS_UNREACHABLE_BELOW( \ Log::Result(TestResults::Skipped, \ std::wstring_convert>().from_bytes(message).c_str()); \ - return; \ - ) + return;) #define WINML_EXPECT_NO_THROW(statement) VERIFY_NO_THROW(statement) #define WINML_EXPECT_TRUE(statement) VERIFY_IS_TRUE(statement) @@ -61,25 +60,26 @@ using namespace WEX::TestExecution; #define WINML_EXPECT_NOT_EQUAL(val1, val2) VERIFY_ARE_NOT_EQUAL(val1, val2) #define WINML_LOG_ERROR(message) \ VERIFY_FAIL(std::wstring_convert>().from_bytes(message).c_str()) -#define WINML_LOG_COMMENT(message)\ +#define WINML_LOG_COMMENT(message) \ WEX::Logging::Log::Comment(std::wstring_convert>().from_bytes(message).c_str()) #define WINML_EXPECT_HRESULT_SUCCEEDED(hresult_expression) VERIFY_SUCCEEDED(hresult_expression) #define WINML_EXPECT_THROW_SPECIFIC(statement, exception, condition) VERIFY_THROWS_SPECIFIC(statement, exception, condition) #define WINML_EXPECT_HRESULT_FAILED(hresult_expression) VERIFY_FAILED(hresult_expression) -#define RUNTIME_PARAMETER_EXISTS(param) \ - bool param_value; \ - SUCCEEDED(RuntimeParameters::TryGetValue(param, param_value)) && param_value +static bool RuntimeParameterExists(std::wstring param) { + bool param_value; + return SUCCEEDED(RuntimeParameters::TryGetValue(param, param_value)) && param_value; +} +static bool SkipGpuTests() { #ifndef USE_DML -#define SKIP_GPU_TESTS \ - true + return true; #else -#define SKIP_GPU_TESTS \ - RUNTIME_PARAMETER_EXISTS(L"noGPUtests") + RuntimeParameterExists(L"noGPUtests"); #endif +} #define GPUTEST \ - if (SKIP_GPU_TESTS) { \ + if (SkipGpuTests()) { \ WINML_SKIP_TEST("Gpu tests disabled"); \ } \ No newline at end of file diff --git a/winml/test/concurrency/ConcurrencyTests.cpp b/winml/test/concurrency/ConcurrencyTests.cpp index 39b5eec17d532..79d32c83cea0c 100644 --- a/winml/test/concurrency/ConcurrencyTests.cpp +++ b/winml/test/concurrency/ConcurrencyTests.cpp @@ -25,7 +25,7 @@ void LoadBindEvalSqueezenetRealDataWithValidationConcurrently() { for (const auto& instance : {"1", "2", "3", "4"}) { threads.emplace_back(load_test_model, instance, LearningModelDeviceKind::Cpu); } - if (SKIP_GPU_TESTS) {} else { + if (SkipGpuTests()) {} else { for (const auto& instance : {"GPU_1", "GPU_2", "GPU_3", "GPU_4"}) { threads.emplace_back(load_test_model, instance, LearningModelDeviceKind::DirectX); } @@ -339,7 +339,7 @@ const ConcurrencyTestsApi& getapi() { EvalAsyncDifferentBindings }; - if (SKIP_GPU_TESTS) { + if (SkipGpuTests()) { api.MultiThreadMultiSessionGpu = SkipTest; api.MultiThreadSingleSessionGpu = SkipTest; } diff --git a/winml/test/scenario/cppwinrt/CustomOps.cpp b/winml/test/scenario/cppwinrt/CustomOps.cpp index 6d3252e2bda28..b87f3a94abe05 100644 --- a/winml/test/scenario/cppwinrt/CustomOps.cpp +++ b/winml/test/scenario/cppwinrt/CustomOps.cpp @@ -680,10 +680,10 @@ const CustomOpsTestsApi& getapi() { CustomKernelWithCustomSchema }; - if (SKIP_GPU_TESTS) { + if (SkipGpuTests()) { api.CustomOperatorFusion = SkipTest; } - if (RUNTIME_PARAMETER_EXISTS("noVideoFrameTests")) { + if (RuntimeParameterExists(L"noVideoFrameTests")) { api.CustomOperatorFusion = SkipTest; } return api; diff --git a/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp index 11a3af8b345d7..a8a3f84db037e 100644 --- a/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp +++ b/winml/test/scenario/cppwinrt/scenariotestscppwinrt.cpp @@ -1435,7 +1435,7 @@ const ScenarioTestsApi& getapi() { D2DInterop, }; - if (SKIP_GPU_TESTS) { + if (SkipGpuTests()) { api.Scenario6BindWithProperties = SkipTest; api.Scenario8SetDeviceSampleDefaultDirectX = SkipTest; api.Scenario8SetDeviceSampleMinPower = SkipTest; @@ -1459,13 +1459,13 @@ const ScenarioTestsApi& getapi() { api.D2DInterop = SkipTest; } - if (RUNTIME_PARAMETER_EXISTS("EdgeCore")) { + if (RuntimeParameterExists(L"EdgeCore")) { api.Scenario8SetDeviceSampleMyCameraDevice = SkipTest; api.Scenario8SetDeviceSampleD3D11Device = SkipTest; api.D2DInterop = SkipTest; } - if (RUNTIME_PARAMETER_EXISTS("noVideoFrameTests")) { + if (RuntimeParameterExists(L"noVideoFrameTests")) { api.Scenario1LoadBindEvalDefault = SkipTest; api.Scenario3SoftwareBitmapInputBinding = SkipTest; api.Scenario5AsyncEval = SkipTest; @@ -1487,11 +1487,11 @@ const ScenarioTestsApi& getapi() { api.QuantizedModels = SkipTest; api.MsftQuantizedModels = SkipTest; } - if (RUNTIME_PARAMETER_EXISTS("noIDXGIFactory6Tests")) { + if (RuntimeParameterExists(L"noIDXGIFactory6Tests")) { api.Scenario8SetDeviceSampleMinPower = SkipTest; api.Scenario8SetDeviceSampleMaxPerf = SkipTest; } - if (RUNTIME_PARAMETER_EXISTS("noID3D12Device5Tests")) { + if (RuntimeParameterExists(L"noID3D12Device5Tests")) { api.DeviceLostRecovery = SkipTest; } return api; From 9a0958b971daed530c4258059f9e9fc7f170a551 Mon Sep 17 00:00:00 2001 From: Ori Levari Date: Mon, 11 May 2020 15:03:50 -0700 Subject: [PATCH 13/15] move skip test into macros files --- winml/test/common/googleTestMacros.h | 5 ++++- winml/test/common/taefTestMacros.h | 6 +++++- winml/test/common/test.h | 8 +------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/winml/test/common/googleTestMacros.h b/winml/test/common/googleTestMacros.h index 74f87c9e7fe54..ec4a70e8daa3c 100644 --- a/winml/test/common/googleTestMacros.h +++ b/winml/test/common/googleTestMacros.h @@ -98,8 +98,11 @@ static bool SkipGpuTests() #endif } - #define GPUTEST \ if (SkipGpuTests()) { \ WINML_SKIP_TEST("Gpu tests disabled"); \ } + +static void SkipTest() { + WINML_SKIP_TEST(""); +} diff --git a/winml/test/common/taefTestMacros.h b/winml/test/common/taefTestMacros.h index 8391d54499719..56d4fc6d87bd3 100644 --- a/winml/test/common/taefTestMacros.h +++ b/winml/test/common/taefTestMacros.h @@ -82,4 +82,8 @@ static bool SkipGpuTests() { #define GPUTEST \ if (SkipGpuTests()) { \ WINML_SKIP_TEST("Gpu tests disabled"); \ - } \ No newline at end of file + } + +static void SkipTest() { + WINML_SKIP_TEST(""); +} \ No newline at end of file diff --git a/winml/test/common/test.h b/winml/test/common/test.h index 8d73710b8f0ad..4fb09ec307112 100644 --- a/winml/test/common/test.h +++ b/winml/test/common/test.h @@ -24,10 +24,4 @@ constexpr bool alwaysFalse() { #ifdef BUILD_TAEF_TEST #include "taefTestMacros.h" #endif -#endif - -#if defined(BUILD_GOOGLE_TEST) || defined(BUILD_TAEF_TEST) -static void SkipTest() { - WINML_SKIP_TEST("") -} -#endif \ No newline at end of file +#endif From 4da83d980ede54e1f59f8c4b24937fa344b7393f Mon Sep 17 00:00:00 2001 From: Ori Levari Date: Mon, 11 May 2020 17:05:28 -0700 Subject: [PATCH 14/15] fix some taefmacro bugs. default to using googletest --- winml/test/common/taefTestMacros.h | 10 +++------- winml/test/common/test.h | 10 ++++++---- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/winml/test/common/taefTestMacros.h b/winml/test/common/taefTestMacros.h index 56d4fc6d87bd3..94451c5a8f6ba 100644 --- a/winml/test/common/taefTestMacros.h +++ b/winml/test/common/taefTestMacros.h @@ -68,22 +68,18 @@ using namespace WEX::TestExecution; static bool RuntimeParameterExists(std::wstring param) { bool param_value; - return SUCCEEDED(RuntimeParameters::TryGetValue(param, param_value)) && param_value; + return SUCCEEDED(RuntimeParameters::TryGetValue(param.c_str(), param_value)) && param_value; } static bool SkipGpuTests() { #ifndef USE_DML return true; #else - RuntimeParameterExists(L"noGPUtests"); + return RuntimeParameterExists(L"noGPUtests"); #endif } #define GPUTEST \ if (SkipGpuTests()) { \ WINML_SKIP_TEST("Gpu tests disabled"); \ - } - -static void SkipTest() { - WINML_SKIP_TEST(""); -} \ No newline at end of file + } \ No newline at end of file diff --git a/winml/test/common/test.h b/winml/test/common/test.h index 4fb09ec307112..1a8380da2bda2 100644 --- a/winml/test/common/test.h +++ b/winml/test/common/test.h @@ -18,10 +18,12 @@ constexpr bool alwaysFalse() { #define WINML_SUPRESS_UNREACHABLE_BELOW(statement) \ if (alwaysTrue()) { statement; } -#ifdef BUILD_GOOGLE_TEST -#include "googleTestMacros.h" -#else #ifdef BUILD_TAEF_TEST #include "taefTestMacros.h" +#else +#include "googleTestMacros.h" #endif -#endif + +static void SkipTest() { + WINML_SKIP_TEST(""); +} From fec0802a79c09fc013218be4826c693a201b4f8a Mon Sep 17 00:00:00 2001 From: Ori Levari Date: Mon, 11 May 2020 17:44:34 -0700 Subject: [PATCH 15/15] add missing change --- winml/test/common/googleTestMacros.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/winml/test/common/googleTestMacros.h b/winml/test/common/googleTestMacros.h index ec4a70e8daa3c..ba487316c8f10 100644 --- a/winml/test/common/googleTestMacros.h +++ b/winml/test/common/googleTestMacros.h @@ -102,7 +102,3 @@ static bool SkipGpuTests() if (SkipGpuTests()) { \ WINML_SKIP_TEST("Gpu tests disabled"); \ } - -static void SkipTest() { - WINML_SKIP_TEST(""); -}