Various changes for automated downlevel test pipeline

cmake/winml_unittests.cmake

@@ -70,7 +70,8 @@ function(get_winml_test_scenario_src
   else()
     set(winml_test_scenario_src
         "${winml_test_src_path}/scenario/cppwinrt/scenariotestscppwinrt.h"
-        "${winml_test_src_path}/scenario/cppwinrt/scenariotestscppwinrt.cpp")
+        "${winml_test_src_path}/scenario/cppwinrt/scenariotestscppwinrt.cpp"
+        )
   endif()
   set(${output_winml_test_scenario_src} ${winml_test_scenario_src} PARENT_SCOPE)
 endfunction()
@@ -168,7 +169,7 @@ add_winml_test(
   SOURCES ${winml_test_api_src} ${winml_test_api_redist_only_src}
   LIBS winml_test_common delayimp.lib
 )
-target_link_options(winml_test_api PRIVATE /DELAYLOAD:dxgi.dll /DELAYLOAD:d3d12.dll /DELAYLOAD:api-ms-win-core-file-l1-2-2.dll /DELAYLOAD:api-ms-win-core-synch-l1-2-1.dll)
+target_link_options(winml_test_api PRIVATE /DELAYLOAD:d3d11.dll /DELAYLOAD:dxgi.dll /DELAYLOAD:d3d12.dll /DELAYLOAD:api-ms-win-core-file-l1-2-2.dll /DELAYLOAD:api-ms-win-core-synch-l1-2-1.dll)
 if (onnxruntime_USE_DML)
   target_link_options(winml_test_api PRIVATE /DELAYLOAD:directml.dll)
 endif()

tools/ci_build/github/azure-pipelines/templates/windowsai-nuget-build.yml

@@ -113,6 +113,8 @@ steps:
        xcopy $(Build.SourcesDirectory)\winml\test\collateral\models\ModelSubdirectory $(Build.ArtifactStagingDirectory)\test_artifact\ModelSubdirectory\ /i
        copy $(Build.SourcesDirectory)\winml\test\collateral\images\*.png $(Build.ArtifactStagingDirectory)\test_artifact\
        copy $(Build.SourcesDirectory)\winml\test\collateral\images\*.jpg $(Build.ArtifactStagingDirectory)\test_artifact\
+       copy $(Build.SourcesDirectory)\onnxruntime\test\testdata\sequence_length.onnx $(Build.ArtifactStagingDirectory)\test_artifact\
+       copy $(Build.SourcesDirectory)\onnxruntime\test\testdata\sequence_construct.onnx $(Build.ArtifactStagingDirectory)\test_artifact\
       displayName: 'Copy WinML test collateral to artifact directory'
   - task: NuGetToolInstaller@0
     displayName: Use Nuget 4.9

winml/test/adapter/AdapterSessionTest.cpp

@@ -84,7 +84,6 @@ UniqueOrtSession CreateUniqueOrtSession(const std::wstring& model_path, const Un
 }
 
 void AppendExecutionProvider_DML() {
-  GPUTEST;
   const auto session_options = CreateUniqueOrtSessionOptions();
 
   const auto device = CreateD3DDevice();
@@ -107,7 +106,6 @@ void GetExecutionProvider() {
 }
 
 void GetExecutionProvider_DML() {
-  GPUTEST;
   const auto session_options = CreateUniqueOrtSessionOptions();
   THROW_IF_NOT_OK_MSG(ort_api->DisableMemPattern(session_options.get()), ort_api);
   const auto device = CreateD3DDevice();
@@ -130,7 +128,6 @@ void RegisterGraphTransformers() {
 }
 
 void RegisterGraphTransformers_DML() {
-  GPUTEST;
   const auto session_options = CreateUniqueOrtSessionOptions();
   auto session = CreateUniqueOrtSession(session_options);
   winml_adapter_api->SessionRegisterGraphTransformers(session.get());
@@ -147,7 +144,6 @@ void RegisterCustomRegistry() {
 }
 
 void RegisterCustomRegistry_DML() {
-  GPUTEST;
   IMLOperatorRegistry* registry;
   THROW_IF_NOT_OK_MSG(winml_adapter_api->CreateCustomRegistry(&registry), ort_api);
   WINML_EXPECT_NOT_EQUAL(nullptr, registry);
@@ -250,8 +246,6 @@ void CopyInputAcrossDevices() {
 }
 
 void CopyInputAcrossDevices_DML() {
-  GPUTEST;
-
   const auto session_options = CreateUniqueOrtSessionOptions();
   THROW_IF_NOT_OK_MSG(ort_api->DisableMemPattern(session_options.get()), ort_api);
   const auto device = CreateD3DDevice();
@@ -287,8 +281,8 @@ void CopyInputAcrossDevices_DML() {
 }
 }
 
-const AdapterSessionTestAPi& getapi() {
-  static constexpr AdapterSessionTestAPi api =
+const AdapterSessionTestAPI& getapi() {
+  static AdapterSessionTestAPI api =
   {
     AdapterSessionTestSetup,
     AdapterSessionTestTeardown,
@@ -307,5 +301,13 @@ const AdapterSessionTestAPi& getapi() {
     CopyInputAcrossDevices,
     CopyInputAcrossDevices_DML
   };
+
+  if (SkipGpuTests()) {
+    api.AppendExecutionProvider_DML = SkipTest;
+    api.GetExecutionProvider_DML = SkipTest;
+    api.RegisterGraphTransformers_DML = SkipTest;
+    api.RegisterCustomRegistry_DML = SkipTest;
+    api.CopyInputAcrossDevices_DML = SkipTest;
+  }
   return api;
 }

winml/test/adapter/AdapterSessionTest.h

@@ -2,7 +2,7 @@
 // Licensed under the MIT License.
 
 #include "test.h"
-struct AdapterSessionTestAPi
+struct AdapterSessionTestAPI
 {
   SetupClass AdapterSessionTestSetup;
   TeardownClass AdapterSessionTestTeardown;
@@ -21,7 +21,7 @@ struct AdapterSessionTestAPi
   VoidTest CopyInputAcrossDevices;
   VoidTest CopyInputAcrossDevices_DML;
 };
-const AdapterSessionTestAPi& getapi();
+const AdapterSessionTestAPI& getapi();
 
 WINML_TEST_CLASS_BEGIN(AdapterSessionTest)
 WINML_TEST_CLASS_SETUP_CLASS(AdapterSessionTestSetup)

winml/test/api/LearningModelAPITest.cpp

@@ -17,10 +17,6 @@ static void LearningModelAPITestsClassSetup() {
   init_apartment();
 }
 
-static void LearningModelAPITestsGpuMethodSetup() {
-  GPUTEST;
-}
-
 static void CreateModelFromFilePath() {
   LearningModel learningModel = nullptr;
   WINML_EXPECT_NO_THROW(APITest::LoadModel(L"squeezenet_modifiedforruntimestests.onnx", learningModel));
@@ -286,10 +282,9 @@ static void CheckMetadataCaseInsensitive() {
 }
 
 const LearningModelApiTestsApi& getapi() {
-  static constexpr LearningModelApiTestsApi api =
+  static LearningModelApiTestsApi api =
   {
     LearningModelAPITestsClassSetup,
-    LearningModelAPITestsGpuMethodSetup,
     CreateModelFromFilePath,
     CreateModelFromUnicodeFilePath,
     CreateModelFileNotFound,
@@ -309,5 +304,9 @@ const LearningModelApiTestsApi& getapi() {
     CheckMetadataCaseInsensitive,
     CreateCorruptModel
   };
+
+  if (RuntimeParameterExists(L"noVideoFrameTests")) {
+    api.CloseModelCheckEval = SkipTest;
+  }
   return api;
 }

winml/test/api/LearningModelAPITest.h

@@ -5,7 +5,6 @@
 struct LearningModelApiTestsApi
 {
   SetupClass LearningModelAPITestsClassSetup;
-  SetupTest LearningModelAPITestsGpuMethodSetup;
   VoidTest CreateModelFromFilePath;
   VoidTest CreateModelFromUnicodeFilePath;
   VoidTest CreateModelFileNotFound;
@@ -45,13 +44,7 @@ WINML_TEST(LearningModelAPITests, EnumerateInputs)
 WINML_TEST(LearningModelAPITests, EnumerateOutputs)
 WINML_TEST(LearningModelAPITests, CloseModelCheckMetadata)
 WINML_TEST(LearningModelAPITests, CloseModelNoNewSessions)
+WINML_TEST(LearningModelAPITests, CloseModelCheckEval)
 WINML_TEST(LearningModelAPITests, CheckMetadataCaseInsensitive)
 WINML_TEST(LearningModelAPITests, CreateCorruptModel)
-WINML_TEST_CLASS_END()
-
-WINML_TEST_CLASS_BEGIN(LearningModelAPITestsGpu)
-WINML_TEST_CLASS_SETUP_CLASS(LearningModelAPITestsClassSetup)
-WINML_TEST_CLASS_SETUP_METHOD(LearningModelAPITestsGpuMethodSetup)
-WINML_TEST_CLASS_BEGIN_TESTS
-WINML_TEST(LearningModelAPITestsGpu, CloseModelCheckEval)
 WINML_TEST_CLASS_END()

winml/test/api/LearningModelBindingAPITest.cpp

@@ -20,10 +20,6 @@ static void LearningModelBindingAPITestsClassSetup() {
   init_apartment();
 }
 
-static void LearningModelBindingAPITestsGpuMethodSetup() {
-  GPUTEST;
-}
-
 static void CpuSqueezeNet()
 {
     std::string cpuInstance("CPU");
@@ -712,10 +708,9 @@ static void SequenceConstructTensorString()
 }
 
 const LearningModelBindingAPITestsApi& getapi() {
-  static constexpr LearningModelBindingAPITestsApi api =
+  static LearningModelBindingAPITestsApi api =
   {
     LearningModelBindingAPITestsClassSetup,
-    LearningModelBindingAPITestsGpuMethodSetup,
     CpuSqueezeNet,
     CpuSqueezeNetEmptyOutputs,
     CpuSqueezeNetUnboundOutputs,
@@ -737,5 +732,17 @@ const LearningModelBindingAPITestsApi& getapi() {
     SequenceLengthTensorFloat,
     SequenceConstructTensorString
   };
+
+  if (SkipGpuTests()) {
+    api.GpuSqueezeNet = SkipTest;
+    api.GpuSqueezeNetEmptyOutputs = SkipTest;
+    api.GpuSqueezeNetUnboundOutputs = SkipTest;
+  }
+  if (RuntimeParameterExists(L"noVideoFrameTests")) {
+    api.ImageBindingDimensions = SkipTest;
+    api.BindInvalidInputName = SkipTest;
+    api.VerifyOutputAfterImageBindCalledTwice = SkipTest;
+    api.VerifyInvalidBindExceptions = SkipTest;
+  }
   return api;
 }

winml/test/api/LearningModelBindingAPITest.h

@@ -5,7 +5,6 @@
 
 struct LearningModelBindingAPITestsApi {
   SetupClass LearningModelBindingAPITestsClassSetup;
-  SetupTest LearningModelBindingAPITestsGpuMethodSetup;
   VoidTest CpuSqueezeNet;
   VoidTest CpuSqueezeNetEmptyOutputs;
   VoidTest CpuSqueezeNetUnboundOutputs;
@@ -46,16 +45,10 @@ WINML_TEST(LearningModelBindingAPITests, VerifyOutputAfterEvaluateAsyncCalledTwi
 WINML_TEST(LearningModelBindingAPITests, VerifyOutputAfterImageBindCalledTwice)
 WINML_TEST(LearningModelBindingAPITests, SequenceLengthTensorFloat)
 WINML_TEST(LearningModelBindingAPITests, SequenceConstructTensorString)
-WINML_TEST_CLASS_END()
-
-WINML_TEST_CLASS_BEGIN(LearningModelBindingAPITestsGPU)
-WINML_TEST_CLASS_SETUP_CLASS(LearningModelBindingAPITestsClassSetup)
-WINML_TEST_CLASS_SETUP_METHOD(LearningModelBindingAPITestsGpuMethodSetup)
-WINML_TEST_CLASS_BEGIN_TESTS
-WINML_TEST(LearningModelBindingAPITestsGPU, GpuSqueezeNet)
-WINML_TEST(LearningModelBindingAPITestsGPU, GpuSqueezeNetEmptyOutputs)
-WINML_TEST(LearningModelBindingAPITestsGPU, GpuSqueezeNetUnboundOutputs)
-WINML_TEST(LearningModelBindingAPITestsGPU, ImageBindingDimensions)
-WINML_TEST(LearningModelBindingAPITestsGPU, VerifyInvalidBindExceptions)
-WINML_TEST(LearningModelBindingAPITestsGPU, BindInvalidInputName)
+WINML_TEST(LearningModelBindingAPITests, GpuSqueezeNet)
+WINML_TEST(LearningModelBindingAPITests, GpuSqueezeNetEmptyOutputs)
+WINML_TEST(LearningModelBindingAPITests, GpuSqueezeNetUnboundOutputs)
+WINML_TEST(LearningModelBindingAPITests, ImageBindingDimensions)
+WINML_TEST(LearningModelBindingAPITests, VerifyInvalidBindExceptions)
+WINML_TEST(LearningModelBindingAPITests, BindInvalidInputName)
 WINML_TEST_CLASS_END()

winml/test/api/LearningModelSessionAPITest.cpp

@@ -23,15 +23,6 @@ static void LearningModelSessionAPITestsClassSetup() {
   init_apartment();
 }
 
-static void LearningModelSessionAPITestsGpuMethodSetup() {
-  GPUTEST;
-}
-
-static void LearningModelSessionAPITestsGpuSkipEdgeCoreMethodSetup() {
-  LearningModelSessionAPITestsGpuMethodSetup();
-  SKIP_EDGECORE;
-}
-
 static void CreateSessionDeviceDefault()
 {
     LearningModel learningModel = nullptr;
@@ -397,12 +388,10 @@ static void CloseSession()
     });
  }
 
-const LearningModelSesssionAPITestsApi& getapi() {
-  static constexpr LearningModelSesssionAPITestsApi api =
+const LearningModelSessionAPITestsApi& getapi() {
+  static LearningModelSessionAPITestsApi api =
   {
     LearningModelSessionAPITestsClassSetup,
-    LearningModelSessionAPITestsGpuMethodSetup,
-    LearningModelSessionAPITestsGpuSkipEdgeCoreMethodSetup,
     CreateSessionDeviceDefault,
     CreateSessionDeviceCpu,
     CreateSessionWithModelLoadedFromStream,
@@ -418,5 +407,22 @@ const LearningModelSesssionAPITestsApi& getapi() {
     EvaluateSessionAndCloseModel,
     CloseSession,
   };
+
+  if (SkipGpuTests()) {
+    api.CreateSessionDeviceDirectX = SkipTest;
+    api.CreateSessionDeviceDirectXHighPerformance = SkipTest;
+    api.CreateSessionDeviceDirectXMinimumPower = SkipTest;
+    api.CreateSessionWithCastToFloat16InModel = SkipTest;
+    api.DISABLED_CreateSessionWithFloat16InitializersInModel = SkipTest;
+    api.AdapterIdAndDevice = SkipTest;
+  }
+  if (RuntimeParameterExists(L"EdgeCore")) {
+    api.AdapterIdAndDevice = SkipTest;
+  }
+  if (RuntimeParameterExists(L"noIDXGIFactory6Tests")) {
+    api.CreateSessionDeviceDirectXHighPerformance = SkipTest;
+    api.CreateSessionDeviceDirectXMinimumPower = SkipTest;
+    api.AdapterIdAndDevice = SkipTest;
+  }
  return api;
 }

winml/test/api/LearningModelSessionAPITest.h

@@ -3,10 +3,8 @@
 
 #include "test.h"
 
-struct LearningModelSesssionAPITestsApi {
+struct LearningModelSessionAPITestsApi {
   SetupClass LearningModelSessionAPITestsClassSetup;
-  SetupTest LearningModelSessionAPITestsGpuMethodSetup;
-  SetupTest LearningModelSessionAPITestsGpuSkipEdgeCoreMethodSetup;
   VoidTest CreateSessionDeviceDefault;
   VoidTest CreateSessionDeviceCpu;
   VoidTest CreateSessionWithModelLoadedFromStream;
@@ -22,7 +20,7 @@ struct LearningModelSesssionAPITestsApi {
   VoidTest EvaluateSessionAndCloseModel;
   VoidTest CloseSession;
 };
-const LearningModelSesssionAPITestsApi& getapi();
+const LearningModelSessionAPITestsApi& getapi();
 
 WINML_TEST_CLASS_BEGIN(LearningModelSessionAPITests)
 WINML_TEST_CLASS_SETUP_CLASS(LearningModelSessionAPITestsClassSetup)
@@ -34,22 +32,10 @@ WINML_TEST(LearningModelSessionAPITests,EvaluateFeatures)
 WINML_TEST(LearningModelSessionAPITests,EvaluateFeaturesAsync)
 WINML_TEST(LearningModelSessionAPITests,EvaluationProperties)
 WINML_TEST(LearningModelSessionAPITests,EvaluateSessionAndCloseModel)
+WINML_TEST(LearningModelSessionAPITests, CreateSessionDeviceDirectX)
+WINML_TEST(LearningModelSessionAPITests, CreateSessionDeviceDirectXHighPerformance)
+WINML_TEST(LearningModelSessionAPITests, CreateSessionDeviceDirectXMinimumPower)
+WINML_TEST(LearningModelSessionAPITests, CreateSessionWithCastToFloat16InModel)
+WINML_TEST(LearningModelSessionAPITests, DISABLED_CreateSessionWithFloat16InitializersInModel)
+WINML_TEST(LearningModelSessionAPITests, AdapterIdAndDevice)
 WINML_TEST_CLASS_END()
-
-WINML_TEST_CLASS_BEGIN(LearningModelSessionAPITestsGpu)
-WINML_TEST_CLASS_SETUP_CLASS(LearningModelSessionAPITestsClassSetup)
-WINML_TEST_CLASS_SETUP_METHOD(LearningModelSessionAPITestsGpuMethodSetup)
-WINML_TEST_CLASS_BEGIN_TESTS
-WINML_TEST(LearningModelSessionAPITestsGpu, CreateSessionDeviceDirectX)
-WINML_TEST(LearningModelSessionAPITestsGpu, CreateSessionDeviceDirectXHighPerformance)
-WINML_TEST(LearningModelSessionAPITestsGpu, CreateSessionDeviceDirectXMinimumPower)
-WINML_TEST(LearningModelSessionAPITestsGpu, CreateSessionWithCastToFloat16InModel)
-WINML_TEST(LearningModelSessionAPITestsGpu, DISABLED_CreateSessionWithFloat16InitializersInModel)
-WINML_TEST_CLASS_END()
-
-WINML_TEST_CLASS_BEGIN(LearningModelSessionAPITestsGpuSkipEdgeCore)
-WINML_TEST_CLASS_SETUP_CLASS(LearningModelSessionAPITestsClassSetup)
-WINML_TEST_CLASS_SETUP_METHOD(LearningModelSessionAPITestsGpuSkipEdgeCoreMethodSetup)
-WINML_TEST_CLASS_BEGIN_TESTS
-WINML_TEST(LearningModelSessionAPITestsGpuSkipEdgeCore, AdapterIdAndDevice)
-WINML_TEST_CLASS_END()

winml/test/api/RawApiTestsGpu.cpp

@@ -104,10 +104,6 @@ static void RawApiTestsGpuApiTestsClassSetup() {
   RoInitialize(RO_INIT_TYPE::RO_INIT_SINGLETHREADED);
 }
 
-static void GpuMethodSetup() {
-  GPUTEST;
-}
-
 static void CreateDirectXDevice() {
   WINML_EXPECT_NO_THROW(CreateDevice(DeviceType::DirectX));
 }
@@ -155,9 +151,8 @@ static void EvaluateNoInputCopy() {
 }
 
 const RawApiTestsGpuApi& getapi() {
-  static constexpr RawApiTestsGpuApi api = {
+  static RawApiTestsGpuApi api = {
       RawApiTestsGpuApiTestsClassSetup,
-      GpuMethodSetup,
       CreateDirectXDevice,
       CreateD3D11DeviceDevice,
       CreateD3D12CommandQueueDevice,
@@ -166,5 +161,15 @@ const RawApiTestsGpuApi& getapi() {
       Evaluate,
       EvaluateNoInputCopy
   };
+
+  if (SkipGpuTests()) {
+    api.CreateDirectXDevice = SkipTest;
+    api.CreateD3D11DeviceDevice = SkipTest;
+    api.CreateD3D12CommandQueueDevice = SkipTest;
+    api.CreateDirectXHighPerformanceDevice = SkipTest;
+    api.CreateDirectXMinPowerDevice = SkipTest;
+    api.Evaluate = SkipTest;
+    api.EvaluateNoInputCopy = SkipTest;
+  }
   return api;
 }