From 0a6fd1a5beef942f5025f6675bfca984ee26b0d0 Mon Sep 17 00:00:00 2001 From: ankitm3k Date: Wed, 11 Dec 2024 20:45:58 +0530 Subject: [PATCH 1/3] update: Update MSFT Contrib Ops from OV --- .../core/providers/openvino/ov_versions/data_ops.cc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc index 2f0dd458cc349..23b430b763a62 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc +++ b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc @@ -95,8 +95,10 @@ std::vector supported_op_mode = { {"Atan", V_2020_4, {"CPU", "GPU"}}, {"Atanh", V_2020_4, {"CPU"}}, {"Atanh", V_2022_1, {"GPU"}}, + {"Attention", V_2024_5, {"CPU","GPU","NPU"}}, {"AveragePool", V_2020_4, {"CPU", "GPU"}}, {"BatchNormalization", V_2020_4, {"CPU", "GPU"}}, + {"BiasGelu", V_2024_5, {"CPU","GPU","NPU"}}, {"BitShift", V_2022_1, {"CPU"}}, {"Cast", V_2020_4, {"CPU", "GPU"}}, {"CastLike", V_2023_1, {"CPU", "GPU"}}, @@ -123,6 +125,7 @@ std::vector supported_op_mode = { {"Dropout", V_2020_4, {"CPU", "GPU"}}, {"Elu", V_2020_4, {"CPU", "GPU"}}, {"Einsum", V_2023_1, {"CPU", "GPU"}}, + {"EmbedLayerNormalization", V_2024_5, {"CPU","GPU","NPU"}}, {"EPContext", V_2024_0, {"CPU", "GPU", "NPU"}}, {"Equal", V_2020_4, {"CPU", "GPU"}}, {"Erf", V_2020_4, {"CPU", "GPU"}}, @@ -131,6 +134,9 @@ std::vector supported_op_mode = { {"EyeLike", V_2022_1, {"CPU"}}, {"Flatten", V_2020_4, {"CPU", "GPU"}}, {"Floor", V_2020_4, {"CPU", "GPU"}}, + {"FusedConv", V_2024_5, {"CPU","GPU","NPU"}}, + {"FusedGemm", V_2024_5, {"CPU","GPU","NPU"}}, + {"FusedMatMul", V_2025_0, {"CPU","GPU","NPU"}}, {"Gather", V_2020_4, {"CPU", "GPU"}}, {"GatherElements", V_2022_2, {"CPU", "GPU"}}, {"GatherND", V_2021_4, {"CPU", "GPU"}}, @@ -164,6 +170,7 @@ std::vector supported_op_mode = { {"LSTM", V_2020_4, {"CPU", "GPU"}}, {"MatMul", V_2020_4, {"CPU", "GPU"}}, {"MatMulInteger", V_2022_1, {"CPU"}}, + {"MatMulNBits", V_2024_5, {"CPU","GPU","NPU"}}, {"Max", V_2020_4, {"CPU", "GPU"}}, {"MaxPool", V_2020_4, {"CPU", "GPU"}}, {"Mean", V_2020_4, {"CPU", "GPU"}}, @@ -184,6 +191,7 @@ std::vector supported_op_mode = { {"PRelu", V_2020_4, {"CPU", "GPU"}}, {"QLinearMatMul", V_2022_3, {"CPU"}}, {"QuantizeLinear", V_2021_4, {"CPU", "GPU"}}, + {"QuickGelu", V_2025_0, {"CPU","GPU","NPU"}}, {"RNN", V_2023_1, {"CPU", "GPU"}}, {"RandomNormalLike", V_2023_0, {"CPU", "GPU"}}, {"RandomNormalLike", V_2023_0, {"CPU", "GPU"}}, @@ -219,9 +227,12 @@ std::vector supported_op_mode = { {"Sigmoid", V_2020_4, {"CPU", "GPU"}}, {"Sign", V_2020_4, {"CPU"}}, {"Sign", V_2022_1, {"GPU"}}, + {"SimplifiedLayerNormalization", V_2025_0, {"CPU","GPU","NPU"}}, {"Sin", V_2022_1, {"CPU", "GPU"}}, {"Sinh", V_2020_4, {"CPU"}}, {"Size", V_2022_1, {"CPU", "GPU"}}, + {"SkipLayerNormalization", V_2024_5, {"CPU","GPU","NPU"}}, + {"SkipSimplifiedLayerNormalization", V_2025_0, {"CPU","GPU","NPU"}}, {"Slice", V_2020_4, {"CPU", "GPU"}}, {"Softmax", V_2020_4, {"CPU", "GPU"}}, {"Softplus", V_2022_1, {"CPU", "GPU"}}, From f8a53305731ce349b4f4642345d79a426473ded0 Mon Sep 17 00:00:00 2001 From: n1harika Date: Fri, 24 Jan 2025 18:15:24 +0530 Subject: [PATCH 2/3] modified data_ops.cc to remove unsupported ops --- .../providers/openvino/ov_versions/data_ops.cc | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc index 23b430b763a62..fd71eebfe533a 100644 --- a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc +++ b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc @@ -95,10 +95,10 @@ std::vector supported_op_mode = { {"Atan", V_2020_4, {"CPU", "GPU"}}, {"Atanh", V_2020_4, {"CPU"}}, {"Atanh", V_2022_1, {"GPU"}}, - {"Attention", V_2024_5, {"CPU","GPU","NPU"}}, + {"Attention", V_2023_0, {"CPU", "GPU"}}, {"AveragePool", V_2020_4, {"CPU", "GPU"}}, {"BatchNormalization", V_2020_4, {"CPU", "GPU"}}, - {"BiasGelu", V_2024_5, {"CPU","GPU","NPU"}}, + {"BiasGelu", V_2023_0, {"CPU", "GPU"}}, {"BitShift", V_2022_1, {"CPU"}}, {"Cast", V_2020_4, {"CPU", "GPU"}}, {"CastLike", V_2023_1, {"CPU", "GPU"}}, @@ -125,7 +125,7 @@ std::vector supported_op_mode = { {"Dropout", V_2020_4, {"CPU", "GPU"}}, {"Elu", V_2020_4, {"CPU", "GPU"}}, {"Einsum", V_2023_1, {"CPU", "GPU"}}, - {"EmbedLayerNormalization", V_2024_5, {"CPU","GPU","NPU"}}, + {"EmbedLayerNormalization", V_2024_5, {"CPU", "GPU"}}, {"EPContext", V_2024_0, {"CPU", "GPU", "NPU"}}, {"Equal", V_2020_4, {"CPU", "GPU"}}, {"Erf", V_2020_4, {"CPU", "GPU"}}, @@ -134,9 +134,8 @@ std::vector supported_op_mode = { {"EyeLike", V_2022_1, {"CPU"}}, {"Flatten", V_2020_4, {"CPU", "GPU"}}, {"Floor", V_2020_4, {"CPU", "GPU"}}, - {"FusedConv", V_2024_5, {"CPU","GPU","NPU"}}, - {"FusedGemm", V_2024_5, {"CPU","GPU","NPU"}}, - {"FusedMatMul", V_2025_0, {"CPU","GPU","NPU"}}, + {"FusedConv", V_2023_0, {"CPU", "GPU"}}, + {"FusedGemm", V_2023_0, {"CPU", "GPU"}}, {"Gather", V_2020_4, {"CPU", "GPU"}}, {"GatherElements", V_2022_2, {"CPU", "GPU"}}, {"GatherND", V_2021_4, {"CPU", "GPU"}}, @@ -170,7 +169,7 @@ std::vector supported_op_mode = { {"LSTM", V_2020_4, {"CPU", "GPU"}}, {"MatMul", V_2020_4, {"CPU", "GPU"}}, {"MatMulInteger", V_2022_1, {"CPU"}}, - {"MatMulNBits", V_2024_5, {"CPU","GPU","NPU"}}, + {"MatMulNBits", V_2024_5, {"CPU", "GPU"}}, {"Max", V_2020_4, {"CPU", "GPU"}}, {"MaxPool", V_2020_4, {"CPU", "GPU"}}, {"Mean", V_2020_4, {"CPU", "GPU"}}, @@ -191,7 +190,6 @@ std::vector supported_op_mode = { {"PRelu", V_2020_4, {"CPU", "GPU"}}, {"QLinearMatMul", V_2022_3, {"CPU"}}, {"QuantizeLinear", V_2021_4, {"CPU", "GPU"}}, - {"QuickGelu", V_2025_0, {"CPU","GPU","NPU"}}, {"RNN", V_2023_1, {"CPU", "GPU"}}, {"RandomNormalLike", V_2023_0, {"CPU", "GPU"}}, {"RandomNormalLike", V_2023_0, {"CPU", "GPU"}}, @@ -227,12 +225,10 @@ std::vector supported_op_mode = { {"Sigmoid", V_2020_4, {"CPU", "GPU"}}, {"Sign", V_2020_4, {"CPU"}}, {"Sign", V_2022_1, {"GPU"}}, - {"SimplifiedLayerNormalization", V_2025_0, {"CPU","GPU","NPU"}}, {"Sin", V_2022_1, {"CPU", "GPU"}}, {"Sinh", V_2020_4, {"CPU"}}, {"Size", V_2022_1, {"CPU", "GPU"}}, - {"SkipLayerNormalization", V_2024_5, {"CPU","GPU","NPU"}}, - {"SkipSimplifiedLayerNormalization", V_2025_0, {"CPU","GPU","NPU"}}, + {"SkipLayerNormalization", V_2024_5, {"CPU", "GPU"}}, {"Slice", V_2020_4, {"CPU", "GPU"}}, {"Softmax", V_2020_4, {"CPU", "GPU"}}, {"Softplus", V_2022_1, {"CPU", "GPU"}}, From 875d055d247382fe7d34c8ebdef60b37fb914010 Mon Sep 17 00:00:00 2001 From: n1harika Date: Mon, 10 Feb 2025 18:42:30 +0530 Subject: [PATCH 3/3] disabled tests for EmbedLayerNormalisation and MatMulNBits --- onnxruntime/test/contrib_ops/embed_layer_norm_op_test.cc | 4 ++++ onnxruntime/test/contrib_ops/matmul_4bits_test.cc | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/onnxruntime/test/contrib_ops/embed_layer_norm_op_test.cc b/onnxruntime/test/contrib_ops/embed_layer_norm_op_test.cc index 0f35a7ff4b36f..043717a9f6e92 100644 --- a/onnxruntime/test/contrib_ops/embed_layer_norm_op_test.cc +++ b/onnxruntime/test/contrib_ops/embed_layer_norm_op_test.cc @@ -158,7 +158,11 @@ static void RunTest(const embedlayernorm::OpData& data, execution_providers.push_back(DefaultDmlExecutionProvider()); tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } else { +#if defined(USE_OPENVINO) + tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); +#else tester.Run(); +#endif } } } diff --git a/onnxruntime/test/contrib_ops/matmul_4bits_test.cc b/onnxruntime/test/contrib_ops/matmul_4bits_test.cc index 9bf08c6350833..b1779ded4a675 100644 --- a/onnxruntime/test/contrib_ops/matmul_4bits_test.cc +++ b/onnxruntime/test/contrib_ops/matmul_4bits_test.cc @@ -85,7 +85,7 @@ struct TestOptions { std::optional output_rel_error{}; }; -std::ostream& operator<<(std::ostream& os, const TestOptions& opts) { +[[maybe_unused]] std::ostream& operator<<(std::ostream& os, const TestOptions& opts) { return os << "M:" << opts.M << ", N:" << opts.N << ", K:" << opts.K << ", block_size:" << opts.block_size << ", accuracy_level:" << opts.accuracy_level @@ -327,6 +327,8 @@ void TestMatMulNBitsTyped() { #endif // !defined(USE_DML) && !defined(USE_WEBGPU) } +#if !defined(USE_OPENVINO) + TEST(MatMulNBits, Float32_Accuracy0) { TestMatMulNBitsTyped(); TestMatMulNBitsTyped(); @@ -462,6 +464,7 @@ TEST(MatMulNBits, Float16_Accuracy4) { } #endif #endif +#endif #if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_DML) || defined(USE_WEBGPU)