From c45d0cb86869d6040ff91bfbbf5f7c0f70aca7a3 Mon Sep 17 00:00:00 2001 From: "Yalachigere, Sreekanth V" Date: Sun, 9 Dec 2018 21:58:54 -0800 Subject: [PATCH 01/56] Filter data (Weights) reorder optimization --- .../mkldnn/mkldnn_execution_provider.h | 13 +++++++ onnxruntime/core/providers/mkldnn/nn/conv.cc | 35 +++++++++++-------- onnxruntime/core/providers/mkldnn/nn/conv.h | 8 +++++ 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h index e5820f8951514..13685856e8e3a 100644 --- a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h +++ b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h @@ -4,10 +4,12 @@ #pragma once #include +#include #include "core/framework/allocatormgr.h" #include "core/framework/execution_provider.h" #include "core/graph/graph_transformer.h" +#include "mkldnn.hpp" namespace onnxruntime { @@ -37,6 +39,17 @@ class MKLDNNExecutionProvider : public IExecutionProvider { } virtual std::shared_ptr GetKernelRegistry() const override; + + std::shared_ptr GetWeightMemory(std::string weightName) { + if (weights_mem_map.find(weightName) != weights_mem_map.end()) + return weights_mem_map[weightName]; + else + return nullptr; + } +public: + // mkldnn formatted weights(filer data) memory from first iteration + // saved by weights name + std::map> weights_mem_map; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.cc b/onnxruntime/core/providers/mkldnn/nn/conv.cc index 355562bf5adcd..edd85eb17a675 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.cc +++ b/onnxruntime/core/providers/mkldnn/nn/conv.cc @@ -402,20 +402,27 @@ Status Conv::Compute(OpKernelContext* context) const { src_data = static_cast(dst.get_data_handle()); } - // Reorder filter memory layout if necessary. - if (filter_format != conv_primitive->GetFilterMemoryFormat()) { - auto pd = mkldnn::memory::primitive_desc(mkldnn::memory::desc(filter_dims_mkl, - MklDnnType(), - filter_format), - cpu_engine); - mkldnn::memory src = mkldnn::memory(pd, (void*)filter_data); - // allocate the size queried from memory primitive desc. it may not match tensor logical size due to - // mkldnn using padding to allow use of blocked format. - filter_reorder_buffer = IAllocator::MakeUniquePtr(alloc, conv_primitive->GetFilterSize()); - mkldnn::memory dst = mkldnn::memory(conv_fwd_pd->weights_primitive_desc(), filter_reorder_buffer.get()); - MemoryReorderParams params(src, dst); - DoReorder(params); - filter_data = static_cast(dst.get_data_handle()); + // Reorder filter memory layout if necessary + // Avoid data reordering. Save filter memory in mkldnn format from first iteration + // in execution provider mapped by weight name. + std::string weightName = OpKernel::Node().InputDefs()[1]->Name(); + std::shared_ptr filter_dst_mem = provider_->GetWeightMemory(weightName); + + if (filter_dst_mem == nullptr) { + if (filter_format != conv_primitive->GetFilterMemoryFormat()) { + auto pd = mkldnn::memory::primitive_desc(mkldnn::memory::desc( + filter_dims_mkl, MklDnnType(), filter_format), cpu_engine); + mkldnn::memory src = mkldnn::memory(pd, (void*)filter_data); + filter_reorder_buffer = IAllocator::MakeUniquePtr(alloc, conv_primitive->GetFilterSize()); + filter_dst_mem.reset( + new mkldnn::memory(conv_fwd_pd->weights_primitive_desc(), filter_reorder_buffer.get())); + MemoryReorderParams params(src, *filter_dst_mem); + DoReorder(params); + filter_data = static_cast(filter_dst_mem->get_data_handle()); + provider_->weights_mem_map[weightName] = filter_dst_mem; + } + } else { + filter_data = static_cast(filter_dst_mem->get_data_handle()); } // Allocate dst buffer if reorder is necessary diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.h b/onnxruntime/core/providers/mkldnn/nn/conv.h index 8189a7ca6ba10..7059f81fc2fe1 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.h +++ b/onnxruntime/core/providers/mkldnn/nn/conv.h @@ -4,18 +4,26 @@ #pragma once #include "core/framework/op_kernel.h" #include "core/providers/cpu/nn/conv.h" +#include "../mkldnn_execution_provider.h" namespace onnxruntime { namespace mkl_dnn { + template class Conv final : public onnxruntime::Conv { public: Conv(const OpKernelInfo& info) : onnxruntime::Conv(info) { + if (info.GetExecutionProvider()->Type() == kMklDnnExecutionProvider) { + provider_ = (const_cast( + dynamic_cast(info.GetExecutionProvider()))); + } } Status Compute(OpKernelContext* context) const override; private: + MKLDNNExecutionProvider * provider_; + }; } // namespace mkl_dnn } // namespace onnxruntime From 8904b82059ee3f8cc4ad33933ee343ac7904d3f9 Mon Sep 17 00:00:00 2001 From: "Yalachigere, Sreekanth V" Date: Mon, 10 Dec 2018 10:31:25 -0800 Subject: [PATCH 02/56] check provider_ for nullptr --- onnxruntime/core/providers/mkldnn/nn/conv.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.cc b/onnxruntime/core/providers/mkldnn/nn/conv.cc index edd85eb17a675..f6de43b6a504e 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.cc +++ b/onnxruntime/core/providers/mkldnn/nn/conv.cc @@ -406,7 +406,9 @@ Status Conv::Compute(OpKernelContext* context) const { // Avoid data reordering. Save filter memory in mkldnn format from first iteration // in execution provider mapped by weight name. std::string weightName = OpKernel::Node().InputDefs()[1]->Name(); - std::shared_ptr filter_dst_mem = provider_->GetWeightMemory(weightName); + std::shared_ptr filter_dst_mem = nullptr; + if(provider_ != nullptr) + filter_dst_mem = provider_->GetWeightMemory(weightName); if (filter_dst_mem == nullptr) { if (filter_format != conv_primitive->GetFilterMemoryFormat()) { @@ -419,7 +421,8 @@ Status Conv::Compute(OpKernelContext* context) const { MemoryReorderParams params(src, *filter_dst_mem); DoReorder(params); filter_data = static_cast(filter_dst_mem->get_data_handle()); - provider_->weights_mem_map[weightName] = filter_dst_mem; + if (provider_ != nullptr) + provider_->weights_mem_map[weightName] = filter_dst_mem; } } else { filter_data = static_cast(filter_dst_mem->get_data_handle()); From e8cdd6350b5710099759b85b10c06dadc740c501 Mon Sep 17 00:00:00 2001 From: "Yalachigere, Sreekanth V" Date: Mon, 10 Dec 2018 16:17:48 -0800 Subject: [PATCH 03/56] PR Review changes: thread safe. weights map object private --- .../providers/mkldnn/mkldnn_execution_provider.h | 16 +++++++++++----- onnxruntime/core/providers/mkldnn/nn/conv.cc | 10 ++++++---- onnxruntime/core/providers/mkldnn/nn/conv.h | 4 +--- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h index 13685856e8e3a..911dd2a1eb22f 100644 --- a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h +++ b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h @@ -41,12 +41,18 @@ class MKLDNNExecutionProvider : public IExecutionProvider { virtual std::shared_ptr GetKernelRegistry() const override; std::shared_ptr GetWeightMemory(std::string weightName) { - if (weights_mem_map.find(weightName) != weights_mem_map.end()) - return weights_mem_map[weightName]; - else - return nullptr; + + auto iter = weights_mem_map.find(weightName); + if (iter != weights_mem_map.end()) + return iter->second; + return nullptr; + } + + std::map>& GetWeightsMap() { + return weights_mem_map; } -public: + +private: // mkldnn formatted weights(filer data) memory from first iteration // saved by weights name std::map> weights_mem_map; diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.cc b/onnxruntime/core/providers/mkldnn/nn/conv.cc index f6de43b6a504e..81ad0b4f1c9e3 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.cc +++ b/onnxruntime/core/providers/mkldnn/nn/conv.cc @@ -407,8 +407,7 @@ Status Conv::Compute(OpKernelContext* context) const { // in execution provider mapped by weight name. std::string weightName = OpKernel::Node().InputDefs()[1]->Name(); std::shared_ptr filter_dst_mem = nullptr; - if(provider_ != nullptr) - filter_dst_mem = provider_->GetWeightMemory(weightName); + filter_dst_mem = provider_->GetWeightMemory(weightName); if (filter_dst_mem == nullptr) { if (filter_format != conv_primitive->GetFilterMemoryFormat()) { @@ -421,8 +420,11 @@ Status Conv::Compute(OpKernelContext* context) const { MemoryReorderParams params(src, *filter_dst_mem); DoReorder(params); filter_data = static_cast(filter_dst_mem->get_data_handle()); - if (provider_ != nullptr) - provider_->weights_mem_map[weightName] = filter_dst_mem; + { + // make assignment threadsafe + std::lock_guard lock(mutex_); + provider_->GetWeightsMap()[weightName] = filter_dst_mem; + } } } else { filter_data = static_cast(filter_dst_mem->get_data_handle()); diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.h b/onnxruntime/core/providers/mkldnn/nn/conv.h index 7059f81fc2fe1..8dde0c899f577 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.h +++ b/onnxruntime/core/providers/mkldnn/nn/conv.h @@ -13,17 +13,15 @@ template class Conv final : public onnxruntime::Conv { public: Conv(const OpKernelInfo& info) : onnxruntime::Conv(info) { - if (info.GetExecutionProvider()->Type() == kMklDnnExecutionProvider) { provider_ = (const_cast( dynamic_cast(info.GetExecutionProvider()))); - } } Status Compute(OpKernelContext* context) const override; private: MKLDNNExecutionProvider * provider_; - + mutable std::mutex mutex_; }; } // namespace mkl_dnn } // namespace onnxruntime From f95cea90f976ed6c0cdf39b4abe1a50096b3fbfb Mon Sep 17 00:00:00 2001 From: "Yalachigere, Sreekanth V" Date: Tue, 11 Dec 2018 09:38:33 -0800 Subject: [PATCH 04/56] using Conv Parameters key to make weights id unique --- onnxruntime/core/providers/mkldnn/nn/conv.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.cc b/onnxruntime/core/providers/mkldnn/nn/conv.cc index 81ad0b4f1c9e3..831c7b620d620 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.cc +++ b/onnxruntime/core/providers/mkldnn/nn/conv.cc @@ -352,6 +352,9 @@ Status Conv::Compute(OpKernelContext* context) const { dst_dims_mkl, strides_mkl, dilations_mkl, padding_left_mkl, padding_right_mkl); ConvPrimitive* conv_primitive = ConvPrimitivePool::Get(conv_params); + + std::string convString = conv_params.ToString(); + auto conv_fwd_pd = conv_primitive->GetPrimitiveDesc(); mkldnn::engine& cpu_engine = GetEngine(); @@ -405,9 +408,9 @@ Status Conv::Compute(OpKernelContext* context) const { // Reorder filter memory layout if necessary // Avoid data reordering. Save filter memory in mkldnn format from first iteration // in execution provider mapped by weight name. - std::string weightName = OpKernel::Node().InputDefs()[1]->Name(); + std::string weightKey = convString + "-" + OpKernel::Node().InputDefs()[1]->Name(); std::shared_ptr filter_dst_mem = nullptr; - filter_dst_mem = provider_->GetWeightMemory(weightName); + filter_dst_mem = provider_->GetWeightMemory(weightKey); if (filter_dst_mem == nullptr) { if (filter_format != conv_primitive->GetFilterMemoryFormat()) { @@ -423,7 +426,7 @@ Status Conv::Compute(OpKernelContext* context) const { { // make assignment threadsafe std::lock_guard lock(mutex_); - provider_->GetWeightsMap()[weightName] = filter_dst_mem; + provider_->GetWeightsMap()[weightKey] = filter_dst_mem; } } } else { From d0f15b1ba57831a2a32e361ec71366c8294d4421 Mon Sep 17 00:00:00 2001 From: "Yalachigere, Sreekanth V" Date: Mon, 10 Dec 2018 16:17:48 -0800 Subject: [PATCH 05/56] PR Review changes: thread safe. weights map object private --- onnxruntime/core/providers/mkldnn/nn/conv.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.cc b/onnxruntime/core/providers/mkldnn/nn/conv.cc index 831c7b620d620..e452937e4d499 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.cc +++ b/onnxruntime/core/providers/mkldnn/nn/conv.cc @@ -408,7 +408,7 @@ Status Conv::Compute(OpKernelContext* context) const { // Reorder filter memory layout if necessary // Avoid data reordering. Save filter memory in mkldnn format from first iteration // in execution provider mapped by weight name. - std::string weightKey = convString + "-" + OpKernel::Node().InputDefs()[1]->Name(); + std::string weightKey = OpKernel::Node().InputDefs()[1]->Name(); std::shared_ptr filter_dst_mem = nullptr; filter_dst_mem = provider_->GetWeightMemory(weightKey); From cf57d17c31f3d1baf205cd854557cc027c05c625 Mon Sep 17 00:00:00 2001 From: "Yalachigere, Sreekanth V" Date: Wed, 12 Dec 2018 09:59:33 -0800 Subject: [PATCH 06/56] removed convParam string from weight key. Weight Id is unique --- onnxruntime/core/providers/mkldnn/nn/conv.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.cc b/onnxruntime/core/providers/mkldnn/nn/conv.cc index e452937e4d499..db3e0e55017de 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.cc +++ b/onnxruntime/core/providers/mkldnn/nn/conv.cc @@ -353,8 +353,6 @@ Status Conv::Compute(OpKernelContext* context) const { padding_left_mkl, padding_right_mkl); ConvPrimitive* conv_primitive = ConvPrimitivePool::Get(conv_params); - std::string convString = conv_params.ToString(); - auto conv_fwd_pd = conv_primitive->GetPrimitiveDesc(); mkldnn::engine& cpu_engine = GetEngine(); From 5a8acd7da8b28a8380829ba859d9d61a02af69b3 Mon Sep 17 00:00:00 2001 From: Ke Zhang Date: Mon, 17 Dec 2018 13:30:16 -0800 Subject: [PATCH 07/56] directly updating dst arg with src arg when adding an edge. no need updating the type and shape info. (#195) --- onnxruntime/core/graph/graph.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/onnxruntime/core/graph/graph.cc b/onnxruntime/core/graph/graph.cc index b593e0b5bb656..08b0a1d6e0b9d 100644 --- a/onnxruntime/core/graph/graph.cc +++ b/onnxruntime/core/graph/graph.cc @@ -830,7 +830,6 @@ void Graph::AddEdge(NodeIndex src_node_index, NodeIndex dst_node_index, int src_ // The output type of source node arg does not match the input type of destination node arg. ORT_THROW("Argument type mismatch when adding edge."); } else { - src_arg->UpdateTypeAndShape(*dst_arg); *dst_arg_pointer = src_arg; } } From 82d04412a0bdfe713a928ce970cfa84b91bf5ffc Mon Sep 17 00:00:00 2001 From: Ke Zhang Date: Mon, 17 Dec 2018 13:30:29 -0800 Subject: [PATCH 08/56] Kezhan/partition logic update (#164) * add check before fusing sub-graph in greedy partitioning * update the partitioning logic to 1) not fuse sub-graph if inner nodes were assigned 2) avoid resolving graph after each provider capability checking and assignment. * resolve conflicts --- .../core/framework/graph_partitioner.cc | 66 +++++++++++++------ 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/onnxruntime/core/framework/graph_partitioner.cc b/onnxruntime/core/framework/graph_partitioner.cc index 203cd365275c7..7ca96978f9960 100644 --- a/onnxruntime/core/framework/graph_partitioner.cc +++ b/onnxruntime/core/framework/graph_partitioner.cc @@ -55,6 +55,11 @@ KernelDefBuilder& BuildFusedKernelDef(KernelDefBuilder& builder, const onnxrunti } Status GraphPartitioner::Partition(onnxruntime::Graph& graph) const { + // It is a greedy partitioning algorithm per provider preferences user provided when calling ONNX RUNTIME right now. + // 1. Execution providers' capabilities are checked one by one. + // 2. All sub-graphs that an execution provider returns will be assigned to it if it's not assigned yet. + // 3. CPU execution provider is expected to be able to run any node and is the last one in execution provider preference. + if (providers_.Empty()) { return Status(ONNXRUNTIME, INVALID_ARGUMENT, "No provider specified."); } @@ -63,10 +68,17 @@ Status GraphPartitioner::Partition(onnxruntime::Graph& graph) const { std::shared_ptr fused_kernel_registry = std::make_shared(); // Partitioning based on provider preference and their capabilities. auto kernel_registries = kernel_registry_mgr_.GetAllKernelRegistries(); + + std::vector>> capabilities_of_all_providers; + GraphViewer graph_viewer(graph); + for (auto& provider : providers_) { + capabilities_of_all_providers.push_back(provider->GetCapability(graph_viewer, kernel_registries)); + } + + int i = 0; for (auto& provider : providers_) { - auto capability_results = provider->GetCapability(GraphViewer(graph), kernel_registries); int count = 0; - for (auto& capability : capability_results) { + for (auto& capability : capabilities_of_all_providers[i++]) { if (nullptr == capability || nullptr == capability->sub_graph) { continue; } @@ -78,30 +90,44 @@ Status GraphPartitioner::Partition(onnxruntime::Graph& graph) const { auto node = graph.GetNode(capability->sub_graph->nodes[0]); if (nullptr != node && node->GetExecutionProviderType().empty()) { + // The node was not fused or assigned. Assign it to this . node->SetExecutionProviderType(provider->Type()); } } else { // The can run a fused in the . - // - // Add fused node into ORT_ENFORCE(nullptr != capability->sub_graph->GetMetaDef()); - std::string node_name = provider->Type() + "_" + capability->sub_graph->GetMetaDef()->name + "_" + std::to_string(count++); - auto& fused_node = graph.FuseSubGraph(std::move(capability->sub_graph), node_name); - fused_node.SetExecutionProviderType(provider->Type()); - auto fused_kernel_func = capability->fuse_kernel_function; - if (fused_kernel_func != nullptr) { - // build the kernel definition on the fly, and register it to the fused_kernel_regisitry. - KernelDefBuilder builder; - BuildFusedKernelDef(builder, fused_node); - fused_kernel_registry->Register(builder, fused_kernel_func); + + // Check whether any node in the was already assigned. + bool sub_graph_available_for_assignment = true; + for (auto node_index : capability->sub_graph->nodes) { + auto node = graph.GetNode(node_index); + if (nullptr == node || !node->GetExecutionProviderType().empty()) { + // The node was fused or assigned, so that the whole sub-graph will not be assigned to this + // The assumption is that this can only run the sub-graph as a whole unit. + sub_graph_available_for_assignment = false; + break; + } + } + + if (sub_graph_available_for_assignment) { + // Add fused node into + std::string node_name = provider->Type() + "_" + capability->sub_graph->GetMetaDef()->name + "_" + std::to_string(count++); + auto& fused_node = graph.FuseSubGraph(std::move(capability->sub_graph), node_name); + fused_node.SetExecutionProviderType(provider->Type()); + auto fused_kernel_func = capability->fuse_kernel_function; + if (fused_kernel_func != nullptr) { + // build the kernel definition on the fly, and register it to the fused_kernel_regisitry. + KernelDefBuilder builder; + BuildFusedKernelDef(builder, fused_node); + fused_kernel_registry->Register(builder, fused_kernel_func); + } } } } - // all done with this provider, resolve the graph before we move on to the next provider. - // This is needed since we create a new GraphViewer() that we pass into the next provider's GetCapability(). - ORT_ENFORCE(graph.Resolve().IsOK()); } + ORT_ENFORCE(graph.Resolve().IsOK()); + // To see if the node with no provider can be inlined. If one such nodes can be // successfully inlined, we re-run the partitioner on the modified graph. bool inline_flag = false; @@ -126,10 +152,10 @@ Status GraphPartitioner::Partition(onnxruntime::Graph& graph) const { this->Partition(graph); } - //For some cases, like fp16 on cpu, right now we don't have any kernel support that. - //But we will insert cast op to run the model, so skip the error checking here. - //If after graph transform phase, the node still not assigned, we will report error - //during kernel creation phase. + //For some cases, like fp16 on cpu, right now we don't have any kernel support that. + //But we will insert cast op to run the model, so skip the error checking here. + //If after graph transform phase, the node still not assigned, we will report error + //during kernel creation phase. #ifdef COUNT_NON_CUDA_OPS for (auto& node : graph.Nodes()) { if (node.GetExecutionProviderType() != kCudaExecutionProvider && From d0544a80824de2fd3d18cc60dd39f99f2c781d9d Mon Sep 17 00:00:00 2001 From: Randy <45701928+RandyShuai@users.noreply.github.com> Date: Mon, 17 Dec 2018 13:47:20 -0800 Subject: [PATCH 09/56] Rashuai/gathernd op (#170) * define gather_nd op * add test cases * add test file * refactor the code and doc * add test cases * fix win compile err * fix win compile err * adjust indent * make constructor explicit * add coment * remove templates * remove wrong def * migrate macros * fix an issue in shape inference --- onnxruntime/contrib_ops/contrib_kernels.cc | 2 + onnxruntime/contrib_ops/cpu/gather_nd.cc | 114 +++++++++++++ onnxruntime/contrib_ops/cpu/gather_nd.h | 49 ++++++ .../core/graph/contrib_ops/contrib_defs.cc | 65 ++++++++ .../test/contrib_ops/gather_nd_op_test.cc | 157 ++++++++++++++++++ 5 files changed, 387 insertions(+) create mode 100644 onnxruntime/contrib_ops/cpu/gather_nd.cc create mode 100644 onnxruntime/contrib_ops/cpu/gather_nd.h create mode 100644 onnxruntime/test/contrib_ops/gather_nd_op_test.cc diff --git a/onnxruntime/contrib_ops/contrib_kernels.cc b/onnxruntime/contrib_ops/contrib_kernels.cc index 11b7e653f3058..a8f8876e771a2 100644 --- a/onnxruntime/contrib_ops/contrib_kernels.cc +++ b/onnxruntime/contrib_ops/contrib_kernels.cc @@ -17,6 +17,7 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, string, StringNormalizer); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, NonMaxSuppression); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, Range); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, GatherND); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, MurmurHash3); void RegisterContribKernels(std::function fn) { @@ -33,6 +34,7 @@ void RegisterContribKernels(std::function fn) { fn(BuildKernel()); fn(BuildKernel()); fn(BuildKernel()); + fn(BuildKernel()); fn(BuildKernel()); } } // namespace contrib diff --git a/onnxruntime/contrib_ops/cpu/gather_nd.cc b/onnxruntime/contrib_ops/cpu/gather_nd.cc new file mode 100644 index 0000000000000..f13794aa6b735 --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/gather_nd.cc @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "contrib_ops/cpu/gather_nd.h" + +namespace onnxruntime { +namespace contrib { + +ONNX_OPERATOR_KERNEL_EX( + GatherND, + kMSDomain, + 1, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::AllTensorTypes()) + .TypeConstraint("Tind", {DataTypeImpl::GetTensorType(),DataTypeImpl::GetTensorType()}), + GatherND); + +template +Status GatherNDBase::PrepareForCompute(OpKernelContext* context, Prepare& p) const { + + auto input_tensor = context->Input(0); + auto indice_tensor = context->Input(1); + ORT_ENFORCE(input_tensor != nullptr); + ORT_ENFORCE(indice_tensor != nullptr); + + auto input_shape = input_tensor->Shape(); + auto indice_shape = indice_tensor->Shape(); + if (indice_shape.NumDimensions() == 0) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "indices tensor must has rank larger than 0"); + } + + auto last_indice_dimension = indice_shape[indice_shape.NumDimensions() - 1]; + if (last_indice_dimension > static_cast(input_shape.NumDimensions())) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "last dimension of indices must not be larger than rank of input tensor"); + } + + std::vector shape(indice_shape.GetDims().begin(), + indice_shape.GetDims().end() - 1); + shape.insert(shape.end(), + input_shape.GetDims().begin() + last_indice_dimension, + input_shape.GetDims().end()); + auto output_tensor = context->Output(0,TensorShape(shape)); + std::vector element_counts(last_indice_dimension, 0LL); // Number of elements for each input dimension + +#pragma omp parallel for + for (int64_t i = 0; i < last_indice_dimension; ++i) { + element_counts[i] = input_shape.SizeFromDimension(i + 1); + } + + int64_t err_indice = 0; + p.element_bytes = input_tensor->DataType()->Size(); + p.element_to_copy = input_shape.SizeFromDimension(last_indice_dimension); + p.bytes_to_copy = p.element_bytes * p.element_to_copy; + auto indice_offset = static_cast(context->Input(1)->DataRaw()); + auto offset_count = indice_shape.Size() / last_indice_dimension; // Times to copy + p.element_offsets.assign(offset_count, 0LL); + + if (input_tensor->DataType() == DataTypeImpl::GetType()) { + p.input_str_base = static_cast(input_tensor->DataRaw()); + p.output_str_base = static_cast(output_tensor->MutableDataRaw()); + } else { + p.input_base = static_cast(context->Input(0)->DataRaw()); + p.output_base = static_cast(output_tensor->MutableDataRaw()); + } + +#pragma omp parallel for + for (int64_t i = 0; i < offset_count; ++i) { + for (int64_t j = 0; j < last_indice_dimension; ++j) { + auto indice = *(indice_offset + i * last_indice_dimension + j); + if (indice < 0 || indice >= input_shape[j]) { + err_indice = indice; + } + p.element_offsets[i] += indice * element_counts[j]; + } + } + return err_indice == 0 ? Status::OK() : + ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "invalid indice found, indice = ", err_indice); +} + +template Status GatherNDBase::PrepareForCompute(OpKernelContext*, Prepare&) const; +template Status GatherNDBase::PrepareForCompute(OpKernelContext*, Prepare&) const; + +Status GatherND::Compute(OpKernelContext* context) const { + Prepare p; + ORT_RETURN_IF_ERROR(context->Input(1)->DataType() == DataTypeImpl::GetType() ? + PrepareForCompute(context, p) : PrepareForCompute(context, p)); + return nullptr == p.input_str_base ? GatherNumber(p) : GatherString(p); +} + +Status GatherND::GatherNumber(const Prepare& p) const { +#pragma omp parallel for + for (int64_t i = 0; i < static_cast(p.element_offsets.size()); ++i) { + memcpy(p.output_base + i * p.bytes_to_copy, + p.input_base + p.element_offsets[i] * p.element_bytes, + p.bytes_to_copy); + } + return Status::OK(); +} + +Status GatherND::GatherString(const Prepare& p) const { +#pragma omp parallel for + for (int64_t i = 0; i < static_cast(p.element_offsets.size()); ++i) { + for (int64_t j = 0; j < static_cast(p.element_to_copy); ++j) { + p.output_str_base[i * p.element_to_copy + j] = p.input_str_base[p.element_offsets[i] + j]; + } + } + return Status::OK(); +} + +} +} diff --git a/onnxruntime/contrib_ops/cpu/gather_nd.h b/onnxruntime/contrib_ops/cpu/gather_nd.h new file mode 100644 index 0000000000000..6d256e07fd8ab --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/gather_nd.h @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { +namespace contrib { + +class GatherNDBase +{ +protected: + struct Prepare { + const uint8_t* input_base; + const std::string* input_str_base; + uint8_t* output_base; + std::string* output_str_base; + uint64_t bytes_to_copy; + uint64_t element_bytes; + uint64_t element_to_copy; + std::vector element_offsets; + + Prepare(): input_base (nullptr), + input_str_base (nullptr), + output_base (nullptr), + output_str_base (nullptr), + bytes_to_copy (0), + element_bytes (0), + element_to_copy (0), + element_offsets (0) {} + }; // struct Prepare + + template + Status PrepareForCompute(OpKernelContext* context, Prepare& p) const; +}; // class GatherNDBase + +class GatherND final : public OpKernel, protected GatherNDBase { +public: + explicit GatherND(const OpKernelInfo& info) : OpKernel(info) {} + Status Compute(OpKernelContext* context) const override; +private: + Status GatherNumber(const Prepare& p) const; + Status GatherString(const Prepare& p) const; +}; + +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc index 7316b85235b8a..a125c720683d4 100644 --- a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc +++ b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc @@ -588,6 +588,71 @@ The bounding box coordinates corresponding to the selected indices can then be o output_elem_type->set_elem_type(ONNX_NAMESPACE::TensorProto::STRING); }) .SetDoc(R"DOC([optional] Step1: Remove elements in X if they match any of the stop words so that the output tensor will not contain any stop words. This operator only accepts [C]- and [1, C]-tensors. If all elements in X are dropped, the output will be the default value of string tensor with shape [1] if input shape is [C] and shape [1, 1] if input shape is [1, C].)DOC"); + + ONNX_CONTRIB_OPERATOR_SCHEMA(GatherND) + .SetDomain(kMSDomain) + .SinceVersion(1) + .Input (0, "data", "Tensor of rank r >= 1.", "T" ) + .Input (1, "indices", "Tensor of rank q >= 1.", "Tind" ) + .Output (0, "output", "Tensor of rank q-1+r-indices[-1].", "T" ) + .TypeConstraint( + "T", + OpSchema::all_tensor_types(), + "Constrain input and output types to any tensor type.") + .TypeConstraint( + "Tind", + {"tensor(int32)", "tensor(int64)"}, + "Constrain indice type to int32 or int64") + .TypeAndShapeInferenceFunction( [] (ONNX_NAMESPACE::InferenceContext& ctx) { + propagateElemTypeFromInputToOutput(ctx, 0, 0); + if (!hasNInputShapes(ctx, 2)) { + fail_shape_inference("GatherND requires two tensor inputs."); + } + auto& data_shape = ctx.getInputType(0)->tensor_type().shape(); + auto& indices_shape = ctx.getInputType(1)->tensor_type().shape(); + auto data_rank = data_shape.dim_size(); + auto indices_rank = indices_shape.dim_size(); + if (data_rank < 1 || indices_rank < 1) { + fail_shape_inference("both data and indices tensor need to have rank larger than zero."); + } + auto last_indice_dimension = indices_shape.dim(indices_rank - 1).dim_value(); + if (last_indice_dimension > data_rank) { + fail_shape_inference("last dimension of indices must not be larger and rank of data tensor"); + } + for (int i = 0; i < indices_rank - 1; ++i) { + *ctx.getOutputType(0) + ->mutable_tensor_type() + ->mutable_shape() + ->add_dim() = indices_shape.dim(i); + } + for (int i = static_cast(last_indice_dimension); i < data_rank; ++i) { + *ctx.getOutputType(0) + ->mutable_tensor_type() + ->mutable_shape() + ->add_dim() = data_shape.dim(i); + } + }) + .SetDoc(R"DOC( +Given `data` tensor of rank r >= 1, and `indices` tensor of rank q >= 1, gather +slices of `data` into an output tensor of rank q - 1 + r - indices[-1]. +Example 1: + data = [[0,1],[2,3]] + indices = [[0,0],[1,1]] + output = [0,3] +Example 2: + data = [[0,1],[2,3]] + indices = [[1],[0]] + output = [[2,3],[0,1]] +Example 3: + data = [[[0,1],[2,3]],[[4,5],[6,7]]] + indices = [[0,1],[1,0]] + output = [[2,3],[4,5]] +Example 4: + data = [[[0,1],[2,3]],[[4,5],[6,7]]] + indices = [[[0,1]],[[1,0]]] + output = [[[2,3]],[[4,5]]] +)DOC"); + } } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/test/contrib_ops/gather_nd_op_test.cc b/onnxruntime/test/contrib_ops/gather_nd_op_test.cc new file mode 100644 index 0000000000000..f7af11bb1f4ec --- /dev/null +++ b/onnxruntime/test/contrib_ops/gather_nd_op_test.cc @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +namespace onnxruntime { +namespace test { + +TEST(GatherNDOpTest, GatherND_scaler_string_int32) { + OpTester test1("GatherND", 1, onnxruntime::kMSDomain); + test1.AddInput("data", {2,2}, {"h","k","o","z"}); + test1.AddInput("indices", {2}, {0,1}); + test1.AddOutput("output", {}, {"k"}); + test1.Run(); + + OpTester test2("GatherND", 1, onnxruntime::kMSDomain); + test2.AddInput("data", {6}, {"h","k","o","z","l","t"}); + test2.AddInput("indices", {1}, {3}); + test2.AddOutput("output", {}, {"z"}); + test2.Run(); + + OpTester test3("GatherND", 1, onnxruntime::kMSDomain); + test3.AddInput("data", {3,2}, {"h","k","o","z","l","t"}); + test3.AddInput("indices", {2}, {2,1}); + test3.AddOutput("output", {}, {"t"}); + test3.Run(); +} + +TEST(GatherNDOpTest, GatherND_matrice_int64_int64) { + OpTester test("GatherND", 1, onnxruntime::kMSDomain); + test.AddInput ("data", {2,2}, {0LL,1LL,2LL,3LL}); + test.AddInput ("indices", {2,2}, {0LL,0LL,1LL,1LL}); + test.AddOutput("output", {2}, {0LL,3LL}); + test.Run(); +} + +TEST(GatherNDOpTest, GatherND_matrice_string_int64) { + OpTester test("GatherND", 1, onnxruntime::kMSDomain); + test.AddInput("data", {2,2}, {"a","b","c","d"}); + test.AddInput("indices", {2,2}, {0LL,0LL,1LL,1LL}); + test.AddOutput("output", {2}, {"a","d"}); + test.Run(); +} + +TEST(GatherNDOpTest, GatherND_matrice_int64_int32) { + OpTester test("GatherND", 1, onnxruntime::kMSDomain); + test.AddInput("data", {2,2}, {0LL,1LL,2LL,3LL}); + test.AddInput("indices", {2,2}, {0,0,1,1}); + test.AddOutput("output", {2}, {0LL,3LL}); + test.Run(); +} + +TEST(GatherNDOpTest, GatherND_matrice_string_int32) { + OpTester test1("GatherND", 1, onnxruntime::kMSDomain); + test1.AddInput("data", {2,2,2}, {"egg","dance","air","bob","terry","smart","laugh","kite"}); + test1.AddInput("indices", {2,1,2}, {0,1,1,0}); + test1.AddOutput("output", {2,1,2}, {"air","bob","terry","smart"}); + test1.Run(); + + OpTester test2("GatherND", 1, onnxruntime::kMSDomain); + test2.AddInput("data", {3,3}, {"egg","dance","air","bob","terry","smart","laugh","kite","hop"}); + test2.AddInput("indices", {3,2}, {2,1,1,0,0,1}); + test2.AddOutput("output", {3}, {"kite","bob","dance"}); + test2.Run(); +} + +TEST(GatherNDOpTest, GatherND_slice_float_int64_t) { + OpTester test("GatherND", 1, onnxruntime::kMSDomain); + test.AddInput("data", {2,2}, {0.0f,0.1f,0.2f,0.3f}); + test.AddInput("indices", {2,1}, {1LL,0LL}); + test.AddOutput("output", {2,2}, {0.2f,0.3f,0.0f,0.1f}); + test.Run(); +} + +TEST(GatherNDOpTest, GatherND_slice_double_int32_t) { + OpTester test("GatherND", 1, onnxruntime::kMSDomain); + test.AddInput("data", {2,2}, {0.0f,0.1f,0.2f,0.3f}); + test.AddInput("indices", {2,1}, {1LL,0LL}); + test.AddOutput("output", {2,2}, {0.2f,0.3f,0.0f,0.1f}); + test.Run(); +} + +TEST(GatherNDOpTest, GatherND_3tensor_int64) { + OpTester test1("GatherND", 1, onnxruntime::kMSDomain); + test1.AddInput("data", {2,2,2}, {0LL,1LL,2LL,3LL,4LL,5LL,6LL,7LL}); + test1.AddInput("indices", {2,2}, {0LL,1LL,1LL,0LL}); + test1.AddOutput("output", {2,2}, {2LL,3LL,4LL,5LL}); + test1.Run(); + + OpTester test2("GatherND", 1, onnxruntime::kMSDomain); + test2.AddInput("data", {2,2,2}, {0,1,2,3,4,5,6,7}); + test2.AddInput("indices", {2,3}, {0,0,1,1,0,1}); + test2.AddOutput("output", {2}, {1,5}); + test2.Run(); + + OpTester test3("GatherND", 1, onnxruntime::kMSDomain); + test3.AddInput("data", {2,2,2}, {0,1,2,3,4,5,6,7}); + test3.AddInput("indices", {1,1}, {1LL}); + test3.AddOutput("output", {1,2,2}, {4,5,6,7}); + test3.Run(); +} + +TEST(GatherNDOpTest, GatherND_batched_index_int64) { + OpTester test("GatherND", 1, onnxruntime::kMSDomain); + test.AddInput("data", {2,2}, {0LL,1LL,2LL,3LL}); + test.AddInput("indices", {2,1,2}, {0LL,0LL,0LL,1LL}); + test.AddOutput("output", {2,1}, {0LL,1LL}); + test.Run(); +} + +TEST(GatherNDOpTest, GatherND_batched_index_bool_int64) { + OpTester test("GatherND", 1, onnxruntime::kMSDomain); + test.AddInput("data", {2,2}, {true,false,false,true}); + test.AddInput("indices", {2,1,2}, {0LL,0LL,0LL,1LL}); + test.AddOutput("output", {2,1}, {true,false}); + test.Run(); +} + +TEST(GatherNDOpTest, GatherND_sliced_index_int64) { + OpTester test("GatherND", 1, onnxruntime::kMSDomain); + test.AddInput("data", {2,2}, {0LL,1LL,2LL,3LL}); + test.AddInput("indices", {2,1,1}, {1LL,0LL}); + test.AddOutput("output", {2,1,2}, {2LL,3LL,0LL,1LL}); + test.Run(); +} + +TEST(GatherNDOpTest, GatherND_sliced_index_string_int32) { + OpTester test("GatherND", 1, onnxruntime::kMSDomain); + test.AddInput("data", {2,2}, {"ab","cde","f","ghi"}); + test.AddInput("indices", {2,1,1}, {1LL,0LL}); + test.AddOutput("output", {2,1,2}, {"f","ghi","ab","cde"}); + test.Run(); +} + +TEST(GatherNDOpTest, GatherND_batched_3tensor_int64) { + OpTester test1("GatherND", 1, onnxruntime::kMSDomain); + test1.AddInput("data", {2,2,2}, {0,1,2,3,4,5,6,7}); + test1.AddInput("indices", {2,2,2}, {0LL,1LL,1LL,0LL,0LL,0LL,1LL,1LL}); + test1.AddOutput("output", {2,2,2}, {2,3,4,5,0,1,6,7}); + test1.Run(); + + OpTester test2("GatherND", 1, onnxruntime::kMSDomain); + test2.AddInput("data", {2,2,2}, {0,1,2,3,4,5,6,7}); + test2.AddInput("indices", {2,2,3}, {0,0,1,1,0,1,0,1,1,1,1,0}); + test2.AddOutput("output", {2,2}, {1,5,3,6}); + test2.Run(); + + OpTester test3("GatherND", 1, onnxruntime::kMSDomain); + test3.AddInput("data", {2,2,2}, {0LL,1LL,2LL,3LL,4LL,5LL,6LL,7LL}); + test3.AddInput("indices", {2,1,1}, {1,0}); + test3.AddOutput("output", {2,1,2,2}, {4LL,5LL,6LL,7LL,0LL,1LL,2LL,3LL}); + test3.Run(); +} + +} // namespace test +} // namespace onnxruntime From b0f27ba0a7cebf54b0c16237b5b81e1191ce8480 Mon Sep 17 00:00:00 2001 From: KeDengMS Date: Mon, 17 Dec 2018 14:41:42 -0800 Subject: [PATCH 10/56] Allow using MKLML header/libs when use_mklml is specified (#178) Allow using MKLML header/libs when use_mklml is specified --- cmake/CMakeLists.txt | 60 ++++++++------ cmake/external/mkldnn.cmake | 83 +++++++++++++------ cmake/onnxruntime_providers.cmake | 2 +- cmake/onnxruntime_python.cmake | 2 +- cmake/onnxruntime_unittests.cmake | 2 +- .../core/providers/cpu/tensor/cast_op.h | 2 +- onnxruntime/core/util/math.h | 10 ++- onnxruntime/core/util/math_cpu.cc | 15 ++-- 8 files changed, 116 insertions(+), 60 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 38e9bd75cfe65..7769cee8f3f0a 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -374,29 +374,6 @@ else() endif() endif() -if (onnxruntime_USE_TVM) - if (WIN32 AND MSVC) - # wd4100: identifier' : unreferenced formal parameter - # wd4244: conversion from 'int' to 'char', possible loss of data - # wd4251: class X needs to have dll-interface to be used by clients of class Y - # wd4267: 'initializing': conversion from 'size_t' to 'int', possible loss of data - # wd4275: non dll-interface class X used as base for dll-interface class Y - # wd4389: signed/unsigned mismatch - # wd4456: declaration of X hides previous local declaration - set(DISABLED_WARNINGS_FOR_TVM "/wd4100" "/wd4244" "/wd4251" "/wd4267" "/wd4275" "/wd4389" "/wd4456") - else() - set(DISABLED_WARNINGS_FOR_TVM "-Wno-error=extra" "-Wno-error=ignored-qualifiers") - if(HAS_UNUSED_PARAMETER) - list(APPEND DISABLED_WARNINGS_FOR_TVM "-Wno-error=unused-parameter") - endif() - if(HAS_CATCH_VALUE) - #TODO: send a PR to TVM and fix it - list(APPEND DISABLED_WARNINGS_FOR_TVM "-Wno-error=catch-value") - endif() - endif() - include(onnxruntime_codegen.cmake) -endif() - if (onnxruntime_USE_JEMALLOC) if (Win32) message( FATAL_ERROR "Jemalloc is not supported on Windows." ) @@ -412,9 +389,21 @@ include_directories( $ ) +if (onnxruntime_USE_MKLDNN OR onnxruntime_USE_MKLML) + include(mkldnn) +endif() + +if (onnxruntime_USE_MKLML) + add_definitions(-DUSE_MKLML=1) + # USE_MKML_FOR_BLAS may cause numerical differences in tests so disable by default + #add_definitions(-DUSE_MKLML_FOR_BLAS=1) + list(APPEND onnxruntime_EXTERNAL_LIBRARIES mklml) + list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES mklml) + link_directories(${MKLML_LIB_DIR}) +endif() + if (onnxruntime_USE_MKLDNN) add_definitions(-DUSE_MKLDNN=1) - include(mkldnn) list(APPEND onnxruntime_EXTERNAL_LIBRARIES mkldnn) list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES mkldnn) link_directories(${MKLDNN_LIB_DIR}) @@ -465,6 +454,29 @@ if (onnxruntime_USE_CUDA) endif() endif() +if (onnxruntime_USE_TVM) + if (WIN32 AND MSVC) + # wd4100: identifier' : unreferenced formal parameter + # wd4244: conversion from 'int' to 'char', possible loss of data + # wd4251: class X needs to have dll-interface to be used by clients of class Y + # wd4267: 'initializing': conversion from 'size_t' to 'int', possible loss of data + # wd4275: non dll-interface class X used as base for dll-interface class Y + # wd4389: signed/unsigned mismatch + # wd4456: declaration of X hides previous local declaration + set(DISABLED_WARNINGS_FOR_TVM "/wd4100" "/wd4244" "/wd4251" "/wd4267" "/wd4275" "/wd4389" "/wd4456") + else() + set(DISABLED_WARNINGS_FOR_TVM "-Wno-error=extra" "-Wno-error=ignored-qualifiers") + if(HAS_UNUSED_PARAMETER) + list(APPEND DISABLED_WARNINGS_FOR_TVM "-Wno-error=unused-parameter") + endif() + if(HAS_CATCH_VALUE) + #TODO: send a PR to TVM and fix it + list(APPEND DISABLED_WARNINGS_FOR_TVM "-Wno-error=catch-value") + endif() + endif() + include(onnxruntime_codegen.cmake) +endif() + #names in this var must match the directory names under onnxruntime/core/providers set(ONNXRUNTIME_PROVIDER_NAMES cpu) diff --git a/cmake/external/mkldnn.cmake b/cmake/external/mkldnn.cmake index a789382bf80ee..659a6560fb929 100644 --- a/cmake/external/mkldnn.cmake +++ b/cmake/external/mkldnn.cmake @@ -1,57 +1,90 @@ include (ExternalProject) set(MKLDNN_URL https://github.com/intel/mkl-dnn.git) -# If MKLDNN_TAG is updated, check if platform.cmake.patch needs to be updated. +# If MKLDNN_TAG is updated, check if MKLML_VERSION and platform.cmake.patch need to be updated. set(MKLDNN_TAG v0.17.1) -set(MKLDNN_SOURCE ${CMAKE_CURRENT_BINARY_DIR}/mkl-dnn/src/mkl-dnn/src) -set(MKLDNN_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/mkl-dnn/install) -set(MKLDNN_LIB_DIR ${MKLDNN_INSTALL}/lib) -set(MKLDNN_INCLUDE_DIR ${MKLDNN_INSTALL}/include) - -set(MKLDNN_PATCH_COMMAND1 git apply ${CMAKE_SOURCE_DIR}/patches/mkldnn/platform.cmake.patch) -# discard prior changes due to patching in mkldnn source to unblock incremental builds. -set(MKLDNN_PATCH_DISCARD_COMMAND cd ${MKLDNN_SOURCE} && git checkout -- .) +set(MKLML_VERSION 2019.0.1.20180928) if(WIN32) + set(MKLML_OS_VERSION_STR "win") + set(MKLML_FILE_EXTENSION "zip") set(MKLDNN_SHARED_LIB mkldnn.dll) set(MKLDNN_IMPORT_LIB mkldnn.lib) if(onnxruntime_USE_MKLML) - set(DOWNLOAD_MKLML ${MKLDNN_SOURCE}/scripts/prepare_mkl.bat) set(MKLML_SHARED_LIB mklml.dll) + set(MKLML_IMPORT_LIB mklml.lib) set(IOMP5MD_SHARED_LIB libiomp5md.dll) + set(IOMP5MD_IMPORT_LIB libiomp5md.lib) endif() else() + set(MKLML_FILE_EXTENSION "tgz") if (APPLE) set(MKLDNN_SHARED_LIB libmkldnn.0.dylib) + set(MKLML_OS_VERSION_STR "mac") else() set(MKLDNN_SHARED_LIB libmkldnn.so.0) + set(MKLML_OS_VERSION_STR "lnx") endif() if(onnxruntime_USE_MKLML) - set(DOWNLOAD_MKLML ${MKLDNN_SOURCE}/scripts/prepare_mkl.sh) set(MKLML_SHARED_LIB libmklml_intel.so) set(IOMP5MD_SHARED_LIB libiomp5.so) endif() endif() -if(NOT onnxruntime_USE_MKLDNN OR EXISTS ${MKLDNN_SOURCE}/external) - set(DOWNLOAD_MKLML "") +if (onnxruntime_USE_MKLML) + set(MKLML_URL https://github.com/intel/mkl-dnn/releases/download/${MKLDNN_TAG}/mklml_${MKLML_OS_VERSION_STR}_${MKLML_VERSION}.${MKLML_FILE_EXTENSION}) + + ExternalProject_Add(project_mklml + PREFIX mklml + URL ${MKLML_URL} + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + UPDATE_COMMAND "" + INSTALL_COMMAND "" ) + + set(MKML_DIR ${CMAKE_CURRENT_BINARY_DIR}/mklml/src/project_mklml) + set(MKLML_INCLUDE_DIR "${MKML_DIR}/include") + set(MKLML_LIB_DIR "${MKML_DIR}/lib") + if(WIN32) + add_library(mklml STATIC IMPORTED) + set_property(TARGET mklml PROPERTY IMPORTED_LOCATION ${MKLML_LIB_DIR}/${MKLML_IMPORT_LIB}) + else() + add_library(mklml SHARED IMPORTED) + set_property(TARGET mklml PROPERTY IMPORTED_LOCATION ${MKLML_LIB_DIR}/${MKLML_SHARED_LIB}) + endif() + add_dependencies(mklml project_mklml) + include_directories(${MKLML_INCLUDE_DIR}) endif() -ExternalProject_Add(project_mkldnn +if (onnxruntime_USE_MKLDNN) + set(MKLDNN_SOURCE ${CMAKE_CURRENT_BINARY_DIR}/mkl-dnn/src/mkl-dnn/src) + set(MKLDNN_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/mkl-dnn/install) + set(MKLDNN_LIB_DIR ${MKLDNN_INSTALL}/lib) + set(MKLDNN_INCLUDE_DIR ${MKLDNN_INSTALL}/include) + + set(MKLDNN_PATCH_COMMAND1 git apply ${CMAKE_SOURCE_DIR}/patches/mkldnn/platform.cmake.patch) + # discard prior changes due to patching in mkldnn source to unblock incremental builds. + set(MKLDNN_PATCH_DISCARD_COMMAND cd ${MKLDNN_SOURCE} && git checkout -- .) + + ExternalProject_Add(project_mkldnn PREFIX mkl-dnn GIT_REPOSITORY ${MKLDNN_URL} GIT_TAG ${MKLDNN_TAG} - PATCH_COMMAND ${DOWNLOAD_MKLML} COMMAND ${MKLDNN_PATCH_DISCARD_COMMAND} COMMAND ${MKLDNN_PATCH_COMMAND1} + PATCH_COMMAND ${MKLDNN_PATCH_DISCARD_COMMAND} COMMAND ${MKLDNN_PATCH_COMMAND1} SOURCE_DIR ${MKLDNN_SOURCE} - CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${MKLDNN_INSTALL} -) + CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${MKLDNN_INSTALL} -DMKLROOT=${MKML_DIR} + ) + if (onnxruntime_USE_MKLML) + add_dependencies(project_mkldnn project_mklml) + endif() -if(WIN32) - add_library(mkldnn STATIC IMPORTED) - set_property(TARGET mkldnn PROPERTY IMPORTED_LOCATION ${MKLDNN_LIB_DIR}/${MKLDNN_IMPORT_LIB}) -else() - add_library(mkldnn SHARED IMPORTED) - set_property(TARGET mkldnn PROPERTY IMPORTED_LOCATION ${MKLDNN_LIB_DIR}/${MKLDNN_SHARED_LIB}) + if(WIN32) + add_library(mkldnn STATIC IMPORTED) + set_property(TARGET mkldnn PROPERTY IMPORTED_LOCATION ${MKLDNN_LIB_DIR}/${MKLDNN_IMPORT_LIB}) + else() + add_library(mkldnn SHARED IMPORTED) + set_property(TARGET mkldnn PROPERTY IMPORTED_LOCATION ${MKLDNN_LIB_DIR}/${MKLDNN_SHARED_LIB}) + endif() + add_dependencies(mkldnn project_mkldnn) + include_directories(${MKLDNN_INCLUDE_DIR}) endif() -add_dependencies(mkldnn project_mkldnn) -include_directories(${MKLDNN_INCLUDE_DIR}) diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index 79734f8a1315d..1386af9de6241 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -33,7 +33,7 @@ source_group(TREE ${ONNXRUNTIME_ROOT} FILES ${onnxruntime_contrib_ops_srcs}) add_library(onnxruntime_providers ${onnxruntime_providers_common_srcs} ${onnxruntime_providers_srcs} ${onnxruntime_contrib_ops_srcs}) onnxruntime_add_include_to_target(onnxruntime_providers onnx protobuf::libprotobuf) target_include_directories(onnxruntime_providers PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS}) -add_dependencies(onnxruntime_providers eigen gsl onnx) +add_dependencies(onnxruntime_providers eigen gsl onnx ${onnxruntime_EXTERNAL_DEPENDENCIES}) install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/cpu DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) set_target_properties(onnxruntime_providers PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(onnxruntime_providers PROPERTIES FOLDER "ONNXRuntime") diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index 22db4d45f2ea3..0de14f9333dc1 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -182,7 +182,7 @@ if (onnxruntime_USE_MKLML) add_custom_command( TARGET onnxruntime_pybind11_state POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy - ${MKLDNN_LIB_DIR}/${MKLML_SHARED_LIB} ${MKLDNN_LIB_DIR}/${IOMP5MD_SHARED_LIB} + ${MKLML_LIB_DIR}/${MKLML_SHARED_LIB} ${MKLML_LIB_DIR}/${IOMP5MD_SHARED_LIB} $/onnxruntime/capi/ ) endif() diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index c11ab46ad9fae..c1a1393bcc973 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -437,7 +437,7 @@ if (onnxruntime_USE_MKLML) add_custom_command( TARGET ${test_data_target} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy - ${MKLDNN_LIB_DIR}/${MKLML_SHARED_LIB} ${MKLDNN_LIB_DIR}/${IOMP5MD_SHARED_LIB} + ${MKLML_LIB_DIR}/${MKLML_SHARED_LIB} ${MKLML_LIB_DIR}/${IOMP5MD_SHARED_LIB} $ ) endif() diff --git a/onnxruntime/core/providers/cpu/tensor/cast_op.h b/onnxruntime/core/providers/cpu/tensor/cast_op.h index e75c08790cb9e..54feb86a0825a 100644 --- a/onnxruntime/core/providers/cpu/tensor/cast_op.h +++ b/onnxruntime/core/providers/cpu/tensor/cast_op.h @@ -5,9 +5,9 @@ #include "core/common/common.h" #include "core/framework/op_kernel.h" +#include "core/util/math.h" #include "core/util/math_cpuonly.h" #include "Eigen/src/Core/arch/CUDA/Half.h" -#include "core/util/math_cpuonly.h" #if defined(USE_MLAS) && defined(_M_AMD64) #include "core/mlas/inc/mlas.h" diff --git a/onnxruntime/core/util/math.h b/onnxruntime/core/util/math.h index 62db5e9fabe51..9e84a41d042b2 100644 --- a/onnxruntime/core/util/math.h +++ b/onnxruntime/core/util/math.h @@ -24,10 +24,16 @@ // still keep it simple, so all platforms would be able to support it fairly // easily. +#ifdef USE_MKLML_FOR_BLAS +// when USE_MKLML is defined, use MKLML cblas for GEMM +#include "mkl_cblas.h" +#define CBLAS_ENUM_DEFINED_H +#else // We include the cblas header here so that we can obtain the macros from cblas. extern "C" { #include "core/framework/cblas.h" } +#endif #include "core/common/common.h" #include "core/framework/data_types.h" @@ -70,7 +76,7 @@ void Not(const int N, const T* x, T* y, Provider* provider); template void Powx(const int N, const T* a, const T b, T* y, Provider* provider); -#define DECLARE_BINARY_OP_BINARY_RESULT(name) \ +#define DECLARE_BINARY_OP_BINARY_RESULT(name) \ template \ void name(const int N, const T* a, const T* b, bool* y, Provider* provider); \ template \ @@ -93,7 +99,7 @@ DECLARE_BINARY_OP_BINARY_RESULT(Xor); #undef DECLARE_BINARY_OP_BINARY_RESULT -#define DECLARE_BINARY_OP(name) \ +#define DECLARE_BINARY_OP(name) \ template \ void name(const int N, const T* a, const T* b, T* y, Provider* provider); \ template \ diff --git a/onnxruntime/core/util/math_cpu.cc b/onnxruntime/core/util/math_cpu.cc index b3b1fbcb70c4b..fc149c415ab25 100644 --- a/onnxruntime/core/util/math_cpu.cc +++ b/onnxruntime/core/util/math_cpu.cc @@ -57,7 +57,8 @@ namespace math { // will delegate the Caffe math functions that are BLAS-related to either the // CBLAS call or the Eigen implementation. //////////////////////////////////////////////////////////////////////////////// -#ifdef USE_EIGEN_FOR_BLAS +// when USE_MKLDNN and USE_MKLML are defined, use cblas APIs for MKLML +#if defined(USE_EIGEN_FOR_BLAS) && !defined(USE_MKLML_FOR_BLAS) // Caffe2 gemm provides a simpler interface to the gemm functions, with the // limitation that the data has to be contiguous in memory. @@ -332,10 +333,14 @@ void Gemm( float* C, CPUMathUtil* /*context*/, MLDataType /*math_type*/) { - int lda = (TransA == CblasNoTrans) ? K : M; - int ldb = (TransB == CblasNoTrans) ? N : K; - cblas_sgemm(CblasRowMajor, TransA, TransB, M, N, K, alpha, A, lda, B, ldb, - beta, C, N); + int lda = gsl::narrow_cast((TransA == CblasNoTrans) ? K : M); + int ldb = gsl::narrow_cast((TransB == CblasNoTrans) ? N : K); + cblas_sgemm(CblasRowMajor, TransA, TransB, + gsl::narrow_cast(M), + gsl::narrow_cast(N), + gsl::narrow_cast(K), + alpha, A, lda, B, ldb, + beta, C, gsl::narrow_cast(N)); } template <> From 8c5d10555784001d893532b62d850916b3c9ad0d Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 17 Dec 2018 13:02:02 -0800 Subject: [PATCH 11/56] fix typo --- onnxruntime/test/onnx/main.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index 8f74bc5ea9a2d..150c8c61bdf53 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -205,7 +205,7 @@ int real_main(int argc, char* argv[]) { sf.AppendExecutionProvider(f); OrtReleaseObject(f); #else - fprintf(stderr, "CUDA is supported in this build"); + fprintf(stderr, "CUDA is not supported in this build"); return -1; #endif } @@ -216,7 +216,7 @@ int real_main(int argc, char* argv[]) { sf.AppendExecutionProvider(f); OrtReleaseObject(f); #else - fprintf(stderr, "Nuphar is supported in this build"); + fprintf(stderr, "Nuphar is not supported in this build"); return -1; #endif } @@ -227,7 +227,7 @@ int real_main(int argc, char* argv[]) { sf.AppendExecutionProvider(f); OrtReleaseObject(f); #else - fprintf(stderr, "MKL-DNN is supported in this build"); + fprintf(stderr, "MKL-DNN is not supported in this build"); return -1; #endif } From 71c56b6d7cc282a21168c74c3b7afd94594a1465 Mon Sep 17 00:00:00 2001 From: edgchen1 Date: Tue, 18 Dec 2018 00:30:07 -0800 Subject: [PATCH 12/56] Fix array feature extractor out of bounds access issue (#194) * Fixed out of bounds access in ArrayFeatureExtractor. * some cleanup * Updated tensor_shape.h comments. * Updated macro name. * Added copy assignment, move assignment/ctor to TensorShape. * Removed i64 literal suffix. * Fixed test. * Fixed type of x_num_dims. --- .../onnxruntime/core/framework/tensor_shape.h | 17 ++-- .../cpu/ml/array_feature_extractor.cc | 29 ++++--- .../cpu/ml/array_feature_extractor_test.cc | 85 +++++++++++++++++-- 3 files changed, 109 insertions(+), 22 deletions(-) diff --git a/include/onnxruntime/core/framework/tensor_shape.h b/include/onnxruntime/core/framework/tensor_shape.h index a012a30d364f8..1007601058640 100644 --- a/include/onnxruntime/core/framework/tensor_shape.h +++ b/include/onnxruntime/core/framework/tensor_shape.h @@ -28,14 +28,18 @@ class TensorShape : private std::vector { public: TensorShape() = default; + TensorShape(const TensorShape& /*other*/) = default; + TensorShape& operator=(const TensorShape& /*other*/) = default; + + TensorShape(TensorShape&& /*other*/) = default; + TensorShape& operator=(TensorShape&& /*other*/) = default; + TensorShape(const int64_t* dimension_sizes, size_t dimension_count); TensorShape(const std::vector& dims); TensorShape(const std::initializer_list& dims); - TensorShape(const TensorShape& /*other*/) = default; - TensorShape(const std::vector& dims, size_t start, size_t end); /** @@ -84,13 +88,16 @@ class TensorShape : private std::vector { /** Return the total number of elements up to the specified dimension. - @param dimension Return size up to this dimension. Value must be >= 0 and < this.Size(). + If the dimension interval is empty (dimension == 0), return 1. + @param dimension Return size up to this dimension. Value must be between 0 and this->NumDimensions(), inclusive. */ int64_t SizeToDimension(size_t dimension) const; /** Return the total number of elements from the specified dimension to the end of the tensor shape. - @param dimension Return size up to this dimension. 0 <= dimension < this.Size(). + If the dimension interval is empty (dimension == this->NumDimensions()), return 1. + @param dimension Return size from this dimension to the end. Value must be between 0 and this->NumDimensions(), + inclusive. */ int64_t SizeFromDimension(size_t dimension) const; @@ -111,7 +118,7 @@ class TensorShape : private std::vector { /** Calculate size between start and end. - Assumes start and end are between 0 and dimensions.size(), inclusive, and that + Assumes start and end are between 0 and this->NumDimensions(), inclusive, and that start < end. */ int64_t SizeHelper(size_t start, size_t end) const; diff --git a/onnxruntime/core/providers/cpu/ml/array_feature_extractor.cc b/onnxruntime/core/providers/cpu/ml/array_feature_extractor.cc index eb79c5b5a62f7..aedd428915038 100644 --- a/onnxruntime/core/providers/cpu/ml/array_feature_extractor.cc +++ b/onnxruntime/core/providers/cpu/ml/array_feature_extractor.cc @@ -53,36 +53,43 @@ template common::Status ArrayFeatureExtractorOp::Compute(OpKernelContext* context) const { const Tensor& X = *context->Input(0); const TensorShape& x_shape = X.Shape(); - const vector& x_dims = x_shape.GetDims(); + const size_t x_num_dims = x_shape.NumDimensions(); const T* x_data = X.template Data(); - if (x_dims.empty()) { + if (x_num_dims == 0) { return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid argument: X input has empty dimensions."); } - int64_t stride = x_dims.size() == 1 ? x_dims[0] : x_dims[1]; - int64_t N = x_dims.size() == 1 ? 1 : x_dims[0]; + const int64_t stride = x_shape[x_num_dims - 1]; const Tensor& Y = *context->Input(1); const TensorShape& y_shape = Y.Shape(); const int64_t* y_data = Y.template Data(); - int64_t num_indices = y_shape.Size(); + const int64_t num_indices = y_shape.Size(); // validate Y if (num_indices == 0) { return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid Y argument: num_indices = 0"); } - if (num_indices - 1 >= stride) { - std::ostringstream err_msg; - err_msg << "Invalid Y argument: num_indices - 1 (" << num_indices - 1 << ") >= stride (" << stride << ")"; - return Status(ONNXRUNTIME, INVALID_ARGUMENT, err_msg.str()); + for (int64_t i = 0; i < num_indices; ++i) { + if (y_data[i] >= stride) { + return ORT_MAKE_STATUS( + ONNXRUNTIME, INVALID_ARGUMENT, + "Invalid Y argument: index is out of range: Y[", i, "] (", y_data[i], ") >=", stride); + } } - Tensor* Z = context->Output(0, TensorShape({N, num_indices})); + const TensorShape z_shape = [num_indices, &x_shape]() { + TensorShape shape{x_shape}; + shape[shape.NumDimensions() - 1] = num_indices; + return shape; + }(); + Tensor* Z = context->Output(0, z_shape); T* z_data = Z->template MutableData(); - for (int64_t n = 0; n < N; ++n) { + const int64_t x_size_until_last_dim = x_shape.SizeToDimension(x_num_dims - 1); + for (int64_t i = 0; i < x_size_until_last_dim; ++i) { for (int64_t j = 0; j < num_indices; ++j) { *z_data++ = x_data[y_data[j]]; } diff --git a/onnxruntime/test/providers/cpu/ml/array_feature_extractor_test.cc b/onnxruntime/test/providers/cpu/ml/array_feature_extractor_test.cc index 164e143fbcf3d..efa78a89a85f5 100644 --- a/onnxruntime/test/providers/cpu/ml/array_feature_extractor_test.cc +++ b/onnxruntime/test/providers/cpu/ml/array_feature_extractor_test.cc @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#include + #include "gtest/gtest.h" #include "test/providers/provider_test_utils.h" #include "gsl/gsl" @@ -8,19 +10,23 @@ using namespace std; namespace onnxruntime { namespace test { -TEST(MLOpTest, ArrayFeatureExtractorTest) { - OpTester test("ArrayFeatureExtractor", 1, onnxruntime::kMLDomain); +class ArrayFeatureExtractorTest : public ::testing::Test { + protected: + OpTester test_{"ArrayFeatureExtractor", 1, onnxruntime::kMLDomain}; +}; + +TEST_F(ArrayFeatureExtractorTest, Basic) { const int N = 3; const std::vector X = {0.8f, -1.5f, 2.0f, 3.8f, -4.0f, 5.0f, 6.8f, -7.5f, 8.0f, 9.8f, -9.0f, 4.0f, 4.8f, -4.5f, 4.0f, 4.8f, -4.0f, 4.0f}; const int kCols = 6; const vector x_dims = {N, kCols}; - test.AddInput("X", x_dims, X); + test_.AddInput("X", x_dims, X); const std::vector Y = {1L, 2L, 4L}; const vector y_dims = {1, 3}; - test.AddInput("Y", y_dims, Y); + test_.AddInput("Y", y_dims, Y); // prepare expected output vector expected_output; @@ -31,9 +37,76 @@ TEST(MLOpTest, ArrayFeatureExtractorTest) { } } const vector expected_dims{N, gsl::narrow_cast(Y.size())}; - test.AddOutput("Z", expected_dims, expected_output); + test_.AddOutput("Z", expected_dims, expected_output); + + test_.Run(); +} + +TEST_F(ArrayFeatureExtractorTest, HigherDimensionalX) { + const std::vector x_dims{2, 3, 4, 5}; + const int64_t x_size = std::accumulate( + x_dims.begin(), x_dims.end(), static_cast(1), std::multiplies<>{}); + const std::vector X = [x_size]() { + std::vector v(x_size); + std::iota(v.begin(), v.end(), 0); + return v; + }(); + test_.AddInput("X", x_dims, X); + + const std::vector Y{0, 1, 1, 2}; + const int64_t y_size = gsl::narrow_cast(Y.size()); + const std::vector y_dims{1, y_size}; + test_.AddInput("Y", y_dims, Y); + + // prepare expected output + const std::vector z_dims = [&x_dims, y_size]() { + std::vector v{x_dims}; + v[v.size() - 1] = y_size; + return v; + }(); + const std::vector Z = [&x_dims, x_size, y_size, &X, &Y]() { + const int64_t x_last_dim_size = x_dims.back(); // stride + const int64_t x_leading_dims_size = x_size / x_last_dim_size; + std::vector v(x_leading_dims_size * y_size); + int32_t* v_output = v.data(); + for (int64_t x_idx = 0; x_idx < x_size; x_idx += x_last_dim_size) { + for (int64_t y_idx = 0; y_idx < y_size; ++y_idx) { + *(v_output++) = X[x_idx + Y[y_idx]]; + } + } + return v; + }(); + test_.AddOutput("Z", z_dims, Z); + + test_.Run(); +} + +TEST_F(ArrayFeatureExtractorTest, OneDimensionalX) { + test_.AddInput("X", {1}, {42}); + test_.AddInput("Y", {1, 3}, {0, 0, 0}); + test_.AddOutput("Z", {3}, {42, 42, 42}); + test_.Run(); +} + +TEST_F(ArrayFeatureExtractorTest, InvalidInputEmptyX) { + test_.AddInput("X", {0}, {}); + test_.AddInput("Y", {1}, {1}); + test_.AddOutput("Z", {0}, {}); + test_.Run(OpTester::ExpectResult::kExpectFailure); +} + +TEST_F(ArrayFeatureExtractorTest, InvalidInputEmptyY) { + test_.AddInput("X", {1}, {1}); + test_.AddInput("Y", {0}, {}); + test_.AddOutput("Z", {0}, {}); + test_.Run(OpTester::ExpectResult::kExpectFailure); +} - test.Run(); +TEST_F(ArrayFeatureExtractorTest, InvalidInputOutOfBoundsY) { + test_.AddInput("X", {2, 2}, {1, 2, 3, 4}); + test_.AddInput("Y", {1}, {10}); + test_.AddOutput("Z", {0}, {}); + test_.Run(OpTester::ExpectResult::kExpectFailure, "index is out of range"); } } // namespace test From 0287019c1df9c242508534831b8401984c393f2d Mon Sep 17 00:00:00 2001 From: jignparm Date: Tue, 18 Dec 2018 08:30:27 +0000 Subject: [PATCH 13/56] C# Gpu : Minor updates to exception message (#201) * Minor updates to exception message * update models folder to new location * update copy to preservenewest --- .../Microsoft.ML.OnnxRuntime/Exceptions.cs | 2 -- .../SessionOptions.cs | 21 ++++++++++++------- .../InferenceTest.cs | 5 ++++- .../Microsoft.ML.OnnxRuntime.Tests.csproj | 6 +++--- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Exceptions.cs b/csharp/src/Microsoft.ML.OnnxRuntime/Exceptions.cs index c7698722d8727..f748ffe08035d 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/Exceptions.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/Exceptions.cs @@ -26,7 +26,6 @@ internal enum ErrorCode InvalidGraph = 10, ShapeInferenceNotRegistered = 11, RequirementNotRegistered = 12, - ExecutionProviderDLLNotFound = 13 } /// @@ -49,7 +48,6 @@ public class OnnxRuntimeException: Exception { ErrorCode.InvalidGraph, "InvalidGraph" }, { ErrorCode.ShapeInferenceNotRegistered, "ShapeInferenceNotRegistered" }, { ErrorCode.RequirementNotRegistered, "RequirementNotRegistered" }, - { ErrorCode.ExecutionProviderDLLNotFound, "ExecutionProviderDLLNotFound" } }; internal OnnxRuntimeException(ErrorCode errorCode, string message) diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs index b1d69104cb188..0e8f88c6dabe9 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System; +using System.Text; using System.Runtime.InteropServices; namespace Microsoft.ML.OnnxRuntime @@ -125,9 +126,12 @@ private void AppendExecutionProvider(NativeOnnxObjectHandle providerFactory) } } + // Declared, but called only if OS = Windows. [DllImport("kernel32.dll")] private static extern IntPtr LoadLibrary(string dllToLoad); + [DllImport("kernel32.dll")] + static extern uint GetSystemDirectory([Out] StringBuilder lpBuffer, uint uSize); private static bool CheckCudaExecutionProviderDLLs() { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) @@ -135,14 +139,15 @@ private static bool CheckCudaExecutionProviderDLLs() foreach (var dll in cudaDelayLoadedLibs) { IntPtr handle = LoadLibrary(dll); - if (handle == IntPtr.Zero) - { - throw new OnnxRuntimeException( - ErrorCode.ExecutionProviderDLLNotFound, - $"Dll not found: {dll}. CUDA 10.0 is required for GPU execution. " + - $"Verify that the library is available on system path." - ); - } + if (handle != IntPtr.Zero) + continue; + var sysdir = new StringBuilder(String.Empty, 2048); + GetSystemDirectory(sysdir, (uint)sysdir.Capacity); + throw new OnnxRuntimeException( + ErrorCode.NoSuchFile, + $"kernel32.LoadLibrary():'{dll}' not found. CUDA 10.0 is required for GPU execution. " + + $". Verify it is available in the system directory={sysdir}. Else copy it to the output folder." + ); } } return true; diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs index 995c6acbb91d1..1edd5d7b6c081 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs @@ -226,7 +226,10 @@ private void TestPreTrainedModelsOpset7And8() if (modelNames[0].ToString() == "._resnet34v2.onnx") modelNames[0] = modelNames[1]; else - throw new Exception($"Opset {opset}: Model {model}: error = can't determine model file name."); + { + var modelNamesList = string.Join(",", modelNames.Select(x => x.ToString())); + throw new Exception($"Opset {opset}: Model {model}. Can't determine model file name. Found these :{modelNamesList}"); + } } var session = new InferenceSession($"{opset}\\{model}\\{modelNames[0].ToString()}"); diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/Microsoft.ML.OnnxRuntime.Tests.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/Microsoft.ML.OnnxRuntime.Tests.csproj index ac3702b7e3df5..252d3ba783874 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/Microsoft.ML.OnnxRuntime.Tests.csproj +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/Microsoft.ML.OnnxRuntime.Tests.csproj @@ -1,4 +1,4 @@ - + netcoreapp2.0 @@ -38,8 +38,8 @@ false - - Always + + PreserveNewest false From 773114a4f125d0cb515526351f5e3defffbca034 Mon Sep 17 00:00:00 2001 From: Ryan Hill <38674843+RyanUnderhill@users.noreply.github.com> Date: Tue, 18 Dec 2018 11:39:46 -0800 Subject: [PATCH 14/56] More C header naming changes (#202) * More Ort prefix changes for consistency * Fix C# methods * More C# fixes --- .../InferenceSession.cs | 8 +- .../NamedOnnxValue.cs | 2 +- .../NativeApiStatus.cs | 2 +- .../Microsoft.ML.OnnxRuntime/NativeMethods.cs | 84 +++---- .../NativeOnnxTensorMemory.cs | 2 +- .../Microsoft.ML.OnnxRuntime/OnnxRuntime.cs | 2 +- .../core/framework/onnx_object_cxx.h | 6 +- .../core/session/onnxruntime_c_api.h | 235 +++++++++--------- .../core/session/onnxruntime_cxx_api.h | 26 +- onnxruntime/core/framework/allocator.cc | 2 +- onnxruntime/core/framework/error_code.cc | 10 +- .../core/framework/error_code_helper.h | 2 +- onnxruntime/core/framework/onnx_object.cc | 6 +- .../core/framework/onnxruntime_typeinfo.cc | 36 +-- .../core/framework/onnxruntime_typeinfo.h | 8 +- .../core/framework/tensor_type_and_shape.cc | 38 +-- .../providers/cpu/cpu_provider_factory.cc | 2 +- onnxruntime/core/providers/cpu/symbols.txt | 16 +- .../providers/cuda/cuda_provider_factory.cc | 2 +- .../mkldnn/mkldnn_provider_factory.cc | 2 +- .../session/default_cpu_allocator_c_api.cc | 4 +- onnxruntime/core/session/onnxruntime_c_api.cc | 166 ++++++------- onnxruntime/test/onnx/TestCase.cc | 26 +- onnxruntime/test/onnx/TestCase.h | 2 +- onnxruntime/test/onnx/main.cc | 8 +- onnxruntime/test/onnx/runner.cc | 38 +-- onnxruntime/test/onnx/runner.h | 10 +- onnxruntime/test/onnxruntime_exec/Runtime.h | 2 +- .../shared_lib/fns_candy_style_transfer.c | 28 +-- onnxruntime/test/shared_lib/test_allocator.cc | 4 +- onnxruntime/test/shared_lib/test_inference.cc | 28 +-- onnxruntime/test/shared_lib/test_io_types.cc | 6 +- onnxruntime/test/util/compare_mlvalue.cc | 8 +- .../test/util/include/test/compare_mlvalue.h | 2 +- .../test/util/include/test_allocator.h | 2 +- 35 files changed, 410 insertions(+), 415 deletions(-) diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs b/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs index c3382f3d4b3dc..8973a982d1094 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs @@ -74,7 +74,7 @@ public InferenceSession(string modelPath, SessionOptions options) { if (_nativeHandle != IntPtr.Zero) { - NativeMethods.ReleaseONNXSession(_nativeHandle); + NativeMethods.OrtReleaseSession(_nativeHandle); _nativeHandle = IntPtr.Zero; } throw e; @@ -178,7 +178,7 @@ internal IReadOnlyCollection Run(IReadOnlyCollection Run(IReadOnlyCollection /// Releases/Unrefs any object, including the Allocator @@ -243,32 +243,32 @@ public enum MemoryType #region Tensor/OnnxValue API [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /* ONNXStatus */ OrtCreateTensorWithDataAsONNXValue( + public static extern IntPtr /* OrtStatus */ OrtCreateTensorWithDataAsOrtValue( IntPtr /* (const OrtAllocatorInfo*) */ allocatorInfo, IntPtr /* (void*) */dataBufferHandle, ulong dataLength, //size_t, TODO: make it portable for x86, arm ulong[] shape, //size_t* or size_t[], TODO: make it portable for x86, arm ulong shapeLength, //size_t, TODO: make it portable for x86, arm TensorElementType type, - out IntPtr /* ONNXValuePtr* */ outputValue); + out IntPtr /* OrtValue** */ outputValue); /// This function doesn't work with string tensor - /// this is a no-copy method whose pointer is only valid until the backing ONNXValuePtr is free'd. + /// this is a no-copy method whose pointer is only valid until the backing OrtValue* is free'd. [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /*(ONNXStatus*)*/ OrtGetTensorMutableData(IntPtr /*(ONNXValue*)*/ value, out IntPtr /* (void**)*/ dataBufferHandle); + public static extern IntPtr /*(OrtStatus*)*/ OrtGetTensorMutableData(IntPtr /*(OrtValue*)*/ value, out IntPtr /* (void**)*/ dataBufferHandle); //[DllImport(nativeLib, CharSet = charSet)] - //public static extern IntPtr /*(ONNXStatus*)*/ OrtGetTensorShapeDimCount(IntPtr /*(ONNXValue*)*/ value, out ulong dimension); //size_t TODO: make it portable for x86, arm + //public static extern IntPtr /*(OrtStatus*)*/ OrtGetTensorShapeDimCount(IntPtr /*(OrtValue*)*/ value, out ulong dimension); //size_t TODO: make it portable for x86, arm //[DllImport(nativeLib, CharSet = charSet)] - //public static extern IntPtr /*(ONNXStatus*)*/ OrtGetTensorShapeElementCount(IntPtr /*(ONNXValue*)*/value, out ulong count); + //public static extern IntPtr /*(OrtStatus*)*/ OrtGetTensorShapeElementCount(IntPtr /*(OrtValue*)*/value, out ulong count); [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(const struct OrtTensorTypeAndShapeInfo*)*/ OrtCastTypeInfoToTensorInfo(IntPtr /*(struct OrtTypeInfo*)*/ typeInfo); [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /*(ONNXStatus*)*/ OrtGetTensorShapeAndType(IntPtr /*(ONNXValue*)*/ value, out IntPtr /*(struct OrtTensorTypeAndShapeInfo*)*/ typeAndShapeInfo); + public static extern IntPtr /*(OrtStatus*)*/ OrtGetTensorShapeAndType(IntPtr /*(OrtValue*)*/ value, out IntPtr /*(struct OrtTensorTypeAndShapeInfo*)*/ typeAndShapeInfo); [DllImport(nativeLib, CharSet = charSet)] public static extern TensorElementType OrtGetTensorElementType(IntPtr /*(const struct OrtTensorTypeAndShapeInfo*)*/ typeAndShapeInfo); @@ -295,7 +295,7 @@ public static extern void OrtGetDimensions( public static extern long OrtGetTensorShapeElementCount(IntPtr /*(const struct OrtTensorTypeAndShapeInfo*)*/ typeAndShapeInfo); [DllImport(nativeLib, CharSet = charSet)] - public static extern void ReleaseONNXValue(IntPtr /*(ONNXValue*)*/ value); + public static extern void OrtReleaseValue(IntPtr /*(OrtValue*)*/ value); #endregion } //class NativeMethods diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxTensorMemory.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxTensorMemory.cs index d48d097240a5e..6b4e22fb71dd5 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxTensorMemory.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxTensorMemory.cs @@ -191,7 +191,7 @@ protected override void Dispose(bool disposing) // do managed objects cleanup } - NativeMethods.ReleaseONNXValue(_onnxValueHandle); + NativeMethods.OrtReleaseValue(_onnxValueHandle); _disposed = true; } diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.cs b/csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.cs index da46fc8a8bb13..1d3abcbf0d8aa 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.cs @@ -70,7 +70,7 @@ public override bool IsInvalid private static void Delete(IntPtr nativePtr) { - NativeMethods.ReleaseONNXEnv(nativePtr); + NativeMethods.OrtReleaseEnv(nativePtr); } protected override bool ReleaseHandle() diff --git a/include/onnxruntime/core/framework/onnx_object_cxx.h b/include/onnxruntime/core/framework/onnx_object_cxx.h index f47389ba0cd2c..9677f8bdda980 100644 --- a/include/onnxruntime/core/framework/onnx_object_cxx.h +++ b/include/onnxruntime/core/framework/onnx_object_cxx.h @@ -16,10 +16,10 @@ namespace onnxruntime { template class ObjectBase { private: - static ONNXObject static_cls; + static OrtObject static_cls; protected: - const ONNXObject* const ORT_ATTRIBUTE_UNUSED cls_; + const OrtObject* const ORT_ATTRIBUTE_UNUSED cls_; std::atomic_int ref_count; ObjectBase() : cls_(&static_cls), ref_count(1) { } @@ -39,7 +39,7 @@ class ObjectBase { }; template -ONNXObject ObjectBase::static_cls = {ObjectBase::OrtAddRefImpl, ObjectBase::OrtReleaseImpl}; +OrtObject ObjectBase::static_cls = {ObjectBase::OrtAddRefImpl, ObjectBase::OrtReleaseImpl}; } // namespace onnxruntime diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 4c9a8b56402f0..1be5004ac1fa6 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -15,7 +15,7 @@ extern "C" { #endif -// SAL2 staffs +// SAL2 Definitions #ifndef _WIN32 #define _In_ #define _In_opt_ @@ -46,7 +46,7 @@ extern "C" { #define ORT_MUST_USE_RESULT __attribute__((warn_unused_result)) #endif -//Any pointer marked with _In_ or _Out_, cannot be NULL. Caller should ensure that. +// Any pointer marked with _In_ or _Out_, cannot be NULL. #ifdef __cplusplus // Windows users should use unicode paths when possible to bypass the MAX_PATH limitation @@ -74,58 +74,75 @@ typedef enum OrtErrorCode { ORT_REQUIREMENT_NOT_REGISTERED = 12 } OrtErrorCode; -// ONNXStatus is always returned as a pointer. nullptr indicates success -typedef void ONNXStatus; +// OrtStatus is always returned as a pointer. nullptr indicates success +typedef void OrtStatus; // __VA_ARGS__ on Windows and Linux are different #define ORT_API(RETURN_TYPE, NAME, ...) \ ORT_EXPORT RETURN_TYPE ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION #define ORT_API_STATUS(NAME, ...) \ - ORT_EXPORT ONNXStatus* ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION ORT_MUST_USE_RESULT + ORT_EXPORT OrtStatus* ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION ORT_MUST_USE_RESULT // Used in *.cc files. Almost as same as ORT_API_STATUS, except without ORT_MUST_USE_RESULT #define ORT_API_STATUS_IMPL(NAME, ...) \ - ORT_EXPORT ONNXStatus* ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION + ORT_EXPORT OrtStatus* ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION -#define DEFINE_RUNTIME_CLASS2(NAME, TYPE) \ - ORT_API(void, Release##NAME, _Frees_ptr_opt_ TYPE* input); +#define ORT_RUNTIME_CLASS2(NAME, TYPE) \ + ORT_API(void, OrtRelease##NAME, _Frees_ptr_opt_ TYPE* input); -#define DEFINE_RUNTIME_CLASS(X) \ - struct X; \ - typedef struct X X; \ - DEFINE_RUNTIME_CLASS2(X, X) +#define ORT_RUNTIME_CLASS(X) \ + struct Ort##X; \ + typedef struct Ort##X Ort##X; \ + ORT_RUNTIME_CLASS2(X, Ort##X) -// ONNXStatus* is pointer to something like this: -// struct ONNXStatus { +// OrtStatus* is pointer to something like this: +// struct OrtStatus { // OrtErrorCode code; // char msg[]; // a null-terminated string, var length // } -DEFINE_RUNTIME_CLASS2(ONNXStatus, void); +ORT_RUNTIME_CLASS2(Status, void); + +// The actual types defined have an Ort prefix +ORT_RUNTIME_CLASS(Provider); +ORT_RUNTIME_CLASS(AllocatorInfo); +ORT_RUNTIME_CLASS(Session); +ORT_RUNTIME_CLASS(Value); +ORT_RUNTIME_CLASS(ValueList); + +struct OrtTypeInfo; +typedef struct OrtTypeInfo OrtTypeInfo; +struct OrtTensorTypeAndShapeInfo; +typedef struct OrtTensorTypeAndShapeInfo OrtTensorTypeAndShapeInfo; +struct OrtRunOptions; +typedef struct OrtRunOptions OrtRunOptions; +struct OrtSessionOptions; +typedef struct OrtSessionOptions OrtSessionOptions; +struct OrtEnv; +typedef struct OrtEnv OrtEnv; /** - * \param msg A null-terminated string. Its content will be copied into the newly created ONNXStatus + * \param msg A null-terminated string. Its content will be copied into the newly created OrtStatus */ -ORT_API(ONNXStatus*, CreateONNXStatus, OrtErrorCode code, _In_ const char* msg) +ORT_API(OrtStatus*, OrtCreateStatus, OrtErrorCode code, _In_ const char* msg) ORT_ALL_ARGS_NONNULL; -ORT_API(OrtErrorCode, OrtGetErrorCode, _In_ const ONNXStatus* status) +ORT_API(OrtErrorCode, OrtGetErrorCode, _In_ const OrtStatus* status) ORT_ALL_ARGS_NONNULL; /** * \param status must not be NULL * \return The error message inside the `status`. Don't free the returned value. */ -ORT_API(const char*, OrtGetErrorMessage, _In_ const ONNXStatus* status) +ORT_API(const char*, OrtGetErrorMessage, _In_ const OrtStatus* status) ORT_ALL_ARGS_NONNULL; // // Tensor Type and Shapes // -struct OrtTensorTypeAndShapeInfo; -//copied from TensorProto::DataType -//Currently, Ort doesn't support complex64, complex128, bfloat16 types -typedef enum OrtTensorElementDataType { +// Copied from TensorProto::DataType +// Currently, Ort doesn't support complex64, complex128, bfloat16 types +typedef enum ONNXTensorElementDataType { ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED = 0, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT = 1, // maps to c type float ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8 = 2, // maps to c type uint8_t @@ -143,42 +160,40 @@ typedef enum OrtTensorElementDataType { ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64 = 14, // complex with float32 real and imaginary components ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128 = 15, // complex with float64 real and imaginary components ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16 = 16, // Non-IEEE floating-point format based on IEEE754 single-precision -} OrtTensorElementDataType; - -//sync with onnx TypeProto oneof -typedef enum OrtType { - ORT_TYPE_UNKNOWN, - ORT_TYPE_TENSOR, - ORT_TYPE_SEQUENCE, - ORT_TYPE_MAP, - ORT_TYPE_OPAQUE, - ORT_TYPE_SPARSETENSOR, -} OrtType; - -struct OrtTypeInfo; +} ONNXTensorElementDataType; + +// Sync with onnx TypeProto oneof +typedef enum ONNXType { + ONNX_TYPE_UNKNOWN, + ONNX_TYPE_TENSOR, + ONNX_TYPE_SEQUENCE, + ONNX_TYPE_MAP, + ONNX_TYPE_OPAQUE, + ONNX_TYPE_SPARSETENSOR, +} ONNXType; /** * Don't free the returned value */ -ORT_API(const struct OrtTensorTypeAndShapeInfo*, OrtCastTypeInfoToTensorInfo, _In_ struct OrtTypeInfo*); +ORT_API(const OrtTensorTypeAndShapeInfo*, OrtCastTypeInfoToTensorInfo, _In_ OrtTypeInfo*); /** * The retured value should be released by calling OrtReleaseObject */ -ORT_API(struct OrtTensorTypeAndShapeInfo*, OrtCreateTensorTypeAndShapeInfo); +ORT_API(OrtTensorTypeAndShapeInfo*, OrtCreateTensorTypeAndShapeInfo); -ORT_API_STATUS(OrtSetTensorElementType, _In_ struct OrtTensorTypeAndShapeInfo*, enum OrtTensorElementDataType type); +ORT_API_STATUS(OrtSetTensorElementType, _In_ OrtTensorTypeAndShapeInfo*, enum ONNXTensorElementDataType type); /** * \param info Created from OrtCreateTensorTypeAndShapeInfo() function * \param dim_values An array with length of `dim_count`. Its elements can contain negative values. * \param dim_count length of dim_values */ -ORT_API_STATUS(OrtSetDims, struct OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count); +ORT_API_STATUS(OrtSetDims, OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count); -ORT_API(enum OrtTensorElementDataType, OrtGetTensorElementType, _In_ const struct OrtTensorTypeAndShapeInfo*); -ORT_API(size_t, OrtGetNumOfDimensions, _In_ const struct OrtTensorTypeAndShapeInfo* info); -ORT_API(void, OrtGetDimensions, _In_ const struct OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, size_t dim_values_length); +ORT_API(enum ONNXTensorElementDataType, OrtGetTensorElementType, _In_ const OrtTensorTypeAndShapeInfo*); +ORT_API(size_t, OrtGetNumOfDimensions, _In_ const OrtTensorTypeAndShapeInfo* info); +ORT_API(void, OrtGetDimensions, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, size_t dim_values_length); /** * How many elements does this tensor have. @@ -191,29 +206,26 @@ ORT_API(void, OrtGetDimensions, _In_ const struct OrtTensorTypeAndShapeInfo* inf * return a negative value if unknown. (That this shape contains a symbolic variable which * represents an unknown dimension.) */ -ORT_API(int64_t, OrtGetTensorShapeElementCount, _In_ const struct OrtTensorTypeAndShapeInfo* info); -struct ONNXValue; +ORT_API(int64_t, OrtGetTensorShapeElementCount, _In_ const OrtTensorTypeAndShapeInfo* info); /** * \param out Should be freed by OrtReleaseObject after use */ -ORT_API_STATUS(OrtGetTensorShapeAndType, _In_ const struct ONNXValue* value, - _Out_ struct OrtTensorTypeAndShapeInfo** out); +ORT_API_STATUS(OrtGetTensorShapeAndType, _In_ const OrtValue* value, _Out_ OrtTensorTypeAndShapeInfo** out); /** - * Get the type information of an ONNXValue + * Get the type information of an OrtValue * \param value * \param out The returned value should be freed by OrtReleaseObject after use */ -ORT_API_STATUS(OrtGetTypeInfo, _In_ const struct ONNXValue* value, struct OrtTypeInfo** out); +ORT_API_STATUS(OrtGetTypeInfo, _In_ const OrtValue* value, OrtTypeInfo** out); -ORT_API(enum OrtType, OrtGetValueType, _In_ const struct ONNXValue* value); +ORT_API(enum ONNXType, OrtGetValueType, _In_ const OrtValue* value); // // OrtRunOptions // -struct OrtRunOptions; -typedef struct OrtRunOptions OrtRunOptions; + /** * \return A pointer of the newly created object. The pointer should be freed by OrtReleaseObject after use */ @@ -229,22 +241,19 @@ ORT_API(const char*, OrtRunOptionsGetRunTag, _In_ OrtRunOptions*); // will exit as soon as possible if the flag is true. ORT_API(void, OrtRunOptionsSetTerminate, _In_ OrtRunOptions*, _In_ bool value); -DEFINE_RUNTIME_CLASS(OrtProvider); - /** - * Just like the IUnknown interface in COM - * Every type inherented from ONNXObject should be deleted by OrtReleaseObject(...). + * Every type inherented from OrtObject should be deleted by OrtReleaseObject(...). */ -typedef struct ONNXObject { - ///returns the new reference count. +typedef struct OrtObject { + // Returns the new reference count. uint32_t(ORT_API_CALL* AddRef)(void* this_); - ///returns the new reference count. + // Returns the new reference count. uint32_t(ORT_API_CALL* Release)(void* this_); - //TODO: implement QueryInterface? -} ONNXObject; + +} OrtObject; /** - * This function is a wrapper to "(*(ONNXObject**)ptr)->AddRef(ptr)" + * This function is a wrapper to "(*(OrtObject**)ptr)->AddRef(ptr)" * WARNING: There is NO type checking in this function. * Before calling this function, caller should make sure current ref count > 0 * \return the new reference count @@ -253,22 +262,19 @@ ORT_API(uint32_t, OrtAddRefToObject, _In_ void* ptr); /** * - * A wrapper to "(*(ONNXObject**)ptr)->Release(ptr)" + * A wrapper to "(*(OrtObject**)ptr)->Release(ptr)" * WARNING: There is NO type checking in this function. * \param ptr Can be NULL. If it's NULL, this function will return zero. * \return the new reference count. */ ORT_API(uint32_t, OrtReleaseObject, _Inout_opt_ void* ptr); -//Inherented from ONNXObject +//Inherented from OrtObject typedef struct OrtProviderFactoryInterface { - ONNXObject parent; - ONNXStatus*(ORT_API_CALL* CreateProvider)(void* this_, OrtProvider** out); + OrtObject parent; + OrtStatus*(ORT_API_CALL* CreateProvider)(void* this_, OrtProvider** out); } OrtProviderFactoryInterface; -struct OrtSessionOptions; -typedef struct OrtSessionOptions OrtSessionOptions; - /** * \return A pointer of the newly created object. The pointer should be freed by OrtReleaseObject after use */ @@ -329,8 +335,6 @@ typedef enum OrtMemType { OrtMemTypeDefault = 0, // the default allocator for execution provider } OrtMemType; -DEFINE_RUNTIME_CLASS(OrtAllocatorInfo); - ORT_API_STATUS(OrtCreateAllocatorInfo, _In_ const char* name1, enum OrtAllocatorType type, int id1, enum OrtMemType mem_type1, _Out_ OrtAllocatorInfo** out); /** @@ -347,9 +351,9 @@ ORT_API(int, OrtAllocatorInfoGetId, _In_ OrtAllocatorInfo* ptr); ORT_API(OrtMemType, OrtAllocatorInfoGetMemType, _In_ OrtAllocatorInfo* ptr); ORT_API(OrtAllocatorType, OrtAllocatorInfoGetType, _In_ OrtAllocatorInfo* ptr); -//inherented from ONNXObject +//inherented from OrtObject typedef struct OrtAllocatorInterface { - struct ONNXObject parent; + struct OrtObject parent; void*(ORT_API_CALL* Alloc)(void* this_, size_t size); void(ORT_API_CALL* Free)(void* this_, void* p); const struct OrtAllocatorInfo*(ORT_API_CALL* Info)(const void* this_); @@ -359,10 +363,7 @@ typedef OrtAllocatorInterface* OrtAllocator; ORT_API(void*, OrtAllocatorAlloc, _Inout_ OrtAllocator* ptr, size_t size); ORT_API(void, OrtAllocatorFree, _Inout_ OrtAllocator* ptr, void* p); -ORT_API(const struct OrtAllocatorInfo*, OrtAllocatorGetInfo, _In_ const OrtAllocator* ptr); - -struct OrtEnv; -typedef struct OrtEnv OrtEnv; +ORT_API(const OrtAllocatorInfo*, OrtAllocatorGetInfo, _In_ const OrtAllocator* ptr); typedef enum OrtLoggingLevel { ORT_LOGGING_LEVEL_kVERBOSE = 0, @@ -391,108 +392,102 @@ ORT_API_STATUS(OrtInitializeWithCustomLogger, OrtLoggingFunction logging_functio _In_ const char* logid, _Out_ OrtEnv** out); -DEFINE_RUNTIME_CLASS(ONNXSession); - -//TODO: document the path separator convention? '/' vs '\' -//TODO: should specify the access characteristics of model_path. Is this read only during the -//execution of OrtCreateInferenceSession, or does the ONNXSession retain a handle to the file/directory -//and continue to access throughout the ONNXSession lifetime? -// What sort of access is needed to model_path : read or read/write? -//TODO: allow loading from an in-memory byte-array +// TODO: document the path separator convention? '/' vs '\' +// TODO: should specify the access characteristics of model_path. Is this read only during the +// execution of OrtCreateInferenceSession, or does the OrtSession retain a handle to the file/directory +// and continue to access throughout the OrtSession lifetime? +// What sort of access is needed to model_path : read or read/write? +// TODO: allow loading from an in-memory byte-array #ifdef _WIN32 ORT_API_STATUS(OrtCreateInferenceSession, _In_ OrtEnv* env, _In_ const wchar_t* model_path, - _In_ const OrtSessionOptions* options, _Out_ ONNXSession** out); + _In_ const OrtSessionOptions* options, _Out_ OrtSession** out); #else ORT_API_STATUS(OrtCreateInferenceSession, _In_ OrtEnv* env, _In_ const char* model_path, - _In_ const OrtSessionOptions* options, _Out_ ONNXSession** out); + _In_ const OrtSessionOptions* options, _Out_ OrtSession** out); #endif -DEFINE_RUNTIME_CLASS(ONNXValue); - -///Call OrtReleaseObject to release the returned value +// Call OrtReleaseObject to release the returned value ORT_API_STATUS(OrtCreateDefaultAllocator, _Out_ OrtAllocator** out); /** - * Create a tensor from an allocator. ReleaseONNXValue will also release the buffer inside the output value + * Create a tensor from an allocator. OrtReleaseValue will also release the buffer inside the output value * \param out will keep a reference to the allocator, without reference counting(will be fixed). Should be freed by - * calling ReleaseONNXValue + * calling OrtReleaseValue * \param type must be one of TENSOR_ELEMENT_DATA_TYPE_xxxx */ -ORT_API_STATUS(OrtCreateTensorAsONNXValue, _Inout_ OrtAllocator* allocator, - _In_ const size_t* shape, size_t shape_len, OrtTensorElementDataType type, - _Out_ ONNXValue** out); +ORT_API_STATUS(OrtCreateTensorAsOrtValue, _Inout_ OrtAllocator* allocator, + _In_ const size_t* shape, size_t shape_len, ONNXTensorElementDataType type, + _Out_ OrtValue** out); /** * Create a tensor with user's buffer. You can fill the buffer either before calling this function or after. - * p_data is owned by caller. ReleaseONNXValue won't release p_data. - * \param out Should be freed by calling ReleaseONNXValue + * p_data is owned by caller. OrtReleaseValue won't release p_data. + * \param out Should be freed by calling OrtReleaseValue */ -ORT_API_STATUS(OrtCreateTensorWithDataAsONNXValue, _In_ const OrtAllocatorInfo* info, +ORT_API_STATUS(OrtCreateTensorWithDataAsOrtValue, _In_ const OrtAllocatorInfo* info, _In_ void* p_data, size_t p_data_len, _In_ const size_t* shape, size_t shape_len, - OrtTensorElementDataType type, _Out_ ONNXValue** out); + ONNXTensorElementDataType type, _Out_ OrtValue** out); /// This function doesn't work with string tensor -/// this is a no-copy method whose pointer is only valid until the backing ONNXValue is free'd. -ORT_API_STATUS(OrtGetTensorMutableData, _Inout_ ONNXValue* value, _Out_ void** out); +/// this is a no-copy method whose pointer is only valid until the backing OrtValue is free'd. +ORT_API_STATUS(OrtGetTensorMutableData, _Inout_ OrtValue* value, _Out_ void** out); /** - * Test if an ONNXValue is a tensor + * Test if an OrtValue is a tensor * \return zero, false. non-zero true */ -ORT_API(int, OrtIsTensor, _In_ const ONNXValue* value); +ORT_API(int, OrtIsTensor, _In_ const OrtValue* value); /** * \param value A tensor created from OrtCreateTensor*** function. * \param s each A string array. Each string in this array must be null terminated. * \param s_len length of s */ -ORT_API_STATUS(OrtFillStringTensor, _In_ ONNXValue* value, _In_ const char* const* s, size_t s_len); +ORT_API_STATUS(OrtFillStringTensor, _In_ OrtValue* value, _In_ const char* const* s, size_t s_len); /** * \param value A tensor created from OrtCreateTensor*** function. * \param len total data length, not including the trailing '\0' chars. */ -ORT_API_STATUS(OrtGetStringTensorDataLength, _In_ const ONNXValue* value, _Out_ size_t* len); +ORT_API_STATUS(OrtGetStringTensorDataLength, _In_ const OrtValue* value, _Out_ size_t* len); /** * \param s string contents. Each string is NOT null-terminated. * \param value A tensor created from OrtCreateTensor*** function. * \param s_len total data length, get it from OrtGetStringTensorDataLength */ -ORT_API_STATUS(OrtGetStringTensorContent, _In_ const ONNXValue* value, _Out_ void* s, size_t s_len, +ORT_API_STATUS(OrtGetStringTensorContent, _In_ const OrtValue* value, _Out_ void* s, size_t s_len, _Out_ size_t* offsets, size_t offsets_len); -DEFINE_RUNTIME_CLASS(ONNXValueList); - -ORT_API_STATUS(OrtRunInference, _Inout_ ONNXSession* sess, +ORT_API_STATUS(OrtRunInference, _Inout_ OrtSession* sess, _In_ OrtRunOptions* run_options, - _In_ const char* const* input_names, _In_ const ONNXValue* const* input, size_t input_len, - _In_ const char* const* output_names, size_t output_names_len, _Out_ ONNXValue** output); + _In_ const char* const* input_names, _In_ const OrtValue* const* input, size_t input_len, + _In_ const char* const* output_names, size_t output_names_len, _Out_ OrtValue** output); -ORT_API_STATUS(OrtInferenceSessionGetInputCount, _In_ const ONNXSession* sess, _Out_ size_t* out); -ORT_API_STATUS(OrtInferenceSessionGetOutputCount, _In_ const ONNXSession* sess, _Out_ size_t* out); +ORT_API_STATUS(OrtInferenceSessionGetInputCount, _In_ const OrtSession* sess, _Out_ size_t* out); +ORT_API_STATUS(OrtInferenceSessionGetOutputCount, _In_ const OrtSession* sess, _Out_ size_t* out); /** * \param out should be freed by OrtReleaseObject after use */ -ORT_API_STATUS(OrtInferenceSessionGetInputTypeInfo, _In_ const ONNXSession* sess, size_t index, _Out_ struct OrtTypeInfo** out); +ORT_API_STATUS(OrtInferenceSessionGetInputTypeInfo, _In_ const OrtSession* sess, size_t index, _Out_ OrtTypeInfo** out); /** * \param out should be freed by OrtReleaseObject after use */ -ORT_API_STATUS(OrtInferenceSessionGetOutputTypeInfo, _In_ const ONNXSession* sess, size_t index, _Out_ struct OrtTypeInfo** out); +ORT_API_STATUS(OrtInferenceSessionGetOutputTypeInfo, _In_ const OrtSession* sess, size_t index, _Out_ OrtTypeInfo** out); -ORT_API_STATUS(OrtInferenceSessionGetInputName, _In_ const ONNXSession* sess, size_t index, +ORT_API_STATUS(OrtInferenceSessionGetInputName, _In_ const OrtSession* sess, size_t index, _Inout_ OrtAllocator* allocator, _Out_ char** value); -ORT_API_STATUS(OrtInferenceSessionGetOutputName, _In_ const ONNXSession* sess, size_t index, +ORT_API_STATUS(OrtInferenceSessionGetOutputName, _In_ const OrtSession* sess, size_t index, _Inout_ OrtAllocator* allocator, _Out_ char** value); -ORT_API_STATUS(OrtTensorProtoToONNXValue, _Inout_ OrtAllocator* allocator, - _In_ const void* input, int input_len, _Out_ ONNXValue** out); +ORT_API_STATUS(OrtTensorProtoToOrtValue, _Inout_ OrtAllocator* allocator, + _In_ const void* input, int input_len, _Out_ OrtValue** out); /** * Deprecated. Please use OrtReleaseObject */ -ORT_API(void, ReleaseONNXEnv, OrtEnv* env); +ORT_API(void, OrtReleaseEnv, OrtEnv* env); #ifdef __cplusplus } diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_api.h b/include/onnxruntime/core/session/onnxruntime_cxx_api.h index d9503688ee125..72ea38c995dbf 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_api.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_api.h @@ -11,10 +11,10 @@ //TODO: encode error code in the message? #define ORT_THROW_ON_ERROR(expr) \ do { \ - ONNXStatus* onnx_status = (expr); \ + OrtStatus* onnx_status = (expr); \ if (onnx_status != nullptr) { \ std::string ort_error_message = OrtGetErrorMessage(onnx_status); \ - ReleaseONNXStatus(onnx_status); \ + OrtReleaseStatus(onnx_status); \ throw std::runtime_error(ort_error_message); \ } \ } while (0); @@ -29,7 +29,7 @@ template <> \ struct default_delete { \ void operator()(Ort##TYPE_NAME* ptr) { \ - (*reinterpret_cast(ptr))->Release(ptr); \ + (*reinterpret_cast(ptr))->Release(ptr); \ } \ }; \ } @@ -89,14 +89,14 @@ class SessionOptionsWrapper { return SessionOptionsWrapper(env_, p); } #ifdef _WIN32 - ONNXSession* OrtCreateInferenceSession(_In_ const wchar_t* model_path) { - ONNXSession* ret; + OrtSession* OrtCreateInferenceSession(_In_ const wchar_t* model_path) { + OrtSession* ret; ORT_THROW_ON_ERROR(::OrtCreateInferenceSession(env_, model_path, value.get(), &ret)); return ret; } #else - ONNXSession* OrtCreateInferenceSession(_In_ const char* model_path) { - ONNXSession* ret; + OrtSession* OrtCreateInferenceSession(_In_ const char* model_path) { + OrtSession* ret; ORT_THROW_ON_ERROR(::OrtCreateInferenceSession(env_, model_path, value.get(), &ret)); return ret; } @@ -105,15 +105,15 @@ class SessionOptionsWrapper { OrtAddCustomOp(value.get(), custom_op_path); } }; -inline ONNXValue* OrtCreateTensorAsONNXValue(_Inout_ OrtAllocator* env, const std::vector& shape, OrtTensorElementDataType type) { - ONNXValue* ret; - ORT_THROW_ON_ERROR(::OrtCreateTensorAsONNXValue(env, shape.data(), shape.size(), type, &ret)); +inline OrtValue* OrtCreateTensorAsOrtValue(_Inout_ OrtAllocator* env, const std::vector& shape, ONNXTensorElementDataType type) { + OrtValue* ret; + ORT_THROW_ON_ERROR(::OrtCreateTensorAsOrtValue(env, shape.data(), shape.size(), type, &ret)); return ret; } -inline ONNXValue* OrtCreateTensorWithDataAsONNXValue(_In_ const OrtAllocatorInfo* info, _In_ void* p_data, size_t p_data_len, const std::vector& shape, OrtTensorElementDataType type) { - ONNXValue* ret; - ORT_THROW_ON_ERROR(::OrtCreateTensorWithDataAsONNXValue(info, p_data, p_data_len, shape.data(), shape.size(), type, &ret)); +inline OrtValue* OrtCreateTensorWithDataAsOrtValue(_In_ const OrtAllocatorInfo* info, _In_ void* p_data, size_t p_data_len, const std::vector& shape, ONNXTensorElementDataType type) { + OrtValue* ret; + ORT_THROW_ON_ERROR(::OrtCreateTensorWithDataAsOrtValue(info, p_data, p_data_len, shape.data(), shape.size(), type, &ret)); return ret; } diff --git a/onnxruntime/core/framework/allocator.cc b/onnxruntime/core/framework/allocator.cc index ca9d5e158e183..26eaac40c7e64 100644 --- a/onnxruntime/core/framework/allocator.cc +++ b/onnxruntime/core/framework/allocator.cc @@ -36,7 +36,7 @@ ORT_API_STATUS_IMPL(OrtCreateAllocatorInfo, const char* name1, OrtAllocatorType return nullptr; } -ORT_API(void, ReleaseOrtAllocatorInfo, OrtAllocatorInfo* p) { +ORT_API(void, OrtReleaseAllocatorInfo, OrtAllocatorInfo* p) { delete p; } diff --git a/onnxruntime/core/framework/error_code.cc b/onnxruntime/core/framework/error_code.cc index 8513905034f43..87cff0ddbfffe 100644 --- a/onnxruntime/core/framework/error_code.cc +++ b/onnxruntime/core/framework/error_code.cc @@ -7,7 +7,7 @@ #include using onnxruntime::common::Status; -ORT_API(ONNXStatus*, CreateONNXStatus, OrtErrorCode code, const char* msg) { +ORT_API(OrtStatus*, OrtCreateStatus, OrtErrorCode code, const char* msg) { assert(!(code == 0 && msg != nullptr)); size_t clen = strlen(msg); size_t len = clen + 1 + sizeof(int); @@ -21,7 +21,7 @@ ORT_API(ONNXStatus*, CreateONNXStatus, OrtErrorCode code, const char* msg) { return ret; } namespace onnxruntime { -ONNXStatus* ToONNXStatus(const Status& st) { +OrtStatus* ToOrtStatus(const Status& st) { if (st.IsOK()) return nullptr; size_t clen = st.ErrorMessage().length(); @@ -36,10 +36,10 @@ ONNXStatus* ToONNXStatus(const Status& st) { return ret; } } // namespace onnxruntime -ORT_API(OrtErrorCode, OrtGetErrorCode, _In_ const ONNXStatus* status) { - return *reinterpret_cast(const_cast(status)); +ORT_API(OrtErrorCode, OrtGetErrorCode, _In_ const OrtStatus* status) { + return *reinterpret_cast(const_cast(status)); } -ORT_API(const char*, OrtGetErrorMessage, _In_ const ONNXStatus* status) { +ORT_API(const char*, OrtGetErrorMessage, _In_ const OrtStatus* status) { return reinterpret_cast(status) + sizeof(int); } diff --git a/onnxruntime/core/framework/error_code_helper.h b/onnxruntime/core/framework/error_code_helper.h index 9b75e7a138e50..90c02f5f7b3e4 100644 --- a/onnxruntime/core/framework/error_code_helper.h +++ b/onnxruntime/core/framework/error_code_helper.h @@ -6,5 +6,5 @@ #include "core/common/status.h" namespace onnxruntime { -ONNXStatus* ToONNXStatus(const onnxruntime::common::Status& st); +OrtStatus* ToOrtStatus(const onnxruntime::common::Status& st); }; diff --git a/onnxruntime/core/framework/onnx_object.cc b/onnxruntime/core/framework/onnx_object.cc index d7232e477c8d7..199b68dc5827b 100644 --- a/onnxruntime/core/framework/onnx_object.cc +++ b/onnxruntime/core/framework/onnx_object.cc @@ -5,17 +5,17 @@ #include ORT_API(uint32_t, OrtAddRefToObject, void* ptr) { - return (*static_cast(ptr))->AddRef(ptr); + return (*static_cast(ptr))->AddRef(ptr); } ORT_API(uint32_t, OrtReleaseObject, void* ptr) { if (ptr == nullptr) return 0; - return (*static_cast(ptr))->Release(ptr); + return (*static_cast(ptr))->Release(ptr); } namespace { struct ObjectImpl { - const ONNXObject* const cls; + const OrtObject* const cls; std::atomic_int ref_count; }; } // namespace diff --git a/onnxruntime/core/framework/onnxruntime_typeinfo.cc b/onnxruntime/core/framework/onnxruntime_typeinfo.cc index 0693daa2f4ff1..250d83e4dcac1 100644 --- a/onnxruntime/core/framework/onnxruntime_typeinfo.cc +++ b/onnxruntime/core/framework/onnxruntime_typeinfo.cc @@ -13,7 +13,7 @@ using onnxruntime::MLFloat16; using onnxruntime::Tensor; using onnxruntime::TensorShape; -OrtTypeInfo::OrtTypeInfo(OrtType type1, void* data1) noexcept : type(type1), data(data1) { +OrtTypeInfo::OrtTypeInfo(ONNXType type1, void* data1) noexcept : type(type1), data(data1) { } OrtTypeInfo::~OrtTypeInfo() { @@ -22,34 +22,34 @@ OrtTypeInfo::~OrtTypeInfo() { } ORT_API(const struct OrtTensorTypeAndShapeInfo*, OrtCastTypeInfoToTensorInfo, _In_ struct OrtTypeInfo* input) { - return input->type == ORT_TYPE_TENSOR ? reinterpret_cast(input->data) : nullptr; + return input->type == ONNX_TYPE_TENSOR ? reinterpret_cast(input->data) : nullptr; } -ONNXStatus* GetTensorShapeAndType(const TensorShape* shape, const onnxruntime::DataTypeImpl* tensor_data_type, OrtTensorTypeAndShapeInfo** out); +OrtStatus* GetTensorShapeAndType(const TensorShape* shape, const onnxruntime::DataTypeImpl* tensor_data_type, OrtTensorTypeAndShapeInfo** out); -ONNXStatus* OrtTypeInfo::FromDataTypeImpl(const onnxruntime::DataTypeImpl* input, const TensorShape* shape, const onnxruntime::DataTypeImpl* tensor_data_type, OrtTypeInfo** out) { +OrtStatus* OrtTypeInfo::FromDataTypeImpl(const onnxruntime::DataTypeImpl* input, const TensorShape* shape, const onnxruntime::DataTypeImpl* tensor_data_type, OrtTypeInfo** out) { if (input == nullptr) { - *out = new OrtTypeInfo(ORT_TYPE_UNKNOWN, nullptr); + *out = new OrtTypeInfo(ONNX_TYPE_UNKNOWN, nullptr); return nullptr; } if (input == DataTypeImpl::GetType()) { OrtTensorTypeAndShapeInfo* info = nullptr; if (tensor_data_type != nullptr) { - ONNXStatus* st = GetTensorShapeAndType(shape, tensor_data_type, &info); + OrtStatus* st = GetTensorShapeAndType(shape, tensor_data_type, &info); if (st != nullptr) return st; } - *out = new OrtTypeInfo(ORT_TYPE_TENSOR, info); + *out = new OrtTypeInfo(ONNX_TYPE_TENSOR, info); return nullptr; } if (input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType()) { - *out = new OrtTypeInfo(ORT_TYPE_MAP, nullptr); + *out = new OrtTypeInfo(ONNX_TYPE_MAP, nullptr); return nullptr; } if (input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType() || input == DataTypeImpl::GetType()) { - *out = new OrtTypeInfo(ORT_TYPE_SEQUENCE, nullptr); + *out = new OrtTypeInfo(ONNX_TYPE_SEQUENCE, nullptr); return nullptr; } - return CreateONNXStatus(ORT_NOT_IMPLEMENTED, "not implemented"); + return OrtCreateStatus(ORT_NOT_IMPLEMENTED, "not implemented"); } const DataTypeImpl* ElementTypeFromProto(ONNX_NAMESPACE::TensorProto_DataType type) { @@ -85,11 +85,11 @@ const DataTypeImpl* ElementTypeFromProto(ONNX_NAMESPACE::TensorProto_DataType ty } } -ONNXStatus* OrtTypeInfo::FromDataTypeImpl(const onnx::TypeProto* input, OrtTypeInfo** out) { +OrtStatus* OrtTypeInfo::FromDataTypeImpl(const onnx::TypeProto* input, OrtTypeInfo** out) { if (input->has_tensor_type()) { const ::onnx::TypeProto_Tensor& onnx_tensor_info = input->tensor_type(); const DataTypeImpl* type = ElementTypeFromProto(onnx_tensor_info.elem_type()); - ONNXStatus* st; + OrtStatus* st; OrtTensorTypeAndShapeInfo* info = nullptr; if (onnx_tensor_info.has_shape()) { const ::onnx::TensorShapeProto& s = onnx_tensor_info.shape(); @@ -104,24 +104,24 @@ ONNXStatus* OrtTypeInfo::FromDataTypeImpl(const onnx::TypeProto* input, OrtTypeI } if (st != nullptr) return st; - *out = new OrtTypeInfo(ORT_TYPE_TENSOR, info); + *out = new OrtTypeInfo(ONNX_TYPE_TENSOR, info); return nullptr; } if (input->has_sequence_type()) { - *out = new OrtTypeInfo(ORT_TYPE_SEQUENCE, nullptr); + *out = new OrtTypeInfo(ONNX_TYPE_SEQUENCE, nullptr); return nullptr; } if (input->has_map_type()) { - *out = new OrtTypeInfo(ORT_TYPE_MAP, nullptr); + *out = new OrtTypeInfo(ONNX_TYPE_MAP, nullptr); return nullptr; } if (input->has_opaque_type()) { - *out = new OrtTypeInfo(ORT_TYPE_OPAQUE, nullptr); + *out = new OrtTypeInfo(ONNX_TYPE_OPAQUE, nullptr); return nullptr; } if (input->has_sparse_tensor_type()) { - *out = new OrtTypeInfo(ORT_TYPE_SPARSETENSOR, nullptr); + *out = new OrtTypeInfo(ONNX_TYPE_SPARSETENSOR, nullptr); return nullptr; } - return CreateONNXStatus(ORT_NOT_IMPLEMENTED, "not implemented"); + return OrtCreateStatus(ORT_NOT_IMPLEMENTED, "not implemented"); } diff --git a/onnxruntime/core/framework/onnxruntime_typeinfo.h b/onnxruntime/core/framework/onnxruntime_typeinfo.h index 7c2f5331bc76c..9cbef90d24bde 100644 --- a/onnxruntime/core/framework/onnxruntime_typeinfo.h +++ b/onnxruntime/core/framework/onnxruntime_typeinfo.h @@ -22,17 +22,17 @@ struct OrtTypeInfo : public onnxruntime::ObjectBase { public: friend class onnxruntime::ObjectBase; - OrtType type = ORT_TYPE_UNKNOWN; + ONNXType type = ONNX_TYPE_UNKNOWN; //owned by this void* data = nullptr; OrtTypeInfo(const OrtTypeInfo& other) = delete; OrtTypeInfo& operator=(const OrtTypeInfo& other) = delete; - static ONNXStatus* FromDataTypeImpl(const onnxruntime::DataTypeImpl* input, const onnxruntime::TensorShape* shape, + static OrtStatus* FromDataTypeImpl(const onnxruntime::DataTypeImpl* input, const onnxruntime::TensorShape* shape, const onnxruntime::DataTypeImpl* tensor_data_type, OrtTypeInfo** out); - static ONNXStatus* FromDataTypeImpl(const onnx::TypeProto*, OrtTypeInfo** out); + static OrtStatus* FromDataTypeImpl(const onnx::TypeProto*, OrtTypeInfo** out); private: - OrtTypeInfo(OrtType type, void* data) noexcept; + OrtTypeInfo(ONNXType type, void* data) noexcept; ~OrtTypeInfo(); }; diff --git a/onnxruntime/core/framework/tensor_type_and_shape.cc b/onnxruntime/core/framework/tensor_type_and_shape.cc index 76f0c88957daa..eb22ef015c12e 100644 --- a/onnxruntime/core/framework/tensor_type_and_shape.cc +++ b/onnxruntime/core/framework/tensor_type_and_shape.cc @@ -18,7 +18,7 @@ struct OrtTensorTypeAndShapeInfo : public onnxruntime::ObjectBase; - OrtTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; + ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; onnxruntime::TensorShape shape; static OrtTensorTypeAndShapeInfo* Create() { @@ -39,14 +39,14 @@ struct OrtTensorTypeAndShapeInfo : public onnxruntime::ObjectBasetype = type; return nullptr; @@ -60,7 +60,7 @@ ORT_API_STATUS_IMPL(OrtSetDims, _In_ OrtTensorTypeAndShapeInfo* this_ptr, _In_ c API_IMPL_END } -ORT_API(enum OrtTensorElementDataType, OrtGetTensorElementType, _In_ const struct OrtTensorTypeAndShapeInfo* info) { +ORT_API(enum ONNXTensorElementDataType, OrtGetTensorElementType, _In_ const struct OrtTensorTypeAndShapeInfo* info) { return info->type; } @@ -76,12 +76,12 @@ ORT_API(int64_t, OrtGetTensorShapeElementCount, _In_ const OrtTensorTypeAndShape return this_ptr->shape.Size(); } -struct ONNXValue; +struct OrtValue; namespace { -inline OrtTensorElementDataType MLDataTypeToOnnxRuntimeTensorElementDataType( +inline ONNXTensorElementDataType MLDataTypeToOnnxRuntimeTensorElementDataType( const onnxruntime::DataTypeImpl* cpp_type) { - OrtTensorElementDataType type; + ONNXTensorElementDataType type; if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; } else if (cpp_type == onnxruntime::DataTypeImpl::GetType()) { @@ -115,10 +115,10 @@ inline OrtTensorElementDataType MLDataTypeToOnnxRuntimeTensorElementDataType( } } // namespace -ONNXStatus* GetTensorShapeAndType(const onnxruntime::TensorShape* shape, const onnxruntime::DataTypeImpl* tensor_data_type, OrtTensorTypeAndShapeInfo** out) { - OrtTensorElementDataType type = MLDataTypeToOnnxRuntimeTensorElementDataType(tensor_data_type); +OrtStatus* GetTensorShapeAndType(const onnxruntime::TensorShape* shape, const onnxruntime::DataTypeImpl* tensor_data_type, OrtTensorTypeAndShapeInfo** out) { + ONNXTensorElementDataType type = MLDataTypeToOnnxRuntimeTensorElementDataType(tensor_data_type); if (ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED == type) { - return CreateONNXStatus(ORT_FAIL, "Not implemented"); + return OrtCreateStatus(ORT_FAIL, "Not implemented"); } OrtTensorTypeAndShapeInfo* ret = OrtCreateTensorTypeAndShapeInfo(); auto status = OrtSetTensorElementType(ret, type); @@ -137,7 +137,7 @@ ONNXStatus* GetTensorShapeAndType(const onnxruntime::TensorShape* shape, const o return nullptr; } -ORT_API_STATUS_IMPL(OrtGetTensorShapeAndType, _In_ const ONNXValue* value, +ORT_API_STATUS_IMPL(OrtGetTensorShapeAndType, _In_ const OrtValue* value, _Out_ OrtTensorTypeAndShapeInfo** out) { API_IMPL_BEGIN auto v = reinterpret_cast(value); @@ -146,30 +146,30 @@ ORT_API_STATUS_IMPL(OrtGetTensorShapeAndType, _In_ const ONNXValue* value, API_IMPL_END } -ORT_API(enum OrtType, OrtGetValueType, _In_ const ONNXValue* value) { +ORT_API(enum ONNXType, OrtGetValueType, _In_ const OrtValue* value) { try { auto v = reinterpret_cast(value); onnxruntime::MLDataType type = v->Type(); OrtTypeInfo* out; - ONNXStatus* ptr = OrtTypeInfo::FromDataTypeImpl(type, nullptr, nullptr, &out); + OrtStatus* ptr = OrtTypeInfo::FromDataTypeImpl(type, nullptr, nullptr, &out); if (ptr != nullptr) { - ReleaseONNXStatus(ptr); - return ORT_TYPE_UNKNOWN; + OrtReleaseStatus(ptr); + return ONNX_TYPE_UNKNOWN; } - OrtType ret = out->type; + ONNXType ret = out->type; OrtReleaseObject(out); return ret; } catch (std::exception&) { - return ORT_TYPE_UNKNOWN; + return ONNX_TYPE_UNKNOWN; } } /** - * Get the type information of an ONNXValue + * Get the type information of an OrtValue * \param value * \return The returned value should be freed by OrtReleaseObject after use */ -ORT_API_STATUS_IMPL(OrtGetTypeInfo, _In_ const ONNXValue* value, struct OrtTypeInfo** out) { +ORT_API_STATUS_IMPL(OrtGetTypeInfo, _In_ const OrtValue* value, struct OrtTypeInfo** out) { auto v = reinterpret_cast(value); onnxruntime::MLDataType type = v->Type(); if (type == nullptr) { diff --git a/onnxruntime/core/providers/cpu/cpu_provider_factory.cc b/onnxruntime/core/providers/cpu/cpu_provider_factory.cc index 98813317834a1..dc8f9532bdd9c 100644 --- a/onnxruntime/core/providers/cpu/cpu_provider_factory.cc +++ b/onnxruntime/core/providers/cpu/cpu_provider_factory.cc @@ -15,7 +15,7 @@ struct CpuProviderFactory { CpuProviderFactory(); }; -ONNXStatus* ORT_API_CALL CreateCpu(void* this_, OrtProvider** out) { +OrtStatus* ORT_API_CALL CreateCpu(void* this_, OrtProvider** out) { CPUExecutionProviderInfo info; CpuProviderFactory* this_ptr = (CpuProviderFactory*)this_; info.create_arena = this_ptr->create_arena; diff --git a/onnxruntime/core/providers/cpu/symbols.txt b/onnxruntime/core/providers/cpu/symbols.txt index f60ba073a64bf..c7955f9b2beab 100644 --- a/onnxruntime/core/providers/cpu/symbols.txt +++ b/onnxruntime/core/providers/cpu/symbols.txt @@ -17,9 +17,9 @@ OrtCreateDefaultAllocator OrtCreateInferenceSession OrtCreateRunOptions OrtCreateSessionOptions -OrtCreateTensorAsONNXValue +OrtCreateTensorAsOrtValue OrtCreateTensorTypeAndShapeInfo -OrtCreateTensorWithDataAsONNXValue +OrtCreateTensorWithDataAsOrtValue OrtDisableCpuMemArena OrtDisableMemPattern OrtDisableProfiling @@ -50,7 +50,12 @@ OrtInferenceSessionGetOutputTypeInfo OrtInitialize OrtInitializeWithCustomLogger OrtIsTensor +OrtReleaseAllocatorInfo +OrtReleaseEnv OrtReleaseObject +OrtReleaseSession +OrtReleaseStatus +OrtReleaseValue OrtRunInference OrtRunOptionsGetRunLogVerbosityLevel OrtRunOptionsGetRunTag @@ -63,9 +68,4 @@ OrtSetSessionLogId OrtSetSessionLogVerbosityLevel OrtSetSessionThreadPoolSize OrtSetTensorElementType -OrtTensorProtoToONNXValue -ReleaseONNXEnv -ReleaseOrtAllocatorInfo -ReleaseONNXSession -ReleaseONNXStatus -ReleaseONNXValue +OrtTensorProtoToOrtValue diff --git a/onnxruntime/core/providers/cuda/cuda_provider_factory.cc b/onnxruntime/core/providers/cuda/cuda_provider_factory.cc index c9030b27c64ff..fb419ac9221d0 100644 --- a/onnxruntime/core/providers/cuda/cuda_provider_factory.cc +++ b/onnxruntime/core/providers/cuda/cuda_provider_factory.cc @@ -15,7 +15,7 @@ struct CUDAProviderFactory { CUDAProviderFactory(); }; -ONNXStatus* ORT_API_CALL CreateCuda(void* this_, OrtProvider** out) { +OrtStatus* ORT_API_CALL CreateCuda(void* this_, OrtProvider** out) { CUDAExecutionProviderInfo info; CUDAProviderFactory* this_ptr = (CUDAProviderFactory*)this_; info.device_id = this_ptr->device_id; diff --git a/onnxruntime/core/providers/mkldnn/mkldnn_provider_factory.cc b/onnxruntime/core/providers/mkldnn/mkldnn_provider_factory.cc index d9d9a543462b6..f9a7b0f063b94 100644 --- a/onnxruntime/core/providers/mkldnn/mkldnn_provider_factory.cc +++ b/onnxruntime/core/providers/mkldnn/mkldnn_provider_factory.cc @@ -15,7 +15,7 @@ struct MkldnnProviderFactory { MkldnnProviderFactory(); }; -ONNXStatus* ORT_API_CALL CreateMkldnn(void* this_, OrtProvider** out) { +OrtStatus* ORT_API_CALL CreateMkldnn(void* this_, OrtProvider** out) { MKLDNNExecutionProviderInfo info; MkldnnProviderFactory* this_ptr = (MkldnnProviderFactory*)this_; info.create_arena = this_ptr->create_arena; diff --git a/onnxruntime/core/session/default_cpu_allocator_c_api.cc b/onnxruntime/core/session/default_cpu_allocator_c_api.cc index 4267a7ce83cbe..8957fbdf4fe63 100644 --- a/onnxruntime/core/session/default_cpu_allocator_c_api.cc +++ b/onnxruntime/core/session/default_cpu_allocator_c_api.cc @@ -44,7 +44,7 @@ OrtDefaultAllocator() : ref_count_(1) { } ~OrtDefaultAllocator() { assert(ref_count_ == 0); - ReleaseOrtAllocatorInfo(cpuAllocatorInfo); + OrtReleaseAllocatorInfo(cpuAllocatorInfo); } public: @@ -71,7 +71,7 @@ ORT_ALLOCATOR_IMPL_END #define API_IMPL_END \ } \ catch (std::exception & ex) { \ - return CreateONNXStatus(ORT_RUNTIME_EXCEPTION, ex.what()); \ + return OrtCreateStatus(ORT_RUNTIME_EXCEPTION, ex.what()); \ } OrtAllocatorInterface OrtDefaultAllocator::table_ = { diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc index fe3daab56735a..627821cb09b6b 100644 --- a/onnxruntime/core/session/onnxruntime_c_api.cc +++ b/onnxruntime/core/session/onnxruntime_c_api.cc @@ -35,7 +35,7 @@ using onnxruntime::MLStatus; using onnxruntime::MLValue; using onnxruntime::OutputDefList; using onnxruntime::Tensor; -using onnxruntime::ToONNXStatus; +using onnxruntime::ToOrtStatus; using onnxruntime::common::Status; #define ORT_API_RETURN_IF_ERROR(expr) \ @@ -65,10 +65,10 @@ struct OrtEnv : public onnxruntime::ObjectBase { }; #define API_IMPL_BEGIN try { -#define API_IMPL_END \ - } \ - catch (std::exception & ex) { \ - return CreateONNXStatus(ORT_RUNTIME_EXCEPTION, ex.what()); \ +#define API_IMPL_END \ + } \ + catch (std::exception & ex) { \ + return OrtCreateStatus(ORT_RUNTIME_EXCEPTION, ex.what()); \ } #define TENSOR_READ_API_BEGIN \ @@ -113,7 +113,7 @@ ORT_API_STATUS_IMPL(OrtInitializeWithCustomLogger, OrtLoggingFunction logging_fu Status status = Environment::Create(env); if (status.IsOK()) *out = new OrtEnv(env.release(), default_logging_manager.release()); - return ToONNXStatus(status); + return ToOrtStatus(status); API_IMPL_END } @@ -129,11 +129,11 @@ ORT_API_STATUS_IMPL(OrtInitialize, OrtLoggingLevel default_warning_level, Status status = Environment::Create(env); if (status.IsOK()) *out = new OrtEnv(env.release(), default_logging_manager.release()); - return ToONNXStatus(status); + return ToOrtStatus(status); API_IMPL_END } -ORT_API_STATUS_IMPL(OrtGetStringTensorDataLength, _In_ const ONNXValue* value, _Out_ size_t* out) { +ORT_API_STATUS_IMPL(OrtGetStringTensorDataLength, _In_ const OrtValue* value, _Out_ size_t* out) { TENSOR_READ_API_BEGIN const auto* src = tensor.Data(); int64_t len = tensor.Shape().Size(); @@ -144,17 +144,17 @@ ORT_API_STATUS_IMPL(OrtGetStringTensorDataLength, _In_ const ONNXValue* value, _ } *out = ret; } else - return CreateONNXStatus(ORT_INVALID_ARGUMENT, "shape is invalid"); + return OrtCreateStatus(ORT_INVALID_ARGUMENT, "shape is invalid"); return nullptr; API_IMPL_END } -ORT_API_STATUS_IMPL(OrtFillStringTensor, _In_ ONNXValue* value, _In_ const char* const* s, size_t s_len) { +ORT_API_STATUS_IMPL(OrtFillStringTensor, _In_ OrtValue* value, _In_ const char* const* s, size_t s_len) { TENSOR_READWRITE_API_BEGIN auto* dst = tensor->MutableData(); auto len = static_cast(tensor->Shape().Size()); if (s_len < len) { - return CreateONNXStatus(ORT_INVALID_ARGUMENT, "input array is too short"); + return OrtCreateStatus(ORT_INVALID_ARGUMENT, "input array is too short"); } for (size_t i = 0; i != len; ++i) { //allocate and copy @@ -165,8 +165,8 @@ ORT_API_STATUS_IMPL(OrtFillStringTensor, _In_ ONNXValue* value, _In_ const char* } template -ONNXStatus* CreateTensorImpl(const size_t* shape, size_t shape_len, OrtAllocatorInterface** allocator, - std::unique_ptr* out) { +OrtStatus* CreateTensorImpl(const size_t* shape, size_t shape_len, OrtAllocatorInterface** allocator, + std::unique_ptr* out) { size_t elem_count = 1; std::vector shapes(shape_len); for (size_t i = 0; i != shape_len; ++i) { @@ -176,11 +176,11 @@ ONNXStatus* CreateTensorImpl(const size_t* shape, size_t shape_len, OrtAllocator size_t size_to_allocate; if (!IAllocator::CalcMemSizeForArray(sizeof(T), elem_count, &size_to_allocate)) { - return CreateONNXStatus(ORT_FAIL, "not enough memory"); + return OrtCreateStatus(ORT_FAIL, "not enough memory"); } void* p_data = (*allocator)->Alloc(allocator, size_to_allocate); if (p_data == nullptr) - return CreateONNXStatus(ORT_FAIL, "size overflow"); + return OrtCreateStatus(ORT_FAIL, "size overflow"); *out = std::make_unique(DataTypeImpl::GetType(), onnxruntime::TensorShape(shapes), static_cast(p_data), @@ -194,8 +194,8 @@ ONNXStatus* CreateTensorImpl(const size_t* shape, size_t shape_len, OrtAllocator * this function will create a copy of the allocator info */ template -ONNXStatus* CreateTensorImpl(const size_t* shape, size_t shape_len, const OrtAllocatorInfo* info, - void* p_data, size_t p_data_len, std::unique_ptr* out) { +OrtStatus* CreateTensorImpl(const size_t* shape, size_t shape_len, const OrtAllocatorInfo* info, + void* p_data, size_t p_data_len, std::unique_ptr* out) { size_t elem_count = 1; std::vector shapes(shape_len); for (size_t i = 0; i != shape_len; ++i) { @@ -205,12 +205,12 @@ ONNXStatus* CreateTensorImpl(const size_t* shape, size_t shape_len, const OrtAll size_t size_to_allocate; if (!IAllocator::CalcMemSizeForArray(sizeof(T), elem_count, &size_to_allocate)) { - return CreateONNXStatus(ORT_INVALID_ARGUMENT, "size overflow"); + return OrtCreateStatus(ORT_INVALID_ARGUMENT, "size overflow"); } if (size_to_allocate > p_data_len) { std::ostringstream oss; oss << "not enough space: expected " << size_to_allocate << ", got " << p_data_len; - return CreateONNXStatus(ORT_INVALID_ARGUMENT, oss.str().c_str()); + return OrtCreateStatus(ORT_INVALID_ARGUMENT, oss.str().c_str()); } *out = std::make_unique(DataTypeImpl::GetType(), onnxruntime::TensorShape(shapes), @@ -223,9 +223,9 @@ ONNXStatus* CreateTensorImpl(const size_t* shape, size_t shape_len, const OrtAll /** * this function will create a copy of the allocator info */ -ORT_API_STATUS_IMPL(OrtCreateTensorWithDataAsONNXValue, _In_ const OrtAllocatorInfo* info, +ORT_API_STATUS_IMPL(OrtCreateTensorWithDataAsOrtValue, _In_ const OrtAllocatorInfo* info, _In_ void* p_data, size_t p_data_len, _In_ const size_t* shape, size_t shape_len, - OrtTensorElementDataType type, _Out_ ONNXValue** out) { + ONNXTensorElementDataType type, _Out_ OrtValue** out) { API_IMPL_BEGIN std::unique_ptr tensor; switch (type) { @@ -274,21 +274,21 @@ ORT_API_STATUS_IMPL(OrtCreateTensorWithDataAsONNXValue, _In_ const OrtAllocatorI std::ostringstream oss; oss << "type " << type << " is not supported in this function"; std::string errmsg = oss.str(); - return CreateONNXStatus(ORT_NOT_IMPLEMENTED, errmsg.c_str()); + return OrtCreateStatus(ORT_NOT_IMPLEMENTED, errmsg.c_str()); } } std::unique_ptr value = std::make_unique(); value->Init(tensor.release(), DataTypeImpl::GetType(), DataTypeImpl::GetType()->GetDeleteFunc()); - *out = reinterpret_cast(value.release()); + *out = reinterpret_cast(value.release()); return nullptr; API_IMPL_END } -ORT_API_STATUS_IMPL(OrtCreateTensorAsONNXValue, _Inout_ OrtAllocator* allocator, - _In_ const size_t* shape, size_t shape_len, OrtTensorElementDataType type, - _Out_ ONNXValue** out) { +ORT_API_STATUS_IMPL(OrtCreateTensorAsOrtValue, _Inout_ OrtAllocator* allocator, + _In_ const size_t* shape, size_t shape_len, ONNXTensorElementDataType type, + _Out_ OrtValue** out) { API_IMPL_BEGIN std::unique_ptr tensor; switch (type) { @@ -338,34 +338,34 @@ ORT_API_STATUS_IMPL(OrtCreateTensorAsONNXValue, _Inout_ OrtAllocator* allocator, std::ostringstream oss; oss << "type " << type << " is not supported in this function"; std::string errmsg = oss.str(); - return CreateONNXStatus(ORT_NOT_IMPLEMENTED, errmsg.c_str()); + return OrtCreateStatus(ORT_NOT_IMPLEMENTED, errmsg.c_str()); } } std::unique_ptr value = std::make_unique(); value->Init(tensor.release(), DataTypeImpl::GetType(), DataTypeImpl::GetType()->GetDeleteFunc()); - *out = reinterpret_cast(value.release()); + *out = reinterpret_cast(value.release()); return nullptr; API_IMPL_END } template -static ONNXStatus* CreateInferenceSessionImpl(_In_ OrtEnv* env, _In_ T model_path, - _In_ const OrtSessionOptions* options, - _Out_ ONNXSession** out) { +static OrtStatus* CreateInferenceSessionImpl(_In_ OrtEnv* env, _In_ T model_path, + _In_ const OrtSessionOptions* options, + _Out_ OrtSession** out) { API_IMPL_BEGIN auto sess = std::make_unique<::onnxruntime::InferenceSession>(options == nullptr ? onnxruntime::SessionOptions() : options->value, env->loggingManager); Status status; if (options != nullptr && !options->custom_op_paths.empty()) { status = sess->LoadCustomOps(options->custom_op_paths); if (!status.IsOK()) - return ToONNXStatus(status); + return ToOrtStatus(status); } if (options != nullptr) for (OrtProviderFactoryInterface** p : options->provider_factories) { OrtProvider* provider; - ONNXStatus* error_code = (*p)->CreateProvider(p, &provider); + OrtStatus* error_code = (*p)->CreateProvider(p, &provider); if (error_code) return error_code; sess->RegisterExecutionProvider(std::unique_ptr( @@ -373,35 +373,35 @@ static ONNXStatus* CreateInferenceSessionImpl(_In_ OrtEnv* env, _In_ T model_pat } status = sess->Load(model_path); if (!status.IsOK()) - return ToONNXStatus(status); + return ToOrtStatus(status); status = sess->Initialize(); if (!status.IsOK()) - return ToONNXStatus(status); - *out = reinterpret_cast(sess.release()); + return ToOrtStatus(status); + *out = reinterpret_cast(sess.release()); return nullptr; API_IMPL_END } #ifdef _WIN32 ORT_API_STATUS_IMPL(OrtCreateInferenceSession, _In_ OrtEnv* env, _In_ const wchar_t* model_path, - _In_ const OrtSessionOptions* options, _Out_ ONNXSession** out) { + _In_ const OrtSessionOptions* options, _Out_ OrtSession** out) { API_IMPL_BEGIN return CreateInferenceSessionImpl(env, model_path, options, out); API_IMPL_END } #else ORT_API_STATUS_IMPL(OrtCreateInferenceSession, _In_ OrtEnv* env, _In_ const char* model_path, - _In_ const OrtSessionOptions* options, _Out_ ONNXSession** out) { + _In_ const OrtSessionOptions* options, _Out_ OrtSession** out) { API_IMPL_BEGIN return CreateInferenceSessionImpl(env, model_path, options, out); API_IMPL_END } #endif -ORT_API_STATUS_IMPL(OrtRunInference, _In_ ONNXSession* sess, +ORT_API_STATUS_IMPL(OrtRunInference, _In_ OrtSession* sess, _In_ OrtRunOptions* run_options, - _In_ const char* const* input_names, _In_ const ONNXValue* const* input, size_t input_len, - _In_ const char* const* output_names1, size_t output_names_len, _Out_ ONNXValue** output) { + _In_ const char* const* input_names, _In_ const OrtValue* const* input, size_t input_len, + _In_ const char* const* output_names1, size_t output_names_len, _Out_ OrtValue** output) { API_IMPL_BEGIN auto session = reinterpret_cast<::onnxruntime::InferenceSession*>(sess); ::onnxruntime::NameMLValMap in; @@ -410,7 +410,7 @@ ORT_API_STATUS_IMPL(OrtRunInference, _In_ ONNXSession* sess, auto kvp = in.insert(std::make_pair(std::string(input_names[i]), *reinterpret_cast(input[i]))); if (!kvp.second) { - return CreateONNXStatus(ORT_INVALID_ARGUMENT, "duplicated input name"); + return OrtCreateStatus(ORT_INVALID_ARGUMENT, "duplicated input name"); } ::onnxruntime::MLValue& value = kvp.first->second; if (value.Fence()) @@ -420,7 +420,7 @@ ORT_API_STATUS_IMPL(OrtRunInference, _In_ ONNXSession* sess, std::vector output_names(output_names_len); for (size_t i = 0; i != output_names_len; ++i) { if (output_names1[i] == nullptr || output_names1[i][0] == '\0') { - return CreateONNXStatus(ORT_INVALID_ARGUMENT, "output name cannot be empty"); + return OrtCreateStatus(ORT_INVALID_ARGUMENT, "output name cannot be empty"); } output_names[i] = output_names1[i]; } @@ -443,20 +443,20 @@ ORT_API_STATUS_IMPL(OrtRunInference, _In_ ONNXSession* sess, } if (!status.IsOK()) - return ToONNXStatus(status); + return ToOrtStatus(status); for (size_t i = 0; i != output_names_len; ++i) { ::onnxruntime::MLValue& value = fetches[i]; if (value.Fence()) value.Fence()->BeforeUsingAsInput(onnxruntime::kCpuExecutionProvider, queue_id); if (output[i] == nullptr) { - output[i] = reinterpret_cast(new MLValue(value)); + output[i] = reinterpret_cast(new MLValue(value)); } } return nullptr; API_IMPL_END } -ORT_API_STATUS_IMPL(OrtGetTensorMutableData, _In_ ONNXValue* value, _Out_ void** output) { +ORT_API_STATUS_IMPL(OrtGetTensorMutableData, _In_ OrtValue* value, _Out_ void** output) { TENSOR_READWRITE_API_BEGIN //TODO: test if it's a string tensor *output = tensor->MutableDataRaw(); @@ -464,13 +464,13 @@ ORT_API_STATUS_IMPL(OrtGetTensorMutableData, _In_ ONNXValue* value, _Out_ void** API_IMPL_END } -ORT_API_STATUS_IMPL(OrtGetStringTensorContent, _In_ const ONNXValue* value, +ORT_API_STATUS_IMPL(OrtGetStringTensorContent, _In_ const OrtValue* value, _Out_ void* s, size_t s_len, _Out_ size_t* offsets, size_t offsets_len) { TENSOR_READ_API_BEGIN const auto* input = tensor.Data(); auto len = static_cast(tensor.Shape().Size()); if (offsets_len < len) { - return CreateONNXStatus(ORT_FAIL, "space is not enough"); + return OrtCreateStatus(ORT_FAIL, "space is not enough"); } { size_t ret = 0; @@ -478,7 +478,7 @@ ORT_API_STATUS_IMPL(OrtGetStringTensorContent, _In_ const ONNXValue* value, ret += input[i].size(); } if (s_len < ret) { - return CreateONNXStatus(ORT_FAIL, "space is not enough"); + return OrtCreateStatus(ORT_FAIL, "space is not enough"); } } size_t f = 0; @@ -493,75 +493,75 @@ ORT_API_STATUS_IMPL(OrtGetStringTensorContent, _In_ const ONNXValue* value, API_IMPL_END } -ORT_API_STATUS_IMPL(OrtTensorProtoToONNXValue, _Inout_ OrtAllocator* allocator, - const void* input, int input_len, _Out_ ONNXValue** out) { +ORT_API_STATUS_IMPL(OrtTensorProtoToOrtValue, _Inout_ OrtAllocator* allocator, + const void* input, int input_len, _Out_ OrtValue** out) { API_IMPL_BEGIN std::shared_ptr allocator_ = std::make_shared(allocator); ::ONNX_NAMESPACE::TensorProto proto; if (!proto.ParseFromArray(input, input_len)) { - return CreateONNXStatus(ORT_FAIL, "parse input tensor proto failed"); + return OrtCreateStatus(ORT_FAIL, "parse input tensor proto failed"); } std::unique_ptr value = std::make_unique(); Status st = onnxruntime::utils::TensorProtoToMLValue(proto, allocator_, nullptr, 0, *value); if (!st.IsOK()) - return ToONNXStatus(st); - *out = reinterpret_cast(value.release()); + return ToOrtStatus(st); + *out = reinterpret_cast(value.release()); return nullptr; API_IMPL_END } -#define DEFINE_RELEASE_ONNX_RUNTIME_OBJECT_FUNCTION(INPUT_TYPE, REAL_TYPE) \ - ORT_API(void, Release##INPUT_TYPE, INPUT_TYPE* value) { \ - delete reinterpret_cast(value); \ +#define DEFINE_RELEASE_ORT_OBJECT_FUNCTION(INPUT_TYPE, REAL_TYPE) \ + ORT_API(void, OrtRelease##INPUT_TYPE, Ort##INPUT_TYPE* value) { \ + delete reinterpret_cast(value); \ } -#define DEFINE_RELEASE_ONNX_RUNTIME_OBJECT_FUNCTION_FOR_ARRAY(INPUT_TYPE, REAL_TYPE) \ - ORT_API(void, Release##INPUT_TYPE, INPUT_TYPE* value) { \ - delete[] reinterpret_cast(value); \ +#define DEFINE_RELEASE_ORT_OBJECT_FUNCTION_FOR_ARRAY(INPUT_TYPE, REAL_TYPE) \ + ORT_API(void, OrtRelease##INPUT_TYPE, Ort##INPUT_TYPE* value) { \ + delete[] reinterpret_cast(value); \ } -ORT_API_STATUS_IMPL(OrtInferenceSessionGetInputCount, _In_ const ONNXSession* sess, _Out_ size_t* out) { +ORT_API_STATUS_IMPL(OrtInferenceSessionGetInputCount, _In_ const OrtSession* sess, _Out_ size_t* out) { API_IMPL_BEGIN auto session = reinterpret_cast(sess); std::pair p = session->GetModelInputs(); if (!p.first.IsOK()) - return ToONNXStatus(p.first); + return ToOrtStatus(p.first); *out = p.second->size(); return nullptr; API_IMPL_END } -ORT_API_STATUS_IMPL(OrtInferenceSessionGetOutputCount, _In_ const ONNXSession* sess, _Out_ size_t* out) { +ORT_API_STATUS_IMPL(OrtInferenceSessionGetOutputCount, _In_ const OrtSession* sess, _Out_ size_t* out) { API_IMPL_BEGIN auto session = reinterpret_cast(sess); std::pair p = session->GetModelOutputs(); if (!p.first.IsOK()) - return ToONNXStatus(p.first); + return ToOrtStatus(p.first); *out = p.second->size(); return nullptr; API_IMPL_END } -ORT_API_STATUS_IMPL(OrtInferenceSessionGetInputTypeInfo, _In_ const ONNXSession* sess, size_t index, _Out_ struct OrtTypeInfo** out) { +ORT_API_STATUS_IMPL(OrtInferenceSessionGetInputTypeInfo, _In_ const OrtSession* sess, size_t index, _Out_ struct OrtTypeInfo** out) { API_IMPL_BEGIN auto session = reinterpret_cast(sess); std::pair p = session->GetModelInputs(); if (!p.first.IsOK()) - return ToONNXStatus(p.first); + return ToOrtStatus(p.first); if (p.second->size() <= index) - return CreateONNXStatus(ORT_FAIL, "out of index"); + return OrtCreateStatus(ORT_FAIL, "out of index"); const ONNX_NAMESPACE::TypeProto* type_proto = (*p.second)[index]->TypeAsProto(); return OrtTypeInfo::FromDataTypeImpl(type_proto, out); API_IMPL_END } -ORT_API_STATUS_IMPL(OrtInferenceSessionGetOutputTypeInfo, _In_ const ONNXSession* sess, size_t index, _Out_ struct OrtTypeInfo** out) { +ORT_API_STATUS_IMPL(OrtInferenceSessionGetOutputTypeInfo, _In_ const OrtSession* sess, size_t index, _Out_ struct OrtTypeInfo** out) { API_IMPL_BEGIN auto session = reinterpret_cast(sess); std::pair p = session->GetModelOutputs(); if (!p.first.IsOK()) - return ToONNXStatus(p.first); + return ToOrtStatus(p.first); if (p.second->size() <= index) - return CreateONNXStatus(ORT_FAIL, "out of index"); + return OrtCreateStatus(ORT_FAIL, "out of index"); const ONNX_NAMESPACE::TypeProto* type_proto = (*p.second)[index]->TypeAsProto(); return OrtTypeInfo::FromDataTypeImpl(type_proto, out); API_IMPL_END @@ -574,23 +574,23 @@ static char* StrDup(const std::string& str, OrtAllocator* allocator) { return output_string; } -static ONNXStatus* GetInputOutputNameImpl(_In_ const ONNXSession* sess, size_t index, - _Inout_ OrtAllocator* allocator, bool is_input, - _Out_ char** output) { +static OrtStatus* GetInputOutputNameImpl(_In_ const OrtSession* sess, size_t index, + _Inout_ OrtAllocator* allocator, bool is_input, + _Out_ char** output) { auto session = reinterpret_cast(sess); std::pair p = is_input ? session->GetModelInputs() : session->GetModelOutputs(); if (!p.first.IsOK()) - return ToONNXStatus(p.first); + return ToOrtStatus(p.first); if (p.second == nullptr) - return CreateONNXStatus(ORT_FAIL, "internal error"); + return OrtCreateStatus(ORT_FAIL, "internal error"); const InputDefList& defs = *p.second; if (index >= defs.size()) - return CreateONNXStatus(ORT_FAIL, "index out of range"); + return OrtCreateStatus(ORT_FAIL, "index out of range"); *output = StrDup(defs[index]->Name(), allocator); return nullptr; } -ORT_API(int, OrtIsTensor, _In_ const ONNXValue* value) { +ORT_API(int, OrtIsTensor, _In_ const OrtValue* value) { auto v = reinterpret_cast(value); return v->IsTensor() ? 1 : 0; } @@ -618,24 +618,24 @@ ORT_API(const struct OrtAllocatorInfo*, OrtAllocatorGetInfo, _In_ const OrtAlloc } } -ORT_API_STATUS_IMPL(OrtInferenceSessionGetInputName, _In_ const ONNXSession* sess, size_t index, +ORT_API_STATUS_IMPL(OrtInferenceSessionGetInputName, _In_ const OrtSession* sess, size_t index, _Inout_ OrtAllocator* allocator, _Out_ char** output) { API_IMPL_BEGIN return GetInputOutputNameImpl(sess, index, allocator, true, output); API_IMPL_END } -ORT_API_STATUS_IMPL(OrtInferenceSessionGetOutputName, _In_ const ONNXSession* sess, size_t index, +ORT_API_STATUS_IMPL(OrtInferenceSessionGetOutputName, _In_ const OrtSession* sess, size_t index, _Inout_ OrtAllocator* allocator, _Out_ char** output) { API_IMPL_BEGIN return GetInputOutputNameImpl(sess, index, allocator, false, output); API_IMPL_END } -DEFINE_RELEASE_ONNX_RUNTIME_OBJECT_FUNCTION(ONNXValue, MLValue) -DEFINE_RELEASE_ONNX_RUNTIME_OBJECT_FUNCTION(ONNXSession, ::onnxruntime::InferenceSession) -DEFINE_RELEASE_ONNX_RUNTIME_OBJECT_FUNCTION_FOR_ARRAY(ONNXStatus, char) +DEFINE_RELEASE_ORT_OBJECT_FUNCTION(Value, MLValue) +DEFINE_RELEASE_ORT_OBJECT_FUNCTION(Session, ::onnxruntime::InferenceSession) +DEFINE_RELEASE_ORT_OBJECT_FUNCTION_FOR_ARRAY(Status, char) -ORT_API(void, ReleaseONNXEnv, OrtEnv* env) { +ORT_API(void, OrtReleaseEnv, OrtEnv* env) { OrtReleaseObject(env); } diff --git a/onnxruntime/test/onnx/TestCase.cc b/onnxruntime/test/onnx/TestCase.cc index 7f7eb686ea39b..b323fa50c2f56 100644 --- a/onnxruntime/test/onnx/TestCase.cc +++ b/onnxruntime/test/onnx/TestCase.cc @@ -180,7 +180,7 @@ static Status SortTensorFileNames(std::vector> } Status LoopDataFile(int test_data_pb_fd, OrtAllocator* env, - const std::vector value_info, std::unordered_map& name_data_map, std::ostringstream& oss) { + const std::vector value_info, std::unordered_map& name_data_map, std::ostringstream& oss) { google::protobuf::io::FileInputStream f(test_data_pb_fd); f.SetCloseOnDelete(true); google::protobuf::io::CodedInputStream coded_input(&f); @@ -188,7 +188,7 @@ Status LoopDataFile(int test_data_pb_fd, OrtAllocator* env, Status st; int item_id = 1; for (proto::TraditionalMLData data; google::protobuf::util::ParseDelimitedFromCodedStream(&data, &coded_input, &clean_eof); ++item_id, data.Clear()) { - std::unique_ptr gvalue(nullptr, ReleaseONNXValue); + std::unique_ptr gvalue(nullptr, OrtReleaseValue); MLValue value; bool is_tensor = false; switch (data.values_case()) { @@ -223,9 +223,9 @@ Status LoopDataFile(int test_data_pb_fd, OrtAllocator* env, st = RichTypeProtoToMLValue(data.map_int64_to_double().v(), value); break; case proto::TraditionalMLData::kTensor: { - ONNXValue* temp_value; + OrtValue* temp_value; std::string s = data.tensor().SerializeAsString(); - ORT_THROW_ON_ERROR(OrtTensorProtoToONNXValue(env, s.data(), (int)s.size(), &temp_value)); + ORT_THROW_ON_ERROR(OrtTensorProtoToOrtValue(env, s.data(), (int)s.size(), &temp_value)); gvalue.reset(temp_value); is_tensor = true; } break; @@ -240,7 +240,7 @@ Status LoopDataFile(int test_data_pb_fd, OrtAllocator* env, if (value_name.empty()) value_name = value_info[name_data_map.size()].name(); - auto pv = name_data_map.insert(std::make_pair(value_name, is_tensor ? gvalue.release() : (ONNXValue*)new MLValue(value))); + auto pv = name_data_map.insert(std::make_pair(value_name, is_tensor ? gvalue.release() : (OrtValue*)new MLValue(value))); if (!pv.second) { st = Status(ONNXRUNTIME, FAIL, "duplicated test data name"); break; @@ -292,8 +292,8 @@ class OnnxTestCase : public ITestCase { } //If we cannot get input name from input_pbs, we'll use names like "data_0","data_1",... It's dirty hack // for https://github.com/onnx/onnx/issues/679 - ::onnxruntime::common::Status ConvertTestData(ONNXSession* session, const std::vector& test_data_pbs, - bool is_input, std::unordered_map& out); + ::onnxruntime::common::Status ConvertTestData(OrtSession* session, const std::vector& test_data_pbs, + bool is_input, std::unordered_map& out); std::string node_name_; std::once_flag model_parsed_; std::once_flag config_parsed_; @@ -333,7 +333,7 @@ class OnnxTestCase : public ITestCase { const std::string& GetTestCaseName() const override { return test_case_name_; } - ::onnxruntime::common::Status LoadTestData(ONNXSession* session, size_t id, std::unordered_map&, bool is_input) override; + ::onnxruntime::common::Status LoadTestData(OrtSession* session, size_t id, std::unordered_map&, bool is_input) override; }; Status OnnxTestCase::loadModelFile(const PATH_CHAR_TYPE* model_url, ONNX_NAMESPACE::ModelProto** model_pb) { @@ -463,7 +463,7 @@ static Status LoadTensors(const std::vector& pb_files, return Status::OK(); } -Status OnnxTestCase::LoadTestData(ONNXSession* session, size_t id, std::unordered_map& name_data_map, bool is_input) { +Status OnnxTestCase::LoadTestData(OrtSession* session, size_t id, std::unordered_map& name_data_map, bool is_input) { if (id >= test_data_dirs_.size()) return Status(ONNXRUNTIME, INVALID_ARGUMENT, "out of bound"); @@ -516,8 +516,8 @@ Status OnnxTestCase::LoadTestData(ONNXSession* session, size_t id, std::unordere return Status::OK(); } -Status OnnxTestCase::ConvertTestData(ONNXSession* session, const std::vector& test_data_pbs, - bool is_input, std::unordered_map& out) { +Status OnnxTestCase::ConvertTestData(OrtSession* session, const std::vector& test_data_pbs, + bool is_input, std::unordered_map& out) { bool has_valid_names = true; std::vector var_names(test_data_pbs.size()); for (size_t input_index = 0; input_index != test_data_pbs.size(); ++input_index) { @@ -553,8 +553,8 @@ Status OnnxTestCase::ConvertTestData(ONNXSession* session, const std::vector& name_data_map, bool is_input) = 0; + virtual ::onnxruntime::common::Status LoadTestData(OrtSession* session, size_t id, std::unordered_map& name_data_map, bool is_input) = 0; virtual const PATH_CHAR_TYPE* GetModelUrl() const = 0; virtual const std::string& GetTestCaseName() const = 0; //a string to help identify the dataset diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index 150c8c61bdf53..3da5bfefd5cb8 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -162,10 +162,10 @@ int real_main(int argc, char* argv[]) { std::unique_ptr env; { OrtEnv* t; - ONNXStatus* ost = OrtInitialize(logging_level, "Default", &t); + OrtStatus* ost = OrtInitialize(logging_level, "Default", &t); if (ost != nullptr) { fprintf(stderr, "Error creating environment: %s \n", OrtGetErrorMessage(ost)); - ReleaseONNXStatus(ost); + OrtReleaseStatus(ost); return -1; } env.reset(t); @@ -176,10 +176,10 @@ int real_main(int argc, char* argv[]) { std::unique_ptr default_allocator; { OrtAllocator* p; - ONNXStatus* ost = OrtCreateDefaultAllocator(&p); + OrtStatus* ost = OrtCreateDefaultAllocator(&p); if (ost != nullptr) { fprintf(stderr, "Error creating environment: %s \n", OrtGetErrorMessage(ost)); - ReleaseONNXStatus(ost); + OrtReleaseStatus(ost); return -1; } default_allocator.reset(p); diff --git a/onnxruntime/test/onnx/runner.cc b/onnxruntime/test/onnx/runner.cc index 7fac9800f13c7..ad3bbfd032bbf 100644 --- a/onnxruntime/test/onnx/runner.cc +++ b/onnxruntime/test/onnx/runner.cc @@ -88,7 +88,7 @@ void PTestRunner::OnTaskFinished(size_t, EXECUTE_RESULT, ORT_CALLBACK_INSTANCE p } } -PTestRunner::PTestRunner(ONNXSession* session1, +PTestRunner::PTestRunner(OrtSession* session1, ITestCase* c, PThreadPool tpool, TestCaseCallBack on_finished1) : DataRunner(session1, c->GetTestCaseName(), c, on_finished1), next_test_to_run(0), finished(0), tpool_(tpool) { } @@ -282,12 +282,12 @@ std::vector LoadTests(const std::vectorGetTestCaseName(), c, on_finished1), repeat_count_(repeat_count) { } -DataRunner::DataRunner(ONNXSession* session1, const std::string& test_case_name1, ITestCase* c, TestCaseCallBack on_finished1) : test_case_name_(test_case_name1), c_(c), session(session1), on_finished(on_finished1), default_allocator(MockedOrtAllocator::Create()) { +DataRunner::DataRunner(OrtSession* session1, const std::string& test_case_name1, ITestCase* c, TestCaseCallBack on_finished1) : test_case_name_(test_case_name1), c_(c), session(session1), on_finished(on_finished1), default_allocator(MockedOrtAllocator::Create()) { std::string s; c->GetNodeName(&s); result = std::make_shared(c->GetDataCount(), EXECUTE_RESULT::UNKNOWN_ERROR, s); @@ -308,12 +308,12 @@ void DataRunner::RunTask(size_t task_id, ORT_CALLBACK_INSTANCE pci, bool store_r OnTaskFinished(task_id, res, pci); } -std::pair CompareGenericValue(const ONNXValue* o, const ONNXValue* expected_mlvalue, double per_sample_tolerance, double relative_per_sample_tolerance, +std::pair CompareGenericValue(const OrtValue* o, const OrtValue* expected_mlvalue, double per_sample_tolerance, double relative_per_sample_tolerance, bool post_processing) { return onnxruntime::CompareMLValue(*(MLValue*)o, *(MLValue*)expected_mlvalue, per_sample_tolerance, relative_per_sample_tolerance, post_processing); } EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) { - std::unordered_map feeds; + std::unordered_map feeds; common::Status status = c_->LoadTestData(session, task_id, feeds, true); if (!status.IsOK()) { LOGF_DEFAULT(ERROR, "%s", status.ErrorMessage().c_str()); @@ -335,14 +335,14 @@ EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) { TIME_SPEC start_time, end_time; GetMonotonicTimeCounter(&start_time); std::vector input_names(feeds.size()); - std::vector input_values(feeds.size()); + std::vector input_values(feeds.size()); size_t input_index = 0; for (auto& kvp : feeds) { input_names[input_index] = kvp.first.c_str(); input_values[input_index] = kvp.second; ++input_index; } - std::vector output_values(output_count); + std::vector output_values(output_count); { std::vector output_names_raw_ptr(output_count); for (size_t i = 0; i != output_count; ++i) { @@ -351,9 +351,9 @@ EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) { auto onnx_status = OrtRunInference(session, nullptr, input_names.data(), input_values.data(), input_index, output_names_raw_ptr.data(), output_count, output_values.data()); if (onnx_status != nullptr) { std::string onnx_runtime_error_message = OrtGetErrorMessage(onnx_status); - ReleaseONNXStatus(onnx_status); + OrtReleaseStatus(onnx_status); for (auto& kvp : feeds) { - ReleaseONNXValue(kvp.second); + OrtReleaseValue(kvp.second); } throw std::runtime_error(onnx_runtime_error_message); } @@ -361,7 +361,7 @@ EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) { GetMonotonicTimeCounter(&end_time); AccumulateTimeSpec(&spent_time_, &start_time, &end_time); for (auto& kvp : feeds) { - ReleaseONNXValue(kvp.second); + OrtReleaseValue(kvp.second); } if (!status.IsOK()) { LOGF_DEFAULT(ERROR, "%s:%s\n", test_case_name_.c_str(), status.ErrorMessage().c_str()); @@ -385,13 +385,13 @@ EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) { } //TODO: if there are no output value files, just skip the validation - std::unordered_map expected_output_values; + std::unordered_map expected_output_values; status = c_->LoadTestData(session, task_id, expected_output_values, false); if (!status.IsOK()) { LOGF_DEFAULT(ERROR, "%s", status.ErrorMessage().c_str()); return StatusCodeToExecuteResult(status.Code()); } - std::unordered_map name_fetch_output_map; + std::unordered_map name_fetch_output_map; std::unordered_map name_output_value_info_proto; int i = 0; for (auto& output_name : output_names) { @@ -404,7 +404,7 @@ EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) { EXECUTE_RESULT res = EXECUTE_RESULT::SUCCESS; for (auto& output : expected_output_values) { - ONNXValue* expected_output_value = output.second; + OrtValue* expected_output_value = output.second; const std::string& output_name = output.first; auto iter = name_fetch_output_map.find(output_name); if (iter == name_fetch_output_map.end()) { @@ -412,7 +412,7 @@ EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) { LOGF_DEFAULT(ERROR, "cannot find %s in the outputs", output_name.c_str()); break; } - ONNXValue* actual_output_value = iter->second; + OrtValue* actual_output_value = iter->second; std::pair ret = CompareGenericValue(actual_output_value, expected_output_value, per_sample_tolerance, relative_per_sample_tolerance, post_procesing); COMPARE_RESULT compare_result = ret.first; if (compare_result == COMPARE_RESULT::SUCCESS) { @@ -461,10 +461,10 @@ EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) { } } for (auto& kvp : expected_output_values) { - ReleaseONNXValue(kvp.second); + OrtReleaseValue(kvp.second); } - for (ONNXValue* p : output_values) { - ReleaseONNXValue(p); + for (OrtValue* p : output_values) { + OrtReleaseValue(p); } return res; } @@ -492,8 +492,8 @@ void RunSingleTestCase(ITestCase* info, const onnxruntime::SessionOptionsWrapper } auto sf2 = sf.clone(); sf2.SetSessionLogId(info->GetTestCaseName().c_str()); - std::unique_ptr session_object( - sf2.OrtCreateInferenceSession(info->GetModelUrl()), ReleaseONNXSession); + std::unique_ptr session_object( + sf2.OrtCreateInferenceSession(info->GetModelUrl()), OrtReleaseSession); LOGF_DEFAULT(INFO, "testing %s\n", info->GetTestCaseName().c_str()); //temp hack. Because we have no resource control. We may not have enough memory to run this test in parallel if (info->GetTestCaseName() == "coreml_FNS-Candy_ImageNet") diff --git a/onnxruntime/test/onnx/runner.h b/onnxruntime/test/onnx/runner.h index 9691e52e3844b..f3dfd852cdf42 100644 --- a/onnxruntime/test/onnx/runner.h +++ b/onnxruntime/test/onnx/runner.h @@ -41,18 +41,18 @@ class DataRunner { ::onnxruntime::TIME_SPEC spent_time_; private: - ONNXSession* session; + OrtSession* session; CALL_BACK on_finished; OrtAllocatorInterface** const default_allocator; EXECUTE_RESULT RunTaskImpl(size_t task_id); ORT_DISALLOW_COPY_AND_ASSIGNMENT(DataRunner); public: - DataRunner(ONNXSession* session1, const std::string& test_case_name1, ITestCase* c, TestCaseCallBack on_finished1); + DataRunner(OrtSession* session1, const std::string& test_case_name1, ITestCase* c, TestCaseCallBack on_finished1); virtual void OnTaskFinished(size_t task_id, EXECUTE_RESULT res, ORT_CALLBACK_INSTANCE pci) noexcept = 0; void RunTask(size_t task_id, ORT_CALLBACK_INSTANCE pci, bool store_result); virtual ~DataRunner() { - ReleaseONNXSession(session); + OrtReleaseSession(session); OrtReleaseObject(default_allocator); } @@ -98,7 +98,7 @@ class SeqTestRunner : public DataRunner { size_t repeat_count_; public: - SeqTestRunner(ONNXSession* session1, + SeqTestRunner(OrtSession* session1, ITestCase* c, size_t repeat_count, TestCaseCallBack on_finished1); @@ -115,7 +115,7 @@ class PTestRunner : public DataRunner { public: void Start(ORT_CALLBACK_INSTANCE pci, size_t concurrent_runs) override; - PTestRunner(ONNXSession* session1, + PTestRunner(OrtSession* session1, ITestCase* c, PThreadPool tpool, TestCaseCallBack on_finished1); diff --git a/onnxruntime/test/onnxruntime_exec/Runtime.h b/onnxruntime/test/onnxruntime_exec/Runtime.h index bbf1aed93479f..b74ee583f19e7 100644 --- a/onnxruntime/test/onnxruntime_exec/Runtime.h +++ b/onnxruntime/test/onnxruntime_exec/Runtime.h @@ -273,7 +273,7 @@ class WinMLRuntime { ctensor = &output.Get(); ONNX_NAMESPACE::ValueInfoProto expected_output_info = (*outputMeta)[i]->ToProto(); - std::pair ret = VerifyValueInfo(expected_output_info, (ONNXValue*)&output); + std::pair ret = VerifyValueInfo(expected_output_info, (OrtValue*)&output); COMPARE_RESULT compare_result = ret.first; compare_result = ret.first; if (compare_result != COMPARE_RESULT::SUCCESS) { diff --git a/onnxruntime/test/shared_lib/fns_candy_style_transfer.c b/onnxruntime/test/shared_lib/fns_candy_style_transfer.c index b26bb4df91857..7aba00362f0ef 100644 --- a/onnxruntime/test/shared_lib/fns_candy_style_transfer.c +++ b/onnxruntime/test/shared_lib/fns_candy_style_transfer.c @@ -8,11 +8,11 @@ #define ORT_ABORT_ON_ERROR(expr) \ do { \ - ONNXStatus* onnx_status = (expr); \ + OrtStatus* onnx_status = (expr); \ if (onnx_status != NULL) { \ const char* msg = OrtGetErrorMessage(onnx_status); \ fprintf(stderr, "%s\n", msg); \ - ReleaseONNXStatus(onnx_status); \ + OrtReleaseStatus(onnx_status); \ abort(); \ } \ } while (0); @@ -94,7 +94,7 @@ static int read_png_file(const char* input_file, size_t* height, size_t* width, /** * \param tensor should be a float tensor in [N,C,H,W] format */ -static int write_tensor_to_png_file(ONNXValue* tensor, const char* output_file) { +static int write_tensor_to_png_file(OrtValue* tensor, const char* output_file) { struct OrtTensorTypeAndShapeInfo* shape_info; ORT_ABORT_ON_ERROR(OrtGetTensorShapeAndType(tensor, &shape_info)); size_t dim_count = OrtGetNumOfDimensions(shape_info); @@ -132,7 +132,7 @@ static void usage() { printf("usage: \n"); } -int run_inference(ONNXSession* session, const char* input_file, const char* output_file) { +int run_inference(OrtSession* session, const char* input_file, const char* output_file) { size_t input_height; size_t input_width; float* model_input; @@ -151,28 +151,28 @@ int run_inference(ONNXSession* session, const char* input_file, const char* outp const size_t input_shape_len = sizeof(input_shape) / sizeof(input_shape[0]); const size_t model_input_len = model_input_ele_count * sizeof(float); - ONNXValue* input_tensor = NULL; - ORT_ABORT_ON_ERROR(OrtCreateTensorWithDataAsONNXValue(allocator_info, model_input, model_input_len, input_shape, input_shape_len, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &input_tensor)); + OrtValue* input_tensor = NULL; + ORT_ABORT_ON_ERROR(OrtCreateTensorWithDataAsOrtValue(allocator_info, model_input, model_input_len, input_shape, input_shape_len, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &input_tensor)); assert(input_tensor != NULL); assert(OrtIsTensor(input_tensor) != 0); - ReleaseOrtAllocatorInfo(allocator_info); + OrtReleaseAllocatorInfo(allocator_info); const char* input_names[] = {"inputImage"}; const char* output_names[] = {"outputImage"}; - ONNXValue* output_tensor = NULL; - ORT_ABORT_ON_ERROR(OrtRunInference(session, NULL, input_names, (const ONNXValue* const*)&input_tensor, 1, output_names, 1, &output_tensor)); + OrtValue* output_tensor = NULL; + ORT_ABORT_ON_ERROR(OrtRunInference(session, NULL, input_names, (const OrtValue* const*)&input_tensor, 1, output_names, 1, &output_tensor)); assert(output_tensor != NULL); assert(OrtIsTensor(output_tensor) != 0); int ret = 0; if (write_tensor_to_png_file(output_tensor, output_file) != 0) { ret = -1; } - ReleaseONNXValue(output_tensor); - ReleaseONNXValue(input_tensor); + OrtReleaseValue(output_tensor); + OrtReleaseValue(input_tensor); free(model_input); return ret; } -void verify_input_output_count(ONNXSession* session) { +void verify_input_output_count(OrtSession* session) { size_t count; ORT_ABORT_ON_ERROR(OrtInferenceSessionGetInputCount(session, &count)); assert(count == 1); @@ -203,12 +203,12 @@ int main(int argc, char* argv[]) { #ifdef USE_CUDA enable_cuda(session_option); #endif - ONNXSession* session; + OrtSession* session; ORT_ABORT_ON_ERROR(OrtCreateInferenceSession(env, model_path, session_option, &session)); verify_input_output_count(session); int ret = run_inference(session, input_file, output_file); OrtReleaseObject(session_option); - ReleaseONNXSession(session); + OrtReleaseSession(session); OrtReleaseObject(env); if (ret != 0) { fprintf(stderr, "fail\n"); diff --git a/onnxruntime/test/shared_lib/test_allocator.cc b/onnxruntime/test/shared_lib/test_allocator.cc index ef4921d5996a9..ab97260e95c4a 100644 --- a/onnxruntime/test/shared_lib/test_allocator.cc +++ b/onnxruntime/test/shared_lib/test_allocator.cc @@ -13,8 +13,8 @@ TEST_F(CApiTest, allocation_info) { ORT_THROW_ON_ERROR(OrtCreateAllocatorInfo("Cpu", OrtArenaAllocator, 0, OrtMemTypeDefault, &info1)); ORT_THROW_ON_ERROR(OrtCreateCpuAllocatorInfo(OrtArenaAllocator, OrtMemTypeDefault, &info2)); ASSERT_EQ(0, OrtCompareAllocatorInfo(info1, info2)); - ReleaseOrtAllocatorInfo(info1); - ReleaseOrtAllocatorInfo(info2); + OrtReleaseAllocatorInfo(info1); + OrtReleaseAllocatorInfo(info2); } TEST_F(CApiTest, DefaultAllocator) { diff --git a/onnxruntime/test/shared_lib/test_inference.cc b/onnxruntime/test/shared_lib/test_inference.cc index b5d3c10fb9791..e1b2156bf9e9e 100644 --- a/onnxruntime/test/shared_lib/test_inference.cc +++ b/onnxruntime/test/shared_lib/test_inference.cc @@ -13,20 +13,20 @@ using namespace onnxruntime; -void RunSession(OrtAllocator* env, ONNXSession* session_object, +void RunSession(OrtAllocator* env, OrtSession* session_object, const std::vector& dims_x, const std::vector& values_x, const std::vector& dims_y, const std::vector& values_y) { - std::unique_ptr value_x(nullptr, ReleaseONNXValue); - std::vector inputs(1); - inputs[0] = OrtCreateTensorAsONNXValue(env, dims_x, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT); + std::unique_ptr value_x(nullptr, OrtReleaseValue); + std::vector inputs(1); + inputs[0] = OrtCreateTensorAsOrtValue(env, dims_x, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT); value_x.reset(inputs[0]); void* raw_data; ORT_THROW_ON_ERROR(OrtGetTensorMutableData(inputs[0], &raw_data)); memcpy(raw_data, values_x.data(), values_x.size() * sizeof(values_x[0])); std::vector input_names{"X"}; - ONNXValue* output_tensor = nullptr; + OrtValue* output_tensor = nullptr; const char* output_names[] = {"Y"}; ORT_THROW_ON_ERROR(OrtRunInference(session_object, NULL, input_names.data(), inputs.data(), inputs.size(), output_names, 1, &output_tensor)); ASSERT_NE(output_tensor, nullptr); @@ -50,7 +50,7 @@ void RunSession(OrtAllocator* env, ONNXSession* session_object, for (size_t i = 0; i != total_len; ++i) { ASSERT_EQ(values_y[i], f[i]); } - ReleaseONNXValue(output_tensor); + OrtReleaseValue(output_tensor); } template @@ -98,7 +98,7 @@ void TestInference(OrtEnv* env, T model_uri, if (custom_op) { sf.AddCustomOp("libonnxruntime_custom_op_shared_lib_test.so"); } - std::unique_ptr inference_session(sf.OrtCreateInferenceSession(model_uri), ReleaseONNXSession); + std::unique_ptr inference_session(sf.OrtCreateInferenceSession(model_uri), OrtReleaseSession); std::unique_ptr default_allocator(MockedOrtAllocator::Create()); // Now run RunSession(default_allocator.get(), inference_session.get(), dims_x, values_x, expected_dims_y, expected_values_y); @@ -147,10 +147,10 @@ TEST_F(CApiTest, DISABLED_custom_op) { #ifdef ORT_RUN_EXTERNAL_ONNX_TESTS TEST_F(CApiTest, create_session_without_session_option) { constexpr PATH_TYPE model_uri = TSTR("../models/opset8/test_squeezenet/model.onnx"); - ONNXSession* ret; + OrtSession* ret; ORT_THROW_ON_ERROR(::OrtCreateInferenceSession(env, model_uri, nullptr, &ret)); ASSERT_NE(nullptr, ret); - ReleaseONNXSession(ret); + OrtReleaseSession(ret); } #endif TEST_F(CApiTest, create_tensor) { @@ -158,8 +158,8 @@ TEST_F(CApiTest, create_tensor) { size_t expected_len = 2; std::unique_ptr default_allocator(MockedOrtAllocator::Create()); { - std::unique_ptr tensor( - OrtCreateTensorAsONNXValue(default_allocator.get(), {expected_len}, ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING), ReleaseONNXValue); + std::unique_ptr tensor( + OrtCreateTensorAsOrtValue(default_allocator.get(), {expected_len}, ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING), OrtReleaseValue); ORT_THROW_ON_ERROR(OrtFillStringTensor(tensor.get(), s, expected_len)); std::unique_ptr shape_info; { @@ -185,9 +185,9 @@ TEST_F(CApiTest, create_tensor_with_data) { OrtAllocatorInfo* info; ORT_THROW_ON_ERROR(OrtCreateAllocatorInfo("Cpu", OrtDeviceAllocator, 0, OrtMemTypeDefault, &info)); std::vector dims = {4}; - std::unique_ptr tensor( - OrtCreateTensorWithDataAsONNXValue(info, values, values_length * sizeof(float), dims, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT), ReleaseONNXValue); - ReleaseOrtAllocatorInfo(info); + std::unique_ptr tensor( + OrtCreateTensorWithDataAsOrtValue(info, values, values_length * sizeof(float), dims, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT), OrtReleaseValue); + OrtReleaseAllocatorInfo(info); void* new_pointer; ORT_THROW_ON_ERROR(OrtGetTensorMutableData(tensor.get(), &new_pointer)); ASSERT_EQ(new_pointer, values); diff --git a/onnxruntime/test/shared_lib/test_io_types.cc b/onnxruntime/test/shared_lib/test_io_types.cc index 60caf4f18c4c1..ceebc782c66aa 100644 --- a/onnxruntime/test/shared_lib/test_io_types.cc +++ b/onnxruntime/test/shared_lib/test_io_types.cc @@ -6,7 +6,7 @@ using namespace onnxruntime; -static void TestModelInfo(const ONNXSession* inference_session, bool is_input, const std::vector& dims) { +static void TestModelInfo(const OrtSession* inference_session, bool is_input, const std::vector& dims) { size_t input_count; if (is_input) { ORT_THROW_ON_ERROR(OrtInferenceSessionGetInputCount(inference_session, &input_count)); @@ -28,7 +28,7 @@ static void TestModelInfo(const ONNXSession* inference_session, bool is_input, c const OrtTensorTypeAndShapeInfo* p = OrtCastTypeInfoToTensorInfo(input_type_info.get()); ASSERT_NE(nullptr, p); - enum OrtTensorElementDataType ele_type = OrtGetTensorElementType(p); + enum ONNXTensorElementDataType ele_type = OrtGetTensorElementType(p); ASSERT_EQ(ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, ele_type); ASSERT_EQ(dims.size(), OrtGetNumOfDimensions(p)); std::vector real_dims(dims.size()); @@ -39,7 +39,7 @@ static void TestModelInfo(const ONNXSession* inference_session, bool is_input, c TEST_F(CApiTest, input_output_type_info) { SessionOptionsWrapper sf(env); constexpr PATH_TYPE model_uri = TSTR("../models/opset8/test_squeezenet/model.onnx"); - std::unique_ptr inference_session(sf.OrtCreateInferenceSession(model_uri), ReleaseONNXSession); + std::unique_ptr inference_session(sf.OrtCreateInferenceSession(model_uri), OrtReleaseSession); TestModelInfo(inference_session.get(), true, {1, 3, 224, 224}); TestModelInfo(inference_session.get(), false, {1, 1000, 1, 1}); } diff --git a/onnxruntime/test/util/compare_mlvalue.cc b/onnxruntime/test/util/compare_mlvalue.cc index 503eef554a155..1b518b7413be7 100644 --- a/onnxruntime/test/util/compare_mlvalue.cc +++ b/onnxruntime/test/util/compare_mlvalue.cc @@ -26,7 +26,7 @@ using namespace onnxruntime; namespace { -OrtTensorElementDataType CApiElementTypeFromProto(ONNX_NAMESPACE::TensorProto_DataType type) { +ONNXTensorElementDataType CApiElementTypeFromProto(ONNX_NAMESPACE::TensorProto_DataType type) { switch (type) { CASE_TYPE(FLOAT) CASE_TYPE(UINT8) @@ -321,7 +321,7 @@ std::pair CompareMLValue(const MLValue& o, const ML per_sample_tolerance, relative_per_sample_tolerance, post_processing); } -std::pair VerifyValueInfo(const ONNX_NAMESPACE::ValueInfoProto& v, const ONNXValue* o) { +std::pair VerifyValueInfo(const ONNX_NAMESPACE::ValueInfoProto& v, const OrtValue* o) { if (!v.has_type()) return std::make_pair(COMPARE_RESULT::SUCCESS, ""); if (v.type().has_tensor_type()) { if (OrtIsTensor(o) == 0) { @@ -339,8 +339,8 @@ std::pair VerifyValueInfo(const ONNX_NAMESPACE::Val ORT_THROW_ON_ERROR(OrtGetTensorShapeAndType(o, &t1)); info.reset(t1); } - OrtTensorElementDataType real_type = OrtGetTensorElementType(info.get()); - OrtTensorElementDataType expected_type = CApiElementTypeFromProto(t.elem_type()); + ONNXTensorElementDataType real_type = OrtGetTensorElementType(info.get()); + ONNXTensorElementDataType expected_type = CApiElementTypeFromProto(t.elem_type()); if (real_type != expected_type) { return std::make_pair(COMPARE_RESULT::TYPE_MISMATCH, ""); } diff --git a/onnxruntime/test/util/include/test/compare_mlvalue.h b/onnxruntime/test/util/include/test/compare_mlvalue.h index 6dbabba1355d1..32024287a76d4 100644 --- a/onnxruntime/test/util/include/test/compare_mlvalue.h +++ b/onnxruntime/test/util/include/test/compare_mlvalue.h @@ -23,5 +23,5 @@ std::pair CompareMLValue(const MLValue& real, const double relative_per_sample_tolerance, bool post_processing); //verify if the 'value' matches the 'expected' ValueInfoProto. 'value' is a model output -std::pair VerifyValueInfo(const ONNX_NAMESPACE::ValueInfoProto& expected, const ONNXValue* value); +std::pair VerifyValueInfo(const ONNX_NAMESPACE::ValueInfoProto& expected, const OrtValue* value); } // namespace onnxruntime diff --git a/onnxruntime/test/util/include/test_allocator.h b/onnxruntime/test/util/include/test_allocator.h index 1ef5929e1d8b2..660320c901647 100644 --- a/onnxruntime/test/util/include/test_allocator.h +++ b/onnxruntime/test/util/include/test_allocator.h @@ -47,7 +47,7 @@ MockedOrtAllocator() : ref_count_(1), memory_inuse(0) { } ~MockedOrtAllocator() { assert(ref_count_ == 0); - ReleaseOrtAllocatorInfo(cpuAllocatorInfo); + OrtReleaseAllocatorInfo(cpuAllocatorInfo); } public: From 0aa1b54aaa85294ee6315d05cd33804e034baba5 Mon Sep 17 00:00:00 2001 From: edgchen1 Date: Tue, 18 Dec 2018 13:23:32 -0800 Subject: [PATCH 15/56] Updated build.py to support relative CMake/CTest paths and did some minor cleanup. (#205) --- tools/ci_build/build.py | 65 +++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 48ee7a01604f4..870c5295e09d3 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -24,6 +24,11 @@ test_data_url = 'https://onnxruntimetestdata.blob.core.windows.net/models/20181210.zip' test_data_checksum = 'a966def7447f4ff04f5665bca235b3f3' +class BuildError(Exception): + """Error from running build steps.""" + def __init__(self, *messages): + super().__init__("\n".join(messages)) + def parse_arguments(): parser = argparse.ArgumentParser(description="ONNXRuntime CI build driver.", usage=''' @@ -84,7 +89,7 @@ def parse_arguments(): parser.add_argument("--ctest_path", default="ctest", help="Path to the CTest program.") parser.add_argument("--skip_submodule_sync", action='store_true', help="Don't do a 'git submodule update'. Makes the Update phase faster.") - parser.add_argument("--use_jemalloc", action='store_true', help="use jemalloc.") + parser.add_argument("--use_jemalloc", action='store_true', help="Use jemalloc.") parser.add_argument("--use_openblas", action='store_true', help="Build with OpenBLAS.") parser.add_argument("--use_mkldnn", action='store_true', help="Build with MKLDNN.") parser.add_argument("--use_mklml", action='store_true', help="Build with MKLML.") @@ -95,14 +100,21 @@ def parse_arguments(): parser.add_argument("--use_llvm", action="store_true", help="Build tvm with llvm") parser.add_argument("--enable_msinternal", action="store_true", help="Enable for Microsoft internal builds only.") parser.add_argument("--llvm_path", help="Path to llvm dir") - parser.add_argument("--azure_sas_key", help="azure storage sas key, starts with '?'") + parser.add_argument("--azure_sas_key", help="Azure storage sas key, starts with '?'") parser.add_argument("--use_brainslice", action="store_true", help="Build with brain slice") - parser.add_argument("--brain_slice_package_path", help="Path to brain slice pacakges") - parser.add_argument("--brain_slice_package_name", help="Name of brain slice pakcages") + parser.add_argument("--brain_slice_package_path", help="Path to brain slice packages") + parser.add_argument("--brain_slice_package_name", help="Name of brain slice packages") parser.add_argument("--brain_slice_client_package_name", help="Name of brainslice client package") parser.add_argument("--use_nuphar", action='store_true', help="Build with nuphar") return parser.parse_args() +def resolve_executable_path(command_or_path): + """Returns the absolute path of an executable.""" + executable_path = shutil.which(command_or_path) + if executable_path is None: + raise BuildError("Failed to resolve executable path for '{}'.".format(command_or_path)) + return os.path.realpath(executable_path) + def is_windows(): return sys.platform.startswith("win") @@ -151,11 +163,10 @@ def install_apt_package(package): if is_sudo(): run_subprocess(['apt-get', 'install', '-y', package]) else: - log.error(package + " APT package missing. Please re-run this script using sudo to install.") - sys.exit(-1) + raise BuildError(package + " APT package missing. Please re-run this script using sudo to install.") def install_ubuntu_deps(args): - 'Check if the necessary Ubuntu dependencies are installed. Not required on docker. Provider help output if missing.' + 'Check if the necessary Ubuntu dependencies are installed. Not required on docker. Provide help output if missing.' # check we need the packages first if not (args.enable_pybind or args.use_openblas): @@ -171,8 +182,7 @@ def install_ubuntu_deps(args): install_apt_package("libopenblas-dev") except Exception as e: - log.error("Error setting up required APT packages. {}".format(str(e))) - sys.exit(-1) + raise BuildError("Error setting up required APT packages. {}".format(str(e))) def install_python_deps(): dep_packages = ['setuptools', 'wheel', 'numpy'] @@ -355,17 +365,15 @@ def setup_cuda_vars(args): cudnn_home_valid = (cudnn_home != None and os.path.exists(cudnn_home)) if (not cuda_home_valid or not cudnn_home_valid): - log.error("cuda_home and cudnn_home paths must be specified and valid.") - log.error("cuda_home='{}' valid={}. cudnn_home='{}' valid={}" - .format(cuda_home, cuda_home_valid, cudnn_home, cudnn_home_valid)) - sys.exit(-1) + raise BuildError("cuda_home and cudnn_home paths must be specified and valid.", + "cuda_home='{}' valid={}. cudnn_home='{}' valid={}" + .format(cuda_home, cuda_home_valid, cudnn_home, cudnn_home_valid)) if (is_windows()): # Validate that the cudnn_home is pointing at the right level if (not os.path.exists(os.path.join(cudnn_home, "bin"))): - log.error("cudnn_home path should include the 'cuda' folder, and must contain the CUDNN 'bin' directory.") - log.error("cudnn_home='{}'".format(cudnn_home)) - sys.exit(-1) + raise BuildError("cudnn_home path should include the 'cuda' folder, and must contain the CUDNN 'bin' directory.", + "cudnn_home='{}'".format(cudnn_home)) os.environ["CUDA_PATH"] = cuda_home os.environ["CUDA_TOOLKIT_ROOT_DIR"] = cuda_home @@ -377,8 +385,7 @@ def setup_cuda_vars(args): # Add version specific CUDA_PATH_Vx_y value as the Visual Studio build files require that version_file = os.path.join(cuda_home, 'version.txt') if not os.path.exists(version_file): - log.error("No version file found in CUDA install directory. Looked for " + version_file) - sys.exit(-1) + raise BuildError("No version file found in CUDA install directory. Looked for " + version_file) cuda_major_version = "unknown" @@ -387,8 +394,7 @@ def setup_cuda_vars(args): first_line = f.readline() m = re.match("CUDA Version (\d+).(\d+)", first_line) if not m: - log.error("Couldn't read version from first line of " + version_file) - sys.exit(-1) + raise BuildError("Couldn't read version from first line of " + version_file) cuda_major_version = m.group(1) minor = m.group(2) @@ -403,11 +409,10 @@ def setup_cuda_vars(args): log.warning("See build.md in the root ONNXRuntime directory for instructions on installing the Visual C++ 2017 14.11 toolset if needed.") elif cuda_major_version == "9" and vc_ver[0] == "14" and int(vc_ver[1]) > 11: - log.error("Visual C++ Tools version not supported by CUDA v9. You must setup the environment to use the 14.11 toolset.") - log.info("Current version is {}. CUDA 9.2 requires version 14.11.*".format(vc_ver_str)) - log.info("If necessary manually install the 14.11 toolset using the Visual Studio 2017 updater.") - log.info("See 'Windows CUDA Build' in build.md in the root directory of this repository.") - sys.exit(-1) + raise BuildError("Visual C++ Tools version not supported by CUDA v9. You must setup the environment to use the 14.11 toolset.", + "Current version is {}. CUDA 9.2 requires version 14.11.*".format(vc_ver_str), + "If necessary manually install the 14.11 toolset using the Visual Studio 2017 updater.", + "See 'Windows CUDA Build' in build.md in the root directory of this repository.") return cuda_home, cudnn_home @@ -480,7 +485,6 @@ def build_python_wheel(source_dir, build_dir, configs, use_cuda): def main(): args = parse_arguments() - cmake_path = args.cmake_path cmake_extra_defines = args.cmake_extra_defines if args.cmake_extra_defines else [] # if there was no explicit argument saying what to do, default to update, build and test. @@ -496,7 +500,8 @@ def main(): configs = set(args.config) # setup paths and directories - ctest_path = args.ctest_path + cmake_path = resolve_executable_path(args.cmake_path) + ctest_path = resolve_executable_path(args.ctest_path) build_dir = args.build_dir script_dir = os.path.realpath(os.path.dirname(__file__)) source_dir = os.path.normpath(os.path.join(script_dir, "..", "..")) @@ -563,4 +568,8 @@ def main(): log.info("Build complete") if __name__ == "__main__": - sys.exit(main()) + try: + sys.exit(main()) + except BuildError as e: + log.error(str(e)) + sys.exit(1) From d4131a31d90125e3d6ff3f710e0a86248020d15b Mon Sep 17 00:00:00 2001 From: jignparm Date: Tue, 18 Dec 2018 21:35:23 +0000 Subject: [PATCH 16/56] Disable csharp pretrained tests temporarily (#207) --- csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs index 1edd5d7b6c081..8044665ab751a 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs @@ -205,7 +205,7 @@ private void TestMultiThreads() session.Dispose(); } - [Fact] + [Fact(Skip = "Disable temporarily")] private void TestPreTrainedModelsOpset7And8() { var opsets = new[] { "opset7", "opset8" }; From c0ec7d56c4a3ea5ec91ec7f1494535c9ec61cb76 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Tue, 18 Dec 2018 13:51:01 -0800 Subject: [PATCH 17/56] Clean up garbage files (#208) --- cmake/CMakeLists.txt | 2 -- cmake/external/cub.cmake | 30 ------------------------------ cmake/patches/cub/CMakeLists.txt | 6 ------ cmake/patches/protobuf.patch | 27 --------------------------- 4 files changed, 65 deletions(-) delete mode 100644 cmake/external/cub.cmake delete mode 100644 cmake/patches/cub/CMakeLists.txt delete mode 100644 cmake/patches/protobuf.patch diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 7769cee8f3f0a..f91c85b38ab4e 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -431,7 +431,6 @@ if (onnxruntime_USE_CUDA) endif () file(TO_CMAKE_PATH ${onnxruntime_CUDNN_HOME} onnxruntime_CUDNN_HOME) find_package(CUDA 9.0 REQUIRED) - include(cub) set(CUDA_LINK_LIBRARIES_KEYWORD PRIVATE) if (WIN32) link_directories(${onnxruntime_CUDNN_HOME}/lib/x64) @@ -442,7 +441,6 @@ if (onnxruntime_USE_CUDA) set(ONNXRUNTIME_CUDA_LIBRARIES cudnn_static cublas_static culibos) endif() list(APPEND onnxruntime_EXTERNAL_LIBRARIES ${ONNXRUNTIME_CUDA_LIBRARIES}) - list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES cub) set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -gencode=arch=compute_30,code=sm_30") # K series set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -gencode=arch=compute_50,code=sm_50") # M series diff --git a/cmake/external/cub.cmake b/cmake/external/cub.cmake deleted file mode 100644 index 080e11c7fd22e..0000000000000 --- a/cmake/external/cub.cmake +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -include (ExternalProject) - -set(cub_URL https://github.com/NVlabs/cub/archive/v1.8.0.zip) -set(cub_HASH SHA256=6bfa06ab52a650ae7ee6963143a0bbc667d6504822cbd9670369b598f18c58c3) -set(cub_BUILD ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub) -set(cub_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub) -set(cub_ARCHIVE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/cub_archive) - -ExternalProject_Add(cub - PREFIX cub - URL ${cub_URL} - URL_HASH ${cub_HASH} - DOWNLOAD_DIR "${DOWNLOAD_LOCATION}" - BUILD_IN_SOURCE 1 - PATCH_COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/cub/CMakeLists.txt ${cub_BUILD} - INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory ${cub_INCLUDE_DIR}/cub ${cub_ARCHIVE_DIR}/cub) diff --git a/cmake/patches/cub/CMakeLists.txt b/cmake/patches/cub/CMakeLists.txt deleted file mode 100644 index 8fbb114c345bd..0000000000000 --- a/cmake/patches/cub/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -cmake_minimum_required(VERSION 2.8.3) - -project(cub) diff --git a/cmake/patches/protobuf.patch b/cmake/patches/protobuf.patch deleted file mode 100644 index 111d5655ca0d2..0000000000000 --- a/cmake/patches/protobuf.patch +++ /dev/null @@ -1,27 +0,0 @@ -diff --git a/src/google/protobuf/compiler/cpp/cpp_file.cc b/src/google/protobuf/compiler/cpp/cpp_file.cc -index a066a6a7..636a864f 100644 ---- a/src/google/protobuf/compiler/cpp/cpp_file.cc -+++ b/src/google/protobuf/compiler/cpp/cpp_file.cc -@@ -972,6 +972,11 @@ void FileGenerator::GenerateTopHeaderGuard(io::Printer* printer, - "#ifndef PROTOBUF_$filename_identifier$__INCLUDED\n" - "#define PROTOBUF_$filename_identifier$__INCLUDED\n" - "\n" -+ "#ifdef _MSC_VER\n" -+ "#pragma warning(push)\n" -+ "#pragma warning(disable: 4800)\n" -+ "#endif // _MSC_VER\n" -+ "\n" - "#include \n", - "filename", file_->name(), "filename_identifier", filename_identifier); - printer->Print("\n"); -@@ -980,6 +985,10 @@ void FileGenerator::GenerateTopHeaderGuard(io::Printer* printer, - void FileGenerator::GenerateBottomHeaderGuard( - io::Printer* printer, const string& filename_identifier) { - printer->Print( -+ "#ifdef _MSC_VER\n" -+ "#pragma warning(pop)\n" -+ "#endif // _MSC_VER\n" -+ "\n" - "#endif // PROTOBUF_$filename_identifier$__INCLUDED\n", - "filename_identifier", filename_identifier); - } From 37b74c771aff134fe6af3715ab687f789b9c2c4e Mon Sep 17 00:00:00 2001 From: Ke Zhang Date: Tue, 18 Dec 2018 13:57:53 -0800 Subject: [PATCH 18/56] add gemmlowp as submodule. (#206) --- .gitmodules | 3 +++ cmake/external/gemmlowp | 1 + 2 files changed, 4 insertions(+) create mode 160000 cmake/external/gemmlowp diff --git a/.gitmodules b/.gitmodules index 004d8dfacb09b..7d34a0dee92d4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,6 @@ [submodule "cmake/external/gsl"] path = cmake/external/gsl url = https://github.com/Microsoft/GSL.git +[submodule "cmake/external/gemmlowp"] + path = cmake/external/gemmlowp + url = https://github.com/google/gemmlowp.git diff --git a/cmake/external/gemmlowp b/cmake/external/gemmlowp new file mode 160000 index 0000000000000..a3df028932a6b --- /dev/null +++ b/cmake/external/gemmlowp @@ -0,0 +1 @@ +Subproject commit a3df028932a6b00e6ef9c4b6f1c6109f696c915d From dc8b37f4c491872f75d4fe1963525f98fdb9974b Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Tue, 18 Dec 2018 14:50:28 -0800 Subject: [PATCH 19/56] update onnx (#209) * update onnx --- cmake/external/onnx | 2 +- onnxruntime/core/framework/onnxruntime_typeinfo.cc | 2 +- .../core/graph/contrib_ops/range_schema_defs.cc | 4 ++-- onnxruntime/core/graph/graph.cc | 8 ++++---- onnxruntime/core/graph/initializer.h | 6 +++--- onnxruntime/core/protobuf/onnx-ml.proto | 14 +++++++------- onnxruntime/test/util/compare_mlvalue.cc | 2 +- .../github/linux/docker/scripts/install_deps.sh | 4 ++-- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/cmake/external/onnx b/cmake/external/onnx index 0a4d5abdf4939..0c8d857bb1624 160000 --- a/cmake/external/onnx +++ b/cmake/external/onnx @@ -1 +1 @@ -Subproject commit 0a4d5abdf4939ab0842a5eadcc16a3bf0738f901 +Subproject commit 0c8d857bb162431912b255d5c0e773fb7c131a65 diff --git a/onnxruntime/core/framework/onnxruntime_typeinfo.cc b/onnxruntime/core/framework/onnxruntime_typeinfo.cc index 250d83e4dcac1..655bc840fceb0 100644 --- a/onnxruntime/core/framework/onnxruntime_typeinfo.cc +++ b/onnxruntime/core/framework/onnxruntime_typeinfo.cc @@ -52,7 +52,7 @@ OrtStatus* OrtTypeInfo::FromDataTypeImpl(const onnxruntime::DataTypeImpl* input, return OrtCreateStatus(ORT_NOT_IMPLEMENTED, "not implemented"); } -const DataTypeImpl* ElementTypeFromProto(ONNX_NAMESPACE::TensorProto_DataType type) { +const DataTypeImpl* ElementTypeFromProto(int type) { switch (type) { case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: return DataTypeImpl::GetType(); diff --git a/onnxruntime/core/graph/contrib_ops/range_schema_defs.cc b/onnxruntime/core/graph/contrib_ops/range_schema_defs.cc index fc31ee4f94371..e5524281b223e 100644 --- a/onnxruntime/core/graph/contrib_ops/range_schema_defs.cc +++ b/onnxruntime/core/graph/contrib_ops/range_schema_defs.cc @@ -81,7 +81,7 @@ static int64_t CalcRangeDim(const TensorProto* startShapeInitializer, static int64_t CalcResultDim(const TensorProto* startShapeInitializer, const TensorProto* limitShapeInitializer, const TensorProto* deltaShapeInitializer, - TensorProto_DataType dtype) { + int dtype) { int64_t dim = -1LL; if (dtype == TensorProto::FLOAT) { dim = CalcRangeDim(startShapeInitializer, limitShapeInitializer, deltaShapeInitializer); @@ -146,7 +146,7 @@ OpSchema& RegisterRangeOpSchema(OpSchema&& op_schema){ const TensorProto* limitShapeInitializer = ctx.getInputData(1); const TensorProto* deltaShapeInitializer = (ctx.getNumInputs() > 2) ? ctx.getInputData(2) : nullptr; const auto& startTensorType = ctx.getInputType(0)->tensor_type(); - TensorProto_DataType dtype = startTensorType.elem_type(); + int dtype = startTensorType.elem_type(); int64_t n = CalcResultDim(startShapeInitializer, limitShapeInitializer, deltaShapeInitializer, dtype); dim.set_dim_value(n); diff --git a/onnxruntime/core/graph/graph.cc b/onnxruntime/core/graph/graph.cc index 08b0a1d6e0b9d..cd3cfaa4c2bde 100644 --- a/onnxruntime/core/graph/graph.cc +++ b/onnxruntime/core/graph/graph.cc @@ -152,8 +152,8 @@ common::Status NodeArg::UpdateTypeAndShape(const ONNX_NAMESPACE::TypeProto& inpu if (input_tensor_elem_type != current_tensor_elem_type) return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Tensor element type mismatch. ", - TensorProto_DataType_Name(input_tensor_elem_type), " != ", - TensorProto_DataType_Name(current_tensor_elem_type)); + TensorProto_DataType_Name(static_cast(input_tensor_elem_type)), " != ", + TensorProto_DataType_Name(static_cast(current_tensor_elem_type))); if (input_tensor_type.has_shape()) { auto& current_tensor_type = *current_type.mutable_tensor_type(); @@ -172,8 +172,8 @@ common::Status NodeArg::UpdateTypeAndShape(const ONNX_NAMESPACE::TypeProto& inpu const auto current_tensor_elem_type = current_type.sparse_tensor_type().elem_type(); if (input_tensor_elem_type != current_tensor_elem_type) { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "SparseTensor element type mismatch. ", - TensorProto_DataType_Name(input_tensor_elem_type), " != ", - TensorProto_DataType_Name(current_tensor_elem_type)); + TensorProto_DataType_Name(static_cast(input_tensor_elem_type)), " != ", + TensorProto_DataType_Name(static_cast(current_tensor_elem_type))); } if (input_tensor_type.has_shape()) { auto& current_tensor_type = *current_type.mutable_sparse_tensor_type(); diff --git a/onnxruntime/core/graph/initializer.h b/onnxruntime/core/graph/initializer.h index 08b7951acda61..d6c15b2cd49ea 100644 --- a/onnxruntime/core/graph/initializer.h +++ b/onnxruntime/core/graph/initializer.h @@ -128,11 +128,11 @@ class Initializer final { } } - ONNX_NAMESPACE::TensorProto_DataType data_type() const { + int data_type() const { return data_type_; } - ONNX_NAMESPACE::TensorProto_DataType& data_type() { + int& data_type() { return data_type_; } @@ -372,7 +372,7 @@ class Initializer final { } private: - ONNX_NAMESPACE::TensorProto_DataType data_type_; + int data_type_; std::string name_; std::vector dims_; int64_t size_; diff --git a/onnxruntime/core/protobuf/onnx-ml.proto b/onnxruntime/core/protobuf/onnx-ml.proto index 79edbb6ef49ce..10463ddf63409 100644 --- a/onnxruntime/core/protobuf/onnx-ml.proto +++ b/onnxruntime/core/protobuf/onnx-ml.proto @@ -330,7 +330,7 @@ message TensorProto { repeated int64 dims = 1; // The data type of the tensor. - optional DataType data_type = 2; + optional int32 data_type = 2; // For very large tensors, we may want to store them in chunks, in which // case the following fields will specify the segment that is stored in @@ -438,7 +438,7 @@ message TypeProto { message Tensor { // This field MUST NOT have the value of UNDEFINED // This field MUST be present for this version of the IR. - optional TensorProto.DataType elem_type = 1; + optional int32 elem_type = 1; optional TensorShapeProto shape = 2; } @@ -454,7 +454,7 @@ message TypeProto { message Map { // This field MUST be present for this version of the IR. // This field MUST refer to an integral type ([U]INT{8|16|32|64}) or STRING - optional TensorProto.DataType key_type = 1; + optional int32 key_type = 1; // This field MUST be present for this version of the IR. optional TypeProto value_type = 2; }; @@ -469,10 +469,10 @@ message TypeProto { // repeated TypeProto parameters = 3; } - message SparseTensor { - // This field MUST NOT have the value of UNDEFINED - // This field MUST be present for this version of the IR. - optional TensorProto.DataType elem_type = 1; + message SparseTensor { + // This field MUST NOT have the value of UNDEFINED + // This field MUST be present for this version of the IR. + optional int32 elem_type = 1; optional TensorShapeProto shape = 2; } diff --git a/onnxruntime/test/util/compare_mlvalue.cc b/onnxruntime/test/util/compare_mlvalue.cc index 1b518b7413be7..a5c6f581d597a 100644 --- a/onnxruntime/test/util/compare_mlvalue.cc +++ b/onnxruntime/test/util/compare_mlvalue.cc @@ -26,7 +26,7 @@ using namespace onnxruntime; namespace { -ONNXTensorElementDataType CApiElementTypeFromProto(ONNX_NAMESPACE::TensorProto_DataType type) { +ONNXTensorElementDataType CApiElementTypeFromProto(int type) { switch (type) { CASE_TYPE(FLOAT) CASE_TYPE(UINT8) diff --git a/tools/ci_build/github/linux/docker/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/scripts/install_deps.sh index 364cbb8bf8d19..d092283f78010 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_deps.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_deps.sh @@ -33,8 +33,8 @@ else #Install ONNX #5af210ca8a1c73aa6bae8754c9346ec54d0a756e is v1.2.3 #bae6333e149a59a3faa9c4d9c44974373dcf5256 is v1.3.0 - #0a4d5abdf4939ab0842a5eadcc16a3bf0738f901 is v1.3.0 latest - for onnx_version in "5af210ca8a1c73aa6bae8754c9346ec54d0a756e" "bae6333e149a59a3faa9c4d9c44974373dcf5256" "0a4d5abdf4939ab0842a5eadcc16a3bf0738f901"; do + #0c8d857bb162431912b255d5c0e773fb7c131a65 is v1.3.0 latest + for onnx_version in "5af210ca8a1c73aa6bae8754c9346ec54d0a756e" "bae6333e149a59a3faa9c4d9c44974373dcf5256" "0c8d857bb162431912b255d5c0e773fb7c131a65"; do if [ -z ${lastest_onnx_version+x} ]; then echo "first pass"; else From beb326f00ed7a3e7ae31c5b7aa7cda8e091beb1e Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Wed, 19 Dec 2018 09:25:42 +1000 Subject: [PATCH 20/56] Simplify logic around creating relationship between nodes for implicit NodeArg usage. Allows using an initializer from multiple levels up to not fail. We would need to accumulate a list of initializers from all levels up otherwise, and doing so doesn't add any value. (#200) Improve a comment to clarify when the parent graph NodeArg lookup kicks in. --- onnxruntime/core/graph/graph.cc | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/onnxruntime/core/graph/graph.cc b/onnxruntime/core/graph/graph.cc index cd3cfaa4c2bde..5547ab91102e9 100644 --- a/onnxruntime/core/graph/graph.cc +++ b/onnxruntime/core/graph/graph.cc @@ -891,12 +891,12 @@ Status Graph::BuildConnections(std::vector& outer_scope_node_args_c subgraph->BuildConnections(node_args_consumed); for (auto& node_arg_name : node_args_consumed) { - bool node_arg_in_parent_graph = false; auto node_arg = GetNodeArg(node_arg_name); if (node_arg == nullptr) { // it's a node arg from outside this graph's scope, so add that to the list we return - // so that we can add the dependency at the next level up + // so that we can add the dependency at the next level up. this happens if you have multiple + // levels of subgraphs between the graph with the original NodeArg and the subgraph with implicit usage. outer_scope_node_args_consumed.push_back(node_arg_name); if (!parent_graph_) { @@ -916,8 +916,6 @@ Status Graph::BuildConnections(std::vector& outer_scope_node_args_c "Failed to find NodeArg in all parent graphs. Name=", node_arg_name, " Graph may not conform to the ONNX spec and contain initializers that are not graph inputs."); } - - node_arg_in_parent_graph = true; } // add it to the Node's list of implicit inputs @@ -931,18 +929,8 @@ Status Graph::BuildConnections(std::vector& outer_scope_node_args_c input_slot_index += static_cast(iter - implicit_inputs.cbegin()); } - if (node_arg_in_parent_graph || - resolve_context_.inputs_and_initializers.find(node_arg_name) != - resolve_context_.inputs_and_initializers.cend()) { - // no connection required if it's an input or initializer. - // if the node arg is from a parent graph we link the nodes in the parent graph by passing the - // node_arg_name back up in outer_scope_node_args_consumed - - } else { - // if it's an output nodearg in this graph we need to create a link to the node the output is coming from - auto entry = resolve_context_.output_args.find(node_arg_name); - ORT_ENFORCE(entry != resolve_context_.output_args.end()); - + auto entry = resolve_context_.output_args.find(node_arg_name); + if (entry != resolve_context_.output_args.end()) { // Create relationship between this node (node), and the node providing the output (output_node). Node& output_node = *entry->second.first; AddEdge(output_node.Index(), node->Index(), entry->second.second, input_slot_index); From 39f47f86eecdd74f7cce831ff42eee49a932964b Mon Sep 17 00:00:00 2001 From: Pranav Sharma Date: Tue, 18 Dec 2018 21:04:42 -0800 Subject: [PATCH 21/56] Adding the include folder for the C Windows pkg. (#198) * Adding the include folder for the C Windows pkg. * Add import lib to the pkg * Disable csharp pretrained tests temporarily --- .../Microsoft.ML.OnnxRuntime.csproj | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj b/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj index 54c17cad3acb5..ec902c46742c0 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj +++ b/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj @@ -5,7 +5,7 @@ true true false - OnnxRuntime.snk + OnnxRuntime.snk Microsoft.ML.OnnxRuntime @@ -31,6 +31,18 @@ + + - - + @(MajorVersionNumber) $(PackageVersion) From 334e329642d6d565755892cefff3b815bb866423 Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Wed, 19 Dec 2018 15:34:37 +1000 Subject: [PATCH 22/56] Increment/decrement UseCount for outputs so that we don't prematurely free a re-used output that is used for a dead output (output with zero users). (#214) --- onnxruntime/core/framework/allocation_planner.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/onnxruntime/core/framework/allocation_planner.cc b/onnxruntime/core/framework/allocation_planner.cc index c13d60b71fab7..277a6ff2ac1e2 100644 --- a/onnxruntime/core/framework/allocation_planner.cc +++ b/onnxruntime/core/framework/allocation_planner.cc @@ -389,6 +389,7 @@ class PlannerImpl { if (node_output->Exists()) { MLValueIndex index = Index(node_output->Name()); ProcessDef(index, node_output); + ++UseCount(index); if (strcmp(default_allocator_info.name, CPU) != 0) { // By default, outputs of this node are allocated on the default device allocator, // except for outputs marked for allocation in MemoryType: @@ -528,7 +529,7 @@ class PlannerImpl { if (node_output->Exists()) { auto& sym = node_output->Name(); auto original = Buffer(Index(sym)); - if (0 == UseCount(original)) + if (0 == --UseCount(original)) freelist_.push_front(FreeBufferInfo(original, program_counter)); } } From ab350fa4c7f3c98a5792a48042673eaaeba99a88 Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Wed, 19 Dec 2018 18:56:35 +1000 Subject: [PATCH 23/56] Re-structure the inference session initialization to (#217) - apply any transforms to the main graph and any subgraphs first - call Graph::Resolve() once on the main graph, which will recurse into the subgraphs - previously it was called after the transform on each subgraph, which results in it traversing up to the main graph to call resolve, and that resolve call recursing into all subgraphs every time. This avoids lots of unnecessary Graph::Resolve calls, and prevents subgraphs from being broken by SessionStateInitializer::InitializeAndSave calling graph_.CleanAllInitializedTensors() prior to final Graph::Resolve call. If a subgraph has optional inputs the backing initializers were removed by CleanAllInitializedTensors causing the next Resolve to incorrectly turn them into required inputs. --- .../framework/session_state_initializer.cc | 56 +------------- .../framework/session_state_initializer.h | 4 +- onnxruntime/core/graph/graph_utils.cc | 74 ++++++++++++++++--- onnxruntime/core/graph/graph_utils.h | 15 ++-- onnxruntime/core/session/inference_session.cc | 61 +++++++++++++-- 5 files changed, 130 insertions(+), 80 deletions(-) diff --git a/onnxruntime/core/framework/session_state_initializer.cc b/onnxruntime/core/framework/session_state_initializer.cc index 6d8df8500ed80..4c2126c48c244 100644 --- a/onnxruntime/core/framework/session_state_initializer.cc +++ b/onnxruntime/core/framework/session_state_initializer.cc @@ -26,12 +26,6 @@ namespace onnxruntime { -static common::Status TransformGraph(onnxruntime::Graph& graph, - const onnxruntime::GraphTransformerManager& graph_transformer_mgr, - const ExecutionProviders& exec_providers, - KernelRegistryManager& kernel_registry_manager, - const InsertCastTransformer& insert_cast_transformer); - static common::Status SaveMLValueNameIndexMapping(const onnxruntime::Graph& graph, MLValueNameIdxMap& mlvalue_name_idx_map, const logging::Logger& logger); @@ -68,15 +62,9 @@ SessionStateInitializer::SessionStateInitializer(onnxruntime::Graph& graph, logger_{logger} { } -common::Status SessionStateInitializer::CreatePlan(const onnxruntime::GraphTransformerManager& graph_transformation_manager, - const InsertCastTransformer& insert_cast_transformer, - const std::vector& outer_scope_node_args, +common::Status SessionStateInitializer::CreatePlan(const std::vector& outer_scope_node_args, bool enable_sequential_execution) { - ORT_RETURN_IF_ERROR(TransformGraph(graph_, graph_transformation_manager, - execution_providers_, kernel_registry_manager_, - insert_cast_transformer)); - - // After transformation/partitioning, the graph now is fixed and graph viewer is created and set for execution. + // the graph now is fixed and graph viewer is created and set for execution. session_state_.SetGraphViewer(std::make_unique(graph_)); auto& mlvalue_name_idx_map = session_state_.GetMLValueNameIdxMap(); @@ -142,46 +130,6 @@ common::Status SessionStateInitializer::InitializeAndSave(bool enable_memory_pat return Status::OK(); } -common::Status TransformGraph(onnxruntime::Graph& graph, - const onnxruntime::GraphTransformerManager& graph_transformer_mgr, - const ExecutionProviders& providers, - KernelRegistryManager& kernel_registry_manager, - const InsertCastTransformer& insert_cast_transformer) { - // The transformer order: - // 1. built-in graph rewriter - // 2. each execution provider's transformer - // 3. do node placement according to kernel definition - // 4. insert copy nodes - // 5. insert cast nodes. - - // first apply the default/system/basic graph to graph optimizations. - ORT_RETURN_IF_ERROR(graph_transformer_mgr.ApplyAll(graph)); - - auto kernels{kernel_registry_manager.GetAllKernelRegistries()}; - - // Do partitioning based on execution providers' capability. - GraphPartitioner partitioner(kernel_registry_manager, providers); - ORT_RETURN_IF_ERROR(partitioner.Partition(graph)); - - // Insert copy nodes. - for (auto& provider : providers) { - if (provider->Type() != onnxruntime::kCpuExecutionProvider && - provider->Type() != onnxruntime::kMklDnnExecutionProvider && - provider->Type() != onnxruntime::kNupharExecutionProvider) { - TransformerMemcpyImpl copy_impl(graph, provider->Type()); - copy_impl.ModifyGraph(kernel_registry_manager); - } - } - - // Insert cast node/s. - bool modified = false; - ORT_RETURN_IF_ERROR(insert_cast_transformer.Apply(graph, modified)); - - ORT_RETURN_IF_ERROR(graph.Resolve()); - - return common::Status::OK(); -} - // Build the MLValue name->idx mapping common::Status SaveMLValueNameIndexMapping(const onnxruntime::Graph& graph, MLValueNameIdxMap& mlvalue_name_idx_map, diff --git a/onnxruntime/core/framework/session_state_initializer.h b/onnxruntime/core/framework/session_state_initializer.h index 95fc97d65b3ab..fa8e4b5f98847 100644 --- a/onnxruntime/core/framework/session_state_initializer.h +++ b/onnxruntime/core/framework/session_state_initializer.h @@ -29,9 +29,7 @@ class SessionStateInitializer { const logging::Logger& logger); // First perform any transformations and create the execution plan - common::Status CreatePlan(const onnxruntime::GraphTransformerManager& graph_transformation_manager, - const InsertCastTransformer& insert_cast_transformer, - const std::vector& outer_scope_node_args, + common::Status CreatePlan(const std::vector& outer_scope_node_args, bool enable_sequential_execution); // initialize tensors, and save. save kernels and input/output node mappings diff --git a/onnxruntime/core/graph/graph_utils.cc b/onnxruntime/core/graph/graph_utils.cc index 7daeba838839d..3dcbc71358136 100644 --- a/onnxruntime/core/graph/graph_utils.cc +++ b/onnxruntime/core/graph/graph_utils.cc @@ -4,18 +4,70 @@ namespace onnxruntime { namespace utils { - // fusion is only done for ONNX domain ops - bool IsSupportedOptypeVersionAndDomain(const Node& node, - const std::string& op_type, - ONNX_NAMESPACE::OperatorSetVersion version, - const std::string& domain) { - if (node.OpType() != op_type || - node.Op()->Deprecated() || node.Op()->SinceVersion() != version || - (!node.Domain().empty() && node.Domain() != domain)) { - return false; +// fusion is only done for ONNX domain ops +bool IsSupportedOptypeVersionAndDomain(const Node& node, + const std::string& op_type, + ONNX_NAMESPACE::OperatorSetVersion version, + const std::string& domain) { + if (node.OpType() != op_type || + node.Op()->Deprecated() || node.Op()->SinceVersion() != version || + (!node.Domain().empty() && node.Domain() != domain)) { + return false; + } + return true; +} + +Status ForAllMutableSubgraphs(Graph& graph, std::function func) { + Status status = Status::OK(); + + for (auto& node : graph.Nodes()) { + for (auto& attribute : node.GetAttributes()) { + auto& name = attribute.first; + auto& proto = attribute.second; + + // check if it has a subgraph + if (proto.has_g()) { + Graph* subgraph = node.GetMutableGraphAttribute(name); + ORT_ENFORCE(subgraph, "Main Graph instance should have populated all subgraphs when being resolved."); + + status = func(*subgraph); + ORT_RETURN_IF_ERROR(status); + + // recurse + status = ForAllMutableSubgraphs(*subgraph, func); + ORT_RETURN_IF_ERROR(status); + } } - return true; } + + return status; +} + +Status ForAllSubgraphs(const Graph& graph, std::function func) { + Status status = Status::OK(); + + for (auto& node : graph.Nodes()) { + for (auto& attribute : node.GetAttributes()) { + auto& name = attribute.first; + auto& proto = attribute.second; + + // check if it has a subgraph + if (proto.has_g()) { + const Graph* subgraph = node.GetGraphAttribute(name); + ORT_ENFORCE(subgraph, "Main Graph instance should have populated all subgraphs when being resolved."); + + status = func(*subgraph); + ORT_RETURN_IF_ERROR(status); + + // recurse + status = ForAllSubgraphs(*subgraph, func); + ORT_RETURN_IF_ERROR(status); + } + } + } + + return status; } -} // namespace onnxruntime \ No newline at end of file +} // namespace utils +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/graph_utils.h b/onnxruntime/core/graph/graph_utils.h index 7ff5e93d3e09a..993a535bab0ff 100644 --- a/onnxruntime/core/graph/graph_utils.h +++ b/onnxruntime/core/graph/graph_utils.h @@ -9,10 +9,13 @@ namespace onnxruntime { namespace utils { - bool IsSupportedOptypeVersionAndDomain(const Node& node, - const std::string& op_type, - ONNX_NAMESPACE::OperatorSetVersion version, - const std::string& domain = kOnnxDomainAlias); -} +bool IsSupportedOptypeVersionAndDomain(const Node& node, + const std::string& op_type, + ONNX_NAMESPACE::OperatorSetVersion version, + const std::string& domain = kOnnxDomainAlias); -} \ No newline at end of file +Status ForAllMutableSubgraphs(Graph& main_graph, std::function func); +Status ForAllSubgraphs(Graph& main_graph, std::function func); + +} // namespace utils +} // namespace onnxruntime diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index 52703a8b2235d..f61cd61122367 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -14,6 +14,7 @@ #include "core/graph/graph_viewer.h" #include "core/graph/graph_transformer.h" #include "core/graph/graph_transformer_mgr.h" +#include "core/graph/graph_utils.h" #include "core/graph/model.h" #include "core/framework/allocatormgr.h" #include "core/framework/customregistry.h" @@ -247,6 +248,44 @@ class InferenceSession::Impl { return common::Status::OK(); } + static common::Status TransformGraph(onnxruntime::Graph& graph, + const onnxruntime::GraphTransformerManager& graph_transformer_mgr, + const ExecutionProviders& providers, + KernelRegistryManager& kernel_registry_manager, + const InsertCastTransformer& insert_cast_transformer) { + // The transformer order: + // 1. built-in graph rewriter + // 2. each execution provider's transformer + // 3. do node placement according to kernel definition + // 4. insert copy nodes + // 5. insert cast nodes. + + // first apply the default/system/basic graph to graph optimizations. + ORT_RETURN_IF_ERROR(graph_transformer_mgr.ApplyAll(graph)); + + auto kernels{kernel_registry_manager.GetAllKernelRegistries()}; + + // Do partitioning based on execution providers' capability. + GraphPartitioner partitioner(kernel_registry_manager, providers); + ORT_RETURN_IF_ERROR(partitioner.Partition(graph)); + + // Insert copy nodes. + for (auto& provider : providers) { + if (provider->Type() != onnxruntime::kCpuExecutionProvider && + provider->Type() != onnxruntime::kMklDnnExecutionProvider && + provider->Type() != onnxruntime::kNupharExecutionProvider) { + TransformerMemcpyImpl copy_impl(graph, provider->Type()); + copy_impl.ModifyGraph(kernel_registry_manager); + } + } + + // Insert cast node/s. + bool modified = false; + ORT_RETURN_IF_ERROR(insert_cast_transformer.Apply(graph, modified)); + + return common::Status::OK(); + } + // memory allocations for a subgraph that are owned by InferenceSession struct SubgraphMemory { std::unique_ptr session_state; @@ -277,10 +316,8 @@ class InferenceSession::Impl { SessionStateInitializer initializer{*subgraph, *subgraph_info.session_state, execution_providers_, kernel_registry_manager_, *session_logger_}; - ORT_RETURN_IF_ERROR( - initializer.CreatePlan(graph_transformation_mgr_, insert_cast_transformer_, - node.ImplicitInputDefs(), - session_options_.enable_sequential_execution)); + ORT_RETURN_IF_ERROR(initializer.CreatePlan(node.ImplicitInputDefs(), + session_options_.enable_sequential_execution)); ORT_RETURN_IF_ERROR(initializer.InitializeAndSave(session_state_.GetEnableMemoryPattern(), subgraph_info.weights_buffers)); @@ -347,9 +384,21 @@ class InferenceSession::Impl { SessionStateInitializer session_initializer{graph, session_state_, execution_providers_, kernel_registry_manager_, *session_logger_}; - ORT_RETURN_IF_ERROR(session_initializer.CreatePlan(graph_transformation_mgr_, insert_cast_transformer_, - {}, session_options_.enable_sequential_execution)); + // apply any transformations to the main graph and any subgraphs + ORT_RETURN_IF_ERROR(TransformGraph(graph, graph_transformation_mgr_, + execution_providers_, kernel_registry_manager_, + insert_cast_transformer_)); + + ORT_RETURN_IF_ERROR(utils::ForAllMutableSubgraphs(graph, [this](Graph& subgraph) { + return TransformGraph(subgraph, graph_transformation_mgr_, + execution_providers_, kernel_registry_manager_, + insert_cast_transformer_); + })); + + // now that all the transforms are done, call Resolve on the main graph. this will recurse into the subgraphs. + ORT_RETURN_IF_ERROR(graph.Resolve()); + ORT_RETURN_IF_ERROR(session_initializer.CreatePlan({}, session_options_.enable_sequential_execution)); ORT_RETURN_IF_ERROR(session_initializer.InitializeAndSave(session_state_.GetEnableMemoryPattern(), weights_buffers_)); From 0248390e4d8c03afe29d0a07e2d0558fed7cb40f Mon Sep 17 00:00:00 2001 From: Jesse Benson Date: Wed, 19 Dec 2018 09:25:25 -0800 Subject: [PATCH 24/56] Add support for checking for F16C support (https://en.wikipedia.org/wiki/F16C). (#212) --- onnxruntime/core/common/cpuid_info.cc | 1 + onnxruntime/core/common/cpuid_info.h | 2 ++ 2 files changed, 3 insertions(+) diff --git a/onnxruntime/core/common/cpuid_info.cc b/onnxruntime/core/common/cpuid_info.cc index dab771b9bc903..c0946b88f48e9 100644 --- a/onnxruntime/core/common/cpuid_info.cc +++ b/onnxruntime/core/common/cpuid_info.cc @@ -56,6 +56,7 @@ CPUIDInfo::CPUIDInfo() noexcept { int value = XGETBV(); bool has_avx = (data[2] & (1 << 28)) && ((value & AVX_MASK) == AVX_MASK); bool has_avx512 = (value & AVX512_MASK) == AVX512_MASK; + has_f16c_ = has_avx && (data[2] & (1 << 29)) && (data[3] & (1 << 26)); if (num_IDs >= 7) { GetCPUID(7, data); diff --git a/onnxruntime/core/common/cpuid_info.h b/onnxruntime/core/common/cpuid_info.h index 2a1b89d9e70fd..de6c57431a13b 100644 --- a/onnxruntime/core/common/cpuid_info.h +++ b/onnxruntime/core/common/cpuid_info.h @@ -14,11 +14,13 @@ class CPUIDInfo { bool HasAVX2() const { return has_avx2_; } bool HasAVX512f() const { return has_avx512f_; } + bool HasF16C() const { return has_f16c_; } private: CPUIDInfo() noexcept; bool has_avx2_{false}; bool has_avx512f_{false}; + bool has_f16c_{false}; }; } From ac3a081ec56bac62adc869b704416bb2f2f209d2 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 19 Dec 2018 11:12:43 -0800 Subject: [PATCH 25/56] Enable release build in Windows CI pipelines (#220) --- tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml | 2 +- tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml index 52a3e6a0004ec..f9fb03e3dca0d 100644 --- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml @@ -5,7 +5,7 @@ jobs: - task: BatchScript@1 inputs: filename: build.bat - arguments: ' --enable_pybind --use_mkldnn --use_mklml --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests' + arguments: ' --config Debug Release --enable_pybind --use_mkldnn --use_mklml --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests' workingFolder: "$(Build.SourcesDirectory)" - task: CmdLine@1 diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml index 53c1f1a9baa14..cd6beffae8354 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml @@ -18,7 +18,7 @@ jobs: - task: BatchScript@1 inputs: filename: build.bat - arguments: ' --enable_onnx_tests --use_mkldnn --build_shared_lib --build_csharp --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda"' + arguments: ' --config Debug Release --enable_onnx_tests --use_mkldnn --build_shared_lib --build_csharp --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda"' workingFolder: "$(Build.SourcesDirectory)" - task: PowerShell@1 displayName: 'Clean up CUDA props files' From 4d010fb1eabaaa38fcd0939043c4a9e54bca1f78 Mon Sep 17 00:00:00 2001 From: Bowen Bao Date: Wed, 19 Dec 2018 11:43:36 -0800 Subject: [PATCH 26/56] Add null check before calling node.op_->Deprecated(). (#211) --- onnxruntime/core/graph/graph.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/core/graph/graph.cc b/onnxruntime/core/graph/graph.cc index 5547ab91102e9..38b2df4c78d8d 100644 --- a/onnxruntime/core/graph/graph.cc +++ b/onnxruntime/core/graph/graph.cc @@ -1637,7 +1637,7 @@ Status Graph::VerifyNodeAndOpMatch() { auto maxInclusiveVersion = DomainToVersionMap().find(domain)->second; node.op_ = schema_registry_->GetSchema(node.OpType(), maxInclusiveVersion, node.Domain()); - if (node.op_->Deprecated()) { + if (node.op_ && node.op_->Deprecated()) { node.op_ = nullptr; } From b9cc134576f5090fc8e0029ff894f471e86c068d Mon Sep 17 00:00:00 2001 From: KeDengMS Date: Wed, 19 Dec 2018 13:45:04 -0800 Subject: [PATCH 27/56] Make sure tensor sizes are 64-byte aligned (#222) This helps reduce misaligned access violation --- onnxruntime/core/framework/execution_frame.cc | 2 +- onnxruntime/test/framework/execution_frame_test.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/onnxruntime/core/framework/execution_frame.cc b/onnxruntime/core/framework/execution_frame.cc index c5323c4e0a0ed..4f4f42c091a51 100644 --- a/onnxruntime/core/framework/execution_frame.cc +++ b/onnxruntime/core/framework/execution_frame.cc @@ -89,7 +89,7 @@ Status ExecutionFrame::AllocateMLValueTensorSelfOwnBufferHelper(int mlvalue_inde if (len < 0) { return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Tensor shape cannot contain any negative value"); } - if (!IAllocator::CalcMemSizeForArray(len, element_type->Size(), &size)) { + if (!IAllocator::CalcMemSizeForArrayWithAlignment<64>(len, element_type->Size(), &size)) { return Status(ONNXRUNTIME, FAIL, "size overflow"); } } diff --git a/onnxruntime/test/framework/execution_frame_test.cc b/onnxruntime/test/framework/execution_frame_test.cc index 8125bcb0ddc32..e2b741cffd44c 100644 --- a/onnxruntime/test/framework/execution_frame_test.cc +++ b/onnxruntime/test/framework/execution_frame_test.cc @@ -257,9 +257,9 @@ TEST(ExecutionFrameTest, MemPatternTest) { EXPECT_EQ(pattern.patterns.size(), pattern.locations.size()); EXPECT_EQ(pattern.patterns.size(), 1); auto p = pattern.GetPatterns(cpu_allocator->Info()); - EXPECT_EQ(p->PeakSize(), sizeof(float) * (4 + 6)); + EXPECT_EQ(p->PeakSize(), 2 * 64); // each allocation is 64-byte aligned EXPECT_EQ(p->GetBlock(3)->offset_, 0); - EXPECT_EQ(p->GetBlock(4)->offset_, sizeof(float) * 4); + EXPECT_EQ(p->GetBlock(4)->offset_, 64); } } // namespace test } // namespace onnxruntime From 94f8f2b05c2ae1a2bdb427919f49d7e2ad263a8c Mon Sep 17 00:00:00 2001 From: "Tang, Cheng" Date: Wed, 19 Dec 2018 13:54:12 -0800 Subject: [PATCH 28/56] placeholder for internal contrib ops (#219) --- .../core/graph/contrib_ops/contrib_defs.cc | 90 ++++++++++--------- .../graph/contrib_ops/internal_schema_defs.cc | 12 +++ .../graph/contrib_ops/internal_schema_defs.h | 22 +++++ 3 files changed, 80 insertions(+), 44 deletions(-) create mode 100644 onnxruntime/core/graph/contrib_ops/internal_schema_defs.cc create mode 100644 onnxruntime/core/graph/contrib_ops/internal_schema_defs.h diff --git a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc index a125c720683d4..e11b3af821632 100644 --- a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc +++ b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc @@ -5,6 +5,7 @@ #include "core/graph/contrib_ops/attn_lstm_schema_defs.h" #include "core/graph/contrib_ops/contrib_defs.h" #include "core/graph/contrib_ops/range_schema_defs.h" +#include "core/graph/contrib_ops/internal_schema_defs.h" #include "core/graph/op.h" #include "onnx/defs/shape_inference.h" @@ -590,49 +591,49 @@ The bounding box coordinates corresponding to the selected indices can then be o .SetDoc(R"DOC([optional] Step1: Remove elements in X if they match any of the stop words so that the output tensor will not contain any stop words. This operator only accepts [C]- and [1, C]-tensors. If all elements in X are dropped, the output will be the default value of string tensor with shape [1] if input shape is [C] and shape [1, 1] if input shape is [1, C].)DOC"); ONNX_CONTRIB_OPERATOR_SCHEMA(GatherND) - .SetDomain(kMSDomain) - .SinceVersion(1) - .Input (0, "data", "Tensor of rank r >= 1.", "T" ) - .Input (1, "indices", "Tensor of rank q >= 1.", "Tind" ) - .Output (0, "output", "Tensor of rank q-1+r-indices[-1].", "T" ) - .TypeConstraint( - "T", - OpSchema::all_tensor_types(), - "Constrain input and output types to any tensor type.") - .TypeConstraint( - "Tind", - {"tensor(int32)", "tensor(int64)"}, - "Constrain indice type to int32 or int64") - .TypeAndShapeInferenceFunction( [] (ONNX_NAMESPACE::InferenceContext& ctx) { - propagateElemTypeFromInputToOutput(ctx, 0, 0); - if (!hasNInputShapes(ctx, 2)) { - fail_shape_inference("GatherND requires two tensor inputs."); - } - auto& data_shape = ctx.getInputType(0)->tensor_type().shape(); - auto& indices_shape = ctx.getInputType(1)->tensor_type().shape(); - auto data_rank = data_shape.dim_size(); - auto indices_rank = indices_shape.dim_size(); - if (data_rank < 1 || indices_rank < 1) { - fail_shape_inference("both data and indices tensor need to have rank larger than zero."); - } - auto last_indice_dimension = indices_shape.dim(indices_rank - 1).dim_value(); - if (last_indice_dimension > data_rank) { - fail_shape_inference("last dimension of indices must not be larger and rank of data tensor"); - } - for (int i = 0; i < indices_rank - 1; ++i) { - *ctx.getOutputType(0) - ->mutable_tensor_type() - ->mutable_shape() - ->add_dim() = indices_shape.dim(i); - } - for (int i = static_cast(last_indice_dimension); i < data_rank; ++i) { - *ctx.getOutputType(0) - ->mutable_tensor_type() - ->mutable_shape() - ->add_dim() = data_shape.dim(i); - } - }) - .SetDoc(R"DOC( + .SetDomain(kMSDomain) + .SinceVersion(1) + .Input(0, "data", "Tensor of rank r >= 1.", "T") + .Input(1, "indices", "Tensor of rank q >= 1.", "Tind") + .Output(0, "output", "Tensor of rank q-1+r-indices[-1].", "T") + .TypeConstraint( + "T", + OpSchema::all_tensor_types(), + "Constrain input and output types to any tensor type.") + .TypeConstraint( + "Tind", + {"tensor(int32)", "tensor(int64)"}, + "Constrain indice type to int32 or int64") + .TypeAndShapeInferenceFunction([](ONNX_NAMESPACE::InferenceContext& ctx) { + propagateElemTypeFromInputToOutput(ctx, 0, 0); + if (!hasNInputShapes(ctx, 2)) { + fail_shape_inference("GatherND requires two tensor inputs."); + } + auto& data_shape = ctx.getInputType(0)->tensor_type().shape(); + auto& indices_shape = ctx.getInputType(1)->tensor_type().shape(); + auto data_rank = data_shape.dim_size(); + auto indices_rank = indices_shape.dim_size(); + if (data_rank < 1 || indices_rank < 1) { + fail_shape_inference("both data and indices tensor need to have rank larger than zero."); + } + auto last_indice_dimension = indices_shape.dim(indices_rank - 1).dim_value(); + if (last_indice_dimension > data_rank) { + fail_shape_inference("last dimension of indices must not be larger and rank of data tensor"); + } + for (int i = 0; i < indices_rank - 1; ++i) { + *ctx.getOutputType(0) + ->mutable_tensor_type() + ->mutable_shape() + ->add_dim() = indices_shape.dim(i); + } + for (int i = static_cast(last_indice_dimension); i < data_rank; ++i) { + *ctx.getOutputType(0) + ->mutable_tensor_type() + ->mutable_shape() + ->add_dim() = data_shape.dim(i); + } + }) + .SetDoc(R"DOC( Given `data` tensor of rank r >= 1, and `indices` tensor of rank q >= 1, gather slices of `data` into an output tensor of rank q - 1 + r - indices[-1]. Example 1: @@ -652,7 +653,8 @@ Example 4: indices = [[[0,1]],[[1,0]]] output = [[[2,3]],[[4,5]]] )DOC"); - + // register internal ops + RegisterInternalSchemas(); } } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/core/graph/contrib_ops/internal_schema_defs.cc b/onnxruntime/core/graph/contrib_ops/internal_schema_defs.cc new file mode 100644 index 0000000000000..f88ed27cfc4ca --- /dev/null +++ b/onnxruntime/core/graph/contrib_ops/internal_schema_defs.cc @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "range_schema_defs.h" + +namespace onnxruntime { +namespace contrib { + +void RegisterInternalSchemas() {} + +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/core/graph/contrib_ops/internal_schema_defs.h b/onnxruntime/core/graph/contrib_ops/internal_schema_defs.h new file mode 100644 index 0000000000000..25333585faf53 --- /dev/null +++ b/onnxruntime/core/graph/contrib_ops/internal_schema_defs.h @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wignored-qualifiers" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif +#include "onnx/defs/schema.h" +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + +namespace onnxruntime { +namespace contrib { + +void RegisterInternalSchemas(); + +} // namespace contrib +} // namespace onnxruntime From e63572c1f3a2922388eca309991008a519435f6c Mon Sep 17 00:00:00 2001 From: Edward Chen Date: Wed, 19 Dec 2018 13:08:01 -0800 Subject: [PATCH 29/56] Updated ArrayFeatureExtractor op to retain old output shape behavior. --- .../core/providers/cpu/ml/array_feature_extractor.cc | 8 ++++++-- .../test/providers/cpu/ml/array_feature_extractor_test.cc | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/onnxruntime/core/providers/cpu/ml/array_feature_extractor.cc b/onnxruntime/core/providers/cpu/ml/array_feature_extractor.cc index aedd428915038..212ca2eef9abb 100644 --- a/onnxruntime/core/providers/cpu/ml/array_feature_extractor.cc +++ b/onnxruntime/core/providers/cpu/ml/array_feature_extractor.cc @@ -80,9 +80,13 @@ common::Status ArrayFeatureExtractorOp::Compute(OpKernelContext* context) con } } - const TensorShape z_shape = [num_indices, &x_shape]() { + const TensorShape z_shape = [num_indices, x_num_dims, &x_shape]() { + if (x_num_dims == 1) { + // special case: for 1D input, return {1, num_indices} for backwards compatibility + return TensorShape{1, num_indices}; + } TensorShape shape{x_shape}; - shape[shape.NumDimensions() - 1] = num_indices; + shape[x_num_dims - 1] = num_indices; return shape; }(); Tensor* Z = context->Output(0, z_shape); diff --git a/onnxruntime/test/providers/cpu/ml/array_feature_extractor_test.cc b/onnxruntime/test/providers/cpu/ml/array_feature_extractor_test.cc index efa78a89a85f5..8d8b5b04e0a16 100644 --- a/onnxruntime/test/providers/cpu/ml/array_feature_extractor_test.cc +++ b/onnxruntime/test/providers/cpu/ml/array_feature_extractor_test.cc @@ -84,7 +84,7 @@ TEST_F(ArrayFeatureExtractorTest, HigherDimensionalX) { TEST_F(ArrayFeatureExtractorTest, OneDimensionalX) { test_.AddInput("X", {1}, {42}); test_.AddInput("Y", {1, 3}, {0, 0, 0}); - test_.AddOutput("Z", {3}, {42, 42, 42}); + test_.AddOutput("Z", {1, 3}, {42, 42, 42}); test_.Run(); } From 84231ba0033ff690773ed46b8dae6f62c8e3549a Mon Sep 17 00:00:00 2001 From: ashku-ms <33531737+ashku-ms@users.noreply.github.com> Date: Wed, 19 Dec 2018 14:23:09 -0800 Subject: [PATCH 30/56] support hyperbolic ops (#223) * support hyperbolic fns This commit adds support for sinh and cosh. Support for hyperbolic inverses is not available in Eigen yet. * Make constructors explicit * remove tests from exclude list * Revert "remove tests from exclude list" This reverts commit 2112a30b57d5a899991de4847e948e700a44e85d. * remove test names from excluded list * remove tanh since its already implemented --- .../providers/cpu/cpu_execution_provider.cc | 8 +++- .../providers/cpu/math/element_wise_ops.cc | 46 +++++++++++++++++-- onnxruntime/test/onnx/main.cc | 2 - .../cpu/math/element_wise_ops_test.cc | 40 ++++++++++------ .../test/python/onnx_backend_test_series.py | 2 - 5 files changed, 73 insertions(+), 25 deletions(-) diff --git a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc index 1e6318949a73e..0a387a92fbcb5 100644 --- a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc +++ b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc @@ -196,6 +196,8 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, Eye class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, float, IsNaN); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, MLFloat16, IsNaN); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, Erf); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, Sinh); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, Cosh); void RegisterOnnxOperatorKernels(std::function fn) { fn(BuildKernel()); @@ -384,6 +386,8 @@ void RegisterOnnxOperatorKernels(std::function fn) { fn(BuildKernel()); fn(BuildKernel()); fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); } // Forward declarations of ml op kernels @@ -485,7 +489,7 @@ static void RegisterCPUKernels(std::function create_fn std::shared_ptr CPUExecutionProvider::GetKernelRegistry() const { static std::shared_ptr - kernel_registry = std::make_shared(RegisterCPUKernels); + kernel_registry = std::make_shared(RegisterCPUKernels); return kernel_registry; } @@ -493,7 +497,7 @@ std::vector> CPUExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph, const std::vector& kernel_registries) const { std::vector> - result = IExecutionProvider::GetCapability(graph, kernel_registries); + result = IExecutionProvider::GetCapability(graph, kernel_registries); for (auto& rule : fuse_rules_) { rule(graph, result); diff --git a/onnxruntime/core/providers/cpu/math/element_wise_ops.cc b/onnxruntime/core/providers/cpu/math/element_wise_ops.cc index f2063f5736a24..1f8324fbefe6e 100644 --- a/onnxruntime/core/providers/cpu/math/element_wise_ops.cc +++ b/onnxruntime/core/providers/cpu/math/element_wise_ops.cc @@ -400,11 +400,10 @@ Status Pow::Compute(OpKernelContext* context) const { std::function, ConstEigenVectorMap, float)> input1scalar = [](EigenVectorMap output, ConstEigenVectorMap input0, float input1) { output = Eigen::pow(input0.array(), input1); }; if (Y.Shape().Size() == 1) { - float value = * Y.Data(); + float value = *Y.Data(); if (value == 2.0) { input1scalar = [](EigenVectorMap output, ConstEigenVectorMap input0, float) { output = Eigen::square(input0.array()); }; - } - else if (value == 3.0) { + } else if (value == 3.0) { input1scalar = [](EigenVectorMap output, ConstEigenVectorMap input0, float) { output = Eigen::cube(input0.array()); }; } } @@ -789,6 +788,46 @@ ONNX_CPU_OPERATOR_KERNEL( KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), Atan); +template +class Sinh final : public OpKernel { + public: + explicit Sinh(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override { + auto& X = *context->Input(0); + auto& Y = *context->Output(0, X.Shape()); + MakeEigenArrayMap(Y) = MakeEigenArrayMap(X).sinh(); + return Status::OK(); + } +}; + +ONNX_CPU_OPERATOR_KERNEL( + Sinh, + 9, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Sinh); + +template +class Cosh final : public OpKernel { + public: + explicit Cosh(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* context) const override { + auto& X = *context->Input(0); + auto& Y = *context->Output(0, X.Shape()); + MakeEigenArrayMap(Y) = MakeEigenArrayMap(X).cosh(); + return Status::OK(); + } +}; + +ONNX_CPU_OPERATOR_KERNEL( + Cosh, + 9, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Cosh); + template <> Status PRelu::Compute(OpKernelContext* context) const { return BroadcastTwo( @@ -887,7 +926,6 @@ Status Erf::Compute(OpKernelContext* context) const { ORT_ENFORCE(X_ptr != nullptr); auto& X = *X_ptr; auto& Y = *context->Output(0, X.Shape()); - EigenMap(Y) = EigenMap(X).array().erf(); return Status::OK(); diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index 3da5bfefd5cb8..966f674ed90d2 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -314,8 +314,6 @@ int real_main(int argc, char* argv[]) { {"upsample_nearest", "opset 9 not supported yet"}, {"onehot_with_axis", "opset 9 not supported yet"}, {"onehot_without_axis", "opset 9 not supported yet"}, // also has bug in current test re: output type. Spandan to fix. - {"sinh", "opset 9 not supported yet"}, - {"cosh", "opset 9 not supported yet"}, {"asinh", "opset 9 not supported yet"}, {"acosh", "opset 9 not supported yet"}, {"atanh", "opset 9 not supported yet"}, diff --git a/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc index e8845542390fe..fa1b369b2d2d6 100644 --- a/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc +++ b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc @@ -163,11 +163,11 @@ TEST(MathOpTest, Sub_int32) { } TEST(MathOpTest, Sub_int64) { - OpTester test("Sub"); - test.AddInput("A", { 3 }, { 1, 5, 6 }); - test.AddInput("B", { 3 }, { 4, 5, 3 }); - test.AddOutput("C", { 3 }, { -3, 0, 3 }); - test.Run(); + OpTester test("Sub"); + test.AddInput("A", {3}, {1, 5, 6}); + test.AddInput("B", {3}, {4, 5, 3}); + test.AddOutput("C", {3}, {-3, 0, 3}); + test.Run(); } TEST(MathOpTest, Sub) { @@ -212,11 +212,11 @@ TEST(MathOpTest, Mul_int32) { } TEST(MathOpTest, Mul_int64) { - OpTester test("Mul"); - test.AddInput("A", { 3 }, { 3, 6, -3 }); - test.AddInput("B", { 3 }, { 4, -3, -2 }); - test.AddOutput("C", { 3 }, { 12, -18, 6 }); - test.Run(); + OpTester test("Mul"); + test.AddInput("A", {3}, {3, 6, -3}); + test.AddInput("B", {3}, {4, -3, -2}); + test.AddOutput("C", {3}, {12, -18, 6}); + test.Run(); } TEST(MathOpTest, Mul) { @@ -246,11 +246,11 @@ TEST(MathOpTest, Div_int32) { } TEST(MathOpTest, Div_int64) { - OpTester test("Div"); - test.AddInput("A", { 3 }, { 4, 8, 8 }); - test.AddInput("B", { 3 }, { 2, 3, 4 }); - test.AddOutput("C", { 3 }, { 2, 2, 2 }); - test.Run(); + OpTester test("Div"); + test.AddInput("A", {3}, {4, 8, 8}); + test.AddInput("B", {3}, {2, 3, 4}); + test.AddOutput("C", {3}, {2, 2, 2}); + test.Run(); } TEST(MathOpTest, Div) { @@ -819,6 +819,16 @@ TEST(MathOpTest, Atan) { TrigTest(test, {-10.0f, -5.0f, 0.0f, 5.0f, 10.0f}); } +TEST(MathOpTest, Sinh) { + OpTester test("Sinh", 9); + TrigTest(test, {-1.0f, -0.5f, 0.0f, 0.5f, 1.0f}); +} + +TEST(MathOpTest, Cosh) { + OpTester test("Cosh", 9); + TrigTest(test, {-1.0f, -0.5f, 0.0f, 0.5f, 1.0f}); +} + TEST(MathOpTest, Expand_8_3x3) { OpTester test("Expand", 8); test.AddInput("data_0", {1}, {1.0f}); diff --git a/onnxruntime/test/python/onnx_backend_test_series.py b/onnxruntime/test/python/onnx_backend_test_series.py index 80f7f6c81f34d..5410c4dbcacc5 100644 --- a/onnxruntime/test/python/onnx_backend_test_series.py +++ b/onnxruntime/test/python/onnx_backend_test_series.py @@ -24,7 +24,6 @@ '|test_atanh_example_cpu.*' '|test_convtranspose_1d_cpu.*' '|test_convtranspose_3d_cpu.*' -'|test_cosh_cpu.*' '|test_cosh_example_cpu.*' '|test_dynamic_slice_cpu.*' '|test_dynamic_slice_default_axes_cpu.*' @@ -43,7 +42,6 @@ '|test_scatter_with_axis_cpu.*' '|test_scatter_without_axis_cpu.*' '|test_sign_cpu.*' -'|test_sinh_cpu.*' '|test_sinh_example_cpu.*' '|test_AvgPool1d_cpu.*' '|test_AvgPool1d_stride_cpu.*' From e97caa77875ea69a1345f68ccb002ac0dbd9ef2c Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 19 Dec 2018 14:45:57 -0800 Subject: [PATCH 31/56] change mkldnn so path (#210) --- cmake/CMakeLists.txt | 10 +++++++--- cmake/external/mkldnn.cmake | 21 ++------------------- cmake/onnxruntime_providers.cmake | 2 +- cmake/onnxruntime_util.cmake | 2 +- 4 files changed, 11 insertions(+), 24 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index f91c85b38ab4e..b42ace5b958c2 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -397,15 +397,19 @@ if (onnxruntime_USE_MKLML) add_definitions(-DUSE_MKLML=1) # USE_MKML_FOR_BLAS may cause numerical differences in tests so disable by default #add_definitions(-DUSE_MKLML_FOR_BLAS=1) - list(APPEND onnxruntime_EXTERNAL_LIBRARIES mklml) - list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES mklml) + if (WIN32 OR APPLE) + list(APPEND onnxruntime_EXTERNAL_LIBRARIES mklml) + else() + list(APPEND onnxruntime_EXTERNAL_LIBRARIES mklml_intel) + endif() + list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES project_mklml) link_directories(${MKLML_LIB_DIR}) endif() if (onnxruntime_USE_MKLDNN) add_definitions(-DUSE_MKLDNN=1) list(APPEND onnxruntime_EXTERNAL_LIBRARIES mkldnn) - list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES mkldnn) + list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES project_mkldnn) link_directories(${MKLDNN_LIB_DIR}) endif() diff --git a/cmake/external/mkldnn.cmake b/cmake/external/mkldnn.cmake index 659a6560fb929..9bd825171b401 100644 --- a/cmake/external/mkldnn.cmake +++ b/cmake/external/mkldnn.cmake @@ -45,15 +45,7 @@ if (onnxruntime_USE_MKLML) set(MKML_DIR ${CMAKE_CURRENT_BINARY_DIR}/mklml/src/project_mklml) set(MKLML_INCLUDE_DIR "${MKML_DIR}/include") set(MKLML_LIB_DIR "${MKML_DIR}/lib") - if(WIN32) - add_library(mklml STATIC IMPORTED) - set_property(TARGET mklml PROPERTY IMPORTED_LOCATION ${MKLML_LIB_DIR}/${MKLML_IMPORT_LIB}) - else() - add_library(mklml SHARED IMPORTED) - set_property(TARGET mklml PROPERTY IMPORTED_LOCATION ${MKLML_LIB_DIR}/${MKLML_SHARED_LIB}) - endif() - add_dependencies(mklml project_mklml) - include_directories(${MKLML_INCLUDE_DIR}) + link_directories(${MKLML_LIB_DIR}) endif() if (onnxruntime_USE_MKLDNN) @@ -74,17 +66,8 @@ if (onnxruntime_USE_MKLDNN) SOURCE_DIR ${MKLDNN_SOURCE} CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${MKLDNN_INSTALL} -DMKLROOT=${MKML_DIR} ) + link_directories(${MKLDNN_LIB_DIR}) if (onnxruntime_USE_MKLML) add_dependencies(project_mkldnn project_mklml) endif() - - if(WIN32) - add_library(mkldnn STATIC IMPORTED) - set_property(TARGET mkldnn PROPERTY IMPORTED_LOCATION ${MKLDNN_LIB_DIR}/${MKLDNN_IMPORT_LIB}) - else() - add_library(mkldnn SHARED IMPORTED) - set_property(TARGET mkldnn PROPERTY IMPORTED_LOCATION ${MKLDNN_LIB_DIR}/${MKLDNN_SHARED_LIB}) - endif() - add_dependencies(mkldnn project_mkldnn) - include_directories(${MKLDNN_INCLUDE_DIR}) endif() diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index 1386af9de6241..db74da04746ca 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -94,7 +94,7 @@ if (onnxruntime_USE_MKLDNN) onnxruntime_add_include_to_target(onnxruntime_providers_mkldnn onnx protobuf::libprotobuf) add_dependencies(onnxruntime_providers_mkldnn eigen ${onnxruntime_EXTERNAL_DEPENDENCIES}) set_target_properties(onnxruntime_providers_mkldnn PROPERTIES FOLDER "ONNXRuntime") - target_include_directories(onnxruntime_providers_mkldnn PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS}) + target_include_directories(onnxruntime_providers_mkldnn PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${MKLDNN_INCLUDE_DIR} ${MKLML_INCLUDE_DIR}) install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/mkldnn DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) set_target_properties(onnxruntime_providers_mkldnn PROPERTIES LINKER_LANGUAGE CXX) endif() diff --git a/cmake/onnxruntime_util.cmake b/cmake/onnxruntime_util.cmake index 525d851a92d70..af1b51326f1bc 100644 --- a/cmake/onnxruntime_util.cmake +++ b/cmake/onnxruntime_util.cmake @@ -9,7 +9,7 @@ file(GLOB_RECURSE onnxruntime_util_srcs source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_util_srcs}) add_library(onnxruntime_util ${onnxruntime_util_srcs}) -target_include_directories(onnxruntime_util PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS}) +target_include_directories(onnxruntime_util PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${MKLDNN_INCLUDE_DIR} ${MKLML_INCLUDE_DIR}) onnxruntime_add_include_to_target(onnxruntime_util onnx protobuf::libprotobuf) set_target_properties(onnxruntime_util PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(onnxruntime_util PROPERTIES FOLDER "ONNXRuntime") From 255ee39af6c68b64dcff27fa82c556a0d7fc5cc4 Mon Sep 17 00:00:00 2001 From: Dmitri Smirnov Date: Wed, 19 Dec 2018 17:46:21 -0800 Subject: [PATCH 32/56] Fix memory leak by improper handling of std::string typed (#227) output buffer. Tensor returns a buffer to fully constructed std::strings and we should treat them as such. --- .../contrib_ops/cpu/string_normalizer.cc | 13 ++++++------- onnxruntime/contrib_ops/cpu/tokenizer.cc | 18 +++++++++--------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/onnxruntime/contrib_ops/cpu/string_normalizer.cc b/onnxruntime/contrib_ops/cpu/string_normalizer.cc index 2b76a7bbb486b..a20eaade10d3b 100644 --- a/onnxruntime/contrib_ops/cpu/string_normalizer.cc +++ b/onnxruntime/contrib_ops/cpu/string_normalizer.cc @@ -42,7 +42,7 @@ class Locale { loc_ = _create_locale(LC_CTYPE, name.c_str()); if (loc_ == nullptr) { ORT_THROW("Failed to construct locale with name:", - name, ":", ":Please, install necessary language-pack-XX and configure locales"); + name, ":", ":Please, install necessary language-pack-XX and configure locales"); } } @@ -78,7 +78,7 @@ class Locale { explicit Locale(const std::string& name) try : loc_(name) { } catch (const std::runtime_error& e) { ORT_THROW("Failed to construct locale with name:", - name, ":", e.what(), ":Please, install necessary language-pack-XX and configure locales"); + name, ":", e.what(), ":Please, install necessary language-pack-XX and configure locales"); } ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(Locale); @@ -118,9 +118,8 @@ Status CopyCaseAction(ForwardIter first, ForwardIter end, OpKernelContext* ctx, if (C == 0) { output_dims.push_back(1); TensorShape output_shape(output_dims); - auto output_ten = ctx->Output(0, output_shape); - auto output_default = output_ten->template MutableData(); - new (output_default) std::string(); + // This will create one empty string + ctx->Output(0, output_shape); return Status::OK(); } @@ -141,11 +140,11 @@ Status CopyCaseAction(ForwardIter first, ForwardIter end, OpKernelContext* ctx, } // In place transform loc.ChangeCase(caseaction, wstr); - new (output_data + output_idx) std::string(converter.to_bytes(wstr)); + *(output_data + output_idx) = converter.to_bytes(wstr); } else { assert(caseaction == StringNormalizer::NONE); // Simple copy or move if the iterator points to a non-const string - new (output_data + output_idx) std::string(std::move(s)); + *(output_data + output_idx) = std::move(s); } ++output_idx; ++first; diff --git a/onnxruntime/contrib_ops/cpu/tokenizer.cc b/onnxruntime/contrib_ops/cpu/tokenizer.cc index 90aaeb9ab8e7e..4c31ec20e596d 100644 --- a/onnxruntime/contrib_ops/cpu/tokenizer.cc +++ b/onnxruntime/contrib_ops/cpu/tokenizer.cc @@ -218,7 +218,7 @@ Tokenizer::Tokenizer(const OpKernelInfo& info) : OpKernel(info) { separators[0].empty()); ORT_ENFORCE(!char_tokenezation_ || mincharnum_ < 2, - "mincharnum is too big for char level tokenezation"); + "mincharnum is too big for char level tokenezation"); // Create TST and insert separators if (!char_tokenezation_) { @@ -284,7 +284,7 @@ Status Tokenizer::CharTokenize(OpKernelContext* ctx, size_t N, size_t C, while (curr_input != last) { const auto& s = *curr_input; if (mark_) { - new (output_data + output_index) std::string(&start_text, 1); + (output_data + output_index)->assign(&start_text, 1); ++output_index; } size_t tokens = 0; @@ -295,20 +295,20 @@ Status Tokenizer::CharTokenize(OpKernelContext* ctx, size_t N, size_t C, assert(result); (void)result; assert(token_idx + tlen <= str_len); - new (output_data + output_index) std::string(s.substr(token_idx, tlen)); + *(output_data + output_index) = s.substr(token_idx, tlen); ++output_index; token_idx += tlen; ++tokens; } if (mark_) { - new (output_data + output_index) std::string(&end_text, 1); + (output_data + output_index)->assign(&end_text, 1); ++output_index; } // Padding strings assert(tokens + (mark_ * 2) <= max_tokens); const size_t pads = max_tokens - (mark_ * 2) - tokens; for (size_t p = 0; p < pads; ++p) { - new (output_data + output_index) std::string(pad_value_); + *(output_data + output_index) = pad_value_; ++output_index; } ++curr_input; @@ -422,21 +422,21 @@ Status Tokenizer::SeparatorTokenize(OpKernelContext* ctx, size_t c_idx = output_index; #endif if (mark_) { - new (output_data + output_index) std::string(&start_text, 1); + (output_data + output_index)->assign(&start_text, 1); ++output_index; } // Output tokens for this row for (auto& token : row) { - new (output_data + output_index) std::string(converter.to_bytes(token)); + *(output_data + output_index) = converter.to_bytes(token); ++output_index; } if (mark_) { - new (output_data + output_index) std::string(&end_text, 1); + (output_data + output_index)->assign(&end_text, 1); ++output_index; } const size_t pads = max_tokens - (mark_ * 2) - row.size(); for (size_t p = 0; p < pads; ++p) { - new (output_data + output_index) std::string(pad_value_); + *(output_data + output_index) = pad_value_; ++output_index; } #ifdef _DEBUG From 1d95c939874d706d591bf1dbb8976792470f5def Mon Sep 17 00:00:00 2001 From: Weixian Date: Wed, 19 Dec 2018 18:16:39 -0800 Subject: [PATCH 33/56] [optimization] avoid vector copy and reduce allocation. (#203) * few. * fix. --- onnxruntime/core/framework/execution_frame.cc | 17 +++++++++++++---- onnxruntime/core/framework/execution_frame.h | 14 +++++++++++++- onnxruntime/core/framework/op_kernel.cc | 4 ++-- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/onnxruntime/core/framework/execution_frame.cc b/onnxruntime/core/framework/execution_frame.cc index 4f4f42c091a51..9e3c122f21c58 100644 --- a/onnxruntime/core/framework/execution_frame.cc +++ b/onnxruntime/core/framework/execution_frame.cc @@ -284,7 +284,7 @@ Status ExecutionFrame::AllocateAsPerAllocationPlan(int mlvalue_index, ORT_RETURN_IF_ERROR(AllocateMLValueTensorSelfOwnBuffer(mlvalue_index, ml_data_type, alloc_info, - parameters.tensor_shape, + parameters.GetTensorShape(), per_alloc_plan.create_fence_if_async)); break; } @@ -294,7 +294,7 @@ Status ExecutionFrame::AllocateAsPerAllocationPlan(int mlvalue_index, reuse_mlvalue_index, ml_data_type, alloc_info, - parameters.tensor_shape, + parameters.GetTensorShape(), per_alloc_plan.create_fence_if_async)); break; } @@ -363,6 +363,15 @@ void ExecutionFrame::Init(const onnxruntime::GraphViewer& graph, } // 5. set node args + std::size_t total_def_count{}; + for (const auto& node : graph.Nodes()) + { + node.ForEachDef([&](const onnxruntime::NodeArg& /*arg*/, bool /*is_input*/) { + ++total_def_count; + }); + } + node_values_.reserve(total_def_count); + for (auto& node : graph.Nodes()) { ORT_ENFORCE(node.Index() < node_offsets_.size()); node_offsets_[node.Index()] = static_cast(node_values_.size()); @@ -451,9 +460,9 @@ static inline void VerifyShape(const MLValue* p_mlvalue, if (p_mlvalue->IsTensor()) { const Tensor* tensor = &p_mlvalue->Get(); - ORT_ENFORCE(tensor->Shape() == parameters.tensor_shape, + ORT_ENFORCE(tensor->Shape() == parameters.GetTensorShape(), "MLValue shape verification failed. Current shape:", tensor->Shape(), - " Requested shape:", parameters.tensor_shape); + " Requested shape:", parameters.GetTensorShape()); } } diff --git a/onnxruntime/core/framework/execution_frame.h b/onnxruntime/core/framework/execution_frame.h index 1d0a410e33ef5..67cfc88cf69f3 100644 --- a/onnxruntime/core/framework/execution_frame.h +++ b/onnxruntime/core/framework/execution_frame.h @@ -20,7 +20,19 @@ class MLValuePatternPlanner; struct MemoryPatternGroup; struct MLValueAllocationParameters { - TensorShape tensor_shape; + MLValueAllocationParameters() = default; + MLValueAllocationParameters(const TensorShape* shape) + : tensor_shape{ shape } + {} + + const TensorShape& GetTensorShape() const + { + static const TensorShape s_empty_tensor_shape; + return tensor_shape != nullptr ? *tensor_shape : s_empty_tensor_shape; + } + +private: + const TensorShape* tensor_shape{}; // todo: is there any parameter needed for ml types? }; diff --git a/onnxruntime/core/framework/op_kernel.cc b/onnxruntime/core/framework/op_kernel.cc index ad1a6d11498d1..65792961197b7 100644 --- a/onnxruntime/core/framework/op_kernel.cc +++ b/onnxruntime/core/framework/op_kernel.cc @@ -29,8 +29,8 @@ Tensor* OpKernelContext::Output(int index, const TensorShape& shape) { // In this case, it's assumed that the tensor hasn't been allocated yet, // so that it's calling ExecutionFrame to create a tensor in the given position with given shape. - MLValueAllocationParameters parameters; - parameters.tensor_shape = shape; + MLValueAllocationParameters parameters{ &shape }; + //: Though we don't need to give 'ret' an initial value, GCC would generate a warning if we don't do that //"error: 'ret' may be used uninitialized in this function" //This warning only exists in Release build. From 0dca08023894857fcc88c4a53ffb9a92b78287c5 Mon Sep 17 00:00:00 2001 From: "Tang, Cheng" Date: Wed, 19 Dec 2018 18:17:20 -0800 Subject: [PATCH 34/56] remove useless internal schema file (#226) * placeholder for internal contrib ops * remove useless internal file * fix build break --- cmake/CMakeLists.txt | 4 ++++ .../core/graph/contrib_ops/contrib_defs.cc | 8 ++++++- .../graph/contrib_ops/internal_schema_defs.cc | 12 ---------- .../graph/contrib_ops/internal_schema_defs.h | 22 ------------------- 4 files changed, 11 insertions(+), 35 deletions(-) delete mode 100644 onnxruntime/core/graph/contrib_ops/internal_schema_defs.cc delete mode 100644 onnxruntime/core/graph/contrib_ops/internal_schema_defs.h diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index b42ace5b958c2..5658f0fd36042 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -479,6 +479,10 @@ if (onnxruntime_USE_TVM) include(onnxruntime_codegen.cmake) endif() +if (onnxruntime_ENABLE_MICROSOFT_INTERNAL) + add_definitions(-DMICROSOFT_INTERNAL) +endif() + #names in this var must match the directory names under onnxruntime/core/providers set(ONNXRUNTIME_PROVIDER_NAMES cpu) diff --git a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc index e11b3af821632..9a2e58057c278 100644 --- a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc +++ b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc @@ -5,10 +5,13 @@ #include "core/graph/contrib_ops/attn_lstm_schema_defs.h" #include "core/graph/contrib_ops/contrib_defs.h" #include "core/graph/contrib_ops/range_schema_defs.h" -#include "core/graph/contrib_ops/internal_schema_defs.h" #include "core/graph/op.h" #include "onnx/defs/shape_inference.h" +#ifdef MICROSOFT_INTERNAL +#include "core/graph/contrib_ops/internal_schema_defs.h" +#endif + namespace ONNX_NAMESPACE { void convPoolTypeAndShapeInference(ONNX_NAMESPACE::InferenceContext& ctx, bool use_dilation, bool require_kernel_shape); } @@ -653,8 +656,11 @@ Example 4: indices = [[[0,1]],[[1,0]]] output = [[[2,3]],[[4,5]]] )DOC"); + +#ifdef MICROSOFT_INTERNAL // register internal ops RegisterInternalSchemas(); +#endif } } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/core/graph/contrib_ops/internal_schema_defs.cc b/onnxruntime/core/graph/contrib_ops/internal_schema_defs.cc deleted file mode 100644 index f88ed27cfc4ca..0000000000000 --- a/onnxruntime/core/graph/contrib_ops/internal_schema_defs.cc +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "range_schema_defs.h" - -namespace onnxruntime { -namespace contrib { - -void RegisterInternalSchemas() {} - -} // namespace contrib -} // namespace onnxruntime diff --git a/onnxruntime/core/graph/contrib_ops/internal_schema_defs.h b/onnxruntime/core/graph/contrib_ops/internal_schema_defs.h deleted file mode 100644 index 25333585faf53..0000000000000 --- a/onnxruntime/core/graph/contrib_ops/internal_schema_defs.h +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#ifdef __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wignored-qualifiers" -#pragma GCC diagnostic ignored "-Wunused-parameter" -#endif -#include "onnx/defs/schema.h" -#ifdef __GNUC__ -#pragma GCC diagnostic pop -#endif - -namespace onnxruntime { -namespace contrib { - -void RegisterInternalSchemas(); - -} // namespace contrib -} // namespace onnxruntime From abce6041c16b97545becacb1df5055bacfa55e99 Mon Sep 17 00:00:00 2001 From: KeDengMS Date: Wed, 19 Dec 2018 21:31:15 -0800 Subject: [PATCH 35/56] Print hex value for float compare when test failed (#228) This helps identify fp accuracy issues --- onnxruntime/test/util/compare_mlvalue.cc | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/onnxruntime/test/util/compare_mlvalue.cc b/onnxruntime/test/util/compare_mlvalue.cc index a5c6f581d597a..24459e5e80803 100644 --- a/onnxruntime/test/util/compare_mlvalue.cc +++ b/onnxruntime/test/util/compare_mlvalue.cc @@ -64,14 +64,20 @@ std::pair CompareFloatResult(const Tensor& outvalue const double real_value = post_processing ? std::max(0.0, std::min(255.0, real_output[di])) : real_output[di]; const double diff = fabs(expected_output[di] - real_value); - const double rtol = per_sample_tolerance + relative_per_sample_tolerance * fabs(expected_output[di]); - if (diff > rtol || (std::isnan(diff) && !std::isnan(expected_output[di]))) { + const double tol = per_sample_tolerance + relative_per_sample_tolerance * fabs(expected_output[di]); + if (diff > tol || (std::isnan(diff) && !std::isnan(expected_output[di]))) { res.first = COMPARE_RESULT::RESULT_DIFFERS; // update error message if this is a larger diff if (diff > max_diff || (std::isnan(diff) && !std::isnan(max_diff))) { + int64_t expected_int = 0; + int64_t real_int = 0; + memcpy(&expected_int, &expected_output[di], sizeof(FLOAT_TYPE)); + memcpy(&real_int, &real_output[di], sizeof(FLOAT_TYPE)); + std::ostringstream oss; - oss << "expected " << expected_output[di] << ", got " << real_value - << ", diff: " << diff << ", tol=" << rtol << "."; + oss << std::hex << "expected " << expected_output[di] << " (" << expected_int << "), got " + << real_value << " (" << real_int << ")" + << ", diff: " << diff << ", tol=" << tol << "."; res.second = oss.str(); max_diff = diff; } From eb867be33190e09b9fdad1fa42a434d24900930a Mon Sep 17 00:00:00 2001 From: jywu-msft <43355415+jywu-msft@users.noreply.github.com> Date: Thu, 20 Dec 2018 07:32:49 -0800 Subject: [PATCH 36/56] update mkldnn to 0.17.2 (#231) --- cmake/external/mkldnn.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/external/mkldnn.cmake b/cmake/external/mkldnn.cmake index 9bd825171b401..72ed6a108d939 100644 --- a/cmake/external/mkldnn.cmake +++ b/cmake/external/mkldnn.cmake @@ -2,7 +2,7 @@ include (ExternalProject) set(MKLDNN_URL https://github.com/intel/mkl-dnn.git) # If MKLDNN_TAG is updated, check if MKLML_VERSION and platform.cmake.patch need to be updated. -set(MKLDNN_TAG v0.17.1) +set(MKLDNN_TAG v0.17.2) set(MKLML_VERSION 2019.0.1.20180928) if(WIN32) From a19b6243025ff40fc1c85fc12f0a3028438e2853 Mon Sep 17 00:00:00 2001 From: ashku-ms <33531737+ashku-ms@users.noreply.github.com> Date: Thu, 20 Dec 2018 09:44:34 -0800 Subject: [PATCH 37/56] MaxUnpool Operator - CPU Implementation (#177) * Initial commit Maxunpool operator * fix gpu build failure * remove op test from excluded list * Change to ORT --- .../providers/cpu/cpu_execution_provider.cc | 2 + onnxruntime/core/providers/cpu/nn/Unpool.cc | 200 +++++++++ onnxruntime/core/providers/cpu/nn/unpool.h | 68 ++++ onnxruntime/test/onnx/main.cc | 2 - .../test/providers/cpu/nn/unpool_op_test.cc | 382 ++++++++++++++++++ .../test/python/onnx_backend_test_series.py | 2 - 6 files changed, 652 insertions(+), 4 deletions(-) create mode 100644 onnxruntime/core/providers/cpu/nn/Unpool.cc create mode 100644 onnxruntime/core/providers/cpu/nn/unpool.h create mode 100644 onnxruntime/test/providers/cpu/nn/unpool_op_test.cc diff --git a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc index 0a387a92fbcb5..f992316cd3108 100644 --- a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc +++ b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc @@ -196,6 +196,7 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, Eye class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, float, IsNaN); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, MLFloat16, IsNaN); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, Erf); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, MaxUnpool); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, Sinh); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, Cosh); @@ -386,6 +387,7 @@ void RegisterOnnxOperatorKernels(std::function fn) { fn(BuildKernel()); fn(BuildKernel()); fn(BuildKernel()); + fn(BuildKernel()); fn(BuildKernel()); fn(BuildKernel()); } diff --git a/onnxruntime/core/providers/cpu/nn/Unpool.cc b/onnxruntime/core/providers/cpu/nn/Unpool.cc new file mode 100644 index 0000000000000..391326dbb2d61 --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/Unpool.cc @@ -0,0 +1,200 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// disable warning because std::copy is used by Sliceiterator +// std::copy_n is not an option for raw pointer destinations as used by gsl::copy. +#ifdef _MSC_VER +#pragma warning(disable : 4996) +#endif +#include "core/providers/cpu/nn/unpool.h" +#include "core/providers/cpu/tensor/utils.h" +#include + +using namespace ::onnxruntime::common; + +namespace onnxruntime { + +ONNX_CPU_OPERATOR_KERNEL( + MaxUnpool, + 9, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::GetTensorType()) + .TypeConstraint("I", DataTypeImpl::GetTensorType()) + .TypeConstraint("Y", DataTypeImpl::GetTensorType()), + MaxUnpool); + +Status MaxUnpool::Compute(OpKernelContext* context) const { + // Get pooled values tensor + const Tensor* X = context->Input(0); + const TensorShape& X_shape = X->Shape(); + const float* X_data = X->template Data(); + + ORT_RETURN_IF_NOT(X_shape.NumDimensions() >= 3, "Input dimension cannot be less than 3."); + + // Supported sizes check + size_t pooling_dims = X_shape.NumDimensions() - 2; + if (pooling_dims > 3) { + return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Unsupported pooling size."); + } + + // Get pooled index tensor + const Tensor* I = context->Input(1); + const TensorShape& I_shape = I->Shape(); + const int64_t* I_data = I->template Data(); + + ORT_RETURN_IF_NOT(I_shape == X_shape, "Index tensor shape should be same as that of the input data tensor to unpool."); + + // Calculate output tensor shape from attributes + std::vector inferredOutputShape(X_shape.NumDimensions()); + + // Copy batch and channel dims + inferredOutputShape[0] = X_shape[0]; + inferredOutputShape[1] = X_shape[1]; + + // For feature dims calculate reversing the formula used for Maxpool + for (auto dim = 0; dim < kernel_shape_.size(); ++dim) { + inferredOutputShape[dim + 2] = (X_shape[dim + 2] - 1) * strides_[dim] - (pads_[dim + 2] + pads_[kernel_shape_.size() + dim + 4]) + kernel_shape_[dim]; + } + + // If outputshape is provided use that to infer additional padding. + std::vector inferredPads; + std::vector givenOutputShape; + bool padsInferred = false; + + if (num_inputs_ == 3) { + auto& tensor_shape = *context->Input(2); + ORT_RETURN_IF_NOT(tensor_shape.Shape().GetDims().size() == 1, "Shape must be 1 dimensional as it's tensor data is a shape"); + + // Turn the shape tensor data into an actual shape + const int64_t* p_shape = tensor_shape.template Data(); + std::vector shape{p_shape, p_shape + tensor_shape.Shape().Size()}; + givenOutputShape = shape; + + inferredPads.resize(inferredOutputShape.size() * 2, 0); + + // calculate if output shape has any padding over the inferred shape for feature dims. + for (auto dim = 2; dim < shape.size(); dim++) { + ORT_RETURN_IF_NOT(inferredOutputShape[dim] <= shape[dim], "Incorrect output shape"); + + int64_t inferredPad = shape[dim] - inferredOutputShape[dim]; + ORT_RETURN_IF_NOT(inferredPad <= kernel_shape_[dim - 2], "Incorrect output shape"); + + if (inferredPad > 0) { + padsInferred = true; + if (inferredPad == kernel_shape_[dim - 2]) { + inferredPads[dim] = 1; + inferredPads[dim + inferredOutputShape.size()] = inferredPad - 1; + } else { + inferredPads[dim + inferredOutputShape.size()] = inferredPad; + } + } + } + } + + // unpool + int64_t totalPooledElem = 1; + int64_t totalOutputElem = 1; + + for (auto dim = 0; dim < X_shape.NumDimensions(); dim++) { + totalPooledElem *= X_shape[dim]; + totalOutputElem *= inferredOutputShape[dim]; + } + + // if there are no pads inferred from outputshape simply create the new unpooled tensor + if (!padsInferred) { + TensorShape shape(inferredOutputShape); + + Tensor* Y = context->Output(0, shape); + auto Y_data = Y->template MutableData(); + auto out = gsl::make_span(Y_data, Y->Shape().Size()); + std::fill_n(out.data(), out.size(), 0.f); + + for (auto curElem = 0; curElem < totalPooledElem; ++curElem) { + out[I_data[curElem]] = X_data[curElem]; + } + } else { + // If the output shape has pads over the inferred dims , first + // create the tensor with the inferred dims and add the padding. + + // Generate tensor with inferred dims. + TensorShape shape(inferredOutputShape); + + AllocatorPtr alloc; + ORT_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&alloc)); + auto element_type = DataTypeImpl::GetType(); + + void* buffer = alloc->Alloc(sizeof(float) * shape.Size()); + std::unique_ptr p_tensor = std::make_unique(element_type, + shape, + buffer, + alloc->Info(), + alloc); + + float* p = p_tensor->template MutableData(); + + auto out = gsl::make_span(p, p_tensor->Shape().Size()); + std::fill_n(out.data(), out.size(), 0.f); + + for (auto curElem = 0; curElem < totalPooledElem; ++curElem) { + out[I_data[curElem]] = X_data[curElem]; + } + + std::vector output_dims(inferredOutputShape); + size_t dimension_count = output_dims.size(); + + std::vector input_starts; + std::vector input_extents; + + // Calculate output dimensions + for (size_t i = 0; i < dimension_count; i++) { + input_starts.push_back(slices_[i]); + input_extents.push_back(output_dims[i] + slices_[i] + slices_[i + dimension_count]); + output_dims[i] += inferredPads[i] + inferredPads[i + dimension_count] + slices_[i] + slices_[i + dimension_count]; + } + + // setup output object + TensorShape output_shape(givenOutputShape); + Tensor* Y = context->Output(0, output_shape); + auto Y_data = Y->template MutableData(); + + auto outData = gsl::make_span(Y_data, Y->Shape().Size()); + + std::fill_n(outData.data(), outData.size(), 0.f); + + // add padding + TensorPitches output_pitches(*Y); + size_t alignSkip = 0; // Amount to skip to align to where the next input tensor data needs to be written + + // Initial skip, sum up the begin padding on each axis + for (size_t i = 0; i < dimension_count; i++) + alignSkip += inferredPads[i] * output_pitches[i]; + + size_t inner_axis = dimension_count - 1; + + TensorAxisCounters input_counters(*p_tensor); + SliceIterator input(*p_tensor, input_starts, input_extents); + + while (input_counters) { + Y_data += alignSkip; + { + Y_data = input.CopyInnermostAxis(Y_data); + int64_t prePad = inferredPads[inner_axis]; + int64_t postPad = inferredPads[inner_axis + dimension_count]; + Y_data += postPad; + alignSkip = prePad; + } + // Calculate the size of the next block of padding (skipping over the innermost axis since that's already done) + while (input_counters.Increment()) { + ptrdiff_t inner_pitch = output_pitches[input_counters.Axis()]; + int64_t prePad = inferredPads[input_counters.Axis()]; + int64_t postPad = inferredPads[input_counters.Axis() + dimension_count]; + Y_data += inner_pitch * postPad; + alignSkip += inner_pitch * prePad; + } + } + } + + return Status::OK(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/nn/unpool.h b/onnxruntime/core/providers/cpu/nn/unpool.h new file mode 100644 index 0000000000000..012c1da30d140 --- /dev/null +++ b/onnxruntime/core/providers/cpu/nn/unpool.h @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include "core/common/common.h" +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/nn/autopad_type.h" + +namespace onnxruntime { + +class MaxUnpool : public OpKernel { + public: + MaxUnpool(const OpKernelInfo& info) : OpKernel(info) { + ORT_ENFORCE(info.GetAttrs("kernel_shape", kernel_shape_).IsOK(), + "No kernel shape is set."); + + num_inputs_ = OpKernel::Node().InputDefs().size(); + + if (num_inputs_ == 3 && !pads_.empty()) { + // ignore pads attribute value + } + + // setup defaults. + if (!info.GetAttrs("pads", pads_).IsOK() || pads_.empty()) { + pads_.resize(kernel_shape_.size() * 2, 0); + } + + if (!info.GetAttrs("strides", strides_).IsOK() || strides_.empty()) { + strides_.resize(kernel_shape_.size(), 1); + } + + for (size_t dim = 0; dim < kernel_shape_.size(); ++dim) { + ORT_ENFORCE(kernel_shape_[dim] > 0); + ORT_ENFORCE(pads_[dim] < kernel_shape_[dim] && pads_[dim + kernel_shape_.size()] < kernel_shape_[dim], + "Pad should be smaller than kernel."); + } + + ORT_ENFORCE(strides_.size() == kernel_shape_.size()); + + // Add 4 pad values (0) for batch and channel dimensions + pads_.insert(pads_.begin(), {0, 0}); + pads_.insert(pads_.begin() + 2 + kernel_shape_.size(), {0, 0}); + + // Separate out any negative pads_ into the slices_ array + slices_.resize(pads_.size(), 0); + for (size_t index = 0; index < pads_.size(); index++) { + if (pads_[index] < 0) { + slices_[index] = pads_[index]; + pads_[index] = 0; + } + } + } + + ~MaxUnpool() override{}; + + Status Compute(OpKernelContext* context) const override; + + private: + std::vector kernel_shape_; + std::vector pads_; + std::vector strides_; + std::vector slices_; // All of the negative padding values are separated out into slices_ + int64_t num_inputs_; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index 966f674ed90d2..93a64012db583 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -309,8 +309,6 @@ int real_main(int argc, char* argv[]) { {"operator_rnn_single_layer", "disable reason"}, {"prelu_broadcast", "disable reason"}, {"prelu_example", "disable reason"}, - {"maxunpool_export_with_output_shape", "opset 9 not supported yet"}, - {"maxunpool_export_without_output_shape", "opset 9 not supported yet"}, {"upsample_nearest", "opset 9 not supported yet"}, {"onehot_with_axis", "opset 9 not supported yet"}, {"onehot_without_axis", "opset 9 not supported yet"}, // also has bug in current test re: output type. Spandan to fix. diff --git a/onnxruntime/test/providers/cpu/nn/unpool_op_test.cc b/onnxruntime/test/providers/cpu/nn/unpool_op_test.cc new file mode 100644 index 0000000000000..fd9b20ba9c3e9 --- /dev/null +++ b/onnxruntime/test/providers/cpu/nn/unpool_op_test.cc @@ -0,0 +1,382 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +using namespace std; +namespace onnxruntime { +namespace test { + +TEST(UnpoolTest, MaxUnPool1D) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2}); + test.AddAttribute("kernel_shape", vector{2}); + + std::vector t_vals = {1, 2, 3, 4}; + std::vector t_dims = {1, 1, 4}; + + std::vector i_vals = {1, 3, 4, 6}; + std::vector i_dims = {1, 1, 4}; + + std::vector expected_dims = {1, 1, 8}; + std::vector expected_vals = {0, 1, 0, 2, 3, 0, 4, 0}; + + std::vector inputDims = {3}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + test.AddInput("output_shape", inputDims, expected_dims); + + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(UnpoolTest, MaxUnPool2D) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2, 2}); + test.AddAttribute("kernel_shape", std::vector{2, 2}); + + std::vector t_vals = {1, 2, 3, 4}; + std::vector t_dims = {1, 1, 2, 2}; + + std::vector i_vals = {1, 3, 4, 6}; + std::vector i_dims = {1, 1, 2, 2}; + + std::vector expected_dims = {1, 1, 4, 4}; + std::vector expected_vals = {0, 1, 0, 2, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + + std::vector inputDims = {4}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + test.AddInput("output_shape", inputDims, expected_dims); + + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(UnpoolTest, MaxUnPool3D) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2, 2, 2}); + test.AddAttribute("kernel_shape", vector{2, 2, 2}); + + std::vector t_vals = {1, 2, 3, 4, 5, 6, 7, 8}; + std::vector t_dims = {1, 1, 2, 2, 2}; + + std::vector i_vals = {1, 3, 24, 30, 32, 38, 60, 62}; + std::vector i_dims = {1, 1, 2, 2, 2}; + + std::vector expected_dims = {1, 1, 4, 4, 4}; + std::vector expectedDims_Size = {5}; + + std::vector expected_vals = + { + //slice 1 + 0, 1, 0, 2, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + + // slice 2 + 0, 0, 0, 0, + 0, 0, 0, 0, + 3, 0, 0, 0, + 0, 0, 4, 0, + + //slice 3 + 5, 0, 0, 0, + 0, 0, 6, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + + // slice 4 + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 7, 0, 8, 0}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + test.AddInput("output_shape", expectedDims_Size, expected_dims); + + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(UnpoolTest, MaxUnPool1D_Without_OutputShape) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2}); + test.AddAttribute("kernel_shape", vector{2}); + + std::vector t_vals = {1, 2, 3, 4}; + std::vector t_dims = {1, 1, 4}; + + std::vector i_vals = {1, 3, 4, 6}; + std::vector i_dims = {1, 1, 4}; + + std::vector expected_dims = {1, 1, 8}; + std::vector expected_vals = {0, 1, 0, 2, 3, 0, 4, 0}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(UnpoolTest, MaxUnPool2D_Without_OutputShape) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2, 2}); + test.AddAttribute("kernel_shape", vector{2, 2}); + + std::vector t_vals = {1, 2, 3, 4}; + std::vector t_dims = {1, 1, 2, 2}; + + std::vector i_vals = {1, 3, 4, 6}; + std::vector i_dims = {1, 1, 2, 2}; + + std::vector expected_dims = {1, 1, 4, 4}; + std::vector expected_vals = {0, 1, 0, 2, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(UnpoolTest, MaxUnPool3D_Without_OutputShape) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2, 2, 2}); + test.AddAttribute("kernel_shape", vector{2, 2, 2}); + + std::vector t_vals = {1, 2, 3, 4, 5, 6, 7, 8}; + std::vector t_dims = {1, 1, 2, 2, 2}; + + std::vector i_vals = {1, 3, 24, 30, 32, 38, 60, 62}; + std::vector i_dims = {1, 1, 2, 2, 2}; + + std::vector expected_dims = {1, 1, 4, 4, 4}; + + std::vector expected_vals = + { + //slice 1 + 0, 1, 0, 2, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + + // slice 2 + 0, 0, 0, 0, + 0, 0, 0, 0, + 3, 0, 0, 0, + 0, 0, 4, 0, + + //slice 3 + 5, 0, 0, 0, + 0, 0, 6, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + + // slice 4 + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 7, 0, 8, 0}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(UnpoolTest, MaxUnPool1D_Padding) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2}); + test.AddAttribute("kernel_shape", vector{2}); + test.AddAttribute("pads", vector{1, 0}); + + std::vector t_vals = {1, 2, 3, 4}; + std::vector t_dims = {1, 1, 4}; + + std::vector i_vals = {1, 3, 4, 6}; + std::vector i_dims = {1, 1, 4}; + + std::vector expected_dims = {1, 1, 7}; + std::vector expected_vals = {0, 1, 0, 2, 3, 0, 4}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + + test.AddOutput("YP", expected_dims, expected_vals); + test.Run(); +} + +TEST(UnpoolTest, MaxUnPool2D_Padding) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2, 2}); + test.AddAttribute("kernel_shape", vector{2, 2}); + test.AddAttribute("pads", vector{1, 1, 0, 0}); + + std::vector t_vals = {1, 2, 3, 4}; + std::vector t_dims = {1, 1, 2, 2}; + + std::vector i_vals = {1, 3, 4, 6}; + std::vector i_dims = {1, 1, 2, 2}; + + std::vector expected_dims = {1, 1, 3, 3}; + std::vector expected_vals = {0, 1, 0, 2, 3, 0, 4, 0, 0}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(UnpoolTest, MaxUnPool3D_Padding) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2, 2, 2}); + test.AddAttribute("kernel_shape", vector{2, 2, 2}); + test.AddAttribute("pads", vector{0, 1, 1, 0, 0, 0}); + + std::vector t_vals = {1, 2, 3, 4}; + std::vector t_dims = {1, 1, 1, 2, 2}; + + std::vector i_vals = {1, 4, 8, 12}; + std::vector i_dims = {1, 1, 1, 2, 2}; + + std::vector expected_dims = {1, 1, 2, 3, 3}; + + std::vector expected_vals = { + 0, 1, 0, + 0, 2, 0, + 0, 0, 3, + 0, 0, 0, + 4, 0, 0, + 0, 0, 0}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(UnpoolTest, MaxUnPool1D_WithPaddedOutput) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2}); + test.AddAttribute("kernel_shape", vector{2}); + + std::vector t_vals = {1, 2, 3, 4}; + std::vector t_dims = {1, 1, 4}; + + std::vector i_vals = {1, 3, 4, 6}; + std::vector i_dims = {1, 1, 4}; + + std::vector expected_dims = {1, 1, 10}; + std::vector expected_vals = {0, 0, 1, 0, 2, 3, 0, 4, 0, 0}; + + std::vector inputDims = {3}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + test.AddInput("output_shape", inputDims, expected_dims); + + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(UnpoolTest, MaxUnPool2D_WithPaddedOutput) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2, 2}); + test.AddAttribute("kernel_shape", std::vector{2, 2}); + + std::vector t_vals = {1, 2, 3, 4}; + std::vector t_dims = {1, 1, 2, 2}; + + std::vector i_vals = {1, 3, 8, 10}; + std::vector i_dims = {1, 1, 2, 2}; + + std::vector expected_dims = {1, 1, 5, 5}; + std::vector expected_vals = { + 0, 1, 0, 2, 0, + 0, 0, 0, 0, 0, + 3, 0, 4, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0}; + + std::vector inputDims = {4}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + test.AddInput("output_shape", inputDims, expected_dims); + + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +TEST(UnpoolTest, MaxUnPool3D_WithPaddedOutput) { + OpTester test("MaxUnpool", 9); + + test.AddAttribute("strides", std::vector{2, 2, 2}); + test.AddAttribute("kernel_shape", vector{2, 2, 2}); + + std::vector t_vals = {1, 2, 3, 4, 5, 6, 7, 8}; + std::vector t_dims = {1, 1, 2, 2, 2}; + + std::vector i_vals = {1, 3, 24, 30, 32, 38, 60, 62}; + std::vector i_dims = {1, 1, 2, 2, 2}; + + std::vector expected_dims = {1, 1, 4, 4, 5}; + std::vector expectedDims_Size = {5}; + + std::vector expected_vals = + { + //slice 1 + 0, 1, 0, 2, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + + // slice 2 + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 3, 0, 0, 0, 0, + 0, 0, 4, 0, 0, + + //slice 3 + 5, 0, 0, 0, 0, + 0, 0, 6, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + + // slice 4 + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 7, 0, 8, 0, 0}; + + test.AddInput("xT", t_dims, t_vals); + test.AddInput("xI", i_dims, i_vals); + test.AddInput("output_shape", expectedDims_Size, expected_dims); + + test.AddOutput("Y", expected_dims, expected_vals); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/python/onnx_backend_test_series.py b/onnxruntime/test/python/onnx_backend_test_series.py index 5410c4dbcacc5..ea6ea135bcd20 100644 --- a/onnxruntime/test/python/onnx_backend_test_series.py +++ b/onnxruntime/test/python/onnx_backend_test_series.py @@ -34,8 +34,6 @@ '|test_eyelike_with_dtype_cpu.*' '|test_eyelike_without_dtype_cpu.*' '|test_gru_seq_length_cpu.*' -'|test_maxunpool_export_with_output_shape_cpu.*' -'|test_maxunpool_export_without_output_shape_cpu.*' '|test_onehot_with_axis_cpu.*' '|test_onehot_without_axis_cpu.*' '|test_scan_sum_cpu.*' From a43382e390dd9244cc5e46f308ff19d9e4aa311c Mon Sep 17 00:00:00 2001 From: jignparm Date: Thu, 20 Dec 2018 09:58:03 -0800 Subject: [PATCH 38/56] Jignparm/csharp gpu (#221) * Minor updates to exception message * update models folder to new location * update copy to preservenewest * reenable pretrained test * added some debugging info for build * update pretrained test, and tensor proto definition --- .../InferenceTest.cs | 29 +++-- .../Microsoft.ML.OnnxRuntime.Tests/OnnxMl.cs | 112 +++++++++--------- 2 files changed, 73 insertions(+), 68 deletions(-) diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs index 8044665ab751a..256792f7343da 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs @@ -205,40 +205,43 @@ private void TestMultiThreads() session.Dispose(); } - [Fact(Skip = "Disable temporarily")] + [Fact] private void TestPreTrainedModelsOpset7And8() { var opsets = new[] { "opset7", "opset8" }; foreach (var opset in opsets) { var modelRoot = new DirectoryInfo(opset); - foreach (var model in modelRoot.EnumerateDirectories()) + foreach (var modelDir in modelRoot.EnumerateDirectories()) { // TODO: dims contains 'None'. Session throws error. - if (model.ToString() == "test_tiny_yolov2") + if (modelDir.Name== "test_tiny_yolov2") continue; + + String onnxModelFileName = null; try { - var modelNames = model.GetFiles("*.onnx"); - if (modelNames.Count() != 1) + var onnxModelNames = modelDir.GetFiles("*.onnx"); + if (onnxModelNames.Count() != 1) { // TODO remove file "._resnet34v2.onnx" from test set - if (modelNames[0].ToString() == "._resnet34v2.onnx") - modelNames[0] = modelNames[1]; + if (onnxModelNames[0].Name == "._resnet34v2.onnx") + onnxModelNames[0] = onnxModelNames[1]; else { - var modelNamesList = string.Join(",", modelNames.Select(x => x.ToString())); - throw new Exception($"Opset {opset}: Model {model}. Can't determine model file name. Found these :{modelNamesList}"); + var modelNamesList = string.Join(",", onnxModelNames.Select(x => x.ToString())); + throw new Exception($"Opset {opset}: Model {modelDir}. Can't determine model file name. Found these :{modelNamesList}"); } } - var session = new InferenceSession($"{opset}\\{model}\\{modelNames[0].ToString()}"); + onnxModelFileName = $"{opset}\\{modelDir.Name}\\{onnxModelNames[0].Name}"; + var session = new InferenceSession(onnxModelFileName); var inMeta = session.InputMetadata; var innodepair = inMeta.First(); var innodename = innodepair.Key; var innodedims = innodepair.Value.Dimensions; - var dataIn = LoadTensorFromFilePb($"{opset}\\{model}\\test_data_set_0\\input_0.pb"); - var dataOut = LoadTensorFromFilePb($"{opset}\\{model}\\test_data_set_0\\output_0.pb"); + var dataIn = LoadTensorFromFilePb($"{opset}\\{modelDir.Name}\\test_data_set_0\\input_0.pb"); + var dataOut = LoadTensorFromFilePb($"{opset}\\{modelDir.Name}\\test_data_set_0\\output_0.pb"); var tensorIn = new DenseTensor(dataIn, innodedims); var nov = new List(); nov.Add(NamedOnnxValue.CreateFromTensor(innodename, tensorIn)); @@ -249,7 +252,7 @@ private void TestPreTrainedModelsOpset7And8() } catch (Exception ex) { - var msg = $"Opset {opset}: Model {model}: error = {ex.Message}"; + var msg = $"Opset {opset}: Model {modelDir}: ModelFile = {onnxModelFileName} error = {ex.Message}"; throw new Exception(msg); } } //model diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/OnnxMl.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/OnnxMl.cs index baf8f0e80e216..209e62c30a5f5 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/OnnxMl.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/OnnxMl.cs @@ -53,41 +53,39 @@ static OnnxMlReflection() { "b3RvEhIKCmRvY19zdHJpbmcYCiABKAkSIwoFaW5wdXQYCyADKAsyFC5vbm54", "LlZhbHVlSW5mb1Byb3RvEiQKBm91dHB1dBgMIAMoCzIULm9ubnguVmFsdWVJ", "bmZvUHJvdG8SKAoKdmFsdWVfaW5mbxgNIAMoCzIULm9ubnguVmFsdWVJbmZv", - "UHJvdG8ivQQKC1RlbnNvclByb3RvEgwKBGRpbXMYASADKAMSLQoJZGF0YV90", - "eXBlGAIgASgOMhoub25ueC5UZW5zb3JQcm90by5EYXRhVHlwZRIqCgdzZWdt", - "ZW50GAMgASgLMhkub25ueC5UZW5zb3JQcm90by5TZWdtZW50EhYKCmZsb2F0", - "X2RhdGEYBCADKAJCAhABEhYKCmludDMyX2RhdGEYBSADKAVCAhABEhMKC3N0", - "cmluZ19kYXRhGAYgAygMEhYKCmludDY0X2RhdGEYByADKANCAhABEgwKBG5h", - "bWUYCCABKAkSEgoKZG9jX3N0cmluZxgMIAEoCRIQCghyYXdfZGF0YRgJIAEo", - "DBIXCgtkb3VibGVfZGF0YRgKIAMoAUICEAESFwoLdWludDY0X2RhdGEYCyAD", - "KARCAhABGiUKB1NlZ21lbnQSDQoFYmVnaW4YASABKAMSCwoDZW5kGAIgASgD", - "ItoBCghEYXRhVHlwZRINCglVTkRFRklORUQQABIJCgVGTE9BVBABEgkKBVVJ", - "TlQ4EAISCAoESU5UOBADEgoKBlVJTlQxNhAEEgkKBUlOVDE2EAUSCQoFSU5U", - "MzIQBhIJCgVJTlQ2NBAHEgoKBlNUUklORxAIEggKBEJPT0wQCRILCgdGTE9B", - "VDE2EAoSCgoGRE9VQkxFEAsSCgoGVUlOVDMyEAwSCgoGVUlOVDY0EA0SDQoJ", - "Q09NUExFWDY0EA4SDgoKQ09NUExFWDEyOBAPEgwKCEJGTE9BVDE2EBAilQEK", - "EFRlbnNvclNoYXBlUHJvdG8SLQoDZGltGAEgAygLMiAub25ueC5UZW5zb3JT", - "aGFwZVByb3RvLkRpbWVuc2lvbhpSCglEaW1lbnNpb24SEwoJZGltX3ZhbHVl", - "GAEgASgDSAASEwoJZGltX3BhcmFtGAIgASgJSAASEgoKZGVub3RhdGlvbhgD", - "IAEoCUIHCgV2YWx1ZSKWBQoJVHlwZVByb3RvEi0KC3RlbnNvcl90eXBlGAEg", - "ASgLMhYub25ueC5UeXBlUHJvdG8uVGVuc29ySAASMQoNc2VxdWVuY2VfdHlw", - "ZRgEIAEoCzIYLm9ubnguVHlwZVByb3RvLlNlcXVlbmNlSAASJwoIbWFwX3R5", - "cGUYBSABKAsyEy5vbm54LlR5cGVQcm90by5NYXBIABItCgtvcGFxdWVfdHlw", - "ZRgHIAEoCzIWLm9ubnguVHlwZVByb3RvLk9wYXF1ZUgAEjoKEnNwYXJzZV90", - "ZW5zb3JfdHlwZRgIIAEoCzIcLm9ubnguVHlwZVByb3RvLlNwYXJzZVRlbnNv", - "ckgAEhIKCmRlbm90YXRpb24YBiABKAkaXgoGVGVuc29yEi0KCWVsZW1fdHlw", - "ZRgBIAEoDjIaLm9ubnguVGVuc29yUHJvdG8uRGF0YVR5cGUSJQoFc2hhcGUY", - "AiABKAsyFi5vbm54LlRlbnNvclNoYXBlUHJvdG8aLgoIU2VxdWVuY2USIgoJ", - "ZWxlbV90eXBlGAEgASgLMg8ub25ueC5UeXBlUHJvdG8aWAoDTWFwEiwKCGtl", - "eV90eXBlGAEgASgOMhoub25ueC5UZW5zb3JQcm90by5EYXRhVHlwZRIjCgp2", - "YWx1ZV90eXBlGAIgASgLMg8ub25ueC5UeXBlUHJvdG8aJgoGT3BhcXVlEg4K", - "BmRvbWFpbhgBIAEoCRIMCgRuYW1lGAIgASgJGmQKDFNwYXJzZVRlbnNvchIt", - "CgllbGVtX3R5cGUYASABKA4yGi5vbm54LlRlbnNvclByb3RvLkRhdGFUeXBl", - "EiUKBXNoYXBlGAIgASgLMhYub25ueC5UZW5zb3JTaGFwZVByb3RvQgcKBXZh", - "bHVlIjUKEk9wZXJhdG9yU2V0SWRQcm90bxIOCgZkb21haW4YASABKAkSDwoH", - "dmVyc2lvbhgCIAEoAypjCgdWZXJzaW9uEhIKDl9TVEFSVF9WRVJTSU9OEAAS", - "GQoVSVJfVkVSU0lPTl8yMDE3XzEwXzEwEAESGQoVSVJfVkVSU0lPTl8yMDE3", - "XzEwXzMwEAISDgoKSVJfVkVSU0lPThADYgZwcm90bzM=")); + "UHJvdG8ioQQKC1RlbnNvclByb3RvEgwKBGRpbXMYASADKAMSEQoJZGF0YV90", + "eXBlGAIgASgFEioKB3NlZ21lbnQYAyABKAsyGS5vbm54LlRlbnNvclByb3Rv", + "LlNlZ21lbnQSFgoKZmxvYXRfZGF0YRgEIAMoAkICEAESFgoKaW50MzJfZGF0", + "YRgFIAMoBUICEAESEwoLc3RyaW5nX2RhdGEYBiADKAwSFgoKaW50NjRfZGF0", + "YRgHIAMoA0ICEAESDAoEbmFtZRgIIAEoCRISCgpkb2Nfc3RyaW5nGAwgASgJ", + "EhAKCHJhd19kYXRhGAkgASgMEhcKC2RvdWJsZV9kYXRhGAogAygBQgIQARIX", + "Cgt1aW50NjRfZGF0YRgLIAMoBEICEAEaJQoHU2VnbWVudBINCgViZWdpbhgB", + "IAEoAxILCgNlbmQYAiABKAMi2gEKCERhdGFUeXBlEg0KCVVOREVGSU5FRBAA", + "EgkKBUZMT0FUEAESCQoFVUlOVDgQAhIICgRJTlQ4EAMSCgoGVUlOVDE2EAQS", + "CQoFSU5UMTYQBRIJCgVJTlQzMhAGEgkKBUlOVDY0EAcSCgoGU1RSSU5HEAgS", + "CAoEQk9PTBAJEgsKB0ZMT0FUMTYQChIKCgZET1VCTEUQCxIKCgZVSU5UMzIQ", + "DBIKCgZVSU5UNjQQDRINCglDT01QTEVYNjQQDhIOCgpDT01QTEVYMTI4EA8S", + "DAoIQkZMT0FUMTYQECKVAQoQVGVuc29yU2hhcGVQcm90bxItCgNkaW0YASAD", + "KAsyIC5vbm54LlRlbnNvclNoYXBlUHJvdG8uRGltZW5zaW9uGlIKCURpbWVu", + "c2lvbhITCglkaW1fdmFsdWUYASABKANIABITCglkaW1fcGFyYW0YAiABKAlI", + "ABISCgpkZW5vdGF0aW9uGAMgASgJQgcKBXZhbHVlIsIECglUeXBlUHJvdG8S", + "LQoLdGVuc29yX3R5cGUYASABKAsyFi5vbm54LlR5cGVQcm90by5UZW5zb3JI", + "ABIxCg1zZXF1ZW5jZV90eXBlGAQgASgLMhgub25ueC5UeXBlUHJvdG8uU2Vx", + "dWVuY2VIABInCghtYXBfdHlwZRgFIAEoCzITLm9ubnguVHlwZVByb3RvLk1h", + "cEgAEi0KC29wYXF1ZV90eXBlGAcgASgLMhYub25ueC5UeXBlUHJvdG8uT3Bh", + "cXVlSAASOgoSc3BhcnNlX3RlbnNvcl90eXBlGAggASgLMhwub25ueC5UeXBl", + "UHJvdG8uU3BhcnNlVGVuc29ySAASEgoKZGVub3RhdGlvbhgGIAEoCRpCCgZU", + "ZW5zb3ISEQoJZWxlbV90eXBlGAEgASgFEiUKBXNoYXBlGAIgASgLMhYub25u", + "eC5UZW5zb3JTaGFwZVByb3RvGi4KCFNlcXVlbmNlEiIKCWVsZW1fdHlwZRgB", + "IAEoCzIPLm9ubnguVHlwZVByb3RvGjwKA01hcBIQCghrZXlfdHlwZRgBIAEo", + "BRIjCgp2YWx1ZV90eXBlGAIgASgLMg8ub25ueC5UeXBlUHJvdG8aJgoGT3Bh", + "cXVlEg4KBmRvbWFpbhgBIAEoCRIMCgRuYW1lGAIgASgJGkgKDFNwYXJzZVRl", + "bnNvchIRCgllbGVtX3R5cGUYASABKAUSJQoFc2hhcGUYAiABKAsyFi5vbm54", + "LlRlbnNvclNoYXBlUHJvdG9CBwoFdmFsdWUiNQoST3BlcmF0b3JTZXRJZFBy", + "b3RvEg4KBmRvbWFpbhgBIAEoCRIPCgd2ZXJzaW9uGAIgASgDKmMKB1ZlcnNp", + "b24SEgoOX1NUQVJUX1ZFUlNJT04QABIZChVJUl9WRVJTSU9OXzIwMTdfMTBf", + "MTAQARIZChVJUl9WRVJTSU9OXzIwMTdfMTBfMzAQAhIOCgpJUl9WRVJTSU9O", + "EANiBnByb3RvMw==")); descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, new pbr::FileDescriptor[] { }, new pbr::GeneratedClrTypeInfo(new[] {typeof(global::Onnx.Version), }, new pbr::GeneratedClrTypeInfo[] { @@ -2116,12 +2114,13 @@ public TensorProto Clone() { /// Field number for the "data_type" field. public const int DataTypeFieldNumber = 2; - private global::Onnx.TensorProto.Types.DataType dataType_ = 0; + private int dataType_; /// /// The data type of the tensor. + /// This field MUST have a valid TensorProto.DataType value /// [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - public global::Onnx.TensorProto.Types.DataType DataType { + public int DataType { get { return dataType_; } set { dataType_ = value; @@ -2355,7 +2354,7 @@ public void WriteTo(pb::CodedOutputStream output) { dims_.WriteTo(output, _repeated_dims_codec); if (DataType != 0) { output.WriteRawTag(16); - output.WriteEnum((int) DataType); + output.WriteInt32(DataType); } if (segment_ != null) { output.WriteRawTag(26); @@ -2389,7 +2388,7 @@ public int CalculateSize() { int size = 0; size += dims_.CalculateSize(_repeated_dims_codec); if (DataType != 0) { - size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) DataType); + size += 1 + pb::CodedOutputStream.ComputeInt32Size(DataType); } if (segment_ != null) { size += 1 + pb::CodedOutputStream.ComputeMessageSize(Segment); @@ -2462,7 +2461,7 @@ public void MergeFrom(pb::CodedInputStream input) { break; } case 16: { - dataType_ = (global::Onnx.TensorProto.Types.DataType) input.ReadEnum(); + DataType = input.ReadInt32(); break; } case 26: { @@ -3517,13 +3516,14 @@ public Tensor Clone() { /// Field number for the "elem_type" field. public const int ElemTypeFieldNumber = 1; - private global::Onnx.TensorProto.Types.DataType elemType_ = 0; + private int elemType_; /// /// This field MUST NOT have the value of UNDEFINED + /// This field MUST have a valid TensorProto.DataType value /// This field MUST be present for this version of the IR. /// [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - public global::Onnx.TensorProto.Types.DataType ElemType { + public int ElemType { get { return elemType_; } set { elemType_ = value; @@ -3579,7 +3579,7 @@ public override string ToString() { public void WriteTo(pb::CodedOutputStream output) { if (ElemType != 0) { output.WriteRawTag(8); - output.WriteEnum((int) ElemType); + output.WriteInt32(ElemType); } if (shape_ != null) { output.WriteRawTag(18); @@ -3594,7 +3594,7 @@ public void WriteTo(pb::CodedOutputStream output) { public int CalculateSize() { int size = 0; if (ElemType != 0) { - size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) ElemType); + size += 1 + pb::CodedOutputStream.ComputeInt32Size(ElemType); } if (shape_ != null) { size += 1 + pb::CodedOutputStream.ComputeMessageSize(Shape); @@ -3631,7 +3631,7 @@ public void MergeFrom(pb::CodedInputStream input) { _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); break; case 8: { - elemType_ = (global::Onnx.TensorProto.Types.DataType) input.ReadEnum(); + ElemType = input.ReadInt32(); break; } case 18: { @@ -3829,13 +3829,14 @@ public Map Clone() { /// Field number for the "key_type" field. public const int KeyTypeFieldNumber = 1; - private global::Onnx.TensorProto.Types.DataType keyType_ = 0; + private int keyType_; /// + /// This field MUST have a valid TensorProto.DataType value /// This field MUST be present for this version of the IR. /// This field MUST refer to an integral type ([U]INT{8|16|32|64}) or STRING /// [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - public global::Onnx.TensorProto.Types.DataType KeyType { + public int KeyType { get { return keyType_; } set { keyType_ = value; @@ -3894,7 +3895,7 @@ public override string ToString() { public void WriteTo(pb::CodedOutputStream output) { if (KeyType != 0) { output.WriteRawTag(8); - output.WriteEnum((int) KeyType); + output.WriteInt32(KeyType); } if (valueType_ != null) { output.WriteRawTag(18); @@ -3909,7 +3910,7 @@ public void WriteTo(pb::CodedOutputStream output) { public int CalculateSize() { int size = 0; if (KeyType != 0) { - size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) KeyType); + size += 1 + pb::CodedOutputStream.ComputeInt32Size(KeyType); } if (valueType_ != null) { size += 1 + pb::CodedOutputStream.ComputeMessageSize(ValueType); @@ -3946,7 +3947,7 @@ public void MergeFrom(pb::CodedInputStream input) { _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); break; case 8: { - keyType_ = (global::Onnx.TensorProto.Types.DataType) input.ReadEnum(); + KeyType = input.ReadInt32(); break; } case 18: { @@ -4162,13 +4163,14 @@ public SparseTensor Clone() { /// Field number for the "elem_type" field. public const int ElemTypeFieldNumber = 1; - private global::Onnx.TensorProto.Types.DataType elemType_ = 0; + private int elemType_; /// /// This field MUST NOT have the value of UNDEFINED + /// This field MUST have a valid TensorProto.DataType value /// This field MUST be present for this version of the IR. /// [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - public global::Onnx.TensorProto.Types.DataType ElemType { + public int ElemType { get { return elemType_; } set { elemType_ = value; @@ -4224,7 +4226,7 @@ public override string ToString() { public void WriteTo(pb::CodedOutputStream output) { if (ElemType != 0) { output.WriteRawTag(8); - output.WriteEnum((int) ElemType); + output.WriteInt32(ElemType); } if (shape_ != null) { output.WriteRawTag(18); @@ -4239,7 +4241,7 @@ public void WriteTo(pb::CodedOutputStream output) { public int CalculateSize() { int size = 0; if (ElemType != 0) { - size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) ElemType); + size += 1 + pb::CodedOutputStream.ComputeInt32Size(ElemType); } if (shape_ != null) { size += 1 + pb::CodedOutputStream.ComputeMessageSize(Shape); @@ -4276,7 +4278,7 @@ public void MergeFrom(pb::CodedInputStream input) { _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); break; case 8: { - elemType_ = (global::Onnx.TensorProto.Types.DataType) input.ReadEnum(); + ElemType = input.ReadInt32(); break; } case 18: { From c453b48b71ae8a858d466b2ed45827b2cc26d591 Mon Sep 17 00:00:00 2001 From: "Tang, Cheng" Date: Thu, 20 Dec 2018 11:11:50 -0800 Subject: [PATCH 39/56] update kernel memory type interface (#225) * refactor the kernel memory type interface * remove useless change * fix comments in PR --- .../core/framework/kernel_def_builder.h | 49 ++++++++++++++----- .../core/framework/allocation_planner.cc | 9 ++-- .../core/framework/kernel_def_builder.cc | 12 ++--- .../core/framework/transformer_memcpy.cc | 23 +++++---- onnxruntime/core/session/IOBinding.cc | 3 +- 5 files changed, 59 insertions(+), 37 deletions(-) diff --git a/include/onnxruntime/core/framework/kernel_def_builder.h b/include/onnxruntime/core/framework/kernel_def_builder.h index dd30683266983..0c5c46eb19af7 100644 --- a/include/onnxruntime/core/framework/kernel_def_builder.h +++ b/include/onnxruntime/core/framework/kernel_def_builder.h @@ -20,13 +20,15 @@ class KernelDefBuilder; typedef std::map MemTypeMap; // note that input/output might be on CPU implicitly when the node is from CPU execution provider -inline bool MemTypeOnCpuExplicitly(const MemTypeMap& mem_type_map, size_t index) { - auto iter = mem_type_map.find(index); - return iter != mem_type_map.end() && (iter->second == OrtMemTypeCPUInput || iter->second == OrtMemTypeCPUOutput); +inline bool MemTypeOnCpuExplicitly(OrtMemType mem_type) { + return mem_type == OrtMemTypeCPUInput || mem_type == OrtMemTypeCPUOutput; } class KernelDef { public: + explicit KernelDef() : default_inputs_mem_type_(OrtMemTypeDefault), default_outputs_mem_type_(OrtMemTypeDefault) { + } + const std::string& OpName() const { return op_name_; } @@ -56,17 +58,20 @@ class KernelDef { return alias_map_; } - const MemTypeMap& InputMemoryType() const { - return input_memory_type_args_; - } - - const MemTypeMap& OutputMemoryType() const { - return output_memory_type_args_; + OrtMemType InputMemoryType(size_t input_index) const { + auto it = input_memory_type_args_.find(input_index); + if (it == input_memory_type_args_.end()) + return default_inputs_mem_type_; + else + return it->second; } - // legacy interface for winml, should not be used in onnxruntime - const MemTypeMap& MemoryType() const { - return output_memory_type_args_; + OrtMemType OutputMemoryType(size_t output_index) const { + auto it = output_memory_type_args_.find(output_index); + if (it == output_memory_type_args_.end()) + return default_outputs_mem_type_; + else + return it->second; } int ExecQueueId() const { @@ -111,6 +116,10 @@ class KernelDef { // execution command queue id, 0 for default queue in execution provider int exec_queue_id_ = 0; + // Default memory type for all inputs + OrtMemType default_inputs_mem_type_; + // Default memory type for all outputs + OrtMemType default_outputs_mem_type_; }; class KernelDefBuilder { @@ -212,6 +221,22 @@ class KernelDefBuilder { return *this; } + /** + Specify the default inputs memory type, if not specified, it is DefaultMemory + */ + KernelDefBuilder& SetDefaultInputsMemoryType(OrtMemType mem_type) { + kernel_def_->default_inputs_mem_type_ = mem_type; + return *this; + } + + /** + Specify the default outputs memory type, if not specified, it is DefaultMemory + */ + KernelDefBuilder& SetDefaultOutputMemoryType(OrtMemType mem_type) { + kernel_def_->default_outputs_mem_type_ = mem_type; + return *this; + } + /** Return the kernel definition, passing ownership of the KernelDef to the caller */ diff --git a/onnxruntime/core/framework/allocation_planner.cc b/onnxruntime/core/framework/allocation_planner.cc index 277a6ff2ac1e2..9bc25e9303fda 100644 --- a/onnxruntime/core/framework/allocation_planner.cc +++ b/onnxruntime/core/framework/allocation_planner.cc @@ -380,7 +380,6 @@ class PlannerImpl { ORT_ENFORCE(exec_provider); auto& default_allocator_info = exec_provider->GetAllocator(0, OrtMemTypeDefault)->Info(); - auto& mem_type_allocated_args = p_kernelDef->OutputMemoryType(); auto& outputs = pnode->OutputDefs(); auto num_outputs = outputs.size(); @@ -393,11 +392,11 @@ class PlannerImpl { if (strcmp(default_allocator_info.name, CPU) != 0) { // By default, outputs of this node are allocated on the default device allocator, // except for outputs marked for allocation in MemoryType: - auto memory_type_iter = mem_type_allocated_args.find(i); - if (memory_type_iter == mem_type_allocated_args.end()) { + auto memory_type = p_kernelDef->OutputMemoryType(i); + if (memory_type == OrtMemTypeDefault) { AllocPlan(index).location = default_allocator_info; } else { - AllocPlan(index).location = exec_provider->GetAllocator(0, memory_type_iter->second)->Info(); + AllocPlan(index).location = exec_provider->GetAllocator(0, memory_type)->Info(); } } } @@ -438,7 +437,7 @@ class PlannerImpl { thisplan.alloc_kind = AllocKind::kAllocateStatically; auto p_opkernelDef = utils::GetKernelDef(kernel_registry_, node); - if (MemTypeOnCpuExplicitly(p_opkernelDef->InputMemoryType(), index)) + if (MemTypeOnCpuExplicitly(p_opkernelDef->InputMemoryType(index))) // weights are not output from any node, so it's OK to put its location on CPU provider thisplan.location = execution_providers_.Get(onnxruntime::kCpuExecutionProvider)->GetAllocator(0, OrtMemTypeDefault)->Info(); else diff --git a/onnxruntime/core/framework/kernel_def_builder.cc b/onnxruntime/core/framework/kernel_def_builder.cc index 8555adc7563de..12d0fd936897c 100644 --- a/onnxruntime/core/framework/kernel_def_builder.cc +++ b/onnxruntime/core/framework/kernel_def_builder.cc @@ -66,20 +66,20 @@ bool KernelDef::IsConflict(const KernelDef& other) const { return false; //check memory type - auto other_input_mem_types = other.InputMemoryType(); + auto& other_input_mem_types = other.input_memory_type_args_; for (auto it : input_memory_type_args_) { - if (other_input_mem_types.count(it.first) && other_input_mem_types[it.first] == it.second) + if (other_input_mem_types.count(it.first) && other_input_mem_types.find(it.first)->second == it.second) return false; } - if (input_memory_type_args_.empty() && !other.InputMemoryType().empty()) + if (input_memory_type_args_.empty() && !other.input_memory_type_args_.empty()) return false; - auto other_output_mem_types = other.OutputMemoryType(); + auto& other_output_mem_types = other.output_memory_type_args_; for (auto it : output_memory_type_args_) { - if (other_output_mem_types.count(it.first) && other_output_mem_types[it.first] == it.second) + if (other_output_mem_types.count(it.first) && other_output_mem_types.find(it.second)->second == it.second) return false; } - return !(output_memory_type_args_.empty() && !other.OutputMemoryType().empty()); + return !(output_memory_type_args_.empty() && !other.output_memory_type_args_.empty()); } KernelDefBuilder& KernelDefBuilder::SetName(const std::string& op_name) { diff --git a/onnxruntime/core/framework/transformer_memcpy.cc b/onnxruntime/core/framework/transformer_memcpy.cc index a4faef010d5c0..b04f278599324 100644 --- a/onnxruntime/core/framework/transformer_memcpy.cc +++ b/onnxruntime/core/framework/transformer_memcpy.cc @@ -68,25 +68,24 @@ void TransformerMemcpyImpl::ProcessDefs(onnxruntime::Node& node, const KernelReg // note KernelCreateInfo might be nullptr for custom kernel const KernelCreateInfo* kci = nullptr; kernel_registries.SearchKernelRegistry(node, &kci); - const auto* input_mem_types = kci ? &kci->kernel_def->InputMemoryType() : nullptr; - const auto* output_mem_types = kci ? &kci->kernel_def->InputMemoryType() : nullptr; + ORT_ENFORCE(onnxruntime::Node::ForEachWithIndex( - node.InputDefs(), - [this, &input_mem_types](const onnxruntime::NodeArg& arg, size_t index) { - if (input_mem_types && MemTypeOnCpuExplicitly(*input_mem_types, index)) - non_provider_input_defs_.insert(&arg); - else - provider_input_defs_.insert(&arg); - return Status::OK(); - }) - .IsOK()); + node.InputDefs(), + [this, &kci](const onnxruntime::NodeArg& arg, size_t index) { + if (kci && MemTypeOnCpuExplicitly(kci->kernel_def->InputMemoryType(index))) + non_provider_input_defs_.insert(&arg); + else + provider_input_defs_.insert(&arg); + return Status::OK(); + }) + .IsOK()); auto& output_defs = node.MutableOutputDefs(); for (size_t i = 0; i < output_defs.size(); ++i) { auto arg = output_defs[i]; if (!arg->Exists()) continue; - if (output_mem_types && MemTypeOnCpuExplicitly(*output_mem_types, i)) + if (kci && MemTypeOnCpuExplicitly(kci->kernel_def->OutputMemoryType(i))) non_provider_output_defs_.insert(arg); else provider_output_defs_.insert(arg); diff --git a/onnxruntime/core/session/IOBinding.cc b/onnxruntime/core/session/IOBinding.cc index 67a33be805612..cd24146d71105 100644 --- a/onnxruntime/core/session/IOBinding.cc +++ b/onnxruntime/core/session/IOBinding.cc @@ -60,10 +60,9 @@ common::Status IOBinding::CopyOneInputAcrossDevices(const SessionState& session_ size_t index = node_info.index; auto& node = *node_info.p_node; const KernelCreateInfo* kci = node_info.kci; - const auto* node_input_mem_types = (kci != nullptr) ? &kci->kernel_def->InputMemoryType() : nullptr; // node may declare input_mem_type to be on CPU explicitly - bool node_input_on_cpu = node_input_mem_types && MemTypeOnCpuExplicitly(*node_input_mem_types, index); + bool node_input_on_cpu = kci && MemTypeOnCpuExplicitly(kci->kernel_def->InputMemoryType(index)); auto& required_provider_type = node_input_on_cpu ? onnxruntime::kCpuExecutionProvider : node.GetExecutionProviderType(); if (!orig_mlvalue.IsTensor()) { // copying not supported for non-tensor types From 4f8af6a867bf1dae91dd32cfd114c73cb17b1201 Mon Sep 17 00:00:00 2001 From: "Yalachigere, Sreekanth V" Date: Sun, 9 Dec 2018 21:58:54 -0800 Subject: [PATCH 40/56] Filter data (Weights) reorder optimization --- .../mkldnn/mkldnn_execution_provider.h | 13 +++++++ onnxruntime/core/providers/mkldnn/nn/conv.cc | 35 +++++++++++-------- onnxruntime/core/providers/mkldnn/nn/conv.h | 8 +++++ 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h index e5820f8951514..13685856e8e3a 100644 --- a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h +++ b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h @@ -4,10 +4,12 @@ #pragma once #include +#include #include "core/framework/allocatormgr.h" #include "core/framework/execution_provider.h" #include "core/graph/graph_transformer.h" +#include "mkldnn.hpp" namespace onnxruntime { @@ -37,6 +39,17 @@ class MKLDNNExecutionProvider : public IExecutionProvider { } virtual std::shared_ptr GetKernelRegistry() const override; + + std::shared_ptr GetWeightMemory(std::string weightName) { + if (weights_mem_map.find(weightName) != weights_mem_map.end()) + return weights_mem_map[weightName]; + else + return nullptr; + } +public: + // mkldnn formatted weights(filer data) memory from first iteration + // saved by weights name + std::map> weights_mem_map; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.cc b/onnxruntime/core/providers/mkldnn/nn/conv.cc index 355562bf5adcd..edd85eb17a675 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.cc +++ b/onnxruntime/core/providers/mkldnn/nn/conv.cc @@ -402,20 +402,27 @@ Status Conv::Compute(OpKernelContext* context) const { src_data = static_cast(dst.get_data_handle()); } - // Reorder filter memory layout if necessary. - if (filter_format != conv_primitive->GetFilterMemoryFormat()) { - auto pd = mkldnn::memory::primitive_desc(mkldnn::memory::desc(filter_dims_mkl, - MklDnnType(), - filter_format), - cpu_engine); - mkldnn::memory src = mkldnn::memory(pd, (void*)filter_data); - // allocate the size queried from memory primitive desc. it may not match tensor logical size due to - // mkldnn using padding to allow use of blocked format. - filter_reorder_buffer = IAllocator::MakeUniquePtr(alloc, conv_primitive->GetFilterSize()); - mkldnn::memory dst = mkldnn::memory(conv_fwd_pd->weights_primitive_desc(), filter_reorder_buffer.get()); - MemoryReorderParams params(src, dst); - DoReorder(params); - filter_data = static_cast(dst.get_data_handle()); + // Reorder filter memory layout if necessary + // Avoid data reordering. Save filter memory in mkldnn format from first iteration + // in execution provider mapped by weight name. + std::string weightName = OpKernel::Node().InputDefs()[1]->Name(); + std::shared_ptr filter_dst_mem = provider_->GetWeightMemory(weightName); + + if (filter_dst_mem == nullptr) { + if (filter_format != conv_primitive->GetFilterMemoryFormat()) { + auto pd = mkldnn::memory::primitive_desc(mkldnn::memory::desc( + filter_dims_mkl, MklDnnType(), filter_format), cpu_engine); + mkldnn::memory src = mkldnn::memory(pd, (void*)filter_data); + filter_reorder_buffer = IAllocator::MakeUniquePtr(alloc, conv_primitive->GetFilterSize()); + filter_dst_mem.reset( + new mkldnn::memory(conv_fwd_pd->weights_primitive_desc(), filter_reorder_buffer.get())); + MemoryReorderParams params(src, *filter_dst_mem); + DoReorder(params); + filter_data = static_cast(filter_dst_mem->get_data_handle()); + provider_->weights_mem_map[weightName] = filter_dst_mem; + } + } else { + filter_data = static_cast(filter_dst_mem->get_data_handle()); } // Allocate dst buffer if reorder is necessary diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.h b/onnxruntime/core/providers/mkldnn/nn/conv.h index 8189a7ca6ba10..7059f81fc2fe1 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.h +++ b/onnxruntime/core/providers/mkldnn/nn/conv.h @@ -4,18 +4,26 @@ #pragma once #include "core/framework/op_kernel.h" #include "core/providers/cpu/nn/conv.h" +#include "../mkldnn_execution_provider.h" namespace onnxruntime { namespace mkl_dnn { + template class Conv final : public onnxruntime::Conv { public: Conv(const OpKernelInfo& info) : onnxruntime::Conv(info) { + if (info.GetExecutionProvider()->Type() == kMklDnnExecutionProvider) { + provider_ = (const_cast( + dynamic_cast(info.GetExecutionProvider()))); + } } Status Compute(OpKernelContext* context) const override; private: + MKLDNNExecutionProvider * provider_; + }; } // namespace mkl_dnn } // namespace onnxruntime From 8808e1476da1c3c4a085e12b76017cc658e305f5 Mon Sep 17 00:00:00 2001 From: "Yalachigere, Sreekanth V" Date: Mon, 10 Dec 2018 10:31:25 -0800 Subject: [PATCH 41/56] check provider_ for nullptr --- onnxruntime/core/providers/mkldnn/nn/conv.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.cc b/onnxruntime/core/providers/mkldnn/nn/conv.cc index edd85eb17a675..f6de43b6a504e 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.cc +++ b/onnxruntime/core/providers/mkldnn/nn/conv.cc @@ -406,7 +406,9 @@ Status Conv::Compute(OpKernelContext* context) const { // Avoid data reordering. Save filter memory in mkldnn format from first iteration // in execution provider mapped by weight name. std::string weightName = OpKernel::Node().InputDefs()[1]->Name(); - std::shared_ptr filter_dst_mem = provider_->GetWeightMemory(weightName); + std::shared_ptr filter_dst_mem = nullptr; + if(provider_ != nullptr) + filter_dst_mem = provider_->GetWeightMemory(weightName); if (filter_dst_mem == nullptr) { if (filter_format != conv_primitive->GetFilterMemoryFormat()) { @@ -419,7 +421,8 @@ Status Conv::Compute(OpKernelContext* context) const { MemoryReorderParams params(src, *filter_dst_mem); DoReorder(params); filter_data = static_cast(filter_dst_mem->get_data_handle()); - provider_->weights_mem_map[weightName] = filter_dst_mem; + if (provider_ != nullptr) + provider_->weights_mem_map[weightName] = filter_dst_mem; } } else { filter_data = static_cast(filter_dst_mem->get_data_handle()); From 7c2a8d84b6a5d72d4ea920fa935871013c0d3a25 Mon Sep 17 00:00:00 2001 From: "Yalachigere, Sreekanth V" Date: Mon, 10 Dec 2018 16:17:48 -0800 Subject: [PATCH 42/56] PR Review changes: thread safe. weights map object private --- .../providers/mkldnn/mkldnn_execution_provider.h | 16 +++++++++++----- onnxruntime/core/providers/mkldnn/nn/conv.cc | 10 ++++++---- onnxruntime/core/providers/mkldnn/nn/conv.h | 4 +--- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h index 13685856e8e3a..911dd2a1eb22f 100644 --- a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h +++ b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h @@ -41,12 +41,18 @@ class MKLDNNExecutionProvider : public IExecutionProvider { virtual std::shared_ptr GetKernelRegistry() const override; std::shared_ptr GetWeightMemory(std::string weightName) { - if (weights_mem_map.find(weightName) != weights_mem_map.end()) - return weights_mem_map[weightName]; - else - return nullptr; + + auto iter = weights_mem_map.find(weightName); + if (iter != weights_mem_map.end()) + return iter->second; + return nullptr; + } + + std::map>& GetWeightsMap() { + return weights_mem_map; } -public: + +private: // mkldnn formatted weights(filer data) memory from first iteration // saved by weights name std::map> weights_mem_map; diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.cc b/onnxruntime/core/providers/mkldnn/nn/conv.cc index f6de43b6a504e..81ad0b4f1c9e3 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.cc +++ b/onnxruntime/core/providers/mkldnn/nn/conv.cc @@ -407,8 +407,7 @@ Status Conv::Compute(OpKernelContext* context) const { // in execution provider mapped by weight name. std::string weightName = OpKernel::Node().InputDefs()[1]->Name(); std::shared_ptr filter_dst_mem = nullptr; - if(provider_ != nullptr) - filter_dst_mem = provider_->GetWeightMemory(weightName); + filter_dst_mem = provider_->GetWeightMemory(weightName); if (filter_dst_mem == nullptr) { if (filter_format != conv_primitive->GetFilterMemoryFormat()) { @@ -421,8 +420,11 @@ Status Conv::Compute(OpKernelContext* context) const { MemoryReorderParams params(src, *filter_dst_mem); DoReorder(params); filter_data = static_cast(filter_dst_mem->get_data_handle()); - if (provider_ != nullptr) - provider_->weights_mem_map[weightName] = filter_dst_mem; + { + // make assignment threadsafe + std::lock_guard lock(mutex_); + provider_->GetWeightsMap()[weightName] = filter_dst_mem; + } } } else { filter_data = static_cast(filter_dst_mem->get_data_handle()); diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.h b/onnxruntime/core/providers/mkldnn/nn/conv.h index 7059f81fc2fe1..8dde0c899f577 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.h +++ b/onnxruntime/core/providers/mkldnn/nn/conv.h @@ -13,17 +13,15 @@ template class Conv final : public onnxruntime::Conv { public: Conv(const OpKernelInfo& info) : onnxruntime::Conv(info) { - if (info.GetExecutionProvider()->Type() == kMklDnnExecutionProvider) { provider_ = (const_cast( dynamic_cast(info.GetExecutionProvider()))); - } } Status Compute(OpKernelContext* context) const override; private: MKLDNNExecutionProvider * provider_; - + mutable std::mutex mutex_; }; } // namespace mkl_dnn } // namespace onnxruntime From 32ee1696e653062025f270b1d653e9d5f08976e5 Mon Sep 17 00:00:00 2001 From: "Yalachigere, Sreekanth V" Date: Tue, 11 Dec 2018 09:38:33 -0800 Subject: [PATCH 43/56] using Conv Parameters key to make weights id unique --- onnxruntime/core/providers/mkldnn/nn/conv.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.cc b/onnxruntime/core/providers/mkldnn/nn/conv.cc index 81ad0b4f1c9e3..831c7b620d620 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.cc +++ b/onnxruntime/core/providers/mkldnn/nn/conv.cc @@ -352,6 +352,9 @@ Status Conv::Compute(OpKernelContext* context) const { dst_dims_mkl, strides_mkl, dilations_mkl, padding_left_mkl, padding_right_mkl); ConvPrimitive* conv_primitive = ConvPrimitivePool::Get(conv_params); + + std::string convString = conv_params.ToString(); + auto conv_fwd_pd = conv_primitive->GetPrimitiveDesc(); mkldnn::engine& cpu_engine = GetEngine(); @@ -405,9 +408,9 @@ Status Conv::Compute(OpKernelContext* context) const { // Reorder filter memory layout if necessary // Avoid data reordering. Save filter memory in mkldnn format from first iteration // in execution provider mapped by weight name. - std::string weightName = OpKernel::Node().InputDefs()[1]->Name(); + std::string weightKey = convString + "-" + OpKernel::Node().InputDefs()[1]->Name(); std::shared_ptr filter_dst_mem = nullptr; - filter_dst_mem = provider_->GetWeightMemory(weightName); + filter_dst_mem = provider_->GetWeightMemory(weightKey); if (filter_dst_mem == nullptr) { if (filter_format != conv_primitive->GetFilterMemoryFormat()) { @@ -423,7 +426,7 @@ Status Conv::Compute(OpKernelContext* context) const { { // make assignment threadsafe std::lock_guard lock(mutex_); - provider_->GetWeightsMap()[weightName] = filter_dst_mem; + provider_->GetWeightsMap()[weightKey] = filter_dst_mem; } } } else { From fe640a803f4c4612e02f0bafb46cb058df538885 Mon Sep 17 00:00:00 2001 From: "Yalachigere, Sreekanth V" Date: Wed, 12 Dec 2018 09:59:33 -0800 Subject: [PATCH 44/56] removed convParam string from weight key. Weight Id is unique --- onnxruntime/core/providers/mkldnn/nn/conv.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.cc b/onnxruntime/core/providers/mkldnn/nn/conv.cc index 831c7b620d620..db3e0e55017de 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.cc +++ b/onnxruntime/core/providers/mkldnn/nn/conv.cc @@ -353,8 +353,6 @@ Status Conv::Compute(OpKernelContext* context) const { padding_left_mkl, padding_right_mkl); ConvPrimitive* conv_primitive = ConvPrimitivePool::Get(conv_params); - std::string convString = conv_params.ToString(); - auto conv_fwd_pd = conv_primitive->GetPrimitiveDesc(); mkldnn::engine& cpu_engine = GetEngine(); @@ -408,7 +406,7 @@ Status Conv::Compute(OpKernelContext* context) const { // Reorder filter memory layout if necessary // Avoid data reordering. Save filter memory in mkldnn format from first iteration // in execution provider mapped by weight name. - std::string weightKey = convString + "-" + OpKernel::Node().InputDefs()[1]->Name(); + std::string weightKey = OpKernel::Node().InputDefs()[1]->Name(); std::shared_ptr filter_dst_mem = nullptr; filter_dst_mem = provider_->GetWeightMemory(weightKey); From a37887cfa1974ac5fdd986ed15eb55b12d1b10b4 Mon Sep 17 00:00:00 2001 From: Ryan Hill <38674843+RyanUnderhill@users.noreply.github.com> Date: Thu, 20 Dec 2018 13:47:48 -0800 Subject: [PATCH 45/56] More intuitive ordering to the API functions (#233) * More intuitive ordering to the API functions * Rename TCHAR_T --- .../core/session/onnxruntime_c_api.h | 461 +++++++++--------- 1 file changed, 226 insertions(+), 235 deletions(-) diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 1be5004ac1fa6..a98a6e0a69aae 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -40,10 +40,12 @@ extern "C" { #endif #define ORT_API_CALL _stdcall #define ORT_MUST_USE_RESULT +#define ORTCHAR_T wchar_t #else #define ORT_EXPORT #define ORT_API_CALL #define ORT_MUST_USE_RESULT __attribute__((warn_unused_result)) +#define ORTCHAR_T char #endif // Any pointer marked with _In_ or _Out_, cannot be NULL. @@ -58,6 +60,46 @@ extern "C" { #define NO_EXCEPTION #endif +// Copied from TensorProto::DataType +// Currently, Ort doesn't support complex64, complex128, bfloat16 types +typedef enum ONNXTensorElementDataType { + ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED = 0, + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT = 1, // maps to c type float + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8 = 2, // maps to c type uint8_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8 = 3, // maps to c type int8_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16 = 4, // maps to c type uint16_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16 = 5, // maps to c type int16_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32 = 6, // maps to c type int32_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64 = 7, // maps to c type int64_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING = 8, // maps to c++ type std::string + ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL = 9, // + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16 = 10, + ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE = 11, // maps to c type double + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32 = 12, // maps to c type uint32_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64 = 13, // maps to c type uint64_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64 = 14, // complex with float32 real and imaginary components + ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128 = 15, // complex with float64 real and imaginary components + ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16 = 16, // Non-IEEE floating-point format based on IEEE754 single-precision +} ONNXTensorElementDataType; + +// Synced with onnx TypeProto oneof +typedef enum ONNXType { + ONNX_TYPE_UNKNOWN, + ONNX_TYPE_TENSOR, + ONNX_TYPE_SEQUENCE, + ONNX_TYPE_MAP, + ONNX_TYPE_OPAQUE, + ONNX_TYPE_SPARSETENSOR, +} ONNXType; + +typedef enum OrtLoggingLevel { + ORT_LOGGING_LEVEL_kVERBOSE = 0, + ORT_LOGGING_LEVEL_kINFO = 1, + ORT_LOGGING_LEVEL_kWARNING = 2, + ORT_LOGGING_LEVEL_kERROR = 3, + ORT_LOGGING_LEVEL_kFATAL = 4 +} OrtLoggingLevel; + typedef enum OrtErrorCode { ORT_OK = 0, ORT_FAIL = 1, @@ -121,126 +163,6 @@ typedef struct OrtSessionOptions OrtSessionOptions; struct OrtEnv; typedef struct OrtEnv OrtEnv; -/** - * \param msg A null-terminated string. Its content will be copied into the newly created OrtStatus - */ -ORT_API(OrtStatus*, OrtCreateStatus, OrtErrorCode code, _In_ const char* msg) -ORT_ALL_ARGS_NONNULL; - -ORT_API(OrtErrorCode, OrtGetErrorCode, _In_ const OrtStatus* status) -ORT_ALL_ARGS_NONNULL; -/** - * \param status must not be NULL - * \return The error message inside the `status`. Don't free the returned value. - */ -ORT_API(const char*, OrtGetErrorMessage, _In_ const OrtStatus* status) -ORT_ALL_ARGS_NONNULL; - -// -// Tensor Type and Shapes -// - -// Copied from TensorProto::DataType -// Currently, Ort doesn't support complex64, complex128, bfloat16 types -typedef enum ONNXTensorElementDataType { - ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED = 0, - ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT = 1, // maps to c type float - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8 = 2, // maps to c type uint8_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8 = 3, // maps to c type int8_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16 = 4, // maps to c type uint16_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16 = 5, // maps to c type int16_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32 = 6, // maps to c type int32_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64 = 7, // maps to c type int64_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING = 8, // maps to c++ type std::string - ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL = 9, // - ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16 = 10, - ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE = 11, // maps to c type double - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32 = 12, // maps to c type uint32_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64 = 13, // maps to c type uint64_t - ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64 = 14, // complex with float32 real and imaginary components - ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128 = 15, // complex with float64 real and imaginary components - ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16 = 16, // Non-IEEE floating-point format based on IEEE754 single-precision -} ONNXTensorElementDataType; - -// Sync with onnx TypeProto oneof -typedef enum ONNXType { - ONNX_TYPE_UNKNOWN, - ONNX_TYPE_TENSOR, - ONNX_TYPE_SEQUENCE, - ONNX_TYPE_MAP, - ONNX_TYPE_OPAQUE, - ONNX_TYPE_SPARSETENSOR, -} ONNXType; - -/** - * Don't free the returned value - */ -ORT_API(const OrtTensorTypeAndShapeInfo*, OrtCastTypeInfoToTensorInfo, _In_ OrtTypeInfo*); - -/** - * The retured value should be released by calling OrtReleaseObject - */ -ORT_API(OrtTensorTypeAndShapeInfo*, OrtCreateTensorTypeAndShapeInfo); - -ORT_API_STATUS(OrtSetTensorElementType, _In_ OrtTensorTypeAndShapeInfo*, enum ONNXTensorElementDataType type); - -/** - * \param info Created from OrtCreateTensorTypeAndShapeInfo() function - * \param dim_values An array with length of `dim_count`. Its elements can contain negative values. - * \param dim_count length of dim_values - */ -ORT_API_STATUS(OrtSetDims, OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count); - -ORT_API(enum ONNXTensorElementDataType, OrtGetTensorElementType, _In_ const OrtTensorTypeAndShapeInfo*); -ORT_API(size_t, OrtGetNumOfDimensions, _In_ const OrtTensorTypeAndShapeInfo* info); -ORT_API(void, OrtGetDimensions, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, size_t dim_values_length); - -/** - * How many elements does this tensor have. - * May return a negative value - * e.g. - * [] -> 1 - * [1,3,4] -> 12 - * [2,0,4] -> 0 - * [-1,3,4] -> -1 - * return a negative value if unknown. (That this shape contains a symbolic variable which - * represents an unknown dimension.) - */ -ORT_API(int64_t, OrtGetTensorShapeElementCount, _In_ const OrtTensorTypeAndShapeInfo* info); - -/** - * \param out Should be freed by OrtReleaseObject after use - */ -ORT_API_STATUS(OrtGetTensorShapeAndType, _In_ const OrtValue* value, _Out_ OrtTensorTypeAndShapeInfo** out); - -/** - * Get the type information of an OrtValue - * \param value - * \param out The returned value should be freed by OrtReleaseObject after use - */ -ORT_API_STATUS(OrtGetTypeInfo, _In_ const OrtValue* value, OrtTypeInfo** out); - -ORT_API(enum ONNXType, OrtGetValueType, _In_ const OrtValue* value); - -// -// OrtRunOptions -// - -/** - * \return A pointer of the newly created object. The pointer should be freed by OrtReleaseObject after use - */ -ORT_API(OrtRunOptions*, OrtCreateRunOptions); - -ORT_API_STATUS(OrtRunOptionsSetRunLogVerbosityLevel, _In_ OrtRunOptions*, unsigned int); -ORT_API_STATUS(OrtRunOptionsSetRunTag, _In_ OrtRunOptions*, _In_ const char* run_tag); - -ORT_API(unsigned int, OrtRunOptionsGetRunLogVerbosityLevel, _In_ OrtRunOptions*); -ORT_API(const char*, OrtRunOptionsGetRunTag, _In_ OrtRunOptions*); - -// set a flag so that any running OrtRunInference* calls that are using this instance of ORtRunOptions -// will exit as soon as possible if the flag is true. -ORT_API(void, OrtRunOptionsSetTerminate, _In_ OrtRunOptions*, _In_ bool value); - /** * Every type inherented from OrtObject should be deleted by OrtReleaseObject(...). */ @@ -252,22 +174,15 @@ typedef struct OrtObject { } OrtObject; -/** - * This function is a wrapper to "(*(OrtObject**)ptr)->AddRef(ptr)" - * WARNING: There is NO type checking in this function. - * Before calling this function, caller should make sure current ref count > 0 - * \return the new reference count - */ -ORT_API(uint32_t, OrtAddRefToObject, _In_ void* ptr); +//inherented from OrtObject +typedef struct OrtAllocatorInterface { + struct OrtObject parent; + void*(ORT_API_CALL* Alloc)(void* this_, size_t size); + void(ORT_API_CALL* Free)(void* this_, void* p); + const struct OrtAllocatorInfo*(ORT_API_CALL* Info)(const void* this_); +} OrtAllocatorInterface; -/** - * - * A wrapper to "(*(OrtObject**)ptr)->Release(ptr)" - * WARNING: There is NO type checking in this function. - * \param ptr Can be NULL. If it's NULL, this function will return zero. - * \return the new reference count. - */ -ORT_API(uint32_t, OrtReleaseObject, _Inout_opt_ void* ptr); +typedef OrtAllocatorInterface* OrtAllocator; //Inherented from OrtObject typedef struct OrtProviderFactoryInterface { @@ -275,6 +190,40 @@ typedef struct OrtProviderFactoryInterface { OrtStatus*(ORT_API_CALL* CreateProvider)(void* this_, OrtProvider** out); } OrtProviderFactoryInterface; +typedef void(ORT_API_CALL* OrtLoggingFunction)( + void* param, OrtLoggingLevel severity, const char* category, const char* logid, const char* code_location, + const char* message); + +/** + * OrtEnv is process-wide. For each process, only one OrtEnv can be created. + * \param out Should be freed by `OrtReleaseObject` after use + */ +ORT_API_STATUS(OrtInitialize, OrtLoggingLevel default_warning_level, _In_ const char* logid, _Out_ OrtEnv** out) +ORT_ALL_ARGS_NONNULL; + +/** + * OrtEnv is process-wise. For each process, only one OrtEnv can be created. Don't do it multiple times + * \param out Should be freed by `OrtReleaseObject` after use + */ +ORT_API_STATUS(OrtInitializeWithCustomLogger, OrtLoggingFunction logging_function, + _In_opt_ void* logger_param, OrtLoggingLevel default_warning_level, + _In_ const char* logid, + _Out_ OrtEnv** out); + +// TODO: document the path separator convention? '/' vs '\' +// TODO: should specify the access characteristics of model_path. Is this read only during the +// execution of OrtCreateInferenceSession, or does the OrtSession retain a handle to the file/directory +// and continue to access throughout the OrtSession lifetime? +// What sort of access is needed to model_path : read or read/write? +// TODO: allow loading from an in-memory byte-array +ORT_API_STATUS(OrtCreateInferenceSession, _In_ OrtEnv* env, _In_ const ORTCHAR_T* model_path, + _In_ const OrtSessionOptions* options, _Out_ OrtSession** out); + +ORT_API_STATUS(OrtRunInference, _Inout_ OrtSession* sess, + _In_ OrtRunOptions* run_options, + _In_ const char* const* input_names, _In_ const OrtValue* const* input, size_t input_len, + _In_ const char* const* output_names, size_t output_names_len, _Out_ OrtValue** output); + /** * \return A pointer of the newly created object. The pointer should be freed by OrtReleaseObject after use */ @@ -285,11 +234,11 @@ ORT_API(OrtSessionOptions*, OrtCloneSessionOptions, OrtSessionOptions*); ORT_API(void, OrtEnableSequentialExecution, _In_ OrtSessionOptions* options); ORT_API(void, OrtDisableSequentialExecution, _In_ OrtSessionOptions* options); -// enable profiling for this session. +// Enable profiling for this session. ORT_API(void, OrtEnableProfiling, _In_ OrtSessionOptions* options, _In_ const char* profile_file_prefix); ORT_API(void, OrtDisableProfiling, _In_ OrtSessionOptions* options); -// enable the memory pattern optimization. +// Enable the memory pattern optimization. // The idea is if the input shapes are the same, we could trace the internal memory allocation // and generate a memory pattern for future request. So next time we could just do one allocation // with a big chunk for all the internal memory allocation. @@ -320,94 +269,38 @@ ORT_API(void, OrtSessionOptionsAppendExecutionProvider, _In_ OrtSessionOptions* ORT_API(void, OrtAddCustomOp, _In_ OrtSessionOptions* options, const char* custom_op_path); -typedef enum OrtAllocatorType { - OrtDeviceAllocator = 0, - OrtArenaAllocator = 1 -} OrtAllocatorType; - -/** - memory types for allocator, exec provider specific types should be extended in each provider -*/ -typedef enum OrtMemType { - OrtMemTypeCPUInput = -2, // Any CPU memory used by non-CPU execution provider - OrtMemTypeCPUOutput = -1, // CPU accessible memory outputted by non-CPU execution provider, i.e. CUDA_PINNED - OrtMemTypeCPU = OrtMemTypeCPUOutput, // temporary CPU accessible memory allocated by non-CPU execution provider, i.e. CUDA_PINNED - OrtMemTypeDefault = 0, // the default allocator for execution provider -} OrtMemType; - -ORT_API_STATUS(OrtCreateAllocatorInfo, _In_ const char* name1, enum OrtAllocatorType type, int id1, enum OrtMemType mem_type1, _Out_ OrtAllocatorInfo** out); +ORT_API_STATUS(OrtInferenceSessionGetInputCount, _In_ const OrtSession* sess, _Out_ size_t* out); +ORT_API_STATUS(OrtInferenceSessionGetOutputCount, _In_ const OrtSession* sess, _Out_ size_t* out); /** - * Test if two allocation info are equal - * \return 0, equal. zero, not equal + * \param out should be freed by OrtReleaseObject after use */ -ORT_API(int, OrtCompareAllocatorInfo, _In_ const OrtAllocatorInfo* info1, _In_ const OrtAllocatorInfo* info2) -ORT_ALL_ARGS_NONNULL; +ORT_API_STATUS(OrtInferenceSessionGetInputTypeInfo, _In_ const OrtSession* sess, size_t index, _Out_ OrtTypeInfo** out); + /** - * Do not free the returned value + * \param out should be freed by OrtReleaseObject after use */ -ORT_API(const char*, OrtAllocatorInfoGetName, _In_ OrtAllocatorInfo* ptr); -ORT_API(int, OrtAllocatorInfoGetId, _In_ OrtAllocatorInfo* ptr); -ORT_API(OrtMemType, OrtAllocatorInfoGetMemType, _In_ OrtAllocatorInfo* ptr); -ORT_API(OrtAllocatorType, OrtAllocatorInfoGetType, _In_ OrtAllocatorInfo* ptr); - -//inherented from OrtObject -typedef struct OrtAllocatorInterface { - struct OrtObject parent; - void*(ORT_API_CALL* Alloc)(void* this_, size_t size); - void(ORT_API_CALL* Free)(void* this_, void* p); - const struct OrtAllocatorInfo*(ORT_API_CALL* Info)(const void* this_); -} OrtAllocatorInterface; - -typedef OrtAllocatorInterface* OrtAllocator; - -ORT_API(void*, OrtAllocatorAlloc, _Inout_ OrtAllocator* ptr, size_t size); -ORT_API(void, OrtAllocatorFree, _Inout_ OrtAllocator* ptr, void* p); -ORT_API(const OrtAllocatorInfo*, OrtAllocatorGetInfo, _In_ const OrtAllocator* ptr); +ORT_API_STATUS(OrtInferenceSessionGetOutputTypeInfo, _In_ const OrtSession* sess, size_t index, _Out_ OrtTypeInfo** out); -typedef enum OrtLoggingLevel { - ORT_LOGGING_LEVEL_kVERBOSE = 0, - ORT_LOGGING_LEVEL_kINFO = 1, - ORT_LOGGING_LEVEL_kWARNING = 2, - ORT_LOGGING_LEVEL_kERROR = 3, - ORT_LOGGING_LEVEL_kFATAL = 4 -} OrtLoggingLevel; +ORT_API_STATUS(OrtInferenceSessionGetInputName, _In_ const OrtSession* sess, size_t index, + _Inout_ OrtAllocator* allocator, _Out_ char** value); +ORT_API_STATUS(OrtInferenceSessionGetOutputName, _In_ const OrtSession* sess, size_t index, + _Inout_ OrtAllocator* allocator, _Out_ char** value); -typedef void(ORT_API_CALL* OrtLoggingFunction)( - void* param, OrtLoggingLevel severity, const char* category, const char* logid, const char* code_location, - const char* message); /** - * OrtEnv is process-wise. For each process, only one OrtEnv can be created. Don't do it multiple times - * \param out Should be freed by `OrtReleaseObject` after use + * \return A pointer to the newly created object. The pointer should be freed by OrtReleaseObject after use */ -ORT_API_STATUS(OrtInitialize, OrtLoggingLevel default_warning_level, _In_ const char* logid, _Out_ OrtEnv** out) -ORT_ALL_ARGS_NONNULL; +ORT_API(OrtRunOptions*, OrtCreateRunOptions); -/** - * OrtEnv is process-wise. For each process, only one OrtEnv can be created. Don't do it multiple times - * \param out Should be freed by `OrtReleaseObject` after use - */ -ORT_API_STATUS(OrtInitializeWithCustomLogger, OrtLoggingFunction logging_function, - _In_opt_ void* logger_param, OrtLoggingLevel default_warning_level, - _In_ const char* logid, - _Out_ OrtEnv** out); +ORT_API_STATUS(OrtRunOptionsSetRunLogVerbosityLevel, _In_ OrtRunOptions*, unsigned int); +ORT_API_STATUS(OrtRunOptionsSetRunTag, _In_ OrtRunOptions*, _In_ const char* run_tag); -// TODO: document the path separator convention? '/' vs '\' -// TODO: should specify the access characteristics of model_path. Is this read only during the -// execution of OrtCreateInferenceSession, or does the OrtSession retain a handle to the file/directory -// and continue to access throughout the OrtSession lifetime? -// What sort of access is needed to model_path : read or read/write? -// TODO: allow loading from an in-memory byte-array -#ifdef _WIN32 -ORT_API_STATUS(OrtCreateInferenceSession, _In_ OrtEnv* env, _In_ const wchar_t* model_path, - _In_ const OrtSessionOptions* options, _Out_ OrtSession** out); -#else -ORT_API_STATUS(OrtCreateInferenceSession, _In_ OrtEnv* env, _In_ const char* model_path, - _In_ const OrtSessionOptions* options, _Out_ OrtSession** out); -#endif +ORT_API(unsigned int, OrtRunOptionsGetRunLogVerbosityLevel, _In_ OrtRunOptions*); +ORT_API(const char*, OrtRunOptionsGetRunTag, _In_ OrtRunOptions*); -// Call OrtReleaseObject to release the returned value -ORT_API_STATUS(OrtCreateDefaultAllocator, _Out_ OrtAllocator** out); +// set a flag so that any running OrtRunInference* calls that are using this instance of ORtRunOptions +// will exit as soon as possible if the flag is true. +ORT_API(void, OrtRunOptionsSetTerminate, _In_ OrtRunOptions*, _In_ bool value); /** * Create a tensor from an allocator. OrtReleaseValue will also release the buffer inside the output value @@ -428,61 +321,159 @@ ORT_API_STATUS(OrtCreateTensorWithDataAsOrtValue, _In_ const OrtAllocatorInfo* i _In_ void* p_data, size_t p_data_len, _In_ const size_t* shape, size_t shape_len, ONNXTensorElementDataType type, _Out_ OrtValue** out); -/// This function doesn't work with string tensor -/// this is a no-copy method whose pointer is only valid until the backing OrtValue is free'd. +// This function doesn't work with string tensor +// this is a no-copy method whose pointer is only valid until the backing OrtValue is free'd. ORT_API_STATUS(OrtGetTensorMutableData, _Inout_ OrtValue* value, _Out_ void** out); /** * Test if an OrtValue is a tensor - * \return zero, false. non-zero true + * \return zero if false. non-zero if true */ ORT_API(int, OrtIsTensor, _In_ const OrtValue* value); /** - * \param value A tensor created from OrtCreateTensor*** function. + * \param value A tensor created from OrtCreateTensor... function. * \param s each A string array. Each string in this array must be null terminated. * \param s_len length of s */ ORT_API_STATUS(OrtFillStringTensor, _In_ OrtValue* value, _In_ const char* const* s, size_t s_len); /** - * \param value A tensor created from OrtCreateTensor*** function. + * \param value A tensor created from OrtCreateTensor... function. * \param len total data length, not including the trailing '\0' chars. */ ORT_API_STATUS(OrtGetStringTensorDataLength, _In_ const OrtValue* value, _Out_ size_t* len); /** * \param s string contents. Each string is NOT null-terminated. - * \param value A tensor created from OrtCreateTensor*** function. + * \param value A tensor created from OrtCreateTensor... function. * \param s_len total data length, get it from OrtGetStringTensorDataLength */ ORT_API_STATUS(OrtGetStringTensorContent, _In_ const OrtValue* value, _Out_ void* s, size_t s_len, _Out_ size_t* offsets, size_t offsets_len); -ORT_API_STATUS(OrtRunInference, _Inout_ OrtSession* sess, - _In_ OrtRunOptions* run_options, - _In_ const char* const* input_names, _In_ const OrtValue* const* input, size_t input_len, - _In_ const char* const* output_names, size_t output_names_len, _Out_ OrtValue** output); +ORT_API_STATUS(OrtTensorProtoToOrtValue, _Inout_ OrtAllocator* allocator, + _In_ const void* input, int input_len, _Out_ OrtValue** out); -ORT_API_STATUS(OrtInferenceSessionGetInputCount, _In_ const OrtSession* sess, _Out_ size_t* out); -ORT_API_STATUS(OrtInferenceSessionGetOutputCount, _In_ const OrtSession* sess, _Out_ size_t* out); +/** + * Don't free the returned value + */ +ORT_API(const OrtTensorTypeAndShapeInfo*, OrtCastTypeInfoToTensorInfo, _In_ OrtTypeInfo*); /** - * \param out should be freed by OrtReleaseObject after use + * The retured value should be released by calling OrtReleaseObject */ -ORT_API_STATUS(OrtInferenceSessionGetInputTypeInfo, _In_ const OrtSession* sess, size_t index, _Out_ OrtTypeInfo** out); +ORT_API(OrtTensorTypeAndShapeInfo*, OrtCreateTensorTypeAndShapeInfo); + +ORT_API_STATUS(OrtSetTensorElementType, _In_ OrtTensorTypeAndShapeInfo*, enum ONNXTensorElementDataType type); /** - * \param out should be freed by OrtReleaseObject after use + * \param info Created from OrtCreateTensorTypeAndShapeInfo() function + * \param dim_values An array with length of `dim_count`. Its elements can contain negative values. + * \param dim_count length of dim_values */ -ORT_API_STATUS(OrtInferenceSessionGetOutputTypeInfo, _In_ const OrtSession* sess, size_t index, _Out_ OrtTypeInfo** out); +ORT_API_STATUS(OrtSetDims, OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count); -ORT_API_STATUS(OrtInferenceSessionGetInputName, _In_ const OrtSession* sess, size_t index, - _Inout_ OrtAllocator* allocator, _Out_ char** value); -ORT_API_STATUS(OrtInferenceSessionGetOutputName, _In_ const OrtSession* sess, size_t index, - _Inout_ OrtAllocator* allocator, _Out_ char** value); +ORT_API(enum ONNXTensorElementDataType, OrtGetTensorElementType, _In_ const OrtTensorTypeAndShapeInfo*); +ORT_API(size_t, OrtGetNumOfDimensions, _In_ const OrtTensorTypeAndShapeInfo* info); +ORT_API(void, OrtGetDimensions, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, size_t dim_values_length); -ORT_API_STATUS(OrtTensorProtoToOrtValue, _Inout_ OrtAllocator* allocator, - _In_ const void* input, int input_len, _Out_ OrtValue** out); +/** + * How many elements does this tensor have. + * May return a negative value + * e.g. + * [] -> 1 + * [1,3,4] -> 12 + * [2,0,4] -> 0 + * [-1,3,4] -> -1 + * return a negative value if unknown. (That this shape contains a symbolic variable which + * represents an unknown dimension.) + */ +ORT_API(int64_t, OrtGetTensorShapeElementCount, _In_ const OrtTensorTypeAndShapeInfo* info); + +/** + * \param out Should be freed by OrtReleaseObject after use + */ +ORT_API_STATUS(OrtGetTensorShapeAndType, _In_ const OrtValue* value, _Out_ OrtTensorTypeAndShapeInfo** out); + +/** + * Get the type information of an OrtValue + * \param value + * \param out The returned value should be freed by OrtReleaseObject after use + */ +ORT_API_STATUS(OrtGetTypeInfo, _In_ const OrtValue* value, OrtTypeInfo** out); + +ORT_API(enum ONNXType, OrtGetValueType, _In_ const OrtValue* value); + +/** + * This function is a wrapper to "(*(OrtObject**)ptr)->AddRef(ptr)" + * WARNING: There is NO type checking in this function. + * Before calling this function, caller should make sure current ref count > 0 + * \return the new reference count + */ +ORT_API(uint32_t, OrtAddRefToObject, _In_ void* ptr); + +/** + * + * A wrapper to "(*(OrtObject**)ptr)->Release(ptr)" + * WARNING: There is NO type checking in this function. + * \param ptr Can be NULL. If it's NULL, this function will return zero. + * \return the new reference count. + */ +ORT_API(uint32_t, OrtReleaseObject, _Inout_opt_ void* ptr); + +typedef enum OrtAllocatorType { + OrtDeviceAllocator = 0, + OrtArenaAllocator = 1 +} OrtAllocatorType; + +/** + memory types for allocator, exec provider specific types should be extended in each provider +*/ +typedef enum OrtMemType { + OrtMemTypeCPUInput = -2, // Any CPU memory used by non-CPU execution provider + OrtMemTypeCPUOutput = -1, // CPU accessible memory outputted by non-CPU execution provider, i.e. CUDA_PINNED + OrtMemTypeCPU = OrtMemTypeCPUOutput, // temporary CPU accessible memory allocated by non-CPU execution provider, i.e. CUDA_PINNED + OrtMemTypeDefault = 0, // the default allocator for execution provider +} OrtMemType; + +ORT_API_STATUS(OrtCreateAllocatorInfo, _In_ const char* name1, enum OrtAllocatorType type, int id1, enum OrtMemType mem_type1, _Out_ OrtAllocatorInfo** out); + +/** + * Test if two allocation info are equal + * \return 0, equal. zero, not equal + */ +ORT_API(int, OrtCompareAllocatorInfo, _In_ const OrtAllocatorInfo* info1, _In_ const OrtAllocatorInfo* info2) +ORT_ALL_ARGS_NONNULL; + +/** + * Do not free the returned value + */ +ORT_API(const char*, OrtAllocatorInfoGetName, _In_ OrtAllocatorInfo* ptr); +ORT_API(int, OrtAllocatorInfoGetId, _In_ OrtAllocatorInfo* ptr); +ORT_API(OrtMemType, OrtAllocatorInfoGetMemType, _In_ OrtAllocatorInfo* ptr); +ORT_API(OrtAllocatorType, OrtAllocatorInfoGetType, _In_ OrtAllocatorInfo* ptr); + +ORT_API(void*, OrtAllocatorAlloc, _Inout_ OrtAllocator* ptr, size_t size); +ORT_API(void, OrtAllocatorFree, _Inout_ OrtAllocator* ptr, void* p); +ORT_API(const OrtAllocatorInfo*, OrtAllocatorGetInfo, _In_ const OrtAllocator* ptr); + +// Call OrtReleaseObject to release the returned value +ORT_API_STATUS(OrtCreateDefaultAllocator, _Out_ OrtAllocator** out); + +/** + * \param msg A null-terminated string. Its content will be copied into the newly created OrtStatus + */ +ORT_API(OrtStatus*, OrtCreateStatus, OrtErrorCode code, _In_ const char* msg) +ORT_ALL_ARGS_NONNULL; + +ORT_API(OrtErrorCode, OrtGetErrorCode, _In_ const OrtStatus* status) +ORT_ALL_ARGS_NONNULL; +/** + * \param status must not be NULL + * \return The error message inside the `status`. Don't free the returned value. + */ +ORT_API(const char*, OrtGetErrorMessage, _In_ const OrtStatus* status) +ORT_ALL_ARGS_NONNULL; /** * Deprecated. Please use OrtReleaseObject From c7ad8d95a694efec8452cd63ae8a878e7ba7ba39 Mon Sep 17 00:00:00 2001 From: "Yalachigere, Sreekanth V" Date: Sun, 9 Dec 2018 21:58:54 -0800 Subject: [PATCH 46/56] Filter data (Weights) reorder optimization --- .../mkldnn/mkldnn_execution_provider.h | 13 +++++++ onnxruntime/core/providers/mkldnn/nn/conv.cc | 35 +++++++++++-------- onnxruntime/core/providers/mkldnn/nn/conv.h | 8 +++++ 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h index e5820f8951514..13685856e8e3a 100644 --- a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h +++ b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h @@ -4,10 +4,12 @@ #pragma once #include +#include #include "core/framework/allocatormgr.h" #include "core/framework/execution_provider.h" #include "core/graph/graph_transformer.h" +#include "mkldnn.hpp" namespace onnxruntime { @@ -37,6 +39,17 @@ class MKLDNNExecutionProvider : public IExecutionProvider { } virtual std::shared_ptr GetKernelRegistry() const override; + + std::shared_ptr GetWeightMemory(std::string weightName) { + if (weights_mem_map.find(weightName) != weights_mem_map.end()) + return weights_mem_map[weightName]; + else + return nullptr; + } +public: + // mkldnn formatted weights(filer data) memory from first iteration + // saved by weights name + std::map> weights_mem_map; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.cc b/onnxruntime/core/providers/mkldnn/nn/conv.cc index 355562bf5adcd..edd85eb17a675 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.cc +++ b/onnxruntime/core/providers/mkldnn/nn/conv.cc @@ -402,20 +402,27 @@ Status Conv::Compute(OpKernelContext* context) const { src_data = static_cast(dst.get_data_handle()); } - // Reorder filter memory layout if necessary. - if (filter_format != conv_primitive->GetFilterMemoryFormat()) { - auto pd = mkldnn::memory::primitive_desc(mkldnn::memory::desc(filter_dims_mkl, - MklDnnType(), - filter_format), - cpu_engine); - mkldnn::memory src = mkldnn::memory(pd, (void*)filter_data); - // allocate the size queried from memory primitive desc. it may not match tensor logical size due to - // mkldnn using padding to allow use of blocked format. - filter_reorder_buffer = IAllocator::MakeUniquePtr(alloc, conv_primitive->GetFilterSize()); - mkldnn::memory dst = mkldnn::memory(conv_fwd_pd->weights_primitive_desc(), filter_reorder_buffer.get()); - MemoryReorderParams params(src, dst); - DoReorder(params); - filter_data = static_cast(dst.get_data_handle()); + // Reorder filter memory layout if necessary + // Avoid data reordering. Save filter memory in mkldnn format from first iteration + // in execution provider mapped by weight name. + std::string weightName = OpKernel::Node().InputDefs()[1]->Name(); + std::shared_ptr filter_dst_mem = provider_->GetWeightMemory(weightName); + + if (filter_dst_mem == nullptr) { + if (filter_format != conv_primitive->GetFilterMemoryFormat()) { + auto pd = mkldnn::memory::primitive_desc(mkldnn::memory::desc( + filter_dims_mkl, MklDnnType(), filter_format), cpu_engine); + mkldnn::memory src = mkldnn::memory(pd, (void*)filter_data); + filter_reorder_buffer = IAllocator::MakeUniquePtr(alloc, conv_primitive->GetFilterSize()); + filter_dst_mem.reset( + new mkldnn::memory(conv_fwd_pd->weights_primitive_desc(), filter_reorder_buffer.get())); + MemoryReorderParams params(src, *filter_dst_mem); + DoReorder(params); + filter_data = static_cast(filter_dst_mem->get_data_handle()); + provider_->weights_mem_map[weightName] = filter_dst_mem; + } + } else { + filter_data = static_cast(filter_dst_mem->get_data_handle()); } // Allocate dst buffer if reorder is necessary diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.h b/onnxruntime/core/providers/mkldnn/nn/conv.h index 8189a7ca6ba10..7059f81fc2fe1 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.h +++ b/onnxruntime/core/providers/mkldnn/nn/conv.h @@ -4,18 +4,26 @@ #pragma once #include "core/framework/op_kernel.h" #include "core/providers/cpu/nn/conv.h" +#include "../mkldnn_execution_provider.h" namespace onnxruntime { namespace mkl_dnn { + template class Conv final : public onnxruntime::Conv { public: Conv(const OpKernelInfo& info) : onnxruntime::Conv(info) { + if (info.GetExecutionProvider()->Type() == kMklDnnExecutionProvider) { + provider_ = (const_cast( + dynamic_cast(info.GetExecutionProvider()))); + } } Status Compute(OpKernelContext* context) const override; private: + MKLDNNExecutionProvider * provider_; + }; } // namespace mkl_dnn } // namespace onnxruntime From 9787cc841c203de141aca980c043004f74ae00a8 Mon Sep 17 00:00:00 2001 From: "Yalachigere, Sreekanth V" Date: Mon, 10 Dec 2018 10:31:25 -0800 Subject: [PATCH 47/56] check provider_ for nullptr --- onnxruntime/core/providers/mkldnn/nn/conv.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.cc b/onnxruntime/core/providers/mkldnn/nn/conv.cc index edd85eb17a675..f6de43b6a504e 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.cc +++ b/onnxruntime/core/providers/mkldnn/nn/conv.cc @@ -406,7 +406,9 @@ Status Conv::Compute(OpKernelContext* context) const { // Avoid data reordering. Save filter memory in mkldnn format from first iteration // in execution provider mapped by weight name. std::string weightName = OpKernel::Node().InputDefs()[1]->Name(); - std::shared_ptr filter_dst_mem = provider_->GetWeightMemory(weightName); + std::shared_ptr filter_dst_mem = nullptr; + if(provider_ != nullptr) + filter_dst_mem = provider_->GetWeightMemory(weightName); if (filter_dst_mem == nullptr) { if (filter_format != conv_primitive->GetFilterMemoryFormat()) { @@ -419,7 +421,8 @@ Status Conv::Compute(OpKernelContext* context) const { MemoryReorderParams params(src, *filter_dst_mem); DoReorder(params); filter_data = static_cast(filter_dst_mem->get_data_handle()); - provider_->weights_mem_map[weightName] = filter_dst_mem; + if (provider_ != nullptr) + provider_->weights_mem_map[weightName] = filter_dst_mem; } } else { filter_data = static_cast(filter_dst_mem->get_data_handle()); From 4460531e712a556198d87db5401e8aa4bcb227dc Mon Sep 17 00:00:00 2001 From: "Yalachigere, Sreekanth V" Date: Mon, 10 Dec 2018 16:17:48 -0800 Subject: [PATCH 48/56] PR Review changes: thread safe. weights map object private --- .../providers/mkldnn/mkldnn_execution_provider.h | 16 +++++++++++----- onnxruntime/core/providers/mkldnn/nn/conv.cc | 10 ++++++---- onnxruntime/core/providers/mkldnn/nn/conv.h | 4 +--- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h index 13685856e8e3a..911dd2a1eb22f 100644 --- a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h +++ b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h @@ -41,12 +41,18 @@ class MKLDNNExecutionProvider : public IExecutionProvider { virtual std::shared_ptr GetKernelRegistry() const override; std::shared_ptr GetWeightMemory(std::string weightName) { - if (weights_mem_map.find(weightName) != weights_mem_map.end()) - return weights_mem_map[weightName]; - else - return nullptr; + + auto iter = weights_mem_map.find(weightName); + if (iter != weights_mem_map.end()) + return iter->second; + return nullptr; + } + + std::map>& GetWeightsMap() { + return weights_mem_map; } -public: + +private: // mkldnn formatted weights(filer data) memory from first iteration // saved by weights name std::map> weights_mem_map; diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.cc b/onnxruntime/core/providers/mkldnn/nn/conv.cc index f6de43b6a504e..81ad0b4f1c9e3 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.cc +++ b/onnxruntime/core/providers/mkldnn/nn/conv.cc @@ -407,8 +407,7 @@ Status Conv::Compute(OpKernelContext* context) const { // in execution provider mapped by weight name. std::string weightName = OpKernel::Node().InputDefs()[1]->Name(); std::shared_ptr filter_dst_mem = nullptr; - if(provider_ != nullptr) - filter_dst_mem = provider_->GetWeightMemory(weightName); + filter_dst_mem = provider_->GetWeightMemory(weightName); if (filter_dst_mem == nullptr) { if (filter_format != conv_primitive->GetFilterMemoryFormat()) { @@ -421,8 +420,11 @@ Status Conv::Compute(OpKernelContext* context) const { MemoryReorderParams params(src, *filter_dst_mem); DoReorder(params); filter_data = static_cast(filter_dst_mem->get_data_handle()); - if (provider_ != nullptr) - provider_->weights_mem_map[weightName] = filter_dst_mem; + { + // make assignment threadsafe + std::lock_guard lock(mutex_); + provider_->GetWeightsMap()[weightName] = filter_dst_mem; + } } } else { filter_data = static_cast(filter_dst_mem->get_data_handle()); diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.h b/onnxruntime/core/providers/mkldnn/nn/conv.h index 7059f81fc2fe1..8dde0c899f577 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.h +++ b/onnxruntime/core/providers/mkldnn/nn/conv.h @@ -13,17 +13,15 @@ template class Conv final : public onnxruntime::Conv { public: Conv(const OpKernelInfo& info) : onnxruntime::Conv(info) { - if (info.GetExecutionProvider()->Type() == kMklDnnExecutionProvider) { provider_ = (const_cast( dynamic_cast(info.GetExecutionProvider()))); - } } Status Compute(OpKernelContext* context) const override; private: MKLDNNExecutionProvider * provider_; - + mutable std::mutex mutex_; }; } // namespace mkl_dnn } // namespace onnxruntime From dbb53a1c3187b18a5e1345a4a0d5daf0c0228900 Mon Sep 17 00:00:00 2001 From: "Yalachigere, Sreekanth V" Date: Tue, 11 Dec 2018 09:38:33 -0800 Subject: [PATCH 49/56] using Conv Parameters key to make weights id unique --- onnxruntime/core/providers/mkldnn/nn/conv.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.cc b/onnxruntime/core/providers/mkldnn/nn/conv.cc index 81ad0b4f1c9e3..831c7b620d620 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.cc +++ b/onnxruntime/core/providers/mkldnn/nn/conv.cc @@ -352,6 +352,9 @@ Status Conv::Compute(OpKernelContext* context) const { dst_dims_mkl, strides_mkl, dilations_mkl, padding_left_mkl, padding_right_mkl); ConvPrimitive* conv_primitive = ConvPrimitivePool::Get(conv_params); + + std::string convString = conv_params.ToString(); + auto conv_fwd_pd = conv_primitive->GetPrimitiveDesc(); mkldnn::engine& cpu_engine = GetEngine(); @@ -405,9 +408,9 @@ Status Conv::Compute(OpKernelContext* context) const { // Reorder filter memory layout if necessary // Avoid data reordering. Save filter memory in mkldnn format from first iteration // in execution provider mapped by weight name. - std::string weightName = OpKernel::Node().InputDefs()[1]->Name(); + std::string weightKey = convString + "-" + OpKernel::Node().InputDefs()[1]->Name(); std::shared_ptr filter_dst_mem = nullptr; - filter_dst_mem = provider_->GetWeightMemory(weightName); + filter_dst_mem = provider_->GetWeightMemory(weightKey); if (filter_dst_mem == nullptr) { if (filter_format != conv_primitive->GetFilterMemoryFormat()) { @@ -423,7 +426,7 @@ Status Conv::Compute(OpKernelContext* context) const { { // make assignment threadsafe std::lock_guard lock(mutex_); - provider_->GetWeightsMap()[weightName] = filter_dst_mem; + provider_->GetWeightsMap()[weightKey] = filter_dst_mem; } } } else { From f321cf9a20495b2103ff34bd5f4c931e225b86a3 Mon Sep 17 00:00:00 2001 From: "Yalachigere, Sreekanth V" Date: Mon, 10 Dec 2018 16:17:48 -0800 Subject: [PATCH 50/56] PR Review changes: thread safe. weights map object private --- onnxruntime/core/providers/mkldnn/nn/conv.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.cc b/onnxruntime/core/providers/mkldnn/nn/conv.cc index 831c7b620d620..e452937e4d499 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.cc +++ b/onnxruntime/core/providers/mkldnn/nn/conv.cc @@ -408,7 +408,7 @@ Status Conv::Compute(OpKernelContext* context) const { // Reorder filter memory layout if necessary // Avoid data reordering. Save filter memory in mkldnn format from first iteration // in execution provider mapped by weight name. - std::string weightKey = convString + "-" + OpKernel::Node().InputDefs()[1]->Name(); + std::string weightKey = OpKernel::Node().InputDefs()[1]->Name(); std::shared_ptr filter_dst_mem = nullptr; filter_dst_mem = provider_->GetWeightMemory(weightKey); From 591fa08e3760eb8564fda2a6d58de39edb6cf0da Mon Sep 17 00:00:00 2001 From: "Yalachigere, Sreekanth V" Date: Wed, 12 Dec 2018 09:59:33 -0800 Subject: [PATCH 51/56] removed convParam string from weight key. Weight Id is unique --- onnxruntime/core/providers/mkldnn/nn/conv.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.cc b/onnxruntime/core/providers/mkldnn/nn/conv.cc index e452937e4d499..db3e0e55017de 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.cc +++ b/onnxruntime/core/providers/mkldnn/nn/conv.cc @@ -353,8 +353,6 @@ Status Conv::Compute(OpKernelContext* context) const { padding_left_mkl, padding_right_mkl); ConvPrimitive* conv_primitive = ConvPrimitivePool::Get(conv_params); - std::string convString = conv_params.ToString(); - auto conv_fwd_pd = conv_primitive->GetPrimitiveDesc(); mkldnn::engine& cpu_engine = GetEngine(); From 42a710ab64903c37884b3d3c4f73c53d4f2f917f Mon Sep 17 00:00:00 2001 From: "Yalachigere, Sreekanth V" Date: Mon, 10 Dec 2018 10:31:25 -0800 Subject: [PATCH 52/56] check provider_ for nullptr --- onnxruntime/core/providers/mkldnn/nn/conv.cc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.cc b/onnxruntime/core/providers/mkldnn/nn/conv.cc index db3e0e55017de..6424742042816 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.cc +++ b/onnxruntime/core/providers/mkldnn/nn/conv.cc @@ -406,9 +406,16 @@ Status Conv::Compute(OpKernelContext* context) const { // Reorder filter memory layout if necessary // Avoid data reordering. Save filter memory in mkldnn format from first iteration // in execution provider mapped by weight name. +<<<<<<< HEAD std::string weightKey = OpKernel::Node().InputDefs()[1]->Name(); std::shared_ptr filter_dst_mem = nullptr; filter_dst_mem = provider_->GetWeightMemory(weightKey); +======= + std::string weightName = OpKernel::Node().InputDefs()[1]->Name(); + std::shared_ptr filter_dst_mem = nullptr; + if(provider_ != nullptr) + filter_dst_mem = provider_->GetWeightMemory(weightName); +>>>>>>> check provider_ for nullptr if (filter_dst_mem == nullptr) { if (filter_format != conv_primitive->GetFilterMemoryFormat()) { @@ -421,11 +428,16 @@ Status Conv::Compute(OpKernelContext* context) const { MemoryReorderParams params(src, *filter_dst_mem); DoReorder(params); filter_data = static_cast(filter_dst_mem->get_data_handle()); +<<<<<<< HEAD { // make assignment threadsafe std::lock_guard lock(mutex_); provider_->GetWeightsMap()[weightKey] = filter_dst_mem; } +======= + if (provider_ != nullptr) + provider_->weights_mem_map[weightName] = filter_dst_mem; +>>>>>>> check provider_ for nullptr } } else { filter_data = static_cast(filter_dst_mem->get_data_handle()); From dff5560b230f298f0f44c6a71dc263203e9343e8 Mon Sep 17 00:00:00 2001 From: "Yalachigere, Sreekanth V" Date: Tue, 11 Dec 2018 09:38:33 -0800 Subject: [PATCH 53/56] using Conv Parameters key to make weights id unique --- onnxruntime/core/providers/mkldnn/nn/conv.cc | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.cc b/onnxruntime/core/providers/mkldnn/nn/conv.cc index 6424742042816..426bac27a93a4 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.cc +++ b/onnxruntime/core/providers/mkldnn/nn/conv.cc @@ -353,6 +353,11 @@ Status Conv::Compute(OpKernelContext* context) const { padding_left_mkl, padding_right_mkl); ConvPrimitive* conv_primitive = ConvPrimitivePool::Get(conv_params); +<<<<<<< HEAD +======= + std::string convString = conv_params.ToString(); + +>>>>>>> using Conv Parameters key to make weights id unique auto conv_fwd_pd = conv_primitive->GetPrimitiveDesc(); mkldnn::engine& cpu_engine = GetEngine(); @@ -406,16 +411,10 @@ Status Conv::Compute(OpKernelContext* context) const { // Reorder filter memory layout if necessary // Avoid data reordering. Save filter memory in mkldnn format from first iteration // in execution provider mapped by weight name. -<<<<<<< HEAD - std::string weightKey = OpKernel::Node().InputDefs()[1]->Name(); + + std::string weightKey = convString + "-" + OpKernel::Node().InputDefs()[1]->Name(); std::shared_ptr filter_dst_mem = nullptr; filter_dst_mem = provider_->GetWeightMemory(weightKey); -======= - std::string weightName = OpKernel::Node().InputDefs()[1]->Name(); - std::shared_ptr filter_dst_mem = nullptr; - if(provider_ != nullptr) - filter_dst_mem = provider_->GetWeightMemory(weightName); ->>>>>>> check provider_ for nullptr if (filter_dst_mem == nullptr) { if (filter_format != conv_primitive->GetFilterMemoryFormat()) { @@ -428,16 +427,11 @@ Status Conv::Compute(OpKernelContext* context) const { MemoryReorderParams params(src, *filter_dst_mem); DoReorder(params); filter_data = static_cast(filter_dst_mem->get_data_handle()); -<<<<<<< HEAD { // make assignment threadsafe std::lock_guard lock(mutex_); provider_->GetWeightsMap()[weightKey] = filter_dst_mem; } -======= - if (provider_ != nullptr) - provider_->weights_mem_map[weightName] = filter_dst_mem; ->>>>>>> check provider_ for nullptr } } else { filter_data = static_cast(filter_dst_mem->get_data_handle()); From 47359cbd4493e7ec0029313ed0b91654d5afc2da Mon Sep 17 00:00:00 2001 From: "Yalachigere, Sreekanth V" Date: Thu, 20 Dec 2018 21:35:58 -0800 Subject: [PATCH 54/56] weight name as key --- onnxruntime/core/providers/mkldnn/nn/conv.cc | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.cc b/onnxruntime/core/providers/mkldnn/nn/conv.cc index 426bac27a93a4..82b9fd42f7e56 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.cc +++ b/onnxruntime/core/providers/mkldnn/nn/conv.cc @@ -353,11 +353,6 @@ Status Conv::Compute(OpKernelContext* context) const { padding_left_mkl, padding_right_mkl); ConvPrimitive* conv_primitive = ConvPrimitivePool::Get(conv_params); -<<<<<<< HEAD -======= - std::string convString = conv_params.ToString(); - ->>>>>>> using Conv Parameters key to make weights id unique auto conv_fwd_pd = conv_primitive->GetPrimitiveDesc(); mkldnn::engine& cpu_engine = GetEngine(); @@ -412,8 +407,8 @@ Status Conv::Compute(OpKernelContext* context) const { // Avoid data reordering. Save filter memory in mkldnn format from first iteration // in execution provider mapped by weight name. - std::string weightKey = convString + "-" + OpKernel::Node().InputDefs()[1]->Name(); - std::shared_ptr filter_dst_mem = nullptr; + std::string weightKey = OpKernel::Node().InputDefs()[1]->Name(); + std::shared_ptr filter_dst_mem =nullptr; filter_dst_mem = provider_->GetWeightMemory(weightKey); if (filter_dst_mem == nullptr) { From 345b440ba36f935d3d4d510e954250cca16a16fb Mon Sep 17 00:00:00 2001 From: "Yalachigere, Sreekanth V" Date: Thu, 20 Dec 2018 21:44:40 -0800 Subject: [PATCH 55/56] weights as key --- cmake/external/onnx | 2 +- onnxruntime/core/providers/mkldnn/nn/conv.cc | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/cmake/external/onnx b/cmake/external/onnx index 0a4d5abdf4939..0c8d857bb1624 160000 --- a/cmake/external/onnx +++ b/cmake/external/onnx @@ -1 +1 @@ -Subproject commit 0a4d5abdf4939ab0842a5eadcc16a3bf0738f901 +Subproject commit 0c8d857bb162431912b255d5c0e773fb7c131a65 diff --git a/onnxruntime/core/providers/mkldnn/nn/conv.cc b/onnxruntime/core/providers/mkldnn/nn/conv.cc index 73a92269c915e..7fe0152d42e9f 100644 --- a/onnxruntime/core/providers/mkldnn/nn/conv.cc +++ b/onnxruntime/core/providers/mkldnn/nn/conv.cc @@ -406,14 +406,8 @@ Status Conv::Compute(OpKernelContext* context) const { // Reorder filter memory layout if necessary // Avoid data reordering. Save filter memory in mkldnn format from first iteration // in execution provider mapped by weight name. -<<<<<<< HEAD - std::string weightKey = OpKernel::Node().InputDefs()[1]->Name(); std::shared_ptr filter_dst_mem =nullptr; -======= - std::string weightKey = OpKernel::Node().InputDefs()[1]->Name(); - std::shared_ptr filter_dst_mem = nullptr; ->>>>>>> e98e362d788712477973eec3373dcbacc81a1fdd filter_dst_mem = provider_->GetWeightMemory(weightKey); if (filter_dst_mem == nullptr) { From 431344e1e7e28f0ce8615d0ebbebafe9a80b4d28 Mon Sep 17 00:00:00 2001 From: "Yalachigere, Sreekanth V" Date: Fri, 21 Dec 2018 00:17:18 -0800 Subject: [PATCH 56/56] Relu and Sum --- .../providers/cpu/activation/activations.h | 2 +- .../providers/cpu/math/element_wise_ops.h | 2 +- .../core/providers/cpu/nn/batch_norm.h | 4 +- .../mkldnn/activation/activations.cc | 204 +++++++++++++ .../providers/mkldnn/activation/activations.h | 20 ++ onnxruntime/core/providers/mkldnn/math/sum.cc | 231 ++++++++++++++ onnxruntime/core/providers/mkldnn/math/sum.h | 22 ++ onnxruntime/core/providers/mkldnn/memcpy_s.h | 13 + .../mkldnn/mkldnn_execution_provider.cc | 6 + .../core/providers/mkldnn/nn/batch_norm.cc | 287 ++++++++++++++++++ .../core/providers/mkldnn/nn/batch_norm.h | 18 ++ 11 files changed, 805 insertions(+), 4 deletions(-) create mode 100644 onnxruntime/core/providers/mkldnn/activation/activations.cc create mode 100644 onnxruntime/core/providers/mkldnn/activation/activations.h create mode 100644 onnxruntime/core/providers/mkldnn/math/sum.cc create mode 100644 onnxruntime/core/providers/mkldnn/math/sum.h create mode 100644 onnxruntime/core/providers/mkldnn/memcpy_s.h create mode 100644 onnxruntime/core/providers/mkldnn/nn/batch_norm.cc create mode 100644 onnxruntime/core/providers/mkldnn/nn/batch_norm.h diff --git a/onnxruntime/core/providers/cpu/activation/activations.h b/onnxruntime/core/providers/cpu/activation/activations.h index 42c434a2e276a..1283f570a1121 100644 --- a/onnxruntime/core/providers/cpu/activation/activations.h +++ b/onnxruntime/core/providers/cpu/activation/activations.h @@ -90,7 +90,7 @@ class ParametricSoftplus final : public OpKernel { }; template -class Relu final : public OpKernel { +class Relu : public OpKernel { public: Relu(const OpKernelInfo& info) : OpKernel(info) {} diff --git a/onnxruntime/core/providers/cpu/math/element_wise_ops.h b/onnxruntime/core/providers/cpu/math/element_wise_ops.h index f36974dec9ec2..ce384776d8d54 100644 --- a/onnxruntime/core/providers/cpu/math/element_wise_ops.h +++ b/onnxruntime/core/providers/cpu/math/element_wise_ops.h @@ -139,7 +139,7 @@ class Log final : public OpKernel { }; template -class Sum_6 final : public OpKernel { +class Sum_6 : public OpKernel { public: Sum_6(const OpKernelInfo& info) : OpKernel(info) { } diff --git a/onnxruntime/core/providers/cpu/nn/batch_norm.h b/onnxruntime/core/providers/cpu/nn/batch_norm.h index f5e66849d6a26..66601c9c2e26d 100644 --- a/onnxruntime/core/providers/cpu/nn/batch_norm.h +++ b/onnxruntime/core/providers/cpu/nn/batch_norm.h @@ -27,7 +27,7 @@ namespace onnxruntime { template -class BatchNorm final : public OpKernel { +class BatchNorm : public OpKernel { public: BatchNorm(const OpKernelInfo& op_kernel_info) : OpKernel(op_kernel_info) { float tmp_eplison; @@ -38,7 +38,7 @@ class BatchNorm final : public OpKernel { Status Compute(OpKernelContext* p_op_kernel_context) const override; - private: + protected: float epsilon_ = 1e-5f; int64_t is_test_; // ignored in this implementation since we're doing inferencing only. }; diff --git a/onnxruntime/core/providers/mkldnn/activation/activations.cc b/onnxruntime/core/providers/mkldnn/activation/activations.cc new file mode 100644 index 0000000000000..e43d7aa3934a3 --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/activation/activations.cc @@ -0,0 +1,204 @@ +// Copyright(C) 2018 Intel Corporation +// Licensed under the MIT License + +#ifdef _WIN32 +#pragma warning(disable : 4244) +#endif + +#include "core/providers/mkldnn/mkldnn_common.h" +#include "core/providers/mkldnn/activation/activations.h" +#include "core/providers/mkldnn/mkldnn_fwd.h" + +namespace onnxruntime { +namespace mkl_dnn { + +namespace { +// Struct which encapsulates parameters for MKLDNN Pool primitive. +struct ReluParams { + mkldnn::memory::dims& src_dims; + mkldnn::memory::dims& dst_dims; + size_t num_dimensions; + + ReluParams(mkldnn::memory::dims& src_dims, mkldnn::memory::dims& dst_dims, + size_t dimensions = 0) + : src_dims(src_dims), + dst_dims(dst_dims), + num_dimensions(dimensions) {} + + // Used as the key for Pool Primitive Reuse Pool. + std::string ToString() const { + std::string key; + key.reserve(64); + key.append("Relu_"); + AddDimsToKey(key, src_dims); + AddDimsToKey(key, dst_dims); + return key; + } +}; + +template +class ReluPrimitive final : public PrimitiveBase { + public: + explicit ReluPrimitive(const ReluParams& params) + : cpu_engine_(GetEngine()) { + context_.stream.reset(new mkldnn::stream(mkldnn::stream::kind::eager)); + if (context_.relu_fwd == nullptr) { + Initialize(params); + } + } + + ~ReluPrimitive() = default; + + void Compute(const T* src_data, const T* dst_data) { + context_.src_mem->set_data_handle( + static_cast(const_cast(src_data))); + context_.dst_mem->set_data_handle( + static_cast(const_cast(dst_data))); + context_.stream->submit(context_.net); + + context_.src_mem->set_data_handle(nullptr); + context_.dst_mem->set_data_handle(nullptr); + return; + } + + std::unique_ptr + GetDstMemoryDesc() const { return context_.dst_md; } + + std::unique_ptr + GetPrimitiveDesc() const { + return context_.relu_fwd_pd; + } + + private: + struct ReluContext { + mkldnn::memory::format src_fmt; + + std::unique_ptr src_mem; + std::unique_ptr dst_mem; + + size_t src_size; + size_t dst_size; + + std::unique_ptr fwd_desc; + std::unique_ptr relu_fwd_pd; + std::unique_ptr relu_fwd; + + std::unique_ptr src_md; + std::unique_ptr dst_md; + + std::unique_ptr stream; + std::vector net; + }; + + void Initialize(const ReluParams& params) { + + mkldnn::memory::format fmt = mkldnn::memory::format::any; + switch (params.num_dimensions) { + case 1: { fmt = mkldnn::memory::format::x; break; } + case 2: { fmt = mkldnn::memory::format::nc; break; } + case 3: { fmt = mkldnn::memory::format::ntc; break; } + case 4: { fmt = mkldnn::memory::format::nchw; break; } + case 5: { fmt = mkldnn::memory::format::ncdhw; break; } + default: { fmt = mkldnn::memory::format::any; break; } + } + + context_.src_md.reset(new mkldnn::memory::desc({ params.src_dims}, MklDnnType(), fmt)); + + mkldnn::algorithm algo = mkldnn::algorithm::eltwise_relu; + context_.fwd_desc.reset(new mkldnn::eltwise_forward::desc( + mkldnn::prop_kind::forward_inference, algo, *context_.src_md, 0)); + + context_.relu_fwd_pd.reset(new mkldnn::eltwise_forward::primitive_desc( + *context_.fwd_desc, cpu_engine_)); + + context_.src_fmt = static_cast( + context_.relu_fwd_pd.get()->src_primitive_desc().desc().data.format); + + context_.src_size = context_.relu_fwd_pd.get()->src_primitive_desc().get_size(); + context_.dst_size = context_.relu_fwd_pd.get()->dst_primitive_desc().get_size(); + + context_.src_mem.reset(new mkldnn::memory(context_.relu_fwd_pd.get()->src_primitive_desc(), nullptr)); + context_.dst_mem.reset(new mkldnn::memory(context_.relu_fwd_pd.get()->dst_primitive_desc(), nullptr)); + context_.relu_fwd.reset( + new mkldnn::eltwise_forward(*context_.relu_fwd_pd, *context_.src_mem, *context_.dst_mem)); + context_.net.push_back(*context_.relu_fwd); + } + + ReluContext context_; + mkldnn::engine& cpu_engine_; +}; + +// Pool which allows for reuse of MKLDNN Relu primitives which are expensive +// to instantiate. To address thread safety, the primitives are stored in a map +// on thread local storage. +template +class ReluPrimitivePool : public PrimitivePool { + public: + static ReluPrimitive* Get(const ReluParams& params) { + ReluPrimitive* primitive = dynamic_cast*>( + ReluPrimitivePool::GetInstance().GetPrimitive(params.ToString())); + + if (primitive == nullptr) { + auto relu_primitive = std::make_unique>(params); + primitive = relu_primitive.get(); + ReluPrimitivePool::GetInstance().SetPrimitive(params.ToString(), + std::move(relu_primitive)); + } + return primitive; + } + + private: + ReluPrimitivePool() = default; + ~ReluPrimitivePool() = default; + + static ReluPrimitivePool& GetInstance() { + static ReluPrimitivePool pool; + return pool; + } +}; +} // namespace + +template +Status Relu::Compute(OpKernelContext* context) const { + const Tensor* X = context->Input(0); + Tensor* Y = context->Output(0, X->Shape()); + + const TensorShape& x_shape = X->Shape(); + const auto& x_dims = x_shape.GetDims(); + + if (X->Shape().NumDimensions() > 5 ) { + return onnxruntime::Relu::Compute(context); + } + + const TensorShape& y_shape = Y->Shape(); + auto& y_dims = y_shape.GetDims(); + + const T* src_data = X->template Data(); + T* dst_data = Y->template MutableData(); + + mkldnn::memory::dims src_dims_mkl(x_dims.begin(), x_dims.end()); + mkldnn::memory::dims dst_dims_mkl(y_dims.begin(), y_dims.end()); + + try { + ReluParams pool_params(src_dims_mkl, dst_dims_mkl, x_shape.NumDimensions()); + ReluPrimitive* relulPrimitive = ReluPrimitivePool::Get(pool_params); + + relulPrimitive->Compute(src_data, dst_data); + } catch (const mkldnn::error& e) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Status: ", e.status, + ", message: ", e.message.c_str()); + } + + return Status::OK(); +} + +ONNX_OPERATOR_KERNEL_EX( + Relu, + kOnnxDomain, + 6, + kMklDnnExecutionProvider, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Relu); + +} // namespace mkl_dnn +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/core/providers/mkldnn/activation/activations.h b/onnxruntime/core/providers/mkldnn/activation/activations.h new file mode 100644 index 0000000000000..be8ba3537b91c --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/activation/activations.h @@ -0,0 +1,20 @@ +// Copyright(C) 2018 Intel Corporation +// Licensed under the MIT License + +#pragma once +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/activation/activations.h" + +namespace onnxruntime { +namespace mkl_dnn { + +template +class Relu : public onnxruntime::Relu { + public: + Relu(const OpKernelInfo& info) : onnxruntime::Relu(info) {} + + Status Compute(OpKernelContext* context) const override; +}; + +} // namespace mkl_dnn +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/core/providers/mkldnn/math/sum.cc b/onnxruntime/core/providers/mkldnn/math/sum.cc new file mode 100644 index 0000000000000..e7d3970f44b98 --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/math/sum.cc @@ -0,0 +1,231 @@ +// Copyright(C) 2018 Intel Corporation +// Licensed under the MIT License + +#ifdef _WIN32 +#pragma warning(disable : 4244) +#endif + +#include "core/providers/mkldnn/mkldnn_common.h" +#include "core/providers/mkldnn/math/sum.h" +#include "core/providers/mkldnn/mkldnn_fwd.h" + +namespace onnxruntime { +namespace mkl_dnn { + +namespace { +// Struct which encapsulates parameters for MKLDNN Sum primitives. +struct SumParams { + const std::vector& src_dims; + const mkldnn::memory::dims& dst_dim; + const int num_inputs; + const int num_dimensions; + + SumParams(const std::vector& dims, + const mkldnn::memory::dims& dst_dims, const int numinputs, + const int dimensions) + : src_dims(dims), + dst_dim(dst_dims), + num_inputs(numinputs), + num_dimensions(dimensions) {} + + // Used as the key for Sum Primitive Reuse Sum. + std::string ToString() const { + std::string key; + key.reserve(64); + key.append("sum_"); + for (size_t i = 0; i < src_dims.size(); i++) { + AddDimsToKey(key, src_dims[i]); + } + AddDimsToKey(key, dst_dim); + return key; + } +}; + +template +class SumPrimitive final : public PrimitiveBase { + public: + explicit SumPrimitive(const SumParams& params) + : cpu_engine_(GetEngine()) { + context_.stream.reset(new mkldnn::stream(mkldnn::stream::kind::eager)); + if (context_.sum_pd == nullptr) { + Initialize(params); + } + } + + ~SumPrimitive() = default; + + void Compute(OpKernelContext* context, int numinputs) { + const Tensor* X1 = context->Input(0); + Tensor* Y = context->Output(0, X1->Shape()); + T* dst_data = Y->template MutableData(); + + context_.dst_mem->set_data_handle( + static_cast(static_cast(dst_data))); + + for (int i = 0; i < numinputs; i++) { + const Tensor* X = context->Input(i); + const T* src_data = X->template Data(); + context_.srcs_memory[i].set_data_handle( + static_cast(const_cast(src_data))); + } + context_.stream->submit(context_.net); + + for (int i = 0; i < numinputs; i++) { + context_.srcs_memory[i].set_data_handle(nullptr); + } + + } + + std::unique_ptr GetDstMemoryDesc() const { + return context_.dst_md; + } + + std::unique_ptr + GetPrimitiveDesc() const { + return context_.sum_pd; + } + + private: + struct SumContext { + std::unique_ptr src_md; + std::unique_ptr dst_md; + + std::vector srcs_memory; + std::unique_ptr dst_mem; + + std::vector srcs_pd; + std::unique_ptr src_mpd; + std::unique_ptr dst_pd; + std::unique_ptr sum_pd; + + std::unique_ptr stream; + std::vector net; + }; + + void Initialize(const SumParams& params) { + std::vector coeff; + + mkldnn::memory::format fmt = mkldnn::memory::format::any; + switch (params.num_dimensions) { + case 1: { fmt = mkldnn::memory::format::x; break; } + case 2: { fmt = mkldnn::memory::format::nc; break; } + case 3: { fmt = mkldnn::memory::format::ntc; break; } + case 4: { fmt = mkldnn::memory::format::nchw; break; } + case 5: { fmt = mkldnn::memory::format::ncdhw; break; } + default: { fmt = mkldnn::memory::format::any; break; } + } + + for (int i = 0; i < params.num_inputs; i++) { + context_.src_md.reset( + new mkldnn::memory::desc({params.src_dims[i]}, MklDnnType(), fmt)); + auto mpd = mkldnn::memory::primitive_desc(*context_.src_md, cpu_engine_); + auto src_memory = mkldnn::memory(mpd, nullptr); + + context_.srcs_pd.push_back(mpd); + context_.srcs_memory.push_back(src_memory); + coeff.push_back(1.0); + } + + std::unique_ptr dst; + context_.dst_md.reset(new mkldnn::memory::desc( + {params.dst_dim}, MklDnnType(), mkldnn::memory::format::any)); + context_.sum_pd.reset(new mkldnn::sum::primitive_desc( + *context_.dst_md, coeff, context_.srcs_pd)); + context_.dst_mem.reset(new mkldnn::memory( + context_.sum_pd->dst_primitive_desc(), nullptr)); + + std::vector inputs; + for (int i = 0; i < params.num_inputs; i++) { + inputs.push_back(context_.srcs_memory[i]); + } + auto c = mkldnn::sum(*context_.sum_pd, inputs, *context_.dst_mem); + context_.net.push_back(c); + } + + SumContext context_; + mkldnn::engine& cpu_engine_; +}; + +// Pool which allows for reuse of MKLDNN Sum primitives which are +// expensive to instantiate. To address thread safety, the primitives +// are stored in a map on thread local storage. + +template +class SumPrimitivePool : public PrimitivePool { + public: + static SumPrimitive* Get(const SumParams& params) { + SumPrimitive* primitive = dynamic_cast*>( + SumPrimitivePool::GetInstance().GetPrimitive(params.ToString())); + + if (primitive == nullptr) { + auto sum_primitive = std::make_unique>(params); + primitive = sum_primitive.get(); + SumPrimitivePool::GetInstance().SetPrimitive( + params.ToString(), std::move(sum_primitive)); + } + return primitive; + } + + private: + SumPrimitivePool() = default; + ~SumPrimitivePool() = default; + + static SumPrimitivePool& GetInstance() { + static SumPrimitivePool pool; + return pool; + } +}; +} // namespace_ + +template +Status Sum::Compute(OpKernelContext* context) const { + int num_inputs = static_cast(OpKernel::Node().InputDefs().size()); + + ORT_ENFORCE(num_inputs > 0, "MKLDNN Sum kernel: Must have at least one input"); + + if (num_inputs == 1) { + return onnxruntime::Sum_6::Compute(context); + } + + std::vector src_dims; + + const Tensor* X1 = context->Input(0); + Tensor* Y = context->Output(0, X1->Shape()); + int dimensions = static_cast(X1->Shape().NumDimensions()); + + const TensorShape& x_shape = X1->Shape(); + const auto& x_dims = x_shape.GetDims(); + mkldnn::memory::dims src_dim(x_dims.begin(), x_dims.end()); + + mkldnn::memory::dims dst_dims_mkl( + Y->Shape().GetDims().begin(), Y->Shape().GetDims().end()); + + for (int i = 0; i < num_inputs; i++) { + const Tensor* X = context->Input(i); + mkldnn::memory::dims src_dims_mkl( + X->Shape().GetDims().begin(), X->Shape().GetDims().end()); + src_dims.push_back(src_dims_mkl); + } + try { + SumParams parameters(src_dims, dst_dims_mkl, num_inputs, dimensions); + SumPrimitive* sum_primitive = SumPrimitivePool::Get(parameters); + ORT_RETURN_IF_NOT(sum_primitive != nullptr); + sum_primitive->Compute(context, num_inputs); + } catch (const mkldnn::error& e) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Status: ", e.status, + ", message: ", e.message.c_str()); + } + + return Status::OK(); +} + +ONNX_OPERATOR_KERNEL_EX( + Sum, + kOnnxDomain, + 6, + kMklDnnExecutionProvider, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + Sum); + +} // namespace mkl_dnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/math/sum.h b/onnxruntime/core/providers/mkldnn/math/sum.h new file mode 100644 index 0000000000000..79d5326c6d70f --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/math/sum.h @@ -0,0 +1,22 @@ +// Copyright(C) 2018 Intel Corporation +// Licensed under the MIT License + +#pragma once +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/math/element_wise_ops.h" +#include "../mkldnn_execution_provider.h" + +namespace onnxruntime { +namespace mkl_dnn { + +template +class Sum final : public onnxruntime::Sum_6 { + public: + Sum(const OpKernelInfo& info) : onnxruntime::Sum_6(info) {} + + Status Compute(OpKernelContext* context) const override; + +private: +}; +} // namespace mkl_dnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/memcpy_s.h b/onnxruntime/core/providers/mkldnn/memcpy_s.h new file mode 100644 index 0000000000000..3db97157118fb --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/memcpy_s.h @@ -0,0 +1,13 @@ +// Copyright(C) 2018 Intel Corporation +// Licensed under the MIT License + +#pragma once + +#ifdef _WIN32 +#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#endif + +// memcpy is deprecated. Replacing it with more secure equivalent memcpy_s +// +#define MEMCPY_S(dest, src, destsz, srcsz) memcpy(dest, src, MIN(destsz, srcsz)) + diff --git a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.cc b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.cc index d8889419a7251..299fcfafbbae0 100644 --- a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.cc +++ b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.cc @@ -65,6 +65,9 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 1, class ONNX_OPERATOR_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 7, Gemm); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 1, MemcpyFromHost); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 1, MemcpyToHost); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 6, Relu); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 6, Sum); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 7, BatchNormalization); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 7, 8, float, AveragePool); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 1, 8, float, GlobalAveragePool); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kMklDnnExecutionProvider, kOnnxDomain, 1, 7, float, MaxPool); @@ -77,6 +80,9 @@ void RegisterMKLDNNKernels(std::function fn) { fn(BuildKernel()); fn(BuildKernel()); fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); + fn(BuildKernel()); fn(BuildKernel()); fn(BuildKernel()); fn(BuildKernel()); diff --git a/onnxruntime/core/providers/mkldnn/nn/batch_norm.cc b/onnxruntime/core/providers/mkldnn/nn/batch_norm.cc new file mode 100644 index 0000000000000..b908415023f3f --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/nn/batch_norm.cc @@ -0,0 +1,287 @@ +// Copyright(C) 2018 Intel Corporation +// Licensed under the MIT License + +#ifdef _WIN32 +#pragma warning(disable : 4244) +#endif + +#include "core/providers/mkldnn/mkldnn_common.h" +#include "core/providers/mkldnn/nn/batch_norm.h" +#include "core/providers/mkldnn/mkldnn_fwd.h" +#include "core/providers/mkldnn/memcpy_s.h" +#include "core/providers/cpu/nn/batch_norm_helper.h" + +namespace onnxruntime { +namespace mkl_dnn { + +namespace { +// Struct which encapsulates parameters for MKLDNN BatchNorm primitive. +struct BatchNormParams { + const mkldnn::memory::dims& src_dims; + const mkldnn::memory::dims& scale_dims; + const mkldnn::memory::dims& b_dims; + const mkldnn::memory::dims& mean_dims; + const mkldnn::memory::dims& var_dims; + const mkldnn::memory::dims& dst_dims; + const float epsilon; + const int num_dimensions; + + BatchNormParams(const mkldnn::memory::dims& src_dims_mkl, + const mkldnn::memory::dims& scale_dims_mkl, + const mkldnn::memory::dims& b_dims_mkl, const mkldnn::memory::dims& mean_dims_mkl, + const mkldnn::memory::dims& var_dims_mkl, const mkldnn::memory::dims& dst_dims_mkl, + const float eps, const int dimensions) + : src_dims(src_dims_mkl), + scale_dims(scale_dims_mkl), + b_dims(b_dims_mkl), + mean_dims(mean_dims_mkl), + var_dims(var_dims_mkl), + dst_dims(dst_dims_mkl), + epsilon(eps), + num_dimensions(dimensions) {} + + // Used as the key for BatchNorm Primitive Reuse Pool. + std::string ToString() const { + std::string key; + key.reserve(128); + key.append("BatchNorm_"); + AddDimsToKey(key, src_dims); + AddDimsToKey(key, scale_dims); + AddDimsToKey(key, b_dims); + AddDimsToKey(key, mean_dims); + AddDimsToKey(key, var_dims); + AddDimsToKey(key, dst_dims); + return key; + } +}; + +template +class BatchNormPrimitive final : public PrimitiveBase { + public: + explicit BatchNormPrimitive(const BatchNormParams& params) + : cpu_engine_(GetEngine()) { + context_.stream.reset(new mkldnn::stream(mkldnn::stream::kind::eager)); + if (context_.batchnorm_fwd == nullptr) { + Initialize(params); + } + } + + ~BatchNormPrimitive() = default; + + void Compute(const T* src_data, const T* scale_data, const T* b_data, + const T* mean_data, const T* var_data, const T* dst_data, + int scale_dims_channels) { + context_.src_mem->set_data_handle( + static_cast(const_cast(src_data))); + context_.mean_mem->set_data_handle( + static_cast(const_cast(mean_data))); + context_.var_mem->set_data_handle( + static_cast(const_cast(var_data))); + context_.dst_mem->set_data_handle( + static_cast(const_cast(dst_data))); + + T* scaleShift_buf = static_cast(context_.scale_shift_mem->get_data_handle()); + + size_t src_bytes = sizeof(T) * scale_dims_channels; + size_t dst_bytes = sizeof(T) * scale_dims_channels; + + MEMCPY_S(scaleShift_buf, scale_data, src_bytes, dst_bytes); + MEMCPY_S(&scaleShift_buf[scale_dims_channels], b_data, src_bytes, dst_bytes); + context_.stream->submit(context_.net); + return; + } + + std::unique_ptr + GetPrimitiveDesc() const { + return context_.conv_fwd_pd; + } + + private: + struct BatchNormContext { + std::unique_ptr src_mem; + std::unique_ptr scale_shift_mem; + std::unique_ptr mean_mem; + std::unique_ptr var_mem; + std::unique_ptr dst_mem; + + std::unique_ptr src_md; + std::unique_ptr scale_shift_md; + std::unique_ptr mean_md; + std::unique_ptr var_md; + std::unique_ptr dst_md; + + std::unique_ptr batchnorm_fwd; + std::unique_ptr + batchnorm_fwd_pd; + + std::unique_ptr stream; + std::vector net; + }; + + void Initialize(const BatchNormParams& params) { + mkldnn::memory::format fmt = mkldnn::memory::format::any; + switch (params.num_dimensions) { + case 1: { fmt = mkldnn::memory::format::x; break; } + case 2: { fmt = mkldnn::memory::format::nc; break; } + case 3: { fmt = mkldnn::memory::format::ntc; break; } + case 4: { fmt = mkldnn::memory::format::nchw; break; } + case 5: { fmt = mkldnn::memory::format::ncdhw; break; } + default: { fmt = mkldnn::memory::format::any; break; } + } + context_.src_md.reset(new mkldnn::memory::desc( + { params.src_dims }, MklDnnType(), fmt)); + + context_.scale_shift_md.reset(new mkldnn::memory::desc( + { 2, params.scale_dims[0] }, MklDnnType(), mkldnn::memory::format::nc)); + + context_.mean_md.reset(new mkldnn::memory::desc( + { params.mean_dims }, MklDnnType(), mkldnn::memory::format::x)); + context_.var_md.reset(new mkldnn::memory::desc( + { params.var_dims }, MklDnnType(), mkldnn::memory::format::x)); + context_.dst_md.reset(new mkldnn::memory::desc( + { params.dst_dims }, MklDnnType(), fmt)); + + context_.src_mem.reset( + new mkldnn::memory({ *context_.src_md, cpu_engine_ }, nullptr)); + + // scale_shift_mem will allocate 2*C*sizeof(float) buffer + // + context_.scale_shift_mem.reset( + new mkldnn::memory({ *context_.scale_shift_md, cpu_engine_ })); + + context_.mean_mem.reset( + new mkldnn::memory({ *context_.mean_md, cpu_engine_ }, nullptr)); + context_.var_mem.reset( + new mkldnn::memory({ *context_.var_md, cpu_engine_ }, nullptr)); + + context_.batchnorm_fwd.reset(new mkldnn::batch_normalization_forward::desc( + mkldnn::prop_kind::forward_inference, *context_.src_md, params.epsilon, + mkldnn::batch_normalization_flag::use_scale_shift | + mkldnn::batch_normalization_flag::use_global_stats)); + + context_.batchnorm_fwd_pd.reset( + new mkldnn::batch_normalization_forward::primitive_desc( + *context_.batchnorm_fwd, cpu_engine_)); + + context_.dst_mem.reset( + new mkldnn::memory( + context_.batchnorm_fwd_pd->dst_primitive_desc(), nullptr)); + + auto bn = mkldnn::batch_normalization_forward( + *context_.batchnorm_fwd_pd, + (const mkldnn::primitive::at)*context_.src_mem, + (const mkldnn::primitive::at)*context_.mean_mem, + (const mkldnn::primitive::at)*context_.var_mem, + (const mkldnn::memory)*context_.scale_shift_mem, + (const mkldnn::memory) *context_.dst_mem); + + context_.net.push_back(bn); + } + + BatchNormContext context_; + mkldnn::engine& cpu_engine_; +}; + +// Pool which allows for reuse of MKLDNN BatchNorm primitives which are +// expensive to instantiate. To address thread safety, the primitives are +// stored in a map on thread local storage. +template +class BatchNormPrimitivePool : public PrimitivePool { + public: + static BatchNormPrimitive* Get(const BatchNormParams& params) { + BatchNormPrimitive* primitive = + dynamic_cast*>( + BatchNormPrimitivePool::GetInstance().GetPrimitive(params.ToString())); + + if (primitive == nullptr) { + auto BatchNorm_primitive = std::make_unique>(params); + primitive = BatchNorm_primitive.get(); + BatchNormPrimitivePool::GetInstance().SetPrimitive( + params.ToString(), std::move(BatchNorm_primitive)); + } + return primitive; + } + + private: + BatchNormPrimitivePool() = default; + ~BatchNormPrimitivePool() = default; + + static BatchNormPrimitivePool& GetInstance() { + static BatchNormPrimitivePool pool; + return pool; + } +}; +} // namespace + +template +Status BatchNorm::Compute(OpKernelContext* context) const { + const Tensor* X = context->Input(0); + + int num_dimensions = static_cast(X->Shape().NumDimensions()); + if (num_dimensions == 3) { + // Fall back CPU implementation + return onnxruntime::BatchNorm::Compute(context); + } + + const T* src_data = X->template Data(); + + const Tensor* scale = context->Input(1); + const T* scale_data = scale->template Data(); + + const Tensor* B = context->Input(2); + const T* b_data = B->template Data(); + + const Tensor* mean = context->Input(3); + const T* mean_data = mean->template Data(); + + const Tensor* var = context->Input(4); + const T* var_data = var->template Data(); + + Tensor* Y = context->Output(0, X->Shape()); + T* dst_data = Y->template MutableData(); + + ORT_RETURN_IF_ERROR( + BatchNormHelper::ValidateInputs(X, scale, B, mean, var)); + + mkldnn::memory::dims src_dims_mkl( + X->Shape().GetDims().begin(), X->Shape().GetDims().end()); + mkldnn::memory::dims scale_dims_mkl( + scale->Shape().GetDims().begin(), scale->Shape().GetDims().end()); + mkldnn::memory::dims b_dims_mkl( + B->Shape().GetDims().begin(), B->Shape().GetDims().end()); + mkldnn::memory::dims mean_dims_mkl( + mean->Shape().GetDims().begin(), mean->Shape().GetDims().end()); + mkldnn::memory::dims var_dims_mkl( + var->Shape().GetDims().begin(), var->Shape().GetDims().end()); + + mkldnn::memory::dims dst_dims_mkl( + Y->Shape().GetDims().begin(), Y->Shape().GetDims().end()); + + try { + BatchNormParams batchNorm_params(src_dims_mkl, scale_dims_mkl, + b_dims_mkl, mean_dims_mkl, var_dims_mkl, dst_dims_mkl, + onnxruntime::BatchNorm::epsilon_, num_dimensions); + BatchNormPrimitive* batchNorm_primitive = + BatchNormPrimitivePool::Get(batchNorm_params); + ORT_RETURN_IF_NOT(batchNorm_primitive != nullptr); + batchNorm_primitive->Compute(src_data, scale_data, b_data, + mean_data, var_data, dst_data, scale_dims_mkl[0]); + + } catch (const mkldnn::error& e) { + return ORT_MAKE_STATUS( + ONNXRUNTIME, FAIL, "Status: ", e.status, ", message: ", e.message.c_str()); + } + + return Status::OK(); +} + +ONNX_OPERATOR_KERNEL_EX( + BatchNormalization, + kOnnxDomain, + 7, + kMklDnnExecutionProvider, + KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + BatchNorm); + +} // namespace mkl_dnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/nn/batch_norm.h b/onnxruntime/core/providers/mkldnn/nn/batch_norm.h new file mode 100644 index 0000000000000..bd4ddd23fae42 --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/nn/batch_norm.h @@ -0,0 +1,18 @@ +// Copyright(C) 2018 Intel Corporation +// Licensed under the MIT License + +#pragma once +#include "core/framework/op_kernel.h" +#include "core/providers/cpu/nn/batch_norm.h" + +namespace onnxruntime { +namespace mkl_dnn { + +template +class BatchNorm final : public onnxruntime::BatchNorm { + public: + BatchNorm(const OpKernelInfo& info) : onnxruntime::BatchNorm(info) {} + Status Compute(OpKernelContext* context) const override; +}; +} // namespace mkl_dnn +} // namespace onnxruntime