Expose parallel execution option in C# API

.gitignore

@@ -31,6 +31,7 @@ onnxruntime_profile*.json
 /docs/python/*_LICENSE
 /csharp/**/obj/
 /csharp/**/bin/
+/csharp/Directory.Build.props
 docs/python/*.onnx
 *.onnx
 onnxprofile_profile_test_*.json

csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs

@@ -50,6 +50,24 @@ public bool SetSessionGraphOptimizationLevel(uint optimization_level)
             return result == 0;
         }
 
+        /// <summary>
+        /// Enable Sequential Execution. By default, it is enabled.
+        /// </summary>
+        /// </param>
+        public void EnableSequentialExecution()
+        {
+                NativeMethods.OrtEnableSequentialExecution(_nativePtr);
+        }
+
+        /// <summary>
+        /// Disable Sequential Execution and enable Parallel Execution.
+        /// </summary>
+        /// </param>
+        public void DisableSequentialExecution()
+        {
+            NativeMethods.OrtDisableSequentialExecution(_nativePtr);
+        }
+
         /// <summary>
         /// Default instance
         /// </summary>

csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs

@@ -51,15 +51,18 @@ public void CanCreateAndDisposeSessionWithModelPath()
         }
 
         [Theory]
-        [InlineData(0)]
-        [InlineData(2)]
-        private void CanRunInferenceOnAModel(uint graphOptimizationLevel)
+        [InlineData(0, true)]
+        [InlineData(0, false)]
+        [InlineData(2, true)]
+        [InlineData(2, false)]
+        private void CanRunInferenceOnAModel(uint graphOptimizationLevel, bool disableSequentialExecution)
         {
             string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx");
 
             // Set the graph optimization level for this session.
             SessionOptions options = new SessionOptions();
             Assert.True(options.SetSessionGraphOptimizationLevel(graphOptimizationLevel));
+            if(disableSequentialExecution) options.DisableSequentialExecution();
 
             using (var session = new InferenceSession(modelPath, options))
             {
@@ -215,11 +218,10 @@ private void TestPreTrainedModelsOpset7And8()
             foreach (var opset in opsets)
             {
                 var modelRoot = new DirectoryInfo(Path.Combine(modelsDir, opset));
-                //var cwd = Directory.GetCurrentDirectory();
                 foreach (var modelDir in modelRoot.EnumerateDirectories())
                 {
                     String onnxModelFileName = null;
-
+        
                     if (skipModels.Contains(modelDir.Name))
                         continue;
 

csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Microsoft.ML.OnnxRuntime.PerfTool.csproj

@@ -32,6 +32,7 @@
 
   <ItemGroup>
     <ProjectReference Include="$(OnnxRuntimeCSharpRoot)\src\Microsoft.ML.OnnxRuntime\Microsoft.ML.OnnxRuntime.csproj" />
+    <PackageReference Include="CommandLineParser" Version="2.4.3" />
     <PackageReference Include="Microsoft.ML.Scoring" Version="1.1.0" />
   </ItemGroup>
 

csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Program.cs

@@ -3,12 +3,9 @@
 
 using System;
 using System.Collections.Generic;
-using System.Text;
-using System.IO;
-using Microsoft.ML.OnnxRuntime;
 using System.Numerics.Tensors;
 using System.Diagnostics;
-
+using CommandLine;
 
 namespace Microsoft.ML.OnnxRuntime.PerfTool
 {
@@ -21,38 +18,51 @@ public enum TimingPoint
         TotalCount = 4
     }
 
-    class Program
+    class CommandOptions
     {
+        [Option('m', "model_file", Required = true, HelpText = "Model Path.")]
+        public string ModelFile { get; set; }
 
-        public static void Main(string[] args)
-        {
-            /*
-             args[0] = model-file-name
-             args[1] = input-file-name
-             args[2] = iteration count
-             */
+        [Option('i', "input_file", Required = true, HelpText = "Input path.")]
+        public string InputFile { get; set; }
 
-            if (args.Length < 3)
-            {
-                PrintUsage();
-                Environment.Exit(1);
-            }
+        [Option('c', "iteration_count", Required = true, HelpText = "Iteration to run.")]
+        public int IterationCount { get; set; }
 
-            string modelPath = args[0];
-            string inputPath = args[1];
-            int iteration = Int32.Parse(args[2]);
-            Console.WriteLine("Running model {0} in OnnxRuntime with input {1} for {2} times", modelPath, inputPath, iteration);
-            DateTime[] timestamps = new DateTime[(int)TimingPoint.TotalCount];
+        [Option('p', Required = false, HelpText = "Run with parallel exection. Default is false")]
+        public bool ParallelExecution { get; set; } = false;
 
-            RunModelOnnxRuntime(modelPath, inputPath, iteration, timestamps);
-            PrintReport(timestamps, iteration);
-            Console.WriteLine("Done");
+        [Option('o', "optimization_level", Required = false, HelpText = "Optimization Level. Default is 1, partial optimization.")]
+        public uint OptimizationLevel { get; set; } = 1;
+    }
 
-            Console.WriteLine("Running model {0} in Sonoma with input {1} for {2} times", modelPath, inputPath, iteration);
-            RunModelOnnxRuntime(modelPath, inputPath, iteration, timestamps);
+    class Program
+    {
+        public static void Main(string[] args)
+        {
+            var cmdOptions = Parser.Default.ParseArguments<CommandOptions>(args);
+            cmdOptions.WithParsed(
+                options => {
+                    Main(options);
+                });
+        }
+        public static void Main(CommandOptions options)
+        {
+            string modelPath = options.ModelFile;
+            string inputPath = options.InputFile;
+            int iteration = options.IterationCount;
+            bool parallelExecution = options.ParallelExecution;
+            uint optLevel = options.OptimizationLevel;
+            Console.WriteLine("Running model {0} in OnnxRuntime:", modelPath);
+            Console.WriteLine("input:{0}", inputPath);
+            Console.WriteLine("iteration count:{0}", iteration);
+            Console.WriteLine("parallel execution:{0}", parallelExecution);
+            Console.WriteLine("optimization level:{0}", optLevel);
+            DateTime[] timestamps = new DateTime[(int)TimingPoint.TotalCount];
+
+            RunModelOnnxRuntime(modelPath, inputPath, iteration, timestamps, parallelExecution, optLevel);
             PrintReport(timestamps, iteration);
             Console.WriteLine("Done");
-
         }
 
 
@@ -74,16 +84,18 @@ public static float[] LoadTensorFromFile(string filename)
             return tensorData.ToArray();
         }
 
-        static void RunModelOnnxRuntime(string modelPath, string inputPath, int iteration, DateTime[] timestamps)
+        static void RunModelOnnxRuntime(string modelPath, string inputPath, int iteration, DateTime[] timestamps, bool parallelExecution, uint optLevel)
         {
             if (timestamps.Length != (int)TimingPoint.TotalCount)
             {
                 throw new ArgumentException("Timestamps array must have "+(int)TimingPoint.TotalCount+" size");
             }
 
             timestamps[(int)TimingPoint.Start] = DateTime.Now;
-
-            using (var session = new InferenceSession(modelPath))
+            SessionOptions options = new SessionOptions();
+            if (parallelExecution) options.DisableSequentialExecution();
+            options.SetSessionGraphOptimizationLevel(optLevel);
+            using (var session = new InferenceSession(modelPath, options))
             {
                 timestamps[(int)TimingPoint.ModelLoaded] = DateTime.Now;
                 var inputMeta = session.InputMetadata;

docs/CSharp_API.md

@@ -111,6 +111,12 @@ Sets the graph optimization level for the session. Default is set to 1. Availabl
  * 1 -> Enable basic optimizations such as redundant node removals and constant folding
  * 2 -> Enable all optimizations (includes Level1 and more complex optimizations such as node fusions)
 
+    EnableSequentialExecution();
+Enable Sequential Execution. By default, it is enabled.
+
+    DisableSequentialExecution();
+Disable Sequential Execution and enable Parallel Execution.
+
     AppendExecutionProvider(ExecutionProvider provider);
 Appends execution provider to the session. For any operator in the graph the first execution provider that implements the operator will be user. ExecutionProvider is defined as the following enum.
 

tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml

@@ -4,6 +4,10 @@ jobs:
     buildDirectory: '$(Build.BinariesDirectory)'
   steps:
     - template: templates/set-test-data-variables-step.yml
+    - task: NuGetToolInstaller@0
+      displayName: Use Nuget 4.3
+      inputs:
+        versionSpec: 4.3.0
     - task: NuGetCommand@2
       displayName: 'NuGet restore'
       inputs:

tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml

@@ -5,6 +5,10 @@ jobs:
     CUDA_VERSION: '10.0'
   steps:
     - template: templates/set-test-data-variables-step.yml
+    - task: NuGetToolInstaller@0
+      displayName: Use Nuget 4.3
+      inputs:
+        versionSpec: 4.3.0
     - task: NuGetCommand@2
       displayName: 'NuGet restore'
       inputs:

tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml

@@ -10,6 +10,10 @@ jobs:
 
   steps:
     # - template: templates/set-test-data-variables-step.yml
+    - task: NuGetToolInstaller@0
+      displayName: Use Nuget 4.3
+      inputs:
+        versionSpec: 4.3.0
     - task: NuGetCommand@2
       displayName: 'NuGet restore'
       inputs: