From ca27a1cd16741b781145f5efa69c84b43ed66930 Mon Sep 17 00:00:00 2001 From: Cecilia Liu Date: Thu, 29 Aug 2019 13:53:50 -0700 Subject: [PATCH] notebook update --- notebook/onnx-pipeline.ipynb | 870 ++++++++++++++++++++--------------- notebook/onnxpipeline.py | 10 +- notebook/readme.md | 3 +- 3 files changed, 516 insertions(+), 367 deletions(-) diff --git a/notebook/onnx-pipeline.ipynb b/notebook/onnx-pipeline.ipynb index 38ba911f77..57afd2074b 100644 --- a/notebook/onnx-pipeline.ipynb +++ b/notebook/onnx-pipeline.ipynb @@ -39,7 +39,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -56,12 +56,13 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import onnxpipeline\n", "\n", + "# Initiate ONNX pipeline with local directory \"model\"\n", "pipeline = onnxpipeline.Pipeline('model')\n", "\n", "# onnx\n", @@ -129,7 +130,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -137,8 +138,8 @@ "output_type": "stream", "text": [ "-----------config----------------\n", - " Container information: \n", - " Local directory path for volume: C:\\Users\\t-chuche\\Documents\\GitHub\\onnx-pipeline\\notebook/model\n", + " Container information: \n", + " Local directory path for volume: E:\\onnx-pipeline\\notebook/model\n", "Volume directory path in dockers: /mnt/model\n", " Result path: result\n", " Converted directory path: test\n", @@ -171,7 +172,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": { "scrolled": true }, @@ -216,8 +217,6 @@ "\n", "\r\n", "\n", - "Check ONNX model for correctness. \r\n", - "\n", "The original model is already onnx. Skipping correctness test. \r\n", "\n", "\r\n", @@ -346,242 +345,316 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "/home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:97 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] /home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:91 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] CUDA failure 35: CUDA driver version is insufficient for CUDA runtime version ; GPU=32646 ; hostname=5306725f2124 ; expr=cudaSetDevice(device_id_); \n", + "Setting thread pool size to 1\n", "\n", - "Stacktrace:\n", + "Total time cost:0.0523975\n", "\n", + "Total iterations:20\n", "\n", + "Average time cost:2.61987 ms\n", "\n", - "Stacktrace:\n", + "Setting thread pool size to 1\n", "\n", + "Total time cost:0.0191098\n", "\n", + "Total iterations:20\n", "\n", - "Setting thread pool size to 1\n", + "Average time cost:0.95549 ms\n", "\n", "Setting thread pool size to 0\n", "\n", - "Total time cost:0.0307762\n", + "Total time cost:0.0180704\n", "\n", "Total iterations:20\n", "\n", - "Average time cost:1.53881 ms\n", + "Average time cost:0.90352 ms\n", "\n", "Setting thread pool size to 0\n", "\n", - "Total time cost:0.0243878\n", + "Total time cost:0.0534911\n", "\n", "Total iterations:20\n", "\n", - "Average time cost:1.21939 ms\n", + "Average time cost:2.67455 ms\n", "\n", "Setting thread pool size to 0\n", "\n", - "Total time cost:0.0238097\n", + "Total time cost:0.0211078\n", "\n", "Total iterations:20\n", "\n", - "Average time cost:1.19049 ms\n", + "Average time cost:1.05539 ms\n", "\n", - "Setting thread pool size to 1\n", + "Setting thread pool size to 0\n", "\n", - "Total time cost:0.159269\n", + "Total time cost:0.0505198\n", "\n", "Total iterations:20\n", "\n", - "Average time cost:7.96343 ms\n", + "Average time cost:2.52599 ms\n", "\n", "Setting thread pool size to 0\n", "\n", - "Total time cost:0.0244207\n", + "Total time cost:0.0175324\n", "\n", "Total iterations:20\n", "\n", - "Average time cost:1.22104 ms\n", + "Average time cost:0.87662 ms\n", "\n", "Setting thread pool size to 0\n", "\n", - "Total time cost:0.0478194\n", + "Total time cost:0.0521924\n", "\n", "Total iterations:20\n", "\n", - "Average time cost:2.39097 ms\n", + "Average time cost:2.60962 ms\n", "\n", "Setting thread pool size to 0\n", "\n", - "Total time cost:0.0381018\n", + "Total time cost:0.0198271\n", "\n", "Total iterations:20\n", "\n", - "Average time cost:1.90509 ms\n", + "Average time cost:0.991355 ms\n", "\n", - "Setting thread pool size to 1\n", + "Setting thread pool size to 0\n", "\n", - "Total time cost:0.0922342\n", + "Total time cost:0.0190172\n", "\n", "Total iterations:20\n", "\n", - "Average time cost:4.61171 ms\n", + "Average time cost:0.95086 ms\n", "\n", - "/home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:97 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] /home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:91 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] CUDA failure 35: CUDA driver version is insufficient for CUDA runtime version ; GPU=32648 ; hostname=5306725f2124 ; expr=cudaSetDevice(device_id_); \n", + "Setting thread pool size to 0\n", "\n", - "Stacktrace:\n", + "Total time cost:0.0255277\n", "\n", + "Total iterations:20\n", "\n", + "Average time cost:1.27639 ms\n", "\n", - "Stacktrace:\n", + "Setting thread pool size to 0\n", "\n", + "Total time cost:0.0212756\n", "\n", + "Total iterations:20\n", + "\n", + "Average time cost:1.06378 ms\n", "\n", "Setting thread pool size to 1\n", "\n", - "/perf_test/bin/RelWithDebInfo/mklml/onnxruntime_perf_test: error while loading shared libraries: libmkldnn.so.0: cannot open shared object file: No such file or directory\n", + "Total time cost:0.0696818\n", "\n", - "/home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:97 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] /home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:91 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] CUDA failure 35: CUDA driver version is insufficient for CUDA runtime version ; GPU=32683 ; hostname=5306725f2124 ; expr=cudaSetDevice(device_id_); \n", + "Total iterations:20\n", "\n", - "Stacktrace:\n", + "Average time cost:3.48409 ms\n", "\n", + "Setting thread pool size to 1\n", "\n", + "Total time cost:0.0118875\n", "\n", - "Stacktrace:\n", + "Total iterations:20\n", + "\n", + "Average time cost:0.594375 ms\n", + "\n", + "Setting thread pool size to 0\n", "\n", + "Total time cost:0.013423\n", "\n", + "Total iterations:20\n", + "\n", + "Average time cost:0.67115 ms\n", "\n", "Setting thread pool size to 0\n", "\n", + "Total time cost:0.0681074\n", + "\n", + "Total iterations:20\n", + "\n", + "Average time cost:3.40537 ms\n", + "\n", "Setting thread pool size to 0\n", "\n", - "Total time cost:0.759284\n", + "Total time cost:0.014876\n", "\n", "Total iterations:20\n", "\n", - "Average time cost:37.9642 ms\n", + "Average time cost:0.7438 ms\n", "\n", "Setting thread pool size to 0\n", "\n", - "Total time cost:0.0485586\n", + "Total time cost:0.063621\n", "\n", "Total iterations:20\n", "\n", - "Average time cost:2.42793 ms\n", + "Average time cost:3.18105 ms\n", "\n", "Setting thread pool size to 0\n", "\n", - "Total time cost:0.214199\n", + "Total time cost:0.0153054\n", "\n", "Total iterations:20\n", "\n", - "Average time cost:10.7099 ms\n", + "Average time cost:0.76527 ms\n", "\n", "Setting thread pool size to 0\n", "\n", - "Total time cost:0.17056\n", + "Total time cost:0.0658572\n", "\n", "Total iterations:20\n", "\n", - "Average time cost:8.52799 ms\n", + "Average time cost:3.29286 ms\n", "\n", "Setting thread pool size to 0\n", "\n", - "Total time cost:0.0520268\n", + "Total time cost:0.014825\n", "\n", "Total iterations:20\n", "\n", - "Average time cost:2.60134 ms\n", + "Average time cost:0.74125 ms\n", "\n", "Setting thread pool size to 0\n", "\n", - "Total time cost:0.0485211\n", + "Total time cost:0.0128891\n", "\n", "Total iterations:20\n", "\n", - "Average time cost:2.42606 ms\n", + "Average time cost:0.644455 ms\n", "\n", "Setting thread pool size to 0\n", "\n", - "Total time cost:0.0278393\n", + "Total time cost:0.0258362\n", "\n", "Total iterations:20\n", "\n", - "Average time cost:1.39196 ms\n", + "Average time cost:1.29181 ms\n", "\n", "Setting thread pool size to 0\n", "\n", - "Total time cost:0.0395186\n", + "Total time cost:0.0129193\n", "\n", "Total iterations:20\n", "\n", - "Average time cost:1.97593 ms\n", + "Average time cost:0.645965 ms\n", "\n", - "/home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:97 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] /home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:91 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] CUDA failure 35: CUDA driver version is insufficient for CUDA runtime version ; GPU=32582 ; hostname=5306725f2124 ; expr=cudaSetDevice(device_id_); \n", + "Setting thread pool size to 1\n", "\n", - "Stacktrace:\n", + "Total time cost:0.0602524\n", "\n", + "Total iterations:20\n", "\n", + "Average time cost:3.01262 ms\n", "\n", - "Stacktrace:\n", + "Setting thread pool size to 1\n", "\n", + "Total time cost:0.017044\n", "\n", + "Total iterations:20\n", + "\n", + "Average time cost:0.8522 ms\n", "\n", "Setting thread pool size to 0\n", "\n", - "/perf_test/bin/RelWithDebInfo/mklml/onnxruntime_perf_test: error while loading shared libraries: libmkldnn.so.0: cannot open shared object file: No such file or directory\n", + "Total time cost:0.0172331\n", + "\n", + "Total iterations:20\n", + "\n", + "Average time cost:0.861655 ms\n", + "\n", + "/home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:97 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] /home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:91 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] CUDA failure 35: CUDA driver version is insufficient for CUDA runtime version ; GPU=32637 ; hostname=f5dd61f60039 ; expr=cudaSetDevice(device_id_); \n", + "\n", + "Stacktrace:\n", + "\n", + "\n", + "\n", + "Stacktrace:\n", + "\n", + "\n", "\n", "Setting thread pool size to 1\n", "\n", - "Total time cost:0.293462\n", + "/home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:97 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] /home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:91 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] CUDA failure 35: CUDA driver version is insufficient for CUDA runtime version ; GPU=32608 ; hostname=f5dd61f60039 ; expr=cudaSetDevice(device_id_); \n", "\n", - "Total iterations:200\n", + "Stacktrace:\n", "\n", - "Average time cost:1.46731 ms\n", "\n", - "Setting thread pool size to 0\n", "\n", - "Total time cost:0.297648\n", + "Stacktrace:\n", "\n", - "Total iterations:200\n", "\n", - "Average time cost:1.48824 ms\n", "\n", "Setting thread pool size to 0\n", "\n", - "Total time cost:0.295176\n", + "/home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:97 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] /home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:91 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] CUDA failure 35: CUDA driver version is insufficient for CUDA runtime version ; GPU=32656 ; hostname=f5dd61f60039 ; expr=cudaSetDevice(device_id_); \n", "\n", - "Total iterations:200\n", + "Stacktrace:\n", + "\n", + "\n", + "\n", + "Stacktrace:\n", "\n", - "Average time cost:1.47588 ms\n", + "\n", + "\n", + "Setting thread pool size to 1\n", + "\n", + "/home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:97 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] /home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:91 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] CUDA failure 35: CUDA driver version is insufficient for CUDA runtime version ; GPU=32692 ; hostname=f5dd61f60039 ; expr=cudaSetDevice(device_id_); \n", + "\n", + "Stacktrace:\n", + "\n", + "\n", + "\n", + "Stacktrace:\n", + "\n", + "\n", + "\n", + "Setting thread pool size to 0\n", "\n", "Setting thread pool size to 0\n", "\n", - "Total time cost:0.433033\n", + "Total time cost:0.180148\n", "\n", "Total iterations:200\n", "\n", - "Average time cost:2.16517 ms\n", + "Average time cost:0.900741 ms\n", "\n", "Setting thread pool size to 0\n", "\n", - "Total time cost:0.277646\n", + "Total time cost:0.21655\n", "\n", "Total iterations:200\n", "\n", - "Average time cost:1.38823 ms\n", + "Average time cost:1.08275 ms\n", "\n", "Cores: 1\n", "\n", - "0\n", + "No GPU found on current device. Cuda and TensorRT performance tuning might not be available. \n", "\n", + "providers ['mklml', 'cpu_openmp', 'mkldnn', 'mkldnn_openmp', 'cpu', 'tensorrt', 'ngraph', 'cuda']\n", "\n", "\n", - "/perf_test/bin/RelWithDebInfo/cuda/onnxruntime_perf_test -e cuda -x 1 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/e2d93036-3be1-43e2-87f1-e832e2432e7f\n", "\n", - "cuda_1_threads None\n", + "/perf_test/bin/RelWithDebInfo/mklml/onnxruntime_perf_test -P -x 1 -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/f2028979-6380-461a-9569-38ef2473c9ce\n", + "\n", + "mklml_parallel_1_threads 2.619875\n", + "\n", + "\n", + "\n", + "/perf_test/bin/RelWithDebInfo/mklml/onnxruntime_perf_test -x 1 -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/01cc797f-ec1a-4844-bf9a-f5a06ee00d98\n", + "\n", + "mklml_1_threads 0.9554900000000001\n", + "\n", + "\n", + "\n", + "/perf_test/bin/RelWithDebInfo/mklml/onnxruntime_perf_test -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/d6f3a345-6b74-4a8f-a553-10406719d4e2\n", + "\n", + "mklml 0.90352\n", "\n", "\n", "\n", @@ -589,9 +662,19 @@ "\n", "OMP_NUM_THREADS=1\n", "\n", - "/perf_test/bin/RelWithDebInfo/mkldnn_openmp/onnxruntime_perf_test -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/644b29d7-9403-41cd-b3b0-4aec6a015875\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -P -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/e01b46e9-919b-454c-976f-58d20289cf09\n", + "\n", + "cpu_openmp_parallel_1_threads_OMP_WAIT_POLICY_active 2.674555\n", + "\n", + "\n", + "\n", + "OMP_WAIT_POLICY=active\n", + "\n", + "OMP_NUM_THREADS=1\n", "\n", - "mkldnn_openmp_1_threads_OMP_WAIT_POLICY_active 0.001195805\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/2c37d398-05c0-4a9b-bf5e-77b19f59f517\n", + "\n", + "cpu_openmp_1_threads_OMP_WAIT_POLICY_active 1.05539\n", "\n", "\n", "\n", @@ -599,189 +682,219 @@ "\n", "OMP_NUM_THREADS=1\n", "\n", - "/perf_test/bin/RelWithDebInfo/mkldnn_openmp/onnxruntime_perf_test -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/0019fde7-0357-4ef0-9aaa-7907e831a973\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -P -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/687cf1c6-91ea-4fc4-a497-deb2b4ff63a8\n", + "\n", + "cpu_openmp_parallel_1_threads_OMP_WAIT_POLICY_passive 2.52599\n", "\n", - "mkldnn_openmp_1_threads_OMP_WAIT_POLICY_passive 0.00137513\n", "\n", "\n", + "OMP_WAIT_POLICY=passive\n", "\n", "OMP_NUM_THREADS=1\n", "\n", - "/perf_test/bin/RelWithDebInfo/mkldnn_openmp/onnxruntime_perf_test -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/fac414a7-d17a-4f6f-942e-f94b6e699f42\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/f51f71b3-1bc2-4a4b-870b-054d8de161c4\n", "\n", - "mkldnn_openmp_1_threads 0.00132002\n", + "cpu_openmp_1_threads_OMP_WAIT_POLICY_passive 0.8766200000000001\n", "\n", "\n", "\n", - "/perf_test/bin/RelWithDebInfo/mkldnn/onnxruntime_perf_test -e mkldnn -x 1 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/4b745e85-6a50-4aff-a8a6-4d8d275861c5\n", + "OMP_NUM_THREADS=1\n", "\n", - "mkldnn_1_threads 0.008280075\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -P -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/550a7ebe-b9cb-49b7-8e5e-0561947cc6da\n", "\n", + "cpu_openmp_parallel_1_threads 2.60962\n", "\n", "\n", - "OMP_WAIT_POLICY=active\n", "\n", "OMP_NUM_THREADS=1\n", "\n", - "/perf_test/bin/RelWithDebInfo/cpu_openmp/onnxruntime_perf_test -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/d40f4872-5139-40b9-b8d6-665cdb297ab6\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/fd37394f-3e3e-43a0-9d28-8e0adcfd001d\n", "\n", - "cpu_openmp_1_threads_OMP_WAIT_POLICY_active 0.00134276\n", + "cpu_openmp_1_threads 0.991355\n", "\n", "\n", "\n", - "OMP_WAIT_POLICY=passive\n", + "OMP_WAIT_POLICY=active\n", "\n", - "OMP_NUM_THREADS=1\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/8e480ab6-2668-426d-b4b3-8a4631d2ec34\n", "\n", - "/perf_test/bin/RelWithDebInfo/cpu_openmp/onnxruntime_perf_test -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/39ee551e-1059-4517-b639-6a6e75ea1f4c\n", + "cpu_openmp_OMP_WAIT_POLICY_active 0.95086\n", "\n", - "cpu_openmp_1_threads_OMP_WAIT_POLICY_passive 0.00297085\n", "\n", "\n", + "OMP_WAIT_POLICY=passive\n", "\n", - "OMP_NUM_THREADS=1\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/385b0da2-128f-43c3-91f0-5f43b62db67e\n", "\n", - "/perf_test/bin/RelWithDebInfo/cpu_openmp/onnxruntime_perf_test -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/735c2c63-1be9-4e84-923b-2c8a4518fb0d\n", + "cpu_openmp_OMP_WAIT_POLICY_passive 1.276385\n", "\n", - "cpu_openmp_1_threads 0.002347575\n", "\n", "\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/0936e4c7-76db-43c1-a294-31c2578af8bf\n", "\n", - "/perf_test/bin/RelWithDebInfo/ngraph/onnxruntime_perf_test -e ngraph -x 1 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/10088b17-ce35-4b44-9222-2821b5ac4428\n", + "cpu_openmp 1.0637800000000002\n", "\n", - "ngraph_1_threads None\n", "\n", "\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -P -x 1 -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/75894cca-846c-4bac-ada2-9ae5b0d118b4\n", "\n", - "/perf_test/bin/RelWithDebInfo/cpu/onnxruntime_perf_test -x 1 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/0a6ea0ab-7f3e-4bae-8964-ba8b5cfbd753\n", + "mkldnn_parallel_1_threads 3.48409\n", "\n", - "cpu_1_threads 0.001122835\n", "\n", "\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -x 1 -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/0af828bb-1c7c-4210-a1e2-fae3977c26cb\n", "\n", - "/perf_test/bin/RelWithDebInfo/tensorrt/onnxruntime_perf_test -e tensorrt -x 1 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/7d3d343c-791b-490a-93af-6fb264b03fcf\n", + "mkldnn_1_threads 0.594375\n", "\n", - "tensorrt_1_threads None\n", "\n", "\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/5157e6d6-4e37-46e1-b690-426da3036bf5\n", "\n", - "/perf_test/bin/RelWithDebInfo/mklml/onnxruntime_perf_test -x 1 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/e6ba1957-641a-4266-87ca-731710070fd2\n", + "mkldnn 0.67115\n", "\n", - "mklml_1_threads None\n", "\n", "\n", + "OMP_WAIT_POLICY=active\n", "\n", - "/perf_test/bin/RelWithDebInfo/cuda/onnxruntime_perf_test -e cuda -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/e3fa3248-229c-42db-ba94-cff031a3ed36\n", + "OMP_NUM_THREADS=1\n", "\n", - "cuda None\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -P -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/c0f86f78-dbc6-4a7c-b2b1-55d7d684027f\n", + "\n", + "mkldnn_openmp_parallel_1_threads_OMP_WAIT_POLICY_active 3.40537\n", "\n", "\n", "\n", "OMP_WAIT_POLICY=active\n", "\n", - "/perf_test/bin/RelWithDebInfo/mkldnn_openmp/onnxruntime_perf_test -e mkldnn -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/e1ee1778-e406-4fce-8b73-48763791f12d\n", + "OMP_NUM_THREADS=1\n", + "\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/b85ec27e-a525-4daa-989d-802f7d83c377\n", + "\n", + "mkldnn_openmp_1_threads_OMP_WAIT_POLICY_active 0.7438\n", + "\n", "\n", - "mkldnn_openmp_OMP_WAIT_POLICY_active 0.05346786\n", + "\n", + "OMP_WAIT_POLICY=passive\n", + "\n", + "OMP_NUM_THREADS=1\n", + "\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -P -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/16ff8b39-d928-497d-95d9-4f969774efba\n", + "\n", + "mkldnn_openmp_parallel_1_threads_OMP_WAIT_POLICY_passive 3.18105\n", "\n", "\n", "\n", "OMP_WAIT_POLICY=passive\n", "\n", - "/perf_test/bin/RelWithDebInfo/mkldnn_openmp/onnxruntime_perf_test -e mkldnn -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/a50a3cb8-6c0d-44d4-8af9-033cef9e73a2\n", + "OMP_NUM_THREADS=1\n", + "\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/dade4860-01e0-4c17-93d9-e30ac39a9281\n", "\n", - "mkldnn_openmp_OMP_WAIT_POLICY_passive 0.002094445\n", + "mkldnn_openmp_1_threads_OMP_WAIT_POLICY_passive 0.76527\n", "\n", "\n", "\n", - "/perf_test/bin/RelWithDebInfo/mkldnn_openmp/onnxruntime_perf_test -e mkldnn -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/9b3f9e13-1410-42c6-973a-fe1a959ba873\n", + "OMP_NUM_THREADS=1\n", "\n", - "mkldnn_openmp 0.008712135\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -P -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/6bbfe707-49d1-432e-8235-cb9791668ecc\n", "\n", + "mkldnn_openmp_parallel_1_threads 3.2928599999999997\n", "\n", "\n", - "/perf_test/bin/RelWithDebInfo/mkldnn/onnxruntime_perf_test -e mkldnn -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/b3e1dbcc-abec-4565-acbd-9737ef1f211c\n", "\n", - "mkldnn 0.01117089\n", + "OMP_NUM_THREADS=1\n", + "\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/4dda93e8-d288-4351-b263-f9920f368de9\n", + "\n", + "mkldnn_openmp_1_threads 0.7412500000000001\n", "\n", "\n", "\n", "OMP_WAIT_POLICY=active\n", "\n", - "/perf_test/bin/RelWithDebInfo/cpu_openmp/onnxruntime_perf_test -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/7431ccbf-dde8-4383-af9a-f707618bdaef\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/69532ad9-7339-48b7-a599-12749ccd0d36\n", "\n", - "cpu_openmp_OMP_WAIT_POLICY_active 0.00139207\n", + "mkldnn_openmp_OMP_WAIT_POLICY_active 0.644455\n", "\n", "\n", "\n", "OMP_WAIT_POLICY=passive\n", "\n", - "/perf_test/bin/RelWithDebInfo/cpu_openmp/onnxruntime_perf_test -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/d529f2ba-2890-4e1a-b4e0-ed2203787f07\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/010a8bfd-d781-4122-bddf-fc030a6b1faf\n", "\n", - "cpu_openmp_OMP_WAIT_POLICY_passive 0.0017405\n", + "mkldnn_openmp_OMP_WAIT_POLICY_passive 1.29181\n", "\n", "\n", "\n", - "/perf_test/bin/RelWithDebInfo/cpu_openmp/onnxruntime_perf_test -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/12d8794f-0935-4b28-ac5b-51f3ccb06aba\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/7f868621-f6c9-42b1-9e10-7fceb1d73092\n", "\n", - "cpu_openmp 0.0013846\n", + "mkldnn_openmp 0.645965\n", "\n", "\n", "\n", - "/perf_test/bin/RelWithDebInfo/ngraph/onnxruntime_perf_test -e ngraph -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/f1f20820-650a-48d1-99e5-60d143604e5a\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -P -x 1 -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/24f4ca53-3257-49c7-b019-86b58efd1908\n", "\n", - "ngraph None\n", + "cpu_parallel_1_threads 3.01262\n", "\n", "\n", "\n", - "/perf_test/bin/RelWithDebInfo/cpu/onnxruntime_perf_test -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/d66083ba-c53a-4692-a542-d97f533a0fa9\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -x 1 -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/479c01c7-9f39-4283-8c24-f384065505ce\n", "\n", - "cpu 0.00196531\n", + "cpu_1_threads 0.8522\n", "\n", "\n", "\n", - "/perf_test/bin/RelWithDebInfo/tensorrt/onnxruntime_perf_test -e tensorrt -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/d9b58d77-0db5-4b6f-9018-066dc27984c7\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/2e63a6f7-594c-48a0-b56c-9e090d727c62\n", "\n", - "tensorrt None\n", + "cpu 0.8616550000000001\n", "\n", "\n", "\n", - "/perf_test/bin/RelWithDebInfo/mklml/onnxruntime_perf_test -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/c524bcd4-07ce-4209-ba74-166f8ec6900d\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e tensorrt -x 1 -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/4216436b-d49e-453a-90eb-ca710f361552\n", "\n", - "mklml None\n", + "tensorrt_1_threads None\n", "\n", "\n", "\n", - "/perf_test/bin/RelWithDebInfo/cpu/onnxruntime_perf_test -x 1 -m times -r 200 /mnt/model/test/model.onnx /mnt/model/result/eea3ede5-6aa5-4ff9-a28d-fc3be214a814\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e tensorrt -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/ec354273-70aa-4a17-a0af-b19659e1e6eb\n", "\n", - "cpu_1_threads 0.001448245\n", + "tensorrt None\n", "\n", "\n", "\n", - "OMP_WAIT_POLICY=active\n", + "/perf_test/bin/RelWithDebInfo/ngraph/onnxruntime_perf_test -e ngraph -P -x 1 -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/dc262eb3-cc03-4b1d-ba8f-a8e2da7a306e\n", "\n", - "OMP_NUM_THREADS=1\n", + "ngraph_parallel_1_threads None\n", "\n", - "/perf_test/bin/RelWithDebInfo/mkldnn_openmp/onnxruntime_perf_test -m times -r 200 /mnt/model/test/model.onnx /mnt/model/result/4add89a9-de41-415d-bdc1-9d90bdfceebc\n", "\n", - "mkldnn_openmp_1_threads_OMP_WAIT_POLICY_active 0.001443092\n", "\n", + "/perf_test/bin/RelWithDebInfo/ngraph/onnxruntime_perf_test -e ngraph -x 1 -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/730eb238-cc58-4e1a-886c-c42e2dc814f0\n", "\n", + "ngraph_1_threads None\n", "\n", - "OMP_NUM_THREADS=1\n", "\n", - "/perf_test/bin/RelWithDebInfo/mkldnn_openmp/onnxruntime_perf_test -m times -r 200 /mnt/model/test/model.onnx /mnt/model/result/77f3cfa8-590d-42a2-bdb5-8f78f039aa88\n", "\n", - "mkldnn_openmp_1_threads 0.001406933\n", + "/perf_test/bin/RelWithDebInfo/ngraph/onnxruntime_perf_test -e ngraph -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/1eae5faf-50f9-448a-b02e-9b26a3ec42e4\n", "\n", + "ngraph None\n", "\n", "\n", - "OMP_WAIT_POLICY=active\n", "\n", - "OMP_NUM_THREADS=1\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e cuda -x 1 -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/986aa192-db8c-4a88-b67a-e6b86de086f5\n", + "\n", + "cuda_1_threads None\n", + "\n", + "\n", + "\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e cuda -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/36cc772f-844d-4d1b-a99a-bb4a8b7cd343\n", + "\n", + "cuda None\n", + "\n", "\n", - "/perf_test/bin/RelWithDebInfo/cpu_openmp/onnxruntime_perf_test -m times -r 200 /mnt/model/test/model.onnx /mnt/model/result/9e23cf82-f19e-4d50-b492-1177d8784d77\n", "\n", - "cpu_openmp_1_threads_OMP_WAIT_POLICY_active 0.001748473\n", + "/perf_test/bin/RelWithDebInfo/mklml/onnxruntime_perf_test -o 3 -m times -r 200 /mnt/model/test/model.onnx /mnt/model/result/b9f62370-b3c1-4d0c-879c-47bb1647f5c0\n", + "\n", + "mklml 0.9007419999999999\n", "\n", "\n", "\n", @@ -789,39 +902,99 @@ "\n", "OMP_NUM_THREADS=1\n", "\n", - "/perf_test/bin/RelWithDebInfo/mkldnn_openmp/onnxruntime_perf_test -m times -r 200 /mnt/model/test/model.onnx /mnt/model/result/8a38b798-442e-4b76-8f97-3e2d4cb36ff6\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -o 3 -m times -r 200 /mnt/model/test/model.onnx /mnt/model/result/8b686e09-33bb-4bda-9a3f-fc507f595924\n", + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cpu_openmp_1_threads_OMP_WAIT_POLICY_passive 1.0827479999999998Setting thread pool size to 1\n", + "\n", + "Total time cost:0.141779\n", "\n", - "mkldnn_openmp_1_threads_OMP_WAIT_POLICY_passive 0.001520645\n", + "Total iterations:200\n", + "\n", + "Average time cost:0.708894 ms\n", + "\n", + "Setting thread pool size to 0\n", + "\n", + "Total time cost:0.146149\n", + "\n", + "Total iterations:200\n", + "\n", + "Average time cost:0.730747 ms\n", + "\n", + "Setting thread pool size to 1\n", + "\n", + "Total time cost:0.192491\n", + "\n", + "Total iterations:200\n", + "\n", + "Average time cost:0.962453 ms\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -x 1 -o 3 -m times -r 200 /mnt/model/test/model.onnx /mnt/model/result/0dc072e1-8cd3-4e2d-9c52-c6ec3bc122a6\n", + "\n", + "mkldnn_1_threads 0.7088939999999999\n", + "\n", + "\n", + "\n", + "OMP_WAIT_POLICY=active\n", + "\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -o 3 -m times -r 200 /mnt/model/test/model.onnx /mnt/model/result/d231523e-0633-43d1-84bf-e6d9b3c684d9\n", + "\n", + "mkldnn_openmp_OMP_WAIT_POLICY_active 0.730747\n", + "\n", + "\n", + "\n", + "/perf_test/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -x 1 -o 3 -m times -r 200 /mnt/model/test/model.onnx /mnt/model/result/342413bd-4b68-4649-9858-881054337737\n", + "\n", + "cpu_1_threads 0.962453\n", "\n", "\n", "\n", "Results:\n", "\n", - "cpu_openmp 0.0013846 s\n", + "mkldnn_1_threads 0.7088939999999999 ms\n", "\n", - "cpu_openmp_OMP_WAIT_POLICY_active 0.00139207 s\n", + "mkldnn 0.67115 ms\n", "\n", - "mkldnn_openmp_1_threads 0.001406933 s\n", + "mkldnn_parallel_1_threads 3.48409 ms\n", "\n", - "mkldnn_openmp_1_threads_OMP_WAIT_POLICY_active 0.001443092 s\n", + "mkldnn_openmp_OMP_WAIT_POLICY_active 0.730747 ms\n", "\n", - "cpu_1_threads 0.001448245 s\n", + "mkldnn_openmp 0.645965 ms\n", "\n", - "cuda_1_threads error\n", + "mkldnn_openmp_1_threads 0.7412500000000001 ms\n", "\n", - "ngraph_1_threads error\n", + "mklml 0.9007419999999999 ms\n", "\n", - "tensorrt_1_threads error\n", + "mklml_1_threads 0.9554900000000001 ms\n", "\n", - "mklml_1_threads error\n", + "mklml_parallel_1_threads 2.619875 ms\n", "\n", - "cuda error\n", + "cpu_1_threads 0.962453 ms\n", "\n", - "ngraph error\n", + "cpu 0.8616550000000001 ms\n", + "\n", + "cpu_parallel_1_threads 3.01262 ms\n", "\n", - "tensorrt error\n", + "cpu_openmp_1_threads_OMP_WAIT_POLICY_passive 1.0827479999999998 ms\n", "\n", - "mklml error\n", + "cpu_openmp_OMP_WAIT_POLICY_active 0.95086 ms\n", + "\n", + "cpu_openmp_1_threads 0.991355 ms\n", + "\n", + "ngraph_parallel_1_threads error\n", + "\n", + "ngraph_1_threads error\n", + "\n", + "ngraph error\n", "\n" ] } @@ -893,38 +1066,48 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "cpu_openmp 0.0013846 s\n", + "mkldnn_1_threads 0.7088939999999999 ms\n", "\n", - "cpu_openmp_OMP_WAIT_POLICY_active 0.00139207 s\n", + "mkldnn 0.67115 ms\n", "\n", - "mkldnn_openmp_1_threads 0.001406933 s\n", + "mkldnn_parallel_1_threads 3.48409 ms\n", "\n", - "mkldnn_openmp_1_threads_OMP_WAIT_POLICY_active 0.001443092 s\n", + "mkldnn_openmp_OMP_WAIT_POLICY_active 0.730747 ms\n", "\n", - "cpu_1_threads 0.001448245 s\n", + "mkldnn_openmp 0.645965 ms\n", "\n", - "cuda_1_threads error\n", + "mkldnn_openmp_1_threads 0.7412500000000001 ms\n", "\n", - "ngraph_1_threads error\n", + "mklml 0.9007419999999999 ms\n", "\n", - "tensorrt_1_threads error\n", + "mklml_1_threads 0.9554900000000001 ms\n", "\n", - "mklml_1_threads error\n", + "mklml_parallel_1_threads 2.619875 ms\n", "\n", - "cuda error\n", + "cpu_1_threads 0.962453 ms\n", "\n", - "ngraph error\n", + "cpu 0.8616550000000001 ms\n", "\n", - "tensorrt error\n", + "cpu_parallel_1_threads 3.01262 ms\n", "\n", - "mklml error\n", + "cpu_openmp_1_threads_OMP_WAIT_POLICY_passive 1.0827479999999998 ms\n", + "\n", + "cpu_openmp_OMP_WAIT_POLICY_active 0.95086 ms\n", + "\n", + "cpu_openmp_1_threads 0.991355 ms\n", + "\n", + "ngraph_parallel_1_threads error\n", + "\n", + "ngraph_1_threads error\n", + "\n", + "ngraph error\n", "\n" ] } @@ -956,14 +1139,14 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "C:\\Users\\t-chuche\\Documents\\GitHub\\onnx-pipeline\\notebook/model/result\n" + "E:\\onnx-pipeline\\notebook/model/result\n" ] } ], @@ -990,7 +1173,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -1027,87 +1210,87 @@ " \n", " \n", " 0\n", - " cpu_openmp\n", - " 0.001385\n", - " 0.001964\n", - " 0.002447\n", - " 0.000000\n", + " mkldnn_1_threads\n", + " 0.708894\n", + " 0.9633\n", + " 1.0281\n", + " 0.94483\n", " 0\n", - " 0.000000\n", - " OrderedDict([('execution_provider', ''), ('env...\n", + " 0.29085\n", + " OrderedDict([('execution_provider', 'mkldnn'),...\n", " \n", " \n", " 1\n", - " cpu_openmp_OMP_WAIT_POLICY_active\n", - " 0.001392\n", - " 0.001825\n", - " 0.002018\n", - " 0.000000\n", + " mkldnn_openmp_OMP_WAIT_POLICY_active\n", + " 0.730747\n", + " 0.9531\n", + " 1.0109\n", + " 0.95975\n", " 0\n", - " 0.000000\n", - " OrderedDict([('execution_provider', ''), ('env...\n", + " 0.29121\n", + " OrderedDict([('execution_provider', 'mkldnn_op...\n", " \n", " \n", " 2\n", - " mkldnn_openmp_1_threads\n", - " 0.001407\n", - " 0.001725\n", - " 0.001858\n", - " 0.873848\n", + " mklml\n", + " 0.900742\n", + " 1.0600\n", + " 1.1543\n", + " 0.91482\n", " 0\n", - " 0.341467\n", - " OrderedDict([('execution_provider', ''), ('env...\n", + " 0.28926\n", + " OrderedDict([('execution_provider', 'mklml'), ...\n", " \n", " \n", " 3\n", - " mkldnn_openmp_1_threads_OMP_WAIT_POLICY_active\n", - " 0.001443\n", - " 0.001790\n", - " 0.001881\n", - " 0.865487\n", + " cpu_1_threads\n", + " 0.962453\n", + " 1.1300\n", + " 1.3482\n", + " 0.89764\n", " 0\n", - " 0.340511\n", - " OrderedDict([('execution_provider', ''), ('env...\n", + " 0.29135\n", + " OrderedDict([('execution_provider', 'cpu'), ('...\n", " \n", " \n", " 4\n", - " cpu_1_threads\n", - " 0.001448\n", - " 0.001811\n", - " 0.001947\n", - " 0.916762\n", + " cpu_openmp_1_threads_OMP_WAIT_POLICY_passive\n", + " 1.082748\n", + " 1.5606\n", + " 1.6307\n", + " 0.94206\n", " 0\n", - " 0.340514\n", - " OrderedDict([('execution_provider', ''), ('env...\n", + " 0.29095\n", + " OrderedDict([('execution_provider', 'cpu_openm...\n", " \n", " \n", "\n", "" ], "text/plain": [ - " name avg p90 \\\n", - "0 cpu_openmp 0.001385 0.001964 \n", - "1 cpu_openmp_OMP_WAIT_POLICY_active 0.001392 0.001825 \n", - "2 mkldnn_openmp_1_threads 0.001407 0.001725 \n", - "3 mkldnn_openmp_1_threads_OMP_WAIT_POLICY_active 0.001443 0.001790 \n", - "4 cpu_1_threads 0.001448 0.001811 \n", + " name avg p90 p95 \\\n", + "0 mkldnn_1_threads 0.708894 0.9633 1.0281 \n", + "1 mkldnn_openmp_OMP_WAIT_POLICY_active 0.730747 0.9531 1.0109 \n", + "2 mklml 0.900742 1.0600 1.1543 \n", + "3 cpu_1_threads 0.962453 1.1300 1.3482 \n", + "4 cpu_openmp_1_threads_OMP_WAIT_POLICY_passive 1.082748 1.5606 1.6307 \n", "\n", - " p95 cpu_usage gpu_usage memory_util \\\n", - "0 0.002447 0.000000 0 0.000000 \n", - "1 0.002018 0.000000 0 0.000000 \n", - "2 0.001858 0.873848 0 0.341467 \n", - "3 0.001881 0.865487 0 0.340511 \n", - "4 0.001947 0.916762 0 0.340514 \n", + " cpu_usage gpu_usage memory_util \\\n", + "0 0.94483 0 0.29085 \n", + "1 0.95975 0 0.29121 \n", + "2 0.91482 0 0.28926 \n", + "3 0.89764 0 0.29135 \n", + "4 0.94206 0 0.29095 \n", "\n", " code_snippet \n", - "0 OrderedDict([('execution_provider', ''), ('env... \n", - "1 OrderedDict([('execution_provider', ''), ('env... \n", - "2 OrderedDict([('execution_provider', ''), ('env... \n", - "3 OrderedDict([('execution_provider', ''), ('env... \n", - "4 OrderedDict([('execution_provider', ''), ('env... " + "0 OrderedDict([('execution_provider', 'mkldnn'),... \n", + "1 OrderedDict([('execution_provider', 'mkldnn_op... \n", + "2 OrderedDict([('execution_provider', 'mklml'), ... \n", + "3 OrderedDict([('execution_provider', 'cpu'), ('... \n", + "4 OrderedDict([('execution_provider', 'cpu_openm... " ] }, - "execution_count": 7, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -1136,7 +1319,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -1161,98 +1344,92 @@ " \n", " \n", " cat\n", - " dur\n", - " name\n", - " op_name\n", - " ph\n", " pid\n", - " provider\n", " tid\n", + " dur\n", " ts\n", + " ph\n", + " name\n", + " args\n", " \n", " \n", " \n", " \n", " 0\n", - " Session\n", - " 73070\n", - " model_loading_uri\n", - " NaN\n", + " Node\n", + " 12367\n", + " 12367\n", + " 1815\n", + " 63814\n", " X\n", - " 136\n", - " NaN\n", - " 136\n", - " 3\n", + " sequential/dense/MatMul\n", + " {'provider': 'CPUExecutionProvider', 'op_name'...\n", " \n", " \n", " 1\n", - " Session\n", - " 6711\n", - " session_initialization\n", - " NaN\n", + " Node\n", + " 12367\n", + " 12367\n", + " 652\n", + " 73670\n", " X\n", - " 136\n", - " NaN\n", - " 136\n", - " 73081\n", + " sequential/dense/MatMul\n", + " {'provider': 'CPUExecutionProvider', 'op_name'...\n", " \n", " \n", " 2\n", " Node\n", - " 1\n", - " sequential/reshape/Shape_fence_before\n", - " Shape\n", + " 12367\n", + " 12367\n", + " 614\n", + " 74898\n", " X\n", - " 136\n", - " NaN\n", - " 136\n", - " 84778\n", + " sequential/dense/MatMul\n", + " {'provider': 'CPUExecutionProvider', 'op_name'...\n", " \n", " \n", " 3\n", " Node\n", - " 61\n", - " sequential/reshape/Shape_kernel_time\n", - " Shape\n", + " 12367\n", + " 12367\n", + " 608\n", + " 80834\n", " X\n", - " 136\n", - " CPUExecutionProvider\n", - " 136\n", - " 84784\n", + " sequential/dense/MatMul\n", + " {'provider': 'CPUExecutionProvider', 'op_name'...\n", " \n", " \n", " 4\n", " Node\n", - " 0\n", - " sequential/reshape/Shape_fence_after\n", - " Shape\n", + " 12367\n", + " 12367\n", + " 604\n", + " 82160\n", " X\n", - " 136\n", - " NaN\n", - " 136\n", - " 84848\n", + " sequential/dense/MatMul\n", + " {'provider': 'CPUExecutionProvider', 'op_name'...\n", " \n", " \n", "\n", "" ], "text/plain": [ - " cat dur name op_name ph pid \\\n", - "0 Session 73070 model_loading_uri NaN X 136 \n", - "1 Session 6711 session_initialization NaN X 136 \n", - "2 Node 1 sequential/reshape/Shape_fence_before Shape X 136 \n", - "3 Node 61 sequential/reshape/Shape_kernel_time Shape X 136 \n", - "4 Node 0 sequential/reshape/Shape_fence_after Shape X 136 \n", + " cat pid tid dur ts ph name \\\n", + "0 Node 12367 12367 1815 63814 X sequential/dense/MatMul \n", + "1 Node 12367 12367 652 73670 X sequential/dense/MatMul \n", + "2 Node 12367 12367 614 74898 X sequential/dense/MatMul \n", + "3 Node 12367 12367 608 80834 X sequential/dense/MatMul \n", + "4 Node 12367 12367 604 82160 X sequential/dense/MatMul \n", "\n", - " provider tid ts \n", - "0 NaN 136 3 \n", - "1 NaN 136 73081 \n", - "2 NaN 136 84778 \n", - "3 CPUExecutionProvider 136 84784 \n", - "4 NaN 136 84848 " + " args \n", + "0 {'provider': 'CPUExecutionProvider', 'op_name'... \n", + "1 {'provider': 'CPUExecutionProvider', 'op_name'... \n", + "2 {'provider': 'CPUExecutionProvider', 'op_name'... \n", + "3 {'provider': 'CPUExecutionProvider', 'op_name'... \n", + "4 {'provider': 'CPUExecutionProvider', 'op_name'... " ] }, - "execution_count": 8, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -1265,90 +1442,85 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Get environment_variables in code_snippet" + "# Get code snippets" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "import onnxruntime as ort\n", + "so = rt.SessionOptions()\n", + "so.set_graph_optimization_level(3)\n", + "so.enable_sequential_execution = False\n", + "so.session_thread_pool_size(0)\n", + "session = rt.Session(\"/mnt/model/test/model.onnx\", so)\n", + "\n" + ] + } + ], + "source": [ + "print(r.get_code(ep='cpu'))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Given the index from latency.json, output the environment variables." + "# netron" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 11, "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Serving 'model/test/model.onnx' at http://localhost:8080\n" + ] + }, { "data": { "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
0
LD_LIBRARY_PATH/perf_test/bin/RelWithDebInfo/mkldnn_openmp
OMP_WAIT_POLICYactive
\n", - "
" + " \n", + " " ], "text/plain": [ - " 0\n", - "LD_LIBRARY_PATH /perf_test/bin/RelWithDebInfo/mkldnn_openmp\n", - "OMP_WAIT_POLICY active" + "" ] }, - "execution_count": 9, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "r.print_environment(3)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Get codes" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Given the index from latency.json, output the codes." + "# only workable for notebook in the local server \n", + "import netron\n", + "netron.start(\"model/test/model.onnx\", browse=False) # 'model.onnx'\n", + "from IPython.display import IFrame\n", + "IFrame('http://localhost:8080', width=\"100%\", height=1000)" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -1356,38 +1528,12 @@ "output_type": "stream", "text": [ "\n", - "import onnxruntime as ort\n", - "so = rt.SessionOptions()\n", - "so.set_graph_optimization_level(2)\n", - "so.enable_sequential_execution = True\n", - "so.session_thread_pool_size(0)\n", - "session = rt.Session(\"/mnt/model/test/model.onnx\", so)\n", - "\n" + "Stopping http://localhost:8080\n" ] } ], "source": [ - "print(r.get_code(3))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# netron" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# only workable for notebook in the local server \n", - "import netron\n", - "netron.start(model) # 'model.onnx'\n", - "from IPython.display import IFrame\n", - "IFrame('http://localhost:8080', width=700, height=350)" + "netron.stop()" ] }, { diff --git a/notebook/onnxpipeline.py b/notebook/onnxpipeline.py index de32bca85f..b7a67c75dd 100644 --- a/notebook/onnxpipeline.py +++ b/notebook/onnxpipeline.py @@ -364,19 +364,21 @@ def __json_to_csv(self, json_data, erase_keys): return csv_name def prints(self, top=5, orient='table'): - json_data = self.latency[:top] + json_data = [] + for ep_name in self.latency.keys(): + json_data.append(self.latency[ep_name][0]) csv_file = self.__json_to_csv(json_data, {'command'}) return pd.read_csv(csv_file) - def print_profiling(self, index, top=10, orient='colums'): + def print_profiling(self, index=0, top=10, orient='colums'): self.__check_profiling_index(index) - return self.__print_json(self.profiling_ops[:top][0], orient) + return self.__print_json(self.profiling_ops[index][:top], orient) def print_environment(self, ep, index, orient='index'): return self.__print_json([self.latency[ep][index]['code_snippet']['environment_variables']], orient) - def get_code(self, ep, index): + def get_code(self, ep, index=0): code = self.latency[ep][index]['code_snippet']['code'] refined_code = code.replace(' ', '\n').replace(' ', '\n') # 4 tabs return refined_code diff --git a/notebook/readme.md b/notebook/readme.md index bb36db367f..77b2ae1159 100644 --- a/notebook/readme.md +++ b/notebook/readme.md @@ -23,7 +23,8 @@ pip install docker pip install jupyter ``` -Type belows in command line, and then choose [onnx-pipeline.ipynb](https://github.com/liuziyue/onnx-pipeline/blob/master/notebook/onnx-pipeline.ipynb) to use the notebook. +# Start Notebook +Type the below in command line, and then choose [onnx-pipeline.ipynb](https://github.com/liuziyue/onnx-pipeline/blob/master/notebook/onnx-pipeline.ipynb) to use the notebook. ``` jupyter notebook