From 2a0765fd548675ea935ead48073ec37187d0c074 Mon Sep 17 00:00:00 2001 From: weilin Date: Thu, 12 Sep 2019 14:08:46 -0700 Subject: [PATCH 1/4] change op_set to 11 --- onnxruntime/python/tools/quantization/quantize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/python/tools/quantization/quantize.py b/onnxruntime/python/tools/quantization/quantize.py index 83df6cd46d99f..56c5870d244e7 100644 --- a/onnxruntime/python/tools/quantization/quantize.py +++ b/onnxruntime/python/tools/quantization/quantize.py @@ -14,7 +14,7 @@ __producer__ = "onnx.quantize" __version__ = "0.1.0" onnx_domain = "ai.onnx" -onnx_op_set_version = 10 +onnx_op_set_version = 11 type_to_name = { 1: "FLOAT", From a0b3a3252c3ab9633894bca63cd1187345fe3c6d Mon Sep 17 00:00:00 2001 From: weilin Date: Thu, 12 Sep 2019 14:11:17 -0700 Subject: [PATCH 2/4] add bias support in convInteger --- .../python/tools/quantization/quantize.py | 50 +++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/onnxruntime/python/tools/quantization/quantize.py b/onnxruntime/python/tools/quantization/quantize.py index 56c5870d244e7..2b6be54193439 100644 --- a/onnxruntime/python/tools/quantization/quantize.py +++ b/onnxruntime/python/tools/quantization/quantize.py @@ -645,6 +645,40 @@ def _get_quantize_input_nodes(self, node, input_index, qType): return nodes + [qlinear_node] + def _get_bias_add_nodes(self, nodes, node, last_output, quantized_bias_name): + ''' + Given a node, this function handles bias add by adding a reshape node on bias and a add node + + parameter nodes: new nodes would be appended into nodes + parameter node: current node(Conv) + parameter last_output: output of previous node(input to bias add) + return: (nodes) + ''' + # Add an Add operation for bias + #Add reshape for correct broadcase + #bias_input = node.input[2] #bias_index == 2 + #initializer = _find_by_name(quantized_bias_name, self.model.graph.initializer) + + #if initializer is not None: + + reshape_input = [quantized_bias_name] + # Add tensors for the shape to be reshaped to + _add_initializer_if_not_present(self.model.graph, "reshape_shape", + [1,-1,1,1], [4], onnx_proto.TensorProto.INT64) + reshape_input.append('reshape_shape') + reshape_op_output = node.output[0] + "_reshape" + reshape_node = onnx.helper.make_node("Reshape", reshape_input, [reshape_op_output], + quantized_bias_name+"reshape") + nodes.append(reshape_node) + + bias_add_input = [last_output] + bias_add_input.append(reshape_op_output) + add_node_output = node.output[0] + "_bias_add" + add_node = onnx.helper.make_node("Add", bias_add_input, [add_node_output], + quantized_bias_name + "bias_add") + nodes.append(add_node) + return add_node_output + def _update_unsupported_nodes_using_weight(self, weight, new_nodes_list): '''Find all nodes using a weight that do not support quantization and add a DequantizeLinear node before those nodes. This includes all nodes except Conv, MatMul. @@ -723,7 +757,7 @@ def _quantize_bias(self, node, new_node_list): # input scale is not provided and this input is dynamically quantized so it is not pre-computed at this point # so resort to dynamic quantization for bias - if node.input[0] not in self.quantization_params and node.input[0] not in self.quantized_value_map: + if self.quantization_params is None or node.input[0] not in self.quantization_params and node.input[0] not in self.quantized_value_map: self._dynamic_quantize_bias(node.input[0], weight_scale_name, bias_name, quantized_bias_name, new_node_list) else: # get scale for input @@ -920,6 +954,13 @@ def _quantize_convolution_integer_ops(self, node, new_nodes_list): (quantized_input_names, zero_point_names, scale_names, nodes) = \ self._quantize_inputs(node, [0, 1], new_nodes_list) + # quantize bias if exist + quantized_bias_name = "" + bias_present = False + if len(node.input) == 3: + quantized_bias_name = self._quantize_bias(node, nodes) + bias_present = True + conv_integer_output = node.output[0] + "_quantized" conv_integer_name = "" if node.name != "": @@ -931,6 +972,10 @@ def _quantize_convolution_integer_ops(self, node, new_nodes_list): [conv_integer_output], conv_integer_name, **kwargs) nodes.append(conv_integer_node) + # Add bias add nodes + if bias_present: + conv_integer_output = self._get_bias_add_nodes(nodes, node, conv_integer_output, quantized_bias_name) + # Add cast operation to cast convInteger output to float. cast_op_output = conv_integer_output + "_cast_output" cast_node = onnx.helper.make_node("Cast", [conv_integer_output], [cast_op_output], @@ -1026,7 +1071,6 @@ def _quantize_convolution_qlinear_ops(self, node, new_nodes_list): if len(node.input) == 3: quantized_bias_name = self._quantize_bias(node, nodes) bias_present = True - data_found, output_scale_name, output_zp_name, output_scale_shape, output_zp_shape = \ self._get_quantization_params(node.output[0]) @@ -1120,7 +1164,7 @@ def _quantize_convolution(self, node, new_nodes_list): ''' assert (node.op_type == "Conv") - if self.mode == QuantizationMode.IntegerOps and len(node.input) == 2: + if self.mode == QuantizationMode.IntegerOps: return self._quantize_convolution_integer_ops(node, new_nodes_list) if self.mode == QuantizationMode.QLinearOps: From ce25bbfe19ce378099318b2d4ee467375a1b97b2 Mon Sep 17 00:00:00 2001 From: Junfeng Dong Date: Thu, 12 Sep 2019 14:56:32 -0700 Subject: [PATCH 3/4] Clean code. --- .../python/tools/quantization/quantize.py | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/onnxruntime/python/tools/quantization/quantize.py b/onnxruntime/python/tools/quantization/quantize.py index 2b6be54193439..7c8f0bd52b232 100644 --- a/onnxruntime/python/tools/quantization/quantize.py +++ b/onnxruntime/python/tools/quantization/quantize.py @@ -647,35 +647,31 @@ def _get_quantize_input_nodes(self, node, input_index, qType): def _get_bias_add_nodes(self, nodes, node, last_output, quantized_bias_name): ''' - Given a node, this function handles bias add by adding a reshape node on bias and a add node + Given a node, this function handles bias add by adding a "reshape" node on bias and an "add" node parameter nodes: new nodes would be appended into nodes - parameter node: current node(Conv) - parameter last_output: output of previous node(input to bias add) - return: (nodes) + parameter node: current node (Conv) + parameter last_output: output of previous node (input to bias add) + return: the name of output ''' # Add an Add operation for bias - #Add reshape for correct broadcase - #bias_input = node.input[2] #bias_index == 2 - #initializer = _find_by_name(quantized_bias_name, self.model.graph.initializer) - - #if initializer is not None: - + # Add reshape for correct broadcase reshape_input = [quantized_bias_name] + # Add tensors for the shape to be reshaped to _add_initializer_if_not_present(self.model.graph, "reshape_shape", - [1,-1,1,1], [4], onnx_proto.TensorProto.INT64) + [1,-1,1,1], [4], onnx_proto.TensorProto.INT64) reshape_input.append('reshape_shape') reshape_op_output = node.output[0] + "_reshape" reshape_node = onnx.helper.make_node("Reshape", reshape_input, [reshape_op_output], - quantized_bias_name+"reshape") + quantized_bias_name+"reshape") nodes.append(reshape_node) bias_add_input = [last_output] bias_add_input.append(reshape_op_output) add_node_output = node.output[0] + "_bias_add" add_node = onnx.helper.make_node("Add", bias_add_input, [add_node_output], - quantized_bias_name + "bias_add") + quantized_bias_name + "bias_add") nodes.append(add_node) return add_node_output From e3b1111c7daee5654d25c870f77115887d9e8d39 Mon Sep 17 00:00:00 2001 From: weilin Date: Tue, 1 Oct 2019 15:30:04 -0700 Subject: [PATCH 4/4] Add function to check opset version of original model and set that for quantized model --- .../python/tools/quantization/quantize.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/onnxruntime/python/tools/quantization/quantize.py b/onnxruntime/python/tools/quantization/quantize.py index 7c8f0bd52b232..4c31c171e8d26 100644 --- a/onnxruntime/python/tools/quantization/quantize.py +++ b/onnxruntime/python/tools/quantization/quantize.py @@ -1185,6 +1185,29 @@ def _quantize_matmul(self, node, new_nodes_list): return [node] +def check_opset_version(org_model, fuse_dynamic_quant): + ''' + Check opset version of original model and set opset version and fuse_dynamic_quant accordingly. + If opset version < 10, set quantized model opset version to 10. + If opset version == 10, do quantization without using dynamicQuantizeLinear operator. + If opset version == 11, do quantization using dynamicQuantizeLinear operator. + + :return: fuse_dynamic_quant boolean value. + ''' + global onnx_op_set_version + opset_version = org_model.opset_import[0].version + if opset_version < 10: + print("Warning: The original model opset version is {}, which does not support quantized operators.\n\ + The opset version of quantized model will be set to 10.".format(opset_version)) + onnx_op_set_version = 10 + fuse_dynamic_quant = False + elif opset_version == 10: + onnx_op_set_version = 10 + fuse_dynamic_quant = False + else: + onnx_op_set_version = 11 + fuse_dynamic_quant = True + return fuse_dynamic_quant def quantize(model, per_channel=False, nbits=8, quantization_mode=QuantizationMode.IntegerOps, static=False, fuse_dynamic_quant=True, asymmetric_input_types=False, @@ -1241,6 +1264,7 @@ def quantize(model, per_channel=False, nbits=8, quantization_mode=QuantizationMo mode = quantization_mode copy_model = onnx_proto.ModelProto() copy_model.CopyFrom(model) + fuse_dynamic_quant = check_opset_version(copy_model, fuse_dynamic_quant) quantizer = ONNXQuantizer(copy_model, per_channel, mode, static, fuse_dynamic_quant, weight_qType, input_qType, quantization_params, nodes_to_quantize) quantizer.quantize_model()