From 99bf8ba82117a115466304bc13a0e754aa9a7c1d Mon Sep 17 00:00:00 2001 From: zhangjun Date: Tue, 29 Nov 2022 10:10:59 +0000 Subject: [PATCH 1/3] add prelu --- .../inference/tensorrt/convert/prelu_op.cc | 74 +++++++++---------- 1 file changed, 34 insertions(+), 40 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/convert/prelu_op.cc b/paddle/fluid/inference/tensorrt/convert/prelu_op.cc index df769a486b4bc..6b246d6427146 100644 --- a/paddle/fluid/inference/tensorrt/convert/prelu_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/prelu_op.cc @@ -31,7 +31,6 @@ class PReluOpConverter : public OpConverter { framework::OpDesc op_desc(op, nullptr); // Declare inputs - size_t input_num = op_desc.Input("X").size(); auto* input = engine_->GetITensor(op_desc.Input("X")[0]); // Get attrs std::string mode = PADDLE_GET_CONST(std::string, op_desc.GetAttr("mode")); @@ -40,49 +39,44 @@ class PReluOpConverter : public OpConverter { data_format = PADDLE_GET_CONST(std::string, op_desc.GetAttr("data_format")); } - auto* alpha_var = scope.FindVar(op_desc.Input("Alpha")[0]); - auto* alpha_tensor = alpha_var->GetMutable(); + auto* alpha_tensor = engine_->GetITensor(op_desc.Input("Alpha")[0]); - auto alpha_weight = - engine_->GetFp32TrtWeight(op_desc.Input("Alpha")[0], *alpha_tensor); - - platform::CPUPlace cpu_place; - - nvinfer1::ILayer* layer = nullptr; - if (engine_->with_dynamic_shape()) { - plugin::PReluPluginDynamic* plugin = new plugin::PReluPluginDynamic( - static_cast(alpha_weight.get().values), - alpha_tensor->numel(), - mode, - data_format); - layer = engine_->AddDynamicPlugin(&input, input_num, plugin); - } else { -#if IS_TRT_VERSION_GE(7000) - nvinfer1::Dims dims; - dims.nbDims = 0; - // jump batch dim - for (int i = 1; i < alpha_tensor->dims().size(); i++) { - dims.d[dims.nbDims++] = alpha_tensor->dims()[i]; - } - for (; dims.nbDims < input->getDimensions().nbDims; dims.nbDims++) { - dims.d[dims.nbDims] = 1; + auto alpha_dims = alpha_tensor->getDimensions(); + auto input_dims = input->getDimensions(); + nvinfer1::ITensor* real_alpha_tensor = alpha_tensor; + if (alpha_dims.nbDims == 1 && alpha_dims.nbDims != input_dims.nbDims) { + auto* reshape_layer = + TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *alpha_tensor); + int c = alpha_dims.d[0]; + if (engine_->with_dynamic_shape()) { + auto* n_tensor = Add1DConstantLayer(1); + auto* c_tensor = Add1DConstantLayer(c); + auto* hw_tensor = Add1DConstantLayer(std::vector{1, 1}); + if (data_format == "NCHW") { + auto* shape_tensor = Concat( + std::vector{n_tensor, c_tensor, hw_tensor}); + reshape_layer->setInput(1, *shape_tensor); + } else { + auto* shape_tensor = Concat( + std::vector{n_tensor, hw_tensor, c_tensor}); + reshape_layer->setInput(1, *shape_tensor); + } + } else { + nvinfer1::Dims3 reshape_dim{1, 1, 1}; + if (data_format == "NCHW") { + reshape_dim.d[0] = c; + } else if (data_format == "NHWC") { + reshape_dim.d[input_dims.nbDims - 1] = c; + } + reshape_layer->setReshapeDimensions(reshape_dim); } + real_alpha_tensor = reshape_layer->getOutput(0); + } - auto alpha_layer = - TRT_ENGINE_ADD_LAYER(engine_, Constant, dims, alpha_weight.get()); - auto alpha_layer_output = alpha_layer->getOutput(0); + nvinfer1::ILayer* layer = nullptr; - layer = TRT_ENGINE_ADD_LAYER( - engine_, ParametricReLU, *input, *alpha_layer_output); -#else - plugin::PReluPlugin* plugin = new plugin::PReluPlugin( - static_cast(alpha_weight.get().values), - alpha_tensor->numel(), - mode, - data_format); - layer = engine_->AddPlugin(&input, input_num, plugin); -#endif - } + layer = TRT_ENGINE_ADD_LAYER( + engine_, ParametricReLU, *input, *real_alpha_tensor); auto output_name = op_desc.Output("Out")[0]; RreplenishLayerAndOutput(layer, "prelu", {output_name}, test_mode); From b41545133374afbc36c5e70513212d43b54831e3 Mon Sep 17 00:00:00 2001 From: zhangjun Date: Tue, 29 Nov 2022 13:27:00 +0000 Subject: [PATCH 2/3] add prelu --- .../inference/tensorrt/convert/prelu_op.cc | 61 ++++++++++++++++--- .../ir/inference/test_trt_convert_prelu.py | 12 ++-- 2 files changed, 57 insertions(+), 16 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/convert/prelu_op.cc b/paddle/fluid/inference/tensorrt/convert/prelu_op.cc index 6b246d6427146..87a17539181e4 100644 --- a/paddle/fluid/inference/tensorrt/convert/prelu_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/prelu_op.cc @@ -39,30 +39,71 @@ class PReluOpConverter : public OpConverter { data_format = PADDLE_GET_CONST(std::string, op_desc.GetAttr("data_format")); } - auto* alpha_tensor = engine_->GetITensor(op_desc.Input("Alpha")[0]); + + auto* alpha_var = scope.FindVar(op_desc.Input("Alpha")[0]); + auto* alpha_weight = alpha_var->GetMutable(); + auto w_dims = alpha_weight->dims(); + auto alpha_data = + engine_->GetFp32TrtWeight(op_desc.Input("Alpha")[0], *alpha_weight); + + nvinfer1::Dims trt_w_dims; + trt_w_dims.nbDims = w_dims.size(); + for (int i = 0; i < trt_w_dims.nbDims; i++) { + trt_w_dims.d[i] = w_dims[i]; + } + + // The `element` mode contains the batch + if (mode == "element" && !engine_->with_dynamic_shape()) { + trt_w_dims.nbDims--; + for (int i = 0; i < trt_w_dims.nbDims; i++) { + trt_w_dims.d[i] = trt_w_dims.d[i + 1]; + } + } + + nvinfer1::ITensor* alpha_tensor = + TRT_ENGINE_ADD_LAYER(engine_, Constant, trt_w_dims, alpha_data.get()) + ->getOutput(0); auto alpha_dims = alpha_tensor->getDimensions(); auto input_dims = input->getDimensions(); nvinfer1::ITensor* real_alpha_tensor = alpha_tensor; - if (alpha_dims.nbDims == 1 && alpha_dims.nbDims != input_dims.nbDims) { + if (alpha_dims.nbDims != input_dims.nbDims) { auto* reshape_layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *alpha_tensor); int c = alpha_dims.d[0]; if (engine_->with_dynamic_shape()) { + std::vector itensors; auto* n_tensor = Add1DConstantLayer(1); auto* c_tensor = Add1DConstantLayer(c); - auto* hw_tensor = Add1DConstantLayer(std::vector{1, 1}); + nvinfer1::ITensor* hw_tensor = nullptr; + nvinfer1::ITensor* shape_tensor = nullptr; + if (input_dims.nbDims - 2 > 0) { + hw_tensor = Add1DConstantLayer( + std::vector(input_dims.nbDims - 2, 1)); + } if (data_format == "NCHW") { - auto* shape_tensor = Concat( - std::vector{n_tensor, c_tensor, hw_tensor}); - reshape_layer->setInput(1, *shape_tensor); + if (hw_tensor != nullptr) { + shape_tensor = Concat( + std::vector{n_tensor, c_tensor, hw_tensor}); + + } else { + shape_tensor = + Concat(std::vector{n_tensor, c_tensor}); + } } else { - auto* shape_tensor = Concat( - std::vector{n_tensor, hw_tensor, c_tensor}); - reshape_layer->setInput(1, *shape_tensor); + if (hw_tensor != nullptr) { + shape_tensor = Concat( + std::vector{n_tensor, hw_tensor, c_tensor}); + } else { + shape_tensor = + Concat(std::vector{n_tensor, c_tensor}); + } } + reshape_layer->setInput(1, *shape_tensor); } else { - nvinfer1::Dims3 reshape_dim{1, 1, 1}; + nvinfer1::Dims reshape_dim; + reshape_dim.nbDims = input_dims.nbDims; + std::fill(reshape_dim.d, reshape_dim.d + input_dims.nbDims, 1); if (data_format == "NCHW") { reshape_dim.d[0] = c; } else if (data_format == "NHWC") { diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_prelu.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_prelu.py index 0e1daa8828b72..14046d45a3b2d 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_prelu.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_prelu.py @@ -49,22 +49,22 @@ def generate_alpha(attrs: List[Dict[str, Any]], dim1, dim2, dim3): if dim1 != 0: shape.append(dim1) if dim2 != 0: - shape.append(1) + shape.append(dim2) if dim3 != 0: - shape.append(1) - return np.random.random(size=shape).astype(np.float32) + shape.append(dim3) + return np.random.random(size=shape[1]).astype(np.float32) elif ( attrs[0]["mode"] == "channel" and attrs[0]["data_format"] == "NHWC" ): shape = [1] if dim1 != 0: - shape.append(1) + shape.append(dim1) if dim2 != 0: - shape.append(1) + shape.append(dim2) if dim3 != 0: shape.append(dim3) - return np.random.random(size=shape).astype(np.float32) + return np.random.random(size=shape[-1]).astype(np.float32) elif attrs[0]["mode"] == "element": shape = [1] if dim1 != 0: From 6d04c4d659b3aef8a8e2553f362da1023a03c02a Mon Sep 17 00:00:00 2001 From: zhangjun Date: Fri, 2 Dec 2022 08:23:57 +0000 Subject: [PATCH 3/3] fix channel mode contains the batch --- paddle/fluid/inference/tensorrt/convert/prelu_op.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/convert/prelu_op.cc b/paddle/fluid/inference/tensorrt/convert/prelu_op.cc index 87a17539181e4..77ad881aa7baa 100644 --- a/paddle/fluid/inference/tensorrt/convert/prelu_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/prelu_op.cc @@ -32,6 +32,7 @@ class PReluOpConverter : public OpConverter { framework::OpDesc op_desc(op, nullptr); // Declare inputs auto* input = engine_->GetITensor(op_desc.Input("X")[0]); + auto input_dims = input->getDimensions(); // Get attrs std::string mode = PADDLE_GET_CONST(std::string, op_desc.GetAttr("mode")); std::string data_format = "NCHW"; @@ -52,8 +53,10 @@ class PReluOpConverter : public OpConverter { trt_w_dims.d[i] = w_dims[i]; } - // The `element` mode contains the batch - if (mode == "element" && !engine_->with_dynamic_shape()) { + // The `element` or `channel` mode contains the batch using static shape. + if ((mode == "element" || mode == "channel") && + !engine_->with_dynamic_shape() && + (trt_w_dims.nbDims - 1 == input_dims.nbDims)) { trt_w_dims.nbDims--; for (int i = 0; i < trt_w_dims.nbDims; i++) { trt_w_dims.d[i] = trt_w_dims.d[i + 1]; @@ -65,7 +68,6 @@ class PReluOpConverter : public OpConverter { ->getOutput(0); auto alpha_dims = alpha_tensor->getDimensions(); - auto input_dims = input->getDimensions(); nvinfer1::ITensor* real_alpha_tensor = alpha_tensor; if (alpha_dims.nbDims != input_dims.nbDims) { auto* reshape_layer =