Skip to content

Commit 190d692

Browse files
rkazantsmvafin
andauthored
[TF FE] Implement translators for ExtractImagePatches and MatrixDiag (openvinotoolkit#12593)
* [TF FE] Implement translators for ExtractImagePatches and MatrixDiag It allows to convert Inpaint model and infer it correctly Signed-off-by: Kazantsev, Roman <roman.kazantsev@intel.com> * Apply code-review feedback: correct comments, use set Signed-off-by: Kazantsev, Roman <roman.kazantsev@intel.com> * Apply suggestions from code review Co-authored-by: Maxim Vafin <maxim.vafin@intel.com> Signed-off-by: Kazantsev, Roman <roman.kazantsev@intel.com> Co-authored-by: Maxim Vafin <maxim.vafin@intel.com>
1 parent 6fd2341 commit 190d692

File tree

5 files changed

+184
-8
lines changed

5 files changed

+184
-8
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
// Copyright (C) 2018-2022 Intel Corporation
2+
// SPDX-License-Identifier: Apache-2.0
3+
//
4+
5+
#include "op_table.hpp"
6+
#include "openvino/op/util/attr_types.hpp"
7+
#include "openvino/opsets/opset8.hpp"
8+
#include "utils.hpp"
9+
10+
using namespace std;
11+
using namespace ov::opset8;
12+
13+
namespace ov {
14+
namespace frontend {
15+
namespace tensorflow {
16+
namespace op {
17+
18+
OutputVector translate_extract_image_patches_op(const NodeContext& node) {
19+
TENSORFLOW_OP_VALIDATION(node, node.get_input_size() >= 0, "ExtractImagePatches must have at least one input.");
20+
auto images = node.get_input(0);
21+
22+
// retrieve attributes for ExtractImagePatches
23+
auto tf_ksizes = node.get_attribute<std::vector<int64_t>>("ksizes");
24+
auto tf_strides = node.get_attribute<std::vector<int64_t>>("strides");
25+
auto tf_rates = node.get_attribute<std::vector<int64_t>>("rates");
26+
auto tf_padding_type = node.get_attribute<std::string>("padding");
27+
ov::op::PadType auto_pad = convert_tf_padding(node, tf_padding_type);
28+
TENSORFLOW_OP_VALIDATION(node,
29+
auto_pad == ov::op::PadType::SAME_UPPER || auto_pad == ov::op::PadType::VALID,
30+
"Only SAME_UPPER and VALID padding modes are supported for ExtractImagePatches.");
31+
32+
// prepare attributes for OpenVINO ExtractImagePatches
33+
Shape sizes(2);
34+
Shape rates(2);
35+
Strides strides(2);
36+
37+
// layout for this operation is always NHWC
38+
bool is_nhwc = true;
39+
convert_nhwc_to_hw(is_nhwc, tf_ksizes, sizes);
40+
convert_nhwc_to_hw(is_nhwc, tf_strides, strides);
41+
convert_nhwc_to_hw(is_nhwc, tf_rates, rates);
42+
43+
// prepare input to ExtractImagePatches
44+
convert_nhwc_to_nchw(is_nhwc, images);
45+
46+
auto extract_image_patches = make_shared<ExtractImagePatches>(images, sizes, strides, rates, auto_pad);
47+
48+
// prepare output to return the original layout NHWC
49+
auto extract_image_patches_output = extract_image_patches->output(0);
50+
convert_nchw_to_nhwc(is_nhwc, extract_image_patches_output);
51+
52+
set_node_name(node.get_name(), extract_image_patches_output.get_node_shared_ptr());
53+
return {extract_image_patches_output};
54+
}
55+
} // namespace op
56+
} // namespace tensorflow
57+
} // namespace frontend
58+
} // namespace ov
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
// Copyright (C) 2018-2022 Intel Corporation
2+
// SPDX-License-Identifier: Apache-2.0
3+
//
4+
5+
#include "op_table.hpp"
6+
#include "openvino/op/util/attr_types.hpp"
7+
#include "openvino/opsets/opset8.hpp"
8+
#include "utils.hpp"
9+
10+
using namespace std;
11+
using namespace ov::opset8;
12+
13+
namespace ov {
14+
namespace frontend {
15+
namespace tensorflow {
16+
namespace op {
17+
18+
OutputVector translate_matrix_diag_op(const NodeContext& node) {
19+
// The translation of MatrixDiag to OpenVINO opset relies on padding of input tensor with zeros,
20+
// reshape to a special form and cutting of unneeded padding part.
21+
// Here is a basic idea described by an example,
22+
// let us have a tensor [1, 2, 3] and generate padding tensor of zeros with a shape [3, 3].
23+
// Concatenate input tensor with padding and get the following:
24+
// [[1, 0, 0, 0]
25+
// [2, 0, 0, 0]
26+
// [3, 0, 0, 0]] of shape [3, 4]
27+
// Reshape to tensor of a shape [12] equal to [1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0]
28+
// Cut off last 3 elements and get [1, 0, 0, 0, 2, 0, 0, 0, 3] and reshape to [3, 3]
29+
// This idea is generalized to higher rank tensors
30+
TENSORFLOW_OP_VALIDATION(node, node.get_input_size() > 0, "MatrixDiag must have at least one input.");
31+
// diagonal is the single input to MatrixDiag operation and has a shape [I, J, ..., M, N]
32+
auto diagonal = node.get_input(0);
33+
auto diagonal_type = diagonal.get_element_type();
34+
35+
// 1. unsqueeze to have at least three rank input of a shape [1, I, J, ..., M, N, 1]
36+
// because dimensions [I, J, ..., M] can be absent
37+
auto unsqueeze_axis = make_shared<Constant>(element::i64, Shape{2}, std::vector<int64_t>{0, -1});
38+
auto unsqueeze_diag = make_shared<Unsqueeze>(diagonal, unsqueeze_axis);
39+
40+
// 2. compute a size of the last dimension of the diagonal input of a shape [I, J, ..., M, N],
41+
// i.e. N that will be diagonalized
42+
auto unsqueeze_diag_shape = make_shared<ShapeOf>(unsqueeze_diag);
43+
auto last_dim = make_shared<StridedSlice>(unsqueeze_diag_shape,
44+
make_shared<Constant>(element::i64, Shape{1}, std::vector<int64_t>{-2}),
45+
make_shared<Constant>(element::i64, Shape{1}, std::vector<int64_t>{-1}),
46+
make_shared<Constant>(element::i64, Shape{1}, std::vector<int64_t>{1}),
47+
std::vector<int64_t>({0}),
48+
std::vector<int64_t>({0}));
49+
50+
// 3. generate a tensor of zeros of a shape [1, I, J, ..., M, N, N]
51+
auto diag_shape = make_shared<ShapeOf>(diagonal);
52+
auto one_dim = make_shared<Constant>(last_dim->get_element_type(), Shape{1}, std::vector<int64_t>{1});
53+
auto padding_shape = make_shared<Concat>(OutputVector({one_dim, diag_shape, last_dim}), 0);
54+
auto padding =
55+
make_shared<Broadcast>(make_shared<Constant>(diagonal_type, Shape{1}, std::vector<int64_t>{0}), padding_shape);
56+
57+
// 4. concatenate to get input tensor with zero padding of a shape [1, I, J, ..., M, N, N + 1]
58+
auto zero_padded_diag = make_shared<Concat>(OutputVector({unsqueeze_diag, padding}), -1);
59+
60+
// reshape padded tensor to get a shape [I, J, ..., M, N * N + N]
61+
// 4.1 retrieve a part of the shape value [1, I, J, ..., M]
62+
auto new_shape_padded_diag1 =
63+
make_shared<StridedSlice>(unsqueeze_diag_shape,
64+
make_shared<Constant>(element::i64, Shape{1}, std::vector<int64_t>{0}),
65+
make_shared<Constant>(element::i64, Shape{1}, std::vector<int64_t>{-2}),
66+
make_shared<Constant>(element::i64, Shape{1}, std::vector<int64_t>{1}),
67+
std::vector<int64_t>({0}),
68+
std::vector<int64_t>({0}));
69+
// 4.2 compute the last part of a shape that is [N * N + N]
70+
auto last_dim_squared = make_shared<Multiply>(last_dim, last_dim);
71+
auto new_shape_padded_diag2 = make_shared<Add>(last_dim_squared, last_dim);
72+
// 4.3 compute a new shape and reshape padded diagonal
73+
auto new_shape_padded_diag = make_shared<Concat>(OutputVector({new_shape_padded_diag1, new_shape_padded_diag2}), 0);
74+
auto reshaped_padded_diag = make_shared<Reshape>(zero_padded_diag, new_shape_padded_diag, false);
75+
76+
// 5. cut off padding in the reshaped padded tensor to get a shape [1, I, J, ..., M, N * N]
77+
auto cut_padded_diag = make_shared<Slice>(
78+
reshaped_padded_diag,
79+
make_shared<Constant>(last_dim_squared->get_element_type(), Shape{1}, std::vector<int64_t>{0}),
80+
last_dim_squared,
81+
make_shared<Constant>(last_dim_squared->get_element_type(), Shape{1}, std::vector<int64_t>{1}),
82+
make_shared<Constant>(element::i64, Shape{1}, std::vector<int64_t>{-1}));
83+
84+
// 6. return the expected shape for the result [I, J, ..., M, N, N]
85+
auto resulted_shape = make_shared<Concat>(OutputVector({diag_shape, last_dim}), 0);
86+
auto resulted_diag = make_shared<Reshape>(cut_padded_diag, resulted_shape, false);
87+
88+
set_node_name(node.get_name(), resulted_diag);
89+
return {resulted_diag};
90+
}
91+
} // namespace op
92+
} // namespace tensorflow
93+
} // namespace frontend
94+
} // namespace ov

src/frontends/tensorflow/src/op/placeholder.cpp

+15-3
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,25 @@ namespace tensorflow {
1414
namespace op {
1515

1616
OutputVector translate_placeholder_op(const NodeContext& node) {
17-
auto ng_et = node.get_attribute<ov::element::Type>("dtype");
18-
auto ng_shape = node.get_attribute<ov::PartialShape>("shape", ov::PartialShape());
17+
auto tf_dtype = node.get_attribute<ov::element::Type>("dtype");
18+
auto tf_shape = node.get_attribute<ov::PartialShape>("shape", ov::PartialShape::dynamic());
1919

20-
auto res = std::make_shared<Parameter>(ng_et, ng_shape);
20+
auto res = std::make_shared<Parameter>(tf_dtype, tf_shape);
2121
set_node_name(node.get_name(), res);
2222
return res->outputs();
2323
}
24+
25+
OutputVector translate_placeholder_with_default_op(const NodeContext& node) {
26+
// For parity with legacy frontend, it creates a constant node with the default value
27+
// As a rule, PlaceholderWithDefault is mainly used for is_training variables in the model
28+
TENSORFLOW_OP_VALIDATION(node,
29+
node.get_input_size() > 0,
30+
"PlaceholderWithDefault must have at least one input that is the default value.");
31+
auto input = node.get_input(0);
32+
set_out_name(node.get_name(), input);
33+
return {input};
34+
}
35+
2436
} // namespace op
2537
} // namespace tensorflow
2638
} // namespace frontend

src/frontends/tensorflow/src/op_table.cpp

+6
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ OP_CONVERTER(translate_depth_to_space_op);
4141
OP_CONVERTER(translate_depthwise_conv_2d_native_op);
4242
OP_CONVERTER(translate_elu_op);
4343
OP_CONVERTER(translate_expand_dims_op);
44+
OP_CONVERTER(translate_extract_image_patches_op);
4445
OP_CONVERTER(translate_fake_quant_op);
4546
OP_CONVERTER(translate_fill_op);
4647
OP_CONVERTER(translate_floor_div_op);
@@ -58,10 +59,12 @@ OP_CONVERTER(translate_log_softmax_op);
5859
OP_CONVERTER(translate_log_1p_op);
5960
OP_CONVERTER(translate_lrn_op);
6061
OP_CONVERTER(translate_mat_mul_op);
62+
OP_CONVERTER(translate_matrix_diag_op);
6163
OP_CONVERTER(translate_max_pool_op);
6264
OP_CONVERTER(translate_non_max_suppression_op);
6365
OP_CONVERTER(translate_pad_op);
6466
OP_CONVERTER(translate_placeholder_op);
67+
OP_CONVERTER(translate_placeholder_with_default_op);
6568
OP_CONVERTER(translate_no_op);
6669
OP_CONVERTER(translate_one_hot_op);
6770
OP_CONVERTER(translate_pack_op);
@@ -181,6 +184,7 @@ const std::map<std::string, CreatorFunction> get_supported_ops() {
181184
{"DepthwiseConv2dNative", translate_depthwise_conv_2d_native_op},
182185
{"Elu", translate_elu_op},
183186
{"ExpandDims", translate_expand_dims_op},
187+
{"ExtractImagePatches", translate_extract_image_patches_op},
184188
{"FakeQuantWithMinMaxVars", translate_fake_quant_op},
185189
{"Fill", translate_fill_op},
186190
{"FloorDiv", translate_floor_div_op},
@@ -200,6 +204,7 @@ const std::map<std::string, CreatorFunction> get_supported_ops() {
200204
{"Log1p", translate_log_1p_op},
201205
{"LRN", translate_lrn_op},
202206
{"MatMul", translate_mat_mul_op},
207+
{"MatrixDiag", translate_matrix_diag_op},
203208
{"MaxPool", translate_max_pool_op},
204209
{"MaxPoolV2", translate_max_pool_op},
205210
{"MaxPool3D", translate_max_pool_op},
@@ -215,6 +220,7 @@ const std::map<std::string, CreatorFunction> get_supported_ops() {
215220
{"Pad", translate_pad_op},
216221
{"PadV2", translate_pad_op},
217222
{"Placeholder", translate_placeholder_op},
223+
{"PlaceholderWithDefault", translate_placeholder_with_default_op},
218224
{"PreventGradient", translate_identity_op},
219225
{"Range", translate_range_op},
220226
{"Rank", translate_rank_op},

src/frontends/tensorflow/src/utils.cpp

+11-5
Original file line numberDiff line numberDiff line change
@@ -34,13 +34,19 @@ void ov::frontend::tensorflow::set_out_name(const std::string& out_name, const o
3434

3535
ov::op::PadType ov::frontend::tensorflow::convert_tf_padding(const ov::frontend::tensorflow::NodeContext& node,
3636
const std::string& tf_padding) {
37+
std::set<std::string> supported_ops = {"Conv2D",
38+
"Conv2DBackpropInput",
39+
"Conv3D",
40+
"Conv3DBackpropInputV2",
41+
"MaxPool",
42+
"MaxPoolV2",
43+
"MaxPool3D",
44+
"ExtractImagePatches"};
3745
auto op_type = node.get_op_type();
3846

3947
TENSORFLOW_OP_VALIDATION(node,
40-
op_type == "Conv2D" || op_type == "Conv2DBackpropInput" || op_type == "Conv3D" ||
41-
op_type == "Conv3DBackpropInputV2" || op_type == "MaxPool" || op_type == "MaxPoolV2" ||
42-
op_type == "MaxPool3D",
43-
"The convert_conv_tf_padding routine supports only convolutional operations.");
48+
supported_ops.count(op_type),
49+
"Conversion of padding mode for " + op_type + " is not supported.");
4450
TENSORFLOW_OP_VALIDATION(
4551
node,
4652
tf_padding == "VALID" || tf_padding == "SAME" || tf_padding == "EXPLICIT",
@@ -57,7 +63,7 @@ ov::op::PadType ov::frontend::tensorflow::convert_tf_padding(const ov::frontend:
5763
return ov::op::PadType::SAME_LOWER;
5864
}
5965
} else if (op_type == "Conv2D" || op_type == "Conv3D" || op_type == "MaxPool" || op_type == "MaxPoolV2" ||
60-
op_type == "MaxPool3D") {
66+
op_type == "MaxPool3D" || op_type == "ExtractImagePatches") {
6167
if (tf_padding == "SAME") {
6268
// According to the formulas for calculating auto_pad values of the
6369
// Conv layer in the Operation specification,

0 commit comments

Comments
 (0)