Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Check and fix tensor and scalar type promotion #28299

Merged
merged 9 commits into from
Oct 30, 2020
Merged

Check and fix tensor and scalar type promotion #28299

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
99c88f3
check and fix tensor and scalar type promotion
chenwhql Oct 28, 2020
9d21619
Merge branch 'develop' into func/check_and_fix_tensor_and_scalar_type…
chenwhql Oct 28, 2020
f153892
fix else branch error
chenwhql Oct 28, 2020
6e1d7dc
Merge branch 'func/check_and_fix_tensor_and_scalar_type_promotion' of…
chenwhql Oct 28, 2020
c8b6485
fix scalar method error
chenwhql Oct 29, 2020
e76b474
fix test_math_op_path unittest
chenwhql Oct 29, 2020
818880f
add future division for unittest
chenwhql Oct 29, 2020
964799c
rm useless bin file
chenwhql Oct 30, 2020
e59c2ef
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…
chenwhql Oct 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 33 additions & 13 deletions python/paddle/fluid/dygraph/math_op_patch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,28 +149,46 @@ def _binary_creator_(method_name,
reverse=False,
scalar_method=None):
def __impl__(self, other_var):
# tensor and ComplexVariable opetator
# 0. check tensor and ComplexVariable opetator
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

opetator -> operator?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thx, I'll fix it in next PR

if isinstance(other_var, ComplexVariable):
# need import paddle in closure
import paddle
math_op = getattr(paddle.incubate.complex.tensor, op_type)
return math_op(self, other_var)

# FIXME(zjl): elementwise_div between integers cannot be converted to scale,
# which may lose accuracy. This is a hot fix for release 1.6.
if scalar_method is not None and not (
op_type == 'elementwise_div' and
self.dtype in _supported_int_dtype_):
if isinstance(other_var, float):
if self.dtype in _supported_int_dtype_:
assert other_var == int(other_var), \
"float value {} cannot convert to integer".format(other_var)
# 1. scalar exists cases
# we need combine the tensor.dtype and scalar.dtype, cast correct object
if isinstance(other_var, float):
# in all cases(+, -, *, /, **, //, %), we need cast tensor.dtype to float
if self.dtype in _supported_int_dtype_:
self = astype(self, 'float32')
# here use `scale` replace `elementwise` to get better performance
# but only +, -, *, / can use this method
if scalar_method is not None:
return scalar_method(self, other_var)
elif isinstance(other_var, int):
return scalar_method(self, float(other_var))
elif isinstance(other_var, int):
# in all cases(+, -, *, /, **, //, %), we can cast it to float
# because the output tensor.dtype depend on the type of input tensor
other_var = float(other_var)
# division is a special case
# NOTE(chenweihang): because we cast tensor to float32 instead float64,
# the division result can only guarantee the numerical accuracy of 6 digits
# after the decimal point. The result of numpy calculation is of float64 type,
# so the calculation result here and the calculation result of numpy are
# different after 6 decimal point. If necessary, we can also use float64 here.
# torch's behavior here is consistent with ours
if op_type == 'elementwise_div' and self.dtype in _supported_int_dtype_:
self = astype(self, 'float32')
# here use `scale` replace `elementwise` to get better performance
# but only +, -, *, / can use this method
if scalar_method is not None:
return scalar_method(self, other_var)
else:
# do nothing
pass

# 2. create varbase for scalar
lhs_dtype = self.dtype

if not isinstance(other_var, core.VarBase):
if reverse:
other_var = create_tensor(
Expand All @@ -179,6 +197,7 @@ def __impl__(self, other_var):
# add fill_op
other_var = create_scalar(value=other_var, dtype=lhs_dtype)

# 3. unify right var type to left var
rhs_dtype = other_var.dtype
if lhs_dtype != rhs_dtype:
other_var = astype(other_var, lhs_dtype)
Expand All @@ -187,6 +206,7 @@ def __impl__(self, other_var):
self = other_var
other_var = tmp

# 4. calculation
axis = -1
math_op = getattr(core.ops, op_type)
return math_op(self, other_var, 'axis', axis)
Expand Down
43 changes: 31 additions & 12 deletions python/paddle/fluid/layers/math_op_patch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,21 +215,39 @@ def _binary_creator_(method_name,
reverse=False,
scalar_method=None):
def __impl__(self, other_var):
# FIXME(zjl): elementwise_div between integers cannot be converted to scale,
# which may lose accuracy. This is a hot fix for release 1.6.
if scalar_method is not None and not (
op_type == 'elementwise_div' and
self.dtype in _supported_int_dtype_):
if isinstance(other_var, float):
if self.dtype in _supported_int_dtype_:
assert other_var == int(other_var), \
"float value {} cannot convert to integer".format(other_var)
# 1. scalar exists cases
# we need combine the tensor.dtype and scalar.dtype, cast correct object
if isinstance(other_var, float):
# in all cases(+, -, *, /, **, //, %), we need cast tensor.dtype to float
if self.dtype in _supported_int_dtype_:
self = astype(self, 'float32')
# here use `scale` replace `elementwise` to get better performance
# but only +, -, *, / can use this method
if scalar_method is not None:
return scalar_method(self, other_var)
elif isinstance(other_var, int):
return scalar_method(self, float(other_var))
elif isinstance(other_var, int):
# in all cases(+, -, *, /, **, //, %), we can cast it to float
# because the output tensor.dtype depend on the type of input tensor
other_var = float(other_var)
# division is a special case
# NOTE(chenweihang): because we cast tensor to float32 instead float64,
# the division result can only guarantee the numerical accuracy of 6 digits
# after the decimal point. The result of numpy calculation is of float64 type,
# so the calculation result here and the calculation result of numpy are
# different after 6 decimal point. If necessary, we can also use float64 here.
# torch's behavior here is consistent with ours
if op_type == 'elementwise_div' and self.dtype in _supported_int_dtype_:
self = astype(self, 'float32')
# here use `scale` replace `elementwise` to get better performance
# but only +, -, *, / can use this method
if scalar_method is not None:
return scalar_method(self, other_var)
else:
# do nothing
pass

# 2. create variable for scalar
lhs_dtype = safe_get_dtype(self)

if not isinstance(other_var, Variable):
if reverse:
has_batch_size = False
Expand All @@ -251,6 +269,7 @@ def __impl__(self, other_var):
other_var = create_scalar(
current_block(self), value=other_var, dtype=lhs_dtype)

# 3. unify right var type to left var
rhs_dtype = safe_get_dtype(other_var)
if lhs_dtype != rhs_dtype:
other_var = astype(other_var, lhs_dtype)
Expand Down
10 changes: 7 additions & 3 deletions python/paddle/fluid/tests/unittests/test_math_op_patch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function
from __future__ import print_function, division

import unittest
from decorator_helper import prog_scope
import paddle
import paddle.fluid as fluid
import numpy


class TestMathOpPatches(unittest.TestCase):
def setUp(self):
paddle.enable_static()

@prog_scope()
def test_add_scalar(self):
a = fluid.layers.data(name="a", shape=[1])
Expand Down Expand Up @@ -197,8 +201,8 @@ def test_integer_div(self):
feed={"a": a_np},
fetch_list=[b])

b_np_actual = (a_np / 7).astype('int64')
self.assertTrue(numpy.array_equal(b_np, b_np_actual))
b_np_actual = (a_np / 7).astype('float32')
self.assertTrue(numpy.allclose(b_np, b_np_actual))

@prog_scope()
def test_equal(self):
Expand Down
Loading