From 71d7106a204e92ebbee3867171631eab55c98f44 Mon Sep 17 00:00:00 2001 From: Alexander Root Date: Fri, 29 Jul 2022 20:40:01 -0400 Subject: [PATCH 01/19] Make saturating_cast an intrinsic --- src/FindIntrinsics.cpp | 45 ++++++++++++++++++++++++++++++++++++++++++ src/IR.cpp | 1 + src/IR.h | 1 + src/IROperator.cpp | 35 +------------------------------- 4 files changed, 48 insertions(+), 34 deletions(-) diff --git a/src/FindIntrinsics.cpp b/src/FindIntrinsics.cpp index fcb6317943b6..8a55a6350749 100644 --- a/src/FindIntrinsics.cpp +++ b/src/FindIntrinsics.cpp @@ -886,6 +886,48 @@ Expr lower_saturating_sub(const Expr &a, const Expr &b) { return simplify(clamp(a, a.type().min() + max(b, 0), a.type().max() + min(b, 0))) - b; } + +Expr lower_saturating_cast(const Type &t, const Expr &a) { + // For float to float, guarantee infinities are always pinned to range. + if (t.is_float() && a.type().is_float()) { + if (t.bits() < a.type().bits()) { + return cast(t, clamp(a, t.min(), t.max())); + } else { + return clamp(cast(t, a), t.min(), t.max()); + } + } else if (a.type() != t) { + // Limits for Int(2^n) or UInt(2^n) are not exactly representable in Float(2^n) + if (a.type().is_float() && !t.is_float() && t.bits() >= a.type().bits()) { + Expr e = max(a, t.min()); // min values turn out to be always representable + + // This line depends on t.max() rounding upward, which should always + // be the case as it is one less than a representable value, thus + // the one larger is always the closest. + e = select(e >= cast(e.type(), t.max()), t.max(), cast(t, e)); + return e; + } else { + Expr min_bound; + if (!a.type().is_uint()) { + min_bound = lossless_cast(a.type(), t.min()); + } + Expr max_bound = lossless_cast(a.type(), t.max()); + + Expr e; + if (min_bound.defined() && max_bound.defined()) { + e = clamp(a, min_bound, max_bound); + } else if (min_bound.defined()) { + e = max(a, min_bound); + } else if (max_bound.defined()) { + e = min(a, max_bound); + } else { + e = a; + } + return cast(t, std::move(e)); + } + } + return a; +} + Expr lower_halving_add(const Expr &a, const Expr &b) { internal_assert(a.type() == b.type()); // Borrowed from http://aggregate.org/MAGIC/#Average%20of%20Integers @@ -1015,6 +1057,9 @@ Expr lower_intrinsic(const Call *op) { } else if (op->is_intrinsic(Call::saturating_sub)) { internal_assert(op->args.size() == 2); return lower_saturating_sub(op->args[0], op->args[1]); + } else if (op->is_intrinsic(Call::saturating_cast)) { + internal_assert(op->args.size() == 1); + return lower_saturating_cast(op->type, op->args[0]); } else if (op->is_intrinsic(Call::widening_shift_left)) { internal_assert(op->args.size() == 2); return lower_widening_shift_left(op->args[0], op->args[1]); diff --git a/src/IR.cpp b/src/IR.cpp index 740234b8e31f..9689292ba6fe 100644 --- a/src/IR.cpp +++ b/src/IR.cpp @@ -654,6 +654,7 @@ const char *const intrinsic_op_names[] = { "rounding_shift_right", "saturating_add", "saturating_sub", + "saturating_cast", "scatter_gather", "select_mask", "shift_left", diff --git a/src/IR.h b/src/IR.h index c6085614b59d..b278c4a2aafb 100644 --- a/src/IR.h +++ b/src/IR.h @@ -556,6 +556,7 @@ struct Call : public ExprNode { rounding_shift_right, saturating_add, saturating_sub, + saturating_cast, scatter_gather, select_mask, shift_left, diff --git a/src/IROperator.cpp b/src/IROperator.cpp index 4693060a8d45..40156dd46d5e 100644 --- a/src/IROperator.cpp +++ b/src/IROperator.cpp @@ -1430,40 +1430,7 @@ Expr require(Expr condition, const std::vector &args) { } Expr saturating_cast(Type t, Expr e) { - // For float to float, guarantee infinities are always pinned to range. - if (t.is_float() && e.type().is_float()) { - if (t.bits() < e.type().bits()) { - e = cast(t, clamp(std::move(e), t.min(), t.max())); - } else { - e = clamp(cast(t, std::move(e)), t.min(), t.max()); - } - } else if (e.type() != t) { - // Limits for Int(2^n) or UInt(2^n) are not exactly representable in Float(2^n) - if (e.type().is_float() && !t.is_float() && t.bits() >= e.type().bits()) { - e = max(std::move(e), t.min()); // min values turn out to be always representable - - // This line depends on t.max() rounding upward, which should always - // be the case as it is one less than a representable value, thus - // the one larger is always the closest. - e = select(e >= cast(e.type(), t.max()), t.max(), cast(t, e)); - } else { - Expr min_bound; - if (!e.type().is_uint()) { - min_bound = lossless_cast(e.type(), t.min()); - } - Expr max_bound = lossless_cast(e.type(), t.max()); - - if (min_bound.defined() && max_bound.defined()) { - e = clamp(std::move(e), min_bound, max_bound); - } else if (min_bound.defined()) { - e = max(std::move(e), min_bound); - } else if (max_bound.defined()) { - e = min(std::move(e), max_bound); - } - e = cast(t, std::move(e)); - } - } - return e; + return Internal::Call::make(t, Internal::Call::saturating_cast, {std::move(e)}, Internal::Call::PureIntrinsic); } Expr select(Expr condition, Expr true_value, Expr false_value) { From 2d398b5a858a35cc6447b0eecfffd329f52ffed0 Mon Sep 17 00:00:00 2001 From: Alexander Root Date: Fri, 29 Jul 2022 20:49:50 -0400 Subject: [PATCH 02/19] clang format --- src/FindIntrinsics.cpp | 1 - src/FindIntrinsics.h | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FindIntrinsics.cpp b/src/FindIntrinsics.cpp index 8a55a6350749..fcf48c4e1afd 100644 --- a/src/FindIntrinsics.cpp +++ b/src/FindIntrinsics.cpp @@ -886,7 +886,6 @@ Expr lower_saturating_sub(const Expr &a, const Expr &b) { return simplify(clamp(a, a.type().min() + max(b, 0), a.type().max() + min(b, 0))) - b; } - Expr lower_saturating_cast(const Type &t, const Expr &a) { // For float to float, guarantee infinities are always pinned to range. if (t.is_float() && a.type().is_float()) { diff --git a/src/FindIntrinsics.h b/src/FindIntrinsics.h index 3d9f955bfb27..07e639117252 100644 --- a/src/FindIntrinsics.h +++ b/src/FindIntrinsics.h @@ -22,6 +22,7 @@ Expr lower_rounding_shift_right(const Expr &a, const Expr &b); Expr lower_saturating_add(const Expr &a, const Expr &b); Expr lower_saturating_sub(const Expr &a, const Expr &b); +Expr lower_saturating_cast(const Type &t, const Expr &a); Expr lower_halving_add(const Expr &a, const Expr &b); Expr lower_halving_sub(const Expr &a, const Expr &b); From 70dcae2ac5526637d2015e7dc17da40e2f7d4e2e Mon Sep 17 00:00:00 2001 From: Alexander Root Date: Fri, 29 Jul 2022 22:59:42 -0400 Subject: [PATCH 03/19] fix saturating_cast intrinsic matching --- src/FindIntrinsics.cpp | 23 +++++++++++++++++++++++ src/IRMatch.h | 9 +++++++++ 2 files changed, 32 insertions(+) diff --git a/src/FindIntrinsics.cpp b/src/FindIntrinsics.cpp index fcf48c4e1afd..dbbb961b2a99 100644 --- a/src/FindIntrinsics.cpp +++ b/src/FindIntrinsics.cpp @@ -575,6 +575,29 @@ class FindIntrinsics : public IRMutator { return rewrite.result; } + const int bits = op->type.bits(); + const auto is_x_same_int = op->type.is_int() && is_int(x, bits); + const auto is_x_same_uint = op->type.is_uint() && is_uint(x, bits); + const auto is_x_same_int_or_uint = is_x_same_int || is_x_same_uint; + auto x_y_same_sign = (is_int(x) == is_int(y)) || (is_uint(x) && is_uint(y)); + Type unsigned_type = op->type.with_code(halide_type_uint); + + if (rewrite(saturating_cast(op->type, widening_add(x, y)), + saturating_add(x, y), + is_x_same_int_or_uint) || + rewrite(saturating_cast(op->type, widening_sub(x, y)), + saturating_sub(x, y), + is_x_same_int_or_uint) || + rewrite(saturating_cast(op->type, shift_right(widening_mul(x, y), z)), + mul_shift_right(x, y, cast(unsigned_type, z)), + is_x_same_int_or_uint && x_y_same_sign && is_uint(z)) || + rewrite(saturating_cast(op->type, rounding_shift_right(widening_mul(x, y), z)), + rounding_mul_shift_right(x, y, cast(unsigned_type, z)), + is_x_same_int_or_uint && x_y_same_sign && is_uint(z)) || + false) { + return mutate(rewrite.result); + } + if (no_overflow(op->type)) { // clang-format off if (rewrite(halving_add(x + y, 1), rounding_halving_add(x, y)) || diff --git a/src/IRMatch.h b/src/IRMatch.h index 756b900e1f4d..9e6e2e76af3c 100644 --- a/src/IRMatch.h +++ b/src/IRMatch.h @@ -1358,6 +1358,7 @@ struct Intrin { struct pattern_tag {}; Call::IntrinsicOp intrin; std::tuple args; + Type optional_type_hint; static constexpr uint32_t binds = bitwise_or_reduce((bindings::mask)...); @@ -1416,6 +1417,8 @@ struct Intrin { return likely_if_innermost(arg0); } else if (intrin == Call::abs) { return abs(arg0); + } else if (intrin == Call::saturating_cast) { + return saturating_cast(optional_type_hint, arg0); } Expr arg1 = std::get(args).make(state, type_hint); @@ -1541,6 +1544,12 @@ template auto saturating_sub(A &&a, B &&b) noexcept -> Intrin { return {Call::saturating_sub, pattern_arg(a), pattern_arg(b)}; } +template +auto saturating_cast(const Type &t, A &&a) noexcept -> Intrin { + Intrin p = {Call::saturating_cast, pattern_arg(a)}; + p.optional_type_hint = t; + return p; +} template auto halving_add(A &&a, B &&b) noexcept -> Intrin { return {Call::halving_add, pattern_arg(a), pattern_arg(b)}; From 19bc9d7dd61d9a2d792c9885e6ccb74abc76e905 Mon Sep 17 00:00:00 2001 From: Alexander Root Date: Tue, 2 Aug 2022 12:52:10 -0400 Subject: [PATCH 04/19] handle saturating_cast in Bounds.cpp + add bounds tests --- src/Bounds.cpp | 113 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/src/Bounds.cpp b/src/Bounds.cpp index eb2f9138268b..9b73cabf917b 100644 --- a/src/Bounds.cpp +++ b/src/Bounds.cpp @@ -1210,6 +1210,103 @@ class Bounds : public IRVisitor { bounds_of_type(t); } } + } else if (op->is_intrinsic(Call::saturating_cast)) { + internal_assert(op->args.size() == 1); + + Expr a = op->args[0]; + a.accept(this); + Interval a_interval = interval; + + bounds_of_type(t); + + // For float to float, guarantee infinities are always pinned to range. + if (t.is_float() && a.type().is_float()) { + if (t.bits() < a.type().bits()) { + // Casting to a smaller float, so clamp and then cast. + if (a_interval.has_lower_bound()) { + // If representable in the type, then use, otherwise return bounds_of_type(t).min + if (can_prove(a_interval.min >= t.min())) { + interval.min = cast(t, a_interval.min); + } + } + if (a_interval.has_upper_bound()) { + if (can_prove(a_interval.max <= t.max())) { + interval.max = cast(t, a_interval.max); + } + } + return; + } else { + // Casting to a wider float, so cast then clamp. + if (a_interval.has_lower_bound()) { + Expr casted_min = cast(t, a_interval.min); + + if (can_prove(casted_min >= t.min())) { + interval.min = casted_min; + } + } + if (a_interval.has_upper_bound()) { + Expr casted_max = cast(t, a_interval.max); + if (can_prove(casted_max <= t.max())) { + interval.max = casted_max; + } + } + return; + } + } else if (a.type() != t) { + // Limits for Int(2^n) or UInt(2^n) are not exactly representable in Float(2^n) + if (a.type().is_float() && !t.is_float() && t.bits() >= a.type().bits()) { + if (a_interval.has_lower_bound()) { + // min values turn out to be always representable + if (can_prove(a_interval.min >= t.min())) { + interval.min = cast(t, a_interval.min); + } + } + if (a_interval.has_upper_bound()) { + // This line depends on t.max() rounding upward, which should always + // be the case as it is one less than a representable value, thus + // the one larger is always the closest. + if (can_prove(a_interval.max <= t.max())) { + interval.max = cast(t, a_interval.max); + } + } + return; + } else { + if (a_interval.has_lower_bound()) { + if (!a.type().is_uint()) { + Expr min_bound = lossless_cast(a.type(), t.min()); + if (min_bound.defined()) { + // Need to prove that cast is safe. + if (can_prove(a_interval.min >= min_bound)) { + interval.min = cast(t, a_interval.min); + } + } else { + // Type of arg cannot represent t.min() (i.e. uint16 -> int8) + // Should be safe to not have the clamp. + interval.min = cast(t, a_interval.min); + } + } else { + // uints are bounded below by 0, so cast is safe. + interval.min = cast(t, a_interval.min); + } + } + if (a_interval.has_upper_bound()) { + Expr max_bound = lossless_cast(a.type(), t.max()); + if (max_bound.defined()) { + // Need to prove that cast is safe. + if (can_prove(a_interval.max <= max_bound)) { + interval.max = cast(t, a_interval.max); + } + } else { + interval.max = cast(t, a_interval.max); + } + } + return; + } + } else { + // a.type() == t + interval = a_interval; + return; + } } else if (op->is_intrinsic(Call::unsafe_promise_clamped) || op->is_intrinsic(Call::promise_clamped)) { // Unlike an explicit clamp, we are also permitted to @@ -3572,6 +3669,22 @@ void bounds_test() { check(scope, cast(u8_1) + cast(u8_2), u16(0), u16(255 * 2)); + check(scope, saturating_cast(clamp(x, 5, 10)), cast(5), cast(10)); + { + Expr imax_p1 = make_const(UInt(32), 0x80000000ull); + scope.push("x", Interval(cast(0), cast(imax_p1))); + check(scope, saturating_cast(max(cast(x), cast(5))), cast(5), Interval::pos_inf()); + scope.pop("x"); + } + { + Expr z = Variable::make(Float(32), "z"); + scope.push("z", Interval(cast(-1), cast(1))); + check(scope, saturating_cast(z), cast(-1), cast(1)); + check(scope, saturating_cast(z), cast(-1), cast(1)); + check(scope, saturating_cast(z), cast(-1), cast(1)); + check(scope, saturating_cast(z), cast(0), cast(1)); + } + { Scope scope; Expr x = Variable::make(UInt(16), "x"); From 48c1acf7cf592a6a88664ff22aa999d7f2dc112d Mon Sep 17 00:00:00 2001 From: Alexander Root Date: Tue, 2 Aug 2022 13:02:39 -0400 Subject: [PATCH 05/19] fix test --- src/Bounds.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Bounds.cpp b/src/Bounds.cpp index 9b73cabf917b..5a4aa876f534 100644 --- a/src/Bounds.cpp +++ b/src/Bounds.cpp @@ -3671,8 +3671,7 @@ void bounds_test() { check(scope, saturating_cast(clamp(x, 5, 10)), cast(5), cast(10)); { - Expr imax_p1 = make_const(UInt(32), 0x80000000ull); - scope.push("x", Interval(cast(0), cast(imax_p1))); + scope.push("x", Interval(cast(0), UInt(32).max())); check(scope, saturating_cast(max(cast(x), cast(5))), cast(5), Interval::pos_inf()); scope.pop("x"); } From b99aa122bf2804fe602a8a7fbc6480c647a186cd Mon Sep 17 00:00:00 2001 From: Alexander Root Date: Wed, 3 Aug 2022 13:23:21 -0400 Subject: [PATCH 06/19] fix saturating_cast bounds inference --- src/Bounds.cpp | 62 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/src/Bounds.cpp b/src/Bounds.cpp index 5a4aa876f534..10833d2b45c0 100644 --- a/src/Bounds.cpp +++ b/src/Bounds.cpp @@ -1271,34 +1271,39 @@ class Bounds : public IRVisitor { } return; } else { - if (a_interval.has_lower_bound()) { - if (!a.type().is_uint()) { - Expr min_bound = lossless_cast(a.type(), t.min()); - if (min_bound.defined()) { - // Need to prove that cast is safe. - if (can_prove(a_interval.min >= min_bound)) { - interval.min = cast(t, a_interval.min); - } - } else { - // Type of arg cannot represent t.min() (i.e. uint16 -> int8) - // Should be safe to not have the clamp. - interval.min = cast(t, a_interval.min); + // We can safely cast a_interval iff we can prove the values + // are within the range of t. + Expr min_bound = lossless_cast(a.type(), t.min()); + Expr max_bound = lossless_cast(a.type(), t.max()); + // If the inner type is not a uint and we can represent t.min() in a.type(), + // then we need to check that value is >= t.min(); + const bool check_lower_bound = !a.type().is_uint() && min_bound.defined(); + // We should always check upper bounds if a.type() can represent t.max(). + const bool check_upper_bound = max_bound.defined(); + + // Define a helper function for performing saturation. + auto check_safe_cast = [&](const Expr &value, const Expr &base) { + if (check_upper_bound && check_lower_bound) { + if (can_prove((value >= min_bound) && (value <= max_bound))) { + return cast(t, value); + } + } else if (check_upper_bound) { + if (can_prove(value <= max_bound)) { + return cast(t, value); + } + } else if (check_lower_bound) { + if (can_prove(a_interval.min >= min_bound)) { + return cast(t, value); } - } else { - // uints are bounded below by 0, so cast is safe. - interval.min = cast(t, a_interval.min); } + return base; + }; + + if (a_interval.has_lower_bound()) { + interval.min = check_safe_cast(a_interval.min, interval.min); } if (a_interval.has_upper_bound()) { - Expr max_bound = lossless_cast(a.type(), t.max()); - if (max_bound.defined()) { - // Need to prove that cast is safe. - if (can_prove(a_interval.max <= max_bound)) { - interval.max = cast(t, a_interval.max); - } - } else { - interval.max = cast(t, a_interval.max); - } + interval.max = check_safe_cast(a_interval.max, interval.max); } return; } @@ -3671,7 +3676,7 @@ void bounds_test() { check(scope, saturating_cast(clamp(x, 5, 10)), cast(5), cast(10)); { - scope.push("x", Interval(cast(0), UInt(32).max())); + scope.push("x", Interval(UInt(32).min(), UInt(32).max())); check(scope, saturating_cast(max(cast(x), cast(5))), cast(5), Interval::pos_inf()); scope.pop("x"); } @@ -3682,6 +3687,13 @@ void bounds_test() { check(scope, saturating_cast(z), cast(-1), cast(1)); check(scope, saturating_cast(z), cast(-1), cast(1)); check(scope, saturating_cast(z), cast(0), cast(1)); + scope.pop("z"); + } + { + Expr z = Variable::make(UInt(32), "z"); + scope.push("z", Interval(UInt(32).max(), UInt(32).max())); + check(scope, saturating_cast(z), Interval::neg_inf(), Interval::pos_inf()); + scope.pop("z"); } { From 9e15b57fc90cbb4516abc3361a8d7751ced635ab Mon Sep 17 00:00:00 2001 From: Alexander Root Date: Wed, 3 Aug 2022 14:01:14 -0400 Subject: [PATCH 07/19] update saturating_cast CodeGen --- src/CodeGen_ARM.cpp | 118 ++++++++++++++++++------------------ src/CodeGen_WebAssembly.cpp | 41 +++++++++++-- src/CodeGen_X86.cpp | 9 ++- 3 files changed, 99 insertions(+), 69 deletions(-) diff --git a/src/CodeGen_ARM.cpp b/src/CodeGen_ARM.cpp index 7fa4bd35b84f..03f2b8daa952 100644 --- a/src/CodeGen_ARM.cpp +++ b/src/CodeGen_ARM.cpp @@ -154,31 +154,31 @@ CodeGen_ARM::CodeGen_ARM(const Target &target) // SQSHL, UQSHL, SQSHLU - Saturating shift left by signed register. for (const Expr &rhs : {wild_i8x_, wild_u8x_}) { - casts.emplace_back("saturating_shift_left", i8_sat(widening_shift_left(wild_i8x_, rhs))); - casts.emplace_back("saturating_shift_left", u8_sat(widening_shift_left(wild_u8x_, rhs))); - casts.emplace_back("saturating_shift_left", u8_sat(widening_shift_left(wild_i8x_, rhs))); + calls.emplace_back("saturating_shift_left", i8_sat(widening_shift_left(wild_i8x_, rhs))); + calls.emplace_back("saturating_shift_left", u8_sat(widening_shift_left(wild_u8x_, rhs))); + calls.emplace_back("saturating_shift_left", u8_sat(widening_shift_left(wild_i8x_, rhs))); } for (const Expr &rhs : {wild_i16x_, wild_u16x_}) { - casts.emplace_back("saturating_shift_left", i16_sat(widening_shift_left(wild_i16x_, rhs))); - casts.emplace_back("saturating_shift_left", u16_sat(widening_shift_left(wild_u16x_, rhs))); - casts.emplace_back("saturating_shift_left", u16_sat(widening_shift_left(wild_i16x_, rhs))); + calls.emplace_back("saturating_shift_left", i16_sat(widening_shift_left(wild_i16x_, rhs))); + calls.emplace_back("saturating_shift_left", u16_sat(widening_shift_left(wild_u16x_, rhs))); + calls.emplace_back("saturating_shift_left", u16_sat(widening_shift_left(wild_i16x_, rhs))); } for (const Expr &rhs : {wild_i32x_, wild_u32x_}) { - casts.emplace_back("saturating_shift_left", i32_sat(widening_shift_left(wild_i32x_, rhs))); - casts.emplace_back("saturating_shift_left", u32_sat(widening_shift_left(wild_u32x_, rhs))); - casts.emplace_back("saturating_shift_left", u32_sat(widening_shift_left(wild_i32x_, rhs))); + calls.emplace_back("saturating_shift_left", i32_sat(widening_shift_left(wild_i32x_, rhs))); + calls.emplace_back("saturating_shift_left", u32_sat(widening_shift_left(wild_u32x_, rhs))); + calls.emplace_back("saturating_shift_left", u32_sat(widening_shift_left(wild_i32x_, rhs))); } // SQSHRN, UQSHRN, SQRSHRUN Saturating narrowing shift right by an (by immediate in [1, output bits]) - casts.emplace_back("saturating_shift_right_narrow", i8_sat(wild_i16x_ >> wild_u16_)); - casts.emplace_back("saturating_shift_right_narrow", u8_sat(wild_u16x_ >> wild_u16_)); - casts.emplace_back("saturating_shift_right_narrow", u8_sat(wild_i16x_ >> wild_u16_)); - casts.emplace_back("saturating_shift_right_narrow", i16_sat(wild_i32x_ >> wild_u32_)); - casts.emplace_back("saturating_shift_right_narrow", u16_sat(wild_u32x_ >> wild_u32_)); - casts.emplace_back("saturating_shift_right_narrow", u16_sat(wild_i32x_ >> wild_u32_)); - casts.emplace_back("saturating_shift_right_narrow", i32_sat(wild_i64x_ >> wild_u64_)); - casts.emplace_back("saturating_shift_right_narrow", u32_sat(wild_u64x_ >> wild_u64_)); - casts.emplace_back("saturating_shift_right_narrow", u32_sat(wild_i64x_ >> wild_u64_)); + calls.emplace_back("saturating_shift_right_narrow", i8_sat(wild_i16x_ >> wild_u16_)); + calls.emplace_back("saturating_shift_right_narrow", u8_sat(wild_u16x_ >> wild_u16_)); + calls.emplace_back("saturating_shift_right_narrow", u8_sat(wild_i16x_ >> wild_u16_)); + calls.emplace_back("saturating_shift_right_narrow", i16_sat(wild_i32x_ >> wild_u32_)); + calls.emplace_back("saturating_shift_right_narrow", u16_sat(wild_u32x_ >> wild_u32_)); + calls.emplace_back("saturating_shift_right_narrow", u16_sat(wild_i32x_ >> wild_u32_)); + calls.emplace_back("saturating_shift_right_narrow", i32_sat(wild_i64x_ >> wild_u64_)); + calls.emplace_back("saturating_shift_right_narrow", u32_sat(wild_u64x_ >> wild_u64_)); + calls.emplace_back("saturating_shift_right_narrow", u32_sat(wild_i64x_ >> wild_u64_)); // SRSHL, URSHL - Rounding shift left (by signed vector) // These are already written as rounding_shift_left @@ -190,15 +190,15 @@ CodeGen_ARM::CodeGen_ARM(const Target &target) // These patterns are almost identity, we just need to strip off the broadcast. // SQXTN, UQXTN, SQXTUN - Saturating narrow. - casts.emplace_back("saturating_narrow", i8_sat(wild_i16x_)); - casts.emplace_back("saturating_narrow", u8_sat(wild_u16x_)); - casts.emplace_back("saturating_narrow", u8_sat(wild_i16x_)); - casts.emplace_back("saturating_narrow", i16_sat(wild_i32x_)); - casts.emplace_back("saturating_narrow", u16_sat(wild_u32x_)); - casts.emplace_back("saturating_narrow", u16_sat(wild_i32x_)); - casts.emplace_back("saturating_narrow", i32_sat(wild_i64x_)); - casts.emplace_back("saturating_narrow", u32_sat(wild_u64x_)); - casts.emplace_back("saturating_narrow", u32_sat(wild_i64x_)); + calls.emplace_back("saturating_narrow", i8_sat(wild_i16x_)); + calls.emplace_back("saturating_narrow", u8_sat(wild_u16x_)); + calls.emplace_back("saturating_narrow", u8_sat(wild_i16x_)); + calls.emplace_back("saturating_narrow", i16_sat(wild_i32x_)); + calls.emplace_back("saturating_narrow", u16_sat(wild_u32x_)); + calls.emplace_back("saturating_narrow", u16_sat(wild_i32x_)); + calls.emplace_back("saturating_narrow", i32_sat(wild_i64x_)); + calls.emplace_back("saturating_narrow", u32_sat(wild_u64x_)); + calls.emplace_back("saturating_narrow", u32_sat(wild_i64x_)); // SQNEG - Saturating negate negations.emplace_back("saturating_negate", -max(wild_i8x_, -127)); @@ -798,38 +798,6 @@ void CodeGen_ARM::visit(const Cast *op) { return; } } - - // If we didn't find a pattern, try rewriting the cast. - static const vector> cast_rewrites = { - // Double or triple narrowing saturating casts are better expressed as - // regular narrowing casts. - {u8_sat(wild_u32x_), u8_sat(u16_sat(wild_u32x_))}, - {u8_sat(wild_i32x_), u8_sat(i16_sat(wild_i32x_))}, - {u8_sat(wild_f32x_), u8_sat(i16_sat(wild_f32x_))}, - {i8_sat(wild_u32x_), i8_sat(u16_sat(wild_u32x_))}, - {i8_sat(wild_i32x_), i8_sat(i16_sat(wild_i32x_))}, - {i8_sat(wild_f32x_), i8_sat(i16_sat(wild_f32x_))}, - {u16_sat(wild_u64x_), u16_sat(u32_sat(wild_u64x_))}, - {u16_sat(wild_i64x_), u16_sat(i32_sat(wild_i64x_))}, - {u16_sat(wild_f64x_), u16_sat(i32_sat(wild_f64x_))}, - {i16_sat(wild_u64x_), i16_sat(u32_sat(wild_u64x_))}, - {i16_sat(wild_i64x_), i16_sat(i32_sat(wild_i64x_))}, - {i16_sat(wild_f64x_), i16_sat(i32_sat(wild_f64x_))}, - {u8_sat(wild_u64x_), u8_sat(u16_sat(u32_sat(wild_u64x_)))}, - {u8_sat(wild_i64x_), u8_sat(i16_sat(i32_sat(wild_i64x_)))}, - {u8_sat(wild_f64x_), u8_sat(i16_sat(i32_sat(wild_f64x_)))}, - {i8_sat(wild_u64x_), i8_sat(u16_sat(u32_sat(wild_u64x_)))}, - {i8_sat(wild_i64x_), i8_sat(i16_sat(i32_sat(wild_i64x_)))}, - {i8_sat(wild_f64x_), i8_sat(i16_sat(i32_sat(wild_f64x_)))}, - }; - for (const auto &i : cast_rewrites) { - if (expr_match(i.first, op, matches)) { - Expr replacement = substitute("*", matches[0], with_lanes(i.second, op->type.lanes())); - debug(3) << "rewriting cast to: " << replacement << " from " << Expr(op) << "\n"; - value = codegen(replacement); - return; - } - } } // LLVM fptoui generates fcvtzs if src is fp16 scalar else fcvtzu. @@ -1183,6 +1151,38 @@ void CodeGen_ARM::visit(const Call *op) { } } } + + // If we didn't find a pattern, try rewriting any saturating casts. + static const vector> cast_rewrites = { + // Double or triple narrowing saturating casts are better expressed as + // regular narrowing casts. + {u8_sat(wild_u32x_), u8_sat(u16_sat(wild_u32x_))}, + {u8_sat(wild_i32x_), u8_sat(i16_sat(wild_i32x_))}, + {u8_sat(wild_f32x_), u8_sat(i16_sat(wild_f32x_))}, + {i8_sat(wild_u32x_), i8_sat(u16_sat(wild_u32x_))}, + {i8_sat(wild_i32x_), i8_sat(i16_sat(wild_i32x_))}, + {i8_sat(wild_f32x_), i8_sat(i16_sat(wild_f32x_))}, + {u16_sat(wild_u64x_), u16_sat(u32_sat(wild_u64x_))}, + {u16_sat(wild_i64x_), u16_sat(i32_sat(wild_i64x_))}, + {u16_sat(wild_f64x_), u16_sat(i32_sat(wild_f64x_))}, + {i16_sat(wild_u64x_), i16_sat(u32_sat(wild_u64x_))}, + {i16_sat(wild_i64x_), i16_sat(i32_sat(wild_i64x_))}, + {i16_sat(wild_f64x_), i16_sat(i32_sat(wild_f64x_))}, + {u8_sat(wild_u64x_), u8_sat(u16_sat(u32_sat(wild_u64x_)))}, + {u8_sat(wild_i64x_), u8_sat(i16_sat(i32_sat(wild_i64x_)))}, + {u8_sat(wild_f64x_), u8_sat(i16_sat(i32_sat(wild_f64x_)))}, + {i8_sat(wild_u64x_), i8_sat(u16_sat(u32_sat(wild_u64x_)))}, + {i8_sat(wild_i64x_), i8_sat(i16_sat(i32_sat(wild_i64x_)))}, + {i8_sat(wild_f64x_), i8_sat(i16_sat(i32_sat(wild_f64x_)))}, + }; + for (const auto &i : cast_rewrites) { + if (expr_match(i.first, op, matches)) { + Expr replacement = substitute("*", matches[0], with_lanes(i.second, op->type.lanes())); + debug(3) << "rewriting cast to: " << replacement << " from " << Expr(op) << "\n"; + value = codegen(replacement); + return; + } + } } if (target.has_feature(Target::ARMFp16)) { diff --git a/src/CodeGen_WebAssembly.cpp b/src/CodeGen_WebAssembly.cpp index 2a63b8df2f36..c9776ddc1e89 100644 --- a/src/CodeGen_WebAssembly.cpp +++ b/src/CodeGen_WebAssembly.cpp @@ -37,6 +37,7 @@ class CodeGen_WebAssembly : public CodeGen_Posix { bool use_pic() const override; void visit(const Cast *) override; + void visit(const Call *) override; void codegen_vector_reduce(const VectorReduce *, const Expr &) override; }; @@ -147,11 +148,6 @@ void CodeGen_WebAssembly::visit(const Cast *op) { // clang-format off static const Pattern patterns[] = { - {"q15mulr_sat_s", i16_sat(rounding_shift_right(widening_mul(wild_i16x_, wild_i16x_), u16(15))), Target::WasmSimd128}, - {"saturating_narrow", i8_sat(wild_i16x_), Target::WasmSimd128}, - {"saturating_narrow", u8_sat(wild_i16x_), Target::WasmSimd128}, - {"saturating_narrow", i16_sat(wild_i32x_), Target::WasmSimd128}, - {"saturating_narrow", u16_sat(wild_i32x_), Target::WasmSimd128}, {"int_to_double", f64(wild_i32x_), Target::WasmSimd128}, {"int_to_double", f64(wild_u32x_), Target::WasmSimd128}, #if LLVM_VERSION == 130 @@ -184,6 +180,41 @@ void CodeGen_WebAssembly::visit(const Cast *op) { CodeGen_Posix::visit(op); } +void CodeGen_WebAssembly::visit(const Call *op) { + struct Pattern { + std::string intrin; ///< Name of the intrinsic + Expr pattern; ///< The pattern to match against + Target::Feature required_feature; + }; + + // clang-format off + static const Pattern patterns[] = { + {"q15mulr_sat_s", i16_sat(rounding_shift_right(widening_mul(wild_i16x_, wild_i16x_), u16(15))), Target::WasmSimd128}, + {"saturating_narrow", i8_sat(wild_i16x_), Target::WasmSimd128}, + {"saturating_narrow", u8_sat(wild_i16x_), Target::WasmSimd128}, + {"saturating_narrow", i16_sat(wild_i32x_), Target::WasmSimd128}, + {"saturating_narrow", u16_sat(wild_i32x_), Target::WasmSimd128}, + }; + // clang-format on + + if (op->type.is_vector()) { + std::vector matches; + for (const Pattern &p : patterns) { + if (!target.has_feature(p.required_feature)) { + continue; + } + if (expr_match(p.pattern, op, matches)) { + value = call_overloaded_intrin(op->type, p.intrin, matches); + if (value) { + return; + } + } + } + } + + CodeGen_Posix::visit(op); +} + void CodeGen_WebAssembly::codegen_vector_reduce(const VectorReduce *op, const Expr &init) { struct Pattern { VectorReduce::Operator reduce_op; diff --git a/src/CodeGen_X86.cpp b/src/CodeGen_X86.cpp index e208b0d10981..5d599409fb61 100644 --- a/src/CodeGen_X86.cpp +++ b/src/CodeGen_X86.cpp @@ -473,11 +473,6 @@ void CodeGen_X86::visit(const Cast *op) { // saturate the result. {"pmulhrs", i16(rounding_shift_right(widening_mul(wild_i16x_, wild_i16x_), 15))}, - {"saturating_narrow", i16_sat(wild_i32x_)}, - {"saturating_narrow", u16_sat(wild_i32x_)}, - {"saturating_narrow", i8_sat(wild_i16x_)}, - {"saturating_narrow", u8_sat(wild_i16x_)}, - {"f32_to_bf16", bf16(wild_f32x_)}, }; // clang-format on @@ -575,6 +570,10 @@ void CodeGen_X86::visit(const Call *op) { {"pmulh", mul_shift_right(wild_i16x_, wild_i16x_, 16)}, {"pmulh", mul_shift_right(wild_u16x_, wild_u16x_, 16)}, {"saturating_pmulhrs", rounding_mul_shift_right(wild_i16x_, wild_i16x_, 15)}, + {"saturating_narrow", i16_sat(wild_i32x_)}, + {"saturating_narrow", u16_sat(wild_i32x_)}, + {"saturating_narrow", i8_sat(wild_i16x_)}, + {"saturating_narrow", u8_sat(wild_i16x_)}, }; // clang-format on From 53e2a644ba693f70149a7da6a710f08c3e3f9cbc Mon Sep 17 00:00:00 2001 From: Alexander Root Date: Wed, 3 Aug 2022 14:06:42 -0400 Subject: [PATCH 08/19] update HexagonOptimize as well --- src/HexagonOptimize.cpp | 101 +++++++++++++++++++++++----------------- 1 file changed, 59 insertions(+), 42 deletions(-) diff --git a/src/HexagonOptimize.cpp b/src/HexagonOptimize.cpp index 3749a9434b42..1cdc525398df 100644 --- a/src/HexagonOptimize.cpp +++ b/src/HexagonOptimize.cpp @@ -797,42 +797,6 @@ class OptimizePatterns : public IRMutator { // Halving unsigned subtract. {"halide.hexagon.navg.vub.vub", i8(widening_sub(wild_u8x, wild_u8x) >> 1)}, - // Saturating narrowing casts with rounding - {"halide.hexagon.trunc_satub_rnd.vh", u8_sat(rounding_shift_right(wild_i16x, 8)), Pattern::DeinterleaveOp0}, - {"halide.hexagon.trunc_satb_rnd.vh", i8_sat(rounding_shift_right(wild_i16x, 8)), Pattern::DeinterleaveOp0}, - {"halide.hexagon.trunc_satub_rnd.vuh", u8_sat(rounding_shift_right(wild_u16x, 8)), Pattern::DeinterleaveOp0}, - {"halide.hexagon.trunc_satuh_rnd.vw", u16_sat(rounding_shift_right(wild_i32x, 16)), Pattern::DeinterleaveOp0}, - {"halide.hexagon.trunc_sath_rnd.vw", i16_sat(rounding_shift_right(wild_i32x, 16)), Pattern::DeinterleaveOp0}, - {"halide.hexagon.trunc_satuh_rnd.vuw", u16_sat(rounding_shift_right(wild_u32x, 16)), Pattern::DeinterleaveOp0}, - - // Saturating narrowing casts with rounding - {"halide.hexagon.trunc_satub_shr_rnd.vh", u8_sat(rounding_shift_right(wild_i16x, wild_u16)), Pattern::DeinterleaveOp0}, - {"halide.hexagon.trunc_satb_shr_rnd.vh", i8_sat(rounding_shift_right(wild_i16x, wild_u16)), Pattern::DeinterleaveOp0}, - {"halide.hexagon.trunc_satub_shr_rnd.vuh", u8_sat(rounding_shift_right(wild_u16x, wild_u16)), Pattern::DeinterleaveOp0 | Pattern::v65orLater}, - {"halide.hexagon.trunc_satuh_shr_rnd.vw", u16_sat(rounding_shift_right(wild_i32x, wild_u32)), Pattern::DeinterleaveOp0}, - {"halide.hexagon.trunc_sath_shr_rnd.vw", i16_sat(rounding_shift_right(wild_i32x, wild_u32)), Pattern::DeinterleaveOp0}, - {"halide.hexagon.trunc_satuh_shr_rnd.vuw", u16_sat(rounding_shift_right(wild_u32x, wild_u32)), Pattern::DeinterleaveOp0}, - - // Saturating narrowing casts - {"halide.hexagon.trunc_satub_shr.vh.uh", u8_sat(wild_i16x >> wild_u16), Pattern::DeinterleaveOp0}, - {"halide.hexagon.trunc_satuh_shr.vw.uw", u16_sat(wild_i32x >> wild_u32), Pattern::DeinterleaveOp0}, - {"halide.hexagon.trunc_sath_shr.vw.uw", i16_sat(wild_i32x >> wild_u32), Pattern::DeinterleaveOp0}, - - // For some of the following narrowing casts, we have the choice of - // non-interleaving or interleaving instructions. Because we don't - // know which one we prefer during pattern matching, we match the - // non-interleaving versions for now and replace them with the - // instructions that interleave later if it makes sense. - - // Saturating narrowing casts. These may interleave later with trunc_sat. - {"halide.hexagon.pack_satub.vh", u8_sat(wild_i16x)}, - {"halide.hexagon.pack_satuh.vw", u16_sat(wild_i32x)}, - {"halide.hexagon.pack_satb.vh", i8_sat(wild_i16x)}, - {"halide.hexagon.pack_sath.vw", i16_sat(wild_i32x)}, - - // We don't have a vpack equivalent to this one, so we match it directly. - {"halide.hexagon.trunc_satuh.vuw", u16_sat(wild_u32x), Pattern::DeinterleaveOp0}, - // Narrowing casts. These may interleave later with trunclo. {"halide.hexagon.packhi.vh", u8(wild_u16x >> 8)}, {"halide.hexagon.packhi.vh", u8(wild_i16x >> 8)}, @@ -872,12 +836,6 @@ class OptimizePatterns : public IRMutator { // fall through to LLVM, which will generate large unoptimized // shuffles. static const vector> cast_rewrites = { - // Saturating narrowing - {u8_sat(wild_u32x), u8_sat(u16_sat(wild_u32x))}, - {u8_sat(wild_i32x), u8_sat(i16_sat(wild_i32x))}, - {i8_sat(wild_u32x), i8_sat(u16_sat(wild_u32x))}, - {i8_sat(wild_i32x), i8_sat(i16_sat(wild_i32x))}, - // Narrowing {u8(wild_u32x), u8(u16(wild_u32x))}, {u8(wild_i32x), u8(i16(wild_i32x))}, @@ -942,6 +900,42 @@ class OptimizePatterns : public IRMutator { } static const vector calls = { + // Saturating narrowing casts with rounding + {"halide.hexagon.trunc_satub_rnd.vh", u8_sat(rounding_shift_right(wild_i16x, 8)), Pattern::DeinterleaveOp0}, + {"halide.hexagon.trunc_satb_rnd.vh", i8_sat(rounding_shift_right(wild_i16x, 8)), Pattern::DeinterleaveOp0}, + {"halide.hexagon.trunc_satub_rnd.vuh", u8_sat(rounding_shift_right(wild_u16x, 8)), Pattern::DeinterleaveOp0}, + {"halide.hexagon.trunc_satuh_rnd.vw", u16_sat(rounding_shift_right(wild_i32x, 16)), Pattern::DeinterleaveOp0}, + {"halide.hexagon.trunc_sath_rnd.vw", i16_sat(rounding_shift_right(wild_i32x, 16)), Pattern::DeinterleaveOp0}, + {"halide.hexagon.trunc_satuh_rnd.vuw", u16_sat(rounding_shift_right(wild_u32x, 16)), Pattern::DeinterleaveOp0}, + + // Saturating narrowing casts with rounding + {"halide.hexagon.trunc_satub_shr_rnd.vh", u8_sat(rounding_shift_right(wild_i16x, wild_u16)), Pattern::DeinterleaveOp0}, + {"halide.hexagon.trunc_satb_shr_rnd.vh", i8_sat(rounding_shift_right(wild_i16x, wild_u16)), Pattern::DeinterleaveOp0}, + {"halide.hexagon.trunc_satub_shr_rnd.vuh", u8_sat(rounding_shift_right(wild_u16x, wild_u16)), Pattern::DeinterleaveOp0 | Pattern::v65orLater}, + {"halide.hexagon.trunc_satuh_shr_rnd.vw", u16_sat(rounding_shift_right(wild_i32x, wild_u32)), Pattern::DeinterleaveOp0}, + {"halide.hexagon.trunc_sath_shr_rnd.vw", i16_sat(rounding_shift_right(wild_i32x, wild_u32)), Pattern::DeinterleaveOp0}, + {"halide.hexagon.trunc_satuh_shr_rnd.vuw", u16_sat(rounding_shift_right(wild_u32x, wild_u32)), Pattern::DeinterleaveOp0}, + + // Saturating narrowing casts + {"halide.hexagon.trunc_satub_shr.vh.uh", u8_sat(wild_i16x >> wild_u16), Pattern::DeinterleaveOp0}, + {"halide.hexagon.trunc_satuh_shr.vw.uw", u16_sat(wild_i32x >> wild_u32), Pattern::DeinterleaveOp0}, + {"halide.hexagon.trunc_sath_shr.vw.uw", i16_sat(wild_i32x >> wild_u32), Pattern::DeinterleaveOp0}, + + // For some of the following narrowing casts, we have the choice of + // non-interleaving or interleaving instructions. Because we don't + // know which one we prefer during pattern matching, we match the + // non-interleaving versions for now and replace them with the + // instructions that interleave later if it makes sense. + + // Saturating narrowing casts. These may interleave later with trunc_sat. + {"halide.hexagon.pack_satub.vh", u8_sat(wild_i16x)}, + {"halide.hexagon.pack_satuh.vw", u16_sat(wild_i32x)}, + {"halide.hexagon.pack_satb.vh", i8_sat(wild_i16x)}, + {"halide.hexagon.pack_sath.vw", i16_sat(wild_i32x)}, + + // We don't have a vpack equivalent to this one, so we match it directly. + {"halide.hexagon.trunc_satuh.vuw", u16_sat(wild_u32x), Pattern::DeinterleaveOp0}, + // Multiply keep high half. {"halide.hexagon.trunc_mpy.vw.vw", mul_shift_right(wild_i32x, wild_i32x, 32)}, @@ -980,11 +974,34 @@ class OptimizePatterns : public IRMutator { {"halide.hexagon.mpy.vh.vuh", widening_mul(wild_u16x, wild_i16x), Pattern::InterleaveResult | Pattern::SwapOps01}, }; + // To hit more of the patterns we want, rewrite "double casts" + // as two stage casts. This also avoids letting vector casts + // fall through to LLVM, which will generate large unoptimized + // shuffles. + static const vector> cast_rewrites = { + // Saturating narrowing + {u8_sat(wild_u32x), u8_sat(u16_sat(wild_u32x))}, + {u8_sat(wild_i32x), u8_sat(i16_sat(wild_i32x))}, + {i8_sat(wild_u32x), i8_sat(u16_sat(wild_u32x))}, + {i8_sat(wild_i32x), i8_sat(i16_sat(wild_i32x))}, + }; + if (op->type.is_vector()) { Expr new_expr = apply_patterns(op, calls, target, this); if (!new_expr.same_as(op)) { return new_expr; } + + // If we didn't find a pattern, try using one of the + // rewrites above. + vector matches; + for (const auto &i : cast_rewrites) { + if (expr_match(i.first, op, matches)) { + Expr replacement = substitute("*", matches[0], with_lanes(i.second, op->type.lanes())); + debug(3) << "rewriting cast to: " << replacement << " from " << Expr(op) << "\n"; + return mutate(replacement); + } + } } if (op->is_intrinsic(Call::lerp)) { From 1cac0280e1a50556ac1936a15ca1031f17ee84e0 Mon Sep 17 00:00:00 2001 From: Alexander Root Date: Wed, 3 Aug 2022 14:07:24 -0400 Subject: [PATCH 09/19] with_lanes should work on intrinsics as well --- src/IRMatch.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/IRMatch.cpp b/src/IRMatch.cpp index 6aba3155777f..fe66579b962a 100644 --- a/src/IRMatch.cpp +++ b/src/IRMatch.cpp @@ -384,6 +384,16 @@ class WithLanes : public IRMutator { } } + Expr visit(const Call *op) override { + if (op->is_intrinsic() && (op->type.lanes() != lanes)) { + auto [new_args, changed] = mutate_with_changes(op->args); + return Call::make(with_lanes(op->type), op->name, new_args, op->call_type, + op->func, op->value_index, op->image, op->param); + } else { + return IRMutator::visit(op); + } + } + public: WithLanes(int lanes) : lanes(lanes) { From 73d2b81219dfffe2455659b0fbabb38f65851548 Mon Sep 17 00:00:00 2001 From: Alexander Root Date: Wed, 3 Aug 2022 14:21:55 -0400 Subject: [PATCH 10/19] fix missed ARM codegen updates --- src/CodeGen_ARM.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/CodeGen_ARM.cpp b/src/CodeGen_ARM.cpp index 03f2b8daa952..c445d3477b81 100644 --- a/src/CodeGen_ARM.cpp +++ b/src/CodeGen_ARM.cpp @@ -142,15 +142,15 @@ CodeGen_ARM::CodeGen_ARM(const Target &target) // TODO: We need to match rounding shift right, and negate the RHS. // SQRSHRN, SQRSHRUN, UQRSHRN - Saturating rounding narrowing shift right narrow (by immediate in [1, output bits]) - casts.emplace_back("saturating_rounding_shift_right_narrow", i8_sat(rounding_shift_right(wild_i16x_, wild_u16_))); - casts.emplace_back("saturating_rounding_shift_right_narrow", u8_sat(rounding_shift_right(wild_u16x_, wild_u16_))); - casts.emplace_back("saturating_rounding_shift_right_narrow", u8_sat(rounding_shift_right(wild_i16x_, wild_u16_))); - casts.emplace_back("saturating_rounding_shift_right_narrow", i16_sat(rounding_shift_right(wild_i32x_, wild_u32_))); - casts.emplace_back("saturating_rounding_shift_right_narrow", u16_sat(rounding_shift_right(wild_u32x_, wild_u32_))); - casts.emplace_back("saturating_rounding_shift_right_narrow", u16_sat(rounding_shift_right(wild_i32x_, wild_u32_))); - casts.emplace_back("saturating_rounding_shift_right_narrow", i32_sat(rounding_shift_right(wild_i64x_, wild_u64_))); - casts.emplace_back("saturating_rounding_shift_right_narrow", u32_sat(rounding_shift_right(wild_u64x_, wild_u64_))); - casts.emplace_back("saturating_rounding_shift_right_narrow", u32_sat(rounding_shift_right(wild_i64x_, wild_u64_))); + calls.emplace_back("saturating_rounding_shift_right_narrow", i8_sat(rounding_shift_right(wild_i16x_, wild_u16_))); + calls.emplace_back("saturating_rounding_shift_right_narrow", u8_sat(rounding_shift_right(wild_u16x_, wild_u16_))); + calls.emplace_back("saturating_rounding_shift_right_narrow", u8_sat(rounding_shift_right(wild_i16x_, wild_u16_))); + calls.emplace_back("saturating_rounding_shift_right_narrow", i16_sat(rounding_shift_right(wild_i32x_, wild_u32_))); + calls.emplace_back("saturating_rounding_shift_right_narrow", u16_sat(rounding_shift_right(wild_u32x_, wild_u32_))); + calls.emplace_back("saturating_rounding_shift_right_narrow", u16_sat(rounding_shift_right(wild_i32x_, wild_u32_))); + calls.emplace_back("saturating_rounding_shift_right_narrow", i32_sat(rounding_shift_right(wild_i64x_, wild_u64_))); + calls.emplace_back("saturating_rounding_shift_right_narrow", u32_sat(rounding_shift_right(wild_u64x_, wild_u64_))); + calls.emplace_back("saturating_rounding_shift_right_narrow", u32_sat(rounding_shift_right(wild_i64x_, wild_u64_))); // SQSHL, UQSHL, SQSHLU - Saturating shift left by signed register. for (const Expr &rhs : {wild_i8x_, wild_u8x_}) { From a86e84f876f415a00f454d6d6773bb1f603c4cb4 Mon Sep 17 00:00:00 2001 From: Alexander Root Date: Wed, 3 Aug 2022 14:22:30 -0400 Subject: [PATCH 11/19] lift to saturating_cast in FindIntrinsics --- src/FindIntrinsics.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/FindIntrinsics.cpp b/src/FindIntrinsics.cpp index dbbb961b2a99..136a9bb59b12 100644 --- a/src/FindIntrinsics.cpp +++ b/src/FindIntrinsics.cpp @@ -410,6 +410,14 @@ class FindIntrinsics : public IRMutator { saturating_sub(x, y), op->type.is_uint() && is_x_same_uint) || + // Saturating narrow patterns. + rewrite(max(min(x, upper), lower), + saturating_cast(op->type, x)) || + + rewrite(min(x, upper), + saturating_cast(op->type, x), + is_uint(x)) || + // Averaging patterns // // We have a slight preference for rounding_halving_add over From 82bc087d029a2efbd65589e77045769c9353b8eb Mon Sep 17 00:00:00 2001 From: Alexander Root Date: Wed, 3 Aug 2022 20:23:03 -0400 Subject: [PATCH 12/19] update intrinsics test for u16_sat --- test/correctness/intrinsics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/correctness/intrinsics.cpp b/test/correctness/intrinsics.cpp index 962b696547c0..068f360d214f 100644 --- a/test/correctness/intrinsics.cpp +++ b/test/correctness/intrinsics.cpp @@ -246,7 +246,7 @@ int main(int argc, char **argv) { check(narrow((u16(u8x) + 500) >> 4), narrow((u16(u8x) + 500) >> 4)); check((u64(u32x) + 8) / 16, u64(rounding_shift_right(u32x, 4))); - check(u16(min((u64(u32x) + 8) / 16, 65535)), u16(min(rounding_shift_right(u32x, 4), 65535))); + check(u16(min((u64(u32x) + 8) / 16, 65535)), u16_sat(rounding_shift_right(u32x, 4))); // And with variable shifts. check(i8(widening_add(i8x, (i8(1) << u8y) / 2) >> u8y), rounding_shift_right(i8x, u8y)); From 880ad1dd4cf243aece174e47b25070cc753f03e8 Mon Sep 17 00:00:00 2001 From: Alexander Root Date: Wed, 3 Aug 2022 21:10:25 -0400 Subject: [PATCH 13/19] better sat_cast(widen(expr)) handling in find_intrinsics --- src/FindIntrinsics.cpp | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/src/FindIntrinsics.cpp b/src/FindIntrinsics.cpp index 136a9bb59b12..0a33b85822aa 100644 --- a/src/FindIntrinsics.cpp +++ b/src/FindIntrinsics.cpp @@ -589,8 +589,13 @@ class FindIntrinsics : public IRMutator { const auto is_x_same_int_or_uint = is_x_same_int || is_x_same_uint; auto x_y_same_sign = (is_int(x) == is_int(y)) || (is_uint(x) && is_uint(y)); Type unsigned_type = op->type.with_code(halide_type_uint); + const auto is_x_wider_int_or_uint = (op->type.is_int() && is_int(x, 2 * bits)) || (op->type.is_uint() && is_uint(x, 2 * bits)); + Type opposite_type = op->type.is_int() ? op->type.with_code(halide_type_uint) : op->type.with_code(halide_type_int); + const auto is_x_wider_opposite_int = (op->type.is_int() && is_uint(x, 2 * bits)) || (op->type.is_uint() && is_int(x, 2 * bits)); - if (rewrite(saturating_cast(op->type, widening_add(x, y)), + if ( + // Saturating patterns. + rewrite(saturating_cast(op->type, widening_add(x, y)), saturating_add(x, y), is_x_same_int_or_uint) || rewrite(saturating_cast(op->type, widening_sub(x, y)), @@ -602,6 +607,29 @@ class FindIntrinsics : public IRMutator { rewrite(saturating_cast(op->type, rounding_shift_right(widening_mul(x, y), z)), rounding_mul_shift_right(x, y, cast(unsigned_type, z)), is_x_same_int_or_uint && x_y_same_sign && is_uint(z)) || + // We can remove unnecessary widening if we are then performing a saturating narrow. + // This is similar to the logic inside `visit_min_or_max`. + (((bits <= 32) && + // Examples: + // i8_sat(int16(i8)) -> i8 + // u8_sat(uint16(u8)) -> u8 + rewrite(saturating_cast(op->type, cast(op->type.widen(), x)), + x, + is_x_same_int_or_uint)) || + ((bits <= 16) && + // Examples: + // i8_sat(int32(i16)) -> i8_sat(i16) + // u8_sat(uint32(u16)) -> u8_sat(u16) + (rewrite(saturating_cast(op->type, cast(op->type.widen().widen(), x)), + saturating_cast(op->type, x), + is_x_wider_int_or_uint) || + // Examples: + // i8_sat(uint32(u16)) -> i8_sat(u16) + // u8_sat(int32(i16)) -> i8_sat(i16) + rewrite(saturating_cast(op->type, cast(opposite_type.widen().widen(), x)), + saturating_cast(op->type, x), + is_x_wider_opposite_int) || + false))) || false) { return mutate(rewrite.result); } From 898438f0bb6389f20becc264d9d612e102439215 Mon Sep 17 00:00:00 2001 From: Alexander Root Date: Wed, 3 Aug 2022 21:20:08 -0400 Subject: [PATCH 14/19] rm unused variable --- src/IRMatch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IRMatch.cpp b/src/IRMatch.cpp index fe66579b962a..8416d223ffc4 100644 --- a/src/IRMatch.cpp +++ b/src/IRMatch.cpp @@ -386,7 +386,7 @@ class WithLanes : public IRMutator { Expr visit(const Call *op) override { if (op->is_intrinsic() && (op->type.lanes() != lanes)) { - auto [new_args, changed] = mutate_with_changes(op->args); + auto new_args = mutate_with_changes(op->args).first; return Call::make(with_lanes(op->type), op->name, new_args, op->call_type, op->func, op->value_index, op->image, op->param); } else { From 5011b1eb3b5b6cde745fbca0834cf3ac7817581a Mon Sep 17 00:00:00 2001 From: Alexander Root Date: Wed, 3 Aug 2022 21:53:52 -0400 Subject: [PATCH 15/19] add shift_right_narrow checks to ARM Call visitor --- src/CodeGen_ARM.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/CodeGen_ARM.cpp b/src/CodeGen_ARM.cpp index c445d3477b81..fbcff72d7db1 100644 --- a/src/CodeGen_ARM.cpp +++ b/src/CodeGen_ARM.cpp @@ -1145,6 +1145,17 @@ void CodeGen_ARM::visit(const Call *op) { vector matches; for (const Pattern &pattern : calls) { if (expr_match(pattern.pattern, op, matches)) { + if (pattern.intrin.find("shift_right_narrow") != string::npos) { + // The shift_right_narrow patterns need the shift to be constant in [1, output_bits]. + const uint64_t *const_b = as_const_uint(matches[1]); + if (!const_b || *const_b == 0 || (int)*const_b > op->type.bits()) { + continue; + } + } + if (target.bits == 32 && pattern.intrin.find("shift_right") != string::npos) { + // The 32-bit ARM backend wants right shifts as negative values. + matches[1] = simplify(-cast(matches[1].type().with_code(halide_type_int), matches[1])); + } value = call_overloaded_intrin(op->type, pattern.intrin, matches); if (value) { return; From 91c13529d73efe5d33e9d57976e71fbf1bf892eb Mon Sep 17 00:00:00 2001 From: Alexander Root Date: Thu, 4 Aug 2022 14:12:53 -0400 Subject: [PATCH 16/19] simplify bounds of saturating_cast + update is_monotonic --- src/Bounds.cpp | 104 ++++-------------------------------------- src/Monotonic.cpp | 3 +- src/Simplify_Call.cpp | 16 +++++++ 3 files changed, 26 insertions(+), 97 deletions(-) diff --git a/src/Bounds.cpp b/src/Bounds.cpp index 10833d2b45c0..beae9bff317d 100644 --- a/src/Bounds.cpp +++ b/src/Bounds.cpp @@ -1216,102 +1216,14 @@ class Bounds : public IRVisitor { Expr a = op->args[0]; a.accept(this); Interval a_interval = interval; - bounds_of_type(t); - - // For float to float, guarantee infinities are always pinned to range. - if (t.is_float() && a.type().is_float()) { - if (t.bits() < a.type().bits()) { - // Casting to a smaller float, so clamp and then cast. - if (a_interval.has_lower_bound()) { - // If representable in the type, then use, otherwise return bounds_of_type(t).min - if (can_prove(a_interval.min >= t.min())) { - interval.min = cast(t, a_interval.min); - } - } - if (a_interval.has_upper_bound()) { - if (can_prove(a_interval.max <= t.max())) { - interval.max = cast(t, a_interval.max); - } - } - return; - } else { - // Casting to a wider float, so cast then clamp. - if (a_interval.has_lower_bound()) { - Expr casted_min = cast(t, a_interval.min); - - if (can_prove(casted_min >= t.min())) { - interval.min = casted_min; - } - } - if (a_interval.has_upper_bound()) { - Expr casted_max = cast(t, a_interval.max); - if (can_prove(casted_max <= t.max())) { - interval.max = casted_max; - } - } - return; - } - } else if (a.type() != t) { - // Limits for Int(2^n) or UInt(2^n) are not exactly representable in Float(2^n) - if (a.type().is_float() && !t.is_float() && t.bits() >= a.type().bits()) { - if (a_interval.has_lower_bound()) { - // min values turn out to be always representable - if (can_prove(a_interval.min >= t.min())) { - interval.min = cast(t, a_interval.min); - } - } - if (a_interval.has_upper_bound()) { - // This line depends on t.max() rounding upward, which should always - // be the case as it is one less than a representable value, thus - // the one larger is always the closest. - if (can_prove(a_interval.max <= t.max())) { - interval.max = cast(t, a_interval.max); - } - } - return; - } else { - // We can safely cast a_interval iff we can prove the values - // are within the range of t. - Expr min_bound = lossless_cast(a.type(), t.min()); - Expr max_bound = lossless_cast(a.type(), t.max()); - // If the inner type is not a uint and we can represent t.min() in a.type(), - // then we need to check that value is >= t.min(); - const bool check_lower_bound = !a.type().is_uint() && min_bound.defined(); - // We should always check upper bounds if a.type() can represent t.max(). - const bool check_upper_bound = max_bound.defined(); - - // Define a helper function for performing saturation. - auto check_safe_cast = [&](const Expr &value, const Expr &base) { - if (check_upper_bound && check_lower_bound) { - if (can_prove((value >= min_bound) && (value <= max_bound))) { - return cast(t, value); - } - } else if (check_upper_bound) { - if (can_prove(value <= max_bound)) { - return cast(t, value); - } - } else if (check_lower_bound) { - if (can_prove(a_interval.min >= min_bound)) { - return cast(t, value); - } - } - return base; - }; - - if (a_interval.has_lower_bound()) { - interval.min = check_safe_cast(a_interval.min, interval.min); - } - if (a_interval.has_upper_bound()) { - interval.max = check_safe_cast(a_interval.max, interval.max); - } - return; - } - } else { - // a.type() == t - interval = a_interval; - return; + if (a_interval.has_lower_bound()) { + interval.min = saturating_cast(op->type, a_interval.min); } + if (a_interval.has_upper_bound()) { + interval.max = saturating_cast(op->type, a_interval.max); + } + return; } else if (op->is_intrinsic(Call::unsafe_promise_clamped) || op->is_intrinsic(Call::promise_clamped)) { // Unlike an explicit clamp, we are also permitted to @@ -3677,7 +3589,7 @@ void bounds_test() { check(scope, saturating_cast(clamp(x, 5, 10)), cast(5), cast(10)); { scope.push("x", Interval(UInt(32).min(), UInt(32).max())); - check(scope, saturating_cast(max(cast(x), cast(5))), cast(5), Interval::pos_inf()); + check(scope, saturating_cast(max(cast(x), cast(5))), cast(5), Int(32).max()); scope.pop("x"); } { @@ -3692,7 +3604,7 @@ void bounds_test() { { Expr z = Variable::make(UInt(32), "z"); scope.push("z", Interval(UInt(32).max(), UInt(32).max())); - check(scope, saturating_cast(z), Interval::neg_inf(), Interval::pos_inf()); + check(scope, saturating_cast(z), Int(32).max(), Int(32).max()); scope.pop("z"); } diff --git a/src/Monotonic.cpp b/src/Monotonic.cpp index ae8978b2cb57..cec309571aa8 100644 --- a/src/Monotonic.cpp +++ b/src/Monotonic.cpp @@ -480,7 +480,8 @@ class DerivativeBounds : public IRVisitor { } if (op->is_intrinsic(Call::unsafe_promise_clamped) || - op->is_intrinsic(Call::promise_clamped)) { + op->is_intrinsic(Call::promise_clamped) || + op->is_intrinsic(Call::saturating_cast)) { op->args[0].accept(this); return; } diff --git a/src/Simplify_Call.cpp b/src/Simplify_Call.cpp index a1ff4c5130fe..6c92fd086405 100644 --- a/src/Simplify_Call.cpp +++ b/src/Simplify_Call.cpp @@ -1,5 +1,6 @@ #include "Simplify_Internal.h" +#include "FindIntrinsics.h" #include "Simplify.h" #ifdef _MSC_VER @@ -351,6 +352,21 @@ Expr Simplify::visit(const Call *op, ExprInfo *bounds) { } else { return absd(a, b); } + } else if (op->is_intrinsic(Call::saturating_cast)) { + internal_assert(op->args.size() == 1); + ExprInfo a_bounds; + Expr a = mutate(op->args[0], &a_bounds); + + // TODO(rootjalex): We could be intelligent about using a_bounds to remove saturating_casts; + + if (is_const(a)) { + a = lower_saturating_cast(op->type, a); + return mutate(a, bounds); + } else if (!a.same_as(op->args[0])) { + return saturating_cast(op->type, a); + } else { + return op; + } } else if (op->is_intrinsic(Call::stringify)) { // Eagerly concat constant arguments to a stringify. bool changed = false; From c16746ff0ac8c2dd2b277222b5151625a6e27588 Mon Sep 17 00:00:00 2001 From: Alexander Root Date: Sun, 7 Aug 2022 15:59:46 -0400 Subject: [PATCH 17/19] use type.element_of() in bounds inference --- src/Bounds.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Bounds.cpp b/src/Bounds.cpp index beae9bff317d..8090a20e721f 100644 --- a/src/Bounds.cpp +++ b/src/Bounds.cpp @@ -1218,10 +1218,10 @@ class Bounds : public IRVisitor { Interval a_interval = interval; bounds_of_type(t); if (a_interval.has_lower_bound()) { - interval.min = saturating_cast(op->type, a_interval.min); + interval.min = saturating_cast(t, a_interval.min); } if (a_interval.has_upper_bound()) { - interval.max = saturating_cast(op->type, a_interval.max); + interval.max = saturating_cast(t, a_interval.max); } return; } else if (op->is_intrinsic(Call::unsafe_promise_clamped) || From ffa96ceeb93d73ecc242a43898854bd54bba3f69 Mon Sep 17 00:00:00 2001 From: Alexander Root Date: Sun, 7 Aug 2022 16:09:08 -0400 Subject: [PATCH 18/19] address nits + check type_hint in Intrin pattern matching --- src/CodeGen_ARM.cpp | 2 +- src/IRMatch.h | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/CodeGen_ARM.cpp b/src/CodeGen_ARM.cpp index fbcff72d7db1..06acff932f9e 100644 --- a/src/CodeGen_ARM.cpp +++ b/src/CodeGen_ARM.cpp @@ -1166,7 +1166,7 @@ void CodeGen_ARM::visit(const Call *op) { // If we didn't find a pattern, try rewriting any saturating casts. static const vector> cast_rewrites = { // Double or triple narrowing saturating casts are better expressed as - // regular narrowing casts. + // combinations of single narrowing saturating casts. {u8_sat(wild_u32x_), u8_sat(u16_sat(wild_u32x_))}, {u8_sat(wild_i32x_), u8_sat(i16_sat(wild_i32x_))}, {u8_sat(wild_f32x_), u8_sat(i16_sat(wild_f32x_))}, diff --git a/src/IRMatch.h b/src/IRMatch.h index 9e6e2e76af3c..3a04c486676e 100644 --- a/src/IRMatch.h +++ b/src/IRMatch.h @@ -1358,6 +1358,9 @@ struct Intrin { struct pattern_tag {}; Call::IntrinsicOp intrin; std::tuple args; + // The type of the output of the intrinsic node. + // Only necessary in cases where it can't be inferred + // from the input types (e.g. saturating_cast). Type optional_type_hint; static constexpr uint32_t binds = bitwise_or_reduce((bindings::mask)...); @@ -1386,7 +1389,9 @@ struct Intrin { return false; } const Call &c = (const Call &)e; - return (c.is_intrinsic(intrin) && match_args<0, bound>(0, c, state)); + return (c.is_intrinsic(intrin) && + ((optional_type_hint == Type()) || optional_type_hint == e.type) && + match_args<0, bound>(0, c, state)); } template Date: Sun, 7 Aug 2022 16:22:03 -0400 Subject: [PATCH 19/19] clang format --- src/IRMatch.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/IRMatch.h b/src/IRMatch.h index 3a04c486676e..0de1c21cee44 100644 --- a/src/IRMatch.h +++ b/src/IRMatch.h @@ -1390,8 +1390,8 @@ struct Intrin { } const Call &c = (const Call &)e; return (c.is_intrinsic(intrin) && - ((optional_type_hint == Type()) || optional_type_hint == e.type) && - match_args<0, bound>(0, c, state)); + ((optional_type_hint == Type()) || optional_type_hint == e.type) && + match_args<0, bound>(0, c, state)); } template