diff --git a/src/CodeGen_ARM.cpp b/src/CodeGen_ARM.cpp index 30ae1a8e18d2..0e56a2c8605b 100644 --- a/src/CodeGen_ARM.cpp +++ b/src/CodeGen_ARM.cpp @@ -1331,6 +1331,7 @@ void CodeGen_ARM::codegen_vector_reduce(const VectorReduce *op, const Expr &init if (narrow.defined()) { if (init.defined() && target.bits == 32) { // On 32-bit, we have an intrinsic for widening add-accumulate. + // TODO: this could be written as a pattern with widen_right_add (#6951). intrin = "pairwise_widening_add_accumulate"; intrin_args = {accumulator, narrow}; accumulator = Expr(); diff --git a/src/FindIntrinsics.cpp b/src/FindIntrinsics.cpp index 0a33b85822aa..9700d06e319b 100644 --- a/src/FindIntrinsics.cpp +++ b/src/FindIntrinsics.cpp @@ -74,7 +74,7 @@ bool is_safe_for_add(const Expr &e, int max_depth) { } else if (cast->type.bits() == cast->value.type().bits()) { return is_safe_for_add(cast->value, max_depth); } - } else if (Call::as_intrinsic(e, {Call::widening_add, Call::widening_sub})) { + } else if (Call::as_intrinsic(e, {Call::widening_add, Call::widening_sub, Call::widen_right_add, Call::widen_right_sub})) { return true; } return false; @@ -131,20 +131,47 @@ Expr to_rounding_shift(const Call *c) { } Expr round; if (c->is_intrinsic(Call::shift_right)) { - round = simplify((make_one(round_type) << max(cast(b.type().with_bits(round_type.bits()), b), 0)) / 2); + round = (make_one(round_type) << max(cast(b.type().with_bits(round_type.bits()), b), 0)) / 2; } else { - round = simplify((make_one(round_type) >> min(cast(b.type().with_bits(round_type.bits()), b), 0)) / 2); + round = (make_one(round_type) >> min(cast(b.type().with_bits(round_type.bits()), b), 0)) / 2; } + // Input expressions are simplified before running find_intrinsics, but b + // has been lifted here so we need to lower_intrinsics before simplifying + // and re-lifting. Should we move this code into the FindIntrinsics class + // to make it easier to lift round? + round = lower_intrinsics(round); + round = simplify(round); + round = find_intrinsics(round); // We can always handle widening adds. if (const Call *add = Call::as_intrinsic(a, {Call::widening_add})) { - if (can_prove(lower_intrinsics(add->args[0]) == round)) { + if (can_prove(lower_intrinsics(add->args[0] == round))) { return rounding_shift(cast(add->type, add->args[1]), b); - } else if (can_prove(lower_intrinsics(add->args[1]) == round)) { + } else if (can_prove(lower_intrinsics(add->args[1] == round))) { return rounding_shift(cast(add->type, add->args[0]), b); } } + if (const Call *add = Call::as_intrinsic(a, {Call::widen_right_add})) { + if (can_prove(lower_intrinsics(add->args[1] == round))) { + return rounding_shift(cast(add->type, add->args[0]), b); + } + } + // Also need to handle the annoying case of a reinterpret wrapping a widen_right_add + // TODO: this pattern makes me want to change the semantics of this op. + if (const Reinterpret *reinterp = a.as()) { + if (reinterp->type.bits() == reinterp->value.type().bits()) { + if (const Call *add = Call::as_intrinsic(reinterp->value, {Call::widen_right_add})) { + if (can_prove(lower_intrinsics(add->args[1] == round))) { + // We expect the first operand to be a reinterpet. + const Reinterpret *reinterp_a = add->args[0].as(); + internal_assert(reinterp_a) << "Failed: " << add->args[0] << "\n"; + return rounding_shift(reinterp_a->value, b); + } + } + } + } + // If it wasn't a widening or saturating add, we might still // be able to safely accept the rounding. Expr a_less_round = find_and_subtract(a, round); @@ -199,6 +226,53 @@ class FindIntrinsics : public IRMutator { } } + if (op->type.is_int_or_uint() && op->type.bits() > 8) { + // Look for widen_right_add intrinsics. + // Yes we do an duplicate code, but we want to check the op->type.code() first, + // and the opposite as well. + for (halide_type_code_t code : {op->type.code(), halide_type_uint, halide_type_int}) { + Type narrow = op->type.narrow().with_code(code); + // Pulling casts out of VectorReduce nodes breaks too much codegen, skip for now. + Expr narrow_a = (a.node_type() == IRNodeType::VectorReduce) ? Expr() : lossless_cast(narrow, a); + Expr narrow_b = (b.node_type() == IRNodeType::VectorReduce) ? Expr() : lossless_cast(narrow, b); + + // This case should have been handled by the above check for widening_add. + internal_assert(!(narrow_a.defined() && narrow_b.defined())) + << "find_intrinsics failed to find a widening_add: " << a << " + " << b << "\n"; + + if (narrow_a.defined()) { + Expr result; + if (b.type().code() != narrow_a.type().code()) { + // Need to do a safe reinterpret. + Type t = b.type().with_code(code); + result = widen_right_add(reinterpret(t, b), narrow_a); + internal_assert(result.type() != op->type); + result = reinterpret(op->type, result); + } else { + result = widen_right_add(b, narrow_a); + } + internal_assert(result.type() == op->type); + return result; + } else if (narrow_b.defined()) { + Expr result; + if (a.type().code() != narrow_b.type().code()) { + // Need to do a safe reinterpret. + Type t = a.type().with_code(code); + result = widen_right_add(reinterpret(t, a), narrow_b); + internal_assert(result.type() != op->type); + result = reinterpret(op->type, result); + } else { + result = widen_right_add(a, narrow_b); + } + internal_assert(result.type() == op->type); + return mutate(result); + } + } + } + + // TODO: there can be widen_right_add + widen_right_add simplification rules. + // i.e. widen_right_add(a, b) + widen_right_add(c, d) = (a + c) + widening_add(b, d) + if (a.same_as(op->a) && b.same_as(op->b)) { return op; } else { @@ -240,6 +314,32 @@ class FindIntrinsics : public IRMutator { return Add::make(a, negative_b); } + // Run after the lossless_negate check, because we want that to turn into an widen_right_add if relevant. + if (op->type.is_int_or_uint() && op->type.bits() > 8) { + // Look for widen_right_sub intrinsics. + // Yes we do an duplicate code, but we want to check the op->type.code() first, + // and the opposite as well. + for (halide_type_code_t code : {op->type.code(), halide_type_uint, halide_type_int}) { + Type narrow = op->type.narrow().with_code(code); + Expr narrow_b = lossless_cast(narrow, b); + + if (narrow_b.defined()) { + Expr result; + if (a.type().code() != narrow_b.type().code()) { + // Need to do a safe reinterpret. + Type t = a.type().with_code(code); + result = widen_right_sub(reinterpret(t, a), narrow_b); + internal_assert(result.type() != op->type); + result = reinterpret(op->type, result); + } else { + result = widen_right_sub(a, narrow_b); + } + internal_assert(result.type() == op->type); + return mutate(result); + } + } + } + if (a.same_as(op->a) && b.same_as(op->b)) { return op; } else { @@ -292,6 +392,49 @@ class FindIntrinsics : public IRMutator { return mutate(result); } + if (op->type.is_int_or_uint() && op->type.bits() > 8) { + // Look for widen_right_mul intrinsics. + // Yes we do an duplicate code, but we want to check the op->type.code() first, + // and the opposite as well. + for (halide_type_code_t code : {op->type.code(), halide_type_uint, halide_type_int}) { + Type narrow = op->type.narrow().with_code(code); + Expr narrow_a = lossless_cast(narrow, a); + Expr narrow_b = lossless_cast(narrow, b); + + // This case should have been handled by the above check for widening_mul. + internal_assert(!(narrow_a.defined() && narrow_b.defined())) + << "find_intrinsics failed to find a widening_mul: " << a << " + " << b << "\n"; + + if (narrow_a.defined()) { + Expr result; + if (b.type().code() != narrow_a.type().code()) { + // Need to do a safe reinterpret. + Type t = b.type().with_code(code); + result = widen_right_mul(reinterpret(t, b), narrow_a); + internal_assert(result.type() != op->type); + result = reinterpret(op->type, result); + } else { + result = widen_right_mul(b, narrow_a); + } + internal_assert(result.type() == op->type); + return result; + } else if (narrow_b.defined()) { + Expr result; + if (a.type().code() != narrow_b.type().code()) { + // Need to do a safe reinterpret. + Type t = a.type().with_code(code); + result = widen_right_mul(reinterpret(t, a), narrow_b); + internal_assert(result.type() != op->type); + result = reinterpret(op->type, result); + } else { + result = widen_right_mul(a, narrow_b); + } + internal_assert(result.type() == op->type); + return mutate(result); + } + } + } + if (a.same_as(op->a) && b.same_as(op->b)) { return op; } else { @@ -594,6 +737,37 @@ class FindIntrinsics : public IRMutator { const auto is_x_wider_opposite_int = (op->type.is_int() && is_uint(x, 2 * bits)) || (op->type.is_uint() && is_int(x, 2 * bits)); if ( + // Simplify extending patterns. + // (x + widen(y)) + widen(z) = x + widening_add(y, z). + rewrite(widen_right_add(widen_right_add(x, y), z), + x + widening_add(y, z), + // We only care about integers, this should be trivially true. + is_x_same_int_or_uint) || + + // (x - widen(y)) - widen(z) = x - widening_add(y, z). + rewrite(widen_right_sub(widen_right_sub(x, y), z), + x - widening_add(y, z), + // We only care about integers, this should be trivially true. + is_x_same_int_or_uint) || + + // (x + widen(y)) - widen(z) = x + cast(t, widening_sub(y, z)) + // cast (reinterpret) is needed only for uints. + rewrite(widen_right_sub(widen_right_add(x, y), z), + x + widening_sub(y, z), + is_x_same_int) || + rewrite(widen_right_sub(widen_right_add(x, y), z), + x + cast(op->type, widening_sub(y, z)), + is_x_same_uint) || + + // (x - widen(y)) + widen(z) = x + cast(t, widening_sub(z, y)) + // cast (reinterpret) is needed only for uints. + rewrite(widen_right_add(widen_right_sub(x, y), z), + x + widening_sub(z, y), + is_x_same_int) || + rewrite(widen_right_add(widen_right_sub(x, y), z), + x + cast(op->type, widening_sub(z, y)), + is_x_same_uint) || + // Saturating patterns. rewrite(saturating_cast(op->type, widening_add(x, y)), saturating_add(x, y), @@ -679,6 +853,7 @@ class FindIntrinsics : public IRMutator { } } } + // TODO: do we want versions of widen_right_add here? if (op->is_intrinsic(Call::shift_right) || op->is_intrinsic(Call::shift_left)) { // Try to turn this into a widening shift. @@ -885,6 +1060,18 @@ Expr find_intrinsics(const Expr &e) { return expr; } +Expr lower_widen_right_add(const Expr &a, const Expr &b) { + return a + widen(b); +} + +Expr lower_widen_right_mul(const Expr &a, const Expr &b) { + return a * widen(b); +} + +Expr lower_widen_right_sub(const Expr &a, const Expr &b) { + return a - widen(b); +} + Expr lower_widening_add(const Expr &a, const Expr &b) { return widen(a) + widen(b); } @@ -1100,7 +1287,16 @@ Expr lower_rounding_mul_shift_right(const Expr &a, const Expr &b, const Expr &q) } Expr lower_intrinsic(const Call *op) { - if (op->is_intrinsic(Call::widening_add)) { + if (op->is_intrinsic(Call::widen_right_add)) { + internal_assert(op->args.size() == 2); + return lower_widen_right_add(op->args[0], op->args[1]); + } else if (op->is_intrinsic(Call::widen_right_mul)) { + internal_assert(op->args.size() == 2); + return lower_widen_right_mul(op->args[0], op->args[1]); + } else if (op->is_intrinsic(Call::widen_right_sub)) { + internal_assert(op->args.size() == 2); + return lower_widen_right_sub(op->args[0], op->args[1]); + } else if (op->is_intrinsic(Call::widening_add)) { internal_assert(op->args.size() == 2); return lower_widening_add(op->args[0], op->args[1]); } else if (op->is_intrinsic(Call::widening_mul)) { diff --git a/src/FindIntrinsics.h b/src/FindIntrinsics.h index 07e639117252..f8ddaf171bc3 100644 --- a/src/FindIntrinsics.h +++ b/src/FindIntrinsics.h @@ -11,6 +11,9 @@ namespace Halide { namespace Internal { /** Implement intrinsics with non-intrinsic using equivalents. */ +Expr lower_widen_right_add(const Expr &a, const Expr &b); +Expr lower_widen_right_mul(const Expr &a, const Expr &b); +Expr lower_widen_right_sub(const Expr &a, const Expr &b); Expr lower_widening_add(const Expr &a, const Expr &b); Expr lower_widening_mul(const Expr &a, const Expr &b); Expr lower_widening_sub(const Expr &a, const Expr &b); diff --git a/src/HexagonOptimize.cpp b/src/HexagonOptimize.cpp index 1cdc525398df..c4ecda2fe70d 100644 --- a/src/HexagonOptimize.cpp +++ b/src/HexagonOptimize.cpp @@ -137,6 +137,8 @@ Expr as_mul(const Expr &a) { Expr b = make_one(s->type) << cast(UInt(s->type.bits()), (int)*log2_b); return simplify(Mul::make(cast(s->type, s->args[0]), b)); } + } else if (const Call *wm = Call::as_intrinsic(a, {Call::widen_right_mul})) { + return simplify(Mul::make(wm->args[0], cast(wm->type, wm->args[1]))); } return Expr(); } @@ -181,18 +183,8 @@ struct Pattern { // re-interleave the result. ReinterleaveOp0 = InterleaveResult | DeinterleaveOp0, - NarrowOp0 = 1 << 10, // Replace operand 0 with its half-width equivalent. - NarrowOp1 = 1 << 11, // Same as above, but for operand 1. - NarrowOp2 = 1 << 12, - NarrowOps = NarrowOp0 | NarrowOp1 | NarrowOp2, - - NarrowUnsignedOp0 = 1 << 15, // Similar to the above, but narrow to an unsigned half width type. - NarrowUnsignedOp1 = 1 << 16, - NarrowUnsignedOp2 = 1 << 17, - NarrowUnsignedOps = NarrowUnsignedOp0 | NarrowUnsignedOp1 | NarrowUnsignedOp2, - - v65orLater = 1 << 21, // Pattern should be matched only for v65 target or later - v66orLater = 1 << 22, // Pattern should be matched only for v66 target or later + v65orLater = 1 << 10, // Pattern should be matched only for v65 target or later + v66orLater = 1 << 11, // Pattern should be matched only for v66 target or later }; string intrin; // Name of the intrinsic @@ -242,14 +234,8 @@ bool process_match_flags(vector &matches, int flags) { // The Pattern::Narrow*Op* flags are ordered such that the operand // corresponds to the bit (with operand 0 corresponding to the least // significant bit), so we can check for them all in a loop. - for (size_t i = 0; i < matches.size(); i++) { - Type t = matches[i].type(); - if (flags & (Pattern::NarrowOp0 << i)) { - matches[i] = lossless_cast(t.narrow(), matches[i]); - } else if (flags & (Pattern::NarrowUnsignedOp0 << i)) { - matches[i] = lossless_cast(t.narrow().with_code(Type::UInt), matches[i]); - } - if (!matches[i].defined()) { + for (const auto &match : matches) { + if (!match.defined()) { return false; } } @@ -426,6 +412,11 @@ int find_mpy_ops(const Expr &op, Type a_ty, Type b_ty, int max_mpy_count, mpy_count += find_mpy_ops(cast(op.type(), add->args[0]), a_ty, b_ty, max_mpy_count, mpys, rest); mpy_count += find_mpy_ops(cast(op.type(), add->args[1]), a_ty, b_ty, max_mpy_count, mpys, rest); return mpy_count; + } else if (const Call *wadd = Call::as_intrinsic(op, {Call::widen_right_add})) { + int mpy_count = 0; + mpy_count += find_mpy_ops(wadd->args[0], a_ty, b_ty, max_mpy_count, mpys, rest); + mpy_count += find_mpy_ops(cast(op.type(), wadd->args[1]), a_ty, b_ty, max_mpy_count, mpys, rest); + return mpy_count; } // Attempt to pretend this op is multiplied by 1. @@ -445,43 +436,12 @@ int find_mpy_ops(const Expr &op, Type a_ty, Type b_ty, int max_mpy_count, // Perform peephole optimizations on the IR, adding appropriate // interleave and deinterleave calls. class OptimizePatterns : public IRMutator { -private: using IRMutator::visit; Scope bounds; const Target ⌖ - Expr visit(const Mul *op) override { - static const vector scalar_muls = { - // Non-widening scalar multiplication. - {"halide.hexagon.mul.vh.b", wild_i16x * wild_i16, Pattern::NarrowOp1}, - {"halide.hexagon.mul.vw.h", wild_i32x * wild_i32, Pattern::NarrowOp1}, - // TODO: There's also mul.vw.b. We currently generate mul.vw.h - // instead. I'm not sure mul.vw.b is faster, it might even be - // slower due to the extra step in broadcasting the scalar up to - // 32 bits. - }; - - static const vector muls = { - // One operand widening multiplication. - {"halide.hexagon.mul.vw.vh", wild_i32x * wild_i32x, Pattern::ReinterleaveOp0 | Pattern::NarrowOp1}, - {"halide.hexagon.mul.vw.vuh", wild_i32x * wild_i32x, Pattern::ReinterleaveOp0 | Pattern::NarrowUnsignedOp1}, - {"halide.hexagon.mul.vuw.vuh", wild_u32x * wild_u32x, Pattern::ReinterleaveOp0 | Pattern::NarrowUnsignedOp1}, - }; - - if (op->type.is_vector()) { - Expr new_expr = apply_commutative_patterns(op, scalar_muls, target, this); - if (!new_expr.same_as(op)) { - return new_expr; - } - - new_expr = apply_commutative_patterns(op, muls, target, this); - if (!new_expr.same_as(op)) { - return new_expr; - } - } - return IRMutator::visit(op); - } + // Interesting muls are handled as widen_right_mul(). // We'll try to sort the mpys based my mpys.first. // But, for this all the mpy.first exprs should either be @@ -743,8 +703,8 @@ class OptimizePatterns : public IRMutator { {"halide.hexagon.add_shl.vh.vh.uh", wild_u16x + (wild_u16x << wild_u16), Pattern::v65orLater}, // Non-widening multiply-accumulates with a scalar. - {"halide.hexagon.add_mul.vh.vh.b", wild_i16x + wild_i16x * wild_i16, Pattern::NarrowOp2}, - {"halide.hexagon.add_mul.vw.vw.h", wild_i32x + wild_i32x * wild_i32, Pattern::NarrowOp2}, + {"halide.hexagon.add_mul.vh.vh.b", wild_i16x + widen_right_mul(wild_i16x, wild_i8)}, + {"halide.hexagon.add_mul.vw.vw.h", wild_i32x + widen_right_mul(wild_i32x, wild_i16)}, // TODO: There's also a add_mul.vw.vw.b // This pattern is very general, so it must come last. @@ -884,6 +844,14 @@ class OptimizePatterns : public IRMutator { return mpyadds; } } + // TODO: There can be better instruction selection for these. + if (op->is_intrinsic(Call::widen_right_add)) { + Expr lowered = Add::make(op->args[0], cast(op->type, op->args[1])); + return mutate(lowered); + } else if (op->is_intrinsic(Call::widen_right_sub)) { + Expr lowered = Sub::make(op->args[0], cast(op->type, op->args[1])); + return mutate(lowered); + } // These intrinsics should get the default lowering, and we need to recursively mutate the // result. We don't want to let these fall through to CodeGen_Hexagon and CodeGen_LLVM, @@ -900,6 +868,18 @@ class OptimizePatterns : public IRMutator { } static const vector calls = { + // Non-widening scalar multiplication. + {"halide.hexagon.mul.vh.b", widen_right_mul(wild_i16x, wild_i8)}, + {"halide.hexagon.mul.vw.h", widen_right_mul(wild_i32x, wild_i16)}, + // TODO: There's also mul.vw.b. We currently generate mul.vw.h + // instead. I'm not sure mul.vw.b is faster, it might even be + // slower due to the extra step in broadcasting the scalar up to + // 32 bits. + + // One operand widening multiplication. + {"halide.hexagon.mul.vw.vh", widen_right_mul(wild_i32x, wild_i16x), Pattern::ReinterleaveOp0}, + {"halide.hexagon.mul.vw.vuh", widen_right_mul(wild_u32x, wild_u16x), Pattern::ReinterleaveOp0}, + // Saturating narrowing casts with rounding {"halide.hexagon.trunc_satub_rnd.vh", u8_sat(rounding_shift_right(wild_i16x, 8)), Pattern::DeinterleaveOp0}, {"halide.hexagon.trunc_satb_rnd.vh", i8_sat(rounding_shift_right(wild_i16x, 8)), Pattern::DeinterleaveOp0}, diff --git a/src/IR.cpp b/src/IR.cpp index 105472ab68d6..e2f54318214a 100644 --- a/src/IR.cpp +++ b/src/IR.cpp @@ -669,6 +669,9 @@ const char *const intrinsic_op_names[] = { "undef", "unreachable", "unsafe_promise_clamped", + "widen_right_add", + "widen_right_mul", + "widen_right_sub", "widening_add", "widening_mul", "widening_shift_left", diff --git a/src/IR.h b/src/IR.h index ff92f38d4107..0da5ffa1aaa6 100644 --- a/src/IR.h +++ b/src/IR.h @@ -583,6 +583,15 @@ struct Call : public ExprNode { undef, unreachable, unsafe_promise_clamped, + + // One-sided variants of widening_add, widening_mul, and widening_sub. + // arg[0] + widen(arg[1]) + widen_right_add, + // arg[0] * widen(arg[1]) + widen_right_mul, + // arg[0] - widen(arg[1]) + widen_right_sub, + widening_add, widening_mul, widening_shift_left, diff --git a/src/IRMatch.h b/src/IRMatch.h index 0de1c21cee44..06d35ba8b4b0 100644 --- a/src/IRMatch.h +++ b/src/IRMatch.h @@ -1429,6 +1429,12 @@ struct Intrin { Expr arg1 = std::get(args).make(state, type_hint); if (intrin == Call::absd) { return absd(arg0, arg1); + } else if (intrin == Call::widen_right_add) { + return widen_right_add(arg0, arg1); + } else if (intrin == Call::widen_right_mul) { + return widen_right_mul(arg0, arg1); + } else if (intrin == Call::widen_right_sub) { + return widen_right_sub(arg0, arg1); } else if (intrin == Call::widening_add) { return widening_add(arg0, arg1); } else if (intrin == Call::widening_sub) { @@ -1529,6 +1535,19 @@ HALIDE_ALWAYS_INLINE auto intrin(Call::IntrinsicOp intrinsic_op, Args... args) n return {intrinsic_op, pattern_arg(args)...}; } +template +auto widen_right_add(A &&a, B &&b) noexcept -> Intrin { + return {Call::widen_right_add, pattern_arg(a), pattern_arg(b)}; +} +template +auto widen_right_mul(A &&a, B &&b) noexcept -> Intrin { + return {Call::widen_right_mul, pattern_arg(a), pattern_arg(b)}; +} +template +auto widen_right_sub(A &&a, B &&b) noexcept -> Intrin { + return {Call::widen_right_sub, pattern_arg(a), pattern_arg(b)}; +} + template auto widening_add(A &&a, B &&b) noexcept -> Intrin { return {Call::widening_add, pattern_arg(a), pattern_arg(b)}; @@ -1859,7 +1878,7 @@ struct VectorReduceOp { A a; B lanes; - constexpr static uint32_t binds = bindings::mask; + constexpr static uint32_t binds = bindings::mask | bindings::mask; constexpr static IRNodeType min_node_type = IRNodeType::VectorReduce; constexpr static IRNodeType max_node_type = IRNodeType::VectorReduce; @@ -2324,7 +2343,7 @@ template struct IsInt { struct pattern_tag {}; A a; - int bits; + int bits, lanes; constexpr static uint32_t binds = bindings::mask; @@ -2339,7 +2358,7 @@ struct IsInt { void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const { // a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method. Type t = a.make(state, {}).type(); - val.u.u64 = t.is_int() && (bits == 0 || t.bits() == bits); + val.u.u64 = t.is_int() && (bits == 0 || t.bits() == bits) && (lanes == 0 || t.lanes() == lanes); ty.code = halide_type_uint; ty.bits = 1; ty.lanes = t.lanes(); @@ -2347,9 +2366,9 @@ struct IsInt { }; template -HALIDE_ALWAYS_INLINE auto is_int(A &&a, int bits = 0) noexcept -> IsInt { +HALIDE_ALWAYS_INLINE auto is_int(A &&a, int bits = 0, int lanes = 0) noexcept -> IsInt { assert_is_lvalue_if_expr(); - return {pattern_arg(a), bits}; + return {pattern_arg(a), bits, lanes}; } template @@ -2358,6 +2377,9 @@ std::ostream &operator<<(std::ostream &s, const IsInt &op) { if (op.bits > 0) { s << ", " << op.bits; } + if (op.lanes > 0) { + s << ", " << op.lanes; + } s << ")"; return s; } @@ -2366,7 +2388,7 @@ template struct IsUInt { struct pattern_tag {}; A a; - int bits; + int bits, lanes; constexpr static uint32_t binds = bindings::mask; @@ -2381,7 +2403,7 @@ struct IsUInt { void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const { // a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method. Type t = a.make(state, {}).type(); - val.u.u64 = t.is_uint() && (bits == 0 || t.bits() == bits); + val.u.u64 = t.is_uint() && (bits == 0 || t.bits() == bits) && (lanes == 0 || t.lanes() == lanes); ty.code = halide_type_uint; ty.bits = 1; ty.lanes = t.lanes(); @@ -2389,9 +2411,9 @@ struct IsUInt { }; template -HALIDE_ALWAYS_INLINE auto is_uint(A &&a, int bits = 0) noexcept -> IsUInt { +HALIDE_ALWAYS_INLINE auto is_uint(A &&a, int bits = 0, int lanes = 0) noexcept -> IsUInt { assert_is_lvalue_if_expr(); - return {pattern_arg(a), bits}; + return {pattern_arg(a), bits, lanes}; } template @@ -2400,6 +2422,9 @@ std::ostream &operator<<(std::ostream &s, const IsUInt &op) { if (op.bits > 0) { s << ", " << op.bits; } + if (op.lanes > 0) { + s << ", " << op.lanes; + } s << ")"; return s; } @@ -2525,6 +2550,43 @@ std::ostream &operator<<(std::ostream &s, const IsMinValue &op) { return s; } +template +struct HasEvenLanes { + struct pattern_tag {}; + A a; + + constexpr static uint32_t binds = bindings::mask; + + // This rule is a boolean-valued predicate. Bools have type UIntImm. + constexpr static IRNodeType min_node_type = IRNodeType::UIntImm; + constexpr static IRNodeType max_node_type = IRNodeType::UIntImm; + constexpr static bool canonical = true; + + constexpr static bool foldable = true; + + HALIDE_ALWAYS_INLINE + void make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const { + // a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method. + Type t = a.make(state, {}).type(); + val.u.u64 = (t.lanes() % 2 == 0); + ty.code = halide_type_uint; + ty.bits = 1; + ty.lanes = t.lanes(); + } +}; + +template +HALIDE_ALWAYS_INLINE auto has_even_lanes(A &&a) noexcept -> HasEvenLanes { + assert_is_lvalue_if_expr(); + return {pattern_arg(a)}; +} + +template +std::ostream &operator<<(std::ostream &s, const HasEvenLanes &op) { + s << "has_even_lanes(" << op.a << ")"; + return s; +} + // Verify properties of each rewrite rule. Currently just fuzz tests them. template= 16) { + if (t.is_int_or_uint() && t.bits() >= 16) { if (const Add *add = e.as()) { // If we can losslessly narrow the args even more // aggressively, we're good. @@ -1104,6 +1104,46 @@ Expr memoize_tag_helper(Expr result, const std::vector &cache_key_values) args, Internal::Call::PureIntrinsic); } +Expr widen_right_add(Expr a, Expr b) { + user_assert(a.defined() && b.defined()) << "widen_right_add of undefined Expr\n" + << a << ", " << b << "\n"; + user_assert(a.type().is_int_or_uint() && b.type().is_int_or_uint()) + << "widen_right_add only defined for integer types, received:\n " << a << "\n " << b << "\n"; + user_assert(b.type().bits() <= 32) << "widen_right_add of large Expr\n" + << a << ", " << b << "\n"; + match_lanes(a, b); + Type wide_type = b.type().widen(); + user_assert(wide_type == a.type()) << "widen_right_add type mismatch\n " << a << "\n " << b << "\n"; + return Call::make(wide_type, Call::widen_right_add, {std::move(a), std::move(b)}, Call::PureIntrinsic); +} + +Expr widen_right_mul(Expr a, Expr b) { + user_assert(a.defined() && b.defined()) << "widen_right_mul of undefined Expr\n" + << a << ", " << b << "\n"; + user_assert(a.type().is_int_or_uint() && b.type().is_int_or_uint()) + << "widen_right_mul only defined for integer types, received:\n " << a << "\n " << b << "\n"; + user_assert(b.type().bits() <= 32) << "widen_right_mul of large Expr\n" + << a << ", " << b << "\n"; + match_lanes(a, b); + Type wide_type = b.type().widen(); + user_assert(wide_type == a.type()) << "widen_right_mul type mismatch\n " << a << "\n " << b << "\n"; + return Call::make(wide_type, Call::widen_right_mul, {std::move(a), std::move(b)}, Call::PureIntrinsic); +} + +Expr widen_right_sub(Expr a, Expr b) { + user_assert(a.defined() && b.defined()) << "widen_right_sub of undefined Expr\n" + << a << ", " << b << "\n"; + user_assert(a.type().is_int_or_uint() && b.type().is_int_or_uint()) + << "widen_right_sub only defined for integer types, received:\n " << a << "\n " << b << "\n"; + user_assert(b.type().bits() <= 32) << "widen_right_sub of large Expr\n" + << a << ", " << b << "\n"; + match_lanes(a, b); + Type wide_type = b.type().widen(); + user_assert(wide_type == a.type()) << "widen_right_sub type mismatch\n" + << a << ", " << b << "\n"; + return Call::make(wide_type, Call::widen_right_sub, {std::move(a), std::move(b)}, Call::PureIntrinsic); +} + Expr widening_add(Expr a, Expr b) { user_assert(a.defined() && b.defined()) << "widening_add of undefined Expr\n"; match_types(a, b); diff --git a/src/IROperator.h b/src/IROperator.h index 048998448c75..d61362b5d13d 100644 --- a/src/IROperator.h +++ b/src/IROperator.h @@ -342,6 +342,13 @@ Expr requirement_failed_error(Expr condition, const std::vector &args); Expr memoize_tag_helper(Expr result, const std::vector &cache_key_values); +/** Compute a + widen(b). */ +Expr widen_right_add(Expr a, Expr b); +/** Compute a * widen(b). */ +Expr widen_right_mul(Expr a, Expr b); +/** Compute a - widen(b). */ +Expr widen_right_sub(Expr a, Expr b); + /** Compute widen(a) + widen(b). */ Expr widening_add(Expr a, Expr b); /** Compute widen(a) * widen(b). a and b may have different signedness. */ diff --git a/src/runtime/hvx_128.ll b/src/runtime/hvx_128.ll index 848f1ab46960..d97982deb620 100644 --- a/src/runtime/hvx_128.ll +++ b/src/runtime/hvx_128.ll @@ -145,18 +145,6 @@ define private <64 x i32> @vaslw.acc.dv.128B(<64 x i32> %a, <64 x i32> %l, i32 % ret <64 x i32> %s } -define weak_odr <64 x i32> @halide.hexagon.mul.vuw.vuh(<64 x i32> %a, <64 x i16> %b) nounwind uwtable readnone alwaysinline { - %a_lo = call <32 x i32> @llvm.hexagon.V6.lo.128B(<64 x i32> %a) - %a_hi = call <32 x i32> @llvm.hexagon.V6.hi.128B(<64 x i32> %a) - %a_e = call <32 x i32> @llvm.hexagon.V6.vshufeh.128B(<32 x i32> %a_hi, <32 x i32> %a_lo) - %a_o = call <32 x i32> @llvm.hexagon.V6.vshufoh.128B(<32 x i32> %a_hi, <32 x i32> %a_lo) - %b_32 = bitcast <64 x i16> %b to <32 x i32> - %ab_e = call <64 x i32> @llvm.hexagon.V6.vmpyuhv.128B(<32 x i32> %a_e, <32 x i32> %b_32) - %ab_o = call <64 x i32> @llvm.hexagon.V6.vmpyuhv.128B(<32 x i32> %a_o, <32 x i32> %b_32) - %ab = call <64 x i32> @vaslw.acc.dv.128B(<64 x i32> %ab_e, <64 x i32> %ab_o, i32 16) - ret <64 x i32> %ab -} - define weak_odr <64 x i32> @halide.hexagon.mul.vuw.vuw(<64 x i32> %a, <64 x i32> %b) nounwind uwtable readnone alwaysinline { %a_lo = call <32 x i32> @llvm.hexagon.V6.lo.128B(<64 x i32> %a) %a_hi = call <32 x i32> @llvm.hexagon.V6.hi.128B(<64 x i32> %a) diff --git a/test/correctness/intrinsics.cpp b/test/correctness/intrinsics.cpp index 068f360d214f..e1a892812df2 100644 --- a/test/correctness/intrinsics.cpp +++ b/test/correctness/intrinsics.cpp @@ -243,7 +243,7 @@ int main(int argc, char **argv) { check(narrow((u32(u16x) + 15) >> 4), rounding_halving_add(u16x, u16(14)) >> u16(3)); // But not if the constant can't fit in the narrower type - check(narrow((u16(u8x) + 500) >> 4), narrow((u16(u8x) + 500) >> 4)); + check(narrow((u16(u8x) + 500) >> 4), narrow((widen_right_add(cast(500), u8x)) >> 4)); check((u64(u32x) + 8) / 16, u64(rounding_shift_right(u32x, 4))); check(u16(min((u64(u32x) + 8) / 16, 65535)), u16_sat(rounding_shift_right(u32x, 4))); diff --git a/test/correctness/simd_op_check_hvx.cpp b/test/correctness/simd_op_check_hvx.cpp index fc35840bcaad..7685da369d18 100644 --- a/test/correctness/simd_op_check_hvx.cpp +++ b/test/correctness/simd_op_check_hvx.cpp @@ -458,7 +458,7 @@ class SimdOpCheckHVX : public SimdOpCheckTest { check("vmpyi(v*.h,v*.h)", hvx_width / 2, i16_1 * i16_2); check("vmpyio(v*.w,v*.h)", hvx_width / 2, i32_1 * i32(i16_1)); check("vmpyie(v*.w,v*.uh)", hvx_width / 2, i32_1 * i32(u16_1)); - check("vmpy(v*.uh,v*.uh)", hvx_width / 2, u32_1 * u32(u16_1)); + check("vmpyie(v*.w,v*.uh)", hvx_width / 2, u32_1 * u32(u16_1)); check("vmpyieo(v*.h,v*.h)", hvx_width / 4, i32_1 * i32_2); // The inconsistency in the expected instructions here is // correct. For bytes, the unsigned value is first, for half