diff --git a/gcc/doc/md.texi b/gcc/doc/md.texi index d166a0debedf4d8edf55c842bcf4ff4690b3e9ce..9fad3322b3f1eb2a836833bb390df78f0cd9734b 100644 --- a/gcc/doc/md.texi +++ b/gcc/doc/md.texi @@ -5438,13 +5438,55 @@ Like @samp{fold_left_plus_@var{m}}, but takes an additional mask operand @cindex @code{sdot_prod@var{m}} instruction pattern @item @samp{sdot_prod@var{m}} + +Compute the sum of the products of two signed elements. +Operand 1 and operand 2 are of the same mode. Their +product, which is of a wider mode, is computed and added to operand 3. +Operand 3 is of a mode equal or wider than the mode of the product. The +result is placed in operand 0, which is of the same mode as operand 3. + +Semantically the expressions perform the multiplication in the following signs + +@smallexample +sdot == + res = sign-ext (a) * sign-ext (b) + c +@dots{} +@end smallexample + @cindex @code{udot_prod@var{m}} instruction pattern -@itemx @samp{udot_prod@var{m}} -Compute the sum of the products of two signed/unsigned elements. -Operand 1 and operand 2 are of the same mode. Their product, which is of a -wider mode, is computed and added to operand 3. Operand 3 is of a mode equal or -wider than the mode of the product. The result is placed in operand 0, which -is of the same mode as operand 3. +@item @samp{udot_prod@var{m}} + +Compute the sum of the products of two unsigned elements. +Operand 1 and operand 2 are of the same mode. Their +product, which is of a wider mode, is computed and added to operand 3. +Operand 3 is of a mode equal or wider than the mode of the product. The +result is placed in operand 0, which is of the same mode as operand 3. + +Semantically the expressions perform the multiplication in the following signs + +@smallexample +udot == + res = zero-ext (a) * zero-ext (b) + c +@dots{} +@end smallexample + + + +@cindex @code{usdot_prod@var{m}} instruction pattern +@item @samp{usdot_prod@var{m}} +Compute the sum of the products of elements of different signs. +Operand 1 must be unsigned and operand 2 signed. Their +product, which is of a wider mode, is computed and added to operand 3. +Operand 3 is of a mode equal or wider than the mode of the product. The +result is placed in operand 0, which is of the same mode as operand 3. + +Semantically the expressions perform the multiplication in the following signs + +@smallexample +usdot == + res = ((unsigned-conv) sign-ext (a)) * zero-ext (b) + c +@dots{} +@end smallexample @cindex @code{ssad@var{m}} instruction pattern @item @samp{ssad@var{m}} diff --git a/gcc/optabs-tree.h b/gcc/optabs-tree.h index c3aaa1a416991e856d3e24da45968a92ebada82c..fbd2b06b8dbfd560dfb66b314830e6b564b37abb 100644 --- a/gcc/optabs-tree.h +++ b/gcc/optabs-tree.h @@ -29,7 +29,8 @@ enum optab_subtype { optab_default, optab_scalar, - optab_vector + optab_vector, + optab_vector_mixed_sign }; /* Return the optab used for computing the given operation on the type given by diff --git a/gcc/optabs-tree.c b/gcc/optabs-tree.c index 95ffe397c23e80c105afea52e9d47216bf52f55a..eeb5aeed3202cc6971b6447994bc5311e9c010bb 100644 --- a/gcc/optabs-tree.c +++ b/gcc/optabs-tree.c @@ -127,7 +127,12 @@ optab_for_tree_code (enum tree_code code, const_tree type, return TYPE_UNSIGNED (type) ? usum_widen_optab : ssum_widen_optab; case DOT_PROD_EXPR: - return TYPE_UNSIGNED (type) ? udot_prod_optab : sdot_prod_optab; + { + if (subtype == optab_vector_mixed_sign) + return usdot_prod_optab; + + return (TYPE_UNSIGNED (type) ? udot_prod_optab : sdot_prod_optab); + } case SAD_EXPR: return TYPE_UNSIGNED (type) ? usad_optab : ssad_optab; diff --git a/gcc/optabs.c b/gcc/optabs.c index f4614a394587787293dc8b680a38901f7906f61c..d9b64441d0e0726afee89dc9c937350451e7670d 100644 --- a/gcc/optabs.c +++ b/gcc/optabs.c @@ -262,6 +262,11 @@ expand_widen_pattern_expr (sepops ops, rtx op0, rtx op1, rtx wide_op, bool sbool = false; oprnd0 = ops->op0; + if (nops >= 2) + oprnd1 = ops->op1; + if (nops >= 3) + oprnd2 = ops->op2; + tmode0 = TYPE_MODE (TREE_TYPE (oprnd0)); if (ops->code == VEC_UNPACK_FIX_TRUNC_HI_EXPR || ops->code == VEC_UNPACK_FIX_TRUNC_LO_EXPR) @@ -285,6 +290,27 @@ expand_widen_pattern_expr (sepops ops, rtx op0, rtx op1, rtx wide_op, ? vec_unpacks_sbool_hi_optab : vec_unpacks_sbool_lo_optab); sbool = true; } + else if (ops->code == DOT_PROD_EXPR) + { + enum optab_subtype subtype = optab_default; + signop sign1 = TYPE_SIGN (TREE_TYPE (oprnd0)); + signop sign2 = TYPE_SIGN (TREE_TYPE (oprnd1)); + if (sign1 == sign2) + ; + else if (sign1 == SIGNED && sign2 == UNSIGNED) + { + subtype = optab_vector_mixed_sign; + /* Same as optab_vector_mixed_sign but flip the operands. */ + std::swap (op0, op1); + } + else if (sign1 == UNSIGNED && sign2 == SIGNED) + subtype = optab_vector_mixed_sign; + else + gcc_unreachable (); + + widen_pattern_optab + = optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), subtype); + } else widen_pattern_optab = optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default); @@ -298,10 +324,7 @@ expand_widen_pattern_expr (sepops ops, rtx op0, rtx op1, rtx wide_op, gcc_assert (icode != CODE_FOR_nothing); if (nops >= 2) - { - oprnd1 = ops->op1; - tmode1 = TYPE_MODE (TREE_TYPE (oprnd1)); - } + tmode1 = TYPE_MODE (TREE_TYPE (oprnd1)); else if (sbool) { nops = 2; @@ -316,7 +339,6 @@ expand_widen_pattern_expr (sepops ops, rtx op0, rtx op1, rtx wide_op, { gcc_assert (tmode1 == tmode0); gcc_assert (op1); - oprnd2 = ops->op2; wmode = TYPE_MODE (TREE_TYPE (oprnd2)); } diff --git a/gcc/optabs.def b/gcc/optabs.def index b192a9d070b8aa72e5676b2eaa020b5bdd7ffcc8..f470c2168378cec840edf7fbdb7c18615baae928 100644 --- a/gcc/optabs.def +++ b/gcc/optabs.def @@ -352,6 +352,7 @@ OPTAB_D (uavg_ceil_optab, "uavg$a3_ceil") OPTAB_D (sdot_prod_optab, "sdot_prod$I$a") OPTAB_D (ssum_widen_optab, "widen_ssum$I$a3") OPTAB_D (udot_prod_optab, "udot_prod$I$a") +OPTAB_D (usdot_prod_optab, "usdot_prod$I$a") OPTAB_D (usum_widen_optab, "widen_usum$I$a3") OPTAB_D (usad_optab, "usad$I$a") OPTAB_D (ssad_optab, "ssad$I$a") diff --git a/gcc/tree-cfg.c b/gcc/tree-cfg.c index 7e3aae5f9c28a49feedc7cc66e8ac0d476b9f28a..0128891852fcd74fe31cd338614e90a26256b4bd 100644 --- a/gcc/tree-cfg.c +++ b/gcc/tree-cfg.c @@ -4421,7 +4421,8 @@ verify_gimple_assign_ternary (gassign *stmt) && !SCALAR_FLOAT_TYPE_P (rhs1_type)) || (!INTEGRAL_TYPE_P (lhs_type) && !SCALAR_FLOAT_TYPE_P (lhs_type)))) - || !types_compatible_p (rhs1_type, rhs2_type) + /* rhs1_type and rhs2_type may differ in sign. */ + || !tree_nop_conversion_p (rhs1_type, rhs2_type) || !useless_type_conversion_p (lhs_type, rhs3_type) || maybe_lt (GET_MODE_SIZE (element_mode (rhs3_type)), 2 * GET_MODE_SIZE (element_mode (rhs1_type)))) diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index 93fa2928e001c154bd4a9a73ac1dbbbf73c456df..756d2867b678d0d8394202c6adb03d9cd26029e7 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -6662,6 +6662,12 @@ vectorizable_reduction (loop_vec_info loop_vinfo, bool lane_reduc_code_p = (code == DOT_PROD_EXPR || code == WIDEN_SUM_EXPR || code == SAD_EXPR); int op_type = TREE_CODE_LENGTH (code); + enum optab_subtype optab_query_kind = optab_vector; + if (code == DOT_PROD_EXPR + && TYPE_SIGN (TREE_TYPE (gimple_assign_rhs1 (stmt))) + != TYPE_SIGN (TREE_TYPE (gimple_assign_rhs2 (stmt)))) + optab_query_kind = optab_vector_mixed_sign; + scalar_dest = gimple_assign_lhs (stmt); scalar_type = TREE_TYPE (scalar_dest); @@ -7189,7 +7195,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo, bool ok = true; /* 4.1. check support for the operation in the loop */ - optab optab = optab_for_tree_code (code, vectype_in, optab_vector); + optab optab = optab_for_tree_code (code, vectype_in, optab_query_kind); if (!optab) { if (dump_enabled_p ()) diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c index 441d6cd28c4eaded7abd756164890dbcffd2f3b8..82123b96313e6783ea214b9259805d65c07d8858 100644 --- a/gcc/tree-vect-patterns.c +++ b/gcc/tree-vect-patterns.c @@ -201,7 +201,8 @@ vect_get_external_def_edge (vec_info *vinfo, tree var) static bool vect_supportable_direct_optab_p (vec_info *vinfo, tree otype, tree_code code, tree itype, tree *vecotype_out, - tree *vecitype_out = NULL) + tree *vecitype_out = NULL, + enum optab_subtype subtype = optab_default) { tree vecitype = get_vectype_for_scalar_type (vinfo, itype); if (!vecitype) @@ -211,7 +212,7 @@ vect_supportable_direct_optab_p (vec_info *vinfo, tree otype, tree_code code, if (!vecotype) return false; - optab optab = optab_for_tree_code (code, vecitype, optab_default); + optab optab = optab_for_tree_code (code, vecitype, subtype); if (!optab) return false; @@ -487,10 +488,14 @@ vect_joust_widened_integer (tree type, bool shift_p, tree op, } /* Return true if the common supertype of NEW_TYPE and *COMMON_TYPE - is narrower than type, storing the supertype in *COMMON_TYPE if so. */ + is narrower than type, storing the supertype in *COMMON_TYPE if so. + If UNPROM_TYPE then accept that *COMMON_TYPE and NEW_TYPE may be of + different signs but equal precision and that the resulting + multiplication of them be compatible with UNPROM_TYPE. */ static bool -vect_joust_widened_type (tree type, tree new_type, tree *common_type) +vect_joust_widened_type (tree type, tree new_type, tree *common_type, + tree unprom_type = NULL) { if (types_compatible_p (*common_type, new_type)) return true; @@ -514,7 +519,18 @@ vect_joust_widened_type (tree type, tree new_type, tree *common_type) unsigned int precision = MAX (TYPE_PRECISION (*common_type), TYPE_PRECISION (new_type)); precision *= 2; - if (precision * 2 > TYPE_PRECISION (type)) + + /* Check if the mismatch is only in the sign and if we have + UNPROM_TYPE then allow it if there is enough precision to + not lose any information during the conversion. */ + if (unprom_type + && TYPE_SIGN (unprom_type) == SIGNED + && tree_nop_conversion_p (*common_type, new_type)) + return true; + + /* The resulting application is unsigned, check if we have enough + precision to perform the operation. */ + if (precision * 2 > TYPE_PRECISION (unprom_type ? unprom_type : type)) return false; *common_type = build_nonstandard_integer_type (precision, false); @@ -532,6 +548,10 @@ vect_joust_widened_type (tree type, tree new_type, tree *common_type) to a type that (a) is narrower than the result of STMT_INFO and (b) can hold all leaf operand values. + If UNPROM_TYPE then allow that the signs of the operands + may differ in signs but not in precision and that the resulting type + of the operation on the operands is compatible with UNPROM_TYPE. + Return 0 if STMT_INFO isn't such a tree, or if no such COMMON_TYPE exists. */ @@ -539,7 +559,8 @@ static unsigned int vect_widened_op_tree (vec_info *vinfo, stmt_vec_info stmt_info, tree_code code, tree_code widened_code, bool shift_p, unsigned int max_nops, - vect_unpromoted_value *unprom, tree *common_type) + vect_unpromoted_value *unprom, tree *common_type, + tree unprom_type = NULL) { /* Check for an integer operation with the right code. */ gassign *assign = dyn_cast (stmt_info->stmt); @@ -600,7 +621,8 @@ vect_widened_op_tree (vec_info *vinfo, stmt_vec_info stmt_info, tree_code code, = vinfo->lookup_def (this_unprom->op); nops = vect_widened_op_tree (vinfo, def_stmt_info, code, widened_code, shift_p, max_nops, - this_unprom, common_type); + this_unprom, common_type, + unprom_type); if (nops == 0) return 0; @@ -617,7 +639,7 @@ vect_widened_op_tree (vec_info *vinfo, stmt_vec_info stmt_info, tree_code code, if (i == 0) *common_type = this_unprom->type; else if (!vect_joust_widened_type (type, this_unprom->type, - common_type)) + common_type, unprom_type)) return 0; } } @@ -799,12 +821,15 @@ vect_convert_input (vec_info *vinfo, stmt_vec_info stmt_info, tree type, } /* Invoke vect_convert_input for N elements of UNPROM and store the - result in the corresponding elements of RESULT. */ + result in the corresponding elements of RESULT. + + If ALLOW_SHORT_SIGN_MISMATCH then don't convert the types if they only + differ by sign. */ static void vect_convert_inputs (vec_info *vinfo, stmt_vec_info stmt_info, unsigned int n, tree *result, tree type, vect_unpromoted_value *unprom, - tree vectype) + tree vectype, bool allow_short_sign_mismatch = false) { for (unsigned int i = 0; i < n; ++i) { @@ -812,8 +837,12 @@ vect_convert_inputs (vec_info *vinfo, stmt_vec_info stmt_info, unsigned int n, for (j = 0; j < i; ++j) if (unprom[j].op == unprom[i].op) break; + if (j < i) result[i] = result[j]; + else if (allow_short_sign_mismatch + && tree_nop_conversion_p (type, unprom[i].type)) + result[i] = unprom[i].op; else result[i] = vect_convert_input (vinfo, stmt_info, type, &unprom[i], vectype); @@ -888,21 +917,24 @@ vect_reassociating_reduction_p (vec_info *vinfo, Try to find the following pattern: - type x_t, y_t; + type1a x_t + type1b y_t; TYPE1 prod; TYPE2 sum = init; loop: sum_0 = phi S1 x_t = ... S2 y_t = ... - S3 x_T = (TYPE1) x_t; - S4 y_T = (TYPE1) y_t; + S3 x_T = (TYPE3) x_t; + S4 y_T = (TYPE4) y_t; S5 prod = x_T * y_T; [S6 prod = (TYPE2) prod; #optional] S7 sum_1 = prod + sum_0; - where 'TYPE1' is exactly double the size of type 'type', and 'TYPE2' is the - same size of 'TYPE1' or bigger. This is a special case of a reduction + where 'TYPE1' is exactly double the size of type 'type1a' and 'type1b', + the sign of 'TYPE1' must be one of 'type1a' or 'type1b' but the sign of + 'type1a' and 'type1b' can differ. 'TYPE2' is the same size of 'TYPE1' or + bigger and must be the same sign. This is a special case of a reduction computation. Input: @@ -939,15 +971,16 @@ vect_recog_dot_prod_pattern (vec_info *vinfo, /* Look for the following pattern DX = (TYPE1) X; - DY = (TYPE1) Y; + DY = (TYPE2) Y; DPROD = DX * DY; - DDPROD = (TYPE2) DPROD; + DDPROD = (TYPE3) DPROD; sum_1 = DDPROD + sum_0; In which - DX is double the size of X - DY is double the size of Y - DX, DY, DPROD all have the same type but the sign - between DX, DY and DPROD can differ. + between DX, DY and DPROD can differ. The sign of DPROD + is one of the signs of DX or DY. - sum is the same size of DPROD or bigger - sum has been recognized as a reduction variable. @@ -986,20 +1019,29 @@ vect_recog_dot_prod_pattern (vec_info *vinfo, inside the loop (in case we are analyzing an outer-loop). */ vect_unpromoted_value unprom0[2]; if (!vect_widened_op_tree (vinfo, mult_vinfo, MULT_EXPR, WIDEN_MULT_EXPR, - false, 2, unprom0, &half_type)) + false, 2, unprom0, &half_type, + TREE_TYPE (unprom_mult.op))) return NULL; + /* Check to see if there is a sign change happening in the operands of the + multiplication and pick the appropriate optab subtype. */ + enum optab_subtype subtype; + if (TYPE_SIGN (unprom0[0].type) == TYPE_SIGN (unprom0[1].type)) + subtype = optab_default; + else + subtype = optab_vector_mixed_sign; + vect_pattern_detected ("vect_recog_dot_prod_pattern", last_stmt); tree half_vectype; if (!vect_supportable_direct_optab_p (vinfo, type, DOT_PROD_EXPR, half_type, - type_out, &half_vectype)) + type_out, &half_vectype, subtype)) return NULL; /* Get the inputs in the appropriate types. */ tree mult_oprnd[2]; vect_convert_inputs (vinfo, stmt_vinfo, 2, mult_oprnd, half_type, - unprom0, half_vectype); + unprom0, half_vectype, true); var = vect_recog_temp_ssa_var (type, NULL); pattern_stmt = gimple_build_assign (var, DOT_PROD_EXPR,