Hi All,
This implements vectorable_early_exit which is used as the codegen part of
vectorizing a gcond.
For the most part it shares the majority of the code with
vectorizable_comparison with addition that it needs to be able to reduce
multiple resulting statements into a single one for use in the gcond, and also
needs to be able to perform masking on the comparisons.
Bootstrapped Regtested on aarch64-none-linux-gnu and no issues.
Ok for master?
Thanks,
Tamar
gcc/ChangeLog:
* tree-vect-stmts.cc (vectorizable_comparison_1): Support stmts without
lhs.
(vectorizable_early_exit): New.
(vect_analyze_stmt, vect_transform_stmt): Use it.
(vect_is_simple_use, vect_get_vector_types_for_stmt): Support gcond.
--- inline copy of patch --
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index
36aeca60a22cfaea8d3b43348000d75de1d525c7..4809b822632279493a843d402a833c9267bb315e
100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -12475,7 +12475,7 @@ vectorizable_comparison_1 (vec_info *vinfo, tree
vectype,
vec<tree> vec_oprnds0 = vNULL;
vec<tree> vec_oprnds1 = vNULL;
tree mask_type;
- tree mask;
+ tree mask = NULL_TREE;
if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
return false;
@@ -12615,8 +12615,9 @@ vectorizable_comparison_1 (vec_info *vinfo, tree
vectype,
/* Transform. */
/* Handle def. */
- lhs = gimple_assign_lhs (STMT_VINFO_STMT (stmt_info));
- mask = vect_create_destination_var (lhs, mask_type);
+ lhs = gimple_get_lhs (STMT_VINFO_STMT (stmt_info));
+ if (lhs)
+ mask = vect_create_destination_var (lhs, mask_type);
vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
rhs1, &vec_oprnds0, vectype,
@@ -12630,7 +12631,10 @@ vectorizable_comparison_1 (vec_info *vinfo, tree
vectype,
gimple *new_stmt;
vec_rhs2 = vec_oprnds1[i];
- new_temp = make_ssa_name (mask);
+ if (lhs)
+ new_temp = make_ssa_name (mask);
+ else
+ new_temp = make_temp_ssa_name (mask_type, NULL, "cmp");
if (bitop1 == NOP_EXPR)
{
new_stmt = gimple_build_assign (new_temp, code,
@@ -12709,6 +12713,196 @@ vectorizable_comparison (vec_info *vinfo,
return true;
}
+/* Check to see if the current early break given in STMT_INFO is valid for
+ vectorization. */
+
+static bool
+vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info,
+ gimple_stmt_iterator *gsi, gimple **vec_stmt,
+ slp_tree slp_node, stmt_vector_for_cost *cost_vec)
+{
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ if (!loop_vinfo
+ || !is_a <gcond *> (STMT_VINFO_STMT (stmt_info)))
+ return false;
+
+ if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_early_exit_def)
+ return false;
+
+ if (!STMT_VINFO_RELEVANT_P (stmt_info))
+ return false;
+
+ gimple_match_op op;
+ if (!gimple_extract_op (stmt_info->stmt, &op))
+ gcc_unreachable ();
+ gcc_assert (op.code.is_tree_code ());
+ auto code = tree_code (op.code);
+
+ tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
+ gcc_assert (vectype_out);
+
+ tree var_op = op.ops[0];
+
+ /* When vectorizing things like pointer comparisons we will assume that
+ the VF of both operands are the same. e.g. a pointer must be compared
+ to a pointer. We'll leave this up to vectorizable_comparison_1 to
+ check further. */
+ tree vectype_op = vectype_out;
+ if (SSA_VAR_P (var_op))
+ {
+ stmt_vec_info operand0_info
+ = loop_vinfo->lookup_stmt (SSA_NAME_DEF_STMT (var_op));
+ if (!operand0_info)
+ return false;
+
+ /* If we're in a pattern get the type of the original statement. */
+ if (STMT_VINFO_IN_PATTERN_P (operand0_info))
+ operand0_info = STMT_VINFO_RELATED_STMT (operand0_info);
+ vectype_op = STMT_VINFO_VECTYPE (operand0_info);
+ }
+
+ tree truth_type = truth_type_for (vectype_op);
+ machine_mode mode = TYPE_MODE (truth_type);
+ int ncopies;
+
+ if (slp_node)
+ ncopies = 1;
+ else
+ ncopies = vect_get_num_copies (loop_vinfo, truth_type);
+
+ vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
+ bool masked_loop_p = LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
+
+ /* Analyze only. */
+ if (!vec_stmt)
+ {
+ if (direct_optab_handler (cbranch_optab, mode) == CODE_FOR_nothing)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "can't vectorize early exit because the "
+ "target doesn't support flag setting vector "
+ "comparisons.\n");
+ return false;
+ }
+
+ if (!expand_vec_cmp_expr_p (vectype_op, truth_type, NE_EXPR))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "can't vectorize early exit because the "
+ "target does not support boolean vector "
+ "comparisons for type %T.\n", truth_type);
+ return false;
+ }
+
+ if (ncopies > 1
+ && direct_optab_handler (ior_optab, mode) == CODE_FOR_nothing)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "can't vectorize early exit because the "
+ "target does not support boolean vector OR for "
+ "type %T.\n", truth_type);
+ return false;
+ }
+
+ if (!vectorizable_comparison_1 (vinfo, truth_type, stmt_info, code, gsi,
+ vec_stmt, slp_node, cost_vec))
+ return false;
+
+ if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
+ vect_record_loop_mask (loop_vinfo, masks, ncopies, truth_type, NULL);
+
+ return true;
+ }
+
+ /* Tranform. */
+
+ tree new_temp = NULL_TREE;
+ gimple *new_stmt = NULL;
+
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location, "transform early-exit.\n");
+
+ if (!vectorizable_comparison_1 (vinfo, truth_type, stmt_info, code, gsi,
+ vec_stmt, slp_node, cost_vec))
+ gcc_unreachable ();
+
+ gimple *stmt = STMT_VINFO_STMT (stmt_info);
+ basic_block cond_bb = gimple_bb (stmt);
+ gimple_stmt_iterator cond_gsi = gsi_last_bb (cond_bb);
+
+ vec<tree> stmts;
+
+ if (slp_node)
+ stmts = SLP_TREE_VEC_DEFS (slp_node);
+ else
+ {
+ auto vec_stmts = STMT_VINFO_VEC_STMTS (stmt_info);
+ stmts.create (vec_stmts.length ());
+ for (auto stmt : vec_stmts)
+ stmts.quick_push (gimple_assign_lhs (stmt));
+ }
+
+ /* Determine if we need to reduce the final value. */
+ if (stmts.length () > 1)
+ {
+ /* We build the reductions in a way to maintain as much parallelism as
+ possible. */
+ auto_vec<tree> workset (stmts.length ());
+ workset.splice (stmts);
+ while (workset.length () > 1)
+ {
+ new_temp = make_temp_ssa_name (truth_type, NULL, "vexit_reduc");
+ tree arg0 = workset.pop ();
+ tree arg1 = workset.pop ();
+ new_stmt = gimple_build_assign (new_temp, BIT_IOR_EXPR, arg0, arg1);
+ vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt,
+ &cond_gsi);
+ if (slp_node)
+ slp_node->push_vec_def (new_stmt);
+ else
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
+ workset.quick_insert (0, new_temp);
+ }
+ }
+ else
+ new_temp = stmts[0];
+
+ gcc_assert (new_temp);
+
+ tree cond = new_temp;
+ if (masked_loop_p)
+ {
+ tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks, ncopies,
truth_type, 0);
+ cond = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask, cond,
+ &cond_gsi);
+ }
+
+ /* Now build the new conditional. Pattern gimple_conds get dropped during
+ codegen so we must replace the original insn. */
+ if (is_pattern_stmt_p (stmt_info))
+ stmt = STMT_VINFO_STMT (STMT_VINFO_RELATED_STMT (stmt_info));
+
+ tree t = fold_build2 (NE_EXPR, boolean_type_node, cond,
+ build_zero_cst (truth_type));
+ t = canonicalize_cond_expr_cond (t);
+ gimple_cond_set_condition_from_tree ((gcond*)stmt, t);
+ update_stmt (stmt);
+
+ if (slp_node)
+ slp_node->push_vec_def (stmt);
+ else
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (stmt);
+
+
+ if (!slp_node)
+ *vec_stmt = stmt;
+
+ return true;
+}
+
/* If SLP_NODE is nonnull, return true if vectorizable_live_operation
can handle all live statements in the node. Otherwise return true
if STMT_INFO is not live or if vectorizable_live_operation can handle it.
@@ -12928,7 +13122,9 @@ vect_analyze_stmt (vec_info *vinfo,
|| vectorizable_lc_phi (as_a <loop_vec_info> (vinfo),
stmt_info, NULL, node)
|| vectorizable_recurr (as_a <loop_vec_info> (vinfo),
- stmt_info, NULL, node, cost_vec));
+ stmt_info, NULL, node, cost_vec)
+ || vectorizable_early_exit (vinfo, stmt_info, NULL, NULL, node,
+ cost_vec));
else
{
if (bb_vinfo)
@@ -12951,7 +13147,10 @@ vect_analyze_stmt (vec_info *vinfo,
NULL, NULL, node, cost_vec)
|| vectorizable_comparison (vinfo, stmt_info, NULL, NULL, node,
cost_vec)
- || vectorizable_phi (vinfo, stmt_info, NULL, node, cost_vec));
+ || vectorizable_phi (vinfo, stmt_info, NULL, node, cost_vec)
+ || vectorizable_early_exit (vinfo, stmt_info, NULL, NULL, node,
+ cost_vec));
+
}
if (node)
@@ -13110,6 +13309,12 @@ vect_transform_stmt (vec_info *vinfo,
gcc_assert (done);
break;
+ case loop_exit_ctrl_vec_info_type:
+ done = vectorizable_early_exit (vinfo, stmt_info, gsi, &vec_stmt,
+ slp_node, NULL);
+ gcc_assert (done);
+ break;
+
default:
if (!STMT_VINFO_LIVE_P (stmt_info))
{
@@ -13511,7 +13716,7 @@ vect_is_simple_use (tree operand, vec_info *vinfo, enum
vect_def_type *dt,
case vect_first_order_recurrence:
dump_printf (MSG_NOTE, "first order recurrence\n");
break;
- case vect_early_exit_def:
+ case vect_early_exit_def:
dump_printf (MSG_NOTE, "early exit\n");
break;
case vect_unknown_def_type:
--
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index
36aeca60a22cfaea8d3b43348000d75de1d525c7..4809b822632279493a843d402a833c9267bb315e
100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -12475,7 +12475,7 @@ vectorizable_comparison_1 (vec_info *vinfo, tree
vectype,
vec<tree> vec_oprnds0 = vNULL;
vec<tree> vec_oprnds1 = vNULL;
tree mask_type;
- tree mask;
+ tree mask = NULL_TREE;
if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
return false;
@@ -12615,8 +12615,9 @@ vectorizable_comparison_1 (vec_info *vinfo, tree
vectype,
/* Transform. */
/* Handle def. */
- lhs = gimple_assign_lhs (STMT_VINFO_STMT (stmt_info));
- mask = vect_create_destination_var (lhs, mask_type);
+ lhs = gimple_get_lhs (STMT_VINFO_STMT (stmt_info));
+ if (lhs)
+ mask = vect_create_destination_var (lhs, mask_type);
vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
rhs1, &vec_oprnds0, vectype,
@@ -12630,7 +12631,10 @@ vectorizable_comparison_1 (vec_info *vinfo, tree
vectype,
gimple *new_stmt;
vec_rhs2 = vec_oprnds1[i];
- new_temp = make_ssa_name (mask);
+ if (lhs)
+ new_temp = make_ssa_name (mask);
+ else
+ new_temp = make_temp_ssa_name (mask_type, NULL, "cmp");
if (bitop1 == NOP_EXPR)
{
new_stmt = gimple_build_assign (new_temp, code,
@@ -12709,6 +12713,196 @@ vectorizable_comparison (vec_info *vinfo,
return true;
}
+/* Check to see if the current early break given in STMT_INFO is valid for
+ vectorization. */
+
+static bool
+vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info,
+ gimple_stmt_iterator *gsi, gimple **vec_stmt,
+ slp_tree slp_node, stmt_vector_for_cost *cost_vec)
+{
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ if (!loop_vinfo
+ || !is_a <gcond *> (STMT_VINFO_STMT (stmt_info)))
+ return false;
+
+ if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_early_exit_def)
+ return false;
+
+ if (!STMT_VINFO_RELEVANT_P (stmt_info))
+ return false;
+
+ gimple_match_op op;
+ if (!gimple_extract_op (stmt_info->stmt, &op))
+ gcc_unreachable ();
+ gcc_assert (op.code.is_tree_code ());
+ auto code = tree_code (op.code);
+
+ tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
+ gcc_assert (vectype_out);
+
+ tree var_op = op.ops[0];
+
+ /* When vectorizing things like pointer comparisons we will assume that
+ the VF of both operands are the same. e.g. a pointer must be compared
+ to a pointer. We'll leave this up to vectorizable_comparison_1 to
+ check further. */
+ tree vectype_op = vectype_out;
+ if (SSA_VAR_P (var_op))
+ {
+ stmt_vec_info operand0_info
+ = loop_vinfo->lookup_stmt (SSA_NAME_DEF_STMT (var_op));
+ if (!operand0_info)
+ return false;
+
+ /* If we're in a pattern get the type of the original statement. */
+ if (STMT_VINFO_IN_PATTERN_P (operand0_info))
+ operand0_info = STMT_VINFO_RELATED_STMT (operand0_info);
+ vectype_op = STMT_VINFO_VECTYPE (operand0_info);
+ }
+
+ tree truth_type = truth_type_for (vectype_op);
+ machine_mode mode = TYPE_MODE (truth_type);
+ int ncopies;
+
+ if (slp_node)
+ ncopies = 1;
+ else
+ ncopies = vect_get_num_copies (loop_vinfo, truth_type);
+
+ vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
+ bool masked_loop_p = LOOP_VINFO_FULLY_MASKED_P (loop_vinfo);
+
+ /* Analyze only. */
+ if (!vec_stmt)
+ {
+ if (direct_optab_handler (cbranch_optab, mode) == CODE_FOR_nothing)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "can't vectorize early exit because the "
+ "target doesn't support flag setting vector "
+ "comparisons.\n");
+ return false;
+ }
+
+ if (!expand_vec_cmp_expr_p (vectype_op, truth_type, NE_EXPR))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "can't vectorize early exit because the "
+ "target does not support boolean vector "
+ "comparisons for type %T.\n", truth_type);
+ return false;
+ }
+
+ if (ncopies > 1
+ && direct_optab_handler (ior_optab, mode) == CODE_FOR_nothing)
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "can't vectorize early exit because the "
+ "target does not support boolean vector OR for "
+ "type %T.\n", truth_type);
+ return false;
+ }
+
+ if (!vectorizable_comparison_1 (vinfo, truth_type, stmt_info, code, gsi,
+ vec_stmt, slp_node, cost_vec))
+ return false;
+
+ if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
+ vect_record_loop_mask (loop_vinfo, masks, ncopies, truth_type, NULL);
+
+ return true;
+ }
+
+ /* Tranform. */
+
+ tree new_temp = NULL_TREE;
+ gimple *new_stmt = NULL;
+
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location, "transform early-exit.\n");
+
+ if (!vectorizable_comparison_1 (vinfo, truth_type, stmt_info, code, gsi,
+ vec_stmt, slp_node, cost_vec))
+ gcc_unreachable ();
+
+ gimple *stmt = STMT_VINFO_STMT (stmt_info);
+ basic_block cond_bb = gimple_bb (stmt);
+ gimple_stmt_iterator cond_gsi = gsi_last_bb (cond_bb);
+
+ vec<tree> stmts;
+
+ if (slp_node)
+ stmts = SLP_TREE_VEC_DEFS (slp_node);
+ else
+ {
+ auto vec_stmts = STMT_VINFO_VEC_STMTS (stmt_info);
+ stmts.create (vec_stmts.length ());
+ for (auto stmt : vec_stmts)
+ stmts.quick_push (gimple_assign_lhs (stmt));
+ }
+
+ /* Determine if we need to reduce the final value. */
+ if (stmts.length () > 1)
+ {
+ /* We build the reductions in a way to maintain as much parallelism as
+ possible. */
+ auto_vec<tree> workset (stmts.length ());
+ workset.splice (stmts);
+ while (workset.length () > 1)
+ {
+ new_temp = make_temp_ssa_name (truth_type, NULL, "vexit_reduc");
+ tree arg0 = workset.pop ();
+ tree arg1 = workset.pop ();
+ new_stmt = gimple_build_assign (new_temp, BIT_IOR_EXPR, arg0, arg1);
+ vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt,
+ &cond_gsi);
+ if (slp_node)
+ slp_node->push_vec_def (new_stmt);
+ else
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
+ workset.quick_insert (0, new_temp);
+ }
+ }
+ else
+ new_temp = stmts[0];
+
+ gcc_assert (new_temp);
+
+ tree cond = new_temp;
+ if (masked_loop_p)
+ {
+ tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks, ncopies,
truth_type, 0);
+ cond = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask, cond,
+ &cond_gsi);
+ }
+
+ /* Now build the new conditional. Pattern gimple_conds get dropped during
+ codegen so we must replace the original insn. */
+ if (is_pattern_stmt_p (stmt_info))
+ stmt = STMT_VINFO_STMT (STMT_VINFO_RELATED_STMT (stmt_info));
+
+ tree t = fold_build2 (NE_EXPR, boolean_type_node, cond,
+ build_zero_cst (truth_type));
+ t = canonicalize_cond_expr_cond (t);
+ gimple_cond_set_condition_from_tree ((gcond*)stmt, t);
+ update_stmt (stmt);
+
+ if (slp_node)
+ slp_node->push_vec_def (stmt);
+ else
+ STMT_VINFO_VEC_STMTS (stmt_info).safe_push (stmt);
+
+
+ if (!slp_node)
+ *vec_stmt = stmt;
+
+ return true;
+}
+
/* If SLP_NODE is nonnull, return true if vectorizable_live_operation
can handle all live statements in the node. Otherwise return true
if STMT_INFO is not live or if vectorizable_live_operation can handle it.
@@ -12928,7 +13122,9 @@ vect_analyze_stmt (vec_info *vinfo,
|| vectorizable_lc_phi (as_a <loop_vec_info> (vinfo),
stmt_info, NULL, node)
|| vectorizable_recurr (as_a <loop_vec_info> (vinfo),
- stmt_info, NULL, node, cost_vec));
+ stmt_info, NULL, node, cost_vec)
+ || vectorizable_early_exit (vinfo, stmt_info, NULL, NULL, node,
+ cost_vec));
else
{
if (bb_vinfo)
@@ -12951,7 +13147,10 @@ vect_analyze_stmt (vec_info *vinfo,
NULL, NULL, node, cost_vec)
|| vectorizable_comparison (vinfo, stmt_info, NULL, NULL, node,
cost_vec)
- || vectorizable_phi (vinfo, stmt_info, NULL, node, cost_vec));
+ || vectorizable_phi (vinfo, stmt_info, NULL, node, cost_vec)
+ || vectorizable_early_exit (vinfo, stmt_info, NULL, NULL, node,
+ cost_vec));
+
}
if (node)
@@ -13110,6 +13309,12 @@ vect_transform_stmt (vec_info *vinfo,
gcc_assert (done);
break;
+ case loop_exit_ctrl_vec_info_type:
+ done = vectorizable_early_exit (vinfo, stmt_info, gsi, &vec_stmt,
+ slp_node, NULL);
+ gcc_assert (done);
+ break;
+
default:
if (!STMT_VINFO_LIVE_P (stmt_info))
{
@@ -13511,7 +13716,7 @@ vect_is_simple_use (tree operand, vec_info *vinfo, enum
vect_def_type *dt,
case vect_first_order_recurrence:
dump_printf (MSG_NOTE, "first order recurrence\n");
break;
- case vect_early_exit_def:
+ case vect_early_exit_def:
dump_printf (MSG_NOTE, "early exit\n");
break;
case vect_unknown_def_type:
