[PATCH 3/3][vect]: use .VARYING for early break IV [PR125597]

[Tamar Christina]

Similar to the mask control variable, this replaces and adds the temporary SSA
var for early break scalar IVs with an IFN_VARYING until the replacement is done
when we start vectorization.

This prevents similar issues as the mask control variable when the IV
expressions are folded.

Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.

Any comments? Does this do what you expected Richi?

Thanks,
Tamar

gcc/ChangeLog:

        PR tree-optimization/125597
        * tree-vect-loop-manip.cc (vect_do_peeling): Create IFN_VARYING for
        early break scalar IV.
        * tree-vect-loop.cc (vect_update_ivs_after_vectorizer_for_early_breaks):
        Replace the IFN_VARYING with actual value.

---
diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
index 
4a3444966acf5d66394196caf24dc3ba2fa63ec3..d2c854e16d4413dc26ab1ec35d20633ef415d01c
 100644
--- a/gcc/tree-vect-loop-manip.cc
+++ b/gcc/tree-vect-loop-manip.cc
@@ -3841,6 +3841,10 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, 
tree nitersm1,
        {
          tree tmp_niters_vf
            = make_ssa_name (LOOP_VINFO_EARLY_BRK_IV_TYPE (loop_vinfo));
+         gcall *tmp_call = gimple_build_call_internal (IFN_VARYING, 0);
+         gimple_call_set_lhs (tmp_call, tmp_niters_vf);
+         auto header_gsi = gsi_after_labels (loop->header);
+         gsi_insert_after (&header_gsi, tmp_call, GSI_SAME_STMT);
 
          if (!(LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo)
                && get_loop_exit_edges (loop).length () == 1)
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 
51e7f05100de7ab31994f3b4a30f5c5be8d8afdd..4d9b691c3299ce08fc0946d9065e3cdead816d8a
 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -11079,6 +11079,11 @@ vect_update_ivs_after_vectorizer_for_early_breaks 
(loop_vec_info loop_vinfo)
   auto loop = LOOP_VINFO_LOOP (loop_vinfo);
   tree induc_var = niters_skip ? copy_ssa_name (phi_var) : phi_var;
 
+  /* Remove the existing dummy GIMPLE statement and just keep the def.  */
+  gimple *def = SSA_NAME_DEF_STMT (phi_var);
+  auto def_gsi = gsi_for_stmt (def);
+  gsi_remove (&def_gsi, true);
+
   auto induction_phi = create_phi_node (induc_var, loop->header);
   tree induc_def = PHI_RESULT (induction_phi);
 
@@ -11149,7 +11154,7 @@ vect_update_ivs_after_vectorizer_for_early_breaks 
(loop_vec_info loop_vinfo)
       gcc_assert (exit_bb);
       auto exit_gsi = gsi_after_labels (exit_bb);
       gsi_insert_seq_before (&exit_gsi, iv_stmts, GSI_SAME_STMT);
-  }
+    }
   /* Write the init_stmts in the loop-preheader block.  */
   auto psi = gsi_last_nondebug_bb (pe->src);
   gsi_insert_seq_after (&psi, init_stmts, GSI_LAST_NEW_STMT);


-- 

diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
index 4a3444966acf5d66394196caf24dc3ba2fa63ec3..d2c854e16d4413dc26ab1ec35d20633ef415d01c 100644
--- a/gcc/tree-vect-loop-manip.cc
+++ b/gcc/tree-vect-loop-manip.cc
@@ -3841,6 +3841,10 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
 	{
 	  tree tmp_niters_vf
 	    = make_ssa_name (LOOP_VINFO_EARLY_BRK_IV_TYPE (loop_vinfo));
+	  gcall *tmp_call = gimple_build_call_internal (IFN_VARYING, 0);
+	  gimple_call_set_lhs (tmp_call, tmp_niters_vf);
+	  auto header_gsi = gsi_after_labels (loop->header);
+	  gsi_insert_after (&header_gsi, tmp_call, GSI_SAME_STMT);
 
 	  if (!(LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo)
 		&& get_loop_exit_edges (loop).length () == 1)
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 51e7f05100de7ab31994f3b4a30f5c5be8d8afdd..4d9b691c3299ce08fc0946d9065e3cdead816d8a 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -11079,6 +11079,11 @@ vect_update_ivs_after_vectorizer_for_early_breaks (loop_vec_info loop_vinfo)
   auto loop = LOOP_VINFO_LOOP (loop_vinfo);
   tree induc_var = niters_skip ? copy_ssa_name (phi_var) : phi_var;
 
+  /* Remove the existing dummy GIMPLE statement and just keep the def.  */
+  gimple *def = SSA_NAME_DEF_STMT (phi_var);
+  auto def_gsi = gsi_for_stmt (def);
+  gsi_remove (&def_gsi, true);
+
   auto induction_phi = create_phi_node (induc_var, loop->header);
   tree induc_def = PHI_RESULT (induction_phi);
 
@@ -11149,7 +11154,7 @@ vect_update_ivs_after_vectorizer_for_early_breaks (loop_vec_info loop_vinfo)
       gcc_assert (exit_bb);
       auto exit_gsi = gsi_after_labels (exit_bb);
       gsi_insert_seq_before (&exit_gsi, iv_stmts, GSI_SAME_STMT);
-  }
+    }
   /* Write the init_stmts in the loop-preheader block.  */
   auto psi = gsi_last_nondebug_bb (pe->src);
   gsi_insert_seq_after (&psi, init_stmts, GSI_LAST_NEW_STMT);