On Tue, 30 Jun 2026, Raghesh Aloor wrote:
> Define IFN_PREDICATED_INDEX_UPDATE and IFN_MASK_COMPRESS_SCATTER_STORE,
> add scalar RTL expanders, and implement phase-2 pattern recognition for
> the predicated-index PHI latch update and paired IFN_MASK_STORE.
>
> During phase-1 detection, record the mask-store base and element scale in
> loop_vec_info so compress-scatter recog can build
> IFN_MASK_COMPRESS_SCATTER_STORE without re-walking the address chain.
IFNs need documenting in ifns.texi, I suppose you want corresponding
optabs as well. I assume the IFN expanders will go away or be
direct optab expanders.
> ---
> gcc/internal-fn.cc | 58 +++++++++
> gcc/internal-fn.def | 4 +
> gcc/tree-vect-loop.cc | 7 +-
> gcc/tree-vect-patterns.cc | 3 +
> gcc/tree-vect-vpcompress.cc | 229 ++++++++++++++++++++++++++++++------
> gcc/tree-vect-vpcompress.h | 6 +
> gcc/tree-vectorizer.h | 7 ++
> 7 files changed, 279 insertions(+), 35 deletions(-)
>
> diff --git a/gcc/internal-fn.cc b/gcc/internal-fn.cc
> index 0138c6f7ef0..bf0f430ec7d 100644
> --- a/gcc/internal-fn.cc
> +++ b/gcc/internal-fn.cc
> @@ -4777,6 +4777,56 @@ set_edom_supported_p (void)
> }
> #include "internal-fn.def"
>
> +/* Expand IFN_PREDICATED_INDEX_UPDATE to mask ? idx + 1 : idx. */
> +
> +static void
> +expand_PREDICATED_INDEX_UPDATE (internal_fn, gcall *stmt)
> +{
> + tree lhs = gimple_call_lhs (stmt);
> + gcc_assert (lhs != NULL_TREE);
> + tree idx = gimple_call_arg (stmt, 0);
> + tree mask = gimple_call_arg (stmt, 1);
> + tree type = TREE_TYPE (lhs);
> + tree one = build_int_cst (type, 1);
> + tree inc = fold_build2 (PLUS_EXPR, type, idx, one);
> + tree cond_expr;
> + if (VECTOR_TYPE_P (TREE_TYPE (mask)))
> + gcc_unreachable ();
> + if (TREE_CODE (TREE_TYPE (mask)) == BOOLEAN_TYPE)
> + cond_expr = fold_build3 (COND_EXPR, type, mask, inc, idx);
> + else
> + {
> + tree cmp = fold_build2 (NE_EXPR, boolean_type_node, mask,
> + build_int_cst (TREE_TYPE (mask), 0));
> + cond_expr = fold_build3 (COND_EXPR, type, cmp, inc, idx);
> + }
> + rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
> + rtx tmp = expand_expr (cond_expr, target, TYPE_MODE (type), EXPAND_NORMAL);
> + if (!rtx_equal_p (tmp, target))
> + emit_move_insn (target, tmp);
> +}
> +
> +/* Lower IFN_MASK_COMPRESS_SCATTER_STORE to IFN_MASK_STORE for now. */
> +
> +static void
> +expand_MASK_COMPRESS_SCATTER_STORE (internal_fn, gcall *stmt)
> +{
> + tree base = gimple_call_arg (stmt, 0);
> + tree align = gimple_call_arg (stmt, 1);
> + tree pack_idx = gimple_call_arg (stmt, 2);
> + tree scale = gimple_call_arg (stmt, 3);
> + tree value = gimple_call_arg (stmt, 4);
> + tree mask = gimple_call_arg (stmt, 5);
> + tree byte_off = fold_build2 (MULT_EXPR, sizetype, pack_idx,
> + fold_convert (sizetype, scale));
> + tree ptr = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (base), base,
> + byte_off);
> + gcall *call
> + = gimple_build_call_internal (IFN_MASK_STORE, 4, ptr, align, mask,
> value);
> + gimple_set_location (call, gimple_location (stmt));
> + expand_internal_call (IFN_MASK_STORE, call);
> +}
> +
> /* Routines to expand each internal function, indexed by function number.
> Each routine has the prototype:
>
> @@ -5104,6 +5154,7 @@ internal_store_fn_p (internal_fn fn)
> case IFN_MASK_LEN_SCATTER_STORE:
> case IFN_LEN_STORE:
> case IFN_MASK_LEN_STORE:
> + case IFN_MASK_COMPRESS_SCATTER_STORE:
> return true;
>
> default:
> @@ -5124,6 +5175,7 @@ internal_gather_scatter_fn_p (internal_fn fn)
> case IFN_SCATTER_STORE:
> case IFN_MASK_SCATTER_STORE:
> case IFN_MASK_LEN_SCATTER_STORE:
> + case IFN_MASK_COMPRESS_SCATTER_STORE:
> return true;
>
> default:
> @@ -5303,6 +5355,7 @@ internal_fn_mask_index (internal_fn fn)
> case IFN_MASK_SCATTER_STORE:
> case IFN_MASK_LEN_GATHER_LOAD:
> case IFN_MASK_LEN_SCATTER_STORE:
> + case IFN_MASK_COMPRESS_SCATTER_STORE:
> return 5;
>
> case IFN_VCOND_MASK:
> @@ -5329,6 +5382,8 @@ internal_fn_stored_value_index (internal_fn fn)
> case IFN_MASK_STORE:
> case IFN_MASK_STORE_LANES:
> return 3;
> +
> + case IFN_MASK_COMPRESS_SCATTER_STORE:
> case IFN_SCATTER_STORE:
> case IFN_MASK_SCATTER_STORE:
> case IFN_MASK_LEN_SCATTER_STORE:
> @@ -5361,6 +5416,7 @@ internal_fn_alias_ptr_index (internal_fn fn)
> case IFN_SCATTER_STORE:
> case IFN_MASK_SCATTER_STORE:
> case IFN_MASK_LEN_SCATTER_STORE:
> + case IFN_MASK_COMPRESS_SCATTER_STORE:
> return 1;
>
> default:
> @@ -5385,6 +5441,7 @@ internal_fn_offset_index (internal_fn fn)
> case IFN_SCATTER_STORE:
> case IFN_MASK_SCATTER_STORE:
> case IFN_MASK_LEN_SCATTER_STORE:
> + case IFN_MASK_COMPRESS_SCATTER_STORE:
> return 2;
>
> default:
> @@ -5409,6 +5466,7 @@ internal_fn_scale_index (internal_fn fn)
> case IFN_SCATTER_STORE:
> case IFN_MASK_SCATTER_STORE:
> case IFN_MASK_LEN_SCATTER_STORE:
> + case IFN_MASK_COMPRESS_SCATTER_STORE:
> return 3;
>
> default:
> diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def
> index af9f92950c7..7bb2ff37edd 100644
> --- a/gcc/internal-fn.def
> +++ b/gcc/internal-fn.def
> @@ -639,6 +639,10 @@ DEF_INTERNAL_FN (ASSUME, ECF_CONST | ECF_LEAF |
> ECF_NOTHROW
> /* For if-conversion of inbranch SIMD clones. */
> DEF_INTERNAL_FN (MASK_CALL, ECF_NOVOPS, NULL)
>
> +/* For Predicated index PHI / compress-scatter . */
> +DEF_INTERNAL_FN (PREDICATED_INDEX_UPDATE, 0, NULL)
> +DEF_INTERNAL_FN (MASK_COMPRESS_SCATTER_STORE, 0, NULL)
> +
> /* _BitInt support. */
> DEF_INTERNAL_FN (MULBITINT, ECF_LEAF | ECF_NOTHROW, ". O . R . R . ")
> DEF_INTERNAL_FN (DIVMODBITINT, ECF_LEAF, ". O . O . R . R . ")
> diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
> index 003f9565a2a..b93a6648acf 100644
> --- a/gcc/tree-vect-loop.cc
> +++ b/gcc/tree-vect-loop.cc
> @@ -788,7 +788,12 @@ _loop_vec_info::_loop_vec_info (class loop *loop_in,
> vec_info_shared *shared)
> early_break_needs_epilogue (false),
> early_break_niters_var (NULL),
> predicated_index_phi (NULL),
> - predicated_index_mask_store (NULL)
> + predicated_index_mask_store (NULL),
> + predicated_index_store_base (NULL_TREE),
> + predicated_index_store_scale (NULL_TREE),
> + predicated_index_mask_compress_scatter_store (NULL),
> + predicated_index_update_ifn_p (false),
> + predicated_index_compress_scatter_ifn_p (false)
> {
> /* CHECKME: We want to visit all BBs before their successors (except for
> latch blocks, for which this assertion wouldn't hold). In the simple
> diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
> index ab7892b6c7b..8aab8a6de1c 100644
> --- a/gcc/tree-vect-patterns.cc
> +++ b/gcc/tree-vect-patterns.cc
> @@ -41,6 +41,7 @@ along with GCC; see the file COPYING3. If not see
> #include "gimplify-me.h"
> #include "cfgloop.h"
> #include "tree-vectorizer.h"
> +#include "tree-vect-vpcompress.h"
> #include "dumpfile.h"
> #include "builtins.h"
> #include "internal-fn.h"
> @@ -7453,6 +7454,7 @@ static vect_recog_func vect_vect_recog_func_ptrs[] = {
> possible beforehand. */
> { vect_recog_average_pattern, "average" },
> { vect_recog_cond_expr_convert_pattern, "cond_expr_convert" },
> + { vect_recog_predicated_index_update_pattern, "predicated_index_update" },
> { vect_recog_mulhs_pattern, "mult_high" },
> { vect_recog_cast_forwprop_pattern, "cast_forwprop" },
> { vect_recog_widen_mult_pattern, "widen_mult" },
> @@ -7477,6 +7479,7 @@ static vect_recog_func vect_vect_recog_func_ptrs[] = {
> of mask conversion that are needed for gather and scatter
> internal functions. */
> { vect_recog_gather_scatter_pattern, "gather_scatter" },
> + { vect_recog_compress_scatter_pattern, "compress_scatter" },
> { vect_recog_cond_store_pattern, "cond_store" },
> { vect_recog_mask_conversion_pattern, "mask_conversion" },
> { vect_recog_widen_plus_pattern, "widen_plus" },
> diff --git a/gcc/tree-vect-vpcompress.cc b/gcc/tree-vect-vpcompress.cc
> index c1a87405b94..0060ad5d780 100644
> --- a/gcc/tree-vect-vpcompress.cc
> +++ b/gcc/tree-vect-vpcompress.cc
> @@ -164,27 +164,31 @@ predicated_index_plus_one (tree &pred_phi,
> return false;
> }
>
> -/* Function pred_phi_in_mask_store_addr_computation.
> +/* Function pred_phi_mask_store_base_and_scale.
>
> - Starting from the store pointer in MASK_STORE_CALL, this function tries to
> - check if the base + offset computation is done and offset finaly reaches
> - PRED_PHI.
> + Starting from the store pointer in MASK_STORE_CALL, check that the address
> + has the form base + pred_phi * element_size and return BASE_OUT and
> + SCALE_OUT on success.
>
> # n = PHI <n_1, ..>
> ...
> a_1 = (long unsigned int) n;
> a_2 = a_1 * 4;
> a_3 = base + a_2;
> - addr = (int *) a_3;
> + addr = (int *) a_3;
> .MASK_STORE (addr, mask, ..)
> n_1 = n + 1;
> ...
> - n_2 = mask ? n_1 : n; */
> + n_2 = mask ? n_1 : n;
> +
> + On success BASE_OUT is base and SCALE_OUT is 4. */
> +
> static bool
> -pred_phi_in_mask_store_addr_computation (class loop *loop,
> - gcall *mask_store_call, tree pred_phi)
> +pred_phi_mask_store_base_and_scale (class loop *loop,
> + gcall *mask_store_call, tree pred_phi,
> + tree *base_out, tree *scale_out)
> {
> - /* Get the pointer operand of MASK_SLP_CALL. 'addr' in the example. */
> + /* Get the pointer operand of MASK_STORE_CALL. 'addr' in the example. */
> tree ptr = gimple_call_arg (mask_store_call, 0);
> if (TREE_CODE (ptr) != SSA_NAME)
> return false;
> @@ -222,21 +226,21 @@ pred_phi_in_mask_store_addr_computation (class loop
> *loop,
> if (!add_stmt)
> return false;
>
> - /* Check if add_stmt is base + offset format. Note that it can be either
> - base + offset or offset + base. We take the loop invariant pointer as
> - the base. Example - a_3 = base + a_2; */
> + /* Identify the loop-invariant base and the varying offset.
> + a_3 = base + a_2; */
> tree add_op0 = gimple_assign_rhs1 (add_stmt);
> tree add_op1 = gimple_assign_rhs2 (add_stmt);
> - tree offset;
> + tree base = add_op0;
> + tree offset = add_op1;
> if (expr_invariant_in_loop_p (loop, add_op1))
> - offset = add_op0;
> - else if (expr_invariant_in_loop_p (loop, add_op0))
> - offset = add_op1;
> - else
> + {
> + base = add_op1;
> + offset = add_op0;
> + }
> + else if (!expr_invariant_in_loop_p (loop, add_op0))
> return false;
>
> - /* Now offset can be a multiplication with the element size as in
> - a_2 = a_1 * 4; */
> + /* Offset should be index * element size, as in a_2 = a_1 * 4. */
> if (TREE_CODE (offset) != SSA_NAME)
> return false;
> gimple *def = SSA_NAME_DEF_STMT (offset);
> @@ -245,7 +249,7 @@ pred_phi_in_mask_store_addr_computation (class loop *loop,
> return false;
> tree mul_op0 = gimple_assign_rhs1 (mul);
> tree mul_op1 = gimple_assign_rhs2 (mul);
> - tree index_op = mul_op0;
> + tree index_op = mul_op0;
> tree size_op = mul_op1;
> if (TREE_CODE (mul_op0) == INTEGER_CST)
> {
> @@ -254,10 +258,11 @@ pred_phi_in_mask_store_addr_computation (class loop
> *loop,
> }
> else if (TREE_CODE (mul_op1) != INTEGER_CST)
> return false;
> - /* The index_op should be an SSA_NAM. */
> + /* The index operand should be an SSA_NAME. */
> if (TREE_CODE (index_op) != SSA_NAME)
> - return false;
> - /* The size_op should be the element size pointed by ptr. */
> + return false;
> +
> + /* SIZE_OP must match the element size of the store pointer type. */
> tree eltype = TREE_TYPE (TREE_TYPE (ptr));
> tree elem_size_unit = TYPE_SIZE_UNIT (eltype);
> if (!elem_size_unit || TREE_CODE (elem_size_unit) != INTEGER_CST)
> @@ -265,12 +270,16 @@ pred_phi_in_mask_store_addr_computation (class loop
> *loop,
> if (!operand_equal_p (size_op, elem_size_unit, OEP_ONLY_CONST))
> return false;
>
> - /* No the index_op can be a casted one. So strip off that cast as in
> - a_1 = (long unsigned int) n; */
> + /* Index may be cast, as in a_1 = (long unsigned int) n. */
> pred_phi_strip_conversion (&index_op);
> + if (index_op != pred_phi)
> + return false;
>
> - /* Finally index_op should be PRED_PHI. */
> - return index_op == pred_phi;
> + if (base_out)
> + *base_out = base;
> + if (scale_out)
> + *scale_out = elem_size_unit;
> + return true;
> }
>
> /* Function vect_find_predicated_index_mask_store.
> @@ -295,7 +304,8 @@ pred_phi_in_mask_store_addr_computation (class loop *loop,
> n_2 = mask ? n_1 : n; */
> static gcall *
> vect_find_predicated_index_mask_store (class loop *loop, gphi *phi,
> - tree pred_phi, tree pred_phi_mask)
> + tree pred_phi, tree pred_phi_mask,
> + tree *base_out, tree *scale_out)
> {
> /* Search only the basic block that holds PHI; TODO: to scan other blocks
> when generalizing. */
> @@ -317,9 +327,8 @@ vect_find_predicated_index_mask_store (class loop *loop,
> gphi *phi,
> continue;
>
> /* Check if PRED_PHI is used to compute mask_store_call address. */
> - /* TODO: Can use any match operator (as in instcombine)?? */
> - if (!pred_phi_in_mask_store_addr_computation(loop, mask_store_call,
> - pred_phi))
> + if (!pred_phi_mask_store_base_and_scale (loop, mask_store_call,
> pred_phi,
> + base_out, scale_out))
> continue;
>
> if (dump_enabled_p ())
> @@ -349,8 +358,8 @@ vect_find_predicated_index_mask_store (class loop *loop,
> gphi *phi,
> ...
> n_2 = mask ? n_1 : n; */
> bool
> -vect_vpcompress_analyze_predicated_index_phi (loop_vec_info loop_vinfo,
> - class loop *loop, gphi *phi)
> +vect_vpcompress_analyze_predicated_index_phi (loop_vec_info loop_vinfo,
> class loop *loop,
> + gphi *phi)
> {
> /* Require a loop-header PHI. */
> if (gimple_bb (phi) != loop->header)
> @@ -397,8 +406,11 @@ vect_vpcompress_analyze_predicated_index_phi
> (loop_vec_info loop_vinfo,
> * computed based on pred-phi and it should use the same mask as that of
> * COND_EXPR. */
> tree mask = gimple_assign_rhs1 (pred_phi_back_edge_asgn_stmt);
> + tree store_base;
> + tree store_scale;
> gcall *mask_store =
> - vect_find_predicated_index_mask_store (loop, phi, pred_phi, mask);
> + vect_find_predicated_index_mask_store (loop, phi, pred_phi, mask,
> + &store_base, &store_scale);
>
> if (dump_enabled_p ())
> dump_printf_loc (MSG_NOTE, vect_location,
> @@ -415,7 +427,156 @@ vect_vpcompress_analyze_predicated_index_phi
> (loop_vec_info loop_vinfo,
>
> loop_vinfo->predicated_index_phi = phi;
> loop_vinfo->predicated_index_mask_store = mask_store;
> + loop_vinfo->predicated_index_store_base = store_base;
> + loop_vinfo->predicated_index_store_scale = store_scale;
>
> /* All requirements satisfiled. */
> return true;
> }
> +
> +/* Function vect_recog_predicated_index_update_pattern.
> +
> + After phase-1 detection, rewrite the predicated-index PHI latch update
> + from a COND_EXPR into IFN_PREDICATED_INDEX_UPDATE. The statement must be
> + the latch definition of the predicated-index PHI and use the same mask as
> + the paired IFN_MASK_STORE.
> +
> + # n = PHI <n_1, 0>
> + ...
> + .MASK_STORE (addr, mask, ..)
> + ...
> + n_2 = mask ? n_1 : n;
> +
> + is rewritten to:
> +
> + n_2 = IFN_PREDICATED_INDEX_UPDATE (n, mask); */
> +
> +gimple *
> +vect_recog_predicated_index_update_pattern (vec_info *vinfo,
> + stmt_vec_info stmt_info,
> + tree *type_out)
> +{
> + loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
> + /* Require a loop that already matched the predicated-index PHI pattern.
> */
> + if (!loop_vinfo
> + || !loop_vinfo->predicated_index_phi
> + || !loop_vinfo->predicated_index_mask_store)
> + return NULL;
> +
> + gphi *phi = loop_vinfo->predicated_index_phi;
> + class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
> + tree pred_phi = PHI_RESULT (phi);
> + /* STMT_INFO must be the latch-edge definition of the predicated-index PHI,
> + e.g. n_2 = mask ? n_1 : n. */
> + tree latch_val = PHI_ARG_DEF_FROM_EDGE (phi, loop_latch_edge (loop));
> + if (TREE_CODE (latch_val) != SSA_NAME
> + || SSA_NAME_DEF_STMT (latch_val) != stmt_info->stmt)
> + return NULL;
> +
> + gassign *asg = dyn_cast <gassign *> (stmt_info->stmt);
> + if (!asg || gimple_assign_rhs_code (asg) != COND_EXPR)
> + return NULL;
> +
> + /* The COND_EXPR condition must be the same mask used by the mask store.
> */
> + tree mask = gimple_assign_rhs1 (asg);
> + if (TREE_CODE (mask) != SSA_NAME)
> + return NULL;
> +
> + gcall *mask_store = loop_vinfo->predicated_index_mask_store;
> + if (gimple_call_arg (mask_store, 2) != mask)
> + return NULL;
> +
> + /* Build IFN_PREDICATED_INDEX_UPDATE (pred_phi, mask). */
> + gcall *call
> + = gimple_build_call_internal (IFN_PREDICATED_INDEX_UPDATE, 2, pred_phi,
> + mask);
> + gimple_call_set_lhs (call, gimple_assign_lhs (asg));
> + gimple_set_location (call, gimple_location (asg));
> + gimple_call_set_nothrow (call, true);
> +
> + /* pred_phi stays scalar even in the vector loop (idx + popcount (mask) per
> + vector iteration). No vector SSA is produced here, so leave type_out
> + unset for the generic pattern machinery. */
> + *type_out = NULL_TREE;
> + loop_vinfo->predicated_index_update_ifn_p = true;
> +
> + if (dump_enabled_p ())
> + dump_printf_loc (MSG_NOTE, vect_location, "%s: detected: %G",
> + "predicated_index_update", stmt_info->stmt);
> +
> + return call;
> +}
> +
> +/* Function vect_recog_compress_scatter_pattern.
> +
> + After phase-1 detection, replace the paired IFN_MASK_STORE with
> + IFN_MASK_COMPRESS_SCATTER_STORE. Base and scale were recorded during
> + phase-1 address analysis.
> +
> + # n = PHI <n_1, ..>
> + ...
> + a_1 = (long unsigned int) n;
> + a_2 = a_1 * 4;
> + a_3 = base + a_2;
> + addr = (int *) a_3;
> + .MASK_STORE (addr, mask, ..);
> +
> + is rewritten to:
> +
> + IFN_MASK_COMPRESS_SCATTER_STORE (base, align, n, 4, .., mask); */
> +
> +gimple *
> +vect_recog_compress_scatter_pattern (vec_info *vinfo,
> + stmt_vec_info stmt_info,
> + tree *type_out)
> +{
> + loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
> + /* Require a loop that already matched the predicated-index PHI pattern.
> */
> + if (!loop_vinfo
> + || !loop_vinfo->predicated_index_phi
> + || !loop_vinfo->predicated_index_mask_store)
> + return NULL;
> +
> + /* STMT_INFO must be the IFN_MASK_STORE recorded during phase-1. */
> + gcall *orig = dyn_cast <gcall *> (stmt_info->stmt);
> + if (!orig || orig != loop_vinfo->predicated_index_mask_store)
> + return NULL;
> + if (!gimple_call_internal_p (orig, IFN_MASK_STORE))
> + return NULL;
> +
> + tree pred_phi = PHI_RESULT (loop_vinfo->predicated_index_phi);
> + tree mask = gimple_call_arg (orig, 2);
> + tree value = gimple_call_arg (orig, 3);
> + tree pack_idx = pred_phi;
> + tree base = loop_vinfo->predicated_index_store_base;
> + tree scale = loop_vinfo->predicated_index_store_scale;
> + if (!base || !scale)
> + return NULL;
> +
> + tree align = gimple_call_arg (orig, 1);
> +
> + /* Build IFN_MASK_COMPRESS_SCATTER_STORE (base, align, pack_idx, scale,
> + value, mask). */
> + gcall *call
> + = gimple_build_call_internal (IFN_MASK_COMPRESS_SCATTER_STORE, 6,
> + base, align, pack_idx, scale, value, mask);
> + gimple_set_location (call, gimple_location (orig));
> + gimple_call_set_nothrow (call, true);
> +
> + tree scalar_type = TREE_TYPE (value);
> + tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
> + if (vectype == NULL_TREE)
> + return NULL;
> +
> + stmt_vec_info pattern_stmt_info = loop_vinfo->add_stmt (call);
> + loop_vinfo->move_dr (pattern_stmt_info, stmt_info);
> +
> + loop_vinfo->predicated_index_mask_compress_scatter_store = call;
> + loop_vinfo->predicated_index_compress_scatter_ifn_p = true;
> +
> + *type_out = vectype;
> + if (dump_enabled_p ())
> + dump_printf_loc (MSG_NOTE, vect_location, "%s: detected: %G",
> + "compress_scatter", stmt_info->stmt);
> + return call;
> +}
> diff --git a/gcc/tree-vect-vpcompress.h b/gcc/tree-vect-vpcompress.h
> index b8d0093831f..0df8bf95ce1 100644
> --- a/gcc/tree-vect-vpcompress.h
> +++ b/gcc/tree-vect-vpcompress.h
> @@ -24,4 +24,10 @@ along with GCC; see the file COPYING3. If not see
> extern bool vect_vpcompress_analyze_predicated_index_phi (loop_vec_info,
> class loop *, gphi *);
>
> +extern gimple *vect_recog_predicated_index_update_pattern (vec_info *,
> + stmt_vec_info,
> + tree *);
> +extern gimple *vect_recog_compress_scatter_pattern (vec_info *,
> + stmt_vec_info,
> + tree *);
> #endif /* GCC_TREE_VECT_VPCOMPRESS_H */
> diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
> index 82c79e101d7..90d9e723ed0 100644
> --- a/gcc/tree-vectorizer.h
> +++ b/gcc/tree-vectorizer.h
> @@ -1278,6 +1278,13 @@ public:
> /* Predicated-index PHI and matching IFN_MASK_STORE. */
> gphi *predicated_index_phi;
> gcall *predicated_index_mask_store;
> + tree predicated_index_store_base;
> + tree predicated_index_store_scale;
> + /* Non-NULL after compress-scatter pattern installs
> IFN_MASK_COMPRESS_SCATTER_STORE. */
> + gcall *predicated_index_mask_compress_scatter_store;
> + /* Set when pattern recog rewrites the idiom to internal functions. */
> + bool predicated_index_update_ifn_p;
> + bool predicated_index_compress_scatter_ifn_p;
> } *loop_vec_info;
>
> /* Access Functions. */
>
--
Richard Biener <[email protected]>
SUSE Software Solutions Germany GmbH,
Frankenstrasse 146, 90461 Nuernberg, Germany;
GF: Jochen Jaser, Andrew McDonald; (HRB 36809, AG Nuernberg)
