This patch adds else-operand handling to the internal functions.
gcc/ChangeLog:
* internal-fn.cc (add_mask_and_len_args): Rename...
(add_mask_else_and_len_args): ...to this and add else handling.
(expand_partial_load_optab_fn): Use adjusted function.
(expand_partial_store_optab_fn): Ditto.
(expand_scatter_store_optab_fn): Ditto.
(expand_gather_load_optab_fn): Ditto.
(internal_fn_len_index): Adjust for masked loads.
(internal_fn_else_index): Add masked loads.
---
gcc/internal-fn.cc | 69 ++++++++++++++++++++++++++++++++++++++--------
1 file changed, 58 insertions(+), 11 deletions(-)
diff --git a/gcc/internal-fn.cc b/gcc/internal-fn.cc
index 8a2e07f2f96..586978e8f3f 100644
--- a/gcc/internal-fn.cc
+++ b/gcc/internal-fn.cc
@@ -331,17 +331,18 @@ get_multi_vector_move (tree array_type, convert_optab
optab)
return convert_optab_handler (optab, imode, vmode);
}
-/* Add mask and len arguments according to the STMT. */
+/* Add mask, else, and len arguments according to the STMT. */
static unsigned int
-add_mask_and_len_args (expand_operand *ops, unsigned int opno, gcall *stmt)
+add_mask_else_and_len_args (expand_operand *ops, unsigned int opno, gcall
*stmt)
{
internal_fn ifn = gimple_call_internal_fn (stmt);
int len_index = internal_fn_len_index (ifn);
/* BIAS is always consecutive next of LEN. */
int bias_index = len_index + 1;
int mask_index = internal_fn_mask_index (ifn);
- /* The order of arguments are always {len,bias,mask}. */
+
+ /* The order of arguments is always {mask, else, len, bias}. */
if (mask_index >= 0)
{
tree mask = gimple_call_arg (stmt, mask_index);
@@ -362,6 +363,23 @@ add_mask_and_len_args (expand_operand *ops, unsigned int
opno, gcall *stmt)
create_input_operand (&ops[opno++], mask_rtx,
TYPE_MODE (TREE_TYPE (mask)));
+
+ }
+
+ int els_index = internal_fn_else_index (ifn);
+ if (els_index >= 0)
+ {
+ tree els = gimple_call_arg (stmt, els_index);
+ tree els_type = TREE_TYPE (els);
+ if (TREE_CODE (els) == SSA_NAME
+ && SSA_NAME_IS_DEFAULT_DEF (els)
+ && VAR_P (SSA_NAME_VAR (els)))
+ create_undefined_input_operand (&ops[opno++], TYPE_MODE (els_type));
+ else
+ {
+ rtx els_rtx = expand_normal (els);
+ create_input_operand (&ops[opno++], els_rtx, TYPE_MODE (els_type));
+ }
}
if (len_index >= 0)
{
@@ -3014,7 +3032,7 @@ static void
expand_partial_load_optab_fn (internal_fn ifn, gcall *stmt, convert_optab
optab)
{
int i = 0;
- class expand_operand ops[5];
+ class expand_operand ops[6];
tree type, lhs, rhs, maskt;
rtx mem, target;
insn_code icode;
@@ -3044,7 +3062,7 @@ expand_partial_load_optab_fn (internal_fn ifn, gcall
*stmt, convert_optab optab)
target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
create_call_lhs_operand (&ops[i++], target, TYPE_MODE (type));
create_fixed_operand (&ops[i++], mem);
- i = add_mask_and_len_args (ops, i, stmt);
+ i = add_mask_else_and_len_args (ops, i, stmt);
expand_insn (icode, i, ops);
assign_call_lhs (lhs, target, &ops[0]);
@@ -3090,7 +3108,7 @@ expand_partial_store_optab_fn (internal_fn ifn, gcall
*stmt, convert_optab optab
reg = expand_normal (rhs);
create_fixed_operand (&ops[i++], mem);
create_input_operand (&ops[i++], reg, TYPE_MODE (type));
- i = add_mask_and_len_args (ops, i, stmt);
+ i = add_mask_else_and_len_args (ops, i, stmt);
expand_insn (icode, i, ops);
}
@@ -3676,7 +3694,7 @@ expand_scatter_store_optab_fn (internal_fn, gcall *stmt,
direct_optab optab)
create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
create_integer_operand (&ops[i++], scale_int);
create_input_operand (&ops[i++], rhs_rtx, TYPE_MODE (TREE_TYPE (rhs)));
- i = add_mask_and_len_args (ops, i, stmt);
+ i = add_mask_else_and_len_args (ops, i, stmt);
insn_code icode = convert_optab_handler (optab, TYPE_MODE (TREE_TYPE (rhs)),
TYPE_MODE (TREE_TYPE (offset)));
@@ -3705,7 +3723,7 @@ expand_gather_load_optab_fn (internal_fn, gcall *stmt,
direct_optab optab)
create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
create_integer_operand (&ops[i++], scale_int);
- i = add_mask_and_len_args (ops, i, stmt);
+ i = add_mask_else_and_len_args (ops, i, stmt);
insn_code icode = convert_optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs)),
TYPE_MODE (TREE_TYPE (offset)));
expand_insn (icode, i, ops);
@@ -4590,6 +4608,18 @@ get_len_internal_fn (internal_fn fn)
case IFN_COND_##NAME:
\
return IFN_COND_LEN_##NAME;
#include "internal-fn.def"
+ default:
+ break;
+ }
+
+ switch (fn)
+ {
+ case IFN_MASK_LOAD:
+ return IFN_MASK_LEN_LOAD;
+ case IFN_MASK_LOAD_LANES:
+ return IFN_MASK_LEN_LOAD_LANES;
+ case IFN_MASK_GATHER_LOAD:
+ return IFN_MASK_LEN_GATHER_LOAD;
default:
return IFN_LAST;
}
@@ -4775,8 +4805,12 @@ internal_fn_len_index (internal_fn fn)
case IFN_LEN_STORE:
return 2;
- case IFN_MASK_LEN_GATHER_LOAD:
case IFN_MASK_LEN_SCATTER_STORE:
+ return 5;
+
+ case IFN_MASK_LEN_GATHER_LOAD:
+ return 6;
+
case IFN_COND_LEN_FMA:
case IFN_COND_LEN_FMS:
case IFN_COND_LEN_FNMA:
@@ -4801,13 +4835,15 @@ internal_fn_len_index (internal_fn fn)
return 4;
case IFN_COND_LEN_NEG:
- case IFN_MASK_LEN_LOAD:
case IFN_MASK_LEN_STORE:
- case IFN_MASK_LEN_LOAD_LANES:
case IFN_MASK_LEN_STORE_LANES:
case IFN_VCOND_MASK_LEN:
return 3;
+ case IFN_MASK_LEN_LOAD:
+ case IFN_MASK_LEN_LOAD_LANES:
+ return 4;
+
default:
return -1;
}
@@ -4857,6 +4893,12 @@ internal_fn_else_index (internal_fn fn)
case IFN_COND_LEN_SHR:
return 3;
+ case IFN_MASK_LOAD:
+ case IFN_MASK_LEN_LOAD:
+ case IFN_MASK_LOAD_LANES:
+ case IFN_MASK_LEN_LOAD_LANES:
+ return 3;
+
case IFN_COND_FMA:
case IFN_COND_FMS:
case IFN_COND_FNMA:
@@ -4867,6 +4909,10 @@ internal_fn_else_index (internal_fn fn)
case IFN_COND_LEN_FNMS:
return 4;
+ case IFN_MASK_GATHER_LOAD:
+ case IFN_MASK_LEN_GATHER_LOAD:
+ return 5;
+
default:
return -1;
}
@@ -4898,6 +4944,7 @@ internal_fn_mask_index (internal_fn fn)
case IFN_MASK_LEN_SCATTER_STORE:
return 4;
+ case IFN_VCOND_MASK:
case IFN_VCOND_MASK_LEN:
return 0;
--
2.45.2
