From: Soumya AR <[email protected]>
This patch implements the expand_ifn_atomic_fetch_minmax function to expand the
IFN_ATOMIC_FETCH_MINMAX internal function to target specific optabs, if
available. It also defines optabs to support atomic fetch min/max operations.
If the target does not support the optab, continue lowering to a CAS loop.
This is done by adding a new callback in atomic_op_lowering alongside the
lowering callback, which checks if backend expansion is possible for that IFN:
If a handler exists, the pass leaves the IFN alone and RTL expand picks it up
via expand_ATOMIC_FETCH_MINMAX -> expand_ifn_atomic_fetch_minmax. Otherwise
we fall back to the CAS loop as before.
----
The expander itself calls expand_atomic_fetch_op rather than the _no_fallback
variant.
This is because can_expand_atomic_minmax in pass_lower_atomic_ifn only confirms
that a backend optab exists; it does not guarantee that maybe_expand_insn's
operand predicates will accept these particular operands.
----
When handling sub-word atomic calls, signed types need special handling.
Consider the following gimple:
short int atomic_var;
short int _1;
atomic_var = -5;
_1 = .ATOMIC_FETCH_MINMAX (&atomic_var, -10, 5, MIN_EXPR, (short) 0);
When expanding lhs (_1), the RTL is generated as a SUBREG with "/s/u" flags set,
which imply that the value is a sign-extended promoted value. An atomic call
returning a non-extended value to this target will violate the contract.
Just emitting a sign extension after the atomic call is not enough since the
compiler will incorrectly assume that the value is already sign-extended, on the
basis of the previously set flags.
To maintain this contract and ensure the sign extension is not optimized away,
we use a temporary non-promoted register for the atomic operation, explicitly
perform the extension, then move to the original promoted target.
----
Bootstrapped and regression tested on aarch64-linux-gnu and x86_64-linux-gnu,
no regression. Cross-compiled and regression tested under qemu for
arm-linux-gnueabihf-armv7-a and aarch64-linux-gnu without LSE.
OK for mainline?
Signed-off-by: Soumya AR <[email protected]>
gcc/ChangeLog:
* atomic-ifn-lowering.cc (struct atomic_op_lowering): Add
can_expand_directly callback.
(can_expand_atomic_minmax): Check whether the target supports
atomic min/max for this datatype.
(lower_atomic_ifn): Honour can_expand_directly: skip lowering
when the IFN can be expanded via an optab.
* builtins.cc (expand_ifn_atomic_fetch_minmax): New.
* builtins.h (expand_ifn_atomic_fetch_minmax): Declare.
* internal-fn.cc (expand_ATOMIC_FETCH_MINMAX): Dispatch to
expand_ifn_atomic_fetch_minmax.
* optabs.cc (get_atomic_op_for_code): Handle SMIN, SMAX, UMIN,
UMAX.
(maybe_emit_op): Likewise.
* optabs.def (OPTAB_NC, OPTAB_D): Add atomic_fetch_{s,u}{min,max}
and atomic_{s,u}{min,max} entries.
* optabs.h (expand_atomic_fetch_op_no_fallback): Export.
---
gcc/atomic-ifn-lowering.cc | 60 +++++++++++++++++++++++++---
gcc/builtins.cc | 82 ++++++++++++++++++++++++++++++++++++++
gcc/builtins.h | 1 +
gcc/internal-fn.cc | 4 +-
gcc/optabs.cc | 38 +++++++++++++++++-
gcc/optabs.def | 24 +++++++++++
gcc/optabs.h | 2 +
7 files changed, 202 insertions(+), 9 deletions(-)
diff --git a/gcc/atomic-ifn-lowering.cc b/gcc/atomic-ifn-lowering.cc
index 77129a1ca7e..ae689dfdae7 100644
--- a/gcc/atomic-ifn-lowering.cc
+++ b/gcc/atomic-ifn-lowering.cc
@@ -50,6 +50,7 @@ After lowering the CFG looks somewhat like:
#include "coretypes.h"
#include "backend.h"
#include "target.h"
+#include "rtl.h"
#include "tree.h"
#include "gimple.h"
#include "gimple-iterator.h"
@@ -59,19 +60,21 @@ After lowering the CFG looks somewhat like:
#include "gimple-pretty-print.h"
#include "memmodel.h"
#include "ssa.h"
+#include "optabs.h"
#include "optabs-query.h"
#include "fold-const.h"
#include "builtins.h"
#include "internal-fn.h"
/* Descriptor for a single atomic-fetch IFN this pass knows how to lower.
- LOWER is called to replace the IFN call: it must remove the call from
- the gimple, typically by emitting a CAS loop via lower_via_cas_loop
- with IFN-specific callbacks. */
+ CAN_EXPAND_DIRECTLY returns true when the backend can expand the IFN via
+ a direct optab, in which case the pass leaves the call alone for the RTL
+ expander. LOWER is called to replace the IFN call with a CAS loop. */
struct atomic_op_lowering
{
internal_fn ifn;
+ bool (*can_expand_directly) (gcall *);
void (*lower) (gcall *);
};
@@ -334,6 +337,48 @@ emit_minmax_lhs_value (tree datatype, tree current_value,
NOP_EXPR, false);
}
+/* Return true when the backend can expand an IFN_ATOMIC_FETCH_MINMAX call
+ directly via a target optab. */
+
+static bool
+can_expand_atomic_minmax (gcall *call)
+{
+ if (!flag_inline_atomics)
+ return false;
+
+ tree datatype = TREE_TYPE (gimple_call_arg (call, 4));
+ /* Arg 3 is the operation tree_code (MIN_EXPR or MAX_EXPR). */
+ tree_code cmp_code
+ = (tree_code) tree_to_uhwi (gimple_call_arg (call, 3));
+ gcc_assert (cmp_code == MIN_EXPR || cmp_code == MAX_EXPR);
+ bool is_min = (cmp_code == MIN_EXPR);
+ bool is_signed = !TYPE_UNSIGNED (datatype);
+ bool unused_result = (gimple_call_lhs (call) == NULL_TREE);
+ machine_mode mode = TYPE_MODE (datatype);
+
+ optab fetch_op_optab
+ = is_signed ? (is_min ? atomic_fetch_smin_optab : atomic_fetch_smax_optab)
+ : (is_min ? atomic_fetch_umin_optab : atomic_fetch_umax_optab);
+
+ if (optab_handler (fetch_op_optab, mode) != CODE_FOR_nothing)
+ return true;
+
+ /* If the result is unused, the no-result optab is also sufficient.
+ MIN/MAX does not support op_fetch in the frontend, so no need to check
+ for those optabs. Even if they were supported by the backend, they
+ would not be helpful here. */
+ if (unused_result)
+ {
+ optab no_result_optab
+ = is_signed ? (is_min ? atomic_smin_optab : atomic_smax_optab)
+ : (is_min ? atomic_umin_optab : atomic_umax_optab);
+ if (direct_optab_handler (no_result_optab, mode) != CODE_FOR_nothing)
+ return true;
+ }
+
+ return false;
+}
+
/* Lower an IFN_ATOMIC_FETCH_MINMAX call to a CAS loop. */
static void
@@ -342,10 +387,11 @@ lower_minmax_call (gcall *call)
lower_via_cas_loop (call, emit_minmax_desired_value, emit_minmax_lhs_value);
}
-/* Registry of atomic-fetch IFNs this pass lowers. */
+/* Registry of atomic-fetch IFNs this pass lowers when the target lacks a
+ direct optab. */
static const atomic_op_lowering atomic_op_table[] = {
- { IFN_ATOMIC_FETCH_MINMAX, lower_minmax_call },
+ {IFN_ATOMIC_FETCH_MINMAX, can_expand_atomic_minmax, lower_minmax_call},
};
static unsigned int
@@ -368,7 +414,9 @@ lower_atomic_ifn (void)
for (const atomic_op_lowering &op : atomic_op_table)
if (gimple_call_internal_p (call, op.ifn))
{
- to_lower.safe_push ({ call, &op });
+ if (op.can_expand_directly && op.can_expand_directly (call))
+ break;
+ to_lower.safe_push ({call, &op});
break;
}
}
diff --git a/gcc/builtins.cc b/gcc/builtins.cc
index 52f3ee029d0..8574abe826a 100644
--- a/gcc/builtins.cc
+++ b/gcc/builtins.cc
@@ -6770,6 +6770,88 @@ expand_builtin_atomic_compare_exchange (machine_mode
mode, tree exp,
return target;
}
+/* Expand IFN_ATOMIC_FETCH_MINMAX internal function. */
+
+void
+expand_ifn_atomic_fetch_minmax (gcall *call)
+{
+ tree ptr = gimple_call_arg (call, 0);
+ tree value = gimple_call_arg (call, 1);
+ tree memorder_tree = gimple_call_arg (call, 2);
+ tree_code cmp_code
+ = (tree_code) tree_to_uhwi (gimple_call_arg (call, 3));
+ gcc_assert (cmp_code == MIN_EXPR || cmp_code == MAX_EXPR);
+ tree datatype = TREE_TYPE (gimple_call_arg (call, 4));
+
+ bool is_min = (cmp_code == MIN_EXPR);
+ bool is_signed = !TYPE_UNSIGNED (datatype);
+
+ memmodel model = get_memmodel (memorder_tree);
+ tree lhs = gimple_call_lhs (call);
+
+ enum rtx_code code;
+ if (is_signed)
+ code = is_min ? SMIN : SMAX;
+ else
+ code = is_min ? UMIN : UMAX;
+
+ machine_mode mode = TYPE_MODE (datatype);
+
+ rtx mem = get_builtin_sync_mem (ptr, mode);
+
+ rtx val = expand_expr_force_mode (value, mode);
+
+ rtx target;
+ if (lhs)
+ target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
+ else
+ target = gen_reg_rtx (mode);
+
+ /* For sub-word signed operations, we need to handle sign extension
+ carefully. The issue is that atomic libcalls may not return properly
+ sign-extended values, but the LHS might be a promoted variable that
+ has been marked as a sign-extended register. A forced sign extension
+ after the atomic call might get optimized away if the target has been
+ marked as sign-extended. */
+ rtx temp_target = target;
+ if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target)
+ && GET_MODE_SIZE (mode).to_constant () < UNITS_PER_WORD)
+ {
+ /* Use a temporary non-promoted register for the atomic operation
+ to avoid incorrect flags for the target register. */
+ temp_target = gen_reg_rtx (mode);
+ }
+
+ /* Even though we are certain at this stage that we have a backend optab
+ for fetch min/max (otherwise pass_lower_atomic_ifn would have lowered
+ this to CAS), still don't use the _no_fallback version since checking
+ for the optab is not enough to guarantee its emission. */
+ rtx result = expand_atomic_fetch_op (temp_target, mem, val, code,
+ model, false);
+
+ if (!result)
+ gcc_unreachable ();
+
+ /* If we used a temporary target due to promotion, handle the conversion
+ properly. */
+ if (temp_target != target)
+ {
+ /* For signed types, explicitly sign-extend the result. */
+ if (is_signed)
+ {
+ rtx extended = gen_reg_rtx (word_mode);
+ emit_insn (
+ gen_rtx_SET (extended, gen_rtx_SIGN_EXTEND (word_mode, result)));
+ result = gen_lowpart (mode, extended);
+ }
+
+ /* Now store to the actual target. */
+ convert_move (SUBREG_REG (target), result, !is_signed);
+ }
+ else if (result != target)
+ emit_move_insn (target, result);
+}
+
/* Helper function for expand_ifn_atomic_compare_exchange - expand
internal ATOMIC_COMPARE_EXCHANGE call into __atomic_compare_exchange_N
call. The weak parameter must be dropped to match the expected parameter
diff --git a/gcc/builtins.h b/gcc/builtins.h
index 9cca88c647c..627f4087dea 100644
--- a/gcc/builtins.h
+++ b/gcc/builtins.h
@@ -133,6 +133,7 @@ extern void expand_builtin_trap (void);
extern void expand_ifn_atomic_bit_test_and (gcall *);
extern void expand_ifn_atomic_compare_exchange (gcall *);
extern void expand_ifn_atomic_op_fetch_cmp_0 (gcall *);
+extern void expand_ifn_atomic_fetch_minmax (gcall *);
extern rtx expand_builtin_crc_table_based (internal_fn, scalar_mode,
scalar_mode, machine_mode,
tree, rtx);
diff --git a/gcc/internal-fn.cc b/gcc/internal-fn.cc
index 692391f67dc..9611d5f922a 100644
--- a/gcc/internal-fn.cc
+++ b/gcc/internal-fn.cc
@@ -3765,9 +3765,9 @@ expand_ATOMIC_XOR_FETCH_CMP_0 (internal_fn, gcall *call)
/* Expand atomic fetch minmax. */
static void
-expand_ATOMIC_FETCH_MINMAX (internal_fn, gcall *)
+expand_ATOMIC_FETCH_MINMAX (internal_fn, gcall *call)
{
- /* Implement this. */
+ expand_ifn_atomic_fetch_minmax (call);
}
/* Expand LAUNDER to assignment, lhs = arg0. */
diff --git a/gcc/optabs.cc b/gcc/optabs.cc
index 57fc9d3306e..828f6bb1089 100644
--- a/gcc/optabs.cc
+++ b/gcc/optabs.cc
@@ -7977,6 +7977,42 @@ get_atomic_op_for_code (struct atomic_op_functions *op,
enum rtx_code code)
op->no_result = sync_nand_optab;
op->reverse_code = UNKNOWN;
break;
+ case SMIN:
+ op->mem_fetch_before = atomic_fetch_smin_optab;
+ op->mem_fetch_after = atomic_smin_fetch_optab;
+ op->mem_no_result = atomic_smin_optab;
+ op->fetch_before = sync_old_smin_optab;
+ op->fetch_after = sync_new_smin_optab;
+ op->no_result = sync_smin_optab;
+ op->reverse_code = UNKNOWN;
+ break;
+ case SMAX:
+ op->mem_fetch_before = atomic_fetch_smax_optab;
+ op->mem_fetch_after = atomic_smax_fetch_optab;
+ op->mem_no_result = atomic_smax_optab;
+ op->fetch_before = sync_old_smax_optab;
+ op->fetch_after = sync_new_smax_optab;
+ op->no_result = sync_smax_optab;
+ op->reverse_code = UNKNOWN;
+ break;
+ case UMIN:
+ op->mem_fetch_before = atomic_fetch_umin_optab;
+ op->mem_fetch_after = atomic_umin_fetch_optab;
+ op->mem_no_result = atomic_umin_optab;
+ op->fetch_before = sync_old_umin_optab;
+ op->fetch_after = sync_new_umin_optab;
+ op->no_result = sync_umin_optab;
+ op->reverse_code = UNKNOWN;
+ break;
+ case UMAX:
+ op->mem_fetch_before = atomic_fetch_umax_optab;
+ op->mem_fetch_after = atomic_umax_fetch_optab;
+ op->mem_no_result = atomic_umax_optab;
+ op->fetch_before = sync_old_umax_optab;
+ op->fetch_after = sync_new_umax_optab;
+ op->no_result = sync_umax_optab;
+ op->reverse_code = UNKNOWN;
+ break;
default:
gcc_unreachable ();
}
@@ -8095,7 +8131,7 @@ maybe_emit_op (const struct atomic_op_functions *optab,
rtx target, rtx mem,
This function will *only* generate instructions if there is a direct
optab. No compare and swap loops or libcalls will be generated. */
-static rtx
+rtx
expand_atomic_fetch_op_no_fallback (rtx target, rtx mem, rtx val,
enum rtx_code code, enum memmodel model,
bool after)
diff --git a/gcc/optabs.def b/gcc/optabs.def
index 7ccea18543f..9d0eb7dd4ca 100644
--- a/gcc/optabs.def
+++ b/gcc/optabs.def
@@ -215,12 +215,20 @@ OPTAB_NC(sync_old_ior_optab, "sync_old_ior$I$a", UNKNOWN)
OPTAB_NC(sync_old_and_optab, "sync_old_and$I$a", UNKNOWN)
OPTAB_NC(sync_old_xor_optab, "sync_old_xor$I$a", UNKNOWN)
OPTAB_NC(sync_old_nand_optab, "sync_old_nand$I$a", UNKNOWN)
+OPTAB_NC(sync_old_smin_optab, "sync_old_smin$I$a", UNKNOWN)
+OPTAB_NC(sync_old_smax_optab, "sync_old_smax$I$a", UNKNOWN)
+OPTAB_NC(sync_old_umin_optab, "sync_old_umin$I$a", UNKNOWN)
+OPTAB_NC(sync_old_umax_optab, "sync_old_umax$I$a", UNKNOWN)
OPTAB_NC(sync_new_add_optab, "sync_new_add$I$a", UNKNOWN)
OPTAB_NC(sync_new_sub_optab, "sync_new_sub$I$a", UNKNOWN)
OPTAB_NC(sync_new_ior_optab, "sync_new_ior$I$a", UNKNOWN)
OPTAB_NC(sync_new_and_optab, "sync_new_and$I$a", UNKNOWN)
OPTAB_NC(sync_new_xor_optab, "sync_new_xor$I$a", UNKNOWN)
OPTAB_NC(sync_new_nand_optab, "sync_new_nand$I$a", UNKNOWN)
+OPTAB_NC(sync_new_smin_optab, "sync_new_smin$I$a", UNKNOWN)
+OPTAB_NC(sync_new_smax_optab, "sync_new_smax$I$a", UNKNOWN)
+OPTAB_NC(sync_new_umin_optab, "sync_new_umin$I$a", UNKNOWN)
+OPTAB_NC(sync_new_umax_optab, "sync_new_umax$I$a", UNKNOWN)
OPTAB_NC(sync_compare_and_swap_optab, "sync_compare_and_swap$I$a", UNKNOWN)
OPTAB_NC(sync_lock_test_and_set_optab, "sync_lock_test_and_set$I$a", UNKNOWN)
@@ -524,6 +532,10 @@ OPTAB_D (sync_and_optab, "sync_and$I$a")
OPTAB_D (sync_ior_optab, "sync_ior$I$a")
OPTAB_D (sync_lock_release_optab, "sync_lock_release$I$a")
OPTAB_D (sync_nand_optab, "sync_nand$I$a")
+OPTAB_D (sync_smin_optab, "sync_smin$I$a")
+OPTAB_D (sync_smax_optab, "sync_smax$I$a")
+OPTAB_D (sync_umin_optab, "sync_umin$I$a")
+OPTAB_D (sync_umax_optab, "sync_umax$I$a")
OPTAB_D (sync_sub_optab, "sync_sub$I$a")
OPTAB_D (sync_xor_optab, "sync_xor$I$a")
@@ -542,6 +554,18 @@ OPTAB_D (atomic_fetch_nand_optab, "atomic_fetch_nand$I$a")
OPTAB_D (atomic_fetch_or_optab, "atomic_fetch_or$I$a")
OPTAB_D (atomic_fetch_sub_optab, "atomic_fetch_sub$I$a")
OPTAB_D (atomic_fetch_xor_optab, "atomic_fetch_xor$I$a")
+OPTAB_D (atomic_fetch_smax_optab, "atomic_fetch_smax$I$a")
+OPTAB_D (atomic_fetch_smin_optab, "atomic_fetch_smin$I$a")
+OPTAB_D (atomic_smax_fetch_optab, "atomic_smax_fetch$I$a")
+OPTAB_D (atomic_smin_fetch_optab, "atomic_smin_fetch$I$a")
+OPTAB_D (atomic_smax_optab, "atomic_smax$I$a")
+OPTAB_D (atomic_smin_optab, "atomic_smin$I$a")
+OPTAB_D (atomic_fetch_umax_optab, "atomic_fetch_umax$I$a")
+OPTAB_D (atomic_fetch_umin_optab, "atomic_fetch_umin$I$a")
+OPTAB_D (atomic_umax_fetch_optab, "atomic_umax_fetch$I$a")
+OPTAB_D (atomic_umin_fetch_optab, "atomic_umin_fetch$I$a")
+OPTAB_D (atomic_umax_optab, "atomic_umax$I$a")
+OPTAB_D (atomic_umin_optab, "atomic_umin$I$a")
OPTAB_D (atomic_load_optab, "atomic_load$I$a")
OPTAB_D (atomic_nand_fetch_optab, "atomic_nand_fetch$I$a")
OPTAB_D (atomic_nand_optab, "atomic_nand$I$a")
diff --git a/gcc/optabs.h b/gcc/optabs.h
index 81f58278887..8647fffaca1 100644
--- a/gcc/optabs.h
+++ b/gcc/optabs.h
@@ -365,6 +365,8 @@ rtx expand_atomic_load (rtx, rtx, enum memmodel);
rtx expand_atomic_store (rtx, rtx, enum memmodel, bool);
rtx expand_atomic_fetch_op (rtx, rtx, rtx, enum rtx_code, enum memmodel,
bool);
+rtx expand_atomic_fetch_op_no_fallback (rtx, rtx, rtx, enum rtx_code,
+ enum memmodel, bool);
extern void expand_asm_reg_clobber_mem_blockage (HARD_REG_SET);
--
2.43.0
