Some targets (riscv, arm) specify in their psABI that the padding bits of
a _BitInt(N) value (the bits above N up to the container width) are sign or
zero extended, and advertise this via bitint_info::extended == bitint_ext_full
(loongarch uses bitint_ext_partial). The middle-end relies on that when it
loads a _BitInt from an ABI-exposed location: it does not re-extend the value
(see the EXTEND_BITINT macro in expr.cc), trusting that whoever wrote the
value already extended the padding bits.
For an atomic (or legacy __sync) read-modify-write whose precision is narrower
than the access mode this trust is broken. A single mode-width atomic, e.g.
unsigned _BitInt(17) b;
__atomic_add_fetch (&b, x, 0);
is expanded on riscv to "amoadd.w", which stores the full 32-bit sum; the
carry out of bit 16 lands in the padding bits, so a subsequent non-extending
load reads garbage and miscompiles e.g. "b == 0x10".
A single atomic instruction cannot keep the padding bits extended, so for such
a _BitInt expand the operation as a compare-and-swap loop that re-extends the
new value to the _BitInt precision (zero extend for unsigned, sign extend for
signed) before storing it. This keeps the padding-bit invariant required by
the ABI and produces correct results for add/sub/nand (and is a no-op for
and/or/xor, which already preserve clean padding). Full-width _BitInt and
ordinary integer types are unaffected and keep using the direct atomic insn.
expand_builtin_atomic_fetch_op recovers the precision and signedness from the
pointee type of the address argument and passes them to a new
expand_atomic_fetch_op_prec entry point, which forces the compare-and-swap
loop and re-extends the new value with reduce_rtx_to_bit_field_precision, the
same reduction the middle-end applies after ordinary _BitInt arithmetic.
Neither a mode-width atomic nor a library call preserves that extension, so
such a _BitInt is expanded inline even with -fno-inline-atomics.
gcc/ChangeLog:
PR target/124948
* builtins.cc (get_bitint_atomic_extend_prec): New function.
(expand_builtin_sync_operation): Use expand_atomic_fetch_op_prec and
pass the _BitInt extension precision and signedness.
(expand_builtin_atomic_fetch_op): Likewise, and expand inline even
with -fno-inline-atomics when the extension is required.
* optabs.h (expand_atomic_fetch_op_prec): Declare.
* optabs.cc (expand_atomic_fetch_op_prec): New function, renamed from
expand_atomic_fetch_op with added PREC and UNSIGNEDP parameters. When
PREC is nonzero, skip the direct optab, reverse and libcall paths and
use a compare-and-swap loop that re-extends the new value to PREC bits
with reduce_rtx_to_bit_field_precision before storing it.
(expand_atomic_fetch_op): Reimplement as a wrapper passing PREC 0.
* expr.cc (reduce_rtx_to_bit_field_precision): New function, split out
of...
(reduce_to_bit_field_precision): ...here. Call it.
* expr.h (reduce_rtx_to_bit_field_precision): Declare.
gcc/testsuite/ChangeLog:
PR target/124948
* gcc.dg/torture/bitint-100.c: New test.
---
gcc/builtins.cc | 55 ++++++++++++++++++++---
gcc/expr.cc | 36 ++++++++++-----
gcc/expr.h | 6 +++
gcc/optabs.cc | 48 ++++++++++++++++----
gcc/optabs.h | 2 +
gcc/testsuite/gcc.dg/torture/bitint-100.c | 47 +++++++++++++++++++
6 files changed, 168 insertions(+), 26 deletions(-)
create mode 100644 gcc/testsuite/gcc.dg/torture/bitint-100.c
diff --git a/gcc/builtins.cc b/gcc/builtins.cc
index 52dd51bad55..466ef78f4fd 100644
--- a/gcc/builtins.cc
+++ b/gcc/builtins.cc
@@ -6520,6 +6520,37 @@ expand_expr_force_mode (tree exp, machine_mode mode)
return val;
}
+/* ADDR is the address argument of an atomic or __sync read-modify-write
+ builtin operating in MODE. If it points to a _BitInt object whose precision
+ is narrower than MODE and the target requires the padding bits to stay
+ sign/zero extended (bitint_info::extended is not bitint_ext_undef), return
+ that precision and set *UNSIGNEDP accordingly; otherwise return 0. A plain
+ mode-width atomic operation would compute the result over the whole mode and
+ leave those padding bits corrupted, so the caller has to keep them extended
+ (see expand_atomic_fetch_op_prec). */
+
+static unsigned int
+get_bitint_atomic_extend_prec (tree addr, machine_mode mode, bool *unsignedp)
+{
+ STRIP_NOPS (addr);
+ scalar_int_mode imode;
+ if (!POINTER_TYPE_P (TREE_TYPE (addr))
+ || TREE_CODE (TREE_TYPE (TREE_TYPE (addr))) != BITINT_TYPE
+ || !is_a <scalar_int_mode> (mode, &imode))
+ return 0;
+
+ tree type = TREE_TYPE (TREE_TYPE (addr));
+ struct bitint_info info;
+ if (TYPE_PRECISION (type) < GET_MODE_PRECISION (imode)
+ && targetm.c.bitint_type_info (TYPE_PRECISION (type), &info)
+ && info.extended != bitint_ext_undef)
+ {
+ *unsignedp = TYPE_UNSIGNED (type);
+ return TYPE_PRECISION (type);
+ }
+ return 0;
+}
+
/* Expand the __sync_xxx_and_fetch and __sync_fetch_and_xxx intrinsics.
EXP is the CALL_EXPR. CODE is the rtx code
@@ -6580,8 +6611,13 @@ expand_builtin_sync_operation (machine_mode mode, tree
exp,
mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 1), mode);
- return expand_atomic_fetch_op (target, mem, val, code, MEMMODEL_SYNC_SEQ_CST,
- after);
+ bool unsignedp = false;
+ unsigned int prec
+ = get_bitint_atomic_extend_prec (CALL_EXPR_ARG (exp, 0), mode, &unsignedp);
+
+ return expand_atomic_fetch_op_prec (target, mem, val, code,
+ MEMMODEL_SYNC_SEQ_CST, after, prec,
+ unsignedp);
}
/* Expand the __sync_val_compare_and_swap and __sync_bool_compare_and_swap
@@ -6952,10 +6988,19 @@ expand_builtin_atomic_fetch_op (machine_mode mode, tree
exp, rtx target,
mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 1), mode);
- /* Only try generating instructions if inlining is turned on. */
- if (flag_inline_atomics)
+ /* PREC is the _BitInt precision to re-extend the result to, or 0. It is
+ nonzero only on targets that require a sub-precision _BitInt's padding
bits
+ to stay sign/zero extended. */
+ bool unsignedp = false;
+ unsigned int prec
+ = get_bitint_atomic_extend_prec (CALL_EXPR_ARG (exp, 0), mode, &unsignedp);
+
+ /* Only try generating instructions if inlining is turned on or handling
+ _BitInt with sign/zero-extend. */
+ if (flag_inline_atomics || prec)
{
- ret = expand_atomic_fetch_op (target, mem, val, code, model,
fetch_after);
+ ret = expand_atomic_fetch_op_prec (target, mem, val, code, model,
+ fetch_after, prec, unsignedp);
if (ret)
return ret;
}
diff --git a/gcc/expr.cc b/gcc/expr.cc
index f42aab21272..c9f8f0fd727 100644
--- a/gcc/expr.cc
+++ b/gcc/expr.cc
@@ -12994,6 +12994,28 @@ expand_expr_real_1 (tree exp, rtx target, machine_mode
tmode,
}
#undef EXTEND_BITINT
+/* Reduce EXP, an integer value in MODE, to PREC bits, sign- or zero-extending
+ (per UNSIGNEDP) the bits at or above PREC so the whole mode holds the value
+ extended as a PREC-bit type; possibly return the result in TARGET. PREC is
+ narrower than the precision of MODE. */
+rtx
+reduce_rtx_to_bit_field_precision (rtx exp, rtx target, scalar_int_mode mode,
+ unsigned int prec, bool unsignedp)
+{
+ if (unsignedp)
+ {
+ rtx mask = immed_wide_int_const
+ (wi::mask (prec, false, GET_MODE_PRECISION (mode)), mode);
+ return expand_and (mode, exp, mask, target);
+ }
+ else
+ {
+ int count = GET_MODE_PRECISION (mode) - prec;
+ exp = expand_shift (LSHIFT_EXPR, mode, exp, count, target, 0);
+ return expand_shift (RSHIFT_EXPR, mode, exp, count, target, 0);
+ }
+}
+
/* Subroutine of above: reduce EXP to the precision of TYPE (in the
signedness of TYPE), possibly returning the result in TARGET.
TYPE is known to be a partial integer type. */
@@ -13012,18 +13034,8 @@ reduce_to_bit_field_precision (rtx exp, rtx target,
tree type)
tree t = wide_int_to_tree (type, value);
return expand_expr (t, target, VOIDmode, EXPAND_NORMAL);
}
- else if (TYPE_UNSIGNED (type))
- {
- rtx mask = immed_wide_int_const
- (wi::mask (prec, false, GET_MODE_PRECISION (mode)), mode);
- return expand_and (mode, exp, mask, target);
- }
- else
- {
- int count = GET_MODE_PRECISION (mode) - prec;
- exp = expand_shift (LSHIFT_EXPR, mode, exp, count, target, 0);
- return expand_shift (RSHIFT_EXPR, mode, exp, count, target, 0);
- }
+ return reduce_rtx_to_bit_field_precision (exp, target, mode, prec,
+ TYPE_UNSIGNED (type));
}
/* Subroutine of above: returns true if OFFSET corresponds to an offset that
diff --git a/gcc/expr.h b/gcc/expr.h
index a54984ea2f8..1ced4105902 100644
--- a/gcc/expr.h
+++ b/gcc/expr.h
@@ -313,6 +313,12 @@ extern rtx expand_expr_real_gassign (gassign *, rtx,
machine_mode,
enum expand_modifier modifier,
rtx * = nullptr, bool = false);
+/* Sign- or zero-extend (per UNSIGNEDP) integer value EXP in MODE from a
+ sub-mode precision PREC, so the whole mode holds it extended as a PREC-bit
+ type; optionally return the result in TARGET. */
+extern rtx reduce_rtx_to_bit_field_precision (rtx, rtx, scalar_int_mode,
+ unsigned int, bool);
+
/* Generate code for computing expression EXP.
An rtx for the computed value is returned. The value is never null.
In the case of a void EXP, const0_rtx is returned. */
diff --git a/gcc/optabs.cc b/gcc/optabs.cc
index 57fc9d3306e..83f2c2e0e4c 100644
--- a/gcc/optabs.cc
+++ b/gcc/optabs.cc
@@ -8184,29 +8184,43 @@ expand_atomic_fetch_op_no_fallback (rtx target, rtx
mem, rtx val,
CODE is the operation being performed (OP)
MEMMODEL is the memory model variant to use.
AFTER is true to return the result of the operation (OP_fetch).
- AFTER is false to return the value before the operation (fetch_OP). */
+ AFTER is false to return the value before the operation (fetch_OP).
+ PREC, if nonzero, is the precision of a _BitInt MEM whose padding bits
+ (above PREC) must stay sign/zero extended in memory; UNSIGNEDP says whether
+ to zero (true) or sign (false) extend them. */
rtx
-expand_atomic_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
- enum memmodel model, bool after)
+expand_atomic_fetch_op_prec (rtx target, rtx mem, rtx val, enum rtx_code code,
+ enum memmodel model, bool after, unsigned int prec,
+ bool unsignedp)
{
machine_mode mode = GET_MODE (mem);
rtx result;
bool unused_result = (target == const0_rtx);
+ /* Nonzero PREC means MEM is a _BitInt whose padding bits (above PREC) must
+ stay sign/zero extended in memory. */
+ bool bitint_p = prec != 0;
+
/* If loads are not atomic for the required size and we are not called to
provide a __sync builtin, do not do anything so that we stay consistent
with atomic loads of the same size. */
if (!can_atomic_load_p (mode) && !is_mm_sync (model))
return NULL_RTX;
- result = expand_atomic_fetch_op_no_fallback (target, mem, val, code, model,
- after);
+ /* For _BitInt, a single atomic instruction or libcall would leave the
+ padding bits holding the result of the full-mode operation, so skip those
+ paths and use the compare-and-swap loop below. */
+ if (!bitint_p)
+ {
+ result = expand_atomic_fetch_op_no_fallback (target, mem, val, code,
+ model, after);
- if (result)
- return result;
+ if (result)
+ return result;
+ }
/* Add/sub can be implemented by doing the reverse operation with -(val). */
- if (code == PLUS || code == MINUS)
+ if (!bitint_p && (code == PLUS || code == MINUS))
{
rtx tmp;
enum rtx_code reverse = (code == PLUS ? MINUS : PLUS);
@@ -8228,7 +8242,7 @@ expand_atomic_fetch_op (rtx target, rtx mem, rtx val,
enum rtx_code code,
}
/* Try the __sync libcalls only if we can't do compare-and-swap inline. */
- if (!can_compare_and_swap_p (mode, false))
+ if (!bitint_p && !can_compare_and_swap_p (mode, false))
{
rtx libfunc;
bool fixup = false;
@@ -8294,6 +8308,12 @@ expand_atomic_fetch_op (rtx target, rtx mem, rtx val,
enum rtx_code code,
t1 = expand_simple_binop (mode, code, t1, val, NULL_RTX, true,
OPTAB_LIB_WIDEN);
+ /* Re-extend the new value to the _BitInt precision. */
+ if (bitint_p && t1 != NULL_RTX)
+ t1 = reduce_rtx_to_bit_field_precision (t1, NULL_RTX,
+ as_a <scalar_int_mode> (mode),
+ prec, unsignedp);
+
/* For after, copy the value now. */
if (!unused_result && after)
emit_move_insn (target, t1);
@@ -8305,6 +8325,16 @@ expand_atomic_fetch_op (rtx target, rtx mem, rtx val,
enum rtx_code code,
return NULL_RTX;
}
+
+/* Likewise, for the common case of a value that occupies the whole mode and
+ so needs no _BitInt padding-bit extension. */
+rtx
+expand_atomic_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
+ enum memmodel model, bool after)
+{
+ return expand_atomic_fetch_op_prec (target, mem, val, code, model, after,
+ 0, false);
+}
/* Return true if OPERAND is suitable for operand number OPNO of
instruction ICODE. */
diff --git a/gcc/optabs.h b/gcc/optabs.h
index 81f58278887..159a19441d0 100644
--- a/gcc/optabs.h
+++ b/gcc/optabs.h
@@ -365,6 +365,8 @@ rtx expand_atomic_load (rtx, rtx, enum memmodel);
rtx expand_atomic_store (rtx, rtx, enum memmodel, bool);
rtx expand_atomic_fetch_op (rtx, rtx, rtx, enum rtx_code, enum memmodel,
bool);
+rtx expand_atomic_fetch_op_prec (rtx, rtx, rtx, enum rtx_code, enum memmodel,
+ bool, unsigned int, bool);
extern void expand_asm_reg_clobber_mem_blockage (HARD_REG_SET);
diff --git a/gcc/testsuite/gcc.dg/torture/bitint-100.c
b/gcc/testsuite/gcc.dg/torture/bitint-100.c
new file mode 100644
index 00000000000..8b5c4b331ad
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/bitint-100.c
@@ -0,0 +1,47 @@
+/* PR target/124948 */
+/* { dg-do run { target bitint } } */
+/* { dg-require-effective-target sync_int_long } */
+/* { dg-options "-std=c23" } */
+
+unsigned _BitInt (17) ub = 0x10020;
+_BitInt (17) sb;
+
+int
+main ()
+{
+ __atomic_add_fetch (&ub, 0x0fff0, 0);
+ if (ub != 0x10)
+ __builtin_abort ();
+
+ ub = 0x10020;
+ if (__atomic_add_fetch (&ub, 0x0fff0, 0) != 0x10)
+ __builtin_abort ();
+
+ ub = 0x10020;
+ if (__atomic_fetch_add (&ub, 0x0fff0, 0) != 0x10020)
+ __builtin_abort ();
+ if (ub != 0x10)
+ __builtin_abort ();
+
+ ub = 0x10020;
+ __sync_add_and_fetch (&ub, 0x0fff0);
+ if (ub != 0x10)
+ __builtin_abort ();
+
+ ub = 0x1ffff;
+ __atomic_nand_fetch (&ub, 0, 0);
+ if (ub != 0x1ffff)
+ __builtin_abort ();
+
+ sb = 0xffff;
+ __atomic_add_fetch (&sb, 1, 0);
+ if (sb != -65536)
+ __builtin_abort ();
+
+ sb = -65536;
+ __atomic_sub_fetch (&sb, 1, 0);
+ if (sb != 65535)
+ __builtin_abort ();
+
+ return 0;
+}
--
2.54.0
