From: Richard Henderson <[email protected]> For native code generation, zero-extending 32-bit addresses for the slow path helpers happens in tcg_out_{ld,st}_helper_args, but there isn't really a slow path for TCI, so that didn't happen.
Make the extension for TCI explicit in the opcode stream, much like we already do for plugins and atomic helpers. Cc: [email protected] Fixes: 24e46e6c9d9 ("accel/tcg: Widen tcg-ldst.h addresses to uint64_t") Signed-off-by: Richard Henderson <[email protected]> (cherry picked from commit 41706d3e72d651edb03b4a06b02b490bec8a3ddf) (Mjt: backport to before v10.0.0-521-gaae2456ac0b4 "tcg: Merge INDEX_op_{ld,st}_{i32,i64,i128}") Signed-off-by: Michael Tokarev <[email protected]> diff --git a/tcg/tcg-op-ldst.c b/tcg/tcg-op-ldst.c index 73838e2701..a8f35e1970 100644 --- a/tcg/tcg-op-ldst.c +++ b/tcg/tcg-op-ldst.c @@ -118,6 +118,30 @@ static void tcg_gen_req_mo(TCGBar type) } } +static TCGTemp *tci_extend_addr(TCGTemp *addr) +{ +#ifdef CONFIG_TCG_INTERPRETER + /* + * 64-bit interpreter requires 64-bit addresses. + * Compare to the extension performed by tcg_out_{ld,st}_helper_args + * for native code generation. + */ + if (TCG_TARGET_REG_BITS == 64 && tcg_ctx->addr_type == TCG_TYPE_I32) { + TCGv_i64 temp = tcg_temp_ebb_new_i64(); + tcg_gen_extu_i32_i64(temp, temp_tcgv_i32(addr)); + return tcgv_i64_temp(temp); + } +#endif + return addr; +} + +static void maybe_free_addr(TCGTemp *addr, TCGTemp *copy) +{ + if (addr != copy) { + tcg_temp_free_internal(copy); + } +} + /* Only required for loads, where value might overlap addr. */ static TCGv_i64 plugin_maybe_preserve_addr(TCGTemp *addr) { @@ -217,6 +241,7 @@ static void tcg_gen_qemu_ld_i32_int(TCGv_i32 val, TCGTemp *addr, MemOp orig_memop; MemOpIdx orig_oi, oi; TCGv_i64 copy_addr; + TCGTemp *addr_new; tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); orig_memop = memop = tcg_canonicalize_memop(memop, 0, 0); @@ -231,11 +256,13 @@ static void tcg_gen_qemu_ld_i32_int(TCGv_i32 val, TCGTemp *addr, oi = make_memop_idx(memop, idx); } + addr_new = tci_extend_addr(addr); copy_addr = plugin_maybe_preserve_addr(addr); gen_ldst(INDEX_op_qemu_ld_i32, TCG_TYPE_I32, - tcgv_i32_temp(val), NULL, addr, oi); + tcgv_i32_temp(val), NULL, addr_new, oi); plugin_gen_mem_callbacks_i32(val, copy_addr, addr, orig_oi, QEMU_PLUGIN_MEM_R); + maybe_free_addr(addr, addr_new); if ((orig_memop ^ memop) & MO_BSWAP) { switch (orig_memop & MO_SIZE) { @@ -267,6 +294,7 @@ static void tcg_gen_qemu_st_i32_int(TCGv_i32 val, TCGTemp *addr, TCGv_i32 swap = NULL; MemOpIdx orig_oi, oi; TCGOpcode opc; + TCGTemp *addr_new; tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); memop = tcg_canonicalize_memop(memop, 0, 1); @@ -294,8 +322,10 @@ static void tcg_gen_qemu_st_i32_int(TCGv_i32 val, TCGTemp *addr, } else { opc = INDEX_op_qemu_st_i32; } - gen_ldst(opc, TCG_TYPE_I32, tcgv_i32_temp(val), NULL, addr, oi); + addr_new = tci_extend_addr(addr); + gen_ldst(opc, TCG_TYPE_I32, tcgv_i32_temp(val), NULL, addr_new, oi); plugin_gen_mem_callbacks_i32(val, NULL, addr, orig_oi, QEMU_PLUGIN_MEM_W); + maybe_free_addr(addr, addr_new); if (swap) { tcg_temp_free_i32(swap); @@ -316,6 +346,7 @@ static void tcg_gen_qemu_ld_i64_int(TCGv_i64 val, TCGTemp *addr, MemOp orig_memop; MemOpIdx orig_oi, oi; TCGv_i64 copy_addr; + TCGTemp *addr_new; if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { tcg_gen_qemu_ld_i32_int(TCGV_LOW(val), addr, idx, memop); @@ -340,10 +371,12 @@ static void tcg_gen_qemu_ld_i64_int(TCGv_i64 val, TCGTemp *addr, oi = make_memop_idx(memop, idx); } + addr_new = tci_extend_addr(addr); copy_addr = plugin_maybe_preserve_addr(addr); - gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, oi); + gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr_new, oi); plugin_gen_mem_callbacks_i64(val, copy_addr, addr, orig_oi, QEMU_PLUGIN_MEM_R); + maybe_free_addr(addr, addr_new); if ((orig_memop ^ memop) & MO_BSWAP) { int flags = (orig_memop & MO_SIGN @@ -378,6 +411,7 @@ static void tcg_gen_qemu_st_i64_int(TCGv_i64 val, TCGTemp *addr, { TCGv_i64 swap = NULL; MemOpIdx orig_oi, oi; + TCGTemp *addr_new; if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { tcg_gen_qemu_st_i32_int(TCGV_LOW(val), addr, idx, memop); @@ -408,8 +442,10 @@ static void tcg_gen_qemu_st_i64_int(TCGv_i64 val, TCGTemp *addr, oi = make_memop_idx(memop, idx); } - gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, oi); + addr_new = tci_extend_addr(addr); + gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr_new, oi); plugin_gen_mem_callbacks_i64(val, NULL, addr, orig_oi, QEMU_PLUGIN_MEM_W); + maybe_free_addr(addr, addr_new); if (swap) { tcg_temp_free_i64(swap); @@ -520,6 +556,7 @@ static void tcg_gen_qemu_ld_i128_int(TCGv_i128 val, TCGTemp *addr, { MemOpIdx orig_oi; TCGv_i64 ext_addr = NULL; + TCGTemp *addr_new; check_max_alignment(memop_alignment_bits(memop)); tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); @@ -547,8 +584,10 @@ static void tcg_gen_qemu_ld_i128_int(TCGv_i128 val, TCGTemp *addr, hi = TCGV128_HIGH(val); } + addr_new = tci_extend_addr(addr); gen_ldst(INDEX_op_qemu_ld_i128, TCG_TYPE_I128, tcgv_i64_temp(lo), - tcgv_i64_temp(hi), addr, oi); + tcgv_i64_temp(hi), addr_new, oi); + maybe_free_addr(addr, addr_new); if (need_bswap) { tcg_gen_bswap64_i64(lo, lo); @@ -576,8 +615,10 @@ static void tcg_gen_qemu_ld_i128_int(TCGv_i128 val, TCGTemp *addr, y = TCGV128_LOW(val); } - gen_ldst_i64(INDEX_op_qemu_ld_i64, x, addr, + addr_new = tci_extend_addr(addr); + gen_ldst_i64(INDEX_op_qemu_ld_i64, x, addr_new, make_memop_idx(mop[0], idx)); + maybe_free_addr(addr, addr_new); if (need_bswap) { tcg_gen_bswap64_i64(x, x); @@ -593,8 +634,10 @@ static void tcg_gen_qemu_ld_i128_int(TCGv_i128 val, TCGTemp *addr, addr_p8 = tcgv_i64_temp(t); } - gen_ldst_i64(INDEX_op_qemu_ld_i64, y, addr_p8, + addr_new = tci_extend_addr(addr_p8); + gen_ldst_i64(INDEX_op_qemu_ld_i64, y, addr_new, make_memop_idx(mop[1], idx)); + maybe_free_addr(addr_p8, addr_new); tcg_temp_free_internal(addr_p8); if (need_bswap) { @@ -628,6 +671,7 @@ static void tcg_gen_qemu_st_i128_int(TCGv_i128 val, TCGTemp *addr, { MemOpIdx orig_oi; TCGv_i64 ext_addr = NULL; + TCGTemp *addr_new; check_max_alignment(memop_alignment_bits(memop)); tcg_gen_req_mo(TCG_MO_ST_LD | TCG_MO_ST_ST); @@ -658,8 +702,10 @@ static void tcg_gen_qemu_st_i128_int(TCGv_i128 val, TCGTemp *addr, hi = TCGV128_HIGH(val); } + addr_new = tci_extend_addr(addr); gen_ldst(INDEX_op_qemu_st_i128, TCG_TYPE_I128, - tcgv_i64_temp(lo), tcgv_i64_temp(hi), addr, oi); + tcgv_i64_temp(lo), tcgv_i64_temp(hi), addr_new, oi); + maybe_free_addr(addr, addr_new); if (need_bswap) { tcg_temp_free_i64(lo); @@ -686,8 +732,10 @@ static void tcg_gen_qemu_st_i128_int(TCGv_i128 val, TCGTemp *addr, x = b; } - gen_ldst_i64(INDEX_op_qemu_st_i64, x, addr, + addr_new = tci_extend_addr(addr); + gen_ldst_i64(INDEX_op_qemu_st_i64, x, addr_new, make_memop_idx(mop[0], idx)); + maybe_free_addr(addr, addr_new); if (tcg_ctx->addr_type == TCG_TYPE_I32) { TCGv_i32 t = tcg_temp_ebb_new_i32(); @@ -699,15 +747,17 @@ static void tcg_gen_qemu_st_i128_int(TCGv_i128 val, TCGTemp *addr, addr_p8 = tcgv_i64_temp(t); } + addr_new = tci_extend_addr(addr_p8); if (b) { tcg_gen_bswap64_i64(b, y); - gen_ldst_i64(INDEX_op_qemu_st_i64, b, addr_p8, + gen_ldst_i64(INDEX_op_qemu_st_i64, b, addr_new, make_memop_idx(mop[1], idx)); tcg_temp_free_i64(b); } else { - gen_ldst_i64(INDEX_op_qemu_st_i64, y, addr_p8, + gen_ldst_i64(INDEX_op_qemu_st_i64, y, addr_new, make_memop_idx(mop[1], idx)); } + maybe_free_addr(addr_p8, addr_new); tcg_temp_free_internal(addr_p8); } else { if (tcg_ctx->addr_type == TCG_TYPE_I32) { -- 2.47.3
