tcg: Add CONFIG_ATOMIC64

Allow qemu to build on 32-bit hosts without 64-bit atomic ops.

Even if we only allow 32-bit hosts to multi-thread emulate 32-bit
guests, we still need some way to handle the 32-bit guest using a
64-bit atomic operation. Do so by dropping back to single-step.

Backports commit df79b996a7b21c6ea7847f7927a2e1a294b86c72 from qemu

qemu/aarch64.h

@@ -679,6 +679,7 @@
 #define gen_helper_double_saturate gen_helper_double_saturate_aarch64
 #define gen_helper_exception_internal gen_helper_exception_internal_aarch64
 #define gen_helper_exception_with_syndrome gen_helper_exception_with_syndrome_aarch64
+#define gen_helper_exit_atomic gen_helper_exit_atomic_aarch64
 #define gen_helper_get_cp_reg gen_helper_get_cp_reg_aarch64
 #define gen_helper_get_cp_reg64 gen_helper_get_cp_reg64_aarch64
 #define gen_helper_get_r13_banked gen_helper_get_r13_banked_aarch64
@@ -1563,6 +1564,7 @@
 #define helper_exception_internal helper_exception_internal_aarch64
 #define helper_exception_return helper_exception_return_aarch64
 #define helper_exception_with_syndrome helper_exception_with_syndrome_aarch64
+#define helper_exit_atomic helper_exit_atomic_aarch64
 #define helper_get_cp_reg helper_get_cp_reg_aarch64
 #define helper_get_cp_reg64 helper_get_cp_reg64_aarch64
 #define helper_get_r13_banked helper_get_r13_banked_aarch64

qemu/aarch64eb.h

@@ -679,6 +679,7 @@
 #define gen_helper_double_saturate gen_helper_double_saturate_aarch64eb
 #define gen_helper_exception_internal gen_helper_exception_internal_aarch64eb
 #define gen_helper_exception_with_syndrome gen_helper_exception_with_syndrome_aarch64eb
+#define gen_helper_exit_atomic gen_helper_exit_atomic_aarch64eb
 #define gen_helper_get_cp_reg gen_helper_get_cp_reg_aarch64eb
 #define gen_helper_get_cp_reg64 gen_helper_get_cp_reg64_aarch64eb
 #define gen_helper_get_r13_banked gen_helper_get_r13_banked_aarch64eb
@@ -1563,6 +1564,7 @@
 #define helper_exception_internal helper_exception_internal_aarch64eb
 #define helper_exception_return helper_exception_return_aarch64eb
 #define helper_exception_with_syndrome helper_exception_with_syndrome_aarch64eb
+#define helper_exit_atomic helper_exit_atomic_aarch64eb
 #define helper_get_cp_reg helper_get_cp_reg_aarch64eb
 #define helper_get_cp_reg64 helper_get_cp_reg64_aarch64eb
 #define helper_get_r13_banked helper_get_r13_banked_aarch64eb

qemu/arm.h

@@ -679,6 +679,7 @@
 #define gen_helper_double_saturate gen_helper_double_saturate_arm
 #define gen_helper_exception_internal gen_helper_exception_internal_arm
 #define gen_helper_exception_with_syndrome gen_helper_exception_with_syndrome_arm
+#define gen_helper_exit_atomic gen_helper_exit_atomic_arm
 #define gen_helper_get_cp_reg gen_helper_get_cp_reg_arm
 #define gen_helper_get_cp_reg64 gen_helper_get_cp_reg64_arm
 #define gen_helper_get_r13_banked gen_helper_get_r13_banked_arm
@@ -1563,6 +1564,7 @@
 #define helper_exception_internal helper_exception_internal_arm
 #define helper_exception_return helper_exception_return_arm
 #define helper_exception_with_syndrome helper_exception_with_syndrome_arm
+#define helper_exit_atomic helper_exit_atomic_arm
 #define helper_get_cp_reg helper_get_cp_reg_arm
 #define helper_get_cp_reg64 helper_get_cp_reg64_arm
 #define helper_get_r13_banked helper_get_r13_banked_arm

qemu/armeb.h

@@ -679,6 +679,7 @@
 #define gen_helper_double_saturate gen_helper_double_saturate_armeb
 #define gen_helper_exception_internal gen_helper_exception_internal_armeb
 #define gen_helper_exception_with_syndrome gen_helper_exception_with_syndrome_armeb
+#define gen_helper_exit_atomic gen_helper_exit_atomic_armeb
 #define gen_helper_get_cp_reg gen_helper_get_cp_reg_armeb
 #define gen_helper_get_cp_reg64 gen_helper_get_cp_reg64_armeb
 #define gen_helper_get_r13_banked gen_helper_get_r13_banked_armeb
@@ -1563,6 +1564,7 @@
 #define helper_exception_internal helper_exception_internal_armeb
 #define helper_exception_return helper_exception_return_armeb
 #define helper_exception_with_syndrome helper_exception_with_syndrome_armeb
+#define helper_exit_atomic helper_exit_atomic_armeb
 #define helper_get_cp_reg helper_get_cp_reg_armeb
 #define helper_get_cp_reg64 helper_get_cp_reg64_armeb
 #define helper_get_r13_banked helper_get_r13_banked_armeb

qemu/configure

@@ -1091,6 +1091,35 @@ EOF
   fi
 fi
 
+#########################################
+# See if 64-bit atomic operations are supported.
+# Note that without __atomic builtins, we can only
+# assume atomic loads/stores max at pointer size.
+
+cat > $TMPC << EOF
+#include <stdint.h>
+int main(void)
+{
+  uint64_t x = 0, y = 0;
+#ifdef __ATOMIC_RELAXED
+  y = __atomic_load_8(&x, 0);
+  __atomic_store_8(&x, y, 0);
+  __atomic_compare_exchange_8(&x, &y, x, 0, 0, 0);
+  __atomic_exchange_8(&x, y, 0);
+  __atomic_fetch_add_8(&x, y, 0);
+#else
+  typedef char is_host64[sizeof(void *) >= sizeof(uint64_t) ? 1 : -1];
+  __sync_lock_test_and_set(&x, y);
+  __sync_val_compare_and_swap(&x, y, 0);
+  __sync_fetch_and_add(&x, y);
+#endif
+  return 0;
+}
+EOF
+if compile_prog "" "" ; then
+  atomic64=yes
+fi
+
 # Now we've finished running tests it's OK to add -Werror to the compiler flags
 if test "$werror" = "yes"; then
     QEMU_CFLAGS="-Werror $QEMU_CFLAGS"
@@ -1216,6 +1245,10 @@ if test "$atomic128" = "yes" ; then
   echo "CONFIG_ATOMIC128=y" >> $config_host_mak
 fi
 
+if test "$atomic64" = "yes" ; then
+  echo "CONFIG_ATOMIC64=y" >> $config_host_mak
+fi
+
 if test "$tcg_interpreter" = "yes"; then
   QEMU_INCLUDES="-I\$(SRC_PATH)/tcg/tci $QEMU_INCLUDES"
 elif test "$ARCH" = "sparc64" ; then

qemu/cputlb.c

@@ -676,8 +676,10 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
 #define DATA_SIZE 4
 #include "atomic_template.h"
 
+#ifdef CONFIG_ATOMIC64
 #define DATA_SIZE 8
 #include "atomic_template.h"
+#endif
 
 #ifdef CONFIG_ATOMIC128
 #define DATA_SIZE 16
@@ -702,8 +704,10 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
 #define DATA_SIZE 4
 #include "atomic_template.h"
 
+#ifdef CONFIG_ATOMIC64
 #define DATA_SIZE 8
 #include "atomic_template.h"
+#endif
 
 /* Code access functions.  */
 

qemu/header_gen.py

@@ -685,6 +685,7 @@ symbols = (
     'gen_helper_double_saturate',
     'gen_helper_exception_internal',
     'gen_helper_exception_with_syndrome',
+    'gen_helper_exit_atomic',
     'gen_helper_get_cp_reg',
     'gen_helper_get_cp_reg64',
     'gen_helper_get_r13_banked',
@@ -1569,6 +1570,7 @@ symbols = (
     'helper_exception_internal',
     'helper_exception_return',
     'helper_exception_with_syndrome',
+    'helper_exit_atomic',
     'helper_get_cp_reg',
     'helper_get_cp_reg64',
     'helper_get_r13_banked',

qemu/m68k.h

@@ -679,6 +679,7 @@
 #define gen_helper_double_saturate gen_helper_double_saturate_m68k
 #define gen_helper_exception_internal gen_helper_exception_internal_m68k
 #define gen_helper_exception_with_syndrome gen_helper_exception_with_syndrome_m68k
+#define gen_helper_exit_atomic gen_helper_exit_atomic_m68k
 #define gen_helper_get_cp_reg gen_helper_get_cp_reg_m68k
 #define gen_helper_get_cp_reg64 gen_helper_get_cp_reg64_m68k
 #define gen_helper_get_r13_banked gen_helper_get_r13_banked_m68k
@@ -1563,6 +1564,7 @@
 #define helper_exception_internal helper_exception_internal_m68k
 #define helper_exception_return helper_exception_return_m68k
 #define helper_exception_with_syndrome helper_exception_with_syndrome_m68k
+#define helper_exit_atomic helper_exit_atomic_m68k
 #define helper_get_cp_reg helper_get_cp_reg_m68k
 #define helper_get_cp_reg64 helper_get_cp_reg64_m68k
 #define helper_get_r13_banked helper_get_r13_banked_m68k

qemu/mips.h

@@ -679,6 +679,7 @@
 #define gen_helper_double_saturate gen_helper_double_saturate_mips
 #define gen_helper_exception_internal gen_helper_exception_internal_mips
 #define gen_helper_exception_with_syndrome gen_helper_exception_with_syndrome_mips
+#define gen_helper_exit_atomic gen_helper_exit_atomic_mips
 #define gen_helper_get_cp_reg gen_helper_get_cp_reg_mips
 #define gen_helper_get_cp_reg64 gen_helper_get_cp_reg64_mips
 #define gen_helper_get_r13_banked gen_helper_get_r13_banked_mips
@@ -1563,6 +1564,7 @@
 #define helper_exception_internal helper_exception_internal_mips
 #define helper_exception_return helper_exception_return_mips
 #define helper_exception_with_syndrome helper_exception_with_syndrome_mips
+#define helper_exit_atomic helper_exit_atomic_mips
 #define helper_get_cp_reg helper_get_cp_reg_mips
 #define helper_get_cp_reg64 helper_get_cp_reg64_mips
 #define helper_get_r13_banked helper_get_r13_banked_mips

qemu/mips64.h

@@ -679,6 +679,7 @@
 #define gen_helper_double_saturate gen_helper_double_saturate_mips64
 #define gen_helper_exception_internal gen_helper_exception_internal_mips64
 #define gen_helper_exception_with_syndrome gen_helper_exception_with_syndrome_mips64
+#define gen_helper_exit_atomic gen_helper_exit_atomic_mips64
 #define gen_helper_get_cp_reg gen_helper_get_cp_reg_mips64
 #define gen_helper_get_cp_reg64 gen_helper_get_cp_reg64_mips64
 #define gen_helper_get_r13_banked gen_helper_get_r13_banked_mips64
@@ -1563,6 +1564,7 @@
 #define helper_exception_internal helper_exception_internal_mips64
 #define helper_exception_return helper_exception_return_mips64
 #define helper_exception_with_syndrome helper_exception_with_syndrome_mips64
+#define helper_exit_atomic helper_exit_atomic_mips64
 #define helper_get_cp_reg helper_get_cp_reg_mips64
 #define helper_get_cp_reg64 helper_get_cp_reg64_mips64
 #define helper_get_r13_banked helper_get_r13_banked_mips64

qemu/mips64el.h

@@ -679,6 +679,7 @@
 #define gen_helper_double_saturate gen_helper_double_saturate_mips64el
 #define gen_helper_exception_internal gen_helper_exception_internal_mips64el
 #define gen_helper_exception_with_syndrome gen_helper_exception_with_syndrome_mips64el
+#define gen_helper_exit_atomic gen_helper_exit_atomic_mips64el
 #define gen_helper_get_cp_reg gen_helper_get_cp_reg_mips64el
 #define gen_helper_get_cp_reg64 gen_helper_get_cp_reg64_mips64el
 #define gen_helper_get_r13_banked gen_helper_get_r13_banked_mips64el
@@ -1563,6 +1564,7 @@
 #define helper_exception_internal helper_exception_internal_mips64el
 #define helper_exception_return helper_exception_return_mips64el
 #define helper_exception_with_syndrome helper_exception_with_syndrome_mips64el
+#define helper_exit_atomic helper_exit_atomic_mips64el
 #define helper_get_cp_reg helper_get_cp_reg_mips64el
 #define helper_get_cp_reg64 helper_get_cp_reg64_mips64el
 #define helper_get_r13_banked helper_get_r13_banked_mips64el

qemu/mipsel.h

@@ -679,6 +679,7 @@
 #define gen_helper_double_saturate gen_helper_double_saturate_mipsel
 #define gen_helper_exception_internal gen_helper_exception_internal_mipsel
 #define gen_helper_exception_with_syndrome gen_helper_exception_with_syndrome_mipsel
+#define gen_helper_exit_atomic gen_helper_exit_atomic_mipsel
 #define gen_helper_get_cp_reg gen_helper_get_cp_reg_mipsel
 #define gen_helper_get_cp_reg64 gen_helper_get_cp_reg64_mipsel
 #define gen_helper_get_r13_banked gen_helper_get_r13_banked_mipsel
@@ -1563,6 +1564,7 @@
 #define helper_exception_internal helper_exception_internal_mipsel
 #define helper_exception_return helper_exception_return_mipsel
 #define helper_exception_with_syndrome helper_exception_with_syndrome_mipsel
+#define helper_exit_atomic helper_exit_atomic_mipsel
 #define helper_get_cp_reg helper_get_cp_reg_mipsel
 #define helper_get_cp_reg64 helper_get_cp_reg64_mipsel
 #define helper_get_r13_banked helper_get_r13_banked_mipsel

qemu/powerpc.h

@@ -679,6 +679,7 @@
 #define gen_helper_double_saturate gen_helper_double_saturate_powerpc
 #define gen_helper_exception_internal gen_helper_exception_internal_powerpc
 #define gen_helper_exception_with_syndrome gen_helper_exception_with_syndrome_powerpc
+#define gen_helper_exit_atomic gen_helper_exit_atomic_powerpc
 #define gen_helper_get_cp_reg gen_helper_get_cp_reg_powerpc
 #define gen_helper_get_cp_reg64 gen_helper_get_cp_reg64_powerpc
 #define gen_helper_get_r13_banked gen_helper_get_r13_banked_powerpc
@@ -1563,6 +1564,7 @@
 #define helper_exception_internal helper_exception_internal_powerpc
 #define helper_exception_return helper_exception_return_powerpc
 #define helper_exception_with_syndrome helper_exception_with_syndrome_powerpc
+#define helper_exit_atomic helper_exit_atomic_powerpc
 #define helper_get_cp_reg helper_get_cp_reg_powerpc
 #define helper_get_cp_reg64 helper_get_cp_reg64_powerpc
 #define helper_get_r13_banked helper_get_r13_banked_powerpc

qemu/sparc.h

@@ -679,6 +679,7 @@
 #define gen_helper_double_saturate gen_helper_double_saturate_sparc
 #define gen_helper_exception_internal gen_helper_exception_internal_sparc
 #define gen_helper_exception_with_syndrome gen_helper_exception_with_syndrome_sparc
+#define gen_helper_exit_atomic gen_helper_exit_atomic_sparc
 #define gen_helper_get_cp_reg gen_helper_get_cp_reg_sparc
 #define gen_helper_get_cp_reg64 gen_helper_get_cp_reg64_sparc
 #define gen_helper_get_r13_banked gen_helper_get_r13_banked_sparc
@@ -1563,6 +1564,7 @@
 #define helper_exception_internal helper_exception_internal_sparc
 #define helper_exception_return helper_exception_return_sparc
 #define helper_exception_with_syndrome helper_exception_with_syndrome_sparc
+#define helper_exit_atomic helper_exit_atomic_sparc
 #define helper_get_cp_reg helper_get_cp_reg_sparc
 #define helper_get_cp_reg64 helper_get_cp_reg64_sparc
 #define helper_get_r13_banked helper_get_r13_banked_sparc

qemu/sparc64.h

@@ -679,6 +679,7 @@
 #define gen_helper_double_saturate gen_helper_double_saturate_sparc64
 #define gen_helper_exception_internal gen_helper_exception_internal_sparc64
 #define gen_helper_exception_with_syndrome gen_helper_exception_with_syndrome_sparc64
+#define gen_helper_exit_atomic gen_helper_exit_atomic_sparc64
 #define gen_helper_get_cp_reg gen_helper_get_cp_reg_sparc64
 #define gen_helper_get_cp_reg64 gen_helper_get_cp_reg64_sparc64
 #define gen_helper_get_r13_banked gen_helper_get_r13_banked_sparc64
@@ -1563,6 +1564,7 @@
 #define helper_exception_internal helper_exception_internal_sparc64
 #define helper_exception_return helper_exception_return_sparc64
 #define helper_exception_with_syndrome helper_exception_with_syndrome_sparc64
+#define helper_exit_atomic helper_exit_atomic_sparc64
 #define helper_get_cp_reg helper_get_cp_reg_sparc64
 #define helper_get_cp_reg64 helper_get_cp_reg64_sparc64
 #define helper_get_r13_banked helper_get_r13_banked_sparc64

qemu/tcg-runtime.c

@@ -102,6 +102,11 @@ int64_t HELPER(mulsh_i64)(int64_t arg1, int64_t arg2)
     return h;
 }
 
+void HELPER(exit_atomic)(CPUArchState *env)
+{
+    cpu_loop_exit_atomic(ENV_GET_CPU(env), GETPC());
+}
+
 #ifndef CONFIG_SOFTMMU
 /* The softmmu versions of these helpers are in cputlb.c.  */
 
@@ -131,8 +136,10 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
 #define DATA_SIZE 4
 #include "atomic_template.h"
 
+#ifdef CONFIG_ATOMIC64
 #define DATA_SIZE 8
 #include "atomic_template.h"
+#endif
 
 /* The following is only callable from other helpers, and matches up
    with the softmmu version.  */

qemu/tcg/tcg-op.c

@@ -1860,8 +1860,8 @@ void tcg_gen_goto_tb(TCGContext *s, unsigned idx)
     tcg_debug_assert(idx <= 1);
 #ifdef CONFIG_DEBUG_TCG
     /* Verify that we havn't seen this numbered exit before.  */
-    tcg_debug_assert((tcg_ctx.goto_tb_issue_mask & (1 << idx)) == 0);
+    tcg_debug_assert((s->goto_tb_issue_mask & (1 << idx)) == 0);
-    tcg_ctx.goto_tb_issue_mask |= 1 << idx;
+    s->goto_tb_issue_mask |= 1 << idx;
 #endif
     tcg_gen_op1i(s, INDEX_op_goto_tb, idx);
 }
@@ -2065,12 +2065,18 @@ typedef void (*gen_atomic_op_i32)(TCGContext *, TCGv_i32, TCGv_env, TCGv, TCGv_i
 typedef void (*gen_atomic_op_i64)(TCGContext *, TCGv_i64, TCGv_env, TCGv, TCGv_i64);
 #endif
 
+#ifdef CONFIG_ATOMIC64
+# define WITH_ATOMIC64(X) X,
+#else
+# define WITH_ATOMIC64(X) NULL,
+#endif
+
 #ifdef HOST_WORDS_BIGENDIAN
 static void * const table_cmpxchg[16] = {
     gen_helper_atomic_cmpxchgb,
     gen_helper_atomic_cmpxchgw_be,
     gen_helper_atomic_cmpxchgl_be,
-    gen_helper_atomic_cmpxchgq_be,
+    WITH_ATOMIC64(gen_helper_atomic_cmpxchgq_be)
     NULL,
     NULL,
     NULL,
@@ -2078,14 +2084,14 @@ static void * const table_cmpxchg[16] = {
     NULL,
     gen_helper_atomic_cmpxchgw_le,
     gen_helper_atomic_cmpxchgl_le,
-    gen_helper_atomic_cmpxchgq_le,
+    WITH_ATOMIC64(gen_helper_atomic_cmpxchgq_le)
 }
 #else
 static void * const table_cmpxchg[16] = {
     gen_helper_atomic_cmpxchgb,
     gen_helper_atomic_cmpxchgw_le,
     gen_helper_atomic_cmpxchgl_le,
-    gen_helper_atomic_cmpxchgq_le,
+    WITH_ATOMIC64(gen_helper_atomic_cmpxchgq_le)
     NULL,
     NULL,
     NULL,
@@ -2093,7 +2099,7 @@ static void * const table_cmpxchg[16] = {
     NULL,
     gen_helper_atomic_cmpxchgw_be,
     gen_helper_atomic_cmpxchgl_be,
-    gen_helper_atomic_cmpxchgq_be,
+    WITH_ATOMIC64(gen_helper_atomic_cmpxchgq_be)
 };
 #endif
 
@@ -2166,6 +2172,7 @@ void tcg_gen_atomic_cmpxchg_i64(TCGContext *s,
         }
         tcg_temp_free_i64(s, t1);
     } else if ((memop & MO_SIZE) == MO_64) {
+#ifdef CONFIG_ATOMIC64
         gen_atomic_cx_i64 gen;
 
         gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
@@ -2178,8 +2185,11 @@ void tcg_gen_atomic_cmpxchg_i64(TCGContext *s,
             tcg_temp_free_i32(s, oi);
         }
 #else
-        gen(s, retv, tcg_ctx.tcg_env, addr, cmpv, newv);
+        gen(s, retv, s->tcg_env, addr, cmpv, newv);
 #endif
+#else
+        gen_helper_exit_atomic(s, s->tcg_env);
+#endif /* CONFIG_ATOMIC64 */
     } else {
         TCGv_i32 c32 = tcg_temp_new_i32(s);
         TCGv_i32 n32 = tcg_temp_new_i32(s);
@@ -2237,7 +2247,7 @@ static void do_atomic_op_i32(TCGContext *s,
         tcg_temp_free_i32(s, oi);
     }
 #else
-    gen(s, ret, tcg_ctx.tcg_env, addr, val);
+    gen(s, ret, s->tcg_env, addr, val);
 #endif
 
     if (memop & MO_SIGN) {
@@ -2271,6 +2281,7 @@ static void do_atomic_op_i64(TCGContext *s,
     memop = tcg_canonicalize_memop(memop, 1, 0);
 
     if ((memop & MO_SIZE) == MO_64) {
+#ifdef CONFIG_ATOMIC64
         gen_atomic_op_i64 gen;
 
         gen = table[memop & (MO_SIZE | MO_BSWAP)];
@@ -2283,8 +2294,11 @@ static void do_atomic_op_i64(TCGContext *s,
             tcg_temp_free_i32(s, oi);
         }
 #else
-        gen(s, ret, tcg_ctx.tcg_env, addr, val);
+        gen(s, ret, s->tcg_env, addr, val);
 #endif
+#else
+        gen_helper_exit_atomic(s, s->tcg_env);
+#endif /* CONFIG_ATOMIC64 */
     } else {
         TCGv_i32 v32 = tcg_temp_new_i32(s);
         TCGv_i32 r32 = tcg_temp_new_i32(s);
@@ -2302,16 +2316,42 @@ static void do_atomic_op_i64(TCGContext *s,
     }
 }
 
-#define GEN_ATOMIC_HELPER(NAME, OP, NEW)                                \
+#ifdef HOST_WORDS_BIGENDIAN
+#define GEN_ATOMIC_TABLE(NAME) \
 static void * const table_##NAME[16] = {    \
-    [MO_8] = gen_helper_atomic_##NAME##b,                               \
+    gen_helper_atomic_##NAME##b,            \
-    [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le,                   \
+    gen_helper_atomic_##NAME##w_be,         \
-    [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be,                   \
+    gen_helper_atomic_##NAME##l_be,         \
-    [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le,                   \
+    gen_helper_atomic_##NAME##q_be,         \
-    [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be,                   \
+    NULL,                                   \
-    [MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le,                   \
+    NULL,                                   \
-    [MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be,                   \
+    NULL,                                   \
-};                                                                      \
+    NULL,                                   \
+    NULL,                                   \
+    gen_helper_atomic_##NAME##w_le,         \
+    gen_helper_atomic_##NAME##l_le,         \
+    gen_helper_atomic_##NAME##q_le,         \
+};
+#else
+#define GEN_ATOMIC_TABLE(NAME) \
+static void * const table_##NAME[16] = {    \
+    gen_helper_atomic_##NAME##b,            \
+    gen_helper_atomic_##NAME##w_le,         \
+    gen_helper_atomic_##NAME##w_be,         \
+    gen_helper_atomic_##NAME##l_le,         \
+    NULL,                                   \
+    NULL,                                   \
+    NULL,                                   \
+    NULL,                                   \
+    NULL,                                   \
+    gen_helper_atomic_##NAME##l_be,         \
+    gen_helper_atomic_##NAME##q_le,         \
+    gen_helper_atomic_##NAME##q_be,         \
+};
+#endif
+
+#define GEN_ATOMIC_HELPER(NAME, OP, NEW)                                \
+GEN_ATOMIC_TABLE(NAME)                                                  \
 void tcg_gen_atomic_##NAME##_i32                                        \
     (TCGContext *s, TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, TCGMemOp memop) \
 {                                                                       \
@@ -2355,4 +2395,6 @@ static void tcg_gen_mov2_i64(TCGContext *s, TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
 
 GEN_ATOMIC_HELPER(xchg, mov2, 0)
 
+#undef GEN_ATOMIC_TABLE
+
 #undef GEN_ATOMIC_HELPER

qemu/tcg/tcg-runtime.h

@@ -15,23 +15,29 @@ DEF_HELPER_FLAGS_2(sar_i64, TCG_CALL_NO_RWG_SE, s64, s64, s64)
 DEF_HELPER_FLAGS_2(mulsh_i64, TCG_CALL_NO_RWG_SE, s64, s64, s64)
 DEF_HELPER_FLAGS_2(muluh_i64, TCG_CALL_NO_RWG_SE, i64, i64, i64)
 
+DEF_HELPER_FLAGS_1(exit_atomic, TCG_CALL_NO_WG, noreturn, env)
+
 #ifdef CONFIG_SOFTMMU
 
 DEF_HELPER_FLAGS_5(atomic_cmpxchgb, TCG_CALL_NO_WG,
                    i32, env, tl, i32, i32, i32)
 DEF_HELPER_FLAGS_5(atomic_cmpxchgw_be, TCG_CALL_NO_WG,
                    i32, env, tl, i32, i32, i32)
-DEF_HELPER_FLAGS_5(atomic_cmpxchgl_be, TCG_CALL_NO_WG,
-                   i32, env, tl, i32, i32, i32)
-DEF_HELPER_FLAGS_5(atomic_cmpxchgq_be, TCG_CALL_NO_WG,
-                   i64, env, tl, i64, i64, i32)
 DEF_HELPER_FLAGS_5(atomic_cmpxchgw_le, TCG_CALL_NO_WG,
                    i32, env, tl, i32, i32, i32)
+DEF_HELPER_FLAGS_5(atomic_cmpxchgl_be, TCG_CALL_NO_WG,
+                   i32, env, tl, i32, i32, i32)
 DEF_HELPER_FLAGS_5(atomic_cmpxchgl_le, TCG_CALL_NO_WG,
                    i32, env, tl, i32, i32, i32)
+
+#ifdef CONFIG_ATOMIC64
+DEF_HELPER_FLAGS_5(atomic_cmpxchgq_be, TCG_CALL_NO_WG,
+                   i64, env, tl, i64, i64, i32)
 DEF_HELPER_FLAGS_5(atomic_cmpxchgq_le, TCG_CALL_NO_WG,
                    i64, env, tl, i64, i64, i32)
+#endif
 
+#ifdef CONFIG_ATOMIC64
 #define GEN_ATOMIC_HELPERS(NAME)                                  \
     DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), b),              \
                        TCG_CALL_NO_WG, i32, env, tl, i32, i32)    \
@@ -47,17 +53,33 @@ DEF_HELPER_FLAGS_5(atomic_cmpxchgq_le, TCG_CALL_NO_WG,
                        TCG_CALL_NO_WG, i64, env, tl, i64, i32)    \
     DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), q_be),           \
                        TCG_CALL_NO_WG, i64, env, tl, i64, i32)
+#else
+#define GEN_ATOMIC_HELPERS(NAME)                                  \
+    DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), b),              \
+                       TCG_CALL_NO_WG, i32, env, tl, i32, i32)    \
+    DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), w_le),           \
+                       TCG_CALL_NO_WG, i32, env, tl, i32, i32)    \
+    DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), w_be),           \
+                       TCG_CALL_NO_WG, i32, env, tl, i32, i32)    \
+    DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), l_le),           \
+                       TCG_CALL_NO_WG, i32, env, tl, i32, i32)    \
+    DEF_HELPER_FLAGS_4(glue(glue(atomic_, NAME), l_be),           \
+                       TCG_CALL_NO_WG, i32, env, tl, i32, i32)
+#endif /* CONFIG_ATOMIC64 */
 
 #else
 
 DEF_HELPER_FLAGS_4(atomic_cmpxchgb, TCG_CALL_NO_WG, i32, env, tl, i32, i32)
 DEF_HELPER_FLAGS_4(atomic_cmpxchgw_be, TCG_CALL_NO_WG, i32, env, tl, i32, i32)
-DEF_HELPER_FLAGS_4(atomic_cmpxchgl_be, TCG_CALL_NO_WG, i32, env, tl, i32, i32)
-DEF_HELPER_FLAGS_4(atomic_cmpxchgq_be, TCG_CALL_NO_WG, i64, env, tl, i64, i64)
 DEF_HELPER_FLAGS_4(atomic_cmpxchgw_le, TCG_CALL_NO_WG, i32, env, tl, i32, i32)
+DEF_HELPER_FLAGS_4(atomic_cmpxchgl_be, TCG_CALL_NO_WG, i32, env, tl, i32, i32)
 DEF_HELPER_FLAGS_4(atomic_cmpxchgl_le, TCG_CALL_NO_WG, i32, env, tl, i32, i32)
+#ifdef CONFIG_ATOMIC64
+DEF_HELPER_FLAGS_4(atomic_cmpxchgq_be, TCG_CALL_NO_WG, i64, env, tl, i64, i64)
 DEF_HELPER_FLAGS_4(atomic_cmpxchgq_le, TCG_CALL_NO_WG, i64, env, tl, i64, i64)
+#endif
 
+#ifdef CONFIG_ATOMIC64
 #define GEN_ATOMIC_HELPERS(NAME)                             \
     DEF_HELPER_FLAGS_3(glue(glue(atomic_, NAME), b),         \
                        TCG_CALL_NO_WG, i32, env, tl, i32)    \
@@ -73,6 +95,19 @@ DEF_HELPER_FLAGS_4(atomic_cmpxchgq_le, TCG_CALL_NO_WG, i64, env, tl, i64, i64)
                        TCG_CALL_NO_WG, i64, env, tl, i64)    \
     DEF_HELPER_FLAGS_3(glue(glue(atomic_, NAME), q_be),      \
                        TCG_CALL_NO_WG, i64, env, tl, i64)
+#else
+#define GEN_ATOMIC_HELPERS(NAME)                             \
+    DEF_HELPER_FLAGS_3(glue(glue(atomic_, NAME), b),         \
+                       TCG_CALL_NO_WG, i32, env, tl, i32)    \
+    DEF_HELPER_FLAGS_3(glue(glue(atomic_, NAME), w_le),      \
+                       TCG_CALL_NO_WG, i32, env, tl, i32)    \
+    DEF_HELPER_FLAGS_3(glue(glue(atomic_, NAME), w_be),      \
+                       TCG_CALL_NO_WG, i32, env, tl, i32)    \
+    DEF_HELPER_FLAGS_3(glue(glue(atomic_, NAME), l_le),      \
+                       TCG_CALL_NO_WG, i32, env, tl, i32)    \
+    DEF_HELPER_FLAGS_3(glue(glue(atomic_, NAME), l_be),      \
+                       TCG_CALL_NO_WG, i32, env, tl, i32)
+#endif /* CONFIG_ATOMIC64 */
 
 #endif /* CONFIG_SOFTMMU */
 

qemu/x86_64.h

@@ -679,6 +679,7 @@
 #define gen_helper_double_saturate gen_helper_double_saturate_x86_64
 #define gen_helper_exception_internal gen_helper_exception_internal_x86_64
 #define gen_helper_exception_with_syndrome gen_helper_exception_with_syndrome_x86_64
+#define gen_helper_exit_atomic gen_helper_exit_atomic_x86_64
 #define gen_helper_get_cp_reg gen_helper_get_cp_reg_x86_64
 #define gen_helper_get_cp_reg64 gen_helper_get_cp_reg64_x86_64
 #define gen_helper_get_r13_banked gen_helper_get_r13_banked_x86_64
@@ -1563,6 +1564,7 @@
 #define helper_exception_internal helper_exception_internal_x86_64
 #define helper_exception_return helper_exception_return_x86_64
 #define helper_exception_with_syndrome helper_exception_with_syndrome_x86_64
+#define helper_exit_atomic helper_exit_atomic_x86_64
 #define helper_get_cp_reg helper_get_cp_reg_x86_64
 #define helper_get_cp_reg64 helper_get_cp_reg64_x86_64
 #define helper_get_r13_banked helper_get_r13_banked_x86_64