target/arm: Simplify DC_ZVA

Now that we know that the operation is on a single page,
we need not loop over pages while probing.

Backports commit e26d0d226892f67435cadcce86df0ddfb9943174 from qemu

qemu/aarch64.h

@@ -327,6 +327,7 @@
 #define cpu_restore_state cpu_restore_state_aarch64
 #define cpu_restore_state_from_tb cpu_restore_state_from_tb_aarch64
 #define cpu_single_step cpu_single_step_aarch64
+#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_aarch64
 #define cpu_tb_exec cpu_tb_exec_aarch64
 #define cpu_to_be64 cpu_to_be64_aarch64
 #define cpu_to_le32 cpu_to_le32_aarch64

qemu/aarch64eb.h

@@ -327,6 +327,7 @@
 #define cpu_restore_state cpu_restore_state_aarch64eb
 #define cpu_restore_state_from_tb cpu_restore_state_from_tb_aarch64eb
 #define cpu_single_step cpu_single_step_aarch64eb
+#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_aarch64eb
 #define cpu_tb_exec cpu_tb_exec_aarch64eb
 #define cpu_to_be64 cpu_to_be64_aarch64eb
 #define cpu_to_le32 cpu_to_le32_aarch64eb

qemu/accel/tcg/cputlb.c

@@ -1532,6 +1532,22 @@ void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
     store_helper(env, addr, val, oi, retaddr, MO_BEQ);
 }
 
+static inline void
+cpu_store_helper(CPUArchState *env, target_ulong addr, uint64_t val,
+                 int mmu_idx, uintptr_t retaddr, MemOp op)
+{
+    TCGMemOpIdx oi;
+
+    oi = make_memop_idx(op, mmu_idx);
+    store_helper(env, addr, val, oi, retaddr, op);
+}
+
+void cpu_stb_mmuidx_ra(CPUArchState *env, target_ulong addr, uint32_t val,
+                       int mmu_idx, uintptr_t retaddr)
+{
+    cpu_store_helper(env, addr, val, mmu_idx, retaddr, MO_UB);
+}
+
 /* First set of helpers allows passing in of OI and RETADDR.  This makes
    them callable from other helpers.  */
 

qemu/arm.h

@@ -327,6 +327,7 @@
 #define cpu_restore_state cpu_restore_state_arm
 #define cpu_restore_state_from_tb cpu_restore_state_from_tb_arm
 #define cpu_single_step cpu_single_step_arm
+#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_arm
 #define cpu_tb_exec cpu_tb_exec_arm
 #define cpu_to_be64 cpu_to_be64_arm
 #define cpu_to_le32 cpu_to_le32_arm

qemu/armeb.h

@@ -327,6 +327,7 @@
 #define cpu_restore_state cpu_restore_state_armeb
 #define cpu_restore_state_from_tb cpu_restore_state_from_tb_armeb
 #define cpu_single_step cpu_single_step_armeb
+#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_armeb
 #define cpu_tb_exec cpu_tb_exec_armeb
 #define cpu_to_be64 cpu_to_be64_armeb
 #define cpu_to_le32 cpu_to_le32_armeb

qemu/header_gen.py

@@ -333,6 +333,7 @@ symbols = (
     'cpu_restore_state',
     'cpu_restore_state_from_tb',
     'cpu_single_step',
+    'cpu_stb_mmuidx_ra',
     'cpu_tb_exec',
     'cpu_to_be64',
     'cpu_to_le32',

qemu/include/exec/cpu_ldst.h

@@ -416,6 +416,9 @@ static inline CPUTLBEntry *tlb_entry(CPUArchState *env, uintptr_t mmu_idx,
 #undef MEMSUFFIX
 #undef SOFTMMU_CODE_ACCESS
 
+void cpu_stb_mmuidx_ra(CPUArchState *env, target_ulong addr, uint32_t val,
+                       int mmu_idx, uintptr_t retaddr);
+
 #endif /* defined(CONFIG_USER_ONLY) */
 
 /**

qemu/m68k.h

@@ -327,6 +327,7 @@
 #define cpu_restore_state cpu_restore_state_m68k
 #define cpu_restore_state_from_tb cpu_restore_state_from_tb_m68k
 #define cpu_single_step cpu_single_step_m68k
+#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_m68k
 #define cpu_tb_exec cpu_tb_exec_m68k
 #define cpu_to_be64 cpu_to_be64_m68k
 #define cpu_to_le32 cpu_to_le32_m68k

qemu/mips.h

@@ -327,6 +327,7 @@
 #define cpu_restore_state cpu_restore_state_mips
 #define cpu_restore_state_from_tb cpu_restore_state_from_tb_mips
 #define cpu_single_step cpu_single_step_mips
+#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_mips
 #define cpu_tb_exec cpu_tb_exec_mips
 #define cpu_to_be64 cpu_to_be64_mips
 #define cpu_to_le32 cpu_to_le32_mips

qemu/mips64.h

@@ -327,6 +327,7 @@
 #define cpu_restore_state cpu_restore_state_mips64
 #define cpu_restore_state_from_tb cpu_restore_state_from_tb_mips64
 #define cpu_single_step cpu_single_step_mips64
+#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_mips64
 #define cpu_tb_exec cpu_tb_exec_mips64
 #define cpu_to_be64 cpu_to_be64_mips64
 #define cpu_to_le32 cpu_to_le32_mips64

qemu/mips64el.h

@@ -327,6 +327,7 @@
 #define cpu_restore_state cpu_restore_state_mips64el
 #define cpu_restore_state_from_tb cpu_restore_state_from_tb_mips64el
 #define cpu_single_step cpu_single_step_mips64el
+#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_mips64el
 #define cpu_tb_exec cpu_tb_exec_mips64el
 #define cpu_to_be64 cpu_to_be64_mips64el
 #define cpu_to_le32 cpu_to_le32_mips64el

qemu/mipsel.h

@@ -327,6 +327,7 @@
 #define cpu_restore_state cpu_restore_state_mipsel
 #define cpu_restore_state_from_tb cpu_restore_state_from_tb_mipsel
 #define cpu_single_step cpu_single_step_mipsel
+#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_mipsel
 #define cpu_tb_exec cpu_tb_exec_mipsel
 #define cpu_to_be64 cpu_to_be64_mipsel
 #define cpu_to_le32 cpu_to_le32_mipsel

qemu/powerpc.h

@@ -327,6 +327,7 @@
 #define cpu_restore_state cpu_restore_state_powerpc
 #define cpu_restore_state_from_tb cpu_restore_state_from_tb_powerpc
 #define cpu_single_step cpu_single_step_powerpc
+#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_powerpc
 #define cpu_tb_exec cpu_tb_exec_powerpc
 #define cpu_to_be64 cpu_to_be64_powerpc
 #define cpu_to_le32 cpu_to_le32_powerpc

qemu/riscv32.h

@@ -327,6 +327,7 @@
 #define cpu_restore_state cpu_restore_state_riscv32
 #define cpu_restore_state_from_tb cpu_restore_state_from_tb_riscv32
 #define cpu_single_step cpu_single_step_riscv32
+#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_riscv32
 #define cpu_tb_exec cpu_tb_exec_riscv32
 #define cpu_to_be64 cpu_to_be64_riscv32
 #define cpu_to_le32 cpu_to_le32_riscv32

qemu/riscv64.h

@@ -327,6 +327,7 @@
 #define cpu_restore_state cpu_restore_state_riscv64
 #define cpu_restore_state_from_tb cpu_restore_state_from_tb_riscv64
 #define cpu_single_step cpu_single_step_riscv64
+#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_riscv64
 #define cpu_tb_exec cpu_tb_exec_riscv64
 #define cpu_to_be64 cpu_to_be64_riscv64
 #define cpu_to_le32 cpu_to_le32_riscv64

qemu/sparc.h

@@ -327,6 +327,7 @@
 #define cpu_restore_state cpu_restore_state_sparc
 #define cpu_restore_state_from_tb cpu_restore_state_from_tb_sparc
 #define cpu_single_step cpu_single_step_sparc
+#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_sparc
 #define cpu_tb_exec cpu_tb_exec_sparc
 #define cpu_to_be64 cpu_to_be64_sparc
 #define cpu_to_le32 cpu_to_le32_sparc

qemu/sparc64.h

@@ -327,6 +327,7 @@
 #define cpu_restore_state cpu_restore_state_sparc64
 #define cpu_restore_state_from_tb cpu_restore_state_from_tb_sparc64
 #define cpu_single_step cpu_single_step_sparc64
+#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_sparc64
 #define cpu_tb_exec cpu_tb_exec_sparc64
 #define cpu_to_be64 cpu_to_be64_sparc64
 #define cpu_to_le32 cpu_to_le32_sparc64

qemu/target/arm/helper-a64.c

@@ -1096,94 +1096,41 @@ void HELPER(dc_zva)(CPUARMState *env, uint64_t vaddr_in)
      * alignment faults or any memory attribute handling).
      */
 
-    ARMCPU *cpu = env_archcpu(env);
+    int blocklen = 4 << env_archcpu(env)->dcz_blocksize;
-    uint64_t blocklen = 4 << cpu->dcz_blocksize;
     uint64_t vaddr = vaddr_in & ~(blocklen - 1);
+    int mmu_idx = cpu_mmu_index(env, false);
+    void *mem;
+
+    /*
+     * Trapless lookup.  In addition to actual invalid page, may
+     * return NULL for I/O, watchpoints, clean pages, etc.
+     */
+    mem = tlb_vaddr_to_host(env, vaddr, MMU_DATA_STORE, mmu_idx);
 
 #ifndef CONFIG_USER_ONLY
-    {
+    if (unlikely(!mem)) {
+        uintptr_t ra = GETPC();
         /*
-         * Slightly awkwardly, QEMU's TARGET_PAGE_SIZE may be less than
+         * Trap if accessing an invalid page.  DC_ZVA requires that we supply
-         * the block size so we might have to do more than one TLB lookup.
+         * the original pointer for an invalid page.  But watchpoints require
-         * We know that in fact for any v8 CPU the page size is at least 4K
+         * that we probe the actual space.  So do both.
-         * and the block size must be 2K or less, but TARGET_PAGE_SIZE is only
-         * 1K as an artefact of legacy v5 subpage support being present in the
-         * same QEMU executable. So in practice the hostaddr[] array has
-         * two entries, given the current setting of TARGET_PAGE_BITS_MIN.
          */
-        int maxidx = DIV_ROUND_UP(blocklen, TARGET_PAGE_SIZE);
+        (void) probe_write(env, vaddr_in, 1, mmu_idx, ra);
-        // msvc doesnt allow non-constant array sizes, so we work out the size it would be
+        mem = probe_write(env, vaddr, blocklen, mmu_idx, ra);
-        // TARGET_PAGE_SIZE is 1024
-        // blocklen is 64
-        // maxidx = (blocklen+TARGET_PAGE_SIZE-1) / TARGET_PAGE_SIZE
-        //        = (64+1024-1) / 1024
-        //        = 1
-#ifdef _MSC_VER
-        void *hostaddr[1];
-#else
-        void *hostaddr[DIV_ROUND_UP(2 * KiB, 1 << TARGET_PAGE_BITS_MIN)];
-#endif
-        int try, i;
-        unsigned mmu_idx = cpu_mmu_index(env, false);
-        TCGMemOpIdx oi = make_memop_idx(MO_UB, mmu_idx);
 
-        assert(maxidx <= ARRAY_SIZE(hostaddr));
+        if (unlikely(!mem)) {
-
-        for (try = 0; try < 2; try++) {
-
-            for (i = 0; i < maxidx; i++) {
-                hostaddr[i] = tlb_vaddr_to_host(env,
-                                                vaddr + TARGET_PAGE_SIZE * i,
-                                                1, mmu_idx);
-                if (!hostaddr[i]) {
-                    break;
-                }
-            }
-            if (i == maxidx) {
             /*
-                 * If it's all in the TLB it's fair game for just writing to;
+             * The only remaining reason for mem == NULL is I/O.
-                 * we know we don't need to update dirty status, etc.
+             * Just do a series of byte writes as the architecture demands.
              */
-                for (i = 0; i < maxidx - 1; i++) {
+            for (int i = 0; i < blocklen; i++) {
-                    memset(hostaddr[i], 0, TARGET_PAGE_SIZE);
+                cpu_stb_mmuidx_ra(env, vaddr + i, 0, mmu_idx, ra);
             }
-                memset(hostaddr[i], 0, blocklen - (i * TARGET_PAGE_SIZE));
+
             return;
         }
-            /*
-             * OK, try a store and see if we can populate the tlb. This
-             * might cause an exception if the memory isn't writable,
-             * in which case we will longjmp out of here. We must for
-             * this purpose use the actual register value passed to us
-             * so that we get the fault address right.
-             */
-            helper_ret_stb_mmu(env, vaddr_in, 0, oi, GETPC());
-            /* Now we can populate the other TLB entries, if any */
-            for (i = 0; i < maxidx; i++) {
-                uint64_t va = vaddr + TARGET_PAGE_SIZE * i;
-                if (va != (vaddr_in & TARGET_PAGE_MASK)) {
-                    helper_ret_stb_mmu(env, va, 0, oi, GETPC());
     }
-            }
-        }
-
-        /*
-         * Slow path (probably attempt to do this to an I/O device or
-         * similar, or clearing of a block of code we have translations
-         * cached for). Just do a series of byte writes as the architecture
-         * demands. It's not worth trying to use a cpu_physical_memory_map(),
-         * memset(), unmap() sequence here because:
-         *  + we'd need to account for the blocksize being larger than a page
-         *  + the direct-RAM access case is almost always going to be dealt
-         *    with in the fastpath code above, so there's no speed benefit
-         *  + we would have to deal with the map returning NULL because the
-         *    bounce buffer was in use
-         */
-        for (i = 0; i < blocklen; i++) {
-            helper_ret_stb_mmu(env, vaddr + i, 0, oi, GETPC());
-        }
-    }
-#else
-    memset(g2h(vaddr), 0, blocklen);
 #endif
+
+    memset(mem, 0, blocklen);
 }

qemu/target/arm/mte_helper.c

@@ -0,0 +1,276 @@
+/*
+ * ARM v8.5-MemTag Operations
+ *
+ * Copyright (c) 2020 Linaro, Ltd.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "internals.h"
+#include "exec/exec-all.h"
+#include "exec/cpu_ldst.h"
+#include "exec/helper-proto.h"
+
+
+static int choose_nonexcluded_tag(int tag, int offset, uint16_t exclude)
+{
+    if (exclude == 0xffff) {
+        return 0;
+    }
+    if (offset == 0) {
+        while (exclude & (1 << tag)) {
+            tag = (tag + 1) & 15;
+        }
+    } else {
+        do {
+            do {
+                tag = (tag + 1) & 15;
+            } while (exclude & (1 << tag));
+        } while (--offset > 0);
+    }
+    return tag;
+}
+
+/**
+ * allocation_tag_mem:
+ * @env: the cpu environment
+ * @ptr_mmu_idx: the addressing regime to use for the virtual address
+ * @ptr: the virtual address for which to look up tag memory
+ * @ptr_access: the access to use for the virtual address
+ * @ptr_size: the number of bytes in the normal memory access
+ * @tag_access: the access to use for the tag memory
+ * @tag_size: the number of bytes in the tag memory access
+ * @ra: the return address for exception handling
+ *
+ * Our tag memory is formatted as a sequence of little-endian nibbles.
+ * That is, the byte at (addr >> (LOG2_TAG_GRANULE + 1)) contains two
+ * tags, with the tag at [3:0] for the lower addr and the tag at [7:4]
+ * for the higher addr.
+ *
+ * Here, resolve the physical address from the virtual address, and return
+ * a pointer to the corresponding tag byte.  Exit with exception if the
+ * virtual address is not accessible for @ptr_access.
+ *
+ * The @ptr_size and @tag_size values may not have an obvious relation
+ * due to the alignment of @ptr, and the number of tag checks required.
+ *
+ * If there is no tag storage corresponding to @ptr, return NULL.
+ */
+static uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx,
+                                   uint64_t ptr, MMUAccessType ptr_access,
+                                   int ptr_size, MMUAccessType tag_access,
+                                   int tag_size, uintptr_t ra)
+{
+    /* Tag storage not implemented.  */
+    return NULL;
+}
+
+uint64_t HELPER(irg)(CPUARMState *env, uint64_t rn, uint64_t rm)
+{
+    int rtag;
+
+    /*
+     * Our IMPDEF choice for GCR_EL1.RRND==1 is to behave as if
+     * GCR_EL1.RRND==0, always producing deterministic results.
+     */
+    uint16_t exclude = extract32(rm | env->cp15.gcr_el1, 0, 16);
+    int start = extract32(env->cp15.rgsr_el1, 0, 4);
+    int seed = extract32(env->cp15.rgsr_el1, 8, 16);
+    int offset, i;
+
+    /* RandomTag */
+    for (i = offset = 0; i < 4; ++i) {
+        /* NextRandomTagBit */
+        int top = (extract32(seed, 5, 1) ^ extract32(seed, 3, 1) ^
+                   extract32(seed, 2, 1) ^ extract32(seed, 0, 1));
+        seed = (top << 15) | (seed >> 1);
+        offset |= top << i;
+    }
+    rtag = choose_nonexcluded_tag(start, offset, exclude);
+    env->cp15.rgsr_el1 = rtag | (seed << 8);
+
+    return address_with_allocation_tag(rn, rtag);
+}
+
+uint64_t HELPER(addsubg)(CPUARMState *env, uint64_t ptr,
+                         int32_t offset, uint32_t tag_offset)
+{
+    int start_tag = allocation_tag_from_addr(ptr);
+    uint16_t exclude = extract32(env->cp15.gcr_el1, 0, 16);
+    int rtag = choose_nonexcluded_tag(start_tag, tag_offset, exclude);
+
+    return address_with_allocation_tag(ptr + offset, rtag);
+}
+
+static int load_tag1(uint64_t ptr, uint8_t *mem)
+{
+    int ofs = extract32(ptr, LOG2_TAG_GRANULE, 1) * 4;
+    return extract32(*mem, ofs, 4);
+}
+
+uint64_t HELPER(ldg)(CPUARMState *env, uint64_t ptr, uint64_t xt)
+{
+    int mmu_idx = cpu_mmu_index(env, false);
+    uint8_t *mem;
+    int rtag = 0;
+
+    /* Trap if accessing an invalid page.  */
+    mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_LOAD, 1,
+                             MMU_DATA_LOAD, 1, GETPC());
+
+    /* Load if page supports tags. */
+    if (mem) {
+        rtag = load_tag1(ptr, mem);
+    }
+
+    return address_with_allocation_tag(xt, rtag);
+}
+
+static void check_tag_aligned(CPUARMState *env, uint64_t ptr, uintptr_t ra)
+{
+    if (unlikely(!QEMU_IS_ALIGNED(ptr, TAG_GRANULE))) {
+        arm_cpu_do_unaligned_access(env_cpu(env), ptr, MMU_DATA_STORE,
+                                    cpu_mmu_index(env, false), ra);
+        g_assert_not_reached();
+    }
+}
+
+/* For use in a non-parallel context, store to the given nibble.  */
+static void store_tag1(uint64_t ptr, uint8_t *mem, int tag)
+{
+    int ofs = extract32(ptr, LOG2_TAG_GRANULE, 1) * 4;
+    *mem = deposit32(*mem, ofs, 4, tag);
+}
+
+/* For use in a parallel context, atomically store to the given nibble.  */
+static void store_tag1_parallel(uint64_t ptr, uint8_t *mem, int tag)
+{
+    int ofs = extract32(ptr, LOG2_TAG_GRANULE, 1) * 4;
+    uint8_t old = atomic_read(mem);
+
+    while (1) {
+        uint8_t new = deposit32(old, ofs, 4, tag);
+        uint8_t cmp = atomic_cmpxchg(mem, old, new);
+        if (likely(cmp == old)) {
+            return;
+        }
+        old = cmp;
+    }
+}
+
+typedef void stg_store1(uint64_t, uint8_t *, int);
+
+static inline void do_stg(CPUARMState *env, uint64_t ptr, uint64_t xt,
+                          uintptr_t ra, stg_store1 store1)
+{
+    int mmu_idx = cpu_mmu_index(env, false);
+    uint8_t *mem;
+
+    check_tag_aligned(env, ptr, ra);
+
+    /* Trap if accessing an invalid page.  */
+    mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE, TAG_GRANULE,
+                             MMU_DATA_STORE, 1, ra);
+
+    /* Store if page supports tags. */
+    if (mem) {
+        store1(ptr, mem, allocation_tag_from_addr(xt));
+    }
+}
+
+void HELPER(stg)(CPUARMState *env, uint64_t ptr, uint64_t xt)
+{
+    do_stg(env, ptr, xt, GETPC(), store_tag1);
+}
+
+void HELPER(stg_parallel)(CPUARMState *env, uint64_t ptr, uint64_t xt)
+{
+    do_stg(env, ptr, xt, GETPC(), store_tag1_parallel);
+}
+
+void HELPER(stg_stub)(CPUARMState *env, uint64_t ptr)
+{
+    int mmu_idx = cpu_mmu_index(env, false);
+    uintptr_t ra = GETPC();
+
+    check_tag_aligned(env, ptr, ra);
+    probe_write(env, ptr, TAG_GRANULE, mmu_idx, ra);
+}
+
+static inline void do_st2g(CPUARMState *env, uint64_t ptr, uint64_t xt,
+                           uintptr_t ra, stg_store1 store1)
+{
+    int mmu_idx = cpu_mmu_index(env, false);
+    int tag = allocation_tag_from_addr(xt);
+    uint8_t *mem1, *mem2;
+
+    check_tag_aligned(env, ptr, ra);
+
+    /*
+     * Trap if accessing an invalid page(s).
+     * This takes priority over !allocation_tag_access_enabled.
+     */
+    if (ptr & TAG_GRANULE) {
+        /* Two stores unaligned mod TAG_GRANULE*2 -- modify two bytes. */
+        mem1 = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE,
+                                  TAG_GRANULE, MMU_DATA_STORE, 1, ra);
+        mem2 = allocation_tag_mem(env, mmu_idx, ptr + TAG_GRANULE,
+                                  MMU_DATA_STORE, TAG_GRANULE,
+                                  MMU_DATA_STORE, 1, ra);
+
+        /* Store if page(s) support tags. */
+        if (mem1) {
+            store1(TAG_GRANULE, mem1, tag);
+        }
+        if (mem2) {
+            store1(0, mem2, tag);
+        }
+    } else {
+        /* Two stores aligned mod TAG_GRANULE*2 -- modify one byte. */
+        mem1 = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE,
+                                  2 * TAG_GRANULE, MMU_DATA_STORE, 1, ra);
+        if (mem1) {
+            tag |= tag << 4;
+            atomic_set(mem1, tag);
+        }
+    }
+}
+
+void HELPER(st2g)(CPUARMState *env, uint64_t ptr, uint64_t xt)
+{
+    do_st2g(env, ptr, xt, GETPC(), store_tag1);
+}
+
+void HELPER(st2g_parallel)(CPUARMState *env, uint64_t ptr, uint64_t xt)
+{
+    do_st2g(env, ptr, xt, GETPC(), store_tag1_parallel);
+}
+
+void HELPER(st2g_stub)(CPUARMState *env, uint64_t ptr)
+{
+    int mmu_idx = cpu_mmu_index(env, false);
+    uintptr_t ra = GETPC();
+    int in_page = -(ptr | TARGET_PAGE_MASK);
+
+    check_tag_aligned(env, ptr, ra);
+
+    if (likely(in_page >= 2 * TAG_GRANULE)) {
+        probe_write(env, ptr, 2 * TAG_GRANULE, mmu_idx, ra);
+    } else {
+        probe_write(env, ptr, TAG_GRANULE, mmu_idx, ra);
+        probe_write(env, ptr + TAG_GRANULE, TAG_GRANULE, mmu_idx, ra);
+    }
+}

qemu/x86_64.h

@@ -327,6 +327,7 @@
 #define cpu_restore_state cpu_restore_state_x86_64
 #define cpu_restore_state_from_tb cpu_restore_state_from_tb_x86_64
 #define cpu_single_step cpu_single_step_x86_64
+#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_x86_64
 #define cpu_tb_exec cpu_tb_exec_x86_64
 #define cpu_to_be64 cpu_to_be64_x86_64
 #define cpu_to_le32 cpu_to_le32_x86_64