diff --git a/qemu/aarch64.h b/qemu/aarch64.h index 6dc87081..2bb895a8 100644 --- a/qemu/aarch64.h +++ b/qemu/aarch64.h @@ -327,6 +327,7 @@ #define cpu_restore_state cpu_restore_state_aarch64 #define cpu_restore_state_from_tb cpu_restore_state_from_tb_aarch64 #define cpu_single_step cpu_single_step_aarch64 +#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_aarch64 #define cpu_tb_exec cpu_tb_exec_aarch64 #define cpu_to_be64 cpu_to_be64_aarch64 #define cpu_to_le32 cpu_to_le32_aarch64 diff --git a/qemu/aarch64eb.h b/qemu/aarch64eb.h index 8cc106b9..2a423505 100644 --- a/qemu/aarch64eb.h +++ b/qemu/aarch64eb.h @@ -327,6 +327,7 @@ #define cpu_restore_state cpu_restore_state_aarch64eb #define cpu_restore_state_from_tb cpu_restore_state_from_tb_aarch64eb #define cpu_single_step cpu_single_step_aarch64eb +#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_aarch64eb #define cpu_tb_exec cpu_tb_exec_aarch64eb #define cpu_to_be64 cpu_to_be64_aarch64eb #define cpu_to_le32 cpu_to_le32_aarch64eb diff --git a/qemu/accel/tcg/cputlb.c b/qemu/accel/tcg/cputlb.c index 9ba57db2..10f755ee 100644 --- a/qemu/accel/tcg/cputlb.c +++ b/qemu/accel/tcg/cputlb.c @@ -1532,6 +1532,22 @@ void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val, store_helper(env, addr, val, oi, retaddr, MO_BEQ); } +static inline void +cpu_store_helper(CPUArchState *env, target_ulong addr, uint64_t val, + int mmu_idx, uintptr_t retaddr, MemOp op) +{ + TCGMemOpIdx oi; + + oi = make_memop_idx(op, mmu_idx); + store_helper(env, addr, val, oi, retaddr, op); +} + +void cpu_stb_mmuidx_ra(CPUArchState *env, target_ulong addr, uint32_t val, + int mmu_idx, uintptr_t retaddr) +{ + cpu_store_helper(env, addr, val, mmu_idx, retaddr, MO_UB); +} + /* First set of helpers allows passing in of OI and RETADDR. This makes them callable from other helpers. */ diff --git a/qemu/arm.h b/qemu/arm.h index c64d5472..1d2b8c71 100644 --- a/qemu/arm.h +++ b/qemu/arm.h @@ -327,6 +327,7 @@ #define cpu_restore_state cpu_restore_state_arm #define cpu_restore_state_from_tb cpu_restore_state_from_tb_arm #define cpu_single_step cpu_single_step_arm +#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_arm #define cpu_tb_exec cpu_tb_exec_arm #define cpu_to_be64 cpu_to_be64_arm #define cpu_to_le32 cpu_to_le32_arm diff --git a/qemu/armeb.h b/qemu/armeb.h index 597671e5..e29393a1 100644 --- a/qemu/armeb.h +++ b/qemu/armeb.h @@ -327,6 +327,7 @@ #define cpu_restore_state cpu_restore_state_armeb #define cpu_restore_state_from_tb cpu_restore_state_from_tb_armeb #define cpu_single_step cpu_single_step_armeb +#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_armeb #define cpu_tb_exec cpu_tb_exec_armeb #define cpu_to_be64 cpu_to_be64_armeb #define cpu_to_le32 cpu_to_le32_armeb diff --git a/qemu/header_gen.py b/qemu/header_gen.py index b53953d1..c7d77472 100644 --- a/qemu/header_gen.py +++ b/qemu/header_gen.py @@ -333,6 +333,7 @@ symbols = ( 'cpu_restore_state', 'cpu_restore_state_from_tb', 'cpu_single_step', + 'cpu_stb_mmuidx_ra', 'cpu_tb_exec', 'cpu_to_be64', 'cpu_to_le32', diff --git a/qemu/include/exec/cpu_ldst.h b/qemu/include/exec/cpu_ldst.h index d9f653fb..e60b8750 100644 --- a/qemu/include/exec/cpu_ldst.h +++ b/qemu/include/exec/cpu_ldst.h @@ -416,6 +416,9 @@ static inline CPUTLBEntry *tlb_entry(CPUArchState *env, uintptr_t mmu_idx, #undef MEMSUFFIX #undef SOFTMMU_CODE_ACCESS +void cpu_stb_mmuidx_ra(CPUArchState *env, target_ulong addr, uint32_t val, + int mmu_idx, uintptr_t retaddr); + #endif /* defined(CONFIG_USER_ONLY) */ /** diff --git a/qemu/m68k.h b/qemu/m68k.h index b39e30be..353d73c0 100644 --- a/qemu/m68k.h +++ b/qemu/m68k.h @@ -327,6 +327,7 @@ #define cpu_restore_state cpu_restore_state_m68k #define cpu_restore_state_from_tb cpu_restore_state_from_tb_m68k #define cpu_single_step cpu_single_step_m68k +#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_m68k #define cpu_tb_exec cpu_tb_exec_m68k #define cpu_to_be64 cpu_to_be64_m68k #define cpu_to_le32 cpu_to_le32_m68k diff --git a/qemu/mips.h b/qemu/mips.h index 80b4d8e9..b2335385 100644 --- a/qemu/mips.h +++ b/qemu/mips.h @@ -327,6 +327,7 @@ #define cpu_restore_state cpu_restore_state_mips #define cpu_restore_state_from_tb cpu_restore_state_from_tb_mips #define cpu_single_step cpu_single_step_mips +#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_mips #define cpu_tb_exec cpu_tb_exec_mips #define cpu_to_be64 cpu_to_be64_mips #define cpu_to_le32 cpu_to_le32_mips diff --git a/qemu/mips64.h b/qemu/mips64.h index 2d925d79..d5998462 100644 --- a/qemu/mips64.h +++ b/qemu/mips64.h @@ -327,6 +327,7 @@ #define cpu_restore_state cpu_restore_state_mips64 #define cpu_restore_state_from_tb cpu_restore_state_from_tb_mips64 #define cpu_single_step cpu_single_step_mips64 +#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_mips64 #define cpu_tb_exec cpu_tb_exec_mips64 #define cpu_to_be64 cpu_to_be64_mips64 #define cpu_to_le32 cpu_to_le32_mips64 diff --git a/qemu/mips64el.h b/qemu/mips64el.h index 53b9cf46..1cd8bdaa 100644 --- a/qemu/mips64el.h +++ b/qemu/mips64el.h @@ -327,6 +327,7 @@ #define cpu_restore_state cpu_restore_state_mips64el #define cpu_restore_state_from_tb cpu_restore_state_from_tb_mips64el #define cpu_single_step cpu_single_step_mips64el +#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_mips64el #define cpu_tb_exec cpu_tb_exec_mips64el #define cpu_to_be64 cpu_to_be64_mips64el #define cpu_to_le32 cpu_to_le32_mips64el diff --git a/qemu/mipsel.h b/qemu/mipsel.h index 2fd410c1..8867df64 100644 --- a/qemu/mipsel.h +++ b/qemu/mipsel.h @@ -327,6 +327,7 @@ #define cpu_restore_state cpu_restore_state_mipsel #define cpu_restore_state_from_tb cpu_restore_state_from_tb_mipsel #define cpu_single_step cpu_single_step_mipsel +#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_mipsel #define cpu_tb_exec cpu_tb_exec_mipsel #define cpu_to_be64 cpu_to_be64_mipsel #define cpu_to_le32 cpu_to_le32_mipsel diff --git a/qemu/powerpc.h b/qemu/powerpc.h index 53cb087c..289e69ea 100644 --- a/qemu/powerpc.h +++ b/qemu/powerpc.h @@ -327,6 +327,7 @@ #define cpu_restore_state cpu_restore_state_powerpc #define cpu_restore_state_from_tb cpu_restore_state_from_tb_powerpc #define cpu_single_step cpu_single_step_powerpc +#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_powerpc #define cpu_tb_exec cpu_tb_exec_powerpc #define cpu_to_be64 cpu_to_be64_powerpc #define cpu_to_le32 cpu_to_le32_powerpc diff --git a/qemu/riscv32.h b/qemu/riscv32.h index ec2d8a9c..4284d119 100644 --- a/qemu/riscv32.h +++ b/qemu/riscv32.h @@ -327,6 +327,7 @@ #define cpu_restore_state cpu_restore_state_riscv32 #define cpu_restore_state_from_tb cpu_restore_state_from_tb_riscv32 #define cpu_single_step cpu_single_step_riscv32 +#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_riscv32 #define cpu_tb_exec cpu_tb_exec_riscv32 #define cpu_to_be64 cpu_to_be64_riscv32 #define cpu_to_le32 cpu_to_le32_riscv32 diff --git a/qemu/riscv64.h b/qemu/riscv64.h index 9d7339c0..c06e9e2d 100644 --- a/qemu/riscv64.h +++ b/qemu/riscv64.h @@ -327,6 +327,7 @@ #define cpu_restore_state cpu_restore_state_riscv64 #define cpu_restore_state_from_tb cpu_restore_state_from_tb_riscv64 #define cpu_single_step cpu_single_step_riscv64 +#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_riscv64 #define cpu_tb_exec cpu_tb_exec_riscv64 #define cpu_to_be64 cpu_to_be64_riscv64 #define cpu_to_le32 cpu_to_le32_riscv64 diff --git a/qemu/sparc.h b/qemu/sparc.h index 7a8710f2..fd492e8d 100644 --- a/qemu/sparc.h +++ b/qemu/sparc.h @@ -327,6 +327,7 @@ #define cpu_restore_state cpu_restore_state_sparc #define cpu_restore_state_from_tb cpu_restore_state_from_tb_sparc #define cpu_single_step cpu_single_step_sparc +#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_sparc #define cpu_tb_exec cpu_tb_exec_sparc #define cpu_to_be64 cpu_to_be64_sparc #define cpu_to_le32 cpu_to_le32_sparc diff --git a/qemu/sparc64.h b/qemu/sparc64.h index e77487d3..7e012a04 100644 --- a/qemu/sparc64.h +++ b/qemu/sparc64.h @@ -327,6 +327,7 @@ #define cpu_restore_state cpu_restore_state_sparc64 #define cpu_restore_state_from_tb cpu_restore_state_from_tb_sparc64 #define cpu_single_step cpu_single_step_sparc64 +#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_sparc64 #define cpu_tb_exec cpu_tb_exec_sparc64 #define cpu_to_be64 cpu_to_be64_sparc64 #define cpu_to_le32 cpu_to_le32_sparc64 diff --git a/qemu/target/arm/helper-a64.c b/qemu/target/arm/helper-a64.c index e80500ab..d66affcd 100644 --- a/qemu/target/arm/helper-a64.c +++ b/qemu/target/arm/helper-a64.c @@ -1096,94 +1096,41 @@ void HELPER(dc_zva)(CPUARMState *env, uint64_t vaddr_in) * alignment faults or any memory attribute handling). */ - ARMCPU *cpu = env_archcpu(env); - uint64_t blocklen = 4 << cpu->dcz_blocksize; + int blocklen = 4 << env_archcpu(env)->dcz_blocksize; uint64_t vaddr = vaddr_in & ~(blocklen - 1); + int mmu_idx = cpu_mmu_index(env, false); + void *mem; + + /* + * Trapless lookup. In addition to actual invalid page, may + * return NULL for I/O, watchpoints, clean pages, etc. + */ + mem = tlb_vaddr_to_host(env, vaddr, MMU_DATA_STORE, mmu_idx); #ifndef CONFIG_USER_ONLY - { + if (unlikely(!mem)) { + uintptr_t ra = GETPC(); /* - * Slightly awkwardly, QEMU's TARGET_PAGE_SIZE may be less than - * the block size so we might have to do more than one TLB lookup. - * We know that in fact for any v8 CPU the page size is at least 4K - * and the block size must be 2K or less, but TARGET_PAGE_SIZE is only - * 1K as an artefact of legacy v5 subpage support being present in the - * same QEMU executable. So in practice the hostaddr[] array has - * two entries, given the current setting of TARGET_PAGE_BITS_MIN. + * Trap if accessing an invalid page. DC_ZVA requires that we supply + * the original pointer for an invalid page. But watchpoints require + * that we probe the actual space. So do both. */ - int maxidx = DIV_ROUND_UP(blocklen, TARGET_PAGE_SIZE); - // msvc doesnt allow non-constant array sizes, so we work out the size it would be - // TARGET_PAGE_SIZE is 1024 - // blocklen is 64 - // maxidx = (blocklen+TARGET_PAGE_SIZE-1) / TARGET_PAGE_SIZE - // = (64+1024-1) / 1024 - // = 1 -#ifdef _MSC_VER - void *hostaddr[1]; -#else - void *hostaddr[DIV_ROUND_UP(2 * KiB, 1 << TARGET_PAGE_BITS_MIN)]; -#endif - int try, i; - unsigned mmu_idx = cpu_mmu_index(env, false); - TCGMemOpIdx oi = make_memop_idx(MO_UB, mmu_idx); + (void) probe_write(env, vaddr_in, 1, mmu_idx, ra); + mem = probe_write(env, vaddr, blocklen, mmu_idx, ra); - assert(maxidx <= ARRAY_SIZE(hostaddr)); - - for (try = 0; try < 2; try++) { - - for (i = 0; i < maxidx; i++) { - hostaddr[i] = tlb_vaddr_to_host(env, - vaddr + TARGET_PAGE_SIZE * i, - 1, mmu_idx); - if (!hostaddr[i]) { - break; - } - } - if (i == maxidx) { - /* - * If it's all in the TLB it's fair game for just writing to; - * we know we don't need to update dirty status, etc. - */ - for (i = 0; i < maxidx - 1; i++) { - memset(hostaddr[i], 0, TARGET_PAGE_SIZE); - } - memset(hostaddr[i], 0, blocklen - (i * TARGET_PAGE_SIZE)); - return; - } + if (unlikely(!mem)) { /* - * OK, try a store and see if we can populate the tlb. This - * might cause an exception if the memory isn't writable, - * in which case we will longjmp out of here. We must for - * this purpose use the actual register value passed to us - * so that we get the fault address right. + * The only remaining reason for mem == NULL is I/O. + * Just do a series of byte writes as the architecture demands. */ - helper_ret_stb_mmu(env, vaddr_in, 0, oi, GETPC()); - /* Now we can populate the other TLB entries, if any */ - for (i = 0; i < maxidx; i++) { - uint64_t va = vaddr + TARGET_PAGE_SIZE * i; - if (va != (vaddr_in & TARGET_PAGE_MASK)) { - helper_ret_stb_mmu(env, va, 0, oi, GETPC()); - } + for (int i = 0; i < blocklen; i++) { + cpu_stb_mmuidx_ra(env, vaddr + i, 0, mmu_idx, ra); } - } - /* - * Slow path (probably attempt to do this to an I/O device or - * similar, or clearing of a block of code we have translations - * cached for). Just do a series of byte writes as the architecture - * demands. It's not worth trying to use a cpu_physical_memory_map(), - * memset(), unmap() sequence here because: - * + we'd need to account for the blocksize being larger than a page - * + the direct-RAM access case is almost always going to be dealt - * with in the fastpath code above, so there's no speed benefit - * + we would have to deal with the map returning NULL because the - * bounce buffer was in use - */ - for (i = 0; i < blocklen; i++) { - helper_ret_stb_mmu(env, vaddr + i, 0, oi, GETPC()); + return; } } -#else - memset(g2h(vaddr), 0, blocklen); #endif + + memset(mem, 0, blocklen); } diff --git a/qemu/target/arm/mte_helper.c b/qemu/target/arm/mte_helper.c new file mode 100644 index 00000000..7ec7930d --- /dev/null +++ b/qemu/target/arm/mte_helper.c @@ -0,0 +1,276 @@ +/* + * ARM v8.5-MemTag Operations + * + * Copyright (c) 2020 Linaro, Ltd. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#include "qemu/osdep.h" +#include "cpu.h" +#include "internals.h" +#include "exec/exec-all.h" +#include "exec/cpu_ldst.h" +#include "exec/helper-proto.h" + + +static int choose_nonexcluded_tag(int tag, int offset, uint16_t exclude) +{ + if (exclude == 0xffff) { + return 0; + } + if (offset == 0) { + while (exclude & (1 << tag)) { + tag = (tag + 1) & 15; + } + } else { + do { + do { + tag = (tag + 1) & 15; + } while (exclude & (1 << tag)); + } while (--offset > 0); + } + return tag; +} + +/** + * allocation_tag_mem: + * @env: the cpu environment + * @ptr_mmu_idx: the addressing regime to use for the virtual address + * @ptr: the virtual address for which to look up tag memory + * @ptr_access: the access to use for the virtual address + * @ptr_size: the number of bytes in the normal memory access + * @tag_access: the access to use for the tag memory + * @tag_size: the number of bytes in the tag memory access + * @ra: the return address for exception handling + * + * Our tag memory is formatted as a sequence of little-endian nibbles. + * That is, the byte at (addr >> (LOG2_TAG_GRANULE + 1)) contains two + * tags, with the tag at [3:0] for the lower addr and the tag at [7:4] + * for the higher addr. + * + * Here, resolve the physical address from the virtual address, and return + * a pointer to the corresponding tag byte. Exit with exception if the + * virtual address is not accessible for @ptr_access. + * + * The @ptr_size and @tag_size values may not have an obvious relation + * due to the alignment of @ptr, and the number of tag checks required. + * + * If there is no tag storage corresponding to @ptr, return NULL. + */ +static uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx, + uint64_t ptr, MMUAccessType ptr_access, + int ptr_size, MMUAccessType tag_access, + int tag_size, uintptr_t ra) +{ + /* Tag storage not implemented. */ + return NULL; +} + +uint64_t HELPER(irg)(CPUARMState *env, uint64_t rn, uint64_t rm) +{ + int rtag; + + /* + * Our IMPDEF choice for GCR_EL1.RRND==1 is to behave as if + * GCR_EL1.RRND==0, always producing deterministic results. + */ + uint16_t exclude = extract32(rm | env->cp15.gcr_el1, 0, 16); + int start = extract32(env->cp15.rgsr_el1, 0, 4); + int seed = extract32(env->cp15.rgsr_el1, 8, 16); + int offset, i; + + /* RandomTag */ + for (i = offset = 0; i < 4; ++i) { + /* NextRandomTagBit */ + int top = (extract32(seed, 5, 1) ^ extract32(seed, 3, 1) ^ + extract32(seed, 2, 1) ^ extract32(seed, 0, 1)); + seed = (top << 15) | (seed >> 1); + offset |= top << i; + } + rtag = choose_nonexcluded_tag(start, offset, exclude); + env->cp15.rgsr_el1 = rtag | (seed << 8); + + return address_with_allocation_tag(rn, rtag); +} + +uint64_t HELPER(addsubg)(CPUARMState *env, uint64_t ptr, + int32_t offset, uint32_t tag_offset) +{ + int start_tag = allocation_tag_from_addr(ptr); + uint16_t exclude = extract32(env->cp15.gcr_el1, 0, 16); + int rtag = choose_nonexcluded_tag(start_tag, tag_offset, exclude); + + return address_with_allocation_tag(ptr + offset, rtag); +} + +static int load_tag1(uint64_t ptr, uint8_t *mem) +{ + int ofs = extract32(ptr, LOG2_TAG_GRANULE, 1) * 4; + return extract32(*mem, ofs, 4); +} + +uint64_t HELPER(ldg)(CPUARMState *env, uint64_t ptr, uint64_t xt) +{ + int mmu_idx = cpu_mmu_index(env, false); + uint8_t *mem; + int rtag = 0; + + /* Trap if accessing an invalid page. */ + mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_LOAD, 1, + MMU_DATA_LOAD, 1, GETPC()); + + /* Load if page supports tags. */ + if (mem) { + rtag = load_tag1(ptr, mem); + } + + return address_with_allocation_tag(xt, rtag); +} + +static void check_tag_aligned(CPUARMState *env, uint64_t ptr, uintptr_t ra) +{ + if (unlikely(!QEMU_IS_ALIGNED(ptr, TAG_GRANULE))) { + arm_cpu_do_unaligned_access(env_cpu(env), ptr, MMU_DATA_STORE, + cpu_mmu_index(env, false), ra); + g_assert_not_reached(); + } +} + +/* For use in a non-parallel context, store to the given nibble. */ +static void store_tag1(uint64_t ptr, uint8_t *mem, int tag) +{ + int ofs = extract32(ptr, LOG2_TAG_GRANULE, 1) * 4; + *mem = deposit32(*mem, ofs, 4, tag); +} + +/* For use in a parallel context, atomically store to the given nibble. */ +static void store_tag1_parallel(uint64_t ptr, uint8_t *mem, int tag) +{ + int ofs = extract32(ptr, LOG2_TAG_GRANULE, 1) * 4; + uint8_t old = atomic_read(mem); + + while (1) { + uint8_t new = deposit32(old, ofs, 4, tag); + uint8_t cmp = atomic_cmpxchg(mem, old, new); + if (likely(cmp == old)) { + return; + } + old = cmp; + } +} + +typedef void stg_store1(uint64_t, uint8_t *, int); + +static inline void do_stg(CPUARMState *env, uint64_t ptr, uint64_t xt, + uintptr_t ra, stg_store1 store1) +{ + int mmu_idx = cpu_mmu_index(env, false); + uint8_t *mem; + + check_tag_aligned(env, ptr, ra); + + /* Trap if accessing an invalid page. */ + mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE, TAG_GRANULE, + MMU_DATA_STORE, 1, ra); + + /* Store if page supports tags. */ + if (mem) { + store1(ptr, mem, allocation_tag_from_addr(xt)); + } +} + +void HELPER(stg)(CPUARMState *env, uint64_t ptr, uint64_t xt) +{ + do_stg(env, ptr, xt, GETPC(), store_tag1); +} + +void HELPER(stg_parallel)(CPUARMState *env, uint64_t ptr, uint64_t xt) +{ + do_stg(env, ptr, xt, GETPC(), store_tag1_parallel); +} + +void HELPER(stg_stub)(CPUARMState *env, uint64_t ptr) +{ + int mmu_idx = cpu_mmu_index(env, false); + uintptr_t ra = GETPC(); + + check_tag_aligned(env, ptr, ra); + probe_write(env, ptr, TAG_GRANULE, mmu_idx, ra); +} + +static inline void do_st2g(CPUARMState *env, uint64_t ptr, uint64_t xt, + uintptr_t ra, stg_store1 store1) +{ + int mmu_idx = cpu_mmu_index(env, false); + int tag = allocation_tag_from_addr(xt); + uint8_t *mem1, *mem2; + + check_tag_aligned(env, ptr, ra); + + /* + * Trap if accessing an invalid page(s). + * This takes priority over !allocation_tag_access_enabled. + */ + if (ptr & TAG_GRANULE) { + /* Two stores unaligned mod TAG_GRANULE*2 -- modify two bytes. */ + mem1 = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE, + TAG_GRANULE, MMU_DATA_STORE, 1, ra); + mem2 = allocation_tag_mem(env, mmu_idx, ptr + TAG_GRANULE, + MMU_DATA_STORE, TAG_GRANULE, + MMU_DATA_STORE, 1, ra); + + /* Store if page(s) support tags. */ + if (mem1) { + store1(TAG_GRANULE, mem1, tag); + } + if (mem2) { + store1(0, mem2, tag); + } + } else { + /* Two stores aligned mod TAG_GRANULE*2 -- modify one byte. */ + mem1 = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE, + 2 * TAG_GRANULE, MMU_DATA_STORE, 1, ra); + if (mem1) { + tag |= tag << 4; + atomic_set(mem1, tag); + } + } +} + +void HELPER(st2g)(CPUARMState *env, uint64_t ptr, uint64_t xt) +{ + do_st2g(env, ptr, xt, GETPC(), store_tag1); +} + +void HELPER(st2g_parallel)(CPUARMState *env, uint64_t ptr, uint64_t xt) +{ + do_st2g(env, ptr, xt, GETPC(), store_tag1_parallel); +} + +void HELPER(st2g_stub)(CPUARMState *env, uint64_t ptr) +{ + int mmu_idx = cpu_mmu_index(env, false); + uintptr_t ra = GETPC(); + int in_page = -(ptr | TARGET_PAGE_MASK); + + check_tag_aligned(env, ptr, ra); + + if (likely(in_page >= 2 * TAG_GRANULE)) { + probe_write(env, ptr, 2 * TAG_GRANULE, mmu_idx, ra); + } else { + probe_write(env, ptr, TAG_GRANULE, mmu_idx, ra); + probe_write(env, ptr + TAG_GRANULE, TAG_GRANULE, mmu_idx, ra); + } +} diff --git a/qemu/x86_64.h b/qemu/x86_64.h index 59f509f2..7c47bd00 100644 --- a/qemu/x86_64.h +++ b/qemu/x86_64.h @@ -327,6 +327,7 @@ #define cpu_restore_state cpu_restore_state_x86_64 #define cpu_restore_state_from_tb cpu_restore_state_from_tb_x86_64 #define cpu_single_step cpu_single_step_x86_64 +#define cpu_stb_mmuidx_ra cpu_stb_mmuidx_ra_x86_64 #define cpu_tb_exec cpu_tb_exec_x86_64 #define cpu_to_be64 cpu_to_be64_x86_64 #define cpu_to_le32 cpu_to_le32_x86_64