From cb31d54b18eccb24d02c5f9946a1a272e06416e4 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 25 Feb 2021 21:53:51 -0500 Subject: [PATCH] target/arm: Add mte helpers for sve scalar + int stores Because the elements are sequential, we can eliminate many tests all at once when the tag hits TCMA, or if the page(s) are not Tagged. Backports commit 71b9f3948c75bb97641a3c8c7de96d1cb47cdc07 from qemu --- qemu/aarch64.h | 37 ++++++++ qemu/aarch64eb.h | 37 ++++++++ qemu/header_gen.py | 37 ++++++++ qemu/target/arm/helper-sve.h | 47 +++++++++ qemu/target/arm/sve_helper.c | 95 ++++++++++++++----- qemu/target/arm/translate-sve.c | 162 +++++++++++++++++++++----------- 6 files changed, 337 insertions(+), 78 deletions(-) diff --git a/qemu/aarch64.h b/qemu/aarch64.h index 8eec2670..91e380ca 100644 --- a/qemu/aarch64.h +++ b/qemu/aarch64.h @@ -4462,6 +4462,43 @@ #define helper_sve_st1bd_r helper_sve_st1bd_r_aarch64 #define helper_sve_st1bh_r helper_sve_st1bh_r_aarch64 #define helper_sve_st1bs_r helper_sve_st1bs_r_aarch64 +#define helper_sve_st1bb_r_mte helper_sve_st1bb_r_mte_aarch64 +#define helper_sve_st2bb_r_mte helper_sve_st2bb_r_mte_aarch64 +#define helper_sve_st3bb_r_mte helper_sve_st3bb_r_mte_aarch64 +#define helper_sve_st4bb_r_mte helper_sve_st4bb_r_mte_aarch64 +#define helper_sve_st1hh_le_r_mte helper_sve_st1hh_le_r_mte_aarch64 +#define helper_sve_st2hh_le_r_mte helper_sve_st2hh_le_r_mte_aarch64 +#define helper_sve_st3hh_le_r_mte helper_sve_st3hh_le_r_mte_aarch64 +#define helper_sve_st4hh_le_r_mte helper_sve_st4hh_le_r_mte_aarch64 +#define helper_sve_st1hh_be_r_mte helper_sve_st1hh_be_r_mte_aarch64 +#define helper_sve_st2hh_be_r_mte helper_sve_st2hh_be_r_mte_aarch64 +#define helper_sve_st3hh_be_r_mte helper_sve_st3hh_be_r_mte_aarch64 +#define helper_sve_st4hh_be_r_mte helper_sve_st4hh_be_r_mte_aarch64 +#define helper_sve_st1ss_le_r_mte helper_sve_st1ss_le_r_mte_aarch64 +#define helper_sve_st2ss_le_r_mte helper_sve_st2ss_le_r_mte_aarch64 +#define helper_sve_st3ss_le_r_mte helper_sve_st3ss_le_r_mte_aarch64 +#define helper_sve_st4ss_le_r_mte helper_sve_st4ss_le_r_mte_aarch64 +#define helper_sve_st1ss_be_r_mte helper_sve_st1ss_be_r_mte_aarch64 +#define helper_sve_st2ss_be_r_mte helper_sve_st2ss_be_r_mte_aarch64 +#define helper_sve_st3ss_be_r_mte helper_sve_st3ss_be_r_mte_aarch64 +#define helper_sve_st4ss_be_r_mte helper_sve_st4ss_be_r_mte_aarch64 +#define helper_sve_st1dd_le_r_mte helper_sve_st1dd_le_r_mte_aarch64 +#define helper_sve_st2dd_le_r_mte helper_sve_st2dd_le_r_mte_aarch64 +#define helper_sve_st3dd_le_r_mte helper_sve_st3dd_le_r_mte_aarch64 +#define helper_sve_st4dd_le_r_mte helper_sve_st4dd_le_r_mte_aarch64 +#define helper_sve_st1dd_be_r_mte helper_sve_st1dd_be_r_mte_aarch64 +#define helper_sve_st2dd_be_r_mte helper_sve_st2dd_be_r_mte_aarch64 +#define helper_sve_st3dd_be_r_mte helper_sve_st3dd_be_r_mte_aarch64 +#define helper_sve_st4dd_be_r_mte helper_sve_st4dd_be_r_mte_aarch64 +#define helper_sve_st1bh_r_mte helper_sve_st1bh_r_mte_aarch64 +#define helper_sve_st1bs_r_mte helper_sve_st1bs_r_mte_aarch64 +#define helper_sve_st1bd_r_mte helper_sve_st1bd_r_mte_aarch64 +#define helper_sve_st1hs_le_r_mte helper_sve_st1hs_le_r_mte_aarch64 +#define helper_sve_st1hd_le_r_mte helper_sve_st1hd_le_r_mte_aarch64 +#define helper_sve_st1hs_be_r_mte helper_sve_st1hs_be_r_mte_aarch64 +#define helper_sve_st1hd_be_r_mte helper_sve_st1hd_be_r_mte_aarch64 +#define helper_sve_st1sd_le_r_mte helper_sve_st1sd_le_r_mte_aarch64 +#define helper_sve_st1sd_be_r_mte helper_sve_st1sd_be_r_mte_aarch64 #define helper_sve_st1dd_be_r helper_sve_st1dd_be_r_aarch64 #define helper_sve_st1dd_le_r helper_sve_st1dd_le_r_aarch64 #define helper_sve_st2dd_be_r helper_sve_st2dd_be_r_aarch64 diff --git a/qemu/aarch64eb.h b/qemu/aarch64eb.h index c6a8315a..cf8ab85e 100644 --- a/qemu/aarch64eb.h +++ b/qemu/aarch64eb.h @@ -4462,6 +4462,43 @@ #define helper_sve_st1bd_r helper_sve_st1bd_r_aarch64eb #define helper_sve_st1bh_r helper_sve_st1bh_r_aarch64eb #define helper_sve_st1bs_r helper_sve_st1bs_r_aarch64eb +#define helper_sve_st1bb_r_mte helper_sve_st1bb_r_mte_aarch64eb +#define helper_sve_st2bb_r_mte helper_sve_st2bb_r_mte_aarch64eb +#define helper_sve_st3bb_r_mte helper_sve_st3bb_r_mte_aarch64eb +#define helper_sve_st4bb_r_mte helper_sve_st4bb_r_mte_aarch64eb +#define helper_sve_st1hh_le_r_mte helper_sve_st1hh_le_r_mte_aarch64eb +#define helper_sve_st2hh_le_r_mte helper_sve_st2hh_le_r_mte_aarch64eb +#define helper_sve_st3hh_le_r_mte helper_sve_st3hh_le_r_mte_aarch64eb +#define helper_sve_st4hh_le_r_mte helper_sve_st4hh_le_r_mte_aarch64eb +#define helper_sve_st1hh_be_r_mte helper_sve_st1hh_be_r_mte_aarch64eb +#define helper_sve_st2hh_be_r_mte helper_sve_st2hh_be_r_mte_aarch64eb +#define helper_sve_st3hh_be_r_mte helper_sve_st3hh_be_r_mte_aarch64eb +#define helper_sve_st4hh_be_r_mte helper_sve_st4hh_be_r_mte_aarch64eb +#define helper_sve_st1ss_le_r_mte helper_sve_st1ss_le_r_mte_aarch64eb +#define helper_sve_st2ss_le_r_mte helper_sve_st2ss_le_r_mte_aarch64eb +#define helper_sve_st3ss_le_r_mte helper_sve_st3ss_le_r_mte_aarch64eb +#define helper_sve_st4ss_le_r_mte helper_sve_st4ss_le_r_mte_aarch64eb +#define helper_sve_st1ss_be_r_mte helper_sve_st1ss_be_r_mte_aarch64eb +#define helper_sve_st2ss_be_r_mte helper_sve_st2ss_be_r_mte_aarch64eb +#define helper_sve_st3ss_be_r_mte helper_sve_st3ss_be_r_mte_aarch64eb +#define helper_sve_st4ss_be_r_mte helper_sve_st4ss_be_r_mte_aarch64eb +#define helper_sve_st1dd_le_r_mte helper_sve_st1dd_le_r_mte_aarch64eb +#define helper_sve_st2dd_le_r_mte helper_sve_st2dd_le_r_mte_aarch64eb +#define helper_sve_st3dd_le_r_mte helper_sve_st3dd_le_r_mte_aarch64eb +#define helper_sve_st4dd_le_r_mte helper_sve_st4dd_le_r_mte_aarch64eb +#define helper_sve_st1dd_be_r_mte helper_sve_st1dd_be_r_mte_aarch64eb +#define helper_sve_st2dd_be_r_mte helper_sve_st2dd_be_r_mte_aarch64eb +#define helper_sve_st3dd_be_r_mte helper_sve_st3dd_be_r_mte_aarch64eb +#define helper_sve_st4dd_be_r_mte helper_sve_st4dd_be_r_mte_aarch64eb +#define helper_sve_st1bh_r_mte helper_sve_st1bh_r_mte_aarch64eb +#define helper_sve_st1bs_r_mte helper_sve_st1bs_r_mte_aarch64eb +#define helper_sve_st1bd_r_mte helper_sve_st1bd_r_mte_aarch64eb +#define helper_sve_st1hs_le_r_mte helper_sve_st1hs_le_r_mte_aarch64eb +#define helper_sve_st1hd_le_r_mte helper_sve_st1hd_le_r_mte_aarch64eb +#define helper_sve_st1hs_be_r_mte helper_sve_st1hs_be_r_mte_aarch64eb +#define helper_sve_st1hd_be_r_mte helper_sve_st1hd_be_r_mte_aarch64eb +#define helper_sve_st1sd_le_r_mte helper_sve_st1sd_le_r_mte_aarch64eb +#define helper_sve_st1sd_be_r_mte helper_sve_st1sd_be_r_mte_aarch64eb #define helper_sve_st1dd_be_r helper_sve_st1dd_be_r_aarch64eb #define helper_sve_st1dd_le_r helper_sve_st1dd_le_r_aarch64eb #define helper_sve_st2dd_be_r helper_sve_st2dd_be_r_aarch64eb diff --git a/qemu/header_gen.py b/qemu/header_gen.py index 1f96b112..b278eccc 100644 --- a/qemu/header_gen.py +++ b/qemu/header_gen.py @@ -4602,6 +4602,43 @@ aarch64_symbols = ( 'helper_sve_st1bd_r', 'helper_sve_st1bh_r', 'helper_sve_st1bs_r', + 'helper_sve_st1bb_r_mte', + 'helper_sve_st2bb_r_mte', + 'helper_sve_st3bb_r_mte', + 'helper_sve_st4bb_r_mte', + 'helper_sve_st1hh_le_r_mte', + 'helper_sve_st2hh_le_r_mte', + 'helper_sve_st3hh_le_r_mte', + 'helper_sve_st4hh_le_r_mte', + 'helper_sve_st1hh_be_r_mte', + 'helper_sve_st2hh_be_r_mte', + 'helper_sve_st3hh_be_r_mte', + 'helper_sve_st4hh_be_r_mte', + 'helper_sve_st1ss_le_r_mte', + 'helper_sve_st2ss_le_r_mte', + 'helper_sve_st3ss_le_r_mte', + 'helper_sve_st4ss_le_r_mte', + 'helper_sve_st1ss_be_r_mte', + 'helper_sve_st2ss_be_r_mte', + 'helper_sve_st3ss_be_r_mte', + 'helper_sve_st4ss_be_r_mte', + 'helper_sve_st1dd_le_r_mte', + 'helper_sve_st2dd_le_r_mte', + 'helper_sve_st3dd_le_r_mte', + 'helper_sve_st4dd_le_r_mte', + 'helper_sve_st1dd_be_r_mte', + 'helper_sve_st2dd_be_r_mte', + 'helper_sve_st3dd_be_r_mte', + 'helper_sve_st4dd_be_r_mte', + 'helper_sve_st1bh_r_mte', + 'helper_sve_st1bs_r_mte', + 'helper_sve_st1bd_r_mte', + 'helper_sve_st1hs_le_r_mte', + 'helper_sve_st1hd_le_r_mte', + 'helper_sve_st1hs_be_r_mte', + 'helper_sve_st1hd_be_r_mte', + 'helper_sve_st1sd_le_r_mte', + 'helper_sve_st1sd_be_r_mte', 'helper_sve_st1dd_be_r', 'helper_sve_st1dd_le_r', 'helper_sve_st2dd_be_r', diff --git a/qemu/target/arm/helper-sve.h b/qemu/target/arm/helper-sve.h index 1bc1974f..1425f33c 100644 --- a/qemu/target/arm/helper-sve.h +++ b/qemu/target/arm/helper-sve.h @@ -1363,6 +1363,53 @@ DEF_HELPER_FLAGS_4(sve_st1hd_be_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32) DEF_HELPER_FLAGS_4(sve_st1sd_le_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32) DEF_HELPER_FLAGS_4(sve_st1sd_be_r, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st1bb_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st2bb_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st3bb_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st4bb_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) + +DEF_HELPER_FLAGS_4(sve_st1hh_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st2hh_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st3hh_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st4hh_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) + +DEF_HELPER_FLAGS_4(sve_st1hh_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st2hh_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st3hh_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st4hh_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) + +DEF_HELPER_FLAGS_4(sve_st1ss_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st2ss_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st3ss_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st4ss_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) + +DEF_HELPER_FLAGS_4(sve_st1ss_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st2ss_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st3ss_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st4ss_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) + +DEF_HELPER_FLAGS_4(sve_st1dd_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st2dd_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st3dd_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st4dd_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) + +DEF_HELPER_FLAGS_4(sve_st1dd_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st2dd_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st3dd_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st4dd_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) + +DEF_HELPER_FLAGS_4(sve_st1bh_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st1bs_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st1bd_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) + +DEF_HELPER_FLAGS_4(sve_st1hs_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st1hd_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st1hs_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st1hd_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) + +DEF_HELPER_FLAGS_4(sve_st1sd_le_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) +DEF_HELPER_FLAGS_4(sve_st1sd_be_r_mte, TCG_CALL_NO_WG, void, env, ptr, tl, i32) + DEF_HELPER_FLAGS_6(sve_ldbsu_zsu, TCG_CALL_NO_WG, void, env, ptr, ptr, ptr, tl, i32) DEF_HELPER_FLAGS_6(sve_ldhsu_le_zsu, TCG_CALL_NO_WG, diff --git a/qemu/target/arm/sve_helper.c b/qemu/target/arm/sve_helper.c index ae42e054..65988c70 100644 --- a/qemu/target/arm/sve_helper.c +++ b/qemu/target/arm/sve_helper.c @@ -5021,11 +5021,12 @@ DO_LDFF1_LDNF1_2(dd, MO_64, MO_64) * Common helper for all contiguous 1,2,3,4-register predicated stores. */ static inline -void sve_stN_r(CPUARMState *env, uint64_t *vg, target_ulong addr, uint32_t desc, - const uintptr_t retaddr, const int esz, - const int msz, const int N, +void sve_stN_r(CPUARMState *env, uint64_t *vg, target_ulong addr, + uint32_t desc, const uintptr_t retaddr, + const int esz, const int msz, const int N, uint32_t mtedesc, sve_ldst1_host_fn *host_fn, - sve_ldst1_tlb_fn *tlb_fn) + sve_ldst1_tlb_fn *tlb_fn, + sve_cont_ldst_mte_check_fn *mte_check_fn) { const unsigned rd = simd_data(desc); const intptr_t reg_max = simd_oprsz(desc); @@ -5047,7 +5048,14 @@ void sve_stN_r(CPUARMState *env, uint64_t *vg, target_ulong addr, uint32_t desc, sve_cont_ldst_watchpoints(&info, env, vg, addr, 1 << esz, N << msz, BP_MEM_WRITE, retaddr); - /* TODO: MTE check. */ + /* + * Handle mte checks for all active elements. + * Since TBI must be set for MTE, !mtedesc => !mte_active. + */ + if (mte_check_fn && mtedesc) { + mte_check_fn(&info, env, vg, addr, 1 << esz, N << msz, + mtedesc, retaddr); + } flags = info.page[0].flags | info.page[1].flags; if (unlikely(flags != 0)) { @@ -5141,26 +5149,67 @@ void sve_stN_r(CPUARMState *env, uint64_t *vg, target_ulong addr, uint32_t desc, } } -#define DO_STN_1(N, NAME, ESZ) \ -void HELPER(sve_st##N##NAME##_r)(CPUARMState *env, void *vg, \ - target_ulong addr, uint32_t desc) \ -{ \ - sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MO_8, N, \ - sve_st1##NAME##_host, sve_st1##NAME##_tlb); \ +static inline +void sve_stN_r_mte(CPUARMState *env, uint64_t *vg, target_ulong addr, + uint32_t desc, const uintptr_t ra, + const int esz, const int msz, const int N, + sve_ldst1_host_fn *host_fn, + sve_ldst1_tlb_fn *tlb_fn) +{ + uint32_t mtedesc = desc >> (SIMD_DATA_SHIFT + SVE_MTEDESC_SHIFT); + int bit55 = extract64(addr, 55, 1); + + /* Remove mtedesc from the normal sve descriptor. */ + desc = extract32(desc, 0, SIMD_DATA_SHIFT + SVE_MTEDESC_SHIFT); + + /* Perform gross MTE suppression early. */ + if (!tbi_check(desc, bit55) || + tcma_check(desc, bit55, allocation_tag_from_addr(addr))) { + mtedesc = 0; + } + + sve_stN_r(env, vg, addr, desc, ra, esz, msz, N, mtedesc, host_fn, tlb_fn, + N == 1 ? sve_cont_ldst_mte_check1 : sve_cont_ldst_mte_checkN); } -#define DO_STN_2(N, NAME, ESZ, MSZ) \ -void HELPER(sve_st##N##NAME##_le_r)(CPUARMState *env, void *vg, \ - target_ulong addr, uint32_t desc) \ -{ \ - sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, N, \ - sve_st1##NAME##_le_host, sve_st1##NAME##_le_tlb); \ -} \ -void HELPER(sve_st##N##NAME##_be_r)(CPUARMState *env, void *vg, \ - target_ulong addr, uint32_t desc) \ -{ \ - sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, N, \ - sve_st1##NAME##_be_host, sve_st1##NAME##_be_tlb); \ +#define DO_STN_1(N, NAME, ESZ) \ +void HELPER(sve_st##N##NAME##_r)(CPUARMState *env, void *vg, \ + target_ulong addr, uint32_t desc) \ +{ \ + sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MO_8, N, 0, \ + sve_st1##NAME##_host, sve_st1##NAME##_tlb, NULL); \ +} \ +void HELPER(sve_st##N##NAME##_r_mte)(CPUARMState *env, void *vg, \ + target_ulong addr, uint32_t desc) \ +{ \ + sve_stN_r_mte(env, vg, addr, desc, GETPC(), ESZ, MO_8, N, \ + sve_st1##NAME##_host, sve_st1##NAME##_tlb); \ +} + +#define DO_STN_2(N, NAME, ESZ, MSZ) \ +void HELPER(sve_st##N##NAME##_le_r)(CPUARMState *env, void *vg, \ + target_ulong addr, uint32_t desc) \ +{ \ + sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, N, 0, \ + sve_st1##NAME##_le_host, sve_st1##NAME##_le_tlb, NULL); \ +} \ +void HELPER(sve_st##N##NAME##_be_r)(CPUARMState *env, void *vg, \ + target_ulong addr, uint32_t desc) \ +{ \ + sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, N, 0, \ + sve_st1##NAME##_be_host, sve_st1##NAME##_be_tlb, NULL); \ +} \ +void HELPER(sve_st##N##NAME##_le_r_mte)(CPUARMState *env, void *vg, \ + target_ulong addr, uint32_t desc) \ +{ \ + sve_stN_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ, N, \ + sve_st1##NAME##_le_host, sve_st1##NAME##_le_tlb); \ +} \ +void HELPER(sve_st##N##NAME##_be_r_mte)(CPUARMState *env, void *vg, \ + target_ulong addr, uint32_t desc) \ +{ \ + sve_stN_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ, N, \ + sve_st1##NAME##_be_host, sve_st1##NAME##_be_tlb); \ } DO_STN_1(1, bb, MO_8) diff --git a/qemu/target/arm/translate-sve.c b/qemu/target/arm/translate-sve.c index 3fd348b4..1f3260ae 100644 --- a/qemu/target/arm/translate-sve.c +++ b/qemu/target/arm/translate-sve.c @@ -5175,73 +5175,125 @@ static bool trans_LD1R_zpri(DisasContext *s, arg_rpri_load *a) static void do_st_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr, int msz, int esz, int nreg) { - static gen_helper_gvec_mem * const fn_single[2][4][4] = { - { { gen_helper_sve_st1bb_r, - gen_helper_sve_st1bh_r, - gen_helper_sve_st1bs_r, - gen_helper_sve_st1bd_r }, - { NULL, - gen_helper_sve_st1hh_le_r, - gen_helper_sve_st1hs_le_r, - gen_helper_sve_st1hd_le_r }, - { NULL, NULL, - gen_helper_sve_st1ss_le_r, - gen_helper_sve_st1sd_le_r }, - { NULL, NULL, NULL, - gen_helper_sve_st1dd_le_r } }, - { { gen_helper_sve_st1bb_r, - gen_helper_sve_st1bh_r, - gen_helper_sve_st1bs_r, - gen_helper_sve_st1bd_r }, - { NULL, - gen_helper_sve_st1hh_be_r, - gen_helper_sve_st1hs_be_r, - gen_helper_sve_st1hd_be_r }, - { NULL, NULL, - gen_helper_sve_st1ss_be_r, - gen_helper_sve_st1sd_be_r }, - { NULL, NULL, NULL, - gen_helper_sve_st1dd_be_r } }, + static gen_helper_gvec_mem * const fn_single[2][2][4][4] = { + { { { gen_helper_sve_st1bb_r, + gen_helper_sve_st1bh_r, + gen_helper_sve_st1bs_r, + gen_helper_sve_st1bd_r }, + { NULL, + gen_helper_sve_st1hh_le_r, + gen_helper_sve_st1hs_le_r, + gen_helper_sve_st1hd_le_r }, + { NULL, NULL, + gen_helper_sve_st1ss_le_r, + gen_helper_sve_st1sd_le_r }, + { NULL, NULL, NULL, + gen_helper_sve_st1dd_le_r } }, + { { gen_helper_sve_st1bb_r, + gen_helper_sve_st1bh_r, + gen_helper_sve_st1bs_r, + gen_helper_sve_st1bd_r }, + { NULL, + gen_helper_sve_st1hh_be_r, + gen_helper_sve_st1hs_be_r, + gen_helper_sve_st1hd_be_r }, + { NULL, NULL, + gen_helper_sve_st1ss_be_r, + gen_helper_sve_st1sd_be_r }, + { NULL, NULL, NULL, + gen_helper_sve_st1dd_be_r } } }, + + { { { gen_helper_sve_st1bb_r_mte, + gen_helper_sve_st1bh_r_mte, + gen_helper_sve_st1bs_r_mte, + gen_helper_sve_st1bd_r_mte }, + { NULL, + gen_helper_sve_st1hh_le_r_mte, + gen_helper_sve_st1hs_le_r_mte, + gen_helper_sve_st1hd_le_r_mte }, + { NULL, NULL, + gen_helper_sve_st1ss_le_r_mte, + gen_helper_sve_st1sd_le_r_mte }, + { NULL, NULL, NULL, + gen_helper_sve_st1dd_le_r_mte } }, + { { gen_helper_sve_st1bb_r_mte, + gen_helper_sve_st1bh_r_mte, + gen_helper_sve_st1bs_r_mte, + gen_helper_sve_st1bd_r_mte }, + { NULL, + gen_helper_sve_st1hh_be_r_mte, + gen_helper_sve_st1hs_be_r_mte, + gen_helper_sve_st1hd_be_r_mte }, + { NULL, NULL, + gen_helper_sve_st1ss_be_r_mte, + gen_helper_sve_st1sd_be_r_mte }, + { NULL, NULL, NULL, + gen_helper_sve_st1dd_be_r_mte } } }, }; - static gen_helper_gvec_mem * const fn_multiple[2][3][4] = { - { { gen_helper_sve_st2bb_r, - gen_helper_sve_st2hh_le_r, - gen_helper_sve_st2ss_le_r, - gen_helper_sve_st2dd_le_r }, - { gen_helper_sve_st3bb_r, - gen_helper_sve_st3hh_le_r, - gen_helper_sve_st3ss_le_r, - gen_helper_sve_st3dd_le_r }, - { gen_helper_sve_st4bb_r, - gen_helper_sve_st4hh_le_r, - gen_helper_sve_st4ss_le_r, - gen_helper_sve_st4dd_le_r } }, - { { gen_helper_sve_st2bb_r, - gen_helper_sve_st2hh_be_r, - gen_helper_sve_st2ss_be_r, - gen_helper_sve_st2dd_be_r }, - { gen_helper_sve_st3bb_r, - gen_helper_sve_st3hh_be_r, - gen_helper_sve_st3ss_be_r, - gen_helper_sve_st3dd_be_r }, - { gen_helper_sve_st4bb_r, - gen_helper_sve_st4hh_be_r, - gen_helper_sve_st4ss_be_r, - gen_helper_sve_st4dd_be_r } }, + static gen_helper_gvec_mem * const fn_multiple[2][2][3][4] = { + { { { gen_helper_sve_st2bb_r, + gen_helper_sve_st2hh_le_r, + gen_helper_sve_st2ss_le_r, + gen_helper_sve_st2dd_le_r }, + { gen_helper_sve_st3bb_r, + gen_helper_sve_st3hh_le_r, + gen_helper_sve_st3ss_le_r, + gen_helper_sve_st3dd_le_r }, + { gen_helper_sve_st4bb_r, + gen_helper_sve_st4hh_le_r, + gen_helper_sve_st4ss_le_r, + gen_helper_sve_st4dd_le_r } }, + { { gen_helper_sve_st2bb_r, + gen_helper_sve_st2hh_be_r, + gen_helper_sve_st2ss_be_r, + gen_helper_sve_st2dd_be_r }, + { gen_helper_sve_st3bb_r, + gen_helper_sve_st3hh_be_r, + gen_helper_sve_st3ss_be_r, + gen_helper_sve_st3dd_be_r }, + { gen_helper_sve_st4bb_r, + gen_helper_sve_st4hh_be_r, + gen_helper_sve_st4ss_be_r, + gen_helper_sve_st4dd_be_r } } }, + { { { gen_helper_sve_st2bb_r_mte, + gen_helper_sve_st2hh_le_r_mte, + gen_helper_sve_st2ss_le_r_mte, + gen_helper_sve_st2dd_le_r_mte }, + { gen_helper_sve_st3bb_r_mte, + gen_helper_sve_st3hh_le_r_mte, + gen_helper_sve_st3ss_le_r_mte, + gen_helper_sve_st3dd_le_r_mte }, + { gen_helper_sve_st4bb_r_mte, + gen_helper_sve_st4hh_le_r_mte, + gen_helper_sve_st4ss_le_r_mte, + gen_helper_sve_st4dd_le_r_mte } }, + { { gen_helper_sve_st2bb_r_mte, + gen_helper_sve_st2hh_be_r_mte, + gen_helper_sve_st2ss_be_r_mte, + gen_helper_sve_st2dd_be_r_mte }, + { gen_helper_sve_st3bb_r_mte, + gen_helper_sve_st3hh_be_r_mte, + gen_helper_sve_st3ss_be_r_mte, + gen_helper_sve_st3dd_be_r_mte }, + { gen_helper_sve_st4bb_r_mte, + gen_helper_sve_st4hh_be_r_mte, + gen_helper_sve_st4ss_be_r_mte, + gen_helper_sve_st4dd_be_r_mte } } }, }; gen_helper_gvec_mem *fn; int be = s->be_data == MO_BE; if (nreg == 0) { /* ST1 */ - fn = fn_single[be][msz][esz]; + fn = fn_single[s->mte_active[0]][be][msz][esz]; + nreg = 1; } else { /* ST2, ST3, ST4 -- msz == esz, enforced by encoding */ assert(msz == esz); - fn = fn_multiple[be][nreg - 1][msz]; + fn = fn_multiple[s->mte_active[0]][be][nreg - 1][msz]; } assert(fn != NULL); - do_mem_zpa(s, zt, pg, addr, msz_dtype(s, msz), 0, true, fn); + do_mem_zpa(s, zt, pg, addr, msz_dtype(s, msz), nreg, true, fn); } static bool trans_ST_zprr(DisasContext *s, arg_rprr_store *a)