target/arm: Implement helper_mte_check1

Fill out the stub that was added earlier.

Backports commit 2e34ff45f32cb032883616a1cc5ea8ac96f546d5 from qemu

qemu/aarch64.h

@@ -3583,7 +3583,7 @@
 #define helper_msr_i_daifclear helper_msr_i_daifclear_aarch64
 #define helper_msr_i_daifset helper_msr_i_daifset_aarch64
 #define helper_msr_i_spsel helper_msr_i_spsel_aarch64
-#define helper_mte_check1 helper_mte_check1_aarch64
+#define helper_mte_check_1 helper_mte_check_1_aarch64
 #define helper_mte_checkN helper_mte_checkN_aarch64
 #define helper_neon_addlp_s16 helper_neon_addlp_s16_aarch64
 #define helper_neon_addlp_s8 helper_neon_addlp_s8_aarch64
@@ -4526,6 +4526,8 @@
 #define helper_xpacd helper_xpacd_aarch64
 #define helper_xpaci helper_xpaci_aarch64
 #define logic_imm_decode_wmask logic_imm_decode_wmask_aarch64
+#define mte_check1 mte_check1_aarch64
+#define mte_probe1 mte_probe1_aarch64
 #define new_tmp_a64 new_tmp_a64_aarch64
 #define new_tmp_a64_zero new_tmp_a64_zero_aarch64
 #define pmsav8_mpu_lookup pmsav8_mpu_lookup_aarch64

qemu/aarch64eb.h

@@ -3583,7 +3583,7 @@
 #define helper_msr_i_daifclear helper_msr_i_daifclear_aarch64eb
 #define helper_msr_i_daifset helper_msr_i_daifset_aarch64eb
 #define helper_msr_i_spsel helper_msr_i_spsel_aarch64eb
-#define helper_mte_check1 helper_mte_check1_aarch64eb
+#define helper_mte_check_1 helper_mte_check_1_aarch64eb
 #define helper_mte_checkN helper_mte_checkN_aarch64eb
 #define helper_neon_addlp_s16 helper_neon_addlp_s16_aarch64eb
 #define helper_neon_addlp_s8 helper_neon_addlp_s8_aarch64eb
@@ -4526,6 +4526,8 @@
 #define helper_xpacd helper_xpacd_aarch64eb
 #define helper_xpaci helper_xpaci_aarch64eb
 #define logic_imm_decode_wmask logic_imm_decode_wmask_aarch64eb
+#define mte_check1 mte_check1_aarch64eb
+#define mte_probe1 mte_probe1_aarch64eb
 #define new_tmp_a64 new_tmp_a64_aarch64eb
 #define new_tmp_a64_zero new_tmp_a64_zero_aarch64eb
 #define pmsav8_mpu_lookup pmsav8_mpu_lookup_aarch64eb

qemu/arm.h

@@ -3515,6 +3515,8 @@
 #define helper_gvec_usra_s helper_gvec_usra_s_arm
 #define helper_probe_access_armfn helper_probe_access_armfn_arm
 #define helper_vjcvt helper_vjcvt_arm
+#define mte_check1 mte_check1_arm
+#define mte_probe1 mte_probe1_arm
 #define pmu_init pmu_init_arm
 #define pmsav8_mpu_lookup pmsav8_mpu_lookup_arm
 #define pmu_op_start pmu_op_start_arm

qemu/armeb.h

@@ -3515,6 +3515,8 @@
 #define helper_gvec_usra_s helper_gvec_usra_s_armeb
 #define helper_probe_access_armfn helper_probe_access_armfn_armeb
 #define helper_vjcvt helper_vjcvt_armeb
+#define mte_check1 mte_check1_armeb
+#define mte_probe1 mte_probe1_armeb
 #define pmu_init pmu_init_armeb
 #define pmsav8_mpu_lookup pmsav8_mpu_lookup_armeb
 #define pmu_op_start pmu_op_start_armeb

qemu/header_gen.py

@@ -3524,6 +3524,8 @@ arm_symbols = (
     'helper_gvec_usra_s',
     'helper_probe_access_armfn',
     'helper_vjcvt',
+    'mte_check1',
+    'mte_probe1',
     'pmu_init',
     'pmsav8_mpu_lookup',
     'pmu_op_start',
@@ -3718,7 +3720,7 @@ aarch64_symbols = (
     'helper_msr_i_daifclear',
     'helper_msr_i_daifset',
     'helper_msr_i_spsel',
-    'helper_mte_check1',
+    'helper_mte_check_1',
     'helper_mte_checkN',
     'helper_neon_addlp_s16',
     'helper_neon_addlp_s8',
@@ -4661,6 +4663,8 @@ aarch64_symbols = (
     'helper_xpacd',
     'helper_xpaci',
     'logic_imm_decode_wmask',
+    'mte_check1',
+    'mte_probe1',
     'new_tmp_a64',
     'new_tmp_a64_zero',
     'pmsav8_mpu_lookup',

qemu/target/arm/helper-a64.h

@@ -104,7 +104,8 @@ DEF_HELPER_FLAGS_3(autdb, TCG_CALL_NO_WG, i64, env, i64, i64)
 DEF_HELPER_FLAGS_2(xpaci, TCG_CALL_NO_RWG_SE, i64, env, i64)
 DEF_HELPER_FLAGS_2(xpacd, TCG_CALL_NO_RWG_SE, i64, env, i64)
 
-DEF_HELPER_FLAGS_3(mte_check1, TCG_CALL_NO_WG, i64, env, i32, i64)
+// Named "mte_check1" in mainline qemu. Renamed to avoid header gen conflicts
+DEF_HELPER_FLAGS_3(mte_check_1, TCG_CALL_NO_WG, i64, env, i32, i64)
 DEF_HELPER_FLAGS_3(mte_checkN, TCG_CALL_NO_WG, i64, env, i32, i64)
 DEF_HELPER_FLAGS_3(irg, TCG_CALL_NO_RWG, i64, env, i64, i64)
 DEF_HELPER_FLAGS_4(addsubg, TCG_CALL_NO_RWG_SE, i64, env, i64, s32, i32)

qemu/target/arm/helper.c

@@ -3164,7 +3164,7 @@ static uint64_t do_ats_write(CPUARMState *env, uint64_t value,
              * Report exception with ESR indicating a fault due to a
              * translation table walk for a cache maintenance instruction.
              */
-            syn = syn_data_abort_no_iss(current_el == target_el,
+            syn = syn_data_abort_no_iss(current_el == target_el, 0,
                                         fi.ea, 1, fi.s1ptw, 1, fsc);
             env->exception.vaddress = value;
             env->exception.fsr = fsr;

qemu/target/arm/internals.h

@@ -453,13 +453,14 @@ static inline uint32_t syn_insn_abort(int same_el, int ea, int s1ptw, int fsc)
         | ARM_EL_IL | (ea << 9) | (s1ptw << 7) | fsc;
 }
 
-static inline uint32_t syn_data_abort_no_iss(int same_el,
+static inline uint32_t syn_data_abort_no_iss(int same_el, int fnv,
                                              int ea, int cm, int s1ptw,
                                              int wnr, int fsc)
 {
     return (EC_DATAABORT << ARM_EL_EC_SHIFT) | (same_el << ARM_EL_EC_SHIFT)
            | ARM_EL_IL
-           | (ea << 9) | (cm << 8) | (s1ptw << 7) | (wnr << 6) | fsc;
+           | (fnv << 10) | (ea << 9) | (cm << 8) | (s1ptw << 7)
+           | (wnr << 6) | fsc;
 }
 
 static inline uint32_t syn_data_abort_with_iss(int same_el,
@@ -1322,6 +1323,11 @@ FIELD(MTEDESC, WRITE, 8, 1)
 FIELD(MTEDESC, ESIZE, 9, 5)
 FIELD(MTEDESC, TSIZE, 14, 10)  /* mte_checkN only */
 
+bool mte_probe1(CPUARMState *env, uint32_t desc, uint64_t ptr);
+uint64_t mte_check1(CPUARMState *env, uint32_t desc,
+                    uint64_t ptr, uintptr_t ra);
+
+
 static inline int allocation_tag_from_addr(uint64_t ptr)
 {
     return extract64(ptr, 56, 4);
@@ -1332,4 +1338,48 @@ static inline uint64_t address_with_allocation_tag(uint64_t ptr, int rtag)
     return deposit64(ptr, 56, 4, rtag);
 }
 
+/* Return true if tbi bits mean that the access is checked.  */
+static inline bool tbi_check(uint32_t desc, int bit55)
+{
+    return (desc >> (R_MTEDESC_TBI_SHIFT + bit55)) & 1;
+}
+
+/* Return true if tcma bits mean that the access is unchecked.  */
+static inline bool tcma_check(uint32_t desc, int bit55, int ptr_tag)
+{
+    /*
+     * We had extracted bit55 and ptr_tag for other reasons, so fold
+     * (ptr<59:55> == 00000 || ptr<59:55> == 11111) into a single test.
+     */
+    bool match = ((ptr_tag + bit55) & 0xf) == 0;
+    bool tcma = (desc >> (R_MTEDESC_TCMA_SHIFT + bit55)) & 1;
+    return tcma && match;
+}
+
+/*
+ * For TBI, ideally, we would do nothing.  Proper behaviour on fault is
+ * for the tag to be present in the FAR_ELx register.  But for user-only
+ * mode, we do not have a TLB with which to implement this, so we must
+ * remove the top byte.
+ */
+static inline uint64_t useronly_clean_ptr(uint64_t ptr)
+{
+    /* TBI is known to be enabled. */
+#ifdef CONFIG_USER_ONLY
+    ptr = sextract64(ptr, 0, 56);
+#endif
+    return ptr;
+}
+
+static inline uint64_t useronly_maybe_clean_ptr(uint32_t desc, uint64_t ptr)
+{
+#ifdef CONFIG_USER_ONLY
+    int64_t clean_ptr = sextract64(ptr, 0, 56);
+    if (tbi_check(desc, clean_ptr < 0)) {
+        ptr = clean_ptr;
+    }
+#endif
+    return ptr;
+}
+
 #endif

qemu/target/arm/mte_helper.c

@@ -359,12 +359,142 @@ void HELPER(stzgm_tags)(CPUARMState *env, uint64_t ptr, uint64_t val)
     }
 }
 
+/* Record a tag check failure.  */
+static void mte_check_fail(CPUARMState *env, int mmu_idx,
+                           uint64_t dirty_ptr, uintptr_t ra)
+{
+    ARMMMUIdx arm_mmu_idx = core_to_aa64_mmu_idx(mmu_idx);
+    int el, reg_el, tcf, select;
+    uint64_t sctlr;
+
+    reg_el = regime_el(env, arm_mmu_idx);
+    sctlr = env->cp15.sctlr_el[reg_el];
+
+    switch (arm_mmu_idx) {
+    case ARMMMUIdx_E10_0:
+    case ARMMMUIdx_E20_0:
+        el = 0;
+        tcf = extract64(sctlr, 38, 2);
+        break;
+    default:
+        el = reg_el;
+        tcf = extract64(sctlr, 40, 2);
+    }
+
+    switch (tcf) {
+    case 1:
+        /*
+         * Tag check fail causes a synchronous exception.
+         *
+         * In restore_state_to_opc, we set the exception syndrome
+         * for the load or store operation.  Unwind first so we
+         * may overwrite that with the syndrome for the tag check.
+         */
+        cpu_restore_state(env_cpu(env), ra, true);
+        env->exception.vaddress = dirty_ptr;
+        raise_exception(env, EXCP_DATA_ABORT,
+                        syn_data_abort_no_iss(el != 0, 0, 0, 0, 0, 0, 0x11),
+                        exception_target_el(env));
+        /* noreturn, but fall through to the assert anyway */
+
+    case 0:
+        /*
+         * Tag check fail does not affect the PE.
+         * We eliminate this case by not setting MTE_ACTIVE
+         * in tb_flags, so that we never make this runtime call.
+         */
+        g_assert_not_reached();
+
+    case 2:
+        /* Tag check fail causes asynchronous flag set.  */
+        mmu_idx = arm_mmu_idx_el(env, el);
+        if (regime_has_2_ranges(mmu_idx)) {
+            select = extract64(dirty_ptr, 55, 1);
+        } else {
+            select = 0;
+        }
+        env->cp15.tfsr_el[el] |= 1 << select;
+        break;
+
+    default:
+        /* Case 3: Reserved. */
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "Tag check failure with SCTLR_EL%d.TCF%s "
+                      "set to reserved value %d\n",
+                      reg_el, el ? "" : "0", tcf);
+        break;
+    }
+}
+
 /*
  * Perform an MTE checked access for a single logical or atomic access.
  */
-uint64_t HELPER(mte_check1)(CPUARMState *env, uint32_t desc, uint64_t ptr)
+static bool mte_probe1_int(CPUARMState *env, uint32_t desc, uint64_t ptr,
+                           uintptr_t ra, int bit55)
 {
-    return ptr;
+    int mem_tag, mmu_idx, ptr_tag, size;
+    MMUAccessType type;
+    uint8_t *mem;
+
+    ptr_tag = allocation_tag_from_addr(ptr);
+
+    if (tcma_check(desc, bit55, ptr_tag)) {
+        return true;
+    }
+
+    mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
+    type = FIELD_EX32(desc, MTEDESC, WRITE) ? MMU_DATA_STORE : MMU_DATA_LOAD;
+    size = FIELD_EX32(desc, MTEDESC, ESIZE);
+
+    mem = allocation_tag_mem(env, mmu_idx, ptr, type, size,
+                             MMU_DATA_LOAD, 1, ra);
+    if (!mem) {
+        return true;
+    }
+
+    mem_tag = load_tag1(ptr, mem);
+    return ptr_tag == mem_tag;
+}
+
+/*
+ * No-fault version of mte_check1, to be used by SVE for MemSingleNF.
+ * Returns false if the access is Checked and the check failed.  This
+ * is only intended to probe the tag -- the validity of the page must
+ * be checked beforehand.
+ */
+bool mte_probe1(CPUARMState *env, uint32_t desc, uint64_t ptr)
+{
+    int bit55 = extract64(ptr, 55, 1);
+
+    /* If TBI is disabled, the access is unchecked. */
+    if (unlikely(!tbi_check(desc, bit55))) {
+        return true;
+    }
+
+    return mte_probe1_int(env, desc, ptr, 0, bit55);
+}
+
+uint64_t mte_check1(CPUARMState *env, uint32_t desc,
+                    uint64_t ptr, uintptr_t ra)
+{
+    int bit55 = extract64(ptr, 55, 1);
+
+    /* If TBI is disabled, the access is unchecked, and ptr is not dirty. */
+    if (unlikely(!tbi_check(desc, bit55))) {
+        return ptr;
+    }
+
+    if (unlikely(!mte_probe1_int(env, desc, ptr, ra, bit55))) {
+        int mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
+        mte_check_fail(env, mmu_idx, ptr, ra);
+    }
+
+    return useronly_clean_ptr(ptr);
+}
+
+uint64_t HELPER(mte_check_1)(CPUARMState *env, uint32_t desc, uint64_t ptr)
+{
+    return mte_check1(env, desc, ptr, GETPC());
 }
 
 /*

qemu/target/arm/tlb_helper.c

@@ -31,7 +31,7 @@ static inline uint32_t merge_syn_data_abort(uint32_t template_syn,
      * ISV field.
      */
     if (!(template_syn & ARM_EL_ISV) || target_el != 2 || s1ptw) {
-        syn = syn_data_abort_no_iss(same_el,
+        syn = syn_data_abort_no_iss(same_el, 0,
                                     ea, 0, s1ptw, is_write, fsc);
     } else {
         /*

qemu/target/arm/translate-a64.c

@@ -407,7 +407,7 @@ static TCGv_i64 gen_mte_check1_mmuidx(DisasContext *s, TCGv_i64 addr,
         tcg_desc = tcg_const_i32(tcg_ctx, desc);
 
         ret = new_tmp_a64(s);
-        gen_helper_mte_check1(tcg_ctx, ret, tcg_ctx->cpu_env, tcg_desc, addr);
+        gen_helper_mte_check_1(tcg_ctx, ret, tcg_ctx->cpu_env, tcg_desc, addr);
         tcg_temp_free_i32(tcg_ctx, tcg_desc);
 
         return ret;