target/arm: Implement LDG, STG, ST2G instructions

Backports commit c15294c1e36a7dd9b25bd54d98178e80f4b64bc1 from qemu

qemu/aarch64.h

@@ -3577,6 +3577,7 @@
 #define helper_gvec_usra_h helper_gvec_usra_h_aarch64
 #define helper_gvec_usra_s helper_gvec_usra_s_aarch64
 #define helper_irg helper_irg_aarch64
+#define helper_ldg helper_ldg_aarch64
 #define helper_msr_i_daifclear helper_msr_i_daifclear_aarch64
 #define helper_msr_i_daifset helper_msr_i_daifset_aarch64
 #define helper_msr_i_spsel helper_msr_i_spsel_aarch64
@@ -3596,6 +3597,7 @@
 #define helper_paired_cmpxchg64_be_parallel helper_paired_cmpxchg64_be_parallel_aarch64
 #define helper_paired_cmpxchg64_le helper_paired_cmpxchg64_le_aarch64
 #define helper_paired_cmpxchg64_le_parallel helper_paired_cmpxchg64_le_parallel_aarch64
+#define helper_probe_access_armfn helper_probe_access_armfn_aarch64
 #define helper_rbit64 helper_rbit64_aarch64
 #define helper_recpsf_f16 helper_recpsf_f16_aarch64
 #define helper_recpsf_f32 helper_recpsf_f32_aarch64
@@ -3606,6 +3608,12 @@
 #define helper_sdiv64 helper_sdiv64_aarch64
 #define helper_simd_tbl helper_simd_tbl_aarch64
 #define helper_sqrt_f16 helper_sqrt_f16_aarch64
+#define helper_st2g helper_st2g_aarch64
+#define helper_st2g_parallel helper_st2g_parallel_aarch64
+#define helper_st2g_stub helper_st2g_stub_aarch64
+#define helper_stg helper_stg_aarch64
+#define helper_stg_parallel helper_stg_parallel_aarch64
+#define helper_stg_stub helper_stg_stub_aarch64
 #define helper_sve_abs_b helper_sve_abs_b_aarch64
 #define helper_sve_abs_d helper_sve_abs_d_aarch64
 #define helper_sve_abs_h helper_sve_abs_h_aarch64

qemu/aarch64eb.h

@@ -3577,6 +3577,7 @@
 #define helper_gvec_usra_h helper_gvec_usra_h_aarch64eb
 #define helper_gvec_usra_s helper_gvec_usra_s_aarch64eb
 #define helper_irg helper_irg_aarch64eb
+#define helper_ldg helper_ldg_aarch64eb
 #define helper_msr_i_daifclear helper_msr_i_daifclear_aarch64eb
 #define helper_msr_i_daifset helper_msr_i_daifset_aarch64eb
 #define helper_msr_i_spsel helper_msr_i_spsel_aarch64eb
@@ -3596,6 +3597,7 @@
 #define helper_paired_cmpxchg64_be_parallel helper_paired_cmpxchg64_be_parallel_aarch64eb
 #define helper_paired_cmpxchg64_le helper_paired_cmpxchg64_le_aarch64eb
 #define helper_paired_cmpxchg64_le_parallel helper_paired_cmpxchg64_le_parallel_aarch64eb
+#define helper_probe_access_armfn helper_probe_access_armfn_aarch64eb
 #define helper_rbit64 helper_rbit64_aarch64eb
 #define helper_recpsf_f16 helper_recpsf_f16_aarch64eb
 #define helper_recpsf_f32 helper_recpsf_f32_aarch64eb
@@ -3606,6 +3608,12 @@
 #define helper_sdiv64 helper_sdiv64_aarch64eb
 #define helper_simd_tbl helper_simd_tbl_aarch64eb
 #define helper_sqrt_f16 helper_sqrt_f16_aarch64eb
+#define helper_st2g helper_st2g_aarch64eb
+#define helper_st2g_parallel helper_st2g_parallel_aarch64eb
+#define helper_st2g_stub helper_st2g_stub_aarch64eb
+#define helper_stg helper_stg_aarch64eb
+#define helper_stg_parallel helper_stg_parallel_aarch64eb
+#define helper_stg_stub helper_stg_stub_aarch64eb
 #define helper_sve_abs_b helper_sve_abs_b_aarch64eb
 #define helper_sve_abs_d helper_sve_abs_d_aarch64eb
 #define helper_sve_abs_h helper_sve_abs_h_aarch64eb

qemu/arm.h

@@ -3512,6 +3512,7 @@
 #define helper_gvec_usra_d helper_gvec_usra_d_arm
 #define helper_gvec_usra_h helper_gvec_usra_h_arm
 #define helper_gvec_usra_s helper_gvec_usra_s_arm
+#define helper_probe_access_armfn helper_probe_access_armfn_arm
 #define helper_vjcvt helper_vjcvt_arm
 #define pmu_init pmu_init_arm
 #define pmsav8_mpu_lookup pmsav8_mpu_lookup_arm

qemu/armeb.h

@@ -3512,6 +3512,7 @@
 #define helper_gvec_usra_d helper_gvec_usra_d_armeb
 #define helper_gvec_usra_h helper_gvec_usra_h_armeb
 #define helper_gvec_usra_s helper_gvec_usra_s_armeb
+#define helper_probe_access_armfn helper_probe_access_armfn_armeb
 #define helper_vjcvt helper_vjcvt_armeb
 #define pmu_init pmu_init_armeb
 #define pmsav8_mpu_lookup pmsav8_mpu_lookup_armeb

qemu/header_gen.py

@@ -3521,6 +3521,7 @@ arm_symbols = (
     'helper_gvec_usra_d',
     'helper_gvec_usra_h',
     'helper_gvec_usra_s',
+    'helper_probe_access_armfn',
     'helper_vjcvt',
     'pmu_init',
     'pmsav8_mpu_lookup',
@@ -3711,6 +3712,7 @@ aarch64_symbols = (
     'helper_gvec_usra_h',
     'helper_gvec_usra_s',
     'helper_irg',
+    'helper_ldg',
     'helper_msr_i_daifclear',
     'helper_msr_i_daifset',
     'helper_msr_i_spsel',
@@ -3730,6 +3732,7 @@ aarch64_symbols = (
     'helper_paired_cmpxchg64_be_parallel',
     'helper_paired_cmpxchg64_le',
     'helper_paired_cmpxchg64_le_parallel',
+    'helper_probe_access_armfn',
     'helper_rbit64',
     'helper_recpsf_f16',
     'helper_recpsf_f32',
@@ -3740,6 +3743,12 @@ aarch64_symbols = (
     'helper_sdiv64',
     'helper_simd_tbl',
     'helper_sqrt_f16',
+    'helper_st2g',
+    'helper_st2g_parallel',
+    'helper_st2g_stub',
+    'helper_stg',
+    'helper_stg_parallel',
+    'helper_stg_stub',
     'helper_sve_abs_b',
     'helper_sve_abs_d',
     'helper_sve_abs_h',

qemu/target/arm/helper-a64.h

@@ -105,4 +105,11 @@ DEF_HELPER_FLAGS_2(xpaci, TCG_CALL_NO_RWG_SE, i64, env, i64)
 DEF_HELPER_FLAGS_2(xpacd, TCG_CALL_NO_RWG_SE, i64, env, i64)
 
 DEF_HELPER_FLAGS_3(irg, TCG_CALL_NO_RWG, i64, env, i64, i64)
-DEF_HELPER_FLAGS_4(addsubg, TCG_CALL_NO_RWG_SE, i64, env, i64, s32, i32)
+DEF_HELPER_FLAGS_4(addsubg, TCG_CALL_NO_RWG_SE, i64, env, i64, s32, i32)
+DEF_HELPER_FLAGS_3(ldg, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(stg, TCG_CALL_NO_WG, void, env, i64, i64)
+DEF_HELPER_FLAGS_3(stg_parallel, TCG_CALL_NO_WG, void, env, i64, i64)
+DEF_HELPER_FLAGS_2(stg_stub, TCG_CALL_NO_WG, void, env, i64)
+DEF_HELPER_FLAGS_3(st2g, TCG_CALL_NO_WG, void, env, i64, i64)
+DEF_HELPER_FLAGS_3(st2g_parallel, TCG_CALL_NO_WG, void, env, i64, i64)
+DEF_HELPER_FLAGS_2(st2g_stub, TCG_CALL_NO_WG, void, env, i64)

qemu/target/arm/helper.h

@@ -92,6 +92,10 @@ DEF_HELPER_4(msr_banked, void, env, i32, i32, i32)
 DEF_HELPER_2(get_user_reg, i32, env, i32)
 DEF_HELPER_3(set_user_reg, void, env, i32, i32)
 
+// Is "probe_access" in mainline qemu. We rename it to work
+// around header_gen.py shenanigans.
+DEF_HELPER_FLAGS_5(probe_access_armfn, TCG_CALL_NO_WG, void, env, tl, i32, i32, i32)
+
 DEF_HELPER_1(vfp_get_fpscr, i32, env)
 DEF_HELPER_2(vfp_set_fpscr, void, env, i32)
 

qemu/target/arm/op_helper.c

@@ -927,3 +927,19 @@ uint32_t HELPER(ror_cc)(CPUARMState *env, uint32_t x, uint32_t i)
         return ((uint32_t)x >> shift) | (x << (32 - shift));
     }
 }
+
+void HELPER(probe_access_armfn)(CPUARMState *env, target_ulong ptr,
+                                uint32_t access_type, uint32_t mmu_idx,
+                                uint32_t size)
+{
+    uint32_t in_page = -((uint32_t)ptr | TARGET_PAGE_SIZE);
+    uintptr_t ra = GETPC();
+
+    if (likely(size <= in_page)) {
+        probe_access(env, ptr, size, access_type, mmu_idx, ra);
+    } else {
+        probe_access(env, ptr, in_page, access_type, mmu_idx, ra);
+        probe_access(env, ptr + in_page, size - in_page,
+                     access_type, mmu_idx, ra);
+    }
+}

qemu/target/arm/translate-a64.c

@@ -369,6 +369,21 @@ static void gen_address_with_allocation_tag0(TCGContext *s, TCGv_i64 dst, TCGv_i
     tcg_gen_andi_i64(s, dst, src, ~MAKE_64BIT_MASK(56, 4));
 }
 
+static void gen_probe_access(DisasContext *s, TCGv_i64 ptr,
+                             MMUAccessType acc, int log2_size)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    TCGv_i32 t_acc = tcg_const_i32(tcg_ctx, acc);
+    TCGv_i32 t_idx = tcg_const_i32(tcg_ctx, get_mem_index(s));
+    TCGv_i32 t_size = tcg_const_i32(tcg_ctx, 1 << log2_size);
+
+    gen_helper_probe_access_armfn(tcg_ctx, tcg_ctx->cpu_env, ptr, t_acc, t_idx, t_size);
+    tcg_temp_free_i32(tcg_ctx, t_acc);
+    tcg_temp_free_i32(tcg_ctx, t_idx);
+    tcg_temp_free_i32(tcg_ctx, t_size);
+}
+
+
 typedef struct DisasCompare64 {
     TCGCond cond;
     TCGv_i64 value;
@@ -3879,6 +3894,155 @@ static void disas_ldst_single_struct(DisasContext *s, uint32_t insn)
     }
 }
 
+/*
+ * Load/Store memory tags
+ *
+ *  31 30 29         24     22  21     12    10      5      0
+ * +-----+-------------+-----+---+------+-----+------+------+
+ * | 1 1 | 0 1 1 0 0 1 | op1 | 1 | imm9 | op2 |  Rn  |  Rt  |
+ * +-----+-------------+-----+---+------+-----+------+------+
+ */
+static void disas_ldst_tag(DisasContext *s, uint32_t insn)
+{
+    TCGContext *tcg_ctx = s->uc->tcg_ctx;
+    int rt = extract32(insn, 0, 5);
+    int rn = extract32(insn, 5, 5);
+    uint64_t offset = sextract64(insn, 12, 9) << LOG2_TAG_GRANULE;
+    int op2 = extract32(insn, 10, 2);
+    int op1 = extract32(insn, 22, 2);
+    bool is_load = false, is_pair = false, is_zero = false;
+    int index = 0;
+    TCGv_i64 addr, clean_addr, tcg_rt;
+
+    /* We checked insn bits [29:24,21] in the caller.  */
+    if (extract32(insn, 30, 2) != 3) {
+        goto do_unallocated;
+    }
+
+    /*
+     * @index is a tri-state variable which has 3 states:
+     * < 0 : post-index, writeback
+     * = 0 : signed offset
+     * > 0 : pre-index, writeback
+     */
+    switch (op1) {
+    case 0:
+        if (op2 != 0) {
+            /* STG */
+            index = op2 - 2;
+            break;
+        }
+        goto do_unallocated;
+    case 1:
+        if (op2 != 0) {
+            /* STZG */
+            is_zero = true;
+            index = op2 - 2;
+        } else {
+            /* LDG */
+            is_load = true;
+        }
+        break;
+    case 2:
+        if (op2 != 0) {
+            /* ST2G */
+            is_pair = true;
+            index = op2 - 2;
+            break;
+        }
+        goto do_unallocated;
+    case 3:
+        if (op2 != 0) {
+            /* STZ2G */
+            is_pair = is_zero = true;
+            index = op2 - 2;
+            break;
+        }
+        goto do_unallocated;
+
+    default:
+    do_unallocated:
+        unallocated_encoding(s);
+        return;
+    }
+
+    if (!dc_isar_feature(aa64_mte_insn_reg, s)) {
+        goto do_unallocated;
+    }
+
+    if (rn == 31) {
+        gen_check_sp_alignment(s);
+    }
+
+    addr = read_cpu_reg_sp(s, rn, true);
+    if (index >= 0) {
+        /* pre-index or signed offset */
+        tcg_gen_addi_i64(tcg_ctx, addr, addr, offset);
+    }
+
+    if (is_load) {
+        tcg_gen_andi_i64(tcg_ctx, addr, addr, -TAG_GRANULE);
+        tcg_rt = cpu_reg(s, rt);
+        if (s->ata) {
+            gen_helper_ldg(tcg_ctx, tcg_rt, tcg_ctx->cpu_env, addr, tcg_rt);
+        } else {
+            clean_addr = clean_data_tbi(s, addr);
+            gen_probe_access(s, clean_addr, MMU_DATA_LOAD, MO_8);
+            gen_address_with_allocation_tag0(tcg_ctx, tcg_rt, addr);
+        }
+    } else {
+        tcg_rt = cpu_reg_sp(s, rt);
+        if (!s->ata) {
+            /*
+             * For STG and ST2G, we need to check alignment and probe memory.
+             * TODO: For STZG and STZ2G, we could rely on the stores below,
+             * at least for system mode; user-only won't enforce alignment.
+             */
+            if (is_pair) {
+                gen_helper_st2g_stub(tcg_ctx, tcg_ctx->cpu_env, addr);
+            } else {
+                gen_helper_stg_stub(tcg_ctx, tcg_ctx->cpu_env, addr);
+            }
+        } else if (tb_cflags(s->base.tb) & CF_PARALLEL) {
+            if (is_pair) {
+                gen_helper_st2g_parallel(tcg_ctx, tcg_ctx->cpu_env, addr, tcg_rt);
+            } else {
+                gen_helper_stg_parallel(tcg_ctx, tcg_ctx->cpu_env, addr, tcg_rt);
+            }
+        } else {
+            if (is_pair) {
+                gen_helper_st2g(tcg_ctx, tcg_ctx->cpu_env, addr, tcg_rt);
+            } else {
+                gen_helper_stg(tcg_ctx, tcg_ctx->cpu_env, addr, tcg_rt);
+            }
+        }
+    }
+
+    if (is_zero) {
+        TCGv_i64 clean_addr = clean_data_tbi(s, addr);
+        TCGv_i64 tcg_zero = tcg_const_i64(tcg_ctx, 0);
+        int mem_index = get_mem_index(s);
+        int i, n = (1 + is_pair) << LOG2_TAG_GRANULE;
+
+        tcg_gen_qemu_st_i64(s->uc, tcg_zero, clean_addr, mem_index,
+                            MO_Q | MO_ALIGN_16);
+        for (i = 8; i < n; i += 8) {
+            tcg_gen_addi_i64(tcg_ctx, clean_addr, clean_addr, 8);
+            tcg_gen_qemu_st_i64(s->uc, tcg_zero, clean_addr, mem_index, MO_Q);
+        }
+        tcg_temp_free_i64(tcg_ctx, tcg_zero);
+    }
+
+    if (index != 0) {
+        /* pre-index or post-index */
+        if (index < 0) {
+            /* post-index */
+            tcg_gen_addi_i64(tcg_ctx, addr, addr, offset);
+        }
+        tcg_gen_mov_i64(tcg_ctx, cpu_reg_sp(s, rn), addr);
+    }
+}
+
 /* Loads and stores */
 static void disas_ldst(DisasContext *s, uint32_t insn)
 {
@@ -3903,13 +4067,14 @@ static void disas_ldst(DisasContext *s, uint32_t insn)
     case 0x0d: /* AdvSIMD load/store single structure */
         disas_ldst_single_struct(s, insn);
         break;
-    case 0x19: /* LDAPR/STLR (unscaled immediate) */
+    case 0x19:
-        if (extract32(insn, 10, 2) != 0 ||
+        if (extract32(insn, 21, 1) != 0) {
-            extract32(insn, 21, 1) != 0) {
+            disas_ldst_tag(s, insn);
+        } else if (extract32(insn, 10, 2) == 0) {
+            disas_ldst_ldapr_stlr(s, insn);
+        } else {
             unallocated_encoding(s);
-            break;
         }
-        disas_ldst_ldapr_stlr(s, insn);
         break;
     default:
         unallocated_encoding(s);