target/arm: Implement fp16 for Neon VFMA, VMFS

Convert the neon floating-point vector operations VFMA and VFMS
to use a gvec helper, and use this to implement the fp16 case.

This is the last use of do_3same_fp() so we can now delete
that function.

Backports commit cf722d75b329ef3f86b869e7e68cbfb1607b3bde

qemu/aarch64.h

@@ -1473,6 +1473,10 @@
 #define helper_gvec_ussub16 helper_gvec_ussub16_aarch64
 #define helper_gvec_ussub32 helper_gvec_ussub32_aarch64
 #define helper_gvec_ussub64 helper_gvec_ussub64_aarch64
+#define helper_gvec_vfma_h helper_gvec_vfma_h_aarch64
+#define helper_gvec_vfma_s helper_gvec_vfma_s_aarch64
+#define helper_gvec_vfms_h helper_gvec_vfms_h_aarch64
+#define helper_gvec_vfms_s helper_gvec_vfms_s_aarch64
 #define helper_gvec_xor helper_gvec_xor_aarch64
 #define helper_gvec_xors helper_gvec_xors_aarch64
 #define helper_iwmmxt_addcb helper_iwmmxt_addcb_aarch64

qemu/aarch64eb.h

@@ -1473,6 +1473,10 @@
 #define helper_gvec_ussub16 helper_gvec_ussub16_aarch64eb
 #define helper_gvec_ussub32 helper_gvec_ussub32_aarch64eb
 #define helper_gvec_ussub64 helper_gvec_ussub64_aarch64eb
+#define helper_gvec_vfma_h helper_gvec_vfma_h_aarch64eb
+#define helper_gvec_vfma_s helper_gvec_vfma_s_aarch64eb
+#define helper_gvec_vfms_h helper_gvec_vfms_h_aarch64eb
+#define helper_gvec_vfms_s helper_gvec_vfms_s_aarch64eb
 #define helper_gvec_xor helper_gvec_xor_aarch64eb
 #define helper_gvec_xors helper_gvec_xors_aarch64eb
 #define helper_iwmmxt_addcb helper_iwmmxt_addcb_aarch64eb

qemu/arm.h

@@ -1473,6 +1473,10 @@
 #define helper_gvec_ussub16 helper_gvec_ussub16_arm
 #define helper_gvec_ussub32 helper_gvec_ussub32_arm
 #define helper_gvec_ussub64 helper_gvec_ussub64_arm
+#define helper_gvec_vfma_h helper_gvec_vfma_h_arm
+#define helper_gvec_vfma_s helper_gvec_vfma_s_arm
+#define helper_gvec_vfms_h helper_gvec_vfms_h_arm
+#define helper_gvec_vfms_s helper_gvec_vfms_s_arm
 #define helper_gvec_xor helper_gvec_xor_arm
 #define helper_gvec_xors helper_gvec_xors_arm
 #define helper_iwmmxt_addcb helper_iwmmxt_addcb_arm

qemu/armeb.h

@@ -1473,6 +1473,10 @@
 #define helper_gvec_ussub16 helper_gvec_ussub16_armeb
 #define helper_gvec_ussub32 helper_gvec_ussub32_armeb
 #define helper_gvec_ussub64 helper_gvec_ussub64_armeb
+#define helper_gvec_vfma_h helper_gvec_vfma_h_armeb
+#define helper_gvec_vfma_s helper_gvec_vfma_s_armeb
+#define helper_gvec_vfms_h helper_gvec_vfms_h_armeb
+#define helper_gvec_vfms_s helper_gvec_vfms_s_armeb
 #define helper_gvec_xor helper_gvec_xor_armeb
 #define helper_gvec_xors helper_gvec_xors_armeb
 #define helper_iwmmxt_addcb helper_iwmmxt_addcb_armeb

qemu/header_gen.py

@@ -1479,6 +1479,10 @@ symbols = (
     'helper_gvec_ussub16',
     'helper_gvec_ussub32',
     'helper_gvec_ussub64',
+    'helper_gvec_vfma_h',
+    'helper_gvec_vfma_s',
+    'helper_gvec_vfms_h',
+    'helper_gvec_vfms_s',
     'helper_gvec_xor',
     'helper_gvec_xors',
     'helper_iwmmxt_addcb',

qemu/m68k.h

@@ -1473,6 +1473,10 @@
 #define helper_gvec_ussub16 helper_gvec_ussub16_m68k
 #define helper_gvec_ussub32 helper_gvec_ussub32_m68k
 #define helper_gvec_ussub64 helper_gvec_ussub64_m68k
+#define helper_gvec_vfma_h helper_gvec_vfma_h_m68k
+#define helper_gvec_vfma_s helper_gvec_vfma_s_m68k
+#define helper_gvec_vfms_h helper_gvec_vfms_h_m68k
+#define helper_gvec_vfms_s helper_gvec_vfms_s_m68k
 #define helper_gvec_xor helper_gvec_xor_m68k
 #define helper_gvec_xors helper_gvec_xors_m68k
 #define helper_iwmmxt_addcb helper_iwmmxt_addcb_m68k

qemu/mips.h

@@ -1473,6 +1473,10 @@
 #define helper_gvec_ussub16 helper_gvec_ussub16_mips
 #define helper_gvec_ussub32 helper_gvec_ussub32_mips
 #define helper_gvec_ussub64 helper_gvec_ussub64_mips
+#define helper_gvec_vfma_h helper_gvec_vfma_h_mips
+#define helper_gvec_vfma_s helper_gvec_vfma_s_mips
+#define helper_gvec_vfms_h helper_gvec_vfms_h_mips
+#define helper_gvec_vfms_s helper_gvec_vfms_s_mips
 #define helper_gvec_xor helper_gvec_xor_mips
 #define helper_gvec_xors helper_gvec_xors_mips
 #define helper_iwmmxt_addcb helper_iwmmxt_addcb_mips

qemu/mips64.h

@@ -1473,6 +1473,10 @@
 #define helper_gvec_ussub16 helper_gvec_ussub16_mips64
 #define helper_gvec_ussub32 helper_gvec_ussub32_mips64
 #define helper_gvec_ussub64 helper_gvec_ussub64_mips64
+#define helper_gvec_vfma_h helper_gvec_vfma_h_mips64
+#define helper_gvec_vfma_s helper_gvec_vfma_s_mips64
+#define helper_gvec_vfms_h helper_gvec_vfms_h_mips64
+#define helper_gvec_vfms_s helper_gvec_vfms_s_mips64
 #define helper_gvec_xor helper_gvec_xor_mips64
 #define helper_gvec_xors helper_gvec_xors_mips64
 #define helper_iwmmxt_addcb helper_iwmmxt_addcb_mips64

qemu/mips64el.h

@@ -1473,6 +1473,10 @@
 #define helper_gvec_ussub16 helper_gvec_ussub16_mips64el
 #define helper_gvec_ussub32 helper_gvec_ussub32_mips64el
 #define helper_gvec_ussub64 helper_gvec_ussub64_mips64el
+#define helper_gvec_vfma_h helper_gvec_vfma_h_mips64el
+#define helper_gvec_vfma_s helper_gvec_vfma_s_mips64el
+#define helper_gvec_vfms_h helper_gvec_vfms_h_mips64el
+#define helper_gvec_vfms_s helper_gvec_vfms_s_mips64el
 #define helper_gvec_xor helper_gvec_xor_mips64el
 #define helper_gvec_xors helper_gvec_xors_mips64el
 #define helper_iwmmxt_addcb helper_iwmmxt_addcb_mips64el

qemu/mipsel.h

@@ -1473,6 +1473,10 @@
 #define helper_gvec_ussub16 helper_gvec_ussub16_mipsel
 #define helper_gvec_ussub32 helper_gvec_ussub32_mipsel
 #define helper_gvec_ussub64 helper_gvec_ussub64_mipsel
+#define helper_gvec_vfma_h helper_gvec_vfma_h_mipsel
+#define helper_gvec_vfma_s helper_gvec_vfma_s_mipsel
+#define helper_gvec_vfms_h helper_gvec_vfms_h_mipsel
+#define helper_gvec_vfms_s helper_gvec_vfms_s_mipsel
 #define helper_gvec_xor helper_gvec_xor_mipsel
 #define helper_gvec_xors helper_gvec_xors_mipsel
 #define helper_iwmmxt_addcb helper_iwmmxt_addcb_mipsel

qemu/powerpc.h

@@ -1473,6 +1473,10 @@
 #define helper_gvec_ussub16 helper_gvec_ussub16_powerpc
 #define helper_gvec_ussub32 helper_gvec_ussub32_powerpc
 #define helper_gvec_ussub64 helper_gvec_ussub64_powerpc
+#define helper_gvec_vfma_h helper_gvec_vfma_h_powerpc
+#define helper_gvec_vfma_s helper_gvec_vfma_s_powerpc
+#define helper_gvec_vfms_h helper_gvec_vfms_h_powerpc
+#define helper_gvec_vfms_s helper_gvec_vfms_s_powerpc
 #define helper_gvec_xor helper_gvec_xor_powerpc
 #define helper_gvec_xors helper_gvec_xors_powerpc
 #define helper_iwmmxt_addcb helper_iwmmxt_addcb_powerpc

qemu/riscv32.h

@@ -1473,6 +1473,10 @@
 #define helper_gvec_ussub16 helper_gvec_ussub16_riscv32
 #define helper_gvec_ussub32 helper_gvec_ussub32_riscv32
 #define helper_gvec_ussub64 helper_gvec_ussub64_riscv32
+#define helper_gvec_vfma_h helper_gvec_vfma_h_riscv32
+#define helper_gvec_vfma_s helper_gvec_vfma_s_riscv32
+#define helper_gvec_vfms_h helper_gvec_vfms_h_riscv32
+#define helper_gvec_vfms_s helper_gvec_vfms_s_riscv32
 #define helper_gvec_xor helper_gvec_xor_riscv32
 #define helper_gvec_xors helper_gvec_xors_riscv32
 #define helper_iwmmxt_addcb helper_iwmmxt_addcb_riscv32

qemu/riscv64.h

@@ -1473,6 +1473,10 @@
 #define helper_gvec_ussub16 helper_gvec_ussub16_riscv64
 #define helper_gvec_ussub32 helper_gvec_ussub32_riscv64
 #define helper_gvec_ussub64 helper_gvec_ussub64_riscv64
+#define helper_gvec_vfma_h helper_gvec_vfma_h_riscv64
+#define helper_gvec_vfma_s helper_gvec_vfma_s_riscv64
+#define helper_gvec_vfms_h helper_gvec_vfms_h_riscv64
+#define helper_gvec_vfms_s helper_gvec_vfms_s_riscv64
 #define helper_gvec_xor helper_gvec_xor_riscv64
 #define helper_gvec_xors helper_gvec_xors_riscv64
 #define helper_iwmmxt_addcb helper_iwmmxt_addcb_riscv64

qemu/sparc.h

@@ -1473,6 +1473,10 @@
 #define helper_gvec_ussub16 helper_gvec_ussub16_sparc
 #define helper_gvec_ussub32 helper_gvec_ussub32_sparc
 #define helper_gvec_ussub64 helper_gvec_ussub64_sparc
+#define helper_gvec_vfma_h helper_gvec_vfma_h_sparc
+#define helper_gvec_vfma_s helper_gvec_vfma_s_sparc
+#define helper_gvec_vfms_h helper_gvec_vfms_h_sparc
+#define helper_gvec_vfms_s helper_gvec_vfms_s_sparc
 #define helper_gvec_xor helper_gvec_xor_sparc
 #define helper_gvec_xors helper_gvec_xors_sparc
 #define helper_iwmmxt_addcb helper_iwmmxt_addcb_sparc

qemu/sparc64.h

@@ -1473,6 +1473,10 @@
 #define helper_gvec_ussub16 helper_gvec_ussub16_sparc64
 #define helper_gvec_ussub32 helper_gvec_ussub32_sparc64
 #define helper_gvec_ussub64 helper_gvec_ussub64_sparc64
+#define helper_gvec_vfma_h helper_gvec_vfma_h_sparc64
+#define helper_gvec_vfma_s helper_gvec_vfma_s_sparc64
+#define helper_gvec_vfms_h helper_gvec_vfms_h_sparc64
+#define helper_gvec_vfms_s helper_gvec_vfms_s_sparc64
 #define helper_gvec_xor helper_gvec_xor_sparc64
 #define helper_gvec_xors helper_gvec_xors_sparc64
 #define helper_iwmmxt_addcb helper_iwmmxt_addcb_sparc64

qemu/target/arm/helper.h

@@ -663,6 +663,12 @@ DEF_HELPER_FLAGS_5(gvec_fmla_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_5(gvec_fmls_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_5(gvec_fmls_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
 
+DEF_HELPER_FLAGS_5(gvec_vfma_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_vfma_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_5(gvec_vfms_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+DEF_HELPER_FLAGS_5(gvec_vfms_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32)
+
 DEF_HELPER_FLAGS_5(gvec_ftsmul_h, TCG_CALL_NO_RWG,
                    void, ptr, ptr, ptr, ptr, i32)
 DEF_HELPER_FLAGS_5(gvec_ftsmul_s, TCG_CALL_NO_RWG,

qemu/target/arm/translate-neon.inc.c

@@ -1051,56 +1051,6 @@ DO_3SAME_PAIR(VPADD, padd_u)
 DO_3SAME_VQDMULH(VQDMULH, qdmulh)
 DO_3SAME_VQDMULH(VQRDMULH, qrdmulh)
 
-static bool do_3same_fp(DisasContext *s, arg_3same *a, VFPGen3OpSPFn *fn,
-                        bool reads_vd)
-{
-    /*
-     * FP operations handled elementwise 32 bits at a time.
-     * If reads_vd is true then the old value of Vd will be
-     * loaded before calling the callback function. This is
-     * used for multiply-accumulate type operations.
-     */
-    TCGv_i32 tmp, tmp2;
-    int pass;
-    TCGContext *tcg_ctx = s->uc->tcg_ctx;
-
-    if (!arm_dc_feature(s, ARM_FEATURE_NEON)) {
-        return false;
-    }
-
-    /* UNDEF accesses to D16-D31 if they don't exist. */
-    if (!dc_isar_feature(aa32_simd_r32, s) &&
-        ((a->vd | a->vn | a->vm) & 0x10)) {
-        return false;
-    }
-
-    if ((a->vn | a->vm | a->vd) & a->q) {
-        return false;
-    }
-
-    if (!vfp_access_check(s)) {
-        return true;
-    }
-
-    TCGv_ptr fpstatus = fpstatus_ptr(tcg_ctx, FPST_STD);
-    for (pass = 0; pass < (a->q ? 4 : 2); pass++) {
-        tmp = neon_load_reg(s, a->vn, pass);
-        tmp2 = neon_load_reg(s, a->vm, pass);
-        if (reads_vd) {
-            TCGv_i32 tmp_rd = neon_load_reg(s, a->vd, pass);
-            fn(tcg_ctx, tmp_rd, tmp, tmp2, fpstatus);
-            neon_store_reg(s, a->vd, pass, tmp_rd);
-            tcg_temp_free_i32(tcg_ctx, tmp);
-        } else {
-            fn(tcg_ctx, tmp, tmp, tmp2, fpstatus);
-            neon_store_reg(s, a->vd, pass, tmp);
-        }
-        tcg_temp_free_i32(tcg_ctx, tmp2);
-    }
-    tcg_temp_free_ptr(tcg_ctx, fpstatus);
-    return true;
-}
-
 #define WRAP_FP_GVEC(WRAPNAME, FPST, FUNC)                              \
     static void WRAPNAME(TCGContext *s, unsigned vece, uint32_t rd_ofs, \
                          uint32_t rn_ofs, uint32_t rm_ofs,              \
@@ -1140,6 +1090,8 @@ DO_3S_FP_GVEC(VMAX, gen_helper_gvec_fmax_s, gen_helper_gvec_fmax_h)
 DO_3S_FP_GVEC(VMIN, gen_helper_gvec_fmin_s, gen_helper_gvec_fmin_h)
 DO_3S_FP_GVEC(VMLA, gen_helper_gvec_fmla_s, gen_helper_gvec_fmla_h)
 DO_3S_FP_GVEC(VMLS, gen_helper_gvec_fmls_s, gen_helper_gvec_fmls_h)
+DO_3S_FP_GVEC(VFMA, gen_helper_gvec_vfma_s, gen_helper_gvec_vfma_h)
+DO_3S_FP_GVEC(VFMS, gen_helper_gvec_vfms_s, gen_helper_gvec_vfms_h)
 
 WRAP_FP_GVEC(gen_VMAXNM_fp32_3s, FPST_STD, gen_helper_gvec_fmaxnum_s)
 WRAP_FP_GVEC(gen_VMAXNM_fp16_3s, FPST_STD_F16, gen_helper_gvec_fmaxnum_h)
@@ -1218,47 +1170,6 @@ static bool trans_VRSQRTS_fp_3s(DisasContext *s, arg_3same *a)
     return do_3same(s, a, gen_VRSQRTS_fp_3s);
 }
 
-static void gen_VFMA_fp_3s(TCGContext *s, TCGv_i32 vd, TCGv_i32 vn, TCGv_i32 vm,
-                           TCGv_ptr fpstatus)
-{
-    gen_helper_vfp_muladds(s, vd, vn, vm, vd, fpstatus);
-}
-
-static bool trans_VFMA_fp_3s(DisasContext *s, arg_3same *a)
-{
-    if (!dc_isar_feature(aa32_simdfmac, s)) {
-        return false;
-    }
-
-    if (a->size != 0) {
-        /* TODO fp16 support */
-        return false;
-    }
-
-    return do_3same_fp(s, a, gen_VFMA_fp_3s, true);
-}
-
-static void gen_VFMS_fp_3s(TCGContext *s, TCGv_i32 vd, TCGv_i32 vn, TCGv_i32 vm,
-                           TCGv_ptr fpstatus)
-{
-    gen_helper_vfp_negs(s, vn, vn);
-    gen_helper_vfp_muladds(s, vd, vn, vm, vd, fpstatus);
-}
-
-static bool trans_VFMS_fp_3s(DisasContext *s, arg_3same *a)
-{
-    if (!dc_isar_feature(aa32_simdfmac, s)) {
-        return false;
-    }
-
-    if (a->size != 0) {
-        /* TODO fp16 support */
-        return false;
-    }
-
-    return do_3same_fp(s, a, gen_VFMS_fp_3s, true);
-}
-
 static bool do_3same_fp_pair(DisasContext *s, arg_3same *a, VFPGen3OpSPFn *fn)
 {
     /* FP operations handled pairwise 32 bits at a time */

qemu/target/arm/vec_helper.c

@@ -868,7 +868,32 @@ static float32 float32_mulsub_nf(float32 dest, float32 op1, float32 op2,
     return float32_sub(dest, float32_mul(op1, op2, stat), stat);
 }
 
-#define DO_MULADD(NAME, FUNC, TYPE) \
+/* Fused versions; these have the semantics Neon VFMA/VFMS want */
+static float16 float16_muladd_f(float16 dest, float16 op1, float16 op2,
+                                float_status *stat)
+{
+    return float16_muladd(op1, op2, dest, 0, stat);
+}
+
+static float32 float32_muladd_f(float32 dest, float32 op1, float32 op2,
+                                 float_status *stat)
+{
+    return float32_muladd(op1, op2, dest, 0, stat);
+}
+
+static float16 float16_mulsub_f(float16 dest, float16 op1, float16 op2,
+                                 float_status *stat)
+{
+    return float16_muladd(float16_chs(op1), op2, dest, 0, stat);
+}
+
+static float32 float32_mulsub_f(float32 dest, float32 op1, float32 op2,
+                                 float_status *stat)
+{
+    return float32_muladd(float32_chs(op1), op2, dest, 0, stat);
+}
+
+#define DO_MULADD(NAME, FUNC, TYPE)                                     \
 void HELPER(NAME)(void *vd, void *vn, void *vm, void *stat, uint32_t desc) \
 {                                                                          \
     intptr_t i, oprsz = simd_oprsz(desc);                                  \
@@ -885,6 +910,12 @@ DO_MULADD(gvec_fmla_s, float32_muladd_nf, float32)
 DO_MULADD(gvec_fmls_h, float16_mulsub_nf, float16)
 DO_MULADD(gvec_fmls_s, float32_mulsub_nf, float32)
 
+DO_MULADD(gvec_vfma_h, float16_muladd_f, float16)
+DO_MULADD(gvec_vfma_s, float32_muladd_f, float32)
+
+DO_MULADD(gvec_vfms_h, float16_mulsub_f, float16)
+DO_MULADD(gvec_vfms_s, float32_mulsub_f, float32)
+
 /* For the indexed ops, SVE applies the index per 128-bit vector segment.
  * For AdvSIMD, there is of course only one such vector segment.
  */

qemu/x86_64.h

@@ -1473,6 +1473,10 @@
 #define helper_gvec_ussub16 helper_gvec_ussub16_x86_64
 #define helper_gvec_ussub32 helper_gvec_ussub32_x86_64
 #define helper_gvec_ussub64 helper_gvec_ussub64_x86_64
+#define helper_gvec_vfma_h helper_gvec_vfma_h_x86_64
+#define helper_gvec_vfma_s helper_gvec_vfma_s_x86_64
+#define helper_gvec_vfms_h helper_gvec_vfms_h_x86_64
+#define helper_gvec_vfms_s helper_gvec_vfms_s_x86_64
 #define helper_gvec_xor helper_gvec_xor_x86_64
 #define helper_gvec_xors helper_gvec_xors_x86_64
 #define helper_iwmmxt_addcb helper_iwmmxt_addcb_x86_64