From 8c6affbca459dd0111ffe662ffe62799fbb5b47b Mon Sep 17 00:00:00 2001 From: Peter Maydell Date: Mon, 1 Mar 2021 17:50:32 -0500 Subject: [PATCH] target/arm/vec_helper: Add gvec fp indexed multiply-and-add operations Add gvec helpers for doing Neon-style indexed non-fused fp multiply-and-accumulate operations. Backports commit c50d8d144098a8261233ca31b47e3bc487e112fe --- qemu/aarch64.h | 4 ++++ qemu/aarch64eb.h | 4 ++++ qemu/arm.h | 4 ++++ qemu/armeb.h | 4 ++++ qemu/header_gen.py | 4 ++++ qemu/m68k.h | 4 ++++ qemu/mips.h | 4 ++++ qemu/mips64.h | 4 ++++ qemu/mips64el.h | 4 ++++ qemu/mipsel.h | 4 ++++ qemu/powerpc.h | 4 ++++ qemu/riscv32.h | 4 ++++ qemu/riscv64.h | 4 ++++ qemu/sparc.h | 4 ++++ qemu/sparc64.h | 4 ++++ qemu/target/arm/helper.h | 10 ++++++++++ qemu/target/arm/vec_helper.c | 28 +++++++++++++++++++++++----- qemu/x86_64.h | 4 ++++ 18 files changed, 97 insertions(+), 5 deletions(-) diff --git a/qemu/aarch64.h b/qemu/aarch64.h index e369df0f..133cf3b3 100644 --- a/qemu/aarch64.h +++ b/qemu/aarch64.h @@ -1852,6 +1852,10 @@ #define helper_gvec_vrint_rm_s helper_gvec_vrint_rm_s_aarch64 #define helper_gvec_vrintx_h helper_gvec_vrintx_h_aarch64 #define helper_gvec_vrintx_s helper_gvec_vrintx_s_aarch64 +#define helper_gvec_fmla_nf_idx_h helper_gvec_fmla_nf_idx_h_aarch64 +#define helper_gvec_fmla_nf_idx_s helper_gvec_fmla_nf_idx_s_aarch64 +#define helper_gvec_fmls_nf_idx_h helper_gvec_fmls_nf_idx_h_aarch64 +#define helper_gvec_fmls_nf_idx_s helper_gvec_fmls_nf_idx_s_aarch64 #define helper_power_down helper_power_down_aarch64 #define helper_pre_hvc helper_pre_hvc_aarch64 #define helper_pre_smc helper_pre_smc_aarch64 diff --git a/qemu/aarch64eb.h b/qemu/aarch64eb.h index 9f9aec0c..0764da58 100644 --- a/qemu/aarch64eb.h +++ b/qemu/aarch64eb.h @@ -1852,6 +1852,10 @@ #define helper_gvec_vrint_rm_s helper_gvec_vrint_rm_s_aarch64eb #define helper_gvec_vrintx_h helper_gvec_vrintx_h_aarch64eb #define helper_gvec_vrintx_s helper_gvec_vrintx_s_aarch64eb +#define helper_gvec_fmla_nf_idx_h helper_gvec_fmla_nf_idx_h_aarch64eb +#define helper_gvec_fmla_nf_idx_s helper_gvec_fmla_nf_idx_s_aarch64eb +#define helper_gvec_fmls_nf_idx_h helper_gvec_fmls_nf_idx_h_aarch64eb +#define helper_gvec_fmls_nf_idx_s helper_gvec_fmls_nf_idx_s_aarch64eb #define helper_power_down helper_power_down_aarch64eb #define helper_pre_hvc helper_pre_hvc_aarch64eb #define helper_pre_smc helper_pre_smc_aarch64eb diff --git a/qemu/arm.h b/qemu/arm.h index 8580732e..75916193 100644 --- a/qemu/arm.h +++ b/qemu/arm.h @@ -1852,6 +1852,10 @@ #define helper_gvec_vrint_rm_s helper_gvec_vrint_rm_s_arm #define helper_gvec_vrintx_h helper_gvec_vrintx_h_arm #define helper_gvec_vrintx_s helper_gvec_vrintx_s_arm +#define helper_gvec_fmla_nf_idx_h helper_gvec_fmla_nf_idx_h_arm +#define helper_gvec_fmla_nf_idx_s helper_gvec_fmla_nf_idx_s_arm +#define helper_gvec_fmls_nf_idx_h helper_gvec_fmls_nf_idx_h_arm +#define helper_gvec_fmls_nf_idx_s helper_gvec_fmls_nf_idx_s_arm #define helper_power_down helper_power_down_arm #define helper_pre_hvc helper_pre_hvc_arm #define helper_pre_smc helper_pre_smc_arm diff --git a/qemu/armeb.h b/qemu/armeb.h index ef394fab..0e8b6f4f 100644 --- a/qemu/armeb.h +++ b/qemu/armeb.h @@ -1852,6 +1852,10 @@ #define helper_gvec_vrint_rm_s helper_gvec_vrint_rm_s_armeb #define helper_gvec_vrintx_h helper_gvec_vrintx_h_armeb #define helper_gvec_vrintx_s helper_gvec_vrintx_s_armeb +#define helper_gvec_fmla_nf_idx_h helper_gvec_fmla_nf_idx_h_armeb +#define helper_gvec_fmla_nf_idx_s helper_gvec_fmla_nf_idx_s_armeb +#define helper_gvec_fmls_nf_idx_h helper_gvec_fmls_nf_idx_h_armeb +#define helper_gvec_fmls_nf_idx_s helper_gvec_fmls_nf_idx_s_armeb #define helper_power_down helper_power_down_armeb #define helper_pre_hvc helper_pre_hvc_armeb #define helper_pre_smc helper_pre_smc_armeb diff --git a/qemu/header_gen.py b/qemu/header_gen.py index 1845d682..fc3de55b 100644 --- a/qemu/header_gen.py +++ b/qemu/header_gen.py @@ -1858,6 +1858,10 @@ symbols = ( 'helper_gvec_vrint_rm_s', 'helper_gvec_vrintx_h', 'helper_gvec_vrintx_s', + 'helper_gvec_fmla_nf_idx_h', + 'helper_gvec_fmla_nf_idx_s', + 'helper_gvec_fmls_nf_idx_h', + 'helper_gvec_fmls_nf_idx_s', 'helper_power_down', 'helper_pre_hvc', 'helper_pre_smc', diff --git a/qemu/m68k.h b/qemu/m68k.h index bc794c91..cbf0eba9 100644 --- a/qemu/m68k.h +++ b/qemu/m68k.h @@ -1852,6 +1852,10 @@ #define helper_gvec_vrint_rm_s helper_gvec_vrint_rm_s_m68k #define helper_gvec_vrintx_h helper_gvec_vrintx_h_m68k #define helper_gvec_vrintx_s helper_gvec_vrintx_s_m68k +#define helper_gvec_fmla_nf_idx_h helper_gvec_fmla_nf_idx_h_m68k +#define helper_gvec_fmla_nf_idx_s helper_gvec_fmla_nf_idx_s_m68k +#define helper_gvec_fmls_nf_idx_h helper_gvec_fmls_nf_idx_h_m68k +#define helper_gvec_fmls_nf_idx_s helper_gvec_fmls_nf_idx_s_m68k #define helper_power_down helper_power_down_m68k #define helper_pre_hvc helper_pre_hvc_m68k #define helper_pre_smc helper_pre_smc_m68k diff --git a/qemu/mips.h b/qemu/mips.h index e2cbb521..ccca0d92 100644 --- a/qemu/mips.h +++ b/qemu/mips.h @@ -1852,6 +1852,10 @@ #define helper_gvec_vrint_rm_s helper_gvec_vrint_rm_s_mips #define helper_gvec_vrintx_h helper_gvec_vrintx_h_mips #define helper_gvec_vrintx_s helper_gvec_vrintx_s_mips +#define helper_gvec_fmla_nf_idx_h helper_gvec_fmla_nf_idx_h_mips +#define helper_gvec_fmla_nf_idx_s helper_gvec_fmla_nf_idx_s_mips +#define helper_gvec_fmls_nf_idx_h helper_gvec_fmls_nf_idx_h_mips +#define helper_gvec_fmls_nf_idx_s helper_gvec_fmls_nf_idx_s_mips #define helper_power_down helper_power_down_mips #define helper_pre_hvc helper_pre_hvc_mips #define helper_pre_smc helper_pre_smc_mips diff --git a/qemu/mips64.h b/qemu/mips64.h index 66439a9c..959babd4 100644 --- a/qemu/mips64.h +++ b/qemu/mips64.h @@ -1852,6 +1852,10 @@ #define helper_gvec_vrint_rm_s helper_gvec_vrint_rm_s_mips64 #define helper_gvec_vrintx_h helper_gvec_vrintx_h_mips64 #define helper_gvec_vrintx_s helper_gvec_vrintx_s_mips64 +#define helper_gvec_fmla_nf_idx_h helper_gvec_fmla_nf_idx_h_mips64 +#define helper_gvec_fmla_nf_idx_s helper_gvec_fmla_nf_idx_s_mips64 +#define helper_gvec_fmls_nf_idx_h helper_gvec_fmls_nf_idx_h_mips64 +#define helper_gvec_fmls_nf_idx_s helper_gvec_fmls_nf_idx_s_mips64 #define helper_power_down helper_power_down_mips64 #define helper_pre_hvc helper_pre_hvc_mips64 #define helper_pre_smc helper_pre_smc_mips64 diff --git a/qemu/mips64el.h b/qemu/mips64el.h index 0a5d3f91..f2af0b30 100644 --- a/qemu/mips64el.h +++ b/qemu/mips64el.h @@ -1852,6 +1852,10 @@ #define helper_gvec_vrint_rm_s helper_gvec_vrint_rm_s_mips64el #define helper_gvec_vrintx_h helper_gvec_vrintx_h_mips64el #define helper_gvec_vrintx_s helper_gvec_vrintx_s_mips64el +#define helper_gvec_fmla_nf_idx_h helper_gvec_fmla_nf_idx_h_mips64el +#define helper_gvec_fmla_nf_idx_s helper_gvec_fmla_nf_idx_s_mips64el +#define helper_gvec_fmls_nf_idx_h helper_gvec_fmls_nf_idx_h_mips64el +#define helper_gvec_fmls_nf_idx_s helper_gvec_fmls_nf_idx_s_mips64el #define helper_power_down helper_power_down_mips64el #define helper_pre_hvc helper_pre_hvc_mips64el #define helper_pre_smc helper_pre_smc_mips64el diff --git a/qemu/mipsel.h b/qemu/mipsel.h index cd673cb3..106cee2f 100644 --- a/qemu/mipsel.h +++ b/qemu/mipsel.h @@ -1852,6 +1852,10 @@ #define helper_gvec_vrint_rm_s helper_gvec_vrint_rm_s_mipsel #define helper_gvec_vrintx_h helper_gvec_vrintx_h_mipsel #define helper_gvec_vrintx_s helper_gvec_vrintx_s_mipsel +#define helper_gvec_fmla_nf_idx_h helper_gvec_fmla_nf_idx_h_mipsel +#define helper_gvec_fmla_nf_idx_s helper_gvec_fmla_nf_idx_s_mipsel +#define helper_gvec_fmls_nf_idx_h helper_gvec_fmls_nf_idx_h_mipsel +#define helper_gvec_fmls_nf_idx_s helper_gvec_fmls_nf_idx_s_mipsel #define helper_power_down helper_power_down_mipsel #define helper_pre_hvc helper_pre_hvc_mipsel #define helper_pre_smc helper_pre_smc_mipsel diff --git a/qemu/powerpc.h b/qemu/powerpc.h index 8afd3500..9d012f19 100644 --- a/qemu/powerpc.h +++ b/qemu/powerpc.h @@ -1852,6 +1852,10 @@ #define helper_gvec_vrint_rm_s helper_gvec_vrint_rm_s_powerpc #define helper_gvec_vrintx_h helper_gvec_vrintx_h_powerpc #define helper_gvec_vrintx_s helper_gvec_vrintx_s_powerpc +#define helper_gvec_fmla_nf_idx_h helper_gvec_fmla_nf_idx_h_powerpc +#define helper_gvec_fmla_nf_idx_s helper_gvec_fmla_nf_idx_s_powerpc +#define helper_gvec_fmls_nf_idx_h helper_gvec_fmls_nf_idx_h_powerpc +#define helper_gvec_fmls_nf_idx_s helper_gvec_fmls_nf_idx_s_powerpc #define helper_power_down helper_power_down_powerpc #define helper_pre_hvc helper_pre_hvc_powerpc #define helper_pre_smc helper_pre_smc_powerpc diff --git a/qemu/riscv32.h b/qemu/riscv32.h index 9a555282..1941e686 100644 --- a/qemu/riscv32.h +++ b/qemu/riscv32.h @@ -1852,6 +1852,10 @@ #define helper_gvec_vrint_rm_s helper_gvec_vrint_rm_s_riscv32 #define helper_gvec_vrintx_h helper_gvec_vrintx_h_riscv32 #define helper_gvec_vrintx_s helper_gvec_vrintx_s_riscv32 +#define helper_gvec_fmla_nf_idx_h helper_gvec_fmla_nf_idx_h_riscv32 +#define helper_gvec_fmla_nf_idx_s helper_gvec_fmla_nf_idx_s_riscv32 +#define helper_gvec_fmls_nf_idx_h helper_gvec_fmls_nf_idx_h_riscv32 +#define helper_gvec_fmls_nf_idx_s helper_gvec_fmls_nf_idx_s_riscv32 #define helper_power_down helper_power_down_riscv32 #define helper_pre_hvc helper_pre_hvc_riscv32 #define helper_pre_smc helper_pre_smc_riscv32 diff --git a/qemu/riscv64.h b/qemu/riscv64.h index 92a8bb20..2277821c 100644 --- a/qemu/riscv64.h +++ b/qemu/riscv64.h @@ -1852,6 +1852,10 @@ #define helper_gvec_vrint_rm_s helper_gvec_vrint_rm_s_riscv64 #define helper_gvec_vrintx_h helper_gvec_vrintx_h_riscv64 #define helper_gvec_vrintx_s helper_gvec_vrintx_s_riscv64 +#define helper_gvec_fmla_nf_idx_h helper_gvec_fmla_nf_idx_h_riscv64 +#define helper_gvec_fmla_nf_idx_s helper_gvec_fmla_nf_idx_s_riscv64 +#define helper_gvec_fmls_nf_idx_h helper_gvec_fmls_nf_idx_h_riscv64 +#define helper_gvec_fmls_nf_idx_s helper_gvec_fmls_nf_idx_s_riscv64 #define helper_power_down helper_power_down_riscv64 #define helper_pre_hvc helper_pre_hvc_riscv64 #define helper_pre_smc helper_pre_smc_riscv64 diff --git a/qemu/sparc.h b/qemu/sparc.h index de57207f..9c5b137a 100644 --- a/qemu/sparc.h +++ b/qemu/sparc.h @@ -1852,6 +1852,10 @@ #define helper_gvec_vrint_rm_s helper_gvec_vrint_rm_s_sparc #define helper_gvec_vrintx_h helper_gvec_vrintx_h_sparc #define helper_gvec_vrintx_s helper_gvec_vrintx_s_sparc +#define helper_gvec_fmla_nf_idx_h helper_gvec_fmla_nf_idx_h_sparc +#define helper_gvec_fmla_nf_idx_s helper_gvec_fmla_nf_idx_s_sparc +#define helper_gvec_fmls_nf_idx_h helper_gvec_fmls_nf_idx_h_sparc +#define helper_gvec_fmls_nf_idx_s helper_gvec_fmls_nf_idx_s_sparc #define helper_power_down helper_power_down_sparc #define helper_pre_hvc helper_pre_hvc_sparc #define helper_pre_smc helper_pre_smc_sparc diff --git a/qemu/sparc64.h b/qemu/sparc64.h index f24591e1..5a2e72a1 100644 --- a/qemu/sparc64.h +++ b/qemu/sparc64.h @@ -1852,6 +1852,10 @@ #define helper_gvec_vrint_rm_s helper_gvec_vrint_rm_s_sparc64 #define helper_gvec_vrintx_h helper_gvec_vrintx_h_sparc64 #define helper_gvec_vrintx_s helper_gvec_vrintx_s_sparc64 +#define helper_gvec_fmla_nf_idx_h helper_gvec_fmla_nf_idx_h_sparc64 +#define helper_gvec_fmla_nf_idx_s helper_gvec_fmla_nf_idx_s_sparc64 +#define helper_gvec_fmls_nf_idx_h helper_gvec_fmls_nf_idx_h_sparc64 +#define helper_gvec_fmls_nf_idx_s helper_gvec_fmls_nf_idx_s_sparc64 #define helper_power_down helper_power_down_sparc64 #define helper_pre_hvc helper_pre_hvc_sparc64 #define helper_pre_smc helper_pre_smc_sparc64 diff --git a/qemu/target/arm/helper.h b/qemu/target/arm/helper.h index 891211cd..73bf0bb8 100644 --- a/qemu/target/arm/helper.h +++ b/qemu/target/arm/helper.h @@ -738,6 +738,16 @@ DEF_HELPER_FLAGS_5(gvec_fmul_idx_s, TCG_CALL_NO_RWG, DEF_HELPER_FLAGS_5(gvec_fmul_idx_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_fmla_nf_idx_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_fmla_nf_idx_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_5(gvec_fmls_nf_idx_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_fmls_nf_idx_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_6(gvec_fmla_idx_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_6(gvec_fmla_idx_s, TCG_CALL_NO_RWG, diff --git a/qemu/target/arm/vec_helper.c b/qemu/target/arm/vec_helper.c index 48fb99ff..91f5dd97 100644 --- a/qemu/target/arm/vec_helper.c +++ b/qemu/target/arm/vec_helper.c @@ -1086,7 +1086,7 @@ DO_MLA_IDX(gvec_mls_idx_d, uint64_t, -, ) #undef DO_MLA_IDX -#define DO_FMUL_IDX(NAME, TYPE, H) \ +#define DO_FMUL_IDX(NAME, ADD, TYPE, H) \ void HELPER(NAME)(void *vd, void *vn, void *vm, void *stat, uint32_t desc) \ { \ intptr_t i, j, oprsz = simd_oprsz(desc); \ @@ -1096,15 +1096,33 @@ void HELPER(NAME)(void *vd, void *vn, void *vm, void *stat, uint32_t desc) \ for (i = 0; i < oprsz / sizeof(TYPE); i += segment) { \ TYPE mm = m[H(i + idx)]; \ for (j = 0; j < segment; j++) { \ - d[i + j] = TYPE##_mul(n[i + j], mm, stat); \ + d[i + j] = TYPE##_##ADD(d[i + j], \ + TYPE##_mul(n[i + j], mm, stat), stat); \ } \ } \ clear_tail(d, oprsz, simd_maxsz(desc)); \ } -DO_FMUL_IDX(gvec_fmul_idx_h, float16, H2) -DO_FMUL_IDX(gvec_fmul_idx_s, float32, H4) -DO_FMUL_IDX(gvec_fmul_idx_d, float64, ) +#define float16_nop(N, M, S) (M) +#define float32_nop(N, M, S) (M) +#define float64_nop(N, M, S) (M) + +DO_FMUL_IDX(gvec_fmul_idx_h, nop, float16, H2) +DO_FMUL_IDX(gvec_fmul_idx_s, nop, float32, H4) +DO_FMUL_IDX(gvec_fmul_idx_d, nop, float64, ) + +/* + * Non-fused multiply-accumulate operations, for Neon. NB that unlike + * the fused ops below they assume accumulate both from and into Vd. + */ +DO_FMUL_IDX(gvec_fmla_nf_idx_h, add, float16, H2) +DO_FMUL_IDX(gvec_fmla_nf_idx_s, add, float32, H4) +DO_FMUL_IDX(gvec_fmls_nf_idx_h, sub, float16, H2) +DO_FMUL_IDX(gvec_fmls_nf_idx_s, sub, float32, H4) + +#undef float16_nop +#undef float32_nop +#undef float64_nop #undef DO_FMUL_IDX diff --git a/qemu/x86_64.h b/qemu/x86_64.h index 6f16f384..a15ae703 100644 --- a/qemu/x86_64.h +++ b/qemu/x86_64.h @@ -1852,6 +1852,10 @@ #define helper_gvec_vrint_rm_s helper_gvec_vrint_rm_s_x86_64 #define helper_gvec_vrintx_h helper_gvec_vrintx_h_x86_64 #define helper_gvec_vrintx_s helper_gvec_vrintx_s_x86_64 +#define helper_gvec_fmla_nf_idx_h helper_gvec_fmla_nf_idx_h_x86_64 +#define helper_gvec_fmla_nf_idx_s helper_gvec_fmla_nf_idx_s_x86_64 +#define helper_gvec_fmls_nf_idx_h helper_gvec_fmls_nf_idx_h_x86_64 +#define helper_gvec_fmls_nf_idx_s helper_gvec_fmls_nf_idx_s_x86_64 #define helper_power_down helper_power_down_x86_64 #define helper_pre_hvc helper_pre_hvc_x86_64 #define helper_pre_smc helper_pre_smc_x86_64