From abd86b228720f29b9a2223212f87a5d855851ae9 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 9 Mar 2018 01:01:42 -0500 Subject: [PATCH] target/arm: Decode aa64 armv8.3 fcmla Backports commit d17b7cdcf4ea3e858ceee8b86fc8544bb71561e6 from qemu Also remember to commit vec_helper. --- qemu/aarch64.h | 5 + qemu/aarch64eb.h | 5 + qemu/arm.h | 5 + qemu/armeb.h | 5 + qemu/header_gen.py | 5 + qemu/m68k.h | 5 + qemu/mips.h | 5 + qemu/mips64.h | 5 + qemu/mips64el.h | 5 + qemu/mipsel.h | 5 + qemu/powerpc.h | 5 + qemu/sparc.h | 5 + qemu/sparc64.h | 5 + qemu/target/arm/helper.h | 11 + qemu/target/arm/translate-a64.c | 94 ++++++- qemu/target/arm/vec_helper.c | 428 ++++++++++++++++++++++++++++++++ qemu/x86_64.h | 5 + 17 files changed, 595 insertions(+), 8 deletions(-) create mode 100644 qemu/target/arm/vec_helper.c diff --git a/qemu/aarch64.h b/qemu/aarch64.h index 6ce3114c..95982fd9 100644 --- a/qemu/aarch64.h +++ b/qemu/aarch64.h @@ -996,6 +996,11 @@ #define helper_gvec_fcaddh helper_gvec_fcaddh_aarch64 #define helper_gvec_fcadds helper_gvec_fcadds_aarch64 #define helper_gvec_fcaddd helper_gvec_fcaddd_aarch64 +#define helper_gvec_fcmlad helper_gvec_fcmlad_aarch64 +#define helper_gvec_fcmlah helper_gvec_fcmlah_aarch64 +#define helper_gvec_fcmlah_idx helper_gvec_fcmlah_idx_aarch64 +#define helper_gvec_fcmlas helper_gvec_fcmlas_aarch64 +#define helper_gvec_fcmlas_idx helper_gvec_fcmlas_idx_aarch64 #define helper_gvec_le8 helper_gvec_le8_aarch64 #define helper_gvec_le16 helper_gvec_le16_aarch64 #define helper_gvec_le32 helper_gvec_le32_aarch64 diff --git a/qemu/aarch64eb.h b/qemu/aarch64eb.h index 1635d33e..915896ed 100644 --- a/qemu/aarch64eb.h +++ b/qemu/aarch64eb.h @@ -996,6 +996,11 @@ #define helper_gvec_fcaddh helper_gvec_fcaddh_aarch64eb #define helper_gvec_fcadds helper_gvec_fcadds_aarch64eb #define helper_gvec_fcaddd helper_gvec_fcaddd_aarch64eb +#define helper_gvec_fcmlad helper_gvec_fcmlad_aarch64eb +#define helper_gvec_fcmlah helper_gvec_fcmlah_aarch64eb +#define helper_gvec_fcmlah_idx helper_gvec_fcmlah_idx_aarch64eb +#define helper_gvec_fcmlas helper_gvec_fcmlas_aarch64eb +#define helper_gvec_fcmlas_idx helper_gvec_fcmlas_idx_aarch64eb #define helper_gvec_le8 helper_gvec_le8_aarch64eb #define helper_gvec_le16 helper_gvec_le16_aarch64eb #define helper_gvec_le32 helper_gvec_le32_aarch64eb diff --git a/qemu/arm.h b/qemu/arm.h index 13a859bb..573fbf70 100644 --- a/qemu/arm.h +++ b/qemu/arm.h @@ -996,6 +996,11 @@ #define helper_gvec_fcaddh helper_gvec_fcaddh_arm #define helper_gvec_fcadds helper_gvec_fcadds_arm #define helper_gvec_fcaddd helper_gvec_fcaddd_arm +#define helper_gvec_fcmlad helper_gvec_fcmlad_arm +#define helper_gvec_fcmlah helper_gvec_fcmlah_arm +#define helper_gvec_fcmlah_idx helper_gvec_fcmlah_idx_arm +#define helper_gvec_fcmlas helper_gvec_fcmlas_arm +#define helper_gvec_fcmlas_idx helper_gvec_fcmlas_idx_arm #define helper_gvec_le8 helper_gvec_le8_arm #define helper_gvec_le16 helper_gvec_le16_arm #define helper_gvec_le32 helper_gvec_le32_arm diff --git a/qemu/armeb.h b/qemu/armeb.h index afef33cc..0e1a4787 100644 --- a/qemu/armeb.h +++ b/qemu/armeb.h @@ -996,6 +996,11 @@ #define helper_gvec_fcaddh helper_gvec_fcaddh_armeb #define helper_gvec_fcadds helper_gvec_fcadds_armeb #define helper_gvec_fcaddd helper_gvec_fcaddd_armeb +#define helper_gvec_fcmlad helper_gvec_fcmlad_armeb +#define helper_gvec_fcmlah helper_gvec_fcmlah_armeb +#define helper_gvec_fcmlah_idx helper_gvec_fcmlah_idx_armeb +#define helper_gvec_fcmlas helper_gvec_fcmlas_armeb +#define helper_gvec_fcmlas_idx helper_gvec_fcmlas_idx_armeb #define helper_gvec_le8 helper_gvec_le8_armeb #define helper_gvec_le16 helper_gvec_le16_armeb #define helper_gvec_le32 helper_gvec_le32_armeb diff --git a/qemu/header_gen.py b/qemu/header_gen.py index 4b5e4491..e37ac3bd 100644 --- a/qemu/header_gen.py +++ b/qemu/header_gen.py @@ -1002,6 +1002,11 @@ symbols = ( 'helper_gvec_fcaddh', 'helper_gvec_fcadds', 'helper_gvec_fcaddd', + 'helper_gvec_fcmlad', + 'helper_gvec_fcmlah', + 'helper_gvec_fcmlah_idx', + 'helper_gvec_fcmlas', + 'helper_gvec_fcmlas_idx', 'helper_gvec_le8', 'helper_gvec_le16', 'helper_gvec_le32', diff --git a/qemu/m68k.h b/qemu/m68k.h index 0c2ae9d8..87f5fb50 100644 --- a/qemu/m68k.h +++ b/qemu/m68k.h @@ -996,6 +996,11 @@ #define helper_gvec_fcaddh helper_gvec_fcaddh_m68k #define helper_gvec_fcadds helper_gvec_fcadds_m68k #define helper_gvec_fcaddd helper_gvec_fcaddd_m68k +#define helper_gvec_fcmlad helper_gvec_fcmlad_m68k +#define helper_gvec_fcmlah helper_gvec_fcmlah_m68k +#define helper_gvec_fcmlah_idx helper_gvec_fcmlah_idx_m68k +#define helper_gvec_fcmlas helper_gvec_fcmlas_m68k +#define helper_gvec_fcmlas_idx helper_gvec_fcmlas_idx_m68k #define helper_gvec_le8 helper_gvec_le8_m68k #define helper_gvec_le16 helper_gvec_le16_m68k #define helper_gvec_le32 helper_gvec_le32_m68k diff --git a/qemu/mips.h b/qemu/mips.h index b804ab95..e430b2dd 100644 --- a/qemu/mips.h +++ b/qemu/mips.h @@ -996,6 +996,11 @@ #define helper_gvec_fcaddh helper_gvec_fcaddh_mips #define helper_gvec_fcadds helper_gvec_fcadds_mips #define helper_gvec_fcaddd helper_gvec_fcaddd_mips +#define helper_gvec_fcmlad helper_gvec_fcmlad_mips +#define helper_gvec_fcmlah helper_gvec_fcmlah_mips +#define helper_gvec_fcmlah_idx helper_gvec_fcmlah_idx_mips +#define helper_gvec_fcmlas helper_gvec_fcmlas_mips +#define helper_gvec_fcmlas_idx helper_gvec_fcmlas_idx_mips #define helper_gvec_le8 helper_gvec_le8_mips #define helper_gvec_le16 helper_gvec_le16_mips #define helper_gvec_le32 helper_gvec_le32_mips diff --git a/qemu/mips64.h b/qemu/mips64.h index 11aec618..fcd0a36f 100644 --- a/qemu/mips64.h +++ b/qemu/mips64.h @@ -996,6 +996,11 @@ #define helper_gvec_fcaddh helper_gvec_fcaddh_mips64 #define helper_gvec_fcadds helper_gvec_fcadds_mips64 #define helper_gvec_fcaddd helper_gvec_fcaddd_mips64 +#define helper_gvec_fcmlad helper_gvec_fcmlad_mips64 +#define helper_gvec_fcmlah helper_gvec_fcmlah_mips64 +#define helper_gvec_fcmlah_idx helper_gvec_fcmlah_idx_mips64 +#define helper_gvec_fcmlas helper_gvec_fcmlas_mips64 +#define helper_gvec_fcmlas_idx helper_gvec_fcmlas_idx_mips64 #define helper_gvec_le8 helper_gvec_le8_mips64 #define helper_gvec_le16 helper_gvec_le16_mips64 #define helper_gvec_le32 helper_gvec_le32_mips64 diff --git a/qemu/mips64el.h b/qemu/mips64el.h index d7e7aede..d0accfe9 100644 --- a/qemu/mips64el.h +++ b/qemu/mips64el.h @@ -996,6 +996,11 @@ #define helper_gvec_fcaddh helper_gvec_fcaddh_mips64el #define helper_gvec_fcadds helper_gvec_fcadds_mips64el #define helper_gvec_fcaddd helper_gvec_fcaddd_mips64el +#define helper_gvec_fcmlad helper_gvec_fcmlad_mips64el +#define helper_gvec_fcmlah helper_gvec_fcmlah_mips64el +#define helper_gvec_fcmlah_idx helper_gvec_fcmlah_idx_mips64el +#define helper_gvec_fcmlas helper_gvec_fcmlas_mips64el +#define helper_gvec_fcmlas_idx helper_gvec_fcmlas_idx_mips64el #define helper_gvec_le8 helper_gvec_le8_mips64el #define helper_gvec_le16 helper_gvec_le16_mips64el #define helper_gvec_le32 helper_gvec_le32_mips64el diff --git a/qemu/mipsel.h b/qemu/mipsel.h index faf1b25e..126c0de6 100644 --- a/qemu/mipsel.h +++ b/qemu/mipsel.h @@ -996,6 +996,11 @@ #define helper_gvec_fcaddh helper_gvec_fcaddh_mipsel #define helper_gvec_fcadds helper_gvec_fcadds_mipsel #define helper_gvec_fcaddd helper_gvec_fcaddd_mipsel +#define helper_gvec_fcmlad helper_gvec_fcmlad_mipsel +#define helper_gvec_fcmlah helper_gvec_fcmlah_mipsel +#define helper_gvec_fcmlah_idx helper_gvec_fcmlah_idx_mipsel +#define helper_gvec_fcmlas helper_gvec_fcmlas_mipsel +#define helper_gvec_fcmlas_idx helper_gvec_fcmlas_idx_mipsel #define helper_gvec_le8 helper_gvec_le8_mipsel #define helper_gvec_le16 helper_gvec_le16_mipsel #define helper_gvec_le32 helper_gvec_le32_mipsel diff --git a/qemu/powerpc.h b/qemu/powerpc.h index 45196045..e904581a 100644 --- a/qemu/powerpc.h +++ b/qemu/powerpc.h @@ -996,6 +996,11 @@ #define helper_gvec_fcaddh helper_gvec_fcaddh_powerpc #define helper_gvec_fcadds helper_gvec_fcadds_powerpc #define helper_gvec_fcaddd helper_gvec_fcaddd_powerpc +#define helper_gvec_fcmlad helper_gvec_fcmlad_powerpc +#define helper_gvec_fcmlah helper_gvec_fcmlah_powerpc +#define helper_gvec_fcmlah_idx helper_gvec_fcmlah_idx_powerpc +#define helper_gvec_fcmlas helper_gvec_fcmlas_powerpc +#define helper_gvec_fcmlas_idx helper_gvec_fcmlas_idx_powerpc #define helper_gvec_le8 helper_gvec_le8_powerpc #define helper_gvec_le16 helper_gvec_le16_powerpc #define helper_gvec_le32 helper_gvec_le32_powerpc diff --git a/qemu/sparc.h b/qemu/sparc.h index 1f3c836f..da4a62f4 100644 --- a/qemu/sparc.h +++ b/qemu/sparc.h @@ -996,6 +996,11 @@ #define helper_gvec_fcaddh helper_gvec_fcaddh_sparc #define helper_gvec_fcadds helper_gvec_fcadds_sparc #define helper_gvec_fcaddd helper_gvec_fcaddd_sparc +#define helper_gvec_fcmlad helper_gvec_fcmlad_sparc +#define helper_gvec_fcmlah helper_gvec_fcmlah_sparc +#define helper_gvec_fcmlah_idx helper_gvec_fcmlah_idx_sparc +#define helper_gvec_fcmlas helper_gvec_fcmlas_sparc +#define helper_gvec_fcmlas_idx helper_gvec_fcmlas_idx_sparc #define helper_gvec_le8 helper_gvec_le8_sparc #define helper_gvec_le16 helper_gvec_le16_sparc #define helper_gvec_le32 helper_gvec_le32_sparc diff --git a/qemu/sparc64.h b/qemu/sparc64.h index 43d36482..4a10acb1 100644 --- a/qemu/sparc64.h +++ b/qemu/sparc64.h @@ -996,6 +996,11 @@ #define helper_gvec_fcaddh helper_gvec_fcaddh_sparc64 #define helper_gvec_fcadds helper_gvec_fcadds_sparc64 #define helper_gvec_fcaddd helper_gvec_fcaddd_sparc64 +#define helper_gvec_fcmlad helper_gvec_fcmlad_sparc64 +#define helper_gvec_fcmlah helper_gvec_fcmlah_sparc64 +#define helper_gvec_fcmlah_idx helper_gvec_fcmlah_idx_sparc64 +#define helper_gvec_fcmlas helper_gvec_fcmlas_sparc64 +#define helper_gvec_fcmlas_idx helper_gvec_fcmlas_idx_sparc64 #define helper_gvec_le8 helper_gvec_le8_sparc64 #define helper_gvec_le16 helper_gvec_le16_sparc64 #define helper_gvec_le32 helper_gvec_le32_sparc64 diff --git a/qemu/target/arm/helper.h b/qemu/target/arm/helper.h index 0812c37b..c25938a7 100644 --- a/qemu/target/arm/helper.h +++ b/qemu/target/arm/helper.h @@ -587,6 +587,17 @@ DEF_HELPER_FLAGS_5(gvec_fcadds, TCG_CALL_NO_RWG, DEF_HELPER_FLAGS_5(gvec_fcaddd, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_fcmlah, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_fcmlah_idx, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_fcmlas, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_fcmlas_idx, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_5(gvec_fcmlad, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, i32) + #ifdef TARGET_ARM #define helper_clz helper_clz_arm #define gen_helper_clz gen_helper_clz_arm diff --git a/qemu/target/arm/translate-a64.c b/qemu/target/arm/translate-a64.c index 478bbd98..f985ab29 100644 --- a/qemu/target/arm/translate-a64.c +++ b/qemu/target/arm/translate-a64.c @@ -10995,6 +10995,10 @@ static void disas_simd_three_reg_same_extra(DisasContext *s, uint32_t insn) } feature = ARM_FEATURE_V8_RDM; break; + case 0x8: /* FCMLA, #0 */ + case 0x9: /* FCMLA, #90 */ + case 0xa: /* FCMLA, #180 */ + case 0xb: /* FCMLA, #270 */ case 0xc: /* FCADD, #90 */ case 0xe: /* FCADD, #270 */ if (size == 0 @@ -11065,6 +11069,29 @@ static void disas_simd_three_reg_same_extra(DisasContext *s, uint32_t insn) } return; + case 0x8: /* FCMLA, #0 */ + case 0x9: /* FCMLA, #90 */ + case 0xa: /* FCMLA, #180 */ + case 0xb: /* FCMLA, #270 */ + rot = extract32(opcode, 0, 2); + switch (size) { + case 1: + gen_gvec_op3_fpst(s, is_q, rd, rn, rm, true, rot, + gen_helper_gvec_fcmlah); + break; + case 2: + gen_gvec_op3_fpst(s, is_q, rd, rn, rm, false, rot, + gen_helper_gvec_fcmlas); + break; + case 3: + gen_gvec_op3_fpst(s, is_q, rd, rn, rm, false, rot, + gen_helper_gvec_fcmlad); + break; + default: + g_assert_not_reached(); + } + return; + default: g_assert_not_reached(); } @@ -12152,7 +12179,7 @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn) int rn = extract32(insn, 5, 5); int rd = extract32(insn, 0, 5); bool is_long = false; - bool is_fp = false; + int is_fp = 0; bool is_fp16 = false; int index; TCGv_ptr fpst; @@ -12190,7 +12217,7 @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn) case 0x05: /* FMLS */ case 0x09: /* FMUL */ case 0x19: /* FMULX */ - is_fp = true; + is_fp = 1; break; case 0x1d: /* SQRDMLAH */ case 0x1f: /* SQRDMLSH */ @@ -12199,20 +12226,28 @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn) return; } break; + case 0x11: /* FCMLA #0 */ + case 0x13: /* FCMLA #90 */ + case 0x15: /* FCMLA #180 */ + case 0x17: /* FCMLA #270 */ + if (!arm_dc_feature(s, ARM_FEATURE_V8_FCMA)) { + unallocated_encoding(s); + return; + } + is_fp = 2; + break; default: unallocated_encoding(s); return; } - if (is_fp) { + switch (is_fp) { + case 1: /* normal fp */ /* convert insn encoded size to TCGMemOp size */ switch (size) { case 0: /* half-precision */ - if (!arm_dc_feature(s, ARM_FEATURE_V8_FP16)) { - unallocated_encoding(s); - return; - } size = MO_16; + is_fp16 = true; break; case MO_32: /* single precision */ case MO_64: /* double precision */ @@ -12221,13 +12256,39 @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn) unallocated_encoding(s); return; } - } else { + break; + + case 2: /* complex fp */ + /* Each indexable element is a complex pair. */ + size <<= 1; + switch (size) { + case MO_32: + if (h && !is_q) { + unallocated_encoding(s); + return; + } + is_fp16 = true; + break; + case MO_64: + break; + default: + unallocated_encoding(s); + return; + } + break; + + default: /* integer */ switch (size) { case MO_8: case MO_64: unallocated_encoding(s); return; } + break; + } + if (is_fp16 && !arm_dc_feature(s, ARM_FEATURE_V8_FP16)) { + unallocated_encoding(s); + return; } /* Given TCGMemOp size, adjust register and indexing. */ @@ -12261,6 +12322,23 @@ static void disas_simd_indexed(DisasContext *s, uint32_t insn) fpst = NULL; } + switch (16 * u + opcode) { + case 0x11: /* FCMLA #0 */ + case 0x13: /* FCMLA #90 */ + case 0x15: /* FCMLA #180 */ + case 0x17: /* FCMLA #270 */ + tcg_gen_gvec_3_ptr(tcg_ctx, vec_full_reg_offset(s, rd), + vec_full_reg_offset(s, rn), + vec_reg_offset(s, rm, index, size), fpst, + is_q ? 16 : 8, vec_full_reg_size(s), + extract32(insn, 13, 2), /* rot */ + size == MO_64 + ? gen_helper_gvec_fcmlas_idx + : gen_helper_gvec_fcmlah_idx); + tcg_temp_free_ptr(tcg_ctx, fpst); + return; + } + if (size == 3) { TCGv_i64 tcg_idx = tcg_temp_new_i64(tcg_ctx); int pass; diff --git a/qemu/target/arm/vec_helper.c b/qemu/target/arm/vec_helper.c new file mode 100644 index 00000000..8dc8015b --- /dev/null +++ b/qemu/target/arm/vec_helper.c @@ -0,0 +1,428 @@ +/* + * ARM AdvSIMD / SVE Vector Operations + * + * Copyright (c) 2018 Linaro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#include "qemu/osdep.h" +#include "cpu.h" +#include "exec/exec-all.h" +#include "exec/helper-proto.h" +#include "tcg/tcg-gvec-desc.h" +#include "fpu/softfloat.h" + +/* Note that vector data is stored in host-endian 64-bit chunks, + so addressing units smaller than that needs a host-endian fixup. */ +#ifdef HOST_WORDS_BIGENDIAN +#define H1(x) ((x) ^ 7) +#define H2(x) ((x) ^ 3) +#define H4(x) ((x) ^ 1) +#else +#define H1(x) (x) +#define H2(x) (x) +#define H4(x) (x) +#endif + +#define SET_QC() env->vfp.xregs[ARM_VFP_FPSCR] |= CPSR_Q + +static void clear_tail(void *vd, uintptr_t opr_sz, uintptr_t max_sz) +{ + uint64_t *d = vd + opr_sz; + uintptr_t i; + + for (i = opr_sz; i < max_sz; i += 8) { + *d++ = 0; + } +} + +/* Signed saturating rounding doubling multiply-accumulate high half, 16-bit */ +static uint16_t inl_qrdmlah_s16(CPUARMState *env, int16_t src1, + int16_t src2, int16_t src3) +{ + /* Simplify: + * = ((a3 << 16) + ((e1 * e2) << 1) + (1 << 15)) >> 16 + * = ((a3 << 15) + (e1 * e2) + (1 << 14)) >> 15 + */ + int32_t ret = (int32_t)src1 * src2; + ret = ((int32_t)src3 << 15) + ret + (1 << 14); + ret >>= 15; + if (ret != (int16_t)ret) { + SET_QC(); + ret = (ret < 0 ? -0x8000 : 0x7fff); + } + return ret; +} + +uint32_t HELPER(neon_qrdmlah_s16)(CPUARMState *env, uint32_t src1, + uint32_t src2, uint32_t src3) +{ + uint16_t e1 = inl_qrdmlah_s16(env, src1, src2, src3); + uint16_t e2 = inl_qrdmlah_s16(env, src1 >> 16, src2 >> 16, src3 >> 16); + return deposit32(e1, 16, 16, e2); +} + +void HELPER(gvec_qrdmlah_s16)(void *vd, void *vn, void *vm, + void *ve, uint32_t desc) +{ + uintptr_t opr_sz = simd_oprsz(desc); + int16_t *d = vd; + int16_t *n = vn; + int16_t *m = vm; + CPUARMState *env = ve; + uintptr_t i; + + for (i = 0; i < opr_sz / 2; ++i) { + d[i] = inl_qrdmlah_s16(env, n[i], m[i], d[i]); + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} + +/* Signed saturating rounding doubling multiply-subtract high half, 16-bit */ +static uint16_t inl_qrdmlsh_s16(CPUARMState *env, int16_t src1, + int16_t src2, int16_t src3) +{ + /* Similarly, using subtraction: + * = ((a3 << 16) - ((e1 * e2) << 1) + (1 << 15)) >> 16 + * = ((a3 << 15) - (e1 * e2) + (1 << 14)) >> 15 + */ + int32_t ret = (int32_t)src1 * src2; + ret = ((int32_t)src3 << 15) - ret + (1 << 14); + ret >>= 15; + if (ret != (int16_t)ret) { + SET_QC(); + ret = (ret < 0 ? -0x8000 : 0x7fff); + } + return ret; +} + +uint32_t HELPER(neon_qrdmlsh_s16)(CPUARMState *env, uint32_t src1, + uint32_t src2, uint32_t src3) +{ + uint16_t e1 = inl_qrdmlsh_s16(env, src1, src2, src3); + uint16_t e2 = inl_qrdmlsh_s16(env, src1 >> 16, src2 >> 16, src3 >> 16); + return deposit32(e1, 16, 16, e2); +} + +void HELPER(gvec_qrdmlsh_s16)(void *vd, void *vn, void *vm, + void *ve, uint32_t desc) +{ + uintptr_t opr_sz = simd_oprsz(desc); + int16_t *d = vd; + int16_t *n = vn; + int16_t *m = vm; + CPUARMState *env = ve; + uintptr_t i; + + for (i = 0; i < opr_sz / 2; ++i) { + d[i] = inl_qrdmlsh_s16(env, n[i], m[i], d[i]); + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} + +/* Signed saturating rounding doubling multiply-accumulate high half, 32-bit */ +uint32_t HELPER(neon_qrdmlah_s32)(CPUARMState *env, int32_t src1, + int32_t src2, int32_t src3) +{ + /* Simplify similarly to int_qrdmlah_s16 above. */ + int64_t ret = (int64_t)src1 * src2; + ret = ((int64_t)src3 << 31) + ret + (1 << 30); + ret >>= 31; + if (ret != (int32_t)ret) { + SET_QC(); + ret = (ret < 0 ? INT32_MIN : INT32_MAX); + } + return ret; +} + +void HELPER(gvec_qrdmlah_s32)(void *vd, void *vn, void *vm, + void *ve, uint32_t desc) +{ + uintptr_t opr_sz = simd_oprsz(desc); + int32_t *d = vd; + int32_t *n = vn; + int32_t *m = vm; + CPUARMState *env = ve; + uintptr_t i; + + for (i = 0; i < opr_sz / 4; ++i) { + d[i] = helper_neon_qrdmlah_s32(env, n[i], m[i], d[i]); + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} + +/* Signed saturating rounding doubling multiply-subtract high half, 32-bit */ +uint32_t HELPER(neon_qrdmlsh_s32)(CPUARMState *env, int32_t src1, + int32_t src2, int32_t src3) +{ + /* Simplify similarly to int_qrdmlsh_s16 above. */ + int64_t ret = (int64_t)src1 * src2; + ret = ((int64_t)src3 << 31) - ret + (1 << 30); + ret >>= 31; + if (ret != (int32_t)ret) { + SET_QC(); + ret = (ret < 0 ? INT32_MIN : INT32_MAX); + } + return ret; +} + +void HELPER(gvec_qrdmlsh_s32)(void *vd, void *vn, void *vm, + void *ve, uint32_t desc) +{ + uintptr_t opr_sz = simd_oprsz(desc); + int32_t *d = vd; + int32_t *n = vn; + int32_t *m = vm; + CPUARMState *env = ve; + uintptr_t i; + + for (i = 0; i < opr_sz / 4; ++i) { + d[i] = helper_neon_qrdmlsh_s32(env, n[i], m[i], d[i]); + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} + +void HELPER(gvec_fcaddh)(void *vd, void *vn, void *vm, + void *vfpst, uint32_t desc) +{ + uintptr_t opr_sz = simd_oprsz(desc); + float16 *d = vd; + float16 *n = vn; + float16 *m = vm; + float_status *fpst = vfpst; + uint32_t neg_real = extract32(desc, SIMD_DATA_SHIFT, 1); + uint32_t neg_imag = neg_real ^ 1; + uintptr_t i; + + /* Shift boolean to the sign bit so we can xor to negate. */ + neg_real <<= 15; + neg_imag <<= 15; + + for (i = 0; i < opr_sz / 2; i += 2) { + float16 e0 = n[H2(i)]; + float16 e1 = m[H2(i + 1)] ^ neg_imag; + float16 e2 = n[H2(i + 1)]; + float16 e3 = m[H2(i)] ^ neg_real; + + d[H2(i)] = float16_add(e0, e1, fpst); + d[H2(i + 1)] = float16_add(e2, e3, fpst); + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} + +void HELPER(gvec_fcadds)(void *vd, void *vn, void *vm, + void *vfpst, uint32_t desc) +{ + uintptr_t opr_sz = simd_oprsz(desc); + float32 *d = vd; + float32 *n = vn; + float32 *m = vm; + float_status *fpst = vfpst; + uint32_t neg_real = extract32(desc, SIMD_DATA_SHIFT, 1); + uint32_t neg_imag = neg_real ^ 1; + uintptr_t i; + + /* Shift boolean to the sign bit so we can xor to negate. */ + neg_real <<= 31; + neg_imag <<= 31; + + for (i = 0; i < opr_sz / 4; i += 2) { + float32 e0 = n[H4(i)]; + float32 e1 = m[H4(i + 1)] ^ neg_imag; + float32 e2 = n[H4(i + 1)]; + float32 e3 = m[H4(i)] ^ neg_real; + + d[H4(i)] = float32_add(e0, e1, fpst); + d[H4(i + 1)] = float32_add(e2, e3, fpst); + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} + +void HELPER(gvec_fcaddd)(void *vd, void *vn, void *vm, + void *vfpst, uint32_t desc) +{ + uintptr_t opr_sz = simd_oprsz(desc); + float64 *d = vd; + float64 *n = vn; + float64 *m = vm; + float_status *fpst = vfpst; + uint64_t neg_real = extract64(desc, SIMD_DATA_SHIFT, 1); + uint64_t neg_imag = neg_real ^ 1; + uintptr_t i; + + /* Shift boolean to the sign bit so we can xor to negate. */ + neg_real <<= 63; + neg_imag <<= 63; + + for (i = 0; i < opr_sz / 8; i += 2) { + float64 e0 = n[i]; + float64 e1 = m[i + 1] ^ neg_imag; + float64 e2 = n[i + 1]; + float64 e3 = m[i] ^ neg_real; + + d[i] = float64_add(e0, e1, fpst); + d[i + 1] = float64_add(e2, e3, fpst); + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} + +void HELPER(gvec_fcmlah)(void *vd, void *vn, void *vm, + void *vfpst, uint32_t desc) +{ + uintptr_t opr_sz = simd_oprsz(desc); + float16 *d = vd; + float16 *n = vn; + float16 *m = vm; + float_status *fpst = vfpst; + intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1); + uint32_t neg_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1); + uint32_t neg_real = flip ^ neg_imag; + uintptr_t i; + + /* Shift boolean to the sign bit so we can xor to negate. */ + neg_real <<= 15; + neg_imag <<= 15; + + for (i = 0; i < opr_sz / 2; i += 2) { + float16 e2 = n[H2(i + flip)]; + float16 e1 = m[H2(i + flip)] ^ neg_real; + float16 e4 = e2; + float16 e3 = m[H2(i + 1 - flip)] ^ neg_imag; + + d[H2(i)] = float16_muladd(e2, e1, d[H2(i)], 0, fpst); + d[H2(i + 1)] = float16_muladd(e4, e3, d[H2(i + 1)], 0, fpst); + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} + +void HELPER(gvec_fcmlah_idx)(void *vd, void *vn, void *vm, + void *vfpst, uint32_t desc) +{ + uintptr_t opr_sz = simd_oprsz(desc); + float16 *d = vd; + float16 *n = vn; + float16 *m = vm; + float_status *fpst = vfpst; + intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1); + uint32_t neg_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1); + uint32_t neg_real = flip ^ neg_imag; + uintptr_t i; + float16 e1 = m[H2(flip)]; + float16 e3 = m[H2(1 - flip)]; + + /* Shift boolean to the sign bit so we can xor to negate. */ + neg_real <<= 15; + neg_imag <<= 15; + e1 ^= neg_real; + e3 ^= neg_imag; + + for (i = 0; i < opr_sz / 2; i += 2) { + float16 e2 = n[H2(i + flip)]; + float16 e4 = e2; + + d[H2(i)] = float16_muladd(e2, e1, d[H2(i)], 0, fpst); + d[H2(i + 1)] = float16_muladd(e4, e3, d[H2(i + 1)], 0, fpst); + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} + +void HELPER(gvec_fcmlas)(void *vd, void *vn, void *vm, + void *vfpst, uint32_t desc) +{ + uintptr_t opr_sz = simd_oprsz(desc); + float32 *d = vd; + float32 *n = vn; + float32 *m = vm; + float_status *fpst = vfpst; + intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1); + uint32_t neg_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1); + uint32_t neg_real = flip ^ neg_imag; + uintptr_t i; + + /* Shift boolean to the sign bit so we can xor to negate. */ + neg_real <<= 31; + neg_imag <<= 31; + + for (i = 0; i < opr_sz / 4; i += 2) { + float32 e2 = n[H4(i + flip)]; + float32 e1 = m[H4(i + flip)] ^ neg_real; + float32 e4 = e2; + float32 e3 = m[H4(i + 1 - flip)] ^ neg_imag; + + d[H4(i)] = float32_muladd(e2, e1, d[H4(i)], 0, fpst); + d[H4(i + 1)] = float32_muladd(e4, e3, d[H4(i + 1)], 0, fpst); + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} + +void HELPER(gvec_fcmlas_idx)(void *vd, void *vn, void *vm, + void *vfpst, uint32_t desc) +{ + uintptr_t opr_sz = simd_oprsz(desc); + float32 *d = vd; + float32 *n = vn; + float32 *m = vm; + float_status *fpst = vfpst; + intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1); + uint32_t neg_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1); + uint32_t neg_real = flip ^ neg_imag; + uintptr_t i; + float32 e1 = m[H4(flip)]; + float32 e3 = m[H4(1 - flip)]; + + /* Shift boolean to the sign bit so we can xor to negate. */ + neg_real <<= 31; + neg_imag <<= 31; + e1 ^= neg_real; + e3 ^= neg_imag; + + for (i = 0; i < opr_sz / 4; i += 2) { + float32 e2 = n[H4(i + flip)]; + float32 e4 = e2; + + d[H4(i)] = float32_muladd(e2, e1, d[H4(i)], 0, fpst); + d[H4(i + 1)] = float32_muladd(e4, e3, d[H4(i + 1)], 0, fpst); + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} + +void HELPER(gvec_fcmlad)(void *vd, void *vn, void *vm, + void *vfpst, uint32_t desc) +{ + uintptr_t opr_sz = simd_oprsz(desc); + float64 *d = vd; + float64 *n = vn; + float64 *m = vm; + float_status *fpst = vfpst; + intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1); + uint64_t neg_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1); + uint64_t neg_real = flip ^ neg_imag; + uintptr_t i; + + /* Shift boolean to the sign bit so we can xor to negate. */ + neg_real <<= 63; + neg_imag <<= 63; + + for (i = 0; i < opr_sz / 8; i += 2) { + float64 e2 = n[i + flip]; + float64 e1 = m[i + flip] ^ neg_real; + float64 e4 = e2; + float64 e3 = m[i + 1 - flip] ^ neg_imag; + + d[i] = float64_muladd(e2, e1, d[i], 0, fpst); + d[i + 1] = float64_muladd(e4, e3, d[i + 1], 0, fpst); + } + clear_tail(d, opr_sz, simd_maxsz(desc)); +} diff --git a/qemu/x86_64.h b/qemu/x86_64.h index f040d880..fd0631c0 100644 --- a/qemu/x86_64.h +++ b/qemu/x86_64.h @@ -996,6 +996,11 @@ #define helper_gvec_fcaddh helper_gvec_fcaddh_x86_64 #define helper_gvec_fcadds helper_gvec_fcadds_x86_64 #define helper_gvec_fcaddd helper_gvec_fcaddd_x86_64 +#define helper_gvec_fcmlad helper_gvec_fcmlad_x86_64 +#define helper_gvec_fcmlah helper_gvec_fcmlah_x86_64 +#define helper_gvec_fcmlah_idx helper_gvec_fcmlah_idx_x86_64 +#define helper_gvec_fcmlas helper_gvec_fcmlas_x86_64 +#define helper_gvec_fcmlas_idx helper_gvec_fcmlas_idx_x86_64 #define helper_gvec_le8 helper_gvec_le8_x86_64 #define helper_gvec_le16 helper_gvec_le16_x86_64 #define helper_gvec_le32 helper_gvec_le32_x86_64