diff --git a/qemu/aarch64.h b/qemu/aarch64.h index f48adcb4..bd1a7ae1 100644 --- a/qemu/aarch64.h +++ b/qemu/aarch64.h @@ -3368,6 +3368,14 @@ #define helper_sve_lsr_zpzz_d helper_sve_lsr_zpzz_d_aarch64 #define helper_sve_lsr_zpzz_h helper_sve_lsr_zpzz_h_aarch64 #define helper_sve_lsr_zpzz_s helper_sve_lsr_zpzz_s_aarch64 +#define helper_sve_mla_b helper_sve_mla_b_aarch64 +#define helper_sve_mla_d helper_sve_mla_d_aarch64 +#define helper_sve_mla_h helper_sve_mla_h_aarch64 +#define helper_sve_mla_s helper_sve_mla_s_aarch64 +#define helper_sve_mls_b helper_sve_mls_b_aarch64 +#define helper_sve_mls_d helper_sve_mls_d_aarch64 +#define helper_sve_mls_h helper_sve_mls_h_aarch64 +#define helper_sve_mls_s helper_sve_mls_s_aarch64 #define helper_sve_mul_zpzz_b helper_sve_mul_zpzz_b_aarch64 #define helper_sve_mul_zpzz_d helper_sve_mul_zpzz_d_aarch64 #define helper_sve_mul_zpzz_h helper_sve_mul_zpzz_h_aarch64 diff --git a/qemu/aarch64eb.h b/qemu/aarch64eb.h index 11059f92..6f41f7e9 100644 --- a/qemu/aarch64eb.h +++ b/qemu/aarch64eb.h @@ -3368,6 +3368,14 @@ #define helper_sve_lsr_zpzz_d helper_sve_lsr_zpzz_d_aarch64eb #define helper_sve_lsr_zpzz_h helper_sve_lsr_zpzz_h_aarch64eb #define helper_sve_lsr_zpzz_s helper_sve_lsr_zpzz_s_aarch64eb +#define helper_sve_mla_b helper_sve_mla_b_aarch64eb +#define helper_sve_mla_d helper_sve_mla_d_aarch64eb +#define helper_sve_mla_h helper_sve_mla_h_aarch64eb +#define helper_sve_mla_s helper_sve_mla_s_aarch64eb +#define helper_sve_mls_b helper_sve_mls_b_aarch64eb +#define helper_sve_mls_d helper_sve_mls_d_aarch64eb +#define helper_sve_mls_h helper_sve_mls_h_aarch64eb +#define helper_sve_mls_s helper_sve_mls_s_aarch64eb #define helper_sve_mul_zpzz_b helper_sve_mul_zpzz_b_aarch64eb #define helper_sve_mul_zpzz_d helper_sve_mul_zpzz_d_aarch64eb #define helper_sve_mul_zpzz_h helper_sve_mul_zpzz_h_aarch64eb diff --git a/qemu/header_gen.py b/qemu/header_gen.py index 3ba7b4e0..60c0a1e0 100644 --- a/qemu/header_gen.py +++ b/qemu/header_gen.py @@ -3389,6 +3389,14 @@ aarch64_symbols = ( 'helper_sve_lsr_zpzz_d', 'helper_sve_lsr_zpzz_h', 'helper_sve_lsr_zpzz_s', + 'helper_sve_mla_b', + 'helper_sve_mla_d', + 'helper_sve_mla_h', + 'helper_sve_mla_s', + 'helper_sve_mls_b', + 'helper_sve_mls_d', + 'helper_sve_mls_h', + 'helper_sve_mls_s', 'helper_sve_mul_zpzz_b', 'helper_sve_mul_zpzz_d', 'helper_sve_mul_zpzz_h', diff --git a/qemu/include/exec/helper-gen.h b/qemu/include/exec/helper-gen.h index 70f11b63..7d37e1fb 100644 --- a/qemu/include/exec/helper-gen.h +++ b/qemu/include/exec/helper-gen.h @@ -57,7 +57,7 @@ static inline void glue(gen_helper_, name)(TCGContext *tcg_ctx, dh_retvar_decl(r } #define DEF_HELPER_FLAGS_6(name, flags, ret, t1, t2, t3, t4, t5, t6) \ -static inline void glue(gen_helper_, name)(TCGContet *tcg_ctx, dh_retvar_decl(ret) \ +static inline void glue(gen_helper_, name)(TCGContext *tcg_ctx, dh_retvar_decl(ret) \ dh_arg_decl(t1, 1), dh_arg_decl(t2, 2), dh_arg_decl(t3, 3), \ dh_arg_decl(t4, 4), dh_arg_decl(t5, 5), dh_arg_decl(t6, 6)) \ { \ diff --git a/qemu/target/arm/helper-sve.h b/qemu/target/arm/helper-sve.h index 11644125..b31d497f 100644 --- a/qemu/target/arm/helper-sve.h +++ b/qemu/target/arm/helper-sve.h @@ -345,6 +345,24 @@ DEF_HELPER_FLAGS_4(sve_neg_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve_neg_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_4(sve_neg_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_6(sve_mla_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_6(sve_mla_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_6(sve_mla_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_6(sve_mla_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) + +DEF_HELPER_FLAGS_6(sve_mls_b, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_6(sve_mls_h, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_6(sve_mls_s, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) +DEF_HELPER_FLAGS_6(sve_mls_d, TCG_CALL_NO_RWG, + void, ptr, ptr, ptr, ptr, ptr, i32) + DEF_HELPER_FLAGS_5(sve_and_pppp, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve_bic_pppp, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) DEF_HELPER_FLAGS_5(sve_eor_pppp, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr, i32) diff --git a/qemu/target/arm/sve.decode b/qemu/target/arm/sve.decode index 0ddc1e96..59a29d9f 100644 --- a/qemu/target/arm/sve.decode +++ b/qemu/target/arm/sve.decode @@ -47,6 +47,7 @@ &rpr_esz rd pg rn esz &rprr_s rd pg rn rm s &rprr_esz rd pg rn rm esz +&rprrr_esz rd pg rn rm ra esz &rpri_esz rd pg rn imm esz ########################################################################### @@ -74,6 +75,12 @@ # One register operand, with governing predicate, vector element size @rd_pg_rn ........ esz:2 ... ... ... pg:3 rn:5 rd:5 &rpr_esz +# Three register operand, with governing predicate, vector element size +@rda_pg_rn_rm ........ esz:2 . rm:5 ... pg:3 rn:5 rd:5 \ + &rprrr_esz ra=%reg_movprfx +@rdn_pg_ra_rm ........ esz:2 . rm:5 ... pg:3 ra:5 rd:5 \ + &rprrr_esz rn=%reg_movprfx + # Two register operand, one immediate operand, with predicate, # element size encoded as TSZHL. User must fill in imm. @rdn_pg_tszimm ........ .. ... ... ... pg:3 ..... rd:5 \ @@ -186,6 +193,16 @@ UXTH 00000100 .. 010 011 101 ... ..... ..... @rd_pg_rn SXTW 00000100 .. 010 100 101 ... ..... ..... @rd_pg_rn UXTW 00000100 .. 010 101 101 ... ..... ..... @rd_pg_rn +### SVE Integer Multiply-Add Group + +# SVE integer multiply-add writing addend (predicated) +MLA 00000100 .. 0 ..... 010 ... ..... ..... @rda_pg_rn_rm +MLS 00000100 .. 0 ..... 011 ... ..... ..... @rda_pg_rn_rm + +# SVE integer multiply-add writing multiplicand (predicated) +MLA 00000100 .. 0 ..... 110 ... ..... ..... @rdn_pg_ra_rm # MAD +MLS 00000100 .. 0 ..... 111 ... ..... ..... @rdn_pg_ra_rm # MSB + ### SVE Logical - Unpredicated Group # SVE bitwise logical operations (unpredicated) diff --git a/qemu/target/arm/sve_helper.c b/qemu/target/arm/sve_helper.c index 2a19bd4e..9ad4e691 100644 --- a/qemu/target/arm/sve_helper.c +++ b/qemu/target/arm/sve_helper.c @@ -934,3 +934,60 @@ DO_ZPZI_D(sve_asrd_d, int64_t, DO_ASRD) #undef DO_ASRD #undef DO_ZPZI #undef DO_ZPZI_D + +/* Fully general four-operand expander, controlled by a predicate. + */ +#define DO_ZPZZZ(NAME, TYPE, H, OP) \ +void HELPER(NAME)(void *vd, void *va, void *vn, void *vm, \ + void *vg, uint32_t desc) \ +{ \ + intptr_t i, opr_sz = simd_oprsz(desc); \ + for (i = 0; i < opr_sz; ) { \ + uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \ + do { \ + if (pg & 1) { \ + TYPE nn = *(TYPE *)(vn + H(i)); \ + TYPE mm = *(TYPE *)(vm + H(i)); \ + TYPE aa = *(TYPE *)(va + H(i)); \ + *(TYPE *)(vd + H(i)) = OP(aa, nn, mm); \ + } \ + i += sizeof(TYPE), pg >>= sizeof(TYPE); \ + } while (i & 15); \ + } \ +} + +/* Similarly, specialized for 64-bit operands. */ +#define DO_ZPZZZ_D(NAME, TYPE, OP) \ +void HELPER(NAME)(void *vd, void *va, void *vn, void *vm, \ + void *vg, uint32_t desc) \ +{ \ + intptr_t i, opr_sz = simd_oprsz(desc) / 8; \ + TYPE *d = vd, *a = va, *n = vn, *m = vm; \ + uint8_t *pg = vg; \ + for (i = 0; i < opr_sz; i += 1) { \ + if (pg[H1(i)] & 1) { \ + TYPE aa = a[i], nn = n[i], mm = m[i]; \ + d[i] = OP(aa, nn, mm); \ + } \ + } \ +} + +#define DO_MLA(A, N, M) (A + N * M) +#define DO_MLS(A, N, M) (A - N * M) + +DO_ZPZZZ(sve_mla_b, uint8_t, H1, DO_MLA) +DO_ZPZZZ(sve_mls_b, uint8_t, H1, DO_MLS) + +DO_ZPZZZ(sve_mla_h, uint16_t, H1_2, DO_MLA) +DO_ZPZZZ(sve_mls_h, uint16_t, H1_2, DO_MLS) + +DO_ZPZZZ(sve_mla_s, uint32_t, H1_4, DO_MLA) +DO_ZPZZZ(sve_mls_s, uint32_t, H1_4, DO_MLS) + +DO_ZPZZZ_D(sve_mla_d, uint64_t, DO_MLA) +DO_ZPZZZ_D(sve_mls_d, uint64_t, DO_MLS) + +#undef DO_MLA +#undef DO_MLS +#undef DO_ZPZZZ +#undef DO_ZPZZZ_D diff --git a/qemu/target/arm/translate-sve.c b/qemu/target/arm/translate-sve.c index d4e32de0..4c98021f 100644 --- a/qemu/target/arm/translate-sve.c +++ b/qemu/target/arm/translate-sve.c @@ -644,6 +644,41 @@ DO_ZPZW(LSL, lsl) #undef DO_ZPZW +/* + *** SVE Integer Multiply-Add Group + */ + +static bool do_zpzzz_ool(DisasContext *s, arg_rprrr_esz *a, + gen_helper_gvec_5 *fn) +{ + if (sve_access_check(s)) { + TCGContext *tcg_ctx = s->uc->tcg_ctx; + unsigned vsz = vec_full_reg_size(s); + tcg_gen_gvec_5_ool(tcg_ctx, vec_full_reg_offset(s, a->rd), + vec_full_reg_offset(s, a->ra), + vec_full_reg_offset(s, a->rn), + vec_full_reg_offset(s, a->rm), + pred_full_reg_offset(s, a->pg), + vsz, vsz, 0, fn); + } + return true; +} + +#define DO_ZPZZZ(NAME, name) \ +static bool trans_##NAME(DisasContext *s, arg_rprrr_esz *a, uint32_t insn) \ +{ \ + static gen_helper_gvec_5 * const fns[4] = { \ + gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \ + gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \ + }; \ + return do_zpzzz_ool(s, a, fns[a->esz]); \ +} + +DO_ZPZZZ(MLA, mla) +DO_ZPZZZ(MLS, mls) + +#undef DO_ZPZZZ + /* *** SVE Predicate Logical Operations Group */