yuzu/unicorn
1
0
Fork 0
mirror of https://github.com/yuzu-emu/unicorn.git synced 2024-12-23 05:35:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

target/arm: Add mte helpers for sve scatter/gather memory ops

Browse source 
Because the elements are non-sequential, we cannot eliminate many
tests straight away like we can for sequential operations. But
we often have the PTE details handy, so we can test for Tagged.

Backports commit d28d12f008ee44dc2cc2ee5d8f673be9febc951e from qemu
This commit is contained in:
Richard Henderson 2021-02-25 22:24:08 -05:00 committed by Lioncash
parent 5698b7badb
commit 732efce958
6 changed files with 1277 additions and 249 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

135
qemu/aarch64.h
View file

@ -4123,6 +4123,58 @@
#define helper_sve_ldbss_zd helper_sve_ldbss_zd_aarch64
#define helper_sve_ldbss_zss helper_sve_ldbss_zss_aarch64
#define helper_sve_ldbss_zsu helper_sve_ldbss_zsu_aarch64
#define helper_sve_ldbsu_zsu_mte helper_sve_ldbsu_zsu_mte_aarch64
#define helper_sve_ldhsu_le_zsu_mte helper_sve_ldhsu_le_zsu_mte_aarch64
#define helper_sve_ldhsu_be_zsu_mte helper_sve_ldhsu_be_zsu_mte_aarch64
#define helper_sve_ldss_le_zsu_mte helper_sve_ldss_le_zsu_mte_aarch64
#define helper_sve_ldss_be_zsu_mte helper_sve_ldss_be_zsu_mte_aarch64
#define helper_sve_ldbss_zsu_mte helper_sve_ldbss_zsu_mte_aarch64
#define helper_sve_ldhss_le_zsu_mte helper_sve_ldhss_le_zsu_mte_aarch64
#define helper_sve_ldhss_be_zsu_mte helper_sve_ldhss_be_zsu_mte_aarch64
#define helper_sve_ldbsu_zss_mte helper_sve_ldbsu_zss_mte_aarch64
#define helper_sve_ldhsu_le_zss_mte helper_sve_ldhsu_le_zss_mte_aarch64
#define helper_sve_ldhsu_be_zss_mte helper_sve_ldhsu_be_zss_mte_aarch64
#define helper_sve_ldss_le_zss_mte helper_sve_ldss_le_zss_mte_aarch64
#define helper_sve_ldss_be_zss_mte helper_sve_ldss_be_zss_mte_aarch64
#define helper_sve_ldbss_zss_mte helper_sve_ldbss_zss_mte_aarch64
#define helper_sve_ldhss_le_zss_mte helper_sve_ldhss_le_zss_mte_aarch64
#define helper_sve_ldhss_be_zss_mte helper_sve_ldhss_be_zss_mte_aarch64
#define helper_sve_ldbdu_zsu_mte helper_sve_ldbdu_zsu_mte_aarch64
#define helper_sve_ldhdu_le_zsu_mte helper_sve_ldhdu_le_zsu_mte_aarch64
#define helper_sve_ldhdu_be_zsu_mte helper_sve_ldhdu_be_zsu_mte_aarch64
#define helper_sve_ldsdu_le_zsu_mte helper_sve_ldsdu_le_zsu_mte_aarch64
#define helper_sve_ldsdu_be_zsu_mte helper_sve_ldsdu_be_zsu_mte_aarch64
#define helper_sve_lddd_le_zsu_mte helper_sve_lddd_le_zsu_mte_aarch64
#define helper_sve_lddd_be_zsu_mte helper_sve_lddd_be_zsu_mte_aarch64
#define helper_sve_ldbds_zsu_mte helper_sve_ldbds_zsu_mte_aarch64
#define helper_sve_ldhds_le_zsu_mte helper_sve_ldhds_le_zsu_mte_aarch64
#define helper_sve_ldhds_be_zsu_mte helper_sve_ldhds_be_zsu_mte_aarch64
#define helper_sve_ldsds_le_zsu_mte helper_sve_ldsds_le_zsu_mte_aarch64
#define helper_sve_ldsds_be_zsu_mte helper_sve_ldsds_be_zsu_mte_aarch64
#define helper_sve_ldbdu_zss_mte helper_sve_ldbdu_zss_mte_aarch64
#define helper_sve_ldhdu_le_zss_mte helper_sve_ldhdu_le_zss_mte_aarch64
#define helper_sve_ldhdu_be_zss_mte helper_sve_ldhdu_be_zss_mte_aarch64
#define helper_sve_ldsdu_le_zss_mte helper_sve_ldsdu_le_zss_mte_aarch64
#define helper_sve_ldsdu_be_zss_mte helper_sve_ldsdu_be_zss_mte_aarch64
#define helper_sve_lddd_le_zss_mte helper_sve_lddd_le_zss_mte_aarch64
#define helper_sve_lddd_be_zss_mte helper_sve_lddd_be_zss_mte_aarch64
#define helper_sve_ldbds_zss_mte helper_sve_ldbds_zss_mte_aarch64
#define helper_sve_ldhds_le_zss_mte helper_sve_ldhds_le_zss_mte_aarch64
#define helper_sve_ldhds_be_zss_mte helper_sve_ldhds_be_zss_mte_aarch64
#define helper_sve_ldsds_le_zss_mte helper_sve_ldsds_le_zss_mte_aarch64
#define helper_sve_ldsds_be_zss_mte helper_sve_ldsds_be_zss_mte_aarch64
#define helper_sve_ldbdu_zd_mte helper_sve_ldbdu_zd_mte_aarch64
#define helper_sve_ldhdu_le_zd_mte helper_sve_ldhdu_le_zd_mte_aarch64
#define helper_sve_ldhdu_be_zd_mte helper_sve_ldhdu_be_zd_mte_aarch64
#define helper_sve_ldsdu_le_zd_mte helper_sve_ldsdu_le_zd_mte_aarch64
#define helper_sve_ldsdu_be_zd_mte helper_sve_ldsdu_be_zd_mte_aarch64
#define helper_sve_lddd_le_zd_mte helper_sve_lddd_le_zd_mte_aarch64
#define helper_sve_lddd_be_zd_mte helper_sve_lddd_be_zd_mte_aarch64
#define helper_sve_ldbds_zd_mte helper_sve_ldbds_zd_mte_aarch64
#define helper_sve_ldhds_le_zd_mte helper_sve_ldhds_le_zd_mte_aarch64
#define helper_sve_ldhds_be_zd_mte helper_sve_ldhds_be_zd_mte_aarch64
#define helper_sve_ldsds_le_zd_mte helper_sve_ldsds_le_zd_mte_aarch64
#define helper_sve_ldsds_be_zd_mte helper_sve_ldsds_be_zd_mte_aarch64
#define helper_sve_ldbsu_zss helper_sve_ldbsu_zss_aarch64
#define helper_sve_ldbsu_zsu helper_sve_ldbsu_zsu_aarch64
#define helper_sve_ldbds_zd helper_sve_ldbds_zd_aarch64
@ -4324,6 +4376,58 @@
#define helper_sve_ldffbdu_zsu helper_sve_ldffbdu_zsu_aarch64
#define helper_sve_ldffbss_zss helper_sve_ldffbss_zss_aarch64
#define helper_sve_ldffbss_zsu helper_sve_ldffbss_zsu_aarch64
#define helper_sve_ldffbsu_zsu_mte helper_sve_ldffbsu_zsu_mte_aarch64
#define helper_sve_ldffhsu_le_zsu_mte helper_sve_ldffhsu_le_zsu_mte_aarch64
#define helper_sve_ldffhsu_be_zsu_mte helper_sve_ldffhsu_be_zsu_mte_aarch64
#define helper_sve_ldffss_le_zsu_mte helper_sve_ldffss_le_zsu_mte_aarch64
#define helper_sve_ldffss_be_zsu_mte helper_sve_ldffss_be_zsu_mte_aarch64
#define helper_sve_ldffbss_zsu_mte helper_sve_ldffbss_zsu_mte_aarch64
#define helper_sve_ldffhss_le_zsu_mte helper_sve_ldffhss_le_zsu_mte_aarch64
#define helper_sve_ldffhss_be_zsu_mte helper_sve_ldffhss_be_zsu_mte_aarch64
#define helper_sve_ldffbsu_zss_mte helper_sve_ldffbsu_zss_mte_aarch64
#define helper_sve_ldffhsu_le_zss_mte helper_sve_ldffhsu_le_zss_mte_aarch64
#define helper_sve_ldffhsu_be_zss_mte helper_sve_ldffhsu_be_zss_mte_aarch64
#define helper_sve_ldffss_le_zss_mte helper_sve_ldffss_le_zss_mte_aarch64
#define helper_sve_ldffss_be_zss_mte helper_sve_ldffss_be_zss_mte_aarch64
#define helper_sve_ldffbss_zss_mte helper_sve_ldffbss_zss_mte_aarch64
#define helper_sve_ldffhss_le_zss_mte helper_sve_ldffhss_le_zss_mte_aarch64
#define helper_sve_ldffhss_be_zss_mte helper_sve_ldffhss_be_zss_mte_aarch64
#define helper_sve_ldffbdu_zsu_mte helper_sve_ldffbdu_zsu_mte_aarch64
#define helper_sve_ldffhdu_le_zsu_mte helper_sve_ldffhdu_le_zsu_mte_aarch64
#define helper_sve_ldffhdu_be_zsu_mte helper_sve_ldffhdu_be_zsu_mte_aarch64
#define helper_sve_ldffsdu_le_zsu_mte helper_sve_ldffsdu_le_zsu_mte_aarch64
#define helper_sve_ldffsdu_be_zsu_mte helper_sve_ldffsdu_be_zsu_mte_aarch64
#define helper_sve_ldffdd_le_zsu_mte helper_sve_ldffdd_le_zsu_mte_aarch64
#define helper_sve_ldffdd_be_zsu_mte helper_sve_ldffdd_be_zsu_mte_aarch64
#define helper_sve_ldffbds_zsu_mte helper_sve_ldffbds_zsu_mte_aarch64
#define helper_sve_ldffhds_le_zsu_mte helper_sve_ldffhds_le_zsu_mte_aarch64
#define helper_sve_ldffhds_be_zsu_mte helper_sve_ldffhds_be_zsu_mte_aarch64
#define helper_sve_ldffsds_le_zsu_mte helper_sve_ldffsds_le_zsu_mte_aarch64
#define helper_sve_ldffsds_be_zsu_mte helper_sve_ldffsds_be_zsu_mte_aarch64
#define helper_sve_ldffbdu_zss_mte helper_sve_ldffbdu_zss_mte_aarch64
#define helper_sve_ldffhdu_le_zss_mte helper_sve_ldffhdu_le_zss_mte_aarch64
#define helper_sve_ldffhdu_be_zss_mte helper_sve_ldffhdu_be_zss_mte_aarch64
#define helper_sve_ldffsdu_le_zss_mte helper_sve_ldffsdu_le_zss_mte_aarch64
#define helper_sve_ldffsdu_be_zss_mte helper_sve_ldffsdu_be_zss_mte_aarch64
#define helper_sve_ldffdd_le_zss_mte helper_sve_ldffdd_le_zss_mte_aarch64
#define helper_sve_ldffdd_be_zss_mte helper_sve_ldffdd_be_zss_mte_aarch64
#define helper_sve_ldffbds_zss_mte helper_sve_ldffbds_zss_mte_aarch64
#define helper_sve_ldffhds_le_zss_mte helper_sve_ldffhds_le_zss_mte_aarch64
#define helper_sve_ldffhds_be_zss_mte helper_sve_ldffhds_be_zss_mte_aarch64
#define helper_sve_ldffsds_le_zss_mte helper_sve_ldffsds_le_zss_mte_aarch64
#define helper_sve_ldffsds_be_zss_mte helper_sve_ldffsds_be_zss_mte_aarch64
#define helper_sve_ldffbdu_zd_mte helper_sve_ldffbdu_zd_mte_aarch64
#define helper_sve_ldffhdu_le_zd_mte helper_sve_ldffhdu_le_zd_mte_aarch64
#define helper_sve_ldffhdu_be_zd_mte helper_sve_ldffhdu_be_zd_mte_aarch64
#define helper_sve_ldffsdu_le_zd_mte helper_sve_ldffsdu_le_zd_mte_aarch64
#define helper_sve_ldffsdu_be_zd_mte helper_sve_ldffsdu_be_zd_mte_aarch64
#define helper_sve_ldffdd_le_zd_mte helper_sve_ldffdd_le_zd_mte_aarch64
#define helper_sve_ldffdd_be_zd_mte helper_sve_ldffdd_be_zd_mte_aarch64
#define helper_sve_ldffbds_zd_mte helper_sve_ldffbds_zd_mte_aarch64
#define helper_sve_ldffhds_le_zd_mte helper_sve_ldffhds_le_zd_mte_aarch64
#define helper_sve_ldffhds_be_zd_mte helper_sve_ldffhds_be_zd_mte_aarch64
#define helper_sve_ldffsds_le_zd_mte helper_sve_ldffsds_le_zd_mte_aarch64
#define helper_sve_ldffsds_be_zd_mte helper_sve_ldffsds_be_zd_mte_aarch64
#define helper_sve_ldffbsu_zss helper_sve_ldffbsu_zss_aarch64
#define helper_sve_ldffbsu_zsu helper_sve_ldffbsu_zsu_aarch64
#define helper_sve_ldffdd_be_zd helper_sve_ldffdd_be_zd_aarch64
@ -4583,6 +4687,37 @@
#define helper_sve_stbd_zd helper_sve_stbd_zd_aarch64
#define helper_sve_stbd_zss helper_sve_stbd_zss_aarch64
#define helper_sve_stbd_zsu helper_sve_stbd_zsu_aarch64
#define helper_sve_stbs_zsu_mte helper_sve_stbs_zsu_mte_aarch64
#define helper_sve_sths_le_zsu_mte helper_sve_sths_le_zsu_mte_aarch64
#define helper_sve_sths_be_zsu_mte helper_sve_sths_be_zsu_mte_aarch64
#define helper_sve_stss_le_zsu_mte helper_sve_stss_le_zsu_mte_aarch64
#define helper_sve_stss_be_zsu_mte helper_sve_stss_be_zsu_mte_aarch64
#define helper_sve_stbs_zss_mte helper_sve_stbs_zss_mte_aarch64
#define helper_sve_sths_le_zss_mte helper_sve_sths_le_zss_mte_aarch64
#define helper_sve_sths_be_zss_mte helper_sve_sths_be_zss_mte_aarch64
#define helper_sve_stss_le_zss_mte helper_sve_stss_le_zss_mte_aarch64
#define helper_sve_stss_be_zss_mte helper_sve_stss_be_zss_mte_aarch64
#define helper_sve_stbd_zsu_mte helper_sve_stbd_zsu_mte_aarch64
#define helper_sve_sthd_le_zsu_mte helper_sve_sthd_le_zsu_mte_aarch64
#define helper_sve_sthd_be_zsu_mte helper_sve_sthd_be_zsu_mte_aarch64
#define helper_sve_stsd_le_zsu_mte helper_sve_stsd_le_zsu_mte_aarch64
#define helper_sve_stsd_be_zsu_mte helper_sve_stsd_be_zsu_mte_aarch64
#define helper_sve_stdd_le_zsu_mte helper_sve_stdd_le_zsu_mte_aarch64
#define helper_sve_stdd_be_zsu_mte helper_sve_stdd_be_zsu_mte_aarch64
#define helper_sve_stbd_zss_mte helper_sve_stbd_zss_mte_aarch64
#define helper_sve_sthd_le_zss_mte helper_sve_sthd_le_zss_mte_aarch64
#define helper_sve_sthd_be_zss_mte helper_sve_sthd_be_zss_mte_aarch64
#define helper_sve_stsd_le_zss_mte helper_sve_stsd_le_zss_mte_aarch64
#define helper_sve_stsd_be_zss_mte helper_sve_stsd_be_zss_mte_aarch64
#define helper_sve_stdd_le_zss_mte helper_sve_stdd_le_zss_mte_aarch64
#define helper_sve_stdd_be_zss_mte helper_sve_stdd_be_zss_mte_aarch64
#define helper_sve_stbd_zd_mte helper_sve_stbd_zd_mte_aarch64
#define helper_sve_sthd_le_zd_mte helper_sve_sthd_le_zd_mte_aarch64
#define helper_sve_sthd_be_zd_mte helper_sve_sthd_be_zd_mte_aarch64
#define helper_sve_stsd_le_zd_mte helper_sve_stsd_le_zd_mte_aarch64
#define helper_sve_stsd_be_zd_mte helper_sve_stsd_be_zd_mte_aarch64
#define helper_sve_stdd_le_zd_mte helper_sve_stdd_le_zd_mte_aarch64
#define helper_sve_stdd_be_zd_mte helper_sve_stdd_be_zd_mte_aarch64
#define helper_sve_stbs_zss helper_sve_stbs_zss_aarch64
#define helper_sve_stbs_zsu helper_sve_stbs_zsu_aarch64
#define helper_sve_stdd_be_zd helper_sve_stdd_be_zd_aarch64

135
qemu/aarch64eb.h
View file

@ -4123,6 +4123,58 @@
#define helper_sve_ldbss_zd helper_sve_ldbss_zd_aarch64eb
#define helper_sve_ldbss_zss helper_sve_ldbss_zss_aarch64eb
#define helper_sve_ldbss_zsu helper_sve_ldbss_zsu_aarch64eb
#define helper_sve_ldbsu_zsu_mte helper_sve_ldbsu_zsu_mte_aarch64eb
#define helper_sve_ldhsu_le_zsu_mte helper_sve_ldhsu_le_zsu_mte_aarch64eb
#define helper_sve_ldhsu_be_zsu_mte helper_sve_ldhsu_be_zsu_mte_aarch64eb
#define helper_sve_ldss_le_zsu_mte helper_sve_ldss_le_zsu_mte_aarch64eb
#define helper_sve_ldss_be_zsu_mte helper_sve_ldss_be_zsu_mte_aarch64eb
#define helper_sve_ldbss_zsu_mte helper_sve_ldbss_zsu_mte_aarch64eb
#define helper_sve_ldhss_le_zsu_mte helper_sve_ldhss_le_zsu_mte_aarch64eb
#define helper_sve_ldhss_be_zsu_mte helper_sve_ldhss_be_zsu_mte_aarch64eb
#define helper_sve_ldbsu_zss_mte helper_sve_ldbsu_zss_mte_aarch64eb
#define helper_sve_ldhsu_le_zss_mte helper_sve_ldhsu_le_zss_mte_aarch64eb
#define helper_sve_ldhsu_be_zss_mte helper_sve_ldhsu_be_zss_mte_aarch64eb
#define helper_sve_ldss_le_zss_mte helper_sve_ldss_le_zss_mte_aarch64eb
#define helper_sve_ldss_be_zss_mte helper_sve_ldss_be_zss_mte_aarch64eb
#define helper_sve_ldbss_zss_mte helper_sve_ldbss_zss_mte_aarch64eb
#define helper_sve_ldhss_le_zss_mte helper_sve_ldhss_le_zss_mte_aarch64eb
#define helper_sve_ldhss_be_zss_mte helper_sve_ldhss_be_zss_mte_aarch64eb
#define helper_sve_ldbdu_zsu_mte helper_sve_ldbdu_zsu_mte_aarch64eb
#define helper_sve_ldhdu_le_zsu_mte helper_sve_ldhdu_le_zsu_mte_aarch64eb
#define helper_sve_ldhdu_be_zsu_mte helper_sve_ldhdu_be_zsu_mte_aarch64eb
#define helper_sve_ldsdu_le_zsu_mte helper_sve_ldsdu_le_zsu_mte_aarch64eb
#define helper_sve_ldsdu_be_zsu_mte helper_sve_ldsdu_be_zsu_mte_aarch64eb
#define helper_sve_lddd_le_zsu_mte helper_sve_lddd_le_zsu_mte_aarch64eb
#define helper_sve_lddd_be_zsu_mte helper_sve_lddd_be_zsu_mte_aarch64eb
#define helper_sve_ldbds_zsu_mte helper_sve_ldbds_zsu_mte_aarch64eb
#define helper_sve_ldhds_le_zsu_mte helper_sve_ldhds_le_zsu_mte_aarch64eb
#define helper_sve_ldhds_be_zsu_mte helper_sve_ldhds_be_zsu_mte_aarch64eb
#define helper_sve_ldsds_le_zsu_mte helper_sve_ldsds_le_zsu_mte_aarch64eb
#define helper_sve_ldsds_be_zsu_mte helper_sve_ldsds_be_zsu_mte_aarch64eb
#define helper_sve_ldbdu_zss_mte helper_sve_ldbdu_zss_mte_aarch64eb
#define helper_sve_ldhdu_le_zss_mte helper_sve_ldhdu_le_zss_mte_aarch64eb
#define helper_sve_ldhdu_be_zss_mte helper_sve_ldhdu_be_zss_mte_aarch64eb
#define helper_sve_ldsdu_le_zss_mte helper_sve_ldsdu_le_zss_mte_aarch64eb
#define helper_sve_ldsdu_be_zss_mte helper_sve_ldsdu_be_zss_mte_aarch64eb
#define helper_sve_lddd_le_zss_mte helper_sve_lddd_le_zss_mte_aarch64eb
#define helper_sve_lddd_be_zss_mte helper_sve_lddd_be_zss_mte_aarch64eb
#define helper_sve_ldbds_zss_mte helper_sve_ldbds_zss_mte_aarch64eb
#define helper_sve_ldhds_le_zss_mte helper_sve_ldhds_le_zss_mte_aarch64eb
#define helper_sve_ldhds_be_zss_mte helper_sve_ldhds_be_zss_mte_aarch64eb
#define helper_sve_ldsds_le_zss_mte helper_sve_ldsds_le_zss_mte_aarch64eb
#define helper_sve_ldsds_be_zss_mte helper_sve_ldsds_be_zss_mte_aarch64eb
#define helper_sve_ldbdu_zd_mte helper_sve_ldbdu_zd_mte_aarch64eb
#define helper_sve_ldhdu_le_zd_mte helper_sve_ldhdu_le_zd_mte_aarch64eb
#define helper_sve_ldhdu_be_zd_mte helper_sve_ldhdu_be_zd_mte_aarch64eb
#define helper_sve_ldsdu_le_zd_mte helper_sve_ldsdu_le_zd_mte_aarch64eb
#define helper_sve_ldsdu_be_zd_mte helper_sve_ldsdu_be_zd_mte_aarch64eb
#define helper_sve_lddd_le_zd_mte helper_sve_lddd_le_zd_mte_aarch64eb
#define helper_sve_lddd_be_zd_mte helper_sve_lddd_be_zd_mte_aarch64eb
#define helper_sve_ldbds_zd_mte helper_sve_ldbds_zd_mte_aarch64eb
#define helper_sve_ldhds_le_zd_mte helper_sve_ldhds_le_zd_mte_aarch64eb
#define helper_sve_ldhds_be_zd_mte helper_sve_ldhds_be_zd_mte_aarch64eb
#define helper_sve_ldsds_le_zd_mte helper_sve_ldsds_le_zd_mte_aarch64eb
#define helper_sve_ldsds_be_zd_mte helper_sve_ldsds_be_zd_mte_aarch64eb
#define helper_sve_ldbsu_zss helper_sve_ldbsu_zss_aarch64eb
#define helper_sve_ldbsu_zsu helper_sve_ldbsu_zsu_aarch64eb
#define helper_sve_ldbds_zd helper_sve_ldbds_zd_aarch64eb
@ -4324,6 +4376,58 @@
#define helper_sve_ldffbdu_zsu helper_sve_ldffbdu_zsu_aarch64eb
#define helper_sve_ldffbss_zss helper_sve_ldffbss_zss_aarch64eb
#define helper_sve_ldffbss_zsu helper_sve_ldffbss_zsu_aarch64eb
#define helper_sve_ldffbsu_zsu_mte helper_sve_ldffbsu_zsu_mte_aarch64eb
#define helper_sve_ldffhsu_le_zsu_mte helper_sve_ldffhsu_le_zsu_mte_aarch64eb
#define helper_sve_ldffhsu_be_zsu_mte helper_sve_ldffhsu_be_zsu_mte_aarch64eb
#define helper_sve_ldffss_le_zsu_mte helper_sve_ldffss_le_zsu_mte_aarch64eb
#define helper_sve_ldffss_be_zsu_mte helper_sve_ldffss_be_zsu_mte_aarch64eb
#define helper_sve_ldffbss_zsu_mte helper_sve_ldffbss_zsu_mte_aarch64eb
#define helper_sve_ldffhss_le_zsu_mte helper_sve_ldffhss_le_zsu_mte_aarch64eb
#define helper_sve_ldffhss_be_zsu_mte helper_sve_ldffhss_be_zsu_mte_aarch64eb
#define helper_sve_ldffbsu_zss_mte helper_sve_ldffbsu_zss_mte_aarch64eb
#define helper_sve_ldffhsu_le_zss_mte helper_sve_ldffhsu_le_zss_mte_aarch64eb
#define helper_sve_ldffhsu_be_zss_mte helper_sve_ldffhsu_be_zss_mte_aarch64eb
#define helper_sve_ldffss_le_zss_mte helper_sve_ldffss_le_zss_mte_aarch64eb
#define helper_sve_ldffss_be_zss_mte helper_sve_ldffss_be_zss_mte_aarch64eb
#define helper_sve_ldffbss_zss_mte helper_sve_ldffbss_zss_mte_aarch64eb
#define helper_sve_ldffhss_le_zss_mte helper_sve_ldffhss_le_zss_mte_aarch64eb
#define helper_sve_ldffhss_be_zss_mte helper_sve_ldffhss_be_zss_mte_aarch64eb
#define helper_sve_ldffbdu_zsu_mte helper_sve_ldffbdu_zsu_mte_aarch64eb
#define helper_sve_ldffhdu_le_zsu_mte helper_sve_ldffhdu_le_zsu_mte_aarch64eb
#define helper_sve_ldffhdu_be_zsu_mte helper_sve_ldffhdu_be_zsu_mte_aarch64eb
#define helper_sve_ldffsdu_le_zsu_mte helper_sve_ldffsdu_le_zsu_mte_aarch64eb
#define helper_sve_ldffsdu_be_zsu_mte helper_sve_ldffsdu_be_zsu_mte_aarch64eb
#define helper_sve_ldffdd_le_zsu_mte helper_sve_ldffdd_le_zsu_mte_aarch64eb
#define helper_sve_ldffdd_be_zsu_mte helper_sve_ldffdd_be_zsu_mte_aarch64eb
#define helper_sve_ldffbds_zsu_mte helper_sve_ldffbds_zsu_mte_aarch64eb
#define helper_sve_ldffhds_le_zsu_mte helper_sve_ldffhds_le_zsu_mte_aarch64eb
#define helper_sve_ldffhds_be_zsu_mte helper_sve_ldffhds_be_zsu_mte_aarch64eb
#define helper_sve_ldffsds_le_zsu_mte helper_sve_ldffsds_le_zsu_mte_aarch64eb
#define helper_sve_ldffsds_be_zsu_mte helper_sve_ldffsds_be_zsu_mte_aarch64eb
#define helper_sve_ldffbdu_zss_mte helper_sve_ldffbdu_zss_mte_aarch64eb
#define helper_sve_ldffhdu_le_zss_mte helper_sve_ldffhdu_le_zss_mte_aarch64eb
#define helper_sve_ldffhdu_be_zss_mte helper_sve_ldffhdu_be_zss_mte_aarch64eb
#define helper_sve_ldffsdu_le_zss_mte helper_sve_ldffsdu_le_zss_mte_aarch64eb
#define helper_sve_ldffsdu_be_zss_mte helper_sve_ldffsdu_be_zss_mte_aarch64eb
#define helper_sve_ldffdd_le_zss_mte helper_sve_ldffdd_le_zss_mte_aarch64eb
#define helper_sve_ldffdd_be_zss_mte helper_sve_ldffdd_be_zss_mte_aarch64eb
#define helper_sve_ldffbds_zss_mte helper_sve_ldffbds_zss_mte_aarch64eb
#define helper_sve_ldffhds_le_zss_mte helper_sve_ldffhds_le_zss_mte_aarch64eb
#define helper_sve_ldffhds_be_zss_mte helper_sve_ldffhds_be_zss_mte_aarch64eb
#define helper_sve_ldffsds_le_zss_mte helper_sve_ldffsds_le_zss_mte_aarch64eb
#define helper_sve_ldffsds_be_zss_mte helper_sve_ldffsds_be_zss_mte_aarch64eb
#define helper_sve_ldffbdu_zd_mte helper_sve_ldffbdu_zd_mte_aarch64eb
#define helper_sve_ldffhdu_le_zd_mte helper_sve_ldffhdu_le_zd_mte_aarch64eb
#define helper_sve_ldffhdu_be_zd_mte helper_sve_ldffhdu_be_zd_mte_aarch64eb
#define helper_sve_ldffsdu_le_zd_mte helper_sve_ldffsdu_le_zd_mte_aarch64eb
#define helper_sve_ldffsdu_be_zd_mte helper_sve_ldffsdu_be_zd_mte_aarch64eb
#define helper_sve_ldffdd_le_zd_mte helper_sve_ldffdd_le_zd_mte_aarch64eb
#define helper_sve_ldffdd_be_zd_mte helper_sve_ldffdd_be_zd_mte_aarch64eb
#define helper_sve_ldffbds_zd_mte helper_sve_ldffbds_zd_mte_aarch64eb
#define helper_sve_ldffhds_le_zd_mte helper_sve_ldffhds_le_zd_mte_aarch64eb
#define helper_sve_ldffhds_be_zd_mte helper_sve_ldffhds_be_zd_mte_aarch64eb
#define helper_sve_ldffsds_le_zd_mte helper_sve_ldffsds_le_zd_mte_aarch64eb
#define helper_sve_ldffsds_be_zd_mte helper_sve_ldffsds_be_zd_mte_aarch64eb
#define helper_sve_ldffbsu_zss helper_sve_ldffbsu_zss_aarch64eb
#define helper_sve_ldffbsu_zsu helper_sve_ldffbsu_zsu_aarch64eb
#define helper_sve_ldffdd_be_zd helper_sve_ldffdd_be_zd_aarch64eb
@ -4583,6 +4687,37 @@
#define helper_sve_stbd_zd helper_sve_stbd_zd_aarch64eb
#define helper_sve_stbd_zss helper_sve_stbd_zss_aarch64eb
#define helper_sve_stbd_zsu helper_sve_stbd_zsu_aarch64eb
#define helper_sve_stbs_zsu_mte helper_sve_stbs_zsu_mte_aarch64eb
#define helper_sve_sths_le_zsu_mte helper_sve_sths_le_zsu_mte_aarch64eb
#define helper_sve_sths_be_zsu_mte helper_sve_sths_be_zsu_mte_aarch64eb
#define helper_sve_stss_le_zsu_mte helper_sve_stss_le_zsu_mte_aarch64eb
#define helper_sve_stss_be_zsu_mte helper_sve_stss_be_zsu_mte_aarch64eb
#define helper_sve_stbs_zss_mte helper_sve_stbs_zss_mte_aarch64eb
#define helper_sve_sths_le_zss_mte helper_sve_sths_le_zss_mte_aarch64eb
#define helper_sve_sths_be_zss_mte helper_sve_sths_be_zss_mte_aarch64eb
#define helper_sve_stss_le_zss_mte helper_sve_stss_le_zss_mte_aarch64eb
#define helper_sve_stss_be_zss_mte helper_sve_stss_be_zss_mte_aarch64eb
#define helper_sve_stbd_zsu_mte helper_sve_stbd_zsu_mte_aarch64eb
#define helper_sve_sthd_le_zsu_mte helper_sve_sthd_le_zsu_mte_aarch64eb
#define helper_sve_sthd_be_zsu_mte helper_sve_sthd_be_zsu_mte_aarch64eb
#define helper_sve_stsd_le_zsu_mte helper_sve_stsd_le_zsu_mte_aarch64eb
#define helper_sve_stsd_be_zsu_mte helper_sve_stsd_be_zsu_mte_aarch64eb
#define helper_sve_stdd_le_zsu_mte helper_sve_stdd_le_zsu_mte_aarch64eb
#define helper_sve_stdd_be_zsu_mte helper_sve_stdd_be_zsu_mte_aarch64eb
#define helper_sve_stbd_zss_mte helper_sve_stbd_zss_mte_aarch64eb
#define helper_sve_sthd_le_zss_mte helper_sve_sthd_le_zss_mte_aarch64eb
#define helper_sve_sthd_be_zss_mte helper_sve_sthd_be_zss_mte_aarch64eb
#define helper_sve_stsd_le_zss_mte helper_sve_stsd_le_zss_mte_aarch64eb
#define helper_sve_stsd_be_zss_mte helper_sve_stsd_be_zss_mte_aarch64eb
#define helper_sve_stdd_le_zss_mte helper_sve_stdd_le_zss_mte_aarch64eb
#define helper_sve_stdd_be_zss_mte helper_sve_stdd_be_zss_mte_aarch64eb
#define helper_sve_stbd_zd_mte helper_sve_stbd_zd_mte_aarch64eb
#define helper_sve_sthd_le_zd_mte helper_sve_sthd_le_zd_mte_aarch64eb
#define helper_sve_sthd_be_zd_mte helper_sve_sthd_be_zd_mte_aarch64eb
#define helper_sve_stsd_le_zd_mte helper_sve_stsd_le_zd_mte_aarch64eb
#define helper_sve_stsd_be_zd_mte helper_sve_stsd_be_zd_mte_aarch64eb
#define helper_sve_stdd_le_zd_mte helper_sve_stdd_le_zd_mte_aarch64eb
#define helper_sve_stdd_be_zd_mte helper_sve_stdd_be_zd_mte_aarch64eb
#define helper_sve_stbs_zss helper_sve_stbs_zss_aarch64eb
#define helper_sve_stbs_zsu helper_sve_stbs_zsu_aarch64eb
#define helper_sve_stdd_be_zd helper_sve_stdd_be_zd_aarch64eb

135
qemu/header_gen.py
View file

@ -4263,6 +4263,58 @@ aarch64_symbols = (
'helper_sve_ldbss_zd',
'helper_sve_ldbss_zss',
'helper_sve_ldbss_zsu',
'helper_sve_ldbsu_zsu_mte',
'helper_sve_ldhsu_le_zsu_mte',
'helper_sve_ldhsu_be_zsu_mte',
'helper_sve_ldss_le_zsu_mte',
'helper_sve_ldss_be_zsu_mte',
'helper_sve_ldbss_zsu_mte',
'helper_sve_ldhss_le_zsu_mte',
'helper_sve_ldhss_be_zsu_mte',
'helper_sve_ldbsu_zss_mte',
'helper_sve_ldhsu_le_zss_mte',
'helper_sve_ldhsu_be_zss_mte',
'helper_sve_ldss_le_zss_mte',
'helper_sve_ldss_be_zss_mte',
'helper_sve_ldbss_zss_mte',
'helper_sve_ldhss_le_zss_mte',
'helper_sve_ldhss_be_zss_mte',
'helper_sve_ldbdu_zsu_mte',
'helper_sve_ldhdu_le_zsu_mte',
'helper_sve_ldhdu_be_zsu_mte',
'helper_sve_ldsdu_le_zsu_mte',
'helper_sve_ldsdu_be_zsu_mte',
'helper_sve_lddd_le_zsu_mte',
'helper_sve_lddd_be_zsu_mte',
'helper_sve_ldbds_zsu_mte',
'helper_sve_ldhds_le_zsu_mte',
'helper_sve_ldhds_be_zsu_mte',
'helper_sve_ldsds_le_zsu_mte',
'helper_sve_ldsds_be_zsu_mte',
'helper_sve_ldbdu_zss_mte',
'helper_sve_ldhdu_le_zss_mte',
'helper_sve_ldhdu_be_zss_mte',
'helper_sve_ldsdu_le_zss_mte',
'helper_sve_ldsdu_be_zss_mte',
'helper_sve_lddd_le_zss_mte',
'helper_sve_lddd_be_zss_mte',
'helper_sve_ldbds_zss_mte',
'helper_sve_ldhds_le_zss_mte',
'helper_sve_ldhds_be_zss_mte',
'helper_sve_ldsds_le_zss_mte',
'helper_sve_ldsds_be_zss_mte',
'helper_sve_ldbdu_zd_mte',
'helper_sve_ldhdu_le_zd_mte',
'helper_sve_ldhdu_be_zd_mte',
'helper_sve_ldsdu_le_zd_mte',
'helper_sve_ldsdu_be_zd_mte',
'helper_sve_lddd_le_zd_mte',
'helper_sve_lddd_be_zd_mte',
'helper_sve_ldbds_zd_mte',
'helper_sve_ldhds_le_zd_mte',
'helper_sve_ldhds_be_zd_mte',
'helper_sve_ldsds_le_zd_mte',
'helper_sve_ldsds_be_zd_mte',
'helper_sve_ldbsu_zss',
'helper_sve_ldbsu_zsu',
'helper_sve_ldbds_zd',
@ -4464,6 +4516,58 @@ aarch64_symbols = (
'helper_sve_ldffbdu_zsu',
'helper_sve_ldffbss_zss',
'helper_sve_ldffbss_zsu',
'helper_sve_ldffbsu_zsu_mte',
'helper_sve_ldffhsu_le_zsu_mte',
'helper_sve_ldffhsu_be_zsu_mte',
'helper_sve_ldffss_le_zsu_mte',
'helper_sve_ldffss_be_zsu_mte',
'helper_sve_ldffbss_zsu_mte',
'helper_sve_ldffhss_le_zsu_mte',
'helper_sve_ldffhss_be_zsu_mte',
'helper_sve_ldffbsu_zss_mte',
'helper_sve_ldffhsu_le_zss_mte',
'helper_sve_ldffhsu_be_zss_mte',
'helper_sve_ldffss_le_zss_mte',
'helper_sve_ldffss_be_zss_mte',
'helper_sve_ldffbss_zss_mte',
'helper_sve_ldffhss_le_zss_mte',
'helper_sve_ldffhss_be_zss_mte',
'helper_sve_ldffbdu_zsu_mte',
'helper_sve_ldffhdu_le_zsu_mte',
'helper_sve_ldffhdu_be_zsu_mte',
'helper_sve_ldffsdu_le_zsu_mte',
'helper_sve_ldffsdu_be_zsu_mte',
'helper_sve_ldffdd_le_zsu_mte',
'helper_sve_ldffdd_be_zsu_mte',
'helper_sve_ldffbds_zsu_mte',
'helper_sve_ldffhds_le_zsu_mte',
'helper_sve_ldffhds_be_zsu_mte',
'helper_sve_ldffsds_le_zsu_mte',
'helper_sve_ldffsds_be_zsu_mte',
'helper_sve_ldffbdu_zss_mte',
'helper_sve_ldffhdu_le_zss_mte',
'helper_sve_ldffhdu_be_zss_mte',
'helper_sve_ldffsdu_le_zss_mte',
'helper_sve_ldffsdu_be_zss_mte',
'helper_sve_ldffdd_le_zss_mte',
'helper_sve_ldffdd_be_zss_mte',
'helper_sve_ldffbds_zss_mte',
'helper_sve_ldffhds_le_zss_mte',
'helper_sve_ldffhds_be_zss_mte',
'helper_sve_ldffsds_le_zss_mte',
'helper_sve_ldffsds_be_zss_mte',
'helper_sve_ldffbdu_zd_mte',
'helper_sve_ldffhdu_le_zd_mte',
'helper_sve_ldffhdu_be_zd_mte',
'helper_sve_ldffsdu_le_zd_mte',
'helper_sve_ldffsdu_be_zd_mte',
'helper_sve_ldffdd_le_zd_mte',
'helper_sve_ldffdd_be_zd_mte',
'helper_sve_ldffbds_zd_mte',
'helper_sve_ldffhds_le_zd_mte',
'helper_sve_ldffhds_be_zd_mte',
'helper_sve_ldffsds_le_zd_mte',
'helper_sve_ldffsds_be_zd_mte',
'helper_sve_ldffbsu_zss',
'helper_sve_ldffbsu_zsu',
'helper_sve_ldffdd_be_zd',
@ -4723,6 +4827,37 @@ aarch64_symbols = (
'helper_sve_stbd_zd',
'helper_sve_stbd_zss',
'helper_sve_stbd_zsu',
'helper_sve_stbs_zsu_mte',
'helper_sve_sths_le_zsu_mte',
'helper_sve_sths_be_zsu_mte',
'helper_sve_stss_le_zsu_mte',
'helper_sve_stss_be_zsu_mte',
'helper_sve_stbs_zss_mte',
'helper_sve_sths_le_zss_mte',
'helper_sve_sths_be_zss_mte',
'helper_sve_stss_le_zss_mte',
'helper_sve_stss_be_zss_mte',
'helper_sve_stbd_zsu_mte',
'helper_sve_sthd_le_zsu_mte',
'helper_sve_sthd_be_zsu_mte',
'helper_sve_stsd_le_zsu_mte',
'helper_sve_stsd_be_zsu_mte',
'helper_sve_stdd_le_zsu_mte',
'helper_sve_stdd_be_zsu_mte',
'helper_sve_stbd_zss_mte',
'helper_sve_sthd_le_zss_mte',
'helper_sve_sthd_be_zss_mte',
'helper_sve_stsd_le_zss_mte',
'helper_sve_stsd_be_zss_mte',
'helper_sve_stdd_le_zss_mte',
'helper_sve_stdd_be_zss_mte',
'helper_sve_stbd_zd_mte',
'helper_sve_sthd_le_zd_mte',
'helper_sve_sthd_be_zd_mte',
'helper_sve_stsd_le_zd_mte',
'helper_sve_stsd_be_zd_mte',
'helper_sve_stdd_le_zd_mte',
'helper_sve_stdd_be_zd_mte',
'helper_sve_stbs_zss',
'helper_sve_stbs_zsu',
'helper_sve_stdd_be_zd',

285
qemu/target/arm/helper-sve.h
View file

@ -1617,6 +1617,115 @@ DEF_HELPER_FLAGS_6(sve_ldsds_le_zd, TCG_CALL_NO_WG,
DEF_HELPER_FLAGS_6(sve_ldsds_be_zd, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldbsu_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldhsu_le_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldhsu_be_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldss_le_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldss_be_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldbss_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldhss_le_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldhss_be_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldbsu_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldhsu_le_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldhsu_be_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldss_le_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldss_be_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldbss_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldhss_le_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldhss_be_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldbdu_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldhdu_le_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldhdu_be_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldsdu_le_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldsdu_be_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_lddd_le_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_lddd_be_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldbds_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldhds_le_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldhds_be_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldsds_le_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldsds_be_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldbdu_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldhdu_le_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldhdu_be_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldsdu_le_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldsdu_be_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_lddd_le_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_lddd_be_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldbds_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldhds_le_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldhds_be_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldsds_le_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldsds_be_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldbdu_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldhdu_le_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldhdu_be_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldsdu_le_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldsdu_be_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_lddd_le_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_lddd_be_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldbds_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldhds_le_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldhds_be_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldsds_le_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldsds_be_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffbsu_zsu, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffhsu_le_zsu, TCG_CALL_NO_WG,
@ -1726,6 +1835,115 @@ DEF_HELPER_FLAGS_6(sve_ldffsds_le_zd, TCG_CALL_NO_WG,
DEF_HELPER_FLAGS_6(sve_ldffsds_be_zd, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffbsu_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffhsu_le_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffhsu_be_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffss_le_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffss_be_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffbss_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffhss_le_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffhss_be_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffbsu_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffhsu_le_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffhsu_be_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffss_le_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffss_be_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffbss_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffhss_le_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffhss_be_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffbdu_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffhdu_le_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffhdu_be_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffsdu_le_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffsdu_be_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffdd_le_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffdd_be_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffbds_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffhds_le_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffhds_be_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffsds_le_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffsds_be_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffbdu_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffhdu_le_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffhdu_be_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffsdu_le_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffsdu_be_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffdd_le_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffdd_be_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffbds_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffhds_le_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffhds_be_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffsds_le_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffsds_be_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffbdu_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffhdu_le_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffhdu_be_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffsdu_le_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffsdu_be_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffdd_le_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffdd_be_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffbds_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffhds_le_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffhds_be_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffsds_le_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_ldffsds_be_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_stbs_zsu, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_sths_le_zsu, TCG_CALL_NO_WG,
@ -1793,4 +2011,71 @@ DEF_HELPER_FLAGS_6(sve_stdd_le_zd, TCG_CALL_NO_WG,
DEF_HELPER_FLAGS_6(sve_stdd_be_zd, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_stbs_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_sths_le_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_sths_be_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_stss_le_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_stss_be_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_stbs_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_sths_le_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_sths_be_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_stss_le_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_stss_be_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_stbd_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_sthd_le_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_sthd_be_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_stsd_le_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_stsd_be_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_stdd_le_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_stdd_be_zsu_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_stbd_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_sthd_le_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_sthd_be_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_stsd_le_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_stsd_be_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_stdd_le_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_stdd_be_zss_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_stbd_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_sthd_le_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_sthd_be_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_stsd_le_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_stsd_be_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_stdd_le_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_6(sve_stdd_be_zd_mte, TCG_CALL_NO_WG,
void, env, ptr, ptr, ptr, tl, i32)
DEF_HELPER_FLAGS_4(sve2_pmull_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)

185
qemu/target/arm/sve_helper.c
View file

@ -5353,7 +5353,8 @@ static target_ulong off_zd_d(void *reg, intptr_t reg_ofs)
static inline
void sve_ld1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm,
target_ulong base, uint32_t desc, uintptr_t retaddr,
int esize, int msize, zreg_off_fn *off_fn,
uint32_t mtedesc, int esize, int msize,
zreg_off_fn *off_fn,
sve_ldst1_host_fn *host_fn,
sve_ldst1_tlb_fn *tlb_fn)
{
@ -5381,7 +5382,9 @@ void sve_ld1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm,
cpu_check_watchpoint(env_cpu(env), addr, msize,
info.attrs, BP_MEM_READ, retaddr);
}
/* TODO: MTE check */
if (mtedesc && arm_tlb_mte_tagged(&info.attrs)) {
mte_check1_(env, mtedesc, addr, retaddr);
}
host_fn(&scratch, reg_off, info.host);
} else {
/* Element crosses the page boundary. */
@ -5392,7 +5395,9 @@ void sve_ld1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm,
msize, info.attrs,
BP_MEM_READ, retaddr);
}
/* TODO: MTE check */
if (mtedesc && arm_tlb_mte_tagged(&info.attrs)) {
mte_check1_(env, mtedesc, addr, retaddr);
}
tlb_fn(env, &scratch, reg_off, addr, retaddr);
}
}
@ -5406,20 +5411,53 @@ void sve_ld1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm,
memcpy(vd, &scratch, reg_max);
}
static inline
void sve_ld1_z_mte(CPUARMState *env, void *vd, uint64_t *vg, void *vm,
target_ulong base, uint32_t desc, uintptr_t retaddr,
int esize, int msize, zreg_off_fn *off_fn,
sve_ldst1_host_fn *host_fn,
sve_ldst1_tlb_fn *tlb_fn)
{
uint32_t mtedesc = desc >> (SIMD_DATA_SHIFT + SVE_MTEDESC_SHIFT);
/* Remove mtedesc from the normal sve descriptor. */
desc = extract32(desc, 0, SIMD_DATA_SHIFT + SVE_MTEDESC_SHIFT);
/*
* ??? TODO: For the 32-bit offset extractions, base + ofs cannot
* offset base entirely over the address space hole to change the
* pointer tag, or change the bit55 selector. So we could here
* examine TBI + TCMA like we do for sve_ldN_r_mte().
*/
sve_ld1_z(env, vd, vg, vm, base, desc, retaddr, mtedesc,
esize, msize, off_fn, host_fn, tlb_fn);
}
#define DO_LD1_ZPZ_S(MEM, OFS, MSZ) \
void HELPER(sve_ld##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \
void *vm, target_ulong base, uint32_t desc) \
{ \
sve_ld1_z(env, vd, vg, vm, base, desc, GETPC(), 4, 1 << MSZ, \
sve_ld1_z(env, vd, vg, vm, base, desc, GETPC(), 0, 4, 1 << MSZ, \
off_##OFS##_s, sve_ld1##MEM##_host, sve_ld1##MEM##_tlb); \
} \
void HELPER(sve_ld##MEM##_##OFS##_mte)(CPUARMState *env, void *vd, void *vg, \
void *vm, target_ulong base, uint32_t desc) \
{ \
sve_ld1_z_mte(env, vd, vg, vm, base, desc, GETPC(), 4, 1 << MSZ, \
off_##OFS##_s, sve_ld1##MEM##_host, sve_ld1##MEM##_tlb); \
}
#define DO_LD1_ZPZ_D(MEM, OFS, MSZ) \
void HELPER(sve_ld##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \
void *vm, target_ulong base, uint32_t desc) \
{ \
sve_ld1_z(env, vd, vg, vm, base, desc, GETPC(), 8, 1 << MSZ, \
sve_ld1_z(env, vd, vg, vm, base, desc, GETPC(), 0, 8, 1 << MSZ, \
off_##OFS##_d, sve_ld1##MEM##_host, sve_ld1##MEM##_tlb); \
} \
void HELPER(sve_ld##MEM##_##OFS##_mte)(CPUARMState *env, void *vd, void *vg, \
void *vm, target_ulong base, uint32_t desc) \
{ \
sve_ld1_z_mte(env, vd, vg, vm, base, desc, GETPC(), 8, 1 << MSZ, \
off_##OFS##_d, sve_ld1##MEM##_host, sve_ld1##MEM##_tlb); \
}
DO_LD1_ZPZ_S(bsu, zsu, MO_8)
@ -5497,7 +5535,8 @@ DO_LD1_ZPZ_D(dd_be, zd, MO_64)
static inline
void sve_ldff1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm,
target_ulong base, uint32_t desc, uintptr_t retaddr,
const int esz, const int msz, zreg_off_fn *off_fn,
uint32_t mtedesc, const int esz, const int msz,
zreg_off_fn *off_fn,
sve_ldst1_host_fn *host_fn,
sve_ldst1_tlb_fn *tlb_fn)
{
@ -5522,6 +5561,9 @@ void sve_ldff1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm,
* Probe the first element, allowing faults.
*/
addr = base + (off_fn(vm, reg_off) << scale);
if (mtedesc) {
mte_check1_(env, mtedesc, addr, retaddr);
}
tlb_fn(env, vd, reg_off, addr, retaddr);
/* After any fault, zero the other elements. */
@ -5554,7 +5596,11 @@ void sve_ldff1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm,
(env_cpu(env), addr, msize) & BP_MEM_READ)) {
goto fault;
}
/* TODO: MTE check. */
if (mtedesc &&
arm_tlb_mte_tagged(&info.attrs) &&
!mte_probe1(env, mtedesc, addr)) {
goto fault;
}
host_fn(vd, reg_off, info.host);
}
@ -5568,20 +5614,58 @@ void sve_ldff1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm,
record_fault(env, reg_off, reg_max);
}
#define DO_LDFF1_ZPZ_S(MEM, OFS, MSZ) \
void HELPER(sve_ldff##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \
void *vm, target_ulong base, uint32_t desc) \
{ \
sve_ldff1_z(env, vd, vg, vm, base, desc, GETPC(), MO_32, MSZ, \
off_##OFS##_s, sve_ld1##MEM##_host, sve_ld1##MEM##_tlb); \
static inline
void sve_ldff1_z_mte(CPUARMState *env, void *vd, uint64_t *vg, void *vm,
target_ulong base, uint32_t desc, uintptr_t retaddr,
const int esz, const int msz,
zreg_off_fn *off_fn,
sve_ldst1_host_fn *host_fn,
sve_ldst1_tlb_fn *tlb_fn)
{
uint32_t mtedesc = desc >> (SIMD_DATA_SHIFT + SVE_MTEDESC_SHIFT);
/* Remove mtedesc from the normal sve descriptor. */
desc = extract32(desc, 0, SIMD_DATA_SHIFT + SVE_MTEDESC_SHIFT);
/*
* ??? TODO: For the 32-bit offset extractions, base + ofs cannot
* offset base entirely over the address space hole to change the
* pointer tag, or change the bit55 selector. So we could here
* examine TBI + TCMA like we do for sve_ldN_r_mte().
*/
sve_ldff1_z(env, vd, vg, vm, base, desc, retaddr, mtedesc,
esz, msz, off_fn, host_fn, tlb_fn);
}
#define DO_LDFF1_ZPZ_D(MEM, OFS, MSZ) \
void HELPER(sve_ldff##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \
void *vm, target_ulong base, uint32_t desc) \
{ \
sve_ldff1_z(env, vd, vg, vm, base, desc, GETPC(), MO_64, MSZ, \
off_##OFS##_d, sve_ld1##MEM##_host, sve_ld1##MEM##_tlb); \
#define DO_LDFF1_ZPZ_S(MEM, OFS, MSZ) \
void HELPER(sve_ldff##MEM##_##OFS) \
(CPUARMState *env, void *vd, void *vg, \
void *vm, target_ulong base, uint32_t desc) \
{ \
sve_ldff1_z(env, vd, vg, vm, base, desc, GETPC(), 0, MO_32, MSZ, \
off_##OFS##_s, sve_ld1##MEM##_host, sve_ld1##MEM##_tlb); \
} \
void HELPER(sve_ldff##MEM##_##OFS##_mte) \
(CPUARMState *env, void *vd, void *vg, \
void *vm, target_ulong base, uint32_t desc) \
{ \
sve_ldff1_z_mte(env, vd, vg, vm, base, desc, GETPC(), MO_32, MSZ, \
off_##OFS##_s, sve_ld1##MEM##_host, sve_ld1##MEM##_tlb); \
}
#define DO_LDFF1_ZPZ_D(MEM, OFS, MSZ) \
void HELPER(sve_ldff##MEM##_##OFS) \
(CPUARMState *env, void *vd, void *vg, \
void *vm, target_ulong base, uint32_t desc) \
{ \
sve_ldff1_z(env, vd, vg, vm, base, desc, GETPC(), 0, MO_64, MSZ, \
off_##OFS##_d, sve_ld1##MEM##_host, sve_ld1##MEM##_tlb); \
} \
void HELPER(sve_ldff##MEM##_##OFS##_mte) \
(CPUARMState *env, void *vd, void *vg, \
void *vm, target_ulong base, uint32_t desc) \
{ \
sve_ldff1_z_mte(env, vd, vg, vm, base, desc, GETPC(), MO_64, MSZ, \
off_##OFS##_d, sve_ld1##MEM##_host, sve_ld1##MEM##_tlb); \
}
DO_LDFF1_ZPZ_S(bsu, zsu, MO_8)
@ -5653,7 +5737,8 @@ DO_LDFF1_ZPZ_D(dd_be, zd, MO_64)
static inline
void sve_st1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm,
target_ulong base, uint32_t desc, uintptr_t retaddr,
int esize, int msize, zreg_off_fn *off_fn,
uint32_t mtedesc, int esize, int msize,
zreg_off_fn *off_fn,
sve_ldst1_host_fn *host_fn,
sve_ldst1_tlb_fn *tlb_fn)
{
@ -5698,7 +5783,10 @@ void sve_st1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm,
cpu_check_watchpoint(env_cpu(env), addr, msize,
info.attrs, BP_MEM_WRITE, retaddr);
}
/* TODO: MTE check. */
if (mtedesc && arm_tlb_mte_tagged(&info.attrs)) {
mte_check1_(env, mtedesc, addr, retaddr);
}
}
i += 1;
reg_off += esize;
@ -5729,20 +5817,53 @@ void sve_st1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm,
} while (reg_off < reg_max);
}
#define DO_ST1_ZPZ_S(MEM, OFS, MSZ) \
void HELPER(sve_st##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \
void *vm, target_ulong base, uint32_t desc) \
{ \
sve_st1_z(env, vd, vg, vm, base, desc, GETPC(), 4, 1 << MSZ, \
off_##OFS##_s, sve_st1##MEM##_host, sve_st1##MEM##_tlb); \
static inline
void sve_st1_z_mte(CPUARMState *env, void *vd, uint64_t *vg, void *vm,
target_ulong base, uint32_t desc, uintptr_t retaddr,
int esize, int msize, zreg_off_fn *off_fn,
sve_ldst1_host_fn *host_fn,
sve_ldst1_tlb_fn *tlb_fn)
{
uint32_t mtedesc = desc >> (SIMD_DATA_SHIFT + SVE_MTEDESC_SHIFT);
/* Remove mtedesc from the normal sve descriptor. */
desc = extract32(desc, 0, SIMD_DATA_SHIFT + SVE_MTEDESC_SHIFT);
/*
* ??? TODO: For the 32-bit offset extractions, base + ofs cannot
* offset base entirely over the address space hole to change the
* pointer tag, or change the bit55 selector. So we could here
* examine TBI + TCMA like we do for sve_ldN_r_mte().
*/
sve_st1_z(env, vd, vg, vm, base, desc, retaddr, mtedesc,
esize, msize, off_fn, host_fn, tlb_fn);
}
#define DO_ST1_ZPZ_D(MEM, OFS, MSZ) \
void HELPER(sve_st##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \
#define DO_ST1_ZPZ_S(MEM, OFS, MSZ) \
void HELPER(sve_st##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \
void *vm, target_ulong base, uint32_t desc) \
{ \
sve_st1_z(env, vd, vg, vm, base, desc, GETPC(), 8, 1 << MSZ, \
off_##OFS##_d, sve_st1##MEM##_host, sve_st1##MEM##_tlb); \
{ \
sve_st1_z(env, vd, vg, vm, base, desc, GETPC(), 0, 4, 1 << MSZ, \
off_##OFS##_s, sve_st1##MEM##_host, sve_st1##MEM##_tlb); \
} \
void HELPER(sve_st##MEM##_##OFS##_mte)(CPUARMState *env, void *vd, void *vg, \
void *vm, target_ulong base, uint32_t desc) \
{ \
sve_st1_z_mte(env, vd, vg, vm, base, desc, GETPC(), 4, 1 << MSZ, \
off_##OFS##_s, sve_st1##MEM##_host, sve_st1##MEM##_tlb); \
}
#define DO_ST1_ZPZ_D(MEM, OFS, MSZ) \
void HELPER(sve_st##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \
void *vm, target_ulong base, uint32_t desc) \
{ \
sve_st1_z(env, vd, vg, vm, base, desc, GETPC(), 0, 8, 1 << MSZ, \
off_##OFS##_d, sve_st1##MEM##_host, sve_st1##MEM##_tlb); \
} \
void HELPER(sve_st##MEM##_##OFS##_mte)(CPUARMState *env, void *vd, void *vg, \
void *vm, target_ulong base, uint32_t desc) \
{ \
sve_st1_z_mte(env, vd, vg, vm, base, desc, GETPC(), 8, 1 << MSZ, \
off_##OFS##_d, sve_st1##MEM##_host, sve_st1##MEM##_tlb); \
}
DO_ST1_ZPZ_S(bs, zsu, MO_8)

651
qemu/target/arm/translate-sve.c
View file

@ -5419,7 +5419,7 @@ static bool trans_ST_zpri(DisasContext *s, arg_rpri_store *a)
*/
static void do_mem_zpz(DisasContext *s, int zt, int pg, int zm,
int scale, TCGv_i64 scalar, int msz,
int scale, TCGv_i64 scalar, int msz, bool is_write,
gen_helper_gvec_mem_scatter *fn)
{
TCGContext *tcg_ctx = s->uc->tcg_ctx;
@ -5428,8 +5428,16 @@ static void do_mem_zpz(DisasContext *s, int zt, int pg, int zm,
TCGv_ptr t_pg = tcg_temp_new_ptr(tcg_ctx);
TCGv_ptr t_zt = tcg_temp_new_ptr(tcg_ctx);
TCGv_i32 t_desc;
int desc;
int desc = 0;
if (s->mte_active[0]) {
desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s));
desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write);
desc = FIELD_DP32(desc, MTEDESC, ESIZE, 1 << msz);
desc <<= SVE_MTEDESC_SHIFT;
}
desc = simd_desc(vsz, vsz, scale);
t_desc = tcg_const_i32(tcg_ctx, desc);
@ -5444,176 +5452,339 @@ static void do_mem_zpz(DisasContext *s, int zt, int pg, int zm,
tcg_temp_free_i32(tcg_ctx, t_desc);
}
/* Indexed by [be][ff][xs][u][msz]. */
static gen_helper_gvec_mem_scatter * const gather_load_fn32[2][2][2][2][3] = {
/* Little-endian */
{ { { { gen_helper_sve_ldbss_zsu,
gen_helper_sve_ldhss_le_zsu,
NULL, },
{ gen_helper_sve_ldbsu_zsu,
gen_helper_sve_ldhsu_le_zsu,
gen_helper_sve_ldss_le_zsu, } },
{ { gen_helper_sve_ldbss_zss,
gen_helper_sve_ldhss_le_zss,
NULL, },
{ gen_helper_sve_ldbsu_zss,
gen_helper_sve_ldhsu_le_zss,
gen_helper_sve_ldss_le_zss, } } },
/* Indexed by [mte][be][ff][xs][u][msz]. */
static gen_helper_gvec_mem_scatter * const
gather_load_fn32[2][2][2][2][2][3] = {
{ /* MTE Inactive */
{ /* Little-endian */
{ { { gen_helper_sve_ldbss_zsu,
gen_helper_sve_ldhss_le_zsu,
NULL, },
{ gen_helper_sve_ldbsu_zsu,
gen_helper_sve_ldhsu_le_zsu,
gen_helper_sve_ldss_le_zsu, } },
{ { gen_helper_sve_ldbss_zss,
gen_helper_sve_ldhss_le_zss,
NULL, },
{ gen_helper_sve_ldbsu_zss,
gen_helper_sve_ldhsu_le_zss,
gen_helper_sve_ldss_le_zss, } } },
/* First-fault */
{ { { gen_helper_sve_ldffbss_zsu,
gen_helper_sve_ldffhss_le_zsu,
NULL, },
{ gen_helper_sve_ldffbsu_zsu,
gen_helper_sve_ldffhsu_le_zsu,
gen_helper_sve_ldffss_le_zsu, } },
{ { gen_helper_sve_ldffbss_zss,
gen_helper_sve_ldffhss_le_zss,
NULL, },
{ gen_helper_sve_ldffbsu_zss,
gen_helper_sve_ldffhsu_le_zss,
gen_helper_sve_ldffss_le_zss, } } } },
/* First-fault */
{ { { gen_helper_sve_ldffbss_zsu,
gen_helper_sve_ldffhss_le_zsu,
NULL, },
{ gen_helper_sve_ldffbsu_zsu,
gen_helper_sve_ldffhsu_le_zsu,
gen_helper_sve_ldffss_le_zsu, } },
{ { gen_helper_sve_ldffbss_zss,
gen_helper_sve_ldffhss_le_zss,
NULL, },
{ gen_helper_sve_ldffbsu_zss,
gen_helper_sve_ldffhsu_le_zss,
gen_helper_sve_ldffss_le_zss, } } } },
/* Big-endian */
{ { { { gen_helper_sve_ldbss_zsu,
gen_helper_sve_ldhss_be_zsu,
NULL, },
{ gen_helper_sve_ldbsu_zsu,
gen_helper_sve_ldhsu_be_zsu,
gen_helper_sve_ldss_be_zsu, } },
{ { gen_helper_sve_ldbss_zss,
gen_helper_sve_ldhss_be_zss,
NULL, },
{ gen_helper_sve_ldbsu_zss,
gen_helper_sve_ldhsu_be_zss,
gen_helper_sve_ldss_be_zss, } } },
{ /* Big-endian */
{ { { gen_helper_sve_ldbss_zsu,
gen_helper_sve_ldhss_be_zsu,
NULL, },
{ gen_helper_sve_ldbsu_zsu,
gen_helper_sve_ldhsu_be_zsu,
gen_helper_sve_ldss_be_zsu, } },
{ { gen_helper_sve_ldbss_zss,
gen_helper_sve_ldhss_be_zss,
NULL, },
{ gen_helper_sve_ldbsu_zss,
gen_helper_sve_ldhsu_be_zss,
gen_helper_sve_ldss_be_zss, } } },
/* First-fault */
{ { { gen_helper_sve_ldffbss_zsu,
gen_helper_sve_ldffhss_be_zsu,
NULL, },
{ gen_helper_sve_ldffbsu_zsu,
gen_helper_sve_ldffhsu_be_zsu,
gen_helper_sve_ldffss_be_zsu, } },
{ { gen_helper_sve_ldffbss_zss,
gen_helper_sve_ldffhss_be_zss,
NULL, },
{ gen_helper_sve_ldffbsu_zss,
gen_helper_sve_ldffhsu_be_zss,
gen_helper_sve_ldffss_be_zss, } } } },
/* First-fault */
{ { { gen_helper_sve_ldffbss_zsu,
gen_helper_sve_ldffhss_be_zsu,
NULL, },
{ gen_helper_sve_ldffbsu_zsu,
gen_helper_sve_ldffhsu_be_zsu,
gen_helper_sve_ldffss_be_zsu, } },
{ { gen_helper_sve_ldffbss_zss,
gen_helper_sve_ldffhss_be_zss,
NULL, },
{ gen_helper_sve_ldffbsu_zss,
gen_helper_sve_ldffhsu_be_zss,
gen_helper_sve_ldffss_be_zss, } } } } },
{ /* MTE Active */
{ /* Little-endian */
{ { { gen_helper_sve_ldbss_zsu_mte,
gen_helper_sve_ldhss_le_zsu_mte,
NULL, },
{ gen_helper_sve_ldbsu_zsu_mte,
gen_helper_sve_ldhsu_le_zsu_mte,
gen_helper_sve_ldss_le_zsu_mte, } },
{ { gen_helper_sve_ldbss_zss_mte,
gen_helper_sve_ldhss_le_zss_mte,
NULL, },
{ gen_helper_sve_ldbsu_zss_mte,
gen_helper_sve_ldhsu_le_zss_mte,
gen_helper_sve_ldss_le_zss_mte, } } },
/* First-fault */
{ { { gen_helper_sve_ldffbss_zsu_mte,
gen_helper_sve_ldffhss_le_zsu_mte,
NULL, },
{ gen_helper_sve_ldffbsu_zsu_mte,
gen_helper_sve_ldffhsu_le_zsu_mte,
gen_helper_sve_ldffss_le_zsu_mte, } },
{ { gen_helper_sve_ldffbss_zss_mte,
gen_helper_sve_ldffhss_le_zss_mte,
NULL, },
{ gen_helper_sve_ldffbsu_zss_mte,
gen_helper_sve_ldffhsu_le_zss_mte,
gen_helper_sve_ldffss_le_zss_mte, } } } },
{ /* Big-endian */
{ { { gen_helper_sve_ldbss_zsu_mte,
gen_helper_sve_ldhss_be_zsu_mte,
NULL, },
{ gen_helper_sve_ldbsu_zsu_mte,
gen_helper_sve_ldhsu_be_zsu_mte,
gen_helper_sve_ldss_be_zsu_mte, } },
{ { gen_helper_sve_ldbss_zss_mte,
gen_helper_sve_ldhss_be_zss_mte,
NULL, },
{ gen_helper_sve_ldbsu_zss_mte,
gen_helper_sve_ldhsu_be_zss_mte,
gen_helper_sve_ldss_be_zss_mte, } } },
/* First-fault */
{ { { gen_helper_sve_ldffbss_zsu_mte,
gen_helper_sve_ldffhss_be_zsu_mte,
NULL, },
{ gen_helper_sve_ldffbsu_zsu_mte,
gen_helper_sve_ldffhsu_be_zsu_mte,
gen_helper_sve_ldffss_be_zsu_mte, } },
{ { gen_helper_sve_ldffbss_zss_mte,
gen_helper_sve_ldffhss_be_zss_mte,
NULL, },
{ gen_helper_sve_ldffbsu_zss_mte,
gen_helper_sve_ldffhsu_be_zss_mte,
gen_helper_sve_ldffss_be_zss_mte, } } } } },
};
/* Note that we overload xs=2 to indicate 64-bit offset. */
static gen_helper_gvec_mem_scatter * const gather_load_fn64[2][2][3][2][4] = {
/* Little-endian */
{ { { { gen_helper_sve_ldbds_zsu,
gen_helper_sve_ldhds_le_zsu,
gen_helper_sve_ldsds_le_zsu,
NULL, },
{ gen_helper_sve_ldbdu_zsu,
gen_helper_sve_ldhdu_le_zsu,
gen_helper_sve_ldsdu_le_zsu,
gen_helper_sve_lddd_le_zsu, } },
{ { gen_helper_sve_ldbds_zss,
gen_helper_sve_ldhds_le_zss,
gen_helper_sve_ldsds_le_zss,
NULL, },
{ gen_helper_sve_ldbdu_zss,
gen_helper_sve_ldhdu_le_zss,
gen_helper_sve_ldsdu_le_zss,
gen_helper_sve_lddd_le_zss, } },
{ { gen_helper_sve_ldbds_zd,
gen_helper_sve_ldhds_le_zd,
gen_helper_sve_ldsds_le_zd,
NULL, },
{ gen_helper_sve_ldbdu_zd,
gen_helper_sve_ldhdu_le_zd,
gen_helper_sve_ldsdu_le_zd,
gen_helper_sve_lddd_le_zd, } } },
static gen_helper_gvec_mem_scatter * const
gather_load_fn64[2][2][2][3][2][4] = {
{ /* MTE Inactive */
{ /* Little-endian */
{ { { gen_helper_sve_ldbds_zsu,
gen_helper_sve_ldhds_le_zsu,
gen_helper_sve_ldsds_le_zsu,
NULL, },
{ gen_helper_sve_ldbdu_zsu,
gen_helper_sve_ldhdu_le_zsu,
gen_helper_sve_ldsdu_le_zsu,
gen_helper_sve_lddd_le_zsu, } },
{ { gen_helper_sve_ldbds_zss,
gen_helper_sve_ldhds_le_zss,
gen_helper_sve_ldsds_le_zss,
NULL, },
{ gen_helper_sve_ldbdu_zss,
gen_helper_sve_ldhdu_le_zss,
gen_helper_sve_ldsdu_le_zss,
gen_helper_sve_lddd_le_zss, } },
{ { gen_helper_sve_ldbds_zd,
gen_helper_sve_ldhds_le_zd,
gen_helper_sve_ldsds_le_zd,
NULL, },
{ gen_helper_sve_ldbdu_zd,
gen_helper_sve_ldhdu_le_zd,
gen_helper_sve_ldsdu_le_zd,
gen_helper_sve_lddd_le_zd, } } },
/* First-fault */
{ { { gen_helper_sve_ldffbds_zsu,
gen_helper_sve_ldffhds_le_zsu,
gen_helper_sve_ldffsds_le_zsu,
NULL, },
{ gen_helper_sve_ldffbdu_zsu,
gen_helper_sve_ldffhdu_le_zsu,
gen_helper_sve_ldffsdu_le_zsu,
gen_helper_sve_ldffdd_le_zsu, } },
{ { gen_helper_sve_ldffbds_zss,
gen_helper_sve_ldffhds_le_zss,
gen_helper_sve_ldffsds_le_zss,
NULL, },
{ gen_helper_sve_ldffbdu_zss,
gen_helper_sve_ldffhdu_le_zss,
gen_helper_sve_ldffsdu_le_zss,
gen_helper_sve_ldffdd_le_zss, } },
{ { gen_helper_sve_ldffbds_zd,
gen_helper_sve_ldffhds_le_zd,
gen_helper_sve_ldffsds_le_zd,
NULL, },
{ gen_helper_sve_ldffbdu_zd,
gen_helper_sve_ldffhdu_le_zd,
gen_helper_sve_ldffsdu_le_zd,
gen_helper_sve_ldffdd_le_zd, } } } },
/* First-fault */
{ { { gen_helper_sve_ldffbds_zsu,
gen_helper_sve_ldffhds_le_zsu,
gen_helper_sve_ldffsds_le_zsu,
NULL, },
{ gen_helper_sve_ldffbdu_zsu,
gen_helper_sve_ldffhdu_le_zsu,
gen_helper_sve_ldffsdu_le_zsu,
gen_helper_sve_ldffdd_le_zsu, } },
{ { gen_helper_sve_ldffbds_zss,
gen_helper_sve_ldffhds_le_zss,
gen_helper_sve_ldffsds_le_zss,
NULL, },
{ gen_helper_sve_ldffbdu_zss,
gen_helper_sve_ldffhdu_le_zss,
gen_helper_sve_ldffsdu_le_zss,
gen_helper_sve_ldffdd_le_zss, } },
{ { gen_helper_sve_ldffbds_zd,
gen_helper_sve_ldffhds_le_zd,
gen_helper_sve_ldffsds_le_zd,
NULL, },
{ gen_helper_sve_ldffbdu_zd,
gen_helper_sve_ldffhdu_le_zd,
gen_helper_sve_ldffsdu_le_zd,
gen_helper_sve_ldffdd_le_zd, } } } },
{ /* Big-endian */
{ { { gen_helper_sve_ldbds_zsu,
gen_helper_sve_ldhds_be_zsu,
gen_helper_sve_ldsds_be_zsu,
NULL, },
{ gen_helper_sve_ldbdu_zsu,
gen_helper_sve_ldhdu_be_zsu,
gen_helper_sve_ldsdu_be_zsu,
gen_helper_sve_lddd_be_zsu, } },
{ { gen_helper_sve_ldbds_zss,
gen_helper_sve_ldhds_be_zss,
gen_helper_sve_ldsds_be_zss,
NULL, },
{ gen_helper_sve_ldbdu_zss,
gen_helper_sve_ldhdu_be_zss,
gen_helper_sve_ldsdu_be_zss,
gen_helper_sve_lddd_be_zss, } },
{ { gen_helper_sve_ldbds_zd,
gen_helper_sve_ldhds_be_zd,
gen_helper_sve_ldsds_be_zd,
NULL, },
{ gen_helper_sve_ldbdu_zd,
gen_helper_sve_ldhdu_be_zd,
gen_helper_sve_ldsdu_be_zd,
gen_helper_sve_lddd_be_zd, } } },
/* Big-endian */
{ { { { gen_helper_sve_ldbds_zsu,
gen_helper_sve_ldhds_be_zsu,
gen_helper_sve_ldsds_be_zsu,
NULL, },
{ gen_helper_sve_ldbdu_zsu,
gen_helper_sve_ldhdu_be_zsu,
gen_helper_sve_ldsdu_be_zsu,
gen_helper_sve_lddd_be_zsu, } },
{ { gen_helper_sve_ldbds_zss,
gen_helper_sve_ldhds_be_zss,
gen_helper_sve_ldsds_be_zss,
NULL, },
{ gen_helper_sve_ldbdu_zss,
gen_helper_sve_ldhdu_be_zss,
gen_helper_sve_ldsdu_be_zss,
gen_helper_sve_lddd_be_zss, } },
{ { gen_helper_sve_ldbds_zd,
gen_helper_sve_ldhds_be_zd,
gen_helper_sve_ldsds_be_zd,
NULL, },
{ gen_helper_sve_ldbdu_zd,
gen_helper_sve_ldhdu_be_zd,
gen_helper_sve_ldsdu_be_zd,
gen_helper_sve_lddd_be_zd, } } },
/* First-fault */
{ { { gen_helper_sve_ldffbds_zsu,
gen_helper_sve_ldffhds_be_zsu,
gen_helper_sve_ldffsds_be_zsu,
NULL, },
{ gen_helper_sve_ldffbdu_zsu,
gen_helper_sve_ldffhdu_be_zsu,
gen_helper_sve_ldffsdu_be_zsu,
gen_helper_sve_ldffdd_be_zsu, } },
{ { gen_helper_sve_ldffbds_zss,
gen_helper_sve_ldffhds_be_zss,
gen_helper_sve_ldffsds_be_zss,
NULL, },
{ gen_helper_sve_ldffbdu_zss,
gen_helper_sve_ldffhdu_be_zss,
gen_helper_sve_ldffsdu_be_zss,
gen_helper_sve_ldffdd_be_zss, } },
{ { gen_helper_sve_ldffbds_zd,
gen_helper_sve_ldffhds_be_zd,
gen_helper_sve_ldffsds_be_zd,
NULL, },
{ gen_helper_sve_ldffbdu_zd,
gen_helper_sve_ldffhdu_be_zd,
gen_helper_sve_ldffsdu_be_zd,
gen_helper_sve_ldffdd_be_zd, } } } } },
{ /* MTE Active */
{ /* Little-endian */
{ { { gen_helper_sve_ldbds_zsu_mte,
gen_helper_sve_ldhds_le_zsu_mte,
gen_helper_sve_ldsds_le_zsu_mte,
NULL, },
{ gen_helper_sve_ldbdu_zsu_mte,
gen_helper_sve_ldhdu_le_zsu_mte,
gen_helper_sve_ldsdu_le_zsu_mte,
gen_helper_sve_lddd_le_zsu_mte, } },
{ { gen_helper_sve_ldbds_zss_mte,
gen_helper_sve_ldhds_le_zss_mte,
gen_helper_sve_ldsds_le_zss_mte,
NULL, },
{ gen_helper_sve_ldbdu_zss_mte,
gen_helper_sve_ldhdu_le_zss_mte,
gen_helper_sve_ldsdu_le_zss_mte,
gen_helper_sve_lddd_le_zss_mte, } },
{ { gen_helper_sve_ldbds_zd_mte,
gen_helper_sve_ldhds_le_zd_mte,
gen_helper_sve_ldsds_le_zd_mte,
NULL, },
{ gen_helper_sve_ldbdu_zd_mte,
gen_helper_sve_ldhdu_le_zd_mte,
gen_helper_sve_ldsdu_le_zd_mte,
gen_helper_sve_lddd_le_zd_mte, } } },
/* First-fault */
{ { { gen_helper_sve_ldffbds_zsu,
gen_helper_sve_ldffhds_be_zsu,
gen_helper_sve_ldffsds_be_zsu,
NULL, },
{ gen_helper_sve_ldffbdu_zsu,
gen_helper_sve_ldffhdu_be_zsu,
gen_helper_sve_ldffsdu_be_zsu,
gen_helper_sve_ldffdd_be_zsu, } },
{ { gen_helper_sve_ldffbds_zss,
gen_helper_sve_ldffhds_be_zss,
gen_helper_sve_ldffsds_be_zss,
NULL, },
{ gen_helper_sve_ldffbdu_zss,
gen_helper_sve_ldffhdu_be_zss,
gen_helper_sve_ldffsdu_be_zss,
gen_helper_sve_ldffdd_be_zss, } },
{ { gen_helper_sve_ldffbds_zd,
gen_helper_sve_ldffhds_be_zd,
gen_helper_sve_ldffsds_be_zd,
NULL, },
{ gen_helper_sve_ldffbdu_zd,
gen_helper_sve_ldffhdu_be_zd,
gen_helper_sve_ldffsdu_be_zd,
gen_helper_sve_ldffdd_be_zd, } } } },
/* First-fault */
{ { { gen_helper_sve_ldffbds_zsu_mte,
gen_helper_sve_ldffhds_le_zsu_mte,
gen_helper_sve_ldffsds_le_zsu_mte,
NULL, },
{ gen_helper_sve_ldffbdu_zsu_mte,
gen_helper_sve_ldffhdu_le_zsu_mte,
gen_helper_sve_ldffsdu_le_zsu_mte,
gen_helper_sve_ldffdd_le_zsu_mte, } },
{ { gen_helper_sve_ldffbds_zss_mte,
gen_helper_sve_ldffhds_le_zss_mte,
gen_helper_sve_ldffsds_le_zss_mte,
NULL, },
{ gen_helper_sve_ldffbdu_zss_mte,
gen_helper_sve_ldffhdu_le_zss_mte,
gen_helper_sve_ldffsdu_le_zss_mte,
gen_helper_sve_ldffdd_le_zss_mte, } },
{ { gen_helper_sve_ldffbds_zd_mte,
gen_helper_sve_ldffhds_le_zd_mte,
gen_helper_sve_ldffsds_le_zd_mte,
NULL, },
{ gen_helper_sve_ldffbdu_zd_mte,
gen_helper_sve_ldffhdu_le_zd_mte,
gen_helper_sve_ldffsdu_le_zd_mte,
gen_helper_sve_ldffdd_le_zd_mte, } } } },
{ /* Big-endian */
{ { { gen_helper_sve_ldbds_zsu_mte,
gen_helper_sve_ldhds_be_zsu_mte,
gen_helper_sve_ldsds_be_zsu_mte,
NULL, },
{ gen_helper_sve_ldbdu_zsu_mte,
gen_helper_sve_ldhdu_be_zsu_mte,
gen_helper_sve_ldsdu_be_zsu_mte,
gen_helper_sve_lddd_be_zsu_mte, } },
{ { gen_helper_sve_ldbds_zss_mte,
gen_helper_sve_ldhds_be_zss_mte,
gen_helper_sve_ldsds_be_zss_mte,
NULL, },
{ gen_helper_sve_ldbdu_zss_mte,
gen_helper_sve_ldhdu_be_zss_mte,
gen_helper_sve_ldsdu_be_zss_mte,
gen_helper_sve_lddd_be_zss_mte, } },
{ { gen_helper_sve_ldbds_zd_mte,
gen_helper_sve_ldhds_be_zd_mte,
gen_helper_sve_ldsds_be_zd_mte,
NULL, },
{ gen_helper_sve_ldbdu_zd_mte,
gen_helper_sve_ldhdu_be_zd_mte,
gen_helper_sve_ldsdu_be_zd_mte,
gen_helper_sve_lddd_be_zd_mte, } } },
/* First-fault */
{ { { gen_helper_sve_ldffbds_zsu_mte,
gen_helper_sve_ldffhds_be_zsu_mte,
gen_helper_sve_ldffsds_be_zsu_mte,
NULL, },
{ gen_helper_sve_ldffbdu_zsu_mte,
gen_helper_sve_ldffhdu_be_zsu_mte,
gen_helper_sve_ldffsdu_be_zsu_mte,
gen_helper_sve_ldffdd_be_zsu_mte, } },
{ { gen_helper_sve_ldffbds_zss_mte,
gen_helper_sve_ldffhds_be_zss_mte,
gen_helper_sve_ldffsds_be_zss_mte,
NULL, },
{ gen_helper_sve_ldffbdu_zss_mte,
gen_helper_sve_ldffhdu_be_zss_mte,
gen_helper_sve_ldffsdu_be_zss_mte,
gen_helper_sve_ldffdd_be_zss_mte, } },
{ { gen_helper_sve_ldffbds_zd_mte,
gen_helper_sve_ldffhds_be_zd_mte,
gen_helper_sve_ldffsds_be_zd_mte,
NULL, },
{ gen_helper_sve_ldffbdu_zd_mte,
gen_helper_sve_ldffhdu_be_zd_mte,
gen_helper_sve_ldffsdu_be_zd_mte,
gen_helper_sve_ldffdd_be_zd_mte, } } } } },
};
static bool trans_LD1_zprz(DisasContext *s, arg_LD1_zprz *a)
{
gen_helper_gvec_mem_scatter *fn = NULL;
int be = s->be_data == MO_BE;
bool be = s->be_data == MO_BE;
bool mte = s->mte_active[0];
if (!sve_access_check(s)) {
return true;
@ -5621,23 +5792,24 @@ static bool trans_LD1_zprz(DisasContext *s, arg_LD1_zprz *a)
switch (a->esz) {
case MO_32:
fn = gather_load_fn32[be][a->ff][a->xs][a->u][a->msz];
fn = gather_load_fn32[mte][be][a->ff][a->xs][a->u][a->msz];
break;
case MO_64:
fn = gather_load_fn64[be][a->ff][a->xs][a->u][a->msz];
fn = gather_load_fn64[mte][be][a->ff][a->xs][a->u][a->msz];
break;
}
assert(fn != NULL);
do_mem_zpz(s, a->rd, a->pg, a->rm, a->scale * a->msz,
cpu_reg_sp(s, a->rn), a->msz, fn);
cpu_reg_sp(s, a->rn), a->msz, false, fn);
return true;
}
static bool trans_LD1_zpiz(DisasContext *s, arg_LD1_zpiz *a)
{
gen_helper_gvec_mem_scatter *fn = NULL;
int be = s->be_data == MO_BE;
bool be = s->be_data == MO_BE;
bool mte = s->mte_active[0];
TCGv_i64 imm;
if (a->esz < a->msz || (a->esz == a->msz && !a->u)) {
@ -5651,10 +5823,10 @@ static bool trans_LD1_zpiz(DisasContext *s, arg_LD1_zpiz *a)
switch (a->esz) {
case MO_32:
fn = gather_load_fn32[be][a->ff][0][a->u][a->msz];
fn = gather_load_fn32[mte][be][a->ff][0][a->u][a->msz];
break;
case MO_64:
fn = gather_load_fn64[be][a->ff][2][a->u][a->msz];
fn = gather_load_fn64[mte][be][a->ff][2][a->u][a->msz];
break;
}
assert(fn != NULL);
@ -5663,64 +5835,108 @@ static bool trans_LD1_zpiz(DisasContext *s, arg_LD1_zpiz *a)
* by loading the immediate into the scalar parameter.
*/
imm = tcg_const_i64(tcg_ctx, a->imm << a->msz);
do_mem_zpz(s, a->rd, a->pg, a->rn, 0, imm, a->msz, fn);
do_mem_zpz(s, a->rd, a->pg, a->rn, 0, imm, a->msz, false, fn);
tcg_temp_free_i64(tcg_ctx, imm);
return true;
}
/* Indexed by [xs][msz]. */
/* Indexed by [be][xs][msz]. */
static gen_helper_gvec_mem_scatter * const scatter_store_fn32[2][2][3] = {
/* Little-endian */
{ { gen_helper_sve_stbs_zsu,
gen_helper_sve_sths_le_zsu,
gen_helper_sve_stss_le_zsu, },
{ gen_helper_sve_stbs_zss,
gen_helper_sve_sths_le_zss,
gen_helper_sve_stss_le_zss, } },
/* Big-endian */
{ { gen_helper_sve_stbs_zsu,
gen_helper_sve_sths_be_zsu,
gen_helper_sve_stss_be_zsu, },
{ gen_helper_sve_stbs_zss,
gen_helper_sve_sths_be_zss,
gen_helper_sve_stss_be_zss, } },
/* Indexed by [mte][be][xs][msz]. */
static gen_helper_gvec_mem_scatter * const scatter_store_fn32[2][2][2][3] = {
{ /* MTE Inactive */
{ /* Little-endian */
{ gen_helper_sve_stbs_zsu,
gen_helper_sve_sths_le_zsu,
gen_helper_sve_stss_le_zsu, },
{ gen_helper_sve_stbs_zss,
gen_helper_sve_sths_le_zss,
gen_helper_sve_stss_le_zss, } },
{ /* Big-endian */
{ gen_helper_sve_stbs_zsu,
gen_helper_sve_sths_be_zsu,
gen_helper_sve_stss_be_zsu, },
{ gen_helper_sve_stbs_zss,
gen_helper_sve_sths_be_zss,
gen_helper_sve_stss_be_zss, } } },
{ /* MTE Active */
{ /* Little-endian */
{ gen_helper_sve_stbs_zsu_mte,
gen_helper_sve_sths_le_zsu_mte,
gen_helper_sve_stss_le_zsu_mte, },
{ gen_helper_sve_stbs_zss_mte,
gen_helper_sve_sths_le_zss_mte,
gen_helper_sve_stss_le_zss_mte, } },
{ /* Big-endian */
{ gen_helper_sve_stbs_zsu_mte,
gen_helper_sve_sths_be_zsu_mte,
gen_helper_sve_stss_be_zsu_mte, },
{ gen_helper_sve_stbs_zss_mte,
gen_helper_sve_sths_be_zss_mte,
gen_helper_sve_stss_be_zss_mte, } } },
};
/* Note that we overload xs=2 to indicate 64-bit offset. */
static gen_helper_gvec_mem_scatter * const scatter_store_fn64[2][3][4] = {
/* Little-endian */
{ { gen_helper_sve_stbd_zsu,
gen_helper_sve_sthd_le_zsu,
gen_helper_sve_stsd_le_zsu,
gen_helper_sve_stdd_le_zsu, },
{ gen_helper_sve_stbd_zss,
gen_helper_sve_sthd_le_zss,
gen_helper_sve_stsd_le_zss,
gen_helper_sve_stdd_le_zss, },
{ gen_helper_sve_stbd_zd,
gen_helper_sve_sthd_le_zd,
gen_helper_sve_stsd_le_zd,
gen_helper_sve_stdd_le_zd, } },
/* Big-endian */
{ { gen_helper_sve_stbd_zsu,
gen_helper_sve_sthd_be_zsu,
gen_helper_sve_stsd_be_zsu,
gen_helper_sve_stdd_be_zsu, },
{ gen_helper_sve_stbd_zss,
gen_helper_sve_sthd_be_zss,
gen_helper_sve_stsd_be_zss,
gen_helper_sve_stdd_be_zss, },
{ gen_helper_sve_stbd_zd,
gen_helper_sve_sthd_be_zd,
gen_helper_sve_stsd_be_zd,
gen_helper_sve_stdd_be_zd, } },
static gen_helper_gvec_mem_scatter * const scatter_store_fn64[2][2][3][4] = {
{ /* MTE Inactive */
{ /* Little-endian */
{ gen_helper_sve_stbd_zsu,
gen_helper_sve_sthd_le_zsu,
gen_helper_sve_stsd_le_zsu,
gen_helper_sve_stdd_le_zsu, },
{ gen_helper_sve_stbd_zss,
gen_helper_sve_sthd_le_zss,
gen_helper_sve_stsd_le_zss,
gen_helper_sve_stdd_le_zss, },
{ gen_helper_sve_stbd_zd,
gen_helper_sve_sthd_le_zd,
gen_helper_sve_stsd_le_zd,
gen_helper_sve_stdd_le_zd, } },
{ /* Big-endian */
{ gen_helper_sve_stbd_zsu,
gen_helper_sve_sthd_be_zsu,
gen_helper_sve_stsd_be_zsu,
gen_helper_sve_stdd_be_zsu, },
{ gen_helper_sve_stbd_zss,
gen_helper_sve_sthd_be_zss,
gen_helper_sve_stsd_be_zss,
gen_helper_sve_stdd_be_zss, },
{ gen_helper_sve_stbd_zd,
gen_helper_sve_sthd_be_zd,
gen_helper_sve_stsd_be_zd,
gen_helper_sve_stdd_be_zd, } } },
{ /* MTE Inactive */
{ /* Little-endian */
{ gen_helper_sve_stbd_zsu_mte,
gen_helper_sve_sthd_le_zsu_mte,
gen_helper_sve_stsd_le_zsu_mte,
gen_helper_sve_stdd_le_zsu_mte, },
{ gen_helper_sve_stbd_zss_mte,
gen_helper_sve_sthd_le_zss_mte,
gen_helper_sve_stsd_le_zss_mte,
gen_helper_sve_stdd_le_zss_mte, },
{ gen_helper_sve_stbd_zd_mte,
gen_helper_sve_sthd_le_zd_mte,
gen_helper_sve_stsd_le_zd_mte,
gen_helper_sve_stdd_le_zd_mte, } },
{ /* Big-endian */
{ gen_helper_sve_stbd_zsu_mte,
gen_helper_sve_sthd_be_zsu_mte,
gen_helper_sve_stsd_be_zsu_mte,
gen_helper_sve_stdd_be_zsu_mte, },
{ gen_helper_sve_stbd_zss_mte,
gen_helper_sve_sthd_be_zss_mte,
gen_helper_sve_stsd_be_zss_mte,
gen_helper_sve_stdd_be_zss_mte, },
{ gen_helper_sve_stbd_zd_mte,
gen_helper_sve_sthd_be_zd_mte,
gen_helper_sve_stsd_be_zd_mte,
gen_helper_sve_stdd_be_zd_mte, } } },
};
static bool trans_ST1_zprz(DisasContext *s, arg_ST1_zprz *a)
{
gen_helper_gvec_mem_scatter *fn;
int be = s->be_data == MO_BE;
bool be = s->be_data == MO_BE;
bool mte = s->mte_active[0];
if (a->esz < a->msz || (a->msz == 0 && a->scale)) {
return false;
@ -5730,23 +5946,24 @@ static bool trans_ST1_zprz(DisasContext *s, arg_ST1_zprz *a)
}
switch (a->esz) {
case MO_32:
fn = scatter_store_fn32[be][a->xs][a->msz];
fn = scatter_store_fn32[mte][be][a->xs][a->msz];
break;
case MO_64:
fn = scatter_store_fn64[be][a->xs][a->msz];
fn = scatter_store_fn64[mte][be][a->xs][a->msz];
break;
default:
g_assert_not_reached();
}
do_mem_zpz(s, a->rd, a->pg, a->rm, a->scale * a->msz,
cpu_reg_sp(s, a->rn), a->msz, fn);
cpu_reg_sp(s, a->rn), a->msz, true, fn);
return true;
}
static bool trans_ST1_zpiz(DisasContext *s, arg_ST1_zpiz *a)
{
gen_helper_gvec_mem_scatter *fn = NULL;
int be = s->be_data == MO_BE;
bool be = s->be_data == MO_BE;
bool mte = s->mte_active[0];
TCGv_i64 imm;
if (a->esz < a->msz) {
@ -5760,10 +5977,10 @@ static bool trans_ST1_zpiz(DisasContext *s, arg_ST1_zpiz *a)
switch (a->esz) {
case MO_32:
fn = scatter_store_fn32[be][0][a->msz];
fn = scatter_store_fn32[mte][be][0][a->msz];
break;
case MO_64:
fn = scatter_store_fn64[be][2][a->msz];
fn = scatter_store_fn64[mte][be][2][a->msz];
break;
}
assert(fn != NULL);
@ -5772,7 +5989,7 @@ static bool trans_ST1_zpiz(DisasContext *s, arg_ST1_zpiz *a)
* by loading the immediate into the scalar parameter.
*/
imm = tcg_const_i64(tcg_ctx, a->imm << a->msz);
do_mem_zpz(s, a->rd, a->pg, a->rn, 0, imm, a->msz, fn);
do_mem_zpz(s, a->rd, a->pg, a->rn, 0, imm, a->msz, true, fn);
tcg_temp_free_i64(tcg_ctx, imm);
return true;
}