target/arm: Handle watchpoints in sve_ld1_r

Handle all of the watchpoints for active elements all at once,
before we've modified the vector register. This removes the
TLB_WATCHPOINT bit from page[].flags, which means that we can
use the normal fast path via RAM.

Backports commit 4bcc3f0ff8e5ae2b17b5aab9aa613ff1b8025896 from qemu
This commit is contained in:
Richard Henderson 2021-02-25 20:44:10 -05:00 committed by Lioncash
parent 0e5aa37c9a
commit 3979c8f73e

View file

@ -4334,6 +4334,70 @@ static bool sve_cont_ldst_pages(SVEContLdSt *info, SVEContFault fault,
return have_work;
}
static void sve_cont_ldst_watchpoints(SVEContLdSt *info, CPUARMState *env,
uint64_t *vg, target_ulong addr,
int esize, int msize, int wp_access,
uintptr_t retaddr)
{
#ifndef CONFIG_USER_ONLY
intptr_t mem_off, reg_off, reg_last;
int flags0 = info->page[0].flags;
int flags1 = info->page[1].flags;
if (likely(!((flags0 | flags1) & TLB_WATCHPOINT))) {
return;
}
/* Indicate that watchpoints are handled. */
info->page[0].flags = flags0 & ~TLB_WATCHPOINT;
info->page[1].flags = flags1 & ~TLB_WATCHPOINT;
if (flags0 & TLB_WATCHPOINT) {
mem_off = info->mem_off_first[0];
reg_off = info->reg_off_first[0];
reg_last = info->reg_off_last[0];
while (reg_off <= reg_last) {
uint64_t pg = vg[reg_off >> 6];
do {
if ((pg >> (reg_off & 63)) & 1) {
cpu_check_watchpoint(env_cpu(env), addr + mem_off,
msize, info->page[0].attrs,
wp_access, retaddr);
}
reg_off += esize;
mem_off += msize;
} while (reg_off <= reg_last && (reg_off & 63));
}
}
mem_off = info->mem_off_split;
if (mem_off >= 0) {
cpu_check_watchpoint(env_cpu(env), addr + mem_off, msize,
info->page[0].attrs, wp_access, retaddr);
}
mem_off = info->mem_off_first[1];
if ((flags1 & TLB_WATCHPOINT) && mem_off >= 0) {
reg_off = info->reg_off_first[1];
reg_last = info->reg_off_last[1];
do {
uint64_t pg = vg[reg_off >> 6];
do {
if ((pg >> (reg_off & 63)) & 1) {
cpu_check_watchpoint(env_cpu(env), addr + mem_off,
msize, info->page[1].attrs,
wp_access, retaddr);
}
reg_off += esize;
mem_off += msize;
} while (reg_off & 63);
} while (reg_off <= reg_last);
}
#endif
}
/*
* The result of tlb_vaddr_to_host for user-only is just g2h(x),
* which is always non-null. Elide the useless test.
@ -4375,13 +4439,19 @@ void sve_ld1_r(CPUARMState *env, uint64_t *vg, const target_ulong addr,
/* Probe the page(s). Exit with exception for any invalid page. */
sve_cont_ldst_pages(&info, FAULT_ALL, env, addr, MMU_DATA_LOAD, retaddr);
/* Handle watchpoints for all active elements. */
sve_cont_ldst_watchpoints(&info, env, vg, addr, 1 << esz, 1 << msz,
BP_MEM_READ, retaddr);
/* TODO: MTE check. */
flags = info.page[0].flags | info.page[1].flags;
if (unlikely(flags != 0)) {
#ifdef CONFIG_USER_ONLY
g_assert_not_reached();
#else
/*
* At least one page includes MMIO (or watchpoints).
* At least one page includes MMIO.
* Any bus operation can fail with cpu_transaction_failed,
* which for ARM will raise SyncExternal. Perform the load
* into scratch memory to preserve register state until the end.