target/arm: Fix sve_uzp_p vs odd vector lengths

Missed out on compressing the second half of a predicate
with length vl % 512 > 256.

Adjust all of the x + (y << s) to x | (y << s) as a
general style fix. Drop the extract64 because the input
uint64_t are known to be already zero-extended from the
current size of the predicate.

Backports 226e6c046c0fce8da32575aad020ca56a5a8064d
This commit is contained in:
Richard Henderson 2021-03-30 14:27:47 -04:00 committed by Lioncash
parent 9777741703
commit 1aed8cee64

View file

@ -1938,7 +1938,7 @@ void HELPER(sve_uzp_p)(void *vd, void *vn, void *vm, uint32_t pred_desc)
if (oprsz <= 8) { if (oprsz <= 8) {
l = compress_bits(n[0] >> odd, esz); l = compress_bits(n[0] >> odd, esz);
h = compress_bits(m[0] >> odd, esz); h = compress_bits(m[0] >> odd, esz);
d[0] = extract64(l + (h << (4 * oprsz)), 0, 8 * oprsz); d[0] = l | (h << (4 * oprsz));
} else { } else {
ARMPredicateReg tmp_m; ARMPredicateReg tmp_m;
intptr_t oprsz_16 = oprsz / 16; intptr_t oprsz_16 = oprsz / 16;
@ -1952,23 +1952,35 @@ void HELPER(sve_uzp_p)(void *vd, void *vn, void *vm, uint32_t pred_desc)
h = n[2 * i + 1]; h = n[2 * i + 1];
l = compress_bits(l >> odd, esz); l = compress_bits(l >> odd, esz);
h = compress_bits(h >> odd, esz); h = compress_bits(h >> odd, esz);
d[i] = l + (h << 32); d[i] = l | (h << 32);
} }
/* For VL which is not a power of 2, the results from M do not /*
align nicely with the uint64_t for D. Put the aligned results * For VL which is not a multiple of 512, the results from M do not
from M into TMP_M and then copy it into place afterward. */ * align nicely with the uint64_t for D. Put the aligned results
* from M into TMP_M and then copy it into place afterward.
*/
if (oprsz & 15) { if (oprsz & 15) {
d[i] = compress_bits(n[2 * i] >> odd, esz); int final_shift = (oprsz & 15) * 2;
l = n[2 * i + 0];
h = n[2 * i + 1];
l = compress_bits(l >> odd, esz);
h = compress_bits(h >> odd, esz);
d[i] = l | (h << final_shift);
for (i = 0; i < oprsz_16; i++) { for (i = 0; i < oprsz_16; i++) {
l = m[2 * i + 0]; l = m[2 * i + 0];
h = m[2 * i + 1]; h = m[2 * i + 1];
l = compress_bits(l >> odd, esz); l = compress_bits(l >> odd, esz);
h = compress_bits(h >> odd, esz); h = compress_bits(h >> odd, esz);
tmp_m.p[i] = l + (h << 32); tmp_m.p[i] = l | (h << 32);
} }
tmp_m.p[i] = compress_bits(m[2 * i] >> odd, esz); l = m[2 * i + 0];
h = m[2 * i + 1];
l = compress_bits(l >> odd, esz);
h = compress_bits(h >> odd, esz);
tmp_m.p[i] = l | (h << final_shift);
swap_memmove(vd + oprsz / 2, &tmp_m, oprsz / 2); swap_memmove(vd + oprsz / 2, &tmp_m, oprsz / 2);
} else { } else {
@ -1977,7 +1989,7 @@ void HELPER(sve_uzp_p)(void *vd, void *vn, void *vm, uint32_t pred_desc)
h = m[2 * i + 1]; h = m[2 * i + 1];
l = compress_bits(l >> odd, esz); l = compress_bits(l >> odd, esz);
h = compress_bits(h >> odd, esz); h = compress_bits(h >> odd, esz);
d[oprsz_16 + i] = l + (h << 32); d[oprsz_16 + i] = l | (h << 32);
} }
} }
} }