2 * AArch64 SVE translation
4 * Copyright (c) 2018 Linaro, Ltd
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
22 #include "exec/exec-all.h"
23 #include "tcg/tcg-op.h"
24 #include "tcg/tcg-op-gvec.h"
25 #include "tcg/tcg-gvec-desc.h"
28 #include "translate.h"
29 #include "internals.h"
30 #include "exec/helper-proto.h"
31 #include "exec/helper-gen.h"
33 #include "translate-a64.h"
34 #include "fpu/softfloat.h"
37 typedef void GVecGen2sFn(unsigned, uint32_t, uint32_t,
38 TCGv_i64
, uint32_t, uint32_t);
40 typedef void gen_helper_gvec_flags_3(TCGv_i32
, TCGv_ptr
, TCGv_ptr
,
42 typedef void gen_helper_gvec_flags_4(TCGv_i32
, TCGv_ptr
, TCGv_ptr
,
43 TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
45 typedef void gen_helper_gvec_mem(TCGv_env
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
46 typedef void gen_helper_gvec_mem_scatter(TCGv_env
, TCGv_ptr
, TCGv_ptr
,
47 TCGv_ptr
, TCGv_i64
, TCGv_i32
);
50 * Helpers for extracting complex instruction fields.
53 /* See e.g. ASR (immediate, predicated).
54 * Returns -1 for unallocated encoding; diagnose later.
56 static int tszimm_esz(DisasContext
*s
, int x
)
58 x
>>= 3; /* discard imm3 */
62 static int tszimm_shr(DisasContext
*s
, int x
)
64 return (16 << tszimm_esz(s
, x
)) - x
;
67 /* See e.g. LSL (immediate, predicated). */
68 static int tszimm_shl(DisasContext
*s
, int x
)
70 return x
- (8 << tszimm_esz(s
, x
));
73 /* The SH bit is in bit 8. Extract the low 8 and shift. */
74 static inline int expand_imm_sh8s(DisasContext
*s
, int x
)
76 return (int8_t)x
<< (x
& 0x100 ? 8 : 0);
79 static inline int expand_imm_sh8u(DisasContext
*s
, int x
)
81 return (uint8_t)x
<< (x
& 0x100 ? 8 : 0);
84 /* Convert a 2-bit memory size (msz) to a 4-bit data type (dtype)
85 * with unsigned data. C.f. SVE Memory Contiguous Load Group.
87 static inline int msz_dtype(DisasContext
*s
, int msz
)
89 static const uint8_t dtype
[4] = { 0, 5, 10, 15 };
94 * Include the generated decoder.
97 #include "decode-sve.c.inc"
100 * Implement all of the translator functions referenced by the decoder.
103 /* Return the offset info CPUARMState of the predicate vector register Pn.
104 * Note for this purpose, FFR is P16.
106 static inline int pred_full_reg_offset(DisasContext
*s
, int regno
)
108 return offsetof(CPUARMState
, vfp
.pregs
[regno
]);
111 /* Return the byte size of the whole predicate register, VL / 64. */
112 static inline int pred_full_reg_size(DisasContext
*s
)
114 return s
->sve_len
>> 3;
117 /* Round up the size of a register to a size allowed by
118 * the tcg vector infrastructure. Any operation which uses this
119 * size may assume that the bits above pred_full_reg_size are zero,
120 * and must leave them the same way.
122 * Note that this is not needed for the vector registers as they
123 * are always properly sized for tcg vectors.
125 static int size_for_gvec(int size
)
130 return QEMU_ALIGN_UP(size
, 16);
134 static int pred_gvec_reg_size(DisasContext
*s
)
136 return size_for_gvec(pred_full_reg_size(s
));
139 /* Invoke an out-of-line helper on 2 Zregs. */
140 static bool gen_gvec_ool_zz(DisasContext
*s
, gen_helper_gvec_2
*fn
,
141 int rd
, int rn
, int data
)
146 if (sve_access_check(s
)) {
147 unsigned vsz
= vec_full_reg_size(s
);
148 tcg_gen_gvec_2_ool(vec_full_reg_offset(s
, rd
),
149 vec_full_reg_offset(s
, rn
),
155 /* Invoke an out-of-line helper on 3 Zregs. */
156 static bool gen_gvec_ool_zzz(DisasContext
*s
, gen_helper_gvec_3
*fn
,
157 int rd
, int rn
, int rm
, int data
)
162 if (sve_access_check(s
)) {
163 unsigned vsz
= vec_full_reg_size(s
);
164 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
165 vec_full_reg_offset(s
, rn
),
166 vec_full_reg_offset(s
, rm
),
172 static bool gen_gvec_ool_arg_zzz(DisasContext
*s
, gen_helper_gvec_3
*fn
,
173 arg_rrr_esz
*a
, int data
)
175 return gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, data
);
178 /* Invoke an out-of-line helper on 4 Zregs. */
179 static bool gen_gvec_ool_zzzz(DisasContext
*s
, gen_helper_gvec_4
*fn
,
180 int rd
, int rn
, int rm
, int ra
, int data
)
185 if (sve_access_check(s
)) {
186 unsigned vsz
= vec_full_reg_size(s
);
187 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
188 vec_full_reg_offset(s
, rn
),
189 vec_full_reg_offset(s
, rm
),
190 vec_full_reg_offset(s
, ra
),
196 static bool gen_gvec_ool_arg_zzzz(DisasContext
*s
, gen_helper_gvec_4
*fn
,
197 arg_rrrr_esz
*a
, int data
)
199 return gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, data
);
202 static bool gen_gvec_ool_arg_zzxz(DisasContext
*s
, gen_helper_gvec_4
*fn
,
205 return gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->index
);
208 /* Invoke an out-of-line helper on 2 Zregs and a predicate. */
209 static void gen_gvec_ool_zzp(DisasContext
*s
, gen_helper_gvec_3
*fn
,
210 int rd
, int rn
, int pg
, int data
)
212 unsigned vsz
= vec_full_reg_size(s
);
213 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
214 vec_full_reg_offset(s
, rn
),
215 pred_full_reg_offset(s
, pg
),
219 /* Invoke an out-of-line helper on 3 Zregs and a predicate. */
220 static void gen_gvec_ool_zzzp(DisasContext
*s
, gen_helper_gvec_4
*fn
,
221 int rd
, int rn
, int rm
, int pg
, int data
)
223 unsigned vsz
= vec_full_reg_size(s
);
224 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
225 vec_full_reg_offset(s
, rn
),
226 vec_full_reg_offset(s
, rm
),
227 pred_full_reg_offset(s
, pg
),
231 /* Invoke a vector expander on two Zregs. */
232 static void gen_gvec_fn_zz(DisasContext
*s
, GVecGen2Fn
*gvec_fn
,
233 int esz
, int rd
, int rn
)
235 unsigned vsz
= vec_full_reg_size(s
);
236 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
237 vec_full_reg_offset(s
, rn
), vsz
, vsz
);
240 /* Invoke a vector expander on three Zregs. */
241 static void gen_gvec_fn_zzz(DisasContext
*s
, GVecGen3Fn
*gvec_fn
,
242 int esz
, int rd
, int rn
, int rm
)
244 unsigned vsz
= vec_full_reg_size(s
);
245 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
246 vec_full_reg_offset(s
, rn
),
247 vec_full_reg_offset(s
, rm
), vsz
, vsz
);
250 /* Invoke a vector expander on four Zregs. */
251 static void gen_gvec_fn_zzzz(DisasContext
*s
, GVecGen4Fn
*gvec_fn
,
252 int esz
, int rd
, int rn
, int rm
, int ra
)
254 unsigned vsz
= vec_full_reg_size(s
);
255 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
256 vec_full_reg_offset(s
, rn
),
257 vec_full_reg_offset(s
, rm
),
258 vec_full_reg_offset(s
, ra
), vsz
, vsz
);
261 /* Invoke a vector move on two Zregs. */
262 static bool do_mov_z(DisasContext
*s
, int rd
, int rn
)
264 if (sve_access_check(s
)) {
265 gen_gvec_fn_zz(s
, tcg_gen_gvec_mov
, MO_8
, rd
, rn
);
270 /* Initialize a Zreg with replications of a 64-bit immediate. */
271 static void do_dupi_z(DisasContext
*s
, int rd
, uint64_t word
)
273 unsigned vsz
= vec_full_reg_size(s
);
274 tcg_gen_gvec_dup_imm(MO_64
, vec_full_reg_offset(s
, rd
), vsz
, vsz
, word
);
277 /* Invoke a vector expander on three Pregs. */
278 static void gen_gvec_fn_ppp(DisasContext
*s
, GVecGen3Fn
*gvec_fn
,
279 int rd
, int rn
, int rm
)
281 unsigned psz
= pred_gvec_reg_size(s
);
282 gvec_fn(MO_64
, pred_full_reg_offset(s
, rd
),
283 pred_full_reg_offset(s
, rn
),
284 pred_full_reg_offset(s
, rm
), psz
, psz
);
287 /* Invoke a vector move on two Pregs. */
288 static bool do_mov_p(DisasContext
*s
, int rd
, int rn
)
290 if (sve_access_check(s
)) {
291 unsigned psz
= pred_gvec_reg_size(s
);
292 tcg_gen_gvec_mov(MO_8
, pred_full_reg_offset(s
, rd
),
293 pred_full_reg_offset(s
, rn
), psz
, psz
);
298 /* Set the cpu flags as per a return from an SVE helper. */
299 static void do_pred_flags(TCGv_i32 t
)
301 tcg_gen_mov_i32(cpu_NF
, t
);
302 tcg_gen_andi_i32(cpu_ZF
, t
, 2);
303 tcg_gen_andi_i32(cpu_CF
, t
, 1);
304 tcg_gen_movi_i32(cpu_VF
, 0);
307 /* Subroutines computing the ARM PredTest psuedofunction. */
308 static void do_predtest1(TCGv_i64 d
, TCGv_i64 g
)
310 TCGv_i32 t
= tcg_temp_new_i32();
312 gen_helper_sve_predtest1(t
, d
, g
);
314 tcg_temp_free_i32(t
);
317 static void do_predtest(DisasContext
*s
, int dofs
, int gofs
, int words
)
319 TCGv_ptr dptr
= tcg_temp_new_ptr();
320 TCGv_ptr gptr
= tcg_temp_new_ptr();
321 TCGv_i32 t
= tcg_temp_new_i32();
323 tcg_gen_addi_ptr(dptr
, cpu_env
, dofs
);
324 tcg_gen_addi_ptr(gptr
, cpu_env
, gofs
);
326 gen_helper_sve_predtest(t
, dptr
, gptr
, tcg_constant_i32(words
));
327 tcg_temp_free_ptr(dptr
);
328 tcg_temp_free_ptr(gptr
);
331 tcg_temp_free_i32(t
);
334 /* For each element size, the bits within a predicate word that are active. */
335 const uint64_t pred_esz_masks
[4] = {
336 0xffffffffffffffffull
, 0x5555555555555555ull
,
337 0x1111111111111111ull
, 0x0101010101010101ull
341 *** SVE Logical - Unpredicated Group
344 static bool do_zzz_fn(DisasContext
*s
, arg_rrr_esz
*a
, GVecGen3Fn
*gvec_fn
)
346 if (sve_access_check(s
)) {
347 gen_gvec_fn_zzz(s
, gvec_fn
, a
->esz
, a
->rd
, a
->rn
, a
->rm
);
352 static bool trans_AND_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
354 return do_zzz_fn(s
, a
, tcg_gen_gvec_and
);
357 static bool trans_ORR_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
359 return do_zzz_fn(s
, a
, tcg_gen_gvec_or
);
362 static bool trans_EOR_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
364 return do_zzz_fn(s
, a
, tcg_gen_gvec_xor
);
367 static bool trans_BIC_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
369 return do_zzz_fn(s
, a
, tcg_gen_gvec_andc
);
372 static void gen_xar8_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
374 TCGv_i64 t
= tcg_temp_new_i64();
375 uint64_t mask
= dup_const(MO_8
, 0xff >> sh
);
377 tcg_gen_xor_i64(t
, n
, m
);
378 tcg_gen_shri_i64(d
, t
, sh
);
379 tcg_gen_shli_i64(t
, t
, 8 - sh
);
380 tcg_gen_andi_i64(d
, d
, mask
);
381 tcg_gen_andi_i64(t
, t
, ~mask
);
382 tcg_gen_or_i64(d
, d
, t
);
383 tcg_temp_free_i64(t
);
386 static void gen_xar16_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
388 TCGv_i64 t
= tcg_temp_new_i64();
389 uint64_t mask
= dup_const(MO_16
, 0xffff >> sh
);
391 tcg_gen_xor_i64(t
, n
, m
);
392 tcg_gen_shri_i64(d
, t
, sh
);
393 tcg_gen_shli_i64(t
, t
, 16 - sh
);
394 tcg_gen_andi_i64(d
, d
, mask
);
395 tcg_gen_andi_i64(t
, t
, ~mask
);
396 tcg_gen_or_i64(d
, d
, t
);
397 tcg_temp_free_i64(t
);
400 static void gen_xar_i32(TCGv_i32 d
, TCGv_i32 n
, TCGv_i32 m
, int32_t sh
)
402 tcg_gen_xor_i32(d
, n
, m
);
403 tcg_gen_rotri_i32(d
, d
, sh
);
406 static void gen_xar_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
408 tcg_gen_xor_i64(d
, n
, m
);
409 tcg_gen_rotri_i64(d
, d
, sh
);
412 static void gen_xar_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
413 TCGv_vec m
, int64_t sh
)
415 tcg_gen_xor_vec(vece
, d
, n
, m
);
416 tcg_gen_rotri_vec(vece
, d
, d
, sh
);
419 void gen_gvec_xar(unsigned vece
, uint32_t rd_ofs
, uint32_t rn_ofs
,
420 uint32_t rm_ofs
, int64_t shift
,
421 uint32_t opr_sz
, uint32_t max_sz
)
423 static const TCGOpcode vecop
[] = { INDEX_op_rotli_vec
, 0 };
424 static const GVecGen3i ops
[4] = {
425 { .fni8
= gen_xar8_i64
,
427 .fno
= gen_helper_sve2_xar_b
,
430 { .fni8
= gen_xar16_i64
,
432 .fno
= gen_helper_sve2_xar_h
,
435 { .fni4
= gen_xar_i32
,
437 .fno
= gen_helper_sve2_xar_s
,
440 { .fni8
= gen_xar_i64
,
442 .fno
= gen_helper_gvec_xar_d
,
446 int esize
= 8 << vece
;
448 /* The SVE2 range is 1 .. esize; the AdvSIMD range is 0 .. esize-1. */
449 tcg_debug_assert(shift
>= 0);
450 tcg_debug_assert(shift
<= esize
);
454 /* xar with no rotate devolves to xor. */
455 tcg_gen_gvec_xor(vece
, rd_ofs
, rn_ofs
, rm_ofs
, opr_sz
, max_sz
);
457 tcg_gen_gvec_3i(rd_ofs
, rn_ofs
, rm_ofs
, opr_sz
, max_sz
,
462 static bool trans_XAR(DisasContext
*s
, arg_rrri_esz
*a
)
464 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
467 if (sve_access_check(s
)) {
468 unsigned vsz
= vec_full_reg_size(s
);
469 gen_gvec_xar(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
470 vec_full_reg_offset(s
, a
->rn
),
471 vec_full_reg_offset(s
, a
->rm
), a
->imm
, vsz
, vsz
);
476 static bool do_sve2_zzzz_fn(DisasContext
*s
, arg_rrrr_esz
*a
, GVecGen4Fn
*fn
)
478 if (!dc_isar_feature(aa64_sve2
, s
)) {
481 if (sve_access_check(s
)) {
482 gen_gvec_fn_zzzz(s
, fn
, a
->esz
, a
->rd
, a
->rn
, a
->rm
, a
->ra
);
487 static void gen_eor3_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
489 tcg_gen_xor_i64(d
, n
, m
);
490 tcg_gen_xor_i64(d
, d
, k
);
493 static void gen_eor3_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
494 TCGv_vec m
, TCGv_vec k
)
496 tcg_gen_xor_vec(vece
, d
, n
, m
);
497 tcg_gen_xor_vec(vece
, d
, d
, k
);
500 static void gen_eor3(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
501 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
503 static const GVecGen4 op
= {
504 .fni8
= gen_eor3_i64
,
505 .fniv
= gen_eor3_vec
,
506 .fno
= gen_helper_sve2_eor3
,
508 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
510 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
513 static bool trans_EOR3(DisasContext
*s
, arg_rrrr_esz
*a
)
515 return do_sve2_zzzz_fn(s
, a
, gen_eor3
);
518 static void gen_bcax_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
520 tcg_gen_andc_i64(d
, m
, k
);
521 tcg_gen_xor_i64(d
, d
, n
);
524 static void gen_bcax_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
525 TCGv_vec m
, TCGv_vec k
)
527 tcg_gen_andc_vec(vece
, d
, m
, k
);
528 tcg_gen_xor_vec(vece
, d
, d
, n
);
531 static void gen_bcax(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
532 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
534 static const GVecGen4 op
= {
535 .fni8
= gen_bcax_i64
,
536 .fniv
= gen_bcax_vec
,
537 .fno
= gen_helper_sve2_bcax
,
539 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
541 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
544 static bool trans_BCAX(DisasContext
*s
, arg_rrrr_esz
*a
)
546 return do_sve2_zzzz_fn(s
, a
, gen_bcax
);
549 static void gen_bsl(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
550 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
552 /* BSL differs from the generic bitsel in argument ordering. */
553 tcg_gen_gvec_bitsel(vece
, d
, a
, n
, m
, oprsz
, maxsz
);
556 static bool trans_BSL(DisasContext
*s
, arg_rrrr_esz
*a
)
558 return do_sve2_zzzz_fn(s
, a
, gen_bsl
);
561 static void gen_bsl1n_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
563 tcg_gen_andc_i64(n
, k
, n
);
564 tcg_gen_andc_i64(m
, m
, k
);
565 tcg_gen_or_i64(d
, n
, m
);
568 static void gen_bsl1n_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
569 TCGv_vec m
, TCGv_vec k
)
571 if (TCG_TARGET_HAS_bitsel_vec
) {
572 tcg_gen_not_vec(vece
, n
, n
);
573 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
575 tcg_gen_andc_vec(vece
, n
, k
, n
);
576 tcg_gen_andc_vec(vece
, m
, m
, k
);
577 tcg_gen_or_vec(vece
, d
, n
, m
);
581 static void gen_bsl1n(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
582 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
584 static const GVecGen4 op
= {
585 .fni8
= gen_bsl1n_i64
,
586 .fniv
= gen_bsl1n_vec
,
587 .fno
= gen_helper_sve2_bsl1n
,
589 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
591 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
594 static bool trans_BSL1N(DisasContext
*s
, arg_rrrr_esz
*a
)
596 return do_sve2_zzzz_fn(s
, a
, gen_bsl1n
);
599 static void gen_bsl2n_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
602 * Z[dn] = (n & k) | (~m & ~k)
605 tcg_gen_and_i64(n
, n
, k
);
606 if (TCG_TARGET_HAS_orc_i64
) {
607 tcg_gen_or_i64(m
, m
, k
);
608 tcg_gen_orc_i64(d
, n
, m
);
610 tcg_gen_nor_i64(m
, m
, k
);
611 tcg_gen_or_i64(d
, n
, m
);
615 static void gen_bsl2n_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
616 TCGv_vec m
, TCGv_vec k
)
618 if (TCG_TARGET_HAS_bitsel_vec
) {
619 tcg_gen_not_vec(vece
, m
, m
);
620 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
622 tcg_gen_and_vec(vece
, n
, n
, k
);
623 tcg_gen_or_vec(vece
, m
, m
, k
);
624 tcg_gen_orc_vec(vece
, d
, n
, m
);
628 static void gen_bsl2n(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
629 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
631 static const GVecGen4 op
= {
632 .fni8
= gen_bsl2n_i64
,
633 .fniv
= gen_bsl2n_vec
,
634 .fno
= gen_helper_sve2_bsl2n
,
636 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
638 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
641 static bool trans_BSL2N(DisasContext
*s
, arg_rrrr_esz
*a
)
643 return do_sve2_zzzz_fn(s
, a
, gen_bsl2n
);
646 static void gen_nbsl_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
648 tcg_gen_and_i64(n
, n
, k
);
649 tcg_gen_andc_i64(m
, m
, k
);
650 tcg_gen_nor_i64(d
, n
, m
);
653 static void gen_nbsl_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
654 TCGv_vec m
, TCGv_vec k
)
656 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
657 tcg_gen_not_vec(vece
, d
, d
);
660 static void gen_nbsl(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
661 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
663 static const GVecGen4 op
= {
664 .fni8
= gen_nbsl_i64
,
665 .fniv
= gen_nbsl_vec
,
666 .fno
= gen_helper_sve2_nbsl
,
668 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
670 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
673 static bool trans_NBSL(DisasContext
*s
, arg_rrrr_esz
*a
)
675 return do_sve2_zzzz_fn(s
, a
, gen_nbsl
);
679 *** SVE Integer Arithmetic - Unpredicated Group
682 static bool trans_ADD_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
684 return do_zzz_fn(s
, a
, tcg_gen_gvec_add
);
687 static bool trans_SUB_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
689 return do_zzz_fn(s
, a
, tcg_gen_gvec_sub
);
692 static bool trans_SQADD_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
694 return do_zzz_fn(s
, a
, tcg_gen_gvec_ssadd
);
697 static bool trans_SQSUB_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
699 return do_zzz_fn(s
, a
, tcg_gen_gvec_sssub
);
702 static bool trans_UQADD_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
704 return do_zzz_fn(s
, a
, tcg_gen_gvec_usadd
);
707 static bool trans_UQSUB_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
709 return do_zzz_fn(s
, a
, tcg_gen_gvec_ussub
);
713 *** SVE Integer Arithmetic - Binary Predicated Group
716 static bool do_zpzz_ool(DisasContext
*s
, arg_rprr_esz
*a
, gen_helper_gvec_4
*fn
)
721 if (sve_access_check(s
)) {
722 gen_gvec_ool_zzzp(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->pg
, 0);
727 /* Select active elememnts from Zn and inactive elements from Zm,
728 * storing the result in Zd.
730 static void do_sel_z(DisasContext
*s
, int rd
, int rn
, int rm
, int pg
, int esz
)
732 static gen_helper_gvec_4
* const fns
[4] = {
733 gen_helper_sve_sel_zpzz_b
, gen_helper_sve_sel_zpzz_h
,
734 gen_helper_sve_sel_zpzz_s
, gen_helper_sve_sel_zpzz_d
736 gen_gvec_ool_zzzp(s
, fns
[esz
], rd
, rn
, rm
, pg
, 0);
739 #define DO_ZPZZ(NAME, name) \
740 static bool trans_##NAME##_zpzz(DisasContext *s, arg_rprr_esz *a) \
742 static gen_helper_gvec_4 * const fns[4] = { \
743 gen_helper_sve_##name##_zpzz_b, gen_helper_sve_##name##_zpzz_h, \
744 gen_helper_sve_##name##_zpzz_s, gen_helper_sve_##name##_zpzz_d, \
746 return do_zpzz_ool(s, a, fns[a->esz]); \
765 DO_ZPZZ(SMULH
, smulh
)
766 DO_ZPZZ(UMULH
, umulh
)
772 static bool trans_SDIV_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
774 static gen_helper_gvec_4
* const fns
[4] = {
775 NULL
, NULL
, gen_helper_sve_sdiv_zpzz_s
, gen_helper_sve_sdiv_zpzz_d
777 return do_zpzz_ool(s
, a
, fns
[a
->esz
]);
780 static bool trans_UDIV_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
782 static gen_helper_gvec_4
* const fns
[4] = {
783 NULL
, NULL
, gen_helper_sve_udiv_zpzz_s
, gen_helper_sve_udiv_zpzz_d
785 return do_zpzz_ool(s
, a
, fns
[a
->esz
]);
788 static bool trans_SEL_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
790 if (sve_access_check(s
)) {
791 do_sel_z(s
, a
->rd
, a
->rn
, a
->rm
, a
->pg
, a
->esz
);
799 *** SVE Integer Arithmetic - Unary Predicated Group
802 static bool do_zpz_ool(DisasContext
*s
, arg_rpr_esz
*a
, gen_helper_gvec_3
*fn
)
807 if (sve_access_check(s
)) {
808 gen_gvec_ool_zzp(s
, fn
, a
->rd
, a
->rn
, a
->pg
, 0);
813 #define DO_ZPZ(NAME, name) \
814 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
816 static gen_helper_gvec_3 * const fns[4] = { \
817 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
818 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
820 return do_zpz_ool(s, a, fns[a->esz]); \
825 DO_ZPZ(CNT_zpz
, cnt_zpz
)
827 DO_ZPZ(NOT_zpz
, not_zpz
)
831 static bool trans_FABS(DisasContext
*s
, arg_rpr_esz
*a
)
833 static gen_helper_gvec_3
* const fns
[4] = {
835 gen_helper_sve_fabs_h
,
836 gen_helper_sve_fabs_s
,
837 gen_helper_sve_fabs_d
839 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
842 static bool trans_FNEG(DisasContext
*s
, arg_rpr_esz
*a
)
844 static gen_helper_gvec_3
* const fns
[4] = {
846 gen_helper_sve_fneg_h
,
847 gen_helper_sve_fneg_s
,
848 gen_helper_sve_fneg_d
850 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
853 static bool trans_SXTB(DisasContext
*s
, arg_rpr_esz
*a
)
855 static gen_helper_gvec_3
* const fns
[4] = {
857 gen_helper_sve_sxtb_h
,
858 gen_helper_sve_sxtb_s
,
859 gen_helper_sve_sxtb_d
861 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
864 static bool trans_UXTB(DisasContext
*s
, arg_rpr_esz
*a
)
866 static gen_helper_gvec_3
* const fns
[4] = {
868 gen_helper_sve_uxtb_h
,
869 gen_helper_sve_uxtb_s
,
870 gen_helper_sve_uxtb_d
872 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
875 static bool trans_SXTH(DisasContext
*s
, arg_rpr_esz
*a
)
877 static gen_helper_gvec_3
* const fns
[4] = {
879 gen_helper_sve_sxth_s
,
880 gen_helper_sve_sxth_d
882 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
885 static bool trans_UXTH(DisasContext
*s
, arg_rpr_esz
*a
)
887 static gen_helper_gvec_3
* const fns
[4] = {
889 gen_helper_sve_uxth_s
,
890 gen_helper_sve_uxth_d
892 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
895 static bool trans_SXTW(DisasContext
*s
, arg_rpr_esz
*a
)
897 return do_zpz_ool(s
, a
, a
->esz
== 3 ? gen_helper_sve_sxtw_d
: NULL
);
900 static bool trans_UXTW(DisasContext
*s
, arg_rpr_esz
*a
)
902 return do_zpz_ool(s
, a
, a
->esz
== 3 ? gen_helper_sve_uxtw_d
: NULL
);
908 *** SVE Integer Reduction Group
911 typedef void gen_helper_gvec_reduc(TCGv_i64
, TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
912 static bool do_vpz_ool(DisasContext
*s
, arg_rpr_esz
*a
,
913 gen_helper_gvec_reduc
*fn
)
915 unsigned vsz
= vec_full_reg_size(s
);
923 if (!sve_access_check(s
)) {
927 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
928 temp
= tcg_temp_new_i64();
929 t_zn
= tcg_temp_new_ptr();
930 t_pg
= tcg_temp_new_ptr();
932 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
933 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
934 fn(temp
, t_zn
, t_pg
, desc
);
935 tcg_temp_free_ptr(t_zn
);
936 tcg_temp_free_ptr(t_pg
);
938 write_fp_dreg(s
, a
->rd
, temp
);
939 tcg_temp_free_i64(temp
);
943 #define DO_VPZ(NAME, name) \
944 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
946 static gen_helper_gvec_reduc * const fns[4] = { \
947 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
948 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
950 return do_vpz_ool(s, a, fns[a->esz]); \
963 static bool trans_SADDV(DisasContext
*s
, arg_rpr_esz
*a
)
965 static gen_helper_gvec_reduc
* const fns
[4] = {
966 gen_helper_sve_saddv_b
, gen_helper_sve_saddv_h
,
967 gen_helper_sve_saddv_s
, NULL
969 return do_vpz_ool(s
, a
, fns
[a
->esz
]);
975 *** SVE Shift by Immediate - Predicated Group
979 * Copy Zn into Zd, storing zeros into inactive elements.
980 * If invert, store zeros into the active elements.
982 static bool do_movz_zpz(DisasContext
*s
, int rd
, int rn
, int pg
,
983 int esz
, bool invert
)
985 static gen_helper_gvec_3
* const fns
[4] = {
986 gen_helper_sve_movz_b
, gen_helper_sve_movz_h
,
987 gen_helper_sve_movz_s
, gen_helper_sve_movz_d
,
990 if (sve_access_check(s
)) {
991 gen_gvec_ool_zzp(s
, fns
[esz
], rd
, rn
, pg
, invert
);
996 static bool do_zpzi_ool(DisasContext
*s
, arg_rpri_esz
*a
,
997 gen_helper_gvec_3
*fn
)
999 if (sve_access_check(s
)) {
1000 gen_gvec_ool_zzp(s
, fn
, a
->rd
, a
->rn
, a
->pg
, a
->imm
);
1005 static bool trans_ASR_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
1007 static gen_helper_gvec_3
* const fns
[4] = {
1008 gen_helper_sve_asr_zpzi_b
, gen_helper_sve_asr_zpzi_h
,
1009 gen_helper_sve_asr_zpzi_s
, gen_helper_sve_asr_zpzi_d
,
1012 /* Invalid tsz encoding -- see tszimm_esz. */
1015 /* Shift by element size is architecturally valid. For
1016 arithmetic right-shift, it's the same as by one less. */
1017 a
->imm
= MIN(a
->imm
, (8 << a
->esz
) - 1);
1018 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1021 static bool trans_LSR_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
1023 static gen_helper_gvec_3
* const fns
[4] = {
1024 gen_helper_sve_lsr_zpzi_b
, gen_helper_sve_lsr_zpzi_h
,
1025 gen_helper_sve_lsr_zpzi_s
, gen_helper_sve_lsr_zpzi_d
,
1030 /* Shift by element size is architecturally valid.
1031 For logical shifts, it is a zeroing operation. */
1032 if (a
->imm
>= (8 << a
->esz
)) {
1033 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
1035 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1039 static bool trans_LSL_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
1041 static gen_helper_gvec_3
* const fns
[4] = {
1042 gen_helper_sve_lsl_zpzi_b
, gen_helper_sve_lsl_zpzi_h
,
1043 gen_helper_sve_lsl_zpzi_s
, gen_helper_sve_lsl_zpzi_d
,
1048 /* Shift by element size is architecturally valid.
1049 For logical shifts, it is a zeroing operation. */
1050 if (a
->imm
>= (8 << a
->esz
)) {
1051 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
1053 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1057 static bool trans_ASRD(DisasContext
*s
, arg_rpri_esz
*a
)
1059 static gen_helper_gvec_3
* const fns
[4] = {
1060 gen_helper_sve_asrd_b
, gen_helper_sve_asrd_h
,
1061 gen_helper_sve_asrd_s
, gen_helper_sve_asrd_d
,
1066 /* Shift by element size is architecturally valid. For arithmetic
1067 right shift for division, it is a zeroing operation. */
1068 if (a
->imm
>= (8 << a
->esz
)) {
1069 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
1071 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1075 static bool trans_SQSHL_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
1077 static gen_helper_gvec_3
* const fns
[4] = {
1078 gen_helper_sve2_sqshl_zpzi_b
, gen_helper_sve2_sqshl_zpzi_h
,
1079 gen_helper_sve2_sqshl_zpzi_s
, gen_helper_sve2_sqshl_zpzi_d
,
1081 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
1084 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1087 static bool trans_UQSHL_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
1089 static gen_helper_gvec_3
* const fns
[4] = {
1090 gen_helper_sve2_uqshl_zpzi_b
, gen_helper_sve2_uqshl_zpzi_h
,
1091 gen_helper_sve2_uqshl_zpzi_s
, gen_helper_sve2_uqshl_zpzi_d
,
1093 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
1096 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1099 static bool trans_SRSHR(DisasContext
*s
, arg_rpri_esz
*a
)
1101 static gen_helper_gvec_3
* const fns
[4] = {
1102 gen_helper_sve2_srshr_b
, gen_helper_sve2_srshr_h
,
1103 gen_helper_sve2_srshr_s
, gen_helper_sve2_srshr_d
,
1105 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
1108 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1111 static bool trans_URSHR(DisasContext
*s
, arg_rpri_esz
*a
)
1113 static gen_helper_gvec_3
* const fns
[4] = {
1114 gen_helper_sve2_urshr_b
, gen_helper_sve2_urshr_h
,
1115 gen_helper_sve2_urshr_s
, gen_helper_sve2_urshr_d
,
1117 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
1120 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1123 static bool trans_SQSHLU(DisasContext
*s
, arg_rpri_esz
*a
)
1125 static gen_helper_gvec_3
* const fns
[4] = {
1126 gen_helper_sve2_sqshlu_b
, gen_helper_sve2_sqshlu_h
,
1127 gen_helper_sve2_sqshlu_s
, gen_helper_sve2_sqshlu_d
,
1129 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
1132 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1136 *** SVE Bitwise Shift - Predicated Group
1139 #define DO_ZPZW(NAME, name) \
1140 static bool trans_##NAME##_zpzw(DisasContext *s, arg_rprr_esz *a) \
1142 static gen_helper_gvec_4 * const fns[3] = { \
1143 gen_helper_sve_##name##_zpzw_b, gen_helper_sve_##name##_zpzw_h, \
1144 gen_helper_sve_##name##_zpzw_s, \
1146 if (a->esz < 0 || a->esz >= 3) { \
1149 return do_zpzz_ool(s, a, fns[a->esz]); \
1159 *** SVE Bitwise Shift - Unpredicated Group
1162 static bool do_shift_imm(DisasContext
*s
, arg_rri_esz
*a
, bool asr
,
1163 void (*gvec_fn
)(unsigned, uint32_t, uint32_t,
1164 int64_t, uint32_t, uint32_t))
1167 /* Invalid tsz encoding -- see tszimm_esz. */
1170 if (sve_access_check(s
)) {
1171 unsigned vsz
= vec_full_reg_size(s
);
1172 /* Shift by element size is architecturally valid. For
1173 arithmetic right-shift, it's the same as by one less.
1174 Otherwise it is a zeroing operation. */
1175 if (a
->imm
>= 8 << a
->esz
) {
1177 a
->imm
= (8 << a
->esz
) - 1;
1179 do_dupi_z(s
, a
->rd
, 0);
1183 gvec_fn(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
1184 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
1189 static bool trans_ASR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
1191 return do_shift_imm(s
, a
, true, tcg_gen_gvec_sari
);
1194 static bool trans_LSR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
1196 return do_shift_imm(s
, a
, false, tcg_gen_gvec_shri
);
1199 static bool trans_LSL_zzi(DisasContext
*s
, arg_rri_esz
*a
)
1201 return do_shift_imm(s
, a
, false, tcg_gen_gvec_shli
);
1204 #define DO_ZZW(NAME, name) \
1205 static gen_helper_gvec_3 * const name##_zzw_fns[4] = { \
1206 gen_helper_sve_##name##_zzw_b, gen_helper_sve_##name##_zzw_h, \
1207 gen_helper_sve_##name##_zzw_s, NULL \
1209 TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_arg_zzz, \
1210 name##_zzw_fns[a->esz], a, 0)
1212 DO_ZZW(ASR_zzw
, asr
)
1213 DO_ZZW(LSR_zzw
, lsr
)
1214 DO_ZZW(LSL_zzw
, lsl
)
1219 *** SVE Integer Multiply-Add Group
1222 static bool do_zpzzz_ool(DisasContext
*s
, arg_rprrr_esz
*a
,
1223 gen_helper_gvec_5
*fn
)
1225 if (sve_access_check(s
)) {
1226 unsigned vsz
= vec_full_reg_size(s
);
1227 tcg_gen_gvec_5_ool(vec_full_reg_offset(s
, a
->rd
),
1228 vec_full_reg_offset(s
, a
->ra
),
1229 vec_full_reg_offset(s
, a
->rn
),
1230 vec_full_reg_offset(s
, a
->rm
),
1231 pred_full_reg_offset(s
, a
->pg
),
1237 #define DO_ZPZZZ(NAME, name) \
1238 static bool trans_##NAME(DisasContext *s, arg_rprrr_esz *a) \
1240 static gen_helper_gvec_5 * const fns[4] = { \
1241 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
1242 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
1244 return do_zpzzz_ool(s, a, fns[a->esz]); \
1253 *** SVE Index Generation Group
1256 static void do_index(DisasContext
*s
, int esz
, int rd
,
1257 TCGv_i64 start
, TCGv_i64 incr
)
1259 unsigned vsz
= vec_full_reg_size(s
);
1260 TCGv_i32 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
1261 TCGv_ptr t_zd
= tcg_temp_new_ptr();
1263 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, rd
));
1265 gen_helper_sve_index_d(t_zd
, start
, incr
, desc
);
1267 typedef void index_fn(TCGv_ptr
, TCGv_i32
, TCGv_i32
, TCGv_i32
);
1268 static index_fn
* const fns
[3] = {
1269 gen_helper_sve_index_b
,
1270 gen_helper_sve_index_h
,
1271 gen_helper_sve_index_s
,
1273 TCGv_i32 s32
= tcg_temp_new_i32();
1274 TCGv_i32 i32
= tcg_temp_new_i32();
1276 tcg_gen_extrl_i64_i32(s32
, start
);
1277 tcg_gen_extrl_i64_i32(i32
, incr
);
1278 fns
[esz
](t_zd
, s32
, i32
, desc
);
1280 tcg_temp_free_i32(s32
);
1281 tcg_temp_free_i32(i32
);
1283 tcg_temp_free_ptr(t_zd
);
1286 static bool trans_INDEX_ii(DisasContext
*s
, arg_INDEX_ii
*a
)
1288 if (sve_access_check(s
)) {
1289 TCGv_i64 start
= tcg_constant_i64(a
->imm1
);
1290 TCGv_i64 incr
= tcg_constant_i64(a
->imm2
);
1291 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1296 static bool trans_INDEX_ir(DisasContext
*s
, arg_INDEX_ir
*a
)
1298 if (sve_access_check(s
)) {
1299 TCGv_i64 start
= tcg_constant_i64(a
->imm
);
1300 TCGv_i64 incr
= cpu_reg(s
, a
->rm
);
1301 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1306 static bool trans_INDEX_ri(DisasContext
*s
, arg_INDEX_ri
*a
)
1308 if (sve_access_check(s
)) {
1309 TCGv_i64 start
= cpu_reg(s
, a
->rn
);
1310 TCGv_i64 incr
= tcg_constant_i64(a
->imm
);
1311 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1316 static bool trans_INDEX_rr(DisasContext
*s
, arg_INDEX_rr
*a
)
1318 if (sve_access_check(s
)) {
1319 TCGv_i64 start
= cpu_reg(s
, a
->rn
);
1320 TCGv_i64 incr
= cpu_reg(s
, a
->rm
);
1321 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1327 *** SVE Stack Allocation Group
1330 static bool trans_ADDVL(DisasContext
*s
, arg_ADDVL
*a
)
1332 if (sve_access_check(s
)) {
1333 TCGv_i64 rd
= cpu_reg_sp(s
, a
->rd
);
1334 TCGv_i64 rn
= cpu_reg_sp(s
, a
->rn
);
1335 tcg_gen_addi_i64(rd
, rn
, a
->imm
* vec_full_reg_size(s
));
1340 static bool trans_ADDPL(DisasContext
*s
, arg_ADDPL
*a
)
1342 if (sve_access_check(s
)) {
1343 TCGv_i64 rd
= cpu_reg_sp(s
, a
->rd
);
1344 TCGv_i64 rn
= cpu_reg_sp(s
, a
->rn
);
1345 tcg_gen_addi_i64(rd
, rn
, a
->imm
* pred_full_reg_size(s
));
1350 static bool trans_RDVL(DisasContext
*s
, arg_RDVL
*a
)
1352 if (sve_access_check(s
)) {
1353 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
1354 tcg_gen_movi_i64(reg
, a
->imm
* vec_full_reg_size(s
));
1360 *** SVE Compute Vector Address Group
1363 static bool do_adr(DisasContext
*s
, arg_rrri
*a
, gen_helper_gvec_3
*fn
)
1365 return gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->imm
);
1368 static bool trans_ADR_p32(DisasContext
*s
, arg_rrri
*a
)
1370 return do_adr(s
, a
, gen_helper_sve_adr_p32
);
1373 static bool trans_ADR_p64(DisasContext
*s
, arg_rrri
*a
)
1375 return do_adr(s
, a
, gen_helper_sve_adr_p64
);
1378 static bool trans_ADR_s32(DisasContext
*s
, arg_rrri
*a
)
1380 return do_adr(s
, a
, gen_helper_sve_adr_s32
);
1383 static bool trans_ADR_u32(DisasContext
*s
, arg_rrri
*a
)
1385 return do_adr(s
, a
, gen_helper_sve_adr_u32
);
1389 *** SVE Integer Misc - Unpredicated Group
1392 static gen_helper_gvec_2
* const fexpa_fns
[4] = {
1393 NULL
, gen_helper_sve_fexpa_h
,
1394 gen_helper_sve_fexpa_s
, gen_helper_sve_fexpa_d
,
1396 TRANS_FEAT(FEXPA
, aa64_sve
, gen_gvec_ool_zz
,
1397 fexpa_fns
[a
->esz
], a
->rd
, a
->rn
, 0)
1399 static gen_helper_gvec_3
* const ftssel_fns
[4] = {
1400 NULL
, gen_helper_sve_ftssel_h
,
1401 gen_helper_sve_ftssel_s
, gen_helper_sve_ftssel_d
,
1403 TRANS_FEAT(FTSSEL
, aa64_sve
, gen_gvec_ool_arg_zzz
, ftssel_fns
[a
->esz
], a
, 0)
1406 *** SVE Predicate Logical Operations Group
1409 static bool do_pppp_flags(DisasContext
*s
, arg_rprr_s
*a
,
1410 const GVecGen4
*gvec_op
)
1412 if (!sve_access_check(s
)) {
1416 unsigned psz
= pred_gvec_reg_size(s
);
1417 int dofs
= pred_full_reg_offset(s
, a
->rd
);
1418 int nofs
= pred_full_reg_offset(s
, a
->rn
);
1419 int mofs
= pred_full_reg_offset(s
, a
->rm
);
1420 int gofs
= pred_full_reg_offset(s
, a
->pg
);
1423 tcg_gen_gvec_4(dofs
, nofs
, mofs
, gofs
, psz
, psz
, gvec_op
);
1428 /* Do the operation and the flags generation in temps. */
1429 TCGv_i64 pd
= tcg_temp_new_i64();
1430 TCGv_i64 pn
= tcg_temp_new_i64();
1431 TCGv_i64 pm
= tcg_temp_new_i64();
1432 TCGv_i64 pg
= tcg_temp_new_i64();
1434 tcg_gen_ld_i64(pn
, cpu_env
, nofs
);
1435 tcg_gen_ld_i64(pm
, cpu_env
, mofs
);
1436 tcg_gen_ld_i64(pg
, cpu_env
, gofs
);
1438 gvec_op
->fni8(pd
, pn
, pm
, pg
);
1439 tcg_gen_st_i64(pd
, cpu_env
, dofs
);
1441 do_predtest1(pd
, pg
);
1443 tcg_temp_free_i64(pd
);
1444 tcg_temp_free_i64(pn
);
1445 tcg_temp_free_i64(pm
);
1446 tcg_temp_free_i64(pg
);
1448 /* The operation and flags generation is large. The computation
1449 * of the flags depends on the original contents of the guarding
1450 * predicate. If the destination overwrites the guarding predicate,
1451 * then the easiest way to get this right is to save a copy.
1454 if (a
->rd
== a
->pg
) {
1455 tofs
= offsetof(CPUARMState
, vfp
.preg_tmp
);
1456 tcg_gen_gvec_mov(0, tofs
, gofs
, psz
, psz
);
1459 tcg_gen_gvec_4(dofs
, nofs
, mofs
, gofs
, psz
, psz
, gvec_op
);
1460 do_predtest(s
, dofs
, tofs
, psz
/ 8);
1465 static void gen_and_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1467 tcg_gen_and_i64(pd
, pn
, pm
);
1468 tcg_gen_and_i64(pd
, pd
, pg
);
1471 static void gen_and_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1472 TCGv_vec pm
, TCGv_vec pg
)
1474 tcg_gen_and_vec(vece
, pd
, pn
, pm
);
1475 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1478 static bool trans_AND_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1480 static const GVecGen4 op
= {
1481 .fni8
= gen_and_pg_i64
,
1482 .fniv
= gen_and_pg_vec
,
1483 .fno
= gen_helper_sve_and_pppp
,
1484 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1488 if (!sve_access_check(s
)) {
1491 if (a
->rn
== a
->rm
) {
1492 if (a
->pg
== a
->rn
) {
1493 do_mov_p(s
, a
->rd
, a
->rn
);
1495 gen_gvec_fn_ppp(s
, tcg_gen_gvec_and
, a
->rd
, a
->rn
, a
->pg
);
1498 } else if (a
->pg
== a
->rn
|| a
->pg
== a
->rm
) {
1499 gen_gvec_fn_ppp(s
, tcg_gen_gvec_and
, a
->rd
, a
->rn
, a
->rm
);
1503 return do_pppp_flags(s
, a
, &op
);
1506 static void gen_bic_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1508 tcg_gen_andc_i64(pd
, pn
, pm
);
1509 tcg_gen_and_i64(pd
, pd
, pg
);
1512 static void gen_bic_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1513 TCGv_vec pm
, TCGv_vec pg
)
1515 tcg_gen_andc_vec(vece
, pd
, pn
, pm
);
1516 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1519 static bool trans_BIC_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1521 static const GVecGen4 op
= {
1522 .fni8
= gen_bic_pg_i64
,
1523 .fniv
= gen_bic_pg_vec
,
1524 .fno
= gen_helper_sve_bic_pppp
,
1525 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1528 if (!a
->s
&& a
->pg
== a
->rn
) {
1529 if (sve_access_check(s
)) {
1530 gen_gvec_fn_ppp(s
, tcg_gen_gvec_andc
, a
->rd
, a
->rn
, a
->rm
);
1534 return do_pppp_flags(s
, a
, &op
);
1537 static void gen_eor_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1539 tcg_gen_xor_i64(pd
, pn
, pm
);
1540 tcg_gen_and_i64(pd
, pd
, pg
);
1543 static void gen_eor_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1544 TCGv_vec pm
, TCGv_vec pg
)
1546 tcg_gen_xor_vec(vece
, pd
, pn
, pm
);
1547 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1550 static bool trans_EOR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1552 static const GVecGen4 op
= {
1553 .fni8
= gen_eor_pg_i64
,
1554 .fniv
= gen_eor_pg_vec
,
1555 .fno
= gen_helper_sve_eor_pppp
,
1556 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1558 return do_pppp_flags(s
, a
, &op
);
1561 static bool trans_SEL_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1566 if (sve_access_check(s
)) {
1567 unsigned psz
= pred_gvec_reg_size(s
);
1568 tcg_gen_gvec_bitsel(MO_8
, pred_full_reg_offset(s
, a
->rd
),
1569 pred_full_reg_offset(s
, a
->pg
),
1570 pred_full_reg_offset(s
, a
->rn
),
1571 pred_full_reg_offset(s
, a
->rm
), psz
, psz
);
1576 static void gen_orr_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1578 tcg_gen_or_i64(pd
, pn
, pm
);
1579 tcg_gen_and_i64(pd
, pd
, pg
);
1582 static void gen_orr_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1583 TCGv_vec pm
, TCGv_vec pg
)
1585 tcg_gen_or_vec(vece
, pd
, pn
, pm
);
1586 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1589 static bool trans_ORR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1591 static const GVecGen4 op
= {
1592 .fni8
= gen_orr_pg_i64
,
1593 .fniv
= gen_orr_pg_vec
,
1594 .fno
= gen_helper_sve_orr_pppp
,
1595 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1598 if (!a
->s
&& a
->pg
== a
->rn
&& a
->rn
== a
->rm
) {
1599 return do_mov_p(s
, a
->rd
, a
->rn
);
1601 return do_pppp_flags(s
, a
, &op
);
1604 static void gen_orn_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1606 tcg_gen_orc_i64(pd
, pn
, pm
);
1607 tcg_gen_and_i64(pd
, pd
, pg
);
1610 static void gen_orn_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1611 TCGv_vec pm
, TCGv_vec pg
)
1613 tcg_gen_orc_vec(vece
, pd
, pn
, pm
);
1614 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1617 static bool trans_ORN_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1619 static const GVecGen4 op
= {
1620 .fni8
= gen_orn_pg_i64
,
1621 .fniv
= gen_orn_pg_vec
,
1622 .fno
= gen_helper_sve_orn_pppp
,
1623 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1625 return do_pppp_flags(s
, a
, &op
);
1628 static void gen_nor_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1630 tcg_gen_or_i64(pd
, pn
, pm
);
1631 tcg_gen_andc_i64(pd
, pg
, pd
);
1634 static void gen_nor_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1635 TCGv_vec pm
, TCGv_vec pg
)
1637 tcg_gen_or_vec(vece
, pd
, pn
, pm
);
1638 tcg_gen_andc_vec(vece
, pd
, pg
, pd
);
1641 static bool trans_NOR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1643 static const GVecGen4 op
= {
1644 .fni8
= gen_nor_pg_i64
,
1645 .fniv
= gen_nor_pg_vec
,
1646 .fno
= gen_helper_sve_nor_pppp
,
1647 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1649 return do_pppp_flags(s
, a
, &op
);
1652 static void gen_nand_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1654 tcg_gen_and_i64(pd
, pn
, pm
);
1655 tcg_gen_andc_i64(pd
, pg
, pd
);
1658 static void gen_nand_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1659 TCGv_vec pm
, TCGv_vec pg
)
1661 tcg_gen_and_vec(vece
, pd
, pn
, pm
);
1662 tcg_gen_andc_vec(vece
, pd
, pg
, pd
);
1665 static bool trans_NAND_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1667 static const GVecGen4 op
= {
1668 .fni8
= gen_nand_pg_i64
,
1669 .fniv
= gen_nand_pg_vec
,
1670 .fno
= gen_helper_sve_nand_pppp
,
1671 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1673 return do_pppp_flags(s
, a
, &op
);
1677 *** SVE Predicate Misc Group
1680 static bool trans_PTEST(DisasContext
*s
, arg_PTEST
*a
)
1682 if (sve_access_check(s
)) {
1683 int nofs
= pred_full_reg_offset(s
, a
->rn
);
1684 int gofs
= pred_full_reg_offset(s
, a
->pg
);
1685 int words
= DIV_ROUND_UP(pred_full_reg_size(s
), 8);
1688 TCGv_i64 pn
= tcg_temp_new_i64();
1689 TCGv_i64 pg
= tcg_temp_new_i64();
1691 tcg_gen_ld_i64(pn
, cpu_env
, nofs
);
1692 tcg_gen_ld_i64(pg
, cpu_env
, gofs
);
1693 do_predtest1(pn
, pg
);
1695 tcg_temp_free_i64(pn
);
1696 tcg_temp_free_i64(pg
);
1698 do_predtest(s
, nofs
, gofs
, words
);
1704 /* See the ARM pseudocode DecodePredCount. */
1705 static unsigned decode_pred_count(unsigned fullsz
, int pattern
, int esz
)
1707 unsigned elements
= fullsz
>> esz
;
1711 case 0x0: /* POW2 */
1712 return pow2floor(elements
);
1723 case 0x9: /* VL16 */
1724 case 0xa: /* VL32 */
1725 case 0xb: /* VL64 */
1726 case 0xc: /* VL128 */
1727 case 0xd: /* VL256 */
1728 bound
= 16 << (pattern
- 9);
1730 case 0x1d: /* MUL4 */
1731 return elements
- elements
% 4;
1732 case 0x1e: /* MUL3 */
1733 return elements
- elements
% 3;
1734 case 0x1f: /* ALL */
1736 default: /* #uimm5 */
1739 return elements
>= bound
? bound
: 0;
1742 /* This handles all of the predicate initialization instructions,
1743 * PTRUE, PFALSE, SETFFR. For PFALSE, we will have set PAT == 32
1744 * so that decode_pred_count returns 0. For SETFFR, we will have
1745 * set RD == 16 == FFR.
1747 static bool do_predset(DisasContext
*s
, int esz
, int rd
, int pat
, bool setflag
)
1749 if (!sve_access_check(s
)) {
1753 unsigned fullsz
= vec_full_reg_size(s
);
1754 unsigned ofs
= pred_full_reg_offset(s
, rd
);
1755 unsigned numelem
, setsz
, i
;
1756 uint64_t word
, lastword
;
1759 numelem
= decode_pred_count(fullsz
, pat
, esz
);
1761 /* Determine what we must store into each bit, and how many. */
1763 lastword
= word
= 0;
1766 setsz
= numelem
<< esz
;
1767 lastword
= word
= pred_esz_masks
[esz
];
1769 lastword
&= MAKE_64BIT_MASK(0, setsz
% 64);
1773 t
= tcg_temp_new_i64();
1775 tcg_gen_movi_i64(t
, lastword
);
1776 tcg_gen_st_i64(t
, cpu_env
, ofs
);
1780 if (word
== lastword
) {
1781 unsigned maxsz
= size_for_gvec(fullsz
/ 8);
1782 unsigned oprsz
= size_for_gvec(setsz
/ 8);
1784 if (oprsz
* 8 == setsz
) {
1785 tcg_gen_gvec_dup_imm(MO_64
, ofs
, oprsz
, maxsz
, word
);
1793 tcg_gen_movi_i64(t
, word
);
1794 for (i
= 0; i
< QEMU_ALIGN_DOWN(setsz
, 8); i
+= 8) {
1795 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1797 if (lastword
!= word
) {
1798 tcg_gen_movi_i64(t
, lastword
);
1799 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1803 tcg_gen_movi_i64(t
, 0);
1804 for (; i
< fullsz
; i
+= 8) {
1805 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1810 tcg_temp_free_i64(t
);
1814 tcg_gen_movi_i32(cpu_NF
, -(word
!= 0));
1815 tcg_gen_movi_i32(cpu_CF
, word
== 0);
1816 tcg_gen_movi_i32(cpu_VF
, 0);
1817 tcg_gen_mov_i32(cpu_ZF
, cpu_NF
);
1822 static bool trans_PTRUE(DisasContext
*s
, arg_PTRUE
*a
)
1824 return do_predset(s
, a
->esz
, a
->rd
, a
->pat
, a
->s
);
1827 static bool trans_SETFFR(DisasContext
*s
, arg_SETFFR
*a
)
1829 /* Note pat == 31 is #all, to set all elements. */
1830 return do_predset(s
, 0, FFR_PRED_NUM
, 31, false);
1833 static bool trans_PFALSE(DisasContext
*s
, arg_PFALSE
*a
)
1835 /* Note pat == 32 is #unimp, to set no elements. */
1836 return do_predset(s
, 0, a
->rd
, 32, false);
1839 static bool trans_RDFFR_p(DisasContext
*s
, arg_RDFFR_p
*a
)
1841 /* The path through do_pppp_flags is complicated enough to want to avoid
1842 * duplication. Frob the arguments into the form of a predicated AND.
1844 arg_rprr_s alt_a
= {
1845 .rd
= a
->rd
, .pg
= a
->pg
, .s
= a
->s
,
1846 .rn
= FFR_PRED_NUM
, .rm
= FFR_PRED_NUM
,
1848 return trans_AND_pppp(s
, &alt_a
);
1851 static bool trans_RDFFR(DisasContext
*s
, arg_RDFFR
*a
)
1853 return do_mov_p(s
, a
->rd
, FFR_PRED_NUM
);
1856 static bool trans_WRFFR(DisasContext
*s
, arg_WRFFR
*a
)
1858 return do_mov_p(s
, FFR_PRED_NUM
, a
->rn
);
1861 static bool do_pfirst_pnext(DisasContext
*s
, arg_rr_esz
*a
,
1862 void (*gen_fn
)(TCGv_i32
, TCGv_ptr
,
1863 TCGv_ptr
, TCGv_i32
))
1865 if (!sve_access_check(s
)) {
1869 TCGv_ptr t_pd
= tcg_temp_new_ptr();
1870 TCGv_ptr t_pg
= tcg_temp_new_ptr();
1874 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, pred_full_reg_size(s
));
1875 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
1877 tcg_gen_addi_ptr(t_pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
1878 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
1879 t
= tcg_temp_new_i32();
1881 gen_fn(t
, t_pd
, t_pg
, tcg_constant_i32(desc
));
1882 tcg_temp_free_ptr(t_pd
);
1883 tcg_temp_free_ptr(t_pg
);
1886 tcg_temp_free_i32(t
);
1890 static bool trans_PFIRST(DisasContext
*s
, arg_rr_esz
*a
)
1892 return do_pfirst_pnext(s
, a
, gen_helper_sve_pfirst
);
1895 static bool trans_PNEXT(DisasContext
*s
, arg_rr_esz
*a
)
1897 return do_pfirst_pnext(s
, a
, gen_helper_sve_pnext
);
1901 *** SVE Element Count Group
1904 /* Perform an inline saturating addition of a 32-bit value within
1905 * a 64-bit register. The second operand is known to be positive,
1906 * which halves the comparisions we must perform to bound the result.
1908 static void do_sat_addsub_32(TCGv_i64 reg
, TCGv_i64 val
, bool u
, bool d
)
1912 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
1914 tcg_gen_ext32u_i64(reg
, reg
);
1916 tcg_gen_ext32s_i64(reg
, reg
);
1919 tcg_gen_sub_i64(reg
, reg
, val
);
1920 ibound
= (u
? 0 : INT32_MIN
);
1921 tcg_gen_smax_i64(reg
, reg
, tcg_constant_i64(ibound
));
1923 tcg_gen_add_i64(reg
, reg
, val
);
1924 ibound
= (u
? UINT32_MAX
: INT32_MAX
);
1925 tcg_gen_smin_i64(reg
, reg
, tcg_constant_i64(ibound
));
1929 /* Similarly with 64-bit values. */
1930 static void do_sat_addsub_64(TCGv_i64 reg
, TCGv_i64 val
, bool u
, bool d
)
1932 TCGv_i64 t0
= tcg_temp_new_i64();
1937 tcg_gen_sub_i64(t0
, reg
, val
);
1938 t2
= tcg_constant_i64(0);
1939 tcg_gen_movcond_i64(TCG_COND_LTU
, reg
, reg
, val
, t2
, t0
);
1941 tcg_gen_add_i64(t0
, reg
, val
);
1942 t2
= tcg_constant_i64(-1);
1943 tcg_gen_movcond_i64(TCG_COND_LTU
, reg
, t0
, reg
, t2
, t0
);
1946 TCGv_i64 t1
= tcg_temp_new_i64();
1948 /* Detect signed overflow for subtraction. */
1949 tcg_gen_xor_i64(t0
, reg
, val
);
1950 tcg_gen_sub_i64(t1
, reg
, val
);
1951 tcg_gen_xor_i64(reg
, reg
, t1
);
1952 tcg_gen_and_i64(t0
, t0
, reg
);
1954 /* Bound the result. */
1955 tcg_gen_movi_i64(reg
, INT64_MIN
);
1956 t2
= tcg_constant_i64(0);
1957 tcg_gen_movcond_i64(TCG_COND_LT
, reg
, t0
, t2
, reg
, t1
);
1959 /* Detect signed overflow for addition. */
1960 tcg_gen_xor_i64(t0
, reg
, val
);
1961 tcg_gen_add_i64(reg
, reg
, val
);
1962 tcg_gen_xor_i64(t1
, reg
, val
);
1963 tcg_gen_andc_i64(t0
, t1
, t0
);
1965 /* Bound the result. */
1966 tcg_gen_movi_i64(t1
, INT64_MAX
);
1967 t2
= tcg_constant_i64(0);
1968 tcg_gen_movcond_i64(TCG_COND_LT
, reg
, t0
, t2
, t1
, reg
);
1970 tcg_temp_free_i64(t1
);
1972 tcg_temp_free_i64(t0
);
1975 /* Similarly with a vector and a scalar operand. */
1976 static void do_sat_addsub_vec(DisasContext
*s
, int esz
, int rd
, int rn
,
1977 TCGv_i64 val
, bool u
, bool d
)
1979 unsigned vsz
= vec_full_reg_size(s
);
1980 TCGv_ptr dptr
, nptr
;
1984 dptr
= tcg_temp_new_ptr();
1985 nptr
= tcg_temp_new_ptr();
1986 tcg_gen_addi_ptr(dptr
, cpu_env
, vec_full_reg_offset(s
, rd
));
1987 tcg_gen_addi_ptr(nptr
, cpu_env
, vec_full_reg_offset(s
, rn
));
1988 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
1992 t32
= tcg_temp_new_i32();
1993 tcg_gen_extrl_i64_i32(t32
, val
);
1995 tcg_gen_neg_i32(t32
, t32
);
1998 gen_helper_sve_uqaddi_b(dptr
, nptr
, t32
, desc
);
2000 gen_helper_sve_sqaddi_b(dptr
, nptr
, t32
, desc
);
2002 tcg_temp_free_i32(t32
);
2006 t32
= tcg_temp_new_i32();
2007 tcg_gen_extrl_i64_i32(t32
, val
);
2009 tcg_gen_neg_i32(t32
, t32
);
2012 gen_helper_sve_uqaddi_h(dptr
, nptr
, t32
, desc
);
2014 gen_helper_sve_sqaddi_h(dptr
, nptr
, t32
, desc
);
2016 tcg_temp_free_i32(t32
);
2020 t64
= tcg_temp_new_i64();
2022 tcg_gen_neg_i64(t64
, val
);
2024 tcg_gen_mov_i64(t64
, val
);
2027 gen_helper_sve_uqaddi_s(dptr
, nptr
, t64
, desc
);
2029 gen_helper_sve_sqaddi_s(dptr
, nptr
, t64
, desc
);
2031 tcg_temp_free_i64(t64
);
2037 gen_helper_sve_uqsubi_d(dptr
, nptr
, val
, desc
);
2039 gen_helper_sve_uqaddi_d(dptr
, nptr
, val
, desc
);
2042 t64
= tcg_temp_new_i64();
2043 tcg_gen_neg_i64(t64
, val
);
2044 gen_helper_sve_sqaddi_d(dptr
, nptr
, t64
, desc
);
2045 tcg_temp_free_i64(t64
);
2047 gen_helper_sve_sqaddi_d(dptr
, nptr
, val
, desc
);
2052 g_assert_not_reached();
2055 tcg_temp_free_ptr(dptr
);
2056 tcg_temp_free_ptr(nptr
);
2059 static bool trans_CNT_r(DisasContext
*s
, arg_CNT_r
*a
)
2061 if (sve_access_check(s
)) {
2062 unsigned fullsz
= vec_full_reg_size(s
);
2063 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2064 tcg_gen_movi_i64(cpu_reg(s
, a
->rd
), numelem
* a
->imm
);
2069 static bool trans_INCDEC_r(DisasContext
*s
, arg_incdec_cnt
*a
)
2071 if (sve_access_check(s
)) {
2072 unsigned fullsz
= vec_full_reg_size(s
);
2073 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2074 int inc
= numelem
* a
->imm
* (a
->d
? -1 : 1);
2075 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
2077 tcg_gen_addi_i64(reg
, reg
, inc
);
2082 static bool trans_SINCDEC_r_32(DisasContext
*s
, arg_incdec_cnt
*a
)
2084 if (!sve_access_check(s
)) {
2088 unsigned fullsz
= vec_full_reg_size(s
);
2089 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2090 int inc
= numelem
* a
->imm
;
2091 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
2093 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
2096 tcg_gen_ext32u_i64(reg
, reg
);
2098 tcg_gen_ext32s_i64(reg
, reg
);
2101 do_sat_addsub_32(reg
, tcg_constant_i64(inc
), a
->u
, a
->d
);
2106 static bool trans_SINCDEC_r_64(DisasContext
*s
, arg_incdec_cnt
*a
)
2108 if (!sve_access_check(s
)) {
2112 unsigned fullsz
= vec_full_reg_size(s
);
2113 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2114 int inc
= numelem
* a
->imm
;
2115 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
2118 do_sat_addsub_64(reg
, tcg_constant_i64(inc
), a
->u
, a
->d
);
2123 static bool trans_INCDEC_v(DisasContext
*s
, arg_incdec2_cnt
*a
)
2129 unsigned fullsz
= vec_full_reg_size(s
);
2130 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2131 int inc
= numelem
* a
->imm
;
2134 if (sve_access_check(s
)) {
2135 tcg_gen_gvec_adds(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
2136 vec_full_reg_offset(s
, a
->rn
),
2137 tcg_constant_i64(a
->d
? -inc
: inc
),
2141 do_mov_z(s
, a
->rd
, a
->rn
);
2146 static bool trans_SINCDEC_v(DisasContext
*s
, arg_incdec2_cnt
*a
)
2152 unsigned fullsz
= vec_full_reg_size(s
);
2153 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2154 int inc
= numelem
* a
->imm
;
2157 if (sve_access_check(s
)) {
2158 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
,
2159 tcg_constant_i64(inc
), a
->u
, a
->d
);
2162 do_mov_z(s
, a
->rd
, a
->rn
);
2168 *** SVE Bitwise Immediate Group
2171 static bool do_zz_dbm(DisasContext
*s
, arg_rr_dbm
*a
, GVecGen2iFn
*gvec_fn
)
2174 if (!logic_imm_decode_wmask(&imm
, extract32(a
->dbm
, 12, 1),
2175 extract32(a
->dbm
, 0, 6),
2176 extract32(a
->dbm
, 6, 6))) {
2179 if (sve_access_check(s
)) {
2180 unsigned vsz
= vec_full_reg_size(s
);
2181 gvec_fn(MO_64
, vec_full_reg_offset(s
, a
->rd
),
2182 vec_full_reg_offset(s
, a
->rn
), imm
, vsz
, vsz
);
2187 static bool trans_AND_zzi(DisasContext
*s
, arg_rr_dbm
*a
)
2189 return do_zz_dbm(s
, a
, tcg_gen_gvec_andi
);
2192 static bool trans_ORR_zzi(DisasContext
*s
, arg_rr_dbm
*a
)
2194 return do_zz_dbm(s
, a
, tcg_gen_gvec_ori
);
2197 static bool trans_EOR_zzi(DisasContext
*s
, arg_rr_dbm
*a
)
2199 return do_zz_dbm(s
, a
, tcg_gen_gvec_xori
);
2202 static bool trans_DUPM(DisasContext
*s
, arg_DUPM
*a
)
2205 if (!logic_imm_decode_wmask(&imm
, extract32(a
->dbm
, 12, 1),
2206 extract32(a
->dbm
, 0, 6),
2207 extract32(a
->dbm
, 6, 6))) {
2210 if (sve_access_check(s
)) {
2211 do_dupi_z(s
, a
->rd
, imm
);
2217 *** SVE Integer Wide Immediate - Predicated Group
2220 /* Implement all merging copies. This is used for CPY (immediate),
2221 * FCPY, CPY (scalar), CPY (SIMD&FP scalar).
2223 static void do_cpy_m(DisasContext
*s
, int esz
, int rd
, int rn
, int pg
,
2226 typedef void gen_cpy(TCGv_ptr
, TCGv_ptr
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
2227 static gen_cpy
* const fns
[4] = {
2228 gen_helper_sve_cpy_m_b
, gen_helper_sve_cpy_m_h
,
2229 gen_helper_sve_cpy_m_s
, gen_helper_sve_cpy_m_d
,
2231 unsigned vsz
= vec_full_reg_size(s
);
2232 TCGv_i32 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
2233 TCGv_ptr t_zd
= tcg_temp_new_ptr();
2234 TCGv_ptr t_zn
= tcg_temp_new_ptr();
2235 TCGv_ptr t_pg
= tcg_temp_new_ptr();
2237 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, rd
));
2238 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, rn
));
2239 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
2241 fns
[esz
](t_zd
, t_zn
, t_pg
, val
, desc
);
2243 tcg_temp_free_ptr(t_zd
);
2244 tcg_temp_free_ptr(t_zn
);
2245 tcg_temp_free_ptr(t_pg
);
2248 static bool trans_FCPY(DisasContext
*s
, arg_FCPY
*a
)
2253 if (sve_access_check(s
)) {
2254 /* Decode the VFP immediate. */
2255 uint64_t imm
= vfp_expand_imm(a
->esz
, a
->imm
);
2256 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rn
, a
->pg
, tcg_constant_i64(imm
));
2261 static bool trans_CPY_m_i(DisasContext
*s
, arg_rpri_esz
*a
)
2263 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
2266 if (sve_access_check(s
)) {
2267 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rn
, a
->pg
, tcg_constant_i64(a
->imm
));
2272 static bool trans_CPY_z_i(DisasContext
*s
, arg_CPY_z_i
*a
)
2274 static gen_helper_gvec_2i
* const fns
[4] = {
2275 gen_helper_sve_cpy_z_b
, gen_helper_sve_cpy_z_h
,
2276 gen_helper_sve_cpy_z_s
, gen_helper_sve_cpy_z_d
,
2279 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
2282 if (sve_access_check(s
)) {
2283 unsigned vsz
= vec_full_reg_size(s
);
2284 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s
, a
->rd
),
2285 pred_full_reg_offset(s
, a
->pg
),
2286 tcg_constant_i64(a
->imm
),
2287 vsz
, vsz
, 0, fns
[a
->esz
]);
2293 *** SVE Permute Extract Group
2296 static bool do_EXT(DisasContext
*s
, int rd
, int rn
, int rm
, int imm
)
2298 if (!sve_access_check(s
)) {
2302 unsigned vsz
= vec_full_reg_size(s
);
2303 unsigned n_ofs
= imm
>= vsz
? 0 : imm
;
2304 unsigned n_siz
= vsz
- n_ofs
;
2305 unsigned d
= vec_full_reg_offset(s
, rd
);
2306 unsigned n
= vec_full_reg_offset(s
, rn
);
2307 unsigned m
= vec_full_reg_offset(s
, rm
);
2309 /* Use host vector move insns if we have appropriate sizes
2310 * and no unfortunate overlap.
2313 && n_ofs
== size_for_gvec(n_ofs
)
2314 && n_siz
== size_for_gvec(n_siz
)
2315 && (d
!= n
|| n_siz
<= n_ofs
)) {
2316 tcg_gen_gvec_mov(0, d
, n
+ n_ofs
, n_siz
, n_siz
);
2318 tcg_gen_gvec_mov(0, d
+ n_siz
, m
, n_ofs
, n_ofs
);
2321 tcg_gen_gvec_3_ool(d
, n
, m
, vsz
, vsz
, n_ofs
, gen_helper_sve_ext
);
2326 static bool trans_EXT(DisasContext
*s
, arg_EXT
*a
)
2328 return do_EXT(s
, a
->rd
, a
->rn
, a
->rm
, a
->imm
);
2331 static bool trans_EXT_sve2(DisasContext
*s
, arg_rri
*a
)
2333 if (!dc_isar_feature(aa64_sve2
, s
)) {
2336 return do_EXT(s
, a
->rd
, a
->rn
, (a
->rn
+ 1) % 32, a
->imm
);
2340 *** SVE Permute - Unpredicated Group
2343 static bool trans_DUP_s(DisasContext
*s
, arg_DUP_s
*a
)
2345 if (sve_access_check(s
)) {
2346 unsigned vsz
= vec_full_reg_size(s
);
2347 tcg_gen_gvec_dup_i64(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
2348 vsz
, vsz
, cpu_reg_sp(s
, a
->rn
));
2353 static bool trans_DUP_x(DisasContext
*s
, arg_DUP_x
*a
)
2355 if ((a
->imm
& 0x1f) == 0) {
2358 if (sve_access_check(s
)) {
2359 unsigned vsz
= vec_full_reg_size(s
);
2360 unsigned dofs
= vec_full_reg_offset(s
, a
->rd
);
2361 unsigned esz
, index
;
2363 esz
= ctz32(a
->imm
);
2364 index
= a
->imm
>> (esz
+ 1);
2366 if ((index
<< esz
) < vsz
) {
2367 unsigned nofs
= vec_reg_offset(s
, a
->rn
, index
, esz
);
2368 tcg_gen_gvec_dup_mem(esz
, dofs
, nofs
, vsz
, vsz
);
2371 * While dup_mem handles 128-bit elements, dup_imm does not.
2372 * Thankfully element size doesn't matter for splatting zero.
2374 tcg_gen_gvec_dup_imm(MO_64
, dofs
, vsz
, vsz
, 0);
2380 static void do_insr_i64(DisasContext
*s
, arg_rrr_esz
*a
, TCGv_i64 val
)
2382 typedef void gen_insr(TCGv_ptr
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
2383 static gen_insr
* const fns
[4] = {
2384 gen_helper_sve_insr_b
, gen_helper_sve_insr_h
,
2385 gen_helper_sve_insr_s
, gen_helper_sve_insr_d
,
2387 unsigned vsz
= vec_full_reg_size(s
);
2388 TCGv_i32 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
2389 TCGv_ptr t_zd
= tcg_temp_new_ptr();
2390 TCGv_ptr t_zn
= tcg_temp_new_ptr();
2392 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, a
->rd
));
2393 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
2395 fns
[a
->esz
](t_zd
, t_zn
, val
, desc
);
2397 tcg_temp_free_ptr(t_zd
);
2398 tcg_temp_free_ptr(t_zn
);
2401 static bool trans_INSR_f(DisasContext
*s
, arg_rrr_esz
*a
)
2403 if (sve_access_check(s
)) {
2404 TCGv_i64 t
= tcg_temp_new_i64();
2405 tcg_gen_ld_i64(t
, cpu_env
, vec_reg_offset(s
, a
->rm
, 0, MO_64
));
2406 do_insr_i64(s
, a
, t
);
2407 tcg_temp_free_i64(t
);
2412 static bool trans_INSR_r(DisasContext
*s
, arg_rrr_esz
*a
)
2414 if (sve_access_check(s
)) {
2415 do_insr_i64(s
, a
, cpu_reg(s
, a
->rm
));
2420 static gen_helper_gvec_2
* const rev_fns
[4] = {
2421 gen_helper_sve_rev_b
, gen_helper_sve_rev_h
,
2422 gen_helper_sve_rev_s
, gen_helper_sve_rev_d
2424 TRANS_FEAT(REV_v
, aa64_sve
, gen_gvec_ool_zz
, rev_fns
[a
->esz
], a
->rd
, a
->rn
, 0)
2426 static gen_helper_gvec_3
* const sve_tbl_fns
[4] = {
2427 gen_helper_sve_tbl_b
, gen_helper_sve_tbl_h
,
2428 gen_helper_sve_tbl_s
, gen_helper_sve_tbl_d
2430 TRANS_FEAT(TBL
, aa64_sve
, gen_gvec_ool_arg_zzz
, sve_tbl_fns
[a
->esz
], a
, 0)
2432 static gen_helper_gvec_4
* const sve2_tbl_fns
[4] = {
2433 gen_helper_sve2_tbl_b
, gen_helper_sve2_tbl_h
,
2434 gen_helper_sve2_tbl_s
, gen_helper_sve2_tbl_d
2436 TRANS_FEAT(TBL_sve2
, aa64_sve2
, gen_gvec_ool_zzzz
, sve2_tbl_fns
[a
->esz
],
2437 a
->rd
, a
->rn
, (a
->rn
+ 1) % 32, a
->rm
, 0)
2439 static gen_helper_gvec_3
* const tbx_fns
[4] = {
2440 gen_helper_sve2_tbx_b
, gen_helper_sve2_tbx_h
,
2441 gen_helper_sve2_tbx_s
, gen_helper_sve2_tbx_d
2443 TRANS_FEAT(TBX
, aa64_sve2
, gen_gvec_ool_arg_zzz
, tbx_fns
[a
->esz
], a
, 0)
2445 static bool trans_UNPK(DisasContext
*s
, arg_UNPK
*a
)
2447 static gen_helper_gvec_2
* const fns
[4][2] = {
2449 { gen_helper_sve_sunpk_h
, gen_helper_sve_uunpk_h
},
2450 { gen_helper_sve_sunpk_s
, gen_helper_sve_uunpk_s
},
2451 { gen_helper_sve_sunpk_d
, gen_helper_sve_uunpk_d
},
2457 if (sve_access_check(s
)) {
2458 unsigned vsz
= vec_full_reg_size(s
);
2459 tcg_gen_gvec_2_ool(vec_full_reg_offset(s
, a
->rd
),
2460 vec_full_reg_offset(s
, a
->rn
)
2461 + (a
->h
? vsz
/ 2 : 0),
2462 vsz
, vsz
, 0, fns
[a
->esz
][a
->u
]);
2468 *** SVE Permute - Predicates Group
2471 static bool do_perm_pred3(DisasContext
*s
, arg_rrr_esz
*a
, bool high_odd
,
2472 gen_helper_gvec_3
*fn
)
2474 if (!sve_access_check(s
)) {
2478 unsigned vsz
= pred_full_reg_size(s
);
2480 TCGv_ptr t_d
= tcg_temp_new_ptr();
2481 TCGv_ptr t_n
= tcg_temp_new_ptr();
2482 TCGv_ptr t_m
= tcg_temp_new_ptr();
2485 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
);
2486 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
2487 desc
= FIELD_DP32(desc
, PREDDESC
, DATA
, high_odd
);
2489 tcg_gen_addi_ptr(t_d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2490 tcg_gen_addi_ptr(t_n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2491 tcg_gen_addi_ptr(t_m
, cpu_env
, pred_full_reg_offset(s
, a
->rm
));
2493 fn(t_d
, t_n
, t_m
, tcg_constant_i32(desc
));
2495 tcg_temp_free_ptr(t_d
);
2496 tcg_temp_free_ptr(t_n
);
2497 tcg_temp_free_ptr(t_m
);
2501 static bool do_perm_pred2(DisasContext
*s
, arg_rr_esz
*a
, bool high_odd
,
2502 gen_helper_gvec_2
*fn
)
2504 if (!sve_access_check(s
)) {
2508 unsigned vsz
= pred_full_reg_size(s
);
2509 TCGv_ptr t_d
= tcg_temp_new_ptr();
2510 TCGv_ptr t_n
= tcg_temp_new_ptr();
2513 tcg_gen_addi_ptr(t_d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2514 tcg_gen_addi_ptr(t_n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2516 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
);
2517 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
2518 desc
= FIELD_DP32(desc
, PREDDESC
, DATA
, high_odd
);
2520 fn(t_d
, t_n
, tcg_constant_i32(desc
));
2522 tcg_temp_free_ptr(t_d
);
2523 tcg_temp_free_ptr(t_n
);
2527 static bool trans_ZIP1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2529 return do_perm_pred3(s
, a
, 0, gen_helper_sve_zip_p
);
2532 static bool trans_ZIP2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2534 return do_perm_pred3(s
, a
, 1, gen_helper_sve_zip_p
);
2537 static bool trans_UZP1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2539 return do_perm_pred3(s
, a
, 0, gen_helper_sve_uzp_p
);
2542 static bool trans_UZP2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2544 return do_perm_pred3(s
, a
, 1, gen_helper_sve_uzp_p
);
2547 static bool trans_TRN1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2549 return do_perm_pred3(s
, a
, 0, gen_helper_sve_trn_p
);
2552 static bool trans_TRN2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2554 return do_perm_pred3(s
, a
, 1, gen_helper_sve_trn_p
);
2557 static bool trans_REV_p(DisasContext
*s
, arg_rr_esz
*a
)
2559 return do_perm_pred2(s
, a
, 0, gen_helper_sve_rev_p
);
2562 static bool trans_PUNPKLO(DisasContext
*s
, arg_PUNPKLO
*a
)
2564 return do_perm_pred2(s
, a
, 0, gen_helper_sve_punpk_p
);
2567 static bool trans_PUNPKHI(DisasContext
*s
, arg_PUNPKHI
*a
)
2569 return do_perm_pred2(s
, a
, 1, gen_helper_sve_punpk_p
);
2573 *** SVE Permute - Interleaving Group
2576 static bool do_zip(DisasContext
*s
, arg_rrr_esz
*a
, bool high
)
2578 static gen_helper_gvec_3
* const fns
[4] = {
2579 gen_helper_sve_zip_b
, gen_helper_sve_zip_h
,
2580 gen_helper_sve_zip_s
, gen_helper_sve_zip_d
,
2583 if (sve_access_check(s
)) {
2584 unsigned vsz
= vec_full_reg_size(s
);
2585 unsigned high_ofs
= high
? vsz
/ 2 : 0;
2586 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, a
->rd
),
2587 vec_full_reg_offset(s
, a
->rn
) + high_ofs
,
2588 vec_full_reg_offset(s
, a
->rm
) + high_ofs
,
2589 vsz
, vsz
, 0, fns
[a
->esz
]);
2594 static bool trans_ZIP1_z(DisasContext
*s
, arg_rrr_esz
*a
)
2596 return do_zip(s
, a
, false);
2599 static bool trans_ZIP2_z(DisasContext
*s
, arg_rrr_esz
*a
)
2601 return do_zip(s
, a
, true);
2604 static bool do_zip_q(DisasContext
*s
, arg_rrr_esz
*a
, bool high
)
2606 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
2609 if (sve_access_check(s
)) {
2610 unsigned vsz
= vec_full_reg_size(s
);
2611 unsigned high_ofs
= high
? QEMU_ALIGN_DOWN(vsz
, 32) / 2 : 0;
2612 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, a
->rd
),
2613 vec_full_reg_offset(s
, a
->rn
) + high_ofs
,
2614 vec_full_reg_offset(s
, a
->rm
) + high_ofs
,
2615 vsz
, vsz
, 0, gen_helper_sve2_zip_q
);
2620 static bool trans_ZIP1_q(DisasContext
*s
, arg_rrr_esz
*a
)
2622 return do_zip_q(s
, a
, false);
2625 static bool trans_ZIP2_q(DisasContext
*s
, arg_rrr_esz
*a
)
2627 return do_zip_q(s
, a
, true);
2630 static gen_helper_gvec_3
* const uzp_fns
[4] = {
2631 gen_helper_sve_uzp_b
, gen_helper_sve_uzp_h
,
2632 gen_helper_sve_uzp_s
, gen_helper_sve_uzp_d
,
2635 TRANS_FEAT(UZP1_z
, aa64_sve
, gen_gvec_ool_arg_zzz
,
2636 uzp_fns
[a
->esz
], a
, 0)
2637 TRANS_FEAT(UZP2_z
, aa64_sve
, gen_gvec_ool_arg_zzz
,
2638 uzp_fns
[a
->esz
], a
, 1 << a
->esz
)
2640 TRANS_FEAT(UZP1_q
, aa64_sve_f64mm
, gen_gvec_ool_arg_zzz
,
2641 gen_helper_sve2_uzp_q
, a
, 0)
2642 TRANS_FEAT(UZP2_q
, aa64_sve_f64mm
, gen_gvec_ool_arg_zzz
,
2643 gen_helper_sve2_uzp_q
, a
, 16)
2645 static gen_helper_gvec_3
* const trn_fns
[4] = {
2646 gen_helper_sve_trn_b
, gen_helper_sve_trn_h
,
2647 gen_helper_sve_trn_s
, gen_helper_sve_trn_d
,
2650 TRANS_FEAT(TRN1_z
, aa64_sve
, gen_gvec_ool_arg_zzz
,
2651 trn_fns
[a
->esz
], a
, 0)
2652 TRANS_FEAT(TRN2_z
, aa64_sve
, gen_gvec_ool_arg_zzz
,
2653 trn_fns
[a
->esz
], a
, 1 << a
->esz
)
2655 TRANS_FEAT(TRN1_q
, aa64_sve_f64mm
, gen_gvec_ool_arg_zzz
,
2656 gen_helper_sve2_trn_q
, a
, 0)
2657 TRANS_FEAT(TRN2_q
, aa64_sve_f64mm
, gen_gvec_ool_arg_zzz
,
2658 gen_helper_sve2_trn_q
, a
, 16)
2661 *** SVE Permute Vector - Predicated Group
2664 static bool trans_COMPACT(DisasContext
*s
, arg_rpr_esz
*a
)
2666 static gen_helper_gvec_3
* const fns
[4] = {
2667 NULL
, NULL
, gen_helper_sve_compact_s
, gen_helper_sve_compact_d
2669 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
2672 /* Call the helper that computes the ARM LastActiveElement pseudocode
2673 * function, scaled by the element size. This includes the not found
2674 * indication; e.g. not found for esz=3 is -8.
2676 static void find_last_active(DisasContext
*s
, TCGv_i32 ret
, int esz
, int pg
)
2678 /* Predicate sizes may be smaller and cannot use simd_desc. We cannot
2679 * round up, as we do elsewhere, because we need the exact size.
2681 TCGv_ptr t_p
= tcg_temp_new_ptr();
2684 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, pred_full_reg_size(s
));
2685 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, esz
);
2687 tcg_gen_addi_ptr(t_p
, cpu_env
, pred_full_reg_offset(s
, pg
));
2689 gen_helper_sve_last_active_element(ret
, t_p
, tcg_constant_i32(desc
));
2691 tcg_temp_free_ptr(t_p
);
2694 /* Increment LAST to the offset of the next element in the vector,
2695 * wrapping around to 0.
2697 static void incr_last_active(DisasContext
*s
, TCGv_i32 last
, int esz
)
2699 unsigned vsz
= vec_full_reg_size(s
);
2701 tcg_gen_addi_i32(last
, last
, 1 << esz
);
2702 if (is_power_of_2(vsz
)) {
2703 tcg_gen_andi_i32(last
, last
, vsz
- 1);
2705 TCGv_i32 max
= tcg_constant_i32(vsz
);
2706 TCGv_i32 zero
= tcg_constant_i32(0);
2707 tcg_gen_movcond_i32(TCG_COND_GEU
, last
, last
, max
, zero
, last
);
2711 /* If LAST < 0, set LAST to the offset of the last element in the vector. */
2712 static void wrap_last_active(DisasContext
*s
, TCGv_i32 last
, int esz
)
2714 unsigned vsz
= vec_full_reg_size(s
);
2716 if (is_power_of_2(vsz
)) {
2717 tcg_gen_andi_i32(last
, last
, vsz
- 1);
2719 TCGv_i32 max
= tcg_constant_i32(vsz
- (1 << esz
));
2720 TCGv_i32 zero
= tcg_constant_i32(0);
2721 tcg_gen_movcond_i32(TCG_COND_LT
, last
, last
, zero
, max
, last
);
2725 /* Load an unsigned element of ESZ from BASE+OFS. */
2726 static TCGv_i64
load_esz(TCGv_ptr base
, int ofs
, int esz
)
2728 TCGv_i64 r
= tcg_temp_new_i64();
2732 tcg_gen_ld8u_i64(r
, base
, ofs
);
2735 tcg_gen_ld16u_i64(r
, base
, ofs
);
2738 tcg_gen_ld32u_i64(r
, base
, ofs
);
2741 tcg_gen_ld_i64(r
, base
, ofs
);
2744 g_assert_not_reached();
2749 /* Load an unsigned element of ESZ from RM[LAST]. */
2750 static TCGv_i64
load_last_active(DisasContext
*s
, TCGv_i32 last
,
2753 TCGv_ptr p
= tcg_temp_new_ptr();
2756 /* Convert offset into vector into offset into ENV.
2757 * The final adjustment for the vector register base
2758 * is added via constant offset to the load.
2761 /* Adjust for element ordering. See vec_reg_offset. */
2763 tcg_gen_xori_i32(last
, last
, 8 - (1 << esz
));
2766 tcg_gen_ext_i32_ptr(p
, last
);
2767 tcg_gen_add_ptr(p
, p
, cpu_env
);
2769 r
= load_esz(p
, vec_full_reg_offset(s
, rm
), esz
);
2770 tcg_temp_free_ptr(p
);
2775 /* Compute CLAST for a Zreg. */
2776 static bool do_clast_vector(DisasContext
*s
, arg_rprr_esz
*a
, bool before
)
2781 unsigned vsz
, esz
= a
->esz
;
2783 if (!sve_access_check(s
)) {
2787 last
= tcg_temp_local_new_i32();
2788 over
= gen_new_label();
2790 find_last_active(s
, last
, esz
, a
->pg
);
2792 /* There is of course no movcond for a 2048-bit vector,
2793 * so we must branch over the actual store.
2795 tcg_gen_brcondi_i32(TCG_COND_LT
, last
, 0, over
);
2798 incr_last_active(s
, last
, esz
);
2801 ele
= load_last_active(s
, last
, a
->rm
, esz
);
2802 tcg_temp_free_i32(last
);
2804 vsz
= vec_full_reg_size(s
);
2805 tcg_gen_gvec_dup_i64(esz
, vec_full_reg_offset(s
, a
->rd
), vsz
, vsz
, ele
);
2806 tcg_temp_free_i64(ele
);
2808 /* If this insn used MOVPRFX, we may need a second move. */
2809 if (a
->rd
!= a
->rn
) {
2810 TCGLabel
*done
= gen_new_label();
2813 gen_set_label(over
);
2814 do_mov_z(s
, a
->rd
, a
->rn
);
2816 gen_set_label(done
);
2818 gen_set_label(over
);
2823 static bool trans_CLASTA_z(DisasContext
*s
, arg_rprr_esz
*a
)
2825 return do_clast_vector(s
, a
, false);
2828 static bool trans_CLASTB_z(DisasContext
*s
, arg_rprr_esz
*a
)
2830 return do_clast_vector(s
, a
, true);
2833 /* Compute CLAST for a scalar. */
2834 static void do_clast_scalar(DisasContext
*s
, int esz
, int pg
, int rm
,
2835 bool before
, TCGv_i64 reg_val
)
2837 TCGv_i32 last
= tcg_temp_new_i32();
2840 find_last_active(s
, last
, esz
, pg
);
2842 /* Extend the original value of last prior to incrementing. */
2843 cmp
= tcg_temp_new_i64();
2844 tcg_gen_ext_i32_i64(cmp
, last
);
2847 incr_last_active(s
, last
, esz
);
2850 /* The conceit here is that while last < 0 indicates not found, after
2851 * adjusting for cpu_env->vfp.zregs[rm], it is still a valid address
2852 * from which we can load garbage. We then discard the garbage with
2853 * a conditional move.
2855 ele
= load_last_active(s
, last
, rm
, esz
);
2856 tcg_temp_free_i32(last
);
2858 tcg_gen_movcond_i64(TCG_COND_GE
, reg_val
, cmp
, tcg_constant_i64(0),
2861 tcg_temp_free_i64(cmp
);
2862 tcg_temp_free_i64(ele
);
2865 /* Compute CLAST for a Vreg. */
2866 static bool do_clast_fp(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2868 if (sve_access_check(s
)) {
2870 int ofs
= vec_reg_offset(s
, a
->rd
, 0, esz
);
2871 TCGv_i64 reg
= load_esz(cpu_env
, ofs
, esz
);
2873 do_clast_scalar(s
, esz
, a
->pg
, a
->rn
, before
, reg
);
2874 write_fp_dreg(s
, a
->rd
, reg
);
2875 tcg_temp_free_i64(reg
);
2880 static bool trans_CLASTA_v(DisasContext
*s
, arg_rpr_esz
*a
)
2882 return do_clast_fp(s
, a
, false);
2885 static bool trans_CLASTB_v(DisasContext
*s
, arg_rpr_esz
*a
)
2887 return do_clast_fp(s
, a
, true);
2890 /* Compute CLAST for a Xreg. */
2891 static bool do_clast_general(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2895 if (!sve_access_check(s
)) {
2899 reg
= cpu_reg(s
, a
->rd
);
2902 tcg_gen_ext8u_i64(reg
, reg
);
2905 tcg_gen_ext16u_i64(reg
, reg
);
2908 tcg_gen_ext32u_i64(reg
, reg
);
2913 g_assert_not_reached();
2916 do_clast_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
, reg
);
2920 static bool trans_CLASTA_r(DisasContext
*s
, arg_rpr_esz
*a
)
2922 return do_clast_general(s
, a
, false);
2925 static bool trans_CLASTB_r(DisasContext
*s
, arg_rpr_esz
*a
)
2927 return do_clast_general(s
, a
, true);
2930 /* Compute LAST for a scalar. */
2931 static TCGv_i64
do_last_scalar(DisasContext
*s
, int esz
,
2932 int pg
, int rm
, bool before
)
2934 TCGv_i32 last
= tcg_temp_new_i32();
2937 find_last_active(s
, last
, esz
, pg
);
2939 wrap_last_active(s
, last
, esz
);
2941 incr_last_active(s
, last
, esz
);
2944 ret
= load_last_active(s
, last
, rm
, esz
);
2945 tcg_temp_free_i32(last
);
2949 /* Compute LAST for a Vreg. */
2950 static bool do_last_fp(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2952 if (sve_access_check(s
)) {
2953 TCGv_i64 val
= do_last_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
);
2954 write_fp_dreg(s
, a
->rd
, val
);
2955 tcg_temp_free_i64(val
);
2960 static bool trans_LASTA_v(DisasContext
*s
, arg_rpr_esz
*a
)
2962 return do_last_fp(s
, a
, false);
2965 static bool trans_LASTB_v(DisasContext
*s
, arg_rpr_esz
*a
)
2967 return do_last_fp(s
, a
, true);
2970 /* Compute LAST for a Xreg. */
2971 static bool do_last_general(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2973 if (sve_access_check(s
)) {
2974 TCGv_i64 val
= do_last_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
);
2975 tcg_gen_mov_i64(cpu_reg(s
, a
->rd
), val
);
2976 tcg_temp_free_i64(val
);
2981 static bool trans_LASTA_r(DisasContext
*s
, arg_rpr_esz
*a
)
2983 return do_last_general(s
, a
, false);
2986 static bool trans_LASTB_r(DisasContext
*s
, arg_rpr_esz
*a
)
2988 return do_last_general(s
, a
, true);
2991 static bool trans_CPY_m_r(DisasContext
*s
, arg_rpr_esz
*a
)
2993 if (sve_access_check(s
)) {
2994 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rd
, a
->pg
, cpu_reg_sp(s
, a
->rn
));
2999 static bool trans_CPY_m_v(DisasContext
*s
, arg_rpr_esz
*a
)
3001 if (sve_access_check(s
)) {
3002 int ofs
= vec_reg_offset(s
, a
->rn
, 0, a
->esz
);
3003 TCGv_i64 t
= load_esz(cpu_env
, ofs
, a
->esz
);
3004 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rd
, a
->pg
, t
);
3005 tcg_temp_free_i64(t
);
3010 static bool trans_REVB(DisasContext
*s
, arg_rpr_esz
*a
)
3012 static gen_helper_gvec_3
* const fns
[4] = {
3014 gen_helper_sve_revb_h
,
3015 gen_helper_sve_revb_s
,
3016 gen_helper_sve_revb_d
,
3018 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
3021 static bool trans_REVH(DisasContext
*s
, arg_rpr_esz
*a
)
3023 static gen_helper_gvec_3
* const fns
[4] = {
3026 gen_helper_sve_revh_s
,
3027 gen_helper_sve_revh_d
,
3029 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
3032 static bool trans_REVW(DisasContext
*s
, arg_rpr_esz
*a
)
3034 return do_zpz_ool(s
, a
, a
->esz
== 3 ? gen_helper_sve_revw_d
: NULL
);
3037 static bool trans_RBIT(DisasContext
*s
, arg_rpr_esz
*a
)
3039 static gen_helper_gvec_3
* const fns
[4] = {
3040 gen_helper_sve_rbit_b
,
3041 gen_helper_sve_rbit_h
,
3042 gen_helper_sve_rbit_s
,
3043 gen_helper_sve_rbit_d
,
3045 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
3048 static bool trans_SPLICE(DisasContext
*s
, arg_rprr_esz
*a
)
3050 if (sve_access_check(s
)) {
3051 gen_gvec_ool_zzzp(s
, gen_helper_sve_splice
,
3052 a
->rd
, a
->rn
, a
->rm
, a
->pg
, a
->esz
);
3057 static bool trans_SPLICE_sve2(DisasContext
*s
, arg_rpr_esz
*a
)
3059 if (!dc_isar_feature(aa64_sve2
, s
)) {
3062 if (sve_access_check(s
)) {
3063 gen_gvec_ool_zzzp(s
, gen_helper_sve_splice
,
3064 a
->rd
, a
->rn
, (a
->rn
+ 1) % 32, a
->pg
, a
->esz
);
3070 *** SVE Integer Compare - Vectors Group
3073 static bool do_ppzz_flags(DisasContext
*s
, arg_rprr_esz
*a
,
3074 gen_helper_gvec_flags_4
*gen_fn
)
3076 TCGv_ptr pd
, zn
, zm
, pg
;
3080 if (gen_fn
== NULL
) {
3083 if (!sve_access_check(s
)) {
3087 vsz
= vec_full_reg_size(s
);
3088 t
= tcg_temp_new_i32();
3089 pd
= tcg_temp_new_ptr();
3090 zn
= tcg_temp_new_ptr();
3091 zm
= tcg_temp_new_ptr();
3092 pg
= tcg_temp_new_ptr();
3094 tcg_gen_addi_ptr(pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3095 tcg_gen_addi_ptr(zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
3096 tcg_gen_addi_ptr(zm
, cpu_env
, vec_full_reg_offset(s
, a
->rm
));
3097 tcg_gen_addi_ptr(pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3099 gen_fn(t
, pd
, zn
, zm
, pg
, tcg_constant_i32(simd_desc(vsz
, vsz
, 0)));
3101 tcg_temp_free_ptr(pd
);
3102 tcg_temp_free_ptr(zn
);
3103 tcg_temp_free_ptr(zm
);
3104 tcg_temp_free_ptr(pg
);
3108 tcg_temp_free_i32(t
);
3112 #define DO_PPZZ(NAME, name) \
3113 static bool trans_##NAME##_ppzz(DisasContext *s, arg_rprr_esz *a) \
3115 static gen_helper_gvec_flags_4 * const fns[4] = { \
3116 gen_helper_sve_##name##_ppzz_b, gen_helper_sve_##name##_ppzz_h, \
3117 gen_helper_sve_##name##_ppzz_s, gen_helper_sve_##name##_ppzz_d, \
3119 return do_ppzz_flags(s, a, fns[a->esz]); \
3122 DO_PPZZ(CMPEQ
, cmpeq
)
3123 DO_PPZZ(CMPNE
, cmpne
)
3124 DO_PPZZ(CMPGT
, cmpgt
)
3125 DO_PPZZ(CMPGE
, cmpge
)
3126 DO_PPZZ(CMPHI
, cmphi
)
3127 DO_PPZZ(CMPHS
, cmphs
)
3131 #define DO_PPZW(NAME, name) \
3132 static bool trans_##NAME##_ppzw(DisasContext *s, arg_rprr_esz *a) \
3134 static gen_helper_gvec_flags_4 * const fns[4] = { \
3135 gen_helper_sve_##name##_ppzw_b, gen_helper_sve_##name##_ppzw_h, \
3136 gen_helper_sve_##name##_ppzw_s, NULL \
3138 return do_ppzz_flags(s, a, fns[a->esz]); \
3141 DO_PPZW(CMPEQ
, cmpeq
)
3142 DO_PPZW(CMPNE
, cmpne
)
3143 DO_PPZW(CMPGT
, cmpgt
)
3144 DO_PPZW(CMPGE
, cmpge
)
3145 DO_PPZW(CMPHI
, cmphi
)
3146 DO_PPZW(CMPHS
, cmphs
)
3147 DO_PPZW(CMPLT
, cmplt
)
3148 DO_PPZW(CMPLE
, cmple
)
3149 DO_PPZW(CMPLO
, cmplo
)
3150 DO_PPZW(CMPLS
, cmpls
)
3155 *** SVE Integer Compare - Immediate Groups
3158 static bool do_ppzi_flags(DisasContext
*s
, arg_rpri_esz
*a
,
3159 gen_helper_gvec_flags_3
*gen_fn
)
3161 TCGv_ptr pd
, zn
, pg
;
3165 if (gen_fn
== NULL
) {
3168 if (!sve_access_check(s
)) {
3172 vsz
= vec_full_reg_size(s
);
3173 t
= tcg_temp_new_i32();
3174 pd
= tcg_temp_new_ptr();
3175 zn
= tcg_temp_new_ptr();
3176 pg
= tcg_temp_new_ptr();
3178 tcg_gen_addi_ptr(pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3179 tcg_gen_addi_ptr(zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
3180 tcg_gen_addi_ptr(pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3182 gen_fn(t
, pd
, zn
, pg
, tcg_constant_i32(simd_desc(vsz
, vsz
, a
->imm
)));
3184 tcg_temp_free_ptr(pd
);
3185 tcg_temp_free_ptr(zn
);
3186 tcg_temp_free_ptr(pg
);
3190 tcg_temp_free_i32(t
);
3194 #define DO_PPZI(NAME, name) \
3195 static bool trans_##NAME##_ppzi(DisasContext *s, arg_rpri_esz *a) \
3197 static gen_helper_gvec_flags_3 * const fns[4] = { \
3198 gen_helper_sve_##name##_ppzi_b, gen_helper_sve_##name##_ppzi_h, \
3199 gen_helper_sve_##name##_ppzi_s, gen_helper_sve_##name##_ppzi_d, \
3201 return do_ppzi_flags(s, a, fns[a->esz]); \
3204 DO_PPZI(CMPEQ
, cmpeq
)
3205 DO_PPZI(CMPNE
, cmpne
)
3206 DO_PPZI(CMPGT
, cmpgt
)
3207 DO_PPZI(CMPGE
, cmpge
)
3208 DO_PPZI(CMPHI
, cmphi
)
3209 DO_PPZI(CMPHS
, cmphs
)
3210 DO_PPZI(CMPLT
, cmplt
)
3211 DO_PPZI(CMPLE
, cmple
)
3212 DO_PPZI(CMPLO
, cmplo
)
3213 DO_PPZI(CMPLS
, cmpls
)
3218 *** SVE Partition Break Group
3221 static bool do_brk3(DisasContext
*s
, arg_rprr_s
*a
,
3222 gen_helper_gvec_4
*fn
, gen_helper_gvec_flags_4
*fn_s
)
3224 if (!sve_access_check(s
)) {
3228 unsigned vsz
= pred_full_reg_size(s
);
3230 /* Predicate sizes may be smaller and cannot use simd_desc. */
3231 TCGv_ptr d
= tcg_temp_new_ptr();
3232 TCGv_ptr n
= tcg_temp_new_ptr();
3233 TCGv_ptr m
= tcg_temp_new_ptr();
3234 TCGv_ptr g
= tcg_temp_new_ptr();
3235 TCGv_i32 desc
= tcg_constant_i32(FIELD_DP32(0, PREDDESC
, OPRSZ
, vsz
));
3237 tcg_gen_addi_ptr(d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3238 tcg_gen_addi_ptr(n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
3239 tcg_gen_addi_ptr(m
, cpu_env
, pred_full_reg_offset(s
, a
->rm
));
3240 tcg_gen_addi_ptr(g
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3243 TCGv_i32 t
= tcg_temp_new_i32();
3244 fn_s(t
, d
, n
, m
, g
, desc
);
3246 tcg_temp_free_i32(t
);
3248 fn(d
, n
, m
, g
, desc
);
3250 tcg_temp_free_ptr(d
);
3251 tcg_temp_free_ptr(n
);
3252 tcg_temp_free_ptr(m
);
3253 tcg_temp_free_ptr(g
);
3257 static bool do_brk2(DisasContext
*s
, arg_rpr_s
*a
,
3258 gen_helper_gvec_3
*fn
, gen_helper_gvec_flags_3
*fn_s
)
3260 if (!sve_access_check(s
)) {
3264 unsigned vsz
= pred_full_reg_size(s
);
3266 /* Predicate sizes may be smaller and cannot use simd_desc. */
3267 TCGv_ptr d
= tcg_temp_new_ptr();
3268 TCGv_ptr n
= tcg_temp_new_ptr();
3269 TCGv_ptr g
= tcg_temp_new_ptr();
3270 TCGv_i32 desc
= tcg_constant_i32(FIELD_DP32(0, PREDDESC
, OPRSZ
, vsz
));
3272 tcg_gen_addi_ptr(d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3273 tcg_gen_addi_ptr(n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
3274 tcg_gen_addi_ptr(g
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3277 TCGv_i32 t
= tcg_temp_new_i32();
3278 fn_s(t
, d
, n
, g
, desc
);
3280 tcg_temp_free_i32(t
);
3284 tcg_temp_free_ptr(d
);
3285 tcg_temp_free_ptr(n
);
3286 tcg_temp_free_ptr(g
);
3290 static bool trans_BRKPA(DisasContext
*s
, arg_rprr_s
*a
)
3292 return do_brk3(s
, a
, gen_helper_sve_brkpa
, gen_helper_sve_brkpas
);
3295 static bool trans_BRKPB(DisasContext
*s
, arg_rprr_s
*a
)
3297 return do_brk3(s
, a
, gen_helper_sve_brkpb
, gen_helper_sve_brkpbs
);
3300 static bool trans_BRKA_m(DisasContext
*s
, arg_rpr_s
*a
)
3302 return do_brk2(s
, a
, gen_helper_sve_brka_m
, gen_helper_sve_brkas_m
);
3305 static bool trans_BRKB_m(DisasContext
*s
, arg_rpr_s
*a
)
3307 return do_brk2(s
, a
, gen_helper_sve_brkb_m
, gen_helper_sve_brkbs_m
);
3310 static bool trans_BRKA_z(DisasContext
*s
, arg_rpr_s
*a
)
3312 return do_brk2(s
, a
, gen_helper_sve_brka_z
, gen_helper_sve_brkas_z
);
3315 static bool trans_BRKB_z(DisasContext
*s
, arg_rpr_s
*a
)
3317 return do_brk2(s
, a
, gen_helper_sve_brkb_z
, gen_helper_sve_brkbs_z
);
3320 static bool trans_BRKN(DisasContext
*s
, arg_rpr_s
*a
)
3322 return do_brk2(s
, a
, gen_helper_sve_brkn
, gen_helper_sve_brkns
);
3326 *** SVE Predicate Count Group
3329 static void do_cntp(DisasContext
*s
, TCGv_i64 val
, int esz
, int pn
, int pg
)
3331 unsigned psz
= pred_full_reg_size(s
);
3336 tcg_gen_ld_i64(val
, cpu_env
, pred_full_reg_offset(s
, pn
));
3338 TCGv_i64 g
= tcg_temp_new_i64();
3339 tcg_gen_ld_i64(g
, cpu_env
, pred_full_reg_offset(s
, pg
));
3340 tcg_gen_and_i64(val
, val
, g
);
3341 tcg_temp_free_i64(g
);
3344 /* Reduce the pred_esz_masks value simply to reduce the
3345 * size of the code generated here.
3347 psz_mask
= MAKE_64BIT_MASK(0, psz
* 8);
3348 tcg_gen_andi_i64(val
, val
, pred_esz_masks
[esz
] & psz_mask
);
3350 tcg_gen_ctpop_i64(val
, val
);
3352 TCGv_ptr t_pn
= tcg_temp_new_ptr();
3353 TCGv_ptr t_pg
= tcg_temp_new_ptr();
3356 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, psz
);
3357 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, esz
);
3359 tcg_gen_addi_ptr(t_pn
, cpu_env
, pred_full_reg_offset(s
, pn
));
3360 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
3362 gen_helper_sve_cntp(val
, t_pn
, t_pg
, tcg_constant_i32(desc
));
3363 tcg_temp_free_ptr(t_pn
);
3364 tcg_temp_free_ptr(t_pg
);
3368 static bool trans_CNTP(DisasContext
*s
, arg_CNTP
*a
)
3370 if (sve_access_check(s
)) {
3371 do_cntp(s
, cpu_reg(s
, a
->rd
), a
->esz
, a
->rn
, a
->pg
);
3376 static bool trans_INCDECP_r(DisasContext
*s
, arg_incdec_pred
*a
)
3378 if (sve_access_check(s
)) {
3379 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3380 TCGv_i64 val
= tcg_temp_new_i64();
3382 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3384 tcg_gen_sub_i64(reg
, reg
, val
);
3386 tcg_gen_add_i64(reg
, reg
, val
);
3388 tcg_temp_free_i64(val
);
3393 static bool trans_INCDECP_z(DisasContext
*s
, arg_incdec2_pred
*a
)
3398 if (sve_access_check(s
)) {
3399 unsigned vsz
= vec_full_reg_size(s
);
3400 TCGv_i64 val
= tcg_temp_new_i64();
3401 GVecGen2sFn
*gvec_fn
= a
->d
? tcg_gen_gvec_subs
: tcg_gen_gvec_adds
;
3403 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3404 gvec_fn(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3405 vec_full_reg_offset(s
, a
->rn
), val
, vsz
, vsz
);
3410 static bool trans_SINCDECP_r_32(DisasContext
*s
, arg_incdec_pred
*a
)
3412 if (sve_access_check(s
)) {
3413 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3414 TCGv_i64 val
= tcg_temp_new_i64();
3416 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3417 do_sat_addsub_32(reg
, val
, a
->u
, a
->d
);
3422 static bool trans_SINCDECP_r_64(DisasContext
*s
, arg_incdec_pred
*a
)
3424 if (sve_access_check(s
)) {
3425 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3426 TCGv_i64 val
= tcg_temp_new_i64();
3428 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3429 do_sat_addsub_64(reg
, val
, a
->u
, a
->d
);
3434 static bool trans_SINCDECP_z(DisasContext
*s
, arg_incdec2_pred
*a
)
3439 if (sve_access_check(s
)) {
3440 TCGv_i64 val
= tcg_temp_new_i64();
3441 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3442 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
, val
, a
->u
, a
->d
);
3448 *** SVE Integer Compare Scalars Group
3451 static bool trans_CTERM(DisasContext
*s
, arg_CTERM
*a
)
3453 if (!sve_access_check(s
)) {
3457 TCGCond cond
= (a
->ne
? TCG_COND_NE
: TCG_COND_EQ
);
3458 TCGv_i64 rn
= read_cpu_reg(s
, a
->rn
, a
->sf
);
3459 TCGv_i64 rm
= read_cpu_reg(s
, a
->rm
, a
->sf
);
3460 TCGv_i64 cmp
= tcg_temp_new_i64();
3462 tcg_gen_setcond_i64(cond
, cmp
, rn
, rm
);
3463 tcg_gen_extrl_i64_i32(cpu_NF
, cmp
);
3464 tcg_temp_free_i64(cmp
);
3466 /* VF = !NF & !CF. */
3467 tcg_gen_xori_i32(cpu_VF
, cpu_NF
, 1);
3468 tcg_gen_andc_i32(cpu_VF
, cpu_VF
, cpu_CF
);
3470 /* Both NF and VF actually look at bit 31. */
3471 tcg_gen_neg_i32(cpu_NF
, cpu_NF
);
3472 tcg_gen_neg_i32(cpu_VF
, cpu_VF
);
3476 static bool trans_WHILE(DisasContext
*s
, arg_WHILE
*a
)
3478 TCGv_i64 op0
, op1
, t0
, t1
, tmax
;
3481 unsigned vsz
= vec_full_reg_size(s
);
3485 /* Note that GE/HS has a->eq == 0 and GT/HI has a->eq == 1. */
3486 bool eq
= a
->eq
== a
->lt
;
3488 /* The greater-than conditions are all SVE2. */
3489 if (!a
->lt
&& !dc_isar_feature(aa64_sve2
, s
)) {
3492 if (!sve_access_check(s
)) {
3496 op0
= read_cpu_reg(s
, a
->rn
, 1);
3497 op1
= read_cpu_reg(s
, a
->rm
, 1);
3501 tcg_gen_ext32u_i64(op0
, op0
);
3502 tcg_gen_ext32u_i64(op1
, op1
);
3504 tcg_gen_ext32s_i64(op0
, op0
);
3505 tcg_gen_ext32s_i64(op1
, op1
);
3509 /* For the helper, compress the different conditions into a computation
3510 * of how many iterations for which the condition is true.
3512 t0
= tcg_temp_new_i64();
3513 t1
= tcg_temp_new_i64();
3516 tcg_gen_sub_i64(t0
, op1
, op0
);
3518 maxval
= a
->sf
? UINT64_MAX
: UINT32_MAX
;
3519 cond
= eq
? TCG_COND_LEU
: TCG_COND_LTU
;
3521 maxval
= a
->sf
? INT64_MAX
: INT32_MAX
;
3522 cond
= eq
? TCG_COND_LE
: TCG_COND_LT
;
3525 tcg_gen_sub_i64(t0
, op0
, op1
);
3528 cond
= eq
? TCG_COND_GEU
: TCG_COND_GTU
;
3530 maxval
= a
->sf
? INT64_MIN
: INT32_MIN
;
3531 cond
= eq
? TCG_COND_GE
: TCG_COND_GT
;
3535 tmax
= tcg_constant_i64(vsz
>> a
->esz
);
3537 /* Equality means one more iteration. */
3538 tcg_gen_addi_i64(t0
, t0
, 1);
3541 * For the less-than while, if op1 is maxval (and the only time
3542 * the addition above could overflow), then we produce an all-true
3543 * predicate by setting the count to the vector length. This is
3544 * because the pseudocode is described as an increment + compare
3545 * loop, and the maximum integer would always compare true.
3546 * Similarly, the greater-than while has the same issue with the
3547 * minimum integer due to the decrement + compare loop.
3549 tcg_gen_movi_i64(t1
, maxval
);
3550 tcg_gen_movcond_i64(TCG_COND_EQ
, t0
, op1
, t1
, tmax
, t0
);
3553 /* Bound to the maximum. */
3554 tcg_gen_umin_i64(t0
, t0
, tmax
);
3556 /* Set the count to zero if the condition is false. */
3557 tcg_gen_movi_i64(t1
, 0);
3558 tcg_gen_movcond_i64(cond
, t0
, op0
, op1
, t0
, t1
);
3559 tcg_temp_free_i64(t1
);
3561 /* Since we're bounded, pass as a 32-bit type. */
3562 t2
= tcg_temp_new_i32();
3563 tcg_gen_extrl_i64_i32(t2
, t0
);
3564 tcg_temp_free_i64(t0
);
3566 /* Scale elements to bits. */
3567 tcg_gen_shli_i32(t2
, t2
, a
->esz
);
3569 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
/ 8);
3570 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
3572 ptr
= tcg_temp_new_ptr();
3573 tcg_gen_addi_ptr(ptr
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3576 gen_helper_sve_whilel(t2
, ptr
, t2
, tcg_constant_i32(desc
));
3578 gen_helper_sve_whileg(t2
, ptr
, t2
, tcg_constant_i32(desc
));
3582 tcg_temp_free_ptr(ptr
);
3583 tcg_temp_free_i32(t2
);
3587 static bool trans_WHILE_ptr(DisasContext
*s
, arg_WHILE_ptr
*a
)
3589 TCGv_i64 op0
, op1
, diff
, t1
, tmax
;
3592 unsigned vsz
= vec_full_reg_size(s
);
3595 if (!dc_isar_feature(aa64_sve2
, s
)) {
3598 if (!sve_access_check(s
)) {
3602 op0
= read_cpu_reg(s
, a
->rn
, 1);
3603 op1
= read_cpu_reg(s
, a
->rm
, 1);
3605 tmax
= tcg_constant_i64(vsz
);
3606 diff
= tcg_temp_new_i64();
3610 /* diff = abs(op1 - op0), noting that op0/1 are unsigned. */
3611 t1
= tcg_temp_new_i64();
3612 tcg_gen_sub_i64(diff
, op0
, op1
);
3613 tcg_gen_sub_i64(t1
, op1
, op0
);
3614 tcg_gen_movcond_i64(TCG_COND_GEU
, diff
, op0
, op1
, diff
, t1
);
3615 tcg_temp_free_i64(t1
);
3616 /* Round down to a multiple of ESIZE. */
3617 tcg_gen_andi_i64(diff
, diff
, -1 << a
->esz
);
3618 /* If op1 == op0, diff == 0, and the condition is always true. */
3619 tcg_gen_movcond_i64(TCG_COND_EQ
, diff
, op0
, op1
, tmax
, diff
);
3622 tcg_gen_sub_i64(diff
, op1
, op0
);
3623 /* Round down to a multiple of ESIZE. */
3624 tcg_gen_andi_i64(diff
, diff
, -1 << a
->esz
);
3625 /* If op0 >= op1, diff <= 0, the condition is always true. */
3626 tcg_gen_movcond_i64(TCG_COND_GEU
, diff
, op0
, op1
, tmax
, diff
);
3629 /* Bound to the maximum. */
3630 tcg_gen_umin_i64(diff
, diff
, tmax
);
3632 /* Since we're bounded, pass as a 32-bit type. */
3633 t2
= tcg_temp_new_i32();
3634 tcg_gen_extrl_i64_i32(t2
, diff
);
3635 tcg_temp_free_i64(diff
);
3637 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
/ 8);
3638 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
3640 ptr
= tcg_temp_new_ptr();
3641 tcg_gen_addi_ptr(ptr
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3643 gen_helper_sve_whilel(t2
, ptr
, t2
, tcg_constant_i32(desc
));
3646 tcg_temp_free_ptr(ptr
);
3647 tcg_temp_free_i32(t2
);
3652 *** SVE Integer Wide Immediate - Unpredicated Group
3655 static bool trans_FDUP(DisasContext
*s
, arg_FDUP
*a
)
3660 if (sve_access_check(s
)) {
3661 unsigned vsz
= vec_full_reg_size(s
);
3662 int dofs
= vec_full_reg_offset(s
, a
->rd
);
3665 /* Decode the VFP immediate. */
3666 imm
= vfp_expand_imm(a
->esz
, a
->imm
);
3667 tcg_gen_gvec_dup_imm(a
->esz
, dofs
, vsz
, vsz
, imm
);
3672 static bool trans_DUP_i(DisasContext
*s
, arg_DUP_i
*a
)
3674 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3677 if (sve_access_check(s
)) {
3678 unsigned vsz
= vec_full_reg_size(s
);
3679 int dofs
= vec_full_reg_offset(s
, a
->rd
);
3681 tcg_gen_gvec_dup_imm(a
->esz
, dofs
, vsz
, vsz
, a
->imm
);
3686 static bool trans_ADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3688 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3691 if (sve_access_check(s
)) {
3692 unsigned vsz
= vec_full_reg_size(s
);
3693 tcg_gen_gvec_addi(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3694 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
3699 static bool trans_SUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3702 return trans_ADD_zzi(s
, a
);
3705 static bool trans_SUBR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3707 static const TCGOpcode vecop_list
[] = { INDEX_op_sub_vec
, 0 };
3708 static const GVecGen2s op
[4] = {
3709 { .fni8
= tcg_gen_vec_sub8_i64
,
3710 .fniv
= tcg_gen_sub_vec
,
3711 .fno
= gen_helper_sve_subri_b
,
3712 .opt_opc
= vecop_list
,
3714 .scalar_first
= true },
3715 { .fni8
= tcg_gen_vec_sub16_i64
,
3716 .fniv
= tcg_gen_sub_vec
,
3717 .fno
= gen_helper_sve_subri_h
,
3718 .opt_opc
= vecop_list
,
3720 .scalar_first
= true },
3721 { .fni4
= tcg_gen_sub_i32
,
3722 .fniv
= tcg_gen_sub_vec
,
3723 .fno
= gen_helper_sve_subri_s
,
3724 .opt_opc
= vecop_list
,
3726 .scalar_first
= true },
3727 { .fni8
= tcg_gen_sub_i64
,
3728 .fniv
= tcg_gen_sub_vec
,
3729 .fno
= gen_helper_sve_subri_d
,
3730 .opt_opc
= vecop_list
,
3731 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
3733 .scalar_first
= true }
3736 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3739 if (sve_access_check(s
)) {
3740 unsigned vsz
= vec_full_reg_size(s
);
3741 tcg_gen_gvec_2s(vec_full_reg_offset(s
, a
->rd
),
3742 vec_full_reg_offset(s
, a
->rn
),
3743 vsz
, vsz
, tcg_constant_i64(a
->imm
), &op
[a
->esz
]);
3748 static bool trans_MUL_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3750 if (sve_access_check(s
)) {
3751 unsigned vsz
= vec_full_reg_size(s
);
3752 tcg_gen_gvec_muli(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3753 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
3758 static bool do_zzi_sat(DisasContext
*s
, arg_rri_esz
*a
, bool u
, bool d
)
3760 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3763 if (sve_access_check(s
)) {
3764 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
,
3765 tcg_constant_i64(a
->imm
), u
, d
);
3770 static bool trans_SQADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3772 return do_zzi_sat(s
, a
, false, false);
3775 static bool trans_UQADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3777 return do_zzi_sat(s
, a
, true, false);
3780 static bool trans_SQSUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3782 return do_zzi_sat(s
, a
, false, true);
3785 static bool trans_UQSUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3787 return do_zzi_sat(s
, a
, true, true);
3790 static bool do_zzi_ool(DisasContext
*s
, arg_rri_esz
*a
, gen_helper_gvec_2i
*fn
)
3792 if (sve_access_check(s
)) {
3793 unsigned vsz
= vec_full_reg_size(s
);
3794 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s
, a
->rd
),
3795 vec_full_reg_offset(s
, a
->rn
),
3796 tcg_constant_i64(a
->imm
), vsz
, vsz
, 0, fn
);
3801 #define DO_ZZI(NAME, name) \
3802 static bool trans_##NAME##_zzi(DisasContext *s, arg_rri_esz *a) \
3804 static gen_helper_gvec_2i * const fns[4] = { \
3805 gen_helper_sve_##name##i_b, gen_helper_sve_##name##i_h, \
3806 gen_helper_sve_##name##i_s, gen_helper_sve_##name##i_d, \
3808 return do_zzi_ool(s, a, fns[a->esz]); \
3818 static gen_helper_gvec_4
* const dot_fns
[2][2] = {
3819 { gen_helper_gvec_sdot_b
, gen_helper_gvec_sdot_h
},
3820 { gen_helper_gvec_udot_b
, gen_helper_gvec_udot_h
}
3822 TRANS_FEAT(DOT_zzzz
, aa64_sve
, gen_gvec_ool_zzzz
,
3823 dot_fns
[a
->u
][a
->sz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, 0)
3826 * SVE Multiply - Indexed
3829 TRANS_FEAT(SDOT_zzxw_s
, aa64_sve
, gen_gvec_ool_arg_zzxz
,
3830 gen_helper_gvec_sdot_idx_b
, a
)
3831 TRANS_FEAT(SDOT_zzxw_d
, aa64_sve
, gen_gvec_ool_arg_zzxz
,
3832 gen_helper_gvec_sdot_idx_h
, a
)
3833 TRANS_FEAT(UDOT_zzxw_s
, aa64_sve
, gen_gvec_ool_arg_zzxz
,
3834 gen_helper_gvec_udot_idx_b
, a
)
3835 TRANS_FEAT(UDOT_zzxw_d
, aa64_sve
, gen_gvec_ool_arg_zzxz
,
3836 gen_helper_gvec_udot_idx_h
, a
)
3838 TRANS_FEAT(SUDOT_zzxw_s
, aa64_sve_i8mm
, gen_gvec_ool_arg_zzxz
,
3839 gen_helper_gvec_sudot_idx_b
, a
)
3840 TRANS_FEAT(USDOT_zzxw_s
, aa64_sve_i8mm
, gen_gvec_ool_arg_zzxz
,
3841 gen_helper_gvec_usdot_idx_b
, a
)
3843 #define DO_SVE2_RRX(NAME, FUNC) \
3844 TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_zzz, FUNC, \
3845 a->rd, a->rn, a->rm, a->index)
3847 DO_SVE2_RRX(MUL_zzx_h
, gen_helper_gvec_mul_idx_h
)
3848 DO_SVE2_RRX(MUL_zzx_s
, gen_helper_gvec_mul_idx_s
)
3849 DO_SVE2_RRX(MUL_zzx_d
, gen_helper_gvec_mul_idx_d
)
3851 DO_SVE2_RRX(SQDMULH_zzx_h
, gen_helper_sve2_sqdmulh_idx_h
)
3852 DO_SVE2_RRX(SQDMULH_zzx_s
, gen_helper_sve2_sqdmulh_idx_s
)
3853 DO_SVE2_RRX(SQDMULH_zzx_d
, gen_helper_sve2_sqdmulh_idx_d
)
3855 DO_SVE2_RRX(SQRDMULH_zzx_h
, gen_helper_sve2_sqrdmulh_idx_h
)
3856 DO_SVE2_RRX(SQRDMULH_zzx_s
, gen_helper_sve2_sqrdmulh_idx_s
)
3857 DO_SVE2_RRX(SQRDMULH_zzx_d
, gen_helper_sve2_sqrdmulh_idx_d
)
3861 #define DO_SVE2_RRX_TB(NAME, FUNC, TOP) \
3862 TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_zzz, FUNC, \
3863 a->rd, a->rn, a->rm, (a->index << 1) | TOP)
3865 DO_SVE2_RRX_TB(SQDMULLB_zzx_s
, gen_helper_sve2_sqdmull_idx_s
, false)
3866 DO_SVE2_RRX_TB(SQDMULLB_zzx_d
, gen_helper_sve2_sqdmull_idx_d
, false)
3867 DO_SVE2_RRX_TB(SQDMULLT_zzx_s
, gen_helper_sve2_sqdmull_idx_s
, true)
3868 DO_SVE2_RRX_TB(SQDMULLT_zzx_d
, gen_helper_sve2_sqdmull_idx_d
, true)
3870 DO_SVE2_RRX_TB(SMULLB_zzx_s
, gen_helper_sve2_smull_idx_s
, false)
3871 DO_SVE2_RRX_TB(SMULLB_zzx_d
, gen_helper_sve2_smull_idx_d
, false)
3872 DO_SVE2_RRX_TB(SMULLT_zzx_s
, gen_helper_sve2_smull_idx_s
, true)
3873 DO_SVE2_RRX_TB(SMULLT_zzx_d
, gen_helper_sve2_smull_idx_d
, true)
3875 DO_SVE2_RRX_TB(UMULLB_zzx_s
, gen_helper_sve2_umull_idx_s
, false)
3876 DO_SVE2_RRX_TB(UMULLB_zzx_d
, gen_helper_sve2_umull_idx_d
, false)
3877 DO_SVE2_RRX_TB(UMULLT_zzx_s
, gen_helper_sve2_umull_idx_s
, true)
3878 DO_SVE2_RRX_TB(UMULLT_zzx_d
, gen_helper_sve2_umull_idx_d
, true)
3880 #undef DO_SVE2_RRX_TB
3882 #define DO_SVE2_RRXR(NAME, FUNC) \
3883 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_arg_zzxz, FUNC, a)
3885 DO_SVE2_RRXR(MLA_zzxz_h
, gen_helper_gvec_mla_idx_h
)
3886 DO_SVE2_RRXR(MLA_zzxz_s
, gen_helper_gvec_mla_idx_s
)
3887 DO_SVE2_RRXR(MLA_zzxz_d
, gen_helper_gvec_mla_idx_d
)
3889 DO_SVE2_RRXR(MLS_zzxz_h
, gen_helper_gvec_mls_idx_h
)
3890 DO_SVE2_RRXR(MLS_zzxz_s
, gen_helper_gvec_mls_idx_s
)
3891 DO_SVE2_RRXR(MLS_zzxz_d
, gen_helper_gvec_mls_idx_d
)
3893 DO_SVE2_RRXR(SQRDMLAH_zzxz_h
, gen_helper_sve2_sqrdmlah_idx_h
)
3894 DO_SVE2_RRXR(SQRDMLAH_zzxz_s
, gen_helper_sve2_sqrdmlah_idx_s
)
3895 DO_SVE2_RRXR(SQRDMLAH_zzxz_d
, gen_helper_sve2_sqrdmlah_idx_d
)
3897 DO_SVE2_RRXR(SQRDMLSH_zzxz_h
, gen_helper_sve2_sqrdmlsh_idx_h
)
3898 DO_SVE2_RRXR(SQRDMLSH_zzxz_s
, gen_helper_sve2_sqrdmlsh_idx_s
)
3899 DO_SVE2_RRXR(SQRDMLSH_zzxz_d
, gen_helper_sve2_sqrdmlsh_idx_d
)
3903 #define DO_SVE2_RRXR_TB(NAME, FUNC, TOP) \
3904 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_zzzz, FUNC, \
3905 a->rd, a->rn, a->rm, a->ra, (a->index << 1) | TOP)
3907 DO_SVE2_RRXR_TB(SQDMLALB_zzxw_s
, gen_helper_sve2_sqdmlal_idx_s
, false)
3908 DO_SVE2_RRXR_TB(SQDMLALB_zzxw_d
, gen_helper_sve2_sqdmlal_idx_d
, false)
3909 DO_SVE2_RRXR_TB(SQDMLALT_zzxw_s
, gen_helper_sve2_sqdmlal_idx_s
, true)
3910 DO_SVE2_RRXR_TB(SQDMLALT_zzxw_d
, gen_helper_sve2_sqdmlal_idx_d
, true)
3912 DO_SVE2_RRXR_TB(SQDMLSLB_zzxw_s
, gen_helper_sve2_sqdmlsl_idx_s
, false)
3913 DO_SVE2_RRXR_TB(SQDMLSLB_zzxw_d
, gen_helper_sve2_sqdmlsl_idx_d
, false)
3914 DO_SVE2_RRXR_TB(SQDMLSLT_zzxw_s
, gen_helper_sve2_sqdmlsl_idx_s
, true)
3915 DO_SVE2_RRXR_TB(SQDMLSLT_zzxw_d
, gen_helper_sve2_sqdmlsl_idx_d
, true)
3917 DO_SVE2_RRXR_TB(SMLALB_zzxw_s
, gen_helper_sve2_smlal_idx_s
, false)
3918 DO_SVE2_RRXR_TB(SMLALB_zzxw_d
, gen_helper_sve2_smlal_idx_d
, false)
3919 DO_SVE2_RRXR_TB(SMLALT_zzxw_s
, gen_helper_sve2_smlal_idx_s
, true)
3920 DO_SVE2_RRXR_TB(SMLALT_zzxw_d
, gen_helper_sve2_smlal_idx_d
, true)
3922 DO_SVE2_RRXR_TB(UMLALB_zzxw_s
, gen_helper_sve2_umlal_idx_s
, false)
3923 DO_SVE2_RRXR_TB(UMLALB_zzxw_d
, gen_helper_sve2_umlal_idx_d
, false)
3924 DO_SVE2_RRXR_TB(UMLALT_zzxw_s
, gen_helper_sve2_umlal_idx_s
, true)
3925 DO_SVE2_RRXR_TB(UMLALT_zzxw_d
, gen_helper_sve2_umlal_idx_d
, true)
3927 DO_SVE2_RRXR_TB(SMLSLB_zzxw_s
, gen_helper_sve2_smlsl_idx_s
, false)
3928 DO_SVE2_RRXR_TB(SMLSLB_zzxw_d
, gen_helper_sve2_smlsl_idx_d
, false)
3929 DO_SVE2_RRXR_TB(SMLSLT_zzxw_s
, gen_helper_sve2_smlsl_idx_s
, true)
3930 DO_SVE2_RRXR_TB(SMLSLT_zzxw_d
, gen_helper_sve2_smlsl_idx_d
, true)
3932 DO_SVE2_RRXR_TB(UMLSLB_zzxw_s
, gen_helper_sve2_umlsl_idx_s
, false)
3933 DO_SVE2_RRXR_TB(UMLSLB_zzxw_d
, gen_helper_sve2_umlsl_idx_d
, false)
3934 DO_SVE2_RRXR_TB(UMLSLT_zzxw_s
, gen_helper_sve2_umlsl_idx_s
, true)
3935 DO_SVE2_RRXR_TB(UMLSLT_zzxw_d
, gen_helper_sve2_umlsl_idx_d
, true)
3937 #undef DO_SVE2_RRXR_TB
3939 #define DO_SVE2_RRXR_ROT(NAME, FUNC) \
3940 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_zzzz, FUNC, \
3941 a->rd, a->rn, a->rm, a->ra, (a->index << 2) | a->rot)
3943 DO_SVE2_RRXR_ROT(CMLA_zzxz_h
, gen_helper_sve2_cmla_idx_h
)
3944 DO_SVE2_RRXR_ROT(CMLA_zzxz_s
, gen_helper_sve2_cmla_idx_s
)
3946 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_h
, gen_helper_sve2_sqrdcmlah_idx_h
)
3947 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_s
, gen_helper_sve2_sqrdcmlah_idx_s
)
3949 DO_SVE2_RRXR_ROT(CDOT_zzxw_s
, gen_helper_sve2_cdot_idx_s
)
3950 DO_SVE2_RRXR_ROT(CDOT_zzxw_d
, gen_helper_sve2_cdot_idx_d
)
3952 #undef DO_SVE2_RRXR_ROT
3955 *** SVE Floating Point Multiply-Add Indexed Group
3958 static bool do_FMLA_zzxz(DisasContext
*s
, arg_rrxr_esz
*a
, bool sub
)
3960 static gen_helper_gvec_4_ptr
* const fns
[3] = {
3961 gen_helper_gvec_fmla_idx_h
,
3962 gen_helper_gvec_fmla_idx_s
,
3963 gen_helper_gvec_fmla_idx_d
,
3966 if (sve_access_check(s
)) {
3967 unsigned vsz
= vec_full_reg_size(s
);
3968 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3969 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
3970 vec_full_reg_offset(s
, a
->rn
),
3971 vec_full_reg_offset(s
, a
->rm
),
3972 vec_full_reg_offset(s
, a
->ra
),
3973 status
, vsz
, vsz
, (a
->index
<< 1) | sub
,
3975 tcg_temp_free_ptr(status
);
3980 static bool trans_FMLA_zzxz(DisasContext
*s
, arg_FMLA_zzxz
*a
)
3982 return do_FMLA_zzxz(s
, a
, false);
3985 static bool trans_FMLS_zzxz(DisasContext
*s
, arg_FMLA_zzxz
*a
)
3987 return do_FMLA_zzxz(s
, a
, true);
3991 *** SVE Floating Point Multiply Indexed Group
3994 static bool trans_FMUL_zzx(DisasContext
*s
, arg_FMUL_zzx
*a
)
3996 static gen_helper_gvec_3_ptr
* const fns
[3] = {
3997 gen_helper_gvec_fmul_idx_h
,
3998 gen_helper_gvec_fmul_idx_s
,
3999 gen_helper_gvec_fmul_idx_d
,
4002 if (sve_access_check(s
)) {
4003 unsigned vsz
= vec_full_reg_size(s
);
4004 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4005 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4006 vec_full_reg_offset(s
, a
->rn
),
4007 vec_full_reg_offset(s
, a
->rm
),
4008 status
, vsz
, vsz
, a
->index
, fns
[a
->esz
- 1]);
4009 tcg_temp_free_ptr(status
);
4015 *** SVE Floating Point Fast Reduction Group
4018 typedef void gen_helper_fp_reduce(TCGv_i64
, TCGv_ptr
, TCGv_ptr
,
4019 TCGv_ptr
, TCGv_i32
);
4021 static void do_reduce(DisasContext
*s
, arg_rpr_esz
*a
,
4022 gen_helper_fp_reduce
*fn
)
4024 unsigned vsz
= vec_full_reg_size(s
);
4025 unsigned p2vsz
= pow2ceil(vsz
);
4026 TCGv_i32 t_desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, p2vsz
));
4027 TCGv_ptr t_zn
, t_pg
, status
;
4030 temp
= tcg_temp_new_i64();
4031 t_zn
= tcg_temp_new_ptr();
4032 t_pg
= tcg_temp_new_ptr();
4034 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
4035 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
4036 status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4038 fn(temp
, t_zn
, t_pg
, status
, t_desc
);
4039 tcg_temp_free_ptr(t_zn
);
4040 tcg_temp_free_ptr(t_pg
);
4041 tcg_temp_free_ptr(status
);
4043 write_fp_dreg(s
, a
->rd
, temp
);
4044 tcg_temp_free_i64(temp
);
4047 #define DO_VPZ(NAME, name) \
4048 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
4050 static gen_helper_fp_reduce * const fns[3] = { \
4051 gen_helper_sve_##name##_h, \
4052 gen_helper_sve_##name##_s, \
4053 gen_helper_sve_##name##_d, \
4055 if (a->esz == 0) { \
4058 if (sve_access_check(s)) { \
4059 do_reduce(s, a, fns[a->esz - 1]); \
4064 DO_VPZ(FADDV
, faddv
)
4065 DO_VPZ(FMINNMV
, fminnmv
)
4066 DO_VPZ(FMAXNMV
, fmaxnmv
)
4067 DO_VPZ(FMINV
, fminv
)
4068 DO_VPZ(FMAXV
, fmaxv
)
4071 *** SVE Floating Point Unary Operations - Unpredicated Group
4074 static void do_zz_fp(DisasContext
*s
, arg_rr_esz
*a
, gen_helper_gvec_2_ptr
*fn
)
4076 unsigned vsz
= vec_full_reg_size(s
);
4077 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4079 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s
, a
->rd
),
4080 vec_full_reg_offset(s
, a
->rn
),
4081 status
, vsz
, vsz
, 0, fn
);
4082 tcg_temp_free_ptr(status
);
4085 static bool trans_FRECPE(DisasContext
*s
, arg_rr_esz
*a
)
4087 static gen_helper_gvec_2_ptr
* const fns
[3] = {
4088 gen_helper_gvec_frecpe_h
,
4089 gen_helper_gvec_frecpe_s
,
4090 gen_helper_gvec_frecpe_d
,
4095 if (sve_access_check(s
)) {
4096 do_zz_fp(s
, a
, fns
[a
->esz
- 1]);
4101 static bool trans_FRSQRTE(DisasContext
*s
, arg_rr_esz
*a
)
4103 static gen_helper_gvec_2_ptr
* const fns
[3] = {
4104 gen_helper_gvec_frsqrte_h
,
4105 gen_helper_gvec_frsqrte_s
,
4106 gen_helper_gvec_frsqrte_d
,
4111 if (sve_access_check(s
)) {
4112 do_zz_fp(s
, a
, fns
[a
->esz
- 1]);
4118 *** SVE Floating Point Compare with Zero Group
4121 static void do_ppz_fp(DisasContext
*s
, arg_rpr_esz
*a
,
4122 gen_helper_gvec_3_ptr
*fn
)
4124 unsigned vsz
= vec_full_reg_size(s
);
4125 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4127 tcg_gen_gvec_3_ptr(pred_full_reg_offset(s
, a
->rd
),
4128 vec_full_reg_offset(s
, a
->rn
),
4129 pred_full_reg_offset(s
, a
->pg
),
4130 status
, vsz
, vsz
, 0, fn
);
4131 tcg_temp_free_ptr(status
);
4134 #define DO_PPZ(NAME, name) \
4135 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
4137 static gen_helper_gvec_3_ptr * const fns[3] = { \
4138 gen_helper_sve_##name##_h, \
4139 gen_helper_sve_##name##_s, \
4140 gen_helper_sve_##name##_d, \
4142 if (a->esz == 0) { \
4145 if (sve_access_check(s)) { \
4146 do_ppz_fp(s, a, fns[a->esz - 1]); \
4151 DO_PPZ(FCMGE_ppz0
, fcmge0
)
4152 DO_PPZ(FCMGT_ppz0
, fcmgt0
)
4153 DO_PPZ(FCMLE_ppz0
, fcmle0
)
4154 DO_PPZ(FCMLT_ppz0
, fcmlt0
)
4155 DO_PPZ(FCMEQ_ppz0
, fcmeq0
)
4156 DO_PPZ(FCMNE_ppz0
, fcmne0
)
4161 *** SVE floating-point trig multiply-add coefficient
4164 static bool trans_FTMAD(DisasContext
*s
, arg_FTMAD
*a
)
4166 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4167 gen_helper_sve_ftmad_h
,
4168 gen_helper_sve_ftmad_s
,
4169 gen_helper_sve_ftmad_d
,
4175 if (sve_access_check(s
)) {
4176 unsigned vsz
= vec_full_reg_size(s
);
4177 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4178 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4179 vec_full_reg_offset(s
, a
->rn
),
4180 vec_full_reg_offset(s
, a
->rm
),
4181 status
, vsz
, vsz
, a
->imm
, fns
[a
->esz
- 1]);
4182 tcg_temp_free_ptr(status
);
4188 *** SVE Floating Point Accumulating Reduction Group
4191 static bool trans_FADDA(DisasContext
*s
, arg_rprr_esz
*a
)
4193 typedef void fadda_fn(TCGv_i64
, TCGv_i64
, TCGv_ptr
,
4194 TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
4195 static fadda_fn
* const fns
[3] = {
4196 gen_helper_sve_fadda_h
,
4197 gen_helper_sve_fadda_s
,
4198 gen_helper_sve_fadda_d
,
4200 unsigned vsz
= vec_full_reg_size(s
);
4201 TCGv_ptr t_rm
, t_pg
, t_fpst
;
4208 if (!sve_access_check(s
)) {
4212 t_val
= load_esz(cpu_env
, vec_reg_offset(s
, a
->rn
, 0, a
->esz
), a
->esz
);
4213 t_rm
= tcg_temp_new_ptr();
4214 t_pg
= tcg_temp_new_ptr();
4215 tcg_gen_addi_ptr(t_rm
, cpu_env
, vec_full_reg_offset(s
, a
->rm
));
4216 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
4217 t_fpst
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4218 t_desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
4220 fns
[a
->esz
- 1](t_val
, t_val
, t_rm
, t_pg
, t_fpst
, t_desc
);
4222 tcg_temp_free_ptr(t_fpst
);
4223 tcg_temp_free_ptr(t_pg
);
4224 tcg_temp_free_ptr(t_rm
);
4226 write_fp_dreg(s
, a
->rd
, t_val
);
4227 tcg_temp_free_i64(t_val
);
4232 *** SVE Floating Point Arithmetic - Unpredicated Group
4235 static bool do_zzz_fp(DisasContext
*s
, arg_rrr_esz
*a
,
4236 gen_helper_gvec_3_ptr
*fn
)
4241 if (sve_access_check(s
)) {
4242 unsigned vsz
= vec_full_reg_size(s
);
4243 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4244 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4245 vec_full_reg_offset(s
, a
->rn
),
4246 vec_full_reg_offset(s
, a
->rm
),
4247 status
, vsz
, vsz
, 0, fn
);
4248 tcg_temp_free_ptr(status
);
4254 #define DO_FP3(NAME, name) \
4255 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
4257 static gen_helper_gvec_3_ptr * const fns[4] = { \
4258 NULL, gen_helper_gvec_##name##_h, \
4259 gen_helper_gvec_##name##_s, gen_helper_gvec_##name##_d \
4261 return do_zzz_fp(s, a, fns[a->esz]); \
4264 DO_FP3(FADD_zzz
, fadd
)
4265 DO_FP3(FSUB_zzz
, fsub
)
4266 DO_FP3(FMUL_zzz
, fmul
)
4267 DO_FP3(FTSMUL
, ftsmul
)
4268 DO_FP3(FRECPS
, recps
)
4269 DO_FP3(FRSQRTS
, rsqrts
)
4274 *** SVE Floating Point Arithmetic - Predicated Group
4277 static bool do_zpzz_fp(DisasContext
*s
, arg_rprr_esz
*a
,
4278 gen_helper_gvec_4_ptr
*fn
)
4283 if (sve_access_check(s
)) {
4284 unsigned vsz
= vec_full_reg_size(s
);
4285 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4286 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4287 vec_full_reg_offset(s
, a
->rn
),
4288 vec_full_reg_offset(s
, a
->rm
),
4289 pred_full_reg_offset(s
, a
->pg
),
4290 status
, vsz
, vsz
, 0, fn
);
4291 tcg_temp_free_ptr(status
);
4296 #define DO_FP3(NAME, name) \
4297 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
4299 static gen_helper_gvec_4_ptr * const fns[4] = { \
4300 NULL, gen_helper_sve_##name##_h, \
4301 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4303 return do_zpzz_fp(s, a, fns[a->esz]); \
4306 DO_FP3(FADD_zpzz
, fadd
)
4307 DO_FP3(FSUB_zpzz
, fsub
)
4308 DO_FP3(FMUL_zpzz
, fmul
)
4309 DO_FP3(FMIN_zpzz
, fmin
)
4310 DO_FP3(FMAX_zpzz
, fmax
)
4311 DO_FP3(FMINNM_zpzz
, fminnum
)
4312 DO_FP3(FMAXNM_zpzz
, fmaxnum
)
4314 DO_FP3(FSCALE
, fscalbn
)
4316 DO_FP3(FMULX
, fmulx
)
4320 typedef void gen_helper_sve_fp2scalar(TCGv_ptr
, TCGv_ptr
, TCGv_ptr
,
4321 TCGv_i64
, TCGv_ptr
, TCGv_i32
);
4323 static void do_fp_scalar(DisasContext
*s
, int zd
, int zn
, int pg
, bool is_fp16
,
4324 TCGv_i64 scalar
, gen_helper_sve_fp2scalar
*fn
)
4326 unsigned vsz
= vec_full_reg_size(s
);
4327 TCGv_ptr t_zd
, t_zn
, t_pg
, status
;
4330 t_zd
= tcg_temp_new_ptr();
4331 t_zn
= tcg_temp_new_ptr();
4332 t_pg
= tcg_temp_new_ptr();
4333 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, zd
));
4334 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, zn
));
4335 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
4337 status
= fpstatus_ptr(is_fp16
? FPST_FPCR_F16
: FPST_FPCR
);
4338 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
4339 fn(t_zd
, t_zn
, t_pg
, scalar
, status
, desc
);
4341 tcg_temp_free_ptr(status
);
4342 tcg_temp_free_ptr(t_pg
);
4343 tcg_temp_free_ptr(t_zn
);
4344 tcg_temp_free_ptr(t_zd
);
4347 static void do_fp_imm(DisasContext
*s
, arg_rpri_esz
*a
, uint64_t imm
,
4348 gen_helper_sve_fp2scalar
*fn
)
4350 do_fp_scalar(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
,
4351 tcg_constant_i64(imm
), fn
);
4354 #define DO_FP_IMM(NAME, name, const0, const1) \
4355 static bool trans_##NAME##_zpzi(DisasContext *s, arg_rpri_esz *a) \
4357 static gen_helper_sve_fp2scalar * const fns[3] = { \
4358 gen_helper_sve_##name##_h, \
4359 gen_helper_sve_##name##_s, \
4360 gen_helper_sve_##name##_d \
4362 static uint64_t const val[3][2] = { \
4363 { float16_##const0, float16_##const1 }, \
4364 { float32_##const0, float32_##const1 }, \
4365 { float64_##const0, float64_##const1 }, \
4367 if (a->esz == 0) { \
4370 if (sve_access_check(s)) { \
4371 do_fp_imm(s, a, val[a->esz - 1][a->imm], fns[a->esz - 1]); \
4376 DO_FP_IMM(FADD
, fadds
, half
, one
)
4377 DO_FP_IMM(FSUB
, fsubs
, half
, one
)
4378 DO_FP_IMM(FMUL
, fmuls
, half
, two
)
4379 DO_FP_IMM(FSUBR
, fsubrs
, half
, one
)
4380 DO_FP_IMM(FMAXNM
, fmaxnms
, zero
, one
)
4381 DO_FP_IMM(FMINNM
, fminnms
, zero
, one
)
4382 DO_FP_IMM(FMAX
, fmaxs
, zero
, one
)
4383 DO_FP_IMM(FMIN
, fmins
, zero
, one
)
4387 static bool do_fp_cmp(DisasContext
*s
, arg_rprr_esz
*a
,
4388 gen_helper_gvec_4_ptr
*fn
)
4393 if (sve_access_check(s
)) {
4394 unsigned vsz
= vec_full_reg_size(s
);
4395 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4396 tcg_gen_gvec_4_ptr(pred_full_reg_offset(s
, a
->rd
),
4397 vec_full_reg_offset(s
, a
->rn
),
4398 vec_full_reg_offset(s
, a
->rm
),
4399 pred_full_reg_offset(s
, a
->pg
),
4400 status
, vsz
, vsz
, 0, fn
);
4401 tcg_temp_free_ptr(status
);
4406 #define DO_FPCMP(NAME, name) \
4407 static bool trans_##NAME##_ppzz(DisasContext *s, arg_rprr_esz *a) \
4409 static gen_helper_gvec_4_ptr * const fns[4] = { \
4410 NULL, gen_helper_sve_##name##_h, \
4411 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4413 return do_fp_cmp(s, a, fns[a->esz]); \
4416 DO_FPCMP(FCMGE
, fcmge
)
4417 DO_FPCMP(FCMGT
, fcmgt
)
4418 DO_FPCMP(FCMEQ
, fcmeq
)
4419 DO_FPCMP(FCMNE
, fcmne
)
4420 DO_FPCMP(FCMUO
, fcmuo
)
4421 DO_FPCMP(FACGE
, facge
)
4422 DO_FPCMP(FACGT
, facgt
)
4426 static bool trans_FCADD(DisasContext
*s
, arg_FCADD
*a
)
4428 static gen_helper_gvec_4_ptr
* const fns
[3] = {
4429 gen_helper_sve_fcadd_h
,
4430 gen_helper_sve_fcadd_s
,
4431 gen_helper_sve_fcadd_d
4437 if (sve_access_check(s
)) {
4438 unsigned vsz
= vec_full_reg_size(s
);
4439 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4440 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4441 vec_full_reg_offset(s
, a
->rn
),
4442 vec_full_reg_offset(s
, a
->rm
),
4443 pred_full_reg_offset(s
, a
->pg
),
4444 status
, vsz
, vsz
, a
->rot
, fns
[a
->esz
- 1]);
4445 tcg_temp_free_ptr(status
);
4450 static bool do_fmla(DisasContext
*s
, arg_rprrr_esz
*a
,
4451 gen_helper_gvec_5_ptr
*fn
)
4456 if (sve_access_check(s
)) {
4457 unsigned vsz
= vec_full_reg_size(s
);
4458 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4459 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s
, a
->rd
),
4460 vec_full_reg_offset(s
, a
->rn
),
4461 vec_full_reg_offset(s
, a
->rm
),
4462 vec_full_reg_offset(s
, a
->ra
),
4463 pred_full_reg_offset(s
, a
->pg
),
4464 status
, vsz
, vsz
, 0, fn
);
4465 tcg_temp_free_ptr(status
);
4470 #define DO_FMLA(NAME, name) \
4471 static bool trans_##NAME(DisasContext *s, arg_rprrr_esz *a) \
4473 static gen_helper_gvec_5_ptr * const fns[4] = { \
4474 NULL, gen_helper_sve_##name##_h, \
4475 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4477 return do_fmla(s, a, fns[a->esz]); \
4480 DO_FMLA(FMLA_zpzzz
, fmla_zpzzz
)
4481 DO_FMLA(FMLS_zpzzz
, fmls_zpzzz
)
4482 DO_FMLA(FNMLA_zpzzz
, fnmla_zpzzz
)
4483 DO_FMLA(FNMLS_zpzzz
, fnmls_zpzzz
)
4487 static bool trans_FCMLA_zpzzz(DisasContext
*s
, arg_FCMLA_zpzzz
*a
)
4489 static gen_helper_gvec_5_ptr
* const fns
[4] = {
4491 gen_helper_sve_fcmla_zpzzz_h
,
4492 gen_helper_sve_fcmla_zpzzz_s
,
4493 gen_helper_sve_fcmla_zpzzz_d
,
4499 if (sve_access_check(s
)) {
4500 unsigned vsz
= vec_full_reg_size(s
);
4501 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4502 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s
, a
->rd
),
4503 vec_full_reg_offset(s
, a
->rn
),
4504 vec_full_reg_offset(s
, a
->rm
),
4505 vec_full_reg_offset(s
, a
->ra
),
4506 pred_full_reg_offset(s
, a
->pg
),
4507 status
, vsz
, vsz
, a
->rot
, fns
[a
->esz
]);
4508 tcg_temp_free_ptr(status
);
4513 static bool trans_FCMLA_zzxz(DisasContext
*s
, arg_FCMLA_zzxz
*a
)
4515 static gen_helper_gvec_4_ptr
* const fns
[2] = {
4516 gen_helper_gvec_fcmlah_idx
,
4517 gen_helper_gvec_fcmlas_idx
,
4520 tcg_debug_assert(a
->esz
== 1 || a
->esz
== 2);
4521 tcg_debug_assert(a
->rd
== a
->ra
);
4522 if (sve_access_check(s
)) {
4523 unsigned vsz
= vec_full_reg_size(s
);
4524 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4525 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4526 vec_full_reg_offset(s
, a
->rn
),
4527 vec_full_reg_offset(s
, a
->rm
),
4528 vec_full_reg_offset(s
, a
->ra
),
4530 a
->index
* 4 + a
->rot
,
4532 tcg_temp_free_ptr(status
);
4538 *** SVE Floating Point Unary Operations Predicated Group
4541 static bool do_zpz_ptr(DisasContext
*s
, int rd
, int rn
, int pg
,
4542 bool is_fp16
, gen_helper_gvec_3_ptr
*fn
)
4544 if (sve_access_check(s
)) {
4545 unsigned vsz
= vec_full_reg_size(s
);
4546 TCGv_ptr status
= fpstatus_ptr(is_fp16
? FPST_FPCR_F16
: FPST_FPCR
);
4547 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, rd
),
4548 vec_full_reg_offset(s
, rn
),
4549 pred_full_reg_offset(s
, pg
),
4550 status
, vsz
, vsz
, 0, fn
);
4551 tcg_temp_free_ptr(status
);
4556 static bool trans_FCVT_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4558 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_sh
);
4561 static bool trans_FCVT_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4563 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_hs
);
4566 static bool trans_BFCVT(DisasContext
*s
, arg_rpr_esz
*a
)
4568 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
4571 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_bfcvt
);
4574 static bool trans_FCVT_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4576 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_dh
);
4579 static bool trans_FCVT_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4581 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_hd
);
4584 static bool trans_FCVT_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4586 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_ds
);
4589 static bool trans_FCVT_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4591 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_sd
);
4594 static bool trans_FCVTZS_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4596 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hh
);
4599 static bool trans_FCVTZU_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4601 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hh
);
4604 static bool trans_FCVTZS_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4606 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hs
);
4609 static bool trans_FCVTZU_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4611 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hs
);
4614 static bool trans_FCVTZS_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4616 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hd
);
4619 static bool trans_FCVTZU_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4621 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hd
);
4624 static bool trans_FCVTZS_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4626 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_ss
);
4629 static bool trans_FCVTZU_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4631 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_ss
);
4634 static bool trans_FCVTZS_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4636 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_sd
);
4639 static bool trans_FCVTZU_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4641 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_sd
);
4644 static bool trans_FCVTZS_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4646 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_ds
);
4649 static bool trans_FCVTZU_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4651 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_ds
);
4654 static bool trans_FCVTZS_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4656 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_dd
);
4659 static bool trans_FCVTZU_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4661 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_dd
);
4664 static gen_helper_gvec_3_ptr
* const frint_fns
[3] = {
4665 gen_helper_sve_frint_h
,
4666 gen_helper_sve_frint_s
,
4667 gen_helper_sve_frint_d
4670 static bool trans_FRINTI(DisasContext
*s
, arg_rpr_esz
*a
)
4675 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
,
4676 frint_fns
[a
->esz
- 1]);
4679 static bool trans_FRINTX(DisasContext
*s
, arg_rpr_esz
*a
)
4681 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4682 gen_helper_sve_frintx_h
,
4683 gen_helper_sve_frintx_s
,
4684 gen_helper_sve_frintx_d
4689 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4692 static bool do_frint_mode(DisasContext
*s
, arg_rpr_esz
*a
,
4693 int mode
, gen_helper_gvec_3_ptr
*fn
)
4695 if (sve_access_check(s
)) {
4696 unsigned vsz
= vec_full_reg_size(s
);
4697 TCGv_i32 tmode
= tcg_const_i32(mode
);
4698 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4700 gen_helper_set_rmode(tmode
, tmode
, status
);
4702 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4703 vec_full_reg_offset(s
, a
->rn
),
4704 pred_full_reg_offset(s
, a
->pg
),
4705 status
, vsz
, vsz
, 0, fn
);
4707 gen_helper_set_rmode(tmode
, tmode
, status
);
4708 tcg_temp_free_i32(tmode
);
4709 tcg_temp_free_ptr(status
);
4714 static bool trans_FRINTN(DisasContext
*s
, arg_rpr_esz
*a
)
4719 return do_frint_mode(s
, a
, float_round_nearest_even
, frint_fns
[a
->esz
- 1]);
4722 static bool trans_FRINTP(DisasContext
*s
, arg_rpr_esz
*a
)
4727 return do_frint_mode(s
, a
, float_round_up
, frint_fns
[a
->esz
- 1]);
4730 static bool trans_FRINTM(DisasContext
*s
, arg_rpr_esz
*a
)
4735 return do_frint_mode(s
, a
, float_round_down
, frint_fns
[a
->esz
- 1]);
4738 static bool trans_FRINTZ(DisasContext
*s
, arg_rpr_esz
*a
)
4743 return do_frint_mode(s
, a
, float_round_to_zero
, frint_fns
[a
->esz
- 1]);
4746 static bool trans_FRINTA(DisasContext
*s
, arg_rpr_esz
*a
)
4751 return do_frint_mode(s
, a
, float_round_ties_away
, frint_fns
[a
->esz
- 1]);
4754 static bool trans_FRECPX(DisasContext
*s
, arg_rpr_esz
*a
)
4756 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4757 gen_helper_sve_frecpx_h
,
4758 gen_helper_sve_frecpx_s
,
4759 gen_helper_sve_frecpx_d
4764 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4767 static bool trans_FSQRT(DisasContext
*s
, arg_rpr_esz
*a
)
4769 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4770 gen_helper_sve_fsqrt_h
,
4771 gen_helper_sve_fsqrt_s
,
4772 gen_helper_sve_fsqrt_d
4777 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4780 static bool trans_SCVTF_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4782 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_hh
);
4785 static bool trans_SCVTF_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4787 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_sh
);
4790 static bool trans_SCVTF_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4792 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_dh
);
4795 static bool trans_SCVTF_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4797 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_ss
);
4800 static bool trans_SCVTF_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4802 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_ds
);
4805 static bool trans_SCVTF_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4807 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_sd
);
4810 static bool trans_SCVTF_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4812 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_dd
);
4815 static bool trans_UCVTF_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4817 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_hh
);
4820 static bool trans_UCVTF_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4822 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_sh
);
4825 static bool trans_UCVTF_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4827 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_dh
);
4830 static bool trans_UCVTF_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4832 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_ss
);
4835 static bool trans_UCVTF_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4837 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_ds
);
4840 static bool trans_UCVTF_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4842 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_sd
);
4845 static bool trans_UCVTF_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4847 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_dd
);
4851 *** SVE Memory - 32-bit Gather and Unsized Contiguous Group
4854 /* Subroutine loading a vector register at VOFS of LEN bytes.
4855 * The load should begin at the address Rn + IMM.
4858 static void do_ldr(DisasContext
*s
, uint32_t vofs
, int len
, int rn
, int imm
)
4860 int len_align
= QEMU_ALIGN_DOWN(len
, 8);
4861 int len_remain
= len
% 8;
4862 int nparts
= len
/ 8 + ctpop8(len_remain
);
4863 int midx
= get_mem_index(s
);
4864 TCGv_i64 dirty_addr
, clean_addr
, t0
, t1
;
4866 dirty_addr
= tcg_temp_new_i64();
4867 tcg_gen_addi_i64(dirty_addr
, cpu_reg_sp(s
, rn
), imm
);
4868 clean_addr
= gen_mte_checkN(s
, dirty_addr
, false, rn
!= 31, len
);
4869 tcg_temp_free_i64(dirty_addr
);
4872 * Note that unpredicated load/store of vector/predicate registers
4873 * are defined as a stream of bytes, which equates to little-endian
4874 * operations on larger quantities.
4875 * Attempt to keep code expansion to a minimum by limiting the
4876 * amount of unrolling done.
4881 t0
= tcg_temp_new_i64();
4882 for (i
= 0; i
< len_align
; i
+= 8) {
4883 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEUQ
);
4884 tcg_gen_st_i64(t0
, cpu_env
, vofs
+ i
);
4885 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4887 tcg_temp_free_i64(t0
);
4889 TCGLabel
*loop
= gen_new_label();
4890 TCGv_ptr tp
, i
= tcg_const_local_ptr(0);
4892 /* Copy the clean address into a local temp, live across the loop. */
4894 clean_addr
= new_tmp_a64_local(s
);
4895 tcg_gen_mov_i64(clean_addr
, t0
);
4897 gen_set_label(loop
);
4899 t0
= tcg_temp_new_i64();
4900 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEUQ
);
4901 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4903 tp
= tcg_temp_new_ptr();
4904 tcg_gen_add_ptr(tp
, cpu_env
, i
);
4905 tcg_gen_addi_ptr(i
, i
, 8);
4906 tcg_gen_st_i64(t0
, tp
, vofs
);
4907 tcg_temp_free_ptr(tp
);
4908 tcg_temp_free_i64(t0
);
4910 tcg_gen_brcondi_ptr(TCG_COND_LTU
, i
, len_align
, loop
);
4911 tcg_temp_free_ptr(i
);
4915 * Predicate register loads can be any multiple of 2.
4916 * Note that we still store the entire 64-bit unit into cpu_env.
4919 t0
= tcg_temp_new_i64();
4920 switch (len_remain
) {
4924 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
,
4925 MO_LE
| ctz32(len_remain
));
4929 t1
= tcg_temp_new_i64();
4930 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEUL
);
4931 tcg_gen_addi_i64(clean_addr
, clean_addr
, 4);
4932 tcg_gen_qemu_ld_i64(t1
, clean_addr
, midx
, MO_LEUW
);
4933 tcg_gen_deposit_i64(t0
, t0
, t1
, 32, 32);
4934 tcg_temp_free_i64(t1
);
4938 g_assert_not_reached();
4940 tcg_gen_st_i64(t0
, cpu_env
, vofs
+ len_align
);
4941 tcg_temp_free_i64(t0
);
4945 /* Similarly for stores. */
4946 static void do_str(DisasContext
*s
, uint32_t vofs
, int len
, int rn
, int imm
)
4948 int len_align
= QEMU_ALIGN_DOWN(len
, 8);
4949 int len_remain
= len
% 8;
4950 int nparts
= len
/ 8 + ctpop8(len_remain
);
4951 int midx
= get_mem_index(s
);
4952 TCGv_i64 dirty_addr
, clean_addr
, t0
;
4954 dirty_addr
= tcg_temp_new_i64();
4955 tcg_gen_addi_i64(dirty_addr
, cpu_reg_sp(s
, rn
), imm
);
4956 clean_addr
= gen_mte_checkN(s
, dirty_addr
, false, rn
!= 31, len
);
4957 tcg_temp_free_i64(dirty_addr
);
4959 /* Note that unpredicated load/store of vector/predicate registers
4960 * are defined as a stream of bytes, which equates to little-endian
4961 * operations on larger quantities. There is no nice way to force
4962 * a little-endian store for aarch64_be-linux-user out of line.
4964 * Attempt to keep code expansion to a minimum by limiting the
4965 * amount of unrolling done.
4970 t0
= tcg_temp_new_i64();
4971 for (i
= 0; i
< len_align
; i
+= 8) {
4972 tcg_gen_ld_i64(t0
, cpu_env
, vofs
+ i
);
4973 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUQ
);
4974 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4976 tcg_temp_free_i64(t0
);
4978 TCGLabel
*loop
= gen_new_label();
4979 TCGv_ptr tp
, i
= tcg_const_local_ptr(0);
4981 /* Copy the clean address into a local temp, live across the loop. */
4983 clean_addr
= new_tmp_a64_local(s
);
4984 tcg_gen_mov_i64(clean_addr
, t0
);
4986 gen_set_label(loop
);
4988 t0
= tcg_temp_new_i64();
4989 tp
= tcg_temp_new_ptr();
4990 tcg_gen_add_ptr(tp
, cpu_env
, i
);
4991 tcg_gen_ld_i64(t0
, tp
, vofs
);
4992 tcg_gen_addi_ptr(i
, i
, 8);
4993 tcg_temp_free_ptr(tp
);
4995 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUQ
);
4996 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4997 tcg_temp_free_i64(t0
);
4999 tcg_gen_brcondi_ptr(TCG_COND_LTU
, i
, len_align
, loop
);
5000 tcg_temp_free_ptr(i
);
5003 /* Predicate register stores can be any multiple of 2. */
5005 t0
= tcg_temp_new_i64();
5006 tcg_gen_ld_i64(t0
, cpu_env
, vofs
+ len_align
);
5008 switch (len_remain
) {
5012 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
,
5013 MO_LE
| ctz32(len_remain
));
5017 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUL
);
5018 tcg_gen_addi_i64(clean_addr
, clean_addr
, 4);
5019 tcg_gen_shri_i64(t0
, t0
, 32);
5020 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUW
);
5024 g_assert_not_reached();
5026 tcg_temp_free_i64(t0
);
5030 static bool trans_LDR_zri(DisasContext
*s
, arg_rri
*a
)
5032 if (sve_access_check(s
)) {
5033 int size
= vec_full_reg_size(s
);
5034 int off
= vec_full_reg_offset(s
, a
->rd
);
5035 do_ldr(s
, off
, size
, a
->rn
, a
->imm
* size
);
5040 static bool trans_LDR_pri(DisasContext
*s
, arg_rri
*a
)
5042 if (sve_access_check(s
)) {
5043 int size
= pred_full_reg_size(s
);
5044 int off
= pred_full_reg_offset(s
, a
->rd
);
5045 do_ldr(s
, off
, size
, a
->rn
, a
->imm
* size
);
5050 static bool trans_STR_zri(DisasContext
*s
, arg_rri
*a
)
5052 if (sve_access_check(s
)) {
5053 int size
= vec_full_reg_size(s
);
5054 int off
= vec_full_reg_offset(s
, a
->rd
);
5055 do_str(s
, off
, size
, a
->rn
, a
->imm
* size
);
5060 static bool trans_STR_pri(DisasContext
*s
, arg_rri
*a
)
5062 if (sve_access_check(s
)) {
5063 int size
= pred_full_reg_size(s
);
5064 int off
= pred_full_reg_offset(s
, a
->rd
);
5065 do_str(s
, off
, size
, a
->rn
, a
->imm
* size
);
5071 *** SVE Memory - Contiguous Load Group
5074 /* The memory mode of the dtype. */
5075 static const MemOp dtype_mop
[16] = {
5076 MO_UB
, MO_UB
, MO_UB
, MO_UB
,
5077 MO_SL
, MO_UW
, MO_UW
, MO_UW
,
5078 MO_SW
, MO_SW
, MO_UL
, MO_UL
,
5079 MO_SB
, MO_SB
, MO_SB
, MO_UQ
5082 #define dtype_msz(x) (dtype_mop[x] & MO_SIZE)
5084 /* The vector element size of dtype. */
5085 static const uint8_t dtype_esz
[16] = {
5092 static void do_mem_zpa(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
,
5093 int dtype
, uint32_t mte_n
, bool is_write
,
5094 gen_helper_gvec_mem
*fn
)
5096 unsigned vsz
= vec_full_reg_size(s
);
5101 * For e.g. LD4, there are not enough arguments to pass all 4
5102 * registers as pointers, so encode the regno into the data field.
5103 * For consistency, do this even for LD1.
5105 if (s
->mte_active
[0]) {
5106 int msz
= dtype_msz(dtype
);
5108 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, get_mem_index(s
));
5109 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
5110 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
5111 desc
= FIELD_DP32(desc
, MTEDESC
, WRITE
, is_write
);
5112 desc
= FIELD_DP32(desc
, MTEDESC
, SIZEM1
, (mte_n
<< msz
) - 1);
5113 desc
<<= SVE_MTEDESC_SHIFT
;
5115 addr
= clean_data_tbi(s
, addr
);
5118 desc
= simd_desc(vsz
, vsz
, zt
| desc
);
5119 t_pg
= tcg_temp_new_ptr();
5121 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
5122 fn(cpu_env
, t_pg
, addr
, tcg_constant_i32(desc
));
5124 tcg_temp_free_ptr(t_pg
);
5127 /* Indexed by [mte][be][dtype][nreg] */
5128 static gen_helper_gvec_mem
* const ldr_fns
[2][2][16][4] = {
5129 { /* mte inactive, little-endian */
5130 { { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld2bb_r
,
5131 gen_helper_sve_ld3bb_r
, gen_helper_sve_ld4bb_r
},
5132 { gen_helper_sve_ld1bhu_r
, NULL
, NULL
, NULL
},
5133 { gen_helper_sve_ld1bsu_r
, NULL
, NULL
, NULL
},
5134 { gen_helper_sve_ld1bdu_r
, NULL
, NULL
, NULL
},
5136 { gen_helper_sve_ld1sds_le_r
, NULL
, NULL
, NULL
},
5137 { gen_helper_sve_ld1hh_le_r
, gen_helper_sve_ld2hh_le_r
,
5138 gen_helper_sve_ld3hh_le_r
, gen_helper_sve_ld4hh_le_r
},
5139 { gen_helper_sve_ld1hsu_le_r
, NULL
, NULL
, NULL
},
5140 { gen_helper_sve_ld1hdu_le_r
, NULL
, NULL
, NULL
},
5142 { gen_helper_sve_ld1hds_le_r
, NULL
, NULL
, NULL
},
5143 { gen_helper_sve_ld1hss_le_r
, NULL
, NULL
, NULL
},
5144 { gen_helper_sve_ld1ss_le_r
, gen_helper_sve_ld2ss_le_r
,
5145 gen_helper_sve_ld3ss_le_r
, gen_helper_sve_ld4ss_le_r
},
5146 { gen_helper_sve_ld1sdu_le_r
, NULL
, NULL
, NULL
},
5148 { gen_helper_sve_ld1bds_r
, NULL
, NULL
, NULL
},
5149 { gen_helper_sve_ld1bss_r
, NULL
, NULL
, NULL
},
5150 { gen_helper_sve_ld1bhs_r
, NULL
, NULL
, NULL
},
5151 { gen_helper_sve_ld1dd_le_r
, gen_helper_sve_ld2dd_le_r
,
5152 gen_helper_sve_ld3dd_le_r
, gen_helper_sve_ld4dd_le_r
} },
5154 /* mte inactive, big-endian */
5155 { { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld2bb_r
,
5156 gen_helper_sve_ld3bb_r
, gen_helper_sve_ld4bb_r
},
5157 { gen_helper_sve_ld1bhu_r
, NULL
, NULL
, NULL
},
5158 { gen_helper_sve_ld1bsu_r
, NULL
, NULL
, NULL
},
5159 { gen_helper_sve_ld1bdu_r
, NULL
, NULL
, NULL
},
5161 { gen_helper_sve_ld1sds_be_r
, NULL
, NULL
, NULL
},
5162 { gen_helper_sve_ld1hh_be_r
, gen_helper_sve_ld2hh_be_r
,
5163 gen_helper_sve_ld3hh_be_r
, gen_helper_sve_ld4hh_be_r
},
5164 { gen_helper_sve_ld1hsu_be_r
, NULL
, NULL
, NULL
},
5165 { gen_helper_sve_ld1hdu_be_r
, NULL
, NULL
, NULL
},
5167 { gen_helper_sve_ld1hds_be_r
, NULL
, NULL
, NULL
},
5168 { gen_helper_sve_ld1hss_be_r
, NULL
, NULL
, NULL
},
5169 { gen_helper_sve_ld1ss_be_r
, gen_helper_sve_ld2ss_be_r
,
5170 gen_helper_sve_ld3ss_be_r
, gen_helper_sve_ld4ss_be_r
},
5171 { gen_helper_sve_ld1sdu_be_r
, NULL
, NULL
, NULL
},
5173 { gen_helper_sve_ld1bds_r
, NULL
, NULL
, NULL
},
5174 { gen_helper_sve_ld1bss_r
, NULL
, NULL
, NULL
},
5175 { gen_helper_sve_ld1bhs_r
, NULL
, NULL
, NULL
},
5176 { gen_helper_sve_ld1dd_be_r
, gen_helper_sve_ld2dd_be_r
,
5177 gen_helper_sve_ld3dd_be_r
, gen_helper_sve_ld4dd_be_r
} } },
5179 { /* mte active, little-endian */
5180 { { gen_helper_sve_ld1bb_r_mte
,
5181 gen_helper_sve_ld2bb_r_mte
,
5182 gen_helper_sve_ld3bb_r_mte
,
5183 gen_helper_sve_ld4bb_r_mte
},
5184 { gen_helper_sve_ld1bhu_r_mte
, NULL
, NULL
, NULL
},
5185 { gen_helper_sve_ld1bsu_r_mte
, NULL
, NULL
, NULL
},
5186 { gen_helper_sve_ld1bdu_r_mte
, NULL
, NULL
, NULL
},
5188 { gen_helper_sve_ld1sds_le_r_mte
, NULL
, NULL
, NULL
},
5189 { gen_helper_sve_ld1hh_le_r_mte
,
5190 gen_helper_sve_ld2hh_le_r_mte
,
5191 gen_helper_sve_ld3hh_le_r_mte
,
5192 gen_helper_sve_ld4hh_le_r_mte
},
5193 { gen_helper_sve_ld1hsu_le_r_mte
, NULL
, NULL
, NULL
},
5194 { gen_helper_sve_ld1hdu_le_r_mte
, NULL
, NULL
, NULL
},
5196 { gen_helper_sve_ld1hds_le_r_mte
, NULL
, NULL
, NULL
},
5197 { gen_helper_sve_ld1hss_le_r_mte
, NULL
, NULL
, NULL
},
5198 { gen_helper_sve_ld1ss_le_r_mte
,
5199 gen_helper_sve_ld2ss_le_r_mte
,
5200 gen_helper_sve_ld3ss_le_r_mte
,
5201 gen_helper_sve_ld4ss_le_r_mte
},
5202 { gen_helper_sve_ld1sdu_le_r_mte
, NULL
, NULL
, NULL
},
5204 { gen_helper_sve_ld1bds_r_mte
, NULL
, NULL
, NULL
},
5205 { gen_helper_sve_ld1bss_r_mte
, NULL
, NULL
, NULL
},
5206 { gen_helper_sve_ld1bhs_r_mte
, NULL
, NULL
, NULL
},
5207 { gen_helper_sve_ld1dd_le_r_mte
,
5208 gen_helper_sve_ld2dd_le_r_mte
,
5209 gen_helper_sve_ld3dd_le_r_mte
,
5210 gen_helper_sve_ld4dd_le_r_mte
} },
5212 /* mte active, big-endian */
5213 { { gen_helper_sve_ld1bb_r_mte
,
5214 gen_helper_sve_ld2bb_r_mte
,
5215 gen_helper_sve_ld3bb_r_mte
,
5216 gen_helper_sve_ld4bb_r_mte
},
5217 { gen_helper_sve_ld1bhu_r_mte
, NULL
, NULL
, NULL
},
5218 { gen_helper_sve_ld1bsu_r_mte
, NULL
, NULL
, NULL
},
5219 { gen_helper_sve_ld1bdu_r_mte
, NULL
, NULL
, NULL
},
5221 { gen_helper_sve_ld1sds_be_r_mte
, NULL
, NULL
, NULL
},
5222 { gen_helper_sve_ld1hh_be_r_mte
,
5223 gen_helper_sve_ld2hh_be_r_mte
,
5224 gen_helper_sve_ld3hh_be_r_mte
,
5225 gen_helper_sve_ld4hh_be_r_mte
},
5226 { gen_helper_sve_ld1hsu_be_r_mte
, NULL
, NULL
, NULL
},
5227 { gen_helper_sve_ld1hdu_be_r_mte
, NULL
, NULL
, NULL
},
5229 { gen_helper_sve_ld1hds_be_r_mte
, NULL
, NULL
, NULL
},
5230 { gen_helper_sve_ld1hss_be_r_mte
, NULL
, NULL
, NULL
},
5231 { gen_helper_sve_ld1ss_be_r_mte
,
5232 gen_helper_sve_ld2ss_be_r_mte
,
5233 gen_helper_sve_ld3ss_be_r_mte
,
5234 gen_helper_sve_ld4ss_be_r_mte
},
5235 { gen_helper_sve_ld1sdu_be_r_mte
, NULL
, NULL
, NULL
},
5237 { gen_helper_sve_ld1bds_r_mte
, NULL
, NULL
, NULL
},
5238 { gen_helper_sve_ld1bss_r_mte
, NULL
, NULL
, NULL
},
5239 { gen_helper_sve_ld1bhs_r_mte
, NULL
, NULL
, NULL
},
5240 { gen_helper_sve_ld1dd_be_r_mte
,
5241 gen_helper_sve_ld2dd_be_r_mte
,
5242 gen_helper_sve_ld3dd_be_r_mte
,
5243 gen_helper_sve_ld4dd_be_r_mte
} } },
5246 static void do_ld_zpa(DisasContext
*s
, int zt
, int pg
,
5247 TCGv_i64 addr
, int dtype
, int nreg
)
5249 gen_helper_gvec_mem
*fn
5250 = ldr_fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][dtype
][nreg
];
5253 * While there are holes in the table, they are not
5254 * accessible via the instruction encoding.
5257 do_mem_zpa(s
, zt
, pg
, addr
, dtype
, nreg
, false, fn
);
5260 static bool trans_LD_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5265 if (sve_access_check(s
)) {
5266 TCGv_i64 addr
= new_tmp_a64(s
);
5267 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
5268 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5269 do_ld_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, a
->nreg
);
5274 static bool trans_LD_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5276 if (sve_access_check(s
)) {
5277 int vsz
= vec_full_reg_size(s
);
5278 int elements
= vsz
>> dtype_esz
[a
->dtype
];
5279 TCGv_i64 addr
= new_tmp_a64(s
);
5281 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
),
5282 (a
->imm
* elements
* (a
->nreg
+ 1))
5283 << dtype_msz(a
->dtype
));
5284 do_ld_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, a
->nreg
);
5289 static bool trans_LDFF1_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5291 static gen_helper_gvec_mem
* const fns
[2][2][16] = {
5292 { /* mte inactive, little-endian */
5293 { gen_helper_sve_ldff1bb_r
,
5294 gen_helper_sve_ldff1bhu_r
,
5295 gen_helper_sve_ldff1bsu_r
,
5296 gen_helper_sve_ldff1bdu_r
,
5298 gen_helper_sve_ldff1sds_le_r
,
5299 gen_helper_sve_ldff1hh_le_r
,
5300 gen_helper_sve_ldff1hsu_le_r
,
5301 gen_helper_sve_ldff1hdu_le_r
,
5303 gen_helper_sve_ldff1hds_le_r
,
5304 gen_helper_sve_ldff1hss_le_r
,
5305 gen_helper_sve_ldff1ss_le_r
,
5306 gen_helper_sve_ldff1sdu_le_r
,
5308 gen_helper_sve_ldff1bds_r
,
5309 gen_helper_sve_ldff1bss_r
,
5310 gen_helper_sve_ldff1bhs_r
,
5311 gen_helper_sve_ldff1dd_le_r
},
5313 /* mte inactive, big-endian */
5314 { gen_helper_sve_ldff1bb_r
,
5315 gen_helper_sve_ldff1bhu_r
,
5316 gen_helper_sve_ldff1bsu_r
,
5317 gen_helper_sve_ldff1bdu_r
,
5319 gen_helper_sve_ldff1sds_be_r
,
5320 gen_helper_sve_ldff1hh_be_r
,
5321 gen_helper_sve_ldff1hsu_be_r
,
5322 gen_helper_sve_ldff1hdu_be_r
,
5324 gen_helper_sve_ldff1hds_be_r
,
5325 gen_helper_sve_ldff1hss_be_r
,
5326 gen_helper_sve_ldff1ss_be_r
,
5327 gen_helper_sve_ldff1sdu_be_r
,
5329 gen_helper_sve_ldff1bds_r
,
5330 gen_helper_sve_ldff1bss_r
,
5331 gen_helper_sve_ldff1bhs_r
,
5332 gen_helper_sve_ldff1dd_be_r
} },
5334 { /* mte active, little-endian */
5335 { gen_helper_sve_ldff1bb_r_mte
,
5336 gen_helper_sve_ldff1bhu_r_mte
,
5337 gen_helper_sve_ldff1bsu_r_mte
,
5338 gen_helper_sve_ldff1bdu_r_mte
,
5340 gen_helper_sve_ldff1sds_le_r_mte
,
5341 gen_helper_sve_ldff1hh_le_r_mte
,
5342 gen_helper_sve_ldff1hsu_le_r_mte
,
5343 gen_helper_sve_ldff1hdu_le_r_mte
,
5345 gen_helper_sve_ldff1hds_le_r_mte
,
5346 gen_helper_sve_ldff1hss_le_r_mte
,
5347 gen_helper_sve_ldff1ss_le_r_mte
,
5348 gen_helper_sve_ldff1sdu_le_r_mte
,
5350 gen_helper_sve_ldff1bds_r_mte
,
5351 gen_helper_sve_ldff1bss_r_mte
,
5352 gen_helper_sve_ldff1bhs_r_mte
,
5353 gen_helper_sve_ldff1dd_le_r_mte
},
5355 /* mte active, big-endian */
5356 { gen_helper_sve_ldff1bb_r_mte
,
5357 gen_helper_sve_ldff1bhu_r_mte
,
5358 gen_helper_sve_ldff1bsu_r_mte
,
5359 gen_helper_sve_ldff1bdu_r_mte
,
5361 gen_helper_sve_ldff1sds_be_r_mte
,
5362 gen_helper_sve_ldff1hh_be_r_mte
,
5363 gen_helper_sve_ldff1hsu_be_r_mte
,
5364 gen_helper_sve_ldff1hdu_be_r_mte
,
5366 gen_helper_sve_ldff1hds_be_r_mte
,
5367 gen_helper_sve_ldff1hss_be_r_mte
,
5368 gen_helper_sve_ldff1ss_be_r_mte
,
5369 gen_helper_sve_ldff1sdu_be_r_mte
,
5371 gen_helper_sve_ldff1bds_r_mte
,
5372 gen_helper_sve_ldff1bss_r_mte
,
5373 gen_helper_sve_ldff1bhs_r_mte
,
5374 gen_helper_sve_ldff1dd_be_r_mte
} },
5377 if (sve_access_check(s
)) {
5378 TCGv_i64 addr
= new_tmp_a64(s
);
5379 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
5380 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5381 do_mem_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, 1, false,
5382 fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][a
->dtype
]);
5387 static bool trans_LDNF1_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5389 static gen_helper_gvec_mem
* const fns
[2][2][16] = {
5390 { /* mte inactive, little-endian */
5391 { gen_helper_sve_ldnf1bb_r
,
5392 gen_helper_sve_ldnf1bhu_r
,
5393 gen_helper_sve_ldnf1bsu_r
,
5394 gen_helper_sve_ldnf1bdu_r
,
5396 gen_helper_sve_ldnf1sds_le_r
,
5397 gen_helper_sve_ldnf1hh_le_r
,
5398 gen_helper_sve_ldnf1hsu_le_r
,
5399 gen_helper_sve_ldnf1hdu_le_r
,
5401 gen_helper_sve_ldnf1hds_le_r
,
5402 gen_helper_sve_ldnf1hss_le_r
,
5403 gen_helper_sve_ldnf1ss_le_r
,
5404 gen_helper_sve_ldnf1sdu_le_r
,
5406 gen_helper_sve_ldnf1bds_r
,
5407 gen_helper_sve_ldnf1bss_r
,
5408 gen_helper_sve_ldnf1bhs_r
,
5409 gen_helper_sve_ldnf1dd_le_r
},
5411 /* mte inactive, big-endian */
5412 { gen_helper_sve_ldnf1bb_r
,
5413 gen_helper_sve_ldnf1bhu_r
,
5414 gen_helper_sve_ldnf1bsu_r
,
5415 gen_helper_sve_ldnf1bdu_r
,
5417 gen_helper_sve_ldnf1sds_be_r
,
5418 gen_helper_sve_ldnf1hh_be_r
,
5419 gen_helper_sve_ldnf1hsu_be_r
,
5420 gen_helper_sve_ldnf1hdu_be_r
,
5422 gen_helper_sve_ldnf1hds_be_r
,
5423 gen_helper_sve_ldnf1hss_be_r
,
5424 gen_helper_sve_ldnf1ss_be_r
,
5425 gen_helper_sve_ldnf1sdu_be_r
,
5427 gen_helper_sve_ldnf1bds_r
,
5428 gen_helper_sve_ldnf1bss_r
,
5429 gen_helper_sve_ldnf1bhs_r
,
5430 gen_helper_sve_ldnf1dd_be_r
} },
5432 { /* mte inactive, little-endian */
5433 { gen_helper_sve_ldnf1bb_r_mte
,
5434 gen_helper_sve_ldnf1bhu_r_mte
,
5435 gen_helper_sve_ldnf1bsu_r_mte
,
5436 gen_helper_sve_ldnf1bdu_r_mte
,
5438 gen_helper_sve_ldnf1sds_le_r_mte
,
5439 gen_helper_sve_ldnf1hh_le_r_mte
,
5440 gen_helper_sve_ldnf1hsu_le_r_mte
,
5441 gen_helper_sve_ldnf1hdu_le_r_mte
,
5443 gen_helper_sve_ldnf1hds_le_r_mte
,
5444 gen_helper_sve_ldnf1hss_le_r_mte
,
5445 gen_helper_sve_ldnf1ss_le_r_mte
,
5446 gen_helper_sve_ldnf1sdu_le_r_mte
,
5448 gen_helper_sve_ldnf1bds_r_mte
,
5449 gen_helper_sve_ldnf1bss_r_mte
,
5450 gen_helper_sve_ldnf1bhs_r_mte
,
5451 gen_helper_sve_ldnf1dd_le_r_mte
},
5453 /* mte inactive, big-endian */
5454 { gen_helper_sve_ldnf1bb_r_mte
,
5455 gen_helper_sve_ldnf1bhu_r_mte
,
5456 gen_helper_sve_ldnf1bsu_r_mte
,
5457 gen_helper_sve_ldnf1bdu_r_mte
,
5459 gen_helper_sve_ldnf1sds_be_r_mte
,
5460 gen_helper_sve_ldnf1hh_be_r_mte
,
5461 gen_helper_sve_ldnf1hsu_be_r_mte
,
5462 gen_helper_sve_ldnf1hdu_be_r_mte
,
5464 gen_helper_sve_ldnf1hds_be_r_mte
,
5465 gen_helper_sve_ldnf1hss_be_r_mte
,
5466 gen_helper_sve_ldnf1ss_be_r_mte
,
5467 gen_helper_sve_ldnf1sdu_be_r_mte
,
5469 gen_helper_sve_ldnf1bds_r_mte
,
5470 gen_helper_sve_ldnf1bss_r_mte
,
5471 gen_helper_sve_ldnf1bhs_r_mte
,
5472 gen_helper_sve_ldnf1dd_be_r_mte
} },
5475 if (sve_access_check(s
)) {
5476 int vsz
= vec_full_reg_size(s
);
5477 int elements
= vsz
>> dtype_esz
[a
->dtype
];
5478 int off
= (a
->imm
* elements
) << dtype_msz(a
->dtype
);
5479 TCGv_i64 addr
= new_tmp_a64(s
);
5481 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), off
);
5482 do_mem_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, 1, false,
5483 fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][a
->dtype
]);
5488 static void do_ldrq(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
, int dtype
)
5490 unsigned vsz
= vec_full_reg_size(s
);
5494 /* Load the first quadword using the normal predicated load helpers. */
5495 poff
= pred_full_reg_offset(s
, pg
);
5498 * Zero-extend the first 16 bits of the predicate into a temporary.
5499 * This avoids triggering an assert making sure we don't have bits
5500 * set within a predicate beyond VQ, but we have lowered VQ to 1
5501 * for this load operation.
5503 TCGv_i64 tmp
= tcg_temp_new_i64();
5507 tcg_gen_ld16u_i64(tmp
, cpu_env
, poff
);
5509 poff
= offsetof(CPUARMState
, vfp
.preg_tmp
);
5510 tcg_gen_st_i64(tmp
, cpu_env
, poff
);
5511 tcg_temp_free_i64(tmp
);
5514 t_pg
= tcg_temp_new_ptr();
5515 tcg_gen_addi_ptr(t_pg
, cpu_env
, poff
);
5517 gen_helper_gvec_mem
*fn
5518 = ldr_fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][dtype
][0];
5519 fn(cpu_env
, t_pg
, addr
, tcg_constant_i32(simd_desc(16, 16, zt
)));
5521 tcg_temp_free_ptr(t_pg
);
5523 /* Replicate that first quadword. */
5525 int doff
= vec_full_reg_offset(s
, zt
);
5526 tcg_gen_gvec_dup_mem(4, doff
+ 16, doff
, vsz
- 16, vsz
- 16);
5530 static bool trans_LD1RQ_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5535 if (sve_access_check(s
)) {
5536 int msz
= dtype_msz(a
->dtype
);
5537 TCGv_i64 addr
= new_tmp_a64(s
);
5538 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), msz
);
5539 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5540 do_ldrq(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5545 static bool trans_LD1RQ_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5547 if (sve_access_check(s
)) {
5548 TCGv_i64 addr
= new_tmp_a64(s
);
5549 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), a
->imm
* 16);
5550 do_ldrq(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5555 static void do_ldro(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
, int dtype
)
5557 unsigned vsz
= vec_full_reg_size(s
);
5564 * Note that this UNDEFINED check comes after CheckSVEEnabled()
5565 * in the ARM pseudocode, which is the sve_access_check() done
5566 * in our caller. We should not now return false from the caller.
5568 unallocated_encoding(s
);
5572 /* Load the first octaword using the normal predicated load helpers. */
5574 poff
= pred_full_reg_offset(s
, pg
);
5577 * Zero-extend the first 32 bits of the predicate into a temporary.
5578 * This avoids triggering an assert making sure we don't have bits
5579 * set within a predicate beyond VQ, but we have lowered VQ to 2
5580 * for this load operation.
5582 TCGv_i64 tmp
= tcg_temp_new_i64();
5586 tcg_gen_ld32u_i64(tmp
, cpu_env
, poff
);
5588 poff
= offsetof(CPUARMState
, vfp
.preg_tmp
);
5589 tcg_gen_st_i64(tmp
, cpu_env
, poff
);
5590 tcg_temp_free_i64(tmp
);
5593 t_pg
= tcg_temp_new_ptr();
5594 tcg_gen_addi_ptr(t_pg
, cpu_env
, poff
);
5596 gen_helper_gvec_mem
*fn
5597 = ldr_fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][dtype
][0];
5598 fn(cpu_env
, t_pg
, addr
, tcg_constant_i32(simd_desc(32, 32, zt
)));
5600 tcg_temp_free_ptr(t_pg
);
5603 * Replicate that first octaword.
5604 * The replication happens in units of 32; if the full vector size
5605 * is not a multiple of 32, the final bits are zeroed.
5607 doff
= vec_full_reg_offset(s
, zt
);
5608 vsz_r32
= QEMU_ALIGN_DOWN(vsz
, 32);
5610 tcg_gen_gvec_dup_mem(5, doff
+ 32, doff
, vsz_r32
- 32, vsz_r32
- 32);
5614 tcg_gen_gvec_dup_imm(MO_64
, doff
+ vsz_r32
, vsz
, vsz
, 0);
5618 static bool trans_LD1RO_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5620 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
5626 if (sve_access_check(s
)) {
5627 TCGv_i64 addr
= new_tmp_a64(s
);
5628 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
5629 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5630 do_ldro(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5635 static bool trans_LD1RO_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5637 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
5640 if (sve_access_check(s
)) {
5641 TCGv_i64 addr
= new_tmp_a64(s
);
5642 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), a
->imm
* 32);
5643 do_ldro(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5648 /* Load and broadcast element. */
5649 static bool trans_LD1R_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5651 unsigned vsz
= vec_full_reg_size(s
);
5652 unsigned psz
= pred_full_reg_size(s
);
5653 unsigned esz
= dtype_esz
[a
->dtype
];
5654 unsigned msz
= dtype_msz(a
->dtype
);
5656 TCGv_i64 temp
, clean_addr
;
5658 if (!sve_access_check(s
)) {
5662 over
= gen_new_label();
5664 /* If the guarding predicate has no bits set, no load occurs. */
5666 /* Reduce the pred_esz_masks value simply to reduce the
5667 * size of the code generated here.
5669 uint64_t psz_mask
= MAKE_64BIT_MASK(0, psz
* 8);
5670 temp
= tcg_temp_new_i64();
5671 tcg_gen_ld_i64(temp
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
5672 tcg_gen_andi_i64(temp
, temp
, pred_esz_masks
[esz
] & psz_mask
);
5673 tcg_gen_brcondi_i64(TCG_COND_EQ
, temp
, 0, over
);
5674 tcg_temp_free_i64(temp
);
5676 TCGv_i32 t32
= tcg_temp_new_i32();
5677 find_last_active(s
, t32
, esz
, a
->pg
);
5678 tcg_gen_brcondi_i32(TCG_COND_LT
, t32
, 0, over
);
5679 tcg_temp_free_i32(t32
);
5682 /* Load the data. */
5683 temp
= tcg_temp_new_i64();
5684 tcg_gen_addi_i64(temp
, cpu_reg_sp(s
, a
->rn
), a
->imm
<< msz
);
5685 clean_addr
= gen_mte_check1(s
, temp
, false, true, msz
);
5687 tcg_gen_qemu_ld_i64(temp
, clean_addr
, get_mem_index(s
),
5688 finalize_memop(s
, dtype_mop
[a
->dtype
]));
5690 /* Broadcast to *all* elements. */
5691 tcg_gen_gvec_dup_i64(esz
, vec_full_reg_offset(s
, a
->rd
),
5693 tcg_temp_free_i64(temp
);
5695 /* Zero the inactive elements. */
5696 gen_set_label(over
);
5697 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, esz
, false);
5700 static void do_st_zpa(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
,
5701 int msz
, int esz
, int nreg
)
5703 static gen_helper_gvec_mem
* const fn_single
[2][2][4][4] = {
5704 { { { gen_helper_sve_st1bb_r
,
5705 gen_helper_sve_st1bh_r
,
5706 gen_helper_sve_st1bs_r
,
5707 gen_helper_sve_st1bd_r
},
5709 gen_helper_sve_st1hh_le_r
,
5710 gen_helper_sve_st1hs_le_r
,
5711 gen_helper_sve_st1hd_le_r
},
5713 gen_helper_sve_st1ss_le_r
,
5714 gen_helper_sve_st1sd_le_r
},
5716 gen_helper_sve_st1dd_le_r
} },
5717 { { gen_helper_sve_st1bb_r
,
5718 gen_helper_sve_st1bh_r
,
5719 gen_helper_sve_st1bs_r
,
5720 gen_helper_sve_st1bd_r
},
5722 gen_helper_sve_st1hh_be_r
,
5723 gen_helper_sve_st1hs_be_r
,
5724 gen_helper_sve_st1hd_be_r
},
5726 gen_helper_sve_st1ss_be_r
,
5727 gen_helper_sve_st1sd_be_r
},
5729 gen_helper_sve_st1dd_be_r
} } },
5731 { { { gen_helper_sve_st1bb_r_mte
,
5732 gen_helper_sve_st1bh_r_mte
,
5733 gen_helper_sve_st1bs_r_mte
,
5734 gen_helper_sve_st1bd_r_mte
},
5736 gen_helper_sve_st1hh_le_r_mte
,
5737 gen_helper_sve_st1hs_le_r_mte
,
5738 gen_helper_sve_st1hd_le_r_mte
},
5740 gen_helper_sve_st1ss_le_r_mte
,
5741 gen_helper_sve_st1sd_le_r_mte
},
5743 gen_helper_sve_st1dd_le_r_mte
} },
5744 { { gen_helper_sve_st1bb_r_mte
,
5745 gen_helper_sve_st1bh_r_mte
,
5746 gen_helper_sve_st1bs_r_mte
,
5747 gen_helper_sve_st1bd_r_mte
},
5749 gen_helper_sve_st1hh_be_r_mte
,
5750 gen_helper_sve_st1hs_be_r_mte
,
5751 gen_helper_sve_st1hd_be_r_mte
},
5753 gen_helper_sve_st1ss_be_r_mte
,
5754 gen_helper_sve_st1sd_be_r_mte
},
5756 gen_helper_sve_st1dd_be_r_mte
} } },
5758 static gen_helper_gvec_mem
* const fn_multiple
[2][2][3][4] = {
5759 { { { gen_helper_sve_st2bb_r
,
5760 gen_helper_sve_st2hh_le_r
,
5761 gen_helper_sve_st2ss_le_r
,
5762 gen_helper_sve_st2dd_le_r
},
5763 { gen_helper_sve_st3bb_r
,
5764 gen_helper_sve_st3hh_le_r
,
5765 gen_helper_sve_st3ss_le_r
,
5766 gen_helper_sve_st3dd_le_r
},
5767 { gen_helper_sve_st4bb_r
,
5768 gen_helper_sve_st4hh_le_r
,
5769 gen_helper_sve_st4ss_le_r
,
5770 gen_helper_sve_st4dd_le_r
} },
5771 { { gen_helper_sve_st2bb_r
,
5772 gen_helper_sve_st2hh_be_r
,
5773 gen_helper_sve_st2ss_be_r
,
5774 gen_helper_sve_st2dd_be_r
},
5775 { gen_helper_sve_st3bb_r
,
5776 gen_helper_sve_st3hh_be_r
,
5777 gen_helper_sve_st3ss_be_r
,
5778 gen_helper_sve_st3dd_be_r
},
5779 { gen_helper_sve_st4bb_r
,
5780 gen_helper_sve_st4hh_be_r
,
5781 gen_helper_sve_st4ss_be_r
,
5782 gen_helper_sve_st4dd_be_r
} } },
5783 { { { gen_helper_sve_st2bb_r_mte
,
5784 gen_helper_sve_st2hh_le_r_mte
,
5785 gen_helper_sve_st2ss_le_r_mte
,
5786 gen_helper_sve_st2dd_le_r_mte
},
5787 { gen_helper_sve_st3bb_r_mte
,
5788 gen_helper_sve_st3hh_le_r_mte
,
5789 gen_helper_sve_st3ss_le_r_mte
,
5790 gen_helper_sve_st3dd_le_r_mte
},
5791 { gen_helper_sve_st4bb_r_mte
,
5792 gen_helper_sve_st4hh_le_r_mte
,
5793 gen_helper_sve_st4ss_le_r_mte
,
5794 gen_helper_sve_st4dd_le_r_mte
} },
5795 { { gen_helper_sve_st2bb_r_mte
,
5796 gen_helper_sve_st2hh_be_r_mte
,
5797 gen_helper_sve_st2ss_be_r_mte
,
5798 gen_helper_sve_st2dd_be_r_mte
},
5799 { gen_helper_sve_st3bb_r_mte
,
5800 gen_helper_sve_st3hh_be_r_mte
,
5801 gen_helper_sve_st3ss_be_r_mte
,
5802 gen_helper_sve_st3dd_be_r_mte
},
5803 { gen_helper_sve_st4bb_r_mte
,
5804 gen_helper_sve_st4hh_be_r_mte
,
5805 gen_helper_sve_st4ss_be_r_mte
,
5806 gen_helper_sve_st4dd_be_r_mte
} } },
5808 gen_helper_gvec_mem
*fn
;
5809 int be
= s
->be_data
== MO_BE
;
5813 fn
= fn_single
[s
->mte_active
[0]][be
][msz
][esz
];
5816 /* ST2, ST3, ST4 -- msz == esz, enforced by encoding */
5818 fn
= fn_multiple
[s
->mte_active
[0]][be
][nreg
- 1][msz
];
5821 do_mem_zpa(s
, zt
, pg
, addr
, msz_dtype(s
, msz
), nreg
, true, fn
);
5824 static bool trans_ST_zprr(DisasContext
*s
, arg_rprr_store
*a
)
5826 if (a
->rm
== 31 || a
->msz
> a
->esz
) {
5829 if (sve_access_check(s
)) {
5830 TCGv_i64 addr
= new_tmp_a64(s
);
5831 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), a
->msz
);
5832 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5833 do_st_zpa(s
, a
->rd
, a
->pg
, addr
, a
->msz
, a
->esz
, a
->nreg
);
5838 static bool trans_ST_zpri(DisasContext
*s
, arg_rpri_store
*a
)
5840 if (a
->msz
> a
->esz
) {
5843 if (sve_access_check(s
)) {
5844 int vsz
= vec_full_reg_size(s
);
5845 int elements
= vsz
>> a
->esz
;
5846 TCGv_i64 addr
= new_tmp_a64(s
);
5848 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
),
5849 (a
->imm
* elements
* (a
->nreg
+ 1)) << a
->msz
);
5850 do_st_zpa(s
, a
->rd
, a
->pg
, addr
, a
->msz
, a
->esz
, a
->nreg
);
5856 *** SVE gather loads / scatter stores
5859 static void do_mem_zpz(DisasContext
*s
, int zt
, int pg
, int zm
,
5860 int scale
, TCGv_i64 scalar
, int msz
, bool is_write
,
5861 gen_helper_gvec_mem_scatter
*fn
)
5863 unsigned vsz
= vec_full_reg_size(s
);
5864 TCGv_ptr t_zm
= tcg_temp_new_ptr();
5865 TCGv_ptr t_pg
= tcg_temp_new_ptr();
5866 TCGv_ptr t_zt
= tcg_temp_new_ptr();
5869 if (s
->mte_active
[0]) {
5870 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, get_mem_index(s
));
5871 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
5872 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
5873 desc
= FIELD_DP32(desc
, MTEDESC
, WRITE
, is_write
);
5874 desc
= FIELD_DP32(desc
, MTEDESC
, SIZEM1
, (1 << msz
) - 1);
5875 desc
<<= SVE_MTEDESC_SHIFT
;
5877 desc
= simd_desc(vsz
, vsz
, desc
| scale
);
5879 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
5880 tcg_gen_addi_ptr(t_zm
, cpu_env
, vec_full_reg_offset(s
, zm
));
5881 tcg_gen_addi_ptr(t_zt
, cpu_env
, vec_full_reg_offset(s
, zt
));
5882 fn(cpu_env
, t_zt
, t_pg
, t_zm
, scalar
, tcg_constant_i32(desc
));
5884 tcg_temp_free_ptr(t_zt
);
5885 tcg_temp_free_ptr(t_zm
);
5886 tcg_temp_free_ptr(t_pg
);
5889 /* Indexed by [mte][be][ff][xs][u][msz]. */
5890 static gen_helper_gvec_mem_scatter
* const
5891 gather_load_fn32
[2][2][2][2][2][3] = {
5892 { /* MTE Inactive */
5893 { /* Little-endian */
5894 { { { gen_helper_sve_ldbss_zsu
,
5895 gen_helper_sve_ldhss_le_zsu
,
5897 { gen_helper_sve_ldbsu_zsu
,
5898 gen_helper_sve_ldhsu_le_zsu
,
5899 gen_helper_sve_ldss_le_zsu
, } },
5900 { { gen_helper_sve_ldbss_zss
,
5901 gen_helper_sve_ldhss_le_zss
,
5903 { gen_helper_sve_ldbsu_zss
,
5904 gen_helper_sve_ldhsu_le_zss
,
5905 gen_helper_sve_ldss_le_zss
, } } },
5908 { { { gen_helper_sve_ldffbss_zsu
,
5909 gen_helper_sve_ldffhss_le_zsu
,
5911 { gen_helper_sve_ldffbsu_zsu
,
5912 gen_helper_sve_ldffhsu_le_zsu
,
5913 gen_helper_sve_ldffss_le_zsu
, } },
5914 { { gen_helper_sve_ldffbss_zss
,
5915 gen_helper_sve_ldffhss_le_zss
,
5917 { gen_helper_sve_ldffbsu_zss
,
5918 gen_helper_sve_ldffhsu_le_zss
,
5919 gen_helper_sve_ldffss_le_zss
, } } } },
5922 { { { gen_helper_sve_ldbss_zsu
,
5923 gen_helper_sve_ldhss_be_zsu
,
5925 { gen_helper_sve_ldbsu_zsu
,
5926 gen_helper_sve_ldhsu_be_zsu
,
5927 gen_helper_sve_ldss_be_zsu
, } },
5928 { { gen_helper_sve_ldbss_zss
,
5929 gen_helper_sve_ldhss_be_zss
,
5931 { gen_helper_sve_ldbsu_zss
,
5932 gen_helper_sve_ldhsu_be_zss
,
5933 gen_helper_sve_ldss_be_zss
, } } },
5936 { { { gen_helper_sve_ldffbss_zsu
,
5937 gen_helper_sve_ldffhss_be_zsu
,
5939 { gen_helper_sve_ldffbsu_zsu
,
5940 gen_helper_sve_ldffhsu_be_zsu
,
5941 gen_helper_sve_ldffss_be_zsu
, } },
5942 { { gen_helper_sve_ldffbss_zss
,
5943 gen_helper_sve_ldffhss_be_zss
,
5945 { gen_helper_sve_ldffbsu_zss
,
5946 gen_helper_sve_ldffhsu_be_zss
,
5947 gen_helper_sve_ldffss_be_zss
, } } } } },
5949 { /* Little-endian */
5950 { { { gen_helper_sve_ldbss_zsu_mte
,
5951 gen_helper_sve_ldhss_le_zsu_mte
,
5953 { gen_helper_sve_ldbsu_zsu_mte
,
5954 gen_helper_sve_ldhsu_le_zsu_mte
,
5955 gen_helper_sve_ldss_le_zsu_mte
, } },
5956 { { gen_helper_sve_ldbss_zss_mte
,
5957 gen_helper_sve_ldhss_le_zss_mte
,
5959 { gen_helper_sve_ldbsu_zss_mte
,
5960 gen_helper_sve_ldhsu_le_zss_mte
,
5961 gen_helper_sve_ldss_le_zss_mte
, } } },
5964 { { { gen_helper_sve_ldffbss_zsu_mte
,
5965 gen_helper_sve_ldffhss_le_zsu_mte
,
5967 { gen_helper_sve_ldffbsu_zsu_mte
,
5968 gen_helper_sve_ldffhsu_le_zsu_mte
,
5969 gen_helper_sve_ldffss_le_zsu_mte
, } },
5970 { { gen_helper_sve_ldffbss_zss_mte
,
5971 gen_helper_sve_ldffhss_le_zss_mte
,
5973 { gen_helper_sve_ldffbsu_zss_mte
,
5974 gen_helper_sve_ldffhsu_le_zss_mte
,
5975 gen_helper_sve_ldffss_le_zss_mte
, } } } },
5978 { { { gen_helper_sve_ldbss_zsu_mte
,
5979 gen_helper_sve_ldhss_be_zsu_mte
,
5981 { gen_helper_sve_ldbsu_zsu_mte
,
5982 gen_helper_sve_ldhsu_be_zsu_mte
,
5983 gen_helper_sve_ldss_be_zsu_mte
, } },
5984 { { gen_helper_sve_ldbss_zss_mte
,
5985 gen_helper_sve_ldhss_be_zss_mte
,
5987 { gen_helper_sve_ldbsu_zss_mte
,
5988 gen_helper_sve_ldhsu_be_zss_mte
,
5989 gen_helper_sve_ldss_be_zss_mte
, } } },
5992 { { { gen_helper_sve_ldffbss_zsu_mte
,
5993 gen_helper_sve_ldffhss_be_zsu_mte
,
5995 { gen_helper_sve_ldffbsu_zsu_mte
,
5996 gen_helper_sve_ldffhsu_be_zsu_mte
,
5997 gen_helper_sve_ldffss_be_zsu_mte
, } },
5998 { { gen_helper_sve_ldffbss_zss_mte
,
5999 gen_helper_sve_ldffhss_be_zss_mte
,
6001 { gen_helper_sve_ldffbsu_zss_mte
,
6002 gen_helper_sve_ldffhsu_be_zss_mte
,
6003 gen_helper_sve_ldffss_be_zss_mte
, } } } } },
6006 /* Note that we overload xs=2 to indicate 64-bit offset. */
6007 static gen_helper_gvec_mem_scatter
* const
6008 gather_load_fn64
[2][2][2][3][2][4] = {
6009 { /* MTE Inactive */
6010 { /* Little-endian */
6011 { { { gen_helper_sve_ldbds_zsu
,
6012 gen_helper_sve_ldhds_le_zsu
,
6013 gen_helper_sve_ldsds_le_zsu
,
6015 { gen_helper_sve_ldbdu_zsu
,
6016 gen_helper_sve_ldhdu_le_zsu
,
6017 gen_helper_sve_ldsdu_le_zsu
,
6018 gen_helper_sve_lddd_le_zsu
, } },
6019 { { gen_helper_sve_ldbds_zss
,
6020 gen_helper_sve_ldhds_le_zss
,
6021 gen_helper_sve_ldsds_le_zss
,
6023 { gen_helper_sve_ldbdu_zss
,
6024 gen_helper_sve_ldhdu_le_zss
,
6025 gen_helper_sve_ldsdu_le_zss
,
6026 gen_helper_sve_lddd_le_zss
, } },
6027 { { gen_helper_sve_ldbds_zd
,
6028 gen_helper_sve_ldhds_le_zd
,
6029 gen_helper_sve_ldsds_le_zd
,
6031 { gen_helper_sve_ldbdu_zd
,
6032 gen_helper_sve_ldhdu_le_zd
,
6033 gen_helper_sve_ldsdu_le_zd
,
6034 gen_helper_sve_lddd_le_zd
, } } },
6037 { { { gen_helper_sve_ldffbds_zsu
,
6038 gen_helper_sve_ldffhds_le_zsu
,
6039 gen_helper_sve_ldffsds_le_zsu
,
6041 { gen_helper_sve_ldffbdu_zsu
,
6042 gen_helper_sve_ldffhdu_le_zsu
,
6043 gen_helper_sve_ldffsdu_le_zsu
,
6044 gen_helper_sve_ldffdd_le_zsu
, } },
6045 { { gen_helper_sve_ldffbds_zss
,
6046 gen_helper_sve_ldffhds_le_zss
,
6047 gen_helper_sve_ldffsds_le_zss
,
6049 { gen_helper_sve_ldffbdu_zss
,
6050 gen_helper_sve_ldffhdu_le_zss
,
6051 gen_helper_sve_ldffsdu_le_zss
,
6052 gen_helper_sve_ldffdd_le_zss
, } },
6053 { { gen_helper_sve_ldffbds_zd
,
6054 gen_helper_sve_ldffhds_le_zd
,
6055 gen_helper_sve_ldffsds_le_zd
,
6057 { gen_helper_sve_ldffbdu_zd
,
6058 gen_helper_sve_ldffhdu_le_zd
,
6059 gen_helper_sve_ldffsdu_le_zd
,
6060 gen_helper_sve_ldffdd_le_zd
, } } } },
6062 { { { gen_helper_sve_ldbds_zsu
,
6063 gen_helper_sve_ldhds_be_zsu
,
6064 gen_helper_sve_ldsds_be_zsu
,
6066 { gen_helper_sve_ldbdu_zsu
,
6067 gen_helper_sve_ldhdu_be_zsu
,
6068 gen_helper_sve_ldsdu_be_zsu
,
6069 gen_helper_sve_lddd_be_zsu
, } },
6070 { { gen_helper_sve_ldbds_zss
,
6071 gen_helper_sve_ldhds_be_zss
,
6072 gen_helper_sve_ldsds_be_zss
,
6074 { gen_helper_sve_ldbdu_zss
,
6075 gen_helper_sve_ldhdu_be_zss
,
6076 gen_helper_sve_ldsdu_be_zss
,
6077 gen_helper_sve_lddd_be_zss
, } },
6078 { { gen_helper_sve_ldbds_zd
,
6079 gen_helper_sve_ldhds_be_zd
,
6080 gen_helper_sve_ldsds_be_zd
,
6082 { gen_helper_sve_ldbdu_zd
,
6083 gen_helper_sve_ldhdu_be_zd
,
6084 gen_helper_sve_ldsdu_be_zd
,
6085 gen_helper_sve_lddd_be_zd
, } } },
6088 { { { gen_helper_sve_ldffbds_zsu
,
6089 gen_helper_sve_ldffhds_be_zsu
,
6090 gen_helper_sve_ldffsds_be_zsu
,
6092 { gen_helper_sve_ldffbdu_zsu
,
6093 gen_helper_sve_ldffhdu_be_zsu
,
6094 gen_helper_sve_ldffsdu_be_zsu
,
6095 gen_helper_sve_ldffdd_be_zsu
, } },
6096 { { gen_helper_sve_ldffbds_zss
,
6097 gen_helper_sve_ldffhds_be_zss
,
6098 gen_helper_sve_ldffsds_be_zss
,
6100 { gen_helper_sve_ldffbdu_zss
,
6101 gen_helper_sve_ldffhdu_be_zss
,
6102 gen_helper_sve_ldffsdu_be_zss
,
6103 gen_helper_sve_ldffdd_be_zss
, } },
6104 { { gen_helper_sve_ldffbds_zd
,
6105 gen_helper_sve_ldffhds_be_zd
,
6106 gen_helper_sve_ldffsds_be_zd
,
6108 { gen_helper_sve_ldffbdu_zd
,
6109 gen_helper_sve_ldffhdu_be_zd
,
6110 gen_helper_sve_ldffsdu_be_zd
,
6111 gen_helper_sve_ldffdd_be_zd
, } } } } },
6113 { /* Little-endian */
6114 { { { gen_helper_sve_ldbds_zsu_mte
,
6115 gen_helper_sve_ldhds_le_zsu_mte
,
6116 gen_helper_sve_ldsds_le_zsu_mte
,
6118 { gen_helper_sve_ldbdu_zsu_mte
,
6119 gen_helper_sve_ldhdu_le_zsu_mte
,
6120 gen_helper_sve_ldsdu_le_zsu_mte
,
6121 gen_helper_sve_lddd_le_zsu_mte
, } },
6122 { { gen_helper_sve_ldbds_zss_mte
,
6123 gen_helper_sve_ldhds_le_zss_mte
,
6124 gen_helper_sve_ldsds_le_zss_mte
,
6126 { gen_helper_sve_ldbdu_zss_mte
,
6127 gen_helper_sve_ldhdu_le_zss_mte
,
6128 gen_helper_sve_ldsdu_le_zss_mte
,
6129 gen_helper_sve_lddd_le_zss_mte
, } },
6130 { { gen_helper_sve_ldbds_zd_mte
,
6131 gen_helper_sve_ldhds_le_zd_mte
,
6132 gen_helper_sve_ldsds_le_zd_mte
,
6134 { gen_helper_sve_ldbdu_zd_mte
,
6135 gen_helper_sve_ldhdu_le_zd_mte
,
6136 gen_helper_sve_ldsdu_le_zd_mte
,
6137 gen_helper_sve_lddd_le_zd_mte
, } } },
6140 { { { gen_helper_sve_ldffbds_zsu_mte
,
6141 gen_helper_sve_ldffhds_le_zsu_mte
,
6142 gen_helper_sve_ldffsds_le_zsu_mte
,
6144 { gen_helper_sve_ldffbdu_zsu_mte
,
6145 gen_helper_sve_ldffhdu_le_zsu_mte
,
6146 gen_helper_sve_ldffsdu_le_zsu_mte
,
6147 gen_helper_sve_ldffdd_le_zsu_mte
, } },
6148 { { gen_helper_sve_ldffbds_zss_mte
,
6149 gen_helper_sve_ldffhds_le_zss_mte
,
6150 gen_helper_sve_ldffsds_le_zss_mte
,
6152 { gen_helper_sve_ldffbdu_zss_mte
,
6153 gen_helper_sve_ldffhdu_le_zss_mte
,
6154 gen_helper_sve_ldffsdu_le_zss_mte
,
6155 gen_helper_sve_ldffdd_le_zss_mte
, } },
6156 { { gen_helper_sve_ldffbds_zd_mte
,
6157 gen_helper_sve_ldffhds_le_zd_mte
,
6158 gen_helper_sve_ldffsds_le_zd_mte
,
6160 { gen_helper_sve_ldffbdu_zd_mte
,
6161 gen_helper_sve_ldffhdu_le_zd_mte
,
6162 gen_helper_sve_ldffsdu_le_zd_mte
,
6163 gen_helper_sve_ldffdd_le_zd_mte
, } } } },
6165 { { { gen_helper_sve_ldbds_zsu_mte
,
6166 gen_helper_sve_ldhds_be_zsu_mte
,
6167 gen_helper_sve_ldsds_be_zsu_mte
,
6169 { gen_helper_sve_ldbdu_zsu_mte
,
6170 gen_helper_sve_ldhdu_be_zsu_mte
,
6171 gen_helper_sve_ldsdu_be_zsu_mte
,
6172 gen_helper_sve_lddd_be_zsu_mte
, } },
6173 { { gen_helper_sve_ldbds_zss_mte
,
6174 gen_helper_sve_ldhds_be_zss_mte
,
6175 gen_helper_sve_ldsds_be_zss_mte
,
6177 { gen_helper_sve_ldbdu_zss_mte
,
6178 gen_helper_sve_ldhdu_be_zss_mte
,
6179 gen_helper_sve_ldsdu_be_zss_mte
,
6180 gen_helper_sve_lddd_be_zss_mte
, } },
6181 { { gen_helper_sve_ldbds_zd_mte
,
6182 gen_helper_sve_ldhds_be_zd_mte
,
6183 gen_helper_sve_ldsds_be_zd_mte
,
6185 { gen_helper_sve_ldbdu_zd_mte
,
6186 gen_helper_sve_ldhdu_be_zd_mte
,
6187 gen_helper_sve_ldsdu_be_zd_mte
,
6188 gen_helper_sve_lddd_be_zd_mte
, } } },
6191 { { { gen_helper_sve_ldffbds_zsu_mte
,
6192 gen_helper_sve_ldffhds_be_zsu_mte
,
6193 gen_helper_sve_ldffsds_be_zsu_mte
,
6195 { gen_helper_sve_ldffbdu_zsu_mte
,
6196 gen_helper_sve_ldffhdu_be_zsu_mte
,
6197 gen_helper_sve_ldffsdu_be_zsu_mte
,
6198 gen_helper_sve_ldffdd_be_zsu_mte
, } },
6199 { { gen_helper_sve_ldffbds_zss_mte
,
6200 gen_helper_sve_ldffhds_be_zss_mte
,
6201 gen_helper_sve_ldffsds_be_zss_mte
,
6203 { gen_helper_sve_ldffbdu_zss_mte
,
6204 gen_helper_sve_ldffhdu_be_zss_mte
,
6205 gen_helper_sve_ldffsdu_be_zss_mte
,
6206 gen_helper_sve_ldffdd_be_zss_mte
, } },
6207 { { gen_helper_sve_ldffbds_zd_mte
,
6208 gen_helper_sve_ldffhds_be_zd_mte
,
6209 gen_helper_sve_ldffsds_be_zd_mte
,
6211 { gen_helper_sve_ldffbdu_zd_mte
,
6212 gen_helper_sve_ldffhdu_be_zd_mte
,
6213 gen_helper_sve_ldffsdu_be_zd_mte
,
6214 gen_helper_sve_ldffdd_be_zd_mte
, } } } } },
6217 static bool trans_LD1_zprz(DisasContext
*s
, arg_LD1_zprz
*a
)
6219 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6220 bool be
= s
->be_data
== MO_BE
;
6221 bool mte
= s
->mte_active
[0];
6223 if (!sve_access_check(s
)) {
6229 fn
= gather_load_fn32
[mte
][be
][a
->ff
][a
->xs
][a
->u
][a
->msz
];
6232 fn
= gather_load_fn64
[mte
][be
][a
->ff
][a
->xs
][a
->u
][a
->msz
];
6237 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rm
, a
->scale
* a
->msz
,
6238 cpu_reg_sp(s
, a
->rn
), a
->msz
, false, fn
);
6242 static bool trans_LD1_zpiz(DisasContext
*s
, arg_LD1_zpiz
*a
)
6244 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6245 bool be
= s
->be_data
== MO_BE
;
6246 bool mte
= s
->mte_active
[0];
6248 if (a
->esz
< a
->msz
|| (a
->esz
== a
->msz
&& !a
->u
)) {
6251 if (!sve_access_check(s
)) {
6257 fn
= gather_load_fn32
[mte
][be
][a
->ff
][0][a
->u
][a
->msz
];
6260 fn
= gather_load_fn64
[mte
][be
][a
->ff
][2][a
->u
][a
->msz
];
6265 /* Treat LD1_zpiz (zn[x] + imm) the same way as LD1_zprz (rn + zm[x])
6266 * by loading the immediate into the scalar parameter.
6268 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0,
6269 tcg_constant_i64(a
->imm
<< a
->msz
), a
->msz
, false, fn
);
6273 static bool trans_LDNT1_zprz(DisasContext
*s
, arg_LD1_zprz
*a
)
6275 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6276 bool be
= s
->be_data
== MO_BE
;
6277 bool mte
= s
->mte_active
[0];
6279 if (a
->esz
< a
->msz
+ !a
->u
) {
6282 if (!dc_isar_feature(aa64_sve2
, s
)) {
6285 if (!sve_access_check(s
)) {
6291 fn
= gather_load_fn32
[mte
][be
][0][0][a
->u
][a
->msz
];
6294 fn
= gather_load_fn64
[mte
][be
][0][2][a
->u
][a
->msz
];
6299 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0,
6300 cpu_reg(s
, a
->rm
), a
->msz
, false, fn
);
6304 /* Indexed by [mte][be][xs][msz]. */
6305 static gen_helper_gvec_mem_scatter
* const scatter_store_fn32
[2][2][2][3] = {
6306 { /* MTE Inactive */
6307 { /* Little-endian */
6308 { gen_helper_sve_stbs_zsu
,
6309 gen_helper_sve_sths_le_zsu
,
6310 gen_helper_sve_stss_le_zsu
, },
6311 { gen_helper_sve_stbs_zss
,
6312 gen_helper_sve_sths_le_zss
,
6313 gen_helper_sve_stss_le_zss
, } },
6315 { gen_helper_sve_stbs_zsu
,
6316 gen_helper_sve_sths_be_zsu
,
6317 gen_helper_sve_stss_be_zsu
, },
6318 { gen_helper_sve_stbs_zss
,
6319 gen_helper_sve_sths_be_zss
,
6320 gen_helper_sve_stss_be_zss
, } } },
6322 { /* Little-endian */
6323 { gen_helper_sve_stbs_zsu_mte
,
6324 gen_helper_sve_sths_le_zsu_mte
,
6325 gen_helper_sve_stss_le_zsu_mte
, },
6326 { gen_helper_sve_stbs_zss_mte
,
6327 gen_helper_sve_sths_le_zss_mte
,
6328 gen_helper_sve_stss_le_zss_mte
, } },
6330 { gen_helper_sve_stbs_zsu_mte
,
6331 gen_helper_sve_sths_be_zsu_mte
,
6332 gen_helper_sve_stss_be_zsu_mte
, },
6333 { gen_helper_sve_stbs_zss_mte
,
6334 gen_helper_sve_sths_be_zss_mte
,
6335 gen_helper_sve_stss_be_zss_mte
, } } },
6338 /* Note that we overload xs=2 to indicate 64-bit offset. */
6339 static gen_helper_gvec_mem_scatter
* const scatter_store_fn64
[2][2][3][4] = {
6340 { /* MTE Inactive */
6341 { /* Little-endian */
6342 { gen_helper_sve_stbd_zsu
,
6343 gen_helper_sve_sthd_le_zsu
,
6344 gen_helper_sve_stsd_le_zsu
,
6345 gen_helper_sve_stdd_le_zsu
, },
6346 { gen_helper_sve_stbd_zss
,
6347 gen_helper_sve_sthd_le_zss
,
6348 gen_helper_sve_stsd_le_zss
,
6349 gen_helper_sve_stdd_le_zss
, },
6350 { gen_helper_sve_stbd_zd
,
6351 gen_helper_sve_sthd_le_zd
,
6352 gen_helper_sve_stsd_le_zd
,
6353 gen_helper_sve_stdd_le_zd
, } },
6355 { gen_helper_sve_stbd_zsu
,
6356 gen_helper_sve_sthd_be_zsu
,
6357 gen_helper_sve_stsd_be_zsu
,
6358 gen_helper_sve_stdd_be_zsu
, },
6359 { gen_helper_sve_stbd_zss
,
6360 gen_helper_sve_sthd_be_zss
,
6361 gen_helper_sve_stsd_be_zss
,
6362 gen_helper_sve_stdd_be_zss
, },
6363 { gen_helper_sve_stbd_zd
,
6364 gen_helper_sve_sthd_be_zd
,
6365 gen_helper_sve_stsd_be_zd
,
6366 gen_helper_sve_stdd_be_zd
, } } },
6367 { /* MTE Inactive */
6368 { /* Little-endian */
6369 { gen_helper_sve_stbd_zsu_mte
,
6370 gen_helper_sve_sthd_le_zsu_mte
,
6371 gen_helper_sve_stsd_le_zsu_mte
,
6372 gen_helper_sve_stdd_le_zsu_mte
, },
6373 { gen_helper_sve_stbd_zss_mte
,
6374 gen_helper_sve_sthd_le_zss_mte
,
6375 gen_helper_sve_stsd_le_zss_mte
,
6376 gen_helper_sve_stdd_le_zss_mte
, },
6377 { gen_helper_sve_stbd_zd_mte
,
6378 gen_helper_sve_sthd_le_zd_mte
,
6379 gen_helper_sve_stsd_le_zd_mte
,
6380 gen_helper_sve_stdd_le_zd_mte
, } },
6382 { gen_helper_sve_stbd_zsu_mte
,
6383 gen_helper_sve_sthd_be_zsu_mte
,
6384 gen_helper_sve_stsd_be_zsu_mte
,
6385 gen_helper_sve_stdd_be_zsu_mte
, },
6386 { gen_helper_sve_stbd_zss_mte
,
6387 gen_helper_sve_sthd_be_zss_mte
,
6388 gen_helper_sve_stsd_be_zss_mte
,
6389 gen_helper_sve_stdd_be_zss_mte
, },
6390 { gen_helper_sve_stbd_zd_mte
,
6391 gen_helper_sve_sthd_be_zd_mte
,
6392 gen_helper_sve_stsd_be_zd_mte
,
6393 gen_helper_sve_stdd_be_zd_mte
, } } },
6396 static bool trans_ST1_zprz(DisasContext
*s
, arg_ST1_zprz
*a
)
6398 gen_helper_gvec_mem_scatter
*fn
;
6399 bool be
= s
->be_data
== MO_BE
;
6400 bool mte
= s
->mte_active
[0];
6402 if (a
->esz
< a
->msz
|| (a
->msz
== 0 && a
->scale
)) {
6405 if (!sve_access_check(s
)) {
6410 fn
= scatter_store_fn32
[mte
][be
][a
->xs
][a
->msz
];
6413 fn
= scatter_store_fn64
[mte
][be
][a
->xs
][a
->msz
];
6416 g_assert_not_reached();
6418 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rm
, a
->scale
* a
->msz
,
6419 cpu_reg_sp(s
, a
->rn
), a
->msz
, true, fn
);
6423 static bool trans_ST1_zpiz(DisasContext
*s
, arg_ST1_zpiz
*a
)
6425 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6426 bool be
= s
->be_data
== MO_BE
;
6427 bool mte
= s
->mte_active
[0];
6429 if (a
->esz
< a
->msz
) {
6432 if (!sve_access_check(s
)) {
6438 fn
= scatter_store_fn32
[mte
][be
][0][a
->msz
];
6441 fn
= scatter_store_fn64
[mte
][be
][2][a
->msz
];
6446 /* Treat ST1_zpiz (zn[x] + imm) the same way as ST1_zprz (rn + zm[x])
6447 * by loading the immediate into the scalar parameter.
6449 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0,
6450 tcg_constant_i64(a
->imm
<< a
->msz
), a
->msz
, true, fn
);
6454 static bool trans_STNT1_zprz(DisasContext
*s
, arg_ST1_zprz
*a
)
6456 gen_helper_gvec_mem_scatter
*fn
;
6457 bool be
= s
->be_data
== MO_BE
;
6458 bool mte
= s
->mte_active
[0];
6460 if (a
->esz
< a
->msz
) {
6463 if (!dc_isar_feature(aa64_sve2
, s
)) {
6466 if (!sve_access_check(s
)) {
6472 fn
= scatter_store_fn32
[mte
][be
][0][a
->msz
];
6475 fn
= scatter_store_fn64
[mte
][be
][2][a
->msz
];
6478 g_assert_not_reached();
6481 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0,
6482 cpu_reg(s
, a
->rm
), a
->msz
, true, fn
);
6490 static bool trans_PRF(DisasContext
*s
, arg_PRF
*a
)
6492 /* Prefetch is a nop within QEMU. */
6493 (void)sve_access_check(s
);
6497 static bool trans_PRF_rr(DisasContext
*s
, arg_PRF_rr
*a
)
6502 /* Prefetch is a nop within QEMU. */
6503 (void)sve_access_check(s
);
6510 * TODO: The implementation so far could handle predicated merging movprfx.
6511 * The helper functions as written take an extra source register to
6512 * use in the operation, but the result is only written when predication
6513 * succeeds. For unpredicated movprfx, we need to rearrange the helpers
6514 * to allow the final write back to the destination to be unconditional.
6515 * For predicated zeroing movprfx, we need to rearrange the helpers to
6516 * allow the final write back to zero inactives.
6518 * In the meantime, just emit the moves.
6521 static bool trans_MOVPRFX(DisasContext
*s
, arg_MOVPRFX
*a
)
6523 return do_mov_z(s
, a
->rd
, a
->rn
);
6526 static bool trans_MOVPRFX_m(DisasContext
*s
, arg_rpr_esz
*a
)
6528 if (sve_access_check(s
)) {
6529 do_sel_z(s
, a
->rd
, a
->rn
, a
->rd
, a
->pg
, a
->esz
);
6534 static bool trans_MOVPRFX_z(DisasContext
*s
, arg_rpr_esz
*a
)
6536 return do_movz_zpz(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
, false);
6540 * SVE2 Integer Multiply - Unpredicated
6543 static bool trans_MUL_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6545 if (!dc_isar_feature(aa64_sve2
, s
)) {
6548 if (sve_access_check(s
)) {
6549 gen_gvec_fn_zzz(s
, tcg_gen_gvec_mul
, a
->esz
, a
->rd
, a
->rn
, a
->rm
);
6554 static gen_helper_gvec_3
* const smulh_zzz_fns
[4] = {
6555 gen_helper_gvec_smulh_b
, gen_helper_gvec_smulh_h
,
6556 gen_helper_gvec_smulh_s
, gen_helper_gvec_smulh_d
,
6558 TRANS_FEAT(SMULH_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6559 smulh_zzz_fns
[a
->esz
], a
, 0)
6561 static gen_helper_gvec_3
* const umulh_zzz_fns
[4] = {
6562 gen_helper_gvec_umulh_b
, gen_helper_gvec_umulh_h
,
6563 gen_helper_gvec_umulh_s
, gen_helper_gvec_umulh_d
,
6565 TRANS_FEAT(UMULH_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6566 umulh_zzz_fns
[a
->esz
], a
, 0)
6568 TRANS_FEAT(PMUL_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6569 gen_helper_gvec_pmul_b
, a
, 0)
6571 static gen_helper_gvec_3
* const sqdmulh_zzz_fns
[4] = {
6572 gen_helper_sve2_sqdmulh_b
, gen_helper_sve2_sqdmulh_h
,
6573 gen_helper_sve2_sqdmulh_s
, gen_helper_sve2_sqdmulh_d
,
6575 TRANS_FEAT(SQDMULH_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6576 sqdmulh_zzz_fns
[a
->esz
], a
, 0)
6578 static gen_helper_gvec_3
* const sqrdmulh_zzz_fns
[4] = {
6579 gen_helper_sve2_sqrdmulh_b
, gen_helper_sve2_sqrdmulh_h
,
6580 gen_helper_sve2_sqrdmulh_s
, gen_helper_sve2_sqrdmulh_d
,
6582 TRANS_FEAT(SQRDMULH_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6583 sqrdmulh_zzz_fns
[a
->esz
], a
, 0)
6586 * SVE2 Integer - Predicated
6589 static bool do_sve2_zpzz_ool(DisasContext
*s
, arg_rprr_esz
*a
,
6590 gen_helper_gvec_4
*fn
)
6592 if (!dc_isar_feature(aa64_sve2
, s
)) {
6595 return do_zpzz_ool(s
, a
, fn
);
6598 static bool trans_SADALP_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
6600 static gen_helper_gvec_4
* const fns
[3] = {
6601 gen_helper_sve2_sadalp_zpzz_h
,
6602 gen_helper_sve2_sadalp_zpzz_s
,
6603 gen_helper_sve2_sadalp_zpzz_d
,
6608 return do_sve2_zpzz_ool(s
, a
, fns
[a
->esz
- 1]);
6611 static bool trans_UADALP_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
6613 static gen_helper_gvec_4
* const fns
[3] = {
6614 gen_helper_sve2_uadalp_zpzz_h
,
6615 gen_helper_sve2_uadalp_zpzz_s
,
6616 gen_helper_sve2_uadalp_zpzz_d
,
6621 return do_sve2_zpzz_ool(s
, a
, fns
[a
->esz
- 1]);
6625 * SVE2 integer unary operations (predicated)
6628 static bool do_sve2_zpz_ool(DisasContext
*s
, arg_rpr_esz
*a
,
6629 gen_helper_gvec_3
*fn
)
6631 if (!dc_isar_feature(aa64_sve2
, s
)) {
6634 return do_zpz_ool(s
, a
, fn
);
6637 static bool trans_URECPE(DisasContext
*s
, arg_rpr_esz
*a
)
6642 return do_sve2_zpz_ool(s
, a
, gen_helper_sve2_urecpe_s
);
6645 static bool trans_URSQRTE(DisasContext
*s
, arg_rpr_esz
*a
)
6650 return do_sve2_zpz_ool(s
, a
, gen_helper_sve2_ursqrte_s
);
6653 static bool trans_SQABS(DisasContext
*s
, arg_rpr_esz
*a
)
6655 static gen_helper_gvec_3
* const fns
[4] = {
6656 gen_helper_sve2_sqabs_b
, gen_helper_sve2_sqabs_h
,
6657 gen_helper_sve2_sqabs_s
, gen_helper_sve2_sqabs_d
,
6659 return do_sve2_zpz_ool(s
, a
, fns
[a
->esz
]);
6662 static bool trans_SQNEG(DisasContext
*s
, arg_rpr_esz
*a
)
6664 static gen_helper_gvec_3
* const fns
[4] = {
6665 gen_helper_sve2_sqneg_b
, gen_helper_sve2_sqneg_h
,
6666 gen_helper_sve2_sqneg_s
, gen_helper_sve2_sqneg_d
,
6668 return do_sve2_zpz_ool(s
, a
, fns
[a
->esz
]);
6671 #define DO_SVE2_ZPZZ(NAME, name) \
6672 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
6674 static gen_helper_gvec_4 * const fns[4] = { \
6675 gen_helper_sve2_##name##_zpzz_b, gen_helper_sve2_##name##_zpzz_h, \
6676 gen_helper_sve2_##name##_zpzz_s, gen_helper_sve2_##name##_zpzz_d, \
6678 return do_sve2_zpzz_ool(s, a, fns[a->esz]); \
6681 DO_SVE2_ZPZZ(SQSHL
, sqshl
)
6682 DO_SVE2_ZPZZ(SQRSHL
, sqrshl
)
6683 DO_SVE2_ZPZZ(SRSHL
, srshl
)
6685 DO_SVE2_ZPZZ(UQSHL
, uqshl
)
6686 DO_SVE2_ZPZZ(UQRSHL
, uqrshl
)
6687 DO_SVE2_ZPZZ(URSHL
, urshl
)
6689 DO_SVE2_ZPZZ(SHADD
, shadd
)
6690 DO_SVE2_ZPZZ(SRHADD
, srhadd
)
6691 DO_SVE2_ZPZZ(SHSUB
, shsub
)
6693 DO_SVE2_ZPZZ(UHADD
, uhadd
)
6694 DO_SVE2_ZPZZ(URHADD
, urhadd
)
6695 DO_SVE2_ZPZZ(UHSUB
, uhsub
)
6697 DO_SVE2_ZPZZ(ADDP
, addp
)
6698 DO_SVE2_ZPZZ(SMAXP
, smaxp
)
6699 DO_SVE2_ZPZZ(UMAXP
, umaxp
)
6700 DO_SVE2_ZPZZ(SMINP
, sminp
)
6701 DO_SVE2_ZPZZ(UMINP
, uminp
)
6703 DO_SVE2_ZPZZ(SQADD_zpzz
, sqadd
)
6704 DO_SVE2_ZPZZ(UQADD_zpzz
, uqadd
)
6705 DO_SVE2_ZPZZ(SQSUB_zpzz
, sqsub
)
6706 DO_SVE2_ZPZZ(UQSUB_zpzz
, uqsub
)
6707 DO_SVE2_ZPZZ(SUQADD
, suqadd
)
6708 DO_SVE2_ZPZZ(USQADD
, usqadd
)
6711 * SVE2 Widening Integer Arithmetic
6714 static gen_helper_gvec_3
* const saddl_fns
[4] = {
6715 NULL
, gen_helper_sve2_saddl_h
,
6716 gen_helper_sve2_saddl_s
, gen_helper_sve2_saddl_d
,
6718 TRANS_FEAT(SADDLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6719 saddl_fns
[a
->esz
], a
, 0)
6720 TRANS_FEAT(SADDLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6721 saddl_fns
[a
->esz
], a
, 3)
6722 TRANS_FEAT(SADDLBT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6723 saddl_fns
[a
->esz
], a
, 2)
6725 static gen_helper_gvec_3
* const ssubl_fns
[4] = {
6726 NULL
, gen_helper_sve2_ssubl_h
,
6727 gen_helper_sve2_ssubl_s
, gen_helper_sve2_ssubl_d
,
6729 TRANS_FEAT(SSUBLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6730 ssubl_fns
[a
->esz
], a
, 0)
6731 TRANS_FEAT(SSUBLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6732 ssubl_fns
[a
->esz
], a
, 3)
6733 TRANS_FEAT(SSUBLBT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6734 ssubl_fns
[a
->esz
], a
, 2)
6735 TRANS_FEAT(SSUBLTB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6736 ssubl_fns
[a
->esz
], a
, 1)
6738 static gen_helper_gvec_3
* const sabdl_fns
[4] = {
6739 NULL
, gen_helper_sve2_sabdl_h
,
6740 gen_helper_sve2_sabdl_s
, gen_helper_sve2_sabdl_d
,
6742 TRANS_FEAT(SABDLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6743 sabdl_fns
[a
->esz
], a
, 0)
6744 TRANS_FEAT(SABDLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6745 sabdl_fns
[a
->esz
], a
, 3)
6747 static gen_helper_gvec_3
* const uaddl_fns
[4] = {
6748 NULL
, gen_helper_sve2_uaddl_h
,
6749 gen_helper_sve2_uaddl_s
, gen_helper_sve2_uaddl_d
,
6751 TRANS_FEAT(UADDLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6752 uaddl_fns
[a
->esz
], a
, 0)
6753 TRANS_FEAT(UADDLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6754 uaddl_fns
[a
->esz
], a
, 3)
6756 static gen_helper_gvec_3
* const usubl_fns
[4] = {
6757 NULL
, gen_helper_sve2_usubl_h
,
6758 gen_helper_sve2_usubl_s
, gen_helper_sve2_usubl_d
,
6760 TRANS_FEAT(USUBLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6761 usubl_fns
[a
->esz
], a
, 0)
6762 TRANS_FEAT(USUBLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6763 usubl_fns
[a
->esz
], a
, 3)
6765 static gen_helper_gvec_3
* const uabdl_fns
[4] = {
6766 NULL
, gen_helper_sve2_uabdl_h
,
6767 gen_helper_sve2_uabdl_s
, gen_helper_sve2_uabdl_d
,
6769 TRANS_FEAT(UABDLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6770 uabdl_fns
[a
->esz
], a
, 0)
6771 TRANS_FEAT(UABDLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6772 uabdl_fns
[a
->esz
], a
, 3)
6774 static gen_helper_gvec_3
* const sqdmull_fns
[4] = {
6775 NULL
, gen_helper_sve2_sqdmull_zzz_h
,
6776 gen_helper_sve2_sqdmull_zzz_s
, gen_helper_sve2_sqdmull_zzz_d
,
6778 TRANS_FEAT(SQDMULLB_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6779 sqdmull_fns
[a
->esz
], a
, 0)
6780 TRANS_FEAT(SQDMULLT_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6781 sqdmull_fns
[a
->esz
], a
, 3)
6783 static gen_helper_gvec_3
* const smull_fns
[4] = {
6784 NULL
, gen_helper_sve2_smull_zzz_h
,
6785 gen_helper_sve2_smull_zzz_s
, gen_helper_sve2_smull_zzz_d
,
6787 TRANS_FEAT(SMULLB_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6788 smull_fns
[a
->esz
], a
, 0)
6789 TRANS_FEAT(SMULLT_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6790 smull_fns
[a
->esz
], a
, 3)
6792 static gen_helper_gvec_3
* const umull_fns
[4] = {
6793 NULL
, gen_helper_sve2_umull_zzz_h
,
6794 gen_helper_sve2_umull_zzz_s
, gen_helper_sve2_umull_zzz_d
,
6796 TRANS_FEAT(UMULLB_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6797 umull_fns
[a
->esz
], a
, 0)
6798 TRANS_FEAT(UMULLT_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6799 umull_fns
[a
->esz
], a
, 3)
6801 static gen_helper_gvec_3
* const eoril_fns
[4] = {
6802 gen_helper_sve2_eoril_b
, gen_helper_sve2_eoril_h
,
6803 gen_helper_sve2_eoril_s
, gen_helper_sve2_eoril_d
,
6805 TRANS_FEAT(EORBT
, aa64_sve2
, gen_gvec_ool_arg_zzz
, eoril_fns
[a
->esz
], a
, 2)
6806 TRANS_FEAT(EORTB
, aa64_sve2
, gen_gvec_ool_arg_zzz
, eoril_fns
[a
->esz
], a
, 1)
6808 static bool do_trans_pmull(DisasContext
*s
, arg_rrr_esz
*a
, bool sel
)
6810 static gen_helper_gvec_3
* const fns
[4] = {
6811 gen_helper_gvec_pmull_q
, gen_helper_sve2_pmull_h
,
6812 NULL
, gen_helper_sve2_pmull_d
,
6814 if (a
->esz
== 0 && !dc_isar_feature(aa64_sve2_pmull128
, s
)) {
6817 return gen_gvec_ool_arg_zzz(s
, fns
[a
->esz
], a
, sel
);
6820 TRANS_FEAT(PMULLB
, aa64_sve2
, do_trans_pmull
, a
, false)
6821 TRANS_FEAT(PMULLT
, aa64_sve2
, do_trans_pmull
, a
, true)
6823 static gen_helper_gvec_3
* const saddw_fns
[4] = {
6824 NULL
, gen_helper_sve2_saddw_h
,
6825 gen_helper_sve2_saddw_s
, gen_helper_sve2_saddw_d
,
6827 TRANS_FEAT(SADDWB
, aa64_sve2
, gen_gvec_ool_arg_zzz
, saddw_fns
[a
->esz
], a
, 0)
6828 TRANS_FEAT(SADDWT
, aa64_sve2
, gen_gvec_ool_arg_zzz
, saddw_fns
[a
->esz
], a
, 1)
6830 static gen_helper_gvec_3
* const ssubw_fns
[4] = {
6831 NULL
, gen_helper_sve2_ssubw_h
,
6832 gen_helper_sve2_ssubw_s
, gen_helper_sve2_ssubw_d
,
6834 TRANS_FEAT(SSUBWB
, aa64_sve2
, gen_gvec_ool_arg_zzz
, ssubw_fns
[a
->esz
], a
, 0)
6835 TRANS_FEAT(SSUBWT
, aa64_sve2
, gen_gvec_ool_arg_zzz
, ssubw_fns
[a
->esz
], a
, 1)
6837 static gen_helper_gvec_3
* const uaddw_fns
[4] = {
6838 NULL
, gen_helper_sve2_uaddw_h
,
6839 gen_helper_sve2_uaddw_s
, gen_helper_sve2_uaddw_d
,
6841 TRANS_FEAT(UADDWB
, aa64_sve2
, gen_gvec_ool_arg_zzz
, uaddw_fns
[a
->esz
], a
, 0)
6842 TRANS_FEAT(UADDWT
, aa64_sve2
, gen_gvec_ool_arg_zzz
, uaddw_fns
[a
->esz
], a
, 1)
6844 static gen_helper_gvec_3
* const usubw_fns
[4] = {
6845 NULL
, gen_helper_sve2_usubw_h
,
6846 gen_helper_sve2_usubw_s
, gen_helper_sve2_usubw_d
,
6848 TRANS_FEAT(USUBWB
, aa64_sve2
, gen_gvec_ool_arg_zzz
, usubw_fns
[a
->esz
], a
, 0)
6849 TRANS_FEAT(USUBWT
, aa64_sve2
, gen_gvec_ool_arg_zzz
, usubw_fns
[a
->esz
], a
, 1)
6851 static void gen_sshll_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t imm
)
6855 int halfbits
= 4 << vece
;
6858 if (shl
== halfbits
) {
6859 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6860 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(halfbits
, halfbits
));
6861 tcg_gen_and_vec(vece
, d
, n
, t
);
6862 tcg_temp_free_vec(t
);
6864 tcg_gen_sari_vec(vece
, d
, n
, halfbits
);
6865 tcg_gen_shli_vec(vece
, d
, d
, shl
);
6868 tcg_gen_shli_vec(vece
, d
, n
, halfbits
);
6869 tcg_gen_sari_vec(vece
, d
, d
, halfbits
- shl
);
6873 static void gen_ushll_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int imm
)
6875 int halfbits
= 4 << vece
;
6877 int shl
= (imm
>> 1);
6881 mask
= MAKE_64BIT_MASK(0, halfbits
);
6883 mask
= dup_const(vece
, mask
);
6885 shift
= shl
- top
* halfbits
;
6887 tcg_gen_shri_i64(d
, n
, -shift
);
6889 tcg_gen_shli_i64(d
, n
, shift
);
6891 tcg_gen_andi_i64(d
, d
, mask
);
6894 static void gen_ushll16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6896 gen_ushll_i64(MO_16
, d
, n
, imm
);
6899 static void gen_ushll32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6901 gen_ushll_i64(MO_32
, d
, n
, imm
);
6904 static void gen_ushll64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6906 gen_ushll_i64(MO_64
, d
, n
, imm
);
6909 static void gen_ushll_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t imm
)
6911 int halfbits
= 4 << vece
;
6916 if (shl
== halfbits
) {
6917 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6918 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(halfbits
, halfbits
));
6919 tcg_gen_and_vec(vece
, d
, n
, t
);
6920 tcg_temp_free_vec(t
);
6922 tcg_gen_shri_vec(vece
, d
, n
, halfbits
);
6923 tcg_gen_shli_vec(vece
, d
, d
, shl
);
6927 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6928 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
6929 tcg_gen_and_vec(vece
, d
, n
, t
);
6930 tcg_temp_free_vec(t
);
6932 tcg_gen_shli_vec(vece
, d
, n
, halfbits
);
6933 tcg_gen_shri_vec(vece
, d
, d
, halfbits
- shl
);
6938 static bool do_sve2_shll_tb(DisasContext
*s
, arg_rri_esz
*a
,
6941 static const TCGOpcode sshll_list
[] = {
6942 INDEX_op_shli_vec
, INDEX_op_sari_vec
, 0
6944 static const TCGOpcode ushll_list
[] = {
6945 INDEX_op_shli_vec
, INDEX_op_shri_vec
, 0
6947 static const GVecGen2i ops
[2][3] = {
6948 { { .fniv
= gen_sshll_vec
,
6949 .opt_opc
= sshll_list
,
6950 .fno
= gen_helper_sve2_sshll_h
,
6952 { .fniv
= gen_sshll_vec
,
6953 .opt_opc
= sshll_list
,
6954 .fno
= gen_helper_sve2_sshll_s
,
6956 { .fniv
= gen_sshll_vec
,
6957 .opt_opc
= sshll_list
,
6958 .fno
= gen_helper_sve2_sshll_d
,
6960 { { .fni8
= gen_ushll16_i64
,
6961 .fniv
= gen_ushll_vec
,
6962 .opt_opc
= ushll_list
,
6963 .fno
= gen_helper_sve2_ushll_h
,
6965 { .fni8
= gen_ushll32_i64
,
6966 .fniv
= gen_ushll_vec
,
6967 .opt_opc
= ushll_list
,
6968 .fno
= gen_helper_sve2_ushll_s
,
6970 { .fni8
= gen_ushll64_i64
,
6971 .fniv
= gen_ushll_vec
,
6972 .opt_opc
= ushll_list
,
6973 .fno
= gen_helper_sve2_ushll_d
,
6977 if (a
->esz
< 0 || a
->esz
> 2 || !dc_isar_feature(aa64_sve2
, s
)) {
6980 if (sve_access_check(s
)) {
6981 unsigned vsz
= vec_full_reg_size(s
);
6982 tcg_gen_gvec_2i(vec_full_reg_offset(s
, a
->rd
),
6983 vec_full_reg_offset(s
, a
->rn
),
6984 vsz
, vsz
, (a
->imm
<< 1) | sel
,
6990 static bool trans_SSHLLB(DisasContext
*s
, arg_rri_esz
*a
)
6992 return do_sve2_shll_tb(s
, a
, false, false);
6995 static bool trans_SSHLLT(DisasContext
*s
, arg_rri_esz
*a
)
6997 return do_sve2_shll_tb(s
, a
, true, false);
7000 static bool trans_USHLLB(DisasContext
*s
, arg_rri_esz
*a
)
7002 return do_sve2_shll_tb(s
, a
, false, true);
7005 static bool trans_USHLLT(DisasContext
*s
, arg_rri_esz
*a
)
7007 return do_sve2_shll_tb(s
, a
, true, true);
7010 static gen_helper_gvec_3
* const bext_fns
[4] = {
7011 gen_helper_sve2_bext_b
, gen_helper_sve2_bext_h
,
7012 gen_helper_sve2_bext_s
, gen_helper_sve2_bext_d
,
7014 TRANS_FEAT(BEXT
, aa64_sve2_bitperm
, gen_gvec_ool_arg_zzz
,
7015 bext_fns
[a
->esz
], a
, 0)
7017 static gen_helper_gvec_3
* const bdep_fns
[4] = {
7018 gen_helper_sve2_bdep_b
, gen_helper_sve2_bdep_h
,
7019 gen_helper_sve2_bdep_s
, gen_helper_sve2_bdep_d
,
7021 TRANS_FEAT(BDEP
, aa64_sve2_bitperm
, gen_gvec_ool_arg_zzz
,
7022 bdep_fns
[a
->esz
], a
, 0)
7024 static gen_helper_gvec_3
* const bgrp_fns
[4] = {
7025 gen_helper_sve2_bgrp_b
, gen_helper_sve2_bgrp_h
,
7026 gen_helper_sve2_bgrp_s
, gen_helper_sve2_bgrp_d
,
7028 TRANS_FEAT(BGRP
, aa64_sve2_bitperm
, gen_gvec_ool_arg_zzz
,
7029 bgrp_fns
[a
->esz
], a
, 0)
7031 static gen_helper_gvec_3
* const cadd_fns
[4] = {
7032 gen_helper_sve2_cadd_b
, gen_helper_sve2_cadd_h
,
7033 gen_helper_sve2_cadd_s
, gen_helper_sve2_cadd_d
,
7035 TRANS_FEAT(CADD_rot90
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
7036 cadd_fns
[a
->esz
], a
, 0)
7037 TRANS_FEAT(CADD_rot270
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
7038 cadd_fns
[a
->esz
], a
, 1)
7040 static gen_helper_gvec_3
* const sqcadd_fns
[4] = {
7041 gen_helper_sve2_sqcadd_b
, gen_helper_sve2_sqcadd_h
,
7042 gen_helper_sve2_sqcadd_s
, gen_helper_sve2_sqcadd_d
,
7044 TRANS_FEAT(SQCADD_rot90
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
7045 sqcadd_fns
[a
->esz
], a
, 0)
7046 TRANS_FEAT(SQCADD_rot270
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
7047 sqcadd_fns
[a
->esz
], a
, 1)
7049 static gen_helper_gvec_4
* const sabal_fns
[4] = {
7050 NULL
, gen_helper_sve2_sabal_h
,
7051 gen_helper_sve2_sabal_s
, gen_helper_sve2_sabal_d
,
7053 TRANS_FEAT(SABALB
, aa64_sve2
, gen_gvec_ool_arg_zzzz
, sabal_fns
[a
->esz
], a
, 0)
7054 TRANS_FEAT(SABALT
, aa64_sve2
, gen_gvec_ool_arg_zzzz
, sabal_fns
[a
->esz
], a
, 1)
7056 static gen_helper_gvec_4
* const uabal_fns
[4] = {
7057 NULL
, gen_helper_sve2_uabal_h
,
7058 gen_helper_sve2_uabal_s
, gen_helper_sve2_uabal_d
,
7060 TRANS_FEAT(UABALB
, aa64_sve2
, gen_gvec_ool_arg_zzzz
, uabal_fns
[a
->esz
], a
, 0)
7061 TRANS_FEAT(UABALT
, aa64_sve2
, gen_gvec_ool_arg_zzzz
, uabal_fns
[a
->esz
], a
, 1)
7063 static bool do_adcl(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
7065 static gen_helper_gvec_4
* const fns
[2] = {
7066 gen_helper_sve2_adcl_s
,
7067 gen_helper_sve2_adcl_d
,
7070 * Note that in this case the ESZ field encodes both size and sign.
7071 * Split out 'subtract' into bit 1 of the data field for the helper.
7073 return gen_gvec_ool_arg_zzzz(s
, fns
[a
->esz
& 1], a
, (a
->esz
& 2) | sel
);
7076 TRANS_FEAT(ADCLB
, aa64_sve2
, do_adcl
, a
, false)
7077 TRANS_FEAT(ADCLT
, aa64_sve2
, do_adcl
, a
, true)
7079 static bool do_sve2_fn2i(DisasContext
*s
, arg_rri_esz
*a
, GVecGen2iFn
*fn
)
7081 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
7084 if (sve_access_check(s
)) {
7085 unsigned vsz
= vec_full_reg_size(s
);
7086 unsigned rd_ofs
= vec_full_reg_offset(s
, a
->rd
);
7087 unsigned rn_ofs
= vec_full_reg_offset(s
, a
->rn
);
7088 fn(a
->esz
, rd_ofs
, rn_ofs
, a
->imm
, vsz
, vsz
);
7093 static bool trans_SSRA(DisasContext
*s
, arg_rri_esz
*a
)
7095 return do_sve2_fn2i(s
, a
, gen_gvec_ssra
);
7098 static bool trans_USRA(DisasContext
*s
, arg_rri_esz
*a
)
7100 return do_sve2_fn2i(s
, a
, gen_gvec_usra
);
7103 static bool trans_SRSRA(DisasContext
*s
, arg_rri_esz
*a
)
7105 return do_sve2_fn2i(s
, a
, gen_gvec_srsra
);
7108 static bool trans_URSRA(DisasContext
*s
, arg_rri_esz
*a
)
7110 return do_sve2_fn2i(s
, a
, gen_gvec_ursra
);
7113 static bool trans_SRI(DisasContext
*s
, arg_rri_esz
*a
)
7115 return do_sve2_fn2i(s
, a
, gen_gvec_sri
);
7118 static bool trans_SLI(DisasContext
*s
, arg_rri_esz
*a
)
7120 return do_sve2_fn2i(s
, a
, gen_gvec_sli
);
7123 static bool do_sve2_fn_zzz(DisasContext
*s
, arg_rrr_esz
*a
, GVecGen3Fn
*fn
)
7125 if (!dc_isar_feature(aa64_sve2
, s
)) {
7128 if (sve_access_check(s
)) {
7129 gen_gvec_fn_zzz(s
, fn
, a
->esz
, a
->rd
, a
->rn
, a
->rm
);
7134 static bool trans_SABA(DisasContext
*s
, arg_rrr_esz
*a
)
7136 return do_sve2_fn_zzz(s
, a
, gen_gvec_saba
);
7139 static bool trans_UABA(DisasContext
*s
, arg_rrr_esz
*a
)
7141 return do_sve2_fn_zzz(s
, a
, gen_gvec_uaba
);
7144 static bool do_sve2_narrow_extract(DisasContext
*s
, arg_rri_esz
*a
,
7145 const GVecGen2 ops
[3])
7147 if (a
->esz
< 0 || a
->esz
> MO_32
|| a
->imm
!= 0 ||
7148 !dc_isar_feature(aa64_sve2
, s
)) {
7151 if (sve_access_check(s
)) {
7152 unsigned vsz
= vec_full_reg_size(s
);
7153 tcg_gen_gvec_2(vec_full_reg_offset(s
, a
->rd
),
7154 vec_full_reg_offset(s
, a
->rn
),
7155 vsz
, vsz
, &ops
[a
->esz
]);
7160 static const TCGOpcode sqxtn_list
[] = {
7161 INDEX_op_shli_vec
, INDEX_op_smin_vec
, INDEX_op_smax_vec
, 0
7164 static void gen_sqxtnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7166 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7167 int halfbits
= 4 << vece
;
7168 int64_t mask
= (1ull << halfbits
) - 1;
7169 int64_t min
= -1ull << (halfbits
- 1);
7170 int64_t max
= -min
- 1;
7172 tcg_gen_dupi_vec(vece
, t
, min
);
7173 tcg_gen_smax_vec(vece
, d
, n
, t
);
7174 tcg_gen_dupi_vec(vece
, t
, max
);
7175 tcg_gen_smin_vec(vece
, d
, d
, t
);
7176 tcg_gen_dupi_vec(vece
, t
, mask
);
7177 tcg_gen_and_vec(vece
, d
, d
, t
);
7178 tcg_temp_free_vec(t
);
7181 static bool trans_SQXTNB(DisasContext
*s
, arg_rri_esz
*a
)
7183 static const GVecGen2 ops
[3] = {
7184 { .fniv
= gen_sqxtnb_vec
,
7185 .opt_opc
= sqxtn_list
,
7186 .fno
= gen_helper_sve2_sqxtnb_h
,
7188 { .fniv
= gen_sqxtnb_vec
,
7189 .opt_opc
= sqxtn_list
,
7190 .fno
= gen_helper_sve2_sqxtnb_s
,
7192 { .fniv
= gen_sqxtnb_vec
,
7193 .opt_opc
= sqxtn_list
,
7194 .fno
= gen_helper_sve2_sqxtnb_d
,
7197 return do_sve2_narrow_extract(s
, a
, ops
);
7200 static void gen_sqxtnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7202 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7203 int halfbits
= 4 << vece
;
7204 int64_t mask
= (1ull << halfbits
) - 1;
7205 int64_t min
= -1ull << (halfbits
- 1);
7206 int64_t max
= -min
- 1;
7208 tcg_gen_dupi_vec(vece
, t
, min
);
7209 tcg_gen_smax_vec(vece
, n
, n
, t
);
7210 tcg_gen_dupi_vec(vece
, t
, max
);
7211 tcg_gen_smin_vec(vece
, n
, n
, t
);
7212 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7213 tcg_gen_dupi_vec(vece
, t
, mask
);
7214 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7215 tcg_temp_free_vec(t
);
7218 static bool trans_SQXTNT(DisasContext
*s
, arg_rri_esz
*a
)
7220 static const GVecGen2 ops
[3] = {
7221 { .fniv
= gen_sqxtnt_vec
,
7222 .opt_opc
= sqxtn_list
,
7224 .fno
= gen_helper_sve2_sqxtnt_h
,
7226 { .fniv
= gen_sqxtnt_vec
,
7227 .opt_opc
= sqxtn_list
,
7229 .fno
= gen_helper_sve2_sqxtnt_s
,
7231 { .fniv
= gen_sqxtnt_vec
,
7232 .opt_opc
= sqxtn_list
,
7234 .fno
= gen_helper_sve2_sqxtnt_d
,
7237 return do_sve2_narrow_extract(s
, a
, ops
);
7240 static const TCGOpcode uqxtn_list
[] = {
7241 INDEX_op_shli_vec
, INDEX_op_umin_vec
, 0
7244 static void gen_uqxtnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7246 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7247 int halfbits
= 4 << vece
;
7248 int64_t max
= (1ull << halfbits
) - 1;
7250 tcg_gen_dupi_vec(vece
, t
, max
);
7251 tcg_gen_umin_vec(vece
, d
, n
, t
);
7252 tcg_temp_free_vec(t
);
7255 static bool trans_UQXTNB(DisasContext
*s
, arg_rri_esz
*a
)
7257 static const GVecGen2 ops
[3] = {
7258 { .fniv
= gen_uqxtnb_vec
,
7259 .opt_opc
= uqxtn_list
,
7260 .fno
= gen_helper_sve2_uqxtnb_h
,
7262 { .fniv
= gen_uqxtnb_vec
,
7263 .opt_opc
= uqxtn_list
,
7264 .fno
= gen_helper_sve2_uqxtnb_s
,
7266 { .fniv
= gen_uqxtnb_vec
,
7267 .opt_opc
= uqxtn_list
,
7268 .fno
= gen_helper_sve2_uqxtnb_d
,
7271 return do_sve2_narrow_extract(s
, a
, ops
);
7274 static void gen_uqxtnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7276 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7277 int halfbits
= 4 << vece
;
7278 int64_t max
= (1ull << halfbits
) - 1;
7280 tcg_gen_dupi_vec(vece
, t
, max
);
7281 tcg_gen_umin_vec(vece
, n
, n
, t
);
7282 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7283 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7284 tcg_temp_free_vec(t
);
7287 static bool trans_UQXTNT(DisasContext
*s
, arg_rri_esz
*a
)
7289 static const GVecGen2 ops
[3] = {
7290 { .fniv
= gen_uqxtnt_vec
,
7291 .opt_opc
= uqxtn_list
,
7293 .fno
= gen_helper_sve2_uqxtnt_h
,
7295 { .fniv
= gen_uqxtnt_vec
,
7296 .opt_opc
= uqxtn_list
,
7298 .fno
= gen_helper_sve2_uqxtnt_s
,
7300 { .fniv
= gen_uqxtnt_vec
,
7301 .opt_opc
= uqxtn_list
,
7303 .fno
= gen_helper_sve2_uqxtnt_d
,
7306 return do_sve2_narrow_extract(s
, a
, ops
);
7309 static const TCGOpcode sqxtun_list
[] = {
7310 INDEX_op_shli_vec
, INDEX_op_umin_vec
, INDEX_op_smax_vec
, 0
7313 static void gen_sqxtunb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7315 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7316 int halfbits
= 4 << vece
;
7317 int64_t max
= (1ull << halfbits
) - 1;
7319 tcg_gen_dupi_vec(vece
, t
, 0);
7320 tcg_gen_smax_vec(vece
, d
, n
, t
);
7321 tcg_gen_dupi_vec(vece
, t
, max
);
7322 tcg_gen_umin_vec(vece
, d
, d
, t
);
7323 tcg_temp_free_vec(t
);
7326 static bool trans_SQXTUNB(DisasContext
*s
, arg_rri_esz
*a
)
7328 static const GVecGen2 ops
[3] = {
7329 { .fniv
= gen_sqxtunb_vec
,
7330 .opt_opc
= sqxtun_list
,
7331 .fno
= gen_helper_sve2_sqxtunb_h
,
7333 { .fniv
= gen_sqxtunb_vec
,
7334 .opt_opc
= sqxtun_list
,
7335 .fno
= gen_helper_sve2_sqxtunb_s
,
7337 { .fniv
= gen_sqxtunb_vec
,
7338 .opt_opc
= sqxtun_list
,
7339 .fno
= gen_helper_sve2_sqxtunb_d
,
7342 return do_sve2_narrow_extract(s
, a
, ops
);
7345 static void gen_sqxtunt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7347 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7348 int halfbits
= 4 << vece
;
7349 int64_t max
= (1ull << halfbits
) - 1;
7351 tcg_gen_dupi_vec(vece
, t
, 0);
7352 tcg_gen_smax_vec(vece
, n
, n
, t
);
7353 tcg_gen_dupi_vec(vece
, t
, max
);
7354 tcg_gen_umin_vec(vece
, n
, n
, t
);
7355 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7356 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7357 tcg_temp_free_vec(t
);
7360 static bool trans_SQXTUNT(DisasContext
*s
, arg_rri_esz
*a
)
7362 static const GVecGen2 ops
[3] = {
7363 { .fniv
= gen_sqxtunt_vec
,
7364 .opt_opc
= sqxtun_list
,
7366 .fno
= gen_helper_sve2_sqxtunt_h
,
7368 { .fniv
= gen_sqxtunt_vec
,
7369 .opt_opc
= sqxtun_list
,
7371 .fno
= gen_helper_sve2_sqxtunt_s
,
7373 { .fniv
= gen_sqxtunt_vec
,
7374 .opt_opc
= sqxtun_list
,
7376 .fno
= gen_helper_sve2_sqxtunt_d
,
7379 return do_sve2_narrow_extract(s
, a
, ops
);
7382 static bool do_sve2_shr_narrow(DisasContext
*s
, arg_rri_esz
*a
,
7383 const GVecGen2i ops
[3])
7385 if (a
->esz
< 0 || a
->esz
> MO_32
|| !dc_isar_feature(aa64_sve2
, s
)) {
7388 assert(a
->imm
> 0 && a
->imm
<= (8 << a
->esz
));
7389 if (sve_access_check(s
)) {
7390 unsigned vsz
= vec_full_reg_size(s
);
7391 tcg_gen_gvec_2i(vec_full_reg_offset(s
, a
->rd
),
7392 vec_full_reg_offset(s
, a
->rn
),
7393 vsz
, vsz
, a
->imm
, &ops
[a
->esz
]);
7398 static void gen_shrnb_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int shr
)
7400 int halfbits
= 4 << vece
;
7401 uint64_t mask
= dup_const(vece
, MAKE_64BIT_MASK(0, halfbits
));
7403 tcg_gen_shri_i64(d
, n
, shr
);
7404 tcg_gen_andi_i64(d
, d
, mask
);
7407 static void gen_shrnb16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7409 gen_shrnb_i64(MO_16
, d
, n
, shr
);
7412 static void gen_shrnb32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7414 gen_shrnb_i64(MO_32
, d
, n
, shr
);
7417 static void gen_shrnb64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7419 gen_shrnb_i64(MO_64
, d
, n
, shr
);
7422 static void gen_shrnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t shr
)
7424 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7425 int halfbits
= 4 << vece
;
7426 uint64_t mask
= MAKE_64BIT_MASK(0, halfbits
);
7428 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7429 tcg_gen_dupi_vec(vece
, t
, mask
);
7430 tcg_gen_and_vec(vece
, d
, n
, t
);
7431 tcg_temp_free_vec(t
);
7434 static bool trans_SHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7436 static const TCGOpcode vec_list
[] = { INDEX_op_shri_vec
, 0 };
7437 static const GVecGen2i ops
[3] = {
7438 { .fni8
= gen_shrnb16_i64
,
7439 .fniv
= gen_shrnb_vec
,
7440 .opt_opc
= vec_list
,
7441 .fno
= gen_helper_sve2_shrnb_h
,
7443 { .fni8
= gen_shrnb32_i64
,
7444 .fniv
= gen_shrnb_vec
,
7445 .opt_opc
= vec_list
,
7446 .fno
= gen_helper_sve2_shrnb_s
,
7448 { .fni8
= gen_shrnb64_i64
,
7449 .fniv
= gen_shrnb_vec
,
7450 .opt_opc
= vec_list
,
7451 .fno
= gen_helper_sve2_shrnb_d
,
7454 return do_sve2_shr_narrow(s
, a
, ops
);
7457 static void gen_shrnt_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int shr
)
7459 int halfbits
= 4 << vece
;
7460 uint64_t mask
= dup_const(vece
, MAKE_64BIT_MASK(0, halfbits
));
7462 tcg_gen_shli_i64(n
, n
, halfbits
- shr
);
7463 tcg_gen_andi_i64(n
, n
, ~mask
);
7464 tcg_gen_andi_i64(d
, d
, mask
);
7465 tcg_gen_or_i64(d
, d
, n
);
7468 static void gen_shrnt16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7470 gen_shrnt_i64(MO_16
, d
, n
, shr
);
7473 static void gen_shrnt32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7475 gen_shrnt_i64(MO_32
, d
, n
, shr
);
7478 static void gen_shrnt64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7480 tcg_gen_shri_i64(n
, n
, shr
);
7481 tcg_gen_deposit_i64(d
, d
, n
, 32, 32);
7484 static void gen_shrnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t shr
)
7486 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7487 int halfbits
= 4 << vece
;
7488 uint64_t mask
= MAKE_64BIT_MASK(0, halfbits
);
7490 tcg_gen_shli_vec(vece
, n
, n
, halfbits
- shr
);
7491 tcg_gen_dupi_vec(vece
, t
, mask
);
7492 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7493 tcg_temp_free_vec(t
);
7496 static bool trans_SHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7498 static const TCGOpcode vec_list
[] = { INDEX_op_shli_vec
, 0 };
7499 static const GVecGen2i ops
[3] = {
7500 { .fni8
= gen_shrnt16_i64
,
7501 .fniv
= gen_shrnt_vec
,
7502 .opt_opc
= vec_list
,
7504 .fno
= gen_helper_sve2_shrnt_h
,
7506 { .fni8
= gen_shrnt32_i64
,
7507 .fniv
= gen_shrnt_vec
,
7508 .opt_opc
= vec_list
,
7510 .fno
= gen_helper_sve2_shrnt_s
,
7512 { .fni8
= gen_shrnt64_i64
,
7513 .fniv
= gen_shrnt_vec
,
7514 .opt_opc
= vec_list
,
7516 .fno
= gen_helper_sve2_shrnt_d
,
7519 return do_sve2_shr_narrow(s
, a
, ops
);
7522 static bool trans_RSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7524 static const GVecGen2i ops
[3] = {
7525 { .fno
= gen_helper_sve2_rshrnb_h
},
7526 { .fno
= gen_helper_sve2_rshrnb_s
},
7527 { .fno
= gen_helper_sve2_rshrnb_d
},
7529 return do_sve2_shr_narrow(s
, a
, ops
);
7532 static bool trans_RSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7534 static const GVecGen2i ops
[3] = {
7535 { .fno
= gen_helper_sve2_rshrnt_h
},
7536 { .fno
= gen_helper_sve2_rshrnt_s
},
7537 { .fno
= gen_helper_sve2_rshrnt_d
},
7539 return do_sve2_shr_narrow(s
, a
, ops
);
7542 static void gen_sqshrunb_vec(unsigned vece
, TCGv_vec d
,
7543 TCGv_vec n
, int64_t shr
)
7545 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7546 int halfbits
= 4 << vece
;
7548 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7549 tcg_gen_dupi_vec(vece
, t
, 0);
7550 tcg_gen_smax_vec(vece
, n
, n
, t
);
7551 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7552 tcg_gen_umin_vec(vece
, d
, n
, t
);
7553 tcg_temp_free_vec(t
);
7556 static bool trans_SQSHRUNB(DisasContext
*s
, arg_rri_esz
*a
)
7558 static const TCGOpcode vec_list
[] = {
7559 INDEX_op_sari_vec
, INDEX_op_smax_vec
, INDEX_op_umin_vec
, 0
7561 static const GVecGen2i ops
[3] = {
7562 { .fniv
= gen_sqshrunb_vec
,
7563 .opt_opc
= vec_list
,
7564 .fno
= gen_helper_sve2_sqshrunb_h
,
7566 { .fniv
= gen_sqshrunb_vec
,
7567 .opt_opc
= vec_list
,
7568 .fno
= gen_helper_sve2_sqshrunb_s
,
7570 { .fniv
= gen_sqshrunb_vec
,
7571 .opt_opc
= vec_list
,
7572 .fno
= gen_helper_sve2_sqshrunb_d
,
7575 return do_sve2_shr_narrow(s
, a
, ops
);
7578 static void gen_sqshrunt_vec(unsigned vece
, TCGv_vec d
,
7579 TCGv_vec n
, int64_t shr
)
7581 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7582 int halfbits
= 4 << vece
;
7584 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7585 tcg_gen_dupi_vec(vece
, t
, 0);
7586 tcg_gen_smax_vec(vece
, n
, n
, t
);
7587 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7588 tcg_gen_umin_vec(vece
, n
, n
, t
);
7589 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7590 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7591 tcg_temp_free_vec(t
);
7594 static bool trans_SQSHRUNT(DisasContext
*s
, arg_rri_esz
*a
)
7596 static const TCGOpcode vec_list
[] = {
7597 INDEX_op_shli_vec
, INDEX_op_sari_vec
,
7598 INDEX_op_smax_vec
, INDEX_op_umin_vec
, 0
7600 static const GVecGen2i ops
[3] = {
7601 { .fniv
= gen_sqshrunt_vec
,
7602 .opt_opc
= vec_list
,
7604 .fno
= gen_helper_sve2_sqshrunt_h
,
7606 { .fniv
= gen_sqshrunt_vec
,
7607 .opt_opc
= vec_list
,
7609 .fno
= gen_helper_sve2_sqshrunt_s
,
7611 { .fniv
= gen_sqshrunt_vec
,
7612 .opt_opc
= vec_list
,
7614 .fno
= gen_helper_sve2_sqshrunt_d
,
7617 return do_sve2_shr_narrow(s
, a
, ops
);
7620 static bool trans_SQRSHRUNB(DisasContext
*s
, arg_rri_esz
*a
)
7622 static const GVecGen2i ops
[3] = {
7623 { .fno
= gen_helper_sve2_sqrshrunb_h
},
7624 { .fno
= gen_helper_sve2_sqrshrunb_s
},
7625 { .fno
= gen_helper_sve2_sqrshrunb_d
},
7627 return do_sve2_shr_narrow(s
, a
, ops
);
7630 static bool trans_SQRSHRUNT(DisasContext
*s
, arg_rri_esz
*a
)
7632 static const GVecGen2i ops
[3] = {
7633 { .fno
= gen_helper_sve2_sqrshrunt_h
},
7634 { .fno
= gen_helper_sve2_sqrshrunt_s
},
7635 { .fno
= gen_helper_sve2_sqrshrunt_d
},
7637 return do_sve2_shr_narrow(s
, a
, ops
);
7640 static void gen_sqshrnb_vec(unsigned vece
, TCGv_vec d
,
7641 TCGv_vec n
, int64_t shr
)
7643 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7644 int halfbits
= 4 << vece
;
7645 int64_t max
= MAKE_64BIT_MASK(0, halfbits
- 1);
7646 int64_t min
= -max
- 1;
7648 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7649 tcg_gen_dupi_vec(vece
, t
, min
);
7650 tcg_gen_smax_vec(vece
, n
, n
, t
);
7651 tcg_gen_dupi_vec(vece
, t
, max
);
7652 tcg_gen_smin_vec(vece
, n
, n
, t
);
7653 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7654 tcg_gen_and_vec(vece
, d
, n
, t
);
7655 tcg_temp_free_vec(t
);
7658 static bool trans_SQSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7660 static const TCGOpcode vec_list
[] = {
7661 INDEX_op_sari_vec
, INDEX_op_smax_vec
, INDEX_op_smin_vec
, 0
7663 static const GVecGen2i ops
[3] = {
7664 { .fniv
= gen_sqshrnb_vec
,
7665 .opt_opc
= vec_list
,
7666 .fno
= gen_helper_sve2_sqshrnb_h
,
7668 { .fniv
= gen_sqshrnb_vec
,
7669 .opt_opc
= vec_list
,
7670 .fno
= gen_helper_sve2_sqshrnb_s
,
7672 { .fniv
= gen_sqshrnb_vec
,
7673 .opt_opc
= vec_list
,
7674 .fno
= gen_helper_sve2_sqshrnb_d
,
7677 return do_sve2_shr_narrow(s
, a
, ops
);
7680 static void gen_sqshrnt_vec(unsigned vece
, TCGv_vec d
,
7681 TCGv_vec n
, int64_t shr
)
7683 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7684 int halfbits
= 4 << vece
;
7685 int64_t max
= MAKE_64BIT_MASK(0, halfbits
- 1);
7686 int64_t min
= -max
- 1;
7688 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7689 tcg_gen_dupi_vec(vece
, t
, min
);
7690 tcg_gen_smax_vec(vece
, n
, n
, t
);
7691 tcg_gen_dupi_vec(vece
, t
, max
);
7692 tcg_gen_smin_vec(vece
, n
, n
, t
);
7693 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7694 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7695 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7696 tcg_temp_free_vec(t
);
7699 static bool trans_SQSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7701 static const TCGOpcode vec_list
[] = {
7702 INDEX_op_shli_vec
, INDEX_op_sari_vec
,
7703 INDEX_op_smax_vec
, INDEX_op_smin_vec
, 0
7705 static const GVecGen2i ops
[3] = {
7706 { .fniv
= gen_sqshrnt_vec
,
7707 .opt_opc
= vec_list
,
7709 .fno
= gen_helper_sve2_sqshrnt_h
,
7711 { .fniv
= gen_sqshrnt_vec
,
7712 .opt_opc
= vec_list
,
7714 .fno
= gen_helper_sve2_sqshrnt_s
,
7716 { .fniv
= gen_sqshrnt_vec
,
7717 .opt_opc
= vec_list
,
7719 .fno
= gen_helper_sve2_sqshrnt_d
,
7722 return do_sve2_shr_narrow(s
, a
, ops
);
7725 static bool trans_SQRSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7727 static const GVecGen2i ops
[3] = {
7728 { .fno
= gen_helper_sve2_sqrshrnb_h
},
7729 { .fno
= gen_helper_sve2_sqrshrnb_s
},
7730 { .fno
= gen_helper_sve2_sqrshrnb_d
},
7732 return do_sve2_shr_narrow(s
, a
, ops
);
7735 static bool trans_SQRSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7737 static const GVecGen2i ops
[3] = {
7738 { .fno
= gen_helper_sve2_sqrshrnt_h
},
7739 { .fno
= gen_helper_sve2_sqrshrnt_s
},
7740 { .fno
= gen_helper_sve2_sqrshrnt_d
},
7742 return do_sve2_shr_narrow(s
, a
, ops
);
7745 static void gen_uqshrnb_vec(unsigned vece
, TCGv_vec d
,
7746 TCGv_vec n
, int64_t shr
)
7748 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7749 int halfbits
= 4 << vece
;
7751 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7752 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7753 tcg_gen_umin_vec(vece
, d
, n
, t
);
7754 tcg_temp_free_vec(t
);
7757 static bool trans_UQSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7759 static const TCGOpcode vec_list
[] = {
7760 INDEX_op_shri_vec
, INDEX_op_umin_vec
, 0
7762 static const GVecGen2i ops
[3] = {
7763 { .fniv
= gen_uqshrnb_vec
,
7764 .opt_opc
= vec_list
,
7765 .fno
= gen_helper_sve2_uqshrnb_h
,
7767 { .fniv
= gen_uqshrnb_vec
,
7768 .opt_opc
= vec_list
,
7769 .fno
= gen_helper_sve2_uqshrnb_s
,
7771 { .fniv
= gen_uqshrnb_vec
,
7772 .opt_opc
= vec_list
,
7773 .fno
= gen_helper_sve2_uqshrnb_d
,
7776 return do_sve2_shr_narrow(s
, a
, ops
);
7779 static void gen_uqshrnt_vec(unsigned vece
, TCGv_vec d
,
7780 TCGv_vec n
, int64_t shr
)
7782 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7783 int halfbits
= 4 << vece
;
7785 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7786 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7787 tcg_gen_umin_vec(vece
, n
, n
, t
);
7788 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7789 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7790 tcg_temp_free_vec(t
);
7793 static bool trans_UQSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7795 static const TCGOpcode vec_list
[] = {
7796 INDEX_op_shli_vec
, INDEX_op_shri_vec
, INDEX_op_umin_vec
, 0
7798 static const GVecGen2i ops
[3] = {
7799 { .fniv
= gen_uqshrnt_vec
,
7800 .opt_opc
= vec_list
,
7802 .fno
= gen_helper_sve2_uqshrnt_h
,
7804 { .fniv
= gen_uqshrnt_vec
,
7805 .opt_opc
= vec_list
,
7807 .fno
= gen_helper_sve2_uqshrnt_s
,
7809 { .fniv
= gen_uqshrnt_vec
,
7810 .opt_opc
= vec_list
,
7812 .fno
= gen_helper_sve2_uqshrnt_d
,
7815 return do_sve2_shr_narrow(s
, a
, ops
);
7818 static bool trans_UQRSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7820 static const GVecGen2i ops
[3] = {
7821 { .fno
= gen_helper_sve2_uqrshrnb_h
},
7822 { .fno
= gen_helper_sve2_uqrshrnb_s
},
7823 { .fno
= gen_helper_sve2_uqrshrnb_d
},
7825 return do_sve2_shr_narrow(s
, a
, ops
);
7828 static bool trans_UQRSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7830 static const GVecGen2i ops
[3] = {
7831 { .fno
= gen_helper_sve2_uqrshrnt_h
},
7832 { .fno
= gen_helper_sve2_uqrshrnt_s
},
7833 { .fno
= gen_helper_sve2_uqrshrnt_d
},
7835 return do_sve2_shr_narrow(s
, a
, ops
);
7838 #define DO_SVE2_ZZZ_NARROW(NAME, name) \
7839 static gen_helper_gvec_3 * const name##_fns[4] = { \
7840 NULL, gen_helper_sve2_##name##_h, \
7841 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
7843 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_arg_zzz, \
7844 name##_fns[a->esz], a, 0)
7846 DO_SVE2_ZZZ_NARROW(ADDHNB
, addhnb
)
7847 DO_SVE2_ZZZ_NARROW(ADDHNT
, addhnt
)
7848 DO_SVE2_ZZZ_NARROW(RADDHNB
, raddhnb
)
7849 DO_SVE2_ZZZ_NARROW(RADDHNT
, raddhnt
)
7851 DO_SVE2_ZZZ_NARROW(SUBHNB
, subhnb
)
7852 DO_SVE2_ZZZ_NARROW(SUBHNT
, subhnt
)
7853 DO_SVE2_ZZZ_NARROW(RSUBHNB
, rsubhnb
)
7854 DO_SVE2_ZZZ_NARROW(RSUBHNT
, rsubhnt
)
7856 static bool do_sve2_ppzz_flags(DisasContext
*s
, arg_rprr_esz
*a
,
7857 gen_helper_gvec_flags_4
*fn
)
7859 if (!dc_isar_feature(aa64_sve2
, s
)) {
7862 return do_ppzz_flags(s
, a
, fn
);
7865 #define DO_SVE2_PPZZ_MATCH(NAME, name) \
7866 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
7868 static gen_helper_gvec_flags_4 * const fns[4] = { \
7869 gen_helper_sve2_##name##_ppzz_b, gen_helper_sve2_##name##_ppzz_h, \
7872 return do_sve2_ppzz_flags(s, a, fns[a->esz]); \
7875 DO_SVE2_PPZZ_MATCH(MATCH
, match
)
7876 DO_SVE2_PPZZ_MATCH(NMATCH
, nmatch
)
7878 static bool trans_HISTCNT(DisasContext
*s
, arg_rprr_esz
*a
)
7880 static gen_helper_gvec_4
* const fns
[2] = {
7881 gen_helper_sve2_histcnt_s
, gen_helper_sve2_histcnt_d
7886 return do_sve2_zpzz_ool(s
, a
, fns
[a
->esz
- 2]);
7889 TRANS_FEAT(HISTSEG
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
7890 a
->esz
== 0 ? gen_helper_sve2_histseg
: NULL
, a
, 0)
7892 static bool do_sve2_zpzz_fp(DisasContext
*s
, arg_rprr_esz
*a
,
7893 gen_helper_gvec_4_ptr
*fn
)
7895 if (!dc_isar_feature(aa64_sve2
, s
)) {
7898 return do_zpzz_fp(s
, a
, fn
);
7901 #define DO_SVE2_ZPZZ_FP(NAME, name) \
7902 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
7904 static gen_helper_gvec_4_ptr * const fns[4] = { \
7905 NULL, gen_helper_sve2_##name##_zpzz_h, \
7906 gen_helper_sve2_##name##_zpzz_s, gen_helper_sve2_##name##_zpzz_d \
7908 return do_sve2_zpzz_fp(s, a, fns[a->esz]); \
7911 DO_SVE2_ZPZZ_FP(FADDP
, faddp
)
7912 DO_SVE2_ZPZZ_FP(FMAXNMP
, fmaxnmp
)
7913 DO_SVE2_ZPZZ_FP(FMINNMP
, fminnmp
)
7914 DO_SVE2_ZPZZ_FP(FMAXP
, fmaxp
)
7915 DO_SVE2_ZPZZ_FP(FMINP
, fminp
)
7918 * SVE Integer Multiply-Add (unpredicated)
7921 static bool trans_FMMLA(DisasContext
*s
, arg_rrrr_esz
*a
)
7923 gen_helper_gvec_4_ptr
*fn
;
7927 if (!dc_isar_feature(aa64_sve_f32mm
, s
)) {
7930 fn
= gen_helper_fmmla_s
;
7933 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
7936 fn
= gen_helper_fmmla_d
;
7942 if (sve_access_check(s
)) {
7943 unsigned vsz
= vec_full_reg_size(s
);
7944 TCGv_ptr status
= fpstatus_ptr(FPST_FPCR
);
7945 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
7946 vec_full_reg_offset(s
, a
->rn
),
7947 vec_full_reg_offset(s
, a
->rm
),
7948 vec_full_reg_offset(s
, a
->ra
),
7949 status
, vsz
, vsz
, 0, fn
);
7950 tcg_temp_free_ptr(status
);
7955 static gen_helper_gvec_4
* const sqdmlal_zzzw_fns
[] = {
7956 NULL
, gen_helper_sve2_sqdmlal_zzzw_h
,
7957 gen_helper_sve2_sqdmlal_zzzw_s
, gen_helper_sve2_sqdmlal_zzzw_d
,
7959 TRANS_FEAT(SQDMLALB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7960 sqdmlal_zzzw_fns
[a
->esz
], a
, 0)
7961 TRANS_FEAT(SQDMLALT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7962 sqdmlal_zzzw_fns
[a
->esz
], a
, 3)
7963 TRANS_FEAT(SQDMLALBT
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7964 sqdmlal_zzzw_fns
[a
->esz
], a
, 2)
7966 static gen_helper_gvec_4
* const sqdmlsl_zzzw_fns
[] = {
7967 NULL
, gen_helper_sve2_sqdmlsl_zzzw_h
,
7968 gen_helper_sve2_sqdmlsl_zzzw_s
, gen_helper_sve2_sqdmlsl_zzzw_d
,
7970 TRANS_FEAT(SQDMLSLB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7971 sqdmlsl_zzzw_fns
[a
->esz
], a
, 0)
7972 TRANS_FEAT(SQDMLSLT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7973 sqdmlsl_zzzw_fns
[a
->esz
], a
, 3)
7974 TRANS_FEAT(SQDMLSLBT
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7975 sqdmlsl_zzzw_fns
[a
->esz
], a
, 2)
7977 static gen_helper_gvec_4
* const sqrdmlah_fns
[] = {
7978 gen_helper_sve2_sqrdmlah_b
, gen_helper_sve2_sqrdmlah_h
,
7979 gen_helper_sve2_sqrdmlah_s
, gen_helper_sve2_sqrdmlah_d
,
7981 TRANS_FEAT(SQRDMLAH_zzzz
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7982 sqrdmlah_fns
[a
->esz
], a
, 0)
7984 static gen_helper_gvec_4
* const sqrdmlsh_fns
[] = {
7985 gen_helper_sve2_sqrdmlsh_b
, gen_helper_sve2_sqrdmlsh_h
,
7986 gen_helper_sve2_sqrdmlsh_s
, gen_helper_sve2_sqrdmlsh_d
,
7988 TRANS_FEAT(SQRDMLSH_zzzz
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7989 sqrdmlsh_fns
[a
->esz
], a
, 0)
7991 static gen_helper_gvec_4
* const smlal_zzzw_fns
[] = {
7992 NULL
, gen_helper_sve2_smlal_zzzw_h
,
7993 gen_helper_sve2_smlal_zzzw_s
, gen_helper_sve2_smlal_zzzw_d
,
7995 TRANS_FEAT(SMLALB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7996 smlal_zzzw_fns
[a
->esz
], a
, 0)
7997 TRANS_FEAT(SMLALT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7998 smlal_zzzw_fns
[a
->esz
], a
, 1)
8000 static gen_helper_gvec_4
* const umlal_zzzw_fns
[] = {
8001 NULL
, gen_helper_sve2_umlal_zzzw_h
,
8002 gen_helper_sve2_umlal_zzzw_s
, gen_helper_sve2_umlal_zzzw_d
,
8004 TRANS_FEAT(UMLALB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
8005 umlal_zzzw_fns
[a
->esz
], a
, 0)
8006 TRANS_FEAT(UMLALT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
8007 umlal_zzzw_fns
[a
->esz
], a
, 1)
8009 static gen_helper_gvec_4
* const smlsl_zzzw_fns
[] = {
8010 NULL
, gen_helper_sve2_smlsl_zzzw_h
,
8011 gen_helper_sve2_smlsl_zzzw_s
, gen_helper_sve2_smlsl_zzzw_d
,
8013 TRANS_FEAT(SMLSLB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
8014 smlsl_zzzw_fns
[a
->esz
], a
, 0)
8015 TRANS_FEAT(SMLSLT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
8016 smlsl_zzzw_fns
[a
->esz
], a
, 1)
8018 static gen_helper_gvec_4
* const umlsl_zzzw_fns
[] = {
8019 NULL
, gen_helper_sve2_umlsl_zzzw_h
,
8020 gen_helper_sve2_umlsl_zzzw_s
, gen_helper_sve2_umlsl_zzzw_d
,
8022 TRANS_FEAT(UMLSLB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
8023 umlsl_zzzw_fns
[a
->esz
], a
, 0)
8024 TRANS_FEAT(UMLSLT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
8025 umlsl_zzzw_fns
[a
->esz
], a
, 1)
8027 static gen_helper_gvec_4
* const cmla_fns
[] = {
8028 gen_helper_sve2_cmla_zzzz_b
, gen_helper_sve2_cmla_zzzz_h
,
8029 gen_helper_sve2_cmla_zzzz_s
, gen_helper_sve2_cmla_zzzz_d
,
8031 TRANS_FEAT(CMLA_zzzz
, aa64_sve2
, gen_gvec_ool_zzzz
,
8032 cmla_fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
)
8034 static gen_helper_gvec_4
* const cdot_fns
[] = {
8035 NULL
, NULL
, gen_helper_sve2_cdot_zzzz_s
, gen_helper_sve2_cdot_zzzz_d
8037 TRANS_FEAT(CDOT_zzzz
, aa64_sve2
, gen_gvec_ool_zzzz
,
8038 cdot_fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
)
8040 static gen_helper_gvec_4
* const sqrdcmlah_fns
[] = {
8041 gen_helper_sve2_sqrdcmlah_zzzz_b
, gen_helper_sve2_sqrdcmlah_zzzz_h
,
8042 gen_helper_sve2_sqrdcmlah_zzzz_s
, gen_helper_sve2_sqrdcmlah_zzzz_d
,
8044 TRANS_FEAT(SQRDCMLAH_zzzz
, aa64_sve2
, gen_gvec_ool_zzzz
,
8045 sqrdcmlah_fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
)
8047 static bool trans_USDOT_zzzz(DisasContext
*s
, arg_USDOT_zzzz
*a
)
8049 if (a
->esz
!= 2 || !dc_isar_feature(aa64_sve_i8mm
, s
)) {
8052 if (sve_access_check(s
)) {
8053 unsigned vsz
= vec_full_reg_size(s
);
8054 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, a
->rd
),
8055 vec_full_reg_offset(s
, a
->rn
),
8056 vec_full_reg_offset(s
, a
->rm
),
8057 vec_full_reg_offset(s
, a
->ra
),
8058 vsz
, vsz
, 0, gen_helper_gvec_usdot_b
);
8063 TRANS_FEAT(AESMC
, aa64_sve2_aes
, gen_gvec_ool_zz
,
8064 gen_helper_crypto_aesmc
, a
->rd
, a
->rd
, a
->decrypt
)
8066 TRANS_FEAT(AESE
, aa64_sve2_aes
, gen_gvec_ool_arg_zzz
,
8067 gen_helper_crypto_aese
, a
, false)
8068 TRANS_FEAT(AESD
, aa64_sve2_aes
, gen_gvec_ool_arg_zzz
,
8069 gen_helper_crypto_aese
, a
, true)
8071 TRANS_FEAT(SM4E
, aa64_sve2_sm4
, gen_gvec_ool_arg_zzz
,
8072 gen_helper_crypto_sm4e
, a
, 0)
8073 TRANS_FEAT(SM4EKEY
, aa64_sve2_sm4
, gen_gvec_ool_arg_zzz
,
8074 gen_helper_crypto_sm4ekey
, a
, 0)
8076 static bool trans_RAX1(DisasContext
*s
, arg_rrr_esz
*a
)
8078 if (!dc_isar_feature(aa64_sve2_sha3
, s
)) {
8081 if (sve_access_check(s
)) {
8082 gen_gvec_fn_zzz(s
, gen_gvec_rax1
, MO_64
, a
->rd
, a
->rn
, a
->rm
);
8087 static bool trans_FCVTNT_sh(DisasContext
*s
, arg_rpr_esz
*a
)
8089 if (!dc_isar_feature(aa64_sve2
, s
)) {
8092 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtnt_sh
);
8095 static bool trans_BFCVTNT(DisasContext
*s
, arg_rpr_esz
*a
)
8097 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
8100 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_bfcvtnt
);
8103 static bool trans_FCVTNT_ds(DisasContext
*s
, arg_rpr_esz
*a
)
8105 if (!dc_isar_feature(aa64_sve2
, s
)) {
8108 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtnt_ds
);
8111 static bool trans_FCVTLT_hs(DisasContext
*s
, arg_rpr_esz
*a
)
8113 if (!dc_isar_feature(aa64_sve2
, s
)) {
8116 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtlt_hs
);
8119 static bool trans_FCVTLT_sd(DisasContext
*s
, arg_rpr_esz
*a
)
8121 if (!dc_isar_feature(aa64_sve2
, s
)) {
8124 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtlt_sd
);
8127 static bool trans_FCVTX_ds(DisasContext
*s
, arg_rpr_esz
*a
)
8129 if (!dc_isar_feature(aa64_sve2
, s
)) {
8132 return do_frint_mode(s
, a
, float_round_to_odd
, gen_helper_sve_fcvt_ds
);
8135 static bool trans_FCVTXNT_ds(DisasContext
*s
, arg_rpr_esz
*a
)
8137 if (!dc_isar_feature(aa64_sve2
, s
)) {
8140 return do_frint_mode(s
, a
, float_round_to_odd
, gen_helper_sve2_fcvtnt_ds
);
8143 static bool trans_FLOGB(DisasContext
*s
, arg_rpr_esz
*a
)
8145 static gen_helper_gvec_3_ptr
* const fns
[] = {
8146 NULL
, gen_helper_flogb_h
,
8147 gen_helper_flogb_s
, gen_helper_flogb_d
8150 if (!dc_isar_feature(aa64_sve2
, s
) || fns
[a
->esz
] == NULL
) {
8153 if (sve_access_check(s
)) {
8155 fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
8156 unsigned vsz
= vec_full_reg_size(s
);
8158 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
8159 vec_full_reg_offset(s
, a
->rn
),
8160 pred_full_reg_offset(s
, a
->pg
),
8161 status
, vsz
, vsz
, 0, fns
[a
->esz
]);
8162 tcg_temp_free_ptr(status
);
8167 static bool do_FMLAL_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sub
, bool sel
)
8169 if (!dc_isar_feature(aa64_sve2
, s
)) {
8172 if (sve_access_check(s
)) {
8173 unsigned vsz
= vec_full_reg_size(s
);
8174 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
8175 vec_full_reg_offset(s
, a
->rn
),
8176 vec_full_reg_offset(s
, a
->rm
),
8177 vec_full_reg_offset(s
, a
->ra
),
8178 cpu_env
, vsz
, vsz
, (sel
<< 1) | sub
,
8179 gen_helper_sve2_fmlal_zzzw_s
);
8184 static bool trans_FMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8186 return do_FMLAL_zzzw(s
, a
, false, false);
8189 static bool trans_FMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8191 return do_FMLAL_zzzw(s
, a
, false, true);
8194 static bool trans_FMLSLB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8196 return do_FMLAL_zzzw(s
, a
, true, false);
8199 static bool trans_FMLSLT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8201 return do_FMLAL_zzzw(s
, a
, true, true);
8204 static bool do_FMLAL_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
, bool sub
, bool sel
)
8206 if (!dc_isar_feature(aa64_sve2
, s
)) {
8209 if (sve_access_check(s
)) {
8210 unsigned vsz
= vec_full_reg_size(s
);
8211 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
8212 vec_full_reg_offset(s
, a
->rn
),
8213 vec_full_reg_offset(s
, a
->rm
),
8214 vec_full_reg_offset(s
, a
->ra
),
8216 (a
->index
<< 2) | (sel
<< 1) | sub
,
8217 gen_helper_sve2_fmlal_zzxw_s
);
8222 static bool trans_FMLALB_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
8224 return do_FMLAL_zzxw(s
, a
, false, false);
8227 static bool trans_FMLALT_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
8229 return do_FMLAL_zzxw(s
, a
, false, true);
8232 static bool trans_FMLSLB_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
8234 return do_FMLAL_zzxw(s
, a
, true, false);
8237 static bool trans_FMLSLT_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
8239 return do_FMLAL_zzxw(s
, a
, true, true);
8242 TRANS_FEAT(SMMLA
, aa64_sve_i8mm
, gen_gvec_ool_arg_zzzz
,
8243 gen_helper_gvec_smmla_b
, a
, 0)
8244 TRANS_FEAT(USMMLA
, aa64_sve_i8mm
, gen_gvec_ool_arg_zzzz
,
8245 gen_helper_gvec_usmmla_b
, a
, 0)
8246 TRANS_FEAT(UMMLA
, aa64_sve_i8mm
, gen_gvec_ool_arg_zzzz
,
8247 gen_helper_gvec_ummla_b
, a
, 0)
8249 TRANS_FEAT(BFDOT_zzzz
, aa64_sve_bf16
, gen_gvec_ool_arg_zzzz
,
8250 gen_helper_gvec_bfdot
, a
, 0)
8251 TRANS_FEAT(BFDOT_zzxz
, aa64_sve_bf16
, gen_gvec_ool_arg_zzxz
,
8252 gen_helper_gvec_bfdot_idx
, a
)
8254 TRANS_FEAT(BFMMLA
, aa64_sve_bf16
, gen_gvec_ool_arg_zzzz
,
8255 gen_helper_gvec_bfmmla
, a
, 0)
8257 static bool do_BFMLAL_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
8259 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
8262 if (sve_access_check(s
)) {
8263 TCGv_ptr status
= fpstatus_ptr(FPST_FPCR
);
8264 unsigned vsz
= vec_full_reg_size(s
);
8266 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
8267 vec_full_reg_offset(s
, a
->rn
),
8268 vec_full_reg_offset(s
, a
->rm
),
8269 vec_full_reg_offset(s
, a
->ra
),
8270 status
, vsz
, vsz
, sel
,
8271 gen_helper_gvec_bfmlal
);
8272 tcg_temp_free_ptr(status
);
8277 static bool trans_BFMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8279 return do_BFMLAL_zzzw(s
, a
, false);
8282 static bool trans_BFMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8284 return do_BFMLAL_zzzw(s
, a
, true);
8287 static bool do_BFMLAL_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
, bool sel
)
8289 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
8292 if (sve_access_check(s
)) {
8293 TCGv_ptr status
= fpstatus_ptr(FPST_FPCR
);
8294 unsigned vsz
= vec_full_reg_size(s
);
8296 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
8297 vec_full_reg_offset(s
, a
->rn
),
8298 vec_full_reg_offset(s
, a
->rm
),
8299 vec_full_reg_offset(s
, a
->ra
),
8300 status
, vsz
, vsz
, (a
->index
<< 1) | sel
,
8301 gen_helper_gvec_bfmlal_idx
);
8302 tcg_temp_free_ptr(status
);
8307 static bool trans_BFMLALB_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
8309 return do_BFMLAL_zzxw(s
, a
, false);
8312 static bool trans_BFMLALT_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
8314 return do_BFMLAL_zzxw(s
, a
, true);