2 * AArch64 SVE translation
4 * Copyright (c) 2018 Linaro, Ltd
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
22 #include "exec/exec-all.h"
23 #include "tcg/tcg-op.h"
24 #include "tcg/tcg-op-gvec.h"
25 #include "tcg/tcg-gvec-desc.h"
28 #include "translate.h"
29 #include "internals.h"
30 #include "exec/helper-proto.h"
31 #include "exec/helper-gen.h"
33 #include "translate-a64.h"
34 #include "fpu/softfloat.h"
37 typedef void GVecGen2sFn(unsigned, uint32_t, uint32_t,
38 TCGv_i64
, uint32_t, uint32_t);
40 typedef void gen_helper_gvec_flags_3(TCGv_i32
, TCGv_ptr
, TCGv_ptr
,
42 typedef void gen_helper_gvec_flags_4(TCGv_i32
, TCGv_ptr
, TCGv_ptr
,
43 TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
45 typedef void gen_helper_gvec_mem(TCGv_env
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
46 typedef void gen_helper_gvec_mem_scatter(TCGv_env
, TCGv_ptr
, TCGv_ptr
,
47 TCGv_ptr
, TCGv_i64
, TCGv_i32
);
50 * Helpers for extracting complex instruction fields.
53 /* See e.g. ASR (immediate, predicated).
54 * Returns -1 for unallocated encoding; diagnose later.
56 static int tszimm_esz(DisasContext
*s
, int x
)
58 x
>>= 3; /* discard imm3 */
62 static int tszimm_shr(DisasContext
*s
, int x
)
64 return (16 << tszimm_esz(s
, x
)) - x
;
67 /* See e.g. LSL (immediate, predicated). */
68 static int tszimm_shl(DisasContext
*s
, int x
)
70 return x
- (8 << tszimm_esz(s
, x
));
73 /* The SH bit is in bit 8. Extract the low 8 and shift. */
74 static inline int expand_imm_sh8s(DisasContext
*s
, int x
)
76 return (int8_t)x
<< (x
& 0x100 ? 8 : 0);
79 static inline int expand_imm_sh8u(DisasContext
*s
, int x
)
81 return (uint8_t)x
<< (x
& 0x100 ? 8 : 0);
84 /* Convert a 2-bit memory size (msz) to a 4-bit data type (dtype)
85 * with unsigned data. C.f. SVE Memory Contiguous Load Group.
87 static inline int msz_dtype(DisasContext
*s
, int msz
)
89 static const uint8_t dtype
[4] = { 0, 5, 10, 15 };
94 * Include the generated decoder.
97 #include "decode-sve.c.inc"
100 * Implement all of the translator functions referenced by the decoder.
103 /* Return the offset info CPUARMState of the predicate vector register Pn.
104 * Note for this purpose, FFR is P16.
106 static inline int pred_full_reg_offset(DisasContext
*s
, int regno
)
108 return offsetof(CPUARMState
, vfp
.pregs
[regno
]);
111 /* Return the byte size of the whole predicate register, VL / 64. */
112 static inline int pred_full_reg_size(DisasContext
*s
)
114 return s
->sve_len
>> 3;
117 /* Round up the size of a register to a size allowed by
118 * the tcg vector infrastructure. Any operation which uses this
119 * size may assume that the bits above pred_full_reg_size are zero,
120 * and must leave them the same way.
122 * Note that this is not needed for the vector registers as they
123 * are always properly sized for tcg vectors.
125 static int size_for_gvec(int size
)
130 return QEMU_ALIGN_UP(size
, 16);
134 static int pred_gvec_reg_size(DisasContext
*s
)
136 return size_for_gvec(pred_full_reg_size(s
));
139 /* Invoke an out-of-line helper on 2 Zregs. */
140 static void gen_gvec_ool_zz(DisasContext
*s
, gen_helper_gvec_2
*fn
,
141 int rd
, int rn
, int data
)
143 unsigned vsz
= vec_full_reg_size(s
);
144 tcg_gen_gvec_2_ool(vec_full_reg_offset(s
, rd
),
145 vec_full_reg_offset(s
, rn
),
149 /* Invoke an out-of-line helper on 3 Zregs. */
150 static void gen_gvec_ool_zzz(DisasContext
*s
, gen_helper_gvec_3
*fn
,
151 int rd
, int rn
, int rm
, int data
)
153 unsigned vsz
= vec_full_reg_size(s
);
154 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
155 vec_full_reg_offset(s
, rn
),
156 vec_full_reg_offset(s
, rm
),
160 /* Invoke an out-of-line helper on 4 Zregs. */
161 static void gen_gvec_ool_zzzz(DisasContext
*s
, gen_helper_gvec_4
*fn
,
162 int rd
, int rn
, int rm
, int ra
, int data
)
164 unsigned vsz
= vec_full_reg_size(s
);
165 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
166 vec_full_reg_offset(s
, rn
),
167 vec_full_reg_offset(s
, rm
),
168 vec_full_reg_offset(s
, ra
),
172 /* Invoke an out-of-line helper on 2 Zregs and a predicate. */
173 static void gen_gvec_ool_zzp(DisasContext
*s
, gen_helper_gvec_3
*fn
,
174 int rd
, int rn
, int pg
, int data
)
176 unsigned vsz
= vec_full_reg_size(s
);
177 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
178 vec_full_reg_offset(s
, rn
),
179 pred_full_reg_offset(s
, pg
),
183 /* Invoke an out-of-line helper on 3 Zregs and a predicate. */
184 static void gen_gvec_ool_zzzp(DisasContext
*s
, gen_helper_gvec_4
*fn
,
185 int rd
, int rn
, int rm
, int pg
, int data
)
187 unsigned vsz
= vec_full_reg_size(s
);
188 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
189 vec_full_reg_offset(s
, rn
),
190 vec_full_reg_offset(s
, rm
),
191 pred_full_reg_offset(s
, pg
),
195 /* Invoke a vector expander on two Zregs. */
196 static void gen_gvec_fn_zz(DisasContext
*s
, GVecGen2Fn
*gvec_fn
,
197 int esz
, int rd
, int rn
)
199 unsigned vsz
= vec_full_reg_size(s
);
200 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
201 vec_full_reg_offset(s
, rn
), vsz
, vsz
);
204 /* Invoke a vector expander on three Zregs. */
205 static void gen_gvec_fn_zzz(DisasContext
*s
, GVecGen3Fn
*gvec_fn
,
206 int esz
, int rd
, int rn
, int rm
)
208 unsigned vsz
= vec_full_reg_size(s
);
209 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
210 vec_full_reg_offset(s
, rn
),
211 vec_full_reg_offset(s
, rm
), vsz
, vsz
);
214 /* Invoke a vector expander on four Zregs. */
215 static void gen_gvec_fn_zzzz(DisasContext
*s
, GVecGen4Fn
*gvec_fn
,
216 int esz
, int rd
, int rn
, int rm
, int ra
)
218 unsigned vsz
= vec_full_reg_size(s
);
219 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
220 vec_full_reg_offset(s
, rn
),
221 vec_full_reg_offset(s
, rm
),
222 vec_full_reg_offset(s
, ra
), vsz
, vsz
);
225 /* Invoke a vector move on two Zregs. */
226 static bool do_mov_z(DisasContext
*s
, int rd
, int rn
)
228 if (sve_access_check(s
)) {
229 gen_gvec_fn_zz(s
, tcg_gen_gvec_mov
, MO_8
, rd
, rn
);
234 /* Initialize a Zreg with replications of a 64-bit immediate. */
235 static void do_dupi_z(DisasContext
*s
, int rd
, uint64_t word
)
237 unsigned vsz
= vec_full_reg_size(s
);
238 tcg_gen_gvec_dup_imm(MO_64
, vec_full_reg_offset(s
, rd
), vsz
, vsz
, word
);
241 /* Invoke a vector expander on three Pregs. */
242 static void gen_gvec_fn_ppp(DisasContext
*s
, GVecGen3Fn
*gvec_fn
,
243 int rd
, int rn
, int rm
)
245 unsigned psz
= pred_gvec_reg_size(s
);
246 gvec_fn(MO_64
, pred_full_reg_offset(s
, rd
),
247 pred_full_reg_offset(s
, rn
),
248 pred_full_reg_offset(s
, rm
), psz
, psz
);
251 /* Invoke a vector move on two Pregs. */
252 static bool do_mov_p(DisasContext
*s
, int rd
, int rn
)
254 if (sve_access_check(s
)) {
255 unsigned psz
= pred_gvec_reg_size(s
);
256 tcg_gen_gvec_mov(MO_8
, pred_full_reg_offset(s
, rd
),
257 pred_full_reg_offset(s
, rn
), psz
, psz
);
262 /* Set the cpu flags as per a return from an SVE helper. */
263 static void do_pred_flags(TCGv_i32 t
)
265 tcg_gen_mov_i32(cpu_NF
, t
);
266 tcg_gen_andi_i32(cpu_ZF
, t
, 2);
267 tcg_gen_andi_i32(cpu_CF
, t
, 1);
268 tcg_gen_movi_i32(cpu_VF
, 0);
271 /* Subroutines computing the ARM PredTest psuedofunction. */
272 static void do_predtest1(TCGv_i64 d
, TCGv_i64 g
)
274 TCGv_i32 t
= tcg_temp_new_i32();
276 gen_helper_sve_predtest1(t
, d
, g
);
278 tcg_temp_free_i32(t
);
281 static void do_predtest(DisasContext
*s
, int dofs
, int gofs
, int words
)
283 TCGv_ptr dptr
= tcg_temp_new_ptr();
284 TCGv_ptr gptr
= tcg_temp_new_ptr();
285 TCGv_i32 t
= tcg_temp_new_i32();
287 tcg_gen_addi_ptr(dptr
, cpu_env
, dofs
);
288 tcg_gen_addi_ptr(gptr
, cpu_env
, gofs
);
290 gen_helper_sve_predtest(t
, dptr
, gptr
, tcg_constant_i32(words
));
291 tcg_temp_free_ptr(dptr
);
292 tcg_temp_free_ptr(gptr
);
295 tcg_temp_free_i32(t
);
298 /* For each element size, the bits within a predicate word that are active. */
299 const uint64_t pred_esz_masks
[4] = {
300 0xffffffffffffffffull
, 0x5555555555555555ull
,
301 0x1111111111111111ull
, 0x0101010101010101ull
305 *** SVE Logical - Unpredicated Group
308 static bool do_zzz_fn(DisasContext
*s
, arg_rrr_esz
*a
, GVecGen3Fn
*gvec_fn
)
310 if (sve_access_check(s
)) {
311 gen_gvec_fn_zzz(s
, gvec_fn
, a
->esz
, a
->rd
, a
->rn
, a
->rm
);
316 static bool trans_AND_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
318 return do_zzz_fn(s
, a
, tcg_gen_gvec_and
);
321 static bool trans_ORR_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
323 return do_zzz_fn(s
, a
, tcg_gen_gvec_or
);
326 static bool trans_EOR_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
328 return do_zzz_fn(s
, a
, tcg_gen_gvec_xor
);
331 static bool trans_BIC_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
333 return do_zzz_fn(s
, a
, tcg_gen_gvec_andc
);
336 static void gen_xar8_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
338 TCGv_i64 t
= tcg_temp_new_i64();
339 uint64_t mask
= dup_const(MO_8
, 0xff >> sh
);
341 tcg_gen_xor_i64(t
, n
, m
);
342 tcg_gen_shri_i64(d
, t
, sh
);
343 tcg_gen_shli_i64(t
, t
, 8 - sh
);
344 tcg_gen_andi_i64(d
, d
, mask
);
345 tcg_gen_andi_i64(t
, t
, ~mask
);
346 tcg_gen_or_i64(d
, d
, t
);
347 tcg_temp_free_i64(t
);
350 static void gen_xar16_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
352 TCGv_i64 t
= tcg_temp_new_i64();
353 uint64_t mask
= dup_const(MO_16
, 0xffff >> sh
);
355 tcg_gen_xor_i64(t
, n
, m
);
356 tcg_gen_shri_i64(d
, t
, sh
);
357 tcg_gen_shli_i64(t
, t
, 16 - sh
);
358 tcg_gen_andi_i64(d
, d
, mask
);
359 tcg_gen_andi_i64(t
, t
, ~mask
);
360 tcg_gen_or_i64(d
, d
, t
);
361 tcg_temp_free_i64(t
);
364 static void gen_xar_i32(TCGv_i32 d
, TCGv_i32 n
, TCGv_i32 m
, int32_t sh
)
366 tcg_gen_xor_i32(d
, n
, m
);
367 tcg_gen_rotri_i32(d
, d
, sh
);
370 static void gen_xar_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
372 tcg_gen_xor_i64(d
, n
, m
);
373 tcg_gen_rotri_i64(d
, d
, sh
);
376 static void gen_xar_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
377 TCGv_vec m
, int64_t sh
)
379 tcg_gen_xor_vec(vece
, d
, n
, m
);
380 tcg_gen_rotri_vec(vece
, d
, d
, sh
);
383 void gen_gvec_xar(unsigned vece
, uint32_t rd_ofs
, uint32_t rn_ofs
,
384 uint32_t rm_ofs
, int64_t shift
,
385 uint32_t opr_sz
, uint32_t max_sz
)
387 static const TCGOpcode vecop
[] = { INDEX_op_rotli_vec
, 0 };
388 static const GVecGen3i ops
[4] = {
389 { .fni8
= gen_xar8_i64
,
391 .fno
= gen_helper_sve2_xar_b
,
394 { .fni8
= gen_xar16_i64
,
396 .fno
= gen_helper_sve2_xar_h
,
399 { .fni4
= gen_xar_i32
,
401 .fno
= gen_helper_sve2_xar_s
,
404 { .fni8
= gen_xar_i64
,
406 .fno
= gen_helper_gvec_xar_d
,
410 int esize
= 8 << vece
;
412 /* The SVE2 range is 1 .. esize; the AdvSIMD range is 0 .. esize-1. */
413 tcg_debug_assert(shift
>= 0);
414 tcg_debug_assert(shift
<= esize
);
418 /* xar with no rotate devolves to xor. */
419 tcg_gen_gvec_xor(vece
, rd_ofs
, rn_ofs
, rm_ofs
, opr_sz
, max_sz
);
421 tcg_gen_gvec_3i(rd_ofs
, rn_ofs
, rm_ofs
, opr_sz
, max_sz
,
426 static bool trans_XAR(DisasContext
*s
, arg_rrri_esz
*a
)
428 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
431 if (sve_access_check(s
)) {
432 unsigned vsz
= vec_full_reg_size(s
);
433 gen_gvec_xar(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
434 vec_full_reg_offset(s
, a
->rn
),
435 vec_full_reg_offset(s
, a
->rm
), a
->imm
, vsz
, vsz
);
440 static bool do_sve2_zzzz_fn(DisasContext
*s
, arg_rrrr_esz
*a
, GVecGen4Fn
*fn
)
442 if (!dc_isar_feature(aa64_sve2
, s
)) {
445 if (sve_access_check(s
)) {
446 gen_gvec_fn_zzzz(s
, fn
, a
->esz
, a
->rd
, a
->rn
, a
->rm
, a
->ra
);
451 static void gen_eor3_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
453 tcg_gen_xor_i64(d
, n
, m
);
454 tcg_gen_xor_i64(d
, d
, k
);
457 static void gen_eor3_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
458 TCGv_vec m
, TCGv_vec k
)
460 tcg_gen_xor_vec(vece
, d
, n
, m
);
461 tcg_gen_xor_vec(vece
, d
, d
, k
);
464 static void gen_eor3(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
465 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
467 static const GVecGen4 op
= {
468 .fni8
= gen_eor3_i64
,
469 .fniv
= gen_eor3_vec
,
470 .fno
= gen_helper_sve2_eor3
,
472 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
474 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
477 static bool trans_EOR3(DisasContext
*s
, arg_rrrr_esz
*a
)
479 return do_sve2_zzzz_fn(s
, a
, gen_eor3
);
482 static void gen_bcax_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
484 tcg_gen_andc_i64(d
, m
, k
);
485 tcg_gen_xor_i64(d
, d
, n
);
488 static void gen_bcax_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
489 TCGv_vec m
, TCGv_vec k
)
491 tcg_gen_andc_vec(vece
, d
, m
, k
);
492 tcg_gen_xor_vec(vece
, d
, d
, n
);
495 static void gen_bcax(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
496 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
498 static const GVecGen4 op
= {
499 .fni8
= gen_bcax_i64
,
500 .fniv
= gen_bcax_vec
,
501 .fno
= gen_helper_sve2_bcax
,
503 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
505 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
508 static bool trans_BCAX(DisasContext
*s
, arg_rrrr_esz
*a
)
510 return do_sve2_zzzz_fn(s
, a
, gen_bcax
);
513 static void gen_bsl(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
514 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
516 /* BSL differs from the generic bitsel in argument ordering. */
517 tcg_gen_gvec_bitsel(vece
, d
, a
, n
, m
, oprsz
, maxsz
);
520 static bool trans_BSL(DisasContext
*s
, arg_rrrr_esz
*a
)
522 return do_sve2_zzzz_fn(s
, a
, gen_bsl
);
525 static void gen_bsl1n_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
527 tcg_gen_andc_i64(n
, k
, n
);
528 tcg_gen_andc_i64(m
, m
, k
);
529 tcg_gen_or_i64(d
, n
, m
);
532 static void gen_bsl1n_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
533 TCGv_vec m
, TCGv_vec k
)
535 if (TCG_TARGET_HAS_bitsel_vec
) {
536 tcg_gen_not_vec(vece
, n
, n
);
537 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
539 tcg_gen_andc_vec(vece
, n
, k
, n
);
540 tcg_gen_andc_vec(vece
, m
, m
, k
);
541 tcg_gen_or_vec(vece
, d
, n
, m
);
545 static void gen_bsl1n(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
546 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
548 static const GVecGen4 op
= {
549 .fni8
= gen_bsl1n_i64
,
550 .fniv
= gen_bsl1n_vec
,
551 .fno
= gen_helper_sve2_bsl1n
,
553 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
555 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
558 static bool trans_BSL1N(DisasContext
*s
, arg_rrrr_esz
*a
)
560 return do_sve2_zzzz_fn(s
, a
, gen_bsl1n
);
563 static void gen_bsl2n_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
566 * Z[dn] = (n & k) | (~m & ~k)
569 tcg_gen_and_i64(n
, n
, k
);
570 if (TCG_TARGET_HAS_orc_i64
) {
571 tcg_gen_or_i64(m
, m
, k
);
572 tcg_gen_orc_i64(d
, n
, m
);
574 tcg_gen_nor_i64(m
, m
, k
);
575 tcg_gen_or_i64(d
, n
, m
);
579 static void gen_bsl2n_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
580 TCGv_vec m
, TCGv_vec k
)
582 if (TCG_TARGET_HAS_bitsel_vec
) {
583 tcg_gen_not_vec(vece
, m
, m
);
584 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
586 tcg_gen_and_vec(vece
, n
, n
, k
);
587 tcg_gen_or_vec(vece
, m
, m
, k
);
588 tcg_gen_orc_vec(vece
, d
, n
, m
);
592 static void gen_bsl2n(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
593 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
595 static const GVecGen4 op
= {
596 .fni8
= gen_bsl2n_i64
,
597 .fniv
= gen_bsl2n_vec
,
598 .fno
= gen_helper_sve2_bsl2n
,
600 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
602 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
605 static bool trans_BSL2N(DisasContext
*s
, arg_rrrr_esz
*a
)
607 return do_sve2_zzzz_fn(s
, a
, gen_bsl2n
);
610 static void gen_nbsl_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
612 tcg_gen_and_i64(n
, n
, k
);
613 tcg_gen_andc_i64(m
, m
, k
);
614 tcg_gen_nor_i64(d
, n
, m
);
617 static void gen_nbsl_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
618 TCGv_vec m
, TCGv_vec k
)
620 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
621 tcg_gen_not_vec(vece
, d
, d
);
624 static void gen_nbsl(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
625 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
627 static const GVecGen4 op
= {
628 .fni8
= gen_nbsl_i64
,
629 .fniv
= gen_nbsl_vec
,
630 .fno
= gen_helper_sve2_nbsl
,
632 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
634 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
637 static bool trans_NBSL(DisasContext
*s
, arg_rrrr_esz
*a
)
639 return do_sve2_zzzz_fn(s
, a
, gen_nbsl
);
643 *** SVE Integer Arithmetic - Unpredicated Group
646 static bool trans_ADD_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
648 return do_zzz_fn(s
, a
, tcg_gen_gvec_add
);
651 static bool trans_SUB_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
653 return do_zzz_fn(s
, a
, tcg_gen_gvec_sub
);
656 static bool trans_SQADD_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
658 return do_zzz_fn(s
, a
, tcg_gen_gvec_ssadd
);
661 static bool trans_SQSUB_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
663 return do_zzz_fn(s
, a
, tcg_gen_gvec_sssub
);
666 static bool trans_UQADD_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
668 return do_zzz_fn(s
, a
, tcg_gen_gvec_usadd
);
671 static bool trans_UQSUB_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
673 return do_zzz_fn(s
, a
, tcg_gen_gvec_ussub
);
677 *** SVE Integer Arithmetic - Binary Predicated Group
680 static bool do_zpzz_ool(DisasContext
*s
, arg_rprr_esz
*a
, gen_helper_gvec_4
*fn
)
685 if (sve_access_check(s
)) {
686 gen_gvec_ool_zzzp(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->pg
, 0);
691 /* Select active elememnts from Zn and inactive elements from Zm,
692 * storing the result in Zd.
694 static void do_sel_z(DisasContext
*s
, int rd
, int rn
, int rm
, int pg
, int esz
)
696 static gen_helper_gvec_4
* const fns
[4] = {
697 gen_helper_sve_sel_zpzz_b
, gen_helper_sve_sel_zpzz_h
,
698 gen_helper_sve_sel_zpzz_s
, gen_helper_sve_sel_zpzz_d
700 gen_gvec_ool_zzzp(s
, fns
[esz
], rd
, rn
, rm
, pg
, 0);
703 #define DO_ZPZZ(NAME, name) \
704 static bool trans_##NAME##_zpzz(DisasContext *s, arg_rprr_esz *a) \
706 static gen_helper_gvec_4 * const fns[4] = { \
707 gen_helper_sve_##name##_zpzz_b, gen_helper_sve_##name##_zpzz_h, \
708 gen_helper_sve_##name##_zpzz_s, gen_helper_sve_##name##_zpzz_d, \
710 return do_zpzz_ool(s, a, fns[a->esz]); \
729 DO_ZPZZ(SMULH
, smulh
)
730 DO_ZPZZ(UMULH
, umulh
)
736 static bool trans_SDIV_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
738 static gen_helper_gvec_4
* const fns
[4] = {
739 NULL
, NULL
, gen_helper_sve_sdiv_zpzz_s
, gen_helper_sve_sdiv_zpzz_d
741 return do_zpzz_ool(s
, a
, fns
[a
->esz
]);
744 static bool trans_UDIV_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
746 static gen_helper_gvec_4
* const fns
[4] = {
747 NULL
, NULL
, gen_helper_sve_udiv_zpzz_s
, gen_helper_sve_udiv_zpzz_d
749 return do_zpzz_ool(s
, a
, fns
[a
->esz
]);
752 static bool trans_SEL_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
754 if (sve_access_check(s
)) {
755 do_sel_z(s
, a
->rd
, a
->rn
, a
->rm
, a
->pg
, a
->esz
);
763 *** SVE Integer Arithmetic - Unary Predicated Group
766 static bool do_zpz_ool(DisasContext
*s
, arg_rpr_esz
*a
, gen_helper_gvec_3
*fn
)
771 if (sve_access_check(s
)) {
772 gen_gvec_ool_zzp(s
, fn
, a
->rd
, a
->rn
, a
->pg
, 0);
777 #define DO_ZPZ(NAME, name) \
778 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
780 static gen_helper_gvec_3 * const fns[4] = { \
781 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
782 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
784 return do_zpz_ool(s, a, fns[a->esz]); \
789 DO_ZPZ(CNT_zpz
, cnt_zpz
)
791 DO_ZPZ(NOT_zpz
, not_zpz
)
795 static bool trans_FABS(DisasContext
*s
, arg_rpr_esz
*a
)
797 static gen_helper_gvec_3
* const fns
[4] = {
799 gen_helper_sve_fabs_h
,
800 gen_helper_sve_fabs_s
,
801 gen_helper_sve_fabs_d
803 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
806 static bool trans_FNEG(DisasContext
*s
, arg_rpr_esz
*a
)
808 static gen_helper_gvec_3
* const fns
[4] = {
810 gen_helper_sve_fneg_h
,
811 gen_helper_sve_fneg_s
,
812 gen_helper_sve_fneg_d
814 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
817 static bool trans_SXTB(DisasContext
*s
, arg_rpr_esz
*a
)
819 static gen_helper_gvec_3
* const fns
[4] = {
821 gen_helper_sve_sxtb_h
,
822 gen_helper_sve_sxtb_s
,
823 gen_helper_sve_sxtb_d
825 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
828 static bool trans_UXTB(DisasContext
*s
, arg_rpr_esz
*a
)
830 static gen_helper_gvec_3
* const fns
[4] = {
832 gen_helper_sve_uxtb_h
,
833 gen_helper_sve_uxtb_s
,
834 gen_helper_sve_uxtb_d
836 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
839 static bool trans_SXTH(DisasContext
*s
, arg_rpr_esz
*a
)
841 static gen_helper_gvec_3
* const fns
[4] = {
843 gen_helper_sve_sxth_s
,
844 gen_helper_sve_sxth_d
846 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
849 static bool trans_UXTH(DisasContext
*s
, arg_rpr_esz
*a
)
851 static gen_helper_gvec_3
* const fns
[4] = {
853 gen_helper_sve_uxth_s
,
854 gen_helper_sve_uxth_d
856 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
859 static bool trans_SXTW(DisasContext
*s
, arg_rpr_esz
*a
)
861 return do_zpz_ool(s
, a
, a
->esz
== 3 ? gen_helper_sve_sxtw_d
: NULL
);
864 static bool trans_UXTW(DisasContext
*s
, arg_rpr_esz
*a
)
866 return do_zpz_ool(s
, a
, a
->esz
== 3 ? gen_helper_sve_uxtw_d
: NULL
);
872 *** SVE Integer Reduction Group
875 typedef void gen_helper_gvec_reduc(TCGv_i64
, TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
876 static bool do_vpz_ool(DisasContext
*s
, arg_rpr_esz
*a
,
877 gen_helper_gvec_reduc
*fn
)
879 unsigned vsz
= vec_full_reg_size(s
);
887 if (!sve_access_check(s
)) {
891 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
892 temp
= tcg_temp_new_i64();
893 t_zn
= tcg_temp_new_ptr();
894 t_pg
= tcg_temp_new_ptr();
896 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
897 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
898 fn(temp
, t_zn
, t_pg
, desc
);
899 tcg_temp_free_ptr(t_zn
);
900 tcg_temp_free_ptr(t_pg
);
901 tcg_temp_free_i32(desc
);
903 write_fp_dreg(s
, a
->rd
, temp
);
904 tcg_temp_free_i64(temp
);
908 #define DO_VPZ(NAME, name) \
909 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
911 static gen_helper_gvec_reduc * const fns[4] = { \
912 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
913 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
915 return do_vpz_ool(s, a, fns[a->esz]); \
928 static bool trans_SADDV(DisasContext
*s
, arg_rpr_esz
*a
)
930 static gen_helper_gvec_reduc
* const fns
[4] = {
931 gen_helper_sve_saddv_b
, gen_helper_sve_saddv_h
,
932 gen_helper_sve_saddv_s
, NULL
934 return do_vpz_ool(s
, a
, fns
[a
->esz
]);
940 *** SVE Shift by Immediate - Predicated Group
944 * Copy Zn into Zd, storing zeros into inactive elements.
945 * If invert, store zeros into the active elements.
947 static bool do_movz_zpz(DisasContext
*s
, int rd
, int rn
, int pg
,
948 int esz
, bool invert
)
950 static gen_helper_gvec_3
* const fns
[4] = {
951 gen_helper_sve_movz_b
, gen_helper_sve_movz_h
,
952 gen_helper_sve_movz_s
, gen_helper_sve_movz_d
,
955 if (sve_access_check(s
)) {
956 gen_gvec_ool_zzp(s
, fns
[esz
], rd
, rn
, pg
, invert
);
961 static bool do_zpzi_ool(DisasContext
*s
, arg_rpri_esz
*a
,
962 gen_helper_gvec_3
*fn
)
964 if (sve_access_check(s
)) {
965 gen_gvec_ool_zzp(s
, fn
, a
->rd
, a
->rn
, a
->pg
, a
->imm
);
970 static bool trans_ASR_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
972 static gen_helper_gvec_3
* const fns
[4] = {
973 gen_helper_sve_asr_zpzi_b
, gen_helper_sve_asr_zpzi_h
,
974 gen_helper_sve_asr_zpzi_s
, gen_helper_sve_asr_zpzi_d
,
977 /* Invalid tsz encoding -- see tszimm_esz. */
980 /* Shift by element size is architecturally valid. For
981 arithmetic right-shift, it's the same as by one less. */
982 a
->imm
= MIN(a
->imm
, (8 << a
->esz
) - 1);
983 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
986 static bool trans_LSR_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
988 static gen_helper_gvec_3
* const fns
[4] = {
989 gen_helper_sve_lsr_zpzi_b
, gen_helper_sve_lsr_zpzi_h
,
990 gen_helper_sve_lsr_zpzi_s
, gen_helper_sve_lsr_zpzi_d
,
995 /* Shift by element size is architecturally valid.
996 For logical shifts, it is a zeroing operation. */
997 if (a
->imm
>= (8 << a
->esz
)) {
998 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
1000 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1004 static bool trans_LSL_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
1006 static gen_helper_gvec_3
* const fns
[4] = {
1007 gen_helper_sve_lsl_zpzi_b
, gen_helper_sve_lsl_zpzi_h
,
1008 gen_helper_sve_lsl_zpzi_s
, gen_helper_sve_lsl_zpzi_d
,
1013 /* Shift by element size is architecturally valid.
1014 For logical shifts, it is a zeroing operation. */
1015 if (a
->imm
>= (8 << a
->esz
)) {
1016 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
1018 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1022 static bool trans_ASRD(DisasContext
*s
, arg_rpri_esz
*a
)
1024 static gen_helper_gvec_3
* const fns
[4] = {
1025 gen_helper_sve_asrd_b
, gen_helper_sve_asrd_h
,
1026 gen_helper_sve_asrd_s
, gen_helper_sve_asrd_d
,
1031 /* Shift by element size is architecturally valid. For arithmetic
1032 right shift for division, it is a zeroing operation. */
1033 if (a
->imm
>= (8 << a
->esz
)) {
1034 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
1036 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1040 static bool trans_SQSHL_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
1042 static gen_helper_gvec_3
* const fns
[4] = {
1043 gen_helper_sve2_sqshl_zpzi_b
, gen_helper_sve2_sqshl_zpzi_h
,
1044 gen_helper_sve2_sqshl_zpzi_s
, gen_helper_sve2_sqshl_zpzi_d
,
1046 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
1049 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1052 static bool trans_UQSHL_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
1054 static gen_helper_gvec_3
* const fns
[4] = {
1055 gen_helper_sve2_uqshl_zpzi_b
, gen_helper_sve2_uqshl_zpzi_h
,
1056 gen_helper_sve2_uqshl_zpzi_s
, gen_helper_sve2_uqshl_zpzi_d
,
1058 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
1061 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1064 static bool trans_SRSHR(DisasContext
*s
, arg_rpri_esz
*a
)
1066 static gen_helper_gvec_3
* const fns
[4] = {
1067 gen_helper_sve2_srshr_b
, gen_helper_sve2_srshr_h
,
1068 gen_helper_sve2_srshr_s
, gen_helper_sve2_srshr_d
,
1070 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
1073 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1076 static bool trans_URSHR(DisasContext
*s
, arg_rpri_esz
*a
)
1078 static gen_helper_gvec_3
* const fns
[4] = {
1079 gen_helper_sve2_urshr_b
, gen_helper_sve2_urshr_h
,
1080 gen_helper_sve2_urshr_s
, gen_helper_sve2_urshr_d
,
1082 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
1085 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1088 static bool trans_SQSHLU(DisasContext
*s
, arg_rpri_esz
*a
)
1090 static gen_helper_gvec_3
* const fns
[4] = {
1091 gen_helper_sve2_sqshlu_b
, gen_helper_sve2_sqshlu_h
,
1092 gen_helper_sve2_sqshlu_s
, gen_helper_sve2_sqshlu_d
,
1094 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
1097 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1101 *** SVE Bitwise Shift - Predicated Group
1104 #define DO_ZPZW(NAME, name) \
1105 static bool trans_##NAME##_zpzw(DisasContext *s, arg_rprr_esz *a) \
1107 static gen_helper_gvec_4 * const fns[3] = { \
1108 gen_helper_sve_##name##_zpzw_b, gen_helper_sve_##name##_zpzw_h, \
1109 gen_helper_sve_##name##_zpzw_s, \
1111 if (a->esz < 0 || a->esz >= 3) { \
1114 return do_zpzz_ool(s, a, fns[a->esz]); \
1124 *** SVE Bitwise Shift - Unpredicated Group
1127 static bool do_shift_imm(DisasContext
*s
, arg_rri_esz
*a
, bool asr
,
1128 void (*gvec_fn
)(unsigned, uint32_t, uint32_t,
1129 int64_t, uint32_t, uint32_t))
1132 /* Invalid tsz encoding -- see tszimm_esz. */
1135 if (sve_access_check(s
)) {
1136 unsigned vsz
= vec_full_reg_size(s
);
1137 /* Shift by element size is architecturally valid. For
1138 arithmetic right-shift, it's the same as by one less.
1139 Otherwise it is a zeroing operation. */
1140 if (a
->imm
>= 8 << a
->esz
) {
1142 a
->imm
= (8 << a
->esz
) - 1;
1144 do_dupi_z(s
, a
->rd
, 0);
1148 gvec_fn(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
1149 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
1154 static bool trans_ASR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
1156 return do_shift_imm(s
, a
, true, tcg_gen_gvec_sari
);
1159 static bool trans_LSR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
1161 return do_shift_imm(s
, a
, false, tcg_gen_gvec_shri
);
1164 static bool trans_LSL_zzi(DisasContext
*s
, arg_rri_esz
*a
)
1166 return do_shift_imm(s
, a
, false, tcg_gen_gvec_shli
);
1169 static bool do_zzw_ool(DisasContext
*s
, arg_rrr_esz
*a
, gen_helper_gvec_3
*fn
)
1174 if (sve_access_check(s
)) {
1175 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, 0);
1180 #define DO_ZZW(NAME, name) \
1181 static bool trans_##NAME##_zzw(DisasContext *s, arg_rrr_esz *a) \
1183 static gen_helper_gvec_3 * const fns[4] = { \
1184 gen_helper_sve_##name##_zzw_b, gen_helper_sve_##name##_zzw_h, \
1185 gen_helper_sve_##name##_zzw_s, NULL \
1187 return do_zzw_ool(s, a, fns[a->esz]); \
1197 *** SVE Integer Multiply-Add Group
1200 static bool do_zpzzz_ool(DisasContext
*s
, arg_rprrr_esz
*a
,
1201 gen_helper_gvec_5
*fn
)
1203 if (sve_access_check(s
)) {
1204 unsigned vsz
= vec_full_reg_size(s
);
1205 tcg_gen_gvec_5_ool(vec_full_reg_offset(s
, a
->rd
),
1206 vec_full_reg_offset(s
, a
->ra
),
1207 vec_full_reg_offset(s
, a
->rn
),
1208 vec_full_reg_offset(s
, a
->rm
),
1209 pred_full_reg_offset(s
, a
->pg
),
1215 #define DO_ZPZZZ(NAME, name) \
1216 static bool trans_##NAME(DisasContext *s, arg_rprrr_esz *a) \
1218 static gen_helper_gvec_5 * const fns[4] = { \
1219 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
1220 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
1222 return do_zpzzz_ool(s, a, fns[a->esz]); \
1231 *** SVE Index Generation Group
1234 static void do_index(DisasContext
*s
, int esz
, int rd
,
1235 TCGv_i64 start
, TCGv_i64 incr
)
1237 unsigned vsz
= vec_full_reg_size(s
);
1238 TCGv_i32 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
1239 TCGv_ptr t_zd
= tcg_temp_new_ptr();
1241 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, rd
));
1243 gen_helper_sve_index_d(t_zd
, start
, incr
, desc
);
1245 typedef void index_fn(TCGv_ptr
, TCGv_i32
, TCGv_i32
, TCGv_i32
);
1246 static index_fn
* const fns
[3] = {
1247 gen_helper_sve_index_b
,
1248 gen_helper_sve_index_h
,
1249 gen_helper_sve_index_s
,
1251 TCGv_i32 s32
= tcg_temp_new_i32();
1252 TCGv_i32 i32
= tcg_temp_new_i32();
1254 tcg_gen_extrl_i64_i32(s32
, start
);
1255 tcg_gen_extrl_i64_i32(i32
, incr
);
1256 fns
[esz
](t_zd
, s32
, i32
, desc
);
1258 tcg_temp_free_i32(s32
);
1259 tcg_temp_free_i32(i32
);
1261 tcg_temp_free_ptr(t_zd
);
1262 tcg_temp_free_i32(desc
);
1265 static bool trans_INDEX_ii(DisasContext
*s
, arg_INDEX_ii
*a
)
1267 if (sve_access_check(s
)) {
1268 TCGv_i64 start
= tcg_constant_i64(a
->imm1
);
1269 TCGv_i64 incr
= tcg_constant_i64(a
->imm2
);
1270 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1275 static bool trans_INDEX_ir(DisasContext
*s
, arg_INDEX_ir
*a
)
1277 if (sve_access_check(s
)) {
1278 TCGv_i64 start
= tcg_constant_i64(a
->imm
);
1279 TCGv_i64 incr
= cpu_reg(s
, a
->rm
);
1280 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1285 static bool trans_INDEX_ri(DisasContext
*s
, arg_INDEX_ri
*a
)
1287 if (sve_access_check(s
)) {
1288 TCGv_i64 start
= cpu_reg(s
, a
->rn
);
1289 TCGv_i64 incr
= tcg_constant_i64(a
->imm
);
1290 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1295 static bool trans_INDEX_rr(DisasContext
*s
, arg_INDEX_rr
*a
)
1297 if (sve_access_check(s
)) {
1298 TCGv_i64 start
= cpu_reg(s
, a
->rn
);
1299 TCGv_i64 incr
= cpu_reg(s
, a
->rm
);
1300 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1306 *** SVE Stack Allocation Group
1309 static bool trans_ADDVL(DisasContext
*s
, arg_ADDVL
*a
)
1311 if (sve_access_check(s
)) {
1312 TCGv_i64 rd
= cpu_reg_sp(s
, a
->rd
);
1313 TCGv_i64 rn
= cpu_reg_sp(s
, a
->rn
);
1314 tcg_gen_addi_i64(rd
, rn
, a
->imm
* vec_full_reg_size(s
));
1319 static bool trans_ADDPL(DisasContext
*s
, arg_ADDPL
*a
)
1321 if (sve_access_check(s
)) {
1322 TCGv_i64 rd
= cpu_reg_sp(s
, a
->rd
);
1323 TCGv_i64 rn
= cpu_reg_sp(s
, a
->rn
);
1324 tcg_gen_addi_i64(rd
, rn
, a
->imm
* pred_full_reg_size(s
));
1329 static bool trans_RDVL(DisasContext
*s
, arg_RDVL
*a
)
1331 if (sve_access_check(s
)) {
1332 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
1333 tcg_gen_movi_i64(reg
, a
->imm
* vec_full_reg_size(s
));
1339 *** SVE Compute Vector Address Group
1342 static bool do_adr(DisasContext
*s
, arg_rrri
*a
, gen_helper_gvec_3
*fn
)
1344 if (sve_access_check(s
)) {
1345 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->imm
);
1350 static bool trans_ADR_p32(DisasContext
*s
, arg_rrri
*a
)
1352 return do_adr(s
, a
, gen_helper_sve_adr_p32
);
1355 static bool trans_ADR_p64(DisasContext
*s
, arg_rrri
*a
)
1357 return do_adr(s
, a
, gen_helper_sve_adr_p64
);
1360 static bool trans_ADR_s32(DisasContext
*s
, arg_rrri
*a
)
1362 return do_adr(s
, a
, gen_helper_sve_adr_s32
);
1365 static bool trans_ADR_u32(DisasContext
*s
, arg_rrri
*a
)
1367 return do_adr(s
, a
, gen_helper_sve_adr_u32
);
1371 *** SVE Integer Misc - Unpredicated Group
1374 static bool trans_FEXPA(DisasContext
*s
, arg_rr_esz
*a
)
1376 static gen_helper_gvec_2
* const fns
[4] = {
1378 gen_helper_sve_fexpa_h
,
1379 gen_helper_sve_fexpa_s
,
1380 gen_helper_sve_fexpa_d
,
1385 if (sve_access_check(s
)) {
1386 gen_gvec_ool_zz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, 0);
1391 static bool trans_FTSSEL(DisasContext
*s
, arg_rrr_esz
*a
)
1393 static gen_helper_gvec_3
* const fns
[4] = {
1395 gen_helper_sve_ftssel_h
,
1396 gen_helper_sve_ftssel_s
,
1397 gen_helper_sve_ftssel_d
,
1402 if (sve_access_check(s
)) {
1403 gen_gvec_ool_zzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, 0);
1409 *** SVE Predicate Logical Operations Group
1412 static bool do_pppp_flags(DisasContext
*s
, arg_rprr_s
*a
,
1413 const GVecGen4
*gvec_op
)
1415 if (!sve_access_check(s
)) {
1419 unsigned psz
= pred_gvec_reg_size(s
);
1420 int dofs
= pred_full_reg_offset(s
, a
->rd
);
1421 int nofs
= pred_full_reg_offset(s
, a
->rn
);
1422 int mofs
= pred_full_reg_offset(s
, a
->rm
);
1423 int gofs
= pred_full_reg_offset(s
, a
->pg
);
1426 tcg_gen_gvec_4(dofs
, nofs
, mofs
, gofs
, psz
, psz
, gvec_op
);
1431 /* Do the operation and the flags generation in temps. */
1432 TCGv_i64 pd
= tcg_temp_new_i64();
1433 TCGv_i64 pn
= tcg_temp_new_i64();
1434 TCGv_i64 pm
= tcg_temp_new_i64();
1435 TCGv_i64 pg
= tcg_temp_new_i64();
1437 tcg_gen_ld_i64(pn
, cpu_env
, nofs
);
1438 tcg_gen_ld_i64(pm
, cpu_env
, mofs
);
1439 tcg_gen_ld_i64(pg
, cpu_env
, gofs
);
1441 gvec_op
->fni8(pd
, pn
, pm
, pg
);
1442 tcg_gen_st_i64(pd
, cpu_env
, dofs
);
1444 do_predtest1(pd
, pg
);
1446 tcg_temp_free_i64(pd
);
1447 tcg_temp_free_i64(pn
);
1448 tcg_temp_free_i64(pm
);
1449 tcg_temp_free_i64(pg
);
1451 /* The operation and flags generation is large. The computation
1452 * of the flags depends on the original contents of the guarding
1453 * predicate. If the destination overwrites the guarding predicate,
1454 * then the easiest way to get this right is to save a copy.
1457 if (a
->rd
== a
->pg
) {
1458 tofs
= offsetof(CPUARMState
, vfp
.preg_tmp
);
1459 tcg_gen_gvec_mov(0, tofs
, gofs
, psz
, psz
);
1462 tcg_gen_gvec_4(dofs
, nofs
, mofs
, gofs
, psz
, psz
, gvec_op
);
1463 do_predtest(s
, dofs
, tofs
, psz
/ 8);
1468 static void gen_and_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1470 tcg_gen_and_i64(pd
, pn
, pm
);
1471 tcg_gen_and_i64(pd
, pd
, pg
);
1474 static void gen_and_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1475 TCGv_vec pm
, TCGv_vec pg
)
1477 tcg_gen_and_vec(vece
, pd
, pn
, pm
);
1478 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1481 static bool trans_AND_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1483 static const GVecGen4 op
= {
1484 .fni8
= gen_and_pg_i64
,
1485 .fniv
= gen_and_pg_vec
,
1486 .fno
= gen_helper_sve_and_pppp
,
1487 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1491 if (!sve_access_check(s
)) {
1494 if (a
->rn
== a
->rm
) {
1495 if (a
->pg
== a
->rn
) {
1496 do_mov_p(s
, a
->rd
, a
->rn
);
1498 gen_gvec_fn_ppp(s
, tcg_gen_gvec_and
, a
->rd
, a
->rn
, a
->pg
);
1501 } else if (a
->pg
== a
->rn
|| a
->pg
== a
->rm
) {
1502 gen_gvec_fn_ppp(s
, tcg_gen_gvec_and
, a
->rd
, a
->rn
, a
->rm
);
1506 return do_pppp_flags(s
, a
, &op
);
1509 static void gen_bic_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1511 tcg_gen_andc_i64(pd
, pn
, pm
);
1512 tcg_gen_and_i64(pd
, pd
, pg
);
1515 static void gen_bic_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1516 TCGv_vec pm
, TCGv_vec pg
)
1518 tcg_gen_andc_vec(vece
, pd
, pn
, pm
);
1519 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1522 static bool trans_BIC_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1524 static const GVecGen4 op
= {
1525 .fni8
= gen_bic_pg_i64
,
1526 .fniv
= gen_bic_pg_vec
,
1527 .fno
= gen_helper_sve_bic_pppp
,
1528 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1531 if (!a
->s
&& a
->pg
== a
->rn
) {
1532 if (sve_access_check(s
)) {
1533 gen_gvec_fn_ppp(s
, tcg_gen_gvec_andc
, a
->rd
, a
->rn
, a
->rm
);
1537 return do_pppp_flags(s
, a
, &op
);
1540 static void gen_eor_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1542 tcg_gen_xor_i64(pd
, pn
, pm
);
1543 tcg_gen_and_i64(pd
, pd
, pg
);
1546 static void gen_eor_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1547 TCGv_vec pm
, TCGv_vec pg
)
1549 tcg_gen_xor_vec(vece
, pd
, pn
, pm
);
1550 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1553 static bool trans_EOR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1555 static const GVecGen4 op
= {
1556 .fni8
= gen_eor_pg_i64
,
1557 .fniv
= gen_eor_pg_vec
,
1558 .fno
= gen_helper_sve_eor_pppp
,
1559 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1561 return do_pppp_flags(s
, a
, &op
);
1564 static bool trans_SEL_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1569 if (sve_access_check(s
)) {
1570 unsigned psz
= pred_gvec_reg_size(s
);
1571 tcg_gen_gvec_bitsel(MO_8
, pred_full_reg_offset(s
, a
->rd
),
1572 pred_full_reg_offset(s
, a
->pg
),
1573 pred_full_reg_offset(s
, a
->rn
),
1574 pred_full_reg_offset(s
, a
->rm
), psz
, psz
);
1579 static void gen_orr_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1581 tcg_gen_or_i64(pd
, pn
, pm
);
1582 tcg_gen_and_i64(pd
, pd
, pg
);
1585 static void gen_orr_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1586 TCGv_vec pm
, TCGv_vec pg
)
1588 tcg_gen_or_vec(vece
, pd
, pn
, pm
);
1589 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1592 static bool trans_ORR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1594 static const GVecGen4 op
= {
1595 .fni8
= gen_orr_pg_i64
,
1596 .fniv
= gen_orr_pg_vec
,
1597 .fno
= gen_helper_sve_orr_pppp
,
1598 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1601 if (!a
->s
&& a
->pg
== a
->rn
&& a
->rn
== a
->rm
) {
1602 return do_mov_p(s
, a
->rd
, a
->rn
);
1604 return do_pppp_flags(s
, a
, &op
);
1607 static void gen_orn_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1609 tcg_gen_orc_i64(pd
, pn
, pm
);
1610 tcg_gen_and_i64(pd
, pd
, pg
);
1613 static void gen_orn_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1614 TCGv_vec pm
, TCGv_vec pg
)
1616 tcg_gen_orc_vec(vece
, pd
, pn
, pm
);
1617 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1620 static bool trans_ORN_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1622 static const GVecGen4 op
= {
1623 .fni8
= gen_orn_pg_i64
,
1624 .fniv
= gen_orn_pg_vec
,
1625 .fno
= gen_helper_sve_orn_pppp
,
1626 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1628 return do_pppp_flags(s
, a
, &op
);
1631 static void gen_nor_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1633 tcg_gen_or_i64(pd
, pn
, pm
);
1634 tcg_gen_andc_i64(pd
, pg
, pd
);
1637 static void gen_nor_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1638 TCGv_vec pm
, TCGv_vec pg
)
1640 tcg_gen_or_vec(vece
, pd
, pn
, pm
);
1641 tcg_gen_andc_vec(vece
, pd
, pg
, pd
);
1644 static bool trans_NOR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1646 static const GVecGen4 op
= {
1647 .fni8
= gen_nor_pg_i64
,
1648 .fniv
= gen_nor_pg_vec
,
1649 .fno
= gen_helper_sve_nor_pppp
,
1650 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1652 return do_pppp_flags(s
, a
, &op
);
1655 static void gen_nand_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1657 tcg_gen_and_i64(pd
, pn
, pm
);
1658 tcg_gen_andc_i64(pd
, pg
, pd
);
1661 static void gen_nand_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1662 TCGv_vec pm
, TCGv_vec pg
)
1664 tcg_gen_and_vec(vece
, pd
, pn
, pm
);
1665 tcg_gen_andc_vec(vece
, pd
, pg
, pd
);
1668 static bool trans_NAND_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1670 static const GVecGen4 op
= {
1671 .fni8
= gen_nand_pg_i64
,
1672 .fniv
= gen_nand_pg_vec
,
1673 .fno
= gen_helper_sve_nand_pppp
,
1674 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1676 return do_pppp_flags(s
, a
, &op
);
1680 *** SVE Predicate Misc Group
1683 static bool trans_PTEST(DisasContext
*s
, arg_PTEST
*a
)
1685 if (sve_access_check(s
)) {
1686 int nofs
= pred_full_reg_offset(s
, a
->rn
);
1687 int gofs
= pred_full_reg_offset(s
, a
->pg
);
1688 int words
= DIV_ROUND_UP(pred_full_reg_size(s
), 8);
1691 TCGv_i64 pn
= tcg_temp_new_i64();
1692 TCGv_i64 pg
= tcg_temp_new_i64();
1694 tcg_gen_ld_i64(pn
, cpu_env
, nofs
);
1695 tcg_gen_ld_i64(pg
, cpu_env
, gofs
);
1696 do_predtest1(pn
, pg
);
1698 tcg_temp_free_i64(pn
);
1699 tcg_temp_free_i64(pg
);
1701 do_predtest(s
, nofs
, gofs
, words
);
1707 /* See the ARM pseudocode DecodePredCount. */
1708 static unsigned decode_pred_count(unsigned fullsz
, int pattern
, int esz
)
1710 unsigned elements
= fullsz
>> esz
;
1714 case 0x0: /* POW2 */
1715 return pow2floor(elements
);
1726 case 0x9: /* VL16 */
1727 case 0xa: /* VL32 */
1728 case 0xb: /* VL64 */
1729 case 0xc: /* VL128 */
1730 case 0xd: /* VL256 */
1731 bound
= 16 << (pattern
- 9);
1733 case 0x1d: /* MUL4 */
1734 return elements
- elements
% 4;
1735 case 0x1e: /* MUL3 */
1736 return elements
- elements
% 3;
1737 case 0x1f: /* ALL */
1739 default: /* #uimm5 */
1742 return elements
>= bound
? bound
: 0;
1745 /* This handles all of the predicate initialization instructions,
1746 * PTRUE, PFALSE, SETFFR. For PFALSE, we will have set PAT == 32
1747 * so that decode_pred_count returns 0. For SETFFR, we will have
1748 * set RD == 16 == FFR.
1750 static bool do_predset(DisasContext
*s
, int esz
, int rd
, int pat
, bool setflag
)
1752 if (!sve_access_check(s
)) {
1756 unsigned fullsz
= vec_full_reg_size(s
);
1757 unsigned ofs
= pred_full_reg_offset(s
, rd
);
1758 unsigned numelem
, setsz
, i
;
1759 uint64_t word
, lastword
;
1762 numelem
= decode_pred_count(fullsz
, pat
, esz
);
1764 /* Determine what we must store into each bit, and how many. */
1766 lastword
= word
= 0;
1769 setsz
= numelem
<< esz
;
1770 lastword
= word
= pred_esz_masks
[esz
];
1772 lastword
&= MAKE_64BIT_MASK(0, setsz
% 64);
1776 t
= tcg_temp_new_i64();
1778 tcg_gen_movi_i64(t
, lastword
);
1779 tcg_gen_st_i64(t
, cpu_env
, ofs
);
1783 if (word
== lastword
) {
1784 unsigned maxsz
= size_for_gvec(fullsz
/ 8);
1785 unsigned oprsz
= size_for_gvec(setsz
/ 8);
1787 if (oprsz
* 8 == setsz
) {
1788 tcg_gen_gvec_dup_imm(MO_64
, ofs
, oprsz
, maxsz
, word
);
1796 tcg_gen_movi_i64(t
, word
);
1797 for (i
= 0; i
< QEMU_ALIGN_DOWN(setsz
, 8); i
+= 8) {
1798 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1800 if (lastword
!= word
) {
1801 tcg_gen_movi_i64(t
, lastword
);
1802 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1806 tcg_gen_movi_i64(t
, 0);
1807 for (; i
< fullsz
; i
+= 8) {
1808 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1813 tcg_temp_free_i64(t
);
1817 tcg_gen_movi_i32(cpu_NF
, -(word
!= 0));
1818 tcg_gen_movi_i32(cpu_CF
, word
== 0);
1819 tcg_gen_movi_i32(cpu_VF
, 0);
1820 tcg_gen_mov_i32(cpu_ZF
, cpu_NF
);
1825 static bool trans_PTRUE(DisasContext
*s
, arg_PTRUE
*a
)
1827 return do_predset(s
, a
->esz
, a
->rd
, a
->pat
, a
->s
);
1830 static bool trans_SETFFR(DisasContext
*s
, arg_SETFFR
*a
)
1832 /* Note pat == 31 is #all, to set all elements. */
1833 return do_predset(s
, 0, FFR_PRED_NUM
, 31, false);
1836 static bool trans_PFALSE(DisasContext
*s
, arg_PFALSE
*a
)
1838 /* Note pat == 32 is #unimp, to set no elements. */
1839 return do_predset(s
, 0, a
->rd
, 32, false);
1842 static bool trans_RDFFR_p(DisasContext
*s
, arg_RDFFR_p
*a
)
1844 /* The path through do_pppp_flags is complicated enough to want to avoid
1845 * duplication. Frob the arguments into the form of a predicated AND.
1847 arg_rprr_s alt_a
= {
1848 .rd
= a
->rd
, .pg
= a
->pg
, .s
= a
->s
,
1849 .rn
= FFR_PRED_NUM
, .rm
= FFR_PRED_NUM
,
1851 return trans_AND_pppp(s
, &alt_a
);
1854 static bool trans_RDFFR(DisasContext
*s
, arg_RDFFR
*a
)
1856 return do_mov_p(s
, a
->rd
, FFR_PRED_NUM
);
1859 static bool trans_WRFFR(DisasContext
*s
, arg_WRFFR
*a
)
1861 return do_mov_p(s
, FFR_PRED_NUM
, a
->rn
);
1864 static bool do_pfirst_pnext(DisasContext
*s
, arg_rr_esz
*a
,
1865 void (*gen_fn
)(TCGv_i32
, TCGv_ptr
,
1866 TCGv_ptr
, TCGv_i32
))
1868 if (!sve_access_check(s
)) {
1872 TCGv_ptr t_pd
= tcg_temp_new_ptr();
1873 TCGv_ptr t_pg
= tcg_temp_new_ptr();
1877 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, pred_full_reg_size(s
));
1878 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
1880 tcg_gen_addi_ptr(t_pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
1881 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
1882 t
= tcg_temp_new_i32();
1884 gen_fn(t
, t_pd
, t_pg
, tcg_constant_i32(desc
));
1885 tcg_temp_free_ptr(t_pd
);
1886 tcg_temp_free_ptr(t_pg
);
1889 tcg_temp_free_i32(t
);
1893 static bool trans_PFIRST(DisasContext
*s
, arg_rr_esz
*a
)
1895 return do_pfirst_pnext(s
, a
, gen_helper_sve_pfirst
);
1898 static bool trans_PNEXT(DisasContext
*s
, arg_rr_esz
*a
)
1900 return do_pfirst_pnext(s
, a
, gen_helper_sve_pnext
);
1904 *** SVE Element Count Group
1907 /* Perform an inline saturating addition of a 32-bit value within
1908 * a 64-bit register. The second operand is known to be positive,
1909 * which halves the comparisions we must perform to bound the result.
1911 static void do_sat_addsub_32(TCGv_i64 reg
, TCGv_i64 val
, bool u
, bool d
)
1915 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
1917 tcg_gen_ext32u_i64(reg
, reg
);
1919 tcg_gen_ext32s_i64(reg
, reg
);
1922 tcg_gen_sub_i64(reg
, reg
, val
);
1923 ibound
= (u
? 0 : INT32_MIN
);
1924 tcg_gen_smax_i64(reg
, reg
, tcg_constant_i64(ibound
));
1926 tcg_gen_add_i64(reg
, reg
, val
);
1927 ibound
= (u
? UINT32_MAX
: INT32_MAX
);
1928 tcg_gen_smin_i64(reg
, reg
, tcg_constant_i64(ibound
));
1932 /* Similarly with 64-bit values. */
1933 static void do_sat_addsub_64(TCGv_i64 reg
, TCGv_i64 val
, bool u
, bool d
)
1935 TCGv_i64 t0
= tcg_temp_new_i64();
1940 tcg_gen_sub_i64(t0
, reg
, val
);
1941 t2
= tcg_constant_i64(0);
1942 tcg_gen_movcond_i64(TCG_COND_LTU
, reg
, reg
, val
, t2
, t0
);
1944 tcg_gen_add_i64(t0
, reg
, val
);
1945 t2
= tcg_constant_i64(-1);
1946 tcg_gen_movcond_i64(TCG_COND_LTU
, reg
, t0
, reg
, t2
, t0
);
1949 TCGv_i64 t1
= tcg_temp_new_i64();
1951 /* Detect signed overflow for subtraction. */
1952 tcg_gen_xor_i64(t0
, reg
, val
);
1953 tcg_gen_sub_i64(t1
, reg
, val
);
1954 tcg_gen_xor_i64(reg
, reg
, t1
);
1955 tcg_gen_and_i64(t0
, t0
, reg
);
1957 /* Bound the result. */
1958 tcg_gen_movi_i64(reg
, INT64_MIN
);
1959 t2
= tcg_constant_i64(0);
1960 tcg_gen_movcond_i64(TCG_COND_LT
, reg
, t0
, t2
, reg
, t1
);
1962 /* Detect signed overflow for addition. */
1963 tcg_gen_xor_i64(t0
, reg
, val
);
1964 tcg_gen_add_i64(reg
, reg
, val
);
1965 tcg_gen_xor_i64(t1
, reg
, val
);
1966 tcg_gen_andc_i64(t0
, t1
, t0
);
1968 /* Bound the result. */
1969 tcg_gen_movi_i64(t1
, INT64_MAX
);
1970 t2
= tcg_constant_i64(0);
1971 tcg_gen_movcond_i64(TCG_COND_LT
, reg
, t0
, t2
, t1
, reg
);
1973 tcg_temp_free_i64(t1
);
1975 tcg_temp_free_i64(t0
);
1978 /* Similarly with a vector and a scalar operand. */
1979 static void do_sat_addsub_vec(DisasContext
*s
, int esz
, int rd
, int rn
,
1980 TCGv_i64 val
, bool u
, bool d
)
1982 unsigned vsz
= vec_full_reg_size(s
);
1983 TCGv_ptr dptr
, nptr
;
1987 dptr
= tcg_temp_new_ptr();
1988 nptr
= tcg_temp_new_ptr();
1989 tcg_gen_addi_ptr(dptr
, cpu_env
, vec_full_reg_offset(s
, rd
));
1990 tcg_gen_addi_ptr(nptr
, cpu_env
, vec_full_reg_offset(s
, rn
));
1991 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
1995 t32
= tcg_temp_new_i32();
1996 tcg_gen_extrl_i64_i32(t32
, val
);
1998 tcg_gen_neg_i32(t32
, t32
);
2001 gen_helper_sve_uqaddi_b(dptr
, nptr
, t32
, desc
);
2003 gen_helper_sve_sqaddi_b(dptr
, nptr
, t32
, desc
);
2005 tcg_temp_free_i32(t32
);
2009 t32
= tcg_temp_new_i32();
2010 tcg_gen_extrl_i64_i32(t32
, val
);
2012 tcg_gen_neg_i32(t32
, t32
);
2015 gen_helper_sve_uqaddi_h(dptr
, nptr
, t32
, desc
);
2017 gen_helper_sve_sqaddi_h(dptr
, nptr
, t32
, desc
);
2019 tcg_temp_free_i32(t32
);
2023 t64
= tcg_temp_new_i64();
2025 tcg_gen_neg_i64(t64
, val
);
2027 tcg_gen_mov_i64(t64
, val
);
2030 gen_helper_sve_uqaddi_s(dptr
, nptr
, t64
, desc
);
2032 gen_helper_sve_sqaddi_s(dptr
, nptr
, t64
, desc
);
2034 tcg_temp_free_i64(t64
);
2040 gen_helper_sve_uqsubi_d(dptr
, nptr
, val
, desc
);
2042 gen_helper_sve_uqaddi_d(dptr
, nptr
, val
, desc
);
2045 t64
= tcg_temp_new_i64();
2046 tcg_gen_neg_i64(t64
, val
);
2047 gen_helper_sve_sqaddi_d(dptr
, nptr
, t64
, desc
);
2048 tcg_temp_free_i64(t64
);
2050 gen_helper_sve_sqaddi_d(dptr
, nptr
, val
, desc
);
2055 g_assert_not_reached();
2058 tcg_temp_free_ptr(dptr
);
2059 tcg_temp_free_ptr(nptr
);
2060 tcg_temp_free_i32(desc
);
2063 static bool trans_CNT_r(DisasContext
*s
, arg_CNT_r
*a
)
2065 if (sve_access_check(s
)) {
2066 unsigned fullsz
= vec_full_reg_size(s
);
2067 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2068 tcg_gen_movi_i64(cpu_reg(s
, a
->rd
), numelem
* a
->imm
);
2073 static bool trans_INCDEC_r(DisasContext
*s
, arg_incdec_cnt
*a
)
2075 if (sve_access_check(s
)) {
2076 unsigned fullsz
= vec_full_reg_size(s
);
2077 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2078 int inc
= numelem
* a
->imm
* (a
->d
? -1 : 1);
2079 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
2081 tcg_gen_addi_i64(reg
, reg
, inc
);
2086 static bool trans_SINCDEC_r_32(DisasContext
*s
, arg_incdec_cnt
*a
)
2088 if (!sve_access_check(s
)) {
2092 unsigned fullsz
= vec_full_reg_size(s
);
2093 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2094 int inc
= numelem
* a
->imm
;
2095 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
2097 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
2100 tcg_gen_ext32u_i64(reg
, reg
);
2102 tcg_gen_ext32s_i64(reg
, reg
);
2105 do_sat_addsub_32(reg
, tcg_constant_i64(inc
), a
->u
, a
->d
);
2110 static bool trans_SINCDEC_r_64(DisasContext
*s
, arg_incdec_cnt
*a
)
2112 if (!sve_access_check(s
)) {
2116 unsigned fullsz
= vec_full_reg_size(s
);
2117 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2118 int inc
= numelem
* a
->imm
;
2119 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
2122 do_sat_addsub_64(reg
, tcg_constant_i64(inc
), a
->u
, a
->d
);
2127 static bool trans_INCDEC_v(DisasContext
*s
, arg_incdec2_cnt
*a
)
2133 unsigned fullsz
= vec_full_reg_size(s
);
2134 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2135 int inc
= numelem
* a
->imm
;
2138 if (sve_access_check(s
)) {
2139 tcg_gen_gvec_adds(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
2140 vec_full_reg_offset(s
, a
->rn
),
2141 tcg_constant_i64(a
->d
? -inc
: inc
),
2145 do_mov_z(s
, a
->rd
, a
->rn
);
2150 static bool trans_SINCDEC_v(DisasContext
*s
, arg_incdec2_cnt
*a
)
2156 unsigned fullsz
= vec_full_reg_size(s
);
2157 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2158 int inc
= numelem
* a
->imm
;
2161 if (sve_access_check(s
)) {
2162 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
,
2163 tcg_constant_i64(inc
), a
->u
, a
->d
);
2166 do_mov_z(s
, a
->rd
, a
->rn
);
2172 *** SVE Bitwise Immediate Group
2175 static bool do_zz_dbm(DisasContext
*s
, arg_rr_dbm
*a
, GVecGen2iFn
*gvec_fn
)
2178 if (!logic_imm_decode_wmask(&imm
, extract32(a
->dbm
, 12, 1),
2179 extract32(a
->dbm
, 0, 6),
2180 extract32(a
->dbm
, 6, 6))) {
2183 if (sve_access_check(s
)) {
2184 unsigned vsz
= vec_full_reg_size(s
);
2185 gvec_fn(MO_64
, vec_full_reg_offset(s
, a
->rd
),
2186 vec_full_reg_offset(s
, a
->rn
), imm
, vsz
, vsz
);
2191 static bool trans_AND_zzi(DisasContext
*s
, arg_rr_dbm
*a
)
2193 return do_zz_dbm(s
, a
, tcg_gen_gvec_andi
);
2196 static bool trans_ORR_zzi(DisasContext
*s
, arg_rr_dbm
*a
)
2198 return do_zz_dbm(s
, a
, tcg_gen_gvec_ori
);
2201 static bool trans_EOR_zzi(DisasContext
*s
, arg_rr_dbm
*a
)
2203 return do_zz_dbm(s
, a
, tcg_gen_gvec_xori
);
2206 static bool trans_DUPM(DisasContext
*s
, arg_DUPM
*a
)
2209 if (!logic_imm_decode_wmask(&imm
, extract32(a
->dbm
, 12, 1),
2210 extract32(a
->dbm
, 0, 6),
2211 extract32(a
->dbm
, 6, 6))) {
2214 if (sve_access_check(s
)) {
2215 do_dupi_z(s
, a
->rd
, imm
);
2221 *** SVE Integer Wide Immediate - Predicated Group
2224 /* Implement all merging copies. This is used for CPY (immediate),
2225 * FCPY, CPY (scalar), CPY (SIMD&FP scalar).
2227 static void do_cpy_m(DisasContext
*s
, int esz
, int rd
, int rn
, int pg
,
2230 typedef void gen_cpy(TCGv_ptr
, TCGv_ptr
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
2231 static gen_cpy
* const fns
[4] = {
2232 gen_helper_sve_cpy_m_b
, gen_helper_sve_cpy_m_h
,
2233 gen_helper_sve_cpy_m_s
, gen_helper_sve_cpy_m_d
,
2235 unsigned vsz
= vec_full_reg_size(s
);
2236 TCGv_i32 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
2237 TCGv_ptr t_zd
= tcg_temp_new_ptr();
2238 TCGv_ptr t_zn
= tcg_temp_new_ptr();
2239 TCGv_ptr t_pg
= tcg_temp_new_ptr();
2241 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, rd
));
2242 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, rn
));
2243 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
2245 fns
[esz
](t_zd
, t_zn
, t_pg
, val
, desc
);
2247 tcg_temp_free_ptr(t_zd
);
2248 tcg_temp_free_ptr(t_zn
);
2249 tcg_temp_free_ptr(t_pg
);
2250 tcg_temp_free_i32(desc
);
2253 static bool trans_FCPY(DisasContext
*s
, arg_FCPY
*a
)
2258 if (sve_access_check(s
)) {
2259 /* Decode the VFP immediate. */
2260 uint64_t imm
= vfp_expand_imm(a
->esz
, a
->imm
);
2261 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rn
, a
->pg
, tcg_constant_i64(imm
));
2266 static bool trans_CPY_m_i(DisasContext
*s
, arg_rpri_esz
*a
)
2268 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
2271 if (sve_access_check(s
)) {
2272 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rn
, a
->pg
, tcg_constant_i64(a
->imm
));
2277 static bool trans_CPY_z_i(DisasContext
*s
, arg_CPY_z_i
*a
)
2279 static gen_helper_gvec_2i
* const fns
[4] = {
2280 gen_helper_sve_cpy_z_b
, gen_helper_sve_cpy_z_h
,
2281 gen_helper_sve_cpy_z_s
, gen_helper_sve_cpy_z_d
,
2284 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
2287 if (sve_access_check(s
)) {
2288 unsigned vsz
= vec_full_reg_size(s
);
2289 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s
, a
->rd
),
2290 pred_full_reg_offset(s
, a
->pg
),
2291 tcg_constant_i64(a
->imm
),
2292 vsz
, vsz
, 0, fns
[a
->esz
]);
2298 *** SVE Permute Extract Group
2301 static bool do_EXT(DisasContext
*s
, int rd
, int rn
, int rm
, int imm
)
2303 if (!sve_access_check(s
)) {
2307 unsigned vsz
= vec_full_reg_size(s
);
2308 unsigned n_ofs
= imm
>= vsz
? 0 : imm
;
2309 unsigned n_siz
= vsz
- n_ofs
;
2310 unsigned d
= vec_full_reg_offset(s
, rd
);
2311 unsigned n
= vec_full_reg_offset(s
, rn
);
2312 unsigned m
= vec_full_reg_offset(s
, rm
);
2314 /* Use host vector move insns if we have appropriate sizes
2315 * and no unfortunate overlap.
2318 && n_ofs
== size_for_gvec(n_ofs
)
2319 && n_siz
== size_for_gvec(n_siz
)
2320 && (d
!= n
|| n_siz
<= n_ofs
)) {
2321 tcg_gen_gvec_mov(0, d
, n
+ n_ofs
, n_siz
, n_siz
);
2323 tcg_gen_gvec_mov(0, d
+ n_siz
, m
, n_ofs
, n_ofs
);
2326 tcg_gen_gvec_3_ool(d
, n
, m
, vsz
, vsz
, n_ofs
, gen_helper_sve_ext
);
2331 static bool trans_EXT(DisasContext
*s
, arg_EXT
*a
)
2333 return do_EXT(s
, a
->rd
, a
->rn
, a
->rm
, a
->imm
);
2336 static bool trans_EXT_sve2(DisasContext
*s
, arg_rri
*a
)
2338 if (!dc_isar_feature(aa64_sve2
, s
)) {
2341 return do_EXT(s
, a
->rd
, a
->rn
, (a
->rn
+ 1) % 32, a
->imm
);
2345 *** SVE Permute - Unpredicated Group
2348 static bool trans_DUP_s(DisasContext
*s
, arg_DUP_s
*a
)
2350 if (sve_access_check(s
)) {
2351 unsigned vsz
= vec_full_reg_size(s
);
2352 tcg_gen_gvec_dup_i64(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
2353 vsz
, vsz
, cpu_reg_sp(s
, a
->rn
));
2358 static bool trans_DUP_x(DisasContext
*s
, arg_DUP_x
*a
)
2360 if ((a
->imm
& 0x1f) == 0) {
2363 if (sve_access_check(s
)) {
2364 unsigned vsz
= vec_full_reg_size(s
);
2365 unsigned dofs
= vec_full_reg_offset(s
, a
->rd
);
2366 unsigned esz
, index
;
2368 esz
= ctz32(a
->imm
);
2369 index
= a
->imm
>> (esz
+ 1);
2371 if ((index
<< esz
) < vsz
) {
2372 unsigned nofs
= vec_reg_offset(s
, a
->rn
, index
, esz
);
2373 tcg_gen_gvec_dup_mem(esz
, dofs
, nofs
, vsz
, vsz
);
2376 * While dup_mem handles 128-bit elements, dup_imm does not.
2377 * Thankfully element size doesn't matter for splatting zero.
2379 tcg_gen_gvec_dup_imm(MO_64
, dofs
, vsz
, vsz
, 0);
2385 static void do_insr_i64(DisasContext
*s
, arg_rrr_esz
*a
, TCGv_i64 val
)
2387 typedef void gen_insr(TCGv_ptr
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
2388 static gen_insr
* const fns
[4] = {
2389 gen_helper_sve_insr_b
, gen_helper_sve_insr_h
,
2390 gen_helper_sve_insr_s
, gen_helper_sve_insr_d
,
2392 unsigned vsz
= vec_full_reg_size(s
);
2393 TCGv_i32 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
2394 TCGv_ptr t_zd
= tcg_temp_new_ptr();
2395 TCGv_ptr t_zn
= tcg_temp_new_ptr();
2397 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, a
->rd
));
2398 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
2400 fns
[a
->esz
](t_zd
, t_zn
, val
, desc
);
2402 tcg_temp_free_ptr(t_zd
);
2403 tcg_temp_free_ptr(t_zn
);
2404 tcg_temp_free_i32(desc
);
2407 static bool trans_INSR_f(DisasContext
*s
, arg_rrr_esz
*a
)
2409 if (sve_access_check(s
)) {
2410 TCGv_i64 t
= tcg_temp_new_i64();
2411 tcg_gen_ld_i64(t
, cpu_env
, vec_reg_offset(s
, a
->rm
, 0, MO_64
));
2412 do_insr_i64(s
, a
, t
);
2413 tcg_temp_free_i64(t
);
2418 static bool trans_INSR_r(DisasContext
*s
, arg_rrr_esz
*a
)
2420 if (sve_access_check(s
)) {
2421 do_insr_i64(s
, a
, cpu_reg(s
, a
->rm
));
2426 static bool trans_REV_v(DisasContext
*s
, arg_rr_esz
*a
)
2428 static gen_helper_gvec_2
* const fns
[4] = {
2429 gen_helper_sve_rev_b
, gen_helper_sve_rev_h
,
2430 gen_helper_sve_rev_s
, gen_helper_sve_rev_d
2433 if (sve_access_check(s
)) {
2434 gen_gvec_ool_zz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, 0);
2439 static bool trans_TBL(DisasContext
*s
, arg_rrr_esz
*a
)
2441 static gen_helper_gvec_3
* const fns
[4] = {
2442 gen_helper_sve_tbl_b
, gen_helper_sve_tbl_h
,
2443 gen_helper_sve_tbl_s
, gen_helper_sve_tbl_d
2446 if (sve_access_check(s
)) {
2447 gen_gvec_ool_zzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, 0);
2452 static bool trans_TBL_sve2(DisasContext
*s
, arg_rrr_esz
*a
)
2454 static gen_helper_gvec_4
* const fns
[4] = {
2455 gen_helper_sve2_tbl_b
, gen_helper_sve2_tbl_h
,
2456 gen_helper_sve2_tbl_s
, gen_helper_sve2_tbl_d
2459 if (!dc_isar_feature(aa64_sve2
, s
)) {
2462 if (sve_access_check(s
)) {
2463 gen_gvec_ool_zzzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
,
2464 (a
->rn
+ 1) % 32, a
->rm
, 0);
2469 static bool trans_TBX(DisasContext
*s
, arg_rrr_esz
*a
)
2471 static gen_helper_gvec_3
* const fns
[4] = {
2472 gen_helper_sve2_tbx_b
, gen_helper_sve2_tbx_h
,
2473 gen_helper_sve2_tbx_s
, gen_helper_sve2_tbx_d
2476 if (!dc_isar_feature(aa64_sve2
, s
)) {
2479 if (sve_access_check(s
)) {
2480 gen_gvec_ool_zzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, 0);
2485 static bool trans_UNPK(DisasContext
*s
, arg_UNPK
*a
)
2487 static gen_helper_gvec_2
* const fns
[4][2] = {
2489 { gen_helper_sve_sunpk_h
, gen_helper_sve_uunpk_h
},
2490 { gen_helper_sve_sunpk_s
, gen_helper_sve_uunpk_s
},
2491 { gen_helper_sve_sunpk_d
, gen_helper_sve_uunpk_d
},
2497 if (sve_access_check(s
)) {
2498 unsigned vsz
= vec_full_reg_size(s
);
2499 tcg_gen_gvec_2_ool(vec_full_reg_offset(s
, a
->rd
),
2500 vec_full_reg_offset(s
, a
->rn
)
2501 + (a
->h
? vsz
/ 2 : 0),
2502 vsz
, vsz
, 0, fns
[a
->esz
][a
->u
]);
2508 *** SVE Permute - Predicates Group
2511 static bool do_perm_pred3(DisasContext
*s
, arg_rrr_esz
*a
, bool high_odd
,
2512 gen_helper_gvec_3
*fn
)
2514 if (!sve_access_check(s
)) {
2518 unsigned vsz
= pred_full_reg_size(s
);
2520 TCGv_ptr t_d
= tcg_temp_new_ptr();
2521 TCGv_ptr t_n
= tcg_temp_new_ptr();
2522 TCGv_ptr t_m
= tcg_temp_new_ptr();
2526 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
);
2527 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
2528 desc
= FIELD_DP32(desc
, PREDDESC
, DATA
, high_odd
);
2530 tcg_gen_addi_ptr(t_d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2531 tcg_gen_addi_ptr(t_n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2532 tcg_gen_addi_ptr(t_m
, cpu_env
, pred_full_reg_offset(s
, a
->rm
));
2533 t_desc
= tcg_const_i32(desc
);
2535 fn(t_d
, t_n
, t_m
, t_desc
);
2537 tcg_temp_free_ptr(t_d
);
2538 tcg_temp_free_ptr(t_n
);
2539 tcg_temp_free_ptr(t_m
);
2540 tcg_temp_free_i32(t_desc
);
2544 static bool do_perm_pred2(DisasContext
*s
, arg_rr_esz
*a
, bool high_odd
,
2545 gen_helper_gvec_2
*fn
)
2547 if (!sve_access_check(s
)) {
2551 unsigned vsz
= pred_full_reg_size(s
);
2552 TCGv_ptr t_d
= tcg_temp_new_ptr();
2553 TCGv_ptr t_n
= tcg_temp_new_ptr();
2557 tcg_gen_addi_ptr(t_d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2558 tcg_gen_addi_ptr(t_n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2560 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
);
2561 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
2562 desc
= FIELD_DP32(desc
, PREDDESC
, DATA
, high_odd
);
2563 t_desc
= tcg_const_i32(desc
);
2565 fn(t_d
, t_n
, t_desc
);
2567 tcg_temp_free_i32(t_desc
);
2568 tcg_temp_free_ptr(t_d
);
2569 tcg_temp_free_ptr(t_n
);
2573 static bool trans_ZIP1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2575 return do_perm_pred3(s
, a
, 0, gen_helper_sve_zip_p
);
2578 static bool trans_ZIP2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2580 return do_perm_pred3(s
, a
, 1, gen_helper_sve_zip_p
);
2583 static bool trans_UZP1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2585 return do_perm_pred3(s
, a
, 0, gen_helper_sve_uzp_p
);
2588 static bool trans_UZP2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2590 return do_perm_pred3(s
, a
, 1, gen_helper_sve_uzp_p
);
2593 static bool trans_TRN1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2595 return do_perm_pred3(s
, a
, 0, gen_helper_sve_trn_p
);
2598 static bool trans_TRN2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2600 return do_perm_pred3(s
, a
, 1, gen_helper_sve_trn_p
);
2603 static bool trans_REV_p(DisasContext
*s
, arg_rr_esz
*a
)
2605 return do_perm_pred2(s
, a
, 0, gen_helper_sve_rev_p
);
2608 static bool trans_PUNPKLO(DisasContext
*s
, arg_PUNPKLO
*a
)
2610 return do_perm_pred2(s
, a
, 0, gen_helper_sve_punpk_p
);
2613 static bool trans_PUNPKHI(DisasContext
*s
, arg_PUNPKHI
*a
)
2615 return do_perm_pred2(s
, a
, 1, gen_helper_sve_punpk_p
);
2619 *** SVE Permute - Interleaving Group
2622 static bool do_zip(DisasContext
*s
, arg_rrr_esz
*a
, bool high
)
2624 static gen_helper_gvec_3
* const fns
[4] = {
2625 gen_helper_sve_zip_b
, gen_helper_sve_zip_h
,
2626 gen_helper_sve_zip_s
, gen_helper_sve_zip_d
,
2629 if (sve_access_check(s
)) {
2630 unsigned vsz
= vec_full_reg_size(s
);
2631 unsigned high_ofs
= high
? vsz
/ 2 : 0;
2632 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, a
->rd
),
2633 vec_full_reg_offset(s
, a
->rn
) + high_ofs
,
2634 vec_full_reg_offset(s
, a
->rm
) + high_ofs
,
2635 vsz
, vsz
, 0, fns
[a
->esz
]);
2640 static bool do_zzz_data_ool(DisasContext
*s
, arg_rrr_esz
*a
, int data
,
2641 gen_helper_gvec_3
*fn
)
2643 if (sve_access_check(s
)) {
2644 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, data
);
2649 static bool trans_ZIP1_z(DisasContext
*s
, arg_rrr_esz
*a
)
2651 return do_zip(s
, a
, false);
2654 static bool trans_ZIP2_z(DisasContext
*s
, arg_rrr_esz
*a
)
2656 return do_zip(s
, a
, true);
2659 static bool do_zip_q(DisasContext
*s
, arg_rrr_esz
*a
, bool high
)
2661 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
2664 if (sve_access_check(s
)) {
2665 unsigned vsz
= vec_full_reg_size(s
);
2666 unsigned high_ofs
= high
? QEMU_ALIGN_DOWN(vsz
, 32) / 2 : 0;
2667 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, a
->rd
),
2668 vec_full_reg_offset(s
, a
->rn
) + high_ofs
,
2669 vec_full_reg_offset(s
, a
->rm
) + high_ofs
,
2670 vsz
, vsz
, 0, gen_helper_sve2_zip_q
);
2675 static bool trans_ZIP1_q(DisasContext
*s
, arg_rrr_esz
*a
)
2677 return do_zip_q(s
, a
, false);
2680 static bool trans_ZIP2_q(DisasContext
*s
, arg_rrr_esz
*a
)
2682 return do_zip_q(s
, a
, true);
2685 static gen_helper_gvec_3
* const uzp_fns
[4] = {
2686 gen_helper_sve_uzp_b
, gen_helper_sve_uzp_h
,
2687 gen_helper_sve_uzp_s
, gen_helper_sve_uzp_d
,
2690 static bool trans_UZP1_z(DisasContext
*s
, arg_rrr_esz
*a
)
2692 return do_zzz_data_ool(s
, a
, 0, uzp_fns
[a
->esz
]);
2695 static bool trans_UZP2_z(DisasContext
*s
, arg_rrr_esz
*a
)
2697 return do_zzz_data_ool(s
, a
, 1 << a
->esz
, uzp_fns
[a
->esz
]);
2700 static bool trans_UZP1_q(DisasContext
*s
, arg_rrr_esz
*a
)
2702 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
2705 return do_zzz_data_ool(s
, a
, 0, gen_helper_sve2_uzp_q
);
2708 static bool trans_UZP2_q(DisasContext
*s
, arg_rrr_esz
*a
)
2710 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
2713 return do_zzz_data_ool(s
, a
, 16, gen_helper_sve2_uzp_q
);
2716 static gen_helper_gvec_3
* const trn_fns
[4] = {
2717 gen_helper_sve_trn_b
, gen_helper_sve_trn_h
,
2718 gen_helper_sve_trn_s
, gen_helper_sve_trn_d
,
2721 static bool trans_TRN1_z(DisasContext
*s
, arg_rrr_esz
*a
)
2723 return do_zzz_data_ool(s
, a
, 0, trn_fns
[a
->esz
]);
2726 static bool trans_TRN2_z(DisasContext
*s
, arg_rrr_esz
*a
)
2728 return do_zzz_data_ool(s
, a
, 1 << a
->esz
, trn_fns
[a
->esz
]);
2731 static bool trans_TRN1_q(DisasContext
*s
, arg_rrr_esz
*a
)
2733 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
2736 return do_zzz_data_ool(s
, a
, 0, gen_helper_sve2_trn_q
);
2739 static bool trans_TRN2_q(DisasContext
*s
, arg_rrr_esz
*a
)
2741 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
2744 return do_zzz_data_ool(s
, a
, 16, gen_helper_sve2_trn_q
);
2748 *** SVE Permute Vector - Predicated Group
2751 static bool trans_COMPACT(DisasContext
*s
, arg_rpr_esz
*a
)
2753 static gen_helper_gvec_3
* const fns
[4] = {
2754 NULL
, NULL
, gen_helper_sve_compact_s
, gen_helper_sve_compact_d
2756 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
2759 /* Call the helper that computes the ARM LastActiveElement pseudocode
2760 * function, scaled by the element size. This includes the not found
2761 * indication; e.g. not found for esz=3 is -8.
2763 static void find_last_active(DisasContext
*s
, TCGv_i32 ret
, int esz
, int pg
)
2765 /* Predicate sizes may be smaller and cannot use simd_desc. We cannot
2766 * round up, as we do elsewhere, because we need the exact size.
2768 TCGv_ptr t_p
= tcg_temp_new_ptr();
2772 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, pred_full_reg_size(s
));
2773 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, esz
);
2775 tcg_gen_addi_ptr(t_p
, cpu_env
, pred_full_reg_offset(s
, pg
));
2776 t_desc
= tcg_const_i32(desc
);
2778 gen_helper_sve_last_active_element(ret
, t_p
, t_desc
);
2780 tcg_temp_free_i32(t_desc
);
2781 tcg_temp_free_ptr(t_p
);
2784 /* Increment LAST to the offset of the next element in the vector,
2785 * wrapping around to 0.
2787 static void incr_last_active(DisasContext
*s
, TCGv_i32 last
, int esz
)
2789 unsigned vsz
= vec_full_reg_size(s
);
2791 tcg_gen_addi_i32(last
, last
, 1 << esz
);
2792 if (is_power_of_2(vsz
)) {
2793 tcg_gen_andi_i32(last
, last
, vsz
- 1);
2795 TCGv_i32 max
= tcg_constant_i32(vsz
);
2796 TCGv_i32 zero
= tcg_constant_i32(0);
2797 tcg_gen_movcond_i32(TCG_COND_GEU
, last
, last
, max
, zero
, last
);
2801 /* If LAST < 0, set LAST to the offset of the last element in the vector. */
2802 static void wrap_last_active(DisasContext
*s
, TCGv_i32 last
, int esz
)
2804 unsigned vsz
= vec_full_reg_size(s
);
2806 if (is_power_of_2(vsz
)) {
2807 tcg_gen_andi_i32(last
, last
, vsz
- 1);
2809 TCGv_i32 max
= tcg_constant_i32(vsz
- (1 << esz
));
2810 TCGv_i32 zero
= tcg_constant_i32(0);
2811 tcg_gen_movcond_i32(TCG_COND_LT
, last
, last
, zero
, max
, last
);
2815 /* Load an unsigned element of ESZ from BASE+OFS. */
2816 static TCGv_i64
load_esz(TCGv_ptr base
, int ofs
, int esz
)
2818 TCGv_i64 r
= tcg_temp_new_i64();
2822 tcg_gen_ld8u_i64(r
, base
, ofs
);
2825 tcg_gen_ld16u_i64(r
, base
, ofs
);
2828 tcg_gen_ld32u_i64(r
, base
, ofs
);
2831 tcg_gen_ld_i64(r
, base
, ofs
);
2834 g_assert_not_reached();
2839 /* Load an unsigned element of ESZ from RM[LAST]. */
2840 static TCGv_i64
load_last_active(DisasContext
*s
, TCGv_i32 last
,
2843 TCGv_ptr p
= tcg_temp_new_ptr();
2846 /* Convert offset into vector into offset into ENV.
2847 * The final adjustment for the vector register base
2848 * is added via constant offset to the load.
2851 /* Adjust for element ordering. See vec_reg_offset. */
2853 tcg_gen_xori_i32(last
, last
, 8 - (1 << esz
));
2856 tcg_gen_ext_i32_ptr(p
, last
);
2857 tcg_gen_add_ptr(p
, p
, cpu_env
);
2859 r
= load_esz(p
, vec_full_reg_offset(s
, rm
), esz
);
2860 tcg_temp_free_ptr(p
);
2865 /* Compute CLAST for a Zreg. */
2866 static bool do_clast_vector(DisasContext
*s
, arg_rprr_esz
*a
, bool before
)
2871 unsigned vsz
, esz
= a
->esz
;
2873 if (!sve_access_check(s
)) {
2877 last
= tcg_temp_local_new_i32();
2878 over
= gen_new_label();
2880 find_last_active(s
, last
, esz
, a
->pg
);
2882 /* There is of course no movcond for a 2048-bit vector,
2883 * so we must branch over the actual store.
2885 tcg_gen_brcondi_i32(TCG_COND_LT
, last
, 0, over
);
2888 incr_last_active(s
, last
, esz
);
2891 ele
= load_last_active(s
, last
, a
->rm
, esz
);
2892 tcg_temp_free_i32(last
);
2894 vsz
= vec_full_reg_size(s
);
2895 tcg_gen_gvec_dup_i64(esz
, vec_full_reg_offset(s
, a
->rd
), vsz
, vsz
, ele
);
2896 tcg_temp_free_i64(ele
);
2898 /* If this insn used MOVPRFX, we may need a second move. */
2899 if (a
->rd
!= a
->rn
) {
2900 TCGLabel
*done
= gen_new_label();
2903 gen_set_label(over
);
2904 do_mov_z(s
, a
->rd
, a
->rn
);
2906 gen_set_label(done
);
2908 gen_set_label(over
);
2913 static bool trans_CLASTA_z(DisasContext
*s
, arg_rprr_esz
*a
)
2915 return do_clast_vector(s
, a
, false);
2918 static bool trans_CLASTB_z(DisasContext
*s
, arg_rprr_esz
*a
)
2920 return do_clast_vector(s
, a
, true);
2923 /* Compute CLAST for a scalar. */
2924 static void do_clast_scalar(DisasContext
*s
, int esz
, int pg
, int rm
,
2925 bool before
, TCGv_i64 reg_val
)
2927 TCGv_i32 last
= tcg_temp_new_i32();
2930 find_last_active(s
, last
, esz
, pg
);
2932 /* Extend the original value of last prior to incrementing. */
2933 cmp
= tcg_temp_new_i64();
2934 tcg_gen_ext_i32_i64(cmp
, last
);
2937 incr_last_active(s
, last
, esz
);
2940 /* The conceit here is that while last < 0 indicates not found, after
2941 * adjusting for cpu_env->vfp.zregs[rm], it is still a valid address
2942 * from which we can load garbage. We then discard the garbage with
2943 * a conditional move.
2945 ele
= load_last_active(s
, last
, rm
, esz
);
2946 tcg_temp_free_i32(last
);
2948 tcg_gen_movcond_i64(TCG_COND_GE
, reg_val
, cmp
, tcg_constant_i64(0),
2951 tcg_temp_free_i64(cmp
);
2952 tcg_temp_free_i64(ele
);
2955 /* Compute CLAST for a Vreg. */
2956 static bool do_clast_fp(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2958 if (sve_access_check(s
)) {
2960 int ofs
= vec_reg_offset(s
, a
->rd
, 0, esz
);
2961 TCGv_i64 reg
= load_esz(cpu_env
, ofs
, esz
);
2963 do_clast_scalar(s
, esz
, a
->pg
, a
->rn
, before
, reg
);
2964 write_fp_dreg(s
, a
->rd
, reg
);
2965 tcg_temp_free_i64(reg
);
2970 static bool trans_CLASTA_v(DisasContext
*s
, arg_rpr_esz
*a
)
2972 return do_clast_fp(s
, a
, false);
2975 static bool trans_CLASTB_v(DisasContext
*s
, arg_rpr_esz
*a
)
2977 return do_clast_fp(s
, a
, true);
2980 /* Compute CLAST for a Xreg. */
2981 static bool do_clast_general(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2985 if (!sve_access_check(s
)) {
2989 reg
= cpu_reg(s
, a
->rd
);
2992 tcg_gen_ext8u_i64(reg
, reg
);
2995 tcg_gen_ext16u_i64(reg
, reg
);
2998 tcg_gen_ext32u_i64(reg
, reg
);
3003 g_assert_not_reached();
3006 do_clast_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
, reg
);
3010 static bool trans_CLASTA_r(DisasContext
*s
, arg_rpr_esz
*a
)
3012 return do_clast_general(s
, a
, false);
3015 static bool trans_CLASTB_r(DisasContext
*s
, arg_rpr_esz
*a
)
3017 return do_clast_general(s
, a
, true);
3020 /* Compute LAST for a scalar. */
3021 static TCGv_i64
do_last_scalar(DisasContext
*s
, int esz
,
3022 int pg
, int rm
, bool before
)
3024 TCGv_i32 last
= tcg_temp_new_i32();
3027 find_last_active(s
, last
, esz
, pg
);
3029 wrap_last_active(s
, last
, esz
);
3031 incr_last_active(s
, last
, esz
);
3034 ret
= load_last_active(s
, last
, rm
, esz
);
3035 tcg_temp_free_i32(last
);
3039 /* Compute LAST for a Vreg. */
3040 static bool do_last_fp(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
3042 if (sve_access_check(s
)) {
3043 TCGv_i64 val
= do_last_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
);
3044 write_fp_dreg(s
, a
->rd
, val
);
3045 tcg_temp_free_i64(val
);
3050 static bool trans_LASTA_v(DisasContext
*s
, arg_rpr_esz
*a
)
3052 return do_last_fp(s
, a
, false);
3055 static bool trans_LASTB_v(DisasContext
*s
, arg_rpr_esz
*a
)
3057 return do_last_fp(s
, a
, true);
3060 /* Compute LAST for a Xreg. */
3061 static bool do_last_general(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
3063 if (sve_access_check(s
)) {
3064 TCGv_i64 val
= do_last_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
);
3065 tcg_gen_mov_i64(cpu_reg(s
, a
->rd
), val
);
3066 tcg_temp_free_i64(val
);
3071 static bool trans_LASTA_r(DisasContext
*s
, arg_rpr_esz
*a
)
3073 return do_last_general(s
, a
, false);
3076 static bool trans_LASTB_r(DisasContext
*s
, arg_rpr_esz
*a
)
3078 return do_last_general(s
, a
, true);
3081 static bool trans_CPY_m_r(DisasContext
*s
, arg_rpr_esz
*a
)
3083 if (sve_access_check(s
)) {
3084 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rd
, a
->pg
, cpu_reg_sp(s
, a
->rn
));
3089 static bool trans_CPY_m_v(DisasContext
*s
, arg_rpr_esz
*a
)
3091 if (sve_access_check(s
)) {
3092 int ofs
= vec_reg_offset(s
, a
->rn
, 0, a
->esz
);
3093 TCGv_i64 t
= load_esz(cpu_env
, ofs
, a
->esz
);
3094 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rd
, a
->pg
, t
);
3095 tcg_temp_free_i64(t
);
3100 static bool trans_REVB(DisasContext
*s
, arg_rpr_esz
*a
)
3102 static gen_helper_gvec_3
* const fns
[4] = {
3104 gen_helper_sve_revb_h
,
3105 gen_helper_sve_revb_s
,
3106 gen_helper_sve_revb_d
,
3108 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
3111 static bool trans_REVH(DisasContext
*s
, arg_rpr_esz
*a
)
3113 static gen_helper_gvec_3
* const fns
[4] = {
3116 gen_helper_sve_revh_s
,
3117 gen_helper_sve_revh_d
,
3119 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
3122 static bool trans_REVW(DisasContext
*s
, arg_rpr_esz
*a
)
3124 return do_zpz_ool(s
, a
, a
->esz
== 3 ? gen_helper_sve_revw_d
: NULL
);
3127 static bool trans_RBIT(DisasContext
*s
, arg_rpr_esz
*a
)
3129 static gen_helper_gvec_3
* const fns
[4] = {
3130 gen_helper_sve_rbit_b
,
3131 gen_helper_sve_rbit_h
,
3132 gen_helper_sve_rbit_s
,
3133 gen_helper_sve_rbit_d
,
3135 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
3138 static bool trans_SPLICE(DisasContext
*s
, arg_rprr_esz
*a
)
3140 if (sve_access_check(s
)) {
3141 gen_gvec_ool_zzzp(s
, gen_helper_sve_splice
,
3142 a
->rd
, a
->rn
, a
->rm
, a
->pg
, a
->esz
);
3147 static bool trans_SPLICE_sve2(DisasContext
*s
, arg_rpr_esz
*a
)
3149 if (!dc_isar_feature(aa64_sve2
, s
)) {
3152 if (sve_access_check(s
)) {
3153 gen_gvec_ool_zzzp(s
, gen_helper_sve_splice
,
3154 a
->rd
, a
->rn
, (a
->rn
+ 1) % 32, a
->pg
, a
->esz
);
3160 *** SVE Integer Compare - Vectors Group
3163 static bool do_ppzz_flags(DisasContext
*s
, arg_rprr_esz
*a
,
3164 gen_helper_gvec_flags_4
*gen_fn
)
3166 TCGv_ptr pd
, zn
, zm
, pg
;
3170 if (gen_fn
== NULL
) {
3173 if (!sve_access_check(s
)) {
3177 vsz
= vec_full_reg_size(s
);
3178 t
= tcg_temp_new_i32();
3179 pd
= tcg_temp_new_ptr();
3180 zn
= tcg_temp_new_ptr();
3181 zm
= tcg_temp_new_ptr();
3182 pg
= tcg_temp_new_ptr();
3184 tcg_gen_addi_ptr(pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3185 tcg_gen_addi_ptr(zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
3186 tcg_gen_addi_ptr(zm
, cpu_env
, vec_full_reg_offset(s
, a
->rm
));
3187 tcg_gen_addi_ptr(pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3189 gen_fn(t
, pd
, zn
, zm
, pg
, tcg_constant_i32(simd_desc(vsz
, vsz
, 0)));
3191 tcg_temp_free_ptr(pd
);
3192 tcg_temp_free_ptr(zn
);
3193 tcg_temp_free_ptr(zm
);
3194 tcg_temp_free_ptr(pg
);
3198 tcg_temp_free_i32(t
);
3202 #define DO_PPZZ(NAME, name) \
3203 static bool trans_##NAME##_ppzz(DisasContext *s, arg_rprr_esz *a) \
3205 static gen_helper_gvec_flags_4 * const fns[4] = { \
3206 gen_helper_sve_##name##_ppzz_b, gen_helper_sve_##name##_ppzz_h, \
3207 gen_helper_sve_##name##_ppzz_s, gen_helper_sve_##name##_ppzz_d, \
3209 return do_ppzz_flags(s, a, fns[a->esz]); \
3212 DO_PPZZ(CMPEQ
, cmpeq
)
3213 DO_PPZZ(CMPNE
, cmpne
)
3214 DO_PPZZ(CMPGT
, cmpgt
)
3215 DO_PPZZ(CMPGE
, cmpge
)
3216 DO_PPZZ(CMPHI
, cmphi
)
3217 DO_PPZZ(CMPHS
, cmphs
)
3221 #define DO_PPZW(NAME, name) \
3222 static bool trans_##NAME##_ppzw(DisasContext *s, arg_rprr_esz *a) \
3224 static gen_helper_gvec_flags_4 * const fns[4] = { \
3225 gen_helper_sve_##name##_ppzw_b, gen_helper_sve_##name##_ppzw_h, \
3226 gen_helper_sve_##name##_ppzw_s, NULL \
3228 return do_ppzz_flags(s, a, fns[a->esz]); \
3231 DO_PPZW(CMPEQ
, cmpeq
)
3232 DO_PPZW(CMPNE
, cmpne
)
3233 DO_PPZW(CMPGT
, cmpgt
)
3234 DO_PPZW(CMPGE
, cmpge
)
3235 DO_PPZW(CMPHI
, cmphi
)
3236 DO_PPZW(CMPHS
, cmphs
)
3237 DO_PPZW(CMPLT
, cmplt
)
3238 DO_PPZW(CMPLE
, cmple
)
3239 DO_PPZW(CMPLO
, cmplo
)
3240 DO_PPZW(CMPLS
, cmpls
)
3245 *** SVE Integer Compare - Immediate Groups
3248 static bool do_ppzi_flags(DisasContext
*s
, arg_rpri_esz
*a
,
3249 gen_helper_gvec_flags_3
*gen_fn
)
3251 TCGv_ptr pd
, zn
, pg
;
3255 if (gen_fn
== NULL
) {
3258 if (!sve_access_check(s
)) {
3262 vsz
= vec_full_reg_size(s
);
3263 t
= tcg_temp_new_i32();
3264 pd
= tcg_temp_new_ptr();
3265 zn
= tcg_temp_new_ptr();
3266 pg
= tcg_temp_new_ptr();
3268 tcg_gen_addi_ptr(pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3269 tcg_gen_addi_ptr(zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
3270 tcg_gen_addi_ptr(pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3272 gen_fn(t
, pd
, zn
, pg
, tcg_constant_i32(simd_desc(vsz
, vsz
, a
->imm
)));
3274 tcg_temp_free_ptr(pd
);
3275 tcg_temp_free_ptr(zn
);
3276 tcg_temp_free_ptr(pg
);
3280 tcg_temp_free_i32(t
);
3284 #define DO_PPZI(NAME, name) \
3285 static bool trans_##NAME##_ppzi(DisasContext *s, arg_rpri_esz *a) \
3287 static gen_helper_gvec_flags_3 * const fns[4] = { \
3288 gen_helper_sve_##name##_ppzi_b, gen_helper_sve_##name##_ppzi_h, \
3289 gen_helper_sve_##name##_ppzi_s, gen_helper_sve_##name##_ppzi_d, \
3291 return do_ppzi_flags(s, a, fns[a->esz]); \
3294 DO_PPZI(CMPEQ
, cmpeq
)
3295 DO_PPZI(CMPNE
, cmpne
)
3296 DO_PPZI(CMPGT
, cmpgt
)
3297 DO_PPZI(CMPGE
, cmpge
)
3298 DO_PPZI(CMPHI
, cmphi
)
3299 DO_PPZI(CMPHS
, cmphs
)
3300 DO_PPZI(CMPLT
, cmplt
)
3301 DO_PPZI(CMPLE
, cmple
)
3302 DO_PPZI(CMPLO
, cmplo
)
3303 DO_PPZI(CMPLS
, cmpls
)
3308 *** SVE Partition Break Group
3311 static bool do_brk3(DisasContext
*s
, arg_rprr_s
*a
,
3312 gen_helper_gvec_4
*fn
, gen_helper_gvec_flags_4
*fn_s
)
3314 if (!sve_access_check(s
)) {
3318 unsigned vsz
= pred_full_reg_size(s
);
3320 /* Predicate sizes may be smaller and cannot use simd_desc. */
3321 TCGv_ptr d
= tcg_temp_new_ptr();
3322 TCGv_ptr n
= tcg_temp_new_ptr();
3323 TCGv_ptr m
= tcg_temp_new_ptr();
3324 TCGv_ptr g
= tcg_temp_new_ptr();
3325 TCGv_i32 t
= tcg_const_i32(FIELD_DP32(0, PREDDESC
, OPRSZ
, vsz
));
3327 tcg_gen_addi_ptr(d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3328 tcg_gen_addi_ptr(n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
3329 tcg_gen_addi_ptr(m
, cpu_env
, pred_full_reg_offset(s
, a
->rm
));
3330 tcg_gen_addi_ptr(g
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3333 fn_s(t
, d
, n
, m
, g
, t
);
3338 tcg_temp_free_ptr(d
);
3339 tcg_temp_free_ptr(n
);
3340 tcg_temp_free_ptr(m
);
3341 tcg_temp_free_ptr(g
);
3342 tcg_temp_free_i32(t
);
3346 static bool do_brk2(DisasContext
*s
, arg_rpr_s
*a
,
3347 gen_helper_gvec_3
*fn
, gen_helper_gvec_flags_3
*fn_s
)
3349 if (!sve_access_check(s
)) {
3353 unsigned vsz
= pred_full_reg_size(s
);
3355 /* Predicate sizes may be smaller and cannot use simd_desc. */
3356 TCGv_ptr d
= tcg_temp_new_ptr();
3357 TCGv_ptr n
= tcg_temp_new_ptr();
3358 TCGv_ptr g
= tcg_temp_new_ptr();
3359 TCGv_i32 t
= tcg_const_i32(FIELD_DP32(0, PREDDESC
, OPRSZ
, vsz
));
3361 tcg_gen_addi_ptr(d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3362 tcg_gen_addi_ptr(n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
3363 tcg_gen_addi_ptr(g
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3366 fn_s(t
, d
, n
, g
, t
);
3371 tcg_temp_free_ptr(d
);
3372 tcg_temp_free_ptr(n
);
3373 tcg_temp_free_ptr(g
);
3374 tcg_temp_free_i32(t
);
3378 static bool trans_BRKPA(DisasContext
*s
, arg_rprr_s
*a
)
3380 return do_brk3(s
, a
, gen_helper_sve_brkpa
, gen_helper_sve_brkpas
);
3383 static bool trans_BRKPB(DisasContext
*s
, arg_rprr_s
*a
)
3385 return do_brk3(s
, a
, gen_helper_sve_brkpb
, gen_helper_sve_brkpbs
);
3388 static bool trans_BRKA_m(DisasContext
*s
, arg_rpr_s
*a
)
3390 return do_brk2(s
, a
, gen_helper_sve_brka_m
, gen_helper_sve_brkas_m
);
3393 static bool trans_BRKB_m(DisasContext
*s
, arg_rpr_s
*a
)
3395 return do_brk2(s
, a
, gen_helper_sve_brkb_m
, gen_helper_sve_brkbs_m
);
3398 static bool trans_BRKA_z(DisasContext
*s
, arg_rpr_s
*a
)
3400 return do_brk2(s
, a
, gen_helper_sve_brka_z
, gen_helper_sve_brkas_z
);
3403 static bool trans_BRKB_z(DisasContext
*s
, arg_rpr_s
*a
)
3405 return do_brk2(s
, a
, gen_helper_sve_brkb_z
, gen_helper_sve_brkbs_z
);
3408 static bool trans_BRKN(DisasContext
*s
, arg_rpr_s
*a
)
3410 return do_brk2(s
, a
, gen_helper_sve_brkn
, gen_helper_sve_brkns
);
3414 *** SVE Predicate Count Group
3417 static void do_cntp(DisasContext
*s
, TCGv_i64 val
, int esz
, int pn
, int pg
)
3419 unsigned psz
= pred_full_reg_size(s
);
3424 tcg_gen_ld_i64(val
, cpu_env
, pred_full_reg_offset(s
, pn
));
3426 TCGv_i64 g
= tcg_temp_new_i64();
3427 tcg_gen_ld_i64(g
, cpu_env
, pred_full_reg_offset(s
, pg
));
3428 tcg_gen_and_i64(val
, val
, g
);
3429 tcg_temp_free_i64(g
);
3432 /* Reduce the pred_esz_masks value simply to reduce the
3433 * size of the code generated here.
3435 psz_mask
= MAKE_64BIT_MASK(0, psz
* 8);
3436 tcg_gen_andi_i64(val
, val
, pred_esz_masks
[esz
] & psz_mask
);
3438 tcg_gen_ctpop_i64(val
, val
);
3440 TCGv_ptr t_pn
= tcg_temp_new_ptr();
3441 TCGv_ptr t_pg
= tcg_temp_new_ptr();
3445 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, psz
);
3446 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, esz
);
3448 tcg_gen_addi_ptr(t_pn
, cpu_env
, pred_full_reg_offset(s
, pn
));
3449 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
3450 t_desc
= tcg_const_i32(desc
);
3452 gen_helper_sve_cntp(val
, t_pn
, t_pg
, t_desc
);
3453 tcg_temp_free_ptr(t_pn
);
3454 tcg_temp_free_ptr(t_pg
);
3455 tcg_temp_free_i32(t_desc
);
3459 static bool trans_CNTP(DisasContext
*s
, arg_CNTP
*a
)
3461 if (sve_access_check(s
)) {
3462 do_cntp(s
, cpu_reg(s
, a
->rd
), a
->esz
, a
->rn
, a
->pg
);
3467 static bool trans_INCDECP_r(DisasContext
*s
, arg_incdec_pred
*a
)
3469 if (sve_access_check(s
)) {
3470 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3471 TCGv_i64 val
= tcg_temp_new_i64();
3473 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3475 tcg_gen_sub_i64(reg
, reg
, val
);
3477 tcg_gen_add_i64(reg
, reg
, val
);
3479 tcg_temp_free_i64(val
);
3484 static bool trans_INCDECP_z(DisasContext
*s
, arg_incdec2_pred
*a
)
3489 if (sve_access_check(s
)) {
3490 unsigned vsz
= vec_full_reg_size(s
);
3491 TCGv_i64 val
= tcg_temp_new_i64();
3492 GVecGen2sFn
*gvec_fn
= a
->d
? tcg_gen_gvec_subs
: tcg_gen_gvec_adds
;
3494 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3495 gvec_fn(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3496 vec_full_reg_offset(s
, a
->rn
), val
, vsz
, vsz
);
3501 static bool trans_SINCDECP_r_32(DisasContext
*s
, arg_incdec_pred
*a
)
3503 if (sve_access_check(s
)) {
3504 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3505 TCGv_i64 val
= tcg_temp_new_i64();
3507 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3508 do_sat_addsub_32(reg
, val
, a
->u
, a
->d
);
3513 static bool trans_SINCDECP_r_64(DisasContext
*s
, arg_incdec_pred
*a
)
3515 if (sve_access_check(s
)) {
3516 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3517 TCGv_i64 val
= tcg_temp_new_i64();
3519 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3520 do_sat_addsub_64(reg
, val
, a
->u
, a
->d
);
3525 static bool trans_SINCDECP_z(DisasContext
*s
, arg_incdec2_pred
*a
)
3530 if (sve_access_check(s
)) {
3531 TCGv_i64 val
= tcg_temp_new_i64();
3532 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3533 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
, val
, a
->u
, a
->d
);
3539 *** SVE Integer Compare Scalars Group
3542 static bool trans_CTERM(DisasContext
*s
, arg_CTERM
*a
)
3544 if (!sve_access_check(s
)) {
3548 TCGCond cond
= (a
->ne
? TCG_COND_NE
: TCG_COND_EQ
);
3549 TCGv_i64 rn
= read_cpu_reg(s
, a
->rn
, a
->sf
);
3550 TCGv_i64 rm
= read_cpu_reg(s
, a
->rm
, a
->sf
);
3551 TCGv_i64 cmp
= tcg_temp_new_i64();
3553 tcg_gen_setcond_i64(cond
, cmp
, rn
, rm
);
3554 tcg_gen_extrl_i64_i32(cpu_NF
, cmp
);
3555 tcg_temp_free_i64(cmp
);
3557 /* VF = !NF & !CF. */
3558 tcg_gen_xori_i32(cpu_VF
, cpu_NF
, 1);
3559 tcg_gen_andc_i32(cpu_VF
, cpu_VF
, cpu_CF
);
3561 /* Both NF and VF actually look at bit 31. */
3562 tcg_gen_neg_i32(cpu_NF
, cpu_NF
);
3563 tcg_gen_neg_i32(cpu_VF
, cpu_VF
);
3567 static bool trans_WHILE(DisasContext
*s
, arg_WHILE
*a
)
3569 TCGv_i64 op0
, op1
, t0
, t1
, tmax
;
3572 unsigned vsz
= vec_full_reg_size(s
);
3576 /* Note that GE/HS has a->eq == 0 and GT/HI has a->eq == 1. */
3577 bool eq
= a
->eq
== a
->lt
;
3579 /* The greater-than conditions are all SVE2. */
3580 if (!a
->lt
&& !dc_isar_feature(aa64_sve2
, s
)) {
3583 if (!sve_access_check(s
)) {
3587 op0
= read_cpu_reg(s
, a
->rn
, 1);
3588 op1
= read_cpu_reg(s
, a
->rm
, 1);
3592 tcg_gen_ext32u_i64(op0
, op0
);
3593 tcg_gen_ext32u_i64(op1
, op1
);
3595 tcg_gen_ext32s_i64(op0
, op0
);
3596 tcg_gen_ext32s_i64(op1
, op1
);
3600 /* For the helper, compress the different conditions into a computation
3601 * of how many iterations for which the condition is true.
3603 t0
= tcg_temp_new_i64();
3604 t1
= tcg_temp_new_i64();
3607 tcg_gen_sub_i64(t0
, op1
, op0
);
3609 maxval
= a
->sf
? UINT64_MAX
: UINT32_MAX
;
3610 cond
= eq
? TCG_COND_LEU
: TCG_COND_LTU
;
3612 maxval
= a
->sf
? INT64_MAX
: INT32_MAX
;
3613 cond
= eq
? TCG_COND_LE
: TCG_COND_LT
;
3616 tcg_gen_sub_i64(t0
, op0
, op1
);
3619 cond
= eq
? TCG_COND_GEU
: TCG_COND_GTU
;
3621 maxval
= a
->sf
? INT64_MIN
: INT32_MIN
;
3622 cond
= eq
? TCG_COND_GE
: TCG_COND_GT
;
3626 tmax
= tcg_constant_i64(vsz
>> a
->esz
);
3628 /* Equality means one more iteration. */
3629 tcg_gen_addi_i64(t0
, t0
, 1);
3632 * For the less-than while, if op1 is maxval (and the only time
3633 * the addition above could overflow), then we produce an all-true
3634 * predicate by setting the count to the vector length. This is
3635 * because the pseudocode is described as an increment + compare
3636 * loop, and the maximum integer would always compare true.
3637 * Similarly, the greater-than while has the same issue with the
3638 * minimum integer due to the decrement + compare loop.
3640 tcg_gen_movi_i64(t1
, maxval
);
3641 tcg_gen_movcond_i64(TCG_COND_EQ
, t0
, op1
, t1
, tmax
, t0
);
3644 /* Bound to the maximum. */
3645 tcg_gen_umin_i64(t0
, t0
, tmax
);
3647 /* Set the count to zero if the condition is false. */
3648 tcg_gen_movi_i64(t1
, 0);
3649 tcg_gen_movcond_i64(cond
, t0
, op0
, op1
, t0
, t1
);
3650 tcg_temp_free_i64(t1
);
3652 /* Since we're bounded, pass as a 32-bit type. */
3653 t2
= tcg_temp_new_i32();
3654 tcg_gen_extrl_i64_i32(t2
, t0
);
3655 tcg_temp_free_i64(t0
);
3657 /* Scale elements to bits. */
3658 tcg_gen_shli_i32(t2
, t2
, a
->esz
);
3660 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
/ 8);
3661 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
3663 ptr
= tcg_temp_new_ptr();
3664 tcg_gen_addi_ptr(ptr
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3667 gen_helper_sve_whilel(t2
, ptr
, t2
, tcg_constant_i32(desc
));
3669 gen_helper_sve_whileg(t2
, ptr
, t2
, tcg_constant_i32(desc
));
3673 tcg_temp_free_ptr(ptr
);
3674 tcg_temp_free_i32(t2
);
3678 static bool trans_WHILE_ptr(DisasContext
*s
, arg_WHILE_ptr
*a
)
3680 TCGv_i64 op0
, op1
, diff
, t1
, tmax
;
3683 unsigned vsz
= vec_full_reg_size(s
);
3686 if (!dc_isar_feature(aa64_sve2
, s
)) {
3689 if (!sve_access_check(s
)) {
3693 op0
= read_cpu_reg(s
, a
->rn
, 1);
3694 op1
= read_cpu_reg(s
, a
->rm
, 1);
3696 tmax
= tcg_constant_i64(vsz
);
3697 diff
= tcg_temp_new_i64();
3701 /* diff = abs(op1 - op0), noting that op0/1 are unsigned. */
3702 t1
= tcg_temp_new_i64();
3703 tcg_gen_sub_i64(diff
, op0
, op1
);
3704 tcg_gen_sub_i64(t1
, op1
, op0
);
3705 tcg_gen_movcond_i64(TCG_COND_GEU
, diff
, op0
, op1
, diff
, t1
);
3706 tcg_temp_free_i64(t1
);
3707 /* Round down to a multiple of ESIZE. */
3708 tcg_gen_andi_i64(diff
, diff
, -1 << a
->esz
);
3709 /* If op1 == op0, diff == 0, and the condition is always true. */
3710 tcg_gen_movcond_i64(TCG_COND_EQ
, diff
, op0
, op1
, tmax
, diff
);
3713 tcg_gen_sub_i64(diff
, op1
, op0
);
3714 /* Round down to a multiple of ESIZE. */
3715 tcg_gen_andi_i64(diff
, diff
, -1 << a
->esz
);
3716 /* If op0 >= op1, diff <= 0, the condition is always true. */
3717 tcg_gen_movcond_i64(TCG_COND_GEU
, diff
, op0
, op1
, tmax
, diff
);
3720 /* Bound to the maximum. */
3721 tcg_gen_umin_i64(diff
, diff
, tmax
);
3723 /* Since we're bounded, pass as a 32-bit type. */
3724 t2
= tcg_temp_new_i32();
3725 tcg_gen_extrl_i64_i32(t2
, diff
);
3726 tcg_temp_free_i64(diff
);
3728 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
/ 8);
3729 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
3731 ptr
= tcg_temp_new_ptr();
3732 tcg_gen_addi_ptr(ptr
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3734 gen_helper_sve_whilel(t2
, ptr
, t2
, tcg_constant_i32(desc
));
3737 tcg_temp_free_ptr(ptr
);
3738 tcg_temp_free_i32(t2
);
3743 *** SVE Integer Wide Immediate - Unpredicated Group
3746 static bool trans_FDUP(DisasContext
*s
, arg_FDUP
*a
)
3751 if (sve_access_check(s
)) {
3752 unsigned vsz
= vec_full_reg_size(s
);
3753 int dofs
= vec_full_reg_offset(s
, a
->rd
);
3756 /* Decode the VFP immediate. */
3757 imm
= vfp_expand_imm(a
->esz
, a
->imm
);
3758 tcg_gen_gvec_dup_imm(a
->esz
, dofs
, vsz
, vsz
, imm
);
3763 static bool trans_DUP_i(DisasContext
*s
, arg_DUP_i
*a
)
3765 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3768 if (sve_access_check(s
)) {
3769 unsigned vsz
= vec_full_reg_size(s
);
3770 int dofs
= vec_full_reg_offset(s
, a
->rd
);
3772 tcg_gen_gvec_dup_imm(a
->esz
, dofs
, vsz
, vsz
, a
->imm
);
3777 static bool trans_ADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3779 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3782 if (sve_access_check(s
)) {
3783 unsigned vsz
= vec_full_reg_size(s
);
3784 tcg_gen_gvec_addi(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3785 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
3790 static bool trans_SUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3793 return trans_ADD_zzi(s
, a
);
3796 static bool trans_SUBR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3798 static const TCGOpcode vecop_list
[] = { INDEX_op_sub_vec
, 0 };
3799 static const GVecGen2s op
[4] = {
3800 { .fni8
= tcg_gen_vec_sub8_i64
,
3801 .fniv
= tcg_gen_sub_vec
,
3802 .fno
= gen_helper_sve_subri_b
,
3803 .opt_opc
= vecop_list
,
3805 .scalar_first
= true },
3806 { .fni8
= tcg_gen_vec_sub16_i64
,
3807 .fniv
= tcg_gen_sub_vec
,
3808 .fno
= gen_helper_sve_subri_h
,
3809 .opt_opc
= vecop_list
,
3811 .scalar_first
= true },
3812 { .fni4
= tcg_gen_sub_i32
,
3813 .fniv
= tcg_gen_sub_vec
,
3814 .fno
= gen_helper_sve_subri_s
,
3815 .opt_opc
= vecop_list
,
3817 .scalar_first
= true },
3818 { .fni8
= tcg_gen_sub_i64
,
3819 .fniv
= tcg_gen_sub_vec
,
3820 .fno
= gen_helper_sve_subri_d
,
3821 .opt_opc
= vecop_list
,
3822 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
3824 .scalar_first
= true }
3827 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3830 if (sve_access_check(s
)) {
3831 unsigned vsz
= vec_full_reg_size(s
);
3832 tcg_gen_gvec_2s(vec_full_reg_offset(s
, a
->rd
),
3833 vec_full_reg_offset(s
, a
->rn
),
3834 vsz
, vsz
, tcg_constant_i64(a
->imm
), &op
[a
->esz
]);
3839 static bool trans_MUL_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3841 if (sve_access_check(s
)) {
3842 unsigned vsz
= vec_full_reg_size(s
);
3843 tcg_gen_gvec_muli(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3844 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
3849 static bool do_zzi_sat(DisasContext
*s
, arg_rri_esz
*a
, bool u
, bool d
)
3851 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3854 if (sve_access_check(s
)) {
3855 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
,
3856 tcg_constant_i64(a
->imm
), u
, d
);
3861 static bool trans_SQADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3863 return do_zzi_sat(s
, a
, false, false);
3866 static bool trans_UQADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3868 return do_zzi_sat(s
, a
, true, false);
3871 static bool trans_SQSUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3873 return do_zzi_sat(s
, a
, false, true);
3876 static bool trans_UQSUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3878 return do_zzi_sat(s
, a
, true, true);
3881 static bool do_zzi_ool(DisasContext
*s
, arg_rri_esz
*a
, gen_helper_gvec_2i
*fn
)
3883 if (sve_access_check(s
)) {
3884 unsigned vsz
= vec_full_reg_size(s
);
3885 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s
, a
->rd
),
3886 vec_full_reg_offset(s
, a
->rn
),
3887 tcg_constant_i64(a
->imm
), vsz
, vsz
, 0, fn
);
3892 #define DO_ZZI(NAME, name) \
3893 static bool trans_##NAME##_zzi(DisasContext *s, arg_rri_esz *a) \
3895 static gen_helper_gvec_2i * const fns[4] = { \
3896 gen_helper_sve_##name##i_b, gen_helper_sve_##name##i_h, \
3897 gen_helper_sve_##name##i_s, gen_helper_sve_##name##i_d, \
3899 return do_zzi_ool(s, a, fns[a->esz]); \
3909 static bool trans_DOT_zzzz(DisasContext
*s
, arg_DOT_zzzz
*a
)
3911 static gen_helper_gvec_4
* const fns
[2][2] = {
3912 { gen_helper_gvec_sdot_b
, gen_helper_gvec_sdot_h
},
3913 { gen_helper_gvec_udot_b
, gen_helper_gvec_udot_h
}
3916 if (sve_access_check(s
)) {
3917 gen_gvec_ool_zzzz(s
, fns
[a
->u
][a
->sz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, 0);
3923 * SVE Multiply - Indexed
3926 static bool do_zzxz_ool(DisasContext
*s
, arg_rrxr_esz
*a
,
3927 gen_helper_gvec_4
*fn
)
3932 if (sve_access_check(s
)) {
3933 gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->index
);
3938 #define DO_RRXR(NAME, FUNC) \
3939 static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
3940 { return do_zzxz_ool(s, a, FUNC); }
3942 DO_RRXR(trans_SDOT_zzxw_s
, gen_helper_gvec_sdot_idx_b
)
3943 DO_RRXR(trans_SDOT_zzxw_d
, gen_helper_gvec_sdot_idx_h
)
3944 DO_RRXR(trans_UDOT_zzxw_s
, gen_helper_gvec_udot_idx_b
)
3945 DO_RRXR(trans_UDOT_zzxw_d
, gen_helper_gvec_udot_idx_h
)
3947 static bool trans_SUDOT_zzxw_s(DisasContext
*s
, arg_rrxr_esz
*a
)
3949 if (!dc_isar_feature(aa64_sve_i8mm
, s
)) {
3952 return do_zzxz_ool(s
, a
, gen_helper_gvec_sudot_idx_b
);
3955 static bool trans_USDOT_zzxw_s(DisasContext
*s
, arg_rrxr_esz
*a
)
3957 if (!dc_isar_feature(aa64_sve_i8mm
, s
)) {
3960 return do_zzxz_ool(s
, a
, gen_helper_gvec_usdot_idx_b
);
3965 static bool do_sve2_zzz_data(DisasContext
*s
, int rd
, int rn
, int rm
, int data
,
3966 gen_helper_gvec_3
*fn
)
3968 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
3971 if (sve_access_check(s
)) {
3972 unsigned vsz
= vec_full_reg_size(s
);
3973 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
3974 vec_full_reg_offset(s
, rn
),
3975 vec_full_reg_offset(s
, rm
),
3976 vsz
, vsz
, data
, fn
);
3981 #define DO_SVE2_RRX(NAME, FUNC) \
3982 static bool NAME(DisasContext *s, arg_rrx_esz *a) \
3983 { return do_sve2_zzz_data(s, a->rd, a->rn, a->rm, a->index, FUNC); }
3985 DO_SVE2_RRX(trans_MUL_zzx_h
, gen_helper_gvec_mul_idx_h
)
3986 DO_SVE2_RRX(trans_MUL_zzx_s
, gen_helper_gvec_mul_idx_s
)
3987 DO_SVE2_RRX(trans_MUL_zzx_d
, gen_helper_gvec_mul_idx_d
)
3989 DO_SVE2_RRX(trans_SQDMULH_zzx_h
, gen_helper_sve2_sqdmulh_idx_h
)
3990 DO_SVE2_RRX(trans_SQDMULH_zzx_s
, gen_helper_sve2_sqdmulh_idx_s
)
3991 DO_SVE2_RRX(trans_SQDMULH_zzx_d
, gen_helper_sve2_sqdmulh_idx_d
)
3993 DO_SVE2_RRX(trans_SQRDMULH_zzx_h
, gen_helper_sve2_sqrdmulh_idx_h
)
3994 DO_SVE2_RRX(trans_SQRDMULH_zzx_s
, gen_helper_sve2_sqrdmulh_idx_s
)
3995 DO_SVE2_RRX(trans_SQRDMULH_zzx_d
, gen_helper_sve2_sqrdmulh_idx_d
)
3999 #define DO_SVE2_RRX_TB(NAME, FUNC, TOP) \
4000 static bool NAME(DisasContext *s, arg_rrx_esz *a) \
4002 return do_sve2_zzz_data(s, a->rd, a->rn, a->rm, \
4003 (a->index << 1) | TOP, FUNC); \
4006 DO_SVE2_RRX_TB(trans_SQDMULLB_zzx_s
, gen_helper_sve2_sqdmull_idx_s
, false)
4007 DO_SVE2_RRX_TB(trans_SQDMULLB_zzx_d
, gen_helper_sve2_sqdmull_idx_d
, false)
4008 DO_SVE2_RRX_TB(trans_SQDMULLT_zzx_s
, gen_helper_sve2_sqdmull_idx_s
, true)
4009 DO_SVE2_RRX_TB(trans_SQDMULLT_zzx_d
, gen_helper_sve2_sqdmull_idx_d
, true)
4011 DO_SVE2_RRX_TB(trans_SMULLB_zzx_s
, gen_helper_sve2_smull_idx_s
, false)
4012 DO_SVE2_RRX_TB(trans_SMULLB_zzx_d
, gen_helper_sve2_smull_idx_d
, false)
4013 DO_SVE2_RRX_TB(trans_SMULLT_zzx_s
, gen_helper_sve2_smull_idx_s
, true)
4014 DO_SVE2_RRX_TB(trans_SMULLT_zzx_d
, gen_helper_sve2_smull_idx_d
, true)
4016 DO_SVE2_RRX_TB(trans_UMULLB_zzx_s
, gen_helper_sve2_umull_idx_s
, false)
4017 DO_SVE2_RRX_TB(trans_UMULLB_zzx_d
, gen_helper_sve2_umull_idx_d
, false)
4018 DO_SVE2_RRX_TB(trans_UMULLT_zzx_s
, gen_helper_sve2_umull_idx_s
, true)
4019 DO_SVE2_RRX_TB(trans_UMULLT_zzx_d
, gen_helper_sve2_umull_idx_d
, true)
4021 #undef DO_SVE2_RRX_TB
4023 static bool do_sve2_zzzz_data(DisasContext
*s
, int rd
, int rn
, int rm
, int ra
,
4024 int data
, gen_helper_gvec_4
*fn
)
4026 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
4029 if (sve_access_check(s
)) {
4030 unsigned vsz
= vec_full_reg_size(s
);
4031 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
4032 vec_full_reg_offset(s
, rn
),
4033 vec_full_reg_offset(s
, rm
),
4034 vec_full_reg_offset(s
, ra
),
4035 vsz
, vsz
, data
, fn
);
4040 #define DO_SVE2_RRXR(NAME, FUNC) \
4041 static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
4042 { return do_sve2_zzzz_data(s, a->rd, a->rn, a->rm, a->ra, a->index, FUNC); }
4044 DO_SVE2_RRXR(trans_MLA_zzxz_h
, gen_helper_gvec_mla_idx_h
)
4045 DO_SVE2_RRXR(trans_MLA_zzxz_s
, gen_helper_gvec_mla_idx_s
)
4046 DO_SVE2_RRXR(trans_MLA_zzxz_d
, gen_helper_gvec_mla_idx_d
)
4048 DO_SVE2_RRXR(trans_MLS_zzxz_h
, gen_helper_gvec_mls_idx_h
)
4049 DO_SVE2_RRXR(trans_MLS_zzxz_s
, gen_helper_gvec_mls_idx_s
)
4050 DO_SVE2_RRXR(trans_MLS_zzxz_d
, gen_helper_gvec_mls_idx_d
)
4052 DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_h
, gen_helper_sve2_sqrdmlah_idx_h
)
4053 DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_s
, gen_helper_sve2_sqrdmlah_idx_s
)
4054 DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_d
, gen_helper_sve2_sqrdmlah_idx_d
)
4056 DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_h
, gen_helper_sve2_sqrdmlsh_idx_h
)
4057 DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_s
, gen_helper_sve2_sqrdmlsh_idx_s
)
4058 DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_d
, gen_helper_sve2_sqrdmlsh_idx_d
)
4062 #define DO_SVE2_RRXR_TB(NAME, FUNC, TOP) \
4063 static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
4065 return do_sve2_zzzz_data(s, a->rd, a->rn, a->rm, a->rd, \
4066 (a->index << 1) | TOP, FUNC); \
4069 DO_SVE2_RRXR_TB(trans_SQDMLALB_zzxw_s
, gen_helper_sve2_sqdmlal_idx_s
, false)
4070 DO_SVE2_RRXR_TB(trans_SQDMLALB_zzxw_d
, gen_helper_sve2_sqdmlal_idx_d
, false)
4071 DO_SVE2_RRXR_TB(trans_SQDMLALT_zzxw_s
, gen_helper_sve2_sqdmlal_idx_s
, true)
4072 DO_SVE2_RRXR_TB(trans_SQDMLALT_zzxw_d
, gen_helper_sve2_sqdmlal_idx_d
, true)
4074 DO_SVE2_RRXR_TB(trans_SQDMLSLB_zzxw_s
, gen_helper_sve2_sqdmlsl_idx_s
, false)
4075 DO_SVE2_RRXR_TB(trans_SQDMLSLB_zzxw_d
, gen_helper_sve2_sqdmlsl_idx_d
, false)
4076 DO_SVE2_RRXR_TB(trans_SQDMLSLT_zzxw_s
, gen_helper_sve2_sqdmlsl_idx_s
, true)
4077 DO_SVE2_RRXR_TB(trans_SQDMLSLT_zzxw_d
, gen_helper_sve2_sqdmlsl_idx_d
, true)
4079 DO_SVE2_RRXR_TB(trans_SMLALB_zzxw_s
, gen_helper_sve2_smlal_idx_s
, false)
4080 DO_SVE2_RRXR_TB(trans_SMLALB_zzxw_d
, gen_helper_sve2_smlal_idx_d
, false)
4081 DO_SVE2_RRXR_TB(trans_SMLALT_zzxw_s
, gen_helper_sve2_smlal_idx_s
, true)
4082 DO_SVE2_RRXR_TB(trans_SMLALT_zzxw_d
, gen_helper_sve2_smlal_idx_d
, true)
4084 DO_SVE2_RRXR_TB(trans_UMLALB_zzxw_s
, gen_helper_sve2_umlal_idx_s
, false)
4085 DO_SVE2_RRXR_TB(trans_UMLALB_zzxw_d
, gen_helper_sve2_umlal_idx_d
, false)
4086 DO_SVE2_RRXR_TB(trans_UMLALT_zzxw_s
, gen_helper_sve2_umlal_idx_s
, true)
4087 DO_SVE2_RRXR_TB(trans_UMLALT_zzxw_d
, gen_helper_sve2_umlal_idx_d
, true)
4089 DO_SVE2_RRXR_TB(trans_SMLSLB_zzxw_s
, gen_helper_sve2_smlsl_idx_s
, false)
4090 DO_SVE2_RRXR_TB(trans_SMLSLB_zzxw_d
, gen_helper_sve2_smlsl_idx_d
, false)
4091 DO_SVE2_RRXR_TB(trans_SMLSLT_zzxw_s
, gen_helper_sve2_smlsl_idx_s
, true)
4092 DO_SVE2_RRXR_TB(trans_SMLSLT_zzxw_d
, gen_helper_sve2_smlsl_idx_d
, true)
4094 DO_SVE2_RRXR_TB(trans_UMLSLB_zzxw_s
, gen_helper_sve2_umlsl_idx_s
, false)
4095 DO_SVE2_RRXR_TB(trans_UMLSLB_zzxw_d
, gen_helper_sve2_umlsl_idx_d
, false)
4096 DO_SVE2_RRXR_TB(trans_UMLSLT_zzxw_s
, gen_helper_sve2_umlsl_idx_s
, true)
4097 DO_SVE2_RRXR_TB(trans_UMLSLT_zzxw_d
, gen_helper_sve2_umlsl_idx_d
, true)
4099 #undef DO_SVE2_RRXR_TB
4101 #define DO_SVE2_RRXR_ROT(NAME, FUNC) \
4102 static bool trans_##NAME(DisasContext *s, arg_##NAME *a) \
4104 return do_sve2_zzzz_data(s, a->rd, a->rn, a->rm, a->ra, \
4105 (a->index << 2) | a->rot, FUNC); \
4108 DO_SVE2_RRXR_ROT(CMLA_zzxz_h
, gen_helper_sve2_cmla_idx_h
)
4109 DO_SVE2_RRXR_ROT(CMLA_zzxz_s
, gen_helper_sve2_cmla_idx_s
)
4111 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_h
, gen_helper_sve2_sqrdcmlah_idx_h
)
4112 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_s
, gen_helper_sve2_sqrdcmlah_idx_s
)
4114 DO_SVE2_RRXR_ROT(CDOT_zzxw_s
, gen_helper_sve2_cdot_idx_s
)
4115 DO_SVE2_RRXR_ROT(CDOT_zzxw_d
, gen_helper_sve2_cdot_idx_d
)
4117 #undef DO_SVE2_RRXR_ROT
4120 *** SVE Floating Point Multiply-Add Indexed Group
4123 static bool do_FMLA_zzxz(DisasContext
*s
, arg_rrxr_esz
*a
, bool sub
)
4125 static gen_helper_gvec_4_ptr
* const fns
[3] = {
4126 gen_helper_gvec_fmla_idx_h
,
4127 gen_helper_gvec_fmla_idx_s
,
4128 gen_helper_gvec_fmla_idx_d
,
4131 if (sve_access_check(s
)) {
4132 unsigned vsz
= vec_full_reg_size(s
);
4133 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4134 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4135 vec_full_reg_offset(s
, a
->rn
),
4136 vec_full_reg_offset(s
, a
->rm
),
4137 vec_full_reg_offset(s
, a
->ra
),
4138 status
, vsz
, vsz
, (a
->index
<< 1) | sub
,
4140 tcg_temp_free_ptr(status
);
4145 static bool trans_FMLA_zzxz(DisasContext
*s
, arg_FMLA_zzxz
*a
)
4147 return do_FMLA_zzxz(s
, a
, false);
4150 static bool trans_FMLS_zzxz(DisasContext
*s
, arg_FMLA_zzxz
*a
)
4152 return do_FMLA_zzxz(s
, a
, true);
4156 *** SVE Floating Point Multiply Indexed Group
4159 static bool trans_FMUL_zzx(DisasContext
*s
, arg_FMUL_zzx
*a
)
4161 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4162 gen_helper_gvec_fmul_idx_h
,
4163 gen_helper_gvec_fmul_idx_s
,
4164 gen_helper_gvec_fmul_idx_d
,
4167 if (sve_access_check(s
)) {
4168 unsigned vsz
= vec_full_reg_size(s
);
4169 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4170 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4171 vec_full_reg_offset(s
, a
->rn
),
4172 vec_full_reg_offset(s
, a
->rm
),
4173 status
, vsz
, vsz
, a
->index
, fns
[a
->esz
- 1]);
4174 tcg_temp_free_ptr(status
);
4180 *** SVE Floating Point Fast Reduction Group
4183 typedef void gen_helper_fp_reduce(TCGv_i64
, TCGv_ptr
, TCGv_ptr
,
4184 TCGv_ptr
, TCGv_i32
);
4186 static void do_reduce(DisasContext
*s
, arg_rpr_esz
*a
,
4187 gen_helper_fp_reduce
*fn
)
4189 unsigned vsz
= vec_full_reg_size(s
);
4190 unsigned p2vsz
= pow2ceil(vsz
);
4191 TCGv_i32 t_desc
= tcg_const_i32(simd_desc(vsz
, vsz
, p2vsz
));
4192 TCGv_ptr t_zn
, t_pg
, status
;
4195 temp
= tcg_temp_new_i64();
4196 t_zn
= tcg_temp_new_ptr();
4197 t_pg
= tcg_temp_new_ptr();
4199 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
4200 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
4201 status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4203 fn(temp
, t_zn
, t_pg
, status
, t_desc
);
4204 tcg_temp_free_ptr(t_zn
);
4205 tcg_temp_free_ptr(t_pg
);
4206 tcg_temp_free_ptr(status
);
4207 tcg_temp_free_i32(t_desc
);
4209 write_fp_dreg(s
, a
->rd
, temp
);
4210 tcg_temp_free_i64(temp
);
4213 #define DO_VPZ(NAME, name) \
4214 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
4216 static gen_helper_fp_reduce * const fns[3] = { \
4217 gen_helper_sve_##name##_h, \
4218 gen_helper_sve_##name##_s, \
4219 gen_helper_sve_##name##_d, \
4221 if (a->esz == 0) { \
4224 if (sve_access_check(s)) { \
4225 do_reduce(s, a, fns[a->esz - 1]); \
4230 DO_VPZ(FADDV
, faddv
)
4231 DO_VPZ(FMINNMV
, fminnmv
)
4232 DO_VPZ(FMAXNMV
, fmaxnmv
)
4233 DO_VPZ(FMINV
, fminv
)
4234 DO_VPZ(FMAXV
, fmaxv
)
4237 *** SVE Floating Point Unary Operations - Unpredicated Group
4240 static void do_zz_fp(DisasContext
*s
, arg_rr_esz
*a
, gen_helper_gvec_2_ptr
*fn
)
4242 unsigned vsz
= vec_full_reg_size(s
);
4243 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4245 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s
, a
->rd
),
4246 vec_full_reg_offset(s
, a
->rn
),
4247 status
, vsz
, vsz
, 0, fn
);
4248 tcg_temp_free_ptr(status
);
4251 static bool trans_FRECPE(DisasContext
*s
, arg_rr_esz
*a
)
4253 static gen_helper_gvec_2_ptr
* const fns
[3] = {
4254 gen_helper_gvec_frecpe_h
,
4255 gen_helper_gvec_frecpe_s
,
4256 gen_helper_gvec_frecpe_d
,
4261 if (sve_access_check(s
)) {
4262 do_zz_fp(s
, a
, fns
[a
->esz
- 1]);
4267 static bool trans_FRSQRTE(DisasContext
*s
, arg_rr_esz
*a
)
4269 static gen_helper_gvec_2_ptr
* const fns
[3] = {
4270 gen_helper_gvec_frsqrte_h
,
4271 gen_helper_gvec_frsqrte_s
,
4272 gen_helper_gvec_frsqrte_d
,
4277 if (sve_access_check(s
)) {
4278 do_zz_fp(s
, a
, fns
[a
->esz
- 1]);
4284 *** SVE Floating Point Compare with Zero Group
4287 static void do_ppz_fp(DisasContext
*s
, arg_rpr_esz
*a
,
4288 gen_helper_gvec_3_ptr
*fn
)
4290 unsigned vsz
= vec_full_reg_size(s
);
4291 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4293 tcg_gen_gvec_3_ptr(pred_full_reg_offset(s
, a
->rd
),
4294 vec_full_reg_offset(s
, a
->rn
),
4295 pred_full_reg_offset(s
, a
->pg
),
4296 status
, vsz
, vsz
, 0, fn
);
4297 tcg_temp_free_ptr(status
);
4300 #define DO_PPZ(NAME, name) \
4301 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
4303 static gen_helper_gvec_3_ptr * const fns[3] = { \
4304 gen_helper_sve_##name##_h, \
4305 gen_helper_sve_##name##_s, \
4306 gen_helper_sve_##name##_d, \
4308 if (a->esz == 0) { \
4311 if (sve_access_check(s)) { \
4312 do_ppz_fp(s, a, fns[a->esz - 1]); \
4317 DO_PPZ(FCMGE_ppz0
, fcmge0
)
4318 DO_PPZ(FCMGT_ppz0
, fcmgt0
)
4319 DO_PPZ(FCMLE_ppz0
, fcmle0
)
4320 DO_PPZ(FCMLT_ppz0
, fcmlt0
)
4321 DO_PPZ(FCMEQ_ppz0
, fcmeq0
)
4322 DO_PPZ(FCMNE_ppz0
, fcmne0
)
4327 *** SVE floating-point trig multiply-add coefficient
4330 static bool trans_FTMAD(DisasContext
*s
, arg_FTMAD
*a
)
4332 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4333 gen_helper_sve_ftmad_h
,
4334 gen_helper_sve_ftmad_s
,
4335 gen_helper_sve_ftmad_d
,
4341 if (sve_access_check(s
)) {
4342 unsigned vsz
= vec_full_reg_size(s
);
4343 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4344 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4345 vec_full_reg_offset(s
, a
->rn
),
4346 vec_full_reg_offset(s
, a
->rm
),
4347 status
, vsz
, vsz
, a
->imm
, fns
[a
->esz
- 1]);
4348 tcg_temp_free_ptr(status
);
4354 *** SVE Floating Point Accumulating Reduction Group
4357 static bool trans_FADDA(DisasContext
*s
, arg_rprr_esz
*a
)
4359 typedef void fadda_fn(TCGv_i64
, TCGv_i64
, TCGv_ptr
,
4360 TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
4361 static fadda_fn
* const fns
[3] = {
4362 gen_helper_sve_fadda_h
,
4363 gen_helper_sve_fadda_s
,
4364 gen_helper_sve_fadda_d
,
4366 unsigned vsz
= vec_full_reg_size(s
);
4367 TCGv_ptr t_rm
, t_pg
, t_fpst
;
4374 if (!sve_access_check(s
)) {
4378 t_val
= load_esz(cpu_env
, vec_reg_offset(s
, a
->rn
, 0, a
->esz
), a
->esz
);
4379 t_rm
= tcg_temp_new_ptr();
4380 t_pg
= tcg_temp_new_ptr();
4381 tcg_gen_addi_ptr(t_rm
, cpu_env
, vec_full_reg_offset(s
, a
->rm
));
4382 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
4383 t_fpst
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4384 t_desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
4386 fns
[a
->esz
- 1](t_val
, t_val
, t_rm
, t_pg
, t_fpst
, t_desc
);
4388 tcg_temp_free_i32(t_desc
);
4389 tcg_temp_free_ptr(t_fpst
);
4390 tcg_temp_free_ptr(t_pg
);
4391 tcg_temp_free_ptr(t_rm
);
4393 write_fp_dreg(s
, a
->rd
, t_val
);
4394 tcg_temp_free_i64(t_val
);
4399 *** SVE Floating Point Arithmetic - Unpredicated Group
4402 static bool do_zzz_fp(DisasContext
*s
, arg_rrr_esz
*a
,
4403 gen_helper_gvec_3_ptr
*fn
)
4408 if (sve_access_check(s
)) {
4409 unsigned vsz
= vec_full_reg_size(s
);
4410 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4411 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4412 vec_full_reg_offset(s
, a
->rn
),
4413 vec_full_reg_offset(s
, a
->rm
),
4414 status
, vsz
, vsz
, 0, fn
);
4415 tcg_temp_free_ptr(status
);
4421 #define DO_FP3(NAME, name) \
4422 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
4424 static gen_helper_gvec_3_ptr * const fns[4] = { \
4425 NULL, gen_helper_gvec_##name##_h, \
4426 gen_helper_gvec_##name##_s, gen_helper_gvec_##name##_d \
4428 return do_zzz_fp(s, a, fns[a->esz]); \
4431 DO_FP3(FADD_zzz
, fadd
)
4432 DO_FP3(FSUB_zzz
, fsub
)
4433 DO_FP3(FMUL_zzz
, fmul
)
4434 DO_FP3(FTSMUL
, ftsmul
)
4435 DO_FP3(FRECPS
, recps
)
4436 DO_FP3(FRSQRTS
, rsqrts
)
4441 *** SVE Floating Point Arithmetic - Predicated Group
4444 static bool do_zpzz_fp(DisasContext
*s
, arg_rprr_esz
*a
,
4445 gen_helper_gvec_4_ptr
*fn
)
4450 if (sve_access_check(s
)) {
4451 unsigned vsz
= vec_full_reg_size(s
);
4452 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4453 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4454 vec_full_reg_offset(s
, a
->rn
),
4455 vec_full_reg_offset(s
, a
->rm
),
4456 pred_full_reg_offset(s
, a
->pg
),
4457 status
, vsz
, vsz
, 0, fn
);
4458 tcg_temp_free_ptr(status
);
4463 #define DO_FP3(NAME, name) \
4464 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
4466 static gen_helper_gvec_4_ptr * const fns[4] = { \
4467 NULL, gen_helper_sve_##name##_h, \
4468 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4470 return do_zpzz_fp(s, a, fns[a->esz]); \
4473 DO_FP3(FADD_zpzz
, fadd
)
4474 DO_FP3(FSUB_zpzz
, fsub
)
4475 DO_FP3(FMUL_zpzz
, fmul
)
4476 DO_FP3(FMIN_zpzz
, fmin
)
4477 DO_FP3(FMAX_zpzz
, fmax
)
4478 DO_FP3(FMINNM_zpzz
, fminnum
)
4479 DO_FP3(FMAXNM_zpzz
, fmaxnum
)
4481 DO_FP3(FSCALE
, fscalbn
)
4483 DO_FP3(FMULX
, fmulx
)
4487 typedef void gen_helper_sve_fp2scalar(TCGv_ptr
, TCGv_ptr
, TCGv_ptr
,
4488 TCGv_i64
, TCGv_ptr
, TCGv_i32
);
4490 static void do_fp_scalar(DisasContext
*s
, int zd
, int zn
, int pg
, bool is_fp16
,
4491 TCGv_i64 scalar
, gen_helper_sve_fp2scalar
*fn
)
4493 unsigned vsz
= vec_full_reg_size(s
);
4494 TCGv_ptr t_zd
, t_zn
, t_pg
, status
;
4497 t_zd
= tcg_temp_new_ptr();
4498 t_zn
= tcg_temp_new_ptr();
4499 t_pg
= tcg_temp_new_ptr();
4500 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, zd
));
4501 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, zn
));
4502 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
4504 status
= fpstatus_ptr(is_fp16
? FPST_FPCR_F16
: FPST_FPCR
);
4505 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
4506 fn(t_zd
, t_zn
, t_pg
, scalar
, status
, desc
);
4508 tcg_temp_free_i32(desc
);
4509 tcg_temp_free_ptr(status
);
4510 tcg_temp_free_ptr(t_pg
);
4511 tcg_temp_free_ptr(t_zn
);
4512 tcg_temp_free_ptr(t_zd
);
4515 static void do_fp_imm(DisasContext
*s
, arg_rpri_esz
*a
, uint64_t imm
,
4516 gen_helper_sve_fp2scalar
*fn
)
4518 do_fp_scalar(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
,
4519 tcg_constant_i64(imm
), fn
);
4522 #define DO_FP_IMM(NAME, name, const0, const1) \
4523 static bool trans_##NAME##_zpzi(DisasContext *s, arg_rpri_esz *a) \
4525 static gen_helper_sve_fp2scalar * const fns[3] = { \
4526 gen_helper_sve_##name##_h, \
4527 gen_helper_sve_##name##_s, \
4528 gen_helper_sve_##name##_d \
4530 static uint64_t const val[3][2] = { \
4531 { float16_##const0, float16_##const1 }, \
4532 { float32_##const0, float32_##const1 }, \
4533 { float64_##const0, float64_##const1 }, \
4535 if (a->esz == 0) { \
4538 if (sve_access_check(s)) { \
4539 do_fp_imm(s, a, val[a->esz - 1][a->imm], fns[a->esz - 1]); \
4544 DO_FP_IMM(FADD
, fadds
, half
, one
)
4545 DO_FP_IMM(FSUB
, fsubs
, half
, one
)
4546 DO_FP_IMM(FMUL
, fmuls
, half
, two
)
4547 DO_FP_IMM(FSUBR
, fsubrs
, half
, one
)
4548 DO_FP_IMM(FMAXNM
, fmaxnms
, zero
, one
)
4549 DO_FP_IMM(FMINNM
, fminnms
, zero
, one
)
4550 DO_FP_IMM(FMAX
, fmaxs
, zero
, one
)
4551 DO_FP_IMM(FMIN
, fmins
, zero
, one
)
4555 static bool do_fp_cmp(DisasContext
*s
, arg_rprr_esz
*a
,
4556 gen_helper_gvec_4_ptr
*fn
)
4561 if (sve_access_check(s
)) {
4562 unsigned vsz
= vec_full_reg_size(s
);
4563 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4564 tcg_gen_gvec_4_ptr(pred_full_reg_offset(s
, a
->rd
),
4565 vec_full_reg_offset(s
, a
->rn
),
4566 vec_full_reg_offset(s
, a
->rm
),
4567 pred_full_reg_offset(s
, a
->pg
),
4568 status
, vsz
, vsz
, 0, fn
);
4569 tcg_temp_free_ptr(status
);
4574 #define DO_FPCMP(NAME, name) \
4575 static bool trans_##NAME##_ppzz(DisasContext *s, arg_rprr_esz *a) \
4577 static gen_helper_gvec_4_ptr * const fns[4] = { \
4578 NULL, gen_helper_sve_##name##_h, \
4579 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4581 return do_fp_cmp(s, a, fns[a->esz]); \
4584 DO_FPCMP(FCMGE
, fcmge
)
4585 DO_FPCMP(FCMGT
, fcmgt
)
4586 DO_FPCMP(FCMEQ
, fcmeq
)
4587 DO_FPCMP(FCMNE
, fcmne
)
4588 DO_FPCMP(FCMUO
, fcmuo
)
4589 DO_FPCMP(FACGE
, facge
)
4590 DO_FPCMP(FACGT
, facgt
)
4594 static bool trans_FCADD(DisasContext
*s
, arg_FCADD
*a
)
4596 static gen_helper_gvec_4_ptr
* const fns
[3] = {
4597 gen_helper_sve_fcadd_h
,
4598 gen_helper_sve_fcadd_s
,
4599 gen_helper_sve_fcadd_d
4605 if (sve_access_check(s
)) {
4606 unsigned vsz
= vec_full_reg_size(s
);
4607 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4608 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4609 vec_full_reg_offset(s
, a
->rn
),
4610 vec_full_reg_offset(s
, a
->rm
),
4611 pred_full_reg_offset(s
, a
->pg
),
4612 status
, vsz
, vsz
, a
->rot
, fns
[a
->esz
- 1]);
4613 tcg_temp_free_ptr(status
);
4618 static bool do_fmla(DisasContext
*s
, arg_rprrr_esz
*a
,
4619 gen_helper_gvec_5_ptr
*fn
)
4624 if (sve_access_check(s
)) {
4625 unsigned vsz
= vec_full_reg_size(s
);
4626 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4627 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s
, a
->rd
),
4628 vec_full_reg_offset(s
, a
->rn
),
4629 vec_full_reg_offset(s
, a
->rm
),
4630 vec_full_reg_offset(s
, a
->ra
),
4631 pred_full_reg_offset(s
, a
->pg
),
4632 status
, vsz
, vsz
, 0, fn
);
4633 tcg_temp_free_ptr(status
);
4638 #define DO_FMLA(NAME, name) \
4639 static bool trans_##NAME(DisasContext *s, arg_rprrr_esz *a) \
4641 static gen_helper_gvec_5_ptr * const fns[4] = { \
4642 NULL, gen_helper_sve_##name##_h, \
4643 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4645 return do_fmla(s, a, fns[a->esz]); \
4648 DO_FMLA(FMLA_zpzzz
, fmla_zpzzz
)
4649 DO_FMLA(FMLS_zpzzz
, fmls_zpzzz
)
4650 DO_FMLA(FNMLA_zpzzz
, fnmla_zpzzz
)
4651 DO_FMLA(FNMLS_zpzzz
, fnmls_zpzzz
)
4655 static bool trans_FCMLA_zpzzz(DisasContext
*s
, arg_FCMLA_zpzzz
*a
)
4657 static gen_helper_gvec_5_ptr
* const fns
[4] = {
4659 gen_helper_sve_fcmla_zpzzz_h
,
4660 gen_helper_sve_fcmla_zpzzz_s
,
4661 gen_helper_sve_fcmla_zpzzz_d
,
4667 if (sve_access_check(s
)) {
4668 unsigned vsz
= vec_full_reg_size(s
);
4669 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4670 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s
, a
->rd
),
4671 vec_full_reg_offset(s
, a
->rn
),
4672 vec_full_reg_offset(s
, a
->rm
),
4673 vec_full_reg_offset(s
, a
->ra
),
4674 pred_full_reg_offset(s
, a
->pg
),
4675 status
, vsz
, vsz
, a
->rot
, fns
[a
->esz
]);
4676 tcg_temp_free_ptr(status
);
4681 static bool trans_FCMLA_zzxz(DisasContext
*s
, arg_FCMLA_zzxz
*a
)
4683 static gen_helper_gvec_4_ptr
* const fns
[2] = {
4684 gen_helper_gvec_fcmlah_idx
,
4685 gen_helper_gvec_fcmlas_idx
,
4688 tcg_debug_assert(a
->esz
== 1 || a
->esz
== 2);
4689 tcg_debug_assert(a
->rd
== a
->ra
);
4690 if (sve_access_check(s
)) {
4691 unsigned vsz
= vec_full_reg_size(s
);
4692 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4693 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4694 vec_full_reg_offset(s
, a
->rn
),
4695 vec_full_reg_offset(s
, a
->rm
),
4696 vec_full_reg_offset(s
, a
->ra
),
4698 a
->index
* 4 + a
->rot
,
4700 tcg_temp_free_ptr(status
);
4706 *** SVE Floating Point Unary Operations Predicated Group
4709 static bool do_zpz_ptr(DisasContext
*s
, int rd
, int rn
, int pg
,
4710 bool is_fp16
, gen_helper_gvec_3_ptr
*fn
)
4712 if (sve_access_check(s
)) {
4713 unsigned vsz
= vec_full_reg_size(s
);
4714 TCGv_ptr status
= fpstatus_ptr(is_fp16
? FPST_FPCR_F16
: FPST_FPCR
);
4715 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, rd
),
4716 vec_full_reg_offset(s
, rn
),
4717 pred_full_reg_offset(s
, pg
),
4718 status
, vsz
, vsz
, 0, fn
);
4719 tcg_temp_free_ptr(status
);
4724 static bool trans_FCVT_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4726 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_sh
);
4729 static bool trans_FCVT_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4731 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_hs
);
4734 static bool trans_BFCVT(DisasContext
*s
, arg_rpr_esz
*a
)
4736 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
4739 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_bfcvt
);
4742 static bool trans_FCVT_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4744 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_dh
);
4747 static bool trans_FCVT_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4749 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_hd
);
4752 static bool trans_FCVT_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4754 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_ds
);
4757 static bool trans_FCVT_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4759 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_sd
);
4762 static bool trans_FCVTZS_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4764 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hh
);
4767 static bool trans_FCVTZU_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4769 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hh
);
4772 static bool trans_FCVTZS_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4774 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hs
);
4777 static bool trans_FCVTZU_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4779 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hs
);
4782 static bool trans_FCVTZS_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4784 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hd
);
4787 static bool trans_FCVTZU_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4789 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hd
);
4792 static bool trans_FCVTZS_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4794 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_ss
);
4797 static bool trans_FCVTZU_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4799 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_ss
);
4802 static bool trans_FCVTZS_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4804 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_sd
);
4807 static bool trans_FCVTZU_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4809 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_sd
);
4812 static bool trans_FCVTZS_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4814 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_ds
);
4817 static bool trans_FCVTZU_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4819 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_ds
);
4822 static bool trans_FCVTZS_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4824 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_dd
);
4827 static bool trans_FCVTZU_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4829 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_dd
);
4832 static gen_helper_gvec_3_ptr
* const frint_fns
[3] = {
4833 gen_helper_sve_frint_h
,
4834 gen_helper_sve_frint_s
,
4835 gen_helper_sve_frint_d
4838 static bool trans_FRINTI(DisasContext
*s
, arg_rpr_esz
*a
)
4843 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
,
4844 frint_fns
[a
->esz
- 1]);
4847 static bool trans_FRINTX(DisasContext
*s
, arg_rpr_esz
*a
)
4849 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4850 gen_helper_sve_frintx_h
,
4851 gen_helper_sve_frintx_s
,
4852 gen_helper_sve_frintx_d
4857 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4860 static bool do_frint_mode(DisasContext
*s
, arg_rpr_esz
*a
,
4861 int mode
, gen_helper_gvec_3_ptr
*fn
)
4863 if (sve_access_check(s
)) {
4864 unsigned vsz
= vec_full_reg_size(s
);
4865 TCGv_i32 tmode
= tcg_const_i32(mode
);
4866 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4868 gen_helper_set_rmode(tmode
, tmode
, status
);
4870 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4871 vec_full_reg_offset(s
, a
->rn
),
4872 pred_full_reg_offset(s
, a
->pg
),
4873 status
, vsz
, vsz
, 0, fn
);
4875 gen_helper_set_rmode(tmode
, tmode
, status
);
4876 tcg_temp_free_i32(tmode
);
4877 tcg_temp_free_ptr(status
);
4882 static bool trans_FRINTN(DisasContext
*s
, arg_rpr_esz
*a
)
4887 return do_frint_mode(s
, a
, float_round_nearest_even
, frint_fns
[a
->esz
- 1]);
4890 static bool trans_FRINTP(DisasContext
*s
, arg_rpr_esz
*a
)
4895 return do_frint_mode(s
, a
, float_round_up
, frint_fns
[a
->esz
- 1]);
4898 static bool trans_FRINTM(DisasContext
*s
, arg_rpr_esz
*a
)
4903 return do_frint_mode(s
, a
, float_round_down
, frint_fns
[a
->esz
- 1]);
4906 static bool trans_FRINTZ(DisasContext
*s
, arg_rpr_esz
*a
)
4911 return do_frint_mode(s
, a
, float_round_to_zero
, frint_fns
[a
->esz
- 1]);
4914 static bool trans_FRINTA(DisasContext
*s
, arg_rpr_esz
*a
)
4919 return do_frint_mode(s
, a
, float_round_ties_away
, frint_fns
[a
->esz
- 1]);
4922 static bool trans_FRECPX(DisasContext
*s
, arg_rpr_esz
*a
)
4924 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4925 gen_helper_sve_frecpx_h
,
4926 gen_helper_sve_frecpx_s
,
4927 gen_helper_sve_frecpx_d
4932 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4935 static bool trans_FSQRT(DisasContext
*s
, arg_rpr_esz
*a
)
4937 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4938 gen_helper_sve_fsqrt_h
,
4939 gen_helper_sve_fsqrt_s
,
4940 gen_helper_sve_fsqrt_d
4945 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4948 static bool trans_SCVTF_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4950 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_hh
);
4953 static bool trans_SCVTF_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4955 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_sh
);
4958 static bool trans_SCVTF_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4960 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_dh
);
4963 static bool trans_SCVTF_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4965 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_ss
);
4968 static bool trans_SCVTF_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4970 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_ds
);
4973 static bool trans_SCVTF_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4975 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_sd
);
4978 static bool trans_SCVTF_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4980 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_dd
);
4983 static bool trans_UCVTF_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4985 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_hh
);
4988 static bool trans_UCVTF_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4990 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_sh
);
4993 static bool trans_UCVTF_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4995 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_dh
);
4998 static bool trans_UCVTF_ss(DisasContext
*s
, arg_rpr_esz
*a
)
5000 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_ss
);
5003 static bool trans_UCVTF_ds(DisasContext
*s
, arg_rpr_esz
*a
)
5005 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_ds
);
5008 static bool trans_UCVTF_sd(DisasContext
*s
, arg_rpr_esz
*a
)
5010 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_sd
);
5013 static bool trans_UCVTF_dd(DisasContext
*s
, arg_rpr_esz
*a
)
5015 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_dd
);
5019 *** SVE Memory - 32-bit Gather and Unsized Contiguous Group
5022 /* Subroutine loading a vector register at VOFS of LEN bytes.
5023 * The load should begin at the address Rn + IMM.
5026 static void do_ldr(DisasContext
*s
, uint32_t vofs
, int len
, int rn
, int imm
)
5028 int len_align
= QEMU_ALIGN_DOWN(len
, 8);
5029 int len_remain
= len
% 8;
5030 int nparts
= len
/ 8 + ctpop8(len_remain
);
5031 int midx
= get_mem_index(s
);
5032 TCGv_i64 dirty_addr
, clean_addr
, t0
, t1
;
5034 dirty_addr
= tcg_temp_new_i64();
5035 tcg_gen_addi_i64(dirty_addr
, cpu_reg_sp(s
, rn
), imm
);
5036 clean_addr
= gen_mte_checkN(s
, dirty_addr
, false, rn
!= 31, len
);
5037 tcg_temp_free_i64(dirty_addr
);
5040 * Note that unpredicated load/store of vector/predicate registers
5041 * are defined as a stream of bytes, which equates to little-endian
5042 * operations on larger quantities.
5043 * Attempt to keep code expansion to a minimum by limiting the
5044 * amount of unrolling done.
5049 t0
= tcg_temp_new_i64();
5050 for (i
= 0; i
< len_align
; i
+= 8) {
5051 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEUQ
);
5052 tcg_gen_st_i64(t0
, cpu_env
, vofs
+ i
);
5053 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
5055 tcg_temp_free_i64(t0
);
5057 TCGLabel
*loop
= gen_new_label();
5058 TCGv_ptr tp
, i
= tcg_const_local_ptr(0);
5060 /* Copy the clean address into a local temp, live across the loop. */
5062 clean_addr
= new_tmp_a64_local(s
);
5063 tcg_gen_mov_i64(clean_addr
, t0
);
5065 gen_set_label(loop
);
5067 t0
= tcg_temp_new_i64();
5068 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEUQ
);
5069 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
5071 tp
= tcg_temp_new_ptr();
5072 tcg_gen_add_ptr(tp
, cpu_env
, i
);
5073 tcg_gen_addi_ptr(i
, i
, 8);
5074 tcg_gen_st_i64(t0
, tp
, vofs
);
5075 tcg_temp_free_ptr(tp
);
5076 tcg_temp_free_i64(t0
);
5078 tcg_gen_brcondi_ptr(TCG_COND_LTU
, i
, len_align
, loop
);
5079 tcg_temp_free_ptr(i
);
5083 * Predicate register loads can be any multiple of 2.
5084 * Note that we still store the entire 64-bit unit into cpu_env.
5087 t0
= tcg_temp_new_i64();
5088 switch (len_remain
) {
5092 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
,
5093 MO_LE
| ctz32(len_remain
));
5097 t1
= tcg_temp_new_i64();
5098 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEUL
);
5099 tcg_gen_addi_i64(clean_addr
, clean_addr
, 4);
5100 tcg_gen_qemu_ld_i64(t1
, clean_addr
, midx
, MO_LEUW
);
5101 tcg_gen_deposit_i64(t0
, t0
, t1
, 32, 32);
5102 tcg_temp_free_i64(t1
);
5106 g_assert_not_reached();
5108 tcg_gen_st_i64(t0
, cpu_env
, vofs
+ len_align
);
5109 tcg_temp_free_i64(t0
);
5113 /* Similarly for stores. */
5114 static void do_str(DisasContext
*s
, uint32_t vofs
, int len
, int rn
, int imm
)
5116 int len_align
= QEMU_ALIGN_DOWN(len
, 8);
5117 int len_remain
= len
% 8;
5118 int nparts
= len
/ 8 + ctpop8(len_remain
);
5119 int midx
= get_mem_index(s
);
5120 TCGv_i64 dirty_addr
, clean_addr
, t0
;
5122 dirty_addr
= tcg_temp_new_i64();
5123 tcg_gen_addi_i64(dirty_addr
, cpu_reg_sp(s
, rn
), imm
);
5124 clean_addr
= gen_mte_checkN(s
, dirty_addr
, false, rn
!= 31, len
);
5125 tcg_temp_free_i64(dirty_addr
);
5127 /* Note that unpredicated load/store of vector/predicate registers
5128 * are defined as a stream of bytes, which equates to little-endian
5129 * operations on larger quantities. There is no nice way to force
5130 * a little-endian store for aarch64_be-linux-user out of line.
5132 * Attempt to keep code expansion to a minimum by limiting the
5133 * amount of unrolling done.
5138 t0
= tcg_temp_new_i64();
5139 for (i
= 0; i
< len_align
; i
+= 8) {
5140 tcg_gen_ld_i64(t0
, cpu_env
, vofs
+ i
);
5141 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUQ
);
5142 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
5144 tcg_temp_free_i64(t0
);
5146 TCGLabel
*loop
= gen_new_label();
5147 TCGv_ptr tp
, i
= tcg_const_local_ptr(0);
5149 /* Copy the clean address into a local temp, live across the loop. */
5151 clean_addr
= new_tmp_a64_local(s
);
5152 tcg_gen_mov_i64(clean_addr
, t0
);
5154 gen_set_label(loop
);
5156 t0
= tcg_temp_new_i64();
5157 tp
= tcg_temp_new_ptr();
5158 tcg_gen_add_ptr(tp
, cpu_env
, i
);
5159 tcg_gen_ld_i64(t0
, tp
, vofs
);
5160 tcg_gen_addi_ptr(i
, i
, 8);
5161 tcg_temp_free_ptr(tp
);
5163 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUQ
);
5164 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
5165 tcg_temp_free_i64(t0
);
5167 tcg_gen_brcondi_ptr(TCG_COND_LTU
, i
, len_align
, loop
);
5168 tcg_temp_free_ptr(i
);
5171 /* Predicate register stores can be any multiple of 2. */
5173 t0
= tcg_temp_new_i64();
5174 tcg_gen_ld_i64(t0
, cpu_env
, vofs
+ len_align
);
5176 switch (len_remain
) {
5180 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
,
5181 MO_LE
| ctz32(len_remain
));
5185 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUL
);
5186 tcg_gen_addi_i64(clean_addr
, clean_addr
, 4);
5187 tcg_gen_shri_i64(t0
, t0
, 32);
5188 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUW
);
5192 g_assert_not_reached();
5194 tcg_temp_free_i64(t0
);
5198 static bool trans_LDR_zri(DisasContext
*s
, arg_rri
*a
)
5200 if (sve_access_check(s
)) {
5201 int size
= vec_full_reg_size(s
);
5202 int off
= vec_full_reg_offset(s
, a
->rd
);
5203 do_ldr(s
, off
, size
, a
->rn
, a
->imm
* size
);
5208 static bool trans_LDR_pri(DisasContext
*s
, arg_rri
*a
)
5210 if (sve_access_check(s
)) {
5211 int size
= pred_full_reg_size(s
);
5212 int off
= pred_full_reg_offset(s
, a
->rd
);
5213 do_ldr(s
, off
, size
, a
->rn
, a
->imm
* size
);
5218 static bool trans_STR_zri(DisasContext
*s
, arg_rri
*a
)
5220 if (sve_access_check(s
)) {
5221 int size
= vec_full_reg_size(s
);
5222 int off
= vec_full_reg_offset(s
, a
->rd
);
5223 do_str(s
, off
, size
, a
->rn
, a
->imm
* size
);
5228 static bool trans_STR_pri(DisasContext
*s
, arg_rri
*a
)
5230 if (sve_access_check(s
)) {
5231 int size
= pred_full_reg_size(s
);
5232 int off
= pred_full_reg_offset(s
, a
->rd
);
5233 do_str(s
, off
, size
, a
->rn
, a
->imm
* size
);
5239 *** SVE Memory - Contiguous Load Group
5242 /* The memory mode of the dtype. */
5243 static const MemOp dtype_mop
[16] = {
5244 MO_UB
, MO_UB
, MO_UB
, MO_UB
,
5245 MO_SL
, MO_UW
, MO_UW
, MO_UW
,
5246 MO_SW
, MO_SW
, MO_UL
, MO_UL
,
5247 MO_SB
, MO_SB
, MO_SB
, MO_UQ
5250 #define dtype_msz(x) (dtype_mop[x] & MO_SIZE)
5252 /* The vector element size of dtype. */
5253 static const uint8_t dtype_esz
[16] = {
5260 static void do_mem_zpa(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
,
5261 int dtype
, uint32_t mte_n
, bool is_write
,
5262 gen_helper_gvec_mem
*fn
)
5264 unsigned vsz
= vec_full_reg_size(s
);
5270 * For e.g. LD4, there are not enough arguments to pass all 4
5271 * registers as pointers, so encode the regno into the data field.
5272 * For consistency, do this even for LD1.
5274 if (s
->mte_active
[0]) {
5275 int msz
= dtype_msz(dtype
);
5277 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, get_mem_index(s
));
5278 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
5279 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
5280 desc
= FIELD_DP32(desc
, MTEDESC
, WRITE
, is_write
);
5281 desc
= FIELD_DP32(desc
, MTEDESC
, SIZEM1
, (mte_n
<< msz
) - 1);
5282 desc
<<= SVE_MTEDESC_SHIFT
;
5284 addr
= clean_data_tbi(s
, addr
);
5287 desc
= simd_desc(vsz
, vsz
, zt
| desc
);
5288 t_desc
= tcg_const_i32(desc
);
5289 t_pg
= tcg_temp_new_ptr();
5291 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
5292 fn(cpu_env
, t_pg
, addr
, t_desc
);
5294 tcg_temp_free_ptr(t_pg
);
5295 tcg_temp_free_i32(t_desc
);
5298 /* Indexed by [mte][be][dtype][nreg] */
5299 static gen_helper_gvec_mem
* const ldr_fns
[2][2][16][4] = {
5300 { /* mte inactive, little-endian */
5301 { { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld2bb_r
,
5302 gen_helper_sve_ld3bb_r
, gen_helper_sve_ld4bb_r
},
5303 { gen_helper_sve_ld1bhu_r
, NULL
, NULL
, NULL
},
5304 { gen_helper_sve_ld1bsu_r
, NULL
, NULL
, NULL
},
5305 { gen_helper_sve_ld1bdu_r
, NULL
, NULL
, NULL
},
5307 { gen_helper_sve_ld1sds_le_r
, NULL
, NULL
, NULL
},
5308 { gen_helper_sve_ld1hh_le_r
, gen_helper_sve_ld2hh_le_r
,
5309 gen_helper_sve_ld3hh_le_r
, gen_helper_sve_ld4hh_le_r
},
5310 { gen_helper_sve_ld1hsu_le_r
, NULL
, NULL
, NULL
},
5311 { gen_helper_sve_ld1hdu_le_r
, NULL
, NULL
, NULL
},
5313 { gen_helper_sve_ld1hds_le_r
, NULL
, NULL
, NULL
},
5314 { gen_helper_sve_ld1hss_le_r
, NULL
, NULL
, NULL
},
5315 { gen_helper_sve_ld1ss_le_r
, gen_helper_sve_ld2ss_le_r
,
5316 gen_helper_sve_ld3ss_le_r
, gen_helper_sve_ld4ss_le_r
},
5317 { gen_helper_sve_ld1sdu_le_r
, NULL
, NULL
, NULL
},
5319 { gen_helper_sve_ld1bds_r
, NULL
, NULL
, NULL
},
5320 { gen_helper_sve_ld1bss_r
, NULL
, NULL
, NULL
},
5321 { gen_helper_sve_ld1bhs_r
, NULL
, NULL
, NULL
},
5322 { gen_helper_sve_ld1dd_le_r
, gen_helper_sve_ld2dd_le_r
,
5323 gen_helper_sve_ld3dd_le_r
, gen_helper_sve_ld4dd_le_r
} },
5325 /* mte inactive, big-endian */
5326 { { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld2bb_r
,
5327 gen_helper_sve_ld3bb_r
, gen_helper_sve_ld4bb_r
},
5328 { gen_helper_sve_ld1bhu_r
, NULL
, NULL
, NULL
},
5329 { gen_helper_sve_ld1bsu_r
, NULL
, NULL
, NULL
},
5330 { gen_helper_sve_ld1bdu_r
, NULL
, NULL
, NULL
},
5332 { gen_helper_sve_ld1sds_be_r
, NULL
, NULL
, NULL
},
5333 { gen_helper_sve_ld1hh_be_r
, gen_helper_sve_ld2hh_be_r
,
5334 gen_helper_sve_ld3hh_be_r
, gen_helper_sve_ld4hh_be_r
},
5335 { gen_helper_sve_ld1hsu_be_r
, NULL
, NULL
, NULL
},
5336 { gen_helper_sve_ld1hdu_be_r
, NULL
, NULL
, NULL
},
5338 { gen_helper_sve_ld1hds_be_r
, NULL
, NULL
, NULL
},
5339 { gen_helper_sve_ld1hss_be_r
, NULL
, NULL
, NULL
},
5340 { gen_helper_sve_ld1ss_be_r
, gen_helper_sve_ld2ss_be_r
,
5341 gen_helper_sve_ld3ss_be_r
, gen_helper_sve_ld4ss_be_r
},
5342 { gen_helper_sve_ld1sdu_be_r
, NULL
, NULL
, NULL
},
5344 { gen_helper_sve_ld1bds_r
, NULL
, NULL
, NULL
},
5345 { gen_helper_sve_ld1bss_r
, NULL
, NULL
, NULL
},
5346 { gen_helper_sve_ld1bhs_r
, NULL
, NULL
, NULL
},
5347 { gen_helper_sve_ld1dd_be_r
, gen_helper_sve_ld2dd_be_r
,
5348 gen_helper_sve_ld3dd_be_r
, gen_helper_sve_ld4dd_be_r
} } },
5350 { /* mte active, little-endian */
5351 { { gen_helper_sve_ld1bb_r_mte
,
5352 gen_helper_sve_ld2bb_r_mte
,
5353 gen_helper_sve_ld3bb_r_mte
,
5354 gen_helper_sve_ld4bb_r_mte
},
5355 { gen_helper_sve_ld1bhu_r_mte
, NULL
, NULL
, NULL
},
5356 { gen_helper_sve_ld1bsu_r_mte
, NULL
, NULL
, NULL
},
5357 { gen_helper_sve_ld1bdu_r_mte
, NULL
, NULL
, NULL
},
5359 { gen_helper_sve_ld1sds_le_r_mte
, NULL
, NULL
, NULL
},
5360 { gen_helper_sve_ld1hh_le_r_mte
,
5361 gen_helper_sve_ld2hh_le_r_mte
,
5362 gen_helper_sve_ld3hh_le_r_mte
,
5363 gen_helper_sve_ld4hh_le_r_mte
},
5364 { gen_helper_sve_ld1hsu_le_r_mte
, NULL
, NULL
, NULL
},
5365 { gen_helper_sve_ld1hdu_le_r_mte
, NULL
, NULL
, NULL
},
5367 { gen_helper_sve_ld1hds_le_r_mte
, NULL
, NULL
, NULL
},
5368 { gen_helper_sve_ld1hss_le_r_mte
, NULL
, NULL
, NULL
},
5369 { gen_helper_sve_ld1ss_le_r_mte
,
5370 gen_helper_sve_ld2ss_le_r_mte
,
5371 gen_helper_sve_ld3ss_le_r_mte
,
5372 gen_helper_sve_ld4ss_le_r_mte
},
5373 { gen_helper_sve_ld1sdu_le_r_mte
, NULL
, NULL
, NULL
},
5375 { gen_helper_sve_ld1bds_r_mte
, NULL
, NULL
, NULL
},
5376 { gen_helper_sve_ld1bss_r_mte
, NULL
, NULL
, NULL
},
5377 { gen_helper_sve_ld1bhs_r_mte
, NULL
, NULL
, NULL
},
5378 { gen_helper_sve_ld1dd_le_r_mte
,
5379 gen_helper_sve_ld2dd_le_r_mte
,
5380 gen_helper_sve_ld3dd_le_r_mte
,
5381 gen_helper_sve_ld4dd_le_r_mte
} },
5383 /* mte active, big-endian */
5384 { { gen_helper_sve_ld1bb_r_mte
,
5385 gen_helper_sve_ld2bb_r_mte
,
5386 gen_helper_sve_ld3bb_r_mte
,
5387 gen_helper_sve_ld4bb_r_mte
},
5388 { gen_helper_sve_ld1bhu_r_mte
, NULL
, NULL
, NULL
},
5389 { gen_helper_sve_ld1bsu_r_mte
, NULL
, NULL
, NULL
},
5390 { gen_helper_sve_ld1bdu_r_mte
, NULL
, NULL
, NULL
},
5392 { gen_helper_sve_ld1sds_be_r_mte
, NULL
, NULL
, NULL
},
5393 { gen_helper_sve_ld1hh_be_r_mte
,
5394 gen_helper_sve_ld2hh_be_r_mte
,
5395 gen_helper_sve_ld3hh_be_r_mte
,
5396 gen_helper_sve_ld4hh_be_r_mte
},
5397 { gen_helper_sve_ld1hsu_be_r_mte
, NULL
, NULL
, NULL
},
5398 { gen_helper_sve_ld1hdu_be_r_mte
, NULL
, NULL
, NULL
},
5400 { gen_helper_sve_ld1hds_be_r_mte
, NULL
, NULL
, NULL
},
5401 { gen_helper_sve_ld1hss_be_r_mte
, NULL
, NULL
, NULL
},
5402 { gen_helper_sve_ld1ss_be_r_mte
,
5403 gen_helper_sve_ld2ss_be_r_mte
,
5404 gen_helper_sve_ld3ss_be_r_mte
,
5405 gen_helper_sve_ld4ss_be_r_mte
},
5406 { gen_helper_sve_ld1sdu_be_r_mte
, NULL
, NULL
, NULL
},
5408 { gen_helper_sve_ld1bds_r_mte
, NULL
, NULL
, NULL
},
5409 { gen_helper_sve_ld1bss_r_mte
, NULL
, NULL
, NULL
},
5410 { gen_helper_sve_ld1bhs_r_mte
, NULL
, NULL
, NULL
},
5411 { gen_helper_sve_ld1dd_be_r_mte
,
5412 gen_helper_sve_ld2dd_be_r_mte
,
5413 gen_helper_sve_ld3dd_be_r_mte
,
5414 gen_helper_sve_ld4dd_be_r_mte
} } },
5417 static void do_ld_zpa(DisasContext
*s
, int zt
, int pg
,
5418 TCGv_i64 addr
, int dtype
, int nreg
)
5420 gen_helper_gvec_mem
*fn
5421 = ldr_fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][dtype
][nreg
];
5424 * While there are holes in the table, they are not
5425 * accessible via the instruction encoding.
5428 do_mem_zpa(s
, zt
, pg
, addr
, dtype
, nreg
, false, fn
);
5431 static bool trans_LD_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5436 if (sve_access_check(s
)) {
5437 TCGv_i64 addr
= new_tmp_a64(s
);
5438 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
5439 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5440 do_ld_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, a
->nreg
);
5445 static bool trans_LD_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5447 if (sve_access_check(s
)) {
5448 int vsz
= vec_full_reg_size(s
);
5449 int elements
= vsz
>> dtype_esz
[a
->dtype
];
5450 TCGv_i64 addr
= new_tmp_a64(s
);
5452 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
),
5453 (a
->imm
* elements
* (a
->nreg
+ 1))
5454 << dtype_msz(a
->dtype
));
5455 do_ld_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, a
->nreg
);
5460 static bool trans_LDFF1_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5462 static gen_helper_gvec_mem
* const fns
[2][2][16] = {
5463 { /* mte inactive, little-endian */
5464 { gen_helper_sve_ldff1bb_r
,
5465 gen_helper_sve_ldff1bhu_r
,
5466 gen_helper_sve_ldff1bsu_r
,
5467 gen_helper_sve_ldff1bdu_r
,
5469 gen_helper_sve_ldff1sds_le_r
,
5470 gen_helper_sve_ldff1hh_le_r
,
5471 gen_helper_sve_ldff1hsu_le_r
,
5472 gen_helper_sve_ldff1hdu_le_r
,
5474 gen_helper_sve_ldff1hds_le_r
,
5475 gen_helper_sve_ldff1hss_le_r
,
5476 gen_helper_sve_ldff1ss_le_r
,
5477 gen_helper_sve_ldff1sdu_le_r
,
5479 gen_helper_sve_ldff1bds_r
,
5480 gen_helper_sve_ldff1bss_r
,
5481 gen_helper_sve_ldff1bhs_r
,
5482 gen_helper_sve_ldff1dd_le_r
},
5484 /* mte inactive, big-endian */
5485 { gen_helper_sve_ldff1bb_r
,
5486 gen_helper_sve_ldff1bhu_r
,
5487 gen_helper_sve_ldff1bsu_r
,
5488 gen_helper_sve_ldff1bdu_r
,
5490 gen_helper_sve_ldff1sds_be_r
,
5491 gen_helper_sve_ldff1hh_be_r
,
5492 gen_helper_sve_ldff1hsu_be_r
,
5493 gen_helper_sve_ldff1hdu_be_r
,
5495 gen_helper_sve_ldff1hds_be_r
,
5496 gen_helper_sve_ldff1hss_be_r
,
5497 gen_helper_sve_ldff1ss_be_r
,
5498 gen_helper_sve_ldff1sdu_be_r
,
5500 gen_helper_sve_ldff1bds_r
,
5501 gen_helper_sve_ldff1bss_r
,
5502 gen_helper_sve_ldff1bhs_r
,
5503 gen_helper_sve_ldff1dd_be_r
} },
5505 { /* mte active, little-endian */
5506 { gen_helper_sve_ldff1bb_r_mte
,
5507 gen_helper_sve_ldff1bhu_r_mte
,
5508 gen_helper_sve_ldff1bsu_r_mte
,
5509 gen_helper_sve_ldff1bdu_r_mte
,
5511 gen_helper_sve_ldff1sds_le_r_mte
,
5512 gen_helper_sve_ldff1hh_le_r_mte
,
5513 gen_helper_sve_ldff1hsu_le_r_mte
,
5514 gen_helper_sve_ldff1hdu_le_r_mte
,
5516 gen_helper_sve_ldff1hds_le_r_mte
,
5517 gen_helper_sve_ldff1hss_le_r_mte
,
5518 gen_helper_sve_ldff1ss_le_r_mte
,
5519 gen_helper_sve_ldff1sdu_le_r_mte
,
5521 gen_helper_sve_ldff1bds_r_mte
,
5522 gen_helper_sve_ldff1bss_r_mte
,
5523 gen_helper_sve_ldff1bhs_r_mte
,
5524 gen_helper_sve_ldff1dd_le_r_mte
},
5526 /* mte active, big-endian */
5527 { gen_helper_sve_ldff1bb_r_mte
,
5528 gen_helper_sve_ldff1bhu_r_mte
,
5529 gen_helper_sve_ldff1bsu_r_mte
,
5530 gen_helper_sve_ldff1bdu_r_mte
,
5532 gen_helper_sve_ldff1sds_be_r_mte
,
5533 gen_helper_sve_ldff1hh_be_r_mte
,
5534 gen_helper_sve_ldff1hsu_be_r_mte
,
5535 gen_helper_sve_ldff1hdu_be_r_mte
,
5537 gen_helper_sve_ldff1hds_be_r_mte
,
5538 gen_helper_sve_ldff1hss_be_r_mte
,
5539 gen_helper_sve_ldff1ss_be_r_mte
,
5540 gen_helper_sve_ldff1sdu_be_r_mte
,
5542 gen_helper_sve_ldff1bds_r_mte
,
5543 gen_helper_sve_ldff1bss_r_mte
,
5544 gen_helper_sve_ldff1bhs_r_mte
,
5545 gen_helper_sve_ldff1dd_be_r_mte
} },
5548 if (sve_access_check(s
)) {
5549 TCGv_i64 addr
= new_tmp_a64(s
);
5550 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
5551 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5552 do_mem_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, 1, false,
5553 fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][a
->dtype
]);
5558 static bool trans_LDNF1_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5560 static gen_helper_gvec_mem
* const fns
[2][2][16] = {
5561 { /* mte inactive, little-endian */
5562 { gen_helper_sve_ldnf1bb_r
,
5563 gen_helper_sve_ldnf1bhu_r
,
5564 gen_helper_sve_ldnf1bsu_r
,
5565 gen_helper_sve_ldnf1bdu_r
,
5567 gen_helper_sve_ldnf1sds_le_r
,
5568 gen_helper_sve_ldnf1hh_le_r
,
5569 gen_helper_sve_ldnf1hsu_le_r
,
5570 gen_helper_sve_ldnf1hdu_le_r
,
5572 gen_helper_sve_ldnf1hds_le_r
,
5573 gen_helper_sve_ldnf1hss_le_r
,
5574 gen_helper_sve_ldnf1ss_le_r
,
5575 gen_helper_sve_ldnf1sdu_le_r
,
5577 gen_helper_sve_ldnf1bds_r
,
5578 gen_helper_sve_ldnf1bss_r
,
5579 gen_helper_sve_ldnf1bhs_r
,
5580 gen_helper_sve_ldnf1dd_le_r
},
5582 /* mte inactive, big-endian */
5583 { gen_helper_sve_ldnf1bb_r
,
5584 gen_helper_sve_ldnf1bhu_r
,
5585 gen_helper_sve_ldnf1bsu_r
,
5586 gen_helper_sve_ldnf1bdu_r
,
5588 gen_helper_sve_ldnf1sds_be_r
,
5589 gen_helper_sve_ldnf1hh_be_r
,
5590 gen_helper_sve_ldnf1hsu_be_r
,
5591 gen_helper_sve_ldnf1hdu_be_r
,
5593 gen_helper_sve_ldnf1hds_be_r
,
5594 gen_helper_sve_ldnf1hss_be_r
,
5595 gen_helper_sve_ldnf1ss_be_r
,
5596 gen_helper_sve_ldnf1sdu_be_r
,
5598 gen_helper_sve_ldnf1bds_r
,
5599 gen_helper_sve_ldnf1bss_r
,
5600 gen_helper_sve_ldnf1bhs_r
,
5601 gen_helper_sve_ldnf1dd_be_r
} },
5603 { /* mte inactive, little-endian */
5604 { gen_helper_sve_ldnf1bb_r_mte
,
5605 gen_helper_sve_ldnf1bhu_r_mte
,
5606 gen_helper_sve_ldnf1bsu_r_mte
,
5607 gen_helper_sve_ldnf1bdu_r_mte
,
5609 gen_helper_sve_ldnf1sds_le_r_mte
,
5610 gen_helper_sve_ldnf1hh_le_r_mte
,
5611 gen_helper_sve_ldnf1hsu_le_r_mte
,
5612 gen_helper_sve_ldnf1hdu_le_r_mte
,
5614 gen_helper_sve_ldnf1hds_le_r_mte
,
5615 gen_helper_sve_ldnf1hss_le_r_mte
,
5616 gen_helper_sve_ldnf1ss_le_r_mte
,
5617 gen_helper_sve_ldnf1sdu_le_r_mte
,
5619 gen_helper_sve_ldnf1bds_r_mte
,
5620 gen_helper_sve_ldnf1bss_r_mte
,
5621 gen_helper_sve_ldnf1bhs_r_mte
,
5622 gen_helper_sve_ldnf1dd_le_r_mte
},
5624 /* mte inactive, big-endian */
5625 { gen_helper_sve_ldnf1bb_r_mte
,
5626 gen_helper_sve_ldnf1bhu_r_mte
,
5627 gen_helper_sve_ldnf1bsu_r_mte
,
5628 gen_helper_sve_ldnf1bdu_r_mte
,
5630 gen_helper_sve_ldnf1sds_be_r_mte
,
5631 gen_helper_sve_ldnf1hh_be_r_mte
,
5632 gen_helper_sve_ldnf1hsu_be_r_mte
,
5633 gen_helper_sve_ldnf1hdu_be_r_mte
,
5635 gen_helper_sve_ldnf1hds_be_r_mte
,
5636 gen_helper_sve_ldnf1hss_be_r_mte
,
5637 gen_helper_sve_ldnf1ss_be_r_mte
,
5638 gen_helper_sve_ldnf1sdu_be_r_mte
,
5640 gen_helper_sve_ldnf1bds_r_mte
,
5641 gen_helper_sve_ldnf1bss_r_mte
,
5642 gen_helper_sve_ldnf1bhs_r_mte
,
5643 gen_helper_sve_ldnf1dd_be_r_mte
} },
5646 if (sve_access_check(s
)) {
5647 int vsz
= vec_full_reg_size(s
);
5648 int elements
= vsz
>> dtype_esz
[a
->dtype
];
5649 int off
= (a
->imm
* elements
) << dtype_msz(a
->dtype
);
5650 TCGv_i64 addr
= new_tmp_a64(s
);
5652 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), off
);
5653 do_mem_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, 1, false,
5654 fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][a
->dtype
]);
5659 static void do_ldrq(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
, int dtype
)
5661 unsigned vsz
= vec_full_reg_size(s
);
5665 /* Load the first quadword using the normal predicated load helpers. */
5666 poff
= pred_full_reg_offset(s
, pg
);
5669 * Zero-extend the first 16 bits of the predicate into a temporary.
5670 * This avoids triggering an assert making sure we don't have bits
5671 * set within a predicate beyond VQ, but we have lowered VQ to 1
5672 * for this load operation.
5674 TCGv_i64 tmp
= tcg_temp_new_i64();
5678 tcg_gen_ld16u_i64(tmp
, cpu_env
, poff
);
5680 poff
= offsetof(CPUARMState
, vfp
.preg_tmp
);
5681 tcg_gen_st_i64(tmp
, cpu_env
, poff
);
5682 tcg_temp_free_i64(tmp
);
5685 t_pg
= tcg_temp_new_ptr();
5686 tcg_gen_addi_ptr(t_pg
, cpu_env
, poff
);
5688 gen_helper_gvec_mem
*fn
5689 = ldr_fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][dtype
][0];
5690 fn(cpu_env
, t_pg
, addr
, tcg_constant_i32(simd_desc(16, 16, zt
)));
5692 tcg_temp_free_ptr(t_pg
);
5694 /* Replicate that first quadword. */
5696 int doff
= vec_full_reg_offset(s
, zt
);
5697 tcg_gen_gvec_dup_mem(4, doff
+ 16, doff
, vsz
- 16, vsz
- 16);
5701 static bool trans_LD1RQ_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5706 if (sve_access_check(s
)) {
5707 int msz
= dtype_msz(a
->dtype
);
5708 TCGv_i64 addr
= new_tmp_a64(s
);
5709 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), msz
);
5710 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5711 do_ldrq(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5716 static bool trans_LD1RQ_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5718 if (sve_access_check(s
)) {
5719 TCGv_i64 addr
= new_tmp_a64(s
);
5720 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), a
->imm
* 16);
5721 do_ldrq(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5726 static void do_ldro(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
, int dtype
)
5728 unsigned vsz
= vec_full_reg_size(s
);
5735 * Note that this UNDEFINED check comes after CheckSVEEnabled()
5736 * in the ARM pseudocode, which is the sve_access_check() done
5737 * in our caller. We should not now return false from the caller.
5739 unallocated_encoding(s
);
5743 /* Load the first octaword using the normal predicated load helpers. */
5745 poff
= pred_full_reg_offset(s
, pg
);
5748 * Zero-extend the first 32 bits of the predicate into a temporary.
5749 * This avoids triggering an assert making sure we don't have bits
5750 * set within a predicate beyond VQ, but we have lowered VQ to 2
5751 * for this load operation.
5753 TCGv_i64 tmp
= tcg_temp_new_i64();
5757 tcg_gen_ld32u_i64(tmp
, cpu_env
, poff
);
5759 poff
= offsetof(CPUARMState
, vfp
.preg_tmp
);
5760 tcg_gen_st_i64(tmp
, cpu_env
, poff
);
5761 tcg_temp_free_i64(tmp
);
5764 t_pg
= tcg_temp_new_ptr();
5765 tcg_gen_addi_ptr(t_pg
, cpu_env
, poff
);
5767 gen_helper_gvec_mem
*fn
5768 = ldr_fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][dtype
][0];
5769 fn(cpu_env
, t_pg
, addr
, tcg_constant_i32(simd_desc(32, 32, zt
)));
5771 tcg_temp_free_ptr(t_pg
);
5774 * Replicate that first octaword.
5775 * The replication happens in units of 32; if the full vector size
5776 * is not a multiple of 32, the final bits are zeroed.
5778 doff
= vec_full_reg_offset(s
, zt
);
5779 vsz_r32
= QEMU_ALIGN_DOWN(vsz
, 32);
5781 tcg_gen_gvec_dup_mem(5, doff
+ 32, doff
, vsz_r32
- 32, vsz_r32
- 32);
5785 tcg_gen_gvec_dup_imm(MO_64
, doff
+ vsz_r32
, vsz
, vsz
, 0);
5789 static bool trans_LD1RO_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5791 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
5797 if (sve_access_check(s
)) {
5798 TCGv_i64 addr
= new_tmp_a64(s
);
5799 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
5800 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5801 do_ldro(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5806 static bool trans_LD1RO_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5808 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
5811 if (sve_access_check(s
)) {
5812 TCGv_i64 addr
= new_tmp_a64(s
);
5813 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), a
->imm
* 32);
5814 do_ldro(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5819 /* Load and broadcast element. */
5820 static bool trans_LD1R_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5822 unsigned vsz
= vec_full_reg_size(s
);
5823 unsigned psz
= pred_full_reg_size(s
);
5824 unsigned esz
= dtype_esz
[a
->dtype
];
5825 unsigned msz
= dtype_msz(a
->dtype
);
5827 TCGv_i64 temp
, clean_addr
;
5829 if (!sve_access_check(s
)) {
5833 over
= gen_new_label();
5835 /* If the guarding predicate has no bits set, no load occurs. */
5837 /* Reduce the pred_esz_masks value simply to reduce the
5838 * size of the code generated here.
5840 uint64_t psz_mask
= MAKE_64BIT_MASK(0, psz
* 8);
5841 temp
= tcg_temp_new_i64();
5842 tcg_gen_ld_i64(temp
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
5843 tcg_gen_andi_i64(temp
, temp
, pred_esz_masks
[esz
] & psz_mask
);
5844 tcg_gen_brcondi_i64(TCG_COND_EQ
, temp
, 0, over
);
5845 tcg_temp_free_i64(temp
);
5847 TCGv_i32 t32
= tcg_temp_new_i32();
5848 find_last_active(s
, t32
, esz
, a
->pg
);
5849 tcg_gen_brcondi_i32(TCG_COND_LT
, t32
, 0, over
);
5850 tcg_temp_free_i32(t32
);
5853 /* Load the data. */
5854 temp
= tcg_temp_new_i64();
5855 tcg_gen_addi_i64(temp
, cpu_reg_sp(s
, a
->rn
), a
->imm
<< msz
);
5856 clean_addr
= gen_mte_check1(s
, temp
, false, true, msz
);
5858 tcg_gen_qemu_ld_i64(temp
, clean_addr
, get_mem_index(s
),
5859 finalize_memop(s
, dtype_mop
[a
->dtype
]));
5861 /* Broadcast to *all* elements. */
5862 tcg_gen_gvec_dup_i64(esz
, vec_full_reg_offset(s
, a
->rd
),
5864 tcg_temp_free_i64(temp
);
5866 /* Zero the inactive elements. */
5867 gen_set_label(over
);
5868 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, esz
, false);
5871 static void do_st_zpa(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
,
5872 int msz
, int esz
, int nreg
)
5874 static gen_helper_gvec_mem
* const fn_single
[2][2][4][4] = {
5875 { { { gen_helper_sve_st1bb_r
,
5876 gen_helper_sve_st1bh_r
,
5877 gen_helper_sve_st1bs_r
,
5878 gen_helper_sve_st1bd_r
},
5880 gen_helper_sve_st1hh_le_r
,
5881 gen_helper_sve_st1hs_le_r
,
5882 gen_helper_sve_st1hd_le_r
},
5884 gen_helper_sve_st1ss_le_r
,
5885 gen_helper_sve_st1sd_le_r
},
5887 gen_helper_sve_st1dd_le_r
} },
5888 { { gen_helper_sve_st1bb_r
,
5889 gen_helper_sve_st1bh_r
,
5890 gen_helper_sve_st1bs_r
,
5891 gen_helper_sve_st1bd_r
},
5893 gen_helper_sve_st1hh_be_r
,
5894 gen_helper_sve_st1hs_be_r
,
5895 gen_helper_sve_st1hd_be_r
},
5897 gen_helper_sve_st1ss_be_r
,
5898 gen_helper_sve_st1sd_be_r
},
5900 gen_helper_sve_st1dd_be_r
} } },
5902 { { { gen_helper_sve_st1bb_r_mte
,
5903 gen_helper_sve_st1bh_r_mte
,
5904 gen_helper_sve_st1bs_r_mte
,
5905 gen_helper_sve_st1bd_r_mte
},
5907 gen_helper_sve_st1hh_le_r_mte
,
5908 gen_helper_sve_st1hs_le_r_mte
,
5909 gen_helper_sve_st1hd_le_r_mte
},
5911 gen_helper_sve_st1ss_le_r_mte
,
5912 gen_helper_sve_st1sd_le_r_mte
},
5914 gen_helper_sve_st1dd_le_r_mte
} },
5915 { { gen_helper_sve_st1bb_r_mte
,
5916 gen_helper_sve_st1bh_r_mte
,
5917 gen_helper_sve_st1bs_r_mte
,
5918 gen_helper_sve_st1bd_r_mte
},
5920 gen_helper_sve_st1hh_be_r_mte
,
5921 gen_helper_sve_st1hs_be_r_mte
,
5922 gen_helper_sve_st1hd_be_r_mte
},
5924 gen_helper_sve_st1ss_be_r_mte
,
5925 gen_helper_sve_st1sd_be_r_mte
},
5927 gen_helper_sve_st1dd_be_r_mte
} } },
5929 static gen_helper_gvec_mem
* const fn_multiple
[2][2][3][4] = {
5930 { { { gen_helper_sve_st2bb_r
,
5931 gen_helper_sve_st2hh_le_r
,
5932 gen_helper_sve_st2ss_le_r
,
5933 gen_helper_sve_st2dd_le_r
},
5934 { gen_helper_sve_st3bb_r
,
5935 gen_helper_sve_st3hh_le_r
,
5936 gen_helper_sve_st3ss_le_r
,
5937 gen_helper_sve_st3dd_le_r
},
5938 { gen_helper_sve_st4bb_r
,
5939 gen_helper_sve_st4hh_le_r
,
5940 gen_helper_sve_st4ss_le_r
,
5941 gen_helper_sve_st4dd_le_r
} },
5942 { { gen_helper_sve_st2bb_r
,
5943 gen_helper_sve_st2hh_be_r
,
5944 gen_helper_sve_st2ss_be_r
,
5945 gen_helper_sve_st2dd_be_r
},
5946 { gen_helper_sve_st3bb_r
,
5947 gen_helper_sve_st3hh_be_r
,
5948 gen_helper_sve_st3ss_be_r
,
5949 gen_helper_sve_st3dd_be_r
},
5950 { gen_helper_sve_st4bb_r
,
5951 gen_helper_sve_st4hh_be_r
,
5952 gen_helper_sve_st4ss_be_r
,
5953 gen_helper_sve_st4dd_be_r
} } },
5954 { { { gen_helper_sve_st2bb_r_mte
,
5955 gen_helper_sve_st2hh_le_r_mte
,
5956 gen_helper_sve_st2ss_le_r_mte
,
5957 gen_helper_sve_st2dd_le_r_mte
},
5958 { gen_helper_sve_st3bb_r_mte
,
5959 gen_helper_sve_st3hh_le_r_mte
,
5960 gen_helper_sve_st3ss_le_r_mte
,
5961 gen_helper_sve_st3dd_le_r_mte
},
5962 { gen_helper_sve_st4bb_r_mte
,
5963 gen_helper_sve_st4hh_le_r_mte
,
5964 gen_helper_sve_st4ss_le_r_mte
,
5965 gen_helper_sve_st4dd_le_r_mte
} },
5966 { { gen_helper_sve_st2bb_r_mte
,
5967 gen_helper_sve_st2hh_be_r_mte
,
5968 gen_helper_sve_st2ss_be_r_mte
,
5969 gen_helper_sve_st2dd_be_r_mte
},
5970 { gen_helper_sve_st3bb_r_mte
,
5971 gen_helper_sve_st3hh_be_r_mte
,
5972 gen_helper_sve_st3ss_be_r_mte
,
5973 gen_helper_sve_st3dd_be_r_mte
},
5974 { gen_helper_sve_st4bb_r_mte
,
5975 gen_helper_sve_st4hh_be_r_mte
,
5976 gen_helper_sve_st4ss_be_r_mte
,
5977 gen_helper_sve_st4dd_be_r_mte
} } },
5979 gen_helper_gvec_mem
*fn
;
5980 int be
= s
->be_data
== MO_BE
;
5984 fn
= fn_single
[s
->mte_active
[0]][be
][msz
][esz
];
5987 /* ST2, ST3, ST4 -- msz == esz, enforced by encoding */
5989 fn
= fn_multiple
[s
->mte_active
[0]][be
][nreg
- 1][msz
];
5992 do_mem_zpa(s
, zt
, pg
, addr
, msz_dtype(s
, msz
), nreg
, true, fn
);
5995 static bool trans_ST_zprr(DisasContext
*s
, arg_rprr_store
*a
)
5997 if (a
->rm
== 31 || a
->msz
> a
->esz
) {
6000 if (sve_access_check(s
)) {
6001 TCGv_i64 addr
= new_tmp_a64(s
);
6002 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), a
->msz
);
6003 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
6004 do_st_zpa(s
, a
->rd
, a
->pg
, addr
, a
->msz
, a
->esz
, a
->nreg
);
6009 static bool trans_ST_zpri(DisasContext
*s
, arg_rpri_store
*a
)
6011 if (a
->msz
> a
->esz
) {
6014 if (sve_access_check(s
)) {
6015 int vsz
= vec_full_reg_size(s
);
6016 int elements
= vsz
>> a
->esz
;
6017 TCGv_i64 addr
= new_tmp_a64(s
);
6019 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
),
6020 (a
->imm
* elements
* (a
->nreg
+ 1)) << a
->msz
);
6021 do_st_zpa(s
, a
->rd
, a
->pg
, addr
, a
->msz
, a
->esz
, a
->nreg
);
6027 *** SVE gather loads / scatter stores
6030 static void do_mem_zpz(DisasContext
*s
, int zt
, int pg
, int zm
,
6031 int scale
, TCGv_i64 scalar
, int msz
, bool is_write
,
6032 gen_helper_gvec_mem_scatter
*fn
)
6034 unsigned vsz
= vec_full_reg_size(s
);
6035 TCGv_ptr t_zm
= tcg_temp_new_ptr();
6036 TCGv_ptr t_pg
= tcg_temp_new_ptr();
6037 TCGv_ptr t_zt
= tcg_temp_new_ptr();
6041 if (s
->mte_active
[0]) {
6042 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, get_mem_index(s
));
6043 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
6044 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
6045 desc
= FIELD_DP32(desc
, MTEDESC
, WRITE
, is_write
);
6046 desc
= FIELD_DP32(desc
, MTEDESC
, SIZEM1
, (1 << msz
) - 1);
6047 desc
<<= SVE_MTEDESC_SHIFT
;
6049 desc
= simd_desc(vsz
, vsz
, desc
| scale
);
6050 t_desc
= tcg_const_i32(desc
);
6052 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
6053 tcg_gen_addi_ptr(t_zm
, cpu_env
, vec_full_reg_offset(s
, zm
));
6054 tcg_gen_addi_ptr(t_zt
, cpu_env
, vec_full_reg_offset(s
, zt
));
6055 fn(cpu_env
, t_zt
, t_pg
, t_zm
, scalar
, t_desc
);
6057 tcg_temp_free_ptr(t_zt
);
6058 tcg_temp_free_ptr(t_zm
);
6059 tcg_temp_free_ptr(t_pg
);
6060 tcg_temp_free_i32(t_desc
);
6063 /* Indexed by [mte][be][ff][xs][u][msz]. */
6064 static gen_helper_gvec_mem_scatter
* const
6065 gather_load_fn32
[2][2][2][2][2][3] = {
6066 { /* MTE Inactive */
6067 { /* Little-endian */
6068 { { { gen_helper_sve_ldbss_zsu
,
6069 gen_helper_sve_ldhss_le_zsu
,
6071 { gen_helper_sve_ldbsu_zsu
,
6072 gen_helper_sve_ldhsu_le_zsu
,
6073 gen_helper_sve_ldss_le_zsu
, } },
6074 { { gen_helper_sve_ldbss_zss
,
6075 gen_helper_sve_ldhss_le_zss
,
6077 { gen_helper_sve_ldbsu_zss
,
6078 gen_helper_sve_ldhsu_le_zss
,
6079 gen_helper_sve_ldss_le_zss
, } } },
6082 { { { gen_helper_sve_ldffbss_zsu
,
6083 gen_helper_sve_ldffhss_le_zsu
,
6085 { gen_helper_sve_ldffbsu_zsu
,
6086 gen_helper_sve_ldffhsu_le_zsu
,
6087 gen_helper_sve_ldffss_le_zsu
, } },
6088 { { gen_helper_sve_ldffbss_zss
,
6089 gen_helper_sve_ldffhss_le_zss
,
6091 { gen_helper_sve_ldffbsu_zss
,
6092 gen_helper_sve_ldffhsu_le_zss
,
6093 gen_helper_sve_ldffss_le_zss
, } } } },
6096 { { { gen_helper_sve_ldbss_zsu
,
6097 gen_helper_sve_ldhss_be_zsu
,
6099 { gen_helper_sve_ldbsu_zsu
,
6100 gen_helper_sve_ldhsu_be_zsu
,
6101 gen_helper_sve_ldss_be_zsu
, } },
6102 { { gen_helper_sve_ldbss_zss
,
6103 gen_helper_sve_ldhss_be_zss
,
6105 { gen_helper_sve_ldbsu_zss
,
6106 gen_helper_sve_ldhsu_be_zss
,
6107 gen_helper_sve_ldss_be_zss
, } } },
6110 { { { gen_helper_sve_ldffbss_zsu
,
6111 gen_helper_sve_ldffhss_be_zsu
,
6113 { gen_helper_sve_ldffbsu_zsu
,
6114 gen_helper_sve_ldffhsu_be_zsu
,
6115 gen_helper_sve_ldffss_be_zsu
, } },
6116 { { gen_helper_sve_ldffbss_zss
,
6117 gen_helper_sve_ldffhss_be_zss
,
6119 { gen_helper_sve_ldffbsu_zss
,
6120 gen_helper_sve_ldffhsu_be_zss
,
6121 gen_helper_sve_ldffss_be_zss
, } } } } },
6123 { /* Little-endian */
6124 { { { gen_helper_sve_ldbss_zsu_mte
,
6125 gen_helper_sve_ldhss_le_zsu_mte
,
6127 { gen_helper_sve_ldbsu_zsu_mte
,
6128 gen_helper_sve_ldhsu_le_zsu_mte
,
6129 gen_helper_sve_ldss_le_zsu_mte
, } },
6130 { { gen_helper_sve_ldbss_zss_mte
,
6131 gen_helper_sve_ldhss_le_zss_mte
,
6133 { gen_helper_sve_ldbsu_zss_mte
,
6134 gen_helper_sve_ldhsu_le_zss_mte
,
6135 gen_helper_sve_ldss_le_zss_mte
, } } },
6138 { { { gen_helper_sve_ldffbss_zsu_mte
,
6139 gen_helper_sve_ldffhss_le_zsu_mte
,
6141 { gen_helper_sve_ldffbsu_zsu_mte
,
6142 gen_helper_sve_ldffhsu_le_zsu_mte
,
6143 gen_helper_sve_ldffss_le_zsu_mte
, } },
6144 { { gen_helper_sve_ldffbss_zss_mte
,
6145 gen_helper_sve_ldffhss_le_zss_mte
,
6147 { gen_helper_sve_ldffbsu_zss_mte
,
6148 gen_helper_sve_ldffhsu_le_zss_mte
,
6149 gen_helper_sve_ldffss_le_zss_mte
, } } } },
6152 { { { gen_helper_sve_ldbss_zsu_mte
,
6153 gen_helper_sve_ldhss_be_zsu_mte
,
6155 { gen_helper_sve_ldbsu_zsu_mte
,
6156 gen_helper_sve_ldhsu_be_zsu_mte
,
6157 gen_helper_sve_ldss_be_zsu_mte
, } },
6158 { { gen_helper_sve_ldbss_zss_mte
,
6159 gen_helper_sve_ldhss_be_zss_mte
,
6161 { gen_helper_sve_ldbsu_zss_mte
,
6162 gen_helper_sve_ldhsu_be_zss_mte
,
6163 gen_helper_sve_ldss_be_zss_mte
, } } },
6166 { { { gen_helper_sve_ldffbss_zsu_mte
,
6167 gen_helper_sve_ldffhss_be_zsu_mte
,
6169 { gen_helper_sve_ldffbsu_zsu_mte
,
6170 gen_helper_sve_ldffhsu_be_zsu_mte
,
6171 gen_helper_sve_ldffss_be_zsu_mte
, } },
6172 { { gen_helper_sve_ldffbss_zss_mte
,
6173 gen_helper_sve_ldffhss_be_zss_mte
,
6175 { gen_helper_sve_ldffbsu_zss_mte
,
6176 gen_helper_sve_ldffhsu_be_zss_mte
,
6177 gen_helper_sve_ldffss_be_zss_mte
, } } } } },
6180 /* Note that we overload xs=2 to indicate 64-bit offset. */
6181 static gen_helper_gvec_mem_scatter
* const
6182 gather_load_fn64
[2][2][2][3][2][4] = {
6183 { /* MTE Inactive */
6184 { /* Little-endian */
6185 { { { gen_helper_sve_ldbds_zsu
,
6186 gen_helper_sve_ldhds_le_zsu
,
6187 gen_helper_sve_ldsds_le_zsu
,
6189 { gen_helper_sve_ldbdu_zsu
,
6190 gen_helper_sve_ldhdu_le_zsu
,
6191 gen_helper_sve_ldsdu_le_zsu
,
6192 gen_helper_sve_lddd_le_zsu
, } },
6193 { { gen_helper_sve_ldbds_zss
,
6194 gen_helper_sve_ldhds_le_zss
,
6195 gen_helper_sve_ldsds_le_zss
,
6197 { gen_helper_sve_ldbdu_zss
,
6198 gen_helper_sve_ldhdu_le_zss
,
6199 gen_helper_sve_ldsdu_le_zss
,
6200 gen_helper_sve_lddd_le_zss
, } },
6201 { { gen_helper_sve_ldbds_zd
,
6202 gen_helper_sve_ldhds_le_zd
,
6203 gen_helper_sve_ldsds_le_zd
,
6205 { gen_helper_sve_ldbdu_zd
,
6206 gen_helper_sve_ldhdu_le_zd
,
6207 gen_helper_sve_ldsdu_le_zd
,
6208 gen_helper_sve_lddd_le_zd
, } } },
6211 { { { gen_helper_sve_ldffbds_zsu
,
6212 gen_helper_sve_ldffhds_le_zsu
,
6213 gen_helper_sve_ldffsds_le_zsu
,
6215 { gen_helper_sve_ldffbdu_zsu
,
6216 gen_helper_sve_ldffhdu_le_zsu
,
6217 gen_helper_sve_ldffsdu_le_zsu
,
6218 gen_helper_sve_ldffdd_le_zsu
, } },
6219 { { gen_helper_sve_ldffbds_zss
,
6220 gen_helper_sve_ldffhds_le_zss
,
6221 gen_helper_sve_ldffsds_le_zss
,
6223 { gen_helper_sve_ldffbdu_zss
,
6224 gen_helper_sve_ldffhdu_le_zss
,
6225 gen_helper_sve_ldffsdu_le_zss
,
6226 gen_helper_sve_ldffdd_le_zss
, } },
6227 { { gen_helper_sve_ldffbds_zd
,
6228 gen_helper_sve_ldffhds_le_zd
,
6229 gen_helper_sve_ldffsds_le_zd
,
6231 { gen_helper_sve_ldffbdu_zd
,
6232 gen_helper_sve_ldffhdu_le_zd
,
6233 gen_helper_sve_ldffsdu_le_zd
,
6234 gen_helper_sve_ldffdd_le_zd
, } } } },
6236 { { { gen_helper_sve_ldbds_zsu
,
6237 gen_helper_sve_ldhds_be_zsu
,
6238 gen_helper_sve_ldsds_be_zsu
,
6240 { gen_helper_sve_ldbdu_zsu
,
6241 gen_helper_sve_ldhdu_be_zsu
,
6242 gen_helper_sve_ldsdu_be_zsu
,
6243 gen_helper_sve_lddd_be_zsu
, } },
6244 { { gen_helper_sve_ldbds_zss
,
6245 gen_helper_sve_ldhds_be_zss
,
6246 gen_helper_sve_ldsds_be_zss
,
6248 { gen_helper_sve_ldbdu_zss
,
6249 gen_helper_sve_ldhdu_be_zss
,
6250 gen_helper_sve_ldsdu_be_zss
,
6251 gen_helper_sve_lddd_be_zss
, } },
6252 { { gen_helper_sve_ldbds_zd
,
6253 gen_helper_sve_ldhds_be_zd
,
6254 gen_helper_sve_ldsds_be_zd
,
6256 { gen_helper_sve_ldbdu_zd
,
6257 gen_helper_sve_ldhdu_be_zd
,
6258 gen_helper_sve_ldsdu_be_zd
,
6259 gen_helper_sve_lddd_be_zd
, } } },
6262 { { { gen_helper_sve_ldffbds_zsu
,
6263 gen_helper_sve_ldffhds_be_zsu
,
6264 gen_helper_sve_ldffsds_be_zsu
,
6266 { gen_helper_sve_ldffbdu_zsu
,
6267 gen_helper_sve_ldffhdu_be_zsu
,
6268 gen_helper_sve_ldffsdu_be_zsu
,
6269 gen_helper_sve_ldffdd_be_zsu
, } },
6270 { { gen_helper_sve_ldffbds_zss
,
6271 gen_helper_sve_ldffhds_be_zss
,
6272 gen_helper_sve_ldffsds_be_zss
,
6274 { gen_helper_sve_ldffbdu_zss
,
6275 gen_helper_sve_ldffhdu_be_zss
,
6276 gen_helper_sve_ldffsdu_be_zss
,
6277 gen_helper_sve_ldffdd_be_zss
, } },
6278 { { gen_helper_sve_ldffbds_zd
,
6279 gen_helper_sve_ldffhds_be_zd
,
6280 gen_helper_sve_ldffsds_be_zd
,
6282 { gen_helper_sve_ldffbdu_zd
,
6283 gen_helper_sve_ldffhdu_be_zd
,
6284 gen_helper_sve_ldffsdu_be_zd
,
6285 gen_helper_sve_ldffdd_be_zd
, } } } } },
6287 { /* Little-endian */
6288 { { { gen_helper_sve_ldbds_zsu_mte
,
6289 gen_helper_sve_ldhds_le_zsu_mte
,
6290 gen_helper_sve_ldsds_le_zsu_mte
,
6292 { gen_helper_sve_ldbdu_zsu_mte
,
6293 gen_helper_sve_ldhdu_le_zsu_mte
,
6294 gen_helper_sve_ldsdu_le_zsu_mte
,
6295 gen_helper_sve_lddd_le_zsu_mte
, } },
6296 { { gen_helper_sve_ldbds_zss_mte
,
6297 gen_helper_sve_ldhds_le_zss_mte
,
6298 gen_helper_sve_ldsds_le_zss_mte
,
6300 { gen_helper_sve_ldbdu_zss_mte
,
6301 gen_helper_sve_ldhdu_le_zss_mte
,
6302 gen_helper_sve_ldsdu_le_zss_mte
,
6303 gen_helper_sve_lddd_le_zss_mte
, } },
6304 { { gen_helper_sve_ldbds_zd_mte
,
6305 gen_helper_sve_ldhds_le_zd_mte
,
6306 gen_helper_sve_ldsds_le_zd_mte
,
6308 { gen_helper_sve_ldbdu_zd_mte
,
6309 gen_helper_sve_ldhdu_le_zd_mte
,
6310 gen_helper_sve_ldsdu_le_zd_mte
,
6311 gen_helper_sve_lddd_le_zd_mte
, } } },
6314 { { { gen_helper_sve_ldffbds_zsu_mte
,
6315 gen_helper_sve_ldffhds_le_zsu_mte
,
6316 gen_helper_sve_ldffsds_le_zsu_mte
,
6318 { gen_helper_sve_ldffbdu_zsu_mte
,
6319 gen_helper_sve_ldffhdu_le_zsu_mte
,
6320 gen_helper_sve_ldffsdu_le_zsu_mte
,
6321 gen_helper_sve_ldffdd_le_zsu_mte
, } },
6322 { { gen_helper_sve_ldffbds_zss_mte
,
6323 gen_helper_sve_ldffhds_le_zss_mte
,
6324 gen_helper_sve_ldffsds_le_zss_mte
,
6326 { gen_helper_sve_ldffbdu_zss_mte
,
6327 gen_helper_sve_ldffhdu_le_zss_mte
,
6328 gen_helper_sve_ldffsdu_le_zss_mte
,
6329 gen_helper_sve_ldffdd_le_zss_mte
, } },
6330 { { gen_helper_sve_ldffbds_zd_mte
,
6331 gen_helper_sve_ldffhds_le_zd_mte
,
6332 gen_helper_sve_ldffsds_le_zd_mte
,
6334 { gen_helper_sve_ldffbdu_zd_mte
,
6335 gen_helper_sve_ldffhdu_le_zd_mte
,
6336 gen_helper_sve_ldffsdu_le_zd_mte
,
6337 gen_helper_sve_ldffdd_le_zd_mte
, } } } },
6339 { { { gen_helper_sve_ldbds_zsu_mte
,
6340 gen_helper_sve_ldhds_be_zsu_mte
,
6341 gen_helper_sve_ldsds_be_zsu_mte
,
6343 { gen_helper_sve_ldbdu_zsu_mte
,
6344 gen_helper_sve_ldhdu_be_zsu_mte
,
6345 gen_helper_sve_ldsdu_be_zsu_mte
,
6346 gen_helper_sve_lddd_be_zsu_mte
, } },
6347 { { gen_helper_sve_ldbds_zss_mte
,
6348 gen_helper_sve_ldhds_be_zss_mte
,
6349 gen_helper_sve_ldsds_be_zss_mte
,
6351 { gen_helper_sve_ldbdu_zss_mte
,
6352 gen_helper_sve_ldhdu_be_zss_mte
,
6353 gen_helper_sve_ldsdu_be_zss_mte
,
6354 gen_helper_sve_lddd_be_zss_mte
, } },
6355 { { gen_helper_sve_ldbds_zd_mte
,
6356 gen_helper_sve_ldhds_be_zd_mte
,
6357 gen_helper_sve_ldsds_be_zd_mte
,
6359 { gen_helper_sve_ldbdu_zd_mte
,
6360 gen_helper_sve_ldhdu_be_zd_mte
,
6361 gen_helper_sve_ldsdu_be_zd_mte
,
6362 gen_helper_sve_lddd_be_zd_mte
, } } },
6365 { { { gen_helper_sve_ldffbds_zsu_mte
,
6366 gen_helper_sve_ldffhds_be_zsu_mte
,
6367 gen_helper_sve_ldffsds_be_zsu_mte
,
6369 { gen_helper_sve_ldffbdu_zsu_mte
,
6370 gen_helper_sve_ldffhdu_be_zsu_mte
,
6371 gen_helper_sve_ldffsdu_be_zsu_mte
,
6372 gen_helper_sve_ldffdd_be_zsu_mte
, } },
6373 { { gen_helper_sve_ldffbds_zss_mte
,
6374 gen_helper_sve_ldffhds_be_zss_mte
,
6375 gen_helper_sve_ldffsds_be_zss_mte
,
6377 { gen_helper_sve_ldffbdu_zss_mte
,
6378 gen_helper_sve_ldffhdu_be_zss_mte
,
6379 gen_helper_sve_ldffsdu_be_zss_mte
,
6380 gen_helper_sve_ldffdd_be_zss_mte
, } },
6381 { { gen_helper_sve_ldffbds_zd_mte
,
6382 gen_helper_sve_ldffhds_be_zd_mte
,
6383 gen_helper_sve_ldffsds_be_zd_mte
,
6385 { gen_helper_sve_ldffbdu_zd_mte
,
6386 gen_helper_sve_ldffhdu_be_zd_mte
,
6387 gen_helper_sve_ldffsdu_be_zd_mte
,
6388 gen_helper_sve_ldffdd_be_zd_mte
, } } } } },
6391 static bool trans_LD1_zprz(DisasContext
*s
, arg_LD1_zprz
*a
)
6393 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6394 bool be
= s
->be_data
== MO_BE
;
6395 bool mte
= s
->mte_active
[0];
6397 if (!sve_access_check(s
)) {
6403 fn
= gather_load_fn32
[mte
][be
][a
->ff
][a
->xs
][a
->u
][a
->msz
];
6406 fn
= gather_load_fn64
[mte
][be
][a
->ff
][a
->xs
][a
->u
][a
->msz
];
6411 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rm
, a
->scale
* a
->msz
,
6412 cpu_reg_sp(s
, a
->rn
), a
->msz
, false, fn
);
6416 static bool trans_LD1_zpiz(DisasContext
*s
, arg_LD1_zpiz
*a
)
6418 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6419 bool be
= s
->be_data
== MO_BE
;
6420 bool mte
= s
->mte_active
[0];
6422 if (a
->esz
< a
->msz
|| (a
->esz
== a
->msz
&& !a
->u
)) {
6425 if (!sve_access_check(s
)) {
6431 fn
= gather_load_fn32
[mte
][be
][a
->ff
][0][a
->u
][a
->msz
];
6434 fn
= gather_load_fn64
[mte
][be
][a
->ff
][2][a
->u
][a
->msz
];
6439 /* Treat LD1_zpiz (zn[x] + imm) the same way as LD1_zprz (rn + zm[x])
6440 * by loading the immediate into the scalar parameter.
6442 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0,
6443 tcg_constant_i64(a
->imm
<< a
->msz
), a
->msz
, false, fn
);
6447 static bool trans_LDNT1_zprz(DisasContext
*s
, arg_LD1_zprz
*a
)
6449 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6450 bool be
= s
->be_data
== MO_BE
;
6451 bool mte
= s
->mte_active
[0];
6453 if (a
->esz
< a
->msz
+ !a
->u
) {
6456 if (!dc_isar_feature(aa64_sve2
, s
)) {
6459 if (!sve_access_check(s
)) {
6465 fn
= gather_load_fn32
[mte
][be
][0][0][a
->u
][a
->msz
];
6468 fn
= gather_load_fn64
[mte
][be
][0][2][a
->u
][a
->msz
];
6473 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0,
6474 cpu_reg(s
, a
->rm
), a
->msz
, false, fn
);
6478 /* Indexed by [mte][be][xs][msz]. */
6479 static gen_helper_gvec_mem_scatter
* const scatter_store_fn32
[2][2][2][3] = {
6480 { /* MTE Inactive */
6481 { /* Little-endian */
6482 { gen_helper_sve_stbs_zsu
,
6483 gen_helper_sve_sths_le_zsu
,
6484 gen_helper_sve_stss_le_zsu
, },
6485 { gen_helper_sve_stbs_zss
,
6486 gen_helper_sve_sths_le_zss
,
6487 gen_helper_sve_stss_le_zss
, } },
6489 { gen_helper_sve_stbs_zsu
,
6490 gen_helper_sve_sths_be_zsu
,
6491 gen_helper_sve_stss_be_zsu
, },
6492 { gen_helper_sve_stbs_zss
,
6493 gen_helper_sve_sths_be_zss
,
6494 gen_helper_sve_stss_be_zss
, } } },
6496 { /* Little-endian */
6497 { gen_helper_sve_stbs_zsu_mte
,
6498 gen_helper_sve_sths_le_zsu_mte
,
6499 gen_helper_sve_stss_le_zsu_mte
, },
6500 { gen_helper_sve_stbs_zss_mte
,
6501 gen_helper_sve_sths_le_zss_mte
,
6502 gen_helper_sve_stss_le_zss_mte
, } },
6504 { gen_helper_sve_stbs_zsu_mte
,
6505 gen_helper_sve_sths_be_zsu_mte
,
6506 gen_helper_sve_stss_be_zsu_mte
, },
6507 { gen_helper_sve_stbs_zss_mte
,
6508 gen_helper_sve_sths_be_zss_mte
,
6509 gen_helper_sve_stss_be_zss_mte
, } } },
6512 /* Note that we overload xs=2 to indicate 64-bit offset. */
6513 static gen_helper_gvec_mem_scatter
* const scatter_store_fn64
[2][2][3][4] = {
6514 { /* MTE Inactive */
6515 { /* Little-endian */
6516 { gen_helper_sve_stbd_zsu
,
6517 gen_helper_sve_sthd_le_zsu
,
6518 gen_helper_sve_stsd_le_zsu
,
6519 gen_helper_sve_stdd_le_zsu
, },
6520 { gen_helper_sve_stbd_zss
,
6521 gen_helper_sve_sthd_le_zss
,
6522 gen_helper_sve_stsd_le_zss
,
6523 gen_helper_sve_stdd_le_zss
, },
6524 { gen_helper_sve_stbd_zd
,
6525 gen_helper_sve_sthd_le_zd
,
6526 gen_helper_sve_stsd_le_zd
,
6527 gen_helper_sve_stdd_le_zd
, } },
6529 { gen_helper_sve_stbd_zsu
,
6530 gen_helper_sve_sthd_be_zsu
,
6531 gen_helper_sve_stsd_be_zsu
,
6532 gen_helper_sve_stdd_be_zsu
, },
6533 { gen_helper_sve_stbd_zss
,
6534 gen_helper_sve_sthd_be_zss
,
6535 gen_helper_sve_stsd_be_zss
,
6536 gen_helper_sve_stdd_be_zss
, },
6537 { gen_helper_sve_stbd_zd
,
6538 gen_helper_sve_sthd_be_zd
,
6539 gen_helper_sve_stsd_be_zd
,
6540 gen_helper_sve_stdd_be_zd
, } } },
6541 { /* MTE Inactive */
6542 { /* Little-endian */
6543 { gen_helper_sve_stbd_zsu_mte
,
6544 gen_helper_sve_sthd_le_zsu_mte
,
6545 gen_helper_sve_stsd_le_zsu_mte
,
6546 gen_helper_sve_stdd_le_zsu_mte
, },
6547 { gen_helper_sve_stbd_zss_mte
,
6548 gen_helper_sve_sthd_le_zss_mte
,
6549 gen_helper_sve_stsd_le_zss_mte
,
6550 gen_helper_sve_stdd_le_zss_mte
, },
6551 { gen_helper_sve_stbd_zd_mte
,
6552 gen_helper_sve_sthd_le_zd_mte
,
6553 gen_helper_sve_stsd_le_zd_mte
,
6554 gen_helper_sve_stdd_le_zd_mte
, } },
6556 { gen_helper_sve_stbd_zsu_mte
,
6557 gen_helper_sve_sthd_be_zsu_mte
,
6558 gen_helper_sve_stsd_be_zsu_mte
,
6559 gen_helper_sve_stdd_be_zsu_mte
, },
6560 { gen_helper_sve_stbd_zss_mte
,
6561 gen_helper_sve_sthd_be_zss_mte
,
6562 gen_helper_sve_stsd_be_zss_mte
,
6563 gen_helper_sve_stdd_be_zss_mte
, },
6564 { gen_helper_sve_stbd_zd_mte
,
6565 gen_helper_sve_sthd_be_zd_mte
,
6566 gen_helper_sve_stsd_be_zd_mte
,
6567 gen_helper_sve_stdd_be_zd_mte
, } } },
6570 static bool trans_ST1_zprz(DisasContext
*s
, arg_ST1_zprz
*a
)
6572 gen_helper_gvec_mem_scatter
*fn
;
6573 bool be
= s
->be_data
== MO_BE
;
6574 bool mte
= s
->mte_active
[0];
6576 if (a
->esz
< a
->msz
|| (a
->msz
== 0 && a
->scale
)) {
6579 if (!sve_access_check(s
)) {
6584 fn
= scatter_store_fn32
[mte
][be
][a
->xs
][a
->msz
];
6587 fn
= scatter_store_fn64
[mte
][be
][a
->xs
][a
->msz
];
6590 g_assert_not_reached();
6592 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rm
, a
->scale
* a
->msz
,
6593 cpu_reg_sp(s
, a
->rn
), a
->msz
, true, fn
);
6597 static bool trans_ST1_zpiz(DisasContext
*s
, arg_ST1_zpiz
*a
)
6599 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6600 bool be
= s
->be_data
== MO_BE
;
6601 bool mte
= s
->mte_active
[0];
6603 if (a
->esz
< a
->msz
) {
6606 if (!sve_access_check(s
)) {
6612 fn
= scatter_store_fn32
[mte
][be
][0][a
->msz
];
6615 fn
= scatter_store_fn64
[mte
][be
][2][a
->msz
];
6620 /* Treat ST1_zpiz (zn[x] + imm) the same way as ST1_zprz (rn + zm[x])
6621 * by loading the immediate into the scalar parameter.
6623 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0,
6624 tcg_constant_i64(a
->imm
<< a
->msz
), a
->msz
, true, fn
);
6628 static bool trans_STNT1_zprz(DisasContext
*s
, arg_ST1_zprz
*a
)
6630 gen_helper_gvec_mem_scatter
*fn
;
6631 bool be
= s
->be_data
== MO_BE
;
6632 bool mte
= s
->mte_active
[0];
6634 if (a
->esz
< a
->msz
) {
6637 if (!dc_isar_feature(aa64_sve2
, s
)) {
6640 if (!sve_access_check(s
)) {
6646 fn
= scatter_store_fn32
[mte
][be
][0][a
->msz
];
6649 fn
= scatter_store_fn64
[mte
][be
][2][a
->msz
];
6652 g_assert_not_reached();
6655 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0,
6656 cpu_reg(s
, a
->rm
), a
->msz
, true, fn
);
6664 static bool trans_PRF(DisasContext
*s
, arg_PRF
*a
)
6666 /* Prefetch is a nop within QEMU. */
6667 (void)sve_access_check(s
);
6671 static bool trans_PRF_rr(DisasContext
*s
, arg_PRF_rr
*a
)
6676 /* Prefetch is a nop within QEMU. */
6677 (void)sve_access_check(s
);
6684 * TODO: The implementation so far could handle predicated merging movprfx.
6685 * The helper functions as written take an extra source register to
6686 * use in the operation, but the result is only written when predication
6687 * succeeds. For unpredicated movprfx, we need to rearrange the helpers
6688 * to allow the final write back to the destination to be unconditional.
6689 * For predicated zeroing movprfx, we need to rearrange the helpers to
6690 * allow the final write back to zero inactives.
6692 * In the meantime, just emit the moves.
6695 static bool trans_MOVPRFX(DisasContext
*s
, arg_MOVPRFX
*a
)
6697 return do_mov_z(s
, a
->rd
, a
->rn
);
6700 static bool trans_MOVPRFX_m(DisasContext
*s
, arg_rpr_esz
*a
)
6702 if (sve_access_check(s
)) {
6703 do_sel_z(s
, a
->rd
, a
->rn
, a
->rd
, a
->pg
, a
->esz
);
6708 static bool trans_MOVPRFX_z(DisasContext
*s
, arg_rpr_esz
*a
)
6710 return do_movz_zpz(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
, false);
6714 * SVE2 Integer Multiply - Unpredicated
6717 static bool trans_MUL_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6719 if (!dc_isar_feature(aa64_sve2
, s
)) {
6722 if (sve_access_check(s
)) {
6723 gen_gvec_fn_zzz(s
, tcg_gen_gvec_mul
, a
->esz
, a
->rd
, a
->rn
, a
->rm
);
6728 static bool do_sve2_zzz_ool(DisasContext
*s
, arg_rrr_esz
*a
,
6729 gen_helper_gvec_3
*fn
)
6731 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
6734 if (sve_access_check(s
)) {
6735 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, 0);
6740 static bool trans_SMULH_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6742 static gen_helper_gvec_3
* const fns
[4] = {
6743 gen_helper_gvec_smulh_b
, gen_helper_gvec_smulh_h
,
6744 gen_helper_gvec_smulh_s
, gen_helper_gvec_smulh_d
,
6746 return do_sve2_zzz_ool(s
, a
, fns
[a
->esz
]);
6749 static bool trans_UMULH_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6751 static gen_helper_gvec_3
* const fns
[4] = {
6752 gen_helper_gvec_umulh_b
, gen_helper_gvec_umulh_h
,
6753 gen_helper_gvec_umulh_s
, gen_helper_gvec_umulh_d
,
6755 return do_sve2_zzz_ool(s
, a
, fns
[a
->esz
]);
6758 static bool trans_PMUL_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6760 return do_sve2_zzz_ool(s
, a
, gen_helper_gvec_pmul_b
);
6763 static bool trans_SQDMULH_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6765 static gen_helper_gvec_3
* const fns
[4] = {
6766 gen_helper_sve2_sqdmulh_b
, gen_helper_sve2_sqdmulh_h
,
6767 gen_helper_sve2_sqdmulh_s
, gen_helper_sve2_sqdmulh_d
,
6769 return do_sve2_zzz_ool(s
, a
, fns
[a
->esz
]);
6772 static bool trans_SQRDMULH_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6774 static gen_helper_gvec_3
* const fns
[4] = {
6775 gen_helper_sve2_sqrdmulh_b
, gen_helper_sve2_sqrdmulh_h
,
6776 gen_helper_sve2_sqrdmulh_s
, gen_helper_sve2_sqrdmulh_d
,
6778 return do_sve2_zzz_ool(s
, a
, fns
[a
->esz
]);
6782 * SVE2 Integer - Predicated
6785 static bool do_sve2_zpzz_ool(DisasContext
*s
, arg_rprr_esz
*a
,
6786 gen_helper_gvec_4
*fn
)
6788 if (!dc_isar_feature(aa64_sve2
, s
)) {
6791 return do_zpzz_ool(s
, a
, fn
);
6794 static bool trans_SADALP_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
6796 static gen_helper_gvec_4
* const fns
[3] = {
6797 gen_helper_sve2_sadalp_zpzz_h
,
6798 gen_helper_sve2_sadalp_zpzz_s
,
6799 gen_helper_sve2_sadalp_zpzz_d
,
6804 return do_sve2_zpzz_ool(s
, a
, fns
[a
->esz
- 1]);
6807 static bool trans_UADALP_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
6809 static gen_helper_gvec_4
* const fns
[3] = {
6810 gen_helper_sve2_uadalp_zpzz_h
,
6811 gen_helper_sve2_uadalp_zpzz_s
,
6812 gen_helper_sve2_uadalp_zpzz_d
,
6817 return do_sve2_zpzz_ool(s
, a
, fns
[a
->esz
- 1]);
6821 * SVE2 integer unary operations (predicated)
6824 static bool do_sve2_zpz_ool(DisasContext
*s
, arg_rpr_esz
*a
,
6825 gen_helper_gvec_3
*fn
)
6827 if (!dc_isar_feature(aa64_sve2
, s
)) {
6830 return do_zpz_ool(s
, a
, fn
);
6833 static bool trans_URECPE(DisasContext
*s
, arg_rpr_esz
*a
)
6838 return do_sve2_zpz_ool(s
, a
, gen_helper_sve2_urecpe_s
);
6841 static bool trans_URSQRTE(DisasContext
*s
, arg_rpr_esz
*a
)
6846 return do_sve2_zpz_ool(s
, a
, gen_helper_sve2_ursqrte_s
);
6849 static bool trans_SQABS(DisasContext
*s
, arg_rpr_esz
*a
)
6851 static gen_helper_gvec_3
* const fns
[4] = {
6852 gen_helper_sve2_sqabs_b
, gen_helper_sve2_sqabs_h
,
6853 gen_helper_sve2_sqabs_s
, gen_helper_sve2_sqabs_d
,
6855 return do_sve2_zpz_ool(s
, a
, fns
[a
->esz
]);
6858 static bool trans_SQNEG(DisasContext
*s
, arg_rpr_esz
*a
)
6860 static gen_helper_gvec_3
* const fns
[4] = {
6861 gen_helper_sve2_sqneg_b
, gen_helper_sve2_sqneg_h
,
6862 gen_helper_sve2_sqneg_s
, gen_helper_sve2_sqneg_d
,
6864 return do_sve2_zpz_ool(s
, a
, fns
[a
->esz
]);
6867 #define DO_SVE2_ZPZZ(NAME, name) \
6868 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
6870 static gen_helper_gvec_4 * const fns[4] = { \
6871 gen_helper_sve2_##name##_zpzz_b, gen_helper_sve2_##name##_zpzz_h, \
6872 gen_helper_sve2_##name##_zpzz_s, gen_helper_sve2_##name##_zpzz_d, \
6874 return do_sve2_zpzz_ool(s, a, fns[a->esz]); \
6877 DO_SVE2_ZPZZ(SQSHL
, sqshl
)
6878 DO_SVE2_ZPZZ(SQRSHL
, sqrshl
)
6879 DO_SVE2_ZPZZ(SRSHL
, srshl
)
6881 DO_SVE2_ZPZZ(UQSHL
, uqshl
)
6882 DO_SVE2_ZPZZ(UQRSHL
, uqrshl
)
6883 DO_SVE2_ZPZZ(URSHL
, urshl
)
6885 DO_SVE2_ZPZZ(SHADD
, shadd
)
6886 DO_SVE2_ZPZZ(SRHADD
, srhadd
)
6887 DO_SVE2_ZPZZ(SHSUB
, shsub
)
6889 DO_SVE2_ZPZZ(UHADD
, uhadd
)
6890 DO_SVE2_ZPZZ(URHADD
, urhadd
)
6891 DO_SVE2_ZPZZ(UHSUB
, uhsub
)
6893 DO_SVE2_ZPZZ(ADDP
, addp
)
6894 DO_SVE2_ZPZZ(SMAXP
, smaxp
)
6895 DO_SVE2_ZPZZ(UMAXP
, umaxp
)
6896 DO_SVE2_ZPZZ(SMINP
, sminp
)
6897 DO_SVE2_ZPZZ(UMINP
, uminp
)
6899 DO_SVE2_ZPZZ(SQADD_zpzz
, sqadd
)
6900 DO_SVE2_ZPZZ(UQADD_zpzz
, uqadd
)
6901 DO_SVE2_ZPZZ(SQSUB_zpzz
, sqsub
)
6902 DO_SVE2_ZPZZ(UQSUB_zpzz
, uqsub
)
6903 DO_SVE2_ZPZZ(SUQADD
, suqadd
)
6904 DO_SVE2_ZPZZ(USQADD
, usqadd
)
6907 * SVE2 Widening Integer Arithmetic
6910 static bool do_sve2_zzw_ool(DisasContext
*s
, arg_rrr_esz
*a
,
6911 gen_helper_gvec_3
*fn
, int data
)
6913 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
6916 if (sve_access_check(s
)) {
6917 unsigned vsz
= vec_full_reg_size(s
);
6918 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, a
->rd
),
6919 vec_full_reg_offset(s
, a
->rn
),
6920 vec_full_reg_offset(s
, a
->rm
),
6921 vsz
, vsz
, data
, fn
);
6926 #define DO_SVE2_ZZZ_TB(NAME, name, SEL1, SEL2) \
6927 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
6929 static gen_helper_gvec_3 * const fns[4] = { \
6930 NULL, gen_helper_sve2_##name##_h, \
6931 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
6933 return do_sve2_zzw_ool(s, a, fns[a->esz], (SEL2 << 1) | SEL1); \
6936 DO_SVE2_ZZZ_TB(SADDLB
, saddl
, false, false)
6937 DO_SVE2_ZZZ_TB(SSUBLB
, ssubl
, false, false)
6938 DO_SVE2_ZZZ_TB(SABDLB
, sabdl
, false, false)
6940 DO_SVE2_ZZZ_TB(UADDLB
, uaddl
, false, false)
6941 DO_SVE2_ZZZ_TB(USUBLB
, usubl
, false, false)
6942 DO_SVE2_ZZZ_TB(UABDLB
, uabdl
, false, false)
6944 DO_SVE2_ZZZ_TB(SADDLT
, saddl
, true, true)
6945 DO_SVE2_ZZZ_TB(SSUBLT
, ssubl
, true, true)
6946 DO_SVE2_ZZZ_TB(SABDLT
, sabdl
, true, true)
6948 DO_SVE2_ZZZ_TB(UADDLT
, uaddl
, true, true)
6949 DO_SVE2_ZZZ_TB(USUBLT
, usubl
, true, true)
6950 DO_SVE2_ZZZ_TB(UABDLT
, uabdl
, true, true)
6952 DO_SVE2_ZZZ_TB(SADDLBT
, saddl
, false, true)
6953 DO_SVE2_ZZZ_TB(SSUBLBT
, ssubl
, false, true)
6954 DO_SVE2_ZZZ_TB(SSUBLTB
, ssubl
, true, false)
6956 DO_SVE2_ZZZ_TB(SQDMULLB_zzz
, sqdmull_zzz
, false, false)
6957 DO_SVE2_ZZZ_TB(SQDMULLT_zzz
, sqdmull_zzz
, true, true)
6959 DO_SVE2_ZZZ_TB(SMULLB_zzz
, smull_zzz
, false, false)
6960 DO_SVE2_ZZZ_TB(SMULLT_zzz
, smull_zzz
, true, true)
6962 DO_SVE2_ZZZ_TB(UMULLB_zzz
, umull_zzz
, false, false)
6963 DO_SVE2_ZZZ_TB(UMULLT_zzz
, umull_zzz
, true, true)
6965 static bool do_eor_tb(DisasContext
*s
, arg_rrr_esz
*a
, bool sel1
)
6967 static gen_helper_gvec_3
* const fns
[4] = {
6968 gen_helper_sve2_eoril_b
, gen_helper_sve2_eoril_h
,
6969 gen_helper_sve2_eoril_s
, gen_helper_sve2_eoril_d
,
6971 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], (!sel1
<< 1) | sel1
);
6974 static bool trans_EORBT(DisasContext
*s
, arg_rrr_esz
*a
)
6976 return do_eor_tb(s
, a
, false);
6979 static bool trans_EORTB(DisasContext
*s
, arg_rrr_esz
*a
)
6981 return do_eor_tb(s
, a
, true);
6984 static bool do_trans_pmull(DisasContext
*s
, arg_rrr_esz
*a
, bool sel
)
6986 static gen_helper_gvec_3
* const fns
[4] = {
6987 gen_helper_gvec_pmull_q
, gen_helper_sve2_pmull_h
,
6988 NULL
, gen_helper_sve2_pmull_d
,
6990 if (a
->esz
== 0 && !dc_isar_feature(aa64_sve2_pmull128
, s
)) {
6993 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], sel
);
6996 static bool trans_PMULLB(DisasContext
*s
, arg_rrr_esz
*a
)
6998 return do_trans_pmull(s
, a
, false);
7001 static bool trans_PMULLT(DisasContext
*s
, arg_rrr_esz
*a
)
7003 return do_trans_pmull(s
, a
, true);
7006 #define DO_SVE2_ZZZ_WTB(NAME, name, SEL2) \
7007 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
7009 static gen_helper_gvec_3 * const fns[4] = { \
7010 NULL, gen_helper_sve2_##name##_h, \
7011 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
7013 return do_sve2_zzw_ool(s, a, fns[a->esz], SEL2); \
7016 DO_SVE2_ZZZ_WTB(SADDWB
, saddw
, false)
7017 DO_SVE2_ZZZ_WTB(SADDWT
, saddw
, true)
7018 DO_SVE2_ZZZ_WTB(SSUBWB
, ssubw
, false)
7019 DO_SVE2_ZZZ_WTB(SSUBWT
, ssubw
, true)
7021 DO_SVE2_ZZZ_WTB(UADDWB
, uaddw
, false)
7022 DO_SVE2_ZZZ_WTB(UADDWT
, uaddw
, true)
7023 DO_SVE2_ZZZ_WTB(USUBWB
, usubw
, false)
7024 DO_SVE2_ZZZ_WTB(USUBWT
, usubw
, true)
7026 static void gen_sshll_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t imm
)
7030 int halfbits
= 4 << vece
;
7033 if (shl
== halfbits
) {
7034 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7035 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(halfbits
, halfbits
));
7036 tcg_gen_and_vec(vece
, d
, n
, t
);
7037 tcg_temp_free_vec(t
);
7039 tcg_gen_sari_vec(vece
, d
, n
, halfbits
);
7040 tcg_gen_shli_vec(vece
, d
, d
, shl
);
7043 tcg_gen_shli_vec(vece
, d
, n
, halfbits
);
7044 tcg_gen_sari_vec(vece
, d
, d
, halfbits
- shl
);
7048 static void gen_ushll_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int imm
)
7050 int halfbits
= 4 << vece
;
7052 int shl
= (imm
>> 1);
7056 mask
= MAKE_64BIT_MASK(0, halfbits
);
7058 mask
= dup_const(vece
, mask
);
7060 shift
= shl
- top
* halfbits
;
7062 tcg_gen_shri_i64(d
, n
, -shift
);
7064 tcg_gen_shli_i64(d
, n
, shift
);
7066 tcg_gen_andi_i64(d
, d
, mask
);
7069 static void gen_ushll16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
7071 gen_ushll_i64(MO_16
, d
, n
, imm
);
7074 static void gen_ushll32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
7076 gen_ushll_i64(MO_32
, d
, n
, imm
);
7079 static void gen_ushll64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
7081 gen_ushll_i64(MO_64
, d
, n
, imm
);
7084 static void gen_ushll_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t imm
)
7086 int halfbits
= 4 << vece
;
7091 if (shl
== halfbits
) {
7092 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7093 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(halfbits
, halfbits
));
7094 tcg_gen_and_vec(vece
, d
, n
, t
);
7095 tcg_temp_free_vec(t
);
7097 tcg_gen_shri_vec(vece
, d
, n
, halfbits
);
7098 tcg_gen_shli_vec(vece
, d
, d
, shl
);
7102 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7103 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7104 tcg_gen_and_vec(vece
, d
, n
, t
);
7105 tcg_temp_free_vec(t
);
7107 tcg_gen_shli_vec(vece
, d
, n
, halfbits
);
7108 tcg_gen_shri_vec(vece
, d
, d
, halfbits
- shl
);
7113 static bool do_sve2_shll_tb(DisasContext
*s
, arg_rri_esz
*a
,
7116 static const TCGOpcode sshll_list
[] = {
7117 INDEX_op_shli_vec
, INDEX_op_sari_vec
, 0
7119 static const TCGOpcode ushll_list
[] = {
7120 INDEX_op_shli_vec
, INDEX_op_shri_vec
, 0
7122 static const GVecGen2i ops
[2][3] = {
7123 { { .fniv
= gen_sshll_vec
,
7124 .opt_opc
= sshll_list
,
7125 .fno
= gen_helper_sve2_sshll_h
,
7127 { .fniv
= gen_sshll_vec
,
7128 .opt_opc
= sshll_list
,
7129 .fno
= gen_helper_sve2_sshll_s
,
7131 { .fniv
= gen_sshll_vec
,
7132 .opt_opc
= sshll_list
,
7133 .fno
= gen_helper_sve2_sshll_d
,
7135 { { .fni8
= gen_ushll16_i64
,
7136 .fniv
= gen_ushll_vec
,
7137 .opt_opc
= ushll_list
,
7138 .fno
= gen_helper_sve2_ushll_h
,
7140 { .fni8
= gen_ushll32_i64
,
7141 .fniv
= gen_ushll_vec
,
7142 .opt_opc
= ushll_list
,
7143 .fno
= gen_helper_sve2_ushll_s
,
7145 { .fni8
= gen_ushll64_i64
,
7146 .fniv
= gen_ushll_vec
,
7147 .opt_opc
= ushll_list
,
7148 .fno
= gen_helper_sve2_ushll_d
,
7152 if (a
->esz
< 0 || a
->esz
> 2 || !dc_isar_feature(aa64_sve2
, s
)) {
7155 if (sve_access_check(s
)) {
7156 unsigned vsz
= vec_full_reg_size(s
);
7157 tcg_gen_gvec_2i(vec_full_reg_offset(s
, a
->rd
),
7158 vec_full_reg_offset(s
, a
->rn
),
7159 vsz
, vsz
, (a
->imm
<< 1) | sel
,
7165 static bool trans_SSHLLB(DisasContext
*s
, arg_rri_esz
*a
)
7167 return do_sve2_shll_tb(s
, a
, false, false);
7170 static bool trans_SSHLLT(DisasContext
*s
, arg_rri_esz
*a
)
7172 return do_sve2_shll_tb(s
, a
, true, false);
7175 static bool trans_USHLLB(DisasContext
*s
, arg_rri_esz
*a
)
7177 return do_sve2_shll_tb(s
, a
, false, true);
7180 static bool trans_USHLLT(DisasContext
*s
, arg_rri_esz
*a
)
7182 return do_sve2_shll_tb(s
, a
, true, true);
7185 static bool trans_BEXT(DisasContext
*s
, arg_rrr_esz
*a
)
7187 static gen_helper_gvec_3
* const fns
[4] = {
7188 gen_helper_sve2_bext_b
, gen_helper_sve2_bext_h
,
7189 gen_helper_sve2_bext_s
, gen_helper_sve2_bext_d
,
7191 if (!dc_isar_feature(aa64_sve2_bitperm
, s
)) {
7194 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], 0);
7197 static bool trans_BDEP(DisasContext
*s
, arg_rrr_esz
*a
)
7199 static gen_helper_gvec_3
* const fns
[4] = {
7200 gen_helper_sve2_bdep_b
, gen_helper_sve2_bdep_h
,
7201 gen_helper_sve2_bdep_s
, gen_helper_sve2_bdep_d
,
7203 if (!dc_isar_feature(aa64_sve2_bitperm
, s
)) {
7206 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], 0);
7209 static bool trans_BGRP(DisasContext
*s
, arg_rrr_esz
*a
)
7211 static gen_helper_gvec_3
* const fns
[4] = {
7212 gen_helper_sve2_bgrp_b
, gen_helper_sve2_bgrp_h
,
7213 gen_helper_sve2_bgrp_s
, gen_helper_sve2_bgrp_d
,
7215 if (!dc_isar_feature(aa64_sve2_bitperm
, s
)) {
7218 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], 0);
7221 static bool do_cadd(DisasContext
*s
, arg_rrr_esz
*a
, bool sq
, bool rot
)
7223 static gen_helper_gvec_3
* const fns
[2][4] = {
7224 { gen_helper_sve2_cadd_b
, gen_helper_sve2_cadd_h
,
7225 gen_helper_sve2_cadd_s
, gen_helper_sve2_cadd_d
},
7226 { gen_helper_sve2_sqcadd_b
, gen_helper_sve2_sqcadd_h
,
7227 gen_helper_sve2_sqcadd_s
, gen_helper_sve2_sqcadd_d
},
7229 return do_sve2_zzw_ool(s
, a
, fns
[sq
][a
->esz
], rot
);
7232 static bool trans_CADD_rot90(DisasContext
*s
, arg_rrr_esz
*a
)
7234 return do_cadd(s
, a
, false, false);
7237 static bool trans_CADD_rot270(DisasContext
*s
, arg_rrr_esz
*a
)
7239 return do_cadd(s
, a
, false, true);
7242 static bool trans_SQCADD_rot90(DisasContext
*s
, arg_rrr_esz
*a
)
7244 return do_cadd(s
, a
, true, false);
7247 static bool trans_SQCADD_rot270(DisasContext
*s
, arg_rrr_esz
*a
)
7249 return do_cadd(s
, a
, true, true);
7252 static bool do_sve2_zzzz_ool(DisasContext
*s
, arg_rrrr_esz
*a
,
7253 gen_helper_gvec_4
*fn
, int data
)
7255 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
7258 if (sve_access_check(s
)) {
7259 gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, data
);
7264 static bool do_abal(DisasContext
*s
, arg_rrrr_esz
*a
, bool uns
, bool sel
)
7266 static gen_helper_gvec_4
* const fns
[2][4] = {
7267 { NULL
, gen_helper_sve2_sabal_h
,
7268 gen_helper_sve2_sabal_s
, gen_helper_sve2_sabal_d
},
7269 { NULL
, gen_helper_sve2_uabal_h
,
7270 gen_helper_sve2_uabal_s
, gen_helper_sve2_uabal_d
},
7272 return do_sve2_zzzz_ool(s
, a
, fns
[uns
][a
->esz
], sel
);
7275 static bool trans_SABALB(DisasContext
*s
, arg_rrrr_esz
*a
)
7277 return do_abal(s
, a
, false, false);
7280 static bool trans_SABALT(DisasContext
*s
, arg_rrrr_esz
*a
)
7282 return do_abal(s
, a
, false, true);
7285 static bool trans_UABALB(DisasContext
*s
, arg_rrrr_esz
*a
)
7287 return do_abal(s
, a
, true, false);
7290 static bool trans_UABALT(DisasContext
*s
, arg_rrrr_esz
*a
)
7292 return do_abal(s
, a
, true, true);
7295 static bool do_adcl(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
7297 static gen_helper_gvec_4
* const fns
[2] = {
7298 gen_helper_sve2_adcl_s
,
7299 gen_helper_sve2_adcl_d
,
7302 * Note that in this case the ESZ field encodes both size and sign.
7303 * Split out 'subtract' into bit 1 of the data field for the helper.
7305 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
& 1], (a
->esz
& 2) | sel
);
7308 static bool trans_ADCLB(DisasContext
*s
, arg_rrrr_esz
*a
)
7310 return do_adcl(s
, a
, false);
7313 static bool trans_ADCLT(DisasContext
*s
, arg_rrrr_esz
*a
)
7315 return do_adcl(s
, a
, true);
7318 static bool do_sve2_fn2i(DisasContext
*s
, arg_rri_esz
*a
, GVecGen2iFn
*fn
)
7320 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
7323 if (sve_access_check(s
)) {
7324 unsigned vsz
= vec_full_reg_size(s
);
7325 unsigned rd_ofs
= vec_full_reg_offset(s
, a
->rd
);
7326 unsigned rn_ofs
= vec_full_reg_offset(s
, a
->rn
);
7327 fn(a
->esz
, rd_ofs
, rn_ofs
, a
->imm
, vsz
, vsz
);
7332 static bool trans_SSRA(DisasContext
*s
, arg_rri_esz
*a
)
7334 return do_sve2_fn2i(s
, a
, gen_gvec_ssra
);
7337 static bool trans_USRA(DisasContext
*s
, arg_rri_esz
*a
)
7339 return do_sve2_fn2i(s
, a
, gen_gvec_usra
);
7342 static bool trans_SRSRA(DisasContext
*s
, arg_rri_esz
*a
)
7344 return do_sve2_fn2i(s
, a
, gen_gvec_srsra
);
7347 static bool trans_URSRA(DisasContext
*s
, arg_rri_esz
*a
)
7349 return do_sve2_fn2i(s
, a
, gen_gvec_ursra
);
7352 static bool trans_SRI(DisasContext
*s
, arg_rri_esz
*a
)
7354 return do_sve2_fn2i(s
, a
, gen_gvec_sri
);
7357 static bool trans_SLI(DisasContext
*s
, arg_rri_esz
*a
)
7359 return do_sve2_fn2i(s
, a
, gen_gvec_sli
);
7362 static bool do_sve2_fn_zzz(DisasContext
*s
, arg_rrr_esz
*a
, GVecGen3Fn
*fn
)
7364 if (!dc_isar_feature(aa64_sve2
, s
)) {
7367 if (sve_access_check(s
)) {
7368 gen_gvec_fn_zzz(s
, fn
, a
->esz
, a
->rd
, a
->rn
, a
->rm
);
7373 static bool trans_SABA(DisasContext
*s
, arg_rrr_esz
*a
)
7375 return do_sve2_fn_zzz(s
, a
, gen_gvec_saba
);
7378 static bool trans_UABA(DisasContext
*s
, arg_rrr_esz
*a
)
7380 return do_sve2_fn_zzz(s
, a
, gen_gvec_uaba
);
7383 static bool do_sve2_narrow_extract(DisasContext
*s
, arg_rri_esz
*a
,
7384 const GVecGen2 ops
[3])
7386 if (a
->esz
< 0 || a
->esz
> MO_32
|| a
->imm
!= 0 ||
7387 !dc_isar_feature(aa64_sve2
, s
)) {
7390 if (sve_access_check(s
)) {
7391 unsigned vsz
= vec_full_reg_size(s
);
7392 tcg_gen_gvec_2(vec_full_reg_offset(s
, a
->rd
),
7393 vec_full_reg_offset(s
, a
->rn
),
7394 vsz
, vsz
, &ops
[a
->esz
]);
7399 static const TCGOpcode sqxtn_list
[] = {
7400 INDEX_op_shli_vec
, INDEX_op_smin_vec
, INDEX_op_smax_vec
, 0
7403 static void gen_sqxtnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7405 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7406 int halfbits
= 4 << vece
;
7407 int64_t mask
= (1ull << halfbits
) - 1;
7408 int64_t min
= -1ull << (halfbits
- 1);
7409 int64_t max
= -min
- 1;
7411 tcg_gen_dupi_vec(vece
, t
, min
);
7412 tcg_gen_smax_vec(vece
, d
, n
, t
);
7413 tcg_gen_dupi_vec(vece
, t
, max
);
7414 tcg_gen_smin_vec(vece
, d
, d
, t
);
7415 tcg_gen_dupi_vec(vece
, t
, mask
);
7416 tcg_gen_and_vec(vece
, d
, d
, t
);
7417 tcg_temp_free_vec(t
);
7420 static bool trans_SQXTNB(DisasContext
*s
, arg_rri_esz
*a
)
7422 static const GVecGen2 ops
[3] = {
7423 { .fniv
= gen_sqxtnb_vec
,
7424 .opt_opc
= sqxtn_list
,
7425 .fno
= gen_helper_sve2_sqxtnb_h
,
7427 { .fniv
= gen_sqxtnb_vec
,
7428 .opt_opc
= sqxtn_list
,
7429 .fno
= gen_helper_sve2_sqxtnb_s
,
7431 { .fniv
= gen_sqxtnb_vec
,
7432 .opt_opc
= sqxtn_list
,
7433 .fno
= gen_helper_sve2_sqxtnb_d
,
7436 return do_sve2_narrow_extract(s
, a
, ops
);
7439 static void gen_sqxtnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7441 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7442 int halfbits
= 4 << vece
;
7443 int64_t mask
= (1ull << halfbits
) - 1;
7444 int64_t min
= -1ull << (halfbits
- 1);
7445 int64_t max
= -min
- 1;
7447 tcg_gen_dupi_vec(vece
, t
, min
);
7448 tcg_gen_smax_vec(vece
, n
, n
, t
);
7449 tcg_gen_dupi_vec(vece
, t
, max
);
7450 tcg_gen_smin_vec(vece
, n
, n
, t
);
7451 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7452 tcg_gen_dupi_vec(vece
, t
, mask
);
7453 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7454 tcg_temp_free_vec(t
);
7457 static bool trans_SQXTNT(DisasContext
*s
, arg_rri_esz
*a
)
7459 static const GVecGen2 ops
[3] = {
7460 { .fniv
= gen_sqxtnt_vec
,
7461 .opt_opc
= sqxtn_list
,
7463 .fno
= gen_helper_sve2_sqxtnt_h
,
7465 { .fniv
= gen_sqxtnt_vec
,
7466 .opt_opc
= sqxtn_list
,
7468 .fno
= gen_helper_sve2_sqxtnt_s
,
7470 { .fniv
= gen_sqxtnt_vec
,
7471 .opt_opc
= sqxtn_list
,
7473 .fno
= gen_helper_sve2_sqxtnt_d
,
7476 return do_sve2_narrow_extract(s
, a
, ops
);
7479 static const TCGOpcode uqxtn_list
[] = {
7480 INDEX_op_shli_vec
, INDEX_op_umin_vec
, 0
7483 static void gen_uqxtnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7485 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7486 int halfbits
= 4 << vece
;
7487 int64_t max
= (1ull << halfbits
) - 1;
7489 tcg_gen_dupi_vec(vece
, t
, max
);
7490 tcg_gen_umin_vec(vece
, d
, n
, t
);
7491 tcg_temp_free_vec(t
);
7494 static bool trans_UQXTNB(DisasContext
*s
, arg_rri_esz
*a
)
7496 static const GVecGen2 ops
[3] = {
7497 { .fniv
= gen_uqxtnb_vec
,
7498 .opt_opc
= uqxtn_list
,
7499 .fno
= gen_helper_sve2_uqxtnb_h
,
7501 { .fniv
= gen_uqxtnb_vec
,
7502 .opt_opc
= uqxtn_list
,
7503 .fno
= gen_helper_sve2_uqxtnb_s
,
7505 { .fniv
= gen_uqxtnb_vec
,
7506 .opt_opc
= uqxtn_list
,
7507 .fno
= gen_helper_sve2_uqxtnb_d
,
7510 return do_sve2_narrow_extract(s
, a
, ops
);
7513 static void gen_uqxtnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7515 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7516 int halfbits
= 4 << vece
;
7517 int64_t max
= (1ull << halfbits
) - 1;
7519 tcg_gen_dupi_vec(vece
, t
, max
);
7520 tcg_gen_umin_vec(vece
, n
, n
, t
);
7521 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7522 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7523 tcg_temp_free_vec(t
);
7526 static bool trans_UQXTNT(DisasContext
*s
, arg_rri_esz
*a
)
7528 static const GVecGen2 ops
[3] = {
7529 { .fniv
= gen_uqxtnt_vec
,
7530 .opt_opc
= uqxtn_list
,
7532 .fno
= gen_helper_sve2_uqxtnt_h
,
7534 { .fniv
= gen_uqxtnt_vec
,
7535 .opt_opc
= uqxtn_list
,
7537 .fno
= gen_helper_sve2_uqxtnt_s
,
7539 { .fniv
= gen_uqxtnt_vec
,
7540 .opt_opc
= uqxtn_list
,
7542 .fno
= gen_helper_sve2_uqxtnt_d
,
7545 return do_sve2_narrow_extract(s
, a
, ops
);
7548 static const TCGOpcode sqxtun_list
[] = {
7549 INDEX_op_shli_vec
, INDEX_op_umin_vec
, INDEX_op_smax_vec
, 0
7552 static void gen_sqxtunb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7554 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7555 int halfbits
= 4 << vece
;
7556 int64_t max
= (1ull << halfbits
) - 1;
7558 tcg_gen_dupi_vec(vece
, t
, 0);
7559 tcg_gen_smax_vec(vece
, d
, n
, t
);
7560 tcg_gen_dupi_vec(vece
, t
, max
);
7561 tcg_gen_umin_vec(vece
, d
, d
, t
);
7562 tcg_temp_free_vec(t
);
7565 static bool trans_SQXTUNB(DisasContext
*s
, arg_rri_esz
*a
)
7567 static const GVecGen2 ops
[3] = {
7568 { .fniv
= gen_sqxtunb_vec
,
7569 .opt_opc
= sqxtun_list
,
7570 .fno
= gen_helper_sve2_sqxtunb_h
,
7572 { .fniv
= gen_sqxtunb_vec
,
7573 .opt_opc
= sqxtun_list
,
7574 .fno
= gen_helper_sve2_sqxtunb_s
,
7576 { .fniv
= gen_sqxtunb_vec
,
7577 .opt_opc
= sqxtun_list
,
7578 .fno
= gen_helper_sve2_sqxtunb_d
,
7581 return do_sve2_narrow_extract(s
, a
, ops
);
7584 static void gen_sqxtunt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7586 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7587 int halfbits
= 4 << vece
;
7588 int64_t max
= (1ull << halfbits
) - 1;
7590 tcg_gen_dupi_vec(vece
, t
, 0);
7591 tcg_gen_smax_vec(vece
, n
, n
, t
);
7592 tcg_gen_dupi_vec(vece
, t
, max
);
7593 tcg_gen_umin_vec(vece
, n
, n
, t
);
7594 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7595 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7596 tcg_temp_free_vec(t
);
7599 static bool trans_SQXTUNT(DisasContext
*s
, arg_rri_esz
*a
)
7601 static const GVecGen2 ops
[3] = {
7602 { .fniv
= gen_sqxtunt_vec
,
7603 .opt_opc
= sqxtun_list
,
7605 .fno
= gen_helper_sve2_sqxtunt_h
,
7607 { .fniv
= gen_sqxtunt_vec
,
7608 .opt_opc
= sqxtun_list
,
7610 .fno
= gen_helper_sve2_sqxtunt_s
,
7612 { .fniv
= gen_sqxtunt_vec
,
7613 .opt_opc
= sqxtun_list
,
7615 .fno
= gen_helper_sve2_sqxtunt_d
,
7618 return do_sve2_narrow_extract(s
, a
, ops
);
7621 static bool do_sve2_shr_narrow(DisasContext
*s
, arg_rri_esz
*a
,
7622 const GVecGen2i ops
[3])
7624 if (a
->esz
< 0 || a
->esz
> MO_32
|| !dc_isar_feature(aa64_sve2
, s
)) {
7627 assert(a
->imm
> 0 && a
->imm
<= (8 << a
->esz
));
7628 if (sve_access_check(s
)) {
7629 unsigned vsz
= vec_full_reg_size(s
);
7630 tcg_gen_gvec_2i(vec_full_reg_offset(s
, a
->rd
),
7631 vec_full_reg_offset(s
, a
->rn
),
7632 vsz
, vsz
, a
->imm
, &ops
[a
->esz
]);
7637 static void gen_shrnb_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int shr
)
7639 int halfbits
= 4 << vece
;
7640 uint64_t mask
= dup_const(vece
, MAKE_64BIT_MASK(0, halfbits
));
7642 tcg_gen_shri_i64(d
, n
, shr
);
7643 tcg_gen_andi_i64(d
, d
, mask
);
7646 static void gen_shrnb16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7648 gen_shrnb_i64(MO_16
, d
, n
, shr
);
7651 static void gen_shrnb32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7653 gen_shrnb_i64(MO_32
, d
, n
, shr
);
7656 static void gen_shrnb64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7658 gen_shrnb_i64(MO_64
, d
, n
, shr
);
7661 static void gen_shrnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t shr
)
7663 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7664 int halfbits
= 4 << vece
;
7665 uint64_t mask
= MAKE_64BIT_MASK(0, halfbits
);
7667 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7668 tcg_gen_dupi_vec(vece
, t
, mask
);
7669 tcg_gen_and_vec(vece
, d
, n
, t
);
7670 tcg_temp_free_vec(t
);
7673 static bool trans_SHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7675 static const TCGOpcode vec_list
[] = { INDEX_op_shri_vec
, 0 };
7676 static const GVecGen2i ops
[3] = {
7677 { .fni8
= gen_shrnb16_i64
,
7678 .fniv
= gen_shrnb_vec
,
7679 .opt_opc
= vec_list
,
7680 .fno
= gen_helper_sve2_shrnb_h
,
7682 { .fni8
= gen_shrnb32_i64
,
7683 .fniv
= gen_shrnb_vec
,
7684 .opt_opc
= vec_list
,
7685 .fno
= gen_helper_sve2_shrnb_s
,
7687 { .fni8
= gen_shrnb64_i64
,
7688 .fniv
= gen_shrnb_vec
,
7689 .opt_opc
= vec_list
,
7690 .fno
= gen_helper_sve2_shrnb_d
,
7693 return do_sve2_shr_narrow(s
, a
, ops
);
7696 static void gen_shrnt_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int shr
)
7698 int halfbits
= 4 << vece
;
7699 uint64_t mask
= dup_const(vece
, MAKE_64BIT_MASK(0, halfbits
));
7701 tcg_gen_shli_i64(n
, n
, halfbits
- shr
);
7702 tcg_gen_andi_i64(n
, n
, ~mask
);
7703 tcg_gen_andi_i64(d
, d
, mask
);
7704 tcg_gen_or_i64(d
, d
, n
);
7707 static void gen_shrnt16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7709 gen_shrnt_i64(MO_16
, d
, n
, shr
);
7712 static void gen_shrnt32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7714 gen_shrnt_i64(MO_32
, d
, n
, shr
);
7717 static void gen_shrnt64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7719 tcg_gen_shri_i64(n
, n
, shr
);
7720 tcg_gen_deposit_i64(d
, d
, n
, 32, 32);
7723 static void gen_shrnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t shr
)
7725 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7726 int halfbits
= 4 << vece
;
7727 uint64_t mask
= MAKE_64BIT_MASK(0, halfbits
);
7729 tcg_gen_shli_vec(vece
, n
, n
, halfbits
- shr
);
7730 tcg_gen_dupi_vec(vece
, t
, mask
);
7731 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7732 tcg_temp_free_vec(t
);
7735 static bool trans_SHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7737 static const TCGOpcode vec_list
[] = { INDEX_op_shli_vec
, 0 };
7738 static const GVecGen2i ops
[3] = {
7739 { .fni8
= gen_shrnt16_i64
,
7740 .fniv
= gen_shrnt_vec
,
7741 .opt_opc
= vec_list
,
7743 .fno
= gen_helper_sve2_shrnt_h
,
7745 { .fni8
= gen_shrnt32_i64
,
7746 .fniv
= gen_shrnt_vec
,
7747 .opt_opc
= vec_list
,
7749 .fno
= gen_helper_sve2_shrnt_s
,
7751 { .fni8
= gen_shrnt64_i64
,
7752 .fniv
= gen_shrnt_vec
,
7753 .opt_opc
= vec_list
,
7755 .fno
= gen_helper_sve2_shrnt_d
,
7758 return do_sve2_shr_narrow(s
, a
, ops
);
7761 static bool trans_RSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7763 static const GVecGen2i ops
[3] = {
7764 { .fno
= gen_helper_sve2_rshrnb_h
},
7765 { .fno
= gen_helper_sve2_rshrnb_s
},
7766 { .fno
= gen_helper_sve2_rshrnb_d
},
7768 return do_sve2_shr_narrow(s
, a
, ops
);
7771 static bool trans_RSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7773 static const GVecGen2i ops
[3] = {
7774 { .fno
= gen_helper_sve2_rshrnt_h
},
7775 { .fno
= gen_helper_sve2_rshrnt_s
},
7776 { .fno
= gen_helper_sve2_rshrnt_d
},
7778 return do_sve2_shr_narrow(s
, a
, ops
);
7781 static void gen_sqshrunb_vec(unsigned vece
, TCGv_vec d
,
7782 TCGv_vec n
, int64_t shr
)
7784 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7785 int halfbits
= 4 << vece
;
7787 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7788 tcg_gen_dupi_vec(vece
, t
, 0);
7789 tcg_gen_smax_vec(vece
, n
, n
, t
);
7790 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7791 tcg_gen_umin_vec(vece
, d
, n
, t
);
7792 tcg_temp_free_vec(t
);
7795 static bool trans_SQSHRUNB(DisasContext
*s
, arg_rri_esz
*a
)
7797 static const TCGOpcode vec_list
[] = {
7798 INDEX_op_sari_vec
, INDEX_op_smax_vec
, INDEX_op_umin_vec
, 0
7800 static const GVecGen2i ops
[3] = {
7801 { .fniv
= gen_sqshrunb_vec
,
7802 .opt_opc
= vec_list
,
7803 .fno
= gen_helper_sve2_sqshrunb_h
,
7805 { .fniv
= gen_sqshrunb_vec
,
7806 .opt_opc
= vec_list
,
7807 .fno
= gen_helper_sve2_sqshrunb_s
,
7809 { .fniv
= gen_sqshrunb_vec
,
7810 .opt_opc
= vec_list
,
7811 .fno
= gen_helper_sve2_sqshrunb_d
,
7814 return do_sve2_shr_narrow(s
, a
, ops
);
7817 static void gen_sqshrunt_vec(unsigned vece
, TCGv_vec d
,
7818 TCGv_vec n
, int64_t shr
)
7820 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7821 int halfbits
= 4 << vece
;
7823 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7824 tcg_gen_dupi_vec(vece
, t
, 0);
7825 tcg_gen_smax_vec(vece
, n
, n
, t
);
7826 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7827 tcg_gen_umin_vec(vece
, n
, n
, t
);
7828 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7829 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7830 tcg_temp_free_vec(t
);
7833 static bool trans_SQSHRUNT(DisasContext
*s
, arg_rri_esz
*a
)
7835 static const TCGOpcode vec_list
[] = {
7836 INDEX_op_shli_vec
, INDEX_op_sari_vec
,
7837 INDEX_op_smax_vec
, INDEX_op_umin_vec
, 0
7839 static const GVecGen2i ops
[3] = {
7840 { .fniv
= gen_sqshrunt_vec
,
7841 .opt_opc
= vec_list
,
7843 .fno
= gen_helper_sve2_sqshrunt_h
,
7845 { .fniv
= gen_sqshrunt_vec
,
7846 .opt_opc
= vec_list
,
7848 .fno
= gen_helper_sve2_sqshrunt_s
,
7850 { .fniv
= gen_sqshrunt_vec
,
7851 .opt_opc
= vec_list
,
7853 .fno
= gen_helper_sve2_sqshrunt_d
,
7856 return do_sve2_shr_narrow(s
, a
, ops
);
7859 static bool trans_SQRSHRUNB(DisasContext
*s
, arg_rri_esz
*a
)
7861 static const GVecGen2i ops
[3] = {
7862 { .fno
= gen_helper_sve2_sqrshrunb_h
},
7863 { .fno
= gen_helper_sve2_sqrshrunb_s
},
7864 { .fno
= gen_helper_sve2_sqrshrunb_d
},
7866 return do_sve2_shr_narrow(s
, a
, ops
);
7869 static bool trans_SQRSHRUNT(DisasContext
*s
, arg_rri_esz
*a
)
7871 static const GVecGen2i ops
[3] = {
7872 { .fno
= gen_helper_sve2_sqrshrunt_h
},
7873 { .fno
= gen_helper_sve2_sqrshrunt_s
},
7874 { .fno
= gen_helper_sve2_sqrshrunt_d
},
7876 return do_sve2_shr_narrow(s
, a
, ops
);
7879 static void gen_sqshrnb_vec(unsigned vece
, TCGv_vec d
,
7880 TCGv_vec n
, int64_t shr
)
7882 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7883 int halfbits
= 4 << vece
;
7884 int64_t max
= MAKE_64BIT_MASK(0, halfbits
- 1);
7885 int64_t min
= -max
- 1;
7887 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7888 tcg_gen_dupi_vec(vece
, t
, min
);
7889 tcg_gen_smax_vec(vece
, n
, n
, t
);
7890 tcg_gen_dupi_vec(vece
, t
, max
);
7891 tcg_gen_smin_vec(vece
, n
, n
, t
);
7892 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7893 tcg_gen_and_vec(vece
, d
, n
, t
);
7894 tcg_temp_free_vec(t
);
7897 static bool trans_SQSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7899 static const TCGOpcode vec_list
[] = {
7900 INDEX_op_sari_vec
, INDEX_op_smax_vec
, INDEX_op_smin_vec
, 0
7902 static const GVecGen2i ops
[3] = {
7903 { .fniv
= gen_sqshrnb_vec
,
7904 .opt_opc
= vec_list
,
7905 .fno
= gen_helper_sve2_sqshrnb_h
,
7907 { .fniv
= gen_sqshrnb_vec
,
7908 .opt_opc
= vec_list
,
7909 .fno
= gen_helper_sve2_sqshrnb_s
,
7911 { .fniv
= gen_sqshrnb_vec
,
7912 .opt_opc
= vec_list
,
7913 .fno
= gen_helper_sve2_sqshrnb_d
,
7916 return do_sve2_shr_narrow(s
, a
, ops
);
7919 static void gen_sqshrnt_vec(unsigned vece
, TCGv_vec d
,
7920 TCGv_vec n
, int64_t shr
)
7922 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7923 int halfbits
= 4 << vece
;
7924 int64_t max
= MAKE_64BIT_MASK(0, halfbits
- 1);
7925 int64_t min
= -max
- 1;
7927 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7928 tcg_gen_dupi_vec(vece
, t
, min
);
7929 tcg_gen_smax_vec(vece
, n
, n
, t
);
7930 tcg_gen_dupi_vec(vece
, t
, max
);
7931 tcg_gen_smin_vec(vece
, n
, n
, t
);
7932 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7933 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7934 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7935 tcg_temp_free_vec(t
);
7938 static bool trans_SQSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7940 static const TCGOpcode vec_list
[] = {
7941 INDEX_op_shli_vec
, INDEX_op_sari_vec
,
7942 INDEX_op_smax_vec
, INDEX_op_smin_vec
, 0
7944 static const GVecGen2i ops
[3] = {
7945 { .fniv
= gen_sqshrnt_vec
,
7946 .opt_opc
= vec_list
,
7948 .fno
= gen_helper_sve2_sqshrnt_h
,
7950 { .fniv
= gen_sqshrnt_vec
,
7951 .opt_opc
= vec_list
,
7953 .fno
= gen_helper_sve2_sqshrnt_s
,
7955 { .fniv
= gen_sqshrnt_vec
,
7956 .opt_opc
= vec_list
,
7958 .fno
= gen_helper_sve2_sqshrnt_d
,
7961 return do_sve2_shr_narrow(s
, a
, ops
);
7964 static bool trans_SQRSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7966 static const GVecGen2i ops
[3] = {
7967 { .fno
= gen_helper_sve2_sqrshrnb_h
},
7968 { .fno
= gen_helper_sve2_sqrshrnb_s
},
7969 { .fno
= gen_helper_sve2_sqrshrnb_d
},
7971 return do_sve2_shr_narrow(s
, a
, ops
);
7974 static bool trans_SQRSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7976 static const GVecGen2i ops
[3] = {
7977 { .fno
= gen_helper_sve2_sqrshrnt_h
},
7978 { .fno
= gen_helper_sve2_sqrshrnt_s
},
7979 { .fno
= gen_helper_sve2_sqrshrnt_d
},
7981 return do_sve2_shr_narrow(s
, a
, ops
);
7984 static void gen_uqshrnb_vec(unsigned vece
, TCGv_vec d
,
7985 TCGv_vec n
, int64_t shr
)
7987 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7988 int halfbits
= 4 << vece
;
7990 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7991 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7992 tcg_gen_umin_vec(vece
, d
, n
, t
);
7993 tcg_temp_free_vec(t
);
7996 static bool trans_UQSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7998 static const TCGOpcode vec_list
[] = {
7999 INDEX_op_shri_vec
, INDEX_op_umin_vec
, 0
8001 static const GVecGen2i ops
[3] = {
8002 { .fniv
= gen_uqshrnb_vec
,
8003 .opt_opc
= vec_list
,
8004 .fno
= gen_helper_sve2_uqshrnb_h
,
8006 { .fniv
= gen_uqshrnb_vec
,
8007 .opt_opc
= vec_list
,
8008 .fno
= gen_helper_sve2_uqshrnb_s
,
8010 { .fniv
= gen_uqshrnb_vec
,
8011 .opt_opc
= vec_list
,
8012 .fno
= gen_helper_sve2_uqshrnb_d
,
8015 return do_sve2_shr_narrow(s
, a
, ops
);
8018 static void gen_uqshrnt_vec(unsigned vece
, TCGv_vec d
,
8019 TCGv_vec n
, int64_t shr
)
8021 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
8022 int halfbits
= 4 << vece
;
8024 tcg_gen_shri_vec(vece
, n
, n
, shr
);
8025 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
8026 tcg_gen_umin_vec(vece
, n
, n
, t
);
8027 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
8028 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
8029 tcg_temp_free_vec(t
);
8032 static bool trans_UQSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
8034 static const TCGOpcode vec_list
[] = {
8035 INDEX_op_shli_vec
, INDEX_op_shri_vec
, INDEX_op_umin_vec
, 0
8037 static const GVecGen2i ops
[3] = {
8038 { .fniv
= gen_uqshrnt_vec
,
8039 .opt_opc
= vec_list
,
8041 .fno
= gen_helper_sve2_uqshrnt_h
,
8043 { .fniv
= gen_uqshrnt_vec
,
8044 .opt_opc
= vec_list
,
8046 .fno
= gen_helper_sve2_uqshrnt_s
,
8048 { .fniv
= gen_uqshrnt_vec
,
8049 .opt_opc
= vec_list
,
8051 .fno
= gen_helper_sve2_uqshrnt_d
,
8054 return do_sve2_shr_narrow(s
, a
, ops
);
8057 static bool trans_UQRSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
8059 static const GVecGen2i ops
[3] = {
8060 { .fno
= gen_helper_sve2_uqrshrnb_h
},
8061 { .fno
= gen_helper_sve2_uqrshrnb_s
},
8062 { .fno
= gen_helper_sve2_uqrshrnb_d
},
8064 return do_sve2_shr_narrow(s
, a
, ops
);
8067 static bool trans_UQRSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
8069 static const GVecGen2i ops
[3] = {
8070 { .fno
= gen_helper_sve2_uqrshrnt_h
},
8071 { .fno
= gen_helper_sve2_uqrshrnt_s
},
8072 { .fno
= gen_helper_sve2_uqrshrnt_d
},
8074 return do_sve2_shr_narrow(s
, a
, ops
);
8077 #define DO_SVE2_ZZZ_NARROW(NAME, name) \
8078 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
8080 static gen_helper_gvec_3 * const fns[4] = { \
8081 NULL, gen_helper_sve2_##name##_h, \
8082 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
8084 return do_sve2_zzz_ool(s, a, fns[a->esz]); \
8087 DO_SVE2_ZZZ_NARROW(ADDHNB
, addhnb
)
8088 DO_SVE2_ZZZ_NARROW(ADDHNT
, addhnt
)
8089 DO_SVE2_ZZZ_NARROW(RADDHNB
, raddhnb
)
8090 DO_SVE2_ZZZ_NARROW(RADDHNT
, raddhnt
)
8092 DO_SVE2_ZZZ_NARROW(SUBHNB
, subhnb
)
8093 DO_SVE2_ZZZ_NARROW(SUBHNT
, subhnt
)
8094 DO_SVE2_ZZZ_NARROW(RSUBHNB
, rsubhnb
)
8095 DO_SVE2_ZZZ_NARROW(RSUBHNT
, rsubhnt
)
8097 static bool do_sve2_ppzz_flags(DisasContext
*s
, arg_rprr_esz
*a
,
8098 gen_helper_gvec_flags_4
*fn
)
8100 if (!dc_isar_feature(aa64_sve2
, s
)) {
8103 return do_ppzz_flags(s
, a
, fn
);
8106 #define DO_SVE2_PPZZ_MATCH(NAME, name) \
8107 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
8109 static gen_helper_gvec_flags_4 * const fns[4] = { \
8110 gen_helper_sve2_##name##_ppzz_b, gen_helper_sve2_##name##_ppzz_h, \
8113 return do_sve2_ppzz_flags(s, a, fns[a->esz]); \
8116 DO_SVE2_PPZZ_MATCH(MATCH
, match
)
8117 DO_SVE2_PPZZ_MATCH(NMATCH
, nmatch
)
8119 static bool trans_HISTCNT(DisasContext
*s
, arg_rprr_esz
*a
)
8121 static gen_helper_gvec_4
* const fns
[2] = {
8122 gen_helper_sve2_histcnt_s
, gen_helper_sve2_histcnt_d
8127 return do_sve2_zpzz_ool(s
, a
, fns
[a
->esz
- 2]);
8130 static bool trans_HISTSEG(DisasContext
*s
, arg_rrr_esz
*a
)
8135 return do_sve2_zzz_ool(s
, a
, gen_helper_sve2_histseg
);
8138 static bool do_sve2_zpzz_fp(DisasContext
*s
, arg_rprr_esz
*a
,
8139 gen_helper_gvec_4_ptr
*fn
)
8141 if (!dc_isar_feature(aa64_sve2
, s
)) {
8144 return do_zpzz_fp(s
, a
, fn
);
8147 #define DO_SVE2_ZPZZ_FP(NAME, name) \
8148 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
8150 static gen_helper_gvec_4_ptr * const fns[4] = { \
8151 NULL, gen_helper_sve2_##name##_zpzz_h, \
8152 gen_helper_sve2_##name##_zpzz_s, gen_helper_sve2_##name##_zpzz_d \
8154 return do_sve2_zpzz_fp(s, a, fns[a->esz]); \
8157 DO_SVE2_ZPZZ_FP(FADDP
, faddp
)
8158 DO_SVE2_ZPZZ_FP(FMAXNMP
, fmaxnmp
)
8159 DO_SVE2_ZPZZ_FP(FMINNMP
, fminnmp
)
8160 DO_SVE2_ZPZZ_FP(FMAXP
, fmaxp
)
8161 DO_SVE2_ZPZZ_FP(FMINP
, fminp
)
8164 * SVE Integer Multiply-Add (unpredicated)
8167 static bool trans_FMMLA(DisasContext
*s
, arg_rrrr_esz
*a
)
8169 gen_helper_gvec_4_ptr
*fn
;
8173 if (!dc_isar_feature(aa64_sve_f32mm
, s
)) {
8176 fn
= gen_helper_fmmla_s
;
8179 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
8182 fn
= gen_helper_fmmla_d
;
8188 if (sve_access_check(s
)) {
8189 unsigned vsz
= vec_full_reg_size(s
);
8190 TCGv_ptr status
= fpstatus_ptr(FPST_FPCR
);
8191 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
8192 vec_full_reg_offset(s
, a
->rn
),
8193 vec_full_reg_offset(s
, a
->rm
),
8194 vec_full_reg_offset(s
, a
->ra
),
8195 status
, vsz
, vsz
, 0, fn
);
8196 tcg_temp_free_ptr(status
);
8201 static bool do_sqdmlal_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
,
8202 bool sel1
, bool sel2
)
8204 static gen_helper_gvec_4
* const fns
[] = {
8205 NULL
, gen_helper_sve2_sqdmlal_zzzw_h
,
8206 gen_helper_sve2_sqdmlal_zzzw_s
, gen_helper_sve2_sqdmlal_zzzw_d
,
8208 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], (sel2
<< 1) | sel1
);
8211 static bool do_sqdmlsl_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
,
8212 bool sel1
, bool sel2
)
8214 static gen_helper_gvec_4
* const fns
[] = {
8215 NULL
, gen_helper_sve2_sqdmlsl_zzzw_h
,
8216 gen_helper_sve2_sqdmlsl_zzzw_s
, gen_helper_sve2_sqdmlsl_zzzw_d
,
8218 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], (sel2
<< 1) | sel1
);
8221 static bool trans_SQDMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8223 return do_sqdmlal_zzzw(s
, a
, false, false);
8226 static bool trans_SQDMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8228 return do_sqdmlal_zzzw(s
, a
, true, true);
8231 static bool trans_SQDMLALBT(DisasContext
*s
, arg_rrrr_esz
*a
)
8233 return do_sqdmlal_zzzw(s
, a
, false, true);
8236 static bool trans_SQDMLSLB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8238 return do_sqdmlsl_zzzw(s
, a
, false, false);
8241 static bool trans_SQDMLSLT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8243 return do_sqdmlsl_zzzw(s
, a
, true, true);
8246 static bool trans_SQDMLSLBT(DisasContext
*s
, arg_rrrr_esz
*a
)
8248 return do_sqdmlsl_zzzw(s
, a
, false, true);
8251 static bool trans_SQRDMLAH_zzzz(DisasContext
*s
, arg_rrrr_esz
*a
)
8253 static gen_helper_gvec_4
* const fns
[] = {
8254 gen_helper_sve2_sqrdmlah_b
, gen_helper_sve2_sqrdmlah_h
,
8255 gen_helper_sve2_sqrdmlah_s
, gen_helper_sve2_sqrdmlah_d
,
8257 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], 0);
8260 static bool trans_SQRDMLSH_zzzz(DisasContext
*s
, arg_rrrr_esz
*a
)
8262 static gen_helper_gvec_4
* const fns
[] = {
8263 gen_helper_sve2_sqrdmlsh_b
, gen_helper_sve2_sqrdmlsh_h
,
8264 gen_helper_sve2_sqrdmlsh_s
, gen_helper_sve2_sqrdmlsh_d
,
8266 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], 0);
8269 static bool do_smlal_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
8271 static gen_helper_gvec_4
* const fns
[] = {
8272 NULL
, gen_helper_sve2_smlal_zzzw_h
,
8273 gen_helper_sve2_smlal_zzzw_s
, gen_helper_sve2_smlal_zzzw_d
,
8275 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], sel
);
8278 static bool trans_SMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8280 return do_smlal_zzzw(s
, a
, false);
8283 static bool trans_SMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8285 return do_smlal_zzzw(s
, a
, true);
8288 static bool do_umlal_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
8290 static gen_helper_gvec_4
* const fns
[] = {
8291 NULL
, gen_helper_sve2_umlal_zzzw_h
,
8292 gen_helper_sve2_umlal_zzzw_s
, gen_helper_sve2_umlal_zzzw_d
,
8294 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], sel
);
8297 static bool trans_UMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8299 return do_umlal_zzzw(s
, a
, false);
8302 static bool trans_UMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8304 return do_umlal_zzzw(s
, a
, true);
8307 static bool do_smlsl_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
8309 static gen_helper_gvec_4
* const fns
[] = {
8310 NULL
, gen_helper_sve2_smlsl_zzzw_h
,
8311 gen_helper_sve2_smlsl_zzzw_s
, gen_helper_sve2_smlsl_zzzw_d
,
8313 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], sel
);
8316 static bool trans_SMLSLB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8318 return do_smlsl_zzzw(s
, a
, false);
8321 static bool trans_SMLSLT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8323 return do_smlsl_zzzw(s
, a
, true);
8326 static bool do_umlsl_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
8328 static gen_helper_gvec_4
* const fns
[] = {
8329 NULL
, gen_helper_sve2_umlsl_zzzw_h
,
8330 gen_helper_sve2_umlsl_zzzw_s
, gen_helper_sve2_umlsl_zzzw_d
,
8332 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], sel
);
8335 static bool trans_UMLSLB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8337 return do_umlsl_zzzw(s
, a
, false);
8340 static bool trans_UMLSLT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8342 return do_umlsl_zzzw(s
, a
, true);
8345 static bool trans_CMLA_zzzz(DisasContext
*s
, arg_CMLA_zzzz
*a
)
8347 static gen_helper_gvec_4
* const fns
[] = {
8348 gen_helper_sve2_cmla_zzzz_b
, gen_helper_sve2_cmla_zzzz_h
,
8349 gen_helper_sve2_cmla_zzzz_s
, gen_helper_sve2_cmla_zzzz_d
,
8352 if (!dc_isar_feature(aa64_sve2
, s
)) {
8355 if (sve_access_check(s
)) {
8356 gen_gvec_ool_zzzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
);
8361 static bool trans_CDOT_zzzz(DisasContext
*s
, arg_CMLA_zzzz
*a
)
8363 if (!dc_isar_feature(aa64_sve2
, s
) || a
->esz
< MO_32
) {
8366 if (sve_access_check(s
)) {
8367 gen_helper_gvec_4
*fn
= (a
->esz
== MO_32
8368 ? gen_helper_sve2_cdot_zzzz_s
8369 : gen_helper_sve2_cdot_zzzz_d
);
8370 gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
);
8375 static bool trans_SQRDCMLAH_zzzz(DisasContext
*s
, arg_SQRDCMLAH_zzzz
*a
)
8377 static gen_helper_gvec_4
* const fns
[] = {
8378 gen_helper_sve2_sqrdcmlah_zzzz_b
, gen_helper_sve2_sqrdcmlah_zzzz_h
,
8379 gen_helper_sve2_sqrdcmlah_zzzz_s
, gen_helper_sve2_sqrdcmlah_zzzz_d
,
8382 if (!dc_isar_feature(aa64_sve2
, s
)) {
8385 if (sve_access_check(s
)) {
8386 gen_gvec_ool_zzzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
);
8391 static bool trans_USDOT_zzzz(DisasContext
*s
, arg_USDOT_zzzz
*a
)
8393 if (a
->esz
!= 2 || !dc_isar_feature(aa64_sve_i8mm
, s
)) {
8396 if (sve_access_check(s
)) {
8397 unsigned vsz
= vec_full_reg_size(s
);
8398 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, a
->rd
),
8399 vec_full_reg_offset(s
, a
->rn
),
8400 vec_full_reg_offset(s
, a
->rm
),
8401 vec_full_reg_offset(s
, a
->ra
),
8402 vsz
, vsz
, 0, gen_helper_gvec_usdot_b
);
8407 static bool trans_AESMC(DisasContext
*s
, arg_AESMC
*a
)
8409 if (!dc_isar_feature(aa64_sve2_aes
, s
)) {
8412 if (sve_access_check(s
)) {
8413 gen_gvec_ool_zz(s
, gen_helper_crypto_aesmc
, a
->rd
, a
->rd
, a
->decrypt
);
8418 static bool do_aese(DisasContext
*s
, arg_rrr_esz
*a
, bool decrypt
)
8420 if (!dc_isar_feature(aa64_sve2_aes
, s
)) {
8423 if (sve_access_check(s
)) {
8424 gen_gvec_ool_zzz(s
, gen_helper_crypto_aese
,
8425 a
->rd
, a
->rn
, a
->rm
, decrypt
);
8430 static bool trans_AESE(DisasContext
*s
, arg_rrr_esz
*a
)
8432 return do_aese(s
, a
, false);
8435 static bool trans_AESD(DisasContext
*s
, arg_rrr_esz
*a
)
8437 return do_aese(s
, a
, true);
8440 static bool do_sm4(DisasContext
*s
, arg_rrr_esz
*a
, gen_helper_gvec_3
*fn
)
8442 if (!dc_isar_feature(aa64_sve2_sm4
, s
)) {
8445 if (sve_access_check(s
)) {
8446 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, 0);
8451 static bool trans_SM4E(DisasContext
*s
, arg_rrr_esz
*a
)
8453 return do_sm4(s
, a
, gen_helper_crypto_sm4e
);
8456 static bool trans_SM4EKEY(DisasContext
*s
, arg_rrr_esz
*a
)
8458 return do_sm4(s
, a
, gen_helper_crypto_sm4ekey
);
8461 static bool trans_RAX1(DisasContext
*s
, arg_rrr_esz
*a
)
8463 if (!dc_isar_feature(aa64_sve2_sha3
, s
)) {
8466 if (sve_access_check(s
)) {
8467 gen_gvec_fn_zzz(s
, gen_gvec_rax1
, MO_64
, a
->rd
, a
->rn
, a
->rm
);
8472 static bool trans_FCVTNT_sh(DisasContext
*s
, arg_rpr_esz
*a
)
8474 if (!dc_isar_feature(aa64_sve2
, s
)) {
8477 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtnt_sh
);
8480 static bool trans_BFCVTNT(DisasContext
*s
, arg_rpr_esz
*a
)
8482 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
8485 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_bfcvtnt
);
8488 static bool trans_FCVTNT_ds(DisasContext
*s
, arg_rpr_esz
*a
)
8490 if (!dc_isar_feature(aa64_sve2
, s
)) {
8493 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtnt_ds
);
8496 static bool trans_FCVTLT_hs(DisasContext
*s
, arg_rpr_esz
*a
)
8498 if (!dc_isar_feature(aa64_sve2
, s
)) {
8501 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtlt_hs
);
8504 static bool trans_FCVTLT_sd(DisasContext
*s
, arg_rpr_esz
*a
)
8506 if (!dc_isar_feature(aa64_sve2
, s
)) {
8509 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtlt_sd
);
8512 static bool trans_FCVTX_ds(DisasContext
*s
, arg_rpr_esz
*a
)
8514 if (!dc_isar_feature(aa64_sve2
, s
)) {
8517 return do_frint_mode(s
, a
, float_round_to_odd
, gen_helper_sve_fcvt_ds
);
8520 static bool trans_FCVTXNT_ds(DisasContext
*s
, arg_rpr_esz
*a
)
8522 if (!dc_isar_feature(aa64_sve2
, s
)) {
8525 return do_frint_mode(s
, a
, float_round_to_odd
, gen_helper_sve2_fcvtnt_ds
);
8528 static bool trans_FLOGB(DisasContext
*s
, arg_rpr_esz
*a
)
8530 static gen_helper_gvec_3_ptr
* const fns
[] = {
8531 NULL
, gen_helper_flogb_h
,
8532 gen_helper_flogb_s
, gen_helper_flogb_d
8535 if (!dc_isar_feature(aa64_sve2
, s
) || fns
[a
->esz
] == NULL
) {
8538 if (sve_access_check(s
)) {
8540 fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
8541 unsigned vsz
= vec_full_reg_size(s
);
8543 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
8544 vec_full_reg_offset(s
, a
->rn
),
8545 pred_full_reg_offset(s
, a
->pg
),
8546 status
, vsz
, vsz
, 0, fns
[a
->esz
]);
8547 tcg_temp_free_ptr(status
);
8552 static bool do_FMLAL_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sub
, bool sel
)
8554 if (!dc_isar_feature(aa64_sve2
, s
)) {
8557 if (sve_access_check(s
)) {
8558 unsigned vsz
= vec_full_reg_size(s
);
8559 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
8560 vec_full_reg_offset(s
, a
->rn
),
8561 vec_full_reg_offset(s
, a
->rm
),
8562 vec_full_reg_offset(s
, a
->ra
),
8563 cpu_env
, vsz
, vsz
, (sel
<< 1) | sub
,
8564 gen_helper_sve2_fmlal_zzzw_s
);
8569 static bool trans_FMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8571 return do_FMLAL_zzzw(s
, a
, false, false);
8574 static bool trans_FMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8576 return do_FMLAL_zzzw(s
, a
, false, true);
8579 static bool trans_FMLSLB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8581 return do_FMLAL_zzzw(s
, a
, true, false);
8584 static bool trans_FMLSLT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8586 return do_FMLAL_zzzw(s
, a
, true, true);
8589 static bool do_FMLAL_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
, bool sub
, bool sel
)
8591 if (!dc_isar_feature(aa64_sve2
, s
)) {
8594 if (sve_access_check(s
)) {
8595 unsigned vsz
= vec_full_reg_size(s
);
8596 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
8597 vec_full_reg_offset(s
, a
->rn
),
8598 vec_full_reg_offset(s
, a
->rm
),
8599 vec_full_reg_offset(s
, a
->ra
),
8601 (a
->index
<< 2) | (sel
<< 1) | sub
,
8602 gen_helper_sve2_fmlal_zzxw_s
);
8607 static bool trans_FMLALB_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
8609 return do_FMLAL_zzxw(s
, a
, false, false);
8612 static bool trans_FMLALT_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
8614 return do_FMLAL_zzxw(s
, a
, false, true);
8617 static bool trans_FMLSLB_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
8619 return do_FMLAL_zzxw(s
, a
, true, false);
8622 static bool trans_FMLSLT_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
8624 return do_FMLAL_zzxw(s
, a
, true, true);
8627 static bool do_i8mm_zzzz_ool(DisasContext
*s
, arg_rrrr_esz
*a
,
8628 gen_helper_gvec_4
*fn
, int data
)
8630 if (!dc_isar_feature(aa64_sve_i8mm
, s
)) {
8633 if (sve_access_check(s
)) {
8634 gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, data
);
8639 static bool trans_SMMLA(DisasContext
*s
, arg_rrrr_esz
*a
)
8641 return do_i8mm_zzzz_ool(s
, a
, gen_helper_gvec_smmla_b
, 0);
8644 static bool trans_USMMLA(DisasContext
*s
, arg_rrrr_esz
*a
)
8646 return do_i8mm_zzzz_ool(s
, a
, gen_helper_gvec_usmmla_b
, 0);
8649 static bool trans_UMMLA(DisasContext
*s
, arg_rrrr_esz
*a
)
8651 return do_i8mm_zzzz_ool(s
, a
, gen_helper_gvec_ummla_b
, 0);
8654 static bool trans_BFDOT_zzzz(DisasContext
*s
, arg_rrrr_esz
*a
)
8656 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
8659 if (sve_access_check(s
)) {
8660 gen_gvec_ool_zzzz(s
, gen_helper_gvec_bfdot
,
8661 a
->rd
, a
->rn
, a
->rm
, a
->ra
, 0);
8666 static bool trans_BFDOT_zzxz(DisasContext
*s
, arg_rrxr_esz
*a
)
8668 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
8671 if (sve_access_check(s
)) {
8672 gen_gvec_ool_zzzz(s
, gen_helper_gvec_bfdot_idx
,
8673 a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->index
);
8678 static bool trans_BFMMLA(DisasContext
*s
, arg_rrrr_esz
*a
)
8680 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
8683 if (sve_access_check(s
)) {
8684 gen_gvec_ool_zzzz(s
, gen_helper_gvec_bfmmla
,
8685 a
->rd
, a
->rn
, a
->rm
, a
->ra
, 0);
8690 static bool do_BFMLAL_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
8692 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
8695 if (sve_access_check(s
)) {
8696 TCGv_ptr status
= fpstatus_ptr(FPST_FPCR
);
8697 unsigned vsz
= vec_full_reg_size(s
);
8699 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
8700 vec_full_reg_offset(s
, a
->rn
),
8701 vec_full_reg_offset(s
, a
->rm
),
8702 vec_full_reg_offset(s
, a
->ra
),
8703 status
, vsz
, vsz
, sel
,
8704 gen_helper_gvec_bfmlal
);
8705 tcg_temp_free_ptr(status
);
8710 static bool trans_BFMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8712 return do_BFMLAL_zzzw(s
, a
, false);
8715 static bool trans_BFMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8717 return do_BFMLAL_zzzw(s
, a
, true);
8720 static bool do_BFMLAL_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
, bool sel
)
8722 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
8725 if (sve_access_check(s
)) {
8726 TCGv_ptr status
= fpstatus_ptr(FPST_FPCR
);
8727 unsigned vsz
= vec_full_reg_size(s
);
8729 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
8730 vec_full_reg_offset(s
, a
->rn
),
8731 vec_full_reg_offset(s
, a
->rm
),
8732 vec_full_reg_offset(s
, a
->ra
),
8733 status
, vsz
, vsz
, (a
->index
<< 1) | sel
,
8734 gen_helper_gvec_bfmlal_idx
);
8735 tcg_temp_free_ptr(status
);
8740 static bool trans_BFMLALB_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
8742 return do_BFMLAL_zzxw(s
, a
, false);
8745 static bool trans_BFMLALT_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
8747 return do_BFMLAL_zzxw(s
, a
, true);