2 * AArch64 SVE translation
4 * Copyright (c) 2018 Linaro, Ltd
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
22 #include "exec/exec-all.h"
23 #include "tcg/tcg-op.h"
24 #include "tcg/tcg-op-gvec.h"
25 #include "tcg/tcg-gvec-desc.h"
28 #include "translate.h"
29 #include "internals.h"
30 #include "exec/helper-proto.h"
31 #include "exec/helper-gen.h"
33 #include "translate-a64.h"
34 #include "fpu/softfloat.h"
37 typedef void GVecGen2sFn(unsigned, uint32_t, uint32_t,
38 TCGv_i64
, uint32_t, uint32_t);
40 typedef void gen_helper_gvec_flags_3(TCGv_i32
, TCGv_ptr
, TCGv_ptr
,
42 typedef void gen_helper_gvec_flags_4(TCGv_i32
, TCGv_ptr
, TCGv_ptr
,
43 TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
45 typedef void gen_helper_gvec_mem(TCGv_env
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
46 typedef void gen_helper_gvec_mem_scatter(TCGv_env
, TCGv_ptr
, TCGv_ptr
,
47 TCGv_ptr
, TCGv_i64
, TCGv_i32
);
50 * Helpers for extracting complex instruction fields.
53 /* See e.g. ASR (immediate, predicated).
54 * Returns -1 for unallocated encoding; diagnose later.
56 static int tszimm_esz(DisasContext
*s
, int x
)
58 x
>>= 3; /* discard imm3 */
62 static int tszimm_shr(DisasContext
*s
, int x
)
64 return (16 << tszimm_esz(s
, x
)) - x
;
67 /* See e.g. LSL (immediate, predicated). */
68 static int tszimm_shl(DisasContext
*s
, int x
)
70 return x
- (8 << tszimm_esz(s
, x
));
73 /* The SH bit is in bit 8. Extract the low 8 and shift. */
74 static inline int expand_imm_sh8s(DisasContext
*s
, int x
)
76 return (int8_t)x
<< (x
& 0x100 ? 8 : 0);
79 static inline int expand_imm_sh8u(DisasContext
*s
, int x
)
81 return (uint8_t)x
<< (x
& 0x100 ? 8 : 0);
84 /* Convert a 2-bit memory size (msz) to a 4-bit data type (dtype)
85 * with unsigned data. C.f. SVE Memory Contiguous Load Group.
87 static inline int msz_dtype(DisasContext
*s
, int msz
)
89 static const uint8_t dtype
[4] = { 0, 5, 10, 15 };
94 * Include the generated decoder.
97 #include "decode-sve.c.inc"
100 * Implement all of the translator functions referenced by the decoder.
103 /* Return the offset info CPUARMState of the predicate vector register Pn.
104 * Note for this purpose, FFR is P16.
106 static inline int pred_full_reg_offset(DisasContext
*s
, int regno
)
108 return offsetof(CPUARMState
, vfp
.pregs
[regno
]);
111 /* Return the byte size of the whole predicate register, VL / 64. */
112 static inline int pred_full_reg_size(DisasContext
*s
)
114 return s
->sve_len
>> 3;
117 /* Round up the size of a register to a size allowed by
118 * the tcg vector infrastructure. Any operation which uses this
119 * size may assume that the bits above pred_full_reg_size are zero,
120 * and must leave them the same way.
122 * Note that this is not needed for the vector registers as they
123 * are always properly sized for tcg vectors.
125 static int size_for_gvec(int size
)
130 return QEMU_ALIGN_UP(size
, 16);
134 static int pred_gvec_reg_size(DisasContext
*s
)
136 return size_for_gvec(pred_full_reg_size(s
));
139 /* Invoke an out-of-line helper on 2 Zregs. */
140 static bool gen_gvec_ool_zz(DisasContext
*s
, gen_helper_gvec_2
*fn
,
141 int rd
, int rn
, int data
)
146 if (sve_access_check(s
)) {
147 unsigned vsz
= vec_full_reg_size(s
);
148 tcg_gen_gvec_2_ool(vec_full_reg_offset(s
, rd
),
149 vec_full_reg_offset(s
, rn
),
155 /* Invoke an out-of-line helper on 3 Zregs. */
156 static bool gen_gvec_ool_zzz(DisasContext
*s
, gen_helper_gvec_3
*fn
,
157 int rd
, int rn
, int rm
, int data
)
162 if (sve_access_check(s
)) {
163 unsigned vsz
= vec_full_reg_size(s
);
164 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
165 vec_full_reg_offset(s
, rn
),
166 vec_full_reg_offset(s
, rm
),
172 static bool gen_gvec_ool_arg_zzz(DisasContext
*s
, gen_helper_gvec_3
*fn
,
173 arg_rrr_esz
*a
, int data
)
175 return gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, data
);
178 /* Invoke an out-of-line helper on 4 Zregs. */
179 static bool gen_gvec_ool_zzzz(DisasContext
*s
, gen_helper_gvec_4
*fn
,
180 int rd
, int rn
, int rm
, int ra
, int data
)
185 if (sve_access_check(s
)) {
186 unsigned vsz
= vec_full_reg_size(s
);
187 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
188 vec_full_reg_offset(s
, rn
),
189 vec_full_reg_offset(s
, rm
),
190 vec_full_reg_offset(s
, ra
),
196 static bool gen_gvec_ool_arg_zzzz(DisasContext
*s
, gen_helper_gvec_4
*fn
,
197 arg_rrrr_esz
*a
, int data
)
199 return gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, data
);
202 static bool gen_gvec_ool_arg_zzxz(DisasContext
*s
, gen_helper_gvec_4
*fn
,
205 return gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->index
);
208 /* Invoke an out-of-line helper on 2 Zregs and a predicate. */
209 static bool gen_gvec_ool_zzp(DisasContext
*s
, gen_helper_gvec_3
*fn
,
210 int rd
, int rn
, int pg
, int data
)
215 if (sve_access_check(s
)) {
216 unsigned vsz
= vec_full_reg_size(s
);
217 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
218 vec_full_reg_offset(s
, rn
),
219 pred_full_reg_offset(s
, pg
),
225 static bool gen_gvec_ool_arg_zpz(DisasContext
*s
, gen_helper_gvec_3
*fn
,
226 arg_rpr_esz
*a
, int data
)
228 return gen_gvec_ool_zzp(s
, fn
, a
->rd
, a
->rn
, a
->pg
, data
);
231 static bool gen_gvec_ool_arg_zpzi(DisasContext
*s
, gen_helper_gvec_3
*fn
,
234 return gen_gvec_ool_zzp(s
, fn
, a
->rd
, a
->rn
, a
->pg
, a
->imm
);
237 /* Invoke an out-of-line helper on 3 Zregs and a predicate. */
238 static bool gen_gvec_ool_zzzp(DisasContext
*s
, gen_helper_gvec_4
*fn
,
239 int rd
, int rn
, int rm
, int pg
, int data
)
244 if (sve_access_check(s
)) {
245 unsigned vsz
= vec_full_reg_size(s
);
246 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
247 vec_full_reg_offset(s
, rn
),
248 vec_full_reg_offset(s
, rm
),
249 pred_full_reg_offset(s
, pg
),
255 static bool gen_gvec_ool_arg_zpzz(DisasContext
*s
, gen_helper_gvec_4
*fn
,
256 arg_rprr_esz
*a
, int data
)
258 return gen_gvec_ool_zzzp(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->pg
, data
);
261 /* Invoke a vector expander on two Zregs and an immediate. */
262 static bool gen_gvec_fn_zzi(DisasContext
*s
, GVecGen2iFn
*gvec_fn
,
263 int esz
, int rd
, int rn
, uint64_t imm
)
265 if (gvec_fn
== NULL
) {
268 if (sve_access_check(s
)) {
269 unsigned vsz
= vec_full_reg_size(s
);
270 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
271 vec_full_reg_offset(s
, rn
), imm
, vsz
, vsz
);
276 static bool gen_gvec_fn_arg_zzi(DisasContext
*s
, GVecGen2iFn
*gvec_fn
,
280 /* Invalid tsz encoding -- see tszimm_esz. */
283 return gen_gvec_fn_zzi(s
, gvec_fn
, a
->esz
, a
->rd
, a
->rn
, a
->imm
);
286 /* Invoke a vector expander on three Zregs. */
287 static bool gen_gvec_fn_zzz(DisasContext
*s
, GVecGen3Fn
*gvec_fn
,
288 int esz
, int rd
, int rn
, int rm
)
290 if (gvec_fn
== NULL
) {
293 if (sve_access_check(s
)) {
294 unsigned vsz
= vec_full_reg_size(s
);
295 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
296 vec_full_reg_offset(s
, rn
),
297 vec_full_reg_offset(s
, rm
), vsz
, vsz
);
302 static bool gen_gvec_fn_arg_zzz(DisasContext
*s
, GVecGen3Fn
*fn
,
305 return gen_gvec_fn_zzz(s
, fn
, a
->esz
, a
->rd
, a
->rn
, a
->rm
);
308 /* Invoke a vector expander on four Zregs. */
309 static bool gen_gvec_fn_arg_zzzz(DisasContext
*s
, GVecGen4Fn
*gvec_fn
,
312 if (gvec_fn
== NULL
) {
315 if (sve_access_check(s
)) {
316 unsigned vsz
= vec_full_reg_size(s
);
317 gvec_fn(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
318 vec_full_reg_offset(s
, a
->rn
),
319 vec_full_reg_offset(s
, a
->rm
),
320 vec_full_reg_offset(s
, a
->ra
), vsz
, vsz
);
325 /* Invoke a vector move on two Zregs. */
326 static bool do_mov_z(DisasContext
*s
, int rd
, int rn
)
328 if (sve_access_check(s
)) {
329 unsigned vsz
= vec_full_reg_size(s
);
330 tcg_gen_gvec_mov(MO_8
, vec_full_reg_offset(s
, rd
),
331 vec_full_reg_offset(s
, rn
), vsz
, vsz
);
336 /* Initialize a Zreg with replications of a 64-bit immediate. */
337 static void do_dupi_z(DisasContext
*s
, int rd
, uint64_t word
)
339 unsigned vsz
= vec_full_reg_size(s
);
340 tcg_gen_gvec_dup_imm(MO_64
, vec_full_reg_offset(s
, rd
), vsz
, vsz
, word
);
343 /* Invoke a vector expander on three Pregs. */
344 static void gen_gvec_fn_ppp(DisasContext
*s
, GVecGen3Fn
*gvec_fn
,
345 int rd
, int rn
, int rm
)
347 unsigned psz
= pred_gvec_reg_size(s
);
348 gvec_fn(MO_64
, pred_full_reg_offset(s
, rd
),
349 pred_full_reg_offset(s
, rn
),
350 pred_full_reg_offset(s
, rm
), psz
, psz
);
353 /* Invoke a vector move on two Pregs. */
354 static bool do_mov_p(DisasContext
*s
, int rd
, int rn
)
356 if (sve_access_check(s
)) {
357 unsigned psz
= pred_gvec_reg_size(s
);
358 tcg_gen_gvec_mov(MO_8
, pred_full_reg_offset(s
, rd
),
359 pred_full_reg_offset(s
, rn
), psz
, psz
);
364 /* Set the cpu flags as per a return from an SVE helper. */
365 static void do_pred_flags(TCGv_i32 t
)
367 tcg_gen_mov_i32(cpu_NF
, t
);
368 tcg_gen_andi_i32(cpu_ZF
, t
, 2);
369 tcg_gen_andi_i32(cpu_CF
, t
, 1);
370 tcg_gen_movi_i32(cpu_VF
, 0);
373 /* Subroutines computing the ARM PredTest psuedofunction. */
374 static void do_predtest1(TCGv_i64 d
, TCGv_i64 g
)
376 TCGv_i32 t
= tcg_temp_new_i32();
378 gen_helper_sve_predtest1(t
, d
, g
);
380 tcg_temp_free_i32(t
);
383 static void do_predtest(DisasContext
*s
, int dofs
, int gofs
, int words
)
385 TCGv_ptr dptr
= tcg_temp_new_ptr();
386 TCGv_ptr gptr
= tcg_temp_new_ptr();
387 TCGv_i32 t
= tcg_temp_new_i32();
389 tcg_gen_addi_ptr(dptr
, cpu_env
, dofs
);
390 tcg_gen_addi_ptr(gptr
, cpu_env
, gofs
);
392 gen_helper_sve_predtest(t
, dptr
, gptr
, tcg_constant_i32(words
));
393 tcg_temp_free_ptr(dptr
);
394 tcg_temp_free_ptr(gptr
);
397 tcg_temp_free_i32(t
);
400 /* For each element size, the bits within a predicate word that are active. */
401 const uint64_t pred_esz_masks
[4] = {
402 0xffffffffffffffffull
, 0x5555555555555555ull
,
403 0x1111111111111111ull
, 0x0101010101010101ull
407 *** SVE Logical - Unpredicated Group
410 TRANS_FEAT(AND_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_and
, a
)
411 TRANS_FEAT(ORR_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_or
, a
)
412 TRANS_FEAT(EOR_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_xor
, a
)
413 TRANS_FEAT(BIC_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_andc
, a
)
415 static void gen_xar8_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
417 TCGv_i64 t
= tcg_temp_new_i64();
418 uint64_t mask
= dup_const(MO_8
, 0xff >> sh
);
420 tcg_gen_xor_i64(t
, n
, m
);
421 tcg_gen_shri_i64(d
, t
, sh
);
422 tcg_gen_shli_i64(t
, t
, 8 - sh
);
423 tcg_gen_andi_i64(d
, d
, mask
);
424 tcg_gen_andi_i64(t
, t
, ~mask
);
425 tcg_gen_or_i64(d
, d
, t
);
426 tcg_temp_free_i64(t
);
429 static void gen_xar16_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
431 TCGv_i64 t
= tcg_temp_new_i64();
432 uint64_t mask
= dup_const(MO_16
, 0xffff >> sh
);
434 tcg_gen_xor_i64(t
, n
, m
);
435 tcg_gen_shri_i64(d
, t
, sh
);
436 tcg_gen_shli_i64(t
, t
, 16 - sh
);
437 tcg_gen_andi_i64(d
, d
, mask
);
438 tcg_gen_andi_i64(t
, t
, ~mask
);
439 tcg_gen_or_i64(d
, d
, t
);
440 tcg_temp_free_i64(t
);
443 static void gen_xar_i32(TCGv_i32 d
, TCGv_i32 n
, TCGv_i32 m
, int32_t sh
)
445 tcg_gen_xor_i32(d
, n
, m
);
446 tcg_gen_rotri_i32(d
, d
, sh
);
449 static void gen_xar_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
451 tcg_gen_xor_i64(d
, n
, m
);
452 tcg_gen_rotri_i64(d
, d
, sh
);
455 static void gen_xar_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
456 TCGv_vec m
, int64_t sh
)
458 tcg_gen_xor_vec(vece
, d
, n
, m
);
459 tcg_gen_rotri_vec(vece
, d
, d
, sh
);
462 void gen_gvec_xar(unsigned vece
, uint32_t rd_ofs
, uint32_t rn_ofs
,
463 uint32_t rm_ofs
, int64_t shift
,
464 uint32_t opr_sz
, uint32_t max_sz
)
466 static const TCGOpcode vecop
[] = { INDEX_op_rotli_vec
, 0 };
467 static const GVecGen3i ops
[4] = {
468 { .fni8
= gen_xar8_i64
,
470 .fno
= gen_helper_sve2_xar_b
,
473 { .fni8
= gen_xar16_i64
,
475 .fno
= gen_helper_sve2_xar_h
,
478 { .fni4
= gen_xar_i32
,
480 .fno
= gen_helper_sve2_xar_s
,
483 { .fni8
= gen_xar_i64
,
485 .fno
= gen_helper_gvec_xar_d
,
489 int esize
= 8 << vece
;
491 /* The SVE2 range is 1 .. esize; the AdvSIMD range is 0 .. esize-1. */
492 tcg_debug_assert(shift
>= 0);
493 tcg_debug_assert(shift
<= esize
);
497 /* xar with no rotate devolves to xor. */
498 tcg_gen_gvec_xor(vece
, rd_ofs
, rn_ofs
, rm_ofs
, opr_sz
, max_sz
);
500 tcg_gen_gvec_3i(rd_ofs
, rn_ofs
, rm_ofs
, opr_sz
, max_sz
,
505 static bool trans_XAR(DisasContext
*s
, arg_rrri_esz
*a
)
507 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
510 if (sve_access_check(s
)) {
511 unsigned vsz
= vec_full_reg_size(s
);
512 gen_gvec_xar(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
513 vec_full_reg_offset(s
, a
->rn
),
514 vec_full_reg_offset(s
, a
->rm
), a
->imm
, vsz
, vsz
);
519 static void gen_eor3_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
521 tcg_gen_xor_i64(d
, n
, m
);
522 tcg_gen_xor_i64(d
, d
, k
);
525 static void gen_eor3_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
526 TCGv_vec m
, TCGv_vec k
)
528 tcg_gen_xor_vec(vece
, d
, n
, m
);
529 tcg_gen_xor_vec(vece
, d
, d
, k
);
532 static void gen_eor3(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
533 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
535 static const GVecGen4 op
= {
536 .fni8
= gen_eor3_i64
,
537 .fniv
= gen_eor3_vec
,
538 .fno
= gen_helper_sve2_eor3
,
540 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
542 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
545 TRANS_FEAT(EOR3
, aa64_sve2
, gen_gvec_fn_arg_zzzz
, gen_eor3
, a
)
547 static void gen_bcax_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
549 tcg_gen_andc_i64(d
, m
, k
);
550 tcg_gen_xor_i64(d
, d
, n
);
553 static void gen_bcax_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
554 TCGv_vec m
, TCGv_vec k
)
556 tcg_gen_andc_vec(vece
, d
, m
, k
);
557 tcg_gen_xor_vec(vece
, d
, d
, n
);
560 static void gen_bcax(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
561 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
563 static const GVecGen4 op
= {
564 .fni8
= gen_bcax_i64
,
565 .fniv
= gen_bcax_vec
,
566 .fno
= gen_helper_sve2_bcax
,
568 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
570 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
573 TRANS_FEAT(BCAX
, aa64_sve2
, gen_gvec_fn_arg_zzzz
, gen_bcax
, a
)
575 static void gen_bsl(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
576 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
578 /* BSL differs from the generic bitsel in argument ordering. */
579 tcg_gen_gvec_bitsel(vece
, d
, a
, n
, m
, oprsz
, maxsz
);
582 TRANS_FEAT(BSL
, aa64_sve2
, gen_gvec_fn_arg_zzzz
, gen_bsl
, a
)
584 static void gen_bsl1n_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
586 tcg_gen_andc_i64(n
, k
, n
);
587 tcg_gen_andc_i64(m
, m
, k
);
588 tcg_gen_or_i64(d
, n
, m
);
591 static void gen_bsl1n_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
592 TCGv_vec m
, TCGv_vec k
)
594 if (TCG_TARGET_HAS_bitsel_vec
) {
595 tcg_gen_not_vec(vece
, n
, n
);
596 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
598 tcg_gen_andc_vec(vece
, n
, k
, n
);
599 tcg_gen_andc_vec(vece
, m
, m
, k
);
600 tcg_gen_or_vec(vece
, d
, n
, m
);
604 static void gen_bsl1n(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
605 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
607 static const GVecGen4 op
= {
608 .fni8
= gen_bsl1n_i64
,
609 .fniv
= gen_bsl1n_vec
,
610 .fno
= gen_helper_sve2_bsl1n
,
612 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
614 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
617 TRANS_FEAT(BSL1N
, aa64_sve2
, gen_gvec_fn_arg_zzzz
, gen_bsl1n
, a
)
619 static void gen_bsl2n_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
622 * Z[dn] = (n & k) | (~m & ~k)
625 tcg_gen_and_i64(n
, n
, k
);
626 if (TCG_TARGET_HAS_orc_i64
) {
627 tcg_gen_or_i64(m
, m
, k
);
628 tcg_gen_orc_i64(d
, n
, m
);
630 tcg_gen_nor_i64(m
, m
, k
);
631 tcg_gen_or_i64(d
, n
, m
);
635 static void gen_bsl2n_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
636 TCGv_vec m
, TCGv_vec k
)
638 if (TCG_TARGET_HAS_bitsel_vec
) {
639 tcg_gen_not_vec(vece
, m
, m
);
640 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
642 tcg_gen_and_vec(vece
, n
, n
, k
);
643 tcg_gen_or_vec(vece
, m
, m
, k
);
644 tcg_gen_orc_vec(vece
, d
, n
, m
);
648 static void gen_bsl2n(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
649 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
651 static const GVecGen4 op
= {
652 .fni8
= gen_bsl2n_i64
,
653 .fniv
= gen_bsl2n_vec
,
654 .fno
= gen_helper_sve2_bsl2n
,
656 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
658 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
661 TRANS_FEAT(BSL2N
, aa64_sve2
, gen_gvec_fn_arg_zzzz
, gen_bsl2n
, a
)
663 static void gen_nbsl_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
665 tcg_gen_and_i64(n
, n
, k
);
666 tcg_gen_andc_i64(m
, m
, k
);
667 tcg_gen_nor_i64(d
, n
, m
);
670 static void gen_nbsl_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
671 TCGv_vec m
, TCGv_vec k
)
673 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
674 tcg_gen_not_vec(vece
, d
, d
);
677 static void gen_nbsl(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
678 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
680 static const GVecGen4 op
= {
681 .fni8
= gen_nbsl_i64
,
682 .fniv
= gen_nbsl_vec
,
683 .fno
= gen_helper_sve2_nbsl
,
685 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
687 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
690 TRANS_FEAT(NBSL
, aa64_sve2
, gen_gvec_fn_arg_zzzz
, gen_nbsl
, a
)
693 *** SVE Integer Arithmetic - Unpredicated Group
696 TRANS_FEAT(ADD_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_add
, a
)
697 TRANS_FEAT(SUB_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_sub
, a
)
698 TRANS_FEAT(SQADD_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_ssadd
, a
)
699 TRANS_FEAT(SQSUB_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_sssub
, a
)
700 TRANS_FEAT(UQADD_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_usadd
, a
)
701 TRANS_FEAT(UQSUB_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_ussub
, a
)
704 *** SVE Integer Arithmetic - Binary Predicated Group
707 /* Select active elememnts from Zn and inactive elements from Zm,
708 * storing the result in Zd.
710 static bool do_sel_z(DisasContext
*s
, int rd
, int rn
, int rm
, int pg
, int esz
)
712 static gen_helper_gvec_4
* const fns
[4] = {
713 gen_helper_sve_sel_zpzz_b
, gen_helper_sve_sel_zpzz_h
,
714 gen_helper_sve_sel_zpzz_s
, gen_helper_sve_sel_zpzz_d
716 return gen_gvec_ool_zzzp(s
, fns
[esz
], rd
, rn
, rm
, pg
, 0);
719 #define DO_ZPZZ(NAME, FEAT, name) \
720 static gen_helper_gvec_4 * const name##_zpzz_fns[4] = { \
721 gen_helper_##name##_zpzz_b, gen_helper_##name##_zpzz_h, \
722 gen_helper_##name##_zpzz_s, gen_helper_##name##_zpzz_d, \
724 TRANS_FEAT(NAME, FEAT, gen_gvec_ool_arg_zpzz, \
725 name##_zpzz_fns[a->esz], a, 0)
727 DO_ZPZZ(AND_zpzz
, aa64_sve
, sve_and
)
728 DO_ZPZZ(EOR_zpzz
, aa64_sve
, sve_eor
)
729 DO_ZPZZ(ORR_zpzz
, aa64_sve
, sve_orr
)
730 DO_ZPZZ(BIC_zpzz
, aa64_sve
, sve_bic
)
732 DO_ZPZZ(ADD_zpzz
, aa64_sve
, sve_add
)
733 DO_ZPZZ(SUB_zpzz
, aa64_sve
, sve_sub
)
735 DO_ZPZZ(SMAX_zpzz
, aa64_sve
, sve_smax
)
736 DO_ZPZZ(UMAX_zpzz
, aa64_sve
, sve_umax
)
737 DO_ZPZZ(SMIN_zpzz
, aa64_sve
, sve_smin
)
738 DO_ZPZZ(UMIN_zpzz
, aa64_sve
, sve_umin
)
739 DO_ZPZZ(SABD_zpzz
, aa64_sve
, sve_sabd
)
740 DO_ZPZZ(UABD_zpzz
, aa64_sve
, sve_uabd
)
742 DO_ZPZZ(MUL_zpzz
, aa64_sve
, sve_mul
)
743 DO_ZPZZ(SMULH_zpzz
, aa64_sve
, sve_smulh
)
744 DO_ZPZZ(UMULH_zpzz
, aa64_sve
, sve_umulh
)
746 DO_ZPZZ(ASR_zpzz
, aa64_sve
, sve_asr
)
747 DO_ZPZZ(LSR_zpzz
, aa64_sve
, sve_lsr
)
748 DO_ZPZZ(LSL_zpzz
, aa64_sve
, sve_lsl
)
750 static gen_helper_gvec_4
* const sdiv_fns
[4] = {
751 NULL
, NULL
, gen_helper_sve_sdiv_zpzz_s
, gen_helper_sve_sdiv_zpzz_d
753 TRANS_FEAT(SDIV_zpzz
, aa64_sve
, gen_gvec_ool_arg_zpzz
, sdiv_fns
[a
->esz
], a
, 0)
755 static gen_helper_gvec_4
* const udiv_fns
[4] = {
756 NULL
, NULL
, gen_helper_sve_udiv_zpzz_s
, gen_helper_sve_udiv_zpzz_d
758 TRANS_FEAT(UDIV_zpzz
, aa64_sve
, gen_gvec_ool_arg_zpzz
, udiv_fns
[a
->esz
], a
, 0)
760 static bool trans_SEL_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
762 return do_sel_z(s
, a
->rd
, a
->rn
, a
->rm
, a
->pg
, a
->esz
);
766 *** SVE Integer Arithmetic - Unary Predicated Group
769 #define DO_ZPZ(NAME, FEAT, name) \
770 static gen_helper_gvec_3 * const name##_fns[4] = { \
771 gen_helper_##name##_b, gen_helper_##name##_h, \
772 gen_helper_##name##_s, gen_helper_##name##_d, \
774 TRANS_FEAT(NAME, FEAT, gen_gvec_ool_arg_zpz, name##_fns[a->esz], a, 0)
776 DO_ZPZ(CLS
, aa64_sve
, sve_cls
)
777 DO_ZPZ(CLZ
, aa64_sve
, sve_clz
)
778 DO_ZPZ(CNT_zpz
, aa64_sve
, sve_cnt_zpz
)
779 DO_ZPZ(CNOT
, aa64_sve
, sve_cnot
)
780 DO_ZPZ(NOT_zpz
, aa64_sve
, sve_not_zpz
)
781 DO_ZPZ(ABS
, aa64_sve
, sve_abs
)
782 DO_ZPZ(NEG
, aa64_sve
, sve_neg
)
783 DO_ZPZ(RBIT
, aa64_sve
, sve_rbit
)
785 static gen_helper_gvec_3
* const fabs_fns
[4] = {
786 NULL
, gen_helper_sve_fabs_h
,
787 gen_helper_sve_fabs_s
, gen_helper_sve_fabs_d
,
789 TRANS_FEAT(FABS
, aa64_sve
, gen_gvec_ool_arg_zpz
, fabs_fns
[a
->esz
], a
, 0)
791 static gen_helper_gvec_3
* const fneg_fns
[4] = {
792 NULL
, gen_helper_sve_fneg_h
,
793 gen_helper_sve_fneg_s
, gen_helper_sve_fneg_d
,
795 TRANS_FEAT(FNEG
, aa64_sve
, gen_gvec_ool_arg_zpz
, fneg_fns
[a
->esz
], a
, 0)
797 static gen_helper_gvec_3
* const sxtb_fns
[4] = {
798 NULL
, gen_helper_sve_sxtb_h
,
799 gen_helper_sve_sxtb_s
, gen_helper_sve_sxtb_d
,
801 TRANS_FEAT(SXTB
, aa64_sve
, gen_gvec_ool_arg_zpz
, sxtb_fns
[a
->esz
], a
, 0)
803 static gen_helper_gvec_3
* const uxtb_fns
[4] = {
804 NULL
, gen_helper_sve_uxtb_h
,
805 gen_helper_sve_uxtb_s
, gen_helper_sve_uxtb_d
,
807 TRANS_FEAT(UXTB
, aa64_sve
, gen_gvec_ool_arg_zpz
, uxtb_fns
[a
->esz
], a
, 0)
809 static gen_helper_gvec_3
* const sxth_fns
[4] = {
810 NULL
, NULL
, gen_helper_sve_sxth_s
, gen_helper_sve_sxth_d
812 TRANS_FEAT(SXTH
, aa64_sve
, gen_gvec_ool_arg_zpz
, sxth_fns
[a
->esz
], a
, 0)
814 static gen_helper_gvec_3
* const uxth_fns
[4] = {
815 NULL
, NULL
, gen_helper_sve_uxth_s
, gen_helper_sve_uxth_d
817 TRANS_FEAT(UXTH
, aa64_sve
, gen_gvec_ool_arg_zpz
, uxth_fns
[a
->esz
], a
, 0)
819 TRANS_FEAT(SXTW
, aa64_sve
, gen_gvec_ool_arg_zpz
,
820 a
->esz
== 3 ? gen_helper_sve_sxtw_d
: NULL
, a
, 0)
821 TRANS_FEAT(UXTW
, aa64_sve
, gen_gvec_ool_arg_zpz
,
822 a
->esz
== 3 ? gen_helper_sve_uxtw_d
: NULL
, a
, 0)
825 *** SVE Integer Reduction Group
828 typedef void gen_helper_gvec_reduc(TCGv_i64
, TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
829 static bool do_vpz_ool(DisasContext
*s
, arg_rpr_esz
*a
,
830 gen_helper_gvec_reduc
*fn
)
832 unsigned vsz
= vec_full_reg_size(s
);
840 if (!sve_access_check(s
)) {
844 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
845 temp
= tcg_temp_new_i64();
846 t_zn
= tcg_temp_new_ptr();
847 t_pg
= tcg_temp_new_ptr();
849 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
850 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
851 fn(temp
, t_zn
, t_pg
, desc
);
852 tcg_temp_free_ptr(t_zn
);
853 tcg_temp_free_ptr(t_pg
);
855 write_fp_dreg(s
, a
->rd
, temp
);
856 tcg_temp_free_i64(temp
);
860 #define DO_VPZ(NAME, name) \
861 static gen_helper_gvec_reduc * const name##_fns[4] = { \
862 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
863 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
865 TRANS_FEAT(NAME, aa64_sve, do_vpz_ool, a, name##_fns[a->esz])
877 static gen_helper_gvec_reduc
* const saddv_fns
[4] = {
878 gen_helper_sve_saddv_b
, gen_helper_sve_saddv_h
,
879 gen_helper_sve_saddv_s
, NULL
881 TRANS_FEAT(SADDV
, aa64_sve
, do_vpz_ool
, a
, saddv_fns
[a
->esz
])
886 *** SVE Shift by Immediate - Predicated Group
890 * Copy Zn into Zd, storing zeros into inactive elements.
891 * If invert, store zeros into the active elements.
893 static bool do_movz_zpz(DisasContext
*s
, int rd
, int rn
, int pg
,
894 int esz
, bool invert
)
896 static gen_helper_gvec_3
* const fns
[4] = {
897 gen_helper_sve_movz_b
, gen_helper_sve_movz_h
,
898 gen_helper_sve_movz_s
, gen_helper_sve_movz_d
,
900 return gen_gvec_ool_zzp(s
, fns
[esz
], rd
, rn
, pg
, invert
);
903 static bool do_shift_zpzi(DisasContext
*s
, arg_rpri_esz
*a
, bool asr
,
904 gen_helper_gvec_3
* const fns
[4])
909 /* Invalid tsz encoding -- see tszimm_esz. */
914 * Shift by element size is architecturally valid.
915 * For arithmetic right-shift, it's the same as by one less.
916 * For logical shifts and ASRD, it is a zeroing operation.
923 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
926 return gen_gvec_ool_arg_zpzi(s
, fns
[a
->esz
], a
);
929 static gen_helper_gvec_3
* const asr_zpzi_fns
[4] = {
930 gen_helper_sve_asr_zpzi_b
, gen_helper_sve_asr_zpzi_h
,
931 gen_helper_sve_asr_zpzi_s
, gen_helper_sve_asr_zpzi_d
,
933 TRANS_FEAT(ASR_zpzi
, aa64_sve
, do_shift_zpzi
, a
, true, asr_zpzi_fns
)
935 static gen_helper_gvec_3
* const lsr_zpzi_fns
[4] = {
936 gen_helper_sve_lsr_zpzi_b
, gen_helper_sve_lsr_zpzi_h
,
937 gen_helper_sve_lsr_zpzi_s
, gen_helper_sve_lsr_zpzi_d
,
939 TRANS_FEAT(LSR_zpzi
, aa64_sve
, do_shift_zpzi
, a
, false, lsr_zpzi_fns
)
941 static gen_helper_gvec_3
* const lsl_zpzi_fns
[4] = {
942 gen_helper_sve_lsl_zpzi_b
, gen_helper_sve_lsl_zpzi_h
,
943 gen_helper_sve_lsl_zpzi_s
, gen_helper_sve_lsl_zpzi_d
,
945 TRANS_FEAT(LSL_zpzi
, aa64_sve
, do_shift_zpzi
, a
, false, lsl_zpzi_fns
)
947 static gen_helper_gvec_3
* const asrd_fns
[4] = {
948 gen_helper_sve_asrd_b
, gen_helper_sve_asrd_h
,
949 gen_helper_sve_asrd_s
, gen_helper_sve_asrd_d
,
951 TRANS_FEAT(ASRD
, aa64_sve
, do_shift_zpzi
, a
, false, asrd_fns
)
953 static gen_helper_gvec_3
* const sqshl_zpzi_fns
[4] = {
954 gen_helper_sve2_sqshl_zpzi_b
, gen_helper_sve2_sqshl_zpzi_h
,
955 gen_helper_sve2_sqshl_zpzi_s
, gen_helper_sve2_sqshl_zpzi_d
,
957 TRANS_FEAT(SQSHL_zpzi
, aa64_sve2
, gen_gvec_ool_arg_zpzi
,
958 a
->esz
< 0 ? NULL
: sqshl_zpzi_fns
[a
->esz
], a
)
960 static gen_helper_gvec_3
* const uqshl_zpzi_fns
[4] = {
961 gen_helper_sve2_uqshl_zpzi_b
, gen_helper_sve2_uqshl_zpzi_h
,
962 gen_helper_sve2_uqshl_zpzi_s
, gen_helper_sve2_uqshl_zpzi_d
,
964 TRANS_FEAT(UQSHL_zpzi
, aa64_sve2
, gen_gvec_ool_arg_zpzi
,
965 a
->esz
< 0 ? NULL
: uqshl_zpzi_fns
[a
->esz
], a
)
967 static gen_helper_gvec_3
* const srshr_fns
[4] = {
968 gen_helper_sve2_srshr_b
, gen_helper_sve2_srshr_h
,
969 gen_helper_sve2_srshr_s
, gen_helper_sve2_srshr_d
,
971 TRANS_FEAT(SRSHR
, aa64_sve2
, gen_gvec_ool_arg_zpzi
,
972 a
->esz
< 0 ? NULL
: srshr_fns
[a
->esz
], a
)
974 static gen_helper_gvec_3
* const urshr_fns
[4] = {
975 gen_helper_sve2_urshr_b
, gen_helper_sve2_urshr_h
,
976 gen_helper_sve2_urshr_s
, gen_helper_sve2_urshr_d
,
978 TRANS_FEAT(URSHR
, aa64_sve2
, gen_gvec_ool_arg_zpzi
,
979 a
->esz
< 0 ? NULL
: urshr_fns
[a
->esz
], a
)
981 static gen_helper_gvec_3
* const sqshlu_fns
[4] = {
982 gen_helper_sve2_sqshlu_b
, gen_helper_sve2_sqshlu_h
,
983 gen_helper_sve2_sqshlu_s
, gen_helper_sve2_sqshlu_d
,
985 TRANS_FEAT(SQSHLU
, aa64_sve2
, gen_gvec_ool_arg_zpzi
,
986 a
->esz
< 0 ? NULL
: sqshlu_fns
[a
->esz
], a
)
989 *** SVE Bitwise Shift - Predicated Group
992 #define DO_ZPZW(NAME, name) \
993 static gen_helper_gvec_4 * const name##_zpzw_fns[4] = { \
994 gen_helper_sve_##name##_zpzw_b, gen_helper_sve_##name##_zpzw_h, \
995 gen_helper_sve_##name##_zpzw_s, NULL \
997 TRANS_FEAT(NAME##_zpzw, aa64_sve, gen_gvec_ool_arg_zpzz, \
998 a->esz < 0 ? NULL : name##_zpzw_fns[a->esz], a, 0)
1007 *** SVE Bitwise Shift - Unpredicated Group
1010 static bool do_shift_imm(DisasContext
*s
, arg_rri_esz
*a
, bool asr
,
1011 void (*gvec_fn
)(unsigned, uint32_t, uint32_t,
1012 int64_t, uint32_t, uint32_t))
1015 /* Invalid tsz encoding -- see tszimm_esz. */
1018 if (sve_access_check(s
)) {
1019 unsigned vsz
= vec_full_reg_size(s
);
1020 /* Shift by element size is architecturally valid. For
1021 arithmetic right-shift, it's the same as by one less.
1022 Otherwise it is a zeroing operation. */
1023 if (a
->imm
>= 8 << a
->esz
) {
1025 a
->imm
= (8 << a
->esz
) - 1;
1027 do_dupi_z(s
, a
->rd
, 0);
1031 gvec_fn(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
1032 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
1037 TRANS_FEAT(ASR_zzi
, aa64_sve
, do_shift_imm
, a
, true, tcg_gen_gvec_sari
)
1038 TRANS_FEAT(LSR_zzi
, aa64_sve
, do_shift_imm
, a
, false, tcg_gen_gvec_shri
)
1039 TRANS_FEAT(LSL_zzi
, aa64_sve
, do_shift_imm
, a
, false, tcg_gen_gvec_shli
)
1041 #define DO_ZZW(NAME, name) \
1042 static gen_helper_gvec_3 * const name##_zzw_fns[4] = { \
1043 gen_helper_sve_##name##_zzw_b, gen_helper_sve_##name##_zzw_h, \
1044 gen_helper_sve_##name##_zzw_s, NULL \
1046 TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_arg_zzz, \
1047 name##_zzw_fns[a->esz], a, 0)
1049 DO_ZZW(ASR_zzw
, asr
)
1050 DO_ZZW(LSR_zzw
, lsr
)
1051 DO_ZZW(LSL_zzw
, lsl
)
1056 *** SVE Integer Multiply-Add Group
1059 static bool do_zpzzz_ool(DisasContext
*s
, arg_rprrr_esz
*a
,
1060 gen_helper_gvec_5
*fn
)
1062 if (sve_access_check(s
)) {
1063 unsigned vsz
= vec_full_reg_size(s
);
1064 tcg_gen_gvec_5_ool(vec_full_reg_offset(s
, a
->rd
),
1065 vec_full_reg_offset(s
, a
->ra
),
1066 vec_full_reg_offset(s
, a
->rn
),
1067 vec_full_reg_offset(s
, a
->rm
),
1068 pred_full_reg_offset(s
, a
->pg
),
1074 static gen_helper_gvec_5
* const mla_fns
[4] = {
1075 gen_helper_sve_mla_b
, gen_helper_sve_mla_h
,
1076 gen_helper_sve_mla_s
, gen_helper_sve_mla_d
,
1078 TRANS_FEAT(MLA
, aa64_sve
, do_zpzzz_ool
, a
, mla_fns
[a
->esz
])
1080 static gen_helper_gvec_5
* const mls_fns
[4] = {
1081 gen_helper_sve_mls_b
, gen_helper_sve_mls_h
,
1082 gen_helper_sve_mls_s
, gen_helper_sve_mls_d
,
1084 TRANS_FEAT(MLS
, aa64_sve
, do_zpzzz_ool
, a
, mls_fns
[a
->esz
])
1087 *** SVE Index Generation Group
1090 static bool do_index(DisasContext
*s
, int esz
, int rd
,
1091 TCGv_i64 start
, TCGv_i64 incr
)
1097 if (!sve_access_check(s
)) {
1101 vsz
= vec_full_reg_size(s
);
1102 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
1103 t_zd
= tcg_temp_new_ptr();
1105 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, rd
));
1107 gen_helper_sve_index_d(t_zd
, start
, incr
, desc
);
1109 typedef void index_fn(TCGv_ptr
, TCGv_i32
, TCGv_i32
, TCGv_i32
);
1110 static index_fn
* const fns
[3] = {
1111 gen_helper_sve_index_b
,
1112 gen_helper_sve_index_h
,
1113 gen_helper_sve_index_s
,
1115 TCGv_i32 s32
= tcg_temp_new_i32();
1116 TCGv_i32 i32
= tcg_temp_new_i32();
1118 tcg_gen_extrl_i64_i32(s32
, start
);
1119 tcg_gen_extrl_i64_i32(i32
, incr
);
1120 fns
[esz
](t_zd
, s32
, i32
, desc
);
1122 tcg_temp_free_i32(s32
);
1123 tcg_temp_free_i32(i32
);
1125 tcg_temp_free_ptr(t_zd
);
1129 TRANS_FEAT(INDEX_ii
, aa64_sve
, do_index
, a
->esz
, a
->rd
,
1130 tcg_constant_i64(a
->imm1
), tcg_constant_i64(a
->imm2
))
1131 TRANS_FEAT(INDEX_ir
, aa64_sve
, do_index
, a
->esz
, a
->rd
,
1132 tcg_constant_i64(a
->imm
), cpu_reg(s
, a
->rm
))
1133 TRANS_FEAT(INDEX_ri
, aa64_sve
, do_index
, a
->esz
, a
->rd
,
1134 cpu_reg(s
, a
->rn
), tcg_constant_i64(a
->imm
))
1135 TRANS_FEAT(INDEX_rr
, aa64_sve
, do_index
, a
->esz
, a
->rd
,
1136 cpu_reg(s
, a
->rn
), cpu_reg(s
, a
->rm
))
1139 *** SVE Stack Allocation Group
1142 static bool trans_ADDVL(DisasContext
*s
, arg_ADDVL
*a
)
1144 if (sve_access_check(s
)) {
1145 TCGv_i64 rd
= cpu_reg_sp(s
, a
->rd
);
1146 TCGv_i64 rn
= cpu_reg_sp(s
, a
->rn
);
1147 tcg_gen_addi_i64(rd
, rn
, a
->imm
* vec_full_reg_size(s
));
1152 static bool trans_ADDPL(DisasContext
*s
, arg_ADDPL
*a
)
1154 if (sve_access_check(s
)) {
1155 TCGv_i64 rd
= cpu_reg_sp(s
, a
->rd
);
1156 TCGv_i64 rn
= cpu_reg_sp(s
, a
->rn
);
1157 tcg_gen_addi_i64(rd
, rn
, a
->imm
* pred_full_reg_size(s
));
1162 static bool trans_RDVL(DisasContext
*s
, arg_RDVL
*a
)
1164 if (sve_access_check(s
)) {
1165 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
1166 tcg_gen_movi_i64(reg
, a
->imm
* vec_full_reg_size(s
));
1172 *** SVE Compute Vector Address Group
1175 static bool do_adr(DisasContext
*s
, arg_rrri
*a
, gen_helper_gvec_3
*fn
)
1177 return gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->imm
);
1180 static bool trans_ADR_p32(DisasContext
*s
, arg_rrri
*a
)
1182 return do_adr(s
, a
, gen_helper_sve_adr_p32
);
1185 static bool trans_ADR_p64(DisasContext
*s
, arg_rrri
*a
)
1187 return do_adr(s
, a
, gen_helper_sve_adr_p64
);
1190 static bool trans_ADR_s32(DisasContext
*s
, arg_rrri
*a
)
1192 return do_adr(s
, a
, gen_helper_sve_adr_s32
);
1195 static bool trans_ADR_u32(DisasContext
*s
, arg_rrri
*a
)
1197 return do_adr(s
, a
, gen_helper_sve_adr_u32
);
1201 *** SVE Integer Misc - Unpredicated Group
1204 static gen_helper_gvec_2
* const fexpa_fns
[4] = {
1205 NULL
, gen_helper_sve_fexpa_h
,
1206 gen_helper_sve_fexpa_s
, gen_helper_sve_fexpa_d
,
1208 TRANS_FEAT(FEXPA
, aa64_sve
, gen_gvec_ool_zz
,
1209 fexpa_fns
[a
->esz
], a
->rd
, a
->rn
, 0)
1211 static gen_helper_gvec_3
* const ftssel_fns
[4] = {
1212 NULL
, gen_helper_sve_ftssel_h
,
1213 gen_helper_sve_ftssel_s
, gen_helper_sve_ftssel_d
,
1215 TRANS_FEAT(FTSSEL
, aa64_sve
, gen_gvec_ool_arg_zzz
, ftssel_fns
[a
->esz
], a
, 0)
1218 *** SVE Predicate Logical Operations Group
1221 static bool do_pppp_flags(DisasContext
*s
, arg_rprr_s
*a
,
1222 const GVecGen4
*gvec_op
)
1224 if (!sve_access_check(s
)) {
1228 unsigned psz
= pred_gvec_reg_size(s
);
1229 int dofs
= pred_full_reg_offset(s
, a
->rd
);
1230 int nofs
= pred_full_reg_offset(s
, a
->rn
);
1231 int mofs
= pred_full_reg_offset(s
, a
->rm
);
1232 int gofs
= pred_full_reg_offset(s
, a
->pg
);
1235 tcg_gen_gvec_4(dofs
, nofs
, mofs
, gofs
, psz
, psz
, gvec_op
);
1240 /* Do the operation and the flags generation in temps. */
1241 TCGv_i64 pd
= tcg_temp_new_i64();
1242 TCGv_i64 pn
= tcg_temp_new_i64();
1243 TCGv_i64 pm
= tcg_temp_new_i64();
1244 TCGv_i64 pg
= tcg_temp_new_i64();
1246 tcg_gen_ld_i64(pn
, cpu_env
, nofs
);
1247 tcg_gen_ld_i64(pm
, cpu_env
, mofs
);
1248 tcg_gen_ld_i64(pg
, cpu_env
, gofs
);
1250 gvec_op
->fni8(pd
, pn
, pm
, pg
);
1251 tcg_gen_st_i64(pd
, cpu_env
, dofs
);
1253 do_predtest1(pd
, pg
);
1255 tcg_temp_free_i64(pd
);
1256 tcg_temp_free_i64(pn
);
1257 tcg_temp_free_i64(pm
);
1258 tcg_temp_free_i64(pg
);
1260 /* The operation and flags generation is large. The computation
1261 * of the flags depends on the original contents of the guarding
1262 * predicate. If the destination overwrites the guarding predicate,
1263 * then the easiest way to get this right is to save a copy.
1266 if (a
->rd
== a
->pg
) {
1267 tofs
= offsetof(CPUARMState
, vfp
.preg_tmp
);
1268 tcg_gen_gvec_mov(0, tofs
, gofs
, psz
, psz
);
1271 tcg_gen_gvec_4(dofs
, nofs
, mofs
, gofs
, psz
, psz
, gvec_op
);
1272 do_predtest(s
, dofs
, tofs
, psz
/ 8);
1277 static void gen_and_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1279 tcg_gen_and_i64(pd
, pn
, pm
);
1280 tcg_gen_and_i64(pd
, pd
, pg
);
1283 static void gen_and_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1284 TCGv_vec pm
, TCGv_vec pg
)
1286 tcg_gen_and_vec(vece
, pd
, pn
, pm
);
1287 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1290 static bool trans_AND_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1292 static const GVecGen4 op
= {
1293 .fni8
= gen_and_pg_i64
,
1294 .fniv
= gen_and_pg_vec
,
1295 .fno
= gen_helper_sve_and_pppp
,
1296 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1300 if (!sve_access_check(s
)) {
1303 if (a
->rn
== a
->rm
) {
1304 if (a
->pg
== a
->rn
) {
1305 do_mov_p(s
, a
->rd
, a
->rn
);
1307 gen_gvec_fn_ppp(s
, tcg_gen_gvec_and
, a
->rd
, a
->rn
, a
->pg
);
1310 } else if (a
->pg
== a
->rn
|| a
->pg
== a
->rm
) {
1311 gen_gvec_fn_ppp(s
, tcg_gen_gvec_and
, a
->rd
, a
->rn
, a
->rm
);
1315 return do_pppp_flags(s
, a
, &op
);
1318 static void gen_bic_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1320 tcg_gen_andc_i64(pd
, pn
, pm
);
1321 tcg_gen_and_i64(pd
, pd
, pg
);
1324 static void gen_bic_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1325 TCGv_vec pm
, TCGv_vec pg
)
1327 tcg_gen_andc_vec(vece
, pd
, pn
, pm
);
1328 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1331 static bool trans_BIC_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1333 static const GVecGen4 op
= {
1334 .fni8
= gen_bic_pg_i64
,
1335 .fniv
= gen_bic_pg_vec
,
1336 .fno
= gen_helper_sve_bic_pppp
,
1337 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1340 if (!a
->s
&& a
->pg
== a
->rn
) {
1341 if (sve_access_check(s
)) {
1342 gen_gvec_fn_ppp(s
, tcg_gen_gvec_andc
, a
->rd
, a
->rn
, a
->rm
);
1346 return do_pppp_flags(s
, a
, &op
);
1349 static void gen_eor_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1351 tcg_gen_xor_i64(pd
, pn
, pm
);
1352 tcg_gen_and_i64(pd
, pd
, pg
);
1355 static void gen_eor_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1356 TCGv_vec pm
, TCGv_vec pg
)
1358 tcg_gen_xor_vec(vece
, pd
, pn
, pm
);
1359 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1362 static bool trans_EOR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1364 static const GVecGen4 op
= {
1365 .fni8
= gen_eor_pg_i64
,
1366 .fniv
= gen_eor_pg_vec
,
1367 .fno
= gen_helper_sve_eor_pppp
,
1368 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1370 return do_pppp_flags(s
, a
, &op
);
1373 static bool trans_SEL_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1378 if (sve_access_check(s
)) {
1379 unsigned psz
= pred_gvec_reg_size(s
);
1380 tcg_gen_gvec_bitsel(MO_8
, pred_full_reg_offset(s
, a
->rd
),
1381 pred_full_reg_offset(s
, a
->pg
),
1382 pred_full_reg_offset(s
, a
->rn
),
1383 pred_full_reg_offset(s
, a
->rm
), psz
, psz
);
1388 static void gen_orr_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1390 tcg_gen_or_i64(pd
, pn
, pm
);
1391 tcg_gen_and_i64(pd
, pd
, pg
);
1394 static void gen_orr_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1395 TCGv_vec pm
, TCGv_vec pg
)
1397 tcg_gen_or_vec(vece
, pd
, pn
, pm
);
1398 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1401 static bool trans_ORR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1403 static const GVecGen4 op
= {
1404 .fni8
= gen_orr_pg_i64
,
1405 .fniv
= gen_orr_pg_vec
,
1406 .fno
= gen_helper_sve_orr_pppp
,
1407 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1410 if (!a
->s
&& a
->pg
== a
->rn
&& a
->rn
== a
->rm
) {
1411 return do_mov_p(s
, a
->rd
, a
->rn
);
1413 return do_pppp_flags(s
, a
, &op
);
1416 static void gen_orn_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1418 tcg_gen_orc_i64(pd
, pn
, pm
);
1419 tcg_gen_and_i64(pd
, pd
, pg
);
1422 static void gen_orn_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1423 TCGv_vec pm
, TCGv_vec pg
)
1425 tcg_gen_orc_vec(vece
, pd
, pn
, pm
);
1426 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1429 static bool trans_ORN_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1431 static const GVecGen4 op
= {
1432 .fni8
= gen_orn_pg_i64
,
1433 .fniv
= gen_orn_pg_vec
,
1434 .fno
= gen_helper_sve_orn_pppp
,
1435 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1437 return do_pppp_flags(s
, a
, &op
);
1440 static void gen_nor_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1442 tcg_gen_or_i64(pd
, pn
, pm
);
1443 tcg_gen_andc_i64(pd
, pg
, pd
);
1446 static void gen_nor_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1447 TCGv_vec pm
, TCGv_vec pg
)
1449 tcg_gen_or_vec(vece
, pd
, pn
, pm
);
1450 tcg_gen_andc_vec(vece
, pd
, pg
, pd
);
1453 static bool trans_NOR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1455 static const GVecGen4 op
= {
1456 .fni8
= gen_nor_pg_i64
,
1457 .fniv
= gen_nor_pg_vec
,
1458 .fno
= gen_helper_sve_nor_pppp
,
1459 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1461 return do_pppp_flags(s
, a
, &op
);
1464 static void gen_nand_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1466 tcg_gen_and_i64(pd
, pn
, pm
);
1467 tcg_gen_andc_i64(pd
, pg
, pd
);
1470 static void gen_nand_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1471 TCGv_vec pm
, TCGv_vec pg
)
1473 tcg_gen_and_vec(vece
, pd
, pn
, pm
);
1474 tcg_gen_andc_vec(vece
, pd
, pg
, pd
);
1477 static bool trans_NAND_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1479 static const GVecGen4 op
= {
1480 .fni8
= gen_nand_pg_i64
,
1481 .fniv
= gen_nand_pg_vec
,
1482 .fno
= gen_helper_sve_nand_pppp
,
1483 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1485 return do_pppp_flags(s
, a
, &op
);
1489 *** SVE Predicate Misc Group
1492 static bool trans_PTEST(DisasContext
*s
, arg_PTEST
*a
)
1494 if (sve_access_check(s
)) {
1495 int nofs
= pred_full_reg_offset(s
, a
->rn
);
1496 int gofs
= pred_full_reg_offset(s
, a
->pg
);
1497 int words
= DIV_ROUND_UP(pred_full_reg_size(s
), 8);
1500 TCGv_i64 pn
= tcg_temp_new_i64();
1501 TCGv_i64 pg
= tcg_temp_new_i64();
1503 tcg_gen_ld_i64(pn
, cpu_env
, nofs
);
1504 tcg_gen_ld_i64(pg
, cpu_env
, gofs
);
1505 do_predtest1(pn
, pg
);
1507 tcg_temp_free_i64(pn
);
1508 tcg_temp_free_i64(pg
);
1510 do_predtest(s
, nofs
, gofs
, words
);
1516 /* See the ARM pseudocode DecodePredCount. */
1517 static unsigned decode_pred_count(unsigned fullsz
, int pattern
, int esz
)
1519 unsigned elements
= fullsz
>> esz
;
1523 case 0x0: /* POW2 */
1524 return pow2floor(elements
);
1535 case 0x9: /* VL16 */
1536 case 0xa: /* VL32 */
1537 case 0xb: /* VL64 */
1538 case 0xc: /* VL128 */
1539 case 0xd: /* VL256 */
1540 bound
= 16 << (pattern
- 9);
1542 case 0x1d: /* MUL4 */
1543 return elements
- elements
% 4;
1544 case 0x1e: /* MUL3 */
1545 return elements
- elements
% 3;
1546 case 0x1f: /* ALL */
1548 default: /* #uimm5 */
1551 return elements
>= bound
? bound
: 0;
1554 /* This handles all of the predicate initialization instructions,
1555 * PTRUE, PFALSE, SETFFR. For PFALSE, we will have set PAT == 32
1556 * so that decode_pred_count returns 0. For SETFFR, we will have
1557 * set RD == 16 == FFR.
1559 static bool do_predset(DisasContext
*s
, int esz
, int rd
, int pat
, bool setflag
)
1561 if (!sve_access_check(s
)) {
1565 unsigned fullsz
= vec_full_reg_size(s
);
1566 unsigned ofs
= pred_full_reg_offset(s
, rd
);
1567 unsigned numelem
, setsz
, i
;
1568 uint64_t word
, lastword
;
1571 numelem
= decode_pred_count(fullsz
, pat
, esz
);
1573 /* Determine what we must store into each bit, and how many. */
1575 lastword
= word
= 0;
1578 setsz
= numelem
<< esz
;
1579 lastword
= word
= pred_esz_masks
[esz
];
1581 lastword
&= MAKE_64BIT_MASK(0, setsz
% 64);
1585 t
= tcg_temp_new_i64();
1587 tcg_gen_movi_i64(t
, lastword
);
1588 tcg_gen_st_i64(t
, cpu_env
, ofs
);
1592 if (word
== lastword
) {
1593 unsigned maxsz
= size_for_gvec(fullsz
/ 8);
1594 unsigned oprsz
= size_for_gvec(setsz
/ 8);
1596 if (oprsz
* 8 == setsz
) {
1597 tcg_gen_gvec_dup_imm(MO_64
, ofs
, oprsz
, maxsz
, word
);
1605 tcg_gen_movi_i64(t
, word
);
1606 for (i
= 0; i
< QEMU_ALIGN_DOWN(setsz
, 8); i
+= 8) {
1607 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1609 if (lastword
!= word
) {
1610 tcg_gen_movi_i64(t
, lastword
);
1611 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1615 tcg_gen_movi_i64(t
, 0);
1616 for (; i
< fullsz
; i
+= 8) {
1617 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1622 tcg_temp_free_i64(t
);
1626 tcg_gen_movi_i32(cpu_NF
, -(word
!= 0));
1627 tcg_gen_movi_i32(cpu_CF
, word
== 0);
1628 tcg_gen_movi_i32(cpu_VF
, 0);
1629 tcg_gen_mov_i32(cpu_ZF
, cpu_NF
);
1634 static bool trans_PTRUE(DisasContext
*s
, arg_PTRUE
*a
)
1636 return do_predset(s
, a
->esz
, a
->rd
, a
->pat
, a
->s
);
1639 static bool trans_SETFFR(DisasContext
*s
, arg_SETFFR
*a
)
1641 /* Note pat == 31 is #all, to set all elements. */
1642 return do_predset(s
, 0, FFR_PRED_NUM
, 31, false);
1645 static bool trans_PFALSE(DisasContext
*s
, arg_PFALSE
*a
)
1647 /* Note pat == 32 is #unimp, to set no elements. */
1648 return do_predset(s
, 0, a
->rd
, 32, false);
1651 static bool trans_RDFFR_p(DisasContext
*s
, arg_RDFFR_p
*a
)
1653 /* The path through do_pppp_flags is complicated enough to want to avoid
1654 * duplication. Frob the arguments into the form of a predicated AND.
1656 arg_rprr_s alt_a
= {
1657 .rd
= a
->rd
, .pg
= a
->pg
, .s
= a
->s
,
1658 .rn
= FFR_PRED_NUM
, .rm
= FFR_PRED_NUM
,
1660 return trans_AND_pppp(s
, &alt_a
);
1663 static bool trans_RDFFR(DisasContext
*s
, arg_RDFFR
*a
)
1665 return do_mov_p(s
, a
->rd
, FFR_PRED_NUM
);
1668 static bool trans_WRFFR(DisasContext
*s
, arg_WRFFR
*a
)
1670 return do_mov_p(s
, FFR_PRED_NUM
, a
->rn
);
1673 static bool do_pfirst_pnext(DisasContext
*s
, arg_rr_esz
*a
,
1674 void (*gen_fn
)(TCGv_i32
, TCGv_ptr
,
1675 TCGv_ptr
, TCGv_i32
))
1677 if (!sve_access_check(s
)) {
1681 TCGv_ptr t_pd
= tcg_temp_new_ptr();
1682 TCGv_ptr t_pg
= tcg_temp_new_ptr();
1686 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, pred_full_reg_size(s
));
1687 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
1689 tcg_gen_addi_ptr(t_pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
1690 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
1691 t
= tcg_temp_new_i32();
1693 gen_fn(t
, t_pd
, t_pg
, tcg_constant_i32(desc
));
1694 tcg_temp_free_ptr(t_pd
);
1695 tcg_temp_free_ptr(t_pg
);
1698 tcg_temp_free_i32(t
);
1702 static bool trans_PFIRST(DisasContext
*s
, arg_rr_esz
*a
)
1704 return do_pfirst_pnext(s
, a
, gen_helper_sve_pfirst
);
1707 static bool trans_PNEXT(DisasContext
*s
, arg_rr_esz
*a
)
1709 return do_pfirst_pnext(s
, a
, gen_helper_sve_pnext
);
1713 *** SVE Element Count Group
1716 /* Perform an inline saturating addition of a 32-bit value within
1717 * a 64-bit register. The second operand is known to be positive,
1718 * which halves the comparisions we must perform to bound the result.
1720 static void do_sat_addsub_32(TCGv_i64 reg
, TCGv_i64 val
, bool u
, bool d
)
1724 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
1726 tcg_gen_ext32u_i64(reg
, reg
);
1728 tcg_gen_ext32s_i64(reg
, reg
);
1731 tcg_gen_sub_i64(reg
, reg
, val
);
1732 ibound
= (u
? 0 : INT32_MIN
);
1733 tcg_gen_smax_i64(reg
, reg
, tcg_constant_i64(ibound
));
1735 tcg_gen_add_i64(reg
, reg
, val
);
1736 ibound
= (u
? UINT32_MAX
: INT32_MAX
);
1737 tcg_gen_smin_i64(reg
, reg
, tcg_constant_i64(ibound
));
1741 /* Similarly with 64-bit values. */
1742 static void do_sat_addsub_64(TCGv_i64 reg
, TCGv_i64 val
, bool u
, bool d
)
1744 TCGv_i64 t0
= tcg_temp_new_i64();
1749 tcg_gen_sub_i64(t0
, reg
, val
);
1750 t2
= tcg_constant_i64(0);
1751 tcg_gen_movcond_i64(TCG_COND_LTU
, reg
, reg
, val
, t2
, t0
);
1753 tcg_gen_add_i64(t0
, reg
, val
);
1754 t2
= tcg_constant_i64(-1);
1755 tcg_gen_movcond_i64(TCG_COND_LTU
, reg
, t0
, reg
, t2
, t0
);
1758 TCGv_i64 t1
= tcg_temp_new_i64();
1760 /* Detect signed overflow for subtraction. */
1761 tcg_gen_xor_i64(t0
, reg
, val
);
1762 tcg_gen_sub_i64(t1
, reg
, val
);
1763 tcg_gen_xor_i64(reg
, reg
, t1
);
1764 tcg_gen_and_i64(t0
, t0
, reg
);
1766 /* Bound the result. */
1767 tcg_gen_movi_i64(reg
, INT64_MIN
);
1768 t2
= tcg_constant_i64(0);
1769 tcg_gen_movcond_i64(TCG_COND_LT
, reg
, t0
, t2
, reg
, t1
);
1771 /* Detect signed overflow for addition. */
1772 tcg_gen_xor_i64(t0
, reg
, val
);
1773 tcg_gen_add_i64(reg
, reg
, val
);
1774 tcg_gen_xor_i64(t1
, reg
, val
);
1775 tcg_gen_andc_i64(t0
, t1
, t0
);
1777 /* Bound the result. */
1778 tcg_gen_movi_i64(t1
, INT64_MAX
);
1779 t2
= tcg_constant_i64(0);
1780 tcg_gen_movcond_i64(TCG_COND_LT
, reg
, t0
, t2
, t1
, reg
);
1782 tcg_temp_free_i64(t1
);
1784 tcg_temp_free_i64(t0
);
1787 /* Similarly with a vector and a scalar operand. */
1788 static void do_sat_addsub_vec(DisasContext
*s
, int esz
, int rd
, int rn
,
1789 TCGv_i64 val
, bool u
, bool d
)
1791 unsigned vsz
= vec_full_reg_size(s
);
1792 TCGv_ptr dptr
, nptr
;
1796 dptr
= tcg_temp_new_ptr();
1797 nptr
= tcg_temp_new_ptr();
1798 tcg_gen_addi_ptr(dptr
, cpu_env
, vec_full_reg_offset(s
, rd
));
1799 tcg_gen_addi_ptr(nptr
, cpu_env
, vec_full_reg_offset(s
, rn
));
1800 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
1804 t32
= tcg_temp_new_i32();
1805 tcg_gen_extrl_i64_i32(t32
, val
);
1807 tcg_gen_neg_i32(t32
, t32
);
1810 gen_helper_sve_uqaddi_b(dptr
, nptr
, t32
, desc
);
1812 gen_helper_sve_sqaddi_b(dptr
, nptr
, t32
, desc
);
1814 tcg_temp_free_i32(t32
);
1818 t32
= tcg_temp_new_i32();
1819 tcg_gen_extrl_i64_i32(t32
, val
);
1821 tcg_gen_neg_i32(t32
, t32
);
1824 gen_helper_sve_uqaddi_h(dptr
, nptr
, t32
, desc
);
1826 gen_helper_sve_sqaddi_h(dptr
, nptr
, t32
, desc
);
1828 tcg_temp_free_i32(t32
);
1832 t64
= tcg_temp_new_i64();
1834 tcg_gen_neg_i64(t64
, val
);
1836 tcg_gen_mov_i64(t64
, val
);
1839 gen_helper_sve_uqaddi_s(dptr
, nptr
, t64
, desc
);
1841 gen_helper_sve_sqaddi_s(dptr
, nptr
, t64
, desc
);
1843 tcg_temp_free_i64(t64
);
1849 gen_helper_sve_uqsubi_d(dptr
, nptr
, val
, desc
);
1851 gen_helper_sve_uqaddi_d(dptr
, nptr
, val
, desc
);
1854 t64
= tcg_temp_new_i64();
1855 tcg_gen_neg_i64(t64
, val
);
1856 gen_helper_sve_sqaddi_d(dptr
, nptr
, t64
, desc
);
1857 tcg_temp_free_i64(t64
);
1859 gen_helper_sve_sqaddi_d(dptr
, nptr
, val
, desc
);
1864 g_assert_not_reached();
1867 tcg_temp_free_ptr(dptr
);
1868 tcg_temp_free_ptr(nptr
);
1871 static bool trans_CNT_r(DisasContext
*s
, arg_CNT_r
*a
)
1873 if (sve_access_check(s
)) {
1874 unsigned fullsz
= vec_full_reg_size(s
);
1875 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
1876 tcg_gen_movi_i64(cpu_reg(s
, a
->rd
), numelem
* a
->imm
);
1881 static bool trans_INCDEC_r(DisasContext
*s
, arg_incdec_cnt
*a
)
1883 if (sve_access_check(s
)) {
1884 unsigned fullsz
= vec_full_reg_size(s
);
1885 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
1886 int inc
= numelem
* a
->imm
* (a
->d
? -1 : 1);
1887 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
1889 tcg_gen_addi_i64(reg
, reg
, inc
);
1894 static bool trans_SINCDEC_r_32(DisasContext
*s
, arg_incdec_cnt
*a
)
1896 if (!sve_access_check(s
)) {
1900 unsigned fullsz
= vec_full_reg_size(s
);
1901 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
1902 int inc
= numelem
* a
->imm
;
1903 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
1905 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
1908 tcg_gen_ext32u_i64(reg
, reg
);
1910 tcg_gen_ext32s_i64(reg
, reg
);
1913 do_sat_addsub_32(reg
, tcg_constant_i64(inc
), a
->u
, a
->d
);
1918 static bool trans_SINCDEC_r_64(DisasContext
*s
, arg_incdec_cnt
*a
)
1920 if (!sve_access_check(s
)) {
1924 unsigned fullsz
= vec_full_reg_size(s
);
1925 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
1926 int inc
= numelem
* a
->imm
;
1927 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
1930 do_sat_addsub_64(reg
, tcg_constant_i64(inc
), a
->u
, a
->d
);
1935 static bool trans_INCDEC_v(DisasContext
*s
, arg_incdec2_cnt
*a
)
1941 unsigned fullsz
= vec_full_reg_size(s
);
1942 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
1943 int inc
= numelem
* a
->imm
;
1946 if (sve_access_check(s
)) {
1947 tcg_gen_gvec_adds(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
1948 vec_full_reg_offset(s
, a
->rn
),
1949 tcg_constant_i64(a
->d
? -inc
: inc
),
1953 do_mov_z(s
, a
->rd
, a
->rn
);
1958 static bool trans_SINCDEC_v(DisasContext
*s
, arg_incdec2_cnt
*a
)
1964 unsigned fullsz
= vec_full_reg_size(s
);
1965 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
1966 int inc
= numelem
* a
->imm
;
1969 if (sve_access_check(s
)) {
1970 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
,
1971 tcg_constant_i64(inc
), a
->u
, a
->d
);
1974 do_mov_z(s
, a
->rd
, a
->rn
);
1980 *** SVE Bitwise Immediate Group
1983 static bool do_zz_dbm(DisasContext
*s
, arg_rr_dbm
*a
, GVecGen2iFn
*gvec_fn
)
1986 if (!logic_imm_decode_wmask(&imm
, extract32(a
->dbm
, 12, 1),
1987 extract32(a
->dbm
, 0, 6),
1988 extract32(a
->dbm
, 6, 6))) {
1991 return gen_gvec_fn_zzi(s
, gvec_fn
, MO_64
, a
->rd
, a
->rn
, imm
);
1994 TRANS_FEAT(AND_zzi
, aa64_sve
, do_zz_dbm
, a
, tcg_gen_gvec_andi
)
1995 TRANS_FEAT(ORR_zzi
, aa64_sve
, do_zz_dbm
, a
, tcg_gen_gvec_ori
)
1996 TRANS_FEAT(EOR_zzi
, aa64_sve
, do_zz_dbm
, a
, tcg_gen_gvec_xori
)
1998 static bool trans_DUPM(DisasContext
*s
, arg_DUPM
*a
)
2001 if (!logic_imm_decode_wmask(&imm
, extract32(a
->dbm
, 12, 1),
2002 extract32(a
->dbm
, 0, 6),
2003 extract32(a
->dbm
, 6, 6))) {
2006 if (sve_access_check(s
)) {
2007 do_dupi_z(s
, a
->rd
, imm
);
2013 *** SVE Integer Wide Immediate - Predicated Group
2016 /* Implement all merging copies. This is used for CPY (immediate),
2017 * FCPY, CPY (scalar), CPY (SIMD&FP scalar).
2019 static void do_cpy_m(DisasContext
*s
, int esz
, int rd
, int rn
, int pg
,
2022 typedef void gen_cpy(TCGv_ptr
, TCGv_ptr
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
2023 static gen_cpy
* const fns
[4] = {
2024 gen_helper_sve_cpy_m_b
, gen_helper_sve_cpy_m_h
,
2025 gen_helper_sve_cpy_m_s
, gen_helper_sve_cpy_m_d
,
2027 unsigned vsz
= vec_full_reg_size(s
);
2028 TCGv_i32 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
2029 TCGv_ptr t_zd
= tcg_temp_new_ptr();
2030 TCGv_ptr t_zn
= tcg_temp_new_ptr();
2031 TCGv_ptr t_pg
= tcg_temp_new_ptr();
2033 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, rd
));
2034 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, rn
));
2035 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
2037 fns
[esz
](t_zd
, t_zn
, t_pg
, val
, desc
);
2039 tcg_temp_free_ptr(t_zd
);
2040 tcg_temp_free_ptr(t_zn
);
2041 tcg_temp_free_ptr(t_pg
);
2044 static bool trans_FCPY(DisasContext
*s
, arg_FCPY
*a
)
2049 if (sve_access_check(s
)) {
2050 /* Decode the VFP immediate. */
2051 uint64_t imm
= vfp_expand_imm(a
->esz
, a
->imm
);
2052 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rn
, a
->pg
, tcg_constant_i64(imm
));
2057 static bool trans_CPY_m_i(DisasContext
*s
, arg_rpri_esz
*a
)
2059 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
2062 if (sve_access_check(s
)) {
2063 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rn
, a
->pg
, tcg_constant_i64(a
->imm
));
2068 static bool trans_CPY_z_i(DisasContext
*s
, arg_CPY_z_i
*a
)
2070 static gen_helper_gvec_2i
* const fns
[4] = {
2071 gen_helper_sve_cpy_z_b
, gen_helper_sve_cpy_z_h
,
2072 gen_helper_sve_cpy_z_s
, gen_helper_sve_cpy_z_d
,
2075 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
2078 if (sve_access_check(s
)) {
2079 unsigned vsz
= vec_full_reg_size(s
);
2080 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s
, a
->rd
),
2081 pred_full_reg_offset(s
, a
->pg
),
2082 tcg_constant_i64(a
->imm
),
2083 vsz
, vsz
, 0, fns
[a
->esz
]);
2089 *** SVE Permute Extract Group
2092 static bool do_EXT(DisasContext
*s
, int rd
, int rn
, int rm
, int imm
)
2094 if (!sve_access_check(s
)) {
2098 unsigned vsz
= vec_full_reg_size(s
);
2099 unsigned n_ofs
= imm
>= vsz
? 0 : imm
;
2100 unsigned n_siz
= vsz
- n_ofs
;
2101 unsigned d
= vec_full_reg_offset(s
, rd
);
2102 unsigned n
= vec_full_reg_offset(s
, rn
);
2103 unsigned m
= vec_full_reg_offset(s
, rm
);
2105 /* Use host vector move insns if we have appropriate sizes
2106 * and no unfortunate overlap.
2109 && n_ofs
== size_for_gvec(n_ofs
)
2110 && n_siz
== size_for_gvec(n_siz
)
2111 && (d
!= n
|| n_siz
<= n_ofs
)) {
2112 tcg_gen_gvec_mov(0, d
, n
+ n_ofs
, n_siz
, n_siz
);
2114 tcg_gen_gvec_mov(0, d
+ n_siz
, m
, n_ofs
, n_ofs
);
2117 tcg_gen_gvec_3_ool(d
, n
, m
, vsz
, vsz
, n_ofs
, gen_helper_sve_ext
);
2122 static bool trans_EXT(DisasContext
*s
, arg_EXT
*a
)
2124 return do_EXT(s
, a
->rd
, a
->rn
, a
->rm
, a
->imm
);
2127 static bool trans_EXT_sve2(DisasContext
*s
, arg_rri
*a
)
2129 if (!dc_isar_feature(aa64_sve2
, s
)) {
2132 return do_EXT(s
, a
->rd
, a
->rn
, (a
->rn
+ 1) % 32, a
->imm
);
2136 *** SVE Permute - Unpredicated Group
2139 static bool trans_DUP_s(DisasContext
*s
, arg_DUP_s
*a
)
2141 if (sve_access_check(s
)) {
2142 unsigned vsz
= vec_full_reg_size(s
);
2143 tcg_gen_gvec_dup_i64(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
2144 vsz
, vsz
, cpu_reg_sp(s
, a
->rn
));
2149 static bool trans_DUP_x(DisasContext
*s
, arg_DUP_x
*a
)
2151 if ((a
->imm
& 0x1f) == 0) {
2154 if (sve_access_check(s
)) {
2155 unsigned vsz
= vec_full_reg_size(s
);
2156 unsigned dofs
= vec_full_reg_offset(s
, a
->rd
);
2157 unsigned esz
, index
;
2159 esz
= ctz32(a
->imm
);
2160 index
= a
->imm
>> (esz
+ 1);
2162 if ((index
<< esz
) < vsz
) {
2163 unsigned nofs
= vec_reg_offset(s
, a
->rn
, index
, esz
);
2164 tcg_gen_gvec_dup_mem(esz
, dofs
, nofs
, vsz
, vsz
);
2167 * While dup_mem handles 128-bit elements, dup_imm does not.
2168 * Thankfully element size doesn't matter for splatting zero.
2170 tcg_gen_gvec_dup_imm(MO_64
, dofs
, vsz
, vsz
, 0);
2176 static void do_insr_i64(DisasContext
*s
, arg_rrr_esz
*a
, TCGv_i64 val
)
2178 typedef void gen_insr(TCGv_ptr
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
2179 static gen_insr
* const fns
[4] = {
2180 gen_helper_sve_insr_b
, gen_helper_sve_insr_h
,
2181 gen_helper_sve_insr_s
, gen_helper_sve_insr_d
,
2183 unsigned vsz
= vec_full_reg_size(s
);
2184 TCGv_i32 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
2185 TCGv_ptr t_zd
= tcg_temp_new_ptr();
2186 TCGv_ptr t_zn
= tcg_temp_new_ptr();
2188 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, a
->rd
));
2189 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
2191 fns
[a
->esz
](t_zd
, t_zn
, val
, desc
);
2193 tcg_temp_free_ptr(t_zd
);
2194 tcg_temp_free_ptr(t_zn
);
2197 static bool trans_INSR_f(DisasContext
*s
, arg_rrr_esz
*a
)
2199 if (sve_access_check(s
)) {
2200 TCGv_i64 t
= tcg_temp_new_i64();
2201 tcg_gen_ld_i64(t
, cpu_env
, vec_reg_offset(s
, a
->rm
, 0, MO_64
));
2202 do_insr_i64(s
, a
, t
);
2203 tcg_temp_free_i64(t
);
2208 static bool trans_INSR_r(DisasContext
*s
, arg_rrr_esz
*a
)
2210 if (sve_access_check(s
)) {
2211 do_insr_i64(s
, a
, cpu_reg(s
, a
->rm
));
2216 static gen_helper_gvec_2
* const rev_fns
[4] = {
2217 gen_helper_sve_rev_b
, gen_helper_sve_rev_h
,
2218 gen_helper_sve_rev_s
, gen_helper_sve_rev_d
2220 TRANS_FEAT(REV_v
, aa64_sve
, gen_gvec_ool_zz
, rev_fns
[a
->esz
], a
->rd
, a
->rn
, 0)
2222 static gen_helper_gvec_3
* const sve_tbl_fns
[4] = {
2223 gen_helper_sve_tbl_b
, gen_helper_sve_tbl_h
,
2224 gen_helper_sve_tbl_s
, gen_helper_sve_tbl_d
2226 TRANS_FEAT(TBL
, aa64_sve
, gen_gvec_ool_arg_zzz
, sve_tbl_fns
[a
->esz
], a
, 0)
2228 static gen_helper_gvec_4
* const sve2_tbl_fns
[4] = {
2229 gen_helper_sve2_tbl_b
, gen_helper_sve2_tbl_h
,
2230 gen_helper_sve2_tbl_s
, gen_helper_sve2_tbl_d
2232 TRANS_FEAT(TBL_sve2
, aa64_sve2
, gen_gvec_ool_zzzz
, sve2_tbl_fns
[a
->esz
],
2233 a
->rd
, a
->rn
, (a
->rn
+ 1) % 32, a
->rm
, 0)
2235 static gen_helper_gvec_3
* const tbx_fns
[4] = {
2236 gen_helper_sve2_tbx_b
, gen_helper_sve2_tbx_h
,
2237 gen_helper_sve2_tbx_s
, gen_helper_sve2_tbx_d
2239 TRANS_FEAT(TBX
, aa64_sve2
, gen_gvec_ool_arg_zzz
, tbx_fns
[a
->esz
], a
, 0)
2241 static bool trans_UNPK(DisasContext
*s
, arg_UNPK
*a
)
2243 static gen_helper_gvec_2
* const fns
[4][2] = {
2245 { gen_helper_sve_sunpk_h
, gen_helper_sve_uunpk_h
},
2246 { gen_helper_sve_sunpk_s
, gen_helper_sve_uunpk_s
},
2247 { gen_helper_sve_sunpk_d
, gen_helper_sve_uunpk_d
},
2253 if (sve_access_check(s
)) {
2254 unsigned vsz
= vec_full_reg_size(s
);
2255 tcg_gen_gvec_2_ool(vec_full_reg_offset(s
, a
->rd
),
2256 vec_full_reg_offset(s
, a
->rn
)
2257 + (a
->h
? vsz
/ 2 : 0),
2258 vsz
, vsz
, 0, fns
[a
->esz
][a
->u
]);
2264 *** SVE Permute - Predicates Group
2267 static bool do_perm_pred3(DisasContext
*s
, arg_rrr_esz
*a
, bool high_odd
,
2268 gen_helper_gvec_3
*fn
)
2270 if (!sve_access_check(s
)) {
2274 unsigned vsz
= pred_full_reg_size(s
);
2276 TCGv_ptr t_d
= tcg_temp_new_ptr();
2277 TCGv_ptr t_n
= tcg_temp_new_ptr();
2278 TCGv_ptr t_m
= tcg_temp_new_ptr();
2281 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
);
2282 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
2283 desc
= FIELD_DP32(desc
, PREDDESC
, DATA
, high_odd
);
2285 tcg_gen_addi_ptr(t_d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2286 tcg_gen_addi_ptr(t_n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2287 tcg_gen_addi_ptr(t_m
, cpu_env
, pred_full_reg_offset(s
, a
->rm
));
2289 fn(t_d
, t_n
, t_m
, tcg_constant_i32(desc
));
2291 tcg_temp_free_ptr(t_d
);
2292 tcg_temp_free_ptr(t_n
);
2293 tcg_temp_free_ptr(t_m
);
2297 static bool do_perm_pred2(DisasContext
*s
, arg_rr_esz
*a
, bool high_odd
,
2298 gen_helper_gvec_2
*fn
)
2300 if (!sve_access_check(s
)) {
2304 unsigned vsz
= pred_full_reg_size(s
);
2305 TCGv_ptr t_d
= tcg_temp_new_ptr();
2306 TCGv_ptr t_n
= tcg_temp_new_ptr();
2309 tcg_gen_addi_ptr(t_d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2310 tcg_gen_addi_ptr(t_n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2312 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
);
2313 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
2314 desc
= FIELD_DP32(desc
, PREDDESC
, DATA
, high_odd
);
2316 fn(t_d
, t_n
, tcg_constant_i32(desc
));
2318 tcg_temp_free_ptr(t_d
);
2319 tcg_temp_free_ptr(t_n
);
2323 static bool trans_ZIP1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2325 return do_perm_pred3(s
, a
, 0, gen_helper_sve_zip_p
);
2328 static bool trans_ZIP2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2330 return do_perm_pred3(s
, a
, 1, gen_helper_sve_zip_p
);
2333 static bool trans_UZP1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2335 return do_perm_pred3(s
, a
, 0, gen_helper_sve_uzp_p
);
2338 static bool trans_UZP2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2340 return do_perm_pred3(s
, a
, 1, gen_helper_sve_uzp_p
);
2343 static bool trans_TRN1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2345 return do_perm_pred3(s
, a
, 0, gen_helper_sve_trn_p
);
2348 static bool trans_TRN2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2350 return do_perm_pred3(s
, a
, 1, gen_helper_sve_trn_p
);
2353 static bool trans_REV_p(DisasContext
*s
, arg_rr_esz
*a
)
2355 return do_perm_pred2(s
, a
, 0, gen_helper_sve_rev_p
);
2358 static bool trans_PUNPKLO(DisasContext
*s
, arg_PUNPKLO
*a
)
2360 return do_perm_pred2(s
, a
, 0, gen_helper_sve_punpk_p
);
2363 static bool trans_PUNPKHI(DisasContext
*s
, arg_PUNPKHI
*a
)
2365 return do_perm_pred2(s
, a
, 1, gen_helper_sve_punpk_p
);
2369 *** SVE Permute - Interleaving Group
2372 static bool do_zip(DisasContext
*s
, arg_rrr_esz
*a
, bool high
)
2374 static gen_helper_gvec_3
* const fns
[4] = {
2375 gen_helper_sve_zip_b
, gen_helper_sve_zip_h
,
2376 gen_helper_sve_zip_s
, gen_helper_sve_zip_d
,
2379 if (sve_access_check(s
)) {
2380 unsigned vsz
= vec_full_reg_size(s
);
2381 unsigned high_ofs
= high
? vsz
/ 2 : 0;
2382 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, a
->rd
),
2383 vec_full_reg_offset(s
, a
->rn
) + high_ofs
,
2384 vec_full_reg_offset(s
, a
->rm
) + high_ofs
,
2385 vsz
, vsz
, 0, fns
[a
->esz
]);
2390 static bool trans_ZIP1_z(DisasContext
*s
, arg_rrr_esz
*a
)
2392 return do_zip(s
, a
, false);
2395 static bool trans_ZIP2_z(DisasContext
*s
, arg_rrr_esz
*a
)
2397 return do_zip(s
, a
, true);
2400 static bool do_zip_q(DisasContext
*s
, arg_rrr_esz
*a
, bool high
)
2402 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
2405 if (sve_access_check(s
)) {
2406 unsigned vsz
= vec_full_reg_size(s
);
2407 unsigned high_ofs
= high
? QEMU_ALIGN_DOWN(vsz
, 32) / 2 : 0;
2408 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, a
->rd
),
2409 vec_full_reg_offset(s
, a
->rn
) + high_ofs
,
2410 vec_full_reg_offset(s
, a
->rm
) + high_ofs
,
2411 vsz
, vsz
, 0, gen_helper_sve2_zip_q
);
2416 static bool trans_ZIP1_q(DisasContext
*s
, arg_rrr_esz
*a
)
2418 return do_zip_q(s
, a
, false);
2421 static bool trans_ZIP2_q(DisasContext
*s
, arg_rrr_esz
*a
)
2423 return do_zip_q(s
, a
, true);
2426 static gen_helper_gvec_3
* const uzp_fns
[4] = {
2427 gen_helper_sve_uzp_b
, gen_helper_sve_uzp_h
,
2428 gen_helper_sve_uzp_s
, gen_helper_sve_uzp_d
,
2431 TRANS_FEAT(UZP1_z
, aa64_sve
, gen_gvec_ool_arg_zzz
,
2432 uzp_fns
[a
->esz
], a
, 0)
2433 TRANS_FEAT(UZP2_z
, aa64_sve
, gen_gvec_ool_arg_zzz
,
2434 uzp_fns
[a
->esz
], a
, 1 << a
->esz
)
2436 TRANS_FEAT(UZP1_q
, aa64_sve_f64mm
, gen_gvec_ool_arg_zzz
,
2437 gen_helper_sve2_uzp_q
, a
, 0)
2438 TRANS_FEAT(UZP2_q
, aa64_sve_f64mm
, gen_gvec_ool_arg_zzz
,
2439 gen_helper_sve2_uzp_q
, a
, 16)
2441 static gen_helper_gvec_3
* const trn_fns
[4] = {
2442 gen_helper_sve_trn_b
, gen_helper_sve_trn_h
,
2443 gen_helper_sve_trn_s
, gen_helper_sve_trn_d
,
2446 TRANS_FEAT(TRN1_z
, aa64_sve
, gen_gvec_ool_arg_zzz
,
2447 trn_fns
[a
->esz
], a
, 0)
2448 TRANS_FEAT(TRN2_z
, aa64_sve
, gen_gvec_ool_arg_zzz
,
2449 trn_fns
[a
->esz
], a
, 1 << a
->esz
)
2451 TRANS_FEAT(TRN1_q
, aa64_sve_f64mm
, gen_gvec_ool_arg_zzz
,
2452 gen_helper_sve2_trn_q
, a
, 0)
2453 TRANS_FEAT(TRN2_q
, aa64_sve_f64mm
, gen_gvec_ool_arg_zzz
,
2454 gen_helper_sve2_trn_q
, a
, 16)
2457 *** SVE Permute Vector - Predicated Group
2460 static gen_helper_gvec_3
* const compact_fns
[4] = {
2461 NULL
, NULL
, gen_helper_sve_compact_s
, gen_helper_sve_compact_d
2463 TRANS_FEAT(COMPACT
, aa64_sve
, gen_gvec_ool_arg_zpz
, compact_fns
[a
->esz
], a
, 0)
2465 /* Call the helper that computes the ARM LastActiveElement pseudocode
2466 * function, scaled by the element size. This includes the not found
2467 * indication; e.g. not found for esz=3 is -8.
2469 static void find_last_active(DisasContext
*s
, TCGv_i32 ret
, int esz
, int pg
)
2471 /* Predicate sizes may be smaller and cannot use simd_desc. We cannot
2472 * round up, as we do elsewhere, because we need the exact size.
2474 TCGv_ptr t_p
= tcg_temp_new_ptr();
2477 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, pred_full_reg_size(s
));
2478 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, esz
);
2480 tcg_gen_addi_ptr(t_p
, cpu_env
, pred_full_reg_offset(s
, pg
));
2482 gen_helper_sve_last_active_element(ret
, t_p
, tcg_constant_i32(desc
));
2484 tcg_temp_free_ptr(t_p
);
2487 /* Increment LAST to the offset of the next element in the vector,
2488 * wrapping around to 0.
2490 static void incr_last_active(DisasContext
*s
, TCGv_i32 last
, int esz
)
2492 unsigned vsz
= vec_full_reg_size(s
);
2494 tcg_gen_addi_i32(last
, last
, 1 << esz
);
2495 if (is_power_of_2(vsz
)) {
2496 tcg_gen_andi_i32(last
, last
, vsz
- 1);
2498 TCGv_i32 max
= tcg_constant_i32(vsz
);
2499 TCGv_i32 zero
= tcg_constant_i32(0);
2500 tcg_gen_movcond_i32(TCG_COND_GEU
, last
, last
, max
, zero
, last
);
2504 /* If LAST < 0, set LAST to the offset of the last element in the vector. */
2505 static void wrap_last_active(DisasContext
*s
, TCGv_i32 last
, int esz
)
2507 unsigned vsz
= vec_full_reg_size(s
);
2509 if (is_power_of_2(vsz
)) {
2510 tcg_gen_andi_i32(last
, last
, vsz
- 1);
2512 TCGv_i32 max
= tcg_constant_i32(vsz
- (1 << esz
));
2513 TCGv_i32 zero
= tcg_constant_i32(0);
2514 tcg_gen_movcond_i32(TCG_COND_LT
, last
, last
, zero
, max
, last
);
2518 /* Load an unsigned element of ESZ from BASE+OFS. */
2519 static TCGv_i64
load_esz(TCGv_ptr base
, int ofs
, int esz
)
2521 TCGv_i64 r
= tcg_temp_new_i64();
2525 tcg_gen_ld8u_i64(r
, base
, ofs
);
2528 tcg_gen_ld16u_i64(r
, base
, ofs
);
2531 tcg_gen_ld32u_i64(r
, base
, ofs
);
2534 tcg_gen_ld_i64(r
, base
, ofs
);
2537 g_assert_not_reached();
2542 /* Load an unsigned element of ESZ from RM[LAST]. */
2543 static TCGv_i64
load_last_active(DisasContext
*s
, TCGv_i32 last
,
2546 TCGv_ptr p
= tcg_temp_new_ptr();
2549 /* Convert offset into vector into offset into ENV.
2550 * The final adjustment for the vector register base
2551 * is added via constant offset to the load.
2554 /* Adjust for element ordering. See vec_reg_offset. */
2556 tcg_gen_xori_i32(last
, last
, 8 - (1 << esz
));
2559 tcg_gen_ext_i32_ptr(p
, last
);
2560 tcg_gen_add_ptr(p
, p
, cpu_env
);
2562 r
= load_esz(p
, vec_full_reg_offset(s
, rm
), esz
);
2563 tcg_temp_free_ptr(p
);
2568 /* Compute CLAST for a Zreg. */
2569 static bool do_clast_vector(DisasContext
*s
, arg_rprr_esz
*a
, bool before
)
2574 unsigned vsz
, esz
= a
->esz
;
2576 if (!sve_access_check(s
)) {
2580 last
= tcg_temp_local_new_i32();
2581 over
= gen_new_label();
2583 find_last_active(s
, last
, esz
, a
->pg
);
2585 /* There is of course no movcond for a 2048-bit vector,
2586 * so we must branch over the actual store.
2588 tcg_gen_brcondi_i32(TCG_COND_LT
, last
, 0, over
);
2591 incr_last_active(s
, last
, esz
);
2594 ele
= load_last_active(s
, last
, a
->rm
, esz
);
2595 tcg_temp_free_i32(last
);
2597 vsz
= vec_full_reg_size(s
);
2598 tcg_gen_gvec_dup_i64(esz
, vec_full_reg_offset(s
, a
->rd
), vsz
, vsz
, ele
);
2599 tcg_temp_free_i64(ele
);
2601 /* If this insn used MOVPRFX, we may need a second move. */
2602 if (a
->rd
!= a
->rn
) {
2603 TCGLabel
*done
= gen_new_label();
2606 gen_set_label(over
);
2607 do_mov_z(s
, a
->rd
, a
->rn
);
2609 gen_set_label(done
);
2611 gen_set_label(over
);
2616 static bool trans_CLASTA_z(DisasContext
*s
, arg_rprr_esz
*a
)
2618 return do_clast_vector(s
, a
, false);
2621 static bool trans_CLASTB_z(DisasContext
*s
, arg_rprr_esz
*a
)
2623 return do_clast_vector(s
, a
, true);
2626 /* Compute CLAST for a scalar. */
2627 static void do_clast_scalar(DisasContext
*s
, int esz
, int pg
, int rm
,
2628 bool before
, TCGv_i64 reg_val
)
2630 TCGv_i32 last
= tcg_temp_new_i32();
2633 find_last_active(s
, last
, esz
, pg
);
2635 /* Extend the original value of last prior to incrementing. */
2636 cmp
= tcg_temp_new_i64();
2637 tcg_gen_ext_i32_i64(cmp
, last
);
2640 incr_last_active(s
, last
, esz
);
2643 /* The conceit here is that while last < 0 indicates not found, after
2644 * adjusting for cpu_env->vfp.zregs[rm], it is still a valid address
2645 * from which we can load garbage. We then discard the garbage with
2646 * a conditional move.
2648 ele
= load_last_active(s
, last
, rm
, esz
);
2649 tcg_temp_free_i32(last
);
2651 tcg_gen_movcond_i64(TCG_COND_GE
, reg_val
, cmp
, tcg_constant_i64(0),
2654 tcg_temp_free_i64(cmp
);
2655 tcg_temp_free_i64(ele
);
2658 /* Compute CLAST for a Vreg. */
2659 static bool do_clast_fp(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2661 if (sve_access_check(s
)) {
2663 int ofs
= vec_reg_offset(s
, a
->rd
, 0, esz
);
2664 TCGv_i64 reg
= load_esz(cpu_env
, ofs
, esz
);
2666 do_clast_scalar(s
, esz
, a
->pg
, a
->rn
, before
, reg
);
2667 write_fp_dreg(s
, a
->rd
, reg
);
2668 tcg_temp_free_i64(reg
);
2673 static bool trans_CLASTA_v(DisasContext
*s
, arg_rpr_esz
*a
)
2675 return do_clast_fp(s
, a
, false);
2678 static bool trans_CLASTB_v(DisasContext
*s
, arg_rpr_esz
*a
)
2680 return do_clast_fp(s
, a
, true);
2683 /* Compute CLAST for a Xreg. */
2684 static bool do_clast_general(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2688 if (!sve_access_check(s
)) {
2692 reg
= cpu_reg(s
, a
->rd
);
2695 tcg_gen_ext8u_i64(reg
, reg
);
2698 tcg_gen_ext16u_i64(reg
, reg
);
2701 tcg_gen_ext32u_i64(reg
, reg
);
2706 g_assert_not_reached();
2709 do_clast_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
, reg
);
2713 static bool trans_CLASTA_r(DisasContext
*s
, arg_rpr_esz
*a
)
2715 return do_clast_general(s
, a
, false);
2718 static bool trans_CLASTB_r(DisasContext
*s
, arg_rpr_esz
*a
)
2720 return do_clast_general(s
, a
, true);
2723 /* Compute LAST for a scalar. */
2724 static TCGv_i64
do_last_scalar(DisasContext
*s
, int esz
,
2725 int pg
, int rm
, bool before
)
2727 TCGv_i32 last
= tcg_temp_new_i32();
2730 find_last_active(s
, last
, esz
, pg
);
2732 wrap_last_active(s
, last
, esz
);
2734 incr_last_active(s
, last
, esz
);
2737 ret
= load_last_active(s
, last
, rm
, esz
);
2738 tcg_temp_free_i32(last
);
2742 /* Compute LAST for a Vreg. */
2743 static bool do_last_fp(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2745 if (sve_access_check(s
)) {
2746 TCGv_i64 val
= do_last_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
);
2747 write_fp_dreg(s
, a
->rd
, val
);
2748 tcg_temp_free_i64(val
);
2753 static bool trans_LASTA_v(DisasContext
*s
, arg_rpr_esz
*a
)
2755 return do_last_fp(s
, a
, false);
2758 static bool trans_LASTB_v(DisasContext
*s
, arg_rpr_esz
*a
)
2760 return do_last_fp(s
, a
, true);
2763 /* Compute LAST for a Xreg. */
2764 static bool do_last_general(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2766 if (sve_access_check(s
)) {
2767 TCGv_i64 val
= do_last_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
);
2768 tcg_gen_mov_i64(cpu_reg(s
, a
->rd
), val
);
2769 tcg_temp_free_i64(val
);
2774 static bool trans_LASTA_r(DisasContext
*s
, arg_rpr_esz
*a
)
2776 return do_last_general(s
, a
, false);
2779 static bool trans_LASTB_r(DisasContext
*s
, arg_rpr_esz
*a
)
2781 return do_last_general(s
, a
, true);
2784 static bool trans_CPY_m_r(DisasContext
*s
, arg_rpr_esz
*a
)
2786 if (sve_access_check(s
)) {
2787 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rd
, a
->pg
, cpu_reg_sp(s
, a
->rn
));
2792 static bool trans_CPY_m_v(DisasContext
*s
, arg_rpr_esz
*a
)
2794 if (sve_access_check(s
)) {
2795 int ofs
= vec_reg_offset(s
, a
->rn
, 0, a
->esz
);
2796 TCGv_i64 t
= load_esz(cpu_env
, ofs
, a
->esz
);
2797 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rd
, a
->pg
, t
);
2798 tcg_temp_free_i64(t
);
2803 static gen_helper_gvec_3
* const revb_fns
[4] = {
2804 NULL
, gen_helper_sve_revb_h
,
2805 gen_helper_sve_revb_s
, gen_helper_sve_revb_d
,
2807 TRANS_FEAT(REVB
, aa64_sve
, gen_gvec_ool_arg_zpz
, revb_fns
[a
->esz
], a
, 0)
2809 static gen_helper_gvec_3
* const revh_fns
[4] = {
2810 NULL
, NULL
, gen_helper_sve_revh_s
, gen_helper_sve_revh_d
,
2812 TRANS_FEAT(REVH
, aa64_sve
, gen_gvec_ool_arg_zpz
, revh_fns
[a
->esz
], a
, 0)
2814 TRANS_FEAT(REVW
, aa64_sve
, gen_gvec_ool_arg_zpz
,
2815 a
->esz
== 3 ? gen_helper_sve_revw_d
: NULL
, a
, 0)
2817 static bool trans_SPLICE(DisasContext
*s
, arg_rprr_esz
*a
)
2819 return gen_gvec_ool_zzzp(s
, gen_helper_sve_splice
,
2820 a
->rd
, a
->rn
, a
->rm
, a
->pg
, a
->esz
);
2823 static bool trans_SPLICE_sve2(DisasContext
*s
, arg_rpr_esz
*a
)
2825 if (!dc_isar_feature(aa64_sve2
, s
)) {
2828 return gen_gvec_ool_zzzp(s
, gen_helper_sve_splice
,
2829 a
->rd
, a
->rn
, (a
->rn
+ 1) % 32, a
->pg
, a
->esz
);
2833 *** SVE Integer Compare - Vectors Group
2836 static bool do_ppzz_flags(DisasContext
*s
, arg_rprr_esz
*a
,
2837 gen_helper_gvec_flags_4
*gen_fn
)
2839 TCGv_ptr pd
, zn
, zm
, pg
;
2843 if (gen_fn
== NULL
) {
2846 if (!sve_access_check(s
)) {
2850 vsz
= vec_full_reg_size(s
);
2851 t
= tcg_temp_new_i32();
2852 pd
= tcg_temp_new_ptr();
2853 zn
= tcg_temp_new_ptr();
2854 zm
= tcg_temp_new_ptr();
2855 pg
= tcg_temp_new_ptr();
2857 tcg_gen_addi_ptr(pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2858 tcg_gen_addi_ptr(zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
2859 tcg_gen_addi_ptr(zm
, cpu_env
, vec_full_reg_offset(s
, a
->rm
));
2860 tcg_gen_addi_ptr(pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
2862 gen_fn(t
, pd
, zn
, zm
, pg
, tcg_constant_i32(simd_desc(vsz
, vsz
, 0)));
2864 tcg_temp_free_ptr(pd
);
2865 tcg_temp_free_ptr(zn
);
2866 tcg_temp_free_ptr(zm
);
2867 tcg_temp_free_ptr(pg
);
2871 tcg_temp_free_i32(t
);
2875 #define DO_PPZZ(NAME, name) \
2876 static bool trans_##NAME##_ppzz(DisasContext *s, arg_rprr_esz *a) \
2878 static gen_helper_gvec_flags_4 * const fns[4] = { \
2879 gen_helper_sve_##name##_ppzz_b, gen_helper_sve_##name##_ppzz_h, \
2880 gen_helper_sve_##name##_ppzz_s, gen_helper_sve_##name##_ppzz_d, \
2882 return do_ppzz_flags(s, a, fns[a->esz]); \
2885 DO_PPZZ(CMPEQ
, cmpeq
)
2886 DO_PPZZ(CMPNE
, cmpne
)
2887 DO_PPZZ(CMPGT
, cmpgt
)
2888 DO_PPZZ(CMPGE
, cmpge
)
2889 DO_PPZZ(CMPHI
, cmphi
)
2890 DO_PPZZ(CMPHS
, cmphs
)
2894 #define DO_PPZW(NAME, name) \
2895 static bool trans_##NAME##_ppzw(DisasContext *s, arg_rprr_esz *a) \
2897 static gen_helper_gvec_flags_4 * const fns[4] = { \
2898 gen_helper_sve_##name##_ppzw_b, gen_helper_sve_##name##_ppzw_h, \
2899 gen_helper_sve_##name##_ppzw_s, NULL \
2901 return do_ppzz_flags(s, a, fns[a->esz]); \
2904 DO_PPZW(CMPEQ
, cmpeq
)
2905 DO_PPZW(CMPNE
, cmpne
)
2906 DO_PPZW(CMPGT
, cmpgt
)
2907 DO_PPZW(CMPGE
, cmpge
)
2908 DO_PPZW(CMPHI
, cmphi
)
2909 DO_PPZW(CMPHS
, cmphs
)
2910 DO_PPZW(CMPLT
, cmplt
)
2911 DO_PPZW(CMPLE
, cmple
)
2912 DO_PPZW(CMPLO
, cmplo
)
2913 DO_PPZW(CMPLS
, cmpls
)
2918 *** SVE Integer Compare - Immediate Groups
2921 static bool do_ppzi_flags(DisasContext
*s
, arg_rpri_esz
*a
,
2922 gen_helper_gvec_flags_3
*gen_fn
)
2924 TCGv_ptr pd
, zn
, pg
;
2928 if (gen_fn
== NULL
) {
2931 if (!sve_access_check(s
)) {
2935 vsz
= vec_full_reg_size(s
);
2936 t
= tcg_temp_new_i32();
2937 pd
= tcg_temp_new_ptr();
2938 zn
= tcg_temp_new_ptr();
2939 pg
= tcg_temp_new_ptr();
2941 tcg_gen_addi_ptr(pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2942 tcg_gen_addi_ptr(zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
2943 tcg_gen_addi_ptr(pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
2945 gen_fn(t
, pd
, zn
, pg
, tcg_constant_i32(simd_desc(vsz
, vsz
, a
->imm
)));
2947 tcg_temp_free_ptr(pd
);
2948 tcg_temp_free_ptr(zn
);
2949 tcg_temp_free_ptr(pg
);
2953 tcg_temp_free_i32(t
);
2957 #define DO_PPZI(NAME, name) \
2958 static bool trans_##NAME##_ppzi(DisasContext *s, arg_rpri_esz *a) \
2960 static gen_helper_gvec_flags_3 * const fns[4] = { \
2961 gen_helper_sve_##name##_ppzi_b, gen_helper_sve_##name##_ppzi_h, \
2962 gen_helper_sve_##name##_ppzi_s, gen_helper_sve_##name##_ppzi_d, \
2964 return do_ppzi_flags(s, a, fns[a->esz]); \
2967 DO_PPZI(CMPEQ
, cmpeq
)
2968 DO_PPZI(CMPNE
, cmpne
)
2969 DO_PPZI(CMPGT
, cmpgt
)
2970 DO_PPZI(CMPGE
, cmpge
)
2971 DO_PPZI(CMPHI
, cmphi
)
2972 DO_PPZI(CMPHS
, cmphs
)
2973 DO_PPZI(CMPLT
, cmplt
)
2974 DO_PPZI(CMPLE
, cmple
)
2975 DO_PPZI(CMPLO
, cmplo
)
2976 DO_PPZI(CMPLS
, cmpls
)
2981 *** SVE Partition Break Group
2984 static bool do_brk3(DisasContext
*s
, arg_rprr_s
*a
,
2985 gen_helper_gvec_4
*fn
, gen_helper_gvec_flags_4
*fn_s
)
2987 if (!sve_access_check(s
)) {
2991 unsigned vsz
= pred_full_reg_size(s
);
2993 /* Predicate sizes may be smaller and cannot use simd_desc. */
2994 TCGv_ptr d
= tcg_temp_new_ptr();
2995 TCGv_ptr n
= tcg_temp_new_ptr();
2996 TCGv_ptr m
= tcg_temp_new_ptr();
2997 TCGv_ptr g
= tcg_temp_new_ptr();
2998 TCGv_i32 desc
= tcg_constant_i32(FIELD_DP32(0, PREDDESC
, OPRSZ
, vsz
));
3000 tcg_gen_addi_ptr(d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3001 tcg_gen_addi_ptr(n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
3002 tcg_gen_addi_ptr(m
, cpu_env
, pred_full_reg_offset(s
, a
->rm
));
3003 tcg_gen_addi_ptr(g
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3006 TCGv_i32 t
= tcg_temp_new_i32();
3007 fn_s(t
, d
, n
, m
, g
, desc
);
3009 tcg_temp_free_i32(t
);
3011 fn(d
, n
, m
, g
, desc
);
3013 tcg_temp_free_ptr(d
);
3014 tcg_temp_free_ptr(n
);
3015 tcg_temp_free_ptr(m
);
3016 tcg_temp_free_ptr(g
);
3020 static bool do_brk2(DisasContext
*s
, arg_rpr_s
*a
,
3021 gen_helper_gvec_3
*fn
, gen_helper_gvec_flags_3
*fn_s
)
3023 if (!sve_access_check(s
)) {
3027 unsigned vsz
= pred_full_reg_size(s
);
3029 /* Predicate sizes may be smaller and cannot use simd_desc. */
3030 TCGv_ptr d
= tcg_temp_new_ptr();
3031 TCGv_ptr n
= tcg_temp_new_ptr();
3032 TCGv_ptr g
= tcg_temp_new_ptr();
3033 TCGv_i32 desc
= tcg_constant_i32(FIELD_DP32(0, PREDDESC
, OPRSZ
, vsz
));
3035 tcg_gen_addi_ptr(d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3036 tcg_gen_addi_ptr(n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
3037 tcg_gen_addi_ptr(g
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3040 TCGv_i32 t
= tcg_temp_new_i32();
3041 fn_s(t
, d
, n
, g
, desc
);
3043 tcg_temp_free_i32(t
);
3047 tcg_temp_free_ptr(d
);
3048 tcg_temp_free_ptr(n
);
3049 tcg_temp_free_ptr(g
);
3053 static bool trans_BRKPA(DisasContext
*s
, arg_rprr_s
*a
)
3055 return do_brk3(s
, a
, gen_helper_sve_brkpa
, gen_helper_sve_brkpas
);
3058 static bool trans_BRKPB(DisasContext
*s
, arg_rprr_s
*a
)
3060 return do_brk3(s
, a
, gen_helper_sve_brkpb
, gen_helper_sve_brkpbs
);
3063 static bool trans_BRKA_m(DisasContext
*s
, arg_rpr_s
*a
)
3065 return do_brk2(s
, a
, gen_helper_sve_brka_m
, gen_helper_sve_brkas_m
);
3068 static bool trans_BRKB_m(DisasContext
*s
, arg_rpr_s
*a
)
3070 return do_brk2(s
, a
, gen_helper_sve_brkb_m
, gen_helper_sve_brkbs_m
);
3073 static bool trans_BRKA_z(DisasContext
*s
, arg_rpr_s
*a
)
3075 return do_brk2(s
, a
, gen_helper_sve_brka_z
, gen_helper_sve_brkas_z
);
3078 static bool trans_BRKB_z(DisasContext
*s
, arg_rpr_s
*a
)
3080 return do_brk2(s
, a
, gen_helper_sve_brkb_z
, gen_helper_sve_brkbs_z
);
3083 static bool trans_BRKN(DisasContext
*s
, arg_rpr_s
*a
)
3085 return do_brk2(s
, a
, gen_helper_sve_brkn
, gen_helper_sve_brkns
);
3089 *** SVE Predicate Count Group
3092 static void do_cntp(DisasContext
*s
, TCGv_i64 val
, int esz
, int pn
, int pg
)
3094 unsigned psz
= pred_full_reg_size(s
);
3099 tcg_gen_ld_i64(val
, cpu_env
, pred_full_reg_offset(s
, pn
));
3101 TCGv_i64 g
= tcg_temp_new_i64();
3102 tcg_gen_ld_i64(g
, cpu_env
, pred_full_reg_offset(s
, pg
));
3103 tcg_gen_and_i64(val
, val
, g
);
3104 tcg_temp_free_i64(g
);
3107 /* Reduce the pred_esz_masks value simply to reduce the
3108 * size of the code generated here.
3110 psz_mask
= MAKE_64BIT_MASK(0, psz
* 8);
3111 tcg_gen_andi_i64(val
, val
, pred_esz_masks
[esz
] & psz_mask
);
3113 tcg_gen_ctpop_i64(val
, val
);
3115 TCGv_ptr t_pn
= tcg_temp_new_ptr();
3116 TCGv_ptr t_pg
= tcg_temp_new_ptr();
3119 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, psz
);
3120 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, esz
);
3122 tcg_gen_addi_ptr(t_pn
, cpu_env
, pred_full_reg_offset(s
, pn
));
3123 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
3125 gen_helper_sve_cntp(val
, t_pn
, t_pg
, tcg_constant_i32(desc
));
3126 tcg_temp_free_ptr(t_pn
);
3127 tcg_temp_free_ptr(t_pg
);
3131 static bool trans_CNTP(DisasContext
*s
, arg_CNTP
*a
)
3133 if (sve_access_check(s
)) {
3134 do_cntp(s
, cpu_reg(s
, a
->rd
), a
->esz
, a
->rn
, a
->pg
);
3139 static bool trans_INCDECP_r(DisasContext
*s
, arg_incdec_pred
*a
)
3141 if (sve_access_check(s
)) {
3142 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3143 TCGv_i64 val
= tcg_temp_new_i64();
3145 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3147 tcg_gen_sub_i64(reg
, reg
, val
);
3149 tcg_gen_add_i64(reg
, reg
, val
);
3151 tcg_temp_free_i64(val
);
3156 static bool trans_INCDECP_z(DisasContext
*s
, arg_incdec2_pred
*a
)
3161 if (sve_access_check(s
)) {
3162 unsigned vsz
= vec_full_reg_size(s
);
3163 TCGv_i64 val
= tcg_temp_new_i64();
3164 GVecGen2sFn
*gvec_fn
= a
->d
? tcg_gen_gvec_subs
: tcg_gen_gvec_adds
;
3166 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3167 gvec_fn(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3168 vec_full_reg_offset(s
, a
->rn
), val
, vsz
, vsz
);
3173 static bool trans_SINCDECP_r_32(DisasContext
*s
, arg_incdec_pred
*a
)
3175 if (sve_access_check(s
)) {
3176 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3177 TCGv_i64 val
= tcg_temp_new_i64();
3179 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3180 do_sat_addsub_32(reg
, val
, a
->u
, a
->d
);
3185 static bool trans_SINCDECP_r_64(DisasContext
*s
, arg_incdec_pred
*a
)
3187 if (sve_access_check(s
)) {
3188 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3189 TCGv_i64 val
= tcg_temp_new_i64();
3191 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3192 do_sat_addsub_64(reg
, val
, a
->u
, a
->d
);
3197 static bool trans_SINCDECP_z(DisasContext
*s
, arg_incdec2_pred
*a
)
3202 if (sve_access_check(s
)) {
3203 TCGv_i64 val
= tcg_temp_new_i64();
3204 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3205 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
, val
, a
->u
, a
->d
);
3211 *** SVE Integer Compare Scalars Group
3214 static bool trans_CTERM(DisasContext
*s
, arg_CTERM
*a
)
3216 if (!sve_access_check(s
)) {
3220 TCGCond cond
= (a
->ne
? TCG_COND_NE
: TCG_COND_EQ
);
3221 TCGv_i64 rn
= read_cpu_reg(s
, a
->rn
, a
->sf
);
3222 TCGv_i64 rm
= read_cpu_reg(s
, a
->rm
, a
->sf
);
3223 TCGv_i64 cmp
= tcg_temp_new_i64();
3225 tcg_gen_setcond_i64(cond
, cmp
, rn
, rm
);
3226 tcg_gen_extrl_i64_i32(cpu_NF
, cmp
);
3227 tcg_temp_free_i64(cmp
);
3229 /* VF = !NF & !CF. */
3230 tcg_gen_xori_i32(cpu_VF
, cpu_NF
, 1);
3231 tcg_gen_andc_i32(cpu_VF
, cpu_VF
, cpu_CF
);
3233 /* Both NF and VF actually look at bit 31. */
3234 tcg_gen_neg_i32(cpu_NF
, cpu_NF
);
3235 tcg_gen_neg_i32(cpu_VF
, cpu_VF
);
3239 static bool trans_WHILE(DisasContext
*s
, arg_WHILE
*a
)
3241 TCGv_i64 op0
, op1
, t0
, t1
, tmax
;
3244 unsigned vsz
= vec_full_reg_size(s
);
3248 /* Note that GE/HS has a->eq == 0 and GT/HI has a->eq == 1. */
3249 bool eq
= a
->eq
== a
->lt
;
3251 /* The greater-than conditions are all SVE2. */
3252 if (!a
->lt
&& !dc_isar_feature(aa64_sve2
, s
)) {
3255 if (!sve_access_check(s
)) {
3259 op0
= read_cpu_reg(s
, a
->rn
, 1);
3260 op1
= read_cpu_reg(s
, a
->rm
, 1);
3264 tcg_gen_ext32u_i64(op0
, op0
);
3265 tcg_gen_ext32u_i64(op1
, op1
);
3267 tcg_gen_ext32s_i64(op0
, op0
);
3268 tcg_gen_ext32s_i64(op1
, op1
);
3272 /* For the helper, compress the different conditions into a computation
3273 * of how many iterations for which the condition is true.
3275 t0
= tcg_temp_new_i64();
3276 t1
= tcg_temp_new_i64();
3279 tcg_gen_sub_i64(t0
, op1
, op0
);
3281 maxval
= a
->sf
? UINT64_MAX
: UINT32_MAX
;
3282 cond
= eq
? TCG_COND_LEU
: TCG_COND_LTU
;
3284 maxval
= a
->sf
? INT64_MAX
: INT32_MAX
;
3285 cond
= eq
? TCG_COND_LE
: TCG_COND_LT
;
3288 tcg_gen_sub_i64(t0
, op0
, op1
);
3291 cond
= eq
? TCG_COND_GEU
: TCG_COND_GTU
;
3293 maxval
= a
->sf
? INT64_MIN
: INT32_MIN
;
3294 cond
= eq
? TCG_COND_GE
: TCG_COND_GT
;
3298 tmax
= tcg_constant_i64(vsz
>> a
->esz
);
3300 /* Equality means one more iteration. */
3301 tcg_gen_addi_i64(t0
, t0
, 1);
3304 * For the less-than while, if op1 is maxval (and the only time
3305 * the addition above could overflow), then we produce an all-true
3306 * predicate by setting the count to the vector length. This is
3307 * because the pseudocode is described as an increment + compare
3308 * loop, and the maximum integer would always compare true.
3309 * Similarly, the greater-than while has the same issue with the
3310 * minimum integer due to the decrement + compare loop.
3312 tcg_gen_movi_i64(t1
, maxval
);
3313 tcg_gen_movcond_i64(TCG_COND_EQ
, t0
, op1
, t1
, tmax
, t0
);
3316 /* Bound to the maximum. */
3317 tcg_gen_umin_i64(t0
, t0
, tmax
);
3319 /* Set the count to zero if the condition is false. */
3320 tcg_gen_movi_i64(t1
, 0);
3321 tcg_gen_movcond_i64(cond
, t0
, op0
, op1
, t0
, t1
);
3322 tcg_temp_free_i64(t1
);
3324 /* Since we're bounded, pass as a 32-bit type. */
3325 t2
= tcg_temp_new_i32();
3326 tcg_gen_extrl_i64_i32(t2
, t0
);
3327 tcg_temp_free_i64(t0
);
3329 /* Scale elements to bits. */
3330 tcg_gen_shli_i32(t2
, t2
, a
->esz
);
3332 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
/ 8);
3333 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
3335 ptr
= tcg_temp_new_ptr();
3336 tcg_gen_addi_ptr(ptr
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3339 gen_helper_sve_whilel(t2
, ptr
, t2
, tcg_constant_i32(desc
));
3341 gen_helper_sve_whileg(t2
, ptr
, t2
, tcg_constant_i32(desc
));
3345 tcg_temp_free_ptr(ptr
);
3346 tcg_temp_free_i32(t2
);
3350 static bool trans_WHILE_ptr(DisasContext
*s
, arg_WHILE_ptr
*a
)
3352 TCGv_i64 op0
, op1
, diff
, t1
, tmax
;
3355 unsigned vsz
= vec_full_reg_size(s
);
3358 if (!dc_isar_feature(aa64_sve2
, s
)) {
3361 if (!sve_access_check(s
)) {
3365 op0
= read_cpu_reg(s
, a
->rn
, 1);
3366 op1
= read_cpu_reg(s
, a
->rm
, 1);
3368 tmax
= tcg_constant_i64(vsz
);
3369 diff
= tcg_temp_new_i64();
3373 /* diff = abs(op1 - op0), noting that op0/1 are unsigned. */
3374 t1
= tcg_temp_new_i64();
3375 tcg_gen_sub_i64(diff
, op0
, op1
);
3376 tcg_gen_sub_i64(t1
, op1
, op0
);
3377 tcg_gen_movcond_i64(TCG_COND_GEU
, diff
, op0
, op1
, diff
, t1
);
3378 tcg_temp_free_i64(t1
);
3379 /* Round down to a multiple of ESIZE. */
3380 tcg_gen_andi_i64(diff
, diff
, -1 << a
->esz
);
3381 /* If op1 == op0, diff == 0, and the condition is always true. */
3382 tcg_gen_movcond_i64(TCG_COND_EQ
, diff
, op0
, op1
, tmax
, diff
);
3385 tcg_gen_sub_i64(diff
, op1
, op0
);
3386 /* Round down to a multiple of ESIZE. */
3387 tcg_gen_andi_i64(diff
, diff
, -1 << a
->esz
);
3388 /* If op0 >= op1, diff <= 0, the condition is always true. */
3389 tcg_gen_movcond_i64(TCG_COND_GEU
, diff
, op0
, op1
, tmax
, diff
);
3392 /* Bound to the maximum. */
3393 tcg_gen_umin_i64(diff
, diff
, tmax
);
3395 /* Since we're bounded, pass as a 32-bit type. */
3396 t2
= tcg_temp_new_i32();
3397 tcg_gen_extrl_i64_i32(t2
, diff
);
3398 tcg_temp_free_i64(diff
);
3400 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
/ 8);
3401 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
3403 ptr
= tcg_temp_new_ptr();
3404 tcg_gen_addi_ptr(ptr
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3406 gen_helper_sve_whilel(t2
, ptr
, t2
, tcg_constant_i32(desc
));
3409 tcg_temp_free_ptr(ptr
);
3410 tcg_temp_free_i32(t2
);
3415 *** SVE Integer Wide Immediate - Unpredicated Group
3418 static bool trans_FDUP(DisasContext
*s
, arg_FDUP
*a
)
3423 if (sve_access_check(s
)) {
3424 unsigned vsz
= vec_full_reg_size(s
);
3425 int dofs
= vec_full_reg_offset(s
, a
->rd
);
3428 /* Decode the VFP immediate. */
3429 imm
= vfp_expand_imm(a
->esz
, a
->imm
);
3430 tcg_gen_gvec_dup_imm(a
->esz
, dofs
, vsz
, vsz
, imm
);
3435 static bool trans_DUP_i(DisasContext
*s
, arg_DUP_i
*a
)
3437 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3440 if (sve_access_check(s
)) {
3441 unsigned vsz
= vec_full_reg_size(s
);
3442 int dofs
= vec_full_reg_offset(s
, a
->rd
);
3444 tcg_gen_gvec_dup_imm(a
->esz
, dofs
, vsz
, vsz
, a
->imm
);
3449 static bool trans_ADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3451 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3454 return gen_gvec_fn_arg_zzi(s
, tcg_gen_gvec_addi
, a
);
3457 static bool trans_SUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3460 return trans_ADD_zzi(s
, a
);
3463 static bool trans_SUBR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3465 static const TCGOpcode vecop_list
[] = { INDEX_op_sub_vec
, 0 };
3466 static const GVecGen2s op
[4] = {
3467 { .fni8
= tcg_gen_vec_sub8_i64
,
3468 .fniv
= tcg_gen_sub_vec
,
3469 .fno
= gen_helper_sve_subri_b
,
3470 .opt_opc
= vecop_list
,
3472 .scalar_first
= true },
3473 { .fni8
= tcg_gen_vec_sub16_i64
,
3474 .fniv
= tcg_gen_sub_vec
,
3475 .fno
= gen_helper_sve_subri_h
,
3476 .opt_opc
= vecop_list
,
3478 .scalar_first
= true },
3479 { .fni4
= tcg_gen_sub_i32
,
3480 .fniv
= tcg_gen_sub_vec
,
3481 .fno
= gen_helper_sve_subri_s
,
3482 .opt_opc
= vecop_list
,
3484 .scalar_first
= true },
3485 { .fni8
= tcg_gen_sub_i64
,
3486 .fniv
= tcg_gen_sub_vec
,
3487 .fno
= gen_helper_sve_subri_d
,
3488 .opt_opc
= vecop_list
,
3489 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
3491 .scalar_first
= true }
3494 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3497 if (sve_access_check(s
)) {
3498 unsigned vsz
= vec_full_reg_size(s
);
3499 tcg_gen_gvec_2s(vec_full_reg_offset(s
, a
->rd
),
3500 vec_full_reg_offset(s
, a
->rn
),
3501 vsz
, vsz
, tcg_constant_i64(a
->imm
), &op
[a
->esz
]);
3506 static bool trans_MUL_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3508 if (sve_access_check(s
)) {
3509 unsigned vsz
= vec_full_reg_size(s
);
3510 tcg_gen_gvec_muli(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3511 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
3516 static bool do_zzi_sat(DisasContext
*s
, arg_rri_esz
*a
, bool u
, bool d
)
3518 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3521 if (sve_access_check(s
)) {
3522 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
,
3523 tcg_constant_i64(a
->imm
), u
, d
);
3528 static bool trans_SQADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3530 return do_zzi_sat(s
, a
, false, false);
3533 static bool trans_UQADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3535 return do_zzi_sat(s
, a
, true, false);
3538 static bool trans_SQSUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3540 return do_zzi_sat(s
, a
, false, true);
3543 static bool trans_UQSUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3545 return do_zzi_sat(s
, a
, true, true);
3548 static bool do_zzi_ool(DisasContext
*s
, arg_rri_esz
*a
, gen_helper_gvec_2i
*fn
)
3550 if (sve_access_check(s
)) {
3551 unsigned vsz
= vec_full_reg_size(s
);
3552 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s
, a
->rd
),
3553 vec_full_reg_offset(s
, a
->rn
),
3554 tcg_constant_i64(a
->imm
), vsz
, vsz
, 0, fn
);
3559 #define DO_ZZI(NAME, name) \
3560 static bool trans_##NAME##_zzi(DisasContext *s, arg_rri_esz *a) \
3562 static gen_helper_gvec_2i * const fns[4] = { \
3563 gen_helper_sve_##name##i_b, gen_helper_sve_##name##i_h, \
3564 gen_helper_sve_##name##i_s, gen_helper_sve_##name##i_d, \
3566 return do_zzi_ool(s, a, fns[a->esz]); \
3576 static gen_helper_gvec_4
* const dot_fns
[2][2] = {
3577 { gen_helper_gvec_sdot_b
, gen_helper_gvec_sdot_h
},
3578 { gen_helper_gvec_udot_b
, gen_helper_gvec_udot_h
}
3580 TRANS_FEAT(DOT_zzzz
, aa64_sve
, gen_gvec_ool_zzzz
,
3581 dot_fns
[a
->u
][a
->sz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, 0)
3584 * SVE Multiply - Indexed
3587 TRANS_FEAT(SDOT_zzxw_s
, aa64_sve
, gen_gvec_ool_arg_zzxz
,
3588 gen_helper_gvec_sdot_idx_b
, a
)
3589 TRANS_FEAT(SDOT_zzxw_d
, aa64_sve
, gen_gvec_ool_arg_zzxz
,
3590 gen_helper_gvec_sdot_idx_h
, a
)
3591 TRANS_FEAT(UDOT_zzxw_s
, aa64_sve
, gen_gvec_ool_arg_zzxz
,
3592 gen_helper_gvec_udot_idx_b
, a
)
3593 TRANS_FEAT(UDOT_zzxw_d
, aa64_sve
, gen_gvec_ool_arg_zzxz
,
3594 gen_helper_gvec_udot_idx_h
, a
)
3596 TRANS_FEAT(SUDOT_zzxw_s
, aa64_sve_i8mm
, gen_gvec_ool_arg_zzxz
,
3597 gen_helper_gvec_sudot_idx_b
, a
)
3598 TRANS_FEAT(USDOT_zzxw_s
, aa64_sve_i8mm
, gen_gvec_ool_arg_zzxz
,
3599 gen_helper_gvec_usdot_idx_b
, a
)
3601 #define DO_SVE2_RRX(NAME, FUNC) \
3602 TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_zzz, FUNC, \
3603 a->rd, a->rn, a->rm, a->index)
3605 DO_SVE2_RRX(MUL_zzx_h
, gen_helper_gvec_mul_idx_h
)
3606 DO_SVE2_RRX(MUL_zzx_s
, gen_helper_gvec_mul_idx_s
)
3607 DO_SVE2_RRX(MUL_zzx_d
, gen_helper_gvec_mul_idx_d
)
3609 DO_SVE2_RRX(SQDMULH_zzx_h
, gen_helper_sve2_sqdmulh_idx_h
)
3610 DO_SVE2_RRX(SQDMULH_zzx_s
, gen_helper_sve2_sqdmulh_idx_s
)
3611 DO_SVE2_RRX(SQDMULH_zzx_d
, gen_helper_sve2_sqdmulh_idx_d
)
3613 DO_SVE2_RRX(SQRDMULH_zzx_h
, gen_helper_sve2_sqrdmulh_idx_h
)
3614 DO_SVE2_RRX(SQRDMULH_zzx_s
, gen_helper_sve2_sqrdmulh_idx_s
)
3615 DO_SVE2_RRX(SQRDMULH_zzx_d
, gen_helper_sve2_sqrdmulh_idx_d
)
3619 #define DO_SVE2_RRX_TB(NAME, FUNC, TOP) \
3620 TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_zzz, FUNC, \
3621 a->rd, a->rn, a->rm, (a->index << 1) | TOP)
3623 DO_SVE2_RRX_TB(SQDMULLB_zzx_s
, gen_helper_sve2_sqdmull_idx_s
, false)
3624 DO_SVE2_RRX_TB(SQDMULLB_zzx_d
, gen_helper_sve2_sqdmull_idx_d
, false)
3625 DO_SVE2_RRX_TB(SQDMULLT_zzx_s
, gen_helper_sve2_sqdmull_idx_s
, true)
3626 DO_SVE2_RRX_TB(SQDMULLT_zzx_d
, gen_helper_sve2_sqdmull_idx_d
, true)
3628 DO_SVE2_RRX_TB(SMULLB_zzx_s
, gen_helper_sve2_smull_idx_s
, false)
3629 DO_SVE2_RRX_TB(SMULLB_zzx_d
, gen_helper_sve2_smull_idx_d
, false)
3630 DO_SVE2_RRX_TB(SMULLT_zzx_s
, gen_helper_sve2_smull_idx_s
, true)
3631 DO_SVE2_RRX_TB(SMULLT_zzx_d
, gen_helper_sve2_smull_idx_d
, true)
3633 DO_SVE2_RRX_TB(UMULLB_zzx_s
, gen_helper_sve2_umull_idx_s
, false)
3634 DO_SVE2_RRX_TB(UMULLB_zzx_d
, gen_helper_sve2_umull_idx_d
, false)
3635 DO_SVE2_RRX_TB(UMULLT_zzx_s
, gen_helper_sve2_umull_idx_s
, true)
3636 DO_SVE2_RRX_TB(UMULLT_zzx_d
, gen_helper_sve2_umull_idx_d
, true)
3638 #undef DO_SVE2_RRX_TB
3640 #define DO_SVE2_RRXR(NAME, FUNC) \
3641 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_arg_zzxz, FUNC, a)
3643 DO_SVE2_RRXR(MLA_zzxz_h
, gen_helper_gvec_mla_idx_h
)
3644 DO_SVE2_RRXR(MLA_zzxz_s
, gen_helper_gvec_mla_idx_s
)
3645 DO_SVE2_RRXR(MLA_zzxz_d
, gen_helper_gvec_mla_idx_d
)
3647 DO_SVE2_RRXR(MLS_zzxz_h
, gen_helper_gvec_mls_idx_h
)
3648 DO_SVE2_RRXR(MLS_zzxz_s
, gen_helper_gvec_mls_idx_s
)
3649 DO_SVE2_RRXR(MLS_zzxz_d
, gen_helper_gvec_mls_idx_d
)
3651 DO_SVE2_RRXR(SQRDMLAH_zzxz_h
, gen_helper_sve2_sqrdmlah_idx_h
)
3652 DO_SVE2_RRXR(SQRDMLAH_zzxz_s
, gen_helper_sve2_sqrdmlah_idx_s
)
3653 DO_SVE2_RRXR(SQRDMLAH_zzxz_d
, gen_helper_sve2_sqrdmlah_idx_d
)
3655 DO_SVE2_RRXR(SQRDMLSH_zzxz_h
, gen_helper_sve2_sqrdmlsh_idx_h
)
3656 DO_SVE2_RRXR(SQRDMLSH_zzxz_s
, gen_helper_sve2_sqrdmlsh_idx_s
)
3657 DO_SVE2_RRXR(SQRDMLSH_zzxz_d
, gen_helper_sve2_sqrdmlsh_idx_d
)
3661 #define DO_SVE2_RRXR_TB(NAME, FUNC, TOP) \
3662 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_zzzz, FUNC, \
3663 a->rd, a->rn, a->rm, a->ra, (a->index << 1) | TOP)
3665 DO_SVE2_RRXR_TB(SQDMLALB_zzxw_s
, gen_helper_sve2_sqdmlal_idx_s
, false)
3666 DO_SVE2_RRXR_TB(SQDMLALB_zzxw_d
, gen_helper_sve2_sqdmlal_idx_d
, false)
3667 DO_SVE2_RRXR_TB(SQDMLALT_zzxw_s
, gen_helper_sve2_sqdmlal_idx_s
, true)
3668 DO_SVE2_RRXR_TB(SQDMLALT_zzxw_d
, gen_helper_sve2_sqdmlal_idx_d
, true)
3670 DO_SVE2_RRXR_TB(SQDMLSLB_zzxw_s
, gen_helper_sve2_sqdmlsl_idx_s
, false)
3671 DO_SVE2_RRXR_TB(SQDMLSLB_zzxw_d
, gen_helper_sve2_sqdmlsl_idx_d
, false)
3672 DO_SVE2_RRXR_TB(SQDMLSLT_zzxw_s
, gen_helper_sve2_sqdmlsl_idx_s
, true)
3673 DO_SVE2_RRXR_TB(SQDMLSLT_zzxw_d
, gen_helper_sve2_sqdmlsl_idx_d
, true)
3675 DO_SVE2_RRXR_TB(SMLALB_zzxw_s
, gen_helper_sve2_smlal_idx_s
, false)
3676 DO_SVE2_RRXR_TB(SMLALB_zzxw_d
, gen_helper_sve2_smlal_idx_d
, false)
3677 DO_SVE2_RRXR_TB(SMLALT_zzxw_s
, gen_helper_sve2_smlal_idx_s
, true)
3678 DO_SVE2_RRXR_TB(SMLALT_zzxw_d
, gen_helper_sve2_smlal_idx_d
, true)
3680 DO_SVE2_RRXR_TB(UMLALB_zzxw_s
, gen_helper_sve2_umlal_idx_s
, false)
3681 DO_SVE2_RRXR_TB(UMLALB_zzxw_d
, gen_helper_sve2_umlal_idx_d
, false)
3682 DO_SVE2_RRXR_TB(UMLALT_zzxw_s
, gen_helper_sve2_umlal_idx_s
, true)
3683 DO_SVE2_RRXR_TB(UMLALT_zzxw_d
, gen_helper_sve2_umlal_idx_d
, true)
3685 DO_SVE2_RRXR_TB(SMLSLB_zzxw_s
, gen_helper_sve2_smlsl_idx_s
, false)
3686 DO_SVE2_RRXR_TB(SMLSLB_zzxw_d
, gen_helper_sve2_smlsl_idx_d
, false)
3687 DO_SVE2_RRXR_TB(SMLSLT_zzxw_s
, gen_helper_sve2_smlsl_idx_s
, true)
3688 DO_SVE2_RRXR_TB(SMLSLT_zzxw_d
, gen_helper_sve2_smlsl_idx_d
, true)
3690 DO_SVE2_RRXR_TB(UMLSLB_zzxw_s
, gen_helper_sve2_umlsl_idx_s
, false)
3691 DO_SVE2_RRXR_TB(UMLSLB_zzxw_d
, gen_helper_sve2_umlsl_idx_d
, false)
3692 DO_SVE2_RRXR_TB(UMLSLT_zzxw_s
, gen_helper_sve2_umlsl_idx_s
, true)
3693 DO_SVE2_RRXR_TB(UMLSLT_zzxw_d
, gen_helper_sve2_umlsl_idx_d
, true)
3695 #undef DO_SVE2_RRXR_TB
3697 #define DO_SVE2_RRXR_ROT(NAME, FUNC) \
3698 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_zzzz, FUNC, \
3699 a->rd, a->rn, a->rm, a->ra, (a->index << 2) | a->rot)
3701 DO_SVE2_RRXR_ROT(CMLA_zzxz_h
, gen_helper_sve2_cmla_idx_h
)
3702 DO_SVE2_RRXR_ROT(CMLA_zzxz_s
, gen_helper_sve2_cmla_idx_s
)
3704 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_h
, gen_helper_sve2_sqrdcmlah_idx_h
)
3705 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_s
, gen_helper_sve2_sqrdcmlah_idx_s
)
3707 DO_SVE2_RRXR_ROT(CDOT_zzxw_s
, gen_helper_sve2_cdot_idx_s
)
3708 DO_SVE2_RRXR_ROT(CDOT_zzxw_d
, gen_helper_sve2_cdot_idx_d
)
3710 #undef DO_SVE2_RRXR_ROT
3713 *** SVE Floating Point Multiply-Add Indexed Group
3716 static bool do_FMLA_zzxz(DisasContext
*s
, arg_rrxr_esz
*a
, bool sub
)
3718 static gen_helper_gvec_4_ptr
* const fns
[3] = {
3719 gen_helper_gvec_fmla_idx_h
,
3720 gen_helper_gvec_fmla_idx_s
,
3721 gen_helper_gvec_fmla_idx_d
,
3724 if (sve_access_check(s
)) {
3725 unsigned vsz
= vec_full_reg_size(s
);
3726 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3727 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
3728 vec_full_reg_offset(s
, a
->rn
),
3729 vec_full_reg_offset(s
, a
->rm
),
3730 vec_full_reg_offset(s
, a
->ra
),
3731 status
, vsz
, vsz
, (a
->index
<< 1) | sub
,
3733 tcg_temp_free_ptr(status
);
3738 static bool trans_FMLA_zzxz(DisasContext
*s
, arg_FMLA_zzxz
*a
)
3740 return do_FMLA_zzxz(s
, a
, false);
3743 static bool trans_FMLS_zzxz(DisasContext
*s
, arg_FMLA_zzxz
*a
)
3745 return do_FMLA_zzxz(s
, a
, true);
3749 *** SVE Floating Point Multiply Indexed Group
3752 static bool trans_FMUL_zzx(DisasContext
*s
, arg_FMUL_zzx
*a
)
3754 static gen_helper_gvec_3_ptr
* const fns
[3] = {
3755 gen_helper_gvec_fmul_idx_h
,
3756 gen_helper_gvec_fmul_idx_s
,
3757 gen_helper_gvec_fmul_idx_d
,
3760 if (sve_access_check(s
)) {
3761 unsigned vsz
= vec_full_reg_size(s
);
3762 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3763 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
3764 vec_full_reg_offset(s
, a
->rn
),
3765 vec_full_reg_offset(s
, a
->rm
),
3766 status
, vsz
, vsz
, a
->index
, fns
[a
->esz
- 1]);
3767 tcg_temp_free_ptr(status
);
3773 *** SVE Floating Point Fast Reduction Group
3776 typedef void gen_helper_fp_reduce(TCGv_i64
, TCGv_ptr
, TCGv_ptr
,
3777 TCGv_ptr
, TCGv_i32
);
3779 static void do_reduce(DisasContext
*s
, arg_rpr_esz
*a
,
3780 gen_helper_fp_reduce
*fn
)
3782 unsigned vsz
= vec_full_reg_size(s
);
3783 unsigned p2vsz
= pow2ceil(vsz
);
3784 TCGv_i32 t_desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, p2vsz
));
3785 TCGv_ptr t_zn
, t_pg
, status
;
3788 temp
= tcg_temp_new_i64();
3789 t_zn
= tcg_temp_new_ptr();
3790 t_pg
= tcg_temp_new_ptr();
3792 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
3793 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3794 status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3796 fn(temp
, t_zn
, t_pg
, status
, t_desc
);
3797 tcg_temp_free_ptr(t_zn
);
3798 tcg_temp_free_ptr(t_pg
);
3799 tcg_temp_free_ptr(status
);
3801 write_fp_dreg(s
, a
->rd
, temp
);
3802 tcg_temp_free_i64(temp
);
3805 #define DO_VPZ(NAME, name) \
3806 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
3808 static gen_helper_fp_reduce * const fns[3] = { \
3809 gen_helper_sve_##name##_h, \
3810 gen_helper_sve_##name##_s, \
3811 gen_helper_sve_##name##_d, \
3813 if (a->esz == 0) { \
3816 if (sve_access_check(s)) { \
3817 do_reduce(s, a, fns[a->esz - 1]); \
3822 DO_VPZ(FADDV
, faddv
)
3823 DO_VPZ(FMINNMV
, fminnmv
)
3824 DO_VPZ(FMAXNMV
, fmaxnmv
)
3825 DO_VPZ(FMINV
, fminv
)
3826 DO_VPZ(FMAXV
, fmaxv
)
3829 *** SVE Floating Point Unary Operations - Unpredicated Group
3832 static void do_zz_fp(DisasContext
*s
, arg_rr_esz
*a
, gen_helper_gvec_2_ptr
*fn
)
3834 unsigned vsz
= vec_full_reg_size(s
);
3835 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3837 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s
, a
->rd
),
3838 vec_full_reg_offset(s
, a
->rn
),
3839 status
, vsz
, vsz
, 0, fn
);
3840 tcg_temp_free_ptr(status
);
3843 static bool trans_FRECPE(DisasContext
*s
, arg_rr_esz
*a
)
3845 static gen_helper_gvec_2_ptr
* const fns
[3] = {
3846 gen_helper_gvec_frecpe_h
,
3847 gen_helper_gvec_frecpe_s
,
3848 gen_helper_gvec_frecpe_d
,
3853 if (sve_access_check(s
)) {
3854 do_zz_fp(s
, a
, fns
[a
->esz
- 1]);
3859 static bool trans_FRSQRTE(DisasContext
*s
, arg_rr_esz
*a
)
3861 static gen_helper_gvec_2_ptr
* const fns
[3] = {
3862 gen_helper_gvec_frsqrte_h
,
3863 gen_helper_gvec_frsqrte_s
,
3864 gen_helper_gvec_frsqrte_d
,
3869 if (sve_access_check(s
)) {
3870 do_zz_fp(s
, a
, fns
[a
->esz
- 1]);
3876 *** SVE Floating Point Compare with Zero Group
3879 static void do_ppz_fp(DisasContext
*s
, arg_rpr_esz
*a
,
3880 gen_helper_gvec_3_ptr
*fn
)
3882 unsigned vsz
= vec_full_reg_size(s
);
3883 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3885 tcg_gen_gvec_3_ptr(pred_full_reg_offset(s
, a
->rd
),
3886 vec_full_reg_offset(s
, a
->rn
),
3887 pred_full_reg_offset(s
, a
->pg
),
3888 status
, vsz
, vsz
, 0, fn
);
3889 tcg_temp_free_ptr(status
);
3892 #define DO_PPZ(NAME, name) \
3893 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
3895 static gen_helper_gvec_3_ptr * const fns[3] = { \
3896 gen_helper_sve_##name##_h, \
3897 gen_helper_sve_##name##_s, \
3898 gen_helper_sve_##name##_d, \
3900 if (a->esz == 0) { \
3903 if (sve_access_check(s)) { \
3904 do_ppz_fp(s, a, fns[a->esz - 1]); \
3909 DO_PPZ(FCMGE_ppz0
, fcmge0
)
3910 DO_PPZ(FCMGT_ppz0
, fcmgt0
)
3911 DO_PPZ(FCMLE_ppz0
, fcmle0
)
3912 DO_PPZ(FCMLT_ppz0
, fcmlt0
)
3913 DO_PPZ(FCMEQ_ppz0
, fcmeq0
)
3914 DO_PPZ(FCMNE_ppz0
, fcmne0
)
3919 *** SVE floating-point trig multiply-add coefficient
3922 static bool trans_FTMAD(DisasContext
*s
, arg_FTMAD
*a
)
3924 static gen_helper_gvec_3_ptr
* const fns
[3] = {
3925 gen_helper_sve_ftmad_h
,
3926 gen_helper_sve_ftmad_s
,
3927 gen_helper_sve_ftmad_d
,
3933 if (sve_access_check(s
)) {
3934 unsigned vsz
= vec_full_reg_size(s
);
3935 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3936 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
3937 vec_full_reg_offset(s
, a
->rn
),
3938 vec_full_reg_offset(s
, a
->rm
),
3939 status
, vsz
, vsz
, a
->imm
, fns
[a
->esz
- 1]);
3940 tcg_temp_free_ptr(status
);
3946 *** SVE Floating Point Accumulating Reduction Group
3949 static bool trans_FADDA(DisasContext
*s
, arg_rprr_esz
*a
)
3951 typedef void fadda_fn(TCGv_i64
, TCGv_i64
, TCGv_ptr
,
3952 TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
3953 static fadda_fn
* const fns
[3] = {
3954 gen_helper_sve_fadda_h
,
3955 gen_helper_sve_fadda_s
,
3956 gen_helper_sve_fadda_d
,
3958 unsigned vsz
= vec_full_reg_size(s
);
3959 TCGv_ptr t_rm
, t_pg
, t_fpst
;
3966 if (!sve_access_check(s
)) {
3970 t_val
= load_esz(cpu_env
, vec_reg_offset(s
, a
->rn
, 0, a
->esz
), a
->esz
);
3971 t_rm
= tcg_temp_new_ptr();
3972 t_pg
= tcg_temp_new_ptr();
3973 tcg_gen_addi_ptr(t_rm
, cpu_env
, vec_full_reg_offset(s
, a
->rm
));
3974 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3975 t_fpst
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3976 t_desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
3978 fns
[a
->esz
- 1](t_val
, t_val
, t_rm
, t_pg
, t_fpst
, t_desc
);
3980 tcg_temp_free_ptr(t_fpst
);
3981 tcg_temp_free_ptr(t_pg
);
3982 tcg_temp_free_ptr(t_rm
);
3984 write_fp_dreg(s
, a
->rd
, t_val
);
3985 tcg_temp_free_i64(t_val
);
3990 *** SVE Floating Point Arithmetic - Unpredicated Group
3993 static bool do_zzz_fp(DisasContext
*s
, arg_rrr_esz
*a
,
3994 gen_helper_gvec_3_ptr
*fn
)
3999 if (sve_access_check(s
)) {
4000 unsigned vsz
= vec_full_reg_size(s
);
4001 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4002 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4003 vec_full_reg_offset(s
, a
->rn
),
4004 vec_full_reg_offset(s
, a
->rm
),
4005 status
, vsz
, vsz
, 0, fn
);
4006 tcg_temp_free_ptr(status
);
4012 #define DO_FP3(NAME, name) \
4013 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
4015 static gen_helper_gvec_3_ptr * const fns[4] = { \
4016 NULL, gen_helper_gvec_##name##_h, \
4017 gen_helper_gvec_##name##_s, gen_helper_gvec_##name##_d \
4019 return do_zzz_fp(s, a, fns[a->esz]); \
4022 DO_FP3(FADD_zzz
, fadd
)
4023 DO_FP3(FSUB_zzz
, fsub
)
4024 DO_FP3(FMUL_zzz
, fmul
)
4025 DO_FP3(FTSMUL
, ftsmul
)
4026 DO_FP3(FRECPS
, recps
)
4027 DO_FP3(FRSQRTS
, rsqrts
)
4032 *** SVE Floating Point Arithmetic - Predicated Group
4035 static bool do_zpzz_fp(DisasContext
*s
, arg_rprr_esz
*a
,
4036 gen_helper_gvec_4_ptr
*fn
)
4041 if (sve_access_check(s
)) {
4042 unsigned vsz
= vec_full_reg_size(s
);
4043 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4044 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4045 vec_full_reg_offset(s
, a
->rn
),
4046 vec_full_reg_offset(s
, a
->rm
),
4047 pred_full_reg_offset(s
, a
->pg
),
4048 status
, vsz
, vsz
, 0, fn
);
4049 tcg_temp_free_ptr(status
);
4054 #define DO_FP3(NAME, name) \
4055 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
4057 static gen_helper_gvec_4_ptr * const fns[4] = { \
4058 NULL, gen_helper_sve_##name##_h, \
4059 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4061 return do_zpzz_fp(s, a, fns[a->esz]); \
4064 DO_FP3(FADD_zpzz
, fadd
)
4065 DO_FP3(FSUB_zpzz
, fsub
)
4066 DO_FP3(FMUL_zpzz
, fmul
)
4067 DO_FP3(FMIN_zpzz
, fmin
)
4068 DO_FP3(FMAX_zpzz
, fmax
)
4069 DO_FP3(FMINNM_zpzz
, fminnum
)
4070 DO_FP3(FMAXNM_zpzz
, fmaxnum
)
4072 DO_FP3(FSCALE
, fscalbn
)
4074 DO_FP3(FMULX
, fmulx
)
4078 typedef void gen_helper_sve_fp2scalar(TCGv_ptr
, TCGv_ptr
, TCGv_ptr
,
4079 TCGv_i64
, TCGv_ptr
, TCGv_i32
);
4081 static void do_fp_scalar(DisasContext
*s
, int zd
, int zn
, int pg
, bool is_fp16
,
4082 TCGv_i64 scalar
, gen_helper_sve_fp2scalar
*fn
)
4084 unsigned vsz
= vec_full_reg_size(s
);
4085 TCGv_ptr t_zd
, t_zn
, t_pg
, status
;
4088 t_zd
= tcg_temp_new_ptr();
4089 t_zn
= tcg_temp_new_ptr();
4090 t_pg
= tcg_temp_new_ptr();
4091 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, zd
));
4092 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, zn
));
4093 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
4095 status
= fpstatus_ptr(is_fp16
? FPST_FPCR_F16
: FPST_FPCR
);
4096 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
4097 fn(t_zd
, t_zn
, t_pg
, scalar
, status
, desc
);
4099 tcg_temp_free_ptr(status
);
4100 tcg_temp_free_ptr(t_pg
);
4101 tcg_temp_free_ptr(t_zn
);
4102 tcg_temp_free_ptr(t_zd
);
4105 static void do_fp_imm(DisasContext
*s
, arg_rpri_esz
*a
, uint64_t imm
,
4106 gen_helper_sve_fp2scalar
*fn
)
4108 do_fp_scalar(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
,
4109 tcg_constant_i64(imm
), fn
);
4112 #define DO_FP_IMM(NAME, name, const0, const1) \
4113 static bool trans_##NAME##_zpzi(DisasContext *s, arg_rpri_esz *a) \
4115 static gen_helper_sve_fp2scalar * const fns[3] = { \
4116 gen_helper_sve_##name##_h, \
4117 gen_helper_sve_##name##_s, \
4118 gen_helper_sve_##name##_d \
4120 static uint64_t const val[3][2] = { \
4121 { float16_##const0, float16_##const1 }, \
4122 { float32_##const0, float32_##const1 }, \
4123 { float64_##const0, float64_##const1 }, \
4125 if (a->esz == 0) { \
4128 if (sve_access_check(s)) { \
4129 do_fp_imm(s, a, val[a->esz - 1][a->imm], fns[a->esz - 1]); \
4134 DO_FP_IMM(FADD
, fadds
, half
, one
)
4135 DO_FP_IMM(FSUB
, fsubs
, half
, one
)
4136 DO_FP_IMM(FMUL
, fmuls
, half
, two
)
4137 DO_FP_IMM(FSUBR
, fsubrs
, half
, one
)
4138 DO_FP_IMM(FMAXNM
, fmaxnms
, zero
, one
)
4139 DO_FP_IMM(FMINNM
, fminnms
, zero
, one
)
4140 DO_FP_IMM(FMAX
, fmaxs
, zero
, one
)
4141 DO_FP_IMM(FMIN
, fmins
, zero
, one
)
4145 static bool do_fp_cmp(DisasContext
*s
, arg_rprr_esz
*a
,
4146 gen_helper_gvec_4_ptr
*fn
)
4151 if (sve_access_check(s
)) {
4152 unsigned vsz
= vec_full_reg_size(s
);
4153 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4154 tcg_gen_gvec_4_ptr(pred_full_reg_offset(s
, a
->rd
),
4155 vec_full_reg_offset(s
, a
->rn
),
4156 vec_full_reg_offset(s
, a
->rm
),
4157 pred_full_reg_offset(s
, a
->pg
),
4158 status
, vsz
, vsz
, 0, fn
);
4159 tcg_temp_free_ptr(status
);
4164 #define DO_FPCMP(NAME, name) \
4165 static bool trans_##NAME##_ppzz(DisasContext *s, arg_rprr_esz *a) \
4167 static gen_helper_gvec_4_ptr * const fns[4] = { \
4168 NULL, gen_helper_sve_##name##_h, \
4169 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4171 return do_fp_cmp(s, a, fns[a->esz]); \
4174 DO_FPCMP(FCMGE
, fcmge
)
4175 DO_FPCMP(FCMGT
, fcmgt
)
4176 DO_FPCMP(FCMEQ
, fcmeq
)
4177 DO_FPCMP(FCMNE
, fcmne
)
4178 DO_FPCMP(FCMUO
, fcmuo
)
4179 DO_FPCMP(FACGE
, facge
)
4180 DO_FPCMP(FACGT
, facgt
)
4184 static bool trans_FCADD(DisasContext
*s
, arg_FCADD
*a
)
4186 static gen_helper_gvec_4_ptr
* const fns
[3] = {
4187 gen_helper_sve_fcadd_h
,
4188 gen_helper_sve_fcadd_s
,
4189 gen_helper_sve_fcadd_d
4195 if (sve_access_check(s
)) {
4196 unsigned vsz
= vec_full_reg_size(s
);
4197 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4198 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4199 vec_full_reg_offset(s
, a
->rn
),
4200 vec_full_reg_offset(s
, a
->rm
),
4201 pred_full_reg_offset(s
, a
->pg
),
4202 status
, vsz
, vsz
, a
->rot
, fns
[a
->esz
- 1]);
4203 tcg_temp_free_ptr(status
);
4208 static bool do_fmla(DisasContext
*s
, arg_rprrr_esz
*a
,
4209 gen_helper_gvec_5_ptr
*fn
)
4214 if (sve_access_check(s
)) {
4215 unsigned vsz
= vec_full_reg_size(s
);
4216 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4217 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s
, a
->rd
),
4218 vec_full_reg_offset(s
, a
->rn
),
4219 vec_full_reg_offset(s
, a
->rm
),
4220 vec_full_reg_offset(s
, a
->ra
),
4221 pred_full_reg_offset(s
, a
->pg
),
4222 status
, vsz
, vsz
, 0, fn
);
4223 tcg_temp_free_ptr(status
);
4228 #define DO_FMLA(NAME, name) \
4229 static bool trans_##NAME(DisasContext *s, arg_rprrr_esz *a) \
4231 static gen_helper_gvec_5_ptr * const fns[4] = { \
4232 NULL, gen_helper_sve_##name##_h, \
4233 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4235 return do_fmla(s, a, fns[a->esz]); \
4238 DO_FMLA(FMLA_zpzzz
, fmla_zpzzz
)
4239 DO_FMLA(FMLS_zpzzz
, fmls_zpzzz
)
4240 DO_FMLA(FNMLA_zpzzz
, fnmla_zpzzz
)
4241 DO_FMLA(FNMLS_zpzzz
, fnmls_zpzzz
)
4245 static bool trans_FCMLA_zpzzz(DisasContext
*s
, arg_FCMLA_zpzzz
*a
)
4247 static gen_helper_gvec_5_ptr
* const fns
[4] = {
4249 gen_helper_sve_fcmla_zpzzz_h
,
4250 gen_helper_sve_fcmla_zpzzz_s
,
4251 gen_helper_sve_fcmla_zpzzz_d
,
4257 if (sve_access_check(s
)) {
4258 unsigned vsz
= vec_full_reg_size(s
);
4259 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4260 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s
, a
->rd
),
4261 vec_full_reg_offset(s
, a
->rn
),
4262 vec_full_reg_offset(s
, a
->rm
),
4263 vec_full_reg_offset(s
, a
->ra
),
4264 pred_full_reg_offset(s
, a
->pg
),
4265 status
, vsz
, vsz
, a
->rot
, fns
[a
->esz
]);
4266 tcg_temp_free_ptr(status
);
4271 static bool trans_FCMLA_zzxz(DisasContext
*s
, arg_FCMLA_zzxz
*a
)
4273 static gen_helper_gvec_4_ptr
* const fns
[2] = {
4274 gen_helper_gvec_fcmlah_idx
,
4275 gen_helper_gvec_fcmlas_idx
,
4278 tcg_debug_assert(a
->esz
== 1 || a
->esz
== 2);
4279 tcg_debug_assert(a
->rd
== a
->ra
);
4280 if (sve_access_check(s
)) {
4281 unsigned vsz
= vec_full_reg_size(s
);
4282 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4283 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4284 vec_full_reg_offset(s
, a
->rn
),
4285 vec_full_reg_offset(s
, a
->rm
),
4286 vec_full_reg_offset(s
, a
->ra
),
4288 a
->index
* 4 + a
->rot
,
4290 tcg_temp_free_ptr(status
);
4296 *** SVE Floating Point Unary Operations Predicated Group
4299 static bool do_zpz_ptr(DisasContext
*s
, int rd
, int rn
, int pg
,
4300 bool is_fp16
, gen_helper_gvec_3_ptr
*fn
)
4302 if (sve_access_check(s
)) {
4303 unsigned vsz
= vec_full_reg_size(s
);
4304 TCGv_ptr status
= fpstatus_ptr(is_fp16
? FPST_FPCR_F16
: FPST_FPCR
);
4305 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, rd
),
4306 vec_full_reg_offset(s
, rn
),
4307 pred_full_reg_offset(s
, pg
),
4308 status
, vsz
, vsz
, 0, fn
);
4309 tcg_temp_free_ptr(status
);
4314 static bool trans_FCVT_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4316 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_sh
);
4319 static bool trans_FCVT_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4321 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_hs
);
4324 static bool trans_BFCVT(DisasContext
*s
, arg_rpr_esz
*a
)
4326 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
4329 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_bfcvt
);
4332 static bool trans_FCVT_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4334 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_dh
);
4337 static bool trans_FCVT_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4339 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_hd
);
4342 static bool trans_FCVT_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4344 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_ds
);
4347 static bool trans_FCVT_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4349 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_sd
);
4352 static bool trans_FCVTZS_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4354 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hh
);
4357 static bool trans_FCVTZU_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4359 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hh
);
4362 static bool trans_FCVTZS_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4364 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hs
);
4367 static bool trans_FCVTZU_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4369 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hs
);
4372 static bool trans_FCVTZS_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4374 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hd
);
4377 static bool trans_FCVTZU_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4379 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hd
);
4382 static bool trans_FCVTZS_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4384 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_ss
);
4387 static bool trans_FCVTZU_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4389 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_ss
);
4392 static bool trans_FCVTZS_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4394 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_sd
);
4397 static bool trans_FCVTZU_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4399 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_sd
);
4402 static bool trans_FCVTZS_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4404 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_ds
);
4407 static bool trans_FCVTZU_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4409 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_ds
);
4412 static bool trans_FCVTZS_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4414 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_dd
);
4417 static bool trans_FCVTZU_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4419 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_dd
);
4422 static gen_helper_gvec_3_ptr
* const frint_fns
[3] = {
4423 gen_helper_sve_frint_h
,
4424 gen_helper_sve_frint_s
,
4425 gen_helper_sve_frint_d
4428 static bool trans_FRINTI(DisasContext
*s
, arg_rpr_esz
*a
)
4433 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
,
4434 frint_fns
[a
->esz
- 1]);
4437 static bool trans_FRINTX(DisasContext
*s
, arg_rpr_esz
*a
)
4439 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4440 gen_helper_sve_frintx_h
,
4441 gen_helper_sve_frintx_s
,
4442 gen_helper_sve_frintx_d
4447 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4450 static bool do_frint_mode(DisasContext
*s
, arg_rpr_esz
*a
,
4451 int mode
, gen_helper_gvec_3_ptr
*fn
)
4453 if (sve_access_check(s
)) {
4454 unsigned vsz
= vec_full_reg_size(s
);
4455 TCGv_i32 tmode
= tcg_const_i32(mode
);
4456 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4458 gen_helper_set_rmode(tmode
, tmode
, status
);
4460 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4461 vec_full_reg_offset(s
, a
->rn
),
4462 pred_full_reg_offset(s
, a
->pg
),
4463 status
, vsz
, vsz
, 0, fn
);
4465 gen_helper_set_rmode(tmode
, tmode
, status
);
4466 tcg_temp_free_i32(tmode
);
4467 tcg_temp_free_ptr(status
);
4472 static bool trans_FRINTN(DisasContext
*s
, arg_rpr_esz
*a
)
4477 return do_frint_mode(s
, a
, float_round_nearest_even
, frint_fns
[a
->esz
- 1]);
4480 static bool trans_FRINTP(DisasContext
*s
, arg_rpr_esz
*a
)
4485 return do_frint_mode(s
, a
, float_round_up
, frint_fns
[a
->esz
- 1]);
4488 static bool trans_FRINTM(DisasContext
*s
, arg_rpr_esz
*a
)
4493 return do_frint_mode(s
, a
, float_round_down
, frint_fns
[a
->esz
- 1]);
4496 static bool trans_FRINTZ(DisasContext
*s
, arg_rpr_esz
*a
)
4501 return do_frint_mode(s
, a
, float_round_to_zero
, frint_fns
[a
->esz
- 1]);
4504 static bool trans_FRINTA(DisasContext
*s
, arg_rpr_esz
*a
)
4509 return do_frint_mode(s
, a
, float_round_ties_away
, frint_fns
[a
->esz
- 1]);
4512 static bool trans_FRECPX(DisasContext
*s
, arg_rpr_esz
*a
)
4514 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4515 gen_helper_sve_frecpx_h
,
4516 gen_helper_sve_frecpx_s
,
4517 gen_helper_sve_frecpx_d
4522 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4525 static bool trans_FSQRT(DisasContext
*s
, arg_rpr_esz
*a
)
4527 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4528 gen_helper_sve_fsqrt_h
,
4529 gen_helper_sve_fsqrt_s
,
4530 gen_helper_sve_fsqrt_d
4535 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4538 static bool trans_SCVTF_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4540 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_hh
);
4543 static bool trans_SCVTF_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4545 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_sh
);
4548 static bool trans_SCVTF_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4550 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_dh
);
4553 static bool trans_SCVTF_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4555 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_ss
);
4558 static bool trans_SCVTF_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4560 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_ds
);
4563 static bool trans_SCVTF_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4565 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_sd
);
4568 static bool trans_SCVTF_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4570 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_dd
);
4573 static bool trans_UCVTF_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4575 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_hh
);
4578 static bool trans_UCVTF_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4580 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_sh
);
4583 static bool trans_UCVTF_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4585 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_dh
);
4588 static bool trans_UCVTF_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4590 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_ss
);
4593 static bool trans_UCVTF_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4595 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_ds
);
4598 static bool trans_UCVTF_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4600 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_sd
);
4603 static bool trans_UCVTF_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4605 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_dd
);
4609 *** SVE Memory - 32-bit Gather and Unsized Contiguous Group
4612 /* Subroutine loading a vector register at VOFS of LEN bytes.
4613 * The load should begin at the address Rn + IMM.
4616 static void do_ldr(DisasContext
*s
, uint32_t vofs
, int len
, int rn
, int imm
)
4618 int len_align
= QEMU_ALIGN_DOWN(len
, 8);
4619 int len_remain
= len
% 8;
4620 int nparts
= len
/ 8 + ctpop8(len_remain
);
4621 int midx
= get_mem_index(s
);
4622 TCGv_i64 dirty_addr
, clean_addr
, t0
, t1
;
4624 dirty_addr
= tcg_temp_new_i64();
4625 tcg_gen_addi_i64(dirty_addr
, cpu_reg_sp(s
, rn
), imm
);
4626 clean_addr
= gen_mte_checkN(s
, dirty_addr
, false, rn
!= 31, len
);
4627 tcg_temp_free_i64(dirty_addr
);
4630 * Note that unpredicated load/store of vector/predicate registers
4631 * are defined as a stream of bytes, which equates to little-endian
4632 * operations on larger quantities.
4633 * Attempt to keep code expansion to a minimum by limiting the
4634 * amount of unrolling done.
4639 t0
= tcg_temp_new_i64();
4640 for (i
= 0; i
< len_align
; i
+= 8) {
4641 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEUQ
);
4642 tcg_gen_st_i64(t0
, cpu_env
, vofs
+ i
);
4643 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4645 tcg_temp_free_i64(t0
);
4647 TCGLabel
*loop
= gen_new_label();
4648 TCGv_ptr tp
, i
= tcg_const_local_ptr(0);
4650 /* Copy the clean address into a local temp, live across the loop. */
4652 clean_addr
= new_tmp_a64_local(s
);
4653 tcg_gen_mov_i64(clean_addr
, t0
);
4655 gen_set_label(loop
);
4657 t0
= tcg_temp_new_i64();
4658 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEUQ
);
4659 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4661 tp
= tcg_temp_new_ptr();
4662 tcg_gen_add_ptr(tp
, cpu_env
, i
);
4663 tcg_gen_addi_ptr(i
, i
, 8);
4664 tcg_gen_st_i64(t0
, tp
, vofs
);
4665 tcg_temp_free_ptr(tp
);
4666 tcg_temp_free_i64(t0
);
4668 tcg_gen_brcondi_ptr(TCG_COND_LTU
, i
, len_align
, loop
);
4669 tcg_temp_free_ptr(i
);
4673 * Predicate register loads can be any multiple of 2.
4674 * Note that we still store the entire 64-bit unit into cpu_env.
4677 t0
= tcg_temp_new_i64();
4678 switch (len_remain
) {
4682 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
,
4683 MO_LE
| ctz32(len_remain
));
4687 t1
= tcg_temp_new_i64();
4688 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEUL
);
4689 tcg_gen_addi_i64(clean_addr
, clean_addr
, 4);
4690 tcg_gen_qemu_ld_i64(t1
, clean_addr
, midx
, MO_LEUW
);
4691 tcg_gen_deposit_i64(t0
, t0
, t1
, 32, 32);
4692 tcg_temp_free_i64(t1
);
4696 g_assert_not_reached();
4698 tcg_gen_st_i64(t0
, cpu_env
, vofs
+ len_align
);
4699 tcg_temp_free_i64(t0
);
4703 /* Similarly for stores. */
4704 static void do_str(DisasContext
*s
, uint32_t vofs
, int len
, int rn
, int imm
)
4706 int len_align
= QEMU_ALIGN_DOWN(len
, 8);
4707 int len_remain
= len
% 8;
4708 int nparts
= len
/ 8 + ctpop8(len_remain
);
4709 int midx
= get_mem_index(s
);
4710 TCGv_i64 dirty_addr
, clean_addr
, t0
;
4712 dirty_addr
= tcg_temp_new_i64();
4713 tcg_gen_addi_i64(dirty_addr
, cpu_reg_sp(s
, rn
), imm
);
4714 clean_addr
= gen_mte_checkN(s
, dirty_addr
, false, rn
!= 31, len
);
4715 tcg_temp_free_i64(dirty_addr
);
4717 /* Note that unpredicated load/store of vector/predicate registers
4718 * are defined as a stream of bytes, which equates to little-endian
4719 * operations on larger quantities. There is no nice way to force
4720 * a little-endian store for aarch64_be-linux-user out of line.
4722 * Attempt to keep code expansion to a minimum by limiting the
4723 * amount of unrolling done.
4728 t0
= tcg_temp_new_i64();
4729 for (i
= 0; i
< len_align
; i
+= 8) {
4730 tcg_gen_ld_i64(t0
, cpu_env
, vofs
+ i
);
4731 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUQ
);
4732 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4734 tcg_temp_free_i64(t0
);
4736 TCGLabel
*loop
= gen_new_label();
4737 TCGv_ptr tp
, i
= tcg_const_local_ptr(0);
4739 /* Copy the clean address into a local temp, live across the loop. */
4741 clean_addr
= new_tmp_a64_local(s
);
4742 tcg_gen_mov_i64(clean_addr
, t0
);
4744 gen_set_label(loop
);
4746 t0
= tcg_temp_new_i64();
4747 tp
= tcg_temp_new_ptr();
4748 tcg_gen_add_ptr(tp
, cpu_env
, i
);
4749 tcg_gen_ld_i64(t0
, tp
, vofs
);
4750 tcg_gen_addi_ptr(i
, i
, 8);
4751 tcg_temp_free_ptr(tp
);
4753 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUQ
);
4754 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4755 tcg_temp_free_i64(t0
);
4757 tcg_gen_brcondi_ptr(TCG_COND_LTU
, i
, len_align
, loop
);
4758 tcg_temp_free_ptr(i
);
4761 /* Predicate register stores can be any multiple of 2. */
4763 t0
= tcg_temp_new_i64();
4764 tcg_gen_ld_i64(t0
, cpu_env
, vofs
+ len_align
);
4766 switch (len_remain
) {
4770 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
,
4771 MO_LE
| ctz32(len_remain
));
4775 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUL
);
4776 tcg_gen_addi_i64(clean_addr
, clean_addr
, 4);
4777 tcg_gen_shri_i64(t0
, t0
, 32);
4778 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUW
);
4782 g_assert_not_reached();
4784 tcg_temp_free_i64(t0
);
4788 static bool trans_LDR_zri(DisasContext
*s
, arg_rri
*a
)
4790 if (sve_access_check(s
)) {
4791 int size
= vec_full_reg_size(s
);
4792 int off
= vec_full_reg_offset(s
, a
->rd
);
4793 do_ldr(s
, off
, size
, a
->rn
, a
->imm
* size
);
4798 static bool trans_LDR_pri(DisasContext
*s
, arg_rri
*a
)
4800 if (sve_access_check(s
)) {
4801 int size
= pred_full_reg_size(s
);
4802 int off
= pred_full_reg_offset(s
, a
->rd
);
4803 do_ldr(s
, off
, size
, a
->rn
, a
->imm
* size
);
4808 static bool trans_STR_zri(DisasContext
*s
, arg_rri
*a
)
4810 if (sve_access_check(s
)) {
4811 int size
= vec_full_reg_size(s
);
4812 int off
= vec_full_reg_offset(s
, a
->rd
);
4813 do_str(s
, off
, size
, a
->rn
, a
->imm
* size
);
4818 static bool trans_STR_pri(DisasContext
*s
, arg_rri
*a
)
4820 if (sve_access_check(s
)) {
4821 int size
= pred_full_reg_size(s
);
4822 int off
= pred_full_reg_offset(s
, a
->rd
);
4823 do_str(s
, off
, size
, a
->rn
, a
->imm
* size
);
4829 *** SVE Memory - Contiguous Load Group
4832 /* The memory mode of the dtype. */
4833 static const MemOp dtype_mop
[16] = {
4834 MO_UB
, MO_UB
, MO_UB
, MO_UB
,
4835 MO_SL
, MO_UW
, MO_UW
, MO_UW
,
4836 MO_SW
, MO_SW
, MO_UL
, MO_UL
,
4837 MO_SB
, MO_SB
, MO_SB
, MO_UQ
4840 #define dtype_msz(x) (dtype_mop[x] & MO_SIZE)
4842 /* The vector element size of dtype. */
4843 static const uint8_t dtype_esz
[16] = {
4850 static void do_mem_zpa(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
,
4851 int dtype
, uint32_t mte_n
, bool is_write
,
4852 gen_helper_gvec_mem
*fn
)
4854 unsigned vsz
= vec_full_reg_size(s
);
4859 * For e.g. LD4, there are not enough arguments to pass all 4
4860 * registers as pointers, so encode the regno into the data field.
4861 * For consistency, do this even for LD1.
4863 if (s
->mte_active
[0]) {
4864 int msz
= dtype_msz(dtype
);
4866 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, get_mem_index(s
));
4867 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
4868 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
4869 desc
= FIELD_DP32(desc
, MTEDESC
, WRITE
, is_write
);
4870 desc
= FIELD_DP32(desc
, MTEDESC
, SIZEM1
, (mte_n
<< msz
) - 1);
4871 desc
<<= SVE_MTEDESC_SHIFT
;
4873 addr
= clean_data_tbi(s
, addr
);
4876 desc
= simd_desc(vsz
, vsz
, zt
| desc
);
4877 t_pg
= tcg_temp_new_ptr();
4879 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
4880 fn(cpu_env
, t_pg
, addr
, tcg_constant_i32(desc
));
4882 tcg_temp_free_ptr(t_pg
);
4885 /* Indexed by [mte][be][dtype][nreg] */
4886 static gen_helper_gvec_mem
* const ldr_fns
[2][2][16][4] = {
4887 { /* mte inactive, little-endian */
4888 { { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld2bb_r
,
4889 gen_helper_sve_ld3bb_r
, gen_helper_sve_ld4bb_r
},
4890 { gen_helper_sve_ld1bhu_r
, NULL
, NULL
, NULL
},
4891 { gen_helper_sve_ld1bsu_r
, NULL
, NULL
, NULL
},
4892 { gen_helper_sve_ld1bdu_r
, NULL
, NULL
, NULL
},
4894 { gen_helper_sve_ld1sds_le_r
, NULL
, NULL
, NULL
},
4895 { gen_helper_sve_ld1hh_le_r
, gen_helper_sve_ld2hh_le_r
,
4896 gen_helper_sve_ld3hh_le_r
, gen_helper_sve_ld4hh_le_r
},
4897 { gen_helper_sve_ld1hsu_le_r
, NULL
, NULL
, NULL
},
4898 { gen_helper_sve_ld1hdu_le_r
, NULL
, NULL
, NULL
},
4900 { gen_helper_sve_ld1hds_le_r
, NULL
, NULL
, NULL
},
4901 { gen_helper_sve_ld1hss_le_r
, NULL
, NULL
, NULL
},
4902 { gen_helper_sve_ld1ss_le_r
, gen_helper_sve_ld2ss_le_r
,
4903 gen_helper_sve_ld3ss_le_r
, gen_helper_sve_ld4ss_le_r
},
4904 { gen_helper_sve_ld1sdu_le_r
, NULL
, NULL
, NULL
},
4906 { gen_helper_sve_ld1bds_r
, NULL
, NULL
, NULL
},
4907 { gen_helper_sve_ld1bss_r
, NULL
, NULL
, NULL
},
4908 { gen_helper_sve_ld1bhs_r
, NULL
, NULL
, NULL
},
4909 { gen_helper_sve_ld1dd_le_r
, gen_helper_sve_ld2dd_le_r
,
4910 gen_helper_sve_ld3dd_le_r
, gen_helper_sve_ld4dd_le_r
} },
4912 /* mte inactive, big-endian */
4913 { { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld2bb_r
,
4914 gen_helper_sve_ld3bb_r
, gen_helper_sve_ld4bb_r
},
4915 { gen_helper_sve_ld1bhu_r
, NULL
, NULL
, NULL
},
4916 { gen_helper_sve_ld1bsu_r
, NULL
, NULL
, NULL
},
4917 { gen_helper_sve_ld1bdu_r
, NULL
, NULL
, NULL
},
4919 { gen_helper_sve_ld1sds_be_r
, NULL
, NULL
, NULL
},
4920 { gen_helper_sve_ld1hh_be_r
, gen_helper_sve_ld2hh_be_r
,
4921 gen_helper_sve_ld3hh_be_r
, gen_helper_sve_ld4hh_be_r
},
4922 { gen_helper_sve_ld1hsu_be_r
, NULL
, NULL
, NULL
},
4923 { gen_helper_sve_ld1hdu_be_r
, NULL
, NULL
, NULL
},
4925 { gen_helper_sve_ld1hds_be_r
, NULL
, NULL
, NULL
},
4926 { gen_helper_sve_ld1hss_be_r
, NULL
, NULL
, NULL
},
4927 { gen_helper_sve_ld1ss_be_r
, gen_helper_sve_ld2ss_be_r
,
4928 gen_helper_sve_ld3ss_be_r
, gen_helper_sve_ld4ss_be_r
},
4929 { gen_helper_sve_ld1sdu_be_r
, NULL
, NULL
, NULL
},
4931 { gen_helper_sve_ld1bds_r
, NULL
, NULL
, NULL
},
4932 { gen_helper_sve_ld1bss_r
, NULL
, NULL
, NULL
},
4933 { gen_helper_sve_ld1bhs_r
, NULL
, NULL
, NULL
},
4934 { gen_helper_sve_ld1dd_be_r
, gen_helper_sve_ld2dd_be_r
,
4935 gen_helper_sve_ld3dd_be_r
, gen_helper_sve_ld4dd_be_r
} } },
4937 { /* mte active, little-endian */
4938 { { gen_helper_sve_ld1bb_r_mte
,
4939 gen_helper_sve_ld2bb_r_mte
,
4940 gen_helper_sve_ld3bb_r_mte
,
4941 gen_helper_sve_ld4bb_r_mte
},
4942 { gen_helper_sve_ld1bhu_r_mte
, NULL
, NULL
, NULL
},
4943 { gen_helper_sve_ld1bsu_r_mte
, NULL
, NULL
, NULL
},
4944 { gen_helper_sve_ld1bdu_r_mte
, NULL
, NULL
, NULL
},
4946 { gen_helper_sve_ld1sds_le_r_mte
, NULL
, NULL
, NULL
},
4947 { gen_helper_sve_ld1hh_le_r_mte
,
4948 gen_helper_sve_ld2hh_le_r_mte
,
4949 gen_helper_sve_ld3hh_le_r_mte
,
4950 gen_helper_sve_ld4hh_le_r_mte
},
4951 { gen_helper_sve_ld1hsu_le_r_mte
, NULL
, NULL
, NULL
},
4952 { gen_helper_sve_ld1hdu_le_r_mte
, NULL
, NULL
, NULL
},
4954 { gen_helper_sve_ld1hds_le_r_mte
, NULL
, NULL
, NULL
},
4955 { gen_helper_sve_ld1hss_le_r_mte
, NULL
, NULL
, NULL
},
4956 { gen_helper_sve_ld1ss_le_r_mte
,
4957 gen_helper_sve_ld2ss_le_r_mte
,
4958 gen_helper_sve_ld3ss_le_r_mte
,
4959 gen_helper_sve_ld4ss_le_r_mte
},
4960 { gen_helper_sve_ld1sdu_le_r_mte
, NULL
, NULL
, NULL
},
4962 { gen_helper_sve_ld1bds_r_mte
, NULL
, NULL
, NULL
},
4963 { gen_helper_sve_ld1bss_r_mte
, NULL
, NULL
, NULL
},
4964 { gen_helper_sve_ld1bhs_r_mte
, NULL
, NULL
, NULL
},
4965 { gen_helper_sve_ld1dd_le_r_mte
,
4966 gen_helper_sve_ld2dd_le_r_mte
,
4967 gen_helper_sve_ld3dd_le_r_mte
,
4968 gen_helper_sve_ld4dd_le_r_mte
} },
4970 /* mte active, big-endian */
4971 { { gen_helper_sve_ld1bb_r_mte
,
4972 gen_helper_sve_ld2bb_r_mte
,
4973 gen_helper_sve_ld3bb_r_mte
,
4974 gen_helper_sve_ld4bb_r_mte
},
4975 { gen_helper_sve_ld1bhu_r_mte
, NULL
, NULL
, NULL
},
4976 { gen_helper_sve_ld1bsu_r_mte
, NULL
, NULL
, NULL
},
4977 { gen_helper_sve_ld1bdu_r_mte
, NULL
, NULL
, NULL
},
4979 { gen_helper_sve_ld1sds_be_r_mte
, NULL
, NULL
, NULL
},
4980 { gen_helper_sve_ld1hh_be_r_mte
,
4981 gen_helper_sve_ld2hh_be_r_mte
,
4982 gen_helper_sve_ld3hh_be_r_mte
,
4983 gen_helper_sve_ld4hh_be_r_mte
},
4984 { gen_helper_sve_ld1hsu_be_r_mte
, NULL
, NULL
, NULL
},
4985 { gen_helper_sve_ld1hdu_be_r_mte
, NULL
, NULL
, NULL
},
4987 { gen_helper_sve_ld1hds_be_r_mte
, NULL
, NULL
, NULL
},
4988 { gen_helper_sve_ld1hss_be_r_mte
, NULL
, NULL
, NULL
},
4989 { gen_helper_sve_ld1ss_be_r_mte
,
4990 gen_helper_sve_ld2ss_be_r_mte
,
4991 gen_helper_sve_ld3ss_be_r_mte
,
4992 gen_helper_sve_ld4ss_be_r_mte
},
4993 { gen_helper_sve_ld1sdu_be_r_mte
, NULL
, NULL
, NULL
},
4995 { gen_helper_sve_ld1bds_r_mte
, NULL
, NULL
, NULL
},
4996 { gen_helper_sve_ld1bss_r_mte
, NULL
, NULL
, NULL
},
4997 { gen_helper_sve_ld1bhs_r_mte
, NULL
, NULL
, NULL
},
4998 { gen_helper_sve_ld1dd_be_r_mte
,
4999 gen_helper_sve_ld2dd_be_r_mte
,
5000 gen_helper_sve_ld3dd_be_r_mte
,
5001 gen_helper_sve_ld4dd_be_r_mte
} } },
5004 static void do_ld_zpa(DisasContext
*s
, int zt
, int pg
,
5005 TCGv_i64 addr
, int dtype
, int nreg
)
5007 gen_helper_gvec_mem
*fn
5008 = ldr_fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][dtype
][nreg
];
5011 * While there are holes in the table, they are not
5012 * accessible via the instruction encoding.
5015 do_mem_zpa(s
, zt
, pg
, addr
, dtype
, nreg
, false, fn
);
5018 static bool trans_LD_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5023 if (sve_access_check(s
)) {
5024 TCGv_i64 addr
= new_tmp_a64(s
);
5025 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
5026 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5027 do_ld_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, a
->nreg
);
5032 static bool trans_LD_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5034 if (sve_access_check(s
)) {
5035 int vsz
= vec_full_reg_size(s
);
5036 int elements
= vsz
>> dtype_esz
[a
->dtype
];
5037 TCGv_i64 addr
= new_tmp_a64(s
);
5039 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
),
5040 (a
->imm
* elements
* (a
->nreg
+ 1))
5041 << dtype_msz(a
->dtype
));
5042 do_ld_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, a
->nreg
);
5047 static bool trans_LDFF1_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5049 static gen_helper_gvec_mem
* const fns
[2][2][16] = {
5050 { /* mte inactive, little-endian */
5051 { gen_helper_sve_ldff1bb_r
,
5052 gen_helper_sve_ldff1bhu_r
,
5053 gen_helper_sve_ldff1bsu_r
,
5054 gen_helper_sve_ldff1bdu_r
,
5056 gen_helper_sve_ldff1sds_le_r
,
5057 gen_helper_sve_ldff1hh_le_r
,
5058 gen_helper_sve_ldff1hsu_le_r
,
5059 gen_helper_sve_ldff1hdu_le_r
,
5061 gen_helper_sve_ldff1hds_le_r
,
5062 gen_helper_sve_ldff1hss_le_r
,
5063 gen_helper_sve_ldff1ss_le_r
,
5064 gen_helper_sve_ldff1sdu_le_r
,
5066 gen_helper_sve_ldff1bds_r
,
5067 gen_helper_sve_ldff1bss_r
,
5068 gen_helper_sve_ldff1bhs_r
,
5069 gen_helper_sve_ldff1dd_le_r
},
5071 /* mte inactive, big-endian */
5072 { gen_helper_sve_ldff1bb_r
,
5073 gen_helper_sve_ldff1bhu_r
,
5074 gen_helper_sve_ldff1bsu_r
,
5075 gen_helper_sve_ldff1bdu_r
,
5077 gen_helper_sve_ldff1sds_be_r
,
5078 gen_helper_sve_ldff1hh_be_r
,
5079 gen_helper_sve_ldff1hsu_be_r
,
5080 gen_helper_sve_ldff1hdu_be_r
,
5082 gen_helper_sve_ldff1hds_be_r
,
5083 gen_helper_sve_ldff1hss_be_r
,
5084 gen_helper_sve_ldff1ss_be_r
,
5085 gen_helper_sve_ldff1sdu_be_r
,
5087 gen_helper_sve_ldff1bds_r
,
5088 gen_helper_sve_ldff1bss_r
,
5089 gen_helper_sve_ldff1bhs_r
,
5090 gen_helper_sve_ldff1dd_be_r
} },
5092 { /* mte active, little-endian */
5093 { gen_helper_sve_ldff1bb_r_mte
,
5094 gen_helper_sve_ldff1bhu_r_mte
,
5095 gen_helper_sve_ldff1bsu_r_mte
,
5096 gen_helper_sve_ldff1bdu_r_mte
,
5098 gen_helper_sve_ldff1sds_le_r_mte
,
5099 gen_helper_sve_ldff1hh_le_r_mte
,
5100 gen_helper_sve_ldff1hsu_le_r_mte
,
5101 gen_helper_sve_ldff1hdu_le_r_mte
,
5103 gen_helper_sve_ldff1hds_le_r_mte
,
5104 gen_helper_sve_ldff1hss_le_r_mte
,
5105 gen_helper_sve_ldff1ss_le_r_mte
,
5106 gen_helper_sve_ldff1sdu_le_r_mte
,
5108 gen_helper_sve_ldff1bds_r_mte
,
5109 gen_helper_sve_ldff1bss_r_mte
,
5110 gen_helper_sve_ldff1bhs_r_mte
,
5111 gen_helper_sve_ldff1dd_le_r_mte
},
5113 /* mte active, big-endian */
5114 { gen_helper_sve_ldff1bb_r_mte
,
5115 gen_helper_sve_ldff1bhu_r_mte
,
5116 gen_helper_sve_ldff1bsu_r_mte
,
5117 gen_helper_sve_ldff1bdu_r_mte
,
5119 gen_helper_sve_ldff1sds_be_r_mte
,
5120 gen_helper_sve_ldff1hh_be_r_mte
,
5121 gen_helper_sve_ldff1hsu_be_r_mte
,
5122 gen_helper_sve_ldff1hdu_be_r_mte
,
5124 gen_helper_sve_ldff1hds_be_r_mte
,
5125 gen_helper_sve_ldff1hss_be_r_mte
,
5126 gen_helper_sve_ldff1ss_be_r_mte
,
5127 gen_helper_sve_ldff1sdu_be_r_mte
,
5129 gen_helper_sve_ldff1bds_r_mte
,
5130 gen_helper_sve_ldff1bss_r_mte
,
5131 gen_helper_sve_ldff1bhs_r_mte
,
5132 gen_helper_sve_ldff1dd_be_r_mte
} },
5135 if (sve_access_check(s
)) {
5136 TCGv_i64 addr
= new_tmp_a64(s
);
5137 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
5138 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5139 do_mem_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, 1, false,
5140 fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][a
->dtype
]);
5145 static bool trans_LDNF1_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5147 static gen_helper_gvec_mem
* const fns
[2][2][16] = {
5148 { /* mte inactive, little-endian */
5149 { gen_helper_sve_ldnf1bb_r
,
5150 gen_helper_sve_ldnf1bhu_r
,
5151 gen_helper_sve_ldnf1bsu_r
,
5152 gen_helper_sve_ldnf1bdu_r
,
5154 gen_helper_sve_ldnf1sds_le_r
,
5155 gen_helper_sve_ldnf1hh_le_r
,
5156 gen_helper_sve_ldnf1hsu_le_r
,
5157 gen_helper_sve_ldnf1hdu_le_r
,
5159 gen_helper_sve_ldnf1hds_le_r
,
5160 gen_helper_sve_ldnf1hss_le_r
,
5161 gen_helper_sve_ldnf1ss_le_r
,
5162 gen_helper_sve_ldnf1sdu_le_r
,
5164 gen_helper_sve_ldnf1bds_r
,
5165 gen_helper_sve_ldnf1bss_r
,
5166 gen_helper_sve_ldnf1bhs_r
,
5167 gen_helper_sve_ldnf1dd_le_r
},
5169 /* mte inactive, big-endian */
5170 { gen_helper_sve_ldnf1bb_r
,
5171 gen_helper_sve_ldnf1bhu_r
,
5172 gen_helper_sve_ldnf1bsu_r
,
5173 gen_helper_sve_ldnf1bdu_r
,
5175 gen_helper_sve_ldnf1sds_be_r
,
5176 gen_helper_sve_ldnf1hh_be_r
,
5177 gen_helper_sve_ldnf1hsu_be_r
,
5178 gen_helper_sve_ldnf1hdu_be_r
,
5180 gen_helper_sve_ldnf1hds_be_r
,
5181 gen_helper_sve_ldnf1hss_be_r
,
5182 gen_helper_sve_ldnf1ss_be_r
,
5183 gen_helper_sve_ldnf1sdu_be_r
,
5185 gen_helper_sve_ldnf1bds_r
,
5186 gen_helper_sve_ldnf1bss_r
,
5187 gen_helper_sve_ldnf1bhs_r
,
5188 gen_helper_sve_ldnf1dd_be_r
} },
5190 { /* mte inactive, little-endian */
5191 { gen_helper_sve_ldnf1bb_r_mte
,
5192 gen_helper_sve_ldnf1bhu_r_mte
,
5193 gen_helper_sve_ldnf1bsu_r_mte
,
5194 gen_helper_sve_ldnf1bdu_r_mte
,
5196 gen_helper_sve_ldnf1sds_le_r_mte
,
5197 gen_helper_sve_ldnf1hh_le_r_mte
,
5198 gen_helper_sve_ldnf1hsu_le_r_mte
,
5199 gen_helper_sve_ldnf1hdu_le_r_mte
,
5201 gen_helper_sve_ldnf1hds_le_r_mte
,
5202 gen_helper_sve_ldnf1hss_le_r_mte
,
5203 gen_helper_sve_ldnf1ss_le_r_mte
,
5204 gen_helper_sve_ldnf1sdu_le_r_mte
,
5206 gen_helper_sve_ldnf1bds_r_mte
,
5207 gen_helper_sve_ldnf1bss_r_mte
,
5208 gen_helper_sve_ldnf1bhs_r_mte
,
5209 gen_helper_sve_ldnf1dd_le_r_mte
},
5211 /* mte inactive, big-endian */
5212 { gen_helper_sve_ldnf1bb_r_mte
,
5213 gen_helper_sve_ldnf1bhu_r_mte
,
5214 gen_helper_sve_ldnf1bsu_r_mte
,
5215 gen_helper_sve_ldnf1bdu_r_mte
,
5217 gen_helper_sve_ldnf1sds_be_r_mte
,
5218 gen_helper_sve_ldnf1hh_be_r_mte
,
5219 gen_helper_sve_ldnf1hsu_be_r_mte
,
5220 gen_helper_sve_ldnf1hdu_be_r_mte
,
5222 gen_helper_sve_ldnf1hds_be_r_mte
,
5223 gen_helper_sve_ldnf1hss_be_r_mte
,
5224 gen_helper_sve_ldnf1ss_be_r_mte
,
5225 gen_helper_sve_ldnf1sdu_be_r_mte
,
5227 gen_helper_sve_ldnf1bds_r_mte
,
5228 gen_helper_sve_ldnf1bss_r_mte
,
5229 gen_helper_sve_ldnf1bhs_r_mte
,
5230 gen_helper_sve_ldnf1dd_be_r_mte
} },
5233 if (sve_access_check(s
)) {
5234 int vsz
= vec_full_reg_size(s
);
5235 int elements
= vsz
>> dtype_esz
[a
->dtype
];
5236 int off
= (a
->imm
* elements
) << dtype_msz(a
->dtype
);
5237 TCGv_i64 addr
= new_tmp_a64(s
);
5239 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), off
);
5240 do_mem_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, 1, false,
5241 fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][a
->dtype
]);
5246 static void do_ldrq(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
, int dtype
)
5248 unsigned vsz
= vec_full_reg_size(s
);
5252 /* Load the first quadword using the normal predicated load helpers. */
5253 poff
= pred_full_reg_offset(s
, pg
);
5256 * Zero-extend the first 16 bits of the predicate into a temporary.
5257 * This avoids triggering an assert making sure we don't have bits
5258 * set within a predicate beyond VQ, but we have lowered VQ to 1
5259 * for this load operation.
5261 TCGv_i64 tmp
= tcg_temp_new_i64();
5265 tcg_gen_ld16u_i64(tmp
, cpu_env
, poff
);
5267 poff
= offsetof(CPUARMState
, vfp
.preg_tmp
);
5268 tcg_gen_st_i64(tmp
, cpu_env
, poff
);
5269 tcg_temp_free_i64(tmp
);
5272 t_pg
= tcg_temp_new_ptr();
5273 tcg_gen_addi_ptr(t_pg
, cpu_env
, poff
);
5275 gen_helper_gvec_mem
*fn
5276 = ldr_fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][dtype
][0];
5277 fn(cpu_env
, t_pg
, addr
, tcg_constant_i32(simd_desc(16, 16, zt
)));
5279 tcg_temp_free_ptr(t_pg
);
5281 /* Replicate that first quadword. */
5283 int doff
= vec_full_reg_offset(s
, zt
);
5284 tcg_gen_gvec_dup_mem(4, doff
+ 16, doff
, vsz
- 16, vsz
- 16);
5288 static bool trans_LD1RQ_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5293 if (sve_access_check(s
)) {
5294 int msz
= dtype_msz(a
->dtype
);
5295 TCGv_i64 addr
= new_tmp_a64(s
);
5296 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), msz
);
5297 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5298 do_ldrq(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5303 static bool trans_LD1RQ_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5305 if (sve_access_check(s
)) {
5306 TCGv_i64 addr
= new_tmp_a64(s
);
5307 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), a
->imm
* 16);
5308 do_ldrq(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5313 static void do_ldro(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
, int dtype
)
5315 unsigned vsz
= vec_full_reg_size(s
);
5322 * Note that this UNDEFINED check comes after CheckSVEEnabled()
5323 * in the ARM pseudocode, which is the sve_access_check() done
5324 * in our caller. We should not now return false from the caller.
5326 unallocated_encoding(s
);
5330 /* Load the first octaword using the normal predicated load helpers. */
5332 poff
= pred_full_reg_offset(s
, pg
);
5335 * Zero-extend the first 32 bits of the predicate into a temporary.
5336 * This avoids triggering an assert making sure we don't have bits
5337 * set within a predicate beyond VQ, but we have lowered VQ to 2
5338 * for this load operation.
5340 TCGv_i64 tmp
= tcg_temp_new_i64();
5344 tcg_gen_ld32u_i64(tmp
, cpu_env
, poff
);
5346 poff
= offsetof(CPUARMState
, vfp
.preg_tmp
);
5347 tcg_gen_st_i64(tmp
, cpu_env
, poff
);
5348 tcg_temp_free_i64(tmp
);
5351 t_pg
= tcg_temp_new_ptr();
5352 tcg_gen_addi_ptr(t_pg
, cpu_env
, poff
);
5354 gen_helper_gvec_mem
*fn
5355 = ldr_fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][dtype
][0];
5356 fn(cpu_env
, t_pg
, addr
, tcg_constant_i32(simd_desc(32, 32, zt
)));
5358 tcg_temp_free_ptr(t_pg
);
5361 * Replicate that first octaword.
5362 * The replication happens in units of 32; if the full vector size
5363 * is not a multiple of 32, the final bits are zeroed.
5365 doff
= vec_full_reg_offset(s
, zt
);
5366 vsz_r32
= QEMU_ALIGN_DOWN(vsz
, 32);
5368 tcg_gen_gvec_dup_mem(5, doff
+ 32, doff
, vsz_r32
- 32, vsz_r32
- 32);
5372 tcg_gen_gvec_dup_imm(MO_64
, doff
+ vsz_r32
, vsz
, vsz
, 0);
5376 static bool trans_LD1RO_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5378 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
5384 if (sve_access_check(s
)) {
5385 TCGv_i64 addr
= new_tmp_a64(s
);
5386 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
5387 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5388 do_ldro(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5393 static bool trans_LD1RO_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5395 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
5398 if (sve_access_check(s
)) {
5399 TCGv_i64 addr
= new_tmp_a64(s
);
5400 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), a
->imm
* 32);
5401 do_ldro(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5406 /* Load and broadcast element. */
5407 static bool trans_LD1R_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5409 unsigned vsz
= vec_full_reg_size(s
);
5410 unsigned psz
= pred_full_reg_size(s
);
5411 unsigned esz
= dtype_esz
[a
->dtype
];
5412 unsigned msz
= dtype_msz(a
->dtype
);
5414 TCGv_i64 temp
, clean_addr
;
5416 if (!sve_access_check(s
)) {
5420 over
= gen_new_label();
5422 /* If the guarding predicate has no bits set, no load occurs. */
5424 /* Reduce the pred_esz_masks value simply to reduce the
5425 * size of the code generated here.
5427 uint64_t psz_mask
= MAKE_64BIT_MASK(0, psz
* 8);
5428 temp
= tcg_temp_new_i64();
5429 tcg_gen_ld_i64(temp
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
5430 tcg_gen_andi_i64(temp
, temp
, pred_esz_masks
[esz
] & psz_mask
);
5431 tcg_gen_brcondi_i64(TCG_COND_EQ
, temp
, 0, over
);
5432 tcg_temp_free_i64(temp
);
5434 TCGv_i32 t32
= tcg_temp_new_i32();
5435 find_last_active(s
, t32
, esz
, a
->pg
);
5436 tcg_gen_brcondi_i32(TCG_COND_LT
, t32
, 0, over
);
5437 tcg_temp_free_i32(t32
);
5440 /* Load the data. */
5441 temp
= tcg_temp_new_i64();
5442 tcg_gen_addi_i64(temp
, cpu_reg_sp(s
, a
->rn
), a
->imm
<< msz
);
5443 clean_addr
= gen_mte_check1(s
, temp
, false, true, msz
);
5445 tcg_gen_qemu_ld_i64(temp
, clean_addr
, get_mem_index(s
),
5446 finalize_memop(s
, dtype_mop
[a
->dtype
]));
5448 /* Broadcast to *all* elements. */
5449 tcg_gen_gvec_dup_i64(esz
, vec_full_reg_offset(s
, a
->rd
),
5451 tcg_temp_free_i64(temp
);
5453 /* Zero the inactive elements. */
5454 gen_set_label(over
);
5455 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, esz
, false);
5458 static void do_st_zpa(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
,
5459 int msz
, int esz
, int nreg
)
5461 static gen_helper_gvec_mem
* const fn_single
[2][2][4][4] = {
5462 { { { gen_helper_sve_st1bb_r
,
5463 gen_helper_sve_st1bh_r
,
5464 gen_helper_sve_st1bs_r
,
5465 gen_helper_sve_st1bd_r
},
5467 gen_helper_sve_st1hh_le_r
,
5468 gen_helper_sve_st1hs_le_r
,
5469 gen_helper_sve_st1hd_le_r
},
5471 gen_helper_sve_st1ss_le_r
,
5472 gen_helper_sve_st1sd_le_r
},
5474 gen_helper_sve_st1dd_le_r
} },
5475 { { gen_helper_sve_st1bb_r
,
5476 gen_helper_sve_st1bh_r
,
5477 gen_helper_sve_st1bs_r
,
5478 gen_helper_sve_st1bd_r
},
5480 gen_helper_sve_st1hh_be_r
,
5481 gen_helper_sve_st1hs_be_r
,
5482 gen_helper_sve_st1hd_be_r
},
5484 gen_helper_sve_st1ss_be_r
,
5485 gen_helper_sve_st1sd_be_r
},
5487 gen_helper_sve_st1dd_be_r
} } },
5489 { { { gen_helper_sve_st1bb_r_mte
,
5490 gen_helper_sve_st1bh_r_mte
,
5491 gen_helper_sve_st1bs_r_mte
,
5492 gen_helper_sve_st1bd_r_mte
},
5494 gen_helper_sve_st1hh_le_r_mte
,
5495 gen_helper_sve_st1hs_le_r_mte
,
5496 gen_helper_sve_st1hd_le_r_mte
},
5498 gen_helper_sve_st1ss_le_r_mte
,
5499 gen_helper_sve_st1sd_le_r_mte
},
5501 gen_helper_sve_st1dd_le_r_mte
} },
5502 { { gen_helper_sve_st1bb_r_mte
,
5503 gen_helper_sve_st1bh_r_mte
,
5504 gen_helper_sve_st1bs_r_mte
,
5505 gen_helper_sve_st1bd_r_mte
},
5507 gen_helper_sve_st1hh_be_r_mte
,
5508 gen_helper_sve_st1hs_be_r_mte
,
5509 gen_helper_sve_st1hd_be_r_mte
},
5511 gen_helper_sve_st1ss_be_r_mte
,
5512 gen_helper_sve_st1sd_be_r_mte
},
5514 gen_helper_sve_st1dd_be_r_mte
} } },
5516 static gen_helper_gvec_mem
* const fn_multiple
[2][2][3][4] = {
5517 { { { gen_helper_sve_st2bb_r
,
5518 gen_helper_sve_st2hh_le_r
,
5519 gen_helper_sve_st2ss_le_r
,
5520 gen_helper_sve_st2dd_le_r
},
5521 { gen_helper_sve_st3bb_r
,
5522 gen_helper_sve_st3hh_le_r
,
5523 gen_helper_sve_st3ss_le_r
,
5524 gen_helper_sve_st3dd_le_r
},
5525 { gen_helper_sve_st4bb_r
,
5526 gen_helper_sve_st4hh_le_r
,
5527 gen_helper_sve_st4ss_le_r
,
5528 gen_helper_sve_st4dd_le_r
} },
5529 { { gen_helper_sve_st2bb_r
,
5530 gen_helper_sve_st2hh_be_r
,
5531 gen_helper_sve_st2ss_be_r
,
5532 gen_helper_sve_st2dd_be_r
},
5533 { gen_helper_sve_st3bb_r
,
5534 gen_helper_sve_st3hh_be_r
,
5535 gen_helper_sve_st3ss_be_r
,
5536 gen_helper_sve_st3dd_be_r
},
5537 { gen_helper_sve_st4bb_r
,
5538 gen_helper_sve_st4hh_be_r
,
5539 gen_helper_sve_st4ss_be_r
,
5540 gen_helper_sve_st4dd_be_r
} } },
5541 { { { gen_helper_sve_st2bb_r_mte
,
5542 gen_helper_sve_st2hh_le_r_mte
,
5543 gen_helper_sve_st2ss_le_r_mte
,
5544 gen_helper_sve_st2dd_le_r_mte
},
5545 { gen_helper_sve_st3bb_r_mte
,
5546 gen_helper_sve_st3hh_le_r_mte
,
5547 gen_helper_sve_st3ss_le_r_mte
,
5548 gen_helper_sve_st3dd_le_r_mte
},
5549 { gen_helper_sve_st4bb_r_mte
,
5550 gen_helper_sve_st4hh_le_r_mte
,
5551 gen_helper_sve_st4ss_le_r_mte
,
5552 gen_helper_sve_st4dd_le_r_mte
} },
5553 { { gen_helper_sve_st2bb_r_mte
,
5554 gen_helper_sve_st2hh_be_r_mte
,
5555 gen_helper_sve_st2ss_be_r_mte
,
5556 gen_helper_sve_st2dd_be_r_mte
},
5557 { gen_helper_sve_st3bb_r_mte
,
5558 gen_helper_sve_st3hh_be_r_mte
,
5559 gen_helper_sve_st3ss_be_r_mte
,
5560 gen_helper_sve_st3dd_be_r_mte
},
5561 { gen_helper_sve_st4bb_r_mte
,
5562 gen_helper_sve_st4hh_be_r_mte
,
5563 gen_helper_sve_st4ss_be_r_mte
,
5564 gen_helper_sve_st4dd_be_r_mte
} } },
5566 gen_helper_gvec_mem
*fn
;
5567 int be
= s
->be_data
== MO_BE
;
5571 fn
= fn_single
[s
->mte_active
[0]][be
][msz
][esz
];
5574 /* ST2, ST3, ST4 -- msz == esz, enforced by encoding */
5576 fn
= fn_multiple
[s
->mte_active
[0]][be
][nreg
- 1][msz
];
5579 do_mem_zpa(s
, zt
, pg
, addr
, msz_dtype(s
, msz
), nreg
, true, fn
);
5582 static bool trans_ST_zprr(DisasContext
*s
, arg_rprr_store
*a
)
5584 if (a
->rm
== 31 || a
->msz
> a
->esz
) {
5587 if (sve_access_check(s
)) {
5588 TCGv_i64 addr
= new_tmp_a64(s
);
5589 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), a
->msz
);
5590 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5591 do_st_zpa(s
, a
->rd
, a
->pg
, addr
, a
->msz
, a
->esz
, a
->nreg
);
5596 static bool trans_ST_zpri(DisasContext
*s
, arg_rpri_store
*a
)
5598 if (a
->msz
> a
->esz
) {
5601 if (sve_access_check(s
)) {
5602 int vsz
= vec_full_reg_size(s
);
5603 int elements
= vsz
>> a
->esz
;
5604 TCGv_i64 addr
= new_tmp_a64(s
);
5606 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
),
5607 (a
->imm
* elements
* (a
->nreg
+ 1)) << a
->msz
);
5608 do_st_zpa(s
, a
->rd
, a
->pg
, addr
, a
->msz
, a
->esz
, a
->nreg
);
5614 *** SVE gather loads / scatter stores
5617 static void do_mem_zpz(DisasContext
*s
, int zt
, int pg
, int zm
,
5618 int scale
, TCGv_i64 scalar
, int msz
, bool is_write
,
5619 gen_helper_gvec_mem_scatter
*fn
)
5621 unsigned vsz
= vec_full_reg_size(s
);
5622 TCGv_ptr t_zm
= tcg_temp_new_ptr();
5623 TCGv_ptr t_pg
= tcg_temp_new_ptr();
5624 TCGv_ptr t_zt
= tcg_temp_new_ptr();
5627 if (s
->mte_active
[0]) {
5628 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, get_mem_index(s
));
5629 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
5630 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
5631 desc
= FIELD_DP32(desc
, MTEDESC
, WRITE
, is_write
);
5632 desc
= FIELD_DP32(desc
, MTEDESC
, SIZEM1
, (1 << msz
) - 1);
5633 desc
<<= SVE_MTEDESC_SHIFT
;
5635 desc
= simd_desc(vsz
, vsz
, desc
| scale
);
5637 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
5638 tcg_gen_addi_ptr(t_zm
, cpu_env
, vec_full_reg_offset(s
, zm
));
5639 tcg_gen_addi_ptr(t_zt
, cpu_env
, vec_full_reg_offset(s
, zt
));
5640 fn(cpu_env
, t_zt
, t_pg
, t_zm
, scalar
, tcg_constant_i32(desc
));
5642 tcg_temp_free_ptr(t_zt
);
5643 tcg_temp_free_ptr(t_zm
);
5644 tcg_temp_free_ptr(t_pg
);
5647 /* Indexed by [mte][be][ff][xs][u][msz]. */
5648 static gen_helper_gvec_mem_scatter
* const
5649 gather_load_fn32
[2][2][2][2][2][3] = {
5650 { /* MTE Inactive */
5651 { /* Little-endian */
5652 { { { gen_helper_sve_ldbss_zsu
,
5653 gen_helper_sve_ldhss_le_zsu
,
5655 { gen_helper_sve_ldbsu_zsu
,
5656 gen_helper_sve_ldhsu_le_zsu
,
5657 gen_helper_sve_ldss_le_zsu
, } },
5658 { { gen_helper_sve_ldbss_zss
,
5659 gen_helper_sve_ldhss_le_zss
,
5661 { gen_helper_sve_ldbsu_zss
,
5662 gen_helper_sve_ldhsu_le_zss
,
5663 gen_helper_sve_ldss_le_zss
, } } },
5666 { { { gen_helper_sve_ldffbss_zsu
,
5667 gen_helper_sve_ldffhss_le_zsu
,
5669 { gen_helper_sve_ldffbsu_zsu
,
5670 gen_helper_sve_ldffhsu_le_zsu
,
5671 gen_helper_sve_ldffss_le_zsu
, } },
5672 { { gen_helper_sve_ldffbss_zss
,
5673 gen_helper_sve_ldffhss_le_zss
,
5675 { gen_helper_sve_ldffbsu_zss
,
5676 gen_helper_sve_ldffhsu_le_zss
,
5677 gen_helper_sve_ldffss_le_zss
, } } } },
5680 { { { gen_helper_sve_ldbss_zsu
,
5681 gen_helper_sve_ldhss_be_zsu
,
5683 { gen_helper_sve_ldbsu_zsu
,
5684 gen_helper_sve_ldhsu_be_zsu
,
5685 gen_helper_sve_ldss_be_zsu
, } },
5686 { { gen_helper_sve_ldbss_zss
,
5687 gen_helper_sve_ldhss_be_zss
,
5689 { gen_helper_sve_ldbsu_zss
,
5690 gen_helper_sve_ldhsu_be_zss
,
5691 gen_helper_sve_ldss_be_zss
, } } },
5694 { { { gen_helper_sve_ldffbss_zsu
,
5695 gen_helper_sve_ldffhss_be_zsu
,
5697 { gen_helper_sve_ldffbsu_zsu
,
5698 gen_helper_sve_ldffhsu_be_zsu
,
5699 gen_helper_sve_ldffss_be_zsu
, } },
5700 { { gen_helper_sve_ldffbss_zss
,
5701 gen_helper_sve_ldffhss_be_zss
,
5703 { gen_helper_sve_ldffbsu_zss
,
5704 gen_helper_sve_ldffhsu_be_zss
,
5705 gen_helper_sve_ldffss_be_zss
, } } } } },
5707 { /* Little-endian */
5708 { { { gen_helper_sve_ldbss_zsu_mte
,
5709 gen_helper_sve_ldhss_le_zsu_mte
,
5711 { gen_helper_sve_ldbsu_zsu_mte
,
5712 gen_helper_sve_ldhsu_le_zsu_mte
,
5713 gen_helper_sve_ldss_le_zsu_mte
, } },
5714 { { gen_helper_sve_ldbss_zss_mte
,
5715 gen_helper_sve_ldhss_le_zss_mte
,
5717 { gen_helper_sve_ldbsu_zss_mte
,
5718 gen_helper_sve_ldhsu_le_zss_mte
,
5719 gen_helper_sve_ldss_le_zss_mte
, } } },
5722 { { { gen_helper_sve_ldffbss_zsu_mte
,
5723 gen_helper_sve_ldffhss_le_zsu_mte
,
5725 { gen_helper_sve_ldffbsu_zsu_mte
,
5726 gen_helper_sve_ldffhsu_le_zsu_mte
,
5727 gen_helper_sve_ldffss_le_zsu_mte
, } },
5728 { { gen_helper_sve_ldffbss_zss_mte
,
5729 gen_helper_sve_ldffhss_le_zss_mte
,
5731 { gen_helper_sve_ldffbsu_zss_mte
,
5732 gen_helper_sve_ldffhsu_le_zss_mte
,
5733 gen_helper_sve_ldffss_le_zss_mte
, } } } },
5736 { { { gen_helper_sve_ldbss_zsu_mte
,
5737 gen_helper_sve_ldhss_be_zsu_mte
,
5739 { gen_helper_sve_ldbsu_zsu_mte
,
5740 gen_helper_sve_ldhsu_be_zsu_mte
,
5741 gen_helper_sve_ldss_be_zsu_mte
, } },
5742 { { gen_helper_sve_ldbss_zss_mte
,
5743 gen_helper_sve_ldhss_be_zss_mte
,
5745 { gen_helper_sve_ldbsu_zss_mte
,
5746 gen_helper_sve_ldhsu_be_zss_mte
,
5747 gen_helper_sve_ldss_be_zss_mte
, } } },
5750 { { { gen_helper_sve_ldffbss_zsu_mte
,
5751 gen_helper_sve_ldffhss_be_zsu_mte
,
5753 { gen_helper_sve_ldffbsu_zsu_mte
,
5754 gen_helper_sve_ldffhsu_be_zsu_mte
,
5755 gen_helper_sve_ldffss_be_zsu_mte
, } },
5756 { { gen_helper_sve_ldffbss_zss_mte
,
5757 gen_helper_sve_ldffhss_be_zss_mte
,
5759 { gen_helper_sve_ldffbsu_zss_mte
,
5760 gen_helper_sve_ldffhsu_be_zss_mte
,
5761 gen_helper_sve_ldffss_be_zss_mte
, } } } } },
5764 /* Note that we overload xs=2 to indicate 64-bit offset. */
5765 static gen_helper_gvec_mem_scatter
* const
5766 gather_load_fn64
[2][2][2][3][2][4] = {
5767 { /* MTE Inactive */
5768 { /* Little-endian */
5769 { { { gen_helper_sve_ldbds_zsu
,
5770 gen_helper_sve_ldhds_le_zsu
,
5771 gen_helper_sve_ldsds_le_zsu
,
5773 { gen_helper_sve_ldbdu_zsu
,
5774 gen_helper_sve_ldhdu_le_zsu
,
5775 gen_helper_sve_ldsdu_le_zsu
,
5776 gen_helper_sve_lddd_le_zsu
, } },
5777 { { gen_helper_sve_ldbds_zss
,
5778 gen_helper_sve_ldhds_le_zss
,
5779 gen_helper_sve_ldsds_le_zss
,
5781 { gen_helper_sve_ldbdu_zss
,
5782 gen_helper_sve_ldhdu_le_zss
,
5783 gen_helper_sve_ldsdu_le_zss
,
5784 gen_helper_sve_lddd_le_zss
, } },
5785 { { gen_helper_sve_ldbds_zd
,
5786 gen_helper_sve_ldhds_le_zd
,
5787 gen_helper_sve_ldsds_le_zd
,
5789 { gen_helper_sve_ldbdu_zd
,
5790 gen_helper_sve_ldhdu_le_zd
,
5791 gen_helper_sve_ldsdu_le_zd
,
5792 gen_helper_sve_lddd_le_zd
, } } },
5795 { { { gen_helper_sve_ldffbds_zsu
,
5796 gen_helper_sve_ldffhds_le_zsu
,
5797 gen_helper_sve_ldffsds_le_zsu
,
5799 { gen_helper_sve_ldffbdu_zsu
,
5800 gen_helper_sve_ldffhdu_le_zsu
,
5801 gen_helper_sve_ldffsdu_le_zsu
,
5802 gen_helper_sve_ldffdd_le_zsu
, } },
5803 { { gen_helper_sve_ldffbds_zss
,
5804 gen_helper_sve_ldffhds_le_zss
,
5805 gen_helper_sve_ldffsds_le_zss
,
5807 { gen_helper_sve_ldffbdu_zss
,
5808 gen_helper_sve_ldffhdu_le_zss
,
5809 gen_helper_sve_ldffsdu_le_zss
,
5810 gen_helper_sve_ldffdd_le_zss
, } },
5811 { { gen_helper_sve_ldffbds_zd
,
5812 gen_helper_sve_ldffhds_le_zd
,
5813 gen_helper_sve_ldffsds_le_zd
,
5815 { gen_helper_sve_ldffbdu_zd
,
5816 gen_helper_sve_ldffhdu_le_zd
,
5817 gen_helper_sve_ldffsdu_le_zd
,
5818 gen_helper_sve_ldffdd_le_zd
, } } } },
5820 { { { gen_helper_sve_ldbds_zsu
,
5821 gen_helper_sve_ldhds_be_zsu
,
5822 gen_helper_sve_ldsds_be_zsu
,
5824 { gen_helper_sve_ldbdu_zsu
,
5825 gen_helper_sve_ldhdu_be_zsu
,
5826 gen_helper_sve_ldsdu_be_zsu
,
5827 gen_helper_sve_lddd_be_zsu
, } },
5828 { { gen_helper_sve_ldbds_zss
,
5829 gen_helper_sve_ldhds_be_zss
,
5830 gen_helper_sve_ldsds_be_zss
,
5832 { gen_helper_sve_ldbdu_zss
,
5833 gen_helper_sve_ldhdu_be_zss
,
5834 gen_helper_sve_ldsdu_be_zss
,
5835 gen_helper_sve_lddd_be_zss
, } },
5836 { { gen_helper_sve_ldbds_zd
,
5837 gen_helper_sve_ldhds_be_zd
,
5838 gen_helper_sve_ldsds_be_zd
,
5840 { gen_helper_sve_ldbdu_zd
,
5841 gen_helper_sve_ldhdu_be_zd
,
5842 gen_helper_sve_ldsdu_be_zd
,
5843 gen_helper_sve_lddd_be_zd
, } } },
5846 { { { gen_helper_sve_ldffbds_zsu
,
5847 gen_helper_sve_ldffhds_be_zsu
,
5848 gen_helper_sve_ldffsds_be_zsu
,
5850 { gen_helper_sve_ldffbdu_zsu
,
5851 gen_helper_sve_ldffhdu_be_zsu
,
5852 gen_helper_sve_ldffsdu_be_zsu
,
5853 gen_helper_sve_ldffdd_be_zsu
, } },
5854 { { gen_helper_sve_ldffbds_zss
,
5855 gen_helper_sve_ldffhds_be_zss
,
5856 gen_helper_sve_ldffsds_be_zss
,
5858 { gen_helper_sve_ldffbdu_zss
,
5859 gen_helper_sve_ldffhdu_be_zss
,
5860 gen_helper_sve_ldffsdu_be_zss
,
5861 gen_helper_sve_ldffdd_be_zss
, } },
5862 { { gen_helper_sve_ldffbds_zd
,
5863 gen_helper_sve_ldffhds_be_zd
,
5864 gen_helper_sve_ldffsds_be_zd
,
5866 { gen_helper_sve_ldffbdu_zd
,
5867 gen_helper_sve_ldffhdu_be_zd
,
5868 gen_helper_sve_ldffsdu_be_zd
,
5869 gen_helper_sve_ldffdd_be_zd
, } } } } },
5871 { /* Little-endian */
5872 { { { gen_helper_sve_ldbds_zsu_mte
,
5873 gen_helper_sve_ldhds_le_zsu_mte
,
5874 gen_helper_sve_ldsds_le_zsu_mte
,
5876 { gen_helper_sve_ldbdu_zsu_mte
,
5877 gen_helper_sve_ldhdu_le_zsu_mte
,
5878 gen_helper_sve_ldsdu_le_zsu_mte
,
5879 gen_helper_sve_lddd_le_zsu_mte
, } },
5880 { { gen_helper_sve_ldbds_zss_mte
,
5881 gen_helper_sve_ldhds_le_zss_mte
,
5882 gen_helper_sve_ldsds_le_zss_mte
,
5884 { gen_helper_sve_ldbdu_zss_mte
,
5885 gen_helper_sve_ldhdu_le_zss_mte
,
5886 gen_helper_sve_ldsdu_le_zss_mte
,
5887 gen_helper_sve_lddd_le_zss_mte
, } },
5888 { { gen_helper_sve_ldbds_zd_mte
,
5889 gen_helper_sve_ldhds_le_zd_mte
,
5890 gen_helper_sve_ldsds_le_zd_mte
,
5892 { gen_helper_sve_ldbdu_zd_mte
,
5893 gen_helper_sve_ldhdu_le_zd_mte
,
5894 gen_helper_sve_ldsdu_le_zd_mte
,
5895 gen_helper_sve_lddd_le_zd_mte
, } } },
5898 { { { gen_helper_sve_ldffbds_zsu_mte
,
5899 gen_helper_sve_ldffhds_le_zsu_mte
,
5900 gen_helper_sve_ldffsds_le_zsu_mte
,
5902 { gen_helper_sve_ldffbdu_zsu_mte
,
5903 gen_helper_sve_ldffhdu_le_zsu_mte
,
5904 gen_helper_sve_ldffsdu_le_zsu_mte
,
5905 gen_helper_sve_ldffdd_le_zsu_mte
, } },
5906 { { gen_helper_sve_ldffbds_zss_mte
,
5907 gen_helper_sve_ldffhds_le_zss_mte
,
5908 gen_helper_sve_ldffsds_le_zss_mte
,
5910 { gen_helper_sve_ldffbdu_zss_mte
,
5911 gen_helper_sve_ldffhdu_le_zss_mte
,
5912 gen_helper_sve_ldffsdu_le_zss_mte
,
5913 gen_helper_sve_ldffdd_le_zss_mte
, } },
5914 { { gen_helper_sve_ldffbds_zd_mte
,
5915 gen_helper_sve_ldffhds_le_zd_mte
,
5916 gen_helper_sve_ldffsds_le_zd_mte
,
5918 { gen_helper_sve_ldffbdu_zd_mte
,
5919 gen_helper_sve_ldffhdu_le_zd_mte
,
5920 gen_helper_sve_ldffsdu_le_zd_mte
,
5921 gen_helper_sve_ldffdd_le_zd_mte
, } } } },
5923 { { { gen_helper_sve_ldbds_zsu_mte
,
5924 gen_helper_sve_ldhds_be_zsu_mte
,
5925 gen_helper_sve_ldsds_be_zsu_mte
,
5927 { gen_helper_sve_ldbdu_zsu_mte
,
5928 gen_helper_sve_ldhdu_be_zsu_mte
,
5929 gen_helper_sve_ldsdu_be_zsu_mte
,
5930 gen_helper_sve_lddd_be_zsu_mte
, } },
5931 { { gen_helper_sve_ldbds_zss_mte
,
5932 gen_helper_sve_ldhds_be_zss_mte
,
5933 gen_helper_sve_ldsds_be_zss_mte
,
5935 { gen_helper_sve_ldbdu_zss_mte
,
5936 gen_helper_sve_ldhdu_be_zss_mte
,
5937 gen_helper_sve_ldsdu_be_zss_mte
,
5938 gen_helper_sve_lddd_be_zss_mte
, } },
5939 { { gen_helper_sve_ldbds_zd_mte
,
5940 gen_helper_sve_ldhds_be_zd_mte
,
5941 gen_helper_sve_ldsds_be_zd_mte
,
5943 { gen_helper_sve_ldbdu_zd_mte
,
5944 gen_helper_sve_ldhdu_be_zd_mte
,
5945 gen_helper_sve_ldsdu_be_zd_mte
,
5946 gen_helper_sve_lddd_be_zd_mte
, } } },
5949 { { { gen_helper_sve_ldffbds_zsu_mte
,
5950 gen_helper_sve_ldffhds_be_zsu_mte
,
5951 gen_helper_sve_ldffsds_be_zsu_mte
,
5953 { gen_helper_sve_ldffbdu_zsu_mte
,
5954 gen_helper_sve_ldffhdu_be_zsu_mte
,
5955 gen_helper_sve_ldffsdu_be_zsu_mte
,
5956 gen_helper_sve_ldffdd_be_zsu_mte
, } },
5957 { { gen_helper_sve_ldffbds_zss_mte
,
5958 gen_helper_sve_ldffhds_be_zss_mte
,
5959 gen_helper_sve_ldffsds_be_zss_mte
,
5961 { gen_helper_sve_ldffbdu_zss_mte
,
5962 gen_helper_sve_ldffhdu_be_zss_mte
,
5963 gen_helper_sve_ldffsdu_be_zss_mte
,
5964 gen_helper_sve_ldffdd_be_zss_mte
, } },
5965 { { gen_helper_sve_ldffbds_zd_mte
,
5966 gen_helper_sve_ldffhds_be_zd_mte
,
5967 gen_helper_sve_ldffsds_be_zd_mte
,
5969 { gen_helper_sve_ldffbdu_zd_mte
,
5970 gen_helper_sve_ldffhdu_be_zd_mte
,
5971 gen_helper_sve_ldffsdu_be_zd_mte
,
5972 gen_helper_sve_ldffdd_be_zd_mte
, } } } } },
5975 static bool trans_LD1_zprz(DisasContext
*s
, arg_LD1_zprz
*a
)
5977 gen_helper_gvec_mem_scatter
*fn
= NULL
;
5978 bool be
= s
->be_data
== MO_BE
;
5979 bool mte
= s
->mte_active
[0];
5981 if (!sve_access_check(s
)) {
5987 fn
= gather_load_fn32
[mte
][be
][a
->ff
][a
->xs
][a
->u
][a
->msz
];
5990 fn
= gather_load_fn64
[mte
][be
][a
->ff
][a
->xs
][a
->u
][a
->msz
];
5995 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rm
, a
->scale
* a
->msz
,
5996 cpu_reg_sp(s
, a
->rn
), a
->msz
, false, fn
);
6000 static bool trans_LD1_zpiz(DisasContext
*s
, arg_LD1_zpiz
*a
)
6002 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6003 bool be
= s
->be_data
== MO_BE
;
6004 bool mte
= s
->mte_active
[0];
6006 if (a
->esz
< a
->msz
|| (a
->esz
== a
->msz
&& !a
->u
)) {
6009 if (!sve_access_check(s
)) {
6015 fn
= gather_load_fn32
[mte
][be
][a
->ff
][0][a
->u
][a
->msz
];
6018 fn
= gather_load_fn64
[mte
][be
][a
->ff
][2][a
->u
][a
->msz
];
6023 /* Treat LD1_zpiz (zn[x] + imm) the same way as LD1_zprz (rn + zm[x])
6024 * by loading the immediate into the scalar parameter.
6026 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0,
6027 tcg_constant_i64(a
->imm
<< a
->msz
), a
->msz
, false, fn
);
6031 static bool trans_LDNT1_zprz(DisasContext
*s
, arg_LD1_zprz
*a
)
6033 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6034 bool be
= s
->be_data
== MO_BE
;
6035 bool mte
= s
->mte_active
[0];
6037 if (a
->esz
< a
->msz
+ !a
->u
) {
6040 if (!dc_isar_feature(aa64_sve2
, s
)) {
6043 if (!sve_access_check(s
)) {
6049 fn
= gather_load_fn32
[mte
][be
][0][0][a
->u
][a
->msz
];
6052 fn
= gather_load_fn64
[mte
][be
][0][2][a
->u
][a
->msz
];
6057 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0,
6058 cpu_reg(s
, a
->rm
), a
->msz
, false, fn
);
6062 /* Indexed by [mte][be][xs][msz]. */
6063 static gen_helper_gvec_mem_scatter
* const scatter_store_fn32
[2][2][2][3] = {
6064 { /* MTE Inactive */
6065 { /* Little-endian */
6066 { gen_helper_sve_stbs_zsu
,
6067 gen_helper_sve_sths_le_zsu
,
6068 gen_helper_sve_stss_le_zsu
, },
6069 { gen_helper_sve_stbs_zss
,
6070 gen_helper_sve_sths_le_zss
,
6071 gen_helper_sve_stss_le_zss
, } },
6073 { gen_helper_sve_stbs_zsu
,
6074 gen_helper_sve_sths_be_zsu
,
6075 gen_helper_sve_stss_be_zsu
, },
6076 { gen_helper_sve_stbs_zss
,
6077 gen_helper_sve_sths_be_zss
,
6078 gen_helper_sve_stss_be_zss
, } } },
6080 { /* Little-endian */
6081 { gen_helper_sve_stbs_zsu_mte
,
6082 gen_helper_sve_sths_le_zsu_mte
,
6083 gen_helper_sve_stss_le_zsu_mte
, },
6084 { gen_helper_sve_stbs_zss_mte
,
6085 gen_helper_sve_sths_le_zss_mte
,
6086 gen_helper_sve_stss_le_zss_mte
, } },
6088 { gen_helper_sve_stbs_zsu_mte
,
6089 gen_helper_sve_sths_be_zsu_mte
,
6090 gen_helper_sve_stss_be_zsu_mte
, },
6091 { gen_helper_sve_stbs_zss_mte
,
6092 gen_helper_sve_sths_be_zss_mte
,
6093 gen_helper_sve_stss_be_zss_mte
, } } },
6096 /* Note that we overload xs=2 to indicate 64-bit offset. */
6097 static gen_helper_gvec_mem_scatter
* const scatter_store_fn64
[2][2][3][4] = {
6098 { /* MTE Inactive */
6099 { /* Little-endian */
6100 { gen_helper_sve_stbd_zsu
,
6101 gen_helper_sve_sthd_le_zsu
,
6102 gen_helper_sve_stsd_le_zsu
,
6103 gen_helper_sve_stdd_le_zsu
, },
6104 { gen_helper_sve_stbd_zss
,
6105 gen_helper_sve_sthd_le_zss
,
6106 gen_helper_sve_stsd_le_zss
,
6107 gen_helper_sve_stdd_le_zss
, },
6108 { gen_helper_sve_stbd_zd
,
6109 gen_helper_sve_sthd_le_zd
,
6110 gen_helper_sve_stsd_le_zd
,
6111 gen_helper_sve_stdd_le_zd
, } },
6113 { gen_helper_sve_stbd_zsu
,
6114 gen_helper_sve_sthd_be_zsu
,
6115 gen_helper_sve_stsd_be_zsu
,
6116 gen_helper_sve_stdd_be_zsu
, },
6117 { gen_helper_sve_stbd_zss
,
6118 gen_helper_sve_sthd_be_zss
,
6119 gen_helper_sve_stsd_be_zss
,
6120 gen_helper_sve_stdd_be_zss
, },
6121 { gen_helper_sve_stbd_zd
,
6122 gen_helper_sve_sthd_be_zd
,
6123 gen_helper_sve_stsd_be_zd
,
6124 gen_helper_sve_stdd_be_zd
, } } },
6125 { /* MTE Inactive */
6126 { /* Little-endian */
6127 { gen_helper_sve_stbd_zsu_mte
,
6128 gen_helper_sve_sthd_le_zsu_mte
,
6129 gen_helper_sve_stsd_le_zsu_mte
,
6130 gen_helper_sve_stdd_le_zsu_mte
, },
6131 { gen_helper_sve_stbd_zss_mte
,
6132 gen_helper_sve_sthd_le_zss_mte
,
6133 gen_helper_sve_stsd_le_zss_mte
,
6134 gen_helper_sve_stdd_le_zss_mte
, },
6135 { gen_helper_sve_stbd_zd_mte
,
6136 gen_helper_sve_sthd_le_zd_mte
,
6137 gen_helper_sve_stsd_le_zd_mte
,
6138 gen_helper_sve_stdd_le_zd_mte
, } },
6140 { gen_helper_sve_stbd_zsu_mte
,
6141 gen_helper_sve_sthd_be_zsu_mte
,
6142 gen_helper_sve_stsd_be_zsu_mte
,
6143 gen_helper_sve_stdd_be_zsu_mte
, },
6144 { gen_helper_sve_stbd_zss_mte
,
6145 gen_helper_sve_sthd_be_zss_mte
,
6146 gen_helper_sve_stsd_be_zss_mte
,
6147 gen_helper_sve_stdd_be_zss_mte
, },
6148 { gen_helper_sve_stbd_zd_mte
,
6149 gen_helper_sve_sthd_be_zd_mte
,
6150 gen_helper_sve_stsd_be_zd_mte
,
6151 gen_helper_sve_stdd_be_zd_mte
, } } },
6154 static bool trans_ST1_zprz(DisasContext
*s
, arg_ST1_zprz
*a
)
6156 gen_helper_gvec_mem_scatter
*fn
;
6157 bool be
= s
->be_data
== MO_BE
;
6158 bool mte
= s
->mte_active
[0];
6160 if (a
->esz
< a
->msz
|| (a
->msz
== 0 && a
->scale
)) {
6163 if (!sve_access_check(s
)) {
6168 fn
= scatter_store_fn32
[mte
][be
][a
->xs
][a
->msz
];
6171 fn
= scatter_store_fn64
[mte
][be
][a
->xs
][a
->msz
];
6174 g_assert_not_reached();
6176 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rm
, a
->scale
* a
->msz
,
6177 cpu_reg_sp(s
, a
->rn
), a
->msz
, true, fn
);
6181 static bool trans_ST1_zpiz(DisasContext
*s
, arg_ST1_zpiz
*a
)
6183 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6184 bool be
= s
->be_data
== MO_BE
;
6185 bool mte
= s
->mte_active
[0];
6187 if (a
->esz
< a
->msz
) {
6190 if (!sve_access_check(s
)) {
6196 fn
= scatter_store_fn32
[mte
][be
][0][a
->msz
];
6199 fn
= scatter_store_fn64
[mte
][be
][2][a
->msz
];
6204 /* Treat ST1_zpiz (zn[x] + imm) the same way as ST1_zprz (rn + zm[x])
6205 * by loading the immediate into the scalar parameter.
6207 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0,
6208 tcg_constant_i64(a
->imm
<< a
->msz
), a
->msz
, true, fn
);
6212 static bool trans_STNT1_zprz(DisasContext
*s
, arg_ST1_zprz
*a
)
6214 gen_helper_gvec_mem_scatter
*fn
;
6215 bool be
= s
->be_data
== MO_BE
;
6216 bool mte
= s
->mte_active
[0];
6218 if (a
->esz
< a
->msz
) {
6221 if (!dc_isar_feature(aa64_sve2
, s
)) {
6224 if (!sve_access_check(s
)) {
6230 fn
= scatter_store_fn32
[mte
][be
][0][a
->msz
];
6233 fn
= scatter_store_fn64
[mte
][be
][2][a
->msz
];
6236 g_assert_not_reached();
6239 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0,
6240 cpu_reg(s
, a
->rm
), a
->msz
, true, fn
);
6248 static bool trans_PRF(DisasContext
*s
, arg_PRF
*a
)
6250 /* Prefetch is a nop within QEMU. */
6251 (void)sve_access_check(s
);
6255 static bool trans_PRF_rr(DisasContext
*s
, arg_PRF_rr
*a
)
6260 /* Prefetch is a nop within QEMU. */
6261 (void)sve_access_check(s
);
6268 * TODO: The implementation so far could handle predicated merging movprfx.
6269 * The helper functions as written take an extra source register to
6270 * use in the operation, but the result is only written when predication
6271 * succeeds. For unpredicated movprfx, we need to rearrange the helpers
6272 * to allow the final write back to the destination to be unconditional.
6273 * For predicated zeroing movprfx, we need to rearrange the helpers to
6274 * allow the final write back to zero inactives.
6276 * In the meantime, just emit the moves.
6279 static bool trans_MOVPRFX(DisasContext
*s
, arg_MOVPRFX
*a
)
6281 return do_mov_z(s
, a
->rd
, a
->rn
);
6284 static bool trans_MOVPRFX_m(DisasContext
*s
, arg_rpr_esz
*a
)
6286 return do_sel_z(s
, a
->rd
, a
->rn
, a
->rd
, a
->pg
, a
->esz
);
6289 static bool trans_MOVPRFX_z(DisasContext
*s
, arg_rpr_esz
*a
)
6291 return do_movz_zpz(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
, false);
6295 * SVE2 Integer Multiply - Unpredicated
6298 TRANS_FEAT(MUL_zzz
, aa64_sve2
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_mul
, a
)
6300 static gen_helper_gvec_3
* const smulh_zzz_fns
[4] = {
6301 gen_helper_gvec_smulh_b
, gen_helper_gvec_smulh_h
,
6302 gen_helper_gvec_smulh_s
, gen_helper_gvec_smulh_d
,
6304 TRANS_FEAT(SMULH_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6305 smulh_zzz_fns
[a
->esz
], a
, 0)
6307 static gen_helper_gvec_3
* const umulh_zzz_fns
[4] = {
6308 gen_helper_gvec_umulh_b
, gen_helper_gvec_umulh_h
,
6309 gen_helper_gvec_umulh_s
, gen_helper_gvec_umulh_d
,
6311 TRANS_FEAT(UMULH_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6312 umulh_zzz_fns
[a
->esz
], a
, 0)
6314 TRANS_FEAT(PMUL_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6315 gen_helper_gvec_pmul_b
, a
, 0)
6317 static gen_helper_gvec_3
* const sqdmulh_zzz_fns
[4] = {
6318 gen_helper_sve2_sqdmulh_b
, gen_helper_sve2_sqdmulh_h
,
6319 gen_helper_sve2_sqdmulh_s
, gen_helper_sve2_sqdmulh_d
,
6321 TRANS_FEAT(SQDMULH_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6322 sqdmulh_zzz_fns
[a
->esz
], a
, 0)
6324 static gen_helper_gvec_3
* const sqrdmulh_zzz_fns
[4] = {
6325 gen_helper_sve2_sqrdmulh_b
, gen_helper_sve2_sqrdmulh_h
,
6326 gen_helper_sve2_sqrdmulh_s
, gen_helper_sve2_sqrdmulh_d
,
6328 TRANS_FEAT(SQRDMULH_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6329 sqrdmulh_zzz_fns
[a
->esz
], a
, 0)
6332 * SVE2 Integer - Predicated
6335 static gen_helper_gvec_4
* const sadlp_fns
[4] = {
6336 NULL
, gen_helper_sve2_sadalp_zpzz_h
,
6337 gen_helper_sve2_sadalp_zpzz_s
, gen_helper_sve2_sadalp_zpzz_d
,
6339 TRANS_FEAT(SADALP_zpzz
, aa64_sve2
, gen_gvec_ool_arg_zpzz
,
6340 sadlp_fns
[a
->esz
], a
, 0)
6342 static gen_helper_gvec_4
* const uadlp_fns
[4] = {
6343 NULL
, gen_helper_sve2_uadalp_zpzz_h
,
6344 gen_helper_sve2_uadalp_zpzz_s
, gen_helper_sve2_uadalp_zpzz_d
,
6346 TRANS_FEAT(UADALP_zpzz
, aa64_sve2
, gen_gvec_ool_arg_zpzz
,
6347 uadlp_fns
[a
->esz
], a
, 0)
6350 * SVE2 integer unary operations (predicated)
6353 TRANS_FEAT(URECPE
, aa64_sve2
, gen_gvec_ool_arg_zpz
,
6354 a
->esz
== 2 ? gen_helper_sve2_urecpe_s
: NULL
, a
, 0)
6356 TRANS_FEAT(URSQRTE
, aa64_sve2
, gen_gvec_ool_arg_zpz
,
6357 a
->esz
== 2 ? gen_helper_sve2_ursqrte_s
: NULL
, a
, 0)
6359 static gen_helper_gvec_3
* const sqabs_fns
[4] = {
6360 gen_helper_sve2_sqabs_b
, gen_helper_sve2_sqabs_h
,
6361 gen_helper_sve2_sqabs_s
, gen_helper_sve2_sqabs_d
,
6363 TRANS_FEAT(SQABS
, aa64_sve2
, gen_gvec_ool_arg_zpz
, sqabs_fns
[a
->esz
], a
, 0)
6365 static gen_helper_gvec_3
* const sqneg_fns
[4] = {
6366 gen_helper_sve2_sqneg_b
, gen_helper_sve2_sqneg_h
,
6367 gen_helper_sve2_sqneg_s
, gen_helper_sve2_sqneg_d
,
6369 TRANS_FEAT(SQNEG
, aa64_sve2
, gen_gvec_ool_arg_zpz
, sqneg_fns
[a
->esz
], a
, 0)
6371 DO_ZPZZ(SQSHL
, aa64_sve2
, sve2_sqshl
)
6372 DO_ZPZZ(SQRSHL
, aa64_sve2
, sve2_sqrshl
)
6373 DO_ZPZZ(SRSHL
, aa64_sve2
, sve2_srshl
)
6375 DO_ZPZZ(UQSHL
, aa64_sve2
, sve2_uqshl
)
6376 DO_ZPZZ(UQRSHL
, aa64_sve2
, sve2_uqrshl
)
6377 DO_ZPZZ(URSHL
, aa64_sve2
, sve2_urshl
)
6379 DO_ZPZZ(SHADD
, aa64_sve2
, sve2_shadd
)
6380 DO_ZPZZ(SRHADD
, aa64_sve2
, sve2_srhadd
)
6381 DO_ZPZZ(SHSUB
, aa64_sve2
, sve2_shsub
)
6383 DO_ZPZZ(UHADD
, aa64_sve2
, sve2_uhadd
)
6384 DO_ZPZZ(URHADD
, aa64_sve2
, sve2_urhadd
)
6385 DO_ZPZZ(UHSUB
, aa64_sve2
, sve2_uhsub
)
6387 DO_ZPZZ(ADDP
, aa64_sve2
, sve2_addp
)
6388 DO_ZPZZ(SMAXP
, aa64_sve2
, sve2_smaxp
)
6389 DO_ZPZZ(UMAXP
, aa64_sve2
, sve2_umaxp
)
6390 DO_ZPZZ(SMINP
, aa64_sve2
, sve2_sminp
)
6391 DO_ZPZZ(UMINP
, aa64_sve2
, sve2_uminp
)
6393 DO_ZPZZ(SQADD_zpzz
, aa64_sve2
, sve2_sqadd
)
6394 DO_ZPZZ(UQADD_zpzz
, aa64_sve2
, sve2_uqadd
)
6395 DO_ZPZZ(SQSUB_zpzz
, aa64_sve2
, sve2_sqsub
)
6396 DO_ZPZZ(UQSUB_zpzz
, aa64_sve2
, sve2_uqsub
)
6397 DO_ZPZZ(SUQADD
, aa64_sve2
, sve2_suqadd
)
6398 DO_ZPZZ(USQADD
, aa64_sve2
, sve2_usqadd
)
6401 * SVE2 Widening Integer Arithmetic
6404 static gen_helper_gvec_3
* const saddl_fns
[4] = {
6405 NULL
, gen_helper_sve2_saddl_h
,
6406 gen_helper_sve2_saddl_s
, gen_helper_sve2_saddl_d
,
6408 TRANS_FEAT(SADDLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6409 saddl_fns
[a
->esz
], a
, 0)
6410 TRANS_FEAT(SADDLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6411 saddl_fns
[a
->esz
], a
, 3)
6412 TRANS_FEAT(SADDLBT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6413 saddl_fns
[a
->esz
], a
, 2)
6415 static gen_helper_gvec_3
* const ssubl_fns
[4] = {
6416 NULL
, gen_helper_sve2_ssubl_h
,
6417 gen_helper_sve2_ssubl_s
, gen_helper_sve2_ssubl_d
,
6419 TRANS_FEAT(SSUBLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6420 ssubl_fns
[a
->esz
], a
, 0)
6421 TRANS_FEAT(SSUBLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6422 ssubl_fns
[a
->esz
], a
, 3)
6423 TRANS_FEAT(SSUBLBT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6424 ssubl_fns
[a
->esz
], a
, 2)
6425 TRANS_FEAT(SSUBLTB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6426 ssubl_fns
[a
->esz
], a
, 1)
6428 static gen_helper_gvec_3
* const sabdl_fns
[4] = {
6429 NULL
, gen_helper_sve2_sabdl_h
,
6430 gen_helper_sve2_sabdl_s
, gen_helper_sve2_sabdl_d
,
6432 TRANS_FEAT(SABDLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6433 sabdl_fns
[a
->esz
], a
, 0)
6434 TRANS_FEAT(SABDLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6435 sabdl_fns
[a
->esz
], a
, 3)
6437 static gen_helper_gvec_3
* const uaddl_fns
[4] = {
6438 NULL
, gen_helper_sve2_uaddl_h
,
6439 gen_helper_sve2_uaddl_s
, gen_helper_sve2_uaddl_d
,
6441 TRANS_FEAT(UADDLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6442 uaddl_fns
[a
->esz
], a
, 0)
6443 TRANS_FEAT(UADDLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6444 uaddl_fns
[a
->esz
], a
, 3)
6446 static gen_helper_gvec_3
* const usubl_fns
[4] = {
6447 NULL
, gen_helper_sve2_usubl_h
,
6448 gen_helper_sve2_usubl_s
, gen_helper_sve2_usubl_d
,
6450 TRANS_FEAT(USUBLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6451 usubl_fns
[a
->esz
], a
, 0)
6452 TRANS_FEAT(USUBLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6453 usubl_fns
[a
->esz
], a
, 3)
6455 static gen_helper_gvec_3
* const uabdl_fns
[4] = {
6456 NULL
, gen_helper_sve2_uabdl_h
,
6457 gen_helper_sve2_uabdl_s
, gen_helper_sve2_uabdl_d
,
6459 TRANS_FEAT(UABDLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6460 uabdl_fns
[a
->esz
], a
, 0)
6461 TRANS_FEAT(UABDLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6462 uabdl_fns
[a
->esz
], a
, 3)
6464 static gen_helper_gvec_3
* const sqdmull_fns
[4] = {
6465 NULL
, gen_helper_sve2_sqdmull_zzz_h
,
6466 gen_helper_sve2_sqdmull_zzz_s
, gen_helper_sve2_sqdmull_zzz_d
,
6468 TRANS_FEAT(SQDMULLB_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6469 sqdmull_fns
[a
->esz
], a
, 0)
6470 TRANS_FEAT(SQDMULLT_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6471 sqdmull_fns
[a
->esz
], a
, 3)
6473 static gen_helper_gvec_3
* const smull_fns
[4] = {
6474 NULL
, gen_helper_sve2_smull_zzz_h
,
6475 gen_helper_sve2_smull_zzz_s
, gen_helper_sve2_smull_zzz_d
,
6477 TRANS_FEAT(SMULLB_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6478 smull_fns
[a
->esz
], a
, 0)
6479 TRANS_FEAT(SMULLT_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6480 smull_fns
[a
->esz
], a
, 3)
6482 static gen_helper_gvec_3
* const umull_fns
[4] = {
6483 NULL
, gen_helper_sve2_umull_zzz_h
,
6484 gen_helper_sve2_umull_zzz_s
, gen_helper_sve2_umull_zzz_d
,
6486 TRANS_FEAT(UMULLB_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6487 umull_fns
[a
->esz
], a
, 0)
6488 TRANS_FEAT(UMULLT_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6489 umull_fns
[a
->esz
], a
, 3)
6491 static gen_helper_gvec_3
* const eoril_fns
[4] = {
6492 gen_helper_sve2_eoril_b
, gen_helper_sve2_eoril_h
,
6493 gen_helper_sve2_eoril_s
, gen_helper_sve2_eoril_d
,
6495 TRANS_FEAT(EORBT
, aa64_sve2
, gen_gvec_ool_arg_zzz
, eoril_fns
[a
->esz
], a
, 2)
6496 TRANS_FEAT(EORTB
, aa64_sve2
, gen_gvec_ool_arg_zzz
, eoril_fns
[a
->esz
], a
, 1)
6498 static bool do_trans_pmull(DisasContext
*s
, arg_rrr_esz
*a
, bool sel
)
6500 static gen_helper_gvec_3
* const fns
[4] = {
6501 gen_helper_gvec_pmull_q
, gen_helper_sve2_pmull_h
,
6502 NULL
, gen_helper_sve2_pmull_d
,
6504 if (a
->esz
== 0 && !dc_isar_feature(aa64_sve2_pmull128
, s
)) {
6507 return gen_gvec_ool_arg_zzz(s
, fns
[a
->esz
], a
, sel
);
6510 TRANS_FEAT(PMULLB
, aa64_sve2
, do_trans_pmull
, a
, false)
6511 TRANS_FEAT(PMULLT
, aa64_sve2
, do_trans_pmull
, a
, true)
6513 static gen_helper_gvec_3
* const saddw_fns
[4] = {
6514 NULL
, gen_helper_sve2_saddw_h
,
6515 gen_helper_sve2_saddw_s
, gen_helper_sve2_saddw_d
,
6517 TRANS_FEAT(SADDWB
, aa64_sve2
, gen_gvec_ool_arg_zzz
, saddw_fns
[a
->esz
], a
, 0)
6518 TRANS_FEAT(SADDWT
, aa64_sve2
, gen_gvec_ool_arg_zzz
, saddw_fns
[a
->esz
], a
, 1)
6520 static gen_helper_gvec_3
* const ssubw_fns
[4] = {
6521 NULL
, gen_helper_sve2_ssubw_h
,
6522 gen_helper_sve2_ssubw_s
, gen_helper_sve2_ssubw_d
,
6524 TRANS_FEAT(SSUBWB
, aa64_sve2
, gen_gvec_ool_arg_zzz
, ssubw_fns
[a
->esz
], a
, 0)
6525 TRANS_FEAT(SSUBWT
, aa64_sve2
, gen_gvec_ool_arg_zzz
, ssubw_fns
[a
->esz
], a
, 1)
6527 static gen_helper_gvec_3
* const uaddw_fns
[4] = {
6528 NULL
, gen_helper_sve2_uaddw_h
,
6529 gen_helper_sve2_uaddw_s
, gen_helper_sve2_uaddw_d
,
6531 TRANS_FEAT(UADDWB
, aa64_sve2
, gen_gvec_ool_arg_zzz
, uaddw_fns
[a
->esz
], a
, 0)
6532 TRANS_FEAT(UADDWT
, aa64_sve2
, gen_gvec_ool_arg_zzz
, uaddw_fns
[a
->esz
], a
, 1)
6534 static gen_helper_gvec_3
* const usubw_fns
[4] = {
6535 NULL
, gen_helper_sve2_usubw_h
,
6536 gen_helper_sve2_usubw_s
, gen_helper_sve2_usubw_d
,
6538 TRANS_FEAT(USUBWB
, aa64_sve2
, gen_gvec_ool_arg_zzz
, usubw_fns
[a
->esz
], a
, 0)
6539 TRANS_FEAT(USUBWT
, aa64_sve2
, gen_gvec_ool_arg_zzz
, usubw_fns
[a
->esz
], a
, 1)
6541 static void gen_sshll_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t imm
)
6545 int halfbits
= 4 << vece
;
6548 if (shl
== halfbits
) {
6549 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6550 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(halfbits
, halfbits
));
6551 tcg_gen_and_vec(vece
, d
, n
, t
);
6552 tcg_temp_free_vec(t
);
6554 tcg_gen_sari_vec(vece
, d
, n
, halfbits
);
6555 tcg_gen_shli_vec(vece
, d
, d
, shl
);
6558 tcg_gen_shli_vec(vece
, d
, n
, halfbits
);
6559 tcg_gen_sari_vec(vece
, d
, d
, halfbits
- shl
);
6563 static void gen_ushll_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int imm
)
6565 int halfbits
= 4 << vece
;
6567 int shl
= (imm
>> 1);
6571 mask
= MAKE_64BIT_MASK(0, halfbits
);
6573 mask
= dup_const(vece
, mask
);
6575 shift
= shl
- top
* halfbits
;
6577 tcg_gen_shri_i64(d
, n
, -shift
);
6579 tcg_gen_shli_i64(d
, n
, shift
);
6581 tcg_gen_andi_i64(d
, d
, mask
);
6584 static void gen_ushll16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6586 gen_ushll_i64(MO_16
, d
, n
, imm
);
6589 static void gen_ushll32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6591 gen_ushll_i64(MO_32
, d
, n
, imm
);
6594 static void gen_ushll64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6596 gen_ushll_i64(MO_64
, d
, n
, imm
);
6599 static void gen_ushll_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t imm
)
6601 int halfbits
= 4 << vece
;
6606 if (shl
== halfbits
) {
6607 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6608 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(halfbits
, halfbits
));
6609 tcg_gen_and_vec(vece
, d
, n
, t
);
6610 tcg_temp_free_vec(t
);
6612 tcg_gen_shri_vec(vece
, d
, n
, halfbits
);
6613 tcg_gen_shli_vec(vece
, d
, d
, shl
);
6617 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6618 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
6619 tcg_gen_and_vec(vece
, d
, n
, t
);
6620 tcg_temp_free_vec(t
);
6622 tcg_gen_shli_vec(vece
, d
, n
, halfbits
);
6623 tcg_gen_shri_vec(vece
, d
, d
, halfbits
- shl
);
6628 static bool do_sve2_shll_tb(DisasContext
*s
, arg_rri_esz
*a
,
6631 static const TCGOpcode sshll_list
[] = {
6632 INDEX_op_shli_vec
, INDEX_op_sari_vec
, 0
6634 static const TCGOpcode ushll_list
[] = {
6635 INDEX_op_shli_vec
, INDEX_op_shri_vec
, 0
6637 static const GVecGen2i ops
[2][3] = {
6638 { { .fniv
= gen_sshll_vec
,
6639 .opt_opc
= sshll_list
,
6640 .fno
= gen_helper_sve2_sshll_h
,
6642 { .fniv
= gen_sshll_vec
,
6643 .opt_opc
= sshll_list
,
6644 .fno
= gen_helper_sve2_sshll_s
,
6646 { .fniv
= gen_sshll_vec
,
6647 .opt_opc
= sshll_list
,
6648 .fno
= gen_helper_sve2_sshll_d
,
6650 { { .fni8
= gen_ushll16_i64
,
6651 .fniv
= gen_ushll_vec
,
6652 .opt_opc
= ushll_list
,
6653 .fno
= gen_helper_sve2_ushll_h
,
6655 { .fni8
= gen_ushll32_i64
,
6656 .fniv
= gen_ushll_vec
,
6657 .opt_opc
= ushll_list
,
6658 .fno
= gen_helper_sve2_ushll_s
,
6660 { .fni8
= gen_ushll64_i64
,
6661 .fniv
= gen_ushll_vec
,
6662 .opt_opc
= ushll_list
,
6663 .fno
= gen_helper_sve2_ushll_d
,
6667 if (a
->esz
< 0 || a
->esz
> 2 || !dc_isar_feature(aa64_sve2
, s
)) {
6670 if (sve_access_check(s
)) {
6671 unsigned vsz
= vec_full_reg_size(s
);
6672 tcg_gen_gvec_2i(vec_full_reg_offset(s
, a
->rd
),
6673 vec_full_reg_offset(s
, a
->rn
),
6674 vsz
, vsz
, (a
->imm
<< 1) | sel
,
6680 static bool trans_SSHLLB(DisasContext
*s
, arg_rri_esz
*a
)
6682 return do_sve2_shll_tb(s
, a
, false, false);
6685 static bool trans_SSHLLT(DisasContext
*s
, arg_rri_esz
*a
)
6687 return do_sve2_shll_tb(s
, a
, true, false);
6690 static bool trans_USHLLB(DisasContext
*s
, arg_rri_esz
*a
)
6692 return do_sve2_shll_tb(s
, a
, false, true);
6695 static bool trans_USHLLT(DisasContext
*s
, arg_rri_esz
*a
)
6697 return do_sve2_shll_tb(s
, a
, true, true);
6700 static gen_helper_gvec_3
* const bext_fns
[4] = {
6701 gen_helper_sve2_bext_b
, gen_helper_sve2_bext_h
,
6702 gen_helper_sve2_bext_s
, gen_helper_sve2_bext_d
,
6704 TRANS_FEAT(BEXT
, aa64_sve2_bitperm
, gen_gvec_ool_arg_zzz
,
6705 bext_fns
[a
->esz
], a
, 0)
6707 static gen_helper_gvec_3
* const bdep_fns
[4] = {
6708 gen_helper_sve2_bdep_b
, gen_helper_sve2_bdep_h
,
6709 gen_helper_sve2_bdep_s
, gen_helper_sve2_bdep_d
,
6711 TRANS_FEAT(BDEP
, aa64_sve2_bitperm
, gen_gvec_ool_arg_zzz
,
6712 bdep_fns
[a
->esz
], a
, 0)
6714 static gen_helper_gvec_3
* const bgrp_fns
[4] = {
6715 gen_helper_sve2_bgrp_b
, gen_helper_sve2_bgrp_h
,
6716 gen_helper_sve2_bgrp_s
, gen_helper_sve2_bgrp_d
,
6718 TRANS_FEAT(BGRP
, aa64_sve2_bitperm
, gen_gvec_ool_arg_zzz
,
6719 bgrp_fns
[a
->esz
], a
, 0)
6721 static gen_helper_gvec_3
* const cadd_fns
[4] = {
6722 gen_helper_sve2_cadd_b
, gen_helper_sve2_cadd_h
,
6723 gen_helper_sve2_cadd_s
, gen_helper_sve2_cadd_d
,
6725 TRANS_FEAT(CADD_rot90
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6726 cadd_fns
[a
->esz
], a
, 0)
6727 TRANS_FEAT(CADD_rot270
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6728 cadd_fns
[a
->esz
], a
, 1)
6730 static gen_helper_gvec_3
* const sqcadd_fns
[4] = {
6731 gen_helper_sve2_sqcadd_b
, gen_helper_sve2_sqcadd_h
,
6732 gen_helper_sve2_sqcadd_s
, gen_helper_sve2_sqcadd_d
,
6734 TRANS_FEAT(SQCADD_rot90
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6735 sqcadd_fns
[a
->esz
], a
, 0)
6736 TRANS_FEAT(SQCADD_rot270
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6737 sqcadd_fns
[a
->esz
], a
, 1)
6739 static gen_helper_gvec_4
* const sabal_fns
[4] = {
6740 NULL
, gen_helper_sve2_sabal_h
,
6741 gen_helper_sve2_sabal_s
, gen_helper_sve2_sabal_d
,
6743 TRANS_FEAT(SABALB
, aa64_sve2
, gen_gvec_ool_arg_zzzz
, sabal_fns
[a
->esz
], a
, 0)
6744 TRANS_FEAT(SABALT
, aa64_sve2
, gen_gvec_ool_arg_zzzz
, sabal_fns
[a
->esz
], a
, 1)
6746 static gen_helper_gvec_4
* const uabal_fns
[4] = {
6747 NULL
, gen_helper_sve2_uabal_h
,
6748 gen_helper_sve2_uabal_s
, gen_helper_sve2_uabal_d
,
6750 TRANS_FEAT(UABALB
, aa64_sve2
, gen_gvec_ool_arg_zzzz
, uabal_fns
[a
->esz
], a
, 0)
6751 TRANS_FEAT(UABALT
, aa64_sve2
, gen_gvec_ool_arg_zzzz
, uabal_fns
[a
->esz
], a
, 1)
6753 static bool do_adcl(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
6755 static gen_helper_gvec_4
* const fns
[2] = {
6756 gen_helper_sve2_adcl_s
,
6757 gen_helper_sve2_adcl_d
,
6760 * Note that in this case the ESZ field encodes both size and sign.
6761 * Split out 'subtract' into bit 1 of the data field for the helper.
6763 return gen_gvec_ool_arg_zzzz(s
, fns
[a
->esz
& 1], a
, (a
->esz
& 2) | sel
);
6766 TRANS_FEAT(ADCLB
, aa64_sve2
, do_adcl
, a
, false)
6767 TRANS_FEAT(ADCLT
, aa64_sve2
, do_adcl
, a
, true)
6769 TRANS_FEAT(SSRA
, aa64_sve2
, gen_gvec_fn_arg_zzi
, gen_gvec_ssra
, a
)
6770 TRANS_FEAT(USRA
, aa64_sve2
, gen_gvec_fn_arg_zzi
, gen_gvec_usra
, a
)
6771 TRANS_FEAT(SRSRA
, aa64_sve2
, gen_gvec_fn_arg_zzi
, gen_gvec_srsra
, a
)
6772 TRANS_FEAT(URSRA
, aa64_sve2
, gen_gvec_fn_arg_zzi
, gen_gvec_ursra
, a
)
6773 TRANS_FEAT(SRI
, aa64_sve2
, gen_gvec_fn_arg_zzi
, gen_gvec_sri
, a
)
6774 TRANS_FEAT(SLI
, aa64_sve2
, gen_gvec_fn_arg_zzi
, gen_gvec_sli
, a
)
6776 TRANS_FEAT(SABA
, aa64_sve2
, gen_gvec_fn_arg_zzz
, gen_gvec_saba
, a
)
6777 TRANS_FEAT(UABA
, aa64_sve2
, gen_gvec_fn_arg_zzz
, gen_gvec_uaba
, a
)
6779 static bool do_sve2_narrow_extract(DisasContext
*s
, arg_rri_esz
*a
,
6780 const GVecGen2 ops
[3])
6782 if (a
->esz
< 0 || a
->esz
> MO_32
|| a
->imm
!= 0 ||
6783 !dc_isar_feature(aa64_sve2
, s
)) {
6786 if (sve_access_check(s
)) {
6787 unsigned vsz
= vec_full_reg_size(s
);
6788 tcg_gen_gvec_2(vec_full_reg_offset(s
, a
->rd
),
6789 vec_full_reg_offset(s
, a
->rn
),
6790 vsz
, vsz
, &ops
[a
->esz
]);
6795 static const TCGOpcode sqxtn_list
[] = {
6796 INDEX_op_shli_vec
, INDEX_op_smin_vec
, INDEX_op_smax_vec
, 0
6799 static void gen_sqxtnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
6801 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6802 int halfbits
= 4 << vece
;
6803 int64_t mask
= (1ull << halfbits
) - 1;
6804 int64_t min
= -1ull << (halfbits
- 1);
6805 int64_t max
= -min
- 1;
6807 tcg_gen_dupi_vec(vece
, t
, min
);
6808 tcg_gen_smax_vec(vece
, d
, n
, t
);
6809 tcg_gen_dupi_vec(vece
, t
, max
);
6810 tcg_gen_smin_vec(vece
, d
, d
, t
);
6811 tcg_gen_dupi_vec(vece
, t
, mask
);
6812 tcg_gen_and_vec(vece
, d
, d
, t
);
6813 tcg_temp_free_vec(t
);
6816 static bool trans_SQXTNB(DisasContext
*s
, arg_rri_esz
*a
)
6818 static const GVecGen2 ops
[3] = {
6819 { .fniv
= gen_sqxtnb_vec
,
6820 .opt_opc
= sqxtn_list
,
6821 .fno
= gen_helper_sve2_sqxtnb_h
,
6823 { .fniv
= gen_sqxtnb_vec
,
6824 .opt_opc
= sqxtn_list
,
6825 .fno
= gen_helper_sve2_sqxtnb_s
,
6827 { .fniv
= gen_sqxtnb_vec
,
6828 .opt_opc
= sqxtn_list
,
6829 .fno
= gen_helper_sve2_sqxtnb_d
,
6832 return do_sve2_narrow_extract(s
, a
, ops
);
6835 static void gen_sqxtnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
6837 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6838 int halfbits
= 4 << vece
;
6839 int64_t mask
= (1ull << halfbits
) - 1;
6840 int64_t min
= -1ull << (halfbits
- 1);
6841 int64_t max
= -min
- 1;
6843 tcg_gen_dupi_vec(vece
, t
, min
);
6844 tcg_gen_smax_vec(vece
, n
, n
, t
);
6845 tcg_gen_dupi_vec(vece
, t
, max
);
6846 tcg_gen_smin_vec(vece
, n
, n
, t
);
6847 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
6848 tcg_gen_dupi_vec(vece
, t
, mask
);
6849 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
6850 tcg_temp_free_vec(t
);
6853 static bool trans_SQXTNT(DisasContext
*s
, arg_rri_esz
*a
)
6855 static const GVecGen2 ops
[3] = {
6856 { .fniv
= gen_sqxtnt_vec
,
6857 .opt_opc
= sqxtn_list
,
6859 .fno
= gen_helper_sve2_sqxtnt_h
,
6861 { .fniv
= gen_sqxtnt_vec
,
6862 .opt_opc
= sqxtn_list
,
6864 .fno
= gen_helper_sve2_sqxtnt_s
,
6866 { .fniv
= gen_sqxtnt_vec
,
6867 .opt_opc
= sqxtn_list
,
6869 .fno
= gen_helper_sve2_sqxtnt_d
,
6872 return do_sve2_narrow_extract(s
, a
, ops
);
6875 static const TCGOpcode uqxtn_list
[] = {
6876 INDEX_op_shli_vec
, INDEX_op_umin_vec
, 0
6879 static void gen_uqxtnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
6881 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6882 int halfbits
= 4 << vece
;
6883 int64_t max
= (1ull << halfbits
) - 1;
6885 tcg_gen_dupi_vec(vece
, t
, max
);
6886 tcg_gen_umin_vec(vece
, d
, n
, t
);
6887 tcg_temp_free_vec(t
);
6890 static bool trans_UQXTNB(DisasContext
*s
, arg_rri_esz
*a
)
6892 static const GVecGen2 ops
[3] = {
6893 { .fniv
= gen_uqxtnb_vec
,
6894 .opt_opc
= uqxtn_list
,
6895 .fno
= gen_helper_sve2_uqxtnb_h
,
6897 { .fniv
= gen_uqxtnb_vec
,
6898 .opt_opc
= uqxtn_list
,
6899 .fno
= gen_helper_sve2_uqxtnb_s
,
6901 { .fniv
= gen_uqxtnb_vec
,
6902 .opt_opc
= uqxtn_list
,
6903 .fno
= gen_helper_sve2_uqxtnb_d
,
6906 return do_sve2_narrow_extract(s
, a
, ops
);
6909 static void gen_uqxtnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
6911 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6912 int halfbits
= 4 << vece
;
6913 int64_t max
= (1ull << halfbits
) - 1;
6915 tcg_gen_dupi_vec(vece
, t
, max
);
6916 tcg_gen_umin_vec(vece
, n
, n
, t
);
6917 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
6918 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
6919 tcg_temp_free_vec(t
);
6922 static bool trans_UQXTNT(DisasContext
*s
, arg_rri_esz
*a
)
6924 static const GVecGen2 ops
[3] = {
6925 { .fniv
= gen_uqxtnt_vec
,
6926 .opt_opc
= uqxtn_list
,
6928 .fno
= gen_helper_sve2_uqxtnt_h
,
6930 { .fniv
= gen_uqxtnt_vec
,
6931 .opt_opc
= uqxtn_list
,
6933 .fno
= gen_helper_sve2_uqxtnt_s
,
6935 { .fniv
= gen_uqxtnt_vec
,
6936 .opt_opc
= uqxtn_list
,
6938 .fno
= gen_helper_sve2_uqxtnt_d
,
6941 return do_sve2_narrow_extract(s
, a
, ops
);
6944 static const TCGOpcode sqxtun_list
[] = {
6945 INDEX_op_shli_vec
, INDEX_op_umin_vec
, INDEX_op_smax_vec
, 0
6948 static void gen_sqxtunb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
6950 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6951 int halfbits
= 4 << vece
;
6952 int64_t max
= (1ull << halfbits
) - 1;
6954 tcg_gen_dupi_vec(vece
, t
, 0);
6955 tcg_gen_smax_vec(vece
, d
, n
, t
);
6956 tcg_gen_dupi_vec(vece
, t
, max
);
6957 tcg_gen_umin_vec(vece
, d
, d
, t
);
6958 tcg_temp_free_vec(t
);
6961 static bool trans_SQXTUNB(DisasContext
*s
, arg_rri_esz
*a
)
6963 static const GVecGen2 ops
[3] = {
6964 { .fniv
= gen_sqxtunb_vec
,
6965 .opt_opc
= sqxtun_list
,
6966 .fno
= gen_helper_sve2_sqxtunb_h
,
6968 { .fniv
= gen_sqxtunb_vec
,
6969 .opt_opc
= sqxtun_list
,
6970 .fno
= gen_helper_sve2_sqxtunb_s
,
6972 { .fniv
= gen_sqxtunb_vec
,
6973 .opt_opc
= sqxtun_list
,
6974 .fno
= gen_helper_sve2_sqxtunb_d
,
6977 return do_sve2_narrow_extract(s
, a
, ops
);
6980 static void gen_sqxtunt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
6982 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6983 int halfbits
= 4 << vece
;
6984 int64_t max
= (1ull << halfbits
) - 1;
6986 tcg_gen_dupi_vec(vece
, t
, 0);
6987 tcg_gen_smax_vec(vece
, n
, n
, t
);
6988 tcg_gen_dupi_vec(vece
, t
, max
);
6989 tcg_gen_umin_vec(vece
, n
, n
, t
);
6990 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
6991 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
6992 tcg_temp_free_vec(t
);
6995 static bool trans_SQXTUNT(DisasContext
*s
, arg_rri_esz
*a
)
6997 static const GVecGen2 ops
[3] = {
6998 { .fniv
= gen_sqxtunt_vec
,
6999 .opt_opc
= sqxtun_list
,
7001 .fno
= gen_helper_sve2_sqxtunt_h
,
7003 { .fniv
= gen_sqxtunt_vec
,
7004 .opt_opc
= sqxtun_list
,
7006 .fno
= gen_helper_sve2_sqxtunt_s
,
7008 { .fniv
= gen_sqxtunt_vec
,
7009 .opt_opc
= sqxtun_list
,
7011 .fno
= gen_helper_sve2_sqxtunt_d
,
7014 return do_sve2_narrow_extract(s
, a
, ops
);
7017 static bool do_sve2_shr_narrow(DisasContext
*s
, arg_rri_esz
*a
,
7018 const GVecGen2i ops
[3])
7020 if (a
->esz
< 0 || a
->esz
> MO_32
|| !dc_isar_feature(aa64_sve2
, s
)) {
7023 assert(a
->imm
> 0 && a
->imm
<= (8 << a
->esz
));
7024 if (sve_access_check(s
)) {
7025 unsigned vsz
= vec_full_reg_size(s
);
7026 tcg_gen_gvec_2i(vec_full_reg_offset(s
, a
->rd
),
7027 vec_full_reg_offset(s
, a
->rn
),
7028 vsz
, vsz
, a
->imm
, &ops
[a
->esz
]);
7033 static void gen_shrnb_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int shr
)
7035 int halfbits
= 4 << vece
;
7036 uint64_t mask
= dup_const(vece
, MAKE_64BIT_MASK(0, halfbits
));
7038 tcg_gen_shri_i64(d
, n
, shr
);
7039 tcg_gen_andi_i64(d
, d
, mask
);
7042 static void gen_shrnb16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7044 gen_shrnb_i64(MO_16
, d
, n
, shr
);
7047 static void gen_shrnb32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7049 gen_shrnb_i64(MO_32
, d
, n
, shr
);
7052 static void gen_shrnb64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7054 gen_shrnb_i64(MO_64
, d
, n
, shr
);
7057 static void gen_shrnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t shr
)
7059 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7060 int halfbits
= 4 << vece
;
7061 uint64_t mask
= MAKE_64BIT_MASK(0, halfbits
);
7063 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7064 tcg_gen_dupi_vec(vece
, t
, mask
);
7065 tcg_gen_and_vec(vece
, d
, n
, t
);
7066 tcg_temp_free_vec(t
);
7069 static bool trans_SHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7071 static const TCGOpcode vec_list
[] = { INDEX_op_shri_vec
, 0 };
7072 static const GVecGen2i ops
[3] = {
7073 { .fni8
= gen_shrnb16_i64
,
7074 .fniv
= gen_shrnb_vec
,
7075 .opt_opc
= vec_list
,
7076 .fno
= gen_helper_sve2_shrnb_h
,
7078 { .fni8
= gen_shrnb32_i64
,
7079 .fniv
= gen_shrnb_vec
,
7080 .opt_opc
= vec_list
,
7081 .fno
= gen_helper_sve2_shrnb_s
,
7083 { .fni8
= gen_shrnb64_i64
,
7084 .fniv
= gen_shrnb_vec
,
7085 .opt_opc
= vec_list
,
7086 .fno
= gen_helper_sve2_shrnb_d
,
7089 return do_sve2_shr_narrow(s
, a
, ops
);
7092 static void gen_shrnt_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int shr
)
7094 int halfbits
= 4 << vece
;
7095 uint64_t mask
= dup_const(vece
, MAKE_64BIT_MASK(0, halfbits
));
7097 tcg_gen_shli_i64(n
, n
, halfbits
- shr
);
7098 tcg_gen_andi_i64(n
, n
, ~mask
);
7099 tcg_gen_andi_i64(d
, d
, mask
);
7100 tcg_gen_or_i64(d
, d
, n
);
7103 static void gen_shrnt16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7105 gen_shrnt_i64(MO_16
, d
, n
, shr
);
7108 static void gen_shrnt32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7110 gen_shrnt_i64(MO_32
, d
, n
, shr
);
7113 static void gen_shrnt64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7115 tcg_gen_shri_i64(n
, n
, shr
);
7116 tcg_gen_deposit_i64(d
, d
, n
, 32, 32);
7119 static void gen_shrnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t shr
)
7121 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7122 int halfbits
= 4 << vece
;
7123 uint64_t mask
= MAKE_64BIT_MASK(0, halfbits
);
7125 tcg_gen_shli_vec(vece
, n
, n
, halfbits
- shr
);
7126 tcg_gen_dupi_vec(vece
, t
, mask
);
7127 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7128 tcg_temp_free_vec(t
);
7131 static bool trans_SHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7133 static const TCGOpcode vec_list
[] = { INDEX_op_shli_vec
, 0 };
7134 static const GVecGen2i ops
[3] = {
7135 { .fni8
= gen_shrnt16_i64
,
7136 .fniv
= gen_shrnt_vec
,
7137 .opt_opc
= vec_list
,
7139 .fno
= gen_helper_sve2_shrnt_h
,
7141 { .fni8
= gen_shrnt32_i64
,
7142 .fniv
= gen_shrnt_vec
,
7143 .opt_opc
= vec_list
,
7145 .fno
= gen_helper_sve2_shrnt_s
,
7147 { .fni8
= gen_shrnt64_i64
,
7148 .fniv
= gen_shrnt_vec
,
7149 .opt_opc
= vec_list
,
7151 .fno
= gen_helper_sve2_shrnt_d
,
7154 return do_sve2_shr_narrow(s
, a
, ops
);
7157 static bool trans_RSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7159 static const GVecGen2i ops
[3] = {
7160 { .fno
= gen_helper_sve2_rshrnb_h
},
7161 { .fno
= gen_helper_sve2_rshrnb_s
},
7162 { .fno
= gen_helper_sve2_rshrnb_d
},
7164 return do_sve2_shr_narrow(s
, a
, ops
);
7167 static bool trans_RSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7169 static const GVecGen2i ops
[3] = {
7170 { .fno
= gen_helper_sve2_rshrnt_h
},
7171 { .fno
= gen_helper_sve2_rshrnt_s
},
7172 { .fno
= gen_helper_sve2_rshrnt_d
},
7174 return do_sve2_shr_narrow(s
, a
, ops
);
7177 static void gen_sqshrunb_vec(unsigned vece
, TCGv_vec d
,
7178 TCGv_vec n
, int64_t shr
)
7180 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7181 int halfbits
= 4 << vece
;
7183 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7184 tcg_gen_dupi_vec(vece
, t
, 0);
7185 tcg_gen_smax_vec(vece
, n
, n
, t
);
7186 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7187 tcg_gen_umin_vec(vece
, d
, n
, t
);
7188 tcg_temp_free_vec(t
);
7191 static bool trans_SQSHRUNB(DisasContext
*s
, arg_rri_esz
*a
)
7193 static const TCGOpcode vec_list
[] = {
7194 INDEX_op_sari_vec
, INDEX_op_smax_vec
, INDEX_op_umin_vec
, 0
7196 static const GVecGen2i ops
[3] = {
7197 { .fniv
= gen_sqshrunb_vec
,
7198 .opt_opc
= vec_list
,
7199 .fno
= gen_helper_sve2_sqshrunb_h
,
7201 { .fniv
= gen_sqshrunb_vec
,
7202 .opt_opc
= vec_list
,
7203 .fno
= gen_helper_sve2_sqshrunb_s
,
7205 { .fniv
= gen_sqshrunb_vec
,
7206 .opt_opc
= vec_list
,
7207 .fno
= gen_helper_sve2_sqshrunb_d
,
7210 return do_sve2_shr_narrow(s
, a
, ops
);
7213 static void gen_sqshrunt_vec(unsigned vece
, TCGv_vec d
,
7214 TCGv_vec n
, int64_t shr
)
7216 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7217 int halfbits
= 4 << vece
;
7219 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7220 tcg_gen_dupi_vec(vece
, t
, 0);
7221 tcg_gen_smax_vec(vece
, n
, n
, t
);
7222 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7223 tcg_gen_umin_vec(vece
, n
, n
, t
);
7224 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7225 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7226 tcg_temp_free_vec(t
);
7229 static bool trans_SQSHRUNT(DisasContext
*s
, arg_rri_esz
*a
)
7231 static const TCGOpcode vec_list
[] = {
7232 INDEX_op_shli_vec
, INDEX_op_sari_vec
,
7233 INDEX_op_smax_vec
, INDEX_op_umin_vec
, 0
7235 static const GVecGen2i ops
[3] = {
7236 { .fniv
= gen_sqshrunt_vec
,
7237 .opt_opc
= vec_list
,
7239 .fno
= gen_helper_sve2_sqshrunt_h
,
7241 { .fniv
= gen_sqshrunt_vec
,
7242 .opt_opc
= vec_list
,
7244 .fno
= gen_helper_sve2_sqshrunt_s
,
7246 { .fniv
= gen_sqshrunt_vec
,
7247 .opt_opc
= vec_list
,
7249 .fno
= gen_helper_sve2_sqshrunt_d
,
7252 return do_sve2_shr_narrow(s
, a
, ops
);
7255 static bool trans_SQRSHRUNB(DisasContext
*s
, arg_rri_esz
*a
)
7257 static const GVecGen2i ops
[3] = {
7258 { .fno
= gen_helper_sve2_sqrshrunb_h
},
7259 { .fno
= gen_helper_sve2_sqrshrunb_s
},
7260 { .fno
= gen_helper_sve2_sqrshrunb_d
},
7262 return do_sve2_shr_narrow(s
, a
, ops
);
7265 static bool trans_SQRSHRUNT(DisasContext
*s
, arg_rri_esz
*a
)
7267 static const GVecGen2i ops
[3] = {
7268 { .fno
= gen_helper_sve2_sqrshrunt_h
},
7269 { .fno
= gen_helper_sve2_sqrshrunt_s
},
7270 { .fno
= gen_helper_sve2_sqrshrunt_d
},
7272 return do_sve2_shr_narrow(s
, a
, ops
);
7275 static void gen_sqshrnb_vec(unsigned vece
, TCGv_vec d
,
7276 TCGv_vec n
, int64_t shr
)
7278 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7279 int halfbits
= 4 << vece
;
7280 int64_t max
= MAKE_64BIT_MASK(0, halfbits
- 1);
7281 int64_t min
= -max
- 1;
7283 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7284 tcg_gen_dupi_vec(vece
, t
, min
);
7285 tcg_gen_smax_vec(vece
, n
, n
, t
);
7286 tcg_gen_dupi_vec(vece
, t
, max
);
7287 tcg_gen_smin_vec(vece
, n
, n
, t
);
7288 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7289 tcg_gen_and_vec(vece
, d
, n
, t
);
7290 tcg_temp_free_vec(t
);
7293 static bool trans_SQSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7295 static const TCGOpcode vec_list
[] = {
7296 INDEX_op_sari_vec
, INDEX_op_smax_vec
, INDEX_op_smin_vec
, 0
7298 static const GVecGen2i ops
[3] = {
7299 { .fniv
= gen_sqshrnb_vec
,
7300 .opt_opc
= vec_list
,
7301 .fno
= gen_helper_sve2_sqshrnb_h
,
7303 { .fniv
= gen_sqshrnb_vec
,
7304 .opt_opc
= vec_list
,
7305 .fno
= gen_helper_sve2_sqshrnb_s
,
7307 { .fniv
= gen_sqshrnb_vec
,
7308 .opt_opc
= vec_list
,
7309 .fno
= gen_helper_sve2_sqshrnb_d
,
7312 return do_sve2_shr_narrow(s
, a
, ops
);
7315 static void gen_sqshrnt_vec(unsigned vece
, TCGv_vec d
,
7316 TCGv_vec n
, int64_t shr
)
7318 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7319 int halfbits
= 4 << vece
;
7320 int64_t max
= MAKE_64BIT_MASK(0, halfbits
- 1);
7321 int64_t min
= -max
- 1;
7323 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7324 tcg_gen_dupi_vec(vece
, t
, min
);
7325 tcg_gen_smax_vec(vece
, n
, n
, t
);
7326 tcg_gen_dupi_vec(vece
, t
, max
);
7327 tcg_gen_smin_vec(vece
, n
, n
, t
);
7328 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7329 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7330 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7331 tcg_temp_free_vec(t
);
7334 static bool trans_SQSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7336 static const TCGOpcode vec_list
[] = {
7337 INDEX_op_shli_vec
, INDEX_op_sari_vec
,
7338 INDEX_op_smax_vec
, INDEX_op_smin_vec
, 0
7340 static const GVecGen2i ops
[3] = {
7341 { .fniv
= gen_sqshrnt_vec
,
7342 .opt_opc
= vec_list
,
7344 .fno
= gen_helper_sve2_sqshrnt_h
,
7346 { .fniv
= gen_sqshrnt_vec
,
7347 .opt_opc
= vec_list
,
7349 .fno
= gen_helper_sve2_sqshrnt_s
,
7351 { .fniv
= gen_sqshrnt_vec
,
7352 .opt_opc
= vec_list
,
7354 .fno
= gen_helper_sve2_sqshrnt_d
,
7357 return do_sve2_shr_narrow(s
, a
, ops
);
7360 static bool trans_SQRSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7362 static const GVecGen2i ops
[3] = {
7363 { .fno
= gen_helper_sve2_sqrshrnb_h
},
7364 { .fno
= gen_helper_sve2_sqrshrnb_s
},
7365 { .fno
= gen_helper_sve2_sqrshrnb_d
},
7367 return do_sve2_shr_narrow(s
, a
, ops
);
7370 static bool trans_SQRSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7372 static const GVecGen2i ops
[3] = {
7373 { .fno
= gen_helper_sve2_sqrshrnt_h
},
7374 { .fno
= gen_helper_sve2_sqrshrnt_s
},
7375 { .fno
= gen_helper_sve2_sqrshrnt_d
},
7377 return do_sve2_shr_narrow(s
, a
, ops
);
7380 static void gen_uqshrnb_vec(unsigned vece
, TCGv_vec d
,
7381 TCGv_vec n
, int64_t shr
)
7383 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7384 int halfbits
= 4 << vece
;
7386 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7387 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7388 tcg_gen_umin_vec(vece
, d
, n
, t
);
7389 tcg_temp_free_vec(t
);
7392 static bool trans_UQSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7394 static const TCGOpcode vec_list
[] = {
7395 INDEX_op_shri_vec
, INDEX_op_umin_vec
, 0
7397 static const GVecGen2i ops
[3] = {
7398 { .fniv
= gen_uqshrnb_vec
,
7399 .opt_opc
= vec_list
,
7400 .fno
= gen_helper_sve2_uqshrnb_h
,
7402 { .fniv
= gen_uqshrnb_vec
,
7403 .opt_opc
= vec_list
,
7404 .fno
= gen_helper_sve2_uqshrnb_s
,
7406 { .fniv
= gen_uqshrnb_vec
,
7407 .opt_opc
= vec_list
,
7408 .fno
= gen_helper_sve2_uqshrnb_d
,
7411 return do_sve2_shr_narrow(s
, a
, ops
);
7414 static void gen_uqshrnt_vec(unsigned vece
, TCGv_vec d
,
7415 TCGv_vec n
, int64_t shr
)
7417 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7418 int halfbits
= 4 << vece
;
7420 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7421 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7422 tcg_gen_umin_vec(vece
, n
, n
, t
);
7423 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7424 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7425 tcg_temp_free_vec(t
);
7428 static bool trans_UQSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7430 static const TCGOpcode vec_list
[] = {
7431 INDEX_op_shli_vec
, INDEX_op_shri_vec
, INDEX_op_umin_vec
, 0
7433 static const GVecGen2i ops
[3] = {
7434 { .fniv
= gen_uqshrnt_vec
,
7435 .opt_opc
= vec_list
,
7437 .fno
= gen_helper_sve2_uqshrnt_h
,
7439 { .fniv
= gen_uqshrnt_vec
,
7440 .opt_opc
= vec_list
,
7442 .fno
= gen_helper_sve2_uqshrnt_s
,
7444 { .fniv
= gen_uqshrnt_vec
,
7445 .opt_opc
= vec_list
,
7447 .fno
= gen_helper_sve2_uqshrnt_d
,
7450 return do_sve2_shr_narrow(s
, a
, ops
);
7453 static bool trans_UQRSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7455 static const GVecGen2i ops
[3] = {
7456 { .fno
= gen_helper_sve2_uqrshrnb_h
},
7457 { .fno
= gen_helper_sve2_uqrshrnb_s
},
7458 { .fno
= gen_helper_sve2_uqrshrnb_d
},
7460 return do_sve2_shr_narrow(s
, a
, ops
);
7463 static bool trans_UQRSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7465 static const GVecGen2i ops
[3] = {
7466 { .fno
= gen_helper_sve2_uqrshrnt_h
},
7467 { .fno
= gen_helper_sve2_uqrshrnt_s
},
7468 { .fno
= gen_helper_sve2_uqrshrnt_d
},
7470 return do_sve2_shr_narrow(s
, a
, ops
);
7473 #define DO_SVE2_ZZZ_NARROW(NAME, name) \
7474 static gen_helper_gvec_3 * const name##_fns[4] = { \
7475 NULL, gen_helper_sve2_##name##_h, \
7476 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
7478 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_arg_zzz, \
7479 name##_fns[a->esz], a, 0)
7481 DO_SVE2_ZZZ_NARROW(ADDHNB
, addhnb
)
7482 DO_SVE2_ZZZ_NARROW(ADDHNT
, addhnt
)
7483 DO_SVE2_ZZZ_NARROW(RADDHNB
, raddhnb
)
7484 DO_SVE2_ZZZ_NARROW(RADDHNT
, raddhnt
)
7486 DO_SVE2_ZZZ_NARROW(SUBHNB
, subhnb
)
7487 DO_SVE2_ZZZ_NARROW(SUBHNT
, subhnt
)
7488 DO_SVE2_ZZZ_NARROW(RSUBHNB
, rsubhnb
)
7489 DO_SVE2_ZZZ_NARROW(RSUBHNT
, rsubhnt
)
7491 static bool do_sve2_ppzz_flags(DisasContext
*s
, arg_rprr_esz
*a
,
7492 gen_helper_gvec_flags_4
*fn
)
7494 if (!dc_isar_feature(aa64_sve2
, s
)) {
7497 return do_ppzz_flags(s
, a
, fn
);
7500 #define DO_SVE2_PPZZ_MATCH(NAME, name) \
7501 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
7503 static gen_helper_gvec_flags_4 * const fns[4] = { \
7504 gen_helper_sve2_##name##_ppzz_b, gen_helper_sve2_##name##_ppzz_h, \
7507 return do_sve2_ppzz_flags(s, a, fns[a->esz]); \
7510 DO_SVE2_PPZZ_MATCH(MATCH
, match
)
7511 DO_SVE2_PPZZ_MATCH(NMATCH
, nmatch
)
7513 static gen_helper_gvec_4
* const histcnt_fns
[4] = {
7514 NULL
, NULL
, gen_helper_sve2_histcnt_s
, gen_helper_sve2_histcnt_d
7516 TRANS_FEAT(HISTCNT
, aa64_sve2
, gen_gvec_ool_arg_zpzz
,
7517 histcnt_fns
[a
->esz
], a
, 0)
7519 TRANS_FEAT(HISTSEG
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
7520 a
->esz
== 0 ? gen_helper_sve2_histseg
: NULL
, a
, 0)
7522 static bool do_sve2_zpzz_fp(DisasContext
*s
, arg_rprr_esz
*a
,
7523 gen_helper_gvec_4_ptr
*fn
)
7525 if (!dc_isar_feature(aa64_sve2
, s
)) {
7528 return do_zpzz_fp(s
, a
, fn
);
7531 #define DO_SVE2_ZPZZ_FP(NAME, name) \
7532 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
7534 static gen_helper_gvec_4_ptr * const fns[4] = { \
7535 NULL, gen_helper_sve2_##name##_zpzz_h, \
7536 gen_helper_sve2_##name##_zpzz_s, gen_helper_sve2_##name##_zpzz_d \
7538 return do_sve2_zpzz_fp(s, a, fns[a->esz]); \
7541 DO_SVE2_ZPZZ_FP(FADDP
, faddp
)
7542 DO_SVE2_ZPZZ_FP(FMAXNMP
, fmaxnmp
)
7543 DO_SVE2_ZPZZ_FP(FMINNMP
, fminnmp
)
7544 DO_SVE2_ZPZZ_FP(FMAXP
, fmaxp
)
7545 DO_SVE2_ZPZZ_FP(FMINP
, fminp
)
7548 * SVE Integer Multiply-Add (unpredicated)
7551 static bool trans_FMMLA(DisasContext
*s
, arg_rrrr_esz
*a
)
7553 gen_helper_gvec_4_ptr
*fn
;
7557 if (!dc_isar_feature(aa64_sve_f32mm
, s
)) {
7560 fn
= gen_helper_fmmla_s
;
7563 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
7566 fn
= gen_helper_fmmla_d
;
7572 if (sve_access_check(s
)) {
7573 unsigned vsz
= vec_full_reg_size(s
);
7574 TCGv_ptr status
= fpstatus_ptr(FPST_FPCR
);
7575 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
7576 vec_full_reg_offset(s
, a
->rn
),
7577 vec_full_reg_offset(s
, a
->rm
),
7578 vec_full_reg_offset(s
, a
->ra
),
7579 status
, vsz
, vsz
, 0, fn
);
7580 tcg_temp_free_ptr(status
);
7585 static gen_helper_gvec_4
* const sqdmlal_zzzw_fns
[] = {
7586 NULL
, gen_helper_sve2_sqdmlal_zzzw_h
,
7587 gen_helper_sve2_sqdmlal_zzzw_s
, gen_helper_sve2_sqdmlal_zzzw_d
,
7589 TRANS_FEAT(SQDMLALB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7590 sqdmlal_zzzw_fns
[a
->esz
], a
, 0)
7591 TRANS_FEAT(SQDMLALT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7592 sqdmlal_zzzw_fns
[a
->esz
], a
, 3)
7593 TRANS_FEAT(SQDMLALBT
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7594 sqdmlal_zzzw_fns
[a
->esz
], a
, 2)
7596 static gen_helper_gvec_4
* const sqdmlsl_zzzw_fns
[] = {
7597 NULL
, gen_helper_sve2_sqdmlsl_zzzw_h
,
7598 gen_helper_sve2_sqdmlsl_zzzw_s
, gen_helper_sve2_sqdmlsl_zzzw_d
,
7600 TRANS_FEAT(SQDMLSLB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7601 sqdmlsl_zzzw_fns
[a
->esz
], a
, 0)
7602 TRANS_FEAT(SQDMLSLT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7603 sqdmlsl_zzzw_fns
[a
->esz
], a
, 3)
7604 TRANS_FEAT(SQDMLSLBT
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7605 sqdmlsl_zzzw_fns
[a
->esz
], a
, 2)
7607 static gen_helper_gvec_4
* const sqrdmlah_fns
[] = {
7608 gen_helper_sve2_sqrdmlah_b
, gen_helper_sve2_sqrdmlah_h
,
7609 gen_helper_sve2_sqrdmlah_s
, gen_helper_sve2_sqrdmlah_d
,
7611 TRANS_FEAT(SQRDMLAH_zzzz
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7612 sqrdmlah_fns
[a
->esz
], a
, 0)
7614 static gen_helper_gvec_4
* const sqrdmlsh_fns
[] = {
7615 gen_helper_sve2_sqrdmlsh_b
, gen_helper_sve2_sqrdmlsh_h
,
7616 gen_helper_sve2_sqrdmlsh_s
, gen_helper_sve2_sqrdmlsh_d
,
7618 TRANS_FEAT(SQRDMLSH_zzzz
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7619 sqrdmlsh_fns
[a
->esz
], a
, 0)
7621 static gen_helper_gvec_4
* const smlal_zzzw_fns
[] = {
7622 NULL
, gen_helper_sve2_smlal_zzzw_h
,
7623 gen_helper_sve2_smlal_zzzw_s
, gen_helper_sve2_smlal_zzzw_d
,
7625 TRANS_FEAT(SMLALB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7626 smlal_zzzw_fns
[a
->esz
], a
, 0)
7627 TRANS_FEAT(SMLALT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7628 smlal_zzzw_fns
[a
->esz
], a
, 1)
7630 static gen_helper_gvec_4
* const umlal_zzzw_fns
[] = {
7631 NULL
, gen_helper_sve2_umlal_zzzw_h
,
7632 gen_helper_sve2_umlal_zzzw_s
, gen_helper_sve2_umlal_zzzw_d
,
7634 TRANS_FEAT(UMLALB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7635 umlal_zzzw_fns
[a
->esz
], a
, 0)
7636 TRANS_FEAT(UMLALT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7637 umlal_zzzw_fns
[a
->esz
], a
, 1)
7639 static gen_helper_gvec_4
* const smlsl_zzzw_fns
[] = {
7640 NULL
, gen_helper_sve2_smlsl_zzzw_h
,
7641 gen_helper_sve2_smlsl_zzzw_s
, gen_helper_sve2_smlsl_zzzw_d
,
7643 TRANS_FEAT(SMLSLB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7644 smlsl_zzzw_fns
[a
->esz
], a
, 0)
7645 TRANS_FEAT(SMLSLT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7646 smlsl_zzzw_fns
[a
->esz
], a
, 1)
7648 static gen_helper_gvec_4
* const umlsl_zzzw_fns
[] = {
7649 NULL
, gen_helper_sve2_umlsl_zzzw_h
,
7650 gen_helper_sve2_umlsl_zzzw_s
, gen_helper_sve2_umlsl_zzzw_d
,
7652 TRANS_FEAT(UMLSLB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7653 umlsl_zzzw_fns
[a
->esz
], a
, 0)
7654 TRANS_FEAT(UMLSLT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7655 umlsl_zzzw_fns
[a
->esz
], a
, 1)
7657 static gen_helper_gvec_4
* const cmla_fns
[] = {
7658 gen_helper_sve2_cmla_zzzz_b
, gen_helper_sve2_cmla_zzzz_h
,
7659 gen_helper_sve2_cmla_zzzz_s
, gen_helper_sve2_cmla_zzzz_d
,
7661 TRANS_FEAT(CMLA_zzzz
, aa64_sve2
, gen_gvec_ool_zzzz
,
7662 cmla_fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
)
7664 static gen_helper_gvec_4
* const cdot_fns
[] = {
7665 NULL
, NULL
, gen_helper_sve2_cdot_zzzz_s
, gen_helper_sve2_cdot_zzzz_d
7667 TRANS_FEAT(CDOT_zzzz
, aa64_sve2
, gen_gvec_ool_zzzz
,
7668 cdot_fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
)
7670 static gen_helper_gvec_4
* const sqrdcmlah_fns
[] = {
7671 gen_helper_sve2_sqrdcmlah_zzzz_b
, gen_helper_sve2_sqrdcmlah_zzzz_h
,
7672 gen_helper_sve2_sqrdcmlah_zzzz_s
, gen_helper_sve2_sqrdcmlah_zzzz_d
,
7674 TRANS_FEAT(SQRDCMLAH_zzzz
, aa64_sve2
, gen_gvec_ool_zzzz
,
7675 sqrdcmlah_fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
)
7677 TRANS_FEAT(USDOT_zzzz
, aa64_sve_i8mm
, gen_gvec_ool_arg_zzzz
,
7678 a
->esz
== 2 ? gen_helper_gvec_usdot_b
: NULL
, a
, 0)
7680 TRANS_FEAT(AESMC
, aa64_sve2_aes
, gen_gvec_ool_zz
,
7681 gen_helper_crypto_aesmc
, a
->rd
, a
->rd
, a
->decrypt
)
7683 TRANS_FEAT(AESE
, aa64_sve2_aes
, gen_gvec_ool_arg_zzz
,
7684 gen_helper_crypto_aese
, a
, false)
7685 TRANS_FEAT(AESD
, aa64_sve2_aes
, gen_gvec_ool_arg_zzz
,
7686 gen_helper_crypto_aese
, a
, true)
7688 TRANS_FEAT(SM4E
, aa64_sve2_sm4
, gen_gvec_ool_arg_zzz
,
7689 gen_helper_crypto_sm4e
, a
, 0)
7690 TRANS_FEAT(SM4EKEY
, aa64_sve2_sm4
, gen_gvec_ool_arg_zzz
,
7691 gen_helper_crypto_sm4ekey
, a
, 0)
7693 TRANS_FEAT(RAX1
, aa64_sve2_sha3
, gen_gvec_fn_arg_zzz
, gen_gvec_rax1
, a
)
7695 static bool trans_FCVTNT_sh(DisasContext
*s
, arg_rpr_esz
*a
)
7697 if (!dc_isar_feature(aa64_sve2
, s
)) {
7700 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtnt_sh
);
7703 static bool trans_BFCVTNT(DisasContext
*s
, arg_rpr_esz
*a
)
7705 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
7708 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_bfcvtnt
);
7711 static bool trans_FCVTNT_ds(DisasContext
*s
, arg_rpr_esz
*a
)
7713 if (!dc_isar_feature(aa64_sve2
, s
)) {
7716 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtnt_ds
);
7719 static bool trans_FCVTLT_hs(DisasContext
*s
, arg_rpr_esz
*a
)
7721 if (!dc_isar_feature(aa64_sve2
, s
)) {
7724 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtlt_hs
);
7727 static bool trans_FCVTLT_sd(DisasContext
*s
, arg_rpr_esz
*a
)
7729 if (!dc_isar_feature(aa64_sve2
, s
)) {
7732 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtlt_sd
);
7735 static bool trans_FCVTX_ds(DisasContext
*s
, arg_rpr_esz
*a
)
7737 if (!dc_isar_feature(aa64_sve2
, s
)) {
7740 return do_frint_mode(s
, a
, float_round_to_odd
, gen_helper_sve_fcvt_ds
);
7743 static bool trans_FCVTXNT_ds(DisasContext
*s
, arg_rpr_esz
*a
)
7745 if (!dc_isar_feature(aa64_sve2
, s
)) {
7748 return do_frint_mode(s
, a
, float_round_to_odd
, gen_helper_sve2_fcvtnt_ds
);
7751 static bool trans_FLOGB(DisasContext
*s
, arg_rpr_esz
*a
)
7753 static gen_helper_gvec_3_ptr
* const fns
[] = {
7754 NULL
, gen_helper_flogb_h
,
7755 gen_helper_flogb_s
, gen_helper_flogb_d
7758 if (!dc_isar_feature(aa64_sve2
, s
) || fns
[a
->esz
] == NULL
) {
7761 if (sve_access_check(s
)) {
7763 fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
7764 unsigned vsz
= vec_full_reg_size(s
);
7766 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
7767 vec_full_reg_offset(s
, a
->rn
),
7768 pred_full_reg_offset(s
, a
->pg
),
7769 status
, vsz
, vsz
, 0, fns
[a
->esz
]);
7770 tcg_temp_free_ptr(status
);
7775 static bool do_FMLAL_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sub
, bool sel
)
7777 if (!dc_isar_feature(aa64_sve2
, s
)) {
7780 if (sve_access_check(s
)) {
7781 unsigned vsz
= vec_full_reg_size(s
);
7782 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
7783 vec_full_reg_offset(s
, a
->rn
),
7784 vec_full_reg_offset(s
, a
->rm
),
7785 vec_full_reg_offset(s
, a
->ra
),
7786 cpu_env
, vsz
, vsz
, (sel
<< 1) | sub
,
7787 gen_helper_sve2_fmlal_zzzw_s
);
7792 static bool trans_FMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7794 return do_FMLAL_zzzw(s
, a
, false, false);
7797 static bool trans_FMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7799 return do_FMLAL_zzzw(s
, a
, false, true);
7802 static bool trans_FMLSLB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7804 return do_FMLAL_zzzw(s
, a
, true, false);
7807 static bool trans_FMLSLT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7809 return do_FMLAL_zzzw(s
, a
, true, true);
7812 static bool do_FMLAL_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
, bool sub
, bool sel
)
7814 if (!dc_isar_feature(aa64_sve2
, s
)) {
7817 if (sve_access_check(s
)) {
7818 unsigned vsz
= vec_full_reg_size(s
);
7819 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
7820 vec_full_reg_offset(s
, a
->rn
),
7821 vec_full_reg_offset(s
, a
->rm
),
7822 vec_full_reg_offset(s
, a
->ra
),
7824 (a
->index
<< 2) | (sel
<< 1) | sub
,
7825 gen_helper_sve2_fmlal_zzxw_s
);
7830 static bool trans_FMLALB_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
7832 return do_FMLAL_zzxw(s
, a
, false, false);
7835 static bool trans_FMLALT_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
7837 return do_FMLAL_zzxw(s
, a
, false, true);
7840 static bool trans_FMLSLB_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
7842 return do_FMLAL_zzxw(s
, a
, true, false);
7845 static bool trans_FMLSLT_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
7847 return do_FMLAL_zzxw(s
, a
, true, true);
7850 TRANS_FEAT(SMMLA
, aa64_sve_i8mm
, gen_gvec_ool_arg_zzzz
,
7851 gen_helper_gvec_smmla_b
, a
, 0)
7852 TRANS_FEAT(USMMLA
, aa64_sve_i8mm
, gen_gvec_ool_arg_zzzz
,
7853 gen_helper_gvec_usmmla_b
, a
, 0)
7854 TRANS_FEAT(UMMLA
, aa64_sve_i8mm
, gen_gvec_ool_arg_zzzz
,
7855 gen_helper_gvec_ummla_b
, a
, 0)
7857 TRANS_FEAT(BFDOT_zzzz
, aa64_sve_bf16
, gen_gvec_ool_arg_zzzz
,
7858 gen_helper_gvec_bfdot
, a
, 0)
7859 TRANS_FEAT(BFDOT_zzxz
, aa64_sve_bf16
, gen_gvec_ool_arg_zzxz
,
7860 gen_helper_gvec_bfdot_idx
, a
)
7862 TRANS_FEAT(BFMMLA
, aa64_sve_bf16
, gen_gvec_ool_arg_zzzz
,
7863 gen_helper_gvec_bfmmla
, a
, 0)
7865 static bool do_BFMLAL_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
7867 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
7870 if (sve_access_check(s
)) {
7871 TCGv_ptr status
= fpstatus_ptr(FPST_FPCR
);
7872 unsigned vsz
= vec_full_reg_size(s
);
7874 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
7875 vec_full_reg_offset(s
, a
->rn
),
7876 vec_full_reg_offset(s
, a
->rm
),
7877 vec_full_reg_offset(s
, a
->ra
),
7878 status
, vsz
, vsz
, sel
,
7879 gen_helper_gvec_bfmlal
);
7880 tcg_temp_free_ptr(status
);
7885 static bool trans_BFMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7887 return do_BFMLAL_zzzw(s
, a
, false);
7890 static bool trans_BFMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7892 return do_BFMLAL_zzzw(s
, a
, true);
7895 static bool do_BFMLAL_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
, bool sel
)
7897 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
7900 if (sve_access_check(s
)) {
7901 TCGv_ptr status
= fpstatus_ptr(FPST_FPCR
);
7902 unsigned vsz
= vec_full_reg_size(s
);
7904 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
7905 vec_full_reg_offset(s
, a
->rn
),
7906 vec_full_reg_offset(s
, a
->rm
),
7907 vec_full_reg_offset(s
, a
->ra
),
7908 status
, vsz
, vsz
, (a
->index
<< 1) | sel
,
7909 gen_helper_gvec_bfmlal_idx
);
7910 tcg_temp_free_ptr(status
);
7915 static bool trans_BFMLALB_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
7917 return do_BFMLAL_zzxw(s
, a
, false);
7920 static bool trans_BFMLALT_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
7922 return do_BFMLAL_zzxw(s
, a
, true);