2 * AArch64 SVE translation
4 * Copyright (c) 2018 Linaro, Ltd
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
22 #include "exec/exec-all.h"
23 #include "tcg/tcg-op.h"
24 #include "tcg/tcg-op-gvec.h"
25 #include "tcg/tcg-gvec-desc.h"
28 #include "translate.h"
29 #include "internals.h"
30 #include "exec/helper-proto.h"
31 #include "exec/helper-gen.h"
33 #include "translate-a64.h"
34 #include "fpu/softfloat.h"
37 typedef void GVecGen2sFn(unsigned, uint32_t, uint32_t,
38 TCGv_i64
, uint32_t, uint32_t);
40 typedef void gen_helper_gvec_flags_3(TCGv_i32
, TCGv_ptr
, TCGv_ptr
,
42 typedef void gen_helper_gvec_flags_4(TCGv_i32
, TCGv_ptr
, TCGv_ptr
,
43 TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
45 typedef void gen_helper_gvec_mem(TCGv_env
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
46 typedef void gen_helper_gvec_mem_scatter(TCGv_env
, TCGv_ptr
, TCGv_ptr
,
47 TCGv_ptr
, TCGv_i64
, TCGv_i32
);
50 * Helpers for extracting complex instruction fields.
53 /* See e.g. ASR (immediate, predicated).
54 * Returns -1 for unallocated encoding; diagnose later.
56 static int tszimm_esz(DisasContext
*s
, int x
)
58 x
>>= 3; /* discard imm3 */
62 static int tszimm_shr(DisasContext
*s
, int x
)
64 return (16 << tszimm_esz(s
, x
)) - x
;
67 /* See e.g. LSL (immediate, predicated). */
68 static int tszimm_shl(DisasContext
*s
, int x
)
70 return x
- (8 << tszimm_esz(s
, x
));
73 /* The SH bit is in bit 8. Extract the low 8 and shift. */
74 static inline int expand_imm_sh8s(DisasContext
*s
, int x
)
76 return (int8_t)x
<< (x
& 0x100 ? 8 : 0);
79 static inline int expand_imm_sh8u(DisasContext
*s
, int x
)
81 return (uint8_t)x
<< (x
& 0x100 ? 8 : 0);
84 /* Convert a 2-bit memory size (msz) to a 4-bit data type (dtype)
85 * with unsigned data. C.f. SVE Memory Contiguous Load Group.
87 static inline int msz_dtype(DisasContext
*s
, int msz
)
89 static const uint8_t dtype
[4] = { 0, 5, 10, 15 };
94 * Include the generated decoder.
97 #include "decode-sve.c.inc"
100 * Implement all of the translator functions referenced by the decoder.
103 /* Return the offset info CPUARMState of the predicate vector register Pn.
104 * Note for this purpose, FFR is P16.
106 static inline int pred_full_reg_offset(DisasContext
*s
, int regno
)
108 return offsetof(CPUARMState
, vfp
.pregs
[regno
]);
111 /* Return the byte size of the whole predicate register, VL / 64. */
112 static inline int pred_full_reg_size(DisasContext
*s
)
114 return s
->sve_len
>> 3;
117 /* Round up the size of a register to a size allowed by
118 * the tcg vector infrastructure. Any operation which uses this
119 * size may assume that the bits above pred_full_reg_size are zero,
120 * and must leave them the same way.
122 * Note that this is not needed for the vector registers as they
123 * are always properly sized for tcg vectors.
125 static int size_for_gvec(int size
)
130 return QEMU_ALIGN_UP(size
, 16);
134 static int pred_gvec_reg_size(DisasContext
*s
)
136 return size_for_gvec(pred_full_reg_size(s
));
139 /* Invoke an out-of-line helper on 2 Zregs. */
140 static bool gen_gvec_ool_zz(DisasContext
*s
, gen_helper_gvec_2
*fn
,
141 int rd
, int rn
, int data
)
146 if (sve_access_check(s
)) {
147 unsigned vsz
= vec_full_reg_size(s
);
148 tcg_gen_gvec_2_ool(vec_full_reg_offset(s
, rd
),
149 vec_full_reg_offset(s
, rn
),
155 /* Invoke an out-of-line helper on 3 Zregs. */
156 static bool gen_gvec_ool_zzz(DisasContext
*s
, gen_helper_gvec_3
*fn
,
157 int rd
, int rn
, int rm
, int data
)
162 if (sve_access_check(s
)) {
163 unsigned vsz
= vec_full_reg_size(s
);
164 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
165 vec_full_reg_offset(s
, rn
),
166 vec_full_reg_offset(s
, rm
),
172 static bool gen_gvec_ool_arg_zzz(DisasContext
*s
, gen_helper_gvec_3
*fn
,
173 arg_rrr_esz
*a
, int data
)
175 return gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, data
);
178 /* Invoke an out-of-line helper on 4 Zregs. */
179 static bool gen_gvec_ool_zzzz(DisasContext
*s
, gen_helper_gvec_4
*fn
,
180 int rd
, int rn
, int rm
, int ra
, int data
)
185 if (sve_access_check(s
)) {
186 unsigned vsz
= vec_full_reg_size(s
);
187 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
188 vec_full_reg_offset(s
, rn
),
189 vec_full_reg_offset(s
, rm
),
190 vec_full_reg_offset(s
, ra
),
196 static bool gen_gvec_ool_arg_zzzz(DisasContext
*s
, gen_helper_gvec_4
*fn
,
197 arg_rrrr_esz
*a
, int data
)
199 return gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, data
);
202 static bool gen_gvec_ool_arg_zzxz(DisasContext
*s
, gen_helper_gvec_4
*fn
,
205 return gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->index
);
208 /* Invoke an out-of-line helper on 2 Zregs and a predicate. */
209 static bool gen_gvec_ool_zzp(DisasContext
*s
, gen_helper_gvec_3
*fn
,
210 int rd
, int rn
, int pg
, int data
)
215 if (sve_access_check(s
)) {
216 unsigned vsz
= vec_full_reg_size(s
);
217 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
218 vec_full_reg_offset(s
, rn
),
219 pred_full_reg_offset(s
, pg
),
225 static bool gen_gvec_ool_arg_zpz(DisasContext
*s
, gen_helper_gvec_3
*fn
,
226 arg_rpr_esz
*a
, int data
)
228 return gen_gvec_ool_zzp(s
, fn
, a
->rd
, a
->rn
, a
->pg
, data
);
231 static bool gen_gvec_ool_arg_zpzi(DisasContext
*s
, gen_helper_gvec_3
*fn
,
234 return gen_gvec_ool_zzp(s
, fn
, a
->rd
, a
->rn
, a
->pg
, a
->imm
);
237 /* Invoke an out-of-line helper on 3 Zregs and a predicate. */
238 static bool gen_gvec_ool_zzzp(DisasContext
*s
, gen_helper_gvec_4
*fn
,
239 int rd
, int rn
, int rm
, int pg
, int data
)
244 if (sve_access_check(s
)) {
245 unsigned vsz
= vec_full_reg_size(s
);
246 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
247 vec_full_reg_offset(s
, rn
),
248 vec_full_reg_offset(s
, rm
),
249 pred_full_reg_offset(s
, pg
),
255 static bool gen_gvec_ool_arg_zpzz(DisasContext
*s
, gen_helper_gvec_4
*fn
,
256 arg_rprr_esz
*a
, int data
)
258 return gen_gvec_ool_zzzp(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->pg
, data
);
261 /* Invoke a vector expander on two Zregs and an immediate. */
262 static bool gen_gvec_fn_zzi(DisasContext
*s
, GVecGen2iFn
*gvec_fn
,
263 int esz
, int rd
, int rn
, uint64_t imm
)
265 if (gvec_fn
== NULL
) {
268 if (sve_access_check(s
)) {
269 unsigned vsz
= vec_full_reg_size(s
);
270 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
271 vec_full_reg_offset(s
, rn
), imm
, vsz
, vsz
);
276 static bool gen_gvec_fn_arg_zzi(DisasContext
*s
, GVecGen2iFn
*gvec_fn
,
280 /* Invalid tsz encoding -- see tszimm_esz. */
283 return gen_gvec_fn_zzi(s
, gvec_fn
, a
->esz
, a
->rd
, a
->rn
, a
->imm
);
286 /* Invoke a vector expander on three Zregs. */
287 static bool gen_gvec_fn_zzz(DisasContext
*s
, GVecGen3Fn
*gvec_fn
,
288 int esz
, int rd
, int rn
, int rm
)
290 if (gvec_fn
== NULL
) {
293 if (sve_access_check(s
)) {
294 unsigned vsz
= vec_full_reg_size(s
);
295 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
296 vec_full_reg_offset(s
, rn
),
297 vec_full_reg_offset(s
, rm
), vsz
, vsz
);
302 static bool gen_gvec_fn_arg_zzz(DisasContext
*s
, GVecGen3Fn
*fn
,
305 return gen_gvec_fn_zzz(s
, fn
, a
->esz
, a
->rd
, a
->rn
, a
->rm
);
308 /* Invoke a vector expander on four Zregs. */
309 static bool gen_gvec_fn_arg_zzzz(DisasContext
*s
, GVecGen4Fn
*gvec_fn
,
312 if (gvec_fn
== NULL
) {
315 if (sve_access_check(s
)) {
316 unsigned vsz
= vec_full_reg_size(s
);
317 gvec_fn(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
318 vec_full_reg_offset(s
, a
->rn
),
319 vec_full_reg_offset(s
, a
->rm
),
320 vec_full_reg_offset(s
, a
->ra
), vsz
, vsz
);
325 /* Invoke a vector move on two Zregs. */
326 static bool do_mov_z(DisasContext
*s
, int rd
, int rn
)
328 if (sve_access_check(s
)) {
329 unsigned vsz
= vec_full_reg_size(s
);
330 tcg_gen_gvec_mov(MO_8
, vec_full_reg_offset(s
, rd
),
331 vec_full_reg_offset(s
, rn
), vsz
, vsz
);
336 /* Initialize a Zreg with replications of a 64-bit immediate. */
337 static void do_dupi_z(DisasContext
*s
, int rd
, uint64_t word
)
339 unsigned vsz
= vec_full_reg_size(s
);
340 tcg_gen_gvec_dup_imm(MO_64
, vec_full_reg_offset(s
, rd
), vsz
, vsz
, word
);
343 /* Invoke a vector expander on three Pregs. */
344 static void gen_gvec_fn_ppp(DisasContext
*s
, GVecGen3Fn
*gvec_fn
,
345 int rd
, int rn
, int rm
)
347 unsigned psz
= pred_gvec_reg_size(s
);
348 gvec_fn(MO_64
, pred_full_reg_offset(s
, rd
),
349 pred_full_reg_offset(s
, rn
),
350 pred_full_reg_offset(s
, rm
), psz
, psz
);
353 /* Invoke a vector move on two Pregs. */
354 static bool do_mov_p(DisasContext
*s
, int rd
, int rn
)
356 if (sve_access_check(s
)) {
357 unsigned psz
= pred_gvec_reg_size(s
);
358 tcg_gen_gvec_mov(MO_8
, pred_full_reg_offset(s
, rd
),
359 pred_full_reg_offset(s
, rn
), psz
, psz
);
364 /* Set the cpu flags as per a return from an SVE helper. */
365 static void do_pred_flags(TCGv_i32 t
)
367 tcg_gen_mov_i32(cpu_NF
, t
);
368 tcg_gen_andi_i32(cpu_ZF
, t
, 2);
369 tcg_gen_andi_i32(cpu_CF
, t
, 1);
370 tcg_gen_movi_i32(cpu_VF
, 0);
373 /* Subroutines computing the ARM PredTest psuedofunction. */
374 static void do_predtest1(TCGv_i64 d
, TCGv_i64 g
)
376 TCGv_i32 t
= tcg_temp_new_i32();
378 gen_helper_sve_predtest1(t
, d
, g
);
380 tcg_temp_free_i32(t
);
383 static void do_predtest(DisasContext
*s
, int dofs
, int gofs
, int words
)
385 TCGv_ptr dptr
= tcg_temp_new_ptr();
386 TCGv_ptr gptr
= tcg_temp_new_ptr();
387 TCGv_i32 t
= tcg_temp_new_i32();
389 tcg_gen_addi_ptr(dptr
, cpu_env
, dofs
);
390 tcg_gen_addi_ptr(gptr
, cpu_env
, gofs
);
392 gen_helper_sve_predtest(t
, dptr
, gptr
, tcg_constant_i32(words
));
393 tcg_temp_free_ptr(dptr
);
394 tcg_temp_free_ptr(gptr
);
397 tcg_temp_free_i32(t
);
400 /* For each element size, the bits within a predicate word that are active. */
401 const uint64_t pred_esz_masks
[4] = {
402 0xffffffffffffffffull
, 0x5555555555555555ull
,
403 0x1111111111111111ull
, 0x0101010101010101ull
406 static bool trans_INVALID(DisasContext
*s
, arg_INVALID
*a
)
408 unallocated_encoding(s
);
413 *** SVE Logical - Unpredicated Group
416 TRANS_FEAT(AND_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_and
, a
)
417 TRANS_FEAT(ORR_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_or
, a
)
418 TRANS_FEAT(EOR_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_xor
, a
)
419 TRANS_FEAT(BIC_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_andc
, a
)
421 static void gen_xar8_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
423 TCGv_i64 t
= tcg_temp_new_i64();
424 uint64_t mask
= dup_const(MO_8
, 0xff >> sh
);
426 tcg_gen_xor_i64(t
, n
, m
);
427 tcg_gen_shri_i64(d
, t
, sh
);
428 tcg_gen_shli_i64(t
, t
, 8 - sh
);
429 tcg_gen_andi_i64(d
, d
, mask
);
430 tcg_gen_andi_i64(t
, t
, ~mask
);
431 tcg_gen_or_i64(d
, d
, t
);
432 tcg_temp_free_i64(t
);
435 static void gen_xar16_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
437 TCGv_i64 t
= tcg_temp_new_i64();
438 uint64_t mask
= dup_const(MO_16
, 0xffff >> sh
);
440 tcg_gen_xor_i64(t
, n
, m
);
441 tcg_gen_shri_i64(d
, t
, sh
);
442 tcg_gen_shli_i64(t
, t
, 16 - sh
);
443 tcg_gen_andi_i64(d
, d
, mask
);
444 tcg_gen_andi_i64(t
, t
, ~mask
);
445 tcg_gen_or_i64(d
, d
, t
);
446 tcg_temp_free_i64(t
);
449 static void gen_xar_i32(TCGv_i32 d
, TCGv_i32 n
, TCGv_i32 m
, int32_t sh
)
451 tcg_gen_xor_i32(d
, n
, m
);
452 tcg_gen_rotri_i32(d
, d
, sh
);
455 static void gen_xar_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
457 tcg_gen_xor_i64(d
, n
, m
);
458 tcg_gen_rotri_i64(d
, d
, sh
);
461 static void gen_xar_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
462 TCGv_vec m
, int64_t sh
)
464 tcg_gen_xor_vec(vece
, d
, n
, m
);
465 tcg_gen_rotri_vec(vece
, d
, d
, sh
);
468 void gen_gvec_xar(unsigned vece
, uint32_t rd_ofs
, uint32_t rn_ofs
,
469 uint32_t rm_ofs
, int64_t shift
,
470 uint32_t opr_sz
, uint32_t max_sz
)
472 static const TCGOpcode vecop
[] = { INDEX_op_rotli_vec
, 0 };
473 static const GVecGen3i ops
[4] = {
474 { .fni8
= gen_xar8_i64
,
476 .fno
= gen_helper_sve2_xar_b
,
479 { .fni8
= gen_xar16_i64
,
481 .fno
= gen_helper_sve2_xar_h
,
484 { .fni4
= gen_xar_i32
,
486 .fno
= gen_helper_sve2_xar_s
,
489 { .fni8
= gen_xar_i64
,
491 .fno
= gen_helper_gvec_xar_d
,
495 int esize
= 8 << vece
;
497 /* The SVE2 range is 1 .. esize; the AdvSIMD range is 0 .. esize-1. */
498 tcg_debug_assert(shift
>= 0);
499 tcg_debug_assert(shift
<= esize
);
503 /* xar with no rotate devolves to xor. */
504 tcg_gen_gvec_xor(vece
, rd_ofs
, rn_ofs
, rm_ofs
, opr_sz
, max_sz
);
506 tcg_gen_gvec_3i(rd_ofs
, rn_ofs
, rm_ofs
, opr_sz
, max_sz
,
511 static bool trans_XAR(DisasContext
*s
, arg_rrri_esz
*a
)
513 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
516 if (sve_access_check(s
)) {
517 unsigned vsz
= vec_full_reg_size(s
);
518 gen_gvec_xar(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
519 vec_full_reg_offset(s
, a
->rn
),
520 vec_full_reg_offset(s
, a
->rm
), a
->imm
, vsz
, vsz
);
525 static void gen_eor3_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
527 tcg_gen_xor_i64(d
, n
, m
);
528 tcg_gen_xor_i64(d
, d
, k
);
531 static void gen_eor3_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
532 TCGv_vec m
, TCGv_vec k
)
534 tcg_gen_xor_vec(vece
, d
, n
, m
);
535 tcg_gen_xor_vec(vece
, d
, d
, k
);
538 static void gen_eor3(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
539 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
541 static const GVecGen4 op
= {
542 .fni8
= gen_eor3_i64
,
543 .fniv
= gen_eor3_vec
,
544 .fno
= gen_helper_sve2_eor3
,
546 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
548 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
551 TRANS_FEAT(EOR3
, aa64_sve2
, gen_gvec_fn_arg_zzzz
, gen_eor3
, a
)
553 static void gen_bcax_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
555 tcg_gen_andc_i64(d
, m
, k
);
556 tcg_gen_xor_i64(d
, d
, n
);
559 static void gen_bcax_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
560 TCGv_vec m
, TCGv_vec k
)
562 tcg_gen_andc_vec(vece
, d
, m
, k
);
563 tcg_gen_xor_vec(vece
, d
, d
, n
);
566 static void gen_bcax(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
567 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
569 static const GVecGen4 op
= {
570 .fni8
= gen_bcax_i64
,
571 .fniv
= gen_bcax_vec
,
572 .fno
= gen_helper_sve2_bcax
,
574 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
576 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
579 TRANS_FEAT(BCAX
, aa64_sve2
, gen_gvec_fn_arg_zzzz
, gen_bcax
, a
)
581 static void gen_bsl(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
582 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
584 /* BSL differs from the generic bitsel in argument ordering. */
585 tcg_gen_gvec_bitsel(vece
, d
, a
, n
, m
, oprsz
, maxsz
);
588 TRANS_FEAT(BSL
, aa64_sve2
, gen_gvec_fn_arg_zzzz
, gen_bsl
, a
)
590 static void gen_bsl1n_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
592 tcg_gen_andc_i64(n
, k
, n
);
593 tcg_gen_andc_i64(m
, m
, k
);
594 tcg_gen_or_i64(d
, n
, m
);
597 static void gen_bsl1n_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
598 TCGv_vec m
, TCGv_vec k
)
600 if (TCG_TARGET_HAS_bitsel_vec
) {
601 tcg_gen_not_vec(vece
, n
, n
);
602 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
604 tcg_gen_andc_vec(vece
, n
, k
, n
);
605 tcg_gen_andc_vec(vece
, m
, m
, k
);
606 tcg_gen_or_vec(vece
, d
, n
, m
);
610 static void gen_bsl1n(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
611 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
613 static const GVecGen4 op
= {
614 .fni8
= gen_bsl1n_i64
,
615 .fniv
= gen_bsl1n_vec
,
616 .fno
= gen_helper_sve2_bsl1n
,
618 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
620 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
623 TRANS_FEAT(BSL1N
, aa64_sve2
, gen_gvec_fn_arg_zzzz
, gen_bsl1n
, a
)
625 static void gen_bsl2n_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
628 * Z[dn] = (n & k) | (~m & ~k)
631 tcg_gen_and_i64(n
, n
, k
);
632 if (TCG_TARGET_HAS_orc_i64
) {
633 tcg_gen_or_i64(m
, m
, k
);
634 tcg_gen_orc_i64(d
, n
, m
);
636 tcg_gen_nor_i64(m
, m
, k
);
637 tcg_gen_or_i64(d
, n
, m
);
641 static void gen_bsl2n_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
642 TCGv_vec m
, TCGv_vec k
)
644 if (TCG_TARGET_HAS_bitsel_vec
) {
645 tcg_gen_not_vec(vece
, m
, m
);
646 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
648 tcg_gen_and_vec(vece
, n
, n
, k
);
649 tcg_gen_or_vec(vece
, m
, m
, k
);
650 tcg_gen_orc_vec(vece
, d
, n
, m
);
654 static void gen_bsl2n(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
655 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
657 static const GVecGen4 op
= {
658 .fni8
= gen_bsl2n_i64
,
659 .fniv
= gen_bsl2n_vec
,
660 .fno
= gen_helper_sve2_bsl2n
,
662 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
664 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
667 TRANS_FEAT(BSL2N
, aa64_sve2
, gen_gvec_fn_arg_zzzz
, gen_bsl2n
, a
)
669 static void gen_nbsl_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
671 tcg_gen_and_i64(n
, n
, k
);
672 tcg_gen_andc_i64(m
, m
, k
);
673 tcg_gen_nor_i64(d
, n
, m
);
676 static void gen_nbsl_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
677 TCGv_vec m
, TCGv_vec k
)
679 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
680 tcg_gen_not_vec(vece
, d
, d
);
683 static void gen_nbsl(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
684 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
686 static const GVecGen4 op
= {
687 .fni8
= gen_nbsl_i64
,
688 .fniv
= gen_nbsl_vec
,
689 .fno
= gen_helper_sve2_nbsl
,
691 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
693 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
696 TRANS_FEAT(NBSL
, aa64_sve2
, gen_gvec_fn_arg_zzzz
, gen_nbsl
, a
)
699 *** SVE Integer Arithmetic - Unpredicated Group
702 TRANS_FEAT(ADD_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_add
, a
)
703 TRANS_FEAT(SUB_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_sub
, a
)
704 TRANS_FEAT(SQADD_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_ssadd
, a
)
705 TRANS_FEAT(SQSUB_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_sssub
, a
)
706 TRANS_FEAT(UQADD_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_usadd
, a
)
707 TRANS_FEAT(UQSUB_zzz
, aa64_sve
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_ussub
, a
)
710 *** SVE Integer Arithmetic - Binary Predicated Group
713 /* Select active elememnts from Zn and inactive elements from Zm,
714 * storing the result in Zd.
716 static bool do_sel_z(DisasContext
*s
, int rd
, int rn
, int rm
, int pg
, int esz
)
718 static gen_helper_gvec_4
* const fns
[4] = {
719 gen_helper_sve_sel_zpzz_b
, gen_helper_sve_sel_zpzz_h
,
720 gen_helper_sve_sel_zpzz_s
, gen_helper_sve_sel_zpzz_d
722 return gen_gvec_ool_zzzp(s
, fns
[esz
], rd
, rn
, rm
, pg
, 0);
725 #define DO_ZPZZ(NAME, FEAT, name) \
726 static gen_helper_gvec_4 * const name##_zpzz_fns[4] = { \
727 gen_helper_##name##_zpzz_b, gen_helper_##name##_zpzz_h, \
728 gen_helper_##name##_zpzz_s, gen_helper_##name##_zpzz_d, \
730 TRANS_FEAT(NAME, FEAT, gen_gvec_ool_arg_zpzz, \
731 name##_zpzz_fns[a->esz], a, 0)
733 DO_ZPZZ(AND_zpzz
, aa64_sve
, sve_and
)
734 DO_ZPZZ(EOR_zpzz
, aa64_sve
, sve_eor
)
735 DO_ZPZZ(ORR_zpzz
, aa64_sve
, sve_orr
)
736 DO_ZPZZ(BIC_zpzz
, aa64_sve
, sve_bic
)
738 DO_ZPZZ(ADD_zpzz
, aa64_sve
, sve_add
)
739 DO_ZPZZ(SUB_zpzz
, aa64_sve
, sve_sub
)
741 DO_ZPZZ(SMAX_zpzz
, aa64_sve
, sve_smax
)
742 DO_ZPZZ(UMAX_zpzz
, aa64_sve
, sve_umax
)
743 DO_ZPZZ(SMIN_zpzz
, aa64_sve
, sve_smin
)
744 DO_ZPZZ(UMIN_zpzz
, aa64_sve
, sve_umin
)
745 DO_ZPZZ(SABD_zpzz
, aa64_sve
, sve_sabd
)
746 DO_ZPZZ(UABD_zpzz
, aa64_sve
, sve_uabd
)
748 DO_ZPZZ(MUL_zpzz
, aa64_sve
, sve_mul
)
749 DO_ZPZZ(SMULH_zpzz
, aa64_sve
, sve_smulh
)
750 DO_ZPZZ(UMULH_zpzz
, aa64_sve
, sve_umulh
)
752 DO_ZPZZ(ASR_zpzz
, aa64_sve
, sve_asr
)
753 DO_ZPZZ(LSR_zpzz
, aa64_sve
, sve_lsr
)
754 DO_ZPZZ(LSL_zpzz
, aa64_sve
, sve_lsl
)
756 static gen_helper_gvec_4
* const sdiv_fns
[4] = {
757 NULL
, NULL
, gen_helper_sve_sdiv_zpzz_s
, gen_helper_sve_sdiv_zpzz_d
759 TRANS_FEAT(SDIV_zpzz
, aa64_sve
, gen_gvec_ool_arg_zpzz
, sdiv_fns
[a
->esz
], a
, 0)
761 static gen_helper_gvec_4
* const udiv_fns
[4] = {
762 NULL
, NULL
, gen_helper_sve_udiv_zpzz_s
, gen_helper_sve_udiv_zpzz_d
764 TRANS_FEAT(UDIV_zpzz
, aa64_sve
, gen_gvec_ool_arg_zpzz
, udiv_fns
[a
->esz
], a
, 0)
766 static bool trans_SEL_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
768 return do_sel_z(s
, a
->rd
, a
->rn
, a
->rm
, a
->pg
, a
->esz
);
772 *** SVE Integer Arithmetic - Unary Predicated Group
775 #define DO_ZPZ(NAME, FEAT, name) \
776 static gen_helper_gvec_3 * const name##_fns[4] = { \
777 gen_helper_##name##_b, gen_helper_##name##_h, \
778 gen_helper_##name##_s, gen_helper_##name##_d, \
780 TRANS_FEAT(NAME, FEAT, gen_gvec_ool_arg_zpz, name##_fns[a->esz], a, 0)
782 DO_ZPZ(CLS
, aa64_sve
, sve_cls
)
783 DO_ZPZ(CLZ
, aa64_sve
, sve_clz
)
784 DO_ZPZ(CNT_zpz
, aa64_sve
, sve_cnt_zpz
)
785 DO_ZPZ(CNOT
, aa64_sve
, sve_cnot
)
786 DO_ZPZ(NOT_zpz
, aa64_sve
, sve_not_zpz
)
787 DO_ZPZ(ABS
, aa64_sve
, sve_abs
)
788 DO_ZPZ(NEG
, aa64_sve
, sve_neg
)
789 DO_ZPZ(RBIT
, aa64_sve
, sve_rbit
)
791 static gen_helper_gvec_3
* const fabs_fns
[4] = {
792 NULL
, gen_helper_sve_fabs_h
,
793 gen_helper_sve_fabs_s
, gen_helper_sve_fabs_d
,
795 TRANS_FEAT(FABS
, aa64_sve
, gen_gvec_ool_arg_zpz
, fabs_fns
[a
->esz
], a
, 0)
797 static gen_helper_gvec_3
* const fneg_fns
[4] = {
798 NULL
, gen_helper_sve_fneg_h
,
799 gen_helper_sve_fneg_s
, gen_helper_sve_fneg_d
,
801 TRANS_FEAT(FNEG
, aa64_sve
, gen_gvec_ool_arg_zpz
, fneg_fns
[a
->esz
], a
, 0)
803 static gen_helper_gvec_3
* const sxtb_fns
[4] = {
804 NULL
, gen_helper_sve_sxtb_h
,
805 gen_helper_sve_sxtb_s
, gen_helper_sve_sxtb_d
,
807 TRANS_FEAT(SXTB
, aa64_sve
, gen_gvec_ool_arg_zpz
, sxtb_fns
[a
->esz
], a
, 0)
809 static gen_helper_gvec_3
* const uxtb_fns
[4] = {
810 NULL
, gen_helper_sve_uxtb_h
,
811 gen_helper_sve_uxtb_s
, gen_helper_sve_uxtb_d
,
813 TRANS_FEAT(UXTB
, aa64_sve
, gen_gvec_ool_arg_zpz
, uxtb_fns
[a
->esz
], a
, 0)
815 static gen_helper_gvec_3
* const sxth_fns
[4] = {
816 NULL
, NULL
, gen_helper_sve_sxth_s
, gen_helper_sve_sxth_d
818 TRANS_FEAT(SXTH
, aa64_sve
, gen_gvec_ool_arg_zpz
, sxth_fns
[a
->esz
], a
, 0)
820 static gen_helper_gvec_3
* const uxth_fns
[4] = {
821 NULL
, NULL
, gen_helper_sve_uxth_s
, gen_helper_sve_uxth_d
823 TRANS_FEAT(UXTH
, aa64_sve
, gen_gvec_ool_arg_zpz
, uxth_fns
[a
->esz
], a
, 0)
825 TRANS_FEAT(SXTW
, aa64_sve
, gen_gvec_ool_arg_zpz
,
826 a
->esz
== 3 ? gen_helper_sve_sxtw_d
: NULL
, a
, 0)
827 TRANS_FEAT(UXTW
, aa64_sve
, gen_gvec_ool_arg_zpz
,
828 a
->esz
== 3 ? gen_helper_sve_uxtw_d
: NULL
, a
, 0)
831 *** SVE Integer Reduction Group
834 typedef void gen_helper_gvec_reduc(TCGv_i64
, TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
835 static bool do_vpz_ool(DisasContext
*s
, arg_rpr_esz
*a
,
836 gen_helper_gvec_reduc
*fn
)
838 unsigned vsz
= vec_full_reg_size(s
);
846 if (!sve_access_check(s
)) {
850 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
851 temp
= tcg_temp_new_i64();
852 t_zn
= tcg_temp_new_ptr();
853 t_pg
= tcg_temp_new_ptr();
855 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
856 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
857 fn(temp
, t_zn
, t_pg
, desc
);
858 tcg_temp_free_ptr(t_zn
);
859 tcg_temp_free_ptr(t_pg
);
861 write_fp_dreg(s
, a
->rd
, temp
);
862 tcg_temp_free_i64(temp
);
866 #define DO_VPZ(NAME, name) \
867 static gen_helper_gvec_reduc * const name##_fns[4] = { \
868 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
869 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
871 TRANS_FEAT(NAME, aa64_sve, do_vpz_ool, a, name##_fns[a->esz])
883 static gen_helper_gvec_reduc
* const saddv_fns
[4] = {
884 gen_helper_sve_saddv_b
, gen_helper_sve_saddv_h
,
885 gen_helper_sve_saddv_s
, NULL
887 TRANS_FEAT(SADDV
, aa64_sve
, do_vpz_ool
, a
, saddv_fns
[a
->esz
])
892 *** SVE Shift by Immediate - Predicated Group
896 * Copy Zn into Zd, storing zeros into inactive elements.
897 * If invert, store zeros into the active elements.
899 static bool do_movz_zpz(DisasContext
*s
, int rd
, int rn
, int pg
,
900 int esz
, bool invert
)
902 static gen_helper_gvec_3
* const fns
[4] = {
903 gen_helper_sve_movz_b
, gen_helper_sve_movz_h
,
904 gen_helper_sve_movz_s
, gen_helper_sve_movz_d
,
906 return gen_gvec_ool_zzp(s
, fns
[esz
], rd
, rn
, pg
, invert
);
909 static bool do_shift_zpzi(DisasContext
*s
, arg_rpri_esz
*a
, bool asr
,
910 gen_helper_gvec_3
* const fns
[4])
915 /* Invalid tsz encoding -- see tszimm_esz. */
920 * Shift by element size is architecturally valid.
921 * For arithmetic right-shift, it's the same as by one less.
922 * For logical shifts and ASRD, it is a zeroing operation.
929 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
932 return gen_gvec_ool_arg_zpzi(s
, fns
[a
->esz
], a
);
935 static gen_helper_gvec_3
* const asr_zpzi_fns
[4] = {
936 gen_helper_sve_asr_zpzi_b
, gen_helper_sve_asr_zpzi_h
,
937 gen_helper_sve_asr_zpzi_s
, gen_helper_sve_asr_zpzi_d
,
939 TRANS_FEAT(ASR_zpzi
, aa64_sve
, do_shift_zpzi
, a
, true, asr_zpzi_fns
)
941 static gen_helper_gvec_3
* const lsr_zpzi_fns
[4] = {
942 gen_helper_sve_lsr_zpzi_b
, gen_helper_sve_lsr_zpzi_h
,
943 gen_helper_sve_lsr_zpzi_s
, gen_helper_sve_lsr_zpzi_d
,
945 TRANS_FEAT(LSR_zpzi
, aa64_sve
, do_shift_zpzi
, a
, false, lsr_zpzi_fns
)
947 static gen_helper_gvec_3
* const lsl_zpzi_fns
[4] = {
948 gen_helper_sve_lsl_zpzi_b
, gen_helper_sve_lsl_zpzi_h
,
949 gen_helper_sve_lsl_zpzi_s
, gen_helper_sve_lsl_zpzi_d
,
951 TRANS_FEAT(LSL_zpzi
, aa64_sve
, do_shift_zpzi
, a
, false, lsl_zpzi_fns
)
953 static gen_helper_gvec_3
* const asrd_fns
[4] = {
954 gen_helper_sve_asrd_b
, gen_helper_sve_asrd_h
,
955 gen_helper_sve_asrd_s
, gen_helper_sve_asrd_d
,
957 TRANS_FEAT(ASRD
, aa64_sve
, do_shift_zpzi
, a
, false, asrd_fns
)
959 static gen_helper_gvec_3
* const sqshl_zpzi_fns
[4] = {
960 gen_helper_sve2_sqshl_zpzi_b
, gen_helper_sve2_sqshl_zpzi_h
,
961 gen_helper_sve2_sqshl_zpzi_s
, gen_helper_sve2_sqshl_zpzi_d
,
963 TRANS_FEAT(SQSHL_zpzi
, aa64_sve2
, gen_gvec_ool_arg_zpzi
,
964 a
->esz
< 0 ? NULL
: sqshl_zpzi_fns
[a
->esz
], a
)
966 static gen_helper_gvec_3
* const uqshl_zpzi_fns
[4] = {
967 gen_helper_sve2_uqshl_zpzi_b
, gen_helper_sve2_uqshl_zpzi_h
,
968 gen_helper_sve2_uqshl_zpzi_s
, gen_helper_sve2_uqshl_zpzi_d
,
970 TRANS_FEAT(UQSHL_zpzi
, aa64_sve2
, gen_gvec_ool_arg_zpzi
,
971 a
->esz
< 0 ? NULL
: uqshl_zpzi_fns
[a
->esz
], a
)
973 static gen_helper_gvec_3
* const srshr_fns
[4] = {
974 gen_helper_sve2_srshr_b
, gen_helper_sve2_srshr_h
,
975 gen_helper_sve2_srshr_s
, gen_helper_sve2_srshr_d
,
977 TRANS_FEAT(SRSHR
, aa64_sve2
, gen_gvec_ool_arg_zpzi
,
978 a
->esz
< 0 ? NULL
: srshr_fns
[a
->esz
], a
)
980 static gen_helper_gvec_3
* const urshr_fns
[4] = {
981 gen_helper_sve2_urshr_b
, gen_helper_sve2_urshr_h
,
982 gen_helper_sve2_urshr_s
, gen_helper_sve2_urshr_d
,
984 TRANS_FEAT(URSHR
, aa64_sve2
, gen_gvec_ool_arg_zpzi
,
985 a
->esz
< 0 ? NULL
: urshr_fns
[a
->esz
], a
)
987 static gen_helper_gvec_3
* const sqshlu_fns
[4] = {
988 gen_helper_sve2_sqshlu_b
, gen_helper_sve2_sqshlu_h
,
989 gen_helper_sve2_sqshlu_s
, gen_helper_sve2_sqshlu_d
,
991 TRANS_FEAT(SQSHLU
, aa64_sve2
, gen_gvec_ool_arg_zpzi
,
992 a
->esz
< 0 ? NULL
: sqshlu_fns
[a
->esz
], a
)
995 *** SVE Bitwise Shift - Predicated Group
998 #define DO_ZPZW(NAME, name) \
999 static gen_helper_gvec_4 * const name##_zpzw_fns[4] = { \
1000 gen_helper_sve_##name##_zpzw_b, gen_helper_sve_##name##_zpzw_h, \
1001 gen_helper_sve_##name##_zpzw_s, NULL \
1003 TRANS_FEAT(NAME##_zpzw, aa64_sve, gen_gvec_ool_arg_zpzz, \
1004 a->esz < 0 ? NULL : name##_zpzw_fns[a->esz], a, 0)
1013 *** SVE Bitwise Shift - Unpredicated Group
1016 static bool do_shift_imm(DisasContext
*s
, arg_rri_esz
*a
, bool asr
,
1017 void (*gvec_fn
)(unsigned, uint32_t, uint32_t,
1018 int64_t, uint32_t, uint32_t))
1021 /* Invalid tsz encoding -- see tszimm_esz. */
1024 if (sve_access_check(s
)) {
1025 unsigned vsz
= vec_full_reg_size(s
);
1026 /* Shift by element size is architecturally valid. For
1027 arithmetic right-shift, it's the same as by one less.
1028 Otherwise it is a zeroing operation. */
1029 if (a
->imm
>= 8 << a
->esz
) {
1031 a
->imm
= (8 << a
->esz
) - 1;
1033 do_dupi_z(s
, a
->rd
, 0);
1037 gvec_fn(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
1038 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
1043 TRANS_FEAT(ASR_zzi
, aa64_sve
, do_shift_imm
, a
, true, tcg_gen_gvec_sari
)
1044 TRANS_FEAT(LSR_zzi
, aa64_sve
, do_shift_imm
, a
, false, tcg_gen_gvec_shri
)
1045 TRANS_FEAT(LSL_zzi
, aa64_sve
, do_shift_imm
, a
, false, tcg_gen_gvec_shli
)
1047 #define DO_ZZW(NAME, name) \
1048 static gen_helper_gvec_3 * const name##_zzw_fns[4] = { \
1049 gen_helper_sve_##name##_zzw_b, gen_helper_sve_##name##_zzw_h, \
1050 gen_helper_sve_##name##_zzw_s, NULL \
1052 TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_arg_zzz, \
1053 name##_zzw_fns[a->esz], a, 0)
1055 DO_ZZW(ASR_zzw
, asr
)
1056 DO_ZZW(LSR_zzw
, lsr
)
1057 DO_ZZW(LSL_zzw
, lsl
)
1062 *** SVE Integer Multiply-Add Group
1065 static bool do_zpzzz_ool(DisasContext
*s
, arg_rprrr_esz
*a
,
1066 gen_helper_gvec_5
*fn
)
1068 if (sve_access_check(s
)) {
1069 unsigned vsz
= vec_full_reg_size(s
);
1070 tcg_gen_gvec_5_ool(vec_full_reg_offset(s
, a
->rd
),
1071 vec_full_reg_offset(s
, a
->ra
),
1072 vec_full_reg_offset(s
, a
->rn
),
1073 vec_full_reg_offset(s
, a
->rm
),
1074 pred_full_reg_offset(s
, a
->pg
),
1080 static gen_helper_gvec_5
* const mla_fns
[4] = {
1081 gen_helper_sve_mla_b
, gen_helper_sve_mla_h
,
1082 gen_helper_sve_mla_s
, gen_helper_sve_mla_d
,
1084 TRANS_FEAT(MLA
, aa64_sve
, do_zpzzz_ool
, a
, mla_fns
[a
->esz
])
1086 static gen_helper_gvec_5
* const mls_fns
[4] = {
1087 gen_helper_sve_mls_b
, gen_helper_sve_mls_h
,
1088 gen_helper_sve_mls_s
, gen_helper_sve_mls_d
,
1090 TRANS_FEAT(MLS
, aa64_sve
, do_zpzzz_ool
, a
, mls_fns
[a
->esz
])
1093 *** SVE Index Generation Group
1096 static bool do_index(DisasContext
*s
, int esz
, int rd
,
1097 TCGv_i64 start
, TCGv_i64 incr
)
1103 if (!sve_access_check(s
)) {
1107 vsz
= vec_full_reg_size(s
);
1108 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
1109 t_zd
= tcg_temp_new_ptr();
1111 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, rd
));
1113 gen_helper_sve_index_d(t_zd
, start
, incr
, desc
);
1115 typedef void index_fn(TCGv_ptr
, TCGv_i32
, TCGv_i32
, TCGv_i32
);
1116 static index_fn
* const fns
[3] = {
1117 gen_helper_sve_index_b
,
1118 gen_helper_sve_index_h
,
1119 gen_helper_sve_index_s
,
1121 TCGv_i32 s32
= tcg_temp_new_i32();
1122 TCGv_i32 i32
= tcg_temp_new_i32();
1124 tcg_gen_extrl_i64_i32(s32
, start
);
1125 tcg_gen_extrl_i64_i32(i32
, incr
);
1126 fns
[esz
](t_zd
, s32
, i32
, desc
);
1128 tcg_temp_free_i32(s32
);
1129 tcg_temp_free_i32(i32
);
1131 tcg_temp_free_ptr(t_zd
);
1135 TRANS_FEAT(INDEX_ii
, aa64_sve
, do_index
, a
->esz
, a
->rd
,
1136 tcg_constant_i64(a
->imm1
), tcg_constant_i64(a
->imm2
))
1137 TRANS_FEAT(INDEX_ir
, aa64_sve
, do_index
, a
->esz
, a
->rd
,
1138 tcg_constant_i64(a
->imm
), cpu_reg(s
, a
->rm
))
1139 TRANS_FEAT(INDEX_ri
, aa64_sve
, do_index
, a
->esz
, a
->rd
,
1140 cpu_reg(s
, a
->rn
), tcg_constant_i64(a
->imm
))
1141 TRANS_FEAT(INDEX_rr
, aa64_sve
, do_index
, a
->esz
, a
->rd
,
1142 cpu_reg(s
, a
->rn
), cpu_reg(s
, a
->rm
))
1145 *** SVE Stack Allocation Group
1148 static bool trans_ADDVL(DisasContext
*s
, arg_ADDVL
*a
)
1150 if (sve_access_check(s
)) {
1151 TCGv_i64 rd
= cpu_reg_sp(s
, a
->rd
);
1152 TCGv_i64 rn
= cpu_reg_sp(s
, a
->rn
);
1153 tcg_gen_addi_i64(rd
, rn
, a
->imm
* vec_full_reg_size(s
));
1158 static bool trans_ADDPL(DisasContext
*s
, arg_ADDPL
*a
)
1160 if (sve_access_check(s
)) {
1161 TCGv_i64 rd
= cpu_reg_sp(s
, a
->rd
);
1162 TCGv_i64 rn
= cpu_reg_sp(s
, a
->rn
);
1163 tcg_gen_addi_i64(rd
, rn
, a
->imm
* pred_full_reg_size(s
));
1168 static bool trans_RDVL(DisasContext
*s
, arg_RDVL
*a
)
1170 if (sve_access_check(s
)) {
1171 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
1172 tcg_gen_movi_i64(reg
, a
->imm
* vec_full_reg_size(s
));
1178 *** SVE Compute Vector Address Group
1181 static bool do_adr(DisasContext
*s
, arg_rrri
*a
, gen_helper_gvec_3
*fn
)
1183 return gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->imm
);
1186 TRANS_FEAT(ADR_p32
, aa64_sve
, do_adr
, a
, gen_helper_sve_adr_p32
)
1187 TRANS_FEAT(ADR_p64
, aa64_sve
, do_adr
, a
, gen_helper_sve_adr_p64
)
1188 TRANS_FEAT(ADR_s32
, aa64_sve
, do_adr
, a
, gen_helper_sve_adr_s32
)
1189 TRANS_FEAT(ADR_u32
, aa64_sve
, do_adr
, a
, gen_helper_sve_adr_u32
)
1192 *** SVE Integer Misc - Unpredicated Group
1195 static gen_helper_gvec_2
* const fexpa_fns
[4] = {
1196 NULL
, gen_helper_sve_fexpa_h
,
1197 gen_helper_sve_fexpa_s
, gen_helper_sve_fexpa_d
,
1199 TRANS_FEAT(FEXPA
, aa64_sve
, gen_gvec_ool_zz
,
1200 fexpa_fns
[a
->esz
], a
->rd
, a
->rn
, 0)
1202 static gen_helper_gvec_3
* const ftssel_fns
[4] = {
1203 NULL
, gen_helper_sve_ftssel_h
,
1204 gen_helper_sve_ftssel_s
, gen_helper_sve_ftssel_d
,
1206 TRANS_FEAT(FTSSEL
, aa64_sve
, gen_gvec_ool_arg_zzz
, ftssel_fns
[a
->esz
], a
, 0)
1209 *** SVE Predicate Logical Operations Group
1212 static bool do_pppp_flags(DisasContext
*s
, arg_rprr_s
*a
,
1213 const GVecGen4
*gvec_op
)
1215 if (!sve_access_check(s
)) {
1219 unsigned psz
= pred_gvec_reg_size(s
);
1220 int dofs
= pred_full_reg_offset(s
, a
->rd
);
1221 int nofs
= pred_full_reg_offset(s
, a
->rn
);
1222 int mofs
= pred_full_reg_offset(s
, a
->rm
);
1223 int gofs
= pred_full_reg_offset(s
, a
->pg
);
1226 tcg_gen_gvec_4(dofs
, nofs
, mofs
, gofs
, psz
, psz
, gvec_op
);
1231 /* Do the operation and the flags generation in temps. */
1232 TCGv_i64 pd
= tcg_temp_new_i64();
1233 TCGv_i64 pn
= tcg_temp_new_i64();
1234 TCGv_i64 pm
= tcg_temp_new_i64();
1235 TCGv_i64 pg
= tcg_temp_new_i64();
1237 tcg_gen_ld_i64(pn
, cpu_env
, nofs
);
1238 tcg_gen_ld_i64(pm
, cpu_env
, mofs
);
1239 tcg_gen_ld_i64(pg
, cpu_env
, gofs
);
1241 gvec_op
->fni8(pd
, pn
, pm
, pg
);
1242 tcg_gen_st_i64(pd
, cpu_env
, dofs
);
1244 do_predtest1(pd
, pg
);
1246 tcg_temp_free_i64(pd
);
1247 tcg_temp_free_i64(pn
);
1248 tcg_temp_free_i64(pm
);
1249 tcg_temp_free_i64(pg
);
1251 /* The operation and flags generation is large. The computation
1252 * of the flags depends on the original contents of the guarding
1253 * predicate. If the destination overwrites the guarding predicate,
1254 * then the easiest way to get this right is to save a copy.
1257 if (a
->rd
== a
->pg
) {
1258 tofs
= offsetof(CPUARMState
, vfp
.preg_tmp
);
1259 tcg_gen_gvec_mov(0, tofs
, gofs
, psz
, psz
);
1262 tcg_gen_gvec_4(dofs
, nofs
, mofs
, gofs
, psz
, psz
, gvec_op
);
1263 do_predtest(s
, dofs
, tofs
, psz
/ 8);
1268 static void gen_and_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1270 tcg_gen_and_i64(pd
, pn
, pm
);
1271 tcg_gen_and_i64(pd
, pd
, pg
);
1274 static void gen_and_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1275 TCGv_vec pm
, TCGv_vec pg
)
1277 tcg_gen_and_vec(vece
, pd
, pn
, pm
);
1278 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1281 static bool trans_AND_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1283 static const GVecGen4 op
= {
1284 .fni8
= gen_and_pg_i64
,
1285 .fniv
= gen_and_pg_vec
,
1286 .fno
= gen_helper_sve_and_pppp
,
1287 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1291 if (!sve_access_check(s
)) {
1294 if (a
->rn
== a
->rm
) {
1295 if (a
->pg
== a
->rn
) {
1296 do_mov_p(s
, a
->rd
, a
->rn
);
1298 gen_gvec_fn_ppp(s
, tcg_gen_gvec_and
, a
->rd
, a
->rn
, a
->pg
);
1301 } else if (a
->pg
== a
->rn
|| a
->pg
== a
->rm
) {
1302 gen_gvec_fn_ppp(s
, tcg_gen_gvec_and
, a
->rd
, a
->rn
, a
->rm
);
1306 return do_pppp_flags(s
, a
, &op
);
1309 static void gen_bic_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1311 tcg_gen_andc_i64(pd
, pn
, pm
);
1312 tcg_gen_and_i64(pd
, pd
, pg
);
1315 static void gen_bic_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1316 TCGv_vec pm
, TCGv_vec pg
)
1318 tcg_gen_andc_vec(vece
, pd
, pn
, pm
);
1319 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1322 static bool trans_BIC_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1324 static const GVecGen4 op
= {
1325 .fni8
= gen_bic_pg_i64
,
1326 .fniv
= gen_bic_pg_vec
,
1327 .fno
= gen_helper_sve_bic_pppp
,
1328 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1331 if (!a
->s
&& a
->pg
== a
->rn
) {
1332 if (sve_access_check(s
)) {
1333 gen_gvec_fn_ppp(s
, tcg_gen_gvec_andc
, a
->rd
, a
->rn
, a
->rm
);
1337 return do_pppp_flags(s
, a
, &op
);
1340 static void gen_eor_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1342 tcg_gen_xor_i64(pd
, pn
, pm
);
1343 tcg_gen_and_i64(pd
, pd
, pg
);
1346 static void gen_eor_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1347 TCGv_vec pm
, TCGv_vec pg
)
1349 tcg_gen_xor_vec(vece
, pd
, pn
, pm
);
1350 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1353 static bool trans_EOR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1355 static const GVecGen4 op
= {
1356 .fni8
= gen_eor_pg_i64
,
1357 .fniv
= gen_eor_pg_vec
,
1358 .fno
= gen_helper_sve_eor_pppp
,
1359 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1361 return do_pppp_flags(s
, a
, &op
);
1364 static bool trans_SEL_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1369 if (sve_access_check(s
)) {
1370 unsigned psz
= pred_gvec_reg_size(s
);
1371 tcg_gen_gvec_bitsel(MO_8
, pred_full_reg_offset(s
, a
->rd
),
1372 pred_full_reg_offset(s
, a
->pg
),
1373 pred_full_reg_offset(s
, a
->rn
),
1374 pred_full_reg_offset(s
, a
->rm
), psz
, psz
);
1379 static void gen_orr_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1381 tcg_gen_or_i64(pd
, pn
, pm
);
1382 tcg_gen_and_i64(pd
, pd
, pg
);
1385 static void gen_orr_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1386 TCGv_vec pm
, TCGv_vec pg
)
1388 tcg_gen_or_vec(vece
, pd
, pn
, pm
);
1389 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1392 static bool trans_ORR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1394 static const GVecGen4 op
= {
1395 .fni8
= gen_orr_pg_i64
,
1396 .fniv
= gen_orr_pg_vec
,
1397 .fno
= gen_helper_sve_orr_pppp
,
1398 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1401 if (!a
->s
&& a
->pg
== a
->rn
&& a
->rn
== a
->rm
) {
1402 return do_mov_p(s
, a
->rd
, a
->rn
);
1404 return do_pppp_flags(s
, a
, &op
);
1407 static void gen_orn_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1409 tcg_gen_orc_i64(pd
, pn
, pm
);
1410 tcg_gen_and_i64(pd
, pd
, pg
);
1413 static void gen_orn_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1414 TCGv_vec pm
, TCGv_vec pg
)
1416 tcg_gen_orc_vec(vece
, pd
, pn
, pm
);
1417 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1420 static bool trans_ORN_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1422 static const GVecGen4 op
= {
1423 .fni8
= gen_orn_pg_i64
,
1424 .fniv
= gen_orn_pg_vec
,
1425 .fno
= gen_helper_sve_orn_pppp
,
1426 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1428 return do_pppp_flags(s
, a
, &op
);
1431 static void gen_nor_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1433 tcg_gen_or_i64(pd
, pn
, pm
);
1434 tcg_gen_andc_i64(pd
, pg
, pd
);
1437 static void gen_nor_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1438 TCGv_vec pm
, TCGv_vec pg
)
1440 tcg_gen_or_vec(vece
, pd
, pn
, pm
);
1441 tcg_gen_andc_vec(vece
, pd
, pg
, pd
);
1444 static bool trans_NOR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1446 static const GVecGen4 op
= {
1447 .fni8
= gen_nor_pg_i64
,
1448 .fniv
= gen_nor_pg_vec
,
1449 .fno
= gen_helper_sve_nor_pppp
,
1450 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1452 return do_pppp_flags(s
, a
, &op
);
1455 static void gen_nand_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1457 tcg_gen_and_i64(pd
, pn
, pm
);
1458 tcg_gen_andc_i64(pd
, pg
, pd
);
1461 static void gen_nand_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1462 TCGv_vec pm
, TCGv_vec pg
)
1464 tcg_gen_and_vec(vece
, pd
, pn
, pm
);
1465 tcg_gen_andc_vec(vece
, pd
, pg
, pd
);
1468 static bool trans_NAND_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1470 static const GVecGen4 op
= {
1471 .fni8
= gen_nand_pg_i64
,
1472 .fniv
= gen_nand_pg_vec
,
1473 .fno
= gen_helper_sve_nand_pppp
,
1474 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1476 return do_pppp_flags(s
, a
, &op
);
1480 *** SVE Predicate Misc Group
1483 static bool trans_PTEST(DisasContext
*s
, arg_PTEST
*a
)
1485 if (sve_access_check(s
)) {
1486 int nofs
= pred_full_reg_offset(s
, a
->rn
);
1487 int gofs
= pred_full_reg_offset(s
, a
->pg
);
1488 int words
= DIV_ROUND_UP(pred_full_reg_size(s
), 8);
1491 TCGv_i64 pn
= tcg_temp_new_i64();
1492 TCGv_i64 pg
= tcg_temp_new_i64();
1494 tcg_gen_ld_i64(pn
, cpu_env
, nofs
);
1495 tcg_gen_ld_i64(pg
, cpu_env
, gofs
);
1496 do_predtest1(pn
, pg
);
1498 tcg_temp_free_i64(pn
);
1499 tcg_temp_free_i64(pg
);
1501 do_predtest(s
, nofs
, gofs
, words
);
1507 /* See the ARM pseudocode DecodePredCount. */
1508 static unsigned decode_pred_count(unsigned fullsz
, int pattern
, int esz
)
1510 unsigned elements
= fullsz
>> esz
;
1514 case 0x0: /* POW2 */
1515 return pow2floor(elements
);
1526 case 0x9: /* VL16 */
1527 case 0xa: /* VL32 */
1528 case 0xb: /* VL64 */
1529 case 0xc: /* VL128 */
1530 case 0xd: /* VL256 */
1531 bound
= 16 << (pattern
- 9);
1533 case 0x1d: /* MUL4 */
1534 return elements
- elements
% 4;
1535 case 0x1e: /* MUL3 */
1536 return elements
- elements
% 3;
1537 case 0x1f: /* ALL */
1539 default: /* #uimm5 */
1542 return elements
>= bound
? bound
: 0;
1545 /* This handles all of the predicate initialization instructions,
1546 * PTRUE, PFALSE, SETFFR. For PFALSE, we will have set PAT == 32
1547 * so that decode_pred_count returns 0. For SETFFR, we will have
1548 * set RD == 16 == FFR.
1550 static bool do_predset(DisasContext
*s
, int esz
, int rd
, int pat
, bool setflag
)
1552 if (!sve_access_check(s
)) {
1556 unsigned fullsz
= vec_full_reg_size(s
);
1557 unsigned ofs
= pred_full_reg_offset(s
, rd
);
1558 unsigned numelem
, setsz
, i
;
1559 uint64_t word
, lastword
;
1562 numelem
= decode_pred_count(fullsz
, pat
, esz
);
1564 /* Determine what we must store into each bit, and how many. */
1566 lastword
= word
= 0;
1569 setsz
= numelem
<< esz
;
1570 lastword
= word
= pred_esz_masks
[esz
];
1572 lastword
&= MAKE_64BIT_MASK(0, setsz
% 64);
1576 t
= tcg_temp_new_i64();
1578 tcg_gen_movi_i64(t
, lastword
);
1579 tcg_gen_st_i64(t
, cpu_env
, ofs
);
1583 if (word
== lastword
) {
1584 unsigned maxsz
= size_for_gvec(fullsz
/ 8);
1585 unsigned oprsz
= size_for_gvec(setsz
/ 8);
1587 if (oprsz
* 8 == setsz
) {
1588 tcg_gen_gvec_dup_imm(MO_64
, ofs
, oprsz
, maxsz
, word
);
1596 tcg_gen_movi_i64(t
, word
);
1597 for (i
= 0; i
< QEMU_ALIGN_DOWN(setsz
, 8); i
+= 8) {
1598 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1600 if (lastword
!= word
) {
1601 tcg_gen_movi_i64(t
, lastword
);
1602 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1606 tcg_gen_movi_i64(t
, 0);
1607 for (; i
< fullsz
; i
+= 8) {
1608 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1613 tcg_temp_free_i64(t
);
1617 tcg_gen_movi_i32(cpu_NF
, -(word
!= 0));
1618 tcg_gen_movi_i32(cpu_CF
, word
== 0);
1619 tcg_gen_movi_i32(cpu_VF
, 0);
1620 tcg_gen_mov_i32(cpu_ZF
, cpu_NF
);
1625 TRANS_FEAT(PTRUE
, aa64_sve
, do_predset
, a
->esz
, a
->rd
, a
->pat
, a
->s
)
1627 /* Note pat == 31 is #all, to set all elements. */
1628 TRANS_FEAT(SETFFR
, aa64_sve
, do_predset
, 0, FFR_PRED_NUM
, 31, false)
1630 /* Note pat == 32 is #unimp, to set no elements. */
1631 TRANS_FEAT(PFALSE
, aa64_sve
, do_predset
, 0, a
->rd
, 32, false)
1633 static bool trans_RDFFR_p(DisasContext
*s
, arg_RDFFR_p
*a
)
1635 /* The path through do_pppp_flags is complicated enough to want to avoid
1636 * duplication. Frob the arguments into the form of a predicated AND.
1638 arg_rprr_s alt_a
= {
1639 .rd
= a
->rd
, .pg
= a
->pg
, .s
= a
->s
,
1640 .rn
= FFR_PRED_NUM
, .rm
= FFR_PRED_NUM
,
1642 return trans_AND_pppp(s
, &alt_a
);
1645 TRANS_FEAT(RDFFR
, aa64_sve
, do_mov_p
, a
->rd
, FFR_PRED_NUM
)
1646 TRANS_FEAT(WRFFR
, aa64_sve
, do_mov_p
, FFR_PRED_NUM
, a
->rn
)
1648 static bool do_pfirst_pnext(DisasContext
*s
, arg_rr_esz
*a
,
1649 void (*gen_fn
)(TCGv_i32
, TCGv_ptr
,
1650 TCGv_ptr
, TCGv_i32
))
1652 if (!sve_access_check(s
)) {
1656 TCGv_ptr t_pd
= tcg_temp_new_ptr();
1657 TCGv_ptr t_pg
= tcg_temp_new_ptr();
1661 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, pred_full_reg_size(s
));
1662 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
1664 tcg_gen_addi_ptr(t_pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
1665 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
1666 t
= tcg_temp_new_i32();
1668 gen_fn(t
, t_pd
, t_pg
, tcg_constant_i32(desc
));
1669 tcg_temp_free_ptr(t_pd
);
1670 tcg_temp_free_ptr(t_pg
);
1673 tcg_temp_free_i32(t
);
1677 TRANS_FEAT(PFIRST
, aa64_sve
, do_pfirst_pnext
, a
, gen_helper_sve_pfirst
)
1678 TRANS_FEAT(PNEXT
, aa64_sve
, do_pfirst_pnext
, a
, gen_helper_sve_pnext
)
1681 *** SVE Element Count Group
1684 /* Perform an inline saturating addition of a 32-bit value within
1685 * a 64-bit register. The second operand is known to be positive,
1686 * which halves the comparisions we must perform to bound the result.
1688 static void do_sat_addsub_32(TCGv_i64 reg
, TCGv_i64 val
, bool u
, bool d
)
1692 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
1694 tcg_gen_ext32u_i64(reg
, reg
);
1696 tcg_gen_ext32s_i64(reg
, reg
);
1699 tcg_gen_sub_i64(reg
, reg
, val
);
1700 ibound
= (u
? 0 : INT32_MIN
);
1701 tcg_gen_smax_i64(reg
, reg
, tcg_constant_i64(ibound
));
1703 tcg_gen_add_i64(reg
, reg
, val
);
1704 ibound
= (u
? UINT32_MAX
: INT32_MAX
);
1705 tcg_gen_smin_i64(reg
, reg
, tcg_constant_i64(ibound
));
1709 /* Similarly with 64-bit values. */
1710 static void do_sat_addsub_64(TCGv_i64 reg
, TCGv_i64 val
, bool u
, bool d
)
1712 TCGv_i64 t0
= tcg_temp_new_i64();
1717 tcg_gen_sub_i64(t0
, reg
, val
);
1718 t2
= tcg_constant_i64(0);
1719 tcg_gen_movcond_i64(TCG_COND_LTU
, reg
, reg
, val
, t2
, t0
);
1721 tcg_gen_add_i64(t0
, reg
, val
);
1722 t2
= tcg_constant_i64(-1);
1723 tcg_gen_movcond_i64(TCG_COND_LTU
, reg
, t0
, reg
, t2
, t0
);
1726 TCGv_i64 t1
= tcg_temp_new_i64();
1728 /* Detect signed overflow for subtraction. */
1729 tcg_gen_xor_i64(t0
, reg
, val
);
1730 tcg_gen_sub_i64(t1
, reg
, val
);
1731 tcg_gen_xor_i64(reg
, reg
, t1
);
1732 tcg_gen_and_i64(t0
, t0
, reg
);
1734 /* Bound the result. */
1735 tcg_gen_movi_i64(reg
, INT64_MIN
);
1736 t2
= tcg_constant_i64(0);
1737 tcg_gen_movcond_i64(TCG_COND_LT
, reg
, t0
, t2
, reg
, t1
);
1739 /* Detect signed overflow for addition. */
1740 tcg_gen_xor_i64(t0
, reg
, val
);
1741 tcg_gen_add_i64(reg
, reg
, val
);
1742 tcg_gen_xor_i64(t1
, reg
, val
);
1743 tcg_gen_andc_i64(t0
, t1
, t0
);
1745 /* Bound the result. */
1746 tcg_gen_movi_i64(t1
, INT64_MAX
);
1747 t2
= tcg_constant_i64(0);
1748 tcg_gen_movcond_i64(TCG_COND_LT
, reg
, t0
, t2
, t1
, reg
);
1750 tcg_temp_free_i64(t1
);
1752 tcg_temp_free_i64(t0
);
1755 /* Similarly with a vector and a scalar operand. */
1756 static void do_sat_addsub_vec(DisasContext
*s
, int esz
, int rd
, int rn
,
1757 TCGv_i64 val
, bool u
, bool d
)
1759 unsigned vsz
= vec_full_reg_size(s
);
1760 TCGv_ptr dptr
, nptr
;
1764 dptr
= tcg_temp_new_ptr();
1765 nptr
= tcg_temp_new_ptr();
1766 tcg_gen_addi_ptr(dptr
, cpu_env
, vec_full_reg_offset(s
, rd
));
1767 tcg_gen_addi_ptr(nptr
, cpu_env
, vec_full_reg_offset(s
, rn
));
1768 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
1772 t32
= tcg_temp_new_i32();
1773 tcg_gen_extrl_i64_i32(t32
, val
);
1775 tcg_gen_neg_i32(t32
, t32
);
1778 gen_helper_sve_uqaddi_b(dptr
, nptr
, t32
, desc
);
1780 gen_helper_sve_sqaddi_b(dptr
, nptr
, t32
, desc
);
1782 tcg_temp_free_i32(t32
);
1786 t32
= tcg_temp_new_i32();
1787 tcg_gen_extrl_i64_i32(t32
, val
);
1789 tcg_gen_neg_i32(t32
, t32
);
1792 gen_helper_sve_uqaddi_h(dptr
, nptr
, t32
, desc
);
1794 gen_helper_sve_sqaddi_h(dptr
, nptr
, t32
, desc
);
1796 tcg_temp_free_i32(t32
);
1800 t64
= tcg_temp_new_i64();
1802 tcg_gen_neg_i64(t64
, val
);
1804 tcg_gen_mov_i64(t64
, val
);
1807 gen_helper_sve_uqaddi_s(dptr
, nptr
, t64
, desc
);
1809 gen_helper_sve_sqaddi_s(dptr
, nptr
, t64
, desc
);
1811 tcg_temp_free_i64(t64
);
1817 gen_helper_sve_uqsubi_d(dptr
, nptr
, val
, desc
);
1819 gen_helper_sve_uqaddi_d(dptr
, nptr
, val
, desc
);
1822 t64
= tcg_temp_new_i64();
1823 tcg_gen_neg_i64(t64
, val
);
1824 gen_helper_sve_sqaddi_d(dptr
, nptr
, t64
, desc
);
1825 tcg_temp_free_i64(t64
);
1827 gen_helper_sve_sqaddi_d(dptr
, nptr
, val
, desc
);
1832 g_assert_not_reached();
1835 tcg_temp_free_ptr(dptr
);
1836 tcg_temp_free_ptr(nptr
);
1839 static bool trans_CNT_r(DisasContext
*s
, arg_CNT_r
*a
)
1841 if (sve_access_check(s
)) {
1842 unsigned fullsz
= vec_full_reg_size(s
);
1843 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
1844 tcg_gen_movi_i64(cpu_reg(s
, a
->rd
), numelem
* a
->imm
);
1849 static bool trans_INCDEC_r(DisasContext
*s
, arg_incdec_cnt
*a
)
1851 if (sve_access_check(s
)) {
1852 unsigned fullsz
= vec_full_reg_size(s
);
1853 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
1854 int inc
= numelem
* a
->imm
* (a
->d
? -1 : 1);
1855 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
1857 tcg_gen_addi_i64(reg
, reg
, inc
);
1862 static bool trans_SINCDEC_r_32(DisasContext
*s
, arg_incdec_cnt
*a
)
1864 if (!sve_access_check(s
)) {
1868 unsigned fullsz
= vec_full_reg_size(s
);
1869 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
1870 int inc
= numelem
* a
->imm
;
1871 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
1873 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
1876 tcg_gen_ext32u_i64(reg
, reg
);
1878 tcg_gen_ext32s_i64(reg
, reg
);
1881 do_sat_addsub_32(reg
, tcg_constant_i64(inc
), a
->u
, a
->d
);
1886 static bool trans_SINCDEC_r_64(DisasContext
*s
, arg_incdec_cnt
*a
)
1888 if (!sve_access_check(s
)) {
1892 unsigned fullsz
= vec_full_reg_size(s
);
1893 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
1894 int inc
= numelem
* a
->imm
;
1895 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
1898 do_sat_addsub_64(reg
, tcg_constant_i64(inc
), a
->u
, a
->d
);
1903 static bool trans_INCDEC_v(DisasContext
*s
, arg_incdec2_cnt
*a
)
1909 unsigned fullsz
= vec_full_reg_size(s
);
1910 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
1911 int inc
= numelem
* a
->imm
;
1914 if (sve_access_check(s
)) {
1915 tcg_gen_gvec_adds(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
1916 vec_full_reg_offset(s
, a
->rn
),
1917 tcg_constant_i64(a
->d
? -inc
: inc
),
1921 do_mov_z(s
, a
->rd
, a
->rn
);
1926 static bool trans_SINCDEC_v(DisasContext
*s
, arg_incdec2_cnt
*a
)
1932 unsigned fullsz
= vec_full_reg_size(s
);
1933 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
1934 int inc
= numelem
* a
->imm
;
1937 if (sve_access_check(s
)) {
1938 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
,
1939 tcg_constant_i64(inc
), a
->u
, a
->d
);
1942 do_mov_z(s
, a
->rd
, a
->rn
);
1948 *** SVE Bitwise Immediate Group
1951 static bool do_zz_dbm(DisasContext
*s
, arg_rr_dbm
*a
, GVecGen2iFn
*gvec_fn
)
1954 if (!logic_imm_decode_wmask(&imm
, extract32(a
->dbm
, 12, 1),
1955 extract32(a
->dbm
, 0, 6),
1956 extract32(a
->dbm
, 6, 6))) {
1959 return gen_gvec_fn_zzi(s
, gvec_fn
, MO_64
, a
->rd
, a
->rn
, imm
);
1962 TRANS_FEAT(AND_zzi
, aa64_sve
, do_zz_dbm
, a
, tcg_gen_gvec_andi
)
1963 TRANS_FEAT(ORR_zzi
, aa64_sve
, do_zz_dbm
, a
, tcg_gen_gvec_ori
)
1964 TRANS_FEAT(EOR_zzi
, aa64_sve
, do_zz_dbm
, a
, tcg_gen_gvec_xori
)
1966 static bool trans_DUPM(DisasContext
*s
, arg_DUPM
*a
)
1969 if (!logic_imm_decode_wmask(&imm
, extract32(a
->dbm
, 12, 1),
1970 extract32(a
->dbm
, 0, 6),
1971 extract32(a
->dbm
, 6, 6))) {
1974 if (sve_access_check(s
)) {
1975 do_dupi_z(s
, a
->rd
, imm
);
1981 *** SVE Integer Wide Immediate - Predicated Group
1984 /* Implement all merging copies. This is used for CPY (immediate),
1985 * FCPY, CPY (scalar), CPY (SIMD&FP scalar).
1987 static void do_cpy_m(DisasContext
*s
, int esz
, int rd
, int rn
, int pg
,
1990 typedef void gen_cpy(TCGv_ptr
, TCGv_ptr
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
1991 static gen_cpy
* const fns
[4] = {
1992 gen_helper_sve_cpy_m_b
, gen_helper_sve_cpy_m_h
,
1993 gen_helper_sve_cpy_m_s
, gen_helper_sve_cpy_m_d
,
1995 unsigned vsz
= vec_full_reg_size(s
);
1996 TCGv_i32 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
1997 TCGv_ptr t_zd
= tcg_temp_new_ptr();
1998 TCGv_ptr t_zn
= tcg_temp_new_ptr();
1999 TCGv_ptr t_pg
= tcg_temp_new_ptr();
2001 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, rd
));
2002 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, rn
));
2003 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
2005 fns
[esz
](t_zd
, t_zn
, t_pg
, val
, desc
);
2007 tcg_temp_free_ptr(t_zd
);
2008 tcg_temp_free_ptr(t_zn
);
2009 tcg_temp_free_ptr(t_pg
);
2012 static bool trans_FCPY(DisasContext
*s
, arg_FCPY
*a
)
2017 if (sve_access_check(s
)) {
2018 /* Decode the VFP immediate. */
2019 uint64_t imm
= vfp_expand_imm(a
->esz
, a
->imm
);
2020 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rn
, a
->pg
, tcg_constant_i64(imm
));
2025 static bool trans_CPY_m_i(DisasContext
*s
, arg_rpri_esz
*a
)
2027 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
2030 if (sve_access_check(s
)) {
2031 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rn
, a
->pg
, tcg_constant_i64(a
->imm
));
2036 static bool trans_CPY_z_i(DisasContext
*s
, arg_CPY_z_i
*a
)
2038 static gen_helper_gvec_2i
* const fns
[4] = {
2039 gen_helper_sve_cpy_z_b
, gen_helper_sve_cpy_z_h
,
2040 gen_helper_sve_cpy_z_s
, gen_helper_sve_cpy_z_d
,
2043 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
2046 if (sve_access_check(s
)) {
2047 unsigned vsz
= vec_full_reg_size(s
);
2048 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s
, a
->rd
),
2049 pred_full_reg_offset(s
, a
->pg
),
2050 tcg_constant_i64(a
->imm
),
2051 vsz
, vsz
, 0, fns
[a
->esz
]);
2057 *** SVE Permute Extract Group
2060 static bool do_EXT(DisasContext
*s
, int rd
, int rn
, int rm
, int imm
)
2062 if (!sve_access_check(s
)) {
2066 unsigned vsz
= vec_full_reg_size(s
);
2067 unsigned n_ofs
= imm
>= vsz
? 0 : imm
;
2068 unsigned n_siz
= vsz
- n_ofs
;
2069 unsigned d
= vec_full_reg_offset(s
, rd
);
2070 unsigned n
= vec_full_reg_offset(s
, rn
);
2071 unsigned m
= vec_full_reg_offset(s
, rm
);
2073 /* Use host vector move insns if we have appropriate sizes
2074 * and no unfortunate overlap.
2077 && n_ofs
== size_for_gvec(n_ofs
)
2078 && n_siz
== size_for_gvec(n_siz
)
2079 && (d
!= n
|| n_siz
<= n_ofs
)) {
2080 tcg_gen_gvec_mov(0, d
, n
+ n_ofs
, n_siz
, n_siz
);
2082 tcg_gen_gvec_mov(0, d
+ n_siz
, m
, n_ofs
, n_ofs
);
2085 tcg_gen_gvec_3_ool(d
, n
, m
, vsz
, vsz
, n_ofs
, gen_helper_sve_ext
);
2090 TRANS_FEAT(EXT
, aa64_sve
, do_EXT
, a
->rd
, a
->rn
, a
->rm
, a
->imm
)
2091 TRANS_FEAT(EXT_sve2
, aa64_sve2
, do_EXT
, a
->rd
, a
->rn
, (a
->rn
+ 1) % 32, a
->imm
)
2094 *** SVE Permute - Unpredicated Group
2097 static bool trans_DUP_s(DisasContext
*s
, arg_DUP_s
*a
)
2099 if (sve_access_check(s
)) {
2100 unsigned vsz
= vec_full_reg_size(s
);
2101 tcg_gen_gvec_dup_i64(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
2102 vsz
, vsz
, cpu_reg_sp(s
, a
->rn
));
2107 static bool trans_DUP_x(DisasContext
*s
, arg_DUP_x
*a
)
2109 if ((a
->imm
& 0x1f) == 0) {
2112 if (sve_access_check(s
)) {
2113 unsigned vsz
= vec_full_reg_size(s
);
2114 unsigned dofs
= vec_full_reg_offset(s
, a
->rd
);
2115 unsigned esz
, index
;
2117 esz
= ctz32(a
->imm
);
2118 index
= a
->imm
>> (esz
+ 1);
2120 if ((index
<< esz
) < vsz
) {
2121 unsigned nofs
= vec_reg_offset(s
, a
->rn
, index
, esz
);
2122 tcg_gen_gvec_dup_mem(esz
, dofs
, nofs
, vsz
, vsz
);
2125 * While dup_mem handles 128-bit elements, dup_imm does not.
2126 * Thankfully element size doesn't matter for splatting zero.
2128 tcg_gen_gvec_dup_imm(MO_64
, dofs
, vsz
, vsz
, 0);
2134 static void do_insr_i64(DisasContext
*s
, arg_rrr_esz
*a
, TCGv_i64 val
)
2136 typedef void gen_insr(TCGv_ptr
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
2137 static gen_insr
* const fns
[4] = {
2138 gen_helper_sve_insr_b
, gen_helper_sve_insr_h
,
2139 gen_helper_sve_insr_s
, gen_helper_sve_insr_d
,
2141 unsigned vsz
= vec_full_reg_size(s
);
2142 TCGv_i32 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
2143 TCGv_ptr t_zd
= tcg_temp_new_ptr();
2144 TCGv_ptr t_zn
= tcg_temp_new_ptr();
2146 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, a
->rd
));
2147 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
2149 fns
[a
->esz
](t_zd
, t_zn
, val
, desc
);
2151 tcg_temp_free_ptr(t_zd
);
2152 tcg_temp_free_ptr(t_zn
);
2155 static bool trans_INSR_f(DisasContext
*s
, arg_rrr_esz
*a
)
2157 if (sve_access_check(s
)) {
2158 TCGv_i64 t
= tcg_temp_new_i64();
2159 tcg_gen_ld_i64(t
, cpu_env
, vec_reg_offset(s
, a
->rm
, 0, MO_64
));
2160 do_insr_i64(s
, a
, t
);
2161 tcg_temp_free_i64(t
);
2166 static bool trans_INSR_r(DisasContext
*s
, arg_rrr_esz
*a
)
2168 if (sve_access_check(s
)) {
2169 do_insr_i64(s
, a
, cpu_reg(s
, a
->rm
));
2174 static gen_helper_gvec_2
* const rev_fns
[4] = {
2175 gen_helper_sve_rev_b
, gen_helper_sve_rev_h
,
2176 gen_helper_sve_rev_s
, gen_helper_sve_rev_d
2178 TRANS_FEAT(REV_v
, aa64_sve
, gen_gvec_ool_zz
, rev_fns
[a
->esz
], a
->rd
, a
->rn
, 0)
2180 static gen_helper_gvec_3
* const sve_tbl_fns
[4] = {
2181 gen_helper_sve_tbl_b
, gen_helper_sve_tbl_h
,
2182 gen_helper_sve_tbl_s
, gen_helper_sve_tbl_d
2184 TRANS_FEAT(TBL
, aa64_sve
, gen_gvec_ool_arg_zzz
, sve_tbl_fns
[a
->esz
], a
, 0)
2186 static gen_helper_gvec_4
* const sve2_tbl_fns
[4] = {
2187 gen_helper_sve2_tbl_b
, gen_helper_sve2_tbl_h
,
2188 gen_helper_sve2_tbl_s
, gen_helper_sve2_tbl_d
2190 TRANS_FEAT(TBL_sve2
, aa64_sve2
, gen_gvec_ool_zzzz
, sve2_tbl_fns
[a
->esz
],
2191 a
->rd
, a
->rn
, (a
->rn
+ 1) % 32, a
->rm
, 0)
2193 static gen_helper_gvec_3
* const tbx_fns
[4] = {
2194 gen_helper_sve2_tbx_b
, gen_helper_sve2_tbx_h
,
2195 gen_helper_sve2_tbx_s
, gen_helper_sve2_tbx_d
2197 TRANS_FEAT(TBX
, aa64_sve2
, gen_gvec_ool_arg_zzz
, tbx_fns
[a
->esz
], a
, 0)
2199 static bool trans_UNPK(DisasContext
*s
, arg_UNPK
*a
)
2201 static gen_helper_gvec_2
* const fns
[4][2] = {
2203 { gen_helper_sve_sunpk_h
, gen_helper_sve_uunpk_h
},
2204 { gen_helper_sve_sunpk_s
, gen_helper_sve_uunpk_s
},
2205 { gen_helper_sve_sunpk_d
, gen_helper_sve_uunpk_d
},
2211 if (sve_access_check(s
)) {
2212 unsigned vsz
= vec_full_reg_size(s
);
2213 tcg_gen_gvec_2_ool(vec_full_reg_offset(s
, a
->rd
),
2214 vec_full_reg_offset(s
, a
->rn
)
2215 + (a
->h
? vsz
/ 2 : 0),
2216 vsz
, vsz
, 0, fns
[a
->esz
][a
->u
]);
2222 *** SVE Permute - Predicates Group
2225 static bool do_perm_pred3(DisasContext
*s
, arg_rrr_esz
*a
, bool high_odd
,
2226 gen_helper_gvec_3
*fn
)
2228 if (!sve_access_check(s
)) {
2232 unsigned vsz
= pred_full_reg_size(s
);
2234 TCGv_ptr t_d
= tcg_temp_new_ptr();
2235 TCGv_ptr t_n
= tcg_temp_new_ptr();
2236 TCGv_ptr t_m
= tcg_temp_new_ptr();
2239 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
);
2240 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
2241 desc
= FIELD_DP32(desc
, PREDDESC
, DATA
, high_odd
);
2243 tcg_gen_addi_ptr(t_d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2244 tcg_gen_addi_ptr(t_n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2245 tcg_gen_addi_ptr(t_m
, cpu_env
, pred_full_reg_offset(s
, a
->rm
));
2247 fn(t_d
, t_n
, t_m
, tcg_constant_i32(desc
));
2249 tcg_temp_free_ptr(t_d
);
2250 tcg_temp_free_ptr(t_n
);
2251 tcg_temp_free_ptr(t_m
);
2255 static bool do_perm_pred2(DisasContext
*s
, arg_rr_esz
*a
, bool high_odd
,
2256 gen_helper_gvec_2
*fn
)
2258 if (!sve_access_check(s
)) {
2262 unsigned vsz
= pred_full_reg_size(s
);
2263 TCGv_ptr t_d
= tcg_temp_new_ptr();
2264 TCGv_ptr t_n
= tcg_temp_new_ptr();
2267 tcg_gen_addi_ptr(t_d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2268 tcg_gen_addi_ptr(t_n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2270 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
);
2271 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
2272 desc
= FIELD_DP32(desc
, PREDDESC
, DATA
, high_odd
);
2274 fn(t_d
, t_n
, tcg_constant_i32(desc
));
2276 tcg_temp_free_ptr(t_d
);
2277 tcg_temp_free_ptr(t_n
);
2281 TRANS_FEAT(ZIP1_p
, aa64_sve
, do_perm_pred3
, a
, 0, gen_helper_sve_zip_p
)
2282 TRANS_FEAT(ZIP2_p
, aa64_sve
, do_perm_pred3
, a
, 1, gen_helper_sve_zip_p
)
2283 TRANS_FEAT(UZP1_p
, aa64_sve
, do_perm_pred3
, a
, 0, gen_helper_sve_uzp_p
)
2284 TRANS_FEAT(UZP2_p
, aa64_sve
, do_perm_pred3
, a
, 1, gen_helper_sve_uzp_p
)
2285 TRANS_FEAT(TRN1_p
, aa64_sve
, do_perm_pred3
, a
, 0, gen_helper_sve_trn_p
)
2286 TRANS_FEAT(TRN2_p
, aa64_sve
, do_perm_pred3
, a
, 1, gen_helper_sve_trn_p
)
2288 TRANS_FEAT(REV_p
, aa64_sve
, do_perm_pred2
, a
, 0, gen_helper_sve_rev_p
)
2289 TRANS_FEAT(PUNPKLO
, aa64_sve
, do_perm_pred2
, a
, 0, gen_helper_sve_punpk_p
)
2290 TRANS_FEAT(PUNPKHI
, aa64_sve
, do_perm_pred2
, a
, 1, gen_helper_sve_punpk_p
)
2293 *** SVE Permute - Interleaving Group
2296 static gen_helper_gvec_3
* const zip_fns
[4] = {
2297 gen_helper_sve_zip_b
, gen_helper_sve_zip_h
,
2298 gen_helper_sve_zip_s
, gen_helper_sve_zip_d
,
2300 TRANS_FEAT(ZIP1_z
, aa64_sve
, gen_gvec_ool_arg_zzz
,
2301 zip_fns
[a
->esz
], a
, 0)
2302 TRANS_FEAT(ZIP2_z
, aa64_sve
, gen_gvec_ool_arg_zzz
,
2303 zip_fns
[a
->esz
], a
, vec_full_reg_size(s
) / 2)
2305 TRANS_FEAT(ZIP1_q
, aa64_sve_f64mm
, gen_gvec_ool_arg_zzz
,
2306 gen_helper_sve2_zip_q
, a
, 0)
2307 TRANS_FEAT(ZIP2_q
, aa64_sve_f64mm
, gen_gvec_ool_arg_zzz
,
2308 gen_helper_sve2_zip_q
, a
,
2309 QEMU_ALIGN_DOWN(vec_full_reg_size(s
), 32) / 2)
2311 static gen_helper_gvec_3
* const uzp_fns
[4] = {
2312 gen_helper_sve_uzp_b
, gen_helper_sve_uzp_h
,
2313 gen_helper_sve_uzp_s
, gen_helper_sve_uzp_d
,
2316 TRANS_FEAT(UZP1_z
, aa64_sve
, gen_gvec_ool_arg_zzz
,
2317 uzp_fns
[a
->esz
], a
, 0)
2318 TRANS_FEAT(UZP2_z
, aa64_sve
, gen_gvec_ool_arg_zzz
,
2319 uzp_fns
[a
->esz
], a
, 1 << a
->esz
)
2321 TRANS_FEAT(UZP1_q
, aa64_sve_f64mm
, gen_gvec_ool_arg_zzz
,
2322 gen_helper_sve2_uzp_q
, a
, 0)
2323 TRANS_FEAT(UZP2_q
, aa64_sve_f64mm
, gen_gvec_ool_arg_zzz
,
2324 gen_helper_sve2_uzp_q
, a
, 16)
2326 static gen_helper_gvec_3
* const trn_fns
[4] = {
2327 gen_helper_sve_trn_b
, gen_helper_sve_trn_h
,
2328 gen_helper_sve_trn_s
, gen_helper_sve_trn_d
,
2331 TRANS_FEAT(TRN1_z
, aa64_sve
, gen_gvec_ool_arg_zzz
,
2332 trn_fns
[a
->esz
], a
, 0)
2333 TRANS_FEAT(TRN2_z
, aa64_sve
, gen_gvec_ool_arg_zzz
,
2334 trn_fns
[a
->esz
], a
, 1 << a
->esz
)
2336 TRANS_FEAT(TRN1_q
, aa64_sve_f64mm
, gen_gvec_ool_arg_zzz
,
2337 gen_helper_sve2_trn_q
, a
, 0)
2338 TRANS_FEAT(TRN2_q
, aa64_sve_f64mm
, gen_gvec_ool_arg_zzz
,
2339 gen_helper_sve2_trn_q
, a
, 16)
2342 *** SVE Permute Vector - Predicated Group
2345 static gen_helper_gvec_3
* const compact_fns
[4] = {
2346 NULL
, NULL
, gen_helper_sve_compact_s
, gen_helper_sve_compact_d
2348 TRANS_FEAT(COMPACT
, aa64_sve
, gen_gvec_ool_arg_zpz
, compact_fns
[a
->esz
], a
, 0)
2350 /* Call the helper that computes the ARM LastActiveElement pseudocode
2351 * function, scaled by the element size. This includes the not found
2352 * indication; e.g. not found for esz=3 is -8.
2354 static void find_last_active(DisasContext
*s
, TCGv_i32 ret
, int esz
, int pg
)
2356 /* Predicate sizes may be smaller and cannot use simd_desc. We cannot
2357 * round up, as we do elsewhere, because we need the exact size.
2359 TCGv_ptr t_p
= tcg_temp_new_ptr();
2362 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, pred_full_reg_size(s
));
2363 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, esz
);
2365 tcg_gen_addi_ptr(t_p
, cpu_env
, pred_full_reg_offset(s
, pg
));
2367 gen_helper_sve_last_active_element(ret
, t_p
, tcg_constant_i32(desc
));
2369 tcg_temp_free_ptr(t_p
);
2372 /* Increment LAST to the offset of the next element in the vector,
2373 * wrapping around to 0.
2375 static void incr_last_active(DisasContext
*s
, TCGv_i32 last
, int esz
)
2377 unsigned vsz
= vec_full_reg_size(s
);
2379 tcg_gen_addi_i32(last
, last
, 1 << esz
);
2380 if (is_power_of_2(vsz
)) {
2381 tcg_gen_andi_i32(last
, last
, vsz
- 1);
2383 TCGv_i32 max
= tcg_constant_i32(vsz
);
2384 TCGv_i32 zero
= tcg_constant_i32(0);
2385 tcg_gen_movcond_i32(TCG_COND_GEU
, last
, last
, max
, zero
, last
);
2389 /* If LAST < 0, set LAST to the offset of the last element in the vector. */
2390 static void wrap_last_active(DisasContext
*s
, TCGv_i32 last
, int esz
)
2392 unsigned vsz
= vec_full_reg_size(s
);
2394 if (is_power_of_2(vsz
)) {
2395 tcg_gen_andi_i32(last
, last
, vsz
- 1);
2397 TCGv_i32 max
= tcg_constant_i32(vsz
- (1 << esz
));
2398 TCGv_i32 zero
= tcg_constant_i32(0);
2399 tcg_gen_movcond_i32(TCG_COND_LT
, last
, last
, zero
, max
, last
);
2403 /* Load an unsigned element of ESZ from BASE+OFS. */
2404 static TCGv_i64
load_esz(TCGv_ptr base
, int ofs
, int esz
)
2406 TCGv_i64 r
= tcg_temp_new_i64();
2410 tcg_gen_ld8u_i64(r
, base
, ofs
);
2413 tcg_gen_ld16u_i64(r
, base
, ofs
);
2416 tcg_gen_ld32u_i64(r
, base
, ofs
);
2419 tcg_gen_ld_i64(r
, base
, ofs
);
2422 g_assert_not_reached();
2427 /* Load an unsigned element of ESZ from RM[LAST]. */
2428 static TCGv_i64
load_last_active(DisasContext
*s
, TCGv_i32 last
,
2431 TCGv_ptr p
= tcg_temp_new_ptr();
2434 /* Convert offset into vector into offset into ENV.
2435 * The final adjustment for the vector register base
2436 * is added via constant offset to the load.
2439 /* Adjust for element ordering. See vec_reg_offset. */
2441 tcg_gen_xori_i32(last
, last
, 8 - (1 << esz
));
2444 tcg_gen_ext_i32_ptr(p
, last
);
2445 tcg_gen_add_ptr(p
, p
, cpu_env
);
2447 r
= load_esz(p
, vec_full_reg_offset(s
, rm
), esz
);
2448 tcg_temp_free_ptr(p
);
2453 /* Compute CLAST for a Zreg. */
2454 static bool do_clast_vector(DisasContext
*s
, arg_rprr_esz
*a
, bool before
)
2459 unsigned vsz
, esz
= a
->esz
;
2461 if (!sve_access_check(s
)) {
2465 last
= tcg_temp_local_new_i32();
2466 over
= gen_new_label();
2468 find_last_active(s
, last
, esz
, a
->pg
);
2470 /* There is of course no movcond for a 2048-bit vector,
2471 * so we must branch over the actual store.
2473 tcg_gen_brcondi_i32(TCG_COND_LT
, last
, 0, over
);
2476 incr_last_active(s
, last
, esz
);
2479 ele
= load_last_active(s
, last
, a
->rm
, esz
);
2480 tcg_temp_free_i32(last
);
2482 vsz
= vec_full_reg_size(s
);
2483 tcg_gen_gvec_dup_i64(esz
, vec_full_reg_offset(s
, a
->rd
), vsz
, vsz
, ele
);
2484 tcg_temp_free_i64(ele
);
2486 /* If this insn used MOVPRFX, we may need a second move. */
2487 if (a
->rd
!= a
->rn
) {
2488 TCGLabel
*done
= gen_new_label();
2491 gen_set_label(over
);
2492 do_mov_z(s
, a
->rd
, a
->rn
);
2494 gen_set_label(done
);
2496 gen_set_label(over
);
2501 TRANS_FEAT(CLASTA_z
, aa64_sve
, do_clast_vector
, a
, false)
2502 TRANS_FEAT(CLASTB_z
, aa64_sve
, do_clast_vector
, a
, true)
2504 /* Compute CLAST for a scalar. */
2505 static void do_clast_scalar(DisasContext
*s
, int esz
, int pg
, int rm
,
2506 bool before
, TCGv_i64 reg_val
)
2508 TCGv_i32 last
= tcg_temp_new_i32();
2511 find_last_active(s
, last
, esz
, pg
);
2513 /* Extend the original value of last prior to incrementing. */
2514 cmp
= tcg_temp_new_i64();
2515 tcg_gen_ext_i32_i64(cmp
, last
);
2518 incr_last_active(s
, last
, esz
);
2521 /* The conceit here is that while last < 0 indicates not found, after
2522 * adjusting for cpu_env->vfp.zregs[rm], it is still a valid address
2523 * from which we can load garbage. We then discard the garbage with
2524 * a conditional move.
2526 ele
= load_last_active(s
, last
, rm
, esz
);
2527 tcg_temp_free_i32(last
);
2529 tcg_gen_movcond_i64(TCG_COND_GE
, reg_val
, cmp
, tcg_constant_i64(0),
2532 tcg_temp_free_i64(cmp
);
2533 tcg_temp_free_i64(ele
);
2536 /* Compute CLAST for a Vreg. */
2537 static bool do_clast_fp(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2539 if (sve_access_check(s
)) {
2541 int ofs
= vec_reg_offset(s
, a
->rd
, 0, esz
);
2542 TCGv_i64 reg
= load_esz(cpu_env
, ofs
, esz
);
2544 do_clast_scalar(s
, esz
, a
->pg
, a
->rn
, before
, reg
);
2545 write_fp_dreg(s
, a
->rd
, reg
);
2546 tcg_temp_free_i64(reg
);
2551 TRANS_FEAT(CLASTA_v
, aa64_sve
, do_clast_fp
, a
, false)
2552 TRANS_FEAT(CLASTB_v
, aa64_sve
, do_clast_fp
, a
, true)
2554 /* Compute CLAST for a Xreg. */
2555 static bool do_clast_general(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2559 if (!sve_access_check(s
)) {
2563 reg
= cpu_reg(s
, a
->rd
);
2566 tcg_gen_ext8u_i64(reg
, reg
);
2569 tcg_gen_ext16u_i64(reg
, reg
);
2572 tcg_gen_ext32u_i64(reg
, reg
);
2577 g_assert_not_reached();
2580 do_clast_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
, reg
);
2584 TRANS_FEAT(CLASTA_r
, aa64_sve
, do_clast_general
, a
, false)
2585 TRANS_FEAT(CLASTB_r
, aa64_sve
, do_clast_general
, a
, true)
2587 /* Compute LAST for a scalar. */
2588 static TCGv_i64
do_last_scalar(DisasContext
*s
, int esz
,
2589 int pg
, int rm
, bool before
)
2591 TCGv_i32 last
= tcg_temp_new_i32();
2594 find_last_active(s
, last
, esz
, pg
);
2596 wrap_last_active(s
, last
, esz
);
2598 incr_last_active(s
, last
, esz
);
2601 ret
= load_last_active(s
, last
, rm
, esz
);
2602 tcg_temp_free_i32(last
);
2606 /* Compute LAST for a Vreg. */
2607 static bool do_last_fp(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2609 if (sve_access_check(s
)) {
2610 TCGv_i64 val
= do_last_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
);
2611 write_fp_dreg(s
, a
->rd
, val
);
2612 tcg_temp_free_i64(val
);
2617 TRANS_FEAT(LASTA_v
, aa64_sve
, do_last_fp
, a
, false)
2618 TRANS_FEAT(LASTB_v
, aa64_sve
, do_last_fp
, a
, true)
2620 /* Compute LAST for a Xreg. */
2621 static bool do_last_general(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2623 if (sve_access_check(s
)) {
2624 TCGv_i64 val
= do_last_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
);
2625 tcg_gen_mov_i64(cpu_reg(s
, a
->rd
), val
);
2626 tcg_temp_free_i64(val
);
2631 TRANS_FEAT(LASTA_r
, aa64_sve
, do_last_general
, a
, false)
2632 TRANS_FEAT(LASTB_r
, aa64_sve
, do_last_general
, a
, true)
2634 static bool trans_CPY_m_r(DisasContext
*s
, arg_rpr_esz
*a
)
2636 if (sve_access_check(s
)) {
2637 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rd
, a
->pg
, cpu_reg_sp(s
, a
->rn
));
2642 static bool trans_CPY_m_v(DisasContext
*s
, arg_rpr_esz
*a
)
2644 if (sve_access_check(s
)) {
2645 int ofs
= vec_reg_offset(s
, a
->rn
, 0, a
->esz
);
2646 TCGv_i64 t
= load_esz(cpu_env
, ofs
, a
->esz
);
2647 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rd
, a
->pg
, t
);
2648 tcg_temp_free_i64(t
);
2653 static gen_helper_gvec_3
* const revb_fns
[4] = {
2654 NULL
, gen_helper_sve_revb_h
,
2655 gen_helper_sve_revb_s
, gen_helper_sve_revb_d
,
2657 TRANS_FEAT(REVB
, aa64_sve
, gen_gvec_ool_arg_zpz
, revb_fns
[a
->esz
], a
, 0)
2659 static gen_helper_gvec_3
* const revh_fns
[4] = {
2660 NULL
, NULL
, gen_helper_sve_revh_s
, gen_helper_sve_revh_d
,
2662 TRANS_FEAT(REVH
, aa64_sve
, gen_gvec_ool_arg_zpz
, revh_fns
[a
->esz
], a
, 0)
2664 TRANS_FEAT(REVW
, aa64_sve
, gen_gvec_ool_arg_zpz
,
2665 a
->esz
== 3 ? gen_helper_sve_revw_d
: NULL
, a
, 0)
2667 TRANS_FEAT(SPLICE
, aa64_sve
, gen_gvec_ool_arg_zpzz
,
2668 gen_helper_sve_splice
, a
, a
->esz
)
2670 TRANS_FEAT(SPLICE_sve2
, aa64_sve2
, gen_gvec_ool_zzzp
, gen_helper_sve_splice
,
2671 a
->rd
, a
->rn
, (a
->rn
+ 1) % 32, a
->pg
, a
->esz
)
2674 *** SVE Integer Compare - Vectors Group
2677 static bool do_ppzz_flags(DisasContext
*s
, arg_rprr_esz
*a
,
2678 gen_helper_gvec_flags_4
*gen_fn
)
2680 TCGv_ptr pd
, zn
, zm
, pg
;
2684 if (gen_fn
== NULL
) {
2687 if (!sve_access_check(s
)) {
2691 vsz
= vec_full_reg_size(s
);
2692 t
= tcg_temp_new_i32();
2693 pd
= tcg_temp_new_ptr();
2694 zn
= tcg_temp_new_ptr();
2695 zm
= tcg_temp_new_ptr();
2696 pg
= tcg_temp_new_ptr();
2698 tcg_gen_addi_ptr(pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2699 tcg_gen_addi_ptr(zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
2700 tcg_gen_addi_ptr(zm
, cpu_env
, vec_full_reg_offset(s
, a
->rm
));
2701 tcg_gen_addi_ptr(pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
2703 gen_fn(t
, pd
, zn
, zm
, pg
, tcg_constant_i32(simd_desc(vsz
, vsz
, 0)));
2705 tcg_temp_free_ptr(pd
);
2706 tcg_temp_free_ptr(zn
);
2707 tcg_temp_free_ptr(zm
);
2708 tcg_temp_free_ptr(pg
);
2712 tcg_temp_free_i32(t
);
2716 #define DO_PPZZ(NAME, name) \
2717 static gen_helper_gvec_flags_4 * const name##_ppzz_fns[4] = { \
2718 gen_helper_sve_##name##_ppzz_b, gen_helper_sve_##name##_ppzz_h, \
2719 gen_helper_sve_##name##_ppzz_s, gen_helper_sve_##name##_ppzz_d, \
2721 TRANS_FEAT(NAME##_ppzz, aa64_sve, do_ppzz_flags, \
2722 a, name##_ppzz_fns[a->esz])
2724 DO_PPZZ(CMPEQ
, cmpeq
)
2725 DO_PPZZ(CMPNE
, cmpne
)
2726 DO_PPZZ(CMPGT
, cmpgt
)
2727 DO_PPZZ(CMPGE
, cmpge
)
2728 DO_PPZZ(CMPHI
, cmphi
)
2729 DO_PPZZ(CMPHS
, cmphs
)
2733 #define DO_PPZW(NAME, name) \
2734 static gen_helper_gvec_flags_4 * const name##_ppzw_fns[4] = { \
2735 gen_helper_sve_##name##_ppzw_b, gen_helper_sve_##name##_ppzw_h, \
2736 gen_helper_sve_##name##_ppzw_s, NULL \
2738 TRANS_FEAT(NAME##_ppzw, aa64_sve, do_ppzz_flags, \
2739 a, name##_ppzw_fns[a->esz])
2741 DO_PPZW(CMPEQ
, cmpeq
)
2742 DO_PPZW(CMPNE
, cmpne
)
2743 DO_PPZW(CMPGT
, cmpgt
)
2744 DO_PPZW(CMPGE
, cmpge
)
2745 DO_PPZW(CMPHI
, cmphi
)
2746 DO_PPZW(CMPHS
, cmphs
)
2747 DO_PPZW(CMPLT
, cmplt
)
2748 DO_PPZW(CMPLE
, cmple
)
2749 DO_PPZW(CMPLO
, cmplo
)
2750 DO_PPZW(CMPLS
, cmpls
)
2755 *** SVE Integer Compare - Immediate Groups
2758 static bool do_ppzi_flags(DisasContext
*s
, arg_rpri_esz
*a
,
2759 gen_helper_gvec_flags_3
*gen_fn
)
2761 TCGv_ptr pd
, zn
, pg
;
2765 if (gen_fn
== NULL
) {
2768 if (!sve_access_check(s
)) {
2772 vsz
= vec_full_reg_size(s
);
2773 t
= tcg_temp_new_i32();
2774 pd
= tcg_temp_new_ptr();
2775 zn
= tcg_temp_new_ptr();
2776 pg
= tcg_temp_new_ptr();
2778 tcg_gen_addi_ptr(pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2779 tcg_gen_addi_ptr(zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
2780 tcg_gen_addi_ptr(pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
2782 gen_fn(t
, pd
, zn
, pg
, tcg_constant_i32(simd_desc(vsz
, vsz
, a
->imm
)));
2784 tcg_temp_free_ptr(pd
);
2785 tcg_temp_free_ptr(zn
);
2786 tcg_temp_free_ptr(pg
);
2790 tcg_temp_free_i32(t
);
2794 #define DO_PPZI(NAME, name) \
2795 static gen_helper_gvec_flags_3 * const name##_ppzi_fns[4] = { \
2796 gen_helper_sve_##name##_ppzi_b, gen_helper_sve_##name##_ppzi_h, \
2797 gen_helper_sve_##name##_ppzi_s, gen_helper_sve_##name##_ppzi_d, \
2799 TRANS_FEAT(NAME##_ppzi, aa64_sve, do_ppzi_flags, a, \
2800 name##_ppzi_fns[a->esz])
2802 DO_PPZI(CMPEQ
, cmpeq
)
2803 DO_PPZI(CMPNE
, cmpne
)
2804 DO_PPZI(CMPGT
, cmpgt
)
2805 DO_PPZI(CMPGE
, cmpge
)
2806 DO_PPZI(CMPHI
, cmphi
)
2807 DO_PPZI(CMPHS
, cmphs
)
2808 DO_PPZI(CMPLT
, cmplt
)
2809 DO_PPZI(CMPLE
, cmple
)
2810 DO_PPZI(CMPLO
, cmplo
)
2811 DO_PPZI(CMPLS
, cmpls
)
2816 *** SVE Partition Break Group
2819 static bool do_brk3(DisasContext
*s
, arg_rprr_s
*a
,
2820 gen_helper_gvec_4
*fn
, gen_helper_gvec_flags_4
*fn_s
)
2822 if (!sve_access_check(s
)) {
2826 unsigned vsz
= pred_full_reg_size(s
);
2828 /* Predicate sizes may be smaller and cannot use simd_desc. */
2829 TCGv_ptr d
= tcg_temp_new_ptr();
2830 TCGv_ptr n
= tcg_temp_new_ptr();
2831 TCGv_ptr m
= tcg_temp_new_ptr();
2832 TCGv_ptr g
= tcg_temp_new_ptr();
2833 TCGv_i32 desc
= tcg_constant_i32(FIELD_DP32(0, PREDDESC
, OPRSZ
, vsz
));
2835 tcg_gen_addi_ptr(d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2836 tcg_gen_addi_ptr(n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2837 tcg_gen_addi_ptr(m
, cpu_env
, pred_full_reg_offset(s
, a
->rm
));
2838 tcg_gen_addi_ptr(g
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
2841 TCGv_i32 t
= tcg_temp_new_i32();
2842 fn_s(t
, d
, n
, m
, g
, desc
);
2844 tcg_temp_free_i32(t
);
2846 fn(d
, n
, m
, g
, desc
);
2848 tcg_temp_free_ptr(d
);
2849 tcg_temp_free_ptr(n
);
2850 tcg_temp_free_ptr(m
);
2851 tcg_temp_free_ptr(g
);
2855 static bool do_brk2(DisasContext
*s
, arg_rpr_s
*a
,
2856 gen_helper_gvec_3
*fn
, gen_helper_gvec_flags_3
*fn_s
)
2858 if (!sve_access_check(s
)) {
2862 unsigned vsz
= pred_full_reg_size(s
);
2864 /* Predicate sizes may be smaller and cannot use simd_desc. */
2865 TCGv_ptr d
= tcg_temp_new_ptr();
2866 TCGv_ptr n
= tcg_temp_new_ptr();
2867 TCGv_ptr g
= tcg_temp_new_ptr();
2868 TCGv_i32 desc
= tcg_constant_i32(FIELD_DP32(0, PREDDESC
, OPRSZ
, vsz
));
2870 tcg_gen_addi_ptr(d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2871 tcg_gen_addi_ptr(n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2872 tcg_gen_addi_ptr(g
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
2875 TCGv_i32 t
= tcg_temp_new_i32();
2876 fn_s(t
, d
, n
, g
, desc
);
2878 tcg_temp_free_i32(t
);
2882 tcg_temp_free_ptr(d
);
2883 tcg_temp_free_ptr(n
);
2884 tcg_temp_free_ptr(g
);
2888 TRANS_FEAT(BRKPA
, aa64_sve
, do_brk3
, a
,
2889 gen_helper_sve_brkpa
, gen_helper_sve_brkpas
)
2890 TRANS_FEAT(BRKPB
, aa64_sve
, do_brk3
, a
,
2891 gen_helper_sve_brkpb
, gen_helper_sve_brkpbs
)
2893 TRANS_FEAT(BRKA_m
, aa64_sve
, do_brk2
, a
,
2894 gen_helper_sve_brka_m
, gen_helper_sve_brkas_m
)
2895 TRANS_FEAT(BRKB_m
, aa64_sve
, do_brk2
, a
,
2896 gen_helper_sve_brkb_m
, gen_helper_sve_brkbs_m
)
2898 TRANS_FEAT(BRKA_z
, aa64_sve
, do_brk2
, a
,
2899 gen_helper_sve_brka_z
, gen_helper_sve_brkas_z
)
2900 TRANS_FEAT(BRKB_z
, aa64_sve
, do_brk2
, a
,
2901 gen_helper_sve_brkb_z
, gen_helper_sve_brkbs_z
)
2903 TRANS_FEAT(BRKN
, aa64_sve
, do_brk2
, a
,
2904 gen_helper_sve_brkn
, gen_helper_sve_brkns
)
2907 *** SVE Predicate Count Group
2910 static void do_cntp(DisasContext
*s
, TCGv_i64 val
, int esz
, int pn
, int pg
)
2912 unsigned psz
= pred_full_reg_size(s
);
2917 tcg_gen_ld_i64(val
, cpu_env
, pred_full_reg_offset(s
, pn
));
2919 TCGv_i64 g
= tcg_temp_new_i64();
2920 tcg_gen_ld_i64(g
, cpu_env
, pred_full_reg_offset(s
, pg
));
2921 tcg_gen_and_i64(val
, val
, g
);
2922 tcg_temp_free_i64(g
);
2925 /* Reduce the pred_esz_masks value simply to reduce the
2926 * size of the code generated here.
2928 psz_mask
= MAKE_64BIT_MASK(0, psz
* 8);
2929 tcg_gen_andi_i64(val
, val
, pred_esz_masks
[esz
] & psz_mask
);
2931 tcg_gen_ctpop_i64(val
, val
);
2933 TCGv_ptr t_pn
= tcg_temp_new_ptr();
2934 TCGv_ptr t_pg
= tcg_temp_new_ptr();
2937 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, psz
);
2938 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, esz
);
2940 tcg_gen_addi_ptr(t_pn
, cpu_env
, pred_full_reg_offset(s
, pn
));
2941 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
2943 gen_helper_sve_cntp(val
, t_pn
, t_pg
, tcg_constant_i32(desc
));
2944 tcg_temp_free_ptr(t_pn
);
2945 tcg_temp_free_ptr(t_pg
);
2949 static bool trans_CNTP(DisasContext
*s
, arg_CNTP
*a
)
2951 if (sve_access_check(s
)) {
2952 do_cntp(s
, cpu_reg(s
, a
->rd
), a
->esz
, a
->rn
, a
->pg
);
2957 static bool trans_INCDECP_r(DisasContext
*s
, arg_incdec_pred
*a
)
2959 if (sve_access_check(s
)) {
2960 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
2961 TCGv_i64 val
= tcg_temp_new_i64();
2963 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
2965 tcg_gen_sub_i64(reg
, reg
, val
);
2967 tcg_gen_add_i64(reg
, reg
, val
);
2969 tcg_temp_free_i64(val
);
2974 static bool trans_INCDECP_z(DisasContext
*s
, arg_incdec2_pred
*a
)
2979 if (sve_access_check(s
)) {
2980 unsigned vsz
= vec_full_reg_size(s
);
2981 TCGv_i64 val
= tcg_temp_new_i64();
2982 GVecGen2sFn
*gvec_fn
= a
->d
? tcg_gen_gvec_subs
: tcg_gen_gvec_adds
;
2984 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
2985 gvec_fn(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
2986 vec_full_reg_offset(s
, a
->rn
), val
, vsz
, vsz
);
2991 static bool trans_SINCDECP_r_32(DisasContext
*s
, arg_incdec_pred
*a
)
2993 if (sve_access_check(s
)) {
2994 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
2995 TCGv_i64 val
= tcg_temp_new_i64();
2997 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
2998 do_sat_addsub_32(reg
, val
, a
->u
, a
->d
);
3003 static bool trans_SINCDECP_r_64(DisasContext
*s
, arg_incdec_pred
*a
)
3005 if (sve_access_check(s
)) {
3006 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3007 TCGv_i64 val
= tcg_temp_new_i64();
3009 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3010 do_sat_addsub_64(reg
, val
, a
->u
, a
->d
);
3015 static bool trans_SINCDECP_z(DisasContext
*s
, arg_incdec2_pred
*a
)
3020 if (sve_access_check(s
)) {
3021 TCGv_i64 val
= tcg_temp_new_i64();
3022 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3023 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
, val
, a
->u
, a
->d
);
3029 *** SVE Integer Compare Scalars Group
3032 static bool trans_CTERM(DisasContext
*s
, arg_CTERM
*a
)
3034 if (!sve_access_check(s
)) {
3038 TCGCond cond
= (a
->ne
? TCG_COND_NE
: TCG_COND_EQ
);
3039 TCGv_i64 rn
= read_cpu_reg(s
, a
->rn
, a
->sf
);
3040 TCGv_i64 rm
= read_cpu_reg(s
, a
->rm
, a
->sf
);
3041 TCGv_i64 cmp
= tcg_temp_new_i64();
3043 tcg_gen_setcond_i64(cond
, cmp
, rn
, rm
);
3044 tcg_gen_extrl_i64_i32(cpu_NF
, cmp
);
3045 tcg_temp_free_i64(cmp
);
3047 /* VF = !NF & !CF. */
3048 tcg_gen_xori_i32(cpu_VF
, cpu_NF
, 1);
3049 tcg_gen_andc_i32(cpu_VF
, cpu_VF
, cpu_CF
);
3051 /* Both NF and VF actually look at bit 31. */
3052 tcg_gen_neg_i32(cpu_NF
, cpu_NF
);
3053 tcg_gen_neg_i32(cpu_VF
, cpu_VF
);
3057 static bool trans_WHILE(DisasContext
*s
, arg_WHILE
*a
)
3059 TCGv_i64 op0
, op1
, t0
, t1
, tmax
;
3062 unsigned vsz
= vec_full_reg_size(s
);
3066 /* Note that GE/HS has a->eq == 0 and GT/HI has a->eq == 1. */
3067 bool eq
= a
->eq
== a
->lt
;
3069 /* The greater-than conditions are all SVE2. */
3070 if (!a
->lt
&& !dc_isar_feature(aa64_sve2
, s
)) {
3073 if (!sve_access_check(s
)) {
3077 op0
= read_cpu_reg(s
, a
->rn
, 1);
3078 op1
= read_cpu_reg(s
, a
->rm
, 1);
3082 tcg_gen_ext32u_i64(op0
, op0
);
3083 tcg_gen_ext32u_i64(op1
, op1
);
3085 tcg_gen_ext32s_i64(op0
, op0
);
3086 tcg_gen_ext32s_i64(op1
, op1
);
3090 /* For the helper, compress the different conditions into a computation
3091 * of how many iterations for which the condition is true.
3093 t0
= tcg_temp_new_i64();
3094 t1
= tcg_temp_new_i64();
3097 tcg_gen_sub_i64(t0
, op1
, op0
);
3099 maxval
= a
->sf
? UINT64_MAX
: UINT32_MAX
;
3100 cond
= eq
? TCG_COND_LEU
: TCG_COND_LTU
;
3102 maxval
= a
->sf
? INT64_MAX
: INT32_MAX
;
3103 cond
= eq
? TCG_COND_LE
: TCG_COND_LT
;
3106 tcg_gen_sub_i64(t0
, op0
, op1
);
3109 cond
= eq
? TCG_COND_GEU
: TCG_COND_GTU
;
3111 maxval
= a
->sf
? INT64_MIN
: INT32_MIN
;
3112 cond
= eq
? TCG_COND_GE
: TCG_COND_GT
;
3116 tmax
= tcg_constant_i64(vsz
>> a
->esz
);
3118 /* Equality means one more iteration. */
3119 tcg_gen_addi_i64(t0
, t0
, 1);
3122 * For the less-than while, if op1 is maxval (and the only time
3123 * the addition above could overflow), then we produce an all-true
3124 * predicate by setting the count to the vector length. This is
3125 * because the pseudocode is described as an increment + compare
3126 * loop, and the maximum integer would always compare true.
3127 * Similarly, the greater-than while has the same issue with the
3128 * minimum integer due to the decrement + compare loop.
3130 tcg_gen_movi_i64(t1
, maxval
);
3131 tcg_gen_movcond_i64(TCG_COND_EQ
, t0
, op1
, t1
, tmax
, t0
);
3134 /* Bound to the maximum. */
3135 tcg_gen_umin_i64(t0
, t0
, tmax
);
3137 /* Set the count to zero if the condition is false. */
3138 tcg_gen_movi_i64(t1
, 0);
3139 tcg_gen_movcond_i64(cond
, t0
, op0
, op1
, t0
, t1
);
3140 tcg_temp_free_i64(t1
);
3142 /* Since we're bounded, pass as a 32-bit type. */
3143 t2
= tcg_temp_new_i32();
3144 tcg_gen_extrl_i64_i32(t2
, t0
);
3145 tcg_temp_free_i64(t0
);
3147 /* Scale elements to bits. */
3148 tcg_gen_shli_i32(t2
, t2
, a
->esz
);
3150 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
/ 8);
3151 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
3153 ptr
= tcg_temp_new_ptr();
3154 tcg_gen_addi_ptr(ptr
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3157 gen_helper_sve_whilel(t2
, ptr
, t2
, tcg_constant_i32(desc
));
3159 gen_helper_sve_whileg(t2
, ptr
, t2
, tcg_constant_i32(desc
));
3163 tcg_temp_free_ptr(ptr
);
3164 tcg_temp_free_i32(t2
);
3168 static bool trans_WHILE_ptr(DisasContext
*s
, arg_WHILE_ptr
*a
)
3170 TCGv_i64 op0
, op1
, diff
, t1
, tmax
;
3173 unsigned vsz
= vec_full_reg_size(s
);
3176 if (!dc_isar_feature(aa64_sve2
, s
)) {
3179 if (!sve_access_check(s
)) {
3183 op0
= read_cpu_reg(s
, a
->rn
, 1);
3184 op1
= read_cpu_reg(s
, a
->rm
, 1);
3186 tmax
= tcg_constant_i64(vsz
);
3187 diff
= tcg_temp_new_i64();
3191 /* diff = abs(op1 - op0), noting that op0/1 are unsigned. */
3192 t1
= tcg_temp_new_i64();
3193 tcg_gen_sub_i64(diff
, op0
, op1
);
3194 tcg_gen_sub_i64(t1
, op1
, op0
);
3195 tcg_gen_movcond_i64(TCG_COND_GEU
, diff
, op0
, op1
, diff
, t1
);
3196 tcg_temp_free_i64(t1
);
3197 /* Round down to a multiple of ESIZE. */
3198 tcg_gen_andi_i64(diff
, diff
, -1 << a
->esz
);
3199 /* If op1 == op0, diff == 0, and the condition is always true. */
3200 tcg_gen_movcond_i64(TCG_COND_EQ
, diff
, op0
, op1
, tmax
, diff
);
3203 tcg_gen_sub_i64(diff
, op1
, op0
);
3204 /* Round down to a multiple of ESIZE. */
3205 tcg_gen_andi_i64(diff
, diff
, -1 << a
->esz
);
3206 /* If op0 >= op1, diff <= 0, the condition is always true. */
3207 tcg_gen_movcond_i64(TCG_COND_GEU
, diff
, op0
, op1
, tmax
, diff
);
3210 /* Bound to the maximum. */
3211 tcg_gen_umin_i64(diff
, diff
, tmax
);
3213 /* Since we're bounded, pass as a 32-bit type. */
3214 t2
= tcg_temp_new_i32();
3215 tcg_gen_extrl_i64_i32(t2
, diff
);
3216 tcg_temp_free_i64(diff
);
3218 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
/ 8);
3219 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
3221 ptr
= tcg_temp_new_ptr();
3222 tcg_gen_addi_ptr(ptr
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3224 gen_helper_sve_whilel(t2
, ptr
, t2
, tcg_constant_i32(desc
));
3227 tcg_temp_free_ptr(ptr
);
3228 tcg_temp_free_i32(t2
);
3233 *** SVE Integer Wide Immediate - Unpredicated Group
3236 static bool trans_FDUP(DisasContext
*s
, arg_FDUP
*a
)
3241 if (sve_access_check(s
)) {
3242 unsigned vsz
= vec_full_reg_size(s
);
3243 int dofs
= vec_full_reg_offset(s
, a
->rd
);
3246 /* Decode the VFP immediate. */
3247 imm
= vfp_expand_imm(a
->esz
, a
->imm
);
3248 tcg_gen_gvec_dup_imm(a
->esz
, dofs
, vsz
, vsz
, imm
);
3253 static bool trans_DUP_i(DisasContext
*s
, arg_DUP_i
*a
)
3255 if (sve_access_check(s
)) {
3256 unsigned vsz
= vec_full_reg_size(s
);
3257 int dofs
= vec_full_reg_offset(s
, a
->rd
);
3258 tcg_gen_gvec_dup_imm(a
->esz
, dofs
, vsz
, vsz
, a
->imm
);
3263 static bool trans_ADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3265 return gen_gvec_fn_arg_zzi(s
, tcg_gen_gvec_addi
, a
);
3268 static bool trans_SUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3271 return trans_ADD_zzi(s
, a
);
3274 static bool trans_SUBR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3276 static const TCGOpcode vecop_list
[] = { INDEX_op_sub_vec
, 0 };
3277 static const GVecGen2s op
[4] = {
3278 { .fni8
= tcg_gen_vec_sub8_i64
,
3279 .fniv
= tcg_gen_sub_vec
,
3280 .fno
= gen_helper_sve_subri_b
,
3281 .opt_opc
= vecop_list
,
3283 .scalar_first
= true },
3284 { .fni8
= tcg_gen_vec_sub16_i64
,
3285 .fniv
= tcg_gen_sub_vec
,
3286 .fno
= gen_helper_sve_subri_h
,
3287 .opt_opc
= vecop_list
,
3289 .scalar_first
= true },
3290 { .fni4
= tcg_gen_sub_i32
,
3291 .fniv
= tcg_gen_sub_vec
,
3292 .fno
= gen_helper_sve_subri_s
,
3293 .opt_opc
= vecop_list
,
3295 .scalar_first
= true },
3296 { .fni8
= tcg_gen_sub_i64
,
3297 .fniv
= tcg_gen_sub_vec
,
3298 .fno
= gen_helper_sve_subri_d
,
3299 .opt_opc
= vecop_list
,
3300 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
3302 .scalar_first
= true }
3305 if (sve_access_check(s
)) {
3306 unsigned vsz
= vec_full_reg_size(s
);
3307 tcg_gen_gvec_2s(vec_full_reg_offset(s
, a
->rd
),
3308 vec_full_reg_offset(s
, a
->rn
),
3309 vsz
, vsz
, tcg_constant_i64(a
->imm
), &op
[a
->esz
]);
3314 TRANS_FEAT(MUL_zzi
, aa64_sve
, gen_gvec_fn_arg_zzi
, tcg_gen_gvec_muli
, a
)
3316 static bool do_zzi_sat(DisasContext
*s
, arg_rri_esz
*a
, bool u
, bool d
)
3318 if (sve_access_check(s
)) {
3319 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
,
3320 tcg_constant_i64(a
->imm
), u
, d
);
3325 static bool trans_SQADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3327 return do_zzi_sat(s
, a
, false, false);
3330 static bool trans_UQADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3332 return do_zzi_sat(s
, a
, true, false);
3335 static bool trans_SQSUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3337 return do_zzi_sat(s
, a
, false, true);
3340 static bool trans_UQSUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3342 return do_zzi_sat(s
, a
, true, true);
3345 static bool do_zzi_ool(DisasContext
*s
, arg_rri_esz
*a
, gen_helper_gvec_2i
*fn
)
3347 if (sve_access_check(s
)) {
3348 unsigned vsz
= vec_full_reg_size(s
);
3349 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s
, a
->rd
),
3350 vec_full_reg_offset(s
, a
->rn
),
3351 tcg_constant_i64(a
->imm
), vsz
, vsz
, 0, fn
);
3356 #define DO_ZZI(NAME, name) \
3357 static bool trans_##NAME##_zzi(DisasContext *s, arg_rri_esz *a) \
3359 static gen_helper_gvec_2i * const fns[4] = { \
3360 gen_helper_sve_##name##i_b, gen_helper_sve_##name##i_h, \
3361 gen_helper_sve_##name##i_s, gen_helper_sve_##name##i_d, \
3363 return do_zzi_ool(s, a, fns[a->esz]); \
3373 static gen_helper_gvec_4
* const dot_fns
[2][2] = {
3374 { gen_helper_gvec_sdot_b
, gen_helper_gvec_sdot_h
},
3375 { gen_helper_gvec_udot_b
, gen_helper_gvec_udot_h
}
3377 TRANS_FEAT(DOT_zzzz
, aa64_sve
, gen_gvec_ool_zzzz
,
3378 dot_fns
[a
->u
][a
->sz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, 0)
3381 * SVE Multiply - Indexed
3384 TRANS_FEAT(SDOT_zzxw_s
, aa64_sve
, gen_gvec_ool_arg_zzxz
,
3385 gen_helper_gvec_sdot_idx_b
, a
)
3386 TRANS_FEAT(SDOT_zzxw_d
, aa64_sve
, gen_gvec_ool_arg_zzxz
,
3387 gen_helper_gvec_sdot_idx_h
, a
)
3388 TRANS_FEAT(UDOT_zzxw_s
, aa64_sve
, gen_gvec_ool_arg_zzxz
,
3389 gen_helper_gvec_udot_idx_b
, a
)
3390 TRANS_FEAT(UDOT_zzxw_d
, aa64_sve
, gen_gvec_ool_arg_zzxz
,
3391 gen_helper_gvec_udot_idx_h
, a
)
3393 TRANS_FEAT(SUDOT_zzxw_s
, aa64_sve_i8mm
, gen_gvec_ool_arg_zzxz
,
3394 gen_helper_gvec_sudot_idx_b
, a
)
3395 TRANS_FEAT(USDOT_zzxw_s
, aa64_sve_i8mm
, gen_gvec_ool_arg_zzxz
,
3396 gen_helper_gvec_usdot_idx_b
, a
)
3398 #define DO_SVE2_RRX(NAME, FUNC) \
3399 TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_zzz, FUNC, \
3400 a->rd, a->rn, a->rm, a->index)
3402 DO_SVE2_RRX(MUL_zzx_h
, gen_helper_gvec_mul_idx_h
)
3403 DO_SVE2_RRX(MUL_zzx_s
, gen_helper_gvec_mul_idx_s
)
3404 DO_SVE2_RRX(MUL_zzx_d
, gen_helper_gvec_mul_idx_d
)
3406 DO_SVE2_RRX(SQDMULH_zzx_h
, gen_helper_sve2_sqdmulh_idx_h
)
3407 DO_SVE2_RRX(SQDMULH_zzx_s
, gen_helper_sve2_sqdmulh_idx_s
)
3408 DO_SVE2_RRX(SQDMULH_zzx_d
, gen_helper_sve2_sqdmulh_idx_d
)
3410 DO_SVE2_RRX(SQRDMULH_zzx_h
, gen_helper_sve2_sqrdmulh_idx_h
)
3411 DO_SVE2_RRX(SQRDMULH_zzx_s
, gen_helper_sve2_sqrdmulh_idx_s
)
3412 DO_SVE2_RRX(SQRDMULH_zzx_d
, gen_helper_sve2_sqrdmulh_idx_d
)
3416 #define DO_SVE2_RRX_TB(NAME, FUNC, TOP) \
3417 TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_zzz, FUNC, \
3418 a->rd, a->rn, a->rm, (a->index << 1) | TOP)
3420 DO_SVE2_RRX_TB(SQDMULLB_zzx_s
, gen_helper_sve2_sqdmull_idx_s
, false)
3421 DO_SVE2_RRX_TB(SQDMULLB_zzx_d
, gen_helper_sve2_sqdmull_idx_d
, false)
3422 DO_SVE2_RRX_TB(SQDMULLT_zzx_s
, gen_helper_sve2_sqdmull_idx_s
, true)
3423 DO_SVE2_RRX_TB(SQDMULLT_zzx_d
, gen_helper_sve2_sqdmull_idx_d
, true)
3425 DO_SVE2_RRX_TB(SMULLB_zzx_s
, gen_helper_sve2_smull_idx_s
, false)
3426 DO_SVE2_RRX_TB(SMULLB_zzx_d
, gen_helper_sve2_smull_idx_d
, false)
3427 DO_SVE2_RRX_TB(SMULLT_zzx_s
, gen_helper_sve2_smull_idx_s
, true)
3428 DO_SVE2_RRX_TB(SMULLT_zzx_d
, gen_helper_sve2_smull_idx_d
, true)
3430 DO_SVE2_RRX_TB(UMULLB_zzx_s
, gen_helper_sve2_umull_idx_s
, false)
3431 DO_SVE2_RRX_TB(UMULLB_zzx_d
, gen_helper_sve2_umull_idx_d
, false)
3432 DO_SVE2_RRX_TB(UMULLT_zzx_s
, gen_helper_sve2_umull_idx_s
, true)
3433 DO_SVE2_RRX_TB(UMULLT_zzx_d
, gen_helper_sve2_umull_idx_d
, true)
3435 #undef DO_SVE2_RRX_TB
3437 #define DO_SVE2_RRXR(NAME, FUNC) \
3438 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_arg_zzxz, FUNC, a)
3440 DO_SVE2_RRXR(MLA_zzxz_h
, gen_helper_gvec_mla_idx_h
)
3441 DO_SVE2_RRXR(MLA_zzxz_s
, gen_helper_gvec_mla_idx_s
)
3442 DO_SVE2_RRXR(MLA_zzxz_d
, gen_helper_gvec_mla_idx_d
)
3444 DO_SVE2_RRXR(MLS_zzxz_h
, gen_helper_gvec_mls_idx_h
)
3445 DO_SVE2_RRXR(MLS_zzxz_s
, gen_helper_gvec_mls_idx_s
)
3446 DO_SVE2_RRXR(MLS_zzxz_d
, gen_helper_gvec_mls_idx_d
)
3448 DO_SVE2_RRXR(SQRDMLAH_zzxz_h
, gen_helper_sve2_sqrdmlah_idx_h
)
3449 DO_SVE2_RRXR(SQRDMLAH_zzxz_s
, gen_helper_sve2_sqrdmlah_idx_s
)
3450 DO_SVE2_RRXR(SQRDMLAH_zzxz_d
, gen_helper_sve2_sqrdmlah_idx_d
)
3452 DO_SVE2_RRXR(SQRDMLSH_zzxz_h
, gen_helper_sve2_sqrdmlsh_idx_h
)
3453 DO_SVE2_RRXR(SQRDMLSH_zzxz_s
, gen_helper_sve2_sqrdmlsh_idx_s
)
3454 DO_SVE2_RRXR(SQRDMLSH_zzxz_d
, gen_helper_sve2_sqrdmlsh_idx_d
)
3458 #define DO_SVE2_RRXR_TB(NAME, FUNC, TOP) \
3459 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_zzzz, FUNC, \
3460 a->rd, a->rn, a->rm, a->ra, (a->index << 1) | TOP)
3462 DO_SVE2_RRXR_TB(SQDMLALB_zzxw_s
, gen_helper_sve2_sqdmlal_idx_s
, false)
3463 DO_SVE2_RRXR_TB(SQDMLALB_zzxw_d
, gen_helper_sve2_sqdmlal_idx_d
, false)
3464 DO_SVE2_RRXR_TB(SQDMLALT_zzxw_s
, gen_helper_sve2_sqdmlal_idx_s
, true)
3465 DO_SVE2_RRXR_TB(SQDMLALT_zzxw_d
, gen_helper_sve2_sqdmlal_idx_d
, true)
3467 DO_SVE2_RRXR_TB(SQDMLSLB_zzxw_s
, gen_helper_sve2_sqdmlsl_idx_s
, false)
3468 DO_SVE2_RRXR_TB(SQDMLSLB_zzxw_d
, gen_helper_sve2_sqdmlsl_idx_d
, false)
3469 DO_SVE2_RRXR_TB(SQDMLSLT_zzxw_s
, gen_helper_sve2_sqdmlsl_idx_s
, true)
3470 DO_SVE2_RRXR_TB(SQDMLSLT_zzxw_d
, gen_helper_sve2_sqdmlsl_idx_d
, true)
3472 DO_SVE2_RRXR_TB(SMLALB_zzxw_s
, gen_helper_sve2_smlal_idx_s
, false)
3473 DO_SVE2_RRXR_TB(SMLALB_zzxw_d
, gen_helper_sve2_smlal_idx_d
, false)
3474 DO_SVE2_RRXR_TB(SMLALT_zzxw_s
, gen_helper_sve2_smlal_idx_s
, true)
3475 DO_SVE2_RRXR_TB(SMLALT_zzxw_d
, gen_helper_sve2_smlal_idx_d
, true)
3477 DO_SVE2_RRXR_TB(UMLALB_zzxw_s
, gen_helper_sve2_umlal_idx_s
, false)
3478 DO_SVE2_RRXR_TB(UMLALB_zzxw_d
, gen_helper_sve2_umlal_idx_d
, false)
3479 DO_SVE2_RRXR_TB(UMLALT_zzxw_s
, gen_helper_sve2_umlal_idx_s
, true)
3480 DO_SVE2_RRXR_TB(UMLALT_zzxw_d
, gen_helper_sve2_umlal_idx_d
, true)
3482 DO_SVE2_RRXR_TB(SMLSLB_zzxw_s
, gen_helper_sve2_smlsl_idx_s
, false)
3483 DO_SVE2_RRXR_TB(SMLSLB_zzxw_d
, gen_helper_sve2_smlsl_idx_d
, false)
3484 DO_SVE2_RRXR_TB(SMLSLT_zzxw_s
, gen_helper_sve2_smlsl_idx_s
, true)
3485 DO_SVE2_RRXR_TB(SMLSLT_zzxw_d
, gen_helper_sve2_smlsl_idx_d
, true)
3487 DO_SVE2_RRXR_TB(UMLSLB_zzxw_s
, gen_helper_sve2_umlsl_idx_s
, false)
3488 DO_SVE2_RRXR_TB(UMLSLB_zzxw_d
, gen_helper_sve2_umlsl_idx_d
, false)
3489 DO_SVE2_RRXR_TB(UMLSLT_zzxw_s
, gen_helper_sve2_umlsl_idx_s
, true)
3490 DO_SVE2_RRXR_TB(UMLSLT_zzxw_d
, gen_helper_sve2_umlsl_idx_d
, true)
3492 #undef DO_SVE2_RRXR_TB
3494 #define DO_SVE2_RRXR_ROT(NAME, FUNC) \
3495 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_zzzz, FUNC, \
3496 a->rd, a->rn, a->rm, a->ra, (a->index << 2) | a->rot)
3498 DO_SVE2_RRXR_ROT(CMLA_zzxz_h
, gen_helper_sve2_cmla_idx_h
)
3499 DO_SVE2_RRXR_ROT(CMLA_zzxz_s
, gen_helper_sve2_cmla_idx_s
)
3501 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_h
, gen_helper_sve2_sqrdcmlah_idx_h
)
3502 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_s
, gen_helper_sve2_sqrdcmlah_idx_s
)
3504 DO_SVE2_RRXR_ROT(CDOT_zzxw_s
, gen_helper_sve2_cdot_idx_s
)
3505 DO_SVE2_RRXR_ROT(CDOT_zzxw_d
, gen_helper_sve2_cdot_idx_d
)
3507 #undef DO_SVE2_RRXR_ROT
3510 *** SVE Floating Point Multiply-Add Indexed Group
3513 static bool do_FMLA_zzxz(DisasContext
*s
, arg_rrxr_esz
*a
, bool sub
)
3515 static gen_helper_gvec_4_ptr
* const fns
[3] = {
3516 gen_helper_gvec_fmla_idx_h
,
3517 gen_helper_gvec_fmla_idx_s
,
3518 gen_helper_gvec_fmla_idx_d
,
3521 if (sve_access_check(s
)) {
3522 unsigned vsz
= vec_full_reg_size(s
);
3523 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3524 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
3525 vec_full_reg_offset(s
, a
->rn
),
3526 vec_full_reg_offset(s
, a
->rm
),
3527 vec_full_reg_offset(s
, a
->ra
),
3528 status
, vsz
, vsz
, (a
->index
<< 1) | sub
,
3530 tcg_temp_free_ptr(status
);
3535 static bool trans_FMLA_zzxz(DisasContext
*s
, arg_FMLA_zzxz
*a
)
3537 return do_FMLA_zzxz(s
, a
, false);
3540 static bool trans_FMLS_zzxz(DisasContext
*s
, arg_FMLA_zzxz
*a
)
3542 return do_FMLA_zzxz(s
, a
, true);
3546 *** SVE Floating Point Multiply Indexed Group
3549 static bool trans_FMUL_zzx(DisasContext
*s
, arg_FMUL_zzx
*a
)
3551 static gen_helper_gvec_3_ptr
* const fns
[3] = {
3552 gen_helper_gvec_fmul_idx_h
,
3553 gen_helper_gvec_fmul_idx_s
,
3554 gen_helper_gvec_fmul_idx_d
,
3557 if (sve_access_check(s
)) {
3558 unsigned vsz
= vec_full_reg_size(s
);
3559 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3560 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
3561 vec_full_reg_offset(s
, a
->rn
),
3562 vec_full_reg_offset(s
, a
->rm
),
3563 status
, vsz
, vsz
, a
->index
, fns
[a
->esz
- 1]);
3564 tcg_temp_free_ptr(status
);
3570 *** SVE Floating Point Fast Reduction Group
3573 typedef void gen_helper_fp_reduce(TCGv_i64
, TCGv_ptr
, TCGv_ptr
,
3574 TCGv_ptr
, TCGv_i32
);
3576 static void do_reduce(DisasContext
*s
, arg_rpr_esz
*a
,
3577 gen_helper_fp_reduce
*fn
)
3579 unsigned vsz
= vec_full_reg_size(s
);
3580 unsigned p2vsz
= pow2ceil(vsz
);
3581 TCGv_i32 t_desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, p2vsz
));
3582 TCGv_ptr t_zn
, t_pg
, status
;
3585 temp
= tcg_temp_new_i64();
3586 t_zn
= tcg_temp_new_ptr();
3587 t_pg
= tcg_temp_new_ptr();
3589 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
3590 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3591 status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3593 fn(temp
, t_zn
, t_pg
, status
, t_desc
);
3594 tcg_temp_free_ptr(t_zn
);
3595 tcg_temp_free_ptr(t_pg
);
3596 tcg_temp_free_ptr(status
);
3598 write_fp_dreg(s
, a
->rd
, temp
);
3599 tcg_temp_free_i64(temp
);
3602 #define DO_VPZ(NAME, name) \
3603 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
3605 static gen_helper_fp_reduce * const fns[3] = { \
3606 gen_helper_sve_##name##_h, \
3607 gen_helper_sve_##name##_s, \
3608 gen_helper_sve_##name##_d, \
3610 if (a->esz == 0) { \
3613 if (sve_access_check(s)) { \
3614 do_reduce(s, a, fns[a->esz - 1]); \
3619 DO_VPZ(FADDV
, faddv
)
3620 DO_VPZ(FMINNMV
, fminnmv
)
3621 DO_VPZ(FMAXNMV
, fmaxnmv
)
3622 DO_VPZ(FMINV
, fminv
)
3623 DO_VPZ(FMAXV
, fmaxv
)
3626 *** SVE Floating Point Unary Operations - Unpredicated Group
3629 static void do_zz_fp(DisasContext
*s
, arg_rr_esz
*a
, gen_helper_gvec_2_ptr
*fn
)
3631 unsigned vsz
= vec_full_reg_size(s
);
3632 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3634 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s
, a
->rd
),
3635 vec_full_reg_offset(s
, a
->rn
),
3636 status
, vsz
, vsz
, 0, fn
);
3637 tcg_temp_free_ptr(status
);
3640 static bool trans_FRECPE(DisasContext
*s
, arg_rr_esz
*a
)
3642 static gen_helper_gvec_2_ptr
* const fns
[3] = {
3643 gen_helper_gvec_frecpe_h
,
3644 gen_helper_gvec_frecpe_s
,
3645 gen_helper_gvec_frecpe_d
,
3650 if (sve_access_check(s
)) {
3651 do_zz_fp(s
, a
, fns
[a
->esz
- 1]);
3656 static bool trans_FRSQRTE(DisasContext
*s
, arg_rr_esz
*a
)
3658 static gen_helper_gvec_2_ptr
* const fns
[3] = {
3659 gen_helper_gvec_frsqrte_h
,
3660 gen_helper_gvec_frsqrte_s
,
3661 gen_helper_gvec_frsqrte_d
,
3666 if (sve_access_check(s
)) {
3667 do_zz_fp(s
, a
, fns
[a
->esz
- 1]);
3673 *** SVE Floating Point Compare with Zero Group
3676 static void do_ppz_fp(DisasContext
*s
, arg_rpr_esz
*a
,
3677 gen_helper_gvec_3_ptr
*fn
)
3679 unsigned vsz
= vec_full_reg_size(s
);
3680 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3682 tcg_gen_gvec_3_ptr(pred_full_reg_offset(s
, a
->rd
),
3683 vec_full_reg_offset(s
, a
->rn
),
3684 pred_full_reg_offset(s
, a
->pg
),
3685 status
, vsz
, vsz
, 0, fn
);
3686 tcg_temp_free_ptr(status
);
3689 #define DO_PPZ(NAME, name) \
3690 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
3692 static gen_helper_gvec_3_ptr * const fns[3] = { \
3693 gen_helper_sve_##name##_h, \
3694 gen_helper_sve_##name##_s, \
3695 gen_helper_sve_##name##_d, \
3697 if (a->esz == 0) { \
3700 if (sve_access_check(s)) { \
3701 do_ppz_fp(s, a, fns[a->esz - 1]); \
3706 DO_PPZ(FCMGE_ppz0
, fcmge0
)
3707 DO_PPZ(FCMGT_ppz0
, fcmgt0
)
3708 DO_PPZ(FCMLE_ppz0
, fcmle0
)
3709 DO_PPZ(FCMLT_ppz0
, fcmlt0
)
3710 DO_PPZ(FCMEQ_ppz0
, fcmeq0
)
3711 DO_PPZ(FCMNE_ppz0
, fcmne0
)
3716 *** SVE floating-point trig multiply-add coefficient
3719 static bool trans_FTMAD(DisasContext
*s
, arg_FTMAD
*a
)
3721 static gen_helper_gvec_3_ptr
* const fns
[3] = {
3722 gen_helper_sve_ftmad_h
,
3723 gen_helper_sve_ftmad_s
,
3724 gen_helper_sve_ftmad_d
,
3730 if (sve_access_check(s
)) {
3731 unsigned vsz
= vec_full_reg_size(s
);
3732 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3733 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
3734 vec_full_reg_offset(s
, a
->rn
),
3735 vec_full_reg_offset(s
, a
->rm
),
3736 status
, vsz
, vsz
, a
->imm
, fns
[a
->esz
- 1]);
3737 tcg_temp_free_ptr(status
);
3743 *** SVE Floating Point Accumulating Reduction Group
3746 static bool trans_FADDA(DisasContext
*s
, arg_rprr_esz
*a
)
3748 typedef void fadda_fn(TCGv_i64
, TCGv_i64
, TCGv_ptr
,
3749 TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
3750 static fadda_fn
* const fns
[3] = {
3751 gen_helper_sve_fadda_h
,
3752 gen_helper_sve_fadda_s
,
3753 gen_helper_sve_fadda_d
,
3755 unsigned vsz
= vec_full_reg_size(s
);
3756 TCGv_ptr t_rm
, t_pg
, t_fpst
;
3763 if (!sve_access_check(s
)) {
3767 t_val
= load_esz(cpu_env
, vec_reg_offset(s
, a
->rn
, 0, a
->esz
), a
->esz
);
3768 t_rm
= tcg_temp_new_ptr();
3769 t_pg
= tcg_temp_new_ptr();
3770 tcg_gen_addi_ptr(t_rm
, cpu_env
, vec_full_reg_offset(s
, a
->rm
));
3771 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3772 t_fpst
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3773 t_desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
3775 fns
[a
->esz
- 1](t_val
, t_val
, t_rm
, t_pg
, t_fpst
, t_desc
);
3777 tcg_temp_free_ptr(t_fpst
);
3778 tcg_temp_free_ptr(t_pg
);
3779 tcg_temp_free_ptr(t_rm
);
3781 write_fp_dreg(s
, a
->rd
, t_val
);
3782 tcg_temp_free_i64(t_val
);
3787 *** SVE Floating Point Arithmetic - Unpredicated Group
3790 static bool do_zzz_fp(DisasContext
*s
, arg_rrr_esz
*a
,
3791 gen_helper_gvec_3_ptr
*fn
)
3796 if (sve_access_check(s
)) {
3797 unsigned vsz
= vec_full_reg_size(s
);
3798 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3799 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
3800 vec_full_reg_offset(s
, a
->rn
),
3801 vec_full_reg_offset(s
, a
->rm
),
3802 status
, vsz
, vsz
, 0, fn
);
3803 tcg_temp_free_ptr(status
);
3809 #define DO_FP3(NAME, name) \
3810 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
3812 static gen_helper_gvec_3_ptr * const fns[4] = { \
3813 NULL, gen_helper_gvec_##name##_h, \
3814 gen_helper_gvec_##name##_s, gen_helper_gvec_##name##_d \
3816 return do_zzz_fp(s, a, fns[a->esz]); \
3819 DO_FP3(FADD_zzz
, fadd
)
3820 DO_FP3(FSUB_zzz
, fsub
)
3821 DO_FP3(FMUL_zzz
, fmul
)
3822 DO_FP3(FTSMUL
, ftsmul
)
3823 DO_FP3(FRECPS
, recps
)
3824 DO_FP3(FRSQRTS
, rsqrts
)
3829 *** SVE Floating Point Arithmetic - Predicated Group
3832 static bool do_zpzz_fp(DisasContext
*s
, arg_rprr_esz
*a
,
3833 gen_helper_gvec_4_ptr
*fn
)
3838 if (sve_access_check(s
)) {
3839 unsigned vsz
= vec_full_reg_size(s
);
3840 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3841 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
3842 vec_full_reg_offset(s
, a
->rn
),
3843 vec_full_reg_offset(s
, a
->rm
),
3844 pred_full_reg_offset(s
, a
->pg
),
3845 status
, vsz
, vsz
, 0, fn
);
3846 tcg_temp_free_ptr(status
);
3851 #define DO_FP3(NAME, name) \
3852 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
3854 static gen_helper_gvec_4_ptr * const fns[4] = { \
3855 NULL, gen_helper_sve_##name##_h, \
3856 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
3858 return do_zpzz_fp(s, a, fns[a->esz]); \
3861 DO_FP3(FADD_zpzz
, fadd
)
3862 DO_FP3(FSUB_zpzz
, fsub
)
3863 DO_FP3(FMUL_zpzz
, fmul
)
3864 DO_FP3(FMIN_zpzz
, fmin
)
3865 DO_FP3(FMAX_zpzz
, fmax
)
3866 DO_FP3(FMINNM_zpzz
, fminnum
)
3867 DO_FP3(FMAXNM_zpzz
, fmaxnum
)
3869 DO_FP3(FSCALE
, fscalbn
)
3871 DO_FP3(FMULX
, fmulx
)
3875 typedef void gen_helper_sve_fp2scalar(TCGv_ptr
, TCGv_ptr
, TCGv_ptr
,
3876 TCGv_i64
, TCGv_ptr
, TCGv_i32
);
3878 static void do_fp_scalar(DisasContext
*s
, int zd
, int zn
, int pg
, bool is_fp16
,
3879 TCGv_i64 scalar
, gen_helper_sve_fp2scalar
*fn
)
3881 unsigned vsz
= vec_full_reg_size(s
);
3882 TCGv_ptr t_zd
, t_zn
, t_pg
, status
;
3885 t_zd
= tcg_temp_new_ptr();
3886 t_zn
= tcg_temp_new_ptr();
3887 t_pg
= tcg_temp_new_ptr();
3888 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, zd
));
3889 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, zn
));
3890 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
3892 status
= fpstatus_ptr(is_fp16
? FPST_FPCR_F16
: FPST_FPCR
);
3893 desc
= tcg_constant_i32(simd_desc(vsz
, vsz
, 0));
3894 fn(t_zd
, t_zn
, t_pg
, scalar
, status
, desc
);
3896 tcg_temp_free_ptr(status
);
3897 tcg_temp_free_ptr(t_pg
);
3898 tcg_temp_free_ptr(t_zn
);
3899 tcg_temp_free_ptr(t_zd
);
3902 static void do_fp_imm(DisasContext
*s
, arg_rpri_esz
*a
, uint64_t imm
,
3903 gen_helper_sve_fp2scalar
*fn
)
3905 do_fp_scalar(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
,
3906 tcg_constant_i64(imm
), fn
);
3909 #define DO_FP_IMM(NAME, name, const0, const1) \
3910 static bool trans_##NAME##_zpzi(DisasContext *s, arg_rpri_esz *a) \
3912 static gen_helper_sve_fp2scalar * const fns[3] = { \
3913 gen_helper_sve_##name##_h, \
3914 gen_helper_sve_##name##_s, \
3915 gen_helper_sve_##name##_d \
3917 static uint64_t const val[3][2] = { \
3918 { float16_##const0, float16_##const1 }, \
3919 { float32_##const0, float32_##const1 }, \
3920 { float64_##const0, float64_##const1 }, \
3922 if (a->esz == 0) { \
3925 if (sve_access_check(s)) { \
3926 do_fp_imm(s, a, val[a->esz - 1][a->imm], fns[a->esz - 1]); \
3931 DO_FP_IMM(FADD
, fadds
, half
, one
)
3932 DO_FP_IMM(FSUB
, fsubs
, half
, one
)
3933 DO_FP_IMM(FMUL
, fmuls
, half
, two
)
3934 DO_FP_IMM(FSUBR
, fsubrs
, half
, one
)
3935 DO_FP_IMM(FMAXNM
, fmaxnms
, zero
, one
)
3936 DO_FP_IMM(FMINNM
, fminnms
, zero
, one
)
3937 DO_FP_IMM(FMAX
, fmaxs
, zero
, one
)
3938 DO_FP_IMM(FMIN
, fmins
, zero
, one
)
3942 static bool do_fp_cmp(DisasContext
*s
, arg_rprr_esz
*a
,
3943 gen_helper_gvec_4_ptr
*fn
)
3948 if (sve_access_check(s
)) {
3949 unsigned vsz
= vec_full_reg_size(s
);
3950 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3951 tcg_gen_gvec_4_ptr(pred_full_reg_offset(s
, a
->rd
),
3952 vec_full_reg_offset(s
, a
->rn
),
3953 vec_full_reg_offset(s
, a
->rm
),
3954 pred_full_reg_offset(s
, a
->pg
),
3955 status
, vsz
, vsz
, 0, fn
);
3956 tcg_temp_free_ptr(status
);
3961 #define DO_FPCMP(NAME, name) \
3962 static bool trans_##NAME##_ppzz(DisasContext *s, arg_rprr_esz *a) \
3964 static gen_helper_gvec_4_ptr * const fns[4] = { \
3965 NULL, gen_helper_sve_##name##_h, \
3966 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
3968 return do_fp_cmp(s, a, fns[a->esz]); \
3971 DO_FPCMP(FCMGE
, fcmge
)
3972 DO_FPCMP(FCMGT
, fcmgt
)
3973 DO_FPCMP(FCMEQ
, fcmeq
)
3974 DO_FPCMP(FCMNE
, fcmne
)
3975 DO_FPCMP(FCMUO
, fcmuo
)
3976 DO_FPCMP(FACGE
, facge
)
3977 DO_FPCMP(FACGT
, facgt
)
3981 static bool trans_FCADD(DisasContext
*s
, arg_FCADD
*a
)
3983 static gen_helper_gvec_4_ptr
* const fns
[3] = {
3984 gen_helper_sve_fcadd_h
,
3985 gen_helper_sve_fcadd_s
,
3986 gen_helper_sve_fcadd_d
3992 if (sve_access_check(s
)) {
3993 unsigned vsz
= vec_full_reg_size(s
);
3994 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3995 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
3996 vec_full_reg_offset(s
, a
->rn
),
3997 vec_full_reg_offset(s
, a
->rm
),
3998 pred_full_reg_offset(s
, a
->pg
),
3999 status
, vsz
, vsz
, a
->rot
, fns
[a
->esz
- 1]);
4000 tcg_temp_free_ptr(status
);
4005 static bool do_fmla(DisasContext
*s
, arg_rprrr_esz
*a
,
4006 gen_helper_gvec_5_ptr
*fn
)
4011 if (sve_access_check(s
)) {
4012 unsigned vsz
= vec_full_reg_size(s
);
4013 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4014 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s
, a
->rd
),
4015 vec_full_reg_offset(s
, a
->rn
),
4016 vec_full_reg_offset(s
, a
->rm
),
4017 vec_full_reg_offset(s
, a
->ra
),
4018 pred_full_reg_offset(s
, a
->pg
),
4019 status
, vsz
, vsz
, 0, fn
);
4020 tcg_temp_free_ptr(status
);
4025 #define DO_FMLA(NAME, name) \
4026 static bool trans_##NAME(DisasContext *s, arg_rprrr_esz *a) \
4028 static gen_helper_gvec_5_ptr * const fns[4] = { \
4029 NULL, gen_helper_sve_##name##_h, \
4030 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4032 return do_fmla(s, a, fns[a->esz]); \
4035 DO_FMLA(FMLA_zpzzz
, fmla_zpzzz
)
4036 DO_FMLA(FMLS_zpzzz
, fmls_zpzzz
)
4037 DO_FMLA(FNMLA_zpzzz
, fnmla_zpzzz
)
4038 DO_FMLA(FNMLS_zpzzz
, fnmls_zpzzz
)
4042 static bool trans_FCMLA_zpzzz(DisasContext
*s
, arg_FCMLA_zpzzz
*a
)
4044 static gen_helper_gvec_5_ptr
* const fns
[4] = {
4046 gen_helper_sve_fcmla_zpzzz_h
,
4047 gen_helper_sve_fcmla_zpzzz_s
,
4048 gen_helper_sve_fcmla_zpzzz_d
,
4054 if (sve_access_check(s
)) {
4055 unsigned vsz
= vec_full_reg_size(s
);
4056 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4057 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s
, a
->rd
),
4058 vec_full_reg_offset(s
, a
->rn
),
4059 vec_full_reg_offset(s
, a
->rm
),
4060 vec_full_reg_offset(s
, a
->ra
),
4061 pred_full_reg_offset(s
, a
->pg
),
4062 status
, vsz
, vsz
, a
->rot
, fns
[a
->esz
]);
4063 tcg_temp_free_ptr(status
);
4068 static bool trans_FCMLA_zzxz(DisasContext
*s
, arg_FCMLA_zzxz
*a
)
4070 static gen_helper_gvec_4_ptr
* const fns
[2] = {
4071 gen_helper_gvec_fcmlah_idx
,
4072 gen_helper_gvec_fcmlas_idx
,
4075 tcg_debug_assert(a
->esz
== 1 || a
->esz
== 2);
4076 tcg_debug_assert(a
->rd
== a
->ra
);
4077 if (sve_access_check(s
)) {
4078 unsigned vsz
= vec_full_reg_size(s
);
4079 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4080 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4081 vec_full_reg_offset(s
, a
->rn
),
4082 vec_full_reg_offset(s
, a
->rm
),
4083 vec_full_reg_offset(s
, a
->ra
),
4085 a
->index
* 4 + a
->rot
,
4087 tcg_temp_free_ptr(status
);
4093 *** SVE Floating Point Unary Operations Predicated Group
4096 static bool do_zpz_ptr(DisasContext
*s
, int rd
, int rn
, int pg
,
4097 bool is_fp16
, gen_helper_gvec_3_ptr
*fn
)
4099 if (sve_access_check(s
)) {
4100 unsigned vsz
= vec_full_reg_size(s
);
4101 TCGv_ptr status
= fpstatus_ptr(is_fp16
? FPST_FPCR_F16
: FPST_FPCR
);
4102 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, rd
),
4103 vec_full_reg_offset(s
, rn
),
4104 pred_full_reg_offset(s
, pg
),
4105 status
, vsz
, vsz
, 0, fn
);
4106 tcg_temp_free_ptr(status
);
4111 static bool trans_FCVT_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4113 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_sh
);
4116 static bool trans_FCVT_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4118 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_hs
);
4121 static bool trans_BFCVT(DisasContext
*s
, arg_rpr_esz
*a
)
4123 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
4126 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_bfcvt
);
4129 static bool trans_FCVT_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4131 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_dh
);
4134 static bool trans_FCVT_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4136 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_hd
);
4139 static bool trans_FCVT_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4141 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_ds
);
4144 static bool trans_FCVT_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4146 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_sd
);
4149 static bool trans_FCVTZS_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4151 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hh
);
4154 static bool trans_FCVTZU_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4156 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hh
);
4159 static bool trans_FCVTZS_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4161 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hs
);
4164 static bool trans_FCVTZU_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4166 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hs
);
4169 static bool trans_FCVTZS_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4171 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hd
);
4174 static bool trans_FCVTZU_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4176 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hd
);
4179 static bool trans_FCVTZS_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4181 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_ss
);
4184 static bool trans_FCVTZU_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4186 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_ss
);
4189 static bool trans_FCVTZS_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4191 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_sd
);
4194 static bool trans_FCVTZU_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4196 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_sd
);
4199 static bool trans_FCVTZS_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4201 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_ds
);
4204 static bool trans_FCVTZU_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4206 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_ds
);
4209 static bool trans_FCVTZS_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4211 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_dd
);
4214 static bool trans_FCVTZU_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4216 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_dd
);
4219 static gen_helper_gvec_3_ptr
* const frint_fns
[3] = {
4220 gen_helper_sve_frint_h
,
4221 gen_helper_sve_frint_s
,
4222 gen_helper_sve_frint_d
4225 static bool trans_FRINTI(DisasContext
*s
, arg_rpr_esz
*a
)
4230 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
,
4231 frint_fns
[a
->esz
- 1]);
4234 static bool trans_FRINTX(DisasContext
*s
, arg_rpr_esz
*a
)
4236 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4237 gen_helper_sve_frintx_h
,
4238 gen_helper_sve_frintx_s
,
4239 gen_helper_sve_frintx_d
4244 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4247 static bool do_frint_mode(DisasContext
*s
, arg_rpr_esz
*a
,
4248 int mode
, gen_helper_gvec_3_ptr
*fn
)
4250 if (sve_access_check(s
)) {
4251 unsigned vsz
= vec_full_reg_size(s
);
4252 TCGv_i32 tmode
= tcg_const_i32(mode
);
4253 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4255 gen_helper_set_rmode(tmode
, tmode
, status
);
4257 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4258 vec_full_reg_offset(s
, a
->rn
),
4259 pred_full_reg_offset(s
, a
->pg
),
4260 status
, vsz
, vsz
, 0, fn
);
4262 gen_helper_set_rmode(tmode
, tmode
, status
);
4263 tcg_temp_free_i32(tmode
);
4264 tcg_temp_free_ptr(status
);
4269 static bool trans_FRINTN(DisasContext
*s
, arg_rpr_esz
*a
)
4274 return do_frint_mode(s
, a
, float_round_nearest_even
, frint_fns
[a
->esz
- 1]);
4277 static bool trans_FRINTP(DisasContext
*s
, arg_rpr_esz
*a
)
4282 return do_frint_mode(s
, a
, float_round_up
, frint_fns
[a
->esz
- 1]);
4285 static bool trans_FRINTM(DisasContext
*s
, arg_rpr_esz
*a
)
4290 return do_frint_mode(s
, a
, float_round_down
, frint_fns
[a
->esz
- 1]);
4293 static bool trans_FRINTZ(DisasContext
*s
, arg_rpr_esz
*a
)
4298 return do_frint_mode(s
, a
, float_round_to_zero
, frint_fns
[a
->esz
- 1]);
4301 static bool trans_FRINTA(DisasContext
*s
, arg_rpr_esz
*a
)
4306 return do_frint_mode(s
, a
, float_round_ties_away
, frint_fns
[a
->esz
- 1]);
4309 static bool trans_FRECPX(DisasContext
*s
, arg_rpr_esz
*a
)
4311 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4312 gen_helper_sve_frecpx_h
,
4313 gen_helper_sve_frecpx_s
,
4314 gen_helper_sve_frecpx_d
4319 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4322 static bool trans_FSQRT(DisasContext
*s
, arg_rpr_esz
*a
)
4324 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4325 gen_helper_sve_fsqrt_h
,
4326 gen_helper_sve_fsqrt_s
,
4327 gen_helper_sve_fsqrt_d
4332 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4335 static bool trans_SCVTF_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4337 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_hh
);
4340 static bool trans_SCVTF_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4342 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_sh
);
4345 static bool trans_SCVTF_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4347 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_dh
);
4350 static bool trans_SCVTF_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4352 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_ss
);
4355 static bool trans_SCVTF_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4357 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_ds
);
4360 static bool trans_SCVTF_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4362 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_sd
);
4365 static bool trans_SCVTF_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4367 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_dd
);
4370 static bool trans_UCVTF_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4372 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_hh
);
4375 static bool trans_UCVTF_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4377 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_sh
);
4380 static bool trans_UCVTF_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4382 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_dh
);
4385 static bool trans_UCVTF_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4387 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_ss
);
4390 static bool trans_UCVTF_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4392 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_ds
);
4395 static bool trans_UCVTF_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4397 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_sd
);
4400 static bool trans_UCVTF_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4402 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_dd
);
4406 *** SVE Memory - 32-bit Gather and Unsized Contiguous Group
4409 /* Subroutine loading a vector register at VOFS of LEN bytes.
4410 * The load should begin at the address Rn + IMM.
4413 static void do_ldr(DisasContext
*s
, uint32_t vofs
, int len
, int rn
, int imm
)
4415 int len_align
= QEMU_ALIGN_DOWN(len
, 8);
4416 int len_remain
= len
% 8;
4417 int nparts
= len
/ 8 + ctpop8(len_remain
);
4418 int midx
= get_mem_index(s
);
4419 TCGv_i64 dirty_addr
, clean_addr
, t0
, t1
;
4421 dirty_addr
= tcg_temp_new_i64();
4422 tcg_gen_addi_i64(dirty_addr
, cpu_reg_sp(s
, rn
), imm
);
4423 clean_addr
= gen_mte_checkN(s
, dirty_addr
, false, rn
!= 31, len
);
4424 tcg_temp_free_i64(dirty_addr
);
4427 * Note that unpredicated load/store of vector/predicate registers
4428 * are defined as a stream of bytes, which equates to little-endian
4429 * operations on larger quantities.
4430 * Attempt to keep code expansion to a minimum by limiting the
4431 * amount of unrolling done.
4436 t0
= tcg_temp_new_i64();
4437 for (i
= 0; i
< len_align
; i
+= 8) {
4438 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEUQ
);
4439 tcg_gen_st_i64(t0
, cpu_env
, vofs
+ i
);
4440 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4442 tcg_temp_free_i64(t0
);
4444 TCGLabel
*loop
= gen_new_label();
4445 TCGv_ptr tp
, i
= tcg_const_local_ptr(0);
4447 /* Copy the clean address into a local temp, live across the loop. */
4449 clean_addr
= new_tmp_a64_local(s
);
4450 tcg_gen_mov_i64(clean_addr
, t0
);
4452 gen_set_label(loop
);
4454 t0
= tcg_temp_new_i64();
4455 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEUQ
);
4456 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4458 tp
= tcg_temp_new_ptr();
4459 tcg_gen_add_ptr(tp
, cpu_env
, i
);
4460 tcg_gen_addi_ptr(i
, i
, 8);
4461 tcg_gen_st_i64(t0
, tp
, vofs
);
4462 tcg_temp_free_ptr(tp
);
4463 tcg_temp_free_i64(t0
);
4465 tcg_gen_brcondi_ptr(TCG_COND_LTU
, i
, len_align
, loop
);
4466 tcg_temp_free_ptr(i
);
4470 * Predicate register loads can be any multiple of 2.
4471 * Note that we still store the entire 64-bit unit into cpu_env.
4474 t0
= tcg_temp_new_i64();
4475 switch (len_remain
) {
4479 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
,
4480 MO_LE
| ctz32(len_remain
));
4484 t1
= tcg_temp_new_i64();
4485 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEUL
);
4486 tcg_gen_addi_i64(clean_addr
, clean_addr
, 4);
4487 tcg_gen_qemu_ld_i64(t1
, clean_addr
, midx
, MO_LEUW
);
4488 tcg_gen_deposit_i64(t0
, t0
, t1
, 32, 32);
4489 tcg_temp_free_i64(t1
);
4493 g_assert_not_reached();
4495 tcg_gen_st_i64(t0
, cpu_env
, vofs
+ len_align
);
4496 tcg_temp_free_i64(t0
);
4500 /* Similarly for stores. */
4501 static void do_str(DisasContext
*s
, uint32_t vofs
, int len
, int rn
, int imm
)
4503 int len_align
= QEMU_ALIGN_DOWN(len
, 8);
4504 int len_remain
= len
% 8;
4505 int nparts
= len
/ 8 + ctpop8(len_remain
);
4506 int midx
= get_mem_index(s
);
4507 TCGv_i64 dirty_addr
, clean_addr
, t0
;
4509 dirty_addr
= tcg_temp_new_i64();
4510 tcg_gen_addi_i64(dirty_addr
, cpu_reg_sp(s
, rn
), imm
);
4511 clean_addr
= gen_mte_checkN(s
, dirty_addr
, false, rn
!= 31, len
);
4512 tcg_temp_free_i64(dirty_addr
);
4514 /* Note that unpredicated load/store of vector/predicate registers
4515 * are defined as a stream of bytes, which equates to little-endian
4516 * operations on larger quantities. There is no nice way to force
4517 * a little-endian store for aarch64_be-linux-user out of line.
4519 * Attempt to keep code expansion to a minimum by limiting the
4520 * amount of unrolling done.
4525 t0
= tcg_temp_new_i64();
4526 for (i
= 0; i
< len_align
; i
+= 8) {
4527 tcg_gen_ld_i64(t0
, cpu_env
, vofs
+ i
);
4528 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUQ
);
4529 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4531 tcg_temp_free_i64(t0
);
4533 TCGLabel
*loop
= gen_new_label();
4534 TCGv_ptr tp
, i
= tcg_const_local_ptr(0);
4536 /* Copy the clean address into a local temp, live across the loop. */
4538 clean_addr
= new_tmp_a64_local(s
);
4539 tcg_gen_mov_i64(clean_addr
, t0
);
4541 gen_set_label(loop
);
4543 t0
= tcg_temp_new_i64();
4544 tp
= tcg_temp_new_ptr();
4545 tcg_gen_add_ptr(tp
, cpu_env
, i
);
4546 tcg_gen_ld_i64(t0
, tp
, vofs
);
4547 tcg_gen_addi_ptr(i
, i
, 8);
4548 tcg_temp_free_ptr(tp
);
4550 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUQ
);
4551 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4552 tcg_temp_free_i64(t0
);
4554 tcg_gen_brcondi_ptr(TCG_COND_LTU
, i
, len_align
, loop
);
4555 tcg_temp_free_ptr(i
);
4558 /* Predicate register stores can be any multiple of 2. */
4560 t0
= tcg_temp_new_i64();
4561 tcg_gen_ld_i64(t0
, cpu_env
, vofs
+ len_align
);
4563 switch (len_remain
) {
4567 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
,
4568 MO_LE
| ctz32(len_remain
));
4572 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUL
);
4573 tcg_gen_addi_i64(clean_addr
, clean_addr
, 4);
4574 tcg_gen_shri_i64(t0
, t0
, 32);
4575 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUW
);
4579 g_assert_not_reached();
4581 tcg_temp_free_i64(t0
);
4585 static bool trans_LDR_zri(DisasContext
*s
, arg_rri
*a
)
4587 if (sve_access_check(s
)) {
4588 int size
= vec_full_reg_size(s
);
4589 int off
= vec_full_reg_offset(s
, a
->rd
);
4590 do_ldr(s
, off
, size
, a
->rn
, a
->imm
* size
);
4595 static bool trans_LDR_pri(DisasContext
*s
, arg_rri
*a
)
4597 if (sve_access_check(s
)) {
4598 int size
= pred_full_reg_size(s
);
4599 int off
= pred_full_reg_offset(s
, a
->rd
);
4600 do_ldr(s
, off
, size
, a
->rn
, a
->imm
* size
);
4605 static bool trans_STR_zri(DisasContext
*s
, arg_rri
*a
)
4607 if (sve_access_check(s
)) {
4608 int size
= vec_full_reg_size(s
);
4609 int off
= vec_full_reg_offset(s
, a
->rd
);
4610 do_str(s
, off
, size
, a
->rn
, a
->imm
* size
);
4615 static bool trans_STR_pri(DisasContext
*s
, arg_rri
*a
)
4617 if (sve_access_check(s
)) {
4618 int size
= pred_full_reg_size(s
);
4619 int off
= pred_full_reg_offset(s
, a
->rd
);
4620 do_str(s
, off
, size
, a
->rn
, a
->imm
* size
);
4626 *** SVE Memory - Contiguous Load Group
4629 /* The memory mode of the dtype. */
4630 static const MemOp dtype_mop
[16] = {
4631 MO_UB
, MO_UB
, MO_UB
, MO_UB
,
4632 MO_SL
, MO_UW
, MO_UW
, MO_UW
,
4633 MO_SW
, MO_SW
, MO_UL
, MO_UL
,
4634 MO_SB
, MO_SB
, MO_SB
, MO_UQ
4637 #define dtype_msz(x) (dtype_mop[x] & MO_SIZE)
4639 /* The vector element size of dtype. */
4640 static const uint8_t dtype_esz
[16] = {
4647 static void do_mem_zpa(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
,
4648 int dtype
, uint32_t mte_n
, bool is_write
,
4649 gen_helper_gvec_mem
*fn
)
4651 unsigned vsz
= vec_full_reg_size(s
);
4656 * For e.g. LD4, there are not enough arguments to pass all 4
4657 * registers as pointers, so encode the regno into the data field.
4658 * For consistency, do this even for LD1.
4660 if (s
->mte_active
[0]) {
4661 int msz
= dtype_msz(dtype
);
4663 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, get_mem_index(s
));
4664 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
4665 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
4666 desc
= FIELD_DP32(desc
, MTEDESC
, WRITE
, is_write
);
4667 desc
= FIELD_DP32(desc
, MTEDESC
, SIZEM1
, (mte_n
<< msz
) - 1);
4668 desc
<<= SVE_MTEDESC_SHIFT
;
4670 addr
= clean_data_tbi(s
, addr
);
4673 desc
= simd_desc(vsz
, vsz
, zt
| desc
);
4674 t_pg
= tcg_temp_new_ptr();
4676 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
4677 fn(cpu_env
, t_pg
, addr
, tcg_constant_i32(desc
));
4679 tcg_temp_free_ptr(t_pg
);
4682 /* Indexed by [mte][be][dtype][nreg] */
4683 static gen_helper_gvec_mem
* const ldr_fns
[2][2][16][4] = {
4684 { /* mte inactive, little-endian */
4685 { { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld2bb_r
,
4686 gen_helper_sve_ld3bb_r
, gen_helper_sve_ld4bb_r
},
4687 { gen_helper_sve_ld1bhu_r
, NULL
, NULL
, NULL
},
4688 { gen_helper_sve_ld1bsu_r
, NULL
, NULL
, NULL
},
4689 { gen_helper_sve_ld1bdu_r
, NULL
, NULL
, NULL
},
4691 { gen_helper_sve_ld1sds_le_r
, NULL
, NULL
, NULL
},
4692 { gen_helper_sve_ld1hh_le_r
, gen_helper_sve_ld2hh_le_r
,
4693 gen_helper_sve_ld3hh_le_r
, gen_helper_sve_ld4hh_le_r
},
4694 { gen_helper_sve_ld1hsu_le_r
, NULL
, NULL
, NULL
},
4695 { gen_helper_sve_ld1hdu_le_r
, NULL
, NULL
, NULL
},
4697 { gen_helper_sve_ld1hds_le_r
, NULL
, NULL
, NULL
},
4698 { gen_helper_sve_ld1hss_le_r
, NULL
, NULL
, NULL
},
4699 { gen_helper_sve_ld1ss_le_r
, gen_helper_sve_ld2ss_le_r
,
4700 gen_helper_sve_ld3ss_le_r
, gen_helper_sve_ld4ss_le_r
},
4701 { gen_helper_sve_ld1sdu_le_r
, NULL
, NULL
, NULL
},
4703 { gen_helper_sve_ld1bds_r
, NULL
, NULL
, NULL
},
4704 { gen_helper_sve_ld1bss_r
, NULL
, NULL
, NULL
},
4705 { gen_helper_sve_ld1bhs_r
, NULL
, NULL
, NULL
},
4706 { gen_helper_sve_ld1dd_le_r
, gen_helper_sve_ld2dd_le_r
,
4707 gen_helper_sve_ld3dd_le_r
, gen_helper_sve_ld4dd_le_r
} },
4709 /* mte inactive, big-endian */
4710 { { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld2bb_r
,
4711 gen_helper_sve_ld3bb_r
, gen_helper_sve_ld4bb_r
},
4712 { gen_helper_sve_ld1bhu_r
, NULL
, NULL
, NULL
},
4713 { gen_helper_sve_ld1bsu_r
, NULL
, NULL
, NULL
},
4714 { gen_helper_sve_ld1bdu_r
, NULL
, NULL
, NULL
},
4716 { gen_helper_sve_ld1sds_be_r
, NULL
, NULL
, NULL
},
4717 { gen_helper_sve_ld1hh_be_r
, gen_helper_sve_ld2hh_be_r
,
4718 gen_helper_sve_ld3hh_be_r
, gen_helper_sve_ld4hh_be_r
},
4719 { gen_helper_sve_ld1hsu_be_r
, NULL
, NULL
, NULL
},
4720 { gen_helper_sve_ld1hdu_be_r
, NULL
, NULL
, NULL
},
4722 { gen_helper_sve_ld1hds_be_r
, NULL
, NULL
, NULL
},
4723 { gen_helper_sve_ld1hss_be_r
, NULL
, NULL
, NULL
},
4724 { gen_helper_sve_ld1ss_be_r
, gen_helper_sve_ld2ss_be_r
,
4725 gen_helper_sve_ld3ss_be_r
, gen_helper_sve_ld4ss_be_r
},
4726 { gen_helper_sve_ld1sdu_be_r
, NULL
, NULL
, NULL
},
4728 { gen_helper_sve_ld1bds_r
, NULL
, NULL
, NULL
},
4729 { gen_helper_sve_ld1bss_r
, NULL
, NULL
, NULL
},
4730 { gen_helper_sve_ld1bhs_r
, NULL
, NULL
, NULL
},
4731 { gen_helper_sve_ld1dd_be_r
, gen_helper_sve_ld2dd_be_r
,
4732 gen_helper_sve_ld3dd_be_r
, gen_helper_sve_ld4dd_be_r
} } },
4734 { /* mte active, little-endian */
4735 { { gen_helper_sve_ld1bb_r_mte
,
4736 gen_helper_sve_ld2bb_r_mte
,
4737 gen_helper_sve_ld3bb_r_mte
,
4738 gen_helper_sve_ld4bb_r_mte
},
4739 { gen_helper_sve_ld1bhu_r_mte
, NULL
, NULL
, NULL
},
4740 { gen_helper_sve_ld1bsu_r_mte
, NULL
, NULL
, NULL
},
4741 { gen_helper_sve_ld1bdu_r_mte
, NULL
, NULL
, NULL
},
4743 { gen_helper_sve_ld1sds_le_r_mte
, NULL
, NULL
, NULL
},
4744 { gen_helper_sve_ld1hh_le_r_mte
,
4745 gen_helper_sve_ld2hh_le_r_mte
,
4746 gen_helper_sve_ld3hh_le_r_mte
,
4747 gen_helper_sve_ld4hh_le_r_mte
},
4748 { gen_helper_sve_ld1hsu_le_r_mte
, NULL
, NULL
, NULL
},
4749 { gen_helper_sve_ld1hdu_le_r_mte
, NULL
, NULL
, NULL
},
4751 { gen_helper_sve_ld1hds_le_r_mte
, NULL
, NULL
, NULL
},
4752 { gen_helper_sve_ld1hss_le_r_mte
, NULL
, NULL
, NULL
},
4753 { gen_helper_sve_ld1ss_le_r_mte
,
4754 gen_helper_sve_ld2ss_le_r_mte
,
4755 gen_helper_sve_ld3ss_le_r_mte
,
4756 gen_helper_sve_ld4ss_le_r_mte
},
4757 { gen_helper_sve_ld1sdu_le_r_mte
, NULL
, NULL
, NULL
},
4759 { gen_helper_sve_ld1bds_r_mte
, NULL
, NULL
, NULL
},
4760 { gen_helper_sve_ld1bss_r_mte
, NULL
, NULL
, NULL
},
4761 { gen_helper_sve_ld1bhs_r_mte
, NULL
, NULL
, NULL
},
4762 { gen_helper_sve_ld1dd_le_r_mte
,
4763 gen_helper_sve_ld2dd_le_r_mte
,
4764 gen_helper_sve_ld3dd_le_r_mte
,
4765 gen_helper_sve_ld4dd_le_r_mte
} },
4767 /* mte active, big-endian */
4768 { { gen_helper_sve_ld1bb_r_mte
,
4769 gen_helper_sve_ld2bb_r_mte
,
4770 gen_helper_sve_ld3bb_r_mte
,
4771 gen_helper_sve_ld4bb_r_mte
},
4772 { gen_helper_sve_ld1bhu_r_mte
, NULL
, NULL
, NULL
},
4773 { gen_helper_sve_ld1bsu_r_mte
, NULL
, NULL
, NULL
},
4774 { gen_helper_sve_ld1bdu_r_mte
, NULL
, NULL
, NULL
},
4776 { gen_helper_sve_ld1sds_be_r_mte
, NULL
, NULL
, NULL
},
4777 { gen_helper_sve_ld1hh_be_r_mte
,
4778 gen_helper_sve_ld2hh_be_r_mte
,
4779 gen_helper_sve_ld3hh_be_r_mte
,
4780 gen_helper_sve_ld4hh_be_r_mte
},
4781 { gen_helper_sve_ld1hsu_be_r_mte
, NULL
, NULL
, NULL
},
4782 { gen_helper_sve_ld1hdu_be_r_mte
, NULL
, NULL
, NULL
},
4784 { gen_helper_sve_ld1hds_be_r_mte
, NULL
, NULL
, NULL
},
4785 { gen_helper_sve_ld1hss_be_r_mte
, NULL
, NULL
, NULL
},
4786 { gen_helper_sve_ld1ss_be_r_mte
,
4787 gen_helper_sve_ld2ss_be_r_mte
,
4788 gen_helper_sve_ld3ss_be_r_mte
,
4789 gen_helper_sve_ld4ss_be_r_mte
},
4790 { gen_helper_sve_ld1sdu_be_r_mte
, NULL
, NULL
, NULL
},
4792 { gen_helper_sve_ld1bds_r_mte
, NULL
, NULL
, NULL
},
4793 { gen_helper_sve_ld1bss_r_mte
, NULL
, NULL
, NULL
},
4794 { gen_helper_sve_ld1bhs_r_mte
, NULL
, NULL
, NULL
},
4795 { gen_helper_sve_ld1dd_be_r_mte
,
4796 gen_helper_sve_ld2dd_be_r_mte
,
4797 gen_helper_sve_ld3dd_be_r_mte
,
4798 gen_helper_sve_ld4dd_be_r_mte
} } },
4801 static void do_ld_zpa(DisasContext
*s
, int zt
, int pg
,
4802 TCGv_i64 addr
, int dtype
, int nreg
)
4804 gen_helper_gvec_mem
*fn
4805 = ldr_fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][dtype
][nreg
];
4808 * While there are holes in the table, they are not
4809 * accessible via the instruction encoding.
4812 do_mem_zpa(s
, zt
, pg
, addr
, dtype
, nreg
, false, fn
);
4815 static bool trans_LD_zprr(DisasContext
*s
, arg_rprr_load
*a
)
4820 if (sve_access_check(s
)) {
4821 TCGv_i64 addr
= new_tmp_a64(s
);
4822 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
4823 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
4824 do_ld_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, a
->nreg
);
4829 static bool trans_LD_zpri(DisasContext
*s
, arg_rpri_load
*a
)
4831 if (sve_access_check(s
)) {
4832 int vsz
= vec_full_reg_size(s
);
4833 int elements
= vsz
>> dtype_esz
[a
->dtype
];
4834 TCGv_i64 addr
= new_tmp_a64(s
);
4836 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
),
4837 (a
->imm
* elements
* (a
->nreg
+ 1))
4838 << dtype_msz(a
->dtype
));
4839 do_ld_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, a
->nreg
);
4844 static bool trans_LDFF1_zprr(DisasContext
*s
, arg_rprr_load
*a
)
4846 static gen_helper_gvec_mem
* const fns
[2][2][16] = {
4847 { /* mte inactive, little-endian */
4848 { gen_helper_sve_ldff1bb_r
,
4849 gen_helper_sve_ldff1bhu_r
,
4850 gen_helper_sve_ldff1bsu_r
,
4851 gen_helper_sve_ldff1bdu_r
,
4853 gen_helper_sve_ldff1sds_le_r
,
4854 gen_helper_sve_ldff1hh_le_r
,
4855 gen_helper_sve_ldff1hsu_le_r
,
4856 gen_helper_sve_ldff1hdu_le_r
,
4858 gen_helper_sve_ldff1hds_le_r
,
4859 gen_helper_sve_ldff1hss_le_r
,
4860 gen_helper_sve_ldff1ss_le_r
,
4861 gen_helper_sve_ldff1sdu_le_r
,
4863 gen_helper_sve_ldff1bds_r
,
4864 gen_helper_sve_ldff1bss_r
,
4865 gen_helper_sve_ldff1bhs_r
,
4866 gen_helper_sve_ldff1dd_le_r
},
4868 /* mte inactive, big-endian */
4869 { gen_helper_sve_ldff1bb_r
,
4870 gen_helper_sve_ldff1bhu_r
,
4871 gen_helper_sve_ldff1bsu_r
,
4872 gen_helper_sve_ldff1bdu_r
,
4874 gen_helper_sve_ldff1sds_be_r
,
4875 gen_helper_sve_ldff1hh_be_r
,
4876 gen_helper_sve_ldff1hsu_be_r
,
4877 gen_helper_sve_ldff1hdu_be_r
,
4879 gen_helper_sve_ldff1hds_be_r
,
4880 gen_helper_sve_ldff1hss_be_r
,
4881 gen_helper_sve_ldff1ss_be_r
,
4882 gen_helper_sve_ldff1sdu_be_r
,
4884 gen_helper_sve_ldff1bds_r
,
4885 gen_helper_sve_ldff1bss_r
,
4886 gen_helper_sve_ldff1bhs_r
,
4887 gen_helper_sve_ldff1dd_be_r
} },
4889 { /* mte active, little-endian */
4890 { gen_helper_sve_ldff1bb_r_mte
,
4891 gen_helper_sve_ldff1bhu_r_mte
,
4892 gen_helper_sve_ldff1bsu_r_mte
,
4893 gen_helper_sve_ldff1bdu_r_mte
,
4895 gen_helper_sve_ldff1sds_le_r_mte
,
4896 gen_helper_sve_ldff1hh_le_r_mte
,
4897 gen_helper_sve_ldff1hsu_le_r_mte
,
4898 gen_helper_sve_ldff1hdu_le_r_mte
,
4900 gen_helper_sve_ldff1hds_le_r_mte
,
4901 gen_helper_sve_ldff1hss_le_r_mte
,
4902 gen_helper_sve_ldff1ss_le_r_mte
,
4903 gen_helper_sve_ldff1sdu_le_r_mte
,
4905 gen_helper_sve_ldff1bds_r_mte
,
4906 gen_helper_sve_ldff1bss_r_mte
,
4907 gen_helper_sve_ldff1bhs_r_mte
,
4908 gen_helper_sve_ldff1dd_le_r_mte
},
4910 /* mte active, big-endian */
4911 { gen_helper_sve_ldff1bb_r_mte
,
4912 gen_helper_sve_ldff1bhu_r_mte
,
4913 gen_helper_sve_ldff1bsu_r_mte
,
4914 gen_helper_sve_ldff1bdu_r_mte
,
4916 gen_helper_sve_ldff1sds_be_r_mte
,
4917 gen_helper_sve_ldff1hh_be_r_mte
,
4918 gen_helper_sve_ldff1hsu_be_r_mte
,
4919 gen_helper_sve_ldff1hdu_be_r_mte
,
4921 gen_helper_sve_ldff1hds_be_r_mte
,
4922 gen_helper_sve_ldff1hss_be_r_mte
,
4923 gen_helper_sve_ldff1ss_be_r_mte
,
4924 gen_helper_sve_ldff1sdu_be_r_mte
,
4926 gen_helper_sve_ldff1bds_r_mte
,
4927 gen_helper_sve_ldff1bss_r_mte
,
4928 gen_helper_sve_ldff1bhs_r_mte
,
4929 gen_helper_sve_ldff1dd_be_r_mte
} },
4932 if (sve_access_check(s
)) {
4933 TCGv_i64 addr
= new_tmp_a64(s
);
4934 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
4935 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
4936 do_mem_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, 1, false,
4937 fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][a
->dtype
]);
4942 static bool trans_LDNF1_zpri(DisasContext
*s
, arg_rpri_load
*a
)
4944 static gen_helper_gvec_mem
* const fns
[2][2][16] = {
4945 { /* mte inactive, little-endian */
4946 { gen_helper_sve_ldnf1bb_r
,
4947 gen_helper_sve_ldnf1bhu_r
,
4948 gen_helper_sve_ldnf1bsu_r
,
4949 gen_helper_sve_ldnf1bdu_r
,
4951 gen_helper_sve_ldnf1sds_le_r
,
4952 gen_helper_sve_ldnf1hh_le_r
,
4953 gen_helper_sve_ldnf1hsu_le_r
,
4954 gen_helper_sve_ldnf1hdu_le_r
,
4956 gen_helper_sve_ldnf1hds_le_r
,
4957 gen_helper_sve_ldnf1hss_le_r
,
4958 gen_helper_sve_ldnf1ss_le_r
,
4959 gen_helper_sve_ldnf1sdu_le_r
,
4961 gen_helper_sve_ldnf1bds_r
,
4962 gen_helper_sve_ldnf1bss_r
,
4963 gen_helper_sve_ldnf1bhs_r
,
4964 gen_helper_sve_ldnf1dd_le_r
},
4966 /* mte inactive, big-endian */
4967 { gen_helper_sve_ldnf1bb_r
,
4968 gen_helper_sve_ldnf1bhu_r
,
4969 gen_helper_sve_ldnf1bsu_r
,
4970 gen_helper_sve_ldnf1bdu_r
,
4972 gen_helper_sve_ldnf1sds_be_r
,
4973 gen_helper_sve_ldnf1hh_be_r
,
4974 gen_helper_sve_ldnf1hsu_be_r
,
4975 gen_helper_sve_ldnf1hdu_be_r
,
4977 gen_helper_sve_ldnf1hds_be_r
,
4978 gen_helper_sve_ldnf1hss_be_r
,
4979 gen_helper_sve_ldnf1ss_be_r
,
4980 gen_helper_sve_ldnf1sdu_be_r
,
4982 gen_helper_sve_ldnf1bds_r
,
4983 gen_helper_sve_ldnf1bss_r
,
4984 gen_helper_sve_ldnf1bhs_r
,
4985 gen_helper_sve_ldnf1dd_be_r
} },
4987 { /* mte inactive, little-endian */
4988 { gen_helper_sve_ldnf1bb_r_mte
,
4989 gen_helper_sve_ldnf1bhu_r_mte
,
4990 gen_helper_sve_ldnf1bsu_r_mte
,
4991 gen_helper_sve_ldnf1bdu_r_mte
,
4993 gen_helper_sve_ldnf1sds_le_r_mte
,
4994 gen_helper_sve_ldnf1hh_le_r_mte
,
4995 gen_helper_sve_ldnf1hsu_le_r_mte
,
4996 gen_helper_sve_ldnf1hdu_le_r_mte
,
4998 gen_helper_sve_ldnf1hds_le_r_mte
,
4999 gen_helper_sve_ldnf1hss_le_r_mte
,
5000 gen_helper_sve_ldnf1ss_le_r_mte
,
5001 gen_helper_sve_ldnf1sdu_le_r_mte
,
5003 gen_helper_sve_ldnf1bds_r_mte
,
5004 gen_helper_sve_ldnf1bss_r_mte
,
5005 gen_helper_sve_ldnf1bhs_r_mte
,
5006 gen_helper_sve_ldnf1dd_le_r_mte
},
5008 /* mte inactive, big-endian */
5009 { gen_helper_sve_ldnf1bb_r_mte
,
5010 gen_helper_sve_ldnf1bhu_r_mte
,
5011 gen_helper_sve_ldnf1bsu_r_mte
,
5012 gen_helper_sve_ldnf1bdu_r_mte
,
5014 gen_helper_sve_ldnf1sds_be_r_mte
,
5015 gen_helper_sve_ldnf1hh_be_r_mte
,
5016 gen_helper_sve_ldnf1hsu_be_r_mte
,
5017 gen_helper_sve_ldnf1hdu_be_r_mte
,
5019 gen_helper_sve_ldnf1hds_be_r_mte
,
5020 gen_helper_sve_ldnf1hss_be_r_mte
,
5021 gen_helper_sve_ldnf1ss_be_r_mte
,
5022 gen_helper_sve_ldnf1sdu_be_r_mte
,
5024 gen_helper_sve_ldnf1bds_r_mte
,
5025 gen_helper_sve_ldnf1bss_r_mte
,
5026 gen_helper_sve_ldnf1bhs_r_mte
,
5027 gen_helper_sve_ldnf1dd_be_r_mte
} },
5030 if (sve_access_check(s
)) {
5031 int vsz
= vec_full_reg_size(s
);
5032 int elements
= vsz
>> dtype_esz
[a
->dtype
];
5033 int off
= (a
->imm
* elements
) << dtype_msz(a
->dtype
);
5034 TCGv_i64 addr
= new_tmp_a64(s
);
5036 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), off
);
5037 do_mem_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, 1, false,
5038 fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][a
->dtype
]);
5043 static void do_ldrq(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
, int dtype
)
5045 unsigned vsz
= vec_full_reg_size(s
);
5049 /* Load the first quadword using the normal predicated load helpers. */
5050 poff
= pred_full_reg_offset(s
, pg
);
5053 * Zero-extend the first 16 bits of the predicate into a temporary.
5054 * This avoids triggering an assert making sure we don't have bits
5055 * set within a predicate beyond VQ, but we have lowered VQ to 1
5056 * for this load operation.
5058 TCGv_i64 tmp
= tcg_temp_new_i64();
5062 tcg_gen_ld16u_i64(tmp
, cpu_env
, poff
);
5064 poff
= offsetof(CPUARMState
, vfp
.preg_tmp
);
5065 tcg_gen_st_i64(tmp
, cpu_env
, poff
);
5066 tcg_temp_free_i64(tmp
);
5069 t_pg
= tcg_temp_new_ptr();
5070 tcg_gen_addi_ptr(t_pg
, cpu_env
, poff
);
5072 gen_helper_gvec_mem
*fn
5073 = ldr_fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][dtype
][0];
5074 fn(cpu_env
, t_pg
, addr
, tcg_constant_i32(simd_desc(16, 16, zt
)));
5076 tcg_temp_free_ptr(t_pg
);
5078 /* Replicate that first quadword. */
5080 int doff
= vec_full_reg_offset(s
, zt
);
5081 tcg_gen_gvec_dup_mem(4, doff
+ 16, doff
, vsz
- 16, vsz
- 16);
5085 static bool trans_LD1RQ_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5090 if (sve_access_check(s
)) {
5091 int msz
= dtype_msz(a
->dtype
);
5092 TCGv_i64 addr
= new_tmp_a64(s
);
5093 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), msz
);
5094 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5095 do_ldrq(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5100 static bool trans_LD1RQ_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5102 if (sve_access_check(s
)) {
5103 TCGv_i64 addr
= new_tmp_a64(s
);
5104 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), a
->imm
* 16);
5105 do_ldrq(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5110 static void do_ldro(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
, int dtype
)
5112 unsigned vsz
= vec_full_reg_size(s
);
5119 * Note that this UNDEFINED check comes after CheckSVEEnabled()
5120 * in the ARM pseudocode, which is the sve_access_check() done
5121 * in our caller. We should not now return false from the caller.
5123 unallocated_encoding(s
);
5127 /* Load the first octaword using the normal predicated load helpers. */
5129 poff
= pred_full_reg_offset(s
, pg
);
5132 * Zero-extend the first 32 bits of the predicate into a temporary.
5133 * This avoids triggering an assert making sure we don't have bits
5134 * set within a predicate beyond VQ, but we have lowered VQ to 2
5135 * for this load operation.
5137 TCGv_i64 tmp
= tcg_temp_new_i64();
5141 tcg_gen_ld32u_i64(tmp
, cpu_env
, poff
);
5143 poff
= offsetof(CPUARMState
, vfp
.preg_tmp
);
5144 tcg_gen_st_i64(tmp
, cpu_env
, poff
);
5145 tcg_temp_free_i64(tmp
);
5148 t_pg
= tcg_temp_new_ptr();
5149 tcg_gen_addi_ptr(t_pg
, cpu_env
, poff
);
5151 gen_helper_gvec_mem
*fn
5152 = ldr_fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][dtype
][0];
5153 fn(cpu_env
, t_pg
, addr
, tcg_constant_i32(simd_desc(32, 32, zt
)));
5155 tcg_temp_free_ptr(t_pg
);
5158 * Replicate that first octaword.
5159 * The replication happens in units of 32; if the full vector size
5160 * is not a multiple of 32, the final bits are zeroed.
5162 doff
= vec_full_reg_offset(s
, zt
);
5163 vsz_r32
= QEMU_ALIGN_DOWN(vsz
, 32);
5165 tcg_gen_gvec_dup_mem(5, doff
+ 32, doff
, vsz_r32
- 32, vsz_r32
- 32);
5169 tcg_gen_gvec_dup_imm(MO_64
, doff
+ vsz_r32
, vsz
, vsz
, 0);
5173 static bool trans_LD1RO_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5175 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
5181 if (sve_access_check(s
)) {
5182 TCGv_i64 addr
= new_tmp_a64(s
);
5183 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
5184 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5185 do_ldro(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5190 static bool trans_LD1RO_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5192 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
5195 if (sve_access_check(s
)) {
5196 TCGv_i64 addr
= new_tmp_a64(s
);
5197 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), a
->imm
* 32);
5198 do_ldro(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5203 /* Load and broadcast element. */
5204 static bool trans_LD1R_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5206 unsigned vsz
= vec_full_reg_size(s
);
5207 unsigned psz
= pred_full_reg_size(s
);
5208 unsigned esz
= dtype_esz
[a
->dtype
];
5209 unsigned msz
= dtype_msz(a
->dtype
);
5211 TCGv_i64 temp
, clean_addr
;
5213 if (!sve_access_check(s
)) {
5217 over
= gen_new_label();
5219 /* If the guarding predicate has no bits set, no load occurs. */
5221 /* Reduce the pred_esz_masks value simply to reduce the
5222 * size of the code generated here.
5224 uint64_t psz_mask
= MAKE_64BIT_MASK(0, psz
* 8);
5225 temp
= tcg_temp_new_i64();
5226 tcg_gen_ld_i64(temp
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
5227 tcg_gen_andi_i64(temp
, temp
, pred_esz_masks
[esz
] & psz_mask
);
5228 tcg_gen_brcondi_i64(TCG_COND_EQ
, temp
, 0, over
);
5229 tcg_temp_free_i64(temp
);
5231 TCGv_i32 t32
= tcg_temp_new_i32();
5232 find_last_active(s
, t32
, esz
, a
->pg
);
5233 tcg_gen_brcondi_i32(TCG_COND_LT
, t32
, 0, over
);
5234 tcg_temp_free_i32(t32
);
5237 /* Load the data. */
5238 temp
= tcg_temp_new_i64();
5239 tcg_gen_addi_i64(temp
, cpu_reg_sp(s
, a
->rn
), a
->imm
<< msz
);
5240 clean_addr
= gen_mte_check1(s
, temp
, false, true, msz
);
5242 tcg_gen_qemu_ld_i64(temp
, clean_addr
, get_mem_index(s
),
5243 finalize_memop(s
, dtype_mop
[a
->dtype
]));
5245 /* Broadcast to *all* elements. */
5246 tcg_gen_gvec_dup_i64(esz
, vec_full_reg_offset(s
, a
->rd
),
5248 tcg_temp_free_i64(temp
);
5250 /* Zero the inactive elements. */
5251 gen_set_label(over
);
5252 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, esz
, false);
5255 static void do_st_zpa(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
,
5256 int msz
, int esz
, int nreg
)
5258 static gen_helper_gvec_mem
* const fn_single
[2][2][4][4] = {
5259 { { { gen_helper_sve_st1bb_r
,
5260 gen_helper_sve_st1bh_r
,
5261 gen_helper_sve_st1bs_r
,
5262 gen_helper_sve_st1bd_r
},
5264 gen_helper_sve_st1hh_le_r
,
5265 gen_helper_sve_st1hs_le_r
,
5266 gen_helper_sve_st1hd_le_r
},
5268 gen_helper_sve_st1ss_le_r
,
5269 gen_helper_sve_st1sd_le_r
},
5271 gen_helper_sve_st1dd_le_r
} },
5272 { { gen_helper_sve_st1bb_r
,
5273 gen_helper_sve_st1bh_r
,
5274 gen_helper_sve_st1bs_r
,
5275 gen_helper_sve_st1bd_r
},
5277 gen_helper_sve_st1hh_be_r
,
5278 gen_helper_sve_st1hs_be_r
,
5279 gen_helper_sve_st1hd_be_r
},
5281 gen_helper_sve_st1ss_be_r
,
5282 gen_helper_sve_st1sd_be_r
},
5284 gen_helper_sve_st1dd_be_r
} } },
5286 { { { gen_helper_sve_st1bb_r_mte
,
5287 gen_helper_sve_st1bh_r_mte
,
5288 gen_helper_sve_st1bs_r_mte
,
5289 gen_helper_sve_st1bd_r_mte
},
5291 gen_helper_sve_st1hh_le_r_mte
,
5292 gen_helper_sve_st1hs_le_r_mte
,
5293 gen_helper_sve_st1hd_le_r_mte
},
5295 gen_helper_sve_st1ss_le_r_mte
,
5296 gen_helper_sve_st1sd_le_r_mte
},
5298 gen_helper_sve_st1dd_le_r_mte
} },
5299 { { gen_helper_sve_st1bb_r_mte
,
5300 gen_helper_sve_st1bh_r_mte
,
5301 gen_helper_sve_st1bs_r_mte
,
5302 gen_helper_sve_st1bd_r_mte
},
5304 gen_helper_sve_st1hh_be_r_mte
,
5305 gen_helper_sve_st1hs_be_r_mte
,
5306 gen_helper_sve_st1hd_be_r_mte
},
5308 gen_helper_sve_st1ss_be_r_mte
,
5309 gen_helper_sve_st1sd_be_r_mte
},
5311 gen_helper_sve_st1dd_be_r_mte
} } },
5313 static gen_helper_gvec_mem
* const fn_multiple
[2][2][3][4] = {
5314 { { { gen_helper_sve_st2bb_r
,
5315 gen_helper_sve_st2hh_le_r
,
5316 gen_helper_sve_st2ss_le_r
,
5317 gen_helper_sve_st2dd_le_r
},
5318 { gen_helper_sve_st3bb_r
,
5319 gen_helper_sve_st3hh_le_r
,
5320 gen_helper_sve_st3ss_le_r
,
5321 gen_helper_sve_st3dd_le_r
},
5322 { gen_helper_sve_st4bb_r
,
5323 gen_helper_sve_st4hh_le_r
,
5324 gen_helper_sve_st4ss_le_r
,
5325 gen_helper_sve_st4dd_le_r
} },
5326 { { gen_helper_sve_st2bb_r
,
5327 gen_helper_sve_st2hh_be_r
,
5328 gen_helper_sve_st2ss_be_r
,
5329 gen_helper_sve_st2dd_be_r
},
5330 { gen_helper_sve_st3bb_r
,
5331 gen_helper_sve_st3hh_be_r
,
5332 gen_helper_sve_st3ss_be_r
,
5333 gen_helper_sve_st3dd_be_r
},
5334 { gen_helper_sve_st4bb_r
,
5335 gen_helper_sve_st4hh_be_r
,
5336 gen_helper_sve_st4ss_be_r
,
5337 gen_helper_sve_st4dd_be_r
} } },
5338 { { { gen_helper_sve_st2bb_r_mte
,
5339 gen_helper_sve_st2hh_le_r_mte
,
5340 gen_helper_sve_st2ss_le_r_mte
,
5341 gen_helper_sve_st2dd_le_r_mte
},
5342 { gen_helper_sve_st3bb_r_mte
,
5343 gen_helper_sve_st3hh_le_r_mte
,
5344 gen_helper_sve_st3ss_le_r_mte
,
5345 gen_helper_sve_st3dd_le_r_mte
},
5346 { gen_helper_sve_st4bb_r_mte
,
5347 gen_helper_sve_st4hh_le_r_mte
,
5348 gen_helper_sve_st4ss_le_r_mte
,
5349 gen_helper_sve_st4dd_le_r_mte
} },
5350 { { gen_helper_sve_st2bb_r_mte
,
5351 gen_helper_sve_st2hh_be_r_mte
,
5352 gen_helper_sve_st2ss_be_r_mte
,
5353 gen_helper_sve_st2dd_be_r_mte
},
5354 { gen_helper_sve_st3bb_r_mte
,
5355 gen_helper_sve_st3hh_be_r_mte
,
5356 gen_helper_sve_st3ss_be_r_mte
,
5357 gen_helper_sve_st3dd_be_r_mte
},
5358 { gen_helper_sve_st4bb_r_mte
,
5359 gen_helper_sve_st4hh_be_r_mte
,
5360 gen_helper_sve_st4ss_be_r_mte
,
5361 gen_helper_sve_st4dd_be_r_mte
} } },
5363 gen_helper_gvec_mem
*fn
;
5364 int be
= s
->be_data
== MO_BE
;
5368 fn
= fn_single
[s
->mte_active
[0]][be
][msz
][esz
];
5371 /* ST2, ST3, ST4 -- msz == esz, enforced by encoding */
5373 fn
= fn_multiple
[s
->mte_active
[0]][be
][nreg
- 1][msz
];
5376 do_mem_zpa(s
, zt
, pg
, addr
, msz_dtype(s
, msz
), nreg
, true, fn
);
5379 static bool trans_ST_zprr(DisasContext
*s
, arg_rprr_store
*a
)
5381 if (a
->rm
== 31 || a
->msz
> a
->esz
) {
5384 if (sve_access_check(s
)) {
5385 TCGv_i64 addr
= new_tmp_a64(s
);
5386 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), a
->msz
);
5387 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5388 do_st_zpa(s
, a
->rd
, a
->pg
, addr
, a
->msz
, a
->esz
, a
->nreg
);
5393 static bool trans_ST_zpri(DisasContext
*s
, arg_rpri_store
*a
)
5395 if (a
->msz
> a
->esz
) {
5398 if (sve_access_check(s
)) {
5399 int vsz
= vec_full_reg_size(s
);
5400 int elements
= vsz
>> a
->esz
;
5401 TCGv_i64 addr
= new_tmp_a64(s
);
5403 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
),
5404 (a
->imm
* elements
* (a
->nreg
+ 1)) << a
->msz
);
5405 do_st_zpa(s
, a
->rd
, a
->pg
, addr
, a
->msz
, a
->esz
, a
->nreg
);
5411 *** SVE gather loads / scatter stores
5414 static void do_mem_zpz(DisasContext
*s
, int zt
, int pg
, int zm
,
5415 int scale
, TCGv_i64 scalar
, int msz
, bool is_write
,
5416 gen_helper_gvec_mem_scatter
*fn
)
5418 unsigned vsz
= vec_full_reg_size(s
);
5419 TCGv_ptr t_zm
= tcg_temp_new_ptr();
5420 TCGv_ptr t_pg
= tcg_temp_new_ptr();
5421 TCGv_ptr t_zt
= tcg_temp_new_ptr();
5424 if (s
->mte_active
[0]) {
5425 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, get_mem_index(s
));
5426 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
5427 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
5428 desc
= FIELD_DP32(desc
, MTEDESC
, WRITE
, is_write
);
5429 desc
= FIELD_DP32(desc
, MTEDESC
, SIZEM1
, (1 << msz
) - 1);
5430 desc
<<= SVE_MTEDESC_SHIFT
;
5432 desc
= simd_desc(vsz
, vsz
, desc
| scale
);
5434 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
5435 tcg_gen_addi_ptr(t_zm
, cpu_env
, vec_full_reg_offset(s
, zm
));
5436 tcg_gen_addi_ptr(t_zt
, cpu_env
, vec_full_reg_offset(s
, zt
));
5437 fn(cpu_env
, t_zt
, t_pg
, t_zm
, scalar
, tcg_constant_i32(desc
));
5439 tcg_temp_free_ptr(t_zt
);
5440 tcg_temp_free_ptr(t_zm
);
5441 tcg_temp_free_ptr(t_pg
);
5444 /* Indexed by [mte][be][ff][xs][u][msz]. */
5445 static gen_helper_gvec_mem_scatter
* const
5446 gather_load_fn32
[2][2][2][2][2][3] = {
5447 { /* MTE Inactive */
5448 { /* Little-endian */
5449 { { { gen_helper_sve_ldbss_zsu
,
5450 gen_helper_sve_ldhss_le_zsu
,
5452 { gen_helper_sve_ldbsu_zsu
,
5453 gen_helper_sve_ldhsu_le_zsu
,
5454 gen_helper_sve_ldss_le_zsu
, } },
5455 { { gen_helper_sve_ldbss_zss
,
5456 gen_helper_sve_ldhss_le_zss
,
5458 { gen_helper_sve_ldbsu_zss
,
5459 gen_helper_sve_ldhsu_le_zss
,
5460 gen_helper_sve_ldss_le_zss
, } } },
5463 { { { gen_helper_sve_ldffbss_zsu
,
5464 gen_helper_sve_ldffhss_le_zsu
,
5466 { gen_helper_sve_ldffbsu_zsu
,
5467 gen_helper_sve_ldffhsu_le_zsu
,
5468 gen_helper_sve_ldffss_le_zsu
, } },
5469 { { gen_helper_sve_ldffbss_zss
,
5470 gen_helper_sve_ldffhss_le_zss
,
5472 { gen_helper_sve_ldffbsu_zss
,
5473 gen_helper_sve_ldffhsu_le_zss
,
5474 gen_helper_sve_ldffss_le_zss
, } } } },
5477 { { { gen_helper_sve_ldbss_zsu
,
5478 gen_helper_sve_ldhss_be_zsu
,
5480 { gen_helper_sve_ldbsu_zsu
,
5481 gen_helper_sve_ldhsu_be_zsu
,
5482 gen_helper_sve_ldss_be_zsu
, } },
5483 { { gen_helper_sve_ldbss_zss
,
5484 gen_helper_sve_ldhss_be_zss
,
5486 { gen_helper_sve_ldbsu_zss
,
5487 gen_helper_sve_ldhsu_be_zss
,
5488 gen_helper_sve_ldss_be_zss
, } } },
5491 { { { gen_helper_sve_ldffbss_zsu
,
5492 gen_helper_sve_ldffhss_be_zsu
,
5494 { gen_helper_sve_ldffbsu_zsu
,
5495 gen_helper_sve_ldffhsu_be_zsu
,
5496 gen_helper_sve_ldffss_be_zsu
, } },
5497 { { gen_helper_sve_ldffbss_zss
,
5498 gen_helper_sve_ldffhss_be_zss
,
5500 { gen_helper_sve_ldffbsu_zss
,
5501 gen_helper_sve_ldffhsu_be_zss
,
5502 gen_helper_sve_ldffss_be_zss
, } } } } },
5504 { /* Little-endian */
5505 { { { gen_helper_sve_ldbss_zsu_mte
,
5506 gen_helper_sve_ldhss_le_zsu_mte
,
5508 { gen_helper_sve_ldbsu_zsu_mte
,
5509 gen_helper_sve_ldhsu_le_zsu_mte
,
5510 gen_helper_sve_ldss_le_zsu_mte
, } },
5511 { { gen_helper_sve_ldbss_zss_mte
,
5512 gen_helper_sve_ldhss_le_zss_mte
,
5514 { gen_helper_sve_ldbsu_zss_mte
,
5515 gen_helper_sve_ldhsu_le_zss_mte
,
5516 gen_helper_sve_ldss_le_zss_mte
, } } },
5519 { { { gen_helper_sve_ldffbss_zsu_mte
,
5520 gen_helper_sve_ldffhss_le_zsu_mte
,
5522 { gen_helper_sve_ldffbsu_zsu_mte
,
5523 gen_helper_sve_ldffhsu_le_zsu_mte
,
5524 gen_helper_sve_ldffss_le_zsu_mte
, } },
5525 { { gen_helper_sve_ldffbss_zss_mte
,
5526 gen_helper_sve_ldffhss_le_zss_mte
,
5528 { gen_helper_sve_ldffbsu_zss_mte
,
5529 gen_helper_sve_ldffhsu_le_zss_mte
,
5530 gen_helper_sve_ldffss_le_zss_mte
, } } } },
5533 { { { gen_helper_sve_ldbss_zsu_mte
,
5534 gen_helper_sve_ldhss_be_zsu_mte
,
5536 { gen_helper_sve_ldbsu_zsu_mte
,
5537 gen_helper_sve_ldhsu_be_zsu_mte
,
5538 gen_helper_sve_ldss_be_zsu_mte
, } },
5539 { { gen_helper_sve_ldbss_zss_mte
,
5540 gen_helper_sve_ldhss_be_zss_mte
,
5542 { gen_helper_sve_ldbsu_zss_mte
,
5543 gen_helper_sve_ldhsu_be_zss_mte
,
5544 gen_helper_sve_ldss_be_zss_mte
, } } },
5547 { { { gen_helper_sve_ldffbss_zsu_mte
,
5548 gen_helper_sve_ldffhss_be_zsu_mte
,
5550 { gen_helper_sve_ldffbsu_zsu_mte
,
5551 gen_helper_sve_ldffhsu_be_zsu_mte
,
5552 gen_helper_sve_ldffss_be_zsu_mte
, } },
5553 { { gen_helper_sve_ldffbss_zss_mte
,
5554 gen_helper_sve_ldffhss_be_zss_mte
,
5556 { gen_helper_sve_ldffbsu_zss_mte
,
5557 gen_helper_sve_ldffhsu_be_zss_mte
,
5558 gen_helper_sve_ldffss_be_zss_mte
, } } } } },
5561 /* Note that we overload xs=2 to indicate 64-bit offset. */
5562 static gen_helper_gvec_mem_scatter
* const
5563 gather_load_fn64
[2][2][2][3][2][4] = {
5564 { /* MTE Inactive */
5565 { /* Little-endian */
5566 { { { gen_helper_sve_ldbds_zsu
,
5567 gen_helper_sve_ldhds_le_zsu
,
5568 gen_helper_sve_ldsds_le_zsu
,
5570 { gen_helper_sve_ldbdu_zsu
,
5571 gen_helper_sve_ldhdu_le_zsu
,
5572 gen_helper_sve_ldsdu_le_zsu
,
5573 gen_helper_sve_lddd_le_zsu
, } },
5574 { { gen_helper_sve_ldbds_zss
,
5575 gen_helper_sve_ldhds_le_zss
,
5576 gen_helper_sve_ldsds_le_zss
,
5578 { gen_helper_sve_ldbdu_zss
,
5579 gen_helper_sve_ldhdu_le_zss
,
5580 gen_helper_sve_ldsdu_le_zss
,
5581 gen_helper_sve_lddd_le_zss
, } },
5582 { { gen_helper_sve_ldbds_zd
,
5583 gen_helper_sve_ldhds_le_zd
,
5584 gen_helper_sve_ldsds_le_zd
,
5586 { gen_helper_sve_ldbdu_zd
,
5587 gen_helper_sve_ldhdu_le_zd
,
5588 gen_helper_sve_ldsdu_le_zd
,
5589 gen_helper_sve_lddd_le_zd
, } } },
5592 { { { gen_helper_sve_ldffbds_zsu
,
5593 gen_helper_sve_ldffhds_le_zsu
,
5594 gen_helper_sve_ldffsds_le_zsu
,
5596 { gen_helper_sve_ldffbdu_zsu
,
5597 gen_helper_sve_ldffhdu_le_zsu
,
5598 gen_helper_sve_ldffsdu_le_zsu
,
5599 gen_helper_sve_ldffdd_le_zsu
, } },
5600 { { gen_helper_sve_ldffbds_zss
,
5601 gen_helper_sve_ldffhds_le_zss
,
5602 gen_helper_sve_ldffsds_le_zss
,
5604 { gen_helper_sve_ldffbdu_zss
,
5605 gen_helper_sve_ldffhdu_le_zss
,
5606 gen_helper_sve_ldffsdu_le_zss
,
5607 gen_helper_sve_ldffdd_le_zss
, } },
5608 { { gen_helper_sve_ldffbds_zd
,
5609 gen_helper_sve_ldffhds_le_zd
,
5610 gen_helper_sve_ldffsds_le_zd
,
5612 { gen_helper_sve_ldffbdu_zd
,
5613 gen_helper_sve_ldffhdu_le_zd
,
5614 gen_helper_sve_ldffsdu_le_zd
,
5615 gen_helper_sve_ldffdd_le_zd
, } } } },
5617 { { { gen_helper_sve_ldbds_zsu
,
5618 gen_helper_sve_ldhds_be_zsu
,
5619 gen_helper_sve_ldsds_be_zsu
,
5621 { gen_helper_sve_ldbdu_zsu
,
5622 gen_helper_sve_ldhdu_be_zsu
,
5623 gen_helper_sve_ldsdu_be_zsu
,
5624 gen_helper_sve_lddd_be_zsu
, } },
5625 { { gen_helper_sve_ldbds_zss
,
5626 gen_helper_sve_ldhds_be_zss
,
5627 gen_helper_sve_ldsds_be_zss
,
5629 { gen_helper_sve_ldbdu_zss
,
5630 gen_helper_sve_ldhdu_be_zss
,
5631 gen_helper_sve_ldsdu_be_zss
,
5632 gen_helper_sve_lddd_be_zss
, } },
5633 { { gen_helper_sve_ldbds_zd
,
5634 gen_helper_sve_ldhds_be_zd
,
5635 gen_helper_sve_ldsds_be_zd
,
5637 { gen_helper_sve_ldbdu_zd
,
5638 gen_helper_sve_ldhdu_be_zd
,
5639 gen_helper_sve_ldsdu_be_zd
,
5640 gen_helper_sve_lddd_be_zd
, } } },
5643 { { { gen_helper_sve_ldffbds_zsu
,
5644 gen_helper_sve_ldffhds_be_zsu
,
5645 gen_helper_sve_ldffsds_be_zsu
,
5647 { gen_helper_sve_ldffbdu_zsu
,
5648 gen_helper_sve_ldffhdu_be_zsu
,
5649 gen_helper_sve_ldffsdu_be_zsu
,
5650 gen_helper_sve_ldffdd_be_zsu
, } },
5651 { { gen_helper_sve_ldffbds_zss
,
5652 gen_helper_sve_ldffhds_be_zss
,
5653 gen_helper_sve_ldffsds_be_zss
,
5655 { gen_helper_sve_ldffbdu_zss
,
5656 gen_helper_sve_ldffhdu_be_zss
,
5657 gen_helper_sve_ldffsdu_be_zss
,
5658 gen_helper_sve_ldffdd_be_zss
, } },
5659 { { gen_helper_sve_ldffbds_zd
,
5660 gen_helper_sve_ldffhds_be_zd
,
5661 gen_helper_sve_ldffsds_be_zd
,
5663 { gen_helper_sve_ldffbdu_zd
,
5664 gen_helper_sve_ldffhdu_be_zd
,
5665 gen_helper_sve_ldffsdu_be_zd
,
5666 gen_helper_sve_ldffdd_be_zd
, } } } } },
5668 { /* Little-endian */
5669 { { { gen_helper_sve_ldbds_zsu_mte
,
5670 gen_helper_sve_ldhds_le_zsu_mte
,
5671 gen_helper_sve_ldsds_le_zsu_mte
,
5673 { gen_helper_sve_ldbdu_zsu_mte
,
5674 gen_helper_sve_ldhdu_le_zsu_mte
,
5675 gen_helper_sve_ldsdu_le_zsu_mte
,
5676 gen_helper_sve_lddd_le_zsu_mte
, } },
5677 { { gen_helper_sve_ldbds_zss_mte
,
5678 gen_helper_sve_ldhds_le_zss_mte
,
5679 gen_helper_sve_ldsds_le_zss_mte
,
5681 { gen_helper_sve_ldbdu_zss_mte
,
5682 gen_helper_sve_ldhdu_le_zss_mte
,
5683 gen_helper_sve_ldsdu_le_zss_mte
,
5684 gen_helper_sve_lddd_le_zss_mte
, } },
5685 { { gen_helper_sve_ldbds_zd_mte
,
5686 gen_helper_sve_ldhds_le_zd_mte
,
5687 gen_helper_sve_ldsds_le_zd_mte
,
5689 { gen_helper_sve_ldbdu_zd_mte
,
5690 gen_helper_sve_ldhdu_le_zd_mte
,
5691 gen_helper_sve_ldsdu_le_zd_mte
,
5692 gen_helper_sve_lddd_le_zd_mte
, } } },
5695 { { { gen_helper_sve_ldffbds_zsu_mte
,
5696 gen_helper_sve_ldffhds_le_zsu_mte
,
5697 gen_helper_sve_ldffsds_le_zsu_mte
,
5699 { gen_helper_sve_ldffbdu_zsu_mte
,
5700 gen_helper_sve_ldffhdu_le_zsu_mte
,
5701 gen_helper_sve_ldffsdu_le_zsu_mte
,
5702 gen_helper_sve_ldffdd_le_zsu_mte
, } },
5703 { { gen_helper_sve_ldffbds_zss_mte
,
5704 gen_helper_sve_ldffhds_le_zss_mte
,
5705 gen_helper_sve_ldffsds_le_zss_mte
,
5707 { gen_helper_sve_ldffbdu_zss_mte
,
5708 gen_helper_sve_ldffhdu_le_zss_mte
,
5709 gen_helper_sve_ldffsdu_le_zss_mte
,
5710 gen_helper_sve_ldffdd_le_zss_mte
, } },
5711 { { gen_helper_sve_ldffbds_zd_mte
,
5712 gen_helper_sve_ldffhds_le_zd_mte
,
5713 gen_helper_sve_ldffsds_le_zd_mte
,
5715 { gen_helper_sve_ldffbdu_zd_mte
,
5716 gen_helper_sve_ldffhdu_le_zd_mte
,
5717 gen_helper_sve_ldffsdu_le_zd_mte
,
5718 gen_helper_sve_ldffdd_le_zd_mte
, } } } },
5720 { { { gen_helper_sve_ldbds_zsu_mte
,
5721 gen_helper_sve_ldhds_be_zsu_mte
,
5722 gen_helper_sve_ldsds_be_zsu_mte
,
5724 { gen_helper_sve_ldbdu_zsu_mte
,
5725 gen_helper_sve_ldhdu_be_zsu_mte
,
5726 gen_helper_sve_ldsdu_be_zsu_mte
,
5727 gen_helper_sve_lddd_be_zsu_mte
, } },
5728 { { gen_helper_sve_ldbds_zss_mte
,
5729 gen_helper_sve_ldhds_be_zss_mte
,
5730 gen_helper_sve_ldsds_be_zss_mte
,
5732 { gen_helper_sve_ldbdu_zss_mte
,
5733 gen_helper_sve_ldhdu_be_zss_mte
,
5734 gen_helper_sve_ldsdu_be_zss_mte
,
5735 gen_helper_sve_lddd_be_zss_mte
, } },
5736 { { gen_helper_sve_ldbds_zd_mte
,
5737 gen_helper_sve_ldhds_be_zd_mte
,
5738 gen_helper_sve_ldsds_be_zd_mte
,
5740 { gen_helper_sve_ldbdu_zd_mte
,
5741 gen_helper_sve_ldhdu_be_zd_mte
,
5742 gen_helper_sve_ldsdu_be_zd_mte
,
5743 gen_helper_sve_lddd_be_zd_mte
, } } },
5746 { { { gen_helper_sve_ldffbds_zsu_mte
,
5747 gen_helper_sve_ldffhds_be_zsu_mte
,
5748 gen_helper_sve_ldffsds_be_zsu_mte
,
5750 { gen_helper_sve_ldffbdu_zsu_mte
,
5751 gen_helper_sve_ldffhdu_be_zsu_mte
,
5752 gen_helper_sve_ldffsdu_be_zsu_mte
,
5753 gen_helper_sve_ldffdd_be_zsu_mte
, } },
5754 { { gen_helper_sve_ldffbds_zss_mte
,
5755 gen_helper_sve_ldffhds_be_zss_mte
,
5756 gen_helper_sve_ldffsds_be_zss_mte
,
5758 { gen_helper_sve_ldffbdu_zss_mte
,
5759 gen_helper_sve_ldffhdu_be_zss_mte
,
5760 gen_helper_sve_ldffsdu_be_zss_mte
,
5761 gen_helper_sve_ldffdd_be_zss_mte
, } },
5762 { { gen_helper_sve_ldffbds_zd_mte
,
5763 gen_helper_sve_ldffhds_be_zd_mte
,
5764 gen_helper_sve_ldffsds_be_zd_mte
,
5766 { gen_helper_sve_ldffbdu_zd_mte
,
5767 gen_helper_sve_ldffhdu_be_zd_mte
,
5768 gen_helper_sve_ldffsdu_be_zd_mte
,
5769 gen_helper_sve_ldffdd_be_zd_mte
, } } } } },
5772 static bool trans_LD1_zprz(DisasContext
*s
, arg_LD1_zprz
*a
)
5774 gen_helper_gvec_mem_scatter
*fn
= NULL
;
5775 bool be
= s
->be_data
== MO_BE
;
5776 bool mte
= s
->mte_active
[0];
5778 if (!sve_access_check(s
)) {
5784 fn
= gather_load_fn32
[mte
][be
][a
->ff
][a
->xs
][a
->u
][a
->msz
];
5787 fn
= gather_load_fn64
[mte
][be
][a
->ff
][a
->xs
][a
->u
][a
->msz
];
5792 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rm
, a
->scale
* a
->msz
,
5793 cpu_reg_sp(s
, a
->rn
), a
->msz
, false, fn
);
5797 static bool trans_LD1_zpiz(DisasContext
*s
, arg_LD1_zpiz
*a
)
5799 gen_helper_gvec_mem_scatter
*fn
= NULL
;
5800 bool be
= s
->be_data
== MO_BE
;
5801 bool mte
= s
->mte_active
[0];
5803 if (a
->esz
< a
->msz
|| (a
->esz
== a
->msz
&& !a
->u
)) {
5806 if (!sve_access_check(s
)) {
5812 fn
= gather_load_fn32
[mte
][be
][a
->ff
][0][a
->u
][a
->msz
];
5815 fn
= gather_load_fn64
[mte
][be
][a
->ff
][2][a
->u
][a
->msz
];
5820 /* Treat LD1_zpiz (zn[x] + imm) the same way as LD1_zprz (rn + zm[x])
5821 * by loading the immediate into the scalar parameter.
5823 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0,
5824 tcg_constant_i64(a
->imm
<< a
->msz
), a
->msz
, false, fn
);
5828 static bool trans_LDNT1_zprz(DisasContext
*s
, arg_LD1_zprz
*a
)
5830 gen_helper_gvec_mem_scatter
*fn
= NULL
;
5831 bool be
= s
->be_data
== MO_BE
;
5832 bool mte
= s
->mte_active
[0];
5834 if (a
->esz
< a
->msz
+ !a
->u
) {
5837 if (!dc_isar_feature(aa64_sve2
, s
)) {
5840 if (!sve_access_check(s
)) {
5846 fn
= gather_load_fn32
[mte
][be
][0][0][a
->u
][a
->msz
];
5849 fn
= gather_load_fn64
[mte
][be
][0][2][a
->u
][a
->msz
];
5854 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0,
5855 cpu_reg(s
, a
->rm
), a
->msz
, false, fn
);
5859 /* Indexed by [mte][be][xs][msz]. */
5860 static gen_helper_gvec_mem_scatter
* const scatter_store_fn32
[2][2][2][3] = {
5861 { /* MTE Inactive */
5862 { /* Little-endian */
5863 { gen_helper_sve_stbs_zsu
,
5864 gen_helper_sve_sths_le_zsu
,
5865 gen_helper_sve_stss_le_zsu
, },
5866 { gen_helper_sve_stbs_zss
,
5867 gen_helper_sve_sths_le_zss
,
5868 gen_helper_sve_stss_le_zss
, } },
5870 { gen_helper_sve_stbs_zsu
,
5871 gen_helper_sve_sths_be_zsu
,
5872 gen_helper_sve_stss_be_zsu
, },
5873 { gen_helper_sve_stbs_zss
,
5874 gen_helper_sve_sths_be_zss
,
5875 gen_helper_sve_stss_be_zss
, } } },
5877 { /* Little-endian */
5878 { gen_helper_sve_stbs_zsu_mte
,
5879 gen_helper_sve_sths_le_zsu_mte
,
5880 gen_helper_sve_stss_le_zsu_mte
, },
5881 { gen_helper_sve_stbs_zss_mte
,
5882 gen_helper_sve_sths_le_zss_mte
,
5883 gen_helper_sve_stss_le_zss_mte
, } },
5885 { gen_helper_sve_stbs_zsu_mte
,
5886 gen_helper_sve_sths_be_zsu_mte
,
5887 gen_helper_sve_stss_be_zsu_mte
, },
5888 { gen_helper_sve_stbs_zss_mte
,
5889 gen_helper_sve_sths_be_zss_mte
,
5890 gen_helper_sve_stss_be_zss_mte
, } } },
5893 /* Note that we overload xs=2 to indicate 64-bit offset. */
5894 static gen_helper_gvec_mem_scatter
* const scatter_store_fn64
[2][2][3][4] = {
5895 { /* MTE Inactive */
5896 { /* Little-endian */
5897 { gen_helper_sve_stbd_zsu
,
5898 gen_helper_sve_sthd_le_zsu
,
5899 gen_helper_sve_stsd_le_zsu
,
5900 gen_helper_sve_stdd_le_zsu
, },
5901 { gen_helper_sve_stbd_zss
,
5902 gen_helper_sve_sthd_le_zss
,
5903 gen_helper_sve_stsd_le_zss
,
5904 gen_helper_sve_stdd_le_zss
, },
5905 { gen_helper_sve_stbd_zd
,
5906 gen_helper_sve_sthd_le_zd
,
5907 gen_helper_sve_stsd_le_zd
,
5908 gen_helper_sve_stdd_le_zd
, } },
5910 { gen_helper_sve_stbd_zsu
,
5911 gen_helper_sve_sthd_be_zsu
,
5912 gen_helper_sve_stsd_be_zsu
,
5913 gen_helper_sve_stdd_be_zsu
, },
5914 { gen_helper_sve_stbd_zss
,
5915 gen_helper_sve_sthd_be_zss
,
5916 gen_helper_sve_stsd_be_zss
,
5917 gen_helper_sve_stdd_be_zss
, },
5918 { gen_helper_sve_stbd_zd
,
5919 gen_helper_sve_sthd_be_zd
,
5920 gen_helper_sve_stsd_be_zd
,
5921 gen_helper_sve_stdd_be_zd
, } } },
5922 { /* MTE Inactive */
5923 { /* Little-endian */
5924 { gen_helper_sve_stbd_zsu_mte
,
5925 gen_helper_sve_sthd_le_zsu_mte
,
5926 gen_helper_sve_stsd_le_zsu_mte
,
5927 gen_helper_sve_stdd_le_zsu_mte
, },
5928 { gen_helper_sve_stbd_zss_mte
,
5929 gen_helper_sve_sthd_le_zss_mte
,
5930 gen_helper_sve_stsd_le_zss_mte
,
5931 gen_helper_sve_stdd_le_zss_mte
, },
5932 { gen_helper_sve_stbd_zd_mte
,
5933 gen_helper_sve_sthd_le_zd_mte
,
5934 gen_helper_sve_stsd_le_zd_mte
,
5935 gen_helper_sve_stdd_le_zd_mte
, } },
5937 { gen_helper_sve_stbd_zsu_mte
,
5938 gen_helper_sve_sthd_be_zsu_mte
,
5939 gen_helper_sve_stsd_be_zsu_mte
,
5940 gen_helper_sve_stdd_be_zsu_mte
, },
5941 { gen_helper_sve_stbd_zss_mte
,
5942 gen_helper_sve_sthd_be_zss_mte
,
5943 gen_helper_sve_stsd_be_zss_mte
,
5944 gen_helper_sve_stdd_be_zss_mte
, },
5945 { gen_helper_sve_stbd_zd_mte
,
5946 gen_helper_sve_sthd_be_zd_mte
,
5947 gen_helper_sve_stsd_be_zd_mte
,
5948 gen_helper_sve_stdd_be_zd_mte
, } } },
5951 static bool trans_ST1_zprz(DisasContext
*s
, arg_ST1_zprz
*a
)
5953 gen_helper_gvec_mem_scatter
*fn
;
5954 bool be
= s
->be_data
== MO_BE
;
5955 bool mte
= s
->mte_active
[0];
5957 if (a
->esz
< a
->msz
|| (a
->msz
== 0 && a
->scale
)) {
5960 if (!sve_access_check(s
)) {
5965 fn
= scatter_store_fn32
[mte
][be
][a
->xs
][a
->msz
];
5968 fn
= scatter_store_fn64
[mte
][be
][a
->xs
][a
->msz
];
5971 g_assert_not_reached();
5973 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rm
, a
->scale
* a
->msz
,
5974 cpu_reg_sp(s
, a
->rn
), a
->msz
, true, fn
);
5978 static bool trans_ST1_zpiz(DisasContext
*s
, arg_ST1_zpiz
*a
)
5980 gen_helper_gvec_mem_scatter
*fn
= NULL
;
5981 bool be
= s
->be_data
== MO_BE
;
5982 bool mte
= s
->mte_active
[0];
5984 if (a
->esz
< a
->msz
) {
5987 if (!sve_access_check(s
)) {
5993 fn
= scatter_store_fn32
[mte
][be
][0][a
->msz
];
5996 fn
= scatter_store_fn64
[mte
][be
][2][a
->msz
];
6001 /* Treat ST1_zpiz (zn[x] + imm) the same way as ST1_zprz (rn + zm[x])
6002 * by loading the immediate into the scalar parameter.
6004 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0,
6005 tcg_constant_i64(a
->imm
<< a
->msz
), a
->msz
, true, fn
);
6009 static bool trans_STNT1_zprz(DisasContext
*s
, arg_ST1_zprz
*a
)
6011 gen_helper_gvec_mem_scatter
*fn
;
6012 bool be
= s
->be_data
== MO_BE
;
6013 bool mte
= s
->mte_active
[0];
6015 if (a
->esz
< a
->msz
) {
6018 if (!dc_isar_feature(aa64_sve2
, s
)) {
6021 if (!sve_access_check(s
)) {
6027 fn
= scatter_store_fn32
[mte
][be
][0][a
->msz
];
6030 fn
= scatter_store_fn64
[mte
][be
][2][a
->msz
];
6033 g_assert_not_reached();
6036 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0,
6037 cpu_reg(s
, a
->rm
), a
->msz
, true, fn
);
6045 static bool trans_PRF(DisasContext
*s
, arg_PRF
*a
)
6047 /* Prefetch is a nop within QEMU. */
6048 (void)sve_access_check(s
);
6052 static bool trans_PRF_rr(DisasContext
*s
, arg_PRF_rr
*a
)
6057 /* Prefetch is a nop within QEMU. */
6058 (void)sve_access_check(s
);
6065 * TODO: The implementation so far could handle predicated merging movprfx.
6066 * The helper functions as written take an extra source register to
6067 * use in the operation, but the result is only written when predication
6068 * succeeds. For unpredicated movprfx, we need to rearrange the helpers
6069 * to allow the final write back to the destination to be unconditional.
6070 * For predicated zeroing movprfx, we need to rearrange the helpers to
6071 * allow the final write back to zero inactives.
6073 * In the meantime, just emit the moves.
6076 static bool trans_MOVPRFX(DisasContext
*s
, arg_MOVPRFX
*a
)
6078 return do_mov_z(s
, a
->rd
, a
->rn
);
6081 static bool trans_MOVPRFX_m(DisasContext
*s
, arg_rpr_esz
*a
)
6083 return do_sel_z(s
, a
->rd
, a
->rn
, a
->rd
, a
->pg
, a
->esz
);
6086 static bool trans_MOVPRFX_z(DisasContext
*s
, arg_rpr_esz
*a
)
6088 return do_movz_zpz(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
, false);
6092 * SVE2 Integer Multiply - Unpredicated
6095 TRANS_FEAT(MUL_zzz
, aa64_sve2
, gen_gvec_fn_arg_zzz
, tcg_gen_gvec_mul
, a
)
6097 static gen_helper_gvec_3
* const smulh_zzz_fns
[4] = {
6098 gen_helper_gvec_smulh_b
, gen_helper_gvec_smulh_h
,
6099 gen_helper_gvec_smulh_s
, gen_helper_gvec_smulh_d
,
6101 TRANS_FEAT(SMULH_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6102 smulh_zzz_fns
[a
->esz
], a
, 0)
6104 static gen_helper_gvec_3
* const umulh_zzz_fns
[4] = {
6105 gen_helper_gvec_umulh_b
, gen_helper_gvec_umulh_h
,
6106 gen_helper_gvec_umulh_s
, gen_helper_gvec_umulh_d
,
6108 TRANS_FEAT(UMULH_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6109 umulh_zzz_fns
[a
->esz
], a
, 0)
6111 TRANS_FEAT(PMUL_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6112 gen_helper_gvec_pmul_b
, a
, 0)
6114 static gen_helper_gvec_3
* const sqdmulh_zzz_fns
[4] = {
6115 gen_helper_sve2_sqdmulh_b
, gen_helper_sve2_sqdmulh_h
,
6116 gen_helper_sve2_sqdmulh_s
, gen_helper_sve2_sqdmulh_d
,
6118 TRANS_FEAT(SQDMULH_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6119 sqdmulh_zzz_fns
[a
->esz
], a
, 0)
6121 static gen_helper_gvec_3
* const sqrdmulh_zzz_fns
[4] = {
6122 gen_helper_sve2_sqrdmulh_b
, gen_helper_sve2_sqrdmulh_h
,
6123 gen_helper_sve2_sqrdmulh_s
, gen_helper_sve2_sqrdmulh_d
,
6125 TRANS_FEAT(SQRDMULH_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6126 sqrdmulh_zzz_fns
[a
->esz
], a
, 0)
6129 * SVE2 Integer - Predicated
6132 static gen_helper_gvec_4
* const sadlp_fns
[4] = {
6133 NULL
, gen_helper_sve2_sadalp_zpzz_h
,
6134 gen_helper_sve2_sadalp_zpzz_s
, gen_helper_sve2_sadalp_zpzz_d
,
6136 TRANS_FEAT(SADALP_zpzz
, aa64_sve2
, gen_gvec_ool_arg_zpzz
,
6137 sadlp_fns
[a
->esz
], a
, 0)
6139 static gen_helper_gvec_4
* const uadlp_fns
[4] = {
6140 NULL
, gen_helper_sve2_uadalp_zpzz_h
,
6141 gen_helper_sve2_uadalp_zpzz_s
, gen_helper_sve2_uadalp_zpzz_d
,
6143 TRANS_FEAT(UADALP_zpzz
, aa64_sve2
, gen_gvec_ool_arg_zpzz
,
6144 uadlp_fns
[a
->esz
], a
, 0)
6147 * SVE2 integer unary operations (predicated)
6150 TRANS_FEAT(URECPE
, aa64_sve2
, gen_gvec_ool_arg_zpz
,
6151 a
->esz
== 2 ? gen_helper_sve2_urecpe_s
: NULL
, a
, 0)
6153 TRANS_FEAT(URSQRTE
, aa64_sve2
, gen_gvec_ool_arg_zpz
,
6154 a
->esz
== 2 ? gen_helper_sve2_ursqrte_s
: NULL
, a
, 0)
6156 static gen_helper_gvec_3
* const sqabs_fns
[4] = {
6157 gen_helper_sve2_sqabs_b
, gen_helper_sve2_sqabs_h
,
6158 gen_helper_sve2_sqabs_s
, gen_helper_sve2_sqabs_d
,
6160 TRANS_FEAT(SQABS
, aa64_sve2
, gen_gvec_ool_arg_zpz
, sqabs_fns
[a
->esz
], a
, 0)
6162 static gen_helper_gvec_3
* const sqneg_fns
[4] = {
6163 gen_helper_sve2_sqneg_b
, gen_helper_sve2_sqneg_h
,
6164 gen_helper_sve2_sqneg_s
, gen_helper_sve2_sqneg_d
,
6166 TRANS_FEAT(SQNEG
, aa64_sve2
, gen_gvec_ool_arg_zpz
, sqneg_fns
[a
->esz
], a
, 0)
6168 DO_ZPZZ(SQSHL
, aa64_sve2
, sve2_sqshl
)
6169 DO_ZPZZ(SQRSHL
, aa64_sve2
, sve2_sqrshl
)
6170 DO_ZPZZ(SRSHL
, aa64_sve2
, sve2_srshl
)
6172 DO_ZPZZ(UQSHL
, aa64_sve2
, sve2_uqshl
)
6173 DO_ZPZZ(UQRSHL
, aa64_sve2
, sve2_uqrshl
)
6174 DO_ZPZZ(URSHL
, aa64_sve2
, sve2_urshl
)
6176 DO_ZPZZ(SHADD
, aa64_sve2
, sve2_shadd
)
6177 DO_ZPZZ(SRHADD
, aa64_sve2
, sve2_srhadd
)
6178 DO_ZPZZ(SHSUB
, aa64_sve2
, sve2_shsub
)
6180 DO_ZPZZ(UHADD
, aa64_sve2
, sve2_uhadd
)
6181 DO_ZPZZ(URHADD
, aa64_sve2
, sve2_urhadd
)
6182 DO_ZPZZ(UHSUB
, aa64_sve2
, sve2_uhsub
)
6184 DO_ZPZZ(ADDP
, aa64_sve2
, sve2_addp
)
6185 DO_ZPZZ(SMAXP
, aa64_sve2
, sve2_smaxp
)
6186 DO_ZPZZ(UMAXP
, aa64_sve2
, sve2_umaxp
)
6187 DO_ZPZZ(SMINP
, aa64_sve2
, sve2_sminp
)
6188 DO_ZPZZ(UMINP
, aa64_sve2
, sve2_uminp
)
6190 DO_ZPZZ(SQADD_zpzz
, aa64_sve2
, sve2_sqadd
)
6191 DO_ZPZZ(UQADD_zpzz
, aa64_sve2
, sve2_uqadd
)
6192 DO_ZPZZ(SQSUB_zpzz
, aa64_sve2
, sve2_sqsub
)
6193 DO_ZPZZ(UQSUB_zpzz
, aa64_sve2
, sve2_uqsub
)
6194 DO_ZPZZ(SUQADD
, aa64_sve2
, sve2_suqadd
)
6195 DO_ZPZZ(USQADD
, aa64_sve2
, sve2_usqadd
)
6198 * SVE2 Widening Integer Arithmetic
6201 static gen_helper_gvec_3
* const saddl_fns
[4] = {
6202 NULL
, gen_helper_sve2_saddl_h
,
6203 gen_helper_sve2_saddl_s
, gen_helper_sve2_saddl_d
,
6205 TRANS_FEAT(SADDLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6206 saddl_fns
[a
->esz
], a
, 0)
6207 TRANS_FEAT(SADDLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6208 saddl_fns
[a
->esz
], a
, 3)
6209 TRANS_FEAT(SADDLBT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6210 saddl_fns
[a
->esz
], a
, 2)
6212 static gen_helper_gvec_3
* const ssubl_fns
[4] = {
6213 NULL
, gen_helper_sve2_ssubl_h
,
6214 gen_helper_sve2_ssubl_s
, gen_helper_sve2_ssubl_d
,
6216 TRANS_FEAT(SSUBLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6217 ssubl_fns
[a
->esz
], a
, 0)
6218 TRANS_FEAT(SSUBLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6219 ssubl_fns
[a
->esz
], a
, 3)
6220 TRANS_FEAT(SSUBLBT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6221 ssubl_fns
[a
->esz
], a
, 2)
6222 TRANS_FEAT(SSUBLTB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6223 ssubl_fns
[a
->esz
], a
, 1)
6225 static gen_helper_gvec_3
* const sabdl_fns
[4] = {
6226 NULL
, gen_helper_sve2_sabdl_h
,
6227 gen_helper_sve2_sabdl_s
, gen_helper_sve2_sabdl_d
,
6229 TRANS_FEAT(SABDLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6230 sabdl_fns
[a
->esz
], a
, 0)
6231 TRANS_FEAT(SABDLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6232 sabdl_fns
[a
->esz
], a
, 3)
6234 static gen_helper_gvec_3
* const uaddl_fns
[4] = {
6235 NULL
, gen_helper_sve2_uaddl_h
,
6236 gen_helper_sve2_uaddl_s
, gen_helper_sve2_uaddl_d
,
6238 TRANS_FEAT(UADDLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6239 uaddl_fns
[a
->esz
], a
, 0)
6240 TRANS_FEAT(UADDLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6241 uaddl_fns
[a
->esz
], a
, 3)
6243 static gen_helper_gvec_3
* const usubl_fns
[4] = {
6244 NULL
, gen_helper_sve2_usubl_h
,
6245 gen_helper_sve2_usubl_s
, gen_helper_sve2_usubl_d
,
6247 TRANS_FEAT(USUBLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6248 usubl_fns
[a
->esz
], a
, 0)
6249 TRANS_FEAT(USUBLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6250 usubl_fns
[a
->esz
], a
, 3)
6252 static gen_helper_gvec_3
* const uabdl_fns
[4] = {
6253 NULL
, gen_helper_sve2_uabdl_h
,
6254 gen_helper_sve2_uabdl_s
, gen_helper_sve2_uabdl_d
,
6256 TRANS_FEAT(UABDLB
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6257 uabdl_fns
[a
->esz
], a
, 0)
6258 TRANS_FEAT(UABDLT
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6259 uabdl_fns
[a
->esz
], a
, 3)
6261 static gen_helper_gvec_3
* const sqdmull_fns
[4] = {
6262 NULL
, gen_helper_sve2_sqdmull_zzz_h
,
6263 gen_helper_sve2_sqdmull_zzz_s
, gen_helper_sve2_sqdmull_zzz_d
,
6265 TRANS_FEAT(SQDMULLB_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6266 sqdmull_fns
[a
->esz
], a
, 0)
6267 TRANS_FEAT(SQDMULLT_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6268 sqdmull_fns
[a
->esz
], a
, 3)
6270 static gen_helper_gvec_3
* const smull_fns
[4] = {
6271 NULL
, gen_helper_sve2_smull_zzz_h
,
6272 gen_helper_sve2_smull_zzz_s
, gen_helper_sve2_smull_zzz_d
,
6274 TRANS_FEAT(SMULLB_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6275 smull_fns
[a
->esz
], a
, 0)
6276 TRANS_FEAT(SMULLT_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6277 smull_fns
[a
->esz
], a
, 3)
6279 static gen_helper_gvec_3
* const umull_fns
[4] = {
6280 NULL
, gen_helper_sve2_umull_zzz_h
,
6281 gen_helper_sve2_umull_zzz_s
, gen_helper_sve2_umull_zzz_d
,
6283 TRANS_FEAT(UMULLB_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6284 umull_fns
[a
->esz
], a
, 0)
6285 TRANS_FEAT(UMULLT_zzz
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6286 umull_fns
[a
->esz
], a
, 3)
6288 static gen_helper_gvec_3
* const eoril_fns
[4] = {
6289 gen_helper_sve2_eoril_b
, gen_helper_sve2_eoril_h
,
6290 gen_helper_sve2_eoril_s
, gen_helper_sve2_eoril_d
,
6292 TRANS_FEAT(EORBT
, aa64_sve2
, gen_gvec_ool_arg_zzz
, eoril_fns
[a
->esz
], a
, 2)
6293 TRANS_FEAT(EORTB
, aa64_sve2
, gen_gvec_ool_arg_zzz
, eoril_fns
[a
->esz
], a
, 1)
6295 static bool do_trans_pmull(DisasContext
*s
, arg_rrr_esz
*a
, bool sel
)
6297 static gen_helper_gvec_3
* const fns
[4] = {
6298 gen_helper_gvec_pmull_q
, gen_helper_sve2_pmull_h
,
6299 NULL
, gen_helper_sve2_pmull_d
,
6301 if (a
->esz
== 0 && !dc_isar_feature(aa64_sve2_pmull128
, s
)) {
6304 return gen_gvec_ool_arg_zzz(s
, fns
[a
->esz
], a
, sel
);
6307 TRANS_FEAT(PMULLB
, aa64_sve2
, do_trans_pmull
, a
, false)
6308 TRANS_FEAT(PMULLT
, aa64_sve2
, do_trans_pmull
, a
, true)
6310 static gen_helper_gvec_3
* const saddw_fns
[4] = {
6311 NULL
, gen_helper_sve2_saddw_h
,
6312 gen_helper_sve2_saddw_s
, gen_helper_sve2_saddw_d
,
6314 TRANS_FEAT(SADDWB
, aa64_sve2
, gen_gvec_ool_arg_zzz
, saddw_fns
[a
->esz
], a
, 0)
6315 TRANS_FEAT(SADDWT
, aa64_sve2
, gen_gvec_ool_arg_zzz
, saddw_fns
[a
->esz
], a
, 1)
6317 static gen_helper_gvec_3
* const ssubw_fns
[4] = {
6318 NULL
, gen_helper_sve2_ssubw_h
,
6319 gen_helper_sve2_ssubw_s
, gen_helper_sve2_ssubw_d
,
6321 TRANS_FEAT(SSUBWB
, aa64_sve2
, gen_gvec_ool_arg_zzz
, ssubw_fns
[a
->esz
], a
, 0)
6322 TRANS_FEAT(SSUBWT
, aa64_sve2
, gen_gvec_ool_arg_zzz
, ssubw_fns
[a
->esz
], a
, 1)
6324 static gen_helper_gvec_3
* const uaddw_fns
[4] = {
6325 NULL
, gen_helper_sve2_uaddw_h
,
6326 gen_helper_sve2_uaddw_s
, gen_helper_sve2_uaddw_d
,
6328 TRANS_FEAT(UADDWB
, aa64_sve2
, gen_gvec_ool_arg_zzz
, uaddw_fns
[a
->esz
], a
, 0)
6329 TRANS_FEAT(UADDWT
, aa64_sve2
, gen_gvec_ool_arg_zzz
, uaddw_fns
[a
->esz
], a
, 1)
6331 static gen_helper_gvec_3
* const usubw_fns
[4] = {
6332 NULL
, gen_helper_sve2_usubw_h
,
6333 gen_helper_sve2_usubw_s
, gen_helper_sve2_usubw_d
,
6335 TRANS_FEAT(USUBWB
, aa64_sve2
, gen_gvec_ool_arg_zzz
, usubw_fns
[a
->esz
], a
, 0)
6336 TRANS_FEAT(USUBWT
, aa64_sve2
, gen_gvec_ool_arg_zzz
, usubw_fns
[a
->esz
], a
, 1)
6338 static void gen_sshll_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t imm
)
6342 int halfbits
= 4 << vece
;
6345 if (shl
== halfbits
) {
6346 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6347 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(halfbits
, halfbits
));
6348 tcg_gen_and_vec(vece
, d
, n
, t
);
6349 tcg_temp_free_vec(t
);
6351 tcg_gen_sari_vec(vece
, d
, n
, halfbits
);
6352 tcg_gen_shli_vec(vece
, d
, d
, shl
);
6355 tcg_gen_shli_vec(vece
, d
, n
, halfbits
);
6356 tcg_gen_sari_vec(vece
, d
, d
, halfbits
- shl
);
6360 static void gen_ushll_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int imm
)
6362 int halfbits
= 4 << vece
;
6364 int shl
= (imm
>> 1);
6368 mask
= MAKE_64BIT_MASK(0, halfbits
);
6370 mask
= dup_const(vece
, mask
);
6372 shift
= shl
- top
* halfbits
;
6374 tcg_gen_shri_i64(d
, n
, -shift
);
6376 tcg_gen_shli_i64(d
, n
, shift
);
6378 tcg_gen_andi_i64(d
, d
, mask
);
6381 static void gen_ushll16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6383 gen_ushll_i64(MO_16
, d
, n
, imm
);
6386 static void gen_ushll32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6388 gen_ushll_i64(MO_32
, d
, n
, imm
);
6391 static void gen_ushll64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6393 gen_ushll_i64(MO_64
, d
, n
, imm
);
6396 static void gen_ushll_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t imm
)
6398 int halfbits
= 4 << vece
;
6403 if (shl
== halfbits
) {
6404 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6405 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(halfbits
, halfbits
));
6406 tcg_gen_and_vec(vece
, d
, n
, t
);
6407 tcg_temp_free_vec(t
);
6409 tcg_gen_shri_vec(vece
, d
, n
, halfbits
);
6410 tcg_gen_shli_vec(vece
, d
, d
, shl
);
6414 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6415 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
6416 tcg_gen_and_vec(vece
, d
, n
, t
);
6417 tcg_temp_free_vec(t
);
6419 tcg_gen_shli_vec(vece
, d
, n
, halfbits
);
6420 tcg_gen_shri_vec(vece
, d
, d
, halfbits
- shl
);
6425 static bool do_sve2_shll_tb(DisasContext
*s
, arg_rri_esz
*a
,
6428 static const TCGOpcode sshll_list
[] = {
6429 INDEX_op_shli_vec
, INDEX_op_sari_vec
, 0
6431 static const TCGOpcode ushll_list
[] = {
6432 INDEX_op_shli_vec
, INDEX_op_shri_vec
, 0
6434 static const GVecGen2i ops
[2][3] = {
6435 { { .fniv
= gen_sshll_vec
,
6436 .opt_opc
= sshll_list
,
6437 .fno
= gen_helper_sve2_sshll_h
,
6439 { .fniv
= gen_sshll_vec
,
6440 .opt_opc
= sshll_list
,
6441 .fno
= gen_helper_sve2_sshll_s
,
6443 { .fniv
= gen_sshll_vec
,
6444 .opt_opc
= sshll_list
,
6445 .fno
= gen_helper_sve2_sshll_d
,
6447 { { .fni8
= gen_ushll16_i64
,
6448 .fniv
= gen_ushll_vec
,
6449 .opt_opc
= ushll_list
,
6450 .fno
= gen_helper_sve2_ushll_h
,
6452 { .fni8
= gen_ushll32_i64
,
6453 .fniv
= gen_ushll_vec
,
6454 .opt_opc
= ushll_list
,
6455 .fno
= gen_helper_sve2_ushll_s
,
6457 { .fni8
= gen_ushll64_i64
,
6458 .fniv
= gen_ushll_vec
,
6459 .opt_opc
= ushll_list
,
6460 .fno
= gen_helper_sve2_ushll_d
,
6464 if (a
->esz
< 0 || a
->esz
> 2 || !dc_isar_feature(aa64_sve2
, s
)) {
6467 if (sve_access_check(s
)) {
6468 unsigned vsz
= vec_full_reg_size(s
);
6469 tcg_gen_gvec_2i(vec_full_reg_offset(s
, a
->rd
),
6470 vec_full_reg_offset(s
, a
->rn
),
6471 vsz
, vsz
, (a
->imm
<< 1) | sel
,
6477 static bool trans_SSHLLB(DisasContext
*s
, arg_rri_esz
*a
)
6479 return do_sve2_shll_tb(s
, a
, false, false);
6482 static bool trans_SSHLLT(DisasContext
*s
, arg_rri_esz
*a
)
6484 return do_sve2_shll_tb(s
, a
, true, false);
6487 static bool trans_USHLLB(DisasContext
*s
, arg_rri_esz
*a
)
6489 return do_sve2_shll_tb(s
, a
, false, true);
6492 static bool trans_USHLLT(DisasContext
*s
, arg_rri_esz
*a
)
6494 return do_sve2_shll_tb(s
, a
, true, true);
6497 static gen_helper_gvec_3
* const bext_fns
[4] = {
6498 gen_helper_sve2_bext_b
, gen_helper_sve2_bext_h
,
6499 gen_helper_sve2_bext_s
, gen_helper_sve2_bext_d
,
6501 TRANS_FEAT(BEXT
, aa64_sve2_bitperm
, gen_gvec_ool_arg_zzz
,
6502 bext_fns
[a
->esz
], a
, 0)
6504 static gen_helper_gvec_3
* const bdep_fns
[4] = {
6505 gen_helper_sve2_bdep_b
, gen_helper_sve2_bdep_h
,
6506 gen_helper_sve2_bdep_s
, gen_helper_sve2_bdep_d
,
6508 TRANS_FEAT(BDEP
, aa64_sve2_bitperm
, gen_gvec_ool_arg_zzz
,
6509 bdep_fns
[a
->esz
], a
, 0)
6511 static gen_helper_gvec_3
* const bgrp_fns
[4] = {
6512 gen_helper_sve2_bgrp_b
, gen_helper_sve2_bgrp_h
,
6513 gen_helper_sve2_bgrp_s
, gen_helper_sve2_bgrp_d
,
6515 TRANS_FEAT(BGRP
, aa64_sve2_bitperm
, gen_gvec_ool_arg_zzz
,
6516 bgrp_fns
[a
->esz
], a
, 0)
6518 static gen_helper_gvec_3
* const cadd_fns
[4] = {
6519 gen_helper_sve2_cadd_b
, gen_helper_sve2_cadd_h
,
6520 gen_helper_sve2_cadd_s
, gen_helper_sve2_cadd_d
,
6522 TRANS_FEAT(CADD_rot90
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6523 cadd_fns
[a
->esz
], a
, 0)
6524 TRANS_FEAT(CADD_rot270
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6525 cadd_fns
[a
->esz
], a
, 1)
6527 static gen_helper_gvec_3
* const sqcadd_fns
[4] = {
6528 gen_helper_sve2_sqcadd_b
, gen_helper_sve2_sqcadd_h
,
6529 gen_helper_sve2_sqcadd_s
, gen_helper_sve2_sqcadd_d
,
6531 TRANS_FEAT(SQCADD_rot90
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6532 sqcadd_fns
[a
->esz
], a
, 0)
6533 TRANS_FEAT(SQCADD_rot270
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
6534 sqcadd_fns
[a
->esz
], a
, 1)
6536 static gen_helper_gvec_4
* const sabal_fns
[4] = {
6537 NULL
, gen_helper_sve2_sabal_h
,
6538 gen_helper_sve2_sabal_s
, gen_helper_sve2_sabal_d
,
6540 TRANS_FEAT(SABALB
, aa64_sve2
, gen_gvec_ool_arg_zzzz
, sabal_fns
[a
->esz
], a
, 0)
6541 TRANS_FEAT(SABALT
, aa64_sve2
, gen_gvec_ool_arg_zzzz
, sabal_fns
[a
->esz
], a
, 1)
6543 static gen_helper_gvec_4
* const uabal_fns
[4] = {
6544 NULL
, gen_helper_sve2_uabal_h
,
6545 gen_helper_sve2_uabal_s
, gen_helper_sve2_uabal_d
,
6547 TRANS_FEAT(UABALB
, aa64_sve2
, gen_gvec_ool_arg_zzzz
, uabal_fns
[a
->esz
], a
, 0)
6548 TRANS_FEAT(UABALT
, aa64_sve2
, gen_gvec_ool_arg_zzzz
, uabal_fns
[a
->esz
], a
, 1)
6550 static bool do_adcl(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
6552 static gen_helper_gvec_4
* const fns
[2] = {
6553 gen_helper_sve2_adcl_s
,
6554 gen_helper_sve2_adcl_d
,
6557 * Note that in this case the ESZ field encodes both size and sign.
6558 * Split out 'subtract' into bit 1 of the data field for the helper.
6560 return gen_gvec_ool_arg_zzzz(s
, fns
[a
->esz
& 1], a
, (a
->esz
& 2) | sel
);
6563 TRANS_FEAT(ADCLB
, aa64_sve2
, do_adcl
, a
, false)
6564 TRANS_FEAT(ADCLT
, aa64_sve2
, do_adcl
, a
, true)
6566 TRANS_FEAT(SSRA
, aa64_sve2
, gen_gvec_fn_arg_zzi
, gen_gvec_ssra
, a
)
6567 TRANS_FEAT(USRA
, aa64_sve2
, gen_gvec_fn_arg_zzi
, gen_gvec_usra
, a
)
6568 TRANS_FEAT(SRSRA
, aa64_sve2
, gen_gvec_fn_arg_zzi
, gen_gvec_srsra
, a
)
6569 TRANS_FEAT(URSRA
, aa64_sve2
, gen_gvec_fn_arg_zzi
, gen_gvec_ursra
, a
)
6570 TRANS_FEAT(SRI
, aa64_sve2
, gen_gvec_fn_arg_zzi
, gen_gvec_sri
, a
)
6571 TRANS_FEAT(SLI
, aa64_sve2
, gen_gvec_fn_arg_zzi
, gen_gvec_sli
, a
)
6573 TRANS_FEAT(SABA
, aa64_sve2
, gen_gvec_fn_arg_zzz
, gen_gvec_saba
, a
)
6574 TRANS_FEAT(UABA
, aa64_sve2
, gen_gvec_fn_arg_zzz
, gen_gvec_uaba
, a
)
6576 static bool do_sve2_narrow_extract(DisasContext
*s
, arg_rri_esz
*a
,
6577 const GVecGen2 ops
[3])
6579 if (a
->esz
< 0 || a
->esz
> MO_32
|| a
->imm
!= 0 ||
6580 !dc_isar_feature(aa64_sve2
, s
)) {
6583 if (sve_access_check(s
)) {
6584 unsigned vsz
= vec_full_reg_size(s
);
6585 tcg_gen_gvec_2(vec_full_reg_offset(s
, a
->rd
),
6586 vec_full_reg_offset(s
, a
->rn
),
6587 vsz
, vsz
, &ops
[a
->esz
]);
6592 static const TCGOpcode sqxtn_list
[] = {
6593 INDEX_op_shli_vec
, INDEX_op_smin_vec
, INDEX_op_smax_vec
, 0
6596 static void gen_sqxtnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
6598 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6599 int halfbits
= 4 << vece
;
6600 int64_t mask
= (1ull << halfbits
) - 1;
6601 int64_t min
= -1ull << (halfbits
- 1);
6602 int64_t max
= -min
- 1;
6604 tcg_gen_dupi_vec(vece
, t
, min
);
6605 tcg_gen_smax_vec(vece
, d
, n
, t
);
6606 tcg_gen_dupi_vec(vece
, t
, max
);
6607 tcg_gen_smin_vec(vece
, d
, d
, t
);
6608 tcg_gen_dupi_vec(vece
, t
, mask
);
6609 tcg_gen_and_vec(vece
, d
, d
, t
);
6610 tcg_temp_free_vec(t
);
6613 static bool trans_SQXTNB(DisasContext
*s
, arg_rri_esz
*a
)
6615 static const GVecGen2 ops
[3] = {
6616 { .fniv
= gen_sqxtnb_vec
,
6617 .opt_opc
= sqxtn_list
,
6618 .fno
= gen_helper_sve2_sqxtnb_h
,
6620 { .fniv
= gen_sqxtnb_vec
,
6621 .opt_opc
= sqxtn_list
,
6622 .fno
= gen_helper_sve2_sqxtnb_s
,
6624 { .fniv
= gen_sqxtnb_vec
,
6625 .opt_opc
= sqxtn_list
,
6626 .fno
= gen_helper_sve2_sqxtnb_d
,
6629 return do_sve2_narrow_extract(s
, a
, ops
);
6632 static void gen_sqxtnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
6634 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6635 int halfbits
= 4 << vece
;
6636 int64_t mask
= (1ull << halfbits
) - 1;
6637 int64_t min
= -1ull << (halfbits
- 1);
6638 int64_t max
= -min
- 1;
6640 tcg_gen_dupi_vec(vece
, t
, min
);
6641 tcg_gen_smax_vec(vece
, n
, n
, t
);
6642 tcg_gen_dupi_vec(vece
, t
, max
);
6643 tcg_gen_smin_vec(vece
, n
, n
, t
);
6644 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
6645 tcg_gen_dupi_vec(vece
, t
, mask
);
6646 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
6647 tcg_temp_free_vec(t
);
6650 static bool trans_SQXTNT(DisasContext
*s
, arg_rri_esz
*a
)
6652 static const GVecGen2 ops
[3] = {
6653 { .fniv
= gen_sqxtnt_vec
,
6654 .opt_opc
= sqxtn_list
,
6656 .fno
= gen_helper_sve2_sqxtnt_h
,
6658 { .fniv
= gen_sqxtnt_vec
,
6659 .opt_opc
= sqxtn_list
,
6661 .fno
= gen_helper_sve2_sqxtnt_s
,
6663 { .fniv
= gen_sqxtnt_vec
,
6664 .opt_opc
= sqxtn_list
,
6666 .fno
= gen_helper_sve2_sqxtnt_d
,
6669 return do_sve2_narrow_extract(s
, a
, ops
);
6672 static const TCGOpcode uqxtn_list
[] = {
6673 INDEX_op_shli_vec
, INDEX_op_umin_vec
, 0
6676 static void gen_uqxtnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
6678 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6679 int halfbits
= 4 << vece
;
6680 int64_t max
= (1ull << halfbits
) - 1;
6682 tcg_gen_dupi_vec(vece
, t
, max
);
6683 tcg_gen_umin_vec(vece
, d
, n
, t
);
6684 tcg_temp_free_vec(t
);
6687 static bool trans_UQXTNB(DisasContext
*s
, arg_rri_esz
*a
)
6689 static const GVecGen2 ops
[3] = {
6690 { .fniv
= gen_uqxtnb_vec
,
6691 .opt_opc
= uqxtn_list
,
6692 .fno
= gen_helper_sve2_uqxtnb_h
,
6694 { .fniv
= gen_uqxtnb_vec
,
6695 .opt_opc
= uqxtn_list
,
6696 .fno
= gen_helper_sve2_uqxtnb_s
,
6698 { .fniv
= gen_uqxtnb_vec
,
6699 .opt_opc
= uqxtn_list
,
6700 .fno
= gen_helper_sve2_uqxtnb_d
,
6703 return do_sve2_narrow_extract(s
, a
, ops
);
6706 static void gen_uqxtnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
6708 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6709 int halfbits
= 4 << vece
;
6710 int64_t max
= (1ull << halfbits
) - 1;
6712 tcg_gen_dupi_vec(vece
, t
, max
);
6713 tcg_gen_umin_vec(vece
, n
, n
, t
);
6714 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
6715 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
6716 tcg_temp_free_vec(t
);
6719 static bool trans_UQXTNT(DisasContext
*s
, arg_rri_esz
*a
)
6721 static const GVecGen2 ops
[3] = {
6722 { .fniv
= gen_uqxtnt_vec
,
6723 .opt_opc
= uqxtn_list
,
6725 .fno
= gen_helper_sve2_uqxtnt_h
,
6727 { .fniv
= gen_uqxtnt_vec
,
6728 .opt_opc
= uqxtn_list
,
6730 .fno
= gen_helper_sve2_uqxtnt_s
,
6732 { .fniv
= gen_uqxtnt_vec
,
6733 .opt_opc
= uqxtn_list
,
6735 .fno
= gen_helper_sve2_uqxtnt_d
,
6738 return do_sve2_narrow_extract(s
, a
, ops
);
6741 static const TCGOpcode sqxtun_list
[] = {
6742 INDEX_op_shli_vec
, INDEX_op_umin_vec
, INDEX_op_smax_vec
, 0
6745 static void gen_sqxtunb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
6747 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6748 int halfbits
= 4 << vece
;
6749 int64_t max
= (1ull << halfbits
) - 1;
6751 tcg_gen_dupi_vec(vece
, t
, 0);
6752 tcg_gen_smax_vec(vece
, d
, n
, t
);
6753 tcg_gen_dupi_vec(vece
, t
, max
);
6754 tcg_gen_umin_vec(vece
, d
, d
, t
);
6755 tcg_temp_free_vec(t
);
6758 static bool trans_SQXTUNB(DisasContext
*s
, arg_rri_esz
*a
)
6760 static const GVecGen2 ops
[3] = {
6761 { .fniv
= gen_sqxtunb_vec
,
6762 .opt_opc
= sqxtun_list
,
6763 .fno
= gen_helper_sve2_sqxtunb_h
,
6765 { .fniv
= gen_sqxtunb_vec
,
6766 .opt_opc
= sqxtun_list
,
6767 .fno
= gen_helper_sve2_sqxtunb_s
,
6769 { .fniv
= gen_sqxtunb_vec
,
6770 .opt_opc
= sqxtun_list
,
6771 .fno
= gen_helper_sve2_sqxtunb_d
,
6774 return do_sve2_narrow_extract(s
, a
, ops
);
6777 static void gen_sqxtunt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
6779 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6780 int halfbits
= 4 << vece
;
6781 int64_t max
= (1ull << halfbits
) - 1;
6783 tcg_gen_dupi_vec(vece
, t
, 0);
6784 tcg_gen_smax_vec(vece
, n
, n
, t
);
6785 tcg_gen_dupi_vec(vece
, t
, max
);
6786 tcg_gen_umin_vec(vece
, n
, n
, t
);
6787 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
6788 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
6789 tcg_temp_free_vec(t
);
6792 static bool trans_SQXTUNT(DisasContext
*s
, arg_rri_esz
*a
)
6794 static const GVecGen2 ops
[3] = {
6795 { .fniv
= gen_sqxtunt_vec
,
6796 .opt_opc
= sqxtun_list
,
6798 .fno
= gen_helper_sve2_sqxtunt_h
,
6800 { .fniv
= gen_sqxtunt_vec
,
6801 .opt_opc
= sqxtun_list
,
6803 .fno
= gen_helper_sve2_sqxtunt_s
,
6805 { .fniv
= gen_sqxtunt_vec
,
6806 .opt_opc
= sqxtun_list
,
6808 .fno
= gen_helper_sve2_sqxtunt_d
,
6811 return do_sve2_narrow_extract(s
, a
, ops
);
6814 static bool do_sve2_shr_narrow(DisasContext
*s
, arg_rri_esz
*a
,
6815 const GVecGen2i ops
[3])
6817 if (a
->esz
< 0 || a
->esz
> MO_32
|| !dc_isar_feature(aa64_sve2
, s
)) {
6820 assert(a
->imm
> 0 && a
->imm
<= (8 << a
->esz
));
6821 if (sve_access_check(s
)) {
6822 unsigned vsz
= vec_full_reg_size(s
);
6823 tcg_gen_gvec_2i(vec_full_reg_offset(s
, a
->rd
),
6824 vec_full_reg_offset(s
, a
->rn
),
6825 vsz
, vsz
, a
->imm
, &ops
[a
->esz
]);
6830 static void gen_shrnb_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int shr
)
6832 int halfbits
= 4 << vece
;
6833 uint64_t mask
= dup_const(vece
, MAKE_64BIT_MASK(0, halfbits
));
6835 tcg_gen_shri_i64(d
, n
, shr
);
6836 tcg_gen_andi_i64(d
, d
, mask
);
6839 static void gen_shrnb16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
6841 gen_shrnb_i64(MO_16
, d
, n
, shr
);
6844 static void gen_shrnb32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
6846 gen_shrnb_i64(MO_32
, d
, n
, shr
);
6849 static void gen_shrnb64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
6851 gen_shrnb_i64(MO_64
, d
, n
, shr
);
6854 static void gen_shrnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t shr
)
6856 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6857 int halfbits
= 4 << vece
;
6858 uint64_t mask
= MAKE_64BIT_MASK(0, halfbits
);
6860 tcg_gen_shri_vec(vece
, n
, n
, shr
);
6861 tcg_gen_dupi_vec(vece
, t
, mask
);
6862 tcg_gen_and_vec(vece
, d
, n
, t
);
6863 tcg_temp_free_vec(t
);
6866 static bool trans_SHRNB(DisasContext
*s
, arg_rri_esz
*a
)
6868 static const TCGOpcode vec_list
[] = { INDEX_op_shri_vec
, 0 };
6869 static const GVecGen2i ops
[3] = {
6870 { .fni8
= gen_shrnb16_i64
,
6871 .fniv
= gen_shrnb_vec
,
6872 .opt_opc
= vec_list
,
6873 .fno
= gen_helper_sve2_shrnb_h
,
6875 { .fni8
= gen_shrnb32_i64
,
6876 .fniv
= gen_shrnb_vec
,
6877 .opt_opc
= vec_list
,
6878 .fno
= gen_helper_sve2_shrnb_s
,
6880 { .fni8
= gen_shrnb64_i64
,
6881 .fniv
= gen_shrnb_vec
,
6882 .opt_opc
= vec_list
,
6883 .fno
= gen_helper_sve2_shrnb_d
,
6886 return do_sve2_shr_narrow(s
, a
, ops
);
6889 static void gen_shrnt_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int shr
)
6891 int halfbits
= 4 << vece
;
6892 uint64_t mask
= dup_const(vece
, MAKE_64BIT_MASK(0, halfbits
));
6894 tcg_gen_shli_i64(n
, n
, halfbits
- shr
);
6895 tcg_gen_andi_i64(n
, n
, ~mask
);
6896 tcg_gen_andi_i64(d
, d
, mask
);
6897 tcg_gen_or_i64(d
, d
, n
);
6900 static void gen_shrnt16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
6902 gen_shrnt_i64(MO_16
, d
, n
, shr
);
6905 static void gen_shrnt32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
6907 gen_shrnt_i64(MO_32
, d
, n
, shr
);
6910 static void gen_shrnt64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
6912 tcg_gen_shri_i64(n
, n
, shr
);
6913 tcg_gen_deposit_i64(d
, d
, n
, 32, 32);
6916 static void gen_shrnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t shr
)
6918 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6919 int halfbits
= 4 << vece
;
6920 uint64_t mask
= MAKE_64BIT_MASK(0, halfbits
);
6922 tcg_gen_shli_vec(vece
, n
, n
, halfbits
- shr
);
6923 tcg_gen_dupi_vec(vece
, t
, mask
);
6924 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
6925 tcg_temp_free_vec(t
);
6928 static bool trans_SHRNT(DisasContext
*s
, arg_rri_esz
*a
)
6930 static const TCGOpcode vec_list
[] = { INDEX_op_shli_vec
, 0 };
6931 static const GVecGen2i ops
[3] = {
6932 { .fni8
= gen_shrnt16_i64
,
6933 .fniv
= gen_shrnt_vec
,
6934 .opt_opc
= vec_list
,
6936 .fno
= gen_helper_sve2_shrnt_h
,
6938 { .fni8
= gen_shrnt32_i64
,
6939 .fniv
= gen_shrnt_vec
,
6940 .opt_opc
= vec_list
,
6942 .fno
= gen_helper_sve2_shrnt_s
,
6944 { .fni8
= gen_shrnt64_i64
,
6945 .fniv
= gen_shrnt_vec
,
6946 .opt_opc
= vec_list
,
6948 .fno
= gen_helper_sve2_shrnt_d
,
6951 return do_sve2_shr_narrow(s
, a
, ops
);
6954 static bool trans_RSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
6956 static const GVecGen2i ops
[3] = {
6957 { .fno
= gen_helper_sve2_rshrnb_h
},
6958 { .fno
= gen_helper_sve2_rshrnb_s
},
6959 { .fno
= gen_helper_sve2_rshrnb_d
},
6961 return do_sve2_shr_narrow(s
, a
, ops
);
6964 static bool trans_RSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
6966 static const GVecGen2i ops
[3] = {
6967 { .fno
= gen_helper_sve2_rshrnt_h
},
6968 { .fno
= gen_helper_sve2_rshrnt_s
},
6969 { .fno
= gen_helper_sve2_rshrnt_d
},
6971 return do_sve2_shr_narrow(s
, a
, ops
);
6974 static void gen_sqshrunb_vec(unsigned vece
, TCGv_vec d
,
6975 TCGv_vec n
, int64_t shr
)
6977 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6978 int halfbits
= 4 << vece
;
6980 tcg_gen_sari_vec(vece
, n
, n
, shr
);
6981 tcg_gen_dupi_vec(vece
, t
, 0);
6982 tcg_gen_smax_vec(vece
, n
, n
, t
);
6983 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
6984 tcg_gen_umin_vec(vece
, d
, n
, t
);
6985 tcg_temp_free_vec(t
);
6988 static bool trans_SQSHRUNB(DisasContext
*s
, arg_rri_esz
*a
)
6990 static const TCGOpcode vec_list
[] = {
6991 INDEX_op_sari_vec
, INDEX_op_smax_vec
, INDEX_op_umin_vec
, 0
6993 static const GVecGen2i ops
[3] = {
6994 { .fniv
= gen_sqshrunb_vec
,
6995 .opt_opc
= vec_list
,
6996 .fno
= gen_helper_sve2_sqshrunb_h
,
6998 { .fniv
= gen_sqshrunb_vec
,
6999 .opt_opc
= vec_list
,
7000 .fno
= gen_helper_sve2_sqshrunb_s
,
7002 { .fniv
= gen_sqshrunb_vec
,
7003 .opt_opc
= vec_list
,
7004 .fno
= gen_helper_sve2_sqshrunb_d
,
7007 return do_sve2_shr_narrow(s
, a
, ops
);
7010 static void gen_sqshrunt_vec(unsigned vece
, TCGv_vec d
,
7011 TCGv_vec n
, int64_t shr
)
7013 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7014 int halfbits
= 4 << vece
;
7016 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7017 tcg_gen_dupi_vec(vece
, t
, 0);
7018 tcg_gen_smax_vec(vece
, n
, n
, t
);
7019 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7020 tcg_gen_umin_vec(vece
, n
, n
, t
);
7021 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7022 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7023 tcg_temp_free_vec(t
);
7026 static bool trans_SQSHRUNT(DisasContext
*s
, arg_rri_esz
*a
)
7028 static const TCGOpcode vec_list
[] = {
7029 INDEX_op_shli_vec
, INDEX_op_sari_vec
,
7030 INDEX_op_smax_vec
, INDEX_op_umin_vec
, 0
7032 static const GVecGen2i ops
[3] = {
7033 { .fniv
= gen_sqshrunt_vec
,
7034 .opt_opc
= vec_list
,
7036 .fno
= gen_helper_sve2_sqshrunt_h
,
7038 { .fniv
= gen_sqshrunt_vec
,
7039 .opt_opc
= vec_list
,
7041 .fno
= gen_helper_sve2_sqshrunt_s
,
7043 { .fniv
= gen_sqshrunt_vec
,
7044 .opt_opc
= vec_list
,
7046 .fno
= gen_helper_sve2_sqshrunt_d
,
7049 return do_sve2_shr_narrow(s
, a
, ops
);
7052 static bool trans_SQRSHRUNB(DisasContext
*s
, arg_rri_esz
*a
)
7054 static const GVecGen2i ops
[3] = {
7055 { .fno
= gen_helper_sve2_sqrshrunb_h
},
7056 { .fno
= gen_helper_sve2_sqrshrunb_s
},
7057 { .fno
= gen_helper_sve2_sqrshrunb_d
},
7059 return do_sve2_shr_narrow(s
, a
, ops
);
7062 static bool trans_SQRSHRUNT(DisasContext
*s
, arg_rri_esz
*a
)
7064 static const GVecGen2i ops
[3] = {
7065 { .fno
= gen_helper_sve2_sqrshrunt_h
},
7066 { .fno
= gen_helper_sve2_sqrshrunt_s
},
7067 { .fno
= gen_helper_sve2_sqrshrunt_d
},
7069 return do_sve2_shr_narrow(s
, a
, ops
);
7072 static void gen_sqshrnb_vec(unsigned vece
, TCGv_vec d
,
7073 TCGv_vec n
, int64_t shr
)
7075 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7076 int halfbits
= 4 << vece
;
7077 int64_t max
= MAKE_64BIT_MASK(0, halfbits
- 1);
7078 int64_t min
= -max
- 1;
7080 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7081 tcg_gen_dupi_vec(vece
, t
, min
);
7082 tcg_gen_smax_vec(vece
, n
, n
, t
);
7083 tcg_gen_dupi_vec(vece
, t
, max
);
7084 tcg_gen_smin_vec(vece
, n
, n
, t
);
7085 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7086 tcg_gen_and_vec(vece
, d
, n
, t
);
7087 tcg_temp_free_vec(t
);
7090 static bool trans_SQSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7092 static const TCGOpcode vec_list
[] = {
7093 INDEX_op_sari_vec
, INDEX_op_smax_vec
, INDEX_op_smin_vec
, 0
7095 static const GVecGen2i ops
[3] = {
7096 { .fniv
= gen_sqshrnb_vec
,
7097 .opt_opc
= vec_list
,
7098 .fno
= gen_helper_sve2_sqshrnb_h
,
7100 { .fniv
= gen_sqshrnb_vec
,
7101 .opt_opc
= vec_list
,
7102 .fno
= gen_helper_sve2_sqshrnb_s
,
7104 { .fniv
= gen_sqshrnb_vec
,
7105 .opt_opc
= vec_list
,
7106 .fno
= gen_helper_sve2_sqshrnb_d
,
7109 return do_sve2_shr_narrow(s
, a
, ops
);
7112 static void gen_sqshrnt_vec(unsigned vece
, TCGv_vec d
,
7113 TCGv_vec n
, int64_t shr
)
7115 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7116 int halfbits
= 4 << vece
;
7117 int64_t max
= MAKE_64BIT_MASK(0, halfbits
- 1);
7118 int64_t min
= -max
- 1;
7120 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7121 tcg_gen_dupi_vec(vece
, t
, min
);
7122 tcg_gen_smax_vec(vece
, n
, n
, t
);
7123 tcg_gen_dupi_vec(vece
, t
, max
);
7124 tcg_gen_smin_vec(vece
, n
, n
, t
);
7125 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7126 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7127 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7128 tcg_temp_free_vec(t
);
7131 static bool trans_SQSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7133 static const TCGOpcode vec_list
[] = {
7134 INDEX_op_shli_vec
, INDEX_op_sari_vec
,
7135 INDEX_op_smax_vec
, INDEX_op_smin_vec
, 0
7137 static const GVecGen2i ops
[3] = {
7138 { .fniv
= gen_sqshrnt_vec
,
7139 .opt_opc
= vec_list
,
7141 .fno
= gen_helper_sve2_sqshrnt_h
,
7143 { .fniv
= gen_sqshrnt_vec
,
7144 .opt_opc
= vec_list
,
7146 .fno
= gen_helper_sve2_sqshrnt_s
,
7148 { .fniv
= gen_sqshrnt_vec
,
7149 .opt_opc
= vec_list
,
7151 .fno
= gen_helper_sve2_sqshrnt_d
,
7154 return do_sve2_shr_narrow(s
, a
, ops
);
7157 static bool trans_SQRSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7159 static const GVecGen2i ops
[3] = {
7160 { .fno
= gen_helper_sve2_sqrshrnb_h
},
7161 { .fno
= gen_helper_sve2_sqrshrnb_s
},
7162 { .fno
= gen_helper_sve2_sqrshrnb_d
},
7164 return do_sve2_shr_narrow(s
, a
, ops
);
7167 static bool trans_SQRSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7169 static const GVecGen2i ops
[3] = {
7170 { .fno
= gen_helper_sve2_sqrshrnt_h
},
7171 { .fno
= gen_helper_sve2_sqrshrnt_s
},
7172 { .fno
= gen_helper_sve2_sqrshrnt_d
},
7174 return do_sve2_shr_narrow(s
, a
, ops
);
7177 static void gen_uqshrnb_vec(unsigned vece
, TCGv_vec d
,
7178 TCGv_vec n
, int64_t shr
)
7180 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7181 int halfbits
= 4 << vece
;
7183 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7184 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7185 tcg_gen_umin_vec(vece
, d
, n
, t
);
7186 tcg_temp_free_vec(t
);
7189 static bool trans_UQSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7191 static const TCGOpcode vec_list
[] = {
7192 INDEX_op_shri_vec
, INDEX_op_umin_vec
, 0
7194 static const GVecGen2i ops
[3] = {
7195 { .fniv
= gen_uqshrnb_vec
,
7196 .opt_opc
= vec_list
,
7197 .fno
= gen_helper_sve2_uqshrnb_h
,
7199 { .fniv
= gen_uqshrnb_vec
,
7200 .opt_opc
= vec_list
,
7201 .fno
= gen_helper_sve2_uqshrnb_s
,
7203 { .fniv
= gen_uqshrnb_vec
,
7204 .opt_opc
= vec_list
,
7205 .fno
= gen_helper_sve2_uqshrnb_d
,
7208 return do_sve2_shr_narrow(s
, a
, ops
);
7211 static void gen_uqshrnt_vec(unsigned vece
, TCGv_vec d
,
7212 TCGv_vec n
, int64_t shr
)
7214 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7215 int halfbits
= 4 << vece
;
7217 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7218 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7219 tcg_gen_umin_vec(vece
, n
, n
, t
);
7220 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7221 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7222 tcg_temp_free_vec(t
);
7225 static bool trans_UQSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7227 static const TCGOpcode vec_list
[] = {
7228 INDEX_op_shli_vec
, INDEX_op_shri_vec
, INDEX_op_umin_vec
, 0
7230 static const GVecGen2i ops
[3] = {
7231 { .fniv
= gen_uqshrnt_vec
,
7232 .opt_opc
= vec_list
,
7234 .fno
= gen_helper_sve2_uqshrnt_h
,
7236 { .fniv
= gen_uqshrnt_vec
,
7237 .opt_opc
= vec_list
,
7239 .fno
= gen_helper_sve2_uqshrnt_s
,
7241 { .fniv
= gen_uqshrnt_vec
,
7242 .opt_opc
= vec_list
,
7244 .fno
= gen_helper_sve2_uqshrnt_d
,
7247 return do_sve2_shr_narrow(s
, a
, ops
);
7250 static bool trans_UQRSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7252 static const GVecGen2i ops
[3] = {
7253 { .fno
= gen_helper_sve2_uqrshrnb_h
},
7254 { .fno
= gen_helper_sve2_uqrshrnb_s
},
7255 { .fno
= gen_helper_sve2_uqrshrnb_d
},
7257 return do_sve2_shr_narrow(s
, a
, ops
);
7260 static bool trans_UQRSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7262 static const GVecGen2i ops
[3] = {
7263 { .fno
= gen_helper_sve2_uqrshrnt_h
},
7264 { .fno
= gen_helper_sve2_uqrshrnt_s
},
7265 { .fno
= gen_helper_sve2_uqrshrnt_d
},
7267 return do_sve2_shr_narrow(s
, a
, ops
);
7270 #define DO_SVE2_ZZZ_NARROW(NAME, name) \
7271 static gen_helper_gvec_3 * const name##_fns[4] = { \
7272 NULL, gen_helper_sve2_##name##_h, \
7273 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
7275 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_arg_zzz, \
7276 name##_fns[a->esz], a, 0)
7278 DO_SVE2_ZZZ_NARROW(ADDHNB
, addhnb
)
7279 DO_SVE2_ZZZ_NARROW(ADDHNT
, addhnt
)
7280 DO_SVE2_ZZZ_NARROW(RADDHNB
, raddhnb
)
7281 DO_SVE2_ZZZ_NARROW(RADDHNT
, raddhnt
)
7283 DO_SVE2_ZZZ_NARROW(SUBHNB
, subhnb
)
7284 DO_SVE2_ZZZ_NARROW(SUBHNT
, subhnt
)
7285 DO_SVE2_ZZZ_NARROW(RSUBHNB
, rsubhnb
)
7286 DO_SVE2_ZZZ_NARROW(RSUBHNT
, rsubhnt
)
7288 static gen_helper_gvec_flags_4
* const match_fns
[4] = {
7289 gen_helper_sve2_match_ppzz_b
, gen_helper_sve2_match_ppzz_h
, NULL
, NULL
7291 TRANS_FEAT(MATCH
, aa64_sve2
, do_ppzz_flags
, a
, match_fns
[a
->esz
])
7293 static gen_helper_gvec_flags_4
* const nmatch_fns
[4] = {
7294 gen_helper_sve2_nmatch_ppzz_b
, gen_helper_sve2_nmatch_ppzz_h
, NULL
, NULL
7296 TRANS_FEAT(NMATCH
, aa64_sve2
, do_ppzz_flags
, a
, nmatch_fns
[a
->esz
])
7298 static gen_helper_gvec_4
* const histcnt_fns
[4] = {
7299 NULL
, NULL
, gen_helper_sve2_histcnt_s
, gen_helper_sve2_histcnt_d
7301 TRANS_FEAT(HISTCNT
, aa64_sve2
, gen_gvec_ool_arg_zpzz
,
7302 histcnt_fns
[a
->esz
], a
, 0)
7304 TRANS_FEAT(HISTSEG
, aa64_sve2
, gen_gvec_ool_arg_zzz
,
7305 a
->esz
== 0 ? gen_helper_sve2_histseg
: NULL
, a
, 0)
7307 static bool do_sve2_zpzz_fp(DisasContext
*s
, arg_rprr_esz
*a
,
7308 gen_helper_gvec_4_ptr
*fn
)
7310 if (!dc_isar_feature(aa64_sve2
, s
)) {
7313 return do_zpzz_fp(s
, a
, fn
);
7316 #define DO_SVE2_ZPZZ_FP(NAME, name) \
7317 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
7319 static gen_helper_gvec_4_ptr * const fns[4] = { \
7320 NULL, gen_helper_sve2_##name##_zpzz_h, \
7321 gen_helper_sve2_##name##_zpzz_s, gen_helper_sve2_##name##_zpzz_d \
7323 return do_sve2_zpzz_fp(s, a, fns[a->esz]); \
7326 DO_SVE2_ZPZZ_FP(FADDP
, faddp
)
7327 DO_SVE2_ZPZZ_FP(FMAXNMP
, fmaxnmp
)
7328 DO_SVE2_ZPZZ_FP(FMINNMP
, fminnmp
)
7329 DO_SVE2_ZPZZ_FP(FMAXP
, fmaxp
)
7330 DO_SVE2_ZPZZ_FP(FMINP
, fminp
)
7333 * SVE Integer Multiply-Add (unpredicated)
7336 static bool trans_FMMLA(DisasContext
*s
, arg_rrrr_esz
*a
)
7338 gen_helper_gvec_4_ptr
*fn
;
7342 if (!dc_isar_feature(aa64_sve_f32mm
, s
)) {
7345 fn
= gen_helper_fmmla_s
;
7348 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
7351 fn
= gen_helper_fmmla_d
;
7357 if (sve_access_check(s
)) {
7358 unsigned vsz
= vec_full_reg_size(s
);
7359 TCGv_ptr status
= fpstatus_ptr(FPST_FPCR
);
7360 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
7361 vec_full_reg_offset(s
, a
->rn
),
7362 vec_full_reg_offset(s
, a
->rm
),
7363 vec_full_reg_offset(s
, a
->ra
),
7364 status
, vsz
, vsz
, 0, fn
);
7365 tcg_temp_free_ptr(status
);
7370 static gen_helper_gvec_4
* const sqdmlal_zzzw_fns
[] = {
7371 NULL
, gen_helper_sve2_sqdmlal_zzzw_h
,
7372 gen_helper_sve2_sqdmlal_zzzw_s
, gen_helper_sve2_sqdmlal_zzzw_d
,
7374 TRANS_FEAT(SQDMLALB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7375 sqdmlal_zzzw_fns
[a
->esz
], a
, 0)
7376 TRANS_FEAT(SQDMLALT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7377 sqdmlal_zzzw_fns
[a
->esz
], a
, 3)
7378 TRANS_FEAT(SQDMLALBT
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7379 sqdmlal_zzzw_fns
[a
->esz
], a
, 2)
7381 static gen_helper_gvec_4
* const sqdmlsl_zzzw_fns
[] = {
7382 NULL
, gen_helper_sve2_sqdmlsl_zzzw_h
,
7383 gen_helper_sve2_sqdmlsl_zzzw_s
, gen_helper_sve2_sqdmlsl_zzzw_d
,
7385 TRANS_FEAT(SQDMLSLB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7386 sqdmlsl_zzzw_fns
[a
->esz
], a
, 0)
7387 TRANS_FEAT(SQDMLSLT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7388 sqdmlsl_zzzw_fns
[a
->esz
], a
, 3)
7389 TRANS_FEAT(SQDMLSLBT
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7390 sqdmlsl_zzzw_fns
[a
->esz
], a
, 2)
7392 static gen_helper_gvec_4
* const sqrdmlah_fns
[] = {
7393 gen_helper_sve2_sqrdmlah_b
, gen_helper_sve2_sqrdmlah_h
,
7394 gen_helper_sve2_sqrdmlah_s
, gen_helper_sve2_sqrdmlah_d
,
7396 TRANS_FEAT(SQRDMLAH_zzzz
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7397 sqrdmlah_fns
[a
->esz
], a
, 0)
7399 static gen_helper_gvec_4
* const sqrdmlsh_fns
[] = {
7400 gen_helper_sve2_sqrdmlsh_b
, gen_helper_sve2_sqrdmlsh_h
,
7401 gen_helper_sve2_sqrdmlsh_s
, gen_helper_sve2_sqrdmlsh_d
,
7403 TRANS_FEAT(SQRDMLSH_zzzz
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7404 sqrdmlsh_fns
[a
->esz
], a
, 0)
7406 static gen_helper_gvec_4
* const smlal_zzzw_fns
[] = {
7407 NULL
, gen_helper_sve2_smlal_zzzw_h
,
7408 gen_helper_sve2_smlal_zzzw_s
, gen_helper_sve2_smlal_zzzw_d
,
7410 TRANS_FEAT(SMLALB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7411 smlal_zzzw_fns
[a
->esz
], a
, 0)
7412 TRANS_FEAT(SMLALT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7413 smlal_zzzw_fns
[a
->esz
], a
, 1)
7415 static gen_helper_gvec_4
* const umlal_zzzw_fns
[] = {
7416 NULL
, gen_helper_sve2_umlal_zzzw_h
,
7417 gen_helper_sve2_umlal_zzzw_s
, gen_helper_sve2_umlal_zzzw_d
,
7419 TRANS_FEAT(UMLALB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7420 umlal_zzzw_fns
[a
->esz
], a
, 0)
7421 TRANS_FEAT(UMLALT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7422 umlal_zzzw_fns
[a
->esz
], a
, 1)
7424 static gen_helper_gvec_4
* const smlsl_zzzw_fns
[] = {
7425 NULL
, gen_helper_sve2_smlsl_zzzw_h
,
7426 gen_helper_sve2_smlsl_zzzw_s
, gen_helper_sve2_smlsl_zzzw_d
,
7428 TRANS_FEAT(SMLSLB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7429 smlsl_zzzw_fns
[a
->esz
], a
, 0)
7430 TRANS_FEAT(SMLSLT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7431 smlsl_zzzw_fns
[a
->esz
], a
, 1)
7433 static gen_helper_gvec_4
* const umlsl_zzzw_fns
[] = {
7434 NULL
, gen_helper_sve2_umlsl_zzzw_h
,
7435 gen_helper_sve2_umlsl_zzzw_s
, gen_helper_sve2_umlsl_zzzw_d
,
7437 TRANS_FEAT(UMLSLB_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7438 umlsl_zzzw_fns
[a
->esz
], a
, 0)
7439 TRANS_FEAT(UMLSLT_zzzw
, aa64_sve2
, gen_gvec_ool_arg_zzzz
,
7440 umlsl_zzzw_fns
[a
->esz
], a
, 1)
7442 static gen_helper_gvec_4
* const cmla_fns
[] = {
7443 gen_helper_sve2_cmla_zzzz_b
, gen_helper_sve2_cmla_zzzz_h
,
7444 gen_helper_sve2_cmla_zzzz_s
, gen_helper_sve2_cmla_zzzz_d
,
7446 TRANS_FEAT(CMLA_zzzz
, aa64_sve2
, gen_gvec_ool_zzzz
,
7447 cmla_fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
)
7449 static gen_helper_gvec_4
* const cdot_fns
[] = {
7450 NULL
, NULL
, gen_helper_sve2_cdot_zzzz_s
, gen_helper_sve2_cdot_zzzz_d
7452 TRANS_FEAT(CDOT_zzzz
, aa64_sve2
, gen_gvec_ool_zzzz
,
7453 cdot_fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
)
7455 static gen_helper_gvec_4
* const sqrdcmlah_fns
[] = {
7456 gen_helper_sve2_sqrdcmlah_zzzz_b
, gen_helper_sve2_sqrdcmlah_zzzz_h
,
7457 gen_helper_sve2_sqrdcmlah_zzzz_s
, gen_helper_sve2_sqrdcmlah_zzzz_d
,
7459 TRANS_FEAT(SQRDCMLAH_zzzz
, aa64_sve2
, gen_gvec_ool_zzzz
,
7460 sqrdcmlah_fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
)
7462 TRANS_FEAT(USDOT_zzzz
, aa64_sve_i8mm
, gen_gvec_ool_arg_zzzz
,
7463 a
->esz
== 2 ? gen_helper_gvec_usdot_b
: NULL
, a
, 0)
7465 TRANS_FEAT(AESMC
, aa64_sve2_aes
, gen_gvec_ool_zz
,
7466 gen_helper_crypto_aesmc
, a
->rd
, a
->rd
, a
->decrypt
)
7468 TRANS_FEAT(AESE
, aa64_sve2_aes
, gen_gvec_ool_arg_zzz
,
7469 gen_helper_crypto_aese
, a
, false)
7470 TRANS_FEAT(AESD
, aa64_sve2_aes
, gen_gvec_ool_arg_zzz
,
7471 gen_helper_crypto_aese
, a
, true)
7473 TRANS_FEAT(SM4E
, aa64_sve2_sm4
, gen_gvec_ool_arg_zzz
,
7474 gen_helper_crypto_sm4e
, a
, 0)
7475 TRANS_FEAT(SM4EKEY
, aa64_sve2_sm4
, gen_gvec_ool_arg_zzz
,
7476 gen_helper_crypto_sm4ekey
, a
, 0)
7478 TRANS_FEAT(RAX1
, aa64_sve2_sha3
, gen_gvec_fn_arg_zzz
, gen_gvec_rax1
, a
)
7480 static bool trans_FCVTNT_sh(DisasContext
*s
, arg_rpr_esz
*a
)
7482 if (!dc_isar_feature(aa64_sve2
, s
)) {
7485 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtnt_sh
);
7488 static bool trans_BFCVTNT(DisasContext
*s
, arg_rpr_esz
*a
)
7490 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
7493 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_bfcvtnt
);
7496 static bool trans_FCVTNT_ds(DisasContext
*s
, arg_rpr_esz
*a
)
7498 if (!dc_isar_feature(aa64_sve2
, s
)) {
7501 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtnt_ds
);
7504 static bool trans_FCVTLT_hs(DisasContext
*s
, arg_rpr_esz
*a
)
7506 if (!dc_isar_feature(aa64_sve2
, s
)) {
7509 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtlt_hs
);
7512 static bool trans_FCVTLT_sd(DisasContext
*s
, arg_rpr_esz
*a
)
7514 if (!dc_isar_feature(aa64_sve2
, s
)) {
7517 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtlt_sd
);
7520 static bool trans_FCVTX_ds(DisasContext
*s
, arg_rpr_esz
*a
)
7522 if (!dc_isar_feature(aa64_sve2
, s
)) {
7525 return do_frint_mode(s
, a
, float_round_to_odd
, gen_helper_sve_fcvt_ds
);
7528 static bool trans_FCVTXNT_ds(DisasContext
*s
, arg_rpr_esz
*a
)
7530 if (!dc_isar_feature(aa64_sve2
, s
)) {
7533 return do_frint_mode(s
, a
, float_round_to_odd
, gen_helper_sve2_fcvtnt_ds
);
7536 static bool trans_FLOGB(DisasContext
*s
, arg_rpr_esz
*a
)
7538 static gen_helper_gvec_3_ptr
* const fns
[] = {
7539 NULL
, gen_helper_flogb_h
,
7540 gen_helper_flogb_s
, gen_helper_flogb_d
7543 if (!dc_isar_feature(aa64_sve2
, s
) || fns
[a
->esz
] == NULL
) {
7546 if (sve_access_check(s
)) {
7548 fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
7549 unsigned vsz
= vec_full_reg_size(s
);
7551 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
7552 vec_full_reg_offset(s
, a
->rn
),
7553 pred_full_reg_offset(s
, a
->pg
),
7554 status
, vsz
, vsz
, 0, fns
[a
->esz
]);
7555 tcg_temp_free_ptr(status
);
7560 static bool do_FMLAL_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sub
, bool sel
)
7562 if (!dc_isar_feature(aa64_sve2
, s
)) {
7565 if (sve_access_check(s
)) {
7566 unsigned vsz
= vec_full_reg_size(s
);
7567 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
7568 vec_full_reg_offset(s
, a
->rn
),
7569 vec_full_reg_offset(s
, a
->rm
),
7570 vec_full_reg_offset(s
, a
->ra
),
7571 cpu_env
, vsz
, vsz
, (sel
<< 1) | sub
,
7572 gen_helper_sve2_fmlal_zzzw_s
);
7577 static bool trans_FMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7579 return do_FMLAL_zzzw(s
, a
, false, false);
7582 static bool trans_FMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7584 return do_FMLAL_zzzw(s
, a
, false, true);
7587 static bool trans_FMLSLB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7589 return do_FMLAL_zzzw(s
, a
, true, false);
7592 static bool trans_FMLSLT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7594 return do_FMLAL_zzzw(s
, a
, true, true);
7597 static bool do_FMLAL_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
, bool sub
, bool sel
)
7599 if (!dc_isar_feature(aa64_sve2
, s
)) {
7602 if (sve_access_check(s
)) {
7603 unsigned vsz
= vec_full_reg_size(s
);
7604 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
7605 vec_full_reg_offset(s
, a
->rn
),
7606 vec_full_reg_offset(s
, a
->rm
),
7607 vec_full_reg_offset(s
, a
->ra
),
7609 (a
->index
<< 2) | (sel
<< 1) | sub
,
7610 gen_helper_sve2_fmlal_zzxw_s
);
7615 static bool trans_FMLALB_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
7617 return do_FMLAL_zzxw(s
, a
, false, false);
7620 static bool trans_FMLALT_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
7622 return do_FMLAL_zzxw(s
, a
, false, true);
7625 static bool trans_FMLSLB_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
7627 return do_FMLAL_zzxw(s
, a
, true, false);
7630 static bool trans_FMLSLT_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
7632 return do_FMLAL_zzxw(s
, a
, true, true);
7635 TRANS_FEAT(SMMLA
, aa64_sve_i8mm
, gen_gvec_ool_arg_zzzz
,
7636 gen_helper_gvec_smmla_b
, a
, 0)
7637 TRANS_FEAT(USMMLA
, aa64_sve_i8mm
, gen_gvec_ool_arg_zzzz
,
7638 gen_helper_gvec_usmmla_b
, a
, 0)
7639 TRANS_FEAT(UMMLA
, aa64_sve_i8mm
, gen_gvec_ool_arg_zzzz
,
7640 gen_helper_gvec_ummla_b
, a
, 0)
7642 TRANS_FEAT(BFDOT_zzzz
, aa64_sve_bf16
, gen_gvec_ool_arg_zzzz
,
7643 gen_helper_gvec_bfdot
, a
, 0)
7644 TRANS_FEAT(BFDOT_zzxz
, aa64_sve_bf16
, gen_gvec_ool_arg_zzxz
,
7645 gen_helper_gvec_bfdot_idx
, a
)
7647 TRANS_FEAT(BFMMLA
, aa64_sve_bf16
, gen_gvec_ool_arg_zzzz
,
7648 gen_helper_gvec_bfmmla
, a
, 0)
7650 static bool do_BFMLAL_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
7652 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
7655 if (sve_access_check(s
)) {
7656 TCGv_ptr status
= fpstatus_ptr(FPST_FPCR
);
7657 unsigned vsz
= vec_full_reg_size(s
);
7659 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
7660 vec_full_reg_offset(s
, a
->rn
),
7661 vec_full_reg_offset(s
, a
->rm
),
7662 vec_full_reg_offset(s
, a
->ra
),
7663 status
, vsz
, vsz
, sel
,
7664 gen_helper_gvec_bfmlal
);
7665 tcg_temp_free_ptr(status
);
7670 static bool trans_BFMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7672 return do_BFMLAL_zzzw(s
, a
, false);
7675 static bool trans_BFMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7677 return do_BFMLAL_zzzw(s
, a
, true);
7680 static bool do_BFMLAL_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
, bool sel
)
7682 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
7685 if (sve_access_check(s
)) {
7686 TCGv_ptr status
= fpstatus_ptr(FPST_FPCR
);
7687 unsigned vsz
= vec_full_reg_size(s
);
7689 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
7690 vec_full_reg_offset(s
, a
->rn
),
7691 vec_full_reg_offset(s
, a
->rm
),
7692 vec_full_reg_offset(s
, a
->ra
),
7693 status
, vsz
, vsz
, (a
->index
<< 1) | sel
,
7694 gen_helper_gvec_bfmlal_idx
);
7695 tcg_temp_free_ptr(status
);
7700 static bool trans_BFMLALB_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
7702 return do_BFMLAL_zzxw(s
, a
, false);
7705 static bool trans_BFMLALT_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
7707 return do_BFMLAL_zzxw(s
, a
, true);