2 * AArch64 SVE translation
4 * Copyright (c) 2018 Linaro, Ltd
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
22 #include "exec/exec-all.h"
23 #include "tcg/tcg-op.h"
24 #include "tcg/tcg-op-gvec.h"
25 #include "tcg/tcg-gvec-desc.h"
28 #include "translate.h"
29 #include "internals.h"
30 #include "exec/helper-proto.h"
31 #include "exec/helper-gen.h"
33 #include "translate-a64.h"
34 #include "fpu/softfloat.h"
37 typedef void GVecGen2sFn(unsigned, uint32_t, uint32_t,
38 TCGv_i64
, uint32_t, uint32_t);
40 typedef void gen_helper_gvec_flags_3(TCGv_i32
, TCGv_ptr
, TCGv_ptr
,
42 typedef void gen_helper_gvec_flags_4(TCGv_i32
, TCGv_ptr
, TCGv_ptr
,
43 TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
45 typedef void gen_helper_gvec_mem(TCGv_env
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
46 typedef void gen_helper_gvec_mem_scatter(TCGv_env
, TCGv_ptr
, TCGv_ptr
,
47 TCGv_ptr
, TCGv_i64
, TCGv_i32
);
50 * Helpers for extracting complex instruction fields.
53 /* See e.g. ASR (immediate, predicated).
54 * Returns -1 for unallocated encoding; diagnose later.
56 static int tszimm_esz(DisasContext
*s
, int x
)
58 x
>>= 3; /* discard imm3 */
62 static int tszimm_shr(DisasContext
*s
, int x
)
64 return (16 << tszimm_esz(s
, x
)) - x
;
67 /* See e.g. LSL (immediate, predicated). */
68 static int tszimm_shl(DisasContext
*s
, int x
)
70 return x
- (8 << tszimm_esz(s
, x
));
73 static inline int plus1(DisasContext
*s
, int x
)
78 /* The SH bit is in bit 8. Extract the low 8 and shift. */
79 static inline int expand_imm_sh8s(DisasContext
*s
, int x
)
81 return (int8_t)x
<< (x
& 0x100 ? 8 : 0);
84 static inline int expand_imm_sh8u(DisasContext
*s
, int x
)
86 return (uint8_t)x
<< (x
& 0x100 ? 8 : 0);
89 /* Convert a 2-bit memory size (msz) to a 4-bit data type (dtype)
90 * with unsigned data. C.f. SVE Memory Contiguous Load Group.
92 static inline int msz_dtype(DisasContext
*s
, int msz
)
94 static const uint8_t dtype
[4] = { 0, 5, 10, 15 };
99 * Include the generated decoder.
102 #include "decode-sve.c.inc"
105 * Implement all of the translator functions referenced by the decoder.
108 /* Return the offset info CPUARMState of the predicate vector register Pn.
109 * Note for this purpose, FFR is P16.
111 static inline int pred_full_reg_offset(DisasContext
*s
, int regno
)
113 return offsetof(CPUARMState
, vfp
.pregs
[regno
]);
116 /* Return the byte size of the whole predicate register, VL / 64. */
117 static inline int pred_full_reg_size(DisasContext
*s
)
119 return s
->sve_len
>> 3;
122 /* Round up the size of a register to a size allowed by
123 * the tcg vector infrastructure. Any operation which uses this
124 * size may assume that the bits above pred_full_reg_size are zero,
125 * and must leave them the same way.
127 * Note that this is not needed for the vector registers as they
128 * are always properly sized for tcg vectors.
130 static int size_for_gvec(int size
)
135 return QEMU_ALIGN_UP(size
, 16);
139 static int pred_gvec_reg_size(DisasContext
*s
)
141 return size_for_gvec(pred_full_reg_size(s
));
144 /* Invoke an out-of-line helper on 2 Zregs. */
145 static void gen_gvec_ool_zz(DisasContext
*s
, gen_helper_gvec_2
*fn
,
146 int rd
, int rn
, int data
)
148 unsigned vsz
= vec_full_reg_size(s
);
149 tcg_gen_gvec_2_ool(vec_full_reg_offset(s
, rd
),
150 vec_full_reg_offset(s
, rn
),
154 /* Invoke an out-of-line helper on 3 Zregs. */
155 static void gen_gvec_ool_zzz(DisasContext
*s
, gen_helper_gvec_3
*fn
,
156 int rd
, int rn
, int rm
, int data
)
158 unsigned vsz
= vec_full_reg_size(s
);
159 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
160 vec_full_reg_offset(s
, rn
),
161 vec_full_reg_offset(s
, rm
),
165 /* Invoke an out-of-line helper on 4 Zregs. */
166 static void gen_gvec_ool_zzzz(DisasContext
*s
, gen_helper_gvec_4
*fn
,
167 int rd
, int rn
, int rm
, int ra
, int data
)
169 unsigned vsz
= vec_full_reg_size(s
);
170 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
171 vec_full_reg_offset(s
, rn
),
172 vec_full_reg_offset(s
, rm
),
173 vec_full_reg_offset(s
, ra
),
177 /* Invoke an out-of-line helper on 2 Zregs and a predicate. */
178 static void gen_gvec_ool_zzp(DisasContext
*s
, gen_helper_gvec_3
*fn
,
179 int rd
, int rn
, int pg
, int data
)
181 unsigned vsz
= vec_full_reg_size(s
);
182 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
183 vec_full_reg_offset(s
, rn
),
184 pred_full_reg_offset(s
, pg
),
188 /* Invoke an out-of-line helper on 3 Zregs and a predicate. */
189 static void gen_gvec_ool_zzzp(DisasContext
*s
, gen_helper_gvec_4
*fn
,
190 int rd
, int rn
, int rm
, int pg
, int data
)
192 unsigned vsz
= vec_full_reg_size(s
);
193 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
194 vec_full_reg_offset(s
, rn
),
195 vec_full_reg_offset(s
, rm
),
196 pred_full_reg_offset(s
, pg
),
200 /* Invoke a vector expander on two Zregs. */
201 static void gen_gvec_fn_zz(DisasContext
*s
, GVecGen2Fn
*gvec_fn
,
202 int esz
, int rd
, int rn
)
204 unsigned vsz
= vec_full_reg_size(s
);
205 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
206 vec_full_reg_offset(s
, rn
), vsz
, vsz
);
209 /* Invoke a vector expander on three Zregs. */
210 static void gen_gvec_fn_zzz(DisasContext
*s
, GVecGen3Fn
*gvec_fn
,
211 int esz
, int rd
, int rn
, int rm
)
213 unsigned vsz
= vec_full_reg_size(s
);
214 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
215 vec_full_reg_offset(s
, rn
),
216 vec_full_reg_offset(s
, rm
), vsz
, vsz
);
219 /* Invoke a vector expander on four Zregs. */
220 static void gen_gvec_fn_zzzz(DisasContext
*s
, GVecGen4Fn
*gvec_fn
,
221 int esz
, int rd
, int rn
, int rm
, int ra
)
223 unsigned vsz
= vec_full_reg_size(s
);
224 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
225 vec_full_reg_offset(s
, rn
),
226 vec_full_reg_offset(s
, rm
),
227 vec_full_reg_offset(s
, ra
), vsz
, vsz
);
230 /* Invoke a vector move on two Zregs. */
231 static bool do_mov_z(DisasContext
*s
, int rd
, int rn
)
233 if (sve_access_check(s
)) {
234 gen_gvec_fn_zz(s
, tcg_gen_gvec_mov
, MO_8
, rd
, rn
);
239 /* Initialize a Zreg with replications of a 64-bit immediate. */
240 static void do_dupi_z(DisasContext
*s
, int rd
, uint64_t word
)
242 unsigned vsz
= vec_full_reg_size(s
);
243 tcg_gen_gvec_dup_imm(MO_64
, vec_full_reg_offset(s
, rd
), vsz
, vsz
, word
);
246 /* Invoke a vector expander on three Pregs. */
247 static void gen_gvec_fn_ppp(DisasContext
*s
, GVecGen3Fn
*gvec_fn
,
248 int rd
, int rn
, int rm
)
250 unsigned psz
= pred_gvec_reg_size(s
);
251 gvec_fn(MO_64
, pred_full_reg_offset(s
, rd
),
252 pred_full_reg_offset(s
, rn
),
253 pred_full_reg_offset(s
, rm
), psz
, psz
);
256 /* Invoke a vector move on two Pregs. */
257 static bool do_mov_p(DisasContext
*s
, int rd
, int rn
)
259 if (sve_access_check(s
)) {
260 unsigned psz
= pred_gvec_reg_size(s
);
261 tcg_gen_gvec_mov(MO_8
, pred_full_reg_offset(s
, rd
),
262 pred_full_reg_offset(s
, rn
), psz
, psz
);
267 /* Set the cpu flags as per a return from an SVE helper. */
268 static void do_pred_flags(TCGv_i32 t
)
270 tcg_gen_mov_i32(cpu_NF
, t
);
271 tcg_gen_andi_i32(cpu_ZF
, t
, 2);
272 tcg_gen_andi_i32(cpu_CF
, t
, 1);
273 tcg_gen_movi_i32(cpu_VF
, 0);
276 /* Subroutines computing the ARM PredTest psuedofunction. */
277 static void do_predtest1(TCGv_i64 d
, TCGv_i64 g
)
279 TCGv_i32 t
= tcg_temp_new_i32();
281 gen_helper_sve_predtest1(t
, d
, g
);
283 tcg_temp_free_i32(t
);
286 static void do_predtest(DisasContext
*s
, int dofs
, int gofs
, int words
)
288 TCGv_ptr dptr
= tcg_temp_new_ptr();
289 TCGv_ptr gptr
= tcg_temp_new_ptr();
292 tcg_gen_addi_ptr(dptr
, cpu_env
, dofs
);
293 tcg_gen_addi_ptr(gptr
, cpu_env
, gofs
);
294 t
= tcg_const_i32(words
);
296 gen_helper_sve_predtest(t
, dptr
, gptr
, t
);
297 tcg_temp_free_ptr(dptr
);
298 tcg_temp_free_ptr(gptr
);
301 tcg_temp_free_i32(t
);
304 /* For each element size, the bits within a predicate word that are active. */
305 const uint64_t pred_esz_masks
[4] = {
306 0xffffffffffffffffull
, 0x5555555555555555ull
,
307 0x1111111111111111ull
, 0x0101010101010101ull
311 *** SVE Logical - Unpredicated Group
314 static bool do_zzz_fn(DisasContext
*s
, arg_rrr_esz
*a
, GVecGen3Fn
*gvec_fn
)
316 if (sve_access_check(s
)) {
317 gen_gvec_fn_zzz(s
, gvec_fn
, a
->esz
, a
->rd
, a
->rn
, a
->rm
);
322 static bool trans_AND_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
324 return do_zzz_fn(s
, a
, tcg_gen_gvec_and
);
327 static bool trans_ORR_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
329 return do_zzz_fn(s
, a
, tcg_gen_gvec_or
);
332 static bool trans_EOR_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
334 return do_zzz_fn(s
, a
, tcg_gen_gvec_xor
);
337 static bool trans_BIC_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
339 return do_zzz_fn(s
, a
, tcg_gen_gvec_andc
);
342 static void gen_xar8_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
344 TCGv_i64 t
= tcg_temp_new_i64();
345 uint64_t mask
= dup_const(MO_8
, 0xff >> sh
);
347 tcg_gen_xor_i64(t
, n
, m
);
348 tcg_gen_shri_i64(d
, t
, sh
);
349 tcg_gen_shli_i64(t
, t
, 8 - sh
);
350 tcg_gen_andi_i64(d
, d
, mask
);
351 tcg_gen_andi_i64(t
, t
, ~mask
);
352 tcg_gen_or_i64(d
, d
, t
);
353 tcg_temp_free_i64(t
);
356 static void gen_xar16_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
358 TCGv_i64 t
= tcg_temp_new_i64();
359 uint64_t mask
= dup_const(MO_16
, 0xffff >> sh
);
361 tcg_gen_xor_i64(t
, n
, m
);
362 tcg_gen_shri_i64(d
, t
, sh
);
363 tcg_gen_shli_i64(t
, t
, 16 - sh
);
364 tcg_gen_andi_i64(d
, d
, mask
);
365 tcg_gen_andi_i64(t
, t
, ~mask
);
366 tcg_gen_or_i64(d
, d
, t
);
367 tcg_temp_free_i64(t
);
370 static void gen_xar_i32(TCGv_i32 d
, TCGv_i32 n
, TCGv_i32 m
, int32_t sh
)
372 tcg_gen_xor_i32(d
, n
, m
);
373 tcg_gen_rotri_i32(d
, d
, sh
);
376 static void gen_xar_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
378 tcg_gen_xor_i64(d
, n
, m
);
379 tcg_gen_rotri_i64(d
, d
, sh
);
382 static void gen_xar_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
383 TCGv_vec m
, int64_t sh
)
385 tcg_gen_xor_vec(vece
, d
, n
, m
);
386 tcg_gen_rotri_vec(vece
, d
, d
, sh
);
389 void gen_gvec_xar(unsigned vece
, uint32_t rd_ofs
, uint32_t rn_ofs
,
390 uint32_t rm_ofs
, int64_t shift
,
391 uint32_t opr_sz
, uint32_t max_sz
)
393 static const TCGOpcode vecop
[] = { INDEX_op_rotli_vec
, 0 };
394 static const GVecGen3i ops
[4] = {
395 { .fni8
= gen_xar8_i64
,
397 .fno
= gen_helper_sve2_xar_b
,
400 { .fni8
= gen_xar16_i64
,
402 .fno
= gen_helper_sve2_xar_h
,
405 { .fni4
= gen_xar_i32
,
407 .fno
= gen_helper_sve2_xar_s
,
410 { .fni8
= gen_xar_i64
,
412 .fno
= gen_helper_gvec_xar_d
,
416 int esize
= 8 << vece
;
418 /* The SVE2 range is 1 .. esize; the AdvSIMD range is 0 .. esize-1. */
419 tcg_debug_assert(shift
>= 0);
420 tcg_debug_assert(shift
<= esize
);
424 /* xar with no rotate devolves to xor. */
425 tcg_gen_gvec_xor(vece
, rd_ofs
, rn_ofs
, rm_ofs
, opr_sz
, max_sz
);
427 tcg_gen_gvec_3i(rd_ofs
, rn_ofs
, rm_ofs
, opr_sz
, max_sz
,
432 static bool trans_XAR(DisasContext
*s
, arg_rrri_esz
*a
)
434 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
437 if (sve_access_check(s
)) {
438 unsigned vsz
= vec_full_reg_size(s
);
439 gen_gvec_xar(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
440 vec_full_reg_offset(s
, a
->rn
),
441 vec_full_reg_offset(s
, a
->rm
), a
->imm
, vsz
, vsz
);
446 static bool do_sve2_zzzz_fn(DisasContext
*s
, arg_rrrr_esz
*a
, GVecGen4Fn
*fn
)
448 if (!dc_isar_feature(aa64_sve2
, s
)) {
451 if (sve_access_check(s
)) {
452 gen_gvec_fn_zzzz(s
, fn
, a
->esz
, a
->rd
, a
->rn
, a
->rm
, a
->ra
);
457 static void gen_eor3_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
459 tcg_gen_xor_i64(d
, n
, m
);
460 tcg_gen_xor_i64(d
, d
, k
);
463 static void gen_eor3_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
464 TCGv_vec m
, TCGv_vec k
)
466 tcg_gen_xor_vec(vece
, d
, n
, m
);
467 tcg_gen_xor_vec(vece
, d
, d
, k
);
470 static void gen_eor3(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
471 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
473 static const GVecGen4 op
= {
474 .fni8
= gen_eor3_i64
,
475 .fniv
= gen_eor3_vec
,
476 .fno
= gen_helper_sve2_eor3
,
478 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
480 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
483 static bool trans_EOR3(DisasContext
*s
, arg_rrrr_esz
*a
)
485 return do_sve2_zzzz_fn(s
, a
, gen_eor3
);
488 static void gen_bcax_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
490 tcg_gen_andc_i64(d
, m
, k
);
491 tcg_gen_xor_i64(d
, d
, n
);
494 static void gen_bcax_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
495 TCGv_vec m
, TCGv_vec k
)
497 tcg_gen_andc_vec(vece
, d
, m
, k
);
498 tcg_gen_xor_vec(vece
, d
, d
, n
);
501 static void gen_bcax(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
502 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
504 static const GVecGen4 op
= {
505 .fni8
= gen_bcax_i64
,
506 .fniv
= gen_bcax_vec
,
507 .fno
= gen_helper_sve2_bcax
,
509 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
511 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
514 static bool trans_BCAX(DisasContext
*s
, arg_rrrr_esz
*a
)
516 return do_sve2_zzzz_fn(s
, a
, gen_bcax
);
519 static void gen_bsl(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
520 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
522 /* BSL differs from the generic bitsel in argument ordering. */
523 tcg_gen_gvec_bitsel(vece
, d
, a
, n
, m
, oprsz
, maxsz
);
526 static bool trans_BSL(DisasContext
*s
, arg_rrrr_esz
*a
)
528 return do_sve2_zzzz_fn(s
, a
, gen_bsl
);
531 static void gen_bsl1n_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
533 tcg_gen_andc_i64(n
, k
, n
);
534 tcg_gen_andc_i64(m
, m
, k
);
535 tcg_gen_or_i64(d
, n
, m
);
538 static void gen_bsl1n_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
539 TCGv_vec m
, TCGv_vec k
)
541 if (TCG_TARGET_HAS_bitsel_vec
) {
542 tcg_gen_not_vec(vece
, n
, n
);
543 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
545 tcg_gen_andc_vec(vece
, n
, k
, n
);
546 tcg_gen_andc_vec(vece
, m
, m
, k
);
547 tcg_gen_or_vec(vece
, d
, n
, m
);
551 static void gen_bsl1n(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
552 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
554 static const GVecGen4 op
= {
555 .fni8
= gen_bsl1n_i64
,
556 .fniv
= gen_bsl1n_vec
,
557 .fno
= gen_helper_sve2_bsl1n
,
559 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
561 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
564 static bool trans_BSL1N(DisasContext
*s
, arg_rrrr_esz
*a
)
566 return do_sve2_zzzz_fn(s
, a
, gen_bsl1n
);
569 static void gen_bsl2n_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
572 * Z[dn] = (n & k) | (~m & ~k)
575 tcg_gen_and_i64(n
, n
, k
);
576 if (TCG_TARGET_HAS_orc_i64
) {
577 tcg_gen_or_i64(m
, m
, k
);
578 tcg_gen_orc_i64(d
, n
, m
);
580 tcg_gen_nor_i64(m
, m
, k
);
581 tcg_gen_or_i64(d
, n
, m
);
585 static void gen_bsl2n_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
586 TCGv_vec m
, TCGv_vec k
)
588 if (TCG_TARGET_HAS_bitsel_vec
) {
589 tcg_gen_not_vec(vece
, m
, m
);
590 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
592 tcg_gen_and_vec(vece
, n
, n
, k
);
593 tcg_gen_or_vec(vece
, m
, m
, k
);
594 tcg_gen_orc_vec(vece
, d
, n
, m
);
598 static void gen_bsl2n(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
599 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
601 static const GVecGen4 op
= {
602 .fni8
= gen_bsl2n_i64
,
603 .fniv
= gen_bsl2n_vec
,
604 .fno
= gen_helper_sve2_bsl2n
,
606 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
608 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
611 static bool trans_BSL2N(DisasContext
*s
, arg_rrrr_esz
*a
)
613 return do_sve2_zzzz_fn(s
, a
, gen_bsl2n
);
616 static void gen_nbsl_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
618 tcg_gen_and_i64(n
, n
, k
);
619 tcg_gen_andc_i64(m
, m
, k
);
620 tcg_gen_nor_i64(d
, n
, m
);
623 static void gen_nbsl_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
624 TCGv_vec m
, TCGv_vec k
)
626 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
627 tcg_gen_not_vec(vece
, d
, d
);
630 static void gen_nbsl(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
631 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
633 static const GVecGen4 op
= {
634 .fni8
= gen_nbsl_i64
,
635 .fniv
= gen_nbsl_vec
,
636 .fno
= gen_helper_sve2_nbsl
,
638 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
640 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
643 static bool trans_NBSL(DisasContext
*s
, arg_rrrr_esz
*a
)
645 return do_sve2_zzzz_fn(s
, a
, gen_nbsl
);
649 *** SVE Integer Arithmetic - Unpredicated Group
652 static bool trans_ADD_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
654 return do_zzz_fn(s
, a
, tcg_gen_gvec_add
);
657 static bool trans_SUB_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
659 return do_zzz_fn(s
, a
, tcg_gen_gvec_sub
);
662 static bool trans_SQADD_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
664 return do_zzz_fn(s
, a
, tcg_gen_gvec_ssadd
);
667 static bool trans_SQSUB_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
669 return do_zzz_fn(s
, a
, tcg_gen_gvec_sssub
);
672 static bool trans_UQADD_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
674 return do_zzz_fn(s
, a
, tcg_gen_gvec_usadd
);
677 static bool trans_UQSUB_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
679 return do_zzz_fn(s
, a
, tcg_gen_gvec_ussub
);
683 *** SVE Integer Arithmetic - Binary Predicated Group
686 static bool do_zpzz_ool(DisasContext
*s
, arg_rprr_esz
*a
, gen_helper_gvec_4
*fn
)
691 if (sve_access_check(s
)) {
692 gen_gvec_ool_zzzp(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->pg
, 0);
697 /* Select active elememnts from Zn and inactive elements from Zm,
698 * storing the result in Zd.
700 static void do_sel_z(DisasContext
*s
, int rd
, int rn
, int rm
, int pg
, int esz
)
702 static gen_helper_gvec_4
* const fns
[4] = {
703 gen_helper_sve_sel_zpzz_b
, gen_helper_sve_sel_zpzz_h
,
704 gen_helper_sve_sel_zpzz_s
, gen_helper_sve_sel_zpzz_d
706 gen_gvec_ool_zzzp(s
, fns
[esz
], rd
, rn
, rm
, pg
, 0);
709 #define DO_ZPZZ(NAME, name) \
710 static bool trans_##NAME##_zpzz(DisasContext *s, arg_rprr_esz *a) \
712 static gen_helper_gvec_4 * const fns[4] = { \
713 gen_helper_sve_##name##_zpzz_b, gen_helper_sve_##name##_zpzz_h, \
714 gen_helper_sve_##name##_zpzz_s, gen_helper_sve_##name##_zpzz_d, \
716 return do_zpzz_ool(s, a, fns[a->esz]); \
735 DO_ZPZZ(SMULH
, smulh
)
736 DO_ZPZZ(UMULH
, umulh
)
742 static bool trans_SDIV_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
744 static gen_helper_gvec_4
* const fns
[4] = {
745 NULL
, NULL
, gen_helper_sve_sdiv_zpzz_s
, gen_helper_sve_sdiv_zpzz_d
747 return do_zpzz_ool(s
, a
, fns
[a
->esz
]);
750 static bool trans_UDIV_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
752 static gen_helper_gvec_4
* const fns
[4] = {
753 NULL
, NULL
, gen_helper_sve_udiv_zpzz_s
, gen_helper_sve_udiv_zpzz_d
755 return do_zpzz_ool(s
, a
, fns
[a
->esz
]);
758 static bool trans_SEL_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
760 if (sve_access_check(s
)) {
761 do_sel_z(s
, a
->rd
, a
->rn
, a
->rm
, a
->pg
, a
->esz
);
769 *** SVE Integer Arithmetic - Unary Predicated Group
772 static bool do_zpz_ool(DisasContext
*s
, arg_rpr_esz
*a
, gen_helper_gvec_3
*fn
)
777 if (sve_access_check(s
)) {
778 gen_gvec_ool_zzp(s
, fn
, a
->rd
, a
->rn
, a
->pg
, 0);
783 #define DO_ZPZ(NAME, name) \
784 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
786 static gen_helper_gvec_3 * const fns[4] = { \
787 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
788 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
790 return do_zpz_ool(s, a, fns[a->esz]); \
795 DO_ZPZ(CNT_zpz
, cnt_zpz
)
797 DO_ZPZ(NOT_zpz
, not_zpz
)
801 static bool trans_FABS(DisasContext
*s
, arg_rpr_esz
*a
)
803 static gen_helper_gvec_3
* const fns
[4] = {
805 gen_helper_sve_fabs_h
,
806 gen_helper_sve_fabs_s
,
807 gen_helper_sve_fabs_d
809 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
812 static bool trans_FNEG(DisasContext
*s
, arg_rpr_esz
*a
)
814 static gen_helper_gvec_3
* const fns
[4] = {
816 gen_helper_sve_fneg_h
,
817 gen_helper_sve_fneg_s
,
818 gen_helper_sve_fneg_d
820 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
823 static bool trans_SXTB(DisasContext
*s
, arg_rpr_esz
*a
)
825 static gen_helper_gvec_3
* const fns
[4] = {
827 gen_helper_sve_sxtb_h
,
828 gen_helper_sve_sxtb_s
,
829 gen_helper_sve_sxtb_d
831 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
834 static bool trans_UXTB(DisasContext
*s
, arg_rpr_esz
*a
)
836 static gen_helper_gvec_3
* const fns
[4] = {
838 gen_helper_sve_uxtb_h
,
839 gen_helper_sve_uxtb_s
,
840 gen_helper_sve_uxtb_d
842 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
845 static bool trans_SXTH(DisasContext
*s
, arg_rpr_esz
*a
)
847 static gen_helper_gvec_3
* const fns
[4] = {
849 gen_helper_sve_sxth_s
,
850 gen_helper_sve_sxth_d
852 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
855 static bool trans_UXTH(DisasContext
*s
, arg_rpr_esz
*a
)
857 static gen_helper_gvec_3
* const fns
[4] = {
859 gen_helper_sve_uxth_s
,
860 gen_helper_sve_uxth_d
862 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
865 static bool trans_SXTW(DisasContext
*s
, arg_rpr_esz
*a
)
867 return do_zpz_ool(s
, a
, a
->esz
== 3 ? gen_helper_sve_sxtw_d
: NULL
);
870 static bool trans_UXTW(DisasContext
*s
, arg_rpr_esz
*a
)
872 return do_zpz_ool(s
, a
, a
->esz
== 3 ? gen_helper_sve_uxtw_d
: NULL
);
878 *** SVE Integer Reduction Group
881 typedef void gen_helper_gvec_reduc(TCGv_i64
, TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
882 static bool do_vpz_ool(DisasContext
*s
, arg_rpr_esz
*a
,
883 gen_helper_gvec_reduc
*fn
)
885 unsigned vsz
= vec_full_reg_size(s
);
893 if (!sve_access_check(s
)) {
897 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
898 temp
= tcg_temp_new_i64();
899 t_zn
= tcg_temp_new_ptr();
900 t_pg
= tcg_temp_new_ptr();
902 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
903 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
904 fn(temp
, t_zn
, t_pg
, desc
);
905 tcg_temp_free_ptr(t_zn
);
906 tcg_temp_free_ptr(t_pg
);
907 tcg_temp_free_i32(desc
);
909 write_fp_dreg(s
, a
->rd
, temp
);
910 tcg_temp_free_i64(temp
);
914 #define DO_VPZ(NAME, name) \
915 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
917 static gen_helper_gvec_reduc * const fns[4] = { \
918 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
919 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
921 return do_vpz_ool(s, a, fns[a->esz]); \
934 static bool trans_SADDV(DisasContext
*s
, arg_rpr_esz
*a
)
936 static gen_helper_gvec_reduc
* const fns
[4] = {
937 gen_helper_sve_saddv_b
, gen_helper_sve_saddv_h
,
938 gen_helper_sve_saddv_s
, NULL
940 return do_vpz_ool(s
, a
, fns
[a
->esz
]);
946 *** SVE Shift by Immediate - Predicated Group
950 * Copy Zn into Zd, storing zeros into inactive elements.
951 * If invert, store zeros into the active elements.
953 static bool do_movz_zpz(DisasContext
*s
, int rd
, int rn
, int pg
,
954 int esz
, bool invert
)
956 static gen_helper_gvec_3
* const fns
[4] = {
957 gen_helper_sve_movz_b
, gen_helper_sve_movz_h
,
958 gen_helper_sve_movz_s
, gen_helper_sve_movz_d
,
961 if (sve_access_check(s
)) {
962 gen_gvec_ool_zzp(s
, fns
[esz
], rd
, rn
, pg
, invert
);
967 static bool do_zpzi_ool(DisasContext
*s
, arg_rpri_esz
*a
,
968 gen_helper_gvec_3
*fn
)
970 if (sve_access_check(s
)) {
971 gen_gvec_ool_zzp(s
, fn
, a
->rd
, a
->rn
, a
->pg
, a
->imm
);
976 static bool trans_ASR_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
978 static gen_helper_gvec_3
* const fns
[4] = {
979 gen_helper_sve_asr_zpzi_b
, gen_helper_sve_asr_zpzi_h
,
980 gen_helper_sve_asr_zpzi_s
, gen_helper_sve_asr_zpzi_d
,
983 /* Invalid tsz encoding -- see tszimm_esz. */
986 /* Shift by element size is architecturally valid. For
987 arithmetic right-shift, it's the same as by one less. */
988 a
->imm
= MIN(a
->imm
, (8 << a
->esz
) - 1);
989 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
992 static bool trans_LSR_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
994 static gen_helper_gvec_3
* const fns
[4] = {
995 gen_helper_sve_lsr_zpzi_b
, gen_helper_sve_lsr_zpzi_h
,
996 gen_helper_sve_lsr_zpzi_s
, gen_helper_sve_lsr_zpzi_d
,
1001 /* Shift by element size is architecturally valid.
1002 For logical shifts, it is a zeroing operation. */
1003 if (a
->imm
>= (8 << a
->esz
)) {
1004 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
1006 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1010 static bool trans_LSL_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
1012 static gen_helper_gvec_3
* const fns
[4] = {
1013 gen_helper_sve_lsl_zpzi_b
, gen_helper_sve_lsl_zpzi_h
,
1014 gen_helper_sve_lsl_zpzi_s
, gen_helper_sve_lsl_zpzi_d
,
1019 /* Shift by element size is architecturally valid.
1020 For logical shifts, it is a zeroing operation. */
1021 if (a
->imm
>= (8 << a
->esz
)) {
1022 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
1024 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1028 static bool trans_ASRD(DisasContext
*s
, arg_rpri_esz
*a
)
1030 static gen_helper_gvec_3
* const fns
[4] = {
1031 gen_helper_sve_asrd_b
, gen_helper_sve_asrd_h
,
1032 gen_helper_sve_asrd_s
, gen_helper_sve_asrd_d
,
1037 /* Shift by element size is architecturally valid. For arithmetic
1038 right shift for division, it is a zeroing operation. */
1039 if (a
->imm
>= (8 << a
->esz
)) {
1040 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
1042 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1046 static bool trans_SQSHL_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
1048 static gen_helper_gvec_3
* const fns
[4] = {
1049 gen_helper_sve2_sqshl_zpzi_b
, gen_helper_sve2_sqshl_zpzi_h
,
1050 gen_helper_sve2_sqshl_zpzi_s
, gen_helper_sve2_sqshl_zpzi_d
,
1052 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
1055 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1058 static bool trans_UQSHL_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
1060 static gen_helper_gvec_3
* const fns
[4] = {
1061 gen_helper_sve2_uqshl_zpzi_b
, gen_helper_sve2_uqshl_zpzi_h
,
1062 gen_helper_sve2_uqshl_zpzi_s
, gen_helper_sve2_uqshl_zpzi_d
,
1064 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
1067 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1070 static bool trans_SRSHR(DisasContext
*s
, arg_rpri_esz
*a
)
1072 static gen_helper_gvec_3
* const fns
[4] = {
1073 gen_helper_sve2_srshr_b
, gen_helper_sve2_srshr_h
,
1074 gen_helper_sve2_srshr_s
, gen_helper_sve2_srshr_d
,
1076 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
1079 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1082 static bool trans_URSHR(DisasContext
*s
, arg_rpri_esz
*a
)
1084 static gen_helper_gvec_3
* const fns
[4] = {
1085 gen_helper_sve2_urshr_b
, gen_helper_sve2_urshr_h
,
1086 gen_helper_sve2_urshr_s
, gen_helper_sve2_urshr_d
,
1088 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
1091 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1094 static bool trans_SQSHLU(DisasContext
*s
, arg_rpri_esz
*a
)
1096 static gen_helper_gvec_3
* const fns
[4] = {
1097 gen_helper_sve2_sqshlu_b
, gen_helper_sve2_sqshlu_h
,
1098 gen_helper_sve2_sqshlu_s
, gen_helper_sve2_sqshlu_d
,
1100 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
1103 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1107 *** SVE Bitwise Shift - Predicated Group
1110 #define DO_ZPZW(NAME, name) \
1111 static bool trans_##NAME##_zpzw(DisasContext *s, arg_rprr_esz *a) \
1113 static gen_helper_gvec_4 * const fns[3] = { \
1114 gen_helper_sve_##name##_zpzw_b, gen_helper_sve_##name##_zpzw_h, \
1115 gen_helper_sve_##name##_zpzw_s, \
1117 if (a->esz < 0 || a->esz >= 3) { \
1120 return do_zpzz_ool(s, a, fns[a->esz]); \
1130 *** SVE Bitwise Shift - Unpredicated Group
1133 static bool do_shift_imm(DisasContext
*s
, arg_rri_esz
*a
, bool asr
,
1134 void (*gvec_fn
)(unsigned, uint32_t, uint32_t,
1135 int64_t, uint32_t, uint32_t))
1138 /* Invalid tsz encoding -- see tszimm_esz. */
1141 if (sve_access_check(s
)) {
1142 unsigned vsz
= vec_full_reg_size(s
);
1143 /* Shift by element size is architecturally valid. For
1144 arithmetic right-shift, it's the same as by one less.
1145 Otherwise it is a zeroing operation. */
1146 if (a
->imm
>= 8 << a
->esz
) {
1148 a
->imm
= (8 << a
->esz
) - 1;
1150 do_dupi_z(s
, a
->rd
, 0);
1154 gvec_fn(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
1155 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
1160 static bool trans_ASR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
1162 return do_shift_imm(s
, a
, true, tcg_gen_gvec_sari
);
1165 static bool trans_LSR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
1167 return do_shift_imm(s
, a
, false, tcg_gen_gvec_shri
);
1170 static bool trans_LSL_zzi(DisasContext
*s
, arg_rri_esz
*a
)
1172 return do_shift_imm(s
, a
, false, tcg_gen_gvec_shli
);
1175 static bool do_zzw_ool(DisasContext
*s
, arg_rrr_esz
*a
, gen_helper_gvec_3
*fn
)
1180 if (sve_access_check(s
)) {
1181 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, 0);
1186 #define DO_ZZW(NAME, name) \
1187 static bool trans_##NAME##_zzw(DisasContext *s, arg_rrr_esz *a) \
1189 static gen_helper_gvec_3 * const fns[4] = { \
1190 gen_helper_sve_##name##_zzw_b, gen_helper_sve_##name##_zzw_h, \
1191 gen_helper_sve_##name##_zzw_s, NULL \
1193 return do_zzw_ool(s, a, fns[a->esz]); \
1203 *** SVE Integer Multiply-Add Group
1206 static bool do_zpzzz_ool(DisasContext
*s
, arg_rprrr_esz
*a
,
1207 gen_helper_gvec_5
*fn
)
1209 if (sve_access_check(s
)) {
1210 unsigned vsz
= vec_full_reg_size(s
);
1211 tcg_gen_gvec_5_ool(vec_full_reg_offset(s
, a
->rd
),
1212 vec_full_reg_offset(s
, a
->ra
),
1213 vec_full_reg_offset(s
, a
->rn
),
1214 vec_full_reg_offset(s
, a
->rm
),
1215 pred_full_reg_offset(s
, a
->pg
),
1221 #define DO_ZPZZZ(NAME, name) \
1222 static bool trans_##NAME(DisasContext *s, arg_rprrr_esz *a) \
1224 static gen_helper_gvec_5 * const fns[4] = { \
1225 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
1226 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
1228 return do_zpzzz_ool(s, a, fns[a->esz]); \
1237 *** SVE Index Generation Group
1240 static void do_index(DisasContext
*s
, int esz
, int rd
,
1241 TCGv_i64 start
, TCGv_i64 incr
)
1243 unsigned vsz
= vec_full_reg_size(s
);
1244 TCGv_i32 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
1245 TCGv_ptr t_zd
= tcg_temp_new_ptr();
1247 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, rd
));
1249 gen_helper_sve_index_d(t_zd
, start
, incr
, desc
);
1251 typedef void index_fn(TCGv_ptr
, TCGv_i32
, TCGv_i32
, TCGv_i32
);
1252 static index_fn
* const fns
[3] = {
1253 gen_helper_sve_index_b
,
1254 gen_helper_sve_index_h
,
1255 gen_helper_sve_index_s
,
1257 TCGv_i32 s32
= tcg_temp_new_i32();
1258 TCGv_i32 i32
= tcg_temp_new_i32();
1260 tcg_gen_extrl_i64_i32(s32
, start
);
1261 tcg_gen_extrl_i64_i32(i32
, incr
);
1262 fns
[esz
](t_zd
, s32
, i32
, desc
);
1264 tcg_temp_free_i32(s32
);
1265 tcg_temp_free_i32(i32
);
1267 tcg_temp_free_ptr(t_zd
);
1268 tcg_temp_free_i32(desc
);
1271 static bool trans_INDEX_ii(DisasContext
*s
, arg_INDEX_ii
*a
)
1273 if (sve_access_check(s
)) {
1274 TCGv_i64 start
= tcg_const_i64(a
->imm1
);
1275 TCGv_i64 incr
= tcg_const_i64(a
->imm2
);
1276 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1277 tcg_temp_free_i64(start
);
1278 tcg_temp_free_i64(incr
);
1283 static bool trans_INDEX_ir(DisasContext
*s
, arg_INDEX_ir
*a
)
1285 if (sve_access_check(s
)) {
1286 TCGv_i64 start
= tcg_const_i64(a
->imm
);
1287 TCGv_i64 incr
= cpu_reg(s
, a
->rm
);
1288 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1289 tcg_temp_free_i64(start
);
1294 static bool trans_INDEX_ri(DisasContext
*s
, arg_INDEX_ri
*a
)
1296 if (sve_access_check(s
)) {
1297 TCGv_i64 start
= cpu_reg(s
, a
->rn
);
1298 TCGv_i64 incr
= tcg_const_i64(a
->imm
);
1299 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1300 tcg_temp_free_i64(incr
);
1305 static bool trans_INDEX_rr(DisasContext
*s
, arg_INDEX_rr
*a
)
1307 if (sve_access_check(s
)) {
1308 TCGv_i64 start
= cpu_reg(s
, a
->rn
);
1309 TCGv_i64 incr
= cpu_reg(s
, a
->rm
);
1310 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1316 *** SVE Stack Allocation Group
1319 static bool trans_ADDVL(DisasContext
*s
, arg_ADDVL
*a
)
1321 if (sve_access_check(s
)) {
1322 TCGv_i64 rd
= cpu_reg_sp(s
, a
->rd
);
1323 TCGv_i64 rn
= cpu_reg_sp(s
, a
->rn
);
1324 tcg_gen_addi_i64(rd
, rn
, a
->imm
* vec_full_reg_size(s
));
1329 static bool trans_ADDPL(DisasContext
*s
, arg_ADDPL
*a
)
1331 if (sve_access_check(s
)) {
1332 TCGv_i64 rd
= cpu_reg_sp(s
, a
->rd
);
1333 TCGv_i64 rn
= cpu_reg_sp(s
, a
->rn
);
1334 tcg_gen_addi_i64(rd
, rn
, a
->imm
* pred_full_reg_size(s
));
1339 static bool trans_RDVL(DisasContext
*s
, arg_RDVL
*a
)
1341 if (sve_access_check(s
)) {
1342 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
1343 tcg_gen_movi_i64(reg
, a
->imm
* vec_full_reg_size(s
));
1349 *** SVE Compute Vector Address Group
1352 static bool do_adr(DisasContext
*s
, arg_rrri
*a
, gen_helper_gvec_3
*fn
)
1354 if (sve_access_check(s
)) {
1355 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->imm
);
1360 static bool trans_ADR_p32(DisasContext
*s
, arg_rrri
*a
)
1362 return do_adr(s
, a
, gen_helper_sve_adr_p32
);
1365 static bool trans_ADR_p64(DisasContext
*s
, arg_rrri
*a
)
1367 return do_adr(s
, a
, gen_helper_sve_adr_p64
);
1370 static bool trans_ADR_s32(DisasContext
*s
, arg_rrri
*a
)
1372 return do_adr(s
, a
, gen_helper_sve_adr_s32
);
1375 static bool trans_ADR_u32(DisasContext
*s
, arg_rrri
*a
)
1377 return do_adr(s
, a
, gen_helper_sve_adr_u32
);
1381 *** SVE Integer Misc - Unpredicated Group
1384 static bool trans_FEXPA(DisasContext
*s
, arg_rr_esz
*a
)
1386 static gen_helper_gvec_2
* const fns
[4] = {
1388 gen_helper_sve_fexpa_h
,
1389 gen_helper_sve_fexpa_s
,
1390 gen_helper_sve_fexpa_d
,
1395 if (sve_access_check(s
)) {
1396 gen_gvec_ool_zz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, 0);
1401 static bool trans_FTSSEL(DisasContext
*s
, arg_rrr_esz
*a
)
1403 static gen_helper_gvec_3
* const fns
[4] = {
1405 gen_helper_sve_ftssel_h
,
1406 gen_helper_sve_ftssel_s
,
1407 gen_helper_sve_ftssel_d
,
1412 if (sve_access_check(s
)) {
1413 gen_gvec_ool_zzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, 0);
1419 *** SVE Predicate Logical Operations Group
1422 static bool do_pppp_flags(DisasContext
*s
, arg_rprr_s
*a
,
1423 const GVecGen4
*gvec_op
)
1425 if (!sve_access_check(s
)) {
1429 unsigned psz
= pred_gvec_reg_size(s
);
1430 int dofs
= pred_full_reg_offset(s
, a
->rd
);
1431 int nofs
= pred_full_reg_offset(s
, a
->rn
);
1432 int mofs
= pred_full_reg_offset(s
, a
->rm
);
1433 int gofs
= pred_full_reg_offset(s
, a
->pg
);
1436 tcg_gen_gvec_4(dofs
, nofs
, mofs
, gofs
, psz
, psz
, gvec_op
);
1441 /* Do the operation and the flags generation in temps. */
1442 TCGv_i64 pd
= tcg_temp_new_i64();
1443 TCGv_i64 pn
= tcg_temp_new_i64();
1444 TCGv_i64 pm
= tcg_temp_new_i64();
1445 TCGv_i64 pg
= tcg_temp_new_i64();
1447 tcg_gen_ld_i64(pn
, cpu_env
, nofs
);
1448 tcg_gen_ld_i64(pm
, cpu_env
, mofs
);
1449 tcg_gen_ld_i64(pg
, cpu_env
, gofs
);
1451 gvec_op
->fni8(pd
, pn
, pm
, pg
);
1452 tcg_gen_st_i64(pd
, cpu_env
, dofs
);
1454 do_predtest1(pd
, pg
);
1456 tcg_temp_free_i64(pd
);
1457 tcg_temp_free_i64(pn
);
1458 tcg_temp_free_i64(pm
);
1459 tcg_temp_free_i64(pg
);
1461 /* The operation and flags generation is large. The computation
1462 * of the flags depends on the original contents of the guarding
1463 * predicate. If the destination overwrites the guarding predicate,
1464 * then the easiest way to get this right is to save a copy.
1467 if (a
->rd
== a
->pg
) {
1468 tofs
= offsetof(CPUARMState
, vfp
.preg_tmp
);
1469 tcg_gen_gvec_mov(0, tofs
, gofs
, psz
, psz
);
1472 tcg_gen_gvec_4(dofs
, nofs
, mofs
, gofs
, psz
, psz
, gvec_op
);
1473 do_predtest(s
, dofs
, tofs
, psz
/ 8);
1478 static void gen_and_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1480 tcg_gen_and_i64(pd
, pn
, pm
);
1481 tcg_gen_and_i64(pd
, pd
, pg
);
1484 static void gen_and_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1485 TCGv_vec pm
, TCGv_vec pg
)
1487 tcg_gen_and_vec(vece
, pd
, pn
, pm
);
1488 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1491 static bool trans_AND_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1493 static const GVecGen4 op
= {
1494 .fni8
= gen_and_pg_i64
,
1495 .fniv
= gen_and_pg_vec
,
1496 .fno
= gen_helper_sve_and_pppp
,
1497 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1501 if (!sve_access_check(s
)) {
1504 if (a
->rn
== a
->rm
) {
1505 if (a
->pg
== a
->rn
) {
1506 do_mov_p(s
, a
->rd
, a
->rn
);
1508 gen_gvec_fn_ppp(s
, tcg_gen_gvec_and
, a
->rd
, a
->rn
, a
->pg
);
1511 } else if (a
->pg
== a
->rn
|| a
->pg
== a
->rm
) {
1512 gen_gvec_fn_ppp(s
, tcg_gen_gvec_and
, a
->rd
, a
->rn
, a
->rm
);
1516 return do_pppp_flags(s
, a
, &op
);
1519 static void gen_bic_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1521 tcg_gen_andc_i64(pd
, pn
, pm
);
1522 tcg_gen_and_i64(pd
, pd
, pg
);
1525 static void gen_bic_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1526 TCGv_vec pm
, TCGv_vec pg
)
1528 tcg_gen_andc_vec(vece
, pd
, pn
, pm
);
1529 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1532 static bool trans_BIC_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1534 static const GVecGen4 op
= {
1535 .fni8
= gen_bic_pg_i64
,
1536 .fniv
= gen_bic_pg_vec
,
1537 .fno
= gen_helper_sve_bic_pppp
,
1538 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1541 if (!a
->s
&& a
->pg
== a
->rn
) {
1542 if (sve_access_check(s
)) {
1543 gen_gvec_fn_ppp(s
, tcg_gen_gvec_andc
, a
->rd
, a
->rn
, a
->rm
);
1547 return do_pppp_flags(s
, a
, &op
);
1550 static void gen_eor_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1552 tcg_gen_xor_i64(pd
, pn
, pm
);
1553 tcg_gen_and_i64(pd
, pd
, pg
);
1556 static void gen_eor_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1557 TCGv_vec pm
, TCGv_vec pg
)
1559 tcg_gen_xor_vec(vece
, pd
, pn
, pm
);
1560 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1563 static bool trans_EOR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1565 static const GVecGen4 op
= {
1566 .fni8
= gen_eor_pg_i64
,
1567 .fniv
= gen_eor_pg_vec
,
1568 .fno
= gen_helper_sve_eor_pppp
,
1569 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1571 return do_pppp_flags(s
, a
, &op
);
1574 static bool trans_SEL_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1579 if (sve_access_check(s
)) {
1580 unsigned psz
= pred_gvec_reg_size(s
);
1581 tcg_gen_gvec_bitsel(MO_8
, pred_full_reg_offset(s
, a
->rd
),
1582 pred_full_reg_offset(s
, a
->pg
),
1583 pred_full_reg_offset(s
, a
->rn
),
1584 pred_full_reg_offset(s
, a
->rm
), psz
, psz
);
1589 static void gen_orr_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1591 tcg_gen_or_i64(pd
, pn
, pm
);
1592 tcg_gen_and_i64(pd
, pd
, pg
);
1595 static void gen_orr_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1596 TCGv_vec pm
, TCGv_vec pg
)
1598 tcg_gen_or_vec(vece
, pd
, pn
, pm
);
1599 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1602 static bool trans_ORR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1604 static const GVecGen4 op
= {
1605 .fni8
= gen_orr_pg_i64
,
1606 .fniv
= gen_orr_pg_vec
,
1607 .fno
= gen_helper_sve_orr_pppp
,
1608 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1611 if (!a
->s
&& a
->pg
== a
->rn
&& a
->rn
== a
->rm
) {
1612 return do_mov_p(s
, a
->rd
, a
->rn
);
1614 return do_pppp_flags(s
, a
, &op
);
1617 static void gen_orn_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1619 tcg_gen_orc_i64(pd
, pn
, pm
);
1620 tcg_gen_and_i64(pd
, pd
, pg
);
1623 static void gen_orn_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1624 TCGv_vec pm
, TCGv_vec pg
)
1626 tcg_gen_orc_vec(vece
, pd
, pn
, pm
);
1627 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1630 static bool trans_ORN_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1632 static const GVecGen4 op
= {
1633 .fni8
= gen_orn_pg_i64
,
1634 .fniv
= gen_orn_pg_vec
,
1635 .fno
= gen_helper_sve_orn_pppp
,
1636 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1638 return do_pppp_flags(s
, a
, &op
);
1641 static void gen_nor_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1643 tcg_gen_or_i64(pd
, pn
, pm
);
1644 tcg_gen_andc_i64(pd
, pg
, pd
);
1647 static void gen_nor_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1648 TCGv_vec pm
, TCGv_vec pg
)
1650 tcg_gen_or_vec(vece
, pd
, pn
, pm
);
1651 tcg_gen_andc_vec(vece
, pd
, pg
, pd
);
1654 static bool trans_NOR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1656 static const GVecGen4 op
= {
1657 .fni8
= gen_nor_pg_i64
,
1658 .fniv
= gen_nor_pg_vec
,
1659 .fno
= gen_helper_sve_nor_pppp
,
1660 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1662 return do_pppp_flags(s
, a
, &op
);
1665 static void gen_nand_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1667 tcg_gen_and_i64(pd
, pn
, pm
);
1668 tcg_gen_andc_i64(pd
, pg
, pd
);
1671 static void gen_nand_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1672 TCGv_vec pm
, TCGv_vec pg
)
1674 tcg_gen_and_vec(vece
, pd
, pn
, pm
);
1675 tcg_gen_andc_vec(vece
, pd
, pg
, pd
);
1678 static bool trans_NAND_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1680 static const GVecGen4 op
= {
1681 .fni8
= gen_nand_pg_i64
,
1682 .fniv
= gen_nand_pg_vec
,
1683 .fno
= gen_helper_sve_nand_pppp
,
1684 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1686 return do_pppp_flags(s
, a
, &op
);
1690 *** SVE Predicate Misc Group
1693 static bool trans_PTEST(DisasContext
*s
, arg_PTEST
*a
)
1695 if (sve_access_check(s
)) {
1696 int nofs
= pred_full_reg_offset(s
, a
->rn
);
1697 int gofs
= pred_full_reg_offset(s
, a
->pg
);
1698 int words
= DIV_ROUND_UP(pred_full_reg_size(s
), 8);
1701 TCGv_i64 pn
= tcg_temp_new_i64();
1702 TCGv_i64 pg
= tcg_temp_new_i64();
1704 tcg_gen_ld_i64(pn
, cpu_env
, nofs
);
1705 tcg_gen_ld_i64(pg
, cpu_env
, gofs
);
1706 do_predtest1(pn
, pg
);
1708 tcg_temp_free_i64(pn
);
1709 tcg_temp_free_i64(pg
);
1711 do_predtest(s
, nofs
, gofs
, words
);
1717 /* See the ARM pseudocode DecodePredCount. */
1718 static unsigned decode_pred_count(unsigned fullsz
, int pattern
, int esz
)
1720 unsigned elements
= fullsz
>> esz
;
1724 case 0x0: /* POW2 */
1725 return pow2floor(elements
);
1736 case 0x9: /* VL16 */
1737 case 0xa: /* VL32 */
1738 case 0xb: /* VL64 */
1739 case 0xc: /* VL128 */
1740 case 0xd: /* VL256 */
1741 bound
= 16 << (pattern
- 9);
1743 case 0x1d: /* MUL4 */
1744 return elements
- elements
% 4;
1745 case 0x1e: /* MUL3 */
1746 return elements
- elements
% 3;
1747 case 0x1f: /* ALL */
1749 default: /* #uimm5 */
1752 return elements
>= bound
? bound
: 0;
1755 /* This handles all of the predicate initialization instructions,
1756 * PTRUE, PFALSE, SETFFR. For PFALSE, we will have set PAT == 32
1757 * so that decode_pred_count returns 0. For SETFFR, we will have
1758 * set RD == 16 == FFR.
1760 static bool do_predset(DisasContext
*s
, int esz
, int rd
, int pat
, bool setflag
)
1762 if (!sve_access_check(s
)) {
1766 unsigned fullsz
= vec_full_reg_size(s
);
1767 unsigned ofs
= pred_full_reg_offset(s
, rd
);
1768 unsigned numelem
, setsz
, i
;
1769 uint64_t word
, lastword
;
1772 numelem
= decode_pred_count(fullsz
, pat
, esz
);
1774 /* Determine what we must store into each bit, and how many. */
1776 lastword
= word
= 0;
1779 setsz
= numelem
<< esz
;
1780 lastword
= word
= pred_esz_masks
[esz
];
1782 lastword
&= MAKE_64BIT_MASK(0, setsz
% 64);
1786 t
= tcg_temp_new_i64();
1788 tcg_gen_movi_i64(t
, lastword
);
1789 tcg_gen_st_i64(t
, cpu_env
, ofs
);
1793 if (word
== lastword
) {
1794 unsigned maxsz
= size_for_gvec(fullsz
/ 8);
1795 unsigned oprsz
= size_for_gvec(setsz
/ 8);
1797 if (oprsz
* 8 == setsz
) {
1798 tcg_gen_gvec_dup_imm(MO_64
, ofs
, oprsz
, maxsz
, word
);
1806 tcg_gen_movi_i64(t
, word
);
1807 for (i
= 0; i
< QEMU_ALIGN_DOWN(setsz
, 8); i
+= 8) {
1808 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1810 if (lastword
!= word
) {
1811 tcg_gen_movi_i64(t
, lastword
);
1812 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1816 tcg_gen_movi_i64(t
, 0);
1817 for (; i
< fullsz
; i
+= 8) {
1818 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1823 tcg_temp_free_i64(t
);
1827 tcg_gen_movi_i32(cpu_NF
, -(word
!= 0));
1828 tcg_gen_movi_i32(cpu_CF
, word
== 0);
1829 tcg_gen_movi_i32(cpu_VF
, 0);
1830 tcg_gen_mov_i32(cpu_ZF
, cpu_NF
);
1835 static bool trans_PTRUE(DisasContext
*s
, arg_PTRUE
*a
)
1837 return do_predset(s
, a
->esz
, a
->rd
, a
->pat
, a
->s
);
1840 static bool trans_SETFFR(DisasContext
*s
, arg_SETFFR
*a
)
1842 /* Note pat == 31 is #all, to set all elements. */
1843 return do_predset(s
, 0, FFR_PRED_NUM
, 31, false);
1846 static bool trans_PFALSE(DisasContext
*s
, arg_PFALSE
*a
)
1848 /* Note pat == 32 is #unimp, to set no elements. */
1849 return do_predset(s
, 0, a
->rd
, 32, false);
1852 static bool trans_RDFFR_p(DisasContext
*s
, arg_RDFFR_p
*a
)
1854 /* The path through do_pppp_flags is complicated enough to want to avoid
1855 * duplication. Frob the arguments into the form of a predicated AND.
1857 arg_rprr_s alt_a
= {
1858 .rd
= a
->rd
, .pg
= a
->pg
, .s
= a
->s
,
1859 .rn
= FFR_PRED_NUM
, .rm
= FFR_PRED_NUM
,
1861 return trans_AND_pppp(s
, &alt_a
);
1864 static bool trans_RDFFR(DisasContext
*s
, arg_RDFFR
*a
)
1866 return do_mov_p(s
, a
->rd
, FFR_PRED_NUM
);
1869 static bool trans_WRFFR(DisasContext
*s
, arg_WRFFR
*a
)
1871 return do_mov_p(s
, FFR_PRED_NUM
, a
->rn
);
1874 static bool do_pfirst_pnext(DisasContext
*s
, arg_rr_esz
*a
,
1875 void (*gen_fn
)(TCGv_i32
, TCGv_ptr
,
1876 TCGv_ptr
, TCGv_i32
))
1878 if (!sve_access_check(s
)) {
1882 TCGv_ptr t_pd
= tcg_temp_new_ptr();
1883 TCGv_ptr t_pg
= tcg_temp_new_ptr();
1887 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, pred_full_reg_size(s
));
1888 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
1890 tcg_gen_addi_ptr(t_pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
1891 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
1892 t
= tcg_const_i32(desc
);
1894 gen_fn(t
, t_pd
, t_pg
, t
);
1895 tcg_temp_free_ptr(t_pd
);
1896 tcg_temp_free_ptr(t_pg
);
1899 tcg_temp_free_i32(t
);
1903 static bool trans_PFIRST(DisasContext
*s
, arg_rr_esz
*a
)
1905 return do_pfirst_pnext(s
, a
, gen_helper_sve_pfirst
);
1908 static bool trans_PNEXT(DisasContext
*s
, arg_rr_esz
*a
)
1910 return do_pfirst_pnext(s
, a
, gen_helper_sve_pnext
);
1914 *** SVE Element Count Group
1917 /* Perform an inline saturating addition of a 32-bit value within
1918 * a 64-bit register. The second operand is known to be positive,
1919 * which halves the comparisions we must perform to bound the result.
1921 static void do_sat_addsub_32(TCGv_i64 reg
, TCGv_i64 val
, bool u
, bool d
)
1927 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
1929 tcg_gen_ext32u_i64(reg
, reg
);
1931 tcg_gen_ext32s_i64(reg
, reg
);
1934 tcg_gen_sub_i64(reg
, reg
, val
);
1935 ibound
= (u
? 0 : INT32_MIN
);
1938 tcg_gen_add_i64(reg
, reg
, val
);
1939 ibound
= (u
? UINT32_MAX
: INT32_MAX
);
1942 bound
= tcg_const_i64(ibound
);
1943 tcg_gen_movcond_i64(cond
, reg
, reg
, bound
, bound
, reg
);
1944 tcg_temp_free_i64(bound
);
1947 /* Similarly with 64-bit values. */
1948 static void do_sat_addsub_64(TCGv_i64 reg
, TCGv_i64 val
, bool u
, bool d
)
1950 TCGv_i64 t0
= tcg_temp_new_i64();
1951 TCGv_i64 t1
= tcg_temp_new_i64();
1956 tcg_gen_sub_i64(t0
, reg
, val
);
1957 tcg_gen_movi_i64(t1
, 0);
1958 tcg_gen_movcond_i64(TCG_COND_LTU
, reg
, reg
, val
, t1
, t0
);
1960 tcg_gen_add_i64(t0
, reg
, val
);
1961 tcg_gen_movi_i64(t1
, -1);
1962 tcg_gen_movcond_i64(TCG_COND_LTU
, reg
, t0
, reg
, t1
, t0
);
1966 /* Detect signed overflow for subtraction. */
1967 tcg_gen_xor_i64(t0
, reg
, val
);
1968 tcg_gen_sub_i64(t1
, reg
, val
);
1969 tcg_gen_xor_i64(reg
, reg
, t1
);
1970 tcg_gen_and_i64(t0
, t0
, reg
);
1972 /* Bound the result. */
1973 tcg_gen_movi_i64(reg
, INT64_MIN
);
1974 t2
= tcg_const_i64(0);
1975 tcg_gen_movcond_i64(TCG_COND_LT
, reg
, t0
, t2
, reg
, t1
);
1977 /* Detect signed overflow for addition. */
1978 tcg_gen_xor_i64(t0
, reg
, val
);
1979 tcg_gen_add_i64(reg
, reg
, val
);
1980 tcg_gen_xor_i64(t1
, reg
, val
);
1981 tcg_gen_andc_i64(t0
, t1
, t0
);
1983 /* Bound the result. */
1984 tcg_gen_movi_i64(t1
, INT64_MAX
);
1985 t2
= tcg_const_i64(0);
1986 tcg_gen_movcond_i64(TCG_COND_LT
, reg
, t0
, t2
, t1
, reg
);
1988 tcg_temp_free_i64(t2
);
1990 tcg_temp_free_i64(t0
);
1991 tcg_temp_free_i64(t1
);
1994 /* Similarly with a vector and a scalar operand. */
1995 static void do_sat_addsub_vec(DisasContext
*s
, int esz
, int rd
, int rn
,
1996 TCGv_i64 val
, bool u
, bool d
)
1998 unsigned vsz
= vec_full_reg_size(s
);
1999 TCGv_ptr dptr
, nptr
;
2003 dptr
= tcg_temp_new_ptr();
2004 nptr
= tcg_temp_new_ptr();
2005 tcg_gen_addi_ptr(dptr
, cpu_env
, vec_full_reg_offset(s
, rd
));
2006 tcg_gen_addi_ptr(nptr
, cpu_env
, vec_full_reg_offset(s
, rn
));
2007 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
2011 t32
= tcg_temp_new_i32();
2012 tcg_gen_extrl_i64_i32(t32
, val
);
2014 tcg_gen_neg_i32(t32
, t32
);
2017 gen_helper_sve_uqaddi_b(dptr
, nptr
, t32
, desc
);
2019 gen_helper_sve_sqaddi_b(dptr
, nptr
, t32
, desc
);
2021 tcg_temp_free_i32(t32
);
2025 t32
= tcg_temp_new_i32();
2026 tcg_gen_extrl_i64_i32(t32
, val
);
2028 tcg_gen_neg_i32(t32
, t32
);
2031 gen_helper_sve_uqaddi_h(dptr
, nptr
, t32
, desc
);
2033 gen_helper_sve_sqaddi_h(dptr
, nptr
, t32
, desc
);
2035 tcg_temp_free_i32(t32
);
2039 t64
= tcg_temp_new_i64();
2041 tcg_gen_neg_i64(t64
, val
);
2043 tcg_gen_mov_i64(t64
, val
);
2046 gen_helper_sve_uqaddi_s(dptr
, nptr
, t64
, desc
);
2048 gen_helper_sve_sqaddi_s(dptr
, nptr
, t64
, desc
);
2050 tcg_temp_free_i64(t64
);
2056 gen_helper_sve_uqsubi_d(dptr
, nptr
, val
, desc
);
2058 gen_helper_sve_uqaddi_d(dptr
, nptr
, val
, desc
);
2061 t64
= tcg_temp_new_i64();
2062 tcg_gen_neg_i64(t64
, val
);
2063 gen_helper_sve_sqaddi_d(dptr
, nptr
, t64
, desc
);
2064 tcg_temp_free_i64(t64
);
2066 gen_helper_sve_sqaddi_d(dptr
, nptr
, val
, desc
);
2071 g_assert_not_reached();
2074 tcg_temp_free_ptr(dptr
);
2075 tcg_temp_free_ptr(nptr
);
2076 tcg_temp_free_i32(desc
);
2079 static bool trans_CNT_r(DisasContext
*s
, arg_CNT_r
*a
)
2081 if (sve_access_check(s
)) {
2082 unsigned fullsz
= vec_full_reg_size(s
);
2083 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2084 tcg_gen_movi_i64(cpu_reg(s
, a
->rd
), numelem
* a
->imm
);
2089 static bool trans_INCDEC_r(DisasContext
*s
, arg_incdec_cnt
*a
)
2091 if (sve_access_check(s
)) {
2092 unsigned fullsz
= vec_full_reg_size(s
);
2093 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2094 int inc
= numelem
* a
->imm
* (a
->d
? -1 : 1);
2095 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
2097 tcg_gen_addi_i64(reg
, reg
, inc
);
2102 static bool trans_SINCDEC_r_32(DisasContext
*s
, arg_incdec_cnt
*a
)
2104 if (!sve_access_check(s
)) {
2108 unsigned fullsz
= vec_full_reg_size(s
);
2109 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2110 int inc
= numelem
* a
->imm
;
2111 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
2113 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
2116 tcg_gen_ext32u_i64(reg
, reg
);
2118 tcg_gen_ext32s_i64(reg
, reg
);
2121 TCGv_i64 t
= tcg_const_i64(inc
);
2122 do_sat_addsub_32(reg
, t
, a
->u
, a
->d
);
2123 tcg_temp_free_i64(t
);
2128 static bool trans_SINCDEC_r_64(DisasContext
*s
, arg_incdec_cnt
*a
)
2130 if (!sve_access_check(s
)) {
2134 unsigned fullsz
= vec_full_reg_size(s
);
2135 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2136 int inc
= numelem
* a
->imm
;
2137 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
2140 TCGv_i64 t
= tcg_const_i64(inc
);
2141 do_sat_addsub_64(reg
, t
, a
->u
, a
->d
);
2142 tcg_temp_free_i64(t
);
2147 static bool trans_INCDEC_v(DisasContext
*s
, arg_incdec2_cnt
*a
)
2153 unsigned fullsz
= vec_full_reg_size(s
);
2154 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2155 int inc
= numelem
* a
->imm
;
2158 if (sve_access_check(s
)) {
2159 TCGv_i64 t
= tcg_const_i64(a
->d
? -inc
: inc
);
2160 tcg_gen_gvec_adds(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
2161 vec_full_reg_offset(s
, a
->rn
),
2163 tcg_temp_free_i64(t
);
2166 do_mov_z(s
, a
->rd
, a
->rn
);
2171 static bool trans_SINCDEC_v(DisasContext
*s
, arg_incdec2_cnt
*a
)
2177 unsigned fullsz
= vec_full_reg_size(s
);
2178 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2179 int inc
= numelem
* a
->imm
;
2182 if (sve_access_check(s
)) {
2183 TCGv_i64 t
= tcg_const_i64(inc
);
2184 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
, t
, a
->u
, a
->d
);
2185 tcg_temp_free_i64(t
);
2188 do_mov_z(s
, a
->rd
, a
->rn
);
2194 *** SVE Bitwise Immediate Group
2197 static bool do_zz_dbm(DisasContext
*s
, arg_rr_dbm
*a
, GVecGen2iFn
*gvec_fn
)
2200 if (!logic_imm_decode_wmask(&imm
, extract32(a
->dbm
, 12, 1),
2201 extract32(a
->dbm
, 0, 6),
2202 extract32(a
->dbm
, 6, 6))) {
2205 if (sve_access_check(s
)) {
2206 unsigned vsz
= vec_full_reg_size(s
);
2207 gvec_fn(MO_64
, vec_full_reg_offset(s
, a
->rd
),
2208 vec_full_reg_offset(s
, a
->rn
), imm
, vsz
, vsz
);
2213 static bool trans_AND_zzi(DisasContext
*s
, arg_rr_dbm
*a
)
2215 return do_zz_dbm(s
, a
, tcg_gen_gvec_andi
);
2218 static bool trans_ORR_zzi(DisasContext
*s
, arg_rr_dbm
*a
)
2220 return do_zz_dbm(s
, a
, tcg_gen_gvec_ori
);
2223 static bool trans_EOR_zzi(DisasContext
*s
, arg_rr_dbm
*a
)
2225 return do_zz_dbm(s
, a
, tcg_gen_gvec_xori
);
2228 static bool trans_DUPM(DisasContext
*s
, arg_DUPM
*a
)
2231 if (!logic_imm_decode_wmask(&imm
, extract32(a
->dbm
, 12, 1),
2232 extract32(a
->dbm
, 0, 6),
2233 extract32(a
->dbm
, 6, 6))) {
2236 if (sve_access_check(s
)) {
2237 do_dupi_z(s
, a
->rd
, imm
);
2243 *** SVE Integer Wide Immediate - Predicated Group
2246 /* Implement all merging copies. This is used for CPY (immediate),
2247 * FCPY, CPY (scalar), CPY (SIMD&FP scalar).
2249 static void do_cpy_m(DisasContext
*s
, int esz
, int rd
, int rn
, int pg
,
2252 typedef void gen_cpy(TCGv_ptr
, TCGv_ptr
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
2253 static gen_cpy
* const fns
[4] = {
2254 gen_helper_sve_cpy_m_b
, gen_helper_sve_cpy_m_h
,
2255 gen_helper_sve_cpy_m_s
, gen_helper_sve_cpy_m_d
,
2257 unsigned vsz
= vec_full_reg_size(s
);
2258 TCGv_i32 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
2259 TCGv_ptr t_zd
= tcg_temp_new_ptr();
2260 TCGv_ptr t_zn
= tcg_temp_new_ptr();
2261 TCGv_ptr t_pg
= tcg_temp_new_ptr();
2263 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, rd
));
2264 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, rn
));
2265 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
2267 fns
[esz
](t_zd
, t_zn
, t_pg
, val
, desc
);
2269 tcg_temp_free_ptr(t_zd
);
2270 tcg_temp_free_ptr(t_zn
);
2271 tcg_temp_free_ptr(t_pg
);
2272 tcg_temp_free_i32(desc
);
2275 static bool trans_FCPY(DisasContext
*s
, arg_FCPY
*a
)
2280 if (sve_access_check(s
)) {
2281 /* Decode the VFP immediate. */
2282 uint64_t imm
= vfp_expand_imm(a
->esz
, a
->imm
);
2283 TCGv_i64 t_imm
= tcg_const_i64(imm
);
2284 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rn
, a
->pg
, t_imm
);
2285 tcg_temp_free_i64(t_imm
);
2290 static bool trans_CPY_m_i(DisasContext
*s
, arg_rpri_esz
*a
)
2292 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
2295 if (sve_access_check(s
)) {
2296 TCGv_i64 t_imm
= tcg_const_i64(a
->imm
);
2297 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rn
, a
->pg
, t_imm
);
2298 tcg_temp_free_i64(t_imm
);
2303 static bool trans_CPY_z_i(DisasContext
*s
, arg_CPY_z_i
*a
)
2305 static gen_helper_gvec_2i
* const fns
[4] = {
2306 gen_helper_sve_cpy_z_b
, gen_helper_sve_cpy_z_h
,
2307 gen_helper_sve_cpy_z_s
, gen_helper_sve_cpy_z_d
,
2310 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
2313 if (sve_access_check(s
)) {
2314 unsigned vsz
= vec_full_reg_size(s
);
2315 TCGv_i64 t_imm
= tcg_const_i64(a
->imm
);
2316 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s
, a
->rd
),
2317 pred_full_reg_offset(s
, a
->pg
),
2318 t_imm
, vsz
, vsz
, 0, fns
[a
->esz
]);
2319 tcg_temp_free_i64(t_imm
);
2325 *** SVE Permute Extract Group
2328 static bool do_EXT(DisasContext
*s
, int rd
, int rn
, int rm
, int imm
)
2330 if (!sve_access_check(s
)) {
2334 unsigned vsz
= vec_full_reg_size(s
);
2335 unsigned n_ofs
= imm
>= vsz
? 0 : imm
;
2336 unsigned n_siz
= vsz
- n_ofs
;
2337 unsigned d
= vec_full_reg_offset(s
, rd
);
2338 unsigned n
= vec_full_reg_offset(s
, rn
);
2339 unsigned m
= vec_full_reg_offset(s
, rm
);
2341 /* Use host vector move insns if we have appropriate sizes
2342 * and no unfortunate overlap.
2345 && n_ofs
== size_for_gvec(n_ofs
)
2346 && n_siz
== size_for_gvec(n_siz
)
2347 && (d
!= n
|| n_siz
<= n_ofs
)) {
2348 tcg_gen_gvec_mov(0, d
, n
+ n_ofs
, n_siz
, n_siz
);
2350 tcg_gen_gvec_mov(0, d
+ n_siz
, m
, n_ofs
, n_ofs
);
2353 tcg_gen_gvec_3_ool(d
, n
, m
, vsz
, vsz
, n_ofs
, gen_helper_sve_ext
);
2358 static bool trans_EXT(DisasContext
*s
, arg_EXT
*a
)
2360 return do_EXT(s
, a
->rd
, a
->rn
, a
->rm
, a
->imm
);
2363 static bool trans_EXT_sve2(DisasContext
*s
, arg_rri
*a
)
2365 if (!dc_isar_feature(aa64_sve2
, s
)) {
2368 return do_EXT(s
, a
->rd
, a
->rn
, (a
->rn
+ 1) % 32, a
->imm
);
2372 *** SVE Permute - Unpredicated Group
2375 static bool trans_DUP_s(DisasContext
*s
, arg_DUP_s
*a
)
2377 if (sve_access_check(s
)) {
2378 unsigned vsz
= vec_full_reg_size(s
);
2379 tcg_gen_gvec_dup_i64(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
2380 vsz
, vsz
, cpu_reg_sp(s
, a
->rn
));
2385 static bool trans_DUP_x(DisasContext
*s
, arg_DUP_x
*a
)
2387 if ((a
->imm
& 0x1f) == 0) {
2390 if (sve_access_check(s
)) {
2391 unsigned vsz
= vec_full_reg_size(s
);
2392 unsigned dofs
= vec_full_reg_offset(s
, a
->rd
);
2393 unsigned esz
, index
;
2395 esz
= ctz32(a
->imm
);
2396 index
= a
->imm
>> (esz
+ 1);
2398 if ((index
<< esz
) < vsz
) {
2399 unsigned nofs
= vec_reg_offset(s
, a
->rn
, index
, esz
);
2400 tcg_gen_gvec_dup_mem(esz
, dofs
, nofs
, vsz
, vsz
);
2403 * While dup_mem handles 128-bit elements, dup_imm does not.
2404 * Thankfully element size doesn't matter for splatting zero.
2406 tcg_gen_gvec_dup_imm(MO_64
, dofs
, vsz
, vsz
, 0);
2412 static void do_insr_i64(DisasContext
*s
, arg_rrr_esz
*a
, TCGv_i64 val
)
2414 typedef void gen_insr(TCGv_ptr
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
2415 static gen_insr
* const fns
[4] = {
2416 gen_helper_sve_insr_b
, gen_helper_sve_insr_h
,
2417 gen_helper_sve_insr_s
, gen_helper_sve_insr_d
,
2419 unsigned vsz
= vec_full_reg_size(s
);
2420 TCGv_i32 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
2421 TCGv_ptr t_zd
= tcg_temp_new_ptr();
2422 TCGv_ptr t_zn
= tcg_temp_new_ptr();
2424 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, a
->rd
));
2425 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
2427 fns
[a
->esz
](t_zd
, t_zn
, val
, desc
);
2429 tcg_temp_free_ptr(t_zd
);
2430 tcg_temp_free_ptr(t_zn
);
2431 tcg_temp_free_i32(desc
);
2434 static bool trans_INSR_f(DisasContext
*s
, arg_rrr_esz
*a
)
2436 if (sve_access_check(s
)) {
2437 TCGv_i64 t
= tcg_temp_new_i64();
2438 tcg_gen_ld_i64(t
, cpu_env
, vec_reg_offset(s
, a
->rm
, 0, MO_64
));
2439 do_insr_i64(s
, a
, t
);
2440 tcg_temp_free_i64(t
);
2445 static bool trans_INSR_r(DisasContext
*s
, arg_rrr_esz
*a
)
2447 if (sve_access_check(s
)) {
2448 do_insr_i64(s
, a
, cpu_reg(s
, a
->rm
));
2453 static bool trans_REV_v(DisasContext
*s
, arg_rr_esz
*a
)
2455 static gen_helper_gvec_2
* const fns
[4] = {
2456 gen_helper_sve_rev_b
, gen_helper_sve_rev_h
,
2457 gen_helper_sve_rev_s
, gen_helper_sve_rev_d
2460 if (sve_access_check(s
)) {
2461 gen_gvec_ool_zz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, 0);
2466 static bool trans_TBL(DisasContext
*s
, arg_rrr_esz
*a
)
2468 static gen_helper_gvec_3
* const fns
[4] = {
2469 gen_helper_sve_tbl_b
, gen_helper_sve_tbl_h
,
2470 gen_helper_sve_tbl_s
, gen_helper_sve_tbl_d
2473 if (sve_access_check(s
)) {
2474 gen_gvec_ool_zzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, 0);
2479 static bool trans_TBL_sve2(DisasContext
*s
, arg_rrr_esz
*a
)
2481 static gen_helper_gvec_4
* const fns
[4] = {
2482 gen_helper_sve2_tbl_b
, gen_helper_sve2_tbl_h
,
2483 gen_helper_sve2_tbl_s
, gen_helper_sve2_tbl_d
2486 if (!dc_isar_feature(aa64_sve2
, s
)) {
2489 if (sve_access_check(s
)) {
2490 gen_gvec_ool_zzzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
,
2491 (a
->rn
+ 1) % 32, a
->rm
, 0);
2496 static bool trans_TBX(DisasContext
*s
, arg_rrr_esz
*a
)
2498 static gen_helper_gvec_3
* const fns
[4] = {
2499 gen_helper_sve2_tbx_b
, gen_helper_sve2_tbx_h
,
2500 gen_helper_sve2_tbx_s
, gen_helper_sve2_tbx_d
2503 if (!dc_isar_feature(aa64_sve2
, s
)) {
2506 if (sve_access_check(s
)) {
2507 gen_gvec_ool_zzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, 0);
2512 static bool trans_UNPK(DisasContext
*s
, arg_UNPK
*a
)
2514 static gen_helper_gvec_2
* const fns
[4][2] = {
2516 { gen_helper_sve_sunpk_h
, gen_helper_sve_uunpk_h
},
2517 { gen_helper_sve_sunpk_s
, gen_helper_sve_uunpk_s
},
2518 { gen_helper_sve_sunpk_d
, gen_helper_sve_uunpk_d
},
2524 if (sve_access_check(s
)) {
2525 unsigned vsz
= vec_full_reg_size(s
);
2526 tcg_gen_gvec_2_ool(vec_full_reg_offset(s
, a
->rd
),
2527 vec_full_reg_offset(s
, a
->rn
)
2528 + (a
->h
? vsz
/ 2 : 0),
2529 vsz
, vsz
, 0, fns
[a
->esz
][a
->u
]);
2535 *** SVE Permute - Predicates Group
2538 static bool do_perm_pred3(DisasContext
*s
, arg_rrr_esz
*a
, bool high_odd
,
2539 gen_helper_gvec_3
*fn
)
2541 if (!sve_access_check(s
)) {
2545 unsigned vsz
= pred_full_reg_size(s
);
2547 TCGv_ptr t_d
= tcg_temp_new_ptr();
2548 TCGv_ptr t_n
= tcg_temp_new_ptr();
2549 TCGv_ptr t_m
= tcg_temp_new_ptr();
2553 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
);
2554 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
2555 desc
= FIELD_DP32(desc
, PREDDESC
, DATA
, high_odd
);
2557 tcg_gen_addi_ptr(t_d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2558 tcg_gen_addi_ptr(t_n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2559 tcg_gen_addi_ptr(t_m
, cpu_env
, pred_full_reg_offset(s
, a
->rm
));
2560 t_desc
= tcg_const_i32(desc
);
2562 fn(t_d
, t_n
, t_m
, t_desc
);
2564 tcg_temp_free_ptr(t_d
);
2565 tcg_temp_free_ptr(t_n
);
2566 tcg_temp_free_ptr(t_m
);
2567 tcg_temp_free_i32(t_desc
);
2571 static bool do_perm_pred2(DisasContext
*s
, arg_rr_esz
*a
, bool high_odd
,
2572 gen_helper_gvec_2
*fn
)
2574 if (!sve_access_check(s
)) {
2578 unsigned vsz
= pred_full_reg_size(s
);
2579 TCGv_ptr t_d
= tcg_temp_new_ptr();
2580 TCGv_ptr t_n
= tcg_temp_new_ptr();
2584 tcg_gen_addi_ptr(t_d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2585 tcg_gen_addi_ptr(t_n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2587 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
);
2588 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
2589 desc
= FIELD_DP32(desc
, PREDDESC
, DATA
, high_odd
);
2590 t_desc
= tcg_const_i32(desc
);
2592 fn(t_d
, t_n
, t_desc
);
2594 tcg_temp_free_i32(t_desc
);
2595 tcg_temp_free_ptr(t_d
);
2596 tcg_temp_free_ptr(t_n
);
2600 static bool trans_ZIP1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2602 return do_perm_pred3(s
, a
, 0, gen_helper_sve_zip_p
);
2605 static bool trans_ZIP2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2607 return do_perm_pred3(s
, a
, 1, gen_helper_sve_zip_p
);
2610 static bool trans_UZP1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2612 return do_perm_pred3(s
, a
, 0, gen_helper_sve_uzp_p
);
2615 static bool trans_UZP2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2617 return do_perm_pred3(s
, a
, 1, gen_helper_sve_uzp_p
);
2620 static bool trans_TRN1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2622 return do_perm_pred3(s
, a
, 0, gen_helper_sve_trn_p
);
2625 static bool trans_TRN2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2627 return do_perm_pred3(s
, a
, 1, gen_helper_sve_trn_p
);
2630 static bool trans_REV_p(DisasContext
*s
, arg_rr_esz
*a
)
2632 return do_perm_pred2(s
, a
, 0, gen_helper_sve_rev_p
);
2635 static bool trans_PUNPKLO(DisasContext
*s
, arg_PUNPKLO
*a
)
2637 return do_perm_pred2(s
, a
, 0, gen_helper_sve_punpk_p
);
2640 static bool trans_PUNPKHI(DisasContext
*s
, arg_PUNPKHI
*a
)
2642 return do_perm_pred2(s
, a
, 1, gen_helper_sve_punpk_p
);
2646 *** SVE Permute - Interleaving Group
2649 static bool do_zip(DisasContext
*s
, arg_rrr_esz
*a
, bool high
)
2651 static gen_helper_gvec_3
* const fns
[4] = {
2652 gen_helper_sve_zip_b
, gen_helper_sve_zip_h
,
2653 gen_helper_sve_zip_s
, gen_helper_sve_zip_d
,
2656 if (sve_access_check(s
)) {
2657 unsigned vsz
= vec_full_reg_size(s
);
2658 unsigned high_ofs
= high
? vsz
/ 2 : 0;
2659 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, a
->rd
),
2660 vec_full_reg_offset(s
, a
->rn
) + high_ofs
,
2661 vec_full_reg_offset(s
, a
->rm
) + high_ofs
,
2662 vsz
, vsz
, 0, fns
[a
->esz
]);
2667 static bool do_zzz_data_ool(DisasContext
*s
, arg_rrr_esz
*a
, int data
,
2668 gen_helper_gvec_3
*fn
)
2670 if (sve_access_check(s
)) {
2671 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, data
);
2676 static bool trans_ZIP1_z(DisasContext
*s
, arg_rrr_esz
*a
)
2678 return do_zip(s
, a
, false);
2681 static bool trans_ZIP2_z(DisasContext
*s
, arg_rrr_esz
*a
)
2683 return do_zip(s
, a
, true);
2686 static bool do_zip_q(DisasContext
*s
, arg_rrr_esz
*a
, bool high
)
2688 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
2691 if (sve_access_check(s
)) {
2692 unsigned vsz
= vec_full_reg_size(s
);
2693 unsigned high_ofs
= high
? QEMU_ALIGN_DOWN(vsz
, 32) / 2 : 0;
2694 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, a
->rd
),
2695 vec_full_reg_offset(s
, a
->rn
) + high_ofs
,
2696 vec_full_reg_offset(s
, a
->rm
) + high_ofs
,
2697 vsz
, vsz
, 0, gen_helper_sve2_zip_q
);
2702 static bool trans_ZIP1_q(DisasContext
*s
, arg_rrr_esz
*a
)
2704 return do_zip_q(s
, a
, false);
2707 static bool trans_ZIP2_q(DisasContext
*s
, arg_rrr_esz
*a
)
2709 return do_zip_q(s
, a
, true);
2712 static gen_helper_gvec_3
* const uzp_fns
[4] = {
2713 gen_helper_sve_uzp_b
, gen_helper_sve_uzp_h
,
2714 gen_helper_sve_uzp_s
, gen_helper_sve_uzp_d
,
2717 static bool trans_UZP1_z(DisasContext
*s
, arg_rrr_esz
*a
)
2719 return do_zzz_data_ool(s
, a
, 0, uzp_fns
[a
->esz
]);
2722 static bool trans_UZP2_z(DisasContext
*s
, arg_rrr_esz
*a
)
2724 return do_zzz_data_ool(s
, a
, 1 << a
->esz
, uzp_fns
[a
->esz
]);
2727 static bool trans_UZP1_q(DisasContext
*s
, arg_rrr_esz
*a
)
2729 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
2732 return do_zzz_data_ool(s
, a
, 0, gen_helper_sve2_uzp_q
);
2735 static bool trans_UZP2_q(DisasContext
*s
, arg_rrr_esz
*a
)
2737 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
2740 return do_zzz_data_ool(s
, a
, 16, gen_helper_sve2_uzp_q
);
2743 static gen_helper_gvec_3
* const trn_fns
[4] = {
2744 gen_helper_sve_trn_b
, gen_helper_sve_trn_h
,
2745 gen_helper_sve_trn_s
, gen_helper_sve_trn_d
,
2748 static bool trans_TRN1_z(DisasContext
*s
, arg_rrr_esz
*a
)
2750 return do_zzz_data_ool(s
, a
, 0, trn_fns
[a
->esz
]);
2753 static bool trans_TRN2_z(DisasContext
*s
, arg_rrr_esz
*a
)
2755 return do_zzz_data_ool(s
, a
, 1 << a
->esz
, trn_fns
[a
->esz
]);
2758 static bool trans_TRN1_q(DisasContext
*s
, arg_rrr_esz
*a
)
2760 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
2763 return do_zzz_data_ool(s
, a
, 0, gen_helper_sve2_trn_q
);
2766 static bool trans_TRN2_q(DisasContext
*s
, arg_rrr_esz
*a
)
2768 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
2771 return do_zzz_data_ool(s
, a
, 16, gen_helper_sve2_trn_q
);
2775 *** SVE Permute Vector - Predicated Group
2778 static bool trans_COMPACT(DisasContext
*s
, arg_rpr_esz
*a
)
2780 static gen_helper_gvec_3
* const fns
[4] = {
2781 NULL
, NULL
, gen_helper_sve_compact_s
, gen_helper_sve_compact_d
2783 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
2786 /* Call the helper that computes the ARM LastActiveElement pseudocode
2787 * function, scaled by the element size. This includes the not found
2788 * indication; e.g. not found for esz=3 is -8.
2790 static void find_last_active(DisasContext
*s
, TCGv_i32 ret
, int esz
, int pg
)
2792 /* Predicate sizes may be smaller and cannot use simd_desc. We cannot
2793 * round up, as we do elsewhere, because we need the exact size.
2795 TCGv_ptr t_p
= tcg_temp_new_ptr();
2799 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, pred_full_reg_size(s
));
2800 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, esz
);
2802 tcg_gen_addi_ptr(t_p
, cpu_env
, pred_full_reg_offset(s
, pg
));
2803 t_desc
= tcg_const_i32(desc
);
2805 gen_helper_sve_last_active_element(ret
, t_p
, t_desc
);
2807 tcg_temp_free_i32(t_desc
);
2808 tcg_temp_free_ptr(t_p
);
2811 /* Increment LAST to the offset of the next element in the vector,
2812 * wrapping around to 0.
2814 static void incr_last_active(DisasContext
*s
, TCGv_i32 last
, int esz
)
2816 unsigned vsz
= vec_full_reg_size(s
);
2818 tcg_gen_addi_i32(last
, last
, 1 << esz
);
2819 if (is_power_of_2(vsz
)) {
2820 tcg_gen_andi_i32(last
, last
, vsz
- 1);
2822 TCGv_i32 max
= tcg_const_i32(vsz
);
2823 TCGv_i32 zero
= tcg_const_i32(0);
2824 tcg_gen_movcond_i32(TCG_COND_GEU
, last
, last
, max
, zero
, last
);
2825 tcg_temp_free_i32(max
);
2826 tcg_temp_free_i32(zero
);
2830 /* If LAST < 0, set LAST to the offset of the last element in the vector. */
2831 static void wrap_last_active(DisasContext
*s
, TCGv_i32 last
, int esz
)
2833 unsigned vsz
= vec_full_reg_size(s
);
2835 if (is_power_of_2(vsz
)) {
2836 tcg_gen_andi_i32(last
, last
, vsz
- 1);
2838 TCGv_i32 max
= tcg_const_i32(vsz
- (1 << esz
));
2839 TCGv_i32 zero
= tcg_const_i32(0);
2840 tcg_gen_movcond_i32(TCG_COND_LT
, last
, last
, zero
, max
, last
);
2841 tcg_temp_free_i32(max
);
2842 tcg_temp_free_i32(zero
);
2846 /* Load an unsigned element of ESZ from BASE+OFS. */
2847 static TCGv_i64
load_esz(TCGv_ptr base
, int ofs
, int esz
)
2849 TCGv_i64 r
= tcg_temp_new_i64();
2853 tcg_gen_ld8u_i64(r
, base
, ofs
);
2856 tcg_gen_ld16u_i64(r
, base
, ofs
);
2859 tcg_gen_ld32u_i64(r
, base
, ofs
);
2862 tcg_gen_ld_i64(r
, base
, ofs
);
2865 g_assert_not_reached();
2870 /* Load an unsigned element of ESZ from RM[LAST]. */
2871 static TCGv_i64
load_last_active(DisasContext
*s
, TCGv_i32 last
,
2874 TCGv_ptr p
= tcg_temp_new_ptr();
2877 /* Convert offset into vector into offset into ENV.
2878 * The final adjustment for the vector register base
2879 * is added via constant offset to the load.
2881 #ifdef HOST_WORDS_BIGENDIAN
2882 /* Adjust for element ordering. See vec_reg_offset. */
2884 tcg_gen_xori_i32(last
, last
, 8 - (1 << esz
));
2887 tcg_gen_ext_i32_ptr(p
, last
);
2888 tcg_gen_add_ptr(p
, p
, cpu_env
);
2890 r
= load_esz(p
, vec_full_reg_offset(s
, rm
), esz
);
2891 tcg_temp_free_ptr(p
);
2896 /* Compute CLAST for a Zreg. */
2897 static bool do_clast_vector(DisasContext
*s
, arg_rprr_esz
*a
, bool before
)
2902 unsigned vsz
, esz
= a
->esz
;
2904 if (!sve_access_check(s
)) {
2908 last
= tcg_temp_local_new_i32();
2909 over
= gen_new_label();
2911 find_last_active(s
, last
, esz
, a
->pg
);
2913 /* There is of course no movcond for a 2048-bit vector,
2914 * so we must branch over the actual store.
2916 tcg_gen_brcondi_i32(TCG_COND_LT
, last
, 0, over
);
2919 incr_last_active(s
, last
, esz
);
2922 ele
= load_last_active(s
, last
, a
->rm
, esz
);
2923 tcg_temp_free_i32(last
);
2925 vsz
= vec_full_reg_size(s
);
2926 tcg_gen_gvec_dup_i64(esz
, vec_full_reg_offset(s
, a
->rd
), vsz
, vsz
, ele
);
2927 tcg_temp_free_i64(ele
);
2929 /* If this insn used MOVPRFX, we may need a second move. */
2930 if (a
->rd
!= a
->rn
) {
2931 TCGLabel
*done
= gen_new_label();
2934 gen_set_label(over
);
2935 do_mov_z(s
, a
->rd
, a
->rn
);
2937 gen_set_label(done
);
2939 gen_set_label(over
);
2944 static bool trans_CLASTA_z(DisasContext
*s
, arg_rprr_esz
*a
)
2946 return do_clast_vector(s
, a
, false);
2949 static bool trans_CLASTB_z(DisasContext
*s
, arg_rprr_esz
*a
)
2951 return do_clast_vector(s
, a
, true);
2954 /* Compute CLAST for a scalar. */
2955 static void do_clast_scalar(DisasContext
*s
, int esz
, int pg
, int rm
,
2956 bool before
, TCGv_i64 reg_val
)
2958 TCGv_i32 last
= tcg_temp_new_i32();
2959 TCGv_i64 ele
, cmp
, zero
;
2961 find_last_active(s
, last
, esz
, pg
);
2963 /* Extend the original value of last prior to incrementing. */
2964 cmp
= tcg_temp_new_i64();
2965 tcg_gen_ext_i32_i64(cmp
, last
);
2968 incr_last_active(s
, last
, esz
);
2971 /* The conceit here is that while last < 0 indicates not found, after
2972 * adjusting for cpu_env->vfp.zregs[rm], it is still a valid address
2973 * from which we can load garbage. We then discard the garbage with
2974 * a conditional move.
2976 ele
= load_last_active(s
, last
, rm
, esz
);
2977 tcg_temp_free_i32(last
);
2979 zero
= tcg_const_i64(0);
2980 tcg_gen_movcond_i64(TCG_COND_GE
, reg_val
, cmp
, zero
, ele
, reg_val
);
2982 tcg_temp_free_i64(zero
);
2983 tcg_temp_free_i64(cmp
);
2984 tcg_temp_free_i64(ele
);
2987 /* Compute CLAST for a Vreg. */
2988 static bool do_clast_fp(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2990 if (sve_access_check(s
)) {
2992 int ofs
= vec_reg_offset(s
, a
->rd
, 0, esz
);
2993 TCGv_i64 reg
= load_esz(cpu_env
, ofs
, esz
);
2995 do_clast_scalar(s
, esz
, a
->pg
, a
->rn
, before
, reg
);
2996 write_fp_dreg(s
, a
->rd
, reg
);
2997 tcg_temp_free_i64(reg
);
3002 static bool trans_CLASTA_v(DisasContext
*s
, arg_rpr_esz
*a
)
3004 return do_clast_fp(s
, a
, false);
3007 static bool trans_CLASTB_v(DisasContext
*s
, arg_rpr_esz
*a
)
3009 return do_clast_fp(s
, a
, true);
3012 /* Compute CLAST for a Xreg. */
3013 static bool do_clast_general(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
3017 if (!sve_access_check(s
)) {
3021 reg
= cpu_reg(s
, a
->rd
);
3024 tcg_gen_ext8u_i64(reg
, reg
);
3027 tcg_gen_ext16u_i64(reg
, reg
);
3030 tcg_gen_ext32u_i64(reg
, reg
);
3035 g_assert_not_reached();
3038 do_clast_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
, reg
);
3042 static bool trans_CLASTA_r(DisasContext
*s
, arg_rpr_esz
*a
)
3044 return do_clast_general(s
, a
, false);
3047 static bool trans_CLASTB_r(DisasContext
*s
, arg_rpr_esz
*a
)
3049 return do_clast_general(s
, a
, true);
3052 /* Compute LAST for a scalar. */
3053 static TCGv_i64
do_last_scalar(DisasContext
*s
, int esz
,
3054 int pg
, int rm
, bool before
)
3056 TCGv_i32 last
= tcg_temp_new_i32();
3059 find_last_active(s
, last
, esz
, pg
);
3061 wrap_last_active(s
, last
, esz
);
3063 incr_last_active(s
, last
, esz
);
3066 ret
= load_last_active(s
, last
, rm
, esz
);
3067 tcg_temp_free_i32(last
);
3071 /* Compute LAST for a Vreg. */
3072 static bool do_last_fp(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
3074 if (sve_access_check(s
)) {
3075 TCGv_i64 val
= do_last_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
);
3076 write_fp_dreg(s
, a
->rd
, val
);
3077 tcg_temp_free_i64(val
);
3082 static bool trans_LASTA_v(DisasContext
*s
, arg_rpr_esz
*a
)
3084 return do_last_fp(s
, a
, false);
3087 static bool trans_LASTB_v(DisasContext
*s
, arg_rpr_esz
*a
)
3089 return do_last_fp(s
, a
, true);
3092 /* Compute LAST for a Xreg. */
3093 static bool do_last_general(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
3095 if (sve_access_check(s
)) {
3096 TCGv_i64 val
= do_last_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
);
3097 tcg_gen_mov_i64(cpu_reg(s
, a
->rd
), val
);
3098 tcg_temp_free_i64(val
);
3103 static bool trans_LASTA_r(DisasContext
*s
, arg_rpr_esz
*a
)
3105 return do_last_general(s
, a
, false);
3108 static bool trans_LASTB_r(DisasContext
*s
, arg_rpr_esz
*a
)
3110 return do_last_general(s
, a
, true);
3113 static bool trans_CPY_m_r(DisasContext
*s
, arg_rpr_esz
*a
)
3115 if (sve_access_check(s
)) {
3116 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rd
, a
->pg
, cpu_reg_sp(s
, a
->rn
));
3121 static bool trans_CPY_m_v(DisasContext
*s
, arg_rpr_esz
*a
)
3123 if (sve_access_check(s
)) {
3124 int ofs
= vec_reg_offset(s
, a
->rn
, 0, a
->esz
);
3125 TCGv_i64 t
= load_esz(cpu_env
, ofs
, a
->esz
);
3126 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rd
, a
->pg
, t
);
3127 tcg_temp_free_i64(t
);
3132 static bool trans_REVB(DisasContext
*s
, arg_rpr_esz
*a
)
3134 static gen_helper_gvec_3
* const fns
[4] = {
3136 gen_helper_sve_revb_h
,
3137 gen_helper_sve_revb_s
,
3138 gen_helper_sve_revb_d
,
3140 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
3143 static bool trans_REVH(DisasContext
*s
, arg_rpr_esz
*a
)
3145 static gen_helper_gvec_3
* const fns
[4] = {
3148 gen_helper_sve_revh_s
,
3149 gen_helper_sve_revh_d
,
3151 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
3154 static bool trans_REVW(DisasContext
*s
, arg_rpr_esz
*a
)
3156 return do_zpz_ool(s
, a
, a
->esz
== 3 ? gen_helper_sve_revw_d
: NULL
);
3159 static bool trans_RBIT(DisasContext
*s
, arg_rpr_esz
*a
)
3161 static gen_helper_gvec_3
* const fns
[4] = {
3162 gen_helper_sve_rbit_b
,
3163 gen_helper_sve_rbit_h
,
3164 gen_helper_sve_rbit_s
,
3165 gen_helper_sve_rbit_d
,
3167 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
3170 static bool trans_SPLICE(DisasContext
*s
, arg_rprr_esz
*a
)
3172 if (sve_access_check(s
)) {
3173 gen_gvec_ool_zzzp(s
, gen_helper_sve_splice
,
3174 a
->rd
, a
->rn
, a
->rm
, a
->pg
, a
->esz
);
3179 static bool trans_SPLICE_sve2(DisasContext
*s
, arg_rpr_esz
*a
)
3181 if (!dc_isar_feature(aa64_sve2
, s
)) {
3184 if (sve_access_check(s
)) {
3185 gen_gvec_ool_zzzp(s
, gen_helper_sve_splice
,
3186 a
->rd
, a
->rn
, (a
->rn
+ 1) % 32, a
->pg
, a
->esz
);
3192 *** SVE Integer Compare - Vectors Group
3195 static bool do_ppzz_flags(DisasContext
*s
, arg_rprr_esz
*a
,
3196 gen_helper_gvec_flags_4
*gen_fn
)
3198 TCGv_ptr pd
, zn
, zm
, pg
;
3202 if (gen_fn
== NULL
) {
3205 if (!sve_access_check(s
)) {
3209 vsz
= vec_full_reg_size(s
);
3210 t
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
3211 pd
= tcg_temp_new_ptr();
3212 zn
= tcg_temp_new_ptr();
3213 zm
= tcg_temp_new_ptr();
3214 pg
= tcg_temp_new_ptr();
3216 tcg_gen_addi_ptr(pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3217 tcg_gen_addi_ptr(zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
3218 tcg_gen_addi_ptr(zm
, cpu_env
, vec_full_reg_offset(s
, a
->rm
));
3219 tcg_gen_addi_ptr(pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3221 gen_fn(t
, pd
, zn
, zm
, pg
, t
);
3223 tcg_temp_free_ptr(pd
);
3224 tcg_temp_free_ptr(zn
);
3225 tcg_temp_free_ptr(zm
);
3226 tcg_temp_free_ptr(pg
);
3230 tcg_temp_free_i32(t
);
3234 #define DO_PPZZ(NAME, name) \
3235 static bool trans_##NAME##_ppzz(DisasContext *s, arg_rprr_esz *a) \
3237 static gen_helper_gvec_flags_4 * const fns[4] = { \
3238 gen_helper_sve_##name##_ppzz_b, gen_helper_sve_##name##_ppzz_h, \
3239 gen_helper_sve_##name##_ppzz_s, gen_helper_sve_##name##_ppzz_d, \
3241 return do_ppzz_flags(s, a, fns[a->esz]); \
3244 DO_PPZZ(CMPEQ
, cmpeq
)
3245 DO_PPZZ(CMPNE
, cmpne
)
3246 DO_PPZZ(CMPGT
, cmpgt
)
3247 DO_PPZZ(CMPGE
, cmpge
)
3248 DO_PPZZ(CMPHI
, cmphi
)
3249 DO_PPZZ(CMPHS
, cmphs
)
3253 #define DO_PPZW(NAME, name) \
3254 static bool trans_##NAME##_ppzw(DisasContext *s, arg_rprr_esz *a) \
3256 static gen_helper_gvec_flags_4 * const fns[4] = { \
3257 gen_helper_sve_##name##_ppzw_b, gen_helper_sve_##name##_ppzw_h, \
3258 gen_helper_sve_##name##_ppzw_s, NULL \
3260 return do_ppzz_flags(s, a, fns[a->esz]); \
3263 DO_PPZW(CMPEQ
, cmpeq
)
3264 DO_PPZW(CMPNE
, cmpne
)
3265 DO_PPZW(CMPGT
, cmpgt
)
3266 DO_PPZW(CMPGE
, cmpge
)
3267 DO_PPZW(CMPHI
, cmphi
)
3268 DO_PPZW(CMPHS
, cmphs
)
3269 DO_PPZW(CMPLT
, cmplt
)
3270 DO_PPZW(CMPLE
, cmple
)
3271 DO_PPZW(CMPLO
, cmplo
)
3272 DO_PPZW(CMPLS
, cmpls
)
3277 *** SVE Integer Compare - Immediate Groups
3280 static bool do_ppzi_flags(DisasContext
*s
, arg_rpri_esz
*a
,
3281 gen_helper_gvec_flags_3
*gen_fn
)
3283 TCGv_ptr pd
, zn
, pg
;
3287 if (gen_fn
== NULL
) {
3290 if (!sve_access_check(s
)) {
3294 vsz
= vec_full_reg_size(s
);
3295 t
= tcg_const_i32(simd_desc(vsz
, vsz
, a
->imm
));
3296 pd
= tcg_temp_new_ptr();
3297 zn
= tcg_temp_new_ptr();
3298 pg
= tcg_temp_new_ptr();
3300 tcg_gen_addi_ptr(pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3301 tcg_gen_addi_ptr(zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
3302 tcg_gen_addi_ptr(pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3304 gen_fn(t
, pd
, zn
, pg
, t
);
3306 tcg_temp_free_ptr(pd
);
3307 tcg_temp_free_ptr(zn
);
3308 tcg_temp_free_ptr(pg
);
3312 tcg_temp_free_i32(t
);
3316 #define DO_PPZI(NAME, name) \
3317 static bool trans_##NAME##_ppzi(DisasContext *s, arg_rpri_esz *a) \
3319 static gen_helper_gvec_flags_3 * const fns[4] = { \
3320 gen_helper_sve_##name##_ppzi_b, gen_helper_sve_##name##_ppzi_h, \
3321 gen_helper_sve_##name##_ppzi_s, gen_helper_sve_##name##_ppzi_d, \
3323 return do_ppzi_flags(s, a, fns[a->esz]); \
3326 DO_PPZI(CMPEQ
, cmpeq
)
3327 DO_PPZI(CMPNE
, cmpne
)
3328 DO_PPZI(CMPGT
, cmpgt
)
3329 DO_PPZI(CMPGE
, cmpge
)
3330 DO_PPZI(CMPHI
, cmphi
)
3331 DO_PPZI(CMPHS
, cmphs
)
3332 DO_PPZI(CMPLT
, cmplt
)
3333 DO_PPZI(CMPLE
, cmple
)
3334 DO_PPZI(CMPLO
, cmplo
)
3335 DO_PPZI(CMPLS
, cmpls
)
3340 *** SVE Partition Break Group
3343 static bool do_brk3(DisasContext
*s
, arg_rprr_s
*a
,
3344 gen_helper_gvec_4
*fn
, gen_helper_gvec_flags_4
*fn_s
)
3346 if (!sve_access_check(s
)) {
3350 unsigned vsz
= pred_full_reg_size(s
);
3352 /* Predicate sizes may be smaller and cannot use simd_desc. */
3353 TCGv_ptr d
= tcg_temp_new_ptr();
3354 TCGv_ptr n
= tcg_temp_new_ptr();
3355 TCGv_ptr m
= tcg_temp_new_ptr();
3356 TCGv_ptr g
= tcg_temp_new_ptr();
3357 TCGv_i32 t
= tcg_const_i32(FIELD_DP32(0, PREDDESC
, OPRSZ
, vsz
));
3359 tcg_gen_addi_ptr(d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3360 tcg_gen_addi_ptr(n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
3361 tcg_gen_addi_ptr(m
, cpu_env
, pred_full_reg_offset(s
, a
->rm
));
3362 tcg_gen_addi_ptr(g
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3365 fn_s(t
, d
, n
, m
, g
, t
);
3370 tcg_temp_free_ptr(d
);
3371 tcg_temp_free_ptr(n
);
3372 tcg_temp_free_ptr(m
);
3373 tcg_temp_free_ptr(g
);
3374 tcg_temp_free_i32(t
);
3378 static bool do_brk2(DisasContext
*s
, arg_rpr_s
*a
,
3379 gen_helper_gvec_3
*fn
, gen_helper_gvec_flags_3
*fn_s
)
3381 if (!sve_access_check(s
)) {
3385 unsigned vsz
= pred_full_reg_size(s
);
3387 /* Predicate sizes may be smaller and cannot use simd_desc. */
3388 TCGv_ptr d
= tcg_temp_new_ptr();
3389 TCGv_ptr n
= tcg_temp_new_ptr();
3390 TCGv_ptr g
= tcg_temp_new_ptr();
3391 TCGv_i32 t
= tcg_const_i32(FIELD_DP32(0, PREDDESC
, OPRSZ
, vsz
));
3393 tcg_gen_addi_ptr(d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3394 tcg_gen_addi_ptr(n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
3395 tcg_gen_addi_ptr(g
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3398 fn_s(t
, d
, n
, g
, t
);
3403 tcg_temp_free_ptr(d
);
3404 tcg_temp_free_ptr(n
);
3405 tcg_temp_free_ptr(g
);
3406 tcg_temp_free_i32(t
);
3410 static bool trans_BRKPA(DisasContext
*s
, arg_rprr_s
*a
)
3412 return do_brk3(s
, a
, gen_helper_sve_brkpa
, gen_helper_sve_brkpas
);
3415 static bool trans_BRKPB(DisasContext
*s
, arg_rprr_s
*a
)
3417 return do_brk3(s
, a
, gen_helper_sve_brkpb
, gen_helper_sve_brkpbs
);
3420 static bool trans_BRKA_m(DisasContext
*s
, arg_rpr_s
*a
)
3422 return do_brk2(s
, a
, gen_helper_sve_brka_m
, gen_helper_sve_brkas_m
);
3425 static bool trans_BRKB_m(DisasContext
*s
, arg_rpr_s
*a
)
3427 return do_brk2(s
, a
, gen_helper_sve_brkb_m
, gen_helper_sve_brkbs_m
);
3430 static bool trans_BRKA_z(DisasContext
*s
, arg_rpr_s
*a
)
3432 return do_brk2(s
, a
, gen_helper_sve_brka_z
, gen_helper_sve_brkas_z
);
3435 static bool trans_BRKB_z(DisasContext
*s
, arg_rpr_s
*a
)
3437 return do_brk2(s
, a
, gen_helper_sve_brkb_z
, gen_helper_sve_brkbs_z
);
3440 static bool trans_BRKN(DisasContext
*s
, arg_rpr_s
*a
)
3442 return do_brk2(s
, a
, gen_helper_sve_brkn
, gen_helper_sve_brkns
);
3446 *** SVE Predicate Count Group
3449 static void do_cntp(DisasContext
*s
, TCGv_i64 val
, int esz
, int pn
, int pg
)
3451 unsigned psz
= pred_full_reg_size(s
);
3456 tcg_gen_ld_i64(val
, cpu_env
, pred_full_reg_offset(s
, pn
));
3458 TCGv_i64 g
= tcg_temp_new_i64();
3459 tcg_gen_ld_i64(g
, cpu_env
, pred_full_reg_offset(s
, pg
));
3460 tcg_gen_and_i64(val
, val
, g
);
3461 tcg_temp_free_i64(g
);
3464 /* Reduce the pred_esz_masks value simply to reduce the
3465 * size of the code generated here.
3467 psz_mask
= MAKE_64BIT_MASK(0, psz
* 8);
3468 tcg_gen_andi_i64(val
, val
, pred_esz_masks
[esz
] & psz_mask
);
3470 tcg_gen_ctpop_i64(val
, val
);
3472 TCGv_ptr t_pn
= tcg_temp_new_ptr();
3473 TCGv_ptr t_pg
= tcg_temp_new_ptr();
3477 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, psz
);
3478 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, esz
);
3480 tcg_gen_addi_ptr(t_pn
, cpu_env
, pred_full_reg_offset(s
, pn
));
3481 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
3482 t_desc
= tcg_const_i32(desc
);
3484 gen_helper_sve_cntp(val
, t_pn
, t_pg
, t_desc
);
3485 tcg_temp_free_ptr(t_pn
);
3486 tcg_temp_free_ptr(t_pg
);
3487 tcg_temp_free_i32(t_desc
);
3491 static bool trans_CNTP(DisasContext
*s
, arg_CNTP
*a
)
3493 if (sve_access_check(s
)) {
3494 do_cntp(s
, cpu_reg(s
, a
->rd
), a
->esz
, a
->rn
, a
->pg
);
3499 static bool trans_INCDECP_r(DisasContext
*s
, arg_incdec_pred
*a
)
3501 if (sve_access_check(s
)) {
3502 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3503 TCGv_i64 val
= tcg_temp_new_i64();
3505 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3507 tcg_gen_sub_i64(reg
, reg
, val
);
3509 tcg_gen_add_i64(reg
, reg
, val
);
3511 tcg_temp_free_i64(val
);
3516 static bool trans_INCDECP_z(DisasContext
*s
, arg_incdec2_pred
*a
)
3521 if (sve_access_check(s
)) {
3522 unsigned vsz
= vec_full_reg_size(s
);
3523 TCGv_i64 val
= tcg_temp_new_i64();
3524 GVecGen2sFn
*gvec_fn
= a
->d
? tcg_gen_gvec_subs
: tcg_gen_gvec_adds
;
3526 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3527 gvec_fn(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3528 vec_full_reg_offset(s
, a
->rn
), val
, vsz
, vsz
);
3533 static bool trans_SINCDECP_r_32(DisasContext
*s
, arg_incdec_pred
*a
)
3535 if (sve_access_check(s
)) {
3536 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3537 TCGv_i64 val
= tcg_temp_new_i64();
3539 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3540 do_sat_addsub_32(reg
, val
, a
->u
, a
->d
);
3545 static bool trans_SINCDECP_r_64(DisasContext
*s
, arg_incdec_pred
*a
)
3547 if (sve_access_check(s
)) {
3548 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3549 TCGv_i64 val
= tcg_temp_new_i64();
3551 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3552 do_sat_addsub_64(reg
, val
, a
->u
, a
->d
);
3557 static bool trans_SINCDECP_z(DisasContext
*s
, arg_incdec2_pred
*a
)
3562 if (sve_access_check(s
)) {
3563 TCGv_i64 val
= tcg_temp_new_i64();
3564 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3565 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
, val
, a
->u
, a
->d
);
3571 *** SVE Integer Compare Scalars Group
3574 static bool trans_CTERM(DisasContext
*s
, arg_CTERM
*a
)
3576 if (!sve_access_check(s
)) {
3580 TCGCond cond
= (a
->ne
? TCG_COND_NE
: TCG_COND_EQ
);
3581 TCGv_i64 rn
= read_cpu_reg(s
, a
->rn
, a
->sf
);
3582 TCGv_i64 rm
= read_cpu_reg(s
, a
->rm
, a
->sf
);
3583 TCGv_i64 cmp
= tcg_temp_new_i64();
3585 tcg_gen_setcond_i64(cond
, cmp
, rn
, rm
);
3586 tcg_gen_extrl_i64_i32(cpu_NF
, cmp
);
3587 tcg_temp_free_i64(cmp
);
3589 /* VF = !NF & !CF. */
3590 tcg_gen_xori_i32(cpu_VF
, cpu_NF
, 1);
3591 tcg_gen_andc_i32(cpu_VF
, cpu_VF
, cpu_CF
);
3593 /* Both NF and VF actually look at bit 31. */
3594 tcg_gen_neg_i32(cpu_NF
, cpu_NF
);
3595 tcg_gen_neg_i32(cpu_VF
, cpu_VF
);
3599 static bool trans_WHILE(DisasContext
*s
, arg_WHILE
*a
)
3601 TCGv_i64 op0
, op1
, t0
, t1
, tmax
;
3604 unsigned vsz
= vec_full_reg_size(s
);
3608 /* Note that GE/HS has a->eq == 0 and GT/HI has a->eq == 1. */
3609 bool eq
= a
->eq
== a
->lt
;
3611 /* The greater-than conditions are all SVE2. */
3612 if (!a
->lt
&& !dc_isar_feature(aa64_sve2
, s
)) {
3615 if (!sve_access_check(s
)) {
3619 op0
= read_cpu_reg(s
, a
->rn
, 1);
3620 op1
= read_cpu_reg(s
, a
->rm
, 1);
3624 tcg_gen_ext32u_i64(op0
, op0
);
3625 tcg_gen_ext32u_i64(op1
, op1
);
3627 tcg_gen_ext32s_i64(op0
, op0
);
3628 tcg_gen_ext32s_i64(op1
, op1
);
3632 /* For the helper, compress the different conditions into a computation
3633 * of how many iterations for which the condition is true.
3635 t0
= tcg_temp_new_i64();
3636 t1
= tcg_temp_new_i64();
3639 tcg_gen_sub_i64(t0
, op1
, op0
);
3641 maxval
= a
->sf
? UINT64_MAX
: UINT32_MAX
;
3642 cond
= eq
? TCG_COND_LEU
: TCG_COND_LTU
;
3644 maxval
= a
->sf
? INT64_MAX
: INT32_MAX
;
3645 cond
= eq
? TCG_COND_LE
: TCG_COND_LT
;
3648 tcg_gen_sub_i64(t0
, op0
, op1
);
3651 cond
= eq
? TCG_COND_GEU
: TCG_COND_GTU
;
3653 maxval
= a
->sf
? INT64_MIN
: INT32_MIN
;
3654 cond
= eq
? TCG_COND_GE
: TCG_COND_GT
;
3658 tmax
= tcg_const_i64(vsz
>> a
->esz
);
3660 /* Equality means one more iteration. */
3661 tcg_gen_addi_i64(t0
, t0
, 1);
3664 * For the less-than while, if op1 is maxval (and the only time
3665 * the addition above could overflow), then we produce an all-true
3666 * predicate by setting the count to the vector length. This is
3667 * because the pseudocode is described as an increment + compare
3668 * loop, and the maximum integer would always compare true.
3669 * Similarly, the greater-than while has the same issue with the
3670 * minimum integer due to the decrement + compare loop.
3672 tcg_gen_movi_i64(t1
, maxval
);
3673 tcg_gen_movcond_i64(TCG_COND_EQ
, t0
, op1
, t1
, tmax
, t0
);
3676 /* Bound to the maximum. */
3677 tcg_gen_umin_i64(t0
, t0
, tmax
);
3678 tcg_temp_free_i64(tmax
);
3680 /* Set the count to zero if the condition is false. */
3681 tcg_gen_movi_i64(t1
, 0);
3682 tcg_gen_movcond_i64(cond
, t0
, op0
, op1
, t0
, t1
);
3683 tcg_temp_free_i64(t1
);
3685 /* Since we're bounded, pass as a 32-bit type. */
3686 t2
= tcg_temp_new_i32();
3687 tcg_gen_extrl_i64_i32(t2
, t0
);
3688 tcg_temp_free_i64(t0
);
3690 /* Scale elements to bits. */
3691 tcg_gen_shli_i32(t2
, t2
, a
->esz
);
3693 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
/ 8);
3694 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
3695 t3
= tcg_const_i32(desc
);
3697 ptr
= tcg_temp_new_ptr();
3698 tcg_gen_addi_ptr(ptr
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3701 gen_helper_sve_whilel(t2
, ptr
, t2
, t3
);
3703 gen_helper_sve_whileg(t2
, ptr
, t2
, t3
);
3707 tcg_temp_free_ptr(ptr
);
3708 tcg_temp_free_i32(t2
);
3709 tcg_temp_free_i32(t3
);
3713 static bool trans_WHILE_ptr(DisasContext
*s
, arg_WHILE_ptr
*a
)
3715 TCGv_i64 op0
, op1
, diff
, t1
, tmax
;
3718 unsigned vsz
= vec_full_reg_size(s
);
3721 if (!dc_isar_feature(aa64_sve2
, s
)) {
3724 if (!sve_access_check(s
)) {
3728 op0
= read_cpu_reg(s
, a
->rn
, 1);
3729 op1
= read_cpu_reg(s
, a
->rm
, 1);
3731 tmax
= tcg_const_i64(vsz
);
3732 diff
= tcg_temp_new_i64();
3736 /* diff = abs(op1 - op0), noting that op0/1 are unsigned. */
3737 t1
= tcg_temp_new_i64();
3738 tcg_gen_sub_i64(diff
, op0
, op1
);
3739 tcg_gen_sub_i64(t1
, op1
, op0
);
3740 tcg_gen_movcond_i64(TCG_COND_GEU
, diff
, op0
, op1
, diff
, t1
);
3741 tcg_temp_free_i64(t1
);
3742 /* Round down to a multiple of ESIZE. */
3743 tcg_gen_andi_i64(diff
, diff
, -1 << a
->esz
);
3744 /* If op1 == op0, diff == 0, and the condition is always true. */
3745 tcg_gen_movcond_i64(TCG_COND_EQ
, diff
, op0
, op1
, tmax
, diff
);
3748 tcg_gen_sub_i64(diff
, op1
, op0
);
3749 /* Round down to a multiple of ESIZE. */
3750 tcg_gen_andi_i64(diff
, diff
, -1 << a
->esz
);
3751 /* If op0 >= op1, diff <= 0, the condition is always true. */
3752 tcg_gen_movcond_i64(TCG_COND_GEU
, diff
, op0
, op1
, tmax
, diff
);
3755 /* Bound to the maximum. */
3756 tcg_gen_umin_i64(diff
, diff
, tmax
);
3757 tcg_temp_free_i64(tmax
);
3759 /* Since we're bounded, pass as a 32-bit type. */
3760 t2
= tcg_temp_new_i32();
3761 tcg_gen_extrl_i64_i32(t2
, diff
);
3762 tcg_temp_free_i64(diff
);
3764 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
/ 8);
3765 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
3766 t3
= tcg_const_i32(desc
);
3768 ptr
= tcg_temp_new_ptr();
3769 tcg_gen_addi_ptr(ptr
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3771 gen_helper_sve_whilel(t2
, ptr
, t2
, t3
);
3774 tcg_temp_free_ptr(ptr
);
3775 tcg_temp_free_i32(t2
);
3776 tcg_temp_free_i32(t3
);
3781 *** SVE Integer Wide Immediate - Unpredicated Group
3784 static bool trans_FDUP(DisasContext
*s
, arg_FDUP
*a
)
3789 if (sve_access_check(s
)) {
3790 unsigned vsz
= vec_full_reg_size(s
);
3791 int dofs
= vec_full_reg_offset(s
, a
->rd
);
3794 /* Decode the VFP immediate. */
3795 imm
= vfp_expand_imm(a
->esz
, a
->imm
);
3796 tcg_gen_gvec_dup_imm(a
->esz
, dofs
, vsz
, vsz
, imm
);
3801 static bool trans_DUP_i(DisasContext
*s
, arg_DUP_i
*a
)
3803 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3806 if (sve_access_check(s
)) {
3807 unsigned vsz
= vec_full_reg_size(s
);
3808 int dofs
= vec_full_reg_offset(s
, a
->rd
);
3810 tcg_gen_gvec_dup_imm(a
->esz
, dofs
, vsz
, vsz
, a
->imm
);
3815 static bool trans_ADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3817 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3820 if (sve_access_check(s
)) {
3821 unsigned vsz
= vec_full_reg_size(s
);
3822 tcg_gen_gvec_addi(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3823 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
3828 static bool trans_SUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3831 return trans_ADD_zzi(s
, a
);
3834 static bool trans_SUBR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3836 static const TCGOpcode vecop_list
[] = { INDEX_op_sub_vec
, 0 };
3837 static const GVecGen2s op
[4] = {
3838 { .fni8
= tcg_gen_vec_sub8_i64
,
3839 .fniv
= tcg_gen_sub_vec
,
3840 .fno
= gen_helper_sve_subri_b
,
3841 .opt_opc
= vecop_list
,
3843 .scalar_first
= true },
3844 { .fni8
= tcg_gen_vec_sub16_i64
,
3845 .fniv
= tcg_gen_sub_vec
,
3846 .fno
= gen_helper_sve_subri_h
,
3847 .opt_opc
= vecop_list
,
3849 .scalar_first
= true },
3850 { .fni4
= tcg_gen_sub_i32
,
3851 .fniv
= tcg_gen_sub_vec
,
3852 .fno
= gen_helper_sve_subri_s
,
3853 .opt_opc
= vecop_list
,
3855 .scalar_first
= true },
3856 { .fni8
= tcg_gen_sub_i64
,
3857 .fniv
= tcg_gen_sub_vec
,
3858 .fno
= gen_helper_sve_subri_d
,
3859 .opt_opc
= vecop_list
,
3860 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
3862 .scalar_first
= true }
3865 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3868 if (sve_access_check(s
)) {
3869 unsigned vsz
= vec_full_reg_size(s
);
3870 TCGv_i64 c
= tcg_const_i64(a
->imm
);
3871 tcg_gen_gvec_2s(vec_full_reg_offset(s
, a
->rd
),
3872 vec_full_reg_offset(s
, a
->rn
),
3873 vsz
, vsz
, c
, &op
[a
->esz
]);
3874 tcg_temp_free_i64(c
);
3879 static bool trans_MUL_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3881 if (sve_access_check(s
)) {
3882 unsigned vsz
= vec_full_reg_size(s
);
3883 tcg_gen_gvec_muli(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3884 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
3889 static bool do_zzi_sat(DisasContext
*s
, arg_rri_esz
*a
, bool u
, bool d
)
3891 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3894 if (sve_access_check(s
)) {
3895 TCGv_i64 val
= tcg_const_i64(a
->imm
);
3896 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
, val
, u
, d
);
3897 tcg_temp_free_i64(val
);
3902 static bool trans_SQADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3904 return do_zzi_sat(s
, a
, false, false);
3907 static bool trans_UQADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3909 return do_zzi_sat(s
, a
, true, false);
3912 static bool trans_SQSUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3914 return do_zzi_sat(s
, a
, false, true);
3917 static bool trans_UQSUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3919 return do_zzi_sat(s
, a
, true, true);
3922 static bool do_zzi_ool(DisasContext
*s
, arg_rri_esz
*a
, gen_helper_gvec_2i
*fn
)
3924 if (sve_access_check(s
)) {
3925 unsigned vsz
= vec_full_reg_size(s
);
3926 TCGv_i64 c
= tcg_const_i64(a
->imm
);
3928 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s
, a
->rd
),
3929 vec_full_reg_offset(s
, a
->rn
),
3930 c
, vsz
, vsz
, 0, fn
);
3931 tcg_temp_free_i64(c
);
3936 #define DO_ZZI(NAME, name) \
3937 static bool trans_##NAME##_zzi(DisasContext *s, arg_rri_esz *a) \
3939 static gen_helper_gvec_2i * const fns[4] = { \
3940 gen_helper_sve_##name##i_b, gen_helper_sve_##name##i_h, \
3941 gen_helper_sve_##name##i_s, gen_helper_sve_##name##i_d, \
3943 return do_zzi_ool(s, a, fns[a->esz]); \
3953 static bool trans_DOT_zzzz(DisasContext
*s
, arg_DOT_zzzz
*a
)
3955 static gen_helper_gvec_4
* const fns
[2][2] = {
3956 { gen_helper_gvec_sdot_b
, gen_helper_gvec_sdot_h
},
3957 { gen_helper_gvec_udot_b
, gen_helper_gvec_udot_h
}
3960 if (sve_access_check(s
)) {
3961 gen_gvec_ool_zzzz(s
, fns
[a
->u
][a
->sz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, 0);
3967 * SVE Multiply - Indexed
3970 static bool do_zzxz_ool(DisasContext
*s
, arg_rrxr_esz
*a
,
3971 gen_helper_gvec_4
*fn
)
3976 if (sve_access_check(s
)) {
3977 gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->index
);
3982 #define DO_RRXR(NAME, FUNC) \
3983 static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
3984 { return do_zzxz_ool(s, a, FUNC); }
3986 DO_RRXR(trans_SDOT_zzxw_s
, gen_helper_gvec_sdot_idx_b
)
3987 DO_RRXR(trans_SDOT_zzxw_d
, gen_helper_gvec_sdot_idx_h
)
3988 DO_RRXR(trans_UDOT_zzxw_s
, gen_helper_gvec_udot_idx_b
)
3989 DO_RRXR(trans_UDOT_zzxw_d
, gen_helper_gvec_udot_idx_h
)
3991 static bool trans_SUDOT_zzxw_s(DisasContext
*s
, arg_rrxr_esz
*a
)
3993 if (!dc_isar_feature(aa64_sve_i8mm
, s
)) {
3996 return do_zzxz_ool(s
, a
, gen_helper_gvec_sudot_idx_b
);
3999 static bool trans_USDOT_zzxw_s(DisasContext
*s
, arg_rrxr_esz
*a
)
4001 if (!dc_isar_feature(aa64_sve_i8mm
, s
)) {
4004 return do_zzxz_ool(s
, a
, gen_helper_gvec_usdot_idx_b
);
4009 static bool do_sve2_zzz_data(DisasContext
*s
, int rd
, int rn
, int rm
, int data
,
4010 gen_helper_gvec_3
*fn
)
4012 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
4015 if (sve_access_check(s
)) {
4016 unsigned vsz
= vec_full_reg_size(s
);
4017 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
4018 vec_full_reg_offset(s
, rn
),
4019 vec_full_reg_offset(s
, rm
),
4020 vsz
, vsz
, data
, fn
);
4025 #define DO_SVE2_RRX(NAME, FUNC) \
4026 static bool NAME(DisasContext *s, arg_rrx_esz *a) \
4027 { return do_sve2_zzz_data(s, a->rd, a->rn, a->rm, a->index, FUNC); }
4029 DO_SVE2_RRX(trans_MUL_zzx_h
, gen_helper_gvec_mul_idx_h
)
4030 DO_SVE2_RRX(trans_MUL_zzx_s
, gen_helper_gvec_mul_idx_s
)
4031 DO_SVE2_RRX(trans_MUL_zzx_d
, gen_helper_gvec_mul_idx_d
)
4033 DO_SVE2_RRX(trans_SQDMULH_zzx_h
, gen_helper_sve2_sqdmulh_idx_h
)
4034 DO_SVE2_RRX(trans_SQDMULH_zzx_s
, gen_helper_sve2_sqdmulh_idx_s
)
4035 DO_SVE2_RRX(trans_SQDMULH_zzx_d
, gen_helper_sve2_sqdmulh_idx_d
)
4037 DO_SVE2_RRX(trans_SQRDMULH_zzx_h
, gen_helper_sve2_sqrdmulh_idx_h
)
4038 DO_SVE2_RRX(trans_SQRDMULH_zzx_s
, gen_helper_sve2_sqrdmulh_idx_s
)
4039 DO_SVE2_RRX(trans_SQRDMULH_zzx_d
, gen_helper_sve2_sqrdmulh_idx_d
)
4043 #define DO_SVE2_RRX_TB(NAME, FUNC, TOP) \
4044 static bool NAME(DisasContext *s, arg_rrx_esz *a) \
4046 return do_sve2_zzz_data(s, a->rd, a->rn, a->rm, \
4047 (a->index << 1) | TOP, FUNC); \
4050 DO_SVE2_RRX_TB(trans_SQDMULLB_zzx_s
, gen_helper_sve2_sqdmull_idx_s
, false)
4051 DO_SVE2_RRX_TB(trans_SQDMULLB_zzx_d
, gen_helper_sve2_sqdmull_idx_d
, false)
4052 DO_SVE2_RRX_TB(trans_SQDMULLT_zzx_s
, gen_helper_sve2_sqdmull_idx_s
, true)
4053 DO_SVE2_RRX_TB(trans_SQDMULLT_zzx_d
, gen_helper_sve2_sqdmull_idx_d
, true)
4055 DO_SVE2_RRX_TB(trans_SMULLB_zzx_s
, gen_helper_sve2_smull_idx_s
, false)
4056 DO_SVE2_RRX_TB(trans_SMULLB_zzx_d
, gen_helper_sve2_smull_idx_d
, false)
4057 DO_SVE2_RRX_TB(trans_SMULLT_zzx_s
, gen_helper_sve2_smull_idx_s
, true)
4058 DO_SVE2_RRX_TB(trans_SMULLT_zzx_d
, gen_helper_sve2_smull_idx_d
, true)
4060 DO_SVE2_RRX_TB(trans_UMULLB_zzx_s
, gen_helper_sve2_umull_idx_s
, false)
4061 DO_SVE2_RRX_TB(trans_UMULLB_zzx_d
, gen_helper_sve2_umull_idx_d
, false)
4062 DO_SVE2_RRX_TB(trans_UMULLT_zzx_s
, gen_helper_sve2_umull_idx_s
, true)
4063 DO_SVE2_RRX_TB(trans_UMULLT_zzx_d
, gen_helper_sve2_umull_idx_d
, true)
4065 #undef DO_SVE2_RRX_TB
4067 static bool do_sve2_zzzz_data(DisasContext
*s
, int rd
, int rn
, int rm
, int ra
,
4068 int data
, gen_helper_gvec_4
*fn
)
4070 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
4073 if (sve_access_check(s
)) {
4074 unsigned vsz
= vec_full_reg_size(s
);
4075 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
4076 vec_full_reg_offset(s
, rn
),
4077 vec_full_reg_offset(s
, rm
),
4078 vec_full_reg_offset(s
, ra
),
4079 vsz
, vsz
, data
, fn
);
4084 #define DO_SVE2_RRXR(NAME, FUNC) \
4085 static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
4086 { return do_sve2_zzzz_data(s, a->rd, a->rn, a->rm, a->ra, a->index, FUNC); }
4088 DO_SVE2_RRXR(trans_MLA_zzxz_h
, gen_helper_gvec_mla_idx_h
)
4089 DO_SVE2_RRXR(trans_MLA_zzxz_s
, gen_helper_gvec_mla_idx_s
)
4090 DO_SVE2_RRXR(trans_MLA_zzxz_d
, gen_helper_gvec_mla_idx_d
)
4092 DO_SVE2_RRXR(trans_MLS_zzxz_h
, gen_helper_gvec_mls_idx_h
)
4093 DO_SVE2_RRXR(trans_MLS_zzxz_s
, gen_helper_gvec_mls_idx_s
)
4094 DO_SVE2_RRXR(trans_MLS_zzxz_d
, gen_helper_gvec_mls_idx_d
)
4096 DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_h
, gen_helper_sve2_sqrdmlah_idx_h
)
4097 DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_s
, gen_helper_sve2_sqrdmlah_idx_s
)
4098 DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_d
, gen_helper_sve2_sqrdmlah_idx_d
)
4100 DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_h
, gen_helper_sve2_sqrdmlsh_idx_h
)
4101 DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_s
, gen_helper_sve2_sqrdmlsh_idx_s
)
4102 DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_d
, gen_helper_sve2_sqrdmlsh_idx_d
)
4106 #define DO_SVE2_RRXR_TB(NAME, FUNC, TOP) \
4107 static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
4109 return do_sve2_zzzz_data(s, a->rd, a->rn, a->rm, a->rd, \
4110 (a->index << 1) | TOP, FUNC); \
4113 DO_SVE2_RRXR_TB(trans_SQDMLALB_zzxw_s
, gen_helper_sve2_sqdmlal_idx_s
, false)
4114 DO_SVE2_RRXR_TB(trans_SQDMLALB_zzxw_d
, gen_helper_sve2_sqdmlal_idx_d
, false)
4115 DO_SVE2_RRXR_TB(trans_SQDMLALT_zzxw_s
, gen_helper_sve2_sqdmlal_idx_s
, true)
4116 DO_SVE2_RRXR_TB(trans_SQDMLALT_zzxw_d
, gen_helper_sve2_sqdmlal_idx_d
, true)
4118 DO_SVE2_RRXR_TB(trans_SQDMLSLB_zzxw_s
, gen_helper_sve2_sqdmlsl_idx_s
, false)
4119 DO_SVE2_RRXR_TB(trans_SQDMLSLB_zzxw_d
, gen_helper_sve2_sqdmlsl_idx_d
, false)
4120 DO_SVE2_RRXR_TB(trans_SQDMLSLT_zzxw_s
, gen_helper_sve2_sqdmlsl_idx_s
, true)
4121 DO_SVE2_RRXR_TB(trans_SQDMLSLT_zzxw_d
, gen_helper_sve2_sqdmlsl_idx_d
, true)
4123 DO_SVE2_RRXR_TB(trans_SMLALB_zzxw_s
, gen_helper_sve2_smlal_idx_s
, false)
4124 DO_SVE2_RRXR_TB(trans_SMLALB_zzxw_d
, gen_helper_sve2_smlal_idx_d
, false)
4125 DO_SVE2_RRXR_TB(trans_SMLALT_zzxw_s
, gen_helper_sve2_smlal_idx_s
, true)
4126 DO_SVE2_RRXR_TB(trans_SMLALT_zzxw_d
, gen_helper_sve2_smlal_idx_d
, true)
4128 DO_SVE2_RRXR_TB(trans_UMLALB_zzxw_s
, gen_helper_sve2_umlal_idx_s
, false)
4129 DO_SVE2_RRXR_TB(trans_UMLALB_zzxw_d
, gen_helper_sve2_umlal_idx_d
, false)
4130 DO_SVE2_RRXR_TB(trans_UMLALT_zzxw_s
, gen_helper_sve2_umlal_idx_s
, true)
4131 DO_SVE2_RRXR_TB(trans_UMLALT_zzxw_d
, gen_helper_sve2_umlal_idx_d
, true)
4133 DO_SVE2_RRXR_TB(trans_SMLSLB_zzxw_s
, gen_helper_sve2_smlsl_idx_s
, false)
4134 DO_SVE2_RRXR_TB(trans_SMLSLB_zzxw_d
, gen_helper_sve2_smlsl_idx_d
, false)
4135 DO_SVE2_RRXR_TB(trans_SMLSLT_zzxw_s
, gen_helper_sve2_smlsl_idx_s
, true)
4136 DO_SVE2_RRXR_TB(trans_SMLSLT_zzxw_d
, gen_helper_sve2_smlsl_idx_d
, true)
4138 DO_SVE2_RRXR_TB(trans_UMLSLB_zzxw_s
, gen_helper_sve2_umlsl_idx_s
, false)
4139 DO_SVE2_RRXR_TB(trans_UMLSLB_zzxw_d
, gen_helper_sve2_umlsl_idx_d
, false)
4140 DO_SVE2_RRXR_TB(trans_UMLSLT_zzxw_s
, gen_helper_sve2_umlsl_idx_s
, true)
4141 DO_SVE2_RRXR_TB(trans_UMLSLT_zzxw_d
, gen_helper_sve2_umlsl_idx_d
, true)
4143 #undef DO_SVE2_RRXR_TB
4145 #define DO_SVE2_RRXR_ROT(NAME, FUNC) \
4146 static bool trans_##NAME(DisasContext *s, arg_##NAME *a) \
4148 return do_sve2_zzzz_data(s, a->rd, a->rn, a->rm, a->ra, \
4149 (a->index << 2) | a->rot, FUNC); \
4152 DO_SVE2_RRXR_ROT(CMLA_zzxz_h
, gen_helper_sve2_cmla_idx_h
)
4153 DO_SVE2_RRXR_ROT(CMLA_zzxz_s
, gen_helper_sve2_cmla_idx_s
)
4155 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_h
, gen_helper_sve2_sqrdcmlah_idx_h
)
4156 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_s
, gen_helper_sve2_sqrdcmlah_idx_s
)
4158 DO_SVE2_RRXR_ROT(CDOT_zzxw_s
, gen_helper_sve2_cdot_idx_s
)
4159 DO_SVE2_RRXR_ROT(CDOT_zzxw_d
, gen_helper_sve2_cdot_idx_d
)
4161 #undef DO_SVE2_RRXR_ROT
4164 *** SVE Floating Point Multiply-Add Indexed Group
4167 static bool do_FMLA_zzxz(DisasContext
*s
, arg_rrxr_esz
*a
, bool sub
)
4169 static gen_helper_gvec_4_ptr
* const fns
[3] = {
4170 gen_helper_gvec_fmla_idx_h
,
4171 gen_helper_gvec_fmla_idx_s
,
4172 gen_helper_gvec_fmla_idx_d
,
4175 if (sve_access_check(s
)) {
4176 unsigned vsz
= vec_full_reg_size(s
);
4177 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4178 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4179 vec_full_reg_offset(s
, a
->rn
),
4180 vec_full_reg_offset(s
, a
->rm
),
4181 vec_full_reg_offset(s
, a
->ra
),
4182 status
, vsz
, vsz
, (a
->index
<< 1) | sub
,
4184 tcg_temp_free_ptr(status
);
4189 static bool trans_FMLA_zzxz(DisasContext
*s
, arg_FMLA_zzxz
*a
)
4191 return do_FMLA_zzxz(s
, a
, false);
4194 static bool trans_FMLS_zzxz(DisasContext
*s
, arg_FMLA_zzxz
*a
)
4196 return do_FMLA_zzxz(s
, a
, true);
4200 *** SVE Floating Point Multiply Indexed Group
4203 static bool trans_FMUL_zzx(DisasContext
*s
, arg_FMUL_zzx
*a
)
4205 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4206 gen_helper_gvec_fmul_idx_h
,
4207 gen_helper_gvec_fmul_idx_s
,
4208 gen_helper_gvec_fmul_idx_d
,
4211 if (sve_access_check(s
)) {
4212 unsigned vsz
= vec_full_reg_size(s
);
4213 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4214 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4215 vec_full_reg_offset(s
, a
->rn
),
4216 vec_full_reg_offset(s
, a
->rm
),
4217 status
, vsz
, vsz
, a
->index
, fns
[a
->esz
- 1]);
4218 tcg_temp_free_ptr(status
);
4224 *** SVE Floating Point Fast Reduction Group
4227 typedef void gen_helper_fp_reduce(TCGv_i64
, TCGv_ptr
, TCGv_ptr
,
4228 TCGv_ptr
, TCGv_i32
);
4230 static void do_reduce(DisasContext
*s
, arg_rpr_esz
*a
,
4231 gen_helper_fp_reduce
*fn
)
4233 unsigned vsz
= vec_full_reg_size(s
);
4234 unsigned p2vsz
= pow2ceil(vsz
);
4235 TCGv_i32 t_desc
= tcg_const_i32(simd_desc(vsz
, vsz
, p2vsz
));
4236 TCGv_ptr t_zn
, t_pg
, status
;
4239 temp
= tcg_temp_new_i64();
4240 t_zn
= tcg_temp_new_ptr();
4241 t_pg
= tcg_temp_new_ptr();
4243 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
4244 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
4245 status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4247 fn(temp
, t_zn
, t_pg
, status
, t_desc
);
4248 tcg_temp_free_ptr(t_zn
);
4249 tcg_temp_free_ptr(t_pg
);
4250 tcg_temp_free_ptr(status
);
4251 tcg_temp_free_i32(t_desc
);
4253 write_fp_dreg(s
, a
->rd
, temp
);
4254 tcg_temp_free_i64(temp
);
4257 #define DO_VPZ(NAME, name) \
4258 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
4260 static gen_helper_fp_reduce * const fns[3] = { \
4261 gen_helper_sve_##name##_h, \
4262 gen_helper_sve_##name##_s, \
4263 gen_helper_sve_##name##_d, \
4265 if (a->esz == 0) { \
4268 if (sve_access_check(s)) { \
4269 do_reduce(s, a, fns[a->esz - 1]); \
4274 DO_VPZ(FADDV
, faddv
)
4275 DO_VPZ(FMINNMV
, fminnmv
)
4276 DO_VPZ(FMAXNMV
, fmaxnmv
)
4277 DO_VPZ(FMINV
, fminv
)
4278 DO_VPZ(FMAXV
, fmaxv
)
4281 *** SVE Floating Point Unary Operations - Unpredicated Group
4284 static void do_zz_fp(DisasContext
*s
, arg_rr_esz
*a
, gen_helper_gvec_2_ptr
*fn
)
4286 unsigned vsz
= vec_full_reg_size(s
);
4287 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4289 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s
, a
->rd
),
4290 vec_full_reg_offset(s
, a
->rn
),
4291 status
, vsz
, vsz
, 0, fn
);
4292 tcg_temp_free_ptr(status
);
4295 static bool trans_FRECPE(DisasContext
*s
, arg_rr_esz
*a
)
4297 static gen_helper_gvec_2_ptr
* const fns
[3] = {
4298 gen_helper_gvec_frecpe_h
,
4299 gen_helper_gvec_frecpe_s
,
4300 gen_helper_gvec_frecpe_d
,
4305 if (sve_access_check(s
)) {
4306 do_zz_fp(s
, a
, fns
[a
->esz
- 1]);
4311 static bool trans_FRSQRTE(DisasContext
*s
, arg_rr_esz
*a
)
4313 static gen_helper_gvec_2_ptr
* const fns
[3] = {
4314 gen_helper_gvec_frsqrte_h
,
4315 gen_helper_gvec_frsqrte_s
,
4316 gen_helper_gvec_frsqrte_d
,
4321 if (sve_access_check(s
)) {
4322 do_zz_fp(s
, a
, fns
[a
->esz
- 1]);
4328 *** SVE Floating Point Compare with Zero Group
4331 static void do_ppz_fp(DisasContext
*s
, arg_rpr_esz
*a
,
4332 gen_helper_gvec_3_ptr
*fn
)
4334 unsigned vsz
= vec_full_reg_size(s
);
4335 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4337 tcg_gen_gvec_3_ptr(pred_full_reg_offset(s
, a
->rd
),
4338 vec_full_reg_offset(s
, a
->rn
),
4339 pred_full_reg_offset(s
, a
->pg
),
4340 status
, vsz
, vsz
, 0, fn
);
4341 tcg_temp_free_ptr(status
);
4344 #define DO_PPZ(NAME, name) \
4345 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
4347 static gen_helper_gvec_3_ptr * const fns[3] = { \
4348 gen_helper_sve_##name##_h, \
4349 gen_helper_sve_##name##_s, \
4350 gen_helper_sve_##name##_d, \
4352 if (a->esz == 0) { \
4355 if (sve_access_check(s)) { \
4356 do_ppz_fp(s, a, fns[a->esz - 1]); \
4361 DO_PPZ(FCMGE_ppz0
, fcmge0
)
4362 DO_PPZ(FCMGT_ppz0
, fcmgt0
)
4363 DO_PPZ(FCMLE_ppz0
, fcmle0
)
4364 DO_PPZ(FCMLT_ppz0
, fcmlt0
)
4365 DO_PPZ(FCMEQ_ppz0
, fcmeq0
)
4366 DO_PPZ(FCMNE_ppz0
, fcmne0
)
4371 *** SVE floating-point trig multiply-add coefficient
4374 static bool trans_FTMAD(DisasContext
*s
, arg_FTMAD
*a
)
4376 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4377 gen_helper_sve_ftmad_h
,
4378 gen_helper_sve_ftmad_s
,
4379 gen_helper_sve_ftmad_d
,
4385 if (sve_access_check(s
)) {
4386 unsigned vsz
= vec_full_reg_size(s
);
4387 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4388 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4389 vec_full_reg_offset(s
, a
->rn
),
4390 vec_full_reg_offset(s
, a
->rm
),
4391 status
, vsz
, vsz
, a
->imm
, fns
[a
->esz
- 1]);
4392 tcg_temp_free_ptr(status
);
4398 *** SVE Floating Point Accumulating Reduction Group
4401 static bool trans_FADDA(DisasContext
*s
, arg_rprr_esz
*a
)
4403 typedef void fadda_fn(TCGv_i64
, TCGv_i64
, TCGv_ptr
,
4404 TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
4405 static fadda_fn
* const fns
[3] = {
4406 gen_helper_sve_fadda_h
,
4407 gen_helper_sve_fadda_s
,
4408 gen_helper_sve_fadda_d
,
4410 unsigned vsz
= vec_full_reg_size(s
);
4411 TCGv_ptr t_rm
, t_pg
, t_fpst
;
4418 if (!sve_access_check(s
)) {
4422 t_val
= load_esz(cpu_env
, vec_reg_offset(s
, a
->rn
, 0, a
->esz
), a
->esz
);
4423 t_rm
= tcg_temp_new_ptr();
4424 t_pg
= tcg_temp_new_ptr();
4425 tcg_gen_addi_ptr(t_rm
, cpu_env
, vec_full_reg_offset(s
, a
->rm
));
4426 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
4427 t_fpst
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4428 t_desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
4430 fns
[a
->esz
- 1](t_val
, t_val
, t_rm
, t_pg
, t_fpst
, t_desc
);
4432 tcg_temp_free_i32(t_desc
);
4433 tcg_temp_free_ptr(t_fpst
);
4434 tcg_temp_free_ptr(t_pg
);
4435 tcg_temp_free_ptr(t_rm
);
4437 write_fp_dreg(s
, a
->rd
, t_val
);
4438 tcg_temp_free_i64(t_val
);
4443 *** SVE Floating Point Arithmetic - Unpredicated Group
4446 static bool do_zzz_fp(DisasContext
*s
, arg_rrr_esz
*a
,
4447 gen_helper_gvec_3_ptr
*fn
)
4452 if (sve_access_check(s
)) {
4453 unsigned vsz
= vec_full_reg_size(s
);
4454 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4455 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4456 vec_full_reg_offset(s
, a
->rn
),
4457 vec_full_reg_offset(s
, a
->rm
),
4458 status
, vsz
, vsz
, 0, fn
);
4459 tcg_temp_free_ptr(status
);
4465 #define DO_FP3(NAME, name) \
4466 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
4468 static gen_helper_gvec_3_ptr * const fns[4] = { \
4469 NULL, gen_helper_gvec_##name##_h, \
4470 gen_helper_gvec_##name##_s, gen_helper_gvec_##name##_d \
4472 return do_zzz_fp(s, a, fns[a->esz]); \
4475 DO_FP3(FADD_zzz
, fadd
)
4476 DO_FP3(FSUB_zzz
, fsub
)
4477 DO_FP3(FMUL_zzz
, fmul
)
4478 DO_FP3(FTSMUL
, ftsmul
)
4479 DO_FP3(FRECPS
, recps
)
4480 DO_FP3(FRSQRTS
, rsqrts
)
4485 *** SVE Floating Point Arithmetic - Predicated Group
4488 static bool do_zpzz_fp(DisasContext
*s
, arg_rprr_esz
*a
,
4489 gen_helper_gvec_4_ptr
*fn
)
4494 if (sve_access_check(s
)) {
4495 unsigned vsz
= vec_full_reg_size(s
);
4496 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4497 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4498 vec_full_reg_offset(s
, a
->rn
),
4499 vec_full_reg_offset(s
, a
->rm
),
4500 pred_full_reg_offset(s
, a
->pg
),
4501 status
, vsz
, vsz
, 0, fn
);
4502 tcg_temp_free_ptr(status
);
4507 #define DO_FP3(NAME, name) \
4508 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
4510 static gen_helper_gvec_4_ptr * const fns[4] = { \
4511 NULL, gen_helper_sve_##name##_h, \
4512 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4514 return do_zpzz_fp(s, a, fns[a->esz]); \
4517 DO_FP3(FADD_zpzz
, fadd
)
4518 DO_FP3(FSUB_zpzz
, fsub
)
4519 DO_FP3(FMUL_zpzz
, fmul
)
4520 DO_FP3(FMIN_zpzz
, fmin
)
4521 DO_FP3(FMAX_zpzz
, fmax
)
4522 DO_FP3(FMINNM_zpzz
, fminnum
)
4523 DO_FP3(FMAXNM_zpzz
, fmaxnum
)
4525 DO_FP3(FSCALE
, fscalbn
)
4527 DO_FP3(FMULX
, fmulx
)
4531 typedef void gen_helper_sve_fp2scalar(TCGv_ptr
, TCGv_ptr
, TCGv_ptr
,
4532 TCGv_i64
, TCGv_ptr
, TCGv_i32
);
4534 static void do_fp_scalar(DisasContext
*s
, int zd
, int zn
, int pg
, bool is_fp16
,
4535 TCGv_i64 scalar
, gen_helper_sve_fp2scalar
*fn
)
4537 unsigned vsz
= vec_full_reg_size(s
);
4538 TCGv_ptr t_zd
, t_zn
, t_pg
, status
;
4541 t_zd
= tcg_temp_new_ptr();
4542 t_zn
= tcg_temp_new_ptr();
4543 t_pg
= tcg_temp_new_ptr();
4544 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, zd
));
4545 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, zn
));
4546 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
4548 status
= fpstatus_ptr(is_fp16
? FPST_FPCR_F16
: FPST_FPCR
);
4549 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
4550 fn(t_zd
, t_zn
, t_pg
, scalar
, status
, desc
);
4552 tcg_temp_free_i32(desc
);
4553 tcg_temp_free_ptr(status
);
4554 tcg_temp_free_ptr(t_pg
);
4555 tcg_temp_free_ptr(t_zn
);
4556 tcg_temp_free_ptr(t_zd
);
4559 static void do_fp_imm(DisasContext
*s
, arg_rpri_esz
*a
, uint64_t imm
,
4560 gen_helper_sve_fp2scalar
*fn
)
4562 TCGv_i64 temp
= tcg_const_i64(imm
);
4563 do_fp_scalar(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, temp
, fn
);
4564 tcg_temp_free_i64(temp
);
4567 #define DO_FP_IMM(NAME, name, const0, const1) \
4568 static bool trans_##NAME##_zpzi(DisasContext *s, arg_rpri_esz *a) \
4570 static gen_helper_sve_fp2scalar * const fns[3] = { \
4571 gen_helper_sve_##name##_h, \
4572 gen_helper_sve_##name##_s, \
4573 gen_helper_sve_##name##_d \
4575 static uint64_t const val[3][2] = { \
4576 { float16_##const0, float16_##const1 }, \
4577 { float32_##const0, float32_##const1 }, \
4578 { float64_##const0, float64_##const1 }, \
4580 if (a->esz == 0) { \
4583 if (sve_access_check(s)) { \
4584 do_fp_imm(s, a, val[a->esz - 1][a->imm], fns[a->esz - 1]); \
4589 DO_FP_IMM(FADD
, fadds
, half
, one
)
4590 DO_FP_IMM(FSUB
, fsubs
, half
, one
)
4591 DO_FP_IMM(FMUL
, fmuls
, half
, two
)
4592 DO_FP_IMM(FSUBR
, fsubrs
, half
, one
)
4593 DO_FP_IMM(FMAXNM
, fmaxnms
, zero
, one
)
4594 DO_FP_IMM(FMINNM
, fminnms
, zero
, one
)
4595 DO_FP_IMM(FMAX
, fmaxs
, zero
, one
)
4596 DO_FP_IMM(FMIN
, fmins
, zero
, one
)
4600 static bool do_fp_cmp(DisasContext
*s
, arg_rprr_esz
*a
,
4601 gen_helper_gvec_4_ptr
*fn
)
4606 if (sve_access_check(s
)) {
4607 unsigned vsz
= vec_full_reg_size(s
);
4608 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4609 tcg_gen_gvec_4_ptr(pred_full_reg_offset(s
, a
->rd
),
4610 vec_full_reg_offset(s
, a
->rn
),
4611 vec_full_reg_offset(s
, a
->rm
),
4612 pred_full_reg_offset(s
, a
->pg
),
4613 status
, vsz
, vsz
, 0, fn
);
4614 tcg_temp_free_ptr(status
);
4619 #define DO_FPCMP(NAME, name) \
4620 static bool trans_##NAME##_ppzz(DisasContext *s, arg_rprr_esz *a) \
4622 static gen_helper_gvec_4_ptr * const fns[4] = { \
4623 NULL, gen_helper_sve_##name##_h, \
4624 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4626 return do_fp_cmp(s, a, fns[a->esz]); \
4629 DO_FPCMP(FCMGE
, fcmge
)
4630 DO_FPCMP(FCMGT
, fcmgt
)
4631 DO_FPCMP(FCMEQ
, fcmeq
)
4632 DO_FPCMP(FCMNE
, fcmne
)
4633 DO_FPCMP(FCMUO
, fcmuo
)
4634 DO_FPCMP(FACGE
, facge
)
4635 DO_FPCMP(FACGT
, facgt
)
4639 static bool trans_FCADD(DisasContext
*s
, arg_FCADD
*a
)
4641 static gen_helper_gvec_4_ptr
* const fns
[3] = {
4642 gen_helper_sve_fcadd_h
,
4643 gen_helper_sve_fcadd_s
,
4644 gen_helper_sve_fcadd_d
4650 if (sve_access_check(s
)) {
4651 unsigned vsz
= vec_full_reg_size(s
);
4652 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4653 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4654 vec_full_reg_offset(s
, a
->rn
),
4655 vec_full_reg_offset(s
, a
->rm
),
4656 pred_full_reg_offset(s
, a
->pg
),
4657 status
, vsz
, vsz
, a
->rot
, fns
[a
->esz
- 1]);
4658 tcg_temp_free_ptr(status
);
4663 static bool do_fmla(DisasContext
*s
, arg_rprrr_esz
*a
,
4664 gen_helper_gvec_5_ptr
*fn
)
4669 if (sve_access_check(s
)) {
4670 unsigned vsz
= vec_full_reg_size(s
);
4671 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4672 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s
, a
->rd
),
4673 vec_full_reg_offset(s
, a
->rn
),
4674 vec_full_reg_offset(s
, a
->rm
),
4675 vec_full_reg_offset(s
, a
->ra
),
4676 pred_full_reg_offset(s
, a
->pg
),
4677 status
, vsz
, vsz
, 0, fn
);
4678 tcg_temp_free_ptr(status
);
4683 #define DO_FMLA(NAME, name) \
4684 static bool trans_##NAME(DisasContext *s, arg_rprrr_esz *a) \
4686 static gen_helper_gvec_5_ptr * const fns[4] = { \
4687 NULL, gen_helper_sve_##name##_h, \
4688 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4690 return do_fmla(s, a, fns[a->esz]); \
4693 DO_FMLA(FMLA_zpzzz
, fmla_zpzzz
)
4694 DO_FMLA(FMLS_zpzzz
, fmls_zpzzz
)
4695 DO_FMLA(FNMLA_zpzzz
, fnmla_zpzzz
)
4696 DO_FMLA(FNMLS_zpzzz
, fnmls_zpzzz
)
4700 static bool trans_FCMLA_zpzzz(DisasContext
*s
, arg_FCMLA_zpzzz
*a
)
4702 static gen_helper_gvec_5_ptr
* const fns
[4] = {
4704 gen_helper_sve_fcmla_zpzzz_h
,
4705 gen_helper_sve_fcmla_zpzzz_s
,
4706 gen_helper_sve_fcmla_zpzzz_d
,
4712 if (sve_access_check(s
)) {
4713 unsigned vsz
= vec_full_reg_size(s
);
4714 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4715 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s
, a
->rd
),
4716 vec_full_reg_offset(s
, a
->rn
),
4717 vec_full_reg_offset(s
, a
->rm
),
4718 vec_full_reg_offset(s
, a
->ra
),
4719 pred_full_reg_offset(s
, a
->pg
),
4720 status
, vsz
, vsz
, a
->rot
, fns
[a
->esz
]);
4721 tcg_temp_free_ptr(status
);
4726 static bool trans_FCMLA_zzxz(DisasContext
*s
, arg_FCMLA_zzxz
*a
)
4728 static gen_helper_gvec_4_ptr
* const fns
[2] = {
4729 gen_helper_gvec_fcmlah_idx
,
4730 gen_helper_gvec_fcmlas_idx
,
4733 tcg_debug_assert(a
->esz
== 1 || a
->esz
== 2);
4734 tcg_debug_assert(a
->rd
== a
->ra
);
4735 if (sve_access_check(s
)) {
4736 unsigned vsz
= vec_full_reg_size(s
);
4737 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4738 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4739 vec_full_reg_offset(s
, a
->rn
),
4740 vec_full_reg_offset(s
, a
->rm
),
4741 vec_full_reg_offset(s
, a
->ra
),
4743 a
->index
* 4 + a
->rot
,
4745 tcg_temp_free_ptr(status
);
4751 *** SVE Floating Point Unary Operations Predicated Group
4754 static bool do_zpz_ptr(DisasContext
*s
, int rd
, int rn
, int pg
,
4755 bool is_fp16
, gen_helper_gvec_3_ptr
*fn
)
4757 if (sve_access_check(s
)) {
4758 unsigned vsz
= vec_full_reg_size(s
);
4759 TCGv_ptr status
= fpstatus_ptr(is_fp16
? FPST_FPCR_F16
: FPST_FPCR
);
4760 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, rd
),
4761 vec_full_reg_offset(s
, rn
),
4762 pred_full_reg_offset(s
, pg
),
4763 status
, vsz
, vsz
, 0, fn
);
4764 tcg_temp_free_ptr(status
);
4769 static bool trans_FCVT_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4771 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_sh
);
4774 static bool trans_FCVT_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4776 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_hs
);
4779 static bool trans_BFCVT(DisasContext
*s
, arg_rpr_esz
*a
)
4781 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
4784 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_bfcvt
);
4787 static bool trans_FCVT_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4789 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_dh
);
4792 static bool trans_FCVT_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4794 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_hd
);
4797 static bool trans_FCVT_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4799 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_ds
);
4802 static bool trans_FCVT_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4804 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_sd
);
4807 static bool trans_FCVTZS_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4809 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hh
);
4812 static bool trans_FCVTZU_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4814 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hh
);
4817 static bool trans_FCVTZS_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4819 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hs
);
4822 static bool trans_FCVTZU_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4824 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hs
);
4827 static bool trans_FCVTZS_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4829 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hd
);
4832 static bool trans_FCVTZU_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4834 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hd
);
4837 static bool trans_FCVTZS_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4839 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_ss
);
4842 static bool trans_FCVTZU_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4844 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_ss
);
4847 static bool trans_FCVTZS_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4849 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_sd
);
4852 static bool trans_FCVTZU_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4854 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_sd
);
4857 static bool trans_FCVTZS_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4859 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_ds
);
4862 static bool trans_FCVTZU_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4864 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_ds
);
4867 static bool trans_FCVTZS_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4869 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_dd
);
4872 static bool trans_FCVTZU_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4874 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_dd
);
4877 static gen_helper_gvec_3_ptr
* const frint_fns
[3] = {
4878 gen_helper_sve_frint_h
,
4879 gen_helper_sve_frint_s
,
4880 gen_helper_sve_frint_d
4883 static bool trans_FRINTI(DisasContext
*s
, arg_rpr_esz
*a
)
4888 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
,
4889 frint_fns
[a
->esz
- 1]);
4892 static bool trans_FRINTX(DisasContext
*s
, arg_rpr_esz
*a
)
4894 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4895 gen_helper_sve_frintx_h
,
4896 gen_helper_sve_frintx_s
,
4897 gen_helper_sve_frintx_d
4902 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4905 static bool do_frint_mode(DisasContext
*s
, arg_rpr_esz
*a
,
4906 int mode
, gen_helper_gvec_3_ptr
*fn
)
4908 if (sve_access_check(s
)) {
4909 unsigned vsz
= vec_full_reg_size(s
);
4910 TCGv_i32 tmode
= tcg_const_i32(mode
);
4911 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4913 gen_helper_set_rmode(tmode
, tmode
, status
);
4915 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4916 vec_full_reg_offset(s
, a
->rn
),
4917 pred_full_reg_offset(s
, a
->pg
),
4918 status
, vsz
, vsz
, 0, fn
);
4920 gen_helper_set_rmode(tmode
, tmode
, status
);
4921 tcg_temp_free_i32(tmode
);
4922 tcg_temp_free_ptr(status
);
4927 static bool trans_FRINTN(DisasContext
*s
, arg_rpr_esz
*a
)
4932 return do_frint_mode(s
, a
, float_round_nearest_even
, frint_fns
[a
->esz
- 1]);
4935 static bool trans_FRINTP(DisasContext
*s
, arg_rpr_esz
*a
)
4940 return do_frint_mode(s
, a
, float_round_up
, frint_fns
[a
->esz
- 1]);
4943 static bool trans_FRINTM(DisasContext
*s
, arg_rpr_esz
*a
)
4948 return do_frint_mode(s
, a
, float_round_down
, frint_fns
[a
->esz
- 1]);
4951 static bool trans_FRINTZ(DisasContext
*s
, arg_rpr_esz
*a
)
4956 return do_frint_mode(s
, a
, float_round_to_zero
, frint_fns
[a
->esz
- 1]);
4959 static bool trans_FRINTA(DisasContext
*s
, arg_rpr_esz
*a
)
4964 return do_frint_mode(s
, a
, float_round_ties_away
, frint_fns
[a
->esz
- 1]);
4967 static bool trans_FRECPX(DisasContext
*s
, arg_rpr_esz
*a
)
4969 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4970 gen_helper_sve_frecpx_h
,
4971 gen_helper_sve_frecpx_s
,
4972 gen_helper_sve_frecpx_d
4977 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4980 static bool trans_FSQRT(DisasContext
*s
, arg_rpr_esz
*a
)
4982 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4983 gen_helper_sve_fsqrt_h
,
4984 gen_helper_sve_fsqrt_s
,
4985 gen_helper_sve_fsqrt_d
4990 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4993 static bool trans_SCVTF_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4995 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_hh
);
4998 static bool trans_SCVTF_sh(DisasContext
*s
, arg_rpr_esz
*a
)
5000 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_sh
);
5003 static bool trans_SCVTF_dh(DisasContext
*s
, arg_rpr_esz
*a
)
5005 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_dh
);
5008 static bool trans_SCVTF_ss(DisasContext
*s
, arg_rpr_esz
*a
)
5010 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_ss
);
5013 static bool trans_SCVTF_ds(DisasContext
*s
, arg_rpr_esz
*a
)
5015 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_ds
);
5018 static bool trans_SCVTF_sd(DisasContext
*s
, arg_rpr_esz
*a
)
5020 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_sd
);
5023 static bool trans_SCVTF_dd(DisasContext
*s
, arg_rpr_esz
*a
)
5025 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_dd
);
5028 static bool trans_UCVTF_hh(DisasContext
*s
, arg_rpr_esz
*a
)
5030 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_hh
);
5033 static bool trans_UCVTF_sh(DisasContext
*s
, arg_rpr_esz
*a
)
5035 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_sh
);
5038 static bool trans_UCVTF_dh(DisasContext
*s
, arg_rpr_esz
*a
)
5040 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_dh
);
5043 static bool trans_UCVTF_ss(DisasContext
*s
, arg_rpr_esz
*a
)
5045 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_ss
);
5048 static bool trans_UCVTF_ds(DisasContext
*s
, arg_rpr_esz
*a
)
5050 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_ds
);
5053 static bool trans_UCVTF_sd(DisasContext
*s
, arg_rpr_esz
*a
)
5055 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_sd
);
5058 static bool trans_UCVTF_dd(DisasContext
*s
, arg_rpr_esz
*a
)
5060 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_dd
);
5064 *** SVE Memory - 32-bit Gather and Unsized Contiguous Group
5067 /* Subroutine loading a vector register at VOFS of LEN bytes.
5068 * The load should begin at the address Rn + IMM.
5071 static void do_ldr(DisasContext
*s
, uint32_t vofs
, int len
, int rn
, int imm
)
5073 int len_align
= QEMU_ALIGN_DOWN(len
, 8);
5074 int len_remain
= len
% 8;
5075 int nparts
= len
/ 8 + ctpop8(len_remain
);
5076 int midx
= get_mem_index(s
);
5077 TCGv_i64 dirty_addr
, clean_addr
, t0
, t1
;
5079 dirty_addr
= tcg_temp_new_i64();
5080 tcg_gen_addi_i64(dirty_addr
, cpu_reg_sp(s
, rn
), imm
);
5081 clean_addr
= gen_mte_checkN(s
, dirty_addr
, false, rn
!= 31, len
);
5082 tcg_temp_free_i64(dirty_addr
);
5085 * Note that unpredicated load/store of vector/predicate registers
5086 * are defined as a stream of bytes, which equates to little-endian
5087 * operations on larger quantities.
5088 * Attempt to keep code expansion to a minimum by limiting the
5089 * amount of unrolling done.
5094 t0
= tcg_temp_new_i64();
5095 for (i
= 0; i
< len_align
; i
+= 8) {
5096 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEQ
);
5097 tcg_gen_st_i64(t0
, cpu_env
, vofs
+ i
);
5098 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
5100 tcg_temp_free_i64(t0
);
5102 TCGLabel
*loop
= gen_new_label();
5103 TCGv_ptr tp
, i
= tcg_const_local_ptr(0);
5105 /* Copy the clean address into a local temp, live across the loop. */
5107 clean_addr
= new_tmp_a64_local(s
);
5108 tcg_gen_mov_i64(clean_addr
, t0
);
5110 gen_set_label(loop
);
5112 t0
= tcg_temp_new_i64();
5113 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEQ
);
5114 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
5116 tp
= tcg_temp_new_ptr();
5117 tcg_gen_add_ptr(tp
, cpu_env
, i
);
5118 tcg_gen_addi_ptr(i
, i
, 8);
5119 tcg_gen_st_i64(t0
, tp
, vofs
);
5120 tcg_temp_free_ptr(tp
);
5121 tcg_temp_free_i64(t0
);
5123 tcg_gen_brcondi_ptr(TCG_COND_LTU
, i
, len_align
, loop
);
5124 tcg_temp_free_ptr(i
);
5128 * Predicate register loads can be any multiple of 2.
5129 * Note that we still store the entire 64-bit unit into cpu_env.
5132 t0
= tcg_temp_new_i64();
5133 switch (len_remain
) {
5137 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
,
5138 MO_LE
| ctz32(len_remain
));
5142 t1
= tcg_temp_new_i64();
5143 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEUL
);
5144 tcg_gen_addi_i64(clean_addr
, clean_addr
, 4);
5145 tcg_gen_qemu_ld_i64(t1
, clean_addr
, midx
, MO_LEUW
);
5146 tcg_gen_deposit_i64(t0
, t0
, t1
, 32, 32);
5147 tcg_temp_free_i64(t1
);
5151 g_assert_not_reached();
5153 tcg_gen_st_i64(t0
, cpu_env
, vofs
+ len_align
);
5154 tcg_temp_free_i64(t0
);
5158 /* Similarly for stores. */
5159 static void do_str(DisasContext
*s
, uint32_t vofs
, int len
, int rn
, int imm
)
5161 int len_align
= QEMU_ALIGN_DOWN(len
, 8);
5162 int len_remain
= len
% 8;
5163 int nparts
= len
/ 8 + ctpop8(len_remain
);
5164 int midx
= get_mem_index(s
);
5165 TCGv_i64 dirty_addr
, clean_addr
, t0
;
5167 dirty_addr
= tcg_temp_new_i64();
5168 tcg_gen_addi_i64(dirty_addr
, cpu_reg_sp(s
, rn
), imm
);
5169 clean_addr
= gen_mte_checkN(s
, dirty_addr
, false, rn
!= 31, len
);
5170 tcg_temp_free_i64(dirty_addr
);
5172 /* Note that unpredicated load/store of vector/predicate registers
5173 * are defined as a stream of bytes, which equates to little-endian
5174 * operations on larger quantities. There is no nice way to force
5175 * a little-endian store for aarch64_be-linux-user out of line.
5177 * Attempt to keep code expansion to a minimum by limiting the
5178 * amount of unrolling done.
5183 t0
= tcg_temp_new_i64();
5184 for (i
= 0; i
< len_align
; i
+= 8) {
5185 tcg_gen_ld_i64(t0
, cpu_env
, vofs
+ i
);
5186 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEQ
);
5187 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
5189 tcg_temp_free_i64(t0
);
5191 TCGLabel
*loop
= gen_new_label();
5192 TCGv_ptr tp
, i
= tcg_const_local_ptr(0);
5194 /* Copy the clean address into a local temp, live across the loop. */
5196 clean_addr
= new_tmp_a64_local(s
);
5197 tcg_gen_mov_i64(clean_addr
, t0
);
5199 gen_set_label(loop
);
5201 t0
= tcg_temp_new_i64();
5202 tp
= tcg_temp_new_ptr();
5203 tcg_gen_add_ptr(tp
, cpu_env
, i
);
5204 tcg_gen_ld_i64(t0
, tp
, vofs
);
5205 tcg_gen_addi_ptr(i
, i
, 8);
5206 tcg_temp_free_ptr(tp
);
5208 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEQ
);
5209 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
5210 tcg_temp_free_i64(t0
);
5212 tcg_gen_brcondi_ptr(TCG_COND_LTU
, i
, len_align
, loop
);
5213 tcg_temp_free_ptr(i
);
5216 /* Predicate register stores can be any multiple of 2. */
5218 t0
= tcg_temp_new_i64();
5219 tcg_gen_ld_i64(t0
, cpu_env
, vofs
+ len_align
);
5221 switch (len_remain
) {
5225 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
,
5226 MO_LE
| ctz32(len_remain
));
5230 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUL
);
5231 tcg_gen_addi_i64(clean_addr
, clean_addr
, 4);
5232 tcg_gen_shri_i64(t0
, t0
, 32);
5233 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUW
);
5237 g_assert_not_reached();
5239 tcg_temp_free_i64(t0
);
5243 static bool trans_LDR_zri(DisasContext
*s
, arg_rri
*a
)
5245 if (sve_access_check(s
)) {
5246 int size
= vec_full_reg_size(s
);
5247 int off
= vec_full_reg_offset(s
, a
->rd
);
5248 do_ldr(s
, off
, size
, a
->rn
, a
->imm
* size
);
5253 static bool trans_LDR_pri(DisasContext
*s
, arg_rri
*a
)
5255 if (sve_access_check(s
)) {
5256 int size
= pred_full_reg_size(s
);
5257 int off
= pred_full_reg_offset(s
, a
->rd
);
5258 do_ldr(s
, off
, size
, a
->rn
, a
->imm
* size
);
5263 static bool trans_STR_zri(DisasContext
*s
, arg_rri
*a
)
5265 if (sve_access_check(s
)) {
5266 int size
= vec_full_reg_size(s
);
5267 int off
= vec_full_reg_offset(s
, a
->rd
);
5268 do_str(s
, off
, size
, a
->rn
, a
->imm
* size
);
5273 static bool trans_STR_pri(DisasContext
*s
, arg_rri
*a
)
5275 if (sve_access_check(s
)) {
5276 int size
= pred_full_reg_size(s
);
5277 int off
= pred_full_reg_offset(s
, a
->rd
);
5278 do_str(s
, off
, size
, a
->rn
, a
->imm
* size
);
5284 *** SVE Memory - Contiguous Load Group
5287 /* The memory mode of the dtype. */
5288 static const MemOp dtype_mop
[16] = {
5289 MO_UB
, MO_UB
, MO_UB
, MO_UB
,
5290 MO_SL
, MO_UW
, MO_UW
, MO_UW
,
5291 MO_SW
, MO_SW
, MO_UL
, MO_UL
,
5292 MO_SB
, MO_SB
, MO_SB
, MO_Q
5295 #define dtype_msz(x) (dtype_mop[x] & MO_SIZE)
5297 /* The vector element size of dtype. */
5298 static const uint8_t dtype_esz
[16] = {
5305 static void do_mem_zpa(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
,
5306 int dtype
, uint32_t mte_n
, bool is_write
,
5307 gen_helper_gvec_mem
*fn
)
5309 unsigned vsz
= vec_full_reg_size(s
);
5315 * For e.g. LD4, there are not enough arguments to pass all 4
5316 * registers as pointers, so encode the regno into the data field.
5317 * For consistency, do this even for LD1.
5319 if (s
->mte_active
[0]) {
5320 int msz
= dtype_msz(dtype
);
5322 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, get_mem_index(s
));
5323 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
5324 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
5325 desc
= FIELD_DP32(desc
, MTEDESC
, WRITE
, is_write
);
5326 desc
= FIELD_DP32(desc
, MTEDESC
, SIZEM1
, (mte_n
<< msz
) - 1);
5327 desc
<<= SVE_MTEDESC_SHIFT
;
5329 addr
= clean_data_tbi(s
, addr
);
5332 desc
= simd_desc(vsz
, vsz
, zt
| desc
);
5333 t_desc
= tcg_const_i32(desc
);
5334 t_pg
= tcg_temp_new_ptr();
5336 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
5337 fn(cpu_env
, t_pg
, addr
, t_desc
);
5339 tcg_temp_free_ptr(t_pg
);
5340 tcg_temp_free_i32(t_desc
);
5343 /* Indexed by [mte][be][dtype][nreg] */
5344 static gen_helper_gvec_mem
* const ldr_fns
[2][2][16][4] = {
5345 { /* mte inactive, little-endian */
5346 { { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld2bb_r
,
5347 gen_helper_sve_ld3bb_r
, gen_helper_sve_ld4bb_r
},
5348 { gen_helper_sve_ld1bhu_r
, NULL
, NULL
, NULL
},
5349 { gen_helper_sve_ld1bsu_r
, NULL
, NULL
, NULL
},
5350 { gen_helper_sve_ld1bdu_r
, NULL
, NULL
, NULL
},
5352 { gen_helper_sve_ld1sds_le_r
, NULL
, NULL
, NULL
},
5353 { gen_helper_sve_ld1hh_le_r
, gen_helper_sve_ld2hh_le_r
,
5354 gen_helper_sve_ld3hh_le_r
, gen_helper_sve_ld4hh_le_r
},
5355 { gen_helper_sve_ld1hsu_le_r
, NULL
, NULL
, NULL
},
5356 { gen_helper_sve_ld1hdu_le_r
, NULL
, NULL
, NULL
},
5358 { gen_helper_sve_ld1hds_le_r
, NULL
, NULL
, NULL
},
5359 { gen_helper_sve_ld1hss_le_r
, NULL
, NULL
, NULL
},
5360 { gen_helper_sve_ld1ss_le_r
, gen_helper_sve_ld2ss_le_r
,
5361 gen_helper_sve_ld3ss_le_r
, gen_helper_sve_ld4ss_le_r
},
5362 { gen_helper_sve_ld1sdu_le_r
, NULL
, NULL
, NULL
},
5364 { gen_helper_sve_ld1bds_r
, NULL
, NULL
, NULL
},
5365 { gen_helper_sve_ld1bss_r
, NULL
, NULL
, NULL
},
5366 { gen_helper_sve_ld1bhs_r
, NULL
, NULL
, NULL
},
5367 { gen_helper_sve_ld1dd_le_r
, gen_helper_sve_ld2dd_le_r
,
5368 gen_helper_sve_ld3dd_le_r
, gen_helper_sve_ld4dd_le_r
} },
5370 /* mte inactive, big-endian */
5371 { { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld2bb_r
,
5372 gen_helper_sve_ld3bb_r
, gen_helper_sve_ld4bb_r
},
5373 { gen_helper_sve_ld1bhu_r
, NULL
, NULL
, NULL
},
5374 { gen_helper_sve_ld1bsu_r
, NULL
, NULL
, NULL
},
5375 { gen_helper_sve_ld1bdu_r
, NULL
, NULL
, NULL
},
5377 { gen_helper_sve_ld1sds_be_r
, NULL
, NULL
, NULL
},
5378 { gen_helper_sve_ld1hh_be_r
, gen_helper_sve_ld2hh_be_r
,
5379 gen_helper_sve_ld3hh_be_r
, gen_helper_sve_ld4hh_be_r
},
5380 { gen_helper_sve_ld1hsu_be_r
, NULL
, NULL
, NULL
},
5381 { gen_helper_sve_ld1hdu_be_r
, NULL
, NULL
, NULL
},
5383 { gen_helper_sve_ld1hds_be_r
, NULL
, NULL
, NULL
},
5384 { gen_helper_sve_ld1hss_be_r
, NULL
, NULL
, NULL
},
5385 { gen_helper_sve_ld1ss_be_r
, gen_helper_sve_ld2ss_be_r
,
5386 gen_helper_sve_ld3ss_be_r
, gen_helper_sve_ld4ss_be_r
},
5387 { gen_helper_sve_ld1sdu_be_r
, NULL
, NULL
, NULL
},
5389 { gen_helper_sve_ld1bds_r
, NULL
, NULL
, NULL
},
5390 { gen_helper_sve_ld1bss_r
, NULL
, NULL
, NULL
},
5391 { gen_helper_sve_ld1bhs_r
, NULL
, NULL
, NULL
},
5392 { gen_helper_sve_ld1dd_be_r
, gen_helper_sve_ld2dd_be_r
,
5393 gen_helper_sve_ld3dd_be_r
, gen_helper_sve_ld4dd_be_r
} } },
5395 { /* mte active, little-endian */
5396 { { gen_helper_sve_ld1bb_r_mte
,
5397 gen_helper_sve_ld2bb_r_mte
,
5398 gen_helper_sve_ld3bb_r_mte
,
5399 gen_helper_sve_ld4bb_r_mte
},
5400 { gen_helper_sve_ld1bhu_r_mte
, NULL
, NULL
, NULL
},
5401 { gen_helper_sve_ld1bsu_r_mte
, NULL
, NULL
, NULL
},
5402 { gen_helper_sve_ld1bdu_r_mte
, NULL
, NULL
, NULL
},
5404 { gen_helper_sve_ld1sds_le_r_mte
, NULL
, NULL
, NULL
},
5405 { gen_helper_sve_ld1hh_le_r_mte
,
5406 gen_helper_sve_ld2hh_le_r_mte
,
5407 gen_helper_sve_ld3hh_le_r_mte
,
5408 gen_helper_sve_ld4hh_le_r_mte
},
5409 { gen_helper_sve_ld1hsu_le_r_mte
, NULL
, NULL
, NULL
},
5410 { gen_helper_sve_ld1hdu_le_r_mte
, NULL
, NULL
, NULL
},
5412 { gen_helper_sve_ld1hds_le_r_mte
, NULL
, NULL
, NULL
},
5413 { gen_helper_sve_ld1hss_le_r_mte
, NULL
, NULL
, NULL
},
5414 { gen_helper_sve_ld1ss_le_r_mte
,
5415 gen_helper_sve_ld2ss_le_r_mte
,
5416 gen_helper_sve_ld3ss_le_r_mte
,
5417 gen_helper_sve_ld4ss_le_r_mte
},
5418 { gen_helper_sve_ld1sdu_le_r_mte
, NULL
, NULL
, NULL
},
5420 { gen_helper_sve_ld1bds_r_mte
, NULL
, NULL
, NULL
},
5421 { gen_helper_sve_ld1bss_r_mte
, NULL
, NULL
, NULL
},
5422 { gen_helper_sve_ld1bhs_r_mte
, NULL
, NULL
, NULL
},
5423 { gen_helper_sve_ld1dd_le_r_mte
,
5424 gen_helper_sve_ld2dd_le_r_mte
,
5425 gen_helper_sve_ld3dd_le_r_mte
,
5426 gen_helper_sve_ld4dd_le_r_mte
} },
5428 /* mte active, big-endian */
5429 { { gen_helper_sve_ld1bb_r_mte
,
5430 gen_helper_sve_ld2bb_r_mte
,
5431 gen_helper_sve_ld3bb_r_mte
,
5432 gen_helper_sve_ld4bb_r_mte
},
5433 { gen_helper_sve_ld1bhu_r_mte
, NULL
, NULL
, NULL
},
5434 { gen_helper_sve_ld1bsu_r_mte
, NULL
, NULL
, NULL
},
5435 { gen_helper_sve_ld1bdu_r_mte
, NULL
, NULL
, NULL
},
5437 { gen_helper_sve_ld1sds_be_r_mte
, NULL
, NULL
, NULL
},
5438 { gen_helper_sve_ld1hh_be_r_mte
,
5439 gen_helper_sve_ld2hh_be_r_mte
,
5440 gen_helper_sve_ld3hh_be_r_mte
,
5441 gen_helper_sve_ld4hh_be_r_mte
},
5442 { gen_helper_sve_ld1hsu_be_r_mte
, NULL
, NULL
, NULL
},
5443 { gen_helper_sve_ld1hdu_be_r_mte
, NULL
, NULL
, NULL
},
5445 { gen_helper_sve_ld1hds_be_r_mte
, NULL
, NULL
, NULL
},
5446 { gen_helper_sve_ld1hss_be_r_mte
, NULL
, NULL
, NULL
},
5447 { gen_helper_sve_ld1ss_be_r_mte
,
5448 gen_helper_sve_ld2ss_be_r_mte
,
5449 gen_helper_sve_ld3ss_be_r_mte
,
5450 gen_helper_sve_ld4ss_be_r_mte
},
5451 { gen_helper_sve_ld1sdu_be_r_mte
, NULL
, NULL
, NULL
},
5453 { gen_helper_sve_ld1bds_r_mte
, NULL
, NULL
, NULL
},
5454 { gen_helper_sve_ld1bss_r_mte
, NULL
, NULL
, NULL
},
5455 { gen_helper_sve_ld1bhs_r_mte
, NULL
, NULL
, NULL
},
5456 { gen_helper_sve_ld1dd_be_r_mte
,
5457 gen_helper_sve_ld2dd_be_r_mte
,
5458 gen_helper_sve_ld3dd_be_r_mte
,
5459 gen_helper_sve_ld4dd_be_r_mte
} } },
5462 static void do_ld_zpa(DisasContext
*s
, int zt
, int pg
,
5463 TCGv_i64 addr
, int dtype
, int nreg
)
5465 gen_helper_gvec_mem
*fn
5466 = ldr_fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][dtype
][nreg
];
5469 * While there are holes in the table, they are not
5470 * accessible via the instruction encoding.
5473 do_mem_zpa(s
, zt
, pg
, addr
, dtype
, nreg
, false, fn
);
5476 static bool trans_LD_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5481 if (sve_access_check(s
)) {
5482 TCGv_i64 addr
= new_tmp_a64(s
);
5483 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
5484 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5485 do_ld_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, a
->nreg
);
5490 static bool trans_LD_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5492 if (sve_access_check(s
)) {
5493 int vsz
= vec_full_reg_size(s
);
5494 int elements
= vsz
>> dtype_esz
[a
->dtype
];
5495 TCGv_i64 addr
= new_tmp_a64(s
);
5497 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
),
5498 (a
->imm
* elements
* (a
->nreg
+ 1))
5499 << dtype_msz(a
->dtype
));
5500 do_ld_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, a
->nreg
);
5505 static bool trans_LDFF1_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5507 static gen_helper_gvec_mem
* const fns
[2][2][16] = {
5508 { /* mte inactive, little-endian */
5509 { gen_helper_sve_ldff1bb_r
,
5510 gen_helper_sve_ldff1bhu_r
,
5511 gen_helper_sve_ldff1bsu_r
,
5512 gen_helper_sve_ldff1bdu_r
,
5514 gen_helper_sve_ldff1sds_le_r
,
5515 gen_helper_sve_ldff1hh_le_r
,
5516 gen_helper_sve_ldff1hsu_le_r
,
5517 gen_helper_sve_ldff1hdu_le_r
,
5519 gen_helper_sve_ldff1hds_le_r
,
5520 gen_helper_sve_ldff1hss_le_r
,
5521 gen_helper_sve_ldff1ss_le_r
,
5522 gen_helper_sve_ldff1sdu_le_r
,
5524 gen_helper_sve_ldff1bds_r
,
5525 gen_helper_sve_ldff1bss_r
,
5526 gen_helper_sve_ldff1bhs_r
,
5527 gen_helper_sve_ldff1dd_le_r
},
5529 /* mte inactive, big-endian */
5530 { gen_helper_sve_ldff1bb_r
,
5531 gen_helper_sve_ldff1bhu_r
,
5532 gen_helper_sve_ldff1bsu_r
,
5533 gen_helper_sve_ldff1bdu_r
,
5535 gen_helper_sve_ldff1sds_be_r
,
5536 gen_helper_sve_ldff1hh_be_r
,
5537 gen_helper_sve_ldff1hsu_be_r
,
5538 gen_helper_sve_ldff1hdu_be_r
,
5540 gen_helper_sve_ldff1hds_be_r
,
5541 gen_helper_sve_ldff1hss_be_r
,
5542 gen_helper_sve_ldff1ss_be_r
,
5543 gen_helper_sve_ldff1sdu_be_r
,
5545 gen_helper_sve_ldff1bds_r
,
5546 gen_helper_sve_ldff1bss_r
,
5547 gen_helper_sve_ldff1bhs_r
,
5548 gen_helper_sve_ldff1dd_be_r
} },
5550 { /* mte active, little-endian */
5551 { gen_helper_sve_ldff1bb_r_mte
,
5552 gen_helper_sve_ldff1bhu_r_mte
,
5553 gen_helper_sve_ldff1bsu_r_mte
,
5554 gen_helper_sve_ldff1bdu_r_mte
,
5556 gen_helper_sve_ldff1sds_le_r_mte
,
5557 gen_helper_sve_ldff1hh_le_r_mte
,
5558 gen_helper_sve_ldff1hsu_le_r_mte
,
5559 gen_helper_sve_ldff1hdu_le_r_mte
,
5561 gen_helper_sve_ldff1hds_le_r_mte
,
5562 gen_helper_sve_ldff1hss_le_r_mte
,
5563 gen_helper_sve_ldff1ss_le_r_mte
,
5564 gen_helper_sve_ldff1sdu_le_r_mte
,
5566 gen_helper_sve_ldff1bds_r_mte
,
5567 gen_helper_sve_ldff1bss_r_mte
,
5568 gen_helper_sve_ldff1bhs_r_mte
,
5569 gen_helper_sve_ldff1dd_le_r_mte
},
5571 /* mte active, big-endian */
5572 { gen_helper_sve_ldff1bb_r_mte
,
5573 gen_helper_sve_ldff1bhu_r_mte
,
5574 gen_helper_sve_ldff1bsu_r_mte
,
5575 gen_helper_sve_ldff1bdu_r_mte
,
5577 gen_helper_sve_ldff1sds_be_r_mte
,
5578 gen_helper_sve_ldff1hh_be_r_mte
,
5579 gen_helper_sve_ldff1hsu_be_r_mte
,
5580 gen_helper_sve_ldff1hdu_be_r_mte
,
5582 gen_helper_sve_ldff1hds_be_r_mte
,
5583 gen_helper_sve_ldff1hss_be_r_mte
,
5584 gen_helper_sve_ldff1ss_be_r_mte
,
5585 gen_helper_sve_ldff1sdu_be_r_mte
,
5587 gen_helper_sve_ldff1bds_r_mte
,
5588 gen_helper_sve_ldff1bss_r_mte
,
5589 gen_helper_sve_ldff1bhs_r_mte
,
5590 gen_helper_sve_ldff1dd_be_r_mte
} },
5593 if (sve_access_check(s
)) {
5594 TCGv_i64 addr
= new_tmp_a64(s
);
5595 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
5596 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5597 do_mem_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, 1, false,
5598 fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][a
->dtype
]);
5603 static bool trans_LDNF1_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5605 static gen_helper_gvec_mem
* const fns
[2][2][16] = {
5606 { /* mte inactive, little-endian */
5607 { gen_helper_sve_ldnf1bb_r
,
5608 gen_helper_sve_ldnf1bhu_r
,
5609 gen_helper_sve_ldnf1bsu_r
,
5610 gen_helper_sve_ldnf1bdu_r
,
5612 gen_helper_sve_ldnf1sds_le_r
,
5613 gen_helper_sve_ldnf1hh_le_r
,
5614 gen_helper_sve_ldnf1hsu_le_r
,
5615 gen_helper_sve_ldnf1hdu_le_r
,
5617 gen_helper_sve_ldnf1hds_le_r
,
5618 gen_helper_sve_ldnf1hss_le_r
,
5619 gen_helper_sve_ldnf1ss_le_r
,
5620 gen_helper_sve_ldnf1sdu_le_r
,
5622 gen_helper_sve_ldnf1bds_r
,
5623 gen_helper_sve_ldnf1bss_r
,
5624 gen_helper_sve_ldnf1bhs_r
,
5625 gen_helper_sve_ldnf1dd_le_r
},
5627 /* mte inactive, big-endian */
5628 { gen_helper_sve_ldnf1bb_r
,
5629 gen_helper_sve_ldnf1bhu_r
,
5630 gen_helper_sve_ldnf1bsu_r
,
5631 gen_helper_sve_ldnf1bdu_r
,
5633 gen_helper_sve_ldnf1sds_be_r
,
5634 gen_helper_sve_ldnf1hh_be_r
,
5635 gen_helper_sve_ldnf1hsu_be_r
,
5636 gen_helper_sve_ldnf1hdu_be_r
,
5638 gen_helper_sve_ldnf1hds_be_r
,
5639 gen_helper_sve_ldnf1hss_be_r
,
5640 gen_helper_sve_ldnf1ss_be_r
,
5641 gen_helper_sve_ldnf1sdu_be_r
,
5643 gen_helper_sve_ldnf1bds_r
,
5644 gen_helper_sve_ldnf1bss_r
,
5645 gen_helper_sve_ldnf1bhs_r
,
5646 gen_helper_sve_ldnf1dd_be_r
} },
5648 { /* mte inactive, little-endian */
5649 { gen_helper_sve_ldnf1bb_r_mte
,
5650 gen_helper_sve_ldnf1bhu_r_mte
,
5651 gen_helper_sve_ldnf1bsu_r_mte
,
5652 gen_helper_sve_ldnf1bdu_r_mte
,
5654 gen_helper_sve_ldnf1sds_le_r_mte
,
5655 gen_helper_sve_ldnf1hh_le_r_mte
,
5656 gen_helper_sve_ldnf1hsu_le_r_mte
,
5657 gen_helper_sve_ldnf1hdu_le_r_mte
,
5659 gen_helper_sve_ldnf1hds_le_r_mte
,
5660 gen_helper_sve_ldnf1hss_le_r_mte
,
5661 gen_helper_sve_ldnf1ss_le_r_mte
,
5662 gen_helper_sve_ldnf1sdu_le_r_mte
,
5664 gen_helper_sve_ldnf1bds_r_mte
,
5665 gen_helper_sve_ldnf1bss_r_mte
,
5666 gen_helper_sve_ldnf1bhs_r_mte
,
5667 gen_helper_sve_ldnf1dd_le_r_mte
},
5669 /* mte inactive, big-endian */
5670 { gen_helper_sve_ldnf1bb_r_mte
,
5671 gen_helper_sve_ldnf1bhu_r_mte
,
5672 gen_helper_sve_ldnf1bsu_r_mte
,
5673 gen_helper_sve_ldnf1bdu_r_mte
,
5675 gen_helper_sve_ldnf1sds_be_r_mte
,
5676 gen_helper_sve_ldnf1hh_be_r_mte
,
5677 gen_helper_sve_ldnf1hsu_be_r_mte
,
5678 gen_helper_sve_ldnf1hdu_be_r_mte
,
5680 gen_helper_sve_ldnf1hds_be_r_mte
,
5681 gen_helper_sve_ldnf1hss_be_r_mte
,
5682 gen_helper_sve_ldnf1ss_be_r_mte
,
5683 gen_helper_sve_ldnf1sdu_be_r_mte
,
5685 gen_helper_sve_ldnf1bds_r_mte
,
5686 gen_helper_sve_ldnf1bss_r_mte
,
5687 gen_helper_sve_ldnf1bhs_r_mte
,
5688 gen_helper_sve_ldnf1dd_be_r_mte
} },
5691 if (sve_access_check(s
)) {
5692 int vsz
= vec_full_reg_size(s
);
5693 int elements
= vsz
>> dtype_esz
[a
->dtype
];
5694 int off
= (a
->imm
* elements
) << dtype_msz(a
->dtype
);
5695 TCGv_i64 addr
= new_tmp_a64(s
);
5697 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), off
);
5698 do_mem_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, 1, false,
5699 fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][a
->dtype
]);
5704 static void do_ldrq(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
, int dtype
)
5706 unsigned vsz
= vec_full_reg_size(s
);
5710 /* Load the first quadword using the normal predicated load helpers. */
5711 poff
= pred_full_reg_offset(s
, pg
);
5714 * Zero-extend the first 16 bits of the predicate into a temporary.
5715 * This avoids triggering an assert making sure we don't have bits
5716 * set within a predicate beyond VQ, but we have lowered VQ to 1
5717 * for this load operation.
5719 TCGv_i64 tmp
= tcg_temp_new_i64();
5720 #ifdef HOST_WORDS_BIGENDIAN
5723 tcg_gen_ld16u_i64(tmp
, cpu_env
, poff
);
5725 poff
= offsetof(CPUARMState
, vfp
.preg_tmp
);
5726 tcg_gen_st_i64(tmp
, cpu_env
, poff
);
5727 tcg_temp_free_i64(tmp
);
5730 t_pg
= tcg_temp_new_ptr();
5731 tcg_gen_addi_ptr(t_pg
, cpu_env
, poff
);
5733 gen_helper_gvec_mem
*fn
5734 = ldr_fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][dtype
][0];
5735 fn(cpu_env
, t_pg
, addr
, tcg_constant_i32(simd_desc(16, 16, zt
)));
5737 tcg_temp_free_ptr(t_pg
);
5739 /* Replicate that first quadword. */
5741 int doff
= vec_full_reg_offset(s
, zt
);
5742 tcg_gen_gvec_dup_mem(4, doff
+ 16, doff
, vsz
- 16, vsz
- 16);
5746 static bool trans_LD1RQ_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5751 if (sve_access_check(s
)) {
5752 int msz
= dtype_msz(a
->dtype
);
5753 TCGv_i64 addr
= new_tmp_a64(s
);
5754 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), msz
);
5755 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5756 do_ldrq(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5761 static bool trans_LD1RQ_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5763 if (sve_access_check(s
)) {
5764 TCGv_i64 addr
= new_tmp_a64(s
);
5765 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), a
->imm
* 16);
5766 do_ldrq(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5771 static void do_ldro(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
, int dtype
)
5773 unsigned vsz
= vec_full_reg_size(s
);
5780 * Note that this UNDEFINED check comes after CheckSVEEnabled()
5781 * in the ARM pseudocode, which is the sve_access_check() done
5782 * in our caller. We should not now return false from the caller.
5784 unallocated_encoding(s
);
5788 /* Load the first octaword using the normal predicated load helpers. */
5790 poff
= pred_full_reg_offset(s
, pg
);
5793 * Zero-extend the first 32 bits of the predicate into a temporary.
5794 * This avoids triggering an assert making sure we don't have bits
5795 * set within a predicate beyond VQ, but we have lowered VQ to 2
5796 * for this load operation.
5798 TCGv_i64 tmp
= tcg_temp_new_i64();
5799 #ifdef HOST_WORDS_BIGENDIAN
5802 tcg_gen_ld32u_i64(tmp
, cpu_env
, poff
);
5804 poff
= offsetof(CPUARMState
, vfp
.preg_tmp
);
5805 tcg_gen_st_i64(tmp
, cpu_env
, poff
);
5806 tcg_temp_free_i64(tmp
);
5809 t_pg
= tcg_temp_new_ptr();
5810 tcg_gen_addi_ptr(t_pg
, cpu_env
, poff
);
5812 gen_helper_gvec_mem
*fn
5813 = ldr_fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][dtype
][0];
5814 fn(cpu_env
, t_pg
, addr
, tcg_constant_i32(simd_desc(32, 32, zt
)));
5816 tcg_temp_free_ptr(t_pg
);
5819 * Replicate that first octaword.
5820 * The replication happens in units of 32; if the full vector size
5821 * is not a multiple of 32, the final bits are zeroed.
5823 doff
= vec_full_reg_offset(s
, zt
);
5824 vsz_r32
= QEMU_ALIGN_DOWN(vsz
, 32);
5826 tcg_gen_gvec_dup_mem(5, doff
+ 32, doff
, vsz_r32
- 32, vsz_r32
- 32);
5830 tcg_gen_gvec_dup_imm(MO_64
, doff
+ vsz_r32
, vsz
, vsz
, 0);
5834 static bool trans_LD1RO_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5836 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
5842 if (sve_access_check(s
)) {
5843 TCGv_i64 addr
= new_tmp_a64(s
);
5844 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
5845 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5846 do_ldro(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5851 static bool trans_LD1RO_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5853 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
5856 if (sve_access_check(s
)) {
5857 TCGv_i64 addr
= new_tmp_a64(s
);
5858 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), a
->imm
* 32);
5859 do_ldro(s
, a
->rd
, a
->pg
, addr
, a
->dtype
);
5864 /* Load and broadcast element. */
5865 static bool trans_LD1R_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5867 unsigned vsz
= vec_full_reg_size(s
);
5868 unsigned psz
= pred_full_reg_size(s
);
5869 unsigned esz
= dtype_esz
[a
->dtype
];
5870 unsigned msz
= dtype_msz(a
->dtype
);
5872 TCGv_i64 temp
, clean_addr
;
5874 if (!sve_access_check(s
)) {
5878 over
= gen_new_label();
5880 /* If the guarding predicate has no bits set, no load occurs. */
5882 /* Reduce the pred_esz_masks value simply to reduce the
5883 * size of the code generated here.
5885 uint64_t psz_mask
= MAKE_64BIT_MASK(0, psz
* 8);
5886 temp
= tcg_temp_new_i64();
5887 tcg_gen_ld_i64(temp
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
5888 tcg_gen_andi_i64(temp
, temp
, pred_esz_masks
[esz
] & psz_mask
);
5889 tcg_gen_brcondi_i64(TCG_COND_EQ
, temp
, 0, over
);
5890 tcg_temp_free_i64(temp
);
5892 TCGv_i32 t32
= tcg_temp_new_i32();
5893 find_last_active(s
, t32
, esz
, a
->pg
);
5894 tcg_gen_brcondi_i32(TCG_COND_LT
, t32
, 0, over
);
5895 tcg_temp_free_i32(t32
);
5898 /* Load the data. */
5899 temp
= tcg_temp_new_i64();
5900 tcg_gen_addi_i64(temp
, cpu_reg_sp(s
, a
->rn
), a
->imm
<< msz
);
5901 clean_addr
= gen_mte_check1(s
, temp
, false, true, msz
);
5903 tcg_gen_qemu_ld_i64(temp
, clean_addr
, get_mem_index(s
),
5904 finalize_memop(s
, dtype_mop
[a
->dtype
]));
5906 /* Broadcast to *all* elements. */
5907 tcg_gen_gvec_dup_i64(esz
, vec_full_reg_offset(s
, a
->rd
),
5909 tcg_temp_free_i64(temp
);
5911 /* Zero the inactive elements. */
5912 gen_set_label(over
);
5913 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, esz
, false);
5916 static void do_st_zpa(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
,
5917 int msz
, int esz
, int nreg
)
5919 static gen_helper_gvec_mem
* const fn_single
[2][2][4][4] = {
5920 { { { gen_helper_sve_st1bb_r
,
5921 gen_helper_sve_st1bh_r
,
5922 gen_helper_sve_st1bs_r
,
5923 gen_helper_sve_st1bd_r
},
5925 gen_helper_sve_st1hh_le_r
,
5926 gen_helper_sve_st1hs_le_r
,
5927 gen_helper_sve_st1hd_le_r
},
5929 gen_helper_sve_st1ss_le_r
,
5930 gen_helper_sve_st1sd_le_r
},
5932 gen_helper_sve_st1dd_le_r
} },
5933 { { gen_helper_sve_st1bb_r
,
5934 gen_helper_sve_st1bh_r
,
5935 gen_helper_sve_st1bs_r
,
5936 gen_helper_sve_st1bd_r
},
5938 gen_helper_sve_st1hh_be_r
,
5939 gen_helper_sve_st1hs_be_r
,
5940 gen_helper_sve_st1hd_be_r
},
5942 gen_helper_sve_st1ss_be_r
,
5943 gen_helper_sve_st1sd_be_r
},
5945 gen_helper_sve_st1dd_be_r
} } },
5947 { { { gen_helper_sve_st1bb_r_mte
,
5948 gen_helper_sve_st1bh_r_mte
,
5949 gen_helper_sve_st1bs_r_mte
,
5950 gen_helper_sve_st1bd_r_mte
},
5952 gen_helper_sve_st1hh_le_r_mte
,
5953 gen_helper_sve_st1hs_le_r_mte
,
5954 gen_helper_sve_st1hd_le_r_mte
},
5956 gen_helper_sve_st1ss_le_r_mte
,
5957 gen_helper_sve_st1sd_le_r_mte
},
5959 gen_helper_sve_st1dd_le_r_mte
} },
5960 { { gen_helper_sve_st1bb_r_mte
,
5961 gen_helper_sve_st1bh_r_mte
,
5962 gen_helper_sve_st1bs_r_mte
,
5963 gen_helper_sve_st1bd_r_mte
},
5965 gen_helper_sve_st1hh_be_r_mte
,
5966 gen_helper_sve_st1hs_be_r_mte
,
5967 gen_helper_sve_st1hd_be_r_mte
},
5969 gen_helper_sve_st1ss_be_r_mte
,
5970 gen_helper_sve_st1sd_be_r_mte
},
5972 gen_helper_sve_st1dd_be_r_mte
} } },
5974 static gen_helper_gvec_mem
* const fn_multiple
[2][2][3][4] = {
5975 { { { gen_helper_sve_st2bb_r
,
5976 gen_helper_sve_st2hh_le_r
,
5977 gen_helper_sve_st2ss_le_r
,
5978 gen_helper_sve_st2dd_le_r
},
5979 { gen_helper_sve_st3bb_r
,
5980 gen_helper_sve_st3hh_le_r
,
5981 gen_helper_sve_st3ss_le_r
,
5982 gen_helper_sve_st3dd_le_r
},
5983 { gen_helper_sve_st4bb_r
,
5984 gen_helper_sve_st4hh_le_r
,
5985 gen_helper_sve_st4ss_le_r
,
5986 gen_helper_sve_st4dd_le_r
} },
5987 { { gen_helper_sve_st2bb_r
,
5988 gen_helper_sve_st2hh_be_r
,
5989 gen_helper_sve_st2ss_be_r
,
5990 gen_helper_sve_st2dd_be_r
},
5991 { gen_helper_sve_st3bb_r
,
5992 gen_helper_sve_st3hh_be_r
,
5993 gen_helper_sve_st3ss_be_r
,
5994 gen_helper_sve_st3dd_be_r
},
5995 { gen_helper_sve_st4bb_r
,
5996 gen_helper_sve_st4hh_be_r
,
5997 gen_helper_sve_st4ss_be_r
,
5998 gen_helper_sve_st4dd_be_r
} } },
5999 { { { gen_helper_sve_st2bb_r_mte
,
6000 gen_helper_sve_st2hh_le_r_mte
,
6001 gen_helper_sve_st2ss_le_r_mte
,
6002 gen_helper_sve_st2dd_le_r_mte
},
6003 { gen_helper_sve_st3bb_r_mte
,
6004 gen_helper_sve_st3hh_le_r_mte
,
6005 gen_helper_sve_st3ss_le_r_mte
,
6006 gen_helper_sve_st3dd_le_r_mte
},
6007 { gen_helper_sve_st4bb_r_mte
,
6008 gen_helper_sve_st4hh_le_r_mte
,
6009 gen_helper_sve_st4ss_le_r_mte
,
6010 gen_helper_sve_st4dd_le_r_mte
} },
6011 { { gen_helper_sve_st2bb_r_mte
,
6012 gen_helper_sve_st2hh_be_r_mte
,
6013 gen_helper_sve_st2ss_be_r_mte
,
6014 gen_helper_sve_st2dd_be_r_mte
},
6015 { gen_helper_sve_st3bb_r_mte
,
6016 gen_helper_sve_st3hh_be_r_mte
,
6017 gen_helper_sve_st3ss_be_r_mte
,
6018 gen_helper_sve_st3dd_be_r_mte
},
6019 { gen_helper_sve_st4bb_r_mte
,
6020 gen_helper_sve_st4hh_be_r_mte
,
6021 gen_helper_sve_st4ss_be_r_mte
,
6022 gen_helper_sve_st4dd_be_r_mte
} } },
6024 gen_helper_gvec_mem
*fn
;
6025 int be
= s
->be_data
== MO_BE
;
6029 fn
= fn_single
[s
->mte_active
[0]][be
][msz
][esz
];
6032 /* ST2, ST3, ST4 -- msz == esz, enforced by encoding */
6034 fn
= fn_multiple
[s
->mte_active
[0]][be
][nreg
- 1][msz
];
6037 do_mem_zpa(s
, zt
, pg
, addr
, msz_dtype(s
, msz
), nreg
, true, fn
);
6040 static bool trans_ST_zprr(DisasContext
*s
, arg_rprr_store
*a
)
6042 if (a
->rm
== 31 || a
->msz
> a
->esz
) {
6045 if (sve_access_check(s
)) {
6046 TCGv_i64 addr
= new_tmp_a64(s
);
6047 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), a
->msz
);
6048 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
6049 do_st_zpa(s
, a
->rd
, a
->pg
, addr
, a
->msz
, a
->esz
, a
->nreg
);
6054 static bool trans_ST_zpri(DisasContext
*s
, arg_rpri_store
*a
)
6056 if (a
->msz
> a
->esz
) {
6059 if (sve_access_check(s
)) {
6060 int vsz
= vec_full_reg_size(s
);
6061 int elements
= vsz
>> a
->esz
;
6062 TCGv_i64 addr
= new_tmp_a64(s
);
6064 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
),
6065 (a
->imm
* elements
* (a
->nreg
+ 1)) << a
->msz
);
6066 do_st_zpa(s
, a
->rd
, a
->pg
, addr
, a
->msz
, a
->esz
, a
->nreg
);
6072 *** SVE gather loads / scatter stores
6075 static void do_mem_zpz(DisasContext
*s
, int zt
, int pg
, int zm
,
6076 int scale
, TCGv_i64 scalar
, int msz
, bool is_write
,
6077 gen_helper_gvec_mem_scatter
*fn
)
6079 unsigned vsz
= vec_full_reg_size(s
);
6080 TCGv_ptr t_zm
= tcg_temp_new_ptr();
6081 TCGv_ptr t_pg
= tcg_temp_new_ptr();
6082 TCGv_ptr t_zt
= tcg_temp_new_ptr();
6086 if (s
->mte_active
[0]) {
6087 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, get_mem_index(s
));
6088 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
6089 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
6090 desc
= FIELD_DP32(desc
, MTEDESC
, WRITE
, is_write
);
6091 desc
= FIELD_DP32(desc
, MTEDESC
, SIZEM1
, (1 << msz
) - 1);
6092 desc
<<= SVE_MTEDESC_SHIFT
;
6094 desc
= simd_desc(vsz
, vsz
, desc
| scale
);
6095 t_desc
= tcg_const_i32(desc
);
6097 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
6098 tcg_gen_addi_ptr(t_zm
, cpu_env
, vec_full_reg_offset(s
, zm
));
6099 tcg_gen_addi_ptr(t_zt
, cpu_env
, vec_full_reg_offset(s
, zt
));
6100 fn(cpu_env
, t_zt
, t_pg
, t_zm
, scalar
, t_desc
);
6102 tcg_temp_free_ptr(t_zt
);
6103 tcg_temp_free_ptr(t_zm
);
6104 tcg_temp_free_ptr(t_pg
);
6105 tcg_temp_free_i32(t_desc
);
6108 /* Indexed by [mte][be][ff][xs][u][msz]. */
6109 static gen_helper_gvec_mem_scatter
* const
6110 gather_load_fn32
[2][2][2][2][2][3] = {
6111 { /* MTE Inactive */
6112 { /* Little-endian */
6113 { { { gen_helper_sve_ldbss_zsu
,
6114 gen_helper_sve_ldhss_le_zsu
,
6116 { gen_helper_sve_ldbsu_zsu
,
6117 gen_helper_sve_ldhsu_le_zsu
,
6118 gen_helper_sve_ldss_le_zsu
, } },
6119 { { gen_helper_sve_ldbss_zss
,
6120 gen_helper_sve_ldhss_le_zss
,
6122 { gen_helper_sve_ldbsu_zss
,
6123 gen_helper_sve_ldhsu_le_zss
,
6124 gen_helper_sve_ldss_le_zss
, } } },
6127 { { { gen_helper_sve_ldffbss_zsu
,
6128 gen_helper_sve_ldffhss_le_zsu
,
6130 { gen_helper_sve_ldffbsu_zsu
,
6131 gen_helper_sve_ldffhsu_le_zsu
,
6132 gen_helper_sve_ldffss_le_zsu
, } },
6133 { { gen_helper_sve_ldffbss_zss
,
6134 gen_helper_sve_ldffhss_le_zss
,
6136 { gen_helper_sve_ldffbsu_zss
,
6137 gen_helper_sve_ldffhsu_le_zss
,
6138 gen_helper_sve_ldffss_le_zss
, } } } },
6141 { { { gen_helper_sve_ldbss_zsu
,
6142 gen_helper_sve_ldhss_be_zsu
,
6144 { gen_helper_sve_ldbsu_zsu
,
6145 gen_helper_sve_ldhsu_be_zsu
,
6146 gen_helper_sve_ldss_be_zsu
, } },
6147 { { gen_helper_sve_ldbss_zss
,
6148 gen_helper_sve_ldhss_be_zss
,
6150 { gen_helper_sve_ldbsu_zss
,
6151 gen_helper_sve_ldhsu_be_zss
,
6152 gen_helper_sve_ldss_be_zss
, } } },
6155 { { { gen_helper_sve_ldffbss_zsu
,
6156 gen_helper_sve_ldffhss_be_zsu
,
6158 { gen_helper_sve_ldffbsu_zsu
,
6159 gen_helper_sve_ldffhsu_be_zsu
,
6160 gen_helper_sve_ldffss_be_zsu
, } },
6161 { { gen_helper_sve_ldffbss_zss
,
6162 gen_helper_sve_ldffhss_be_zss
,
6164 { gen_helper_sve_ldffbsu_zss
,
6165 gen_helper_sve_ldffhsu_be_zss
,
6166 gen_helper_sve_ldffss_be_zss
, } } } } },
6168 { /* Little-endian */
6169 { { { gen_helper_sve_ldbss_zsu_mte
,
6170 gen_helper_sve_ldhss_le_zsu_mte
,
6172 { gen_helper_sve_ldbsu_zsu_mte
,
6173 gen_helper_sve_ldhsu_le_zsu_mte
,
6174 gen_helper_sve_ldss_le_zsu_mte
, } },
6175 { { gen_helper_sve_ldbss_zss_mte
,
6176 gen_helper_sve_ldhss_le_zss_mte
,
6178 { gen_helper_sve_ldbsu_zss_mte
,
6179 gen_helper_sve_ldhsu_le_zss_mte
,
6180 gen_helper_sve_ldss_le_zss_mte
, } } },
6183 { { { gen_helper_sve_ldffbss_zsu_mte
,
6184 gen_helper_sve_ldffhss_le_zsu_mte
,
6186 { gen_helper_sve_ldffbsu_zsu_mte
,
6187 gen_helper_sve_ldffhsu_le_zsu_mte
,
6188 gen_helper_sve_ldffss_le_zsu_mte
, } },
6189 { { gen_helper_sve_ldffbss_zss_mte
,
6190 gen_helper_sve_ldffhss_le_zss_mte
,
6192 { gen_helper_sve_ldffbsu_zss_mte
,
6193 gen_helper_sve_ldffhsu_le_zss_mte
,
6194 gen_helper_sve_ldffss_le_zss_mte
, } } } },
6197 { { { gen_helper_sve_ldbss_zsu_mte
,
6198 gen_helper_sve_ldhss_be_zsu_mte
,
6200 { gen_helper_sve_ldbsu_zsu_mte
,
6201 gen_helper_sve_ldhsu_be_zsu_mte
,
6202 gen_helper_sve_ldss_be_zsu_mte
, } },
6203 { { gen_helper_sve_ldbss_zss_mte
,
6204 gen_helper_sve_ldhss_be_zss_mte
,
6206 { gen_helper_sve_ldbsu_zss_mte
,
6207 gen_helper_sve_ldhsu_be_zss_mte
,
6208 gen_helper_sve_ldss_be_zss_mte
, } } },
6211 { { { gen_helper_sve_ldffbss_zsu_mte
,
6212 gen_helper_sve_ldffhss_be_zsu_mte
,
6214 { gen_helper_sve_ldffbsu_zsu_mte
,
6215 gen_helper_sve_ldffhsu_be_zsu_mte
,
6216 gen_helper_sve_ldffss_be_zsu_mte
, } },
6217 { { gen_helper_sve_ldffbss_zss_mte
,
6218 gen_helper_sve_ldffhss_be_zss_mte
,
6220 { gen_helper_sve_ldffbsu_zss_mte
,
6221 gen_helper_sve_ldffhsu_be_zss_mte
,
6222 gen_helper_sve_ldffss_be_zss_mte
, } } } } },
6225 /* Note that we overload xs=2 to indicate 64-bit offset. */
6226 static gen_helper_gvec_mem_scatter
* const
6227 gather_load_fn64
[2][2][2][3][2][4] = {
6228 { /* MTE Inactive */
6229 { /* Little-endian */
6230 { { { gen_helper_sve_ldbds_zsu
,
6231 gen_helper_sve_ldhds_le_zsu
,
6232 gen_helper_sve_ldsds_le_zsu
,
6234 { gen_helper_sve_ldbdu_zsu
,
6235 gen_helper_sve_ldhdu_le_zsu
,
6236 gen_helper_sve_ldsdu_le_zsu
,
6237 gen_helper_sve_lddd_le_zsu
, } },
6238 { { gen_helper_sve_ldbds_zss
,
6239 gen_helper_sve_ldhds_le_zss
,
6240 gen_helper_sve_ldsds_le_zss
,
6242 { gen_helper_sve_ldbdu_zss
,
6243 gen_helper_sve_ldhdu_le_zss
,
6244 gen_helper_sve_ldsdu_le_zss
,
6245 gen_helper_sve_lddd_le_zss
, } },
6246 { { gen_helper_sve_ldbds_zd
,
6247 gen_helper_sve_ldhds_le_zd
,
6248 gen_helper_sve_ldsds_le_zd
,
6250 { gen_helper_sve_ldbdu_zd
,
6251 gen_helper_sve_ldhdu_le_zd
,
6252 gen_helper_sve_ldsdu_le_zd
,
6253 gen_helper_sve_lddd_le_zd
, } } },
6256 { { { gen_helper_sve_ldffbds_zsu
,
6257 gen_helper_sve_ldffhds_le_zsu
,
6258 gen_helper_sve_ldffsds_le_zsu
,
6260 { gen_helper_sve_ldffbdu_zsu
,
6261 gen_helper_sve_ldffhdu_le_zsu
,
6262 gen_helper_sve_ldffsdu_le_zsu
,
6263 gen_helper_sve_ldffdd_le_zsu
, } },
6264 { { gen_helper_sve_ldffbds_zss
,
6265 gen_helper_sve_ldffhds_le_zss
,
6266 gen_helper_sve_ldffsds_le_zss
,
6268 { gen_helper_sve_ldffbdu_zss
,
6269 gen_helper_sve_ldffhdu_le_zss
,
6270 gen_helper_sve_ldffsdu_le_zss
,
6271 gen_helper_sve_ldffdd_le_zss
, } },
6272 { { gen_helper_sve_ldffbds_zd
,
6273 gen_helper_sve_ldffhds_le_zd
,
6274 gen_helper_sve_ldffsds_le_zd
,
6276 { gen_helper_sve_ldffbdu_zd
,
6277 gen_helper_sve_ldffhdu_le_zd
,
6278 gen_helper_sve_ldffsdu_le_zd
,
6279 gen_helper_sve_ldffdd_le_zd
, } } } },
6281 { { { gen_helper_sve_ldbds_zsu
,
6282 gen_helper_sve_ldhds_be_zsu
,
6283 gen_helper_sve_ldsds_be_zsu
,
6285 { gen_helper_sve_ldbdu_zsu
,
6286 gen_helper_sve_ldhdu_be_zsu
,
6287 gen_helper_sve_ldsdu_be_zsu
,
6288 gen_helper_sve_lddd_be_zsu
, } },
6289 { { gen_helper_sve_ldbds_zss
,
6290 gen_helper_sve_ldhds_be_zss
,
6291 gen_helper_sve_ldsds_be_zss
,
6293 { gen_helper_sve_ldbdu_zss
,
6294 gen_helper_sve_ldhdu_be_zss
,
6295 gen_helper_sve_ldsdu_be_zss
,
6296 gen_helper_sve_lddd_be_zss
, } },
6297 { { gen_helper_sve_ldbds_zd
,
6298 gen_helper_sve_ldhds_be_zd
,
6299 gen_helper_sve_ldsds_be_zd
,
6301 { gen_helper_sve_ldbdu_zd
,
6302 gen_helper_sve_ldhdu_be_zd
,
6303 gen_helper_sve_ldsdu_be_zd
,
6304 gen_helper_sve_lddd_be_zd
, } } },
6307 { { { gen_helper_sve_ldffbds_zsu
,
6308 gen_helper_sve_ldffhds_be_zsu
,
6309 gen_helper_sve_ldffsds_be_zsu
,
6311 { gen_helper_sve_ldffbdu_zsu
,
6312 gen_helper_sve_ldffhdu_be_zsu
,
6313 gen_helper_sve_ldffsdu_be_zsu
,
6314 gen_helper_sve_ldffdd_be_zsu
, } },
6315 { { gen_helper_sve_ldffbds_zss
,
6316 gen_helper_sve_ldffhds_be_zss
,
6317 gen_helper_sve_ldffsds_be_zss
,
6319 { gen_helper_sve_ldffbdu_zss
,
6320 gen_helper_sve_ldffhdu_be_zss
,
6321 gen_helper_sve_ldffsdu_be_zss
,
6322 gen_helper_sve_ldffdd_be_zss
, } },
6323 { { gen_helper_sve_ldffbds_zd
,
6324 gen_helper_sve_ldffhds_be_zd
,
6325 gen_helper_sve_ldffsds_be_zd
,
6327 { gen_helper_sve_ldffbdu_zd
,
6328 gen_helper_sve_ldffhdu_be_zd
,
6329 gen_helper_sve_ldffsdu_be_zd
,
6330 gen_helper_sve_ldffdd_be_zd
, } } } } },
6332 { /* Little-endian */
6333 { { { gen_helper_sve_ldbds_zsu_mte
,
6334 gen_helper_sve_ldhds_le_zsu_mte
,
6335 gen_helper_sve_ldsds_le_zsu_mte
,
6337 { gen_helper_sve_ldbdu_zsu_mte
,
6338 gen_helper_sve_ldhdu_le_zsu_mte
,
6339 gen_helper_sve_ldsdu_le_zsu_mte
,
6340 gen_helper_sve_lddd_le_zsu_mte
, } },
6341 { { gen_helper_sve_ldbds_zss_mte
,
6342 gen_helper_sve_ldhds_le_zss_mte
,
6343 gen_helper_sve_ldsds_le_zss_mte
,
6345 { gen_helper_sve_ldbdu_zss_mte
,
6346 gen_helper_sve_ldhdu_le_zss_mte
,
6347 gen_helper_sve_ldsdu_le_zss_mte
,
6348 gen_helper_sve_lddd_le_zss_mte
, } },
6349 { { gen_helper_sve_ldbds_zd_mte
,
6350 gen_helper_sve_ldhds_le_zd_mte
,
6351 gen_helper_sve_ldsds_le_zd_mte
,
6353 { gen_helper_sve_ldbdu_zd_mte
,
6354 gen_helper_sve_ldhdu_le_zd_mte
,
6355 gen_helper_sve_ldsdu_le_zd_mte
,
6356 gen_helper_sve_lddd_le_zd_mte
, } } },
6359 { { { gen_helper_sve_ldffbds_zsu_mte
,
6360 gen_helper_sve_ldffhds_le_zsu_mte
,
6361 gen_helper_sve_ldffsds_le_zsu_mte
,
6363 { gen_helper_sve_ldffbdu_zsu_mte
,
6364 gen_helper_sve_ldffhdu_le_zsu_mte
,
6365 gen_helper_sve_ldffsdu_le_zsu_mte
,
6366 gen_helper_sve_ldffdd_le_zsu_mte
, } },
6367 { { gen_helper_sve_ldffbds_zss_mte
,
6368 gen_helper_sve_ldffhds_le_zss_mte
,
6369 gen_helper_sve_ldffsds_le_zss_mte
,
6371 { gen_helper_sve_ldffbdu_zss_mte
,
6372 gen_helper_sve_ldffhdu_le_zss_mte
,
6373 gen_helper_sve_ldffsdu_le_zss_mte
,
6374 gen_helper_sve_ldffdd_le_zss_mte
, } },
6375 { { gen_helper_sve_ldffbds_zd_mte
,
6376 gen_helper_sve_ldffhds_le_zd_mte
,
6377 gen_helper_sve_ldffsds_le_zd_mte
,
6379 { gen_helper_sve_ldffbdu_zd_mte
,
6380 gen_helper_sve_ldffhdu_le_zd_mte
,
6381 gen_helper_sve_ldffsdu_le_zd_mte
,
6382 gen_helper_sve_ldffdd_le_zd_mte
, } } } },
6384 { { { gen_helper_sve_ldbds_zsu_mte
,
6385 gen_helper_sve_ldhds_be_zsu_mte
,
6386 gen_helper_sve_ldsds_be_zsu_mte
,
6388 { gen_helper_sve_ldbdu_zsu_mte
,
6389 gen_helper_sve_ldhdu_be_zsu_mte
,
6390 gen_helper_sve_ldsdu_be_zsu_mte
,
6391 gen_helper_sve_lddd_be_zsu_mte
, } },
6392 { { gen_helper_sve_ldbds_zss_mte
,
6393 gen_helper_sve_ldhds_be_zss_mte
,
6394 gen_helper_sve_ldsds_be_zss_mte
,
6396 { gen_helper_sve_ldbdu_zss_mte
,
6397 gen_helper_sve_ldhdu_be_zss_mte
,
6398 gen_helper_sve_ldsdu_be_zss_mte
,
6399 gen_helper_sve_lddd_be_zss_mte
, } },
6400 { { gen_helper_sve_ldbds_zd_mte
,
6401 gen_helper_sve_ldhds_be_zd_mte
,
6402 gen_helper_sve_ldsds_be_zd_mte
,
6404 { gen_helper_sve_ldbdu_zd_mte
,
6405 gen_helper_sve_ldhdu_be_zd_mte
,
6406 gen_helper_sve_ldsdu_be_zd_mte
,
6407 gen_helper_sve_lddd_be_zd_mte
, } } },
6410 { { { gen_helper_sve_ldffbds_zsu_mte
,
6411 gen_helper_sve_ldffhds_be_zsu_mte
,
6412 gen_helper_sve_ldffsds_be_zsu_mte
,
6414 { gen_helper_sve_ldffbdu_zsu_mte
,
6415 gen_helper_sve_ldffhdu_be_zsu_mte
,
6416 gen_helper_sve_ldffsdu_be_zsu_mte
,
6417 gen_helper_sve_ldffdd_be_zsu_mte
, } },
6418 { { gen_helper_sve_ldffbds_zss_mte
,
6419 gen_helper_sve_ldffhds_be_zss_mte
,
6420 gen_helper_sve_ldffsds_be_zss_mte
,
6422 { gen_helper_sve_ldffbdu_zss_mte
,
6423 gen_helper_sve_ldffhdu_be_zss_mte
,
6424 gen_helper_sve_ldffsdu_be_zss_mte
,
6425 gen_helper_sve_ldffdd_be_zss_mte
, } },
6426 { { gen_helper_sve_ldffbds_zd_mte
,
6427 gen_helper_sve_ldffhds_be_zd_mte
,
6428 gen_helper_sve_ldffsds_be_zd_mte
,
6430 { gen_helper_sve_ldffbdu_zd_mte
,
6431 gen_helper_sve_ldffhdu_be_zd_mte
,
6432 gen_helper_sve_ldffsdu_be_zd_mte
,
6433 gen_helper_sve_ldffdd_be_zd_mte
, } } } } },
6436 static bool trans_LD1_zprz(DisasContext
*s
, arg_LD1_zprz
*a
)
6438 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6439 bool be
= s
->be_data
== MO_BE
;
6440 bool mte
= s
->mte_active
[0];
6442 if (!sve_access_check(s
)) {
6448 fn
= gather_load_fn32
[mte
][be
][a
->ff
][a
->xs
][a
->u
][a
->msz
];
6451 fn
= gather_load_fn64
[mte
][be
][a
->ff
][a
->xs
][a
->u
][a
->msz
];
6456 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rm
, a
->scale
* a
->msz
,
6457 cpu_reg_sp(s
, a
->rn
), a
->msz
, false, fn
);
6461 static bool trans_LD1_zpiz(DisasContext
*s
, arg_LD1_zpiz
*a
)
6463 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6464 bool be
= s
->be_data
== MO_BE
;
6465 bool mte
= s
->mte_active
[0];
6468 if (a
->esz
< a
->msz
|| (a
->esz
== a
->msz
&& !a
->u
)) {
6471 if (!sve_access_check(s
)) {
6477 fn
= gather_load_fn32
[mte
][be
][a
->ff
][0][a
->u
][a
->msz
];
6480 fn
= gather_load_fn64
[mte
][be
][a
->ff
][2][a
->u
][a
->msz
];
6485 /* Treat LD1_zpiz (zn[x] + imm) the same way as LD1_zprz (rn + zm[x])
6486 * by loading the immediate into the scalar parameter.
6488 imm
= tcg_const_i64(a
->imm
<< a
->msz
);
6489 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0, imm
, a
->msz
, false, fn
);
6490 tcg_temp_free_i64(imm
);
6494 static bool trans_LDNT1_zprz(DisasContext
*s
, arg_LD1_zprz
*a
)
6496 if (!dc_isar_feature(aa64_sve2
, s
)) {
6499 return trans_LD1_zprz(s
, a
);
6502 /* Indexed by [mte][be][xs][msz]. */
6503 static gen_helper_gvec_mem_scatter
* const scatter_store_fn32
[2][2][2][3] = {
6504 { /* MTE Inactive */
6505 { /* Little-endian */
6506 { gen_helper_sve_stbs_zsu
,
6507 gen_helper_sve_sths_le_zsu
,
6508 gen_helper_sve_stss_le_zsu
, },
6509 { gen_helper_sve_stbs_zss
,
6510 gen_helper_sve_sths_le_zss
,
6511 gen_helper_sve_stss_le_zss
, } },
6513 { gen_helper_sve_stbs_zsu
,
6514 gen_helper_sve_sths_be_zsu
,
6515 gen_helper_sve_stss_be_zsu
, },
6516 { gen_helper_sve_stbs_zss
,
6517 gen_helper_sve_sths_be_zss
,
6518 gen_helper_sve_stss_be_zss
, } } },
6520 { /* Little-endian */
6521 { gen_helper_sve_stbs_zsu_mte
,
6522 gen_helper_sve_sths_le_zsu_mte
,
6523 gen_helper_sve_stss_le_zsu_mte
, },
6524 { gen_helper_sve_stbs_zss_mte
,
6525 gen_helper_sve_sths_le_zss_mte
,
6526 gen_helper_sve_stss_le_zss_mte
, } },
6528 { gen_helper_sve_stbs_zsu_mte
,
6529 gen_helper_sve_sths_be_zsu_mte
,
6530 gen_helper_sve_stss_be_zsu_mte
, },
6531 { gen_helper_sve_stbs_zss_mte
,
6532 gen_helper_sve_sths_be_zss_mte
,
6533 gen_helper_sve_stss_be_zss_mte
, } } },
6536 /* Note that we overload xs=2 to indicate 64-bit offset. */
6537 static gen_helper_gvec_mem_scatter
* const scatter_store_fn64
[2][2][3][4] = {
6538 { /* MTE Inactive */
6539 { /* Little-endian */
6540 { gen_helper_sve_stbd_zsu
,
6541 gen_helper_sve_sthd_le_zsu
,
6542 gen_helper_sve_stsd_le_zsu
,
6543 gen_helper_sve_stdd_le_zsu
, },
6544 { gen_helper_sve_stbd_zss
,
6545 gen_helper_sve_sthd_le_zss
,
6546 gen_helper_sve_stsd_le_zss
,
6547 gen_helper_sve_stdd_le_zss
, },
6548 { gen_helper_sve_stbd_zd
,
6549 gen_helper_sve_sthd_le_zd
,
6550 gen_helper_sve_stsd_le_zd
,
6551 gen_helper_sve_stdd_le_zd
, } },
6553 { gen_helper_sve_stbd_zsu
,
6554 gen_helper_sve_sthd_be_zsu
,
6555 gen_helper_sve_stsd_be_zsu
,
6556 gen_helper_sve_stdd_be_zsu
, },
6557 { gen_helper_sve_stbd_zss
,
6558 gen_helper_sve_sthd_be_zss
,
6559 gen_helper_sve_stsd_be_zss
,
6560 gen_helper_sve_stdd_be_zss
, },
6561 { gen_helper_sve_stbd_zd
,
6562 gen_helper_sve_sthd_be_zd
,
6563 gen_helper_sve_stsd_be_zd
,
6564 gen_helper_sve_stdd_be_zd
, } } },
6565 { /* MTE Inactive */
6566 { /* Little-endian */
6567 { gen_helper_sve_stbd_zsu_mte
,
6568 gen_helper_sve_sthd_le_zsu_mte
,
6569 gen_helper_sve_stsd_le_zsu_mte
,
6570 gen_helper_sve_stdd_le_zsu_mte
, },
6571 { gen_helper_sve_stbd_zss_mte
,
6572 gen_helper_sve_sthd_le_zss_mte
,
6573 gen_helper_sve_stsd_le_zss_mte
,
6574 gen_helper_sve_stdd_le_zss_mte
, },
6575 { gen_helper_sve_stbd_zd_mte
,
6576 gen_helper_sve_sthd_le_zd_mte
,
6577 gen_helper_sve_stsd_le_zd_mte
,
6578 gen_helper_sve_stdd_le_zd_mte
, } },
6580 { gen_helper_sve_stbd_zsu_mte
,
6581 gen_helper_sve_sthd_be_zsu_mte
,
6582 gen_helper_sve_stsd_be_zsu_mte
,
6583 gen_helper_sve_stdd_be_zsu_mte
, },
6584 { gen_helper_sve_stbd_zss_mte
,
6585 gen_helper_sve_sthd_be_zss_mte
,
6586 gen_helper_sve_stsd_be_zss_mte
,
6587 gen_helper_sve_stdd_be_zss_mte
, },
6588 { gen_helper_sve_stbd_zd_mte
,
6589 gen_helper_sve_sthd_be_zd_mte
,
6590 gen_helper_sve_stsd_be_zd_mte
,
6591 gen_helper_sve_stdd_be_zd_mte
, } } },
6594 static bool trans_ST1_zprz(DisasContext
*s
, arg_ST1_zprz
*a
)
6596 gen_helper_gvec_mem_scatter
*fn
;
6597 bool be
= s
->be_data
== MO_BE
;
6598 bool mte
= s
->mte_active
[0];
6600 if (a
->esz
< a
->msz
|| (a
->msz
== 0 && a
->scale
)) {
6603 if (!sve_access_check(s
)) {
6608 fn
= scatter_store_fn32
[mte
][be
][a
->xs
][a
->msz
];
6611 fn
= scatter_store_fn64
[mte
][be
][a
->xs
][a
->msz
];
6614 g_assert_not_reached();
6616 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rm
, a
->scale
* a
->msz
,
6617 cpu_reg_sp(s
, a
->rn
), a
->msz
, true, fn
);
6621 static bool trans_ST1_zpiz(DisasContext
*s
, arg_ST1_zpiz
*a
)
6623 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6624 bool be
= s
->be_data
== MO_BE
;
6625 bool mte
= s
->mte_active
[0];
6628 if (a
->esz
< a
->msz
) {
6631 if (!sve_access_check(s
)) {
6637 fn
= scatter_store_fn32
[mte
][be
][0][a
->msz
];
6640 fn
= scatter_store_fn64
[mte
][be
][2][a
->msz
];
6645 /* Treat ST1_zpiz (zn[x] + imm) the same way as ST1_zprz (rn + zm[x])
6646 * by loading the immediate into the scalar parameter.
6648 imm
= tcg_const_i64(a
->imm
<< a
->msz
);
6649 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0, imm
, a
->msz
, true, fn
);
6650 tcg_temp_free_i64(imm
);
6654 static bool trans_STNT1_zprz(DisasContext
*s
, arg_ST1_zprz
*a
)
6656 if (!dc_isar_feature(aa64_sve2
, s
)) {
6659 return trans_ST1_zprz(s
, a
);
6666 static bool trans_PRF(DisasContext
*s
, arg_PRF
*a
)
6668 /* Prefetch is a nop within QEMU. */
6669 (void)sve_access_check(s
);
6673 static bool trans_PRF_rr(DisasContext
*s
, arg_PRF_rr
*a
)
6678 /* Prefetch is a nop within QEMU. */
6679 (void)sve_access_check(s
);
6686 * TODO: The implementation so far could handle predicated merging movprfx.
6687 * The helper functions as written take an extra source register to
6688 * use in the operation, but the result is only written when predication
6689 * succeeds. For unpredicated movprfx, we need to rearrange the helpers
6690 * to allow the final write back to the destination to be unconditional.
6691 * For predicated zeroing movprfx, we need to rearrange the helpers to
6692 * allow the final write back to zero inactives.
6694 * In the meantime, just emit the moves.
6697 static bool trans_MOVPRFX(DisasContext
*s
, arg_MOVPRFX
*a
)
6699 return do_mov_z(s
, a
->rd
, a
->rn
);
6702 static bool trans_MOVPRFX_m(DisasContext
*s
, arg_rpr_esz
*a
)
6704 if (sve_access_check(s
)) {
6705 do_sel_z(s
, a
->rd
, a
->rn
, a
->rd
, a
->pg
, a
->esz
);
6710 static bool trans_MOVPRFX_z(DisasContext
*s
, arg_rpr_esz
*a
)
6712 return do_movz_zpz(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
, false);
6716 * SVE2 Integer Multiply - Unpredicated
6719 static bool trans_MUL_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6721 if (!dc_isar_feature(aa64_sve2
, s
)) {
6724 if (sve_access_check(s
)) {
6725 gen_gvec_fn_zzz(s
, tcg_gen_gvec_mul
, a
->esz
, a
->rd
, a
->rn
, a
->rm
);
6730 static bool do_sve2_zzz_ool(DisasContext
*s
, arg_rrr_esz
*a
,
6731 gen_helper_gvec_3
*fn
)
6733 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
6736 if (sve_access_check(s
)) {
6737 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, 0);
6742 static bool trans_SMULH_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6744 static gen_helper_gvec_3
* const fns
[4] = {
6745 gen_helper_gvec_smulh_b
, gen_helper_gvec_smulh_h
,
6746 gen_helper_gvec_smulh_s
, gen_helper_gvec_smulh_d
,
6748 return do_sve2_zzz_ool(s
, a
, fns
[a
->esz
]);
6751 static bool trans_UMULH_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6753 static gen_helper_gvec_3
* const fns
[4] = {
6754 gen_helper_gvec_umulh_b
, gen_helper_gvec_umulh_h
,
6755 gen_helper_gvec_umulh_s
, gen_helper_gvec_umulh_d
,
6757 return do_sve2_zzz_ool(s
, a
, fns
[a
->esz
]);
6760 static bool trans_PMUL_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6762 return do_sve2_zzz_ool(s
, a
, gen_helper_gvec_pmul_b
);
6765 static bool trans_SQDMULH_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6767 static gen_helper_gvec_3
* const fns
[4] = {
6768 gen_helper_sve2_sqdmulh_b
, gen_helper_sve2_sqdmulh_h
,
6769 gen_helper_sve2_sqdmulh_s
, gen_helper_sve2_sqdmulh_d
,
6771 return do_sve2_zzz_ool(s
, a
, fns
[a
->esz
]);
6774 static bool trans_SQRDMULH_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6776 static gen_helper_gvec_3
* const fns
[4] = {
6777 gen_helper_sve2_sqrdmulh_b
, gen_helper_sve2_sqrdmulh_h
,
6778 gen_helper_sve2_sqrdmulh_s
, gen_helper_sve2_sqrdmulh_d
,
6780 return do_sve2_zzz_ool(s
, a
, fns
[a
->esz
]);
6784 * SVE2 Integer - Predicated
6787 static bool do_sve2_zpzz_ool(DisasContext
*s
, arg_rprr_esz
*a
,
6788 gen_helper_gvec_4
*fn
)
6790 if (!dc_isar_feature(aa64_sve2
, s
)) {
6793 return do_zpzz_ool(s
, a
, fn
);
6796 static bool trans_SADALP_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
6798 static gen_helper_gvec_4
* const fns
[3] = {
6799 gen_helper_sve2_sadalp_zpzz_h
,
6800 gen_helper_sve2_sadalp_zpzz_s
,
6801 gen_helper_sve2_sadalp_zpzz_d
,
6806 return do_sve2_zpzz_ool(s
, a
, fns
[a
->esz
- 1]);
6809 static bool trans_UADALP_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
6811 static gen_helper_gvec_4
* const fns
[3] = {
6812 gen_helper_sve2_uadalp_zpzz_h
,
6813 gen_helper_sve2_uadalp_zpzz_s
,
6814 gen_helper_sve2_uadalp_zpzz_d
,
6819 return do_sve2_zpzz_ool(s
, a
, fns
[a
->esz
- 1]);
6823 * SVE2 integer unary operations (predicated)
6826 static bool do_sve2_zpz_ool(DisasContext
*s
, arg_rpr_esz
*a
,
6827 gen_helper_gvec_3
*fn
)
6829 if (!dc_isar_feature(aa64_sve2
, s
)) {
6832 return do_zpz_ool(s
, a
, fn
);
6835 static bool trans_URECPE(DisasContext
*s
, arg_rpr_esz
*a
)
6840 return do_sve2_zpz_ool(s
, a
, gen_helper_sve2_urecpe_s
);
6843 static bool trans_URSQRTE(DisasContext
*s
, arg_rpr_esz
*a
)
6848 return do_sve2_zpz_ool(s
, a
, gen_helper_sve2_ursqrte_s
);
6851 static bool trans_SQABS(DisasContext
*s
, arg_rpr_esz
*a
)
6853 static gen_helper_gvec_3
* const fns
[4] = {
6854 gen_helper_sve2_sqabs_b
, gen_helper_sve2_sqabs_h
,
6855 gen_helper_sve2_sqabs_s
, gen_helper_sve2_sqabs_d
,
6857 return do_sve2_zpz_ool(s
, a
, fns
[a
->esz
]);
6860 static bool trans_SQNEG(DisasContext
*s
, arg_rpr_esz
*a
)
6862 static gen_helper_gvec_3
* const fns
[4] = {
6863 gen_helper_sve2_sqneg_b
, gen_helper_sve2_sqneg_h
,
6864 gen_helper_sve2_sqneg_s
, gen_helper_sve2_sqneg_d
,
6866 return do_sve2_zpz_ool(s
, a
, fns
[a
->esz
]);
6869 #define DO_SVE2_ZPZZ(NAME, name) \
6870 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
6872 static gen_helper_gvec_4 * const fns[4] = { \
6873 gen_helper_sve2_##name##_zpzz_b, gen_helper_sve2_##name##_zpzz_h, \
6874 gen_helper_sve2_##name##_zpzz_s, gen_helper_sve2_##name##_zpzz_d, \
6876 return do_sve2_zpzz_ool(s, a, fns[a->esz]); \
6879 DO_SVE2_ZPZZ(SQSHL
, sqshl
)
6880 DO_SVE2_ZPZZ(SQRSHL
, sqrshl
)
6881 DO_SVE2_ZPZZ(SRSHL
, srshl
)
6883 DO_SVE2_ZPZZ(UQSHL
, uqshl
)
6884 DO_SVE2_ZPZZ(UQRSHL
, uqrshl
)
6885 DO_SVE2_ZPZZ(URSHL
, urshl
)
6887 DO_SVE2_ZPZZ(SHADD
, shadd
)
6888 DO_SVE2_ZPZZ(SRHADD
, srhadd
)
6889 DO_SVE2_ZPZZ(SHSUB
, shsub
)
6891 DO_SVE2_ZPZZ(UHADD
, uhadd
)
6892 DO_SVE2_ZPZZ(URHADD
, urhadd
)
6893 DO_SVE2_ZPZZ(UHSUB
, uhsub
)
6895 DO_SVE2_ZPZZ(ADDP
, addp
)
6896 DO_SVE2_ZPZZ(SMAXP
, smaxp
)
6897 DO_SVE2_ZPZZ(UMAXP
, umaxp
)
6898 DO_SVE2_ZPZZ(SMINP
, sminp
)
6899 DO_SVE2_ZPZZ(UMINP
, uminp
)
6901 DO_SVE2_ZPZZ(SQADD_zpzz
, sqadd
)
6902 DO_SVE2_ZPZZ(UQADD_zpzz
, uqadd
)
6903 DO_SVE2_ZPZZ(SQSUB_zpzz
, sqsub
)
6904 DO_SVE2_ZPZZ(UQSUB_zpzz
, uqsub
)
6905 DO_SVE2_ZPZZ(SUQADD
, suqadd
)
6906 DO_SVE2_ZPZZ(USQADD
, usqadd
)
6909 * SVE2 Widening Integer Arithmetic
6912 static bool do_sve2_zzw_ool(DisasContext
*s
, arg_rrr_esz
*a
,
6913 gen_helper_gvec_3
*fn
, int data
)
6915 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
6918 if (sve_access_check(s
)) {
6919 unsigned vsz
= vec_full_reg_size(s
);
6920 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, a
->rd
),
6921 vec_full_reg_offset(s
, a
->rn
),
6922 vec_full_reg_offset(s
, a
->rm
),
6923 vsz
, vsz
, data
, fn
);
6928 #define DO_SVE2_ZZZ_TB(NAME, name, SEL1, SEL2) \
6929 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
6931 static gen_helper_gvec_3 * const fns[4] = { \
6932 NULL, gen_helper_sve2_##name##_h, \
6933 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
6935 return do_sve2_zzw_ool(s, a, fns[a->esz], (SEL2 << 1) | SEL1); \
6938 DO_SVE2_ZZZ_TB(SADDLB
, saddl
, false, false)
6939 DO_SVE2_ZZZ_TB(SSUBLB
, ssubl
, false, false)
6940 DO_SVE2_ZZZ_TB(SABDLB
, sabdl
, false, false)
6942 DO_SVE2_ZZZ_TB(UADDLB
, uaddl
, false, false)
6943 DO_SVE2_ZZZ_TB(USUBLB
, usubl
, false, false)
6944 DO_SVE2_ZZZ_TB(UABDLB
, uabdl
, false, false)
6946 DO_SVE2_ZZZ_TB(SADDLT
, saddl
, true, true)
6947 DO_SVE2_ZZZ_TB(SSUBLT
, ssubl
, true, true)
6948 DO_SVE2_ZZZ_TB(SABDLT
, sabdl
, true, true)
6950 DO_SVE2_ZZZ_TB(UADDLT
, uaddl
, true, true)
6951 DO_SVE2_ZZZ_TB(USUBLT
, usubl
, true, true)
6952 DO_SVE2_ZZZ_TB(UABDLT
, uabdl
, true, true)
6954 DO_SVE2_ZZZ_TB(SADDLBT
, saddl
, false, true)
6955 DO_SVE2_ZZZ_TB(SSUBLBT
, ssubl
, false, true)
6956 DO_SVE2_ZZZ_TB(SSUBLTB
, ssubl
, true, false)
6958 DO_SVE2_ZZZ_TB(SQDMULLB_zzz
, sqdmull_zzz
, false, false)
6959 DO_SVE2_ZZZ_TB(SQDMULLT_zzz
, sqdmull_zzz
, true, true)
6961 DO_SVE2_ZZZ_TB(SMULLB_zzz
, smull_zzz
, false, false)
6962 DO_SVE2_ZZZ_TB(SMULLT_zzz
, smull_zzz
, true, true)
6964 DO_SVE2_ZZZ_TB(UMULLB_zzz
, umull_zzz
, false, false)
6965 DO_SVE2_ZZZ_TB(UMULLT_zzz
, umull_zzz
, true, true)
6967 static bool do_eor_tb(DisasContext
*s
, arg_rrr_esz
*a
, bool sel1
)
6969 static gen_helper_gvec_3
* const fns
[4] = {
6970 gen_helper_sve2_eoril_b
, gen_helper_sve2_eoril_h
,
6971 gen_helper_sve2_eoril_s
, gen_helper_sve2_eoril_d
,
6973 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], (!sel1
<< 1) | sel1
);
6976 static bool trans_EORBT(DisasContext
*s
, arg_rrr_esz
*a
)
6978 return do_eor_tb(s
, a
, false);
6981 static bool trans_EORTB(DisasContext
*s
, arg_rrr_esz
*a
)
6983 return do_eor_tb(s
, a
, true);
6986 static bool do_trans_pmull(DisasContext
*s
, arg_rrr_esz
*a
, bool sel
)
6988 static gen_helper_gvec_3
* const fns
[4] = {
6989 gen_helper_gvec_pmull_q
, gen_helper_sve2_pmull_h
,
6990 NULL
, gen_helper_sve2_pmull_d
,
6992 if (a
->esz
== 0 && !dc_isar_feature(aa64_sve2_pmull128
, s
)) {
6995 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], sel
);
6998 static bool trans_PMULLB(DisasContext
*s
, arg_rrr_esz
*a
)
7000 return do_trans_pmull(s
, a
, false);
7003 static bool trans_PMULLT(DisasContext
*s
, arg_rrr_esz
*a
)
7005 return do_trans_pmull(s
, a
, true);
7008 #define DO_SVE2_ZZZ_WTB(NAME, name, SEL2) \
7009 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
7011 static gen_helper_gvec_3 * const fns[4] = { \
7012 NULL, gen_helper_sve2_##name##_h, \
7013 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
7015 return do_sve2_zzw_ool(s, a, fns[a->esz], SEL2); \
7018 DO_SVE2_ZZZ_WTB(SADDWB
, saddw
, false)
7019 DO_SVE2_ZZZ_WTB(SADDWT
, saddw
, true)
7020 DO_SVE2_ZZZ_WTB(SSUBWB
, ssubw
, false)
7021 DO_SVE2_ZZZ_WTB(SSUBWT
, ssubw
, true)
7023 DO_SVE2_ZZZ_WTB(UADDWB
, uaddw
, false)
7024 DO_SVE2_ZZZ_WTB(UADDWT
, uaddw
, true)
7025 DO_SVE2_ZZZ_WTB(USUBWB
, usubw
, false)
7026 DO_SVE2_ZZZ_WTB(USUBWT
, usubw
, true)
7028 static void gen_sshll_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t imm
)
7032 int halfbits
= 4 << vece
;
7035 if (shl
== halfbits
) {
7036 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7037 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(halfbits
, halfbits
));
7038 tcg_gen_and_vec(vece
, d
, n
, t
);
7039 tcg_temp_free_vec(t
);
7041 tcg_gen_sari_vec(vece
, d
, n
, halfbits
);
7042 tcg_gen_shli_vec(vece
, d
, d
, shl
);
7045 tcg_gen_shli_vec(vece
, d
, n
, halfbits
);
7046 tcg_gen_sari_vec(vece
, d
, d
, halfbits
- shl
);
7050 static void gen_ushll_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int imm
)
7052 int halfbits
= 4 << vece
;
7054 int shl
= (imm
>> 1);
7058 mask
= MAKE_64BIT_MASK(0, halfbits
);
7060 mask
= dup_const(vece
, mask
);
7062 shift
= shl
- top
* halfbits
;
7064 tcg_gen_shri_i64(d
, n
, -shift
);
7066 tcg_gen_shli_i64(d
, n
, shift
);
7068 tcg_gen_andi_i64(d
, d
, mask
);
7071 static void gen_ushll16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
7073 gen_ushll_i64(MO_16
, d
, n
, imm
);
7076 static void gen_ushll32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
7078 gen_ushll_i64(MO_32
, d
, n
, imm
);
7081 static void gen_ushll64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
7083 gen_ushll_i64(MO_64
, d
, n
, imm
);
7086 static void gen_ushll_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t imm
)
7088 int halfbits
= 4 << vece
;
7093 if (shl
== halfbits
) {
7094 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7095 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(halfbits
, halfbits
));
7096 tcg_gen_and_vec(vece
, d
, n
, t
);
7097 tcg_temp_free_vec(t
);
7099 tcg_gen_shri_vec(vece
, d
, n
, halfbits
);
7100 tcg_gen_shli_vec(vece
, d
, d
, shl
);
7104 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7105 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7106 tcg_gen_and_vec(vece
, d
, n
, t
);
7107 tcg_temp_free_vec(t
);
7109 tcg_gen_shli_vec(vece
, d
, n
, halfbits
);
7110 tcg_gen_shri_vec(vece
, d
, d
, halfbits
- shl
);
7115 static bool do_sve2_shll_tb(DisasContext
*s
, arg_rri_esz
*a
,
7118 static const TCGOpcode sshll_list
[] = {
7119 INDEX_op_shli_vec
, INDEX_op_sari_vec
, 0
7121 static const TCGOpcode ushll_list
[] = {
7122 INDEX_op_shli_vec
, INDEX_op_shri_vec
, 0
7124 static const GVecGen2i ops
[2][3] = {
7125 { { .fniv
= gen_sshll_vec
,
7126 .opt_opc
= sshll_list
,
7127 .fno
= gen_helper_sve2_sshll_h
,
7129 { .fniv
= gen_sshll_vec
,
7130 .opt_opc
= sshll_list
,
7131 .fno
= gen_helper_sve2_sshll_s
,
7133 { .fniv
= gen_sshll_vec
,
7134 .opt_opc
= sshll_list
,
7135 .fno
= gen_helper_sve2_sshll_d
,
7137 { { .fni8
= gen_ushll16_i64
,
7138 .fniv
= gen_ushll_vec
,
7139 .opt_opc
= ushll_list
,
7140 .fno
= gen_helper_sve2_ushll_h
,
7142 { .fni8
= gen_ushll32_i64
,
7143 .fniv
= gen_ushll_vec
,
7144 .opt_opc
= ushll_list
,
7145 .fno
= gen_helper_sve2_ushll_s
,
7147 { .fni8
= gen_ushll64_i64
,
7148 .fniv
= gen_ushll_vec
,
7149 .opt_opc
= ushll_list
,
7150 .fno
= gen_helper_sve2_ushll_d
,
7154 if (a
->esz
< 0 || a
->esz
> 2 || !dc_isar_feature(aa64_sve2
, s
)) {
7157 if (sve_access_check(s
)) {
7158 unsigned vsz
= vec_full_reg_size(s
);
7159 tcg_gen_gvec_2i(vec_full_reg_offset(s
, a
->rd
),
7160 vec_full_reg_offset(s
, a
->rn
),
7161 vsz
, vsz
, (a
->imm
<< 1) | sel
,
7167 static bool trans_SSHLLB(DisasContext
*s
, arg_rri_esz
*a
)
7169 return do_sve2_shll_tb(s
, a
, false, false);
7172 static bool trans_SSHLLT(DisasContext
*s
, arg_rri_esz
*a
)
7174 return do_sve2_shll_tb(s
, a
, true, false);
7177 static bool trans_USHLLB(DisasContext
*s
, arg_rri_esz
*a
)
7179 return do_sve2_shll_tb(s
, a
, false, true);
7182 static bool trans_USHLLT(DisasContext
*s
, arg_rri_esz
*a
)
7184 return do_sve2_shll_tb(s
, a
, true, true);
7187 static bool trans_BEXT(DisasContext
*s
, arg_rrr_esz
*a
)
7189 static gen_helper_gvec_3
* const fns
[4] = {
7190 gen_helper_sve2_bext_b
, gen_helper_sve2_bext_h
,
7191 gen_helper_sve2_bext_s
, gen_helper_sve2_bext_d
,
7193 if (!dc_isar_feature(aa64_sve2_bitperm
, s
)) {
7196 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], 0);
7199 static bool trans_BDEP(DisasContext
*s
, arg_rrr_esz
*a
)
7201 static gen_helper_gvec_3
* const fns
[4] = {
7202 gen_helper_sve2_bdep_b
, gen_helper_sve2_bdep_h
,
7203 gen_helper_sve2_bdep_s
, gen_helper_sve2_bdep_d
,
7205 if (!dc_isar_feature(aa64_sve2_bitperm
, s
)) {
7208 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], 0);
7211 static bool trans_BGRP(DisasContext
*s
, arg_rrr_esz
*a
)
7213 static gen_helper_gvec_3
* const fns
[4] = {
7214 gen_helper_sve2_bgrp_b
, gen_helper_sve2_bgrp_h
,
7215 gen_helper_sve2_bgrp_s
, gen_helper_sve2_bgrp_d
,
7217 if (!dc_isar_feature(aa64_sve2_bitperm
, s
)) {
7220 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], 0);
7223 static bool do_cadd(DisasContext
*s
, arg_rrr_esz
*a
, bool sq
, bool rot
)
7225 static gen_helper_gvec_3
* const fns
[2][4] = {
7226 { gen_helper_sve2_cadd_b
, gen_helper_sve2_cadd_h
,
7227 gen_helper_sve2_cadd_s
, gen_helper_sve2_cadd_d
},
7228 { gen_helper_sve2_sqcadd_b
, gen_helper_sve2_sqcadd_h
,
7229 gen_helper_sve2_sqcadd_s
, gen_helper_sve2_sqcadd_d
},
7231 return do_sve2_zzw_ool(s
, a
, fns
[sq
][a
->esz
], rot
);
7234 static bool trans_CADD_rot90(DisasContext
*s
, arg_rrr_esz
*a
)
7236 return do_cadd(s
, a
, false, false);
7239 static bool trans_CADD_rot270(DisasContext
*s
, arg_rrr_esz
*a
)
7241 return do_cadd(s
, a
, false, true);
7244 static bool trans_SQCADD_rot90(DisasContext
*s
, arg_rrr_esz
*a
)
7246 return do_cadd(s
, a
, true, false);
7249 static bool trans_SQCADD_rot270(DisasContext
*s
, arg_rrr_esz
*a
)
7251 return do_cadd(s
, a
, true, true);
7254 static bool do_sve2_zzzz_ool(DisasContext
*s
, arg_rrrr_esz
*a
,
7255 gen_helper_gvec_4
*fn
, int data
)
7257 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
7260 if (sve_access_check(s
)) {
7261 gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, data
);
7266 static bool do_abal(DisasContext
*s
, arg_rrrr_esz
*a
, bool uns
, bool sel
)
7268 static gen_helper_gvec_4
* const fns
[2][4] = {
7269 { NULL
, gen_helper_sve2_sabal_h
,
7270 gen_helper_sve2_sabal_s
, gen_helper_sve2_sabal_d
},
7271 { NULL
, gen_helper_sve2_uabal_h
,
7272 gen_helper_sve2_uabal_s
, gen_helper_sve2_uabal_d
},
7274 return do_sve2_zzzz_ool(s
, a
, fns
[uns
][a
->esz
], sel
);
7277 static bool trans_SABALB(DisasContext
*s
, arg_rrrr_esz
*a
)
7279 return do_abal(s
, a
, false, false);
7282 static bool trans_SABALT(DisasContext
*s
, arg_rrrr_esz
*a
)
7284 return do_abal(s
, a
, false, true);
7287 static bool trans_UABALB(DisasContext
*s
, arg_rrrr_esz
*a
)
7289 return do_abal(s
, a
, true, false);
7292 static bool trans_UABALT(DisasContext
*s
, arg_rrrr_esz
*a
)
7294 return do_abal(s
, a
, true, true);
7297 static bool do_adcl(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
7299 static gen_helper_gvec_4
* const fns
[2] = {
7300 gen_helper_sve2_adcl_s
,
7301 gen_helper_sve2_adcl_d
,
7304 * Note that in this case the ESZ field encodes both size and sign.
7305 * Split out 'subtract' into bit 1 of the data field for the helper.
7307 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
& 1], (a
->esz
& 2) | sel
);
7310 static bool trans_ADCLB(DisasContext
*s
, arg_rrrr_esz
*a
)
7312 return do_adcl(s
, a
, false);
7315 static bool trans_ADCLT(DisasContext
*s
, arg_rrrr_esz
*a
)
7317 return do_adcl(s
, a
, true);
7320 static bool do_sve2_fn2i(DisasContext
*s
, arg_rri_esz
*a
, GVecGen2iFn
*fn
)
7322 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
7325 if (sve_access_check(s
)) {
7326 unsigned vsz
= vec_full_reg_size(s
);
7327 unsigned rd_ofs
= vec_full_reg_offset(s
, a
->rd
);
7328 unsigned rn_ofs
= vec_full_reg_offset(s
, a
->rn
);
7329 fn(a
->esz
, rd_ofs
, rn_ofs
, a
->imm
, vsz
, vsz
);
7334 static bool trans_SSRA(DisasContext
*s
, arg_rri_esz
*a
)
7336 return do_sve2_fn2i(s
, a
, gen_gvec_ssra
);
7339 static bool trans_USRA(DisasContext
*s
, arg_rri_esz
*a
)
7341 return do_sve2_fn2i(s
, a
, gen_gvec_usra
);
7344 static bool trans_SRSRA(DisasContext
*s
, arg_rri_esz
*a
)
7346 return do_sve2_fn2i(s
, a
, gen_gvec_srsra
);
7349 static bool trans_URSRA(DisasContext
*s
, arg_rri_esz
*a
)
7351 return do_sve2_fn2i(s
, a
, gen_gvec_ursra
);
7354 static bool trans_SRI(DisasContext
*s
, arg_rri_esz
*a
)
7356 return do_sve2_fn2i(s
, a
, gen_gvec_sri
);
7359 static bool trans_SLI(DisasContext
*s
, arg_rri_esz
*a
)
7361 return do_sve2_fn2i(s
, a
, gen_gvec_sli
);
7364 static bool do_sve2_fn_zzz(DisasContext
*s
, arg_rrr_esz
*a
, GVecGen3Fn
*fn
)
7366 if (!dc_isar_feature(aa64_sve2
, s
)) {
7369 if (sve_access_check(s
)) {
7370 gen_gvec_fn_zzz(s
, fn
, a
->esz
, a
->rd
, a
->rn
, a
->rm
);
7375 static bool trans_SABA(DisasContext
*s
, arg_rrr_esz
*a
)
7377 return do_sve2_fn_zzz(s
, a
, gen_gvec_saba
);
7380 static bool trans_UABA(DisasContext
*s
, arg_rrr_esz
*a
)
7382 return do_sve2_fn_zzz(s
, a
, gen_gvec_uaba
);
7385 static bool do_sve2_narrow_extract(DisasContext
*s
, arg_rri_esz
*a
,
7386 const GVecGen2 ops
[3])
7388 if (a
->esz
< 0 || a
->esz
> MO_32
|| a
->imm
!= 0 ||
7389 !dc_isar_feature(aa64_sve2
, s
)) {
7392 if (sve_access_check(s
)) {
7393 unsigned vsz
= vec_full_reg_size(s
);
7394 tcg_gen_gvec_2(vec_full_reg_offset(s
, a
->rd
),
7395 vec_full_reg_offset(s
, a
->rn
),
7396 vsz
, vsz
, &ops
[a
->esz
]);
7401 static const TCGOpcode sqxtn_list
[] = {
7402 INDEX_op_shli_vec
, INDEX_op_smin_vec
, INDEX_op_smax_vec
, 0
7405 static void gen_sqxtnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7407 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7408 int halfbits
= 4 << vece
;
7409 int64_t mask
= (1ull << halfbits
) - 1;
7410 int64_t min
= -1ull << (halfbits
- 1);
7411 int64_t max
= -min
- 1;
7413 tcg_gen_dupi_vec(vece
, t
, min
);
7414 tcg_gen_smax_vec(vece
, d
, n
, t
);
7415 tcg_gen_dupi_vec(vece
, t
, max
);
7416 tcg_gen_smin_vec(vece
, d
, d
, t
);
7417 tcg_gen_dupi_vec(vece
, t
, mask
);
7418 tcg_gen_and_vec(vece
, d
, d
, t
);
7419 tcg_temp_free_vec(t
);
7422 static bool trans_SQXTNB(DisasContext
*s
, arg_rri_esz
*a
)
7424 static const GVecGen2 ops
[3] = {
7425 { .fniv
= gen_sqxtnb_vec
,
7426 .opt_opc
= sqxtn_list
,
7427 .fno
= gen_helper_sve2_sqxtnb_h
,
7429 { .fniv
= gen_sqxtnb_vec
,
7430 .opt_opc
= sqxtn_list
,
7431 .fno
= gen_helper_sve2_sqxtnb_s
,
7433 { .fniv
= gen_sqxtnb_vec
,
7434 .opt_opc
= sqxtn_list
,
7435 .fno
= gen_helper_sve2_sqxtnb_d
,
7438 return do_sve2_narrow_extract(s
, a
, ops
);
7441 static void gen_sqxtnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7443 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7444 int halfbits
= 4 << vece
;
7445 int64_t mask
= (1ull << halfbits
) - 1;
7446 int64_t min
= -1ull << (halfbits
- 1);
7447 int64_t max
= -min
- 1;
7449 tcg_gen_dupi_vec(vece
, t
, min
);
7450 tcg_gen_smax_vec(vece
, n
, n
, t
);
7451 tcg_gen_dupi_vec(vece
, t
, max
);
7452 tcg_gen_smin_vec(vece
, n
, n
, t
);
7453 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7454 tcg_gen_dupi_vec(vece
, t
, mask
);
7455 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7456 tcg_temp_free_vec(t
);
7459 static bool trans_SQXTNT(DisasContext
*s
, arg_rri_esz
*a
)
7461 static const GVecGen2 ops
[3] = {
7462 { .fniv
= gen_sqxtnt_vec
,
7463 .opt_opc
= sqxtn_list
,
7465 .fno
= gen_helper_sve2_sqxtnt_h
,
7467 { .fniv
= gen_sqxtnt_vec
,
7468 .opt_opc
= sqxtn_list
,
7470 .fno
= gen_helper_sve2_sqxtnt_s
,
7472 { .fniv
= gen_sqxtnt_vec
,
7473 .opt_opc
= sqxtn_list
,
7475 .fno
= gen_helper_sve2_sqxtnt_d
,
7478 return do_sve2_narrow_extract(s
, a
, ops
);
7481 static const TCGOpcode uqxtn_list
[] = {
7482 INDEX_op_shli_vec
, INDEX_op_umin_vec
, 0
7485 static void gen_uqxtnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7487 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7488 int halfbits
= 4 << vece
;
7489 int64_t max
= (1ull << halfbits
) - 1;
7491 tcg_gen_dupi_vec(vece
, t
, max
);
7492 tcg_gen_umin_vec(vece
, d
, n
, t
);
7493 tcg_temp_free_vec(t
);
7496 static bool trans_UQXTNB(DisasContext
*s
, arg_rri_esz
*a
)
7498 static const GVecGen2 ops
[3] = {
7499 { .fniv
= gen_uqxtnb_vec
,
7500 .opt_opc
= uqxtn_list
,
7501 .fno
= gen_helper_sve2_uqxtnb_h
,
7503 { .fniv
= gen_uqxtnb_vec
,
7504 .opt_opc
= uqxtn_list
,
7505 .fno
= gen_helper_sve2_uqxtnb_s
,
7507 { .fniv
= gen_uqxtnb_vec
,
7508 .opt_opc
= uqxtn_list
,
7509 .fno
= gen_helper_sve2_uqxtnb_d
,
7512 return do_sve2_narrow_extract(s
, a
, ops
);
7515 static void gen_uqxtnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7517 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7518 int halfbits
= 4 << vece
;
7519 int64_t max
= (1ull << halfbits
) - 1;
7521 tcg_gen_dupi_vec(vece
, t
, max
);
7522 tcg_gen_umin_vec(vece
, n
, n
, t
);
7523 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7524 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7525 tcg_temp_free_vec(t
);
7528 static bool trans_UQXTNT(DisasContext
*s
, arg_rri_esz
*a
)
7530 static const GVecGen2 ops
[3] = {
7531 { .fniv
= gen_uqxtnt_vec
,
7532 .opt_opc
= uqxtn_list
,
7534 .fno
= gen_helper_sve2_uqxtnt_h
,
7536 { .fniv
= gen_uqxtnt_vec
,
7537 .opt_opc
= uqxtn_list
,
7539 .fno
= gen_helper_sve2_uqxtnt_s
,
7541 { .fniv
= gen_uqxtnt_vec
,
7542 .opt_opc
= uqxtn_list
,
7544 .fno
= gen_helper_sve2_uqxtnt_d
,
7547 return do_sve2_narrow_extract(s
, a
, ops
);
7550 static const TCGOpcode sqxtun_list
[] = {
7551 INDEX_op_shli_vec
, INDEX_op_umin_vec
, INDEX_op_smax_vec
, 0
7554 static void gen_sqxtunb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7556 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7557 int halfbits
= 4 << vece
;
7558 int64_t max
= (1ull << halfbits
) - 1;
7560 tcg_gen_dupi_vec(vece
, t
, 0);
7561 tcg_gen_smax_vec(vece
, d
, n
, t
);
7562 tcg_gen_dupi_vec(vece
, t
, max
);
7563 tcg_gen_umin_vec(vece
, d
, d
, t
);
7564 tcg_temp_free_vec(t
);
7567 static bool trans_SQXTUNB(DisasContext
*s
, arg_rri_esz
*a
)
7569 static const GVecGen2 ops
[3] = {
7570 { .fniv
= gen_sqxtunb_vec
,
7571 .opt_opc
= sqxtun_list
,
7572 .fno
= gen_helper_sve2_sqxtunb_h
,
7574 { .fniv
= gen_sqxtunb_vec
,
7575 .opt_opc
= sqxtun_list
,
7576 .fno
= gen_helper_sve2_sqxtunb_s
,
7578 { .fniv
= gen_sqxtunb_vec
,
7579 .opt_opc
= sqxtun_list
,
7580 .fno
= gen_helper_sve2_sqxtunb_d
,
7583 return do_sve2_narrow_extract(s
, a
, ops
);
7586 static void gen_sqxtunt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7588 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7589 int halfbits
= 4 << vece
;
7590 int64_t max
= (1ull << halfbits
) - 1;
7592 tcg_gen_dupi_vec(vece
, t
, 0);
7593 tcg_gen_smax_vec(vece
, n
, n
, t
);
7594 tcg_gen_dupi_vec(vece
, t
, max
);
7595 tcg_gen_umin_vec(vece
, n
, n
, t
);
7596 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7597 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7598 tcg_temp_free_vec(t
);
7601 static bool trans_SQXTUNT(DisasContext
*s
, arg_rri_esz
*a
)
7603 static const GVecGen2 ops
[3] = {
7604 { .fniv
= gen_sqxtunt_vec
,
7605 .opt_opc
= sqxtun_list
,
7607 .fno
= gen_helper_sve2_sqxtunt_h
,
7609 { .fniv
= gen_sqxtunt_vec
,
7610 .opt_opc
= sqxtun_list
,
7612 .fno
= gen_helper_sve2_sqxtunt_s
,
7614 { .fniv
= gen_sqxtunt_vec
,
7615 .opt_opc
= sqxtun_list
,
7617 .fno
= gen_helper_sve2_sqxtunt_d
,
7620 return do_sve2_narrow_extract(s
, a
, ops
);
7623 static bool do_sve2_shr_narrow(DisasContext
*s
, arg_rri_esz
*a
,
7624 const GVecGen2i ops
[3])
7626 if (a
->esz
< 0 || a
->esz
> MO_32
|| !dc_isar_feature(aa64_sve2
, s
)) {
7629 assert(a
->imm
> 0 && a
->imm
<= (8 << a
->esz
));
7630 if (sve_access_check(s
)) {
7631 unsigned vsz
= vec_full_reg_size(s
);
7632 tcg_gen_gvec_2i(vec_full_reg_offset(s
, a
->rd
),
7633 vec_full_reg_offset(s
, a
->rn
),
7634 vsz
, vsz
, a
->imm
, &ops
[a
->esz
]);
7639 static void gen_shrnb_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int shr
)
7641 int halfbits
= 4 << vece
;
7642 uint64_t mask
= dup_const(vece
, MAKE_64BIT_MASK(0, halfbits
));
7644 tcg_gen_shri_i64(d
, n
, shr
);
7645 tcg_gen_andi_i64(d
, d
, mask
);
7648 static void gen_shrnb16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7650 gen_shrnb_i64(MO_16
, d
, n
, shr
);
7653 static void gen_shrnb32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7655 gen_shrnb_i64(MO_32
, d
, n
, shr
);
7658 static void gen_shrnb64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7660 gen_shrnb_i64(MO_64
, d
, n
, shr
);
7663 static void gen_shrnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t shr
)
7665 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7666 int halfbits
= 4 << vece
;
7667 uint64_t mask
= MAKE_64BIT_MASK(0, halfbits
);
7669 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7670 tcg_gen_dupi_vec(vece
, t
, mask
);
7671 tcg_gen_and_vec(vece
, d
, n
, t
);
7672 tcg_temp_free_vec(t
);
7675 static bool trans_SHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7677 static const TCGOpcode vec_list
[] = { INDEX_op_shri_vec
, 0 };
7678 static const GVecGen2i ops
[3] = {
7679 { .fni8
= gen_shrnb16_i64
,
7680 .fniv
= gen_shrnb_vec
,
7681 .opt_opc
= vec_list
,
7682 .fno
= gen_helper_sve2_shrnb_h
,
7684 { .fni8
= gen_shrnb32_i64
,
7685 .fniv
= gen_shrnb_vec
,
7686 .opt_opc
= vec_list
,
7687 .fno
= gen_helper_sve2_shrnb_s
,
7689 { .fni8
= gen_shrnb64_i64
,
7690 .fniv
= gen_shrnb_vec
,
7691 .opt_opc
= vec_list
,
7692 .fno
= gen_helper_sve2_shrnb_d
,
7695 return do_sve2_shr_narrow(s
, a
, ops
);
7698 static void gen_shrnt_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int shr
)
7700 int halfbits
= 4 << vece
;
7701 uint64_t mask
= dup_const(vece
, MAKE_64BIT_MASK(0, halfbits
));
7703 tcg_gen_shli_i64(n
, n
, halfbits
- shr
);
7704 tcg_gen_andi_i64(n
, n
, ~mask
);
7705 tcg_gen_andi_i64(d
, d
, mask
);
7706 tcg_gen_or_i64(d
, d
, n
);
7709 static void gen_shrnt16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7711 gen_shrnt_i64(MO_16
, d
, n
, shr
);
7714 static void gen_shrnt32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7716 gen_shrnt_i64(MO_32
, d
, n
, shr
);
7719 static void gen_shrnt64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7721 tcg_gen_shri_i64(n
, n
, shr
);
7722 tcg_gen_deposit_i64(d
, d
, n
, 32, 32);
7725 static void gen_shrnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t shr
)
7727 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7728 int halfbits
= 4 << vece
;
7729 uint64_t mask
= MAKE_64BIT_MASK(0, halfbits
);
7731 tcg_gen_shli_vec(vece
, n
, n
, halfbits
- shr
);
7732 tcg_gen_dupi_vec(vece
, t
, mask
);
7733 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7734 tcg_temp_free_vec(t
);
7737 static bool trans_SHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7739 static const TCGOpcode vec_list
[] = { INDEX_op_shli_vec
, 0 };
7740 static const GVecGen2i ops
[3] = {
7741 { .fni8
= gen_shrnt16_i64
,
7742 .fniv
= gen_shrnt_vec
,
7743 .opt_opc
= vec_list
,
7745 .fno
= gen_helper_sve2_shrnt_h
,
7747 { .fni8
= gen_shrnt32_i64
,
7748 .fniv
= gen_shrnt_vec
,
7749 .opt_opc
= vec_list
,
7751 .fno
= gen_helper_sve2_shrnt_s
,
7753 { .fni8
= gen_shrnt64_i64
,
7754 .fniv
= gen_shrnt_vec
,
7755 .opt_opc
= vec_list
,
7757 .fno
= gen_helper_sve2_shrnt_d
,
7760 return do_sve2_shr_narrow(s
, a
, ops
);
7763 static bool trans_RSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7765 static const GVecGen2i ops
[3] = {
7766 { .fno
= gen_helper_sve2_rshrnb_h
},
7767 { .fno
= gen_helper_sve2_rshrnb_s
},
7768 { .fno
= gen_helper_sve2_rshrnb_d
},
7770 return do_sve2_shr_narrow(s
, a
, ops
);
7773 static bool trans_RSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7775 static const GVecGen2i ops
[3] = {
7776 { .fno
= gen_helper_sve2_rshrnt_h
},
7777 { .fno
= gen_helper_sve2_rshrnt_s
},
7778 { .fno
= gen_helper_sve2_rshrnt_d
},
7780 return do_sve2_shr_narrow(s
, a
, ops
);
7783 static void gen_sqshrunb_vec(unsigned vece
, TCGv_vec d
,
7784 TCGv_vec n
, int64_t shr
)
7786 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7787 int halfbits
= 4 << vece
;
7789 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7790 tcg_gen_dupi_vec(vece
, t
, 0);
7791 tcg_gen_smax_vec(vece
, n
, n
, t
);
7792 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7793 tcg_gen_umin_vec(vece
, d
, n
, t
);
7794 tcg_temp_free_vec(t
);
7797 static bool trans_SQSHRUNB(DisasContext
*s
, arg_rri_esz
*a
)
7799 static const TCGOpcode vec_list
[] = {
7800 INDEX_op_sari_vec
, INDEX_op_smax_vec
, INDEX_op_umin_vec
, 0
7802 static const GVecGen2i ops
[3] = {
7803 { .fniv
= gen_sqshrunb_vec
,
7804 .opt_opc
= vec_list
,
7805 .fno
= gen_helper_sve2_sqshrunb_h
,
7807 { .fniv
= gen_sqshrunb_vec
,
7808 .opt_opc
= vec_list
,
7809 .fno
= gen_helper_sve2_sqshrunb_s
,
7811 { .fniv
= gen_sqshrunb_vec
,
7812 .opt_opc
= vec_list
,
7813 .fno
= gen_helper_sve2_sqshrunb_d
,
7816 return do_sve2_shr_narrow(s
, a
, ops
);
7819 static void gen_sqshrunt_vec(unsigned vece
, TCGv_vec d
,
7820 TCGv_vec n
, int64_t shr
)
7822 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7823 int halfbits
= 4 << vece
;
7825 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7826 tcg_gen_dupi_vec(vece
, t
, 0);
7827 tcg_gen_smax_vec(vece
, n
, n
, t
);
7828 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7829 tcg_gen_umin_vec(vece
, n
, n
, t
);
7830 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7831 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7832 tcg_temp_free_vec(t
);
7835 static bool trans_SQSHRUNT(DisasContext
*s
, arg_rri_esz
*a
)
7837 static const TCGOpcode vec_list
[] = {
7838 INDEX_op_shli_vec
, INDEX_op_sari_vec
,
7839 INDEX_op_smax_vec
, INDEX_op_umin_vec
, 0
7841 static const GVecGen2i ops
[3] = {
7842 { .fniv
= gen_sqshrunt_vec
,
7843 .opt_opc
= vec_list
,
7845 .fno
= gen_helper_sve2_sqshrunt_h
,
7847 { .fniv
= gen_sqshrunt_vec
,
7848 .opt_opc
= vec_list
,
7850 .fno
= gen_helper_sve2_sqshrunt_s
,
7852 { .fniv
= gen_sqshrunt_vec
,
7853 .opt_opc
= vec_list
,
7855 .fno
= gen_helper_sve2_sqshrunt_d
,
7858 return do_sve2_shr_narrow(s
, a
, ops
);
7861 static bool trans_SQRSHRUNB(DisasContext
*s
, arg_rri_esz
*a
)
7863 static const GVecGen2i ops
[3] = {
7864 { .fno
= gen_helper_sve2_sqrshrunb_h
},
7865 { .fno
= gen_helper_sve2_sqrshrunb_s
},
7866 { .fno
= gen_helper_sve2_sqrshrunb_d
},
7868 return do_sve2_shr_narrow(s
, a
, ops
);
7871 static bool trans_SQRSHRUNT(DisasContext
*s
, arg_rri_esz
*a
)
7873 static const GVecGen2i ops
[3] = {
7874 { .fno
= gen_helper_sve2_sqrshrunt_h
},
7875 { .fno
= gen_helper_sve2_sqrshrunt_s
},
7876 { .fno
= gen_helper_sve2_sqrshrunt_d
},
7878 return do_sve2_shr_narrow(s
, a
, ops
);
7881 static void gen_sqshrnb_vec(unsigned vece
, TCGv_vec d
,
7882 TCGv_vec n
, int64_t shr
)
7884 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7885 int halfbits
= 4 << vece
;
7886 int64_t max
= MAKE_64BIT_MASK(0, halfbits
- 1);
7887 int64_t min
= -max
- 1;
7889 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7890 tcg_gen_dupi_vec(vece
, t
, min
);
7891 tcg_gen_smax_vec(vece
, n
, n
, t
);
7892 tcg_gen_dupi_vec(vece
, t
, max
);
7893 tcg_gen_smin_vec(vece
, n
, n
, t
);
7894 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7895 tcg_gen_and_vec(vece
, d
, n
, t
);
7896 tcg_temp_free_vec(t
);
7899 static bool trans_SQSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7901 static const TCGOpcode vec_list
[] = {
7902 INDEX_op_sari_vec
, INDEX_op_smax_vec
, INDEX_op_smin_vec
, 0
7904 static const GVecGen2i ops
[3] = {
7905 { .fniv
= gen_sqshrnb_vec
,
7906 .opt_opc
= vec_list
,
7907 .fno
= gen_helper_sve2_sqshrnb_h
,
7909 { .fniv
= gen_sqshrnb_vec
,
7910 .opt_opc
= vec_list
,
7911 .fno
= gen_helper_sve2_sqshrnb_s
,
7913 { .fniv
= gen_sqshrnb_vec
,
7914 .opt_opc
= vec_list
,
7915 .fno
= gen_helper_sve2_sqshrnb_d
,
7918 return do_sve2_shr_narrow(s
, a
, ops
);
7921 static void gen_sqshrnt_vec(unsigned vece
, TCGv_vec d
,
7922 TCGv_vec n
, int64_t shr
)
7924 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7925 int halfbits
= 4 << vece
;
7926 int64_t max
= MAKE_64BIT_MASK(0, halfbits
- 1);
7927 int64_t min
= -max
- 1;
7929 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7930 tcg_gen_dupi_vec(vece
, t
, min
);
7931 tcg_gen_smax_vec(vece
, n
, n
, t
);
7932 tcg_gen_dupi_vec(vece
, t
, max
);
7933 tcg_gen_smin_vec(vece
, n
, n
, t
);
7934 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7935 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7936 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7937 tcg_temp_free_vec(t
);
7940 static bool trans_SQSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7942 static const TCGOpcode vec_list
[] = {
7943 INDEX_op_shli_vec
, INDEX_op_sari_vec
,
7944 INDEX_op_smax_vec
, INDEX_op_smin_vec
, 0
7946 static const GVecGen2i ops
[3] = {
7947 { .fniv
= gen_sqshrnt_vec
,
7948 .opt_opc
= vec_list
,
7950 .fno
= gen_helper_sve2_sqshrnt_h
,
7952 { .fniv
= gen_sqshrnt_vec
,
7953 .opt_opc
= vec_list
,
7955 .fno
= gen_helper_sve2_sqshrnt_s
,
7957 { .fniv
= gen_sqshrnt_vec
,
7958 .opt_opc
= vec_list
,
7960 .fno
= gen_helper_sve2_sqshrnt_d
,
7963 return do_sve2_shr_narrow(s
, a
, ops
);
7966 static bool trans_SQRSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7968 static const GVecGen2i ops
[3] = {
7969 { .fno
= gen_helper_sve2_sqrshrnb_h
},
7970 { .fno
= gen_helper_sve2_sqrshrnb_s
},
7971 { .fno
= gen_helper_sve2_sqrshrnb_d
},
7973 return do_sve2_shr_narrow(s
, a
, ops
);
7976 static bool trans_SQRSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7978 static const GVecGen2i ops
[3] = {
7979 { .fno
= gen_helper_sve2_sqrshrnt_h
},
7980 { .fno
= gen_helper_sve2_sqrshrnt_s
},
7981 { .fno
= gen_helper_sve2_sqrshrnt_d
},
7983 return do_sve2_shr_narrow(s
, a
, ops
);
7986 static void gen_uqshrnb_vec(unsigned vece
, TCGv_vec d
,
7987 TCGv_vec n
, int64_t shr
)
7989 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7990 int halfbits
= 4 << vece
;
7992 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7993 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7994 tcg_gen_umin_vec(vece
, d
, n
, t
);
7995 tcg_temp_free_vec(t
);
7998 static bool trans_UQSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
8000 static const TCGOpcode vec_list
[] = {
8001 INDEX_op_shri_vec
, INDEX_op_umin_vec
, 0
8003 static const GVecGen2i ops
[3] = {
8004 { .fniv
= gen_uqshrnb_vec
,
8005 .opt_opc
= vec_list
,
8006 .fno
= gen_helper_sve2_uqshrnb_h
,
8008 { .fniv
= gen_uqshrnb_vec
,
8009 .opt_opc
= vec_list
,
8010 .fno
= gen_helper_sve2_uqshrnb_s
,
8012 { .fniv
= gen_uqshrnb_vec
,
8013 .opt_opc
= vec_list
,
8014 .fno
= gen_helper_sve2_uqshrnb_d
,
8017 return do_sve2_shr_narrow(s
, a
, ops
);
8020 static void gen_uqshrnt_vec(unsigned vece
, TCGv_vec d
,
8021 TCGv_vec n
, int64_t shr
)
8023 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
8024 int halfbits
= 4 << vece
;
8026 tcg_gen_shri_vec(vece
, n
, n
, shr
);
8027 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
8028 tcg_gen_umin_vec(vece
, n
, n
, t
);
8029 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
8030 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
8031 tcg_temp_free_vec(t
);
8034 static bool trans_UQSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
8036 static const TCGOpcode vec_list
[] = {
8037 INDEX_op_shli_vec
, INDEX_op_shri_vec
, INDEX_op_umin_vec
, 0
8039 static const GVecGen2i ops
[3] = {
8040 { .fniv
= gen_uqshrnt_vec
,
8041 .opt_opc
= vec_list
,
8043 .fno
= gen_helper_sve2_uqshrnt_h
,
8045 { .fniv
= gen_uqshrnt_vec
,
8046 .opt_opc
= vec_list
,
8048 .fno
= gen_helper_sve2_uqshrnt_s
,
8050 { .fniv
= gen_uqshrnt_vec
,
8051 .opt_opc
= vec_list
,
8053 .fno
= gen_helper_sve2_uqshrnt_d
,
8056 return do_sve2_shr_narrow(s
, a
, ops
);
8059 static bool trans_UQRSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
8061 static const GVecGen2i ops
[3] = {
8062 { .fno
= gen_helper_sve2_uqrshrnb_h
},
8063 { .fno
= gen_helper_sve2_uqrshrnb_s
},
8064 { .fno
= gen_helper_sve2_uqrshrnb_d
},
8066 return do_sve2_shr_narrow(s
, a
, ops
);
8069 static bool trans_UQRSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
8071 static const GVecGen2i ops
[3] = {
8072 { .fno
= gen_helper_sve2_uqrshrnt_h
},
8073 { .fno
= gen_helper_sve2_uqrshrnt_s
},
8074 { .fno
= gen_helper_sve2_uqrshrnt_d
},
8076 return do_sve2_shr_narrow(s
, a
, ops
);
8079 #define DO_SVE2_ZZZ_NARROW(NAME, name) \
8080 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
8082 static gen_helper_gvec_3 * const fns[4] = { \
8083 NULL, gen_helper_sve2_##name##_h, \
8084 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
8086 return do_sve2_zzz_ool(s, a, fns[a->esz]); \
8089 DO_SVE2_ZZZ_NARROW(ADDHNB
, addhnb
)
8090 DO_SVE2_ZZZ_NARROW(ADDHNT
, addhnt
)
8091 DO_SVE2_ZZZ_NARROW(RADDHNB
, raddhnb
)
8092 DO_SVE2_ZZZ_NARROW(RADDHNT
, raddhnt
)
8094 DO_SVE2_ZZZ_NARROW(SUBHNB
, subhnb
)
8095 DO_SVE2_ZZZ_NARROW(SUBHNT
, subhnt
)
8096 DO_SVE2_ZZZ_NARROW(RSUBHNB
, rsubhnb
)
8097 DO_SVE2_ZZZ_NARROW(RSUBHNT
, rsubhnt
)
8099 static bool do_sve2_ppzz_flags(DisasContext
*s
, arg_rprr_esz
*a
,
8100 gen_helper_gvec_flags_4
*fn
)
8102 if (!dc_isar_feature(aa64_sve2
, s
)) {
8105 return do_ppzz_flags(s
, a
, fn
);
8108 #define DO_SVE2_PPZZ_MATCH(NAME, name) \
8109 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
8111 static gen_helper_gvec_flags_4 * const fns[4] = { \
8112 gen_helper_sve2_##name##_ppzz_b, gen_helper_sve2_##name##_ppzz_h, \
8115 return do_sve2_ppzz_flags(s, a, fns[a->esz]); \
8118 DO_SVE2_PPZZ_MATCH(MATCH
, match
)
8119 DO_SVE2_PPZZ_MATCH(NMATCH
, nmatch
)
8121 static bool trans_HISTCNT(DisasContext
*s
, arg_rprr_esz
*a
)
8123 static gen_helper_gvec_4
* const fns
[2] = {
8124 gen_helper_sve2_histcnt_s
, gen_helper_sve2_histcnt_d
8129 return do_sve2_zpzz_ool(s
, a
, fns
[a
->esz
- 2]);
8132 static bool trans_HISTSEG(DisasContext
*s
, arg_rrr_esz
*a
)
8137 return do_sve2_zzz_ool(s
, a
, gen_helper_sve2_histseg
);
8140 static bool do_sve2_zpzz_fp(DisasContext
*s
, arg_rprr_esz
*a
,
8141 gen_helper_gvec_4_ptr
*fn
)
8143 if (!dc_isar_feature(aa64_sve2
, s
)) {
8146 return do_zpzz_fp(s
, a
, fn
);
8149 #define DO_SVE2_ZPZZ_FP(NAME, name) \
8150 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
8152 static gen_helper_gvec_4_ptr * const fns[4] = { \
8153 NULL, gen_helper_sve2_##name##_zpzz_h, \
8154 gen_helper_sve2_##name##_zpzz_s, gen_helper_sve2_##name##_zpzz_d \
8156 return do_sve2_zpzz_fp(s, a, fns[a->esz]); \
8159 DO_SVE2_ZPZZ_FP(FADDP
, faddp
)
8160 DO_SVE2_ZPZZ_FP(FMAXNMP
, fmaxnmp
)
8161 DO_SVE2_ZPZZ_FP(FMINNMP
, fminnmp
)
8162 DO_SVE2_ZPZZ_FP(FMAXP
, fmaxp
)
8163 DO_SVE2_ZPZZ_FP(FMINP
, fminp
)
8166 * SVE Integer Multiply-Add (unpredicated)
8169 static bool trans_FMMLA(DisasContext
*s
, arg_rrrr_esz
*a
)
8171 gen_helper_gvec_4_ptr
*fn
;
8175 if (!dc_isar_feature(aa64_sve_f32mm
, s
)) {
8178 fn
= gen_helper_fmmla_s
;
8181 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
8184 fn
= gen_helper_fmmla_d
;
8190 if (sve_access_check(s
)) {
8191 unsigned vsz
= vec_full_reg_size(s
);
8192 TCGv_ptr status
= fpstatus_ptr(FPST_FPCR
);
8193 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
8194 vec_full_reg_offset(s
, a
->rn
),
8195 vec_full_reg_offset(s
, a
->rm
),
8196 vec_full_reg_offset(s
, a
->ra
),
8197 status
, vsz
, vsz
, 0, fn
);
8198 tcg_temp_free_ptr(status
);
8203 static bool do_sqdmlal_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
,
8204 bool sel1
, bool sel2
)
8206 static gen_helper_gvec_4
* const fns
[] = {
8207 NULL
, gen_helper_sve2_sqdmlal_zzzw_h
,
8208 gen_helper_sve2_sqdmlal_zzzw_s
, gen_helper_sve2_sqdmlal_zzzw_d
,
8210 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], (sel2
<< 1) | sel1
);
8213 static bool do_sqdmlsl_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
,
8214 bool sel1
, bool sel2
)
8216 static gen_helper_gvec_4
* const fns
[] = {
8217 NULL
, gen_helper_sve2_sqdmlsl_zzzw_h
,
8218 gen_helper_sve2_sqdmlsl_zzzw_s
, gen_helper_sve2_sqdmlsl_zzzw_d
,
8220 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], (sel2
<< 1) | sel1
);
8223 static bool trans_SQDMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8225 return do_sqdmlal_zzzw(s
, a
, false, false);
8228 static bool trans_SQDMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8230 return do_sqdmlal_zzzw(s
, a
, true, true);
8233 static bool trans_SQDMLALBT(DisasContext
*s
, arg_rrrr_esz
*a
)
8235 return do_sqdmlal_zzzw(s
, a
, false, true);
8238 static bool trans_SQDMLSLB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8240 return do_sqdmlsl_zzzw(s
, a
, false, false);
8243 static bool trans_SQDMLSLT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8245 return do_sqdmlsl_zzzw(s
, a
, true, true);
8248 static bool trans_SQDMLSLBT(DisasContext
*s
, arg_rrrr_esz
*a
)
8250 return do_sqdmlsl_zzzw(s
, a
, false, true);
8253 static bool trans_SQRDMLAH_zzzz(DisasContext
*s
, arg_rrrr_esz
*a
)
8255 static gen_helper_gvec_4
* const fns
[] = {
8256 gen_helper_sve2_sqrdmlah_b
, gen_helper_sve2_sqrdmlah_h
,
8257 gen_helper_sve2_sqrdmlah_s
, gen_helper_sve2_sqrdmlah_d
,
8259 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], 0);
8262 static bool trans_SQRDMLSH_zzzz(DisasContext
*s
, arg_rrrr_esz
*a
)
8264 static gen_helper_gvec_4
* const fns
[] = {
8265 gen_helper_sve2_sqrdmlsh_b
, gen_helper_sve2_sqrdmlsh_h
,
8266 gen_helper_sve2_sqrdmlsh_s
, gen_helper_sve2_sqrdmlsh_d
,
8268 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], 0);
8271 static bool do_smlal_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
8273 static gen_helper_gvec_4
* const fns
[] = {
8274 NULL
, gen_helper_sve2_smlal_zzzw_h
,
8275 gen_helper_sve2_smlal_zzzw_s
, gen_helper_sve2_smlal_zzzw_d
,
8277 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], sel
);
8280 static bool trans_SMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8282 return do_smlal_zzzw(s
, a
, false);
8285 static bool trans_SMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8287 return do_smlal_zzzw(s
, a
, true);
8290 static bool do_umlal_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
8292 static gen_helper_gvec_4
* const fns
[] = {
8293 NULL
, gen_helper_sve2_umlal_zzzw_h
,
8294 gen_helper_sve2_umlal_zzzw_s
, gen_helper_sve2_umlal_zzzw_d
,
8296 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], sel
);
8299 static bool trans_UMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8301 return do_umlal_zzzw(s
, a
, false);
8304 static bool trans_UMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8306 return do_umlal_zzzw(s
, a
, true);
8309 static bool do_smlsl_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
8311 static gen_helper_gvec_4
* const fns
[] = {
8312 NULL
, gen_helper_sve2_smlsl_zzzw_h
,
8313 gen_helper_sve2_smlsl_zzzw_s
, gen_helper_sve2_smlsl_zzzw_d
,
8315 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], sel
);
8318 static bool trans_SMLSLB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8320 return do_smlsl_zzzw(s
, a
, false);
8323 static bool trans_SMLSLT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8325 return do_smlsl_zzzw(s
, a
, true);
8328 static bool do_umlsl_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
8330 static gen_helper_gvec_4
* const fns
[] = {
8331 NULL
, gen_helper_sve2_umlsl_zzzw_h
,
8332 gen_helper_sve2_umlsl_zzzw_s
, gen_helper_sve2_umlsl_zzzw_d
,
8334 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], sel
);
8337 static bool trans_UMLSLB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8339 return do_umlsl_zzzw(s
, a
, false);
8342 static bool trans_UMLSLT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8344 return do_umlsl_zzzw(s
, a
, true);
8347 static bool trans_CMLA_zzzz(DisasContext
*s
, arg_CMLA_zzzz
*a
)
8349 static gen_helper_gvec_4
* const fns
[] = {
8350 gen_helper_sve2_cmla_zzzz_b
, gen_helper_sve2_cmla_zzzz_h
,
8351 gen_helper_sve2_cmla_zzzz_s
, gen_helper_sve2_cmla_zzzz_d
,
8354 if (!dc_isar_feature(aa64_sve2
, s
)) {
8357 if (sve_access_check(s
)) {
8358 gen_gvec_ool_zzzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
);
8363 static bool trans_CDOT_zzzz(DisasContext
*s
, arg_CMLA_zzzz
*a
)
8365 if (!dc_isar_feature(aa64_sve2
, s
) || a
->esz
< MO_32
) {
8368 if (sve_access_check(s
)) {
8369 gen_helper_gvec_4
*fn
= (a
->esz
== MO_32
8370 ? gen_helper_sve2_cdot_zzzz_s
8371 : gen_helper_sve2_cdot_zzzz_d
);
8372 gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
);
8377 static bool trans_SQRDCMLAH_zzzz(DisasContext
*s
, arg_SQRDCMLAH_zzzz
*a
)
8379 static gen_helper_gvec_4
* const fns
[] = {
8380 gen_helper_sve2_sqrdcmlah_zzzz_b
, gen_helper_sve2_sqrdcmlah_zzzz_h
,
8381 gen_helper_sve2_sqrdcmlah_zzzz_s
, gen_helper_sve2_sqrdcmlah_zzzz_d
,
8384 if (!dc_isar_feature(aa64_sve2
, s
)) {
8387 if (sve_access_check(s
)) {
8388 gen_gvec_ool_zzzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
);
8393 static bool trans_USDOT_zzzz(DisasContext
*s
, arg_USDOT_zzzz
*a
)
8395 if (a
->esz
!= 2 || !dc_isar_feature(aa64_sve_i8mm
, s
)) {
8398 if (sve_access_check(s
)) {
8399 unsigned vsz
= vec_full_reg_size(s
);
8400 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, a
->rd
),
8401 vec_full_reg_offset(s
, a
->rn
),
8402 vec_full_reg_offset(s
, a
->rm
),
8403 vec_full_reg_offset(s
, a
->ra
),
8404 vsz
, vsz
, 0, gen_helper_gvec_usdot_b
);
8409 static bool trans_AESMC(DisasContext
*s
, arg_AESMC
*a
)
8411 if (!dc_isar_feature(aa64_sve2_aes
, s
)) {
8414 if (sve_access_check(s
)) {
8415 gen_gvec_ool_zz(s
, gen_helper_crypto_aesmc
, a
->rd
, a
->rd
, a
->decrypt
);
8420 static bool do_aese(DisasContext
*s
, arg_rrr_esz
*a
, bool decrypt
)
8422 if (!dc_isar_feature(aa64_sve2_aes
, s
)) {
8425 if (sve_access_check(s
)) {
8426 gen_gvec_ool_zzz(s
, gen_helper_crypto_aese
,
8427 a
->rd
, a
->rn
, a
->rm
, decrypt
);
8432 static bool trans_AESE(DisasContext
*s
, arg_rrr_esz
*a
)
8434 return do_aese(s
, a
, false);
8437 static bool trans_AESD(DisasContext
*s
, arg_rrr_esz
*a
)
8439 return do_aese(s
, a
, true);
8442 static bool do_sm4(DisasContext
*s
, arg_rrr_esz
*a
, gen_helper_gvec_3
*fn
)
8444 if (!dc_isar_feature(aa64_sve2_sm4
, s
)) {
8447 if (sve_access_check(s
)) {
8448 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, 0);
8453 static bool trans_SM4E(DisasContext
*s
, arg_rrr_esz
*a
)
8455 return do_sm4(s
, a
, gen_helper_crypto_sm4e
);
8458 static bool trans_SM4EKEY(DisasContext
*s
, arg_rrr_esz
*a
)
8460 return do_sm4(s
, a
, gen_helper_crypto_sm4ekey
);
8463 static bool trans_RAX1(DisasContext
*s
, arg_rrr_esz
*a
)
8465 if (!dc_isar_feature(aa64_sve2_sha3
, s
)) {
8468 if (sve_access_check(s
)) {
8469 gen_gvec_fn_zzz(s
, gen_gvec_rax1
, MO_64
, a
->rd
, a
->rn
, a
->rm
);
8474 static bool trans_FCVTNT_sh(DisasContext
*s
, arg_rpr_esz
*a
)
8476 if (!dc_isar_feature(aa64_sve2
, s
)) {
8479 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtnt_sh
);
8482 static bool trans_BFCVTNT(DisasContext
*s
, arg_rpr_esz
*a
)
8484 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
8487 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_bfcvtnt
);
8490 static bool trans_FCVTNT_ds(DisasContext
*s
, arg_rpr_esz
*a
)
8492 if (!dc_isar_feature(aa64_sve2
, s
)) {
8495 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtnt_ds
);
8498 static bool trans_FCVTLT_hs(DisasContext
*s
, arg_rpr_esz
*a
)
8500 if (!dc_isar_feature(aa64_sve2
, s
)) {
8503 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtlt_hs
);
8506 static bool trans_FCVTLT_sd(DisasContext
*s
, arg_rpr_esz
*a
)
8508 if (!dc_isar_feature(aa64_sve2
, s
)) {
8511 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtlt_sd
);
8514 static bool trans_FCVTX_ds(DisasContext
*s
, arg_rpr_esz
*a
)
8516 if (!dc_isar_feature(aa64_sve2
, s
)) {
8519 return do_frint_mode(s
, a
, float_round_to_odd
, gen_helper_sve_fcvt_ds
);
8522 static bool trans_FCVTXNT_ds(DisasContext
*s
, arg_rpr_esz
*a
)
8524 if (!dc_isar_feature(aa64_sve2
, s
)) {
8527 return do_frint_mode(s
, a
, float_round_to_odd
, gen_helper_sve2_fcvtnt_ds
);
8530 static bool trans_FLOGB(DisasContext
*s
, arg_rpr_esz
*a
)
8532 static gen_helper_gvec_3_ptr
* const fns
[] = {
8533 NULL
, gen_helper_flogb_h
,
8534 gen_helper_flogb_s
, gen_helper_flogb_d
8537 if (!dc_isar_feature(aa64_sve2
, s
) || fns
[a
->esz
] == NULL
) {
8540 if (sve_access_check(s
)) {
8542 fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
8543 unsigned vsz
= vec_full_reg_size(s
);
8545 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
8546 vec_full_reg_offset(s
, a
->rn
),
8547 pred_full_reg_offset(s
, a
->pg
),
8548 status
, vsz
, vsz
, 0, fns
[a
->esz
]);
8549 tcg_temp_free_ptr(status
);
8554 static bool do_FMLAL_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sub
, bool sel
)
8556 if (!dc_isar_feature(aa64_sve2
, s
)) {
8559 if (sve_access_check(s
)) {
8560 unsigned vsz
= vec_full_reg_size(s
);
8561 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
8562 vec_full_reg_offset(s
, a
->rn
),
8563 vec_full_reg_offset(s
, a
->rm
),
8564 vec_full_reg_offset(s
, a
->ra
),
8565 cpu_env
, vsz
, vsz
, (sel
<< 1) | sub
,
8566 gen_helper_sve2_fmlal_zzzw_s
);
8571 static bool trans_FMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8573 return do_FMLAL_zzzw(s
, a
, false, false);
8576 static bool trans_FMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8578 return do_FMLAL_zzzw(s
, a
, false, true);
8581 static bool trans_FMLSLB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8583 return do_FMLAL_zzzw(s
, a
, true, false);
8586 static bool trans_FMLSLT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8588 return do_FMLAL_zzzw(s
, a
, true, true);
8591 static bool do_FMLAL_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
, bool sub
, bool sel
)
8593 if (!dc_isar_feature(aa64_sve2
, s
)) {
8596 if (sve_access_check(s
)) {
8597 unsigned vsz
= vec_full_reg_size(s
);
8598 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
8599 vec_full_reg_offset(s
, a
->rn
),
8600 vec_full_reg_offset(s
, a
->rm
),
8601 vec_full_reg_offset(s
, a
->ra
),
8603 (a
->index
<< 2) | (sel
<< 1) | sub
,
8604 gen_helper_sve2_fmlal_zzxw_s
);
8609 static bool trans_FMLALB_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
8611 return do_FMLAL_zzxw(s
, a
, false, false);
8614 static bool trans_FMLALT_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
8616 return do_FMLAL_zzxw(s
, a
, false, true);
8619 static bool trans_FMLSLB_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
8621 return do_FMLAL_zzxw(s
, a
, true, false);
8624 static bool trans_FMLSLT_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
8626 return do_FMLAL_zzxw(s
, a
, true, true);
8629 static bool do_i8mm_zzzz_ool(DisasContext
*s
, arg_rrrr_esz
*a
,
8630 gen_helper_gvec_4
*fn
, int data
)
8632 if (!dc_isar_feature(aa64_sve_i8mm
, s
)) {
8635 if (sve_access_check(s
)) {
8636 gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, data
);
8641 static bool trans_SMMLA(DisasContext
*s
, arg_rrrr_esz
*a
)
8643 return do_i8mm_zzzz_ool(s
, a
, gen_helper_gvec_smmla_b
, 0);
8646 static bool trans_USMMLA(DisasContext
*s
, arg_rrrr_esz
*a
)
8648 return do_i8mm_zzzz_ool(s
, a
, gen_helper_gvec_usmmla_b
, 0);
8651 static bool trans_UMMLA(DisasContext
*s
, arg_rrrr_esz
*a
)
8653 return do_i8mm_zzzz_ool(s
, a
, gen_helper_gvec_ummla_b
, 0);
8656 static bool trans_BFDOT_zzzz(DisasContext
*s
, arg_rrrr_esz
*a
)
8658 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
8661 if (sve_access_check(s
)) {
8662 gen_gvec_ool_zzzz(s
, gen_helper_gvec_bfdot
,
8663 a
->rd
, a
->rn
, a
->rm
, a
->ra
, 0);
8668 static bool trans_BFDOT_zzxz(DisasContext
*s
, arg_rrxr_esz
*a
)
8670 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
8673 if (sve_access_check(s
)) {
8674 gen_gvec_ool_zzzz(s
, gen_helper_gvec_bfdot_idx
,
8675 a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->index
);
8680 static bool trans_BFMMLA(DisasContext
*s
, arg_rrrr_esz
*a
)
8682 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
8685 if (sve_access_check(s
)) {
8686 gen_gvec_ool_zzzz(s
, gen_helper_gvec_bfmmla
,
8687 a
->rd
, a
->rn
, a
->rm
, a
->ra
, 0);
8692 static bool do_BFMLAL_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
8694 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
8697 if (sve_access_check(s
)) {
8698 TCGv_ptr status
= fpstatus_ptr(FPST_FPCR
);
8699 unsigned vsz
= vec_full_reg_size(s
);
8701 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
8702 vec_full_reg_offset(s
, a
->rn
),
8703 vec_full_reg_offset(s
, a
->rm
),
8704 vec_full_reg_offset(s
, a
->ra
),
8705 status
, vsz
, vsz
, sel
,
8706 gen_helper_gvec_bfmlal
);
8707 tcg_temp_free_ptr(status
);
8712 static bool trans_BFMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8714 return do_BFMLAL_zzzw(s
, a
, false);
8717 static bool trans_BFMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8719 return do_BFMLAL_zzzw(s
, a
, true);
8722 static bool do_BFMLAL_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
, bool sel
)
8724 if (!dc_isar_feature(aa64_sve_bf16
, s
)) {
8727 if (sve_access_check(s
)) {
8728 TCGv_ptr status
= fpstatus_ptr(FPST_FPCR
);
8729 unsigned vsz
= vec_full_reg_size(s
);
8731 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
8732 vec_full_reg_offset(s
, a
->rn
),
8733 vec_full_reg_offset(s
, a
->rm
),
8734 vec_full_reg_offset(s
, a
->ra
),
8735 status
, vsz
, vsz
, (a
->index
<< 1) | sel
,
8736 gen_helper_gvec_bfmlal_idx
);
8737 tcg_temp_free_ptr(status
);
8742 static bool trans_BFMLALB_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
8744 return do_BFMLAL_zzxw(s
, a
, false);
8747 static bool trans_BFMLALT_zzxw(DisasContext
*s
, arg_rrxr_esz
*a
)
8749 return do_BFMLAL_zzxw(s
, a
, true);