2 * AArch64 SVE translation
4 * Copyright (c) 2018 Linaro, Ltd
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
22 #include "exec/exec-all.h"
23 #include "tcg/tcg-op.h"
24 #include "tcg/tcg-op-gvec.h"
25 #include "tcg/tcg-gvec-desc.h"
28 #include "translate.h"
29 #include "internals.h"
30 #include "exec/helper-proto.h"
31 #include "exec/helper-gen.h"
33 #include "trace-tcg.h"
34 #include "translate-a64.h"
35 #include "fpu/softfloat.h"
38 typedef void GVecGen2sFn(unsigned, uint32_t, uint32_t,
39 TCGv_i64
, uint32_t, uint32_t);
41 typedef void gen_helper_gvec_flags_3(TCGv_i32
, TCGv_ptr
, TCGv_ptr
,
43 typedef void gen_helper_gvec_flags_4(TCGv_i32
, TCGv_ptr
, TCGv_ptr
,
44 TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
46 typedef void gen_helper_gvec_mem(TCGv_env
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
47 typedef void gen_helper_gvec_mem_scatter(TCGv_env
, TCGv_ptr
, TCGv_ptr
,
48 TCGv_ptr
, TCGv_i64
, TCGv_i32
);
51 * Helpers for extracting complex instruction fields.
54 /* See e.g. ASR (immediate, predicated).
55 * Returns -1 for unallocated encoding; diagnose later.
57 static int tszimm_esz(DisasContext
*s
, int x
)
59 x
>>= 3; /* discard imm3 */
63 static int tszimm_shr(DisasContext
*s
, int x
)
65 return (16 << tszimm_esz(s
, x
)) - x
;
68 /* See e.g. LSL (immediate, predicated). */
69 static int tszimm_shl(DisasContext
*s
, int x
)
71 return x
- (8 << tszimm_esz(s
, x
));
74 static inline int plus1(DisasContext
*s
, int x
)
79 /* The SH bit is in bit 8. Extract the low 8 and shift. */
80 static inline int expand_imm_sh8s(DisasContext
*s
, int x
)
82 return (int8_t)x
<< (x
& 0x100 ? 8 : 0);
85 static inline int expand_imm_sh8u(DisasContext
*s
, int x
)
87 return (uint8_t)x
<< (x
& 0x100 ? 8 : 0);
90 /* Convert a 2-bit memory size (msz) to a 4-bit data type (dtype)
91 * with unsigned data. C.f. SVE Memory Contiguous Load Group.
93 static inline int msz_dtype(DisasContext
*s
, int msz
)
95 static const uint8_t dtype
[4] = { 0, 5, 10, 15 };
100 * Include the generated decoder.
103 #include "decode-sve.c.inc"
106 * Implement all of the translator functions referenced by the decoder.
109 /* Return the offset info CPUARMState of the predicate vector register Pn.
110 * Note for this purpose, FFR is P16.
112 static inline int pred_full_reg_offset(DisasContext
*s
, int regno
)
114 return offsetof(CPUARMState
, vfp
.pregs
[regno
]);
117 /* Return the byte size of the whole predicate register, VL / 64. */
118 static inline int pred_full_reg_size(DisasContext
*s
)
120 return s
->sve_len
>> 3;
123 /* Round up the size of a register to a size allowed by
124 * the tcg vector infrastructure. Any operation which uses this
125 * size may assume that the bits above pred_full_reg_size are zero,
126 * and must leave them the same way.
128 * Note that this is not needed for the vector registers as they
129 * are always properly sized for tcg vectors.
131 static int size_for_gvec(int size
)
136 return QEMU_ALIGN_UP(size
, 16);
140 static int pred_gvec_reg_size(DisasContext
*s
)
142 return size_for_gvec(pred_full_reg_size(s
));
145 /* Invoke an out-of-line helper on 2 Zregs. */
146 static void gen_gvec_ool_zz(DisasContext
*s
, gen_helper_gvec_2
*fn
,
147 int rd
, int rn
, int data
)
149 unsigned vsz
= vec_full_reg_size(s
);
150 tcg_gen_gvec_2_ool(vec_full_reg_offset(s
, rd
),
151 vec_full_reg_offset(s
, rn
),
155 /* Invoke an out-of-line helper on 3 Zregs. */
156 static void gen_gvec_ool_zzz(DisasContext
*s
, gen_helper_gvec_3
*fn
,
157 int rd
, int rn
, int rm
, int data
)
159 unsigned vsz
= vec_full_reg_size(s
);
160 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
161 vec_full_reg_offset(s
, rn
),
162 vec_full_reg_offset(s
, rm
),
166 /* Invoke an out-of-line helper on 4 Zregs. */
167 static void gen_gvec_ool_zzzz(DisasContext
*s
, gen_helper_gvec_4
*fn
,
168 int rd
, int rn
, int rm
, int ra
, int data
)
170 unsigned vsz
= vec_full_reg_size(s
);
171 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
172 vec_full_reg_offset(s
, rn
),
173 vec_full_reg_offset(s
, rm
),
174 vec_full_reg_offset(s
, ra
),
178 /* Invoke an out-of-line helper on 2 Zregs and a predicate. */
179 static void gen_gvec_ool_zzp(DisasContext
*s
, gen_helper_gvec_3
*fn
,
180 int rd
, int rn
, int pg
, int data
)
182 unsigned vsz
= vec_full_reg_size(s
);
183 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
184 vec_full_reg_offset(s
, rn
),
185 pred_full_reg_offset(s
, pg
),
189 /* Invoke an out-of-line helper on 3 Zregs and a predicate. */
190 static void gen_gvec_ool_zzzp(DisasContext
*s
, gen_helper_gvec_4
*fn
,
191 int rd
, int rn
, int rm
, int pg
, int data
)
193 unsigned vsz
= vec_full_reg_size(s
);
194 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
195 vec_full_reg_offset(s
, rn
),
196 vec_full_reg_offset(s
, rm
),
197 pred_full_reg_offset(s
, pg
),
201 /* Invoke a vector expander on two Zregs. */
202 static void gen_gvec_fn_zz(DisasContext
*s
, GVecGen2Fn
*gvec_fn
,
203 int esz
, int rd
, int rn
)
205 unsigned vsz
= vec_full_reg_size(s
);
206 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
207 vec_full_reg_offset(s
, rn
), vsz
, vsz
);
210 /* Invoke a vector expander on three Zregs. */
211 static void gen_gvec_fn_zzz(DisasContext
*s
, GVecGen3Fn
*gvec_fn
,
212 int esz
, int rd
, int rn
, int rm
)
214 unsigned vsz
= vec_full_reg_size(s
);
215 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
216 vec_full_reg_offset(s
, rn
),
217 vec_full_reg_offset(s
, rm
), vsz
, vsz
);
220 /* Invoke a vector expander on four Zregs. */
221 static void gen_gvec_fn_zzzz(DisasContext
*s
, GVecGen4Fn
*gvec_fn
,
222 int esz
, int rd
, int rn
, int rm
, int ra
)
224 unsigned vsz
= vec_full_reg_size(s
);
225 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
226 vec_full_reg_offset(s
, rn
),
227 vec_full_reg_offset(s
, rm
),
228 vec_full_reg_offset(s
, ra
), vsz
, vsz
);
231 /* Invoke a vector move on two Zregs. */
232 static bool do_mov_z(DisasContext
*s
, int rd
, int rn
)
234 if (sve_access_check(s
)) {
235 gen_gvec_fn_zz(s
, tcg_gen_gvec_mov
, MO_8
, rd
, rn
);
240 /* Initialize a Zreg with replications of a 64-bit immediate. */
241 static void do_dupi_z(DisasContext
*s
, int rd
, uint64_t word
)
243 unsigned vsz
= vec_full_reg_size(s
);
244 tcg_gen_gvec_dup_imm(MO_64
, vec_full_reg_offset(s
, rd
), vsz
, vsz
, word
);
247 /* Invoke a vector expander on three Pregs. */
248 static void gen_gvec_fn_ppp(DisasContext
*s
, GVecGen3Fn
*gvec_fn
,
249 int rd
, int rn
, int rm
)
251 unsigned psz
= pred_gvec_reg_size(s
);
252 gvec_fn(MO_64
, pred_full_reg_offset(s
, rd
),
253 pred_full_reg_offset(s
, rn
),
254 pred_full_reg_offset(s
, rm
), psz
, psz
);
257 /* Invoke a vector move on two Pregs. */
258 static bool do_mov_p(DisasContext
*s
, int rd
, int rn
)
260 if (sve_access_check(s
)) {
261 unsigned psz
= pred_gvec_reg_size(s
);
262 tcg_gen_gvec_mov(MO_8
, pred_full_reg_offset(s
, rd
),
263 pred_full_reg_offset(s
, rn
), psz
, psz
);
268 /* Set the cpu flags as per a return from an SVE helper. */
269 static void do_pred_flags(TCGv_i32 t
)
271 tcg_gen_mov_i32(cpu_NF
, t
);
272 tcg_gen_andi_i32(cpu_ZF
, t
, 2);
273 tcg_gen_andi_i32(cpu_CF
, t
, 1);
274 tcg_gen_movi_i32(cpu_VF
, 0);
277 /* Subroutines computing the ARM PredTest psuedofunction. */
278 static void do_predtest1(TCGv_i64 d
, TCGv_i64 g
)
280 TCGv_i32 t
= tcg_temp_new_i32();
282 gen_helper_sve_predtest1(t
, d
, g
);
284 tcg_temp_free_i32(t
);
287 static void do_predtest(DisasContext
*s
, int dofs
, int gofs
, int words
)
289 TCGv_ptr dptr
= tcg_temp_new_ptr();
290 TCGv_ptr gptr
= tcg_temp_new_ptr();
293 tcg_gen_addi_ptr(dptr
, cpu_env
, dofs
);
294 tcg_gen_addi_ptr(gptr
, cpu_env
, gofs
);
295 t
= tcg_const_i32(words
);
297 gen_helper_sve_predtest(t
, dptr
, gptr
, t
);
298 tcg_temp_free_ptr(dptr
);
299 tcg_temp_free_ptr(gptr
);
302 tcg_temp_free_i32(t
);
305 /* For each element size, the bits within a predicate word that are active. */
306 const uint64_t pred_esz_masks
[4] = {
307 0xffffffffffffffffull
, 0x5555555555555555ull
,
308 0x1111111111111111ull
, 0x0101010101010101ull
312 *** SVE Logical - Unpredicated Group
315 static bool do_zzz_fn(DisasContext
*s
, arg_rrr_esz
*a
, GVecGen3Fn
*gvec_fn
)
317 if (sve_access_check(s
)) {
318 gen_gvec_fn_zzz(s
, gvec_fn
, a
->esz
, a
->rd
, a
->rn
, a
->rm
);
323 static bool trans_AND_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
325 return do_zzz_fn(s
, a
, tcg_gen_gvec_and
);
328 static bool trans_ORR_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
330 return do_zzz_fn(s
, a
, tcg_gen_gvec_or
);
333 static bool trans_EOR_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
335 return do_zzz_fn(s
, a
, tcg_gen_gvec_xor
);
338 static bool trans_BIC_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
340 return do_zzz_fn(s
, a
, tcg_gen_gvec_andc
);
343 static void gen_xar8_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
345 TCGv_i64 t
= tcg_temp_new_i64();
346 uint64_t mask
= dup_const(MO_8
, 0xff >> sh
);
348 tcg_gen_xor_i64(t
, n
, m
);
349 tcg_gen_shri_i64(d
, t
, sh
);
350 tcg_gen_shli_i64(t
, t
, 8 - sh
);
351 tcg_gen_andi_i64(d
, d
, mask
);
352 tcg_gen_andi_i64(t
, t
, ~mask
);
353 tcg_gen_or_i64(d
, d
, t
);
354 tcg_temp_free_i64(t
);
357 static void gen_xar16_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
359 TCGv_i64 t
= tcg_temp_new_i64();
360 uint64_t mask
= dup_const(MO_16
, 0xffff >> sh
);
362 tcg_gen_xor_i64(t
, n
, m
);
363 tcg_gen_shri_i64(d
, t
, sh
);
364 tcg_gen_shli_i64(t
, t
, 16 - sh
);
365 tcg_gen_andi_i64(d
, d
, mask
);
366 tcg_gen_andi_i64(t
, t
, ~mask
);
367 tcg_gen_or_i64(d
, d
, t
);
368 tcg_temp_free_i64(t
);
371 static void gen_xar_i32(TCGv_i32 d
, TCGv_i32 n
, TCGv_i32 m
, int32_t sh
)
373 tcg_gen_xor_i32(d
, n
, m
);
374 tcg_gen_rotri_i32(d
, d
, sh
);
377 static void gen_xar_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
379 tcg_gen_xor_i64(d
, n
, m
);
380 tcg_gen_rotri_i64(d
, d
, sh
);
383 static void gen_xar_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
384 TCGv_vec m
, int64_t sh
)
386 tcg_gen_xor_vec(vece
, d
, n
, m
);
387 tcg_gen_rotri_vec(vece
, d
, d
, sh
);
390 void gen_gvec_xar(unsigned vece
, uint32_t rd_ofs
, uint32_t rn_ofs
,
391 uint32_t rm_ofs
, int64_t shift
,
392 uint32_t opr_sz
, uint32_t max_sz
)
394 static const TCGOpcode vecop
[] = { INDEX_op_rotli_vec
, 0 };
395 static const GVecGen3i ops
[4] = {
396 { .fni8
= gen_xar8_i64
,
398 .fno
= gen_helper_sve2_xar_b
,
401 { .fni8
= gen_xar16_i64
,
403 .fno
= gen_helper_sve2_xar_h
,
406 { .fni4
= gen_xar_i32
,
408 .fno
= gen_helper_sve2_xar_s
,
411 { .fni8
= gen_xar_i64
,
413 .fno
= gen_helper_gvec_xar_d
,
417 int esize
= 8 << vece
;
419 /* The SVE2 range is 1 .. esize; the AdvSIMD range is 0 .. esize-1. */
420 tcg_debug_assert(shift
>= 0);
421 tcg_debug_assert(shift
<= esize
);
425 /* xar with no rotate devolves to xor. */
426 tcg_gen_gvec_xor(vece
, rd_ofs
, rn_ofs
, rm_ofs
, opr_sz
, max_sz
);
428 tcg_gen_gvec_3i(rd_ofs
, rn_ofs
, rm_ofs
, opr_sz
, max_sz
,
433 static bool trans_XAR(DisasContext
*s
, arg_rrri_esz
*a
)
435 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
438 if (sve_access_check(s
)) {
439 unsigned vsz
= vec_full_reg_size(s
);
440 gen_gvec_xar(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
441 vec_full_reg_offset(s
, a
->rn
),
442 vec_full_reg_offset(s
, a
->rm
), a
->imm
, vsz
, vsz
);
447 static bool do_sve2_zzzz_fn(DisasContext
*s
, arg_rrrr_esz
*a
, GVecGen4Fn
*fn
)
449 if (!dc_isar_feature(aa64_sve2
, s
)) {
452 if (sve_access_check(s
)) {
453 gen_gvec_fn_zzzz(s
, fn
, a
->esz
, a
->rd
, a
->rn
, a
->rm
, a
->ra
);
458 static void gen_eor3_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
460 tcg_gen_xor_i64(d
, n
, m
);
461 tcg_gen_xor_i64(d
, d
, k
);
464 static void gen_eor3_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
465 TCGv_vec m
, TCGv_vec k
)
467 tcg_gen_xor_vec(vece
, d
, n
, m
);
468 tcg_gen_xor_vec(vece
, d
, d
, k
);
471 static void gen_eor3(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
472 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
474 static const GVecGen4 op
= {
475 .fni8
= gen_eor3_i64
,
476 .fniv
= gen_eor3_vec
,
477 .fno
= gen_helper_sve2_eor3
,
479 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
481 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
484 static bool trans_EOR3(DisasContext
*s
, arg_rrrr_esz
*a
)
486 return do_sve2_zzzz_fn(s
, a
, gen_eor3
);
489 static void gen_bcax_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
491 tcg_gen_andc_i64(d
, m
, k
);
492 tcg_gen_xor_i64(d
, d
, n
);
495 static void gen_bcax_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
496 TCGv_vec m
, TCGv_vec k
)
498 tcg_gen_andc_vec(vece
, d
, m
, k
);
499 tcg_gen_xor_vec(vece
, d
, d
, n
);
502 static void gen_bcax(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
503 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
505 static const GVecGen4 op
= {
506 .fni8
= gen_bcax_i64
,
507 .fniv
= gen_bcax_vec
,
508 .fno
= gen_helper_sve2_bcax
,
510 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
512 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
515 static bool trans_BCAX(DisasContext
*s
, arg_rrrr_esz
*a
)
517 return do_sve2_zzzz_fn(s
, a
, gen_bcax
);
520 static void gen_bsl(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
521 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
523 /* BSL differs from the generic bitsel in argument ordering. */
524 tcg_gen_gvec_bitsel(vece
, d
, a
, n
, m
, oprsz
, maxsz
);
527 static bool trans_BSL(DisasContext
*s
, arg_rrrr_esz
*a
)
529 return do_sve2_zzzz_fn(s
, a
, gen_bsl
);
532 static void gen_bsl1n_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
534 tcg_gen_andc_i64(n
, k
, n
);
535 tcg_gen_andc_i64(m
, m
, k
);
536 tcg_gen_or_i64(d
, n
, m
);
539 static void gen_bsl1n_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
540 TCGv_vec m
, TCGv_vec k
)
542 if (TCG_TARGET_HAS_bitsel_vec
) {
543 tcg_gen_not_vec(vece
, n
, n
);
544 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
546 tcg_gen_andc_vec(vece
, n
, k
, n
);
547 tcg_gen_andc_vec(vece
, m
, m
, k
);
548 tcg_gen_or_vec(vece
, d
, n
, m
);
552 static void gen_bsl1n(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
553 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
555 static const GVecGen4 op
= {
556 .fni8
= gen_bsl1n_i64
,
557 .fniv
= gen_bsl1n_vec
,
558 .fno
= gen_helper_sve2_bsl1n
,
560 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
562 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
565 static bool trans_BSL1N(DisasContext
*s
, arg_rrrr_esz
*a
)
567 return do_sve2_zzzz_fn(s
, a
, gen_bsl1n
);
570 static void gen_bsl2n_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
573 * Z[dn] = (n & k) | (~m & ~k)
576 tcg_gen_and_i64(n
, n
, k
);
577 if (TCG_TARGET_HAS_orc_i64
) {
578 tcg_gen_or_i64(m
, m
, k
);
579 tcg_gen_orc_i64(d
, n
, m
);
581 tcg_gen_nor_i64(m
, m
, k
);
582 tcg_gen_or_i64(d
, n
, m
);
586 static void gen_bsl2n_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
587 TCGv_vec m
, TCGv_vec k
)
589 if (TCG_TARGET_HAS_bitsel_vec
) {
590 tcg_gen_not_vec(vece
, m
, m
);
591 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
593 tcg_gen_and_vec(vece
, n
, n
, k
);
594 tcg_gen_or_vec(vece
, m
, m
, k
);
595 tcg_gen_orc_vec(vece
, d
, n
, m
);
599 static void gen_bsl2n(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
600 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
602 static const GVecGen4 op
= {
603 .fni8
= gen_bsl2n_i64
,
604 .fniv
= gen_bsl2n_vec
,
605 .fno
= gen_helper_sve2_bsl2n
,
607 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
609 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
612 static bool trans_BSL2N(DisasContext
*s
, arg_rrrr_esz
*a
)
614 return do_sve2_zzzz_fn(s
, a
, gen_bsl2n
);
617 static void gen_nbsl_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
619 tcg_gen_and_i64(n
, n
, k
);
620 tcg_gen_andc_i64(m
, m
, k
);
621 tcg_gen_nor_i64(d
, n
, m
);
624 static void gen_nbsl_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
625 TCGv_vec m
, TCGv_vec k
)
627 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
628 tcg_gen_not_vec(vece
, d
, d
);
631 static void gen_nbsl(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
632 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
634 static const GVecGen4 op
= {
635 .fni8
= gen_nbsl_i64
,
636 .fniv
= gen_nbsl_vec
,
637 .fno
= gen_helper_sve2_nbsl
,
639 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
641 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
644 static bool trans_NBSL(DisasContext
*s
, arg_rrrr_esz
*a
)
646 return do_sve2_zzzz_fn(s
, a
, gen_nbsl
);
650 *** SVE Integer Arithmetic - Unpredicated Group
653 static bool trans_ADD_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
655 return do_zzz_fn(s
, a
, tcg_gen_gvec_add
);
658 static bool trans_SUB_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
660 return do_zzz_fn(s
, a
, tcg_gen_gvec_sub
);
663 static bool trans_SQADD_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
665 return do_zzz_fn(s
, a
, tcg_gen_gvec_ssadd
);
668 static bool trans_SQSUB_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
670 return do_zzz_fn(s
, a
, tcg_gen_gvec_sssub
);
673 static bool trans_UQADD_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
675 return do_zzz_fn(s
, a
, tcg_gen_gvec_usadd
);
678 static bool trans_UQSUB_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
680 return do_zzz_fn(s
, a
, tcg_gen_gvec_ussub
);
684 *** SVE Integer Arithmetic - Binary Predicated Group
687 static bool do_zpzz_ool(DisasContext
*s
, arg_rprr_esz
*a
, gen_helper_gvec_4
*fn
)
692 if (sve_access_check(s
)) {
693 gen_gvec_ool_zzzp(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->pg
, 0);
698 /* Select active elememnts from Zn and inactive elements from Zm,
699 * storing the result in Zd.
701 static void do_sel_z(DisasContext
*s
, int rd
, int rn
, int rm
, int pg
, int esz
)
703 static gen_helper_gvec_4
* const fns
[4] = {
704 gen_helper_sve_sel_zpzz_b
, gen_helper_sve_sel_zpzz_h
,
705 gen_helper_sve_sel_zpzz_s
, gen_helper_sve_sel_zpzz_d
707 gen_gvec_ool_zzzp(s
, fns
[esz
], rd
, rn
, rm
, pg
, 0);
710 #define DO_ZPZZ(NAME, name) \
711 static bool trans_##NAME##_zpzz(DisasContext *s, arg_rprr_esz *a) \
713 static gen_helper_gvec_4 * const fns[4] = { \
714 gen_helper_sve_##name##_zpzz_b, gen_helper_sve_##name##_zpzz_h, \
715 gen_helper_sve_##name##_zpzz_s, gen_helper_sve_##name##_zpzz_d, \
717 return do_zpzz_ool(s, a, fns[a->esz]); \
736 DO_ZPZZ(SMULH
, smulh
)
737 DO_ZPZZ(UMULH
, umulh
)
743 static bool trans_SDIV_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
745 static gen_helper_gvec_4
* const fns
[4] = {
746 NULL
, NULL
, gen_helper_sve_sdiv_zpzz_s
, gen_helper_sve_sdiv_zpzz_d
748 return do_zpzz_ool(s
, a
, fns
[a
->esz
]);
751 static bool trans_UDIV_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
753 static gen_helper_gvec_4
* const fns
[4] = {
754 NULL
, NULL
, gen_helper_sve_udiv_zpzz_s
, gen_helper_sve_udiv_zpzz_d
756 return do_zpzz_ool(s
, a
, fns
[a
->esz
]);
759 static bool trans_SEL_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
761 if (sve_access_check(s
)) {
762 do_sel_z(s
, a
->rd
, a
->rn
, a
->rm
, a
->pg
, a
->esz
);
770 *** SVE Integer Arithmetic - Unary Predicated Group
773 static bool do_zpz_ool(DisasContext
*s
, arg_rpr_esz
*a
, gen_helper_gvec_3
*fn
)
778 if (sve_access_check(s
)) {
779 gen_gvec_ool_zzp(s
, fn
, a
->rd
, a
->rn
, a
->pg
, 0);
784 #define DO_ZPZ(NAME, name) \
785 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
787 static gen_helper_gvec_3 * const fns[4] = { \
788 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
789 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
791 return do_zpz_ool(s, a, fns[a->esz]); \
796 DO_ZPZ(CNT_zpz
, cnt_zpz
)
798 DO_ZPZ(NOT_zpz
, not_zpz
)
802 static bool trans_FABS(DisasContext
*s
, arg_rpr_esz
*a
)
804 static gen_helper_gvec_3
* const fns
[4] = {
806 gen_helper_sve_fabs_h
,
807 gen_helper_sve_fabs_s
,
808 gen_helper_sve_fabs_d
810 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
813 static bool trans_FNEG(DisasContext
*s
, arg_rpr_esz
*a
)
815 static gen_helper_gvec_3
* const fns
[4] = {
817 gen_helper_sve_fneg_h
,
818 gen_helper_sve_fneg_s
,
819 gen_helper_sve_fneg_d
821 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
824 static bool trans_SXTB(DisasContext
*s
, arg_rpr_esz
*a
)
826 static gen_helper_gvec_3
* const fns
[4] = {
828 gen_helper_sve_sxtb_h
,
829 gen_helper_sve_sxtb_s
,
830 gen_helper_sve_sxtb_d
832 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
835 static bool trans_UXTB(DisasContext
*s
, arg_rpr_esz
*a
)
837 static gen_helper_gvec_3
* const fns
[4] = {
839 gen_helper_sve_uxtb_h
,
840 gen_helper_sve_uxtb_s
,
841 gen_helper_sve_uxtb_d
843 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
846 static bool trans_SXTH(DisasContext
*s
, arg_rpr_esz
*a
)
848 static gen_helper_gvec_3
* const fns
[4] = {
850 gen_helper_sve_sxth_s
,
851 gen_helper_sve_sxth_d
853 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
856 static bool trans_UXTH(DisasContext
*s
, arg_rpr_esz
*a
)
858 static gen_helper_gvec_3
* const fns
[4] = {
860 gen_helper_sve_uxth_s
,
861 gen_helper_sve_uxth_d
863 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
866 static bool trans_SXTW(DisasContext
*s
, arg_rpr_esz
*a
)
868 return do_zpz_ool(s
, a
, a
->esz
== 3 ? gen_helper_sve_sxtw_d
: NULL
);
871 static bool trans_UXTW(DisasContext
*s
, arg_rpr_esz
*a
)
873 return do_zpz_ool(s
, a
, a
->esz
== 3 ? gen_helper_sve_uxtw_d
: NULL
);
879 *** SVE Integer Reduction Group
882 typedef void gen_helper_gvec_reduc(TCGv_i64
, TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
883 static bool do_vpz_ool(DisasContext
*s
, arg_rpr_esz
*a
,
884 gen_helper_gvec_reduc
*fn
)
886 unsigned vsz
= vec_full_reg_size(s
);
894 if (!sve_access_check(s
)) {
898 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
899 temp
= tcg_temp_new_i64();
900 t_zn
= tcg_temp_new_ptr();
901 t_pg
= tcg_temp_new_ptr();
903 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
904 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
905 fn(temp
, t_zn
, t_pg
, desc
);
906 tcg_temp_free_ptr(t_zn
);
907 tcg_temp_free_ptr(t_pg
);
908 tcg_temp_free_i32(desc
);
910 write_fp_dreg(s
, a
->rd
, temp
);
911 tcg_temp_free_i64(temp
);
915 #define DO_VPZ(NAME, name) \
916 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
918 static gen_helper_gvec_reduc * const fns[4] = { \
919 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
920 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
922 return do_vpz_ool(s, a, fns[a->esz]); \
935 static bool trans_SADDV(DisasContext
*s
, arg_rpr_esz
*a
)
937 static gen_helper_gvec_reduc
* const fns
[4] = {
938 gen_helper_sve_saddv_b
, gen_helper_sve_saddv_h
,
939 gen_helper_sve_saddv_s
, NULL
941 return do_vpz_ool(s
, a
, fns
[a
->esz
]);
947 *** SVE Shift by Immediate - Predicated Group
951 * Copy Zn into Zd, storing zeros into inactive elements.
952 * If invert, store zeros into the active elements.
954 static bool do_movz_zpz(DisasContext
*s
, int rd
, int rn
, int pg
,
955 int esz
, bool invert
)
957 static gen_helper_gvec_3
* const fns
[4] = {
958 gen_helper_sve_movz_b
, gen_helper_sve_movz_h
,
959 gen_helper_sve_movz_s
, gen_helper_sve_movz_d
,
962 if (sve_access_check(s
)) {
963 gen_gvec_ool_zzp(s
, fns
[esz
], rd
, rn
, pg
, invert
);
968 static bool do_zpzi_ool(DisasContext
*s
, arg_rpri_esz
*a
,
969 gen_helper_gvec_3
*fn
)
971 if (sve_access_check(s
)) {
972 gen_gvec_ool_zzp(s
, fn
, a
->rd
, a
->rn
, a
->pg
, a
->imm
);
977 static bool trans_ASR_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
979 static gen_helper_gvec_3
* const fns
[4] = {
980 gen_helper_sve_asr_zpzi_b
, gen_helper_sve_asr_zpzi_h
,
981 gen_helper_sve_asr_zpzi_s
, gen_helper_sve_asr_zpzi_d
,
984 /* Invalid tsz encoding -- see tszimm_esz. */
987 /* Shift by element size is architecturally valid. For
988 arithmetic right-shift, it's the same as by one less. */
989 a
->imm
= MIN(a
->imm
, (8 << a
->esz
) - 1);
990 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
993 static bool trans_LSR_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
995 static gen_helper_gvec_3
* const fns
[4] = {
996 gen_helper_sve_lsr_zpzi_b
, gen_helper_sve_lsr_zpzi_h
,
997 gen_helper_sve_lsr_zpzi_s
, gen_helper_sve_lsr_zpzi_d
,
1002 /* Shift by element size is architecturally valid.
1003 For logical shifts, it is a zeroing operation. */
1004 if (a
->imm
>= (8 << a
->esz
)) {
1005 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
1007 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1011 static bool trans_LSL_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
1013 static gen_helper_gvec_3
* const fns
[4] = {
1014 gen_helper_sve_lsl_zpzi_b
, gen_helper_sve_lsl_zpzi_h
,
1015 gen_helper_sve_lsl_zpzi_s
, gen_helper_sve_lsl_zpzi_d
,
1020 /* Shift by element size is architecturally valid.
1021 For logical shifts, it is a zeroing operation. */
1022 if (a
->imm
>= (8 << a
->esz
)) {
1023 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
1025 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1029 static bool trans_ASRD(DisasContext
*s
, arg_rpri_esz
*a
)
1031 static gen_helper_gvec_3
* const fns
[4] = {
1032 gen_helper_sve_asrd_b
, gen_helper_sve_asrd_h
,
1033 gen_helper_sve_asrd_s
, gen_helper_sve_asrd_d
,
1038 /* Shift by element size is architecturally valid. For arithmetic
1039 right shift for division, it is a zeroing operation. */
1040 if (a
->imm
>= (8 << a
->esz
)) {
1041 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
1043 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1048 *** SVE Bitwise Shift - Predicated Group
1051 #define DO_ZPZW(NAME, name) \
1052 static bool trans_##NAME##_zpzw(DisasContext *s, arg_rprr_esz *a) \
1054 static gen_helper_gvec_4 * const fns[3] = { \
1055 gen_helper_sve_##name##_zpzw_b, gen_helper_sve_##name##_zpzw_h, \
1056 gen_helper_sve_##name##_zpzw_s, \
1058 if (a->esz < 0 || a->esz >= 3) { \
1061 return do_zpzz_ool(s, a, fns[a->esz]); \
1071 *** SVE Bitwise Shift - Unpredicated Group
1074 static bool do_shift_imm(DisasContext
*s
, arg_rri_esz
*a
, bool asr
,
1075 void (*gvec_fn
)(unsigned, uint32_t, uint32_t,
1076 int64_t, uint32_t, uint32_t))
1079 /* Invalid tsz encoding -- see tszimm_esz. */
1082 if (sve_access_check(s
)) {
1083 unsigned vsz
= vec_full_reg_size(s
);
1084 /* Shift by element size is architecturally valid. For
1085 arithmetic right-shift, it's the same as by one less.
1086 Otherwise it is a zeroing operation. */
1087 if (a
->imm
>= 8 << a
->esz
) {
1089 a
->imm
= (8 << a
->esz
) - 1;
1091 do_dupi_z(s
, a
->rd
, 0);
1095 gvec_fn(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
1096 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
1101 static bool trans_ASR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
1103 return do_shift_imm(s
, a
, true, tcg_gen_gvec_sari
);
1106 static bool trans_LSR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
1108 return do_shift_imm(s
, a
, false, tcg_gen_gvec_shri
);
1111 static bool trans_LSL_zzi(DisasContext
*s
, arg_rri_esz
*a
)
1113 return do_shift_imm(s
, a
, false, tcg_gen_gvec_shli
);
1116 static bool do_zzw_ool(DisasContext
*s
, arg_rrr_esz
*a
, gen_helper_gvec_3
*fn
)
1121 if (sve_access_check(s
)) {
1122 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, 0);
1127 #define DO_ZZW(NAME, name) \
1128 static bool trans_##NAME##_zzw(DisasContext *s, arg_rrr_esz *a) \
1130 static gen_helper_gvec_3 * const fns[4] = { \
1131 gen_helper_sve_##name##_zzw_b, gen_helper_sve_##name##_zzw_h, \
1132 gen_helper_sve_##name##_zzw_s, NULL \
1134 return do_zzw_ool(s, a, fns[a->esz]); \
1144 *** SVE Integer Multiply-Add Group
1147 static bool do_zpzzz_ool(DisasContext
*s
, arg_rprrr_esz
*a
,
1148 gen_helper_gvec_5
*fn
)
1150 if (sve_access_check(s
)) {
1151 unsigned vsz
= vec_full_reg_size(s
);
1152 tcg_gen_gvec_5_ool(vec_full_reg_offset(s
, a
->rd
),
1153 vec_full_reg_offset(s
, a
->ra
),
1154 vec_full_reg_offset(s
, a
->rn
),
1155 vec_full_reg_offset(s
, a
->rm
),
1156 pred_full_reg_offset(s
, a
->pg
),
1162 #define DO_ZPZZZ(NAME, name) \
1163 static bool trans_##NAME(DisasContext *s, arg_rprrr_esz *a) \
1165 static gen_helper_gvec_5 * const fns[4] = { \
1166 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
1167 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
1169 return do_zpzzz_ool(s, a, fns[a->esz]); \
1178 *** SVE Index Generation Group
1181 static void do_index(DisasContext
*s
, int esz
, int rd
,
1182 TCGv_i64 start
, TCGv_i64 incr
)
1184 unsigned vsz
= vec_full_reg_size(s
);
1185 TCGv_i32 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
1186 TCGv_ptr t_zd
= tcg_temp_new_ptr();
1188 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, rd
));
1190 gen_helper_sve_index_d(t_zd
, start
, incr
, desc
);
1192 typedef void index_fn(TCGv_ptr
, TCGv_i32
, TCGv_i32
, TCGv_i32
);
1193 static index_fn
* const fns
[3] = {
1194 gen_helper_sve_index_b
,
1195 gen_helper_sve_index_h
,
1196 gen_helper_sve_index_s
,
1198 TCGv_i32 s32
= tcg_temp_new_i32();
1199 TCGv_i32 i32
= tcg_temp_new_i32();
1201 tcg_gen_extrl_i64_i32(s32
, start
);
1202 tcg_gen_extrl_i64_i32(i32
, incr
);
1203 fns
[esz
](t_zd
, s32
, i32
, desc
);
1205 tcg_temp_free_i32(s32
);
1206 tcg_temp_free_i32(i32
);
1208 tcg_temp_free_ptr(t_zd
);
1209 tcg_temp_free_i32(desc
);
1212 static bool trans_INDEX_ii(DisasContext
*s
, arg_INDEX_ii
*a
)
1214 if (sve_access_check(s
)) {
1215 TCGv_i64 start
= tcg_const_i64(a
->imm1
);
1216 TCGv_i64 incr
= tcg_const_i64(a
->imm2
);
1217 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1218 tcg_temp_free_i64(start
);
1219 tcg_temp_free_i64(incr
);
1224 static bool trans_INDEX_ir(DisasContext
*s
, arg_INDEX_ir
*a
)
1226 if (sve_access_check(s
)) {
1227 TCGv_i64 start
= tcg_const_i64(a
->imm
);
1228 TCGv_i64 incr
= cpu_reg(s
, a
->rm
);
1229 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1230 tcg_temp_free_i64(start
);
1235 static bool trans_INDEX_ri(DisasContext
*s
, arg_INDEX_ri
*a
)
1237 if (sve_access_check(s
)) {
1238 TCGv_i64 start
= cpu_reg(s
, a
->rn
);
1239 TCGv_i64 incr
= tcg_const_i64(a
->imm
);
1240 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1241 tcg_temp_free_i64(incr
);
1246 static bool trans_INDEX_rr(DisasContext
*s
, arg_INDEX_rr
*a
)
1248 if (sve_access_check(s
)) {
1249 TCGv_i64 start
= cpu_reg(s
, a
->rn
);
1250 TCGv_i64 incr
= cpu_reg(s
, a
->rm
);
1251 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1257 *** SVE Stack Allocation Group
1260 static bool trans_ADDVL(DisasContext
*s
, arg_ADDVL
*a
)
1262 if (sve_access_check(s
)) {
1263 TCGv_i64 rd
= cpu_reg_sp(s
, a
->rd
);
1264 TCGv_i64 rn
= cpu_reg_sp(s
, a
->rn
);
1265 tcg_gen_addi_i64(rd
, rn
, a
->imm
* vec_full_reg_size(s
));
1270 static bool trans_ADDPL(DisasContext
*s
, arg_ADDPL
*a
)
1272 if (sve_access_check(s
)) {
1273 TCGv_i64 rd
= cpu_reg_sp(s
, a
->rd
);
1274 TCGv_i64 rn
= cpu_reg_sp(s
, a
->rn
);
1275 tcg_gen_addi_i64(rd
, rn
, a
->imm
* pred_full_reg_size(s
));
1280 static bool trans_RDVL(DisasContext
*s
, arg_RDVL
*a
)
1282 if (sve_access_check(s
)) {
1283 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
1284 tcg_gen_movi_i64(reg
, a
->imm
* vec_full_reg_size(s
));
1290 *** SVE Compute Vector Address Group
1293 static bool do_adr(DisasContext
*s
, arg_rrri
*a
, gen_helper_gvec_3
*fn
)
1295 if (sve_access_check(s
)) {
1296 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->imm
);
1301 static bool trans_ADR_p32(DisasContext
*s
, arg_rrri
*a
)
1303 return do_adr(s
, a
, gen_helper_sve_adr_p32
);
1306 static bool trans_ADR_p64(DisasContext
*s
, arg_rrri
*a
)
1308 return do_adr(s
, a
, gen_helper_sve_adr_p64
);
1311 static bool trans_ADR_s32(DisasContext
*s
, arg_rrri
*a
)
1313 return do_adr(s
, a
, gen_helper_sve_adr_s32
);
1316 static bool trans_ADR_u32(DisasContext
*s
, arg_rrri
*a
)
1318 return do_adr(s
, a
, gen_helper_sve_adr_u32
);
1322 *** SVE Integer Misc - Unpredicated Group
1325 static bool trans_FEXPA(DisasContext
*s
, arg_rr_esz
*a
)
1327 static gen_helper_gvec_2
* const fns
[4] = {
1329 gen_helper_sve_fexpa_h
,
1330 gen_helper_sve_fexpa_s
,
1331 gen_helper_sve_fexpa_d
,
1336 if (sve_access_check(s
)) {
1337 gen_gvec_ool_zz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, 0);
1342 static bool trans_FTSSEL(DisasContext
*s
, arg_rrr_esz
*a
)
1344 static gen_helper_gvec_3
* const fns
[4] = {
1346 gen_helper_sve_ftssel_h
,
1347 gen_helper_sve_ftssel_s
,
1348 gen_helper_sve_ftssel_d
,
1353 if (sve_access_check(s
)) {
1354 gen_gvec_ool_zzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, 0);
1360 *** SVE Predicate Logical Operations Group
1363 static bool do_pppp_flags(DisasContext
*s
, arg_rprr_s
*a
,
1364 const GVecGen4
*gvec_op
)
1366 if (!sve_access_check(s
)) {
1370 unsigned psz
= pred_gvec_reg_size(s
);
1371 int dofs
= pred_full_reg_offset(s
, a
->rd
);
1372 int nofs
= pred_full_reg_offset(s
, a
->rn
);
1373 int mofs
= pred_full_reg_offset(s
, a
->rm
);
1374 int gofs
= pred_full_reg_offset(s
, a
->pg
);
1377 tcg_gen_gvec_4(dofs
, nofs
, mofs
, gofs
, psz
, psz
, gvec_op
);
1382 /* Do the operation and the flags generation in temps. */
1383 TCGv_i64 pd
= tcg_temp_new_i64();
1384 TCGv_i64 pn
= tcg_temp_new_i64();
1385 TCGv_i64 pm
= tcg_temp_new_i64();
1386 TCGv_i64 pg
= tcg_temp_new_i64();
1388 tcg_gen_ld_i64(pn
, cpu_env
, nofs
);
1389 tcg_gen_ld_i64(pm
, cpu_env
, mofs
);
1390 tcg_gen_ld_i64(pg
, cpu_env
, gofs
);
1392 gvec_op
->fni8(pd
, pn
, pm
, pg
);
1393 tcg_gen_st_i64(pd
, cpu_env
, dofs
);
1395 do_predtest1(pd
, pg
);
1397 tcg_temp_free_i64(pd
);
1398 tcg_temp_free_i64(pn
);
1399 tcg_temp_free_i64(pm
);
1400 tcg_temp_free_i64(pg
);
1402 /* The operation and flags generation is large. The computation
1403 * of the flags depends on the original contents of the guarding
1404 * predicate. If the destination overwrites the guarding predicate,
1405 * then the easiest way to get this right is to save a copy.
1408 if (a
->rd
== a
->pg
) {
1409 tofs
= offsetof(CPUARMState
, vfp
.preg_tmp
);
1410 tcg_gen_gvec_mov(0, tofs
, gofs
, psz
, psz
);
1413 tcg_gen_gvec_4(dofs
, nofs
, mofs
, gofs
, psz
, psz
, gvec_op
);
1414 do_predtest(s
, dofs
, tofs
, psz
/ 8);
1419 static void gen_and_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1421 tcg_gen_and_i64(pd
, pn
, pm
);
1422 tcg_gen_and_i64(pd
, pd
, pg
);
1425 static void gen_and_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1426 TCGv_vec pm
, TCGv_vec pg
)
1428 tcg_gen_and_vec(vece
, pd
, pn
, pm
);
1429 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1432 static bool trans_AND_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1434 static const GVecGen4 op
= {
1435 .fni8
= gen_and_pg_i64
,
1436 .fniv
= gen_and_pg_vec
,
1437 .fno
= gen_helper_sve_and_pppp
,
1438 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1442 if (!sve_access_check(s
)) {
1445 if (a
->rn
== a
->rm
) {
1446 if (a
->pg
== a
->rn
) {
1447 do_mov_p(s
, a
->rd
, a
->rn
);
1449 gen_gvec_fn_ppp(s
, tcg_gen_gvec_and
, a
->rd
, a
->rn
, a
->pg
);
1452 } else if (a
->pg
== a
->rn
|| a
->pg
== a
->rm
) {
1453 gen_gvec_fn_ppp(s
, tcg_gen_gvec_and
, a
->rd
, a
->rn
, a
->rm
);
1457 return do_pppp_flags(s
, a
, &op
);
1460 static void gen_bic_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1462 tcg_gen_andc_i64(pd
, pn
, pm
);
1463 tcg_gen_and_i64(pd
, pd
, pg
);
1466 static void gen_bic_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1467 TCGv_vec pm
, TCGv_vec pg
)
1469 tcg_gen_andc_vec(vece
, pd
, pn
, pm
);
1470 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1473 static bool trans_BIC_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1475 static const GVecGen4 op
= {
1476 .fni8
= gen_bic_pg_i64
,
1477 .fniv
= gen_bic_pg_vec
,
1478 .fno
= gen_helper_sve_bic_pppp
,
1479 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1482 if (!a
->s
&& a
->pg
== a
->rn
) {
1483 if (sve_access_check(s
)) {
1484 gen_gvec_fn_ppp(s
, tcg_gen_gvec_andc
, a
->rd
, a
->rn
, a
->rm
);
1488 return do_pppp_flags(s
, a
, &op
);
1491 static void gen_eor_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1493 tcg_gen_xor_i64(pd
, pn
, pm
);
1494 tcg_gen_and_i64(pd
, pd
, pg
);
1497 static void gen_eor_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1498 TCGv_vec pm
, TCGv_vec pg
)
1500 tcg_gen_xor_vec(vece
, pd
, pn
, pm
);
1501 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1504 static bool trans_EOR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1506 static const GVecGen4 op
= {
1507 .fni8
= gen_eor_pg_i64
,
1508 .fniv
= gen_eor_pg_vec
,
1509 .fno
= gen_helper_sve_eor_pppp
,
1510 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1512 return do_pppp_flags(s
, a
, &op
);
1515 static bool trans_SEL_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1520 if (sve_access_check(s
)) {
1521 unsigned psz
= pred_gvec_reg_size(s
);
1522 tcg_gen_gvec_bitsel(MO_8
, pred_full_reg_offset(s
, a
->rd
),
1523 pred_full_reg_offset(s
, a
->pg
),
1524 pred_full_reg_offset(s
, a
->rn
),
1525 pred_full_reg_offset(s
, a
->rm
), psz
, psz
);
1530 static void gen_orr_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1532 tcg_gen_or_i64(pd
, pn
, pm
);
1533 tcg_gen_and_i64(pd
, pd
, pg
);
1536 static void gen_orr_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1537 TCGv_vec pm
, TCGv_vec pg
)
1539 tcg_gen_or_vec(vece
, pd
, pn
, pm
);
1540 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1543 static bool trans_ORR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1545 static const GVecGen4 op
= {
1546 .fni8
= gen_orr_pg_i64
,
1547 .fniv
= gen_orr_pg_vec
,
1548 .fno
= gen_helper_sve_orr_pppp
,
1549 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1552 if (!a
->s
&& a
->pg
== a
->rn
&& a
->rn
== a
->rm
) {
1553 return do_mov_p(s
, a
->rd
, a
->rn
);
1555 return do_pppp_flags(s
, a
, &op
);
1558 static void gen_orn_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1560 tcg_gen_orc_i64(pd
, pn
, pm
);
1561 tcg_gen_and_i64(pd
, pd
, pg
);
1564 static void gen_orn_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1565 TCGv_vec pm
, TCGv_vec pg
)
1567 tcg_gen_orc_vec(vece
, pd
, pn
, pm
);
1568 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1571 static bool trans_ORN_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1573 static const GVecGen4 op
= {
1574 .fni8
= gen_orn_pg_i64
,
1575 .fniv
= gen_orn_pg_vec
,
1576 .fno
= gen_helper_sve_orn_pppp
,
1577 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1579 return do_pppp_flags(s
, a
, &op
);
1582 static void gen_nor_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1584 tcg_gen_or_i64(pd
, pn
, pm
);
1585 tcg_gen_andc_i64(pd
, pg
, pd
);
1588 static void gen_nor_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1589 TCGv_vec pm
, TCGv_vec pg
)
1591 tcg_gen_or_vec(vece
, pd
, pn
, pm
);
1592 tcg_gen_andc_vec(vece
, pd
, pg
, pd
);
1595 static bool trans_NOR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1597 static const GVecGen4 op
= {
1598 .fni8
= gen_nor_pg_i64
,
1599 .fniv
= gen_nor_pg_vec
,
1600 .fno
= gen_helper_sve_nor_pppp
,
1601 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1603 return do_pppp_flags(s
, a
, &op
);
1606 static void gen_nand_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1608 tcg_gen_and_i64(pd
, pn
, pm
);
1609 tcg_gen_andc_i64(pd
, pg
, pd
);
1612 static void gen_nand_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1613 TCGv_vec pm
, TCGv_vec pg
)
1615 tcg_gen_and_vec(vece
, pd
, pn
, pm
);
1616 tcg_gen_andc_vec(vece
, pd
, pg
, pd
);
1619 static bool trans_NAND_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1621 static const GVecGen4 op
= {
1622 .fni8
= gen_nand_pg_i64
,
1623 .fniv
= gen_nand_pg_vec
,
1624 .fno
= gen_helper_sve_nand_pppp
,
1625 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1627 return do_pppp_flags(s
, a
, &op
);
1631 *** SVE Predicate Misc Group
1634 static bool trans_PTEST(DisasContext
*s
, arg_PTEST
*a
)
1636 if (sve_access_check(s
)) {
1637 int nofs
= pred_full_reg_offset(s
, a
->rn
);
1638 int gofs
= pred_full_reg_offset(s
, a
->pg
);
1639 int words
= DIV_ROUND_UP(pred_full_reg_size(s
), 8);
1642 TCGv_i64 pn
= tcg_temp_new_i64();
1643 TCGv_i64 pg
= tcg_temp_new_i64();
1645 tcg_gen_ld_i64(pn
, cpu_env
, nofs
);
1646 tcg_gen_ld_i64(pg
, cpu_env
, gofs
);
1647 do_predtest1(pn
, pg
);
1649 tcg_temp_free_i64(pn
);
1650 tcg_temp_free_i64(pg
);
1652 do_predtest(s
, nofs
, gofs
, words
);
1658 /* See the ARM pseudocode DecodePredCount. */
1659 static unsigned decode_pred_count(unsigned fullsz
, int pattern
, int esz
)
1661 unsigned elements
= fullsz
>> esz
;
1665 case 0x0: /* POW2 */
1666 return pow2floor(elements
);
1677 case 0x9: /* VL16 */
1678 case 0xa: /* VL32 */
1679 case 0xb: /* VL64 */
1680 case 0xc: /* VL128 */
1681 case 0xd: /* VL256 */
1682 bound
= 16 << (pattern
- 9);
1684 case 0x1d: /* MUL4 */
1685 return elements
- elements
% 4;
1686 case 0x1e: /* MUL3 */
1687 return elements
- elements
% 3;
1688 case 0x1f: /* ALL */
1690 default: /* #uimm5 */
1693 return elements
>= bound
? bound
: 0;
1696 /* This handles all of the predicate initialization instructions,
1697 * PTRUE, PFALSE, SETFFR. For PFALSE, we will have set PAT == 32
1698 * so that decode_pred_count returns 0. For SETFFR, we will have
1699 * set RD == 16 == FFR.
1701 static bool do_predset(DisasContext
*s
, int esz
, int rd
, int pat
, bool setflag
)
1703 if (!sve_access_check(s
)) {
1707 unsigned fullsz
= vec_full_reg_size(s
);
1708 unsigned ofs
= pred_full_reg_offset(s
, rd
);
1709 unsigned numelem
, setsz
, i
;
1710 uint64_t word
, lastword
;
1713 numelem
= decode_pred_count(fullsz
, pat
, esz
);
1715 /* Determine what we must store into each bit, and how many. */
1717 lastword
= word
= 0;
1720 setsz
= numelem
<< esz
;
1721 lastword
= word
= pred_esz_masks
[esz
];
1723 lastword
&= MAKE_64BIT_MASK(0, setsz
% 64);
1727 t
= tcg_temp_new_i64();
1729 tcg_gen_movi_i64(t
, lastword
);
1730 tcg_gen_st_i64(t
, cpu_env
, ofs
);
1734 if (word
== lastword
) {
1735 unsigned maxsz
= size_for_gvec(fullsz
/ 8);
1736 unsigned oprsz
= size_for_gvec(setsz
/ 8);
1738 if (oprsz
* 8 == setsz
) {
1739 tcg_gen_gvec_dup_imm(MO_64
, ofs
, oprsz
, maxsz
, word
);
1747 tcg_gen_movi_i64(t
, word
);
1748 for (i
= 0; i
< QEMU_ALIGN_DOWN(setsz
, 8); i
+= 8) {
1749 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1751 if (lastword
!= word
) {
1752 tcg_gen_movi_i64(t
, lastword
);
1753 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1757 tcg_gen_movi_i64(t
, 0);
1758 for (; i
< fullsz
; i
+= 8) {
1759 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1764 tcg_temp_free_i64(t
);
1768 tcg_gen_movi_i32(cpu_NF
, -(word
!= 0));
1769 tcg_gen_movi_i32(cpu_CF
, word
== 0);
1770 tcg_gen_movi_i32(cpu_VF
, 0);
1771 tcg_gen_mov_i32(cpu_ZF
, cpu_NF
);
1776 static bool trans_PTRUE(DisasContext
*s
, arg_PTRUE
*a
)
1778 return do_predset(s
, a
->esz
, a
->rd
, a
->pat
, a
->s
);
1781 static bool trans_SETFFR(DisasContext
*s
, arg_SETFFR
*a
)
1783 /* Note pat == 31 is #all, to set all elements. */
1784 return do_predset(s
, 0, FFR_PRED_NUM
, 31, false);
1787 static bool trans_PFALSE(DisasContext
*s
, arg_PFALSE
*a
)
1789 /* Note pat == 32 is #unimp, to set no elements. */
1790 return do_predset(s
, 0, a
->rd
, 32, false);
1793 static bool trans_RDFFR_p(DisasContext
*s
, arg_RDFFR_p
*a
)
1795 /* The path through do_pppp_flags is complicated enough to want to avoid
1796 * duplication. Frob the arguments into the form of a predicated AND.
1798 arg_rprr_s alt_a
= {
1799 .rd
= a
->rd
, .pg
= a
->pg
, .s
= a
->s
,
1800 .rn
= FFR_PRED_NUM
, .rm
= FFR_PRED_NUM
,
1802 return trans_AND_pppp(s
, &alt_a
);
1805 static bool trans_RDFFR(DisasContext
*s
, arg_RDFFR
*a
)
1807 return do_mov_p(s
, a
->rd
, FFR_PRED_NUM
);
1810 static bool trans_WRFFR(DisasContext
*s
, arg_WRFFR
*a
)
1812 return do_mov_p(s
, FFR_PRED_NUM
, a
->rn
);
1815 static bool do_pfirst_pnext(DisasContext
*s
, arg_rr_esz
*a
,
1816 void (*gen_fn
)(TCGv_i32
, TCGv_ptr
,
1817 TCGv_ptr
, TCGv_i32
))
1819 if (!sve_access_check(s
)) {
1823 TCGv_ptr t_pd
= tcg_temp_new_ptr();
1824 TCGv_ptr t_pg
= tcg_temp_new_ptr();
1828 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, pred_full_reg_size(s
));
1829 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
1831 tcg_gen_addi_ptr(t_pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
1832 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
1833 t
= tcg_const_i32(desc
);
1835 gen_fn(t
, t_pd
, t_pg
, t
);
1836 tcg_temp_free_ptr(t_pd
);
1837 tcg_temp_free_ptr(t_pg
);
1840 tcg_temp_free_i32(t
);
1844 static bool trans_PFIRST(DisasContext
*s
, arg_rr_esz
*a
)
1846 return do_pfirst_pnext(s
, a
, gen_helper_sve_pfirst
);
1849 static bool trans_PNEXT(DisasContext
*s
, arg_rr_esz
*a
)
1851 return do_pfirst_pnext(s
, a
, gen_helper_sve_pnext
);
1855 *** SVE Element Count Group
1858 /* Perform an inline saturating addition of a 32-bit value within
1859 * a 64-bit register. The second operand is known to be positive,
1860 * which halves the comparisions we must perform to bound the result.
1862 static void do_sat_addsub_32(TCGv_i64 reg
, TCGv_i64 val
, bool u
, bool d
)
1868 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
1870 tcg_gen_ext32u_i64(reg
, reg
);
1872 tcg_gen_ext32s_i64(reg
, reg
);
1875 tcg_gen_sub_i64(reg
, reg
, val
);
1876 ibound
= (u
? 0 : INT32_MIN
);
1879 tcg_gen_add_i64(reg
, reg
, val
);
1880 ibound
= (u
? UINT32_MAX
: INT32_MAX
);
1883 bound
= tcg_const_i64(ibound
);
1884 tcg_gen_movcond_i64(cond
, reg
, reg
, bound
, bound
, reg
);
1885 tcg_temp_free_i64(bound
);
1888 /* Similarly with 64-bit values. */
1889 static void do_sat_addsub_64(TCGv_i64 reg
, TCGv_i64 val
, bool u
, bool d
)
1891 TCGv_i64 t0
= tcg_temp_new_i64();
1892 TCGv_i64 t1
= tcg_temp_new_i64();
1897 tcg_gen_sub_i64(t0
, reg
, val
);
1898 tcg_gen_movi_i64(t1
, 0);
1899 tcg_gen_movcond_i64(TCG_COND_LTU
, reg
, reg
, val
, t1
, t0
);
1901 tcg_gen_add_i64(t0
, reg
, val
);
1902 tcg_gen_movi_i64(t1
, -1);
1903 tcg_gen_movcond_i64(TCG_COND_LTU
, reg
, t0
, reg
, t1
, t0
);
1907 /* Detect signed overflow for subtraction. */
1908 tcg_gen_xor_i64(t0
, reg
, val
);
1909 tcg_gen_sub_i64(t1
, reg
, val
);
1910 tcg_gen_xor_i64(reg
, reg
, t1
);
1911 tcg_gen_and_i64(t0
, t0
, reg
);
1913 /* Bound the result. */
1914 tcg_gen_movi_i64(reg
, INT64_MIN
);
1915 t2
= tcg_const_i64(0);
1916 tcg_gen_movcond_i64(TCG_COND_LT
, reg
, t0
, t2
, reg
, t1
);
1918 /* Detect signed overflow for addition. */
1919 tcg_gen_xor_i64(t0
, reg
, val
);
1920 tcg_gen_add_i64(reg
, reg
, val
);
1921 tcg_gen_xor_i64(t1
, reg
, val
);
1922 tcg_gen_andc_i64(t0
, t1
, t0
);
1924 /* Bound the result. */
1925 tcg_gen_movi_i64(t1
, INT64_MAX
);
1926 t2
= tcg_const_i64(0);
1927 tcg_gen_movcond_i64(TCG_COND_LT
, reg
, t0
, t2
, t1
, reg
);
1929 tcg_temp_free_i64(t2
);
1931 tcg_temp_free_i64(t0
);
1932 tcg_temp_free_i64(t1
);
1935 /* Similarly with a vector and a scalar operand. */
1936 static void do_sat_addsub_vec(DisasContext
*s
, int esz
, int rd
, int rn
,
1937 TCGv_i64 val
, bool u
, bool d
)
1939 unsigned vsz
= vec_full_reg_size(s
);
1940 TCGv_ptr dptr
, nptr
;
1944 dptr
= tcg_temp_new_ptr();
1945 nptr
= tcg_temp_new_ptr();
1946 tcg_gen_addi_ptr(dptr
, cpu_env
, vec_full_reg_offset(s
, rd
));
1947 tcg_gen_addi_ptr(nptr
, cpu_env
, vec_full_reg_offset(s
, rn
));
1948 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
1952 t32
= tcg_temp_new_i32();
1953 tcg_gen_extrl_i64_i32(t32
, val
);
1955 tcg_gen_neg_i32(t32
, t32
);
1958 gen_helper_sve_uqaddi_b(dptr
, nptr
, t32
, desc
);
1960 gen_helper_sve_sqaddi_b(dptr
, nptr
, t32
, desc
);
1962 tcg_temp_free_i32(t32
);
1966 t32
= tcg_temp_new_i32();
1967 tcg_gen_extrl_i64_i32(t32
, val
);
1969 tcg_gen_neg_i32(t32
, t32
);
1972 gen_helper_sve_uqaddi_h(dptr
, nptr
, t32
, desc
);
1974 gen_helper_sve_sqaddi_h(dptr
, nptr
, t32
, desc
);
1976 tcg_temp_free_i32(t32
);
1980 t64
= tcg_temp_new_i64();
1982 tcg_gen_neg_i64(t64
, val
);
1984 tcg_gen_mov_i64(t64
, val
);
1987 gen_helper_sve_uqaddi_s(dptr
, nptr
, t64
, desc
);
1989 gen_helper_sve_sqaddi_s(dptr
, nptr
, t64
, desc
);
1991 tcg_temp_free_i64(t64
);
1997 gen_helper_sve_uqsubi_d(dptr
, nptr
, val
, desc
);
1999 gen_helper_sve_uqaddi_d(dptr
, nptr
, val
, desc
);
2002 t64
= tcg_temp_new_i64();
2003 tcg_gen_neg_i64(t64
, val
);
2004 gen_helper_sve_sqaddi_d(dptr
, nptr
, t64
, desc
);
2005 tcg_temp_free_i64(t64
);
2007 gen_helper_sve_sqaddi_d(dptr
, nptr
, val
, desc
);
2012 g_assert_not_reached();
2015 tcg_temp_free_ptr(dptr
);
2016 tcg_temp_free_ptr(nptr
);
2017 tcg_temp_free_i32(desc
);
2020 static bool trans_CNT_r(DisasContext
*s
, arg_CNT_r
*a
)
2022 if (sve_access_check(s
)) {
2023 unsigned fullsz
= vec_full_reg_size(s
);
2024 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2025 tcg_gen_movi_i64(cpu_reg(s
, a
->rd
), numelem
* a
->imm
);
2030 static bool trans_INCDEC_r(DisasContext
*s
, arg_incdec_cnt
*a
)
2032 if (sve_access_check(s
)) {
2033 unsigned fullsz
= vec_full_reg_size(s
);
2034 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2035 int inc
= numelem
* a
->imm
* (a
->d
? -1 : 1);
2036 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
2038 tcg_gen_addi_i64(reg
, reg
, inc
);
2043 static bool trans_SINCDEC_r_32(DisasContext
*s
, arg_incdec_cnt
*a
)
2045 if (!sve_access_check(s
)) {
2049 unsigned fullsz
= vec_full_reg_size(s
);
2050 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2051 int inc
= numelem
* a
->imm
;
2052 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
2054 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
2057 tcg_gen_ext32u_i64(reg
, reg
);
2059 tcg_gen_ext32s_i64(reg
, reg
);
2062 TCGv_i64 t
= tcg_const_i64(inc
);
2063 do_sat_addsub_32(reg
, t
, a
->u
, a
->d
);
2064 tcg_temp_free_i64(t
);
2069 static bool trans_SINCDEC_r_64(DisasContext
*s
, arg_incdec_cnt
*a
)
2071 if (!sve_access_check(s
)) {
2075 unsigned fullsz
= vec_full_reg_size(s
);
2076 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2077 int inc
= numelem
* a
->imm
;
2078 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
2081 TCGv_i64 t
= tcg_const_i64(inc
);
2082 do_sat_addsub_64(reg
, t
, a
->u
, a
->d
);
2083 tcg_temp_free_i64(t
);
2088 static bool trans_INCDEC_v(DisasContext
*s
, arg_incdec2_cnt
*a
)
2094 unsigned fullsz
= vec_full_reg_size(s
);
2095 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2096 int inc
= numelem
* a
->imm
;
2099 if (sve_access_check(s
)) {
2100 TCGv_i64 t
= tcg_const_i64(a
->d
? -inc
: inc
);
2101 tcg_gen_gvec_adds(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
2102 vec_full_reg_offset(s
, a
->rn
),
2104 tcg_temp_free_i64(t
);
2107 do_mov_z(s
, a
->rd
, a
->rn
);
2112 static bool trans_SINCDEC_v(DisasContext
*s
, arg_incdec2_cnt
*a
)
2118 unsigned fullsz
= vec_full_reg_size(s
);
2119 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2120 int inc
= numelem
* a
->imm
;
2123 if (sve_access_check(s
)) {
2124 TCGv_i64 t
= tcg_const_i64(inc
);
2125 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
, t
, a
->u
, a
->d
);
2126 tcg_temp_free_i64(t
);
2129 do_mov_z(s
, a
->rd
, a
->rn
);
2135 *** SVE Bitwise Immediate Group
2138 static bool do_zz_dbm(DisasContext
*s
, arg_rr_dbm
*a
, GVecGen2iFn
*gvec_fn
)
2141 if (!logic_imm_decode_wmask(&imm
, extract32(a
->dbm
, 12, 1),
2142 extract32(a
->dbm
, 0, 6),
2143 extract32(a
->dbm
, 6, 6))) {
2146 if (sve_access_check(s
)) {
2147 unsigned vsz
= vec_full_reg_size(s
);
2148 gvec_fn(MO_64
, vec_full_reg_offset(s
, a
->rd
),
2149 vec_full_reg_offset(s
, a
->rn
), imm
, vsz
, vsz
);
2154 static bool trans_AND_zzi(DisasContext
*s
, arg_rr_dbm
*a
)
2156 return do_zz_dbm(s
, a
, tcg_gen_gvec_andi
);
2159 static bool trans_ORR_zzi(DisasContext
*s
, arg_rr_dbm
*a
)
2161 return do_zz_dbm(s
, a
, tcg_gen_gvec_ori
);
2164 static bool trans_EOR_zzi(DisasContext
*s
, arg_rr_dbm
*a
)
2166 return do_zz_dbm(s
, a
, tcg_gen_gvec_xori
);
2169 static bool trans_DUPM(DisasContext
*s
, arg_DUPM
*a
)
2172 if (!logic_imm_decode_wmask(&imm
, extract32(a
->dbm
, 12, 1),
2173 extract32(a
->dbm
, 0, 6),
2174 extract32(a
->dbm
, 6, 6))) {
2177 if (sve_access_check(s
)) {
2178 do_dupi_z(s
, a
->rd
, imm
);
2184 *** SVE Integer Wide Immediate - Predicated Group
2187 /* Implement all merging copies. This is used for CPY (immediate),
2188 * FCPY, CPY (scalar), CPY (SIMD&FP scalar).
2190 static void do_cpy_m(DisasContext
*s
, int esz
, int rd
, int rn
, int pg
,
2193 typedef void gen_cpy(TCGv_ptr
, TCGv_ptr
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
2194 static gen_cpy
* const fns
[4] = {
2195 gen_helper_sve_cpy_m_b
, gen_helper_sve_cpy_m_h
,
2196 gen_helper_sve_cpy_m_s
, gen_helper_sve_cpy_m_d
,
2198 unsigned vsz
= vec_full_reg_size(s
);
2199 TCGv_i32 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
2200 TCGv_ptr t_zd
= tcg_temp_new_ptr();
2201 TCGv_ptr t_zn
= tcg_temp_new_ptr();
2202 TCGv_ptr t_pg
= tcg_temp_new_ptr();
2204 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, rd
));
2205 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, rn
));
2206 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
2208 fns
[esz
](t_zd
, t_zn
, t_pg
, val
, desc
);
2210 tcg_temp_free_ptr(t_zd
);
2211 tcg_temp_free_ptr(t_zn
);
2212 tcg_temp_free_ptr(t_pg
);
2213 tcg_temp_free_i32(desc
);
2216 static bool trans_FCPY(DisasContext
*s
, arg_FCPY
*a
)
2221 if (sve_access_check(s
)) {
2222 /* Decode the VFP immediate. */
2223 uint64_t imm
= vfp_expand_imm(a
->esz
, a
->imm
);
2224 TCGv_i64 t_imm
= tcg_const_i64(imm
);
2225 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rn
, a
->pg
, t_imm
);
2226 tcg_temp_free_i64(t_imm
);
2231 static bool trans_CPY_m_i(DisasContext
*s
, arg_rpri_esz
*a
)
2233 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
2236 if (sve_access_check(s
)) {
2237 TCGv_i64 t_imm
= tcg_const_i64(a
->imm
);
2238 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rn
, a
->pg
, t_imm
);
2239 tcg_temp_free_i64(t_imm
);
2244 static bool trans_CPY_z_i(DisasContext
*s
, arg_CPY_z_i
*a
)
2246 static gen_helper_gvec_2i
* const fns
[4] = {
2247 gen_helper_sve_cpy_z_b
, gen_helper_sve_cpy_z_h
,
2248 gen_helper_sve_cpy_z_s
, gen_helper_sve_cpy_z_d
,
2251 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
2254 if (sve_access_check(s
)) {
2255 unsigned vsz
= vec_full_reg_size(s
);
2256 TCGv_i64 t_imm
= tcg_const_i64(a
->imm
);
2257 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s
, a
->rd
),
2258 pred_full_reg_offset(s
, a
->pg
),
2259 t_imm
, vsz
, vsz
, 0, fns
[a
->esz
]);
2260 tcg_temp_free_i64(t_imm
);
2266 *** SVE Permute Extract Group
2269 static bool do_EXT(DisasContext
*s
, int rd
, int rn
, int rm
, int imm
)
2271 if (!sve_access_check(s
)) {
2275 unsigned vsz
= vec_full_reg_size(s
);
2276 unsigned n_ofs
= imm
>= vsz
? 0 : imm
;
2277 unsigned n_siz
= vsz
- n_ofs
;
2278 unsigned d
= vec_full_reg_offset(s
, rd
);
2279 unsigned n
= vec_full_reg_offset(s
, rn
);
2280 unsigned m
= vec_full_reg_offset(s
, rm
);
2282 /* Use host vector move insns if we have appropriate sizes
2283 * and no unfortunate overlap.
2286 && n_ofs
== size_for_gvec(n_ofs
)
2287 && n_siz
== size_for_gvec(n_siz
)
2288 && (d
!= n
|| n_siz
<= n_ofs
)) {
2289 tcg_gen_gvec_mov(0, d
, n
+ n_ofs
, n_siz
, n_siz
);
2291 tcg_gen_gvec_mov(0, d
+ n_siz
, m
, n_ofs
, n_ofs
);
2294 tcg_gen_gvec_3_ool(d
, n
, m
, vsz
, vsz
, n_ofs
, gen_helper_sve_ext
);
2299 static bool trans_EXT(DisasContext
*s
, arg_EXT
*a
)
2301 return do_EXT(s
, a
->rd
, a
->rn
, a
->rm
, a
->imm
);
2304 static bool trans_EXT_sve2(DisasContext
*s
, arg_rri
*a
)
2306 if (!dc_isar_feature(aa64_sve2
, s
)) {
2309 return do_EXT(s
, a
->rd
, a
->rn
, (a
->rn
+ 1) % 32, a
->imm
);
2313 *** SVE Permute - Unpredicated Group
2316 static bool trans_DUP_s(DisasContext
*s
, arg_DUP_s
*a
)
2318 if (sve_access_check(s
)) {
2319 unsigned vsz
= vec_full_reg_size(s
);
2320 tcg_gen_gvec_dup_i64(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
2321 vsz
, vsz
, cpu_reg_sp(s
, a
->rn
));
2326 static bool trans_DUP_x(DisasContext
*s
, arg_DUP_x
*a
)
2328 if ((a
->imm
& 0x1f) == 0) {
2331 if (sve_access_check(s
)) {
2332 unsigned vsz
= vec_full_reg_size(s
);
2333 unsigned dofs
= vec_full_reg_offset(s
, a
->rd
);
2334 unsigned esz
, index
;
2336 esz
= ctz32(a
->imm
);
2337 index
= a
->imm
>> (esz
+ 1);
2339 if ((index
<< esz
) < vsz
) {
2340 unsigned nofs
= vec_reg_offset(s
, a
->rn
, index
, esz
);
2341 tcg_gen_gvec_dup_mem(esz
, dofs
, nofs
, vsz
, vsz
);
2344 * While dup_mem handles 128-bit elements, dup_imm does not.
2345 * Thankfully element size doesn't matter for splatting zero.
2347 tcg_gen_gvec_dup_imm(MO_64
, dofs
, vsz
, vsz
, 0);
2353 static void do_insr_i64(DisasContext
*s
, arg_rrr_esz
*a
, TCGv_i64 val
)
2355 typedef void gen_insr(TCGv_ptr
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
2356 static gen_insr
* const fns
[4] = {
2357 gen_helper_sve_insr_b
, gen_helper_sve_insr_h
,
2358 gen_helper_sve_insr_s
, gen_helper_sve_insr_d
,
2360 unsigned vsz
= vec_full_reg_size(s
);
2361 TCGv_i32 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
2362 TCGv_ptr t_zd
= tcg_temp_new_ptr();
2363 TCGv_ptr t_zn
= tcg_temp_new_ptr();
2365 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, a
->rd
));
2366 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
2368 fns
[a
->esz
](t_zd
, t_zn
, val
, desc
);
2370 tcg_temp_free_ptr(t_zd
);
2371 tcg_temp_free_ptr(t_zn
);
2372 tcg_temp_free_i32(desc
);
2375 static bool trans_INSR_f(DisasContext
*s
, arg_rrr_esz
*a
)
2377 if (sve_access_check(s
)) {
2378 TCGv_i64 t
= tcg_temp_new_i64();
2379 tcg_gen_ld_i64(t
, cpu_env
, vec_reg_offset(s
, a
->rm
, 0, MO_64
));
2380 do_insr_i64(s
, a
, t
);
2381 tcg_temp_free_i64(t
);
2386 static bool trans_INSR_r(DisasContext
*s
, arg_rrr_esz
*a
)
2388 if (sve_access_check(s
)) {
2389 do_insr_i64(s
, a
, cpu_reg(s
, a
->rm
));
2394 static bool trans_REV_v(DisasContext
*s
, arg_rr_esz
*a
)
2396 static gen_helper_gvec_2
* const fns
[4] = {
2397 gen_helper_sve_rev_b
, gen_helper_sve_rev_h
,
2398 gen_helper_sve_rev_s
, gen_helper_sve_rev_d
2401 if (sve_access_check(s
)) {
2402 gen_gvec_ool_zz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, 0);
2407 static bool trans_TBL(DisasContext
*s
, arg_rrr_esz
*a
)
2409 static gen_helper_gvec_3
* const fns
[4] = {
2410 gen_helper_sve_tbl_b
, gen_helper_sve_tbl_h
,
2411 gen_helper_sve_tbl_s
, gen_helper_sve_tbl_d
2414 if (sve_access_check(s
)) {
2415 gen_gvec_ool_zzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, 0);
2420 static bool trans_UNPK(DisasContext
*s
, arg_UNPK
*a
)
2422 static gen_helper_gvec_2
* const fns
[4][2] = {
2424 { gen_helper_sve_sunpk_h
, gen_helper_sve_uunpk_h
},
2425 { gen_helper_sve_sunpk_s
, gen_helper_sve_uunpk_s
},
2426 { gen_helper_sve_sunpk_d
, gen_helper_sve_uunpk_d
},
2432 if (sve_access_check(s
)) {
2433 unsigned vsz
= vec_full_reg_size(s
);
2434 tcg_gen_gvec_2_ool(vec_full_reg_offset(s
, a
->rd
),
2435 vec_full_reg_offset(s
, a
->rn
)
2436 + (a
->h
? vsz
/ 2 : 0),
2437 vsz
, vsz
, 0, fns
[a
->esz
][a
->u
]);
2443 *** SVE Permute - Predicates Group
2446 static bool do_perm_pred3(DisasContext
*s
, arg_rrr_esz
*a
, bool high_odd
,
2447 gen_helper_gvec_3
*fn
)
2449 if (!sve_access_check(s
)) {
2453 unsigned vsz
= pred_full_reg_size(s
);
2455 TCGv_ptr t_d
= tcg_temp_new_ptr();
2456 TCGv_ptr t_n
= tcg_temp_new_ptr();
2457 TCGv_ptr t_m
= tcg_temp_new_ptr();
2461 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
);
2462 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
2463 desc
= FIELD_DP32(desc
, PREDDESC
, DATA
, high_odd
);
2465 tcg_gen_addi_ptr(t_d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2466 tcg_gen_addi_ptr(t_n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2467 tcg_gen_addi_ptr(t_m
, cpu_env
, pred_full_reg_offset(s
, a
->rm
));
2468 t_desc
= tcg_const_i32(desc
);
2470 fn(t_d
, t_n
, t_m
, t_desc
);
2472 tcg_temp_free_ptr(t_d
);
2473 tcg_temp_free_ptr(t_n
);
2474 tcg_temp_free_ptr(t_m
);
2475 tcg_temp_free_i32(t_desc
);
2479 static bool do_perm_pred2(DisasContext
*s
, arg_rr_esz
*a
, bool high_odd
,
2480 gen_helper_gvec_2
*fn
)
2482 if (!sve_access_check(s
)) {
2486 unsigned vsz
= pred_full_reg_size(s
);
2487 TCGv_ptr t_d
= tcg_temp_new_ptr();
2488 TCGv_ptr t_n
= tcg_temp_new_ptr();
2492 tcg_gen_addi_ptr(t_d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2493 tcg_gen_addi_ptr(t_n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2495 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
);
2496 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
2497 desc
= FIELD_DP32(desc
, PREDDESC
, DATA
, high_odd
);
2498 t_desc
= tcg_const_i32(desc
);
2500 fn(t_d
, t_n
, t_desc
);
2502 tcg_temp_free_i32(t_desc
);
2503 tcg_temp_free_ptr(t_d
);
2504 tcg_temp_free_ptr(t_n
);
2508 static bool trans_ZIP1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2510 return do_perm_pred3(s
, a
, 0, gen_helper_sve_zip_p
);
2513 static bool trans_ZIP2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2515 return do_perm_pred3(s
, a
, 1, gen_helper_sve_zip_p
);
2518 static bool trans_UZP1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2520 return do_perm_pred3(s
, a
, 0, gen_helper_sve_uzp_p
);
2523 static bool trans_UZP2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2525 return do_perm_pred3(s
, a
, 1, gen_helper_sve_uzp_p
);
2528 static bool trans_TRN1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2530 return do_perm_pred3(s
, a
, 0, gen_helper_sve_trn_p
);
2533 static bool trans_TRN2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2535 return do_perm_pred3(s
, a
, 1, gen_helper_sve_trn_p
);
2538 static bool trans_REV_p(DisasContext
*s
, arg_rr_esz
*a
)
2540 return do_perm_pred2(s
, a
, 0, gen_helper_sve_rev_p
);
2543 static bool trans_PUNPKLO(DisasContext
*s
, arg_PUNPKLO
*a
)
2545 return do_perm_pred2(s
, a
, 0, gen_helper_sve_punpk_p
);
2548 static bool trans_PUNPKHI(DisasContext
*s
, arg_PUNPKHI
*a
)
2550 return do_perm_pred2(s
, a
, 1, gen_helper_sve_punpk_p
);
2554 *** SVE Permute - Interleaving Group
2557 static bool do_zip(DisasContext
*s
, arg_rrr_esz
*a
, bool high
)
2559 static gen_helper_gvec_3
* const fns
[4] = {
2560 gen_helper_sve_zip_b
, gen_helper_sve_zip_h
,
2561 gen_helper_sve_zip_s
, gen_helper_sve_zip_d
,
2564 if (sve_access_check(s
)) {
2565 unsigned vsz
= vec_full_reg_size(s
);
2566 unsigned high_ofs
= high
? vsz
/ 2 : 0;
2567 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, a
->rd
),
2568 vec_full_reg_offset(s
, a
->rn
) + high_ofs
,
2569 vec_full_reg_offset(s
, a
->rm
) + high_ofs
,
2570 vsz
, vsz
, 0, fns
[a
->esz
]);
2575 static bool do_zzz_data_ool(DisasContext
*s
, arg_rrr_esz
*a
, int data
,
2576 gen_helper_gvec_3
*fn
)
2578 if (sve_access_check(s
)) {
2579 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, data
);
2584 static bool trans_ZIP1_z(DisasContext
*s
, arg_rrr_esz
*a
)
2586 return do_zip(s
, a
, false);
2589 static bool trans_ZIP2_z(DisasContext
*s
, arg_rrr_esz
*a
)
2591 return do_zip(s
, a
, true);
2594 static gen_helper_gvec_3
* const uzp_fns
[4] = {
2595 gen_helper_sve_uzp_b
, gen_helper_sve_uzp_h
,
2596 gen_helper_sve_uzp_s
, gen_helper_sve_uzp_d
,
2599 static bool trans_UZP1_z(DisasContext
*s
, arg_rrr_esz
*a
)
2601 return do_zzz_data_ool(s
, a
, 0, uzp_fns
[a
->esz
]);
2604 static bool trans_UZP2_z(DisasContext
*s
, arg_rrr_esz
*a
)
2606 return do_zzz_data_ool(s
, a
, 1 << a
->esz
, uzp_fns
[a
->esz
]);
2609 static gen_helper_gvec_3
* const trn_fns
[4] = {
2610 gen_helper_sve_trn_b
, gen_helper_sve_trn_h
,
2611 gen_helper_sve_trn_s
, gen_helper_sve_trn_d
,
2614 static bool trans_TRN1_z(DisasContext
*s
, arg_rrr_esz
*a
)
2616 return do_zzz_data_ool(s
, a
, 0, trn_fns
[a
->esz
]);
2619 static bool trans_TRN2_z(DisasContext
*s
, arg_rrr_esz
*a
)
2621 return do_zzz_data_ool(s
, a
, 1 << a
->esz
, trn_fns
[a
->esz
]);
2625 *** SVE Permute Vector - Predicated Group
2628 static bool trans_COMPACT(DisasContext
*s
, arg_rpr_esz
*a
)
2630 static gen_helper_gvec_3
* const fns
[4] = {
2631 NULL
, NULL
, gen_helper_sve_compact_s
, gen_helper_sve_compact_d
2633 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
2636 /* Call the helper that computes the ARM LastActiveElement pseudocode
2637 * function, scaled by the element size. This includes the not found
2638 * indication; e.g. not found for esz=3 is -8.
2640 static void find_last_active(DisasContext
*s
, TCGv_i32 ret
, int esz
, int pg
)
2642 /* Predicate sizes may be smaller and cannot use simd_desc. We cannot
2643 * round up, as we do elsewhere, because we need the exact size.
2645 TCGv_ptr t_p
= tcg_temp_new_ptr();
2649 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, pred_full_reg_size(s
));
2650 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, esz
);
2652 tcg_gen_addi_ptr(t_p
, cpu_env
, pred_full_reg_offset(s
, pg
));
2653 t_desc
= tcg_const_i32(desc
);
2655 gen_helper_sve_last_active_element(ret
, t_p
, t_desc
);
2657 tcg_temp_free_i32(t_desc
);
2658 tcg_temp_free_ptr(t_p
);
2661 /* Increment LAST to the offset of the next element in the vector,
2662 * wrapping around to 0.
2664 static void incr_last_active(DisasContext
*s
, TCGv_i32 last
, int esz
)
2666 unsigned vsz
= vec_full_reg_size(s
);
2668 tcg_gen_addi_i32(last
, last
, 1 << esz
);
2669 if (is_power_of_2(vsz
)) {
2670 tcg_gen_andi_i32(last
, last
, vsz
- 1);
2672 TCGv_i32 max
= tcg_const_i32(vsz
);
2673 TCGv_i32 zero
= tcg_const_i32(0);
2674 tcg_gen_movcond_i32(TCG_COND_GEU
, last
, last
, max
, zero
, last
);
2675 tcg_temp_free_i32(max
);
2676 tcg_temp_free_i32(zero
);
2680 /* If LAST < 0, set LAST to the offset of the last element in the vector. */
2681 static void wrap_last_active(DisasContext
*s
, TCGv_i32 last
, int esz
)
2683 unsigned vsz
= vec_full_reg_size(s
);
2685 if (is_power_of_2(vsz
)) {
2686 tcg_gen_andi_i32(last
, last
, vsz
- 1);
2688 TCGv_i32 max
= tcg_const_i32(vsz
- (1 << esz
));
2689 TCGv_i32 zero
= tcg_const_i32(0);
2690 tcg_gen_movcond_i32(TCG_COND_LT
, last
, last
, zero
, max
, last
);
2691 tcg_temp_free_i32(max
);
2692 tcg_temp_free_i32(zero
);
2696 /* Load an unsigned element of ESZ from BASE+OFS. */
2697 static TCGv_i64
load_esz(TCGv_ptr base
, int ofs
, int esz
)
2699 TCGv_i64 r
= tcg_temp_new_i64();
2703 tcg_gen_ld8u_i64(r
, base
, ofs
);
2706 tcg_gen_ld16u_i64(r
, base
, ofs
);
2709 tcg_gen_ld32u_i64(r
, base
, ofs
);
2712 tcg_gen_ld_i64(r
, base
, ofs
);
2715 g_assert_not_reached();
2720 /* Load an unsigned element of ESZ from RM[LAST]. */
2721 static TCGv_i64
load_last_active(DisasContext
*s
, TCGv_i32 last
,
2724 TCGv_ptr p
= tcg_temp_new_ptr();
2727 /* Convert offset into vector into offset into ENV.
2728 * The final adjustment for the vector register base
2729 * is added via constant offset to the load.
2731 #ifdef HOST_WORDS_BIGENDIAN
2732 /* Adjust for element ordering. See vec_reg_offset. */
2734 tcg_gen_xori_i32(last
, last
, 8 - (1 << esz
));
2737 tcg_gen_ext_i32_ptr(p
, last
);
2738 tcg_gen_add_ptr(p
, p
, cpu_env
);
2740 r
= load_esz(p
, vec_full_reg_offset(s
, rm
), esz
);
2741 tcg_temp_free_ptr(p
);
2746 /* Compute CLAST for a Zreg. */
2747 static bool do_clast_vector(DisasContext
*s
, arg_rprr_esz
*a
, bool before
)
2752 unsigned vsz
, esz
= a
->esz
;
2754 if (!sve_access_check(s
)) {
2758 last
= tcg_temp_local_new_i32();
2759 over
= gen_new_label();
2761 find_last_active(s
, last
, esz
, a
->pg
);
2763 /* There is of course no movcond for a 2048-bit vector,
2764 * so we must branch over the actual store.
2766 tcg_gen_brcondi_i32(TCG_COND_LT
, last
, 0, over
);
2769 incr_last_active(s
, last
, esz
);
2772 ele
= load_last_active(s
, last
, a
->rm
, esz
);
2773 tcg_temp_free_i32(last
);
2775 vsz
= vec_full_reg_size(s
);
2776 tcg_gen_gvec_dup_i64(esz
, vec_full_reg_offset(s
, a
->rd
), vsz
, vsz
, ele
);
2777 tcg_temp_free_i64(ele
);
2779 /* If this insn used MOVPRFX, we may need a second move. */
2780 if (a
->rd
!= a
->rn
) {
2781 TCGLabel
*done
= gen_new_label();
2784 gen_set_label(over
);
2785 do_mov_z(s
, a
->rd
, a
->rn
);
2787 gen_set_label(done
);
2789 gen_set_label(over
);
2794 static bool trans_CLASTA_z(DisasContext
*s
, arg_rprr_esz
*a
)
2796 return do_clast_vector(s
, a
, false);
2799 static bool trans_CLASTB_z(DisasContext
*s
, arg_rprr_esz
*a
)
2801 return do_clast_vector(s
, a
, true);
2804 /* Compute CLAST for a scalar. */
2805 static void do_clast_scalar(DisasContext
*s
, int esz
, int pg
, int rm
,
2806 bool before
, TCGv_i64 reg_val
)
2808 TCGv_i32 last
= tcg_temp_new_i32();
2809 TCGv_i64 ele
, cmp
, zero
;
2811 find_last_active(s
, last
, esz
, pg
);
2813 /* Extend the original value of last prior to incrementing. */
2814 cmp
= tcg_temp_new_i64();
2815 tcg_gen_ext_i32_i64(cmp
, last
);
2818 incr_last_active(s
, last
, esz
);
2821 /* The conceit here is that while last < 0 indicates not found, after
2822 * adjusting for cpu_env->vfp.zregs[rm], it is still a valid address
2823 * from which we can load garbage. We then discard the garbage with
2824 * a conditional move.
2826 ele
= load_last_active(s
, last
, rm
, esz
);
2827 tcg_temp_free_i32(last
);
2829 zero
= tcg_const_i64(0);
2830 tcg_gen_movcond_i64(TCG_COND_GE
, reg_val
, cmp
, zero
, ele
, reg_val
);
2832 tcg_temp_free_i64(zero
);
2833 tcg_temp_free_i64(cmp
);
2834 tcg_temp_free_i64(ele
);
2837 /* Compute CLAST for a Vreg. */
2838 static bool do_clast_fp(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2840 if (sve_access_check(s
)) {
2842 int ofs
= vec_reg_offset(s
, a
->rd
, 0, esz
);
2843 TCGv_i64 reg
= load_esz(cpu_env
, ofs
, esz
);
2845 do_clast_scalar(s
, esz
, a
->pg
, a
->rn
, before
, reg
);
2846 write_fp_dreg(s
, a
->rd
, reg
);
2847 tcg_temp_free_i64(reg
);
2852 static bool trans_CLASTA_v(DisasContext
*s
, arg_rpr_esz
*a
)
2854 return do_clast_fp(s
, a
, false);
2857 static bool trans_CLASTB_v(DisasContext
*s
, arg_rpr_esz
*a
)
2859 return do_clast_fp(s
, a
, true);
2862 /* Compute CLAST for a Xreg. */
2863 static bool do_clast_general(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2867 if (!sve_access_check(s
)) {
2871 reg
= cpu_reg(s
, a
->rd
);
2874 tcg_gen_ext8u_i64(reg
, reg
);
2877 tcg_gen_ext16u_i64(reg
, reg
);
2880 tcg_gen_ext32u_i64(reg
, reg
);
2885 g_assert_not_reached();
2888 do_clast_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
, reg
);
2892 static bool trans_CLASTA_r(DisasContext
*s
, arg_rpr_esz
*a
)
2894 return do_clast_general(s
, a
, false);
2897 static bool trans_CLASTB_r(DisasContext
*s
, arg_rpr_esz
*a
)
2899 return do_clast_general(s
, a
, true);
2902 /* Compute LAST for a scalar. */
2903 static TCGv_i64
do_last_scalar(DisasContext
*s
, int esz
,
2904 int pg
, int rm
, bool before
)
2906 TCGv_i32 last
= tcg_temp_new_i32();
2909 find_last_active(s
, last
, esz
, pg
);
2911 wrap_last_active(s
, last
, esz
);
2913 incr_last_active(s
, last
, esz
);
2916 ret
= load_last_active(s
, last
, rm
, esz
);
2917 tcg_temp_free_i32(last
);
2921 /* Compute LAST for a Vreg. */
2922 static bool do_last_fp(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2924 if (sve_access_check(s
)) {
2925 TCGv_i64 val
= do_last_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
);
2926 write_fp_dreg(s
, a
->rd
, val
);
2927 tcg_temp_free_i64(val
);
2932 static bool trans_LASTA_v(DisasContext
*s
, arg_rpr_esz
*a
)
2934 return do_last_fp(s
, a
, false);
2937 static bool trans_LASTB_v(DisasContext
*s
, arg_rpr_esz
*a
)
2939 return do_last_fp(s
, a
, true);
2942 /* Compute LAST for a Xreg. */
2943 static bool do_last_general(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2945 if (sve_access_check(s
)) {
2946 TCGv_i64 val
= do_last_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
);
2947 tcg_gen_mov_i64(cpu_reg(s
, a
->rd
), val
);
2948 tcg_temp_free_i64(val
);
2953 static bool trans_LASTA_r(DisasContext
*s
, arg_rpr_esz
*a
)
2955 return do_last_general(s
, a
, false);
2958 static bool trans_LASTB_r(DisasContext
*s
, arg_rpr_esz
*a
)
2960 return do_last_general(s
, a
, true);
2963 static bool trans_CPY_m_r(DisasContext
*s
, arg_rpr_esz
*a
)
2965 if (sve_access_check(s
)) {
2966 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rd
, a
->pg
, cpu_reg_sp(s
, a
->rn
));
2971 static bool trans_CPY_m_v(DisasContext
*s
, arg_rpr_esz
*a
)
2973 if (sve_access_check(s
)) {
2974 int ofs
= vec_reg_offset(s
, a
->rn
, 0, a
->esz
);
2975 TCGv_i64 t
= load_esz(cpu_env
, ofs
, a
->esz
);
2976 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rd
, a
->pg
, t
);
2977 tcg_temp_free_i64(t
);
2982 static bool trans_REVB(DisasContext
*s
, arg_rpr_esz
*a
)
2984 static gen_helper_gvec_3
* const fns
[4] = {
2986 gen_helper_sve_revb_h
,
2987 gen_helper_sve_revb_s
,
2988 gen_helper_sve_revb_d
,
2990 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
2993 static bool trans_REVH(DisasContext
*s
, arg_rpr_esz
*a
)
2995 static gen_helper_gvec_3
* const fns
[4] = {
2998 gen_helper_sve_revh_s
,
2999 gen_helper_sve_revh_d
,
3001 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
3004 static bool trans_REVW(DisasContext
*s
, arg_rpr_esz
*a
)
3006 return do_zpz_ool(s
, a
, a
->esz
== 3 ? gen_helper_sve_revw_d
: NULL
);
3009 static bool trans_RBIT(DisasContext
*s
, arg_rpr_esz
*a
)
3011 static gen_helper_gvec_3
* const fns
[4] = {
3012 gen_helper_sve_rbit_b
,
3013 gen_helper_sve_rbit_h
,
3014 gen_helper_sve_rbit_s
,
3015 gen_helper_sve_rbit_d
,
3017 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
3020 static bool trans_SPLICE(DisasContext
*s
, arg_rprr_esz
*a
)
3022 if (sve_access_check(s
)) {
3023 gen_gvec_ool_zzzp(s
, gen_helper_sve_splice
,
3024 a
->rd
, a
->rn
, a
->rm
, a
->pg
, a
->esz
);
3029 static bool trans_SPLICE_sve2(DisasContext
*s
, arg_rpr_esz
*a
)
3031 if (!dc_isar_feature(aa64_sve2
, s
)) {
3034 if (sve_access_check(s
)) {
3035 gen_gvec_ool_zzzp(s
, gen_helper_sve_splice
,
3036 a
->rd
, a
->rn
, (a
->rn
+ 1) % 32, a
->pg
, a
->esz
);
3042 *** SVE Integer Compare - Vectors Group
3045 static bool do_ppzz_flags(DisasContext
*s
, arg_rprr_esz
*a
,
3046 gen_helper_gvec_flags_4
*gen_fn
)
3048 TCGv_ptr pd
, zn
, zm
, pg
;
3052 if (gen_fn
== NULL
) {
3055 if (!sve_access_check(s
)) {
3059 vsz
= vec_full_reg_size(s
);
3060 t
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
3061 pd
= tcg_temp_new_ptr();
3062 zn
= tcg_temp_new_ptr();
3063 zm
= tcg_temp_new_ptr();
3064 pg
= tcg_temp_new_ptr();
3066 tcg_gen_addi_ptr(pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3067 tcg_gen_addi_ptr(zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
3068 tcg_gen_addi_ptr(zm
, cpu_env
, vec_full_reg_offset(s
, a
->rm
));
3069 tcg_gen_addi_ptr(pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3071 gen_fn(t
, pd
, zn
, zm
, pg
, t
);
3073 tcg_temp_free_ptr(pd
);
3074 tcg_temp_free_ptr(zn
);
3075 tcg_temp_free_ptr(zm
);
3076 tcg_temp_free_ptr(pg
);
3080 tcg_temp_free_i32(t
);
3084 #define DO_PPZZ(NAME, name) \
3085 static bool trans_##NAME##_ppzz(DisasContext *s, arg_rprr_esz *a) \
3087 static gen_helper_gvec_flags_4 * const fns[4] = { \
3088 gen_helper_sve_##name##_ppzz_b, gen_helper_sve_##name##_ppzz_h, \
3089 gen_helper_sve_##name##_ppzz_s, gen_helper_sve_##name##_ppzz_d, \
3091 return do_ppzz_flags(s, a, fns[a->esz]); \
3094 DO_PPZZ(CMPEQ
, cmpeq
)
3095 DO_PPZZ(CMPNE
, cmpne
)
3096 DO_PPZZ(CMPGT
, cmpgt
)
3097 DO_PPZZ(CMPGE
, cmpge
)
3098 DO_PPZZ(CMPHI
, cmphi
)
3099 DO_PPZZ(CMPHS
, cmphs
)
3103 #define DO_PPZW(NAME, name) \
3104 static bool trans_##NAME##_ppzw(DisasContext *s, arg_rprr_esz *a) \
3106 static gen_helper_gvec_flags_4 * const fns[4] = { \
3107 gen_helper_sve_##name##_ppzw_b, gen_helper_sve_##name##_ppzw_h, \
3108 gen_helper_sve_##name##_ppzw_s, NULL \
3110 return do_ppzz_flags(s, a, fns[a->esz]); \
3113 DO_PPZW(CMPEQ
, cmpeq
)
3114 DO_PPZW(CMPNE
, cmpne
)
3115 DO_PPZW(CMPGT
, cmpgt
)
3116 DO_PPZW(CMPGE
, cmpge
)
3117 DO_PPZW(CMPHI
, cmphi
)
3118 DO_PPZW(CMPHS
, cmphs
)
3119 DO_PPZW(CMPLT
, cmplt
)
3120 DO_PPZW(CMPLE
, cmple
)
3121 DO_PPZW(CMPLO
, cmplo
)
3122 DO_PPZW(CMPLS
, cmpls
)
3127 *** SVE Integer Compare - Immediate Groups
3130 static bool do_ppzi_flags(DisasContext
*s
, arg_rpri_esz
*a
,
3131 gen_helper_gvec_flags_3
*gen_fn
)
3133 TCGv_ptr pd
, zn
, pg
;
3137 if (gen_fn
== NULL
) {
3140 if (!sve_access_check(s
)) {
3144 vsz
= vec_full_reg_size(s
);
3145 t
= tcg_const_i32(simd_desc(vsz
, vsz
, a
->imm
));
3146 pd
= tcg_temp_new_ptr();
3147 zn
= tcg_temp_new_ptr();
3148 pg
= tcg_temp_new_ptr();
3150 tcg_gen_addi_ptr(pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3151 tcg_gen_addi_ptr(zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
3152 tcg_gen_addi_ptr(pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3154 gen_fn(t
, pd
, zn
, pg
, t
);
3156 tcg_temp_free_ptr(pd
);
3157 tcg_temp_free_ptr(zn
);
3158 tcg_temp_free_ptr(pg
);
3162 tcg_temp_free_i32(t
);
3166 #define DO_PPZI(NAME, name) \
3167 static bool trans_##NAME##_ppzi(DisasContext *s, arg_rpri_esz *a) \
3169 static gen_helper_gvec_flags_3 * const fns[4] = { \
3170 gen_helper_sve_##name##_ppzi_b, gen_helper_sve_##name##_ppzi_h, \
3171 gen_helper_sve_##name##_ppzi_s, gen_helper_sve_##name##_ppzi_d, \
3173 return do_ppzi_flags(s, a, fns[a->esz]); \
3176 DO_PPZI(CMPEQ
, cmpeq
)
3177 DO_PPZI(CMPNE
, cmpne
)
3178 DO_PPZI(CMPGT
, cmpgt
)
3179 DO_PPZI(CMPGE
, cmpge
)
3180 DO_PPZI(CMPHI
, cmphi
)
3181 DO_PPZI(CMPHS
, cmphs
)
3182 DO_PPZI(CMPLT
, cmplt
)
3183 DO_PPZI(CMPLE
, cmple
)
3184 DO_PPZI(CMPLO
, cmplo
)
3185 DO_PPZI(CMPLS
, cmpls
)
3190 *** SVE Partition Break Group
3193 static bool do_brk3(DisasContext
*s
, arg_rprr_s
*a
,
3194 gen_helper_gvec_4
*fn
, gen_helper_gvec_flags_4
*fn_s
)
3196 if (!sve_access_check(s
)) {
3200 unsigned vsz
= pred_full_reg_size(s
);
3202 /* Predicate sizes may be smaller and cannot use simd_desc. */
3203 TCGv_ptr d
= tcg_temp_new_ptr();
3204 TCGv_ptr n
= tcg_temp_new_ptr();
3205 TCGv_ptr m
= tcg_temp_new_ptr();
3206 TCGv_ptr g
= tcg_temp_new_ptr();
3207 TCGv_i32 t
= tcg_const_i32(FIELD_DP32(0, PREDDESC
, OPRSZ
, vsz
));
3209 tcg_gen_addi_ptr(d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3210 tcg_gen_addi_ptr(n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
3211 tcg_gen_addi_ptr(m
, cpu_env
, pred_full_reg_offset(s
, a
->rm
));
3212 tcg_gen_addi_ptr(g
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3215 fn_s(t
, d
, n
, m
, g
, t
);
3220 tcg_temp_free_ptr(d
);
3221 tcg_temp_free_ptr(n
);
3222 tcg_temp_free_ptr(m
);
3223 tcg_temp_free_ptr(g
);
3224 tcg_temp_free_i32(t
);
3228 static bool do_brk2(DisasContext
*s
, arg_rpr_s
*a
,
3229 gen_helper_gvec_3
*fn
, gen_helper_gvec_flags_3
*fn_s
)
3231 if (!sve_access_check(s
)) {
3235 unsigned vsz
= pred_full_reg_size(s
);
3237 /* Predicate sizes may be smaller and cannot use simd_desc. */
3238 TCGv_ptr d
= tcg_temp_new_ptr();
3239 TCGv_ptr n
= tcg_temp_new_ptr();
3240 TCGv_ptr g
= tcg_temp_new_ptr();
3241 TCGv_i32 t
= tcg_const_i32(FIELD_DP32(0, PREDDESC
, OPRSZ
, vsz
));
3243 tcg_gen_addi_ptr(d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3244 tcg_gen_addi_ptr(n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
3245 tcg_gen_addi_ptr(g
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3248 fn_s(t
, d
, n
, g
, t
);
3253 tcg_temp_free_ptr(d
);
3254 tcg_temp_free_ptr(n
);
3255 tcg_temp_free_ptr(g
);
3256 tcg_temp_free_i32(t
);
3260 static bool trans_BRKPA(DisasContext
*s
, arg_rprr_s
*a
)
3262 return do_brk3(s
, a
, gen_helper_sve_brkpa
, gen_helper_sve_brkpas
);
3265 static bool trans_BRKPB(DisasContext
*s
, arg_rprr_s
*a
)
3267 return do_brk3(s
, a
, gen_helper_sve_brkpb
, gen_helper_sve_brkpbs
);
3270 static bool trans_BRKA_m(DisasContext
*s
, arg_rpr_s
*a
)
3272 return do_brk2(s
, a
, gen_helper_sve_brka_m
, gen_helper_sve_brkas_m
);
3275 static bool trans_BRKB_m(DisasContext
*s
, arg_rpr_s
*a
)
3277 return do_brk2(s
, a
, gen_helper_sve_brkb_m
, gen_helper_sve_brkbs_m
);
3280 static bool trans_BRKA_z(DisasContext
*s
, arg_rpr_s
*a
)
3282 return do_brk2(s
, a
, gen_helper_sve_brka_z
, gen_helper_sve_brkas_z
);
3285 static bool trans_BRKB_z(DisasContext
*s
, arg_rpr_s
*a
)
3287 return do_brk2(s
, a
, gen_helper_sve_brkb_z
, gen_helper_sve_brkbs_z
);
3290 static bool trans_BRKN(DisasContext
*s
, arg_rpr_s
*a
)
3292 return do_brk2(s
, a
, gen_helper_sve_brkn
, gen_helper_sve_brkns
);
3296 *** SVE Predicate Count Group
3299 static void do_cntp(DisasContext
*s
, TCGv_i64 val
, int esz
, int pn
, int pg
)
3301 unsigned psz
= pred_full_reg_size(s
);
3306 tcg_gen_ld_i64(val
, cpu_env
, pred_full_reg_offset(s
, pn
));
3308 TCGv_i64 g
= tcg_temp_new_i64();
3309 tcg_gen_ld_i64(g
, cpu_env
, pred_full_reg_offset(s
, pg
));
3310 tcg_gen_and_i64(val
, val
, g
);
3311 tcg_temp_free_i64(g
);
3314 /* Reduce the pred_esz_masks value simply to reduce the
3315 * size of the code generated here.
3317 psz_mask
= MAKE_64BIT_MASK(0, psz
* 8);
3318 tcg_gen_andi_i64(val
, val
, pred_esz_masks
[esz
] & psz_mask
);
3320 tcg_gen_ctpop_i64(val
, val
);
3322 TCGv_ptr t_pn
= tcg_temp_new_ptr();
3323 TCGv_ptr t_pg
= tcg_temp_new_ptr();
3327 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, psz
);
3328 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, esz
);
3330 tcg_gen_addi_ptr(t_pn
, cpu_env
, pred_full_reg_offset(s
, pn
));
3331 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
3332 t_desc
= tcg_const_i32(desc
);
3334 gen_helper_sve_cntp(val
, t_pn
, t_pg
, t_desc
);
3335 tcg_temp_free_ptr(t_pn
);
3336 tcg_temp_free_ptr(t_pg
);
3337 tcg_temp_free_i32(t_desc
);
3341 static bool trans_CNTP(DisasContext
*s
, arg_CNTP
*a
)
3343 if (sve_access_check(s
)) {
3344 do_cntp(s
, cpu_reg(s
, a
->rd
), a
->esz
, a
->rn
, a
->pg
);
3349 static bool trans_INCDECP_r(DisasContext
*s
, arg_incdec_pred
*a
)
3351 if (sve_access_check(s
)) {
3352 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3353 TCGv_i64 val
= tcg_temp_new_i64();
3355 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3357 tcg_gen_sub_i64(reg
, reg
, val
);
3359 tcg_gen_add_i64(reg
, reg
, val
);
3361 tcg_temp_free_i64(val
);
3366 static bool trans_INCDECP_z(DisasContext
*s
, arg_incdec2_pred
*a
)
3371 if (sve_access_check(s
)) {
3372 unsigned vsz
= vec_full_reg_size(s
);
3373 TCGv_i64 val
= tcg_temp_new_i64();
3374 GVecGen2sFn
*gvec_fn
= a
->d
? tcg_gen_gvec_subs
: tcg_gen_gvec_adds
;
3376 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3377 gvec_fn(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3378 vec_full_reg_offset(s
, a
->rn
), val
, vsz
, vsz
);
3383 static bool trans_SINCDECP_r_32(DisasContext
*s
, arg_incdec_pred
*a
)
3385 if (sve_access_check(s
)) {
3386 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3387 TCGv_i64 val
= tcg_temp_new_i64();
3389 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3390 do_sat_addsub_32(reg
, val
, a
->u
, a
->d
);
3395 static bool trans_SINCDECP_r_64(DisasContext
*s
, arg_incdec_pred
*a
)
3397 if (sve_access_check(s
)) {
3398 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3399 TCGv_i64 val
= tcg_temp_new_i64();
3401 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3402 do_sat_addsub_64(reg
, val
, a
->u
, a
->d
);
3407 static bool trans_SINCDECP_z(DisasContext
*s
, arg_incdec2_pred
*a
)
3412 if (sve_access_check(s
)) {
3413 TCGv_i64 val
= tcg_temp_new_i64();
3414 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3415 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
, val
, a
->u
, a
->d
);
3421 *** SVE Integer Compare Scalars Group
3424 static bool trans_CTERM(DisasContext
*s
, arg_CTERM
*a
)
3426 if (!sve_access_check(s
)) {
3430 TCGCond cond
= (a
->ne
? TCG_COND_NE
: TCG_COND_EQ
);
3431 TCGv_i64 rn
= read_cpu_reg(s
, a
->rn
, a
->sf
);
3432 TCGv_i64 rm
= read_cpu_reg(s
, a
->rm
, a
->sf
);
3433 TCGv_i64 cmp
= tcg_temp_new_i64();
3435 tcg_gen_setcond_i64(cond
, cmp
, rn
, rm
);
3436 tcg_gen_extrl_i64_i32(cpu_NF
, cmp
);
3437 tcg_temp_free_i64(cmp
);
3439 /* VF = !NF & !CF. */
3440 tcg_gen_xori_i32(cpu_VF
, cpu_NF
, 1);
3441 tcg_gen_andc_i32(cpu_VF
, cpu_VF
, cpu_CF
);
3443 /* Both NF and VF actually look at bit 31. */
3444 tcg_gen_neg_i32(cpu_NF
, cpu_NF
);
3445 tcg_gen_neg_i32(cpu_VF
, cpu_VF
);
3449 static bool trans_WHILE(DisasContext
*s
, arg_WHILE
*a
)
3451 TCGv_i64 op0
, op1
, t0
, t1
, tmax
;
3454 unsigned vsz
= vec_full_reg_size(s
);
3458 /* Note that GE/HS has a->eq == 0 and GT/HI has a->eq == 1. */
3459 bool eq
= a
->eq
== a
->lt
;
3461 /* The greater-than conditions are all SVE2. */
3462 if (!a
->lt
&& !dc_isar_feature(aa64_sve2
, s
)) {
3465 if (!sve_access_check(s
)) {
3469 op0
= read_cpu_reg(s
, a
->rn
, 1);
3470 op1
= read_cpu_reg(s
, a
->rm
, 1);
3474 tcg_gen_ext32u_i64(op0
, op0
);
3475 tcg_gen_ext32u_i64(op1
, op1
);
3477 tcg_gen_ext32s_i64(op0
, op0
);
3478 tcg_gen_ext32s_i64(op1
, op1
);
3482 /* For the helper, compress the different conditions into a computation
3483 * of how many iterations for which the condition is true.
3485 t0
= tcg_temp_new_i64();
3486 t1
= tcg_temp_new_i64();
3489 tcg_gen_sub_i64(t0
, op1
, op0
);
3491 maxval
= a
->sf
? UINT64_MAX
: UINT32_MAX
;
3492 cond
= eq
? TCG_COND_LEU
: TCG_COND_LTU
;
3494 maxval
= a
->sf
? INT64_MAX
: INT32_MAX
;
3495 cond
= eq
? TCG_COND_LE
: TCG_COND_LT
;
3498 tcg_gen_sub_i64(t0
, op0
, op1
);
3501 cond
= eq
? TCG_COND_GEU
: TCG_COND_GTU
;
3503 maxval
= a
->sf
? INT64_MIN
: INT32_MIN
;
3504 cond
= eq
? TCG_COND_GE
: TCG_COND_GT
;
3508 tmax
= tcg_const_i64(vsz
>> a
->esz
);
3510 /* Equality means one more iteration. */
3511 tcg_gen_addi_i64(t0
, t0
, 1);
3514 * For the less-than while, if op1 is maxval (and the only time
3515 * the addition above could overflow), then we produce an all-true
3516 * predicate by setting the count to the vector length. This is
3517 * because the pseudocode is described as an increment + compare
3518 * loop, and the maximum integer would always compare true.
3519 * Similarly, the greater-than while has the same issue with the
3520 * minimum integer due to the decrement + compare loop.
3522 tcg_gen_movi_i64(t1
, maxval
);
3523 tcg_gen_movcond_i64(TCG_COND_EQ
, t0
, op1
, t1
, tmax
, t0
);
3526 /* Bound to the maximum. */
3527 tcg_gen_umin_i64(t0
, t0
, tmax
);
3528 tcg_temp_free_i64(tmax
);
3530 /* Set the count to zero if the condition is false. */
3531 tcg_gen_movi_i64(t1
, 0);
3532 tcg_gen_movcond_i64(cond
, t0
, op0
, op1
, t0
, t1
);
3533 tcg_temp_free_i64(t1
);
3535 /* Since we're bounded, pass as a 32-bit type. */
3536 t2
= tcg_temp_new_i32();
3537 tcg_gen_extrl_i64_i32(t2
, t0
);
3538 tcg_temp_free_i64(t0
);
3540 /* Scale elements to bits. */
3541 tcg_gen_shli_i32(t2
, t2
, a
->esz
);
3543 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
/ 8);
3544 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
3545 t3
= tcg_const_i32(desc
);
3547 ptr
= tcg_temp_new_ptr();
3548 tcg_gen_addi_ptr(ptr
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3551 gen_helper_sve_whilel(t2
, ptr
, t2
, t3
);
3553 gen_helper_sve_whileg(t2
, ptr
, t2
, t3
);
3557 tcg_temp_free_ptr(ptr
);
3558 tcg_temp_free_i32(t2
);
3559 tcg_temp_free_i32(t3
);
3563 static bool trans_WHILE_ptr(DisasContext
*s
, arg_WHILE_ptr
*a
)
3565 TCGv_i64 op0
, op1
, diff
, t1
, tmax
;
3568 unsigned vsz
= vec_full_reg_size(s
);
3571 if (!dc_isar_feature(aa64_sve2
, s
)) {
3574 if (!sve_access_check(s
)) {
3578 op0
= read_cpu_reg(s
, a
->rn
, 1);
3579 op1
= read_cpu_reg(s
, a
->rm
, 1);
3581 tmax
= tcg_const_i64(vsz
);
3582 diff
= tcg_temp_new_i64();
3586 /* diff = abs(op1 - op0), noting that op0/1 are unsigned. */
3587 t1
= tcg_temp_new_i64();
3588 tcg_gen_sub_i64(diff
, op0
, op1
);
3589 tcg_gen_sub_i64(t1
, op1
, op0
);
3590 tcg_gen_movcond_i64(TCG_COND_GEU
, diff
, op0
, op1
, diff
, t1
);
3591 tcg_temp_free_i64(t1
);
3592 /* Round down to a multiple of ESIZE. */
3593 tcg_gen_andi_i64(diff
, diff
, -1 << a
->esz
);
3594 /* If op1 == op0, diff == 0, and the condition is always true. */
3595 tcg_gen_movcond_i64(TCG_COND_EQ
, diff
, op0
, op1
, tmax
, diff
);
3598 tcg_gen_sub_i64(diff
, op1
, op0
);
3599 /* Round down to a multiple of ESIZE. */
3600 tcg_gen_andi_i64(diff
, diff
, -1 << a
->esz
);
3601 /* If op0 >= op1, diff <= 0, the condition is always true. */
3602 tcg_gen_movcond_i64(TCG_COND_GEU
, diff
, op0
, op1
, tmax
, diff
);
3605 /* Bound to the maximum. */
3606 tcg_gen_umin_i64(diff
, diff
, tmax
);
3607 tcg_temp_free_i64(tmax
);
3609 /* Since we're bounded, pass as a 32-bit type. */
3610 t2
= tcg_temp_new_i32();
3611 tcg_gen_extrl_i64_i32(t2
, diff
);
3612 tcg_temp_free_i64(diff
);
3614 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
/ 8);
3615 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
3616 t3
= tcg_const_i32(desc
);
3618 ptr
= tcg_temp_new_ptr();
3619 tcg_gen_addi_ptr(ptr
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3621 gen_helper_sve_whilel(t2
, ptr
, t2
, t3
);
3624 tcg_temp_free_ptr(ptr
);
3625 tcg_temp_free_i32(t2
);
3626 tcg_temp_free_i32(t3
);
3631 *** SVE Integer Wide Immediate - Unpredicated Group
3634 static bool trans_FDUP(DisasContext
*s
, arg_FDUP
*a
)
3639 if (sve_access_check(s
)) {
3640 unsigned vsz
= vec_full_reg_size(s
);
3641 int dofs
= vec_full_reg_offset(s
, a
->rd
);
3644 /* Decode the VFP immediate. */
3645 imm
= vfp_expand_imm(a
->esz
, a
->imm
);
3646 tcg_gen_gvec_dup_imm(a
->esz
, dofs
, vsz
, vsz
, imm
);
3651 static bool trans_DUP_i(DisasContext
*s
, arg_DUP_i
*a
)
3653 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3656 if (sve_access_check(s
)) {
3657 unsigned vsz
= vec_full_reg_size(s
);
3658 int dofs
= vec_full_reg_offset(s
, a
->rd
);
3660 tcg_gen_gvec_dup_imm(a
->esz
, dofs
, vsz
, vsz
, a
->imm
);
3665 static bool trans_ADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3667 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3670 if (sve_access_check(s
)) {
3671 unsigned vsz
= vec_full_reg_size(s
);
3672 tcg_gen_gvec_addi(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3673 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
3678 static bool trans_SUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3681 return trans_ADD_zzi(s
, a
);
3684 static bool trans_SUBR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3686 static const TCGOpcode vecop_list
[] = { INDEX_op_sub_vec
, 0 };
3687 static const GVecGen2s op
[4] = {
3688 { .fni8
= tcg_gen_vec_sub8_i64
,
3689 .fniv
= tcg_gen_sub_vec
,
3690 .fno
= gen_helper_sve_subri_b
,
3691 .opt_opc
= vecop_list
,
3693 .scalar_first
= true },
3694 { .fni8
= tcg_gen_vec_sub16_i64
,
3695 .fniv
= tcg_gen_sub_vec
,
3696 .fno
= gen_helper_sve_subri_h
,
3697 .opt_opc
= vecop_list
,
3699 .scalar_first
= true },
3700 { .fni4
= tcg_gen_sub_i32
,
3701 .fniv
= tcg_gen_sub_vec
,
3702 .fno
= gen_helper_sve_subri_s
,
3703 .opt_opc
= vecop_list
,
3705 .scalar_first
= true },
3706 { .fni8
= tcg_gen_sub_i64
,
3707 .fniv
= tcg_gen_sub_vec
,
3708 .fno
= gen_helper_sve_subri_d
,
3709 .opt_opc
= vecop_list
,
3710 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
3712 .scalar_first
= true }
3715 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3718 if (sve_access_check(s
)) {
3719 unsigned vsz
= vec_full_reg_size(s
);
3720 TCGv_i64 c
= tcg_const_i64(a
->imm
);
3721 tcg_gen_gvec_2s(vec_full_reg_offset(s
, a
->rd
),
3722 vec_full_reg_offset(s
, a
->rn
),
3723 vsz
, vsz
, c
, &op
[a
->esz
]);
3724 tcg_temp_free_i64(c
);
3729 static bool trans_MUL_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3731 if (sve_access_check(s
)) {
3732 unsigned vsz
= vec_full_reg_size(s
);
3733 tcg_gen_gvec_muli(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3734 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
3739 static bool do_zzi_sat(DisasContext
*s
, arg_rri_esz
*a
, bool u
, bool d
)
3741 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3744 if (sve_access_check(s
)) {
3745 TCGv_i64 val
= tcg_const_i64(a
->imm
);
3746 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
, val
, u
, d
);
3747 tcg_temp_free_i64(val
);
3752 static bool trans_SQADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3754 return do_zzi_sat(s
, a
, false, false);
3757 static bool trans_UQADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3759 return do_zzi_sat(s
, a
, true, false);
3762 static bool trans_SQSUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3764 return do_zzi_sat(s
, a
, false, true);
3767 static bool trans_UQSUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3769 return do_zzi_sat(s
, a
, true, true);
3772 static bool do_zzi_ool(DisasContext
*s
, arg_rri_esz
*a
, gen_helper_gvec_2i
*fn
)
3774 if (sve_access_check(s
)) {
3775 unsigned vsz
= vec_full_reg_size(s
);
3776 TCGv_i64 c
= tcg_const_i64(a
->imm
);
3778 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s
, a
->rd
),
3779 vec_full_reg_offset(s
, a
->rn
),
3780 c
, vsz
, vsz
, 0, fn
);
3781 tcg_temp_free_i64(c
);
3786 #define DO_ZZI(NAME, name) \
3787 static bool trans_##NAME##_zzi(DisasContext *s, arg_rri_esz *a) \
3789 static gen_helper_gvec_2i * const fns[4] = { \
3790 gen_helper_sve_##name##i_b, gen_helper_sve_##name##i_h, \
3791 gen_helper_sve_##name##i_s, gen_helper_sve_##name##i_d, \
3793 return do_zzi_ool(s, a, fns[a->esz]); \
3803 static bool trans_DOT_zzzz(DisasContext
*s
, arg_DOT_zzzz
*a
)
3805 static gen_helper_gvec_4
* const fns
[2][2] = {
3806 { gen_helper_gvec_sdot_b
, gen_helper_gvec_sdot_h
},
3807 { gen_helper_gvec_udot_b
, gen_helper_gvec_udot_h
}
3810 if (sve_access_check(s
)) {
3811 gen_gvec_ool_zzzz(s
, fns
[a
->u
][a
->sz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, 0);
3817 * SVE Multiply - Indexed
3820 static bool do_zzxz_ool(DisasContext
*s
, arg_rrxr_esz
*a
,
3821 gen_helper_gvec_4
*fn
)
3826 if (sve_access_check(s
)) {
3827 gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->index
);
3832 #define DO_RRXR(NAME, FUNC) \
3833 static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
3834 { return do_zzxz_ool(s, a, FUNC); }
3836 DO_RRXR(trans_SDOT_zzxw_s
, gen_helper_gvec_sdot_idx_b
)
3837 DO_RRXR(trans_SDOT_zzxw_d
, gen_helper_gvec_sdot_idx_h
)
3838 DO_RRXR(trans_UDOT_zzxw_s
, gen_helper_gvec_udot_idx_b
)
3839 DO_RRXR(trans_UDOT_zzxw_d
, gen_helper_gvec_udot_idx_h
)
3843 static bool do_sve2_zzz_data(DisasContext
*s
, int rd
, int rn
, int rm
, int data
,
3844 gen_helper_gvec_3
*fn
)
3846 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
3849 if (sve_access_check(s
)) {
3850 unsigned vsz
= vec_full_reg_size(s
);
3851 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
3852 vec_full_reg_offset(s
, rn
),
3853 vec_full_reg_offset(s
, rm
),
3854 vsz
, vsz
, data
, fn
);
3859 #define DO_SVE2_RRX(NAME, FUNC) \
3860 static bool NAME(DisasContext *s, arg_rrx_esz *a) \
3861 { return do_sve2_zzz_data(s, a->rd, a->rn, a->rm, a->index, FUNC); }
3863 DO_SVE2_RRX(trans_MUL_zzx_h
, gen_helper_gvec_mul_idx_h
)
3864 DO_SVE2_RRX(trans_MUL_zzx_s
, gen_helper_gvec_mul_idx_s
)
3865 DO_SVE2_RRX(trans_MUL_zzx_d
, gen_helper_gvec_mul_idx_d
)
3869 #define DO_SVE2_RRX_TB(NAME, FUNC, TOP) \
3870 static bool NAME(DisasContext *s, arg_rrx_esz *a) \
3872 return do_sve2_zzz_data(s, a->rd, a->rn, a->rm, \
3873 (a->index << 1) | TOP, FUNC); \
3876 DO_SVE2_RRX_TB(trans_SQDMULLB_zzx_s
, gen_helper_sve2_sqdmull_idx_s
, false)
3877 DO_SVE2_RRX_TB(trans_SQDMULLB_zzx_d
, gen_helper_sve2_sqdmull_idx_d
, false)
3878 DO_SVE2_RRX_TB(trans_SQDMULLT_zzx_s
, gen_helper_sve2_sqdmull_idx_s
, true)
3879 DO_SVE2_RRX_TB(trans_SQDMULLT_zzx_d
, gen_helper_sve2_sqdmull_idx_d
, true)
3881 #undef DO_SVE2_RRX_TB
3883 static bool do_sve2_zzzz_data(DisasContext
*s
, int rd
, int rn
, int rm
, int ra
,
3884 int data
, gen_helper_gvec_4
*fn
)
3886 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
3889 if (sve_access_check(s
)) {
3890 unsigned vsz
= vec_full_reg_size(s
);
3891 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
3892 vec_full_reg_offset(s
, rn
),
3893 vec_full_reg_offset(s
, rm
),
3894 vec_full_reg_offset(s
, ra
),
3895 vsz
, vsz
, data
, fn
);
3900 #define DO_SVE2_RRXR(NAME, FUNC) \
3901 static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
3902 { return do_sve2_zzzz_data(s, a->rd, a->rn, a->rm, a->ra, a->index, FUNC); }
3904 DO_SVE2_RRXR(trans_MLA_zzxz_h
, gen_helper_gvec_mla_idx_h
)
3905 DO_SVE2_RRXR(trans_MLA_zzxz_s
, gen_helper_gvec_mla_idx_s
)
3906 DO_SVE2_RRXR(trans_MLA_zzxz_d
, gen_helper_gvec_mla_idx_d
)
3908 DO_SVE2_RRXR(trans_MLS_zzxz_h
, gen_helper_gvec_mls_idx_h
)
3909 DO_SVE2_RRXR(trans_MLS_zzxz_s
, gen_helper_gvec_mls_idx_s
)
3910 DO_SVE2_RRXR(trans_MLS_zzxz_d
, gen_helper_gvec_mls_idx_d
)
3912 DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_h
, gen_helper_sve2_sqrdmlah_idx_h
)
3913 DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_s
, gen_helper_sve2_sqrdmlah_idx_s
)
3914 DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_d
, gen_helper_sve2_sqrdmlah_idx_d
)
3916 DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_h
, gen_helper_sve2_sqrdmlsh_idx_h
)
3917 DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_s
, gen_helper_sve2_sqrdmlsh_idx_s
)
3918 DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_d
, gen_helper_sve2_sqrdmlsh_idx_d
)
3922 #define DO_SVE2_RRXR_TB(NAME, FUNC, TOP) \
3923 static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
3925 return do_sve2_zzzz_data(s, a->rd, a->rn, a->rm, a->rd, \
3926 (a->index << 1) | TOP, FUNC); \
3929 DO_SVE2_RRXR_TB(trans_SQDMLALB_zzxw_s
, gen_helper_sve2_sqdmlal_idx_s
, false)
3930 DO_SVE2_RRXR_TB(trans_SQDMLALB_zzxw_d
, gen_helper_sve2_sqdmlal_idx_d
, false)
3931 DO_SVE2_RRXR_TB(trans_SQDMLALT_zzxw_s
, gen_helper_sve2_sqdmlal_idx_s
, true)
3932 DO_SVE2_RRXR_TB(trans_SQDMLALT_zzxw_d
, gen_helper_sve2_sqdmlal_idx_d
, true)
3934 DO_SVE2_RRXR_TB(trans_SQDMLSLB_zzxw_s
, gen_helper_sve2_sqdmlsl_idx_s
, false)
3935 DO_SVE2_RRXR_TB(trans_SQDMLSLB_zzxw_d
, gen_helper_sve2_sqdmlsl_idx_d
, false)
3936 DO_SVE2_RRXR_TB(trans_SQDMLSLT_zzxw_s
, gen_helper_sve2_sqdmlsl_idx_s
, true)
3937 DO_SVE2_RRXR_TB(trans_SQDMLSLT_zzxw_d
, gen_helper_sve2_sqdmlsl_idx_d
, true)
3939 #undef DO_SVE2_RRXR_TB
3942 *** SVE Floating Point Multiply-Add Indexed Group
3945 static bool do_FMLA_zzxz(DisasContext
*s
, arg_rrxr_esz
*a
, bool sub
)
3947 static gen_helper_gvec_4_ptr
* const fns
[3] = {
3948 gen_helper_gvec_fmla_idx_h
,
3949 gen_helper_gvec_fmla_idx_s
,
3950 gen_helper_gvec_fmla_idx_d
,
3953 if (sve_access_check(s
)) {
3954 unsigned vsz
= vec_full_reg_size(s
);
3955 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3956 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
3957 vec_full_reg_offset(s
, a
->rn
),
3958 vec_full_reg_offset(s
, a
->rm
),
3959 vec_full_reg_offset(s
, a
->ra
),
3960 status
, vsz
, vsz
, (a
->index
<< 1) | sub
,
3962 tcg_temp_free_ptr(status
);
3967 static bool trans_FMLA_zzxz(DisasContext
*s
, arg_FMLA_zzxz
*a
)
3969 return do_FMLA_zzxz(s
, a
, false);
3972 static bool trans_FMLS_zzxz(DisasContext
*s
, arg_FMLA_zzxz
*a
)
3974 return do_FMLA_zzxz(s
, a
, true);
3978 *** SVE Floating Point Multiply Indexed Group
3981 static bool trans_FMUL_zzx(DisasContext
*s
, arg_FMUL_zzx
*a
)
3983 static gen_helper_gvec_3_ptr
* const fns
[3] = {
3984 gen_helper_gvec_fmul_idx_h
,
3985 gen_helper_gvec_fmul_idx_s
,
3986 gen_helper_gvec_fmul_idx_d
,
3989 if (sve_access_check(s
)) {
3990 unsigned vsz
= vec_full_reg_size(s
);
3991 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3992 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
3993 vec_full_reg_offset(s
, a
->rn
),
3994 vec_full_reg_offset(s
, a
->rm
),
3995 status
, vsz
, vsz
, a
->index
, fns
[a
->esz
- 1]);
3996 tcg_temp_free_ptr(status
);
4002 *** SVE Floating Point Fast Reduction Group
4005 typedef void gen_helper_fp_reduce(TCGv_i64
, TCGv_ptr
, TCGv_ptr
,
4006 TCGv_ptr
, TCGv_i32
);
4008 static void do_reduce(DisasContext
*s
, arg_rpr_esz
*a
,
4009 gen_helper_fp_reduce
*fn
)
4011 unsigned vsz
= vec_full_reg_size(s
);
4012 unsigned p2vsz
= pow2ceil(vsz
);
4013 TCGv_i32 t_desc
= tcg_const_i32(simd_desc(vsz
, vsz
, p2vsz
));
4014 TCGv_ptr t_zn
, t_pg
, status
;
4017 temp
= tcg_temp_new_i64();
4018 t_zn
= tcg_temp_new_ptr();
4019 t_pg
= tcg_temp_new_ptr();
4021 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
4022 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
4023 status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4025 fn(temp
, t_zn
, t_pg
, status
, t_desc
);
4026 tcg_temp_free_ptr(t_zn
);
4027 tcg_temp_free_ptr(t_pg
);
4028 tcg_temp_free_ptr(status
);
4029 tcg_temp_free_i32(t_desc
);
4031 write_fp_dreg(s
, a
->rd
, temp
);
4032 tcg_temp_free_i64(temp
);
4035 #define DO_VPZ(NAME, name) \
4036 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
4038 static gen_helper_fp_reduce * const fns[3] = { \
4039 gen_helper_sve_##name##_h, \
4040 gen_helper_sve_##name##_s, \
4041 gen_helper_sve_##name##_d, \
4043 if (a->esz == 0) { \
4046 if (sve_access_check(s)) { \
4047 do_reduce(s, a, fns[a->esz - 1]); \
4052 DO_VPZ(FADDV
, faddv
)
4053 DO_VPZ(FMINNMV
, fminnmv
)
4054 DO_VPZ(FMAXNMV
, fmaxnmv
)
4055 DO_VPZ(FMINV
, fminv
)
4056 DO_VPZ(FMAXV
, fmaxv
)
4059 *** SVE Floating Point Unary Operations - Unpredicated Group
4062 static void do_zz_fp(DisasContext
*s
, arg_rr_esz
*a
, gen_helper_gvec_2_ptr
*fn
)
4064 unsigned vsz
= vec_full_reg_size(s
);
4065 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4067 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s
, a
->rd
),
4068 vec_full_reg_offset(s
, a
->rn
),
4069 status
, vsz
, vsz
, 0, fn
);
4070 tcg_temp_free_ptr(status
);
4073 static bool trans_FRECPE(DisasContext
*s
, arg_rr_esz
*a
)
4075 static gen_helper_gvec_2_ptr
* const fns
[3] = {
4076 gen_helper_gvec_frecpe_h
,
4077 gen_helper_gvec_frecpe_s
,
4078 gen_helper_gvec_frecpe_d
,
4083 if (sve_access_check(s
)) {
4084 do_zz_fp(s
, a
, fns
[a
->esz
- 1]);
4089 static bool trans_FRSQRTE(DisasContext
*s
, arg_rr_esz
*a
)
4091 static gen_helper_gvec_2_ptr
* const fns
[3] = {
4092 gen_helper_gvec_frsqrte_h
,
4093 gen_helper_gvec_frsqrte_s
,
4094 gen_helper_gvec_frsqrte_d
,
4099 if (sve_access_check(s
)) {
4100 do_zz_fp(s
, a
, fns
[a
->esz
- 1]);
4106 *** SVE Floating Point Compare with Zero Group
4109 static void do_ppz_fp(DisasContext
*s
, arg_rpr_esz
*a
,
4110 gen_helper_gvec_3_ptr
*fn
)
4112 unsigned vsz
= vec_full_reg_size(s
);
4113 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4115 tcg_gen_gvec_3_ptr(pred_full_reg_offset(s
, a
->rd
),
4116 vec_full_reg_offset(s
, a
->rn
),
4117 pred_full_reg_offset(s
, a
->pg
),
4118 status
, vsz
, vsz
, 0, fn
);
4119 tcg_temp_free_ptr(status
);
4122 #define DO_PPZ(NAME, name) \
4123 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
4125 static gen_helper_gvec_3_ptr * const fns[3] = { \
4126 gen_helper_sve_##name##_h, \
4127 gen_helper_sve_##name##_s, \
4128 gen_helper_sve_##name##_d, \
4130 if (a->esz == 0) { \
4133 if (sve_access_check(s)) { \
4134 do_ppz_fp(s, a, fns[a->esz - 1]); \
4139 DO_PPZ(FCMGE_ppz0
, fcmge0
)
4140 DO_PPZ(FCMGT_ppz0
, fcmgt0
)
4141 DO_PPZ(FCMLE_ppz0
, fcmle0
)
4142 DO_PPZ(FCMLT_ppz0
, fcmlt0
)
4143 DO_PPZ(FCMEQ_ppz0
, fcmeq0
)
4144 DO_PPZ(FCMNE_ppz0
, fcmne0
)
4149 *** SVE floating-point trig multiply-add coefficient
4152 static bool trans_FTMAD(DisasContext
*s
, arg_FTMAD
*a
)
4154 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4155 gen_helper_sve_ftmad_h
,
4156 gen_helper_sve_ftmad_s
,
4157 gen_helper_sve_ftmad_d
,
4163 if (sve_access_check(s
)) {
4164 unsigned vsz
= vec_full_reg_size(s
);
4165 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4166 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4167 vec_full_reg_offset(s
, a
->rn
),
4168 vec_full_reg_offset(s
, a
->rm
),
4169 status
, vsz
, vsz
, a
->imm
, fns
[a
->esz
- 1]);
4170 tcg_temp_free_ptr(status
);
4176 *** SVE Floating Point Accumulating Reduction Group
4179 static bool trans_FADDA(DisasContext
*s
, arg_rprr_esz
*a
)
4181 typedef void fadda_fn(TCGv_i64
, TCGv_i64
, TCGv_ptr
,
4182 TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
4183 static fadda_fn
* const fns
[3] = {
4184 gen_helper_sve_fadda_h
,
4185 gen_helper_sve_fadda_s
,
4186 gen_helper_sve_fadda_d
,
4188 unsigned vsz
= vec_full_reg_size(s
);
4189 TCGv_ptr t_rm
, t_pg
, t_fpst
;
4196 if (!sve_access_check(s
)) {
4200 t_val
= load_esz(cpu_env
, vec_reg_offset(s
, a
->rn
, 0, a
->esz
), a
->esz
);
4201 t_rm
= tcg_temp_new_ptr();
4202 t_pg
= tcg_temp_new_ptr();
4203 tcg_gen_addi_ptr(t_rm
, cpu_env
, vec_full_reg_offset(s
, a
->rm
));
4204 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
4205 t_fpst
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4206 t_desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
4208 fns
[a
->esz
- 1](t_val
, t_val
, t_rm
, t_pg
, t_fpst
, t_desc
);
4210 tcg_temp_free_i32(t_desc
);
4211 tcg_temp_free_ptr(t_fpst
);
4212 tcg_temp_free_ptr(t_pg
);
4213 tcg_temp_free_ptr(t_rm
);
4215 write_fp_dreg(s
, a
->rd
, t_val
);
4216 tcg_temp_free_i64(t_val
);
4221 *** SVE Floating Point Arithmetic - Unpredicated Group
4224 static bool do_zzz_fp(DisasContext
*s
, arg_rrr_esz
*a
,
4225 gen_helper_gvec_3_ptr
*fn
)
4230 if (sve_access_check(s
)) {
4231 unsigned vsz
= vec_full_reg_size(s
);
4232 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4233 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4234 vec_full_reg_offset(s
, a
->rn
),
4235 vec_full_reg_offset(s
, a
->rm
),
4236 status
, vsz
, vsz
, 0, fn
);
4237 tcg_temp_free_ptr(status
);
4243 #define DO_FP3(NAME, name) \
4244 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
4246 static gen_helper_gvec_3_ptr * const fns[4] = { \
4247 NULL, gen_helper_gvec_##name##_h, \
4248 gen_helper_gvec_##name##_s, gen_helper_gvec_##name##_d \
4250 return do_zzz_fp(s, a, fns[a->esz]); \
4253 DO_FP3(FADD_zzz
, fadd
)
4254 DO_FP3(FSUB_zzz
, fsub
)
4255 DO_FP3(FMUL_zzz
, fmul
)
4256 DO_FP3(FTSMUL
, ftsmul
)
4257 DO_FP3(FRECPS
, recps
)
4258 DO_FP3(FRSQRTS
, rsqrts
)
4263 *** SVE Floating Point Arithmetic - Predicated Group
4266 static bool do_zpzz_fp(DisasContext
*s
, arg_rprr_esz
*a
,
4267 gen_helper_gvec_4_ptr
*fn
)
4272 if (sve_access_check(s
)) {
4273 unsigned vsz
= vec_full_reg_size(s
);
4274 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4275 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4276 vec_full_reg_offset(s
, a
->rn
),
4277 vec_full_reg_offset(s
, a
->rm
),
4278 pred_full_reg_offset(s
, a
->pg
),
4279 status
, vsz
, vsz
, 0, fn
);
4280 tcg_temp_free_ptr(status
);
4285 #define DO_FP3(NAME, name) \
4286 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
4288 static gen_helper_gvec_4_ptr * const fns[4] = { \
4289 NULL, gen_helper_sve_##name##_h, \
4290 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4292 return do_zpzz_fp(s, a, fns[a->esz]); \
4295 DO_FP3(FADD_zpzz
, fadd
)
4296 DO_FP3(FSUB_zpzz
, fsub
)
4297 DO_FP3(FMUL_zpzz
, fmul
)
4298 DO_FP3(FMIN_zpzz
, fmin
)
4299 DO_FP3(FMAX_zpzz
, fmax
)
4300 DO_FP3(FMINNM_zpzz
, fminnum
)
4301 DO_FP3(FMAXNM_zpzz
, fmaxnum
)
4303 DO_FP3(FSCALE
, fscalbn
)
4305 DO_FP3(FMULX
, fmulx
)
4309 typedef void gen_helper_sve_fp2scalar(TCGv_ptr
, TCGv_ptr
, TCGv_ptr
,
4310 TCGv_i64
, TCGv_ptr
, TCGv_i32
);
4312 static void do_fp_scalar(DisasContext
*s
, int zd
, int zn
, int pg
, bool is_fp16
,
4313 TCGv_i64 scalar
, gen_helper_sve_fp2scalar
*fn
)
4315 unsigned vsz
= vec_full_reg_size(s
);
4316 TCGv_ptr t_zd
, t_zn
, t_pg
, status
;
4319 t_zd
= tcg_temp_new_ptr();
4320 t_zn
= tcg_temp_new_ptr();
4321 t_pg
= tcg_temp_new_ptr();
4322 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, zd
));
4323 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, zn
));
4324 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
4326 status
= fpstatus_ptr(is_fp16
? FPST_FPCR_F16
: FPST_FPCR
);
4327 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
4328 fn(t_zd
, t_zn
, t_pg
, scalar
, status
, desc
);
4330 tcg_temp_free_i32(desc
);
4331 tcg_temp_free_ptr(status
);
4332 tcg_temp_free_ptr(t_pg
);
4333 tcg_temp_free_ptr(t_zn
);
4334 tcg_temp_free_ptr(t_zd
);
4337 static void do_fp_imm(DisasContext
*s
, arg_rpri_esz
*a
, uint64_t imm
,
4338 gen_helper_sve_fp2scalar
*fn
)
4340 TCGv_i64 temp
= tcg_const_i64(imm
);
4341 do_fp_scalar(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, temp
, fn
);
4342 tcg_temp_free_i64(temp
);
4345 #define DO_FP_IMM(NAME, name, const0, const1) \
4346 static bool trans_##NAME##_zpzi(DisasContext *s, arg_rpri_esz *a) \
4348 static gen_helper_sve_fp2scalar * const fns[3] = { \
4349 gen_helper_sve_##name##_h, \
4350 gen_helper_sve_##name##_s, \
4351 gen_helper_sve_##name##_d \
4353 static uint64_t const val[3][2] = { \
4354 { float16_##const0, float16_##const1 }, \
4355 { float32_##const0, float32_##const1 }, \
4356 { float64_##const0, float64_##const1 }, \
4358 if (a->esz == 0) { \
4361 if (sve_access_check(s)) { \
4362 do_fp_imm(s, a, val[a->esz - 1][a->imm], fns[a->esz - 1]); \
4367 DO_FP_IMM(FADD
, fadds
, half
, one
)
4368 DO_FP_IMM(FSUB
, fsubs
, half
, one
)
4369 DO_FP_IMM(FMUL
, fmuls
, half
, two
)
4370 DO_FP_IMM(FSUBR
, fsubrs
, half
, one
)
4371 DO_FP_IMM(FMAXNM
, fmaxnms
, zero
, one
)
4372 DO_FP_IMM(FMINNM
, fminnms
, zero
, one
)
4373 DO_FP_IMM(FMAX
, fmaxs
, zero
, one
)
4374 DO_FP_IMM(FMIN
, fmins
, zero
, one
)
4378 static bool do_fp_cmp(DisasContext
*s
, arg_rprr_esz
*a
,
4379 gen_helper_gvec_4_ptr
*fn
)
4384 if (sve_access_check(s
)) {
4385 unsigned vsz
= vec_full_reg_size(s
);
4386 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4387 tcg_gen_gvec_4_ptr(pred_full_reg_offset(s
, a
->rd
),
4388 vec_full_reg_offset(s
, a
->rn
),
4389 vec_full_reg_offset(s
, a
->rm
),
4390 pred_full_reg_offset(s
, a
->pg
),
4391 status
, vsz
, vsz
, 0, fn
);
4392 tcg_temp_free_ptr(status
);
4397 #define DO_FPCMP(NAME, name) \
4398 static bool trans_##NAME##_ppzz(DisasContext *s, arg_rprr_esz *a) \
4400 static gen_helper_gvec_4_ptr * const fns[4] = { \
4401 NULL, gen_helper_sve_##name##_h, \
4402 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4404 return do_fp_cmp(s, a, fns[a->esz]); \
4407 DO_FPCMP(FCMGE
, fcmge
)
4408 DO_FPCMP(FCMGT
, fcmgt
)
4409 DO_FPCMP(FCMEQ
, fcmeq
)
4410 DO_FPCMP(FCMNE
, fcmne
)
4411 DO_FPCMP(FCMUO
, fcmuo
)
4412 DO_FPCMP(FACGE
, facge
)
4413 DO_FPCMP(FACGT
, facgt
)
4417 static bool trans_FCADD(DisasContext
*s
, arg_FCADD
*a
)
4419 static gen_helper_gvec_4_ptr
* const fns
[3] = {
4420 gen_helper_sve_fcadd_h
,
4421 gen_helper_sve_fcadd_s
,
4422 gen_helper_sve_fcadd_d
4428 if (sve_access_check(s
)) {
4429 unsigned vsz
= vec_full_reg_size(s
);
4430 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4431 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4432 vec_full_reg_offset(s
, a
->rn
),
4433 vec_full_reg_offset(s
, a
->rm
),
4434 pred_full_reg_offset(s
, a
->pg
),
4435 status
, vsz
, vsz
, a
->rot
, fns
[a
->esz
- 1]);
4436 tcg_temp_free_ptr(status
);
4441 static bool do_fmla(DisasContext
*s
, arg_rprrr_esz
*a
,
4442 gen_helper_gvec_5_ptr
*fn
)
4447 if (sve_access_check(s
)) {
4448 unsigned vsz
= vec_full_reg_size(s
);
4449 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4450 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s
, a
->rd
),
4451 vec_full_reg_offset(s
, a
->rn
),
4452 vec_full_reg_offset(s
, a
->rm
),
4453 vec_full_reg_offset(s
, a
->ra
),
4454 pred_full_reg_offset(s
, a
->pg
),
4455 status
, vsz
, vsz
, 0, fn
);
4456 tcg_temp_free_ptr(status
);
4461 #define DO_FMLA(NAME, name) \
4462 static bool trans_##NAME(DisasContext *s, arg_rprrr_esz *a) \
4464 static gen_helper_gvec_5_ptr * const fns[4] = { \
4465 NULL, gen_helper_sve_##name##_h, \
4466 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4468 return do_fmla(s, a, fns[a->esz]); \
4471 DO_FMLA(FMLA_zpzzz
, fmla_zpzzz
)
4472 DO_FMLA(FMLS_zpzzz
, fmls_zpzzz
)
4473 DO_FMLA(FNMLA_zpzzz
, fnmla_zpzzz
)
4474 DO_FMLA(FNMLS_zpzzz
, fnmls_zpzzz
)
4478 static bool trans_FCMLA_zpzzz(DisasContext
*s
, arg_FCMLA_zpzzz
*a
)
4480 static gen_helper_gvec_5_ptr
* const fns
[4] = {
4482 gen_helper_sve_fcmla_zpzzz_h
,
4483 gen_helper_sve_fcmla_zpzzz_s
,
4484 gen_helper_sve_fcmla_zpzzz_d
,
4490 if (sve_access_check(s
)) {
4491 unsigned vsz
= vec_full_reg_size(s
);
4492 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4493 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s
, a
->rd
),
4494 vec_full_reg_offset(s
, a
->rn
),
4495 vec_full_reg_offset(s
, a
->rm
),
4496 vec_full_reg_offset(s
, a
->ra
),
4497 pred_full_reg_offset(s
, a
->pg
),
4498 status
, vsz
, vsz
, a
->rot
, fns
[a
->esz
]);
4499 tcg_temp_free_ptr(status
);
4504 static bool trans_FCMLA_zzxz(DisasContext
*s
, arg_FCMLA_zzxz
*a
)
4506 static gen_helper_gvec_4_ptr
* const fns
[2] = {
4507 gen_helper_gvec_fcmlah_idx
,
4508 gen_helper_gvec_fcmlas_idx
,
4511 tcg_debug_assert(a
->esz
== 1 || a
->esz
== 2);
4512 tcg_debug_assert(a
->rd
== a
->ra
);
4513 if (sve_access_check(s
)) {
4514 unsigned vsz
= vec_full_reg_size(s
);
4515 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4516 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4517 vec_full_reg_offset(s
, a
->rn
),
4518 vec_full_reg_offset(s
, a
->rm
),
4519 vec_full_reg_offset(s
, a
->ra
),
4521 a
->index
* 4 + a
->rot
,
4523 tcg_temp_free_ptr(status
);
4529 *** SVE Floating Point Unary Operations Predicated Group
4532 static bool do_zpz_ptr(DisasContext
*s
, int rd
, int rn
, int pg
,
4533 bool is_fp16
, gen_helper_gvec_3_ptr
*fn
)
4535 if (sve_access_check(s
)) {
4536 unsigned vsz
= vec_full_reg_size(s
);
4537 TCGv_ptr status
= fpstatus_ptr(is_fp16
? FPST_FPCR_F16
: FPST_FPCR
);
4538 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, rd
),
4539 vec_full_reg_offset(s
, rn
),
4540 pred_full_reg_offset(s
, pg
),
4541 status
, vsz
, vsz
, 0, fn
);
4542 tcg_temp_free_ptr(status
);
4547 static bool trans_FCVT_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4549 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_sh
);
4552 static bool trans_FCVT_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4554 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_hs
);
4557 static bool trans_FCVT_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4559 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_dh
);
4562 static bool trans_FCVT_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4564 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_hd
);
4567 static bool trans_FCVT_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4569 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_ds
);
4572 static bool trans_FCVT_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4574 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_sd
);
4577 static bool trans_FCVTZS_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4579 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hh
);
4582 static bool trans_FCVTZU_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4584 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hh
);
4587 static bool trans_FCVTZS_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4589 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hs
);
4592 static bool trans_FCVTZU_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4594 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hs
);
4597 static bool trans_FCVTZS_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4599 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hd
);
4602 static bool trans_FCVTZU_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4604 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hd
);
4607 static bool trans_FCVTZS_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4609 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_ss
);
4612 static bool trans_FCVTZU_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4614 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_ss
);
4617 static bool trans_FCVTZS_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4619 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_sd
);
4622 static bool trans_FCVTZU_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4624 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_sd
);
4627 static bool trans_FCVTZS_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4629 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_ds
);
4632 static bool trans_FCVTZU_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4634 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_ds
);
4637 static bool trans_FCVTZS_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4639 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_dd
);
4642 static bool trans_FCVTZU_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4644 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_dd
);
4647 static gen_helper_gvec_3_ptr
* const frint_fns
[3] = {
4648 gen_helper_sve_frint_h
,
4649 gen_helper_sve_frint_s
,
4650 gen_helper_sve_frint_d
4653 static bool trans_FRINTI(DisasContext
*s
, arg_rpr_esz
*a
)
4658 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
,
4659 frint_fns
[a
->esz
- 1]);
4662 static bool trans_FRINTX(DisasContext
*s
, arg_rpr_esz
*a
)
4664 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4665 gen_helper_sve_frintx_h
,
4666 gen_helper_sve_frintx_s
,
4667 gen_helper_sve_frintx_d
4672 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4675 static bool do_frint_mode(DisasContext
*s
, arg_rpr_esz
*a
, int mode
)
4680 if (sve_access_check(s
)) {
4681 unsigned vsz
= vec_full_reg_size(s
);
4682 TCGv_i32 tmode
= tcg_const_i32(mode
);
4683 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4685 gen_helper_set_rmode(tmode
, tmode
, status
);
4687 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4688 vec_full_reg_offset(s
, a
->rn
),
4689 pred_full_reg_offset(s
, a
->pg
),
4690 status
, vsz
, vsz
, 0, frint_fns
[a
->esz
- 1]);
4692 gen_helper_set_rmode(tmode
, tmode
, status
);
4693 tcg_temp_free_i32(tmode
);
4694 tcg_temp_free_ptr(status
);
4699 static bool trans_FRINTN(DisasContext
*s
, arg_rpr_esz
*a
)
4701 return do_frint_mode(s
, a
, float_round_nearest_even
);
4704 static bool trans_FRINTP(DisasContext
*s
, arg_rpr_esz
*a
)
4706 return do_frint_mode(s
, a
, float_round_up
);
4709 static bool trans_FRINTM(DisasContext
*s
, arg_rpr_esz
*a
)
4711 return do_frint_mode(s
, a
, float_round_down
);
4714 static bool trans_FRINTZ(DisasContext
*s
, arg_rpr_esz
*a
)
4716 return do_frint_mode(s
, a
, float_round_to_zero
);
4719 static bool trans_FRINTA(DisasContext
*s
, arg_rpr_esz
*a
)
4721 return do_frint_mode(s
, a
, float_round_ties_away
);
4724 static bool trans_FRECPX(DisasContext
*s
, arg_rpr_esz
*a
)
4726 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4727 gen_helper_sve_frecpx_h
,
4728 gen_helper_sve_frecpx_s
,
4729 gen_helper_sve_frecpx_d
4734 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4737 static bool trans_FSQRT(DisasContext
*s
, arg_rpr_esz
*a
)
4739 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4740 gen_helper_sve_fsqrt_h
,
4741 gen_helper_sve_fsqrt_s
,
4742 gen_helper_sve_fsqrt_d
4747 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4750 static bool trans_SCVTF_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4752 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_hh
);
4755 static bool trans_SCVTF_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4757 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_sh
);
4760 static bool trans_SCVTF_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4762 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_dh
);
4765 static bool trans_SCVTF_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4767 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_ss
);
4770 static bool trans_SCVTF_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4772 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_ds
);
4775 static bool trans_SCVTF_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4777 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_sd
);
4780 static bool trans_SCVTF_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4782 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_dd
);
4785 static bool trans_UCVTF_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4787 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_hh
);
4790 static bool trans_UCVTF_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4792 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_sh
);
4795 static bool trans_UCVTF_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4797 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_dh
);
4800 static bool trans_UCVTF_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4802 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_ss
);
4805 static bool trans_UCVTF_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4807 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_ds
);
4810 static bool trans_UCVTF_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4812 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_sd
);
4815 static bool trans_UCVTF_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4817 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_dd
);
4821 *** SVE Memory - 32-bit Gather and Unsized Contiguous Group
4824 /* Subroutine loading a vector register at VOFS of LEN bytes.
4825 * The load should begin at the address Rn + IMM.
4828 static void do_ldr(DisasContext
*s
, uint32_t vofs
, int len
, int rn
, int imm
)
4830 int len_align
= QEMU_ALIGN_DOWN(len
, 8);
4831 int len_remain
= len
% 8;
4832 int nparts
= len
/ 8 + ctpop8(len_remain
);
4833 int midx
= get_mem_index(s
);
4834 TCGv_i64 dirty_addr
, clean_addr
, t0
, t1
;
4836 dirty_addr
= tcg_temp_new_i64();
4837 tcg_gen_addi_i64(dirty_addr
, cpu_reg_sp(s
, rn
), imm
);
4838 clean_addr
= gen_mte_checkN(s
, dirty_addr
, false, rn
!= 31, len
);
4839 tcg_temp_free_i64(dirty_addr
);
4842 * Note that unpredicated load/store of vector/predicate registers
4843 * are defined as a stream of bytes, which equates to little-endian
4844 * operations on larger quantities.
4845 * Attempt to keep code expansion to a minimum by limiting the
4846 * amount of unrolling done.
4851 t0
= tcg_temp_new_i64();
4852 for (i
= 0; i
< len_align
; i
+= 8) {
4853 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEQ
);
4854 tcg_gen_st_i64(t0
, cpu_env
, vofs
+ i
);
4855 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4857 tcg_temp_free_i64(t0
);
4859 TCGLabel
*loop
= gen_new_label();
4860 TCGv_ptr tp
, i
= tcg_const_local_ptr(0);
4862 /* Copy the clean address into a local temp, live across the loop. */
4864 clean_addr
= new_tmp_a64_local(s
);
4865 tcg_gen_mov_i64(clean_addr
, t0
);
4867 gen_set_label(loop
);
4869 t0
= tcg_temp_new_i64();
4870 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEQ
);
4871 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4873 tp
= tcg_temp_new_ptr();
4874 tcg_gen_add_ptr(tp
, cpu_env
, i
);
4875 tcg_gen_addi_ptr(i
, i
, 8);
4876 tcg_gen_st_i64(t0
, tp
, vofs
);
4877 tcg_temp_free_ptr(tp
);
4878 tcg_temp_free_i64(t0
);
4880 tcg_gen_brcondi_ptr(TCG_COND_LTU
, i
, len_align
, loop
);
4881 tcg_temp_free_ptr(i
);
4885 * Predicate register loads can be any multiple of 2.
4886 * Note that we still store the entire 64-bit unit into cpu_env.
4889 t0
= tcg_temp_new_i64();
4890 switch (len_remain
) {
4894 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
,
4895 MO_LE
| ctz32(len_remain
));
4899 t1
= tcg_temp_new_i64();
4900 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEUL
);
4901 tcg_gen_addi_i64(clean_addr
, clean_addr
, 4);
4902 tcg_gen_qemu_ld_i64(t1
, clean_addr
, midx
, MO_LEUW
);
4903 tcg_gen_deposit_i64(t0
, t0
, t1
, 32, 32);
4904 tcg_temp_free_i64(t1
);
4908 g_assert_not_reached();
4910 tcg_gen_st_i64(t0
, cpu_env
, vofs
+ len_align
);
4911 tcg_temp_free_i64(t0
);
4915 /* Similarly for stores. */
4916 static void do_str(DisasContext
*s
, uint32_t vofs
, int len
, int rn
, int imm
)
4918 int len_align
= QEMU_ALIGN_DOWN(len
, 8);
4919 int len_remain
= len
% 8;
4920 int nparts
= len
/ 8 + ctpop8(len_remain
);
4921 int midx
= get_mem_index(s
);
4922 TCGv_i64 dirty_addr
, clean_addr
, t0
;
4924 dirty_addr
= tcg_temp_new_i64();
4925 tcg_gen_addi_i64(dirty_addr
, cpu_reg_sp(s
, rn
), imm
);
4926 clean_addr
= gen_mte_checkN(s
, dirty_addr
, false, rn
!= 31, len
);
4927 tcg_temp_free_i64(dirty_addr
);
4929 /* Note that unpredicated load/store of vector/predicate registers
4930 * are defined as a stream of bytes, which equates to little-endian
4931 * operations on larger quantities. There is no nice way to force
4932 * a little-endian store for aarch64_be-linux-user out of line.
4934 * Attempt to keep code expansion to a minimum by limiting the
4935 * amount of unrolling done.
4940 t0
= tcg_temp_new_i64();
4941 for (i
= 0; i
< len_align
; i
+= 8) {
4942 tcg_gen_ld_i64(t0
, cpu_env
, vofs
+ i
);
4943 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEQ
);
4944 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4946 tcg_temp_free_i64(t0
);
4948 TCGLabel
*loop
= gen_new_label();
4949 TCGv_ptr tp
, i
= tcg_const_local_ptr(0);
4951 /* Copy the clean address into a local temp, live across the loop. */
4953 clean_addr
= new_tmp_a64_local(s
);
4954 tcg_gen_mov_i64(clean_addr
, t0
);
4956 gen_set_label(loop
);
4958 t0
= tcg_temp_new_i64();
4959 tp
= tcg_temp_new_ptr();
4960 tcg_gen_add_ptr(tp
, cpu_env
, i
);
4961 tcg_gen_ld_i64(t0
, tp
, vofs
);
4962 tcg_gen_addi_ptr(i
, i
, 8);
4963 tcg_temp_free_ptr(tp
);
4965 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEQ
);
4966 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4967 tcg_temp_free_i64(t0
);
4969 tcg_gen_brcondi_ptr(TCG_COND_LTU
, i
, len_align
, loop
);
4970 tcg_temp_free_ptr(i
);
4973 /* Predicate register stores can be any multiple of 2. */
4975 t0
= tcg_temp_new_i64();
4976 tcg_gen_ld_i64(t0
, cpu_env
, vofs
+ len_align
);
4978 switch (len_remain
) {
4982 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
,
4983 MO_LE
| ctz32(len_remain
));
4987 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUL
);
4988 tcg_gen_addi_i64(clean_addr
, clean_addr
, 4);
4989 tcg_gen_shri_i64(t0
, t0
, 32);
4990 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUW
);
4994 g_assert_not_reached();
4996 tcg_temp_free_i64(t0
);
5000 static bool trans_LDR_zri(DisasContext
*s
, arg_rri
*a
)
5002 if (sve_access_check(s
)) {
5003 int size
= vec_full_reg_size(s
);
5004 int off
= vec_full_reg_offset(s
, a
->rd
);
5005 do_ldr(s
, off
, size
, a
->rn
, a
->imm
* size
);
5010 static bool trans_LDR_pri(DisasContext
*s
, arg_rri
*a
)
5012 if (sve_access_check(s
)) {
5013 int size
= pred_full_reg_size(s
);
5014 int off
= pred_full_reg_offset(s
, a
->rd
);
5015 do_ldr(s
, off
, size
, a
->rn
, a
->imm
* size
);
5020 static bool trans_STR_zri(DisasContext
*s
, arg_rri
*a
)
5022 if (sve_access_check(s
)) {
5023 int size
= vec_full_reg_size(s
);
5024 int off
= vec_full_reg_offset(s
, a
->rd
);
5025 do_str(s
, off
, size
, a
->rn
, a
->imm
* size
);
5030 static bool trans_STR_pri(DisasContext
*s
, arg_rri
*a
)
5032 if (sve_access_check(s
)) {
5033 int size
= pred_full_reg_size(s
);
5034 int off
= pred_full_reg_offset(s
, a
->rd
);
5035 do_str(s
, off
, size
, a
->rn
, a
->imm
* size
);
5041 *** SVE Memory - Contiguous Load Group
5044 /* The memory mode of the dtype. */
5045 static const MemOp dtype_mop
[16] = {
5046 MO_UB
, MO_UB
, MO_UB
, MO_UB
,
5047 MO_SL
, MO_UW
, MO_UW
, MO_UW
,
5048 MO_SW
, MO_SW
, MO_UL
, MO_UL
,
5049 MO_SB
, MO_SB
, MO_SB
, MO_Q
5052 #define dtype_msz(x) (dtype_mop[x] & MO_SIZE)
5054 /* The vector element size of dtype. */
5055 static const uint8_t dtype_esz
[16] = {
5062 static void do_mem_zpa(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
,
5063 int dtype
, uint32_t mte_n
, bool is_write
,
5064 gen_helper_gvec_mem
*fn
)
5066 unsigned vsz
= vec_full_reg_size(s
);
5072 * For e.g. LD4, there are not enough arguments to pass all 4
5073 * registers as pointers, so encode the regno into the data field.
5074 * For consistency, do this even for LD1.
5076 if (s
->mte_active
[0]) {
5077 int msz
= dtype_msz(dtype
);
5079 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, get_mem_index(s
));
5080 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
5081 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
5082 desc
= FIELD_DP32(desc
, MTEDESC
, WRITE
, is_write
);
5083 desc
= FIELD_DP32(desc
, MTEDESC
, SIZEM1
, (mte_n
<< msz
) - 1);
5084 desc
<<= SVE_MTEDESC_SHIFT
;
5086 addr
= clean_data_tbi(s
, addr
);
5089 desc
= simd_desc(vsz
, vsz
, zt
| desc
);
5090 t_desc
= tcg_const_i32(desc
);
5091 t_pg
= tcg_temp_new_ptr();
5093 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
5094 fn(cpu_env
, t_pg
, addr
, t_desc
);
5096 tcg_temp_free_ptr(t_pg
);
5097 tcg_temp_free_i32(t_desc
);
5100 static void do_ld_zpa(DisasContext
*s
, int zt
, int pg
,
5101 TCGv_i64 addr
, int dtype
, int nreg
)
5103 static gen_helper_gvec_mem
* const fns
[2][2][16][4] = {
5104 { /* mte inactive, little-endian */
5105 { { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld2bb_r
,
5106 gen_helper_sve_ld3bb_r
, gen_helper_sve_ld4bb_r
},
5107 { gen_helper_sve_ld1bhu_r
, NULL
, NULL
, NULL
},
5108 { gen_helper_sve_ld1bsu_r
, NULL
, NULL
, NULL
},
5109 { gen_helper_sve_ld1bdu_r
, NULL
, NULL
, NULL
},
5111 { gen_helper_sve_ld1sds_le_r
, NULL
, NULL
, NULL
},
5112 { gen_helper_sve_ld1hh_le_r
, gen_helper_sve_ld2hh_le_r
,
5113 gen_helper_sve_ld3hh_le_r
, gen_helper_sve_ld4hh_le_r
},
5114 { gen_helper_sve_ld1hsu_le_r
, NULL
, NULL
, NULL
},
5115 { gen_helper_sve_ld1hdu_le_r
, NULL
, NULL
, NULL
},
5117 { gen_helper_sve_ld1hds_le_r
, NULL
, NULL
, NULL
},
5118 { gen_helper_sve_ld1hss_le_r
, NULL
, NULL
, NULL
},
5119 { gen_helper_sve_ld1ss_le_r
, gen_helper_sve_ld2ss_le_r
,
5120 gen_helper_sve_ld3ss_le_r
, gen_helper_sve_ld4ss_le_r
},
5121 { gen_helper_sve_ld1sdu_le_r
, NULL
, NULL
, NULL
},
5123 { gen_helper_sve_ld1bds_r
, NULL
, NULL
, NULL
},
5124 { gen_helper_sve_ld1bss_r
, NULL
, NULL
, NULL
},
5125 { gen_helper_sve_ld1bhs_r
, NULL
, NULL
, NULL
},
5126 { gen_helper_sve_ld1dd_le_r
, gen_helper_sve_ld2dd_le_r
,
5127 gen_helper_sve_ld3dd_le_r
, gen_helper_sve_ld4dd_le_r
} },
5129 /* mte inactive, big-endian */
5130 { { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld2bb_r
,
5131 gen_helper_sve_ld3bb_r
, gen_helper_sve_ld4bb_r
},
5132 { gen_helper_sve_ld1bhu_r
, NULL
, NULL
, NULL
},
5133 { gen_helper_sve_ld1bsu_r
, NULL
, NULL
, NULL
},
5134 { gen_helper_sve_ld1bdu_r
, NULL
, NULL
, NULL
},
5136 { gen_helper_sve_ld1sds_be_r
, NULL
, NULL
, NULL
},
5137 { gen_helper_sve_ld1hh_be_r
, gen_helper_sve_ld2hh_be_r
,
5138 gen_helper_sve_ld3hh_be_r
, gen_helper_sve_ld4hh_be_r
},
5139 { gen_helper_sve_ld1hsu_be_r
, NULL
, NULL
, NULL
},
5140 { gen_helper_sve_ld1hdu_be_r
, NULL
, NULL
, NULL
},
5142 { gen_helper_sve_ld1hds_be_r
, NULL
, NULL
, NULL
},
5143 { gen_helper_sve_ld1hss_be_r
, NULL
, NULL
, NULL
},
5144 { gen_helper_sve_ld1ss_be_r
, gen_helper_sve_ld2ss_be_r
,
5145 gen_helper_sve_ld3ss_be_r
, gen_helper_sve_ld4ss_be_r
},
5146 { gen_helper_sve_ld1sdu_be_r
, NULL
, NULL
, NULL
},
5148 { gen_helper_sve_ld1bds_r
, NULL
, NULL
, NULL
},
5149 { gen_helper_sve_ld1bss_r
, NULL
, NULL
, NULL
},
5150 { gen_helper_sve_ld1bhs_r
, NULL
, NULL
, NULL
},
5151 { gen_helper_sve_ld1dd_be_r
, gen_helper_sve_ld2dd_be_r
,
5152 gen_helper_sve_ld3dd_be_r
, gen_helper_sve_ld4dd_be_r
} } },
5154 { /* mte active, little-endian */
5155 { { gen_helper_sve_ld1bb_r_mte
,
5156 gen_helper_sve_ld2bb_r_mte
,
5157 gen_helper_sve_ld3bb_r_mte
,
5158 gen_helper_sve_ld4bb_r_mte
},
5159 { gen_helper_sve_ld1bhu_r_mte
, NULL
, NULL
, NULL
},
5160 { gen_helper_sve_ld1bsu_r_mte
, NULL
, NULL
, NULL
},
5161 { gen_helper_sve_ld1bdu_r_mte
, NULL
, NULL
, NULL
},
5163 { gen_helper_sve_ld1sds_le_r_mte
, NULL
, NULL
, NULL
},
5164 { gen_helper_sve_ld1hh_le_r_mte
,
5165 gen_helper_sve_ld2hh_le_r_mte
,
5166 gen_helper_sve_ld3hh_le_r_mte
,
5167 gen_helper_sve_ld4hh_le_r_mte
},
5168 { gen_helper_sve_ld1hsu_le_r_mte
, NULL
, NULL
, NULL
},
5169 { gen_helper_sve_ld1hdu_le_r_mte
, NULL
, NULL
, NULL
},
5171 { gen_helper_sve_ld1hds_le_r_mte
, NULL
, NULL
, NULL
},
5172 { gen_helper_sve_ld1hss_le_r_mte
, NULL
, NULL
, NULL
},
5173 { gen_helper_sve_ld1ss_le_r_mte
,
5174 gen_helper_sve_ld2ss_le_r_mte
,
5175 gen_helper_sve_ld3ss_le_r_mte
,
5176 gen_helper_sve_ld4ss_le_r_mte
},
5177 { gen_helper_sve_ld1sdu_le_r_mte
, NULL
, NULL
, NULL
},
5179 { gen_helper_sve_ld1bds_r_mte
, NULL
, NULL
, NULL
},
5180 { gen_helper_sve_ld1bss_r_mte
, NULL
, NULL
, NULL
},
5181 { gen_helper_sve_ld1bhs_r_mte
, NULL
, NULL
, NULL
},
5182 { gen_helper_sve_ld1dd_le_r_mte
,
5183 gen_helper_sve_ld2dd_le_r_mte
,
5184 gen_helper_sve_ld3dd_le_r_mte
,
5185 gen_helper_sve_ld4dd_le_r_mte
} },
5187 /* mte active, big-endian */
5188 { { gen_helper_sve_ld1bb_r_mte
,
5189 gen_helper_sve_ld2bb_r_mte
,
5190 gen_helper_sve_ld3bb_r_mte
,
5191 gen_helper_sve_ld4bb_r_mte
},
5192 { gen_helper_sve_ld1bhu_r_mte
, NULL
, NULL
, NULL
},
5193 { gen_helper_sve_ld1bsu_r_mte
, NULL
, NULL
, NULL
},
5194 { gen_helper_sve_ld1bdu_r_mte
, NULL
, NULL
, NULL
},
5196 { gen_helper_sve_ld1sds_be_r_mte
, NULL
, NULL
, NULL
},
5197 { gen_helper_sve_ld1hh_be_r_mte
,
5198 gen_helper_sve_ld2hh_be_r_mte
,
5199 gen_helper_sve_ld3hh_be_r_mte
,
5200 gen_helper_sve_ld4hh_be_r_mte
},
5201 { gen_helper_sve_ld1hsu_be_r_mte
, NULL
, NULL
, NULL
},
5202 { gen_helper_sve_ld1hdu_be_r_mte
, NULL
, NULL
, NULL
},
5204 { gen_helper_sve_ld1hds_be_r_mte
, NULL
, NULL
, NULL
},
5205 { gen_helper_sve_ld1hss_be_r_mte
, NULL
, NULL
, NULL
},
5206 { gen_helper_sve_ld1ss_be_r_mte
,
5207 gen_helper_sve_ld2ss_be_r_mte
,
5208 gen_helper_sve_ld3ss_be_r_mte
,
5209 gen_helper_sve_ld4ss_be_r_mte
},
5210 { gen_helper_sve_ld1sdu_be_r_mte
, NULL
, NULL
, NULL
},
5212 { gen_helper_sve_ld1bds_r_mte
, NULL
, NULL
, NULL
},
5213 { gen_helper_sve_ld1bss_r_mte
, NULL
, NULL
, NULL
},
5214 { gen_helper_sve_ld1bhs_r_mte
, NULL
, NULL
, NULL
},
5215 { gen_helper_sve_ld1dd_be_r_mte
,
5216 gen_helper_sve_ld2dd_be_r_mte
,
5217 gen_helper_sve_ld3dd_be_r_mte
,
5218 gen_helper_sve_ld4dd_be_r_mte
} } },
5220 gen_helper_gvec_mem
*fn
5221 = fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][dtype
][nreg
];
5224 * While there are holes in the table, they are not
5225 * accessible via the instruction encoding.
5228 do_mem_zpa(s
, zt
, pg
, addr
, dtype
, nreg
, false, fn
);
5231 static bool trans_LD_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5236 if (sve_access_check(s
)) {
5237 TCGv_i64 addr
= new_tmp_a64(s
);
5238 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
5239 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5240 do_ld_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, a
->nreg
);
5245 static bool trans_LD_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5247 if (sve_access_check(s
)) {
5248 int vsz
= vec_full_reg_size(s
);
5249 int elements
= vsz
>> dtype_esz
[a
->dtype
];
5250 TCGv_i64 addr
= new_tmp_a64(s
);
5252 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
),
5253 (a
->imm
* elements
* (a
->nreg
+ 1))
5254 << dtype_msz(a
->dtype
));
5255 do_ld_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, a
->nreg
);
5260 static bool trans_LDFF1_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5262 static gen_helper_gvec_mem
* const fns
[2][2][16] = {
5263 { /* mte inactive, little-endian */
5264 { gen_helper_sve_ldff1bb_r
,
5265 gen_helper_sve_ldff1bhu_r
,
5266 gen_helper_sve_ldff1bsu_r
,
5267 gen_helper_sve_ldff1bdu_r
,
5269 gen_helper_sve_ldff1sds_le_r
,
5270 gen_helper_sve_ldff1hh_le_r
,
5271 gen_helper_sve_ldff1hsu_le_r
,
5272 gen_helper_sve_ldff1hdu_le_r
,
5274 gen_helper_sve_ldff1hds_le_r
,
5275 gen_helper_sve_ldff1hss_le_r
,
5276 gen_helper_sve_ldff1ss_le_r
,
5277 gen_helper_sve_ldff1sdu_le_r
,
5279 gen_helper_sve_ldff1bds_r
,
5280 gen_helper_sve_ldff1bss_r
,
5281 gen_helper_sve_ldff1bhs_r
,
5282 gen_helper_sve_ldff1dd_le_r
},
5284 /* mte inactive, big-endian */
5285 { gen_helper_sve_ldff1bb_r
,
5286 gen_helper_sve_ldff1bhu_r
,
5287 gen_helper_sve_ldff1bsu_r
,
5288 gen_helper_sve_ldff1bdu_r
,
5290 gen_helper_sve_ldff1sds_be_r
,
5291 gen_helper_sve_ldff1hh_be_r
,
5292 gen_helper_sve_ldff1hsu_be_r
,
5293 gen_helper_sve_ldff1hdu_be_r
,
5295 gen_helper_sve_ldff1hds_be_r
,
5296 gen_helper_sve_ldff1hss_be_r
,
5297 gen_helper_sve_ldff1ss_be_r
,
5298 gen_helper_sve_ldff1sdu_be_r
,
5300 gen_helper_sve_ldff1bds_r
,
5301 gen_helper_sve_ldff1bss_r
,
5302 gen_helper_sve_ldff1bhs_r
,
5303 gen_helper_sve_ldff1dd_be_r
} },
5305 { /* mte active, little-endian */
5306 { gen_helper_sve_ldff1bb_r_mte
,
5307 gen_helper_sve_ldff1bhu_r_mte
,
5308 gen_helper_sve_ldff1bsu_r_mte
,
5309 gen_helper_sve_ldff1bdu_r_mte
,
5311 gen_helper_sve_ldff1sds_le_r_mte
,
5312 gen_helper_sve_ldff1hh_le_r_mte
,
5313 gen_helper_sve_ldff1hsu_le_r_mte
,
5314 gen_helper_sve_ldff1hdu_le_r_mte
,
5316 gen_helper_sve_ldff1hds_le_r_mte
,
5317 gen_helper_sve_ldff1hss_le_r_mte
,
5318 gen_helper_sve_ldff1ss_le_r_mte
,
5319 gen_helper_sve_ldff1sdu_le_r_mte
,
5321 gen_helper_sve_ldff1bds_r_mte
,
5322 gen_helper_sve_ldff1bss_r_mte
,
5323 gen_helper_sve_ldff1bhs_r_mte
,
5324 gen_helper_sve_ldff1dd_le_r_mte
},
5326 /* mte active, big-endian */
5327 { gen_helper_sve_ldff1bb_r_mte
,
5328 gen_helper_sve_ldff1bhu_r_mte
,
5329 gen_helper_sve_ldff1bsu_r_mte
,
5330 gen_helper_sve_ldff1bdu_r_mte
,
5332 gen_helper_sve_ldff1sds_be_r_mte
,
5333 gen_helper_sve_ldff1hh_be_r_mte
,
5334 gen_helper_sve_ldff1hsu_be_r_mte
,
5335 gen_helper_sve_ldff1hdu_be_r_mte
,
5337 gen_helper_sve_ldff1hds_be_r_mte
,
5338 gen_helper_sve_ldff1hss_be_r_mte
,
5339 gen_helper_sve_ldff1ss_be_r_mte
,
5340 gen_helper_sve_ldff1sdu_be_r_mte
,
5342 gen_helper_sve_ldff1bds_r_mte
,
5343 gen_helper_sve_ldff1bss_r_mte
,
5344 gen_helper_sve_ldff1bhs_r_mte
,
5345 gen_helper_sve_ldff1dd_be_r_mte
} },
5348 if (sve_access_check(s
)) {
5349 TCGv_i64 addr
= new_tmp_a64(s
);
5350 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
5351 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5352 do_mem_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, 1, false,
5353 fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][a
->dtype
]);
5358 static bool trans_LDNF1_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5360 static gen_helper_gvec_mem
* const fns
[2][2][16] = {
5361 { /* mte inactive, little-endian */
5362 { gen_helper_sve_ldnf1bb_r
,
5363 gen_helper_sve_ldnf1bhu_r
,
5364 gen_helper_sve_ldnf1bsu_r
,
5365 gen_helper_sve_ldnf1bdu_r
,
5367 gen_helper_sve_ldnf1sds_le_r
,
5368 gen_helper_sve_ldnf1hh_le_r
,
5369 gen_helper_sve_ldnf1hsu_le_r
,
5370 gen_helper_sve_ldnf1hdu_le_r
,
5372 gen_helper_sve_ldnf1hds_le_r
,
5373 gen_helper_sve_ldnf1hss_le_r
,
5374 gen_helper_sve_ldnf1ss_le_r
,
5375 gen_helper_sve_ldnf1sdu_le_r
,
5377 gen_helper_sve_ldnf1bds_r
,
5378 gen_helper_sve_ldnf1bss_r
,
5379 gen_helper_sve_ldnf1bhs_r
,
5380 gen_helper_sve_ldnf1dd_le_r
},
5382 /* mte inactive, big-endian */
5383 { gen_helper_sve_ldnf1bb_r
,
5384 gen_helper_sve_ldnf1bhu_r
,
5385 gen_helper_sve_ldnf1bsu_r
,
5386 gen_helper_sve_ldnf1bdu_r
,
5388 gen_helper_sve_ldnf1sds_be_r
,
5389 gen_helper_sve_ldnf1hh_be_r
,
5390 gen_helper_sve_ldnf1hsu_be_r
,
5391 gen_helper_sve_ldnf1hdu_be_r
,
5393 gen_helper_sve_ldnf1hds_be_r
,
5394 gen_helper_sve_ldnf1hss_be_r
,
5395 gen_helper_sve_ldnf1ss_be_r
,
5396 gen_helper_sve_ldnf1sdu_be_r
,
5398 gen_helper_sve_ldnf1bds_r
,
5399 gen_helper_sve_ldnf1bss_r
,
5400 gen_helper_sve_ldnf1bhs_r
,
5401 gen_helper_sve_ldnf1dd_be_r
} },
5403 { /* mte inactive, little-endian */
5404 { gen_helper_sve_ldnf1bb_r_mte
,
5405 gen_helper_sve_ldnf1bhu_r_mte
,
5406 gen_helper_sve_ldnf1bsu_r_mte
,
5407 gen_helper_sve_ldnf1bdu_r_mte
,
5409 gen_helper_sve_ldnf1sds_le_r_mte
,
5410 gen_helper_sve_ldnf1hh_le_r_mte
,
5411 gen_helper_sve_ldnf1hsu_le_r_mte
,
5412 gen_helper_sve_ldnf1hdu_le_r_mte
,
5414 gen_helper_sve_ldnf1hds_le_r_mte
,
5415 gen_helper_sve_ldnf1hss_le_r_mte
,
5416 gen_helper_sve_ldnf1ss_le_r_mte
,
5417 gen_helper_sve_ldnf1sdu_le_r_mte
,
5419 gen_helper_sve_ldnf1bds_r_mte
,
5420 gen_helper_sve_ldnf1bss_r_mte
,
5421 gen_helper_sve_ldnf1bhs_r_mte
,
5422 gen_helper_sve_ldnf1dd_le_r_mte
},
5424 /* mte inactive, big-endian */
5425 { gen_helper_sve_ldnf1bb_r_mte
,
5426 gen_helper_sve_ldnf1bhu_r_mte
,
5427 gen_helper_sve_ldnf1bsu_r_mte
,
5428 gen_helper_sve_ldnf1bdu_r_mte
,
5430 gen_helper_sve_ldnf1sds_be_r_mte
,
5431 gen_helper_sve_ldnf1hh_be_r_mte
,
5432 gen_helper_sve_ldnf1hsu_be_r_mte
,
5433 gen_helper_sve_ldnf1hdu_be_r_mte
,
5435 gen_helper_sve_ldnf1hds_be_r_mte
,
5436 gen_helper_sve_ldnf1hss_be_r_mte
,
5437 gen_helper_sve_ldnf1ss_be_r_mte
,
5438 gen_helper_sve_ldnf1sdu_be_r_mte
,
5440 gen_helper_sve_ldnf1bds_r_mte
,
5441 gen_helper_sve_ldnf1bss_r_mte
,
5442 gen_helper_sve_ldnf1bhs_r_mte
,
5443 gen_helper_sve_ldnf1dd_be_r_mte
} },
5446 if (sve_access_check(s
)) {
5447 int vsz
= vec_full_reg_size(s
);
5448 int elements
= vsz
>> dtype_esz
[a
->dtype
];
5449 int off
= (a
->imm
* elements
) << dtype_msz(a
->dtype
);
5450 TCGv_i64 addr
= new_tmp_a64(s
);
5452 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), off
);
5453 do_mem_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, 1, false,
5454 fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][a
->dtype
]);
5459 static void do_ldrq(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
, int msz
)
5461 static gen_helper_gvec_mem
* const fns
[2][4] = {
5462 { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld1hh_le_r
,
5463 gen_helper_sve_ld1ss_le_r
, gen_helper_sve_ld1dd_le_r
},
5464 { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld1hh_be_r
,
5465 gen_helper_sve_ld1ss_be_r
, gen_helper_sve_ld1dd_be_r
},
5467 unsigned vsz
= vec_full_reg_size(s
);
5472 /* Load the first quadword using the normal predicated load helpers. */
5473 desc
= simd_desc(16, 16, zt
);
5474 t_desc
= tcg_const_i32(desc
);
5476 poff
= pred_full_reg_offset(s
, pg
);
5479 * Zero-extend the first 16 bits of the predicate into a temporary.
5480 * This avoids triggering an assert making sure we don't have bits
5481 * set within a predicate beyond VQ, but we have lowered VQ to 1
5482 * for this load operation.
5484 TCGv_i64 tmp
= tcg_temp_new_i64();
5485 #ifdef HOST_WORDS_BIGENDIAN
5488 tcg_gen_ld16u_i64(tmp
, cpu_env
, poff
);
5490 poff
= offsetof(CPUARMState
, vfp
.preg_tmp
);
5491 tcg_gen_st_i64(tmp
, cpu_env
, poff
);
5492 tcg_temp_free_i64(tmp
);
5495 t_pg
= tcg_temp_new_ptr();
5496 tcg_gen_addi_ptr(t_pg
, cpu_env
, poff
);
5498 fns
[s
->be_data
== MO_BE
][msz
](cpu_env
, t_pg
, addr
, t_desc
);
5500 tcg_temp_free_ptr(t_pg
);
5501 tcg_temp_free_i32(t_desc
);
5503 /* Replicate that first quadword. */
5505 unsigned dofs
= vec_full_reg_offset(s
, zt
);
5506 tcg_gen_gvec_dup_mem(4, dofs
+ 16, dofs
, vsz
- 16, vsz
- 16);
5510 static bool trans_LD1RQ_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5515 if (sve_access_check(s
)) {
5516 int msz
= dtype_msz(a
->dtype
);
5517 TCGv_i64 addr
= new_tmp_a64(s
);
5518 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), msz
);
5519 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5520 do_ldrq(s
, a
->rd
, a
->pg
, addr
, msz
);
5525 static bool trans_LD1RQ_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5527 if (sve_access_check(s
)) {
5528 TCGv_i64 addr
= new_tmp_a64(s
);
5529 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), a
->imm
* 16);
5530 do_ldrq(s
, a
->rd
, a
->pg
, addr
, dtype_msz(a
->dtype
));
5535 /* Load and broadcast element. */
5536 static bool trans_LD1R_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5538 unsigned vsz
= vec_full_reg_size(s
);
5539 unsigned psz
= pred_full_reg_size(s
);
5540 unsigned esz
= dtype_esz
[a
->dtype
];
5541 unsigned msz
= dtype_msz(a
->dtype
);
5543 TCGv_i64 temp
, clean_addr
;
5545 if (!sve_access_check(s
)) {
5549 over
= gen_new_label();
5551 /* If the guarding predicate has no bits set, no load occurs. */
5553 /* Reduce the pred_esz_masks value simply to reduce the
5554 * size of the code generated here.
5556 uint64_t psz_mask
= MAKE_64BIT_MASK(0, psz
* 8);
5557 temp
= tcg_temp_new_i64();
5558 tcg_gen_ld_i64(temp
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
5559 tcg_gen_andi_i64(temp
, temp
, pred_esz_masks
[esz
] & psz_mask
);
5560 tcg_gen_brcondi_i64(TCG_COND_EQ
, temp
, 0, over
);
5561 tcg_temp_free_i64(temp
);
5563 TCGv_i32 t32
= tcg_temp_new_i32();
5564 find_last_active(s
, t32
, esz
, a
->pg
);
5565 tcg_gen_brcondi_i32(TCG_COND_LT
, t32
, 0, over
);
5566 tcg_temp_free_i32(t32
);
5569 /* Load the data. */
5570 temp
= tcg_temp_new_i64();
5571 tcg_gen_addi_i64(temp
, cpu_reg_sp(s
, a
->rn
), a
->imm
<< msz
);
5572 clean_addr
= gen_mte_check1(s
, temp
, false, true, msz
);
5574 tcg_gen_qemu_ld_i64(temp
, clean_addr
, get_mem_index(s
),
5575 finalize_memop(s
, dtype_mop
[a
->dtype
]));
5577 /* Broadcast to *all* elements. */
5578 tcg_gen_gvec_dup_i64(esz
, vec_full_reg_offset(s
, a
->rd
),
5580 tcg_temp_free_i64(temp
);
5582 /* Zero the inactive elements. */
5583 gen_set_label(over
);
5584 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, esz
, false);
5587 static void do_st_zpa(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
,
5588 int msz
, int esz
, int nreg
)
5590 static gen_helper_gvec_mem
* const fn_single
[2][2][4][4] = {
5591 { { { gen_helper_sve_st1bb_r
,
5592 gen_helper_sve_st1bh_r
,
5593 gen_helper_sve_st1bs_r
,
5594 gen_helper_sve_st1bd_r
},
5596 gen_helper_sve_st1hh_le_r
,
5597 gen_helper_sve_st1hs_le_r
,
5598 gen_helper_sve_st1hd_le_r
},
5600 gen_helper_sve_st1ss_le_r
,
5601 gen_helper_sve_st1sd_le_r
},
5603 gen_helper_sve_st1dd_le_r
} },
5604 { { gen_helper_sve_st1bb_r
,
5605 gen_helper_sve_st1bh_r
,
5606 gen_helper_sve_st1bs_r
,
5607 gen_helper_sve_st1bd_r
},
5609 gen_helper_sve_st1hh_be_r
,
5610 gen_helper_sve_st1hs_be_r
,
5611 gen_helper_sve_st1hd_be_r
},
5613 gen_helper_sve_st1ss_be_r
,
5614 gen_helper_sve_st1sd_be_r
},
5616 gen_helper_sve_st1dd_be_r
} } },
5618 { { { gen_helper_sve_st1bb_r_mte
,
5619 gen_helper_sve_st1bh_r_mte
,
5620 gen_helper_sve_st1bs_r_mte
,
5621 gen_helper_sve_st1bd_r_mte
},
5623 gen_helper_sve_st1hh_le_r_mte
,
5624 gen_helper_sve_st1hs_le_r_mte
,
5625 gen_helper_sve_st1hd_le_r_mte
},
5627 gen_helper_sve_st1ss_le_r_mte
,
5628 gen_helper_sve_st1sd_le_r_mte
},
5630 gen_helper_sve_st1dd_le_r_mte
} },
5631 { { gen_helper_sve_st1bb_r_mte
,
5632 gen_helper_sve_st1bh_r_mte
,
5633 gen_helper_sve_st1bs_r_mte
,
5634 gen_helper_sve_st1bd_r_mte
},
5636 gen_helper_sve_st1hh_be_r_mte
,
5637 gen_helper_sve_st1hs_be_r_mte
,
5638 gen_helper_sve_st1hd_be_r_mte
},
5640 gen_helper_sve_st1ss_be_r_mte
,
5641 gen_helper_sve_st1sd_be_r_mte
},
5643 gen_helper_sve_st1dd_be_r_mte
} } },
5645 static gen_helper_gvec_mem
* const fn_multiple
[2][2][3][4] = {
5646 { { { gen_helper_sve_st2bb_r
,
5647 gen_helper_sve_st2hh_le_r
,
5648 gen_helper_sve_st2ss_le_r
,
5649 gen_helper_sve_st2dd_le_r
},
5650 { gen_helper_sve_st3bb_r
,
5651 gen_helper_sve_st3hh_le_r
,
5652 gen_helper_sve_st3ss_le_r
,
5653 gen_helper_sve_st3dd_le_r
},
5654 { gen_helper_sve_st4bb_r
,
5655 gen_helper_sve_st4hh_le_r
,
5656 gen_helper_sve_st4ss_le_r
,
5657 gen_helper_sve_st4dd_le_r
} },
5658 { { gen_helper_sve_st2bb_r
,
5659 gen_helper_sve_st2hh_be_r
,
5660 gen_helper_sve_st2ss_be_r
,
5661 gen_helper_sve_st2dd_be_r
},
5662 { gen_helper_sve_st3bb_r
,
5663 gen_helper_sve_st3hh_be_r
,
5664 gen_helper_sve_st3ss_be_r
,
5665 gen_helper_sve_st3dd_be_r
},
5666 { gen_helper_sve_st4bb_r
,
5667 gen_helper_sve_st4hh_be_r
,
5668 gen_helper_sve_st4ss_be_r
,
5669 gen_helper_sve_st4dd_be_r
} } },
5670 { { { gen_helper_sve_st2bb_r_mte
,
5671 gen_helper_sve_st2hh_le_r_mte
,
5672 gen_helper_sve_st2ss_le_r_mte
,
5673 gen_helper_sve_st2dd_le_r_mte
},
5674 { gen_helper_sve_st3bb_r_mte
,
5675 gen_helper_sve_st3hh_le_r_mte
,
5676 gen_helper_sve_st3ss_le_r_mte
,
5677 gen_helper_sve_st3dd_le_r_mte
},
5678 { gen_helper_sve_st4bb_r_mte
,
5679 gen_helper_sve_st4hh_le_r_mte
,
5680 gen_helper_sve_st4ss_le_r_mte
,
5681 gen_helper_sve_st4dd_le_r_mte
} },
5682 { { gen_helper_sve_st2bb_r_mte
,
5683 gen_helper_sve_st2hh_be_r_mte
,
5684 gen_helper_sve_st2ss_be_r_mte
,
5685 gen_helper_sve_st2dd_be_r_mte
},
5686 { gen_helper_sve_st3bb_r_mte
,
5687 gen_helper_sve_st3hh_be_r_mte
,
5688 gen_helper_sve_st3ss_be_r_mte
,
5689 gen_helper_sve_st3dd_be_r_mte
},
5690 { gen_helper_sve_st4bb_r_mte
,
5691 gen_helper_sve_st4hh_be_r_mte
,
5692 gen_helper_sve_st4ss_be_r_mte
,
5693 gen_helper_sve_st4dd_be_r_mte
} } },
5695 gen_helper_gvec_mem
*fn
;
5696 int be
= s
->be_data
== MO_BE
;
5700 fn
= fn_single
[s
->mte_active
[0]][be
][msz
][esz
];
5703 /* ST2, ST3, ST4 -- msz == esz, enforced by encoding */
5705 fn
= fn_multiple
[s
->mte_active
[0]][be
][nreg
- 1][msz
];
5708 do_mem_zpa(s
, zt
, pg
, addr
, msz_dtype(s
, msz
), nreg
, true, fn
);
5711 static bool trans_ST_zprr(DisasContext
*s
, arg_rprr_store
*a
)
5713 if (a
->rm
== 31 || a
->msz
> a
->esz
) {
5716 if (sve_access_check(s
)) {
5717 TCGv_i64 addr
= new_tmp_a64(s
);
5718 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), a
->msz
);
5719 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5720 do_st_zpa(s
, a
->rd
, a
->pg
, addr
, a
->msz
, a
->esz
, a
->nreg
);
5725 static bool trans_ST_zpri(DisasContext
*s
, arg_rpri_store
*a
)
5727 if (a
->msz
> a
->esz
) {
5730 if (sve_access_check(s
)) {
5731 int vsz
= vec_full_reg_size(s
);
5732 int elements
= vsz
>> a
->esz
;
5733 TCGv_i64 addr
= new_tmp_a64(s
);
5735 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
),
5736 (a
->imm
* elements
* (a
->nreg
+ 1)) << a
->msz
);
5737 do_st_zpa(s
, a
->rd
, a
->pg
, addr
, a
->msz
, a
->esz
, a
->nreg
);
5743 *** SVE gather loads / scatter stores
5746 static void do_mem_zpz(DisasContext
*s
, int zt
, int pg
, int zm
,
5747 int scale
, TCGv_i64 scalar
, int msz
, bool is_write
,
5748 gen_helper_gvec_mem_scatter
*fn
)
5750 unsigned vsz
= vec_full_reg_size(s
);
5751 TCGv_ptr t_zm
= tcg_temp_new_ptr();
5752 TCGv_ptr t_pg
= tcg_temp_new_ptr();
5753 TCGv_ptr t_zt
= tcg_temp_new_ptr();
5757 if (s
->mte_active
[0]) {
5758 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, get_mem_index(s
));
5759 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
5760 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
5761 desc
= FIELD_DP32(desc
, MTEDESC
, WRITE
, is_write
);
5762 desc
= FIELD_DP32(desc
, MTEDESC
, SIZEM1
, (1 << msz
) - 1);
5763 desc
<<= SVE_MTEDESC_SHIFT
;
5765 desc
= simd_desc(vsz
, vsz
, desc
| scale
);
5766 t_desc
= tcg_const_i32(desc
);
5768 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
5769 tcg_gen_addi_ptr(t_zm
, cpu_env
, vec_full_reg_offset(s
, zm
));
5770 tcg_gen_addi_ptr(t_zt
, cpu_env
, vec_full_reg_offset(s
, zt
));
5771 fn(cpu_env
, t_zt
, t_pg
, t_zm
, scalar
, t_desc
);
5773 tcg_temp_free_ptr(t_zt
);
5774 tcg_temp_free_ptr(t_zm
);
5775 tcg_temp_free_ptr(t_pg
);
5776 tcg_temp_free_i32(t_desc
);
5779 /* Indexed by [mte][be][ff][xs][u][msz]. */
5780 static gen_helper_gvec_mem_scatter
* const
5781 gather_load_fn32
[2][2][2][2][2][3] = {
5782 { /* MTE Inactive */
5783 { /* Little-endian */
5784 { { { gen_helper_sve_ldbss_zsu
,
5785 gen_helper_sve_ldhss_le_zsu
,
5787 { gen_helper_sve_ldbsu_zsu
,
5788 gen_helper_sve_ldhsu_le_zsu
,
5789 gen_helper_sve_ldss_le_zsu
, } },
5790 { { gen_helper_sve_ldbss_zss
,
5791 gen_helper_sve_ldhss_le_zss
,
5793 { gen_helper_sve_ldbsu_zss
,
5794 gen_helper_sve_ldhsu_le_zss
,
5795 gen_helper_sve_ldss_le_zss
, } } },
5798 { { { gen_helper_sve_ldffbss_zsu
,
5799 gen_helper_sve_ldffhss_le_zsu
,
5801 { gen_helper_sve_ldffbsu_zsu
,
5802 gen_helper_sve_ldffhsu_le_zsu
,
5803 gen_helper_sve_ldffss_le_zsu
, } },
5804 { { gen_helper_sve_ldffbss_zss
,
5805 gen_helper_sve_ldffhss_le_zss
,
5807 { gen_helper_sve_ldffbsu_zss
,
5808 gen_helper_sve_ldffhsu_le_zss
,
5809 gen_helper_sve_ldffss_le_zss
, } } } },
5812 { { { gen_helper_sve_ldbss_zsu
,
5813 gen_helper_sve_ldhss_be_zsu
,
5815 { gen_helper_sve_ldbsu_zsu
,
5816 gen_helper_sve_ldhsu_be_zsu
,
5817 gen_helper_sve_ldss_be_zsu
, } },
5818 { { gen_helper_sve_ldbss_zss
,
5819 gen_helper_sve_ldhss_be_zss
,
5821 { gen_helper_sve_ldbsu_zss
,
5822 gen_helper_sve_ldhsu_be_zss
,
5823 gen_helper_sve_ldss_be_zss
, } } },
5826 { { { gen_helper_sve_ldffbss_zsu
,
5827 gen_helper_sve_ldffhss_be_zsu
,
5829 { gen_helper_sve_ldffbsu_zsu
,
5830 gen_helper_sve_ldffhsu_be_zsu
,
5831 gen_helper_sve_ldffss_be_zsu
, } },
5832 { { gen_helper_sve_ldffbss_zss
,
5833 gen_helper_sve_ldffhss_be_zss
,
5835 { gen_helper_sve_ldffbsu_zss
,
5836 gen_helper_sve_ldffhsu_be_zss
,
5837 gen_helper_sve_ldffss_be_zss
, } } } } },
5839 { /* Little-endian */
5840 { { { gen_helper_sve_ldbss_zsu_mte
,
5841 gen_helper_sve_ldhss_le_zsu_mte
,
5843 { gen_helper_sve_ldbsu_zsu_mte
,
5844 gen_helper_sve_ldhsu_le_zsu_mte
,
5845 gen_helper_sve_ldss_le_zsu_mte
, } },
5846 { { gen_helper_sve_ldbss_zss_mte
,
5847 gen_helper_sve_ldhss_le_zss_mte
,
5849 { gen_helper_sve_ldbsu_zss_mte
,
5850 gen_helper_sve_ldhsu_le_zss_mte
,
5851 gen_helper_sve_ldss_le_zss_mte
, } } },
5854 { { { gen_helper_sve_ldffbss_zsu_mte
,
5855 gen_helper_sve_ldffhss_le_zsu_mte
,
5857 { gen_helper_sve_ldffbsu_zsu_mte
,
5858 gen_helper_sve_ldffhsu_le_zsu_mte
,
5859 gen_helper_sve_ldffss_le_zsu_mte
, } },
5860 { { gen_helper_sve_ldffbss_zss_mte
,
5861 gen_helper_sve_ldffhss_le_zss_mte
,
5863 { gen_helper_sve_ldffbsu_zss_mte
,
5864 gen_helper_sve_ldffhsu_le_zss_mte
,
5865 gen_helper_sve_ldffss_le_zss_mte
, } } } },
5868 { { { gen_helper_sve_ldbss_zsu_mte
,
5869 gen_helper_sve_ldhss_be_zsu_mte
,
5871 { gen_helper_sve_ldbsu_zsu_mte
,
5872 gen_helper_sve_ldhsu_be_zsu_mte
,
5873 gen_helper_sve_ldss_be_zsu_mte
, } },
5874 { { gen_helper_sve_ldbss_zss_mte
,
5875 gen_helper_sve_ldhss_be_zss_mte
,
5877 { gen_helper_sve_ldbsu_zss_mte
,
5878 gen_helper_sve_ldhsu_be_zss_mte
,
5879 gen_helper_sve_ldss_be_zss_mte
, } } },
5882 { { { gen_helper_sve_ldffbss_zsu_mte
,
5883 gen_helper_sve_ldffhss_be_zsu_mte
,
5885 { gen_helper_sve_ldffbsu_zsu_mte
,
5886 gen_helper_sve_ldffhsu_be_zsu_mte
,
5887 gen_helper_sve_ldffss_be_zsu_mte
, } },
5888 { { gen_helper_sve_ldffbss_zss_mte
,
5889 gen_helper_sve_ldffhss_be_zss_mte
,
5891 { gen_helper_sve_ldffbsu_zss_mte
,
5892 gen_helper_sve_ldffhsu_be_zss_mte
,
5893 gen_helper_sve_ldffss_be_zss_mte
, } } } } },
5896 /* Note that we overload xs=2 to indicate 64-bit offset. */
5897 static gen_helper_gvec_mem_scatter
* const
5898 gather_load_fn64
[2][2][2][3][2][4] = {
5899 { /* MTE Inactive */
5900 { /* Little-endian */
5901 { { { gen_helper_sve_ldbds_zsu
,
5902 gen_helper_sve_ldhds_le_zsu
,
5903 gen_helper_sve_ldsds_le_zsu
,
5905 { gen_helper_sve_ldbdu_zsu
,
5906 gen_helper_sve_ldhdu_le_zsu
,
5907 gen_helper_sve_ldsdu_le_zsu
,
5908 gen_helper_sve_lddd_le_zsu
, } },
5909 { { gen_helper_sve_ldbds_zss
,
5910 gen_helper_sve_ldhds_le_zss
,
5911 gen_helper_sve_ldsds_le_zss
,
5913 { gen_helper_sve_ldbdu_zss
,
5914 gen_helper_sve_ldhdu_le_zss
,
5915 gen_helper_sve_ldsdu_le_zss
,
5916 gen_helper_sve_lddd_le_zss
, } },
5917 { { gen_helper_sve_ldbds_zd
,
5918 gen_helper_sve_ldhds_le_zd
,
5919 gen_helper_sve_ldsds_le_zd
,
5921 { gen_helper_sve_ldbdu_zd
,
5922 gen_helper_sve_ldhdu_le_zd
,
5923 gen_helper_sve_ldsdu_le_zd
,
5924 gen_helper_sve_lddd_le_zd
, } } },
5927 { { { gen_helper_sve_ldffbds_zsu
,
5928 gen_helper_sve_ldffhds_le_zsu
,
5929 gen_helper_sve_ldffsds_le_zsu
,
5931 { gen_helper_sve_ldffbdu_zsu
,
5932 gen_helper_sve_ldffhdu_le_zsu
,
5933 gen_helper_sve_ldffsdu_le_zsu
,
5934 gen_helper_sve_ldffdd_le_zsu
, } },
5935 { { gen_helper_sve_ldffbds_zss
,
5936 gen_helper_sve_ldffhds_le_zss
,
5937 gen_helper_sve_ldffsds_le_zss
,
5939 { gen_helper_sve_ldffbdu_zss
,
5940 gen_helper_sve_ldffhdu_le_zss
,
5941 gen_helper_sve_ldffsdu_le_zss
,
5942 gen_helper_sve_ldffdd_le_zss
, } },
5943 { { gen_helper_sve_ldffbds_zd
,
5944 gen_helper_sve_ldffhds_le_zd
,
5945 gen_helper_sve_ldffsds_le_zd
,
5947 { gen_helper_sve_ldffbdu_zd
,
5948 gen_helper_sve_ldffhdu_le_zd
,
5949 gen_helper_sve_ldffsdu_le_zd
,
5950 gen_helper_sve_ldffdd_le_zd
, } } } },
5952 { { { gen_helper_sve_ldbds_zsu
,
5953 gen_helper_sve_ldhds_be_zsu
,
5954 gen_helper_sve_ldsds_be_zsu
,
5956 { gen_helper_sve_ldbdu_zsu
,
5957 gen_helper_sve_ldhdu_be_zsu
,
5958 gen_helper_sve_ldsdu_be_zsu
,
5959 gen_helper_sve_lddd_be_zsu
, } },
5960 { { gen_helper_sve_ldbds_zss
,
5961 gen_helper_sve_ldhds_be_zss
,
5962 gen_helper_sve_ldsds_be_zss
,
5964 { gen_helper_sve_ldbdu_zss
,
5965 gen_helper_sve_ldhdu_be_zss
,
5966 gen_helper_sve_ldsdu_be_zss
,
5967 gen_helper_sve_lddd_be_zss
, } },
5968 { { gen_helper_sve_ldbds_zd
,
5969 gen_helper_sve_ldhds_be_zd
,
5970 gen_helper_sve_ldsds_be_zd
,
5972 { gen_helper_sve_ldbdu_zd
,
5973 gen_helper_sve_ldhdu_be_zd
,
5974 gen_helper_sve_ldsdu_be_zd
,
5975 gen_helper_sve_lddd_be_zd
, } } },
5978 { { { gen_helper_sve_ldffbds_zsu
,
5979 gen_helper_sve_ldffhds_be_zsu
,
5980 gen_helper_sve_ldffsds_be_zsu
,
5982 { gen_helper_sve_ldffbdu_zsu
,
5983 gen_helper_sve_ldffhdu_be_zsu
,
5984 gen_helper_sve_ldffsdu_be_zsu
,
5985 gen_helper_sve_ldffdd_be_zsu
, } },
5986 { { gen_helper_sve_ldffbds_zss
,
5987 gen_helper_sve_ldffhds_be_zss
,
5988 gen_helper_sve_ldffsds_be_zss
,
5990 { gen_helper_sve_ldffbdu_zss
,
5991 gen_helper_sve_ldffhdu_be_zss
,
5992 gen_helper_sve_ldffsdu_be_zss
,
5993 gen_helper_sve_ldffdd_be_zss
, } },
5994 { { gen_helper_sve_ldffbds_zd
,
5995 gen_helper_sve_ldffhds_be_zd
,
5996 gen_helper_sve_ldffsds_be_zd
,
5998 { gen_helper_sve_ldffbdu_zd
,
5999 gen_helper_sve_ldffhdu_be_zd
,
6000 gen_helper_sve_ldffsdu_be_zd
,
6001 gen_helper_sve_ldffdd_be_zd
, } } } } },
6003 { /* Little-endian */
6004 { { { gen_helper_sve_ldbds_zsu_mte
,
6005 gen_helper_sve_ldhds_le_zsu_mte
,
6006 gen_helper_sve_ldsds_le_zsu_mte
,
6008 { gen_helper_sve_ldbdu_zsu_mte
,
6009 gen_helper_sve_ldhdu_le_zsu_mte
,
6010 gen_helper_sve_ldsdu_le_zsu_mte
,
6011 gen_helper_sve_lddd_le_zsu_mte
, } },
6012 { { gen_helper_sve_ldbds_zss_mte
,
6013 gen_helper_sve_ldhds_le_zss_mte
,
6014 gen_helper_sve_ldsds_le_zss_mte
,
6016 { gen_helper_sve_ldbdu_zss_mte
,
6017 gen_helper_sve_ldhdu_le_zss_mte
,
6018 gen_helper_sve_ldsdu_le_zss_mte
,
6019 gen_helper_sve_lddd_le_zss_mte
, } },
6020 { { gen_helper_sve_ldbds_zd_mte
,
6021 gen_helper_sve_ldhds_le_zd_mte
,
6022 gen_helper_sve_ldsds_le_zd_mte
,
6024 { gen_helper_sve_ldbdu_zd_mte
,
6025 gen_helper_sve_ldhdu_le_zd_mte
,
6026 gen_helper_sve_ldsdu_le_zd_mte
,
6027 gen_helper_sve_lddd_le_zd_mte
, } } },
6030 { { { gen_helper_sve_ldffbds_zsu_mte
,
6031 gen_helper_sve_ldffhds_le_zsu_mte
,
6032 gen_helper_sve_ldffsds_le_zsu_mte
,
6034 { gen_helper_sve_ldffbdu_zsu_mte
,
6035 gen_helper_sve_ldffhdu_le_zsu_mte
,
6036 gen_helper_sve_ldffsdu_le_zsu_mte
,
6037 gen_helper_sve_ldffdd_le_zsu_mte
, } },
6038 { { gen_helper_sve_ldffbds_zss_mte
,
6039 gen_helper_sve_ldffhds_le_zss_mte
,
6040 gen_helper_sve_ldffsds_le_zss_mte
,
6042 { gen_helper_sve_ldffbdu_zss_mte
,
6043 gen_helper_sve_ldffhdu_le_zss_mte
,
6044 gen_helper_sve_ldffsdu_le_zss_mte
,
6045 gen_helper_sve_ldffdd_le_zss_mte
, } },
6046 { { gen_helper_sve_ldffbds_zd_mte
,
6047 gen_helper_sve_ldffhds_le_zd_mte
,
6048 gen_helper_sve_ldffsds_le_zd_mte
,
6050 { gen_helper_sve_ldffbdu_zd_mte
,
6051 gen_helper_sve_ldffhdu_le_zd_mte
,
6052 gen_helper_sve_ldffsdu_le_zd_mte
,
6053 gen_helper_sve_ldffdd_le_zd_mte
, } } } },
6055 { { { gen_helper_sve_ldbds_zsu_mte
,
6056 gen_helper_sve_ldhds_be_zsu_mte
,
6057 gen_helper_sve_ldsds_be_zsu_mte
,
6059 { gen_helper_sve_ldbdu_zsu_mte
,
6060 gen_helper_sve_ldhdu_be_zsu_mte
,
6061 gen_helper_sve_ldsdu_be_zsu_mte
,
6062 gen_helper_sve_lddd_be_zsu_mte
, } },
6063 { { gen_helper_sve_ldbds_zss_mte
,
6064 gen_helper_sve_ldhds_be_zss_mte
,
6065 gen_helper_sve_ldsds_be_zss_mte
,
6067 { gen_helper_sve_ldbdu_zss_mte
,
6068 gen_helper_sve_ldhdu_be_zss_mte
,
6069 gen_helper_sve_ldsdu_be_zss_mte
,
6070 gen_helper_sve_lddd_be_zss_mte
, } },
6071 { { gen_helper_sve_ldbds_zd_mte
,
6072 gen_helper_sve_ldhds_be_zd_mte
,
6073 gen_helper_sve_ldsds_be_zd_mte
,
6075 { gen_helper_sve_ldbdu_zd_mte
,
6076 gen_helper_sve_ldhdu_be_zd_mte
,
6077 gen_helper_sve_ldsdu_be_zd_mte
,
6078 gen_helper_sve_lddd_be_zd_mte
, } } },
6081 { { { gen_helper_sve_ldffbds_zsu_mte
,
6082 gen_helper_sve_ldffhds_be_zsu_mte
,
6083 gen_helper_sve_ldffsds_be_zsu_mte
,
6085 { gen_helper_sve_ldffbdu_zsu_mte
,
6086 gen_helper_sve_ldffhdu_be_zsu_mte
,
6087 gen_helper_sve_ldffsdu_be_zsu_mte
,
6088 gen_helper_sve_ldffdd_be_zsu_mte
, } },
6089 { { gen_helper_sve_ldffbds_zss_mte
,
6090 gen_helper_sve_ldffhds_be_zss_mte
,
6091 gen_helper_sve_ldffsds_be_zss_mte
,
6093 { gen_helper_sve_ldffbdu_zss_mte
,
6094 gen_helper_sve_ldffhdu_be_zss_mte
,
6095 gen_helper_sve_ldffsdu_be_zss_mte
,
6096 gen_helper_sve_ldffdd_be_zss_mte
, } },
6097 { { gen_helper_sve_ldffbds_zd_mte
,
6098 gen_helper_sve_ldffhds_be_zd_mte
,
6099 gen_helper_sve_ldffsds_be_zd_mte
,
6101 { gen_helper_sve_ldffbdu_zd_mte
,
6102 gen_helper_sve_ldffhdu_be_zd_mte
,
6103 gen_helper_sve_ldffsdu_be_zd_mte
,
6104 gen_helper_sve_ldffdd_be_zd_mte
, } } } } },
6107 static bool trans_LD1_zprz(DisasContext
*s
, arg_LD1_zprz
*a
)
6109 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6110 bool be
= s
->be_data
== MO_BE
;
6111 bool mte
= s
->mte_active
[0];
6113 if (!sve_access_check(s
)) {
6119 fn
= gather_load_fn32
[mte
][be
][a
->ff
][a
->xs
][a
->u
][a
->msz
];
6122 fn
= gather_load_fn64
[mte
][be
][a
->ff
][a
->xs
][a
->u
][a
->msz
];
6127 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rm
, a
->scale
* a
->msz
,
6128 cpu_reg_sp(s
, a
->rn
), a
->msz
, false, fn
);
6132 static bool trans_LD1_zpiz(DisasContext
*s
, arg_LD1_zpiz
*a
)
6134 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6135 bool be
= s
->be_data
== MO_BE
;
6136 bool mte
= s
->mte_active
[0];
6139 if (a
->esz
< a
->msz
|| (a
->esz
== a
->msz
&& !a
->u
)) {
6142 if (!sve_access_check(s
)) {
6148 fn
= gather_load_fn32
[mte
][be
][a
->ff
][0][a
->u
][a
->msz
];
6151 fn
= gather_load_fn64
[mte
][be
][a
->ff
][2][a
->u
][a
->msz
];
6156 /* Treat LD1_zpiz (zn[x] + imm) the same way as LD1_zprz (rn + zm[x])
6157 * by loading the immediate into the scalar parameter.
6159 imm
= tcg_const_i64(a
->imm
<< a
->msz
);
6160 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0, imm
, a
->msz
, false, fn
);
6161 tcg_temp_free_i64(imm
);
6165 static bool trans_LDNT1_zprz(DisasContext
*s
, arg_LD1_zprz
*a
)
6167 if (!dc_isar_feature(aa64_sve2
, s
)) {
6170 return trans_LD1_zprz(s
, a
);
6173 /* Indexed by [mte][be][xs][msz]. */
6174 static gen_helper_gvec_mem_scatter
* const scatter_store_fn32
[2][2][2][3] = {
6175 { /* MTE Inactive */
6176 { /* Little-endian */
6177 { gen_helper_sve_stbs_zsu
,
6178 gen_helper_sve_sths_le_zsu
,
6179 gen_helper_sve_stss_le_zsu
, },
6180 { gen_helper_sve_stbs_zss
,
6181 gen_helper_sve_sths_le_zss
,
6182 gen_helper_sve_stss_le_zss
, } },
6184 { gen_helper_sve_stbs_zsu
,
6185 gen_helper_sve_sths_be_zsu
,
6186 gen_helper_sve_stss_be_zsu
, },
6187 { gen_helper_sve_stbs_zss
,
6188 gen_helper_sve_sths_be_zss
,
6189 gen_helper_sve_stss_be_zss
, } } },
6191 { /* Little-endian */
6192 { gen_helper_sve_stbs_zsu_mte
,
6193 gen_helper_sve_sths_le_zsu_mte
,
6194 gen_helper_sve_stss_le_zsu_mte
, },
6195 { gen_helper_sve_stbs_zss_mte
,
6196 gen_helper_sve_sths_le_zss_mte
,
6197 gen_helper_sve_stss_le_zss_mte
, } },
6199 { gen_helper_sve_stbs_zsu_mte
,
6200 gen_helper_sve_sths_be_zsu_mte
,
6201 gen_helper_sve_stss_be_zsu_mte
, },
6202 { gen_helper_sve_stbs_zss_mte
,
6203 gen_helper_sve_sths_be_zss_mte
,
6204 gen_helper_sve_stss_be_zss_mte
, } } },
6207 /* Note that we overload xs=2 to indicate 64-bit offset. */
6208 static gen_helper_gvec_mem_scatter
* const scatter_store_fn64
[2][2][3][4] = {
6209 { /* MTE Inactive */
6210 { /* Little-endian */
6211 { gen_helper_sve_stbd_zsu
,
6212 gen_helper_sve_sthd_le_zsu
,
6213 gen_helper_sve_stsd_le_zsu
,
6214 gen_helper_sve_stdd_le_zsu
, },
6215 { gen_helper_sve_stbd_zss
,
6216 gen_helper_sve_sthd_le_zss
,
6217 gen_helper_sve_stsd_le_zss
,
6218 gen_helper_sve_stdd_le_zss
, },
6219 { gen_helper_sve_stbd_zd
,
6220 gen_helper_sve_sthd_le_zd
,
6221 gen_helper_sve_stsd_le_zd
,
6222 gen_helper_sve_stdd_le_zd
, } },
6224 { gen_helper_sve_stbd_zsu
,
6225 gen_helper_sve_sthd_be_zsu
,
6226 gen_helper_sve_stsd_be_zsu
,
6227 gen_helper_sve_stdd_be_zsu
, },
6228 { gen_helper_sve_stbd_zss
,
6229 gen_helper_sve_sthd_be_zss
,
6230 gen_helper_sve_stsd_be_zss
,
6231 gen_helper_sve_stdd_be_zss
, },
6232 { gen_helper_sve_stbd_zd
,
6233 gen_helper_sve_sthd_be_zd
,
6234 gen_helper_sve_stsd_be_zd
,
6235 gen_helper_sve_stdd_be_zd
, } } },
6236 { /* MTE Inactive */
6237 { /* Little-endian */
6238 { gen_helper_sve_stbd_zsu_mte
,
6239 gen_helper_sve_sthd_le_zsu_mte
,
6240 gen_helper_sve_stsd_le_zsu_mte
,
6241 gen_helper_sve_stdd_le_zsu_mte
, },
6242 { gen_helper_sve_stbd_zss_mte
,
6243 gen_helper_sve_sthd_le_zss_mte
,
6244 gen_helper_sve_stsd_le_zss_mte
,
6245 gen_helper_sve_stdd_le_zss_mte
, },
6246 { gen_helper_sve_stbd_zd_mte
,
6247 gen_helper_sve_sthd_le_zd_mte
,
6248 gen_helper_sve_stsd_le_zd_mte
,
6249 gen_helper_sve_stdd_le_zd_mte
, } },
6251 { gen_helper_sve_stbd_zsu_mte
,
6252 gen_helper_sve_sthd_be_zsu_mte
,
6253 gen_helper_sve_stsd_be_zsu_mte
,
6254 gen_helper_sve_stdd_be_zsu_mte
, },
6255 { gen_helper_sve_stbd_zss_mte
,
6256 gen_helper_sve_sthd_be_zss_mte
,
6257 gen_helper_sve_stsd_be_zss_mte
,
6258 gen_helper_sve_stdd_be_zss_mte
, },
6259 { gen_helper_sve_stbd_zd_mte
,
6260 gen_helper_sve_sthd_be_zd_mte
,
6261 gen_helper_sve_stsd_be_zd_mte
,
6262 gen_helper_sve_stdd_be_zd_mte
, } } },
6265 static bool trans_ST1_zprz(DisasContext
*s
, arg_ST1_zprz
*a
)
6267 gen_helper_gvec_mem_scatter
*fn
;
6268 bool be
= s
->be_data
== MO_BE
;
6269 bool mte
= s
->mte_active
[0];
6271 if (a
->esz
< a
->msz
|| (a
->msz
== 0 && a
->scale
)) {
6274 if (!sve_access_check(s
)) {
6279 fn
= scatter_store_fn32
[mte
][be
][a
->xs
][a
->msz
];
6282 fn
= scatter_store_fn64
[mte
][be
][a
->xs
][a
->msz
];
6285 g_assert_not_reached();
6287 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rm
, a
->scale
* a
->msz
,
6288 cpu_reg_sp(s
, a
->rn
), a
->msz
, true, fn
);
6292 static bool trans_ST1_zpiz(DisasContext
*s
, arg_ST1_zpiz
*a
)
6294 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6295 bool be
= s
->be_data
== MO_BE
;
6296 bool mte
= s
->mte_active
[0];
6299 if (a
->esz
< a
->msz
) {
6302 if (!sve_access_check(s
)) {
6308 fn
= scatter_store_fn32
[mte
][be
][0][a
->msz
];
6311 fn
= scatter_store_fn64
[mte
][be
][2][a
->msz
];
6316 /* Treat ST1_zpiz (zn[x] + imm) the same way as ST1_zprz (rn + zm[x])
6317 * by loading the immediate into the scalar parameter.
6319 imm
= tcg_const_i64(a
->imm
<< a
->msz
);
6320 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0, imm
, a
->msz
, true, fn
);
6321 tcg_temp_free_i64(imm
);
6325 static bool trans_STNT1_zprz(DisasContext
*s
, arg_ST1_zprz
*a
)
6327 if (!dc_isar_feature(aa64_sve2
, s
)) {
6330 return trans_ST1_zprz(s
, a
);
6337 static bool trans_PRF(DisasContext
*s
, arg_PRF
*a
)
6339 /* Prefetch is a nop within QEMU. */
6340 (void)sve_access_check(s
);
6344 static bool trans_PRF_rr(DisasContext
*s
, arg_PRF_rr
*a
)
6349 /* Prefetch is a nop within QEMU. */
6350 (void)sve_access_check(s
);
6357 * TODO: The implementation so far could handle predicated merging movprfx.
6358 * The helper functions as written take an extra source register to
6359 * use in the operation, but the result is only written when predication
6360 * succeeds. For unpredicated movprfx, we need to rearrange the helpers
6361 * to allow the final write back to the destination to be unconditional.
6362 * For predicated zeroing movprfx, we need to rearrange the helpers to
6363 * allow the final write back to zero inactives.
6365 * In the meantime, just emit the moves.
6368 static bool trans_MOVPRFX(DisasContext
*s
, arg_MOVPRFX
*a
)
6370 return do_mov_z(s
, a
->rd
, a
->rn
);
6373 static bool trans_MOVPRFX_m(DisasContext
*s
, arg_rpr_esz
*a
)
6375 if (sve_access_check(s
)) {
6376 do_sel_z(s
, a
->rd
, a
->rn
, a
->rd
, a
->pg
, a
->esz
);
6381 static bool trans_MOVPRFX_z(DisasContext
*s
, arg_rpr_esz
*a
)
6383 return do_movz_zpz(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
, false);
6387 * SVE2 Integer Multiply - Unpredicated
6390 static bool trans_MUL_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6392 if (!dc_isar_feature(aa64_sve2
, s
)) {
6395 if (sve_access_check(s
)) {
6396 gen_gvec_fn_zzz(s
, tcg_gen_gvec_mul
, a
->esz
, a
->rd
, a
->rn
, a
->rm
);
6401 static bool do_sve2_zzz_ool(DisasContext
*s
, arg_rrr_esz
*a
,
6402 gen_helper_gvec_3
*fn
)
6404 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
6407 if (sve_access_check(s
)) {
6408 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, 0);
6413 static bool trans_SMULH_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6415 static gen_helper_gvec_3
* const fns
[4] = {
6416 gen_helper_gvec_smulh_b
, gen_helper_gvec_smulh_h
,
6417 gen_helper_gvec_smulh_s
, gen_helper_gvec_smulh_d
,
6419 return do_sve2_zzz_ool(s
, a
, fns
[a
->esz
]);
6422 static bool trans_UMULH_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6424 static gen_helper_gvec_3
* const fns
[4] = {
6425 gen_helper_gvec_umulh_b
, gen_helper_gvec_umulh_h
,
6426 gen_helper_gvec_umulh_s
, gen_helper_gvec_umulh_d
,
6428 return do_sve2_zzz_ool(s
, a
, fns
[a
->esz
]);
6431 static bool trans_PMUL_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6433 return do_sve2_zzz_ool(s
, a
, gen_helper_gvec_pmul_b
);
6436 static bool trans_SQDMULH_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6438 static gen_helper_gvec_3
* const fns
[4] = {
6439 gen_helper_sve2_sqdmulh_b
, gen_helper_sve2_sqdmulh_h
,
6440 gen_helper_sve2_sqdmulh_s
, gen_helper_sve2_sqdmulh_d
,
6442 return do_sve2_zzz_ool(s
, a
, fns
[a
->esz
]);
6445 static bool trans_SQRDMULH_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6447 static gen_helper_gvec_3
* const fns
[4] = {
6448 gen_helper_sve2_sqrdmulh_b
, gen_helper_sve2_sqrdmulh_h
,
6449 gen_helper_sve2_sqrdmulh_s
, gen_helper_sve2_sqrdmulh_d
,
6451 return do_sve2_zzz_ool(s
, a
, fns
[a
->esz
]);
6455 * SVE2 Integer - Predicated
6458 static bool do_sve2_zpzz_ool(DisasContext
*s
, arg_rprr_esz
*a
,
6459 gen_helper_gvec_4
*fn
)
6461 if (!dc_isar_feature(aa64_sve2
, s
)) {
6464 return do_zpzz_ool(s
, a
, fn
);
6467 static bool trans_SADALP_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
6469 static gen_helper_gvec_4
* const fns
[3] = {
6470 gen_helper_sve2_sadalp_zpzz_h
,
6471 gen_helper_sve2_sadalp_zpzz_s
,
6472 gen_helper_sve2_sadalp_zpzz_d
,
6477 return do_sve2_zpzz_ool(s
, a
, fns
[a
->esz
- 1]);
6480 static bool trans_UADALP_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
6482 static gen_helper_gvec_4
* const fns
[3] = {
6483 gen_helper_sve2_uadalp_zpzz_h
,
6484 gen_helper_sve2_uadalp_zpzz_s
,
6485 gen_helper_sve2_uadalp_zpzz_d
,
6490 return do_sve2_zpzz_ool(s
, a
, fns
[a
->esz
- 1]);
6494 * SVE2 integer unary operations (predicated)
6497 static bool do_sve2_zpz_ool(DisasContext
*s
, arg_rpr_esz
*a
,
6498 gen_helper_gvec_3
*fn
)
6500 if (!dc_isar_feature(aa64_sve2
, s
)) {
6503 return do_zpz_ool(s
, a
, fn
);
6506 static bool trans_URECPE(DisasContext
*s
, arg_rpr_esz
*a
)
6511 return do_sve2_zpz_ool(s
, a
, gen_helper_sve2_urecpe_s
);
6514 static bool trans_URSQRTE(DisasContext
*s
, arg_rpr_esz
*a
)
6519 return do_sve2_zpz_ool(s
, a
, gen_helper_sve2_ursqrte_s
);
6522 static bool trans_SQABS(DisasContext
*s
, arg_rpr_esz
*a
)
6524 static gen_helper_gvec_3
* const fns
[4] = {
6525 gen_helper_sve2_sqabs_b
, gen_helper_sve2_sqabs_h
,
6526 gen_helper_sve2_sqabs_s
, gen_helper_sve2_sqabs_d
,
6528 return do_sve2_zpz_ool(s
, a
, fns
[a
->esz
]);
6531 static bool trans_SQNEG(DisasContext
*s
, arg_rpr_esz
*a
)
6533 static gen_helper_gvec_3
* const fns
[4] = {
6534 gen_helper_sve2_sqneg_b
, gen_helper_sve2_sqneg_h
,
6535 gen_helper_sve2_sqneg_s
, gen_helper_sve2_sqneg_d
,
6537 return do_sve2_zpz_ool(s
, a
, fns
[a
->esz
]);
6540 #define DO_SVE2_ZPZZ(NAME, name) \
6541 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
6543 static gen_helper_gvec_4 * const fns[4] = { \
6544 gen_helper_sve2_##name##_zpzz_b, gen_helper_sve2_##name##_zpzz_h, \
6545 gen_helper_sve2_##name##_zpzz_s, gen_helper_sve2_##name##_zpzz_d, \
6547 return do_sve2_zpzz_ool(s, a, fns[a->esz]); \
6550 DO_SVE2_ZPZZ(SQSHL
, sqshl
)
6551 DO_SVE2_ZPZZ(SQRSHL
, sqrshl
)
6552 DO_SVE2_ZPZZ(SRSHL
, srshl
)
6554 DO_SVE2_ZPZZ(UQSHL
, uqshl
)
6555 DO_SVE2_ZPZZ(UQRSHL
, uqrshl
)
6556 DO_SVE2_ZPZZ(URSHL
, urshl
)
6558 DO_SVE2_ZPZZ(SHADD
, shadd
)
6559 DO_SVE2_ZPZZ(SRHADD
, srhadd
)
6560 DO_SVE2_ZPZZ(SHSUB
, shsub
)
6562 DO_SVE2_ZPZZ(UHADD
, uhadd
)
6563 DO_SVE2_ZPZZ(URHADD
, urhadd
)
6564 DO_SVE2_ZPZZ(UHSUB
, uhsub
)
6566 DO_SVE2_ZPZZ(ADDP
, addp
)
6567 DO_SVE2_ZPZZ(SMAXP
, smaxp
)
6568 DO_SVE2_ZPZZ(UMAXP
, umaxp
)
6569 DO_SVE2_ZPZZ(SMINP
, sminp
)
6570 DO_SVE2_ZPZZ(UMINP
, uminp
)
6572 DO_SVE2_ZPZZ(SQADD_zpzz
, sqadd
)
6573 DO_SVE2_ZPZZ(UQADD_zpzz
, uqadd
)
6574 DO_SVE2_ZPZZ(SQSUB_zpzz
, sqsub
)
6575 DO_SVE2_ZPZZ(UQSUB_zpzz
, uqsub
)
6576 DO_SVE2_ZPZZ(SUQADD
, suqadd
)
6577 DO_SVE2_ZPZZ(USQADD
, usqadd
)
6580 * SVE2 Widening Integer Arithmetic
6583 static bool do_sve2_zzw_ool(DisasContext
*s
, arg_rrr_esz
*a
,
6584 gen_helper_gvec_3
*fn
, int data
)
6586 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
6589 if (sve_access_check(s
)) {
6590 unsigned vsz
= vec_full_reg_size(s
);
6591 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, a
->rd
),
6592 vec_full_reg_offset(s
, a
->rn
),
6593 vec_full_reg_offset(s
, a
->rm
),
6594 vsz
, vsz
, data
, fn
);
6599 #define DO_SVE2_ZZZ_TB(NAME, name, SEL1, SEL2) \
6600 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
6602 static gen_helper_gvec_3 * const fns[4] = { \
6603 NULL, gen_helper_sve2_##name##_h, \
6604 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
6606 return do_sve2_zzw_ool(s, a, fns[a->esz], (SEL2 << 1) | SEL1); \
6609 DO_SVE2_ZZZ_TB(SADDLB
, saddl
, false, false)
6610 DO_SVE2_ZZZ_TB(SSUBLB
, ssubl
, false, false)
6611 DO_SVE2_ZZZ_TB(SABDLB
, sabdl
, false, false)
6613 DO_SVE2_ZZZ_TB(UADDLB
, uaddl
, false, false)
6614 DO_SVE2_ZZZ_TB(USUBLB
, usubl
, false, false)
6615 DO_SVE2_ZZZ_TB(UABDLB
, uabdl
, false, false)
6617 DO_SVE2_ZZZ_TB(SADDLT
, saddl
, true, true)
6618 DO_SVE2_ZZZ_TB(SSUBLT
, ssubl
, true, true)
6619 DO_SVE2_ZZZ_TB(SABDLT
, sabdl
, true, true)
6621 DO_SVE2_ZZZ_TB(UADDLT
, uaddl
, true, true)
6622 DO_SVE2_ZZZ_TB(USUBLT
, usubl
, true, true)
6623 DO_SVE2_ZZZ_TB(UABDLT
, uabdl
, true, true)
6625 DO_SVE2_ZZZ_TB(SADDLBT
, saddl
, false, true)
6626 DO_SVE2_ZZZ_TB(SSUBLBT
, ssubl
, false, true)
6627 DO_SVE2_ZZZ_TB(SSUBLTB
, ssubl
, true, false)
6629 DO_SVE2_ZZZ_TB(SQDMULLB_zzz
, sqdmull_zzz
, false, false)
6630 DO_SVE2_ZZZ_TB(SQDMULLT_zzz
, sqdmull_zzz
, true, true)
6632 DO_SVE2_ZZZ_TB(SMULLB_zzz
, smull_zzz
, false, false)
6633 DO_SVE2_ZZZ_TB(SMULLT_zzz
, smull_zzz
, true, true)
6635 DO_SVE2_ZZZ_TB(UMULLB_zzz
, umull_zzz
, false, false)
6636 DO_SVE2_ZZZ_TB(UMULLT_zzz
, umull_zzz
, true, true)
6638 static bool do_eor_tb(DisasContext
*s
, arg_rrr_esz
*a
, bool sel1
)
6640 static gen_helper_gvec_3
* const fns
[4] = {
6641 gen_helper_sve2_eoril_b
, gen_helper_sve2_eoril_h
,
6642 gen_helper_sve2_eoril_s
, gen_helper_sve2_eoril_d
,
6644 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], (!sel1
<< 1) | sel1
);
6647 static bool trans_EORBT(DisasContext
*s
, arg_rrr_esz
*a
)
6649 return do_eor_tb(s
, a
, false);
6652 static bool trans_EORTB(DisasContext
*s
, arg_rrr_esz
*a
)
6654 return do_eor_tb(s
, a
, true);
6657 static bool do_trans_pmull(DisasContext
*s
, arg_rrr_esz
*a
, bool sel
)
6659 static gen_helper_gvec_3
* const fns
[4] = {
6660 gen_helper_gvec_pmull_q
, gen_helper_sve2_pmull_h
,
6661 NULL
, gen_helper_sve2_pmull_d
,
6663 if (a
->esz
== 0 && !dc_isar_feature(aa64_sve2_pmull128
, s
)) {
6666 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], sel
);
6669 static bool trans_PMULLB(DisasContext
*s
, arg_rrr_esz
*a
)
6671 return do_trans_pmull(s
, a
, false);
6674 static bool trans_PMULLT(DisasContext
*s
, arg_rrr_esz
*a
)
6676 return do_trans_pmull(s
, a
, true);
6679 #define DO_SVE2_ZZZ_WTB(NAME, name, SEL2) \
6680 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
6682 static gen_helper_gvec_3 * const fns[4] = { \
6683 NULL, gen_helper_sve2_##name##_h, \
6684 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
6686 return do_sve2_zzw_ool(s, a, fns[a->esz], SEL2); \
6689 DO_SVE2_ZZZ_WTB(SADDWB
, saddw
, false)
6690 DO_SVE2_ZZZ_WTB(SADDWT
, saddw
, true)
6691 DO_SVE2_ZZZ_WTB(SSUBWB
, ssubw
, false)
6692 DO_SVE2_ZZZ_WTB(SSUBWT
, ssubw
, true)
6694 DO_SVE2_ZZZ_WTB(UADDWB
, uaddw
, false)
6695 DO_SVE2_ZZZ_WTB(UADDWT
, uaddw
, true)
6696 DO_SVE2_ZZZ_WTB(USUBWB
, usubw
, false)
6697 DO_SVE2_ZZZ_WTB(USUBWT
, usubw
, true)
6699 static void gen_sshll_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t imm
)
6703 int halfbits
= 4 << vece
;
6706 if (shl
== halfbits
) {
6707 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6708 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(halfbits
, halfbits
));
6709 tcg_gen_and_vec(vece
, d
, n
, t
);
6710 tcg_temp_free_vec(t
);
6712 tcg_gen_sari_vec(vece
, d
, n
, halfbits
);
6713 tcg_gen_shli_vec(vece
, d
, d
, shl
);
6716 tcg_gen_shli_vec(vece
, d
, n
, halfbits
);
6717 tcg_gen_sari_vec(vece
, d
, d
, halfbits
- shl
);
6721 static void gen_ushll_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int imm
)
6723 int halfbits
= 4 << vece
;
6725 int shl
= (imm
>> 1);
6729 mask
= MAKE_64BIT_MASK(0, halfbits
);
6731 mask
= dup_const(vece
, mask
);
6733 shift
= shl
- top
* halfbits
;
6735 tcg_gen_shri_i64(d
, n
, -shift
);
6737 tcg_gen_shli_i64(d
, n
, shift
);
6739 tcg_gen_andi_i64(d
, d
, mask
);
6742 static void gen_ushll16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6744 gen_ushll_i64(MO_16
, d
, n
, imm
);
6747 static void gen_ushll32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6749 gen_ushll_i64(MO_32
, d
, n
, imm
);
6752 static void gen_ushll64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6754 gen_ushll_i64(MO_64
, d
, n
, imm
);
6757 static void gen_ushll_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t imm
)
6759 int halfbits
= 4 << vece
;
6764 if (shl
== halfbits
) {
6765 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6766 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(halfbits
, halfbits
));
6767 tcg_gen_and_vec(vece
, d
, n
, t
);
6768 tcg_temp_free_vec(t
);
6770 tcg_gen_shri_vec(vece
, d
, n
, halfbits
);
6771 tcg_gen_shli_vec(vece
, d
, d
, shl
);
6775 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6776 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
6777 tcg_gen_and_vec(vece
, d
, n
, t
);
6778 tcg_temp_free_vec(t
);
6780 tcg_gen_shli_vec(vece
, d
, n
, halfbits
);
6781 tcg_gen_shri_vec(vece
, d
, d
, halfbits
- shl
);
6786 static bool do_sve2_shll_tb(DisasContext
*s
, arg_rri_esz
*a
,
6789 static const TCGOpcode sshll_list
[] = {
6790 INDEX_op_shli_vec
, INDEX_op_sari_vec
, 0
6792 static const TCGOpcode ushll_list
[] = {
6793 INDEX_op_shli_vec
, INDEX_op_shri_vec
, 0
6795 static const GVecGen2i ops
[2][3] = {
6796 { { .fniv
= gen_sshll_vec
,
6797 .opt_opc
= sshll_list
,
6798 .fno
= gen_helper_sve2_sshll_h
,
6800 { .fniv
= gen_sshll_vec
,
6801 .opt_opc
= sshll_list
,
6802 .fno
= gen_helper_sve2_sshll_s
,
6804 { .fniv
= gen_sshll_vec
,
6805 .opt_opc
= sshll_list
,
6806 .fno
= gen_helper_sve2_sshll_d
,
6808 { { .fni8
= gen_ushll16_i64
,
6809 .fniv
= gen_ushll_vec
,
6810 .opt_opc
= ushll_list
,
6811 .fno
= gen_helper_sve2_ushll_h
,
6813 { .fni8
= gen_ushll32_i64
,
6814 .fniv
= gen_ushll_vec
,
6815 .opt_opc
= ushll_list
,
6816 .fno
= gen_helper_sve2_ushll_s
,
6818 { .fni8
= gen_ushll64_i64
,
6819 .fniv
= gen_ushll_vec
,
6820 .opt_opc
= ushll_list
,
6821 .fno
= gen_helper_sve2_ushll_d
,
6825 if (a
->esz
< 0 || a
->esz
> 2 || !dc_isar_feature(aa64_sve2
, s
)) {
6828 if (sve_access_check(s
)) {
6829 unsigned vsz
= vec_full_reg_size(s
);
6830 tcg_gen_gvec_2i(vec_full_reg_offset(s
, a
->rd
),
6831 vec_full_reg_offset(s
, a
->rn
),
6832 vsz
, vsz
, (a
->imm
<< 1) | sel
,
6838 static bool trans_SSHLLB(DisasContext
*s
, arg_rri_esz
*a
)
6840 return do_sve2_shll_tb(s
, a
, false, false);
6843 static bool trans_SSHLLT(DisasContext
*s
, arg_rri_esz
*a
)
6845 return do_sve2_shll_tb(s
, a
, true, false);
6848 static bool trans_USHLLB(DisasContext
*s
, arg_rri_esz
*a
)
6850 return do_sve2_shll_tb(s
, a
, false, true);
6853 static bool trans_USHLLT(DisasContext
*s
, arg_rri_esz
*a
)
6855 return do_sve2_shll_tb(s
, a
, true, true);
6858 static bool trans_BEXT(DisasContext
*s
, arg_rrr_esz
*a
)
6860 static gen_helper_gvec_3
* const fns
[4] = {
6861 gen_helper_sve2_bext_b
, gen_helper_sve2_bext_h
,
6862 gen_helper_sve2_bext_s
, gen_helper_sve2_bext_d
,
6864 if (!dc_isar_feature(aa64_sve2_bitperm
, s
)) {
6867 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], 0);
6870 static bool trans_BDEP(DisasContext
*s
, arg_rrr_esz
*a
)
6872 static gen_helper_gvec_3
* const fns
[4] = {
6873 gen_helper_sve2_bdep_b
, gen_helper_sve2_bdep_h
,
6874 gen_helper_sve2_bdep_s
, gen_helper_sve2_bdep_d
,
6876 if (!dc_isar_feature(aa64_sve2_bitperm
, s
)) {
6879 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], 0);
6882 static bool trans_BGRP(DisasContext
*s
, arg_rrr_esz
*a
)
6884 static gen_helper_gvec_3
* const fns
[4] = {
6885 gen_helper_sve2_bgrp_b
, gen_helper_sve2_bgrp_h
,
6886 gen_helper_sve2_bgrp_s
, gen_helper_sve2_bgrp_d
,
6888 if (!dc_isar_feature(aa64_sve2_bitperm
, s
)) {
6891 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], 0);
6894 static bool do_cadd(DisasContext
*s
, arg_rrr_esz
*a
, bool sq
, bool rot
)
6896 static gen_helper_gvec_3
* const fns
[2][4] = {
6897 { gen_helper_sve2_cadd_b
, gen_helper_sve2_cadd_h
,
6898 gen_helper_sve2_cadd_s
, gen_helper_sve2_cadd_d
},
6899 { gen_helper_sve2_sqcadd_b
, gen_helper_sve2_sqcadd_h
,
6900 gen_helper_sve2_sqcadd_s
, gen_helper_sve2_sqcadd_d
},
6902 return do_sve2_zzw_ool(s
, a
, fns
[sq
][a
->esz
], rot
);
6905 static bool trans_CADD_rot90(DisasContext
*s
, arg_rrr_esz
*a
)
6907 return do_cadd(s
, a
, false, false);
6910 static bool trans_CADD_rot270(DisasContext
*s
, arg_rrr_esz
*a
)
6912 return do_cadd(s
, a
, false, true);
6915 static bool trans_SQCADD_rot90(DisasContext
*s
, arg_rrr_esz
*a
)
6917 return do_cadd(s
, a
, true, false);
6920 static bool trans_SQCADD_rot270(DisasContext
*s
, arg_rrr_esz
*a
)
6922 return do_cadd(s
, a
, true, true);
6925 static bool do_sve2_zzzz_ool(DisasContext
*s
, arg_rrrr_esz
*a
,
6926 gen_helper_gvec_4
*fn
, int data
)
6928 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
6931 if (sve_access_check(s
)) {
6932 gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, data
);
6937 static bool do_abal(DisasContext
*s
, arg_rrrr_esz
*a
, bool uns
, bool sel
)
6939 static gen_helper_gvec_4
* const fns
[2][4] = {
6940 { NULL
, gen_helper_sve2_sabal_h
,
6941 gen_helper_sve2_sabal_s
, gen_helper_sve2_sabal_d
},
6942 { NULL
, gen_helper_sve2_uabal_h
,
6943 gen_helper_sve2_uabal_s
, gen_helper_sve2_uabal_d
},
6945 return do_sve2_zzzz_ool(s
, a
, fns
[uns
][a
->esz
], sel
);
6948 static bool trans_SABALB(DisasContext
*s
, arg_rrrr_esz
*a
)
6950 return do_abal(s
, a
, false, false);
6953 static bool trans_SABALT(DisasContext
*s
, arg_rrrr_esz
*a
)
6955 return do_abal(s
, a
, false, true);
6958 static bool trans_UABALB(DisasContext
*s
, arg_rrrr_esz
*a
)
6960 return do_abal(s
, a
, true, false);
6963 static bool trans_UABALT(DisasContext
*s
, arg_rrrr_esz
*a
)
6965 return do_abal(s
, a
, true, true);
6968 static bool do_adcl(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
6970 static gen_helper_gvec_4
* const fns
[2] = {
6971 gen_helper_sve2_adcl_s
,
6972 gen_helper_sve2_adcl_d
,
6975 * Note that in this case the ESZ field encodes both size and sign.
6976 * Split out 'subtract' into bit 1 of the data field for the helper.
6978 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
& 1], (a
->esz
& 2) | sel
);
6981 static bool trans_ADCLB(DisasContext
*s
, arg_rrrr_esz
*a
)
6983 return do_adcl(s
, a
, false);
6986 static bool trans_ADCLT(DisasContext
*s
, arg_rrrr_esz
*a
)
6988 return do_adcl(s
, a
, true);
6991 static bool do_sve2_fn2i(DisasContext
*s
, arg_rri_esz
*a
, GVecGen2iFn
*fn
)
6993 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
6996 if (sve_access_check(s
)) {
6997 unsigned vsz
= vec_full_reg_size(s
);
6998 unsigned rd_ofs
= vec_full_reg_offset(s
, a
->rd
);
6999 unsigned rn_ofs
= vec_full_reg_offset(s
, a
->rn
);
7000 fn(a
->esz
, rd_ofs
, rn_ofs
, a
->imm
, vsz
, vsz
);
7005 static bool trans_SSRA(DisasContext
*s
, arg_rri_esz
*a
)
7007 return do_sve2_fn2i(s
, a
, gen_gvec_ssra
);
7010 static bool trans_USRA(DisasContext
*s
, arg_rri_esz
*a
)
7012 return do_sve2_fn2i(s
, a
, gen_gvec_usra
);
7015 static bool trans_SRSRA(DisasContext
*s
, arg_rri_esz
*a
)
7017 return do_sve2_fn2i(s
, a
, gen_gvec_srsra
);
7020 static bool trans_URSRA(DisasContext
*s
, arg_rri_esz
*a
)
7022 return do_sve2_fn2i(s
, a
, gen_gvec_ursra
);
7025 static bool trans_SRI(DisasContext
*s
, arg_rri_esz
*a
)
7027 return do_sve2_fn2i(s
, a
, gen_gvec_sri
);
7030 static bool trans_SLI(DisasContext
*s
, arg_rri_esz
*a
)
7032 return do_sve2_fn2i(s
, a
, gen_gvec_sli
);
7035 static bool do_sve2_fn_zzz(DisasContext
*s
, arg_rrr_esz
*a
, GVecGen3Fn
*fn
)
7037 if (!dc_isar_feature(aa64_sve2
, s
)) {
7040 if (sve_access_check(s
)) {
7041 gen_gvec_fn_zzz(s
, fn
, a
->esz
, a
->rd
, a
->rn
, a
->rm
);
7046 static bool trans_SABA(DisasContext
*s
, arg_rrr_esz
*a
)
7048 return do_sve2_fn_zzz(s
, a
, gen_gvec_saba
);
7051 static bool trans_UABA(DisasContext
*s
, arg_rrr_esz
*a
)
7053 return do_sve2_fn_zzz(s
, a
, gen_gvec_uaba
);
7056 static bool do_sve2_narrow_extract(DisasContext
*s
, arg_rri_esz
*a
,
7057 const GVecGen2 ops
[3])
7059 if (a
->esz
< 0 || a
->esz
> MO_32
|| a
->imm
!= 0 ||
7060 !dc_isar_feature(aa64_sve2
, s
)) {
7063 if (sve_access_check(s
)) {
7064 unsigned vsz
= vec_full_reg_size(s
);
7065 tcg_gen_gvec_2(vec_full_reg_offset(s
, a
->rd
),
7066 vec_full_reg_offset(s
, a
->rn
),
7067 vsz
, vsz
, &ops
[a
->esz
]);
7072 static const TCGOpcode sqxtn_list
[] = {
7073 INDEX_op_shli_vec
, INDEX_op_smin_vec
, INDEX_op_smax_vec
, 0
7076 static void gen_sqxtnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7078 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7079 int halfbits
= 4 << vece
;
7080 int64_t mask
= (1ull << halfbits
) - 1;
7081 int64_t min
= -1ull << (halfbits
- 1);
7082 int64_t max
= -min
- 1;
7084 tcg_gen_dupi_vec(vece
, t
, min
);
7085 tcg_gen_smax_vec(vece
, d
, n
, t
);
7086 tcg_gen_dupi_vec(vece
, t
, max
);
7087 tcg_gen_smin_vec(vece
, d
, d
, t
);
7088 tcg_gen_dupi_vec(vece
, t
, mask
);
7089 tcg_gen_and_vec(vece
, d
, d
, t
);
7090 tcg_temp_free_vec(t
);
7093 static bool trans_SQXTNB(DisasContext
*s
, arg_rri_esz
*a
)
7095 static const GVecGen2 ops
[3] = {
7096 { .fniv
= gen_sqxtnb_vec
,
7097 .opt_opc
= sqxtn_list
,
7098 .fno
= gen_helper_sve2_sqxtnb_h
,
7100 { .fniv
= gen_sqxtnb_vec
,
7101 .opt_opc
= sqxtn_list
,
7102 .fno
= gen_helper_sve2_sqxtnb_s
,
7104 { .fniv
= gen_sqxtnb_vec
,
7105 .opt_opc
= sqxtn_list
,
7106 .fno
= gen_helper_sve2_sqxtnb_d
,
7109 return do_sve2_narrow_extract(s
, a
, ops
);
7112 static void gen_sqxtnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7114 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7115 int halfbits
= 4 << vece
;
7116 int64_t mask
= (1ull << halfbits
) - 1;
7117 int64_t min
= -1ull << (halfbits
- 1);
7118 int64_t max
= -min
- 1;
7120 tcg_gen_dupi_vec(vece
, t
, min
);
7121 tcg_gen_smax_vec(vece
, n
, n
, t
);
7122 tcg_gen_dupi_vec(vece
, t
, max
);
7123 tcg_gen_smin_vec(vece
, n
, n
, t
);
7124 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7125 tcg_gen_dupi_vec(vece
, t
, mask
);
7126 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7127 tcg_temp_free_vec(t
);
7130 static bool trans_SQXTNT(DisasContext
*s
, arg_rri_esz
*a
)
7132 static const GVecGen2 ops
[3] = {
7133 { .fniv
= gen_sqxtnt_vec
,
7134 .opt_opc
= sqxtn_list
,
7136 .fno
= gen_helper_sve2_sqxtnt_h
,
7138 { .fniv
= gen_sqxtnt_vec
,
7139 .opt_opc
= sqxtn_list
,
7141 .fno
= gen_helper_sve2_sqxtnt_s
,
7143 { .fniv
= gen_sqxtnt_vec
,
7144 .opt_opc
= sqxtn_list
,
7146 .fno
= gen_helper_sve2_sqxtnt_d
,
7149 return do_sve2_narrow_extract(s
, a
, ops
);
7152 static const TCGOpcode uqxtn_list
[] = {
7153 INDEX_op_shli_vec
, INDEX_op_umin_vec
, 0
7156 static void gen_uqxtnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7158 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7159 int halfbits
= 4 << vece
;
7160 int64_t max
= (1ull << halfbits
) - 1;
7162 tcg_gen_dupi_vec(vece
, t
, max
);
7163 tcg_gen_umin_vec(vece
, d
, n
, t
);
7164 tcg_temp_free_vec(t
);
7167 static bool trans_UQXTNB(DisasContext
*s
, arg_rri_esz
*a
)
7169 static const GVecGen2 ops
[3] = {
7170 { .fniv
= gen_uqxtnb_vec
,
7171 .opt_opc
= uqxtn_list
,
7172 .fno
= gen_helper_sve2_uqxtnb_h
,
7174 { .fniv
= gen_uqxtnb_vec
,
7175 .opt_opc
= uqxtn_list
,
7176 .fno
= gen_helper_sve2_uqxtnb_s
,
7178 { .fniv
= gen_uqxtnb_vec
,
7179 .opt_opc
= uqxtn_list
,
7180 .fno
= gen_helper_sve2_uqxtnb_d
,
7183 return do_sve2_narrow_extract(s
, a
, ops
);
7186 static void gen_uqxtnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7188 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7189 int halfbits
= 4 << vece
;
7190 int64_t max
= (1ull << halfbits
) - 1;
7192 tcg_gen_dupi_vec(vece
, t
, max
);
7193 tcg_gen_umin_vec(vece
, n
, n
, t
);
7194 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7195 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7196 tcg_temp_free_vec(t
);
7199 static bool trans_UQXTNT(DisasContext
*s
, arg_rri_esz
*a
)
7201 static const GVecGen2 ops
[3] = {
7202 { .fniv
= gen_uqxtnt_vec
,
7203 .opt_opc
= uqxtn_list
,
7205 .fno
= gen_helper_sve2_uqxtnt_h
,
7207 { .fniv
= gen_uqxtnt_vec
,
7208 .opt_opc
= uqxtn_list
,
7210 .fno
= gen_helper_sve2_uqxtnt_s
,
7212 { .fniv
= gen_uqxtnt_vec
,
7213 .opt_opc
= uqxtn_list
,
7215 .fno
= gen_helper_sve2_uqxtnt_d
,
7218 return do_sve2_narrow_extract(s
, a
, ops
);
7221 static const TCGOpcode sqxtun_list
[] = {
7222 INDEX_op_shli_vec
, INDEX_op_umin_vec
, INDEX_op_smax_vec
, 0
7225 static void gen_sqxtunb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7227 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7228 int halfbits
= 4 << vece
;
7229 int64_t max
= (1ull << halfbits
) - 1;
7231 tcg_gen_dupi_vec(vece
, t
, 0);
7232 tcg_gen_smax_vec(vece
, d
, n
, t
);
7233 tcg_gen_dupi_vec(vece
, t
, max
);
7234 tcg_gen_umin_vec(vece
, d
, d
, t
);
7235 tcg_temp_free_vec(t
);
7238 static bool trans_SQXTUNB(DisasContext
*s
, arg_rri_esz
*a
)
7240 static const GVecGen2 ops
[3] = {
7241 { .fniv
= gen_sqxtunb_vec
,
7242 .opt_opc
= sqxtun_list
,
7243 .fno
= gen_helper_sve2_sqxtunb_h
,
7245 { .fniv
= gen_sqxtunb_vec
,
7246 .opt_opc
= sqxtun_list
,
7247 .fno
= gen_helper_sve2_sqxtunb_s
,
7249 { .fniv
= gen_sqxtunb_vec
,
7250 .opt_opc
= sqxtun_list
,
7251 .fno
= gen_helper_sve2_sqxtunb_d
,
7254 return do_sve2_narrow_extract(s
, a
, ops
);
7257 static void gen_sqxtunt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7259 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7260 int halfbits
= 4 << vece
;
7261 int64_t max
= (1ull << halfbits
) - 1;
7263 tcg_gen_dupi_vec(vece
, t
, 0);
7264 tcg_gen_smax_vec(vece
, n
, n
, t
);
7265 tcg_gen_dupi_vec(vece
, t
, max
);
7266 tcg_gen_umin_vec(vece
, n
, n
, t
);
7267 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7268 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7269 tcg_temp_free_vec(t
);
7272 static bool trans_SQXTUNT(DisasContext
*s
, arg_rri_esz
*a
)
7274 static const GVecGen2 ops
[3] = {
7275 { .fniv
= gen_sqxtunt_vec
,
7276 .opt_opc
= sqxtun_list
,
7278 .fno
= gen_helper_sve2_sqxtunt_h
,
7280 { .fniv
= gen_sqxtunt_vec
,
7281 .opt_opc
= sqxtun_list
,
7283 .fno
= gen_helper_sve2_sqxtunt_s
,
7285 { .fniv
= gen_sqxtunt_vec
,
7286 .opt_opc
= sqxtun_list
,
7288 .fno
= gen_helper_sve2_sqxtunt_d
,
7291 return do_sve2_narrow_extract(s
, a
, ops
);
7294 static bool do_sve2_shr_narrow(DisasContext
*s
, arg_rri_esz
*a
,
7295 const GVecGen2i ops
[3])
7297 if (a
->esz
< 0 || a
->esz
> MO_32
|| !dc_isar_feature(aa64_sve2
, s
)) {
7300 assert(a
->imm
> 0 && a
->imm
<= (8 << a
->esz
));
7301 if (sve_access_check(s
)) {
7302 unsigned vsz
= vec_full_reg_size(s
);
7303 tcg_gen_gvec_2i(vec_full_reg_offset(s
, a
->rd
),
7304 vec_full_reg_offset(s
, a
->rn
),
7305 vsz
, vsz
, a
->imm
, &ops
[a
->esz
]);
7310 static void gen_shrnb_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int shr
)
7312 int halfbits
= 4 << vece
;
7313 uint64_t mask
= dup_const(vece
, MAKE_64BIT_MASK(0, halfbits
));
7315 tcg_gen_shri_i64(d
, n
, shr
);
7316 tcg_gen_andi_i64(d
, d
, mask
);
7319 static void gen_shrnb16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7321 gen_shrnb_i64(MO_16
, d
, n
, shr
);
7324 static void gen_shrnb32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7326 gen_shrnb_i64(MO_32
, d
, n
, shr
);
7329 static void gen_shrnb64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7331 gen_shrnb_i64(MO_64
, d
, n
, shr
);
7334 static void gen_shrnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t shr
)
7336 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7337 int halfbits
= 4 << vece
;
7338 uint64_t mask
= MAKE_64BIT_MASK(0, halfbits
);
7340 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7341 tcg_gen_dupi_vec(vece
, t
, mask
);
7342 tcg_gen_and_vec(vece
, d
, n
, t
);
7343 tcg_temp_free_vec(t
);
7346 static bool trans_SHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7348 static const TCGOpcode vec_list
[] = { INDEX_op_shri_vec
, 0 };
7349 static const GVecGen2i ops
[3] = {
7350 { .fni8
= gen_shrnb16_i64
,
7351 .fniv
= gen_shrnb_vec
,
7352 .opt_opc
= vec_list
,
7353 .fno
= gen_helper_sve2_shrnb_h
,
7355 { .fni8
= gen_shrnb32_i64
,
7356 .fniv
= gen_shrnb_vec
,
7357 .opt_opc
= vec_list
,
7358 .fno
= gen_helper_sve2_shrnb_s
,
7360 { .fni8
= gen_shrnb64_i64
,
7361 .fniv
= gen_shrnb_vec
,
7362 .opt_opc
= vec_list
,
7363 .fno
= gen_helper_sve2_shrnb_d
,
7366 return do_sve2_shr_narrow(s
, a
, ops
);
7369 static void gen_shrnt_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int shr
)
7371 int halfbits
= 4 << vece
;
7372 uint64_t mask
= dup_const(vece
, MAKE_64BIT_MASK(0, halfbits
));
7374 tcg_gen_shli_i64(n
, n
, halfbits
- shr
);
7375 tcg_gen_andi_i64(n
, n
, ~mask
);
7376 tcg_gen_andi_i64(d
, d
, mask
);
7377 tcg_gen_or_i64(d
, d
, n
);
7380 static void gen_shrnt16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7382 gen_shrnt_i64(MO_16
, d
, n
, shr
);
7385 static void gen_shrnt32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7387 gen_shrnt_i64(MO_32
, d
, n
, shr
);
7390 static void gen_shrnt64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7392 tcg_gen_shri_i64(n
, n
, shr
);
7393 tcg_gen_deposit_i64(d
, d
, n
, 32, 32);
7396 static void gen_shrnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t shr
)
7398 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7399 int halfbits
= 4 << vece
;
7400 uint64_t mask
= MAKE_64BIT_MASK(0, halfbits
);
7402 tcg_gen_shli_vec(vece
, n
, n
, halfbits
- shr
);
7403 tcg_gen_dupi_vec(vece
, t
, mask
);
7404 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7405 tcg_temp_free_vec(t
);
7408 static bool trans_SHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7410 static const TCGOpcode vec_list
[] = { INDEX_op_shli_vec
, 0 };
7411 static const GVecGen2i ops
[3] = {
7412 { .fni8
= gen_shrnt16_i64
,
7413 .fniv
= gen_shrnt_vec
,
7414 .opt_opc
= vec_list
,
7416 .fno
= gen_helper_sve2_shrnt_h
,
7418 { .fni8
= gen_shrnt32_i64
,
7419 .fniv
= gen_shrnt_vec
,
7420 .opt_opc
= vec_list
,
7422 .fno
= gen_helper_sve2_shrnt_s
,
7424 { .fni8
= gen_shrnt64_i64
,
7425 .fniv
= gen_shrnt_vec
,
7426 .opt_opc
= vec_list
,
7428 .fno
= gen_helper_sve2_shrnt_d
,
7431 return do_sve2_shr_narrow(s
, a
, ops
);
7434 static bool trans_RSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7436 static const GVecGen2i ops
[3] = {
7437 { .fno
= gen_helper_sve2_rshrnb_h
},
7438 { .fno
= gen_helper_sve2_rshrnb_s
},
7439 { .fno
= gen_helper_sve2_rshrnb_d
},
7441 return do_sve2_shr_narrow(s
, a
, ops
);
7444 static bool trans_RSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7446 static const GVecGen2i ops
[3] = {
7447 { .fno
= gen_helper_sve2_rshrnt_h
},
7448 { .fno
= gen_helper_sve2_rshrnt_s
},
7449 { .fno
= gen_helper_sve2_rshrnt_d
},
7451 return do_sve2_shr_narrow(s
, a
, ops
);
7454 static void gen_sqshrunb_vec(unsigned vece
, TCGv_vec d
,
7455 TCGv_vec n
, int64_t shr
)
7457 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7458 int halfbits
= 4 << vece
;
7460 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7461 tcg_gen_dupi_vec(vece
, t
, 0);
7462 tcg_gen_smax_vec(vece
, n
, n
, t
);
7463 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7464 tcg_gen_umin_vec(vece
, d
, n
, t
);
7465 tcg_temp_free_vec(t
);
7468 static bool trans_SQSHRUNB(DisasContext
*s
, arg_rri_esz
*a
)
7470 static const TCGOpcode vec_list
[] = {
7471 INDEX_op_sari_vec
, INDEX_op_smax_vec
, INDEX_op_umin_vec
, 0
7473 static const GVecGen2i ops
[3] = {
7474 { .fniv
= gen_sqshrunb_vec
,
7475 .opt_opc
= vec_list
,
7476 .fno
= gen_helper_sve2_sqshrunb_h
,
7478 { .fniv
= gen_sqshrunb_vec
,
7479 .opt_opc
= vec_list
,
7480 .fno
= gen_helper_sve2_sqshrunb_s
,
7482 { .fniv
= gen_sqshrunb_vec
,
7483 .opt_opc
= vec_list
,
7484 .fno
= gen_helper_sve2_sqshrunb_d
,
7487 return do_sve2_shr_narrow(s
, a
, ops
);
7490 static void gen_sqshrunt_vec(unsigned vece
, TCGv_vec d
,
7491 TCGv_vec n
, int64_t shr
)
7493 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7494 int halfbits
= 4 << vece
;
7496 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7497 tcg_gen_dupi_vec(vece
, t
, 0);
7498 tcg_gen_smax_vec(vece
, n
, n
, t
);
7499 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7500 tcg_gen_umin_vec(vece
, n
, n
, t
);
7501 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7502 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7503 tcg_temp_free_vec(t
);
7506 static bool trans_SQSHRUNT(DisasContext
*s
, arg_rri_esz
*a
)
7508 static const TCGOpcode vec_list
[] = {
7509 INDEX_op_shli_vec
, INDEX_op_sari_vec
,
7510 INDEX_op_smax_vec
, INDEX_op_umin_vec
, 0
7512 static const GVecGen2i ops
[3] = {
7513 { .fniv
= gen_sqshrunt_vec
,
7514 .opt_opc
= vec_list
,
7516 .fno
= gen_helper_sve2_sqshrunt_h
,
7518 { .fniv
= gen_sqshrunt_vec
,
7519 .opt_opc
= vec_list
,
7521 .fno
= gen_helper_sve2_sqshrunt_s
,
7523 { .fniv
= gen_sqshrunt_vec
,
7524 .opt_opc
= vec_list
,
7526 .fno
= gen_helper_sve2_sqshrunt_d
,
7529 return do_sve2_shr_narrow(s
, a
, ops
);
7532 static bool trans_SQRSHRUNB(DisasContext
*s
, arg_rri_esz
*a
)
7534 static const GVecGen2i ops
[3] = {
7535 { .fno
= gen_helper_sve2_sqrshrunb_h
},
7536 { .fno
= gen_helper_sve2_sqrshrunb_s
},
7537 { .fno
= gen_helper_sve2_sqrshrunb_d
},
7539 return do_sve2_shr_narrow(s
, a
, ops
);
7542 static bool trans_SQRSHRUNT(DisasContext
*s
, arg_rri_esz
*a
)
7544 static const GVecGen2i ops
[3] = {
7545 { .fno
= gen_helper_sve2_sqrshrunt_h
},
7546 { .fno
= gen_helper_sve2_sqrshrunt_s
},
7547 { .fno
= gen_helper_sve2_sqrshrunt_d
},
7549 return do_sve2_shr_narrow(s
, a
, ops
);
7552 static void gen_sqshrnb_vec(unsigned vece
, TCGv_vec d
,
7553 TCGv_vec n
, int64_t shr
)
7555 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7556 int halfbits
= 4 << vece
;
7557 int64_t max
= MAKE_64BIT_MASK(0, halfbits
- 1);
7558 int64_t min
= -max
- 1;
7560 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7561 tcg_gen_dupi_vec(vece
, t
, min
);
7562 tcg_gen_smax_vec(vece
, n
, n
, t
);
7563 tcg_gen_dupi_vec(vece
, t
, max
);
7564 tcg_gen_smin_vec(vece
, n
, n
, t
);
7565 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7566 tcg_gen_and_vec(vece
, d
, n
, t
);
7567 tcg_temp_free_vec(t
);
7570 static bool trans_SQSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7572 static const TCGOpcode vec_list
[] = {
7573 INDEX_op_sari_vec
, INDEX_op_smax_vec
, INDEX_op_smin_vec
, 0
7575 static const GVecGen2i ops
[3] = {
7576 { .fniv
= gen_sqshrnb_vec
,
7577 .opt_opc
= vec_list
,
7578 .fno
= gen_helper_sve2_sqshrnb_h
,
7580 { .fniv
= gen_sqshrnb_vec
,
7581 .opt_opc
= vec_list
,
7582 .fno
= gen_helper_sve2_sqshrnb_s
,
7584 { .fniv
= gen_sqshrnb_vec
,
7585 .opt_opc
= vec_list
,
7586 .fno
= gen_helper_sve2_sqshrnb_d
,
7589 return do_sve2_shr_narrow(s
, a
, ops
);
7592 static void gen_sqshrnt_vec(unsigned vece
, TCGv_vec d
,
7593 TCGv_vec n
, int64_t shr
)
7595 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7596 int halfbits
= 4 << vece
;
7597 int64_t max
= MAKE_64BIT_MASK(0, halfbits
- 1);
7598 int64_t min
= -max
- 1;
7600 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7601 tcg_gen_dupi_vec(vece
, t
, min
);
7602 tcg_gen_smax_vec(vece
, n
, n
, t
);
7603 tcg_gen_dupi_vec(vece
, t
, max
);
7604 tcg_gen_smin_vec(vece
, n
, n
, t
);
7605 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7606 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7607 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7608 tcg_temp_free_vec(t
);
7611 static bool trans_SQSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7613 static const TCGOpcode vec_list
[] = {
7614 INDEX_op_shli_vec
, INDEX_op_sari_vec
,
7615 INDEX_op_smax_vec
, INDEX_op_smin_vec
, 0
7617 static const GVecGen2i ops
[3] = {
7618 { .fniv
= gen_sqshrnt_vec
,
7619 .opt_opc
= vec_list
,
7621 .fno
= gen_helper_sve2_sqshrnt_h
,
7623 { .fniv
= gen_sqshrnt_vec
,
7624 .opt_opc
= vec_list
,
7626 .fno
= gen_helper_sve2_sqshrnt_s
,
7628 { .fniv
= gen_sqshrnt_vec
,
7629 .opt_opc
= vec_list
,
7631 .fno
= gen_helper_sve2_sqshrnt_d
,
7634 return do_sve2_shr_narrow(s
, a
, ops
);
7637 static bool trans_SQRSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7639 static const GVecGen2i ops
[3] = {
7640 { .fno
= gen_helper_sve2_sqrshrnb_h
},
7641 { .fno
= gen_helper_sve2_sqrshrnb_s
},
7642 { .fno
= gen_helper_sve2_sqrshrnb_d
},
7644 return do_sve2_shr_narrow(s
, a
, ops
);
7647 static bool trans_SQRSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7649 static const GVecGen2i ops
[3] = {
7650 { .fno
= gen_helper_sve2_sqrshrnt_h
},
7651 { .fno
= gen_helper_sve2_sqrshrnt_s
},
7652 { .fno
= gen_helper_sve2_sqrshrnt_d
},
7654 return do_sve2_shr_narrow(s
, a
, ops
);
7657 static void gen_uqshrnb_vec(unsigned vece
, TCGv_vec d
,
7658 TCGv_vec n
, int64_t shr
)
7660 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7661 int halfbits
= 4 << vece
;
7663 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7664 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7665 tcg_gen_umin_vec(vece
, d
, n
, t
);
7666 tcg_temp_free_vec(t
);
7669 static bool trans_UQSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7671 static const TCGOpcode vec_list
[] = {
7672 INDEX_op_shri_vec
, INDEX_op_umin_vec
, 0
7674 static const GVecGen2i ops
[3] = {
7675 { .fniv
= gen_uqshrnb_vec
,
7676 .opt_opc
= vec_list
,
7677 .fno
= gen_helper_sve2_uqshrnb_h
,
7679 { .fniv
= gen_uqshrnb_vec
,
7680 .opt_opc
= vec_list
,
7681 .fno
= gen_helper_sve2_uqshrnb_s
,
7683 { .fniv
= gen_uqshrnb_vec
,
7684 .opt_opc
= vec_list
,
7685 .fno
= gen_helper_sve2_uqshrnb_d
,
7688 return do_sve2_shr_narrow(s
, a
, ops
);
7691 static void gen_uqshrnt_vec(unsigned vece
, TCGv_vec d
,
7692 TCGv_vec n
, int64_t shr
)
7694 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7695 int halfbits
= 4 << vece
;
7697 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7698 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7699 tcg_gen_umin_vec(vece
, n
, n
, t
);
7700 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7701 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7702 tcg_temp_free_vec(t
);
7705 static bool trans_UQSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7707 static const TCGOpcode vec_list
[] = {
7708 INDEX_op_shli_vec
, INDEX_op_shri_vec
, INDEX_op_umin_vec
, 0
7710 static const GVecGen2i ops
[3] = {
7711 { .fniv
= gen_uqshrnt_vec
,
7712 .opt_opc
= vec_list
,
7714 .fno
= gen_helper_sve2_uqshrnt_h
,
7716 { .fniv
= gen_uqshrnt_vec
,
7717 .opt_opc
= vec_list
,
7719 .fno
= gen_helper_sve2_uqshrnt_s
,
7721 { .fniv
= gen_uqshrnt_vec
,
7722 .opt_opc
= vec_list
,
7724 .fno
= gen_helper_sve2_uqshrnt_d
,
7727 return do_sve2_shr_narrow(s
, a
, ops
);
7730 static bool trans_UQRSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7732 static const GVecGen2i ops
[3] = {
7733 { .fno
= gen_helper_sve2_uqrshrnb_h
},
7734 { .fno
= gen_helper_sve2_uqrshrnb_s
},
7735 { .fno
= gen_helper_sve2_uqrshrnb_d
},
7737 return do_sve2_shr_narrow(s
, a
, ops
);
7740 static bool trans_UQRSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7742 static const GVecGen2i ops
[3] = {
7743 { .fno
= gen_helper_sve2_uqrshrnt_h
},
7744 { .fno
= gen_helper_sve2_uqrshrnt_s
},
7745 { .fno
= gen_helper_sve2_uqrshrnt_d
},
7747 return do_sve2_shr_narrow(s
, a
, ops
);
7750 #define DO_SVE2_ZZZ_NARROW(NAME, name) \
7751 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
7753 static gen_helper_gvec_3 * const fns[4] = { \
7754 NULL, gen_helper_sve2_##name##_h, \
7755 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
7757 return do_sve2_zzz_ool(s, a, fns[a->esz]); \
7760 DO_SVE2_ZZZ_NARROW(ADDHNB
, addhnb
)
7761 DO_SVE2_ZZZ_NARROW(ADDHNT
, addhnt
)
7762 DO_SVE2_ZZZ_NARROW(RADDHNB
, raddhnb
)
7763 DO_SVE2_ZZZ_NARROW(RADDHNT
, raddhnt
)
7765 DO_SVE2_ZZZ_NARROW(SUBHNB
, subhnb
)
7766 DO_SVE2_ZZZ_NARROW(SUBHNT
, subhnt
)
7767 DO_SVE2_ZZZ_NARROW(RSUBHNB
, rsubhnb
)
7768 DO_SVE2_ZZZ_NARROW(RSUBHNT
, rsubhnt
)
7770 static bool do_sve2_ppzz_flags(DisasContext
*s
, arg_rprr_esz
*a
,
7771 gen_helper_gvec_flags_4
*fn
)
7773 if (!dc_isar_feature(aa64_sve2
, s
)) {
7776 return do_ppzz_flags(s
, a
, fn
);
7779 #define DO_SVE2_PPZZ_MATCH(NAME, name) \
7780 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
7782 static gen_helper_gvec_flags_4 * const fns[4] = { \
7783 gen_helper_sve2_##name##_ppzz_b, gen_helper_sve2_##name##_ppzz_h, \
7786 return do_sve2_ppzz_flags(s, a, fns[a->esz]); \
7789 DO_SVE2_PPZZ_MATCH(MATCH
, match
)
7790 DO_SVE2_PPZZ_MATCH(NMATCH
, nmatch
)
7792 static bool trans_HISTCNT(DisasContext
*s
, arg_rprr_esz
*a
)
7794 static gen_helper_gvec_4
* const fns
[2] = {
7795 gen_helper_sve2_histcnt_s
, gen_helper_sve2_histcnt_d
7800 return do_sve2_zpzz_ool(s
, a
, fns
[a
->esz
- 2]);
7803 static bool trans_HISTSEG(DisasContext
*s
, arg_rrr_esz
*a
)
7808 return do_sve2_zzz_ool(s
, a
, gen_helper_sve2_histseg
);
7811 static bool do_sve2_zpzz_fp(DisasContext
*s
, arg_rprr_esz
*a
,
7812 gen_helper_gvec_4_ptr
*fn
)
7814 if (!dc_isar_feature(aa64_sve2
, s
)) {
7817 return do_zpzz_fp(s
, a
, fn
);
7820 #define DO_SVE2_ZPZZ_FP(NAME, name) \
7821 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
7823 static gen_helper_gvec_4_ptr * const fns[4] = { \
7824 NULL, gen_helper_sve2_##name##_zpzz_h, \
7825 gen_helper_sve2_##name##_zpzz_s, gen_helper_sve2_##name##_zpzz_d \
7827 return do_sve2_zpzz_fp(s, a, fns[a->esz]); \
7830 DO_SVE2_ZPZZ_FP(FADDP
, faddp
)
7831 DO_SVE2_ZPZZ_FP(FMAXNMP
, fmaxnmp
)
7832 DO_SVE2_ZPZZ_FP(FMINNMP
, fminnmp
)
7833 DO_SVE2_ZPZZ_FP(FMAXP
, fmaxp
)
7834 DO_SVE2_ZPZZ_FP(FMINP
, fminp
)
7837 * SVE Integer Multiply-Add (unpredicated)
7840 static bool trans_FMMLA(DisasContext
*s
, arg_rrrr_esz
*a
)
7842 gen_helper_gvec_4_ptr
*fn
;
7846 if (!dc_isar_feature(aa64_sve_f32mm
, s
)) {
7849 fn
= gen_helper_fmmla_s
;
7852 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
7855 fn
= gen_helper_fmmla_d
;
7861 if (sve_access_check(s
)) {
7862 unsigned vsz
= vec_full_reg_size(s
);
7863 TCGv_ptr status
= fpstatus_ptr(FPST_FPCR
);
7864 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
7865 vec_full_reg_offset(s
, a
->rn
),
7866 vec_full_reg_offset(s
, a
->rm
),
7867 vec_full_reg_offset(s
, a
->ra
),
7868 status
, vsz
, vsz
, 0, fn
);
7869 tcg_temp_free_ptr(status
);
7874 static bool do_sqdmlal_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
,
7875 bool sel1
, bool sel2
)
7877 static gen_helper_gvec_4
* const fns
[] = {
7878 NULL
, gen_helper_sve2_sqdmlal_zzzw_h
,
7879 gen_helper_sve2_sqdmlal_zzzw_s
, gen_helper_sve2_sqdmlal_zzzw_d
,
7881 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], (sel2
<< 1) | sel1
);
7884 static bool do_sqdmlsl_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
,
7885 bool sel1
, bool sel2
)
7887 static gen_helper_gvec_4
* const fns
[] = {
7888 NULL
, gen_helper_sve2_sqdmlsl_zzzw_h
,
7889 gen_helper_sve2_sqdmlsl_zzzw_s
, gen_helper_sve2_sqdmlsl_zzzw_d
,
7891 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], (sel2
<< 1) | sel1
);
7894 static bool trans_SQDMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7896 return do_sqdmlal_zzzw(s
, a
, false, false);
7899 static bool trans_SQDMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7901 return do_sqdmlal_zzzw(s
, a
, true, true);
7904 static bool trans_SQDMLALBT(DisasContext
*s
, arg_rrrr_esz
*a
)
7906 return do_sqdmlal_zzzw(s
, a
, false, true);
7909 static bool trans_SQDMLSLB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7911 return do_sqdmlsl_zzzw(s
, a
, false, false);
7914 static bool trans_SQDMLSLT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7916 return do_sqdmlsl_zzzw(s
, a
, true, true);
7919 static bool trans_SQDMLSLBT(DisasContext
*s
, arg_rrrr_esz
*a
)
7921 return do_sqdmlsl_zzzw(s
, a
, false, true);
7924 static bool trans_SQRDMLAH_zzzz(DisasContext
*s
, arg_rrrr_esz
*a
)
7926 static gen_helper_gvec_4
* const fns
[] = {
7927 gen_helper_sve2_sqrdmlah_b
, gen_helper_sve2_sqrdmlah_h
,
7928 gen_helper_sve2_sqrdmlah_s
, gen_helper_sve2_sqrdmlah_d
,
7930 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], 0);
7933 static bool trans_SQRDMLSH_zzzz(DisasContext
*s
, arg_rrrr_esz
*a
)
7935 static gen_helper_gvec_4
* const fns
[] = {
7936 gen_helper_sve2_sqrdmlsh_b
, gen_helper_sve2_sqrdmlsh_h
,
7937 gen_helper_sve2_sqrdmlsh_s
, gen_helper_sve2_sqrdmlsh_d
,
7939 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], 0);
7942 static bool do_smlal_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
7944 static gen_helper_gvec_4
* const fns
[] = {
7945 NULL
, gen_helper_sve2_smlal_zzzw_h
,
7946 gen_helper_sve2_smlal_zzzw_s
, gen_helper_sve2_smlal_zzzw_d
,
7948 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], sel
);
7951 static bool trans_SMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7953 return do_smlal_zzzw(s
, a
, false);
7956 static bool trans_SMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7958 return do_smlal_zzzw(s
, a
, true);
7961 static bool do_umlal_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
7963 static gen_helper_gvec_4
* const fns
[] = {
7964 NULL
, gen_helper_sve2_umlal_zzzw_h
,
7965 gen_helper_sve2_umlal_zzzw_s
, gen_helper_sve2_umlal_zzzw_d
,
7967 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], sel
);
7970 static bool trans_UMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7972 return do_umlal_zzzw(s
, a
, false);
7975 static bool trans_UMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7977 return do_umlal_zzzw(s
, a
, true);
7980 static bool do_smlsl_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
7982 static gen_helper_gvec_4
* const fns
[] = {
7983 NULL
, gen_helper_sve2_smlsl_zzzw_h
,
7984 gen_helper_sve2_smlsl_zzzw_s
, gen_helper_sve2_smlsl_zzzw_d
,
7986 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], sel
);
7989 static bool trans_SMLSLB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7991 return do_smlsl_zzzw(s
, a
, false);
7994 static bool trans_SMLSLT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7996 return do_smlsl_zzzw(s
, a
, true);
7999 static bool do_umlsl_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
8001 static gen_helper_gvec_4
* const fns
[] = {
8002 NULL
, gen_helper_sve2_umlsl_zzzw_h
,
8003 gen_helper_sve2_umlsl_zzzw_s
, gen_helper_sve2_umlsl_zzzw_d
,
8005 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], sel
);
8008 static bool trans_UMLSLB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8010 return do_umlsl_zzzw(s
, a
, false);
8013 static bool trans_UMLSLT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8015 return do_umlsl_zzzw(s
, a
, true);
8018 static bool trans_CMLA_zzzz(DisasContext
*s
, arg_CMLA_zzzz
*a
)
8020 static gen_helper_gvec_4
* const fns
[] = {
8021 gen_helper_sve2_cmla_zzzz_b
, gen_helper_sve2_cmla_zzzz_h
,
8022 gen_helper_sve2_cmla_zzzz_s
, gen_helper_sve2_cmla_zzzz_d
,
8025 if (!dc_isar_feature(aa64_sve2
, s
)) {
8028 if (sve_access_check(s
)) {
8029 gen_gvec_ool_zzzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
);
8034 static bool trans_SQRDCMLAH_zzzz(DisasContext
*s
, arg_SQRDCMLAH_zzzz
*a
)
8036 static gen_helper_gvec_4
* const fns
[] = {
8037 gen_helper_sve2_sqrdcmlah_zzzz_b
, gen_helper_sve2_sqrdcmlah_zzzz_h
,
8038 gen_helper_sve2_sqrdcmlah_zzzz_s
, gen_helper_sve2_sqrdcmlah_zzzz_d
,
8041 if (!dc_isar_feature(aa64_sve2
, s
)) {
8044 if (sve_access_check(s
)) {
8045 gen_gvec_ool_zzzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
);