2 * AArch64 SVE translation
4 * Copyright (c) 2018 Linaro, Ltd
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
22 #include "exec/exec-all.h"
23 #include "tcg/tcg-op.h"
24 #include "tcg/tcg-op-gvec.h"
25 #include "tcg/tcg-gvec-desc.h"
28 #include "translate.h"
29 #include "internals.h"
30 #include "exec/helper-proto.h"
31 #include "exec/helper-gen.h"
33 #include "trace-tcg.h"
34 #include "translate-a64.h"
35 #include "fpu/softfloat.h"
38 typedef void GVecGen2sFn(unsigned, uint32_t, uint32_t,
39 TCGv_i64
, uint32_t, uint32_t);
41 typedef void gen_helper_gvec_flags_3(TCGv_i32
, TCGv_ptr
, TCGv_ptr
,
43 typedef void gen_helper_gvec_flags_4(TCGv_i32
, TCGv_ptr
, TCGv_ptr
,
44 TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
46 typedef void gen_helper_gvec_mem(TCGv_env
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
47 typedef void gen_helper_gvec_mem_scatter(TCGv_env
, TCGv_ptr
, TCGv_ptr
,
48 TCGv_ptr
, TCGv_i64
, TCGv_i32
);
51 * Helpers for extracting complex instruction fields.
54 /* See e.g. ASR (immediate, predicated).
55 * Returns -1 for unallocated encoding; diagnose later.
57 static int tszimm_esz(DisasContext
*s
, int x
)
59 x
>>= 3; /* discard imm3 */
63 static int tszimm_shr(DisasContext
*s
, int x
)
65 return (16 << tszimm_esz(s
, x
)) - x
;
68 /* See e.g. LSL (immediate, predicated). */
69 static int tszimm_shl(DisasContext
*s
, int x
)
71 return x
- (8 << tszimm_esz(s
, x
));
74 static inline int plus1(DisasContext
*s
, int x
)
79 /* The SH bit is in bit 8. Extract the low 8 and shift. */
80 static inline int expand_imm_sh8s(DisasContext
*s
, int x
)
82 return (int8_t)x
<< (x
& 0x100 ? 8 : 0);
85 static inline int expand_imm_sh8u(DisasContext
*s
, int x
)
87 return (uint8_t)x
<< (x
& 0x100 ? 8 : 0);
90 /* Convert a 2-bit memory size (msz) to a 4-bit data type (dtype)
91 * with unsigned data. C.f. SVE Memory Contiguous Load Group.
93 static inline int msz_dtype(DisasContext
*s
, int msz
)
95 static const uint8_t dtype
[4] = { 0, 5, 10, 15 };
100 * Include the generated decoder.
103 #include "decode-sve.c.inc"
106 * Implement all of the translator functions referenced by the decoder.
109 /* Return the offset info CPUARMState of the predicate vector register Pn.
110 * Note for this purpose, FFR is P16.
112 static inline int pred_full_reg_offset(DisasContext
*s
, int regno
)
114 return offsetof(CPUARMState
, vfp
.pregs
[regno
]);
117 /* Return the byte size of the whole predicate register, VL / 64. */
118 static inline int pred_full_reg_size(DisasContext
*s
)
120 return s
->sve_len
>> 3;
123 /* Round up the size of a register to a size allowed by
124 * the tcg vector infrastructure. Any operation which uses this
125 * size may assume that the bits above pred_full_reg_size are zero,
126 * and must leave them the same way.
128 * Note that this is not needed for the vector registers as they
129 * are always properly sized for tcg vectors.
131 static int size_for_gvec(int size
)
136 return QEMU_ALIGN_UP(size
, 16);
140 static int pred_gvec_reg_size(DisasContext
*s
)
142 return size_for_gvec(pred_full_reg_size(s
));
145 /* Invoke an out-of-line helper on 2 Zregs. */
146 static void gen_gvec_ool_zz(DisasContext
*s
, gen_helper_gvec_2
*fn
,
147 int rd
, int rn
, int data
)
149 unsigned vsz
= vec_full_reg_size(s
);
150 tcg_gen_gvec_2_ool(vec_full_reg_offset(s
, rd
),
151 vec_full_reg_offset(s
, rn
),
155 /* Invoke an out-of-line helper on 3 Zregs. */
156 static void gen_gvec_ool_zzz(DisasContext
*s
, gen_helper_gvec_3
*fn
,
157 int rd
, int rn
, int rm
, int data
)
159 unsigned vsz
= vec_full_reg_size(s
);
160 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
161 vec_full_reg_offset(s
, rn
),
162 vec_full_reg_offset(s
, rm
),
166 /* Invoke an out-of-line helper on 4 Zregs. */
167 static void gen_gvec_ool_zzzz(DisasContext
*s
, gen_helper_gvec_4
*fn
,
168 int rd
, int rn
, int rm
, int ra
, int data
)
170 unsigned vsz
= vec_full_reg_size(s
);
171 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
172 vec_full_reg_offset(s
, rn
),
173 vec_full_reg_offset(s
, rm
),
174 vec_full_reg_offset(s
, ra
),
178 /* Invoke an out-of-line helper on 2 Zregs and a predicate. */
179 static void gen_gvec_ool_zzp(DisasContext
*s
, gen_helper_gvec_3
*fn
,
180 int rd
, int rn
, int pg
, int data
)
182 unsigned vsz
= vec_full_reg_size(s
);
183 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
184 vec_full_reg_offset(s
, rn
),
185 pred_full_reg_offset(s
, pg
),
189 /* Invoke an out-of-line helper on 3 Zregs and a predicate. */
190 static void gen_gvec_ool_zzzp(DisasContext
*s
, gen_helper_gvec_4
*fn
,
191 int rd
, int rn
, int rm
, int pg
, int data
)
193 unsigned vsz
= vec_full_reg_size(s
);
194 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
195 vec_full_reg_offset(s
, rn
),
196 vec_full_reg_offset(s
, rm
),
197 pred_full_reg_offset(s
, pg
),
201 /* Invoke a vector expander on two Zregs. */
202 static void gen_gvec_fn_zz(DisasContext
*s
, GVecGen2Fn
*gvec_fn
,
203 int esz
, int rd
, int rn
)
205 unsigned vsz
= vec_full_reg_size(s
);
206 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
207 vec_full_reg_offset(s
, rn
), vsz
, vsz
);
210 /* Invoke a vector expander on three Zregs. */
211 static void gen_gvec_fn_zzz(DisasContext
*s
, GVecGen3Fn
*gvec_fn
,
212 int esz
, int rd
, int rn
, int rm
)
214 unsigned vsz
= vec_full_reg_size(s
);
215 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
216 vec_full_reg_offset(s
, rn
),
217 vec_full_reg_offset(s
, rm
), vsz
, vsz
);
220 /* Invoke a vector expander on four Zregs. */
221 static void gen_gvec_fn_zzzz(DisasContext
*s
, GVecGen4Fn
*gvec_fn
,
222 int esz
, int rd
, int rn
, int rm
, int ra
)
224 unsigned vsz
= vec_full_reg_size(s
);
225 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
226 vec_full_reg_offset(s
, rn
),
227 vec_full_reg_offset(s
, rm
),
228 vec_full_reg_offset(s
, ra
), vsz
, vsz
);
231 /* Invoke a vector move on two Zregs. */
232 static bool do_mov_z(DisasContext
*s
, int rd
, int rn
)
234 if (sve_access_check(s
)) {
235 gen_gvec_fn_zz(s
, tcg_gen_gvec_mov
, MO_8
, rd
, rn
);
240 /* Initialize a Zreg with replications of a 64-bit immediate. */
241 static void do_dupi_z(DisasContext
*s
, int rd
, uint64_t word
)
243 unsigned vsz
= vec_full_reg_size(s
);
244 tcg_gen_gvec_dup_imm(MO_64
, vec_full_reg_offset(s
, rd
), vsz
, vsz
, word
);
247 /* Invoke a vector expander on three Pregs. */
248 static void gen_gvec_fn_ppp(DisasContext
*s
, GVecGen3Fn
*gvec_fn
,
249 int rd
, int rn
, int rm
)
251 unsigned psz
= pred_gvec_reg_size(s
);
252 gvec_fn(MO_64
, pred_full_reg_offset(s
, rd
),
253 pred_full_reg_offset(s
, rn
),
254 pred_full_reg_offset(s
, rm
), psz
, psz
);
257 /* Invoke a vector move on two Pregs. */
258 static bool do_mov_p(DisasContext
*s
, int rd
, int rn
)
260 if (sve_access_check(s
)) {
261 unsigned psz
= pred_gvec_reg_size(s
);
262 tcg_gen_gvec_mov(MO_8
, pred_full_reg_offset(s
, rd
),
263 pred_full_reg_offset(s
, rn
), psz
, psz
);
268 /* Set the cpu flags as per a return from an SVE helper. */
269 static void do_pred_flags(TCGv_i32 t
)
271 tcg_gen_mov_i32(cpu_NF
, t
);
272 tcg_gen_andi_i32(cpu_ZF
, t
, 2);
273 tcg_gen_andi_i32(cpu_CF
, t
, 1);
274 tcg_gen_movi_i32(cpu_VF
, 0);
277 /* Subroutines computing the ARM PredTest psuedofunction. */
278 static void do_predtest1(TCGv_i64 d
, TCGv_i64 g
)
280 TCGv_i32 t
= tcg_temp_new_i32();
282 gen_helper_sve_predtest1(t
, d
, g
);
284 tcg_temp_free_i32(t
);
287 static void do_predtest(DisasContext
*s
, int dofs
, int gofs
, int words
)
289 TCGv_ptr dptr
= tcg_temp_new_ptr();
290 TCGv_ptr gptr
= tcg_temp_new_ptr();
293 tcg_gen_addi_ptr(dptr
, cpu_env
, dofs
);
294 tcg_gen_addi_ptr(gptr
, cpu_env
, gofs
);
295 t
= tcg_const_i32(words
);
297 gen_helper_sve_predtest(t
, dptr
, gptr
, t
);
298 tcg_temp_free_ptr(dptr
);
299 tcg_temp_free_ptr(gptr
);
302 tcg_temp_free_i32(t
);
305 /* For each element size, the bits within a predicate word that are active. */
306 const uint64_t pred_esz_masks
[4] = {
307 0xffffffffffffffffull
, 0x5555555555555555ull
,
308 0x1111111111111111ull
, 0x0101010101010101ull
312 *** SVE Logical - Unpredicated Group
315 static bool do_zzz_fn(DisasContext
*s
, arg_rrr_esz
*a
, GVecGen3Fn
*gvec_fn
)
317 if (sve_access_check(s
)) {
318 gen_gvec_fn_zzz(s
, gvec_fn
, a
->esz
, a
->rd
, a
->rn
, a
->rm
);
323 static bool trans_AND_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
325 return do_zzz_fn(s
, a
, tcg_gen_gvec_and
);
328 static bool trans_ORR_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
330 return do_zzz_fn(s
, a
, tcg_gen_gvec_or
);
333 static bool trans_EOR_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
335 return do_zzz_fn(s
, a
, tcg_gen_gvec_xor
);
338 static bool trans_BIC_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
340 return do_zzz_fn(s
, a
, tcg_gen_gvec_andc
);
343 static void gen_xar8_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
345 TCGv_i64 t
= tcg_temp_new_i64();
346 uint64_t mask
= dup_const(MO_8
, 0xff >> sh
);
348 tcg_gen_xor_i64(t
, n
, m
);
349 tcg_gen_shri_i64(d
, t
, sh
);
350 tcg_gen_shli_i64(t
, t
, 8 - sh
);
351 tcg_gen_andi_i64(d
, d
, mask
);
352 tcg_gen_andi_i64(t
, t
, ~mask
);
353 tcg_gen_or_i64(d
, d
, t
);
354 tcg_temp_free_i64(t
);
357 static void gen_xar16_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
359 TCGv_i64 t
= tcg_temp_new_i64();
360 uint64_t mask
= dup_const(MO_16
, 0xffff >> sh
);
362 tcg_gen_xor_i64(t
, n
, m
);
363 tcg_gen_shri_i64(d
, t
, sh
);
364 tcg_gen_shli_i64(t
, t
, 16 - sh
);
365 tcg_gen_andi_i64(d
, d
, mask
);
366 tcg_gen_andi_i64(t
, t
, ~mask
);
367 tcg_gen_or_i64(d
, d
, t
);
368 tcg_temp_free_i64(t
);
371 static void gen_xar_i32(TCGv_i32 d
, TCGv_i32 n
, TCGv_i32 m
, int32_t sh
)
373 tcg_gen_xor_i32(d
, n
, m
);
374 tcg_gen_rotri_i32(d
, d
, sh
);
377 static void gen_xar_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
379 tcg_gen_xor_i64(d
, n
, m
);
380 tcg_gen_rotri_i64(d
, d
, sh
);
383 static void gen_xar_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
384 TCGv_vec m
, int64_t sh
)
386 tcg_gen_xor_vec(vece
, d
, n
, m
);
387 tcg_gen_rotri_vec(vece
, d
, d
, sh
);
390 void gen_gvec_xar(unsigned vece
, uint32_t rd_ofs
, uint32_t rn_ofs
,
391 uint32_t rm_ofs
, int64_t shift
,
392 uint32_t opr_sz
, uint32_t max_sz
)
394 static const TCGOpcode vecop
[] = { INDEX_op_rotli_vec
, 0 };
395 static const GVecGen3i ops
[4] = {
396 { .fni8
= gen_xar8_i64
,
398 .fno
= gen_helper_sve2_xar_b
,
401 { .fni8
= gen_xar16_i64
,
403 .fno
= gen_helper_sve2_xar_h
,
406 { .fni4
= gen_xar_i32
,
408 .fno
= gen_helper_sve2_xar_s
,
411 { .fni8
= gen_xar_i64
,
413 .fno
= gen_helper_gvec_xar_d
,
417 int esize
= 8 << vece
;
419 /* The SVE2 range is 1 .. esize; the AdvSIMD range is 0 .. esize-1. */
420 tcg_debug_assert(shift
>= 0);
421 tcg_debug_assert(shift
<= esize
);
425 /* xar with no rotate devolves to xor. */
426 tcg_gen_gvec_xor(vece
, rd_ofs
, rn_ofs
, rm_ofs
, opr_sz
, max_sz
);
428 tcg_gen_gvec_3i(rd_ofs
, rn_ofs
, rm_ofs
, opr_sz
, max_sz
,
433 static bool trans_XAR(DisasContext
*s
, arg_rrri_esz
*a
)
435 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
438 if (sve_access_check(s
)) {
439 unsigned vsz
= vec_full_reg_size(s
);
440 gen_gvec_xar(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
441 vec_full_reg_offset(s
, a
->rn
),
442 vec_full_reg_offset(s
, a
->rm
), a
->imm
, vsz
, vsz
);
447 static bool do_sve2_zzzz_fn(DisasContext
*s
, arg_rrrr_esz
*a
, GVecGen4Fn
*fn
)
449 if (!dc_isar_feature(aa64_sve2
, s
)) {
452 if (sve_access_check(s
)) {
453 gen_gvec_fn_zzzz(s
, fn
, a
->esz
, a
->rd
, a
->rn
, a
->rm
, a
->ra
);
458 static void gen_eor3_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
460 tcg_gen_xor_i64(d
, n
, m
);
461 tcg_gen_xor_i64(d
, d
, k
);
464 static void gen_eor3_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
465 TCGv_vec m
, TCGv_vec k
)
467 tcg_gen_xor_vec(vece
, d
, n
, m
);
468 tcg_gen_xor_vec(vece
, d
, d
, k
);
471 static void gen_eor3(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
472 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
474 static const GVecGen4 op
= {
475 .fni8
= gen_eor3_i64
,
476 .fniv
= gen_eor3_vec
,
477 .fno
= gen_helper_sve2_eor3
,
479 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
481 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
484 static bool trans_EOR3(DisasContext
*s
, arg_rrrr_esz
*a
)
486 return do_sve2_zzzz_fn(s
, a
, gen_eor3
);
489 static void gen_bcax_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
491 tcg_gen_andc_i64(d
, m
, k
);
492 tcg_gen_xor_i64(d
, d
, n
);
495 static void gen_bcax_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
496 TCGv_vec m
, TCGv_vec k
)
498 tcg_gen_andc_vec(vece
, d
, m
, k
);
499 tcg_gen_xor_vec(vece
, d
, d
, n
);
502 static void gen_bcax(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
503 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
505 static const GVecGen4 op
= {
506 .fni8
= gen_bcax_i64
,
507 .fniv
= gen_bcax_vec
,
508 .fno
= gen_helper_sve2_bcax
,
510 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
512 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
515 static bool trans_BCAX(DisasContext
*s
, arg_rrrr_esz
*a
)
517 return do_sve2_zzzz_fn(s
, a
, gen_bcax
);
520 static void gen_bsl(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
521 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
523 /* BSL differs from the generic bitsel in argument ordering. */
524 tcg_gen_gvec_bitsel(vece
, d
, a
, n
, m
, oprsz
, maxsz
);
527 static bool trans_BSL(DisasContext
*s
, arg_rrrr_esz
*a
)
529 return do_sve2_zzzz_fn(s
, a
, gen_bsl
);
532 static void gen_bsl1n_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
534 tcg_gen_andc_i64(n
, k
, n
);
535 tcg_gen_andc_i64(m
, m
, k
);
536 tcg_gen_or_i64(d
, n
, m
);
539 static void gen_bsl1n_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
540 TCGv_vec m
, TCGv_vec k
)
542 if (TCG_TARGET_HAS_bitsel_vec
) {
543 tcg_gen_not_vec(vece
, n
, n
);
544 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
546 tcg_gen_andc_vec(vece
, n
, k
, n
);
547 tcg_gen_andc_vec(vece
, m
, m
, k
);
548 tcg_gen_or_vec(vece
, d
, n
, m
);
552 static void gen_bsl1n(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
553 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
555 static const GVecGen4 op
= {
556 .fni8
= gen_bsl1n_i64
,
557 .fniv
= gen_bsl1n_vec
,
558 .fno
= gen_helper_sve2_bsl1n
,
560 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
562 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
565 static bool trans_BSL1N(DisasContext
*s
, arg_rrrr_esz
*a
)
567 return do_sve2_zzzz_fn(s
, a
, gen_bsl1n
);
570 static void gen_bsl2n_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
573 * Z[dn] = (n & k) | (~m & ~k)
576 tcg_gen_and_i64(n
, n
, k
);
577 if (TCG_TARGET_HAS_orc_i64
) {
578 tcg_gen_or_i64(m
, m
, k
);
579 tcg_gen_orc_i64(d
, n
, m
);
581 tcg_gen_nor_i64(m
, m
, k
);
582 tcg_gen_or_i64(d
, n
, m
);
586 static void gen_bsl2n_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
587 TCGv_vec m
, TCGv_vec k
)
589 if (TCG_TARGET_HAS_bitsel_vec
) {
590 tcg_gen_not_vec(vece
, m
, m
);
591 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
593 tcg_gen_and_vec(vece
, n
, n
, k
);
594 tcg_gen_or_vec(vece
, m
, m
, k
);
595 tcg_gen_orc_vec(vece
, d
, n
, m
);
599 static void gen_bsl2n(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
600 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
602 static const GVecGen4 op
= {
603 .fni8
= gen_bsl2n_i64
,
604 .fniv
= gen_bsl2n_vec
,
605 .fno
= gen_helper_sve2_bsl2n
,
607 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
609 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
612 static bool trans_BSL2N(DisasContext
*s
, arg_rrrr_esz
*a
)
614 return do_sve2_zzzz_fn(s
, a
, gen_bsl2n
);
617 static void gen_nbsl_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
619 tcg_gen_and_i64(n
, n
, k
);
620 tcg_gen_andc_i64(m
, m
, k
);
621 tcg_gen_nor_i64(d
, n
, m
);
624 static void gen_nbsl_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
625 TCGv_vec m
, TCGv_vec k
)
627 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
628 tcg_gen_not_vec(vece
, d
, d
);
631 static void gen_nbsl(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
632 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
634 static const GVecGen4 op
= {
635 .fni8
= gen_nbsl_i64
,
636 .fniv
= gen_nbsl_vec
,
637 .fno
= gen_helper_sve2_nbsl
,
639 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
641 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
644 static bool trans_NBSL(DisasContext
*s
, arg_rrrr_esz
*a
)
646 return do_sve2_zzzz_fn(s
, a
, gen_nbsl
);
650 *** SVE Integer Arithmetic - Unpredicated Group
653 static bool trans_ADD_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
655 return do_zzz_fn(s
, a
, tcg_gen_gvec_add
);
658 static bool trans_SUB_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
660 return do_zzz_fn(s
, a
, tcg_gen_gvec_sub
);
663 static bool trans_SQADD_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
665 return do_zzz_fn(s
, a
, tcg_gen_gvec_ssadd
);
668 static bool trans_SQSUB_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
670 return do_zzz_fn(s
, a
, tcg_gen_gvec_sssub
);
673 static bool trans_UQADD_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
675 return do_zzz_fn(s
, a
, tcg_gen_gvec_usadd
);
678 static bool trans_UQSUB_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
680 return do_zzz_fn(s
, a
, tcg_gen_gvec_ussub
);
684 *** SVE Integer Arithmetic - Binary Predicated Group
687 static bool do_zpzz_ool(DisasContext
*s
, arg_rprr_esz
*a
, gen_helper_gvec_4
*fn
)
692 if (sve_access_check(s
)) {
693 gen_gvec_ool_zzzp(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->pg
, 0);
698 /* Select active elememnts from Zn and inactive elements from Zm,
699 * storing the result in Zd.
701 static void do_sel_z(DisasContext
*s
, int rd
, int rn
, int rm
, int pg
, int esz
)
703 static gen_helper_gvec_4
* const fns
[4] = {
704 gen_helper_sve_sel_zpzz_b
, gen_helper_sve_sel_zpzz_h
,
705 gen_helper_sve_sel_zpzz_s
, gen_helper_sve_sel_zpzz_d
707 gen_gvec_ool_zzzp(s
, fns
[esz
], rd
, rn
, rm
, pg
, 0);
710 #define DO_ZPZZ(NAME, name) \
711 static bool trans_##NAME##_zpzz(DisasContext *s, arg_rprr_esz *a) \
713 static gen_helper_gvec_4 * const fns[4] = { \
714 gen_helper_sve_##name##_zpzz_b, gen_helper_sve_##name##_zpzz_h, \
715 gen_helper_sve_##name##_zpzz_s, gen_helper_sve_##name##_zpzz_d, \
717 return do_zpzz_ool(s, a, fns[a->esz]); \
736 DO_ZPZZ(SMULH
, smulh
)
737 DO_ZPZZ(UMULH
, umulh
)
743 static bool trans_SDIV_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
745 static gen_helper_gvec_4
* const fns
[4] = {
746 NULL
, NULL
, gen_helper_sve_sdiv_zpzz_s
, gen_helper_sve_sdiv_zpzz_d
748 return do_zpzz_ool(s
, a
, fns
[a
->esz
]);
751 static bool trans_UDIV_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
753 static gen_helper_gvec_4
* const fns
[4] = {
754 NULL
, NULL
, gen_helper_sve_udiv_zpzz_s
, gen_helper_sve_udiv_zpzz_d
756 return do_zpzz_ool(s
, a
, fns
[a
->esz
]);
759 static bool trans_SEL_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
761 if (sve_access_check(s
)) {
762 do_sel_z(s
, a
->rd
, a
->rn
, a
->rm
, a
->pg
, a
->esz
);
770 *** SVE Integer Arithmetic - Unary Predicated Group
773 static bool do_zpz_ool(DisasContext
*s
, arg_rpr_esz
*a
, gen_helper_gvec_3
*fn
)
778 if (sve_access_check(s
)) {
779 gen_gvec_ool_zzp(s
, fn
, a
->rd
, a
->rn
, a
->pg
, 0);
784 #define DO_ZPZ(NAME, name) \
785 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
787 static gen_helper_gvec_3 * const fns[4] = { \
788 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
789 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
791 return do_zpz_ool(s, a, fns[a->esz]); \
796 DO_ZPZ(CNT_zpz
, cnt_zpz
)
798 DO_ZPZ(NOT_zpz
, not_zpz
)
802 static bool trans_FABS(DisasContext
*s
, arg_rpr_esz
*a
)
804 static gen_helper_gvec_3
* const fns
[4] = {
806 gen_helper_sve_fabs_h
,
807 gen_helper_sve_fabs_s
,
808 gen_helper_sve_fabs_d
810 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
813 static bool trans_FNEG(DisasContext
*s
, arg_rpr_esz
*a
)
815 static gen_helper_gvec_3
* const fns
[4] = {
817 gen_helper_sve_fneg_h
,
818 gen_helper_sve_fneg_s
,
819 gen_helper_sve_fneg_d
821 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
824 static bool trans_SXTB(DisasContext
*s
, arg_rpr_esz
*a
)
826 static gen_helper_gvec_3
* const fns
[4] = {
828 gen_helper_sve_sxtb_h
,
829 gen_helper_sve_sxtb_s
,
830 gen_helper_sve_sxtb_d
832 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
835 static bool trans_UXTB(DisasContext
*s
, arg_rpr_esz
*a
)
837 static gen_helper_gvec_3
* const fns
[4] = {
839 gen_helper_sve_uxtb_h
,
840 gen_helper_sve_uxtb_s
,
841 gen_helper_sve_uxtb_d
843 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
846 static bool trans_SXTH(DisasContext
*s
, arg_rpr_esz
*a
)
848 static gen_helper_gvec_3
* const fns
[4] = {
850 gen_helper_sve_sxth_s
,
851 gen_helper_sve_sxth_d
853 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
856 static bool trans_UXTH(DisasContext
*s
, arg_rpr_esz
*a
)
858 static gen_helper_gvec_3
* const fns
[4] = {
860 gen_helper_sve_uxth_s
,
861 gen_helper_sve_uxth_d
863 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
866 static bool trans_SXTW(DisasContext
*s
, arg_rpr_esz
*a
)
868 return do_zpz_ool(s
, a
, a
->esz
== 3 ? gen_helper_sve_sxtw_d
: NULL
);
871 static bool trans_UXTW(DisasContext
*s
, arg_rpr_esz
*a
)
873 return do_zpz_ool(s
, a
, a
->esz
== 3 ? gen_helper_sve_uxtw_d
: NULL
);
879 *** SVE Integer Reduction Group
882 typedef void gen_helper_gvec_reduc(TCGv_i64
, TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
883 static bool do_vpz_ool(DisasContext
*s
, arg_rpr_esz
*a
,
884 gen_helper_gvec_reduc
*fn
)
886 unsigned vsz
= vec_full_reg_size(s
);
894 if (!sve_access_check(s
)) {
898 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
899 temp
= tcg_temp_new_i64();
900 t_zn
= tcg_temp_new_ptr();
901 t_pg
= tcg_temp_new_ptr();
903 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
904 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
905 fn(temp
, t_zn
, t_pg
, desc
);
906 tcg_temp_free_ptr(t_zn
);
907 tcg_temp_free_ptr(t_pg
);
908 tcg_temp_free_i32(desc
);
910 write_fp_dreg(s
, a
->rd
, temp
);
911 tcg_temp_free_i64(temp
);
915 #define DO_VPZ(NAME, name) \
916 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
918 static gen_helper_gvec_reduc * const fns[4] = { \
919 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
920 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
922 return do_vpz_ool(s, a, fns[a->esz]); \
935 static bool trans_SADDV(DisasContext
*s
, arg_rpr_esz
*a
)
937 static gen_helper_gvec_reduc
* const fns
[4] = {
938 gen_helper_sve_saddv_b
, gen_helper_sve_saddv_h
,
939 gen_helper_sve_saddv_s
, NULL
941 return do_vpz_ool(s
, a
, fns
[a
->esz
]);
947 *** SVE Shift by Immediate - Predicated Group
951 * Copy Zn into Zd, storing zeros into inactive elements.
952 * If invert, store zeros into the active elements.
954 static bool do_movz_zpz(DisasContext
*s
, int rd
, int rn
, int pg
,
955 int esz
, bool invert
)
957 static gen_helper_gvec_3
* const fns
[4] = {
958 gen_helper_sve_movz_b
, gen_helper_sve_movz_h
,
959 gen_helper_sve_movz_s
, gen_helper_sve_movz_d
,
962 if (sve_access_check(s
)) {
963 gen_gvec_ool_zzp(s
, fns
[esz
], rd
, rn
, pg
, invert
);
968 static bool do_zpzi_ool(DisasContext
*s
, arg_rpri_esz
*a
,
969 gen_helper_gvec_3
*fn
)
971 if (sve_access_check(s
)) {
972 gen_gvec_ool_zzp(s
, fn
, a
->rd
, a
->rn
, a
->pg
, a
->imm
);
977 static bool trans_ASR_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
979 static gen_helper_gvec_3
* const fns
[4] = {
980 gen_helper_sve_asr_zpzi_b
, gen_helper_sve_asr_zpzi_h
,
981 gen_helper_sve_asr_zpzi_s
, gen_helper_sve_asr_zpzi_d
,
984 /* Invalid tsz encoding -- see tszimm_esz. */
987 /* Shift by element size is architecturally valid. For
988 arithmetic right-shift, it's the same as by one less. */
989 a
->imm
= MIN(a
->imm
, (8 << a
->esz
) - 1);
990 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
993 static bool trans_LSR_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
995 static gen_helper_gvec_3
* const fns
[4] = {
996 gen_helper_sve_lsr_zpzi_b
, gen_helper_sve_lsr_zpzi_h
,
997 gen_helper_sve_lsr_zpzi_s
, gen_helper_sve_lsr_zpzi_d
,
1002 /* Shift by element size is architecturally valid.
1003 For logical shifts, it is a zeroing operation. */
1004 if (a
->imm
>= (8 << a
->esz
)) {
1005 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
1007 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1011 static bool trans_LSL_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
1013 static gen_helper_gvec_3
* const fns
[4] = {
1014 gen_helper_sve_lsl_zpzi_b
, gen_helper_sve_lsl_zpzi_h
,
1015 gen_helper_sve_lsl_zpzi_s
, gen_helper_sve_lsl_zpzi_d
,
1020 /* Shift by element size is architecturally valid.
1021 For logical shifts, it is a zeroing operation. */
1022 if (a
->imm
>= (8 << a
->esz
)) {
1023 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
1025 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1029 static bool trans_ASRD(DisasContext
*s
, arg_rpri_esz
*a
)
1031 static gen_helper_gvec_3
* const fns
[4] = {
1032 gen_helper_sve_asrd_b
, gen_helper_sve_asrd_h
,
1033 gen_helper_sve_asrd_s
, gen_helper_sve_asrd_d
,
1038 /* Shift by element size is architecturally valid. For arithmetic
1039 right shift for division, it is a zeroing operation. */
1040 if (a
->imm
>= (8 << a
->esz
)) {
1041 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
1043 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1048 *** SVE Bitwise Shift - Predicated Group
1051 #define DO_ZPZW(NAME, name) \
1052 static bool trans_##NAME##_zpzw(DisasContext *s, arg_rprr_esz *a) \
1054 static gen_helper_gvec_4 * const fns[3] = { \
1055 gen_helper_sve_##name##_zpzw_b, gen_helper_sve_##name##_zpzw_h, \
1056 gen_helper_sve_##name##_zpzw_s, \
1058 if (a->esz < 0 || a->esz >= 3) { \
1061 return do_zpzz_ool(s, a, fns[a->esz]); \
1071 *** SVE Bitwise Shift - Unpredicated Group
1074 static bool do_shift_imm(DisasContext
*s
, arg_rri_esz
*a
, bool asr
,
1075 void (*gvec_fn
)(unsigned, uint32_t, uint32_t,
1076 int64_t, uint32_t, uint32_t))
1079 /* Invalid tsz encoding -- see tszimm_esz. */
1082 if (sve_access_check(s
)) {
1083 unsigned vsz
= vec_full_reg_size(s
);
1084 /* Shift by element size is architecturally valid. For
1085 arithmetic right-shift, it's the same as by one less.
1086 Otherwise it is a zeroing operation. */
1087 if (a
->imm
>= 8 << a
->esz
) {
1089 a
->imm
= (8 << a
->esz
) - 1;
1091 do_dupi_z(s
, a
->rd
, 0);
1095 gvec_fn(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
1096 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
1101 static bool trans_ASR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
1103 return do_shift_imm(s
, a
, true, tcg_gen_gvec_sari
);
1106 static bool trans_LSR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
1108 return do_shift_imm(s
, a
, false, tcg_gen_gvec_shri
);
1111 static bool trans_LSL_zzi(DisasContext
*s
, arg_rri_esz
*a
)
1113 return do_shift_imm(s
, a
, false, tcg_gen_gvec_shli
);
1116 static bool do_zzw_ool(DisasContext
*s
, arg_rrr_esz
*a
, gen_helper_gvec_3
*fn
)
1121 if (sve_access_check(s
)) {
1122 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, 0);
1127 #define DO_ZZW(NAME, name) \
1128 static bool trans_##NAME##_zzw(DisasContext *s, arg_rrr_esz *a) \
1130 static gen_helper_gvec_3 * const fns[4] = { \
1131 gen_helper_sve_##name##_zzw_b, gen_helper_sve_##name##_zzw_h, \
1132 gen_helper_sve_##name##_zzw_s, NULL \
1134 return do_zzw_ool(s, a, fns[a->esz]); \
1144 *** SVE Integer Multiply-Add Group
1147 static bool do_zpzzz_ool(DisasContext
*s
, arg_rprrr_esz
*a
,
1148 gen_helper_gvec_5
*fn
)
1150 if (sve_access_check(s
)) {
1151 unsigned vsz
= vec_full_reg_size(s
);
1152 tcg_gen_gvec_5_ool(vec_full_reg_offset(s
, a
->rd
),
1153 vec_full_reg_offset(s
, a
->ra
),
1154 vec_full_reg_offset(s
, a
->rn
),
1155 vec_full_reg_offset(s
, a
->rm
),
1156 pred_full_reg_offset(s
, a
->pg
),
1162 #define DO_ZPZZZ(NAME, name) \
1163 static bool trans_##NAME(DisasContext *s, arg_rprrr_esz *a) \
1165 static gen_helper_gvec_5 * const fns[4] = { \
1166 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
1167 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
1169 return do_zpzzz_ool(s, a, fns[a->esz]); \
1178 *** SVE Index Generation Group
1181 static void do_index(DisasContext
*s
, int esz
, int rd
,
1182 TCGv_i64 start
, TCGv_i64 incr
)
1184 unsigned vsz
= vec_full_reg_size(s
);
1185 TCGv_i32 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
1186 TCGv_ptr t_zd
= tcg_temp_new_ptr();
1188 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, rd
));
1190 gen_helper_sve_index_d(t_zd
, start
, incr
, desc
);
1192 typedef void index_fn(TCGv_ptr
, TCGv_i32
, TCGv_i32
, TCGv_i32
);
1193 static index_fn
* const fns
[3] = {
1194 gen_helper_sve_index_b
,
1195 gen_helper_sve_index_h
,
1196 gen_helper_sve_index_s
,
1198 TCGv_i32 s32
= tcg_temp_new_i32();
1199 TCGv_i32 i32
= tcg_temp_new_i32();
1201 tcg_gen_extrl_i64_i32(s32
, start
);
1202 tcg_gen_extrl_i64_i32(i32
, incr
);
1203 fns
[esz
](t_zd
, s32
, i32
, desc
);
1205 tcg_temp_free_i32(s32
);
1206 tcg_temp_free_i32(i32
);
1208 tcg_temp_free_ptr(t_zd
);
1209 tcg_temp_free_i32(desc
);
1212 static bool trans_INDEX_ii(DisasContext
*s
, arg_INDEX_ii
*a
)
1214 if (sve_access_check(s
)) {
1215 TCGv_i64 start
= tcg_const_i64(a
->imm1
);
1216 TCGv_i64 incr
= tcg_const_i64(a
->imm2
);
1217 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1218 tcg_temp_free_i64(start
);
1219 tcg_temp_free_i64(incr
);
1224 static bool trans_INDEX_ir(DisasContext
*s
, arg_INDEX_ir
*a
)
1226 if (sve_access_check(s
)) {
1227 TCGv_i64 start
= tcg_const_i64(a
->imm
);
1228 TCGv_i64 incr
= cpu_reg(s
, a
->rm
);
1229 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1230 tcg_temp_free_i64(start
);
1235 static bool trans_INDEX_ri(DisasContext
*s
, arg_INDEX_ri
*a
)
1237 if (sve_access_check(s
)) {
1238 TCGv_i64 start
= cpu_reg(s
, a
->rn
);
1239 TCGv_i64 incr
= tcg_const_i64(a
->imm
);
1240 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1241 tcg_temp_free_i64(incr
);
1246 static bool trans_INDEX_rr(DisasContext
*s
, arg_INDEX_rr
*a
)
1248 if (sve_access_check(s
)) {
1249 TCGv_i64 start
= cpu_reg(s
, a
->rn
);
1250 TCGv_i64 incr
= cpu_reg(s
, a
->rm
);
1251 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1257 *** SVE Stack Allocation Group
1260 static bool trans_ADDVL(DisasContext
*s
, arg_ADDVL
*a
)
1262 if (sve_access_check(s
)) {
1263 TCGv_i64 rd
= cpu_reg_sp(s
, a
->rd
);
1264 TCGv_i64 rn
= cpu_reg_sp(s
, a
->rn
);
1265 tcg_gen_addi_i64(rd
, rn
, a
->imm
* vec_full_reg_size(s
));
1270 static bool trans_ADDPL(DisasContext
*s
, arg_ADDPL
*a
)
1272 if (sve_access_check(s
)) {
1273 TCGv_i64 rd
= cpu_reg_sp(s
, a
->rd
);
1274 TCGv_i64 rn
= cpu_reg_sp(s
, a
->rn
);
1275 tcg_gen_addi_i64(rd
, rn
, a
->imm
* pred_full_reg_size(s
));
1280 static bool trans_RDVL(DisasContext
*s
, arg_RDVL
*a
)
1282 if (sve_access_check(s
)) {
1283 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
1284 tcg_gen_movi_i64(reg
, a
->imm
* vec_full_reg_size(s
));
1290 *** SVE Compute Vector Address Group
1293 static bool do_adr(DisasContext
*s
, arg_rrri
*a
, gen_helper_gvec_3
*fn
)
1295 if (sve_access_check(s
)) {
1296 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->imm
);
1301 static bool trans_ADR_p32(DisasContext
*s
, arg_rrri
*a
)
1303 return do_adr(s
, a
, gen_helper_sve_adr_p32
);
1306 static bool trans_ADR_p64(DisasContext
*s
, arg_rrri
*a
)
1308 return do_adr(s
, a
, gen_helper_sve_adr_p64
);
1311 static bool trans_ADR_s32(DisasContext
*s
, arg_rrri
*a
)
1313 return do_adr(s
, a
, gen_helper_sve_adr_s32
);
1316 static bool trans_ADR_u32(DisasContext
*s
, arg_rrri
*a
)
1318 return do_adr(s
, a
, gen_helper_sve_adr_u32
);
1322 *** SVE Integer Misc - Unpredicated Group
1325 static bool trans_FEXPA(DisasContext
*s
, arg_rr_esz
*a
)
1327 static gen_helper_gvec_2
* const fns
[4] = {
1329 gen_helper_sve_fexpa_h
,
1330 gen_helper_sve_fexpa_s
,
1331 gen_helper_sve_fexpa_d
,
1336 if (sve_access_check(s
)) {
1337 gen_gvec_ool_zz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, 0);
1342 static bool trans_FTSSEL(DisasContext
*s
, arg_rrr_esz
*a
)
1344 static gen_helper_gvec_3
* const fns
[4] = {
1346 gen_helper_sve_ftssel_h
,
1347 gen_helper_sve_ftssel_s
,
1348 gen_helper_sve_ftssel_d
,
1353 if (sve_access_check(s
)) {
1354 gen_gvec_ool_zzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, 0);
1360 *** SVE Predicate Logical Operations Group
1363 static bool do_pppp_flags(DisasContext
*s
, arg_rprr_s
*a
,
1364 const GVecGen4
*gvec_op
)
1366 if (!sve_access_check(s
)) {
1370 unsigned psz
= pred_gvec_reg_size(s
);
1371 int dofs
= pred_full_reg_offset(s
, a
->rd
);
1372 int nofs
= pred_full_reg_offset(s
, a
->rn
);
1373 int mofs
= pred_full_reg_offset(s
, a
->rm
);
1374 int gofs
= pred_full_reg_offset(s
, a
->pg
);
1377 tcg_gen_gvec_4(dofs
, nofs
, mofs
, gofs
, psz
, psz
, gvec_op
);
1382 /* Do the operation and the flags generation in temps. */
1383 TCGv_i64 pd
= tcg_temp_new_i64();
1384 TCGv_i64 pn
= tcg_temp_new_i64();
1385 TCGv_i64 pm
= tcg_temp_new_i64();
1386 TCGv_i64 pg
= tcg_temp_new_i64();
1388 tcg_gen_ld_i64(pn
, cpu_env
, nofs
);
1389 tcg_gen_ld_i64(pm
, cpu_env
, mofs
);
1390 tcg_gen_ld_i64(pg
, cpu_env
, gofs
);
1392 gvec_op
->fni8(pd
, pn
, pm
, pg
);
1393 tcg_gen_st_i64(pd
, cpu_env
, dofs
);
1395 do_predtest1(pd
, pg
);
1397 tcg_temp_free_i64(pd
);
1398 tcg_temp_free_i64(pn
);
1399 tcg_temp_free_i64(pm
);
1400 tcg_temp_free_i64(pg
);
1402 /* The operation and flags generation is large. The computation
1403 * of the flags depends on the original contents of the guarding
1404 * predicate. If the destination overwrites the guarding predicate,
1405 * then the easiest way to get this right is to save a copy.
1408 if (a
->rd
== a
->pg
) {
1409 tofs
= offsetof(CPUARMState
, vfp
.preg_tmp
);
1410 tcg_gen_gvec_mov(0, tofs
, gofs
, psz
, psz
);
1413 tcg_gen_gvec_4(dofs
, nofs
, mofs
, gofs
, psz
, psz
, gvec_op
);
1414 do_predtest(s
, dofs
, tofs
, psz
/ 8);
1419 static void gen_and_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1421 tcg_gen_and_i64(pd
, pn
, pm
);
1422 tcg_gen_and_i64(pd
, pd
, pg
);
1425 static void gen_and_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1426 TCGv_vec pm
, TCGv_vec pg
)
1428 tcg_gen_and_vec(vece
, pd
, pn
, pm
);
1429 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1432 static bool trans_AND_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1434 static const GVecGen4 op
= {
1435 .fni8
= gen_and_pg_i64
,
1436 .fniv
= gen_and_pg_vec
,
1437 .fno
= gen_helper_sve_and_pppp
,
1438 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1442 if (!sve_access_check(s
)) {
1445 if (a
->rn
== a
->rm
) {
1446 if (a
->pg
== a
->rn
) {
1447 do_mov_p(s
, a
->rd
, a
->rn
);
1449 gen_gvec_fn_ppp(s
, tcg_gen_gvec_and
, a
->rd
, a
->rn
, a
->pg
);
1452 } else if (a
->pg
== a
->rn
|| a
->pg
== a
->rm
) {
1453 gen_gvec_fn_ppp(s
, tcg_gen_gvec_and
, a
->rd
, a
->rn
, a
->rm
);
1457 return do_pppp_flags(s
, a
, &op
);
1460 static void gen_bic_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1462 tcg_gen_andc_i64(pd
, pn
, pm
);
1463 tcg_gen_and_i64(pd
, pd
, pg
);
1466 static void gen_bic_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1467 TCGv_vec pm
, TCGv_vec pg
)
1469 tcg_gen_andc_vec(vece
, pd
, pn
, pm
);
1470 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1473 static bool trans_BIC_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1475 static const GVecGen4 op
= {
1476 .fni8
= gen_bic_pg_i64
,
1477 .fniv
= gen_bic_pg_vec
,
1478 .fno
= gen_helper_sve_bic_pppp
,
1479 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1482 if (!a
->s
&& a
->pg
== a
->rn
) {
1483 if (sve_access_check(s
)) {
1484 gen_gvec_fn_ppp(s
, tcg_gen_gvec_andc
, a
->rd
, a
->rn
, a
->rm
);
1488 return do_pppp_flags(s
, a
, &op
);
1491 static void gen_eor_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1493 tcg_gen_xor_i64(pd
, pn
, pm
);
1494 tcg_gen_and_i64(pd
, pd
, pg
);
1497 static void gen_eor_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1498 TCGv_vec pm
, TCGv_vec pg
)
1500 tcg_gen_xor_vec(vece
, pd
, pn
, pm
);
1501 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1504 static bool trans_EOR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1506 static const GVecGen4 op
= {
1507 .fni8
= gen_eor_pg_i64
,
1508 .fniv
= gen_eor_pg_vec
,
1509 .fno
= gen_helper_sve_eor_pppp
,
1510 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1512 return do_pppp_flags(s
, a
, &op
);
1515 static bool trans_SEL_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1520 if (sve_access_check(s
)) {
1521 unsigned psz
= pred_gvec_reg_size(s
);
1522 tcg_gen_gvec_bitsel(MO_8
, pred_full_reg_offset(s
, a
->rd
),
1523 pred_full_reg_offset(s
, a
->pg
),
1524 pred_full_reg_offset(s
, a
->rn
),
1525 pred_full_reg_offset(s
, a
->rm
), psz
, psz
);
1530 static void gen_orr_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1532 tcg_gen_or_i64(pd
, pn
, pm
);
1533 tcg_gen_and_i64(pd
, pd
, pg
);
1536 static void gen_orr_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1537 TCGv_vec pm
, TCGv_vec pg
)
1539 tcg_gen_or_vec(vece
, pd
, pn
, pm
);
1540 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1543 static bool trans_ORR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1545 static const GVecGen4 op
= {
1546 .fni8
= gen_orr_pg_i64
,
1547 .fniv
= gen_orr_pg_vec
,
1548 .fno
= gen_helper_sve_orr_pppp
,
1549 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1552 if (!a
->s
&& a
->pg
== a
->rn
&& a
->rn
== a
->rm
) {
1553 return do_mov_p(s
, a
->rd
, a
->rn
);
1555 return do_pppp_flags(s
, a
, &op
);
1558 static void gen_orn_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1560 tcg_gen_orc_i64(pd
, pn
, pm
);
1561 tcg_gen_and_i64(pd
, pd
, pg
);
1564 static void gen_orn_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1565 TCGv_vec pm
, TCGv_vec pg
)
1567 tcg_gen_orc_vec(vece
, pd
, pn
, pm
);
1568 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1571 static bool trans_ORN_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1573 static const GVecGen4 op
= {
1574 .fni8
= gen_orn_pg_i64
,
1575 .fniv
= gen_orn_pg_vec
,
1576 .fno
= gen_helper_sve_orn_pppp
,
1577 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1579 return do_pppp_flags(s
, a
, &op
);
1582 static void gen_nor_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1584 tcg_gen_or_i64(pd
, pn
, pm
);
1585 tcg_gen_andc_i64(pd
, pg
, pd
);
1588 static void gen_nor_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1589 TCGv_vec pm
, TCGv_vec pg
)
1591 tcg_gen_or_vec(vece
, pd
, pn
, pm
);
1592 tcg_gen_andc_vec(vece
, pd
, pg
, pd
);
1595 static bool trans_NOR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1597 static const GVecGen4 op
= {
1598 .fni8
= gen_nor_pg_i64
,
1599 .fniv
= gen_nor_pg_vec
,
1600 .fno
= gen_helper_sve_nor_pppp
,
1601 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1603 return do_pppp_flags(s
, a
, &op
);
1606 static void gen_nand_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1608 tcg_gen_and_i64(pd
, pn
, pm
);
1609 tcg_gen_andc_i64(pd
, pg
, pd
);
1612 static void gen_nand_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1613 TCGv_vec pm
, TCGv_vec pg
)
1615 tcg_gen_and_vec(vece
, pd
, pn
, pm
);
1616 tcg_gen_andc_vec(vece
, pd
, pg
, pd
);
1619 static bool trans_NAND_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1621 static const GVecGen4 op
= {
1622 .fni8
= gen_nand_pg_i64
,
1623 .fniv
= gen_nand_pg_vec
,
1624 .fno
= gen_helper_sve_nand_pppp
,
1625 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1627 return do_pppp_flags(s
, a
, &op
);
1631 *** SVE Predicate Misc Group
1634 static bool trans_PTEST(DisasContext
*s
, arg_PTEST
*a
)
1636 if (sve_access_check(s
)) {
1637 int nofs
= pred_full_reg_offset(s
, a
->rn
);
1638 int gofs
= pred_full_reg_offset(s
, a
->pg
);
1639 int words
= DIV_ROUND_UP(pred_full_reg_size(s
), 8);
1642 TCGv_i64 pn
= tcg_temp_new_i64();
1643 TCGv_i64 pg
= tcg_temp_new_i64();
1645 tcg_gen_ld_i64(pn
, cpu_env
, nofs
);
1646 tcg_gen_ld_i64(pg
, cpu_env
, gofs
);
1647 do_predtest1(pn
, pg
);
1649 tcg_temp_free_i64(pn
);
1650 tcg_temp_free_i64(pg
);
1652 do_predtest(s
, nofs
, gofs
, words
);
1658 /* See the ARM pseudocode DecodePredCount. */
1659 static unsigned decode_pred_count(unsigned fullsz
, int pattern
, int esz
)
1661 unsigned elements
= fullsz
>> esz
;
1665 case 0x0: /* POW2 */
1666 return pow2floor(elements
);
1677 case 0x9: /* VL16 */
1678 case 0xa: /* VL32 */
1679 case 0xb: /* VL64 */
1680 case 0xc: /* VL128 */
1681 case 0xd: /* VL256 */
1682 bound
= 16 << (pattern
- 9);
1684 case 0x1d: /* MUL4 */
1685 return elements
- elements
% 4;
1686 case 0x1e: /* MUL3 */
1687 return elements
- elements
% 3;
1688 case 0x1f: /* ALL */
1690 default: /* #uimm5 */
1693 return elements
>= bound
? bound
: 0;
1696 /* This handles all of the predicate initialization instructions,
1697 * PTRUE, PFALSE, SETFFR. For PFALSE, we will have set PAT == 32
1698 * so that decode_pred_count returns 0. For SETFFR, we will have
1699 * set RD == 16 == FFR.
1701 static bool do_predset(DisasContext
*s
, int esz
, int rd
, int pat
, bool setflag
)
1703 if (!sve_access_check(s
)) {
1707 unsigned fullsz
= vec_full_reg_size(s
);
1708 unsigned ofs
= pred_full_reg_offset(s
, rd
);
1709 unsigned numelem
, setsz
, i
;
1710 uint64_t word
, lastword
;
1713 numelem
= decode_pred_count(fullsz
, pat
, esz
);
1715 /* Determine what we must store into each bit, and how many. */
1717 lastword
= word
= 0;
1720 setsz
= numelem
<< esz
;
1721 lastword
= word
= pred_esz_masks
[esz
];
1723 lastword
&= MAKE_64BIT_MASK(0, setsz
% 64);
1727 t
= tcg_temp_new_i64();
1729 tcg_gen_movi_i64(t
, lastword
);
1730 tcg_gen_st_i64(t
, cpu_env
, ofs
);
1734 if (word
== lastword
) {
1735 unsigned maxsz
= size_for_gvec(fullsz
/ 8);
1736 unsigned oprsz
= size_for_gvec(setsz
/ 8);
1738 if (oprsz
* 8 == setsz
) {
1739 tcg_gen_gvec_dup_imm(MO_64
, ofs
, oprsz
, maxsz
, word
);
1747 tcg_gen_movi_i64(t
, word
);
1748 for (i
= 0; i
< QEMU_ALIGN_DOWN(setsz
, 8); i
+= 8) {
1749 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1751 if (lastword
!= word
) {
1752 tcg_gen_movi_i64(t
, lastword
);
1753 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1757 tcg_gen_movi_i64(t
, 0);
1758 for (; i
< fullsz
; i
+= 8) {
1759 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1764 tcg_temp_free_i64(t
);
1768 tcg_gen_movi_i32(cpu_NF
, -(word
!= 0));
1769 tcg_gen_movi_i32(cpu_CF
, word
== 0);
1770 tcg_gen_movi_i32(cpu_VF
, 0);
1771 tcg_gen_mov_i32(cpu_ZF
, cpu_NF
);
1776 static bool trans_PTRUE(DisasContext
*s
, arg_PTRUE
*a
)
1778 return do_predset(s
, a
->esz
, a
->rd
, a
->pat
, a
->s
);
1781 static bool trans_SETFFR(DisasContext
*s
, arg_SETFFR
*a
)
1783 /* Note pat == 31 is #all, to set all elements. */
1784 return do_predset(s
, 0, FFR_PRED_NUM
, 31, false);
1787 static bool trans_PFALSE(DisasContext
*s
, arg_PFALSE
*a
)
1789 /* Note pat == 32 is #unimp, to set no elements. */
1790 return do_predset(s
, 0, a
->rd
, 32, false);
1793 static bool trans_RDFFR_p(DisasContext
*s
, arg_RDFFR_p
*a
)
1795 /* The path through do_pppp_flags is complicated enough to want to avoid
1796 * duplication. Frob the arguments into the form of a predicated AND.
1798 arg_rprr_s alt_a
= {
1799 .rd
= a
->rd
, .pg
= a
->pg
, .s
= a
->s
,
1800 .rn
= FFR_PRED_NUM
, .rm
= FFR_PRED_NUM
,
1802 return trans_AND_pppp(s
, &alt_a
);
1805 static bool trans_RDFFR(DisasContext
*s
, arg_RDFFR
*a
)
1807 return do_mov_p(s
, a
->rd
, FFR_PRED_NUM
);
1810 static bool trans_WRFFR(DisasContext
*s
, arg_WRFFR
*a
)
1812 return do_mov_p(s
, FFR_PRED_NUM
, a
->rn
);
1815 static bool do_pfirst_pnext(DisasContext
*s
, arg_rr_esz
*a
,
1816 void (*gen_fn
)(TCGv_i32
, TCGv_ptr
,
1817 TCGv_ptr
, TCGv_i32
))
1819 if (!sve_access_check(s
)) {
1823 TCGv_ptr t_pd
= tcg_temp_new_ptr();
1824 TCGv_ptr t_pg
= tcg_temp_new_ptr();
1828 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, pred_full_reg_size(s
));
1829 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
1831 tcg_gen_addi_ptr(t_pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
1832 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
1833 t
= tcg_const_i32(desc
);
1835 gen_fn(t
, t_pd
, t_pg
, t
);
1836 tcg_temp_free_ptr(t_pd
);
1837 tcg_temp_free_ptr(t_pg
);
1840 tcg_temp_free_i32(t
);
1844 static bool trans_PFIRST(DisasContext
*s
, arg_rr_esz
*a
)
1846 return do_pfirst_pnext(s
, a
, gen_helper_sve_pfirst
);
1849 static bool trans_PNEXT(DisasContext
*s
, arg_rr_esz
*a
)
1851 return do_pfirst_pnext(s
, a
, gen_helper_sve_pnext
);
1855 *** SVE Element Count Group
1858 /* Perform an inline saturating addition of a 32-bit value within
1859 * a 64-bit register. The second operand is known to be positive,
1860 * which halves the comparisions we must perform to bound the result.
1862 static void do_sat_addsub_32(TCGv_i64 reg
, TCGv_i64 val
, bool u
, bool d
)
1868 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
1870 tcg_gen_ext32u_i64(reg
, reg
);
1872 tcg_gen_ext32s_i64(reg
, reg
);
1875 tcg_gen_sub_i64(reg
, reg
, val
);
1876 ibound
= (u
? 0 : INT32_MIN
);
1879 tcg_gen_add_i64(reg
, reg
, val
);
1880 ibound
= (u
? UINT32_MAX
: INT32_MAX
);
1883 bound
= tcg_const_i64(ibound
);
1884 tcg_gen_movcond_i64(cond
, reg
, reg
, bound
, bound
, reg
);
1885 tcg_temp_free_i64(bound
);
1888 /* Similarly with 64-bit values. */
1889 static void do_sat_addsub_64(TCGv_i64 reg
, TCGv_i64 val
, bool u
, bool d
)
1891 TCGv_i64 t0
= tcg_temp_new_i64();
1892 TCGv_i64 t1
= tcg_temp_new_i64();
1897 tcg_gen_sub_i64(t0
, reg
, val
);
1898 tcg_gen_movi_i64(t1
, 0);
1899 tcg_gen_movcond_i64(TCG_COND_LTU
, reg
, reg
, val
, t1
, t0
);
1901 tcg_gen_add_i64(t0
, reg
, val
);
1902 tcg_gen_movi_i64(t1
, -1);
1903 tcg_gen_movcond_i64(TCG_COND_LTU
, reg
, t0
, reg
, t1
, t0
);
1907 /* Detect signed overflow for subtraction. */
1908 tcg_gen_xor_i64(t0
, reg
, val
);
1909 tcg_gen_sub_i64(t1
, reg
, val
);
1910 tcg_gen_xor_i64(reg
, reg
, t1
);
1911 tcg_gen_and_i64(t0
, t0
, reg
);
1913 /* Bound the result. */
1914 tcg_gen_movi_i64(reg
, INT64_MIN
);
1915 t2
= tcg_const_i64(0);
1916 tcg_gen_movcond_i64(TCG_COND_LT
, reg
, t0
, t2
, reg
, t1
);
1918 /* Detect signed overflow for addition. */
1919 tcg_gen_xor_i64(t0
, reg
, val
);
1920 tcg_gen_add_i64(reg
, reg
, val
);
1921 tcg_gen_xor_i64(t1
, reg
, val
);
1922 tcg_gen_andc_i64(t0
, t1
, t0
);
1924 /* Bound the result. */
1925 tcg_gen_movi_i64(t1
, INT64_MAX
);
1926 t2
= tcg_const_i64(0);
1927 tcg_gen_movcond_i64(TCG_COND_LT
, reg
, t0
, t2
, t1
, reg
);
1929 tcg_temp_free_i64(t2
);
1931 tcg_temp_free_i64(t0
);
1932 tcg_temp_free_i64(t1
);
1935 /* Similarly with a vector and a scalar operand. */
1936 static void do_sat_addsub_vec(DisasContext
*s
, int esz
, int rd
, int rn
,
1937 TCGv_i64 val
, bool u
, bool d
)
1939 unsigned vsz
= vec_full_reg_size(s
);
1940 TCGv_ptr dptr
, nptr
;
1944 dptr
= tcg_temp_new_ptr();
1945 nptr
= tcg_temp_new_ptr();
1946 tcg_gen_addi_ptr(dptr
, cpu_env
, vec_full_reg_offset(s
, rd
));
1947 tcg_gen_addi_ptr(nptr
, cpu_env
, vec_full_reg_offset(s
, rn
));
1948 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
1952 t32
= tcg_temp_new_i32();
1953 tcg_gen_extrl_i64_i32(t32
, val
);
1955 tcg_gen_neg_i32(t32
, t32
);
1958 gen_helper_sve_uqaddi_b(dptr
, nptr
, t32
, desc
);
1960 gen_helper_sve_sqaddi_b(dptr
, nptr
, t32
, desc
);
1962 tcg_temp_free_i32(t32
);
1966 t32
= tcg_temp_new_i32();
1967 tcg_gen_extrl_i64_i32(t32
, val
);
1969 tcg_gen_neg_i32(t32
, t32
);
1972 gen_helper_sve_uqaddi_h(dptr
, nptr
, t32
, desc
);
1974 gen_helper_sve_sqaddi_h(dptr
, nptr
, t32
, desc
);
1976 tcg_temp_free_i32(t32
);
1980 t64
= tcg_temp_new_i64();
1982 tcg_gen_neg_i64(t64
, val
);
1984 tcg_gen_mov_i64(t64
, val
);
1987 gen_helper_sve_uqaddi_s(dptr
, nptr
, t64
, desc
);
1989 gen_helper_sve_sqaddi_s(dptr
, nptr
, t64
, desc
);
1991 tcg_temp_free_i64(t64
);
1997 gen_helper_sve_uqsubi_d(dptr
, nptr
, val
, desc
);
1999 gen_helper_sve_uqaddi_d(dptr
, nptr
, val
, desc
);
2002 t64
= tcg_temp_new_i64();
2003 tcg_gen_neg_i64(t64
, val
);
2004 gen_helper_sve_sqaddi_d(dptr
, nptr
, t64
, desc
);
2005 tcg_temp_free_i64(t64
);
2007 gen_helper_sve_sqaddi_d(dptr
, nptr
, val
, desc
);
2012 g_assert_not_reached();
2015 tcg_temp_free_ptr(dptr
);
2016 tcg_temp_free_ptr(nptr
);
2017 tcg_temp_free_i32(desc
);
2020 static bool trans_CNT_r(DisasContext
*s
, arg_CNT_r
*a
)
2022 if (sve_access_check(s
)) {
2023 unsigned fullsz
= vec_full_reg_size(s
);
2024 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2025 tcg_gen_movi_i64(cpu_reg(s
, a
->rd
), numelem
* a
->imm
);
2030 static bool trans_INCDEC_r(DisasContext
*s
, arg_incdec_cnt
*a
)
2032 if (sve_access_check(s
)) {
2033 unsigned fullsz
= vec_full_reg_size(s
);
2034 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2035 int inc
= numelem
* a
->imm
* (a
->d
? -1 : 1);
2036 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
2038 tcg_gen_addi_i64(reg
, reg
, inc
);
2043 static bool trans_SINCDEC_r_32(DisasContext
*s
, arg_incdec_cnt
*a
)
2045 if (!sve_access_check(s
)) {
2049 unsigned fullsz
= vec_full_reg_size(s
);
2050 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2051 int inc
= numelem
* a
->imm
;
2052 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
2054 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
2057 tcg_gen_ext32u_i64(reg
, reg
);
2059 tcg_gen_ext32s_i64(reg
, reg
);
2062 TCGv_i64 t
= tcg_const_i64(inc
);
2063 do_sat_addsub_32(reg
, t
, a
->u
, a
->d
);
2064 tcg_temp_free_i64(t
);
2069 static bool trans_SINCDEC_r_64(DisasContext
*s
, arg_incdec_cnt
*a
)
2071 if (!sve_access_check(s
)) {
2075 unsigned fullsz
= vec_full_reg_size(s
);
2076 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2077 int inc
= numelem
* a
->imm
;
2078 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
2081 TCGv_i64 t
= tcg_const_i64(inc
);
2082 do_sat_addsub_64(reg
, t
, a
->u
, a
->d
);
2083 tcg_temp_free_i64(t
);
2088 static bool trans_INCDEC_v(DisasContext
*s
, arg_incdec2_cnt
*a
)
2094 unsigned fullsz
= vec_full_reg_size(s
);
2095 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2096 int inc
= numelem
* a
->imm
;
2099 if (sve_access_check(s
)) {
2100 TCGv_i64 t
= tcg_const_i64(a
->d
? -inc
: inc
);
2101 tcg_gen_gvec_adds(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
2102 vec_full_reg_offset(s
, a
->rn
),
2104 tcg_temp_free_i64(t
);
2107 do_mov_z(s
, a
->rd
, a
->rn
);
2112 static bool trans_SINCDEC_v(DisasContext
*s
, arg_incdec2_cnt
*a
)
2118 unsigned fullsz
= vec_full_reg_size(s
);
2119 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2120 int inc
= numelem
* a
->imm
;
2123 if (sve_access_check(s
)) {
2124 TCGv_i64 t
= tcg_const_i64(inc
);
2125 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
, t
, a
->u
, a
->d
);
2126 tcg_temp_free_i64(t
);
2129 do_mov_z(s
, a
->rd
, a
->rn
);
2135 *** SVE Bitwise Immediate Group
2138 static bool do_zz_dbm(DisasContext
*s
, arg_rr_dbm
*a
, GVecGen2iFn
*gvec_fn
)
2141 if (!logic_imm_decode_wmask(&imm
, extract32(a
->dbm
, 12, 1),
2142 extract32(a
->dbm
, 0, 6),
2143 extract32(a
->dbm
, 6, 6))) {
2146 if (sve_access_check(s
)) {
2147 unsigned vsz
= vec_full_reg_size(s
);
2148 gvec_fn(MO_64
, vec_full_reg_offset(s
, a
->rd
),
2149 vec_full_reg_offset(s
, a
->rn
), imm
, vsz
, vsz
);
2154 static bool trans_AND_zzi(DisasContext
*s
, arg_rr_dbm
*a
)
2156 return do_zz_dbm(s
, a
, tcg_gen_gvec_andi
);
2159 static bool trans_ORR_zzi(DisasContext
*s
, arg_rr_dbm
*a
)
2161 return do_zz_dbm(s
, a
, tcg_gen_gvec_ori
);
2164 static bool trans_EOR_zzi(DisasContext
*s
, arg_rr_dbm
*a
)
2166 return do_zz_dbm(s
, a
, tcg_gen_gvec_xori
);
2169 static bool trans_DUPM(DisasContext
*s
, arg_DUPM
*a
)
2172 if (!logic_imm_decode_wmask(&imm
, extract32(a
->dbm
, 12, 1),
2173 extract32(a
->dbm
, 0, 6),
2174 extract32(a
->dbm
, 6, 6))) {
2177 if (sve_access_check(s
)) {
2178 do_dupi_z(s
, a
->rd
, imm
);
2184 *** SVE Integer Wide Immediate - Predicated Group
2187 /* Implement all merging copies. This is used for CPY (immediate),
2188 * FCPY, CPY (scalar), CPY (SIMD&FP scalar).
2190 static void do_cpy_m(DisasContext
*s
, int esz
, int rd
, int rn
, int pg
,
2193 typedef void gen_cpy(TCGv_ptr
, TCGv_ptr
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
2194 static gen_cpy
* const fns
[4] = {
2195 gen_helper_sve_cpy_m_b
, gen_helper_sve_cpy_m_h
,
2196 gen_helper_sve_cpy_m_s
, gen_helper_sve_cpy_m_d
,
2198 unsigned vsz
= vec_full_reg_size(s
);
2199 TCGv_i32 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
2200 TCGv_ptr t_zd
= tcg_temp_new_ptr();
2201 TCGv_ptr t_zn
= tcg_temp_new_ptr();
2202 TCGv_ptr t_pg
= tcg_temp_new_ptr();
2204 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, rd
));
2205 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, rn
));
2206 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
2208 fns
[esz
](t_zd
, t_zn
, t_pg
, val
, desc
);
2210 tcg_temp_free_ptr(t_zd
);
2211 tcg_temp_free_ptr(t_zn
);
2212 tcg_temp_free_ptr(t_pg
);
2213 tcg_temp_free_i32(desc
);
2216 static bool trans_FCPY(DisasContext
*s
, arg_FCPY
*a
)
2221 if (sve_access_check(s
)) {
2222 /* Decode the VFP immediate. */
2223 uint64_t imm
= vfp_expand_imm(a
->esz
, a
->imm
);
2224 TCGv_i64 t_imm
= tcg_const_i64(imm
);
2225 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rn
, a
->pg
, t_imm
);
2226 tcg_temp_free_i64(t_imm
);
2231 static bool trans_CPY_m_i(DisasContext
*s
, arg_rpri_esz
*a
)
2233 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
2236 if (sve_access_check(s
)) {
2237 TCGv_i64 t_imm
= tcg_const_i64(a
->imm
);
2238 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rn
, a
->pg
, t_imm
);
2239 tcg_temp_free_i64(t_imm
);
2244 static bool trans_CPY_z_i(DisasContext
*s
, arg_CPY_z_i
*a
)
2246 static gen_helper_gvec_2i
* const fns
[4] = {
2247 gen_helper_sve_cpy_z_b
, gen_helper_sve_cpy_z_h
,
2248 gen_helper_sve_cpy_z_s
, gen_helper_sve_cpy_z_d
,
2251 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
2254 if (sve_access_check(s
)) {
2255 unsigned vsz
= vec_full_reg_size(s
);
2256 TCGv_i64 t_imm
= tcg_const_i64(a
->imm
);
2257 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s
, a
->rd
),
2258 pred_full_reg_offset(s
, a
->pg
),
2259 t_imm
, vsz
, vsz
, 0, fns
[a
->esz
]);
2260 tcg_temp_free_i64(t_imm
);
2266 *** SVE Permute Extract Group
2269 static bool do_EXT(DisasContext
*s
, int rd
, int rn
, int rm
, int imm
)
2271 if (!sve_access_check(s
)) {
2275 unsigned vsz
= vec_full_reg_size(s
);
2276 unsigned n_ofs
= imm
>= vsz
? 0 : imm
;
2277 unsigned n_siz
= vsz
- n_ofs
;
2278 unsigned d
= vec_full_reg_offset(s
, rd
);
2279 unsigned n
= vec_full_reg_offset(s
, rn
);
2280 unsigned m
= vec_full_reg_offset(s
, rm
);
2282 /* Use host vector move insns if we have appropriate sizes
2283 * and no unfortunate overlap.
2286 && n_ofs
== size_for_gvec(n_ofs
)
2287 && n_siz
== size_for_gvec(n_siz
)
2288 && (d
!= n
|| n_siz
<= n_ofs
)) {
2289 tcg_gen_gvec_mov(0, d
, n
+ n_ofs
, n_siz
, n_siz
);
2291 tcg_gen_gvec_mov(0, d
+ n_siz
, m
, n_ofs
, n_ofs
);
2294 tcg_gen_gvec_3_ool(d
, n
, m
, vsz
, vsz
, n_ofs
, gen_helper_sve_ext
);
2299 static bool trans_EXT(DisasContext
*s
, arg_EXT
*a
)
2301 return do_EXT(s
, a
->rd
, a
->rn
, a
->rm
, a
->imm
);
2304 static bool trans_EXT_sve2(DisasContext
*s
, arg_rri
*a
)
2306 if (!dc_isar_feature(aa64_sve2
, s
)) {
2309 return do_EXT(s
, a
->rd
, a
->rn
, (a
->rn
+ 1) % 32, a
->imm
);
2313 *** SVE Permute - Unpredicated Group
2316 static bool trans_DUP_s(DisasContext
*s
, arg_DUP_s
*a
)
2318 if (sve_access_check(s
)) {
2319 unsigned vsz
= vec_full_reg_size(s
);
2320 tcg_gen_gvec_dup_i64(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
2321 vsz
, vsz
, cpu_reg_sp(s
, a
->rn
));
2326 static bool trans_DUP_x(DisasContext
*s
, arg_DUP_x
*a
)
2328 if ((a
->imm
& 0x1f) == 0) {
2331 if (sve_access_check(s
)) {
2332 unsigned vsz
= vec_full_reg_size(s
);
2333 unsigned dofs
= vec_full_reg_offset(s
, a
->rd
);
2334 unsigned esz
, index
;
2336 esz
= ctz32(a
->imm
);
2337 index
= a
->imm
>> (esz
+ 1);
2339 if ((index
<< esz
) < vsz
) {
2340 unsigned nofs
= vec_reg_offset(s
, a
->rn
, index
, esz
);
2341 tcg_gen_gvec_dup_mem(esz
, dofs
, nofs
, vsz
, vsz
);
2344 * While dup_mem handles 128-bit elements, dup_imm does not.
2345 * Thankfully element size doesn't matter for splatting zero.
2347 tcg_gen_gvec_dup_imm(MO_64
, dofs
, vsz
, vsz
, 0);
2353 static void do_insr_i64(DisasContext
*s
, arg_rrr_esz
*a
, TCGv_i64 val
)
2355 typedef void gen_insr(TCGv_ptr
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
2356 static gen_insr
* const fns
[4] = {
2357 gen_helper_sve_insr_b
, gen_helper_sve_insr_h
,
2358 gen_helper_sve_insr_s
, gen_helper_sve_insr_d
,
2360 unsigned vsz
= vec_full_reg_size(s
);
2361 TCGv_i32 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
2362 TCGv_ptr t_zd
= tcg_temp_new_ptr();
2363 TCGv_ptr t_zn
= tcg_temp_new_ptr();
2365 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, a
->rd
));
2366 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
2368 fns
[a
->esz
](t_zd
, t_zn
, val
, desc
);
2370 tcg_temp_free_ptr(t_zd
);
2371 tcg_temp_free_ptr(t_zn
);
2372 tcg_temp_free_i32(desc
);
2375 static bool trans_INSR_f(DisasContext
*s
, arg_rrr_esz
*a
)
2377 if (sve_access_check(s
)) {
2378 TCGv_i64 t
= tcg_temp_new_i64();
2379 tcg_gen_ld_i64(t
, cpu_env
, vec_reg_offset(s
, a
->rm
, 0, MO_64
));
2380 do_insr_i64(s
, a
, t
);
2381 tcg_temp_free_i64(t
);
2386 static bool trans_INSR_r(DisasContext
*s
, arg_rrr_esz
*a
)
2388 if (sve_access_check(s
)) {
2389 do_insr_i64(s
, a
, cpu_reg(s
, a
->rm
));
2394 static bool trans_REV_v(DisasContext
*s
, arg_rr_esz
*a
)
2396 static gen_helper_gvec_2
* const fns
[4] = {
2397 gen_helper_sve_rev_b
, gen_helper_sve_rev_h
,
2398 gen_helper_sve_rev_s
, gen_helper_sve_rev_d
2401 if (sve_access_check(s
)) {
2402 gen_gvec_ool_zz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, 0);
2407 static bool trans_TBL(DisasContext
*s
, arg_rrr_esz
*a
)
2409 static gen_helper_gvec_3
* const fns
[4] = {
2410 gen_helper_sve_tbl_b
, gen_helper_sve_tbl_h
,
2411 gen_helper_sve_tbl_s
, gen_helper_sve_tbl_d
2414 if (sve_access_check(s
)) {
2415 gen_gvec_ool_zzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, 0);
2420 static bool trans_UNPK(DisasContext
*s
, arg_UNPK
*a
)
2422 static gen_helper_gvec_2
* const fns
[4][2] = {
2424 { gen_helper_sve_sunpk_h
, gen_helper_sve_uunpk_h
},
2425 { gen_helper_sve_sunpk_s
, gen_helper_sve_uunpk_s
},
2426 { gen_helper_sve_sunpk_d
, gen_helper_sve_uunpk_d
},
2432 if (sve_access_check(s
)) {
2433 unsigned vsz
= vec_full_reg_size(s
);
2434 tcg_gen_gvec_2_ool(vec_full_reg_offset(s
, a
->rd
),
2435 vec_full_reg_offset(s
, a
->rn
)
2436 + (a
->h
? vsz
/ 2 : 0),
2437 vsz
, vsz
, 0, fns
[a
->esz
][a
->u
]);
2443 *** SVE Permute - Predicates Group
2446 static bool do_perm_pred3(DisasContext
*s
, arg_rrr_esz
*a
, bool high_odd
,
2447 gen_helper_gvec_3
*fn
)
2449 if (!sve_access_check(s
)) {
2453 unsigned vsz
= pred_full_reg_size(s
);
2455 TCGv_ptr t_d
= tcg_temp_new_ptr();
2456 TCGv_ptr t_n
= tcg_temp_new_ptr();
2457 TCGv_ptr t_m
= tcg_temp_new_ptr();
2461 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
);
2462 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
2463 desc
= FIELD_DP32(desc
, PREDDESC
, DATA
, high_odd
);
2465 tcg_gen_addi_ptr(t_d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2466 tcg_gen_addi_ptr(t_n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2467 tcg_gen_addi_ptr(t_m
, cpu_env
, pred_full_reg_offset(s
, a
->rm
));
2468 t_desc
= tcg_const_i32(desc
);
2470 fn(t_d
, t_n
, t_m
, t_desc
);
2472 tcg_temp_free_ptr(t_d
);
2473 tcg_temp_free_ptr(t_n
);
2474 tcg_temp_free_ptr(t_m
);
2475 tcg_temp_free_i32(t_desc
);
2479 static bool do_perm_pred2(DisasContext
*s
, arg_rr_esz
*a
, bool high_odd
,
2480 gen_helper_gvec_2
*fn
)
2482 if (!sve_access_check(s
)) {
2486 unsigned vsz
= pred_full_reg_size(s
);
2487 TCGv_ptr t_d
= tcg_temp_new_ptr();
2488 TCGv_ptr t_n
= tcg_temp_new_ptr();
2492 tcg_gen_addi_ptr(t_d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2493 tcg_gen_addi_ptr(t_n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2495 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
);
2496 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
2497 desc
= FIELD_DP32(desc
, PREDDESC
, DATA
, high_odd
);
2498 t_desc
= tcg_const_i32(desc
);
2500 fn(t_d
, t_n
, t_desc
);
2502 tcg_temp_free_i32(t_desc
);
2503 tcg_temp_free_ptr(t_d
);
2504 tcg_temp_free_ptr(t_n
);
2508 static bool trans_ZIP1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2510 return do_perm_pred3(s
, a
, 0, gen_helper_sve_zip_p
);
2513 static bool trans_ZIP2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2515 return do_perm_pred3(s
, a
, 1, gen_helper_sve_zip_p
);
2518 static bool trans_UZP1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2520 return do_perm_pred3(s
, a
, 0, gen_helper_sve_uzp_p
);
2523 static bool trans_UZP2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2525 return do_perm_pred3(s
, a
, 1, gen_helper_sve_uzp_p
);
2528 static bool trans_TRN1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2530 return do_perm_pred3(s
, a
, 0, gen_helper_sve_trn_p
);
2533 static bool trans_TRN2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2535 return do_perm_pred3(s
, a
, 1, gen_helper_sve_trn_p
);
2538 static bool trans_REV_p(DisasContext
*s
, arg_rr_esz
*a
)
2540 return do_perm_pred2(s
, a
, 0, gen_helper_sve_rev_p
);
2543 static bool trans_PUNPKLO(DisasContext
*s
, arg_PUNPKLO
*a
)
2545 return do_perm_pred2(s
, a
, 0, gen_helper_sve_punpk_p
);
2548 static bool trans_PUNPKHI(DisasContext
*s
, arg_PUNPKHI
*a
)
2550 return do_perm_pred2(s
, a
, 1, gen_helper_sve_punpk_p
);
2554 *** SVE Permute - Interleaving Group
2557 static bool do_zip(DisasContext
*s
, arg_rrr_esz
*a
, bool high
)
2559 static gen_helper_gvec_3
* const fns
[4] = {
2560 gen_helper_sve_zip_b
, gen_helper_sve_zip_h
,
2561 gen_helper_sve_zip_s
, gen_helper_sve_zip_d
,
2564 if (sve_access_check(s
)) {
2565 unsigned vsz
= vec_full_reg_size(s
);
2566 unsigned high_ofs
= high
? vsz
/ 2 : 0;
2567 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, a
->rd
),
2568 vec_full_reg_offset(s
, a
->rn
) + high_ofs
,
2569 vec_full_reg_offset(s
, a
->rm
) + high_ofs
,
2570 vsz
, vsz
, 0, fns
[a
->esz
]);
2575 static bool do_zzz_data_ool(DisasContext
*s
, arg_rrr_esz
*a
, int data
,
2576 gen_helper_gvec_3
*fn
)
2578 if (sve_access_check(s
)) {
2579 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, data
);
2584 static bool trans_ZIP1_z(DisasContext
*s
, arg_rrr_esz
*a
)
2586 return do_zip(s
, a
, false);
2589 static bool trans_ZIP2_z(DisasContext
*s
, arg_rrr_esz
*a
)
2591 return do_zip(s
, a
, true);
2594 static gen_helper_gvec_3
* const uzp_fns
[4] = {
2595 gen_helper_sve_uzp_b
, gen_helper_sve_uzp_h
,
2596 gen_helper_sve_uzp_s
, gen_helper_sve_uzp_d
,
2599 static bool trans_UZP1_z(DisasContext
*s
, arg_rrr_esz
*a
)
2601 return do_zzz_data_ool(s
, a
, 0, uzp_fns
[a
->esz
]);
2604 static bool trans_UZP2_z(DisasContext
*s
, arg_rrr_esz
*a
)
2606 return do_zzz_data_ool(s
, a
, 1 << a
->esz
, uzp_fns
[a
->esz
]);
2609 static gen_helper_gvec_3
* const trn_fns
[4] = {
2610 gen_helper_sve_trn_b
, gen_helper_sve_trn_h
,
2611 gen_helper_sve_trn_s
, gen_helper_sve_trn_d
,
2614 static bool trans_TRN1_z(DisasContext
*s
, arg_rrr_esz
*a
)
2616 return do_zzz_data_ool(s
, a
, 0, trn_fns
[a
->esz
]);
2619 static bool trans_TRN2_z(DisasContext
*s
, arg_rrr_esz
*a
)
2621 return do_zzz_data_ool(s
, a
, 1 << a
->esz
, trn_fns
[a
->esz
]);
2625 *** SVE Permute Vector - Predicated Group
2628 static bool trans_COMPACT(DisasContext
*s
, arg_rpr_esz
*a
)
2630 static gen_helper_gvec_3
* const fns
[4] = {
2631 NULL
, NULL
, gen_helper_sve_compact_s
, gen_helper_sve_compact_d
2633 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
2636 /* Call the helper that computes the ARM LastActiveElement pseudocode
2637 * function, scaled by the element size. This includes the not found
2638 * indication; e.g. not found for esz=3 is -8.
2640 static void find_last_active(DisasContext
*s
, TCGv_i32 ret
, int esz
, int pg
)
2642 /* Predicate sizes may be smaller and cannot use simd_desc. We cannot
2643 * round up, as we do elsewhere, because we need the exact size.
2645 TCGv_ptr t_p
= tcg_temp_new_ptr();
2649 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, pred_full_reg_size(s
));
2650 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, esz
);
2652 tcg_gen_addi_ptr(t_p
, cpu_env
, pred_full_reg_offset(s
, pg
));
2653 t_desc
= tcg_const_i32(desc
);
2655 gen_helper_sve_last_active_element(ret
, t_p
, t_desc
);
2657 tcg_temp_free_i32(t_desc
);
2658 tcg_temp_free_ptr(t_p
);
2661 /* Increment LAST to the offset of the next element in the vector,
2662 * wrapping around to 0.
2664 static void incr_last_active(DisasContext
*s
, TCGv_i32 last
, int esz
)
2666 unsigned vsz
= vec_full_reg_size(s
);
2668 tcg_gen_addi_i32(last
, last
, 1 << esz
);
2669 if (is_power_of_2(vsz
)) {
2670 tcg_gen_andi_i32(last
, last
, vsz
- 1);
2672 TCGv_i32 max
= tcg_const_i32(vsz
);
2673 TCGv_i32 zero
= tcg_const_i32(0);
2674 tcg_gen_movcond_i32(TCG_COND_GEU
, last
, last
, max
, zero
, last
);
2675 tcg_temp_free_i32(max
);
2676 tcg_temp_free_i32(zero
);
2680 /* If LAST < 0, set LAST to the offset of the last element in the vector. */
2681 static void wrap_last_active(DisasContext
*s
, TCGv_i32 last
, int esz
)
2683 unsigned vsz
= vec_full_reg_size(s
);
2685 if (is_power_of_2(vsz
)) {
2686 tcg_gen_andi_i32(last
, last
, vsz
- 1);
2688 TCGv_i32 max
= tcg_const_i32(vsz
- (1 << esz
));
2689 TCGv_i32 zero
= tcg_const_i32(0);
2690 tcg_gen_movcond_i32(TCG_COND_LT
, last
, last
, zero
, max
, last
);
2691 tcg_temp_free_i32(max
);
2692 tcg_temp_free_i32(zero
);
2696 /* Load an unsigned element of ESZ from BASE+OFS. */
2697 static TCGv_i64
load_esz(TCGv_ptr base
, int ofs
, int esz
)
2699 TCGv_i64 r
= tcg_temp_new_i64();
2703 tcg_gen_ld8u_i64(r
, base
, ofs
);
2706 tcg_gen_ld16u_i64(r
, base
, ofs
);
2709 tcg_gen_ld32u_i64(r
, base
, ofs
);
2712 tcg_gen_ld_i64(r
, base
, ofs
);
2715 g_assert_not_reached();
2720 /* Load an unsigned element of ESZ from RM[LAST]. */
2721 static TCGv_i64
load_last_active(DisasContext
*s
, TCGv_i32 last
,
2724 TCGv_ptr p
= tcg_temp_new_ptr();
2727 /* Convert offset into vector into offset into ENV.
2728 * The final adjustment for the vector register base
2729 * is added via constant offset to the load.
2731 #ifdef HOST_WORDS_BIGENDIAN
2732 /* Adjust for element ordering. See vec_reg_offset. */
2734 tcg_gen_xori_i32(last
, last
, 8 - (1 << esz
));
2737 tcg_gen_ext_i32_ptr(p
, last
);
2738 tcg_gen_add_ptr(p
, p
, cpu_env
);
2740 r
= load_esz(p
, vec_full_reg_offset(s
, rm
), esz
);
2741 tcg_temp_free_ptr(p
);
2746 /* Compute CLAST for a Zreg. */
2747 static bool do_clast_vector(DisasContext
*s
, arg_rprr_esz
*a
, bool before
)
2752 unsigned vsz
, esz
= a
->esz
;
2754 if (!sve_access_check(s
)) {
2758 last
= tcg_temp_local_new_i32();
2759 over
= gen_new_label();
2761 find_last_active(s
, last
, esz
, a
->pg
);
2763 /* There is of course no movcond for a 2048-bit vector,
2764 * so we must branch over the actual store.
2766 tcg_gen_brcondi_i32(TCG_COND_LT
, last
, 0, over
);
2769 incr_last_active(s
, last
, esz
);
2772 ele
= load_last_active(s
, last
, a
->rm
, esz
);
2773 tcg_temp_free_i32(last
);
2775 vsz
= vec_full_reg_size(s
);
2776 tcg_gen_gvec_dup_i64(esz
, vec_full_reg_offset(s
, a
->rd
), vsz
, vsz
, ele
);
2777 tcg_temp_free_i64(ele
);
2779 /* If this insn used MOVPRFX, we may need a second move. */
2780 if (a
->rd
!= a
->rn
) {
2781 TCGLabel
*done
= gen_new_label();
2784 gen_set_label(over
);
2785 do_mov_z(s
, a
->rd
, a
->rn
);
2787 gen_set_label(done
);
2789 gen_set_label(over
);
2794 static bool trans_CLASTA_z(DisasContext
*s
, arg_rprr_esz
*a
)
2796 return do_clast_vector(s
, a
, false);
2799 static bool trans_CLASTB_z(DisasContext
*s
, arg_rprr_esz
*a
)
2801 return do_clast_vector(s
, a
, true);
2804 /* Compute CLAST for a scalar. */
2805 static void do_clast_scalar(DisasContext
*s
, int esz
, int pg
, int rm
,
2806 bool before
, TCGv_i64 reg_val
)
2808 TCGv_i32 last
= tcg_temp_new_i32();
2809 TCGv_i64 ele
, cmp
, zero
;
2811 find_last_active(s
, last
, esz
, pg
);
2813 /* Extend the original value of last prior to incrementing. */
2814 cmp
= tcg_temp_new_i64();
2815 tcg_gen_ext_i32_i64(cmp
, last
);
2818 incr_last_active(s
, last
, esz
);
2821 /* The conceit here is that while last < 0 indicates not found, after
2822 * adjusting for cpu_env->vfp.zregs[rm], it is still a valid address
2823 * from which we can load garbage. We then discard the garbage with
2824 * a conditional move.
2826 ele
= load_last_active(s
, last
, rm
, esz
);
2827 tcg_temp_free_i32(last
);
2829 zero
= tcg_const_i64(0);
2830 tcg_gen_movcond_i64(TCG_COND_GE
, reg_val
, cmp
, zero
, ele
, reg_val
);
2832 tcg_temp_free_i64(zero
);
2833 tcg_temp_free_i64(cmp
);
2834 tcg_temp_free_i64(ele
);
2837 /* Compute CLAST for a Vreg. */
2838 static bool do_clast_fp(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2840 if (sve_access_check(s
)) {
2842 int ofs
= vec_reg_offset(s
, a
->rd
, 0, esz
);
2843 TCGv_i64 reg
= load_esz(cpu_env
, ofs
, esz
);
2845 do_clast_scalar(s
, esz
, a
->pg
, a
->rn
, before
, reg
);
2846 write_fp_dreg(s
, a
->rd
, reg
);
2847 tcg_temp_free_i64(reg
);
2852 static bool trans_CLASTA_v(DisasContext
*s
, arg_rpr_esz
*a
)
2854 return do_clast_fp(s
, a
, false);
2857 static bool trans_CLASTB_v(DisasContext
*s
, arg_rpr_esz
*a
)
2859 return do_clast_fp(s
, a
, true);
2862 /* Compute CLAST for a Xreg. */
2863 static bool do_clast_general(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2867 if (!sve_access_check(s
)) {
2871 reg
= cpu_reg(s
, a
->rd
);
2874 tcg_gen_ext8u_i64(reg
, reg
);
2877 tcg_gen_ext16u_i64(reg
, reg
);
2880 tcg_gen_ext32u_i64(reg
, reg
);
2885 g_assert_not_reached();
2888 do_clast_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
, reg
);
2892 static bool trans_CLASTA_r(DisasContext
*s
, arg_rpr_esz
*a
)
2894 return do_clast_general(s
, a
, false);
2897 static bool trans_CLASTB_r(DisasContext
*s
, arg_rpr_esz
*a
)
2899 return do_clast_general(s
, a
, true);
2902 /* Compute LAST for a scalar. */
2903 static TCGv_i64
do_last_scalar(DisasContext
*s
, int esz
,
2904 int pg
, int rm
, bool before
)
2906 TCGv_i32 last
= tcg_temp_new_i32();
2909 find_last_active(s
, last
, esz
, pg
);
2911 wrap_last_active(s
, last
, esz
);
2913 incr_last_active(s
, last
, esz
);
2916 ret
= load_last_active(s
, last
, rm
, esz
);
2917 tcg_temp_free_i32(last
);
2921 /* Compute LAST for a Vreg. */
2922 static bool do_last_fp(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2924 if (sve_access_check(s
)) {
2925 TCGv_i64 val
= do_last_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
);
2926 write_fp_dreg(s
, a
->rd
, val
);
2927 tcg_temp_free_i64(val
);
2932 static bool trans_LASTA_v(DisasContext
*s
, arg_rpr_esz
*a
)
2934 return do_last_fp(s
, a
, false);
2937 static bool trans_LASTB_v(DisasContext
*s
, arg_rpr_esz
*a
)
2939 return do_last_fp(s
, a
, true);
2942 /* Compute LAST for a Xreg. */
2943 static bool do_last_general(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2945 if (sve_access_check(s
)) {
2946 TCGv_i64 val
= do_last_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
);
2947 tcg_gen_mov_i64(cpu_reg(s
, a
->rd
), val
);
2948 tcg_temp_free_i64(val
);
2953 static bool trans_LASTA_r(DisasContext
*s
, arg_rpr_esz
*a
)
2955 return do_last_general(s
, a
, false);
2958 static bool trans_LASTB_r(DisasContext
*s
, arg_rpr_esz
*a
)
2960 return do_last_general(s
, a
, true);
2963 static bool trans_CPY_m_r(DisasContext
*s
, arg_rpr_esz
*a
)
2965 if (sve_access_check(s
)) {
2966 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rd
, a
->pg
, cpu_reg_sp(s
, a
->rn
));
2971 static bool trans_CPY_m_v(DisasContext
*s
, arg_rpr_esz
*a
)
2973 if (sve_access_check(s
)) {
2974 int ofs
= vec_reg_offset(s
, a
->rn
, 0, a
->esz
);
2975 TCGv_i64 t
= load_esz(cpu_env
, ofs
, a
->esz
);
2976 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rd
, a
->pg
, t
);
2977 tcg_temp_free_i64(t
);
2982 static bool trans_REVB(DisasContext
*s
, arg_rpr_esz
*a
)
2984 static gen_helper_gvec_3
* const fns
[4] = {
2986 gen_helper_sve_revb_h
,
2987 gen_helper_sve_revb_s
,
2988 gen_helper_sve_revb_d
,
2990 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
2993 static bool trans_REVH(DisasContext
*s
, arg_rpr_esz
*a
)
2995 static gen_helper_gvec_3
* const fns
[4] = {
2998 gen_helper_sve_revh_s
,
2999 gen_helper_sve_revh_d
,
3001 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
3004 static bool trans_REVW(DisasContext
*s
, arg_rpr_esz
*a
)
3006 return do_zpz_ool(s
, a
, a
->esz
== 3 ? gen_helper_sve_revw_d
: NULL
);
3009 static bool trans_RBIT(DisasContext
*s
, arg_rpr_esz
*a
)
3011 static gen_helper_gvec_3
* const fns
[4] = {
3012 gen_helper_sve_rbit_b
,
3013 gen_helper_sve_rbit_h
,
3014 gen_helper_sve_rbit_s
,
3015 gen_helper_sve_rbit_d
,
3017 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
3020 static bool trans_SPLICE(DisasContext
*s
, arg_rprr_esz
*a
)
3022 if (sve_access_check(s
)) {
3023 gen_gvec_ool_zzzp(s
, gen_helper_sve_splice
,
3024 a
->rd
, a
->rn
, a
->rm
, a
->pg
, a
->esz
);
3029 static bool trans_SPLICE_sve2(DisasContext
*s
, arg_rpr_esz
*a
)
3031 if (!dc_isar_feature(aa64_sve2
, s
)) {
3034 if (sve_access_check(s
)) {
3035 gen_gvec_ool_zzzp(s
, gen_helper_sve_splice
,
3036 a
->rd
, a
->rn
, (a
->rn
+ 1) % 32, a
->pg
, a
->esz
);
3042 *** SVE Integer Compare - Vectors Group
3045 static bool do_ppzz_flags(DisasContext
*s
, arg_rprr_esz
*a
,
3046 gen_helper_gvec_flags_4
*gen_fn
)
3048 TCGv_ptr pd
, zn
, zm
, pg
;
3052 if (gen_fn
== NULL
) {
3055 if (!sve_access_check(s
)) {
3059 vsz
= vec_full_reg_size(s
);
3060 t
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
3061 pd
= tcg_temp_new_ptr();
3062 zn
= tcg_temp_new_ptr();
3063 zm
= tcg_temp_new_ptr();
3064 pg
= tcg_temp_new_ptr();
3066 tcg_gen_addi_ptr(pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3067 tcg_gen_addi_ptr(zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
3068 tcg_gen_addi_ptr(zm
, cpu_env
, vec_full_reg_offset(s
, a
->rm
));
3069 tcg_gen_addi_ptr(pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3071 gen_fn(t
, pd
, zn
, zm
, pg
, t
);
3073 tcg_temp_free_ptr(pd
);
3074 tcg_temp_free_ptr(zn
);
3075 tcg_temp_free_ptr(zm
);
3076 tcg_temp_free_ptr(pg
);
3080 tcg_temp_free_i32(t
);
3084 #define DO_PPZZ(NAME, name) \
3085 static bool trans_##NAME##_ppzz(DisasContext *s, arg_rprr_esz *a) \
3087 static gen_helper_gvec_flags_4 * const fns[4] = { \
3088 gen_helper_sve_##name##_ppzz_b, gen_helper_sve_##name##_ppzz_h, \
3089 gen_helper_sve_##name##_ppzz_s, gen_helper_sve_##name##_ppzz_d, \
3091 return do_ppzz_flags(s, a, fns[a->esz]); \
3094 DO_PPZZ(CMPEQ
, cmpeq
)
3095 DO_PPZZ(CMPNE
, cmpne
)
3096 DO_PPZZ(CMPGT
, cmpgt
)
3097 DO_PPZZ(CMPGE
, cmpge
)
3098 DO_PPZZ(CMPHI
, cmphi
)
3099 DO_PPZZ(CMPHS
, cmphs
)
3103 #define DO_PPZW(NAME, name) \
3104 static bool trans_##NAME##_ppzw(DisasContext *s, arg_rprr_esz *a) \
3106 static gen_helper_gvec_flags_4 * const fns[4] = { \
3107 gen_helper_sve_##name##_ppzw_b, gen_helper_sve_##name##_ppzw_h, \
3108 gen_helper_sve_##name##_ppzw_s, NULL \
3110 return do_ppzz_flags(s, a, fns[a->esz]); \
3113 DO_PPZW(CMPEQ
, cmpeq
)
3114 DO_PPZW(CMPNE
, cmpne
)
3115 DO_PPZW(CMPGT
, cmpgt
)
3116 DO_PPZW(CMPGE
, cmpge
)
3117 DO_PPZW(CMPHI
, cmphi
)
3118 DO_PPZW(CMPHS
, cmphs
)
3119 DO_PPZW(CMPLT
, cmplt
)
3120 DO_PPZW(CMPLE
, cmple
)
3121 DO_PPZW(CMPLO
, cmplo
)
3122 DO_PPZW(CMPLS
, cmpls
)
3127 *** SVE Integer Compare - Immediate Groups
3130 static bool do_ppzi_flags(DisasContext
*s
, arg_rpri_esz
*a
,
3131 gen_helper_gvec_flags_3
*gen_fn
)
3133 TCGv_ptr pd
, zn
, pg
;
3137 if (gen_fn
== NULL
) {
3140 if (!sve_access_check(s
)) {
3144 vsz
= vec_full_reg_size(s
);
3145 t
= tcg_const_i32(simd_desc(vsz
, vsz
, a
->imm
));
3146 pd
= tcg_temp_new_ptr();
3147 zn
= tcg_temp_new_ptr();
3148 pg
= tcg_temp_new_ptr();
3150 tcg_gen_addi_ptr(pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3151 tcg_gen_addi_ptr(zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
3152 tcg_gen_addi_ptr(pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3154 gen_fn(t
, pd
, zn
, pg
, t
);
3156 tcg_temp_free_ptr(pd
);
3157 tcg_temp_free_ptr(zn
);
3158 tcg_temp_free_ptr(pg
);
3162 tcg_temp_free_i32(t
);
3166 #define DO_PPZI(NAME, name) \
3167 static bool trans_##NAME##_ppzi(DisasContext *s, arg_rpri_esz *a) \
3169 static gen_helper_gvec_flags_3 * const fns[4] = { \
3170 gen_helper_sve_##name##_ppzi_b, gen_helper_sve_##name##_ppzi_h, \
3171 gen_helper_sve_##name##_ppzi_s, gen_helper_sve_##name##_ppzi_d, \
3173 return do_ppzi_flags(s, a, fns[a->esz]); \
3176 DO_PPZI(CMPEQ
, cmpeq
)
3177 DO_PPZI(CMPNE
, cmpne
)
3178 DO_PPZI(CMPGT
, cmpgt
)
3179 DO_PPZI(CMPGE
, cmpge
)
3180 DO_PPZI(CMPHI
, cmphi
)
3181 DO_PPZI(CMPHS
, cmphs
)
3182 DO_PPZI(CMPLT
, cmplt
)
3183 DO_PPZI(CMPLE
, cmple
)
3184 DO_PPZI(CMPLO
, cmplo
)
3185 DO_PPZI(CMPLS
, cmpls
)
3190 *** SVE Partition Break Group
3193 static bool do_brk3(DisasContext
*s
, arg_rprr_s
*a
,
3194 gen_helper_gvec_4
*fn
, gen_helper_gvec_flags_4
*fn_s
)
3196 if (!sve_access_check(s
)) {
3200 unsigned vsz
= pred_full_reg_size(s
);
3202 /* Predicate sizes may be smaller and cannot use simd_desc. */
3203 TCGv_ptr d
= tcg_temp_new_ptr();
3204 TCGv_ptr n
= tcg_temp_new_ptr();
3205 TCGv_ptr m
= tcg_temp_new_ptr();
3206 TCGv_ptr g
= tcg_temp_new_ptr();
3207 TCGv_i32 t
= tcg_const_i32(FIELD_DP32(0, PREDDESC
, OPRSZ
, vsz
));
3209 tcg_gen_addi_ptr(d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3210 tcg_gen_addi_ptr(n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
3211 tcg_gen_addi_ptr(m
, cpu_env
, pred_full_reg_offset(s
, a
->rm
));
3212 tcg_gen_addi_ptr(g
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3215 fn_s(t
, d
, n
, m
, g
, t
);
3220 tcg_temp_free_ptr(d
);
3221 tcg_temp_free_ptr(n
);
3222 tcg_temp_free_ptr(m
);
3223 tcg_temp_free_ptr(g
);
3224 tcg_temp_free_i32(t
);
3228 static bool do_brk2(DisasContext
*s
, arg_rpr_s
*a
,
3229 gen_helper_gvec_3
*fn
, gen_helper_gvec_flags_3
*fn_s
)
3231 if (!sve_access_check(s
)) {
3235 unsigned vsz
= pred_full_reg_size(s
);
3237 /* Predicate sizes may be smaller and cannot use simd_desc. */
3238 TCGv_ptr d
= tcg_temp_new_ptr();
3239 TCGv_ptr n
= tcg_temp_new_ptr();
3240 TCGv_ptr g
= tcg_temp_new_ptr();
3241 TCGv_i32 t
= tcg_const_i32(FIELD_DP32(0, PREDDESC
, OPRSZ
, vsz
));
3243 tcg_gen_addi_ptr(d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3244 tcg_gen_addi_ptr(n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
3245 tcg_gen_addi_ptr(g
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3248 fn_s(t
, d
, n
, g
, t
);
3253 tcg_temp_free_ptr(d
);
3254 tcg_temp_free_ptr(n
);
3255 tcg_temp_free_ptr(g
);
3256 tcg_temp_free_i32(t
);
3260 static bool trans_BRKPA(DisasContext
*s
, arg_rprr_s
*a
)
3262 return do_brk3(s
, a
, gen_helper_sve_brkpa
, gen_helper_sve_brkpas
);
3265 static bool trans_BRKPB(DisasContext
*s
, arg_rprr_s
*a
)
3267 return do_brk3(s
, a
, gen_helper_sve_brkpb
, gen_helper_sve_brkpbs
);
3270 static bool trans_BRKA_m(DisasContext
*s
, arg_rpr_s
*a
)
3272 return do_brk2(s
, a
, gen_helper_sve_brka_m
, gen_helper_sve_brkas_m
);
3275 static bool trans_BRKB_m(DisasContext
*s
, arg_rpr_s
*a
)
3277 return do_brk2(s
, a
, gen_helper_sve_brkb_m
, gen_helper_sve_brkbs_m
);
3280 static bool trans_BRKA_z(DisasContext
*s
, arg_rpr_s
*a
)
3282 return do_brk2(s
, a
, gen_helper_sve_brka_z
, gen_helper_sve_brkas_z
);
3285 static bool trans_BRKB_z(DisasContext
*s
, arg_rpr_s
*a
)
3287 return do_brk2(s
, a
, gen_helper_sve_brkb_z
, gen_helper_sve_brkbs_z
);
3290 static bool trans_BRKN(DisasContext
*s
, arg_rpr_s
*a
)
3292 return do_brk2(s
, a
, gen_helper_sve_brkn
, gen_helper_sve_brkns
);
3296 *** SVE Predicate Count Group
3299 static void do_cntp(DisasContext
*s
, TCGv_i64 val
, int esz
, int pn
, int pg
)
3301 unsigned psz
= pred_full_reg_size(s
);
3306 tcg_gen_ld_i64(val
, cpu_env
, pred_full_reg_offset(s
, pn
));
3308 TCGv_i64 g
= tcg_temp_new_i64();
3309 tcg_gen_ld_i64(g
, cpu_env
, pred_full_reg_offset(s
, pg
));
3310 tcg_gen_and_i64(val
, val
, g
);
3311 tcg_temp_free_i64(g
);
3314 /* Reduce the pred_esz_masks value simply to reduce the
3315 * size of the code generated here.
3317 psz_mask
= MAKE_64BIT_MASK(0, psz
* 8);
3318 tcg_gen_andi_i64(val
, val
, pred_esz_masks
[esz
] & psz_mask
);
3320 tcg_gen_ctpop_i64(val
, val
);
3322 TCGv_ptr t_pn
= tcg_temp_new_ptr();
3323 TCGv_ptr t_pg
= tcg_temp_new_ptr();
3327 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, psz
);
3328 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, esz
);
3330 tcg_gen_addi_ptr(t_pn
, cpu_env
, pred_full_reg_offset(s
, pn
));
3331 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
3332 t_desc
= tcg_const_i32(desc
);
3334 gen_helper_sve_cntp(val
, t_pn
, t_pg
, t_desc
);
3335 tcg_temp_free_ptr(t_pn
);
3336 tcg_temp_free_ptr(t_pg
);
3337 tcg_temp_free_i32(t_desc
);
3341 static bool trans_CNTP(DisasContext
*s
, arg_CNTP
*a
)
3343 if (sve_access_check(s
)) {
3344 do_cntp(s
, cpu_reg(s
, a
->rd
), a
->esz
, a
->rn
, a
->pg
);
3349 static bool trans_INCDECP_r(DisasContext
*s
, arg_incdec_pred
*a
)
3351 if (sve_access_check(s
)) {
3352 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3353 TCGv_i64 val
= tcg_temp_new_i64();
3355 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3357 tcg_gen_sub_i64(reg
, reg
, val
);
3359 tcg_gen_add_i64(reg
, reg
, val
);
3361 tcg_temp_free_i64(val
);
3366 static bool trans_INCDECP_z(DisasContext
*s
, arg_incdec2_pred
*a
)
3371 if (sve_access_check(s
)) {
3372 unsigned vsz
= vec_full_reg_size(s
);
3373 TCGv_i64 val
= tcg_temp_new_i64();
3374 GVecGen2sFn
*gvec_fn
= a
->d
? tcg_gen_gvec_subs
: tcg_gen_gvec_adds
;
3376 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3377 gvec_fn(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3378 vec_full_reg_offset(s
, a
->rn
), val
, vsz
, vsz
);
3383 static bool trans_SINCDECP_r_32(DisasContext
*s
, arg_incdec_pred
*a
)
3385 if (sve_access_check(s
)) {
3386 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3387 TCGv_i64 val
= tcg_temp_new_i64();
3389 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3390 do_sat_addsub_32(reg
, val
, a
->u
, a
->d
);
3395 static bool trans_SINCDECP_r_64(DisasContext
*s
, arg_incdec_pred
*a
)
3397 if (sve_access_check(s
)) {
3398 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3399 TCGv_i64 val
= tcg_temp_new_i64();
3401 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3402 do_sat_addsub_64(reg
, val
, a
->u
, a
->d
);
3407 static bool trans_SINCDECP_z(DisasContext
*s
, arg_incdec2_pred
*a
)
3412 if (sve_access_check(s
)) {
3413 TCGv_i64 val
= tcg_temp_new_i64();
3414 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3415 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
, val
, a
->u
, a
->d
);
3421 *** SVE Integer Compare Scalars Group
3424 static bool trans_CTERM(DisasContext
*s
, arg_CTERM
*a
)
3426 if (!sve_access_check(s
)) {
3430 TCGCond cond
= (a
->ne
? TCG_COND_NE
: TCG_COND_EQ
);
3431 TCGv_i64 rn
= read_cpu_reg(s
, a
->rn
, a
->sf
);
3432 TCGv_i64 rm
= read_cpu_reg(s
, a
->rm
, a
->sf
);
3433 TCGv_i64 cmp
= tcg_temp_new_i64();
3435 tcg_gen_setcond_i64(cond
, cmp
, rn
, rm
);
3436 tcg_gen_extrl_i64_i32(cpu_NF
, cmp
);
3437 tcg_temp_free_i64(cmp
);
3439 /* VF = !NF & !CF. */
3440 tcg_gen_xori_i32(cpu_VF
, cpu_NF
, 1);
3441 tcg_gen_andc_i32(cpu_VF
, cpu_VF
, cpu_CF
);
3443 /* Both NF and VF actually look at bit 31. */
3444 tcg_gen_neg_i32(cpu_NF
, cpu_NF
);
3445 tcg_gen_neg_i32(cpu_VF
, cpu_VF
);
3449 static bool trans_WHILE(DisasContext
*s
, arg_WHILE
*a
)
3451 TCGv_i64 op0
, op1
, t0
, t1
, tmax
;
3454 unsigned vsz
= vec_full_reg_size(s
);
3458 /* Note that GE/HS has a->eq == 0 and GT/HI has a->eq == 1. */
3459 bool eq
= a
->eq
== a
->lt
;
3461 /* The greater-than conditions are all SVE2. */
3462 if (!a
->lt
&& !dc_isar_feature(aa64_sve2
, s
)) {
3465 if (!sve_access_check(s
)) {
3469 op0
= read_cpu_reg(s
, a
->rn
, 1);
3470 op1
= read_cpu_reg(s
, a
->rm
, 1);
3474 tcg_gen_ext32u_i64(op0
, op0
);
3475 tcg_gen_ext32u_i64(op1
, op1
);
3477 tcg_gen_ext32s_i64(op0
, op0
);
3478 tcg_gen_ext32s_i64(op1
, op1
);
3482 /* For the helper, compress the different conditions into a computation
3483 * of how many iterations for which the condition is true.
3485 t0
= tcg_temp_new_i64();
3486 t1
= tcg_temp_new_i64();
3489 tcg_gen_sub_i64(t0
, op1
, op0
);
3491 maxval
= a
->sf
? UINT64_MAX
: UINT32_MAX
;
3492 cond
= eq
? TCG_COND_LEU
: TCG_COND_LTU
;
3494 maxval
= a
->sf
? INT64_MAX
: INT32_MAX
;
3495 cond
= eq
? TCG_COND_LE
: TCG_COND_LT
;
3498 tcg_gen_sub_i64(t0
, op0
, op1
);
3501 cond
= eq
? TCG_COND_GEU
: TCG_COND_GTU
;
3503 maxval
= a
->sf
? INT64_MIN
: INT32_MIN
;
3504 cond
= eq
? TCG_COND_GE
: TCG_COND_GT
;
3508 tmax
= tcg_const_i64(vsz
>> a
->esz
);
3510 /* Equality means one more iteration. */
3511 tcg_gen_addi_i64(t0
, t0
, 1);
3514 * For the less-than while, if op1 is maxval (and the only time
3515 * the addition above could overflow), then we produce an all-true
3516 * predicate by setting the count to the vector length. This is
3517 * because the pseudocode is described as an increment + compare
3518 * loop, and the maximum integer would always compare true.
3519 * Similarly, the greater-than while has the same issue with the
3520 * minimum integer due to the decrement + compare loop.
3522 tcg_gen_movi_i64(t1
, maxval
);
3523 tcg_gen_movcond_i64(TCG_COND_EQ
, t0
, op1
, t1
, tmax
, t0
);
3526 /* Bound to the maximum. */
3527 tcg_gen_umin_i64(t0
, t0
, tmax
);
3528 tcg_temp_free_i64(tmax
);
3530 /* Set the count to zero if the condition is false. */
3531 tcg_gen_movi_i64(t1
, 0);
3532 tcg_gen_movcond_i64(cond
, t0
, op0
, op1
, t0
, t1
);
3533 tcg_temp_free_i64(t1
);
3535 /* Since we're bounded, pass as a 32-bit type. */
3536 t2
= tcg_temp_new_i32();
3537 tcg_gen_extrl_i64_i32(t2
, t0
);
3538 tcg_temp_free_i64(t0
);
3540 /* Scale elements to bits. */
3541 tcg_gen_shli_i32(t2
, t2
, a
->esz
);
3543 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
/ 8);
3544 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
3545 t3
= tcg_const_i32(desc
);
3547 ptr
= tcg_temp_new_ptr();
3548 tcg_gen_addi_ptr(ptr
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3551 gen_helper_sve_whilel(t2
, ptr
, t2
, t3
);
3553 gen_helper_sve_whileg(t2
, ptr
, t2
, t3
);
3557 tcg_temp_free_ptr(ptr
);
3558 tcg_temp_free_i32(t2
);
3559 tcg_temp_free_i32(t3
);
3563 static bool trans_WHILE_ptr(DisasContext
*s
, arg_WHILE_ptr
*a
)
3565 TCGv_i64 op0
, op1
, diff
, t1
, tmax
;
3568 unsigned vsz
= vec_full_reg_size(s
);
3571 if (!dc_isar_feature(aa64_sve2
, s
)) {
3574 if (!sve_access_check(s
)) {
3578 op0
= read_cpu_reg(s
, a
->rn
, 1);
3579 op1
= read_cpu_reg(s
, a
->rm
, 1);
3581 tmax
= tcg_const_i64(vsz
);
3582 diff
= tcg_temp_new_i64();
3586 /* diff = abs(op1 - op0), noting that op0/1 are unsigned. */
3587 t1
= tcg_temp_new_i64();
3588 tcg_gen_sub_i64(diff
, op0
, op1
);
3589 tcg_gen_sub_i64(t1
, op1
, op0
);
3590 tcg_gen_movcond_i64(TCG_COND_GEU
, diff
, op0
, op1
, diff
, t1
);
3591 tcg_temp_free_i64(t1
);
3592 /* Round down to a multiple of ESIZE. */
3593 tcg_gen_andi_i64(diff
, diff
, -1 << a
->esz
);
3594 /* If op1 == op0, diff == 0, and the condition is always true. */
3595 tcg_gen_movcond_i64(TCG_COND_EQ
, diff
, op0
, op1
, tmax
, diff
);
3598 tcg_gen_sub_i64(diff
, op1
, op0
);
3599 /* Round down to a multiple of ESIZE. */
3600 tcg_gen_andi_i64(diff
, diff
, -1 << a
->esz
);
3601 /* If op0 >= op1, diff <= 0, the condition is always true. */
3602 tcg_gen_movcond_i64(TCG_COND_GEU
, diff
, op0
, op1
, tmax
, diff
);
3605 /* Bound to the maximum. */
3606 tcg_gen_umin_i64(diff
, diff
, tmax
);
3607 tcg_temp_free_i64(tmax
);
3609 /* Since we're bounded, pass as a 32-bit type. */
3610 t2
= tcg_temp_new_i32();
3611 tcg_gen_extrl_i64_i32(t2
, diff
);
3612 tcg_temp_free_i64(diff
);
3614 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
/ 8);
3615 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
3616 t3
= tcg_const_i32(desc
);
3618 ptr
= tcg_temp_new_ptr();
3619 tcg_gen_addi_ptr(ptr
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3621 gen_helper_sve_whilel(t2
, ptr
, t2
, t3
);
3624 tcg_temp_free_ptr(ptr
);
3625 tcg_temp_free_i32(t2
);
3626 tcg_temp_free_i32(t3
);
3631 *** SVE Integer Wide Immediate - Unpredicated Group
3634 static bool trans_FDUP(DisasContext
*s
, arg_FDUP
*a
)
3639 if (sve_access_check(s
)) {
3640 unsigned vsz
= vec_full_reg_size(s
);
3641 int dofs
= vec_full_reg_offset(s
, a
->rd
);
3644 /* Decode the VFP immediate. */
3645 imm
= vfp_expand_imm(a
->esz
, a
->imm
);
3646 tcg_gen_gvec_dup_imm(a
->esz
, dofs
, vsz
, vsz
, imm
);
3651 static bool trans_DUP_i(DisasContext
*s
, arg_DUP_i
*a
)
3653 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3656 if (sve_access_check(s
)) {
3657 unsigned vsz
= vec_full_reg_size(s
);
3658 int dofs
= vec_full_reg_offset(s
, a
->rd
);
3660 tcg_gen_gvec_dup_imm(a
->esz
, dofs
, vsz
, vsz
, a
->imm
);
3665 static bool trans_ADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3667 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3670 if (sve_access_check(s
)) {
3671 unsigned vsz
= vec_full_reg_size(s
);
3672 tcg_gen_gvec_addi(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3673 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
3678 static bool trans_SUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3681 return trans_ADD_zzi(s
, a
);
3684 static bool trans_SUBR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3686 static const TCGOpcode vecop_list
[] = { INDEX_op_sub_vec
, 0 };
3687 static const GVecGen2s op
[4] = {
3688 { .fni8
= tcg_gen_vec_sub8_i64
,
3689 .fniv
= tcg_gen_sub_vec
,
3690 .fno
= gen_helper_sve_subri_b
,
3691 .opt_opc
= vecop_list
,
3693 .scalar_first
= true },
3694 { .fni8
= tcg_gen_vec_sub16_i64
,
3695 .fniv
= tcg_gen_sub_vec
,
3696 .fno
= gen_helper_sve_subri_h
,
3697 .opt_opc
= vecop_list
,
3699 .scalar_first
= true },
3700 { .fni4
= tcg_gen_sub_i32
,
3701 .fniv
= tcg_gen_sub_vec
,
3702 .fno
= gen_helper_sve_subri_s
,
3703 .opt_opc
= vecop_list
,
3705 .scalar_first
= true },
3706 { .fni8
= tcg_gen_sub_i64
,
3707 .fniv
= tcg_gen_sub_vec
,
3708 .fno
= gen_helper_sve_subri_d
,
3709 .opt_opc
= vecop_list
,
3710 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
3712 .scalar_first
= true }
3715 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3718 if (sve_access_check(s
)) {
3719 unsigned vsz
= vec_full_reg_size(s
);
3720 TCGv_i64 c
= tcg_const_i64(a
->imm
);
3721 tcg_gen_gvec_2s(vec_full_reg_offset(s
, a
->rd
),
3722 vec_full_reg_offset(s
, a
->rn
),
3723 vsz
, vsz
, c
, &op
[a
->esz
]);
3724 tcg_temp_free_i64(c
);
3729 static bool trans_MUL_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3731 if (sve_access_check(s
)) {
3732 unsigned vsz
= vec_full_reg_size(s
);
3733 tcg_gen_gvec_muli(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3734 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
3739 static bool do_zzi_sat(DisasContext
*s
, arg_rri_esz
*a
, bool u
, bool d
)
3741 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3744 if (sve_access_check(s
)) {
3745 TCGv_i64 val
= tcg_const_i64(a
->imm
);
3746 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
, val
, u
, d
);
3747 tcg_temp_free_i64(val
);
3752 static bool trans_SQADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3754 return do_zzi_sat(s
, a
, false, false);
3757 static bool trans_UQADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3759 return do_zzi_sat(s
, a
, true, false);
3762 static bool trans_SQSUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3764 return do_zzi_sat(s
, a
, false, true);
3767 static bool trans_UQSUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3769 return do_zzi_sat(s
, a
, true, true);
3772 static bool do_zzi_ool(DisasContext
*s
, arg_rri_esz
*a
, gen_helper_gvec_2i
*fn
)
3774 if (sve_access_check(s
)) {
3775 unsigned vsz
= vec_full_reg_size(s
);
3776 TCGv_i64 c
= tcg_const_i64(a
->imm
);
3778 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s
, a
->rd
),
3779 vec_full_reg_offset(s
, a
->rn
),
3780 c
, vsz
, vsz
, 0, fn
);
3781 tcg_temp_free_i64(c
);
3786 #define DO_ZZI(NAME, name) \
3787 static bool trans_##NAME##_zzi(DisasContext *s, arg_rri_esz *a) \
3789 static gen_helper_gvec_2i * const fns[4] = { \
3790 gen_helper_sve_##name##i_b, gen_helper_sve_##name##i_h, \
3791 gen_helper_sve_##name##i_s, gen_helper_sve_##name##i_d, \
3793 return do_zzi_ool(s, a, fns[a->esz]); \
3803 static bool trans_DOT_zzzz(DisasContext
*s
, arg_DOT_zzzz
*a
)
3805 static gen_helper_gvec_4
* const fns
[2][2] = {
3806 { gen_helper_gvec_sdot_b
, gen_helper_gvec_sdot_h
},
3807 { gen_helper_gvec_udot_b
, gen_helper_gvec_udot_h
}
3810 if (sve_access_check(s
)) {
3811 gen_gvec_ool_zzzz(s
, fns
[a
->u
][a
->sz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, 0);
3817 * SVE Multiply - Indexed
3820 static bool do_zzxz_ool(DisasContext
*s
, arg_rrxr_esz
*a
,
3821 gen_helper_gvec_4
*fn
)
3826 if (sve_access_check(s
)) {
3827 gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->index
);
3832 #define DO_RRXR(NAME, FUNC) \
3833 static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
3834 { return do_zzxz_ool(s, a, FUNC); }
3836 DO_RRXR(trans_SDOT_zzxw_s
, gen_helper_gvec_sdot_idx_b
)
3837 DO_RRXR(trans_SDOT_zzxw_d
, gen_helper_gvec_sdot_idx_h
)
3838 DO_RRXR(trans_UDOT_zzxw_s
, gen_helper_gvec_udot_idx_b
)
3839 DO_RRXR(trans_UDOT_zzxw_d
, gen_helper_gvec_udot_idx_h
)
3841 static bool trans_SUDOT_zzxw_s(DisasContext
*s
, arg_rrxr_esz
*a
)
3843 if (!dc_isar_feature(aa64_sve_i8mm
, s
)) {
3846 return do_zzxz_ool(s
, a
, gen_helper_gvec_sudot_idx_b
);
3849 static bool trans_USDOT_zzxw_s(DisasContext
*s
, arg_rrxr_esz
*a
)
3851 if (!dc_isar_feature(aa64_sve_i8mm
, s
)) {
3854 return do_zzxz_ool(s
, a
, gen_helper_gvec_usdot_idx_b
);
3859 static bool do_sve2_zzz_data(DisasContext
*s
, int rd
, int rn
, int rm
, int data
,
3860 gen_helper_gvec_3
*fn
)
3862 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
3865 if (sve_access_check(s
)) {
3866 unsigned vsz
= vec_full_reg_size(s
);
3867 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
3868 vec_full_reg_offset(s
, rn
),
3869 vec_full_reg_offset(s
, rm
),
3870 vsz
, vsz
, data
, fn
);
3875 #define DO_SVE2_RRX(NAME, FUNC) \
3876 static bool NAME(DisasContext *s, arg_rrx_esz *a) \
3877 { return do_sve2_zzz_data(s, a->rd, a->rn, a->rm, a->index, FUNC); }
3879 DO_SVE2_RRX(trans_MUL_zzx_h
, gen_helper_gvec_mul_idx_h
)
3880 DO_SVE2_RRX(trans_MUL_zzx_s
, gen_helper_gvec_mul_idx_s
)
3881 DO_SVE2_RRX(trans_MUL_zzx_d
, gen_helper_gvec_mul_idx_d
)
3883 DO_SVE2_RRX(trans_SQDMULH_zzx_h
, gen_helper_sve2_sqdmulh_idx_h
)
3884 DO_SVE2_RRX(trans_SQDMULH_zzx_s
, gen_helper_sve2_sqdmulh_idx_s
)
3885 DO_SVE2_RRX(trans_SQDMULH_zzx_d
, gen_helper_sve2_sqdmulh_idx_d
)
3887 DO_SVE2_RRX(trans_SQRDMULH_zzx_h
, gen_helper_sve2_sqrdmulh_idx_h
)
3888 DO_SVE2_RRX(trans_SQRDMULH_zzx_s
, gen_helper_sve2_sqrdmulh_idx_s
)
3889 DO_SVE2_RRX(trans_SQRDMULH_zzx_d
, gen_helper_sve2_sqrdmulh_idx_d
)
3893 #define DO_SVE2_RRX_TB(NAME, FUNC, TOP) \
3894 static bool NAME(DisasContext *s, arg_rrx_esz *a) \
3896 return do_sve2_zzz_data(s, a->rd, a->rn, a->rm, \
3897 (a->index << 1) | TOP, FUNC); \
3900 DO_SVE2_RRX_TB(trans_SQDMULLB_zzx_s
, gen_helper_sve2_sqdmull_idx_s
, false)
3901 DO_SVE2_RRX_TB(trans_SQDMULLB_zzx_d
, gen_helper_sve2_sqdmull_idx_d
, false)
3902 DO_SVE2_RRX_TB(trans_SQDMULLT_zzx_s
, gen_helper_sve2_sqdmull_idx_s
, true)
3903 DO_SVE2_RRX_TB(trans_SQDMULLT_zzx_d
, gen_helper_sve2_sqdmull_idx_d
, true)
3905 DO_SVE2_RRX_TB(trans_SMULLB_zzx_s
, gen_helper_sve2_smull_idx_s
, false)
3906 DO_SVE2_RRX_TB(trans_SMULLB_zzx_d
, gen_helper_sve2_smull_idx_d
, false)
3907 DO_SVE2_RRX_TB(trans_SMULLT_zzx_s
, gen_helper_sve2_smull_idx_s
, true)
3908 DO_SVE2_RRX_TB(trans_SMULLT_zzx_d
, gen_helper_sve2_smull_idx_d
, true)
3910 DO_SVE2_RRX_TB(trans_UMULLB_zzx_s
, gen_helper_sve2_umull_idx_s
, false)
3911 DO_SVE2_RRX_TB(trans_UMULLB_zzx_d
, gen_helper_sve2_umull_idx_d
, false)
3912 DO_SVE2_RRX_TB(trans_UMULLT_zzx_s
, gen_helper_sve2_umull_idx_s
, true)
3913 DO_SVE2_RRX_TB(trans_UMULLT_zzx_d
, gen_helper_sve2_umull_idx_d
, true)
3915 #undef DO_SVE2_RRX_TB
3917 static bool do_sve2_zzzz_data(DisasContext
*s
, int rd
, int rn
, int rm
, int ra
,
3918 int data
, gen_helper_gvec_4
*fn
)
3920 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
3923 if (sve_access_check(s
)) {
3924 unsigned vsz
= vec_full_reg_size(s
);
3925 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
3926 vec_full_reg_offset(s
, rn
),
3927 vec_full_reg_offset(s
, rm
),
3928 vec_full_reg_offset(s
, ra
),
3929 vsz
, vsz
, data
, fn
);
3934 #define DO_SVE2_RRXR(NAME, FUNC) \
3935 static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
3936 { return do_sve2_zzzz_data(s, a->rd, a->rn, a->rm, a->ra, a->index, FUNC); }
3938 DO_SVE2_RRXR(trans_MLA_zzxz_h
, gen_helper_gvec_mla_idx_h
)
3939 DO_SVE2_RRXR(trans_MLA_zzxz_s
, gen_helper_gvec_mla_idx_s
)
3940 DO_SVE2_RRXR(trans_MLA_zzxz_d
, gen_helper_gvec_mla_idx_d
)
3942 DO_SVE2_RRXR(trans_MLS_zzxz_h
, gen_helper_gvec_mls_idx_h
)
3943 DO_SVE2_RRXR(trans_MLS_zzxz_s
, gen_helper_gvec_mls_idx_s
)
3944 DO_SVE2_RRXR(trans_MLS_zzxz_d
, gen_helper_gvec_mls_idx_d
)
3946 DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_h
, gen_helper_sve2_sqrdmlah_idx_h
)
3947 DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_s
, gen_helper_sve2_sqrdmlah_idx_s
)
3948 DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_d
, gen_helper_sve2_sqrdmlah_idx_d
)
3950 DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_h
, gen_helper_sve2_sqrdmlsh_idx_h
)
3951 DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_s
, gen_helper_sve2_sqrdmlsh_idx_s
)
3952 DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_d
, gen_helper_sve2_sqrdmlsh_idx_d
)
3956 #define DO_SVE2_RRXR_TB(NAME, FUNC, TOP) \
3957 static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
3959 return do_sve2_zzzz_data(s, a->rd, a->rn, a->rm, a->rd, \
3960 (a->index << 1) | TOP, FUNC); \
3963 DO_SVE2_RRXR_TB(trans_SQDMLALB_zzxw_s
, gen_helper_sve2_sqdmlal_idx_s
, false)
3964 DO_SVE2_RRXR_TB(trans_SQDMLALB_zzxw_d
, gen_helper_sve2_sqdmlal_idx_d
, false)
3965 DO_SVE2_RRXR_TB(trans_SQDMLALT_zzxw_s
, gen_helper_sve2_sqdmlal_idx_s
, true)
3966 DO_SVE2_RRXR_TB(trans_SQDMLALT_zzxw_d
, gen_helper_sve2_sqdmlal_idx_d
, true)
3968 DO_SVE2_RRXR_TB(trans_SQDMLSLB_zzxw_s
, gen_helper_sve2_sqdmlsl_idx_s
, false)
3969 DO_SVE2_RRXR_TB(trans_SQDMLSLB_zzxw_d
, gen_helper_sve2_sqdmlsl_idx_d
, false)
3970 DO_SVE2_RRXR_TB(trans_SQDMLSLT_zzxw_s
, gen_helper_sve2_sqdmlsl_idx_s
, true)
3971 DO_SVE2_RRXR_TB(trans_SQDMLSLT_zzxw_d
, gen_helper_sve2_sqdmlsl_idx_d
, true)
3973 DO_SVE2_RRXR_TB(trans_SMLALB_zzxw_s
, gen_helper_sve2_smlal_idx_s
, false)
3974 DO_SVE2_RRXR_TB(trans_SMLALB_zzxw_d
, gen_helper_sve2_smlal_idx_d
, false)
3975 DO_SVE2_RRXR_TB(trans_SMLALT_zzxw_s
, gen_helper_sve2_smlal_idx_s
, true)
3976 DO_SVE2_RRXR_TB(trans_SMLALT_zzxw_d
, gen_helper_sve2_smlal_idx_d
, true)
3978 DO_SVE2_RRXR_TB(trans_UMLALB_zzxw_s
, gen_helper_sve2_umlal_idx_s
, false)
3979 DO_SVE2_RRXR_TB(trans_UMLALB_zzxw_d
, gen_helper_sve2_umlal_idx_d
, false)
3980 DO_SVE2_RRXR_TB(trans_UMLALT_zzxw_s
, gen_helper_sve2_umlal_idx_s
, true)
3981 DO_SVE2_RRXR_TB(trans_UMLALT_zzxw_d
, gen_helper_sve2_umlal_idx_d
, true)
3983 DO_SVE2_RRXR_TB(trans_SMLSLB_zzxw_s
, gen_helper_sve2_smlsl_idx_s
, false)
3984 DO_SVE2_RRXR_TB(trans_SMLSLB_zzxw_d
, gen_helper_sve2_smlsl_idx_d
, false)
3985 DO_SVE2_RRXR_TB(trans_SMLSLT_zzxw_s
, gen_helper_sve2_smlsl_idx_s
, true)
3986 DO_SVE2_RRXR_TB(trans_SMLSLT_zzxw_d
, gen_helper_sve2_smlsl_idx_d
, true)
3988 DO_SVE2_RRXR_TB(trans_UMLSLB_zzxw_s
, gen_helper_sve2_umlsl_idx_s
, false)
3989 DO_SVE2_RRXR_TB(trans_UMLSLB_zzxw_d
, gen_helper_sve2_umlsl_idx_d
, false)
3990 DO_SVE2_RRXR_TB(trans_UMLSLT_zzxw_s
, gen_helper_sve2_umlsl_idx_s
, true)
3991 DO_SVE2_RRXR_TB(trans_UMLSLT_zzxw_d
, gen_helper_sve2_umlsl_idx_d
, true)
3993 #undef DO_SVE2_RRXR_TB
3995 #define DO_SVE2_RRXR_ROT(NAME, FUNC) \
3996 static bool trans_##NAME(DisasContext *s, arg_##NAME *a) \
3998 return do_sve2_zzzz_data(s, a->rd, a->rn, a->rm, a->ra, \
3999 (a->index << 2) | a->rot, FUNC); \
4002 DO_SVE2_RRXR_ROT(CMLA_zzxz_h
, gen_helper_sve2_cmla_idx_h
)
4003 DO_SVE2_RRXR_ROT(CMLA_zzxz_s
, gen_helper_sve2_cmla_idx_s
)
4005 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_h
, gen_helper_sve2_sqrdcmlah_idx_h
)
4006 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_s
, gen_helper_sve2_sqrdcmlah_idx_s
)
4008 DO_SVE2_RRXR_ROT(CDOT_zzxw_s
, gen_helper_sve2_cdot_idx_s
)
4009 DO_SVE2_RRXR_ROT(CDOT_zzxw_d
, gen_helper_sve2_cdot_idx_d
)
4011 #undef DO_SVE2_RRXR_ROT
4014 *** SVE Floating Point Multiply-Add Indexed Group
4017 static bool do_FMLA_zzxz(DisasContext
*s
, arg_rrxr_esz
*a
, bool sub
)
4019 static gen_helper_gvec_4_ptr
* const fns
[3] = {
4020 gen_helper_gvec_fmla_idx_h
,
4021 gen_helper_gvec_fmla_idx_s
,
4022 gen_helper_gvec_fmla_idx_d
,
4025 if (sve_access_check(s
)) {
4026 unsigned vsz
= vec_full_reg_size(s
);
4027 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4028 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4029 vec_full_reg_offset(s
, a
->rn
),
4030 vec_full_reg_offset(s
, a
->rm
),
4031 vec_full_reg_offset(s
, a
->ra
),
4032 status
, vsz
, vsz
, (a
->index
<< 1) | sub
,
4034 tcg_temp_free_ptr(status
);
4039 static bool trans_FMLA_zzxz(DisasContext
*s
, arg_FMLA_zzxz
*a
)
4041 return do_FMLA_zzxz(s
, a
, false);
4044 static bool trans_FMLS_zzxz(DisasContext
*s
, arg_FMLA_zzxz
*a
)
4046 return do_FMLA_zzxz(s
, a
, true);
4050 *** SVE Floating Point Multiply Indexed Group
4053 static bool trans_FMUL_zzx(DisasContext
*s
, arg_FMUL_zzx
*a
)
4055 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4056 gen_helper_gvec_fmul_idx_h
,
4057 gen_helper_gvec_fmul_idx_s
,
4058 gen_helper_gvec_fmul_idx_d
,
4061 if (sve_access_check(s
)) {
4062 unsigned vsz
= vec_full_reg_size(s
);
4063 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4064 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4065 vec_full_reg_offset(s
, a
->rn
),
4066 vec_full_reg_offset(s
, a
->rm
),
4067 status
, vsz
, vsz
, a
->index
, fns
[a
->esz
- 1]);
4068 tcg_temp_free_ptr(status
);
4074 *** SVE Floating Point Fast Reduction Group
4077 typedef void gen_helper_fp_reduce(TCGv_i64
, TCGv_ptr
, TCGv_ptr
,
4078 TCGv_ptr
, TCGv_i32
);
4080 static void do_reduce(DisasContext
*s
, arg_rpr_esz
*a
,
4081 gen_helper_fp_reduce
*fn
)
4083 unsigned vsz
= vec_full_reg_size(s
);
4084 unsigned p2vsz
= pow2ceil(vsz
);
4085 TCGv_i32 t_desc
= tcg_const_i32(simd_desc(vsz
, vsz
, p2vsz
));
4086 TCGv_ptr t_zn
, t_pg
, status
;
4089 temp
= tcg_temp_new_i64();
4090 t_zn
= tcg_temp_new_ptr();
4091 t_pg
= tcg_temp_new_ptr();
4093 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
4094 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
4095 status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4097 fn(temp
, t_zn
, t_pg
, status
, t_desc
);
4098 tcg_temp_free_ptr(t_zn
);
4099 tcg_temp_free_ptr(t_pg
);
4100 tcg_temp_free_ptr(status
);
4101 tcg_temp_free_i32(t_desc
);
4103 write_fp_dreg(s
, a
->rd
, temp
);
4104 tcg_temp_free_i64(temp
);
4107 #define DO_VPZ(NAME, name) \
4108 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
4110 static gen_helper_fp_reduce * const fns[3] = { \
4111 gen_helper_sve_##name##_h, \
4112 gen_helper_sve_##name##_s, \
4113 gen_helper_sve_##name##_d, \
4115 if (a->esz == 0) { \
4118 if (sve_access_check(s)) { \
4119 do_reduce(s, a, fns[a->esz - 1]); \
4124 DO_VPZ(FADDV
, faddv
)
4125 DO_VPZ(FMINNMV
, fminnmv
)
4126 DO_VPZ(FMAXNMV
, fmaxnmv
)
4127 DO_VPZ(FMINV
, fminv
)
4128 DO_VPZ(FMAXV
, fmaxv
)
4131 *** SVE Floating Point Unary Operations - Unpredicated Group
4134 static void do_zz_fp(DisasContext
*s
, arg_rr_esz
*a
, gen_helper_gvec_2_ptr
*fn
)
4136 unsigned vsz
= vec_full_reg_size(s
);
4137 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4139 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s
, a
->rd
),
4140 vec_full_reg_offset(s
, a
->rn
),
4141 status
, vsz
, vsz
, 0, fn
);
4142 tcg_temp_free_ptr(status
);
4145 static bool trans_FRECPE(DisasContext
*s
, arg_rr_esz
*a
)
4147 static gen_helper_gvec_2_ptr
* const fns
[3] = {
4148 gen_helper_gvec_frecpe_h
,
4149 gen_helper_gvec_frecpe_s
,
4150 gen_helper_gvec_frecpe_d
,
4155 if (sve_access_check(s
)) {
4156 do_zz_fp(s
, a
, fns
[a
->esz
- 1]);
4161 static bool trans_FRSQRTE(DisasContext
*s
, arg_rr_esz
*a
)
4163 static gen_helper_gvec_2_ptr
* const fns
[3] = {
4164 gen_helper_gvec_frsqrte_h
,
4165 gen_helper_gvec_frsqrte_s
,
4166 gen_helper_gvec_frsqrte_d
,
4171 if (sve_access_check(s
)) {
4172 do_zz_fp(s
, a
, fns
[a
->esz
- 1]);
4178 *** SVE Floating Point Compare with Zero Group
4181 static void do_ppz_fp(DisasContext
*s
, arg_rpr_esz
*a
,
4182 gen_helper_gvec_3_ptr
*fn
)
4184 unsigned vsz
= vec_full_reg_size(s
);
4185 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4187 tcg_gen_gvec_3_ptr(pred_full_reg_offset(s
, a
->rd
),
4188 vec_full_reg_offset(s
, a
->rn
),
4189 pred_full_reg_offset(s
, a
->pg
),
4190 status
, vsz
, vsz
, 0, fn
);
4191 tcg_temp_free_ptr(status
);
4194 #define DO_PPZ(NAME, name) \
4195 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
4197 static gen_helper_gvec_3_ptr * const fns[3] = { \
4198 gen_helper_sve_##name##_h, \
4199 gen_helper_sve_##name##_s, \
4200 gen_helper_sve_##name##_d, \
4202 if (a->esz == 0) { \
4205 if (sve_access_check(s)) { \
4206 do_ppz_fp(s, a, fns[a->esz - 1]); \
4211 DO_PPZ(FCMGE_ppz0
, fcmge0
)
4212 DO_PPZ(FCMGT_ppz0
, fcmgt0
)
4213 DO_PPZ(FCMLE_ppz0
, fcmle0
)
4214 DO_PPZ(FCMLT_ppz0
, fcmlt0
)
4215 DO_PPZ(FCMEQ_ppz0
, fcmeq0
)
4216 DO_PPZ(FCMNE_ppz0
, fcmne0
)
4221 *** SVE floating-point trig multiply-add coefficient
4224 static bool trans_FTMAD(DisasContext
*s
, arg_FTMAD
*a
)
4226 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4227 gen_helper_sve_ftmad_h
,
4228 gen_helper_sve_ftmad_s
,
4229 gen_helper_sve_ftmad_d
,
4235 if (sve_access_check(s
)) {
4236 unsigned vsz
= vec_full_reg_size(s
);
4237 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4238 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4239 vec_full_reg_offset(s
, a
->rn
),
4240 vec_full_reg_offset(s
, a
->rm
),
4241 status
, vsz
, vsz
, a
->imm
, fns
[a
->esz
- 1]);
4242 tcg_temp_free_ptr(status
);
4248 *** SVE Floating Point Accumulating Reduction Group
4251 static bool trans_FADDA(DisasContext
*s
, arg_rprr_esz
*a
)
4253 typedef void fadda_fn(TCGv_i64
, TCGv_i64
, TCGv_ptr
,
4254 TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
4255 static fadda_fn
* const fns
[3] = {
4256 gen_helper_sve_fadda_h
,
4257 gen_helper_sve_fadda_s
,
4258 gen_helper_sve_fadda_d
,
4260 unsigned vsz
= vec_full_reg_size(s
);
4261 TCGv_ptr t_rm
, t_pg
, t_fpst
;
4268 if (!sve_access_check(s
)) {
4272 t_val
= load_esz(cpu_env
, vec_reg_offset(s
, a
->rn
, 0, a
->esz
), a
->esz
);
4273 t_rm
= tcg_temp_new_ptr();
4274 t_pg
= tcg_temp_new_ptr();
4275 tcg_gen_addi_ptr(t_rm
, cpu_env
, vec_full_reg_offset(s
, a
->rm
));
4276 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
4277 t_fpst
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4278 t_desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
4280 fns
[a
->esz
- 1](t_val
, t_val
, t_rm
, t_pg
, t_fpst
, t_desc
);
4282 tcg_temp_free_i32(t_desc
);
4283 tcg_temp_free_ptr(t_fpst
);
4284 tcg_temp_free_ptr(t_pg
);
4285 tcg_temp_free_ptr(t_rm
);
4287 write_fp_dreg(s
, a
->rd
, t_val
);
4288 tcg_temp_free_i64(t_val
);
4293 *** SVE Floating Point Arithmetic - Unpredicated Group
4296 static bool do_zzz_fp(DisasContext
*s
, arg_rrr_esz
*a
,
4297 gen_helper_gvec_3_ptr
*fn
)
4302 if (sve_access_check(s
)) {
4303 unsigned vsz
= vec_full_reg_size(s
);
4304 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4305 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4306 vec_full_reg_offset(s
, a
->rn
),
4307 vec_full_reg_offset(s
, a
->rm
),
4308 status
, vsz
, vsz
, 0, fn
);
4309 tcg_temp_free_ptr(status
);
4315 #define DO_FP3(NAME, name) \
4316 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
4318 static gen_helper_gvec_3_ptr * const fns[4] = { \
4319 NULL, gen_helper_gvec_##name##_h, \
4320 gen_helper_gvec_##name##_s, gen_helper_gvec_##name##_d \
4322 return do_zzz_fp(s, a, fns[a->esz]); \
4325 DO_FP3(FADD_zzz
, fadd
)
4326 DO_FP3(FSUB_zzz
, fsub
)
4327 DO_FP3(FMUL_zzz
, fmul
)
4328 DO_FP3(FTSMUL
, ftsmul
)
4329 DO_FP3(FRECPS
, recps
)
4330 DO_FP3(FRSQRTS
, rsqrts
)
4335 *** SVE Floating Point Arithmetic - Predicated Group
4338 static bool do_zpzz_fp(DisasContext
*s
, arg_rprr_esz
*a
,
4339 gen_helper_gvec_4_ptr
*fn
)
4344 if (sve_access_check(s
)) {
4345 unsigned vsz
= vec_full_reg_size(s
);
4346 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4347 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4348 vec_full_reg_offset(s
, a
->rn
),
4349 vec_full_reg_offset(s
, a
->rm
),
4350 pred_full_reg_offset(s
, a
->pg
),
4351 status
, vsz
, vsz
, 0, fn
);
4352 tcg_temp_free_ptr(status
);
4357 #define DO_FP3(NAME, name) \
4358 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
4360 static gen_helper_gvec_4_ptr * const fns[4] = { \
4361 NULL, gen_helper_sve_##name##_h, \
4362 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4364 return do_zpzz_fp(s, a, fns[a->esz]); \
4367 DO_FP3(FADD_zpzz
, fadd
)
4368 DO_FP3(FSUB_zpzz
, fsub
)
4369 DO_FP3(FMUL_zpzz
, fmul
)
4370 DO_FP3(FMIN_zpzz
, fmin
)
4371 DO_FP3(FMAX_zpzz
, fmax
)
4372 DO_FP3(FMINNM_zpzz
, fminnum
)
4373 DO_FP3(FMAXNM_zpzz
, fmaxnum
)
4375 DO_FP3(FSCALE
, fscalbn
)
4377 DO_FP3(FMULX
, fmulx
)
4381 typedef void gen_helper_sve_fp2scalar(TCGv_ptr
, TCGv_ptr
, TCGv_ptr
,
4382 TCGv_i64
, TCGv_ptr
, TCGv_i32
);
4384 static void do_fp_scalar(DisasContext
*s
, int zd
, int zn
, int pg
, bool is_fp16
,
4385 TCGv_i64 scalar
, gen_helper_sve_fp2scalar
*fn
)
4387 unsigned vsz
= vec_full_reg_size(s
);
4388 TCGv_ptr t_zd
, t_zn
, t_pg
, status
;
4391 t_zd
= tcg_temp_new_ptr();
4392 t_zn
= tcg_temp_new_ptr();
4393 t_pg
= tcg_temp_new_ptr();
4394 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, zd
));
4395 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, zn
));
4396 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
4398 status
= fpstatus_ptr(is_fp16
? FPST_FPCR_F16
: FPST_FPCR
);
4399 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
4400 fn(t_zd
, t_zn
, t_pg
, scalar
, status
, desc
);
4402 tcg_temp_free_i32(desc
);
4403 tcg_temp_free_ptr(status
);
4404 tcg_temp_free_ptr(t_pg
);
4405 tcg_temp_free_ptr(t_zn
);
4406 tcg_temp_free_ptr(t_zd
);
4409 static void do_fp_imm(DisasContext
*s
, arg_rpri_esz
*a
, uint64_t imm
,
4410 gen_helper_sve_fp2scalar
*fn
)
4412 TCGv_i64 temp
= tcg_const_i64(imm
);
4413 do_fp_scalar(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, temp
, fn
);
4414 tcg_temp_free_i64(temp
);
4417 #define DO_FP_IMM(NAME, name, const0, const1) \
4418 static bool trans_##NAME##_zpzi(DisasContext *s, arg_rpri_esz *a) \
4420 static gen_helper_sve_fp2scalar * const fns[3] = { \
4421 gen_helper_sve_##name##_h, \
4422 gen_helper_sve_##name##_s, \
4423 gen_helper_sve_##name##_d \
4425 static uint64_t const val[3][2] = { \
4426 { float16_##const0, float16_##const1 }, \
4427 { float32_##const0, float32_##const1 }, \
4428 { float64_##const0, float64_##const1 }, \
4430 if (a->esz == 0) { \
4433 if (sve_access_check(s)) { \
4434 do_fp_imm(s, a, val[a->esz - 1][a->imm], fns[a->esz - 1]); \
4439 DO_FP_IMM(FADD
, fadds
, half
, one
)
4440 DO_FP_IMM(FSUB
, fsubs
, half
, one
)
4441 DO_FP_IMM(FMUL
, fmuls
, half
, two
)
4442 DO_FP_IMM(FSUBR
, fsubrs
, half
, one
)
4443 DO_FP_IMM(FMAXNM
, fmaxnms
, zero
, one
)
4444 DO_FP_IMM(FMINNM
, fminnms
, zero
, one
)
4445 DO_FP_IMM(FMAX
, fmaxs
, zero
, one
)
4446 DO_FP_IMM(FMIN
, fmins
, zero
, one
)
4450 static bool do_fp_cmp(DisasContext
*s
, arg_rprr_esz
*a
,
4451 gen_helper_gvec_4_ptr
*fn
)
4456 if (sve_access_check(s
)) {
4457 unsigned vsz
= vec_full_reg_size(s
);
4458 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4459 tcg_gen_gvec_4_ptr(pred_full_reg_offset(s
, a
->rd
),
4460 vec_full_reg_offset(s
, a
->rn
),
4461 vec_full_reg_offset(s
, a
->rm
),
4462 pred_full_reg_offset(s
, a
->pg
),
4463 status
, vsz
, vsz
, 0, fn
);
4464 tcg_temp_free_ptr(status
);
4469 #define DO_FPCMP(NAME, name) \
4470 static bool trans_##NAME##_ppzz(DisasContext *s, arg_rprr_esz *a) \
4472 static gen_helper_gvec_4_ptr * const fns[4] = { \
4473 NULL, gen_helper_sve_##name##_h, \
4474 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4476 return do_fp_cmp(s, a, fns[a->esz]); \
4479 DO_FPCMP(FCMGE
, fcmge
)
4480 DO_FPCMP(FCMGT
, fcmgt
)
4481 DO_FPCMP(FCMEQ
, fcmeq
)
4482 DO_FPCMP(FCMNE
, fcmne
)
4483 DO_FPCMP(FCMUO
, fcmuo
)
4484 DO_FPCMP(FACGE
, facge
)
4485 DO_FPCMP(FACGT
, facgt
)
4489 static bool trans_FCADD(DisasContext
*s
, arg_FCADD
*a
)
4491 static gen_helper_gvec_4_ptr
* const fns
[3] = {
4492 gen_helper_sve_fcadd_h
,
4493 gen_helper_sve_fcadd_s
,
4494 gen_helper_sve_fcadd_d
4500 if (sve_access_check(s
)) {
4501 unsigned vsz
= vec_full_reg_size(s
);
4502 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4503 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4504 vec_full_reg_offset(s
, a
->rn
),
4505 vec_full_reg_offset(s
, a
->rm
),
4506 pred_full_reg_offset(s
, a
->pg
),
4507 status
, vsz
, vsz
, a
->rot
, fns
[a
->esz
- 1]);
4508 tcg_temp_free_ptr(status
);
4513 static bool do_fmla(DisasContext
*s
, arg_rprrr_esz
*a
,
4514 gen_helper_gvec_5_ptr
*fn
)
4519 if (sve_access_check(s
)) {
4520 unsigned vsz
= vec_full_reg_size(s
);
4521 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4522 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s
, a
->rd
),
4523 vec_full_reg_offset(s
, a
->rn
),
4524 vec_full_reg_offset(s
, a
->rm
),
4525 vec_full_reg_offset(s
, a
->ra
),
4526 pred_full_reg_offset(s
, a
->pg
),
4527 status
, vsz
, vsz
, 0, fn
);
4528 tcg_temp_free_ptr(status
);
4533 #define DO_FMLA(NAME, name) \
4534 static bool trans_##NAME(DisasContext *s, arg_rprrr_esz *a) \
4536 static gen_helper_gvec_5_ptr * const fns[4] = { \
4537 NULL, gen_helper_sve_##name##_h, \
4538 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4540 return do_fmla(s, a, fns[a->esz]); \
4543 DO_FMLA(FMLA_zpzzz
, fmla_zpzzz
)
4544 DO_FMLA(FMLS_zpzzz
, fmls_zpzzz
)
4545 DO_FMLA(FNMLA_zpzzz
, fnmla_zpzzz
)
4546 DO_FMLA(FNMLS_zpzzz
, fnmls_zpzzz
)
4550 static bool trans_FCMLA_zpzzz(DisasContext
*s
, arg_FCMLA_zpzzz
*a
)
4552 static gen_helper_gvec_5_ptr
* const fns
[4] = {
4554 gen_helper_sve_fcmla_zpzzz_h
,
4555 gen_helper_sve_fcmla_zpzzz_s
,
4556 gen_helper_sve_fcmla_zpzzz_d
,
4562 if (sve_access_check(s
)) {
4563 unsigned vsz
= vec_full_reg_size(s
);
4564 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4565 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s
, a
->rd
),
4566 vec_full_reg_offset(s
, a
->rn
),
4567 vec_full_reg_offset(s
, a
->rm
),
4568 vec_full_reg_offset(s
, a
->ra
),
4569 pred_full_reg_offset(s
, a
->pg
),
4570 status
, vsz
, vsz
, a
->rot
, fns
[a
->esz
]);
4571 tcg_temp_free_ptr(status
);
4576 static bool trans_FCMLA_zzxz(DisasContext
*s
, arg_FCMLA_zzxz
*a
)
4578 static gen_helper_gvec_4_ptr
* const fns
[2] = {
4579 gen_helper_gvec_fcmlah_idx
,
4580 gen_helper_gvec_fcmlas_idx
,
4583 tcg_debug_assert(a
->esz
== 1 || a
->esz
== 2);
4584 tcg_debug_assert(a
->rd
== a
->ra
);
4585 if (sve_access_check(s
)) {
4586 unsigned vsz
= vec_full_reg_size(s
);
4587 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4588 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4589 vec_full_reg_offset(s
, a
->rn
),
4590 vec_full_reg_offset(s
, a
->rm
),
4591 vec_full_reg_offset(s
, a
->ra
),
4593 a
->index
* 4 + a
->rot
,
4595 tcg_temp_free_ptr(status
);
4601 *** SVE Floating Point Unary Operations Predicated Group
4604 static bool do_zpz_ptr(DisasContext
*s
, int rd
, int rn
, int pg
,
4605 bool is_fp16
, gen_helper_gvec_3_ptr
*fn
)
4607 if (sve_access_check(s
)) {
4608 unsigned vsz
= vec_full_reg_size(s
);
4609 TCGv_ptr status
= fpstatus_ptr(is_fp16
? FPST_FPCR_F16
: FPST_FPCR
);
4610 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, rd
),
4611 vec_full_reg_offset(s
, rn
),
4612 pred_full_reg_offset(s
, pg
),
4613 status
, vsz
, vsz
, 0, fn
);
4614 tcg_temp_free_ptr(status
);
4619 static bool trans_FCVT_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4621 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_sh
);
4624 static bool trans_FCVT_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4626 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_hs
);
4629 static bool trans_FCVT_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4631 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_dh
);
4634 static bool trans_FCVT_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4636 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_hd
);
4639 static bool trans_FCVT_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4641 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_ds
);
4644 static bool trans_FCVT_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4646 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_sd
);
4649 static bool trans_FCVTZS_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4651 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hh
);
4654 static bool trans_FCVTZU_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4656 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hh
);
4659 static bool trans_FCVTZS_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4661 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hs
);
4664 static bool trans_FCVTZU_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4666 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hs
);
4669 static bool trans_FCVTZS_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4671 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hd
);
4674 static bool trans_FCVTZU_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4676 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hd
);
4679 static bool trans_FCVTZS_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4681 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_ss
);
4684 static bool trans_FCVTZU_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4686 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_ss
);
4689 static bool trans_FCVTZS_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4691 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_sd
);
4694 static bool trans_FCVTZU_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4696 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_sd
);
4699 static bool trans_FCVTZS_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4701 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_ds
);
4704 static bool trans_FCVTZU_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4706 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_ds
);
4709 static bool trans_FCVTZS_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4711 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_dd
);
4714 static bool trans_FCVTZU_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4716 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_dd
);
4719 static gen_helper_gvec_3_ptr
* const frint_fns
[3] = {
4720 gen_helper_sve_frint_h
,
4721 gen_helper_sve_frint_s
,
4722 gen_helper_sve_frint_d
4725 static bool trans_FRINTI(DisasContext
*s
, arg_rpr_esz
*a
)
4730 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
,
4731 frint_fns
[a
->esz
- 1]);
4734 static bool trans_FRINTX(DisasContext
*s
, arg_rpr_esz
*a
)
4736 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4737 gen_helper_sve_frintx_h
,
4738 gen_helper_sve_frintx_s
,
4739 gen_helper_sve_frintx_d
4744 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4747 static bool do_frint_mode(DisasContext
*s
, arg_rpr_esz
*a
, int mode
)
4752 if (sve_access_check(s
)) {
4753 unsigned vsz
= vec_full_reg_size(s
);
4754 TCGv_i32 tmode
= tcg_const_i32(mode
);
4755 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4757 gen_helper_set_rmode(tmode
, tmode
, status
);
4759 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4760 vec_full_reg_offset(s
, a
->rn
),
4761 pred_full_reg_offset(s
, a
->pg
),
4762 status
, vsz
, vsz
, 0, frint_fns
[a
->esz
- 1]);
4764 gen_helper_set_rmode(tmode
, tmode
, status
);
4765 tcg_temp_free_i32(tmode
);
4766 tcg_temp_free_ptr(status
);
4771 static bool trans_FRINTN(DisasContext
*s
, arg_rpr_esz
*a
)
4773 return do_frint_mode(s
, a
, float_round_nearest_even
);
4776 static bool trans_FRINTP(DisasContext
*s
, arg_rpr_esz
*a
)
4778 return do_frint_mode(s
, a
, float_round_up
);
4781 static bool trans_FRINTM(DisasContext
*s
, arg_rpr_esz
*a
)
4783 return do_frint_mode(s
, a
, float_round_down
);
4786 static bool trans_FRINTZ(DisasContext
*s
, arg_rpr_esz
*a
)
4788 return do_frint_mode(s
, a
, float_round_to_zero
);
4791 static bool trans_FRINTA(DisasContext
*s
, arg_rpr_esz
*a
)
4793 return do_frint_mode(s
, a
, float_round_ties_away
);
4796 static bool trans_FRECPX(DisasContext
*s
, arg_rpr_esz
*a
)
4798 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4799 gen_helper_sve_frecpx_h
,
4800 gen_helper_sve_frecpx_s
,
4801 gen_helper_sve_frecpx_d
4806 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4809 static bool trans_FSQRT(DisasContext
*s
, arg_rpr_esz
*a
)
4811 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4812 gen_helper_sve_fsqrt_h
,
4813 gen_helper_sve_fsqrt_s
,
4814 gen_helper_sve_fsqrt_d
4819 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4822 static bool trans_SCVTF_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4824 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_hh
);
4827 static bool trans_SCVTF_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4829 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_sh
);
4832 static bool trans_SCVTF_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4834 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_dh
);
4837 static bool trans_SCVTF_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4839 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_ss
);
4842 static bool trans_SCVTF_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4844 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_ds
);
4847 static bool trans_SCVTF_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4849 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_sd
);
4852 static bool trans_SCVTF_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4854 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_dd
);
4857 static bool trans_UCVTF_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4859 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_hh
);
4862 static bool trans_UCVTF_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4864 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_sh
);
4867 static bool trans_UCVTF_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4869 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_dh
);
4872 static bool trans_UCVTF_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4874 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_ss
);
4877 static bool trans_UCVTF_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4879 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_ds
);
4882 static bool trans_UCVTF_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4884 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_sd
);
4887 static bool trans_UCVTF_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4889 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_dd
);
4893 *** SVE Memory - 32-bit Gather and Unsized Contiguous Group
4896 /* Subroutine loading a vector register at VOFS of LEN bytes.
4897 * The load should begin at the address Rn + IMM.
4900 static void do_ldr(DisasContext
*s
, uint32_t vofs
, int len
, int rn
, int imm
)
4902 int len_align
= QEMU_ALIGN_DOWN(len
, 8);
4903 int len_remain
= len
% 8;
4904 int nparts
= len
/ 8 + ctpop8(len_remain
);
4905 int midx
= get_mem_index(s
);
4906 TCGv_i64 dirty_addr
, clean_addr
, t0
, t1
;
4908 dirty_addr
= tcg_temp_new_i64();
4909 tcg_gen_addi_i64(dirty_addr
, cpu_reg_sp(s
, rn
), imm
);
4910 clean_addr
= gen_mte_checkN(s
, dirty_addr
, false, rn
!= 31, len
);
4911 tcg_temp_free_i64(dirty_addr
);
4914 * Note that unpredicated load/store of vector/predicate registers
4915 * are defined as a stream of bytes, which equates to little-endian
4916 * operations on larger quantities.
4917 * Attempt to keep code expansion to a minimum by limiting the
4918 * amount of unrolling done.
4923 t0
= tcg_temp_new_i64();
4924 for (i
= 0; i
< len_align
; i
+= 8) {
4925 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEQ
);
4926 tcg_gen_st_i64(t0
, cpu_env
, vofs
+ i
);
4927 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4929 tcg_temp_free_i64(t0
);
4931 TCGLabel
*loop
= gen_new_label();
4932 TCGv_ptr tp
, i
= tcg_const_local_ptr(0);
4934 /* Copy the clean address into a local temp, live across the loop. */
4936 clean_addr
= new_tmp_a64_local(s
);
4937 tcg_gen_mov_i64(clean_addr
, t0
);
4939 gen_set_label(loop
);
4941 t0
= tcg_temp_new_i64();
4942 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEQ
);
4943 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4945 tp
= tcg_temp_new_ptr();
4946 tcg_gen_add_ptr(tp
, cpu_env
, i
);
4947 tcg_gen_addi_ptr(i
, i
, 8);
4948 tcg_gen_st_i64(t0
, tp
, vofs
);
4949 tcg_temp_free_ptr(tp
);
4950 tcg_temp_free_i64(t0
);
4952 tcg_gen_brcondi_ptr(TCG_COND_LTU
, i
, len_align
, loop
);
4953 tcg_temp_free_ptr(i
);
4957 * Predicate register loads can be any multiple of 2.
4958 * Note that we still store the entire 64-bit unit into cpu_env.
4961 t0
= tcg_temp_new_i64();
4962 switch (len_remain
) {
4966 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
,
4967 MO_LE
| ctz32(len_remain
));
4971 t1
= tcg_temp_new_i64();
4972 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEUL
);
4973 tcg_gen_addi_i64(clean_addr
, clean_addr
, 4);
4974 tcg_gen_qemu_ld_i64(t1
, clean_addr
, midx
, MO_LEUW
);
4975 tcg_gen_deposit_i64(t0
, t0
, t1
, 32, 32);
4976 tcg_temp_free_i64(t1
);
4980 g_assert_not_reached();
4982 tcg_gen_st_i64(t0
, cpu_env
, vofs
+ len_align
);
4983 tcg_temp_free_i64(t0
);
4987 /* Similarly for stores. */
4988 static void do_str(DisasContext
*s
, uint32_t vofs
, int len
, int rn
, int imm
)
4990 int len_align
= QEMU_ALIGN_DOWN(len
, 8);
4991 int len_remain
= len
% 8;
4992 int nparts
= len
/ 8 + ctpop8(len_remain
);
4993 int midx
= get_mem_index(s
);
4994 TCGv_i64 dirty_addr
, clean_addr
, t0
;
4996 dirty_addr
= tcg_temp_new_i64();
4997 tcg_gen_addi_i64(dirty_addr
, cpu_reg_sp(s
, rn
), imm
);
4998 clean_addr
= gen_mte_checkN(s
, dirty_addr
, false, rn
!= 31, len
);
4999 tcg_temp_free_i64(dirty_addr
);
5001 /* Note that unpredicated load/store of vector/predicate registers
5002 * are defined as a stream of bytes, which equates to little-endian
5003 * operations on larger quantities. There is no nice way to force
5004 * a little-endian store for aarch64_be-linux-user out of line.
5006 * Attempt to keep code expansion to a minimum by limiting the
5007 * amount of unrolling done.
5012 t0
= tcg_temp_new_i64();
5013 for (i
= 0; i
< len_align
; i
+= 8) {
5014 tcg_gen_ld_i64(t0
, cpu_env
, vofs
+ i
);
5015 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEQ
);
5016 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
5018 tcg_temp_free_i64(t0
);
5020 TCGLabel
*loop
= gen_new_label();
5021 TCGv_ptr tp
, i
= tcg_const_local_ptr(0);
5023 /* Copy the clean address into a local temp, live across the loop. */
5025 clean_addr
= new_tmp_a64_local(s
);
5026 tcg_gen_mov_i64(clean_addr
, t0
);
5028 gen_set_label(loop
);
5030 t0
= tcg_temp_new_i64();
5031 tp
= tcg_temp_new_ptr();
5032 tcg_gen_add_ptr(tp
, cpu_env
, i
);
5033 tcg_gen_ld_i64(t0
, tp
, vofs
);
5034 tcg_gen_addi_ptr(i
, i
, 8);
5035 tcg_temp_free_ptr(tp
);
5037 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEQ
);
5038 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
5039 tcg_temp_free_i64(t0
);
5041 tcg_gen_brcondi_ptr(TCG_COND_LTU
, i
, len_align
, loop
);
5042 tcg_temp_free_ptr(i
);
5045 /* Predicate register stores can be any multiple of 2. */
5047 t0
= tcg_temp_new_i64();
5048 tcg_gen_ld_i64(t0
, cpu_env
, vofs
+ len_align
);
5050 switch (len_remain
) {
5054 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
,
5055 MO_LE
| ctz32(len_remain
));
5059 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUL
);
5060 tcg_gen_addi_i64(clean_addr
, clean_addr
, 4);
5061 tcg_gen_shri_i64(t0
, t0
, 32);
5062 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUW
);
5066 g_assert_not_reached();
5068 tcg_temp_free_i64(t0
);
5072 static bool trans_LDR_zri(DisasContext
*s
, arg_rri
*a
)
5074 if (sve_access_check(s
)) {
5075 int size
= vec_full_reg_size(s
);
5076 int off
= vec_full_reg_offset(s
, a
->rd
);
5077 do_ldr(s
, off
, size
, a
->rn
, a
->imm
* size
);
5082 static bool trans_LDR_pri(DisasContext
*s
, arg_rri
*a
)
5084 if (sve_access_check(s
)) {
5085 int size
= pred_full_reg_size(s
);
5086 int off
= pred_full_reg_offset(s
, a
->rd
);
5087 do_ldr(s
, off
, size
, a
->rn
, a
->imm
* size
);
5092 static bool trans_STR_zri(DisasContext
*s
, arg_rri
*a
)
5094 if (sve_access_check(s
)) {
5095 int size
= vec_full_reg_size(s
);
5096 int off
= vec_full_reg_offset(s
, a
->rd
);
5097 do_str(s
, off
, size
, a
->rn
, a
->imm
* size
);
5102 static bool trans_STR_pri(DisasContext
*s
, arg_rri
*a
)
5104 if (sve_access_check(s
)) {
5105 int size
= pred_full_reg_size(s
);
5106 int off
= pred_full_reg_offset(s
, a
->rd
);
5107 do_str(s
, off
, size
, a
->rn
, a
->imm
* size
);
5113 *** SVE Memory - Contiguous Load Group
5116 /* The memory mode of the dtype. */
5117 static const MemOp dtype_mop
[16] = {
5118 MO_UB
, MO_UB
, MO_UB
, MO_UB
,
5119 MO_SL
, MO_UW
, MO_UW
, MO_UW
,
5120 MO_SW
, MO_SW
, MO_UL
, MO_UL
,
5121 MO_SB
, MO_SB
, MO_SB
, MO_Q
5124 #define dtype_msz(x) (dtype_mop[x] & MO_SIZE)
5126 /* The vector element size of dtype. */
5127 static const uint8_t dtype_esz
[16] = {
5134 static void do_mem_zpa(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
,
5135 int dtype
, uint32_t mte_n
, bool is_write
,
5136 gen_helper_gvec_mem
*fn
)
5138 unsigned vsz
= vec_full_reg_size(s
);
5144 * For e.g. LD4, there are not enough arguments to pass all 4
5145 * registers as pointers, so encode the regno into the data field.
5146 * For consistency, do this even for LD1.
5148 if (s
->mte_active
[0]) {
5149 int msz
= dtype_msz(dtype
);
5151 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, get_mem_index(s
));
5152 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
5153 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
5154 desc
= FIELD_DP32(desc
, MTEDESC
, WRITE
, is_write
);
5155 desc
= FIELD_DP32(desc
, MTEDESC
, SIZEM1
, (mte_n
<< msz
) - 1);
5156 desc
<<= SVE_MTEDESC_SHIFT
;
5158 addr
= clean_data_tbi(s
, addr
);
5161 desc
= simd_desc(vsz
, vsz
, zt
| desc
);
5162 t_desc
= tcg_const_i32(desc
);
5163 t_pg
= tcg_temp_new_ptr();
5165 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
5166 fn(cpu_env
, t_pg
, addr
, t_desc
);
5168 tcg_temp_free_ptr(t_pg
);
5169 tcg_temp_free_i32(t_desc
);
5172 static void do_ld_zpa(DisasContext
*s
, int zt
, int pg
,
5173 TCGv_i64 addr
, int dtype
, int nreg
)
5175 static gen_helper_gvec_mem
* const fns
[2][2][16][4] = {
5176 { /* mte inactive, little-endian */
5177 { { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld2bb_r
,
5178 gen_helper_sve_ld3bb_r
, gen_helper_sve_ld4bb_r
},
5179 { gen_helper_sve_ld1bhu_r
, NULL
, NULL
, NULL
},
5180 { gen_helper_sve_ld1bsu_r
, NULL
, NULL
, NULL
},
5181 { gen_helper_sve_ld1bdu_r
, NULL
, NULL
, NULL
},
5183 { gen_helper_sve_ld1sds_le_r
, NULL
, NULL
, NULL
},
5184 { gen_helper_sve_ld1hh_le_r
, gen_helper_sve_ld2hh_le_r
,
5185 gen_helper_sve_ld3hh_le_r
, gen_helper_sve_ld4hh_le_r
},
5186 { gen_helper_sve_ld1hsu_le_r
, NULL
, NULL
, NULL
},
5187 { gen_helper_sve_ld1hdu_le_r
, NULL
, NULL
, NULL
},
5189 { gen_helper_sve_ld1hds_le_r
, NULL
, NULL
, NULL
},
5190 { gen_helper_sve_ld1hss_le_r
, NULL
, NULL
, NULL
},
5191 { gen_helper_sve_ld1ss_le_r
, gen_helper_sve_ld2ss_le_r
,
5192 gen_helper_sve_ld3ss_le_r
, gen_helper_sve_ld4ss_le_r
},
5193 { gen_helper_sve_ld1sdu_le_r
, NULL
, NULL
, NULL
},
5195 { gen_helper_sve_ld1bds_r
, NULL
, NULL
, NULL
},
5196 { gen_helper_sve_ld1bss_r
, NULL
, NULL
, NULL
},
5197 { gen_helper_sve_ld1bhs_r
, NULL
, NULL
, NULL
},
5198 { gen_helper_sve_ld1dd_le_r
, gen_helper_sve_ld2dd_le_r
,
5199 gen_helper_sve_ld3dd_le_r
, gen_helper_sve_ld4dd_le_r
} },
5201 /* mte inactive, big-endian */
5202 { { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld2bb_r
,
5203 gen_helper_sve_ld3bb_r
, gen_helper_sve_ld4bb_r
},
5204 { gen_helper_sve_ld1bhu_r
, NULL
, NULL
, NULL
},
5205 { gen_helper_sve_ld1bsu_r
, NULL
, NULL
, NULL
},
5206 { gen_helper_sve_ld1bdu_r
, NULL
, NULL
, NULL
},
5208 { gen_helper_sve_ld1sds_be_r
, NULL
, NULL
, NULL
},
5209 { gen_helper_sve_ld1hh_be_r
, gen_helper_sve_ld2hh_be_r
,
5210 gen_helper_sve_ld3hh_be_r
, gen_helper_sve_ld4hh_be_r
},
5211 { gen_helper_sve_ld1hsu_be_r
, NULL
, NULL
, NULL
},
5212 { gen_helper_sve_ld1hdu_be_r
, NULL
, NULL
, NULL
},
5214 { gen_helper_sve_ld1hds_be_r
, NULL
, NULL
, NULL
},
5215 { gen_helper_sve_ld1hss_be_r
, NULL
, NULL
, NULL
},
5216 { gen_helper_sve_ld1ss_be_r
, gen_helper_sve_ld2ss_be_r
,
5217 gen_helper_sve_ld3ss_be_r
, gen_helper_sve_ld4ss_be_r
},
5218 { gen_helper_sve_ld1sdu_be_r
, NULL
, NULL
, NULL
},
5220 { gen_helper_sve_ld1bds_r
, NULL
, NULL
, NULL
},
5221 { gen_helper_sve_ld1bss_r
, NULL
, NULL
, NULL
},
5222 { gen_helper_sve_ld1bhs_r
, NULL
, NULL
, NULL
},
5223 { gen_helper_sve_ld1dd_be_r
, gen_helper_sve_ld2dd_be_r
,
5224 gen_helper_sve_ld3dd_be_r
, gen_helper_sve_ld4dd_be_r
} } },
5226 { /* mte active, little-endian */
5227 { { gen_helper_sve_ld1bb_r_mte
,
5228 gen_helper_sve_ld2bb_r_mte
,
5229 gen_helper_sve_ld3bb_r_mte
,
5230 gen_helper_sve_ld4bb_r_mte
},
5231 { gen_helper_sve_ld1bhu_r_mte
, NULL
, NULL
, NULL
},
5232 { gen_helper_sve_ld1bsu_r_mte
, NULL
, NULL
, NULL
},
5233 { gen_helper_sve_ld1bdu_r_mte
, NULL
, NULL
, NULL
},
5235 { gen_helper_sve_ld1sds_le_r_mte
, NULL
, NULL
, NULL
},
5236 { gen_helper_sve_ld1hh_le_r_mte
,
5237 gen_helper_sve_ld2hh_le_r_mte
,
5238 gen_helper_sve_ld3hh_le_r_mte
,
5239 gen_helper_sve_ld4hh_le_r_mte
},
5240 { gen_helper_sve_ld1hsu_le_r_mte
, NULL
, NULL
, NULL
},
5241 { gen_helper_sve_ld1hdu_le_r_mte
, NULL
, NULL
, NULL
},
5243 { gen_helper_sve_ld1hds_le_r_mte
, NULL
, NULL
, NULL
},
5244 { gen_helper_sve_ld1hss_le_r_mte
, NULL
, NULL
, NULL
},
5245 { gen_helper_sve_ld1ss_le_r_mte
,
5246 gen_helper_sve_ld2ss_le_r_mte
,
5247 gen_helper_sve_ld3ss_le_r_mte
,
5248 gen_helper_sve_ld4ss_le_r_mte
},
5249 { gen_helper_sve_ld1sdu_le_r_mte
, NULL
, NULL
, NULL
},
5251 { gen_helper_sve_ld1bds_r_mte
, NULL
, NULL
, NULL
},
5252 { gen_helper_sve_ld1bss_r_mte
, NULL
, NULL
, NULL
},
5253 { gen_helper_sve_ld1bhs_r_mte
, NULL
, NULL
, NULL
},
5254 { gen_helper_sve_ld1dd_le_r_mte
,
5255 gen_helper_sve_ld2dd_le_r_mte
,
5256 gen_helper_sve_ld3dd_le_r_mte
,
5257 gen_helper_sve_ld4dd_le_r_mte
} },
5259 /* mte active, big-endian */
5260 { { gen_helper_sve_ld1bb_r_mte
,
5261 gen_helper_sve_ld2bb_r_mte
,
5262 gen_helper_sve_ld3bb_r_mte
,
5263 gen_helper_sve_ld4bb_r_mte
},
5264 { gen_helper_sve_ld1bhu_r_mte
, NULL
, NULL
, NULL
},
5265 { gen_helper_sve_ld1bsu_r_mte
, NULL
, NULL
, NULL
},
5266 { gen_helper_sve_ld1bdu_r_mte
, NULL
, NULL
, NULL
},
5268 { gen_helper_sve_ld1sds_be_r_mte
, NULL
, NULL
, NULL
},
5269 { gen_helper_sve_ld1hh_be_r_mte
,
5270 gen_helper_sve_ld2hh_be_r_mte
,
5271 gen_helper_sve_ld3hh_be_r_mte
,
5272 gen_helper_sve_ld4hh_be_r_mte
},
5273 { gen_helper_sve_ld1hsu_be_r_mte
, NULL
, NULL
, NULL
},
5274 { gen_helper_sve_ld1hdu_be_r_mte
, NULL
, NULL
, NULL
},
5276 { gen_helper_sve_ld1hds_be_r_mte
, NULL
, NULL
, NULL
},
5277 { gen_helper_sve_ld1hss_be_r_mte
, NULL
, NULL
, NULL
},
5278 { gen_helper_sve_ld1ss_be_r_mte
,
5279 gen_helper_sve_ld2ss_be_r_mte
,
5280 gen_helper_sve_ld3ss_be_r_mte
,
5281 gen_helper_sve_ld4ss_be_r_mte
},
5282 { gen_helper_sve_ld1sdu_be_r_mte
, NULL
, NULL
, NULL
},
5284 { gen_helper_sve_ld1bds_r_mte
, NULL
, NULL
, NULL
},
5285 { gen_helper_sve_ld1bss_r_mte
, NULL
, NULL
, NULL
},
5286 { gen_helper_sve_ld1bhs_r_mte
, NULL
, NULL
, NULL
},
5287 { gen_helper_sve_ld1dd_be_r_mte
,
5288 gen_helper_sve_ld2dd_be_r_mte
,
5289 gen_helper_sve_ld3dd_be_r_mte
,
5290 gen_helper_sve_ld4dd_be_r_mte
} } },
5292 gen_helper_gvec_mem
*fn
5293 = fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][dtype
][nreg
];
5296 * While there are holes in the table, they are not
5297 * accessible via the instruction encoding.
5300 do_mem_zpa(s
, zt
, pg
, addr
, dtype
, nreg
, false, fn
);
5303 static bool trans_LD_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5308 if (sve_access_check(s
)) {
5309 TCGv_i64 addr
= new_tmp_a64(s
);
5310 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
5311 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5312 do_ld_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, a
->nreg
);
5317 static bool trans_LD_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5319 if (sve_access_check(s
)) {
5320 int vsz
= vec_full_reg_size(s
);
5321 int elements
= vsz
>> dtype_esz
[a
->dtype
];
5322 TCGv_i64 addr
= new_tmp_a64(s
);
5324 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
),
5325 (a
->imm
* elements
* (a
->nreg
+ 1))
5326 << dtype_msz(a
->dtype
));
5327 do_ld_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, a
->nreg
);
5332 static bool trans_LDFF1_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5334 static gen_helper_gvec_mem
* const fns
[2][2][16] = {
5335 { /* mte inactive, little-endian */
5336 { gen_helper_sve_ldff1bb_r
,
5337 gen_helper_sve_ldff1bhu_r
,
5338 gen_helper_sve_ldff1bsu_r
,
5339 gen_helper_sve_ldff1bdu_r
,
5341 gen_helper_sve_ldff1sds_le_r
,
5342 gen_helper_sve_ldff1hh_le_r
,
5343 gen_helper_sve_ldff1hsu_le_r
,
5344 gen_helper_sve_ldff1hdu_le_r
,
5346 gen_helper_sve_ldff1hds_le_r
,
5347 gen_helper_sve_ldff1hss_le_r
,
5348 gen_helper_sve_ldff1ss_le_r
,
5349 gen_helper_sve_ldff1sdu_le_r
,
5351 gen_helper_sve_ldff1bds_r
,
5352 gen_helper_sve_ldff1bss_r
,
5353 gen_helper_sve_ldff1bhs_r
,
5354 gen_helper_sve_ldff1dd_le_r
},
5356 /* mte inactive, big-endian */
5357 { gen_helper_sve_ldff1bb_r
,
5358 gen_helper_sve_ldff1bhu_r
,
5359 gen_helper_sve_ldff1bsu_r
,
5360 gen_helper_sve_ldff1bdu_r
,
5362 gen_helper_sve_ldff1sds_be_r
,
5363 gen_helper_sve_ldff1hh_be_r
,
5364 gen_helper_sve_ldff1hsu_be_r
,
5365 gen_helper_sve_ldff1hdu_be_r
,
5367 gen_helper_sve_ldff1hds_be_r
,
5368 gen_helper_sve_ldff1hss_be_r
,
5369 gen_helper_sve_ldff1ss_be_r
,
5370 gen_helper_sve_ldff1sdu_be_r
,
5372 gen_helper_sve_ldff1bds_r
,
5373 gen_helper_sve_ldff1bss_r
,
5374 gen_helper_sve_ldff1bhs_r
,
5375 gen_helper_sve_ldff1dd_be_r
} },
5377 { /* mte active, little-endian */
5378 { gen_helper_sve_ldff1bb_r_mte
,
5379 gen_helper_sve_ldff1bhu_r_mte
,
5380 gen_helper_sve_ldff1bsu_r_mte
,
5381 gen_helper_sve_ldff1bdu_r_mte
,
5383 gen_helper_sve_ldff1sds_le_r_mte
,
5384 gen_helper_sve_ldff1hh_le_r_mte
,
5385 gen_helper_sve_ldff1hsu_le_r_mte
,
5386 gen_helper_sve_ldff1hdu_le_r_mte
,
5388 gen_helper_sve_ldff1hds_le_r_mte
,
5389 gen_helper_sve_ldff1hss_le_r_mte
,
5390 gen_helper_sve_ldff1ss_le_r_mte
,
5391 gen_helper_sve_ldff1sdu_le_r_mte
,
5393 gen_helper_sve_ldff1bds_r_mte
,
5394 gen_helper_sve_ldff1bss_r_mte
,
5395 gen_helper_sve_ldff1bhs_r_mte
,
5396 gen_helper_sve_ldff1dd_le_r_mte
},
5398 /* mte active, big-endian */
5399 { gen_helper_sve_ldff1bb_r_mte
,
5400 gen_helper_sve_ldff1bhu_r_mte
,
5401 gen_helper_sve_ldff1bsu_r_mte
,
5402 gen_helper_sve_ldff1bdu_r_mte
,
5404 gen_helper_sve_ldff1sds_be_r_mte
,
5405 gen_helper_sve_ldff1hh_be_r_mte
,
5406 gen_helper_sve_ldff1hsu_be_r_mte
,
5407 gen_helper_sve_ldff1hdu_be_r_mte
,
5409 gen_helper_sve_ldff1hds_be_r_mte
,
5410 gen_helper_sve_ldff1hss_be_r_mte
,
5411 gen_helper_sve_ldff1ss_be_r_mte
,
5412 gen_helper_sve_ldff1sdu_be_r_mte
,
5414 gen_helper_sve_ldff1bds_r_mte
,
5415 gen_helper_sve_ldff1bss_r_mte
,
5416 gen_helper_sve_ldff1bhs_r_mte
,
5417 gen_helper_sve_ldff1dd_be_r_mte
} },
5420 if (sve_access_check(s
)) {
5421 TCGv_i64 addr
= new_tmp_a64(s
);
5422 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
5423 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5424 do_mem_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, 1, false,
5425 fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][a
->dtype
]);
5430 static bool trans_LDNF1_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5432 static gen_helper_gvec_mem
* const fns
[2][2][16] = {
5433 { /* mte inactive, little-endian */
5434 { gen_helper_sve_ldnf1bb_r
,
5435 gen_helper_sve_ldnf1bhu_r
,
5436 gen_helper_sve_ldnf1bsu_r
,
5437 gen_helper_sve_ldnf1bdu_r
,
5439 gen_helper_sve_ldnf1sds_le_r
,
5440 gen_helper_sve_ldnf1hh_le_r
,
5441 gen_helper_sve_ldnf1hsu_le_r
,
5442 gen_helper_sve_ldnf1hdu_le_r
,
5444 gen_helper_sve_ldnf1hds_le_r
,
5445 gen_helper_sve_ldnf1hss_le_r
,
5446 gen_helper_sve_ldnf1ss_le_r
,
5447 gen_helper_sve_ldnf1sdu_le_r
,
5449 gen_helper_sve_ldnf1bds_r
,
5450 gen_helper_sve_ldnf1bss_r
,
5451 gen_helper_sve_ldnf1bhs_r
,
5452 gen_helper_sve_ldnf1dd_le_r
},
5454 /* mte inactive, big-endian */
5455 { gen_helper_sve_ldnf1bb_r
,
5456 gen_helper_sve_ldnf1bhu_r
,
5457 gen_helper_sve_ldnf1bsu_r
,
5458 gen_helper_sve_ldnf1bdu_r
,
5460 gen_helper_sve_ldnf1sds_be_r
,
5461 gen_helper_sve_ldnf1hh_be_r
,
5462 gen_helper_sve_ldnf1hsu_be_r
,
5463 gen_helper_sve_ldnf1hdu_be_r
,
5465 gen_helper_sve_ldnf1hds_be_r
,
5466 gen_helper_sve_ldnf1hss_be_r
,
5467 gen_helper_sve_ldnf1ss_be_r
,
5468 gen_helper_sve_ldnf1sdu_be_r
,
5470 gen_helper_sve_ldnf1bds_r
,
5471 gen_helper_sve_ldnf1bss_r
,
5472 gen_helper_sve_ldnf1bhs_r
,
5473 gen_helper_sve_ldnf1dd_be_r
} },
5475 { /* mte inactive, little-endian */
5476 { gen_helper_sve_ldnf1bb_r_mte
,
5477 gen_helper_sve_ldnf1bhu_r_mte
,
5478 gen_helper_sve_ldnf1bsu_r_mte
,
5479 gen_helper_sve_ldnf1bdu_r_mte
,
5481 gen_helper_sve_ldnf1sds_le_r_mte
,
5482 gen_helper_sve_ldnf1hh_le_r_mte
,
5483 gen_helper_sve_ldnf1hsu_le_r_mte
,
5484 gen_helper_sve_ldnf1hdu_le_r_mte
,
5486 gen_helper_sve_ldnf1hds_le_r_mte
,
5487 gen_helper_sve_ldnf1hss_le_r_mte
,
5488 gen_helper_sve_ldnf1ss_le_r_mte
,
5489 gen_helper_sve_ldnf1sdu_le_r_mte
,
5491 gen_helper_sve_ldnf1bds_r_mte
,
5492 gen_helper_sve_ldnf1bss_r_mte
,
5493 gen_helper_sve_ldnf1bhs_r_mte
,
5494 gen_helper_sve_ldnf1dd_le_r_mte
},
5496 /* mte inactive, big-endian */
5497 { gen_helper_sve_ldnf1bb_r_mte
,
5498 gen_helper_sve_ldnf1bhu_r_mte
,
5499 gen_helper_sve_ldnf1bsu_r_mte
,
5500 gen_helper_sve_ldnf1bdu_r_mte
,
5502 gen_helper_sve_ldnf1sds_be_r_mte
,
5503 gen_helper_sve_ldnf1hh_be_r_mte
,
5504 gen_helper_sve_ldnf1hsu_be_r_mte
,
5505 gen_helper_sve_ldnf1hdu_be_r_mte
,
5507 gen_helper_sve_ldnf1hds_be_r_mte
,
5508 gen_helper_sve_ldnf1hss_be_r_mte
,
5509 gen_helper_sve_ldnf1ss_be_r_mte
,
5510 gen_helper_sve_ldnf1sdu_be_r_mte
,
5512 gen_helper_sve_ldnf1bds_r_mte
,
5513 gen_helper_sve_ldnf1bss_r_mte
,
5514 gen_helper_sve_ldnf1bhs_r_mte
,
5515 gen_helper_sve_ldnf1dd_be_r_mte
} },
5518 if (sve_access_check(s
)) {
5519 int vsz
= vec_full_reg_size(s
);
5520 int elements
= vsz
>> dtype_esz
[a
->dtype
];
5521 int off
= (a
->imm
* elements
) << dtype_msz(a
->dtype
);
5522 TCGv_i64 addr
= new_tmp_a64(s
);
5524 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), off
);
5525 do_mem_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, 1, false,
5526 fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][a
->dtype
]);
5531 static void do_ldrq(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
, int msz
)
5533 static gen_helper_gvec_mem
* const fns
[2][4] = {
5534 { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld1hh_le_r
,
5535 gen_helper_sve_ld1ss_le_r
, gen_helper_sve_ld1dd_le_r
},
5536 { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld1hh_be_r
,
5537 gen_helper_sve_ld1ss_be_r
, gen_helper_sve_ld1dd_be_r
},
5539 unsigned vsz
= vec_full_reg_size(s
);
5544 /* Load the first quadword using the normal predicated load helpers. */
5545 desc
= simd_desc(16, 16, zt
);
5546 t_desc
= tcg_const_i32(desc
);
5548 poff
= pred_full_reg_offset(s
, pg
);
5551 * Zero-extend the first 16 bits of the predicate into a temporary.
5552 * This avoids triggering an assert making sure we don't have bits
5553 * set within a predicate beyond VQ, but we have lowered VQ to 1
5554 * for this load operation.
5556 TCGv_i64 tmp
= tcg_temp_new_i64();
5557 #ifdef HOST_WORDS_BIGENDIAN
5560 tcg_gen_ld16u_i64(tmp
, cpu_env
, poff
);
5562 poff
= offsetof(CPUARMState
, vfp
.preg_tmp
);
5563 tcg_gen_st_i64(tmp
, cpu_env
, poff
);
5564 tcg_temp_free_i64(tmp
);
5567 t_pg
= tcg_temp_new_ptr();
5568 tcg_gen_addi_ptr(t_pg
, cpu_env
, poff
);
5570 fns
[s
->be_data
== MO_BE
][msz
](cpu_env
, t_pg
, addr
, t_desc
);
5572 tcg_temp_free_ptr(t_pg
);
5573 tcg_temp_free_i32(t_desc
);
5575 /* Replicate that first quadword. */
5577 unsigned dofs
= vec_full_reg_offset(s
, zt
);
5578 tcg_gen_gvec_dup_mem(4, dofs
+ 16, dofs
, vsz
- 16, vsz
- 16);
5582 static bool trans_LD1RQ_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5587 if (sve_access_check(s
)) {
5588 int msz
= dtype_msz(a
->dtype
);
5589 TCGv_i64 addr
= new_tmp_a64(s
);
5590 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), msz
);
5591 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5592 do_ldrq(s
, a
->rd
, a
->pg
, addr
, msz
);
5597 static bool trans_LD1RQ_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5599 if (sve_access_check(s
)) {
5600 TCGv_i64 addr
= new_tmp_a64(s
);
5601 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), a
->imm
* 16);
5602 do_ldrq(s
, a
->rd
, a
->pg
, addr
, dtype_msz(a
->dtype
));
5607 /* Load and broadcast element. */
5608 static bool trans_LD1R_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5610 unsigned vsz
= vec_full_reg_size(s
);
5611 unsigned psz
= pred_full_reg_size(s
);
5612 unsigned esz
= dtype_esz
[a
->dtype
];
5613 unsigned msz
= dtype_msz(a
->dtype
);
5615 TCGv_i64 temp
, clean_addr
;
5617 if (!sve_access_check(s
)) {
5621 over
= gen_new_label();
5623 /* If the guarding predicate has no bits set, no load occurs. */
5625 /* Reduce the pred_esz_masks value simply to reduce the
5626 * size of the code generated here.
5628 uint64_t psz_mask
= MAKE_64BIT_MASK(0, psz
* 8);
5629 temp
= tcg_temp_new_i64();
5630 tcg_gen_ld_i64(temp
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
5631 tcg_gen_andi_i64(temp
, temp
, pred_esz_masks
[esz
] & psz_mask
);
5632 tcg_gen_brcondi_i64(TCG_COND_EQ
, temp
, 0, over
);
5633 tcg_temp_free_i64(temp
);
5635 TCGv_i32 t32
= tcg_temp_new_i32();
5636 find_last_active(s
, t32
, esz
, a
->pg
);
5637 tcg_gen_brcondi_i32(TCG_COND_LT
, t32
, 0, over
);
5638 tcg_temp_free_i32(t32
);
5641 /* Load the data. */
5642 temp
= tcg_temp_new_i64();
5643 tcg_gen_addi_i64(temp
, cpu_reg_sp(s
, a
->rn
), a
->imm
<< msz
);
5644 clean_addr
= gen_mte_check1(s
, temp
, false, true, msz
);
5646 tcg_gen_qemu_ld_i64(temp
, clean_addr
, get_mem_index(s
),
5647 finalize_memop(s
, dtype_mop
[a
->dtype
]));
5649 /* Broadcast to *all* elements. */
5650 tcg_gen_gvec_dup_i64(esz
, vec_full_reg_offset(s
, a
->rd
),
5652 tcg_temp_free_i64(temp
);
5654 /* Zero the inactive elements. */
5655 gen_set_label(over
);
5656 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, esz
, false);
5659 static void do_st_zpa(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
,
5660 int msz
, int esz
, int nreg
)
5662 static gen_helper_gvec_mem
* const fn_single
[2][2][4][4] = {
5663 { { { gen_helper_sve_st1bb_r
,
5664 gen_helper_sve_st1bh_r
,
5665 gen_helper_sve_st1bs_r
,
5666 gen_helper_sve_st1bd_r
},
5668 gen_helper_sve_st1hh_le_r
,
5669 gen_helper_sve_st1hs_le_r
,
5670 gen_helper_sve_st1hd_le_r
},
5672 gen_helper_sve_st1ss_le_r
,
5673 gen_helper_sve_st1sd_le_r
},
5675 gen_helper_sve_st1dd_le_r
} },
5676 { { gen_helper_sve_st1bb_r
,
5677 gen_helper_sve_st1bh_r
,
5678 gen_helper_sve_st1bs_r
,
5679 gen_helper_sve_st1bd_r
},
5681 gen_helper_sve_st1hh_be_r
,
5682 gen_helper_sve_st1hs_be_r
,
5683 gen_helper_sve_st1hd_be_r
},
5685 gen_helper_sve_st1ss_be_r
,
5686 gen_helper_sve_st1sd_be_r
},
5688 gen_helper_sve_st1dd_be_r
} } },
5690 { { { gen_helper_sve_st1bb_r_mte
,
5691 gen_helper_sve_st1bh_r_mte
,
5692 gen_helper_sve_st1bs_r_mte
,
5693 gen_helper_sve_st1bd_r_mte
},
5695 gen_helper_sve_st1hh_le_r_mte
,
5696 gen_helper_sve_st1hs_le_r_mte
,
5697 gen_helper_sve_st1hd_le_r_mte
},
5699 gen_helper_sve_st1ss_le_r_mte
,
5700 gen_helper_sve_st1sd_le_r_mte
},
5702 gen_helper_sve_st1dd_le_r_mte
} },
5703 { { gen_helper_sve_st1bb_r_mte
,
5704 gen_helper_sve_st1bh_r_mte
,
5705 gen_helper_sve_st1bs_r_mte
,
5706 gen_helper_sve_st1bd_r_mte
},
5708 gen_helper_sve_st1hh_be_r_mte
,
5709 gen_helper_sve_st1hs_be_r_mte
,
5710 gen_helper_sve_st1hd_be_r_mte
},
5712 gen_helper_sve_st1ss_be_r_mte
,
5713 gen_helper_sve_st1sd_be_r_mte
},
5715 gen_helper_sve_st1dd_be_r_mte
} } },
5717 static gen_helper_gvec_mem
* const fn_multiple
[2][2][3][4] = {
5718 { { { gen_helper_sve_st2bb_r
,
5719 gen_helper_sve_st2hh_le_r
,
5720 gen_helper_sve_st2ss_le_r
,
5721 gen_helper_sve_st2dd_le_r
},
5722 { gen_helper_sve_st3bb_r
,
5723 gen_helper_sve_st3hh_le_r
,
5724 gen_helper_sve_st3ss_le_r
,
5725 gen_helper_sve_st3dd_le_r
},
5726 { gen_helper_sve_st4bb_r
,
5727 gen_helper_sve_st4hh_le_r
,
5728 gen_helper_sve_st4ss_le_r
,
5729 gen_helper_sve_st4dd_le_r
} },
5730 { { gen_helper_sve_st2bb_r
,
5731 gen_helper_sve_st2hh_be_r
,
5732 gen_helper_sve_st2ss_be_r
,
5733 gen_helper_sve_st2dd_be_r
},
5734 { gen_helper_sve_st3bb_r
,
5735 gen_helper_sve_st3hh_be_r
,
5736 gen_helper_sve_st3ss_be_r
,
5737 gen_helper_sve_st3dd_be_r
},
5738 { gen_helper_sve_st4bb_r
,
5739 gen_helper_sve_st4hh_be_r
,
5740 gen_helper_sve_st4ss_be_r
,
5741 gen_helper_sve_st4dd_be_r
} } },
5742 { { { gen_helper_sve_st2bb_r_mte
,
5743 gen_helper_sve_st2hh_le_r_mte
,
5744 gen_helper_sve_st2ss_le_r_mte
,
5745 gen_helper_sve_st2dd_le_r_mte
},
5746 { gen_helper_sve_st3bb_r_mte
,
5747 gen_helper_sve_st3hh_le_r_mte
,
5748 gen_helper_sve_st3ss_le_r_mte
,
5749 gen_helper_sve_st3dd_le_r_mte
},
5750 { gen_helper_sve_st4bb_r_mte
,
5751 gen_helper_sve_st4hh_le_r_mte
,
5752 gen_helper_sve_st4ss_le_r_mte
,
5753 gen_helper_sve_st4dd_le_r_mte
} },
5754 { { gen_helper_sve_st2bb_r_mte
,
5755 gen_helper_sve_st2hh_be_r_mte
,
5756 gen_helper_sve_st2ss_be_r_mte
,
5757 gen_helper_sve_st2dd_be_r_mte
},
5758 { gen_helper_sve_st3bb_r_mte
,
5759 gen_helper_sve_st3hh_be_r_mte
,
5760 gen_helper_sve_st3ss_be_r_mte
,
5761 gen_helper_sve_st3dd_be_r_mte
},
5762 { gen_helper_sve_st4bb_r_mte
,
5763 gen_helper_sve_st4hh_be_r_mte
,
5764 gen_helper_sve_st4ss_be_r_mte
,
5765 gen_helper_sve_st4dd_be_r_mte
} } },
5767 gen_helper_gvec_mem
*fn
;
5768 int be
= s
->be_data
== MO_BE
;
5772 fn
= fn_single
[s
->mte_active
[0]][be
][msz
][esz
];
5775 /* ST2, ST3, ST4 -- msz == esz, enforced by encoding */
5777 fn
= fn_multiple
[s
->mte_active
[0]][be
][nreg
- 1][msz
];
5780 do_mem_zpa(s
, zt
, pg
, addr
, msz_dtype(s
, msz
), nreg
, true, fn
);
5783 static bool trans_ST_zprr(DisasContext
*s
, arg_rprr_store
*a
)
5785 if (a
->rm
== 31 || a
->msz
> a
->esz
) {
5788 if (sve_access_check(s
)) {
5789 TCGv_i64 addr
= new_tmp_a64(s
);
5790 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), a
->msz
);
5791 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5792 do_st_zpa(s
, a
->rd
, a
->pg
, addr
, a
->msz
, a
->esz
, a
->nreg
);
5797 static bool trans_ST_zpri(DisasContext
*s
, arg_rpri_store
*a
)
5799 if (a
->msz
> a
->esz
) {
5802 if (sve_access_check(s
)) {
5803 int vsz
= vec_full_reg_size(s
);
5804 int elements
= vsz
>> a
->esz
;
5805 TCGv_i64 addr
= new_tmp_a64(s
);
5807 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
),
5808 (a
->imm
* elements
* (a
->nreg
+ 1)) << a
->msz
);
5809 do_st_zpa(s
, a
->rd
, a
->pg
, addr
, a
->msz
, a
->esz
, a
->nreg
);
5815 *** SVE gather loads / scatter stores
5818 static void do_mem_zpz(DisasContext
*s
, int zt
, int pg
, int zm
,
5819 int scale
, TCGv_i64 scalar
, int msz
, bool is_write
,
5820 gen_helper_gvec_mem_scatter
*fn
)
5822 unsigned vsz
= vec_full_reg_size(s
);
5823 TCGv_ptr t_zm
= tcg_temp_new_ptr();
5824 TCGv_ptr t_pg
= tcg_temp_new_ptr();
5825 TCGv_ptr t_zt
= tcg_temp_new_ptr();
5829 if (s
->mte_active
[0]) {
5830 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, get_mem_index(s
));
5831 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
5832 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
5833 desc
= FIELD_DP32(desc
, MTEDESC
, WRITE
, is_write
);
5834 desc
= FIELD_DP32(desc
, MTEDESC
, SIZEM1
, (1 << msz
) - 1);
5835 desc
<<= SVE_MTEDESC_SHIFT
;
5837 desc
= simd_desc(vsz
, vsz
, desc
| scale
);
5838 t_desc
= tcg_const_i32(desc
);
5840 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
5841 tcg_gen_addi_ptr(t_zm
, cpu_env
, vec_full_reg_offset(s
, zm
));
5842 tcg_gen_addi_ptr(t_zt
, cpu_env
, vec_full_reg_offset(s
, zt
));
5843 fn(cpu_env
, t_zt
, t_pg
, t_zm
, scalar
, t_desc
);
5845 tcg_temp_free_ptr(t_zt
);
5846 tcg_temp_free_ptr(t_zm
);
5847 tcg_temp_free_ptr(t_pg
);
5848 tcg_temp_free_i32(t_desc
);
5851 /* Indexed by [mte][be][ff][xs][u][msz]. */
5852 static gen_helper_gvec_mem_scatter
* const
5853 gather_load_fn32
[2][2][2][2][2][3] = {
5854 { /* MTE Inactive */
5855 { /* Little-endian */
5856 { { { gen_helper_sve_ldbss_zsu
,
5857 gen_helper_sve_ldhss_le_zsu
,
5859 { gen_helper_sve_ldbsu_zsu
,
5860 gen_helper_sve_ldhsu_le_zsu
,
5861 gen_helper_sve_ldss_le_zsu
, } },
5862 { { gen_helper_sve_ldbss_zss
,
5863 gen_helper_sve_ldhss_le_zss
,
5865 { gen_helper_sve_ldbsu_zss
,
5866 gen_helper_sve_ldhsu_le_zss
,
5867 gen_helper_sve_ldss_le_zss
, } } },
5870 { { { gen_helper_sve_ldffbss_zsu
,
5871 gen_helper_sve_ldffhss_le_zsu
,
5873 { gen_helper_sve_ldffbsu_zsu
,
5874 gen_helper_sve_ldffhsu_le_zsu
,
5875 gen_helper_sve_ldffss_le_zsu
, } },
5876 { { gen_helper_sve_ldffbss_zss
,
5877 gen_helper_sve_ldffhss_le_zss
,
5879 { gen_helper_sve_ldffbsu_zss
,
5880 gen_helper_sve_ldffhsu_le_zss
,
5881 gen_helper_sve_ldffss_le_zss
, } } } },
5884 { { { gen_helper_sve_ldbss_zsu
,
5885 gen_helper_sve_ldhss_be_zsu
,
5887 { gen_helper_sve_ldbsu_zsu
,
5888 gen_helper_sve_ldhsu_be_zsu
,
5889 gen_helper_sve_ldss_be_zsu
, } },
5890 { { gen_helper_sve_ldbss_zss
,
5891 gen_helper_sve_ldhss_be_zss
,
5893 { gen_helper_sve_ldbsu_zss
,
5894 gen_helper_sve_ldhsu_be_zss
,
5895 gen_helper_sve_ldss_be_zss
, } } },
5898 { { { gen_helper_sve_ldffbss_zsu
,
5899 gen_helper_sve_ldffhss_be_zsu
,
5901 { gen_helper_sve_ldffbsu_zsu
,
5902 gen_helper_sve_ldffhsu_be_zsu
,
5903 gen_helper_sve_ldffss_be_zsu
, } },
5904 { { gen_helper_sve_ldffbss_zss
,
5905 gen_helper_sve_ldffhss_be_zss
,
5907 { gen_helper_sve_ldffbsu_zss
,
5908 gen_helper_sve_ldffhsu_be_zss
,
5909 gen_helper_sve_ldffss_be_zss
, } } } } },
5911 { /* Little-endian */
5912 { { { gen_helper_sve_ldbss_zsu_mte
,
5913 gen_helper_sve_ldhss_le_zsu_mte
,
5915 { gen_helper_sve_ldbsu_zsu_mte
,
5916 gen_helper_sve_ldhsu_le_zsu_mte
,
5917 gen_helper_sve_ldss_le_zsu_mte
, } },
5918 { { gen_helper_sve_ldbss_zss_mte
,
5919 gen_helper_sve_ldhss_le_zss_mte
,
5921 { gen_helper_sve_ldbsu_zss_mte
,
5922 gen_helper_sve_ldhsu_le_zss_mte
,
5923 gen_helper_sve_ldss_le_zss_mte
, } } },
5926 { { { gen_helper_sve_ldffbss_zsu_mte
,
5927 gen_helper_sve_ldffhss_le_zsu_mte
,
5929 { gen_helper_sve_ldffbsu_zsu_mte
,
5930 gen_helper_sve_ldffhsu_le_zsu_mte
,
5931 gen_helper_sve_ldffss_le_zsu_mte
, } },
5932 { { gen_helper_sve_ldffbss_zss_mte
,
5933 gen_helper_sve_ldffhss_le_zss_mte
,
5935 { gen_helper_sve_ldffbsu_zss_mte
,
5936 gen_helper_sve_ldffhsu_le_zss_mte
,
5937 gen_helper_sve_ldffss_le_zss_mte
, } } } },
5940 { { { gen_helper_sve_ldbss_zsu_mte
,
5941 gen_helper_sve_ldhss_be_zsu_mte
,
5943 { gen_helper_sve_ldbsu_zsu_mte
,
5944 gen_helper_sve_ldhsu_be_zsu_mte
,
5945 gen_helper_sve_ldss_be_zsu_mte
, } },
5946 { { gen_helper_sve_ldbss_zss_mte
,
5947 gen_helper_sve_ldhss_be_zss_mte
,
5949 { gen_helper_sve_ldbsu_zss_mte
,
5950 gen_helper_sve_ldhsu_be_zss_mte
,
5951 gen_helper_sve_ldss_be_zss_mte
, } } },
5954 { { { gen_helper_sve_ldffbss_zsu_mte
,
5955 gen_helper_sve_ldffhss_be_zsu_mte
,
5957 { gen_helper_sve_ldffbsu_zsu_mte
,
5958 gen_helper_sve_ldffhsu_be_zsu_mte
,
5959 gen_helper_sve_ldffss_be_zsu_mte
, } },
5960 { { gen_helper_sve_ldffbss_zss_mte
,
5961 gen_helper_sve_ldffhss_be_zss_mte
,
5963 { gen_helper_sve_ldffbsu_zss_mte
,
5964 gen_helper_sve_ldffhsu_be_zss_mte
,
5965 gen_helper_sve_ldffss_be_zss_mte
, } } } } },
5968 /* Note that we overload xs=2 to indicate 64-bit offset. */
5969 static gen_helper_gvec_mem_scatter
* const
5970 gather_load_fn64
[2][2][2][3][2][4] = {
5971 { /* MTE Inactive */
5972 { /* Little-endian */
5973 { { { gen_helper_sve_ldbds_zsu
,
5974 gen_helper_sve_ldhds_le_zsu
,
5975 gen_helper_sve_ldsds_le_zsu
,
5977 { gen_helper_sve_ldbdu_zsu
,
5978 gen_helper_sve_ldhdu_le_zsu
,
5979 gen_helper_sve_ldsdu_le_zsu
,
5980 gen_helper_sve_lddd_le_zsu
, } },
5981 { { gen_helper_sve_ldbds_zss
,
5982 gen_helper_sve_ldhds_le_zss
,
5983 gen_helper_sve_ldsds_le_zss
,
5985 { gen_helper_sve_ldbdu_zss
,
5986 gen_helper_sve_ldhdu_le_zss
,
5987 gen_helper_sve_ldsdu_le_zss
,
5988 gen_helper_sve_lddd_le_zss
, } },
5989 { { gen_helper_sve_ldbds_zd
,
5990 gen_helper_sve_ldhds_le_zd
,
5991 gen_helper_sve_ldsds_le_zd
,
5993 { gen_helper_sve_ldbdu_zd
,
5994 gen_helper_sve_ldhdu_le_zd
,
5995 gen_helper_sve_ldsdu_le_zd
,
5996 gen_helper_sve_lddd_le_zd
, } } },
5999 { { { gen_helper_sve_ldffbds_zsu
,
6000 gen_helper_sve_ldffhds_le_zsu
,
6001 gen_helper_sve_ldffsds_le_zsu
,
6003 { gen_helper_sve_ldffbdu_zsu
,
6004 gen_helper_sve_ldffhdu_le_zsu
,
6005 gen_helper_sve_ldffsdu_le_zsu
,
6006 gen_helper_sve_ldffdd_le_zsu
, } },
6007 { { gen_helper_sve_ldffbds_zss
,
6008 gen_helper_sve_ldffhds_le_zss
,
6009 gen_helper_sve_ldffsds_le_zss
,
6011 { gen_helper_sve_ldffbdu_zss
,
6012 gen_helper_sve_ldffhdu_le_zss
,
6013 gen_helper_sve_ldffsdu_le_zss
,
6014 gen_helper_sve_ldffdd_le_zss
, } },
6015 { { gen_helper_sve_ldffbds_zd
,
6016 gen_helper_sve_ldffhds_le_zd
,
6017 gen_helper_sve_ldffsds_le_zd
,
6019 { gen_helper_sve_ldffbdu_zd
,
6020 gen_helper_sve_ldffhdu_le_zd
,
6021 gen_helper_sve_ldffsdu_le_zd
,
6022 gen_helper_sve_ldffdd_le_zd
, } } } },
6024 { { { gen_helper_sve_ldbds_zsu
,
6025 gen_helper_sve_ldhds_be_zsu
,
6026 gen_helper_sve_ldsds_be_zsu
,
6028 { gen_helper_sve_ldbdu_zsu
,
6029 gen_helper_sve_ldhdu_be_zsu
,
6030 gen_helper_sve_ldsdu_be_zsu
,
6031 gen_helper_sve_lddd_be_zsu
, } },
6032 { { gen_helper_sve_ldbds_zss
,
6033 gen_helper_sve_ldhds_be_zss
,
6034 gen_helper_sve_ldsds_be_zss
,
6036 { gen_helper_sve_ldbdu_zss
,
6037 gen_helper_sve_ldhdu_be_zss
,
6038 gen_helper_sve_ldsdu_be_zss
,
6039 gen_helper_sve_lddd_be_zss
, } },
6040 { { gen_helper_sve_ldbds_zd
,
6041 gen_helper_sve_ldhds_be_zd
,
6042 gen_helper_sve_ldsds_be_zd
,
6044 { gen_helper_sve_ldbdu_zd
,
6045 gen_helper_sve_ldhdu_be_zd
,
6046 gen_helper_sve_ldsdu_be_zd
,
6047 gen_helper_sve_lddd_be_zd
, } } },
6050 { { { gen_helper_sve_ldffbds_zsu
,
6051 gen_helper_sve_ldffhds_be_zsu
,
6052 gen_helper_sve_ldffsds_be_zsu
,
6054 { gen_helper_sve_ldffbdu_zsu
,
6055 gen_helper_sve_ldffhdu_be_zsu
,
6056 gen_helper_sve_ldffsdu_be_zsu
,
6057 gen_helper_sve_ldffdd_be_zsu
, } },
6058 { { gen_helper_sve_ldffbds_zss
,
6059 gen_helper_sve_ldffhds_be_zss
,
6060 gen_helper_sve_ldffsds_be_zss
,
6062 { gen_helper_sve_ldffbdu_zss
,
6063 gen_helper_sve_ldffhdu_be_zss
,
6064 gen_helper_sve_ldffsdu_be_zss
,
6065 gen_helper_sve_ldffdd_be_zss
, } },
6066 { { gen_helper_sve_ldffbds_zd
,
6067 gen_helper_sve_ldffhds_be_zd
,
6068 gen_helper_sve_ldffsds_be_zd
,
6070 { gen_helper_sve_ldffbdu_zd
,
6071 gen_helper_sve_ldffhdu_be_zd
,
6072 gen_helper_sve_ldffsdu_be_zd
,
6073 gen_helper_sve_ldffdd_be_zd
, } } } } },
6075 { /* Little-endian */
6076 { { { gen_helper_sve_ldbds_zsu_mte
,
6077 gen_helper_sve_ldhds_le_zsu_mte
,
6078 gen_helper_sve_ldsds_le_zsu_mte
,
6080 { gen_helper_sve_ldbdu_zsu_mte
,
6081 gen_helper_sve_ldhdu_le_zsu_mte
,
6082 gen_helper_sve_ldsdu_le_zsu_mte
,
6083 gen_helper_sve_lddd_le_zsu_mte
, } },
6084 { { gen_helper_sve_ldbds_zss_mte
,
6085 gen_helper_sve_ldhds_le_zss_mte
,
6086 gen_helper_sve_ldsds_le_zss_mte
,
6088 { gen_helper_sve_ldbdu_zss_mte
,
6089 gen_helper_sve_ldhdu_le_zss_mte
,
6090 gen_helper_sve_ldsdu_le_zss_mte
,
6091 gen_helper_sve_lddd_le_zss_mte
, } },
6092 { { gen_helper_sve_ldbds_zd_mte
,
6093 gen_helper_sve_ldhds_le_zd_mte
,
6094 gen_helper_sve_ldsds_le_zd_mte
,
6096 { gen_helper_sve_ldbdu_zd_mte
,
6097 gen_helper_sve_ldhdu_le_zd_mte
,
6098 gen_helper_sve_ldsdu_le_zd_mte
,
6099 gen_helper_sve_lddd_le_zd_mte
, } } },
6102 { { { gen_helper_sve_ldffbds_zsu_mte
,
6103 gen_helper_sve_ldffhds_le_zsu_mte
,
6104 gen_helper_sve_ldffsds_le_zsu_mte
,
6106 { gen_helper_sve_ldffbdu_zsu_mte
,
6107 gen_helper_sve_ldffhdu_le_zsu_mte
,
6108 gen_helper_sve_ldffsdu_le_zsu_mte
,
6109 gen_helper_sve_ldffdd_le_zsu_mte
, } },
6110 { { gen_helper_sve_ldffbds_zss_mte
,
6111 gen_helper_sve_ldffhds_le_zss_mte
,
6112 gen_helper_sve_ldffsds_le_zss_mte
,
6114 { gen_helper_sve_ldffbdu_zss_mte
,
6115 gen_helper_sve_ldffhdu_le_zss_mte
,
6116 gen_helper_sve_ldffsdu_le_zss_mte
,
6117 gen_helper_sve_ldffdd_le_zss_mte
, } },
6118 { { gen_helper_sve_ldffbds_zd_mte
,
6119 gen_helper_sve_ldffhds_le_zd_mte
,
6120 gen_helper_sve_ldffsds_le_zd_mte
,
6122 { gen_helper_sve_ldffbdu_zd_mte
,
6123 gen_helper_sve_ldffhdu_le_zd_mte
,
6124 gen_helper_sve_ldffsdu_le_zd_mte
,
6125 gen_helper_sve_ldffdd_le_zd_mte
, } } } },
6127 { { { gen_helper_sve_ldbds_zsu_mte
,
6128 gen_helper_sve_ldhds_be_zsu_mte
,
6129 gen_helper_sve_ldsds_be_zsu_mte
,
6131 { gen_helper_sve_ldbdu_zsu_mte
,
6132 gen_helper_sve_ldhdu_be_zsu_mte
,
6133 gen_helper_sve_ldsdu_be_zsu_mte
,
6134 gen_helper_sve_lddd_be_zsu_mte
, } },
6135 { { gen_helper_sve_ldbds_zss_mte
,
6136 gen_helper_sve_ldhds_be_zss_mte
,
6137 gen_helper_sve_ldsds_be_zss_mte
,
6139 { gen_helper_sve_ldbdu_zss_mte
,
6140 gen_helper_sve_ldhdu_be_zss_mte
,
6141 gen_helper_sve_ldsdu_be_zss_mte
,
6142 gen_helper_sve_lddd_be_zss_mte
, } },
6143 { { gen_helper_sve_ldbds_zd_mte
,
6144 gen_helper_sve_ldhds_be_zd_mte
,
6145 gen_helper_sve_ldsds_be_zd_mte
,
6147 { gen_helper_sve_ldbdu_zd_mte
,
6148 gen_helper_sve_ldhdu_be_zd_mte
,
6149 gen_helper_sve_ldsdu_be_zd_mte
,
6150 gen_helper_sve_lddd_be_zd_mte
, } } },
6153 { { { gen_helper_sve_ldffbds_zsu_mte
,
6154 gen_helper_sve_ldffhds_be_zsu_mte
,
6155 gen_helper_sve_ldffsds_be_zsu_mte
,
6157 { gen_helper_sve_ldffbdu_zsu_mte
,
6158 gen_helper_sve_ldffhdu_be_zsu_mte
,
6159 gen_helper_sve_ldffsdu_be_zsu_mte
,
6160 gen_helper_sve_ldffdd_be_zsu_mte
, } },
6161 { { gen_helper_sve_ldffbds_zss_mte
,
6162 gen_helper_sve_ldffhds_be_zss_mte
,
6163 gen_helper_sve_ldffsds_be_zss_mte
,
6165 { gen_helper_sve_ldffbdu_zss_mte
,
6166 gen_helper_sve_ldffhdu_be_zss_mte
,
6167 gen_helper_sve_ldffsdu_be_zss_mte
,
6168 gen_helper_sve_ldffdd_be_zss_mte
, } },
6169 { { gen_helper_sve_ldffbds_zd_mte
,
6170 gen_helper_sve_ldffhds_be_zd_mte
,
6171 gen_helper_sve_ldffsds_be_zd_mte
,
6173 { gen_helper_sve_ldffbdu_zd_mte
,
6174 gen_helper_sve_ldffhdu_be_zd_mte
,
6175 gen_helper_sve_ldffsdu_be_zd_mte
,
6176 gen_helper_sve_ldffdd_be_zd_mte
, } } } } },
6179 static bool trans_LD1_zprz(DisasContext
*s
, arg_LD1_zprz
*a
)
6181 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6182 bool be
= s
->be_data
== MO_BE
;
6183 bool mte
= s
->mte_active
[0];
6185 if (!sve_access_check(s
)) {
6191 fn
= gather_load_fn32
[mte
][be
][a
->ff
][a
->xs
][a
->u
][a
->msz
];
6194 fn
= gather_load_fn64
[mte
][be
][a
->ff
][a
->xs
][a
->u
][a
->msz
];
6199 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rm
, a
->scale
* a
->msz
,
6200 cpu_reg_sp(s
, a
->rn
), a
->msz
, false, fn
);
6204 static bool trans_LD1_zpiz(DisasContext
*s
, arg_LD1_zpiz
*a
)
6206 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6207 bool be
= s
->be_data
== MO_BE
;
6208 bool mte
= s
->mte_active
[0];
6211 if (a
->esz
< a
->msz
|| (a
->esz
== a
->msz
&& !a
->u
)) {
6214 if (!sve_access_check(s
)) {
6220 fn
= gather_load_fn32
[mte
][be
][a
->ff
][0][a
->u
][a
->msz
];
6223 fn
= gather_load_fn64
[mte
][be
][a
->ff
][2][a
->u
][a
->msz
];
6228 /* Treat LD1_zpiz (zn[x] + imm) the same way as LD1_zprz (rn + zm[x])
6229 * by loading the immediate into the scalar parameter.
6231 imm
= tcg_const_i64(a
->imm
<< a
->msz
);
6232 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0, imm
, a
->msz
, false, fn
);
6233 tcg_temp_free_i64(imm
);
6237 static bool trans_LDNT1_zprz(DisasContext
*s
, arg_LD1_zprz
*a
)
6239 if (!dc_isar_feature(aa64_sve2
, s
)) {
6242 return trans_LD1_zprz(s
, a
);
6245 /* Indexed by [mte][be][xs][msz]. */
6246 static gen_helper_gvec_mem_scatter
* const scatter_store_fn32
[2][2][2][3] = {
6247 { /* MTE Inactive */
6248 { /* Little-endian */
6249 { gen_helper_sve_stbs_zsu
,
6250 gen_helper_sve_sths_le_zsu
,
6251 gen_helper_sve_stss_le_zsu
, },
6252 { gen_helper_sve_stbs_zss
,
6253 gen_helper_sve_sths_le_zss
,
6254 gen_helper_sve_stss_le_zss
, } },
6256 { gen_helper_sve_stbs_zsu
,
6257 gen_helper_sve_sths_be_zsu
,
6258 gen_helper_sve_stss_be_zsu
, },
6259 { gen_helper_sve_stbs_zss
,
6260 gen_helper_sve_sths_be_zss
,
6261 gen_helper_sve_stss_be_zss
, } } },
6263 { /* Little-endian */
6264 { gen_helper_sve_stbs_zsu_mte
,
6265 gen_helper_sve_sths_le_zsu_mte
,
6266 gen_helper_sve_stss_le_zsu_mte
, },
6267 { gen_helper_sve_stbs_zss_mte
,
6268 gen_helper_sve_sths_le_zss_mte
,
6269 gen_helper_sve_stss_le_zss_mte
, } },
6271 { gen_helper_sve_stbs_zsu_mte
,
6272 gen_helper_sve_sths_be_zsu_mte
,
6273 gen_helper_sve_stss_be_zsu_mte
, },
6274 { gen_helper_sve_stbs_zss_mte
,
6275 gen_helper_sve_sths_be_zss_mte
,
6276 gen_helper_sve_stss_be_zss_mte
, } } },
6279 /* Note that we overload xs=2 to indicate 64-bit offset. */
6280 static gen_helper_gvec_mem_scatter
* const scatter_store_fn64
[2][2][3][4] = {
6281 { /* MTE Inactive */
6282 { /* Little-endian */
6283 { gen_helper_sve_stbd_zsu
,
6284 gen_helper_sve_sthd_le_zsu
,
6285 gen_helper_sve_stsd_le_zsu
,
6286 gen_helper_sve_stdd_le_zsu
, },
6287 { gen_helper_sve_stbd_zss
,
6288 gen_helper_sve_sthd_le_zss
,
6289 gen_helper_sve_stsd_le_zss
,
6290 gen_helper_sve_stdd_le_zss
, },
6291 { gen_helper_sve_stbd_zd
,
6292 gen_helper_sve_sthd_le_zd
,
6293 gen_helper_sve_stsd_le_zd
,
6294 gen_helper_sve_stdd_le_zd
, } },
6296 { gen_helper_sve_stbd_zsu
,
6297 gen_helper_sve_sthd_be_zsu
,
6298 gen_helper_sve_stsd_be_zsu
,
6299 gen_helper_sve_stdd_be_zsu
, },
6300 { gen_helper_sve_stbd_zss
,
6301 gen_helper_sve_sthd_be_zss
,
6302 gen_helper_sve_stsd_be_zss
,
6303 gen_helper_sve_stdd_be_zss
, },
6304 { gen_helper_sve_stbd_zd
,
6305 gen_helper_sve_sthd_be_zd
,
6306 gen_helper_sve_stsd_be_zd
,
6307 gen_helper_sve_stdd_be_zd
, } } },
6308 { /* MTE Inactive */
6309 { /* Little-endian */
6310 { gen_helper_sve_stbd_zsu_mte
,
6311 gen_helper_sve_sthd_le_zsu_mte
,
6312 gen_helper_sve_stsd_le_zsu_mte
,
6313 gen_helper_sve_stdd_le_zsu_mte
, },
6314 { gen_helper_sve_stbd_zss_mte
,
6315 gen_helper_sve_sthd_le_zss_mte
,
6316 gen_helper_sve_stsd_le_zss_mte
,
6317 gen_helper_sve_stdd_le_zss_mte
, },
6318 { gen_helper_sve_stbd_zd_mte
,
6319 gen_helper_sve_sthd_le_zd_mte
,
6320 gen_helper_sve_stsd_le_zd_mte
,
6321 gen_helper_sve_stdd_le_zd_mte
, } },
6323 { gen_helper_sve_stbd_zsu_mte
,
6324 gen_helper_sve_sthd_be_zsu_mte
,
6325 gen_helper_sve_stsd_be_zsu_mte
,
6326 gen_helper_sve_stdd_be_zsu_mte
, },
6327 { gen_helper_sve_stbd_zss_mte
,
6328 gen_helper_sve_sthd_be_zss_mte
,
6329 gen_helper_sve_stsd_be_zss_mte
,
6330 gen_helper_sve_stdd_be_zss_mte
, },
6331 { gen_helper_sve_stbd_zd_mte
,
6332 gen_helper_sve_sthd_be_zd_mte
,
6333 gen_helper_sve_stsd_be_zd_mte
,
6334 gen_helper_sve_stdd_be_zd_mte
, } } },
6337 static bool trans_ST1_zprz(DisasContext
*s
, arg_ST1_zprz
*a
)
6339 gen_helper_gvec_mem_scatter
*fn
;
6340 bool be
= s
->be_data
== MO_BE
;
6341 bool mte
= s
->mte_active
[0];
6343 if (a
->esz
< a
->msz
|| (a
->msz
== 0 && a
->scale
)) {
6346 if (!sve_access_check(s
)) {
6351 fn
= scatter_store_fn32
[mte
][be
][a
->xs
][a
->msz
];
6354 fn
= scatter_store_fn64
[mte
][be
][a
->xs
][a
->msz
];
6357 g_assert_not_reached();
6359 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rm
, a
->scale
* a
->msz
,
6360 cpu_reg_sp(s
, a
->rn
), a
->msz
, true, fn
);
6364 static bool trans_ST1_zpiz(DisasContext
*s
, arg_ST1_zpiz
*a
)
6366 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6367 bool be
= s
->be_data
== MO_BE
;
6368 bool mte
= s
->mte_active
[0];
6371 if (a
->esz
< a
->msz
) {
6374 if (!sve_access_check(s
)) {
6380 fn
= scatter_store_fn32
[mte
][be
][0][a
->msz
];
6383 fn
= scatter_store_fn64
[mte
][be
][2][a
->msz
];
6388 /* Treat ST1_zpiz (zn[x] + imm) the same way as ST1_zprz (rn + zm[x])
6389 * by loading the immediate into the scalar parameter.
6391 imm
= tcg_const_i64(a
->imm
<< a
->msz
);
6392 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0, imm
, a
->msz
, true, fn
);
6393 tcg_temp_free_i64(imm
);
6397 static bool trans_STNT1_zprz(DisasContext
*s
, arg_ST1_zprz
*a
)
6399 if (!dc_isar_feature(aa64_sve2
, s
)) {
6402 return trans_ST1_zprz(s
, a
);
6409 static bool trans_PRF(DisasContext
*s
, arg_PRF
*a
)
6411 /* Prefetch is a nop within QEMU. */
6412 (void)sve_access_check(s
);
6416 static bool trans_PRF_rr(DisasContext
*s
, arg_PRF_rr
*a
)
6421 /* Prefetch is a nop within QEMU. */
6422 (void)sve_access_check(s
);
6429 * TODO: The implementation so far could handle predicated merging movprfx.
6430 * The helper functions as written take an extra source register to
6431 * use in the operation, but the result is only written when predication
6432 * succeeds. For unpredicated movprfx, we need to rearrange the helpers
6433 * to allow the final write back to the destination to be unconditional.
6434 * For predicated zeroing movprfx, we need to rearrange the helpers to
6435 * allow the final write back to zero inactives.
6437 * In the meantime, just emit the moves.
6440 static bool trans_MOVPRFX(DisasContext
*s
, arg_MOVPRFX
*a
)
6442 return do_mov_z(s
, a
->rd
, a
->rn
);
6445 static bool trans_MOVPRFX_m(DisasContext
*s
, arg_rpr_esz
*a
)
6447 if (sve_access_check(s
)) {
6448 do_sel_z(s
, a
->rd
, a
->rn
, a
->rd
, a
->pg
, a
->esz
);
6453 static bool trans_MOVPRFX_z(DisasContext
*s
, arg_rpr_esz
*a
)
6455 return do_movz_zpz(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
, false);
6459 * SVE2 Integer Multiply - Unpredicated
6462 static bool trans_MUL_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6464 if (!dc_isar_feature(aa64_sve2
, s
)) {
6467 if (sve_access_check(s
)) {
6468 gen_gvec_fn_zzz(s
, tcg_gen_gvec_mul
, a
->esz
, a
->rd
, a
->rn
, a
->rm
);
6473 static bool do_sve2_zzz_ool(DisasContext
*s
, arg_rrr_esz
*a
,
6474 gen_helper_gvec_3
*fn
)
6476 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
6479 if (sve_access_check(s
)) {
6480 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, 0);
6485 static bool trans_SMULH_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6487 static gen_helper_gvec_3
* const fns
[4] = {
6488 gen_helper_gvec_smulh_b
, gen_helper_gvec_smulh_h
,
6489 gen_helper_gvec_smulh_s
, gen_helper_gvec_smulh_d
,
6491 return do_sve2_zzz_ool(s
, a
, fns
[a
->esz
]);
6494 static bool trans_UMULH_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6496 static gen_helper_gvec_3
* const fns
[4] = {
6497 gen_helper_gvec_umulh_b
, gen_helper_gvec_umulh_h
,
6498 gen_helper_gvec_umulh_s
, gen_helper_gvec_umulh_d
,
6500 return do_sve2_zzz_ool(s
, a
, fns
[a
->esz
]);
6503 static bool trans_PMUL_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6505 return do_sve2_zzz_ool(s
, a
, gen_helper_gvec_pmul_b
);
6508 static bool trans_SQDMULH_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6510 static gen_helper_gvec_3
* const fns
[4] = {
6511 gen_helper_sve2_sqdmulh_b
, gen_helper_sve2_sqdmulh_h
,
6512 gen_helper_sve2_sqdmulh_s
, gen_helper_sve2_sqdmulh_d
,
6514 return do_sve2_zzz_ool(s
, a
, fns
[a
->esz
]);
6517 static bool trans_SQRDMULH_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6519 static gen_helper_gvec_3
* const fns
[4] = {
6520 gen_helper_sve2_sqrdmulh_b
, gen_helper_sve2_sqrdmulh_h
,
6521 gen_helper_sve2_sqrdmulh_s
, gen_helper_sve2_sqrdmulh_d
,
6523 return do_sve2_zzz_ool(s
, a
, fns
[a
->esz
]);
6527 * SVE2 Integer - Predicated
6530 static bool do_sve2_zpzz_ool(DisasContext
*s
, arg_rprr_esz
*a
,
6531 gen_helper_gvec_4
*fn
)
6533 if (!dc_isar_feature(aa64_sve2
, s
)) {
6536 return do_zpzz_ool(s
, a
, fn
);
6539 static bool trans_SADALP_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
6541 static gen_helper_gvec_4
* const fns
[3] = {
6542 gen_helper_sve2_sadalp_zpzz_h
,
6543 gen_helper_sve2_sadalp_zpzz_s
,
6544 gen_helper_sve2_sadalp_zpzz_d
,
6549 return do_sve2_zpzz_ool(s
, a
, fns
[a
->esz
- 1]);
6552 static bool trans_UADALP_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
6554 static gen_helper_gvec_4
* const fns
[3] = {
6555 gen_helper_sve2_uadalp_zpzz_h
,
6556 gen_helper_sve2_uadalp_zpzz_s
,
6557 gen_helper_sve2_uadalp_zpzz_d
,
6562 return do_sve2_zpzz_ool(s
, a
, fns
[a
->esz
- 1]);
6566 * SVE2 integer unary operations (predicated)
6569 static bool do_sve2_zpz_ool(DisasContext
*s
, arg_rpr_esz
*a
,
6570 gen_helper_gvec_3
*fn
)
6572 if (!dc_isar_feature(aa64_sve2
, s
)) {
6575 return do_zpz_ool(s
, a
, fn
);
6578 static bool trans_URECPE(DisasContext
*s
, arg_rpr_esz
*a
)
6583 return do_sve2_zpz_ool(s
, a
, gen_helper_sve2_urecpe_s
);
6586 static bool trans_URSQRTE(DisasContext
*s
, arg_rpr_esz
*a
)
6591 return do_sve2_zpz_ool(s
, a
, gen_helper_sve2_ursqrte_s
);
6594 static bool trans_SQABS(DisasContext
*s
, arg_rpr_esz
*a
)
6596 static gen_helper_gvec_3
* const fns
[4] = {
6597 gen_helper_sve2_sqabs_b
, gen_helper_sve2_sqabs_h
,
6598 gen_helper_sve2_sqabs_s
, gen_helper_sve2_sqabs_d
,
6600 return do_sve2_zpz_ool(s
, a
, fns
[a
->esz
]);
6603 static bool trans_SQNEG(DisasContext
*s
, arg_rpr_esz
*a
)
6605 static gen_helper_gvec_3
* const fns
[4] = {
6606 gen_helper_sve2_sqneg_b
, gen_helper_sve2_sqneg_h
,
6607 gen_helper_sve2_sqneg_s
, gen_helper_sve2_sqneg_d
,
6609 return do_sve2_zpz_ool(s
, a
, fns
[a
->esz
]);
6612 #define DO_SVE2_ZPZZ(NAME, name) \
6613 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
6615 static gen_helper_gvec_4 * const fns[4] = { \
6616 gen_helper_sve2_##name##_zpzz_b, gen_helper_sve2_##name##_zpzz_h, \
6617 gen_helper_sve2_##name##_zpzz_s, gen_helper_sve2_##name##_zpzz_d, \
6619 return do_sve2_zpzz_ool(s, a, fns[a->esz]); \
6622 DO_SVE2_ZPZZ(SQSHL
, sqshl
)
6623 DO_SVE2_ZPZZ(SQRSHL
, sqrshl
)
6624 DO_SVE2_ZPZZ(SRSHL
, srshl
)
6626 DO_SVE2_ZPZZ(UQSHL
, uqshl
)
6627 DO_SVE2_ZPZZ(UQRSHL
, uqrshl
)
6628 DO_SVE2_ZPZZ(URSHL
, urshl
)
6630 DO_SVE2_ZPZZ(SHADD
, shadd
)
6631 DO_SVE2_ZPZZ(SRHADD
, srhadd
)
6632 DO_SVE2_ZPZZ(SHSUB
, shsub
)
6634 DO_SVE2_ZPZZ(UHADD
, uhadd
)
6635 DO_SVE2_ZPZZ(URHADD
, urhadd
)
6636 DO_SVE2_ZPZZ(UHSUB
, uhsub
)
6638 DO_SVE2_ZPZZ(ADDP
, addp
)
6639 DO_SVE2_ZPZZ(SMAXP
, smaxp
)
6640 DO_SVE2_ZPZZ(UMAXP
, umaxp
)
6641 DO_SVE2_ZPZZ(SMINP
, sminp
)
6642 DO_SVE2_ZPZZ(UMINP
, uminp
)
6644 DO_SVE2_ZPZZ(SQADD_zpzz
, sqadd
)
6645 DO_SVE2_ZPZZ(UQADD_zpzz
, uqadd
)
6646 DO_SVE2_ZPZZ(SQSUB_zpzz
, sqsub
)
6647 DO_SVE2_ZPZZ(UQSUB_zpzz
, uqsub
)
6648 DO_SVE2_ZPZZ(SUQADD
, suqadd
)
6649 DO_SVE2_ZPZZ(USQADD
, usqadd
)
6652 * SVE2 Widening Integer Arithmetic
6655 static bool do_sve2_zzw_ool(DisasContext
*s
, arg_rrr_esz
*a
,
6656 gen_helper_gvec_3
*fn
, int data
)
6658 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
6661 if (sve_access_check(s
)) {
6662 unsigned vsz
= vec_full_reg_size(s
);
6663 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, a
->rd
),
6664 vec_full_reg_offset(s
, a
->rn
),
6665 vec_full_reg_offset(s
, a
->rm
),
6666 vsz
, vsz
, data
, fn
);
6671 #define DO_SVE2_ZZZ_TB(NAME, name, SEL1, SEL2) \
6672 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
6674 static gen_helper_gvec_3 * const fns[4] = { \
6675 NULL, gen_helper_sve2_##name##_h, \
6676 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
6678 return do_sve2_zzw_ool(s, a, fns[a->esz], (SEL2 << 1) | SEL1); \
6681 DO_SVE2_ZZZ_TB(SADDLB
, saddl
, false, false)
6682 DO_SVE2_ZZZ_TB(SSUBLB
, ssubl
, false, false)
6683 DO_SVE2_ZZZ_TB(SABDLB
, sabdl
, false, false)
6685 DO_SVE2_ZZZ_TB(UADDLB
, uaddl
, false, false)
6686 DO_SVE2_ZZZ_TB(USUBLB
, usubl
, false, false)
6687 DO_SVE2_ZZZ_TB(UABDLB
, uabdl
, false, false)
6689 DO_SVE2_ZZZ_TB(SADDLT
, saddl
, true, true)
6690 DO_SVE2_ZZZ_TB(SSUBLT
, ssubl
, true, true)
6691 DO_SVE2_ZZZ_TB(SABDLT
, sabdl
, true, true)
6693 DO_SVE2_ZZZ_TB(UADDLT
, uaddl
, true, true)
6694 DO_SVE2_ZZZ_TB(USUBLT
, usubl
, true, true)
6695 DO_SVE2_ZZZ_TB(UABDLT
, uabdl
, true, true)
6697 DO_SVE2_ZZZ_TB(SADDLBT
, saddl
, false, true)
6698 DO_SVE2_ZZZ_TB(SSUBLBT
, ssubl
, false, true)
6699 DO_SVE2_ZZZ_TB(SSUBLTB
, ssubl
, true, false)
6701 DO_SVE2_ZZZ_TB(SQDMULLB_zzz
, sqdmull_zzz
, false, false)
6702 DO_SVE2_ZZZ_TB(SQDMULLT_zzz
, sqdmull_zzz
, true, true)
6704 DO_SVE2_ZZZ_TB(SMULLB_zzz
, smull_zzz
, false, false)
6705 DO_SVE2_ZZZ_TB(SMULLT_zzz
, smull_zzz
, true, true)
6707 DO_SVE2_ZZZ_TB(UMULLB_zzz
, umull_zzz
, false, false)
6708 DO_SVE2_ZZZ_TB(UMULLT_zzz
, umull_zzz
, true, true)
6710 static bool do_eor_tb(DisasContext
*s
, arg_rrr_esz
*a
, bool sel1
)
6712 static gen_helper_gvec_3
* const fns
[4] = {
6713 gen_helper_sve2_eoril_b
, gen_helper_sve2_eoril_h
,
6714 gen_helper_sve2_eoril_s
, gen_helper_sve2_eoril_d
,
6716 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], (!sel1
<< 1) | sel1
);
6719 static bool trans_EORBT(DisasContext
*s
, arg_rrr_esz
*a
)
6721 return do_eor_tb(s
, a
, false);
6724 static bool trans_EORTB(DisasContext
*s
, arg_rrr_esz
*a
)
6726 return do_eor_tb(s
, a
, true);
6729 static bool do_trans_pmull(DisasContext
*s
, arg_rrr_esz
*a
, bool sel
)
6731 static gen_helper_gvec_3
* const fns
[4] = {
6732 gen_helper_gvec_pmull_q
, gen_helper_sve2_pmull_h
,
6733 NULL
, gen_helper_sve2_pmull_d
,
6735 if (a
->esz
== 0 && !dc_isar_feature(aa64_sve2_pmull128
, s
)) {
6738 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], sel
);
6741 static bool trans_PMULLB(DisasContext
*s
, arg_rrr_esz
*a
)
6743 return do_trans_pmull(s
, a
, false);
6746 static bool trans_PMULLT(DisasContext
*s
, arg_rrr_esz
*a
)
6748 return do_trans_pmull(s
, a
, true);
6751 #define DO_SVE2_ZZZ_WTB(NAME, name, SEL2) \
6752 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
6754 static gen_helper_gvec_3 * const fns[4] = { \
6755 NULL, gen_helper_sve2_##name##_h, \
6756 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
6758 return do_sve2_zzw_ool(s, a, fns[a->esz], SEL2); \
6761 DO_SVE2_ZZZ_WTB(SADDWB
, saddw
, false)
6762 DO_SVE2_ZZZ_WTB(SADDWT
, saddw
, true)
6763 DO_SVE2_ZZZ_WTB(SSUBWB
, ssubw
, false)
6764 DO_SVE2_ZZZ_WTB(SSUBWT
, ssubw
, true)
6766 DO_SVE2_ZZZ_WTB(UADDWB
, uaddw
, false)
6767 DO_SVE2_ZZZ_WTB(UADDWT
, uaddw
, true)
6768 DO_SVE2_ZZZ_WTB(USUBWB
, usubw
, false)
6769 DO_SVE2_ZZZ_WTB(USUBWT
, usubw
, true)
6771 static void gen_sshll_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t imm
)
6775 int halfbits
= 4 << vece
;
6778 if (shl
== halfbits
) {
6779 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6780 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(halfbits
, halfbits
));
6781 tcg_gen_and_vec(vece
, d
, n
, t
);
6782 tcg_temp_free_vec(t
);
6784 tcg_gen_sari_vec(vece
, d
, n
, halfbits
);
6785 tcg_gen_shli_vec(vece
, d
, d
, shl
);
6788 tcg_gen_shli_vec(vece
, d
, n
, halfbits
);
6789 tcg_gen_sari_vec(vece
, d
, d
, halfbits
- shl
);
6793 static void gen_ushll_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int imm
)
6795 int halfbits
= 4 << vece
;
6797 int shl
= (imm
>> 1);
6801 mask
= MAKE_64BIT_MASK(0, halfbits
);
6803 mask
= dup_const(vece
, mask
);
6805 shift
= shl
- top
* halfbits
;
6807 tcg_gen_shri_i64(d
, n
, -shift
);
6809 tcg_gen_shli_i64(d
, n
, shift
);
6811 tcg_gen_andi_i64(d
, d
, mask
);
6814 static void gen_ushll16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6816 gen_ushll_i64(MO_16
, d
, n
, imm
);
6819 static void gen_ushll32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6821 gen_ushll_i64(MO_32
, d
, n
, imm
);
6824 static void gen_ushll64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6826 gen_ushll_i64(MO_64
, d
, n
, imm
);
6829 static void gen_ushll_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t imm
)
6831 int halfbits
= 4 << vece
;
6836 if (shl
== halfbits
) {
6837 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6838 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(halfbits
, halfbits
));
6839 tcg_gen_and_vec(vece
, d
, n
, t
);
6840 tcg_temp_free_vec(t
);
6842 tcg_gen_shri_vec(vece
, d
, n
, halfbits
);
6843 tcg_gen_shli_vec(vece
, d
, d
, shl
);
6847 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6848 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
6849 tcg_gen_and_vec(vece
, d
, n
, t
);
6850 tcg_temp_free_vec(t
);
6852 tcg_gen_shli_vec(vece
, d
, n
, halfbits
);
6853 tcg_gen_shri_vec(vece
, d
, d
, halfbits
- shl
);
6858 static bool do_sve2_shll_tb(DisasContext
*s
, arg_rri_esz
*a
,
6861 static const TCGOpcode sshll_list
[] = {
6862 INDEX_op_shli_vec
, INDEX_op_sari_vec
, 0
6864 static const TCGOpcode ushll_list
[] = {
6865 INDEX_op_shli_vec
, INDEX_op_shri_vec
, 0
6867 static const GVecGen2i ops
[2][3] = {
6868 { { .fniv
= gen_sshll_vec
,
6869 .opt_opc
= sshll_list
,
6870 .fno
= gen_helper_sve2_sshll_h
,
6872 { .fniv
= gen_sshll_vec
,
6873 .opt_opc
= sshll_list
,
6874 .fno
= gen_helper_sve2_sshll_s
,
6876 { .fniv
= gen_sshll_vec
,
6877 .opt_opc
= sshll_list
,
6878 .fno
= gen_helper_sve2_sshll_d
,
6880 { { .fni8
= gen_ushll16_i64
,
6881 .fniv
= gen_ushll_vec
,
6882 .opt_opc
= ushll_list
,
6883 .fno
= gen_helper_sve2_ushll_h
,
6885 { .fni8
= gen_ushll32_i64
,
6886 .fniv
= gen_ushll_vec
,
6887 .opt_opc
= ushll_list
,
6888 .fno
= gen_helper_sve2_ushll_s
,
6890 { .fni8
= gen_ushll64_i64
,
6891 .fniv
= gen_ushll_vec
,
6892 .opt_opc
= ushll_list
,
6893 .fno
= gen_helper_sve2_ushll_d
,
6897 if (a
->esz
< 0 || a
->esz
> 2 || !dc_isar_feature(aa64_sve2
, s
)) {
6900 if (sve_access_check(s
)) {
6901 unsigned vsz
= vec_full_reg_size(s
);
6902 tcg_gen_gvec_2i(vec_full_reg_offset(s
, a
->rd
),
6903 vec_full_reg_offset(s
, a
->rn
),
6904 vsz
, vsz
, (a
->imm
<< 1) | sel
,
6910 static bool trans_SSHLLB(DisasContext
*s
, arg_rri_esz
*a
)
6912 return do_sve2_shll_tb(s
, a
, false, false);
6915 static bool trans_SSHLLT(DisasContext
*s
, arg_rri_esz
*a
)
6917 return do_sve2_shll_tb(s
, a
, true, false);
6920 static bool trans_USHLLB(DisasContext
*s
, arg_rri_esz
*a
)
6922 return do_sve2_shll_tb(s
, a
, false, true);
6925 static bool trans_USHLLT(DisasContext
*s
, arg_rri_esz
*a
)
6927 return do_sve2_shll_tb(s
, a
, true, true);
6930 static bool trans_BEXT(DisasContext
*s
, arg_rrr_esz
*a
)
6932 static gen_helper_gvec_3
* const fns
[4] = {
6933 gen_helper_sve2_bext_b
, gen_helper_sve2_bext_h
,
6934 gen_helper_sve2_bext_s
, gen_helper_sve2_bext_d
,
6936 if (!dc_isar_feature(aa64_sve2_bitperm
, s
)) {
6939 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], 0);
6942 static bool trans_BDEP(DisasContext
*s
, arg_rrr_esz
*a
)
6944 static gen_helper_gvec_3
* const fns
[4] = {
6945 gen_helper_sve2_bdep_b
, gen_helper_sve2_bdep_h
,
6946 gen_helper_sve2_bdep_s
, gen_helper_sve2_bdep_d
,
6948 if (!dc_isar_feature(aa64_sve2_bitperm
, s
)) {
6951 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], 0);
6954 static bool trans_BGRP(DisasContext
*s
, arg_rrr_esz
*a
)
6956 static gen_helper_gvec_3
* const fns
[4] = {
6957 gen_helper_sve2_bgrp_b
, gen_helper_sve2_bgrp_h
,
6958 gen_helper_sve2_bgrp_s
, gen_helper_sve2_bgrp_d
,
6960 if (!dc_isar_feature(aa64_sve2_bitperm
, s
)) {
6963 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], 0);
6966 static bool do_cadd(DisasContext
*s
, arg_rrr_esz
*a
, bool sq
, bool rot
)
6968 static gen_helper_gvec_3
* const fns
[2][4] = {
6969 { gen_helper_sve2_cadd_b
, gen_helper_sve2_cadd_h
,
6970 gen_helper_sve2_cadd_s
, gen_helper_sve2_cadd_d
},
6971 { gen_helper_sve2_sqcadd_b
, gen_helper_sve2_sqcadd_h
,
6972 gen_helper_sve2_sqcadd_s
, gen_helper_sve2_sqcadd_d
},
6974 return do_sve2_zzw_ool(s
, a
, fns
[sq
][a
->esz
], rot
);
6977 static bool trans_CADD_rot90(DisasContext
*s
, arg_rrr_esz
*a
)
6979 return do_cadd(s
, a
, false, false);
6982 static bool trans_CADD_rot270(DisasContext
*s
, arg_rrr_esz
*a
)
6984 return do_cadd(s
, a
, false, true);
6987 static bool trans_SQCADD_rot90(DisasContext
*s
, arg_rrr_esz
*a
)
6989 return do_cadd(s
, a
, true, false);
6992 static bool trans_SQCADD_rot270(DisasContext
*s
, arg_rrr_esz
*a
)
6994 return do_cadd(s
, a
, true, true);
6997 static bool do_sve2_zzzz_ool(DisasContext
*s
, arg_rrrr_esz
*a
,
6998 gen_helper_gvec_4
*fn
, int data
)
7000 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
7003 if (sve_access_check(s
)) {
7004 gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, data
);
7009 static bool do_abal(DisasContext
*s
, arg_rrrr_esz
*a
, bool uns
, bool sel
)
7011 static gen_helper_gvec_4
* const fns
[2][4] = {
7012 { NULL
, gen_helper_sve2_sabal_h
,
7013 gen_helper_sve2_sabal_s
, gen_helper_sve2_sabal_d
},
7014 { NULL
, gen_helper_sve2_uabal_h
,
7015 gen_helper_sve2_uabal_s
, gen_helper_sve2_uabal_d
},
7017 return do_sve2_zzzz_ool(s
, a
, fns
[uns
][a
->esz
], sel
);
7020 static bool trans_SABALB(DisasContext
*s
, arg_rrrr_esz
*a
)
7022 return do_abal(s
, a
, false, false);
7025 static bool trans_SABALT(DisasContext
*s
, arg_rrrr_esz
*a
)
7027 return do_abal(s
, a
, false, true);
7030 static bool trans_UABALB(DisasContext
*s
, arg_rrrr_esz
*a
)
7032 return do_abal(s
, a
, true, false);
7035 static bool trans_UABALT(DisasContext
*s
, arg_rrrr_esz
*a
)
7037 return do_abal(s
, a
, true, true);
7040 static bool do_adcl(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
7042 static gen_helper_gvec_4
* const fns
[2] = {
7043 gen_helper_sve2_adcl_s
,
7044 gen_helper_sve2_adcl_d
,
7047 * Note that in this case the ESZ field encodes both size and sign.
7048 * Split out 'subtract' into bit 1 of the data field for the helper.
7050 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
& 1], (a
->esz
& 2) | sel
);
7053 static bool trans_ADCLB(DisasContext
*s
, arg_rrrr_esz
*a
)
7055 return do_adcl(s
, a
, false);
7058 static bool trans_ADCLT(DisasContext
*s
, arg_rrrr_esz
*a
)
7060 return do_adcl(s
, a
, true);
7063 static bool do_sve2_fn2i(DisasContext
*s
, arg_rri_esz
*a
, GVecGen2iFn
*fn
)
7065 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
7068 if (sve_access_check(s
)) {
7069 unsigned vsz
= vec_full_reg_size(s
);
7070 unsigned rd_ofs
= vec_full_reg_offset(s
, a
->rd
);
7071 unsigned rn_ofs
= vec_full_reg_offset(s
, a
->rn
);
7072 fn(a
->esz
, rd_ofs
, rn_ofs
, a
->imm
, vsz
, vsz
);
7077 static bool trans_SSRA(DisasContext
*s
, arg_rri_esz
*a
)
7079 return do_sve2_fn2i(s
, a
, gen_gvec_ssra
);
7082 static bool trans_USRA(DisasContext
*s
, arg_rri_esz
*a
)
7084 return do_sve2_fn2i(s
, a
, gen_gvec_usra
);
7087 static bool trans_SRSRA(DisasContext
*s
, arg_rri_esz
*a
)
7089 return do_sve2_fn2i(s
, a
, gen_gvec_srsra
);
7092 static bool trans_URSRA(DisasContext
*s
, arg_rri_esz
*a
)
7094 return do_sve2_fn2i(s
, a
, gen_gvec_ursra
);
7097 static bool trans_SRI(DisasContext
*s
, arg_rri_esz
*a
)
7099 return do_sve2_fn2i(s
, a
, gen_gvec_sri
);
7102 static bool trans_SLI(DisasContext
*s
, arg_rri_esz
*a
)
7104 return do_sve2_fn2i(s
, a
, gen_gvec_sli
);
7107 static bool do_sve2_fn_zzz(DisasContext
*s
, arg_rrr_esz
*a
, GVecGen3Fn
*fn
)
7109 if (!dc_isar_feature(aa64_sve2
, s
)) {
7112 if (sve_access_check(s
)) {
7113 gen_gvec_fn_zzz(s
, fn
, a
->esz
, a
->rd
, a
->rn
, a
->rm
);
7118 static bool trans_SABA(DisasContext
*s
, arg_rrr_esz
*a
)
7120 return do_sve2_fn_zzz(s
, a
, gen_gvec_saba
);
7123 static bool trans_UABA(DisasContext
*s
, arg_rrr_esz
*a
)
7125 return do_sve2_fn_zzz(s
, a
, gen_gvec_uaba
);
7128 static bool do_sve2_narrow_extract(DisasContext
*s
, arg_rri_esz
*a
,
7129 const GVecGen2 ops
[3])
7131 if (a
->esz
< 0 || a
->esz
> MO_32
|| a
->imm
!= 0 ||
7132 !dc_isar_feature(aa64_sve2
, s
)) {
7135 if (sve_access_check(s
)) {
7136 unsigned vsz
= vec_full_reg_size(s
);
7137 tcg_gen_gvec_2(vec_full_reg_offset(s
, a
->rd
),
7138 vec_full_reg_offset(s
, a
->rn
),
7139 vsz
, vsz
, &ops
[a
->esz
]);
7144 static const TCGOpcode sqxtn_list
[] = {
7145 INDEX_op_shli_vec
, INDEX_op_smin_vec
, INDEX_op_smax_vec
, 0
7148 static void gen_sqxtnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7150 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7151 int halfbits
= 4 << vece
;
7152 int64_t mask
= (1ull << halfbits
) - 1;
7153 int64_t min
= -1ull << (halfbits
- 1);
7154 int64_t max
= -min
- 1;
7156 tcg_gen_dupi_vec(vece
, t
, min
);
7157 tcg_gen_smax_vec(vece
, d
, n
, t
);
7158 tcg_gen_dupi_vec(vece
, t
, max
);
7159 tcg_gen_smin_vec(vece
, d
, d
, t
);
7160 tcg_gen_dupi_vec(vece
, t
, mask
);
7161 tcg_gen_and_vec(vece
, d
, d
, t
);
7162 tcg_temp_free_vec(t
);
7165 static bool trans_SQXTNB(DisasContext
*s
, arg_rri_esz
*a
)
7167 static const GVecGen2 ops
[3] = {
7168 { .fniv
= gen_sqxtnb_vec
,
7169 .opt_opc
= sqxtn_list
,
7170 .fno
= gen_helper_sve2_sqxtnb_h
,
7172 { .fniv
= gen_sqxtnb_vec
,
7173 .opt_opc
= sqxtn_list
,
7174 .fno
= gen_helper_sve2_sqxtnb_s
,
7176 { .fniv
= gen_sqxtnb_vec
,
7177 .opt_opc
= sqxtn_list
,
7178 .fno
= gen_helper_sve2_sqxtnb_d
,
7181 return do_sve2_narrow_extract(s
, a
, ops
);
7184 static void gen_sqxtnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7186 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7187 int halfbits
= 4 << vece
;
7188 int64_t mask
= (1ull << halfbits
) - 1;
7189 int64_t min
= -1ull << (halfbits
- 1);
7190 int64_t max
= -min
- 1;
7192 tcg_gen_dupi_vec(vece
, t
, min
);
7193 tcg_gen_smax_vec(vece
, n
, n
, t
);
7194 tcg_gen_dupi_vec(vece
, t
, max
);
7195 tcg_gen_smin_vec(vece
, n
, n
, t
);
7196 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7197 tcg_gen_dupi_vec(vece
, t
, mask
);
7198 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7199 tcg_temp_free_vec(t
);
7202 static bool trans_SQXTNT(DisasContext
*s
, arg_rri_esz
*a
)
7204 static const GVecGen2 ops
[3] = {
7205 { .fniv
= gen_sqxtnt_vec
,
7206 .opt_opc
= sqxtn_list
,
7208 .fno
= gen_helper_sve2_sqxtnt_h
,
7210 { .fniv
= gen_sqxtnt_vec
,
7211 .opt_opc
= sqxtn_list
,
7213 .fno
= gen_helper_sve2_sqxtnt_s
,
7215 { .fniv
= gen_sqxtnt_vec
,
7216 .opt_opc
= sqxtn_list
,
7218 .fno
= gen_helper_sve2_sqxtnt_d
,
7221 return do_sve2_narrow_extract(s
, a
, ops
);
7224 static const TCGOpcode uqxtn_list
[] = {
7225 INDEX_op_shli_vec
, INDEX_op_umin_vec
, 0
7228 static void gen_uqxtnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7230 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7231 int halfbits
= 4 << vece
;
7232 int64_t max
= (1ull << halfbits
) - 1;
7234 tcg_gen_dupi_vec(vece
, t
, max
);
7235 tcg_gen_umin_vec(vece
, d
, n
, t
);
7236 tcg_temp_free_vec(t
);
7239 static bool trans_UQXTNB(DisasContext
*s
, arg_rri_esz
*a
)
7241 static const GVecGen2 ops
[3] = {
7242 { .fniv
= gen_uqxtnb_vec
,
7243 .opt_opc
= uqxtn_list
,
7244 .fno
= gen_helper_sve2_uqxtnb_h
,
7246 { .fniv
= gen_uqxtnb_vec
,
7247 .opt_opc
= uqxtn_list
,
7248 .fno
= gen_helper_sve2_uqxtnb_s
,
7250 { .fniv
= gen_uqxtnb_vec
,
7251 .opt_opc
= uqxtn_list
,
7252 .fno
= gen_helper_sve2_uqxtnb_d
,
7255 return do_sve2_narrow_extract(s
, a
, ops
);
7258 static void gen_uqxtnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7260 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7261 int halfbits
= 4 << vece
;
7262 int64_t max
= (1ull << halfbits
) - 1;
7264 tcg_gen_dupi_vec(vece
, t
, max
);
7265 tcg_gen_umin_vec(vece
, n
, n
, t
);
7266 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7267 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7268 tcg_temp_free_vec(t
);
7271 static bool trans_UQXTNT(DisasContext
*s
, arg_rri_esz
*a
)
7273 static const GVecGen2 ops
[3] = {
7274 { .fniv
= gen_uqxtnt_vec
,
7275 .opt_opc
= uqxtn_list
,
7277 .fno
= gen_helper_sve2_uqxtnt_h
,
7279 { .fniv
= gen_uqxtnt_vec
,
7280 .opt_opc
= uqxtn_list
,
7282 .fno
= gen_helper_sve2_uqxtnt_s
,
7284 { .fniv
= gen_uqxtnt_vec
,
7285 .opt_opc
= uqxtn_list
,
7287 .fno
= gen_helper_sve2_uqxtnt_d
,
7290 return do_sve2_narrow_extract(s
, a
, ops
);
7293 static const TCGOpcode sqxtun_list
[] = {
7294 INDEX_op_shli_vec
, INDEX_op_umin_vec
, INDEX_op_smax_vec
, 0
7297 static void gen_sqxtunb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7299 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7300 int halfbits
= 4 << vece
;
7301 int64_t max
= (1ull << halfbits
) - 1;
7303 tcg_gen_dupi_vec(vece
, t
, 0);
7304 tcg_gen_smax_vec(vece
, d
, n
, t
);
7305 tcg_gen_dupi_vec(vece
, t
, max
);
7306 tcg_gen_umin_vec(vece
, d
, d
, t
);
7307 tcg_temp_free_vec(t
);
7310 static bool trans_SQXTUNB(DisasContext
*s
, arg_rri_esz
*a
)
7312 static const GVecGen2 ops
[3] = {
7313 { .fniv
= gen_sqxtunb_vec
,
7314 .opt_opc
= sqxtun_list
,
7315 .fno
= gen_helper_sve2_sqxtunb_h
,
7317 { .fniv
= gen_sqxtunb_vec
,
7318 .opt_opc
= sqxtun_list
,
7319 .fno
= gen_helper_sve2_sqxtunb_s
,
7321 { .fniv
= gen_sqxtunb_vec
,
7322 .opt_opc
= sqxtun_list
,
7323 .fno
= gen_helper_sve2_sqxtunb_d
,
7326 return do_sve2_narrow_extract(s
, a
, ops
);
7329 static void gen_sqxtunt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7331 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7332 int halfbits
= 4 << vece
;
7333 int64_t max
= (1ull << halfbits
) - 1;
7335 tcg_gen_dupi_vec(vece
, t
, 0);
7336 tcg_gen_smax_vec(vece
, n
, n
, t
);
7337 tcg_gen_dupi_vec(vece
, t
, max
);
7338 tcg_gen_umin_vec(vece
, n
, n
, t
);
7339 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7340 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7341 tcg_temp_free_vec(t
);
7344 static bool trans_SQXTUNT(DisasContext
*s
, arg_rri_esz
*a
)
7346 static const GVecGen2 ops
[3] = {
7347 { .fniv
= gen_sqxtunt_vec
,
7348 .opt_opc
= sqxtun_list
,
7350 .fno
= gen_helper_sve2_sqxtunt_h
,
7352 { .fniv
= gen_sqxtunt_vec
,
7353 .opt_opc
= sqxtun_list
,
7355 .fno
= gen_helper_sve2_sqxtunt_s
,
7357 { .fniv
= gen_sqxtunt_vec
,
7358 .opt_opc
= sqxtun_list
,
7360 .fno
= gen_helper_sve2_sqxtunt_d
,
7363 return do_sve2_narrow_extract(s
, a
, ops
);
7366 static bool do_sve2_shr_narrow(DisasContext
*s
, arg_rri_esz
*a
,
7367 const GVecGen2i ops
[3])
7369 if (a
->esz
< 0 || a
->esz
> MO_32
|| !dc_isar_feature(aa64_sve2
, s
)) {
7372 assert(a
->imm
> 0 && a
->imm
<= (8 << a
->esz
));
7373 if (sve_access_check(s
)) {
7374 unsigned vsz
= vec_full_reg_size(s
);
7375 tcg_gen_gvec_2i(vec_full_reg_offset(s
, a
->rd
),
7376 vec_full_reg_offset(s
, a
->rn
),
7377 vsz
, vsz
, a
->imm
, &ops
[a
->esz
]);
7382 static void gen_shrnb_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int shr
)
7384 int halfbits
= 4 << vece
;
7385 uint64_t mask
= dup_const(vece
, MAKE_64BIT_MASK(0, halfbits
));
7387 tcg_gen_shri_i64(d
, n
, shr
);
7388 tcg_gen_andi_i64(d
, d
, mask
);
7391 static void gen_shrnb16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7393 gen_shrnb_i64(MO_16
, d
, n
, shr
);
7396 static void gen_shrnb32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7398 gen_shrnb_i64(MO_32
, d
, n
, shr
);
7401 static void gen_shrnb64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7403 gen_shrnb_i64(MO_64
, d
, n
, shr
);
7406 static void gen_shrnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t shr
)
7408 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7409 int halfbits
= 4 << vece
;
7410 uint64_t mask
= MAKE_64BIT_MASK(0, halfbits
);
7412 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7413 tcg_gen_dupi_vec(vece
, t
, mask
);
7414 tcg_gen_and_vec(vece
, d
, n
, t
);
7415 tcg_temp_free_vec(t
);
7418 static bool trans_SHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7420 static const TCGOpcode vec_list
[] = { INDEX_op_shri_vec
, 0 };
7421 static const GVecGen2i ops
[3] = {
7422 { .fni8
= gen_shrnb16_i64
,
7423 .fniv
= gen_shrnb_vec
,
7424 .opt_opc
= vec_list
,
7425 .fno
= gen_helper_sve2_shrnb_h
,
7427 { .fni8
= gen_shrnb32_i64
,
7428 .fniv
= gen_shrnb_vec
,
7429 .opt_opc
= vec_list
,
7430 .fno
= gen_helper_sve2_shrnb_s
,
7432 { .fni8
= gen_shrnb64_i64
,
7433 .fniv
= gen_shrnb_vec
,
7434 .opt_opc
= vec_list
,
7435 .fno
= gen_helper_sve2_shrnb_d
,
7438 return do_sve2_shr_narrow(s
, a
, ops
);
7441 static void gen_shrnt_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int shr
)
7443 int halfbits
= 4 << vece
;
7444 uint64_t mask
= dup_const(vece
, MAKE_64BIT_MASK(0, halfbits
));
7446 tcg_gen_shli_i64(n
, n
, halfbits
- shr
);
7447 tcg_gen_andi_i64(n
, n
, ~mask
);
7448 tcg_gen_andi_i64(d
, d
, mask
);
7449 tcg_gen_or_i64(d
, d
, n
);
7452 static void gen_shrnt16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7454 gen_shrnt_i64(MO_16
, d
, n
, shr
);
7457 static void gen_shrnt32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7459 gen_shrnt_i64(MO_32
, d
, n
, shr
);
7462 static void gen_shrnt64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7464 tcg_gen_shri_i64(n
, n
, shr
);
7465 tcg_gen_deposit_i64(d
, d
, n
, 32, 32);
7468 static void gen_shrnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t shr
)
7470 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7471 int halfbits
= 4 << vece
;
7472 uint64_t mask
= MAKE_64BIT_MASK(0, halfbits
);
7474 tcg_gen_shli_vec(vece
, n
, n
, halfbits
- shr
);
7475 tcg_gen_dupi_vec(vece
, t
, mask
);
7476 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7477 tcg_temp_free_vec(t
);
7480 static bool trans_SHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7482 static const TCGOpcode vec_list
[] = { INDEX_op_shli_vec
, 0 };
7483 static const GVecGen2i ops
[3] = {
7484 { .fni8
= gen_shrnt16_i64
,
7485 .fniv
= gen_shrnt_vec
,
7486 .opt_opc
= vec_list
,
7488 .fno
= gen_helper_sve2_shrnt_h
,
7490 { .fni8
= gen_shrnt32_i64
,
7491 .fniv
= gen_shrnt_vec
,
7492 .opt_opc
= vec_list
,
7494 .fno
= gen_helper_sve2_shrnt_s
,
7496 { .fni8
= gen_shrnt64_i64
,
7497 .fniv
= gen_shrnt_vec
,
7498 .opt_opc
= vec_list
,
7500 .fno
= gen_helper_sve2_shrnt_d
,
7503 return do_sve2_shr_narrow(s
, a
, ops
);
7506 static bool trans_RSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7508 static const GVecGen2i ops
[3] = {
7509 { .fno
= gen_helper_sve2_rshrnb_h
},
7510 { .fno
= gen_helper_sve2_rshrnb_s
},
7511 { .fno
= gen_helper_sve2_rshrnb_d
},
7513 return do_sve2_shr_narrow(s
, a
, ops
);
7516 static bool trans_RSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7518 static const GVecGen2i ops
[3] = {
7519 { .fno
= gen_helper_sve2_rshrnt_h
},
7520 { .fno
= gen_helper_sve2_rshrnt_s
},
7521 { .fno
= gen_helper_sve2_rshrnt_d
},
7523 return do_sve2_shr_narrow(s
, a
, ops
);
7526 static void gen_sqshrunb_vec(unsigned vece
, TCGv_vec d
,
7527 TCGv_vec n
, int64_t shr
)
7529 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7530 int halfbits
= 4 << vece
;
7532 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7533 tcg_gen_dupi_vec(vece
, t
, 0);
7534 tcg_gen_smax_vec(vece
, n
, n
, t
);
7535 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7536 tcg_gen_umin_vec(vece
, d
, n
, t
);
7537 tcg_temp_free_vec(t
);
7540 static bool trans_SQSHRUNB(DisasContext
*s
, arg_rri_esz
*a
)
7542 static const TCGOpcode vec_list
[] = {
7543 INDEX_op_sari_vec
, INDEX_op_smax_vec
, INDEX_op_umin_vec
, 0
7545 static const GVecGen2i ops
[3] = {
7546 { .fniv
= gen_sqshrunb_vec
,
7547 .opt_opc
= vec_list
,
7548 .fno
= gen_helper_sve2_sqshrunb_h
,
7550 { .fniv
= gen_sqshrunb_vec
,
7551 .opt_opc
= vec_list
,
7552 .fno
= gen_helper_sve2_sqshrunb_s
,
7554 { .fniv
= gen_sqshrunb_vec
,
7555 .opt_opc
= vec_list
,
7556 .fno
= gen_helper_sve2_sqshrunb_d
,
7559 return do_sve2_shr_narrow(s
, a
, ops
);
7562 static void gen_sqshrunt_vec(unsigned vece
, TCGv_vec d
,
7563 TCGv_vec n
, int64_t shr
)
7565 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7566 int halfbits
= 4 << vece
;
7568 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7569 tcg_gen_dupi_vec(vece
, t
, 0);
7570 tcg_gen_smax_vec(vece
, n
, n
, t
);
7571 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7572 tcg_gen_umin_vec(vece
, n
, n
, t
);
7573 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7574 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7575 tcg_temp_free_vec(t
);
7578 static bool trans_SQSHRUNT(DisasContext
*s
, arg_rri_esz
*a
)
7580 static const TCGOpcode vec_list
[] = {
7581 INDEX_op_shli_vec
, INDEX_op_sari_vec
,
7582 INDEX_op_smax_vec
, INDEX_op_umin_vec
, 0
7584 static const GVecGen2i ops
[3] = {
7585 { .fniv
= gen_sqshrunt_vec
,
7586 .opt_opc
= vec_list
,
7588 .fno
= gen_helper_sve2_sqshrunt_h
,
7590 { .fniv
= gen_sqshrunt_vec
,
7591 .opt_opc
= vec_list
,
7593 .fno
= gen_helper_sve2_sqshrunt_s
,
7595 { .fniv
= gen_sqshrunt_vec
,
7596 .opt_opc
= vec_list
,
7598 .fno
= gen_helper_sve2_sqshrunt_d
,
7601 return do_sve2_shr_narrow(s
, a
, ops
);
7604 static bool trans_SQRSHRUNB(DisasContext
*s
, arg_rri_esz
*a
)
7606 static const GVecGen2i ops
[3] = {
7607 { .fno
= gen_helper_sve2_sqrshrunb_h
},
7608 { .fno
= gen_helper_sve2_sqrshrunb_s
},
7609 { .fno
= gen_helper_sve2_sqrshrunb_d
},
7611 return do_sve2_shr_narrow(s
, a
, ops
);
7614 static bool trans_SQRSHRUNT(DisasContext
*s
, arg_rri_esz
*a
)
7616 static const GVecGen2i ops
[3] = {
7617 { .fno
= gen_helper_sve2_sqrshrunt_h
},
7618 { .fno
= gen_helper_sve2_sqrshrunt_s
},
7619 { .fno
= gen_helper_sve2_sqrshrunt_d
},
7621 return do_sve2_shr_narrow(s
, a
, ops
);
7624 static void gen_sqshrnb_vec(unsigned vece
, TCGv_vec d
,
7625 TCGv_vec n
, int64_t shr
)
7627 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7628 int halfbits
= 4 << vece
;
7629 int64_t max
= MAKE_64BIT_MASK(0, halfbits
- 1);
7630 int64_t min
= -max
- 1;
7632 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7633 tcg_gen_dupi_vec(vece
, t
, min
);
7634 tcg_gen_smax_vec(vece
, n
, n
, t
);
7635 tcg_gen_dupi_vec(vece
, t
, max
);
7636 tcg_gen_smin_vec(vece
, n
, n
, t
);
7637 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7638 tcg_gen_and_vec(vece
, d
, n
, t
);
7639 tcg_temp_free_vec(t
);
7642 static bool trans_SQSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7644 static const TCGOpcode vec_list
[] = {
7645 INDEX_op_sari_vec
, INDEX_op_smax_vec
, INDEX_op_smin_vec
, 0
7647 static const GVecGen2i ops
[3] = {
7648 { .fniv
= gen_sqshrnb_vec
,
7649 .opt_opc
= vec_list
,
7650 .fno
= gen_helper_sve2_sqshrnb_h
,
7652 { .fniv
= gen_sqshrnb_vec
,
7653 .opt_opc
= vec_list
,
7654 .fno
= gen_helper_sve2_sqshrnb_s
,
7656 { .fniv
= gen_sqshrnb_vec
,
7657 .opt_opc
= vec_list
,
7658 .fno
= gen_helper_sve2_sqshrnb_d
,
7661 return do_sve2_shr_narrow(s
, a
, ops
);
7664 static void gen_sqshrnt_vec(unsigned vece
, TCGv_vec d
,
7665 TCGv_vec n
, int64_t shr
)
7667 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7668 int halfbits
= 4 << vece
;
7669 int64_t max
= MAKE_64BIT_MASK(0, halfbits
- 1);
7670 int64_t min
= -max
- 1;
7672 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7673 tcg_gen_dupi_vec(vece
, t
, min
);
7674 tcg_gen_smax_vec(vece
, n
, n
, t
);
7675 tcg_gen_dupi_vec(vece
, t
, max
);
7676 tcg_gen_smin_vec(vece
, n
, n
, t
);
7677 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7678 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7679 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7680 tcg_temp_free_vec(t
);
7683 static bool trans_SQSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7685 static const TCGOpcode vec_list
[] = {
7686 INDEX_op_shli_vec
, INDEX_op_sari_vec
,
7687 INDEX_op_smax_vec
, INDEX_op_smin_vec
, 0
7689 static const GVecGen2i ops
[3] = {
7690 { .fniv
= gen_sqshrnt_vec
,
7691 .opt_opc
= vec_list
,
7693 .fno
= gen_helper_sve2_sqshrnt_h
,
7695 { .fniv
= gen_sqshrnt_vec
,
7696 .opt_opc
= vec_list
,
7698 .fno
= gen_helper_sve2_sqshrnt_s
,
7700 { .fniv
= gen_sqshrnt_vec
,
7701 .opt_opc
= vec_list
,
7703 .fno
= gen_helper_sve2_sqshrnt_d
,
7706 return do_sve2_shr_narrow(s
, a
, ops
);
7709 static bool trans_SQRSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7711 static const GVecGen2i ops
[3] = {
7712 { .fno
= gen_helper_sve2_sqrshrnb_h
},
7713 { .fno
= gen_helper_sve2_sqrshrnb_s
},
7714 { .fno
= gen_helper_sve2_sqrshrnb_d
},
7716 return do_sve2_shr_narrow(s
, a
, ops
);
7719 static bool trans_SQRSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7721 static const GVecGen2i ops
[3] = {
7722 { .fno
= gen_helper_sve2_sqrshrnt_h
},
7723 { .fno
= gen_helper_sve2_sqrshrnt_s
},
7724 { .fno
= gen_helper_sve2_sqrshrnt_d
},
7726 return do_sve2_shr_narrow(s
, a
, ops
);
7729 static void gen_uqshrnb_vec(unsigned vece
, TCGv_vec d
,
7730 TCGv_vec n
, int64_t shr
)
7732 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7733 int halfbits
= 4 << vece
;
7735 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7736 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7737 tcg_gen_umin_vec(vece
, d
, n
, t
);
7738 tcg_temp_free_vec(t
);
7741 static bool trans_UQSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7743 static const TCGOpcode vec_list
[] = {
7744 INDEX_op_shri_vec
, INDEX_op_umin_vec
, 0
7746 static const GVecGen2i ops
[3] = {
7747 { .fniv
= gen_uqshrnb_vec
,
7748 .opt_opc
= vec_list
,
7749 .fno
= gen_helper_sve2_uqshrnb_h
,
7751 { .fniv
= gen_uqshrnb_vec
,
7752 .opt_opc
= vec_list
,
7753 .fno
= gen_helper_sve2_uqshrnb_s
,
7755 { .fniv
= gen_uqshrnb_vec
,
7756 .opt_opc
= vec_list
,
7757 .fno
= gen_helper_sve2_uqshrnb_d
,
7760 return do_sve2_shr_narrow(s
, a
, ops
);
7763 static void gen_uqshrnt_vec(unsigned vece
, TCGv_vec d
,
7764 TCGv_vec n
, int64_t shr
)
7766 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7767 int halfbits
= 4 << vece
;
7769 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7770 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7771 tcg_gen_umin_vec(vece
, n
, n
, t
);
7772 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7773 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7774 tcg_temp_free_vec(t
);
7777 static bool trans_UQSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7779 static const TCGOpcode vec_list
[] = {
7780 INDEX_op_shli_vec
, INDEX_op_shri_vec
, INDEX_op_umin_vec
, 0
7782 static const GVecGen2i ops
[3] = {
7783 { .fniv
= gen_uqshrnt_vec
,
7784 .opt_opc
= vec_list
,
7786 .fno
= gen_helper_sve2_uqshrnt_h
,
7788 { .fniv
= gen_uqshrnt_vec
,
7789 .opt_opc
= vec_list
,
7791 .fno
= gen_helper_sve2_uqshrnt_s
,
7793 { .fniv
= gen_uqshrnt_vec
,
7794 .opt_opc
= vec_list
,
7796 .fno
= gen_helper_sve2_uqshrnt_d
,
7799 return do_sve2_shr_narrow(s
, a
, ops
);
7802 static bool trans_UQRSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7804 static const GVecGen2i ops
[3] = {
7805 { .fno
= gen_helper_sve2_uqrshrnb_h
},
7806 { .fno
= gen_helper_sve2_uqrshrnb_s
},
7807 { .fno
= gen_helper_sve2_uqrshrnb_d
},
7809 return do_sve2_shr_narrow(s
, a
, ops
);
7812 static bool trans_UQRSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7814 static const GVecGen2i ops
[3] = {
7815 { .fno
= gen_helper_sve2_uqrshrnt_h
},
7816 { .fno
= gen_helper_sve2_uqrshrnt_s
},
7817 { .fno
= gen_helper_sve2_uqrshrnt_d
},
7819 return do_sve2_shr_narrow(s
, a
, ops
);
7822 #define DO_SVE2_ZZZ_NARROW(NAME, name) \
7823 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
7825 static gen_helper_gvec_3 * const fns[4] = { \
7826 NULL, gen_helper_sve2_##name##_h, \
7827 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
7829 return do_sve2_zzz_ool(s, a, fns[a->esz]); \
7832 DO_SVE2_ZZZ_NARROW(ADDHNB
, addhnb
)
7833 DO_SVE2_ZZZ_NARROW(ADDHNT
, addhnt
)
7834 DO_SVE2_ZZZ_NARROW(RADDHNB
, raddhnb
)
7835 DO_SVE2_ZZZ_NARROW(RADDHNT
, raddhnt
)
7837 DO_SVE2_ZZZ_NARROW(SUBHNB
, subhnb
)
7838 DO_SVE2_ZZZ_NARROW(SUBHNT
, subhnt
)
7839 DO_SVE2_ZZZ_NARROW(RSUBHNB
, rsubhnb
)
7840 DO_SVE2_ZZZ_NARROW(RSUBHNT
, rsubhnt
)
7842 static bool do_sve2_ppzz_flags(DisasContext
*s
, arg_rprr_esz
*a
,
7843 gen_helper_gvec_flags_4
*fn
)
7845 if (!dc_isar_feature(aa64_sve2
, s
)) {
7848 return do_ppzz_flags(s
, a
, fn
);
7851 #define DO_SVE2_PPZZ_MATCH(NAME, name) \
7852 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
7854 static gen_helper_gvec_flags_4 * const fns[4] = { \
7855 gen_helper_sve2_##name##_ppzz_b, gen_helper_sve2_##name##_ppzz_h, \
7858 return do_sve2_ppzz_flags(s, a, fns[a->esz]); \
7861 DO_SVE2_PPZZ_MATCH(MATCH
, match
)
7862 DO_SVE2_PPZZ_MATCH(NMATCH
, nmatch
)
7864 static bool trans_HISTCNT(DisasContext
*s
, arg_rprr_esz
*a
)
7866 static gen_helper_gvec_4
* const fns
[2] = {
7867 gen_helper_sve2_histcnt_s
, gen_helper_sve2_histcnt_d
7872 return do_sve2_zpzz_ool(s
, a
, fns
[a
->esz
- 2]);
7875 static bool trans_HISTSEG(DisasContext
*s
, arg_rrr_esz
*a
)
7880 return do_sve2_zzz_ool(s
, a
, gen_helper_sve2_histseg
);
7883 static bool do_sve2_zpzz_fp(DisasContext
*s
, arg_rprr_esz
*a
,
7884 gen_helper_gvec_4_ptr
*fn
)
7886 if (!dc_isar_feature(aa64_sve2
, s
)) {
7889 return do_zpzz_fp(s
, a
, fn
);
7892 #define DO_SVE2_ZPZZ_FP(NAME, name) \
7893 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
7895 static gen_helper_gvec_4_ptr * const fns[4] = { \
7896 NULL, gen_helper_sve2_##name##_zpzz_h, \
7897 gen_helper_sve2_##name##_zpzz_s, gen_helper_sve2_##name##_zpzz_d \
7899 return do_sve2_zpzz_fp(s, a, fns[a->esz]); \
7902 DO_SVE2_ZPZZ_FP(FADDP
, faddp
)
7903 DO_SVE2_ZPZZ_FP(FMAXNMP
, fmaxnmp
)
7904 DO_SVE2_ZPZZ_FP(FMINNMP
, fminnmp
)
7905 DO_SVE2_ZPZZ_FP(FMAXP
, fmaxp
)
7906 DO_SVE2_ZPZZ_FP(FMINP
, fminp
)
7909 * SVE Integer Multiply-Add (unpredicated)
7912 static bool trans_FMMLA(DisasContext
*s
, arg_rrrr_esz
*a
)
7914 gen_helper_gvec_4_ptr
*fn
;
7918 if (!dc_isar_feature(aa64_sve_f32mm
, s
)) {
7921 fn
= gen_helper_fmmla_s
;
7924 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
7927 fn
= gen_helper_fmmla_d
;
7933 if (sve_access_check(s
)) {
7934 unsigned vsz
= vec_full_reg_size(s
);
7935 TCGv_ptr status
= fpstatus_ptr(FPST_FPCR
);
7936 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
7937 vec_full_reg_offset(s
, a
->rn
),
7938 vec_full_reg_offset(s
, a
->rm
),
7939 vec_full_reg_offset(s
, a
->ra
),
7940 status
, vsz
, vsz
, 0, fn
);
7941 tcg_temp_free_ptr(status
);
7946 static bool do_sqdmlal_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
,
7947 bool sel1
, bool sel2
)
7949 static gen_helper_gvec_4
* const fns
[] = {
7950 NULL
, gen_helper_sve2_sqdmlal_zzzw_h
,
7951 gen_helper_sve2_sqdmlal_zzzw_s
, gen_helper_sve2_sqdmlal_zzzw_d
,
7953 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], (sel2
<< 1) | sel1
);
7956 static bool do_sqdmlsl_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
,
7957 bool sel1
, bool sel2
)
7959 static gen_helper_gvec_4
* const fns
[] = {
7960 NULL
, gen_helper_sve2_sqdmlsl_zzzw_h
,
7961 gen_helper_sve2_sqdmlsl_zzzw_s
, gen_helper_sve2_sqdmlsl_zzzw_d
,
7963 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], (sel2
<< 1) | sel1
);
7966 static bool trans_SQDMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7968 return do_sqdmlal_zzzw(s
, a
, false, false);
7971 static bool trans_SQDMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7973 return do_sqdmlal_zzzw(s
, a
, true, true);
7976 static bool trans_SQDMLALBT(DisasContext
*s
, arg_rrrr_esz
*a
)
7978 return do_sqdmlal_zzzw(s
, a
, false, true);
7981 static bool trans_SQDMLSLB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7983 return do_sqdmlsl_zzzw(s
, a
, false, false);
7986 static bool trans_SQDMLSLT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
7988 return do_sqdmlsl_zzzw(s
, a
, true, true);
7991 static bool trans_SQDMLSLBT(DisasContext
*s
, arg_rrrr_esz
*a
)
7993 return do_sqdmlsl_zzzw(s
, a
, false, true);
7996 static bool trans_SQRDMLAH_zzzz(DisasContext
*s
, arg_rrrr_esz
*a
)
7998 static gen_helper_gvec_4
* const fns
[] = {
7999 gen_helper_sve2_sqrdmlah_b
, gen_helper_sve2_sqrdmlah_h
,
8000 gen_helper_sve2_sqrdmlah_s
, gen_helper_sve2_sqrdmlah_d
,
8002 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], 0);
8005 static bool trans_SQRDMLSH_zzzz(DisasContext
*s
, arg_rrrr_esz
*a
)
8007 static gen_helper_gvec_4
* const fns
[] = {
8008 gen_helper_sve2_sqrdmlsh_b
, gen_helper_sve2_sqrdmlsh_h
,
8009 gen_helper_sve2_sqrdmlsh_s
, gen_helper_sve2_sqrdmlsh_d
,
8011 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], 0);
8014 static bool do_smlal_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
8016 static gen_helper_gvec_4
* const fns
[] = {
8017 NULL
, gen_helper_sve2_smlal_zzzw_h
,
8018 gen_helper_sve2_smlal_zzzw_s
, gen_helper_sve2_smlal_zzzw_d
,
8020 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], sel
);
8023 static bool trans_SMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8025 return do_smlal_zzzw(s
, a
, false);
8028 static bool trans_SMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8030 return do_smlal_zzzw(s
, a
, true);
8033 static bool do_umlal_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
8035 static gen_helper_gvec_4
* const fns
[] = {
8036 NULL
, gen_helper_sve2_umlal_zzzw_h
,
8037 gen_helper_sve2_umlal_zzzw_s
, gen_helper_sve2_umlal_zzzw_d
,
8039 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], sel
);
8042 static bool trans_UMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8044 return do_umlal_zzzw(s
, a
, false);
8047 static bool trans_UMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8049 return do_umlal_zzzw(s
, a
, true);
8052 static bool do_smlsl_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
8054 static gen_helper_gvec_4
* const fns
[] = {
8055 NULL
, gen_helper_sve2_smlsl_zzzw_h
,
8056 gen_helper_sve2_smlsl_zzzw_s
, gen_helper_sve2_smlsl_zzzw_d
,
8058 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], sel
);
8061 static bool trans_SMLSLB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8063 return do_smlsl_zzzw(s
, a
, false);
8066 static bool trans_SMLSLT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8068 return do_smlsl_zzzw(s
, a
, true);
8071 static bool do_umlsl_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
8073 static gen_helper_gvec_4
* const fns
[] = {
8074 NULL
, gen_helper_sve2_umlsl_zzzw_h
,
8075 gen_helper_sve2_umlsl_zzzw_s
, gen_helper_sve2_umlsl_zzzw_d
,
8077 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], sel
);
8080 static bool trans_UMLSLB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8082 return do_umlsl_zzzw(s
, a
, false);
8085 static bool trans_UMLSLT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8087 return do_umlsl_zzzw(s
, a
, true);
8090 static bool trans_CMLA_zzzz(DisasContext
*s
, arg_CMLA_zzzz
*a
)
8092 static gen_helper_gvec_4
* const fns
[] = {
8093 gen_helper_sve2_cmla_zzzz_b
, gen_helper_sve2_cmla_zzzz_h
,
8094 gen_helper_sve2_cmla_zzzz_s
, gen_helper_sve2_cmla_zzzz_d
,
8097 if (!dc_isar_feature(aa64_sve2
, s
)) {
8100 if (sve_access_check(s
)) {
8101 gen_gvec_ool_zzzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
);
8106 static bool trans_CDOT_zzzz(DisasContext
*s
, arg_CMLA_zzzz
*a
)
8108 if (!dc_isar_feature(aa64_sve2
, s
) || a
->esz
< MO_32
) {
8111 if (sve_access_check(s
)) {
8112 gen_helper_gvec_4
*fn
= (a
->esz
== MO_32
8113 ? gen_helper_sve2_cdot_zzzz_s
8114 : gen_helper_sve2_cdot_zzzz_d
);
8115 gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
);
8120 static bool trans_SQRDCMLAH_zzzz(DisasContext
*s
, arg_SQRDCMLAH_zzzz
*a
)
8122 static gen_helper_gvec_4
* const fns
[] = {
8123 gen_helper_sve2_sqrdcmlah_zzzz_b
, gen_helper_sve2_sqrdcmlah_zzzz_h
,
8124 gen_helper_sve2_sqrdcmlah_zzzz_s
, gen_helper_sve2_sqrdcmlah_zzzz_d
,
8127 if (!dc_isar_feature(aa64_sve2
, s
)) {
8130 if (sve_access_check(s
)) {
8131 gen_gvec_ool_zzzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
);
8136 static bool trans_USDOT_zzzz(DisasContext
*s
, arg_USDOT_zzzz
*a
)
8138 if (a
->esz
!= 2 || !dc_isar_feature(aa64_sve_i8mm
, s
)) {
8141 if (sve_access_check(s
)) {
8142 unsigned vsz
= vec_full_reg_size(s
);
8143 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, a
->rd
),
8144 vec_full_reg_offset(s
, a
->rn
),
8145 vec_full_reg_offset(s
, a
->rm
),
8146 vec_full_reg_offset(s
, a
->ra
),
8147 vsz
, vsz
, 0, gen_helper_gvec_usdot_b
);