2 * AArch64 SVE translation
4 * Copyright (c) 2018 Linaro, Ltd
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
22 #include "exec/exec-all.h"
23 #include "tcg/tcg-op.h"
24 #include "tcg/tcg-op-gvec.h"
25 #include "tcg/tcg-gvec-desc.h"
28 #include "translate.h"
29 #include "internals.h"
30 #include "exec/helper-proto.h"
31 #include "exec/helper-gen.h"
33 #include "trace-tcg.h"
34 #include "translate-a64.h"
35 #include "fpu/softfloat.h"
38 typedef void GVecGen2sFn(unsigned, uint32_t, uint32_t,
39 TCGv_i64
, uint32_t, uint32_t);
41 typedef void gen_helper_gvec_flags_3(TCGv_i32
, TCGv_ptr
, TCGv_ptr
,
43 typedef void gen_helper_gvec_flags_4(TCGv_i32
, TCGv_ptr
, TCGv_ptr
,
44 TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
46 typedef void gen_helper_gvec_mem(TCGv_env
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
47 typedef void gen_helper_gvec_mem_scatter(TCGv_env
, TCGv_ptr
, TCGv_ptr
,
48 TCGv_ptr
, TCGv_i64
, TCGv_i32
);
51 * Helpers for extracting complex instruction fields.
54 /* See e.g. ASR (immediate, predicated).
55 * Returns -1 for unallocated encoding; diagnose later.
57 static int tszimm_esz(DisasContext
*s
, int x
)
59 x
>>= 3; /* discard imm3 */
63 static int tszimm_shr(DisasContext
*s
, int x
)
65 return (16 << tszimm_esz(s
, x
)) - x
;
68 /* See e.g. LSL (immediate, predicated). */
69 static int tszimm_shl(DisasContext
*s
, int x
)
71 return x
- (8 << tszimm_esz(s
, x
));
74 static inline int plus1(DisasContext
*s
, int x
)
79 /* The SH bit is in bit 8. Extract the low 8 and shift. */
80 static inline int expand_imm_sh8s(DisasContext
*s
, int x
)
82 return (int8_t)x
<< (x
& 0x100 ? 8 : 0);
85 static inline int expand_imm_sh8u(DisasContext
*s
, int x
)
87 return (uint8_t)x
<< (x
& 0x100 ? 8 : 0);
90 /* Convert a 2-bit memory size (msz) to a 4-bit data type (dtype)
91 * with unsigned data. C.f. SVE Memory Contiguous Load Group.
93 static inline int msz_dtype(DisasContext
*s
, int msz
)
95 static const uint8_t dtype
[4] = { 0, 5, 10, 15 };
100 * Include the generated decoder.
103 #include "decode-sve.c.inc"
106 * Implement all of the translator functions referenced by the decoder.
109 /* Return the offset info CPUARMState of the predicate vector register Pn.
110 * Note for this purpose, FFR is P16.
112 static inline int pred_full_reg_offset(DisasContext
*s
, int regno
)
114 return offsetof(CPUARMState
, vfp
.pregs
[regno
]);
117 /* Return the byte size of the whole predicate register, VL / 64. */
118 static inline int pred_full_reg_size(DisasContext
*s
)
120 return s
->sve_len
>> 3;
123 /* Round up the size of a register to a size allowed by
124 * the tcg vector infrastructure. Any operation which uses this
125 * size may assume that the bits above pred_full_reg_size are zero,
126 * and must leave them the same way.
128 * Note that this is not needed for the vector registers as they
129 * are always properly sized for tcg vectors.
131 static int size_for_gvec(int size
)
136 return QEMU_ALIGN_UP(size
, 16);
140 static int pred_gvec_reg_size(DisasContext
*s
)
142 return size_for_gvec(pred_full_reg_size(s
));
145 /* Invoke an out-of-line helper on 2 Zregs. */
146 static void gen_gvec_ool_zz(DisasContext
*s
, gen_helper_gvec_2
*fn
,
147 int rd
, int rn
, int data
)
149 unsigned vsz
= vec_full_reg_size(s
);
150 tcg_gen_gvec_2_ool(vec_full_reg_offset(s
, rd
),
151 vec_full_reg_offset(s
, rn
),
155 /* Invoke an out-of-line helper on 3 Zregs. */
156 static void gen_gvec_ool_zzz(DisasContext
*s
, gen_helper_gvec_3
*fn
,
157 int rd
, int rn
, int rm
, int data
)
159 unsigned vsz
= vec_full_reg_size(s
);
160 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
161 vec_full_reg_offset(s
, rn
),
162 vec_full_reg_offset(s
, rm
),
166 /* Invoke an out-of-line helper on 4 Zregs. */
167 static void gen_gvec_ool_zzzz(DisasContext
*s
, gen_helper_gvec_4
*fn
,
168 int rd
, int rn
, int rm
, int ra
, int data
)
170 unsigned vsz
= vec_full_reg_size(s
);
171 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
172 vec_full_reg_offset(s
, rn
),
173 vec_full_reg_offset(s
, rm
),
174 vec_full_reg_offset(s
, ra
),
178 /* Invoke an out-of-line helper on 2 Zregs and a predicate. */
179 static void gen_gvec_ool_zzp(DisasContext
*s
, gen_helper_gvec_3
*fn
,
180 int rd
, int rn
, int pg
, int data
)
182 unsigned vsz
= vec_full_reg_size(s
);
183 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
184 vec_full_reg_offset(s
, rn
),
185 pred_full_reg_offset(s
, pg
),
189 /* Invoke an out-of-line helper on 3 Zregs and a predicate. */
190 static void gen_gvec_ool_zzzp(DisasContext
*s
, gen_helper_gvec_4
*fn
,
191 int rd
, int rn
, int rm
, int pg
, int data
)
193 unsigned vsz
= vec_full_reg_size(s
);
194 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
195 vec_full_reg_offset(s
, rn
),
196 vec_full_reg_offset(s
, rm
),
197 pred_full_reg_offset(s
, pg
),
201 /* Invoke a vector expander on two Zregs. */
202 static void gen_gvec_fn_zz(DisasContext
*s
, GVecGen2Fn
*gvec_fn
,
203 int esz
, int rd
, int rn
)
205 unsigned vsz
= vec_full_reg_size(s
);
206 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
207 vec_full_reg_offset(s
, rn
), vsz
, vsz
);
210 /* Invoke a vector expander on three Zregs. */
211 static void gen_gvec_fn_zzz(DisasContext
*s
, GVecGen3Fn
*gvec_fn
,
212 int esz
, int rd
, int rn
, int rm
)
214 unsigned vsz
= vec_full_reg_size(s
);
215 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
216 vec_full_reg_offset(s
, rn
),
217 vec_full_reg_offset(s
, rm
), vsz
, vsz
);
220 /* Invoke a vector expander on four Zregs. */
221 static void gen_gvec_fn_zzzz(DisasContext
*s
, GVecGen4Fn
*gvec_fn
,
222 int esz
, int rd
, int rn
, int rm
, int ra
)
224 unsigned vsz
= vec_full_reg_size(s
);
225 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
226 vec_full_reg_offset(s
, rn
),
227 vec_full_reg_offset(s
, rm
),
228 vec_full_reg_offset(s
, ra
), vsz
, vsz
);
231 /* Invoke a vector move on two Zregs. */
232 static bool do_mov_z(DisasContext
*s
, int rd
, int rn
)
234 if (sve_access_check(s
)) {
235 gen_gvec_fn_zz(s
, tcg_gen_gvec_mov
, MO_8
, rd
, rn
);
240 /* Initialize a Zreg with replications of a 64-bit immediate. */
241 static void do_dupi_z(DisasContext
*s
, int rd
, uint64_t word
)
243 unsigned vsz
= vec_full_reg_size(s
);
244 tcg_gen_gvec_dup_imm(MO_64
, vec_full_reg_offset(s
, rd
), vsz
, vsz
, word
);
247 /* Invoke a vector expander on three Pregs. */
248 static void gen_gvec_fn_ppp(DisasContext
*s
, GVecGen3Fn
*gvec_fn
,
249 int rd
, int rn
, int rm
)
251 unsigned psz
= pred_gvec_reg_size(s
);
252 gvec_fn(MO_64
, pred_full_reg_offset(s
, rd
),
253 pred_full_reg_offset(s
, rn
),
254 pred_full_reg_offset(s
, rm
), psz
, psz
);
257 /* Invoke a vector move on two Pregs. */
258 static bool do_mov_p(DisasContext
*s
, int rd
, int rn
)
260 if (sve_access_check(s
)) {
261 unsigned psz
= pred_gvec_reg_size(s
);
262 tcg_gen_gvec_mov(MO_8
, pred_full_reg_offset(s
, rd
),
263 pred_full_reg_offset(s
, rn
), psz
, psz
);
268 /* Set the cpu flags as per a return from an SVE helper. */
269 static void do_pred_flags(TCGv_i32 t
)
271 tcg_gen_mov_i32(cpu_NF
, t
);
272 tcg_gen_andi_i32(cpu_ZF
, t
, 2);
273 tcg_gen_andi_i32(cpu_CF
, t
, 1);
274 tcg_gen_movi_i32(cpu_VF
, 0);
277 /* Subroutines computing the ARM PredTest psuedofunction. */
278 static void do_predtest1(TCGv_i64 d
, TCGv_i64 g
)
280 TCGv_i32 t
= tcg_temp_new_i32();
282 gen_helper_sve_predtest1(t
, d
, g
);
284 tcg_temp_free_i32(t
);
287 static void do_predtest(DisasContext
*s
, int dofs
, int gofs
, int words
)
289 TCGv_ptr dptr
= tcg_temp_new_ptr();
290 TCGv_ptr gptr
= tcg_temp_new_ptr();
293 tcg_gen_addi_ptr(dptr
, cpu_env
, dofs
);
294 tcg_gen_addi_ptr(gptr
, cpu_env
, gofs
);
295 t
= tcg_const_i32(words
);
297 gen_helper_sve_predtest(t
, dptr
, gptr
, t
);
298 tcg_temp_free_ptr(dptr
);
299 tcg_temp_free_ptr(gptr
);
302 tcg_temp_free_i32(t
);
305 /* For each element size, the bits within a predicate word that are active. */
306 const uint64_t pred_esz_masks
[4] = {
307 0xffffffffffffffffull
, 0x5555555555555555ull
,
308 0x1111111111111111ull
, 0x0101010101010101ull
312 *** SVE Logical - Unpredicated Group
315 static bool do_zzz_fn(DisasContext
*s
, arg_rrr_esz
*a
, GVecGen3Fn
*gvec_fn
)
317 if (sve_access_check(s
)) {
318 gen_gvec_fn_zzz(s
, gvec_fn
, a
->esz
, a
->rd
, a
->rn
, a
->rm
);
323 static bool trans_AND_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
325 return do_zzz_fn(s
, a
, tcg_gen_gvec_and
);
328 static bool trans_ORR_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
330 return do_zzz_fn(s
, a
, tcg_gen_gvec_or
);
333 static bool trans_EOR_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
335 return do_zzz_fn(s
, a
, tcg_gen_gvec_xor
);
338 static bool trans_BIC_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
340 return do_zzz_fn(s
, a
, tcg_gen_gvec_andc
);
343 static void gen_xar8_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
345 TCGv_i64 t
= tcg_temp_new_i64();
346 uint64_t mask
= dup_const(MO_8
, 0xff >> sh
);
348 tcg_gen_xor_i64(t
, n
, m
);
349 tcg_gen_shri_i64(d
, t
, sh
);
350 tcg_gen_shli_i64(t
, t
, 8 - sh
);
351 tcg_gen_andi_i64(d
, d
, mask
);
352 tcg_gen_andi_i64(t
, t
, ~mask
);
353 tcg_gen_or_i64(d
, d
, t
);
354 tcg_temp_free_i64(t
);
357 static void gen_xar16_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
359 TCGv_i64 t
= tcg_temp_new_i64();
360 uint64_t mask
= dup_const(MO_16
, 0xffff >> sh
);
362 tcg_gen_xor_i64(t
, n
, m
);
363 tcg_gen_shri_i64(d
, t
, sh
);
364 tcg_gen_shli_i64(t
, t
, 16 - sh
);
365 tcg_gen_andi_i64(d
, d
, mask
);
366 tcg_gen_andi_i64(t
, t
, ~mask
);
367 tcg_gen_or_i64(d
, d
, t
);
368 tcg_temp_free_i64(t
);
371 static void gen_xar_i32(TCGv_i32 d
, TCGv_i32 n
, TCGv_i32 m
, int32_t sh
)
373 tcg_gen_xor_i32(d
, n
, m
);
374 tcg_gen_rotri_i32(d
, d
, sh
);
377 static void gen_xar_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, int64_t sh
)
379 tcg_gen_xor_i64(d
, n
, m
);
380 tcg_gen_rotri_i64(d
, d
, sh
);
383 static void gen_xar_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
384 TCGv_vec m
, int64_t sh
)
386 tcg_gen_xor_vec(vece
, d
, n
, m
);
387 tcg_gen_rotri_vec(vece
, d
, d
, sh
);
390 void gen_gvec_xar(unsigned vece
, uint32_t rd_ofs
, uint32_t rn_ofs
,
391 uint32_t rm_ofs
, int64_t shift
,
392 uint32_t opr_sz
, uint32_t max_sz
)
394 static const TCGOpcode vecop
[] = { INDEX_op_rotli_vec
, 0 };
395 static const GVecGen3i ops
[4] = {
396 { .fni8
= gen_xar8_i64
,
398 .fno
= gen_helper_sve2_xar_b
,
401 { .fni8
= gen_xar16_i64
,
403 .fno
= gen_helper_sve2_xar_h
,
406 { .fni4
= gen_xar_i32
,
408 .fno
= gen_helper_sve2_xar_s
,
411 { .fni8
= gen_xar_i64
,
413 .fno
= gen_helper_gvec_xar_d
,
417 int esize
= 8 << vece
;
419 /* The SVE2 range is 1 .. esize; the AdvSIMD range is 0 .. esize-1. */
420 tcg_debug_assert(shift
>= 0);
421 tcg_debug_assert(shift
<= esize
);
425 /* xar with no rotate devolves to xor. */
426 tcg_gen_gvec_xor(vece
, rd_ofs
, rn_ofs
, rm_ofs
, opr_sz
, max_sz
);
428 tcg_gen_gvec_3i(rd_ofs
, rn_ofs
, rm_ofs
, opr_sz
, max_sz
,
433 static bool trans_XAR(DisasContext
*s
, arg_rrri_esz
*a
)
435 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
438 if (sve_access_check(s
)) {
439 unsigned vsz
= vec_full_reg_size(s
);
440 gen_gvec_xar(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
441 vec_full_reg_offset(s
, a
->rn
),
442 vec_full_reg_offset(s
, a
->rm
), a
->imm
, vsz
, vsz
);
447 static bool do_sve2_zzzz_fn(DisasContext
*s
, arg_rrrr_esz
*a
, GVecGen4Fn
*fn
)
449 if (!dc_isar_feature(aa64_sve2
, s
)) {
452 if (sve_access_check(s
)) {
453 gen_gvec_fn_zzzz(s
, fn
, a
->esz
, a
->rd
, a
->rn
, a
->rm
, a
->ra
);
458 static void gen_eor3_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
460 tcg_gen_xor_i64(d
, n
, m
);
461 tcg_gen_xor_i64(d
, d
, k
);
464 static void gen_eor3_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
465 TCGv_vec m
, TCGv_vec k
)
467 tcg_gen_xor_vec(vece
, d
, n
, m
);
468 tcg_gen_xor_vec(vece
, d
, d
, k
);
471 static void gen_eor3(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
472 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
474 static const GVecGen4 op
= {
475 .fni8
= gen_eor3_i64
,
476 .fniv
= gen_eor3_vec
,
477 .fno
= gen_helper_sve2_eor3
,
479 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
481 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
484 static bool trans_EOR3(DisasContext
*s
, arg_rrrr_esz
*a
)
486 return do_sve2_zzzz_fn(s
, a
, gen_eor3
);
489 static void gen_bcax_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
491 tcg_gen_andc_i64(d
, m
, k
);
492 tcg_gen_xor_i64(d
, d
, n
);
495 static void gen_bcax_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
496 TCGv_vec m
, TCGv_vec k
)
498 tcg_gen_andc_vec(vece
, d
, m
, k
);
499 tcg_gen_xor_vec(vece
, d
, d
, n
);
502 static void gen_bcax(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
503 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
505 static const GVecGen4 op
= {
506 .fni8
= gen_bcax_i64
,
507 .fniv
= gen_bcax_vec
,
508 .fno
= gen_helper_sve2_bcax
,
510 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
512 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
515 static bool trans_BCAX(DisasContext
*s
, arg_rrrr_esz
*a
)
517 return do_sve2_zzzz_fn(s
, a
, gen_bcax
);
520 static void gen_bsl(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
521 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
523 /* BSL differs from the generic bitsel in argument ordering. */
524 tcg_gen_gvec_bitsel(vece
, d
, a
, n
, m
, oprsz
, maxsz
);
527 static bool trans_BSL(DisasContext
*s
, arg_rrrr_esz
*a
)
529 return do_sve2_zzzz_fn(s
, a
, gen_bsl
);
532 static void gen_bsl1n_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
534 tcg_gen_andc_i64(n
, k
, n
);
535 tcg_gen_andc_i64(m
, m
, k
);
536 tcg_gen_or_i64(d
, n
, m
);
539 static void gen_bsl1n_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
540 TCGv_vec m
, TCGv_vec k
)
542 if (TCG_TARGET_HAS_bitsel_vec
) {
543 tcg_gen_not_vec(vece
, n
, n
);
544 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
546 tcg_gen_andc_vec(vece
, n
, k
, n
);
547 tcg_gen_andc_vec(vece
, m
, m
, k
);
548 tcg_gen_or_vec(vece
, d
, n
, m
);
552 static void gen_bsl1n(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
553 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
555 static const GVecGen4 op
= {
556 .fni8
= gen_bsl1n_i64
,
557 .fniv
= gen_bsl1n_vec
,
558 .fno
= gen_helper_sve2_bsl1n
,
560 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
562 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
565 static bool trans_BSL1N(DisasContext
*s
, arg_rrrr_esz
*a
)
567 return do_sve2_zzzz_fn(s
, a
, gen_bsl1n
);
570 static void gen_bsl2n_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
573 * Z[dn] = (n & k) | (~m & ~k)
576 tcg_gen_and_i64(n
, n
, k
);
577 if (TCG_TARGET_HAS_orc_i64
) {
578 tcg_gen_or_i64(m
, m
, k
);
579 tcg_gen_orc_i64(d
, n
, m
);
581 tcg_gen_nor_i64(m
, m
, k
);
582 tcg_gen_or_i64(d
, n
, m
);
586 static void gen_bsl2n_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
587 TCGv_vec m
, TCGv_vec k
)
589 if (TCG_TARGET_HAS_bitsel_vec
) {
590 tcg_gen_not_vec(vece
, m
, m
);
591 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
593 tcg_gen_and_vec(vece
, n
, n
, k
);
594 tcg_gen_or_vec(vece
, m
, m
, k
);
595 tcg_gen_orc_vec(vece
, d
, n
, m
);
599 static void gen_bsl2n(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
600 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
602 static const GVecGen4 op
= {
603 .fni8
= gen_bsl2n_i64
,
604 .fniv
= gen_bsl2n_vec
,
605 .fno
= gen_helper_sve2_bsl2n
,
607 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
609 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
612 static bool trans_BSL2N(DisasContext
*s
, arg_rrrr_esz
*a
)
614 return do_sve2_zzzz_fn(s
, a
, gen_bsl2n
);
617 static void gen_nbsl_i64(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_i64 k
)
619 tcg_gen_and_i64(n
, n
, k
);
620 tcg_gen_andc_i64(m
, m
, k
);
621 tcg_gen_nor_i64(d
, n
, m
);
624 static void gen_nbsl_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
,
625 TCGv_vec m
, TCGv_vec k
)
627 tcg_gen_bitsel_vec(vece
, d
, k
, n
, m
);
628 tcg_gen_not_vec(vece
, d
, d
);
631 static void gen_nbsl(unsigned vece
, uint32_t d
, uint32_t n
, uint32_t m
,
632 uint32_t a
, uint32_t oprsz
, uint32_t maxsz
)
634 static const GVecGen4 op
= {
635 .fni8
= gen_nbsl_i64
,
636 .fniv
= gen_nbsl_vec
,
637 .fno
= gen_helper_sve2_nbsl
,
639 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
641 tcg_gen_gvec_4(d
, n
, m
, a
, oprsz
, maxsz
, &op
);
644 static bool trans_NBSL(DisasContext
*s
, arg_rrrr_esz
*a
)
646 return do_sve2_zzzz_fn(s
, a
, gen_nbsl
);
650 *** SVE Integer Arithmetic - Unpredicated Group
653 static bool trans_ADD_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
655 return do_zzz_fn(s
, a
, tcg_gen_gvec_add
);
658 static bool trans_SUB_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
660 return do_zzz_fn(s
, a
, tcg_gen_gvec_sub
);
663 static bool trans_SQADD_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
665 return do_zzz_fn(s
, a
, tcg_gen_gvec_ssadd
);
668 static bool trans_SQSUB_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
670 return do_zzz_fn(s
, a
, tcg_gen_gvec_sssub
);
673 static bool trans_UQADD_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
675 return do_zzz_fn(s
, a
, tcg_gen_gvec_usadd
);
678 static bool trans_UQSUB_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
680 return do_zzz_fn(s
, a
, tcg_gen_gvec_ussub
);
684 *** SVE Integer Arithmetic - Binary Predicated Group
687 static bool do_zpzz_ool(DisasContext
*s
, arg_rprr_esz
*a
, gen_helper_gvec_4
*fn
)
692 if (sve_access_check(s
)) {
693 gen_gvec_ool_zzzp(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->pg
, 0);
698 /* Select active elememnts from Zn and inactive elements from Zm,
699 * storing the result in Zd.
701 static void do_sel_z(DisasContext
*s
, int rd
, int rn
, int rm
, int pg
, int esz
)
703 static gen_helper_gvec_4
* const fns
[4] = {
704 gen_helper_sve_sel_zpzz_b
, gen_helper_sve_sel_zpzz_h
,
705 gen_helper_sve_sel_zpzz_s
, gen_helper_sve_sel_zpzz_d
707 gen_gvec_ool_zzzp(s
, fns
[esz
], rd
, rn
, rm
, pg
, 0);
710 #define DO_ZPZZ(NAME, name) \
711 static bool trans_##NAME##_zpzz(DisasContext *s, arg_rprr_esz *a) \
713 static gen_helper_gvec_4 * const fns[4] = { \
714 gen_helper_sve_##name##_zpzz_b, gen_helper_sve_##name##_zpzz_h, \
715 gen_helper_sve_##name##_zpzz_s, gen_helper_sve_##name##_zpzz_d, \
717 return do_zpzz_ool(s, a, fns[a->esz]); \
736 DO_ZPZZ(SMULH
, smulh
)
737 DO_ZPZZ(UMULH
, umulh
)
743 static bool trans_SDIV_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
745 static gen_helper_gvec_4
* const fns
[4] = {
746 NULL
, NULL
, gen_helper_sve_sdiv_zpzz_s
, gen_helper_sve_sdiv_zpzz_d
748 return do_zpzz_ool(s
, a
, fns
[a
->esz
]);
751 static bool trans_UDIV_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
753 static gen_helper_gvec_4
* const fns
[4] = {
754 NULL
, NULL
, gen_helper_sve_udiv_zpzz_s
, gen_helper_sve_udiv_zpzz_d
756 return do_zpzz_ool(s
, a
, fns
[a
->esz
]);
759 static bool trans_SEL_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
761 if (sve_access_check(s
)) {
762 do_sel_z(s
, a
->rd
, a
->rn
, a
->rm
, a
->pg
, a
->esz
);
770 *** SVE Integer Arithmetic - Unary Predicated Group
773 static bool do_zpz_ool(DisasContext
*s
, arg_rpr_esz
*a
, gen_helper_gvec_3
*fn
)
778 if (sve_access_check(s
)) {
779 gen_gvec_ool_zzp(s
, fn
, a
->rd
, a
->rn
, a
->pg
, 0);
784 #define DO_ZPZ(NAME, name) \
785 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
787 static gen_helper_gvec_3 * const fns[4] = { \
788 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
789 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
791 return do_zpz_ool(s, a, fns[a->esz]); \
796 DO_ZPZ(CNT_zpz
, cnt_zpz
)
798 DO_ZPZ(NOT_zpz
, not_zpz
)
802 static bool trans_FABS(DisasContext
*s
, arg_rpr_esz
*a
)
804 static gen_helper_gvec_3
* const fns
[4] = {
806 gen_helper_sve_fabs_h
,
807 gen_helper_sve_fabs_s
,
808 gen_helper_sve_fabs_d
810 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
813 static bool trans_FNEG(DisasContext
*s
, arg_rpr_esz
*a
)
815 static gen_helper_gvec_3
* const fns
[4] = {
817 gen_helper_sve_fneg_h
,
818 gen_helper_sve_fneg_s
,
819 gen_helper_sve_fneg_d
821 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
824 static bool trans_SXTB(DisasContext
*s
, arg_rpr_esz
*a
)
826 static gen_helper_gvec_3
* const fns
[4] = {
828 gen_helper_sve_sxtb_h
,
829 gen_helper_sve_sxtb_s
,
830 gen_helper_sve_sxtb_d
832 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
835 static bool trans_UXTB(DisasContext
*s
, arg_rpr_esz
*a
)
837 static gen_helper_gvec_3
* const fns
[4] = {
839 gen_helper_sve_uxtb_h
,
840 gen_helper_sve_uxtb_s
,
841 gen_helper_sve_uxtb_d
843 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
846 static bool trans_SXTH(DisasContext
*s
, arg_rpr_esz
*a
)
848 static gen_helper_gvec_3
* const fns
[4] = {
850 gen_helper_sve_sxth_s
,
851 gen_helper_sve_sxth_d
853 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
856 static bool trans_UXTH(DisasContext
*s
, arg_rpr_esz
*a
)
858 static gen_helper_gvec_3
* const fns
[4] = {
860 gen_helper_sve_uxth_s
,
861 gen_helper_sve_uxth_d
863 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
866 static bool trans_SXTW(DisasContext
*s
, arg_rpr_esz
*a
)
868 return do_zpz_ool(s
, a
, a
->esz
== 3 ? gen_helper_sve_sxtw_d
: NULL
);
871 static bool trans_UXTW(DisasContext
*s
, arg_rpr_esz
*a
)
873 return do_zpz_ool(s
, a
, a
->esz
== 3 ? gen_helper_sve_uxtw_d
: NULL
);
879 *** SVE Integer Reduction Group
882 typedef void gen_helper_gvec_reduc(TCGv_i64
, TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
883 static bool do_vpz_ool(DisasContext
*s
, arg_rpr_esz
*a
,
884 gen_helper_gvec_reduc
*fn
)
886 unsigned vsz
= vec_full_reg_size(s
);
894 if (!sve_access_check(s
)) {
898 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
899 temp
= tcg_temp_new_i64();
900 t_zn
= tcg_temp_new_ptr();
901 t_pg
= tcg_temp_new_ptr();
903 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
904 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
905 fn(temp
, t_zn
, t_pg
, desc
);
906 tcg_temp_free_ptr(t_zn
);
907 tcg_temp_free_ptr(t_pg
);
908 tcg_temp_free_i32(desc
);
910 write_fp_dreg(s
, a
->rd
, temp
);
911 tcg_temp_free_i64(temp
);
915 #define DO_VPZ(NAME, name) \
916 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
918 static gen_helper_gvec_reduc * const fns[4] = { \
919 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
920 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
922 return do_vpz_ool(s, a, fns[a->esz]); \
935 static bool trans_SADDV(DisasContext
*s
, arg_rpr_esz
*a
)
937 static gen_helper_gvec_reduc
* const fns
[4] = {
938 gen_helper_sve_saddv_b
, gen_helper_sve_saddv_h
,
939 gen_helper_sve_saddv_s
, NULL
941 return do_vpz_ool(s
, a
, fns
[a
->esz
]);
947 *** SVE Shift by Immediate - Predicated Group
951 * Copy Zn into Zd, storing zeros into inactive elements.
952 * If invert, store zeros into the active elements.
954 static bool do_movz_zpz(DisasContext
*s
, int rd
, int rn
, int pg
,
955 int esz
, bool invert
)
957 static gen_helper_gvec_3
* const fns
[4] = {
958 gen_helper_sve_movz_b
, gen_helper_sve_movz_h
,
959 gen_helper_sve_movz_s
, gen_helper_sve_movz_d
,
962 if (sve_access_check(s
)) {
963 gen_gvec_ool_zzp(s
, fns
[esz
], rd
, rn
, pg
, invert
);
968 static bool do_zpzi_ool(DisasContext
*s
, arg_rpri_esz
*a
,
969 gen_helper_gvec_3
*fn
)
971 if (sve_access_check(s
)) {
972 gen_gvec_ool_zzp(s
, fn
, a
->rd
, a
->rn
, a
->pg
, a
->imm
);
977 static bool trans_ASR_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
979 static gen_helper_gvec_3
* const fns
[4] = {
980 gen_helper_sve_asr_zpzi_b
, gen_helper_sve_asr_zpzi_h
,
981 gen_helper_sve_asr_zpzi_s
, gen_helper_sve_asr_zpzi_d
,
984 /* Invalid tsz encoding -- see tszimm_esz. */
987 /* Shift by element size is architecturally valid. For
988 arithmetic right-shift, it's the same as by one less. */
989 a
->imm
= MIN(a
->imm
, (8 << a
->esz
) - 1);
990 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
993 static bool trans_LSR_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
995 static gen_helper_gvec_3
* const fns
[4] = {
996 gen_helper_sve_lsr_zpzi_b
, gen_helper_sve_lsr_zpzi_h
,
997 gen_helper_sve_lsr_zpzi_s
, gen_helper_sve_lsr_zpzi_d
,
1002 /* Shift by element size is architecturally valid.
1003 For logical shifts, it is a zeroing operation. */
1004 if (a
->imm
>= (8 << a
->esz
)) {
1005 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
1007 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1011 static bool trans_LSL_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
1013 static gen_helper_gvec_3
* const fns
[4] = {
1014 gen_helper_sve_lsl_zpzi_b
, gen_helper_sve_lsl_zpzi_h
,
1015 gen_helper_sve_lsl_zpzi_s
, gen_helper_sve_lsl_zpzi_d
,
1020 /* Shift by element size is architecturally valid.
1021 For logical shifts, it is a zeroing operation. */
1022 if (a
->imm
>= (8 << a
->esz
)) {
1023 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
1025 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1029 static bool trans_ASRD(DisasContext
*s
, arg_rpri_esz
*a
)
1031 static gen_helper_gvec_3
* const fns
[4] = {
1032 gen_helper_sve_asrd_b
, gen_helper_sve_asrd_h
,
1033 gen_helper_sve_asrd_s
, gen_helper_sve_asrd_d
,
1038 /* Shift by element size is architecturally valid. For arithmetic
1039 right shift for division, it is a zeroing operation. */
1040 if (a
->imm
>= (8 << a
->esz
)) {
1041 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
1043 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
1048 *** SVE Bitwise Shift - Predicated Group
1051 #define DO_ZPZW(NAME, name) \
1052 static bool trans_##NAME##_zpzw(DisasContext *s, arg_rprr_esz *a) \
1054 static gen_helper_gvec_4 * const fns[3] = { \
1055 gen_helper_sve_##name##_zpzw_b, gen_helper_sve_##name##_zpzw_h, \
1056 gen_helper_sve_##name##_zpzw_s, \
1058 if (a->esz < 0 || a->esz >= 3) { \
1061 return do_zpzz_ool(s, a, fns[a->esz]); \
1071 *** SVE Bitwise Shift - Unpredicated Group
1074 static bool do_shift_imm(DisasContext
*s
, arg_rri_esz
*a
, bool asr
,
1075 void (*gvec_fn
)(unsigned, uint32_t, uint32_t,
1076 int64_t, uint32_t, uint32_t))
1079 /* Invalid tsz encoding -- see tszimm_esz. */
1082 if (sve_access_check(s
)) {
1083 unsigned vsz
= vec_full_reg_size(s
);
1084 /* Shift by element size is architecturally valid. For
1085 arithmetic right-shift, it's the same as by one less.
1086 Otherwise it is a zeroing operation. */
1087 if (a
->imm
>= 8 << a
->esz
) {
1089 a
->imm
= (8 << a
->esz
) - 1;
1091 do_dupi_z(s
, a
->rd
, 0);
1095 gvec_fn(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
1096 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
1101 static bool trans_ASR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
1103 return do_shift_imm(s
, a
, true, tcg_gen_gvec_sari
);
1106 static bool trans_LSR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
1108 return do_shift_imm(s
, a
, false, tcg_gen_gvec_shri
);
1111 static bool trans_LSL_zzi(DisasContext
*s
, arg_rri_esz
*a
)
1113 return do_shift_imm(s
, a
, false, tcg_gen_gvec_shli
);
1116 static bool do_zzw_ool(DisasContext
*s
, arg_rrr_esz
*a
, gen_helper_gvec_3
*fn
)
1121 if (sve_access_check(s
)) {
1122 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, 0);
1127 #define DO_ZZW(NAME, name) \
1128 static bool trans_##NAME##_zzw(DisasContext *s, arg_rrr_esz *a) \
1130 static gen_helper_gvec_3 * const fns[4] = { \
1131 gen_helper_sve_##name##_zzw_b, gen_helper_sve_##name##_zzw_h, \
1132 gen_helper_sve_##name##_zzw_s, NULL \
1134 return do_zzw_ool(s, a, fns[a->esz]); \
1144 *** SVE Integer Multiply-Add Group
1147 static bool do_zpzzz_ool(DisasContext
*s
, arg_rprrr_esz
*a
,
1148 gen_helper_gvec_5
*fn
)
1150 if (sve_access_check(s
)) {
1151 unsigned vsz
= vec_full_reg_size(s
);
1152 tcg_gen_gvec_5_ool(vec_full_reg_offset(s
, a
->rd
),
1153 vec_full_reg_offset(s
, a
->ra
),
1154 vec_full_reg_offset(s
, a
->rn
),
1155 vec_full_reg_offset(s
, a
->rm
),
1156 pred_full_reg_offset(s
, a
->pg
),
1162 #define DO_ZPZZZ(NAME, name) \
1163 static bool trans_##NAME(DisasContext *s, arg_rprrr_esz *a) \
1165 static gen_helper_gvec_5 * const fns[4] = { \
1166 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
1167 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
1169 return do_zpzzz_ool(s, a, fns[a->esz]); \
1178 *** SVE Index Generation Group
1181 static void do_index(DisasContext
*s
, int esz
, int rd
,
1182 TCGv_i64 start
, TCGv_i64 incr
)
1184 unsigned vsz
= vec_full_reg_size(s
);
1185 TCGv_i32 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
1186 TCGv_ptr t_zd
= tcg_temp_new_ptr();
1188 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, rd
));
1190 gen_helper_sve_index_d(t_zd
, start
, incr
, desc
);
1192 typedef void index_fn(TCGv_ptr
, TCGv_i32
, TCGv_i32
, TCGv_i32
);
1193 static index_fn
* const fns
[3] = {
1194 gen_helper_sve_index_b
,
1195 gen_helper_sve_index_h
,
1196 gen_helper_sve_index_s
,
1198 TCGv_i32 s32
= tcg_temp_new_i32();
1199 TCGv_i32 i32
= tcg_temp_new_i32();
1201 tcg_gen_extrl_i64_i32(s32
, start
);
1202 tcg_gen_extrl_i64_i32(i32
, incr
);
1203 fns
[esz
](t_zd
, s32
, i32
, desc
);
1205 tcg_temp_free_i32(s32
);
1206 tcg_temp_free_i32(i32
);
1208 tcg_temp_free_ptr(t_zd
);
1209 tcg_temp_free_i32(desc
);
1212 static bool trans_INDEX_ii(DisasContext
*s
, arg_INDEX_ii
*a
)
1214 if (sve_access_check(s
)) {
1215 TCGv_i64 start
= tcg_const_i64(a
->imm1
);
1216 TCGv_i64 incr
= tcg_const_i64(a
->imm2
);
1217 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1218 tcg_temp_free_i64(start
);
1219 tcg_temp_free_i64(incr
);
1224 static bool trans_INDEX_ir(DisasContext
*s
, arg_INDEX_ir
*a
)
1226 if (sve_access_check(s
)) {
1227 TCGv_i64 start
= tcg_const_i64(a
->imm
);
1228 TCGv_i64 incr
= cpu_reg(s
, a
->rm
);
1229 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1230 tcg_temp_free_i64(start
);
1235 static bool trans_INDEX_ri(DisasContext
*s
, arg_INDEX_ri
*a
)
1237 if (sve_access_check(s
)) {
1238 TCGv_i64 start
= cpu_reg(s
, a
->rn
);
1239 TCGv_i64 incr
= tcg_const_i64(a
->imm
);
1240 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1241 tcg_temp_free_i64(incr
);
1246 static bool trans_INDEX_rr(DisasContext
*s
, arg_INDEX_rr
*a
)
1248 if (sve_access_check(s
)) {
1249 TCGv_i64 start
= cpu_reg(s
, a
->rn
);
1250 TCGv_i64 incr
= cpu_reg(s
, a
->rm
);
1251 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
1257 *** SVE Stack Allocation Group
1260 static bool trans_ADDVL(DisasContext
*s
, arg_ADDVL
*a
)
1262 if (sve_access_check(s
)) {
1263 TCGv_i64 rd
= cpu_reg_sp(s
, a
->rd
);
1264 TCGv_i64 rn
= cpu_reg_sp(s
, a
->rn
);
1265 tcg_gen_addi_i64(rd
, rn
, a
->imm
* vec_full_reg_size(s
));
1270 static bool trans_ADDPL(DisasContext
*s
, arg_ADDPL
*a
)
1272 if (sve_access_check(s
)) {
1273 TCGv_i64 rd
= cpu_reg_sp(s
, a
->rd
);
1274 TCGv_i64 rn
= cpu_reg_sp(s
, a
->rn
);
1275 tcg_gen_addi_i64(rd
, rn
, a
->imm
* pred_full_reg_size(s
));
1280 static bool trans_RDVL(DisasContext
*s
, arg_RDVL
*a
)
1282 if (sve_access_check(s
)) {
1283 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
1284 tcg_gen_movi_i64(reg
, a
->imm
* vec_full_reg_size(s
));
1290 *** SVE Compute Vector Address Group
1293 static bool do_adr(DisasContext
*s
, arg_rrri
*a
, gen_helper_gvec_3
*fn
)
1295 if (sve_access_check(s
)) {
1296 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->imm
);
1301 static bool trans_ADR_p32(DisasContext
*s
, arg_rrri
*a
)
1303 return do_adr(s
, a
, gen_helper_sve_adr_p32
);
1306 static bool trans_ADR_p64(DisasContext
*s
, arg_rrri
*a
)
1308 return do_adr(s
, a
, gen_helper_sve_adr_p64
);
1311 static bool trans_ADR_s32(DisasContext
*s
, arg_rrri
*a
)
1313 return do_adr(s
, a
, gen_helper_sve_adr_s32
);
1316 static bool trans_ADR_u32(DisasContext
*s
, arg_rrri
*a
)
1318 return do_adr(s
, a
, gen_helper_sve_adr_u32
);
1322 *** SVE Integer Misc - Unpredicated Group
1325 static bool trans_FEXPA(DisasContext
*s
, arg_rr_esz
*a
)
1327 static gen_helper_gvec_2
* const fns
[4] = {
1329 gen_helper_sve_fexpa_h
,
1330 gen_helper_sve_fexpa_s
,
1331 gen_helper_sve_fexpa_d
,
1336 if (sve_access_check(s
)) {
1337 gen_gvec_ool_zz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, 0);
1342 static bool trans_FTSSEL(DisasContext
*s
, arg_rrr_esz
*a
)
1344 static gen_helper_gvec_3
* const fns
[4] = {
1346 gen_helper_sve_ftssel_h
,
1347 gen_helper_sve_ftssel_s
,
1348 gen_helper_sve_ftssel_d
,
1353 if (sve_access_check(s
)) {
1354 gen_gvec_ool_zzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, 0);
1360 *** SVE Predicate Logical Operations Group
1363 static bool do_pppp_flags(DisasContext
*s
, arg_rprr_s
*a
,
1364 const GVecGen4
*gvec_op
)
1366 if (!sve_access_check(s
)) {
1370 unsigned psz
= pred_gvec_reg_size(s
);
1371 int dofs
= pred_full_reg_offset(s
, a
->rd
);
1372 int nofs
= pred_full_reg_offset(s
, a
->rn
);
1373 int mofs
= pred_full_reg_offset(s
, a
->rm
);
1374 int gofs
= pred_full_reg_offset(s
, a
->pg
);
1377 tcg_gen_gvec_4(dofs
, nofs
, mofs
, gofs
, psz
, psz
, gvec_op
);
1382 /* Do the operation and the flags generation in temps. */
1383 TCGv_i64 pd
= tcg_temp_new_i64();
1384 TCGv_i64 pn
= tcg_temp_new_i64();
1385 TCGv_i64 pm
= tcg_temp_new_i64();
1386 TCGv_i64 pg
= tcg_temp_new_i64();
1388 tcg_gen_ld_i64(pn
, cpu_env
, nofs
);
1389 tcg_gen_ld_i64(pm
, cpu_env
, mofs
);
1390 tcg_gen_ld_i64(pg
, cpu_env
, gofs
);
1392 gvec_op
->fni8(pd
, pn
, pm
, pg
);
1393 tcg_gen_st_i64(pd
, cpu_env
, dofs
);
1395 do_predtest1(pd
, pg
);
1397 tcg_temp_free_i64(pd
);
1398 tcg_temp_free_i64(pn
);
1399 tcg_temp_free_i64(pm
);
1400 tcg_temp_free_i64(pg
);
1402 /* The operation and flags generation is large. The computation
1403 * of the flags depends on the original contents of the guarding
1404 * predicate. If the destination overwrites the guarding predicate,
1405 * then the easiest way to get this right is to save a copy.
1408 if (a
->rd
== a
->pg
) {
1409 tofs
= offsetof(CPUARMState
, vfp
.preg_tmp
);
1410 tcg_gen_gvec_mov(0, tofs
, gofs
, psz
, psz
);
1413 tcg_gen_gvec_4(dofs
, nofs
, mofs
, gofs
, psz
, psz
, gvec_op
);
1414 do_predtest(s
, dofs
, tofs
, psz
/ 8);
1419 static void gen_and_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1421 tcg_gen_and_i64(pd
, pn
, pm
);
1422 tcg_gen_and_i64(pd
, pd
, pg
);
1425 static void gen_and_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1426 TCGv_vec pm
, TCGv_vec pg
)
1428 tcg_gen_and_vec(vece
, pd
, pn
, pm
);
1429 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1432 static bool trans_AND_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1434 static const GVecGen4 op
= {
1435 .fni8
= gen_and_pg_i64
,
1436 .fniv
= gen_and_pg_vec
,
1437 .fno
= gen_helper_sve_and_pppp
,
1438 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1442 if (!sve_access_check(s
)) {
1445 if (a
->rn
== a
->rm
) {
1446 if (a
->pg
== a
->rn
) {
1447 do_mov_p(s
, a
->rd
, a
->rn
);
1449 gen_gvec_fn_ppp(s
, tcg_gen_gvec_and
, a
->rd
, a
->rn
, a
->pg
);
1452 } else if (a
->pg
== a
->rn
|| a
->pg
== a
->rm
) {
1453 gen_gvec_fn_ppp(s
, tcg_gen_gvec_and
, a
->rd
, a
->rn
, a
->rm
);
1457 return do_pppp_flags(s
, a
, &op
);
1460 static void gen_bic_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1462 tcg_gen_andc_i64(pd
, pn
, pm
);
1463 tcg_gen_and_i64(pd
, pd
, pg
);
1466 static void gen_bic_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1467 TCGv_vec pm
, TCGv_vec pg
)
1469 tcg_gen_andc_vec(vece
, pd
, pn
, pm
);
1470 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1473 static bool trans_BIC_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1475 static const GVecGen4 op
= {
1476 .fni8
= gen_bic_pg_i64
,
1477 .fniv
= gen_bic_pg_vec
,
1478 .fno
= gen_helper_sve_bic_pppp
,
1479 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1482 if (!a
->s
&& a
->pg
== a
->rn
) {
1483 if (sve_access_check(s
)) {
1484 gen_gvec_fn_ppp(s
, tcg_gen_gvec_andc
, a
->rd
, a
->rn
, a
->rm
);
1488 return do_pppp_flags(s
, a
, &op
);
1491 static void gen_eor_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1493 tcg_gen_xor_i64(pd
, pn
, pm
);
1494 tcg_gen_and_i64(pd
, pd
, pg
);
1497 static void gen_eor_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1498 TCGv_vec pm
, TCGv_vec pg
)
1500 tcg_gen_xor_vec(vece
, pd
, pn
, pm
);
1501 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1504 static bool trans_EOR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1506 static const GVecGen4 op
= {
1507 .fni8
= gen_eor_pg_i64
,
1508 .fniv
= gen_eor_pg_vec
,
1509 .fno
= gen_helper_sve_eor_pppp
,
1510 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1512 return do_pppp_flags(s
, a
, &op
);
1515 static bool trans_SEL_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1520 if (sve_access_check(s
)) {
1521 unsigned psz
= pred_gvec_reg_size(s
);
1522 tcg_gen_gvec_bitsel(MO_8
, pred_full_reg_offset(s
, a
->rd
),
1523 pred_full_reg_offset(s
, a
->pg
),
1524 pred_full_reg_offset(s
, a
->rn
),
1525 pred_full_reg_offset(s
, a
->rm
), psz
, psz
);
1530 static void gen_orr_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1532 tcg_gen_or_i64(pd
, pn
, pm
);
1533 tcg_gen_and_i64(pd
, pd
, pg
);
1536 static void gen_orr_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1537 TCGv_vec pm
, TCGv_vec pg
)
1539 tcg_gen_or_vec(vece
, pd
, pn
, pm
);
1540 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1543 static bool trans_ORR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1545 static const GVecGen4 op
= {
1546 .fni8
= gen_orr_pg_i64
,
1547 .fniv
= gen_orr_pg_vec
,
1548 .fno
= gen_helper_sve_orr_pppp
,
1549 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1552 if (!a
->s
&& a
->pg
== a
->rn
&& a
->rn
== a
->rm
) {
1553 return do_mov_p(s
, a
->rd
, a
->rn
);
1555 return do_pppp_flags(s
, a
, &op
);
1558 static void gen_orn_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1560 tcg_gen_orc_i64(pd
, pn
, pm
);
1561 tcg_gen_and_i64(pd
, pd
, pg
);
1564 static void gen_orn_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1565 TCGv_vec pm
, TCGv_vec pg
)
1567 tcg_gen_orc_vec(vece
, pd
, pn
, pm
);
1568 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1571 static bool trans_ORN_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1573 static const GVecGen4 op
= {
1574 .fni8
= gen_orn_pg_i64
,
1575 .fniv
= gen_orn_pg_vec
,
1576 .fno
= gen_helper_sve_orn_pppp
,
1577 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1579 return do_pppp_flags(s
, a
, &op
);
1582 static void gen_nor_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1584 tcg_gen_or_i64(pd
, pn
, pm
);
1585 tcg_gen_andc_i64(pd
, pg
, pd
);
1588 static void gen_nor_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1589 TCGv_vec pm
, TCGv_vec pg
)
1591 tcg_gen_or_vec(vece
, pd
, pn
, pm
);
1592 tcg_gen_andc_vec(vece
, pd
, pg
, pd
);
1595 static bool trans_NOR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1597 static const GVecGen4 op
= {
1598 .fni8
= gen_nor_pg_i64
,
1599 .fniv
= gen_nor_pg_vec
,
1600 .fno
= gen_helper_sve_nor_pppp
,
1601 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1603 return do_pppp_flags(s
, a
, &op
);
1606 static void gen_nand_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1608 tcg_gen_and_i64(pd
, pn
, pm
);
1609 tcg_gen_andc_i64(pd
, pg
, pd
);
1612 static void gen_nand_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1613 TCGv_vec pm
, TCGv_vec pg
)
1615 tcg_gen_and_vec(vece
, pd
, pn
, pm
);
1616 tcg_gen_andc_vec(vece
, pd
, pg
, pd
);
1619 static bool trans_NAND_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1621 static const GVecGen4 op
= {
1622 .fni8
= gen_nand_pg_i64
,
1623 .fniv
= gen_nand_pg_vec
,
1624 .fno
= gen_helper_sve_nand_pppp
,
1625 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1627 return do_pppp_flags(s
, a
, &op
);
1631 *** SVE Predicate Misc Group
1634 static bool trans_PTEST(DisasContext
*s
, arg_PTEST
*a
)
1636 if (sve_access_check(s
)) {
1637 int nofs
= pred_full_reg_offset(s
, a
->rn
);
1638 int gofs
= pred_full_reg_offset(s
, a
->pg
);
1639 int words
= DIV_ROUND_UP(pred_full_reg_size(s
), 8);
1642 TCGv_i64 pn
= tcg_temp_new_i64();
1643 TCGv_i64 pg
= tcg_temp_new_i64();
1645 tcg_gen_ld_i64(pn
, cpu_env
, nofs
);
1646 tcg_gen_ld_i64(pg
, cpu_env
, gofs
);
1647 do_predtest1(pn
, pg
);
1649 tcg_temp_free_i64(pn
);
1650 tcg_temp_free_i64(pg
);
1652 do_predtest(s
, nofs
, gofs
, words
);
1658 /* See the ARM pseudocode DecodePredCount. */
1659 static unsigned decode_pred_count(unsigned fullsz
, int pattern
, int esz
)
1661 unsigned elements
= fullsz
>> esz
;
1665 case 0x0: /* POW2 */
1666 return pow2floor(elements
);
1677 case 0x9: /* VL16 */
1678 case 0xa: /* VL32 */
1679 case 0xb: /* VL64 */
1680 case 0xc: /* VL128 */
1681 case 0xd: /* VL256 */
1682 bound
= 16 << (pattern
- 9);
1684 case 0x1d: /* MUL4 */
1685 return elements
- elements
% 4;
1686 case 0x1e: /* MUL3 */
1687 return elements
- elements
% 3;
1688 case 0x1f: /* ALL */
1690 default: /* #uimm5 */
1693 return elements
>= bound
? bound
: 0;
1696 /* This handles all of the predicate initialization instructions,
1697 * PTRUE, PFALSE, SETFFR. For PFALSE, we will have set PAT == 32
1698 * so that decode_pred_count returns 0. For SETFFR, we will have
1699 * set RD == 16 == FFR.
1701 static bool do_predset(DisasContext
*s
, int esz
, int rd
, int pat
, bool setflag
)
1703 if (!sve_access_check(s
)) {
1707 unsigned fullsz
= vec_full_reg_size(s
);
1708 unsigned ofs
= pred_full_reg_offset(s
, rd
);
1709 unsigned numelem
, setsz
, i
;
1710 uint64_t word
, lastword
;
1713 numelem
= decode_pred_count(fullsz
, pat
, esz
);
1715 /* Determine what we must store into each bit, and how many. */
1717 lastword
= word
= 0;
1720 setsz
= numelem
<< esz
;
1721 lastword
= word
= pred_esz_masks
[esz
];
1723 lastword
&= MAKE_64BIT_MASK(0, setsz
% 64);
1727 t
= tcg_temp_new_i64();
1729 tcg_gen_movi_i64(t
, lastword
);
1730 tcg_gen_st_i64(t
, cpu_env
, ofs
);
1734 if (word
== lastword
) {
1735 unsigned maxsz
= size_for_gvec(fullsz
/ 8);
1736 unsigned oprsz
= size_for_gvec(setsz
/ 8);
1738 if (oprsz
* 8 == setsz
) {
1739 tcg_gen_gvec_dup_imm(MO_64
, ofs
, oprsz
, maxsz
, word
);
1747 tcg_gen_movi_i64(t
, word
);
1748 for (i
= 0; i
< QEMU_ALIGN_DOWN(setsz
, 8); i
+= 8) {
1749 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1751 if (lastword
!= word
) {
1752 tcg_gen_movi_i64(t
, lastword
);
1753 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1757 tcg_gen_movi_i64(t
, 0);
1758 for (; i
< fullsz
; i
+= 8) {
1759 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1764 tcg_temp_free_i64(t
);
1768 tcg_gen_movi_i32(cpu_NF
, -(word
!= 0));
1769 tcg_gen_movi_i32(cpu_CF
, word
== 0);
1770 tcg_gen_movi_i32(cpu_VF
, 0);
1771 tcg_gen_mov_i32(cpu_ZF
, cpu_NF
);
1776 static bool trans_PTRUE(DisasContext
*s
, arg_PTRUE
*a
)
1778 return do_predset(s
, a
->esz
, a
->rd
, a
->pat
, a
->s
);
1781 static bool trans_SETFFR(DisasContext
*s
, arg_SETFFR
*a
)
1783 /* Note pat == 31 is #all, to set all elements. */
1784 return do_predset(s
, 0, FFR_PRED_NUM
, 31, false);
1787 static bool trans_PFALSE(DisasContext
*s
, arg_PFALSE
*a
)
1789 /* Note pat == 32 is #unimp, to set no elements. */
1790 return do_predset(s
, 0, a
->rd
, 32, false);
1793 static bool trans_RDFFR_p(DisasContext
*s
, arg_RDFFR_p
*a
)
1795 /* The path through do_pppp_flags is complicated enough to want to avoid
1796 * duplication. Frob the arguments into the form of a predicated AND.
1798 arg_rprr_s alt_a
= {
1799 .rd
= a
->rd
, .pg
= a
->pg
, .s
= a
->s
,
1800 .rn
= FFR_PRED_NUM
, .rm
= FFR_PRED_NUM
,
1802 return trans_AND_pppp(s
, &alt_a
);
1805 static bool trans_RDFFR(DisasContext
*s
, arg_RDFFR
*a
)
1807 return do_mov_p(s
, a
->rd
, FFR_PRED_NUM
);
1810 static bool trans_WRFFR(DisasContext
*s
, arg_WRFFR
*a
)
1812 return do_mov_p(s
, FFR_PRED_NUM
, a
->rn
);
1815 static bool do_pfirst_pnext(DisasContext
*s
, arg_rr_esz
*a
,
1816 void (*gen_fn
)(TCGv_i32
, TCGv_ptr
,
1817 TCGv_ptr
, TCGv_i32
))
1819 if (!sve_access_check(s
)) {
1823 TCGv_ptr t_pd
= tcg_temp_new_ptr();
1824 TCGv_ptr t_pg
= tcg_temp_new_ptr();
1828 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, pred_full_reg_size(s
));
1829 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
1831 tcg_gen_addi_ptr(t_pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
1832 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
1833 t
= tcg_const_i32(desc
);
1835 gen_fn(t
, t_pd
, t_pg
, t
);
1836 tcg_temp_free_ptr(t_pd
);
1837 tcg_temp_free_ptr(t_pg
);
1840 tcg_temp_free_i32(t
);
1844 static bool trans_PFIRST(DisasContext
*s
, arg_rr_esz
*a
)
1846 return do_pfirst_pnext(s
, a
, gen_helper_sve_pfirst
);
1849 static bool trans_PNEXT(DisasContext
*s
, arg_rr_esz
*a
)
1851 return do_pfirst_pnext(s
, a
, gen_helper_sve_pnext
);
1855 *** SVE Element Count Group
1858 /* Perform an inline saturating addition of a 32-bit value within
1859 * a 64-bit register. The second operand is known to be positive,
1860 * which halves the comparisions we must perform to bound the result.
1862 static void do_sat_addsub_32(TCGv_i64 reg
, TCGv_i64 val
, bool u
, bool d
)
1868 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
1870 tcg_gen_ext32u_i64(reg
, reg
);
1872 tcg_gen_ext32s_i64(reg
, reg
);
1875 tcg_gen_sub_i64(reg
, reg
, val
);
1876 ibound
= (u
? 0 : INT32_MIN
);
1879 tcg_gen_add_i64(reg
, reg
, val
);
1880 ibound
= (u
? UINT32_MAX
: INT32_MAX
);
1883 bound
= tcg_const_i64(ibound
);
1884 tcg_gen_movcond_i64(cond
, reg
, reg
, bound
, bound
, reg
);
1885 tcg_temp_free_i64(bound
);
1888 /* Similarly with 64-bit values. */
1889 static void do_sat_addsub_64(TCGv_i64 reg
, TCGv_i64 val
, bool u
, bool d
)
1891 TCGv_i64 t0
= tcg_temp_new_i64();
1892 TCGv_i64 t1
= tcg_temp_new_i64();
1897 tcg_gen_sub_i64(t0
, reg
, val
);
1898 tcg_gen_movi_i64(t1
, 0);
1899 tcg_gen_movcond_i64(TCG_COND_LTU
, reg
, reg
, val
, t1
, t0
);
1901 tcg_gen_add_i64(t0
, reg
, val
);
1902 tcg_gen_movi_i64(t1
, -1);
1903 tcg_gen_movcond_i64(TCG_COND_LTU
, reg
, t0
, reg
, t1
, t0
);
1907 /* Detect signed overflow for subtraction. */
1908 tcg_gen_xor_i64(t0
, reg
, val
);
1909 tcg_gen_sub_i64(t1
, reg
, val
);
1910 tcg_gen_xor_i64(reg
, reg
, t1
);
1911 tcg_gen_and_i64(t0
, t0
, reg
);
1913 /* Bound the result. */
1914 tcg_gen_movi_i64(reg
, INT64_MIN
);
1915 t2
= tcg_const_i64(0);
1916 tcg_gen_movcond_i64(TCG_COND_LT
, reg
, t0
, t2
, reg
, t1
);
1918 /* Detect signed overflow for addition. */
1919 tcg_gen_xor_i64(t0
, reg
, val
);
1920 tcg_gen_add_i64(reg
, reg
, val
);
1921 tcg_gen_xor_i64(t1
, reg
, val
);
1922 tcg_gen_andc_i64(t0
, t1
, t0
);
1924 /* Bound the result. */
1925 tcg_gen_movi_i64(t1
, INT64_MAX
);
1926 t2
= tcg_const_i64(0);
1927 tcg_gen_movcond_i64(TCG_COND_LT
, reg
, t0
, t2
, t1
, reg
);
1929 tcg_temp_free_i64(t2
);
1931 tcg_temp_free_i64(t0
);
1932 tcg_temp_free_i64(t1
);
1935 /* Similarly with a vector and a scalar operand. */
1936 static void do_sat_addsub_vec(DisasContext
*s
, int esz
, int rd
, int rn
,
1937 TCGv_i64 val
, bool u
, bool d
)
1939 unsigned vsz
= vec_full_reg_size(s
);
1940 TCGv_ptr dptr
, nptr
;
1944 dptr
= tcg_temp_new_ptr();
1945 nptr
= tcg_temp_new_ptr();
1946 tcg_gen_addi_ptr(dptr
, cpu_env
, vec_full_reg_offset(s
, rd
));
1947 tcg_gen_addi_ptr(nptr
, cpu_env
, vec_full_reg_offset(s
, rn
));
1948 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
1952 t32
= tcg_temp_new_i32();
1953 tcg_gen_extrl_i64_i32(t32
, val
);
1955 tcg_gen_neg_i32(t32
, t32
);
1958 gen_helper_sve_uqaddi_b(dptr
, nptr
, t32
, desc
);
1960 gen_helper_sve_sqaddi_b(dptr
, nptr
, t32
, desc
);
1962 tcg_temp_free_i32(t32
);
1966 t32
= tcg_temp_new_i32();
1967 tcg_gen_extrl_i64_i32(t32
, val
);
1969 tcg_gen_neg_i32(t32
, t32
);
1972 gen_helper_sve_uqaddi_h(dptr
, nptr
, t32
, desc
);
1974 gen_helper_sve_sqaddi_h(dptr
, nptr
, t32
, desc
);
1976 tcg_temp_free_i32(t32
);
1980 t64
= tcg_temp_new_i64();
1982 tcg_gen_neg_i64(t64
, val
);
1984 tcg_gen_mov_i64(t64
, val
);
1987 gen_helper_sve_uqaddi_s(dptr
, nptr
, t64
, desc
);
1989 gen_helper_sve_sqaddi_s(dptr
, nptr
, t64
, desc
);
1991 tcg_temp_free_i64(t64
);
1997 gen_helper_sve_uqsubi_d(dptr
, nptr
, val
, desc
);
1999 gen_helper_sve_uqaddi_d(dptr
, nptr
, val
, desc
);
2002 t64
= tcg_temp_new_i64();
2003 tcg_gen_neg_i64(t64
, val
);
2004 gen_helper_sve_sqaddi_d(dptr
, nptr
, t64
, desc
);
2005 tcg_temp_free_i64(t64
);
2007 gen_helper_sve_sqaddi_d(dptr
, nptr
, val
, desc
);
2012 g_assert_not_reached();
2015 tcg_temp_free_ptr(dptr
);
2016 tcg_temp_free_ptr(nptr
);
2017 tcg_temp_free_i32(desc
);
2020 static bool trans_CNT_r(DisasContext
*s
, arg_CNT_r
*a
)
2022 if (sve_access_check(s
)) {
2023 unsigned fullsz
= vec_full_reg_size(s
);
2024 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2025 tcg_gen_movi_i64(cpu_reg(s
, a
->rd
), numelem
* a
->imm
);
2030 static bool trans_INCDEC_r(DisasContext
*s
, arg_incdec_cnt
*a
)
2032 if (sve_access_check(s
)) {
2033 unsigned fullsz
= vec_full_reg_size(s
);
2034 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2035 int inc
= numelem
* a
->imm
* (a
->d
? -1 : 1);
2036 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
2038 tcg_gen_addi_i64(reg
, reg
, inc
);
2043 static bool trans_SINCDEC_r_32(DisasContext
*s
, arg_incdec_cnt
*a
)
2045 if (!sve_access_check(s
)) {
2049 unsigned fullsz
= vec_full_reg_size(s
);
2050 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2051 int inc
= numelem
* a
->imm
;
2052 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
2054 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
2057 tcg_gen_ext32u_i64(reg
, reg
);
2059 tcg_gen_ext32s_i64(reg
, reg
);
2062 TCGv_i64 t
= tcg_const_i64(inc
);
2063 do_sat_addsub_32(reg
, t
, a
->u
, a
->d
);
2064 tcg_temp_free_i64(t
);
2069 static bool trans_SINCDEC_r_64(DisasContext
*s
, arg_incdec_cnt
*a
)
2071 if (!sve_access_check(s
)) {
2075 unsigned fullsz
= vec_full_reg_size(s
);
2076 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2077 int inc
= numelem
* a
->imm
;
2078 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
2081 TCGv_i64 t
= tcg_const_i64(inc
);
2082 do_sat_addsub_64(reg
, t
, a
->u
, a
->d
);
2083 tcg_temp_free_i64(t
);
2088 static bool trans_INCDEC_v(DisasContext
*s
, arg_incdec2_cnt
*a
)
2094 unsigned fullsz
= vec_full_reg_size(s
);
2095 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2096 int inc
= numelem
* a
->imm
;
2099 if (sve_access_check(s
)) {
2100 TCGv_i64 t
= tcg_const_i64(a
->d
? -inc
: inc
);
2101 tcg_gen_gvec_adds(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
2102 vec_full_reg_offset(s
, a
->rn
),
2104 tcg_temp_free_i64(t
);
2107 do_mov_z(s
, a
->rd
, a
->rn
);
2112 static bool trans_SINCDEC_v(DisasContext
*s
, arg_incdec2_cnt
*a
)
2118 unsigned fullsz
= vec_full_reg_size(s
);
2119 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
2120 int inc
= numelem
* a
->imm
;
2123 if (sve_access_check(s
)) {
2124 TCGv_i64 t
= tcg_const_i64(inc
);
2125 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
, t
, a
->u
, a
->d
);
2126 tcg_temp_free_i64(t
);
2129 do_mov_z(s
, a
->rd
, a
->rn
);
2135 *** SVE Bitwise Immediate Group
2138 static bool do_zz_dbm(DisasContext
*s
, arg_rr_dbm
*a
, GVecGen2iFn
*gvec_fn
)
2141 if (!logic_imm_decode_wmask(&imm
, extract32(a
->dbm
, 12, 1),
2142 extract32(a
->dbm
, 0, 6),
2143 extract32(a
->dbm
, 6, 6))) {
2146 if (sve_access_check(s
)) {
2147 unsigned vsz
= vec_full_reg_size(s
);
2148 gvec_fn(MO_64
, vec_full_reg_offset(s
, a
->rd
),
2149 vec_full_reg_offset(s
, a
->rn
), imm
, vsz
, vsz
);
2154 static bool trans_AND_zzi(DisasContext
*s
, arg_rr_dbm
*a
)
2156 return do_zz_dbm(s
, a
, tcg_gen_gvec_andi
);
2159 static bool trans_ORR_zzi(DisasContext
*s
, arg_rr_dbm
*a
)
2161 return do_zz_dbm(s
, a
, tcg_gen_gvec_ori
);
2164 static bool trans_EOR_zzi(DisasContext
*s
, arg_rr_dbm
*a
)
2166 return do_zz_dbm(s
, a
, tcg_gen_gvec_xori
);
2169 static bool trans_DUPM(DisasContext
*s
, arg_DUPM
*a
)
2172 if (!logic_imm_decode_wmask(&imm
, extract32(a
->dbm
, 12, 1),
2173 extract32(a
->dbm
, 0, 6),
2174 extract32(a
->dbm
, 6, 6))) {
2177 if (sve_access_check(s
)) {
2178 do_dupi_z(s
, a
->rd
, imm
);
2184 *** SVE Integer Wide Immediate - Predicated Group
2187 /* Implement all merging copies. This is used for CPY (immediate),
2188 * FCPY, CPY (scalar), CPY (SIMD&FP scalar).
2190 static void do_cpy_m(DisasContext
*s
, int esz
, int rd
, int rn
, int pg
,
2193 typedef void gen_cpy(TCGv_ptr
, TCGv_ptr
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
2194 static gen_cpy
* const fns
[4] = {
2195 gen_helper_sve_cpy_m_b
, gen_helper_sve_cpy_m_h
,
2196 gen_helper_sve_cpy_m_s
, gen_helper_sve_cpy_m_d
,
2198 unsigned vsz
= vec_full_reg_size(s
);
2199 TCGv_i32 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
2200 TCGv_ptr t_zd
= tcg_temp_new_ptr();
2201 TCGv_ptr t_zn
= tcg_temp_new_ptr();
2202 TCGv_ptr t_pg
= tcg_temp_new_ptr();
2204 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, rd
));
2205 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, rn
));
2206 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
2208 fns
[esz
](t_zd
, t_zn
, t_pg
, val
, desc
);
2210 tcg_temp_free_ptr(t_zd
);
2211 tcg_temp_free_ptr(t_zn
);
2212 tcg_temp_free_ptr(t_pg
);
2213 tcg_temp_free_i32(desc
);
2216 static bool trans_FCPY(DisasContext
*s
, arg_FCPY
*a
)
2221 if (sve_access_check(s
)) {
2222 /* Decode the VFP immediate. */
2223 uint64_t imm
= vfp_expand_imm(a
->esz
, a
->imm
);
2224 TCGv_i64 t_imm
= tcg_const_i64(imm
);
2225 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rn
, a
->pg
, t_imm
);
2226 tcg_temp_free_i64(t_imm
);
2231 static bool trans_CPY_m_i(DisasContext
*s
, arg_rpri_esz
*a
)
2233 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
2236 if (sve_access_check(s
)) {
2237 TCGv_i64 t_imm
= tcg_const_i64(a
->imm
);
2238 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rn
, a
->pg
, t_imm
);
2239 tcg_temp_free_i64(t_imm
);
2244 static bool trans_CPY_z_i(DisasContext
*s
, arg_CPY_z_i
*a
)
2246 static gen_helper_gvec_2i
* const fns
[4] = {
2247 gen_helper_sve_cpy_z_b
, gen_helper_sve_cpy_z_h
,
2248 gen_helper_sve_cpy_z_s
, gen_helper_sve_cpy_z_d
,
2251 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
2254 if (sve_access_check(s
)) {
2255 unsigned vsz
= vec_full_reg_size(s
);
2256 TCGv_i64 t_imm
= tcg_const_i64(a
->imm
);
2257 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s
, a
->rd
),
2258 pred_full_reg_offset(s
, a
->pg
),
2259 t_imm
, vsz
, vsz
, 0, fns
[a
->esz
]);
2260 tcg_temp_free_i64(t_imm
);
2266 *** SVE Permute Extract Group
2269 static bool do_EXT(DisasContext
*s
, int rd
, int rn
, int rm
, int imm
)
2271 if (!sve_access_check(s
)) {
2275 unsigned vsz
= vec_full_reg_size(s
);
2276 unsigned n_ofs
= imm
>= vsz
? 0 : imm
;
2277 unsigned n_siz
= vsz
- n_ofs
;
2278 unsigned d
= vec_full_reg_offset(s
, rd
);
2279 unsigned n
= vec_full_reg_offset(s
, rn
);
2280 unsigned m
= vec_full_reg_offset(s
, rm
);
2282 /* Use host vector move insns if we have appropriate sizes
2283 * and no unfortunate overlap.
2286 && n_ofs
== size_for_gvec(n_ofs
)
2287 && n_siz
== size_for_gvec(n_siz
)
2288 && (d
!= n
|| n_siz
<= n_ofs
)) {
2289 tcg_gen_gvec_mov(0, d
, n
+ n_ofs
, n_siz
, n_siz
);
2291 tcg_gen_gvec_mov(0, d
+ n_siz
, m
, n_ofs
, n_ofs
);
2294 tcg_gen_gvec_3_ool(d
, n
, m
, vsz
, vsz
, n_ofs
, gen_helper_sve_ext
);
2299 static bool trans_EXT(DisasContext
*s
, arg_EXT
*a
)
2301 return do_EXT(s
, a
->rd
, a
->rn
, a
->rm
, a
->imm
);
2304 static bool trans_EXT_sve2(DisasContext
*s
, arg_rri
*a
)
2306 if (!dc_isar_feature(aa64_sve2
, s
)) {
2309 return do_EXT(s
, a
->rd
, a
->rn
, (a
->rn
+ 1) % 32, a
->imm
);
2313 *** SVE Permute - Unpredicated Group
2316 static bool trans_DUP_s(DisasContext
*s
, arg_DUP_s
*a
)
2318 if (sve_access_check(s
)) {
2319 unsigned vsz
= vec_full_reg_size(s
);
2320 tcg_gen_gvec_dup_i64(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
2321 vsz
, vsz
, cpu_reg_sp(s
, a
->rn
));
2326 static bool trans_DUP_x(DisasContext
*s
, arg_DUP_x
*a
)
2328 if ((a
->imm
& 0x1f) == 0) {
2331 if (sve_access_check(s
)) {
2332 unsigned vsz
= vec_full_reg_size(s
);
2333 unsigned dofs
= vec_full_reg_offset(s
, a
->rd
);
2334 unsigned esz
, index
;
2336 esz
= ctz32(a
->imm
);
2337 index
= a
->imm
>> (esz
+ 1);
2339 if ((index
<< esz
) < vsz
) {
2340 unsigned nofs
= vec_reg_offset(s
, a
->rn
, index
, esz
);
2341 tcg_gen_gvec_dup_mem(esz
, dofs
, nofs
, vsz
, vsz
);
2344 * While dup_mem handles 128-bit elements, dup_imm does not.
2345 * Thankfully element size doesn't matter for splatting zero.
2347 tcg_gen_gvec_dup_imm(MO_64
, dofs
, vsz
, vsz
, 0);
2353 static void do_insr_i64(DisasContext
*s
, arg_rrr_esz
*a
, TCGv_i64 val
)
2355 typedef void gen_insr(TCGv_ptr
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
2356 static gen_insr
* const fns
[4] = {
2357 gen_helper_sve_insr_b
, gen_helper_sve_insr_h
,
2358 gen_helper_sve_insr_s
, gen_helper_sve_insr_d
,
2360 unsigned vsz
= vec_full_reg_size(s
);
2361 TCGv_i32 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
2362 TCGv_ptr t_zd
= tcg_temp_new_ptr();
2363 TCGv_ptr t_zn
= tcg_temp_new_ptr();
2365 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, a
->rd
));
2366 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
2368 fns
[a
->esz
](t_zd
, t_zn
, val
, desc
);
2370 tcg_temp_free_ptr(t_zd
);
2371 tcg_temp_free_ptr(t_zn
);
2372 tcg_temp_free_i32(desc
);
2375 static bool trans_INSR_f(DisasContext
*s
, arg_rrr_esz
*a
)
2377 if (sve_access_check(s
)) {
2378 TCGv_i64 t
= tcg_temp_new_i64();
2379 tcg_gen_ld_i64(t
, cpu_env
, vec_reg_offset(s
, a
->rm
, 0, MO_64
));
2380 do_insr_i64(s
, a
, t
);
2381 tcg_temp_free_i64(t
);
2386 static bool trans_INSR_r(DisasContext
*s
, arg_rrr_esz
*a
)
2388 if (sve_access_check(s
)) {
2389 do_insr_i64(s
, a
, cpu_reg(s
, a
->rm
));
2394 static bool trans_REV_v(DisasContext
*s
, arg_rr_esz
*a
)
2396 static gen_helper_gvec_2
* const fns
[4] = {
2397 gen_helper_sve_rev_b
, gen_helper_sve_rev_h
,
2398 gen_helper_sve_rev_s
, gen_helper_sve_rev_d
2401 if (sve_access_check(s
)) {
2402 gen_gvec_ool_zz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, 0);
2407 static bool trans_TBL(DisasContext
*s
, arg_rrr_esz
*a
)
2409 static gen_helper_gvec_3
* const fns
[4] = {
2410 gen_helper_sve_tbl_b
, gen_helper_sve_tbl_h
,
2411 gen_helper_sve_tbl_s
, gen_helper_sve_tbl_d
2414 if (sve_access_check(s
)) {
2415 gen_gvec_ool_zzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, 0);
2420 static bool trans_TBL_sve2(DisasContext
*s
, arg_rrr_esz
*a
)
2422 static gen_helper_gvec_4
* const fns
[4] = {
2423 gen_helper_sve2_tbl_b
, gen_helper_sve2_tbl_h
,
2424 gen_helper_sve2_tbl_s
, gen_helper_sve2_tbl_d
2427 if (!dc_isar_feature(aa64_sve2
, s
)) {
2430 if (sve_access_check(s
)) {
2431 gen_gvec_ool_zzzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
,
2432 (a
->rn
+ 1) % 32, a
->rm
, 0);
2437 static bool trans_TBX(DisasContext
*s
, arg_rrr_esz
*a
)
2439 static gen_helper_gvec_3
* const fns
[4] = {
2440 gen_helper_sve2_tbx_b
, gen_helper_sve2_tbx_h
,
2441 gen_helper_sve2_tbx_s
, gen_helper_sve2_tbx_d
2444 if (!dc_isar_feature(aa64_sve2
, s
)) {
2447 if (sve_access_check(s
)) {
2448 gen_gvec_ool_zzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, 0);
2453 static bool trans_UNPK(DisasContext
*s
, arg_UNPK
*a
)
2455 static gen_helper_gvec_2
* const fns
[4][2] = {
2457 { gen_helper_sve_sunpk_h
, gen_helper_sve_uunpk_h
},
2458 { gen_helper_sve_sunpk_s
, gen_helper_sve_uunpk_s
},
2459 { gen_helper_sve_sunpk_d
, gen_helper_sve_uunpk_d
},
2465 if (sve_access_check(s
)) {
2466 unsigned vsz
= vec_full_reg_size(s
);
2467 tcg_gen_gvec_2_ool(vec_full_reg_offset(s
, a
->rd
),
2468 vec_full_reg_offset(s
, a
->rn
)
2469 + (a
->h
? vsz
/ 2 : 0),
2470 vsz
, vsz
, 0, fns
[a
->esz
][a
->u
]);
2476 *** SVE Permute - Predicates Group
2479 static bool do_perm_pred3(DisasContext
*s
, arg_rrr_esz
*a
, bool high_odd
,
2480 gen_helper_gvec_3
*fn
)
2482 if (!sve_access_check(s
)) {
2486 unsigned vsz
= pred_full_reg_size(s
);
2488 TCGv_ptr t_d
= tcg_temp_new_ptr();
2489 TCGv_ptr t_n
= tcg_temp_new_ptr();
2490 TCGv_ptr t_m
= tcg_temp_new_ptr();
2494 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
);
2495 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
2496 desc
= FIELD_DP32(desc
, PREDDESC
, DATA
, high_odd
);
2498 tcg_gen_addi_ptr(t_d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2499 tcg_gen_addi_ptr(t_n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2500 tcg_gen_addi_ptr(t_m
, cpu_env
, pred_full_reg_offset(s
, a
->rm
));
2501 t_desc
= tcg_const_i32(desc
);
2503 fn(t_d
, t_n
, t_m
, t_desc
);
2505 tcg_temp_free_ptr(t_d
);
2506 tcg_temp_free_ptr(t_n
);
2507 tcg_temp_free_ptr(t_m
);
2508 tcg_temp_free_i32(t_desc
);
2512 static bool do_perm_pred2(DisasContext
*s
, arg_rr_esz
*a
, bool high_odd
,
2513 gen_helper_gvec_2
*fn
)
2515 if (!sve_access_check(s
)) {
2519 unsigned vsz
= pred_full_reg_size(s
);
2520 TCGv_ptr t_d
= tcg_temp_new_ptr();
2521 TCGv_ptr t_n
= tcg_temp_new_ptr();
2525 tcg_gen_addi_ptr(t_d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2526 tcg_gen_addi_ptr(t_n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2528 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
);
2529 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
2530 desc
= FIELD_DP32(desc
, PREDDESC
, DATA
, high_odd
);
2531 t_desc
= tcg_const_i32(desc
);
2533 fn(t_d
, t_n
, t_desc
);
2535 tcg_temp_free_i32(t_desc
);
2536 tcg_temp_free_ptr(t_d
);
2537 tcg_temp_free_ptr(t_n
);
2541 static bool trans_ZIP1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2543 return do_perm_pred3(s
, a
, 0, gen_helper_sve_zip_p
);
2546 static bool trans_ZIP2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2548 return do_perm_pred3(s
, a
, 1, gen_helper_sve_zip_p
);
2551 static bool trans_UZP1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2553 return do_perm_pred3(s
, a
, 0, gen_helper_sve_uzp_p
);
2556 static bool trans_UZP2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2558 return do_perm_pred3(s
, a
, 1, gen_helper_sve_uzp_p
);
2561 static bool trans_TRN1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2563 return do_perm_pred3(s
, a
, 0, gen_helper_sve_trn_p
);
2566 static bool trans_TRN2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2568 return do_perm_pred3(s
, a
, 1, gen_helper_sve_trn_p
);
2571 static bool trans_REV_p(DisasContext
*s
, arg_rr_esz
*a
)
2573 return do_perm_pred2(s
, a
, 0, gen_helper_sve_rev_p
);
2576 static bool trans_PUNPKLO(DisasContext
*s
, arg_PUNPKLO
*a
)
2578 return do_perm_pred2(s
, a
, 0, gen_helper_sve_punpk_p
);
2581 static bool trans_PUNPKHI(DisasContext
*s
, arg_PUNPKHI
*a
)
2583 return do_perm_pred2(s
, a
, 1, gen_helper_sve_punpk_p
);
2587 *** SVE Permute - Interleaving Group
2590 static bool do_zip(DisasContext
*s
, arg_rrr_esz
*a
, bool high
)
2592 static gen_helper_gvec_3
* const fns
[4] = {
2593 gen_helper_sve_zip_b
, gen_helper_sve_zip_h
,
2594 gen_helper_sve_zip_s
, gen_helper_sve_zip_d
,
2597 if (sve_access_check(s
)) {
2598 unsigned vsz
= vec_full_reg_size(s
);
2599 unsigned high_ofs
= high
? vsz
/ 2 : 0;
2600 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, a
->rd
),
2601 vec_full_reg_offset(s
, a
->rn
) + high_ofs
,
2602 vec_full_reg_offset(s
, a
->rm
) + high_ofs
,
2603 vsz
, vsz
, 0, fns
[a
->esz
]);
2608 static bool do_zzz_data_ool(DisasContext
*s
, arg_rrr_esz
*a
, int data
,
2609 gen_helper_gvec_3
*fn
)
2611 if (sve_access_check(s
)) {
2612 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, data
);
2617 static bool trans_ZIP1_z(DisasContext
*s
, arg_rrr_esz
*a
)
2619 return do_zip(s
, a
, false);
2622 static bool trans_ZIP2_z(DisasContext
*s
, arg_rrr_esz
*a
)
2624 return do_zip(s
, a
, true);
2627 static gen_helper_gvec_3
* const uzp_fns
[4] = {
2628 gen_helper_sve_uzp_b
, gen_helper_sve_uzp_h
,
2629 gen_helper_sve_uzp_s
, gen_helper_sve_uzp_d
,
2632 static bool trans_UZP1_z(DisasContext
*s
, arg_rrr_esz
*a
)
2634 return do_zzz_data_ool(s
, a
, 0, uzp_fns
[a
->esz
]);
2637 static bool trans_UZP2_z(DisasContext
*s
, arg_rrr_esz
*a
)
2639 return do_zzz_data_ool(s
, a
, 1 << a
->esz
, uzp_fns
[a
->esz
]);
2642 static gen_helper_gvec_3
* const trn_fns
[4] = {
2643 gen_helper_sve_trn_b
, gen_helper_sve_trn_h
,
2644 gen_helper_sve_trn_s
, gen_helper_sve_trn_d
,
2647 static bool trans_TRN1_z(DisasContext
*s
, arg_rrr_esz
*a
)
2649 return do_zzz_data_ool(s
, a
, 0, trn_fns
[a
->esz
]);
2652 static bool trans_TRN2_z(DisasContext
*s
, arg_rrr_esz
*a
)
2654 return do_zzz_data_ool(s
, a
, 1 << a
->esz
, trn_fns
[a
->esz
]);
2658 *** SVE Permute Vector - Predicated Group
2661 static bool trans_COMPACT(DisasContext
*s
, arg_rpr_esz
*a
)
2663 static gen_helper_gvec_3
* const fns
[4] = {
2664 NULL
, NULL
, gen_helper_sve_compact_s
, gen_helper_sve_compact_d
2666 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
2669 /* Call the helper that computes the ARM LastActiveElement pseudocode
2670 * function, scaled by the element size. This includes the not found
2671 * indication; e.g. not found for esz=3 is -8.
2673 static void find_last_active(DisasContext
*s
, TCGv_i32 ret
, int esz
, int pg
)
2675 /* Predicate sizes may be smaller and cannot use simd_desc. We cannot
2676 * round up, as we do elsewhere, because we need the exact size.
2678 TCGv_ptr t_p
= tcg_temp_new_ptr();
2682 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, pred_full_reg_size(s
));
2683 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, esz
);
2685 tcg_gen_addi_ptr(t_p
, cpu_env
, pred_full_reg_offset(s
, pg
));
2686 t_desc
= tcg_const_i32(desc
);
2688 gen_helper_sve_last_active_element(ret
, t_p
, t_desc
);
2690 tcg_temp_free_i32(t_desc
);
2691 tcg_temp_free_ptr(t_p
);
2694 /* Increment LAST to the offset of the next element in the vector,
2695 * wrapping around to 0.
2697 static void incr_last_active(DisasContext
*s
, TCGv_i32 last
, int esz
)
2699 unsigned vsz
= vec_full_reg_size(s
);
2701 tcg_gen_addi_i32(last
, last
, 1 << esz
);
2702 if (is_power_of_2(vsz
)) {
2703 tcg_gen_andi_i32(last
, last
, vsz
- 1);
2705 TCGv_i32 max
= tcg_const_i32(vsz
);
2706 TCGv_i32 zero
= tcg_const_i32(0);
2707 tcg_gen_movcond_i32(TCG_COND_GEU
, last
, last
, max
, zero
, last
);
2708 tcg_temp_free_i32(max
);
2709 tcg_temp_free_i32(zero
);
2713 /* If LAST < 0, set LAST to the offset of the last element in the vector. */
2714 static void wrap_last_active(DisasContext
*s
, TCGv_i32 last
, int esz
)
2716 unsigned vsz
= vec_full_reg_size(s
);
2718 if (is_power_of_2(vsz
)) {
2719 tcg_gen_andi_i32(last
, last
, vsz
- 1);
2721 TCGv_i32 max
= tcg_const_i32(vsz
- (1 << esz
));
2722 TCGv_i32 zero
= tcg_const_i32(0);
2723 tcg_gen_movcond_i32(TCG_COND_LT
, last
, last
, zero
, max
, last
);
2724 tcg_temp_free_i32(max
);
2725 tcg_temp_free_i32(zero
);
2729 /* Load an unsigned element of ESZ from BASE+OFS. */
2730 static TCGv_i64
load_esz(TCGv_ptr base
, int ofs
, int esz
)
2732 TCGv_i64 r
= tcg_temp_new_i64();
2736 tcg_gen_ld8u_i64(r
, base
, ofs
);
2739 tcg_gen_ld16u_i64(r
, base
, ofs
);
2742 tcg_gen_ld32u_i64(r
, base
, ofs
);
2745 tcg_gen_ld_i64(r
, base
, ofs
);
2748 g_assert_not_reached();
2753 /* Load an unsigned element of ESZ from RM[LAST]. */
2754 static TCGv_i64
load_last_active(DisasContext
*s
, TCGv_i32 last
,
2757 TCGv_ptr p
= tcg_temp_new_ptr();
2760 /* Convert offset into vector into offset into ENV.
2761 * The final adjustment for the vector register base
2762 * is added via constant offset to the load.
2764 #ifdef HOST_WORDS_BIGENDIAN
2765 /* Adjust for element ordering. See vec_reg_offset. */
2767 tcg_gen_xori_i32(last
, last
, 8 - (1 << esz
));
2770 tcg_gen_ext_i32_ptr(p
, last
);
2771 tcg_gen_add_ptr(p
, p
, cpu_env
);
2773 r
= load_esz(p
, vec_full_reg_offset(s
, rm
), esz
);
2774 tcg_temp_free_ptr(p
);
2779 /* Compute CLAST for a Zreg. */
2780 static bool do_clast_vector(DisasContext
*s
, arg_rprr_esz
*a
, bool before
)
2785 unsigned vsz
, esz
= a
->esz
;
2787 if (!sve_access_check(s
)) {
2791 last
= tcg_temp_local_new_i32();
2792 over
= gen_new_label();
2794 find_last_active(s
, last
, esz
, a
->pg
);
2796 /* There is of course no movcond for a 2048-bit vector,
2797 * so we must branch over the actual store.
2799 tcg_gen_brcondi_i32(TCG_COND_LT
, last
, 0, over
);
2802 incr_last_active(s
, last
, esz
);
2805 ele
= load_last_active(s
, last
, a
->rm
, esz
);
2806 tcg_temp_free_i32(last
);
2808 vsz
= vec_full_reg_size(s
);
2809 tcg_gen_gvec_dup_i64(esz
, vec_full_reg_offset(s
, a
->rd
), vsz
, vsz
, ele
);
2810 tcg_temp_free_i64(ele
);
2812 /* If this insn used MOVPRFX, we may need a second move. */
2813 if (a
->rd
!= a
->rn
) {
2814 TCGLabel
*done
= gen_new_label();
2817 gen_set_label(over
);
2818 do_mov_z(s
, a
->rd
, a
->rn
);
2820 gen_set_label(done
);
2822 gen_set_label(over
);
2827 static bool trans_CLASTA_z(DisasContext
*s
, arg_rprr_esz
*a
)
2829 return do_clast_vector(s
, a
, false);
2832 static bool trans_CLASTB_z(DisasContext
*s
, arg_rprr_esz
*a
)
2834 return do_clast_vector(s
, a
, true);
2837 /* Compute CLAST for a scalar. */
2838 static void do_clast_scalar(DisasContext
*s
, int esz
, int pg
, int rm
,
2839 bool before
, TCGv_i64 reg_val
)
2841 TCGv_i32 last
= tcg_temp_new_i32();
2842 TCGv_i64 ele
, cmp
, zero
;
2844 find_last_active(s
, last
, esz
, pg
);
2846 /* Extend the original value of last prior to incrementing. */
2847 cmp
= tcg_temp_new_i64();
2848 tcg_gen_ext_i32_i64(cmp
, last
);
2851 incr_last_active(s
, last
, esz
);
2854 /* The conceit here is that while last < 0 indicates not found, after
2855 * adjusting for cpu_env->vfp.zregs[rm], it is still a valid address
2856 * from which we can load garbage. We then discard the garbage with
2857 * a conditional move.
2859 ele
= load_last_active(s
, last
, rm
, esz
);
2860 tcg_temp_free_i32(last
);
2862 zero
= tcg_const_i64(0);
2863 tcg_gen_movcond_i64(TCG_COND_GE
, reg_val
, cmp
, zero
, ele
, reg_val
);
2865 tcg_temp_free_i64(zero
);
2866 tcg_temp_free_i64(cmp
);
2867 tcg_temp_free_i64(ele
);
2870 /* Compute CLAST for a Vreg. */
2871 static bool do_clast_fp(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2873 if (sve_access_check(s
)) {
2875 int ofs
= vec_reg_offset(s
, a
->rd
, 0, esz
);
2876 TCGv_i64 reg
= load_esz(cpu_env
, ofs
, esz
);
2878 do_clast_scalar(s
, esz
, a
->pg
, a
->rn
, before
, reg
);
2879 write_fp_dreg(s
, a
->rd
, reg
);
2880 tcg_temp_free_i64(reg
);
2885 static bool trans_CLASTA_v(DisasContext
*s
, arg_rpr_esz
*a
)
2887 return do_clast_fp(s
, a
, false);
2890 static bool trans_CLASTB_v(DisasContext
*s
, arg_rpr_esz
*a
)
2892 return do_clast_fp(s
, a
, true);
2895 /* Compute CLAST for a Xreg. */
2896 static bool do_clast_general(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2900 if (!sve_access_check(s
)) {
2904 reg
= cpu_reg(s
, a
->rd
);
2907 tcg_gen_ext8u_i64(reg
, reg
);
2910 tcg_gen_ext16u_i64(reg
, reg
);
2913 tcg_gen_ext32u_i64(reg
, reg
);
2918 g_assert_not_reached();
2921 do_clast_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
, reg
);
2925 static bool trans_CLASTA_r(DisasContext
*s
, arg_rpr_esz
*a
)
2927 return do_clast_general(s
, a
, false);
2930 static bool trans_CLASTB_r(DisasContext
*s
, arg_rpr_esz
*a
)
2932 return do_clast_general(s
, a
, true);
2935 /* Compute LAST for a scalar. */
2936 static TCGv_i64
do_last_scalar(DisasContext
*s
, int esz
,
2937 int pg
, int rm
, bool before
)
2939 TCGv_i32 last
= tcg_temp_new_i32();
2942 find_last_active(s
, last
, esz
, pg
);
2944 wrap_last_active(s
, last
, esz
);
2946 incr_last_active(s
, last
, esz
);
2949 ret
= load_last_active(s
, last
, rm
, esz
);
2950 tcg_temp_free_i32(last
);
2954 /* Compute LAST for a Vreg. */
2955 static bool do_last_fp(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2957 if (sve_access_check(s
)) {
2958 TCGv_i64 val
= do_last_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
);
2959 write_fp_dreg(s
, a
->rd
, val
);
2960 tcg_temp_free_i64(val
);
2965 static bool trans_LASTA_v(DisasContext
*s
, arg_rpr_esz
*a
)
2967 return do_last_fp(s
, a
, false);
2970 static bool trans_LASTB_v(DisasContext
*s
, arg_rpr_esz
*a
)
2972 return do_last_fp(s
, a
, true);
2975 /* Compute LAST for a Xreg. */
2976 static bool do_last_general(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2978 if (sve_access_check(s
)) {
2979 TCGv_i64 val
= do_last_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
);
2980 tcg_gen_mov_i64(cpu_reg(s
, a
->rd
), val
);
2981 tcg_temp_free_i64(val
);
2986 static bool trans_LASTA_r(DisasContext
*s
, arg_rpr_esz
*a
)
2988 return do_last_general(s
, a
, false);
2991 static bool trans_LASTB_r(DisasContext
*s
, arg_rpr_esz
*a
)
2993 return do_last_general(s
, a
, true);
2996 static bool trans_CPY_m_r(DisasContext
*s
, arg_rpr_esz
*a
)
2998 if (sve_access_check(s
)) {
2999 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rd
, a
->pg
, cpu_reg_sp(s
, a
->rn
));
3004 static bool trans_CPY_m_v(DisasContext
*s
, arg_rpr_esz
*a
)
3006 if (sve_access_check(s
)) {
3007 int ofs
= vec_reg_offset(s
, a
->rn
, 0, a
->esz
);
3008 TCGv_i64 t
= load_esz(cpu_env
, ofs
, a
->esz
);
3009 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rd
, a
->pg
, t
);
3010 tcg_temp_free_i64(t
);
3015 static bool trans_REVB(DisasContext
*s
, arg_rpr_esz
*a
)
3017 static gen_helper_gvec_3
* const fns
[4] = {
3019 gen_helper_sve_revb_h
,
3020 gen_helper_sve_revb_s
,
3021 gen_helper_sve_revb_d
,
3023 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
3026 static bool trans_REVH(DisasContext
*s
, arg_rpr_esz
*a
)
3028 static gen_helper_gvec_3
* const fns
[4] = {
3031 gen_helper_sve_revh_s
,
3032 gen_helper_sve_revh_d
,
3034 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
3037 static bool trans_REVW(DisasContext
*s
, arg_rpr_esz
*a
)
3039 return do_zpz_ool(s
, a
, a
->esz
== 3 ? gen_helper_sve_revw_d
: NULL
);
3042 static bool trans_RBIT(DisasContext
*s
, arg_rpr_esz
*a
)
3044 static gen_helper_gvec_3
* const fns
[4] = {
3045 gen_helper_sve_rbit_b
,
3046 gen_helper_sve_rbit_h
,
3047 gen_helper_sve_rbit_s
,
3048 gen_helper_sve_rbit_d
,
3050 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
3053 static bool trans_SPLICE(DisasContext
*s
, arg_rprr_esz
*a
)
3055 if (sve_access_check(s
)) {
3056 gen_gvec_ool_zzzp(s
, gen_helper_sve_splice
,
3057 a
->rd
, a
->rn
, a
->rm
, a
->pg
, a
->esz
);
3062 static bool trans_SPLICE_sve2(DisasContext
*s
, arg_rpr_esz
*a
)
3064 if (!dc_isar_feature(aa64_sve2
, s
)) {
3067 if (sve_access_check(s
)) {
3068 gen_gvec_ool_zzzp(s
, gen_helper_sve_splice
,
3069 a
->rd
, a
->rn
, (a
->rn
+ 1) % 32, a
->pg
, a
->esz
);
3075 *** SVE Integer Compare - Vectors Group
3078 static bool do_ppzz_flags(DisasContext
*s
, arg_rprr_esz
*a
,
3079 gen_helper_gvec_flags_4
*gen_fn
)
3081 TCGv_ptr pd
, zn
, zm
, pg
;
3085 if (gen_fn
== NULL
) {
3088 if (!sve_access_check(s
)) {
3092 vsz
= vec_full_reg_size(s
);
3093 t
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
3094 pd
= tcg_temp_new_ptr();
3095 zn
= tcg_temp_new_ptr();
3096 zm
= tcg_temp_new_ptr();
3097 pg
= tcg_temp_new_ptr();
3099 tcg_gen_addi_ptr(pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3100 tcg_gen_addi_ptr(zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
3101 tcg_gen_addi_ptr(zm
, cpu_env
, vec_full_reg_offset(s
, a
->rm
));
3102 tcg_gen_addi_ptr(pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3104 gen_fn(t
, pd
, zn
, zm
, pg
, t
);
3106 tcg_temp_free_ptr(pd
);
3107 tcg_temp_free_ptr(zn
);
3108 tcg_temp_free_ptr(zm
);
3109 tcg_temp_free_ptr(pg
);
3113 tcg_temp_free_i32(t
);
3117 #define DO_PPZZ(NAME, name) \
3118 static bool trans_##NAME##_ppzz(DisasContext *s, arg_rprr_esz *a) \
3120 static gen_helper_gvec_flags_4 * const fns[4] = { \
3121 gen_helper_sve_##name##_ppzz_b, gen_helper_sve_##name##_ppzz_h, \
3122 gen_helper_sve_##name##_ppzz_s, gen_helper_sve_##name##_ppzz_d, \
3124 return do_ppzz_flags(s, a, fns[a->esz]); \
3127 DO_PPZZ(CMPEQ
, cmpeq
)
3128 DO_PPZZ(CMPNE
, cmpne
)
3129 DO_PPZZ(CMPGT
, cmpgt
)
3130 DO_PPZZ(CMPGE
, cmpge
)
3131 DO_PPZZ(CMPHI
, cmphi
)
3132 DO_PPZZ(CMPHS
, cmphs
)
3136 #define DO_PPZW(NAME, name) \
3137 static bool trans_##NAME##_ppzw(DisasContext *s, arg_rprr_esz *a) \
3139 static gen_helper_gvec_flags_4 * const fns[4] = { \
3140 gen_helper_sve_##name##_ppzw_b, gen_helper_sve_##name##_ppzw_h, \
3141 gen_helper_sve_##name##_ppzw_s, NULL \
3143 return do_ppzz_flags(s, a, fns[a->esz]); \
3146 DO_PPZW(CMPEQ
, cmpeq
)
3147 DO_PPZW(CMPNE
, cmpne
)
3148 DO_PPZW(CMPGT
, cmpgt
)
3149 DO_PPZW(CMPGE
, cmpge
)
3150 DO_PPZW(CMPHI
, cmphi
)
3151 DO_PPZW(CMPHS
, cmphs
)
3152 DO_PPZW(CMPLT
, cmplt
)
3153 DO_PPZW(CMPLE
, cmple
)
3154 DO_PPZW(CMPLO
, cmplo
)
3155 DO_PPZW(CMPLS
, cmpls
)
3160 *** SVE Integer Compare - Immediate Groups
3163 static bool do_ppzi_flags(DisasContext
*s
, arg_rpri_esz
*a
,
3164 gen_helper_gvec_flags_3
*gen_fn
)
3166 TCGv_ptr pd
, zn
, pg
;
3170 if (gen_fn
== NULL
) {
3173 if (!sve_access_check(s
)) {
3177 vsz
= vec_full_reg_size(s
);
3178 t
= tcg_const_i32(simd_desc(vsz
, vsz
, a
->imm
));
3179 pd
= tcg_temp_new_ptr();
3180 zn
= tcg_temp_new_ptr();
3181 pg
= tcg_temp_new_ptr();
3183 tcg_gen_addi_ptr(pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3184 tcg_gen_addi_ptr(zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
3185 tcg_gen_addi_ptr(pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3187 gen_fn(t
, pd
, zn
, pg
, t
);
3189 tcg_temp_free_ptr(pd
);
3190 tcg_temp_free_ptr(zn
);
3191 tcg_temp_free_ptr(pg
);
3195 tcg_temp_free_i32(t
);
3199 #define DO_PPZI(NAME, name) \
3200 static bool trans_##NAME##_ppzi(DisasContext *s, arg_rpri_esz *a) \
3202 static gen_helper_gvec_flags_3 * const fns[4] = { \
3203 gen_helper_sve_##name##_ppzi_b, gen_helper_sve_##name##_ppzi_h, \
3204 gen_helper_sve_##name##_ppzi_s, gen_helper_sve_##name##_ppzi_d, \
3206 return do_ppzi_flags(s, a, fns[a->esz]); \
3209 DO_PPZI(CMPEQ
, cmpeq
)
3210 DO_PPZI(CMPNE
, cmpne
)
3211 DO_PPZI(CMPGT
, cmpgt
)
3212 DO_PPZI(CMPGE
, cmpge
)
3213 DO_PPZI(CMPHI
, cmphi
)
3214 DO_PPZI(CMPHS
, cmphs
)
3215 DO_PPZI(CMPLT
, cmplt
)
3216 DO_PPZI(CMPLE
, cmple
)
3217 DO_PPZI(CMPLO
, cmplo
)
3218 DO_PPZI(CMPLS
, cmpls
)
3223 *** SVE Partition Break Group
3226 static bool do_brk3(DisasContext
*s
, arg_rprr_s
*a
,
3227 gen_helper_gvec_4
*fn
, gen_helper_gvec_flags_4
*fn_s
)
3229 if (!sve_access_check(s
)) {
3233 unsigned vsz
= pred_full_reg_size(s
);
3235 /* Predicate sizes may be smaller and cannot use simd_desc. */
3236 TCGv_ptr d
= tcg_temp_new_ptr();
3237 TCGv_ptr n
= tcg_temp_new_ptr();
3238 TCGv_ptr m
= tcg_temp_new_ptr();
3239 TCGv_ptr g
= tcg_temp_new_ptr();
3240 TCGv_i32 t
= tcg_const_i32(FIELD_DP32(0, PREDDESC
, OPRSZ
, vsz
));
3242 tcg_gen_addi_ptr(d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3243 tcg_gen_addi_ptr(n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
3244 tcg_gen_addi_ptr(m
, cpu_env
, pred_full_reg_offset(s
, a
->rm
));
3245 tcg_gen_addi_ptr(g
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3248 fn_s(t
, d
, n
, m
, g
, t
);
3253 tcg_temp_free_ptr(d
);
3254 tcg_temp_free_ptr(n
);
3255 tcg_temp_free_ptr(m
);
3256 tcg_temp_free_ptr(g
);
3257 tcg_temp_free_i32(t
);
3261 static bool do_brk2(DisasContext
*s
, arg_rpr_s
*a
,
3262 gen_helper_gvec_3
*fn
, gen_helper_gvec_flags_3
*fn_s
)
3264 if (!sve_access_check(s
)) {
3268 unsigned vsz
= pred_full_reg_size(s
);
3270 /* Predicate sizes may be smaller and cannot use simd_desc. */
3271 TCGv_ptr d
= tcg_temp_new_ptr();
3272 TCGv_ptr n
= tcg_temp_new_ptr();
3273 TCGv_ptr g
= tcg_temp_new_ptr();
3274 TCGv_i32 t
= tcg_const_i32(FIELD_DP32(0, PREDDESC
, OPRSZ
, vsz
));
3276 tcg_gen_addi_ptr(d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3277 tcg_gen_addi_ptr(n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
3278 tcg_gen_addi_ptr(g
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3281 fn_s(t
, d
, n
, g
, t
);
3286 tcg_temp_free_ptr(d
);
3287 tcg_temp_free_ptr(n
);
3288 tcg_temp_free_ptr(g
);
3289 tcg_temp_free_i32(t
);
3293 static bool trans_BRKPA(DisasContext
*s
, arg_rprr_s
*a
)
3295 return do_brk3(s
, a
, gen_helper_sve_brkpa
, gen_helper_sve_brkpas
);
3298 static bool trans_BRKPB(DisasContext
*s
, arg_rprr_s
*a
)
3300 return do_brk3(s
, a
, gen_helper_sve_brkpb
, gen_helper_sve_brkpbs
);
3303 static bool trans_BRKA_m(DisasContext
*s
, arg_rpr_s
*a
)
3305 return do_brk2(s
, a
, gen_helper_sve_brka_m
, gen_helper_sve_brkas_m
);
3308 static bool trans_BRKB_m(DisasContext
*s
, arg_rpr_s
*a
)
3310 return do_brk2(s
, a
, gen_helper_sve_brkb_m
, gen_helper_sve_brkbs_m
);
3313 static bool trans_BRKA_z(DisasContext
*s
, arg_rpr_s
*a
)
3315 return do_brk2(s
, a
, gen_helper_sve_brka_z
, gen_helper_sve_brkas_z
);
3318 static bool trans_BRKB_z(DisasContext
*s
, arg_rpr_s
*a
)
3320 return do_brk2(s
, a
, gen_helper_sve_brkb_z
, gen_helper_sve_brkbs_z
);
3323 static bool trans_BRKN(DisasContext
*s
, arg_rpr_s
*a
)
3325 return do_brk2(s
, a
, gen_helper_sve_brkn
, gen_helper_sve_brkns
);
3329 *** SVE Predicate Count Group
3332 static void do_cntp(DisasContext
*s
, TCGv_i64 val
, int esz
, int pn
, int pg
)
3334 unsigned psz
= pred_full_reg_size(s
);
3339 tcg_gen_ld_i64(val
, cpu_env
, pred_full_reg_offset(s
, pn
));
3341 TCGv_i64 g
= tcg_temp_new_i64();
3342 tcg_gen_ld_i64(g
, cpu_env
, pred_full_reg_offset(s
, pg
));
3343 tcg_gen_and_i64(val
, val
, g
);
3344 tcg_temp_free_i64(g
);
3347 /* Reduce the pred_esz_masks value simply to reduce the
3348 * size of the code generated here.
3350 psz_mask
= MAKE_64BIT_MASK(0, psz
* 8);
3351 tcg_gen_andi_i64(val
, val
, pred_esz_masks
[esz
] & psz_mask
);
3353 tcg_gen_ctpop_i64(val
, val
);
3355 TCGv_ptr t_pn
= tcg_temp_new_ptr();
3356 TCGv_ptr t_pg
= tcg_temp_new_ptr();
3360 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, psz
);
3361 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, esz
);
3363 tcg_gen_addi_ptr(t_pn
, cpu_env
, pred_full_reg_offset(s
, pn
));
3364 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
3365 t_desc
= tcg_const_i32(desc
);
3367 gen_helper_sve_cntp(val
, t_pn
, t_pg
, t_desc
);
3368 tcg_temp_free_ptr(t_pn
);
3369 tcg_temp_free_ptr(t_pg
);
3370 tcg_temp_free_i32(t_desc
);
3374 static bool trans_CNTP(DisasContext
*s
, arg_CNTP
*a
)
3376 if (sve_access_check(s
)) {
3377 do_cntp(s
, cpu_reg(s
, a
->rd
), a
->esz
, a
->rn
, a
->pg
);
3382 static bool trans_INCDECP_r(DisasContext
*s
, arg_incdec_pred
*a
)
3384 if (sve_access_check(s
)) {
3385 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3386 TCGv_i64 val
= tcg_temp_new_i64();
3388 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3390 tcg_gen_sub_i64(reg
, reg
, val
);
3392 tcg_gen_add_i64(reg
, reg
, val
);
3394 tcg_temp_free_i64(val
);
3399 static bool trans_INCDECP_z(DisasContext
*s
, arg_incdec2_pred
*a
)
3404 if (sve_access_check(s
)) {
3405 unsigned vsz
= vec_full_reg_size(s
);
3406 TCGv_i64 val
= tcg_temp_new_i64();
3407 GVecGen2sFn
*gvec_fn
= a
->d
? tcg_gen_gvec_subs
: tcg_gen_gvec_adds
;
3409 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3410 gvec_fn(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3411 vec_full_reg_offset(s
, a
->rn
), val
, vsz
, vsz
);
3416 static bool trans_SINCDECP_r_32(DisasContext
*s
, arg_incdec_pred
*a
)
3418 if (sve_access_check(s
)) {
3419 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3420 TCGv_i64 val
= tcg_temp_new_i64();
3422 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3423 do_sat_addsub_32(reg
, val
, a
->u
, a
->d
);
3428 static bool trans_SINCDECP_r_64(DisasContext
*s
, arg_incdec_pred
*a
)
3430 if (sve_access_check(s
)) {
3431 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3432 TCGv_i64 val
= tcg_temp_new_i64();
3434 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3435 do_sat_addsub_64(reg
, val
, a
->u
, a
->d
);
3440 static bool trans_SINCDECP_z(DisasContext
*s
, arg_incdec2_pred
*a
)
3445 if (sve_access_check(s
)) {
3446 TCGv_i64 val
= tcg_temp_new_i64();
3447 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3448 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
, val
, a
->u
, a
->d
);
3454 *** SVE Integer Compare Scalars Group
3457 static bool trans_CTERM(DisasContext
*s
, arg_CTERM
*a
)
3459 if (!sve_access_check(s
)) {
3463 TCGCond cond
= (a
->ne
? TCG_COND_NE
: TCG_COND_EQ
);
3464 TCGv_i64 rn
= read_cpu_reg(s
, a
->rn
, a
->sf
);
3465 TCGv_i64 rm
= read_cpu_reg(s
, a
->rm
, a
->sf
);
3466 TCGv_i64 cmp
= tcg_temp_new_i64();
3468 tcg_gen_setcond_i64(cond
, cmp
, rn
, rm
);
3469 tcg_gen_extrl_i64_i32(cpu_NF
, cmp
);
3470 tcg_temp_free_i64(cmp
);
3472 /* VF = !NF & !CF. */
3473 tcg_gen_xori_i32(cpu_VF
, cpu_NF
, 1);
3474 tcg_gen_andc_i32(cpu_VF
, cpu_VF
, cpu_CF
);
3476 /* Both NF and VF actually look at bit 31. */
3477 tcg_gen_neg_i32(cpu_NF
, cpu_NF
);
3478 tcg_gen_neg_i32(cpu_VF
, cpu_VF
);
3482 static bool trans_WHILE(DisasContext
*s
, arg_WHILE
*a
)
3484 TCGv_i64 op0
, op1
, t0
, t1
, tmax
;
3487 unsigned vsz
= vec_full_reg_size(s
);
3491 /* Note that GE/HS has a->eq == 0 and GT/HI has a->eq == 1. */
3492 bool eq
= a
->eq
== a
->lt
;
3494 /* The greater-than conditions are all SVE2. */
3495 if (!a
->lt
&& !dc_isar_feature(aa64_sve2
, s
)) {
3498 if (!sve_access_check(s
)) {
3502 op0
= read_cpu_reg(s
, a
->rn
, 1);
3503 op1
= read_cpu_reg(s
, a
->rm
, 1);
3507 tcg_gen_ext32u_i64(op0
, op0
);
3508 tcg_gen_ext32u_i64(op1
, op1
);
3510 tcg_gen_ext32s_i64(op0
, op0
);
3511 tcg_gen_ext32s_i64(op1
, op1
);
3515 /* For the helper, compress the different conditions into a computation
3516 * of how many iterations for which the condition is true.
3518 t0
= tcg_temp_new_i64();
3519 t1
= tcg_temp_new_i64();
3522 tcg_gen_sub_i64(t0
, op1
, op0
);
3524 maxval
= a
->sf
? UINT64_MAX
: UINT32_MAX
;
3525 cond
= eq
? TCG_COND_LEU
: TCG_COND_LTU
;
3527 maxval
= a
->sf
? INT64_MAX
: INT32_MAX
;
3528 cond
= eq
? TCG_COND_LE
: TCG_COND_LT
;
3531 tcg_gen_sub_i64(t0
, op0
, op1
);
3534 cond
= eq
? TCG_COND_GEU
: TCG_COND_GTU
;
3536 maxval
= a
->sf
? INT64_MIN
: INT32_MIN
;
3537 cond
= eq
? TCG_COND_GE
: TCG_COND_GT
;
3541 tmax
= tcg_const_i64(vsz
>> a
->esz
);
3543 /* Equality means one more iteration. */
3544 tcg_gen_addi_i64(t0
, t0
, 1);
3547 * For the less-than while, if op1 is maxval (and the only time
3548 * the addition above could overflow), then we produce an all-true
3549 * predicate by setting the count to the vector length. This is
3550 * because the pseudocode is described as an increment + compare
3551 * loop, and the maximum integer would always compare true.
3552 * Similarly, the greater-than while has the same issue with the
3553 * minimum integer due to the decrement + compare loop.
3555 tcg_gen_movi_i64(t1
, maxval
);
3556 tcg_gen_movcond_i64(TCG_COND_EQ
, t0
, op1
, t1
, tmax
, t0
);
3559 /* Bound to the maximum. */
3560 tcg_gen_umin_i64(t0
, t0
, tmax
);
3561 tcg_temp_free_i64(tmax
);
3563 /* Set the count to zero if the condition is false. */
3564 tcg_gen_movi_i64(t1
, 0);
3565 tcg_gen_movcond_i64(cond
, t0
, op0
, op1
, t0
, t1
);
3566 tcg_temp_free_i64(t1
);
3568 /* Since we're bounded, pass as a 32-bit type. */
3569 t2
= tcg_temp_new_i32();
3570 tcg_gen_extrl_i64_i32(t2
, t0
);
3571 tcg_temp_free_i64(t0
);
3573 /* Scale elements to bits. */
3574 tcg_gen_shli_i32(t2
, t2
, a
->esz
);
3576 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
/ 8);
3577 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
3578 t3
= tcg_const_i32(desc
);
3580 ptr
= tcg_temp_new_ptr();
3581 tcg_gen_addi_ptr(ptr
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3584 gen_helper_sve_whilel(t2
, ptr
, t2
, t3
);
3586 gen_helper_sve_whileg(t2
, ptr
, t2
, t3
);
3590 tcg_temp_free_ptr(ptr
);
3591 tcg_temp_free_i32(t2
);
3592 tcg_temp_free_i32(t3
);
3596 static bool trans_WHILE_ptr(DisasContext
*s
, arg_WHILE_ptr
*a
)
3598 TCGv_i64 op0
, op1
, diff
, t1
, tmax
;
3601 unsigned vsz
= vec_full_reg_size(s
);
3604 if (!dc_isar_feature(aa64_sve2
, s
)) {
3607 if (!sve_access_check(s
)) {
3611 op0
= read_cpu_reg(s
, a
->rn
, 1);
3612 op1
= read_cpu_reg(s
, a
->rm
, 1);
3614 tmax
= tcg_const_i64(vsz
);
3615 diff
= tcg_temp_new_i64();
3619 /* diff = abs(op1 - op0), noting that op0/1 are unsigned. */
3620 t1
= tcg_temp_new_i64();
3621 tcg_gen_sub_i64(diff
, op0
, op1
);
3622 tcg_gen_sub_i64(t1
, op1
, op0
);
3623 tcg_gen_movcond_i64(TCG_COND_GEU
, diff
, op0
, op1
, diff
, t1
);
3624 tcg_temp_free_i64(t1
);
3625 /* Round down to a multiple of ESIZE. */
3626 tcg_gen_andi_i64(diff
, diff
, -1 << a
->esz
);
3627 /* If op1 == op0, diff == 0, and the condition is always true. */
3628 tcg_gen_movcond_i64(TCG_COND_EQ
, diff
, op0
, op1
, tmax
, diff
);
3631 tcg_gen_sub_i64(diff
, op1
, op0
);
3632 /* Round down to a multiple of ESIZE. */
3633 tcg_gen_andi_i64(diff
, diff
, -1 << a
->esz
);
3634 /* If op0 >= op1, diff <= 0, the condition is always true. */
3635 tcg_gen_movcond_i64(TCG_COND_GEU
, diff
, op0
, op1
, tmax
, diff
);
3638 /* Bound to the maximum. */
3639 tcg_gen_umin_i64(diff
, diff
, tmax
);
3640 tcg_temp_free_i64(tmax
);
3642 /* Since we're bounded, pass as a 32-bit type. */
3643 t2
= tcg_temp_new_i32();
3644 tcg_gen_extrl_i64_i32(t2
, diff
);
3645 tcg_temp_free_i64(diff
);
3647 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
/ 8);
3648 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
3649 t3
= tcg_const_i32(desc
);
3651 ptr
= tcg_temp_new_ptr();
3652 tcg_gen_addi_ptr(ptr
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3654 gen_helper_sve_whilel(t2
, ptr
, t2
, t3
);
3657 tcg_temp_free_ptr(ptr
);
3658 tcg_temp_free_i32(t2
);
3659 tcg_temp_free_i32(t3
);
3664 *** SVE Integer Wide Immediate - Unpredicated Group
3667 static bool trans_FDUP(DisasContext
*s
, arg_FDUP
*a
)
3672 if (sve_access_check(s
)) {
3673 unsigned vsz
= vec_full_reg_size(s
);
3674 int dofs
= vec_full_reg_offset(s
, a
->rd
);
3677 /* Decode the VFP immediate. */
3678 imm
= vfp_expand_imm(a
->esz
, a
->imm
);
3679 tcg_gen_gvec_dup_imm(a
->esz
, dofs
, vsz
, vsz
, imm
);
3684 static bool trans_DUP_i(DisasContext
*s
, arg_DUP_i
*a
)
3686 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3689 if (sve_access_check(s
)) {
3690 unsigned vsz
= vec_full_reg_size(s
);
3691 int dofs
= vec_full_reg_offset(s
, a
->rd
);
3693 tcg_gen_gvec_dup_imm(a
->esz
, dofs
, vsz
, vsz
, a
->imm
);
3698 static bool trans_ADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3700 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3703 if (sve_access_check(s
)) {
3704 unsigned vsz
= vec_full_reg_size(s
);
3705 tcg_gen_gvec_addi(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3706 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
3711 static bool trans_SUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3714 return trans_ADD_zzi(s
, a
);
3717 static bool trans_SUBR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3719 static const TCGOpcode vecop_list
[] = { INDEX_op_sub_vec
, 0 };
3720 static const GVecGen2s op
[4] = {
3721 { .fni8
= tcg_gen_vec_sub8_i64
,
3722 .fniv
= tcg_gen_sub_vec
,
3723 .fno
= gen_helper_sve_subri_b
,
3724 .opt_opc
= vecop_list
,
3726 .scalar_first
= true },
3727 { .fni8
= tcg_gen_vec_sub16_i64
,
3728 .fniv
= tcg_gen_sub_vec
,
3729 .fno
= gen_helper_sve_subri_h
,
3730 .opt_opc
= vecop_list
,
3732 .scalar_first
= true },
3733 { .fni4
= tcg_gen_sub_i32
,
3734 .fniv
= tcg_gen_sub_vec
,
3735 .fno
= gen_helper_sve_subri_s
,
3736 .opt_opc
= vecop_list
,
3738 .scalar_first
= true },
3739 { .fni8
= tcg_gen_sub_i64
,
3740 .fniv
= tcg_gen_sub_vec
,
3741 .fno
= gen_helper_sve_subri_d
,
3742 .opt_opc
= vecop_list
,
3743 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
3745 .scalar_first
= true }
3748 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3751 if (sve_access_check(s
)) {
3752 unsigned vsz
= vec_full_reg_size(s
);
3753 TCGv_i64 c
= tcg_const_i64(a
->imm
);
3754 tcg_gen_gvec_2s(vec_full_reg_offset(s
, a
->rd
),
3755 vec_full_reg_offset(s
, a
->rn
),
3756 vsz
, vsz
, c
, &op
[a
->esz
]);
3757 tcg_temp_free_i64(c
);
3762 static bool trans_MUL_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3764 if (sve_access_check(s
)) {
3765 unsigned vsz
= vec_full_reg_size(s
);
3766 tcg_gen_gvec_muli(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3767 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
3772 static bool do_zzi_sat(DisasContext
*s
, arg_rri_esz
*a
, bool u
, bool d
)
3774 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3777 if (sve_access_check(s
)) {
3778 TCGv_i64 val
= tcg_const_i64(a
->imm
);
3779 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
, val
, u
, d
);
3780 tcg_temp_free_i64(val
);
3785 static bool trans_SQADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3787 return do_zzi_sat(s
, a
, false, false);
3790 static bool trans_UQADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3792 return do_zzi_sat(s
, a
, true, false);
3795 static bool trans_SQSUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3797 return do_zzi_sat(s
, a
, false, true);
3800 static bool trans_UQSUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3802 return do_zzi_sat(s
, a
, true, true);
3805 static bool do_zzi_ool(DisasContext
*s
, arg_rri_esz
*a
, gen_helper_gvec_2i
*fn
)
3807 if (sve_access_check(s
)) {
3808 unsigned vsz
= vec_full_reg_size(s
);
3809 TCGv_i64 c
= tcg_const_i64(a
->imm
);
3811 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s
, a
->rd
),
3812 vec_full_reg_offset(s
, a
->rn
),
3813 c
, vsz
, vsz
, 0, fn
);
3814 tcg_temp_free_i64(c
);
3819 #define DO_ZZI(NAME, name) \
3820 static bool trans_##NAME##_zzi(DisasContext *s, arg_rri_esz *a) \
3822 static gen_helper_gvec_2i * const fns[4] = { \
3823 gen_helper_sve_##name##i_b, gen_helper_sve_##name##i_h, \
3824 gen_helper_sve_##name##i_s, gen_helper_sve_##name##i_d, \
3826 return do_zzi_ool(s, a, fns[a->esz]); \
3836 static bool trans_DOT_zzzz(DisasContext
*s
, arg_DOT_zzzz
*a
)
3838 static gen_helper_gvec_4
* const fns
[2][2] = {
3839 { gen_helper_gvec_sdot_b
, gen_helper_gvec_sdot_h
},
3840 { gen_helper_gvec_udot_b
, gen_helper_gvec_udot_h
}
3843 if (sve_access_check(s
)) {
3844 gen_gvec_ool_zzzz(s
, fns
[a
->u
][a
->sz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, 0);
3850 * SVE Multiply - Indexed
3853 static bool do_zzxz_ool(DisasContext
*s
, arg_rrxr_esz
*a
,
3854 gen_helper_gvec_4
*fn
)
3859 if (sve_access_check(s
)) {
3860 gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->index
);
3865 #define DO_RRXR(NAME, FUNC) \
3866 static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
3867 { return do_zzxz_ool(s, a, FUNC); }
3869 DO_RRXR(trans_SDOT_zzxw_s
, gen_helper_gvec_sdot_idx_b
)
3870 DO_RRXR(trans_SDOT_zzxw_d
, gen_helper_gvec_sdot_idx_h
)
3871 DO_RRXR(trans_UDOT_zzxw_s
, gen_helper_gvec_udot_idx_b
)
3872 DO_RRXR(trans_UDOT_zzxw_d
, gen_helper_gvec_udot_idx_h
)
3874 static bool trans_SUDOT_zzxw_s(DisasContext
*s
, arg_rrxr_esz
*a
)
3876 if (!dc_isar_feature(aa64_sve_i8mm
, s
)) {
3879 return do_zzxz_ool(s
, a
, gen_helper_gvec_sudot_idx_b
);
3882 static bool trans_USDOT_zzxw_s(DisasContext
*s
, arg_rrxr_esz
*a
)
3884 if (!dc_isar_feature(aa64_sve_i8mm
, s
)) {
3887 return do_zzxz_ool(s
, a
, gen_helper_gvec_usdot_idx_b
);
3892 static bool do_sve2_zzz_data(DisasContext
*s
, int rd
, int rn
, int rm
, int data
,
3893 gen_helper_gvec_3
*fn
)
3895 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
3898 if (sve_access_check(s
)) {
3899 unsigned vsz
= vec_full_reg_size(s
);
3900 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
3901 vec_full_reg_offset(s
, rn
),
3902 vec_full_reg_offset(s
, rm
),
3903 vsz
, vsz
, data
, fn
);
3908 #define DO_SVE2_RRX(NAME, FUNC) \
3909 static bool NAME(DisasContext *s, arg_rrx_esz *a) \
3910 { return do_sve2_zzz_data(s, a->rd, a->rn, a->rm, a->index, FUNC); }
3912 DO_SVE2_RRX(trans_MUL_zzx_h
, gen_helper_gvec_mul_idx_h
)
3913 DO_SVE2_RRX(trans_MUL_zzx_s
, gen_helper_gvec_mul_idx_s
)
3914 DO_SVE2_RRX(trans_MUL_zzx_d
, gen_helper_gvec_mul_idx_d
)
3916 DO_SVE2_RRX(trans_SQDMULH_zzx_h
, gen_helper_sve2_sqdmulh_idx_h
)
3917 DO_SVE2_RRX(trans_SQDMULH_zzx_s
, gen_helper_sve2_sqdmulh_idx_s
)
3918 DO_SVE2_RRX(trans_SQDMULH_zzx_d
, gen_helper_sve2_sqdmulh_idx_d
)
3920 DO_SVE2_RRX(trans_SQRDMULH_zzx_h
, gen_helper_sve2_sqrdmulh_idx_h
)
3921 DO_SVE2_RRX(trans_SQRDMULH_zzx_s
, gen_helper_sve2_sqrdmulh_idx_s
)
3922 DO_SVE2_RRX(trans_SQRDMULH_zzx_d
, gen_helper_sve2_sqrdmulh_idx_d
)
3926 #define DO_SVE2_RRX_TB(NAME, FUNC, TOP) \
3927 static bool NAME(DisasContext *s, arg_rrx_esz *a) \
3929 return do_sve2_zzz_data(s, a->rd, a->rn, a->rm, \
3930 (a->index << 1) | TOP, FUNC); \
3933 DO_SVE2_RRX_TB(trans_SQDMULLB_zzx_s
, gen_helper_sve2_sqdmull_idx_s
, false)
3934 DO_SVE2_RRX_TB(trans_SQDMULLB_zzx_d
, gen_helper_sve2_sqdmull_idx_d
, false)
3935 DO_SVE2_RRX_TB(trans_SQDMULLT_zzx_s
, gen_helper_sve2_sqdmull_idx_s
, true)
3936 DO_SVE2_RRX_TB(trans_SQDMULLT_zzx_d
, gen_helper_sve2_sqdmull_idx_d
, true)
3938 DO_SVE2_RRX_TB(trans_SMULLB_zzx_s
, gen_helper_sve2_smull_idx_s
, false)
3939 DO_SVE2_RRX_TB(trans_SMULLB_zzx_d
, gen_helper_sve2_smull_idx_d
, false)
3940 DO_SVE2_RRX_TB(trans_SMULLT_zzx_s
, gen_helper_sve2_smull_idx_s
, true)
3941 DO_SVE2_RRX_TB(trans_SMULLT_zzx_d
, gen_helper_sve2_smull_idx_d
, true)
3943 DO_SVE2_RRX_TB(trans_UMULLB_zzx_s
, gen_helper_sve2_umull_idx_s
, false)
3944 DO_SVE2_RRX_TB(trans_UMULLB_zzx_d
, gen_helper_sve2_umull_idx_d
, false)
3945 DO_SVE2_RRX_TB(trans_UMULLT_zzx_s
, gen_helper_sve2_umull_idx_s
, true)
3946 DO_SVE2_RRX_TB(trans_UMULLT_zzx_d
, gen_helper_sve2_umull_idx_d
, true)
3948 #undef DO_SVE2_RRX_TB
3950 static bool do_sve2_zzzz_data(DisasContext
*s
, int rd
, int rn
, int rm
, int ra
,
3951 int data
, gen_helper_gvec_4
*fn
)
3953 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
3956 if (sve_access_check(s
)) {
3957 unsigned vsz
= vec_full_reg_size(s
);
3958 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
3959 vec_full_reg_offset(s
, rn
),
3960 vec_full_reg_offset(s
, rm
),
3961 vec_full_reg_offset(s
, ra
),
3962 vsz
, vsz
, data
, fn
);
3967 #define DO_SVE2_RRXR(NAME, FUNC) \
3968 static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
3969 { return do_sve2_zzzz_data(s, a->rd, a->rn, a->rm, a->ra, a->index, FUNC); }
3971 DO_SVE2_RRXR(trans_MLA_zzxz_h
, gen_helper_gvec_mla_idx_h
)
3972 DO_SVE2_RRXR(trans_MLA_zzxz_s
, gen_helper_gvec_mla_idx_s
)
3973 DO_SVE2_RRXR(trans_MLA_zzxz_d
, gen_helper_gvec_mla_idx_d
)
3975 DO_SVE2_RRXR(trans_MLS_zzxz_h
, gen_helper_gvec_mls_idx_h
)
3976 DO_SVE2_RRXR(trans_MLS_zzxz_s
, gen_helper_gvec_mls_idx_s
)
3977 DO_SVE2_RRXR(trans_MLS_zzxz_d
, gen_helper_gvec_mls_idx_d
)
3979 DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_h
, gen_helper_sve2_sqrdmlah_idx_h
)
3980 DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_s
, gen_helper_sve2_sqrdmlah_idx_s
)
3981 DO_SVE2_RRXR(trans_SQRDMLAH_zzxz_d
, gen_helper_sve2_sqrdmlah_idx_d
)
3983 DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_h
, gen_helper_sve2_sqrdmlsh_idx_h
)
3984 DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_s
, gen_helper_sve2_sqrdmlsh_idx_s
)
3985 DO_SVE2_RRXR(trans_SQRDMLSH_zzxz_d
, gen_helper_sve2_sqrdmlsh_idx_d
)
3989 #define DO_SVE2_RRXR_TB(NAME, FUNC, TOP) \
3990 static bool NAME(DisasContext *s, arg_rrxr_esz *a) \
3992 return do_sve2_zzzz_data(s, a->rd, a->rn, a->rm, a->rd, \
3993 (a->index << 1) | TOP, FUNC); \
3996 DO_SVE2_RRXR_TB(trans_SQDMLALB_zzxw_s
, gen_helper_sve2_sqdmlal_idx_s
, false)
3997 DO_SVE2_RRXR_TB(trans_SQDMLALB_zzxw_d
, gen_helper_sve2_sqdmlal_idx_d
, false)
3998 DO_SVE2_RRXR_TB(trans_SQDMLALT_zzxw_s
, gen_helper_sve2_sqdmlal_idx_s
, true)
3999 DO_SVE2_RRXR_TB(trans_SQDMLALT_zzxw_d
, gen_helper_sve2_sqdmlal_idx_d
, true)
4001 DO_SVE2_RRXR_TB(trans_SQDMLSLB_zzxw_s
, gen_helper_sve2_sqdmlsl_idx_s
, false)
4002 DO_SVE2_RRXR_TB(trans_SQDMLSLB_zzxw_d
, gen_helper_sve2_sqdmlsl_idx_d
, false)
4003 DO_SVE2_RRXR_TB(trans_SQDMLSLT_zzxw_s
, gen_helper_sve2_sqdmlsl_idx_s
, true)
4004 DO_SVE2_RRXR_TB(trans_SQDMLSLT_zzxw_d
, gen_helper_sve2_sqdmlsl_idx_d
, true)
4006 DO_SVE2_RRXR_TB(trans_SMLALB_zzxw_s
, gen_helper_sve2_smlal_idx_s
, false)
4007 DO_SVE2_RRXR_TB(trans_SMLALB_zzxw_d
, gen_helper_sve2_smlal_idx_d
, false)
4008 DO_SVE2_RRXR_TB(trans_SMLALT_zzxw_s
, gen_helper_sve2_smlal_idx_s
, true)
4009 DO_SVE2_RRXR_TB(trans_SMLALT_zzxw_d
, gen_helper_sve2_smlal_idx_d
, true)
4011 DO_SVE2_RRXR_TB(trans_UMLALB_zzxw_s
, gen_helper_sve2_umlal_idx_s
, false)
4012 DO_SVE2_RRXR_TB(trans_UMLALB_zzxw_d
, gen_helper_sve2_umlal_idx_d
, false)
4013 DO_SVE2_RRXR_TB(trans_UMLALT_zzxw_s
, gen_helper_sve2_umlal_idx_s
, true)
4014 DO_SVE2_RRXR_TB(trans_UMLALT_zzxw_d
, gen_helper_sve2_umlal_idx_d
, true)
4016 DO_SVE2_RRXR_TB(trans_SMLSLB_zzxw_s
, gen_helper_sve2_smlsl_idx_s
, false)
4017 DO_SVE2_RRXR_TB(trans_SMLSLB_zzxw_d
, gen_helper_sve2_smlsl_idx_d
, false)
4018 DO_SVE2_RRXR_TB(trans_SMLSLT_zzxw_s
, gen_helper_sve2_smlsl_idx_s
, true)
4019 DO_SVE2_RRXR_TB(trans_SMLSLT_zzxw_d
, gen_helper_sve2_smlsl_idx_d
, true)
4021 DO_SVE2_RRXR_TB(trans_UMLSLB_zzxw_s
, gen_helper_sve2_umlsl_idx_s
, false)
4022 DO_SVE2_RRXR_TB(trans_UMLSLB_zzxw_d
, gen_helper_sve2_umlsl_idx_d
, false)
4023 DO_SVE2_RRXR_TB(trans_UMLSLT_zzxw_s
, gen_helper_sve2_umlsl_idx_s
, true)
4024 DO_SVE2_RRXR_TB(trans_UMLSLT_zzxw_d
, gen_helper_sve2_umlsl_idx_d
, true)
4026 #undef DO_SVE2_RRXR_TB
4028 #define DO_SVE2_RRXR_ROT(NAME, FUNC) \
4029 static bool trans_##NAME(DisasContext *s, arg_##NAME *a) \
4031 return do_sve2_zzzz_data(s, a->rd, a->rn, a->rm, a->ra, \
4032 (a->index << 2) | a->rot, FUNC); \
4035 DO_SVE2_RRXR_ROT(CMLA_zzxz_h
, gen_helper_sve2_cmla_idx_h
)
4036 DO_SVE2_RRXR_ROT(CMLA_zzxz_s
, gen_helper_sve2_cmla_idx_s
)
4038 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_h
, gen_helper_sve2_sqrdcmlah_idx_h
)
4039 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_s
, gen_helper_sve2_sqrdcmlah_idx_s
)
4041 DO_SVE2_RRXR_ROT(CDOT_zzxw_s
, gen_helper_sve2_cdot_idx_s
)
4042 DO_SVE2_RRXR_ROT(CDOT_zzxw_d
, gen_helper_sve2_cdot_idx_d
)
4044 #undef DO_SVE2_RRXR_ROT
4047 *** SVE Floating Point Multiply-Add Indexed Group
4050 static bool do_FMLA_zzxz(DisasContext
*s
, arg_rrxr_esz
*a
, bool sub
)
4052 static gen_helper_gvec_4_ptr
* const fns
[3] = {
4053 gen_helper_gvec_fmla_idx_h
,
4054 gen_helper_gvec_fmla_idx_s
,
4055 gen_helper_gvec_fmla_idx_d
,
4058 if (sve_access_check(s
)) {
4059 unsigned vsz
= vec_full_reg_size(s
);
4060 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4061 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4062 vec_full_reg_offset(s
, a
->rn
),
4063 vec_full_reg_offset(s
, a
->rm
),
4064 vec_full_reg_offset(s
, a
->ra
),
4065 status
, vsz
, vsz
, (a
->index
<< 1) | sub
,
4067 tcg_temp_free_ptr(status
);
4072 static bool trans_FMLA_zzxz(DisasContext
*s
, arg_FMLA_zzxz
*a
)
4074 return do_FMLA_zzxz(s
, a
, false);
4077 static bool trans_FMLS_zzxz(DisasContext
*s
, arg_FMLA_zzxz
*a
)
4079 return do_FMLA_zzxz(s
, a
, true);
4083 *** SVE Floating Point Multiply Indexed Group
4086 static bool trans_FMUL_zzx(DisasContext
*s
, arg_FMUL_zzx
*a
)
4088 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4089 gen_helper_gvec_fmul_idx_h
,
4090 gen_helper_gvec_fmul_idx_s
,
4091 gen_helper_gvec_fmul_idx_d
,
4094 if (sve_access_check(s
)) {
4095 unsigned vsz
= vec_full_reg_size(s
);
4096 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4097 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4098 vec_full_reg_offset(s
, a
->rn
),
4099 vec_full_reg_offset(s
, a
->rm
),
4100 status
, vsz
, vsz
, a
->index
, fns
[a
->esz
- 1]);
4101 tcg_temp_free_ptr(status
);
4107 *** SVE Floating Point Fast Reduction Group
4110 typedef void gen_helper_fp_reduce(TCGv_i64
, TCGv_ptr
, TCGv_ptr
,
4111 TCGv_ptr
, TCGv_i32
);
4113 static void do_reduce(DisasContext
*s
, arg_rpr_esz
*a
,
4114 gen_helper_fp_reduce
*fn
)
4116 unsigned vsz
= vec_full_reg_size(s
);
4117 unsigned p2vsz
= pow2ceil(vsz
);
4118 TCGv_i32 t_desc
= tcg_const_i32(simd_desc(vsz
, vsz
, p2vsz
));
4119 TCGv_ptr t_zn
, t_pg
, status
;
4122 temp
= tcg_temp_new_i64();
4123 t_zn
= tcg_temp_new_ptr();
4124 t_pg
= tcg_temp_new_ptr();
4126 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
4127 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
4128 status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4130 fn(temp
, t_zn
, t_pg
, status
, t_desc
);
4131 tcg_temp_free_ptr(t_zn
);
4132 tcg_temp_free_ptr(t_pg
);
4133 tcg_temp_free_ptr(status
);
4134 tcg_temp_free_i32(t_desc
);
4136 write_fp_dreg(s
, a
->rd
, temp
);
4137 tcg_temp_free_i64(temp
);
4140 #define DO_VPZ(NAME, name) \
4141 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
4143 static gen_helper_fp_reduce * const fns[3] = { \
4144 gen_helper_sve_##name##_h, \
4145 gen_helper_sve_##name##_s, \
4146 gen_helper_sve_##name##_d, \
4148 if (a->esz == 0) { \
4151 if (sve_access_check(s)) { \
4152 do_reduce(s, a, fns[a->esz - 1]); \
4157 DO_VPZ(FADDV
, faddv
)
4158 DO_VPZ(FMINNMV
, fminnmv
)
4159 DO_VPZ(FMAXNMV
, fmaxnmv
)
4160 DO_VPZ(FMINV
, fminv
)
4161 DO_VPZ(FMAXV
, fmaxv
)
4164 *** SVE Floating Point Unary Operations - Unpredicated Group
4167 static void do_zz_fp(DisasContext
*s
, arg_rr_esz
*a
, gen_helper_gvec_2_ptr
*fn
)
4169 unsigned vsz
= vec_full_reg_size(s
);
4170 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4172 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s
, a
->rd
),
4173 vec_full_reg_offset(s
, a
->rn
),
4174 status
, vsz
, vsz
, 0, fn
);
4175 tcg_temp_free_ptr(status
);
4178 static bool trans_FRECPE(DisasContext
*s
, arg_rr_esz
*a
)
4180 static gen_helper_gvec_2_ptr
* const fns
[3] = {
4181 gen_helper_gvec_frecpe_h
,
4182 gen_helper_gvec_frecpe_s
,
4183 gen_helper_gvec_frecpe_d
,
4188 if (sve_access_check(s
)) {
4189 do_zz_fp(s
, a
, fns
[a
->esz
- 1]);
4194 static bool trans_FRSQRTE(DisasContext
*s
, arg_rr_esz
*a
)
4196 static gen_helper_gvec_2_ptr
* const fns
[3] = {
4197 gen_helper_gvec_frsqrte_h
,
4198 gen_helper_gvec_frsqrte_s
,
4199 gen_helper_gvec_frsqrte_d
,
4204 if (sve_access_check(s
)) {
4205 do_zz_fp(s
, a
, fns
[a
->esz
- 1]);
4211 *** SVE Floating Point Compare with Zero Group
4214 static void do_ppz_fp(DisasContext
*s
, arg_rpr_esz
*a
,
4215 gen_helper_gvec_3_ptr
*fn
)
4217 unsigned vsz
= vec_full_reg_size(s
);
4218 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4220 tcg_gen_gvec_3_ptr(pred_full_reg_offset(s
, a
->rd
),
4221 vec_full_reg_offset(s
, a
->rn
),
4222 pred_full_reg_offset(s
, a
->pg
),
4223 status
, vsz
, vsz
, 0, fn
);
4224 tcg_temp_free_ptr(status
);
4227 #define DO_PPZ(NAME, name) \
4228 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
4230 static gen_helper_gvec_3_ptr * const fns[3] = { \
4231 gen_helper_sve_##name##_h, \
4232 gen_helper_sve_##name##_s, \
4233 gen_helper_sve_##name##_d, \
4235 if (a->esz == 0) { \
4238 if (sve_access_check(s)) { \
4239 do_ppz_fp(s, a, fns[a->esz - 1]); \
4244 DO_PPZ(FCMGE_ppz0
, fcmge0
)
4245 DO_PPZ(FCMGT_ppz0
, fcmgt0
)
4246 DO_PPZ(FCMLE_ppz0
, fcmle0
)
4247 DO_PPZ(FCMLT_ppz0
, fcmlt0
)
4248 DO_PPZ(FCMEQ_ppz0
, fcmeq0
)
4249 DO_PPZ(FCMNE_ppz0
, fcmne0
)
4254 *** SVE floating-point trig multiply-add coefficient
4257 static bool trans_FTMAD(DisasContext
*s
, arg_FTMAD
*a
)
4259 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4260 gen_helper_sve_ftmad_h
,
4261 gen_helper_sve_ftmad_s
,
4262 gen_helper_sve_ftmad_d
,
4268 if (sve_access_check(s
)) {
4269 unsigned vsz
= vec_full_reg_size(s
);
4270 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4271 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4272 vec_full_reg_offset(s
, a
->rn
),
4273 vec_full_reg_offset(s
, a
->rm
),
4274 status
, vsz
, vsz
, a
->imm
, fns
[a
->esz
- 1]);
4275 tcg_temp_free_ptr(status
);
4281 *** SVE Floating Point Accumulating Reduction Group
4284 static bool trans_FADDA(DisasContext
*s
, arg_rprr_esz
*a
)
4286 typedef void fadda_fn(TCGv_i64
, TCGv_i64
, TCGv_ptr
,
4287 TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
4288 static fadda_fn
* const fns
[3] = {
4289 gen_helper_sve_fadda_h
,
4290 gen_helper_sve_fadda_s
,
4291 gen_helper_sve_fadda_d
,
4293 unsigned vsz
= vec_full_reg_size(s
);
4294 TCGv_ptr t_rm
, t_pg
, t_fpst
;
4301 if (!sve_access_check(s
)) {
4305 t_val
= load_esz(cpu_env
, vec_reg_offset(s
, a
->rn
, 0, a
->esz
), a
->esz
);
4306 t_rm
= tcg_temp_new_ptr();
4307 t_pg
= tcg_temp_new_ptr();
4308 tcg_gen_addi_ptr(t_rm
, cpu_env
, vec_full_reg_offset(s
, a
->rm
));
4309 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
4310 t_fpst
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4311 t_desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
4313 fns
[a
->esz
- 1](t_val
, t_val
, t_rm
, t_pg
, t_fpst
, t_desc
);
4315 tcg_temp_free_i32(t_desc
);
4316 tcg_temp_free_ptr(t_fpst
);
4317 tcg_temp_free_ptr(t_pg
);
4318 tcg_temp_free_ptr(t_rm
);
4320 write_fp_dreg(s
, a
->rd
, t_val
);
4321 tcg_temp_free_i64(t_val
);
4326 *** SVE Floating Point Arithmetic - Unpredicated Group
4329 static bool do_zzz_fp(DisasContext
*s
, arg_rrr_esz
*a
,
4330 gen_helper_gvec_3_ptr
*fn
)
4335 if (sve_access_check(s
)) {
4336 unsigned vsz
= vec_full_reg_size(s
);
4337 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4338 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4339 vec_full_reg_offset(s
, a
->rn
),
4340 vec_full_reg_offset(s
, a
->rm
),
4341 status
, vsz
, vsz
, 0, fn
);
4342 tcg_temp_free_ptr(status
);
4348 #define DO_FP3(NAME, name) \
4349 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
4351 static gen_helper_gvec_3_ptr * const fns[4] = { \
4352 NULL, gen_helper_gvec_##name##_h, \
4353 gen_helper_gvec_##name##_s, gen_helper_gvec_##name##_d \
4355 return do_zzz_fp(s, a, fns[a->esz]); \
4358 DO_FP3(FADD_zzz
, fadd
)
4359 DO_FP3(FSUB_zzz
, fsub
)
4360 DO_FP3(FMUL_zzz
, fmul
)
4361 DO_FP3(FTSMUL
, ftsmul
)
4362 DO_FP3(FRECPS
, recps
)
4363 DO_FP3(FRSQRTS
, rsqrts
)
4368 *** SVE Floating Point Arithmetic - Predicated Group
4371 static bool do_zpzz_fp(DisasContext
*s
, arg_rprr_esz
*a
,
4372 gen_helper_gvec_4_ptr
*fn
)
4377 if (sve_access_check(s
)) {
4378 unsigned vsz
= vec_full_reg_size(s
);
4379 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4380 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4381 vec_full_reg_offset(s
, a
->rn
),
4382 vec_full_reg_offset(s
, a
->rm
),
4383 pred_full_reg_offset(s
, a
->pg
),
4384 status
, vsz
, vsz
, 0, fn
);
4385 tcg_temp_free_ptr(status
);
4390 #define DO_FP3(NAME, name) \
4391 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
4393 static gen_helper_gvec_4_ptr * const fns[4] = { \
4394 NULL, gen_helper_sve_##name##_h, \
4395 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4397 return do_zpzz_fp(s, a, fns[a->esz]); \
4400 DO_FP3(FADD_zpzz
, fadd
)
4401 DO_FP3(FSUB_zpzz
, fsub
)
4402 DO_FP3(FMUL_zpzz
, fmul
)
4403 DO_FP3(FMIN_zpzz
, fmin
)
4404 DO_FP3(FMAX_zpzz
, fmax
)
4405 DO_FP3(FMINNM_zpzz
, fminnum
)
4406 DO_FP3(FMAXNM_zpzz
, fmaxnum
)
4408 DO_FP3(FSCALE
, fscalbn
)
4410 DO_FP3(FMULX
, fmulx
)
4414 typedef void gen_helper_sve_fp2scalar(TCGv_ptr
, TCGv_ptr
, TCGv_ptr
,
4415 TCGv_i64
, TCGv_ptr
, TCGv_i32
);
4417 static void do_fp_scalar(DisasContext
*s
, int zd
, int zn
, int pg
, bool is_fp16
,
4418 TCGv_i64 scalar
, gen_helper_sve_fp2scalar
*fn
)
4420 unsigned vsz
= vec_full_reg_size(s
);
4421 TCGv_ptr t_zd
, t_zn
, t_pg
, status
;
4424 t_zd
= tcg_temp_new_ptr();
4425 t_zn
= tcg_temp_new_ptr();
4426 t_pg
= tcg_temp_new_ptr();
4427 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, zd
));
4428 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, zn
));
4429 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
4431 status
= fpstatus_ptr(is_fp16
? FPST_FPCR_F16
: FPST_FPCR
);
4432 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
4433 fn(t_zd
, t_zn
, t_pg
, scalar
, status
, desc
);
4435 tcg_temp_free_i32(desc
);
4436 tcg_temp_free_ptr(status
);
4437 tcg_temp_free_ptr(t_pg
);
4438 tcg_temp_free_ptr(t_zn
);
4439 tcg_temp_free_ptr(t_zd
);
4442 static void do_fp_imm(DisasContext
*s
, arg_rpri_esz
*a
, uint64_t imm
,
4443 gen_helper_sve_fp2scalar
*fn
)
4445 TCGv_i64 temp
= tcg_const_i64(imm
);
4446 do_fp_scalar(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, temp
, fn
);
4447 tcg_temp_free_i64(temp
);
4450 #define DO_FP_IMM(NAME, name, const0, const1) \
4451 static bool trans_##NAME##_zpzi(DisasContext *s, arg_rpri_esz *a) \
4453 static gen_helper_sve_fp2scalar * const fns[3] = { \
4454 gen_helper_sve_##name##_h, \
4455 gen_helper_sve_##name##_s, \
4456 gen_helper_sve_##name##_d \
4458 static uint64_t const val[3][2] = { \
4459 { float16_##const0, float16_##const1 }, \
4460 { float32_##const0, float32_##const1 }, \
4461 { float64_##const0, float64_##const1 }, \
4463 if (a->esz == 0) { \
4466 if (sve_access_check(s)) { \
4467 do_fp_imm(s, a, val[a->esz - 1][a->imm], fns[a->esz - 1]); \
4472 DO_FP_IMM(FADD
, fadds
, half
, one
)
4473 DO_FP_IMM(FSUB
, fsubs
, half
, one
)
4474 DO_FP_IMM(FMUL
, fmuls
, half
, two
)
4475 DO_FP_IMM(FSUBR
, fsubrs
, half
, one
)
4476 DO_FP_IMM(FMAXNM
, fmaxnms
, zero
, one
)
4477 DO_FP_IMM(FMINNM
, fminnms
, zero
, one
)
4478 DO_FP_IMM(FMAX
, fmaxs
, zero
, one
)
4479 DO_FP_IMM(FMIN
, fmins
, zero
, one
)
4483 static bool do_fp_cmp(DisasContext
*s
, arg_rprr_esz
*a
,
4484 gen_helper_gvec_4_ptr
*fn
)
4489 if (sve_access_check(s
)) {
4490 unsigned vsz
= vec_full_reg_size(s
);
4491 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4492 tcg_gen_gvec_4_ptr(pred_full_reg_offset(s
, a
->rd
),
4493 vec_full_reg_offset(s
, a
->rn
),
4494 vec_full_reg_offset(s
, a
->rm
),
4495 pred_full_reg_offset(s
, a
->pg
),
4496 status
, vsz
, vsz
, 0, fn
);
4497 tcg_temp_free_ptr(status
);
4502 #define DO_FPCMP(NAME, name) \
4503 static bool trans_##NAME##_ppzz(DisasContext *s, arg_rprr_esz *a) \
4505 static gen_helper_gvec_4_ptr * const fns[4] = { \
4506 NULL, gen_helper_sve_##name##_h, \
4507 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4509 return do_fp_cmp(s, a, fns[a->esz]); \
4512 DO_FPCMP(FCMGE
, fcmge
)
4513 DO_FPCMP(FCMGT
, fcmgt
)
4514 DO_FPCMP(FCMEQ
, fcmeq
)
4515 DO_FPCMP(FCMNE
, fcmne
)
4516 DO_FPCMP(FCMUO
, fcmuo
)
4517 DO_FPCMP(FACGE
, facge
)
4518 DO_FPCMP(FACGT
, facgt
)
4522 static bool trans_FCADD(DisasContext
*s
, arg_FCADD
*a
)
4524 static gen_helper_gvec_4_ptr
* const fns
[3] = {
4525 gen_helper_sve_fcadd_h
,
4526 gen_helper_sve_fcadd_s
,
4527 gen_helper_sve_fcadd_d
4533 if (sve_access_check(s
)) {
4534 unsigned vsz
= vec_full_reg_size(s
);
4535 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4536 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4537 vec_full_reg_offset(s
, a
->rn
),
4538 vec_full_reg_offset(s
, a
->rm
),
4539 pred_full_reg_offset(s
, a
->pg
),
4540 status
, vsz
, vsz
, a
->rot
, fns
[a
->esz
- 1]);
4541 tcg_temp_free_ptr(status
);
4546 static bool do_fmla(DisasContext
*s
, arg_rprrr_esz
*a
,
4547 gen_helper_gvec_5_ptr
*fn
)
4552 if (sve_access_check(s
)) {
4553 unsigned vsz
= vec_full_reg_size(s
);
4554 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4555 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s
, a
->rd
),
4556 vec_full_reg_offset(s
, a
->rn
),
4557 vec_full_reg_offset(s
, a
->rm
),
4558 vec_full_reg_offset(s
, a
->ra
),
4559 pred_full_reg_offset(s
, a
->pg
),
4560 status
, vsz
, vsz
, 0, fn
);
4561 tcg_temp_free_ptr(status
);
4566 #define DO_FMLA(NAME, name) \
4567 static bool trans_##NAME(DisasContext *s, arg_rprrr_esz *a) \
4569 static gen_helper_gvec_5_ptr * const fns[4] = { \
4570 NULL, gen_helper_sve_##name##_h, \
4571 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
4573 return do_fmla(s, a, fns[a->esz]); \
4576 DO_FMLA(FMLA_zpzzz
, fmla_zpzzz
)
4577 DO_FMLA(FMLS_zpzzz
, fmls_zpzzz
)
4578 DO_FMLA(FNMLA_zpzzz
, fnmla_zpzzz
)
4579 DO_FMLA(FNMLS_zpzzz
, fnmls_zpzzz
)
4583 static bool trans_FCMLA_zpzzz(DisasContext
*s
, arg_FCMLA_zpzzz
*a
)
4585 static gen_helper_gvec_5_ptr
* const fns
[4] = {
4587 gen_helper_sve_fcmla_zpzzz_h
,
4588 gen_helper_sve_fcmla_zpzzz_s
,
4589 gen_helper_sve_fcmla_zpzzz_d
,
4595 if (sve_access_check(s
)) {
4596 unsigned vsz
= vec_full_reg_size(s
);
4597 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4598 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s
, a
->rd
),
4599 vec_full_reg_offset(s
, a
->rn
),
4600 vec_full_reg_offset(s
, a
->rm
),
4601 vec_full_reg_offset(s
, a
->ra
),
4602 pred_full_reg_offset(s
, a
->pg
),
4603 status
, vsz
, vsz
, a
->rot
, fns
[a
->esz
]);
4604 tcg_temp_free_ptr(status
);
4609 static bool trans_FCMLA_zzxz(DisasContext
*s
, arg_FCMLA_zzxz
*a
)
4611 static gen_helper_gvec_4_ptr
* const fns
[2] = {
4612 gen_helper_gvec_fcmlah_idx
,
4613 gen_helper_gvec_fcmlas_idx
,
4616 tcg_debug_assert(a
->esz
== 1 || a
->esz
== 2);
4617 tcg_debug_assert(a
->rd
== a
->ra
);
4618 if (sve_access_check(s
)) {
4619 unsigned vsz
= vec_full_reg_size(s
);
4620 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4621 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
4622 vec_full_reg_offset(s
, a
->rn
),
4623 vec_full_reg_offset(s
, a
->rm
),
4624 vec_full_reg_offset(s
, a
->ra
),
4626 a
->index
* 4 + a
->rot
,
4628 tcg_temp_free_ptr(status
);
4634 *** SVE Floating Point Unary Operations Predicated Group
4637 static bool do_zpz_ptr(DisasContext
*s
, int rd
, int rn
, int pg
,
4638 bool is_fp16
, gen_helper_gvec_3_ptr
*fn
)
4640 if (sve_access_check(s
)) {
4641 unsigned vsz
= vec_full_reg_size(s
);
4642 TCGv_ptr status
= fpstatus_ptr(is_fp16
? FPST_FPCR_F16
: FPST_FPCR
);
4643 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, rd
),
4644 vec_full_reg_offset(s
, rn
),
4645 pred_full_reg_offset(s
, pg
),
4646 status
, vsz
, vsz
, 0, fn
);
4647 tcg_temp_free_ptr(status
);
4652 static bool trans_FCVT_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4654 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_sh
);
4657 static bool trans_FCVT_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4659 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_hs
);
4662 static bool trans_FCVT_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4664 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_dh
);
4667 static bool trans_FCVT_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4669 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_hd
);
4672 static bool trans_FCVT_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4674 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_ds
);
4677 static bool trans_FCVT_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4679 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_sd
);
4682 static bool trans_FCVTZS_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4684 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hh
);
4687 static bool trans_FCVTZU_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4689 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hh
);
4692 static bool trans_FCVTZS_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4694 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hs
);
4697 static bool trans_FCVTZU_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4699 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hs
);
4702 static bool trans_FCVTZS_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4704 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hd
);
4707 static bool trans_FCVTZU_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4709 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hd
);
4712 static bool trans_FCVTZS_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4714 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_ss
);
4717 static bool trans_FCVTZU_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4719 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_ss
);
4722 static bool trans_FCVTZS_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4724 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_sd
);
4727 static bool trans_FCVTZU_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4729 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_sd
);
4732 static bool trans_FCVTZS_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4734 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_ds
);
4737 static bool trans_FCVTZU_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4739 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_ds
);
4742 static bool trans_FCVTZS_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4744 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_dd
);
4747 static bool trans_FCVTZU_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4749 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_dd
);
4752 static gen_helper_gvec_3_ptr
* const frint_fns
[3] = {
4753 gen_helper_sve_frint_h
,
4754 gen_helper_sve_frint_s
,
4755 gen_helper_sve_frint_d
4758 static bool trans_FRINTI(DisasContext
*s
, arg_rpr_esz
*a
)
4763 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
,
4764 frint_fns
[a
->esz
- 1]);
4767 static bool trans_FRINTX(DisasContext
*s
, arg_rpr_esz
*a
)
4769 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4770 gen_helper_sve_frintx_h
,
4771 gen_helper_sve_frintx_s
,
4772 gen_helper_sve_frintx_d
4777 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4780 static bool do_frint_mode(DisasContext
*s
, arg_rpr_esz
*a
, int mode
)
4785 if (sve_access_check(s
)) {
4786 unsigned vsz
= vec_full_reg_size(s
);
4787 TCGv_i32 tmode
= tcg_const_i32(mode
);
4788 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4790 gen_helper_set_rmode(tmode
, tmode
, status
);
4792 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4793 vec_full_reg_offset(s
, a
->rn
),
4794 pred_full_reg_offset(s
, a
->pg
),
4795 status
, vsz
, vsz
, 0, frint_fns
[a
->esz
- 1]);
4797 gen_helper_set_rmode(tmode
, tmode
, status
);
4798 tcg_temp_free_i32(tmode
);
4799 tcg_temp_free_ptr(status
);
4804 static bool trans_FRINTN(DisasContext
*s
, arg_rpr_esz
*a
)
4806 return do_frint_mode(s
, a
, float_round_nearest_even
);
4809 static bool trans_FRINTP(DisasContext
*s
, arg_rpr_esz
*a
)
4811 return do_frint_mode(s
, a
, float_round_up
);
4814 static bool trans_FRINTM(DisasContext
*s
, arg_rpr_esz
*a
)
4816 return do_frint_mode(s
, a
, float_round_down
);
4819 static bool trans_FRINTZ(DisasContext
*s
, arg_rpr_esz
*a
)
4821 return do_frint_mode(s
, a
, float_round_to_zero
);
4824 static bool trans_FRINTA(DisasContext
*s
, arg_rpr_esz
*a
)
4826 return do_frint_mode(s
, a
, float_round_ties_away
);
4829 static bool trans_FRECPX(DisasContext
*s
, arg_rpr_esz
*a
)
4831 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4832 gen_helper_sve_frecpx_h
,
4833 gen_helper_sve_frecpx_s
,
4834 gen_helper_sve_frecpx_d
4839 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4842 static bool trans_FSQRT(DisasContext
*s
, arg_rpr_esz
*a
)
4844 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4845 gen_helper_sve_fsqrt_h
,
4846 gen_helper_sve_fsqrt_s
,
4847 gen_helper_sve_fsqrt_d
4852 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4855 static bool trans_SCVTF_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4857 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_hh
);
4860 static bool trans_SCVTF_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4862 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_sh
);
4865 static bool trans_SCVTF_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4867 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_dh
);
4870 static bool trans_SCVTF_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4872 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_ss
);
4875 static bool trans_SCVTF_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4877 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_ds
);
4880 static bool trans_SCVTF_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4882 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_sd
);
4885 static bool trans_SCVTF_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4887 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_dd
);
4890 static bool trans_UCVTF_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4892 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_hh
);
4895 static bool trans_UCVTF_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4897 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_sh
);
4900 static bool trans_UCVTF_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4902 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_dh
);
4905 static bool trans_UCVTF_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4907 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_ss
);
4910 static bool trans_UCVTF_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4912 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_ds
);
4915 static bool trans_UCVTF_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4917 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_sd
);
4920 static bool trans_UCVTF_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4922 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_dd
);
4926 *** SVE Memory - 32-bit Gather and Unsized Contiguous Group
4929 /* Subroutine loading a vector register at VOFS of LEN bytes.
4930 * The load should begin at the address Rn + IMM.
4933 static void do_ldr(DisasContext
*s
, uint32_t vofs
, int len
, int rn
, int imm
)
4935 int len_align
= QEMU_ALIGN_DOWN(len
, 8);
4936 int len_remain
= len
% 8;
4937 int nparts
= len
/ 8 + ctpop8(len_remain
);
4938 int midx
= get_mem_index(s
);
4939 TCGv_i64 dirty_addr
, clean_addr
, t0
, t1
;
4941 dirty_addr
= tcg_temp_new_i64();
4942 tcg_gen_addi_i64(dirty_addr
, cpu_reg_sp(s
, rn
), imm
);
4943 clean_addr
= gen_mte_checkN(s
, dirty_addr
, false, rn
!= 31, len
);
4944 tcg_temp_free_i64(dirty_addr
);
4947 * Note that unpredicated load/store of vector/predicate registers
4948 * are defined as a stream of bytes, which equates to little-endian
4949 * operations on larger quantities.
4950 * Attempt to keep code expansion to a minimum by limiting the
4951 * amount of unrolling done.
4956 t0
= tcg_temp_new_i64();
4957 for (i
= 0; i
< len_align
; i
+= 8) {
4958 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEQ
);
4959 tcg_gen_st_i64(t0
, cpu_env
, vofs
+ i
);
4960 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4962 tcg_temp_free_i64(t0
);
4964 TCGLabel
*loop
= gen_new_label();
4965 TCGv_ptr tp
, i
= tcg_const_local_ptr(0);
4967 /* Copy the clean address into a local temp, live across the loop. */
4969 clean_addr
= new_tmp_a64_local(s
);
4970 tcg_gen_mov_i64(clean_addr
, t0
);
4972 gen_set_label(loop
);
4974 t0
= tcg_temp_new_i64();
4975 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEQ
);
4976 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4978 tp
= tcg_temp_new_ptr();
4979 tcg_gen_add_ptr(tp
, cpu_env
, i
);
4980 tcg_gen_addi_ptr(i
, i
, 8);
4981 tcg_gen_st_i64(t0
, tp
, vofs
);
4982 tcg_temp_free_ptr(tp
);
4983 tcg_temp_free_i64(t0
);
4985 tcg_gen_brcondi_ptr(TCG_COND_LTU
, i
, len_align
, loop
);
4986 tcg_temp_free_ptr(i
);
4990 * Predicate register loads can be any multiple of 2.
4991 * Note that we still store the entire 64-bit unit into cpu_env.
4994 t0
= tcg_temp_new_i64();
4995 switch (len_remain
) {
4999 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
,
5000 MO_LE
| ctz32(len_remain
));
5004 t1
= tcg_temp_new_i64();
5005 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEUL
);
5006 tcg_gen_addi_i64(clean_addr
, clean_addr
, 4);
5007 tcg_gen_qemu_ld_i64(t1
, clean_addr
, midx
, MO_LEUW
);
5008 tcg_gen_deposit_i64(t0
, t0
, t1
, 32, 32);
5009 tcg_temp_free_i64(t1
);
5013 g_assert_not_reached();
5015 tcg_gen_st_i64(t0
, cpu_env
, vofs
+ len_align
);
5016 tcg_temp_free_i64(t0
);
5020 /* Similarly for stores. */
5021 static void do_str(DisasContext
*s
, uint32_t vofs
, int len
, int rn
, int imm
)
5023 int len_align
= QEMU_ALIGN_DOWN(len
, 8);
5024 int len_remain
= len
% 8;
5025 int nparts
= len
/ 8 + ctpop8(len_remain
);
5026 int midx
= get_mem_index(s
);
5027 TCGv_i64 dirty_addr
, clean_addr
, t0
;
5029 dirty_addr
= tcg_temp_new_i64();
5030 tcg_gen_addi_i64(dirty_addr
, cpu_reg_sp(s
, rn
), imm
);
5031 clean_addr
= gen_mte_checkN(s
, dirty_addr
, false, rn
!= 31, len
);
5032 tcg_temp_free_i64(dirty_addr
);
5034 /* Note that unpredicated load/store of vector/predicate registers
5035 * are defined as a stream of bytes, which equates to little-endian
5036 * operations on larger quantities. There is no nice way to force
5037 * a little-endian store for aarch64_be-linux-user out of line.
5039 * Attempt to keep code expansion to a minimum by limiting the
5040 * amount of unrolling done.
5045 t0
= tcg_temp_new_i64();
5046 for (i
= 0; i
< len_align
; i
+= 8) {
5047 tcg_gen_ld_i64(t0
, cpu_env
, vofs
+ i
);
5048 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEQ
);
5049 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
5051 tcg_temp_free_i64(t0
);
5053 TCGLabel
*loop
= gen_new_label();
5054 TCGv_ptr tp
, i
= tcg_const_local_ptr(0);
5056 /* Copy the clean address into a local temp, live across the loop. */
5058 clean_addr
= new_tmp_a64_local(s
);
5059 tcg_gen_mov_i64(clean_addr
, t0
);
5061 gen_set_label(loop
);
5063 t0
= tcg_temp_new_i64();
5064 tp
= tcg_temp_new_ptr();
5065 tcg_gen_add_ptr(tp
, cpu_env
, i
);
5066 tcg_gen_ld_i64(t0
, tp
, vofs
);
5067 tcg_gen_addi_ptr(i
, i
, 8);
5068 tcg_temp_free_ptr(tp
);
5070 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEQ
);
5071 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
5072 tcg_temp_free_i64(t0
);
5074 tcg_gen_brcondi_ptr(TCG_COND_LTU
, i
, len_align
, loop
);
5075 tcg_temp_free_ptr(i
);
5078 /* Predicate register stores can be any multiple of 2. */
5080 t0
= tcg_temp_new_i64();
5081 tcg_gen_ld_i64(t0
, cpu_env
, vofs
+ len_align
);
5083 switch (len_remain
) {
5087 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
,
5088 MO_LE
| ctz32(len_remain
));
5092 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUL
);
5093 tcg_gen_addi_i64(clean_addr
, clean_addr
, 4);
5094 tcg_gen_shri_i64(t0
, t0
, 32);
5095 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUW
);
5099 g_assert_not_reached();
5101 tcg_temp_free_i64(t0
);
5105 static bool trans_LDR_zri(DisasContext
*s
, arg_rri
*a
)
5107 if (sve_access_check(s
)) {
5108 int size
= vec_full_reg_size(s
);
5109 int off
= vec_full_reg_offset(s
, a
->rd
);
5110 do_ldr(s
, off
, size
, a
->rn
, a
->imm
* size
);
5115 static bool trans_LDR_pri(DisasContext
*s
, arg_rri
*a
)
5117 if (sve_access_check(s
)) {
5118 int size
= pred_full_reg_size(s
);
5119 int off
= pred_full_reg_offset(s
, a
->rd
);
5120 do_ldr(s
, off
, size
, a
->rn
, a
->imm
* size
);
5125 static bool trans_STR_zri(DisasContext
*s
, arg_rri
*a
)
5127 if (sve_access_check(s
)) {
5128 int size
= vec_full_reg_size(s
);
5129 int off
= vec_full_reg_offset(s
, a
->rd
);
5130 do_str(s
, off
, size
, a
->rn
, a
->imm
* size
);
5135 static bool trans_STR_pri(DisasContext
*s
, arg_rri
*a
)
5137 if (sve_access_check(s
)) {
5138 int size
= pred_full_reg_size(s
);
5139 int off
= pred_full_reg_offset(s
, a
->rd
);
5140 do_str(s
, off
, size
, a
->rn
, a
->imm
* size
);
5146 *** SVE Memory - Contiguous Load Group
5149 /* The memory mode of the dtype. */
5150 static const MemOp dtype_mop
[16] = {
5151 MO_UB
, MO_UB
, MO_UB
, MO_UB
,
5152 MO_SL
, MO_UW
, MO_UW
, MO_UW
,
5153 MO_SW
, MO_SW
, MO_UL
, MO_UL
,
5154 MO_SB
, MO_SB
, MO_SB
, MO_Q
5157 #define dtype_msz(x) (dtype_mop[x] & MO_SIZE)
5159 /* The vector element size of dtype. */
5160 static const uint8_t dtype_esz
[16] = {
5167 static void do_mem_zpa(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
,
5168 int dtype
, uint32_t mte_n
, bool is_write
,
5169 gen_helper_gvec_mem
*fn
)
5171 unsigned vsz
= vec_full_reg_size(s
);
5177 * For e.g. LD4, there are not enough arguments to pass all 4
5178 * registers as pointers, so encode the regno into the data field.
5179 * For consistency, do this even for LD1.
5181 if (s
->mte_active
[0]) {
5182 int msz
= dtype_msz(dtype
);
5184 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, get_mem_index(s
));
5185 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
5186 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
5187 desc
= FIELD_DP32(desc
, MTEDESC
, WRITE
, is_write
);
5188 desc
= FIELD_DP32(desc
, MTEDESC
, SIZEM1
, (mte_n
<< msz
) - 1);
5189 desc
<<= SVE_MTEDESC_SHIFT
;
5191 addr
= clean_data_tbi(s
, addr
);
5194 desc
= simd_desc(vsz
, vsz
, zt
| desc
);
5195 t_desc
= tcg_const_i32(desc
);
5196 t_pg
= tcg_temp_new_ptr();
5198 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
5199 fn(cpu_env
, t_pg
, addr
, t_desc
);
5201 tcg_temp_free_ptr(t_pg
);
5202 tcg_temp_free_i32(t_desc
);
5205 static void do_ld_zpa(DisasContext
*s
, int zt
, int pg
,
5206 TCGv_i64 addr
, int dtype
, int nreg
)
5208 static gen_helper_gvec_mem
* const fns
[2][2][16][4] = {
5209 { /* mte inactive, little-endian */
5210 { { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld2bb_r
,
5211 gen_helper_sve_ld3bb_r
, gen_helper_sve_ld4bb_r
},
5212 { gen_helper_sve_ld1bhu_r
, NULL
, NULL
, NULL
},
5213 { gen_helper_sve_ld1bsu_r
, NULL
, NULL
, NULL
},
5214 { gen_helper_sve_ld1bdu_r
, NULL
, NULL
, NULL
},
5216 { gen_helper_sve_ld1sds_le_r
, NULL
, NULL
, NULL
},
5217 { gen_helper_sve_ld1hh_le_r
, gen_helper_sve_ld2hh_le_r
,
5218 gen_helper_sve_ld3hh_le_r
, gen_helper_sve_ld4hh_le_r
},
5219 { gen_helper_sve_ld1hsu_le_r
, NULL
, NULL
, NULL
},
5220 { gen_helper_sve_ld1hdu_le_r
, NULL
, NULL
, NULL
},
5222 { gen_helper_sve_ld1hds_le_r
, NULL
, NULL
, NULL
},
5223 { gen_helper_sve_ld1hss_le_r
, NULL
, NULL
, NULL
},
5224 { gen_helper_sve_ld1ss_le_r
, gen_helper_sve_ld2ss_le_r
,
5225 gen_helper_sve_ld3ss_le_r
, gen_helper_sve_ld4ss_le_r
},
5226 { gen_helper_sve_ld1sdu_le_r
, NULL
, NULL
, NULL
},
5228 { gen_helper_sve_ld1bds_r
, NULL
, NULL
, NULL
},
5229 { gen_helper_sve_ld1bss_r
, NULL
, NULL
, NULL
},
5230 { gen_helper_sve_ld1bhs_r
, NULL
, NULL
, NULL
},
5231 { gen_helper_sve_ld1dd_le_r
, gen_helper_sve_ld2dd_le_r
,
5232 gen_helper_sve_ld3dd_le_r
, gen_helper_sve_ld4dd_le_r
} },
5234 /* mte inactive, big-endian */
5235 { { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld2bb_r
,
5236 gen_helper_sve_ld3bb_r
, gen_helper_sve_ld4bb_r
},
5237 { gen_helper_sve_ld1bhu_r
, NULL
, NULL
, NULL
},
5238 { gen_helper_sve_ld1bsu_r
, NULL
, NULL
, NULL
},
5239 { gen_helper_sve_ld1bdu_r
, NULL
, NULL
, NULL
},
5241 { gen_helper_sve_ld1sds_be_r
, NULL
, NULL
, NULL
},
5242 { gen_helper_sve_ld1hh_be_r
, gen_helper_sve_ld2hh_be_r
,
5243 gen_helper_sve_ld3hh_be_r
, gen_helper_sve_ld4hh_be_r
},
5244 { gen_helper_sve_ld1hsu_be_r
, NULL
, NULL
, NULL
},
5245 { gen_helper_sve_ld1hdu_be_r
, NULL
, NULL
, NULL
},
5247 { gen_helper_sve_ld1hds_be_r
, NULL
, NULL
, NULL
},
5248 { gen_helper_sve_ld1hss_be_r
, NULL
, NULL
, NULL
},
5249 { gen_helper_sve_ld1ss_be_r
, gen_helper_sve_ld2ss_be_r
,
5250 gen_helper_sve_ld3ss_be_r
, gen_helper_sve_ld4ss_be_r
},
5251 { gen_helper_sve_ld1sdu_be_r
, NULL
, NULL
, NULL
},
5253 { gen_helper_sve_ld1bds_r
, NULL
, NULL
, NULL
},
5254 { gen_helper_sve_ld1bss_r
, NULL
, NULL
, NULL
},
5255 { gen_helper_sve_ld1bhs_r
, NULL
, NULL
, NULL
},
5256 { gen_helper_sve_ld1dd_be_r
, gen_helper_sve_ld2dd_be_r
,
5257 gen_helper_sve_ld3dd_be_r
, gen_helper_sve_ld4dd_be_r
} } },
5259 { /* mte active, little-endian */
5260 { { gen_helper_sve_ld1bb_r_mte
,
5261 gen_helper_sve_ld2bb_r_mte
,
5262 gen_helper_sve_ld3bb_r_mte
,
5263 gen_helper_sve_ld4bb_r_mte
},
5264 { gen_helper_sve_ld1bhu_r_mte
, NULL
, NULL
, NULL
},
5265 { gen_helper_sve_ld1bsu_r_mte
, NULL
, NULL
, NULL
},
5266 { gen_helper_sve_ld1bdu_r_mte
, NULL
, NULL
, NULL
},
5268 { gen_helper_sve_ld1sds_le_r_mte
, NULL
, NULL
, NULL
},
5269 { gen_helper_sve_ld1hh_le_r_mte
,
5270 gen_helper_sve_ld2hh_le_r_mte
,
5271 gen_helper_sve_ld3hh_le_r_mte
,
5272 gen_helper_sve_ld4hh_le_r_mte
},
5273 { gen_helper_sve_ld1hsu_le_r_mte
, NULL
, NULL
, NULL
},
5274 { gen_helper_sve_ld1hdu_le_r_mte
, NULL
, NULL
, NULL
},
5276 { gen_helper_sve_ld1hds_le_r_mte
, NULL
, NULL
, NULL
},
5277 { gen_helper_sve_ld1hss_le_r_mte
, NULL
, NULL
, NULL
},
5278 { gen_helper_sve_ld1ss_le_r_mte
,
5279 gen_helper_sve_ld2ss_le_r_mte
,
5280 gen_helper_sve_ld3ss_le_r_mte
,
5281 gen_helper_sve_ld4ss_le_r_mte
},
5282 { gen_helper_sve_ld1sdu_le_r_mte
, NULL
, NULL
, NULL
},
5284 { gen_helper_sve_ld1bds_r_mte
, NULL
, NULL
, NULL
},
5285 { gen_helper_sve_ld1bss_r_mte
, NULL
, NULL
, NULL
},
5286 { gen_helper_sve_ld1bhs_r_mte
, NULL
, NULL
, NULL
},
5287 { gen_helper_sve_ld1dd_le_r_mte
,
5288 gen_helper_sve_ld2dd_le_r_mte
,
5289 gen_helper_sve_ld3dd_le_r_mte
,
5290 gen_helper_sve_ld4dd_le_r_mte
} },
5292 /* mte active, big-endian */
5293 { { gen_helper_sve_ld1bb_r_mte
,
5294 gen_helper_sve_ld2bb_r_mte
,
5295 gen_helper_sve_ld3bb_r_mte
,
5296 gen_helper_sve_ld4bb_r_mte
},
5297 { gen_helper_sve_ld1bhu_r_mte
, NULL
, NULL
, NULL
},
5298 { gen_helper_sve_ld1bsu_r_mte
, NULL
, NULL
, NULL
},
5299 { gen_helper_sve_ld1bdu_r_mte
, NULL
, NULL
, NULL
},
5301 { gen_helper_sve_ld1sds_be_r_mte
, NULL
, NULL
, NULL
},
5302 { gen_helper_sve_ld1hh_be_r_mte
,
5303 gen_helper_sve_ld2hh_be_r_mte
,
5304 gen_helper_sve_ld3hh_be_r_mte
,
5305 gen_helper_sve_ld4hh_be_r_mte
},
5306 { gen_helper_sve_ld1hsu_be_r_mte
, NULL
, NULL
, NULL
},
5307 { gen_helper_sve_ld1hdu_be_r_mte
, NULL
, NULL
, NULL
},
5309 { gen_helper_sve_ld1hds_be_r_mte
, NULL
, NULL
, NULL
},
5310 { gen_helper_sve_ld1hss_be_r_mte
, NULL
, NULL
, NULL
},
5311 { gen_helper_sve_ld1ss_be_r_mte
,
5312 gen_helper_sve_ld2ss_be_r_mte
,
5313 gen_helper_sve_ld3ss_be_r_mte
,
5314 gen_helper_sve_ld4ss_be_r_mte
},
5315 { gen_helper_sve_ld1sdu_be_r_mte
, NULL
, NULL
, NULL
},
5317 { gen_helper_sve_ld1bds_r_mte
, NULL
, NULL
, NULL
},
5318 { gen_helper_sve_ld1bss_r_mte
, NULL
, NULL
, NULL
},
5319 { gen_helper_sve_ld1bhs_r_mte
, NULL
, NULL
, NULL
},
5320 { gen_helper_sve_ld1dd_be_r_mte
,
5321 gen_helper_sve_ld2dd_be_r_mte
,
5322 gen_helper_sve_ld3dd_be_r_mte
,
5323 gen_helper_sve_ld4dd_be_r_mte
} } },
5325 gen_helper_gvec_mem
*fn
5326 = fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][dtype
][nreg
];
5329 * While there are holes in the table, they are not
5330 * accessible via the instruction encoding.
5333 do_mem_zpa(s
, zt
, pg
, addr
, dtype
, nreg
, false, fn
);
5336 static bool trans_LD_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5341 if (sve_access_check(s
)) {
5342 TCGv_i64 addr
= new_tmp_a64(s
);
5343 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
5344 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5345 do_ld_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, a
->nreg
);
5350 static bool trans_LD_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5352 if (sve_access_check(s
)) {
5353 int vsz
= vec_full_reg_size(s
);
5354 int elements
= vsz
>> dtype_esz
[a
->dtype
];
5355 TCGv_i64 addr
= new_tmp_a64(s
);
5357 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
),
5358 (a
->imm
* elements
* (a
->nreg
+ 1))
5359 << dtype_msz(a
->dtype
));
5360 do_ld_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, a
->nreg
);
5365 static bool trans_LDFF1_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5367 static gen_helper_gvec_mem
* const fns
[2][2][16] = {
5368 { /* mte inactive, little-endian */
5369 { gen_helper_sve_ldff1bb_r
,
5370 gen_helper_sve_ldff1bhu_r
,
5371 gen_helper_sve_ldff1bsu_r
,
5372 gen_helper_sve_ldff1bdu_r
,
5374 gen_helper_sve_ldff1sds_le_r
,
5375 gen_helper_sve_ldff1hh_le_r
,
5376 gen_helper_sve_ldff1hsu_le_r
,
5377 gen_helper_sve_ldff1hdu_le_r
,
5379 gen_helper_sve_ldff1hds_le_r
,
5380 gen_helper_sve_ldff1hss_le_r
,
5381 gen_helper_sve_ldff1ss_le_r
,
5382 gen_helper_sve_ldff1sdu_le_r
,
5384 gen_helper_sve_ldff1bds_r
,
5385 gen_helper_sve_ldff1bss_r
,
5386 gen_helper_sve_ldff1bhs_r
,
5387 gen_helper_sve_ldff1dd_le_r
},
5389 /* mte inactive, big-endian */
5390 { gen_helper_sve_ldff1bb_r
,
5391 gen_helper_sve_ldff1bhu_r
,
5392 gen_helper_sve_ldff1bsu_r
,
5393 gen_helper_sve_ldff1bdu_r
,
5395 gen_helper_sve_ldff1sds_be_r
,
5396 gen_helper_sve_ldff1hh_be_r
,
5397 gen_helper_sve_ldff1hsu_be_r
,
5398 gen_helper_sve_ldff1hdu_be_r
,
5400 gen_helper_sve_ldff1hds_be_r
,
5401 gen_helper_sve_ldff1hss_be_r
,
5402 gen_helper_sve_ldff1ss_be_r
,
5403 gen_helper_sve_ldff1sdu_be_r
,
5405 gen_helper_sve_ldff1bds_r
,
5406 gen_helper_sve_ldff1bss_r
,
5407 gen_helper_sve_ldff1bhs_r
,
5408 gen_helper_sve_ldff1dd_be_r
} },
5410 { /* mte active, little-endian */
5411 { gen_helper_sve_ldff1bb_r_mte
,
5412 gen_helper_sve_ldff1bhu_r_mte
,
5413 gen_helper_sve_ldff1bsu_r_mte
,
5414 gen_helper_sve_ldff1bdu_r_mte
,
5416 gen_helper_sve_ldff1sds_le_r_mte
,
5417 gen_helper_sve_ldff1hh_le_r_mte
,
5418 gen_helper_sve_ldff1hsu_le_r_mte
,
5419 gen_helper_sve_ldff1hdu_le_r_mte
,
5421 gen_helper_sve_ldff1hds_le_r_mte
,
5422 gen_helper_sve_ldff1hss_le_r_mte
,
5423 gen_helper_sve_ldff1ss_le_r_mte
,
5424 gen_helper_sve_ldff1sdu_le_r_mte
,
5426 gen_helper_sve_ldff1bds_r_mte
,
5427 gen_helper_sve_ldff1bss_r_mte
,
5428 gen_helper_sve_ldff1bhs_r_mte
,
5429 gen_helper_sve_ldff1dd_le_r_mte
},
5431 /* mte active, big-endian */
5432 { gen_helper_sve_ldff1bb_r_mte
,
5433 gen_helper_sve_ldff1bhu_r_mte
,
5434 gen_helper_sve_ldff1bsu_r_mte
,
5435 gen_helper_sve_ldff1bdu_r_mte
,
5437 gen_helper_sve_ldff1sds_be_r_mte
,
5438 gen_helper_sve_ldff1hh_be_r_mte
,
5439 gen_helper_sve_ldff1hsu_be_r_mte
,
5440 gen_helper_sve_ldff1hdu_be_r_mte
,
5442 gen_helper_sve_ldff1hds_be_r_mte
,
5443 gen_helper_sve_ldff1hss_be_r_mte
,
5444 gen_helper_sve_ldff1ss_be_r_mte
,
5445 gen_helper_sve_ldff1sdu_be_r_mte
,
5447 gen_helper_sve_ldff1bds_r_mte
,
5448 gen_helper_sve_ldff1bss_r_mte
,
5449 gen_helper_sve_ldff1bhs_r_mte
,
5450 gen_helper_sve_ldff1dd_be_r_mte
} },
5453 if (sve_access_check(s
)) {
5454 TCGv_i64 addr
= new_tmp_a64(s
);
5455 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
5456 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5457 do_mem_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, 1, false,
5458 fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][a
->dtype
]);
5463 static bool trans_LDNF1_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5465 static gen_helper_gvec_mem
* const fns
[2][2][16] = {
5466 { /* mte inactive, little-endian */
5467 { gen_helper_sve_ldnf1bb_r
,
5468 gen_helper_sve_ldnf1bhu_r
,
5469 gen_helper_sve_ldnf1bsu_r
,
5470 gen_helper_sve_ldnf1bdu_r
,
5472 gen_helper_sve_ldnf1sds_le_r
,
5473 gen_helper_sve_ldnf1hh_le_r
,
5474 gen_helper_sve_ldnf1hsu_le_r
,
5475 gen_helper_sve_ldnf1hdu_le_r
,
5477 gen_helper_sve_ldnf1hds_le_r
,
5478 gen_helper_sve_ldnf1hss_le_r
,
5479 gen_helper_sve_ldnf1ss_le_r
,
5480 gen_helper_sve_ldnf1sdu_le_r
,
5482 gen_helper_sve_ldnf1bds_r
,
5483 gen_helper_sve_ldnf1bss_r
,
5484 gen_helper_sve_ldnf1bhs_r
,
5485 gen_helper_sve_ldnf1dd_le_r
},
5487 /* mte inactive, big-endian */
5488 { gen_helper_sve_ldnf1bb_r
,
5489 gen_helper_sve_ldnf1bhu_r
,
5490 gen_helper_sve_ldnf1bsu_r
,
5491 gen_helper_sve_ldnf1bdu_r
,
5493 gen_helper_sve_ldnf1sds_be_r
,
5494 gen_helper_sve_ldnf1hh_be_r
,
5495 gen_helper_sve_ldnf1hsu_be_r
,
5496 gen_helper_sve_ldnf1hdu_be_r
,
5498 gen_helper_sve_ldnf1hds_be_r
,
5499 gen_helper_sve_ldnf1hss_be_r
,
5500 gen_helper_sve_ldnf1ss_be_r
,
5501 gen_helper_sve_ldnf1sdu_be_r
,
5503 gen_helper_sve_ldnf1bds_r
,
5504 gen_helper_sve_ldnf1bss_r
,
5505 gen_helper_sve_ldnf1bhs_r
,
5506 gen_helper_sve_ldnf1dd_be_r
} },
5508 { /* mte inactive, little-endian */
5509 { gen_helper_sve_ldnf1bb_r_mte
,
5510 gen_helper_sve_ldnf1bhu_r_mte
,
5511 gen_helper_sve_ldnf1bsu_r_mte
,
5512 gen_helper_sve_ldnf1bdu_r_mte
,
5514 gen_helper_sve_ldnf1sds_le_r_mte
,
5515 gen_helper_sve_ldnf1hh_le_r_mte
,
5516 gen_helper_sve_ldnf1hsu_le_r_mte
,
5517 gen_helper_sve_ldnf1hdu_le_r_mte
,
5519 gen_helper_sve_ldnf1hds_le_r_mte
,
5520 gen_helper_sve_ldnf1hss_le_r_mte
,
5521 gen_helper_sve_ldnf1ss_le_r_mte
,
5522 gen_helper_sve_ldnf1sdu_le_r_mte
,
5524 gen_helper_sve_ldnf1bds_r_mte
,
5525 gen_helper_sve_ldnf1bss_r_mte
,
5526 gen_helper_sve_ldnf1bhs_r_mte
,
5527 gen_helper_sve_ldnf1dd_le_r_mte
},
5529 /* mte inactive, big-endian */
5530 { gen_helper_sve_ldnf1bb_r_mte
,
5531 gen_helper_sve_ldnf1bhu_r_mte
,
5532 gen_helper_sve_ldnf1bsu_r_mte
,
5533 gen_helper_sve_ldnf1bdu_r_mte
,
5535 gen_helper_sve_ldnf1sds_be_r_mte
,
5536 gen_helper_sve_ldnf1hh_be_r_mte
,
5537 gen_helper_sve_ldnf1hsu_be_r_mte
,
5538 gen_helper_sve_ldnf1hdu_be_r_mte
,
5540 gen_helper_sve_ldnf1hds_be_r_mte
,
5541 gen_helper_sve_ldnf1hss_be_r_mte
,
5542 gen_helper_sve_ldnf1ss_be_r_mte
,
5543 gen_helper_sve_ldnf1sdu_be_r_mte
,
5545 gen_helper_sve_ldnf1bds_r_mte
,
5546 gen_helper_sve_ldnf1bss_r_mte
,
5547 gen_helper_sve_ldnf1bhs_r_mte
,
5548 gen_helper_sve_ldnf1dd_be_r_mte
} },
5551 if (sve_access_check(s
)) {
5552 int vsz
= vec_full_reg_size(s
);
5553 int elements
= vsz
>> dtype_esz
[a
->dtype
];
5554 int off
= (a
->imm
* elements
) << dtype_msz(a
->dtype
);
5555 TCGv_i64 addr
= new_tmp_a64(s
);
5557 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), off
);
5558 do_mem_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, 1, false,
5559 fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][a
->dtype
]);
5564 static void do_ldrq(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
, int msz
)
5566 static gen_helper_gvec_mem
* const fns
[2][4] = {
5567 { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld1hh_le_r
,
5568 gen_helper_sve_ld1ss_le_r
, gen_helper_sve_ld1dd_le_r
},
5569 { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld1hh_be_r
,
5570 gen_helper_sve_ld1ss_be_r
, gen_helper_sve_ld1dd_be_r
},
5572 unsigned vsz
= vec_full_reg_size(s
);
5577 /* Load the first quadword using the normal predicated load helpers. */
5578 desc
= simd_desc(16, 16, zt
);
5579 t_desc
= tcg_const_i32(desc
);
5581 poff
= pred_full_reg_offset(s
, pg
);
5584 * Zero-extend the first 16 bits of the predicate into a temporary.
5585 * This avoids triggering an assert making sure we don't have bits
5586 * set within a predicate beyond VQ, but we have lowered VQ to 1
5587 * for this load operation.
5589 TCGv_i64 tmp
= tcg_temp_new_i64();
5590 #ifdef HOST_WORDS_BIGENDIAN
5593 tcg_gen_ld16u_i64(tmp
, cpu_env
, poff
);
5595 poff
= offsetof(CPUARMState
, vfp
.preg_tmp
);
5596 tcg_gen_st_i64(tmp
, cpu_env
, poff
);
5597 tcg_temp_free_i64(tmp
);
5600 t_pg
= tcg_temp_new_ptr();
5601 tcg_gen_addi_ptr(t_pg
, cpu_env
, poff
);
5603 fns
[s
->be_data
== MO_BE
][msz
](cpu_env
, t_pg
, addr
, t_desc
);
5605 tcg_temp_free_ptr(t_pg
);
5606 tcg_temp_free_i32(t_desc
);
5608 /* Replicate that first quadword. */
5610 unsigned dofs
= vec_full_reg_offset(s
, zt
);
5611 tcg_gen_gvec_dup_mem(4, dofs
+ 16, dofs
, vsz
- 16, vsz
- 16);
5615 static bool trans_LD1RQ_zprr(DisasContext
*s
, arg_rprr_load
*a
)
5620 if (sve_access_check(s
)) {
5621 int msz
= dtype_msz(a
->dtype
);
5622 TCGv_i64 addr
= new_tmp_a64(s
);
5623 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), msz
);
5624 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5625 do_ldrq(s
, a
->rd
, a
->pg
, addr
, msz
);
5630 static bool trans_LD1RQ_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5632 if (sve_access_check(s
)) {
5633 TCGv_i64 addr
= new_tmp_a64(s
);
5634 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), a
->imm
* 16);
5635 do_ldrq(s
, a
->rd
, a
->pg
, addr
, dtype_msz(a
->dtype
));
5640 /* Load and broadcast element. */
5641 static bool trans_LD1R_zpri(DisasContext
*s
, arg_rpri_load
*a
)
5643 unsigned vsz
= vec_full_reg_size(s
);
5644 unsigned psz
= pred_full_reg_size(s
);
5645 unsigned esz
= dtype_esz
[a
->dtype
];
5646 unsigned msz
= dtype_msz(a
->dtype
);
5648 TCGv_i64 temp
, clean_addr
;
5650 if (!sve_access_check(s
)) {
5654 over
= gen_new_label();
5656 /* If the guarding predicate has no bits set, no load occurs. */
5658 /* Reduce the pred_esz_masks value simply to reduce the
5659 * size of the code generated here.
5661 uint64_t psz_mask
= MAKE_64BIT_MASK(0, psz
* 8);
5662 temp
= tcg_temp_new_i64();
5663 tcg_gen_ld_i64(temp
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
5664 tcg_gen_andi_i64(temp
, temp
, pred_esz_masks
[esz
] & psz_mask
);
5665 tcg_gen_brcondi_i64(TCG_COND_EQ
, temp
, 0, over
);
5666 tcg_temp_free_i64(temp
);
5668 TCGv_i32 t32
= tcg_temp_new_i32();
5669 find_last_active(s
, t32
, esz
, a
->pg
);
5670 tcg_gen_brcondi_i32(TCG_COND_LT
, t32
, 0, over
);
5671 tcg_temp_free_i32(t32
);
5674 /* Load the data. */
5675 temp
= tcg_temp_new_i64();
5676 tcg_gen_addi_i64(temp
, cpu_reg_sp(s
, a
->rn
), a
->imm
<< msz
);
5677 clean_addr
= gen_mte_check1(s
, temp
, false, true, msz
);
5679 tcg_gen_qemu_ld_i64(temp
, clean_addr
, get_mem_index(s
),
5680 finalize_memop(s
, dtype_mop
[a
->dtype
]));
5682 /* Broadcast to *all* elements. */
5683 tcg_gen_gvec_dup_i64(esz
, vec_full_reg_offset(s
, a
->rd
),
5685 tcg_temp_free_i64(temp
);
5687 /* Zero the inactive elements. */
5688 gen_set_label(over
);
5689 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, esz
, false);
5692 static void do_st_zpa(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
,
5693 int msz
, int esz
, int nreg
)
5695 static gen_helper_gvec_mem
* const fn_single
[2][2][4][4] = {
5696 { { { gen_helper_sve_st1bb_r
,
5697 gen_helper_sve_st1bh_r
,
5698 gen_helper_sve_st1bs_r
,
5699 gen_helper_sve_st1bd_r
},
5701 gen_helper_sve_st1hh_le_r
,
5702 gen_helper_sve_st1hs_le_r
,
5703 gen_helper_sve_st1hd_le_r
},
5705 gen_helper_sve_st1ss_le_r
,
5706 gen_helper_sve_st1sd_le_r
},
5708 gen_helper_sve_st1dd_le_r
} },
5709 { { gen_helper_sve_st1bb_r
,
5710 gen_helper_sve_st1bh_r
,
5711 gen_helper_sve_st1bs_r
,
5712 gen_helper_sve_st1bd_r
},
5714 gen_helper_sve_st1hh_be_r
,
5715 gen_helper_sve_st1hs_be_r
,
5716 gen_helper_sve_st1hd_be_r
},
5718 gen_helper_sve_st1ss_be_r
,
5719 gen_helper_sve_st1sd_be_r
},
5721 gen_helper_sve_st1dd_be_r
} } },
5723 { { { gen_helper_sve_st1bb_r_mte
,
5724 gen_helper_sve_st1bh_r_mte
,
5725 gen_helper_sve_st1bs_r_mte
,
5726 gen_helper_sve_st1bd_r_mte
},
5728 gen_helper_sve_st1hh_le_r_mte
,
5729 gen_helper_sve_st1hs_le_r_mte
,
5730 gen_helper_sve_st1hd_le_r_mte
},
5732 gen_helper_sve_st1ss_le_r_mte
,
5733 gen_helper_sve_st1sd_le_r_mte
},
5735 gen_helper_sve_st1dd_le_r_mte
} },
5736 { { gen_helper_sve_st1bb_r_mte
,
5737 gen_helper_sve_st1bh_r_mte
,
5738 gen_helper_sve_st1bs_r_mte
,
5739 gen_helper_sve_st1bd_r_mte
},
5741 gen_helper_sve_st1hh_be_r_mte
,
5742 gen_helper_sve_st1hs_be_r_mte
,
5743 gen_helper_sve_st1hd_be_r_mte
},
5745 gen_helper_sve_st1ss_be_r_mte
,
5746 gen_helper_sve_st1sd_be_r_mte
},
5748 gen_helper_sve_st1dd_be_r_mte
} } },
5750 static gen_helper_gvec_mem
* const fn_multiple
[2][2][3][4] = {
5751 { { { gen_helper_sve_st2bb_r
,
5752 gen_helper_sve_st2hh_le_r
,
5753 gen_helper_sve_st2ss_le_r
,
5754 gen_helper_sve_st2dd_le_r
},
5755 { gen_helper_sve_st3bb_r
,
5756 gen_helper_sve_st3hh_le_r
,
5757 gen_helper_sve_st3ss_le_r
,
5758 gen_helper_sve_st3dd_le_r
},
5759 { gen_helper_sve_st4bb_r
,
5760 gen_helper_sve_st4hh_le_r
,
5761 gen_helper_sve_st4ss_le_r
,
5762 gen_helper_sve_st4dd_le_r
} },
5763 { { gen_helper_sve_st2bb_r
,
5764 gen_helper_sve_st2hh_be_r
,
5765 gen_helper_sve_st2ss_be_r
,
5766 gen_helper_sve_st2dd_be_r
},
5767 { gen_helper_sve_st3bb_r
,
5768 gen_helper_sve_st3hh_be_r
,
5769 gen_helper_sve_st3ss_be_r
,
5770 gen_helper_sve_st3dd_be_r
},
5771 { gen_helper_sve_st4bb_r
,
5772 gen_helper_sve_st4hh_be_r
,
5773 gen_helper_sve_st4ss_be_r
,
5774 gen_helper_sve_st4dd_be_r
} } },
5775 { { { gen_helper_sve_st2bb_r_mte
,
5776 gen_helper_sve_st2hh_le_r_mte
,
5777 gen_helper_sve_st2ss_le_r_mte
,
5778 gen_helper_sve_st2dd_le_r_mte
},
5779 { gen_helper_sve_st3bb_r_mte
,
5780 gen_helper_sve_st3hh_le_r_mte
,
5781 gen_helper_sve_st3ss_le_r_mte
,
5782 gen_helper_sve_st3dd_le_r_mte
},
5783 { gen_helper_sve_st4bb_r_mte
,
5784 gen_helper_sve_st4hh_le_r_mte
,
5785 gen_helper_sve_st4ss_le_r_mte
,
5786 gen_helper_sve_st4dd_le_r_mte
} },
5787 { { gen_helper_sve_st2bb_r_mte
,
5788 gen_helper_sve_st2hh_be_r_mte
,
5789 gen_helper_sve_st2ss_be_r_mte
,
5790 gen_helper_sve_st2dd_be_r_mte
},
5791 { gen_helper_sve_st3bb_r_mte
,
5792 gen_helper_sve_st3hh_be_r_mte
,
5793 gen_helper_sve_st3ss_be_r_mte
,
5794 gen_helper_sve_st3dd_be_r_mte
},
5795 { gen_helper_sve_st4bb_r_mte
,
5796 gen_helper_sve_st4hh_be_r_mte
,
5797 gen_helper_sve_st4ss_be_r_mte
,
5798 gen_helper_sve_st4dd_be_r_mte
} } },
5800 gen_helper_gvec_mem
*fn
;
5801 int be
= s
->be_data
== MO_BE
;
5805 fn
= fn_single
[s
->mte_active
[0]][be
][msz
][esz
];
5808 /* ST2, ST3, ST4 -- msz == esz, enforced by encoding */
5810 fn
= fn_multiple
[s
->mte_active
[0]][be
][nreg
- 1][msz
];
5813 do_mem_zpa(s
, zt
, pg
, addr
, msz_dtype(s
, msz
), nreg
, true, fn
);
5816 static bool trans_ST_zprr(DisasContext
*s
, arg_rprr_store
*a
)
5818 if (a
->rm
== 31 || a
->msz
> a
->esz
) {
5821 if (sve_access_check(s
)) {
5822 TCGv_i64 addr
= new_tmp_a64(s
);
5823 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), a
->msz
);
5824 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5825 do_st_zpa(s
, a
->rd
, a
->pg
, addr
, a
->msz
, a
->esz
, a
->nreg
);
5830 static bool trans_ST_zpri(DisasContext
*s
, arg_rpri_store
*a
)
5832 if (a
->msz
> a
->esz
) {
5835 if (sve_access_check(s
)) {
5836 int vsz
= vec_full_reg_size(s
);
5837 int elements
= vsz
>> a
->esz
;
5838 TCGv_i64 addr
= new_tmp_a64(s
);
5840 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
),
5841 (a
->imm
* elements
* (a
->nreg
+ 1)) << a
->msz
);
5842 do_st_zpa(s
, a
->rd
, a
->pg
, addr
, a
->msz
, a
->esz
, a
->nreg
);
5848 *** SVE gather loads / scatter stores
5851 static void do_mem_zpz(DisasContext
*s
, int zt
, int pg
, int zm
,
5852 int scale
, TCGv_i64 scalar
, int msz
, bool is_write
,
5853 gen_helper_gvec_mem_scatter
*fn
)
5855 unsigned vsz
= vec_full_reg_size(s
);
5856 TCGv_ptr t_zm
= tcg_temp_new_ptr();
5857 TCGv_ptr t_pg
= tcg_temp_new_ptr();
5858 TCGv_ptr t_zt
= tcg_temp_new_ptr();
5862 if (s
->mte_active
[0]) {
5863 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, get_mem_index(s
));
5864 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
5865 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
5866 desc
= FIELD_DP32(desc
, MTEDESC
, WRITE
, is_write
);
5867 desc
= FIELD_DP32(desc
, MTEDESC
, SIZEM1
, (1 << msz
) - 1);
5868 desc
<<= SVE_MTEDESC_SHIFT
;
5870 desc
= simd_desc(vsz
, vsz
, desc
| scale
);
5871 t_desc
= tcg_const_i32(desc
);
5873 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
5874 tcg_gen_addi_ptr(t_zm
, cpu_env
, vec_full_reg_offset(s
, zm
));
5875 tcg_gen_addi_ptr(t_zt
, cpu_env
, vec_full_reg_offset(s
, zt
));
5876 fn(cpu_env
, t_zt
, t_pg
, t_zm
, scalar
, t_desc
);
5878 tcg_temp_free_ptr(t_zt
);
5879 tcg_temp_free_ptr(t_zm
);
5880 tcg_temp_free_ptr(t_pg
);
5881 tcg_temp_free_i32(t_desc
);
5884 /* Indexed by [mte][be][ff][xs][u][msz]. */
5885 static gen_helper_gvec_mem_scatter
* const
5886 gather_load_fn32
[2][2][2][2][2][3] = {
5887 { /* MTE Inactive */
5888 { /* Little-endian */
5889 { { { gen_helper_sve_ldbss_zsu
,
5890 gen_helper_sve_ldhss_le_zsu
,
5892 { gen_helper_sve_ldbsu_zsu
,
5893 gen_helper_sve_ldhsu_le_zsu
,
5894 gen_helper_sve_ldss_le_zsu
, } },
5895 { { gen_helper_sve_ldbss_zss
,
5896 gen_helper_sve_ldhss_le_zss
,
5898 { gen_helper_sve_ldbsu_zss
,
5899 gen_helper_sve_ldhsu_le_zss
,
5900 gen_helper_sve_ldss_le_zss
, } } },
5903 { { { gen_helper_sve_ldffbss_zsu
,
5904 gen_helper_sve_ldffhss_le_zsu
,
5906 { gen_helper_sve_ldffbsu_zsu
,
5907 gen_helper_sve_ldffhsu_le_zsu
,
5908 gen_helper_sve_ldffss_le_zsu
, } },
5909 { { gen_helper_sve_ldffbss_zss
,
5910 gen_helper_sve_ldffhss_le_zss
,
5912 { gen_helper_sve_ldffbsu_zss
,
5913 gen_helper_sve_ldffhsu_le_zss
,
5914 gen_helper_sve_ldffss_le_zss
, } } } },
5917 { { { gen_helper_sve_ldbss_zsu
,
5918 gen_helper_sve_ldhss_be_zsu
,
5920 { gen_helper_sve_ldbsu_zsu
,
5921 gen_helper_sve_ldhsu_be_zsu
,
5922 gen_helper_sve_ldss_be_zsu
, } },
5923 { { gen_helper_sve_ldbss_zss
,
5924 gen_helper_sve_ldhss_be_zss
,
5926 { gen_helper_sve_ldbsu_zss
,
5927 gen_helper_sve_ldhsu_be_zss
,
5928 gen_helper_sve_ldss_be_zss
, } } },
5931 { { { gen_helper_sve_ldffbss_zsu
,
5932 gen_helper_sve_ldffhss_be_zsu
,
5934 { gen_helper_sve_ldffbsu_zsu
,
5935 gen_helper_sve_ldffhsu_be_zsu
,
5936 gen_helper_sve_ldffss_be_zsu
, } },
5937 { { gen_helper_sve_ldffbss_zss
,
5938 gen_helper_sve_ldffhss_be_zss
,
5940 { gen_helper_sve_ldffbsu_zss
,
5941 gen_helper_sve_ldffhsu_be_zss
,
5942 gen_helper_sve_ldffss_be_zss
, } } } } },
5944 { /* Little-endian */
5945 { { { gen_helper_sve_ldbss_zsu_mte
,
5946 gen_helper_sve_ldhss_le_zsu_mte
,
5948 { gen_helper_sve_ldbsu_zsu_mte
,
5949 gen_helper_sve_ldhsu_le_zsu_mte
,
5950 gen_helper_sve_ldss_le_zsu_mte
, } },
5951 { { gen_helper_sve_ldbss_zss_mte
,
5952 gen_helper_sve_ldhss_le_zss_mte
,
5954 { gen_helper_sve_ldbsu_zss_mte
,
5955 gen_helper_sve_ldhsu_le_zss_mte
,
5956 gen_helper_sve_ldss_le_zss_mte
, } } },
5959 { { { gen_helper_sve_ldffbss_zsu_mte
,
5960 gen_helper_sve_ldffhss_le_zsu_mte
,
5962 { gen_helper_sve_ldffbsu_zsu_mte
,
5963 gen_helper_sve_ldffhsu_le_zsu_mte
,
5964 gen_helper_sve_ldffss_le_zsu_mte
, } },
5965 { { gen_helper_sve_ldffbss_zss_mte
,
5966 gen_helper_sve_ldffhss_le_zss_mte
,
5968 { gen_helper_sve_ldffbsu_zss_mte
,
5969 gen_helper_sve_ldffhsu_le_zss_mte
,
5970 gen_helper_sve_ldffss_le_zss_mte
, } } } },
5973 { { { gen_helper_sve_ldbss_zsu_mte
,
5974 gen_helper_sve_ldhss_be_zsu_mte
,
5976 { gen_helper_sve_ldbsu_zsu_mte
,
5977 gen_helper_sve_ldhsu_be_zsu_mte
,
5978 gen_helper_sve_ldss_be_zsu_mte
, } },
5979 { { gen_helper_sve_ldbss_zss_mte
,
5980 gen_helper_sve_ldhss_be_zss_mte
,
5982 { gen_helper_sve_ldbsu_zss_mte
,
5983 gen_helper_sve_ldhsu_be_zss_mte
,
5984 gen_helper_sve_ldss_be_zss_mte
, } } },
5987 { { { gen_helper_sve_ldffbss_zsu_mte
,
5988 gen_helper_sve_ldffhss_be_zsu_mte
,
5990 { gen_helper_sve_ldffbsu_zsu_mte
,
5991 gen_helper_sve_ldffhsu_be_zsu_mte
,
5992 gen_helper_sve_ldffss_be_zsu_mte
, } },
5993 { { gen_helper_sve_ldffbss_zss_mte
,
5994 gen_helper_sve_ldffhss_be_zss_mte
,
5996 { gen_helper_sve_ldffbsu_zss_mte
,
5997 gen_helper_sve_ldffhsu_be_zss_mte
,
5998 gen_helper_sve_ldffss_be_zss_mte
, } } } } },
6001 /* Note that we overload xs=2 to indicate 64-bit offset. */
6002 static gen_helper_gvec_mem_scatter
* const
6003 gather_load_fn64
[2][2][2][3][2][4] = {
6004 { /* MTE Inactive */
6005 { /* Little-endian */
6006 { { { gen_helper_sve_ldbds_zsu
,
6007 gen_helper_sve_ldhds_le_zsu
,
6008 gen_helper_sve_ldsds_le_zsu
,
6010 { gen_helper_sve_ldbdu_zsu
,
6011 gen_helper_sve_ldhdu_le_zsu
,
6012 gen_helper_sve_ldsdu_le_zsu
,
6013 gen_helper_sve_lddd_le_zsu
, } },
6014 { { gen_helper_sve_ldbds_zss
,
6015 gen_helper_sve_ldhds_le_zss
,
6016 gen_helper_sve_ldsds_le_zss
,
6018 { gen_helper_sve_ldbdu_zss
,
6019 gen_helper_sve_ldhdu_le_zss
,
6020 gen_helper_sve_ldsdu_le_zss
,
6021 gen_helper_sve_lddd_le_zss
, } },
6022 { { gen_helper_sve_ldbds_zd
,
6023 gen_helper_sve_ldhds_le_zd
,
6024 gen_helper_sve_ldsds_le_zd
,
6026 { gen_helper_sve_ldbdu_zd
,
6027 gen_helper_sve_ldhdu_le_zd
,
6028 gen_helper_sve_ldsdu_le_zd
,
6029 gen_helper_sve_lddd_le_zd
, } } },
6032 { { { gen_helper_sve_ldffbds_zsu
,
6033 gen_helper_sve_ldffhds_le_zsu
,
6034 gen_helper_sve_ldffsds_le_zsu
,
6036 { gen_helper_sve_ldffbdu_zsu
,
6037 gen_helper_sve_ldffhdu_le_zsu
,
6038 gen_helper_sve_ldffsdu_le_zsu
,
6039 gen_helper_sve_ldffdd_le_zsu
, } },
6040 { { gen_helper_sve_ldffbds_zss
,
6041 gen_helper_sve_ldffhds_le_zss
,
6042 gen_helper_sve_ldffsds_le_zss
,
6044 { gen_helper_sve_ldffbdu_zss
,
6045 gen_helper_sve_ldffhdu_le_zss
,
6046 gen_helper_sve_ldffsdu_le_zss
,
6047 gen_helper_sve_ldffdd_le_zss
, } },
6048 { { gen_helper_sve_ldffbds_zd
,
6049 gen_helper_sve_ldffhds_le_zd
,
6050 gen_helper_sve_ldffsds_le_zd
,
6052 { gen_helper_sve_ldffbdu_zd
,
6053 gen_helper_sve_ldffhdu_le_zd
,
6054 gen_helper_sve_ldffsdu_le_zd
,
6055 gen_helper_sve_ldffdd_le_zd
, } } } },
6057 { { { gen_helper_sve_ldbds_zsu
,
6058 gen_helper_sve_ldhds_be_zsu
,
6059 gen_helper_sve_ldsds_be_zsu
,
6061 { gen_helper_sve_ldbdu_zsu
,
6062 gen_helper_sve_ldhdu_be_zsu
,
6063 gen_helper_sve_ldsdu_be_zsu
,
6064 gen_helper_sve_lddd_be_zsu
, } },
6065 { { gen_helper_sve_ldbds_zss
,
6066 gen_helper_sve_ldhds_be_zss
,
6067 gen_helper_sve_ldsds_be_zss
,
6069 { gen_helper_sve_ldbdu_zss
,
6070 gen_helper_sve_ldhdu_be_zss
,
6071 gen_helper_sve_ldsdu_be_zss
,
6072 gen_helper_sve_lddd_be_zss
, } },
6073 { { gen_helper_sve_ldbds_zd
,
6074 gen_helper_sve_ldhds_be_zd
,
6075 gen_helper_sve_ldsds_be_zd
,
6077 { gen_helper_sve_ldbdu_zd
,
6078 gen_helper_sve_ldhdu_be_zd
,
6079 gen_helper_sve_ldsdu_be_zd
,
6080 gen_helper_sve_lddd_be_zd
, } } },
6083 { { { gen_helper_sve_ldffbds_zsu
,
6084 gen_helper_sve_ldffhds_be_zsu
,
6085 gen_helper_sve_ldffsds_be_zsu
,
6087 { gen_helper_sve_ldffbdu_zsu
,
6088 gen_helper_sve_ldffhdu_be_zsu
,
6089 gen_helper_sve_ldffsdu_be_zsu
,
6090 gen_helper_sve_ldffdd_be_zsu
, } },
6091 { { gen_helper_sve_ldffbds_zss
,
6092 gen_helper_sve_ldffhds_be_zss
,
6093 gen_helper_sve_ldffsds_be_zss
,
6095 { gen_helper_sve_ldffbdu_zss
,
6096 gen_helper_sve_ldffhdu_be_zss
,
6097 gen_helper_sve_ldffsdu_be_zss
,
6098 gen_helper_sve_ldffdd_be_zss
, } },
6099 { { gen_helper_sve_ldffbds_zd
,
6100 gen_helper_sve_ldffhds_be_zd
,
6101 gen_helper_sve_ldffsds_be_zd
,
6103 { gen_helper_sve_ldffbdu_zd
,
6104 gen_helper_sve_ldffhdu_be_zd
,
6105 gen_helper_sve_ldffsdu_be_zd
,
6106 gen_helper_sve_ldffdd_be_zd
, } } } } },
6108 { /* Little-endian */
6109 { { { gen_helper_sve_ldbds_zsu_mte
,
6110 gen_helper_sve_ldhds_le_zsu_mte
,
6111 gen_helper_sve_ldsds_le_zsu_mte
,
6113 { gen_helper_sve_ldbdu_zsu_mte
,
6114 gen_helper_sve_ldhdu_le_zsu_mte
,
6115 gen_helper_sve_ldsdu_le_zsu_mte
,
6116 gen_helper_sve_lddd_le_zsu_mte
, } },
6117 { { gen_helper_sve_ldbds_zss_mte
,
6118 gen_helper_sve_ldhds_le_zss_mte
,
6119 gen_helper_sve_ldsds_le_zss_mte
,
6121 { gen_helper_sve_ldbdu_zss_mte
,
6122 gen_helper_sve_ldhdu_le_zss_mte
,
6123 gen_helper_sve_ldsdu_le_zss_mte
,
6124 gen_helper_sve_lddd_le_zss_mte
, } },
6125 { { gen_helper_sve_ldbds_zd_mte
,
6126 gen_helper_sve_ldhds_le_zd_mte
,
6127 gen_helper_sve_ldsds_le_zd_mte
,
6129 { gen_helper_sve_ldbdu_zd_mte
,
6130 gen_helper_sve_ldhdu_le_zd_mte
,
6131 gen_helper_sve_ldsdu_le_zd_mte
,
6132 gen_helper_sve_lddd_le_zd_mte
, } } },
6135 { { { gen_helper_sve_ldffbds_zsu_mte
,
6136 gen_helper_sve_ldffhds_le_zsu_mte
,
6137 gen_helper_sve_ldffsds_le_zsu_mte
,
6139 { gen_helper_sve_ldffbdu_zsu_mte
,
6140 gen_helper_sve_ldffhdu_le_zsu_mte
,
6141 gen_helper_sve_ldffsdu_le_zsu_mte
,
6142 gen_helper_sve_ldffdd_le_zsu_mte
, } },
6143 { { gen_helper_sve_ldffbds_zss_mte
,
6144 gen_helper_sve_ldffhds_le_zss_mte
,
6145 gen_helper_sve_ldffsds_le_zss_mte
,
6147 { gen_helper_sve_ldffbdu_zss_mte
,
6148 gen_helper_sve_ldffhdu_le_zss_mte
,
6149 gen_helper_sve_ldffsdu_le_zss_mte
,
6150 gen_helper_sve_ldffdd_le_zss_mte
, } },
6151 { { gen_helper_sve_ldffbds_zd_mte
,
6152 gen_helper_sve_ldffhds_le_zd_mte
,
6153 gen_helper_sve_ldffsds_le_zd_mte
,
6155 { gen_helper_sve_ldffbdu_zd_mte
,
6156 gen_helper_sve_ldffhdu_le_zd_mte
,
6157 gen_helper_sve_ldffsdu_le_zd_mte
,
6158 gen_helper_sve_ldffdd_le_zd_mte
, } } } },
6160 { { { gen_helper_sve_ldbds_zsu_mte
,
6161 gen_helper_sve_ldhds_be_zsu_mte
,
6162 gen_helper_sve_ldsds_be_zsu_mte
,
6164 { gen_helper_sve_ldbdu_zsu_mte
,
6165 gen_helper_sve_ldhdu_be_zsu_mte
,
6166 gen_helper_sve_ldsdu_be_zsu_mte
,
6167 gen_helper_sve_lddd_be_zsu_mte
, } },
6168 { { gen_helper_sve_ldbds_zss_mte
,
6169 gen_helper_sve_ldhds_be_zss_mte
,
6170 gen_helper_sve_ldsds_be_zss_mte
,
6172 { gen_helper_sve_ldbdu_zss_mte
,
6173 gen_helper_sve_ldhdu_be_zss_mte
,
6174 gen_helper_sve_ldsdu_be_zss_mte
,
6175 gen_helper_sve_lddd_be_zss_mte
, } },
6176 { { gen_helper_sve_ldbds_zd_mte
,
6177 gen_helper_sve_ldhds_be_zd_mte
,
6178 gen_helper_sve_ldsds_be_zd_mte
,
6180 { gen_helper_sve_ldbdu_zd_mte
,
6181 gen_helper_sve_ldhdu_be_zd_mte
,
6182 gen_helper_sve_ldsdu_be_zd_mte
,
6183 gen_helper_sve_lddd_be_zd_mte
, } } },
6186 { { { gen_helper_sve_ldffbds_zsu_mte
,
6187 gen_helper_sve_ldffhds_be_zsu_mte
,
6188 gen_helper_sve_ldffsds_be_zsu_mte
,
6190 { gen_helper_sve_ldffbdu_zsu_mte
,
6191 gen_helper_sve_ldffhdu_be_zsu_mte
,
6192 gen_helper_sve_ldffsdu_be_zsu_mte
,
6193 gen_helper_sve_ldffdd_be_zsu_mte
, } },
6194 { { gen_helper_sve_ldffbds_zss_mte
,
6195 gen_helper_sve_ldffhds_be_zss_mte
,
6196 gen_helper_sve_ldffsds_be_zss_mte
,
6198 { gen_helper_sve_ldffbdu_zss_mte
,
6199 gen_helper_sve_ldffhdu_be_zss_mte
,
6200 gen_helper_sve_ldffsdu_be_zss_mte
,
6201 gen_helper_sve_ldffdd_be_zss_mte
, } },
6202 { { gen_helper_sve_ldffbds_zd_mte
,
6203 gen_helper_sve_ldffhds_be_zd_mte
,
6204 gen_helper_sve_ldffsds_be_zd_mte
,
6206 { gen_helper_sve_ldffbdu_zd_mte
,
6207 gen_helper_sve_ldffhdu_be_zd_mte
,
6208 gen_helper_sve_ldffsdu_be_zd_mte
,
6209 gen_helper_sve_ldffdd_be_zd_mte
, } } } } },
6212 static bool trans_LD1_zprz(DisasContext
*s
, arg_LD1_zprz
*a
)
6214 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6215 bool be
= s
->be_data
== MO_BE
;
6216 bool mte
= s
->mte_active
[0];
6218 if (!sve_access_check(s
)) {
6224 fn
= gather_load_fn32
[mte
][be
][a
->ff
][a
->xs
][a
->u
][a
->msz
];
6227 fn
= gather_load_fn64
[mte
][be
][a
->ff
][a
->xs
][a
->u
][a
->msz
];
6232 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rm
, a
->scale
* a
->msz
,
6233 cpu_reg_sp(s
, a
->rn
), a
->msz
, false, fn
);
6237 static bool trans_LD1_zpiz(DisasContext
*s
, arg_LD1_zpiz
*a
)
6239 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6240 bool be
= s
->be_data
== MO_BE
;
6241 bool mte
= s
->mte_active
[0];
6244 if (a
->esz
< a
->msz
|| (a
->esz
== a
->msz
&& !a
->u
)) {
6247 if (!sve_access_check(s
)) {
6253 fn
= gather_load_fn32
[mte
][be
][a
->ff
][0][a
->u
][a
->msz
];
6256 fn
= gather_load_fn64
[mte
][be
][a
->ff
][2][a
->u
][a
->msz
];
6261 /* Treat LD1_zpiz (zn[x] + imm) the same way as LD1_zprz (rn + zm[x])
6262 * by loading the immediate into the scalar parameter.
6264 imm
= tcg_const_i64(a
->imm
<< a
->msz
);
6265 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0, imm
, a
->msz
, false, fn
);
6266 tcg_temp_free_i64(imm
);
6270 static bool trans_LDNT1_zprz(DisasContext
*s
, arg_LD1_zprz
*a
)
6272 if (!dc_isar_feature(aa64_sve2
, s
)) {
6275 return trans_LD1_zprz(s
, a
);
6278 /* Indexed by [mte][be][xs][msz]. */
6279 static gen_helper_gvec_mem_scatter
* const scatter_store_fn32
[2][2][2][3] = {
6280 { /* MTE Inactive */
6281 { /* Little-endian */
6282 { gen_helper_sve_stbs_zsu
,
6283 gen_helper_sve_sths_le_zsu
,
6284 gen_helper_sve_stss_le_zsu
, },
6285 { gen_helper_sve_stbs_zss
,
6286 gen_helper_sve_sths_le_zss
,
6287 gen_helper_sve_stss_le_zss
, } },
6289 { gen_helper_sve_stbs_zsu
,
6290 gen_helper_sve_sths_be_zsu
,
6291 gen_helper_sve_stss_be_zsu
, },
6292 { gen_helper_sve_stbs_zss
,
6293 gen_helper_sve_sths_be_zss
,
6294 gen_helper_sve_stss_be_zss
, } } },
6296 { /* Little-endian */
6297 { gen_helper_sve_stbs_zsu_mte
,
6298 gen_helper_sve_sths_le_zsu_mte
,
6299 gen_helper_sve_stss_le_zsu_mte
, },
6300 { gen_helper_sve_stbs_zss_mte
,
6301 gen_helper_sve_sths_le_zss_mte
,
6302 gen_helper_sve_stss_le_zss_mte
, } },
6304 { gen_helper_sve_stbs_zsu_mte
,
6305 gen_helper_sve_sths_be_zsu_mte
,
6306 gen_helper_sve_stss_be_zsu_mte
, },
6307 { gen_helper_sve_stbs_zss_mte
,
6308 gen_helper_sve_sths_be_zss_mte
,
6309 gen_helper_sve_stss_be_zss_mte
, } } },
6312 /* Note that we overload xs=2 to indicate 64-bit offset. */
6313 static gen_helper_gvec_mem_scatter
* const scatter_store_fn64
[2][2][3][4] = {
6314 { /* MTE Inactive */
6315 { /* Little-endian */
6316 { gen_helper_sve_stbd_zsu
,
6317 gen_helper_sve_sthd_le_zsu
,
6318 gen_helper_sve_stsd_le_zsu
,
6319 gen_helper_sve_stdd_le_zsu
, },
6320 { gen_helper_sve_stbd_zss
,
6321 gen_helper_sve_sthd_le_zss
,
6322 gen_helper_sve_stsd_le_zss
,
6323 gen_helper_sve_stdd_le_zss
, },
6324 { gen_helper_sve_stbd_zd
,
6325 gen_helper_sve_sthd_le_zd
,
6326 gen_helper_sve_stsd_le_zd
,
6327 gen_helper_sve_stdd_le_zd
, } },
6329 { gen_helper_sve_stbd_zsu
,
6330 gen_helper_sve_sthd_be_zsu
,
6331 gen_helper_sve_stsd_be_zsu
,
6332 gen_helper_sve_stdd_be_zsu
, },
6333 { gen_helper_sve_stbd_zss
,
6334 gen_helper_sve_sthd_be_zss
,
6335 gen_helper_sve_stsd_be_zss
,
6336 gen_helper_sve_stdd_be_zss
, },
6337 { gen_helper_sve_stbd_zd
,
6338 gen_helper_sve_sthd_be_zd
,
6339 gen_helper_sve_stsd_be_zd
,
6340 gen_helper_sve_stdd_be_zd
, } } },
6341 { /* MTE Inactive */
6342 { /* Little-endian */
6343 { gen_helper_sve_stbd_zsu_mte
,
6344 gen_helper_sve_sthd_le_zsu_mte
,
6345 gen_helper_sve_stsd_le_zsu_mte
,
6346 gen_helper_sve_stdd_le_zsu_mte
, },
6347 { gen_helper_sve_stbd_zss_mte
,
6348 gen_helper_sve_sthd_le_zss_mte
,
6349 gen_helper_sve_stsd_le_zss_mte
,
6350 gen_helper_sve_stdd_le_zss_mte
, },
6351 { gen_helper_sve_stbd_zd_mte
,
6352 gen_helper_sve_sthd_le_zd_mte
,
6353 gen_helper_sve_stsd_le_zd_mte
,
6354 gen_helper_sve_stdd_le_zd_mte
, } },
6356 { gen_helper_sve_stbd_zsu_mte
,
6357 gen_helper_sve_sthd_be_zsu_mte
,
6358 gen_helper_sve_stsd_be_zsu_mte
,
6359 gen_helper_sve_stdd_be_zsu_mte
, },
6360 { gen_helper_sve_stbd_zss_mte
,
6361 gen_helper_sve_sthd_be_zss_mte
,
6362 gen_helper_sve_stsd_be_zss_mte
,
6363 gen_helper_sve_stdd_be_zss_mte
, },
6364 { gen_helper_sve_stbd_zd_mte
,
6365 gen_helper_sve_sthd_be_zd_mte
,
6366 gen_helper_sve_stsd_be_zd_mte
,
6367 gen_helper_sve_stdd_be_zd_mte
, } } },
6370 static bool trans_ST1_zprz(DisasContext
*s
, arg_ST1_zprz
*a
)
6372 gen_helper_gvec_mem_scatter
*fn
;
6373 bool be
= s
->be_data
== MO_BE
;
6374 bool mte
= s
->mte_active
[0];
6376 if (a
->esz
< a
->msz
|| (a
->msz
== 0 && a
->scale
)) {
6379 if (!sve_access_check(s
)) {
6384 fn
= scatter_store_fn32
[mte
][be
][a
->xs
][a
->msz
];
6387 fn
= scatter_store_fn64
[mte
][be
][a
->xs
][a
->msz
];
6390 g_assert_not_reached();
6392 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rm
, a
->scale
* a
->msz
,
6393 cpu_reg_sp(s
, a
->rn
), a
->msz
, true, fn
);
6397 static bool trans_ST1_zpiz(DisasContext
*s
, arg_ST1_zpiz
*a
)
6399 gen_helper_gvec_mem_scatter
*fn
= NULL
;
6400 bool be
= s
->be_data
== MO_BE
;
6401 bool mte
= s
->mte_active
[0];
6404 if (a
->esz
< a
->msz
) {
6407 if (!sve_access_check(s
)) {
6413 fn
= scatter_store_fn32
[mte
][be
][0][a
->msz
];
6416 fn
= scatter_store_fn64
[mte
][be
][2][a
->msz
];
6421 /* Treat ST1_zpiz (zn[x] + imm) the same way as ST1_zprz (rn + zm[x])
6422 * by loading the immediate into the scalar parameter.
6424 imm
= tcg_const_i64(a
->imm
<< a
->msz
);
6425 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0, imm
, a
->msz
, true, fn
);
6426 tcg_temp_free_i64(imm
);
6430 static bool trans_STNT1_zprz(DisasContext
*s
, arg_ST1_zprz
*a
)
6432 if (!dc_isar_feature(aa64_sve2
, s
)) {
6435 return trans_ST1_zprz(s
, a
);
6442 static bool trans_PRF(DisasContext
*s
, arg_PRF
*a
)
6444 /* Prefetch is a nop within QEMU. */
6445 (void)sve_access_check(s
);
6449 static bool trans_PRF_rr(DisasContext
*s
, arg_PRF_rr
*a
)
6454 /* Prefetch is a nop within QEMU. */
6455 (void)sve_access_check(s
);
6462 * TODO: The implementation so far could handle predicated merging movprfx.
6463 * The helper functions as written take an extra source register to
6464 * use in the operation, but the result is only written when predication
6465 * succeeds. For unpredicated movprfx, we need to rearrange the helpers
6466 * to allow the final write back to the destination to be unconditional.
6467 * For predicated zeroing movprfx, we need to rearrange the helpers to
6468 * allow the final write back to zero inactives.
6470 * In the meantime, just emit the moves.
6473 static bool trans_MOVPRFX(DisasContext
*s
, arg_MOVPRFX
*a
)
6475 return do_mov_z(s
, a
->rd
, a
->rn
);
6478 static bool trans_MOVPRFX_m(DisasContext
*s
, arg_rpr_esz
*a
)
6480 if (sve_access_check(s
)) {
6481 do_sel_z(s
, a
->rd
, a
->rn
, a
->rd
, a
->pg
, a
->esz
);
6486 static bool trans_MOVPRFX_z(DisasContext
*s
, arg_rpr_esz
*a
)
6488 return do_movz_zpz(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
, false);
6492 * SVE2 Integer Multiply - Unpredicated
6495 static bool trans_MUL_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6497 if (!dc_isar_feature(aa64_sve2
, s
)) {
6500 if (sve_access_check(s
)) {
6501 gen_gvec_fn_zzz(s
, tcg_gen_gvec_mul
, a
->esz
, a
->rd
, a
->rn
, a
->rm
);
6506 static bool do_sve2_zzz_ool(DisasContext
*s
, arg_rrr_esz
*a
,
6507 gen_helper_gvec_3
*fn
)
6509 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
6512 if (sve_access_check(s
)) {
6513 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, 0);
6518 static bool trans_SMULH_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6520 static gen_helper_gvec_3
* const fns
[4] = {
6521 gen_helper_gvec_smulh_b
, gen_helper_gvec_smulh_h
,
6522 gen_helper_gvec_smulh_s
, gen_helper_gvec_smulh_d
,
6524 return do_sve2_zzz_ool(s
, a
, fns
[a
->esz
]);
6527 static bool trans_UMULH_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6529 static gen_helper_gvec_3
* const fns
[4] = {
6530 gen_helper_gvec_umulh_b
, gen_helper_gvec_umulh_h
,
6531 gen_helper_gvec_umulh_s
, gen_helper_gvec_umulh_d
,
6533 return do_sve2_zzz_ool(s
, a
, fns
[a
->esz
]);
6536 static bool trans_PMUL_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6538 return do_sve2_zzz_ool(s
, a
, gen_helper_gvec_pmul_b
);
6541 static bool trans_SQDMULH_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6543 static gen_helper_gvec_3
* const fns
[4] = {
6544 gen_helper_sve2_sqdmulh_b
, gen_helper_sve2_sqdmulh_h
,
6545 gen_helper_sve2_sqdmulh_s
, gen_helper_sve2_sqdmulh_d
,
6547 return do_sve2_zzz_ool(s
, a
, fns
[a
->esz
]);
6550 static bool trans_SQRDMULH_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
6552 static gen_helper_gvec_3
* const fns
[4] = {
6553 gen_helper_sve2_sqrdmulh_b
, gen_helper_sve2_sqrdmulh_h
,
6554 gen_helper_sve2_sqrdmulh_s
, gen_helper_sve2_sqrdmulh_d
,
6556 return do_sve2_zzz_ool(s
, a
, fns
[a
->esz
]);
6560 * SVE2 Integer - Predicated
6563 static bool do_sve2_zpzz_ool(DisasContext
*s
, arg_rprr_esz
*a
,
6564 gen_helper_gvec_4
*fn
)
6566 if (!dc_isar_feature(aa64_sve2
, s
)) {
6569 return do_zpzz_ool(s
, a
, fn
);
6572 static bool trans_SADALP_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
6574 static gen_helper_gvec_4
* const fns
[3] = {
6575 gen_helper_sve2_sadalp_zpzz_h
,
6576 gen_helper_sve2_sadalp_zpzz_s
,
6577 gen_helper_sve2_sadalp_zpzz_d
,
6582 return do_sve2_zpzz_ool(s
, a
, fns
[a
->esz
- 1]);
6585 static bool trans_UADALP_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
6587 static gen_helper_gvec_4
* const fns
[3] = {
6588 gen_helper_sve2_uadalp_zpzz_h
,
6589 gen_helper_sve2_uadalp_zpzz_s
,
6590 gen_helper_sve2_uadalp_zpzz_d
,
6595 return do_sve2_zpzz_ool(s
, a
, fns
[a
->esz
- 1]);
6599 * SVE2 integer unary operations (predicated)
6602 static bool do_sve2_zpz_ool(DisasContext
*s
, arg_rpr_esz
*a
,
6603 gen_helper_gvec_3
*fn
)
6605 if (!dc_isar_feature(aa64_sve2
, s
)) {
6608 return do_zpz_ool(s
, a
, fn
);
6611 static bool trans_URECPE(DisasContext
*s
, arg_rpr_esz
*a
)
6616 return do_sve2_zpz_ool(s
, a
, gen_helper_sve2_urecpe_s
);
6619 static bool trans_URSQRTE(DisasContext
*s
, arg_rpr_esz
*a
)
6624 return do_sve2_zpz_ool(s
, a
, gen_helper_sve2_ursqrte_s
);
6627 static bool trans_SQABS(DisasContext
*s
, arg_rpr_esz
*a
)
6629 static gen_helper_gvec_3
* const fns
[4] = {
6630 gen_helper_sve2_sqabs_b
, gen_helper_sve2_sqabs_h
,
6631 gen_helper_sve2_sqabs_s
, gen_helper_sve2_sqabs_d
,
6633 return do_sve2_zpz_ool(s
, a
, fns
[a
->esz
]);
6636 static bool trans_SQNEG(DisasContext
*s
, arg_rpr_esz
*a
)
6638 static gen_helper_gvec_3
* const fns
[4] = {
6639 gen_helper_sve2_sqneg_b
, gen_helper_sve2_sqneg_h
,
6640 gen_helper_sve2_sqneg_s
, gen_helper_sve2_sqneg_d
,
6642 return do_sve2_zpz_ool(s
, a
, fns
[a
->esz
]);
6645 #define DO_SVE2_ZPZZ(NAME, name) \
6646 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
6648 static gen_helper_gvec_4 * const fns[4] = { \
6649 gen_helper_sve2_##name##_zpzz_b, gen_helper_sve2_##name##_zpzz_h, \
6650 gen_helper_sve2_##name##_zpzz_s, gen_helper_sve2_##name##_zpzz_d, \
6652 return do_sve2_zpzz_ool(s, a, fns[a->esz]); \
6655 DO_SVE2_ZPZZ(SQSHL
, sqshl
)
6656 DO_SVE2_ZPZZ(SQRSHL
, sqrshl
)
6657 DO_SVE2_ZPZZ(SRSHL
, srshl
)
6659 DO_SVE2_ZPZZ(UQSHL
, uqshl
)
6660 DO_SVE2_ZPZZ(UQRSHL
, uqrshl
)
6661 DO_SVE2_ZPZZ(URSHL
, urshl
)
6663 DO_SVE2_ZPZZ(SHADD
, shadd
)
6664 DO_SVE2_ZPZZ(SRHADD
, srhadd
)
6665 DO_SVE2_ZPZZ(SHSUB
, shsub
)
6667 DO_SVE2_ZPZZ(UHADD
, uhadd
)
6668 DO_SVE2_ZPZZ(URHADD
, urhadd
)
6669 DO_SVE2_ZPZZ(UHSUB
, uhsub
)
6671 DO_SVE2_ZPZZ(ADDP
, addp
)
6672 DO_SVE2_ZPZZ(SMAXP
, smaxp
)
6673 DO_SVE2_ZPZZ(UMAXP
, umaxp
)
6674 DO_SVE2_ZPZZ(SMINP
, sminp
)
6675 DO_SVE2_ZPZZ(UMINP
, uminp
)
6677 DO_SVE2_ZPZZ(SQADD_zpzz
, sqadd
)
6678 DO_SVE2_ZPZZ(UQADD_zpzz
, uqadd
)
6679 DO_SVE2_ZPZZ(SQSUB_zpzz
, sqsub
)
6680 DO_SVE2_ZPZZ(UQSUB_zpzz
, uqsub
)
6681 DO_SVE2_ZPZZ(SUQADD
, suqadd
)
6682 DO_SVE2_ZPZZ(USQADD
, usqadd
)
6685 * SVE2 Widening Integer Arithmetic
6688 static bool do_sve2_zzw_ool(DisasContext
*s
, arg_rrr_esz
*a
,
6689 gen_helper_gvec_3
*fn
, int data
)
6691 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
6694 if (sve_access_check(s
)) {
6695 unsigned vsz
= vec_full_reg_size(s
);
6696 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, a
->rd
),
6697 vec_full_reg_offset(s
, a
->rn
),
6698 vec_full_reg_offset(s
, a
->rm
),
6699 vsz
, vsz
, data
, fn
);
6704 #define DO_SVE2_ZZZ_TB(NAME, name, SEL1, SEL2) \
6705 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
6707 static gen_helper_gvec_3 * const fns[4] = { \
6708 NULL, gen_helper_sve2_##name##_h, \
6709 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
6711 return do_sve2_zzw_ool(s, a, fns[a->esz], (SEL2 << 1) | SEL1); \
6714 DO_SVE2_ZZZ_TB(SADDLB
, saddl
, false, false)
6715 DO_SVE2_ZZZ_TB(SSUBLB
, ssubl
, false, false)
6716 DO_SVE2_ZZZ_TB(SABDLB
, sabdl
, false, false)
6718 DO_SVE2_ZZZ_TB(UADDLB
, uaddl
, false, false)
6719 DO_SVE2_ZZZ_TB(USUBLB
, usubl
, false, false)
6720 DO_SVE2_ZZZ_TB(UABDLB
, uabdl
, false, false)
6722 DO_SVE2_ZZZ_TB(SADDLT
, saddl
, true, true)
6723 DO_SVE2_ZZZ_TB(SSUBLT
, ssubl
, true, true)
6724 DO_SVE2_ZZZ_TB(SABDLT
, sabdl
, true, true)
6726 DO_SVE2_ZZZ_TB(UADDLT
, uaddl
, true, true)
6727 DO_SVE2_ZZZ_TB(USUBLT
, usubl
, true, true)
6728 DO_SVE2_ZZZ_TB(UABDLT
, uabdl
, true, true)
6730 DO_SVE2_ZZZ_TB(SADDLBT
, saddl
, false, true)
6731 DO_SVE2_ZZZ_TB(SSUBLBT
, ssubl
, false, true)
6732 DO_SVE2_ZZZ_TB(SSUBLTB
, ssubl
, true, false)
6734 DO_SVE2_ZZZ_TB(SQDMULLB_zzz
, sqdmull_zzz
, false, false)
6735 DO_SVE2_ZZZ_TB(SQDMULLT_zzz
, sqdmull_zzz
, true, true)
6737 DO_SVE2_ZZZ_TB(SMULLB_zzz
, smull_zzz
, false, false)
6738 DO_SVE2_ZZZ_TB(SMULLT_zzz
, smull_zzz
, true, true)
6740 DO_SVE2_ZZZ_TB(UMULLB_zzz
, umull_zzz
, false, false)
6741 DO_SVE2_ZZZ_TB(UMULLT_zzz
, umull_zzz
, true, true)
6743 static bool do_eor_tb(DisasContext
*s
, arg_rrr_esz
*a
, bool sel1
)
6745 static gen_helper_gvec_3
* const fns
[4] = {
6746 gen_helper_sve2_eoril_b
, gen_helper_sve2_eoril_h
,
6747 gen_helper_sve2_eoril_s
, gen_helper_sve2_eoril_d
,
6749 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], (!sel1
<< 1) | sel1
);
6752 static bool trans_EORBT(DisasContext
*s
, arg_rrr_esz
*a
)
6754 return do_eor_tb(s
, a
, false);
6757 static bool trans_EORTB(DisasContext
*s
, arg_rrr_esz
*a
)
6759 return do_eor_tb(s
, a
, true);
6762 static bool do_trans_pmull(DisasContext
*s
, arg_rrr_esz
*a
, bool sel
)
6764 static gen_helper_gvec_3
* const fns
[4] = {
6765 gen_helper_gvec_pmull_q
, gen_helper_sve2_pmull_h
,
6766 NULL
, gen_helper_sve2_pmull_d
,
6768 if (a
->esz
== 0 && !dc_isar_feature(aa64_sve2_pmull128
, s
)) {
6771 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], sel
);
6774 static bool trans_PMULLB(DisasContext
*s
, arg_rrr_esz
*a
)
6776 return do_trans_pmull(s
, a
, false);
6779 static bool trans_PMULLT(DisasContext
*s
, arg_rrr_esz
*a
)
6781 return do_trans_pmull(s
, a
, true);
6784 #define DO_SVE2_ZZZ_WTB(NAME, name, SEL2) \
6785 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
6787 static gen_helper_gvec_3 * const fns[4] = { \
6788 NULL, gen_helper_sve2_##name##_h, \
6789 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
6791 return do_sve2_zzw_ool(s, a, fns[a->esz], SEL2); \
6794 DO_SVE2_ZZZ_WTB(SADDWB
, saddw
, false)
6795 DO_SVE2_ZZZ_WTB(SADDWT
, saddw
, true)
6796 DO_SVE2_ZZZ_WTB(SSUBWB
, ssubw
, false)
6797 DO_SVE2_ZZZ_WTB(SSUBWT
, ssubw
, true)
6799 DO_SVE2_ZZZ_WTB(UADDWB
, uaddw
, false)
6800 DO_SVE2_ZZZ_WTB(UADDWT
, uaddw
, true)
6801 DO_SVE2_ZZZ_WTB(USUBWB
, usubw
, false)
6802 DO_SVE2_ZZZ_WTB(USUBWT
, usubw
, true)
6804 static void gen_sshll_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t imm
)
6808 int halfbits
= 4 << vece
;
6811 if (shl
== halfbits
) {
6812 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6813 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(halfbits
, halfbits
));
6814 tcg_gen_and_vec(vece
, d
, n
, t
);
6815 tcg_temp_free_vec(t
);
6817 tcg_gen_sari_vec(vece
, d
, n
, halfbits
);
6818 tcg_gen_shli_vec(vece
, d
, d
, shl
);
6821 tcg_gen_shli_vec(vece
, d
, n
, halfbits
);
6822 tcg_gen_sari_vec(vece
, d
, d
, halfbits
- shl
);
6826 static void gen_ushll_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int imm
)
6828 int halfbits
= 4 << vece
;
6830 int shl
= (imm
>> 1);
6834 mask
= MAKE_64BIT_MASK(0, halfbits
);
6836 mask
= dup_const(vece
, mask
);
6838 shift
= shl
- top
* halfbits
;
6840 tcg_gen_shri_i64(d
, n
, -shift
);
6842 tcg_gen_shli_i64(d
, n
, shift
);
6844 tcg_gen_andi_i64(d
, d
, mask
);
6847 static void gen_ushll16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6849 gen_ushll_i64(MO_16
, d
, n
, imm
);
6852 static void gen_ushll32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6854 gen_ushll_i64(MO_32
, d
, n
, imm
);
6857 static void gen_ushll64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6859 gen_ushll_i64(MO_64
, d
, n
, imm
);
6862 static void gen_ushll_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t imm
)
6864 int halfbits
= 4 << vece
;
6869 if (shl
== halfbits
) {
6870 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6871 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(halfbits
, halfbits
));
6872 tcg_gen_and_vec(vece
, d
, n
, t
);
6873 tcg_temp_free_vec(t
);
6875 tcg_gen_shri_vec(vece
, d
, n
, halfbits
);
6876 tcg_gen_shli_vec(vece
, d
, d
, shl
);
6880 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6881 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
6882 tcg_gen_and_vec(vece
, d
, n
, t
);
6883 tcg_temp_free_vec(t
);
6885 tcg_gen_shli_vec(vece
, d
, n
, halfbits
);
6886 tcg_gen_shri_vec(vece
, d
, d
, halfbits
- shl
);
6891 static bool do_sve2_shll_tb(DisasContext
*s
, arg_rri_esz
*a
,
6894 static const TCGOpcode sshll_list
[] = {
6895 INDEX_op_shli_vec
, INDEX_op_sari_vec
, 0
6897 static const TCGOpcode ushll_list
[] = {
6898 INDEX_op_shli_vec
, INDEX_op_shri_vec
, 0
6900 static const GVecGen2i ops
[2][3] = {
6901 { { .fniv
= gen_sshll_vec
,
6902 .opt_opc
= sshll_list
,
6903 .fno
= gen_helper_sve2_sshll_h
,
6905 { .fniv
= gen_sshll_vec
,
6906 .opt_opc
= sshll_list
,
6907 .fno
= gen_helper_sve2_sshll_s
,
6909 { .fniv
= gen_sshll_vec
,
6910 .opt_opc
= sshll_list
,
6911 .fno
= gen_helper_sve2_sshll_d
,
6913 { { .fni8
= gen_ushll16_i64
,
6914 .fniv
= gen_ushll_vec
,
6915 .opt_opc
= ushll_list
,
6916 .fno
= gen_helper_sve2_ushll_h
,
6918 { .fni8
= gen_ushll32_i64
,
6919 .fniv
= gen_ushll_vec
,
6920 .opt_opc
= ushll_list
,
6921 .fno
= gen_helper_sve2_ushll_s
,
6923 { .fni8
= gen_ushll64_i64
,
6924 .fniv
= gen_ushll_vec
,
6925 .opt_opc
= ushll_list
,
6926 .fno
= gen_helper_sve2_ushll_d
,
6930 if (a
->esz
< 0 || a
->esz
> 2 || !dc_isar_feature(aa64_sve2
, s
)) {
6933 if (sve_access_check(s
)) {
6934 unsigned vsz
= vec_full_reg_size(s
);
6935 tcg_gen_gvec_2i(vec_full_reg_offset(s
, a
->rd
),
6936 vec_full_reg_offset(s
, a
->rn
),
6937 vsz
, vsz
, (a
->imm
<< 1) | sel
,
6943 static bool trans_SSHLLB(DisasContext
*s
, arg_rri_esz
*a
)
6945 return do_sve2_shll_tb(s
, a
, false, false);
6948 static bool trans_SSHLLT(DisasContext
*s
, arg_rri_esz
*a
)
6950 return do_sve2_shll_tb(s
, a
, true, false);
6953 static bool trans_USHLLB(DisasContext
*s
, arg_rri_esz
*a
)
6955 return do_sve2_shll_tb(s
, a
, false, true);
6958 static bool trans_USHLLT(DisasContext
*s
, arg_rri_esz
*a
)
6960 return do_sve2_shll_tb(s
, a
, true, true);
6963 static bool trans_BEXT(DisasContext
*s
, arg_rrr_esz
*a
)
6965 static gen_helper_gvec_3
* const fns
[4] = {
6966 gen_helper_sve2_bext_b
, gen_helper_sve2_bext_h
,
6967 gen_helper_sve2_bext_s
, gen_helper_sve2_bext_d
,
6969 if (!dc_isar_feature(aa64_sve2_bitperm
, s
)) {
6972 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], 0);
6975 static bool trans_BDEP(DisasContext
*s
, arg_rrr_esz
*a
)
6977 static gen_helper_gvec_3
* const fns
[4] = {
6978 gen_helper_sve2_bdep_b
, gen_helper_sve2_bdep_h
,
6979 gen_helper_sve2_bdep_s
, gen_helper_sve2_bdep_d
,
6981 if (!dc_isar_feature(aa64_sve2_bitperm
, s
)) {
6984 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], 0);
6987 static bool trans_BGRP(DisasContext
*s
, arg_rrr_esz
*a
)
6989 static gen_helper_gvec_3
* const fns
[4] = {
6990 gen_helper_sve2_bgrp_b
, gen_helper_sve2_bgrp_h
,
6991 gen_helper_sve2_bgrp_s
, gen_helper_sve2_bgrp_d
,
6993 if (!dc_isar_feature(aa64_sve2_bitperm
, s
)) {
6996 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], 0);
6999 static bool do_cadd(DisasContext
*s
, arg_rrr_esz
*a
, bool sq
, bool rot
)
7001 static gen_helper_gvec_3
* const fns
[2][4] = {
7002 { gen_helper_sve2_cadd_b
, gen_helper_sve2_cadd_h
,
7003 gen_helper_sve2_cadd_s
, gen_helper_sve2_cadd_d
},
7004 { gen_helper_sve2_sqcadd_b
, gen_helper_sve2_sqcadd_h
,
7005 gen_helper_sve2_sqcadd_s
, gen_helper_sve2_sqcadd_d
},
7007 return do_sve2_zzw_ool(s
, a
, fns
[sq
][a
->esz
], rot
);
7010 static bool trans_CADD_rot90(DisasContext
*s
, arg_rrr_esz
*a
)
7012 return do_cadd(s
, a
, false, false);
7015 static bool trans_CADD_rot270(DisasContext
*s
, arg_rrr_esz
*a
)
7017 return do_cadd(s
, a
, false, true);
7020 static bool trans_SQCADD_rot90(DisasContext
*s
, arg_rrr_esz
*a
)
7022 return do_cadd(s
, a
, true, false);
7025 static bool trans_SQCADD_rot270(DisasContext
*s
, arg_rrr_esz
*a
)
7027 return do_cadd(s
, a
, true, true);
7030 static bool do_sve2_zzzz_ool(DisasContext
*s
, arg_rrrr_esz
*a
,
7031 gen_helper_gvec_4
*fn
, int data
)
7033 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
7036 if (sve_access_check(s
)) {
7037 gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, data
);
7042 static bool do_abal(DisasContext
*s
, arg_rrrr_esz
*a
, bool uns
, bool sel
)
7044 static gen_helper_gvec_4
* const fns
[2][4] = {
7045 { NULL
, gen_helper_sve2_sabal_h
,
7046 gen_helper_sve2_sabal_s
, gen_helper_sve2_sabal_d
},
7047 { NULL
, gen_helper_sve2_uabal_h
,
7048 gen_helper_sve2_uabal_s
, gen_helper_sve2_uabal_d
},
7050 return do_sve2_zzzz_ool(s
, a
, fns
[uns
][a
->esz
], sel
);
7053 static bool trans_SABALB(DisasContext
*s
, arg_rrrr_esz
*a
)
7055 return do_abal(s
, a
, false, false);
7058 static bool trans_SABALT(DisasContext
*s
, arg_rrrr_esz
*a
)
7060 return do_abal(s
, a
, false, true);
7063 static bool trans_UABALB(DisasContext
*s
, arg_rrrr_esz
*a
)
7065 return do_abal(s
, a
, true, false);
7068 static bool trans_UABALT(DisasContext
*s
, arg_rrrr_esz
*a
)
7070 return do_abal(s
, a
, true, true);
7073 static bool do_adcl(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
7075 static gen_helper_gvec_4
* const fns
[2] = {
7076 gen_helper_sve2_adcl_s
,
7077 gen_helper_sve2_adcl_d
,
7080 * Note that in this case the ESZ field encodes both size and sign.
7081 * Split out 'subtract' into bit 1 of the data field for the helper.
7083 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
& 1], (a
->esz
& 2) | sel
);
7086 static bool trans_ADCLB(DisasContext
*s
, arg_rrrr_esz
*a
)
7088 return do_adcl(s
, a
, false);
7091 static bool trans_ADCLT(DisasContext
*s
, arg_rrrr_esz
*a
)
7093 return do_adcl(s
, a
, true);
7096 static bool do_sve2_fn2i(DisasContext
*s
, arg_rri_esz
*a
, GVecGen2iFn
*fn
)
7098 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
7101 if (sve_access_check(s
)) {
7102 unsigned vsz
= vec_full_reg_size(s
);
7103 unsigned rd_ofs
= vec_full_reg_offset(s
, a
->rd
);
7104 unsigned rn_ofs
= vec_full_reg_offset(s
, a
->rn
);
7105 fn(a
->esz
, rd_ofs
, rn_ofs
, a
->imm
, vsz
, vsz
);
7110 static bool trans_SSRA(DisasContext
*s
, arg_rri_esz
*a
)
7112 return do_sve2_fn2i(s
, a
, gen_gvec_ssra
);
7115 static bool trans_USRA(DisasContext
*s
, arg_rri_esz
*a
)
7117 return do_sve2_fn2i(s
, a
, gen_gvec_usra
);
7120 static bool trans_SRSRA(DisasContext
*s
, arg_rri_esz
*a
)
7122 return do_sve2_fn2i(s
, a
, gen_gvec_srsra
);
7125 static bool trans_URSRA(DisasContext
*s
, arg_rri_esz
*a
)
7127 return do_sve2_fn2i(s
, a
, gen_gvec_ursra
);
7130 static bool trans_SRI(DisasContext
*s
, arg_rri_esz
*a
)
7132 return do_sve2_fn2i(s
, a
, gen_gvec_sri
);
7135 static bool trans_SLI(DisasContext
*s
, arg_rri_esz
*a
)
7137 return do_sve2_fn2i(s
, a
, gen_gvec_sli
);
7140 static bool do_sve2_fn_zzz(DisasContext
*s
, arg_rrr_esz
*a
, GVecGen3Fn
*fn
)
7142 if (!dc_isar_feature(aa64_sve2
, s
)) {
7145 if (sve_access_check(s
)) {
7146 gen_gvec_fn_zzz(s
, fn
, a
->esz
, a
->rd
, a
->rn
, a
->rm
);
7151 static bool trans_SABA(DisasContext
*s
, arg_rrr_esz
*a
)
7153 return do_sve2_fn_zzz(s
, a
, gen_gvec_saba
);
7156 static bool trans_UABA(DisasContext
*s
, arg_rrr_esz
*a
)
7158 return do_sve2_fn_zzz(s
, a
, gen_gvec_uaba
);
7161 static bool do_sve2_narrow_extract(DisasContext
*s
, arg_rri_esz
*a
,
7162 const GVecGen2 ops
[3])
7164 if (a
->esz
< 0 || a
->esz
> MO_32
|| a
->imm
!= 0 ||
7165 !dc_isar_feature(aa64_sve2
, s
)) {
7168 if (sve_access_check(s
)) {
7169 unsigned vsz
= vec_full_reg_size(s
);
7170 tcg_gen_gvec_2(vec_full_reg_offset(s
, a
->rd
),
7171 vec_full_reg_offset(s
, a
->rn
),
7172 vsz
, vsz
, &ops
[a
->esz
]);
7177 static const TCGOpcode sqxtn_list
[] = {
7178 INDEX_op_shli_vec
, INDEX_op_smin_vec
, INDEX_op_smax_vec
, 0
7181 static void gen_sqxtnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7183 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7184 int halfbits
= 4 << vece
;
7185 int64_t mask
= (1ull << halfbits
) - 1;
7186 int64_t min
= -1ull << (halfbits
- 1);
7187 int64_t max
= -min
- 1;
7189 tcg_gen_dupi_vec(vece
, t
, min
);
7190 tcg_gen_smax_vec(vece
, d
, n
, t
);
7191 tcg_gen_dupi_vec(vece
, t
, max
);
7192 tcg_gen_smin_vec(vece
, d
, d
, t
);
7193 tcg_gen_dupi_vec(vece
, t
, mask
);
7194 tcg_gen_and_vec(vece
, d
, d
, t
);
7195 tcg_temp_free_vec(t
);
7198 static bool trans_SQXTNB(DisasContext
*s
, arg_rri_esz
*a
)
7200 static const GVecGen2 ops
[3] = {
7201 { .fniv
= gen_sqxtnb_vec
,
7202 .opt_opc
= sqxtn_list
,
7203 .fno
= gen_helper_sve2_sqxtnb_h
,
7205 { .fniv
= gen_sqxtnb_vec
,
7206 .opt_opc
= sqxtn_list
,
7207 .fno
= gen_helper_sve2_sqxtnb_s
,
7209 { .fniv
= gen_sqxtnb_vec
,
7210 .opt_opc
= sqxtn_list
,
7211 .fno
= gen_helper_sve2_sqxtnb_d
,
7214 return do_sve2_narrow_extract(s
, a
, ops
);
7217 static void gen_sqxtnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7219 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7220 int halfbits
= 4 << vece
;
7221 int64_t mask
= (1ull << halfbits
) - 1;
7222 int64_t min
= -1ull << (halfbits
- 1);
7223 int64_t max
= -min
- 1;
7225 tcg_gen_dupi_vec(vece
, t
, min
);
7226 tcg_gen_smax_vec(vece
, n
, n
, t
);
7227 tcg_gen_dupi_vec(vece
, t
, max
);
7228 tcg_gen_smin_vec(vece
, n
, n
, t
);
7229 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7230 tcg_gen_dupi_vec(vece
, t
, mask
);
7231 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7232 tcg_temp_free_vec(t
);
7235 static bool trans_SQXTNT(DisasContext
*s
, arg_rri_esz
*a
)
7237 static const GVecGen2 ops
[3] = {
7238 { .fniv
= gen_sqxtnt_vec
,
7239 .opt_opc
= sqxtn_list
,
7241 .fno
= gen_helper_sve2_sqxtnt_h
,
7243 { .fniv
= gen_sqxtnt_vec
,
7244 .opt_opc
= sqxtn_list
,
7246 .fno
= gen_helper_sve2_sqxtnt_s
,
7248 { .fniv
= gen_sqxtnt_vec
,
7249 .opt_opc
= sqxtn_list
,
7251 .fno
= gen_helper_sve2_sqxtnt_d
,
7254 return do_sve2_narrow_extract(s
, a
, ops
);
7257 static const TCGOpcode uqxtn_list
[] = {
7258 INDEX_op_shli_vec
, INDEX_op_umin_vec
, 0
7261 static void gen_uqxtnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7263 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7264 int halfbits
= 4 << vece
;
7265 int64_t max
= (1ull << halfbits
) - 1;
7267 tcg_gen_dupi_vec(vece
, t
, max
);
7268 tcg_gen_umin_vec(vece
, d
, n
, t
);
7269 tcg_temp_free_vec(t
);
7272 static bool trans_UQXTNB(DisasContext
*s
, arg_rri_esz
*a
)
7274 static const GVecGen2 ops
[3] = {
7275 { .fniv
= gen_uqxtnb_vec
,
7276 .opt_opc
= uqxtn_list
,
7277 .fno
= gen_helper_sve2_uqxtnb_h
,
7279 { .fniv
= gen_uqxtnb_vec
,
7280 .opt_opc
= uqxtn_list
,
7281 .fno
= gen_helper_sve2_uqxtnb_s
,
7283 { .fniv
= gen_uqxtnb_vec
,
7284 .opt_opc
= uqxtn_list
,
7285 .fno
= gen_helper_sve2_uqxtnb_d
,
7288 return do_sve2_narrow_extract(s
, a
, ops
);
7291 static void gen_uqxtnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7293 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7294 int halfbits
= 4 << vece
;
7295 int64_t max
= (1ull << halfbits
) - 1;
7297 tcg_gen_dupi_vec(vece
, t
, max
);
7298 tcg_gen_umin_vec(vece
, n
, n
, t
);
7299 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7300 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7301 tcg_temp_free_vec(t
);
7304 static bool trans_UQXTNT(DisasContext
*s
, arg_rri_esz
*a
)
7306 static const GVecGen2 ops
[3] = {
7307 { .fniv
= gen_uqxtnt_vec
,
7308 .opt_opc
= uqxtn_list
,
7310 .fno
= gen_helper_sve2_uqxtnt_h
,
7312 { .fniv
= gen_uqxtnt_vec
,
7313 .opt_opc
= uqxtn_list
,
7315 .fno
= gen_helper_sve2_uqxtnt_s
,
7317 { .fniv
= gen_uqxtnt_vec
,
7318 .opt_opc
= uqxtn_list
,
7320 .fno
= gen_helper_sve2_uqxtnt_d
,
7323 return do_sve2_narrow_extract(s
, a
, ops
);
7326 static const TCGOpcode sqxtun_list
[] = {
7327 INDEX_op_shli_vec
, INDEX_op_umin_vec
, INDEX_op_smax_vec
, 0
7330 static void gen_sqxtunb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7332 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7333 int halfbits
= 4 << vece
;
7334 int64_t max
= (1ull << halfbits
) - 1;
7336 tcg_gen_dupi_vec(vece
, t
, 0);
7337 tcg_gen_smax_vec(vece
, d
, n
, t
);
7338 tcg_gen_dupi_vec(vece
, t
, max
);
7339 tcg_gen_umin_vec(vece
, d
, d
, t
);
7340 tcg_temp_free_vec(t
);
7343 static bool trans_SQXTUNB(DisasContext
*s
, arg_rri_esz
*a
)
7345 static const GVecGen2 ops
[3] = {
7346 { .fniv
= gen_sqxtunb_vec
,
7347 .opt_opc
= sqxtun_list
,
7348 .fno
= gen_helper_sve2_sqxtunb_h
,
7350 { .fniv
= gen_sqxtunb_vec
,
7351 .opt_opc
= sqxtun_list
,
7352 .fno
= gen_helper_sve2_sqxtunb_s
,
7354 { .fniv
= gen_sqxtunb_vec
,
7355 .opt_opc
= sqxtun_list
,
7356 .fno
= gen_helper_sve2_sqxtunb_d
,
7359 return do_sve2_narrow_extract(s
, a
, ops
);
7362 static void gen_sqxtunt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
)
7364 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7365 int halfbits
= 4 << vece
;
7366 int64_t max
= (1ull << halfbits
) - 1;
7368 tcg_gen_dupi_vec(vece
, t
, 0);
7369 tcg_gen_smax_vec(vece
, n
, n
, t
);
7370 tcg_gen_dupi_vec(vece
, t
, max
);
7371 tcg_gen_umin_vec(vece
, n
, n
, t
);
7372 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7373 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7374 tcg_temp_free_vec(t
);
7377 static bool trans_SQXTUNT(DisasContext
*s
, arg_rri_esz
*a
)
7379 static const GVecGen2 ops
[3] = {
7380 { .fniv
= gen_sqxtunt_vec
,
7381 .opt_opc
= sqxtun_list
,
7383 .fno
= gen_helper_sve2_sqxtunt_h
,
7385 { .fniv
= gen_sqxtunt_vec
,
7386 .opt_opc
= sqxtun_list
,
7388 .fno
= gen_helper_sve2_sqxtunt_s
,
7390 { .fniv
= gen_sqxtunt_vec
,
7391 .opt_opc
= sqxtun_list
,
7393 .fno
= gen_helper_sve2_sqxtunt_d
,
7396 return do_sve2_narrow_extract(s
, a
, ops
);
7399 static bool do_sve2_shr_narrow(DisasContext
*s
, arg_rri_esz
*a
,
7400 const GVecGen2i ops
[3])
7402 if (a
->esz
< 0 || a
->esz
> MO_32
|| !dc_isar_feature(aa64_sve2
, s
)) {
7405 assert(a
->imm
> 0 && a
->imm
<= (8 << a
->esz
));
7406 if (sve_access_check(s
)) {
7407 unsigned vsz
= vec_full_reg_size(s
);
7408 tcg_gen_gvec_2i(vec_full_reg_offset(s
, a
->rd
),
7409 vec_full_reg_offset(s
, a
->rn
),
7410 vsz
, vsz
, a
->imm
, &ops
[a
->esz
]);
7415 static void gen_shrnb_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int shr
)
7417 int halfbits
= 4 << vece
;
7418 uint64_t mask
= dup_const(vece
, MAKE_64BIT_MASK(0, halfbits
));
7420 tcg_gen_shri_i64(d
, n
, shr
);
7421 tcg_gen_andi_i64(d
, d
, mask
);
7424 static void gen_shrnb16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7426 gen_shrnb_i64(MO_16
, d
, n
, shr
);
7429 static void gen_shrnb32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7431 gen_shrnb_i64(MO_32
, d
, n
, shr
);
7434 static void gen_shrnb64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7436 gen_shrnb_i64(MO_64
, d
, n
, shr
);
7439 static void gen_shrnb_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t shr
)
7441 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7442 int halfbits
= 4 << vece
;
7443 uint64_t mask
= MAKE_64BIT_MASK(0, halfbits
);
7445 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7446 tcg_gen_dupi_vec(vece
, t
, mask
);
7447 tcg_gen_and_vec(vece
, d
, n
, t
);
7448 tcg_temp_free_vec(t
);
7451 static bool trans_SHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7453 static const TCGOpcode vec_list
[] = { INDEX_op_shri_vec
, 0 };
7454 static const GVecGen2i ops
[3] = {
7455 { .fni8
= gen_shrnb16_i64
,
7456 .fniv
= gen_shrnb_vec
,
7457 .opt_opc
= vec_list
,
7458 .fno
= gen_helper_sve2_shrnb_h
,
7460 { .fni8
= gen_shrnb32_i64
,
7461 .fniv
= gen_shrnb_vec
,
7462 .opt_opc
= vec_list
,
7463 .fno
= gen_helper_sve2_shrnb_s
,
7465 { .fni8
= gen_shrnb64_i64
,
7466 .fniv
= gen_shrnb_vec
,
7467 .opt_opc
= vec_list
,
7468 .fno
= gen_helper_sve2_shrnb_d
,
7471 return do_sve2_shr_narrow(s
, a
, ops
);
7474 static void gen_shrnt_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int shr
)
7476 int halfbits
= 4 << vece
;
7477 uint64_t mask
= dup_const(vece
, MAKE_64BIT_MASK(0, halfbits
));
7479 tcg_gen_shli_i64(n
, n
, halfbits
- shr
);
7480 tcg_gen_andi_i64(n
, n
, ~mask
);
7481 tcg_gen_andi_i64(d
, d
, mask
);
7482 tcg_gen_or_i64(d
, d
, n
);
7485 static void gen_shrnt16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7487 gen_shrnt_i64(MO_16
, d
, n
, shr
);
7490 static void gen_shrnt32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7492 gen_shrnt_i64(MO_32
, d
, n
, shr
);
7495 static void gen_shrnt64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t shr
)
7497 tcg_gen_shri_i64(n
, n
, shr
);
7498 tcg_gen_deposit_i64(d
, d
, n
, 32, 32);
7501 static void gen_shrnt_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t shr
)
7503 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7504 int halfbits
= 4 << vece
;
7505 uint64_t mask
= MAKE_64BIT_MASK(0, halfbits
);
7507 tcg_gen_shli_vec(vece
, n
, n
, halfbits
- shr
);
7508 tcg_gen_dupi_vec(vece
, t
, mask
);
7509 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7510 tcg_temp_free_vec(t
);
7513 static bool trans_SHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7515 static const TCGOpcode vec_list
[] = { INDEX_op_shli_vec
, 0 };
7516 static const GVecGen2i ops
[3] = {
7517 { .fni8
= gen_shrnt16_i64
,
7518 .fniv
= gen_shrnt_vec
,
7519 .opt_opc
= vec_list
,
7521 .fno
= gen_helper_sve2_shrnt_h
,
7523 { .fni8
= gen_shrnt32_i64
,
7524 .fniv
= gen_shrnt_vec
,
7525 .opt_opc
= vec_list
,
7527 .fno
= gen_helper_sve2_shrnt_s
,
7529 { .fni8
= gen_shrnt64_i64
,
7530 .fniv
= gen_shrnt_vec
,
7531 .opt_opc
= vec_list
,
7533 .fno
= gen_helper_sve2_shrnt_d
,
7536 return do_sve2_shr_narrow(s
, a
, ops
);
7539 static bool trans_RSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7541 static const GVecGen2i ops
[3] = {
7542 { .fno
= gen_helper_sve2_rshrnb_h
},
7543 { .fno
= gen_helper_sve2_rshrnb_s
},
7544 { .fno
= gen_helper_sve2_rshrnb_d
},
7546 return do_sve2_shr_narrow(s
, a
, ops
);
7549 static bool trans_RSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7551 static const GVecGen2i ops
[3] = {
7552 { .fno
= gen_helper_sve2_rshrnt_h
},
7553 { .fno
= gen_helper_sve2_rshrnt_s
},
7554 { .fno
= gen_helper_sve2_rshrnt_d
},
7556 return do_sve2_shr_narrow(s
, a
, ops
);
7559 static void gen_sqshrunb_vec(unsigned vece
, TCGv_vec d
,
7560 TCGv_vec n
, int64_t shr
)
7562 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7563 int halfbits
= 4 << vece
;
7565 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7566 tcg_gen_dupi_vec(vece
, t
, 0);
7567 tcg_gen_smax_vec(vece
, n
, n
, t
);
7568 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7569 tcg_gen_umin_vec(vece
, d
, n
, t
);
7570 tcg_temp_free_vec(t
);
7573 static bool trans_SQSHRUNB(DisasContext
*s
, arg_rri_esz
*a
)
7575 static const TCGOpcode vec_list
[] = {
7576 INDEX_op_sari_vec
, INDEX_op_smax_vec
, INDEX_op_umin_vec
, 0
7578 static const GVecGen2i ops
[3] = {
7579 { .fniv
= gen_sqshrunb_vec
,
7580 .opt_opc
= vec_list
,
7581 .fno
= gen_helper_sve2_sqshrunb_h
,
7583 { .fniv
= gen_sqshrunb_vec
,
7584 .opt_opc
= vec_list
,
7585 .fno
= gen_helper_sve2_sqshrunb_s
,
7587 { .fniv
= gen_sqshrunb_vec
,
7588 .opt_opc
= vec_list
,
7589 .fno
= gen_helper_sve2_sqshrunb_d
,
7592 return do_sve2_shr_narrow(s
, a
, ops
);
7595 static void gen_sqshrunt_vec(unsigned vece
, TCGv_vec d
,
7596 TCGv_vec n
, int64_t shr
)
7598 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7599 int halfbits
= 4 << vece
;
7601 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7602 tcg_gen_dupi_vec(vece
, t
, 0);
7603 tcg_gen_smax_vec(vece
, n
, n
, t
);
7604 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7605 tcg_gen_umin_vec(vece
, n
, n
, t
);
7606 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7607 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7608 tcg_temp_free_vec(t
);
7611 static bool trans_SQSHRUNT(DisasContext
*s
, arg_rri_esz
*a
)
7613 static const TCGOpcode vec_list
[] = {
7614 INDEX_op_shli_vec
, INDEX_op_sari_vec
,
7615 INDEX_op_smax_vec
, INDEX_op_umin_vec
, 0
7617 static const GVecGen2i ops
[3] = {
7618 { .fniv
= gen_sqshrunt_vec
,
7619 .opt_opc
= vec_list
,
7621 .fno
= gen_helper_sve2_sqshrunt_h
,
7623 { .fniv
= gen_sqshrunt_vec
,
7624 .opt_opc
= vec_list
,
7626 .fno
= gen_helper_sve2_sqshrunt_s
,
7628 { .fniv
= gen_sqshrunt_vec
,
7629 .opt_opc
= vec_list
,
7631 .fno
= gen_helper_sve2_sqshrunt_d
,
7634 return do_sve2_shr_narrow(s
, a
, ops
);
7637 static bool trans_SQRSHRUNB(DisasContext
*s
, arg_rri_esz
*a
)
7639 static const GVecGen2i ops
[3] = {
7640 { .fno
= gen_helper_sve2_sqrshrunb_h
},
7641 { .fno
= gen_helper_sve2_sqrshrunb_s
},
7642 { .fno
= gen_helper_sve2_sqrshrunb_d
},
7644 return do_sve2_shr_narrow(s
, a
, ops
);
7647 static bool trans_SQRSHRUNT(DisasContext
*s
, arg_rri_esz
*a
)
7649 static const GVecGen2i ops
[3] = {
7650 { .fno
= gen_helper_sve2_sqrshrunt_h
},
7651 { .fno
= gen_helper_sve2_sqrshrunt_s
},
7652 { .fno
= gen_helper_sve2_sqrshrunt_d
},
7654 return do_sve2_shr_narrow(s
, a
, ops
);
7657 static void gen_sqshrnb_vec(unsigned vece
, TCGv_vec d
,
7658 TCGv_vec n
, int64_t shr
)
7660 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7661 int halfbits
= 4 << vece
;
7662 int64_t max
= MAKE_64BIT_MASK(0, halfbits
- 1);
7663 int64_t min
= -max
- 1;
7665 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7666 tcg_gen_dupi_vec(vece
, t
, min
);
7667 tcg_gen_smax_vec(vece
, n
, n
, t
);
7668 tcg_gen_dupi_vec(vece
, t
, max
);
7669 tcg_gen_smin_vec(vece
, n
, n
, t
);
7670 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7671 tcg_gen_and_vec(vece
, d
, n
, t
);
7672 tcg_temp_free_vec(t
);
7675 static bool trans_SQSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7677 static const TCGOpcode vec_list
[] = {
7678 INDEX_op_sari_vec
, INDEX_op_smax_vec
, INDEX_op_smin_vec
, 0
7680 static const GVecGen2i ops
[3] = {
7681 { .fniv
= gen_sqshrnb_vec
,
7682 .opt_opc
= vec_list
,
7683 .fno
= gen_helper_sve2_sqshrnb_h
,
7685 { .fniv
= gen_sqshrnb_vec
,
7686 .opt_opc
= vec_list
,
7687 .fno
= gen_helper_sve2_sqshrnb_s
,
7689 { .fniv
= gen_sqshrnb_vec
,
7690 .opt_opc
= vec_list
,
7691 .fno
= gen_helper_sve2_sqshrnb_d
,
7694 return do_sve2_shr_narrow(s
, a
, ops
);
7697 static void gen_sqshrnt_vec(unsigned vece
, TCGv_vec d
,
7698 TCGv_vec n
, int64_t shr
)
7700 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7701 int halfbits
= 4 << vece
;
7702 int64_t max
= MAKE_64BIT_MASK(0, halfbits
- 1);
7703 int64_t min
= -max
- 1;
7705 tcg_gen_sari_vec(vece
, n
, n
, shr
);
7706 tcg_gen_dupi_vec(vece
, t
, min
);
7707 tcg_gen_smax_vec(vece
, n
, n
, t
);
7708 tcg_gen_dupi_vec(vece
, t
, max
);
7709 tcg_gen_smin_vec(vece
, n
, n
, t
);
7710 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7711 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7712 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7713 tcg_temp_free_vec(t
);
7716 static bool trans_SQSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7718 static const TCGOpcode vec_list
[] = {
7719 INDEX_op_shli_vec
, INDEX_op_sari_vec
,
7720 INDEX_op_smax_vec
, INDEX_op_smin_vec
, 0
7722 static const GVecGen2i ops
[3] = {
7723 { .fniv
= gen_sqshrnt_vec
,
7724 .opt_opc
= vec_list
,
7726 .fno
= gen_helper_sve2_sqshrnt_h
,
7728 { .fniv
= gen_sqshrnt_vec
,
7729 .opt_opc
= vec_list
,
7731 .fno
= gen_helper_sve2_sqshrnt_s
,
7733 { .fniv
= gen_sqshrnt_vec
,
7734 .opt_opc
= vec_list
,
7736 .fno
= gen_helper_sve2_sqshrnt_d
,
7739 return do_sve2_shr_narrow(s
, a
, ops
);
7742 static bool trans_SQRSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7744 static const GVecGen2i ops
[3] = {
7745 { .fno
= gen_helper_sve2_sqrshrnb_h
},
7746 { .fno
= gen_helper_sve2_sqrshrnb_s
},
7747 { .fno
= gen_helper_sve2_sqrshrnb_d
},
7749 return do_sve2_shr_narrow(s
, a
, ops
);
7752 static bool trans_SQRSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7754 static const GVecGen2i ops
[3] = {
7755 { .fno
= gen_helper_sve2_sqrshrnt_h
},
7756 { .fno
= gen_helper_sve2_sqrshrnt_s
},
7757 { .fno
= gen_helper_sve2_sqrshrnt_d
},
7759 return do_sve2_shr_narrow(s
, a
, ops
);
7762 static void gen_uqshrnb_vec(unsigned vece
, TCGv_vec d
,
7763 TCGv_vec n
, int64_t shr
)
7765 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7766 int halfbits
= 4 << vece
;
7768 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7769 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7770 tcg_gen_umin_vec(vece
, d
, n
, t
);
7771 tcg_temp_free_vec(t
);
7774 static bool trans_UQSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7776 static const TCGOpcode vec_list
[] = {
7777 INDEX_op_shri_vec
, INDEX_op_umin_vec
, 0
7779 static const GVecGen2i ops
[3] = {
7780 { .fniv
= gen_uqshrnb_vec
,
7781 .opt_opc
= vec_list
,
7782 .fno
= gen_helper_sve2_uqshrnb_h
,
7784 { .fniv
= gen_uqshrnb_vec
,
7785 .opt_opc
= vec_list
,
7786 .fno
= gen_helper_sve2_uqshrnb_s
,
7788 { .fniv
= gen_uqshrnb_vec
,
7789 .opt_opc
= vec_list
,
7790 .fno
= gen_helper_sve2_uqshrnb_d
,
7793 return do_sve2_shr_narrow(s
, a
, ops
);
7796 static void gen_uqshrnt_vec(unsigned vece
, TCGv_vec d
,
7797 TCGv_vec n
, int64_t shr
)
7799 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
7800 int halfbits
= 4 << vece
;
7802 tcg_gen_shri_vec(vece
, n
, n
, shr
);
7803 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
7804 tcg_gen_umin_vec(vece
, n
, n
, t
);
7805 tcg_gen_shli_vec(vece
, n
, n
, halfbits
);
7806 tcg_gen_bitsel_vec(vece
, d
, t
, d
, n
);
7807 tcg_temp_free_vec(t
);
7810 static bool trans_UQSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7812 static const TCGOpcode vec_list
[] = {
7813 INDEX_op_shli_vec
, INDEX_op_shri_vec
, INDEX_op_umin_vec
, 0
7815 static const GVecGen2i ops
[3] = {
7816 { .fniv
= gen_uqshrnt_vec
,
7817 .opt_opc
= vec_list
,
7819 .fno
= gen_helper_sve2_uqshrnt_h
,
7821 { .fniv
= gen_uqshrnt_vec
,
7822 .opt_opc
= vec_list
,
7824 .fno
= gen_helper_sve2_uqshrnt_s
,
7826 { .fniv
= gen_uqshrnt_vec
,
7827 .opt_opc
= vec_list
,
7829 .fno
= gen_helper_sve2_uqshrnt_d
,
7832 return do_sve2_shr_narrow(s
, a
, ops
);
7835 static bool trans_UQRSHRNB(DisasContext
*s
, arg_rri_esz
*a
)
7837 static const GVecGen2i ops
[3] = {
7838 { .fno
= gen_helper_sve2_uqrshrnb_h
},
7839 { .fno
= gen_helper_sve2_uqrshrnb_s
},
7840 { .fno
= gen_helper_sve2_uqrshrnb_d
},
7842 return do_sve2_shr_narrow(s
, a
, ops
);
7845 static bool trans_UQRSHRNT(DisasContext
*s
, arg_rri_esz
*a
)
7847 static const GVecGen2i ops
[3] = {
7848 { .fno
= gen_helper_sve2_uqrshrnt_h
},
7849 { .fno
= gen_helper_sve2_uqrshrnt_s
},
7850 { .fno
= gen_helper_sve2_uqrshrnt_d
},
7852 return do_sve2_shr_narrow(s
, a
, ops
);
7855 #define DO_SVE2_ZZZ_NARROW(NAME, name) \
7856 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
7858 static gen_helper_gvec_3 * const fns[4] = { \
7859 NULL, gen_helper_sve2_##name##_h, \
7860 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
7862 return do_sve2_zzz_ool(s, a, fns[a->esz]); \
7865 DO_SVE2_ZZZ_NARROW(ADDHNB
, addhnb
)
7866 DO_SVE2_ZZZ_NARROW(ADDHNT
, addhnt
)
7867 DO_SVE2_ZZZ_NARROW(RADDHNB
, raddhnb
)
7868 DO_SVE2_ZZZ_NARROW(RADDHNT
, raddhnt
)
7870 DO_SVE2_ZZZ_NARROW(SUBHNB
, subhnb
)
7871 DO_SVE2_ZZZ_NARROW(SUBHNT
, subhnt
)
7872 DO_SVE2_ZZZ_NARROW(RSUBHNB
, rsubhnb
)
7873 DO_SVE2_ZZZ_NARROW(RSUBHNT
, rsubhnt
)
7875 static bool do_sve2_ppzz_flags(DisasContext
*s
, arg_rprr_esz
*a
,
7876 gen_helper_gvec_flags_4
*fn
)
7878 if (!dc_isar_feature(aa64_sve2
, s
)) {
7881 return do_ppzz_flags(s
, a
, fn
);
7884 #define DO_SVE2_PPZZ_MATCH(NAME, name) \
7885 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
7887 static gen_helper_gvec_flags_4 * const fns[4] = { \
7888 gen_helper_sve2_##name##_ppzz_b, gen_helper_sve2_##name##_ppzz_h, \
7891 return do_sve2_ppzz_flags(s, a, fns[a->esz]); \
7894 DO_SVE2_PPZZ_MATCH(MATCH
, match
)
7895 DO_SVE2_PPZZ_MATCH(NMATCH
, nmatch
)
7897 static bool trans_HISTCNT(DisasContext
*s
, arg_rprr_esz
*a
)
7899 static gen_helper_gvec_4
* const fns
[2] = {
7900 gen_helper_sve2_histcnt_s
, gen_helper_sve2_histcnt_d
7905 return do_sve2_zpzz_ool(s
, a
, fns
[a
->esz
- 2]);
7908 static bool trans_HISTSEG(DisasContext
*s
, arg_rrr_esz
*a
)
7913 return do_sve2_zzz_ool(s
, a
, gen_helper_sve2_histseg
);
7916 static bool do_sve2_zpzz_fp(DisasContext
*s
, arg_rprr_esz
*a
,
7917 gen_helper_gvec_4_ptr
*fn
)
7919 if (!dc_isar_feature(aa64_sve2
, s
)) {
7922 return do_zpzz_fp(s
, a
, fn
);
7925 #define DO_SVE2_ZPZZ_FP(NAME, name) \
7926 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
7928 static gen_helper_gvec_4_ptr * const fns[4] = { \
7929 NULL, gen_helper_sve2_##name##_zpzz_h, \
7930 gen_helper_sve2_##name##_zpzz_s, gen_helper_sve2_##name##_zpzz_d \
7932 return do_sve2_zpzz_fp(s, a, fns[a->esz]); \
7935 DO_SVE2_ZPZZ_FP(FADDP
, faddp
)
7936 DO_SVE2_ZPZZ_FP(FMAXNMP
, fmaxnmp
)
7937 DO_SVE2_ZPZZ_FP(FMINNMP
, fminnmp
)
7938 DO_SVE2_ZPZZ_FP(FMAXP
, fmaxp
)
7939 DO_SVE2_ZPZZ_FP(FMINP
, fminp
)
7942 * SVE Integer Multiply-Add (unpredicated)
7945 static bool trans_FMMLA(DisasContext
*s
, arg_rrrr_esz
*a
)
7947 gen_helper_gvec_4_ptr
*fn
;
7951 if (!dc_isar_feature(aa64_sve_f32mm
, s
)) {
7954 fn
= gen_helper_fmmla_s
;
7957 if (!dc_isar_feature(aa64_sve_f64mm
, s
)) {
7960 fn
= gen_helper_fmmla_d
;
7966 if (sve_access_check(s
)) {
7967 unsigned vsz
= vec_full_reg_size(s
);
7968 TCGv_ptr status
= fpstatus_ptr(FPST_FPCR
);
7969 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
7970 vec_full_reg_offset(s
, a
->rn
),
7971 vec_full_reg_offset(s
, a
->rm
),
7972 vec_full_reg_offset(s
, a
->ra
),
7973 status
, vsz
, vsz
, 0, fn
);
7974 tcg_temp_free_ptr(status
);
7979 static bool do_sqdmlal_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
,
7980 bool sel1
, bool sel2
)
7982 static gen_helper_gvec_4
* const fns
[] = {
7983 NULL
, gen_helper_sve2_sqdmlal_zzzw_h
,
7984 gen_helper_sve2_sqdmlal_zzzw_s
, gen_helper_sve2_sqdmlal_zzzw_d
,
7986 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], (sel2
<< 1) | sel1
);
7989 static bool do_sqdmlsl_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
,
7990 bool sel1
, bool sel2
)
7992 static gen_helper_gvec_4
* const fns
[] = {
7993 NULL
, gen_helper_sve2_sqdmlsl_zzzw_h
,
7994 gen_helper_sve2_sqdmlsl_zzzw_s
, gen_helper_sve2_sqdmlsl_zzzw_d
,
7996 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], (sel2
<< 1) | sel1
);
7999 static bool trans_SQDMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8001 return do_sqdmlal_zzzw(s
, a
, false, false);
8004 static bool trans_SQDMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8006 return do_sqdmlal_zzzw(s
, a
, true, true);
8009 static bool trans_SQDMLALBT(DisasContext
*s
, arg_rrrr_esz
*a
)
8011 return do_sqdmlal_zzzw(s
, a
, false, true);
8014 static bool trans_SQDMLSLB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8016 return do_sqdmlsl_zzzw(s
, a
, false, false);
8019 static bool trans_SQDMLSLT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8021 return do_sqdmlsl_zzzw(s
, a
, true, true);
8024 static bool trans_SQDMLSLBT(DisasContext
*s
, arg_rrrr_esz
*a
)
8026 return do_sqdmlsl_zzzw(s
, a
, false, true);
8029 static bool trans_SQRDMLAH_zzzz(DisasContext
*s
, arg_rrrr_esz
*a
)
8031 static gen_helper_gvec_4
* const fns
[] = {
8032 gen_helper_sve2_sqrdmlah_b
, gen_helper_sve2_sqrdmlah_h
,
8033 gen_helper_sve2_sqrdmlah_s
, gen_helper_sve2_sqrdmlah_d
,
8035 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], 0);
8038 static bool trans_SQRDMLSH_zzzz(DisasContext
*s
, arg_rrrr_esz
*a
)
8040 static gen_helper_gvec_4
* const fns
[] = {
8041 gen_helper_sve2_sqrdmlsh_b
, gen_helper_sve2_sqrdmlsh_h
,
8042 gen_helper_sve2_sqrdmlsh_s
, gen_helper_sve2_sqrdmlsh_d
,
8044 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], 0);
8047 static bool do_smlal_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
8049 static gen_helper_gvec_4
* const fns
[] = {
8050 NULL
, gen_helper_sve2_smlal_zzzw_h
,
8051 gen_helper_sve2_smlal_zzzw_s
, gen_helper_sve2_smlal_zzzw_d
,
8053 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], sel
);
8056 static bool trans_SMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8058 return do_smlal_zzzw(s
, a
, false);
8061 static bool trans_SMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8063 return do_smlal_zzzw(s
, a
, true);
8066 static bool do_umlal_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
8068 static gen_helper_gvec_4
* const fns
[] = {
8069 NULL
, gen_helper_sve2_umlal_zzzw_h
,
8070 gen_helper_sve2_umlal_zzzw_s
, gen_helper_sve2_umlal_zzzw_d
,
8072 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], sel
);
8075 static bool trans_UMLALB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8077 return do_umlal_zzzw(s
, a
, false);
8080 static bool trans_UMLALT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8082 return do_umlal_zzzw(s
, a
, true);
8085 static bool do_smlsl_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
8087 static gen_helper_gvec_4
* const fns
[] = {
8088 NULL
, gen_helper_sve2_smlsl_zzzw_h
,
8089 gen_helper_sve2_smlsl_zzzw_s
, gen_helper_sve2_smlsl_zzzw_d
,
8091 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], sel
);
8094 static bool trans_SMLSLB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8096 return do_smlsl_zzzw(s
, a
, false);
8099 static bool trans_SMLSLT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8101 return do_smlsl_zzzw(s
, a
, true);
8104 static bool do_umlsl_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
8106 static gen_helper_gvec_4
* const fns
[] = {
8107 NULL
, gen_helper_sve2_umlsl_zzzw_h
,
8108 gen_helper_sve2_umlsl_zzzw_s
, gen_helper_sve2_umlsl_zzzw_d
,
8110 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
], sel
);
8113 static bool trans_UMLSLB_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8115 return do_umlsl_zzzw(s
, a
, false);
8118 static bool trans_UMLSLT_zzzw(DisasContext
*s
, arg_rrrr_esz
*a
)
8120 return do_umlsl_zzzw(s
, a
, true);
8123 static bool trans_CMLA_zzzz(DisasContext
*s
, arg_CMLA_zzzz
*a
)
8125 static gen_helper_gvec_4
* const fns
[] = {
8126 gen_helper_sve2_cmla_zzzz_b
, gen_helper_sve2_cmla_zzzz_h
,
8127 gen_helper_sve2_cmla_zzzz_s
, gen_helper_sve2_cmla_zzzz_d
,
8130 if (!dc_isar_feature(aa64_sve2
, s
)) {
8133 if (sve_access_check(s
)) {
8134 gen_gvec_ool_zzzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
);
8139 static bool trans_CDOT_zzzz(DisasContext
*s
, arg_CMLA_zzzz
*a
)
8141 if (!dc_isar_feature(aa64_sve2
, s
) || a
->esz
< MO_32
) {
8144 if (sve_access_check(s
)) {
8145 gen_helper_gvec_4
*fn
= (a
->esz
== MO_32
8146 ? gen_helper_sve2_cdot_zzzz_s
8147 : gen_helper_sve2_cdot_zzzz_d
);
8148 gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
);
8153 static bool trans_SQRDCMLAH_zzzz(DisasContext
*s
, arg_SQRDCMLAH_zzzz
*a
)
8155 static gen_helper_gvec_4
* const fns
[] = {
8156 gen_helper_sve2_sqrdcmlah_zzzz_b
, gen_helper_sve2_sqrdcmlah_zzzz_h
,
8157 gen_helper_sve2_sqrdcmlah_zzzz_s
, gen_helper_sve2_sqrdcmlah_zzzz_d
,
8160 if (!dc_isar_feature(aa64_sve2
, s
)) {
8163 if (sve_access_check(s
)) {
8164 gen_gvec_ool_zzzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, a
->ra
, a
->rot
);
8169 static bool trans_USDOT_zzzz(DisasContext
*s
, arg_USDOT_zzzz
*a
)
8171 if (a
->esz
!= 2 || !dc_isar_feature(aa64_sve_i8mm
, s
)) {
8174 if (sve_access_check(s
)) {
8175 unsigned vsz
= vec_full_reg_size(s
);
8176 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, a
->rd
),
8177 vec_full_reg_offset(s
, a
->rn
),
8178 vec_full_reg_offset(s
, a
->rm
),
8179 vec_full_reg_offset(s
, a
->ra
),
8180 vsz
, vsz
, 0, gen_helper_gvec_usdot_b
);
8185 static bool trans_AESMC(DisasContext
*s
, arg_AESMC
*a
)
8187 if (!dc_isar_feature(aa64_sve2_aes
, s
)) {
8190 if (sve_access_check(s
)) {
8191 gen_gvec_ool_zz(s
, gen_helper_crypto_aesmc
, a
->rd
, a
->rd
, a
->decrypt
);
8196 static bool do_aese(DisasContext
*s
, arg_rrr_esz
*a
, bool decrypt
)
8198 if (!dc_isar_feature(aa64_sve2_aes
, s
)) {
8201 if (sve_access_check(s
)) {
8202 gen_gvec_ool_zzz(s
, gen_helper_crypto_aese
,
8203 a
->rd
, a
->rn
, a
->rm
, decrypt
);
8208 static bool trans_AESE(DisasContext
*s
, arg_rrr_esz
*a
)
8210 return do_aese(s
, a
, false);
8213 static bool trans_AESD(DisasContext
*s
, arg_rrr_esz
*a
)
8215 return do_aese(s
, a
, true);
8218 static bool do_sm4(DisasContext
*s
, arg_rrr_esz
*a
, gen_helper_gvec_3
*fn
)
8220 if (!dc_isar_feature(aa64_sve2_sm4
, s
)) {
8223 if (sve_access_check(s
)) {
8224 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, 0);
8229 static bool trans_SM4E(DisasContext
*s
, arg_rrr_esz
*a
)
8231 return do_sm4(s
, a
, gen_helper_crypto_sm4e
);
8234 static bool trans_SM4EKEY(DisasContext
*s
, arg_rrr_esz
*a
)
8236 return do_sm4(s
, a
, gen_helper_crypto_sm4ekey
);
8239 static bool trans_RAX1(DisasContext
*s
, arg_rrr_esz
*a
)
8241 if (!dc_isar_feature(aa64_sve2_sha3
, s
)) {
8244 if (sve_access_check(s
)) {
8245 gen_gvec_fn_zzz(s
, gen_gvec_rax1
, MO_64
, a
->rd
, a
->rn
, a
->rm
);
8250 static bool trans_FCVTNT_sh(DisasContext
*s
, arg_rpr_esz
*a
)
8252 if (!dc_isar_feature(aa64_sve2
, s
)) {
8255 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtnt_sh
);
8258 static bool trans_FCVTNT_ds(DisasContext
*s
, arg_rpr_esz
*a
)
8260 if (!dc_isar_feature(aa64_sve2
, s
)) {
8263 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve2_fcvtnt_ds
);