2 * AArch64 SVE translation
4 * Copyright (c) 2018 Linaro, Ltd
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
22 #include "exec/exec-all.h"
23 #include "tcg/tcg-op.h"
24 #include "tcg/tcg-op-gvec.h"
25 #include "tcg/tcg-gvec-desc.h"
28 #include "translate.h"
29 #include "internals.h"
30 #include "exec/helper-proto.h"
31 #include "exec/helper-gen.h"
33 #include "trace-tcg.h"
34 #include "translate-a64.h"
35 #include "fpu/softfloat.h"
38 typedef void GVecGen2sFn(unsigned, uint32_t, uint32_t,
39 TCGv_i64
, uint32_t, uint32_t);
41 typedef void gen_helper_gvec_flags_3(TCGv_i32
, TCGv_ptr
, TCGv_ptr
,
43 typedef void gen_helper_gvec_flags_4(TCGv_i32
, TCGv_ptr
, TCGv_ptr
,
44 TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
46 typedef void gen_helper_gvec_mem(TCGv_env
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
47 typedef void gen_helper_gvec_mem_scatter(TCGv_env
, TCGv_ptr
, TCGv_ptr
,
48 TCGv_ptr
, TCGv_i64
, TCGv_i32
);
51 * Helpers for extracting complex instruction fields.
54 /* See e.g. ASR (immediate, predicated).
55 * Returns -1 for unallocated encoding; diagnose later.
57 static int tszimm_esz(DisasContext
*s
, int x
)
59 x
>>= 3; /* discard imm3 */
63 static int tszimm_shr(DisasContext
*s
, int x
)
65 return (16 << tszimm_esz(s
, x
)) - x
;
68 /* See e.g. LSL (immediate, predicated). */
69 static int tszimm_shl(DisasContext
*s
, int x
)
71 return x
- (8 << tszimm_esz(s
, x
));
74 static inline int plus1(DisasContext
*s
, int x
)
79 /* The SH bit is in bit 8. Extract the low 8 and shift. */
80 static inline int expand_imm_sh8s(DisasContext
*s
, int x
)
82 return (int8_t)x
<< (x
& 0x100 ? 8 : 0);
85 static inline int expand_imm_sh8u(DisasContext
*s
, int x
)
87 return (uint8_t)x
<< (x
& 0x100 ? 8 : 0);
90 /* Convert a 2-bit memory size (msz) to a 4-bit data type (dtype)
91 * with unsigned data. C.f. SVE Memory Contiguous Load Group.
93 static inline int msz_dtype(DisasContext
*s
, int msz
)
95 static const uint8_t dtype
[4] = { 0, 5, 10, 15 };
100 * Include the generated decoder.
103 #include "decode-sve.c.inc"
106 * Implement all of the translator functions referenced by the decoder.
109 /* Return the offset info CPUARMState of the predicate vector register Pn.
110 * Note for this purpose, FFR is P16.
112 static inline int pred_full_reg_offset(DisasContext
*s
, int regno
)
114 return offsetof(CPUARMState
, vfp
.pregs
[regno
]);
117 /* Return the byte size of the whole predicate register, VL / 64. */
118 static inline int pred_full_reg_size(DisasContext
*s
)
120 return s
->sve_len
>> 3;
123 /* Round up the size of a register to a size allowed by
124 * the tcg vector infrastructure. Any operation which uses this
125 * size may assume that the bits above pred_full_reg_size are zero,
126 * and must leave them the same way.
128 * Note that this is not needed for the vector registers as they
129 * are always properly sized for tcg vectors.
131 static int size_for_gvec(int size
)
136 return QEMU_ALIGN_UP(size
, 16);
140 static int pred_gvec_reg_size(DisasContext
*s
)
142 return size_for_gvec(pred_full_reg_size(s
));
145 /* Invoke an out-of-line helper on 2 Zregs. */
146 static void gen_gvec_ool_zz(DisasContext
*s
, gen_helper_gvec_2
*fn
,
147 int rd
, int rn
, int data
)
149 unsigned vsz
= vec_full_reg_size(s
);
150 tcg_gen_gvec_2_ool(vec_full_reg_offset(s
, rd
),
151 vec_full_reg_offset(s
, rn
),
155 /* Invoke an out-of-line helper on 3 Zregs. */
156 static void gen_gvec_ool_zzz(DisasContext
*s
, gen_helper_gvec_3
*fn
,
157 int rd
, int rn
, int rm
, int data
)
159 unsigned vsz
= vec_full_reg_size(s
);
160 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
161 vec_full_reg_offset(s
, rn
),
162 vec_full_reg_offset(s
, rm
),
166 /* Invoke an out-of-line helper on 4 Zregs. */
167 static void gen_gvec_ool_zzzz(DisasContext
*s
, gen_helper_gvec_4
*fn
,
168 int rd
, int rn
, int rm
, int ra
, int data
)
170 unsigned vsz
= vec_full_reg_size(s
);
171 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
172 vec_full_reg_offset(s
, rn
),
173 vec_full_reg_offset(s
, rm
),
174 vec_full_reg_offset(s
, ra
),
178 /* Invoke an out-of-line helper on 2 Zregs and a predicate. */
179 static void gen_gvec_ool_zzp(DisasContext
*s
, gen_helper_gvec_3
*fn
,
180 int rd
, int rn
, int pg
, int data
)
182 unsigned vsz
= vec_full_reg_size(s
);
183 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
184 vec_full_reg_offset(s
, rn
),
185 pred_full_reg_offset(s
, pg
),
189 /* Invoke an out-of-line helper on 3 Zregs and a predicate. */
190 static void gen_gvec_ool_zzzp(DisasContext
*s
, gen_helper_gvec_4
*fn
,
191 int rd
, int rn
, int rm
, int pg
, int data
)
193 unsigned vsz
= vec_full_reg_size(s
);
194 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
195 vec_full_reg_offset(s
, rn
),
196 vec_full_reg_offset(s
, rm
),
197 pred_full_reg_offset(s
, pg
),
201 /* Invoke a vector expander on two Zregs. */
202 static void gen_gvec_fn_zz(DisasContext
*s
, GVecGen2Fn
*gvec_fn
,
203 int esz
, int rd
, int rn
)
205 unsigned vsz
= vec_full_reg_size(s
);
206 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
207 vec_full_reg_offset(s
, rn
), vsz
, vsz
);
210 /* Invoke a vector expander on three Zregs. */
211 static void gen_gvec_fn_zzz(DisasContext
*s
, GVecGen3Fn
*gvec_fn
,
212 int esz
, int rd
, int rn
, int rm
)
214 unsigned vsz
= vec_full_reg_size(s
);
215 gvec_fn(esz
, vec_full_reg_offset(s
, rd
),
216 vec_full_reg_offset(s
, rn
),
217 vec_full_reg_offset(s
, rm
), vsz
, vsz
);
220 /* Invoke a vector move on two Zregs. */
221 static bool do_mov_z(DisasContext
*s
, int rd
, int rn
)
223 if (sve_access_check(s
)) {
224 gen_gvec_fn_zz(s
, tcg_gen_gvec_mov
, MO_8
, rd
, rn
);
229 /* Initialize a Zreg with replications of a 64-bit immediate. */
230 static void do_dupi_z(DisasContext
*s
, int rd
, uint64_t word
)
232 unsigned vsz
= vec_full_reg_size(s
);
233 tcg_gen_gvec_dup_imm(MO_64
, vec_full_reg_offset(s
, rd
), vsz
, vsz
, word
);
236 /* Invoke a vector expander on three Pregs. */
237 static void gen_gvec_fn_ppp(DisasContext
*s
, GVecGen3Fn
*gvec_fn
,
238 int rd
, int rn
, int rm
)
240 unsigned psz
= pred_gvec_reg_size(s
);
241 gvec_fn(MO_64
, pred_full_reg_offset(s
, rd
),
242 pred_full_reg_offset(s
, rn
),
243 pred_full_reg_offset(s
, rm
), psz
, psz
);
246 /* Invoke a vector move on two Pregs. */
247 static bool do_mov_p(DisasContext
*s
, int rd
, int rn
)
249 if (sve_access_check(s
)) {
250 unsigned psz
= pred_gvec_reg_size(s
);
251 tcg_gen_gvec_mov(MO_8
, pred_full_reg_offset(s
, rd
),
252 pred_full_reg_offset(s
, rn
), psz
, psz
);
257 /* Set the cpu flags as per a return from an SVE helper. */
258 static void do_pred_flags(TCGv_i32 t
)
260 tcg_gen_mov_i32(cpu_NF
, t
);
261 tcg_gen_andi_i32(cpu_ZF
, t
, 2);
262 tcg_gen_andi_i32(cpu_CF
, t
, 1);
263 tcg_gen_movi_i32(cpu_VF
, 0);
266 /* Subroutines computing the ARM PredTest psuedofunction. */
267 static void do_predtest1(TCGv_i64 d
, TCGv_i64 g
)
269 TCGv_i32 t
= tcg_temp_new_i32();
271 gen_helper_sve_predtest1(t
, d
, g
);
273 tcg_temp_free_i32(t
);
276 static void do_predtest(DisasContext
*s
, int dofs
, int gofs
, int words
)
278 TCGv_ptr dptr
= tcg_temp_new_ptr();
279 TCGv_ptr gptr
= tcg_temp_new_ptr();
282 tcg_gen_addi_ptr(dptr
, cpu_env
, dofs
);
283 tcg_gen_addi_ptr(gptr
, cpu_env
, gofs
);
284 t
= tcg_const_i32(words
);
286 gen_helper_sve_predtest(t
, dptr
, gptr
, t
);
287 tcg_temp_free_ptr(dptr
);
288 tcg_temp_free_ptr(gptr
);
291 tcg_temp_free_i32(t
);
294 /* For each element size, the bits within a predicate word that are active. */
295 const uint64_t pred_esz_masks
[4] = {
296 0xffffffffffffffffull
, 0x5555555555555555ull
,
297 0x1111111111111111ull
, 0x0101010101010101ull
301 *** SVE Logical - Unpredicated Group
304 static bool do_zzz_fn(DisasContext
*s
, arg_rrr_esz
*a
, GVecGen3Fn
*gvec_fn
)
306 if (sve_access_check(s
)) {
307 gen_gvec_fn_zzz(s
, gvec_fn
, a
->esz
, a
->rd
, a
->rn
, a
->rm
);
312 static bool trans_AND_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
314 return do_zzz_fn(s
, a
, tcg_gen_gvec_and
);
317 static bool trans_ORR_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
319 return do_zzz_fn(s
, a
, tcg_gen_gvec_or
);
322 static bool trans_EOR_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
324 return do_zzz_fn(s
, a
, tcg_gen_gvec_xor
);
327 static bool trans_BIC_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
329 return do_zzz_fn(s
, a
, tcg_gen_gvec_andc
);
333 *** SVE Integer Arithmetic - Unpredicated Group
336 static bool trans_ADD_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
338 return do_zzz_fn(s
, a
, tcg_gen_gvec_add
);
341 static bool trans_SUB_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
343 return do_zzz_fn(s
, a
, tcg_gen_gvec_sub
);
346 static bool trans_SQADD_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
348 return do_zzz_fn(s
, a
, tcg_gen_gvec_ssadd
);
351 static bool trans_SQSUB_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
353 return do_zzz_fn(s
, a
, tcg_gen_gvec_sssub
);
356 static bool trans_UQADD_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
358 return do_zzz_fn(s
, a
, tcg_gen_gvec_usadd
);
361 static bool trans_UQSUB_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
363 return do_zzz_fn(s
, a
, tcg_gen_gvec_ussub
);
367 *** SVE Integer Arithmetic - Binary Predicated Group
370 static bool do_zpzz_ool(DisasContext
*s
, arg_rprr_esz
*a
, gen_helper_gvec_4
*fn
)
375 if (sve_access_check(s
)) {
376 gen_gvec_ool_zzzp(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->pg
, 0);
381 /* Select active elememnts from Zn and inactive elements from Zm,
382 * storing the result in Zd.
384 static void do_sel_z(DisasContext
*s
, int rd
, int rn
, int rm
, int pg
, int esz
)
386 static gen_helper_gvec_4
* const fns
[4] = {
387 gen_helper_sve_sel_zpzz_b
, gen_helper_sve_sel_zpzz_h
,
388 gen_helper_sve_sel_zpzz_s
, gen_helper_sve_sel_zpzz_d
390 gen_gvec_ool_zzzp(s
, fns
[esz
], rd
, rn
, rm
, pg
, 0);
393 #define DO_ZPZZ(NAME, name) \
394 static bool trans_##NAME##_zpzz(DisasContext *s, arg_rprr_esz *a) \
396 static gen_helper_gvec_4 * const fns[4] = { \
397 gen_helper_sve_##name##_zpzz_b, gen_helper_sve_##name##_zpzz_h, \
398 gen_helper_sve_##name##_zpzz_s, gen_helper_sve_##name##_zpzz_d, \
400 return do_zpzz_ool(s, a, fns[a->esz]); \
419 DO_ZPZZ(SMULH
, smulh
)
420 DO_ZPZZ(UMULH
, umulh
)
426 static bool trans_SDIV_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
428 static gen_helper_gvec_4
* const fns
[4] = {
429 NULL
, NULL
, gen_helper_sve_sdiv_zpzz_s
, gen_helper_sve_sdiv_zpzz_d
431 return do_zpzz_ool(s
, a
, fns
[a
->esz
]);
434 static bool trans_UDIV_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
436 static gen_helper_gvec_4
* const fns
[4] = {
437 NULL
, NULL
, gen_helper_sve_udiv_zpzz_s
, gen_helper_sve_udiv_zpzz_d
439 return do_zpzz_ool(s
, a
, fns
[a
->esz
]);
442 static bool trans_SEL_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
444 if (sve_access_check(s
)) {
445 do_sel_z(s
, a
->rd
, a
->rn
, a
->rm
, a
->pg
, a
->esz
);
453 *** SVE Integer Arithmetic - Unary Predicated Group
456 static bool do_zpz_ool(DisasContext
*s
, arg_rpr_esz
*a
, gen_helper_gvec_3
*fn
)
461 if (sve_access_check(s
)) {
462 gen_gvec_ool_zzp(s
, fn
, a
->rd
, a
->rn
, a
->pg
, 0);
467 #define DO_ZPZ(NAME, name) \
468 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
470 static gen_helper_gvec_3 * const fns[4] = { \
471 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
472 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
474 return do_zpz_ool(s, a, fns[a->esz]); \
479 DO_ZPZ(CNT_zpz
, cnt_zpz
)
481 DO_ZPZ(NOT_zpz
, not_zpz
)
485 static bool trans_FABS(DisasContext
*s
, arg_rpr_esz
*a
)
487 static gen_helper_gvec_3
* const fns
[4] = {
489 gen_helper_sve_fabs_h
,
490 gen_helper_sve_fabs_s
,
491 gen_helper_sve_fabs_d
493 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
496 static bool trans_FNEG(DisasContext
*s
, arg_rpr_esz
*a
)
498 static gen_helper_gvec_3
* const fns
[4] = {
500 gen_helper_sve_fneg_h
,
501 gen_helper_sve_fneg_s
,
502 gen_helper_sve_fneg_d
504 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
507 static bool trans_SXTB(DisasContext
*s
, arg_rpr_esz
*a
)
509 static gen_helper_gvec_3
* const fns
[4] = {
511 gen_helper_sve_sxtb_h
,
512 gen_helper_sve_sxtb_s
,
513 gen_helper_sve_sxtb_d
515 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
518 static bool trans_UXTB(DisasContext
*s
, arg_rpr_esz
*a
)
520 static gen_helper_gvec_3
* const fns
[4] = {
522 gen_helper_sve_uxtb_h
,
523 gen_helper_sve_uxtb_s
,
524 gen_helper_sve_uxtb_d
526 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
529 static bool trans_SXTH(DisasContext
*s
, arg_rpr_esz
*a
)
531 static gen_helper_gvec_3
* const fns
[4] = {
533 gen_helper_sve_sxth_s
,
534 gen_helper_sve_sxth_d
536 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
539 static bool trans_UXTH(DisasContext
*s
, arg_rpr_esz
*a
)
541 static gen_helper_gvec_3
* const fns
[4] = {
543 gen_helper_sve_uxth_s
,
544 gen_helper_sve_uxth_d
546 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
549 static bool trans_SXTW(DisasContext
*s
, arg_rpr_esz
*a
)
551 return do_zpz_ool(s
, a
, a
->esz
== 3 ? gen_helper_sve_sxtw_d
: NULL
);
554 static bool trans_UXTW(DisasContext
*s
, arg_rpr_esz
*a
)
556 return do_zpz_ool(s
, a
, a
->esz
== 3 ? gen_helper_sve_uxtw_d
: NULL
);
562 *** SVE Integer Reduction Group
565 typedef void gen_helper_gvec_reduc(TCGv_i64
, TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
566 static bool do_vpz_ool(DisasContext
*s
, arg_rpr_esz
*a
,
567 gen_helper_gvec_reduc
*fn
)
569 unsigned vsz
= vec_full_reg_size(s
);
577 if (!sve_access_check(s
)) {
581 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
582 temp
= tcg_temp_new_i64();
583 t_zn
= tcg_temp_new_ptr();
584 t_pg
= tcg_temp_new_ptr();
586 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
587 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
588 fn(temp
, t_zn
, t_pg
, desc
);
589 tcg_temp_free_ptr(t_zn
);
590 tcg_temp_free_ptr(t_pg
);
591 tcg_temp_free_i32(desc
);
593 write_fp_dreg(s
, a
->rd
, temp
);
594 tcg_temp_free_i64(temp
);
598 #define DO_VPZ(NAME, name) \
599 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
601 static gen_helper_gvec_reduc * const fns[4] = { \
602 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
603 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
605 return do_vpz_ool(s, a, fns[a->esz]); \
618 static bool trans_SADDV(DisasContext
*s
, arg_rpr_esz
*a
)
620 static gen_helper_gvec_reduc
* const fns
[4] = {
621 gen_helper_sve_saddv_b
, gen_helper_sve_saddv_h
,
622 gen_helper_sve_saddv_s
, NULL
624 return do_vpz_ool(s
, a
, fns
[a
->esz
]);
630 *** SVE Shift by Immediate - Predicated Group
634 * Copy Zn into Zd, storing zeros into inactive elements.
635 * If invert, store zeros into the active elements.
637 static bool do_movz_zpz(DisasContext
*s
, int rd
, int rn
, int pg
,
638 int esz
, bool invert
)
640 static gen_helper_gvec_3
* const fns
[4] = {
641 gen_helper_sve_movz_b
, gen_helper_sve_movz_h
,
642 gen_helper_sve_movz_s
, gen_helper_sve_movz_d
,
645 if (sve_access_check(s
)) {
646 gen_gvec_ool_zzp(s
, fns
[esz
], rd
, rn
, pg
, invert
);
651 static bool do_zpzi_ool(DisasContext
*s
, arg_rpri_esz
*a
,
652 gen_helper_gvec_3
*fn
)
654 if (sve_access_check(s
)) {
655 gen_gvec_ool_zzp(s
, fn
, a
->rd
, a
->rn
, a
->pg
, a
->imm
);
660 static bool trans_ASR_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
662 static gen_helper_gvec_3
* const fns
[4] = {
663 gen_helper_sve_asr_zpzi_b
, gen_helper_sve_asr_zpzi_h
,
664 gen_helper_sve_asr_zpzi_s
, gen_helper_sve_asr_zpzi_d
,
667 /* Invalid tsz encoding -- see tszimm_esz. */
670 /* Shift by element size is architecturally valid. For
671 arithmetic right-shift, it's the same as by one less. */
672 a
->imm
= MIN(a
->imm
, (8 << a
->esz
) - 1);
673 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
676 static bool trans_LSR_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
678 static gen_helper_gvec_3
* const fns
[4] = {
679 gen_helper_sve_lsr_zpzi_b
, gen_helper_sve_lsr_zpzi_h
,
680 gen_helper_sve_lsr_zpzi_s
, gen_helper_sve_lsr_zpzi_d
,
685 /* Shift by element size is architecturally valid.
686 For logical shifts, it is a zeroing operation. */
687 if (a
->imm
>= (8 << a
->esz
)) {
688 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
690 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
694 static bool trans_LSL_zpzi(DisasContext
*s
, arg_rpri_esz
*a
)
696 static gen_helper_gvec_3
* const fns
[4] = {
697 gen_helper_sve_lsl_zpzi_b
, gen_helper_sve_lsl_zpzi_h
,
698 gen_helper_sve_lsl_zpzi_s
, gen_helper_sve_lsl_zpzi_d
,
703 /* Shift by element size is architecturally valid.
704 For logical shifts, it is a zeroing operation. */
705 if (a
->imm
>= (8 << a
->esz
)) {
706 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
708 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
712 static bool trans_ASRD(DisasContext
*s
, arg_rpri_esz
*a
)
714 static gen_helper_gvec_3
* const fns
[4] = {
715 gen_helper_sve_asrd_b
, gen_helper_sve_asrd_h
,
716 gen_helper_sve_asrd_s
, gen_helper_sve_asrd_d
,
721 /* Shift by element size is architecturally valid. For arithmetic
722 right shift for division, it is a zeroing operation. */
723 if (a
->imm
>= (8 << a
->esz
)) {
724 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, a
->esz
, true);
726 return do_zpzi_ool(s
, a
, fns
[a
->esz
]);
731 *** SVE Bitwise Shift - Predicated Group
734 #define DO_ZPZW(NAME, name) \
735 static bool trans_##NAME##_zpzw(DisasContext *s, arg_rprr_esz *a) \
737 static gen_helper_gvec_4 * const fns[3] = { \
738 gen_helper_sve_##name##_zpzw_b, gen_helper_sve_##name##_zpzw_h, \
739 gen_helper_sve_##name##_zpzw_s, \
741 if (a->esz < 0 || a->esz >= 3) { \
744 return do_zpzz_ool(s, a, fns[a->esz]); \
754 *** SVE Bitwise Shift - Unpredicated Group
757 static bool do_shift_imm(DisasContext
*s
, arg_rri_esz
*a
, bool asr
,
758 void (*gvec_fn
)(unsigned, uint32_t, uint32_t,
759 int64_t, uint32_t, uint32_t))
762 /* Invalid tsz encoding -- see tszimm_esz. */
765 if (sve_access_check(s
)) {
766 unsigned vsz
= vec_full_reg_size(s
);
767 /* Shift by element size is architecturally valid. For
768 arithmetic right-shift, it's the same as by one less.
769 Otherwise it is a zeroing operation. */
770 if (a
->imm
>= 8 << a
->esz
) {
772 a
->imm
= (8 << a
->esz
) - 1;
774 do_dupi_z(s
, a
->rd
, 0);
778 gvec_fn(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
779 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
784 static bool trans_ASR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
786 return do_shift_imm(s
, a
, true, tcg_gen_gvec_sari
);
789 static bool trans_LSR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
791 return do_shift_imm(s
, a
, false, tcg_gen_gvec_shri
);
794 static bool trans_LSL_zzi(DisasContext
*s
, arg_rri_esz
*a
)
796 return do_shift_imm(s
, a
, false, tcg_gen_gvec_shli
);
799 static bool do_zzw_ool(DisasContext
*s
, arg_rrr_esz
*a
, gen_helper_gvec_3
*fn
)
804 if (sve_access_check(s
)) {
805 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, 0);
810 #define DO_ZZW(NAME, name) \
811 static bool trans_##NAME##_zzw(DisasContext *s, arg_rrr_esz *a) \
813 static gen_helper_gvec_3 * const fns[4] = { \
814 gen_helper_sve_##name##_zzw_b, gen_helper_sve_##name##_zzw_h, \
815 gen_helper_sve_##name##_zzw_s, NULL \
817 return do_zzw_ool(s, a, fns[a->esz]); \
827 *** SVE Integer Multiply-Add Group
830 static bool do_zpzzz_ool(DisasContext
*s
, arg_rprrr_esz
*a
,
831 gen_helper_gvec_5
*fn
)
833 if (sve_access_check(s
)) {
834 unsigned vsz
= vec_full_reg_size(s
);
835 tcg_gen_gvec_5_ool(vec_full_reg_offset(s
, a
->rd
),
836 vec_full_reg_offset(s
, a
->ra
),
837 vec_full_reg_offset(s
, a
->rn
),
838 vec_full_reg_offset(s
, a
->rm
),
839 pred_full_reg_offset(s
, a
->pg
),
845 #define DO_ZPZZZ(NAME, name) \
846 static bool trans_##NAME(DisasContext *s, arg_rprrr_esz *a) \
848 static gen_helper_gvec_5 * const fns[4] = { \
849 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
850 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
852 return do_zpzzz_ool(s, a, fns[a->esz]); \
861 *** SVE Index Generation Group
864 static void do_index(DisasContext
*s
, int esz
, int rd
,
865 TCGv_i64 start
, TCGv_i64 incr
)
867 unsigned vsz
= vec_full_reg_size(s
);
868 TCGv_i32 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
869 TCGv_ptr t_zd
= tcg_temp_new_ptr();
871 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, rd
));
873 gen_helper_sve_index_d(t_zd
, start
, incr
, desc
);
875 typedef void index_fn(TCGv_ptr
, TCGv_i32
, TCGv_i32
, TCGv_i32
);
876 static index_fn
* const fns
[3] = {
877 gen_helper_sve_index_b
,
878 gen_helper_sve_index_h
,
879 gen_helper_sve_index_s
,
881 TCGv_i32 s32
= tcg_temp_new_i32();
882 TCGv_i32 i32
= tcg_temp_new_i32();
884 tcg_gen_extrl_i64_i32(s32
, start
);
885 tcg_gen_extrl_i64_i32(i32
, incr
);
886 fns
[esz
](t_zd
, s32
, i32
, desc
);
888 tcg_temp_free_i32(s32
);
889 tcg_temp_free_i32(i32
);
891 tcg_temp_free_ptr(t_zd
);
892 tcg_temp_free_i32(desc
);
895 static bool trans_INDEX_ii(DisasContext
*s
, arg_INDEX_ii
*a
)
897 if (sve_access_check(s
)) {
898 TCGv_i64 start
= tcg_const_i64(a
->imm1
);
899 TCGv_i64 incr
= tcg_const_i64(a
->imm2
);
900 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
901 tcg_temp_free_i64(start
);
902 tcg_temp_free_i64(incr
);
907 static bool trans_INDEX_ir(DisasContext
*s
, arg_INDEX_ir
*a
)
909 if (sve_access_check(s
)) {
910 TCGv_i64 start
= tcg_const_i64(a
->imm
);
911 TCGv_i64 incr
= cpu_reg(s
, a
->rm
);
912 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
913 tcg_temp_free_i64(start
);
918 static bool trans_INDEX_ri(DisasContext
*s
, arg_INDEX_ri
*a
)
920 if (sve_access_check(s
)) {
921 TCGv_i64 start
= cpu_reg(s
, a
->rn
);
922 TCGv_i64 incr
= tcg_const_i64(a
->imm
);
923 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
924 tcg_temp_free_i64(incr
);
929 static bool trans_INDEX_rr(DisasContext
*s
, arg_INDEX_rr
*a
)
931 if (sve_access_check(s
)) {
932 TCGv_i64 start
= cpu_reg(s
, a
->rn
);
933 TCGv_i64 incr
= cpu_reg(s
, a
->rm
);
934 do_index(s
, a
->esz
, a
->rd
, start
, incr
);
940 *** SVE Stack Allocation Group
943 static bool trans_ADDVL(DisasContext
*s
, arg_ADDVL
*a
)
945 if (sve_access_check(s
)) {
946 TCGv_i64 rd
= cpu_reg_sp(s
, a
->rd
);
947 TCGv_i64 rn
= cpu_reg_sp(s
, a
->rn
);
948 tcg_gen_addi_i64(rd
, rn
, a
->imm
* vec_full_reg_size(s
));
953 static bool trans_ADDPL(DisasContext
*s
, arg_ADDPL
*a
)
955 if (sve_access_check(s
)) {
956 TCGv_i64 rd
= cpu_reg_sp(s
, a
->rd
);
957 TCGv_i64 rn
= cpu_reg_sp(s
, a
->rn
);
958 tcg_gen_addi_i64(rd
, rn
, a
->imm
* pred_full_reg_size(s
));
963 static bool trans_RDVL(DisasContext
*s
, arg_RDVL
*a
)
965 if (sve_access_check(s
)) {
966 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
967 tcg_gen_movi_i64(reg
, a
->imm
* vec_full_reg_size(s
));
973 *** SVE Compute Vector Address Group
976 static bool do_adr(DisasContext
*s
, arg_rrri
*a
, gen_helper_gvec_3
*fn
)
978 if (sve_access_check(s
)) {
979 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->imm
);
984 static bool trans_ADR_p32(DisasContext
*s
, arg_rrri
*a
)
986 return do_adr(s
, a
, gen_helper_sve_adr_p32
);
989 static bool trans_ADR_p64(DisasContext
*s
, arg_rrri
*a
)
991 return do_adr(s
, a
, gen_helper_sve_adr_p64
);
994 static bool trans_ADR_s32(DisasContext
*s
, arg_rrri
*a
)
996 return do_adr(s
, a
, gen_helper_sve_adr_s32
);
999 static bool trans_ADR_u32(DisasContext
*s
, arg_rrri
*a
)
1001 return do_adr(s
, a
, gen_helper_sve_adr_u32
);
1005 *** SVE Integer Misc - Unpredicated Group
1008 static bool trans_FEXPA(DisasContext
*s
, arg_rr_esz
*a
)
1010 static gen_helper_gvec_2
* const fns
[4] = {
1012 gen_helper_sve_fexpa_h
,
1013 gen_helper_sve_fexpa_s
,
1014 gen_helper_sve_fexpa_d
,
1019 if (sve_access_check(s
)) {
1020 gen_gvec_ool_zz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, 0);
1025 static bool trans_FTSSEL(DisasContext
*s
, arg_rrr_esz
*a
)
1027 static gen_helper_gvec_3
* const fns
[4] = {
1029 gen_helper_sve_ftssel_h
,
1030 gen_helper_sve_ftssel_s
,
1031 gen_helper_sve_ftssel_d
,
1036 if (sve_access_check(s
)) {
1037 gen_gvec_ool_zzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, 0);
1043 *** SVE Predicate Logical Operations Group
1046 static bool do_pppp_flags(DisasContext
*s
, arg_rprr_s
*a
,
1047 const GVecGen4
*gvec_op
)
1049 if (!sve_access_check(s
)) {
1053 unsigned psz
= pred_gvec_reg_size(s
);
1054 int dofs
= pred_full_reg_offset(s
, a
->rd
);
1055 int nofs
= pred_full_reg_offset(s
, a
->rn
);
1056 int mofs
= pred_full_reg_offset(s
, a
->rm
);
1057 int gofs
= pred_full_reg_offset(s
, a
->pg
);
1060 tcg_gen_gvec_4(dofs
, nofs
, mofs
, gofs
, psz
, psz
, gvec_op
);
1065 /* Do the operation and the flags generation in temps. */
1066 TCGv_i64 pd
= tcg_temp_new_i64();
1067 TCGv_i64 pn
= tcg_temp_new_i64();
1068 TCGv_i64 pm
= tcg_temp_new_i64();
1069 TCGv_i64 pg
= tcg_temp_new_i64();
1071 tcg_gen_ld_i64(pn
, cpu_env
, nofs
);
1072 tcg_gen_ld_i64(pm
, cpu_env
, mofs
);
1073 tcg_gen_ld_i64(pg
, cpu_env
, gofs
);
1075 gvec_op
->fni8(pd
, pn
, pm
, pg
);
1076 tcg_gen_st_i64(pd
, cpu_env
, dofs
);
1078 do_predtest1(pd
, pg
);
1080 tcg_temp_free_i64(pd
);
1081 tcg_temp_free_i64(pn
);
1082 tcg_temp_free_i64(pm
);
1083 tcg_temp_free_i64(pg
);
1085 /* The operation and flags generation is large. The computation
1086 * of the flags depends on the original contents of the guarding
1087 * predicate. If the destination overwrites the guarding predicate,
1088 * then the easiest way to get this right is to save a copy.
1091 if (a
->rd
== a
->pg
) {
1092 tofs
= offsetof(CPUARMState
, vfp
.preg_tmp
);
1093 tcg_gen_gvec_mov(0, tofs
, gofs
, psz
, psz
);
1096 tcg_gen_gvec_4(dofs
, nofs
, mofs
, gofs
, psz
, psz
, gvec_op
);
1097 do_predtest(s
, dofs
, tofs
, psz
/ 8);
1102 static void gen_and_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1104 tcg_gen_and_i64(pd
, pn
, pm
);
1105 tcg_gen_and_i64(pd
, pd
, pg
);
1108 static void gen_and_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1109 TCGv_vec pm
, TCGv_vec pg
)
1111 tcg_gen_and_vec(vece
, pd
, pn
, pm
);
1112 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1115 static bool trans_AND_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1117 static const GVecGen4 op
= {
1118 .fni8
= gen_and_pg_i64
,
1119 .fniv
= gen_and_pg_vec
,
1120 .fno
= gen_helper_sve_and_pppp
,
1121 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1125 if (!sve_access_check(s
)) {
1128 if (a
->rn
== a
->rm
) {
1129 if (a
->pg
== a
->rn
) {
1130 do_mov_p(s
, a
->rd
, a
->rn
);
1132 gen_gvec_fn_ppp(s
, tcg_gen_gvec_and
, a
->rd
, a
->rn
, a
->pg
);
1135 } else if (a
->pg
== a
->rn
|| a
->pg
== a
->rm
) {
1136 gen_gvec_fn_ppp(s
, tcg_gen_gvec_and
, a
->rd
, a
->rn
, a
->rm
);
1140 return do_pppp_flags(s
, a
, &op
);
1143 static void gen_bic_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1145 tcg_gen_andc_i64(pd
, pn
, pm
);
1146 tcg_gen_and_i64(pd
, pd
, pg
);
1149 static void gen_bic_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1150 TCGv_vec pm
, TCGv_vec pg
)
1152 tcg_gen_andc_vec(vece
, pd
, pn
, pm
);
1153 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1156 static bool trans_BIC_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1158 static const GVecGen4 op
= {
1159 .fni8
= gen_bic_pg_i64
,
1160 .fniv
= gen_bic_pg_vec
,
1161 .fno
= gen_helper_sve_bic_pppp
,
1162 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1165 if (!a
->s
&& a
->pg
== a
->rn
) {
1166 if (sve_access_check(s
)) {
1167 gen_gvec_fn_ppp(s
, tcg_gen_gvec_andc
, a
->rd
, a
->rn
, a
->rm
);
1171 return do_pppp_flags(s
, a
, &op
);
1174 static void gen_eor_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1176 tcg_gen_xor_i64(pd
, pn
, pm
);
1177 tcg_gen_and_i64(pd
, pd
, pg
);
1180 static void gen_eor_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1181 TCGv_vec pm
, TCGv_vec pg
)
1183 tcg_gen_xor_vec(vece
, pd
, pn
, pm
);
1184 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1187 static bool trans_EOR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1189 static const GVecGen4 op
= {
1190 .fni8
= gen_eor_pg_i64
,
1191 .fniv
= gen_eor_pg_vec
,
1192 .fno
= gen_helper_sve_eor_pppp
,
1193 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1195 return do_pppp_flags(s
, a
, &op
);
1198 static bool trans_SEL_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1203 if (sve_access_check(s
)) {
1204 unsigned psz
= pred_gvec_reg_size(s
);
1205 tcg_gen_gvec_bitsel(MO_8
, pred_full_reg_offset(s
, a
->rd
),
1206 pred_full_reg_offset(s
, a
->pg
),
1207 pred_full_reg_offset(s
, a
->rn
),
1208 pred_full_reg_offset(s
, a
->rm
), psz
, psz
);
1213 static void gen_orr_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1215 tcg_gen_or_i64(pd
, pn
, pm
);
1216 tcg_gen_and_i64(pd
, pd
, pg
);
1219 static void gen_orr_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1220 TCGv_vec pm
, TCGv_vec pg
)
1222 tcg_gen_or_vec(vece
, pd
, pn
, pm
);
1223 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1226 static bool trans_ORR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1228 static const GVecGen4 op
= {
1229 .fni8
= gen_orr_pg_i64
,
1230 .fniv
= gen_orr_pg_vec
,
1231 .fno
= gen_helper_sve_orr_pppp
,
1232 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1235 if (!a
->s
&& a
->pg
== a
->rn
&& a
->rn
== a
->rm
) {
1236 return do_mov_p(s
, a
->rd
, a
->rn
);
1238 return do_pppp_flags(s
, a
, &op
);
1241 static void gen_orn_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1243 tcg_gen_orc_i64(pd
, pn
, pm
);
1244 tcg_gen_and_i64(pd
, pd
, pg
);
1247 static void gen_orn_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1248 TCGv_vec pm
, TCGv_vec pg
)
1250 tcg_gen_orc_vec(vece
, pd
, pn
, pm
);
1251 tcg_gen_and_vec(vece
, pd
, pd
, pg
);
1254 static bool trans_ORN_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1256 static const GVecGen4 op
= {
1257 .fni8
= gen_orn_pg_i64
,
1258 .fniv
= gen_orn_pg_vec
,
1259 .fno
= gen_helper_sve_orn_pppp
,
1260 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1262 return do_pppp_flags(s
, a
, &op
);
1265 static void gen_nor_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1267 tcg_gen_or_i64(pd
, pn
, pm
);
1268 tcg_gen_andc_i64(pd
, pg
, pd
);
1271 static void gen_nor_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1272 TCGv_vec pm
, TCGv_vec pg
)
1274 tcg_gen_or_vec(vece
, pd
, pn
, pm
);
1275 tcg_gen_andc_vec(vece
, pd
, pg
, pd
);
1278 static bool trans_NOR_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1280 static const GVecGen4 op
= {
1281 .fni8
= gen_nor_pg_i64
,
1282 .fniv
= gen_nor_pg_vec
,
1283 .fno
= gen_helper_sve_nor_pppp
,
1284 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1286 return do_pppp_flags(s
, a
, &op
);
1289 static void gen_nand_pg_i64(TCGv_i64 pd
, TCGv_i64 pn
, TCGv_i64 pm
, TCGv_i64 pg
)
1291 tcg_gen_and_i64(pd
, pn
, pm
);
1292 tcg_gen_andc_i64(pd
, pg
, pd
);
1295 static void gen_nand_pg_vec(unsigned vece
, TCGv_vec pd
, TCGv_vec pn
,
1296 TCGv_vec pm
, TCGv_vec pg
)
1298 tcg_gen_and_vec(vece
, pd
, pn
, pm
);
1299 tcg_gen_andc_vec(vece
, pd
, pg
, pd
);
1302 static bool trans_NAND_pppp(DisasContext
*s
, arg_rprr_s
*a
)
1304 static const GVecGen4 op
= {
1305 .fni8
= gen_nand_pg_i64
,
1306 .fniv
= gen_nand_pg_vec
,
1307 .fno
= gen_helper_sve_nand_pppp
,
1308 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
1310 return do_pppp_flags(s
, a
, &op
);
1314 *** SVE Predicate Misc Group
1317 static bool trans_PTEST(DisasContext
*s
, arg_PTEST
*a
)
1319 if (sve_access_check(s
)) {
1320 int nofs
= pred_full_reg_offset(s
, a
->rn
);
1321 int gofs
= pred_full_reg_offset(s
, a
->pg
);
1322 int words
= DIV_ROUND_UP(pred_full_reg_size(s
), 8);
1325 TCGv_i64 pn
= tcg_temp_new_i64();
1326 TCGv_i64 pg
= tcg_temp_new_i64();
1328 tcg_gen_ld_i64(pn
, cpu_env
, nofs
);
1329 tcg_gen_ld_i64(pg
, cpu_env
, gofs
);
1330 do_predtest1(pn
, pg
);
1332 tcg_temp_free_i64(pn
);
1333 tcg_temp_free_i64(pg
);
1335 do_predtest(s
, nofs
, gofs
, words
);
1341 /* See the ARM pseudocode DecodePredCount. */
1342 static unsigned decode_pred_count(unsigned fullsz
, int pattern
, int esz
)
1344 unsigned elements
= fullsz
>> esz
;
1348 case 0x0: /* POW2 */
1349 return pow2floor(elements
);
1360 case 0x9: /* VL16 */
1361 case 0xa: /* VL32 */
1362 case 0xb: /* VL64 */
1363 case 0xc: /* VL128 */
1364 case 0xd: /* VL256 */
1365 bound
= 16 << (pattern
- 9);
1367 case 0x1d: /* MUL4 */
1368 return elements
- elements
% 4;
1369 case 0x1e: /* MUL3 */
1370 return elements
- elements
% 3;
1371 case 0x1f: /* ALL */
1373 default: /* #uimm5 */
1376 return elements
>= bound
? bound
: 0;
1379 /* This handles all of the predicate initialization instructions,
1380 * PTRUE, PFALSE, SETFFR. For PFALSE, we will have set PAT == 32
1381 * so that decode_pred_count returns 0. For SETFFR, we will have
1382 * set RD == 16 == FFR.
1384 static bool do_predset(DisasContext
*s
, int esz
, int rd
, int pat
, bool setflag
)
1386 if (!sve_access_check(s
)) {
1390 unsigned fullsz
= vec_full_reg_size(s
);
1391 unsigned ofs
= pred_full_reg_offset(s
, rd
);
1392 unsigned numelem
, setsz
, i
;
1393 uint64_t word
, lastword
;
1396 numelem
= decode_pred_count(fullsz
, pat
, esz
);
1398 /* Determine what we must store into each bit, and how many. */
1400 lastword
= word
= 0;
1403 setsz
= numelem
<< esz
;
1404 lastword
= word
= pred_esz_masks
[esz
];
1406 lastword
&= MAKE_64BIT_MASK(0, setsz
% 64);
1410 t
= tcg_temp_new_i64();
1412 tcg_gen_movi_i64(t
, lastword
);
1413 tcg_gen_st_i64(t
, cpu_env
, ofs
);
1417 if (word
== lastword
) {
1418 unsigned maxsz
= size_for_gvec(fullsz
/ 8);
1419 unsigned oprsz
= size_for_gvec(setsz
/ 8);
1421 if (oprsz
* 8 == setsz
) {
1422 tcg_gen_gvec_dup_imm(MO_64
, ofs
, oprsz
, maxsz
, word
);
1430 tcg_gen_movi_i64(t
, word
);
1431 for (i
= 0; i
< QEMU_ALIGN_DOWN(setsz
, 8); i
+= 8) {
1432 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1434 if (lastword
!= word
) {
1435 tcg_gen_movi_i64(t
, lastword
);
1436 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1440 tcg_gen_movi_i64(t
, 0);
1441 for (; i
< fullsz
; i
+= 8) {
1442 tcg_gen_st_i64(t
, cpu_env
, ofs
+ i
);
1447 tcg_temp_free_i64(t
);
1451 tcg_gen_movi_i32(cpu_NF
, -(word
!= 0));
1452 tcg_gen_movi_i32(cpu_CF
, word
== 0);
1453 tcg_gen_movi_i32(cpu_VF
, 0);
1454 tcg_gen_mov_i32(cpu_ZF
, cpu_NF
);
1459 static bool trans_PTRUE(DisasContext
*s
, arg_PTRUE
*a
)
1461 return do_predset(s
, a
->esz
, a
->rd
, a
->pat
, a
->s
);
1464 static bool trans_SETFFR(DisasContext
*s
, arg_SETFFR
*a
)
1466 /* Note pat == 31 is #all, to set all elements. */
1467 return do_predset(s
, 0, FFR_PRED_NUM
, 31, false);
1470 static bool trans_PFALSE(DisasContext
*s
, arg_PFALSE
*a
)
1472 /* Note pat == 32 is #unimp, to set no elements. */
1473 return do_predset(s
, 0, a
->rd
, 32, false);
1476 static bool trans_RDFFR_p(DisasContext
*s
, arg_RDFFR_p
*a
)
1478 /* The path through do_pppp_flags is complicated enough to want to avoid
1479 * duplication. Frob the arguments into the form of a predicated AND.
1481 arg_rprr_s alt_a
= {
1482 .rd
= a
->rd
, .pg
= a
->pg
, .s
= a
->s
,
1483 .rn
= FFR_PRED_NUM
, .rm
= FFR_PRED_NUM
,
1485 return trans_AND_pppp(s
, &alt_a
);
1488 static bool trans_RDFFR(DisasContext
*s
, arg_RDFFR
*a
)
1490 return do_mov_p(s
, a
->rd
, FFR_PRED_NUM
);
1493 static bool trans_WRFFR(DisasContext
*s
, arg_WRFFR
*a
)
1495 return do_mov_p(s
, FFR_PRED_NUM
, a
->rn
);
1498 static bool do_pfirst_pnext(DisasContext
*s
, arg_rr_esz
*a
,
1499 void (*gen_fn
)(TCGv_i32
, TCGv_ptr
,
1500 TCGv_ptr
, TCGv_i32
))
1502 if (!sve_access_check(s
)) {
1506 TCGv_ptr t_pd
= tcg_temp_new_ptr();
1507 TCGv_ptr t_pg
= tcg_temp_new_ptr();
1511 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, pred_full_reg_size(s
));
1512 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
1514 tcg_gen_addi_ptr(t_pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
1515 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
1516 t
= tcg_const_i32(desc
);
1518 gen_fn(t
, t_pd
, t_pg
, t
);
1519 tcg_temp_free_ptr(t_pd
);
1520 tcg_temp_free_ptr(t_pg
);
1523 tcg_temp_free_i32(t
);
1527 static bool trans_PFIRST(DisasContext
*s
, arg_rr_esz
*a
)
1529 return do_pfirst_pnext(s
, a
, gen_helper_sve_pfirst
);
1532 static bool trans_PNEXT(DisasContext
*s
, arg_rr_esz
*a
)
1534 return do_pfirst_pnext(s
, a
, gen_helper_sve_pnext
);
1538 *** SVE Element Count Group
1541 /* Perform an inline saturating addition of a 32-bit value within
1542 * a 64-bit register. The second operand is known to be positive,
1543 * which halves the comparisions we must perform to bound the result.
1545 static void do_sat_addsub_32(TCGv_i64 reg
, TCGv_i64 val
, bool u
, bool d
)
1551 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
1553 tcg_gen_ext32u_i64(reg
, reg
);
1555 tcg_gen_ext32s_i64(reg
, reg
);
1558 tcg_gen_sub_i64(reg
, reg
, val
);
1559 ibound
= (u
? 0 : INT32_MIN
);
1562 tcg_gen_add_i64(reg
, reg
, val
);
1563 ibound
= (u
? UINT32_MAX
: INT32_MAX
);
1566 bound
= tcg_const_i64(ibound
);
1567 tcg_gen_movcond_i64(cond
, reg
, reg
, bound
, bound
, reg
);
1568 tcg_temp_free_i64(bound
);
1571 /* Similarly with 64-bit values. */
1572 static void do_sat_addsub_64(TCGv_i64 reg
, TCGv_i64 val
, bool u
, bool d
)
1574 TCGv_i64 t0
= tcg_temp_new_i64();
1575 TCGv_i64 t1
= tcg_temp_new_i64();
1580 tcg_gen_sub_i64(t0
, reg
, val
);
1581 tcg_gen_movi_i64(t1
, 0);
1582 tcg_gen_movcond_i64(TCG_COND_LTU
, reg
, reg
, val
, t1
, t0
);
1584 tcg_gen_add_i64(t0
, reg
, val
);
1585 tcg_gen_movi_i64(t1
, -1);
1586 tcg_gen_movcond_i64(TCG_COND_LTU
, reg
, t0
, reg
, t1
, t0
);
1590 /* Detect signed overflow for subtraction. */
1591 tcg_gen_xor_i64(t0
, reg
, val
);
1592 tcg_gen_sub_i64(t1
, reg
, val
);
1593 tcg_gen_xor_i64(reg
, reg
, t1
);
1594 tcg_gen_and_i64(t0
, t0
, reg
);
1596 /* Bound the result. */
1597 tcg_gen_movi_i64(reg
, INT64_MIN
);
1598 t2
= tcg_const_i64(0);
1599 tcg_gen_movcond_i64(TCG_COND_LT
, reg
, t0
, t2
, reg
, t1
);
1601 /* Detect signed overflow for addition. */
1602 tcg_gen_xor_i64(t0
, reg
, val
);
1603 tcg_gen_add_i64(reg
, reg
, val
);
1604 tcg_gen_xor_i64(t1
, reg
, val
);
1605 tcg_gen_andc_i64(t0
, t1
, t0
);
1607 /* Bound the result. */
1608 tcg_gen_movi_i64(t1
, INT64_MAX
);
1609 t2
= tcg_const_i64(0);
1610 tcg_gen_movcond_i64(TCG_COND_LT
, reg
, t0
, t2
, t1
, reg
);
1612 tcg_temp_free_i64(t2
);
1614 tcg_temp_free_i64(t0
);
1615 tcg_temp_free_i64(t1
);
1618 /* Similarly with a vector and a scalar operand. */
1619 static void do_sat_addsub_vec(DisasContext
*s
, int esz
, int rd
, int rn
,
1620 TCGv_i64 val
, bool u
, bool d
)
1622 unsigned vsz
= vec_full_reg_size(s
);
1623 TCGv_ptr dptr
, nptr
;
1627 dptr
= tcg_temp_new_ptr();
1628 nptr
= tcg_temp_new_ptr();
1629 tcg_gen_addi_ptr(dptr
, cpu_env
, vec_full_reg_offset(s
, rd
));
1630 tcg_gen_addi_ptr(nptr
, cpu_env
, vec_full_reg_offset(s
, rn
));
1631 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
1635 t32
= tcg_temp_new_i32();
1636 tcg_gen_extrl_i64_i32(t32
, val
);
1638 tcg_gen_neg_i32(t32
, t32
);
1641 gen_helper_sve_uqaddi_b(dptr
, nptr
, t32
, desc
);
1643 gen_helper_sve_sqaddi_b(dptr
, nptr
, t32
, desc
);
1645 tcg_temp_free_i32(t32
);
1649 t32
= tcg_temp_new_i32();
1650 tcg_gen_extrl_i64_i32(t32
, val
);
1652 tcg_gen_neg_i32(t32
, t32
);
1655 gen_helper_sve_uqaddi_h(dptr
, nptr
, t32
, desc
);
1657 gen_helper_sve_sqaddi_h(dptr
, nptr
, t32
, desc
);
1659 tcg_temp_free_i32(t32
);
1663 t64
= tcg_temp_new_i64();
1665 tcg_gen_neg_i64(t64
, val
);
1667 tcg_gen_mov_i64(t64
, val
);
1670 gen_helper_sve_uqaddi_s(dptr
, nptr
, t64
, desc
);
1672 gen_helper_sve_sqaddi_s(dptr
, nptr
, t64
, desc
);
1674 tcg_temp_free_i64(t64
);
1680 gen_helper_sve_uqsubi_d(dptr
, nptr
, val
, desc
);
1682 gen_helper_sve_uqaddi_d(dptr
, nptr
, val
, desc
);
1685 t64
= tcg_temp_new_i64();
1686 tcg_gen_neg_i64(t64
, val
);
1687 gen_helper_sve_sqaddi_d(dptr
, nptr
, t64
, desc
);
1688 tcg_temp_free_i64(t64
);
1690 gen_helper_sve_sqaddi_d(dptr
, nptr
, val
, desc
);
1695 g_assert_not_reached();
1698 tcg_temp_free_ptr(dptr
);
1699 tcg_temp_free_ptr(nptr
);
1700 tcg_temp_free_i32(desc
);
1703 static bool trans_CNT_r(DisasContext
*s
, arg_CNT_r
*a
)
1705 if (sve_access_check(s
)) {
1706 unsigned fullsz
= vec_full_reg_size(s
);
1707 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
1708 tcg_gen_movi_i64(cpu_reg(s
, a
->rd
), numelem
* a
->imm
);
1713 static bool trans_INCDEC_r(DisasContext
*s
, arg_incdec_cnt
*a
)
1715 if (sve_access_check(s
)) {
1716 unsigned fullsz
= vec_full_reg_size(s
);
1717 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
1718 int inc
= numelem
* a
->imm
* (a
->d
? -1 : 1);
1719 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
1721 tcg_gen_addi_i64(reg
, reg
, inc
);
1726 static bool trans_SINCDEC_r_32(DisasContext
*s
, arg_incdec_cnt
*a
)
1728 if (!sve_access_check(s
)) {
1732 unsigned fullsz
= vec_full_reg_size(s
);
1733 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
1734 int inc
= numelem
* a
->imm
;
1735 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
1737 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
1740 tcg_gen_ext32u_i64(reg
, reg
);
1742 tcg_gen_ext32s_i64(reg
, reg
);
1745 TCGv_i64 t
= tcg_const_i64(inc
);
1746 do_sat_addsub_32(reg
, t
, a
->u
, a
->d
);
1747 tcg_temp_free_i64(t
);
1752 static bool trans_SINCDEC_r_64(DisasContext
*s
, arg_incdec_cnt
*a
)
1754 if (!sve_access_check(s
)) {
1758 unsigned fullsz
= vec_full_reg_size(s
);
1759 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
1760 int inc
= numelem
* a
->imm
;
1761 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
1764 TCGv_i64 t
= tcg_const_i64(inc
);
1765 do_sat_addsub_64(reg
, t
, a
->u
, a
->d
);
1766 tcg_temp_free_i64(t
);
1771 static bool trans_INCDEC_v(DisasContext
*s
, arg_incdec2_cnt
*a
)
1777 unsigned fullsz
= vec_full_reg_size(s
);
1778 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
1779 int inc
= numelem
* a
->imm
;
1782 if (sve_access_check(s
)) {
1783 TCGv_i64 t
= tcg_const_i64(a
->d
? -inc
: inc
);
1784 tcg_gen_gvec_adds(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
1785 vec_full_reg_offset(s
, a
->rn
),
1787 tcg_temp_free_i64(t
);
1790 do_mov_z(s
, a
->rd
, a
->rn
);
1795 static bool trans_SINCDEC_v(DisasContext
*s
, arg_incdec2_cnt
*a
)
1801 unsigned fullsz
= vec_full_reg_size(s
);
1802 unsigned numelem
= decode_pred_count(fullsz
, a
->pat
, a
->esz
);
1803 int inc
= numelem
* a
->imm
;
1806 if (sve_access_check(s
)) {
1807 TCGv_i64 t
= tcg_const_i64(inc
);
1808 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
, t
, a
->u
, a
->d
);
1809 tcg_temp_free_i64(t
);
1812 do_mov_z(s
, a
->rd
, a
->rn
);
1818 *** SVE Bitwise Immediate Group
1821 static bool do_zz_dbm(DisasContext
*s
, arg_rr_dbm
*a
, GVecGen2iFn
*gvec_fn
)
1824 if (!logic_imm_decode_wmask(&imm
, extract32(a
->dbm
, 12, 1),
1825 extract32(a
->dbm
, 0, 6),
1826 extract32(a
->dbm
, 6, 6))) {
1829 if (sve_access_check(s
)) {
1830 unsigned vsz
= vec_full_reg_size(s
);
1831 gvec_fn(MO_64
, vec_full_reg_offset(s
, a
->rd
),
1832 vec_full_reg_offset(s
, a
->rn
), imm
, vsz
, vsz
);
1837 static bool trans_AND_zzi(DisasContext
*s
, arg_rr_dbm
*a
)
1839 return do_zz_dbm(s
, a
, tcg_gen_gvec_andi
);
1842 static bool trans_ORR_zzi(DisasContext
*s
, arg_rr_dbm
*a
)
1844 return do_zz_dbm(s
, a
, tcg_gen_gvec_ori
);
1847 static bool trans_EOR_zzi(DisasContext
*s
, arg_rr_dbm
*a
)
1849 return do_zz_dbm(s
, a
, tcg_gen_gvec_xori
);
1852 static bool trans_DUPM(DisasContext
*s
, arg_DUPM
*a
)
1855 if (!logic_imm_decode_wmask(&imm
, extract32(a
->dbm
, 12, 1),
1856 extract32(a
->dbm
, 0, 6),
1857 extract32(a
->dbm
, 6, 6))) {
1860 if (sve_access_check(s
)) {
1861 do_dupi_z(s
, a
->rd
, imm
);
1867 *** SVE Integer Wide Immediate - Predicated Group
1870 /* Implement all merging copies. This is used for CPY (immediate),
1871 * FCPY, CPY (scalar), CPY (SIMD&FP scalar).
1873 static void do_cpy_m(DisasContext
*s
, int esz
, int rd
, int rn
, int pg
,
1876 typedef void gen_cpy(TCGv_ptr
, TCGv_ptr
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
1877 static gen_cpy
* const fns
[4] = {
1878 gen_helper_sve_cpy_m_b
, gen_helper_sve_cpy_m_h
,
1879 gen_helper_sve_cpy_m_s
, gen_helper_sve_cpy_m_d
,
1881 unsigned vsz
= vec_full_reg_size(s
);
1882 TCGv_i32 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
1883 TCGv_ptr t_zd
= tcg_temp_new_ptr();
1884 TCGv_ptr t_zn
= tcg_temp_new_ptr();
1885 TCGv_ptr t_pg
= tcg_temp_new_ptr();
1887 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, rd
));
1888 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, rn
));
1889 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
1891 fns
[esz
](t_zd
, t_zn
, t_pg
, val
, desc
);
1893 tcg_temp_free_ptr(t_zd
);
1894 tcg_temp_free_ptr(t_zn
);
1895 tcg_temp_free_ptr(t_pg
);
1896 tcg_temp_free_i32(desc
);
1899 static bool trans_FCPY(DisasContext
*s
, arg_FCPY
*a
)
1904 if (sve_access_check(s
)) {
1905 /* Decode the VFP immediate. */
1906 uint64_t imm
= vfp_expand_imm(a
->esz
, a
->imm
);
1907 TCGv_i64 t_imm
= tcg_const_i64(imm
);
1908 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rn
, a
->pg
, t_imm
);
1909 tcg_temp_free_i64(t_imm
);
1914 static bool trans_CPY_m_i(DisasContext
*s
, arg_rpri_esz
*a
)
1916 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
1919 if (sve_access_check(s
)) {
1920 TCGv_i64 t_imm
= tcg_const_i64(a
->imm
);
1921 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rn
, a
->pg
, t_imm
);
1922 tcg_temp_free_i64(t_imm
);
1927 static bool trans_CPY_z_i(DisasContext
*s
, arg_CPY_z_i
*a
)
1929 static gen_helper_gvec_2i
* const fns
[4] = {
1930 gen_helper_sve_cpy_z_b
, gen_helper_sve_cpy_z_h
,
1931 gen_helper_sve_cpy_z_s
, gen_helper_sve_cpy_z_d
,
1934 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
1937 if (sve_access_check(s
)) {
1938 unsigned vsz
= vec_full_reg_size(s
);
1939 TCGv_i64 t_imm
= tcg_const_i64(a
->imm
);
1940 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s
, a
->rd
),
1941 pred_full_reg_offset(s
, a
->pg
),
1942 t_imm
, vsz
, vsz
, 0, fns
[a
->esz
]);
1943 tcg_temp_free_i64(t_imm
);
1949 *** SVE Permute Extract Group
1952 static bool trans_EXT(DisasContext
*s
, arg_EXT
*a
)
1954 if (!sve_access_check(s
)) {
1958 unsigned vsz
= vec_full_reg_size(s
);
1959 unsigned n_ofs
= a
->imm
>= vsz
? 0 : a
->imm
;
1960 unsigned n_siz
= vsz
- n_ofs
;
1961 unsigned d
= vec_full_reg_offset(s
, a
->rd
);
1962 unsigned n
= vec_full_reg_offset(s
, a
->rn
);
1963 unsigned m
= vec_full_reg_offset(s
, a
->rm
);
1965 /* Use host vector move insns if we have appropriate sizes
1966 * and no unfortunate overlap.
1969 && n_ofs
== size_for_gvec(n_ofs
)
1970 && n_siz
== size_for_gvec(n_siz
)
1971 && (d
!= n
|| n_siz
<= n_ofs
)) {
1972 tcg_gen_gvec_mov(0, d
, n
+ n_ofs
, n_siz
, n_siz
);
1974 tcg_gen_gvec_mov(0, d
+ n_siz
, m
, n_ofs
, n_ofs
);
1977 tcg_gen_gvec_3_ool(d
, n
, m
, vsz
, vsz
, n_ofs
, gen_helper_sve_ext
);
1983 *** SVE Permute - Unpredicated Group
1986 static bool trans_DUP_s(DisasContext
*s
, arg_DUP_s
*a
)
1988 if (sve_access_check(s
)) {
1989 unsigned vsz
= vec_full_reg_size(s
);
1990 tcg_gen_gvec_dup_i64(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
1991 vsz
, vsz
, cpu_reg_sp(s
, a
->rn
));
1996 static bool trans_DUP_x(DisasContext
*s
, arg_DUP_x
*a
)
1998 if ((a
->imm
& 0x1f) == 0) {
2001 if (sve_access_check(s
)) {
2002 unsigned vsz
= vec_full_reg_size(s
);
2003 unsigned dofs
= vec_full_reg_offset(s
, a
->rd
);
2004 unsigned esz
, index
;
2006 esz
= ctz32(a
->imm
);
2007 index
= a
->imm
>> (esz
+ 1);
2009 if ((index
<< esz
) < vsz
) {
2010 unsigned nofs
= vec_reg_offset(s
, a
->rn
, index
, esz
);
2011 tcg_gen_gvec_dup_mem(esz
, dofs
, nofs
, vsz
, vsz
);
2014 * While dup_mem handles 128-bit elements, dup_imm does not.
2015 * Thankfully element size doesn't matter for splatting zero.
2017 tcg_gen_gvec_dup_imm(MO_64
, dofs
, vsz
, vsz
, 0);
2023 static void do_insr_i64(DisasContext
*s
, arg_rrr_esz
*a
, TCGv_i64 val
)
2025 typedef void gen_insr(TCGv_ptr
, TCGv_ptr
, TCGv_i64
, TCGv_i32
);
2026 static gen_insr
* const fns
[4] = {
2027 gen_helper_sve_insr_b
, gen_helper_sve_insr_h
,
2028 gen_helper_sve_insr_s
, gen_helper_sve_insr_d
,
2030 unsigned vsz
= vec_full_reg_size(s
);
2031 TCGv_i32 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
2032 TCGv_ptr t_zd
= tcg_temp_new_ptr();
2033 TCGv_ptr t_zn
= tcg_temp_new_ptr();
2035 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, a
->rd
));
2036 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
2038 fns
[a
->esz
](t_zd
, t_zn
, val
, desc
);
2040 tcg_temp_free_ptr(t_zd
);
2041 tcg_temp_free_ptr(t_zn
);
2042 tcg_temp_free_i32(desc
);
2045 static bool trans_INSR_f(DisasContext
*s
, arg_rrr_esz
*a
)
2047 if (sve_access_check(s
)) {
2048 TCGv_i64 t
= tcg_temp_new_i64();
2049 tcg_gen_ld_i64(t
, cpu_env
, vec_reg_offset(s
, a
->rm
, 0, MO_64
));
2050 do_insr_i64(s
, a
, t
);
2051 tcg_temp_free_i64(t
);
2056 static bool trans_INSR_r(DisasContext
*s
, arg_rrr_esz
*a
)
2058 if (sve_access_check(s
)) {
2059 do_insr_i64(s
, a
, cpu_reg(s
, a
->rm
));
2064 static bool trans_REV_v(DisasContext
*s
, arg_rr_esz
*a
)
2066 static gen_helper_gvec_2
* const fns
[4] = {
2067 gen_helper_sve_rev_b
, gen_helper_sve_rev_h
,
2068 gen_helper_sve_rev_s
, gen_helper_sve_rev_d
2071 if (sve_access_check(s
)) {
2072 gen_gvec_ool_zz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, 0);
2077 static bool trans_TBL(DisasContext
*s
, arg_rrr_esz
*a
)
2079 static gen_helper_gvec_3
* const fns
[4] = {
2080 gen_helper_sve_tbl_b
, gen_helper_sve_tbl_h
,
2081 gen_helper_sve_tbl_s
, gen_helper_sve_tbl_d
2084 if (sve_access_check(s
)) {
2085 gen_gvec_ool_zzz(s
, fns
[a
->esz
], a
->rd
, a
->rn
, a
->rm
, 0);
2090 static bool trans_UNPK(DisasContext
*s
, arg_UNPK
*a
)
2092 static gen_helper_gvec_2
* const fns
[4][2] = {
2094 { gen_helper_sve_sunpk_h
, gen_helper_sve_uunpk_h
},
2095 { gen_helper_sve_sunpk_s
, gen_helper_sve_uunpk_s
},
2096 { gen_helper_sve_sunpk_d
, gen_helper_sve_uunpk_d
},
2102 if (sve_access_check(s
)) {
2103 unsigned vsz
= vec_full_reg_size(s
);
2104 tcg_gen_gvec_2_ool(vec_full_reg_offset(s
, a
->rd
),
2105 vec_full_reg_offset(s
, a
->rn
)
2106 + (a
->h
? vsz
/ 2 : 0),
2107 vsz
, vsz
, 0, fns
[a
->esz
][a
->u
]);
2113 *** SVE Permute - Predicates Group
2116 static bool do_perm_pred3(DisasContext
*s
, arg_rrr_esz
*a
, bool high_odd
,
2117 gen_helper_gvec_3
*fn
)
2119 if (!sve_access_check(s
)) {
2123 unsigned vsz
= pred_full_reg_size(s
);
2125 TCGv_ptr t_d
= tcg_temp_new_ptr();
2126 TCGv_ptr t_n
= tcg_temp_new_ptr();
2127 TCGv_ptr t_m
= tcg_temp_new_ptr();
2131 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
);
2132 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
2133 desc
= FIELD_DP32(desc
, PREDDESC
, DATA
, high_odd
);
2135 tcg_gen_addi_ptr(t_d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2136 tcg_gen_addi_ptr(t_n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2137 tcg_gen_addi_ptr(t_m
, cpu_env
, pred_full_reg_offset(s
, a
->rm
));
2138 t_desc
= tcg_const_i32(desc
);
2140 fn(t_d
, t_n
, t_m
, t_desc
);
2142 tcg_temp_free_ptr(t_d
);
2143 tcg_temp_free_ptr(t_n
);
2144 tcg_temp_free_ptr(t_m
);
2145 tcg_temp_free_i32(t_desc
);
2149 static bool do_perm_pred2(DisasContext
*s
, arg_rr_esz
*a
, bool high_odd
,
2150 gen_helper_gvec_2
*fn
)
2152 if (!sve_access_check(s
)) {
2156 unsigned vsz
= pred_full_reg_size(s
);
2157 TCGv_ptr t_d
= tcg_temp_new_ptr();
2158 TCGv_ptr t_n
= tcg_temp_new_ptr();
2162 tcg_gen_addi_ptr(t_d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2163 tcg_gen_addi_ptr(t_n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2165 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
);
2166 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
2167 desc
= FIELD_DP32(desc
, PREDDESC
, DATA
, high_odd
);
2168 t_desc
= tcg_const_i32(desc
);
2170 fn(t_d
, t_n
, t_desc
);
2172 tcg_temp_free_i32(t_desc
);
2173 tcg_temp_free_ptr(t_d
);
2174 tcg_temp_free_ptr(t_n
);
2178 static bool trans_ZIP1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2180 return do_perm_pred3(s
, a
, 0, gen_helper_sve_zip_p
);
2183 static bool trans_ZIP2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2185 return do_perm_pred3(s
, a
, 1, gen_helper_sve_zip_p
);
2188 static bool trans_UZP1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2190 return do_perm_pred3(s
, a
, 0, gen_helper_sve_uzp_p
);
2193 static bool trans_UZP2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2195 return do_perm_pred3(s
, a
, 1, gen_helper_sve_uzp_p
);
2198 static bool trans_TRN1_p(DisasContext
*s
, arg_rrr_esz
*a
)
2200 return do_perm_pred3(s
, a
, 0, gen_helper_sve_trn_p
);
2203 static bool trans_TRN2_p(DisasContext
*s
, arg_rrr_esz
*a
)
2205 return do_perm_pred3(s
, a
, 1, gen_helper_sve_trn_p
);
2208 static bool trans_REV_p(DisasContext
*s
, arg_rr_esz
*a
)
2210 return do_perm_pred2(s
, a
, 0, gen_helper_sve_rev_p
);
2213 static bool trans_PUNPKLO(DisasContext
*s
, arg_PUNPKLO
*a
)
2215 return do_perm_pred2(s
, a
, 0, gen_helper_sve_punpk_p
);
2218 static bool trans_PUNPKHI(DisasContext
*s
, arg_PUNPKHI
*a
)
2220 return do_perm_pred2(s
, a
, 1, gen_helper_sve_punpk_p
);
2224 *** SVE Permute - Interleaving Group
2227 static bool do_zip(DisasContext
*s
, arg_rrr_esz
*a
, bool high
)
2229 static gen_helper_gvec_3
* const fns
[4] = {
2230 gen_helper_sve_zip_b
, gen_helper_sve_zip_h
,
2231 gen_helper_sve_zip_s
, gen_helper_sve_zip_d
,
2234 if (sve_access_check(s
)) {
2235 unsigned vsz
= vec_full_reg_size(s
);
2236 unsigned high_ofs
= high
? vsz
/ 2 : 0;
2237 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, a
->rd
),
2238 vec_full_reg_offset(s
, a
->rn
) + high_ofs
,
2239 vec_full_reg_offset(s
, a
->rm
) + high_ofs
,
2240 vsz
, vsz
, 0, fns
[a
->esz
]);
2245 static bool do_zzz_data_ool(DisasContext
*s
, arg_rrr_esz
*a
, int data
,
2246 gen_helper_gvec_3
*fn
)
2248 if (sve_access_check(s
)) {
2249 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, data
);
2254 static bool trans_ZIP1_z(DisasContext
*s
, arg_rrr_esz
*a
)
2256 return do_zip(s
, a
, false);
2259 static bool trans_ZIP2_z(DisasContext
*s
, arg_rrr_esz
*a
)
2261 return do_zip(s
, a
, true);
2264 static gen_helper_gvec_3
* const uzp_fns
[4] = {
2265 gen_helper_sve_uzp_b
, gen_helper_sve_uzp_h
,
2266 gen_helper_sve_uzp_s
, gen_helper_sve_uzp_d
,
2269 static bool trans_UZP1_z(DisasContext
*s
, arg_rrr_esz
*a
)
2271 return do_zzz_data_ool(s
, a
, 0, uzp_fns
[a
->esz
]);
2274 static bool trans_UZP2_z(DisasContext
*s
, arg_rrr_esz
*a
)
2276 return do_zzz_data_ool(s
, a
, 1 << a
->esz
, uzp_fns
[a
->esz
]);
2279 static gen_helper_gvec_3
* const trn_fns
[4] = {
2280 gen_helper_sve_trn_b
, gen_helper_sve_trn_h
,
2281 gen_helper_sve_trn_s
, gen_helper_sve_trn_d
,
2284 static bool trans_TRN1_z(DisasContext
*s
, arg_rrr_esz
*a
)
2286 return do_zzz_data_ool(s
, a
, 0, trn_fns
[a
->esz
]);
2289 static bool trans_TRN2_z(DisasContext
*s
, arg_rrr_esz
*a
)
2291 return do_zzz_data_ool(s
, a
, 1 << a
->esz
, trn_fns
[a
->esz
]);
2295 *** SVE Permute Vector - Predicated Group
2298 static bool trans_COMPACT(DisasContext
*s
, arg_rpr_esz
*a
)
2300 static gen_helper_gvec_3
* const fns
[4] = {
2301 NULL
, NULL
, gen_helper_sve_compact_s
, gen_helper_sve_compact_d
2303 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
2306 /* Call the helper that computes the ARM LastActiveElement pseudocode
2307 * function, scaled by the element size. This includes the not found
2308 * indication; e.g. not found for esz=3 is -8.
2310 static void find_last_active(DisasContext
*s
, TCGv_i32 ret
, int esz
, int pg
)
2312 /* Predicate sizes may be smaller and cannot use simd_desc. We cannot
2313 * round up, as we do elsewhere, because we need the exact size.
2315 TCGv_ptr t_p
= tcg_temp_new_ptr();
2319 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, pred_full_reg_size(s
));
2320 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, esz
);
2322 tcg_gen_addi_ptr(t_p
, cpu_env
, pred_full_reg_offset(s
, pg
));
2323 t_desc
= tcg_const_i32(desc
);
2325 gen_helper_sve_last_active_element(ret
, t_p
, t_desc
);
2327 tcg_temp_free_i32(t_desc
);
2328 tcg_temp_free_ptr(t_p
);
2331 /* Increment LAST to the offset of the next element in the vector,
2332 * wrapping around to 0.
2334 static void incr_last_active(DisasContext
*s
, TCGv_i32 last
, int esz
)
2336 unsigned vsz
= vec_full_reg_size(s
);
2338 tcg_gen_addi_i32(last
, last
, 1 << esz
);
2339 if (is_power_of_2(vsz
)) {
2340 tcg_gen_andi_i32(last
, last
, vsz
- 1);
2342 TCGv_i32 max
= tcg_const_i32(vsz
);
2343 TCGv_i32 zero
= tcg_const_i32(0);
2344 tcg_gen_movcond_i32(TCG_COND_GEU
, last
, last
, max
, zero
, last
);
2345 tcg_temp_free_i32(max
);
2346 tcg_temp_free_i32(zero
);
2350 /* If LAST < 0, set LAST to the offset of the last element in the vector. */
2351 static void wrap_last_active(DisasContext
*s
, TCGv_i32 last
, int esz
)
2353 unsigned vsz
= vec_full_reg_size(s
);
2355 if (is_power_of_2(vsz
)) {
2356 tcg_gen_andi_i32(last
, last
, vsz
- 1);
2358 TCGv_i32 max
= tcg_const_i32(vsz
- (1 << esz
));
2359 TCGv_i32 zero
= tcg_const_i32(0);
2360 tcg_gen_movcond_i32(TCG_COND_LT
, last
, last
, zero
, max
, last
);
2361 tcg_temp_free_i32(max
);
2362 tcg_temp_free_i32(zero
);
2366 /* Load an unsigned element of ESZ from BASE+OFS. */
2367 static TCGv_i64
load_esz(TCGv_ptr base
, int ofs
, int esz
)
2369 TCGv_i64 r
= tcg_temp_new_i64();
2373 tcg_gen_ld8u_i64(r
, base
, ofs
);
2376 tcg_gen_ld16u_i64(r
, base
, ofs
);
2379 tcg_gen_ld32u_i64(r
, base
, ofs
);
2382 tcg_gen_ld_i64(r
, base
, ofs
);
2385 g_assert_not_reached();
2390 /* Load an unsigned element of ESZ from RM[LAST]. */
2391 static TCGv_i64
load_last_active(DisasContext
*s
, TCGv_i32 last
,
2394 TCGv_ptr p
= tcg_temp_new_ptr();
2397 /* Convert offset into vector into offset into ENV.
2398 * The final adjustment for the vector register base
2399 * is added via constant offset to the load.
2401 #ifdef HOST_WORDS_BIGENDIAN
2402 /* Adjust for element ordering. See vec_reg_offset. */
2404 tcg_gen_xori_i32(last
, last
, 8 - (1 << esz
));
2407 tcg_gen_ext_i32_ptr(p
, last
);
2408 tcg_gen_add_ptr(p
, p
, cpu_env
);
2410 r
= load_esz(p
, vec_full_reg_offset(s
, rm
), esz
);
2411 tcg_temp_free_ptr(p
);
2416 /* Compute CLAST for a Zreg. */
2417 static bool do_clast_vector(DisasContext
*s
, arg_rprr_esz
*a
, bool before
)
2422 unsigned vsz
, esz
= a
->esz
;
2424 if (!sve_access_check(s
)) {
2428 last
= tcg_temp_local_new_i32();
2429 over
= gen_new_label();
2431 find_last_active(s
, last
, esz
, a
->pg
);
2433 /* There is of course no movcond for a 2048-bit vector,
2434 * so we must branch over the actual store.
2436 tcg_gen_brcondi_i32(TCG_COND_LT
, last
, 0, over
);
2439 incr_last_active(s
, last
, esz
);
2442 ele
= load_last_active(s
, last
, a
->rm
, esz
);
2443 tcg_temp_free_i32(last
);
2445 vsz
= vec_full_reg_size(s
);
2446 tcg_gen_gvec_dup_i64(esz
, vec_full_reg_offset(s
, a
->rd
), vsz
, vsz
, ele
);
2447 tcg_temp_free_i64(ele
);
2449 /* If this insn used MOVPRFX, we may need a second move. */
2450 if (a
->rd
!= a
->rn
) {
2451 TCGLabel
*done
= gen_new_label();
2454 gen_set_label(over
);
2455 do_mov_z(s
, a
->rd
, a
->rn
);
2457 gen_set_label(done
);
2459 gen_set_label(over
);
2464 static bool trans_CLASTA_z(DisasContext
*s
, arg_rprr_esz
*a
)
2466 return do_clast_vector(s
, a
, false);
2469 static bool trans_CLASTB_z(DisasContext
*s
, arg_rprr_esz
*a
)
2471 return do_clast_vector(s
, a
, true);
2474 /* Compute CLAST for a scalar. */
2475 static void do_clast_scalar(DisasContext
*s
, int esz
, int pg
, int rm
,
2476 bool before
, TCGv_i64 reg_val
)
2478 TCGv_i32 last
= tcg_temp_new_i32();
2479 TCGv_i64 ele
, cmp
, zero
;
2481 find_last_active(s
, last
, esz
, pg
);
2483 /* Extend the original value of last prior to incrementing. */
2484 cmp
= tcg_temp_new_i64();
2485 tcg_gen_ext_i32_i64(cmp
, last
);
2488 incr_last_active(s
, last
, esz
);
2491 /* The conceit here is that while last < 0 indicates not found, after
2492 * adjusting for cpu_env->vfp.zregs[rm], it is still a valid address
2493 * from which we can load garbage. We then discard the garbage with
2494 * a conditional move.
2496 ele
= load_last_active(s
, last
, rm
, esz
);
2497 tcg_temp_free_i32(last
);
2499 zero
= tcg_const_i64(0);
2500 tcg_gen_movcond_i64(TCG_COND_GE
, reg_val
, cmp
, zero
, ele
, reg_val
);
2502 tcg_temp_free_i64(zero
);
2503 tcg_temp_free_i64(cmp
);
2504 tcg_temp_free_i64(ele
);
2507 /* Compute CLAST for a Vreg. */
2508 static bool do_clast_fp(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2510 if (sve_access_check(s
)) {
2512 int ofs
= vec_reg_offset(s
, a
->rd
, 0, esz
);
2513 TCGv_i64 reg
= load_esz(cpu_env
, ofs
, esz
);
2515 do_clast_scalar(s
, esz
, a
->pg
, a
->rn
, before
, reg
);
2516 write_fp_dreg(s
, a
->rd
, reg
);
2517 tcg_temp_free_i64(reg
);
2522 static bool trans_CLASTA_v(DisasContext
*s
, arg_rpr_esz
*a
)
2524 return do_clast_fp(s
, a
, false);
2527 static bool trans_CLASTB_v(DisasContext
*s
, arg_rpr_esz
*a
)
2529 return do_clast_fp(s
, a
, true);
2532 /* Compute CLAST for a Xreg. */
2533 static bool do_clast_general(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2537 if (!sve_access_check(s
)) {
2541 reg
= cpu_reg(s
, a
->rd
);
2544 tcg_gen_ext8u_i64(reg
, reg
);
2547 tcg_gen_ext16u_i64(reg
, reg
);
2550 tcg_gen_ext32u_i64(reg
, reg
);
2555 g_assert_not_reached();
2558 do_clast_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
, reg
);
2562 static bool trans_CLASTA_r(DisasContext
*s
, arg_rpr_esz
*a
)
2564 return do_clast_general(s
, a
, false);
2567 static bool trans_CLASTB_r(DisasContext
*s
, arg_rpr_esz
*a
)
2569 return do_clast_general(s
, a
, true);
2572 /* Compute LAST for a scalar. */
2573 static TCGv_i64
do_last_scalar(DisasContext
*s
, int esz
,
2574 int pg
, int rm
, bool before
)
2576 TCGv_i32 last
= tcg_temp_new_i32();
2579 find_last_active(s
, last
, esz
, pg
);
2581 wrap_last_active(s
, last
, esz
);
2583 incr_last_active(s
, last
, esz
);
2586 ret
= load_last_active(s
, last
, rm
, esz
);
2587 tcg_temp_free_i32(last
);
2591 /* Compute LAST for a Vreg. */
2592 static bool do_last_fp(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2594 if (sve_access_check(s
)) {
2595 TCGv_i64 val
= do_last_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
);
2596 write_fp_dreg(s
, a
->rd
, val
);
2597 tcg_temp_free_i64(val
);
2602 static bool trans_LASTA_v(DisasContext
*s
, arg_rpr_esz
*a
)
2604 return do_last_fp(s
, a
, false);
2607 static bool trans_LASTB_v(DisasContext
*s
, arg_rpr_esz
*a
)
2609 return do_last_fp(s
, a
, true);
2612 /* Compute LAST for a Xreg. */
2613 static bool do_last_general(DisasContext
*s
, arg_rpr_esz
*a
, bool before
)
2615 if (sve_access_check(s
)) {
2616 TCGv_i64 val
= do_last_scalar(s
, a
->esz
, a
->pg
, a
->rn
, before
);
2617 tcg_gen_mov_i64(cpu_reg(s
, a
->rd
), val
);
2618 tcg_temp_free_i64(val
);
2623 static bool trans_LASTA_r(DisasContext
*s
, arg_rpr_esz
*a
)
2625 return do_last_general(s
, a
, false);
2628 static bool trans_LASTB_r(DisasContext
*s
, arg_rpr_esz
*a
)
2630 return do_last_general(s
, a
, true);
2633 static bool trans_CPY_m_r(DisasContext
*s
, arg_rpr_esz
*a
)
2635 if (sve_access_check(s
)) {
2636 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rd
, a
->pg
, cpu_reg_sp(s
, a
->rn
));
2641 static bool trans_CPY_m_v(DisasContext
*s
, arg_rpr_esz
*a
)
2643 if (sve_access_check(s
)) {
2644 int ofs
= vec_reg_offset(s
, a
->rn
, 0, a
->esz
);
2645 TCGv_i64 t
= load_esz(cpu_env
, ofs
, a
->esz
);
2646 do_cpy_m(s
, a
->esz
, a
->rd
, a
->rd
, a
->pg
, t
);
2647 tcg_temp_free_i64(t
);
2652 static bool trans_REVB(DisasContext
*s
, arg_rpr_esz
*a
)
2654 static gen_helper_gvec_3
* const fns
[4] = {
2656 gen_helper_sve_revb_h
,
2657 gen_helper_sve_revb_s
,
2658 gen_helper_sve_revb_d
,
2660 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
2663 static bool trans_REVH(DisasContext
*s
, arg_rpr_esz
*a
)
2665 static gen_helper_gvec_3
* const fns
[4] = {
2668 gen_helper_sve_revh_s
,
2669 gen_helper_sve_revh_d
,
2671 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
2674 static bool trans_REVW(DisasContext
*s
, arg_rpr_esz
*a
)
2676 return do_zpz_ool(s
, a
, a
->esz
== 3 ? gen_helper_sve_revw_d
: NULL
);
2679 static bool trans_RBIT(DisasContext
*s
, arg_rpr_esz
*a
)
2681 static gen_helper_gvec_3
* const fns
[4] = {
2682 gen_helper_sve_rbit_b
,
2683 gen_helper_sve_rbit_h
,
2684 gen_helper_sve_rbit_s
,
2685 gen_helper_sve_rbit_d
,
2687 return do_zpz_ool(s
, a
, fns
[a
->esz
]);
2690 static bool trans_SPLICE(DisasContext
*s
, arg_rprr_esz
*a
)
2692 if (sve_access_check(s
)) {
2693 gen_gvec_ool_zzzp(s
, gen_helper_sve_splice
,
2694 a
->rd
, a
->rn
, a
->rm
, a
->pg
, a
->esz
);
2700 *** SVE Integer Compare - Vectors Group
2703 static bool do_ppzz_flags(DisasContext
*s
, arg_rprr_esz
*a
,
2704 gen_helper_gvec_flags_4
*gen_fn
)
2706 TCGv_ptr pd
, zn
, zm
, pg
;
2710 if (gen_fn
== NULL
) {
2713 if (!sve_access_check(s
)) {
2717 vsz
= vec_full_reg_size(s
);
2718 t
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
2719 pd
= tcg_temp_new_ptr();
2720 zn
= tcg_temp_new_ptr();
2721 zm
= tcg_temp_new_ptr();
2722 pg
= tcg_temp_new_ptr();
2724 tcg_gen_addi_ptr(pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2725 tcg_gen_addi_ptr(zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
2726 tcg_gen_addi_ptr(zm
, cpu_env
, vec_full_reg_offset(s
, a
->rm
));
2727 tcg_gen_addi_ptr(pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
2729 gen_fn(t
, pd
, zn
, zm
, pg
, t
);
2731 tcg_temp_free_ptr(pd
);
2732 tcg_temp_free_ptr(zn
);
2733 tcg_temp_free_ptr(zm
);
2734 tcg_temp_free_ptr(pg
);
2738 tcg_temp_free_i32(t
);
2742 #define DO_PPZZ(NAME, name) \
2743 static bool trans_##NAME##_ppzz(DisasContext *s, arg_rprr_esz *a) \
2745 static gen_helper_gvec_flags_4 * const fns[4] = { \
2746 gen_helper_sve_##name##_ppzz_b, gen_helper_sve_##name##_ppzz_h, \
2747 gen_helper_sve_##name##_ppzz_s, gen_helper_sve_##name##_ppzz_d, \
2749 return do_ppzz_flags(s, a, fns[a->esz]); \
2752 DO_PPZZ(CMPEQ
, cmpeq
)
2753 DO_PPZZ(CMPNE
, cmpne
)
2754 DO_PPZZ(CMPGT
, cmpgt
)
2755 DO_PPZZ(CMPGE
, cmpge
)
2756 DO_PPZZ(CMPHI
, cmphi
)
2757 DO_PPZZ(CMPHS
, cmphs
)
2761 #define DO_PPZW(NAME, name) \
2762 static bool trans_##NAME##_ppzw(DisasContext *s, arg_rprr_esz *a) \
2764 static gen_helper_gvec_flags_4 * const fns[4] = { \
2765 gen_helper_sve_##name##_ppzw_b, gen_helper_sve_##name##_ppzw_h, \
2766 gen_helper_sve_##name##_ppzw_s, NULL \
2768 return do_ppzz_flags(s, a, fns[a->esz]); \
2771 DO_PPZW(CMPEQ
, cmpeq
)
2772 DO_PPZW(CMPNE
, cmpne
)
2773 DO_PPZW(CMPGT
, cmpgt
)
2774 DO_PPZW(CMPGE
, cmpge
)
2775 DO_PPZW(CMPHI
, cmphi
)
2776 DO_PPZW(CMPHS
, cmphs
)
2777 DO_PPZW(CMPLT
, cmplt
)
2778 DO_PPZW(CMPLE
, cmple
)
2779 DO_PPZW(CMPLO
, cmplo
)
2780 DO_PPZW(CMPLS
, cmpls
)
2785 *** SVE Integer Compare - Immediate Groups
2788 static bool do_ppzi_flags(DisasContext
*s
, arg_rpri_esz
*a
,
2789 gen_helper_gvec_flags_3
*gen_fn
)
2791 TCGv_ptr pd
, zn
, pg
;
2795 if (gen_fn
== NULL
) {
2798 if (!sve_access_check(s
)) {
2802 vsz
= vec_full_reg_size(s
);
2803 t
= tcg_const_i32(simd_desc(vsz
, vsz
, a
->imm
));
2804 pd
= tcg_temp_new_ptr();
2805 zn
= tcg_temp_new_ptr();
2806 pg
= tcg_temp_new_ptr();
2808 tcg_gen_addi_ptr(pd
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2809 tcg_gen_addi_ptr(zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
2810 tcg_gen_addi_ptr(pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
2812 gen_fn(t
, pd
, zn
, pg
, t
);
2814 tcg_temp_free_ptr(pd
);
2815 tcg_temp_free_ptr(zn
);
2816 tcg_temp_free_ptr(pg
);
2820 tcg_temp_free_i32(t
);
2824 #define DO_PPZI(NAME, name) \
2825 static bool trans_##NAME##_ppzi(DisasContext *s, arg_rpri_esz *a) \
2827 static gen_helper_gvec_flags_3 * const fns[4] = { \
2828 gen_helper_sve_##name##_ppzi_b, gen_helper_sve_##name##_ppzi_h, \
2829 gen_helper_sve_##name##_ppzi_s, gen_helper_sve_##name##_ppzi_d, \
2831 return do_ppzi_flags(s, a, fns[a->esz]); \
2834 DO_PPZI(CMPEQ
, cmpeq
)
2835 DO_PPZI(CMPNE
, cmpne
)
2836 DO_PPZI(CMPGT
, cmpgt
)
2837 DO_PPZI(CMPGE
, cmpge
)
2838 DO_PPZI(CMPHI
, cmphi
)
2839 DO_PPZI(CMPHS
, cmphs
)
2840 DO_PPZI(CMPLT
, cmplt
)
2841 DO_PPZI(CMPLE
, cmple
)
2842 DO_PPZI(CMPLO
, cmplo
)
2843 DO_PPZI(CMPLS
, cmpls
)
2848 *** SVE Partition Break Group
2851 static bool do_brk3(DisasContext
*s
, arg_rprr_s
*a
,
2852 gen_helper_gvec_4
*fn
, gen_helper_gvec_flags_4
*fn_s
)
2854 if (!sve_access_check(s
)) {
2858 unsigned vsz
= pred_full_reg_size(s
);
2860 /* Predicate sizes may be smaller and cannot use simd_desc. */
2861 TCGv_ptr d
= tcg_temp_new_ptr();
2862 TCGv_ptr n
= tcg_temp_new_ptr();
2863 TCGv_ptr m
= tcg_temp_new_ptr();
2864 TCGv_ptr g
= tcg_temp_new_ptr();
2865 TCGv_i32 t
= tcg_const_i32(FIELD_DP32(0, PREDDESC
, OPRSZ
, vsz
));
2867 tcg_gen_addi_ptr(d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2868 tcg_gen_addi_ptr(n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2869 tcg_gen_addi_ptr(m
, cpu_env
, pred_full_reg_offset(s
, a
->rm
));
2870 tcg_gen_addi_ptr(g
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
2873 fn_s(t
, d
, n
, m
, g
, t
);
2878 tcg_temp_free_ptr(d
);
2879 tcg_temp_free_ptr(n
);
2880 tcg_temp_free_ptr(m
);
2881 tcg_temp_free_ptr(g
);
2882 tcg_temp_free_i32(t
);
2886 static bool do_brk2(DisasContext
*s
, arg_rpr_s
*a
,
2887 gen_helper_gvec_3
*fn
, gen_helper_gvec_flags_3
*fn_s
)
2889 if (!sve_access_check(s
)) {
2893 unsigned vsz
= pred_full_reg_size(s
);
2895 /* Predicate sizes may be smaller and cannot use simd_desc. */
2896 TCGv_ptr d
= tcg_temp_new_ptr();
2897 TCGv_ptr n
= tcg_temp_new_ptr();
2898 TCGv_ptr g
= tcg_temp_new_ptr();
2899 TCGv_i32 t
= tcg_const_i32(FIELD_DP32(0, PREDDESC
, OPRSZ
, vsz
));
2901 tcg_gen_addi_ptr(d
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
2902 tcg_gen_addi_ptr(n
, cpu_env
, pred_full_reg_offset(s
, a
->rn
));
2903 tcg_gen_addi_ptr(g
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
2906 fn_s(t
, d
, n
, g
, t
);
2911 tcg_temp_free_ptr(d
);
2912 tcg_temp_free_ptr(n
);
2913 tcg_temp_free_ptr(g
);
2914 tcg_temp_free_i32(t
);
2918 static bool trans_BRKPA(DisasContext
*s
, arg_rprr_s
*a
)
2920 return do_brk3(s
, a
, gen_helper_sve_brkpa
, gen_helper_sve_brkpas
);
2923 static bool trans_BRKPB(DisasContext
*s
, arg_rprr_s
*a
)
2925 return do_brk3(s
, a
, gen_helper_sve_brkpb
, gen_helper_sve_brkpbs
);
2928 static bool trans_BRKA_m(DisasContext
*s
, arg_rpr_s
*a
)
2930 return do_brk2(s
, a
, gen_helper_sve_brka_m
, gen_helper_sve_brkas_m
);
2933 static bool trans_BRKB_m(DisasContext
*s
, arg_rpr_s
*a
)
2935 return do_brk2(s
, a
, gen_helper_sve_brkb_m
, gen_helper_sve_brkbs_m
);
2938 static bool trans_BRKA_z(DisasContext
*s
, arg_rpr_s
*a
)
2940 return do_brk2(s
, a
, gen_helper_sve_brka_z
, gen_helper_sve_brkas_z
);
2943 static bool trans_BRKB_z(DisasContext
*s
, arg_rpr_s
*a
)
2945 return do_brk2(s
, a
, gen_helper_sve_brkb_z
, gen_helper_sve_brkbs_z
);
2948 static bool trans_BRKN(DisasContext
*s
, arg_rpr_s
*a
)
2950 return do_brk2(s
, a
, gen_helper_sve_brkn
, gen_helper_sve_brkns
);
2954 *** SVE Predicate Count Group
2957 static void do_cntp(DisasContext
*s
, TCGv_i64 val
, int esz
, int pn
, int pg
)
2959 unsigned psz
= pred_full_reg_size(s
);
2964 tcg_gen_ld_i64(val
, cpu_env
, pred_full_reg_offset(s
, pn
));
2966 TCGv_i64 g
= tcg_temp_new_i64();
2967 tcg_gen_ld_i64(g
, cpu_env
, pred_full_reg_offset(s
, pg
));
2968 tcg_gen_and_i64(val
, val
, g
);
2969 tcg_temp_free_i64(g
);
2972 /* Reduce the pred_esz_masks value simply to reduce the
2973 * size of the code generated here.
2975 psz_mask
= MAKE_64BIT_MASK(0, psz
* 8);
2976 tcg_gen_andi_i64(val
, val
, pred_esz_masks
[esz
] & psz_mask
);
2978 tcg_gen_ctpop_i64(val
, val
);
2980 TCGv_ptr t_pn
= tcg_temp_new_ptr();
2981 TCGv_ptr t_pg
= tcg_temp_new_ptr();
2985 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, psz
);
2986 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, esz
);
2988 tcg_gen_addi_ptr(t_pn
, cpu_env
, pred_full_reg_offset(s
, pn
));
2989 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
2990 t_desc
= tcg_const_i32(desc
);
2992 gen_helper_sve_cntp(val
, t_pn
, t_pg
, t_desc
);
2993 tcg_temp_free_ptr(t_pn
);
2994 tcg_temp_free_ptr(t_pg
);
2995 tcg_temp_free_i32(t_desc
);
2999 static bool trans_CNTP(DisasContext
*s
, arg_CNTP
*a
)
3001 if (sve_access_check(s
)) {
3002 do_cntp(s
, cpu_reg(s
, a
->rd
), a
->esz
, a
->rn
, a
->pg
);
3007 static bool trans_INCDECP_r(DisasContext
*s
, arg_incdec_pred
*a
)
3009 if (sve_access_check(s
)) {
3010 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3011 TCGv_i64 val
= tcg_temp_new_i64();
3013 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3015 tcg_gen_sub_i64(reg
, reg
, val
);
3017 tcg_gen_add_i64(reg
, reg
, val
);
3019 tcg_temp_free_i64(val
);
3024 static bool trans_INCDECP_z(DisasContext
*s
, arg_incdec2_pred
*a
)
3029 if (sve_access_check(s
)) {
3030 unsigned vsz
= vec_full_reg_size(s
);
3031 TCGv_i64 val
= tcg_temp_new_i64();
3032 GVecGen2sFn
*gvec_fn
= a
->d
? tcg_gen_gvec_subs
: tcg_gen_gvec_adds
;
3034 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3035 gvec_fn(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3036 vec_full_reg_offset(s
, a
->rn
), val
, vsz
, vsz
);
3041 static bool trans_SINCDECP_r_32(DisasContext
*s
, arg_incdec_pred
*a
)
3043 if (sve_access_check(s
)) {
3044 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3045 TCGv_i64 val
= tcg_temp_new_i64();
3047 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3048 do_sat_addsub_32(reg
, val
, a
->u
, a
->d
);
3053 static bool trans_SINCDECP_r_64(DisasContext
*s
, arg_incdec_pred
*a
)
3055 if (sve_access_check(s
)) {
3056 TCGv_i64 reg
= cpu_reg(s
, a
->rd
);
3057 TCGv_i64 val
= tcg_temp_new_i64();
3059 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3060 do_sat_addsub_64(reg
, val
, a
->u
, a
->d
);
3065 static bool trans_SINCDECP_z(DisasContext
*s
, arg_incdec2_pred
*a
)
3070 if (sve_access_check(s
)) {
3071 TCGv_i64 val
= tcg_temp_new_i64();
3072 do_cntp(s
, val
, a
->esz
, a
->pg
, a
->pg
);
3073 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
, val
, a
->u
, a
->d
);
3079 *** SVE Integer Compare Scalars Group
3082 static bool trans_CTERM(DisasContext
*s
, arg_CTERM
*a
)
3084 if (!sve_access_check(s
)) {
3088 TCGCond cond
= (a
->ne
? TCG_COND_NE
: TCG_COND_EQ
);
3089 TCGv_i64 rn
= read_cpu_reg(s
, a
->rn
, a
->sf
);
3090 TCGv_i64 rm
= read_cpu_reg(s
, a
->rm
, a
->sf
);
3091 TCGv_i64 cmp
= tcg_temp_new_i64();
3093 tcg_gen_setcond_i64(cond
, cmp
, rn
, rm
);
3094 tcg_gen_extrl_i64_i32(cpu_NF
, cmp
);
3095 tcg_temp_free_i64(cmp
);
3097 /* VF = !NF & !CF. */
3098 tcg_gen_xori_i32(cpu_VF
, cpu_NF
, 1);
3099 tcg_gen_andc_i32(cpu_VF
, cpu_VF
, cpu_CF
);
3101 /* Both NF and VF actually look at bit 31. */
3102 tcg_gen_neg_i32(cpu_NF
, cpu_NF
);
3103 tcg_gen_neg_i32(cpu_VF
, cpu_VF
);
3107 static bool trans_WHILE(DisasContext
*s
, arg_WHILE
*a
)
3109 TCGv_i64 op0
, op1
, t0
, t1
, tmax
;
3112 unsigned vsz
= vec_full_reg_size(s
);
3116 if (!sve_access_check(s
)) {
3120 op0
= read_cpu_reg(s
, a
->rn
, 1);
3121 op1
= read_cpu_reg(s
, a
->rm
, 1);
3125 tcg_gen_ext32u_i64(op0
, op0
);
3126 tcg_gen_ext32u_i64(op1
, op1
);
3128 tcg_gen_ext32s_i64(op0
, op0
);
3129 tcg_gen_ext32s_i64(op1
, op1
);
3133 /* For the helper, compress the different conditions into a computation
3134 * of how many iterations for which the condition is true.
3136 t0
= tcg_temp_new_i64();
3137 t1
= tcg_temp_new_i64();
3138 tcg_gen_sub_i64(t0
, op1
, op0
);
3140 tmax
= tcg_const_i64(vsz
>> a
->esz
);
3142 /* Equality means one more iteration. */
3143 tcg_gen_addi_i64(t0
, t0
, 1);
3145 /* If op1 is max (un)signed integer (and the only time the addition
3146 * above could overflow), then we produce an all-true predicate by
3147 * setting the count to the vector length. This is because the
3148 * pseudocode is described as an increment + compare loop, and the
3149 * max integer would always compare true.
3151 tcg_gen_movi_i64(t1
, (a
->sf
3152 ? (a
->u
? UINT64_MAX
: INT64_MAX
)
3153 : (a
->u
? UINT32_MAX
: INT32_MAX
)));
3154 tcg_gen_movcond_i64(TCG_COND_EQ
, t0
, op1
, t1
, tmax
, t0
);
3157 /* Bound to the maximum. */
3158 tcg_gen_umin_i64(t0
, t0
, tmax
);
3159 tcg_temp_free_i64(tmax
);
3161 /* Set the count to zero if the condition is false. */
3163 ? (a
->eq
? TCG_COND_LEU
: TCG_COND_LTU
)
3164 : (a
->eq
? TCG_COND_LE
: TCG_COND_LT
));
3165 tcg_gen_movi_i64(t1
, 0);
3166 tcg_gen_movcond_i64(cond
, t0
, op0
, op1
, t0
, t1
);
3167 tcg_temp_free_i64(t1
);
3169 /* Since we're bounded, pass as a 32-bit type. */
3170 t2
= tcg_temp_new_i32();
3171 tcg_gen_extrl_i64_i32(t2
, t0
);
3172 tcg_temp_free_i64(t0
);
3174 /* Scale elements to bits. */
3175 tcg_gen_shli_i32(t2
, t2
, a
->esz
);
3177 desc
= FIELD_DP32(desc
, PREDDESC
, OPRSZ
, vsz
/ 8);
3178 desc
= FIELD_DP32(desc
, PREDDESC
, ESZ
, a
->esz
);
3179 t3
= tcg_const_i32(desc
);
3181 ptr
= tcg_temp_new_ptr();
3182 tcg_gen_addi_ptr(ptr
, cpu_env
, pred_full_reg_offset(s
, a
->rd
));
3184 gen_helper_sve_while(t2
, ptr
, t2
, t3
);
3187 tcg_temp_free_ptr(ptr
);
3188 tcg_temp_free_i32(t2
);
3189 tcg_temp_free_i32(t3
);
3194 *** SVE Integer Wide Immediate - Unpredicated Group
3197 static bool trans_FDUP(DisasContext
*s
, arg_FDUP
*a
)
3202 if (sve_access_check(s
)) {
3203 unsigned vsz
= vec_full_reg_size(s
);
3204 int dofs
= vec_full_reg_offset(s
, a
->rd
);
3207 /* Decode the VFP immediate. */
3208 imm
= vfp_expand_imm(a
->esz
, a
->imm
);
3209 tcg_gen_gvec_dup_imm(a
->esz
, dofs
, vsz
, vsz
, imm
);
3214 static bool trans_DUP_i(DisasContext
*s
, arg_DUP_i
*a
)
3216 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3219 if (sve_access_check(s
)) {
3220 unsigned vsz
= vec_full_reg_size(s
);
3221 int dofs
= vec_full_reg_offset(s
, a
->rd
);
3223 tcg_gen_gvec_dup_imm(a
->esz
, dofs
, vsz
, vsz
, a
->imm
);
3228 static bool trans_ADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3230 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3233 if (sve_access_check(s
)) {
3234 unsigned vsz
= vec_full_reg_size(s
);
3235 tcg_gen_gvec_addi(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3236 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
3241 static bool trans_SUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3244 return trans_ADD_zzi(s
, a
);
3247 static bool trans_SUBR_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3249 static const TCGOpcode vecop_list
[] = { INDEX_op_sub_vec
, 0 };
3250 static const GVecGen2s op
[4] = {
3251 { .fni8
= tcg_gen_vec_sub8_i64
,
3252 .fniv
= tcg_gen_sub_vec
,
3253 .fno
= gen_helper_sve_subri_b
,
3254 .opt_opc
= vecop_list
,
3256 .scalar_first
= true },
3257 { .fni8
= tcg_gen_vec_sub16_i64
,
3258 .fniv
= tcg_gen_sub_vec
,
3259 .fno
= gen_helper_sve_subri_h
,
3260 .opt_opc
= vecop_list
,
3262 .scalar_first
= true },
3263 { .fni4
= tcg_gen_sub_i32
,
3264 .fniv
= tcg_gen_sub_vec
,
3265 .fno
= gen_helper_sve_subri_s
,
3266 .opt_opc
= vecop_list
,
3268 .scalar_first
= true },
3269 { .fni8
= tcg_gen_sub_i64
,
3270 .fniv
= tcg_gen_sub_vec
,
3271 .fno
= gen_helper_sve_subri_d
,
3272 .opt_opc
= vecop_list
,
3273 .prefer_i64
= TCG_TARGET_REG_BITS
== 64,
3275 .scalar_first
= true }
3278 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3281 if (sve_access_check(s
)) {
3282 unsigned vsz
= vec_full_reg_size(s
);
3283 TCGv_i64 c
= tcg_const_i64(a
->imm
);
3284 tcg_gen_gvec_2s(vec_full_reg_offset(s
, a
->rd
),
3285 vec_full_reg_offset(s
, a
->rn
),
3286 vsz
, vsz
, c
, &op
[a
->esz
]);
3287 tcg_temp_free_i64(c
);
3292 static bool trans_MUL_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3294 if (sve_access_check(s
)) {
3295 unsigned vsz
= vec_full_reg_size(s
);
3296 tcg_gen_gvec_muli(a
->esz
, vec_full_reg_offset(s
, a
->rd
),
3297 vec_full_reg_offset(s
, a
->rn
), a
->imm
, vsz
, vsz
);
3302 static bool do_zzi_sat(DisasContext
*s
, arg_rri_esz
*a
, bool u
, bool d
)
3304 if (a
->esz
== 0 && extract32(s
->insn
, 13, 1)) {
3307 if (sve_access_check(s
)) {
3308 TCGv_i64 val
= tcg_const_i64(a
->imm
);
3309 do_sat_addsub_vec(s
, a
->esz
, a
->rd
, a
->rn
, val
, u
, d
);
3310 tcg_temp_free_i64(val
);
3315 static bool trans_SQADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3317 return do_zzi_sat(s
, a
, false, false);
3320 static bool trans_UQADD_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3322 return do_zzi_sat(s
, a
, true, false);
3325 static bool trans_SQSUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3327 return do_zzi_sat(s
, a
, false, true);
3330 static bool trans_UQSUB_zzi(DisasContext
*s
, arg_rri_esz
*a
)
3332 return do_zzi_sat(s
, a
, true, true);
3335 static bool do_zzi_ool(DisasContext
*s
, arg_rri_esz
*a
, gen_helper_gvec_2i
*fn
)
3337 if (sve_access_check(s
)) {
3338 unsigned vsz
= vec_full_reg_size(s
);
3339 TCGv_i64 c
= tcg_const_i64(a
->imm
);
3341 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s
, a
->rd
),
3342 vec_full_reg_offset(s
, a
->rn
),
3343 c
, vsz
, vsz
, 0, fn
);
3344 tcg_temp_free_i64(c
);
3349 #define DO_ZZI(NAME, name) \
3350 static bool trans_##NAME##_zzi(DisasContext *s, arg_rri_esz *a) \
3352 static gen_helper_gvec_2i * const fns[4] = { \
3353 gen_helper_sve_##name##i_b, gen_helper_sve_##name##i_h, \
3354 gen_helper_sve_##name##i_s, gen_helper_sve_##name##i_d, \
3356 return do_zzi_ool(s, a, fns[a->esz]); \
3366 static bool trans_DOT_zzz(DisasContext
*s
, arg_DOT_zzz
*a
)
3368 static gen_helper_gvec_3
* const fns
[2][2] = {
3369 { gen_helper_gvec_sdot_b
, gen_helper_gvec_sdot_h
},
3370 { gen_helper_gvec_udot_b
, gen_helper_gvec_udot_h
}
3373 if (sve_access_check(s
)) {
3374 gen_gvec_ool_zzz(s
, fns
[a
->u
][a
->sz
], a
->rd
, a
->rn
, a
->rm
, 0);
3379 static bool trans_DOT_zzx(DisasContext
*s
, arg_DOT_zzx
*a
)
3381 static gen_helper_gvec_3
* const fns
[2][2] = {
3382 { gen_helper_gvec_sdot_idx_b
, gen_helper_gvec_sdot_idx_h
},
3383 { gen_helper_gvec_udot_idx_b
, gen_helper_gvec_udot_idx_h
}
3386 if (sve_access_check(s
)) {
3387 gen_gvec_ool_zzz(s
, fns
[a
->u
][a
->sz
], a
->rd
, a
->rn
, a
->rm
, a
->index
);
3394 *** SVE Floating Point Multiply-Add Indexed Group
3397 static bool trans_FMLA_zzxz(DisasContext
*s
, arg_FMLA_zzxz
*a
)
3399 static gen_helper_gvec_4_ptr
* const fns
[3] = {
3400 gen_helper_gvec_fmla_idx_h
,
3401 gen_helper_gvec_fmla_idx_s
,
3402 gen_helper_gvec_fmla_idx_d
,
3405 if (sve_access_check(s
)) {
3406 unsigned vsz
= vec_full_reg_size(s
);
3407 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3408 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
3409 vec_full_reg_offset(s
, a
->rn
),
3410 vec_full_reg_offset(s
, a
->rm
),
3411 vec_full_reg_offset(s
, a
->ra
),
3412 status
, vsz
, vsz
, (a
->index
<< 1) | a
->sub
,
3414 tcg_temp_free_ptr(status
);
3420 *** SVE Floating Point Multiply Indexed Group
3423 static bool trans_FMUL_zzx(DisasContext
*s
, arg_FMUL_zzx
*a
)
3425 static gen_helper_gvec_3_ptr
* const fns
[3] = {
3426 gen_helper_gvec_fmul_idx_h
,
3427 gen_helper_gvec_fmul_idx_s
,
3428 gen_helper_gvec_fmul_idx_d
,
3431 if (sve_access_check(s
)) {
3432 unsigned vsz
= vec_full_reg_size(s
);
3433 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3434 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
3435 vec_full_reg_offset(s
, a
->rn
),
3436 vec_full_reg_offset(s
, a
->rm
),
3437 status
, vsz
, vsz
, a
->index
, fns
[a
->esz
- 1]);
3438 tcg_temp_free_ptr(status
);
3444 *** SVE Floating Point Fast Reduction Group
3447 typedef void gen_helper_fp_reduce(TCGv_i64
, TCGv_ptr
, TCGv_ptr
,
3448 TCGv_ptr
, TCGv_i32
);
3450 static void do_reduce(DisasContext
*s
, arg_rpr_esz
*a
,
3451 gen_helper_fp_reduce
*fn
)
3453 unsigned vsz
= vec_full_reg_size(s
);
3454 unsigned p2vsz
= pow2ceil(vsz
);
3455 TCGv_i32 t_desc
= tcg_const_i32(simd_desc(vsz
, vsz
, p2vsz
));
3456 TCGv_ptr t_zn
, t_pg
, status
;
3459 temp
= tcg_temp_new_i64();
3460 t_zn
= tcg_temp_new_ptr();
3461 t_pg
= tcg_temp_new_ptr();
3463 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, a
->rn
));
3464 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3465 status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3467 fn(temp
, t_zn
, t_pg
, status
, t_desc
);
3468 tcg_temp_free_ptr(t_zn
);
3469 tcg_temp_free_ptr(t_pg
);
3470 tcg_temp_free_ptr(status
);
3471 tcg_temp_free_i32(t_desc
);
3473 write_fp_dreg(s
, a
->rd
, temp
);
3474 tcg_temp_free_i64(temp
);
3477 #define DO_VPZ(NAME, name) \
3478 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
3480 static gen_helper_fp_reduce * const fns[3] = { \
3481 gen_helper_sve_##name##_h, \
3482 gen_helper_sve_##name##_s, \
3483 gen_helper_sve_##name##_d, \
3485 if (a->esz == 0) { \
3488 if (sve_access_check(s)) { \
3489 do_reduce(s, a, fns[a->esz - 1]); \
3494 DO_VPZ(FADDV
, faddv
)
3495 DO_VPZ(FMINNMV
, fminnmv
)
3496 DO_VPZ(FMAXNMV
, fmaxnmv
)
3497 DO_VPZ(FMINV
, fminv
)
3498 DO_VPZ(FMAXV
, fmaxv
)
3501 *** SVE Floating Point Unary Operations - Unpredicated Group
3504 static void do_zz_fp(DisasContext
*s
, arg_rr_esz
*a
, gen_helper_gvec_2_ptr
*fn
)
3506 unsigned vsz
= vec_full_reg_size(s
);
3507 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3509 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s
, a
->rd
),
3510 vec_full_reg_offset(s
, a
->rn
),
3511 status
, vsz
, vsz
, 0, fn
);
3512 tcg_temp_free_ptr(status
);
3515 static bool trans_FRECPE(DisasContext
*s
, arg_rr_esz
*a
)
3517 static gen_helper_gvec_2_ptr
* const fns
[3] = {
3518 gen_helper_gvec_frecpe_h
,
3519 gen_helper_gvec_frecpe_s
,
3520 gen_helper_gvec_frecpe_d
,
3525 if (sve_access_check(s
)) {
3526 do_zz_fp(s
, a
, fns
[a
->esz
- 1]);
3531 static bool trans_FRSQRTE(DisasContext
*s
, arg_rr_esz
*a
)
3533 static gen_helper_gvec_2_ptr
* const fns
[3] = {
3534 gen_helper_gvec_frsqrte_h
,
3535 gen_helper_gvec_frsqrte_s
,
3536 gen_helper_gvec_frsqrte_d
,
3541 if (sve_access_check(s
)) {
3542 do_zz_fp(s
, a
, fns
[a
->esz
- 1]);
3548 *** SVE Floating Point Compare with Zero Group
3551 static void do_ppz_fp(DisasContext
*s
, arg_rpr_esz
*a
,
3552 gen_helper_gvec_3_ptr
*fn
)
3554 unsigned vsz
= vec_full_reg_size(s
);
3555 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3557 tcg_gen_gvec_3_ptr(pred_full_reg_offset(s
, a
->rd
),
3558 vec_full_reg_offset(s
, a
->rn
),
3559 pred_full_reg_offset(s
, a
->pg
),
3560 status
, vsz
, vsz
, 0, fn
);
3561 tcg_temp_free_ptr(status
);
3564 #define DO_PPZ(NAME, name) \
3565 static bool trans_##NAME(DisasContext *s, arg_rpr_esz *a) \
3567 static gen_helper_gvec_3_ptr * const fns[3] = { \
3568 gen_helper_sve_##name##_h, \
3569 gen_helper_sve_##name##_s, \
3570 gen_helper_sve_##name##_d, \
3572 if (a->esz == 0) { \
3575 if (sve_access_check(s)) { \
3576 do_ppz_fp(s, a, fns[a->esz - 1]); \
3581 DO_PPZ(FCMGE_ppz0
, fcmge0
)
3582 DO_PPZ(FCMGT_ppz0
, fcmgt0
)
3583 DO_PPZ(FCMLE_ppz0
, fcmle0
)
3584 DO_PPZ(FCMLT_ppz0
, fcmlt0
)
3585 DO_PPZ(FCMEQ_ppz0
, fcmeq0
)
3586 DO_PPZ(FCMNE_ppz0
, fcmne0
)
3591 *** SVE floating-point trig multiply-add coefficient
3594 static bool trans_FTMAD(DisasContext
*s
, arg_FTMAD
*a
)
3596 static gen_helper_gvec_3_ptr
* const fns
[3] = {
3597 gen_helper_sve_ftmad_h
,
3598 gen_helper_sve_ftmad_s
,
3599 gen_helper_sve_ftmad_d
,
3605 if (sve_access_check(s
)) {
3606 unsigned vsz
= vec_full_reg_size(s
);
3607 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3608 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
3609 vec_full_reg_offset(s
, a
->rn
),
3610 vec_full_reg_offset(s
, a
->rm
),
3611 status
, vsz
, vsz
, a
->imm
, fns
[a
->esz
- 1]);
3612 tcg_temp_free_ptr(status
);
3618 *** SVE Floating Point Accumulating Reduction Group
3621 static bool trans_FADDA(DisasContext
*s
, arg_rprr_esz
*a
)
3623 typedef void fadda_fn(TCGv_i64
, TCGv_i64
, TCGv_ptr
,
3624 TCGv_ptr
, TCGv_ptr
, TCGv_i32
);
3625 static fadda_fn
* const fns
[3] = {
3626 gen_helper_sve_fadda_h
,
3627 gen_helper_sve_fadda_s
,
3628 gen_helper_sve_fadda_d
,
3630 unsigned vsz
= vec_full_reg_size(s
);
3631 TCGv_ptr t_rm
, t_pg
, t_fpst
;
3638 if (!sve_access_check(s
)) {
3642 t_val
= load_esz(cpu_env
, vec_reg_offset(s
, a
->rn
, 0, a
->esz
), a
->esz
);
3643 t_rm
= tcg_temp_new_ptr();
3644 t_pg
= tcg_temp_new_ptr();
3645 tcg_gen_addi_ptr(t_rm
, cpu_env
, vec_full_reg_offset(s
, a
->rm
));
3646 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
3647 t_fpst
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3648 t_desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
3650 fns
[a
->esz
- 1](t_val
, t_val
, t_rm
, t_pg
, t_fpst
, t_desc
);
3652 tcg_temp_free_i32(t_desc
);
3653 tcg_temp_free_ptr(t_fpst
);
3654 tcg_temp_free_ptr(t_pg
);
3655 tcg_temp_free_ptr(t_rm
);
3657 write_fp_dreg(s
, a
->rd
, t_val
);
3658 tcg_temp_free_i64(t_val
);
3663 *** SVE Floating Point Arithmetic - Unpredicated Group
3666 static bool do_zzz_fp(DisasContext
*s
, arg_rrr_esz
*a
,
3667 gen_helper_gvec_3_ptr
*fn
)
3672 if (sve_access_check(s
)) {
3673 unsigned vsz
= vec_full_reg_size(s
);
3674 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3675 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
3676 vec_full_reg_offset(s
, a
->rn
),
3677 vec_full_reg_offset(s
, a
->rm
),
3678 status
, vsz
, vsz
, 0, fn
);
3679 tcg_temp_free_ptr(status
);
3685 #define DO_FP3(NAME, name) \
3686 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
3688 static gen_helper_gvec_3_ptr * const fns[4] = { \
3689 NULL, gen_helper_gvec_##name##_h, \
3690 gen_helper_gvec_##name##_s, gen_helper_gvec_##name##_d \
3692 return do_zzz_fp(s, a, fns[a->esz]); \
3695 DO_FP3(FADD_zzz
, fadd
)
3696 DO_FP3(FSUB_zzz
, fsub
)
3697 DO_FP3(FMUL_zzz
, fmul
)
3698 DO_FP3(FTSMUL
, ftsmul
)
3699 DO_FP3(FRECPS
, recps
)
3700 DO_FP3(FRSQRTS
, rsqrts
)
3705 *** SVE Floating Point Arithmetic - Predicated Group
3708 static bool do_zpzz_fp(DisasContext
*s
, arg_rprr_esz
*a
,
3709 gen_helper_gvec_4_ptr
*fn
)
3714 if (sve_access_check(s
)) {
3715 unsigned vsz
= vec_full_reg_size(s
);
3716 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3717 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
3718 vec_full_reg_offset(s
, a
->rn
),
3719 vec_full_reg_offset(s
, a
->rm
),
3720 pred_full_reg_offset(s
, a
->pg
),
3721 status
, vsz
, vsz
, 0, fn
);
3722 tcg_temp_free_ptr(status
);
3727 #define DO_FP3(NAME, name) \
3728 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
3730 static gen_helper_gvec_4_ptr * const fns[4] = { \
3731 NULL, gen_helper_sve_##name##_h, \
3732 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
3734 return do_zpzz_fp(s, a, fns[a->esz]); \
3737 DO_FP3(FADD_zpzz
, fadd
)
3738 DO_FP3(FSUB_zpzz
, fsub
)
3739 DO_FP3(FMUL_zpzz
, fmul
)
3740 DO_FP3(FMIN_zpzz
, fmin
)
3741 DO_FP3(FMAX_zpzz
, fmax
)
3742 DO_FP3(FMINNM_zpzz
, fminnum
)
3743 DO_FP3(FMAXNM_zpzz
, fmaxnum
)
3745 DO_FP3(FSCALE
, fscalbn
)
3747 DO_FP3(FMULX
, fmulx
)
3751 typedef void gen_helper_sve_fp2scalar(TCGv_ptr
, TCGv_ptr
, TCGv_ptr
,
3752 TCGv_i64
, TCGv_ptr
, TCGv_i32
);
3754 static void do_fp_scalar(DisasContext
*s
, int zd
, int zn
, int pg
, bool is_fp16
,
3755 TCGv_i64 scalar
, gen_helper_sve_fp2scalar
*fn
)
3757 unsigned vsz
= vec_full_reg_size(s
);
3758 TCGv_ptr t_zd
, t_zn
, t_pg
, status
;
3761 t_zd
= tcg_temp_new_ptr();
3762 t_zn
= tcg_temp_new_ptr();
3763 t_pg
= tcg_temp_new_ptr();
3764 tcg_gen_addi_ptr(t_zd
, cpu_env
, vec_full_reg_offset(s
, zd
));
3765 tcg_gen_addi_ptr(t_zn
, cpu_env
, vec_full_reg_offset(s
, zn
));
3766 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
3768 status
= fpstatus_ptr(is_fp16
? FPST_FPCR_F16
: FPST_FPCR
);
3769 desc
= tcg_const_i32(simd_desc(vsz
, vsz
, 0));
3770 fn(t_zd
, t_zn
, t_pg
, scalar
, status
, desc
);
3772 tcg_temp_free_i32(desc
);
3773 tcg_temp_free_ptr(status
);
3774 tcg_temp_free_ptr(t_pg
);
3775 tcg_temp_free_ptr(t_zn
);
3776 tcg_temp_free_ptr(t_zd
);
3779 static void do_fp_imm(DisasContext
*s
, arg_rpri_esz
*a
, uint64_t imm
,
3780 gen_helper_sve_fp2scalar
*fn
)
3782 TCGv_i64 temp
= tcg_const_i64(imm
);
3783 do_fp_scalar(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, temp
, fn
);
3784 tcg_temp_free_i64(temp
);
3787 #define DO_FP_IMM(NAME, name, const0, const1) \
3788 static bool trans_##NAME##_zpzi(DisasContext *s, arg_rpri_esz *a) \
3790 static gen_helper_sve_fp2scalar * const fns[3] = { \
3791 gen_helper_sve_##name##_h, \
3792 gen_helper_sve_##name##_s, \
3793 gen_helper_sve_##name##_d \
3795 static uint64_t const val[3][2] = { \
3796 { float16_##const0, float16_##const1 }, \
3797 { float32_##const0, float32_##const1 }, \
3798 { float64_##const0, float64_##const1 }, \
3800 if (a->esz == 0) { \
3803 if (sve_access_check(s)) { \
3804 do_fp_imm(s, a, val[a->esz - 1][a->imm], fns[a->esz - 1]); \
3809 DO_FP_IMM(FADD
, fadds
, half
, one
)
3810 DO_FP_IMM(FSUB
, fsubs
, half
, one
)
3811 DO_FP_IMM(FMUL
, fmuls
, half
, two
)
3812 DO_FP_IMM(FSUBR
, fsubrs
, half
, one
)
3813 DO_FP_IMM(FMAXNM
, fmaxnms
, zero
, one
)
3814 DO_FP_IMM(FMINNM
, fminnms
, zero
, one
)
3815 DO_FP_IMM(FMAX
, fmaxs
, zero
, one
)
3816 DO_FP_IMM(FMIN
, fmins
, zero
, one
)
3820 static bool do_fp_cmp(DisasContext
*s
, arg_rprr_esz
*a
,
3821 gen_helper_gvec_4_ptr
*fn
)
3826 if (sve_access_check(s
)) {
3827 unsigned vsz
= vec_full_reg_size(s
);
3828 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3829 tcg_gen_gvec_4_ptr(pred_full_reg_offset(s
, a
->rd
),
3830 vec_full_reg_offset(s
, a
->rn
),
3831 vec_full_reg_offset(s
, a
->rm
),
3832 pred_full_reg_offset(s
, a
->pg
),
3833 status
, vsz
, vsz
, 0, fn
);
3834 tcg_temp_free_ptr(status
);
3839 #define DO_FPCMP(NAME, name) \
3840 static bool trans_##NAME##_ppzz(DisasContext *s, arg_rprr_esz *a) \
3842 static gen_helper_gvec_4_ptr * const fns[4] = { \
3843 NULL, gen_helper_sve_##name##_h, \
3844 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
3846 return do_fp_cmp(s, a, fns[a->esz]); \
3849 DO_FPCMP(FCMGE
, fcmge
)
3850 DO_FPCMP(FCMGT
, fcmgt
)
3851 DO_FPCMP(FCMEQ
, fcmeq
)
3852 DO_FPCMP(FCMNE
, fcmne
)
3853 DO_FPCMP(FCMUO
, fcmuo
)
3854 DO_FPCMP(FACGE
, facge
)
3855 DO_FPCMP(FACGT
, facgt
)
3859 static bool trans_FCADD(DisasContext
*s
, arg_FCADD
*a
)
3861 static gen_helper_gvec_4_ptr
* const fns
[3] = {
3862 gen_helper_sve_fcadd_h
,
3863 gen_helper_sve_fcadd_s
,
3864 gen_helper_sve_fcadd_d
3870 if (sve_access_check(s
)) {
3871 unsigned vsz
= vec_full_reg_size(s
);
3872 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3873 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, a
->rd
),
3874 vec_full_reg_offset(s
, a
->rn
),
3875 vec_full_reg_offset(s
, a
->rm
),
3876 pred_full_reg_offset(s
, a
->pg
),
3877 status
, vsz
, vsz
, a
->rot
, fns
[a
->esz
- 1]);
3878 tcg_temp_free_ptr(status
);
3883 static bool do_fmla(DisasContext
*s
, arg_rprrr_esz
*a
,
3884 gen_helper_gvec_5_ptr
*fn
)
3889 if (sve_access_check(s
)) {
3890 unsigned vsz
= vec_full_reg_size(s
);
3891 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3892 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s
, a
->rd
),
3893 vec_full_reg_offset(s
, a
->rn
),
3894 vec_full_reg_offset(s
, a
->rm
),
3895 vec_full_reg_offset(s
, a
->ra
),
3896 pred_full_reg_offset(s
, a
->pg
),
3897 status
, vsz
, vsz
, 0, fn
);
3898 tcg_temp_free_ptr(status
);
3903 #define DO_FMLA(NAME, name) \
3904 static bool trans_##NAME(DisasContext *s, arg_rprrr_esz *a) \
3906 static gen_helper_gvec_5_ptr * const fns[4] = { \
3907 NULL, gen_helper_sve_##name##_h, \
3908 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
3910 return do_fmla(s, a, fns[a->esz]); \
3913 DO_FMLA(FMLA_zpzzz
, fmla_zpzzz
)
3914 DO_FMLA(FMLS_zpzzz
, fmls_zpzzz
)
3915 DO_FMLA(FNMLA_zpzzz
, fnmla_zpzzz
)
3916 DO_FMLA(FNMLS_zpzzz
, fnmls_zpzzz
)
3920 static bool trans_FCMLA_zpzzz(DisasContext
*s
, arg_FCMLA_zpzzz
*a
)
3922 static gen_helper_gvec_5_ptr
* const fns
[4] = {
3924 gen_helper_sve_fcmla_zpzzz_h
,
3925 gen_helper_sve_fcmla_zpzzz_s
,
3926 gen_helper_sve_fcmla_zpzzz_d
,
3932 if (sve_access_check(s
)) {
3933 unsigned vsz
= vec_full_reg_size(s
);
3934 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3935 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s
, a
->rd
),
3936 vec_full_reg_offset(s
, a
->rn
),
3937 vec_full_reg_offset(s
, a
->rm
),
3938 vec_full_reg_offset(s
, a
->ra
),
3939 pred_full_reg_offset(s
, a
->pg
),
3940 status
, vsz
, vsz
, a
->rot
, fns
[a
->esz
]);
3941 tcg_temp_free_ptr(status
);
3946 static bool trans_FCMLA_zzxz(DisasContext
*s
, arg_FCMLA_zzxz
*a
)
3948 static gen_helper_gvec_3_ptr
* const fns
[2] = {
3949 gen_helper_gvec_fcmlah_idx
,
3950 gen_helper_gvec_fcmlas_idx
,
3953 tcg_debug_assert(a
->esz
== 1 || a
->esz
== 2);
3954 tcg_debug_assert(a
->rd
== a
->ra
);
3955 if (sve_access_check(s
)) {
3956 unsigned vsz
= vec_full_reg_size(s
);
3957 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
3958 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
3959 vec_full_reg_offset(s
, a
->rn
),
3960 vec_full_reg_offset(s
, a
->rm
),
3962 a
->index
* 4 + a
->rot
,
3964 tcg_temp_free_ptr(status
);
3970 *** SVE Floating Point Unary Operations Predicated Group
3973 static bool do_zpz_ptr(DisasContext
*s
, int rd
, int rn
, int pg
,
3974 bool is_fp16
, gen_helper_gvec_3_ptr
*fn
)
3976 if (sve_access_check(s
)) {
3977 unsigned vsz
= vec_full_reg_size(s
);
3978 TCGv_ptr status
= fpstatus_ptr(is_fp16
? FPST_FPCR_F16
: FPST_FPCR
);
3979 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, rd
),
3980 vec_full_reg_offset(s
, rn
),
3981 pred_full_reg_offset(s
, pg
),
3982 status
, vsz
, vsz
, 0, fn
);
3983 tcg_temp_free_ptr(status
);
3988 static bool trans_FCVT_sh(DisasContext
*s
, arg_rpr_esz
*a
)
3990 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_sh
);
3993 static bool trans_FCVT_hs(DisasContext
*s
, arg_rpr_esz
*a
)
3995 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_hs
);
3998 static bool trans_FCVT_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4000 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_dh
);
4003 static bool trans_FCVT_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4005 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_hd
);
4008 static bool trans_FCVT_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4010 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_ds
);
4013 static bool trans_FCVT_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4015 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvt_sd
);
4018 static bool trans_FCVTZS_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4020 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hh
);
4023 static bool trans_FCVTZU_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4025 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hh
);
4028 static bool trans_FCVTZS_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4030 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hs
);
4033 static bool trans_FCVTZU_hs(DisasContext
*s
, arg_rpr_esz
*a
)
4035 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hs
);
4038 static bool trans_FCVTZS_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4040 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzs_hd
);
4043 static bool trans_FCVTZU_hd(DisasContext
*s
, arg_rpr_esz
*a
)
4045 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_fcvtzu_hd
);
4048 static bool trans_FCVTZS_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4050 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_ss
);
4053 static bool trans_FCVTZU_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4055 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_ss
);
4058 static bool trans_FCVTZS_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4060 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_sd
);
4063 static bool trans_FCVTZU_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4065 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_sd
);
4068 static bool trans_FCVTZS_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4070 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_ds
);
4073 static bool trans_FCVTZU_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4075 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_ds
);
4078 static bool trans_FCVTZS_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4080 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzs_dd
);
4083 static bool trans_FCVTZU_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4085 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_fcvtzu_dd
);
4088 static gen_helper_gvec_3_ptr
* const frint_fns
[3] = {
4089 gen_helper_sve_frint_h
,
4090 gen_helper_sve_frint_s
,
4091 gen_helper_sve_frint_d
4094 static bool trans_FRINTI(DisasContext
*s
, arg_rpr_esz
*a
)
4099 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
,
4100 frint_fns
[a
->esz
- 1]);
4103 static bool trans_FRINTX(DisasContext
*s
, arg_rpr_esz
*a
)
4105 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4106 gen_helper_sve_frintx_h
,
4107 gen_helper_sve_frintx_s
,
4108 gen_helper_sve_frintx_d
4113 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4116 static bool do_frint_mode(DisasContext
*s
, arg_rpr_esz
*a
, int mode
)
4121 if (sve_access_check(s
)) {
4122 unsigned vsz
= vec_full_reg_size(s
);
4123 TCGv_i32 tmode
= tcg_const_i32(mode
);
4124 TCGv_ptr status
= fpstatus_ptr(a
->esz
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
4126 gen_helper_set_rmode(tmode
, tmode
, status
);
4128 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
4129 vec_full_reg_offset(s
, a
->rn
),
4130 pred_full_reg_offset(s
, a
->pg
),
4131 status
, vsz
, vsz
, 0, frint_fns
[a
->esz
- 1]);
4133 gen_helper_set_rmode(tmode
, tmode
, status
);
4134 tcg_temp_free_i32(tmode
);
4135 tcg_temp_free_ptr(status
);
4140 static bool trans_FRINTN(DisasContext
*s
, arg_rpr_esz
*a
)
4142 return do_frint_mode(s
, a
, float_round_nearest_even
);
4145 static bool trans_FRINTP(DisasContext
*s
, arg_rpr_esz
*a
)
4147 return do_frint_mode(s
, a
, float_round_up
);
4150 static bool trans_FRINTM(DisasContext
*s
, arg_rpr_esz
*a
)
4152 return do_frint_mode(s
, a
, float_round_down
);
4155 static bool trans_FRINTZ(DisasContext
*s
, arg_rpr_esz
*a
)
4157 return do_frint_mode(s
, a
, float_round_to_zero
);
4160 static bool trans_FRINTA(DisasContext
*s
, arg_rpr_esz
*a
)
4162 return do_frint_mode(s
, a
, float_round_ties_away
);
4165 static bool trans_FRECPX(DisasContext
*s
, arg_rpr_esz
*a
)
4167 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4168 gen_helper_sve_frecpx_h
,
4169 gen_helper_sve_frecpx_s
,
4170 gen_helper_sve_frecpx_d
4175 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4178 static bool trans_FSQRT(DisasContext
*s
, arg_rpr_esz
*a
)
4180 static gen_helper_gvec_3_ptr
* const fns
[3] = {
4181 gen_helper_sve_fsqrt_h
,
4182 gen_helper_sve_fsqrt_s
,
4183 gen_helper_sve_fsqrt_d
4188 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
== MO_16
, fns
[a
->esz
- 1]);
4191 static bool trans_SCVTF_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4193 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_hh
);
4196 static bool trans_SCVTF_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4198 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_sh
);
4201 static bool trans_SCVTF_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4203 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_scvt_dh
);
4206 static bool trans_SCVTF_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4208 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_ss
);
4211 static bool trans_SCVTF_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4213 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_ds
);
4216 static bool trans_SCVTF_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4218 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_sd
);
4221 static bool trans_SCVTF_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4223 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_scvt_dd
);
4226 static bool trans_UCVTF_hh(DisasContext
*s
, arg_rpr_esz
*a
)
4228 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_hh
);
4231 static bool trans_UCVTF_sh(DisasContext
*s
, arg_rpr_esz
*a
)
4233 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_sh
);
4236 static bool trans_UCVTF_dh(DisasContext
*s
, arg_rpr_esz
*a
)
4238 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, true, gen_helper_sve_ucvt_dh
);
4241 static bool trans_UCVTF_ss(DisasContext
*s
, arg_rpr_esz
*a
)
4243 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_ss
);
4246 static bool trans_UCVTF_ds(DisasContext
*s
, arg_rpr_esz
*a
)
4248 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_ds
);
4251 static bool trans_UCVTF_sd(DisasContext
*s
, arg_rpr_esz
*a
)
4253 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_sd
);
4256 static bool trans_UCVTF_dd(DisasContext
*s
, arg_rpr_esz
*a
)
4258 return do_zpz_ptr(s
, a
->rd
, a
->rn
, a
->pg
, false, gen_helper_sve_ucvt_dd
);
4262 *** SVE Memory - 32-bit Gather and Unsized Contiguous Group
4265 /* Subroutine loading a vector register at VOFS of LEN bytes.
4266 * The load should begin at the address Rn + IMM.
4269 static void do_ldr(DisasContext
*s
, uint32_t vofs
, int len
, int rn
, int imm
)
4271 int len_align
= QEMU_ALIGN_DOWN(len
, 8);
4272 int len_remain
= len
% 8;
4273 int nparts
= len
/ 8 + ctpop8(len_remain
);
4274 int midx
= get_mem_index(s
);
4275 TCGv_i64 dirty_addr
, clean_addr
, t0
, t1
;
4277 dirty_addr
= tcg_temp_new_i64();
4278 tcg_gen_addi_i64(dirty_addr
, cpu_reg_sp(s
, rn
), imm
);
4279 clean_addr
= gen_mte_checkN(s
, dirty_addr
, false, rn
!= 31, len
);
4280 tcg_temp_free_i64(dirty_addr
);
4283 * Note that unpredicated load/store of vector/predicate registers
4284 * are defined as a stream of bytes, which equates to little-endian
4285 * operations on larger quantities.
4286 * Attempt to keep code expansion to a minimum by limiting the
4287 * amount of unrolling done.
4292 t0
= tcg_temp_new_i64();
4293 for (i
= 0; i
< len_align
; i
+= 8) {
4294 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEQ
);
4295 tcg_gen_st_i64(t0
, cpu_env
, vofs
+ i
);
4296 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4298 tcg_temp_free_i64(t0
);
4300 TCGLabel
*loop
= gen_new_label();
4301 TCGv_ptr tp
, i
= tcg_const_local_ptr(0);
4303 /* Copy the clean address into a local temp, live across the loop. */
4305 clean_addr
= new_tmp_a64_local(s
);
4306 tcg_gen_mov_i64(clean_addr
, t0
);
4308 gen_set_label(loop
);
4310 t0
= tcg_temp_new_i64();
4311 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEQ
);
4312 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4314 tp
= tcg_temp_new_ptr();
4315 tcg_gen_add_ptr(tp
, cpu_env
, i
);
4316 tcg_gen_addi_ptr(i
, i
, 8);
4317 tcg_gen_st_i64(t0
, tp
, vofs
);
4318 tcg_temp_free_ptr(tp
);
4319 tcg_temp_free_i64(t0
);
4321 tcg_gen_brcondi_ptr(TCG_COND_LTU
, i
, len_align
, loop
);
4322 tcg_temp_free_ptr(i
);
4326 * Predicate register loads can be any multiple of 2.
4327 * Note that we still store the entire 64-bit unit into cpu_env.
4330 t0
= tcg_temp_new_i64();
4331 switch (len_remain
) {
4335 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
,
4336 MO_LE
| ctz32(len_remain
));
4340 t1
= tcg_temp_new_i64();
4341 tcg_gen_qemu_ld_i64(t0
, clean_addr
, midx
, MO_LEUL
);
4342 tcg_gen_addi_i64(clean_addr
, clean_addr
, 4);
4343 tcg_gen_qemu_ld_i64(t1
, clean_addr
, midx
, MO_LEUW
);
4344 tcg_gen_deposit_i64(t0
, t0
, t1
, 32, 32);
4345 tcg_temp_free_i64(t1
);
4349 g_assert_not_reached();
4351 tcg_gen_st_i64(t0
, cpu_env
, vofs
+ len_align
);
4352 tcg_temp_free_i64(t0
);
4356 /* Similarly for stores. */
4357 static void do_str(DisasContext
*s
, uint32_t vofs
, int len
, int rn
, int imm
)
4359 int len_align
= QEMU_ALIGN_DOWN(len
, 8);
4360 int len_remain
= len
% 8;
4361 int nparts
= len
/ 8 + ctpop8(len_remain
);
4362 int midx
= get_mem_index(s
);
4363 TCGv_i64 dirty_addr
, clean_addr
, t0
;
4365 dirty_addr
= tcg_temp_new_i64();
4366 tcg_gen_addi_i64(dirty_addr
, cpu_reg_sp(s
, rn
), imm
);
4367 clean_addr
= gen_mte_checkN(s
, dirty_addr
, false, rn
!= 31, len
);
4368 tcg_temp_free_i64(dirty_addr
);
4370 /* Note that unpredicated load/store of vector/predicate registers
4371 * are defined as a stream of bytes, which equates to little-endian
4372 * operations on larger quantities. There is no nice way to force
4373 * a little-endian store for aarch64_be-linux-user out of line.
4375 * Attempt to keep code expansion to a minimum by limiting the
4376 * amount of unrolling done.
4381 t0
= tcg_temp_new_i64();
4382 for (i
= 0; i
< len_align
; i
+= 8) {
4383 tcg_gen_ld_i64(t0
, cpu_env
, vofs
+ i
);
4384 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEQ
);
4385 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4387 tcg_temp_free_i64(t0
);
4389 TCGLabel
*loop
= gen_new_label();
4390 TCGv_ptr tp
, i
= tcg_const_local_ptr(0);
4392 /* Copy the clean address into a local temp, live across the loop. */
4394 clean_addr
= new_tmp_a64_local(s
);
4395 tcg_gen_mov_i64(clean_addr
, t0
);
4397 gen_set_label(loop
);
4399 t0
= tcg_temp_new_i64();
4400 tp
= tcg_temp_new_ptr();
4401 tcg_gen_add_ptr(tp
, cpu_env
, i
);
4402 tcg_gen_ld_i64(t0
, tp
, vofs
);
4403 tcg_gen_addi_ptr(i
, i
, 8);
4404 tcg_temp_free_ptr(tp
);
4406 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEQ
);
4407 tcg_gen_addi_i64(clean_addr
, clean_addr
, 8);
4408 tcg_temp_free_i64(t0
);
4410 tcg_gen_brcondi_ptr(TCG_COND_LTU
, i
, len_align
, loop
);
4411 tcg_temp_free_ptr(i
);
4414 /* Predicate register stores can be any multiple of 2. */
4416 t0
= tcg_temp_new_i64();
4417 tcg_gen_ld_i64(t0
, cpu_env
, vofs
+ len_align
);
4419 switch (len_remain
) {
4423 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
,
4424 MO_LE
| ctz32(len_remain
));
4428 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUL
);
4429 tcg_gen_addi_i64(clean_addr
, clean_addr
, 4);
4430 tcg_gen_shri_i64(t0
, t0
, 32);
4431 tcg_gen_qemu_st_i64(t0
, clean_addr
, midx
, MO_LEUW
);
4435 g_assert_not_reached();
4437 tcg_temp_free_i64(t0
);
4441 static bool trans_LDR_zri(DisasContext
*s
, arg_rri
*a
)
4443 if (sve_access_check(s
)) {
4444 int size
= vec_full_reg_size(s
);
4445 int off
= vec_full_reg_offset(s
, a
->rd
);
4446 do_ldr(s
, off
, size
, a
->rn
, a
->imm
* size
);
4451 static bool trans_LDR_pri(DisasContext
*s
, arg_rri
*a
)
4453 if (sve_access_check(s
)) {
4454 int size
= pred_full_reg_size(s
);
4455 int off
= pred_full_reg_offset(s
, a
->rd
);
4456 do_ldr(s
, off
, size
, a
->rn
, a
->imm
* size
);
4461 static bool trans_STR_zri(DisasContext
*s
, arg_rri
*a
)
4463 if (sve_access_check(s
)) {
4464 int size
= vec_full_reg_size(s
);
4465 int off
= vec_full_reg_offset(s
, a
->rd
);
4466 do_str(s
, off
, size
, a
->rn
, a
->imm
* size
);
4471 static bool trans_STR_pri(DisasContext
*s
, arg_rri
*a
)
4473 if (sve_access_check(s
)) {
4474 int size
= pred_full_reg_size(s
);
4475 int off
= pred_full_reg_offset(s
, a
->rd
);
4476 do_str(s
, off
, size
, a
->rn
, a
->imm
* size
);
4482 *** SVE Memory - Contiguous Load Group
4485 /* The memory mode of the dtype. */
4486 static const MemOp dtype_mop
[16] = {
4487 MO_UB
, MO_UB
, MO_UB
, MO_UB
,
4488 MO_SL
, MO_UW
, MO_UW
, MO_UW
,
4489 MO_SW
, MO_SW
, MO_UL
, MO_UL
,
4490 MO_SB
, MO_SB
, MO_SB
, MO_Q
4493 #define dtype_msz(x) (dtype_mop[x] & MO_SIZE)
4495 /* The vector element size of dtype. */
4496 static const uint8_t dtype_esz
[16] = {
4503 static void do_mem_zpa(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
,
4504 int dtype
, uint32_t mte_n
, bool is_write
,
4505 gen_helper_gvec_mem
*fn
)
4507 unsigned vsz
= vec_full_reg_size(s
);
4513 * For e.g. LD4, there are not enough arguments to pass all 4
4514 * registers as pointers, so encode the regno into the data field.
4515 * For consistency, do this even for LD1.
4517 if (s
->mte_active
[0]) {
4518 int msz
= dtype_msz(dtype
);
4520 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, get_mem_index(s
));
4521 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
4522 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
4523 desc
= FIELD_DP32(desc
, MTEDESC
, WRITE
, is_write
);
4524 desc
= FIELD_DP32(desc
, MTEDESC
, SIZEM1
, (mte_n
<< msz
) - 1);
4525 desc
<<= SVE_MTEDESC_SHIFT
;
4527 addr
= clean_data_tbi(s
, addr
);
4530 desc
= simd_desc(vsz
, vsz
, zt
| desc
);
4531 t_desc
= tcg_const_i32(desc
);
4532 t_pg
= tcg_temp_new_ptr();
4534 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
4535 fn(cpu_env
, t_pg
, addr
, t_desc
);
4537 tcg_temp_free_ptr(t_pg
);
4538 tcg_temp_free_i32(t_desc
);
4541 static void do_ld_zpa(DisasContext
*s
, int zt
, int pg
,
4542 TCGv_i64 addr
, int dtype
, int nreg
)
4544 static gen_helper_gvec_mem
* const fns
[2][2][16][4] = {
4545 { /* mte inactive, little-endian */
4546 { { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld2bb_r
,
4547 gen_helper_sve_ld3bb_r
, gen_helper_sve_ld4bb_r
},
4548 { gen_helper_sve_ld1bhu_r
, NULL
, NULL
, NULL
},
4549 { gen_helper_sve_ld1bsu_r
, NULL
, NULL
, NULL
},
4550 { gen_helper_sve_ld1bdu_r
, NULL
, NULL
, NULL
},
4552 { gen_helper_sve_ld1sds_le_r
, NULL
, NULL
, NULL
},
4553 { gen_helper_sve_ld1hh_le_r
, gen_helper_sve_ld2hh_le_r
,
4554 gen_helper_sve_ld3hh_le_r
, gen_helper_sve_ld4hh_le_r
},
4555 { gen_helper_sve_ld1hsu_le_r
, NULL
, NULL
, NULL
},
4556 { gen_helper_sve_ld1hdu_le_r
, NULL
, NULL
, NULL
},
4558 { gen_helper_sve_ld1hds_le_r
, NULL
, NULL
, NULL
},
4559 { gen_helper_sve_ld1hss_le_r
, NULL
, NULL
, NULL
},
4560 { gen_helper_sve_ld1ss_le_r
, gen_helper_sve_ld2ss_le_r
,
4561 gen_helper_sve_ld3ss_le_r
, gen_helper_sve_ld4ss_le_r
},
4562 { gen_helper_sve_ld1sdu_le_r
, NULL
, NULL
, NULL
},
4564 { gen_helper_sve_ld1bds_r
, NULL
, NULL
, NULL
},
4565 { gen_helper_sve_ld1bss_r
, NULL
, NULL
, NULL
},
4566 { gen_helper_sve_ld1bhs_r
, NULL
, NULL
, NULL
},
4567 { gen_helper_sve_ld1dd_le_r
, gen_helper_sve_ld2dd_le_r
,
4568 gen_helper_sve_ld3dd_le_r
, gen_helper_sve_ld4dd_le_r
} },
4570 /* mte inactive, big-endian */
4571 { { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld2bb_r
,
4572 gen_helper_sve_ld3bb_r
, gen_helper_sve_ld4bb_r
},
4573 { gen_helper_sve_ld1bhu_r
, NULL
, NULL
, NULL
},
4574 { gen_helper_sve_ld1bsu_r
, NULL
, NULL
, NULL
},
4575 { gen_helper_sve_ld1bdu_r
, NULL
, NULL
, NULL
},
4577 { gen_helper_sve_ld1sds_be_r
, NULL
, NULL
, NULL
},
4578 { gen_helper_sve_ld1hh_be_r
, gen_helper_sve_ld2hh_be_r
,
4579 gen_helper_sve_ld3hh_be_r
, gen_helper_sve_ld4hh_be_r
},
4580 { gen_helper_sve_ld1hsu_be_r
, NULL
, NULL
, NULL
},
4581 { gen_helper_sve_ld1hdu_be_r
, NULL
, NULL
, NULL
},
4583 { gen_helper_sve_ld1hds_be_r
, NULL
, NULL
, NULL
},
4584 { gen_helper_sve_ld1hss_be_r
, NULL
, NULL
, NULL
},
4585 { gen_helper_sve_ld1ss_be_r
, gen_helper_sve_ld2ss_be_r
,
4586 gen_helper_sve_ld3ss_be_r
, gen_helper_sve_ld4ss_be_r
},
4587 { gen_helper_sve_ld1sdu_be_r
, NULL
, NULL
, NULL
},
4589 { gen_helper_sve_ld1bds_r
, NULL
, NULL
, NULL
},
4590 { gen_helper_sve_ld1bss_r
, NULL
, NULL
, NULL
},
4591 { gen_helper_sve_ld1bhs_r
, NULL
, NULL
, NULL
},
4592 { gen_helper_sve_ld1dd_be_r
, gen_helper_sve_ld2dd_be_r
,
4593 gen_helper_sve_ld3dd_be_r
, gen_helper_sve_ld4dd_be_r
} } },
4595 { /* mte active, little-endian */
4596 { { gen_helper_sve_ld1bb_r_mte
,
4597 gen_helper_sve_ld2bb_r_mte
,
4598 gen_helper_sve_ld3bb_r_mte
,
4599 gen_helper_sve_ld4bb_r_mte
},
4600 { gen_helper_sve_ld1bhu_r_mte
, NULL
, NULL
, NULL
},
4601 { gen_helper_sve_ld1bsu_r_mte
, NULL
, NULL
, NULL
},
4602 { gen_helper_sve_ld1bdu_r_mte
, NULL
, NULL
, NULL
},
4604 { gen_helper_sve_ld1sds_le_r_mte
, NULL
, NULL
, NULL
},
4605 { gen_helper_sve_ld1hh_le_r_mte
,
4606 gen_helper_sve_ld2hh_le_r_mte
,
4607 gen_helper_sve_ld3hh_le_r_mte
,
4608 gen_helper_sve_ld4hh_le_r_mte
},
4609 { gen_helper_sve_ld1hsu_le_r_mte
, NULL
, NULL
, NULL
},
4610 { gen_helper_sve_ld1hdu_le_r_mte
, NULL
, NULL
, NULL
},
4612 { gen_helper_sve_ld1hds_le_r_mte
, NULL
, NULL
, NULL
},
4613 { gen_helper_sve_ld1hss_le_r_mte
, NULL
, NULL
, NULL
},
4614 { gen_helper_sve_ld1ss_le_r_mte
,
4615 gen_helper_sve_ld2ss_le_r_mte
,
4616 gen_helper_sve_ld3ss_le_r_mte
,
4617 gen_helper_sve_ld4ss_le_r_mte
},
4618 { gen_helper_sve_ld1sdu_le_r_mte
, NULL
, NULL
, NULL
},
4620 { gen_helper_sve_ld1bds_r_mte
, NULL
, NULL
, NULL
},
4621 { gen_helper_sve_ld1bss_r_mte
, NULL
, NULL
, NULL
},
4622 { gen_helper_sve_ld1bhs_r_mte
, NULL
, NULL
, NULL
},
4623 { gen_helper_sve_ld1dd_le_r_mte
,
4624 gen_helper_sve_ld2dd_le_r_mte
,
4625 gen_helper_sve_ld3dd_le_r_mte
,
4626 gen_helper_sve_ld4dd_le_r_mte
} },
4628 /* mte active, big-endian */
4629 { { gen_helper_sve_ld1bb_r_mte
,
4630 gen_helper_sve_ld2bb_r_mte
,
4631 gen_helper_sve_ld3bb_r_mte
,
4632 gen_helper_sve_ld4bb_r_mte
},
4633 { gen_helper_sve_ld1bhu_r_mte
, NULL
, NULL
, NULL
},
4634 { gen_helper_sve_ld1bsu_r_mte
, NULL
, NULL
, NULL
},
4635 { gen_helper_sve_ld1bdu_r_mte
, NULL
, NULL
, NULL
},
4637 { gen_helper_sve_ld1sds_be_r_mte
, NULL
, NULL
, NULL
},
4638 { gen_helper_sve_ld1hh_be_r_mte
,
4639 gen_helper_sve_ld2hh_be_r_mte
,
4640 gen_helper_sve_ld3hh_be_r_mte
,
4641 gen_helper_sve_ld4hh_be_r_mte
},
4642 { gen_helper_sve_ld1hsu_be_r_mte
, NULL
, NULL
, NULL
},
4643 { gen_helper_sve_ld1hdu_be_r_mte
, NULL
, NULL
, NULL
},
4645 { gen_helper_sve_ld1hds_be_r_mte
, NULL
, NULL
, NULL
},
4646 { gen_helper_sve_ld1hss_be_r_mte
, NULL
, NULL
, NULL
},
4647 { gen_helper_sve_ld1ss_be_r_mte
,
4648 gen_helper_sve_ld2ss_be_r_mte
,
4649 gen_helper_sve_ld3ss_be_r_mte
,
4650 gen_helper_sve_ld4ss_be_r_mte
},
4651 { gen_helper_sve_ld1sdu_be_r_mte
, NULL
, NULL
, NULL
},
4653 { gen_helper_sve_ld1bds_r_mte
, NULL
, NULL
, NULL
},
4654 { gen_helper_sve_ld1bss_r_mte
, NULL
, NULL
, NULL
},
4655 { gen_helper_sve_ld1bhs_r_mte
, NULL
, NULL
, NULL
},
4656 { gen_helper_sve_ld1dd_be_r_mte
,
4657 gen_helper_sve_ld2dd_be_r_mte
,
4658 gen_helper_sve_ld3dd_be_r_mte
,
4659 gen_helper_sve_ld4dd_be_r_mte
} } },
4661 gen_helper_gvec_mem
*fn
4662 = fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][dtype
][nreg
];
4665 * While there are holes in the table, they are not
4666 * accessible via the instruction encoding.
4669 do_mem_zpa(s
, zt
, pg
, addr
, dtype
, nreg
, false, fn
);
4672 static bool trans_LD_zprr(DisasContext
*s
, arg_rprr_load
*a
)
4677 if (sve_access_check(s
)) {
4678 TCGv_i64 addr
= new_tmp_a64(s
);
4679 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
4680 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
4681 do_ld_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, a
->nreg
);
4686 static bool trans_LD_zpri(DisasContext
*s
, arg_rpri_load
*a
)
4688 if (sve_access_check(s
)) {
4689 int vsz
= vec_full_reg_size(s
);
4690 int elements
= vsz
>> dtype_esz
[a
->dtype
];
4691 TCGv_i64 addr
= new_tmp_a64(s
);
4693 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
),
4694 (a
->imm
* elements
* (a
->nreg
+ 1))
4695 << dtype_msz(a
->dtype
));
4696 do_ld_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, a
->nreg
);
4701 static bool trans_LDFF1_zprr(DisasContext
*s
, arg_rprr_load
*a
)
4703 static gen_helper_gvec_mem
* const fns
[2][2][16] = {
4704 { /* mte inactive, little-endian */
4705 { gen_helper_sve_ldff1bb_r
,
4706 gen_helper_sve_ldff1bhu_r
,
4707 gen_helper_sve_ldff1bsu_r
,
4708 gen_helper_sve_ldff1bdu_r
,
4710 gen_helper_sve_ldff1sds_le_r
,
4711 gen_helper_sve_ldff1hh_le_r
,
4712 gen_helper_sve_ldff1hsu_le_r
,
4713 gen_helper_sve_ldff1hdu_le_r
,
4715 gen_helper_sve_ldff1hds_le_r
,
4716 gen_helper_sve_ldff1hss_le_r
,
4717 gen_helper_sve_ldff1ss_le_r
,
4718 gen_helper_sve_ldff1sdu_le_r
,
4720 gen_helper_sve_ldff1bds_r
,
4721 gen_helper_sve_ldff1bss_r
,
4722 gen_helper_sve_ldff1bhs_r
,
4723 gen_helper_sve_ldff1dd_le_r
},
4725 /* mte inactive, big-endian */
4726 { gen_helper_sve_ldff1bb_r
,
4727 gen_helper_sve_ldff1bhu_r
,
4728 gen_helper_sve_ldff1bsu_r
,
4729 gen_helper_sve_ldff1bdu_r
,
4731 gen_helper_sve_ldff1sds_be_r
,
4732 gen_helper_sve_ldff1hh_be_r
,
4733 gen_helper_sve_ldff1hsu_be_r
,
4734 gen_helper_sve_ldff1hdu_be_r
,
4736 gen_helper_sve_ldff1hds_be_r
,
4737 gen_helper_sve_ldff1hss_be_r
,
4738 gen_helper_sve_ldff1ss_be_r
,
4739 gen_helper_sve_ldff1sdu_be_r
,
4741 gen_helper_sve_ldff1bds_r
,
4742 gen_helper_sve_ldff1bss_r
,
4743 gen_helper_sve_ldff1bhs_r
,
4744 gen_helper_sve_ldff1dd_be_r
} },
4746 { /* mte active, little-endian */
4747 { gen_helper_sve_ldff1bb_r_mte
,
4748 gen_helper_sve_ldff1bhu_r_mte
,
4749 gen_helper_sve_ldff1bsu_r_mte
,
4750 gen_helper_sve_ldff1bdu_r_mte
,
4752 gen_helper_sve_ldff1sds_le_r_mte
,
4753 gen_helper_sve_ldff1hh_le_r_mte
,
4754 gen_helper_sve_ldff1hsu_le_r_mte
,
4755 gen_helper_sve_ldff1hdu_le_r_mte
,
4757 gen_helper_sve_ldff1hds_le_r_mte
,
4758 gen_helper_sve_ldff1hss_le_r_mte
,
4759 gen_helper_sve_ldff1ss_le_r_mte
,
4760 gen_helper_sve_ldff1sdu_le_r_mte
,
4762 gen_helper_sve_ldff1bds_r_mte
,
4763 gen_helper_sve_ldff1bss_r_mte
,
4764 gen_helper_sve_ldff1bhs_r_mte
,
4765 gen_helper_sve_ldff1dd_le_r_mte
},
4767 /* mte active, big-endian */
4768 { gen_helper_sve_ldff1bb_r_mte
,
4769 gen_helper_sve_ldff1bhu_r_mte
,
4770 gen_helper_sve_ldff1bsu_r_mte
,
4771 gen_helper_sve_ldff1bdu_r_mte
,
4773 gen_helper_sve_ldff1sds_be_r_mte
,
4774 gen_helper_sve_ldff1hh_be_r_mte
,
4775 gen_helper_sve_ldff1hsu_be_r_mte
,
4776 gen_helper_sve_ldff1hdu_be_r_mte
,
4778 gen_helper_sve_ldff1hds_be_r_mte
,
4779 gen_helper_sve_ldff1hss_be_r_mte
,
4780 gen_helper_sve_ldff1ss_be_r_mte
,
4781 gen_helper_sve_ldff1sdu_be_r_mte
,
4783 gen_helper_sve_ldff1bds_r_mte
,
4784 gen_helper_sve_ldff1bss_r_mte
,
4785 gen_helper_sve_ldff1bhs_r_mte
,
4786 gen_helper_sve_ldff1dd_be_r_mte
} },
4789 if (sve_access_check(s
)) {
4790 TCGv_i64 addr
= new_tmp_a64(s
);
4791 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), dtype_msz(a
->dtype
));
4792 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
4793 do_mem_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, 1, false,
4794 fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][a
->dtype
]);
4799 static bool trans_LDNF1_zpri(DisasContext
*s
, arg_rpri_load
*a
)
4801 static gen_helper_gvec_mem
* const fns
[2][2][16] = {
4802 { /* mte inactive, little-endian */
4803 { gen_helper_sve_ldnf1bb_r
,
4804 gen_helper_sve_ldnf1bhu_r
,
4805 gen_helper_sve_ldnf1bsu_r
,
4806 gen_helper_sve_ldnf1bdu_r
,
4808 gen_helper_sve_ldnf1sds_le_r
,
4809 gen_helper_sve_ldnf1hh_le_r
,
4810 gen_helper_sve_ldnf1hsu_le_r
,
4811 gen_helper_sve_ldnf1hdu_le_r
,
4813 gen_helper_sve_ldnf1hds_le_r
,
4814 gen_helper_sve_ldnf1hss_le_r
,
4815 gen_helper_sve_ldnf1ss_le_r
,
4816 gen_helper_sve_ldnf1sdu_le_r
,
4818 gen_helper_sve_ldnf1bds_r
,
4819 gen_helper_sve_ldnf1bss_r
,
4820 gen_helper_sve_ldnf1bhs_r
,
4821 gen_helper_sve_ldnf1dd_le_r
},
4823 /* mte inactive, big-endian */
4824 { gen_helper_sve_ldnf1bb_r
,
4825 gen_helper_sve_ldnf1bhu_r
,
4826 gen_helper_sve_ldnf1bsu_r
,
4827 gen_helper_sve_ldnf1bdu_r
,
4829 gen_helper_sve_ldnf1sds_be_r
,
4830 gen_helper_sve_ldnf1hh_be_r
,
4831 gen_helper_sve_ldnf1hsu_be_r
,
4832 gen_helper_sve_ldnf1hdu_be_r
,
4834 gen_helper_sve_ldnf1hds_be_r
,
4835 gen_helper_sve_ldnf1hss_be_r
,
4836 gen_helper_sve_ldnf1ss_be_r
,
4837 gen_helper_sve_ldnf1sdu_be_r
,
4839 gen_helper_sve_ldnf1bds_r
,
4840 gen_helper_sve_ldnf1bss_r
,
4841 gen_helper_sve_ldnf1bhs_r
,
4842 gen_helper_sve_ldnf1dd_be_r
} },
4844 { /* mte inactive, little-endian */
4845 { gen_helper_sve_ldnf1bb_r_mte
,
4846 gen_helper_sve_ldnf1bhu_r_mte
,
4847 gen_helper_sve_ldnf1bsu_r_mte
,
4848 gen_helper_sve_ldnf1bdu_r_mte
,
4850 gen_helper_sve_ldnf1sds_le_r_mte
,
4851 gen_helper_sve_ldnf1hh_le_r_mte
,
4852 gen_helper_sve_ldnf1hsu_le_r_mte
,
4853 gen_helper_sve_ldnf1hdu_le_r_mte
,
4855 gen_helper_sve_ldnf1hds_le_r_mte
,
4856 gen_helper_sve_ldnf1hss_le_r_mte
,
4857 gen_helper_sve_ldnf1ss_le_r_mte
,
4858 gen_helper_sve_ldnf1sdu_le_r_mte
,
4860 gen_helper_sve_ldnf1bds_r_mte
,
4861 gen_helper_sve_ldnf1bss_r_mte
,
4862 gen_helper_sve_ldnf1bhs_r_mte
,
4863 gen_helper_sve_ldnf1dd_le_r_mte
},
4865 /* mte inactive, big-endian */
4866 { gen_helper_sve_ldnf1bb_r_mte
,
4867 gen_helper_sve_ldnf1bhu_r_mte
,
4868 gen_helper_sve_ldnf1bsu_r_mte
,
4869 gen_helper_sve_ldnf1bdu_r_mte
,
4871 gen_helper_sve_ldnf1sds_be_r_mte
,
4872 gen_helper_sve_ldnf1hh_be_r_mte
,
4873 gen_helper_sve_ldnf1hsu_be_r_mte
,
4874 gen_helper_sve_ldnf1hdu_be_r_mte
,
4876 gen_helper_sve_ldnf1hds_be_r_mte
,
4877 gen_helper_sve_ldnf1hss_be_r_mte
,
4878 gen_helper_sve_ldnf1ss_be_r_mte
,
4879 gen_helper_sve_ldnf1sdu_be_r_mte
,
4881 gen_helper_sve_ldnf1bds_r_mte
,
4882 gen_helper_sve_ldnf1bss_r_mte
,
4883 gen_helper_sve_ldnf1bhs_r_mte
,
4884 gen_helper_sve_ldnf1dd_be_r_mte
} },
4887 if (sve_access_check(s
)) {
4888 int vsz
= vec_full_reg_size(s
);
4889 int elements
= vsz
>> dtype_esz
[a
->dtype
];
4890 int off
= (a
->imm
* elements
) << dtype_msz(a
->dtype
);
4891 TCGv_i64 addr
= new_tmp_a64(s
);
4893 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), off
);
4894 do_mem_zpa(s
, a
->rd
, a
->pg
, addr
, a
->dtype
, 1, false,
4895 fns
[s
->mte_active
[0]][s
->be_data
== MO_BE
][a
->dtype
]);
4900 static void do_ldrq(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
, int msz
)
4902 static gen_helper_gvec_mem
* const fns
[2][4] = {
4903 { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld1hh_le_r
,
4904 gen_helper_sve_ld1ss_le_r
, gen_helper_sve_ld1dd_le_r
},
4905 { gen_helper_sve_ld1bb_r
, gen_helper_sve_ld1hh_be_r
,
4906 gen_helper_sve_ld1ss_be_r
, gen_helper_sve_ld1dd_be_r
},
4908 unsigned vsz
= vec_full_reg_size(s
);
4913 /* Load the first quadword using the normal predicated load helpers. */
4914 desc
= simd_desc(16, 16, zt
);
4915 t_desc
= tcg_const_i32(desc
);
4917 poff
= pred_full_reg_offset(s
, pg
);
4920 * Zero-extend the first 16 bits of the predicate into a temporary.
4921 * This avoids triggering an assert making sure we don't have bits
4922 * set within a predicate beyond VQ, but we have lowered VQ to 1
4923 * for this load operation.
4925 TCGv_i64 tmp
= tcg_temp_new_i64();
4926 #ifdef HOST_WORDS_BIGENDIAN
4929 tcg_gen_ld16u_i64(tmp
, cpu_env
, poff
);
4931 poff
= offsetof(CPUARMState
, vfp
.preg_tmp
);
4932 tcg_gen_st_i64(tmp
, cpu_env
, poff
);
4933 tcg_temp_free_i64(tmp
);
4936 t_pg
= tcg_temp_new_ptr();
4937 tcg_gen_addi_ptr(t_pg
, cpu_env
, poff
);
4939 fns
[s
->be_data
== MO_BE
][msz
](cpu_env
, t_pg
, addr
, t_desc
);
4941 tcg_temp_free_ptr(t_pg
);
4942 tcg_temp_free_i32(t_desc
);
4944 /* Replicate that first quadword. */
4946 unsigned dofs
= vec_full_reg_offset(s
, zt
);
4947 tcg_gen_gvec_dup_mem(4, dofs
+ 16, dofs
, vsz
- 16, vsz
- 16);
4951 static bool trans_LD1RQ_zprr(DisasContext
*s
, arg_rprr_load
*a
)
4956 if (sve_access_check(s
)) {
4957 int msz
= dtype_msz(a
->dtype
);
4958 TCGv_i64 addr
= new_tmp_a64(s
);
4959 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), msz
);
4960 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
4961 do_ldrq(s
, a
->rd
, a
->pg
, addr
, msz
);
4966 static bool trans_LD1RQ_zpri(DisasContext
*s
, arg_rpri_load
*a
)
4968 if (sve_access_check(s
)) {
4969 TCGv_i64 addr
= new_tmp_a64(s
);
4970 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
), a
->imm
* 16);
4971 do_ldrq(s
, a
->rd
, a
->pg
, addr
, dtype_msz(a
->dtype
));
4976 /* Load and broadcast element. */
4977 static bool trans_LD1R_zpri(DisasContext
*s
, arg_rpri_load
*a
)
4979 unsigned vsz
= vec_full_reg_size(s
);
4980 unsigned psz
= pred_full_reg_size(s
);
4981 unsigned esz
= dtype_esz
[a
->dtype
];
4982 unsigned msz
= dtype_msz(a
->dtype
);
4984 TCGv_i64 temp
, clean_addr
;
4986 if (!sve_access_check(s
)) {
4990 over
= gen_new_label();
4992 /* If the guarding predicate has no bits set, no load occurs. */
4994 /* Reduce the pred_esz_masks value simply to reduce the
4995 * size of the code generated here.
4997 uint64_t psz_mask
= MAKE_64BIT_MASK(0, psz
* 8);
4998 temp
= tcg_temp_new_i64();
4999 tcg_gen_ld_i64(temp
, cpu_env
, pred_full_reg_offset(s
, a
->pg
));
5000 tcg_gen_andi_i64(temp
, temp
, pred_esz_masks
[esz
] & psz_mask
);
5001 tcg_gen_brcondi_i64(TCG_COND_EQ
, temp
, 0, over
);
5002 tcg_temp_free_i64(temp
);
5004 TCGv_i32 t32
= tcg_temp_new_i32();
5005 find_last_active(s
, t32
, esz
, a
->pg
);
5006 tcg_gen_brcondi_i32(TCG_COND_LT
, t32
, 0, over
);
5007 tcg_temp_free_i32(t32
);
5010 /* Load the data. */
5011 temp
= tcg_temp_new_i64();
5012 tcg_gen_addi_i64(temp
, cpu_reg_sp(s
, a
->rn
), a
->imm
<< msz
);
5013 clean_addr
= gen_mte_check1(s
, temp
, false, true, msz
);
5015 tcg_gen_qemu_ld_i64(temp
, clean_addr
, get_mem_index(s
),
5016 finalize_memop(s
, dtype_mop
[a
->dtype
]));
5018 /* Broadcast to *all* elements. */
5019 tcg_gen_gvec_dup_i64(esz
, vec_full_reg_offset(s
, a
->rd
),
5021 tcg_temp_free_i64(temp
);
5023 /* Zero the inactive elements. */
5024 gen_set_label(over
);
5025 return do_movz_zpz(s
, a
->rd
, a
->rd
, a
->pg
, esz
, false);
5028 static void do_st_zpa(DisasContext
*s
, int zt
, int pg
, TCGv_i64 addr
,
5029 int msz
, int esz
, int nreg
)
5031 static gen_helper_gvec_mem
* const fn_single
[2][2][4][4] = {
5032 { { { gen_helper_sve_st1bb_r
,
5033 gen_helper_sve_st1bh_r
,
5034 gen_helper_sve_st1bs_r
,
5035 gen_helper_sve_st1bd_r
},
5037 gen_helper_sve_st1hh_le_r
,
5038 gen_helper_sve_st1hs_le_r
,
5039 gen_helper_sve_st1hd_le_r
},
5041 gen_helper_sve_st1ss_le_r
,
5042 gen_helper_sve_st1sd_le_r
},
5044 gen_helper_sve_st1dd_le_r
} },
5045 { { gen_helper_sve_st1bb_r
,
5046 gen_helper_sve_st1bh_r
,
5047 gen_helper_sve_st1bs_r
,
5048 gen_helper_sve_st1bd_r
},
5050 gen_helper_sve_st1hh_be_r
,
5051 gen_helper_sve_st1hs_be_r
,
5052 gen_helper_sve_st1hd_be_r
},
5054 gen_helper_sve_st1ss_be_r
,
5055 gen_helper_sve_st1sd_be_r
},
5057 gen_helper_sve_st1dd_be_r
} } },
5059 { { { gen_helper_sve_st1bb_r_mte
,
5060 gen_helper_sve_st1bh_r_mte
,
5061 gen_helper_sve_st1bs_r_mte
,
5062 gen_helper_sve_st1bd_r_mte
},
5064 gen_helper_sve_st1hh_le_r_mte
,
5065 gen_helper_sve_st1hs_le_r_mte
,
5066 gen_helper_sve_st1hd_le_r_mte
},
5068 gen_helper_sve_st1ss_le_r_mte
,
5069 gen_helper_sve_st1sd_le_r_mte
},
5071 gen_helper_sve_st1dd_le_r_mte
} },
5072 { { gen_helper_sve_st1bb_r_mte
,
5073 gen_helper_sve_st1bh_r_mte
,
5074 gen_helper_sve_st1bs_r_mte
,
5075 gen_helper_sve_st1bd_r_mte
},
5077 gen_helper_sve_st1hh_be_r_mte
,
5078 gen_helper_sve_st1hs_be_r_mte
,
5079 gen_helper_sve_st1hd_be_r_mte
},
5081 gen_helper_sve_st1ss_be_r_mte
,
5082 gen_helper_sve_st1sd_be_r_mte
},
5084 gen_helper_sve_st1dd_be_r_mte
} } },
5086 static gen_helper_gvec_mem
* const fn_multiple
[2][2][3][4] = {
5087 { { { gen_helper_sve_st2bb_r
,
5088 gen_helper_sve_st2hh_le_r
,
5089 gen_helper_sve_st2ss_le_r
,
5090 gen_helper_sve_st2dd_le_r
},
5091 { gen_helper_sve_st3bb_r
,
5092 gen_helper_sve_st3hh_le_r
,
5093 gen_helper_sve_st3ss_le_r
,
5094 gen_helper_sve_st3dd_le_r
},
5095 { gen_helper_sve_st4bb_r
,
5096 gen_helper_sve_st4hh_le_r
,
5097 gen_helper_sve_st4ss_le_r
,
5098 gen_helper_sve_st4dd_le_r
} },
5099 { { gen_helper_sve_st2bb_r
,
5100 gen_helper_sve_st2hh_be_r
,
5101 gen_helper_sve_st2ss_be_r
,
5102 gen_helper_sve_st2dd_be_r
},
5103 { gen_helper_sve_st3bb_r
,
5104 gen_helper_sve_st3hh_be_r
,
5105 gen_helper_sve_st3ss_be_r
,
5106 gen_helper_sve_st3dd_be_r
},
5107 { gen_helper_sve_st4bb_r
,
5108 gen_helper_sve_st4hh_be_r
,
5109 gen_helper_sve_st4ss_be_r
,
5110 gen_helper_sve_st4dd_be_r
} } },
5111 { { { gen_helper_sve_st2bb_r_mte
,
5112 gen_helper_sve_st2hh_le_r_mte
,
5113 gen_helper_sve_st2ss_le_r_mte
,
5114 gen_helper_sve_st2dd_le_r_mte
},
5115 { gen_helper_sve_st3bb_r_mte
,
5116 gen_helper_sve_st3hh_le_r_mte
,
5117 gen_helper_sve_st3ss_le_r_mte
,
5118 gen_helper_sve_st3dd_le_r_mte
},
5119 { gen_helper_sve_st4bb_r_mte
,
5120 gen_helper_sve_st4hh_le_r_mte
,
5121 gen_helper_sve_st4ss_le_r_mte
,
5122 gen_helper_sve_st4dd_le_r_mte
} },
5123 { { gen_helper_sve_st2bb_r_mte
,
5124 gen_helper_sve_st2hh_be_r_mte
,
5125 gen_helper_sve_st2ss_be_r_mte
,
5126 gen_helper_sve_st2dd_be_r_mte
},
5127 { gen_helper_sve_st3bb_r_mte
,
5128 gen_helper_sve_st3hh_be_r_mte
,
5129 gen_helper_sve_st3ss_be_r_mte
,
5130 gen_helper_sve_st3dd_be_r_mte
},
5131 { gen_helper_sve_st4bb_r_mte
,
5132 gen_helper_sve_st4hh_be_r_mte
,
5133 gen_helper_sve_st4ss_be_r_mte
,
5134 gen_helper_sve_st4dd_be_r_mte
} } },
5136 gen_helper_gvec_mem
*fn
;
5137 int be
= s
->be_data
== MO_BE
;
5141 fn
= fn_single
[s
->mte_active
[0]][be
][msz
][esz
];
5144 /* ST2, ST3, ST4 -- msz == esz, enforced by encoding */
5146 fn
= fn_multiple
[s
->mte_active
[0]][be
][nreg
- 1][msz
];
5149 do_mem_zpa(s
, zt
, pg
, addr
, msz_dtype(s
, msz
), nreg
, true, fn
);
5152 static bool trans_ST_zprr(DisasContext
*s
, arg_rprr_store
*a
)
5154 if (a
->rm
== 31 || a
->msz
> a
->esz
) {
5157 if (sve_access_check(s
)) {
5158 TCGv_i64 addr
= new_tmp_a64(s
);
5159 tcg_gen_shli_i64(addr
, cpu_reg(s
, a
->rm
), a
->msz
);
5160 tcg_gen_add_i64(addr
, addr
, cpu_reg_sp(s
, a
->rn
));
5161 do_st_zpa(s
, a
->rd
, a
->pg
, addr
, a
->msz
, a
->esz
, a
->nreg
);
5166 static bool trans_ST_zpri(DisasContext
*s
, arg_rpri_store
*a
)
5168 if (a
->msz
> a
->esz
) {
5171 if (sve_access_check(s
)) {
5172 int vsz
= vec_full_reg_size(s
);
5173 int elements
= vsz
>> a
->esz
;
5174 TCGv_i64 addr
= new_tmp_a64(s
);
5176 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, a
->rn
),
5177 (a
->imm
* elements
* (a
->nreg
+ 1)) << a
->msz
);
5178 do_st_zpa(s
, a
->rd
, a
->pg
, addr
, a
->msz
, a
->esz
, a
->nreg
);
5184 *** SVE gather loads / scatter stores
5187 static void do_mem_zpz(DisasContext
*s
, int zt
, int pg
, int zm
,
5188 int scale
, TCGv_i64 scalar
, int msz
, bool is_write
,
5189 gen_helper_gvec_mem_scatter
*fn
)
5191 unsigned vsz
= vec_full_reg_size(s
);
5192 TCGv_ptr t_zm
= tcg_temp_new_ptr();
5193 TCGv_ptr t_pg
= tcg_temp_new_ptr();
5194 TCGv_ptr t_zt
= tcg_temp_new_ptr();
5198 if (s
->mte_active
[0]) {
5199 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, get_mem_index(s
));
5200 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
5201 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
5202 desc
= FIELD_DP32(desc
, MTEDESC
, WRITE
, is_write
);
5203 desc
= FIELD_DP32(desc
, MTEDESC
, SIZEM1
, (1 << msz
) - 1);
5204 desc
<<= SVE_MTEDESC_SHIFT
;
5206 desc
= simd_desc(vsz
, vsz
, desc
| scale
);
5207 t_desc
= tcg_const_i32(desc
);
5209 tcg_gen_addi_ptr(t_pg
, cpu_env
, pred_full_reg_offset(s
, pg
));
5210 tcg_gen_addi_ptr(t_zm
, cpu_env
, vec_full_reg_offset(s
, zm
));
5211 tcg_gen_addi_ptr(t_zt
, cpu_env
, vec_full_reg_offset(s
, zt
));
5212 fn(cpu_env
, t_zt
, t_pg
, t_zm
, scalar
, t_desc
);
5214 tcg_temp_free_ptr(t_zt
);
5215 tcg_temp_free_ptr(t_zm
);
5216 tcg_temp_free_ptr(t_pg
);
5217 tcg_temp_free_i32(t_desc
);
5220 /* Indexed by [mte][be][ff][xs][u][msz]. */
5221 static gen_helper_gvec_mem_scatter
* const
5222 gather_load_fn32
[2][2][2][2][2][3] = {
5223 { /* MTE Inactive */
5224 { /* Little-endian */
5225 { { { gen_helper_sve_ldbss_zsu
,
5226 gen_helper_sve_ldhss_le_zsu
,
5228 { gen_helper_sve_ldbsu_zsu
,
5229 gen_helper_sve_ldhsu_le_zsu
,
5230 gen_helper_sve_ldss_le_zsu
, } },
5231 { { gen_helper_sve_ldbss_zss
,
5232 gen_helper_sve_ldhss_le_zss
,
5234 { gen_helper_sve_ldbsu_zss
,
5235 gen_helper_sve_ldhsu_le_zss
,
5236 gen_helper_sve_ldss_le_zss
, } } },
5239 { { { gen_helper_sve_ldffbss_zsu
,
5240 gen_helper_sve_ldffhss_le_zsu
,
5242 { gen_helper_sve_ldffbsu_zsu
,
5243 gen_helper_sve_ldffhsu_le_zsu
,
5244 gen_helper_sve_ldffss_le_zsu
, } },
5245 { { gen_helper_sve_ldffbss_zss
,
5246 gen_helper_sve_ldffhss_le_zss
,
5248 { gen_helper_sve_ldffbsu_zss
,
5249 gen_helper_sve_ldffhsu_le_zss
,
5250 gen_helper_sve_ldffss_le_zss
, } } } },
5253 { { { gen_helper_sve_ldbss_zsu
,
5254 gen_helper_sve_ldhss_be_zsu
,
5256 { gen_helper_sve_ldbsu_zsu
,
5257 gen_helper_sve_ldhsu_be_zsu
,
5258 gen_helper_sve_ldss_be_zsu
, } },
5259 { { gen_helper_sve_ldbss_zss
,
5260 gen_helper_sve_ldhss_be_zss
,
5262 { gen_helper_sve_ldbsu_zss
,
5263 gen_helper_sve_ldhsu_be_zss
,
5264 gen_helper_sve_ldss_be_zss
, } } },
5267 { { { gen_helper_sve_ldffbss_zsu
,
5268 gen_helper_sve_ldffhss_be_zsu
,
5270 { gen_helper_sve_ldffbsu_zsu
,
5271 gen_helper_sve_ldffhsu_be_zsu
,
5272 gen_helper_sve_ldffss_be_zsu
, } },
5273 { { gen_helper_sve_ldffbss_zss
,
5274 gen_helper_sve_ldffhss_be_zss
,
5276 { gen_helper_sve_ldffbsu_zss
,
5277 gen_helper_sve_ldffhsu_be_zss
,
5278 gen_helper_sve_ldffss_be_zss
, } } } } },
5280 { /* Little-endian */
5281 { { { gen_helper_sve_ldbss_zsu_mte
,
5282 gen_helper_sve_ldhss_le_zsu_mte
,
5284 { gen_helper_sve_ldbsu_zsu_mte
,
5285 gen_helper_sve_ldhsu_le_zsu_mte
,
5286 gen_helper_sve_ldss_le_zsu_mte
, } },
5287 { { gen_helper_sve_ldbss_zss_mte
,
5288 gen_helper_sve_ldhss_le_zss_mte
,
5290 { gen_helper_sve_ldbsu_zss_mte
,
5291 gen_helper_sve_ldhsu_le_zss_mte
,
5292 gen_helper_sve_ldss_le_zss_mte
, } } },
5295 { { { gen_helper_sve_ldffbss_zsu_mte
,
5296 gen_helper_sve_ldffhss_le_zsu_mte
,
5298 { gen_helper_sve_ldffbsu_zsu_mte
,
5299 gen_helper_sve_ldffhsu_le_zsu_mte
,
5300 gen_helper_sve_ldffss_le_zsu_mte
, } },
5301 { { gen_helper_sve_ldffbss_zss_mte
,
5302 gen_helper_sve_ldffhss_le_zss_mte
,
5304 { gen_helper_sve_ldffbsu_zss_mte
,
5305 gen_helper_sve_ldffhsu_le_zss_mte
,
5306 gen_helper_sve_ldffss_le_zss_mte
, } } } },
5309 { { { gen_helper_sve_ldbss_zsu_mte
,
5310 gen_helper_sve_ldhss_be_zsu_mte
,
5312 { gen_helper_sve_ldbsu_zsu_mte
,
5313 gen_helper_sve_ldhsu_be_zsu_mte
,
5314 gen_helper_sve_ldss_be_zsu_mte
, } },
5315 { { gen_helper_sve_ldbss_zss_mte
,
5316 gen_helper_sve_ldhss_be_zss_mte
,
5318 { gen_helper_sve_ldbsu_zss_mte
,
5319 gen_helper_sve_ldhsu_be_zss_mte
,
5320 gen_helper_sve_ldss_be_zss_mte
, } } },
5323 { { { gen_helper_sve_ldffbss_zsu_mte
,
5324 gen_helper_sve_ldffhss_be_zsu_mte
,
5326 { gen_helper_sve_ldffbsu_zsu_mte
,
5327 gen_helper_sve_ldffhsu_be_zsu_mte
,
5328 gen_helper_sve_ldffss_be_zsu_mte
, } },
5329 { { gen_helper_sve_ldffbss_zss_mte
,
5330 gen_helper_sve_ldffhss_be_zss_mte
,
5332 { gen_helper_sve_ldffbsu_zss_mte
,
5333 gen_helper_sve_ldffhsu_be_zss_mte
,
5334 gen_helper_sve_ldffss_be_zss_mte
, } } } } },
5337 /* Note that we overload xs=2 to indicate 64-bit offset. */
5338 static gen_helper_gvec_mem_scatter
* const
5339 gather_load_fn64
[2][2][2][3][2][4] = {
5340 { /* MTE Inactive */
5341 { /* Little-endian */
5342 { { { gen_helper_sve_ldbds_zsu
,
5343 gen_helper_sve_ldhds_le_zsu
,
5344 gen_helper_sve_ldsds_le_zsu
,
5346 { gen_helper_sve_ldbdu_zsu
,
5347 gen_helper_sve_ldhdu_le_zsu
,
5348 gen_helper_sve_ldsdu_le_zsu
,
5349 gen_helper_sve_lddd_le_zsu
, } },
5350 { { gen_helper_sve_ldbds_zss
,
5351 gen_helper_sve_ldhds_le_zss
,
5352 gen_helper_sve_ldsds_le_zss
,
5354 { gen_helper_sve_ldbdu_zss
,
5355 gen_helper_sve_ldhdu_le_zss
,
5356 gen_helper_sve_ldsdu_le_zss
,
5357 gen_helper_sve_lddd_le_zss
, } },
5358 { { gen_helper_sve_ldbds_zd
,
5359 gen_helper_sve_ldhds_le_zd
,
5360 gen_helper_sve_ldsds_le_zd
,
5362 { gen_helper_sve_ldbdu_zd
,
5363 gen_helper_sve_ldhdu_le_zd
,
5364 gen_helper_sve_ldsdu_le_zd
,
5365 gen_helper_sve_lddd_le_zd
, } } },
5368 { { { gen_helper_sve_ldffbds_zsu
,
5369 gen_helper_sve_ldffhds_le_zsu
,
5370 gen_helper_sve_ldffsds_le_zsu
,
5372 { gen_helper_sve_ldffbdu_zsu
,
5373 gen_helper_sve_ldffhdu_le_zsu
,
5374 gen_helper_sve_ldffsdu_le_zsu
,
5375 gen_helper_sve_ldffdd_le_zsu
, } },
5376 { { gen_helper_sve_ldffbds_zss
,
5377 gen_helper_sve_ldffhds_le_zss
,
5378 gen_helper_sve_ldffsds_le_zss
,
5380 { gen_helper_sve_ldffbdu_zss
,
5381 gen_helper_sve_ldffhdu_le_zss
,
5382 gen_helper_sve_ldffsdu_le_zss
,
5383 gen_helper_sve_ldffdd_le_zss
, } },
5384 { { gen_helper_sve_ldffbds_zd
,
5385 gen_helper_sve_ldffhds_le_zd
,
5386 gen_helper_sve_ldffsds_le_zd
,
5388 { gen_helper_sve_ldffbdu_zd
,
5389 gen_helper_sve_ldffhdu_le_zd
,
5390 gen_helper_sve_ldffsdu_le_zd
,
5391 gen_helper_sve_ldffdd_le_zd
, } } } },
5393 { { { gen_helper_sve_ldbds_zsu
,
5394 gen_helper_sve_ldhds_be_zsu
,
5395 gen_helper_sve_ldsds_be_zsu
,
5397 { gen_helper_sve_ldbdu_zsu
,
5398 gen_helper_sve_ldhdu_be_zsu
,
5399 gen_helper_sve_ldsdu_be_zsu
,
5400 gen_helper_sve_lddd_be_zsu
, } },
5401 { { gen_helper_sve_ldbds_zss
,
5402 gen_helper_sve_ldhds_be_zss
,
5403 gen_helper_sve_ldsds_be_zss
,
5405 { gen_helper_sve_ldbdu_zss
,
5406 gen_helper_sve_ldhdu_be_zss
,
5407 gen_helper_sve_ldsdu_be_zss
,
5408 gen_helper_sve_lddd_be_zss
, } },
5409 { { gen_helper_sve_ldbds_zd
,
5410 gen_helper_sve_ldhds_be_zd
,
5411 gen_helper_sve_ldsds_be_zd
,
5413 { gen_helper_sve_ldbdu_zd
,
5414 gen_helper_sve_ldhdu_be_zd
,
5415 gen_helper_sve_ldsdu_be_zd
,
5416 gen_helper_sve_lddd_be_zd
, } } },
5419 { { { gen_helper_sve_ldffbds_zsu
,
5420 gen_helper_sve_ldffhds_be_zsu
,
5421 gen_helper_sve_ldffsds_be_zsu
,
5423 { gen_helper_sve_ldffbdu_zsu
,
5424 gen_helper_sve_ldffhdu_be_zsu
,
5425 gen_helper_sve_ldffsdu_be_zsu
,
5426 gen_helper_sve_ldffdd_be_zsu
, } },
5427 { { gen_helper_sve_ldffbds_zss
,
5428 gen_helper_sve_ldffhds_be_zss
,
5429 gen_helper_sve_ldffsds_be_zss
,
5431 { gen_helper_sve_ldffbdu_zss
,
5432 gen_helper_sve_ldffhdu_be_zss
,
5433 gen_helper_sve_ldffsdu_be_zss
,
5434 gen_helper_sve_ldffdd_be_zss
, } },
5435 { { gen_helper_sve_ldffbds_zd
,
5436 gen_helper_sve_ldffhds_be_zd
,
5437 gen_helper_sve_ldffsds_be_zd
,
5439 { gen_helper_sve_ldffbdu_zd
,
5440 gen_helper_sve_ldffhdu_be_zd
,
5441 gen_helper_sve_ldffsdu_be_zd
,
5442 gen_helper_sve_ldffdd_be_zd
, } } } } },
5444 { /* Little-endian */
5445 { { { gen_helper_sve_ldbds_zsu_mte
,
5446 gen_helper_sve_ldhds_le_zsu_mte
,
5447 gen_helper_sve_ldsds_le_zsu_mte
,
5449 { gen_helper_sve_ldbdu_zsu_mte
,
5450 gen_helper_sve_ldhdu_le_zsu_mte
,
5451 gen_helper_sve_ldsdu_le_zsu_mte
,
5452 gen_helper_sve_lddd_le_zsu_mte
, } },
5453 { { gen_helper_sve_ldbds_zss_mte
,
5454 gen_helper_sve_ldhds_le_zss_mte
,
5455 gen_helper_sve_ldsds_le_zss_mte
,
5457 { gen_helper_sve_ldbdu_zss_mte
,
5458 gen_helper_sve_ldhdu_le_zss_mte
,
5459 gen_helper_sve_ldsdu_le_zss_mte
,
5460 gen_helper_sve_lddd_le_zss_mte
, } },
5461 { { gen_helper_sve_ldbds_zd_mte
,
5462 gen_helper_sve_ldhds_le_zd_mte
,
5463 gen_helper_sve_ldsds_le_zd_mte
,
5465 { gen_helper_sve_ldbdu_zd_mte
,
5466 gen_helper_sve_ldhdu_le_zd_mte
,
5467 gen_helper_sve_ldsdu_le_zd_mte
,
5468 gen_helper_sve_lddd_le_zd_mte
, } } },
5471 { { { gen_helper_sve_ldffbds_zsu_mte
,
5472 gen_helper_sve_ldffhds_le_zsu_mte
,
5473 gen_helper_sve_ldffsds_le_zsu_mte
,
5475 { gen_helper_sve_ldffbdu_zsu_mte
,
5476 gen_helper_sve_ldffhdu_le_zsu_mte
,
5477 gen_helper_sve_ldffsdu_le_zsu_mte
,
5478 gen_helper_sve_ldffdd_le_zsu_mte
, } },
5479 { { gen_helper_sve_ldffbds_zss_mte
,
5480 gen_helper_sve_ldffhds_le_zss_mte
,
5481 gen_helper_sve_ldffsds_le_zss_mte
,
5483 { gen_helper_sve_ldffbdu_zss_mte
,
5484 gen_helper_sve_ldffhdu_le_zss_mte
,
5485 gen_helper_sve_ldffsdu_le_zss_mte
,
5486 gen_helper_sve_ldffdd_le_zss_mte
, } },
5487 { { gen_helper_sve_ldffbds_zd_mte
,
5488 gen_helper_sve_ldffhds_le_zd_mte
,
5489 gen_helper_sve_ldffsds_le_zd_mte
,
5491 { gen_helper_sve_ldffbdu_zd_mte
,
5492 gen_helper_sve_ldffhdu_le_zd_mte
,
5493 gen_helper_sve_ldffsdu_le_zd_mte
,
5494 gen_helper_sve_ldffdd_le_zd_mte
, } } } },
5496 { { { gen_helper_sve_ldbds_zsu_mte
,
5497 gen_helper_sve_ldhds_be_zsu_mte
,
5498 gen_helper_sve_ldsds_be_zsu_mte
,
5500 { gen_helper_sve_ldbdu_zsu_mte
,
5501 gen_helper_sve_ldhdu_be_zsu_mte
,
5502 gen_helper_sve_ldsdu_be_zsu_mte
,
5503 gen_helper_sve_lddd_be_zsu_mte
, } },
5504 { { gen_helper_sve_ldbds_zss_mte
,
5505 gen_helper_sve_ldhds_be_zss_mte
,
5506 gen_helper_sve_ldsds_be_zss_mte
,
5508 { gen_helper_sve_ldbdu_zss_mte
,
5509 gen_helper_sve_ldhdu_be_zss_mte
,
5510 gen_helper_sve_ldsdu_be_zss_mte
,
5511 gen_helper_sve_lddd_be_zss_mte
, } },
5512 { { gen_helper_sve_ldbds_zd_mte
,
5513 gen_helper_sve_ldhds_be_zd_mte
,
5514 gen_helper_sve_ldsds_be_zd_mte
,
5516 { gen_helper_sve_ldbdu_zd_mte
,
5517 gen_helper_sve_ldhdu_be_zd_mte
,
5518 gen_helper_sve_ldsdu_be_zd_mte
,
5519 gen_helper_sve_lddd_be_zd_mte
, } } },
5522 { { { gen_helper_sve_ldffbds_zsu_mte
,
5523 gen_helper_sve_ldffhds_be_zsu_mte
,
5524 gen_helper_sve_ldffsds_be_zsu_mte
,
5526 { gen_helper_sve_ldffbdu_zsu_mte
,
5527 gen_helper_sve_ldffhdu_be_zsu_mte
,
5528 gen_helper_sve_ldffsdu_be_zsu_mte
,
5529 gen_helper_sve_ldffdd_be_zsu_mte
, } },
5530 { { gen_helper_sve_ldffbds_zss_mte
,
5531 gen_helper_sve_ldffhds_be_zss_mte
,
5532 gen_helper_sve_ldffsds_be_zss_mte
,
5534 { gen_helper_sve_ldffbdu_zss_mte
,
5535 gen_helper_sve_ldffhdu_be_zss_mte
,
5536 gen_helper_sve_ldffsdu_be_zss_mte
,
5537 gen_helper_sve_ldffdd_be_zss_mte
, } },
5538 { { gen_helper_sve_ldffbds_zd_mte
,
5539 gen_helper_sve_ldffhds_be_zd_mte
,
5540 gen_helper_sve_ldffsds_be_zd_mte
,
5542 { gen_helper_sve_ldffbdu_zd_mte
,
5543 gen_helper_sve_ldffhdu_be_zd_mte
,
5544 gen_helper_sve_ldffsdu_be_zd_mte
,
5545 gen_helper_sve_ldffdd_be_zd_mte
, } } } } },
5548 static bool trans_LD1_zprz(DisasContext
*s
, arg_LD1_zprz
*a
)
5550 gen_helper_gvec_mem_scatter
*fn
= NULL
;
5551 bool be
= s
->be_data
== MO_BE
;
5552 bool mte
= s
->mte_active
[0];
5554 if (!sve_access_check(s
)) {
5560 fn
= gather_load_fn32
[mte
][be
][a
->ff
][a
->xs
][a
->u
][a
->msz
];
5563 fn
= gather_load_fn64
[mte
][be
][a
->ff
][a
->xs
][a
->u
][a
->msz
];
5568 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rm
, a
->scale
* a
->msz
,
5569 cpu_reg_sp(s
, a
->rn
), a
->msz
, false, fn
);
5573 static bool trans_LD1_zpiz(DisasContext
*s
, arg_LD1_zpiz
*a
)
5575 gen_helper_gvec_mem_scatter
*fn
= NULL
;
5576 bool be
= s
->be_data
== MO_BE
;
5577 bool mte
= s
->mte_active
[0];
5580 if (a
->esz
< a
->msz
|| (a
->esz
== a
->msz
&& !a
->u
)) {
5583 if (!sve_access_check(s
)) {
5589 fn
= gather_load_fn32
[mte
][be
][a
->ff
][0][a
->u
][a
->msz
];
5592 fn
= gather_load_fn64
[mte
][be
][a
->ff
][2][a
->u
][a
->msz
];
5597 /* Treat LD1_zpiz (zn[x] + imm) the same way as LD1_zprz (rn + zm[x])
5598 * by loading the immediate into the scalar parameter.
5600 imm
= tcg_const_i64(a
->imm
<< a
->msz
);
5601 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0, imm
, a
->msz
, false, fn
);
5602 tcg_temp_free_i64(imm
);
5606 /* Indexed by [mte][be][xs][msz]. */
5607 static gen_helper_gvec_mem_scatter
* const scatter_store_fn32
[2][2][2][3] = {
5608 { /* MTE Inactive */
5609 { /* Little-endian */
5610 { gen_helper_sve_stbs_zsu
,
5611 gen_helper_sve_sths_le_zsu
,
5612 gen_helper_sve_stss_le_zsu
, },
5613 { gen_helper_sve_stbs_zss
,
5614 gen_helper_sve_sths_le_zss
,
5615 gen_helper_sve_stss_le_zss
, } },
5617 { gen_helper_sve_stbs_zsu
,
5618 gen_helper_sve_sths_be_zsu
,
5619 gen_helper_sve_stss_be_zsu
, },
5620 { gen_helper_sve_stbs_zss
,
5621 gen_helper_sve_sths_be_zss
,
5622 gen_helper_sve_stss_be_zss
, } } },
5624 { /* Little-endian */
5625 { gen_helper_sve_stbs_zsu_mte
,
5626 gen_helper_sve_sths_le_zsu_mte
,
5627 gen_helper_sve_stss_le_zsu_mte
, },
5628 { gen_helper_sve_stbs_zss_mte
,
5629 gen_helper_sve_sths_le_zss_mte
,
5630 gen_helper_sve_stss_le_zss_mte
, } },
5632 { gen_helper_sve_stbs_zsu_mte
,
5633 gen_helper_sve_sths_be_zsu_mte
,
5634 gen_helper_sve_stss_be_zsu_mte
, },
5635 { gen_helper_sve_stbs_zss_mte
,
5636 gen_helper_sve_sths_be_zss_mte
,
5637 gen_helper_sve_stss_be_zss_mte
, } } },
5640 /* Note that we overload xs=2 to indicate 64-bit offset. */
5641 static gen_helper_gvec_mem_scatter
* const scatter_store_fn64
[2][2][3][4] = {
5642 { /* MTE Inactive */
5643 { /* Little-endian */
5644 { gen_helper_sve_stbd_zsu
,
5645 gen_helper_sve_sthd_le_zsu
,
5646 gen_helper_sve_stsd_le_zsu
,
5647 gen_helper_sve_stdd_le_zsu
, },
5648 { gen_helper_sve_stbd_zss
,
5649 gen_helper_sve_sthd_le_zss
,
5650 gen_helper_sve_stsd_le_zss
,
5651 gen_helper_sve_stdd_le_zss
, },
5652 { gen_helper_sve_stbd_zd
,
5653 gen_helper_sve_sthd_le_zd
,
5654 gen_helper_sve_stsd_le_zd
,
5655 gen_helper_sve_stdd_le_zd
, } },
5657 { gen_helper_sve_stbd_zsu
,
5658 gen_helper_sve_sthd_be_zsu
,
5659 gen_helper_sve_stsd_be_zsu
,
5660 gen_helper_sve_stdd_be_zsu
, },
5661 { gen_helper_sve_stbd_zss
,
5662 gen_helper_sve_sthd_be_zss
,
5663 gen_helper_sve_stsd_be_zss
,
5664 gen_helper_sve_stdd_be_zss
, },
5665 { gen_helper_sve_stbd_zd
,
5666 gen_helper_sve_sthd_be_zd
,
5667 gen_helper_sve_stsd_be_zd
,
5668 gen_helper_sve_stdd_be_zd
, } } },
5669 { /* MTE Inactive */
5670 { /* Little-endian */
5671 { gen_helper_sve_stbd_zsu_mte
,
5672 gen_helper_sve_sthd_le_zsu_mte
,
5673 gen_helper_sve_stsd_le_zsu_mte
,
5674 gen_helper_sve_stdd_le_zsu_mte
, },
5675 { gen_helper_sve_stbd_zss_mte
,
5676 gen_helper_sve_sthd_le_zss_mte
,
5677 gen_helper_sve_stsd_le_zss_mte
,
5678 gen_helper_sve_stdd_le_zss_mte
, },
5679 { gen_helper_sve_stbd_zd_mte
,
5680 gen_helper_sve_sthd_le_zd_mte
,
5681 gen_helper_sve_stsd_le_zd_mte
,
5682 gen_helper_sve_stdd_le_zd_mte
, } },
5684 { gen_helper_sve_stbd_zsu_mte
,
5685 gen_helper_sve_sthd_be_zsu_mte
,
5686 gen_helper_sve_stsd_be_zsu_mte
,
5687 gen_helper_sve_stdd_be_zsu_mte
, },
5688 { gen_helper_sve_stbd_zss_mte
,
5689 gen_helper_sve_sthd_be_zss_mte
,
5690 gen_helper_sve_stsd_be_zss_mte
,
5691 gen_helper_sve_stdd_be_zss_mte
, },
5692 { gen_helper_sve_stbd_zd_mte
,
5693 gen_helper_sve_sthd_be_zd_mte
,
5694 gen_helper_sve_stsd_be_zd_mte
,
5695 gen_helper_sve_stdd_be_zd_mte
, } } },
5698 static bool trans_ST1_zprz(DisasContext
*s
, arg_ST1_zprz
*a
)
5700 gen_helper_gvec_mem_scatter
*fn
;
5701 bool be
= s
->be_data
== MO_BE
;
5702 bool mte
= s
->mte_active
[0];
5704 if (a
->esz
< a
->msz
|| (a
->msz
== 0 && a
->scale
)) {
5707 if (!sve_access_check(s
)) {
5712 fn
= scatter_store_fn32
[mte
][be
][a
->xs
][a
->msz
];
5715 fn
= scatter_store_fn64
[mte
][be
][a
->xs
][a
->msz
];
5718 g_assert_not_reached();
5720 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rm
, a
->scale
* a
->msz
,
5721 cpu_reg_sp(s
, a
->rn
), a
->msz
, true, fn
);
5725 static bool trans_ST1_zpiz(DisasContext
*s
, arg_ST1_zpiz
*a
)
5727 gen_helper_gvec_mem_scatter
*fn
= NULL
;
5728 bool be
= s
->be_data
== MO_BE
;
5729 bool mte
= s
->mte_active
[0];
5732 if (a
->esz
< a
->msz
) {
5735 if (!sve_access_check(s
)) {
5741 fn
= scatter_store_fn32
[mte
][be
][0][a
->msz
];
5744 fn
= scatter_store_fn64
[mte
][be
][2][a
->msz
];
5749 /* Treat ST1_zpiz (zn[x] + imm) the same way as ST1_zprz (rn + zm[x])
5750 * by loading the immediate into the scalar parameter.
5752 imm
= tcg_const_i64(a
->imm
<< a
->msz
);
5753 do_mem_zpz(s
, a
->rd
, a
->pg
, a
->rn
, 0, imm
, a
->msz
, true, fn
);
5754 tcg_temp_free_i64(imm
);
5762 static bool trans_PRF(DisasContext
*s
, arg_PRF
*a
)
5764 /* Prefetch is a nop within QEMU. */
5765 (void)sve_access_check(s
);
5769 static bool trans_PRF_rr(DisasContext
*s
, arg_PRF_rr
*a
)
5774 /* Prefetch is a nop within QEMU. */
5775 (void)sve_access_check(s
);
5782 * TODO: The implementation so far could handle predicated merging movprfx.
5783 * The helper functions as written take an extra source register to
5784 * use in the operation, but the result is only written when predication
5785 * succeeds. For unpredicated movprfx, we need to rearrange the helpers
5786 * to allow the final write back to the destination to be unconditional.
5787 * For predicated zeroing movprfx, we need to rearrange the helpers to
5788 * allow the final write back to zero inactives.
5790 * In the meantime, just emit the moves.
5793 static bool trans_MOVPRFX(DisasContext
*s
, arg_MOVPRFX
*a
)
5795 return do_mov_z(s
, a
->rd
, a
->rn
);
5798 static bool trans_MOVPRFX_m(DisasContext
*s
, arg_rpr_esz
*a
)
5800 if (sve_access_check(s
)) {
5801 do_sel_z(s
, a
->rd
, a
->rn
, a
->rd
, a
->pg
, a
->esz
);
5806 static bool trans_MOVPRFX_z(DisasContext
*s
, arg_rpr_esz
*a
)
5808 return do_movz_zpz(s
, a
->rd
, a
->rn
, a
->pg
, a
->esz
, false);
5812 * SVE2 Integer Multiply - Unpredicated
5815 static bool trans_MUL_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
5817 if (!dc_isar_feature(aa64_sve2
, s
)) {
5820 if (sve_access_check(s
)) {
5821 gen_gvec_fn_zzz(s
, tcg_gen_gvec_mul
, a
->esz
, a
->rd
, a
->rn
, a
->rm
);
5826 static bool do_sve2_zzz_ool(DisasContext
*s
, arg_rrr_esz
*a
,
5827 gen_helper_gvec_3
*fn
)
5829 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
5832 if (sve_access_check(s
)) {
5833 gen_gvec_ool_zzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, 0);
5838 static bool trans_SMULH_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
5840 static gen_helper_gvec_3
* const fns
[4] = {
5841 gen_helper_gvec_smulh_b
, gen_helper_gvec_smulh_h
,
5842 gen_helper_gvec_smulh_s
, gen_helper_gvec_smulh_d
,
5844 return do_sve2_zzz_ool(s
, a
, fns
[a
->esz
]);
5847 static bool trans_UMULH_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
5849 static gen_helper_gvec_3
* const fns
[4] = {
5850 gen_helper_gvec_umulh_b
, gen_helper_gvec_umulh_h
,
5851 gen_helper_gvec_umulh_s
, gen_helper_gvec_umulh_d
,
5853 return do_sve2_zzz_ool(s
, a
, fns
[a
->esz
]);
5856 static bool trans_PMUL_zzz(DisasContext
*s
, arg_rrr_esz
*a
)
5858 return do_sve2_zzz_ool(s
, a
, gen_helper_gvec_pmul_b
);
5862 * SVE2 Integer - Predicated
5865 static bool do_sve2_zpzz_ool(DisasContext
*s
, arg_rprr_esz
*a
,
5866 gen_helper_gvec_4
*fn
)
5868 if (!dc_isar_feature(aa64_sve2
, s
)) {
5871 return do_zpzz_ool(s
, a
, fn
);
5874 static bool trans_SADALP_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
5876 static gen_helper_gvec_4
* const fns
[3] = {
5877 gen_helper_sve2_sadalp_zpzz_h
,
5878 gen_helper_sve2_sadalp_zpzz_s
,
5879 gen_helper_sve2_sadalp_zpzz_d
,
5884 return do_sve2_zpzz_ool(s
, a
, fns
[a
->esz
- 1]);
5887 static bool trans_UADALP_zpzz(DisasContext
*s
, arg_rprr_esz
*a
)
5889 static gen_helper_gvec_4
* const fns
[3] = {
5890 gen_helper_sve2_uadalp_zpzz_h
,
5891 gen_helper_sve2_uadalp_zpzz_s
,
5892 gen_helper_sve2_uadalp_zpzz_d
,
5897 return do_sve2_zpzz_ool(s
, a
, fns
[a
->esz
- 1]);
5901 * SVE2 integer unary operations (predicated)
5904 static bool do_sve2_zpz_ool(DisasContext
*s
, arg_rpr_esz
*a
,
5905 gen_helper_gvec_3
*fn
)
5907 if (!dc_isar_feature(aa64_sve2
, s
)) {
5910 return do_zpz_ool(s
, a
, fn
);
5913 static bool trans_URECPE(DisasContext
*s
, arg_rpr_esz
*a
)
5918 return do_sve2_zpz_ool(s
, a
, gen_helper_sve2_urecpe_s
);
5921 static bool trans_URSQRTE(DisasContext
*s
, arg_rpr_esz
*a
)
5926 return do_sve2_zpz_ool(s
, a
, gen_helper_sve2_ursqrte_s
);
5929 static bool trans_SQABS(DisasContext
*s
, arg_rpr_esz
*a
)
5931 static gen_helper_gvec_3
* const fns
[4] = {
5932 gen_helper_sve2_sqabs_b
, gen_helper_sve2_sqabs_h
,
5933 gen_helper_sve2_sqabs_s
, gen_helper_sve2_sqabs_d
,
5935 return do_sve2_zpz_ool(s
, a
, fns
[a
->esz
]);
5938 static bool trans_SQNEG(DisasContext
*s
, arg_rpr_esz
*a
)
5940 static gen_helper_gvec_3
* const fns
[4] = {
5941 gen_helper_sve2_sqneg_b
, gen_helper_sve2_sqneg_h
,
5942 gen_helper_sve2_sqneg_s
, gen_helper_sve2_sqneg_d
,
5944 return do_sve2_zpz_ool(s
, a
, fns
[a
->esz
]);
5947 #define DO_SVE2_ZPZZ(NAME, name) \
5948 static bool trans_##NAME(DisasContext *s, arg_rprr_esz *a) \
5950 static gen_helper_gvec_4 * const fns[4] = { \
5951 gen_helper_sve2_##name##_zpzz_b, gen_helper_sve2_##name##_zpzz_h, \
5952 gen_helper_sve2_##name##_zpzz_s, gen_helper_sve2_##name##_zpzz_d, \
5954 return do_sve2_zpzz_ool(s, a, fns[a->esz]); \
5957 DO_SVE2_ZPZZ(SQSHL
, sqshl
)
5958 DO_SVE2_ZPZZ(SQRSHL
, sqrshl
)
5959 DO_SVE2_ZPZZ(SRSHL
, srshl
)
5961 DO_SVE2_ZPZZ(UQSHL
, uqshl
)
5962 DO_SVE2_ZPZZ(UQRSHL
, uqrshl
)
5963 DO_SVE2_ZPZZ(URSHL
, urshl
)
5965 DO_SVE2_ZPZZ(SHADD
, shadd
)
5966 DO_SVE2_ZPZZ(SRHADD
, srhadd
)
5967 DO_SVE2_ZPZZ(SHSUB
, shsub
)
5969 DO_SVE2_ZPZZ(UHADD
, uhadd
)
5970 DO_SVE2_ZPZZ(URHADD
, urhadd
)
5971 DO_SVE2_ZPZZ(UHSUB
, uhsub
)
5973 DO_SVE2_ZPZZ(ADDP
, addp
)
5974 DO_SVE2_ZPZZ(SMAXP
, smaxp
)
5975 DO_SVE2_ZPZZ(UMAXP
, umaxp
)
5976 DO_SVE2_ZPZZ(SMINP
, sminp
)
5977 DO_SVE2_ZPZZ(UMINP
, uminp
)
5979 DO_SVE2_ZPZZ(SQADD_zpzz
, sqadd
)
5980 DO_SVE2_ZPZZ(UQADD_zpzz
, uqadd
)
5981 DO_SVE2_ZPZZ(SQSUB_zpzz
, sqsub
)
5982 DO_SVE2_ZPZZ(UQSUB_zpzz
, uqsub
)
5983 DO_SVE2_ZPZZ(SUQADD
, suqadd
)
5984 DO_SVE2_ZPZZ(USQADD
, usqadd
)
5987 * SVE2 Widening Integer Arithmetic
5990 static bool do_sve2_zzw_ool(DisasContext
*s
, arg_rrr_esz
*a
,
5991 gen_helper_gvec_3
*fn
, int data
)
5993 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
5996 if (sve_access_check(s
)) {
5997 unsigned vsz
= vec_full_reg_size(s
);
5998 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, a
->rd
),
5999 vec_full_reg_offset(s
, a
->rn
),
6000 vec_full_reg_offset(s
, a
->rm
),
6001 vsz
, vsz
, data
, fn
);
6006 #define DO_SVE2_ZZZ_TB(NAME, name, SEL1, SEL2) \
6007 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
6009 static gen_helper_gvec_3 * const fns[4] = { \
6010 NULL, gen_helper_sve2_##name##_h, \
6011 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
6013 return do_sve2_zzw_ool(s, a, fns[a->esz], (SEL2 << 1) | SEL1); \
6016 DO_SVE2_ZZZ_TB(SADDLB
, saddl
, false, false)
6017 DO_SVE2_ZZZ_TB(SSUBLB
, ssubl
, false, false)
6018 DO_SVE2_ZZZ_TB(SABDLB
, sabdl
, false, false)
6020 DO_SVE2_ZZZ_TB(UADDLB
, uaddl
, false, false)
6021 DO_SVE2_ZZZ_TB(USUBLB
, usubl
, false, false)
6022 DO_SVE2_ZZZ_TB(UABDLB
, uabdl
, false, false)
6024 DO_SVE2_ZZZ_TB(SADDLT
, saddl
, true, true)
6025 DO_SVE2_ZZZ_TB(SSUBLT
, ssubl
, true, true)
6026 DO_SVE2_ZZZ_TB(SABDLT
, sabdl
, true, true)
6028 DO_SVE2_ZZZ_TB(UADDLT
, uaddl
, true, true)
6029 DO_SVE2_ZZZ_TB(USUBLT
, usubl
, true, true)
6030 DO_SVE2_ZZZ_TB(UABDLT
, uabdl
, true, true)
6032 DO_SVE2_ZZZ_TB(SADDLBT
, saddl
, false, true)
6033 DO_SVE2_ZZZ_TB(SSUBLBT
, ssubl
, false, true)
6034 DO_SVE2_ZZZ_TB(SSUBLTB
, ssubl
, true, false)
6036 DO_SVE2_ZZZ_TB(SQDMULLB_zzz
, sqdmull_zzz
, false, false)
6037 DO_SVE2_ZZZ_TB(SQDMULLT_zzz
, sqdmull_zzz
, true, true)
6039 DO_SVE2_ZZZ_TB(SMULLB_zzz
, smull_zzz
, false, false)
6040 DO_SVE2_ZZZ_TB(SMULLT_zzz
, smull_zzz
, true, true)
6042 DO_SVE2_ZZZ_TB(UMULLB_zzz
, umull_zzz
, false, false)
6043 DO_SVE2_ZZZ_TB(UMULLT_zzz
, umull_zzz
, true, true)
6045 static bool do_eor_tb(DisasContext
*s
, arg_rrr_esz
*a
, bool sel1
)
6047 static gen_helper_gvec_3
* const fns
[4] = {
6048 gen_helper_sve2_eoril_b
, gen_helper_sve2_eoril_h
,
6049 gen_helper_sve2_eoril_s
, gen_helper_sve2_eoril_d
,
6051 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], (!sel1
<< 1) | sel1
);
6054 static bool trans_EORBT(DisasContext
*s
, arg_rrr_esz
*a
)
6056 return do_eor_tb(s
, a
, false);
6059 static bool trans_EORTB(DisasContext
*s
, arg_rrr_esz
*a
)
6061 return do_eor_tb(s
, a
, true);
6064 static bool do_trans_pmull(DisasContext
*s
, arg_rrr_esz
*a
, bool sel
)
6066 static gen_helper_gvec_3
* const fns
[4] = {
6067 gen_helper_gvec_pmull_q
, gen_helper_sve2_pmull_h
,
6068 NULL
, gen_helper_sve2_pmull_d
,
6070 if (a
->esz
== 0 && !dc_isar_feature(aa64_sve2_pmull128
, s
)) {
6073 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], sel
);
6076 static bool trans_PMULLB(DisasContext
*s
, arg_rrr_esz
*a
)
6078 return do_trans_pmull(s
, a
, false);
6081 static bool trans_PMULLT(DisasContext
*s
, arg_rrr_esz
*a
)
6083 return do_trans_pmull(s
, a
, true);
6086 #define DO_SVE2_ZZZ_WTB(NAME, name, SEL2) \
6087 static bool trans_##NAME(DisasContext *s, arg_rrr_esz *a) \
6089 static gen_helper_gvec_3 * const fns[4] = { \
6090 NULL, gen_helper_sve2_##name##_h, \
6091 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
6093 return do_sve2_zzw_ool(s, a, fns[a->esz], SEL2); \
6096 DO_SVE2_ZZZ_WTB(SADDWB
, saddw
, false)
6097 DO_SVE2_ZZZ_WTB(SADDWT
, saddw
, true)
6098 DO_SVE2_ZZZ_WTB(SSUBWB
, ssubw
, false)
6099 DO_SVE2_ZZZ_WTB(SSUBWT
, ssubw
, true)
6101 DO_SVE2_ZZZ_WTB(UADDWB
, uaddw
, false)
6102 DO_SVE2_ZZZ_WTB(UADDWT
, uaddw
, true)
6103 DO_SVE2_ZZZ_WTB(USUBWB
, usubw
, false)
6104 DO_SVE2_ZZZ_WTB(USUBWT
, usubw
, true)
6106 static void gen_sshll_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t imm
)
6110 int halfbits
= 4 << vece
;
6113 if (shl
== halfbits
) {
6114 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6115 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(halfbits
, halfbits
));
6116 tcg_gen_and_vec(vece
, d
, n
, t
);
6117 tcg_temp_free_vec(t
);
6119 tcg_gen_sari_vec(vece
, d
, n
, halfbits
);
6120 tcg_gen_shli_vec(vece
, d
, d
, shl
);
6123 tcg_gen_shli_vec(vece
, d
, n
, halfbits
);
6124 tcg_gen_sari_vec(vece
, d
, d
, halfbits
- shl
);
6128 static void gen_ushll_i64(unsigned vece
, TCGv_i64 d
, TCGv_i64 n
, int imm
)
6130 int halfbits
= 4 << vece
;
6132 int shl
= (imm
>> 1);
6136 mask
= MAKE_64BIT_MASK(0, halfbits
);
6138 mask
= dup_const(vece
, mask
);
6140 shift
= shl
- top
* halfbits
;
6142 tcg_gen_shri_i64(d
, n
, -shift
);
6144 tcg_gen_shli_i64(d
, n
, shift
);
6146 tcg_gen_andi_i64(d
, d
, mask
);
6149 static void gen_ushll16_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6151 gen_ushll_i64(MO_16
, d
, n
, imm
);
6154 static void gen_ushll32_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6156 gen_ushll_i64(MO_32
, d
, n
, imm
);
6159 static void gen_ushll64_i64(TCGv_i64 d
, TCGv_i64 n
, int64_t imm
)
6161 gen_ushll_i64(MO_64
, d
, n
, imm
);
6164 static void gen_ushll_vec(unsigned vece
, TCGv_vec d
, TCGv_vec n
, int64_t imm
)
6166 int halfbits
= 4 << vece
;
6171 if (shl
== halfbits
) {
6172 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6173 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(halfbits
, halfbits
));
6174 tcg_gen_and_vec(vece
, d
, n
, t
);
6175 tcg_temp_free_vec(t
);
6177 tcg_gen_shri_vec(vece
, d
, n
, halfbits
);
6178 tcg_gen_shli_vec(vece
, d
, d
, shl
);
6182 TCGv_vec t
= tcg_temp_new_vec_matching(d
);
6183 tcg_gen_dupi_vec(vece
, t
, MAKE_64BIT_MASK(0, halfbits
));
6184 tcg_gen_and_vec(vece
, d
, n
, t
);
6185 tcg_temp_free_vec(t
);
6187 tcg_gen_shli_vec(vece
, d
, n
, halfbits
);
6188 tcg_gen_shri_vec(vece
, d
, d
, halfbits
- shl
);
6193 static bool do_sve2_shll_tb(DisasContext
*s
, arg_rri_esz
*a
,
6196 static const TCGOpcode sshll_list
[] = {
6197 INDEX_op_shli_vec
, INDEX_op_sari_vec
, 0
6199 static const TCGOpcode ushll_list
[] = {
6200 INDEX_op_shli_vec
, INDEX_op_shri_vec
, 0
6202 static const GVecGen2i ops
[2][3] = {
6203 { { .fniv
= gen_sshll_vec
,
6204 .opt_opc
= sshll_list
,
6205 .fno
= gen_helper_sve2_sshll_h
,
6207 { .fniv
= gen_sshll_vec
,
6208 .opt_opc
= sshll_list
,
6209 .fno
= gen_helper_sve2_sshll_s
,
6211 { .fniv
= gen_sshll_vec
,
6212 .opt_opc
= sshll_list
,
6213 .fno
= gen_helper_sve2_sshll_d
,
6215 { { .fni8
= gen_ushll16_i64
,
6216 .fniv
= gen_ushll_vec
,
6217 .opt_opc
= ushll_list
,
6218 .fno
= gen_helper_sve2_ushll_h
,
6220 { .fni8
= gen_ushll32_i64
,
6221 .fniv
= gen_ushll_vec
,
6222 .opt_opc
= ushll_list
,
6223 .fno
= gen_helper_sve2_ushll_s
,
6225 { .fni8
= gen_ushll64_i64
,
6226 .fniv
= gen_ushll_vec
,
6227 .opt_opc
= ushll_list
,
6228 .fno
= gen_helper_sve2_ushll_d
,
6232 if (a
->esz
< 0 || a
->esz
> 2 || !dc_isar_feature(aa64_sve2
, s
)) {
6235 if (sve_access_check(s
)) {
6236 unsigned vsz
= vec_full_reg_size(s
);
6237 tcg_gen_gvec_2i(vec_full_reg_offset(s
, a
->rd
),
6238 vec_full_reg_offset(s
, a
->rn
),
6239 vsz
, vsz
, (a
->imm
<< 1) | sel
,
6245 static bool trans_SSHLLB(DisasContext
*s
, arg_rri_esz
*a
)
6247 return do_sve2_shll_tb(s
, a
, false, false);
6250 static bool trans_SSHLLT(DisasContext
*s
, arg_rri_esz
*a
)
6252 return do_sve2_shll_tb(s
, a
, true, false);
6255 static bool trans_USHLLB(DisasContext
*s
, arg_rri_esz
*a
)
6257 return do_sve2_shll_tb(s
, a
, false, true);
6260 static bool trans_USHLLT(DisasContext
*s
, arg_rri_esz
*a
)
6262 return do_sve2_shll_tb(s
, a
, true, true);
6265 static bool trans_BEXT(DisasContext
*s
, arg_rrr_esz
*a
)
6267 static gen_helper_gvec_3
* const fns
[4] = {
6268 gen_helper_sve2_bext_b
, gen_helper_sve2_bext_h
,
6269 gen_helper_sve2_bext_s
, gen_helper_sve2_bext_d
,
6271 if (!dc_isar_feature(aa64_sve2_bitperm
, s
)) {
6274 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], 0);
6277 static bool trans_BDEP(DisasContext
*s
, arg_rrr_esz
*a
)
6279 static gen_helper_gvec_3
* const fns
[4] = {
6280 gen_helper_sve2_bdep_b
, gen_helper_sve2_bdep_h
,
6281 gen_helper_sve2_bdep_s
, gen_helper_sve2_bdep_d
,
6283 if (!dc_isar_feature(aa64_sve2_bitperm
, s
)) {
6286 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], 0);
6289 static bool trans_BGRP(DisasContext
*s
, arg_rrr_esz
*a
)
6291 static gen_helper_gvec_3
* const fns
[4] = {
6292 gen_helper_sve2_bgrp_b
, gen_helper_sve2_bgrp_h
,
6293 gen_helper_sve2_bgrp_s
, gen_helper_sve2_bgrp_d
,
6295 if (!dc_isar_feature(aa64_sve2_bitperm
, s
)) {
6298 return do_sve2_zzw_ool(s
, a
, fns
[a
->esz
], 0);
6301 static bool do_cadd(DisasContext
*s
, arg_rrr_esz
*a
, bool sq
, bool rot
)
6303 static gen_helper_gvec_3
* const fns
[2][4] = {
6304 { gen_helper_sve2_cadd_b
, gen_helper_sve2_cadd_h
,
6305 gen_helper_sve2_cadd_s
, gen_helper_sve2_cadd_d
},
6306 { gen_helper_sve2_sqcadd_b
, gen_helper_sve2_sqcadd_h
,
6307 gen_helper_sve2_sqcadd_s
, gen_helper_sve2_sqcadd_d
},
6309 return do_sve2_zzw_ool(s
, a
, fns
[sq
][a
->esz
], rot
);
6312 static bool trans_CADD_rot90(DisasContext
*s
, arg_rrr_esz
*a
)
6314 return do_cadd(s
, a
, false, false);
6317 static bool trans_CADD_rot270(DisasContext
*s
, arg_rrr_esz
*a
)
6319 return do_cadd(s
, a
, false, true);
6322 static bool trans_SQCADD_rot90(DisasContext
*s
, arg_rrr_esz
*a
)
6324 return do_cadd(s
, a
, true, false);
6327 static bool trans_SQCADD_rot270(DisasContext
*s
, arg_rrr_esz
*a
)
6329 return do_cadd(s
, a
, true, true);
6332 static bool do_sve2_zzzz_ool(DisasContext
*s
, arg_rrrr_esz
*a
,
6333 gen_helper_gvec_4
*fn
, int data
)
6335 if (fn
== NULL
|| !dc_isar_feature(aa64_sve2
, s
)) {
6338 if (sve_access_check(s
)) {
6339 gen_gvec_ool_zzzz(s
, fn
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, data
);
6344 static bool do_abal(DisasContext
*s
, arg_rrrr_esz
*a
, bool uns
, bool sel
)
6346 static gen_helper_gvec_4
* const fns
[2][4] = {
6347 { NULL
, gen_helper_sve2_sabal_h
,
6348 gen_helper_sve2_sabal_s
, gen_helper_sve2_sabal_d
},
6349 { NULL
, gen_helper_sve2_uabal_h
,
6350 gen_helper_sve2_uabal_s
, gen_helper_sve2_uabal_d
},
6352 return do_sve2_zzzz_ool(s
, a
, fns
[uns
][a
->esz
], sel
);
6355 static bool trans_SABALB(DisasContext
*s
, arg_rrrr_esz
*a
)
6357 return do_abal(s
, a
, false, false);
6360 static bool trans_SABALT(DisasContext
*s
, arg_rrrr_esz
*a
)
6362 return do_abal(s
, a
, false, true);
6365 static bool trans_UABALB(DisasContext
*s
, arg_rrrr_esz
*a
)
6367 return do_abal(s
, a
, true, false);
6370 static bool trans_UABALT(DisasContext
*s
, arg_rrrr_esz
*a
)
6372 return do_abal(s
, a
, true, true);
6375 static bool do_adcl(DisasContext
*s
, arg_rrrr_esz
*a
, bool sel
)
6377 static gen_helper_gvec_4
* const fns
[2] = {
6378 gen_helper_sve2_adcl_s
,
6379 gen_helper_sve2_adcl_d
,
6382 * Note that in this case the ESZ field encodes both size and sign.
6383 * Split out 'subtract' into bit 1 of the data field for the helper.
6385 return do_sve2_zzzz_ool(s
, a
, fns
[a
->esz
& 1], (a
->esz
& 2) | sel
);
6388 static bool trans_ADCLB(DisasContext
*s
, arg_rrrr_esz
*a
)
6390 return do_adcl(s
, a
, false);
6393 static bool trans_ADCLT(DisasContext
*s
, arg_rrrr_esz
*a
)
6395 return do_adcl(s
, a
, true);
6398 static bool do_sve2_fn2i(DisasContext
*s
, arg_rri_esz
*a
, GVecGen2iFn
*fn
)
6400 if (a
->esz
< 0 || !dc_isar_feature(aa64_sve2
, s
)) {
6403 if (sve_access_check(s
)) {
6404 unsigned vsz
= vec_full_reg_size(s
);
6405 unsigned rd_ofs
= vec_full_reg_offset(s
, a
->rd
);
6406 unsigned rn_ofs
= vec_full_reg_offset(s
, a
->rn
);
6407 fn(a
->esz
, rd_ofs
, rn_ofs
, a
->imm
, vsz
, vsz
);
6412 static bool trans_SSRA(DisasContext
*s
, arg_rri_esz
*a
)
6414 return do_sve2_fn2i(s
, a
, gen_gvec_ssra
);
6417 static bool trans_USRA(DisasContext
*s
, arg_rri_esz
*a
)
6419 return do_sve2_fn2i(s
, a
, gen_gvec_usra
);
6422 static bool trans_SRSRA(DisasContext
*s
, arg_rri_esz
*a
)
6424 return do_sve2_fn2i(s
, a
, gen_gvec_srsra
);
6427 static bool trans_URSRA(DisasContext
*s
, arg_rri_esz
*a
)
6429 return do_sve2_fn2i(s
, a
, gen_gvec_ursra
);