4 * Copyright (c) 2013 Alexander Graf <agraf@suse.de>
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #include "qemu/osdep.h"
21 #include "exec/exec-all.h"
22 #include "translate.h"
23 #include "translate-a64.h"
26 #include "semihosting/semihost.h"
29 static TCGv_i64 cpu_X
[32];
30 static TCGv_i64 cpu_pc
;
32 /* Load/store exclusive handling */
33 static TCGv_i64 cpu_exclusive_high
;
35 static const char *regnames
[] = {
36 "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7",
37 "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15",
38 "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23",
39 "x24", "x25", "x26", "x27", "x28", "x29", "lr", "sp"
43 A64_SHIFT_TYPE_LSL
= 0,
44 A64_SHIFT_TYPE_LSR
= 1,
45 A64_SHIFT_TYPE_ASR
= 2,
46 A64_SHIFT_TYPE_ROR
= 3
50 * Helpers for extracting complex instruction fields
54 * For load/store with an unsigned 12 bit immediate scaled by the element
55 * size. The input has the immediate field in bits [14:3] and the element
58 static int uimm_scaled(DisasContext
*s
, int x
)
60 unsigned imm
= x
>> 3;
61 unsigned scale
= extract32(x
, 0, 3);
65 /* For load/store memory tags: scale offset by LOG2_TAG_GRANULE */
66 static int scale_by_log2_tag_granule(DisasContext
*s
, int x
)
68 return x
<< LOG2_TAG_GRANULE
;
72 * Include the generated decoders.
75 #include "decode-sme-fa64.c.inc"
76 #include "decode-a64.c.inc"
78 /* Table based decoder typedefs - used when the relevant bits for decode
79 * are too awkwardly scattered across the instruction (eg SIMD).
81 typedef void AArch64DecodeFn(DisasContext
*s
, uint32_t insn
);
83 typedef struct AArch64DecodeTable
{
86 AArch64DecodeFn
*disas_fn
;
89 /* initialize TCG globals. */
90 void a64_translate_init(void)
94 cpu_pc
= tcg_global_mem_new_i64(tcg_env
,
95 offsetof(CPUARMState
, pc
),
97 for (i
= 0; i
< 32; i
++) {
98 cpu_X
[i
] = tcg_global_mem_new_i64(tcg_env
,
99 offsetof(CPUARMState
, xregs
[i
]),
103 cpu_exclusive_high
= tcg_global_mem_new_i64(tcg_env
,
104 offsetof(CPUARMState
, exclusive_high
), "exclusive_high");
108 * Return the core mmu_idx to use for A64 load/store insns which
109 * have a "unprivileged load/store" variant. Those insns access
110 * EL0 if executed from an EL which has control over EL0 (usually
111 * EL1) but behave like normal loads and stores if executed from
112 * elsewhere (eg EL3).
114 * @unpriv : true for the unprivileged encoding; false for the
115 * normal encoding (in which case we will return the same
116 * thing as get_mem_index().
118 static int get_a64_user_mem_index(DisasContext
*s
, bool unpriv
)
121 * If AccType_UNPRIV is not used, the insn uses AccType_NORMAL,
122 * which is the usual mmu_idx for this cpu state.
124 ARMMMUIdx useridx
= s
->mmu_idx
;
126 if (unpriv
&& s
->unpriv
) {
128 * We have pre-computed the condition for AccType_UNPRIV.
129 * Therefore we should never get here with a mmu_idx for
130 * which we do not know the corresponding user mmu_idx.
133 case ARMMMUIdx_E10_1
:
134 case ARMMMUIdx_E10_1_PAN
:
135 useridx
= ARMMMUIdx_E10_0
;
137 case ARMMMUIdx_E20_2
:
138 case ARMMMUIdx_E20_2_PAN
:
139 useridx
= ARMMMUIdx_E20_0
;
142 g_assert_not_reached();
145 return arm_to_core_mmu_idx(useridx
);
148 static void set_btype_raw(int val
)
150 tcg_gen_st_i32(tcg_constant_i32(val
), tcg_env
,
151 offsetof(CPUARMState
, btype
));
154 static void set_btype(DisasContext
*s
, int val
)
156 /* BTYPE is a 2-bit field, and 0 should be done with reset_btype. */
157 tcg_debug_assert(val
>= 1 && val
<= 3);
162 static void reset_btype(DisasContext
*s
)
170 static void gen_pc_plus_diff(DisasContext
*s
, TCGv_i64 dest
, target_long diff
)
172 assert(s
->pc_save
!= -1);
173 if (tb_cflags(s
->base
.tb
) & CF_PCREL
) {
174 tcg_gen_addi_i64(dest
, cpu_pc
, (s
->pc_curr
- s
->pc_save
) + diff
);
176 tcg_gen_movi_i64(dest
, s
->pc_curr
+ diff
);
180 void gen_a64_update_pc(DisasContext
*s
, target_long diff
)
182 gen_pc_plus_diff(s
, cpu_pc
, diff
);
183 s
->pc_save
= s
->pc_curr
+ diff
;
187 * Handle Top Byte Ignore (TBI) bits.
189 * If address tagging is enabled via the TCR TBI bits:
190 * + for EL2 and EL3 there is only one TBI bit, and if it is set
191 * then the address is zero-extended, clearing bits [63:56]
192 * + for EL0 and EL1, TBI0 controls addresses with bit 55 == 0
193 * and TBI1 controls addresses with bit 55 == 1.
194 * If the appropriate TBI bit is set for the address then
195 * the address is sign-extended from bit 55 into bits [63:56]
197 * Here We have concatenated TBI{1,0} into tbi.
199 static void gen_top_byte_ignore(DisasContext
*s
, TCGv_i64 dst
,
200 TCGv_i64 src
, int tbi
)
203 /* Load unmodified address */
204 tcg_gen_mov_i64(dst
, src
);
205 } else if (!regime_has_2_ranges(s
->mmu_idx
)) {
206 /* Force tag byte to all zero */
207 tcg_gen_extract_i64(dst
, src
, 0, 56);
209 /* Sign-extend from bit 55. */
210 tcg_gen_sextract_i64(dst
, src
, 0, 56);
214 /* tbi0 but !tbi1: only use the extension if positive */
215 tcg_gen_and_i64(dst
, dst
, src
);
218 /* !tbi0 but tbi1: only use the extension if negative */
219 tcg_gen_or_i64(dst
, dst
, src
);
222 /* tbi0 and tbi1: always use the extension */
225 g_assert_not_reached();
230 static void gen_a64_set_pc(DisasContext
*s
, TCGv_i64 src
)
233 * If address tagging is enabled for instructions via the TCR TBI bits,
234 * then loading an address into the PC will clear out any tag.
236 gen_top_byte_ignore(s
, cpu_pc
, src
, s
->tbii
);
241 * Handle MTE and/or TBI.
243 * For TBI, ideally, we would do nothing. Proper behaviour on fault is
244 * for the tag to be present in the FAR_ELx register. But for user-only
245 * mode we do not have a TLB with which to implement this, so we must
246 * remove the top byte now.
248 * Always return a fresh temporary that we can increment independently
249 * of the write-back address.
252 TCGv_i64
clean_data_tbi(DisasContext
*s
, TCGv_i64 addr
)
254 TCGv_i64 clean
= tcg_temp_new_i64();
255 #ifdef CONFIG_USER_ONLY
256 gen_top_byte_ignore(s
, clean
, addr
, s
->tbid
);
258 tcg_gen_mov_i64(clean
, addr
);
263 /* Insert a zero tag into src, with the result at dst. */
264 static void gen_address_with_allocation_tag0(TCGv_i64 dst
, TCGv_i64 src
)
266 tcg_gen_andi_i64(dst
, src
, ~MAKE_64BIT_MASK(56, 4));
269 static void gen_probe_access(DisasContext
*s
, TCGv_i64 ptr
,
270 MMUAccessType acc
, int log2_size
)
272 gen_helper_probe_access(tcg_env
, ptr
,
273 tcg_constant_i32(acc
),
274 tcg_constant_i32(get_mem_index(s
)),
275 tcg_constant_i32(1 << log2_size
));
279 * For MTE, check a single logical or atomic access. This probes a single
280 * address, the exact one specified. The size and alignment of the access
281 * is not relevant to MTE, per se, but watchpoints do require the size,
282 * and we want to recognize those before making any other changes to state.
284 static TCGv_i64
gen_mte_check1_mmuidx(DisasContext
*s
, TCGv_i64 addr
,
285 bool is_write
, bool tag_checked
,
286 MemOp memop
, bool is_unpriv
,
289 if (tag_checked
&& s
->mte_active
[is_unpriv
]) {
293 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, core_idx
);
294 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
295 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
296 desc
= FIELD_DP32(desc
, MTEDESC
, WRITE
, is_write
);
297 desc
= FIELD_DP32(desc
, MTEDESC
, ALIGN
, get_alignment_bits(memop
));
298 desc
= FIELD_DP32(desc
, MTEDESC
, SIZEM1
, memop_size(memop
) - 1);
300 ret
= tcg_temp_new_i64();
301 gen_helper_mte_check(ret
, tcg_env
, tcg_constant_i32(desc
), addr
);
305 return clean_data_tbi(s
, addr
);
308 TCGv_i64
gen_mte_check1(DisasContext
*s
, TCGv_i64 addr
, bool is_write
,
309 bool tag_checked
, MemOp memop
)
311 return gen_mte_check1_mmuidx(s
, addr
, is_write
, tag_checked
, memop
,
312 false, get_mem_index(s
));
316 * For MTE, check multiple logical sequential accesses.
318 TCGv_i64
gen_mte_checkN(DisasContext
*s
, TCGv_i64 addr
, bool is_write
,
319 bool tag_checked
, int total_size
, MemOp single_mop
)
321 if (tag_checked
&& s
->mte_active
[0]) {
325 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, get_mem_index(s
));
326 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
327 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
328 desc
= FIELD_DP32(desc
, MTEDESC
, WRITE
, is_write
);
329 desc
= FIELD_DP32(desc
, MTEDESC
, ALIGN
, get_alignment_bits(single_mop
));
330 desc
= FIELD_DP32(desc
, MTEDESC
, SIZEM1
, total_size
- 1);
332 ret
= tcg_temp_new_i64();
333 gen_helper_mte_check(ret
, tcg_env
, tcg_constant_i32(desc
), addr
);
337 return clean_data_tbi(s
, addr
);
341 * Generate the special alignment check that applies to AccType_ATOMIC
342 * and AccType_ORDERED insns under FEAT_LSE2: the access need not be
343 * naturally aligned, but it must not cross a 16-byte boundary.
344 * See AArch64.CheckAlignment().
346 static void check_lse2_align(DisasContext
*s
, int rn
, int imm
,
347 bool is_write
, MemOp mop
)
351 TCGLabel
*over_label
;
355 tmp
= tcg_temp_new_i32();
356 tcg_gen_extrl_i64_i32(tmp
, cpu_reg_sp(s
, rn
));
357 tcg_gen_addi_i32(tmp
, tmp
, imm
& 15);
358 tcg_gen_andi_i32(tmp
, tmp
, 15);
359 tcg_gen_addi_i32(tmp
, tmp
, memop_size(mop
));
361 over_label
= gen_new_label();
362 tcg_gen_brcondi_i32(TCG_COND_LEU
, tmp
, 16, over_label
);
364 addr
= tcg_temp_new_i64();
365 tcg_gen_addi_i64(addr
, cpu_reg_sp(s
, rn
), imm
);
367 type
= is_write
? MMU_DATA_STORE
: MMU_DATA_LOAD
,
368 mmu_idx
= get_mem_index(s
);
369 gen_helper_unaligned_access(tcg_env
, addr
, tcg_constant_i32(type
),
370 tcg_constant_i32(mmu_idx
));
372 gen_set_label(over_label
);
376 /* Handle the alignment check for AccType_ATOMIC instructions. */
377 static MemOp
check_atomic_align(DisasContext
*s
, int rn
, MemOp mop
)
379 MemOp size
= mop
& MO_SIZE
;
386 * If size == MO_128, this is a LDXP, and the operation is single-copy
387 * atomic for each doubleword, not the entire quadword; it still must
388 * be quadword aligned.
390 if (size
== MO_128
) {
391 return finalize_memop_atom(s
, MO_128
| MO_ALIGN
,
392 MO_ATOM_IFALIGN_PAIR
);
394 if (dc_isar_feature(aa64_lse2
, s
)) {
395 check_lse2_align(s
, rn
, 0, true, mop
);
399 return finalize_memop(s
, mop
);
402 /* Handle the alignment check for AccType_ORDERED instructions. */
403 static MemOp
check_ordered_align(DisasContext
*s
, int rn
, int imm
,
404 bool is_write
, MemOp mop
)
406 MemOp size
= mop
& MO_SIZE
;
411 if (size
== MO_128
) {
412 return finalize_memop_atom(s
, MO_128
| MO_ALIGN
,
413 MO_ATOM_IFALIGN_PAIR
);
415 if (!dc_isar_feature(aa64_lse2
, s
)) {
417 } else if (!s
->naa
) {
418 check_lse2_align(s
, rn
, imm
, is_write
, mop
);
420 return finalize_memop(s
, mop
);
423 typedef struct DisasCompare64
{
428 static void a64_test_cc(DisasCompare64
*c64
, int cc
)
432 arm_test_cc(&c32
, cc
);
435 * Sign-extend the 32-bit value so that the GE/LT comparisons work
436 * properly. The NE/EQ comparisons are also fine with this choice.
438 c64
->cond
= c32
.cond
;
439 c64
->value
= tcg_temp_new_i64();
440 tcg_gen_ext_i32_i64(c64
->value
, c32
.value
);
443 static void gen_rebuild_hflags(DisasContext
*s
)
445 gen_helper_rebuild_hflags_a64(tcg_env
, tcg_constant_i32(s
->current_el
));
448 static void gen_exception_internal(int excp
)
450 assert(excp_is_internal(excp
));
451 gen_helper_exception_internal(tcg_env
, tcg_constant_i32(excp
));
454 static void gen_exception_internal_insn(DisasContext
*s
, int excp
)
456 gen_a64_update_pc(s
, 0);
457 gen_exception_internal(excp
);
458 s
->base
.is_jmp
= DISAS_NORETURN
;
461 static void gen_exception_bkpt_insn(DisasContext
*s
, uint32_t syndrome
)
463 gen_a64_update_pc(s
, 0);
464 gen_helper_exception_bkpt_insn(tcg_env
, tcg_constant_i32(syndrome
));
465 s
->base
.is_jmp
= DISAS_NORETURN
;
468 static void gen_step_complete_exception(DisasContext
*s
)
470 /* We just completed step of an insn. Move from Active-not-pending
471 * to Active-pending, and then also take the swstep exception.
472 * This corresponds to making the (IMPDEF) choice to prioritize
473 * swstep exceptions over asynchronous exceptions taken to an exception
474 * level where debug is disabled. This choice has the advantage that
475 * we do not need to maintain internal state corresponding to the
476 * ISV/EX syndrome bits between completion of the step and generation
477 * of the exception, and our syndrome information is always correct.
480 gen_swstep_exception(s
, 1, s
->is_ldex
);
481 s
->base
.is_jmp
= DISAS_NORETURN
;
484 static inline bool use_goto_tb(DisasContext
*s
, uint64_t dest
)
489 return translator_use_goto_tb(&s
->base
, dest
);
492 static void gen_goto_tb(DisasContext
*s
, int n
, int64_t diff
)
494 if (use_goto_tb(s
, s
->pc_curr
+ diff
)) {
496 * For pcrel, the pc must always be up-to-date on entry to
497 * the linked TB, so that it can use simple additions for all
498 * further adjustments. For !pcrel, the linked TB is compiled
499 * to know its full virtual address, so we can delay the
500 * update to pc to the unlinked path. A long chain of links
501 * can thus avoid many updates to the PC.
503 if (tb_cflags(s
->base
.tb
) & CF_PCREL
) {
504 gen_a64_update_pc(s
, diff
);
508 gen_a64_update_pc(s
, diff
);
510 tcg_gen_exit_tb(s
->base
.tb
, n
);
511 s
->base
.is_jmp
= DISAS_NORETURN
;
513 gen_a64_update_pc(s
, diff
);
515 gen_step_complete_exception(s
);
517 tcg_gen_lookup_and_goto_ptr();
518 s
->base
.is_jmp
= DISAS_NORETURN
;
524 * Register access functions
526 * These functions are used for directly accessing a register in where
527 * changes to the final register value are likely to be made. If you
528 * need to use a register for temporary calculation (e.g. index type
529 * operations) use the read_* form.
531 * B1.2.1 Register mappings
533 * In instruction register encoding 31 can refer to ZR (zero register) or
534 * the SP (stack pointer) depending on context. In QEMU's case we map SP
535 * to cpu_X[31] and ZR accesses to a temporary which can be discarded.
536 * This is the point of the _sp forms.
538 TCGv_i64
cpu_reg(DisasContext
*s
, int reg
)
541 TCGv_i64 t
= tcg_temp_new_i64();
542 tcg_gen_movi_i64(t
, 0);
549 /* register access for when 31 == SP */
550 TCGv_i64
cpu_reg_sp(DisasContext
*s
, int reg
)
555 /* read a cpu register in 32bit/64bit mode. Returns a TCGv_i64
556 * representing the register contents. This TCGv is an auto-freed
557 * temporary so it need not be explicitly freed, and may be modified.
559 TCGv_i64
read_cpu_reg(DisasContext
*s
, int reg
, int sf
)
561 TCGv_i64 v
= tcg_temp_new_i64();
564 tcg_gen_mov_i64(v
, cpu_X
[reg
]);
566 tcg_gen_ext32u_i64(v
, cpu_X
[reg
]);
569 tcg_gen_movi_i64(v
, 0);
574 TCGv_i64
read_cpu_reg_sp(DisasContext
*s
, int reg
, int sf
)
576 TCGv_i64 v
= tcg_temp_new_i64();
578 tcg_gen_mov_i64(v
, cpu_X
[reg
]);
580 tcg_gen_ext32u_i64(v
, cpu_X
[reg
]);
585 /* Return the offset into CPUARMState of a slice (from
586 * the least significant end) of FP register Qn (ie
588 * (Note that this is not the same mapping as for A32; see cpu.h)
590 static inline int fp_reg_offset(DisasContext
*s
, int regno
, MemOp size
)
592 return vec_reg_offset(s
, regno
, 0, size
);
595 /* Offset of the high half of the 128 bit vector Qn */
596 static inline int fp_reg_hi_offset(DisasContext
*s
, int regno
)
598 return vec_reg_offset(s
, regno
, 1, MO_64
);
601 /* Convenience accessors for reading and writing single and double
602 * FP registers. Writing clears the upper parts of the associated
603 * 128 bit vector register, as required by the architecture.
604 * Note that unlike the GP register accessors, the values returned
605 * by the read functions must be manually freed.
607 static TCGv_i64
read_fp_dreg(DisasContext
*s
, int reg
)
609 TCGv_i64 v
= tcg_temp_new_i64();
611 tcg_gen_ld_i64(v
, tcg_env
, fp_reg_offset(s
, reg
, MO_64
));
615 static TCGv_i32
read_fp_sreg(DisasContext
*s
, int reg
)
617 TCGv_i32 v
= tcg_temp_new_i32();
619 tcg_gen_ld_i32(v
, tcg_env
, fp_reg_offset(s
, reg
, MO_32
));
623 static TCGv_i32
read_fp_hreg(DisasContext
*s
, int reg
)
625 TCGv_i32 v
= tcg_temp_new_i32();
627 tcg_gen_ld16u_i32(v
, tcg_env
, fp_reg_offset(s
, reg
, MO_16
));
631 /* Clear the bits above an N-bit vector, for N = (is_q ? 128 : 64).
632 * If SVE is not enabled, then there are only 128 bits in the vector.
634 static void clear_vec_high(DisasContext
*s
, bool is_q
, int rd
)
636 unsigned ofs
= fp_reg_offset(s
, rd
, MO_64
);
637 unsigned vsz
= vec_full_reg_size(s
);
639 /* Nop move, with side effect of clearing the tail. */
640 tcg_gen_gvec_mov(MO_64
, ofs
, ofs
, is_q
? 16 : 8, vsz
);
643 void write_fp_dreg(DisasContext
*s
, int reg
, TCGv_i64 v
)
645 unsigned ofs
= fp_reg_offset(s
, reg
, MO_64
);
647 tcg_gen_st_i64(v
, tcg_env
, ofs
);
648 clear_vec_high(s
, false, reg
);
651 static void write_fp_sreg(DisasContext
*s
, int reg
, TCGv_i32 v
)
653 TCGv_i64 tmp
= tcg_temp_new_i64();
655 tcg_gen_extu_i32_i64(tmp
, v
);
656 write_fp_dreg(s
, reg
, tmp
);
659 /* Expand a 2-operand AdvSIMD vector operation using an expander function. */
660 static void gen_gvec_fn2(DisasContext
*s
, bool is_q
, int rd
, int rn
,
661 GVecGen2Fn
*gvec_fn
, int vece
)
663 gvec_fn(vece
, vec_full_reg_offset(s
, rd
), vec_full_reg_offset(s
, rn
),
664 is_q
? 16 : 8, vec_full_reg_size(s
));
667 /* Expand a 2-operand + immediate AdvSIMD vector operation using
668 * an expander function.
670 static void gen_gvec_fn2i(DisasContext
*s
, bool is_q
, int rd
, int rn
,
671 int64_t imm
, GVecGen2iFn
*gvec_fn
, int vece
)
673 gvec_fn(vece
, vec_full_reg_offset(s
, rd
), vec_full_reg_offset(s
, rn
),
674 imm
, is_q
? 16 : 8, vec_full_reg_size(s
));
677 /* Expand a 3-operand AdvSIMD vector operation using an expander function. */
678 static void gen_gvec_fn3(DisasContext
*s
, bool is_q
, int rd
, int rn
, int rm
,
679 GVecGen3Fn
*gvec_fn
, int vece
)
681 gvec_fn(vece
, vec_full_reg_offset(s
, rd
), vec_full_reg_offset(s
, rn
),
682 vec_full_reg_offset(s
, rm
), is_q
? 16 : 8, vec_full_reg_size(s
));
685 /* Expand a 4-operand AdvSIMD vector operation using an expander function. */
686 static void gen_gvec_fn4(DisasContext
*s
, bool is_q
, int rd
, int rn
, int rm
,
687 int rx
, GVecGen4Fn
*gvec_fn
, int vece
)
689 gvec_fn(vece
, vec_full_reg_offset(s
, rd
), vec_full_reg_offset(s
, rn
),
690 vec_full_reg_offset(s
, rm
), vec_full_reg_offset(s
, rx
),
691 is_q
? 16 : 8, vec_full_reg_size(s
));
694 /* Expand a 2-operand operation using an out-of-line helper. */
695 static void gen_gvec_op2_ool(DisasContext
*s
, bool is_q
, int rd
,
696 int rn
, int data
, gen_helper_gvec_2
*fn
)
698 tcg_gen_gvec_2_ool(vec_full_reg_offset(s
, rd
),
699 vec_full_reg_offset(s
, rn
),
700 is_q
? 16 : 8, vec_full_reg_size(s
), data
, fn
);
703 /* Expand a 3-operand operation using an out-of-line helper. */
704 static void gen_gvec_op3_ool(DisasContext
*s
, bool is_q
, int rd
,
705 int rn
, int rm
, int data
, gen_helper_gvec_3
*fn
)
707 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
708 vec_full_reg_offset(s
, rn
),
709 vec_full_reg_offset(s
, rm
),
710 is_q
? 16 : 8, vec_full_reg_size(s
), data
, fn
);
713 /* Expand a 3-operand + fpstatus pointer + simd data value operation using
714 * an out-of-line helper.
716 static void gen_gvec_op3_fpst(DisasContext
*s
, bool is_q
, int rd
, int rn
,
717 int rm
, bool is_fp16
, int data
,
718 gen_helper_gvec_3_ptr
*fn
)
720 TCGv_ptr fpst
= fpstatus_ptr(is_fp16
? FPST_FPCR_F16
: FPST_FPCR
);
721 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, rd
),
722 vec_full_reg_offset(s
, rn
),
723 vec_full_reg_offset(s
, rm
), fpst
,
724 is_q
? 16 : 8, vec_full_reg_size(s
), data
, fn
);
727 /* Expand a 3-operand + qc + operation using an out-of-line helper. */
728 static void gen_gvec_op3_qc(DisasContext
*s
, bool is_q
, int rd
, int rn
,
729 int rm
, gen_helper_gvec_3_ptr
*fn
)
731 TCGv_ptr qc_ptr
= tcg_temp_new_ptr();
733 tcg_gen_addi_ptr(qc_ptr
, tcg_env
, offsetof(CPUARMState
, vfp
.qc
));
734 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, rd
),
735 vec_full_reg_offset(s
, rn
),
736 vec_full_reg_offset(s
, rm
), qc_ptr
,
737 is_q
? 16 : 8, vec_full_reg_size(s
), 0, fn
);
740 /* Expand a 4-operand operation using an out-of-line helper. */
741 static void gen_gvec_op4_ool(DisasContext
*s
, bool is_q
, int rd
, int rn
,
742 int rm
, int ra
, int data
, gen_helper_gvec_4
*fn
)
744 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
745 vec_full_reg_offset(s
, rn
),
746 vec_full_reg_offset(s
, rm
),
747 vec_full_reg_offset(s
, ra
),
748 is_q
? 16 : 8, vec_full_reg_size(s
), data
, fn
);
752 * Expand a 4-operand + fpstatus pointer + simd data value operation using
753 * an out-of-line helper.
755 static void gen_gvec_op4_fpst(DisasContext
*s
, bool is_q
, int rd
, int rn
,
756 int rm
, int ra
, bool is_fp16
, int data
,
757 gen_helper_gvec_4_ptr
*fn
)
759 TCGv_ptr fpst
= fpstatus_ptr(is_fp16
? FPST_FPCR_F16
: FPST_FPCR
);
760 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, rd
),
761 vec_full_reg_offset(s
, rn
),
762 vec_full_reg_offset(s
, rm
),
763 vec_full_reg_offset(s
, ra
), fpst
,
764 is_q
? 16 : 8, vec_full_reg_size(s
), data
, fn
);
767 /* Set ZF and NF based on a 64 bit result. This is alas fiddlier
768 * than the 32 bit equivalent.
770 static inline void gen_set_NZ64(TCGv_i64 result
)
772 tcg_gen_extr_i64_i32(cpu_ZF
, cpu_NF
, result
);
773 tcg_gen_or_i32(cpu_ZF
, cpu_ZF
, cpu_NF
);
776 /* Set NZCV as for a logical operation: NZ as per result, CV cleared. */
777 static inline void gen_logic_CC(int sf
, TCGv_i64 result
)
780 gen_set_NZ64(result
);
782 tcg_gen_extrl_i64_i32(cpu_ZF
, result
);
783 tcg_gen_mov_i32(cpu_NF
, cpu_ZF
);
785 tcg_gen_movi_i32(cpu_CF
, 0);
786 tcg_gen_movi_i32(cpu_VF
, 0);
789 /* dest = T0 + T1; compute C, N, V and Z flags */
790 static void gen_add64_CC(TCGv_i64 dest
, TCGv_i64 t0
, TCGv_i64 t1
)
792 TCGv_i64 result
, flag
, tmp
;
793 result
= tcg_temp_new_i64();
794 flag
= tcg_temp_new_i64();
795 tmp
= tcg_temp_new_i64();
797 tcg_gen_movi_i64(tmp
, 0);
798 tcg_gen_add2_i64(result
, flag
, t0
, tmp
, t1
, tmp
);
800 tcg_gen_extrl_i64_i32(cpu_CF
, flag
);
802 gen_set_NZ64(result
);
804 tcg_gen_xor_i64(flag
, result
, t0
);
805 tcg_gen_xor_i64(tmp
, t0
, t1
);
806 tcg_gen_andc_i64(flag
, flag
, tmp
);
807 tcg_gen_extrh_i64_i32(cpu_VF
, flag
);
809 tcg_gen_mov_i64(dest
, result
);
812 static void gen_add32_CC(TCGv_i64 dest
, TCGv_i64 t0
, TCGv_i64 t1
)
814 TCGv_i32 t0_32
= tcg_temp_new_i32();
815 TCGv_i32 t1_32
= tcg_temp_new_i32();
816 TCGv_i32 tmp
= tcg_temp_new_i32();
818 tcg_gen_movi_i32(tmp
, 0);
819 tcg_gen_extrl_i64_i32(t0_32
, t0
);
820 tcg_gen_extrl_i64_i32(t1_32
, t1
);
821 tcg_gen_add2_i32(cpu_NF
, cpu_CF
, t0_32
, tmp
, t1_32
, tmp
);
822 tcg_gen_mov_i32(cpu_ZF
, cpu_NF
);
823 tcg_gen_xor_i32(cpu_VF
, cpu_NF
, t0_32
);
824 tcg_gen_xor_i32(tmp
, t0_32
, t1_32
);
825 tcg_gen_andc_i32(cpu_VF
, cpu_VF
, tmp
);
826 tcg_gen_extu_i32_i64(dest
, cpu_NF
);
829 static void gen_add_CC(int sf
, TCGv_i64 dest
, TCGv_i64 t0
, TCGv_i64 t1
)
832 gen_add64_CC(dest
, t0
, t1
);
834 gen_add32_CC(dest
, t0
, t1
);
838 /* dest = T0 - T1; compute C, N, V and Z flags */
839 static void gen_sub64_CC(TCGv_i64 dest
, TCGv_i64 t0
, TCGv_i64 t1
)
841 /* 64 bit arithmetic */
842 TCGv_i64 result
, flag
, tmp
;
844 result
= tcg_temp_new_i64();
845 flag
= tcg_temp_new_i64();
846 tcg_gen_sub_i64(result
, t0
, t1
);
848 gen_set_NZ64(result
);
850 tcg_gen_setcond_i64(TCG_COND_GEU
, flag
, t0
, t1
);
851 tcg_gen_extrl_i64_i32(cpu_CF
, flag
);
853 tcg_gen_xor_i64(flag
, result
, t0
);
854 tmp
= tcg_temp_new_i64();
855 tcg_gen_xor_i64(tmp
, t0
, t1
);
856 tcg_gen_and_i64(flag
, flag
, tmp
);
857 tcg_gen_extrh_i64_i32(cpu_VF
, flag
);
858 tcg_gen_mov_i64(dest
, result
);
861 static void gen_sub32_CC(TCGv_i64 dest
, TCGv_i64 t0
, TCGv_i64 t1
)
863 /* 32 bit arithmetic */
864 TCGv_i32 t0_32
= tcg_temp_new_i32();
865 TCGv_i32 t1_32
= tcg_temp_new_i32();
868 tcg_gen_extrl_i64_i32(t0_32
, t0
);
869 tcg_gen_extrl_i64_i32(t1_32
, t1
);
870 tcg_gen_sub_i32(cpu_NF
, t0_32
, t1_32
);
871 tcg_gen_mov_i32(cpu_ZF
, cpu_NF
);
872 tcg_gen_setcond_i32(TCG_COND_GEU
, cpu_CF
, t0_32
, t1_32
);
873 tcg_gen_xor_i32(cpu_VF
, cpu_NF
, t0_32
);
874 tmp
= tcg_temp_new_i32();
875 tcg_gen_xor_i32(tmp
, t0_32
, t1_32
);
876 tcg_gen_and_i32(cpu_VF
, cpu_VF
, tmp
);
877 tcg_gen_extu_i32_i64(dest
, cpu_NF
);
880 static void gen_sub_CC(int sf
, TCGv_i64 dest
, TCGv_i64 t0
, TCGv_i64 t1
)
883 gen_sub64_CC(dest
, t0
, t1
);
885 gen_sub32_CC(dest
, t0
, t1
);
889 /* dest = T0 + T1 + CF; do not compute flags. */
890 static void gen_adc(int sf
, TCGv_i64 dest
, TCGv_i64 t0
, TCGv_i64 t1
)
892 TCGv_i64 flag
= tcg_temp_new_i64();
893 tcg_gen_extu_i32_i64(flag
, cpu_CF
);
894 tcg_gen_add_i64(dest
, t0
, t1
);
895 tcg_gen_add_i64(dest
, dest
, flag
);
898 tcg_gen_ext32u_i64(dest
, dest
);
902 /* dest = T0 + T1 + CF; compute C, N, V and Z flags. */
903 static void gen_adc_CC(int sf
, TCGv_i64 dest
, TCGv_i64 t0
, TCGv_i64 t1
)
906 TCGv_i64 result
= tcg_temp_new_i64();
907 TCGv_i64 cf_64
= tcg_temp_new_i64();
908 TCGv_i64 vf_64
= tcg_temp_new_i64();
909 TCGv_i64 tmp
= tcg_temp_new_i64();
910 TCGv_i64 zero
= tcg_constant_i64(0);
912 tcg_gen_extu_i32_i64(cf_64
, cpu_CF
);
913 tcg_gen_add2_i64(result
, cf_64
, t0
, zero
, cf_64
, zero
);
914 tcg_gen_add2_i64(result
, cf_64
, result
, cf_64
, t1
, zero
);
915 tcg_gen_extrl_i64_i32(cpu_CF
, cf_64
);
916 gen_set_NZ64(result
);
918 tcg_gen_xor_i64(vf_64
, result
, t0
);
919 tcg_gen_xor_i64(tmp
, t0
, t1
);
920 tcg_gen_andc_i64(vf_64
, vf_64
, tmp
);
921 tcg_gen_extrh_i64_i32(cpu_VF
, vf_64
);
923 tcg_gen_mov_i64(dest
, result
);
925 TCGv_i32 t0_32
= tcg_temp_new_i32();
926 TCGv_i32 t1_32
= tcg_temp_new_i32();
927 TCGv_i32 tmp
= tcg_temp_new_i32();
928 TCGv_i32 zero
= tcg_constant_i32(0);
930 tcg_gen_extrl_i64_i32(t0_32
, t0
);
931 tcg_gen_extrl_i64_i32(t1_32
, t1
);
932 tcg_gen_add2_i32(cpu_NF
, cpu_CF
, t0_32
, zero
, cpu_CF
, zero
);
933 tcg_gen_add2_i32(cpu_NF
, cpu_CF
, cpu_NF
, cpu_CF
, t1_32
, zero
);
935 tcg_gen_mov_i32(cpu_ZF
, cpu_NF
);
936 tcg_gen_xor_i32(cpu_VF
, cpu_NF
, t0_32
);
937 tcg_gen_xor_i32(tmp
, t0_32
, t1_32
);
938 tcg_gen_andc_i32(cpu_VF
, cpu_VF
, tmp
);
939 tcg_gen_extu_i32_i64(dest
, cpu_NF
);
944 * Load/Store generators
948 * Store from GPR register to memory.
950 static void do_gpr_st_memidx(DisasContext
*s
, TCGv_i64 source
,
951 TCGv_i64 tcg_addr
, MemOp memop
, int memidx
,
953 unsigned int iss_srt
,
954 bool iss_sf
, bool iss_ar
)
956 tcg_gen_qemu_st_i64(source
, tcg_addr
, memidx
, memop
);
961 syn
= syn_data_abort_with_iss(0,
967 0, 0, 0, 0, 0, false);
968 disas_set_insn_syndrome(s
, syn
);
972 static void do_gpr_st(DisasContext
*s
, TCGv_i64 source
,
973 TCGv_i64 tcg_addr
, MemOp memop
,
975 unsigned int iss_srt
,
976 bool iss_sf
, bool iss_ar
)
978 do_gpr_st_memidx(s
, source
, tcg_addr
, memop
, get_mem_index(s
),
979 iss_valid
, iss_srt
, iss_sf
, iss_ar
);
983 * Load from memory to GPR register
985 static void do_gpr_ld_memidx(DisasContext
*s
, TCGv_i64 dest
, TCGv_i64 tcg_addr
,
986 MemOp memop
, bool extend
, int memidx
,
987 bool iss_valid
, unsigned int iss_srt
,
988 bool iss_sf
, bool iss_ar
)
990 tcg_gen_qemu_ld_i64(dest
, tcg_addr
, memidx
, memop
);
992 if (extend
&& (memop
& MO_SIGN
)) {
993 g_assert((memop
& MO_SIZE
) <= MO_32
);
994 tcg_gen_ext32u_i64(dest
, dest
);
1000 syn
= syn_data_abort_with_iss(0,
1002 (memop
& MO_SIGN
) != 0,
1006 0, 0, 0, 0, 0, false);
1007 disas_set_insn_syndrome(s
, syn
);
1011 static void do_gpr_ld(DisasContext
*s
, TCGv_i64 dest
, TCGv_i64 tcg_addr
,
1012 MemOp memop
, bool extend
,
1013 bool iss_valid
, unsigned int iss_srt
,
1014 bool iss_sf
, bool iss_ar
)
1016 do_gpr_ld_memidx(s
, dest
, tcg_addr
, memop
, extend
, get_mem_index(s
),
1017 iss_valid
, iss_srt
, iss_sf
, iss_ar
);
1021 * Store from FP register to memory
1023 static void do_fp_st(DisasContext
*s
, int srcidx
, TCGv_i64 tcg_addr
, MemOp mop
)
1025 /* This writes the bottom N bits of a 128 bit wide vector to memory */
1026 TCGv_i64 tmplo
= tcg_temp_new_i64();
1028 tcg_gen_ld_i64(tmplo
, tcg_env
, fp_reg_offset(s
, srcidx
, MO_64
));
1030 if ((mop
& MO_SIZE
) < MO_128
) {
1031 tcg_gen_qemu_st_i64(tmplo
, tcg_addr
, get_mem_index(s
), mop
);
1033 TCGv_i64 tmphi
= tcg_temp_new_i64();
1034 TCGv_i128 t16
= tcg_temp_new_i128();
1036 tcg_gen_ld_i64(tmphi
, tcg_env
, fp_reg_hi_offset(s
, srcidx
));
1037 tcg_gen_concat_i64_i128(t16
, tmplo
, tmphi
);
1039 tcg_gen_qemu_st_i128(t16
, tcg_addr
, get_mem_index(s
), mop
);
1044 * Load from memory to FP register
1046 static void do_fp_ld(DisasContext
*s
, int destidx
, TCGv_i64 tcg_addr
, MemOp mop
)
1048 /* This always zero-extends and writes to a full 128 bit wide vector */
1049 TCGv_i64 tmplo
= tcg_temp_new_i64();
1050 TCGv_i64 tmphi
= NULL
;
1052 if ((mop
& MO_SIZE
) < MO_128
) {
1053 tcg_gen_qemu_ld_i64(tmplo
, tcg_addr
, get_mem_index(s
), mop
);
1055 TCGv_i128 t16
= tcg_temp_new_i128();
1057 tcg_gen_qemu_ld_i128(t16
, tcg_addr
, get_mem_index(s
), mop
);
1059 tmphi
= tcg_temp_new_i64();
1060 tcg_gen_extr_i128_i64(tmplo
, tmphi
, t16
);
1063 tcg_gen_st_i64(tmplo
, tcg_env
, fp_reg_offset(s
, destidx
, MO_64
));
1066 tcg_gen_st_i64(tmphi
, tcg_env
, fp_reg_hi_offset(s
, destidx
));
1068 clear_vec_high(s
, tmphi
!= NULL
, destidx
);
1072 * Vector load/store helpers.
1074 * The principal difference between this and a FP load is that we don't
1075 * zero extend as we are filling a partial chunk of the vector register.
1076 * These functions don't support 128 bit loads/stores, which would be
1077 * normal load/store operations.
1079 * The _i32 versions are useful when operating on 32 bit quantities
1080 * (eg for floating point single or using Neon helper functions).
1083 /* Get value of an element within a vector register */
1084 static void read_vec_element(DisasContext
*s
, TCGv_i64 tcg_dest
, int srcidx
,
1085 int element
, MemOp memop
)
1087 int vect_off
= vec_reg_offset(s
, srcidx
, element
, memop
& MO_SIZE
);
1088 switch ((unsigned)memop
) {
1090 tcg_gen_ld8u_i64(tcg_dest
, tcg_env
, vect_off
);
1093 tcg_gen_ld16u_i64(tcg_dest
, tcg_env
, vect_off
);
1096 tcg_gen_ld32u_i64(tcg_dest
, tcg_env
, vect_off
);
1099 tcg_gen_ld8s_i64(tcg_dest
, tcg_env
, vect_off
);
1102 tcg_gen_ld16s_i64(tcg_dest
, tcg_env
, vect_off
);
1105 tcg_gen_ld32s_i64(tcg_dest
, tcg_env
, vect_off
);
1109 tcg_gen_ld_i64(tcg_dest
, tcg_env
, vect_off
);
1112 g_assert_not_reached();
1116 static void read_vec_element_i32(DisasContext
*s
, TCGv_i32 tcg_dest
, int srcidx
,
1117 int element
, MemOp memop
)
1119 int vect_off
= vec_reg_offset(s
, srcidx
, element
, memop
& MO_SIZE
);
1122 tcg_gen_ld8u_i32(tcg_dest
, tcg_env
, vect_off
);
1125 tcg_gen_ld16u_i32(tcg_dest
, tcg_env
, vect_off
);
1128 tcg_gen_ld8s_i32(tcg_dest
, tcg_env
, vect_off
);
1131 tcg_gen_ld16s_i32(tcg_dest
, tcg_env
, vect_off
);
1135 tcg_gen_ld_i32(tcg_dest
, tcg_env
, vect_off
);
1138 g_assert_not_reached();
1142 /* Set value of an element within a vector register */
1143 static void write_vec_element(DisasContext
*s
, TCGv_i64 tcg_src
, int destidx
,
1144 int element
, MemOp memop
)
1146 int vect_off
= vec_reg_offset(s
, destidx
, element
, memop
& MO_SIZE
);
1149 tcg_gen_st8_i64(tcg_src
, tcg_env
, vect_off
);
1152 tcg_gen_st16_i64(tcg_src
, tcg_env
, vect_off
);
1155 tcg_gen_st32_i64(tcg_src
, tcg_env
, vect_off
);
1158 tcg_gen_st_i64(tcg_src
, tcg_env
, vect_off
);
1161 g_assert_not_reached();
1165 static void write_vec_element_i32(DisasContext
*s
, TCGv_i32 tcg_src
,
1166 int destidx
, int element
, MemOp memop
)
1168 int vect_off
= vec_reg_offset(s
, destidx
, element
, memop
& MO_SIZE
);
1171 tcg_gen_st8_i32(tcg_src
, tcg_env
, vect_off
);
1174 tcg_gen_st16_i32(tcg_src
, tcg_env
, vect_off
);
1177 tcg_gen_st_i32(tcg_src
, tcg_env
, vect_off
);
1180 g_assert_not_reached();
1184 /* Store from vector register to memory */
1185 static void do_vec_st(DisasContext
*s
, int srcidx
, int element
,
1186 TCGv_i64 tcg_addr
, MemOp mop
)
1188 TCGv_i64 tcg_tmp
= tcg_temp_new_i64();
1190 read_vec_element(s
, tcg_tmp
, srcidx
, element
, mop
& MO_SIZE
);
1191 tcg_gen_qemu_st_i64(tcg_tmp
, tcg_addr
, get_mem_index(s
), mop
);
1194 /* Load from memory to vector register */
1195 static void do_vec_ld(DisasContext
*s
, int destidx
, int element
,
1196 TCGv_i64 tcg_addr
, MemOp mop
)
1198 TCGv_i64 tcg_tmp
= tcg_temp_new_i64();
1200 tcg_gen_qemu_ld_i64(tcg_tmp
, tcg_addr
, get_mem_index(s
), mop
);
1201 write_vec_element(s
, tcg_tmp
, destidx
, element
, mop
& MO_SIZE
);
1204 /* Check that FP/Neon access is enabled. If it is, return
1205 * true. If not, emit code to generate an appropriate exception,
1206 * and return false; the caller should not emit any code for
1207 * the instruction. Note that this check must happen after all
1208 * unallocated-encoding checks (otherwise the syndrome information
1209 * for the resulting exception will be incorrect).
1211 static bool fp_access_check_only(DisasContext
*s
)
1213 if (s
->fp_excp_el
) {
1214 assert(!s
->fp_access_checked
);
1215 s
->fp_access_checked
= true;
1217 gen_exception_insn_el(s
, 0, EXCP_UDEF
,
1218 syn_fp_access_trap(1, 0xe, false, 0),
1222 s
->fp_access_checked
= true;
1226 static bool fp_access_check(DisasContext
*s
)
1228 if (!fp_access_check_only(s
)) {
1231 if (s
->sme_trap_nonstreaming
&& s
->is_nonstreaming
) {
1232 gen_exception_insn(s
, 0, EXCP_UDEF
,
1233 syn_smetrap(SME_ET_Streaming
, false));
1240 * Check that SVE access is enabled. If it is, return true.
1241 * If not, emit code to generate an appropriate exception and return false.
1242 * This function corresponds to CheckSVEEnabled().
1244 bool sve_access_check(DisasContext
*s
)
1246 if (s
->pstate_sm
|| !dc_isar_feature(aa64_sve
, s
)) {
1247 assert(dc_isar_feature(aa64_sme
, s
));
1248 if (!sme_sm_enabled_check(s
)) {
1251 } else if (s
->sve_excp_el
) {
1252 gen_exception_insn_el(s
, 0, EXCP_UDEF
,
1253 syn_sve_access_trap(), s
->sve_excp_el
);
1256 s
->sve_access_checked
= true;
1257 return fp_access_check(s
);
1260 /* Assert that we only raise one exception per instruction. */
1261 assert(!s
->sve_access_checked
);
1262 s
->sve_access_checked
= true;
1267 * Check that SME access is enabled, raise an exception if not.
1268 * Note that this function corresponds to CheckSMEAccess and is
1269 * only used directly for cpregs.
1271 static bool sme_access_check(DisasContext
*s
)
1273 if (s
->sme_excp_el
) {
1274 gen_exception_insn_el(s
, 0, EXCP_UDEF
,
1275 syn_smetrap(SME_ET_AccessTrap
, false),
1282 /* This function corresponds to CheckSMEEnabled. */
1283 bool sme_enabled_check(DisasContext
*s
)
1286 * Note that unlike sve_excp_el, we have not constrained sme_excp_el
1287 * to be zero when fp_excp_el has priority. This is because we need
1288 * sme_excp_el by itself for cpregs access checks.
1290 if (!s
->fp_excp_el
|| s
->sme_excp_el
< s
->fp_excp_el
) {
1291 s
->fp_access_checked
= true;
1292 return sme_access_check(s
);
1294 return fp_access_check_only(s
);
1297 /* Common subroutine for CheckSMEAnd*Enabled. */
1298 bool sme_enabled_check_with_svcr(DisasContext
*s
, unsigned req
)
1300 if (!sme_enabled_check(s
)) {
1303 if (FIELD_EX64(req
, SVCR
, SM
) && !s
->pstate_sm
) {
1304 gen_exception_insn(s
, 0, EXCP_UDEF
,
1305 syn_smetrap(SME_ET_NotStreaming
, false));
1308 if (FIELD_EX64(req
, SVCR
, ZA
) && !s
->pstate_za
) {
1309 gen_exception_insn(s
, 0, EXCP_UDEF
,
1310 syn_smetrap(SME_ET_InactiveZA
, false));
1317 * Expanders for AdvSIMD translation functions.
1320 static bool do_gvec_op2_ool(DisasContext
*s
, arg_qrr_e
*a
, int data
,
1321 gen_helper_gvec_2
*fn
)
1323 if (!a
->q
&& a
->esz
== MO_64
) {
1326 if (fp_access_check(s
)) {
1327 gen_gvec_op2_ool(s
, a
->q
, a
->rd
, a
->rn
, data
, fn
);
1332 static bool do_gvec_op3_ool(DisasContext
*s
, arg_qrrr_e
*a
, int data
,
1333 gen_helper_gvec_3
*fn
)
1335 if (!a
->q
&& a
->esz
== MO_64
) {
1338 if (fp_access_check(s
)) {
1339 gen_gvec_op3_ool(s
, a
->q
, a
->rd
, a
->rn
, a
->rm
, data
, fn
);
1344 static bool do_gvec_fn3(DisasContext
*s
, arg_qrrr_e
*a
, GVecGen3Fn
*fn
)
1346 if (!a
->q
&& a
->esz
== MO_64
) {
1349 if (fp_access_check(s
)) {
1350 gen_gvec_fn3(s
, a
->q
, a
->rd
, a
->rn
, a
->rm
, fn
, a
->esz
);
1355 static bool do_gvec_fn3_no64(DisasContext
*s
, arg_qrrr_e
*a
, GVecGen3Fn
*fn
)
1357 if (a
->esz
== MO_64
) {
1360 if (fp_access_check(s
)) {
1361 gen_gvec_fn3(s
, a
->q
, a
->rd
, a
->rn
, a
->rm
, fn
, a
->esz
);
1366 static bool do_gvec_fn4(DisasContext
*s
, arg_qrrrr_e
*a
, GVecGen4Fn
*fn
)
1368 if (!a
->q
&& a
->esz
== MO_64
) {
1371 if (fp_access_check(s
)) {
1372 gen_gvec_fn4(s
, a
->q
, a
->rd
, a
->rn
, a
->rm
, a
->ra
, fn
, a
->esz
);
1378 * This utility function is for doing register extension with an
1379 * optional shift. You will likely want to pass a temporary for the
1380 * destination register. See DecodeRegExtend() in the ARM ARM.
1382 static void ext_and_shift_reg(TCGv_i64 tcg_out
, TCGv_i64 tcg_in
,
1383 int option
, unsigned int shift
)
1385 int extsize
= extract32(option
, 0, 2);
1386 bool is_signed
= extract32(option
, 2, 1);
1388 tcg_gen_ext_i64(tcg_out
, tcg_in
, extsize
| (is_signed
? MO_SIGN
: 0));
1389 tcg_gen_shli_i64(tcg_out
, tcg_out
, shift
);
1392 static inline void gen_check_sp_alignment(DisasContext
*s
)
1394 /* The AArch64 architecture mandates that (if enabled via PSTATE
1395 * or SCTLR bits) there is a check that SP is 16-aligned on every
1396 * SP-relative load or store (with an exception generated if it is not).
1397 * In line with general QEMU practice regarding misaligned accesses,
1398 * we omit these checks for the sake of guest program performance.
1399 * This function is provided as a hook so we can more easily add these
1400 * checks in future (possibly as a "favour catching guest program bugs
1401 * over speed" user selectable option).
1406 * This provides a simple table based table lookup decoder. It is
1407 * intended to be used when the relevant bits for decode are too
1408 * awkwardly placed and switch/if based logic would be confusing and
1409 * deeply nested. Since it's a linear search through the table, tables
1410 * should be kept small.
1412 * It returns the first handler where insn & mask == pattern, or
1413 * NULL if there is no match.
1414 * The table is terminated by an empty mask (i.e. 0)
1416 static inline AArch64DecodeFn
*lookup_disas_fn(const AArch64DecodeTable
*table
,
1419 const AArch64DecodeTable
*tptr
= table
;
1421 while (tptr
->mask
) {
1422 if ((insn
& tptr
->mask
) == tptr
->pattern
) {
1423 return tptr
->disas_fn
;
1431 * The instruction disassembly implemented here matches
1432 * the instruction encoding classifications in chapter C4
1433 * of the ARM Architecture Reference Manual (DDI0487B_a);
1434 * classification names and decode diagrams here should generally
1435 * match up with those in the manual.
1438 static bool trans_B(DisasContext
*s
, arg_i
*a
)
1441 gen_goto_tb(s
, 0, a
->imm
);
1445 static bool trans_BL(DisasContext
*s
, arg_i
*a
)
1447 gen_pc_plus_diff(s
, cpu_reg(s
, 30), curr_insn_len(s
));
1449 gen_goto_tb(s
, 0, a
->imm
);
1454 static bool trans_CBZ(DisasContext
*s
, arg_cbz
*a
)
1459 tcg_cmp
= read_cpu_reg(s
, a
->rt
, a
->sf
);
1462 match
= gen_disas_label(s
);
1463 tcg_gen_brcondi_i64(a
->nz
? TCG_COND_NE
: TCG_COND_EQ
,
1464 tcg_cmp
, 0, match
.label
);
1465 gen_goto_tb(s
, 0, 4);
1466 set_disas_label(s
, match
);
1467 gen_goto_tb(s
, 1, a
->imm
);
1471 static bool trans_TBZ(DisasContext
*s
, arg_tbz
*a
)
1476 tcg_cmp
= tcg_temp_new_i64();
1477 tcg_gen_andi_i64(tcg_cmp
, cpu_reg(s
, a
->rt
), 1ULL << a
->bitpos
);
1481 match
= gen_disas_label(s
);
1482 tcg_gen_brcondi_i64(a
->nz
? TCG_COND_NE
: TCG_COND_EQ
,
1483 tcg_cmp
, 0, match
.label
);
1484 gen_goto_tb(s
, 0, 4);
1485 set_disas_label(s
, match
);
1486 gen_goto_tb(s
, 1, a
->imm
);
1490 static bool trans_B_cond(DisasContext
*s
, arg_B_cond
*a
)
1492 /* BC.cond is only present with FEAT_HBC */
1493 if (a
->c
&& !dc_isar_feature(aa64_hbc
, s
)) {
1497 if (a
->cond
< 0x0e) {
1498 /* genuinely conditional branches */
1499 DisasLabel match
= gen_disas_label(s
);
1500 arm_gen_test_cc(a
->cond
, match
.label
);
1501 gen_goto_tb(s
, 0, 4);
1502 set_disas_label(s
, match
);
1503 gen_goto_tb(s
, 1, a
->imm
);
1505 /* 0xe and 0xf are both "always" conditions */
1506 gen_goto_tb(s
, 0, a
->imm
);
1511 static void set_btype_for_br(DisasContext
*s
, int rn
)
1513 if (dc_isar_feature(aa64_bti
, s
)) {
1514 /* BR to {x16,x17} or !guard -> 1, else 3. */
1515 set_btype(s
, rn
== 16 || rn
== 17 || !s
->guarded_page
? 1 : 3);
1519 static void set_btype_for_blr(DisasContext
*s
)
1521 if (dc_isar_feature(aa64_bti
, s
)) {
1522 /* BLR sets BTYPE to 2, regardless of source guarded page. */
1527 static bool trans_BR(DisasContext
*s
, arg_r
*a
)
1529 gen_a64_set_pc(s
, cpu_reg(s
, a
->rn
));
1530 set_btype_for_br(s
, a
->rn
);
1531 s
->base
.is_jmp
= DISAS_JUMP
;
1535 static bool trans_BLR(DisasContext
*s
, arg_r
*a
)
1537 TCGv_i64 dst
= cpu_reg(s
, a
->rn
);
1538 TCGv_i64 lr
= cpu_reg(s
, 30);
1540 TCGv_i64 tmp
= tcg_temp_new_i64();
1541 tcg_gen_mov_i64(tmp
, dst
);
1544 gen_pc_plus_diff(s
, lr
, curr_insn_len(s
));
1545 gen_a64_set_pc(s
, dst
);
1546 set_btype_for_blr(s
);
1547 s
->base
.is_jmp
= DISAS_JUMP
;
1551 static bool trans_RET(DisasContext
*s
, arg_r
*a
)
1553 gen_a64_set_pc(s
, cpu_reg(s
, a
->rn
));
1554 s
->base
.is_jmp
= DISAS_JUMP
;
1558 static TCGv_i64
auth_branch_target(DisasContext
*s
, TCGv_i64 dst
,
1559 TCGv_i64 modifier
, bool use_key_a
)
1563 * Return the branch target for a BRAA/RETA/etc, which is either
1564 * just the destination dst, or that value with the pauth check
1565 * done and the code removed from the high bits.
1567 if (!s
->pauth_active
) {
1571 truedst
= tcg_temp_new_i64();
1573 gen_helper_autia_combined(truedst
, tcg_env
, dst
, modifier
);
1575 gen_helper_autib_combined(truedst
, tcg_env
, dst
, modifier
);
1580 static bool trans_BRAZ(DisasContext
*s
, arg_braz
*a
)
1584 if (!dc_isar_feature(aa64_pauth
, s
)) {
1588 dst
= auth_branch_target(s
, cpu_reg(s
, a
->rn
), tcg_constant_i64(0), !a
->m
);
1589 gen_a64_set_pc(s
, dst
);
1590 set_btype_for_br(s
, a
->rn
);
1591 s
->base
.is_jmp
= DISAS_JUMP
;
1595 static bool trans_BLRAZ(DisasContext
*s
, arg_braz
*a
)
1599 if (!dc_isar_feature(aa64_pauth
, s
)) {
1603 dst
= auth_branch_target(s
, cpu_reg(s
, a
->rn
), tcg_constant_i64(0), !a
->m
);
1604 lr
= cpu_reg(s
, 30);
1606 TCGv_i64 tmp
= tcg_temp_new_i64();
1607 tcg_gen_mov_i64(tmp
, dst
);
1610 gen_pc_plus_diff(s
, lr
, curr_insn_len(s
));
1611 gen_a64_set_pc(s
, dst
);
1612 set_btype_for_blr(s
);
1613 s
->base
.is_jmp
= DISAS_JUMP
;
1617 static bool trans_RETA(DisasContext
*s
, arg_reta
*a
)
1621 dst
= auth_branch_target(s
, cpu_reg(s
, 30), cpu_X
[31], !a
->m
);
1622 gen_a64_set_pc(s
, dst
);
1623 s
->base
.is_jmp
= DISAS_JUMP
;
1627 static bool trans_BRA(DisasContext
*s
, arg_bra
*a
)
1631 if (!dc_isar_feature(aa64_pauth
, s
)) {
1634 dst
= auth_branch_target(s
, cpu_reg(s
,a
->rn
), cpu_reg_sp(s
, a
->rm
), !a
->m
);
1635 gen_a64_set_pc(s
, dst
);
1636 set_btype_for_br(s
, a
->rn
);
1637 s
->base
.is_jmp
= DISAS_JUMP
;
1641 static bool trans_BLRA(DisasContext
*s
, arg_bra
*a
)
1645 if (!dc_isar_feature(aa64_pauth
, s
)) {
1648 dst
= auth_branch_target(s
, cpu_reg(s
, a
->rn
), cpu_reg_sp(s
, a
->rm
), !a
->m
);
1649 lr
= cpu_reg(s
, 30);
1651 TCGv_i64 tmp
= tcg_temp_new_i64();
1652 tcg_gen_mov_i64(tmp
, dst
);
1655 gen_pc_plus_diff(s
, lr
, curr_insn_len(s
));
1656 gen_a64_set_pc(s
, dst
);
1657 set_btype_for_blr(s
);
1658 s
->base
.is_jmp
= DISAS_JUMP
;
1662 static bool trans_ERET(DisasContext
*s
, arg_ERET
*a
)
1666 if (s
->current_el
== 0) {
1670 gen_exception_insn_el(s
, 0, EXCP_UDEF
, syn_erettrap(0), 2);
1673 dst
= tcg_temp_new_i64();
1674 tcg_gen_ld_i64(dst
, tcg_env
,
1675 offsetof(CPUARMState
, elr_el
[s
->current_el
]));
1677 translator_io_start(&s
->base
);
1679 gen_helper_exception_return(tcg_env
, dst
);
1680 /* Must exit loop to check un-masked IRQs */
1681 s
->base
.is_jmp
= DISAS_EXIT
;
1685 static bool trans_ERETA(DisasContext
*s
, arg_reta
*a
)
1689 if (!dc_isar_feature(aa64_pauth
, s
)) {
1692 if (s
->current_el
== 0) {
1695 /* The FGT trap takes precedence over an auth trap. */
1697 gen_exception_insn_el(s
, 0, EXCP_UDEF
, syn_erettrap(a
->m
? 3 : 2), 2);
1700 dst
= tcg_temp_new_i64();
1701 tcg_gen_ld_i64(dst
, tcg_env
,
1702 offsetof(CPUARMState
, elr_el
[s
->current_el
]));
1704 dst
= auth_branch_target(s
, dst
, cpu_X
[31], !a
->m
);
1706 translator_io_start(&s
->base
);
1708 gen_helper_exception_return(tcg_env
, dst
);
1709 /* Must exit loop to check un-masked IRQs */
1710 s
->base
.is_jmp
= DISAS_EXIT
;
1714 static bool trans_NOP(DisasContext
*s
, arg_NOP
*a
)
1719 static bool trans_YIELD(DisasContext
*s
, arg_YIELD
*a
)
1722 * When running in MTTCG we don't generate jumps to the yield and
1723 * WFE helpers as it won't affect the scheduling of other vCPUs.
1724 * If we wanted to more completely model WFE/SEV so we don't busy
1725 * spin unnecessarily we would need to do something more involved.
1727 if (!(tb_cflags(s
->base
.tb
) & CF_PARALLEL
)) {
1728 s
->base
.is_jmp
= DISAS_YIELD
;
1733 static bool trans_WFI(DisasContext
*s
, arg_WFI
*a
)
1735 s
->base
.is_jmp
= DISAS_WFI
;
1739 static bool trans_WFE(DisasContext
*s
, arg_WFI
*a
)
1742 * When running in MTTCG we don't generate jumps to the yield and
1743 * WFE helpers as it won't affect the scheduling of other vCPUs.
1744 * If we wanted to more completely model WFE/SEV so we don't busy
1745 * spin unnecessarily we would need to do something more involved.
1747 if (!(tb_cflags(s
->base
.tb
) & CF_PARALLEL
)) {
1748 s
->base
.is_jmp
= DISAS_WFE
;
1753 static bool trans_XPACLRI(DisasContext
*s
, arg_XPACLRI
*a
)
1755 if (s
->pauth_active
) {
1756 gen_helper_xpaci(cpu_X
[30], tcg_env
, cpu_X
[30]);
1761 static bool trans_PACIA1716(DisasContext
*s
, arg_PACIA1716
*a
)
1763 if (s
->pauth_active
) {
1764 gen_helper_pacia(cpu_X
[17], tcg_env
, cpu_X
[17], cpu_X
[16]);
1769 static bool trans_PACIB1716(DisasContext
*s
, arg_PACIB1716
*a
)
1771 if (s
->pauth_active
) {
1772 gen_helper_pacib(cpu_X
[17], tcg_env
, cpu_X
[17], cpu_X
[16]);
1777 static bool trans_AUTIA1716(DisasContext
*s
, arg_AUTIA1716
*a
)
1779 if (s
->pauth_active
) {
1780 gen_helper_autia(cpu_X
[17], tcg_env
, cpu_X
[17], cpu_X
[16]);
1785 static bool trans_AUTIB1716(DisasContext
*s
, arg_AUTIB1716
*a
)
1787 if (s
->pauth_active
) {
1788 gen_helper_autib(cpu_X
[17], tcg_env
, cpu_X
[17], cpu_X
[16]);
1793 static bool trans_ESB(DisasContext
*s
, arg_ESB
*a
)
1795 /* Without RAS, we must implement this as NOP. */
1796 if (dc_isar_feature(aa64_ras
, s
)) {
1798 * QEMU does not have a source of physical SErrors,
1799 * so we are only concerned with virtual SErrors.
1800 * The pseudocode in the ARM for this case is
1801 * if PSTATE.EL IN {EL0, EL1} && EL2Enabled() then
1802 * AArch64.vESBOperation();
1803 * Most of the condition can be evaluated at translation time.
1804 * Test for EL2 present, and defer test for SEL2 to runtime.
1806 if (s
->current_el
<= 1 && arm_dc_feature(s
, ARM_FEATURE_EL2
)) {
1807 gen_helper_vesb(tcg_env
);
1813 static bool trans_PACIAZ(DisasContext
*s
, arg_PACIAZ
*a
)
1815 if (s
->pauth_active
) {
1816 gen_helper_pacia(cpu_X
[30], tcg_env
, cpu_X
[30], tcg_constant_i64(0));
1821 static bool trans_PACIASP(DisasContext
*s
, arg_PACIASP
*a
)
1823 if (s
->pauth_active
) {
1824 gen_helper_pacia(cpu_X
[30], tcg_env
, cpu_X
[30], cpu_X
[31]);
1829 static bool trans_PACIBZ(DisasContext
*s
, arg_PACIBZ
*a
)
1831 if (s
->pauth_active
) {
1832 gen_helper_pacib(cpu_X
[30], tcg_env
, cpu_X
[30], tcg_constant_i64(0));
1837 static bool trans_PACIBSP(DisasContext
*s
, arg_PACIBSP
*a
)
1839 if (s
->pauth_active
) {
1840 gen_helper_pacib(cpu_X
[30], tcg_env
, cpu_X
[30], cpu_X
[31]);
1845 static bool trans_AUTIAZ(DisasContext
*s
, arg_AUTIAZ
*a
)
1847 if (s
->pauth_active
) {
1848 gen_helper_autia(cpu_X
[30], tcg_env
, cpu_X
[30], tcg_constant_i64(0));
1853 static bool trans_AUTIASP(DisasContext
*s
, arg_AUTIASP
*a
)
1855 if (s
->pauth_active
) {
1856 gen_helper_autia(cpu_X
[30], tcg_env
, cpu_X
[30], cpu_X
[31]);
1861 static bool trans_AUTIBZ(DisasContext
*s
, arg_AUTIBZ
*a
)
1863 if (s
->pauth_active
) {
1864 gen_helper_autib(cpu_X
[30], tcg_env
, cpu_X
[30], tcg_constant_i64(0));
1869 static bool trans_AUTIBSP(DisasContext
*s
, arg_AUTIBSP
*a
)
1871 if (s
->pauth_active
) {
1872 gen_helper_autib(cpu_X
[30], tcg_env
, cpu_X
[30], cpu_X
[31]);
1877 static bool trans_CLREX(DisasContext
*s
, arg_CLREX
*a
)
1879 tcg_gen_movi_i64(cpu_exclusive_addr
, -1);
1883 static bool trans_DSB_DMB(DisasContext
*s
, arg_DSB_DMB
*a
)
1885 /* We handle DSB and DMB the same way */
1889 case 1: /* MBReqTypes_Reads */
1890 bar
= TCG_BAR_SC
| TCG_MO_LD_LD
| TCG_MO_LD_ST
;
1892 case 2: /* MBReqTypes_Writes */
1893 bar
= TCG_BAR_SC
| TCG_MO_ST_ST
;
1895 default: /* MBReqTypes_All */
1896 bar
= TCG_BAR_SC
| TCG_MO_ALL
;
1903 static bool trans_ISB(DisasContext
*s
, arg_ISB
*a
)
1906 * We need to break the TB after this insn to execute
1907 * self-modifying code correctly and also to take
1908 * any pending interrupts immediately.
1911 gen_goto_tb(s
, 0, 4);
1915 static bool trans_SB(DisasContext
*s
, arg_SB
*a
)
1917 if (!dc_isar_feature(aa64_sb
, s
)) {
1921 * TODO: There is no speculation barrier opcode for TCG;
1922 * MB and end the TB instead.
1924 tcg_gen_mb(TCG_MO_ALL
| TCG_BAR_SC
);
1925 gen_goto_tb(s
, 0, 4);
1929 static bool trans_CFINV(DisasContext
*s
, arg_CFINV
*a
)
1931 if (!dc_isar_feature(aa64_condm_4
, s
)) {
1934 tcg_gen_xori_i32(cpu_CF
, cpu_CF
, 1);
1938 static bool trans_XAFLAG(DisasContext
*s
, arg_XAFLAG
*a
)
1942 if (!dc_isar_feature(aa64_condm_5
, s
)) {
1946 z
= tcg_temp_new_i32();
1948 tcg_gen_setcondi_i32(TCG_COND_EQ
, z
, cpu_ZF
, 0);
1957 tcg_gen_or_i32(cpu_NF
, cpu_CF
, z
);
1958 tcg_gen_subi_i32(cpu_NF
, cpu_NF
, 1);
1961 tcg_gen_and_i32(cpu_ZF
, z
, cpu_CF
);
1962 tcg_gen_xori_i32(cpu_ZF
, cpu_ZF
, 1);
1964 /* (!C & Z) << 31 -> -(Z & ~C) */
1965 tcg_gen_andc_i32(cpu_VF
, z
, cpu_CF
);
1966 tcg_gen_neg_i32(cpu_VF
, cpu_VF
);
1969 tcg_gen_or_i32(cpu_CF
, cpu_CF
, z
);
1974 static bool trans_AXFLAG(DisasContext
*s
, arg_AXFLAG
*a
)
1976 if (!dc_isar_feature(aa64_condm_5
, s
)) {
1980 tcg_gen_sari_i32(cpu_VF
, cpu_VF
, 31); /* V ? -1 : 0 */
1981 tcg_gen_andc_i32(cpu_CF
, cpu_CF
, cpu_VF
); /* C & !V */
1983 /* !(Z | V) -> !(!ZF | V) -> ZF & !V -> ZF & ~VF */
1984 tcg_gen_andc_i32(cpu_ZF
, cpu_ZF
, cpu_VF
);
1986 tcg_gen_movi_i32(cpu_NF
, 0);
1987 tcg_gen_movi_i32(cpu_VF
, 0);
1992 static bool trans_MSR_i_UAO(DisasContext
*s
, arg_i
*a
)
1994 if (!dc_isar_feature(aa64_uao
, s
) || s
->current_el
== 0) {
1998 set_pstate_bits(PSTATE_UAO
);
2000 clear_pstate_bits(PSTATE_UAO
);
2002 gen_rebuild_hflags(s
);
2003 s
->base
.is_jmp
= DISAS_TOO_MANY
;
2007 static bool trans_MSR_i_PAN(DisasContext
*s
, arg_i
*a
)
2009 if (!dc_isar_feature(aa64_pan
, s
) || s
->current_el
== 0) {
2013 set_pstate_bits(PSTATE_PAN
);
2015 clear_pstate_bits(PSTATE_PAN
);
2017 gen_rebuild_hflags(s
);
2018 s
->base
.is_jmp
= DISAS_TOO_MANY
;
2022 static bool trans_MSR_i_SPSEL(DisasContext
*s
, arg_i
*a
)
2024 if (s
->current_el
== 0) {
2027 gen_helper_msr_i_spsel(tcg_env
, tcg_constant_i32(a
->imm
& PSTATE_SP
));
2028 s
->base
.is_jmp
= DISAS_TOO_MANY
;
2032 static bool trans_MSR_i_SBSS(DisasContext
*s
, arg_i
*a
)
2034 if (!dc_isar_feature(aa64_ssbs
, s
)) {
2038 set_pstate_bits(PSTATE_SSBS
);
2040 clear_pstate_bits(PSTATE_SSBS
);
2042 /* Don't need to rebuild hflags since SSBS is a nop */
2043 s
->base
.is_jmp
= DISAS_TOO_MANY
;
2047 static bool trans_MSR_i_DIT(DisasContext
*s
, arg_i
*a
)
2049 if (!dc_isar_feature(aa64_dit
, s
)) {
2053 set_pstate_bits(PSTATE_DIT
);
2055 clear_pstate_bits(PSTATE_DIT
);
2057 /* There's no need to rebuild hflags because DIT is a nop */
2058 s
->base
.is_jmp
= DISAS_TOO_MANY
;
2062 static bool trans_MSR_i_TCO(DisasContext
*s
, arg_i
*a
)
2064 if (dc_isar_feature(aa64_mte
, s
)) {
2065 /* Full MTE is enabled -- set the TCO bit as directed. */
2067 set_pstate_bits(PSTATE_TCO
);
2069 clear_pstate_bits(PSTATE_TCO
);
2071 gen_rebuild_hflags(s
);
2072 /* Many factors, including TCO, go into MTE_ACTIVE. */
2073 s
->base
.is_jmp
= DISAS_UPDATE_NOCHAIN
;
2075 } else if (dc_isar_feature(aa64_mte_insn_reg
, s
)) {
2076 /* Only "instructions accessible at EL0" -- PSTATE.TCO is WI. */
2079 /* Insn not present */
2084 static bool trans_MSR_i_DAIFSET(DisasContext
*s
, arg_i
*a
)
2086 gen_helper_msr_i_daifset(tcg_env
, tcg_constant_i32(a
->imm
));
2087 s
->base
.is_jmp
= DISAS_TOO_MANY
;
2091 static bool trans_MSR_i_DAIFCLEAR(DisasContext
*s
, arg_i
*a
)
2093 gen_helper_msr_i_daifclear(tcg_env
, tcg_constant_i32(a
->imm
));
2094 /* Exit the cpu loop to re-evaluate pending IRQs. */
2095 s
->base
.is_jmp
= DISAS_UPDATE_EXIT
;
2099 static bool trans_MSR_i_ALLINT(DisasContext
*s
, arg_i
*a
)
2101 if (!dc_isar_feature(aa64_nmi
, s
) || s
->current_el
== 0) {
2106 clear_pstate_bits(PSTATE_ALLINT
);
2107 } else if (s
->current_el
> 1) {
2108 set_pstate_bits(PSTATE_ALLINT
);
2110 gen_helper_msr_set_allint_el1(tcg_env
);
2113 /* Exit the cpu loop to re-evaluate pending IRQs. */
2114 s
->base
.is_jmp
= DISAS_UPDATE_EXIT
;
2118 static bool trans_MSR_i_SVCR(DisasContext
*s
, arg_MSR_i_SVCR
*a
)
2120 if (!dc_isar_feature(aa64_sme
, s
) || a
->mask
== 0) {
2123 if (sme_access_check(s
)) {
2124 int old
= s
->pstate_sm
| (s
->pstate_za
<< 1);
2125 int new = a
->imm
* 3;
2127 if ((old
^ new) & a
->mask
) {
2128 /* At least one bit changes. */
2129 gen_helper_set_svcr(tcg_env
, tcg_constant_i32(new),
2130 tcg_constant_i32(a
->mask
));
2131 s
->base
.is_jmp
= DISAS_TOO_MANY
;
2137 static void gen_get_nzcv(TCGv_i64 tcg_rt
)
2139 TCGv_i32 tmp
= tcg_temp_new_i32();
2140 TCGv_i32 nzcv
= tcg_temp_new_i32();
2142 /* build bit 31, N */
2143 tcg_gen_andi_i32(nzcv
, cpu_NF
, (1U << 31));
2144 /* build bit 30, Z */
2145 tcg_gen_setcondi_i32(TCG_COND_EQ
, tmp
, cpu_ZF
, 0);
2146 tcg_gen_deposit_i32(nzcv
, nzcv
, tmp
, 30, 1);
2147 /* build bit 29, C */
2148 tcg_gen_deposit_i32(nzcv
, nzcv
, cpu_CF
, 29, 1);
2149 /* build bit 28, V */
2150 tcg_gen_shri_i32(tmp
, cpu_VF
, 31);
2151 tcg_gen_deposit_i32(nzcv
, nzcv
, tmp
, 28, 1);
2152 /* generate result */
2153 tcg_gen_extu_i32_i64(tcg_rt
, nzcv
);
2156 static void gen_set_nzcv(TCGv_i64 tcg_rt
)
2158 TCGv_i32 nzcv
= tcg_temp_new_i32();
2160 /* take NZCV from R[t] */
2161 tcg_gen_extrl_i64_i32(nzcv
, tcg_rt
);
2164 tcg_gen_andi_i32(cpu_NF
, nzcv
, (1U << 31));
2166 tcg_gen_andi_i32(cpu_ZF
, nzcv
, (1 << 30));
2167 tcg_gen_setcondi_i32(TCG_COND_EQ
, cpu_ZF
, cpu_ZF
, 0);
2169 tcg_gen_andi_i32(cpu_CF
, nzcv
, (1 << 29));
2170 tcg_gen_shri_i32(cpu_CF
, cpu_CF
, 29);
2172 tcg_gen_andi_i32(cpu_VF
, nzcv
, (1 << 28));
2173 tcg_gen_shli_i32(cpu_VF
, cpu_VF
, 3);
2176 static void gen_sysreg_undef(DisasContext
*s
, bool isread
,
2177 uint8_t op0
, uint8_t op1
, uint8_t op2
,
2178 uint8_t crn
, uint8_t crm
, uint8_t rt
)
2181 * Generate code to emit an UNDEF with correct syndrome
2182 * information for a failed system register access.
2183 * This is EC_UNCATEGORIZED (ie a standard UNDEF) in most cases,
2184 * but if FEAT_IDST is implemented then read accesses to registers
2185 * in the feature ID space are reported with the EC_SYSTEMREGISTERTRAP
2190 if (isread
&& dc_isar_feature(aa64_ids
, s
) &&
2191 arm_cpreg_encoding_in_idspace(op0
, op1
, op2
, crn
, crm
)) {
2192 syndrome
= syn_aa64_sysregtrap(op0
, op1
, op2
, crn
, crm
, rt
, isread
);
2194 syndrome
= syn_uncategorized();
2196 gen_exception_insn(s
, 0, EXCP_UDEF
, syndrome
);
2199 /* MRS - move from system register
2200 * MSR (register) - move to system register
2203 * These are all essentially the same insn in 'read' and 'write'
2204 * versions, with varying op0 fields.
2206 static void handle_sys(DisasContext
*s
, bool isread
,
2207 unsigned int op0
, unsigned int op1
, unsigned int op2
,
2208 unsigned int crn
, unsigned int crm
, unsigned int rt
)
2210 uint32_t key
= ENCODE_AA64_CP_REG(CP_REG_ARM64_SYSREG_CP
,
2211 crn
, crm
, op0
, op1
, op2
);
2212 const ARMCPRegInfo
*ri
= get_arm_cp_reginfo(s
->cp_regs
, key
);
2213 bool need_exit_tb
= false;
2214 bool nv_trap_to_el2
= false;
2215 bool nv_redirect_reg
= false;
2216 bool skip_fp_access_checks
= false;
2217 bool nv2_mem_redirect
= false;
2218 TCGv_ptr tcg_ri
= NULL
;
2220 uint32_t syndrome
= syn_aa64_sysregtrap(op0
, op1
, op2
, crn
, crm
, rt
, isread
);
2222 if (crn
== 11 || crn
== 15) {
2224 * Check for TIDCP trap, which must take precedence over
2225 * the UNDEF for "no such register" etc.
2227 switch (s
->current_el
) {
2229 if (dc_isar_feature(aa64_tidcp1
, s
)) {
2230 gen_helper_tidcp_el0(tcg_env
, tcg_constant_i32(syndrome
));
2234 gen_helper_tidcp_el1(tcg_env
, tcg_constant_i32(syndrome
));
2240 /* Unknown register; this might be a guest error or a QEMU
2241 * unimplemented feature.
2243 qemu_log_mask(LOG_UNIMP
, "%s access to unsupported AArch64 "
2244 "system register op0:%d op1:%d crn:%d crm:%d op2:%d\n",
2245 isread
? "read" : "write", op0
, op1
, crn
, crm
, op2
);
2246 gen_sysreg_undef(s
, isread
, op0
, op1
, op2
, crn
, crm
, rt
);
2250 if (s
->nv2
&& ri
->nv2_redirect_offset
) {
2252 * Some registers always redirect to memory; some only do so if
2253 * HCR_EL2.NV1 is 0, and some only if NV1 is 1 (these come in
2254 * pairs which share an offset; see the table in R_CSRPQ).
2256 if (ri
->nv2_redirect_offset
& NV2_REDIR_NV1
) {
2257 nv2_mem_redirect
= s
->nv1
;
2258 } else if (ri
->nv2_redirect_offset
& NV2_REDIR_NO_NV1
) {
2259 nv2_mem_redirect
= !s
->nv1
;
2261 nv2_mem_redirect
= true;
2265 /* Check access permissions */
2266 if (!cp_access_ok(s
->current_el
, ri
, isread
)) {
2268 * FEAT_NV/NV2 handling does not do the usual FP access checks
2269 * for registers only accessible at EL2 (though it *does* do them
2270 * for registers accessible at EL1).
2272 skip_fp_access_checks
= true;
2273 if (s
->nv2
&& (ri
->type
& ARM_CP_NV2_REDIRECT
)) {
2275 * This is one of the few EL2 registers which should redirect
2276 * to the equivalent EL1 register. We do that after running
2277 * the EL2 register's accessfn.
2279 nv_redirect_reg
= true;
2280 assert(!nv2_mem_redirect
);
2281 } else if (nv2_mem_redirect
) {
2283 * NV2 redirect-to-memory takes precedence over trap to EL2 or
2286 } else if (s
->nv
&& arm_cpreg_traps_in_nv(ri
)) {
2288 * This register / instruction exists and is an EL2 register, so
2289 * we must trap to EL2 if accessed in nested virtualization EL1
2290 * instead of UNDEFing. We'll do that after the usual access checks.
2291 * (This makes a difference only for a couple of registers like
2292 * VSTTBR_EL2 where the "UNDEF if NonSecure" should take priority
2293 * over the trap-to-EL2. Most trapped-by-FEAT_NV registers have
2294 * an accessfn which does nothing when called from EL1, because
2295 * the trap-to-EL3 controls which would apply to that register
2296 * at EL2 don't take priority over the FEAT_NV trap-to-EL2.)
2298 nv_trap_to_el2
= true;
2300 gen_sysreg_undef(s
, isread
, op0
, op1
, op2
, crn
, crm
, rt
);
2305 if (ri
->accessfn
|| (ri
->fgt
&& s
->fgt_active
)) {
2306 /* Emit code to perform further access permissions checks at
2307 * runtime; this may result in an exception.
2309 gen_a64_update_pc(s
, 0);
2310 tcg_ri
= tcg_temp_new_ptr();
2311 gen_helper_access_check_cp_reg(tcg_ri
, tcg_env
,
2312 tcg_constant_i32(key
),
2313 tcg_constant_i32(syndrome
),
2314 tcg_constant_i32(isread
));
2315 } else if (ri
->type
& ARM_CP_RAISES_EXC
) {
2317 * The readfn or writefn might raise an exception;
2318 * synchronize the CPU state in case it does.
2320 gen_a64_update_pc(s
, 0);
2323 if (!skip_fp_access_checks
) {
2324 if ((ri
->type
& ARM_CP_FPU
) && !fp_access_check_only(s
)) {
2326 } else if ((ri
->type
& ARM_CP_SVE
) && !sve_access_check(s
)) {
2328 } else if ((ri
->type
& ARM_CP_SME
) && !sme_access_check(s
)) {
2333 if (nv_trap_to_el2
) {
2334 gen_exception_insn_el(s
, 0, EXCP_UDEF
, syndrome
, 2);
2338 if (nv_redirect_reg
) {
2340 * FEAT_NV2 redirection of an EL2 register to an EL1 register.
2341 * Conveniently in all cases the encoding of the EL1 register is
2342 * identical to the EL2 register except that opc1 is 0.
2343 * Get the reginfo for the EL1 register to use for the actual access.
2344 * We don't use the EL1 register's access function, and
2345 * fine-grained-traps on EL1 also do not apply here.
2347 key
= ENCODE_AA64_CP_REG(CP_REG_ARM64_SYSREG_CP
,
2348 crn
, crm
, op0
, 0, op2
);
2349 ri
= get_arm_cp_reginfo(s
->cp_regs
, key
);
2351 assert(cp_access_ok(s
->current_el
, ri
, isread
));
2353 * We might not have done an update_pc earlier, so check we don't
2354 * need it. We could support this in future if necessary.
2356 assert(!(ri
->type
& ARM_CP_RAISES_EXC
));
2359 if (nv2_mem_redirect
) {
2361 * This system register is being redirected into an EL2 memory access.
2362 * This means it is not an IO operation, doesn't change hflags,
2363 * and need not end the TB, because it has no side effects.
2365 * The access is 64-bit single copy atomic, guaranteed aligned because
2366 * of the definition of VCNR_EL2. Its endianness depends on
2367 * SCTLR_EL2.EE, not on the data endianness of EL1.
2368 * It is done under either the EL2 translation regime or the EL2&0
2369 * translation regime, depending on HCR_EL2.E2H. It behaves as if
2372 TCGv_i64 ptr
= tcg_temp_new_i64();
2373 MemOp mop
= MO_64
| MO_ALIGN
| MO_ATOM_IFALIGN
;
2374 ARMMMUIdx armmemidx
= s
->nv2_mem_e20
? ARMMMUIdx_E20_2
: ARMMMUIdx_E2
;
2375 int memidx
= arm_to_core_mmu_idx(armmemidx
);
2378 mop
|= (s
->nv2_mem_be
? MO_BE
: MO_LE
);
2380 tcg_gen_ld_i64(ptr
, tcg_env
, offsetof(CPUARMState
, cp15
.vncr_el2
));
2381 tcg_gen_addi_i64(ptr
, ptr
,
2382 (ri
->nv2_redirect_offset
& ~NV2_REDIR_FLAG_MASK
));
2383 tcg_rt
= cpu_reg(s
, rt
);
2385 syn
= syn_data_abort_vncr(0, !isread
, 0);
2386 disas_set_insn_syndrome(s
, syn
);
2388 tcg_gen_qemu_ld_i64(tcg_rt
, ptr
, memidx
, mop
);
2390 tcg_gen_qemu_st_i64(tcg_rt
, ptr
, memidx
, mop
);
2395 /* Handle special cases first */
2396 switch (ri
->type
& ARM_CP_SPECIAL_MASK
) {
2402 tcg_rt
= cpu_reg(s
, rt
);
2404 gen_get_nzcv(tcg_rt
);
2406 gen_set_nzcv(tcg_rt
);
2409 case ARM_CP_CURRENTEL
:
2412 * Reads as current EL value from pstate, which is
2413 * guaranteed to be constant by the tb flags.
2414 * For nested virt we should report EL2.
2416 int el
= s
->nv
? 2 : s
->current_el
;
2417 tcg_rt
= cpu_reg(s
, rt
);
2418 tcg_gen_movi_i64(tcg_rt
, el
<< 2);
2422 /* Writes clear the aligned block of memory which rt points into. */
2423 if (s
->mte_active
[0]) {
2426 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, get_mem_index(s
));
2427 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
2428 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
2430 tcg_rt
= tcg_temp_new_i64();
2431 gen_helper_mte_check_zva(tcg_rt
, tcg_env
,
2432 tcg_constant_i32(desc
), cpu_reg(s
, rt
));
2434 tcg_rt
= clean_data_tbi(s
, cpu_reg(s
, rt
));
2436 gen_helper_dc_zva(tcg_env
, tcg_rt
);
2440 TCGv_i64 clean_addr
, tag
;
2443 * DC_GVA, like DC_ZVA, requires that we supply the original
2444 * pointer for an invalid page. Probe that address first.
2446 tcg_rt
= cpu_reg(s
, rt
);
2447 clean_addr
= clean_data_tbi(s
, tcg_rt
);
2448 gen_probe_access(s
, clean_addr
, MMU_DATA_STORE
, MO_8
);
2451 /* Extract the tag from the register to match STZGM. */
2452 tag
= tcg_temp_new_i64();
2453 tcg_gen_shri_i64(tag
, tcg_rt
, 56);
2454 gen_helper_stzgm_tags(tcg_env
, clean_addr
, tag
);
2458 case ARM_CP_DC_GZVA
:
2460 TCGv_i64 clean_addr
, tag
;
2462 /* For DC_GZVA, we can rely on DC_ZVA for the proper fault. */
2463 tcg_rt
= cpu_reg(s
, rt
);
2464 clean_addr
= clean_data_tbi(s
, tcg_rt
);
2465 gen_helper_dc_zva(tcg_env
, clean_addr
);
2468 /* Extract the tag from the register to match STZGM. */
2469 tag
= tcg_temp_new_i64();
2470 tcg_gen_shri_i64(tag
, tcg_rt
, 56);
2471 gen_helper_stzgm_tags(tcg_env
, clean_addr
, tag
);
2476 g_assert_not_reached();
2479 if (ri
->type
& ARM_CP_IO
) {
2480 /* I/O operations must end the TB here (whether read or write) */
2481 need_exit_tb
= translator_io_start(&s
->base
);
2484 tcg_rt
= cpu_reg(s
, rt
);
2487 if (ri
->type
& ARM_CP_CONST
) {
2488 tcg_gen_movi_i64(tcg_rt
, ri
->resetvalue
);
2489 } else if (ri
->readfn
) {
2491 tcg_ri
= gen_lookup_cp_reg(key
);
2493 gen_helper_get_cp_reg64(tcg_rt
, tcg_env
, tcg_ri
);
2495 tcg_gen_ld_i64(tcg_rt
, tcg_env
, ri
->fieldoffset
);
2498 if (ri
->type
& ARM_CP_CONST
) {
2499 /* If not forbidden by access permissions, treat as WI */
2501 } else if (ri
->writefn
) {
2503 tcg_ri
= gen_lookup_cp_reg(key
);
2505 gen_helper_set_cp_reg64(tcg_env
, tcg_ri
, tcg_rt
);
2507 tcg_gen_st_i64(tcg_rt
, tcg_env
, ri
->fieldoffset
);
2511 if (!isread
&& !(ri
->type
& ARM_CP_SUPPRESS_TB_END
)) {
2513 * A write to any coprocessor register that ends a TB
2514 * must rebuild the hflags for the next TB.
2516 gen_rebuild_hflags(s
);
2518 * We default to ending the TB on a coprocessor register write,
2519 * but allow this to be suppressed by the register definition
2520 * (usually only necessary to work around guest bugs).
2522 need_exit_tb
= true;
2525 s
->base
.is_jmp
= DISAS_UPDATE_EXIT
;
2529 static bool trans_SYS(DisasContext
*s
, arg_SYS
*a
)
2531 handle_sys(s
, a
->l
, a
->op0
, a
->op1
, a
->op2
, a
->crn
, a
->crm
, a
->rt
);
2535 static bool trans_SVC(DisasContext
*s
, arg_i
*a
)
2538 * For SVC, HVC and SMC we advance the single-step state
2539 * machine before taking the exception. This is architecturally
2540 * mandated, to ensure that single-stepping a system call
2541 * instruction works properly.
2543 uint32_t syndrome
= syn_aa64_svc(a
->imm
);
2545 gen_exception_insn_el(s
, 0, EXCP_UDEF
, syndrome
, 2);
2549 gen_exception_insn(s
, 4, EXCP_SWI
, syndrome
);
2553 static bool trans_HVC(DisasContext
*s
, arg_i
*a
)
2555 int target_el
= s
->current_el
== 3 ? 3 : 2;
2557 if (s
->current_el
== 0) {
2558 unallocated_encoding(s
);
2562 * The pre HVC helper handles cases when HVC gets trapped
2563 * as an undefined insn by runtime configuration.
2565 gen_a64_update_pc(s
, 0);
2566 gen_helper_pre_hvc(tcg_env
);
2567 /* Architecture requires ss advance before we do the actual work */
2569 gen_exception_insn_el(s
, 4, EXCP_HVC
, syn_aa64_hvc(a
->imm
), target_el
);
2573 static bool trans_SMC(DisasContext
*s
, arg_i
*a
)
2575 if (s
->current_el
== 0) {
2576 unallocated_encoding(s
);
2579 gen_a64_update_pc(s
, 0);
2580 gen_helper_pre_smc(tcg_env
, tcg_constant_i32(syn_aa64_smc(a
->imm
)));
2581 /* Architecture requires ss advance before we do the actual work */
2583 gen_exception_insn_el(s
, 4, EXCP_SMC
, syn_aa64_smc(a
->imm
), 3);
2587 static bool trans_BRK(DisasContext
*s
, arg_i
*a
)
2589 gen_exception_bkpt_insn(s
, syn_aa64_bkpt(a
->imm
));
2593 static bool trans_HLT(DisasContext
*s
, arg_i
*a
)
2596 * HLT. This has two purposes.
2597 * Architecturally, it is an external halting debug instruction.
2598 * Since QEMU doesn't implement external debug, we treat this as
2599 * it is required for halting debug disabled: it will UNDEF.
2600 * Secondly, "HLT 0xf000" is the A64 semihosting syscall instruction.
2602 if (semihosting_enabled(s
->current_el
== 0) && a
->imm
== 0xf000) {
2603 gen_exception_internal_insn(s
, EXCP_SEMIHOST
);
2605 unallocated_encoding(s
);
2611 * Load/Store exclusive instructions are implemented by remembering
2612 * the value/address loaded, and seeing if these are the same
2613 * when the store is performed. This is not actually the architecturally
2614 * mandated semantics, but it works for typical guest code sequences
2615 * and avoids having to monitor regular stores.
2617 * The store exclusive uses the atomic cmpxchg primitives to avoid
2618 * races in multi-threaded linux-user and when MTTCG softmmu is
2621 static void gen_load_exclusive(DisasContext
*s
, int rt
, int rt2
, int rn
,
2622 int size
, bool is_pair
)
2624 int idx
= get_mem_index(s
);
2625 TCGv_i64 dirty_addr
, clean_addr
;
2626 MemOp memop
= check_atomic_align(s
, rn
, size
+ is_pair
);
2629 dirty_addr
= cpu_reg_sp(s
, rn
);
2630 clean_addr
= gen_mte_check1(s
, dirty_addr
, false, rn
!= 31, memop
);
2632 g_assert(size
<= 3);
2634 g_assert(size
>= 2);
2636 tcg_gen_qemu_ld_i64(cpu_exclusive_val
, clean_addr
, idx
, memop
);
2637 if (s
->be_data
== MO_LE
) {
2638 tcg_gen_extract_i64(cpu_reg(s
, rt
), cpu_exclusive_val
, 0, 32);
2639 tcg_gen_extract_i64(cpu_reg(s
, rt2
), cpu_exclusive_val
, 32, 32);
2641 tcg_gen_extract_i64(cpu_reg(s
, rt
), cpu_exclusive_val
, 32, 32);
2642 tcg_gen_extract_i64(cpu_reg(s
, rt2
), cpu_exclusive_val
, 0, 32);
2645 TCGv_i128 t16
= tcg_temp_new_i128();
2647 tcg_gen_qemu_ld_i128(t16
, clean_addr
, idx
, memop
);
2649 if (s
->be_data
== MO_LE
) {
2650 tcg_gen_extr_i128_i64(cpu_exclusive_val
,
2651 cpu_exclusive_high
, t16
);
2653 tcg_gen_extr_i128_i64(cpu_exclusive_high
,
2654 cpu_exclusive_val
, t16
);
2656 tcg_gen_mov_i64(cpu_reg(s
, rt
), cpu_exclusive_val
);
2657 tcg_gen_mov_i64(cpu_reg(s
, rt2
), cpu_exclusive_high
);
2660 tcg_gen_qemu_ld_i64(cpu_exclusive_val
, clean_addr
, idx
, memop
);
2661 tcg_gen_mov_i64(cpu_reg(s
, rt
), cpu_exclusive_val
);
2663 tcg_gen_mov_i64(cpu_exclusive_addr
, clean_addr
);
2666 static void gen_store_exclusive(DisasContext
*s
, int rd
, int rt
, int rt2
,
2667 int rn
, int size
, int is_pair
)
2669 /* if (env->exclusive_addr == addr && env->exclusive_val == [addr]
2670 * && (!is_pair || env->exclusive_high == [addr + datasize])) {
2673 * [addr + datasize] = {Rt2};
2679 * env->exclusive_addr = -1;
2681 TCGLabel
*fail_label
= gen_new_label();
2682 TCGLabel
*done_label
= gen_new_label();
2683 TCGv_i64 tmp
, clean_addr
;
2687 * FIXME: We are out of spec here. We have recorded only the address
2688 * from load_exclusive, not the entire range, and we assume that the
2689 * size of the access on both sides match. The architecture allows the
2690 * store to be smaller than the load, so long as the stored bytes are
2691 * within the range recorded by the load.
2694 /* See AArch64.ExclusiveMonitorsPass() and AArch64.IsExclusiveVA(). */
2695 clean_addr
= clean_data_tbi(s
, cpu_reg_sp(s
, rn
));
2696 tcg_gen_brcond_i64(TCG_COND_NE
, clean_addr
, cpu_exclusive_addr
, fail_label
);
2699 * The write, and any associated faults, only happen if the virtual
2700 * and physical addresses pass the exclusive monitor check. These
2701 * faults are exceedingly unlikely, because normally the guest uses
2702 * the exact same address register for the load_exclusive, and we
2703 * would have recognized these faults there.
2705 * It is possible to trigger an alignment fault pre-LSE2, e.g. with an
2706 * unaligned 4-byte write within the range of an aligned 8-byte load.
2707 * With LSE2, the store would need to cross a 16-byte boundary when the
2708 * load did not, which would mean the store is outside the range
2709 * recorded for the monitor, which would have failed a corrected monitor
2710 * check above. For now, we assume no size change and retain the
2711 * MO_ALIGN to let tcg know what we checked in the load_exclusive.
2713 * It is possible to trigger an MTE fault, by performing the load with
2714 * a virtual address with a valid tag and performing the store with the
2715 * same virtual address and a different invalid tag.
2717 memop
= size
+ is_pair
;
2718 if (memop
== MO_128
|| !dc_isar_feature(aa64_lse2
, s
)) {
2721 memop
= finalize_memop(s
, memop
);
2722 gen_mte_check1(s
, cpu_reg_sp(s
, rn
), true, rn
!= 31, memop
);
2724 tmp
= tcg_temp_new_i64();
2727 if (s
->be_data
== MO_LE
) {
2728 tcg_gen_concat32_i64(tmp
, cpu_reg(s
, rt
), cpu_reg(s
, rt2
));
2730 tcg_gen_concat32_i64(tmp
, cpu_reg(s
, rt2
), cpu_reg(s
, rt
));
2732 tcg_gen_atomic_cmpxchg_i64(tmp
, cpu_exclusive_addr
,
2733 cpu_exclusive_val
, tmp
,
2734 get_mem_index(s
), memop
);
2735 tcg_gen_setcond_i64(TCG_COND_NE
, tmp
, tmp
, cpu_exclusive_val
);
2737 TCGv_i128 t16
= tcg_temp_new_i128();
2738 TCGv_i128 c16
= tcg_temp_new_i128();
2741 if (s
->be_data
== MO_LE
) {
2742 tcg_gen_concat_i64_i128(t16
, cpu_reg(s
, rt
), cpu_reg(s
, rt2
));
2743 tcg_gen_concat_i64_i128(c16
, cpu_exclusive_val
,
2744 cpu_exclusive_high
);
2746 tcg_gen_concat_i64_i128(t16
, cpu_reg(s
, rt2
), cpu_reg(s
, rt
));
2747 tcg_gen_concat_i64_i128(c16
, cpu_exclusive_high
,
2751 tcg_gen_atomic_cmpxchg_i128(t16
, cpu_exclusive_addr
, c16
, t16
,
2752 get_mem_index(s
), memop
);
2754 a
= tcg_temp_new_i64();
2755 b
= tcg_temp_new_i64();
2756 if (s
->be_data
== MO_LE
) {
2757 tcg_gen_extr_i128_i64(a
, b
, t16
);
2759 tcg_gen_extr_i128_i64(b
, a
, t16
);
2762 tcg_gen_xor_i64(a
, a
, cpu_exclusive_val
);
2763 tcg_gen_xor_i64(b
, b
, cpu_exclusive_high
);
2764 tcg_gen_or_i64(tmp
, a
, b
);
2766 tcg_gen_setcondi_i64(TCG_COND_NE
, tmp
, tmp
, 0);
2769 tcg_gen_atomic_cmpxchg_i64(tmp
, cpu_exclusive_addr
, cpu_exclusive_val
,
2770 cpu_reg(s
, rt
), get_mem_index(s
), memop
);
2771 tcg_gen_setcond_i64(TCG_COND_NE
, tmp
, tmp
, cpu_exclusive_val
);
2773 tcg_gen_mov_i64(cpu_reg(s
, rd
), tmp
);
2774 tcg_gen_br(done_label
);
2776 gen_set_label(fail_label
);
2777 tcg_gen_movi_i64(cpu_reg(s
, rd
), 1);
2778 gen_set_label(done_label
);
2779 tcg_gen_movi_i64(cpu_exclusive_addr
, -1);
2782 static void gen_compare_and_swap(DisasContext
*s
, int rs
, int rt
,
2785 TCGv_i64 tcg_rs
= cpu_reg(s
, rs
);
2786 TCGv_i64 tcg_rt
= cpu_reg(s
, rt
);
2787 int memidx
= get_mem_index(s
);
2788 TCGv_i64 clean_addr
;
2792 gen_check_sp_alignment(s
);
2794 memop
= check_atomic_align(s
, rn
, size
);
2795 clean_addr
= gen_mte_check1(s
, cpu_reg_sp(s
, rn
), true, rn
!= 31, memop
);
2796 tcg_gen_atomic_cmpxchg_i64(tcg_rs
, clean_addr
, tcg_rs
, tcg_rt
,
2800 static void gen_compare_and_swap_pair(DisasContext
*s
, int rs
, int rt
,
2803 TCGv_i64 s1
= cpu_reg(s
, rs
);
2804 TCGv_i64 s2
= cpu_reg(s
, rs
+ 1);
2805 TCGv_i64 t1
= cpu_reg(s
, rt
);
2806 TCGv_i64 t2
= cpu_reg(s
, rt
+ 1);
2807 TCGv_i64 clean_addr
;
2808 int memidx
= get_mem_index(s
);
2812 gen_check_sp_alignment(s
);
2815 /* This is a single atomic access, despite the "pair". */
2816 memop
= check_atomic_align(s
, rn
, size
+ 1);
2817 clean_addr
= gen_mte_check1(s
, cpu_reg_sp(s
, rn
), true, rn
!= 31, memop
);
2820 TCGv_i64 cmp
= tcg_temp_new_i64();
2821 TCGv_i64 val
= tcg_temp_new_i64();
2823 if (s
->be_data
== MO_LE
) {
2824 tcg_gen_concat32_i64(val
, t1
, t2
);
2825 tcg_gen_concat32_i64(cmp
, s1
, s2
);
2827 tcg_gen_concat32_i64(val
, t2
, t1
);
2828 tcg_gen_concat32_i64(cmp
, s2
, s1
);
2831 tcg_gen_atomic_cmpxchg_i64(cmp
, clean_addr
, cmp
, val
, memidx
, memop
);
2833 if (s
->be_data
== MO_LE
) {
2834 tcg_gen_extr32_i64(s1
, s2
, cmp
);
2836 tcg_gen_extr32_i64(s2
, s1
, cmp
);
2839 TCGv_i128 cmp
= tcg_temp_new_i128();
2840 TCGv_i128 val
= tcg_temp_new_i128();
2842 if (s
->be_data
== MO_LE
) {
2843 tcg_gen_concat_i64_i128(val
, t1
, t2
);
2844 tcg_gen_concat_i64_i128(cmp
, s1
, s2
);
2846 tcg_gen_concat_i64_i128(val
, t2
, t1
);
2847 tcg_gen_concat_i64_i128(cmp
, s2
, s1
);
2850 tcg_gen_atomic_cmpxchg_i128(cmp
, clean_addr
, cmp
, val
, memidx
, memop
);
2852 if (s
->be_data
== MO_LE
) {
2853 tcg_gen_extr_i128_i64(s1
, s2
, cmp
);
2855 tcg_gen_extr_i128_i64(s2
, s1
, cmp
);
2861 * Compute the ISS.SF bit for syndrome information if an exception
2862 * is taken on a load or store. This indicates whether the instruction
2863 * is accessing a 32-bit or 64-bit register. This logic is derived
2864 * from the ARMv8 specs for LDR (Shared decode for all encodings).
2866 static bool ldst_iss_sf(int size
, bool sign
, bool ext
)
2871 * Signed loads are 64 bit results if we are not going to
2872 * do a zero-extend from 32 to 64 after the load.
2873 * (For a store, sign and ext are always false.)
2877 /* Unsigned loads/stores work at the specified size */
2878 return size
== MO_64
;
2882 static bool trans_STXR(DisasContext
*s
, arg_stxr
*a
)
2885 gen_check_sp_alignment(s
);
2888 tcg_gen_mb(TCG_MO_ALL
| TCG_BAR_STRL
);
2890 gen_store_exclusive(s
, a
->rs
, a
->rt
, a
->rt2
, a
->rn
, a
->sz
, false);
2894 static bool trans_LDXR(DisasContext
*s
, arg_stxr
*a
)
2897 gen_check_sp_alignment(s
);
2899 gen_load_exclusive(s
, a
->rt
, a
->rt2
, a
->rn
, a
->sz
, false);
2901 tcg_gen_mb(TCG_MO_ALL
| TCG_BAR_LDAQ
);
2906 static bool trans_STLR(DisasContext
*s
, arg_stlr
*a
)
2908 TCGv_i64 clean_addr
;
2910 bool iss_sf
= ldst_iss_sf(a
->sz
, false, false);
2913 * StoreLORelease is the same as Store-Release for QEMU, but
2914 * needs the feature-test.
2916 if (!a
->lasr
&& !dc_isar_feature(aa64_lor
, s
)) {
2919 /* Generate ISS for non-exclusive accesses including LASR. */
2921 gen_check_sp_alignment(s
);
2923 tcg_gen_mb(TCG_MO_ALL
| TCG_BAR_STRL
);
2924 memop
= check_ordered_align(s
, a
->rn
, 0, true, a
->sz
);
2925 clean_addr
= gen_mte_check1(s
, cpu_reg_sp(s
, a
->rn
),
2926 true, a
->rn
!= 31, memop
);
2927 do_gpr_st(s
, cpu_reg(s
, a
->rt
), clean_addr
, memop
, true, a
->rt
,
2932 static bool trans_LDAR(DisasContext
*s
, arg_stlr
*a
)
2934 TCGv_i64 clean_addr
;
2936 bool iss_sf
= ldst_iss_sf(a
->sz
, false, false);
2938 /* LoadLOAcquire is the same as Load-Acquire for QEMU. */
2939 if (!a
->lasr
&& !dc_isar_feature(aa64_lor
, s
)) {
2942 /* Generate ISS for non-exclusive accesses including LASR. */
2944 gen_check_sp_alignment(s
);
2946 memop
= check_ordered_align(s
, a
->rn
, 0, false, a
->sz
);
2947 clean_addr
= gen_mte_check1(s
, cpu_reg_sp(s
, a
->rn
),
2948 false, a
->rn
!= 31, memop
);
2949 do_gpr_ld(s
, cpu_reg(s
, a
->rt
), clean_addr
, memop
, false, true,
2950 a
->rt
, iss_sf
, a
->lasr
);
2951 tcg_gen_mb(TCG_MO_ALL
| TCG_BAR_LDAQ
);
2955 static bool trans_STXP(DisasContext
*s
, arg_stxr
*a
)
2958 gen_check_sp_alignment(s
);
2961 tcg_gen_mb(TCG_MO_ALL
| TCG_BAR_STRL
);
2963 gen_store_exclusive(s
, a
->rs
, a
->rt
, a
->rt2
, a
->rn
, a
->sz
, true);
2967 static bool trans_LDXP(DisasContext
*s
, arg_stxr
*a
)
2970 gen_check_sp_alignment(s
);
2972 gen_load_exclusive(s
, a
->rt
, a
->rt2
, a
->rn
, a
->sz
, true);
2974 tcg_gen_mb(TCG_MO_ALL
| TCG_BAR_LDAQ
);
2979 static bool trans_CASP(DisasContext
*s
, arg_CASP
*a
)
2981 if (!dc_isar_feature(aa64_atomics
, s
)) {
2984 if (((a
->rt
| a
->rs
) & 1) != 0) {
2988 gen_compare_and_swap_pair(s
, a
->rs
, a
->rt
, a
->rn
, a
->sz
);
2992 static bool trans_CAS(DisasContext
*s
, arg_CAS
*a
)
2994 if (!dc_isar_feature(aa64_atomics
, s
)) {
2997 gen_compare_and_swap(s
, a
->rs
, a
->rt
, a
->rn
, a
->sz
);
3001 static bool trans_LD_lit(DisasContext
*s
, arg_ldlit
*a
)
3003 bool iss_sf
= ldst_iss_sf(a
->sz
, a
->sign
, false);
3004 TCGv_i64 tcg_rt
= cpu_reg(s
, a
->rt
);
3005 TCGv_i64 clean_addr
= tcg_temp_new_i64();
3006 MemOp memop
= finalize_memop(s
, a
->sz
+ a
->sign
* MO_SIGN
);
3008 gen_pc_plus_diff(s
, clean_addr
, a
->imm
);
3009 do_gpr_ld(s
, tcg_rt
, clean_addr
, memop
,
3010 false, true, a
->rt
, iss_sf
, false);
3014 static bool trans_LD_lit_v(DisasContext
*s
, arg_ldlit
*a
)
3016 /* Load register (literal), vector version */
3017 TCGv_i64 clean_addr
;
3020 if (!fp_access_check(s
)) {
3023 memop
= finalize_memop_asimd(s
, a
->sz
);
3024 clean_addr
= tcg_temp_new_i64();
3025 gen_pc_plus_diff(s
, clean_addr
, a
->imm
);
3026 do_fp_ld(s
, a
->rt
, clean_addr
, memop
);
3030 static void op_addr_ldstpair_pre(DisasContext
*s
, arg_ldstpair
*a
,
3031 TCGv_i64
*clean_addr
, TCGv_i64
*dirty_addr
,
3032 uint64_t offset
, bool is_store
, MemOp mop
)
3035 gen_check_sp_alignment(s
);
3038 *dirty_addr
= read_cpu_reg_sp(s
, a
->rn
, 1);
3040 tcg_gen_addi_i64(*dirty_addr
, *dirty_addr
, offset
);
3043 *clean_addr
= gen_mte_checkN(s
, *dirty_addr
, is_store
,
3044 (a
->w
|| a
->rn
!= 31), 2 << a
->sz
, mop
);
3047 static void op_addr_ldstpair_post(DisasContext
*s
, arg_ldstpair
*a
,
3048 TCGv_i64 dirty_addr
, uint64_t offset
)
3052 tcg_gen_addi_i64(dirty_addr
, dirty_addr
, offset
);
3054 tcg_gen_mov_i64(cpu_reg_sp(s
, a
->rn
), dirty_addr
);
3058 static bool trans_STP(DisasContext
*s
, arg_ldstpair
*a
)
3060 uint64_t offset
= a
->imm
<< a
->sz
;
3061 TCGv_i64 clean_addr
, dirty_addr
, tcg_rt
, tcg_rt2
;
3062 MemOp mop
= finalize_memop(s
, a
->sz
);
3064 op_addr_ldstpair_pre(s
, a
, &clean_addr
, &dirty_addr
, offset
, true, mop
);
3065 tcg_rt
= cpu_reg(s
, a
->rt
);
3066 tcg_rt2
= cpu_reg(s
, a
->rt2
);
3068 * We built mop above for the single logical access -- rebuild it
3069 * now for the paired operation.
3071 * With LSE2, non-sign-extending pairs are treated atomically if
3072 * aligned, and if unaligned one of the pair will be completely
3073 * within a 16-byte block and that element will be atomic.
3074 * Otherwise each element is separately atomic.
3075 * In all cases, issue one operation with the correct atomicity.
3079 mop
|= (a
->sz
== 2 ? MO_ALIGN_4
: MO_ALIGN_8
);
3081 mop
= finalize_memop_pair(s
, mop
);
3083 TCGv_i64 tmp
= tcg_temp_new_i64();
3085 if (s
->be_data
== MO_LE
) {
3086 tcg_gen_concat32_i64(tmp
, tcg_rt
, tcg_rt2
);
3088 tcg_gen_concat32_i64(tmp
, tcg_rt2
, tcg_rt
);
3090 tcg_gen_qemu_st_i64(tmp
, clean_addr
, get_mem_index(s
), mop
);
3092 TCGv_i128 tmp
= tcg_temp_new_i128();
3094 if (s
->be_data
== MO_LE
) {
3095 tcg_gen_concat_i64_i128(tmp
, tcg_rt
, tcg_rt2
);
3097 tcg_gen_concat_i64_i128(tmp
, tcg_rt2
, tcg_rt
);
3099 tcg_gen_qemu_st_i128(tmp
, clean_addr
, get_mem_index(s
), mop
);
3101 op_addr_ldstpair_post(s
, a
, dirty_addr
, offset
);
3105 static bool trans_LDP(DisasContext
*s
, arg_ldstpair
*a
)
3107 uint64_t offset
= a
->imm
<< a
->sz
;
3108 TCGv_i64 clean_addr
, dirty_addr
, tcg_rt
, tcg_rt2
;
3109 MemOp mop
= finalize_memop(s
, a
->sz
);
3111 op_addr_ldstpair_pre(s
, a
, &clean_addr
, &dirty_addr
, offset
, false, mop
);
3112 tcg_rt
= cpu_reg(s
, a
->rt
);
3113 tcg_rt2
= cpu_reg(s
, a
->rt2
);
3116 * We built mop above for the single logical access -- rebuild it
3117 * now for the paired operation.
3119 * With LSE2, non-sign-extending pairs are treated atomically if
3120 * aligned, and if unaligned one of the pair will be completely
3121 * within a 16-byte block and that element will be atomic.
3122 * Otherwise each element is separately atomic.
3123 * In all cases, issue one operation with the correct atomicity.
3125 * This treats sign-extending loads like zero-extending loads,
3126 * since that reuses the most code below.
3130 mop
|= (a
->sz
== 2 ? MO_ALIGN_4
: MO_ALIGN_8
);
3132 mop
= finalize_memop_pair(s
, mop
);
3134 int o2
= s
->be_data
== MO_LE
? 32 : 0;
3137 tcg_gen_qemu_ld_i64(tcg_rt
, clean_addr
, get_mem_index(s
), mop
);
3139 tcg_gen_sextract_i64(tcg_rt2
, tcg_rt
, o2
, 32);
3140 tcg_gen_sextract_i64(tcg_rt
, tcg_rt
, o1
, 32);
3142 tcg_gen_extract_i64(tcg_rt2
, tcg_rt
, o2
, 32);
3143 tcg_gen_extract_i64(tcg_rt
, tcg_rt
, o1
, 32);
3146 TCGv_i128 tmp
= tcg_temp_new_i128();
3148 tcg_gen_qemu_ld_i128(tmp
, clean_addr
, get_mem_index(s
), mop
);
3149 if (s
->be_data
== MO_LE
) {
3150 tcg_gen_extr_i128_i64(tcg_rt
, tcg_rt2
, tmp
);
3152 tcg_gen_extr_i128_i64(tcg_rt2
, tcg_rt
, tmp
);
3155 op_addr_ldstpair_post(s
, a
, dirty_addr
, offset
);
3159 static bool trans_STP_v(DisasContext
*s
, arg_ldstpair
*a
)
3161 uint64_t offset
= a
->imm
<< a
->sz
;
3162 TCGv_i64 clean_addr
, dirty_addr
;
3165 if (!fp_access_check(s
)) {
3169 /* LSE2 does not merge FP pairs; leave these as separate operations. */
3170 mop
= finalize_memop_asimd(s
, a
->sz
);
3171 op_addr_ldstpair_pre(s
, a
, &clean_addr
, &dirty_addr
, offset
, true, mop
);
3172 do_fp_st(s
, a
->rt
, clean_addr
, mop
);
3173 tcg_gen_addi_i64(clean_addr
, clean_addr
, 1 << a
->sz
);
3174 do_fp_st(s
, a
->rt2
, clean_addr
, mop
);
3175 op_addr_ldstpair_post(s
, a
, dirty_addr
, offset
);
3179 static bool trans_LDP_v(DisasContext
*s
, arg_ldstpair
*a
)
3181 uint64_t offset
= a
->imm
<< a
->sz
;
3182 TCGv_i64 clean_addr
, dirty_addr
;
3185 if (!fp_access_check(s
)) {
3189 /* LSE2 does not merge FP pairs; leave these as separate operations. */
3190 mop
= finalize_memop_asimd(s
, a
->sz
);
3191 op_addr_ldstpair_pre(s
, a
, &clean_addr
, &dirty_addr
, offset
, false, mop
);
3192 do_fp_ld(s
, a
->rt
, clean_addr
, mop
);
3193 tcg_gen_addi_i64(clean_addr
, clean_addr
, 1 << a
->sz
);
3194 do_fp_ld(s
, a
->rt2
, clean_addr
, mop
);
3195 op_addr_ldstpair_post(s
, a
, dirty_addr
, offset
);
3199 static bool trans_STGP(DisasContext
*s
, arg_ldstpair
*a
)
3201 TCGv_i64 clean_addr
, dirty_addr
, tcg_rt
, tcg_rt2
;
3202 uint64_t offset
= a
->imm
<< LOG2_TAG_GRANULE
;
3206 /* STGP only comes in one size. */
3207 tcg_debug_assert(a
->sz
== MO_64
);
3209 if (!dc_isar_feature(aa64_mte_insn_reg
, s
)) {
3214 gen_check_sp_alignment(s
);
3217 dirty_addr
= read_cpu_reg_sp(s
, a
->rn
, 1);
3219 tcg_gen_addi_i64(dirty_addr
, dirty_addr
, offset
);
3222 clean_addr
= clean_data_tbi(s
, dirty_addr
);
3223 tcg_rt
= cpu_reg(s
, a
->rt
);
3224 tcg_rt2
= cpu_reg(s
, a
->rt2
);
3227 * STGP is defined as two 8-byte memory operations, aligned to TAG_GRANULE,
3228 * and one tag operation. We implement it as one single aligned 16-byte
3229 * memory operation for convenience. Note that the alignment ensures
3230 * MO_ATOM_IFALIGN_PAIR produces 8-byte atomicity for the memory store.
3232 mop
= finalize_memop_atom(s
, MO_128
| MO_ALIGN
, MO_ATOM_IFALIGN_PAIR
);
3234 tmp
= tcg_temp_new_i128();
3235 if (s
->be_data
== MO_LE
) {
3236 tcg_gen_concat_i64_i128(tmp
, tcg_rt
, tcg_rt2
);
3238 tcg_gen_concat_i64_i128(tmp
, tcg_rt2
, tcg_rt
);
3240 tcg_gen_qemu_st_i128(tmp
, clean_addr
, get_mem_index(s
), mop
);
3242 /* Perform the tag store, if tag access enabled. */
3244 if (tb_cflags(s
->base
.tb
) & CF_PARALLEL
) {
3245 gen_helper_stg_parallel(tcg_env
, dirty_addr
, dirty_addr
);
3247 gen_helper_stg(tcg_env
, dirty_addr
, dirty_addr
);
3251 op_addr_ldstpair_post(s
, a
, dirty_addr
, offset
);
3255 static void op_addr_ldst_imm_pre(DisasContext
*s
, arg_ldst_imm
*a
,
3256 TCGv_i64
*clean_addr
, TCGv_i64
*dirty_addr
,
3257 uint64_t offset
, bool is_store
, MemOp mop
)
3262 gen_check_sp_alignment(s
);
3265 *dirty_addr
= read_cpu_reg_sp(s
, a
->rn
, 1);
3267 tcg_gen_addi_i64(*dirty_addr
, *dirty_addr
, offset
);
3269 memidx
= get_a64_user_mem_index(s
, a
->unpriv
);
3270 *clean_addr
= gen_mte_check1_mmuidx(s
, *dirty_addr
, is_store
,
3271 a
->w
|| a
->rn
!= 31,
3272 mop
, a
->unpriv
, memidx
);
3275 static void op_addr_ldst_imm_post(DisasContext
*s
, arg_ldst_imm
*a
,
3276 TCGv_i64 dirty_addr
, uint64_t offset
)
3280 tcg_gen_addi_i64(dirty_addr
, dirty_addr
, offset
);
3282 tcg_gen_mov_i64(cpu_reg_sp(s
, a
->rn
), dirty_addr
);
3286 static bool trans_STR_i(DisasContext
*s
, arg_ldst_imm
*a
)
3288 bool iss_sf
, iss_valid
= !a
->w
;
3289 TCGv_i64 clean_addr
, dirty_addr
, tcg_rt
;
3290 int memidx
= get_a64_user_mem_index(s
, a
->unpriv
);
3291 MemOp mop
= finalize_memop(s
, a
->sz
+ a
->sign
* MO_SIGN
);
3293 op_addr_ldst_imm_pre(s
, a
, &clean_addr
, &dirty_addr
, a
->imm
, true, mop
);
3295 tcg_rt
= cpu_reg(s
, a
->rt
);
3296 iss_sf
= ldst_iss_sf(a
->sz
, a
->sign
, a
->ext
);
3298 do_gpr_st_memidx(s
, tcg_rt
, clean_addr
, mop
, memidx
,
3299 iss_valid
, a
->rt
, iss_sf
, false);
3300 op_addr_ldst_imm_post(s
, a
, dirty_addr
, a
->imm
);
3304 static bool trans_LDR_i(DisasContext
*s
, arg_ldst_imm
*a
)
3306 bool iss_sf
, iss_valid
= !a
->w
;
3307 TCGv_i64 clean_addr
, dirty_addr
, tcg_rt
;
3308 int memidx
= get_a64_user_mem_index(s
, a
->unpriv
);
3309 MemOp mop
= finalize_memop(s
, a
->sz
+ a
->sign
* MO_SIGN
);
3311 op_addr_ldst_imm_pre(s
, a
, &clean_addr
, &dirty_addr
, a
->imm
, false, mop
);
3313 tcg_rt
= cpu_reg(s
, a
->rt
);
3314 iss_sf
= ldst_iss_sf(a
->sz
, a
->sign
, a
->ext
);
3316 do_gpr_ld_memidx(s
, tcg_rt
, clean_addr
, mop
,
3317 a
->ext
, memidx
, iss_valid
, a
->rt
, iss_sf
, false);
3318 op_addr_ldst_imm_post(s
, a
, dirty_addr
, a
->imm
);
3322 static bool trans_STR_v_i(DisasContext
*s
, arg_ldst_imm
*a
)
3324 TCGv_i64 clean_addr
, dirty_addr
;
3327 if (!fp_access_check(s
)) {
3330 mop
= finalize_memop_asimd(s
, a
->sz
);
3331 op_addr_ldst_imm_pre(s
, a
, &clean_addr
, &dirty_addr
, a
->imm
, true, mop
);
3332 do_fp_st(s
, a
->rt
, clean_addr
, mop
);
3333 op_addr_ldst_imm_post(s
, a
, dirty_addr
, a
->imm
);
3337 static bool trans_LDR_v_i(DisasContext
*s
, arg_ldst_imm
*a
)
3339 TCGv_i64 clean_addr
, dirty_addr
;
3342 if (!fp_access_check(s
)) {
3345 mop
= finalize_memop_asimd(s
, a
->sz
);
3346 op_addr_ldst_imm_pre(s
, a
, &clean_addr
, &dirty_addr
, a
->imm
, false, mop
);
3347 do_fp_ld(s
, a
->rt
, clean_addr
, mop
);
3348 op_addr_ldst_imm_post(s
, a
, dirty_addr
, a
->imm
);
3352 static void op_addr_ldst_pre(DisasContext
*s
, arg_ldst
*a
,
3353 TCGv_i64
*clean_addr
, TCGv_i64
*dirty_addr
,
3354 bool is_store
, MemOp memop
)
3359 gen_check_sp_alignment(s
);
3361 *dirty_addr
= read_cpu_reg_sp(s
, a
->rn
, 1);
3363 tcg_rm
= read_cpu_reg(s
, a
->rm
, 1);
3364 ext_and_shift_reg(tcg_rm
, tcg_rm
, a
->opt
, a
->s
? a
->sz
: 0);
3366 tcg_gen_add_i64(*dirty_addr
, *dirty_addr
, tcg_rm
);
3367 *clean_addr
= gen_mte_check1(s
, *dirty_addr
, is_store
, true, memop
);
3370 static bool trans_LDR(DisasContext
*s
, arg_ldst
*a
)
3372 TCGv_i64 clean_addr
, dirty_addr
, tcg_rt
;
3373 bool iss_sf
= ldst_iss_sf(a
->sz
, a
->sign
, a
->ext
);
3376 if (extract32(a
->opt
, 1, 1) == 0) {
3380 memop
= finalize_memop(s
, a
->sz
+ a
->sign
* MO_SIGN
);
3381 op_addr_ldst_pre(s
, a
, &clean_addr
, &dirty_addr
, false, memop
);
3382 tcg_rt
= cpu_reg(s
, a
->rt
);
3383 do_gpr_ld(s
, tcg_rt
, clean_addr
, memop
,
3384 a
->ext
, true, a
->rt
, iss_sf
, false);
3388 static bool trans_STR(DisasContext
*s
, arg_ldst
*a
)
3390 TCGv_i64 clean_addr
, dirty_addr
, tcg_rt
;
3391 bool iss_sf
= ldst_iss_sf(a
->sz
, a
->sign
, a
->ext
);
3394 if (extract32(a
->opt
, 1, 1) == 0) {
3398 memop
= finalize_memop(s
, a
->sz
);
3399 op_addr_ldst_pre(s
, a
, &clean_addr
, &dirty_addr
, true, memop
);
3400 tcg_rt
= cpu_reg(s
, a
->rt
);
3401 do_gpr_st(s
, tcg_rt
, clean_addr
, memop
, true, a
->rt
, iss_sf
, false);
3405 static bool trans_LDR_v(DisasContext
*s
, arg_ldst
*a
)
3407 TCGv_i64 clean_addr
, dirty_addr
;
3410 if (extract32(a
->opt
, 1, 1) == 0) {
3414 if (!fp_access_check(s
)) {
3418 memop
= finalize_memop_asimd(s
, a
->sz
);
3419 op_addr_ldst_pre(s
, a
, &clean_addr
, &dirty_addr
, false, memop
);
3420 do_fp_ld(s
, a
->rt
, clean_addr
, memop
);
3424 static bool trans_STR_v(DisasContext
*s
, arg_ldst
*a
)
3426 TCGv_i64 clean_addr
, dirty_addr
;
3429 if (extract32(a
->opt
, 1, 1) == 0) {
3433 if (!fp_access_check(s
)) {
3437 memop
= finalize_memop_asimd(s
, a
->sz
);
3438 op_addr_ldst_pre(s
, a
, &clean_addr
, &dirty_addr
, true, memop
);
3439 do_fp_st(s
, a
->rt
, clean_addr
, memop
);
3444 static bool do_atomic_ld(DisasContext
*s
, arg_atomic
*a
, AtomicThreeOpFn
*fn
,
3445 int sign
, bool invert
)
3447 MemOp mop
= a
->sz
| sign
;
3448 TCGv_i64 clean_addr
, tcg_rs
, tcg_rt
;
3451 gen_check_sp_alignment(s
);
3453 mop
= check_atomic_align(s
, a
->rn
, mop
);
3454 clean_addr
= gen_mte_check1(s
, cpu_reg_sp(s
, a
->rn
), false,
3456 tcg_rs
= read_cpu_reg(s
, a
->rs
, true);
3457 tcg_rt
= cpu_reg(s
, a
->rt
);
3459 tcg_gen_not_i64(tcg_rs
, tcg_rs
);
3462 * The tcg atomic primitives are all full barriers. Therefore we
3463 * can ignore the Acquire and Release bits of this instruction.
3465 fn(tcg_rt
, clean_addr
, tcg_rs
, get_mem_index(s
), mop
);
3467 if (mop
& MO_SIGN
) {
3470 tcg_gen_ext8u_i64(tcg_rt
, tcg_rt
);
3473 tcg_gen_ext16u_i64(tcg_rt
, tcg_rt
);
3476 tcg_gen_ext32u_i64(tcg_rt
, tcg_rt
);
3481 g_assert_not_reached();
3487 TRANS_FEAT(LDADD
, aa64_atomics
, do_atomic_ld
, a
, tcg_gen_atomic_fetch_add_i64
, 0, false)
3488 TRANS_FEAT(LDCLR
, aa64_atomics
, do_atomic_ld
, a
, tcg_gen_atomic_fetch_and_i64
, 0, true)
3489 TRANS_FEAT(LDEOR
, aa64_atomics
, do_atomic_ld
, a
, tcg_gen_atomic_fetch_xor_i64
, 0, false)
3490 TRANS_FEAT(LDSET
, aa64_atomics
, do_atomic_ld
, a
, tcg_gen_atomic_fetch_or_i64
, 0, false)
3491 TRANS_FEAT(LDSMAX
, aa64_atomics
, do_atomic_ld
, a
, tcg_gen_atomic_fetch_smax_i64
, MO_SIGN
, false)
3492 TRANS_FEAT(LDSMIN
, aa64_atomics
, do_atomic_ld
, a
, tcg_gen_atomic_fetch_smin_i64
, MO_SIGN
, false)
3493 TRANS_FEAT(LDUMAX
, aa64_atomics
, do_atomic_ld
, a
, tcg_gen_atomic_fetch_umax_i64
, 0, false)
3494 TRANS_FEAT(LDUMIN
, aa64_atomics
, do_atomic_ld
, a
, tcg_gen_atomic_fetch_umin_i64
, 0, false)
3495 TRANS_FEAT(SWP
, aa64_atomics
, do_atomic_ld
, a
, tcg_gen_atomic_xchg_i64
, 0, false)
3497 static bool trans_LDAPR(DisasContext
*s
, arg_LDAPR
*a
)
3499 bool iss_sf
= ldst_iss_sf(a
->sz
, false, false);
3500 TCGv_i64 clean_addr
;
3503 if (!dc_isar_feature(aa64_atomics
, s
) ||
3504 !dc_isar_feature(aa64_rcpc_8_3
, s
)) {
3508 gen_check_sp_alignment(s
);
3510 mop
= check_atomic_align(s
, a
->rn
, a
->sz
);
3511 clean_addr
= gen_mte_check1(s
, cpu_reg_sp(s
, a
->rn
), false,
3514 * LDAPR* are a special case because they are a simple load, not a
3515 * fetch-and-do-something op.
3516 * The architectural consistency requirements here are weaker than
3517 * full load-acquire (we only need "load-acquire processor consistent"),
3518 * but we choose to implement them as full LDAQ.
3520 do_gpr_ld(s
, cpu_reg(s
, a
->rt
), clean_addr
, mop
, false,
3521 true, a
->rt
, iss_sf
, true);
3522 tcg_gen_mb(TCG_MO_ALL
| TCG_BAR_LDAQ
);
3526 static bool trans_LDRA(DisasContext
*s
, arg_LDRA
*a
)
3528 TCGv_i64 clean_addr
, dirty_addr
, tcg_rt
;
3531 /* Load with pointer authentication */
3532 if (!dc_isar_feature(aa64_pauth
, s
)) {
3537 gen_check_sp_alignment(s
);
3539 dirty_addr
= read_cpu_reg_sp(s
, a
->rn
, 1);
3541 if (s
->pauth_active
) {
3543 gen_helper_autda_combined(dirty_addr
, tcg_env
, dirty_addr
,
3544 tcg_constant_i64(0));
3546 gen_helper_autdb_combined(dirty_addr
, tcg_env
, dirty_addr
,
3547 tcg_constant_i64(0));
3551 tcg_gen_addi_i64(dirty_addr
, dirty_addr
, a
->imm
);
3553 memop
= finalize_memop(s
, MO_64
);
3555 /* Note that "clean" and "dirty" here refer to TBI not PAC. */
3556 clean_addr
= gen_mte_check1(s
, dirty_addr
, false,
3557 a
->w
|| a
->rn
!= 31, memop
);
3559 tcg_rt
= cpu_reg(s
, a
->rt
);
3560 do_gpr_ld(s
, tcg_rt
, clean_addr
, memop
,
3561 /* extend */ false, /* iss_valid */ !a
->w
,
3562 /* iss_srt */ a
->rt
, /* iss_sf */ true, /* iss_ar */ false);
3565 tcg_gen_mov_i64(cpu_reg_sp(s
, a
->rn
), dirty_addr
);
3570 static bool trans_LDAPR_i(DisasContext
*s
, arg_ldapr_stlr_i
*a
)
3572 TCGv_i64 clean_addr
, dirty_addr
;
3573 MemOp mop
= a
->sz
| (a
->sign
? MO_SIGN
: 0);
3574 bool iss_sf
= ldst_iss_sf(a
->sz
, a
->sign
, a
->ext
);
3576 if (!dc_isar_feature(aa64_rcpc_8_4
, s
)) {
3581 gen_check_sp_alignment(s
);
3584 mop
= check_ordered_align(s
, a
->rn
, a
->imm
, false, mop
);
3585 dirty_addr
= read_cpu_reg_sp(s
, a
->rn
, 1);
3586 tcg_gen_addi_i64(dirty_addr
, dirty_addr
, a
->imm
);
3587 clean_addr
= clean_data_tbi(s
, dirty_addr
);
3590 * Load-AcquirePC semantics; we implement as the slightly more
3591 * restrictive Load-Acquire.
3593 do_gpr_ld(s
, cpu_reg(s
, a
->rt
), clean_addr
, mop
, a
->ext
, true,
3594 a
->rt
, iss_sf
, true);
3595 tcg_gen_mb(TCG_MO_ALL
| TCG_BAR_LDAQ
);
3599 static bool trans_STLR_i(DisasContext
*s
, arg_ldapr_stlr_i
*a
)
3601 TCGv_i64 clean_addr
, dirty_addr
;
3603 bool iss_sf
= ldst_iss_sf(a
->sz
, a
->sign
, a
->ext
);
3605 if (!dc_isar_feature(aa64_rcpc_8_4
, s
)) {
3609 /* TODO: ARMv8.4-LSE SCTLR.nAA */
3612 gen_check_sp_alignment(s
);
3615 mop
= check_ordered_align(s
, a
->rn
, a
->imm
, true, mop
);
3616 dirty_addr
= read_cpu_reg_sp(s
, a
->rn
, 1);
3617 tcg_gen_addi_i64(dirty_addr
, dirty_addr
, a
->imm
);
3618 clean_addr
= clean_data_tbi(s
, dirty_addr
);
3620 /* Store-Release semantics */
3621 tcg_gen_mb(TCG_MO_ALL
| TCG_BAR_STRL
);
3622 do_gpr_st(s
, cpu_reg(s
, a
->rt
), clean_addr
, mop
, true, a
->rt
, iss_sf
, true);
3626 static bool trans_LD_mult(DisasContext
*s
, arg_ldst_mult
*a
)
3628 TCGv_i64 clean_addr
, tcg_rn
, tcg_ebytes
;
3629 MemOp endian
, align
, mop
;
3631 int total
; /* total bytes */
3632 int elements
; /* elements per vector */
3636 if (!a
->p
&& a
->rm
!= 0) {
3637 /* For non-postindexed accesses the Rm field must be 0 */
3640 if (size
== 3 && !a
->q
&& a
->selem
!= 1) {
3643 if (!fp_access_check(s
)) {
3648 gen_check_sp_alignment(s
);
3651 /* For our purposes, bytes are always little-endian. */
3652 endian
= s
->be_data
;
3657 total
= a
->rpt
* a
->selem
* (a
->q
? 16 : 8);
3658 tcg_rn
= cpu_reg_sp(s
, a
->rn
);
3661 * Issue the MTE check vs the logical repeat count, before we
3662 * promote consecutive little-endian elements below.
3664 clean_addr
= gen_mte_checkN(s
, tcg_rn
, false, a
->p
|| a
->rn
!= 31, total
,
3665 finalize_memop_asimd(s
, size
));
3668 * Consecutive little-endian elements from a single register
3669 * can be promoted to a larger little-endian operation.
3672 if (a
->selem
== 1 && endian
== MO_LE
) {
3673 align
= pow2_align(size
);
3676 if (!s
->align_mem
) {
3679 mop
= endian
| size
| align
;
3681 elements
= (a
->q
? 16 : 8) >> size
;
3682 tcg_ebytes
= tcg_constant_i64(1 << size
);
3683 for (r
= 0; r
< a
->rpt
; r
++) {
3685 for (e
= 0; e
< elements
; e
++) {
3687 for (xs
= 0; xs
< a
->selem
; xs
++) {
3688 int tt
= (a
->rt
+ r
+ xs
) % 32;
3689 do_vec_ld(s
, tt
, e
, clean_addr
, mop
);
3690 tcg_gen_add_i64(clean_addr
, clean_addr
, tcg_ebytes
);
3696 * For non-quad operations, setting a slice of the low 64 bits of
3697 * the register clears the high 64 bits (in the ARM ARM pseudocode
3698 * this is implicit in the fact that 'rval' is a 64 bit wide
3699 * variable). For quad operations, we might still need to zero
3700 * the high bits of SVE.
3702 for (r
= 0; r
< a
->rpt
* a
->selem
; r
++) {
3703 int tt
= (a
->rt
+ r
) % 32;
3704 clear_vec_high(s
, a
->q
, tt
);
3709 tcg_gen_addi_i64(tcg_rn
, tcg_rn
, total
);
3711 tcg_gen_add_i64(tcg_rn
, tcg_rn
, cpu_reg(s
, a
->rm
));
3717 static bool trans_ST_mult(DisasContext
*s
, arg_ldst_mult
*a
)
3719 TCGv_i64 clean_addr
, tcg_rn
, tcg_ebytes
;
3720 MemOp endian
, align
, mop
;
3722 int total
; /* total bytes */
3723 int elements
; /* elements per vector */
3727 if (!a
->p
&& a
->rm
!= 0) {
3728 /* For non-postindexed accesses the Rm field must be 0 */
3731 if (size
== 3 && !a
->q
&& a
->selem
!= 1) {
3734 if (!fp_access_check(s
)) {
3739 gen_check_sp_alignment(s
);
3742 /* For our purposes, bytes are always little-endian. */
3743 endian
= s
->be_data
;
3748 total
= a
->rpt
* a
->selem
* (a
->q
? 16 : 8);
3749 tcg_rn
= cpu_reg_sp(s
, a
->rn
);
3752 * Issue the MTE check vs the logical repeat count, before we
3753 * promote consecutive little-endian elements below.
3755 clean_addr
= gen_mte_checkN(s
, tcg_rn
, true, a
->p
|| a
->rn
!= 31, total
,
3756 finalize_memop_asimd(s
, size
));
3759 * Consecutive little-endian elements from a single register
3760 * can be promoted to a larger little-endian operation.
3763 if (a
->selem
== 1 && endian
== MO_LE
) {
3764 align
= pow2_align(size
);
3767 if (!s
->align_mem
) {
3770 mop
= endian
| size
| align
;
3772 elements
= (a
->q
? 16 : 8) >> size
;
3773 tcg_ebytes
= tcg_constant_i64(1 << size
);
3774 for (r
= 0; r
< a
->rpt
; r
++) {
3776 for (e
= 0; e
< elements
; e
++) {
3778 for (xs
= 0; xs
< a
->selem
; xs
++) {
3779 int tt
= (a
->rt
+ r
+ xs
) % 32;
3780 do_vec_st(s
, tt
, e
, clean_addr
, mop
);
3781 tcg_gen_add_i64(clean_addr
, clean_addr
, tcg_ebytes
);
3788 tcg_gen_addi_i64(tcg_rn
, tcg_rn
, total
);
3790 tcg_gen_add_i64(tcg_rn
, tcg_rn
, cpu_reg(s
, a
->rm
));
3796 static bool trans_ST_single(DisasContext
*s
, arg_ldst_single
*a
)
3799 TCGv_i64 clean_addr
, tcg_rn
, tcg_ebytes
;
3802 if (!a
->p
&& a
->rm
!= 0) {
3805 if (!fp_access_check(s
)) {
3810 gen_check_sp_alignment(s
);
3813 total
= a
->selem
<< a
->scale
;
3814 tcg_rn
= cpu_reg_sp(s
, a
->rn
);
3816 mop
= finalize_memop_asimd(s
, a
->scale
);
3817 clean_addr
= gen_mte_checkN(s
, tcg_rn
, true, a
->p
|| a
->rn
!= 31,
3820 tcg_ebytes
= tcg_constant_i64(1 << a
->scale
);
3821 for (xs
= 0, rt
= a
->rt
; xs
< a
->selem
; xs
++, rt
= (rt
+ 1) % 32) {
3822 do_vec_st(s
, rt
, a
->index
, clean_addr
, mop
);
3823 tcg_gen_add_i64(clean_addr
, clean_addr
, tcg_ebytes
);
3828 tcg_gen_addi_i64(tcg_rn
, tcg_rn
, total
);
3830 tcg_gen_add_i64(tcg_rn
, tcg_rn
, cpu_reg(s
, a
->rm
));
3836 static bool trans_LD_single(DisasContext
*s
, arg_ldst_single
*a
)
3839 TCGv_i64 clean_addr
, tcg_rn
, tcg_ebytes
;
3842 if (!a
->p
&& a
->rm
!= 0) {
3845 if (!fp_access_check(s
)) {
3850 gen_check_sp_alignment(s
);
3853 total
= a
->selem
<< a
->scale
;
3854 tcg_rn
= cpu_reg_sp(s
, a
->rn
);
3856 mop
= finalize_memop_asimd(s
, a
->scale
);
3857 clean_addr
= gen_mte_checkN(s
, tcg_rn
, false, a
->p
|| a
->rn
!= 31,
3860 tcg_ebytes
= tcg_constant_i64(1 << a
->scale
);
3861 for (xs
= 0, rt
= a
->rt
; xs
< a
->selem
; xs
++, rt
= (rt
+ 1) % 32) {
3862 do_vec_ld(s
, rt
, a
->index
, clean_addr
, mop
);
3863 tcg_gen_add_i64(clean_addr
, clean_addr
, tcg_ebytes
);
3868 tcg_gen_addi_i64(tcg_rn
, tcg_rn
, total
);
3870 tcg_gen_add_i64(tcg_rn
, tcg_rn
, cpu_reg(s
, a
->rm
));
3876 static bool trans_LD_single_repl(DisasContext
*s
, arg_LD_single_repl
*a
)
3879 TCGv_i64 clean_addr
, tcg_rn
, tcg_ebytes
;
3882 if (!a
->p
&& a
->rm
!= 0) {
3885 if (!fp_access_check(s
)) {
3890 gen_check_sp_alignment(s
);
3893 total
= a
->selem
<< a
->scale
;
3894 tcg_rn
= cpu_reg_sp(s
, a
->rn
);
3896 mop
= finalize_memop_asimd(s
, a
->scale
);
3897 clean_addr
= gen_mte_checkN(s
, tcg_rn
, false, a
->p
|| a
->rn
!= 31,
3900 tcg_ebytes
= tcg_constant_i64(1 << a
->scale
);
3901 for (xs
= 0, rt
= a
->rt
; xs
< a
->selem
; xs
++, rt
= (rt
+ 1) % 32) {
3902 /* Load and replicate to all elements */
3903 TCGv_i64 tcg_tmp
= tcg_temp_new_i64();
3905 tcg_gen_qemu_ld_i64(tcg_tmp
, clean_addr
, get_mem_index(s
), mop
);
3906 tcg_gen_gvec_dup_i64(a
->scale
, vec_full_reg_offset(s
, rt
),
3907 (a
->q
+ 1) * 8, vec_full_reg_size(s
), tcg_tmp
);
3908 tcg_gen_add_i64(clean_addr
, clean_addr
, tcg_ebytes
);
3913 tcg_gen_addi_i64(tcg_rn
, tcg_rn
, total
);
3915 tcg_gen_add_i64(tcg_rn
, tcg_rn
, cpu_reg(s
, a
->rm
));
3921 static bool trans_STZGM(DisasContext
*s
, arg_ldst_tag
*a
)
3923 TCGv_i64 addr
, clean_addr
, tcg_rt
;
3924 int size
= 4 << s
->dcz_blocksize
;
3926 if (!dc_isar_feature(aa64_mte
, s
)) {
3929 if (s
->current_el
== 0) {
3934 gen_check_sp_alignment(s
);
3937 addr
= read_cpu_reg_sp(s
, a
->rn
, true);
3938 tcg_gen_addi_i64(addr
, addr
, a
->imm
);
3939 tcg_rt
= cpu_reg(s
, a
->rt
);
3942 gen_helper_stzgm_tags(tcg_env
, addr
, tcg_rt
);
3945 * The non-tags portion of STZGM is mostly like DC_ZVA,
3946 * except the alignment happens before the access.
3948 clean_addr
= clean_data_tbi(s
, addr
);
3949 tcg_gen_andi_i64(clean_addr
, clean_addr
, -size
);
3950 gen_helper_dc_zva(tcg_env
, clean_addr
);
3954 static bool trans_STGM(DisasContext
*s
, arg_ldst_tag
*a
)
3956 TCGv_i64 addr
, clean_addr
, tcg_rt
;
3958 if (!dc_isar_feature(aa64_mte
, s
)) {
3961 if (s
->current_el
== 0) {
3966 gen_check_sp_alignment(s
);
3969 addr
= read_cpu_reg_sp(s
, a
->rn
, true);
3970 tcg_gen_addi_i64(addr
, addr
, a
->imm
);
3971 tcg_rt
= cpu_reg(s
, a
->rt
);
3974 gen_helper_stgm(tcg_env
, addr
, tcg_rt
);
3976 MMUAccessType acc
= MMU_DATA_STORE
;
3977 int size
= 4 << s
->gm_blocksize
;
3979 clean_addr
= clean_data_tbi(s
, addr
);
3980 tcg_gen_andi_i64(clean_addr
, clean_addr
, -size
);
3981 gen_probe_access(s
, clean_addr
, acc
, size
);
3986 static bool trans_LDGM(DisasContext
*s
, arg_ldst_tag
*a
)
3988 TCGv_i64 addr
, clean_addr
, tcg_rt
;
3990 if (!dc_isar_feature(aa64_mte
, s
)) {
3993 if (s
->current_el
== 0) {
3998 gen_check_sp_alignment(s
);
4001 addr
= read_cpu_reg_sp(s
, a
->rn
, true);
4002 tcg_gen_addi_i64(addr
, addr
, a
->imm
);
4003 tcg_rt
= cpu_reg(s
, a
->rt
);
4006 gen_helper_ldgm(tcg_rt
, tcg_env
, addr
);
4008 MMUAccessType acc
= MMU_DATA_LOAD
;
4009 int size
= 4 << s
->gm_blocksize
;
4011 clean_addr
= clean_data_tbi(s
, addr
);
4012 tcg_gen_andi_i64(clean_addr
, clean_addr
, -size
);
4013 gen_probe_access(s
, clean_addr
, acc
, size
);
4014 /* The result tags are zeros. */
4015 tcg_gen_movi_i64(tcg_rt
, 0);
4020 static bool trans_LDG(DisasContext
*s
, arg_ldst_tag
*a
)
4022 TCGv_i64 addr
, clean_addr
, tcg_rt
;
4024 if (!dc_isar_feature(aa64_mte_insn_reg
, s
)) {
4029 gen_check_sp_alignment(s
);
4032 addr
= read_cpu_reg_sp(s
, a
->rn
, true);
4034 /* pre-index or signed offset */
4035 tcg_gen_addi_i64(addr
, addr
, a
->imm
);
4038 tcg_gen_andi_i64(addr
, addr
, -TAG_GRANULE
);
4039 tcg_rt
= cpu_reg(s
, a
->rt
);
4041 gen_helper_ldg(tcg_rt
, tcg_env
, addr
, tcg_rt
);
4044 * Tag access disabled: we must check for aborts on the load
4045 * load from [rn+offset], and then insert a 0 tag into rt.
4047 clean_addr
= clean_data_tbi(s
, addr
);
4048 gen_probe_access(s
, clean_addr
, MMU_DATA_LOAD
, MO_8
);
4049 gen_address_with_allocation_tag0(tcg_rt
, tcg_rt
);
4053 /* pre-index or post-index */
4056 tcg_gen_addi_i64(addr
, addr
, a
->imm
);
4058 tcg_gen_mov_i64(cpu_reg_sp(s
, a
->rn
), addr
);
4063 static bool do_STG(DisasContext
*s
, arg_ldst_tag
*a
, bool is_zero
, bool is_pair
)
4065 TCGv_i64 addr
, tcg_rt
;
4068 gen_check_sp_alignment(s
);
4071 addr
= read_cpu_reg_sp(s
, a
->rn
, true);
4073 /* pre-index or signed offset */
4074 tcg_gen_addi_i64(addr
, addr
, a
->imm
);
4076 tcg_rt
= cpu_reg_sp(s
, a
->rt
);
4079 * For STG and ST2G, we need to check alignment and probe memory.
4080 * TODO: For STZG and STZ2G, we could rely on the stores below,
4081 * at least for system mode; user-only won't enforce alignment.
4084 gen_helper_st2g_stub(tcg_env
, addr
);
4086 gen_helper_stg_stub(tcg_env
, addr
);
4088 } else if (tb_cflags(s
->base
.tb
) & CF_PARALLEL
) {
4090 gen_helper_st2g_parallel(tcg_env
, addr
, tcg_rt
);
4092 gen_helper_stg_parallel(tcg_env
, addr
, tcg_rt
);
4096 gen_helper_st2g(tcg_env
, addr
, tcg_rt
);
4098 gen_helper_stg(tcg_env
, addr
, tcg_rt
);
4103 TCGv_i64 clean_addr
= clean_data_tbi(s
, addr
);
4104 TCGv_i64 zero64
= tcg_constant_i64(0);
4105 TCGv_i128 zero128
= tcg_temp_new_i128();
4106 int mem_index
= get_mem_index(s
);
4107 MemOp mop
= finalize_memop(s
, MO_128
| MO_ALIGN
);
4109 tcg_gen_concat_i64_i128(zero128
, zero64
, zero64
);
4111 /* This is 1 or 2 atomic 16-byte operations. */
4112 tcg_gen_qemu_st_i128(zero128
, clean_addr
, mem_index
, mop
);
4114 tcg_gen_addi_i64(clean_addr
, clean_addr
, 16);
4115 tcg_gen_qemu_st_i128(zero128
, clean_addr
, mem_index
, mop
);
4120 /* pre-index or post-index */
4123 tcg_gen_addi_i64(addr
, addr
, a
->imm
);
4125 tcg_gen_mov_i64(cpu_reg_sp(s
, a
->rn
), addr
);
4130 TRANS_FEAT(STG
, aa64_mte_insn_reg
, do_STG
, a
, false, false)
4131 TRANS_FEAT(STZG
, aa64_mte_insn_reg
, do_STG
, a
, true, false)
4132 TRANS_FEAT(ST2G
, aa64_mte_insn_reg
, do_STG
, a
, false, true)
4133 TRANS_FEAT(STZ2G
, aa64_mte_insn_reg
, do_STG
, a
, true, true)
4135 typedef void SetFn(TCGv_env
, TCGv_i32
, TCGv_i32
);
4137 static bool do_SET(DisasContext
*s
, arg_set
*a
, bool is_epilogue
,
4138 bool is_setg
, SetFn fn
)
4141 uint32_t syndrome
, desc
= 0;
4143 if (is_setg
&& !dc_isar_feature(aa64_mte
, s
)) {
4148 * UNPREDICTABLE cases: we choose to UNDEF, which allows
4149 * us to pull this check before the CheckMOPSEnabled() test
4150 * (which we do in the helper function)
4152 if (a
->rs
== a
->rn
|| a
->rs
== a
->rd
|| a
->rn
== a
->rd
||
4153 a
->rd
== 31 || a
->rn
== 31) {
4157 memidx
= get_a64_user_mem_index(s
, a
->unpriv
);
4160 * We pass option_a == true, matching our implementation;
4161 * we pass wrong_option == false: helper function may set that bit.
4163 syndrome
= syn_mop(true, is_setg
, (a
->nontemp
<< 1) | a
->unpriv
,
4164 is_epilogue
, false, true, a
->rd
, a
->rs
, a
->rn
);
4166 if (is_setg
? s
->ata
[a
->unpriv
] : s
->mte_active
[a
->unpriv
]) {
4167 /* We may need to do MTE tag checking, so assemble the descriptor */
4168 desc
= FIELD_DP32(desc
, MTEDESC
, TBI
, s
->tbid
);
4169 desc
= FIELD_DP32(desc
, MTEDESC
, TCMA
, s
->tcma
);
4170 desc
= FIELD_DP32(desc
, MTEDESC
, WRITE
, true);
4171 /* SIZEM1 and ALIGN we leave 0 (byte write) */
4173 /* The helper function always needs the memidx even with MTE disabled */
4174 desc
= FIELD_DP32(desc
, MTEDESC
, MIDX
, memidx
);
4177 * The helper needs the register numbers, but since they're in
4178 * the syndrome anyway, we let it extract them from there rather
4179 * than passing in an extra three integer arguments.
4181 fn(tcg_env
, tcg_constant_i32(syndrome
), tcg_constant_i32(desc
));
4185 TRANS_FEAT(SETP
, aa64_mops
, do_SET
, a
, false, false, gen_helper_setp
)
4186 TRANS_FEAT(SETM
, aa64_mops
, do_SET
, a
, false, false, gen_helper_setm
)
4187 TRANS_FEAT(SETE
, aa64_mops
, do_SET
, a
, true, false, gen_helper_sete
)
4188 TRANS_FEAT(SETGP
, aa64_mops
, do_SET
, a
, false, true, gen_helper_setgp
)
4189 TRANS_FEAT(SETGM
, aa64_mops
, do_SET
, a
, false, true, gen_helper_setgm
)
4190 TRANS_FEAT(SETGE
, aa64_mops
, do_SET
, a
, true, true, gen_helper_setge
)
4192 typedef void CpyFn(TCGv_env
, TCGv_i32
, TCGv_i32
, TCGv_i32
);
4194 static bool do_CPY(DisasContext
*s
, arg_cpy
*a
, bool is_epilogue
, CpyFn fn
)
4196 int rmemidx
, wmemidx
;
4197 uint32_t syndrome
, rdesc
= 0, wdesc
= 0;
4198 bool wunpriv
= extract32(a
->options
, 0, 1);
4199 bool runpriv
= extract32(a
->options
, 1, 1);
4202 * UNPREDICTABLE cases: we choose to UNDEF, which allows
4203 * us to pull this check before the CheckMOPSEnabled() test
4204 * (which we do in the helper function)
4206 if (a
->rs
== a
->rn
|| a
->rs
== a
->rd
|| a
->rn
== a
->rd
||
4207 a
->rd
== 31 || a
->rs
== 31 || a
->rn
== 31) {
4211 rmemidx
= get_a64_user_mem_index(s
, runpriv
);
4212 wmemidx
= get_a64_user_mem_index(s
, wunpriv
);
4215 * We pass option_a == true, matching our implementation;
4216 * we pass wrong_option == false: helper function may set that bit.
4218 syndrome
= syn_mop(false, false, a
->options
, is_epilogue
,
4219 false, true, a
->rd
, a
->rs
, a
->rn
);
4221 /* If we need to do MTE tag checking, assemble the descriptors */
4222 if (s
->mte_active
[runpriv
]) {
4223 rdesc
= FIELD_DP32(rdesc
, MTEDESC
, TBI
, s
->tbid
);
4224 rdesc
= FIELD_DP32(rdesc
, MTEDESC
, TCMA
, s
->tcma
);
4226 if (s
->mte_active
[wunpriv
]) {
4227 wdesc
= FIELD_DP32(wdesc
, MTEDESC
, TBI
, s
->tbid
);
4228 wdesc
= FIELD_DP32(wdesc
, MTEDESC
, TCMA
, s
->tcma
);
4229 wdesc
= FIELD_DP32(wdesc
, MTEDESC
, WRITE
, true);
4231 /* The helper function needs these parts of the descriptor regardless */
4232 rdesc
= FIELD_DP32(rdesc
, MTEDESC
, MIDX
, rmemidx
);
4233 wdesc
= FIELD_DP32(wdesc
, MTEDESC
, MIDX
, wmemidx
);
4236 * The helper needs the register numbers, but since they're in
4237 * the syndrome anyway, we let it extract them from there rather
4238 * than passing in an extra three integer arguments.
4240 fn(tcg_env
, tcg_constant_i32(syndrome
), tcg_constant_i32(wdesc
),
4241 tcg_constant_i32(rdesc
));
4245 TRANS_FEAT(CPYP
, aa64_mops
, do_CPY
, a
, false, gen_helper_cpyp
)
4246 TRANS_FEAT(CPYM
, aa64_mops
, do_CPY
, a
, false, gen_helper_cpym
)
4247 TRANS_FEAT(CPYE
, aa64_mops
, do_CPY
, a
, true, gen_helper_cpye
)
4248 TRANS_FEAT(CPYFP
, aa64_mops
, do_CPY
, a
, false, gen_helper_cpyfp
)
4249 TRANS_FEAT(CPYFM
, aa64_mops
, do_CPY
, a
, false, gen_helper_cpyfm
)
4250 TRANS_FEAT(CPYFE
, aa64_mops
, do_CPY
, a
, true, gen_helper_cpyfe
)
4252 typedef void ArithTwoOp(TCGv_i64
, TCGv_i64
, TCGv_i64
);
4254 static bool gen_rri(DisasContext
*s
, arg_rri_sf
*a
,
4255 bool rd_sp
, bool rn_sp
, ArithTwoOp
*fn
)
4257 TCGv_i64 tcg_rn
= rn_sp
? cpu_reg_sp(s
, a
->rn
) : cpu_reg(s
, a
->rn
);
4258 TCGv_i64 tcg_rd
= rd_sp
? cpu_reg_sp(s
, a
->rd
) : cpu_reg(s
, a
->rd
);
4259 TCGv_i64 tcg_imm
= tcg_constant_i64(a
->imm
);
4261 fn(tcg_rd
, tcg_rn
, tcg_imm
);
4263 tcg_gen_ext32u_i64(tcg_rd
, tcg_rd
);
4269 * PC-rel. addressing
4272 static bool trans_ADR(DisasContext
*s
, arg_ri
*a
)
4274 gen_pc_plus_diff(s
, cpu_reg(s
, a
->rd
), a
->imm
);
4278 static bool trans_ADRP(DisasContext
*s
, arg_ri
*a
)
4280 int64_t offset
= (int64_t)a
->imm
<< 12;
4282 /* The page offset is ok for CF_PCREL. */
4283 offset
-= s
->pc_curr
& 0xfff;
4284 gen_pc_plus_diff(s
, cpu_reg(s
, a
->rd
), offset
);
4289 * Add/subtract (immediate)
4291 TRANS(ADD_i
, gen_rri
, a
, 1, 1, tcg_gen_add_i64
)
4292 TRANS(SUB_i
, gen_rri
, a
, 1, 1, tcg_gen_sub_i64
)
4293 TRANS(ADDS_i
, gen_rri
, a
, 0, 1, a
->sf
? gen_add64_CC
: gen_add32_CC
)
4294 TRANS(SUBS_i
, gen_rri
, a
, 0, 1, a
->sf
? gen_sub64_CC
: gen_sub32_CC
)
4297 * Add/subtract (immediate, with tags)
4300 static bool gen_add_sub_imm_with_tags(DisasContext
*s
, arg_rri_tag
*a
,
4303 TCGv_i64 tcg_rn
, tcg_rd
;
4306 imm
= a
->uimm6
<< LOG2_TAG_GRANULE
;
4311 tcg_rn
= cpu_reg_sp(s
, a
->rn
);
4312 tcg_rd
= cpu_reg_sp(s
, a
->rd
);
4315 gen_helper_addsubg(tcg_rd
, tcg_env
, tcg_rn
,
4316 tcg_constant_i32(imm
),
4317 tcg_constant_i32(a
->uimm4
));
4319 tcg_gen_addi_i64(tcg_rd
, tcg_rn
, imm
);
4320 gen_address_with_allocation_tag0(tcg_rd
, tcg_rd
);
4325 TRANS_FEAT(ADDG_i
, aa64_mte_insn_reg
, gen_add_sub_imm_with_tags
, a
, false)
4326 TRANS_FEAT(SUBG_i
, aa64_mte_insn_reg
, gen_add_sub_imm_with_tags
, a
, true)
4328 /* The input should be a value in the bottom e bits (with higher
4329 * bits zero); returns that value replicated into every element
4330 * of size e in a 64 bit integer.
4332 static uint64_t bitfield_replicate(uint64_t mask
, unsigned int e
)
4343 * Logical (immediate)
4347 * Simplified variant of pseudocode DecodeBitMasks() for the case where we
4348 * only require the wmask. Returns false if the imms/immr/immn are a reserved
4349 * value (ie should cause a guest UNDEF exception), and true if they are
4350 * valid, in which case the decoded bit pattern is written to result.
4352 bool logic_imm_decode_wmask(uint64_t *result
, unsigned int immn
,
4353 unsigned int imms
, unsigned int immr
)
4356 unsigned e
, levels
, s
, r
;
4359 assert(immn
< 2 && imms
< 64 && immr
< 64);
4361 /* The bit patterns we create here are 64 bit patterns which
4362 * are vectors of identical elements of size e = 2, 4, 8, 16, 32 or
4363 * 64 bits each. Each element contains the same value: a run
4364 * of between 1 and e-1 non-zero bits, rotated within the
4365 * element by between 0 and e-1 bits.
4367 * The element size and run length are encoded into immn (1 bit)
4368 * and imms (6 bits) as follows:
4369 * 64 bit elements: immn = 1, imms = <length of run - 1>
4370 * 32 bit elements: immn = 0, imms = 0 : <length of run - 1>
4371 * 16 bit elements: immn = 0, imms = 10 : <length of run - 1>
4372 * 8 bit elements: immn = 0, imms = 110 : <length of run - 1>
4373 * 4 bit elements: immn = 0, imms = 1110 : <length of run - 1>
4374 * 2 bit elements: immn = 0, imms = 11110 : <length of run - 1>
4375 * Notice that immn = 0, imms = 11111x is the only combination
4376 * not covered by one of the above options; this is reserved.
4377 * Further, <length of run - 1> all-ones is a reserved pattern.
4379 * In all cases the rotation is by immr % e (and immr is 6 bits).
4382 /* First determine the element size */
4383 len
= 31 - clz32((immn
<< 6) | (~imms
& 0x3f));
4385 /* This is the immn == 0, imms == 0x11111x case */
4395 /* <length of run - 1> mustn't be all-ones. */
4399 /* Create the value of one element: s+1 set bits rotated
4400 * by r within the element (which is e bits wide)...
4402 mask
= MAKE_64BIT_MASK(0, s
+ 1);
4404 mask
= (mask
>> r
) | (mask
<< (e
- r
));
4405 mask
&= MAKE_64BIT_MASK(0, e
);
4407 /* ...then replicate the element over the whole 64 bit value */
4408 mask
= bitfield_replicate(mask
, e
);
4413 static bool gen_rri_log(DisasContext
*s
, arg_rri_log
*a
, bool set_cc
,
4414 void (*fn
)(TCGv_i64
, TCGv_i64
, int64_t))
4416 TCGv_i64 tcg_rd
, tcg_rn
;
4419 /* Some immediate field values are reserved. */
4420 if (!logic_imm_decode_wmask(&imm
, extract32(a
->dbm
, 12, 1),
4421 extract32(a
->dbm
, 0, 6),
4422 extract32(a
->dbm
, 6, 6))) {
4426 imm
&= 0xffffffffull
;
4429 tcg_rd
= set_cc
? cpu_reg(s
, a
->rd
) : cpu_reg_sp(s
, a
->rd
);
4430 tcg_rn
= cpu_reg(s
, a
->rn
);
4432 fn(tcg_rd
, tcg_rn
, imm
);
4434 gen_logic_CC(a
->sf
, tcg_rd
);
4437 tcg_gen_ext32u_i64(tcg_rd
, tcg_rd
);
4442 TRANS(AND_i
, gen_rri_log
, a
, false, tcg_gen_andi_i64
)
4443 TRANS(ORR_i
, gen_rri_log
, a
, false, tcg_gen_ori_i64
)
4444 TRANS(EOR_i
, gen_rri_log
, a
, false, tcg_gen_xori_i64
)
4445 TRANS(ANDS_i
, gen_rri_log
, a
, true, tcg_gen_andi_i64
)
4448 * Move wide (immediate)
4451 static bool trans_MOVZ(DisasContext
*s
, arg_movw
*a
)
4453 int pos
= a
->hw
<< 4;
4454 tcg_gen_movi_i64(cpu_reg(s
, a
->rd
), (uint64_t)a
->imm
<< pos
);
4458 static bool trans_MOVN(DisasContext
*s
, arg_movw
*a
)
4460 int pos
= a
->hw
<< 4;
4461 uint64_t imm
= a
->imm
;
4463 imm
= ~(imm
<< pos
);
4465 imm
= (uint32_t)imm
;
4467 tcg_gen_movi_i64(cpu_reg(s
, a
->rd
), imm
);
4471 static bool trans_MOVK(DisasContext
*s
, arg_movw
*a
)
4473 int pos
= a
->hw
<< 4;
4474 TCGv_i64 tcg_rd
, tcg_im
;
4476 tcg_rd
= cpu_reg(s
, a
->rd
);
4477 tcg_im
= tcg_constant_i64(a
->imm
);
4478 tcg_gen_deposit_i64(tcg_rd
, tcg_rd
, tcg_im
, pos
, 16);
4480 tcg_gen_ext32u_i64(tcg_rd
, tcg_rd
);
4489 static bool trans_SBFM(DisasContext
*s
, arg_SBFM
*a
)
4491 TCGv_i64 tcg_rd
= cpu_reg(s
, a
->rd
);
4492 TCGv_i64 tcg_tmp
= read_cpu_reg(s
, a
->rn
, 1);
4493 unsigned int bitsize
= a
->sf
? 64 : 32;
4494 unsigned int ri
= a
->immr
;
4495 unsigned int si
= a
->imms
;
4496 unsigned int pos
, len
;
4499 /* Wd<s-r:0> = Wn<s:r> */
4500 len
= (si
- ri
) + 1;
4501 tcg_gen_sextract_i64(tcg_rd
, tcg_tmp
, ri
, len
);
4503 tcg_gen_ext32u_i64(tcg_rd
, tcg_rd
);
4506 /* Wd<32+s-r,32-r> = Wn<s:0> */
4508 pos
= (bitsize
- ri
) & (bitsize
- 1);
4512 * Sign extend the destination field from len to fill the
4513 * balance of the word. Let the deposit below insert all
4514 * of those sign bits.
4516 tcg_gen_sextract_i64(tcg_tmp
, tcg_tmp
, 0, len
);
4521 * We start with zero, and we haven't modified any bits outside
4522 * bitsize, therefore no final zero-extension is unneeded for !sf.
4524 tcg_gen_deposit_z_i64(tcg_rd
, tcg_tmp
, pos
, len
);
4529 static bool trans_UBFM(DisasContext
*s
, arg_UBFM
*a
)
4531 TCGv_i64 tcg_rd
= cpu_reg(s
, a
->rd
);
4532 TCGv_i64 tcg_tmp
= read_cpu_reg(s
, a
->rn
, 1);
4533 unsigned int bitsize
= a
->sf
? 64 : 32;
4534 unsigned int ri
= a
->immr
;
4535 unsigned int si
= a
->imms
;
4536 unsigned int pos
, len
;
4538 tcg_rd
= cpu_reg(s
, a
->rd
);
4539 tcg_tmp
= read_cpu_reg(s
, a
->rn
, 1);
4542 /* Wd<s-r:0> = Wn<s:r> */
4543 len
= (si
- ri
) + 1;
4544 tcg_gen_extract_i64(tcg_rd
, tcg_tmp
, ri
, len
);
4546 /* Wd<32+s-r,32-r> = Wn<s:0> */
4548 pos
= (bitsize
- ri
) & (bitsize
- 1);
4549 tcg_gen_deposit_z_i64(tcg_rd
, tcg_tmp
, pos
, len
);
4554 static bool trans_BFM(DisasContext
*s
, arg_BFM
*a
)
4556 TCGv_i64 tcg_rd
= cpu_reg(s
, a
->rd
);
4557 TCGv_i64 tcg_tmp
= read_cpu_reg(s
, a
->rn
, 1);
4558 unsigned int bitsize
= a
->sf
? 64 : 32;
4559 unsigned int ri
= a
->immr
;
4560 unsigned int si
= a
->imms
;
4561 unsigned int pos
, len
;
4563 tcg_rd
= cpu_reg(s
, a
->rd
);
4564 tcg_tmp
= read_cpu_reg(s
, a
->rn
, 1);
4567 /* Wd<s-r:0> = Wn<s:r> */
4568 tcg_gen_shri_i64(tcg_tmp
, tcg_tmp
, ri
);
4569 len
= (si
- ri
) + 1;
4572 /* Wd<32+s-r,32-r> = Wn<s:0> */
4574 pos
= (bitsize
- ri
) & (bitsize
- 1);
4577 tcg_gen_deposit_i64(tcg_rd
, tcg_rd
, tcg_tmp
, pos
, len
);
4579 tcg_gen_ext32u_i64(tcg_rd
, tcg_rd
);
4584 static bool trans_EXTR(DisasContext
*s
, arg_extract
*a
)
4586 TCGv_i64 tcg_rd
, tcg_rm
, tcg_rn
;
4588 tcg_rd
= cpu_reg(s
, a
->rd
);
4590 if (unlikely(a
->imm
== 0)) {
4592 * tcg shl_i32/shl_i64 is undefined for 32/64 bit shifts,
4593 * so an extract from bit 0 is a special case.
4596 tcg_gen_mov_i64(tcg_rd
, cpu_reg(s
, a
->rm
));
4598 tcg_gen_ext32u_i64(tcg_rd
, cpu_reg(s
, a
->rm
));
4601 tcg_rm
= cpu_reg(s
, a
->rm
);
4602 tcg_rn
= cpu_reg(s
, a
->rn
);
4605 /* Specialization to ROR happens in EXTRACT2. */
4606 tcg_gen_extract2_i64(tcg_rd
, tcg_rm
, tcg_rn
, a
->imm
);
4608 TCGv_i32 t0
= tcg_temp_new_i32();
4610 tcg_gen_extrl_i64_i32(t0
, tcg_rm
);
4611 if (a
->rm
== a
->rn
) {
4612 tcg_gen_rotri_i32(t0
, t0
, a
->imm
);
4614 TCGv_i32 t1
= tcg_temp_new_i32();
4615 tcg_gen_extrl_i64_i32(t1
, tcg_rn
);
4616 tcg_gen_extract2_i32(t0
, t0
, t1
, a
->imm
);
4618 tcg_gen_extu_i32_i64(tcg_rd
, t0
);
4625 * Cryptographic AES, SHA, SHA512
4628 TRANS_FEAT(AESE
, aa64_aes
, do_gvec_op3_ool
, a
, 0, gen_helper_crypto_aese
)
4629 TRANS_FEAT(AESD
, aa64_aes
, do_gvec_op3_ool
, a
, 0, gen_helper_crypto_aesd
)
4630 TRANS_FEAT(AESMC
, aa64_aes
, do_gvec_op2_ool
, a
, 0, gen_helper_crypto_aesmc
)
4631 TRANS_FEAT(AESIMC
, aa64_aes
, do_gvec_op2_ool
, a
, 0, gen_helper_crypto_aesimc
)
4633 TRANS_FEAT(SHA1C
, aa64_sha1
, do_gvec_op3_ool
, a
, 0, gen_helper_crypto_sha1c
)
4634 TRANS_FEAT(SHA1P
, aa64_sha1
, do_gvec_op3_ool
, a
, 0, gen_helper_crypto_sha1p
)
4635 TRANS_FEAT(SHA1M
, aa64_sha1
, do_gvec_op3_ool
, a
, 0, gen_helper_crypto_sha1m
)
4636 TRANS_FEAT(SHA1SU0
, aa64_sha1
, do_gvec_op3_ool
, a
, 0, gen_helper_crypto_sha1su0
)
4638 TRANS_FEAT(SHA256H
, aa64_sha256
, do_gvec_op3_ool
, a
, 0, gen_helper_crypto_sha256h
)
4639 TRANS_FEAT(SHA256H2
, aa64_sha256
, do_gvec_op3_ool
, a
, 0, gen_helper_crypto_sha256h2
)
4640 TRANS_FEAT(SHA256SU1
, aa64_sha256
, do_gvec_op3_ool
, a
, 0, gen_helper_crypto_sha256su1
)
4642 TRANS_FEAT(SHA1H
, aa64_sha1
, do_gvec_op2_ool
, a
, 0, gen_helper_crypto_sha1h
)
4643 TRANS_FEAT(SHA1SU1
, aa64_sha1
, do_gvec_op2_ool
, a
, 0, gen_helper_crypto_sha1su1
)
4644 TRANS_FEAT(SHA256SU0
, aa64_sha256
, do_gvec_op2_ool
, a
, 0, gen_helper_crypto_sha256su0
)
4646 TRANS_FEAT(SHA512H
, aa64_sha512
, do_gvec_op3_ool
, a
, 0, gen_helper_crypto_sha512h
)
4647 TRANS_FEAT(SHA512H2
, aa64_sha512
, do_gvec_op3_ool
, a
, 0, gen_helper_crypto_sha512h2
)
4648 TRANS_FEAT(SHA512SU1
, aa64_sha512
, do_gvec_op3_ool
, a
, 0, gen_helper_crypto_sha512su1
)
4649 TRANS_FEAT(RAX1
, aa64_sha3
, do_gvec_fn3
, a
, gen_gvec_rax1
)
4650 TRANS_FEAT(SM3PARTW1
, aa64_sm3
, do_gvec_op3_ool
, a
, 0, gen_helper_crypto_sm3partw1
)
4651 TRANS_FEAT(SM3PARTW2
, aa64_sm3
, do_gvec_op3_ool
, a
, 0, gen_helper_crypto_sm3partw2
)
4652 TRANS_FEAT(SM4EKEY
, aa64_sm4
, do_gvec_op3_ool
, a
, 0, gen_helper_crypto_sm4ekey
)
4654 TRANS_FEAT(SHA512SU0
, aa64_sha512
, do_gvec_op2_ool
, a
, 0, gen_helper_crypto_sha512su0
)
4655 TRANS_FEAT(SM4E
, aa64_sm4
, do_gvec_op3_ool
, a
, 0, gen_helper_crypto_sm4e
)
4657 TRANS_FEAT(EOR3
, aa64_sha3
, do_gvec_fn4
, a
, gen_gvec_eor3
)
4658 TRANS_FEAT(BCAX
, aa64_sha3
, do_gvec_fn4
, a
, gen_gvec_bcax
)
4660 static bool trans_SM3SS1(DisasContext
*s
, arg_SM3SS1
*a
)
4662 if (!dc_isar_feature(aa64_sm3
, s
)) {
4665 if (fp_access_check(s
)) {
4666 TCGv_i32 tcg_op1
= tcg_temp_new_i32();
4667 TCGv_i32 tcg_op2
= tcg_temp_new_i32();
4668 TCGv_i32 tcg_op3
= tcg_temp_new_i32();
4669 TCGv_i32 tcg_res
= tcg_temp_new_i32();
4672 read_vec_element_i32(s
, tcg_op1
, a
->rn
, 3, MO_32
);
4673 read_vec_element_i32(s
, tcg_op2
, a
->rm
, 3, MO_32
);
4674 read_vec_element_i32(s
, tcg_op3
, a
->ra
, 3, MO_32
);
4676 tcg_gen_rotri_i32(tcg_res
, tcg_op1
, 20);
4677 tcg_gen_add_i32(tcg_res
, tcg_res
, tcg_op2
);
4678 tcg_gen_add_i32(tcg_res
, tcg_res
, tcg_op3
);
4679 tcg_gen_rotri_i32(tcg_res
, tcg_res
, 25);
4681 /* Clear the whole register first, then store bits [127:96]. */
4682 vsz
= vec_full_reg_size(s
);
4683 dofs
= vec_full_reg_offset(s
, a
->rd
);
4684 tcg_gen_gvec_dup_imm(MO_64
, dofs
, vsz
, vsz
, 0);
4685 write_vec_element_i32(s
, tcg_res
, a
->rd
, 3, MO_32
);
4690 static bool do_crypto3i(DisasContext
*s
, arg_crypto3i
*a
, gen_helper_gvec_3
*fn
)
4692 if (fp_access_check(s
)) {
4693 gen_gvec_op3_ool(s
, true, a
->rd
, a
->rn
, a
->rm
, a
->imm
, fn
);
4697 TRANS_FEAT(SM3TT1A
, aa64_sm3
, do_crypto3i
, a
, gen_helper_crypto_sm3tt1a
)
4698 TRANS_FEAT(SM3TT1B
, aa64_sm3
, do_crypto3i
, a
, gen_helper_crypto_sm3tt1b
)
4699 TRANS_FEAT(SM3TT2A
, aa64_sm3
, do_crypto3i
, a
, gen_helper_crypto_sm3tt2a
)
4700 TRANS_FEAT(SM3TT2B
, aa64_sm3
, do_crypto3i
, a
, gen_helper_crypto_sm3tt2b
)
4702 static bool trans_XAR(DisasContext
*s
, arg_XAR
*a
)
4704 if (!dc_isar_feature(aa64_sha3
, s
)) {
4707 if (fp_access_check(s
)) {
4708 gen_gvec_xar(MO_64
, vec_full_reg_offset(s
, a
->rd
),
4709 vec_full_reg_offset(s
, a
->rn
),
4710 vec_full_reg_offset(s
, a
->rm
), a
->imm
, 16,
4711 vec_full_reg_size(s
));
4717 * Advanced SIMD copy
4720 static bool decode_esz_idx(int imm
, MemOp
*pesz
, unsigned *pidx
)
4722 unsigned esz
= ctz32(imm
);
4725 *pidx
= imm
>> (esz
+ 1);
4731 static bool trans_DUP_element_s(DisasContext
*s
, arg_DUP_element_s
*a
)
4736 if (!decode_esz_idx(a
->imm
, &esz
, &idx
)) {
4739 if (fp_access_check(s
)) {
4741 * This instruction just extracts the specified element and
4742 * zero-extends it into the bottom of the destination register.
4744 TCGv_i64 tmp
= tcg_temp_new_i64();
4745 read_vec_element(s
, tmp
, a
->rn
, idx
, esz
);
4746 write_fp_dreg(s
, a
->rd
, tmp
);
4751 static bool trans_DUP_element_v(DisasContext
*s
, arg_DUP_element_v
*a
)
4756 if (!decode_esz_idx(a
->imm
, &esz
, &idx
)) {
4759 if (esz
== MO_64
&& !a
->q
) {
4762 if (fp_access_check(s
)) {
4763 tcg_gen_gvec_dup_mem(esz
, vec_full_reg_offset(s
, a
->rd
),
4764 vec_reg_offset(s
, a
->rn
, idx
, esz
),
4765 a
->q
? 16 : 8, vec_full_reg_size(s
));
4770 static bool trans_DUP_general(DisasContext
*s
, arg_DUP_general
*a
)
4775 if (!decode_esz_idx(a
->imm
, &esz
, &idx
)) {
4778 if (esz
== MO_64
&& !a
->q
) {
4781 if (fp_access_check(s
)) {
4782 tcg_gen_gvec_dup_i64(esz
, vec_full_reg_offset(s
, a
->rd
),
4783 a
->q
? 16 : 8, vec_full_reg_size(s
),
4789 static bool do_smov_umov(DisasContext
*s
, arg_SMOV
*a
, MemOp is_signed
)
4794 if (!decode_esz_idx(a
->imm
, &esz
, &idx
)) {
4798 if (esz
== MO_64
|| (esz
== MO_32
&& !a
->q
)) {
4802 if (esz
== MO_64
? !a
->q
: a
->q
) {
4806 if (fp_access_check(s
)) {
4807 TCGv_i64 tcg_rd
= cpu_reg(s
, a
->rd
);
4808 read_vec_element(s
, tcg_rd
, a
->rn
, idx
, esz
| is_signed
);
4809 if (is_signed
&& !a
->q
) {
4810 tcg_gen_ext32u_i64(tcg_rd
, tcg_rd
);
4816 TRANS(SMOV
, do_smov_umov
, a
, MO_SIGN
)
4817 TRANS(UMOV
, do_smov_umov
, a
, 0)
4819 static bool trans_INS_general(DisasContext
*s
, arg_INS_general
*a
)
4824 if (!decode_esz_idx(a
->imm
, &esz
, &idx
)) {
4827 if (fp_access_check(s
)) {
4828 write_vec_element(s
, cpu_reg(s
, a
->rn
), a
->rd
, idx
, esz
);
4829 clear_vec_high(s
, true, a
->rd
);
4834 static bool trans_INS_element(DisasContext
*s
, arg_INS_element
*a
)
4837 unsigned didx
, sidx
;
4839 if (!decode_esz_idx(a
->di
, &esz
, &didx
)) {
4842 sidx
= a
->si
>> esz
;
4843 if (fp_access_check(s
)) {
4844 TCGv_i64 tmp
= tcg_temp_new_i64();
4846 read_vec_element(s
, tmp
, a
->rn
, sidx
, esz
);
4847 write_vec_element(s
, tmp
, a
->rd
, didx
, esz
);
4849 /* INS is considered a 128-bit write for SVE. */
4850 clear_vec_high(s
, true, a
->rd
);
4856 * Advanced SIMD three same
4859 typedef struct FPScalar
{
4860 void (*gen_h
)(TCGv_i32
, TCGv_i32
, TCGv_i32
, TCGv_ptr
);
4861 void (*gen_s
)(TCGv_i32
, TCGv_i32
, TCGv_i32
, TCGv_ptr
);
4862 void (*gen_d
)(TCGv_i64
, TCGv_i64
, TCGv_i64
, TCGv_ptr
);
4865 static bool do_fp3_scalar(DisasContext
*s
, arg_rrr_e
*a
, const FPScalar
*f
)
4869 if (fp_access_check(s
)) {
4870 TCGv_i64 t0
= read_fp_dreg(s
, a
->rn
);
4871 TCGv_i64 t1
= read_fp_dreg(s
, a
->rm
);
4872 f
->gen_d(t0
, t0
, t1
, fpstatus_ptr(FPST_FPCR
));
4873 write_fp_dreg(s
, a
->rd
, t0
);
4877 if (fp_access_check(s
)) {
4878 TCGv_i32 t0
= read_fp_sreg(s
, a
->rn
);
4879 TCGv_i32 t1
= read_fp_sreg(s
, a
->rm
);
4880 f
->gen_s(t0
, t0
, t1
, fpstatus_ptr(FPST_FPCR
));
4881 write_fp_sreg(s
, a
->rd
, t0
);
4885 if (!dc_isar_feature(aa64_fp16
, s
)) {
4888 if (fp_access_check(s
)) {
4889 TCGv_i32 t0
= read_fp_hreg(s
, a
->rn
);
4890 TCGv_i32 t1
= read_fp_hreg(s
, a
->rm
);
4891 f
->gen_h(t0
, t0
, t1
, fpstatus_ptr(FPST_FPCR_F16
));
4892 write_fp_sreg(s
, a
->rd
, t0
);
4901 static const FPScalar f_scalar_fadd
= {
4902 gen_helper_vfp_addh
,
4903 gen_helper_vfp_adds
,
4904 gen_helper_vfp_addd
,
4906 TRANS(FADD_s
, do_fp3_scalar
, a
, &f_scalar_fadd
)
4908 static const FPScalar f_scalar_fsub
= {
4909 gen_helper_vfp_subh
,
4910 gen_helper_vfp_subs
,
4911 gen_helper_vfp_subd
,
4913 TRANS(FSUB_s
, do_fp3_scalar
, a
, &f_scalar_fsub
)
4915 static const FPScalar f_scalar_fdiv
= {
4916 gen_helper_vfp_divh
,
4917 gen_helper_vfp_divs
,
4918 gen_helper_vfp_divd
,
4920 TRANS(FDIV_s
, do_fp3_scalar
, a
, &f_scalar_fdiv
)
4922 static const FPScalar f_scalar_fmul
= {
4923 gen_helper_vfp_mulh
,
4924 gen_helper_vfp_muls
,
4925 gen_helper_vfp_muld
,
4927 TRANS(FMUL_s
, do_fp3_scalar
, a
, &f_scalar_fmul
)
4929 static const FPScalar f_scalar_fmax
= {
4930 gen_helper_advsimd_maxh
,
4931 gen_helper_vfp_maxs
,
4932 gen_helper_vfp_maxd
,
4934 TRANS(FMAX_s
, do_fp3_scalar
, a
, &f_scalar_fmax
)
4936 static const FPScalar f_scalar_fmin
= {
4937 gen_helper_advsimd_minh
,
4938 gen_helper_vfp_mins
,
4939 gen_helper_vfp_mind
,
4941 TRANS(FMIN_s
, do_fp3_scalar
, a
, &f_scalar_fmin
)
4943 static const FPScalar f_scalar_fmaxnm
= {
4944 gen_helper_advsimd_maxnumh
,
4945 gen_helper_vfp_maxnums
,
4946 gen_helper_vfp_maxnumd
,
4948 TRANS(FMAXNM_s
, do_fp3_scalar
, a
, &f_scalar_fmaxnm
)
4950 static const FPScalar f_scalar_fminnm
= {
4951 gen_helper_advsimd_minnumh
,
4952 gen_helper_vfp_minnums
,
4953 gen_helper_vfp_minnumd
,
4955 TRANS(FMINNM_s
, do_fp3_scalar
, a
, &f_scalar_fminnm
)
4957 static const FPScalar f_scalar_fmulx
= {
4958 gen_helper_advsimd_mulxh
,
4959 gen_helper_vfp_mulxs
,
4960 gen_helper_vfp_mulxd
,
4962 TRANS(FMULX_s
, do_fp3_scalar
, a
, &f_scalar_fmulx
)
4964 static void gen_fnmul_h(TCGv_i32 d
, TCGv_i32 n
, TCGv_i32 m
, TCGv_ptr s
)
4966 gen_helper_vfp_mulh(d
, n
, m
, s
);
4970 static void gen_fnmul_s(TCGv_i32 d
, TCGv_i32 n
, TCGv_i32 m
, TCGv_ptr s
)
4972 gen_helper_vfp_muls(d
, n
, m
, s
);
4976 static void gen_fnmul_d(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_ptr s
)
4978 gen_helper_vfp_muld(d
, n
, m
, s
);
4982 static const FPScalar f_scalar_fnmul
= {
4987 TRANS(FNMUL_s
, do_fp3_scalar
, a
, &f_scalar_fnmul
)
4989 static const FPScalar f_scalar_fcmeq
= {
4990 gen_helper_advsimd_ceq_f16
,
4991 gen_helper_neon_ceq_f32
,
4992 gen_helper_neon_ceq_f64
,
4994 TRANS(FCMEQ_s
, do_fp3_scalar
, a
, &f_scalar_fcmeq
)
4996 static const FPScalar f_scalar_fcmge
= {
4997 gen_helper_advsimd_cge_f16
,
4998 gen_helper_neon_cge_f32
,
4999 gen_helper_neon_cge_f64
,
5001 TRANS(FCMGE_s
, do_fp3_scalar
, a
, &f_scalar_fcmge
)
5003 static const FPScalar f_scalar_fcmgt
= {
5004 gen_helper_advsimd_cgt_f16
,
5005 gen_helper_neon_cgt_f32
,
5006 gen_helper_neon_cgt_f64
,
5008 TRANS(FCMGT_s
, do_fp3_scalar
, a
, &f_scalar_fcmgt
)
5010 static const FPScalar f_scalar_facge
= {
5011 gen_helper_advsimd_acge_f16
,
5012 gen_helper_neon_acge_f32
,
5013 gen_helper_neon_acge_f64
,
5015 TRANS(FACGE_s
, do_fp3_scalar
, a
, &f_scalar_facge
)
5017 static const FPScalar f_scalar_facgt
= {
5018 gen_helper_advsimd_acgt_f16
,
5019 gen_helper_neon_acgt_f32
,
5020 gen_helper_neon_acgt_f64
,
5022 TRANS(FACGT_s
, do_fp3_scalar
, a
, &f_scalar_facgt
)
5024 static void gen_fabd_h(TCGv_i32 d
, TCGv_i32 n
, TCGv_i32 m
, TCGv_ptr s
)
5026 gen_helper_vfp_subh(d
, n
, m
, s
);
5030 static void gen_fabd_s(TCGv_i32 d
, TCGv_i32 n
, TCGv_i32 m
, TCGv_ptr s
)
5032 gen_helper_vfp_subs(d
, n
, m
, s
);
5036 static void gen_fabd_d(TCGv_i64 d
, TCGv_i64 n
, TCGv_i64 m
, TCGv_ptr s
)
5038 gen_helper_vfp_subd(d
, n
, m
, s
);
5042 static const FPScalar f_scalar_fabd
= {
5047 TRANS(FABD_s
, do_fp3_scalar
, a
, &f_scalar_fabd
)
5049 static const FPScalar f_scalar_frecps
= {
5050 gen_helper_recpsf_f16
,
5051 gen_helper_recpsf_f32
,
5052 gen_helper_recpsf_f64
,
5054 TRANS(FRECPS_s
, do_fp3_scalar
, a
, &f_scalar_frecps
)
5056 static const FPScalar f_scalar_frsqrts
= {
5057 gen_helper_rsqrtsf_f16
,
5058 gen_helper_rsqrtsf_f32
,
5059 gen_helper_rsqrtsf_f64
,
5061 TRANS(FRSQRTS_s
, do_fp3_scalar
, a
, &f_scalar_frsqrts
)
5063 static bool do_satacc_s(DisasContext
*s
, arg_rrr_e
*a
,
5064 MemOp sgn_n
, MemOp sgn_m
,
5065 void (*gen_bhs
)(TCGv_i64
, TCGv_i64
, TCGv_i64
, TCGv_i64
, MemOp
),
5066 void (*gen_d
)(TCGv_i64
, TCGv_i64
, TCGv_i64
, TCGv_i64
))
5068 TCGv_i64 t0
, t1
, t2
, qc
;
5071 if (!fp_access_check(s
)) {
5075 t0
= tcg_temp_new_i64();
5076 t1
= tcg_temp_new_i64();
5077 t2
= tcg_temp_new_i64();
5078 qc
= tcg_temp_new_i64();
5079 read_vec_element(s
, t1
, a
->rn
, 0, esz
| sgn_n
);
5080 read_vec_element(s
, t2
, a
->rm
, 0, esz
| sgn_m
);
5081 tcg_gen_ld_i64(qc
, tcg_env
, offsetof(CPUARMState
, vfp
.qc
));
5084 gen_d(t0
, qc
, t1
, t2
);
5086 gen_bhs(t0
, qc
, t1
, t2
, esz
);
5087 tcg_gen_ext_i64(t0
, t0
, esz
);
5090 write_fp_dreg(s
, a
->rd
, t0
);
5091 tcg_gen_st_i64(qc
, tcg_env
, offsetof(CPUARMState
, vfp
.qc
));
5095 TRANS(SQADD_s
, do_satacc_s
, a
, MO_SIGN
, MO_SIGN
, gen_sqadd_bhs
, gen_sqadd_d
)
5096 TRANS(SQSUB_s
, do_satacc_s
, a
, MO_SIGN
, MO_SIGN
, gen_sqsub_bhs
, gen_sqsub_d
)
5097 TRANS(UQADD_s
, do_satacc_s
, a
, 0, 0, gen_uqadd_bhs
, gen_uqadd_d
)
5098 TRANS(UQSUB_s
, do_satacc_s
, a
, 0, 0, gen_uqsub_bhs
, gen_uqsub_d
)
5099 TRANS(SUQADD_s
, do_satacc_s
, a
, MO_SIGN
, 0, gen_suqadd_bhs
, gen_suqadd_d
)
5100 TRANS(USQADD_s
, do_satacc_s
, a
, 0, MO_SIGN
, gen_usqadd_bhs
, gen_usqadd_d
)
5102 static bool do_int3_scalar_d(DisasContext
*s
, arg_rrr_e
*a
,
5103 void (*fn
)(TCGv_i64
, TCGv_i64
, TCGv_i64
))
5105 if (fp_access_check(s
)) {
5106 TCGv_i64 t0
= tcg_temp_new_i64();
5107 TCGv_i64 t1
= tcg_temp_new_i64();
5109 read_vec_element(s
, t0
, a
->rn
, 0, MO_64
);
5110 read_vec_element(s
, t1
, a
->rm
, 0, MO_64
);
5112 write_fp_dreg(s
, a
->rd
, t0
);
5117 TRANS(SSHL_s
, do_int3_scalar_d
, a
, gen_sshl_i64
)
5118 TRANS(USHL_s
, do_int3_scalar_d
, a
, gen_ushl_i64
)
5119 TRANS(SRSHL_s
, do_int3_scalar_d
, a
, gen_helper_neon_rshl_s64
)
5120 TRANS(URSHL_s
, do_int3_scalar_d
, a
, gen_helper_neon_rshl_u64
)
5122 static bool do_fp3_vector(DisasContext
*s
, arg_qrrr_e
*a
,
5123 gen_helper_gvec_3_ptr
* const fns
[3])
5136 if (!dc_isar_feature(aa64_fp16
, s
)) {
5143 if (fp_access_check(s
)) {
5144 gen_gvec_op3_fpst(s
, a
->q
, a
->rd
, a
->rn
, a
->rm
,
5145 esz
== MO_16
, 0, fns
[esz
- 1]);
5150 static gen_helper_gvec_3_ptr
* const f_vector_fadd
[3] = {
5151 gen_helper_gvec_fadd_h
,
5152 gen_helper_gvec_fadd_s
,
5153 gen_helper_gvec_fadd_d
,
5155 TRANS(FADD_v
, do_fp3_vector
, a
, f_vector_fadd
)
5157 static gen_helper_gvec_3_ptr
* const f_vector_fsub
[3] = {
5158 gen_helper_gvec_fsub_h
,
5159 gen_helper_gvec_fsub_s
,
5160 gen_helper_gvec_fsub_d
,
5162 TRANS(FSUB_v
, do_fp3_vector
, a
, f_vector_fsub
)
5164 static gen_helper_gvec_3_ptr
* const f_vector_fdiv
[3] = {
5165 gen_helper_gvec_fdiv_h
,
5166 gen_helper_gvec_fdiv_s
,
5167 gen_helper_gvec_fdiv_d
,
5169 TRANS(FDIV_v
, do_fp3_vector
, a
, f_vector_fdiv
)
5171 static gen_helper_gvec_3_ptr
* const f_vector_fmul
[3] = {
5172 gen_helper_gvec_fmul_h
,
5173 gen_helper_gvec_fmul_s
,
5174 gen_helper_gvec_fmul_d
,
5176 TRANS(FMUL_v
, do_fp3_vector
, a
, f_vector_fmul
)
5178 static gen_helper_gvec_3_ptr
* const f_vector_fmax
[3] = {
5179 gen_helper_gvec_fmax_h
,
5180 gen_helper_gvec_fmax_s
,
5181 gen_helper_gvec_fmax_d
,
5183 TRANS(FMAX_v
, do_fp3_vector
, a
, f_vector_fmax
)
5185 static gen_helper_gvec_3_ptr
* const f_vector_fmin
[3] = {
5186 gen_helper_gvec_fmin_h
,
5187 gen_helper_gvec_fmin_s
,
5188 gen_helper_gvec_fmin_d
,
5190 TRANS(FMIN_v
, do_fp3_vector
, a
, f_vector_fmin
)
5192 static gen_helper_gvec_3_ptr
* const f_vector_fmaxnm
[3] = {
5193 gen_helper_gvec_fmaxnum_h
,
5194 gen_helper_gvec_fmaxnum_s
,
5195 gen_helper_gvec_fmaxnum_d
,
5197 TRANS(FMAXNM_v
, do_fp3_vector
, a
, f_vector_fmaxnm
)
5199 static gen_helper_gvec_3_ptr
* const f_vector_fminnm
[3] = {
5200 gen_helper_gvec_fminnum_h
,
5201 gen_helper_gvec_fminnum_s
,
5202 gen_helper_gvec_fminnum_d
,
5204 TRANS(FMINNM_v
, do_fp3_vector
, a
, f_vector_fminnm
)
5206 static gen_helper_gvec_3_ptr
* const f_vector_fmulx
[3] = {
5207 gen_helper_gvec_fmulx_h
,
5208 gen_helper_gvec_fmulx_s
,
5209 gen_helper_gvec_fmulx_d
,
5211 TRANS(FMULX_v
, do_fp3_vector
, a
, f_vector_fmulx
)
5213 static gen_helper_gvec_3_ptr
* const f_vector_fmla
[3] = {
5214 gen_helper_gvec_vfma_h
,
5215 gen_helper_gvec_vfma_s
,
5216 gen_helper_gvec_vfma_d
,
5218 TRANS(FMLA_v
, do_fp3_vector
, a
, f_vector_fmla
)
5220 static gen_helper_gvec_3_ptr
* const f_vector_fmls
[3] = {
5221 gen_helper_gvec_vfms_h
,
5222 gen_helper_gvec_vfms_s
,
5223 gen_helper_gvec_vfms_d
,
5225 TRANS(FMLS_v
, do_fp3_vector
, a
, f_vector_fmls
)
5227 static gen_helper_gvec_3_ptr
* const f_vector_fcmeq
[3] = {
5228 gen_helper_gvec_fceq_h
,
5229 gen_helper_gvec_fceq_s
,
5230 gen_helper_gvec_fceq_d
,
5232 TRANS(FCMEQ_v
, do_fp3_vector
, a
, f_vector_fcmeq
)
5234 static gen_helper_gvec_3_ptr
* const f_vector_fcmge
[3] = {
5235 gen_helper_gvec_fcge_h
,
5236 gen_helper_gvec_fcge_s
,
5237 gen_helper_gvec_fcge_d
,
5239 TRANS(FCMGE_v
, do_fp3_vector
, a
, f_vector_fcmge
)
5241 static gen_helper_gvec_3_ptr
* const f_vector_fcmgt
[3] = {
5242 gen_helper_gvec_fcgt_h
,
5243 gen_helper_gvec_fcgt_s
,
5244 gen_helper_gvec_fcgt_d
,
5246 TRANS(FCMGT_v
, do_fp3_vector
, a
, f_vector_fcmgt
)
5248 static gen_helper_gvec_3_ptr
* const f_vector_facge
[3] = {
5249 gen_helper_gvec_facge_h
,
5250 gen_helper_gvec_facge_s
,
5251 gen_helper_gvec_facge_d
,
5253 TRANS(FACGE_v
, do_fp3_vector
, a
, f_vector_facge
)
5255 static gen_helper_gvec_3_ptr
* const f_vector_facgt
[3] = {
5256 gen_helper_gvec_facgt_h
,
5257 gen_helper_gvec_facgt_s
,
5258 gen_helper_gvec_facgt_d
,
5260 TRANS(FACGT_v
, do_fp3_vector
, a
, f_vector_facgt
)
5262 static gen_helper_gvec_3_ptr
* const f_vector_fabd
[3] = {
5263 gen_helper_gvec_fabd_h
,
5264 gen_helper_gvec_fabd_s
,
5265 gen_helper_gvec_fabd_d
,
5267 TRANS(FABD_v
, do_fp3_vector
, a
, f_vector_fabd
)
5269 static gen_helper_gvec_3_ptr
* const f_vector_frecps
[3] = {
5270 gen_helper_gvec_recps_h
,
5271 gen_helper_gvec_recps_s
,
5272 gen_helper_gvec_recps_d
,
5274 TRANS(FRECPS_v
, do_fp3_vector
, a
, f_vector_frecps
)
5276 static gen_helper_gvec_3_ptr
* const f_vector_frsqrts
[3] = {
5277 gen_helper_gvec_rsqrts_h
,
5278 gen_helper_gvec_rsqrts_s
,
5279 gen_helper_gvec_rsqrts_d
,
5281 TRANS(FRSQRTS_v
, do_fp3_vector
, a
, f_vector_frsqrts
)
5283 static gen_helper_gvec_3_ptr
* const f_vector_faddp
[3] = {
5284 gen_helper_gvec_faddp_h
,
5285 gen_helper_gvec_faddp_s
,
5286 gen_helper_gvec_faddp_d
,
5288 TRANS(FADDP_v
, do_fp3_vector
, a
, f_vector_faddp
)
5290 static gen_helper_gvec_3_ptr
* const f_vector_fmaxp
[3] = {
5291 gen_helper_gvec_fmaxp_h
,
5292 gen_helper_gvec_fmaxp_s
,
5293 gen_helper_gvec_fmaxp_d
,
5295 TRANS(FMAXP_v
, do_fp3_vector
, a
, f_vector_fmaxp
)
5297 static gen_helper_gvec_3_ptr
* const f_vector_fminp
[3] = {
5298 gen_helper_gvec_fminp_h
,
5299 gen_helper_gvec_fminp_s
,
5300 gen_helper_gvec_fminp_d
,
5302 TRANS(FMINP_v
, do_fp3_vector
, a
, f_vector_fminp
)
5304 static gen_helper_gvec_3_ptr
* const f_vector_fmaxnmp
[3] = {
5305 gen_helper_gvec_fmaxnump_h
,
5306 gen_helper_gvec_fmaxnump_s
,
5307 gen_helper_gvec_fmaxnump_d
,
5309 TRANS(FMAXNMP_v
, do_fp3_vector
, a
, f_vector_fmaxnmp
)
5311 static gen_helper_gvec_3_ptr
* const f_vector_fminnmp
[3] = {
5312 gen_helper_gvec_fminnump_h
,
5313 gen_helper_gvec_fminnump_s
,
5314 gen_helper_gvec_fminnump_d
,
5316 TRANS(FMINNMP_v
, do_fp3_vector
, a
, f_vector_fminnmp
)
5318 static bool do_fmlal(DisasContext
*s
, arg_qrrr_e
*a
, bool is_s
, bool is_2
)
5320 if (fp_access_check(s
)) {
5321 int data
= (is_2
<< 1) | is_s
;
5322 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
5323 vec_full_reg_offset(s
, a
->rn
),
5324 vec_full_reg_offset(s
, a
->rm
), tcg_env
,
5325 a
->q
? 16 : 8, vec_full_reg_size(s
),
5326 data
, gen_helper_gvec_fmlal_a64
);
5331 TRANS_FEAT(FMLAL_v
, aa64_fhm
, do_fmlal
, a
, false, false)
5332 TRANS_FEAT(FMLSL_v
, aa64_fhm
, do_fmlal
, a
, true, false)
5333 TRANS_FEAT(FMLAL2_v
, aa64_fhm
, do_fmlal
, a
, false, true)
5334 TRANS_FEAT(FMLSL2_v
, aa64_fhm
, do_fmlal
, a
, true, true)
5336 TRANS(ADDP_v
, do_gvec_fn3
, a
, gen_gvec_addp
)
5337 TRANS(SMAXP_v
, do_gvec_fn3_no64
, a
, gen_gvec_smaxp
)
5338 TRANS(SMINP_v
, do_gvec_fn3_no64
, a
, gen_gvec_sminp
)
5339 TRANS(UMAXP_v
, do_gvec_fn3_no64
, a
, gen_gvec_umaxp
)
5340 TRANS(UMINP_v
, do_gvec_fn3_no64
, a
, gen_gvec_uminp
)
5342 TRANS(AND_v
, do_gvec_fn3
, a
, tcg_gen_gvec_and
)
5343 TRANS(BIC_v
, do_gvec_fn3
, a
, tcg_gen_gvec_andc
)
5344 TRANS(ORR_v
, do_gvec_fn3
, a
, tcg_gen_gvec_or
)
5345 TRANS(ORN_v
, do_gvec_fn3
, a
, tcg_gen_gvec_orc
)
5346 TRANS(EOR_v
, do_gvec_fn3
, a
, tcg_gen_gvec_xor
)
5348 static bool do_bitsel(DisasContext
*s
, bool is_q
, int d
, int a
, int b
, int c
)
5350 if (fp_access_check(s
)) {
5351 gen_gvec_fn4(s
, is_q
, d
, a
, b
, c
, tcg_gen_gvec_bitsel
, 0);
5356 TRANS(BSL_v
, do_bitsel
, a
->q
, a
->rd
, a
->rd
, a
->rn
, a
->rm
)
5357 TRANS(BIT_v
, do_bitsel
, a
->q
, a
->rd
, a
->rm
, a
->rn
, a
->rd
)
5358 TRANS(BIF_v
, do_bitsel
, a
->q
, a
->rd
, a
->rm
, a
->rd
, a
->rn
)
5360 TRANS(SQADD_v
, do_gvec_fn3
, a
, gen_gvec_sqadd_qc
)
5361 TRANS(UQADD_v
, do_gvec_fn3
, a
, gen_gvec_uqadd_qc
)
5362 TRANS(SQSUB_v
, do_gvec_fn3
, a
, gen_gvec_sqsub_qc
)
5363 TRANS(UQSUB_v
, do_gvec_fn3
, a
, gen_gvec_uqsub_qc
)
5364 TRANS(SUQADD_v
, do_gvec_fn3
, a
, gen_gvec_suqadd_qc
)
5365 TRANS(USQADD_v
, do_gvec_fn3
, a
, gen_gvec_usqadd_qc
)
5367 TRANS(SSHL_v
, do_gvec_fn3
, a
, gen_gvec_sshl
)
5368 TRANS(USHL_v
, do_gvec_fn3
, a
, gen_gvec_ushl
)
5369 TRANS(SRSHL_v
, do_gvec_fn3
, a
, gen_gvec_srshl
)
5370 TRANS(URSHL_v
, do_gvec_fn3
, a
, gen_gvec_urshl
)
5374 * Advanced SIMD scalar/vector x indexed element
5377 static bool do_fp3_scalar_idx(DisasContext
*s
, arg_rrx_e
*a
, const FPScalar
*f
)
5381 if (fp_access_check(s
)) {
5382 TCGv_i64 t0
= read_fp_dreg(s
, a
->rn
);
5383 TCGv_i64 t1
= tcg_temp_new_i64();
5385 read_vec_element(s
, t1
, a
->rm
, a
->idx
, MO_64
);
5386 f
->gen_d(t0
, t0
, t1
, fpstatus_ptr(FPST_FPCR
));
5387 write_fp_dreg(s
, a
->rd
, t0
);
5391 if (fp_access_check(s
)) {
5392 TCGv_i32 t0
= read_fp_sreg(s
, a
->rn
);
5393 TCGv_i32 t1
= tcg_temp_new_i32();
5395 read_vec_element_i32(s
, t1
, a
->rm
, a
->idx
, MO_32
);
5396 f
->gen_s(t0
, t0
, t1
, fpstatus_ptr(FPST_FPCR
));
5397 write_fp_sreg(s
, a
->rd
, t0
);
5401 if (!dc_isar_feature(aa64_fp16
, s
)) {
5404 if (fp_access_check(s
)) {
5405 TCGv_i32 t0
= read_fp_hreg(s
, a
->rn
);
5406 TCGv_i32 t1
= tcg_temp_new_i32();
5408 read_vec_element_i32(s
, t1
, a
->rm
, a
->idx
, MO_16
);
5409 f
->gen_h(t0
, t0
, t1
, fpstatus_ptr(FPST_FPCR_F16
));
5410 write_fp_sreg(s
, a
->rd
, t0
);
5414 g_assert_not_reached();
5419 TRANS(FMUL_si
, do_fp3_scalar_idx
, a
, &f_scalar_fmul
)
5420 TRANS(FMULX_si
, do_fp3_scalar_idx
, a
, &f_scalar_fmulx
)
5422 static bool do_fmla_scalar_idx(DisasContext
*s
, arg_rrx_e
*a
, bool neg
)
5426 if (fp_access_check(s
)) {
5427 TCGv_i64 t0
= read_fp_dreg(s
, a
->rd
);
5428 TCGv_i64 t1
= read_fp_dreg(s
, a
->rn
);
5429 TCGv_i64 t2
= tcg_temp_new_i64();
5431 read_vec_element(s
, t2
, a
->rm
, a
->idx
, MO_64
);
5433 gen_vfp_negd(t1
, t1
);
5435 gen_helper_vfp_muladdd(t0
, t1
, t2
, t0
, fpstatus_ptr(FPST_FPCR
));
5436 write_fp_dreg(s
, a
->rd
, t0
);
5440 if (fp_access_check(s
)) {
5441 TCGv_i32 t0
= read_fp_sreg(s
, a
->rd
);
5442 TCGv_i32 t1
= read_fp_sreg(s
, a
->rn
);
5443 TCGv_i32 t2
= tcg_temp_new_i32();
5445 read_vec_element_i32(s
, t2
, a
->rm
, a
->idx
, MO_32
);
5447 gen_vfp_negs(t1
, t1
);
5449 gen_helper_vfp_muladds(t0
, t1
, t2
, t0
, fpstatus_ptr(FPST_FPCR
));
5450 write_fp_sreg(s
, a
->rd
, t0
);
5454 if (!dc_isar_feature(aa64_fp16
, s
)) {
5457 if (fp_access_check(s
)) {
5458 TCGv_i32 t0
= read_fp_hreg(s
, a
->rd
);
5459 TCGv_i32 t1
= read_fp_hreg(s
, a
->rn
);
5460 TCGv_i32 t2
= tcg_temp_new_i32();
5462 read_vec_element_i32(s
, t2
, a
->rm
, a
->idx
, MO_16
);
5464 gen_vfp_negh(t1
, t1
);
5466 gen_helper_advsimd_muladdh(t0
, t1
, t2
, t0
,
5467 fpstatus_ptr(FPST_FPCR_F16
));
5468 write_fp_sreg(s
, a
->rd
, t0
);
5472 g_assert_not_reached();
5477 TRANS(FMLA_si
, do_fmla_scalar_idx
, a
, false)
5478 TRANS(FMLS_si
, do_fmla_scalar_idx
, a
, true)
5480 static bool do_fp3_vector_idx(DisasContext
*s
, arg_qrrx_e
*a
,
5481 gen_helper_gvec_3_ptr
* const fns
[3])
5494 if (!dc_isar_feature(aa64_fp16
, s
)) {
5499 g_assert_not_reached();
5501 if (fp_access_check(s
)) {
5502 gen_gvec_op3_fpst(s
, a
->q
, a
->rd
, a
->rn
, a
->rm
,
5503 esz
== MO_16
, a
->idx
, fns
[esz
- 1]);
5508 static gen_helper_gvec_3_ptr
* const f_vector_idx_fmul
[3] = {
5509 gen_helper_gvec_fmul_idx_h
,
5510 gen_helper_gvec_fmul_idx_s
,
5511 gen_helper_gvec_fmul_idx_d
,
5513 TRANS(FMUL_vi
, do_fp3_vector_idx
, a
, f_vector_idx_fmul
)
5515 static gen_helper_gvec_3_ptr
* const f_vector_idx_fmulx
[3] = {
5516 gen_helper_gvec_fmulx_idx_h
,
5517 gen_helper_gvec_fmulx_idx_s
,
5518 gen_helper_gvec_fmulx_idx_d
,
5520 TRANS(FMULX_vi
, do_fp3_vector_idx
, a
, f_vector_idx_fmulx
)
5522 static bool do_fmla_vector_idx(DisasContext
*s
, arg_qrrx_e
*a
, bool neg
)
5524 static gen_helper_gvec_4_ptr
* const fns
[3] = {
5525 gen_helper_gvec_fmla_idx_h
,
5526 gen_helper_gvec_fmla_idx_s
,
5527 gen_helper_gvec_fmla_idx_d
,
5540 if (!dc_isar_feature(aa64_fp16
, s
)) {
5545 g_assert_not_reached();
5547 if (fp_access_check(s
)) {
5548 gen_gvec_op4_fpst(s
, a
->q
, a
->rd
, a
->rn
, a
->rm
, a
->rd
,
5549 esz
== MO_16
, (a
->idx
<< 1) | neg
,
5555 TRANS(FMLA_vi
, do_fmla_vector_idx
, a
, false)
5556 TRANS(FMLS_vi
, do_fmla_vector_idx
, a
, true)
5558 static bool do_fmlal_idx(DisasContext
*s
, arg_qrrx_e
*a
, bool is_s
, bool is_2
)
5560 if (fp_access_check(s
)) {
5561 int data
= (a
->idx
<< 2) | (is_2
<< 1) | is_s
;
5562 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s
, a
->rd
),
5563 vec_full_reg_offset(s
, a
->rn
),
5564 vec_full_reg_offset(s
, a
->rm
), tcg_env
,
5565 a
->q
? 16 : 8, vec_full_reg_size(s
),
5566 data
, gen_helper_gvec_fmlal_idx_a64
);
5571 TRANS_FEAT(FMLAL_vi
, aa64_fhm
, do_fmlal_idx
, a
, false, false)
5572 TRANS_FEAT(FMLSL_vi
, aa64_fhm
, do_fmlal_idx
, a
, true, false)
5573 TRANS_FEAT(FMLAL2_vi
, aa64_fhm
, do_fmlal_idx
, a
, false, true)
5574 TRANS_FEAT(FMLSL2_vi
, aa64_fhm
, do_fmlal_idx
, a
, true, true)
5577 * Advanced SIMD scalar pairwise
5580 static bool do_fp3_scalar_pair(DisasContext
*s
, arg_rr_e
*a
, const FPScalar
*f
)
5584 if (fp_access_check(s
)) {
5585 TCGv_i64 t0
= tcg_temp_new_i64();
5586 TCGv_i64 t1
= tcg_temp_new_i64();
5588 read_vec_element(s
, t0
, a
->rn
, 0, MO_64
);
5589 read_vec_element(s
, t1
, a
->rn
, 1, MO_64
);
5590 f
->gen_d(t0
, t0
, t1
, fpstatus_ptr(FPST_FPCR
));
5591 write_fp_dreg(s
, a
->rd
, t0
);
5595 if (fp_access_check(s
)) {
5596 TCGv_i32 t0
= tcg_temp_new_i32();
5597 TCGv_i32 t1
= tcg_temp_new_i32();
5599 read_vec_element_i32(s
, t0
, a
->rn
, 0, MO_32
);
5600 read_vec_element_i32(s
, t1
, a
->rn
, 1, MO_32
);
5601 f
->gen_s(t0
, t0
, t1
, fpstatus_ptr(FPST_FPCR
));
5602 write_fp_sreg(s
, a
->rd
, t0
);
5606 if (!dc_isar_feature(aa64_fp16
, s
)) {
5609 if (fp_access_check(s
)) {
5610 TCGv_i32 t0
= tcg_temp_new_i32();
5611 TCGv_i32 t1
= tcg_temp_new_i32();
5613 read_vec_element_i32(s
, t0
, a
->rn
, 0, MO_16
);
5614 read_vec_element_i32(s
, t1
, a
->rn
, 1, MO_16
);
5615 f
->gen_h(t0
, t0
, t1
, fpstatus_ptr(FPST_FPCR_F16
));
5616 write_fp_sreg(s
, a
->rd
, t0
);
5620 g_assert_not_reached();
5625 TRANS(FADDP_s
, do_fp3_scalar_pair
, a
, &f_scalar_fadd
)
5626 TRANS(FMAXP_s
, do_fp3_scalar_pair
, a
, &f_scalar_fmax
)
5627 TRANS(FMINP_s
, do_fp3_scalar_pair
, a
, &f_scalar_fmin
)
5628 TRANS(FMAXNMP_s
, do_fp3_scalar_pair
, a
, &f_scalar_fmaxnm
)
5629 TRANS(FMINNMP_s
, do_fp3_scalar_pair
, a
, &f_scalar_fminnm
)
5631 static bool trans_ADDP_s(DisasContext
*s
, arg_rr_e
*a
)
5633 if (fp_access_check(s
)) {
5634 TCGv_i64 t0
= tcg_temp_new_i64();
5635 TCGv_i64 t1
= tcg_temp_new_i64();
5637 read_vec_element(s
, t0
, a
->rn
, 0, MO_64
);
5638 read_vec_element(s
, t1
, a
->rn
, 1, MO_64
);
5639 tcg_gen_add_i64(t0
, t0
, t1
);
5640 write_fp_dreg(s
, a
->rd
, t0
);
5645 /* Shift a TCGv src by TCGv shift_amount, put result in dst.
5646 * Note that it is the caller's responsibility to ensure that the
5647 * shift amount is in range (ie 0..31 or 0..63) and provide the ARM
5648 * mandated semantics for out of range shifts.
5650 static void shift_reg(TCGv_i64 dst
, TCGv_i64 src
, int sf
,
5651 enum a64_shift_type shift_type
, TCGv_i64 shift_amount
)
5653 switch (shift_type
) {
5654 case A64_SHIFT_TYPE_LSL
:
5655 tcg_gen_shl_i64(dst
, src
, shift_amount
);
5657 case A64_SHIFT_TYPE_LSR
:
5658 tcg_gen_shr_i64(dst
, src
, shift_amount
);
5660 case A64_SHIFT_TYPE_ASR
:
5662 tcg_gen_ext32s_i64(dst
, src
);
5664 tcg_gen_sar_i64(dst
, sf
? src
: dst
, shift_amount
);
5666 case A64_SHIFT_TYPE_ROR
:
5668 tcg_gen_rotr_i64(dst
, src
, shift_amount
);
5671 t0
= tcg_temp_new_i32();
5672 t1
= tcg_temp_new_i32();
5673 tcg_gen_extrl_i64_i32(t0
, src
);
5674 tcg_gen_extrl_i64_i32(t1
, shift_amount
);
5675 tcg_gen_rotr_i32(t0
, t0
, t1
);
5676 tcg_gen_extu_i32_i64(dst
, t0
);
5680 assert(FALSE
); /* all shift types should be handled */
5684 if (!sf
) { /* zero extend final result */
5685 tcg_gen_ext32u_i64(dst
, dst
);
5689 /* Shift a TCGv src by immediate, put result in dst.
5690 * The shift amount must be in range (this should always be true as the
5691 * relevant instructions will UNDEF on bad shift immediates).
5693 static void shift_reg_imm(TCGv_i64 dst
, TCGv_i64 src
, int sf
,
5694 enum a64_shift_type shift_type
, unsigned int shift_i
)
5696 assert(shift_i
< (sf
? 64 : 32));
5699 tcg_gen_mov_i64(dst
, src
);
5701 shift_reg(dst
, src
, sf
, shift_type
, tcg_constant_i64(shift_i
));
5705 /* Logical (shifted register)
5706 * 31 30 29 28 24 23 22 21 20 16 15 10 9 5 4 0
5707 * +----+-----+-----------+-------+---+------+--------+------+------+
5708 * | sf | opc | 0 1 0 1 0 | shift | N | Rm | imm6 | Rn | Rd |
5709 * +----+-----+-----------+-------+---+------+--------+------+------+
5711 static void disas_logic_reg(DisasContext
*s
, uint32_t insn
)
5713 TCGv_i64 tcg_rd
, tcg_rn
, tcg_rm
;
5714 unsigned int sf
, opc
, shift_type
, invert
, rm
, shift_amount
, rn
, rd
;
5716 sf
= extract32(insn
, 31, 1);
5717 opc
= extract32(insn
, 29, 2);
5718 shift_type
= extract32(insn
, 22, 2);
5719 invert
= extract32(insn
, 21, 1);
5720 rm
= extract32(insn
, 16, 5);
5721 shift_amount
= extract32(insn
, 10, 6);
5722 rn
= extract32(insn
, 5, 5);
5723 rd
= extract32(insn
, 0, 5);
5725 if (!sf
&& (shift_amount
& (1 << 5))) {
5726 unallocated_encoding(s
);
5730 tcg_rd
= cpu_reg(s
, rd
);
5732 if (opc
== 1 && shift_amount
== 0 && shift_type
== 0 && rn
== 31) {
5733 /* Unshifted ORR and ORN with WZR/XZR is the standard encoding for
5734 * register-register MOV and MVN, so it is worth special casing.
5736 tcg_rm
= cpu_reg(s
, rm
);
5738 tcg_gen_not_i64(tcg_rd
, tcg_rm
);
5740 tcg_gen_ext32u_i64(tcg_rd
, tcg_rd
);
5744 tcg_gen_mov_i64(tcg_rd
, tcg_rm
);
5746 tcg_gen_ext32u_i64(tcg_rd
, tcg_rm
);
5752 tcg_rm
= read_cpu_reg(s
, rm
, sf
);
5755 shift_reg_imm(tcg_rm
, tcg_rm
, sf
, shift_type
, shift_amount
);
5758 tcg_rn
= cpu_reg(s
, rn
);
5760 switch (opc
| (invert
<< 2)) {
5763 tcg_gen_and_i64(tcg_rd
, tcg_rn
, tcg_rm
);
5766 tcg_gen_or_i64(tcg_rd
, tcg_rn
, tcg_rm
);
5769 tcg_gen_xor_i64(tcg_rd
, tcg_rn
, tcg_rm
);
5773 tcg_gen_andc_i64(tcg_rd
, tcg_rn
, tcg_rm
);
5776 tcg_gen_orc_i64(tcg_rd
, tcg_rn
, tcg_rm
);
5779 tcg_gen_eqv_i64(tcg_rd
, tcg_rn
, tcg_rm
);
5787 tcg_gen_ext32u_i64(tcg_rd
, tcg_rd
);
5791 gen_logic_CC(sf
, tcg_rd
);
5796 * Add/subtract (extended register)
5798 * 31|30|29|28 24|23 22|21|20 16|15 13|12 10|9 5|4 0|
5799 * +--+--+--+-----------+-----+--+-------+------+------+----+----+
5800 * |sf|op| S| 0 1 0 1 1 | opt | 1| Rm |option| imm3 | Rn | Rd |
5801 * +--+--+--+-----------+-----+--+-------+------+------+----+----+
5803 * sf: 0 -> 32bit, 1 -> 64bit
5804 * op: 0 -> add , 1 -> sub
5807 * option: extension type (see DecodeRegExtend)
5808 * imm3: optional shift to Rm
5810 * Rd = Rn + LSL(extend(Rm), amount)
5812 static void disas_add_sub_ext_reg(DisasContext
*s
, uint32_t insn
)
5814 int rd
= extract32(insn
, 0, 5);
5815 int rn
= extract32(insn
, 5, 5);
5816 int imm3
= extract32(insn
, 10, 3);
5817 int option
= extract32(insn
, 13, 3);
5818 int rm
= extract32(insn
, 16, 5);
5819 int opt
= extract32(insn
, 22, 2);
5820 bool setflags
= extract32(insn
, 29, 1);
5821 bool sub_op
= extract32(insn
, 30, 1);
5822 bool sf
= extract32(insn
, 31, 1);
5824 TCGv_i64 tcg_rm
, tcg_rn
; /* temps */
5826 TCGv_i64 tcg_result
;
5828 if (imm3
> 4 || opt
!= 0) {
5829 unallocated_encoding(s
);
5833 /* non-flag setting ops may use SP */
5835 tcg_rd
= cpu_reg_sp(s
, rd
);
5837 tcg_rd
= cpu_reg(s
, rd
);
5839 tcg_rn
= read_cpu_reg_sp(s
, rn
, sf
);
5841 tcg_rm
= read_cpu_reg(s
, rm
, sf
);
5842 ext_and_shift_reg(tcg_rm
, tcg_rm
, option
, imm3
);
5844 tcg_result
= tcg_temp_new_i64();
5848 tcg_gen_sub_i64(tcg_result
, tcg_rn
, tcg_rm
);
5850 tcg_gen_add_i64(tcg_result
, tcg_rn
, tcg_rm
);
5854 gen_sub_CC(sf
, tcg_result
, tcg_rn
, tcg_rm
);
5856 gen_add_CC(sf
, tcg_result
, tcg_rn
, tcg_rm
);
5861 tcg_gen_mov_i64(tcg_rd
, tcg_result
);
5863 tcg_gen_ext32u_i64(tcg_rd
, tcg_result
);
5868 * Add/subtract (shifted register)
5870 * 31 30 29 28 24 23 22 21 20 16 15 10 9 5 4 0
5871 * +--+--+--+-----------+-----+--+-------+---------+------+------+
5872 * |sf|op| S| 0 1 0 1 1 |shift| 0| Rm | imm6 | Rn | Rd |
5873 * +--+--+--+-----------+-----+--+-------+---------+------+------+
5875 * sf: 0 -> 32bit, 1 -> 64bit
5876 * op: 0 -> add , 1 -> sub
5878 * shift: 00 -> LSL, 01 -> LSR, 10 -> ASR, 11 -> RESERVED
5879 * imm6: Shift amount to apply to Rm before the add/sub
5881 static void disas_add_sub_reg(DisasContext
*s
, uint32_t insn
)
5883 int rd
= extract32(insn
, 0, 5);
5884 int rn
= extract32(insn
, 5, 5);
5885 int imm6
= extract32(insn
, 10, 6);
5886 int rm
= extract32(insn
, 16, 5);
5887 int shift_type
= extract32(insn
, 22, 2);
5888 bool setflags
= extract32(insn
, 29, 1);
5889 bool sub_op
= extract32(insn
, 30, 1);
5890 bool sf
= extract32(insn
, 31, 1);
5892 TCGv_i64 tcg_rd
= cpu_reg(s
, rd
);
5893 TCGv_i64 tcg_rn
, tcg_rm
;
5894 TCGv_i64 tcg_result
;
5896 if ((shift_type
== 3) || (!sf
&& (imm6
> 31))) {
5897 unallocated_encoding(s
);
5901 tcg_rn
= read_cpu_reg(s
, rn
, sf
);
5902 tcg_rm
= read_cpu_reg(s
, rm
, sf
);
5904 shift_reg_imm(tcg_rm
, tcg_rm
, sf
, shift_type
, imm6
);
5906 tcg_result
= tcg_temp_new_i64();
5910 tcg_gen_sub_i64(tcg_result
, tcg_rn
, tcg_rm
);
5912 tcg_gen_add_i64(tcg_result
, tcg_rn
, tcg_rm
);
5916 gen_sub_CC(sf
, tcg_result
, tcg_rn
, tcg_rm
);
5918 gen_add_CC(sf
, tcg_result
, tcg_rn
, tcg_rm
);
5923 tcg_gen_mov_i64(tcg_rd
, tcg_result
);
5925 tcg_gen_ext32u_i64(tcg_rd
, tcg_result
);
5929 /* Data-processing (3 source)
5931 * 31 30 29 28 24 23 21 20 16 15 14 10 9 5 4 0
5932 * +--+------+-----------+------+------+----+------+------+------+
5933 * |sf| op54 | 1 1 0 1 1 | op31 | Rm | o0 | Ra | Rn | Rd |
5934 * +--+------+-----------+------+------+----+------+------+------+
5936 static void disas_data_proc_3src(DisasContext
*s
, uint32_t insn
)
5938 int rd
= extract32(insn
, 0, 5);
5939 int rn
= extract32(insn
, 5, 5);
5940 int ra
= extract32(insn
, 10, 5);
5941 int rm
= extract32(insn
, 16, 5);
5942 int op_id
= (extract32(insn
, 29, 3) << 4) |
5943 (extract32(insn
, 21, 3) << 1) |
5944 extract32(insn
, 15, 1);
5945 bool sf
= extract32(insn
, 31, 1);
5946 bool is_sub
= extract32(op_id
, 0, 1);
5947 bool is_high
= extract32(op_id
, 2, 1);
5948 bool is_signed
= false;
5953 /* Note that op_id is sf:op54:op31:o0 so it includes the 32/64 size flag */
5955 case 0x42: /* SMADDL */
5956 case 0x43: /* SMSUBL */
5957 case 0x44: /* SMULH */
5960 case 0x0: /* MADD (32bit) */
5961 case 0x1: /* MSUB (32bit) */
5962 case 0x40: /* MADD (64bit) */
5963 case 0x41: /* MSUB (64bit) */
5964 case 0x4a: /* UMADDL */
5965 case 0x4b: /* UMSUBL */
5966 case 0x4c: /* UMULH */
5969 unallocated_encoding(s
);
5974 TCGv_i64 low_bits
= tcg_temp_new_i64(); /* low bits discarded */
5975 TCGv_i64 tcg_rd
= cpu_reg(s
, rd
);
5976 TCGv_i64 tcg_rn
= cpu_reg(s
, rn
);
5977 TCGv_i64 tcg_rm
= cpu_reg(s
, rm
);
5980 tcg_gen_muls2_i64(low_bits
, tcg_rd
, tcg_rn
, tcg_rm
);
5982 tcg_gen_mulu2_i64(low_bits
, tcg_rd
, tcg_rn
, tcg_rm
);
5987 tcg_op1
= tcg_temp_new_i64();
5988 tcg_op2
= tcg_temp_new_i64();
5989 tcg_tmp
= tcg_temp_new_i64();
5992 tcg_gen_mov_i64(tcg_op1
, cpu_reg(s
, rn
));
5993 tcg_gen_mov_i64(tcg_op2
, cpu_reg(s
, rm
));
5996 tcg_gen_ext32s_i64(tcg_op1
, cpu_reg(s
, rn
));
5997 tcg_gen_ext32s_i64(tcg_op2
, cpu_reg(s
, rm
));
5999 tcg_gen_ext32u_i64(tcg_op1
, cpu_reg(s
, rn
));
6000 tcg_gen_ext32u_i64(tcg_op2
, cpu_reg(s
, rm
));
6004 if (ra
== 31 && !is_sub
) {
6005 /* Special-case MADD with rA == XZR; it is the standard MUL alias */
6006 tcg_gen_mul_i64(cpu_reg(s
, rd
), tcg_op1
, tcg_op2
);
6008 tcg_gen_mul_i64(tcg_tmp
, tcg_op1
, tcg_op2
);
6010 tcg_gen_sub_i64(cpu_reg(s
, rd
), cpu_reg(s
, ra
), tcg_tmp
);
6012 tcg_gen_add_i64(cpu_reg(s
, rd
), cpu_reg(s
, ra
), tcg_tmp
);
6017 tcg_gen_ext32u_i64(cpu_reg(s
, rd
), cpu_reg(s
, rd
));
6021 /* Add/subtract (with carry)
6022 * 31 30 29 28 27 26 25 24 23 22 21 20 16 15 10 9 5 4 0
6023 * +--+--+--+------------------------+------+-------------+------+-----+
6024 * |sf|op| S| 1 1 0 1 0 0 0 0 | rm | 0 0 0 0 0 0 | Rn | Rd |
6025 * +--+--+--+------------------------+------+-------------+------+-----+
6028 static void disas_adc_sbc(DisasContext
*s
, uint32_t insn
)
6030 unsigned int sf
, op
, setflags
, rm
, rn
, rd
;
6031 TCGv_i64 tcg_y
, tcg_rn
, tcg_rd
;
6033 sf
= extract32(insn
, 31, 1);
6034 op
= extract32(insn
, 30, 1);
6035 setflags
= extract32(insn
, 29, 1);
6036 rm
= extract32(insn
, 16, 5);
6037 rn
= extract32(insn
, 5, 5);
6038 rd
= extract32(insn
, 0, 5);
6040 tcg_rd
= cpu_reg(s
, rd
);
6041 tcg_rn
= cpu_reg(s
, rn
);
6044 tcg_y
= tcg_temp_new_i64();
6045 tcg_gen_not_i64(tcg_y
, cpu_reg(s
, rm
));
6047 tcg_y
= cpu_reg(s
, rm
);
6051 gen_adc_CC(sf
, tcg_rd
, tcg_rn
, tcg_y
);
6053 gen_adc(sf
, tcg_rd
, tcg_rn
, tcg_y
);
6058 * Rotate right into flags
6059 * 31 30 29 21 15 10 5 4 0
6060 * +--+--+--+-----------------+--------+-----------+------+--+------+
6061 * |sf|op| S| 1 1 0 1 0 0 0 0 | imm6 | 0 0 0 0 1 | Rn |o2| mask |
6062 * +--+--+--+-----------------+--------+-----------+------+--+------+
6064 static void disas_rotate_right_into_flags(DisasContext
*s
, uint32_t insn
)
6066 int mask
= extract32(insn
, 0, 4);
6067 int o2
= extract32(insn
, 4, 1);
6068 int rn
= extract32(insn
, 5, 5);
6069 int imm6
= extract32(insn
, 15, 6);
6070 int sf_op_s
= extract32(insn
, 29, 3);
6074 if (sf_op_s
!= 5 || o2
!= 0 || !dc_isar_feature(aa64_condm_4
, s
)) {
6075 unallocated_encoding(s
);
6079 tcg_rn
= read_cpu_reg(s
, rn
, 1);
6080 tcg_gen_rotri_i64(tcg_rn
, tcg_rn
, imm6
);
6082 nzcv
= tcg_temp_new_i32();
6083 tcg_gen_extrl_i64_i32(nzcv
, tcg_rn
);
6085 if (mask
& 8) { /* N */
6086 tcg_gen_shli_i32(cpu_NF
, nzcv
, 31 - 3);
6088 if (mask
& 4) { /* Z */
6089 tcg_gen_not_i32(cpu_ZF
, nzcv
);
6090 tcg_gen_andi_i32(cpu_ZF
, cpu_ZF
, 4);
6092 if (mask
& 2) { /* C */
6093 tcg_gen_extract_i32(cpu_CF
, nzcv
, 1, 1);
6095 if (mask
& 1) { /* V */
6096 tcg_gen_shli_i32(cpu_VF
, nzcv
, 31 - 0);
6101 * Evaluate into flags
6102 * 31 30 29 21 15 14 10 5 4 0
6103 * +--+--+--+-----------------+---------+----+---------+------+--+------+
6104 * |sf|op| S| 1 1 0 1 0 0 0 0 | opcode2 | sz | 0 0 1 0 | Rn |o3| mask |
6105 * +--+--+--+-----------------+---------+----+---------+------+--+------+
6107 static void disas_evaluate_into_flags(DisasContext
*s
, uint32_t insn
)
6109 int o3_mask
= extract32(insn
, 0, 5);
6110 int rn
= extract32(insn
, 5, 5);
6111 int o2
= extract32(insn
, 15, 6);
6112 int sz
= extract32(insn
, 14, 1);
6113 int sf_op_s
= extract32(insn
, 29, 3);
6117 if (sf_op_s
!= 1 || o2
!= 0 || o3_mask
!= 0xd ||
6118 !dc_isar_feature(aa64_condm_4
, s
)) {
6119 unallocated_encoding(s
);
6122 shift
= sz
? 16 : 24; /* SETF16 or SETF8 */
6124 tmp
= tcg_temp_new_i32();
6125 tcg_gen_extrl_i64_i32(tmp
, cpu_reg(s
, rn
));
6126 tcg_gen_shli_i32(cpu_NF
, tmp
, shift
);
6127 tcg_gen_shli_i32(cpu_VF
, tmp
, shift
- 1);
6128 tcg_gen_mov_i32(cpu_ZF
, cpu_NF
);
6129 tcg_gen_xor_i32(cpu_VF
, cpu_VF
, cpu_NF
);
6132 /* Conditional compare (immediate / register)
6133 * 31 30 29 28 27 26 25 24 23 22 21 20 16 15 12 11 10 9 5 4 3 0
6134 * +--+--+--+------------------------+--------+------+----+--+------+--+-----+
6135 * |sf|op| S| 1 1 0 1 0 0 1 0 |imm5/rm | cond |i/r |o2| Rn |o3|nzcv |
6136 * +--+--+--+------------------------+--------+------+----+--+------+--+-----+
6139 static void disas_cc(DisasContext
*s
, uint32_t insn
)
6141 unsigned int sf
, op
, y
, cond
, rn
, nzcv
, is_imm
;
6142 TCGv_i32 tcg_t0
, tcg_t1
, tcg_t2
;
6143 TCGv_i64 tcg_tmp
, tcg_y
, tcg_rn
;
6146 if (!extract32(insn
, 29, 1)) {
6147 unallocated_encoding(s
);
6150 if (insn
& (1 << 10 | 1 << 4)) {
6151 unallocated_encoding(s
);
6154 sf
= extract32(insn
, 31, 1);
6155 op
= extract32(insn
, 30, 1);
6156 is_imm
= extract32(insn
, 11, 1);
6157 y
= extract32(insn
, 16, 5); /* y = rm (reg) or imm5 (imm) */
6158 cond
= extract32(insn
, 12, 4);
6159 rn
= extract32(insn
, 5, 5);
6160 nzcv
= extract32(insn
, 0, 4);
6162 /* Set T0 = !COND. */
6163 tcg_t0
= tcg_temp_new_i32();
6164 arm_test_cc(&c
, cond
);
6165 tcg_gen_setcondi_i32(tcg_invert_cond(c
.cond
), tcg_t0
, c
.value
, 0);
6167 /* Load the arguments for the new comparison. */
6169 tcg_y
= tcg_temp_new_i64();
6170 tcg_gen_movi_i64(tcg_y
, y
);
6172 tcg_y
= cpu_reg(s
, y
);
6174 tcg_rn
= cpu_reg(s
, rn
);
6176 /* Set the flags for the new comparison. */
6177 tcg_tmp
= tcg_temp_new_i64();
6179 gen_sub_CC(sf
, tcg_tmp
, tcg_rn
, tcg_y
);
6181 gen_add_CC(sf
, tcg_tmp
, tcg_rn
, tcg_y
);
6184 /* If COND was false, force the flags to #nzcv. Compute two masks
6185 * to help with this: T1 = (COND ? 0 : -1), T2 = (COND ? -1 : 0).
6186 * For tcg hosts that support ANDC, we can make do with just T1.
6187 * In either case, allow the tcg optimizer to delete any unused mask.
6189 tcg_t1
= tcg_temp_new_i32();
6190 tcg_t2
= tcg_temp_new_i32();
6191 tcg_gen_neg_i32(tcg_t1
, tcg_t0
);
6192 tcg_gen_subi_i32(tcg_t2
, tcg_t0
, 1);
6194 if (nzcv
& 8) { /* N */
6195 tcg_gen_or_i32(cpu_NF
, cpu_NF
, tcg_t1
);
6197 if (TCG_TARGET_HAS_andc_i32
) {
6198 tcg_gen_andc_i32(cpu_NF
, cpu_NF
, tcg_t1
);
6200 tcg_gen_and_i32(cpu_NF
, cpu_NF
, tcg_t2
);
6203 if (nzcv
& 4) { /* Z */
6204 if (TCG_TARGET_HAS_andc_i32
) {
6205 tcg_gen_andc_i32(cpu_ZF
, cpu_ZF
, tcg_t1
);
6207 tcg_gen_and_i32(cpu_ZF
, cpu_ZF
, tcg_t2
);
6210 tcg_gen_or_i32(cpu_ZF
, cpu_ZF
, tcg_t0
);
6212 if (nzcv
& 2) { /* C */
6213 tcg_gen_or_i32(cpu_CF
, cpu_CF
, tcg_t0
);
6215 if (TCG_TARGET_HAS_andc_i32
) {
6216 tcg_gen_andc_i32(cpu_CF
, cpu_CF
, tcg_t1
);
6218 tcg_gen_and_i32(cpu_CF
, cpu_CF
, tcg_t2
);
6221 if (nzcv
& 1) { /* V */
6222 tcg_gen_or_i32(cpu_VF
, cpu_VF
, tcg_t1
);
6224 if (TCG_TARGET_HAS_andc_i32
) {
6225 tcg_gen_andc_i32(cpu_VF
, cpu_VF
, tcg_t1
);
6227 tcg_gen_and_i32(cpu_VF
, cpu_VF
, tcg_t2
);
6232 /* Conditional select
6233 * 31 30 29 28 21 20 16 15 12 11 10 9 5 4 0
6234 * +----+----+---+-----------------+------+------+-----+------+------+
6235 * | sf | op | S | 1 1 0 1 0 1 0 0 | Rm | cond | op2 | Rn | Rd |
6236 * +----+----+---+-----------------+------+------+-----+------+------+
6238 static void disas_cond_select(DisasContext
*s
, uint32_t insn
)
6240 unsigned int sf
, else_inv
, rm
, cond
, else_inc
, rn
, rd
;
6241 TCGv_i64 tcg_rd
, zero
;
6244 if (extract32(insn
, 29, 1) || extract32(insn
, 11, 1)) {
6245 /* S == 1 or op2<1> == 1 */
6246 unallocated_encoding(s
);
6249 sf
= extract32(insn
, 31, 1);
6250 else_inv
= extract32(insn
, 30, 1);
6251 rm
= extract32(insn
, 16, 5);
6252 cond
= extract32(insn
, 12, 4);
6253 else_inc
= extract32(insn
, 10, 1);
6254 rn
= extract32(insn
, 5, 5);
6255 rd
= extract32(insn
, 0, 5);
6257 tcg_rd
= cpu_reg(s
, rd
);
6259 a64_test_cc(&c
, cond
);
6260 zero
= tcg_constant_i64(0);
6262 if (rn
== 31 && rm
== 31 && (else_inc
^ else_inv
)) {
6265 tcg_gen_negsetcond_i64(tcg_invert_cond(c
.cond
),
6266 tcg_rd
, c
.value
, zero
);
6268 tcg_gen_setcond_i64(tcg_invert_cond(c
.cond
),
6269 tcg_rd
, c
.value
, zero
);
6272 TCGv_i64 t_true
= cpu_reg(s
, rn
);
6273 TCGv_i64 t_false
= read_cpu_reg(s
, rm
, 1);
6274 if (else_inv
&& else_inc
) {
6275 tcg_gen_neg_i64(t_false
, t_false
);
6276 } else if (else_inv
) {
6277 tcg_gen_not_i64(t_false
, t_false
);
6278 } else if (else_inc
) {
6279 tcg_gen_addi_i64(t_false
, t_false
, 1);
6281 tcg_gen_movcond_i64(c
.cond
, tcg_rd
, c
.value
, zero
, t_true
, t_false
);
6285 tcg_gen_ext32u_i64(tcg_rd
, tcg_rd
);
6289 static void handle_clz(DisasContext
*s
, unsigned int sf
,
6290 unsigned int rn
, unsigned int rd
)
6292 TCGv_i64 tcg_rd
, tcg_rn
;
6293 tcg_rd
= cpu_reg(s
, rd
);
6294 tcg_rn
= cpu_reg(s
, rn
);
6297 tcg_gen_clzi_i64(tcg_rd
, tcg_rn
, 64);
6299 TCGv_i32 tcg_tmp32
= tcg_temp_new_i32();
6300 tcg_gen_extrl_i64_i32(tcg_tmp32
, tcg_rn
);
6301 tcg_gen_clzi_i32(tcg_tmp32
, tcg_tmp32
, 32);
6302 tcg_gen_extu_i32_i64(tcg_rd
, tcg_tmp32
);
6306 static void handle_cls(DisasContext
*s
, unsigned int sf
,
6307 unsigned int rn
, unsigned int rd
)
6309 TCGv_i64 tcg_rd
, tcg_rn
;
6310 tcg_rd
= cpu_reg(s
, rd
);
6311 tcg_rn
= cpu_reg(s
, rn
);
6314 tcg_gen_clrsb_i64(tcg_rd
, tcg_rn
);
6316 TCGv_i32 tcg_tmp32
= tcg_temp_new_i32();
6317 tcg_gen_extrl_i64_i32(tcg_tmp32
, tcg_rn
);
6318 tcg_gen_clrsb_i32(tcg_tmp32
, tcg_tmp32
);
6319 tcg_gen_extu_i32_i64(tcg_rd
, tcg_tmp32
);
6323 static void handle_rbit(DisasContext
*s
, unsigned int sf
,
6324 unsigned int rn
, unsigned int rd
)
6326 TCGv_i64 tcg_rd
, tcg_rn
;
6327 tcg_rd
= cpu_reg(s
, rd
);
6328 tcg_rn
= cpu_reg(s
, rn
);
6331 gen_helper_rbit64(tcg_rd
, tcg_rn
);
6333 TCGv_i32 tcg_tmp32
= tcg_temp_new_i32();
6334 tcg_gen_extrl_i64_i32(tcg_tmp32
, tcg_rn
);
6335 gen_helper_rbit(tcg_tmp32
, tcg_tmp32
);
6336 tcg_gen_extu_i32_i64(tcg_rd
, tcg_tmp32
);
6340 /* REV with sf==1, opcode==3 ("REV64") */
6341 static void handle_rev64(DisasContext
*s
, unsigned int sf
,
6342 unsigned int rn
, unsigned int rd
)
6345 unallocated_encoding(s
);
6348 tcg_gen_bswap64_i64(cpu_reg(s
, rd
), cpu_reg(s
, rn
));
6351 /* REV with sf==0, opcode==2
6352 * REV32 (sf==1, opcode==2)
6354 static void handle_rev32(DisasContext
*s
, unsigned int sf
,
6355 unsigned int rn
, unsigned int rd
)
6357 TCGv_i64 tcg_rd
= cpu_reg(s
, rd
);
6358 TCGv_i64 tcg_rn
= cpu_reg(s
, rn
);
6361 tcg_gen_bswap64_i64(tcg_rd
, tcg_rn
);
6362 tcg_gen_rotri_i64(tcg_rd
, tcg_rd
, 32);
6364 tcg_gen_bswap32_i64(tcg_rd
, tcg_rn
, TCG_BSWAP_OZ
);
6368 /* REV16 (opcode==1) */
6369 static void handle_rev16(DisasContext
*s
, unsigned int sf
,
6370 unsigned int rn
, unsigned int rd
)
6372 TCGv_i64 tcg_rd
= cpu_reg(s
, rd
);
6373 TCGv_i64 tcg_tmp
= tcg_temp_new_i64();
6374 TCGv_i64 tcg_rn
= read_cpu_reg(s
, rn
, sf
);
6375 TCGv_i64 mask
= tcg_constant_i64(sf
? 0x00ff00ff00ff00ffull
: 0x00ff00ff);
6377 tcg_gen_shri_i64(tcg_tmp
, tcg_rn
, 8);
6378 tcg_gen_and_i64(tcg_rd
, tcg_rn
, mask
);
6379 tcg_gen_and_i64(tcg_tmp
, tcg_tmp
, mask
);
6380 tcg_gen_shli_i64(tcg_rd
, tcg_rd
, 8);
6381 tcg_gen_or_i64(tcg_rd
, tcg_rd
, tcg_tmp
);
6384 /* Data-processing (1 source)
6385 * 31 30 29 28 21 20 16 15 10 9 5 4 0
6386 * +----+---+---+-----------------+---------+--------+------+------+
6387 * | sf | 1 | S | 1 1 0 1 0 1 1 0 | opcode2 | opcode | Rn | Rd |
6388 * +----+---+---+-----------------+---------+--------+------+------+
6390 static void disas_data_proc_1src(DisasContext
*s
, uint32_t insn
)
6392 unsigned int sf
, opcode
, opcode2
, rn
, rd
;
6395 if (extract32(insn
, 29, 1)) {
6396 unallocated_encoding(s
);
6400 sf
= extract32(insn
, 31, 1);
6401 opcode
= extract32(insn
, 10, 6);
6402 opcode2
= extract32(insn
, 16, 5);
6403 rn
= extract32(insn
, 5, 5);
6404 rd
= extract32(insn
, 0, 5);
6406 #define MAP(SF, O2, O1) ((SF) | (O1 << 1) | (O2 << 7))
6408 switch (MAP(sf
, opcode2
, opcode
)) {
6409 case MAP(0, 0x00, 0x00): /* RBIT */
6410 case MAP(1, 0x00, 0x00):
6411 handle_rbit(s
, sf
, rn
, rd
);
6413 case MAP(0, 0x00, 0x01): /* REV16 */
6414 case MAP(1, 0x00, 0x01):
6415 handle_rev16(s
, sf
, rn
, rd
);
6417 case MAP(0, 0x00, 0x02): /* REV/REV32 */
6418 case MAP(1, 0x00, 0x02):
6419 handle_rev32(s
, sf
, rn
, rd
);
6421 case MAP(1, 0x00, 0x03): /* REV64 */
6422 handle_rev64(s
, sf
, rn
, rd
);
6424 case MAP(0, 0x00, 0x04): /* CLZ */
6425 case MAP(1, 0x00, 0x04):
6426 handle_clz(s
, sf
, rn
, rd
);
6428 case MAP(0, 0x00, 0x05): /* CLS */
6429 case MAP(1, 0x00, 0x05):
6430 handle_cls(s
, sf
, rn
, rd
);
6432 case MAP(1, 0x01, 0x00): /* PACIA */
6433 if (s
->pauth_active
) {
6434 tcg_rd
= cpu_reg(s
, rd
);
6435 gen_helper_pacia(tcg_rd
, tcg_env
, tcg_rd
, cpu_reg_sp(s
, rn
));
6436 } else if (!dc_isar_feature(aa64_pauth
, s
)) {
6437 goto do_unallocated
;
6440 case MAP(1, 0x01, 0x01): /* PACIB */
6441 if (s
->pauth_active
) {
6442 tcg_rd
= cpu_reg(s
, rd
);
6443 gen_helper_pacib(tcg_rd
, tcg_env
, tcg_rd
, cpu_reg_sp(s
, rn
));
6444 } else if (!dc_isar_feature(aa64_pauth
, s
)) {
6445 goto do_unallocated
;
6448 case MAP(1, 0x01, 0x02): /* PACDA */
6449 if (s
->pauth_active
) {
6450 tcg_rd
= cpu_reg(s
, rd
);
6451 gen_helper_pacda(tcg_rd
, tcg_env
, tcg_rd
, cpu_reg_sp(s
, rn
));
6452 } else if (!dc_isar_feature(aa64_pauth
, s
)) {
6453 goto do_unallocated
;
6456 case MAP(1, 0x01, 0x03): /* PACDB */
6457 if (s
->pauth_active
) {
6458 tcg_rd
= cpu_reg(s
, rd
);
6459 gen_helper_pacdb(tcg_rd
, tcg_env
, tcg_rd
, cpu_reg_sp(s
, rn
));
6460 } else if (!dc_isar_feature(aa64_pauth
, s
)) {
6461 goto do_unallocated
;
6464 case MAP(1, 0x01, 0x04): /* AUTIA */
6465 if (s
->pauth_active
) {
6466 tcg_rd
= cpu_reg(s
, rd
);
6467 gen_helper_autia(tcg_rd
, tcg_env
, tcg_rd
, cpu_reg_sp(s
, rn
));
6468 } else if (!dc_isar_feature(aa64_pauth
, s
)) {
6469 goto do_unallocated
;
6472 case MAP(1, 0x01, 0x05): /* AUTIB */
6473 if (s
->pauth_active
) {
6474 tcg_rd
= cpu_reg(s
, rd
);
6475 gen_helper_autib(tcg_rd
, tcg_env
, tcg_rd
, cpu_reg_sp(s
, rn
));
6476 } else if (!dc_isar_feature(aa64_pauth
, s
)) {
6477 goto do_unallocated
;
6480 case MAP(1, 0x01, 0x06): /* AUTDA */
6481 if (s
->pauth_active
) {
6482 tcg_rd
= cpu_reg(s
, rd
);
6483 gen_helper_autda(tcg_rd
, tcg_env
, tcg_rd
, cpu_reg_sp(s
, rn
));
6484 } else if (!dc_isar_feature(aa64_pauth
, s
)) {
6485 goto do_unallocated
;
6488 case MAP(1, 0x01, 0x07): /* AUTDB */
6489 if (s
->pauth_active
) {
6490 tcg_rd
= cpu_reg(s
, rd
);
6491 gen_helper_autdb(tcg_rd
, tcg_env
, tcg_rd
, cpu_reg_sp(s
, rn
));
6492 } else if (!dc_isar_feature(aa64_pauth
, s
)) {
6493 goto do_unallocated
;
6496 case MAP(1, 0x01, 0x08): /* PACIZA */
6497 if (!dc_isar_feature(aa64_pauth
, s
) || rn
!= 31) {
6498 goto do_unallocated
;
6499 } else if (s
->pauth_active
) {
6500 tcg_rd
= cpu_reg(s
, rd
);
6501 gen_helper_pacia(tcg_rd
, tcg_env
, tcg_rd
, tcg_constant_i64(0));
6504 case MAP(1, 0x01, 0x09): /* PACIZB */
6505 if (!dc_isar_feature(aa64_pauth
, s
) || rn
!= 31) {
6506 goto do_unallocated
;
6507 } else if (s
->pauth_active
) {
6508 tcg_rd
= cpu_reg(s
, rd
);
6509 gen_helper_pacib(tcg_rd
, tcg_env
, tcg_rd
, tcg_constant_i64(0));
6512 case MAP(1, 0x01, 0x0a): /* PACDZA */
6513 if (!dc_isar_feature(aa64_pauth
, s
) || rn
!= 31) {
6514 goto do_unallocated
;
6515 } else if (s
->pauth_active
) {
6516 tcg_rd
= cpu_reg(s
, rd
);
6517 gen_helper_pacda(tcg_rd
, tcg_env
, tcg_rd
, tcg_constant_i64(0));
6520 case MAP(1, 0x01, 0x0b): /* PACDZB */
6521 if (!dc_isar_feature(aa64_pauth
, s
) || rn
!= 31) {
6522 goto do_unallocated
;
6523 } else if (s
->pauth_active
) {
6524 tcg_rd
= cpu_reg(s
, rd
);
6525 gen_helper_pacdb(tcg_rd
, tcg_env
, tcg_rd
, tcg_constant_i64(0));
6528 case MAP(1, 0x01, 0x0c): /* AUTIZA */
6529 if (!dc_isar_feature(aa64_pauth
, s
) || rn
!= 31) {
6530 goto do_unallocated
;
6531 } else if (s
->pauth_active
) {
6532 tcg_rd
= cpu_reg(s
, rd
);
6533 gen_helper_autia(tcg_rd
, tcg_env
, tcg_rd
, tcg_constant_i64(0));
6536 case MAP(1, 0x01, 0x0d): /* AUTIZB */
6537 if (!dc_isar_feature(aa64_pauth
, s
) || rn
!= 31) {
6538 goto do_unallocated
;
6539 } else if (s
->pauth_active
) {
6540 tcg_rd
= cpu_reg(s
, rd
);
6541 gen_helper_autib(tcg_rd
, tcg_env
, tcg_rd
, tcg_constant_i64(0));
6544 case MAP(1, 0x01, 0x0e): /* AUTDZA */
6545 if (!dc_isar_feature(aa64_pauth
, s
) || rn
!= 31) {
6546 goto do_unallocated
;
6547 } else if (s
->pauth_active
) {
6548 tcg_rd
= cpu_reg(s
, rd
);
6549 gen_helper_autda(tcg_rd
, tcg_env
, tcg_rd
, tcg_constant_i64(0));
6552 case MAP(1, 0x01, 0x0f): /* AUTDZB */
6553 if (!dc_isar_feature(aa64_pauth
, s
) || rn
!= 31) {
6554 goto do_unallocated
;
6555 } else if (s
->pauth_active
) {
6556 tcg_rd
= cpu_reg(s
, rd
);
6557 gen_helper_autdb(tcg_rd
, tcg_env
, tcg_rd
, tcg_constant_i64(0));
6560 case MAP(1, 0x01, 0x10): /* XPACI */
6561 if (!dc_isar_feature(aa64_pauth
, s
) || rn
!= 31) {
6562 goto do_unallocated
;
6563 } else if (s
->pauth_active
) {
6564 tcg_rd
= cpu_reg(s
, rd
);
6565 gen_helper_xpaci(tcg_rd
, tcg_env
, tcg_rd
);
6568 case MAP(1, 0x01, 0x11): /* XPACD */
6569 if (!dc_isar_feature(aa64_pauth
, s
) || rn
!= 31) {
6570 goto do_unallocated
;
6571 } else if (s
->pauth_active
) {
6572 tcg_rd
= cpu_reg(s
, rd
);
6573 gen_helper_xpacd(tcg_rd
, tcg_env
, tcg_rd
);
6578 unallocated_encoding(s
);
6585 static void handle_div(DisasContext
*s
, bool is_signed
, unsigned int sf
,
6586 unsigned int rm
, unsigned int rn
, unsigned int rd
)
6588 TCGv_i64 tcg_n
, tcg_m
, tcg_rd
;
6589 tcg_rd
= cpu_reg(s
, rd
);
6591 if (!sf
&& is_signed
) {
6592 tcg_n
= tcg_temp_new_i64();
6593 tcg_m
= tcg_temp_new_i64();
6594 tcg_gen_ext32s_i64(tcg_n
, cpu_reg(s
, rn
));
6595 tcg_gen_ext32s_i64(tcg_m
, cpu_reg(s
, rm
));
6597 tcg_n
= read_cpu_reg(s
, rn
, sf
);
6598 tcg_m
= read_cpu_reg(s
, rm
, sf
);
6602 gen_helper_sdiv64(tcg_rd
, tcg_n
, tcg_m
);
6604 gen_helper_udiv64(tcg_rd
, tcg_n
, tcg_m
);
6607 if (!sf
) { /* zero extend final result */
6608 tcg_gen_ext32u_i64(tcg_rd
, tcg_rd
);
6612 /* LSLV, LSRV, ASRV, RORV */
6613 static void handle_shift_reg(DisasContext
*s
,
6614 enum a64_shift_type shift_type
, unsigned int sf
,
6615 unsigned int rm
, unsigned int rn
, unsigned int rd
)
6617 TCGv_i64 tcg_shift
= tcg_temp_new_i64();
6618 TCGv_i64 tcg_rd
= cpu_reg(s
, rd
);
6619 TCGv_i64 tcg_rn
= read_cpu_reg(s
, rn
, sf
);
6621 tcg_gen_andi_i64(tcg_shift
, cpu_reg(s
, rm
), sf
? 63 : 31);
6622 shift_reg(tcg_rd
, tcg_rn
, sf
, shift_type
, tcg_shift
);
6625 /* CRC32[BHWX], CRC32C[BHWX] */
6626 static void handle_crc32(DisasContext
*s
,
6627 unsigned int sf
, unsigned int sz
, bool crc32c
,
6628 unsigned int rm
, unsigned int rn
, unsigned int rd
)
6630 TCGv_i64 tcg_acc
, tcg_val
;
6633 if (!dc_isar_feature(aa64_crc32
, s
)
6634 || (sf
== 1 && sz
!= 3)
6635 || (sf
== 0 && sz
== 3)) {
6636 unallocated_encoding(s
);
6641 tcg_val
= cpu_reg(s
, rm
);
6655 g_assert_not_reached();
6657 tcg_val
= tcg_temp_new_i64();
6658 tcg_gen_andi_i64(tcg_val
, cpu_reg(s
, rm
), mask
);
6661 tcg_acc
= cpu_reg(s
, rn
);
6662 tcg_bytes
= tcg_constant_i32(1 << sz
);
6665 gen_helper_crc32c_64(cpu_reg(s
, rd
), tcg_acc
, tcg_val
, tcg_bytes
);
6667 gen_helper_crc32_64(cpu_reg(s
, rd
), tcg_acc
, tcg_val
, tcg_bytes
);
6671 /* Data-processing (2 source)
6672 * 31 30 29 28 21 20 16 15 10 9 5 4 0
6673 * +----+---+---+-----------------+------+--------+------+------+
6674 * | sf | 0 | S | 1 1 0 1 0 1 1 0 | Rm | opcode | Rn | Rd |
6675 * +----+---+---+-----------------+------+--------+------+------+
6677 static void disas_data_proc_2src(DisasContext
*s
, uint32_t insn
)
6679 unsigned int sf
, rm
, opcode
, rn
, rd
, setflag
;
6680 sf
= extract32(insn
, 31, 1);
6681 setflag
= extract32(insn
, 29, 1);
6682 rm
= extract32(insn
, 16, 5);
6683 opcode
= extract32(insn
, 10, 6);
6684 rn
= extract32(insn
, 5, 5);
6685 rd
= extract32(insn
, 0, 5);
6687 if (setflag
&& opcode
!= 0) {
6688 unallocated_encoding(s
);
6693 case 0: /* SUBP(S) */
6694 if (sf
== 0 || !dc_isar_feature(aa64_mte_insn_reg
, s
)) {
6695 goto do_unallocated
;
6697 TCGv_i64 tcg_n
, tcg_m
, tcg_d
;
6699 tcg_n
= read_cpu_reg_sp(s
, rn
, true);
6700 tcg_m
= read_cpu_reg_sp(s
, rm
, true);
6701 tcg_gen_sextract_i64(tcg_n
, tcg_n
, 0, 56);
6702 tcg_gen_sextract_i64(tcg_m
, tcg_m
, 0, 56);
6703 tcg_d
= cpu_reg(s
, rd
);
6706 gen_sub_CC(true, tcg_d
, tcg_n
, tcg_m
);
6708 tcg_gen_sub_i64(tcg_d
, tcg_n
, tcg_m
);
6713 handle_div(s
, false, sf
, rm
, rn
, rd
);
6716 handle_div(s
, true, sf
, rm
, rn
, rd
);
6719 if (sf
== 0 || !dc_isar_feature(aa64_mte_insn_reg
, s
)) {
6720 goto do_unallocated
;
6723 gen_helper_irg(cpu_reg_sp(s
, rd
), tcg_env
,
6724 cpu_reg_sp(s
, rn
), cpu_reg(s
, rm
));
6726 gen_address_with_allocation_tag0(cpu_reg_sp(s
, rd
),
6731 if (sf
== 0 || !dc_isar_feature(aa64_mte_insn_reg
, s
)) {
6732 goto do_unallocated
;
6734 TCGv_i64 t
= tcg_temp_new_i64();
6736 tcg_gen_extract_i64(t
, cpu_reg_sp(s
, rn
), 56, 4);
6737 tcg_gen_shl_i64(t
, tcg_constant_i64(1), t
);
6738 tcg_gen_or_i64(cpu_reg(s
, rd
), cpu_reg(s
, rm
), t
);
6742 handle_shift_reg(s
, A64_SHIFT_TYPE_LSL
, sf
, rm
, rn
, rd
);
6745 handle_shift_reg(s
, A64_SHIFT_TYPE_LSR
, sf
, rm
, rn
, rd
);
6748 handle_shift_reg(s
, A64_SHIFT_TYPE_ASR
, sf
, rm
, rn
, rd
);
6751 handle_shift_reg(s
, A64_SHIFT_TYPE_ROR
, sf
, rm
, rn
, rd
);
6753 case 12: /* PACGA */
6754 if (sf
== 0 || !dc_isar_feature(aa64_pauth
, s
)) {
6755 goto do_unallocated
;
6757 gen_helper_pacga(cpu_reg(s
, rd
), tcg_env
,
6758 cpu_reg(s
, rn
), cpu_reg_sp(s
, rm
));
6767 case 23: /* CRC32 */
6769 int sz
= extract32(opcode
, 0, 2);
6770 bool crc32c
= extract32(opcode
, 2, 1);
6771 handle_crc32(s
, sf
, sz
, crc32c
, rm
, rn
, rd
);
6776 unallocated_encoding(s
);
6782 * Data processing - register
6783 * 31 30 29 28 25 21 20 16 10 0
6784 * +--+---+--+---+-------+-----+-------+-------+---------+
6785 * | |op0| |op1| 1 0 1 | op2 | | op3 | |
6786 * +--+---+--+---+-------+-----+-------+-------+---------+
6788 static void disas_data_proc_reg(DisasContext
*s
, uint32_t insn
)
6790 int op0
= extract32(insn
, 30, 1);
6791 int op1
= extract32(insn
, 28, 1);
6792 int op2
= extract32(insn
, 21, 4);
6793 int op3
= extract32(insn
, 10, 6);
6798 /* Add/sub (extended register) */
6799 disas_add_sub_ext_reg(s
, insn
);
6801 /* Add/sub (shifted register) */
6802 disas_add_sub_reg(s
, insn
);
6805 /* Logical (shifted register) */
6806 disas_logic_reg(s
, insn
);
6814 case 0x00: /* Add/subtract (with carry) */
6815 disas_adc_sbc(s
, insn
);
6818 case 0x01: /* Rotate right into flags */
6820 disas_rotate_right_into_flags(s
, insn
);
6823 case 0x02: /* Evaluate into flags */
6827 disas_evaluate_into_flags(s
, insn
);
6831 goto do_unallocated
;
6835 case 0x2: /* Conditional compare */
6836 disas_cc(s
, insn
); /* both imm and reg forms */
6839 case 0x4: /* Conditional select */
6840 disas_cond_select(s
, insn
);
6843 case 0x6: /* Data-processing */
6844 if (op0
) { /* (1 source) */
6845 disas_data_proc_1src(s
, insn
);
6846 } else { /* (2 source) */
6847 disas_data_proc_2src(s
, insn
);
6850 case 0x8 ... 0xf: /* (3 source) */
6851 disas_data_proc_3src(s
, insn
);
6856 unallocated_encoding(s
);
6861 static void handle_fp_compare(DisasContext
*s
, int size
,
6862 unsigned int rn
, unsigned int rm
,
6863 bool cmp_with_zero
, bool signal_all_nans
)
6865 TCGv_i64 tcg_flags
= tcg_temp_new_i64();
6866 TCGv_ptr fpst
= fpstatus_ptr(size
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
6868 if (size
== MO_64
) {
6869 TCGv_i64 tcg_vn
, tcg_vm
;
6871 tcg_vn
= read_fp_dreg(s
, rn
);
6872 if (cmp_with_zero
) {
6873 tcg_vm
= tcg_constant_i64(0);
6875 tcg_vm
= read_fp_dreg(s
, rm
);
6877 if (signal_all_nans
) {
6878 gen_helper_vfp_cmped_a64(tcg_flags
, tcg_vn
, tcg_vm
, fpst
);
6880 gen_helper_vfp_cmpd_a64(tcg_flags
, tcg_vn
, tcg_vm
, fpst
);
6883 TCGv_i32 tcg_vn
= tcg_temp_new_i32();
6884 TCGv_i32 tcg_vm
= tcg_temp_new_i32();
6886 read_vec_element_i32(s
, tcg_vn
, rn
, 0, size
);
6887 if (cmp_with_zero
) {
6888 tcg_gen_movi_i32(tcg_vm
, 0);
6890 read_vec_element_i32(s
, tcg_vm
, rm
, 0, size
);
6895 if (signal_all_nans
) {
6896 gen_helper_vfp_cmpes_a64(tcg_flags
, tcg_vn
, tcg_vm
, fpst
);
6898 gen_helper_vfp_cmps_a64(tcg_flags
, tcg_vn
, tcg_vm
, fpst
);
6902 if (signal_all_nans
) {
6903 gen_helper_vfp_cmpeh_a64(tcg_flags
, tcg_vn
, tcg_vm
, fpst
);
6905 gen_helper_vfp_cmph_a64(tcg_flags
, tcg_vn
, tcg_vm
, fpst
);
6909 g_assert_not_reached();
6913 gen_set_nzcv(tcg_flags
);
6916 /* Floating point compare
6917 * 31 30 29 28 24 23 22 21 20 16 15 14 13 10 9 5 4 0
6918 * +---+---+---+-----------+------+---+------+-----+---------+------+-------+
6919 * | M | 0 | S | 1 1 1 1 0 | type | 1 | Rm | op | 1 0 0 0 | Rn | op2 |
6920 * +---+---+---+-----------+------+---+------+-----+---------+------+-------+
6922 static void disas_fp_compare(DisasContext
*s
, uint32_t insn
)
6924 unsigned int mos
, type
, rm
, op
, rn
, opc
, op2r
;
6927 mos
= extract32(insn
, 29, 3);
6928 type
= extract32(insn
, 22, 2);
6929 rm
= extract32(insn
, 16, 5);
6930 op
= extract32(insn
, 14, 2);
6931 rn
= extract32(insn
, 5, 5);
6932 opc
= extract32(insn
, 3, 2);
6933 op2r
= extract32(insn
, 0, 3);
6935 if (mos
|| op
|| op2r
) {
6936 unallocated_encoding(s
);
6949 if (dc_isar_feature(aa64_fp16
, s
)) {
6954 unallocated_encoding(s
);
6958 if (!fp_access_check(s
)) {
6962 handle_fp_compare(s
, size
, rn
, rm
, opc
& 1, opc
& 2);
6965 /* Floating point conditional compare
6966 * 31 30 29 28 24 23 22 21 20 16 15 12 11 10 9 5 4 3 0
6967 * +---+---+---+-----------+------+---+------+------+-----+------+----+------+
6968 * | M | 0 | S | 1 1 1 1 0 | type | 1 | Rm | cond | 0 1 | Rn | op | nzcv |
6969 * +---+---+---+-----------+------+---+------+------+-----+------+----+------+
6971 static void disas_fp_ccomp(DisasContext
*s
, uint32_t insn
)
6973 unsigned int mos
, type
, rm
, cond
, rn
, op
, nzcv
;
6974 TCGLabel
*label_continue
= NULL
;
6977 mos
= extract32(insn
, 29, 3);
6978 type
= extract32(insn
, 22, 2);
6979 rm
= extract32(insn
, 16, 5);
6980 cond
= extract32(insn
, 12, 4);
6981 rn
= extract32(insn
, 5, 5);
6982 op
= extract32(insn
, 4, 1);
6983 nzcv
= extract32(insn
, 0, 4);
6986 unallocated_encoding(s
);
6999 if (dc_isar_feature(aa64_fp16
, s
)) {
7004 unallocated_encoding(s
);
7008 if (!fp_access_check(s
)) {
7012 if (cond
< 0x0e) { /* not always */
7013 TCGLabel
*label_match
= gen_new_label();
7014 label_continue
= gen_new_label();
7015 arm_gen_test_cc(cond
, label_match
);
7017 gen_set_nzcv(tcg_constant_i64(nzcv
<< 28));
7018 tcg_gen_br(label_continue
);
7019 gen_set_label(label_match
);
7022 handle_fp_compare(s
, size
, rn
, rm
, false, op
);
7025 gen_set_label(label_continue
);
7029 /* Floating point conditional select
7030 * 31 30 29 28 24 23 22 21 20 16 15 12 11 10 9 5 4 0
7031 * +---+---+---+-----------+------+---+------+------+-----+------+------+
7032 * | M | 0 | S | 1 1 1 1 0 | type | 1 | Rm | cond | 1 1 | Rn | Rd |
7033 * +---+---+---+-----------+------+---+------+------+-----+------+------+
7035 static void disas_fp_csel(DisasContext
*s
, uint32_t insn
)
7037 unsigned int mos
, type
, rm
, cond
, rn
, rd
;
7038 TCGv_i64 t_true
, t_false
;
7042 mos
= extract32(insn
, 29, 3);
7043 type
= extract32(insn
, 22, 2);
7044 rm
= extract32(insn
, 16, 5);
7045 cond
= extract32(insn
, 12, 4);
7046 rn
= extract32(insn
, 5, 5);
7047 rd
= extract32(insn
, 0, 5);
7050 unallocated_encoding(s
);
7063 if (dc_isar_feature(aa64_fp16
, s
)) {
7068 unallocated_encoding(s
);
7072 if (!fp_access_check(s
)) {
7076 /* Zero extend sreg & hreg inputs to 64 bits now. */
7077 t_true
= tcg_temp_new_i64();
7078 t_false
= tcg_temp_new_i64();
7079 read_vec_element(s
, t_true
, rn
, 0, sz
);
7080 read_vec_element(s
, t_false
, rm
, 0, sz
);
7082 a64_test_cc(&c
, cond
);
7083 tcg_gen_movcond_i64(c
.cond
, t_true
, c
.value
, tcg_constant_i64(0),
7086 /* Note that sregs & hregs write back zeros to the high bits,
7087 and we've already done the zero-extension. */
7088 write_fp_dreg(s
, rd
, t_true
);
7091 /* Floating-point data-processing (1 source) - half precision */
7092 static void handle_fp_1src_half(DisasContext
*s
, int opcode
, int rd
, int rn
)
7094 TCGv_ptr fpst
= NULL
;
7095 TCGv_i32 tcg_op
= read_fp_hreg(s
, rn
);
7096 TCGv_i32 tcg_res
= tcg_temp_new_i32();
7099 case 0x0: /* FMOV */
7100 tcg_gen_mov_i32(tcg_res
, tcg_op
);
7102 case 0x1: /* FABS */
7103 gen_vfp_absh(tcg_res
, tcg_op
);
7105 case 0x2: /* FNEG */
7106 gen_vfp_negh(tcg_res
, tcg_op
);
7108 case 0x3: /* FSQRT */
7109 fpst
= fpstatus_ptr(FPST_FPCR_F16
);
7110 gen_helper_sqrt_f16(tcg_res
, tcg_op
, fpst
);
7112 case 0x8: /* FRINTN */
7113 case 0x9: /* FRINTP */
7114 case 0xa: /* FRINTM */
7115 case 0xb: /* FRINTZ */
7116 case 0xc: /* FRINTA */
7120 fpst
= fpstatus_ptr(FPST_FPCR_F16
);
7121 tcg_rmode
= gen_set_rmode(opcode
& 7, fpst
);
7122 gen_helper_advsimd_rinth(tcg_res
, tcg_op
, fpst
);
7123 gen_restore_rmode(tcg_rmode
, fpst
);
7126 case 0xe: /* FRINTX */
7127 fpst
= fpstatus_ptr(FPST_FPCR_F16
);
7128 gen_helper_advsimd_rinth_exact(tcg_res
, tcg_op
, fpst
);
7130 case 0xf: /* FRINTI */
7131 fpst
= fpstatus_ptr(FPST_FPCR_F16
);
7132 gen_helper_advsimd_rinth(tcg_res
, tcg_op
, fpst
);
7135 g_assert_not_reached();
7138 write_fp_sreg(s
, rd
, tcg_res
);
7141 /* Floating-point data-processing (1 source) - single precision */
7142 static void handle_fp_1src_single(DisasContext
*s
, int opcode
, int rd
, int rn
)
7144 void (*gen_fpst
)(TCGv_i32
, TCGv_i32
, TCGv_ptr
);
7145 TCGv_i32 tcg_op
, tcg_res
;
7149 tcg_op
= read_fp_sreg(s
, rn
);
7150 tcg_res
= tcg_temp_new_i32();
7153 case 0x0: /* FMOV */
7154 tcg_gen_mov_i32(tcg_res
, tcg_op
);
7156 case 0x1: /* FABS */
7157 gen_vfp_abss(tcg_res
, tcg_op
);
7159 case 0x2: /* FNEG */
7160 gen_vfp_negs(tcg_res
, tcg_op
);
7162 case 0x3: /* FSQRT */
7163 gen_helper_vfp_sqrts(tcg_res
, tcg_op
, tcg_env
);
7165 case 0x6: /* BFCVT */
7166 gen_fpst
= gen_helper_bfcvt
;
7168 case 0x8: /* FRINTN */
7169 case 0x9: /* FRINTP */
7170 case 0xa: /* FRINTM */
7171 case 0xb: /* FRINTZ */
7172 case 0xc: /* FRINTA */
7174 gen_fpst
= gen_helper_rints
;
7176 case 0xe: /* FRINTX */
7177 gen_fpst
= gen_helper_rints_exact
;
7179 case 0xf: /* FRINTI */
7180 gen_fpst
= gen_helper_rints
;
7182 case 0x10: /* FRINT32Z */
7183 rmode
= FPROUNDING_ZERO
;
7184 gen_fpst
= gen_helper_frint32_s
;
7186 case 0x11: /* FRINT32X */
7187 gen_fpst
= gen_helper_frint32_s
;
7189 case 0x12: /* FRINT64Z */
7190 rmode
= FPROUNDING_ZERO
;
7191 gen_fpst
= gen_helper_frint64_s
;
7193 case 0x13: /* FRINT64X */
7194 gen_fpst
= gen_helper_frint64_s
;
7197 g_assert_not_reached();
7200 fpst
= fpstatus_ptr(FPST_FPCR
);
7202 TCGv_i32 tcg_rmode
= gen_set_rmode(rmode
, fpst
);
7203 gen_fpst(tcg_res
, tcg_op
, fpst
);
7204 gen_restore_rmode(tcg_rmode
, fpst
);
7206 gen_fpst(tcg_res
, tcg_op
, fpst
);
7210 write_fp_sreg(s
, rd
, tcg_res
);
7213 /* Floating-point data-processing (1 source) - double precision */
7214 static void handle_fp_1src_double(DisasContext
*s
, int opcode
, int rd
, int rn
)
7216 void (*gen_fpst
)(TCGv_i64
, TCGv_i64
, TCGv_ptr
);
7217 TCGv_i64 tcg_op
, tcg_res
;
7222 case 0x0: /* FMOV */
7223 gen_gvec_fn2(s
, false, rd
, rn
, tcg_gen_gvec_mov
, 0);
7227 tcg_op
= read_fp_dreg(s
, rn
);
7228 tcg_res
= tcg_temp_new_i64();
7231 case 0x1: /* FABS */
7232 gen_vfp_absd(tcg_res
, tcg_op
);
7234 case 0x2: /* FNEG */
7235 gen_vfp_negd(tcg_res
, tcg_op
);
7237 case 0x3: /* FSQRT */
7238 gen_helper_vfp_sqrtd(tcg_res
, tcg_op
, tcg_env
);
7240 case 0x8: /* FRINTN */
7241 case 0x9: /* FRINTP */
7242 case 0xa: /* FRINTM */
7243 case 0xb: /* FRINTZ */
7244 case 0xc: /* FRINTA */
7246 gen_fpst
= gen_helper_rintd
;
7248 case 0xe: /* FRINTX */
7249 gen_fpst
= gen_helper_rintd_exact
;
7251 case 0xf: /* FRINTI */
7252 gen_fpst
= gen_helper_rintd
;
7254 case 0x10: /* FRINT32Z */
7255 rmode
= FPROUNDING_ZERO
;
7256 gen_fpst
= gen_helper_frint32_d
;
7258 case 0x11: /* FRINT32X */
7259 gen_fpst
= gen_helper_frint32_d
;
7261 case 0x12: /* FRINT64Z */
7262 rmode
= FPROUNDING_ZERO
;
7263 gen_fpst
= gen_helper_frint64_d
;
7265 case 0x13: /* FRINT64X */
7266 gen_fpst
= gen_helper_frint64_d
;
7269 g_assert_not_reached();
7272 fpst
= fpstatus_ptr(FPST_FPCR
);
7274 TCGv_i32 tcg_rmode
= gen_set_rmode(rmode
, fpst
);
7275 gen_fpst(tcg_res
, tcg_op
, fpst
);
7276 gen_restore_rmode(tcg_rmode
, fpst
);
7278 gen_fpst(tcg_res
, tcg_op
, fpst
);
7282 write_fp_dreg(s
, rd
, tcg_res
);
7285 static void handle_fp_fcvt(DisasContext
*s
, int opcode
,
7286 int rd
, int rn
, int dtype
, int ntype
)
7291 TCGv_i32 tcg_rn
= read_fp_sreg(s
, rn
);
7293 /* Single to double */
7294 TCGv_i64 tcg_rd
= tcg_temp_new_i64();
7295 gen_helper_vfp_fcvtds(tcg_rd
, tcg_rn
, tcg_env
);
7296 write_fp_dreg(s
, rd
, tcg_rd
);
7298 /* Single to half */
7299 TCGv_i32 tcg_rd
= tcg_temp_new_i32();
7300 TCGv_i32 ahp
= get_ahp_flag();
7301 TCGv_ptr fpst
= fpstatus_ptr(FPST_FPCR
);
7303 gen_helper_vfp_fcvt_f32_to_f16(tcg_rd
, tcg_rn
, fpst
, ahp
);
7304 /* write_fp_sreg is OK here because top half of tcg_rd is zero */
7305 write_fp_sreg(s
, rd
, tcg_rd
);
7311 TCGv_i64 tcg_rn
= read_fp_dreg(s
, rn
);
7312 TCGv_i32 tcg_rd
= tcg_temp_new_i32();
7314 /* Double to single */
7315 gen_helper_vfp_fcvtsd(tcg_rd
, tcg_rn
, tcg_env
);
7317 TCGv_ptr fpst
= fpstatus_ptr(FPST_FPCR
);
7318 TCGv_i32 ahp
= get_ahp_flag();
7319 /* Double to half */
7320 gen_helper_vfp_fcvt_f64_to_f16(tcg_rd
, tcg_rn
, fpst
, ahp
);
7321 /* write_fp_sreg is OK here because top half of tcg_rd is zero */
7323 write_fp_sreg(s
, rd
, tcg_rd
);
7328 TCGv_i32 tcg_rn
= read_fp_sreg(s
, rn
);
7329 TCGv_ptr tcg_fpst
= fpstatus_ptr(FPST_FPCR
);
7330 TCGv_i32 tcg_ahp
= get_ahp_flag();
7331 tcg_gen_ext16u_i32(tcg_rn
, tcg_rn
);
7333 /* Half to single */
7334 TCGv_i32 tcg_rd
= tcg_temp_new_i32();
7335 gen_helper_vfp_fcvt_f16_to_f32(tcg_rd
, tcg_rn
, tcg_fpst
, tcg_ahp
);
7336 write_fp_sreg(s
, rd
, tcg_rd
);
7338 /* Half to double */
7339 TCGv_i64 tcg_rd
= tcg_temp_new_i64();
7340 gen_helper_vfp_fcvt_f16_to_f64(tcg_rd
, tcg_rn
, tcg_fpst
, tcg_ahp
);
7341 write_fp_dreg(s
, rd
, tcg_rd
);
7346 g_assert_not_reached();
7350 /* Floating point data-processing (1 source)
7351 * 31 30 29 28 24 23 22 21 20 15 14 10 9 5 4 0
7352 * +---+---+---+-----------+------+---+--------+-----------+------+------+
7353 * | M | 0 | S | 1 1 1 1 0 | type | 1 | opcode | 1 0 0 0 0 | Rn | Rd |
7354 * +---+---+---+-----------+------+---+--------+-----------+------+------+
7356 static void disas_fp_1src(DisasContext
*s
, uint32_t insn
)
7358 int mos
= extract32(insn
, 29, 3);
7359 int type
= extract32(insn
, 22, 2);
7360 int opcode
= extract32(insn
, 15, 6);
7361 int rn
= extract32(insn
, 5, 5);
7362 int rd
= extract32(insn
, 0, 5);
7365 goto do_unallocated
;
7369 case 0x4: case 0x5: case 0x7:
7371 /* FCVT between half, single and double precision */
7372 int dtype
= extract32(opcode
, 0, 2);
7373 if (type
== 2 || dtype
== type
) {
7374 goto do_unallocated
;
7376 if (!fp_access_check(s
)) {
7380 handle_fp_fcvt(s
, opcode
, rd
, rn
, dtype
, type
);
7384 case 0x10 ... 0x13: /* FRINT{32,64}{X,Z} */
7385 if (type
> 1 || !dc_isar_feature(aa64_frint
, s
)) {
7386 goto do_unallocated
;
7392 /* 32-to-32 and 64-to-64 ops */
7395 if (!fp_access_check(s
)) {
7398 handle_fp_1src_single(s
, opcode
, rd
, rn
);
7401 if (!fp_access_check(s
)) {
7404 handle_fp_1src_double(s
, opcode
, rd
, rn
);
7407 if (!dc_isar_feature(aa64_fp16
, s
)) {
7408 goto do_unallocated
;
7411 if (!fp_access_check(s
)) {
7414 handle_fp_1src_half(s
, opcode
, rd
, rn
);
7417 goto do_unallocated
;
7424 if (!dc_isar_feature(aa64_bf16
, s
)) {
7425 goto do_unallocated
;
7427 if (!fp_access_check(s
)) {
7430 handle_fp_1src_single(s
, opcode
, rd
, rn
);
7433 goto do_unallocated
;
7439 unallocated_encoding(s
);
7444 /* Floating-point data-processing (3 source) - single precision */
7445 static void handle_fp_3src_single(DisasContext
*s
, bool o0
, bool o1
,
7446 int rd
, int rn
, int rm
, int ra
)
7448 TCGv_i32 tcg_op1
, tcg_op2
, tcg_op3
;
7449 TCGv_i32 tcg_res
= tcg_temp_new_i32();
7450 TCGv_ptr fpst
= fpstatus_ptr(FPST_FPCR
);
7452 tcg_op1
= read_fp_sreg(s
, rn
);
7453 tcg_op2
= read_fp_sreg(s
, rm
);
7454 tcg_op3
= read_fp_sreg(s
, ra
);
7456 /* These are fused multiply-add, and must be done as one
7457 * floating point operation with no rounding between the
7458 * multiplication and addition steps.
7459 * NB that doing the negations here as separate steps is
7460 * correct : an input NaN should come out with its sign bit
7461 * flipped if it is a negated-input.
7464 gen_vfp_negs(tcg_op3
, tcg_op3
);
7468 gen_vfp_negs(tcg_op1
, tcg_op1
);
7471 gen_helper_vfp_muladds(tcg_res
, tcg_op1
, tcg_op2
, tcg_op3
, fpst
);
7473 write_fp_sreg(s
, rd
, tcg_res
);
7476 /* Floating-point data-processing (3 source) - double precision */
7477 static void handle_fp_3src_double(DisasContext
*s
, bool o0
, bool o1
,
7478 int rd
, int rn
, int rm
, int ra
)
7480 TCGv_i64 tcg_op1
, tcg_op2
, tcg_op3
;
7481 TCGv_i64 tcg_res
= tcg_temp_new_i64();
7482 TCGv_ptr fpst
= fpstatus_ptr(FPST_FPCR
);
7484 tcg_op1
= read_fp_dreg(s
, rn
);
7485 tcg_op2
= read_fp_dreg(s
, rm
);
7486 tcg_op3
= read_fp_dreg(s
, ra
);
7488 /* These are fused multiply-add, and must be done as one
7489 * floating point operation with no rounding between the
7490 * multiplication and addition steps.
7491 * NB that doing the negations here as separate steps is
7492 * correct : an input NaN should come out with its sign bit
7493 * flipped if it is a negated-input.
7496 gen_vfp_negd(tcg_op3
, tcg_op3
);
7500 gen_vfp_negd(tcg_op1
, tcg_op1
);
7503 gen_helper_vfp_muladdd(tcg_res
, tcg_op1
, tcg_op2
, tcg_op3
, fpst
);
7505 write_fp_dreg(s
, rd
, tcg_res
);
7508 /* Floating-point data-processing (3 source) - half precision */
7509 static void handle_fp_3src_half(DisasContext
*s
, bool o0
, bool o1
,
7510 int rd
, int rn
, int rm
, int ra
)
7512 TCGv_i32 tcg_op1
, tcg_op2
, tcg_op3
;
7513 TCGv_i32 tcg_res
= tcg_temp_new_i32();
7514 TCGv_ptr fpst
= fpstatus_ptr(FPST_FPCR_F16
);
7516 tcg_op1
= read_fp_hreg(s
, rn
);
7517 tcg_op2
= read_fp_hreg(s
, rm
);
7518 tcg_op3
= read_fp_hreg(s
, ra
);
7520 /* These are fused multiply-add, and must be done as one
7521 * floating point operation with no rounding between the
7522 * multiplication and addition steps.
7523 * NB that doing the negations here as separate steps is
7524 * correct : an input NaN should come out with its sign bit
7525 * flipped if it is a negated-input.
7528 tcg_gen_xori_i32(tcg_op3
, tcg_op3
, 0x8000);
7532 tcg_gen_xori_i32(tcg_op1
, tcg_op1
, 0x8000);
7535 gen_helper_advsimd_muladdh(tcg_res
, tcg_op1
, tcg_op2
, tcg_op3
, fpst
);
7537 write_fp_sreg(s
, rd
, tcg_res
);
7540 /* Floating point data-processing (3 source)
7541 * 31 30 29 28 24 23 22 21 20 16 15 14 10 9 5 4 0
7542 * +---+---+---+-----------+------+----+------+----+------+------+------+
7543 * | M | 0 | S | 1 1 1 1 1 | type | o1 | Rm | o0 | Ra | Rn | Rd |
7544 * +---+---+---+-----------+------+----+------+----+------+------+------+
7546 static void disas_fp_3src(DisasContext
*s
, uint32_t insn
)
7548 int mos
= extract32(insn
, 29, 3);
7549 int type
= extract32(insn
, 22, 2);
7550 int rd
= extract32(insn
, 0, 5);
7551 int rn
= extract32(insn
, 5, 5);
7552 int ra
= extract32(insn
, 10, 5);
7553 int rm
= extract32(insn
, 16, 5);
7554 bool o0
= extract32(insn
, 15, 1);
7555 bool o1
= extract32(insn
, 21, 1);
7558 unallocated_encoding(s
);
7564 if (!fp_access_check(s
)) {
7567 handle_fp_3src_single(s
, o0
, o1
, rd
, rn
, rm
, ra
);
7570 if (!fp_access_check(s
)) {
7573 handle_fp_3src_double(s
, o0
, o1
, rd
, rn
, rm
, ra
);
7576 if (!dc_isar_feature(aa64_fp16
, s
)) {
7577 unallocated_encoding(s
);
7580 if (!fp_access_check(s
)) {
7583 handle_fp_3src_half(s
, o0
, o1
, rd
, rn
, rm
, ra
);
7586 unallocated_encoding(s
);
7590 /* Floating point immediate
7591 * 31 30 29 28 24 23 22 21 20 13 12 10 9 5 4 0
7592 * +---+---+---+-----------+------+---+------------+-------+------+------+
7593 * | M | 0 | S | 1 1 1 1 0 | type | 1 | imm8 | 1 0 0 | imm5 | Rd |
7594 * +---+---+---+-----------+------+---+------------+-------+------+------+
7596 static void disas_fp_imm(DisasContext
*s
, uint32_t insn
)
7598 int rd
= extract32(insn
, 0, 5);
7599 int imm5
= extract32(insn
, 5, 5);
7600 int imm8
= extract32(insn
, 13, 8);
7601 int type
= extract32(insn
, 22, 2);
7602 int mos
= extract32(insn
, 29, 3);
7607 unallocated_encoding(s
);
7620 if (dc_isar_feature(aa64_fp16
, s
)) {
7625 unallocated_encoding(s
);
7629 if (!fp_access_check(s
)) {
7633 imm
= vfp_expand_imm(sz
, imm8
);
7634 write_fp_dreg(s
, rd
, tcg_constant_i64(imm
));
7637 /* Handle floating point <=> fixed point conversions. Note that we can
7638 * also deal with fp <=> integer conversions as a special case (scale == 64)
7639 * OPTME: consider handling that special case specially or at least skipping
7640 * the call to scalbn in the helpers for zero shifts.
7642 static void handle_fpfpcvt(DisasContext
*s
, int rd
, int rn
, int opcode
,
7643 bool itof
, int rmode
, int scale
, int sf
, int type
)
7645 bool is_signed
= !(opcode
& 1);
7646 TCGv_ptr tcg_fpstatus
;
7647 TCGv_i32 tcg_shift
, tcg_single
;
7648 TCGv_i64 tcg_double
;
7650 tcg_fpstatus
= fpstatus_ptr(type
== 3 ? FPST_FPCR_F16
: FPST_FPCR
);
7652 tcg_shift
= tcg_constant_i32(64 - scale
);
7655 TCGv_i64 tcg_int
= cpu_reg(s
, rn
);
7657 TCGv_i64 tcg_extend
= tcg_temp_new_i64();
7660 tcg_gen_ext32s_i64(tcg_extend
, tcg_int
);
7662 tcg_gen_ext32u_i64(tcg_extend
, tcg_int
);
7665 tcg_int
= tcg_extend
;
7669 case 1: /* float64 */
7670 tcg_double
= tcg_temp_new_i64();
7672 gen_helper_vfp_sqtod(tcg_double
, tcg_int
,
7673 tcg_shift
, tcg_fpstatus
);
7675 gen_helper_vfp_uqtod(tcg_double
, tcg_int
,
7676 tcg_shift
, tcg_fpstatus
);
7678 write_fp_dreg(s
, rd
, tcg_double
);
7681 case 0: /* float32 */
7682 tcg_single
= tcg_temp_new_i32();
7684 gen_helper_vfp_sqtos(tcg_single
, tcg_int
,
7685 tcg_shift
, tcg_fpstatus
);
7687 gen_helper_vfp_uqtos(tcg_single
, tcg_int
,
7688 tcg_shift
, tcg_fpstatus
);
7690 write_fp_sreg(s
, rd
, tcg_single
);
7693 case 3: /* float16 */
7694 tcg_single
= tcg_temp_new_i32();
7696 gen_helper_vfp_sqtoh(tcg_single
, tcg_int
,
7697 tcg_shift
, tcg_fpstatus
);
7699 gen_helper_vfp_uqtoh(tcg_single
, tcg_int
,
7700 tcg_shift
, tcg_fpstatus
);
7702 write_fp_sreg(s
, rd
, tcg_single
);
7706 g_assert_not_reached();
7709 TCGv_i64 tcg_int
= cpu_reg(s
, rd
);
7712 if (extract32(opcode
, 2, 1)) {
7713 /* There are too many rounding modes to all fit into rmode,
7714 * so FCVTA[US] is a special case.
7716 rmode
= FPROUNDING_TIEAWAY
;
7719 tcg_rmode
= gen_set_rmode(rmode
, tcg_fpstatus
);
7722 case 1: /* float64 */
7723 tcg_double
= read_fp_dreg(s
, rn
);
7726 gen_helper_vfp_tosld(tcg_int
, tcg_double
,
7727 tcg_shift
, tcg_fpstatus
);
7729 gen_helper_vfp_tosqd(tcg_int
, tcg_double
,
7730 tcg_shift
, tcg_fpstatus
);
7734 gen_helper_vfp_tould(tcg_int
, tcg_double
,
7735 tcg_shift
, tcg_fpstatus
);
7737 gen_helper_vfp_touqd(tcg_int
, tcg_double
,
7738 tcg_shift
, tcg_fpstatus
);
7742 tcg_gen_ext32u_i64(tcg_int
, tcg_int
);
7746 case 0: /* float32 */
7747 tcg_single
= read_fp_sreg(s
, rn
);
7750 gen_helper_vfp_tosqs(tcg_int
, tcg_single
,
7751 tcg_shift
, tcg_fpstatus
);
7753 gen_helper_vfp_touqs(tcg_int
, tcg_single
,
7754 tcg_shift
, tcg_fpstatus
);
7757 TCGv_i32 tcg_dest
= tcg_temp_new_i32();
7759 gen_helper_vfp_tosls(tcg_dest
, tcg_single
,
7760 tcg_shift
, tcg_fpstatus
);
7762 gen_helper_vfp_touls(tcg_dest
, tcg_single
,
7763 tcg_shift
, tcg_fpstatus
);
7765 tcg_gen_extu_i32_i64(tcg_int
, tcg_dest
);
7769 case 3: /* float16 */
7770 tcg_single
= read_fp_sreg(s
, rn
);
7773 gen_helper_vfp_tosqh(tcg_int
, tcg_single
,
7774 tcg_shift
, tcg_fpstatus
);
7776 gen_helper_vfp_touqh(tcg_int
, tcg_single
,
7777 tcg_shift
, tcg_fpstatus
);
7780 TCGv_i32 tcg_dest
= tcg_temp_new_i32();
7782 gen_helper_vfp_toslh(tcg_dest
, tcg_single
,
7783 tcg_shift
, tcg_fpstatus
);
7785 gen_helper_vfp_toulh(tcg_dest
, tcg_single
,
7786 tcg_shift
, tcg_fpstatus
);
7788 tcg_gen_extu_i32_i64(tcg_int
, tcg_dest
);
7793 g_assert_not_reached();
7796 gen_restore_rmode(tcg_rmode
, tcg_fpstatus
);
7800 /* Floating point <-> fixed point conversions
7801 * 31 30 29 28 24 23 22 21 20 19 18 16 15 10 9 5 4 0
7802 * +----+---+---+-----------+------+---+-------+--------+-------+------+------+
7803 * | sf | 0 | S | 1 1 1 1 0 | type | 0 | rmode | opcode | scale | Rn | Rd |
7804 * +----+---+---+-----------+------+---+-------+--------+-------+------+------+
7806 static void disas_fp_fixed_conv(DisasContext
*s
, uint32_t insn
)
7808 int rd
= extract32(insn
, 0, 5);
7809 int rn
= extract32(insn
, 5, 5);
7810 int scale
= extract32(insn
, 10, 6);
7811 int opcode
= extract32(insn
, 16, 3);
7812 int rmode
= extract32(insn
, 19, 2);
7813 int type
= extract32(insn
, 22, 2);
7814 bool sbit
= extract32(insn
, 29, 1);
7815 bool sf
= extract32(insn
, 31, 1);
7818 if (sbit
|| (!sf
&& scale
< 32)) {
7819 unallocated_encoding(s
);
7824 case 0: /* float32 */
7825 case 1: /* float64 */
7827 case 3: /* float16 */
7828 if (dc_isar_feature(aa64_fp16
, s
)) {
7833 unallocated_encoding(s
);
7837 switch ((rmode
<< 3) | opcode
) {
7838 case 0x2: /* SCVTF */
7839 case 0x3: /* UCVTF */
7842 case 0x18: /* FCVTZS */
7843 case 0x19: /* FCVTZU */
7847 unallocated_encoding(s
);
7851 if (!fp_access_check(s
)) {
7855 handle_fpfpcvt(s
, rd
, rn
, opcode
, itof
, FPROUNDING_ZERO
, scale
, sf
, type
);
7858 static void handle_fmov(DisasContext
*s
, int rd
, int rn
, int type
, bool itof
)
7860 /* FMOV: gpr to or from float, double, or top half of quad fp reg,
7861 * without conversion.
7865 TCGv_i64 tcg_rn
= cpu_reg(s
, rn
);
7871 tmp
= tcg_temp_new_i64();
7872 tcg_gen_ext32u_i64(tmp
, tcg_rn
);
7873 write_fp_dreg(s
, rd
, tmp
);
7877 write_fp_dreg(s
, rd
, tcg_rn
);
7880 /* 64 bit to top half. */
7881 tcg_gen_st_i64(tcg_rn
, tcg_env
, fp_reg_hi_offset(s
, rd
));
7882 clear_vec_high(s
, true, rd
);
7886 tmp
= tcg_temp_new_i64();
7887 tcg_gen_ext16u_i64(tmp
, tcg_rn
);
7888 write_fp_dreg(s
, rd
, tmp
);
7891 g_assert_not_reached();
7894 TCGv_i64 tcg_rd
= cpu_reg(s
, rd
);
7899 tcg_gen_ld32u_i64(tcg_rd
, tcg_env
, fp_reg_offset(s
, rn
, MO_32
));
7903 tcg_gen_ld_i64(tcg_rd
, tcg_env
, fp_reg_offset(s
, rn
, MO_64
));
7906 /* 64 bits from top half */
7907 tcg_gen_ld_i64(tcg_rd
, tcg_env
, fp_reg_hi_offset(s
, rn
));
7911 tcg_gen_ld16u_i64(tcg_rd
, tcg_env
, fp_reg_offset(s
, rn
, MO_16
));
7914 g_assert_not_reached();
7919 static void handle_fjcvtzs(DisasContext
*s
, int rd
, int rn
)
7921 TCGv_i64 t
= read_fp_dreg(s
, rn
);
7922 TCGv_ptr fpstatus
= fpstatus_ptr(FPST_FPCR
);
7924 gen_helper_fjcvtzs(t
, t
, fpstatus
);
7926 tcg_gen_ext32u_i64(cpu_reg(s
, rd
), t
);
7927 tcg_gen_extrh_i64_i32(cpu_ZF
, t
);
7928 tcg_gen_movi_i32(cpu_CF
, 0);
7929 tcg_gen_movi_i32(cpu_NF
, 0);
7930 tcg_gen_movi_i32(cpu_VF
, 0);
7933 /* Floating point <-> integer conversions
7934 * 31 30 29 28 24 23 22 21 20 19 18 16 15 10 9 5 4 0
7935 * +----+---+---+-----------+------+---+-------+-----+-------------+----+----+
7936 * | sf | 0 | S | 1 1 1 1 0 | type | 1 | rmode | opc | 0 0 0 0 0 0 | Rn | Rd |
7937 * +----+---+---+-----------+------+---+-------+-----+-------------+----+----+
7939 static void disas_fp_int_conv(DisasContext
*s
, uint32_t insn
)
7941 int rd
= extract32(insn
, 0, 5);
7942 int rn
= extract32(insn
, 5, 5);
7943 int opcode
= extract32(insn
, 16, 3);
7944 int rmode
= extract32(insn
, 19, 2);
7945 int type
= extract32(insn
, 22, 2);
7946 bool sbit
= extract32(insn
, 29, 1);
7947 bool sf
= extract32(insn
, 31, 1);
7951 goto do_unallocated
;
7959 case 4: /* FCVTAS */
7960 case 5: /* FCVTAU */
7962 goto do_unallocated
;
7965 case 0: /* FCVT[NPMZ]S */
7966 case 1: /* FCVT[NPMZ]U */
7968 case 0: /* float32 */
7969 case 1: /* float64 */
7971 case 3: /* float16 */
7972 if (!dc_isar_feature(aa64_fp16
, s
)) {
7973 goto do_unallocated
;
7977 goto do_unallocated
;
7979 if (!fp_access_check(s
)) {
7982 handle_fpfpcvt(s
, rd
, rn
, opcode
, itof
, rmode
, 64, sf
, type
);
7986 switch (sf
<< 7 | type
<< 5 | rmode
<< 3 | opcode
) {
7987 case 0b01100110: /* FMOV half <-> 32-bit int */
7989 case 0b11100110: /* FMOV half <-> 64-bit int */
7991 if (!dc_isar_feature(aa64_fp16
, s
)) {
7992 goto do_unallocated
;
7995 case 0b00000110: /* FMOV 32-bit */
7997 case 0b10100110: /* FMOV 64-bit */
7999 case 0b11001110: /* FMOV top half of 128-bit */
8001 if (!fp_access_check(s
)) {
8005 handle_fmov(s
, rd
, rn
, type
, itof
);
8008 case 0b00111110: /* FJCVTZS */
8009 if (!dc_isar_feature(aa64_jscvt
, s
)) {
8010 goto do_unallocated
;
8011 } else if (fp_access_check(s
)) {
8012 handle_fjcvtzs(s
, rd
, rn
);
8018 unallocated_encoding(s
);
8025 /* FP-specific subcases of table C3-6 (SIMD and FP data processing)
8026 * 31 30 29 28 25 24 0
8027 * +---+---+---+---------+-----------------------------+
8028 * | | 0 | | 1 1 1 1 | |
8029 * +---+---+---+---------+-----------------------------+
8031 static void disas_data_proc_fp(DisasContext
*s
, uint32_t insn
)
8033 if (extract32(insn
, 24, 1)) {
8034 /* Floating point data-processing (3 source) */
8035 disas_fp_3src(s
, insn
);
8036 } else if (extract32(insn
, 21, 1) == 0) {
8037 /* Floating point to fixed point conversions */
8038 disas_fp_fixed_conv(s
, insn
);
8040 switch (extract32(insn
, 10, 2)) {
8042 /* Floating point conditional compare */
8043 disas_fp_ccomp(s
, insn
);
8046 /* Floating point data-processing (2 source) */
8047 unallocated_encoding(s
); /* in decodetree */
8050 /* Floating point conditional select */
8051 disas_fp_csel(s
, insn
);
8054 switch (ctz32(extract32(insn
, 12, 4))) {
8055 case 0: /* [15:12] == xxx1 */
8056 /* Floating point immediate */
8057 disas_fp_imm(s
, insn
);
8059 case 1: /* [15:12] == xx10 */
8060 /* Floating point compare */
8061 disas_fp_compare(s
, insn
);
8063 case 2: /* [15:12] == x100 */
8064 /* Floating point data-processing (1 source) */
8065 disas_fp_1src(s
, insn
);
8067 case 3: /* [15:12] == 1000 */
8068 unallocated_encoding(s
);
8070 default: /* [15:12] == 0000 */
8071 /* Floating point <-> integer conversions */
8072 disas_fp_int_conv(s
, insn
);
8080 static void do_ext64(DisasContext
*s
, TCGv_i64 tcg_left
, TCGv_i64 tcg_right
,
8083 /* Extract 64 bits from the middle of two concatenated 64 bit
8084 * vector register slices left:right. The extracted bits start
8085 * at 'pos' bits into the right (least significant) side.
8086 * We return the result in tcg_right, and guarantee not to
8089 TCGv_i64 tcg_tmp
= tcg_temp_new_i64();
8090 assert(pos
> 0 && pos
< 64);
8092 tcg_gen_shri_i64(tcg_right
, tcg_right
, pos
);
8093 tcg_gen_shli_i64(tcg_tmp
, tcg_left
, 64 - pos
);
8094 tcg_gen_or_i64(tcg_right
, tcg_right
, tcg_tmp
);
8098 * 31 30 29 24 23 22 21 20 16 15 14 11 10 9 5 4 0
8099 * +---+---+-------------+-----+---+------+---+------+---+------+------+
8100 * | 0 | Q | 1 0 1 1 1 0 | op2 | 0 | Rm | 0 | imm4 | 0 | Rn | Rd |
8101 * +---+---+-------------+-----+---+------+---+------+---+------+------+
8103 static void disas_simd_ext(DisasContext
*s
, uint32_t insn
)
8105 int is_q
= extract32(insn
, 30, 1);
8106 int op2
= extract32(insn
, 22, 2);
8107 int imm4
= extract32(insn
, 11, 4);
8108 int rm
= extract32(insn
, 16, 5);
8109 int rn
= extract32(insn
, 5, 5);
8110 int rd
= extract32(insn
, 0, 5);
8111 int pos
= imm4
<< 3;
8112 TCGv_i64 tcg_resl
, tcg_resh
;
8114 if (op2
!= 0 || (!is_q
&& extract32(imm4
, 3, 1))) {
8115 unallocated_encoding(s
);
8119 if (!fp_access_check(s
)) {
8123 tcg_resh
= tcg_temp_new_i64();
8124 tcg_resl
= tcg_temp_new_i64();
8126 /* Vd gets bits starting at pos bits into Vm:Vn. This is
8127 * either extracting 128 bits from a 128:128 concatenation, or
8128 * extracting 64 bits from a 64:64 concatenation.
8131 read_vec_element(s
, tcg_resl
, rn
, 0, MO_64
);
8133 read_vec_element(s
, tcg_resh
, rm
, 0, MO_64
);
8134 do_ext64(s
, tcg_resh
, tcg_resl
, pos
);
8142 EltPosns eltposns
[] = { {rn
, 0}, {rn
, 1}, {rm
, 0}, {rm
, 1} };
8143 EltPosns
*elt
= eltposns
;
8150 read_vec_element(s
, tcg_resl
, elt
->reg
, elt
->elt
, MO_64
);
8152 read_vec_element(s
, tcg_resh
, elt
->reg
, elt
->elt
, MO_64
);
8155 do_ext64(s
, tcg_resh
, tcg_resl
, pos
);
8156 tcg_hh
= tcg_temp_new_i64();
8157 read_vec_element(s
, tcg_hh
, elt
->reg
, elt
->elt
, MO_64
);
8158 do_ext64(s
, tcg_hh
, tcg_resh
, pos
);
8162 write_vec_element(s
, tcg_resl
, rd
, 0, MO_64
);
8164 write_vec_element(s
, tcg_resh
, rd
, 1, MO_64
);
8166 clear_vec_high(s
, is_q
, rd
);
8170 * 31 30 29 24 23 22 21 20 16 15 14 13 12 11 10 9 5 4 0
8171 * +---+---+-------------+-----+---+------+---+-----+----+-----+------+------+
8172 * | 0 | Q | 0 0 1 1 1 0 | op2 | 0 | Rm | 0 | len | op | 0 0 | Rn | Rd |
8173 * +---+---+-------------+-----+---+------+---+-----+----+-----+------+------+
8175 static void disas_simd_tb(DisasContext
*s
, uint32_t insn
)
8177 int op2
= extract32(insn
, 22, 2);
8178 int is_q
= extract32(insn
, 30, 1);
8179 int rm
= extract32(insn
, 16, 5);
8180 int rn
= extract32(insn
, 5, 5);
8181 int rd
= extract32(insn
, 0, 5);
8182 int is_tbx
= extract32(insn
, 12, 1);
8183 int len
= (extract32(insn
, 13, 2) + 1) * 16;
8186 unallocated_encoding(s
);
8190 if (!fp_access_check(s
)) {
8194 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s
, rd
),
8195 vec_full_reg_offset(s
, rm
), tcg_env
,
8196 is_q
? 16 : 8, vec_full_reg_size(s
),
8197 (len
<< 6) | (is_tbx
<< 5) | rn
,
8198 gen_helper_simd_tblx
);
8202 * 31 30 29 24 23 22 21 20 16 15 14 12 11 10 9 5 4 0
8203 * +---+---+-------------+------+---+------+---+------------------+------+
8204 * | 0 | Q | 0 0 1 1 1 0 | size | 0 | Rm | 0 | opc | 1 0 | Rn | Rd |
8205 * +---+---+-------------+------+---+------+---+------------------+------+
8207 static void disas_simd_zip_trn(DisasContext
*s
, uint32_t insn
)
8209 int rd
= extract32(insn
, 0, 5);
8210 int rn
= extract32(insn
, 5, 5);
8211 int rm
= extract32(insn
, 16, 5);
8212 int size
= extract32(insn
, 22, 2);
8213 /* opc field bits [1:0] indicate ZIP/UZP/TRN;
8214 * bit 2 indicates 1 vs 2 variant of the insn.
8216 int opcode
= extract32(insn
, 12, 2);
8217 bool part
= extract32(insn
, 14, 1);
8218 bool is_q
= extract32(insn
, 30, 1);
8219 int esize
= 8 << size
;
8221 int datasize
= is_q
? 128 : 64;
8222 int elements
= datasize
/ esize
;
8223 TCGv_i64 tcg_res
[2], tcg_ele
;
8225 if (opcode
== 0 || (size
== 3 && !is_q
)) {
8226 unallocated_encoding(s
);
8230 if (!fp_access_check(s
)) {
8234 tcg_res
[0] = tcg_temp_new_i64();
8235 tcg_res
[1] = is_q
? tcg_temp_new_i64() : NULL
;
8236 tcg_ele
= tcg_temp_new_i64();
8238 for (i
= 0; i
< elements
; i
++) {
8242 case 1: /* UZP1/2 */
8244 int midpoint
= elements
/ 2;
8246 read_vec_element(s
, tcg_ele
, rn
, 2 * i
+ part
, size
);
8248 read_vec_element(s
, tcg_ele
, rm
,
8249 2 * (i
- midpoint
) + part
, size
);
8253 case 2: /* TRN1/2 */
8255 read_vec_element(s
, tcg_ele
, rm
, (i
& ~1) + part
, size
);
8257 read_vec_element(s
, tcg_ele
, rn
, (i
& ~1) + part
, size
);
8260 case 3: /* ZIP1/2 */
8262 int base
= part
* elements
/ 2;
8264 read_vec_element(s
, tcg_ele
, rm
, base
+ (i
>> 1), size
);
8266 read_vec_element(s
, tcg_ele
, rn
, base
+ (i
>> 1), size
);
8271 g_assert_not_reached();
8274 w
= (i
* esize
) / 64;
8275 o
= (i
* esize
) % 64;
8277 tcg_gen_mov_i64(tcg_res
[w
], tcg_ele
);
8279 tcg_gen_shli_i64(tcg_ele
, tcg_ele
, o
);
8280 tcg_gen_or_i64(tcg_res
[w
], tcg_res
[w
], tcg_ele
);
8284 for (i
= 0; i
<= is_q
; ++i
) {
8285 write_vec_element(s
, tcg_res
[i
], rd
, i
, MO_64
);
8287 clear_vec_high(s
, is_q
, rd
);
8291 * do_reduction_op helper
8293 * This mirrors the Reduce() pseudocode in the ARM ARM. It is
8294 * important for correct NaN propagation that we do these
8295 * operations in exactly the order specified by the pseudocode.
8297 * This is a recursive function, TCG temps should be freed by the
8298 * calling function once it is done with the values.
8300 static TCGv_i32
do_reduction_op(DisasContext
*s
, int fpopcode
, int rn
,
8301 int esize
, int size
, int vmap
, TCGv_ptr fpst
)
8303 if (esize
== size
) {
8305 MemOp msize
= esize
== 16 ? MO_16
: MO_32
;
8308 /* We should have one register left here */
8309 assert(ctpop8(vmap
) == 1);
8310 element
= ctz32(vmap
);
8311 assert(element
< 8);
8313 tcg_elem
= tcg_temp_new_i32();
8314 read_vec_element_i32(s
, tcg_elem
, rn
, element
, msize
);
8317 int bits
= size
/ 2;
8318 int shift
= ctpop8(vmap
) / 2;
8319 int vmap_lo
= (vmap
>> shift
) & vmap
;
8320 int vmap_hi
= (vmap
& ~vmap_lo
);
8321 TCGv_i32 tcg_hi
, tcg_lo
, tcg_res
;
8323 tcg_hi
= do_reduction_op(s
, fpopcode
, rn
, esize
, bits
, vmap_hi
, fpst
);
8324 tcg_lo
= do_reduction_op(s
, fpopcode
, rn
, esize
, bits
, vmap_lo
, fpst
);
8325 tcg_res
= tcg_temp_new_i32();
8328 case 0x0c: /* fmaxnmv half-precision */
8329 gen_helper_advsimd_maxnumh(tcg_res
, tcg_lo
, tcg_hi
, fpst
);
8331 case 0x0f: /* fmaxv half-precision */
8332 gen_helper_advsimd_maxh(tcg_res
, tcg_lo
, tcg_hi
, fpst
);
8334 case 0x1c: /* fminnmv half-precision */
8335 gen_helper_advsimd_minnumh(tcg_res
, tcg_lo
, tcg_hi
, fpst
);
8337 case 0x1f: /* fminv half-precision */
8338 gen_helper_advsimd_minh(tcg_res
, tcg_lo
, tcg_hi
, fpst
);
8340 case 0x2c: /* fmaxnmv */
8341 gen_helper_vfp_maxnums(tcg_res
, tcg_lo
, tcg_hi
, fpst
);
8343 case 0x2f: /* fmaxv */
8344 gen_helper_vfp_maxs(tcg_res
, tcg_lo
, tcg_hi
, fpst
);
8346 case 0x3c: /* fminnmv */
8347 gen_helper_vfp_minnums(tcg_res
, tcg_lo
, tcg_hi
, fpst
);
8349 case 0x3f: /* fminv */
8350 gen_helper_vfp_mins(tcg_res
, tcg_lo
, tcg_hi
, fpst
);
8353 g_assert_not_reached();
8359 /* AdvSIMD across lanes
8360 * 31 30 29 28 24 23 22 21 17 16 12 11 10 9 5 4 0
8361 * +---+---+---+-----------+------+-----------+--------+-----+------+------+
8362 * | 0 | Q | U | 0 1 1 1 0 | size | 1 1 0 0 0 | opcode | 1 0 | Rn | Rd |
8363 * +---+---+---+-----------+------+-----------+--------+-----+------+------+
8365 static void disas_simd_across_lanes(DisasContext
*s
, uint32_t insn
)
8367 int rd
= extract32(insn
, 0, 5);
8368 int rn
= extract32(insn
, 5, 5);
8369 int size
= extract32(insn
, 22, 2);
8370 int opcode
= extract32(insn
, 12, 5);
8371 bool is_q
= extract32(insn
, 30, 1);
8372 bool is_u
= extract32(insn
, 29, 1);
8374 bool is_min
= false;
8378 TCGv_i64 tcg_res
, tcg_elt
;
8381 case 0x1b: /* ADDV */
8383 unallocated_encoding(s
);
8387 case 0x3: /* SADDLV, UADDLV */
8388 case 0xa: /* SMAXV, UMAXV */
8389 case 0x1a: /* SMINV, UMINV */
8390 if (size
== 3 || (size
== 2 && !is_q
)) {
8391 unallocated_encoding(s
);
8395 case 0xc: /* FMAXNMV, FMINNMV */
8396 case 0xf: /* FMAXV, FMINV */
8397 /* Bit 1 of size field encodes min vs max and the actual size
8398 * depends on the encoding of the U bit. If not set (and FP16
8399 * enabled) then we do half-precision float instead of single
8402 is_min
= extract32(size
, 1, 1);
8404 if (!is_u
&& dc_isar_feature(aa64_fp16
, s
)) {
8406 } else if (!is_u
|| !is_q
|| extract32(size
, 0, 1)) {
8407 unallocated_encoding(s
);
8414 unallocated_encoding(s
);
8418 if (!fp_access_check(s
)) {
8423 elements
= (is_q
? 128 : 64) / esize
;
8425 tcg_res
= tcg_temp_new_i64();
8426 tcg_elt
= tcg_temp_new_i64();
8428 /* These instructions operate across all lanes of a vector
8429 * to produce a single result. We can guarantee that a 64
8430 * bit intermediate is sufficient:
8431 * + for [US]ADDLV the maximum element size is 32 bits, and
8432 * the result type is 64 bits
8433 * + for FMAX*V, FMIN*V, ADDV the intermediate type is the
8434 * same as the element size, which is 32 bits at most
8435 * For the integer operations we can choose to work at 64
8436 * or 32 bits and truncate at the end; for simplicity
8437 * we use 64 bits always. The floating point
8438 * ops do require 32 bit intermediates, though.
8441 read_vec_element(s
, tcg_res
, rn
, 0, size
| (is_u
? 0 : MO_SIGN
));
8443 for (i
= 1; i
< elements
; i
++) {
8444 read_vec_element(s
, tcg_elt
, rn
, i
, size
| (is_u
? 0 : MO_SIGN
));
8447 case 0x03: /* SADDLV / UADDLV */
8448 case 0x1b: /* ADDV */
8449 tcg_gen_add_i64(tcg_res
, tcg_res
, tcg_elt
);
8451 case 0x0a: /* SMAXV / UMAXV */
8453 tcg_gen_umax_i64(tcg_res
, tcg_res
, tcg_elt
);
8455 tcg_gen_smax_i64(tcg_res
, tcg_res
, tcg_elt
);
8458 case 0x1a: /* SMINV / UMINV */
8460 tcg_gen_umin_i64(tcg_res
, tcg_res
, tcg_elt
);
8462 tcg_gen_smin_i64(tcg_res
, tcg_res
, tcg_elt
);
8466 g_assert_not_reached();
8471 /* Floating point vector reduction ops which work across 32
8472 * bit (single) or 16 bit (half-precision) intermediates.
8473 * Note that correct NaN propagation requires that we do these
8474 * operations in exactly the order specified by the pseudocode.
8476 TCGv_ptr fpst
= fpstatus_ptr(size
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
8477 int fpopcode
= opcode
| is_min
<< 4 | is_u
<< 5;
8478 int vmap
= (1 << elements
) - 1;
8479 TCGv_i32 tcg_res32
= do_reduction_op(s
, fpopcode
, rn
, esize
,
8480 (is_q
? 128 : 64), vmap
, fpst
);
8481 tcg_gen_extu_i32_i64(tcg_res
, tcg_res32
);
8484 /* Now truncate the result to the width required for the final output */
8485 if (opcode
== 0x03) {
8486 /* SADDLV, UADDLV: result is 2*esize */
8492 tcg_gen_ext8u_i64(tcg_res
, tcg_res
);
8495 tcg_gen_ext16u_i64(tcg_res
, tcg_res
);
8498 tcg_gen_ext32u_i64(tcg_res
, tcg_res
);
8503 g_assert_not_reached();
8506 write_fp_dreg(s
, rd
, tcg_res
);
8509 /* AdvSIMD modified immediate
8510 * 31 30 29 28 19 18 16 15 12 11 10 9 5 4 0
8511 * +---+---+----+---------------------+-----+-------+----+---+-------+------+
8512 * | 0 | Q | op | 0 1 1 1 1 0 0 0 0 0 | abc | cmode | o2 | 1 | defgh | Rd |
8513 * +---+---+----+---------------------+-----+-------+----+---+-------+------+
8515 * There are a number of operations that can be carried out here:
8516 * MOVI - move (shifted) imm into register
8517 * MVNI - move inverted (shifted) imm into register
8518 * ORR - bitwise OR of (shifted) imm with register
8519 * BIC - bitwise clear of (shifted) imm with register
8520 * With ARMv8.2 we also have:
8521 * FMOV half-precision
8523 static void disas_simd_mod_imm(DisasContext
*s
, uint32_t insn
)
8525 int rd
= extract32(insn
, 0, 5);
8526 int cmode
= extract32(insn
, 12, 4);
8527 int o2
= extract32(insn
, 11, 1);
8528 uint64_t abcdefgh
= extract32(insn
, 5, 5) | (extract32(insn
, 16, 3) << 5);
8529 bool is_neg
= extract32(insn
, 29, 1);
8530 bool is_q
= extract32(insn
, 30, 1);
8534 if (cmode
!= 0xf || is_neg
) {
8535 unallocated_encoding(s
);
8538 /* FMOV (vector, immediate) - half-precision */
8539 if (!dc_isar_feature(aa64_fp16
, s
)) {
8540 unallocated_encoding(s
);
8543 imm
= vfp_expand_imm(MO_16
, abcdefgh
);
8544 /* now duplicate across the lanes */
8545 imm
= dup_const(MO_16
, imm
);
8547 if (cmode
== 0xf && is_neg
&& !is_q
) {
8548 unallocated_encoding(s
);
8551 imm
= asimd_imm_const(abcdefgh
, cmode
, is_neg
);
8554 if (!fp_access_check(s
)) {
8558 if (!((cmode
& 0x9) == 0x1 || (cmode
& 0xd) == 0x9)) {
8559 /* MOVI or MVNI, with MVNI negation handled above. */
8560 tcg_gen_gvec_dup_imm(MO_64
, vec_full_reg_offset(s
, rd
), is_q
? 16 : 8,
8561 vec_full_reg_size(s
), imm
);
8563 /* ORR or BIC, with BIC negation to AND handled above. */
8565 gen_gvec_fn2i(s
, is_q
, rd
, rd
, imm
, tcg_gen_gvec_andi
, MO_64
);
8567 gen_gvec_fn2i(s
, is_q
, rd
, rd
, imm
, tcg_gen_gvec_ori
, MO_64
);
8573 * Common SSHR[RA]/USHR[RA] - Shift right (optional rounding/accumulate)
8575 * This code is handles the common shifting code and is used by both
8576 * the vector and scalar code.
8578 static void handle_shri_with_rndacc(TCGv_i64 tcg_res
, TCGv_i64 tcg_src
,
8579 TCGv_i64 tcg_rnd
, bool accumulate
,
8580 bool is_u
, int size
, int shift
)
8582 bool extended_result
= false;
8583 bool round
= tcg_rnd
!= NULL
;
8585 TCGv_i64 tcg_src_hi
;
8587 if (round
&& size
== 3) {
8588 extended_result
= true;
8589 ext_lshift
= 64 - shift
;
8590 tcg_src_hi
= tcg_temp_new_i64();
8591 } else if (shift
== 64) {
8592 if (!accumulate
&& is_u
) {
8593 /* result is zero */
8594 tcg_gen_movi_i64(tcg_res
, 0);
8599 /* Deal with the rounding step */
8601 if (extended_result
) {
8602 TCGv_i64 tcg_zero
= tcg_constant_i64(0);
8604 /* take care of sign extending tcg_res */
8605 tcg_gen_sari_i64(tcg_src_hi
, tcg_src
, 63);
8606 tcg_gen_add2_i64(tcg_src
, tcg_src_hi
,
8607 tcg_src
, tcg_src_hi
,
8610 tcg_gen_add2_i64(tcg_src
, tcg_src_hi
,
8615 tcg_gen_add_i64(tcg_src
, tcg_src
, tcg_rnd
);
8619 /* Now do the shift right */
8620 if (round
&& extended_result
) {
8621 /* extended case, >64 bit precision required */
8622 if (ext_lshift
== 0) {
8623 /* special case, only high bits matter */
8624 tcg_gen_mov_i64(tcg_src
, tcg_src_hi
);
8626 tcg_gen_shri_i64(tcg_src
, tcg_src
, shift
);
8627 tcg_gen_shli_i64(tcg_src_hi
, tcg_src_hi
, ext_lshift
);
8628 tcg_gen_or_i64(tcg_src
, tcg_src
, tcg_src_hi
);
8633 /* essentially shifting in 64 zeros */
8634 tcg_gen_movi_i64(tcg_src
, 0);
8636 tcg_gen_shri_i64(tcg_src
, tcg_src
, shift
);
8640 /* effectively extending the sign-bit */
8641 tcg_gen_sari_i64(tcg_src
, tcg_src
, 63);
8643 tcg_gen_sari_i64(tcg_src
, tcg_src
, shift
);
8649 tcg_gen_add_i64(tcg_res
, tcg_res
, tcg_src
);
8651 tcg_gen_mov_i64(tcg_res
, tcg_src
);
8655 /* SSHR[RA]/USHR[RA] - Scalar shift right (optional rounding/accumulate) */
8656 static void handle_scalar_simd_shri(DisasContext
*s
,
8657 bool is_u
, int immh
, int immb
,
8658 int opcode
, int rn
, int rd
)
8661 int immhb
= immh
<< 3 | immb
;
8662 int shift
= 2 * (8 << size
) - immhb
;
8663 bool accumulate
= false;
8665 bool insert
= false;
8670 if (!extract32(immh
, 3, 1)) {
8671 unallocated_encoding(s
);
8675 if (!fp_access_check(s
)) {
8680 case 0x02: /* SSRA / USRA (accumulate) */
8683 case 0x04: /* SRSHR / URSHR (rounding) */
8686 case 0x06: /* SRSRA / URSRA (accum + rounding) */
8687 accumulate
= round
= true;
8689 case 0x08: /* SRI */
8695 tcg_round
= tcg_constant_i64(1ULL << (shift
- 1));
8700 tcg_rn
= read_fp_dreg(s
, rn
);
8701 tcg_rd
= (accumulate
|| insert
) ? read_fp_dreg(s
, rd
) : tcg_temp_new_i64();
8704 /* shift count same as element size is valid but does nothing;
8705 * special case to avoid potential shift by 64.
8707 int esize
= 8 << size
;
8708 if (shift
!= esize
) {
8709 tcg_gen_shri_i64(tcg_rn
, tcg_rn
, shift
);
8710 tcg_gen_deposit_i64(tcg_rd
, tcg_rd
, tcg_rn
, 0, esize
- shift
);
8713 handle_shri_with_rndacc(tcg_rd
, tcg_rn
, tcg_round
,
8714 accumulate
, is_u
, size
, shift
);
8717 write_fp_dreg(s
, rd
, tcg_rd
);
8720 /* SHL/SLI - Scalar shift left */
8721 static void handle_scalar_simd_shli(DisasContext
*s
, bool insert
,
8722 int immh
, int immb
, int opcode
,
8725 int size
= 32 - clz32(immh
) - 1;
8726 int immhb
= immh
<< 3 | immb
;
8727 int shift
= immhb
- (8 << size
);
8731 if (!extract32(immh
, 3, 1)) {
8732 unallocated_encoding(s
);
8736 if (!fp_access_check(s
)) {
8740 tcg_rn
= read_fp_dreg(s
, rn
);
8741 tcg_rd
= insert
? read_fp_dreg(s
, rd
) : tcg_temp_new_i64();
8744 tcg_gen_deposit_i64(tcg_rd
, tcg_rd
, tcg_rn
, shift
, 64 - shift
);
8746 tcg_gen_shli_i64(tcg_rd
, tcg_rn
, shift
);
8749 write_fp_dreg(s
, rd
, tcg_rd
);
8752 /* SQSHRN/SQSHRUN - Saturating (signed/unsigned) shift right with
8753 * (signed/unsigned) narrowing */
8754 static void handle_vec_simd_sqshrn(DisasContext
*s
, bool is_scalar
, bool is_q
,
8755 bool is_u_shift
, bool is_u_narrow
,
8756 int immh
, int immb
, int opcode
,
8759 int immhb
= immh
<< 3 | immb
;
8760 int size
= 32 - clz32(immh
) - 1;
8761 int esize
= 8 << size
;
8762 int shift
= (2 * esize
) - immhb
;
8763 int elements
= is_scalar
? 1 : (64 / esize
);
8764 bool round
= extract32(opcode
, 0, 1);
8765 MemOp ldop
= (size
+ 1) | (is_u_shift
? 0 : MO_SIGN
);
8766 TCGv_i64 tcg_rn
, tcg_rd
, tcg_round
;
8767 TCGv_i32 tcg_rd_narrowed
;
8770 static NeonGenNarrowEnvFn
* const signed_narrow_fns
[4][2] = {
8771 { gen_helper_neon_narrow_sat_s8
,
8772 gen_helper_neon_unarrow_sat8
},
8773 { gen_helper_neon_narrow_sat_s16
,
8774 gen_helper_neon_unarrow_sat16
},
8775 { gen_helper_neon_narrow_sat_s32
,
8776 gen_helper_neon_unarrow_sat32
},
8779 static NeonGenNarrowEnvFn
* const unsigned_narrow_fns
[4] = {
8780 gen_helper_neon_narrow_sat_u8
,
8781 gen_helper_neon_narrow_sat_u16
,
8782 gen_helper_neon_narrow_sat_u32
,
8785 NeonGenNarrowEnvFn
*narrowfn
;
8791 if (extract32(immh
, 3, 1)) {
8792 unallocated_encoding(s
);
8796 if (!fp_access_check(s
)) {
8801 narrowfn
= unsigned_narrow_fns
[size
];
8803 narrowfn
= signed_narrow_fns
[size
][is_u_narrow
? 1 : 0];
8806 tcg_rn
= tcg_temp_new_i64();
8807 tcg_rd
= tcg_temp_new_i64();
8808 tcg_rd_narrowed
= tcg_temp_new_i32();
8809 tcg_final
= tcg_temp_new_i64();
8812 tcg_round
= tcg_constant_i64(1ULL << (shift
- 1));
8817 for (i
= 0; i
< elements
; i
++) {
8818 read_vec_element(s
, tcg_rn
, rn
, i
, ldop
);
8819 handle_shri_with_rndacc(tcg_rd
, tcg_rn
, tcg_round
,
8820 false, is_u_shift
, size
+1, shift
);
8821 narrowfn(tcg_rd_narrowed
, tcg_env
, tcg_rd
);
8822 tcg_gen_extu_i32_i64(tcg_rd
, tcg_rd_narrowed
);
8824 tcg_gen_extract_i64(tcg_final
, tcg_rd
, 0, esize
);
8826 tcg_gen_deposit_i64(tcg_final
, tcg_final
, tcg_rd
, esize
* i
, esize
);
8831 write_vec_element(s
, tcg_final
, rd
, 0, MO_64
);
8833 write_vec_element(s
, tcg_final
, rd
, 1, MO_64
);
8835 clear_vec_high(s
, is_q
, rd
);
8838 /* SQSHLU, UQSHL, SQSHL: saturating left shifts */
8839 static void handle_simd_qshl(DisasContext
*s
, bool scalar
, bool is_q
,
8840 bool src_unsigned
, bool dst_unsigned
,
8841 int immh
, int immb
, int rn
, int rd
)
8843 int immhb
= immh
<< 3 | immb
;
8844 int size
= 32 - clz32(immh
) - 1;
8845 int shift
= immhb
- (8 << size
);
8849 assert(!(scalar
&& is_q
));
8852 if (!is_q
&& extract32(immh
, 3, 1)) {
8853 unallocated_encoding(s
);
8857 /* Since we use the variable-shift helpers we must
8858 * replicate the shift count into each element of
8859 * the tcg_shift value.
8863 shift
|= shift
<< 8;
8866 shift
|= shift
<< 16;
8872 g_assert_not_reached();
8876 if (!fp_access_check(s
)) {
8881 TCGv_i64 tcg_shift
= tcg_constant_i64(shift
);
8882 static NeonGenTwo64OpEnvFn
* const fns
[2][2] = {
8883 { gen_helper_neon_qshl_s64
, gen_helper_neon_qshlu_s64
},
8884 { NULL
, gen_helper_neon_qshl_u64
},
8886 NeonGenTwo64OpEnvFn
*genfn
= fns
[src_unsigned
][dst_unsigned
];
8887 int maxpass
= is_q
? 2 : 1;
8889 for (pass
= 0; pass
< maxpass
; pass
++) {
8890 TCGv_i64 tcg_op
= tcg_temp_new_i64();
8892 read_vec_element(s
, tcg_op
, rn
, pass
, MO_64
);
8893 genfn(tcg_op
, tcg_env
, tcg_op
, tcg_shift
);
8894 write_vec_element(s
, tcg_op
, rd
, pass
, MO_64
);
8896 clear_vec_high(s
, is_q
, rd
);
8898 TCGv_i32 tcg_shift
= tcg_constant_i32(shift
);
8899 static NeonGenTwoOpEnvFn
* const fns
[2][2][3] = {
8901 { gen_helper_neon_qshl_s8
,
8902 gen_helper_neon_qshl_s16
,
8903 gen_helper_neon_qshl_s32
},
8904 { gen_helper_neon_qshlu_s8
,
8905 gen_helper_neon_qshlu_s16
,
8906 gen_helper_neon_qshlu_s32
}
8908 { NULL
, NULL
, NULL
},
8909 { gen_helper_neon_qshl_u8
,
8910 gen_helper_neon_qshl_u16
,
8911 gen_helper_neon_qshl_u32
}
8914 NeonGenTwoOpEnvFn
*genfn
= fns
[src_unsigned
][dst_unsigned
][size
];
8915 MemOp memop
= scalar
? size
: MO_32
;
8916 int maxpass
= scalar
? 1 : is_q
? 4 : 2;
8918 for (pass
= 0; pass
< maxpass
; pass
++) {
8919 TCGv_i32 tcg_op
= tcg_temp_new_i32();
8921 read_vec_element_i32(s
, tcg_op
, rn
, pass
, memop
);
8922 genfn(tcg_op
, tcg_env
, tcg_op
, tcg_shift
);
8926 tcg_gen_ext8u_i32(tcg_op
, tcg_op
);
8929 tcg_gen_ext16u_i32(tcg_op
, tcg_op
);
8934 g_assert_not_reached();
8936 write_fp_sreg(s
, rd
, tcg_op
);
8938 write_vec_element_i32(s
, tcg_op
, rd
, pass
, MO_32
);
8943 clear_vec_high(s
, is_q
, rd
);
8948 /* Common vector code for handling integer to FP conversion */
8949 static void handle_simd_intfp_conv(DisasContext
*s
, int rd
, int rn
,
8950 int elements
, int is_signed
,
8951 int fracbits
, int size
)
8953 TCGv_ptr tcg_fpst
= fpstatus_ptr(size
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
8954 TCGv_i32 tcg_shift
= NULL
;
8956 MemOp mop
= size
| (is_signed
? MO_SIGN
: 0);
8959 if (fracbits
|| size
== MO_64
) {
8960 tcg_shift
= tcg_constant_i32(fracbits
);
8963 if (size
== MO_64
) {
8964 TCGv_i64 tcg_int64
= tcg_temp_new_i64();
8965 TCGv_i64 tcg_double
= tcg_temp_new_i64();
8967 for (pass
= 0; pass
< elements
; pass
++) {
8968 read_vec_element(s
, tcg_int64
, rn
, pass
, mop
);
8971 gen_helper_vfp_sqtod(tcg_double
, tcg_int64
,
8972 tcg_shift
, tcg_fpst
);
8974 gen_helper_vfp_uqtod(tcg_double
, tcg_int64
,
8975 tcg_shift
, tcg_fpst
);
8977 if (elements
== 1) {
8978 write_fp_dreg(s
, rd
, tcg_double
);
8980 write_vec_element(s
, tcg_double
, rd
, pass
, MO_64
);
8984 TCGv_i32 tcg_int32
= tcg_temp_new_i32();
8985 TCGv_i32 tcg_float
= tcg_temp_new_i32();
8987 for (pass
= 0; pass
< elements
; pass
++) {
8988 read_vec_element_i32(s
, tcg_int32
, rn
, pass
, mop
);
8994 gen_helper_vfp_sltos(tcg_float
, tcg_int32
,
8995 tcg_shift
, tcg_fpst
);
8997 gen_helper_vfp_ultos(tcg_float
, tcg_int32
,
8998 tcg_shift
, tcg_fpst
);
9002 gen_helper_vfp_sitos(tcg_float
, tcg_int32
, tcg_fpst
);
9004 gen_helper_vfp_uitos(tcg_float
, tcg_int32
, tcg_fpst
);
9011 gen_helper_vfp_sltoh(tcg_float
, tcg_int32
,
9012 tcg_shift
, tcg_fpst
);
9014 gen_helper_vfp_ultoh(tcg_float
, tcg_int32
,
9015 tcg_shift
, tcg_fpst
);
9019 gen_helper_vfp_sitoh(tcg_float
, tcg_int32
, tcg_fpst
);
9021 gen_helper_vfp_uitoh(tcg_float
, tcg_int32
, tcg_fpst
);
9026 g_assert_not_reached();
9029 if (elements
== 1) {
9030 write_fp_sreg(s
, rd
, tcg_float
);
9032 write_vec_element_i32(s
, tcg_float
, rd
, pass
, size
);
9037 clear_vec_high(s
, elements
<< size
== 16, rd
);
9040 /* UCVTF/SCVTF - Integer to FP conversion */
9041 static void handle_simd_shift_intfp_conv(DisasContext
*s
, bool is_scalar
,
9042 bool is_q
, bool is_u
,
9043 int immh
, int immb
, int opcode
,
9046 int size
, elements
, fracbits
;
9047 int immhb
= immh
<< 3 | immb
;
9051 if (!is_scalar
&& !is_q
) {
9052 unallocated_encoding(s
);
9055 } else if (immh
& 4) {
9057 } else if (immh
& 2) {
9059 if (!dc_isar_feature(aa64_fp16
, s
)) {
9060 unallocated_encoding(s
);
9064 /* immh == 0 would be a failure of the decode logic */
9065 g_assert(immh
== 1);
9066 unallocated_encoding(s
);
9073 elements
= (8 << is_q
) >> size
;
9075 fracbits
= (16 << size
) - immhb
;
9077 if (!fp_access_check(s
)) {
9081 handle_simd_intfp_conv(s
, rd
, rn
, elements
, !is_u
, fracbits
, size
);
9084 /* FCVTZS, FVCVTZU - FP to fixedpoint conversion */
9085 static void handle_simd_shift_fpint_conv(DisasContext
*s
, bool is_scalar
,
9086 bool is_q
, bool is_u
,
9087 int immh
, int immb
, int rn
, int rd
)
9089 int immhb
= immh
<< 3 | immb
;
9090 int pass
, size
, fracbits
;
9091 TCGv_ptr tcg_fpstatus
;
9092 TCGv_i32 tcg_rmode
, tcg_shift
;
9096 if (!is_scalar
&& !is_q
) {
9097 unallocated_encoding(s
);
9100 } else if (immh
& 0x4) {
9102 } else if (immh
& 0x2) {
9104 if (!dc_isar_feature(aa64_fp16
, s
)) {
9105 unallocated_encoding(s
);
9109 /* Should have split out AdvSIMD modified immediate earlier. */
9111 unallocated_encoding(s
);
9115 if (!fp_access_check(s
)) {
9119 assert(!(is_scalar
&& is_q
));
9121 tcg_fpstatus
= fpstatus_ptr(size
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
9122 tcg_rmode
= gen_set_rmode(FPROUNDING_ZERO
, tcg_fpstatus
);
9123 fracbits
= (16 << size
) - immhb
;
9124 tcg_shift
= tcg_constant_i32(fracbits
);
9126 if (size
== MO_64
) {
9127 int maxpass
= is_scalar
? 1 : 2;
9129 for (pass
= 0; pass
< maxpass
; pass
++) {
9130 TCGv_i64 tcg_op
= tcg_temp_new_i64();
9132 read_vec_element(s
, tcg_op
, rn
, pass
, MO_64
);
9134 gen_helper_vfp_touqd(tcg_op
, tcg_op
, tcg_shift
, tcg_fpstatus
);
9136 gen_helper_vfp_tosqd(tcg_op
, tcg_op
, tcg_shift
, tcg_fpstatus
);
9138 write_vec_element(s
, tcg_op
, rd
, pass
, MO_64
);
9140 clear_vec_high(s
, is_q
, rd
);
9142 void (*fn
)(TCGv_i32
, TCGv_i32
, TCGv_i32
, TCGv_ptr
);
9143 int maxpass
= is_scalar
? 1 : ((8 << is_q
) >> size
);
9148 fn
= gen_helper_vfp_touhh
;
9150 fn
= gen_helper_vfp_toshh
;
9155 fn
= gen_helper_vfp_touls
;
9157 fn
= gen_helper_vfp_tosls
;
9161 g_assert_not_reached();
9164 for (pass
= 0; pass
< maxpass
; pass
++) {
9165 TCGv_i32 tcg_op
= tcg_temp_new_i32();
9167 read_vec_element_i32(s
, tcg_op
, rn
, pass
, size
);
9168 fn(tcg_op
, tcg_op
, tcg_shift
, tcg_fpstatus
);
9170 if (size
== MO_16
&& !is_u
) {
9171 tcg_gen_ext16u_i32(tcg_op
, tcg_op
);
9173 write_fp_sreg(s
, rd
, tcg_op
);
9175 write_vec_element_i32(s
, tcg_op
, rd
, pass
, size
);
9179 clear_vec_high(s
, is_q
, rd
);
9183 gen_restore_rmode(tcg_rmode
, tcg_fpstatus
);
9186 /* AdvSIMD scalar shift by immediate
9187 * 31 30 29 28 23 22 19 18 16 15 11 10 9 5 4 0
9188 * +-----+---+-------------+------+------+--------+---+------+------+
9189 * | 0 1 | U | 1 1 1 1 1 0 | immh | immb | opcode | 1 | Rn | Rd |
9190 * +-----+---+-------------+------+------+--------+---+------+------+
9192 * This is the scalar version so it works on a fixed sized registers
9194 static void disas_simd_scalar_shift_imm(DisasContext
*s
, uint32_t insn
)
9196 int rd
= extract32(insn
, 0, 5);
9197 int rn
= extract32(insn
, 5, 5);
9198 int opcode
= extract32(insn
, 11, 5);
9199 int immb
= extract32(insn
, 16, 3);
9200 int immh
= extract32(insn
, 19, 4);
9201 bool is_u
= extract32(insn
, 29, 1);
9204 unallocated_encoding(s
);
9209 case 0x08: /* SRI */
9211 unallocated_encoding(s
);
9215 case 0x00: /* SSHR / USHR */
9216 case 0x02: /* SSRA / USRA */
9217 case 0x04: /* SRSHR / URSHR */
9218 case 0x06: /* SRSRA / URSRA */
9219 handle_scalar_simd_shri(s
, is_u
, immh
, immb
, opcode
, rn
, rd
);
9221 case 0x0a: /* SHL / SLI */
9222 handle_scalar_simd_shli(s
, is_u
, immh
, immb
, opcode
, rn
, rd
);
9224 case 0x1c: /* SCVTF, UCVTF */
9225 handle_simd_shift_intfp_conv(s
, true, false, is_u
, immh
, immb
,
9228 case 0x10: /* SQSHRUN, SQSHRUN2 */
9229 case 0x11: /* SQRSHRUN, SQRSHRUN2 */
9231 unallocated_encoding(s
);
9234 handle_vec_simd_sqshrn(s
, true, false, false, true,
9235 immh
, immb
, opcode
, rn
, rd
);
9237 case 0x12: /* SQSHRN, SQSHRN2, UQSHRN */
9238 case 0x13: /* SQRSHRN, SQRSHRN2, UQRSHRN, UQRSHRN2 */
9239 handle_vec_simd_sqshrn(s
, true, false, is_u
, is_u
,
9240 immh
, immb
, opcode
, rn
, rd
);
9242 case 0xc: /* SQSHLU */
9244 unallocated_encoding(s
);
9247 handle_simd_qshl(s
, true, false, false, true, immh
, immb
, rn
, rd
);
9249 case 0xe: /* SQSHL, UQSHL */
9250 handle_simd_qshl(s
, true, false, is_u
, is_u
, immh
, immb
, rn
, rd
);
9252 case 0x1f: /* FCVTZS, FCVTZU */
9253 handle_simd_shift_fpint_conv(s
, true, false, is_u
, immh
, immb
, rn
, rd
);
9256 unallocated_encoding(s
);
9261 /* AdvSIMD scalar three different
9262 * 31 30 29 28 24 23 22 21 20 16 15 12 11 10 9 5 4 0
9263 * +-----+---+-----------+------+---+------+--------+-----+------+------+
9264 * | 0 1 | U | 1 1 1 1 0 | size | 1 | Rm | opcode | 0 0 | Rn | Rd |
9265 * +-----+---+-----------+------+---+------+--------+-----+------+------+
9267 static void disas_simd_scalar_three_reg_diff(DisasContext
*s
, uint32_t insn
)
9269 bool is_u
= extract32(insn
, 29, 1);
9270 int size
= extract32(insn
, 22, 2);
9271 int opcode
= extract32(insn
, 12, 4);
9272 int rm
= extract32(insn
, 16, 5);
9273 int rn
= extract32(insn
, 5, 5);
9274 int rd
= extract32(insn
, 0, 5);
9277 unallocated_encoding(s
);
9282 case 0x9: /* SQDMLAL, SQDMLAL2 */
9283 case 0xb: /* SQDMLSL, SQDMLSL2 */
9284 case 0xd: /* SQDMULL, SQDMULL2 */
9285 if (size
== 0 || size
== 3) {
9286 unallocated_encoding(s
);
9291 unallocated_encoding(s
);
9295 if (!fp_access_check(s
)) {
9300 TCGv_i64 tcg_op1
= tcg_temp_new_i64();
9301 TCGv_i64 tcg_op2
= tcg_temp_new_i64();
9302 TCGv_i64 tcg_res
= tcg_temp_new_i64();
9304 read_vec_element(s
, tcg_op1
, rn
, 0, MO_32
| MO_SIGN
);
9305 read_vec_element(s
, tcg_op2
, rm
, 0, MO_32
| MO_SIGN
);
9307 tcg_gen_mul_i64(tcg_res
, tcg_op1
, tcg_op2
);
9308 gen_helper_neon_addl_saturate_s64(tcg_res
, tcg_env
, tcg_res
, tcg_res
);
9311 case 0xd: /* SQDMULL, SQDMULL2 */
9313 case 0xb: /* SQDMLSL, SQDMLSL2 */
9314 tcg_gen_neg_i64(tcg_res
, tcg_res
);
9316 case 0x9: /* SQDMLAL, SQDMLAL2 */
9317 read_vec_element(s
, tcg_op1
, rd
, 0, MO_64
);
9318 gen_helper_neon_addl_saturate_s64(tcg_res
, tcg_env
,
9322 g_assert_not_reached();
9325 write_fp_dreg(s
, rd
, tcg_res
);
9327 TCGv_i32 tcg_op1
= read_fp_hreg(s
, rn
);
9328 TCGv_i32 tcg_op2
= read_fp_hreg(s
, rm
);
9329 TCGv_i64 tcg_res
= tcg_temp_new_i64();
9331 gen_helper_neon_mull_s16(tcg_res
, tcg_op1
, tcg_op2
);
9332 gen_helper_neon_addl_saturate_s32(tcg_res
, tcg_env
, tcg_res
, tcg_res
);
9335 case 0xd: /* SQDMULL, SQDMULL2 */
9337 case 0xb: /* SQDMLSL, SQDMLSL2 */
9338 gen_helper_neon_negl_u32(tcg_res
, tcg_res
);
9340 case 0x9: /* SQDMLAL, SQDMLAL2 */
9342 TCGv_i64 tcg_op3
= tcg_temp_new_i64();
9343 read_vec_element(s
, tcg_op3
, rd
, 0, MO_32
);
9344 gen_helper_neon_addl_saturate_s32(tcg_res
, tcg_env
,
9349 g_assert_not_reached();
9352 tcg_gen_ext32u_i64(tcg_res
, tcg_res
);
9353 write_fp_dreg(s
, rd
, tcg_res
);
9357 static void handle_3same_64(DisasContext
*s
, int opcode
, bool u
,
9358 TCGv_i64 tcg_rd
, TCGv_i64 tcg_rn
, TCGv_i64 tcg_rm
)
9360 /* Handle 64x64->64 opcodes which are shared between the scalar
9361 * and vector 3-same groups. We cover every opcode where size == 3
9362 * is valid in either the three-reg-same (integer, not pairwise)
9363 * or scalar-three-reg-same groups.
9368 case 0x6: /* CMGT, CMHI */
9369 cond
= u
? TCG_COND_GTU
: TCG_COND_GT
;
9371 /* 64 bit integer comparison, result = test ? -1 : 0. */
9372 tcg_gen_negsetcond_i64(cond
, tcg_rd
, tcg_rn
, tcg_rm
);
9374 case 0x7: /* CMGE, CMHS */
9375 cond
= u
? TCG_COND_GEU
: TCG_COND_GE
;
9377 case 0x11: /* CMTST, CMEQ */
9382 gen_cmtst_i64(tcg_rd
, tcg_rn
, tcg_rm
);
9384 case 0x9: /* SQSHL, UQSHL */
9386 gen_helper_neon_qshl_u64(tcg_rd
, tcg_env
, tcg_rn
, tcg_rm
);
9388 gen_helper_neon_qshl_s64(tcg_rd
, tcg_env
, tcg_rn
, tcg_rm
);
9391 case 0xb: /* SQRSHL, UQRSHL */
9393 gen_helper_neon_qrshl_u64(tcg_rd
, tcg_env
, tcg_rn
, tcg_rm
);
9395 gen_helper_neon_qrshl_s64(tcg_rd
, tcg_env
, tcg_rn
, tcg_rm
);
9398 case 0x10: /* ADD, SUB */
9400 tcg_gen_sub_i64(tcg_rd
, tcg_rn
, tcg_rm
);
9402 tcg_gen_add_i64(tcg_rd
, tcg_rn
, tcg_rm
);
9406 case 0x1: /* SQADD / UQADD */
9407 case 0x5: /* SQSUB / UQSUB */
9408 case 0x8: /* SSHL, USHL */
9409 case 0xa: /* SRSHL, URSHL */
9410 g_assert_not_reached();
9414 /* AdvSIMD scalar three same
9415 * 31 30 29 28 24 23 22 21 20 16 15 11 10 9 5 4 0
9416 * +-----+---+-----------+------+---+------+--------+---+------+------+
9417 * | 0 1 | U | 1 1 1 1 0 | size | 1 | Rm | opcode | 1 | Rn | Rd |
9418 * +-----+---+-----------+------+---+------+--------+---+------+------+
9420 static void disas_simd_scalar_three_reg_same(DisasContext
*s
, uint32_t insn
)
9422 int rd
= extract32(insn
, 0, 5);
9423 int rn
= extract32(insn
, 5, 5);
9424 int opcode
= extract32(insn
, 11, 5);
9425 int rm
= extract32(insn
, 16, 5);
9426 int size
= extract32(insn
, 22, 2);
9427 bool u
= extract32(insn
, 29, 1);
9431 case 0x9: /* SQSHL, UQSHL */
9432 case 0xb: /* SQRSHL, UQRSHL */
9434 case 0x6: /* CMGT, CMHI */
9435 case 0x7: /* CMGE, CMHS */
9436 case 0x11: /* CMTST, CMEQ */
9437 case 0x10: /* ADD, SUB (vector) */
9439 unallocated_encoding(s
);
9443 case 0x16: /* SQDMULH, SQRDMULH (vector) */
9444 if (size
!= 1 && size
!= 2) {
9445 unallocated_encoding(s
);
9450 case 0x1: /* SQADD, UQADD */
9451 case 0x5: /* SQSUB, UQSUB */
9452 case 0x8: /* SSHL, USHL */
9453 case 0xa: /* SRSHL, URSHL */
9454 unallocated_encoding(s
);
9458 if (!fp_access_check(s
)) {
9462 tcg_rd
= tcg_temp_new_i64();
9465 TCGv_i64 tcg_rn
= read_fp_dreg(s
, rn
);
9466 TCGv_i64 tcg_rm
= read_fp_dreg(s
, rm
);
9468 handle_3same_64(s
, opcode
, u
, tcg_rd
, tcg_rn
, tcg_rm
);
9470 /* Do a single operation on the lowest element in the vector.
9471 * We use the standard Neon helpers and rely on 0 OP 0 == 0 with
9472 * no side effects for all these operations.
9473 * OPTME: special-purpose helpers would avoid doing some
9474 * unnecessary work in the helper for the 8 and 16 bit cases.
9476 NeonGenTwoOpEnvFn
*genenvfn
= NULL
;
9477 void (*genfn
)(TCGv_i64
, TCGv_i64
, TCGv_i64
, TCGv_i64
, MemOp
) = NULL
;
9480 case 0x9: /* SQSHL, UQSHL */
9482 static NeonGenTwoOpEnvFn
* const fns
[3][2] = {
9483 { gen_helper_neon_qshl_s8
, gen_helper_neon_qshl_u8
},
9484 { gen_helper_neon_qshl_s16
, gen_helper_neon_qshl_u16
},
9485 { gen_helper_neon_qshl_s32
, gen_helper_neon_qshl_u32
},
9487 genenvfn
= fns
[size
][u
];
9490 case 0xb: /* SQRSHL, UQRSHL */
9492 static NeonGenTwoOpEnvFn
* const fns
[3][2] = {
9493 { gen_helper_neon_qrshl_s8
, gen_helper_neon_qrshl_u8
},
9494 { gen_helper_neon_qrshl_s16
, gen_helper_neon_qrshl_u16
},
9495 { gen_helper_neon_qrshl_s32
, gen_helper_neon_qrshl_u32
},
9497 genenvfn
= fns
[size
][u
];
9500 case 0x16: /* SQDMULH, SQRDMULH */
9502 static NeonGenTwoOpEnvFn
* const fns
[2][2] = {
9503 { gen_helper_neon_qdmulh_s16
, gen_helper_neon_qrdmulh_s16
},
9504 { gen_helper_neon_qdmulh_s32
, gen_helper_neon_qrdmulh_s32
},
9506 assert(size
== 1 || size
== 2);
9507 genenvfn
= fns
[size
- 1][u
];
9511 case 0x1: /* SQADD, UQADD */
9512 case 0x5: /* SQSUB, UQSUB */
9513 g_assert_not_reached();
9517 TCGv_i32 tcg_rn
= tcg_temp_new_i32();
9518 TCGv_i32 tcg_rm
= tcg_temp_new_i32();
9520 read_vec_element_i32(s
, tcg_rn
, rn
, 0, size
);
9521 read_vec_element_i32(s
, tcg_rm
, rm
, 0, size
);
9522 genenvfn(tcg_rn
, tcg_env
, tcg_rn
, tcg_rm
);
9523 tcg_gen_extu_i32_i64(tcg_rd
, tcg_rn
);
9525 TCGv_i64 tcg_rn
= tcg_temp_new_i64();
9526 TCGv_i64 tcg_rm
= tcg_temp_new_i64();
9527 TCGv_i64 qc
= tcg_temp_new_i64();
9529 read_vec_element(s
, tcg_rn
, rn
, 0, size
| (u
? 0 : MO_SIGN
));
9530 read_vec_element(s
, tcg_rm
, rm
, 0, size
| (u
? 0 : MO_SIGN
));
9531 tcg_gen_ld_i64(qc
, tcg_env
, offsetof(CPUARMState
, vfp
.qc
));
9532 genfn(tcg_rd
, qc
, tcg_rn
, tcg_rm
, size
);
9533 tcg_gen_st_i64(qc
, tcg_env
, offsetof(CPUARMState
, vfp
.qc
));
9535 /* Truncate signed 64-bit result for writeback. */
9536 tcg_gen_ext_i64(tcg_rd
, tcg_rd
, size
);
9541 write_fp_dreg(s
, rd
, tcg_rd
);
9544 /* AdvSIMD scalar three same extra
9545 * 31 30 29 28 24 23 22 21 20 16 15 14 11 10 9 5 4 0
9546 * +-----+---+-----------+------+---+------+---+--------+---+----+----+
9547 * | 0 1 | U | 1 1 1 1 0 | size | 0 | Rm | 1 | opcode | 1 | Rn | Rd |
9548 * +-----+---+-----------+------+---+------+---+--------+---+----+----+
9550 static void disas_simd_scalar_three_reg_same_extra(DisasContext
*s
,
9553 int rd
= extract32(insn
, 0, 5);
9554 int rn
= extract32(insn
, 5, 5);
9555 int opcode
= extract32(insn
, 11, 4);
9556 int rm
= extract32(insn
, 16, 5);
9557 int size
= extract32(insn
, 22, 2);
9558 bool u
= extract32(insn
, 29, 1);
9559 TCGv_i32 ele1
, ele2
, ele3
;
9563 switch (u
* 16 + opcode
) {
9564 case 0x10: /* SQRDMLAH (vector) */
9565 case 0x11: /* SQRDMLSH (vector) */
9566 if (size
!= 1 && size
!= 2) {
9567 unallocated_encoding(s
);
9570 feature
= dc_isar_feature(aa64_rdm
, s
);
9573 unallocated_encoding(s
);
9577 unallocated_encoding(s
);
9580 if (!fp_access_check(s
)) {
9584 /* Do a single operation on the lowest element in the vector.
9585 * We use the standard Neon helpers and rely on 0 OP 0 == 0
9586 * with no side effects for all these operations.
9587 * OPTME: special-purpose helpers would avoid doing some
9588 * unnecessary work in the helper for the 16 bit cases.
9590 ele1
= tcg_temp_new_i32();
9591 ele2
= tcg_temp_new_i32();
9592 ele3
= tcg_temp_new_i32();
9594 read_vec_element_i32(s
, ele1
, rn
, 0, size
);
9595 read_vec_element_i32(s
, ele2
, rm
, 0, size
);
9596 read_vec_element_i32(s
, ele3
, rd
, 0, size
);
9599 case 0x0: /* SQRDMLAH */
9601 gen_helper_neon_qrdmlah_s16(ele3
, tcg_env
, ele1
, ele2
, ele3
);
9603 gen_helper_neon_qrdmlah_s32(ele3
, tcg_env
, ele1
, ele2
, ele3
);
9606 case 0x1: /* SQRDMLSH */
9608 gen_helper_neon_qrdmlsh_s16(ele3
, tcg_env
, ele1
, ele2
, ele3
);
9610 gen_helper_neon_qrdmlsh_s32(ele3
, tcg_env
, ele1
, ele2
, ele3
);
9614 g_assert_not_reached();
9617 res
= tcg_temp_new_i64();
9618 tcg_gen_extu_i32_i64(res
, ele3
);
9619 write_fp_dreg(s
, rd
, res
);
9622 static void handle_2misc_64(DisasContext
*s
, int opcode
, bool u
,
9623 TCGv_i64 tcg_rd
, TCGv_i64 tcg_rn
,
9624 TCGv_i32 tcg_rmode
, TCGv_ptr tcg_fpstatus
)
9626 /* Handle 64->64 opcodes which are shared between the scalar and
9627 * vector 2-reg-misc groups. We cover every integer opcode where size == 3
9628 * is valid in either group and also the double-precision fp ops.
9629 * The caller only need provide tcg_rmode and tcg_fpstatus if the op
9635 case 0x4: /* CLS, CLZ */
9637 tcg_gen_clzi_i64(tcg_rd
, tcg_rn
, 64);
9639 tcg_gen_clrsb_i64(tcg_rd
, tcg_rn
);
9643 /* This opcode is shared with CNT and RBIT but we have earlier
9644 * enforced that size == 3 if and only if this is the NOT insn.
9646 tcg_gen_not_i64(tcg_rd
, tcg_rn
);
9648 case 0x7: /* SQABS, SQNEG */
9650 gen_helper_neon_qneg_s64(tcg_rd
, tcg_env
, tcg_rn
);
9652 gen_helper_neon_qabs_s64(tcg_rd
, tcg_env
, tcg_rn
);
9655 case 0xa: /* CMLT */
9658 /* 64 bit integer comparison against zero, result is test ? -1 : 0. */
9659 tcg_gen_negsetcond_i64(cond
, tcg_rd
, tcg_rn
, tcg_constant_i64(0));
9661 case 0x8: /* CMGT, CMGE */
9662 cond
= u
? TCG_COND_GE
: TCG_COND_GT
;
9664 case 0x9: /* CMEQ, CMLE */
9665 cond
= u
? TCG_COND_LE
: TCG_COND_EQ
;
9667 case 0xb: /* ABS, NEG */
9669 tcg_gen_neg_i64(tcg_rd
, tcg_rn
);
9671 tcg_gen_abs_i64(tcg_rd
, tcg_rn
);
9674 case 0x2f: /* FABS */
9675 gen_vfp_absd(tcg_rd
, tcg_rn
);
9677 case 0x6f: /* FNEG */
9678 gen_vfp_negd(tcg_rd
, tcg_rn
);
9680 case 0x7f: /* FSQRT */
9681 gen_helper_vfp_sqrtd(tcg_rd
, tcg_rn
, tcg_env
);
9683 case 0x1a: /* FCVTNS */
9684 case 0x1b: /* FCVTMS */
9685 case 0x1c: /* FCVTAS */
9686 case 0x3a: /* FCVTPS */
9687 case 0x3b: /* FCVTZS */
9688 gen_helper_vfp_tosqd(tcg_rd
, tcg_rn
, tcg_constant_i32(0), tcg_fpstatus
);
9690 case 0x5a: /* FCVTNU */
9691 case 0x5b: /* FCVTMU */
9692 case 0x5c: /* FCVTAU */
9693 case 0x7a: /* FCVTPU */
9694 case 0x7b: /* FCVTZU */
9695 gen_helper_vfp_touqd(tcg_rd
, tcg_rn
, tcg_constant_i32(0), tcg_fpstatus
);
9697 case 0x18: /* FRINTN */
9698 case 0x19: /* FRINTM */
9699 case 0x38: /* FRINTP */
9700 case 0x39: /* FRINTZ */
9701 case 0x58: /* FRINTA */
9702 case 0x79: /* FRINTI */
9703 gen_helper_rintd(tcg_rd
, tcg_rn
, tcg_fpstatus
);
9705 case 0x59: /* FRINTX */
9706 gen_helper_rintd_exact(tcg_rd
, tcg_rn
, tcg_fpstatus
);
9708 case 0x1e: /* FRINT32Z */
9709 case 0x5e: /* FRINT32X */
9710 gen_helper_frint32_d(tcg_rd
, tcg_rn
, tcg_fpstatus
);
9712 case 0x1f: /* FRINT64Z */
9713 case 0x5f: /* FRINT64X */
9714 gen_helper_frint64_d(tcg_rd
, tcg_rn
, tcg_fpstatus
);
9717 g_assert_not_reached();
9721 static void handle_2misc_fcmp_zero(DisasContext
*s
, int opcode
,
9722 bool is_scalar
, bool is_u
, bool is_q
,
9723 int size
, int rn
, int rd
)
9725 bool is_double
= (size
== MO_64
);
9728 if (!fp_access_check(s
)) {
9732 fpst
= fpstatus_ptr(size
== MO_16
? FPST_FPCR_F16
: FPST_FPCR
);
9735 TCGv_i64 tcg_op
= tcg_temp_new_i64();
9736 TCGv_i64 tcg_zero
= tcg_constant_i64(0);
9737 TCGv_i64 tcg_res
= tcg_temp_new_i64();
9738 NeonGenTwoDoubleOpFn
*genfn
;
9743 case 0x2e: /* FCMLT (zero) */
9746 case 0x2c: /* FCMGT (zero) */
9747 genfn
= gen_helper_neon_cgt_f64
;
9749 case 0x2d: /* FCMEQ (zero) */
9750 genfn
= gen_helper_neon_ceq_f64
;
9752 case 0x6d: /* FCMLE (zero) */
9755 case 0x6c: /* FCMGE (zero) */
9756 genfn
= gen_helper_neon_cge_f64
;
9759 g_assert_not_reached();
9762 for (pass
= 0; pass
< (is_scalar
? 1 : 2); pass
++) {
9763 read_vec_element(s
, tcg_op
, rn
, pass
, MO_64
);
9765 genfn(tcg_res
, tcg_zero
, tcg_op
, fpst
);
9767 genfn(tcg_res
, tcg_op
, tcg_zero
, fpst
);
9769 write_vec_element(s
, tcg_res
, rd
, pass
, MO_64
);
9772 clear_vec_high(s
, !is_scalar
, rd
);
9774 TCGv_i32 tcg_op
= tcg_temp_new_i32();
9775 TCGv_i32 tcg_zero
= tcg_constant_i32(0);
9776 TCGv_i32 tcg_res
= tcg_temp_new_i32();
9777 NeonGenTwoSingleOpFn
*genfn
;
9779 int pass
, maxpasses
;
9781 if (size
== MO_16
) {
9783 case 0x2e: /* FCMLT (zero) */
9786 case 0x2c: /* FCMGT (zero) */
9787 genfn
= gen_helper_advsimd_cgt_f16
;
9789 case 0x2d: /* FCMEQ (zero) */
9790 genfn
= gen_helper_advsimd_ceq_f16
;
9792 case 0x6d: /* FCMLE (zero) */
9795 case 0x6c: /* FCMGE (zero) */
9796 genfn
= gen_helper_advsimd_cge_f16
;
9799 g_assert_not_reached();
9803 case 0x2e: /* FCMLT (zero) */
9806 case 0x2c: /* FCMGT (zero) */
9807 genfn
= gen_helper_neon_cgt_f32
;
9809 case 0x2d: /* FCMEQ (zero) */
9810 genfn
= gen_helper_neon_ceq_f32
;
9812 case 0x6d: /* FCMLE (zero) */
9815 case 0x6c: /* FCMGE (zero) */
9816 genfn
= gen_helper_neon_cge_f32
;
9819 g_assert_not_reached();
9826 int vector_size
= 8 << is_q
;
9827 maxpasses
= vector_size
>> size
;
9830 for (pass
= 0; pass
< maxpasses
; pass
++) {
9831 read_vec_element_i32(s
, tcg_op
, rn
, pass
, size
);
9833 genfn(tcg_res
, tcg_zero
, tcg_op
, fpst
);
9835 genfn(tcg_res
, tcg_op
, tcg_zero
, fpst
);
9838 write_fp_sreg(s
, rd
, tcg_res
);
9840 write_vec_element_i32(s
, tcg_res
, rd
, pass
, size
);
9845 clear_vec_high(s
, is_q
, rd
);
9850 static void handle_2misc_reciprocal(DisasContext
*s
, int opcode
,
9851 bool is_scalar
, bool is_u
, bool is_q
,
9852 int size
, int rn
, int rd
)
9854 bool is_double
= (size
== 3);
9855 TCGv_ptr fpst
= fpstatus_ptr(FPST_FPCR
);
9858 TCGv_i64 tcg_op
= tcg_temp_new_i64();
9859 TCGv_i64 tcg_res
= tcg_temp_new_i64();
9862 for (pass
= 0; pass
< (is_scalar
? 1 : 2); pass
++) {
9863 read_vec_element(s
, tcg_op
, rn
, pass
, MO_64
);
9865 case 0x3d: /* FRECPE */
9866 gen_helper_recpe_f64(tcg_res
, tcg_op
, fpst
);
9868 case 0x3f: /* FRECPX */
9869 gen_helper_frecpx_f64(tcg_res
, tcg_op
, fpst
);
9871 case 0x7d: /* FRSQRTE */
9872 gen_helper_rsqrte_f64(tcg_res
, tcg_op
, fpst
);
9875 g_assert_not_reached();
9877 write_vec_element(s
, tcg_res
, rd
, pass
, MO_64
);
9879 clear_vec_high(s
, !is_scalar
, rd
);
9881 TCGv_i32 tcg_op
= tcg_temp_new_i32();
9882 TCGv_i32 tcg_res
= tcg_temp_new_i32();
9883 int pass
, maxpasses
;
9888 maxpasses
= is_q
? 4 : 2;
9891 for (pass
= 0; pass
< maxpasses
; pass
++) {
9892 read_vec_element_i32(s
, tcg_op
, rn
, pass
, MO_32
);
9895 case 0x3c: /* URECPE */
9896 gen_helper_recpe_u32(tcg_res
, tcg_op
);
9898 case 0x3d: /* FRECPE */
9899 gen_helper_recpe_f32(tcg_res
, tcg_op
, fpst
);
9901 case 0x3f: /* FRECPX */
9902 gen_helper_frecpx_f32(tcg_res
, tcg_op
, fpst
);
9904 case 0x7d: /* FRSQRTE */
9905 gen_helper_rsqrte_f32(tcg_res
, tcg_op
, fpst
);
9908 g_assert_not_reached();
9912 write_fp_sreg(s
, rd
, tcg_res
);
9914 write_vec_element_i32(s
, tcg_res
, rd
, pass
, MO_32
);
9918 clear_vec_high(s
, is_q
, rd
);
9923 static void handle_2misc_narrow(DisasContext
*s
, bool scalar
,
9924 int opcode
, bool u
, bool is_q
,
9925 int size
, int rn
, int rd
)
9927 /* Handle 2-reg-misc ops which are narrowing (so each 2*size element
9928 * in the source becomes a size element in the destination).
9931 TCGv_i32 tcg_res
[2];
9932 int destelt
= is_q
? 2 : 0;
9933 int passes
= scalar
? 1 : 2;
9936 tcg_res
[1] = tcg_constant_i32(0);
9939 for (pass
= 0; pass
< passes
; pass
++) {
9940 TCGv_i64 tcg_op
= tcg_temp_new_i64();
9941 NeonGenNarrowFn
*genfn
= NULL
;
9942 NeonGenNarrowEnvFn
*genenvfn
= NULL
;
9945 read_vec_element(s
, tcg_op
, rn
, pass
, size
+ 1);
9947 read_vec_element(s
, tcg_op
, rn
, pass
, MO_64
);
9949 tcg_res
[pass
] = tcg_temp_new_i32();
9952 case 0x12: /* XTN, SQXTUN */
9954 static NeonGenNarrowFn
* const xtnfns
[3] = {
9955 gen_helper_neon_narrow_u8
,
9956 gen_helper_neon_narrow_u16
,
9957 tcg_gen_extrl_i64_i32
,
9959 static NeonGenNarrowEnvFn
* const sqxtunfns
[3] = {
9960 gen_helper_neon_unarrow_sat8
,
9961 gen_helper_neon_unarrow_sat16
,
9962 gen_helper_neon_unarrow_sat32
,
9965 genenvfn
= sqxtunfns
[size
];
9967 genfn
= xtnfns
[size
];
9971 case 0x14: /* SQXTN, UQXTN */
9973 static NeonGenNarrowEnvFn
* const fns
[3][2] = {
9974 { gen_helper_neon_narrow_sat_s8
,
9975 gen_helper_neon_narrow_sat_u8
},
9976 { gen_helper_neon_narrow_sat_s16
,
9977 gen_helper_neon_narrow_sat_u16
},
9978 { gen_helper_neon_narrow_sat_s32
,
9979 gen_helper_neon_narrow_sat_u32
},
9981 genenvfn
= fns
[size
][u
];
9984 case 0x16: /* FCVTN, FCVTN2 */
9985 /* 32 bit to 16 bit or 64 bit to 32 bit float conversion */
9987 gen_helper_vfp_fcvtsd(tcg_res
[pass
], tcg_op
, tcg_env
);
9989 TCGv_i32 tcg_lo
= tcg_temp_new_i32();
9990 TCGv_i32 tcg_hi
= tcg_temp_new_i32();
9991 TCGv_ptr fpst
= fpstatus_ptr(FPST_FPCR
);
9992 TCGv_i32 ahp
= get_ahp_flag();
9994 tcg_gen_extr_i64_i32(tcg_lo
, tcg_hi
, tcg_op
);
9995 gen_helper_vfp_fcvt_f32_to_f16(tcg_lo
, tcg_lo
, fpst
, ahp
);
9996 gen_helper_vfp_fcvt_f32_to_f16(tcg_hi
, tcg_hi
, fpst
, ahp
);
9997 tcg_gen_deposit_i32(tcg_res
[pass
], tcg_lo
, tcg_hi
, 16, 16);
10000 case 0x36: /* BFCVTN, BFCVTN2 */
10002 TCGv_ptr fpst
= fpstatus_ptr(FPST_FPCR
);
10003 gen_helper_bfcvt_pair(tcg_res
[pass
], tcg_op
, fpst
);
10006 case 0x56: /* FCVTXN, FCVTXN2 */
10007 /* 64 bit to 32 bit float conversion
10008 * with von Neumann rounding (round to odd)
10011 gen_helper_fcvtx_f64_to_f32(tcg_res
[pass
], tcg_op
, tcg_env
);
10014 g_assert_not_reached();
10018 genfn(tcg_res
[pass
], tcg_op
);
10019 } else if (genenvfn
) {
10020 genenvfn(tcg_res
[pass
], tcg_env
, tcg_op
);
10024 for (pass
= 0; pass
< 2; pass
++) {
10025 write_vec_element_i32(s
, tcg_res
[pass
], rd
, destelt
+ pass
, MO_32
);
10027 clear_vec_high(s
, is_q
, rd
);
10030 /* AdvSIMD scalar two reg misc
10031 * 31 30 29 28 24 23 22 21 17 16 12 11 10 9 5 4 0
10032 * +-----+---+-----------+------+-----------+--------+-----+------+------+
10033 * | 0 1 | U | 1 1 1 1 0 | size | 1 0 0 0 0 | opcode | 1 0 | Rn | Rd |
10034 * +-----+---+-----------+------+-----------+--------+-----+------+------+
10036 static void disas_simd_scalar_two_reg_misc(DisasContext
*s
, uint32_t insn
)
10038 int rd
= extract32(insn
, 0, 5);
10039 int rn
= extract32(insn
, 5, 5);
10040 int opcode
= extract32(insn
, 12, 5);
10041 int size
= extract32(insn
, 22, 2);
10042 bool u
= extract32(insn
, 29, 1);
10043 bool is_fcvt
= false;
10045 TCGv_i32 tcg_rmode
;
10046 TCGv_ptr tcg_fpstatus
;
10049 case 0x7: /* SQABS / SQNEG */
10051 case 0xa: /* CMLT */
10053 unallocated_encoding(s
);
10057 case 0x8: /* CMGT, CMGE */
10058 case 0x9: /* CMEQ, CMLE */
10059 case 0xb: /* ABS, NEG */
10061 unallocated_encoding(s
);
10065 case 0x12: /* SQXTUN */
10067 unallocated_encoding(s
);
10071 case 0x14: /* SQXTN, UQXTN */
10073 unallocated_encoding(s
);
10076 if (!fp_access_check(s
)) {
10079 handle_2misc_narrow(s
, true, opcode
, u
, false, size
, rn
, rd
);
10082 case 0x16 ... 0x1d:
10084 /* Floating point: U, size[1] and opcode indicate operation;
10085 * size[0] indicates single or double precision.
10087 opcode
|= (extract32(size
, 1, 1) << 5) | (u
<< 6);
10088 size
= extract32(size
, 0, 1) ? 3 : 2;
10090 case 0x2c: /* FCMGT (zero) */
10091 case 0x2d: /* FCMEQ (zero) */
10092 case 0x2e: /* FCMLT (zero) */
10093 case 0x6c: /* FCMGE (zero) */
10094 case 0x6d: /* FCMLE (zero) */
10095 handle_2misc_fcmp_zero(s
, opcode
, true, u
, true, size
, rn
, rd
);
10097 case 0x1d: /* SCVTF */
10098 case 0x5d: /* UCVTF */
10100 bool is_signed
= (opcode
== 0x1d);
10101 if (!fp_access_check(s
)) {
10104 handle_simd_intfp_conv(s
, rd
, rn
, 1, is_signed
, 0, size
);
10107 case 0x3d: /* FRECPE */
10108 case 0x3f: /* FRECPX */
10109 case 0x7d: /* FRSQRTE */
10110 if (!fp_access_check(s
)) {
10113 handle_2misc_reciprocal(s
, opcode
, true, u
, true, size
, rn
, rd
);
10115 case 0x1a: /* FCVTNS */
10116 case 0x1b: /* FCVTMS */
10117 case 0x3a: /* FCVTPS */
10118 case 0x3b: /* FCVTZS */
10119 case 0x5a: /* FCVTNU */
10120 case 0x5b: /* FCVTMU */
10121 case 0x7a: /* FCVTPU */
10122 case 0x7b: /* FCVTZU */
10124 rmode
= extract32(opcode
, 5, 1) | (extract32(opcode
, 0, 1) << 1);
10126 case 0x1c: /* FCVTAS */
10127 case 0x5c: /* FCVTAU */
10128 /* TIEAWAY doesn't fit in the usual rounding mode encoding */
10130 rmode
= FPROUNDING_TIEAWAY
;
10132 case 0x56: /* FCVTXN, FCVTXN2 */
10134 unallocated_encoding(s
);
10137 if (!fp_access_check(s
)) {
10140 handle_2misc_narrow(s
, true, opcode
, u
, false, size
- 1, rn
, rd
);
10143 unallocated_encoding(s
);
10148 case 0x3: /* USQADD / SUQADD */
10149 unallocated_encoding(s
);
10153 if (!fp_access_check(s
)) {
10158 tcg_fpstatus
= fpstatus_ptr(FPST_FPCR
);
10159 tcg_rmode
= gen_set_rmode(rmode
, tcg_fpstatus
);
10161 tcg_fpstatus
= NULL
;
10166 TCGv_i64 tcg_rn
= read_fp_dreg(s
, rn
);
10167 TCGv_i64 tcg_rd
= tcg_temp_new_i64();
10169 handle_2misc_64(s
, opcode
, u
, tcg_rd
, tcg_rn
, tcg_rmode
, tcg_fpstatus
);
10170 write_fp_dreg(s
, rd
, tcg_rd
);
10172 TCGv_i32 tcg_rn
= tcg_temp_new_i32();
10173 TCGv_i32 tcg_rd
= tcg_temp_new_i32();
10175 read_vec_element_i32(s
, tcg_rn
, rn
, 0, size
);
10178 case 0x7: /* SQABS, SQNEG */
10180 NeonGenOneOpEnvFn
*genfn
;
10181 static NeonGenOneOpEnvFn
* const fns
[3][2] = {
10182 { gen_helper_neon_qabs_s8
, gen_helper_neon_qneg_s8
},
10183 { gen_helper_neon_qabs_s16
, gen_helper_neon_qneg_s16
},
10184 { gen_helper_neon_qabs_s32
, gen_helper_neon_qneg_s32
},
10186 genfn
= fns
[size
][u
];
10187 genfn(tcg_rd
, tcg_env
, tcg_rn
);
10190 case 0x1a: /* FCVTNS */
10191 case 0x1b: /* FCVTMS */
10192 case 0x1c: /* FCVTAS */
10193 case 0x3a: /* FCVTPS */
10194 case 0x3b: /* FCVTZS */
10195 gen_helper_vfp_tosls(tcg_rd
, tcg_rn
, tcg_constant_i32(0),
10198 case 0x5a: /* FCVTNU */
10199 case 0x5b: /* FCVTMU */
10200 case 0x5c: /* FCVTAU */
10201 case 0x7a: /* FCVTPU */
10202 case 0x7b: /* FCVTZU */
10203 gen_helper_vfp_touls(tcg_rd
, tcg_rn
, tcg_constant_i32(0),
10207 g_assert_not_reached();
10210 write_fp_sreg(s
, rd
, tcg_rd
);
10214 gen_restore_rmode(tcg_rmode
, tcg_fpstatus
);
10218 /* SSHR[RA]/USHR[RA] - Vector shift right (optional rounding/accumulate) */
10219 static void handle_vec_simd_shri(DisasContext
*s
, bool is_q
, bool is_u
,
10220 int immh
, int immb
, int opcode
, int rn
, int rd
)
10222 int size
= 32 - clz32(immh
) - 1;
10223 int immhb
= immh
<< 3 | immb
;
10224 int shift
= 2 * (8 << size
) - immhb
;
10225 GVecGen2iFn
*gvec_fn
;
10227 if (extract32(immh
, 3, 1) && !is_q
) {
10228 unallocated_encoding(s
);
10231 tcg_debug_assert(size
<= 3);
10233 if (!fp_access_check(s
)) {
10238 case 0x02: /* SSRA / USRA (accumulate) */
10239 gvec_fn
= is_u
? gen_gvec_usra
: gen_gvec_ssra
;
10242 case 0x08: /* SRI */
10243 gvec_fn
= gen_gvec_sri
;
10246 case 0x00: /* SSHR / USHR */
10248 if (shift
== 8 << size
) {
10249 /* Shift count the same size as element size produces zero. */
10250 tcg_gen_gvec_dup_imm(size
, vec_full_reg_offset(s
, rd
),
10251 is_q
? 16 : 8, vec_full_reg_size(s
), 0);
10254 gvec_fn
= tcg_gen_gvec_shri
;
10256 /* Shift count the same size as element size produces all sign. */
10257 if (shift
== 8 << size
) {
10260 gvec_fn
= tcg_gen_gvec_sari
;
10264 case 0x04: /* SRSHR / URSHR (rounding) */
10265 gvec_fn
= is_u
? gen_gvec_urshr
: gen_gvec_srshr
;
10268 case 0x06: /* SRSRA / URSRA (accum + rounding) */
10269 gvec_fn
= is_u
? gen_gvec_ursra
: gen_gvec_srsra
;
10273 g_assert_not_reached();
10276 gen_gvec_fn2i(s
, is_q
, rd
, rn
, shift
, gvec_fn
, size
);
10279 /* SHL/SLI - Vector shift left */
10280 static void handle_vec_simd_shli(DisasContext
*s
, bool is_q
, bool insert
,
10281 int immh
, int immb
, int opcode
, int rn
, int rd
)
10283 int size
= 32 - clz32(immh
) - 1;
10284 int immhb
= immh
<< 3 | immb
;
10285 int shift
= immhb
- (8 << size
);
10287 /* Range of size is limited by decode: immh is a non-zero 4 bit field */
10288 assert(size
>= 0 && size
<= 3);
10290 if (extract32(immh
, 3, 1) && !is_q
) {
10291 unallocated_encoding(s
);
10295 if (!fp_access_check(s
)) {
10300 gen_gvec_fn2i(s
, is_q
, rd
, rn
, shift
, gen_gvec_sli
, size
);
10302 gen_gvec_fn2i(s
, is_q
, rd
, rn
, shift
, tcg_gen_gvec_shli
, size
);
10306 /* USHLL/SHLL - Vector shift left with widening */
10307 static void handle_vec_simd_wshli(DisasContext
*s
, bool is_q
, bool is_u
,
10308 int immh
, int immb
, int opcode
, int rn
, int rd
)
10310 int size
= 32 - clz32(immh
) - 1;
10311 int immhb
= immh
<< 3 | immb
;
10312 int shift
= immhb
- (8 << size
);
10314 int esize
= 8 << size
;
10315 int elements
= dsize
/esize
;
10316 TCGv_i64 tcg_rn
= tcg_temp_new_i64();
10317 TCGv_i64 tcg_rd
= tcg_temp_new_i64();
10321 unallocated_encoding(s
);
10325 if (!fp_access_check(s
)) {
10329 /* For the LL variants the store is larger than the load,
10330 * so if rd == rn we would overwrite parts of our input.
10331 * So load everything right now and use shifts in the main loop.
10333 read_vec_element(s
, tcg_rn
, rn
, is_q
? 1 : 0, MO_64
);
10335 for (i
= 0; i
< elements
; i
++) {
10336 tcg_gen_shri_i64(tcg_rd
, tcg_rn
, i
* esize
);
10337 ext_and_shift_reg(tcg_rd
, tcg_rd
, size
| (!is_u
<< 2), 0);
10338 tcg_gen_shli_i64(tcg_rd
, tcg_rd
, shift
);
10339 write_vec_element(s
, tcg_rd
, rd
, i
, size
+ 1);
10343 /* SHRN/RSHRN - Shift right with narrowing (and potential rounding) */
10344 static void handle_vec_simd_shrn(DisasContext
*s
, bool is_q
,
10345 int immh
, int immb
, int opcode
, int rn
, int rd
)
10347 int immhb
= immh
<< 3 | immb
;
10348 int size
= 32 - clz32(immh
) - 1;
10350 int esize
= 8 << size
;
10351 int elements
= dsize
/esize
;
10352 int shift
= (2 * esize
) - immhb
;
10353 bool round
= extract32(opcode
, 0, 1);
10354 TCGv_i64 tcg_rn
, tcg_rd
, tcg_final
;
10355 TCGv_i64 tcg_round
;
10358 if (extract32(immh
, 3, 1)) {
10359 unallocated_encoding(s
);
10363 if (!fp_access_check(s
)) {
10367 tcg_rn
= tcg_temp_new_i64();
10368 tcg_rd
= tcg_temp_new_i64();
10369 tcg_final
= tcg_temp_new_i64();
10370 read_vec_element(s
, tcg_final
, rd
, is_q
? 1 : 0, MO_64
);
10373 tcg_round
= tcg_constant_i64(1ULL << (shift
- 1));
10378 for (i
= 0; i
< elements
; i
++) {
10379 read_vec_element(s
, tcg_rn
, rn
, i
, size
+1);
10380 handle_shri_with_rndacc(tcg_rd
, tcg_rn
, tcg_round
,
10381 false, true, size
+1, shift
);
10383 tcg_gen_deposit_i64(tcg_final
, tcg_final
, tcg_rd
, esize
* i
, esize
);
10387 write_vec_element(s
, tcg_final
, rd
, 0, MO_64
);
10389 write_vec_element(s
, tcg_final
, rd
, 1, MO_64
);
10392 clear_vec_high(s
, is_q
, rd
);
10396 /* AdvSIMD shift by immediate
10397 * 31 30 29 28 23 22 19 18 16 15 11 10 9 5 4 0
10398 * +---+---+---+-------------+------+------+--------+---+------+------+
10399 * | 0 | Q | U | 0 1 1 1 1 0 | immh | immb | opcode | 1 | Rn | Rd |
10400 * +---+---+---+-------------+------+------+--------+---+------+------+
10402 static void disas_simd_shift_imm(DisasContext
*s
, uint32_t insn
)
10404 int rd
= extract32(insn
, 0, 5);
10405 int rn
= extract32(insn
, 5, 5);
10406 int opcode
= extract32(insn
, 11, 5);
10407 int immb
= extract32(insn
, 16, 3);
10408 int immh
= extract32(insn
, 19, 4);
10409 bool is_u
= extract32(insn
, 29, 1);
10410 bool is_q
= extract32(insn
, 30, 1);
10412 /* data_proc_simd[] has sent immh == 0 to disas_simd_mod_imm. */
10416 case 0x08: /* SRI */
10418 unallocated_encoding(s
);
10422 case 0x00: /* SSHR / USHR */
10423 case 0x02: /* SSRA / USRA (accumulate) */
10424 case 0x04: /* SRSHR / URSHR (rounding) */
10425 case 0x06: /* SRSRA / URSRA (accum + rounding) */
10426 handle_vec_simd_shri(s
, is_q
, is_u
, immh
, immb
, opcode
, rn
, rd
);
10428 case 0x0a: /* SHL / SLI */
10429 handle_vec_simd_shli(s
, is_q
, is_u
, immh
, immb
, opcode
, rn
, rd
);
10431 case 0x10: /* SHRN */
10432 case 0x11: /* RSHRN / SQRSHRUN */
10434 handle_vec_simd_sqshrn(s
, false, is_q
, false, true, immh
, immb
,
10437 handle_vec_simd_shrn(s
, is_q
, immh
, immb
, opcode
, rn
, rd
);
10440 case 0x12: /* SQSHRN / UQSHRN */
10441 case 0x13: /* SQRSHRN / UQRSHRN */
10442 handle_vec_simd_sqshrn(s
, false, is_q
, is_u
, is_u
, immh
, immb
,
10445 case 0x14: /* SSHLL / USHLL */
10446 handle_vec_simd_wshli(s
, is_q
, is_u
, immh
, immb
, opcode
, rn
, rd
);
10448 case 0x1c: /* SCVTF / UCVTF */
10449 handle_simd_shift_intfp_conv(s
, false, is_q
, is_u
, immh
, immb
,
10452 case 0xc: /* SQSHLU */
10454 unallocated_encoding(s
);
10457 handle_simd_qshl(s
, false, is_q
, false, true, immh
, immb
, rn
, rd
);
10459 case 0xe: /* SQSHL, UQSHL */
10460 handle_simd_qshl(s
, false, is_q
, is_u
, is_u
, immh
, immb
, rn
, rd
);
10462 case 0x1f: /* FCVTZS/ FCVTZU */
10463 handle_simd_shift_fpint_conv(s
, false, is_q
, is_u
, immh
, immb
, rn
, rd
);
10466 unallocated_encoding(s
);
10471 /* Generate code to do a "long" addition or subtraction, ie one done in
10472 * TCGv_i64 on vector lanes twice the width specified by size.
10474 static void gen_neon_addl(int size
, bool is_sub
, TCGv_i64 tcg_res
,
10475 TCGv_i64 tcg_op1
, TCGv_i64 tcg_op2
)
10477 static NeonGenTwo64OpFn
* const fns
[3][2] = {
10478 { gen_helper_neon_addl_u16
, gen_helper_neon_subl_u16
},
10479 { gen_helper_neon_addl_u32
, gen_helper_neon_subl_u32
},
10480 { tcg_gen_add_i64
, tcg_gen_sub_i64
},
10482 NeonGenTwo64OpFn
*genfn
;
10485 genfn
= fns
[size
][is_sub
];
10486 genfn(tcg_res
, tcg_op1
, tcg_op2
);
10489 static void handle_3rd_widening(DisasContext
*s
, int is_q
, int is_u
, int size
,
10490 int opcode
, int rd
, int rn
, int rm
)
10492 /* 3-reg-different widening insns: 64 x 64 -> 128 */
10493 TCGv_i64 tcg_res
[2];
10496 tcg_res
[0] = tcg_temp_new_i64();
10497 tcg_res
[1] = tcg_temp_new_i64();
10499 /* Does this op do an adding accumulate, a subtracting accumulate,
10500 * or no accumulate at all?
10518 read_vec_element(s
, tcg_res
[0], rd
, 0, MO_64
);
10519 read_vec_element(s
, tcg_res
[1], rd
, 1, MO_64
);
10522 /* size == 2 means two 32x32->64 operations; this is worth special
10523 * casing because we can generally handle it inline.
10526 for (pass
= 0; pass
< 2; pass
++) {
10527 TCGv_i64 tcg_op1
= tcg_temp_new_i64();
10528 TCGv_i64 tcg_op2
= tcg_temp_new_i64();
10529 TCGv_i64 tcg_passres
;
10530 MemOp memop
= MO_32
| (is_u
? 0 : MO_SIGN
);
10532 int elt
= pass
+ is_q
* 2;
10534 read_vec_element(s
, tcg_op1
, rn
, elt
, memop
);
10535 read_vec_element(s
, tcg_op2
, rm
, elt
, memop
);
10538 tcg_passres
= tcg_res
[pass
];
10540 tcg_passres
= tcg_temp_new_i64();
10544 case 0: /* SADDL, SADDL2, UADDL, UADDL2 */
10545 tcg_gen_add_i64(tcg_passres
, tcg_op1
, tcg_op2
);
10547 case 2: /* SSUBL, SSUBL2, USUBL, USUBL2 */
10548 tcg_gen_sub_i64(tcg_passres
, tcg_op1
, tcg_op2
);
10550 case 5: /* SABAL, SABAL2, UABAL, UABAL2 */
10551 case 7: /* SABDL, SABDL2, UABDL, UABDL2 */
10553 TCGv_i64 tcg_tmp1
= tcg_temp_new_i64();
10554 TCGv_i64 tcg_tmp2
= tcg_temp_new_i64();
10556 tcg_gen_sub_i64(tcg_tmp1
, tcg_op1
, tcg_op2
);
10557 tcg_gen_sub_i64(tcg_tmp2
, tcg_op2
, tcg_op1
);
10558 tcg_gen_movcond_i64(is_u
? TCG_COND_GEU
: TCG_COND_GE
,
10560 tcg_op1
, tcg_op2
, tcg_tmp1
, tcg_tmp2
);
10563 case 8: /* SMLAL, SMLAL2, UMLAL, UMLAL2 */
10564 case 10: /* SMLSL, SMLSL2, UMLSL, UMLSL2 */
10565 case 12: /* UMULL, UMULL2, SMULL, SMULL2 */
10566 tcg_gen_mul_i64(tcg_passres
, tcg_op1
, tcg_op2
);
10568 case 9: /* SQDMLAL, SQDMLAL2 */
10569 case 11: /* SQDMLSL, SQDMLSL2 */
10570 case 13: /* SQDMULL, SQDMULL2 */
10571 tcg_gen_mul_i64(tcg_passres
, tcg_op1
, tcg_op2
);
10572 gen_helper_neon_addl_saturate_s64(tcg_passres
, tcg_env
,
10573 tcg_passres
, tcg_passres
);
10576 g_assert_not_reached();
10579 if (opcode
== 9 || opcode
== 11) {
10580 /* saturating accumulate ops */
10582 tcg_gen_neg_i64(tcg_passres
, tcg_passres
);
10584 gen_helper_neon_addl_saturate_s64(tcg_res
[pass
], tcg_env
,
10585 tcg_res
[pass
], tcg_passres
);
10586 } else if (accop
> 0) {
10587 tcg_gen_add_i64(tcg_res
[pass
], tcg_res
[pass
], tcg_passres
);
10588 } else if (accop
< 0) {
10589 tcg_gen_sub_i64(tcg_res
[pass
], tcg_res
[pass
], tcg_passres
);
10593 /* size 0 or 1, generally helper functions */
10594 for (pass
= 0; pass
< 2; pass
++) {
10595 TCGv_i32 tcg_op1
= tcg_temp_new_i32();
10596 TCGv_i32 tcg_op2
= tcg_temp_new_i32();
10597 TCGv_i64 tcg_passres
;
10598 int elt
= pass
+ is_q
* 2;
10600 read_vec_element_i32(s
, tcg_op1
, rn
, elt
, MO_32
);
10601 read_vec_element_i32(s
, tcg_op2
, rm
, elt
, MO_32
);
10604 tcg_passres
= tcg_res
[pass
];
10606 tcg_passres
= tcg_temp_new_i64();
10610 case 0: /* SADDL, SADDL2, UADDL, UADDL2 */
10611 case 2: /* SSUBL, SSUBL2, USUBL, USUBL2 */
10613 TCGv_i64 tcg_op2_64
= tcg_temp_new_i64();
10614 static NeonGenWidenFn
* const widenfns
[2][2] = {
10615 { gen_helper_neon_widen_s8
, gen_helper_neon_widen_u8
},
10616 { gen_helper_neon_widen_s16
, gen_helper_neon_widen_u16
},
10618 NeonGenWidenFn
*widenfn
= widenfns
[size
][is_u
];
10620 widenfn(tcg_op2_64
, tcg_op2
);
10621 widenfn(tcg_passres
, tcg_op1
);
10622 gen_neon_addl(size
, (opcode
== 2), tcg_passres
,
10623 tcg_passres
, tcg_op2_64
);
10626 case 5: /* SABAL, SABAL2, UABAL, UABAL2 */
10627 case 7: /* SABDL, SABDL2, UABDL, UABDL2 */
10630 gen_helper_neon_abdl_u16(tcg_passres
, tcg_op1
, tcg_op2
);
10632 gen_helper_neon_abdl_s16(tcg_passres
, tcg_op1
, tcg_op2
);
10636 gen_helper_neon_abdl_u32(tcg_passres
, tcg_op1
, tcg_op2
);
10638 gen_helper_neon_abdl_s32(tcg_passres
, tcg_op1
, tcg_op2
);
10642 case 8: /* SMLAL, SMLAL2, UMLAL, UMLAL2 */
10643 case 10: /* SMLSL, SMLSL2, UMLSL, UMLSL2 */
10644 case 12: /* UMULL, UMULL2, SMULL, SMULL2 */
10647 gen_helper_neon_mull_u8(tcg_passres
, tcg_op1
, tcg_op2
);
10649 gen_helper_neon_mull_s8(tcg_passres
, tcg_op1
, tcg_op2
);
10653 gen_helper_neon_mull_u16(tcg_passres
, tcg_op1
, tcg_op2
);
10655 gen_helper_neon_mull_s16(tcg_passres
, tcg_op1
, tcg_op2
);
10659 case 9: /* SQDMLAL, SQDMLAL2 */
10660 case 11: /* SQDMLSL, SQDMLSL2 */
10661 case 13: /* SQDMULL, SQDMULL2 */
10663 gen_helper_neon_mull_s16(tcg_passres
, tcg_op1
, tcg_op2
);
10664 gen_helper_neon_addl_saturate_s32(tcg_passres
, tcg_env
,
10665 tcg_passres
, tcg_passres
);
10668 g_assert_not_reached();
10672 if (opcode
== 9 || opcode
== 11) {
10673 /* saturating accumulate ops */
10675 gen_helper_neon_negl_u32(tcg_passres
, tcg_passres
);
10677 gen_helper_neon_addl_saturate_s32(tcg_res
[pass
], tcg_env
,
10681 gen_neon_addl(size
, (accop
< 0), tcg_res
[pass
],
10682 tcg_res
[pass
], tcg_passres
);
10688 write_vec_element(s
, tcg_res
[0], rd
, 0, MO_64
);
10689 write_vec_element(s
, tcg_res
[1], rd
, 1, MO_64
);
10692 static void handle_3rd_wide(DisasContext
*s
, int is_q
, int is_u
, int size
,
10693 int opcode
, int rd
, int rn
, int rm
)
10695 TCGv_i64 tcg_res
[2];
10696 int part
= is_q
? 2 : 0;
10699 for (pass
= 0; pass
< 2; pass
++) {
10700 TCGv_i64 tcg_op1
= tcg_temp_new_i64();
10701 TCGv_i32 tcg_op2
= tcg_temp_new_i32();
10702 TCGv_i64 tcg_op2_wide
= tcg_temp_new_i64();
10703 static NeonGenWidenFn
* const widenfns
[3][2] = {
10704 { gen_helper_neon_widen_s8
, gen_helper_neon_widen_u8
},
10705 { gen_helper_neon_widen_s16
, gen_helper_neon_widen_u16
},
10706 { tcg_gen_ext_i32_i64
, tcg_gen_extu_i32_i64
},
10708 NeonGenWidenFn
*widenfn
= widenfns
[size
][is_u
];
10710 read_vec_element(s
, tcg_op1
, rn
, pass
, MO_64
);
10711 read_vec_element_i32(s
, tcg_op2
, rm
, part
+ pass
, MO_32
);
10712 widenfn(tcg_op2_wide
, tcg_op2
);
10713 tcg_res
[pass
] = tcg_temp_new_i64();
10714 gen_neon_addl(size
, (opcode
== 3),
10715 tcg_res
[pass
], tcg_op1
, tcg_op2_wide
);
10718 for (pass
= 0; pass
< 2; pass
++) {
10719 write_vec_element(s
, tcg_res
[pass
], rd
, pass
, MO_64
);
10723 static void do_narrow_round_high_u32(TCGv_i32 res
, TCGv_i64 in
)
10725 tcg_gen_addi_i64(in
, in
, 1U << 31);
10726 tcg_gen_extrh_i64_i32(res
, in
);
10729 static void handle_3rd_narrowing(DisasContext
*s
, int is_q
, int is_u
, int size
,
10730 int opcode
, int rd
, int rn
, int rm
)
10732 TCGv_i32 tcg_res
[2];
10733 int part
= is_q
? 2 : 0;
10736 for (pass
= 0; pass
< 2; pass
++) {
10737 TCGv_i64 tcg_op1
= tcg_temp_new_i64();
10738 TCGv_i64 tcg_op2
= tcg_temp_new_i64();
10739 TCGv_i64 tcg_wideres
= tcg_temp_new_i64();
10740 static NeonGenNarrowFn
* const narrowfns
[3][2] = {
10741 { gen_helper_neon_narrow_high_u8
,
10742 gen_helper_neon_narrow_round_high_u8
},
10743 { gen_helper_neon_narrow_high_u16
,
10744 gen_helper_neon_narrow_round_high_u16
},
10745 { tcg_gen_extrh_i64_i32
, do_narrow_round_high_u32
},
10747 NeonGenNarrowFn
*gennarrow
= narrowfns
[size
][is_u
];
10749 read_vec_element(s
, tcg_op1
, rn
, pass
, MO_64
);
10750 read_vec_element(s
, tcg_op2
, rm
, pass
, MO_64
);
10752 gen_neon_addl(size
, (opcode
== 6), tcg_wideres
, tcg_op1
, tcg_op2
);
10754 tcg_res
[pass
] = tcg_temp_new_i32();
10755 gennarrow(tcg_res
[pass
], tcg_wideres
);
10758 for (pass
= 0; pass
< 2; pass
++) {
10759 write_vec_element_i32(s
, tcg_res
[pass
], rd
, pass
+ part
, MO_32
);
10761 clear_vec_high(s
, is_q
, rd
);
10764 /* AdvSIMD three different
10765 * 31 30 29 28 24 23 22 21 20 16 15 12 11 10 9 5 4 0
10766 * +---+---+---+-----------+------+---+------+--------+-----+------+------+
10767 * | 0 | Q | U | 0 1 1 1 0 | size | 1 | Rm | opcode | 0 0 | Rn | Rd |
10768 * +---+---+---+-----------+------+---+------+--------+-----+------+------+
10770 static void disas_simd_three_reg_diff(DisasContext
*s
, uint32_t insn
)
10772 /* Instructions in this group fall into three basic classes
10773 * (in each case with the operation working on each element in
10774 * the input vectors):
10775 * (1) widening 64 x 64 -> 128 (with possibly Vd as an extra
10777 * (2) wide 64 x 128 -> 128
10778 * (3) narrowing 128 x 128 -> 64
10779 * Here we do initial decode, catch unallocated cases and
10780 * dispatch to separate functions for each class.
10782 int is_q
= extract32(insn
, 30, 1);
10783 int is_u
= extract32(insn
, 29, 1);
10784 int size
= extract32(insn
, 22, 2);
10785 int opcode
= extract32(insn
, 12, 4);
10786 int rm
= extract32(insn
, 16, 5);
10787 int rn
= extract32(insn
, 5, 5);
10788 int rd
= extract32(insn
, 0, 5);
10791 case 1: /* SADDW, SADDW2, UADDW, UADDW2 */
10792 case 3: /* SSUBW, SSUBW2, USUBW, USUBW2 */
10793 /* 64 x 128 -> 128 */
10795 unallocated_encoding(s
);
10798 if (!fp_access_check(s
)) {
10801 handle_3rd_wide(s
, is_q
, is_u
, size
, opcode
, rd
, rn
, rm
);
10803 case 4: /* ADDHN, ADDHN2, RADDHN, RADDHN2 */
10804 case 6: /* SUBHN, SUBHN2, RSUBHN, RSUBHN2 */
10805 /* 128 x 128 -> 64 */
10807 unallocated_encoding(s
);
10810 if (!fp_access_check(s
)) {
10813 handle_3rd_narrowing(s
, is_q
, is_u
, size
, opcode
, rd
, rn
, rm
);
10815 case 14: /* PMULL, PMULL2 */
10817 unallocated_encoding(s
);
10821 case 0: /* PMULL.P8 */
10822 if (!fp_access_check(s
)) {
10825 /* The Q field specifies lo/hi half input for this insn. */
10826 gen_gvec_op3_ool(s
, true, rd
, rn
, rm
, is_q
,
10827 gen_helper_neon_pmull_h
);
10830 case 3: /* PMULL.P64 */
10831 if (!dc_isar_feature(aa64_pmull
, s
)) {
10832 unallocated_encoding(s
);
10835 if (!fp_access_check(s
)) {
10838 /* The Q field specifies lo/hi half input for this insn. */
10839 gen_gvec_op3_ool(s
, true, rd
, rn
, rm
, is_q
,
10840 gen_helper_gvec_pmull_q
);
10844 unallocated_encoding(s
);
10848 case 9: /* SQDMLAL, SQDMLAL2 */
10849 case 11: /* SQDMLSL, SQDMLSL2 */
10850 case 13: /* SQDMULL, SQDMULL2 */
10851 if (is_u
|| size
== 0) {
10852 unallocated_encoding(s
);
10856 case 0: /* SADDL, SADDL2, UADDL, UADDL2 */
10857 case 2: /* SSUBL, SSUBL2, USUBL, USUBL2 */
10858 case 5: /* SABAL, SABAL2, UABAL, UABAL2 */
10859 case 7: /* SABDL, SABDL2, UABDL, UABDL2 */
10860 case 8: /* SMLAL, SMLAL2, UMLAL, UMLAL2 */
10861 case 10: /* SMLSL, SMLSL2, UMLSL, UMLSL2 */
10862 case 12: /* SMULL, SMULL2, UMULL, UMULL2 */
10863 /* 64 x 64 -> 128 */
10865 unallocated_encoding(s
);
10868 if (!fp_access_check(s
)) {
10872 handle_3rd_widening(s
, is_q
, is_u
, size
, opcode
, rd
, rn
, rm
);
10875 /* opcode 15 not allocated */
10876 unallocated_encoding(s
);
10881 /* Integer op subgroup of C3.6.16. */
10882 static void disas_simd_3same_int(DisasContext
*s
, uint32_t insn
)
10884 int is_q
= extract32(insn
, 30, 1);
10885 int u
= extract32(insn
, 29, 1);
10886 int size
= extract32(insn
, 22, 2);
10887 int opcode
= extract32(insn
, 11, 5);
10888 int rm
= extract32(insn
, 16, 5);
10889 int rn
= extract32(insn
, 5, 5);
10890 int rd
= extract32(insn
, 0, 5);
10895 case 0x13: /* MUL, PMUL */
10896 if (u
&& size
!= 0) {
10897 unallocated_encoding(s
);
10901 case 0x0: /* SHADD, UHADD */
10902 case 0x2: /* SRHADD, URHADD */
10903 case 0x4: /* SHSUB, UHSUB */
10904 case 0xc: /* SMAX, UMAX */
10905 case 0xd: /* SMIN, UMIN */
10906 case 0xe: /* SABD, UABD */
10907 case 0xf: /* SABA, UABA */
10908 case 0x12: /* MLA, MLS */
10910 unallocated_encoding(s
);
10914 case 0x16: /* SQDMULH, SQRDMULH */
10915 if (size
== 0 || size
== 3) {
10916 unallocated_encoding(s
);
10921 if (size
== 3 && !is_q
) {
10922 unallocated_encoding(s
);
10927 case 0x01: /* SQADD, UQADD */
10928 case 0x05: /* SQSUB, UQSUB */
10929 case 0x08: /* SSHL, USHL */
10930 case 0x0a: /* SRSHL, URSHL */
10931 unallocated_encoding(s
);
10935 if (!fp_access_check(s
)) {
10940 case 0x09: /* SQSHL, UQSHL */
10942 gen_gvec_fn3(s
, is_q
, rd
, rn
, rm
, gen_neon_uqshl
, size
);
10944 gen_gvec_fn3(s
, is_q
, rd
, rn
, rm
, gen_neon_sqshl
, size
);
10947 case 0x0c: /* SMAX, UMAX */
10949 gen_gvec_fn3(s
, is_q
, rd
, rn
, rm
, tcg_gen_gvec_umax
, size
);
10951 gen_gvec_fn3(s
, is_q
, rd
, rn
, rm
, tcg_gen_gvec_smax
, size
);
10954 case 0x0d: /* SMIN, UMIN */
10956 gen_gvec_fn3(s
, is_q
, rd
, rn
, rm
, tcg_gen_gvec_umin
, size
);
10958 gen_gvec_fn3(s
, is_q
, rd
, rn
, rm
, tcg_gen_gvec_smin
, size
);
10961 case 0xe: /* SABD, UABD */
10963 gen_gvec_fn3(s
, is_q
, rd
, rn
, rm
, gen_gvec_uabd
, size
);
10965 gen_gvec_fn3(s
, is_q
, rd
, rn
, rm
, gen_gvec_sabd
, size
);
10968 case 0xf: /* SABA, UABA */
10970 gen_gvec_fn3(s
, is_q
, rd
, rn
, rm
, gen_gvec_uaba
, size
);
10972 gen_gvec_fn3(s
, is_q
, rd
, rn
, rm
, gen_gvec_saba
, size
);
10975 case 0x10: /* ADD, SUB */
10977 gen_gvec_fn3(s
, is_q
, rd
, rn
, rm
, tcg_gen_gvec_sub
, size
);
10979 gen_gvec_fn3(s
, is_q
, rd
, rn
, rm
, tcg_gen_gvec_add
, size
);
10982 case 0x13: /* MUL, PMUL */
10983 if (!u
) { /* MUL */
10984 gen_gvec_fn3(s
, is_q
, rd
, rn
, rm
, tcg_gen_gvec_mul
, size
);
10985 } else { /* PMUL */
10986 gen_gvec_op3_ool(s
, is_q
, rd
, rn
, rm
, 0, gen_helper_gvec_pmul_b
);
10989 case 0x12: /* MLA, MLS */
10991 gen_gvec_fn3(s
, is_q
, rd
, rn
, rm
, gen_gvec_mls
, size
);
10993 gen_gvec_fn3(s
, is_q
, rd
, rn
, rm
, gen_gvec_mla
, size
);
10996 case 0x16: /* SQDMULH, SQRDMULH */
10998 static gen_helper_gvec_3_ptr
* const fns
[2][2] = {
10999 { gen_helper_neon_sqdmulh_h
, gen_helper_neon_sqrdmulh_h
},
11000 { gen_helper_neon_sqdmulh_s
, gen_helper_neon_sqrdmulh_s
},
11002 gen_gvec_op3_qc(s
, is_q
, rd
, rn
, rm
, fns
[size
- 1][u
]);
11006 if (!u
) { /* CMTST */
11007 gen_gvec_fn3(s
, is_q
, rd
, rn
, rm
, gen_gvec_cmtst
, size
);
11011 cond
= TCG_COND_EQ
;
11013 case 0x06: /* CMGT, CMHI */
11014 cond
= u
? TCG_COND_GTU
: TCG_COND_GT
;
11016 case 0x07: /* CMGE, CMHS */
11017 cond
= u
? TCG_COND_GEU
: TCG_COND_GE
;
11019 tcg_gen_gvec_cmp(cond
, size
, vec_full_reg_offset(s
, rd
),
11020 vec_full_reg_offset(s
, rn
),
11021 vec_full_reg_offset(s
, rm
),
11022 is_q
? 16 : 8, vec_full_reg_size(s
));
11028 for (pass
= 0; pass
< 2; pass
++) {
11029 TCGv_i64 tcg_op1
= tcg_temp_new_i64();
11030 TCGv_i64 tcg_op2
= tcg_temp_new_i64();
11031 TCGv_i64 tcg_res
= tcg_temp_new_i64();
11033 read_vec_element(s
, tcg_op1
, rn
, pass
, MO_64
);
11034 read_vec_element(s
, tcg_op2
, rm
, pass
, MO_64
);
11036 handle_3same_64(s
, opcode
, u
, tcg_res
, tcg_op1
, tcg_op2
);
11038 write_vec_element(s
, tcg_res
, rd
, pass
, MO_64
);
11041 for (pass
= 0; pass
< (is_q
? 4 : 2); pass
++) {
11042 TCGv_i32 tcg_op1
= tcg_temp_new_i32();
11043 TCGv_i32 tcg_op2
= tcg_temp_new_i32();
11044 TCGv_i32 tcg_res
= tcg_temp_new_i32();
11045 NeonGenTwoOpFn
*genfn
= NULL
;
11046 NeonGenTwoOpEnvFn
*genenvfn
= NULL
;
11048 read_vec_element_i32(s
, tcg_op1
, rn
, pass
, MO_32
);
11049 read_vec_element_i32(s
, tcg_op2
, rm
, pass
, MO_32
);
11052 case 0x0: /* SHADD, UHADD */
11054 static NeonGenTwoOpFn
* const fns
[3][2] = {
11055 { gen_helper_neon_hadd_s8
, gen_helper_neon_hadd_u8
},
11056 { gen_helper_neon_hadd_s16
, gen_helper_neon_hadd_u16
},
11057 { gen_helper_neon_hadd_s32
, gen_helper_neon_hadd_u32
},
11059 genfn
= fns
[size
][u
];
11062 case 0x2: /* SRHADD, URHADD */
11064 static NeonGenTwoOpFn
* const fns
[3][2] = {
11065 { gen_helper_neon_rhadd_s8
, gen_helper_neon_rhadd_u8
},
11066 { gen_helper_neon_rhadd_s16
, gen_helper_neon_rhadd_u16
},
11067 { gen_helper_neon_rhadd_s32
, gen_helper_neon_rhadd_u32
},
11069 genfn
= fns
[size
][u
];
11072 case 0x4: /* SHSUB, UHSUB */
11074 static NeonGenTwoOpFn
* const fns
[3][2] = {
11075 { gen_helper_neon_hsub_s8
, gen_helper_neon_hsub_u8
},
11076 { gen_helper_neon_hsub_s16
, gen_helper_neon_hsub_u16
},
11077 { gen_helper_neon_hsub_s32
, gen_helper_neon_hsub_u32
},
11079 genfn
= fns
[size
][u
];
11082 case 0xb: /* SQRSHL, UQRSHL */
11084 static NeonGenTwoOpEnvFn
* const fns
[3][2] = {
11085 { gen_helper_neon_qrshl_s8
, gen_helper_neon_qrshl_u8
},
11086 { gen_helper_neon_qrshl_s16
, gen_helper_neon_qrshl_u16
},
11087 { gen_helper_neon_qrshl_s32
, gen_helper_neon_qrshl_u32
},
11089 genenvfn
= fns
[size
][u
];
11093 g_assert_not_reached();
11097 genenvfn(tcg_res
, tcg_env
, tcg_op1
, tcg_op2
);
11099 genfn(tcg_res
, tcg_op1
, tcg_op2
);
11102 write_vec_element_i32(s
, tcg_res
, rd
, pass
, MO_32
);
11105 clear_vec_high(s
, is_q
, rd
);
11108 /* AdvSIMD three same
11109 * 31 30 29 28 24 23 22 21 20 16 15 11 10 9 5 4 0
11110 * +---+---+---+-----------+------+---+------+--------+---+------+------+
11111 * | 0 | Q | U | 0 1 1 1 0 | size | 1 | Rm | opcode | 1 | Rn | Rd |
11112 * +---+---+---+-----------+------+---+------+--------+---+------+------+
11114 static void disas_simd_three_reg_same(DisasContext
*s
, uint32_t insn
)
11116 int opcode
= extract32(insn
, 11, 5);
11120 disas_simd_3same_int(s
, insn
);
11122 case 0x3: /* logic ops */
11123 case 0x14: /* SMAXP, UMAXP */
11124 case 0x15: /* SMINP, UMINP */
11125 case 0x17: /* ADDP */
11126 case 0x18 ... 0x31: /* floating point ops */
11127 unallocated_encoding(s
);
11132 /* AdvSIMD three same extra
11133 * 31 30 29 28 24 23 22 21 20 16 15 14 11 10 9 5 4 0
11134 * +---+---+---+-----------+------+---+------+---+--------+---+----+----+
11135 * | 0 | Q | U | 0 1 1 1 0 | size | 0 | Rm | 1 | opcode | 1 | Rn | Rd |
11136 * +---+---+---+-----------+------+---+------+---+--------+---+----+----+
11138 static void disas_simd_three_reg_same_extra(DisasContext
*s
, uint32_t insn
)
11140 int rd
= extract32(insn
, 0, 5);
11141 int rn
= extract32(insn
, 5, 5);
11142 int opcode
= extract32(insn
, 11, 4);
11143 int rm
= extract32(insn
, 16, 5);
11144 int size
= extract32(insn
, 22, 2);
11145 bool u
= extract32(insn
, 29, 1);
11146 bool is_q
= extract32(insn
, 30, 1);
11150 switch (u
* 16 + opcode
) {
11151 case 0x10: /* SQRDMLAH (vector) */
11152 case 0x11: /* SQRDMLSH (vector) */
11153 if (size
!= 1 && size
!= 2) {
11154 unallocated_encoding(s
);
11157 feature
= dc_isar_feature(aa64_rdm
, s
);
11159 case 0x02: /* SDOT (vector) */
11160 case 0x12: /* UDOT (vector) */
11161 if (size
!= MO_32
) {
11162 unallocated_encoding(s
);
11165 feature
= dc_isar_feature(aa64_dp
, s
);
11167 case 0x03: /* USDOT */
11168 if (size
!= MO_32
) {
11169 unallocated_encoding(s
);
11172 feature
= dc_isar_feature(aa64_i8mm
, s
);
11174 case 0x04: /* SMMLA */
11175 case 0x14: /* UMMLA */
11176 case 0x05: /* USMMLA */
11177 if (!is_q
|| size
!= MO_32
) {
11178 unallocated_encoding(s
);
11181 feature
= dc_isar_feature(aa64_i8mm
, s
);
11183 case 0x18: /* FCMLA, #0 */
11184 case 0x19: /* FCMLA, #90 */
11185 case 0x1a: /* FCMLA, #180 */
11186 case 0x1b: /* FCMLA, #270 */
11187 case 0x1c: /* FCADD, #90 */
11188 case 0x1e: /* FCADD, #270 */
11190 || (size
== 1 && !dc_isar_feature(aa64_fp16
, s
))
11191 || (size
== 3 && !is_q
)) {
11192 unallocated_encoding(s
);
11195 feature
= dc_isar_feature(aa64_fcma
, s
);
11197 case 0x1d: /* BFMMLA */
11198 if (size
!= MO_16
|| !is_q
) {
11199 unallocated_encoding(s
);
11202 feature
= dc_isar_feature(aa64_bf16
, s
);
11206 case 1: /* BFDOT */
11207 case 3: /* BFMLAL{B,T} */
11208 feature
= dc_isar_feature(aa64_bf16
, s
);
11211 unallocated_encoding(s
);
11216 unallocated_encoding(s
);
11220 unallocated_encoding(s
);
11223 if (!fp_access_check(s
)) {
11228 case 0x0: /* SQRDMLAH (vector) */
11229 gen_gvec_fn3(s
, is_q
, rd
, rn
, rm
, gen_gvec_sqrdmlah_qc
, size
);
11232 case 0x1: /* SQRDMLSH (vector) */
11233 gen_gvec_fn3(s
, is_q
, rd
, rn
, rm
, gen_gvec_sqrdmlsh_qc
, size
);
11236 case 0x2: /* SDOT / UDOT */
11237 gen_gvec_op4_ool(s
, is_q
, rd
, rn
, rm
, rd
, 0,
11238 u
? gen_helper_gvec_udot_b
: gen_helper_gvec_sdot_b
);
11241 case 0x3: /* USDOT */
11242 gen_gvec_op4_ool(s
, is_q
, rd
, rn
, rm
, rd
, 0, gen_helper_gvec_usdot_b
);
11245 case 0x04: /* SMMLA, UMMLA */
11246 gen_gvec_op4_ool(s
, 1, rd
, rn
, rm
, rd
, 0,
11247 u
? gen_helper_gvec_ummla_b
11248 : gen_helper_gvec_smmla_b
);
11250 case 0x05: /* USMMLA */
11251 gen_gvec_op4_ool(s
, 1, rd
, rn
, rm
, rd
, 0, gen_helper_gvec_usmmla_b
);
11254 case 0x8: /* FCMLA, #0 */
11255 case 0x9: /* FCMLA, #90 */
11256 case 0xa: /* FCMLA, #180 */
11257 case 0xb: /* FCMLA, #270 */
11258 rot
= extract32(opcode
, 0, 2);
11261 gen_gvec_op4_fpst(s
, is_q
, rd
, rn
, rm
, rd
, true, rot
,
11262 gen_helper_gvec_fcmlah
);
11265 gen_gvec_op4_fpst(s
, is_q
, rd
, rn
, rm
, rd
, false, rot
,
11266 gen_helper_gvec_fcmlas
);
11269 gen_gvec_op4_fpst(s
, is_q
, rd
, rn
, rm
, rd
, false, rot
,
11270 gen_helper_gvec_fcmlad
);
11273 g_assert_not_reached();
11277 case 0xc: /* FCADD, #90 */
11278 case 0xe: /* FCADD, #270 */
11279 rot
= extract32(opcode
, 1, 1);
11282 gen_gvec_op3_fpst(s
, is_q
, rd
, rn
, rm
, size
== 1, rot
,
11283 gen_helper_gvec_fcaddh
);
11286 gen_gvec_op3_fpst(s
, is_q
, rd
, rn
, rm
, size
== 1, rot
,
11287 gen_helper_gvec_fcadds
);
11290 gen_gvec_op3_fpst(s
, is_q
, rd
, rn
, rm
, size
== 1, rot
,
11291 gen_helper_gvec_fcaddd
);
11294 g_assert_not_reached();
11298 case 0xd: /* BFMMLA */
11299 gen_gvec_op4_ool(s
, is_q
, rd
, rn
, rm
, rd
, 0, gen_helper_gvec_bfmmla
);
11303 case 1: /* BFDOT */
11304 gen_gvec_op4_ool(s
, is_q
, rd
, rn
, rm
, rd
, 0, gen_helper_gvec_bfdot
);
11306 case 3: /* BFMLAL{B,T} */
11307 gen_gvec_op4_fpst(s
, 1, rd
, rn
, rm
, rd
, false, is_q
,
11308 gen_helper_gvec_bfmlal
);
11311 g_assert_not_reached();
11316 g_assert_not_reached();
11320 static void handle_2misc_widening(DisasContext
*s
, int opcode
, bool is_q
,
11321 int size
, int rn
, int rd
)
11323 /* Handle 2-reg-misc ops which are widening (so each size element
11324 * in the source becomes a 2*size element in the destination.
11325 * The only instruction like this is FCVTL.
11330 /* 32 -> 64 bit fp conversion */
11331 TCGv_i64 tcg_res
[2];
11332 int srcelt
= is_q
? 2 : 0;
11334 for (pass
= 0; pass
< 2; pass
++) {
11335 TCGv_i32 tcg_op
= tcg_temp_new_i32();
11336 tcg_res
[pass
] = tcg_temp_new_i64();
11338 read_vec_element_i32(s
, tcg_op
, rn
, srcelt
+ pass
, MO_32
);
11339 gen_helper_vfp_fcvtds(tcg_res
[pass
], tcg_op
, tcg_env
);
11341 for (pass
= 0; pass
< 2; pass
++) {
11342 write_vec_element(s
, tcg_res
[pass
], rd
, pass
, MO_64
);
11345 /* 16 -> 32 bit fp conversion */
11346 int srcelt
= is_q
? 4 : 0;
11347 TCGv_i32 tcg_res
[4];
11348 TCGv_ptr fpst
= fpstatus_ptr(FPST_FPCR
);
11349 TCGv_i32 ahp
= get_ahp_flag();
11351 for (pass
= 0; pass
< 4; pass
++) {
11352 tcg_res
[pass
] = tcg_temp_new_i32();
11354 read_vec_element_i32(s
, tcg_res
[pass
], rn
, srcelt
+ pass
, MO_16
);
11355 gen_helper_vfp_fcvt_f16_to_f32(tcg_res
[pass
], tcg_res
[pass
],
11358 for (pass
= 0; pass
< 4; pass
++) {
11359 write_vec_element_i32(s
, tcg_res
[pass
], rd
, pass
, MO_32
);
11364 static void handle_rev(DisasContext
*s
, int opcode
, bool u
,
11365 bool is_q
, int size
, int rn
, int rd
)
11367 int op
= (opcode
<< 1) | u
;
11368 int opsz
= op
+ size
;
11369 int grp_size
= 3 - opsz
;
11370 int dsize
= is_q
? 128 : 64;
11374 unallocated_encoding(s
);
11378 if (!fp_access_check(s
)) {
11383 /* Special case bytes, use bswap op on each group of elements */
11384 int groups
= dsize
/ (8 << grp_size
);
11386 for (i
= 0; i
< groups
; i
++) {
11387 TCGv_i64 tcg_tmp
= tcg_temp_new_i64();
11389 read_vec_element(s
, tcg_tmp
, rn
, i
, grp_size
);
11390 switch (grp_size
) {
11392 tcg_gen_bswap16_i64(tcg_tmp
, tcg_tmp
, TCG_BSWAP_IZ
);
11395 tcg_gen_bswap32_i64(tcg_tmp
, tcg_tmp
, TCG_BSWAP_IZ
);
11398 tcg_gen_bswap64_i64(tcg_tmp
, tcg_tmp
);
11401 g_assert_not_reached();
11403 write_vec_element(s
, tcg_tmp
, rd
, i
, grp_size
);
11405 clear_vec_high(s
, is_q
, rd
);
11407 int revmask
= (1 << grp_size
) - 1;
11408 int esize
= 8 << size
;
11409 int elements
= dsize
/ esize
;
11410 TCGv_i64 tcg_rn
= tcg_temp_new_i64();
11411 TCGv_i64 tcg_rd
[2];
11413 for (i
= 0; i
< 2; i
++) {
11414 tcg_rd
[i
] = tcg_temp_new_i64();
11415 tcg_gen_movi_i64(tcg_rd
[i
], 0);
11418 for (i
= 0; i
< elements
; i
++) {
11419 int e_rev
= (i
& 0xf) ^ revmask
;
11420 int w
= (e_rev
* esize
) / 64;
11421 int o
= (e_rev
* esize
) % 64;
11423 read_vec_element(s
, tcg_rn
, rn
, i
, size
);
11424 tcg_gen_deposit_i64(tcg_rd
[w
], tcg_rd
[w
], tcg_rn
, o
, esize
);
11427 for (i
= 0; i
< 2; i
++) {
11428 write_vec_element(s
, tcg_rd
[i
], rd
, i
, MO_64
);
11430 clear_vec_high(s
, true, rd
);
11434 static void handle_2misc_pairwise(DisasContext
*s
, int opcode
, bool u
,
11435 bool is_q
, int size
, int rn
, int rd
)
11437 /* Implement the pairwise operations from 2-misc:
11438 * SADDLP, UADDLP, SADALP, UADALP.
11439 * These all add pairs of elements in the input to produce a
11440 * double-width result element in the output (possibly accumulating).
11442 bool accum
= (opcode
== 0x6);
11443 int maxpass
= is_q
? 2 : 1;
11445 TCGv_i64 tcg_res
[2];
11448 /* 32 + 32 -> 64 op */
11449 MemOp memop
= size
+ (u
? 0 : MO_SIGN
);
11451 for (pass
= 0; pass
< maxpass
; pass
++) {
11452 TCGv_i64 tcg_op1
= tcg_temp_new_i64();
11453 TCGv_i64 tcg_op2
= tcg_temp_new_i64();
11455 tcg_res
[pass
] = tcg_temp_new_i64();
11457 read_vec_element(s
, tcg_op1
, rn
, pass
* 2, memop
);
11458 read_vec_element(s
, tcg_op2
, rn
, pass
* 2 + 1, memop
);
11459 tcg_gen_add_i64(tcg_res
[pass
], tcg_op1
, tcg_op2
);
11461 read_vec_element(s
, tcg_op1
, rd
, pass
, MO_64
);
11462 tcg_gen_add_i64(tcg_res
[pass
], tcg_res
[pass
], tcg_op1
);
11466 for (pass
= 0; pass
< maxpass
; pass
++) {
11467 TCGv_i64 tcg_op
= tcg_temp_new_i64();
11468 NeonGenOne64OpFn
*genfn
;
11469 static NeonGenOne64OpFn
* const fns
[2][2] = {
11470 { gen_helper_neon_addlp_s8
, gen_helper_neon_addlp_u8
},
11471 { gen_helper_neon_addlp_s16
, gen_helper_neon_addlp_u16
},
11474 genfn
= fns
[size
][u
];
11476 tcg_res
[pass
] = tcg_temp_new_i64();
11478 read_vec_element(s
, tcg_op
, rn
, pass
, MO_64
);
11479 genfn(tcg_res
[pass
], tcg_op
);
11482 read_vec_element(s
, tcg_op
, rd
, pass
, MO_64
);
11484 gen_helper_neon_addl_u16(tcg_res
[pass
],
11485 tcg_res
[pass
], tcg_op
);
11487 gen_helper_neon_addl_u32(tcg_res
[pass
],
11488 tcg_res
[pass
], tcg_op
);
11494 tcg_res
[1] = tcg_constant_i64(0);
11496 for (pass
= 0; pass
< 2; pass
++) {
11497 write_vec_element(s
, tcg_res
[pass
], rd
, pass
, MO_64
);
11501 static void handle_shll(DisasContext
*s
, bool is_q
, int size
, int rn
, int rd
)
11503 /* Implement SHLL and SHLL2 */
11505 int part
= is_q
? 2 : 0;
11506 TCGv_i64 tcg_res
[2];
11508 for (pass
= 0; pass
< 2; pass
++) {
11509 static NeonGenWidenFn
* const widenfns
[3] = {
11510 gen_helper_neon_widen_u8
,
11511 gen_helper_neon_widen_u16
,
11512 tcg_gen_extu_i32_i64
,
11514 NeonGenWidenFn
*widenfn
= widenfns
[size
];
11515 TCGv_i32 tcg_op
= tcg_temp_new_i32();
11517 read_vec_element_i32(s
, tcg_op
, rn
, part
+ pass
, MO_32
);
11518 tcg_res
[pass
] = tcg_temp_new_i64();
11519 widenfn(tcg_res
[pass
], tcg_op
);
11520 tcg_gen_shli_i64(tcg_res
[pass
], tcg_res
[pass
], 8 << size
);
11523 for (pass
= 0; pass
< 2; pass
++) {
11524 write_vec_element(s
, tcg_res
[pass
], rd
, pass
, MO_64
);
11528 /* AdvSIMD two reg misc
11529 * 31 30 29 28 24 23 22 21 17 16 12 11 10 9 5 4 0
11530 * +---+---+---+-----------+------+-----------+--------+-----+------+------+
11531 * | 0 | Q | U | 0 1 1 1 0 | size | 1 0 0 0 0 | opcode | 1 0 | Rn | Rd |
11532 * +---+---+---+-----------+------+-----------+--------+-----+------+------+
11534 static void disas_simd_two_reg_misc(DisasContext
*s
, uint32_t insn
)
11536 int size
= extract32(insn
, 22, 2);
11537 int opcode
= extract32(insn
, 12, 5);
11538 bool u
= extract32(insn
, 29, 1);
11539 bool is_q
= extract32(insn
, 30, 1);
11540 int rn
= extract32(insn
, 5, 5);
11541 int rd
= extract32(insn
, 0, 5);
11542 bool need_fpstatus
= false;
11544 TCGv_i32 tcg_rmode
;
11545 TCGv_ptr tcg_fpstatus
;
11548 case 0x0: /* REV64, REV32 */
11549 case 0x1: /* REV16 */
11550 handle_rev(s
, opcode
, u
, is_q
, size
, rn
, rd
);
11552 case 0x5: /* CNT, NOT, RBIT */
11553 if (u
&& size
== 0) {
11556 } else if (u
&& size
== 1) {
11559 } else if (!u
&& size
== 0) {
11563 unallocated_encoding(s
);
11565 case 0x12: /* XTN, XTN2, SQXTUN, SQXTUN2 */
11566 case 0x14: /* SQXTN, SQXTN2, UQXTN, UQXTN2 */
11568 unallocated_encoding(s
);
11571 if (!fp_access_check(s
)) {
11575 handle_2misc_narrow(s
, false, opcode
, u
, is_q
, size
, rn
, rd
);
11577 case 0x4: /* CLS, CLZ */
11579 unallocated_encoding(s
);
11583 case 0x2: /* SADDLP, UADDLP */
11584 case 0x6: /* SADALP, UADALP */
11586 unallocated_encoding(s
);
11589 if (!fp_access_check(s
)) {
11592 handle_2misc_pairwise(s
, opcode
, u
, is_q
, size
, rn
, rd
);
11594 case 0x13: /* SHLL, SHLL2 */
11595 if (u
== 0 || size
== 3) {
11596 unallocated_encoding(s
);
11599 if (!fp_access_check(s
)) {
11602 handle_shll(s
, is_q
, size
, rn
, rd
);
11604 case 0xa: /* CMLT */
11606 unallocated_encoding(s
);
11610 case 0x8: /* CMGT, CMGE */
11611 case 0x9: /* CMEQ, CMLE */
11612 case 0xb: /* ABS, NEG */
11613 if (size
== 3 && !is_q
) {
11614 unallocated_encoding(s
);
11618 case 0x7: /* SQABS, SQNEG */
11619 if (size
== 3 && !is_q
) {
11620 unallocated_encoding(s
);
11625 case 0x16 ... 0x1f:
11627 /* Floating point: U, size[1] and opcode indicate operation;
11628 * size[0] indicates single or double precision.
11630 int is_double
= extract32(size
, 0, 1);
11631 opcode
|= (extract32(size
, 1, 1) << 5) | (u
<< 6);
11632 size
= is_double
? 3 : 2;
11634 case 0x2f: /* FABS */
11635 case 0x6f: /* FNEG */
11636 if (size
== 3 && !is_q
) {
11637 unallocated_encoding(s
);
11641 case 0x1d: /* SCVTF */
11642 case 0x5d: /* UCVTF */
11644 bool is_signed
= (opcode
== 0x1d) ? true : false;
11645 int elements
= is_double
? 2 : is_q
? 4 : 2;
11646 if (is_double
&& !is_q
) {
11647 unallocated_encoding(s
);
11650 if (!fp_access_check(s
)) {
11653 handle_simd_intfp_conv(s
, rd
, rn
, elements
, is_signed
, 0, size
);
11656 case 0x2c: /* FCMGT (zero) */
11657 case 0x2d: /* FCMEQ (zero) */
11658 case 0x2e: /* FCMLT (zero) */
11659 case 0x6c: /* FCMGE (zero) */
11660 case 0x6d: /* FCMLE (zero) */
11661 if (size
== 3 && !is_q
) {
11662 unallocated_encoding(s
);
11665 handle_2misc_fcmp_zero(s
, opcode
, false, u
, is_q
, size
, rn
, rd
);
11667 case 0x7f: /* FSQRT */
11668 if (size
== 3 && !is_q
) {
11669 unallocated_encoding(s
);
11673 case 0x1a: /* FCVTNS */
11674 case 0x1b: /* FCVTMS */
11675 case 0x3a: /* FCVTPS */
11676 case 0x3b: /* FCVTZS */
11677 case 0x5a: /* FCVTNU */
11678 case 0x5b: /* FCVTMU */
11679 case 0x7a: /* FCVTPU */
11680 case 0x7b: /* FCVTZU */
11681 need_fpstatus
= true;
11682 rmode
= extract32(opcode
, 5, 1) | (extract32(opcode
, 0, 1) << 1);
11683 if (size
== 3 && !is_q
) {
11684 unallocated_encoding(s
);
11688 case 0x5c: /* FCVTAU */
11689 case 0x1c: /* FCVTAS */
11690 need_fpstatus
= true;
11691 rmode
= FPROUNDING_TIEAWAY
;
11692 if (size
== 3 && !is_q
) {
11693 unallocated_encoding(s
);
11697 case 0x3c: /* URECPE */
11699 unallocated_encoding(s
);
11703 case 0x3d: /* FRECPE */
11704 case 0x7d: /* FRSQRTE */
11705 if (size
== 3 && !is_q
) {
11706 unallocated_encoding(s
);
11709 if (!fp_access_check(s
)) {
11712 handle_2misc_reciprocal(s
, opcode
, false, u
, is_q
, size
, rn
, rd
);
11714 case 0x56: /* FCVTXN, FCVTXN2 */
11716 unallocated_encoding(s
);
11720 case 0x16: /* FCVTN, FCVTN2 */
11721 /* handle_2misc_narrow does a 2*size -> size operation, but these
11722 * instructions encode the source size rather than dest size.
11724 if (!fp_access_check(s
)) {
11727 handle_2misc_narrow(s
, false, opcode
, 0, is_q
, size
- 1, rn
, rd
);
11729 case 0x36: /* BFCVTN, BFCVTN2 */
11730 if (!dc_isar_feature(aa64_bf16
, s
) || size
!= 2) {
11731 unallocated_encoding(s
);
11734 if (!fp_access_check(s
)) {
11737 handle_2misc_narrow(s
, false, opcode
, 0, is_q
, size
- 1, rn
, rd
);
11739 case 0x17: /* FCVTL, FCVTL2 */
11740 if (!fp_access_check(s
)) {
11743 handle_2misc_widening(s
, opcode
, is_q
, size
, rn
, rd
);
11745 case 0x18: /* FRINTN */
11746 case 0x19: /* FRINTM */
11747 case 0x38: /* FRINTP */
11748 case 0x39: /* FRINTZ */
11749 rmode
= extract32(opcode
, 5, 1) | (extract32(opcode
, 0, 1) << 1);
11751 case 0x59: /* FRINTX */
11752 case 0x79: /* FRINTI */
11753 need_fpstatus
= true;
11754 if (size
== 3 && !is_q
) {
11755 unallocated_encoding(s
);
11759 case 0x58: /* FRINTA */
11760 rmode
= FPROUNDING_TIEAWAY
;
11761 need_fpstatus
= true;
11762 if (size
== 3 && !is_q
) {
11763 unallocated_encoding(s
);
11767 case 0x7c: /* URSQRTE */
11769 unallocated_encoding(s
);
11773 case 0x1e: /* FRINT32Z */
11774 case 0x1f: /* FRINT64Z */
11775 rmode
= FPROUNDING_ZERO
;
11777 case 0x5e: /* FRINT32X */
11778 case 0x5f: /* FRINT64X */
11779 need_fpstatus
= true;
11780 if ((size
== 3 && !is_q
) || !dc_isar_feature(aa64_frint
, s
)) {
11781 unallocated_encoding(s
);
11786 unallocated_encoding(s
);
11792 case 0x3: /* SUQADD, USQADD */
11793 unallocated_encoding(s
);
11797 if (!fp_access_check(s
)) {
11801 if (need_fpstatus
|| rmode
>= 0) {
11802 tcg_fpstatus
= fpstatus_ptr(FPST_FPCR
);
11804 tcg_fpstatus
= NULL
;
11807 tcg_rmode
= gen_set_rmode(rmode
, tcg_fpstatus
);
11814 if (u
&& size
== 0) { /* NOT */
11815 gen_gvec_fn2(s
, is_q
, rd
, rn
, tcg_gen_gvec_not
, 0);
11819 case 0x8: /* CMGT, CMGE */
11821 gen_gvec_fn2(s
, is_q
, rd
, rn
, gen_gvec_cge0
, size
);
11823 gen_gvec_fn2(s
, is_q
, rd
, rn
, gen_gvec_cgt0
, size
);
11826 case 0x9: /* CMEQ, CMLE */
11828 gen_gvec_fn2(s
, is_q
, rd
, rn
, gen_gvec_cle0
, size
);
11830 gen_gvec_fn2(s
, is_q
, rd
, rn
, gen_gvec_ceq0
, size
);
11833 case 0xa: /* CMLT */
11834 gen_gvec_fn2(s
, is_q
, rd
, rn
, gen_gvec_clt0
, size
);
11837 if (u
) { /* ABS, NEG */
11838 gen_gvec_fn2(s
, is_q
, rd
, rn
, tcg_gen_gvec_neg
, size
);
11840 gen_gvec_fn2(s
, is_q
, rd
, rn
, tcg_gen_gvec_abs
, size
);
11846 /* All 64-bit element operations can be shared with scalar 2misc */
11849 /* Coverity claims (size == 3 && !is_q) has been eliminated
11850 * from all paths leading to here.
11852 tcg_debug_assert(is_q
);
11853 for (pass
= 0; pass
< 2; pass
++) {
11854 TCGv_i64 tcg_op
= tcg_temp_new_i64();
11855 TCGv_i64 tcg_res
= tcg_temp_new_i64();
11857 read_vec_element(s
, tcg_op
, rn
, pass
, MO_64
);
11859 handle_2misc_64(s
, opcode
, u
, tcg_res
, tcg_op
,
11860 tcg_rmode
, tcg_fpstatus
);
11862 write_vec_element(s
, tcg_res
, rd
, pass
, MO_64
);
11867 for (pass
= 0; pass
< (is_q
? 4 : 2); pass
++) {
11868 TCGv_i32 tcg_op
= tcg_temp_new_i32();
11869 TCGv_i32 tcg_res
= tcg_temp_new_i32();
11871 read_vec_element_i32(s
, tcg_op
, rn
, pass
, MO_32
);
11874 /* Special cases for 32 bit elements */
11876 case 0x4: /* CLS */
11878 tcg_gen_clzi_i32(tcg_res
, tcg_op
, 32);
11880 tcg_gen_clrsb_i32(tcg_res
, tcg_op
);
11883 case 0x7: /* SQABS, SQNEG */
11885 gen_helper_neon_qneg_s32(tcg_res
, tcg_env
, tcg_op
);
11887 gen_helper_neon_qabs_s32(tcg_res
, tcg_env
, tcg_op
);
11890 case 0x2f: /* FABS */
11891 gen_vfp_abss(tcg_res
, tcg_op
);
11893 case 0x6f: /* FNEG */
11894 gen_vfp_negs(tcg_res
, tcg_op
);
11896 case 0x7f: /* FSQRT */
11897 gen_helper_vfp_sqrts(tcg_res
, tcg_op
, tcg_env
);
11899 case 0x1a: /* FCVTNS */
11900 case 0x1b: /* FCVTMS */
11901 case 0x1c: /* FCVTAS */
11902 case 0x3a: /* FCVTPS */
11903 case 0x3b: /* FCVTZS */
11904 gen_helper_vfp_tosls(tcg_res
, tcg_op
,
11905 tcg_constant_i32(0), tcg_fpstatus
);
11907 case 0x5a: /* FCVTNU */
11908 case 0x5b: /* FCVTMU */
11909 case 0x5c: /* FCVTAU */
11910 case 0x7a: /* FCVTPU */
11911 case 0x7b: /* FCVTZU */
11912 gen_helper_vfp_touls(tcg_res
, tcg_op
,
11913 tcg_constant_i32(0), tcg_fpstatus
);
11915 case 0x18: /* FRINTN */
11916 case 0x19: /* FRINTM */
11917 case 0x38: /* FRINTP */
11918 case 0x39: /* FRINTZ */
11919 case 0x58: /* FRINTA */
11920 case 0x79: /* FRINTI */
11921 gen_helper_rints(tcg_res
, tcg_op
, tcg_fpstatus
);
11923 case 0x59: /* FRINTX */
11924 gen_helper_rints_exact(tcg_res
, tcg_op
, tcg_fpstatus
);
11926 case 0x7c: /* URSQRTE */
11927 gen_helper_rsqrte_u32(tcg_res
, tcg_op
);
11929 case 0x1e: /* FRINT32Z */
11930 case 0x5e: /* FRINT32X */
11931 gen_helper_frint32_s(tcg_res
, tcg_op
, tcg_fpstatus
);
11933 case 0x1f: /* FRINT64Z */
11934 case 0x5f: /* FRINT64X */
11935 gen_helper_frint64_s(tcg_res
, tcg_op
, tcg_fpstatus
);
11938 g_assert_not_reached();
11941 /* Use helpers for 8 and 16 bit elements */
11943 case 0x5: /* CNT, RBIT */
11944 /* For these two insns size is part of the opcode specifier
11945 * (handled earlier); they always operate on byte elements.
11948 gen_helper_neon_rbit_u8(tcg_res
, tcg_op
);
11950 gen_helper_neon_cnt_u8(tcg_res
, tcg_op
);
11953 case 0x7: /* SQABS, SQNEG */
11955 NeonGenOneOpEnvFn
*genfn
;
11956 static NeonGenOneOpEnvFn
* const fns
[2][2] = {
11957 { gen_helper_neon_qabs_s8
, gen_helper_neon_qneg_s8
},
11958 { gen_helper_neon_qabs_s16
, gen_helper_neon_qneg_s16
},
11960 genfn
= fns
[size
][u
];
11961 genfn(tcg_res
, tcg_env
, tcg_op
);
11964 case 0x4: /* CLS, CLZ */
11967 gen_helper_neon_clz_u8(tcg_res
, tcg_op
);
11969 gen_helper_neon_clz_u16(tcg_res
, tcg_op
);
11973 gen_helper_neon_cls_s8(tcg_res
, tcg_op
);
11975 gen_helper_neon_cls_s16(tcg_res
, tcg_op
);
11980 g_assert_not_reached();
11984 write_vec_element_i32(s
, tcg_res
, rd
, pass
, MO_32
);
11987 clear_vec_high(s
, is_q
, rd
);
11990 gen_restore_rmode(tcg_rmode
, tcg_fpstatus
);
11994 /* AdvSIMD [scalar] two register miscellaneous (FP16)
11996 * 31 30 29 28 27 24 23 22 21 17 16 12 11 10 9 5 4 0
11997 * +---+---+---+---+---------+---+-------------+--------+-----+------+------+
11998 * | 0 | Q | U | S | 1 1 1 0 | a | 1 1 1 1 0 0 | opcode | 1 0 | Rn | Rd |
11999 * +---+---+---+---+---------+---+-------------+--------+-----+------+------+
12000 * mask: 1000 1111 0111 1110 0000 1100 0000 0000 0x8f7e 0c00
12001 * val: 0000 1110 0111 1000 0000 1000 0000 0000 0x0e78 0800
12003 * This actually covers two groups where scalar access is governed by
12004 * bit 28. A bunch of the instructions (float to integral) only exist
12005 * in the vector form and are un-allocated for the scalar decode. Also
12006 * in the scalar decode Q is always 1.
12008 static void disas_simd_two_reg_misc_fp16(DisasContext
*s
, uint32_t insn
)
12010 int fpop
, opcode
, a
, u
;
12014 bool only_in_vector
= false;
12017 TCGv_i32 tcg_rmode
= NULL
;
12018 TCGv_ptr tcg_fpstatus
= NULL
;
12019 bool need_fpst
= true;
12022 if (!dc_isar_feature(aa64_fp16
, s
)) {
12023 unallocated_encoding(s
);
12027 rd
= extract32(insn
, 0, 5);
12028 rn
= extract32(insn
, 5, 5);
12030 a
= extract32(insn
, 23, 1);
12031 u
= extract32(insn
, 29, 1);
12032 is_scalar
= extract32(insn
, 28, 1);
12033 is_q
= extract32(insn
, 30, 1);
12035 opcode
= extract32(insn
, 12, 5);
12036 fpop
= deposit32(opcode
, 5, 1, a
);
12037 fpop
= deposit32(fpop
, 6, 1, u
);
12040 case 0x1d: /* SCVTF */
12041 case 0x5d: /* UCVTF */
12048 elements
= (is_q
? 8 : 4);
12051 if (!fp_access_check(s
)) {
12054 handle_simd_intfp_conv(s
, rd
, rn
, elements
, !u
, 0, MO_16
);
12058 case 0x2c: /* FCMGT (zero) */
12059 case 0x2d: /* FCMEQ (zero) */
12060 case 0x2e: /* FCMLT (zero) */
12061 case 0x6c: /* FCMGE (zero) */
12062 case 0x6d: /* FCMLE (zero) */
12063 handle_2misc_fcmp_zero(s
, fpop
, is_scalar
, 0, is_q
, MO_16
, rn
, rd
);
12065 case 0x3d: /* FRECPE */
12066 case 0x3f: /* FRECPX */
12068 case 0x18: /* FRINTN */
12069 only_in_vector
= true;
12070 rmode
= FPROUNDING_TIEEVEN
;
12072 case 0x19: /* FRINTM */
12073 only_in_vector
= true;
12074 rmode
= FPROUNDING_NEGINF
;
12076 case 0x38: /* FRINTP */
12077 only_in_vector
= true;
12078 rmode
= FPROUNDING_POSINF
;
12080 case 0x39: /* FRINTZ */
12081 only_in_vector
= true;
12082 rmode
= FPROUNDING_ZERO
;
12084 case 0x58: /* FRINTA */
12085 only_in_vector
= true;
12086 rmode
= FPROUNDING_TIEAWAY
;
12088 case 0x59: /* FRINTX */
12089 case 0x79: /* FRINTI */
12090 only_in_vector
= true;
12091 /* current rounding mode */
12093 case 0x1a: /* FCVTNS */
12094 rmode
= FPROUNDING_TIEEVEN
;
12096 case 0x1b: /* FCVTMS */
12097 rmode
= FPROUNDING_NEGINF
;
12099 case 0x1c: /* FCVTAS */
12100 rmode
= FPROUNDING_TIEAWAY
;
12102 case 0x3a: /* FCVTPS */
12103 rmode
= FPROUNDING_POSINF
;
12105 case 0x3b: /* FCVTZS */
12106 rmode
= FPROUNDING_ZERO
;
12108 case 0x5a: /* FCVTNU */
12109 rmode
= FPROUNDING_TIEEVEN
;
12111 case 0x5b: /* FCVTMU */
12112 rmode
= FPROUNDING_NEGINF
;
12114 case 0x5c: /* FCVTAU */
12115 rmode
= FPROUNDING_TIEAWAY
;
12117 case 0x7a: /* FCVTPU */
12118 rmode
= FPROUNDING_POSINF
;
12120 case 0x7b: /* FCVTZU */
12121 rmode
= FPROUNDING_ZERO
;
12123 case 0x2f: /* FABS */
12124 case 0x6f: /* FNEG */
12127 case 0x7d: /* FRSQRTE */
12128 case 0x7f: /* FSQRT (vector) */
12131 unallocated_encoding(s
);
12136 /* Check additional constraints for the scalar encoding */
12139 unallocated_encoding(s
);
12142 /* FRINTxx is only in the vector form */
12143 if (only_in_vector
) {
12144 unallocated_encoding(s
);
12149 if (!fp_access_check(s
)) {
12153 if (rmode
>= 0 || need_fpst
) {
12154 tcg_fpstatus
= fpstatus_ptr(FPST_FPCR_F16
);
12158 tcg_rmode
= gen_set_rmode(rmode
, tcg_fpstatus
);
12162 TCGv_i32 tcg_op
= read_fp_hreg(s
, rn
);
12163 TCGv_i32 tcg_res
= tcg_temp_new_i32();
12166 case 0x1a: /* FCVTNS */
12167 case 0x1b: /* FCVTMS */
12168 case 0x1c: /* FCVTAS */
12169 case 0x3a: /* FCVTPS */
12170 case 0x3b: /* FCVTZS */
12171 gen_helper_advsimd_f16tosinth(tcg_res
, tcg_op
, tcg_fpstatus
);
12173 case 0x3d: /* FRECPE */
12174 gen_helper_recpe_f16(tcg_res
, tcg_op
, tcg_fpstatus
);
12176 case 0x3f: /* FRECPX */
12177 gen_helper_frecpx_f16(tcg_res
, tcg_op
, tcg_fpstatus
);
12179 case 0x5a: /* FCVTNU */
12180 case 0x5b: /* FCVTMU */
12181 case 0x5c: /* FCVTAU */
12182 case 0x7a: /* FCVTPU */
12183 case 0x7b: /* FCVTZU */
12184 gen_helper_advsimd_f16touinth(tcg_res
, tcg_op
, tcg_fpstatus
);
12186 case 0x6f: /* FNEG */
12187 tcg_gen_xori_i32(tcg_res
, tcg_op
, 0x8000);
12189 case 0x7d: /* FRSQRTE */
12190 gen_helper_rsqrte_f16(tcg_res
, tcg_op
, tcg_fpstatus
);
12193 g_assert_not_reached();
12196 /* limit any sign extension going on */
12197 tcg_gen_andi_i32(tcg_res
, tcg_res
, 0xffff);
12198 write_fp_sreg(s
, rd
, tcg_res
);
12200 for (pass
= 0; pass
< (is_q
? 8 : 4); pass
++) {
12201 TCGv_i32 tcg_op
= tcg_temp_new_i32();
12202 TCGv_i32 tcg_res
= tcg_temp_new_i32();
12204 read_vec_element_i32(s
, tcg_op
, rn
, pass
, MO_16
);
12207 case 0x1a: /* FCVTNS */
12208 case 0x1b: /* FCVTMS */
12209 case 0x1c: /* FCVTAS */
12210 case 0x3a: /* FCVTPS */
12211 case 0x3b: /* FCVTZS */
12212 gen_helper_advsimd_f16tosinth(tcg_res
, tcg_op
, tcg_fpstatus
);
12214 case 0x3d: /* FRECPE */
12215 gen_helper_recpe_f16(tcg_res
, tcg_op
, tcg_fpstatus
);
12217 case 0x5a: /* FCVTNU */
12218 case 0x5b: /* FCVTMU */
12219 case 0x5c: /* FCVTAU */
12220 case 0x7a: /* FCVTPU */
12221 case 0x7b: /* FCVTZU */
12222 gen_helper_advsimd_f16touinth(tcg_res
, tcg_op
, tcg_fpstatus
);
12224 case 0x18: /* FRINTN */
12225 case 0x19: /* FRINTM */
12226 case 0x38: /* FRINTP */
12227 case 0x39: /* FRINTZ */
12228 case 0x58: /* FRINTA */
12229 case 0x79: /* FRINTI */
12230 gen_helper_advsimd_rinth(tcg_res
, tcg_op
, tcg_fpstatus
);
12232 case 0x59: /* FRINTX */
12233 gen_helper_advsimd_rinth_exact(tcg_res
, tcg_op
, tcg_fpstatus
);
12235 case 0x2f: /* FABS */
12236 tcg_gen_andi_i32(tcg_res
, tcg_op
, 0x7fff);
12238 case 0x6f: /* FNEG */
12239 tcg_gen_xori_i32(tcg_res
, tcg_op
, 0x8000);
12241 case 0x7d: /* FRSQRTE */
12242 gen_helper_rsqrte_f16(tcg_res
, tcg_op
, tcg_fpstatus
);
12244 case 0x7f: /* FSQRT */
12245 gen_helper_sqrt_f16(tcg_res
, tcg_op
, tcg_fpstatus
);
12248 g_assert_not_reached();
12251 write_vec_element_i32(s
, tcg_res
, rd
, pass
, MO_16
);
12254 clear_vec_high(s
, is_q
, rd
);
12258 gen_restore_rmode(tcg_rmode
, tcg_fpstatus
);
12262 /* AdvSIMD scalar x indexed element
12263 * 31 30 29 28 24 23 22 21 20 19 16 15 12 11 10 9 5 4 0
12264 * +-----+---+-----------+------+---+---+------+-----+---+---+------+------+
12265 * | 0 1 | U | 1 1 1 1 1 | size | L | M | Rm | opc | H | 0 | Rn | Rd |
12266 * +-----+---+-----------+------+---+---+------+-----+---+---+------+------+
12267 * AdvSIMD vector x indexed element
12268 * 31 30 29 28 24 23 22 21 20 19 16 15 12 11 10 9 5 4 0
12269 * +---+---+---+-----------+------+---+---+------+-----+---+---+------+------+
12270 * | 0 | Q | U | 0 1 1 1 1 | size | L | M | Rm | opc | H | 0 | Rn | Rd |
12271 * +---+---+---+-----------+------+---+---+------+-----+---+---+------+------+
12273 static void disas_simd_indexed(DisasContext
*s
, uint32_t insn
)
12275 /* This encoding has two kinds of instruction:
12276 * normal, where we perform elt x idxelt => elt for each
12277 * element in the vector
12278 * long, where we perform elt x idxelt and generate a result of
12279 * double the width of the input element
12280 * The long ops have a 'part' specifier (ie come in INSN, INSN2 pairs).
12282 bool is_scalar
= extract32(insn
, 28, 1);
12283 bool is_q
= extract32(insn
, 30, 1);
12284 bool u
= extract32(insn
, 29, 1);
12285 int size
= extract32(insn
, 22, 2);
12286 int l
= extract32(insn
, 21, 1);
12287 int m
= extract32(insn
, 20, 1);
12288 /* Note that the Rm field here is only 4 bits, not 5 as it usually is */
12289 int rm
= extract32(insn
, 16, 4);
12290 int opcode
= extract32(insn
, 12, 4);
12291 int h
= extract32(insn
, 11, 1);
12292 int rn
= extract32(insn
, 5, 5);
12293 int rd
= extract32(insn
, 0, 5);
12294 bool is_long
= false;
12296 bool is_fp16
= false;
12300 switch (16 * u
+ opcode
) {
12301 case 0x08: /* MUL */
12302 case 0x10: /* MLA */
12303 case 0x14: /* MLS */
12305 unallocated_encoding(s
);
12309 case 0x02: /* SMLAL, SMLAL2 */
12310 case 0x12: /* UMLAL, UMLAL2 */
12311 case 0x06: /* SMLSL, SMLSL2 */
12312 case 0x16: /* UMLSL, UMLSL2 */
12313 case 0x0a: /* SMULL, SMULL2 */
12314 case 0x1a: /* UMULL, UMULL2 */
12316 unallocated_encoding(s
);
12321 case 0x03: /* SQDMLAL, SQDMLAL2 */
12322 case 0x07: /* SQDMLSL, SQDMLSL2 */
12323 case 0x0b: /* SQDMULL, SQDMULL2 */
12326 case 0x0c: /* SQDMULH */
12327 case 0x0d: /* SQRDMULH */
12329 case 0x1d: /* SQRDMLAH */
12330 case 0x1f: /* SQRDMLSH */
12331 if (!dc_isar_feature(aa64_rdm
, s
)) {
12332 unallocated_encoding(s
);
12336 case 0x0e: /* SDOT */
12337 case 0x1e: /* UDOT */
12338 if (is_scalar
|| size
!= MO_32
|| !dc_isar_feature(aa64_dp
, s
)) {
12339 unallocated_encoding(s
);
12345 case 0: /* SUDOT */
12346 case 2: /* USDOT */
12347 if (is_scalar
|| !dc_isar_feature(aa64_i8mm
, s
)) {
12348 unallocated_encoding(s
);
12353 case 1: /* BFDOT */
12354 if (is_scalar
|| !dc_isar_feature(aa64_bf16
, s
)) {
12355 unallocated_encoding(s
);
12360 case 3: /* BFMLAL{B,T} */
12361 if (is_scalar
|| !dc_isar_feature(aa64_bf16
, s
)) {
12362 unallocated_encoding(s
);
12365 /* can't set is_fp without other incorrect size checks */
12369 unallocated_encoding(s
);
12373 case 0x11: /* FCMLA #0 */
12374 case 0x13: /* FCMLA #90 */
12375 case 0x15: /* FCMLA #180 */
12376 case 0x17: /* FCMLA #270 */
12377 if (is_scalar
|| !dc_isar_feature(aa64_fcma
, s
)) {
12378 unallocated_encoding(s
);
12384 case 0x00: /* FMLAL */
12385 case 0x01: /* FMLA */
12386 case 0x04: /* FMLSL */
12387 case 0x05: /* FMLS */
12388 case 0x09: /* FMUL */
12389 case 0x18: /* FMLAL2 */
12390 case 0x19: /* FMULX */
12391 case 0x1c: /* FMLSL2 */
12392 unallocated_encoding(s
);
12397 case 1: /* normal fp */
12398 unallocated_encoding(s
); /* in decodetree */
12401 case 2: /* complex fp */
12402 /* Each indexable element is a complex pair. */
12407 unallocated_encoding(s
);
12415 unallocated_encoding(s
);
12420 default: /* integer */
12424 unallocated_encoding(s
);
12429 if (is_fp16
&& !dc_isar_feature(aa64_fp16
, s
)) {
12430 unallocated_encoding(s
);
12434 /* Given MemOp size, adjust register and indexing. */
12437 index
= h
<< 2 | l
<< 1 | m
;
12440 index
= h
<< 1 | l
;
12445 unallocated_encoding(s
);
12452 g_assert_not_reached();
12455 if (!fp_access_check(s
)) {
12460 fpst
= fpstatus_ptr(is_fp16
? FPST_FPCR_F16
: FPST_FPCR
);
12465 switch (16 * u
+ opcode
) {
12466 case 0x0e: /* SDOT */
12467 case 0x1e: /* UDOT */
12468 gen_gvec_op4_ool(s
, is_q
, rd
, rn
, rm
, rd
, index
,
12469 u
? gen_helper_gvec_udot_idx_b
12470 : gen_helper_gvec_sdot_idx_b
);
12473 switch (extract32(insn
, 22, 2)) {
12474 case 0: /* SUDOT */
12475 gen_gvec_op4_ool(s
, is_q
, rd
, rn
, rm
, rd
, index
,
12476 gen_helper_gvec_sudot_idx_b
);
12478 case 1: /* BFDOT */
12479 gen_gvec_op4_ool(s
, is_q
, rd
, rn
, rm
, rd
, index
,
12480 gen_helper_gvec_bfdot_idx
);
12482 case 2: /* USDOT */
12483 gen_gvec_op4_ool(s
, is_q
, rd
, rn
, rm
, rd
, index
,
12484 gen_helper_gvec_usdot_idx_b
);
12486 case 3: /* BFMLAL{B,T} */
12487 gen_gvec_op4_fpst(s
, 1, rd
, rn
, rm
, rd
, 0, (index
<< 1) | is_q
,
12488 gen_helper_gvec_bfmlal_idx
);
12491 g_assert_not_reached();
12492 case 0x11: /* FCMLA #0 */
12493 case 0x13: /* FCMLA #90 */
12494 case 0x15: /* FCMLA #180 */
12495 case 0x17: /* FCMLA #270 */
12497 int rot
= extract32(insn
, 13, 2);
12498 int data
= (index
<< 2) | rot
;
12499 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s
, rd
),
12500 vec_full_reg_offset(s
, rn
),
12501 vec_full_reg_offset(s
, rm
),
12502 vec_full_reg_offset(s
, rd
), fpst
,
12503 is_q
? 16 : 8, vec_full_reg_size(s
), data
,
12505 ? gen_helper_gvec_fcmlas_idx
12506 : gen_helper_gvec_fcmlah_idx
);
12510 case 0x08: /* MUL */
12511 if (!is_long
&& !is_scalar
) {
12512 static gen_helper_gvec_3
* const fns
[3] = {
12513 gen_helper_gvec_mul_idx_h
,
12514 gen_helper_gvec_mul_idx_s
,
12515 gen_helper_gvec_mul_idx_d
,
12517 tcg_gen_gvec_3_ool(vec_full_reg_offset(s
, rd
),
12518 vec_full_reg_offset(s
, rn
),
12519 vec_full_reg_offset(s
, rm
),
12520 is_q
? 16 : 8, vec_full_reg_size(s
),
12521 index
, fns
[size
- 1]);
12526 case 0x10: /* MLA */
12527 if (!is_long
&& !is_scalar
) {
12528 static gen_helper_gvec_4
* const fns
[3] = {
12529 gen_helper_gvec_mla_idx_h
,
12530 gen_helper_gvec_mla_idx_s
,
12531 gen_helper_gvec_mla_idx_d
,
12533 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
12534 vec_full_reg_offset(s
, rn
),
12535 vec_full_reg_offset(s
, rm
),
12536 vec_full_reg_offset(s
, rd
),
12537 is_q
? 16 : 8, vec_full_reg_size(s
),
12538 index
, fns
[size
- 1]);
12543 case 0x14: /* MLS */
12544 if (!is_long
&& !is_scalar
) {
12545 static gen_helper_gvec_4
* const fns
[3] = {
12546 gen_helper_gvec_mls_idx_h
,
12547 gen_helper_gvec_mls_idx_s
,
12548 gen_helper_gvec_mls_idx_d
,
12550 tcg_gen_gvec_4_ool(vec_full_reg_offset(s
, rd
),
12551 vec_full_reg_offset(s
, rn
),
12552 vec_full_reg_offset(s
, rm
),
12553 vec_full_reg_offset(s
, rd
),
12554 is_q
? 16 : 8, vec_full_reg_size(s
),
12555 index
, fns
[size
- 1]);
12562 g_assert_not_reached();
12563 } else if (!is_long
) {
12564 /* 32 bit floating point, or 16 or 32 bit integer.
12565 * For the 16 bit scalar case we use the usual Neon helpers and
12566 * rely on the fact that 0 op 0 == 0 with no side effects.
12568 TCGv_i32 tcg_idx
= tcg_temp_new_i32();
12569 int pass
, maxpasses
;
12574 maxpasses
= is_q
? 4 : 2;
12577 read_vec_element_i32(s
, tcg_idx
, rm
, index
, size
);
12579 if (size
== 1 && !is_scalar
) {
12580 /* The simplest way to handle the 16x16 indexed ops is to duplicate
12581 * the index into both halves of the 32 bit tcg_idx and then use
12582 * the usual Neon helpers.
12584 tcg_gen_deposit_i32(tcg_idx
, tcg_idx
, tcg_idx
, 16, 16);
12587 for (pass
= 0; pass
< maxpasses
; pass
++) {
12588 TCGv_i32 tcg_op
= tcg_temp_new_i32();
12589 TCGv_i32 tcg_res
= tcg_temp_new_i32();
12591 read_vec_element_i32(s
, tcg_op
, rn
, pass
, is_scalar
? size
: MO_32
);
12593 switch (16 * u
+ opcode
) {
12594 case 0x08: /* MUL */
12595 case 0x10: /* MLA */
12596 case 0x14: /* MLS */
12598 static NeonGenTwoOpFn
* const fns
[2][2] = {
12599 { gen_helper_neon_add_u16
, gen_helper_neon_sub_u16
},
12600 { tcg_gen_add_i32
, tcg_gen_sub_i32
},
12602 NeonGenTwoOpFn
*genfn
;
12603 bool is_sub
= opcode
== 0x4;
12606 gen_helper_neon_mul_u16(tcg_res
, tcg_op
, tcg_idx
);
12608 tcg_gen_mul_i32(tcg_res
, tcg_op
, tcg_idx
);
12610 if (opcode
== 0x8) {
12613 read_vec_element_i32(s
, tcg_op
, rd
, pass
, MO_32
);
12614 genfn
= fns
[size
- 1][is_sub
];
12615 genfn(tcg_res
, tcg_op
, tcg_res
);
12618 case 0x0c: /* SQDMULH */
12620 gen_helper_neon_qdmulh_s16(tcg_res
, tcg_env
,
12623 gen_helper_neon_qdmulh_s32(tcg_res
, tcg_env
,
12627 case 0x0d: /* SQRDMULH */
12629 gen_helper_neon_qrdmulh_s16(tcg_res
, tcg_env
,
12632 gen_helper_neon_qrdmulh_s32(tcg_res
, tcg_env
,
12636 case 0x1d: /* SQRDMLAH */
12637 read_vec_element_i32(s
, tcg_res
, rd
, pass
,
12638 is_scalar
? size
: MO_32
);
12640 gen_helper_neon_qrdmlah_s16(tcg_res
, tcg_env
,
12641 tcg_op
, tcg_idx
, tcg_res
);
12643 gen_helper_neon_qrdmlah_s32(tcg_res
, tcg_env
,
12644 tcg_op
, tcg_idx
, tcg_res
);
12647 case 0x1f: /* SQRDMLSH */
12648 read_vec_element_i32(s
, tcg_res
, rd
, pass
,
12649 is_scalar
? size
: MO_32
);
12651 gen_helper_neon_qrdmlsh_s16(tcg_res
, tcg_env
,
12652 tcg_op
, tcg_idx
, tcg_res
);
12654 gen_helper_neon_qrdmlsh_s32(tcg_res
, tcg_env
,
12655 tcg_op
, tcg_idx
, tcg_res
);
12659 case 0x01: /* FMLA */
12660 case 0x05: /* FMLS */
12661 case 0x09: /* FMUL */
12662 case 0x19: /* FMULX */
12663 g_assert_not_reached();
12667 write_fp_sreg(s
, rd
, tcg_res
);
12669 write_vec_element_i32(s
, tcg_res
, rd
, pass
, MO_32
);
12673 clear_vec_high(s
, is_q
, rd
);
12675 /* long ops: 16x16->32 or 32x32->64 */
12676 TCGv_i64 tcg_res
[2];
12678 bool satop
= extract32(opcode
, 0, 1);
12679 MemOp memop
= MO_32
;
12686 TCGv_i64 tcg_idx
= tcg_temp_new_i64();
12688 read_vec_element(s
, tcg_idx
, rm
, index
, memop
);
12690 for (pass
= 0; pass
< (is_scalar
? 1 : 2); pass
++) {
12691 TCGv_i64 tcg_op
= tcg_temp_new_i64();
12692 TCGv_i64 tcg_passres
;
12698 passelt
= pass
+ (is_q
* 2);
12701 read_vec_element(s
, tcg_op
, rn
, passelt
, memop
);
12703 tcg_res
[pass
] = tcg_temp_new_i64();
12705 if (opcode
== 0xa || opcode
== 0xb) {
12706 /* Non-accumulating ops */
12707 tcg_passres
= tcg_res
[pass
];
12709 tcg_passres
= tcg_temp_new_i64();
12712 tcg_gen_mul_i64(tcg_passres
, tcg_op
, tcg_idx
);
12715 /* saturating, doubling */
12716 gen_helper_neon_addl_saturate_s64(tcg_passres
, tcg_env
,
12717 tcg_passres
, tcg_passres
);
12720 if (opcode
== 0xa || opcode
== 0xb) {
12724 /* Accumulating op: handle accumulate step */
12725 read_vec_element(s
, tcg_res
[pass
], rd
, pass
, MO_64
);
12728 case 0x2: /* SMLAL, SMLAL2, UMLAL, UMLAL2 */
12729 tcg_gen_add_i64(tcg_res
[pass
], tcg_res
[pass
], tcg_passres
);
12731 case 0x6: /* SMLSL, SMLSL2, UMLSL, UMLSL2 */
12732 tcg_gen_sub_i64(tcg_res
[pass
], tcg_res
[pass
], tcg_passres
);
12734 case 0x7: /* SQDMLSL, SQDMLSL2 */
12735 tcg_gen_neg_i64(tcg_passres
, tcg_passres
);
12737 case 0x3: /* SQDMLAL, SQDMLAL2 */
12738 gen_helper_neon_addl_saturate_s64(tcg_res
[pass
], tcg_env
,
12743 g_assert_not_reached();
12747 clear_vec_high(s
, !is_scalar
, rd
);
12749 TCGv_i32 tcg_idx
= tcg_temp_new_i32();
12752 read_vec_element_i32(s
, tcg_idx
, rm
, index
, size
);
12755 /* The simplest way to handle the 16x16 indexed ops is to
12756 * duplicate the index into both halves of the 32 bit tcg_idx
12757 * and then use the usual Neon helpers.
12759 tcg_gen_deposit_i32(tcg_idx
, tcg_idx
, tcg_idx
, 16, 16);
12762 for (pass
= 0; pass
< (is_scalar
? 1 : 2); pass
++) {
12763 TCGv_i32 tcg_op
= tcg_temp_new_i32();
12764 TCGv_i64 tcg_passres
;
12767 read_vec_element_i32(s
, tcg_op
, rn
, pass
, size
);
12769 read_vec_element_i32(s
, tcg_op
, rn
,
12770 pass
+ (is_q
* 2), MO_32
);
12773 tcg_res
[pass
] = tcg_temp_new_i64();
12775 if (opcode
== 0xa || opcode
== 0xb) {
12776 /* Non-accumulating ops */
12777 tcg_passres
= tcg_res
[pass
];
12779 tcg_passres
= tcg_temp_new_i64();
12782 if (memop
& MO_SIGN
) {
12783 gen_helper_neon_mull_s16(tcg_passres
, tcg_op
, tcg_idx
);
12785 gen_helper_neon_mull_u16(tcg_passres
, tcg_op
, tcg_idx
);
12788 gen_helper_neon_addl_saturate_s32(tcg_passres
, tcg_env
,
12789 tcg_passres
, tcg_passres
);
12792 if (opcode
== 0xa || opcode
== 0xb) {
12796 /* Accumulating op: handle accumulate step */
12797 read_vec_element(s
, tcg_res
[pass
], rd
, pass
, MO_64
);
12800 case 0x2: /* SMLAL, SMLAL2, UMLAL, UMLAL2 */
12801 gen_helper_neon_addl_u32(tcg_res
[pass
], tcg_res
[pass
],
12804 case 0x6: /* SMLSL, SMLSL2, UMLSL, UMLSL2 */
12805 gen_helper_neon_subl_u32(tcg_res
[pass
], tcg_res
[pass
],
12808 case 0x7: /* SQDMLSL, SQDMLSL2 */
12809 gen_helper_neon_negl_u32(tcg_passres
, tcg_passres
);
12811 case 0x3: /* SQDMLAL, SQDMLAL2 */
12812 gen_helper_neon_addl_saturate_s32(tcg_res
[pass
], tcg_env
,
12817 g_assert_not_reached();
12822 tcg_gen_ext32u_i64(tcg_res
[0], tcg_res
[0]);
12827 tcg_res
[1] = tcg_constant_i64(0);
12830 for (pass
= 0; pass
< 2; pass
++) {
12831 write_vec_element(s
, tcg_res
[pass
], rd
, pass
, MO_64
);
12836 /* C3.6 Data processing - SIMD, inc Crypto
12838 * As the decode gets a little complex we are using a table based
12839 * approach for this part of the decode.
12841 static const AArch64DecodeTable data_proc_simd
[] = {
12842 /* pattern , mask , fn */
12843 { 0x0e200400, 0x9f200400, disas_simd_three_reg_same
},
12844 { 0x0e008400, 0x9f208400, disas_simd_three_reg_same_extra
},
12845 { 0x0e200000, 0x9f200c00, disas_simd_three_reg_diff
},
12846 { 0x0e200800, 0x9f3e0c00, disas_simd_two_reg_misc
},
12847 { 0x0e300800, 0x9f3e0c00, disas_simd_across_lanes
},
12848 { 0x0f000000, 0x9f000400, disas_simd_indexed
}, /* vector indexed */
12849 /* simd_mod_imm decode is a subset of simd_shift_imm, so must precede it */
12850 { 0x0f000400, 0x9ff80400, disas_simd_mod_imm
},
12851 { 0x0f000400, 0x9f800400, disas_simd_shift_imm
},
12852 { 0x0e000000, 0xbf208c00, disas_simd_tb
},
12853 { 0x0e000800, 0xbf208c00, disas_simd_zip_trn
},
12854 { 0x2e000000, 0xbf208400, disas_simd_ext
},
12855 { 0x5e200400, 0xdf200400, disas_simd_scalar_three_reg_same
},
12856 { 0x5e008400, 0xdf208400, disas_simd_scalar_three_reg_same_extra
},
12857 { 0x5e200000, 0xdf200c00, disas_simd_scalar_three_reg_diff
},
12858 { 0x5e200800, 0xdf3e0c00, disas_simd_scalar_two_reg_misc
},
12859 { 0x5f000000, 0xdf000400, disas_simd_indexed
}, /* scalar indexed */
12860 { 0x5f000400, 0xdf800400, disas_simd_scalar_shift_imm
},
12861 { 0x0e780800, 0x8f7e0c00, disas_simd_two_reg_misc_fp16
},
12862 { 0x00000000, 0x00000000, NULL
}
12865 static void disas_data_proc_simd(DisasContext
*s
, uint32_t insn
)
12867 /* Note that this is called with all non-FP cases from
12868 * table C3-6 so it must UNDEF for entries not specifically
12869 * allocated to instructions in that table.
12871 AArch64DecodeFn
*fn
= lookup_disas_fn(&data_proc_simd
[0], insn
);
12875 unallocated_encoding(s
);
12879 /* C3.6 Data processing - SIMD and floating point */
12880 static void disas_data_proc_simd_fp(DisasContext
*s
, uint32_t insn
)
12882 if (extract32(insn
, 28, 1) == 1 && extract32(insn
, 30, 1) == 0) {
12883 disas_data_proc_fp(s
, insn
);
12885 /* SIMD, including crypto */
12886 disas_data_proc_simd(s
, insn
);
12890 static bool trans_OK(DisasContext
*s
, arg_OK
*a
)
12895 static bool trans_FAIL(DisasContext
*s
, arg_OK
*a
)
12897 s
->is_nonstreaming
= true;
12903 * @env: The cpu environment
12904 * @s: The DisasContext
12906 * Return true if the page is guarded.
12908 static bool is_guarded_page(CPUARMState
*env
, DisasContext
*s
)
12910 uint64_t addr
= s
->base
.pc_first
;
12911 #ifdef CONFIG_USER_ONLY
12912 return page_get_flags(addr
) & PAGE_BTI
;
12914 CPUTLBEntryFull
*full
;
12916 int mmu_idx
= arm_to_core_mmu_idx(s
->mmu_idx
);
12920 * We test this immediately after reading an insn, which means
12921 * that the TLB entry must be present and valid, and thus this
12922 * access will never raise an exception.
12924 flags
= probe_access_full(env
, addr
, 0, MMU_INST_FETCH
, mmu_idx
,
12925 false, &host
, &full
, 0);
12926 assert(!(flags
& TLB_INVALID_MASK
));
12928 return full
->extra
.arm
.guarded
;
12933 * btype_destination_ok:
12934 * @insn: The instruction at the branch destination
12935 * @bt: SCTLR_ELx.BT
12936 * @btype: PSTATE.BTYPE, and is non-zero
12938 * On a guarded page, there are a limited number of insns
12939 * that may be present at the branch target:
12940 * - branch target identifiers,
12941 * - paciasp, pacibsp,
12944 * Anything else causes a Branch Target Exception.
12946 * Return true if the branch is compatible, false to raise BTITRAP.
12948 static bool btype_destination_ok(uint32_t insn
, bool bt
, int btype
)
12950 if ((insn
& 0xfffff01fu
) == 0xd503201fu
) {
12952 switch (extract32(insn
, 5, 7)) {
12953 case 0b011001: /* PACIASP */
12954 case 0b011011: /* PACIBSP */
12956 * If SCTLR_ELx.BT, then PACI*SP are not compatible
12957 * with btype == 3. Otherwise all btype are ok.
12959 return !bt
|| btype
!= 3;
12960 case 0b100000: /* BTI */
12961 /* Not compatible with any btype. */
12963 case 0b100010: /* BTI c */
12964 /* Not compatible with btype == 3 */
12966 case 0b100100: /* BTI j */
12967 /* Not compatible with btype == 2 */
12969 case 0b100110: /* BTI jc */
12970 /* Compatible with any btype. */
12974 switch (insn
& 0xffe0001fu
) {
12975 case 0xd4200000u
: /* BRK */
12976 case 0xd4400000u
: /* HLT */
12977 /* Give priority to the breakpoint exception. */
12984 /* C3.1 A64 instruction index by encoding */
12985 static void disas_a64_legacy(DisasContext
*s
, uint32_t insn
)
12987 switch (extract32(insn
, 25, 4)) {
12989 case 0xd: /* Data processing - register */
12990 disas_data_proc_reg(s
, insn
);
12993 case 0xf: /* Data processing - SIMD and floating point */
12994 disas_data_proc_simd_fp(s
, insn
);
12997 unallocated_encoding(s
);
13002 static void aarch64_tr_init_disas_context(DisasContextBase
*dcbase
,
13005 DisasContext
*dc
= container_of(dcbase
, DisasContext
, base
);
13006 CPUARMState
*env
= cpu_env(cpu
);
13007 ARMCPU
*arm_cpu
= env_archcpu(env
);
13008 CPUARMTBFlags tb_flags
= arm_tbflags_from_tb(dc
->base
.tb
);
13009 int bound
, core_mmu_idx
;
13011 dc
->isar
= &arm_cpu
->isar
;
13013 dc
->pc_save
= dc
->base
.pc_first
;
13014 dc
->aarch64
= true;
13017 dc
->be_data
= EX_TBFLAG_ANY(tb_flags
, BE_DATA
) ? MO_BE
: MO_LE
;
13018 dc
->condexec_mask
= 0;
13019 dc
->condexec_cond
= 0;
13020 core_mmu_idx
= EX_TBFLAG_ANY(tb_flags
, MMUIDX
);
13021 dc
->mmu_idx
= core_to_aa64_mmu_idx(core_mmu_idx
);
13022 dc
->tbii
= EX_TBFLAG_A64(tb_flags
, TBII
);
13023 dc
->tbid
= EX_TBFLAG_A64(tb_flags
, TBID
);
13024 dc
->tcma
= EX_TBFLAG_A64(tb_flags
, TCMA
);
13025 dc
->current_el
= arm_mmu_idx_to_el(dc
->mmu_idx
);
13026 #if !defined(CONFIG_USER_ONLY)
13027 dc
->user
= (dc
->current_el
== 0);
13029 dc
->fp_excp_el
= EX_TBFLAG_ANY(tb_flags
, FPEXC_EL
);
13030 dc
->align_mem
= EX_TBFLAG_ANY(tb_flags
, ALIGN_MEM
);
13031 dc
->pstate_il
= EX_TBFLAG_ANY(tb_flags
, PSTATE__IL
);
13032 dc
->fgt_active
= EX_TBFLAG_ANY(tb_flags
, FGT_ACTIVE
);
13033 dc
->fgt_svc
= EX_TBFLAG_ANY(tb_flags
, FGT_SVC
);
13034 dc
->trap_eret
= EX_TBFLAG_A64(tb_flags
, TRAP_ERET
);
13035 dc
->sve_excp_el
= EX_TBFLAG_A64(tb_flags
, SVEEXC_EL
);
13036 dc
->sme_excp_el
= EX_TBFLAG_A64(tb_flags
, SMEEXC_EL
);
13037 dc
->vl
= (EX_TBFLAG_A64(tb_flags
, VL
) + 1) * 16;
13038 dc
->svl
= (EX_TBFLAG_A64(tb_flags
, SVL
) + 1) * 16;
13039 dc
->pauth_active
= EX_TBFLAG_A64(tb_flags
, PAUTH_ACTIVE
);
13040 dc
->bt
= EX_TBFLAG_A64(tb_flags
, BT
);
13041 dc
->btype
= EX_TBFLAG_A64(tb_flags
, BTYPE
);
13042 dc
->unpriv
= EX_TBFLAG_A64(tb_flags
, UNPRIV
);
13043 dc
->ata
[0] = EX_TBFLAG_A64(tb_flags
, ATA
);
13044 dc
->ata
[1] = EX_TBFLAG_A64(tb_flags
, ATA0
);
13045 dc
->mte_active
[0] = EX_TBFLAG_A64(tb_flags
, MTE_ACTIVE
);
13046 dc
->mte_active
[1] = EX_TBFLAG_A64(tb_flags
, MTE0_ACTIVE
);
13047 dc
->pstate_sm
= EX_TBFLAG_A64(tb_flags
, PSTATE_SM
);
13048 dc
->pstate_za
= EX_TBFLAG_A64(tb_flags
, PSTATE_ZA
);
13049 dc
->sme_trap_nonstreaming
= EX_TBFLAG_A64(tb_flags
, SME_TRAP_NONSTREAMING
);
13050 dc
->naa
= EX_TBFLAG_A64(tb_flags
, NAA
);
13051 dc
->nv
= EX_TBFLAG_A64(tb_flags
, NV
);
13052 dc
->nv1
= EX_TBFLAG_A64(tb_flags
, NV1
);
13053 dc
->nv2
= EX_TBFLAG_A64(tb_flags
, NV2
);
13054 dc
->nv2_mem_e20
= EX_TBFLAG_A64(tb_flags
, NV2_MEM_E20
);
13055 dc
->nv2_mem_be
= EX_TBFLAG_A64(tb_flags
, NV2_MEM_BE
);
13057 dc
->vec_stride
= 0;
13058 dc
->cp_regs
= arm_cpu
->cp_regs
;
13059 dc
->features
= env
->features
;
13060 dc
->dcz_blocksize
= arm_cpu
->dcz_blocksize
;
13061 dc
->gm_blocksize
= arm_cpu
->gm_blocksize
;
13063 #ifdef CONFIG_USER_ONLY
13064 /* In sve_probe_page, we assume TBI is enabled. */
13065 tcg_debug_assert(dc
->tbid
& 1);
13068 dc
->lse2
= dc_isar_feature(aa64_lse2
, dc
);
13070 /* Single step state. The code-generation logic here is:
13072 * generate code with no special handling for single-stepping (except
13073 * that anything that can make us go to SS_ACTIVE == 1 must end the TB;
13074 * this happens anyway because those changes are all system register or
13076 * SS_ACTIVE == 1, PSTATE.SS == 1: (active-not-pending)
13077 * emit code for one insn
13078 * emit code to clear PSTATE.SS
13079 * emit code to generate software step exception for completed step
13080 * end TB (as usual for having generated an exception)
13081 * SS_ACTIVE == 1, PSTATE.SS == 0: (active-pending)
13082 * emit code to generate a software step exception
13085 dc
->ss_active
= EX_TBFLAG_ANY(tb_flags
, SS_ACTIVE
);
13086 dc
->pstate_ss
= EX_TBFLAG_ANY(tb_flags
, PSTATE__SS
);
13087 dc
->is_ldex
= false;
13089 /* Bound the number of insns to execute to those left on the page. */
13090 bound
= -(dc
->base
.pc_first
| TARGET_PAGE_MASK
) / 4;
13092 /* If architectural single step active, limit to 1. */
13093 if (dc
->ss_active
) {
13096 dc
->base
.max_insns
= MIN(dc
->base
.max_insns
, bound
);
13099 static void aarch64_tr_tb_start(DisasContextBase
*db
, CPUState
*cpu
)
13103 static void aarch64_tr_insn_start(DisasContextBase
*dcbase
, CPUState
*cpu
)
13105 DisasContext
*dc
= container_of(dcbase
, DisasContext
, base
);
13106 target_ulong pc_arg
= dc
->base
.pc_next
;
13108 if (tb_cflags(dcbase
->tb
) & CF_PCREL
) {
13109 pc_arg
&= ~TARGET_PAGE_MASK
;
13111 tcg_gen_insn_start(pc_arg
, 0, 0);
13112 dc
->insn_start_updated
= false;
13115 static void aarch64_tr_translate_insn(DisasContextBase
*dcbase
, CPUState
*cpu
)
13117 DisasContext
*s
= container_of(dcbase
, DisasContext
, base
);
13118 CPUARMState
*env
= cpu_env(cpu
);
13119 uint64_t pc
= s
->base
.pc_next
;
13122 /* Singlestep exceptions have the highest priority. */
13123 if (s
->ss_active
&& !s
->pstate_ss
) {
13124 /* Singlestep state is Active-pending.
13125 * If we're in this state at the start of a TB then either
13126 * a) we just took an exception to an EL which is being debugged
13127 * and this is the first insn in the exception handler
13128 * b) debug exceptions were masked and we just unmasked them
13129 * without changing EL (eg by clearing PSTATE.D)
13130 * In either case we're going to take a swstep exception in the
13131 * "did not step an insn" case, and so the syndrome ISV and EX
13132 * bits should be zero.
13134 assert(s
->base
.num_insns
== 1);
13135 gen_swstep_exception(s
, 0, 0);
13136 s
->base
.is_jmp
= DISAS_NORETURN
;
13137 s
->base
.pc_next
= pc
+ 4;
13143 * PC alignment fault. This has priority over the instruction abort
13144 * that we would receive from a translation fault via arm_ldl_code.
13145 * This should only be possible after an indirect branch, at the
13148 assert(s
->base
.num_insns
== 1);
13149 gen_helper_exception_pc_alignment(tcg_env
, tcg_constant_tl(pc
));
13150 s
->base
.is_jmp
= DISAS_NORETURN
;
13151 s
->base
.pc_next
= QEMU_ALIGN_UP(pc
, 4);
13156 insn
= arm_ldl_code(env
, &s
->base
, pc
, s
->sctlr_b
);
13158 s
->base
.pc_next
= pc
+ 4;
13160 s
->fp_access_checked
= false;
13161 s
->sve_access_checked
= false;
13163 if (s
->pstate_il
) {
13165 * Illegal execution state. This has priority over BTI
13166 * exceptions, but comes after instruction abort exceptions.
13168 gen_exception_insn(s
, 0, EXCP_UDEF
, syn_illegalstate());
13172 if (dc_isar_feature(aa64_bti
, s
)) {
13173 if (s
->base
.num_insns
== 1) {
13175 * At the first insn of the TB, compute s->guarded_page.
13176 * We delayed computing this until successfully reading
13177 * the first insn of the TB, above. This (mostly) ensures
13178 * that the softmmu tlb entry has been populated, and the
13179 * page table GP bit is available.
13181 * Note that we need to compute this even if btype == 0,
13182 * because this value is used for BR instructions later
13183 * where ENV is not available.
13185 s
->guarded_page
= is_guarded_page(env
, s
);
13187 /* First insn can have btype set to non-zero. */
13188 tcg_debug_assert(s
->btype
>= 0);
13191 * Note that the Branch Target Exception has fairly high
13192 * priority -- below debugging exceptions but above most
13193 * everything else. This allows us to handle this now
13194 * instead of waiting until the insn is otherwise decoded.
13198 && !btype_destination_ok(insn
, s
->bt
, s
->btype
)) {
13199 gen_exception_insn(s
, 0, EXCP_UDEF
, syn_btitrap(s
->btype
));
13203 /* Not the first insn: btype must be 0. */
13204 tcg_debug_assert(s
->btype
== 0);
13208 s
->is_nonstreaming
= false;
13209 if (s
->sme_trap_nonstreaming
) {
13210 disas_sme_fa64(s
, insn
);
13213 if (!disas_a64(s
, insn
) &&
13214 !disas_sme(s
, insn
) &&
13215 !disas_sve(s
, insn
)) {
13216 disas_a64_legacy(s
, insn
);
13220 * After execution of most insns, btype is reset to 0.
13221 * Note that we set btype == -1 when the insn sets btype.
13223 if (s
->btype
> 0 && s
->base
.is_jmp
!= DISAS_NORETURN
) {
13228 static void aarch64_tr_tb_stop(DisasContextBase
*dcbase
, CPUState
*cpu
)
13230 DisasContext
*dc
= container_of(dcbase
, DisasContext
, base
);
13232 if (unlikely(dc
->ss_active
)) {
13233 /* Note that this means single stepping WFI doesn't halt the CPU.
13234 * For conditional branch insns this is harmless unreachable code as
13235 * gen_goto_tb() has already handled emitting the debug exception
13236 * (and thus a tb-jump is not possible when singlestepping).
13238 switch (dc
->base
.is_jmp
) {
13240 gen_a64_update_pc(dc
, 4);
13244 gen_step_complete_exception(dc
);
13246 case DISAS_NORETURN
:
13250 switch (dc
->base
.is_jmp
) {
13252 case DISAS_TOO_MANY
:
13253 gen_goto_tb(dc
, 1, 4);
13256 case DISAS_UPDATE_EXIT
:
13257 gen_a64_update_pc(dc
, 4);
13260 tcg_gen_exit_tb(NULL
, 0);
13262 case DISAS_UPDATE_NOCHAIN
:
13263 gen_a64_update_pc(dc
, 4);
13266 tcg_gen_lookup_and_goto_ptr();
13268 case DISAS_NORETURN
:
13272 gen_a64_update_pc(dc
, 4);
13273 gen_helper_wfe(tcg_env
);
13276 gen_a64_update_pc(dc
, 4);
13277 gen_helper_yield(tcg_env
);
13281 * This is a special case because we don't want to just halt
13282 * the CPU if trying to debug across a WFI.
13284 gen_a64_update_pc(dc
, 4);
13285 gen_helper_wfi(tcg_env
, tcg_constant_i32(4));
13287 * The helper doesn't necessarily throw an exception, but we
13288 * must go back to the main loop to check for interrupts anyway.
13290 tcg_gen_exit_tb(NULL
, 0);
13296 const TranslatorOps aarch64_translator_ops
= {
13297 .init_disas_context
= aarch64_tr_init_disas_context
,
13298 .tb_start
= aarch64_tr_tb_start
,
13299 .insn_start
= aarch64_tr_insn_start
,
13300 .translate_insn
= aarch64_tr_translate_insn
,
13301 .tb_stop
= aarch64_tr_tb_stop
,