2 * ARM v8.5-MemTag Operations
4 * Copyright (c) 2020 Linaro, Ltd.
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
22 #include "internals.h"
23 #include "exec/exec-all.h"
24 #include "exec/ram_addr.h"
25 #include "exec/cpu_ldst.h"
26 #include "exec/helper-proto.h"
27 #include "qapi/error.h"
28 #include "qemu/guest-random.h"
31 static int choose_nonexcluded_tag(int tag
, int offset
, uint16_t exclude
)
33 if (exclude
== 0xffff) {
37 while (exclude
& (1 << tag
)) {
44 } while (exclude
& (1 << tag
));
45 } while (--offset
> 0);
52 * @env: the cpu environment
53 * @ptr_mmu_idx: the addressing regime to use for the virtual address
54 * @ptr: the virtual address for which to look up tag memory
55 * @ptr_access: the access to use for the virtual address
56 * @ptr_size: the number of bytes in the normal memory access
57 * @tag_access: the access to use for the tag memory
58 * @tag_size: the number of bytes in the tag memory access
59 * @ra: the return address for exception handling
61 * Our tag memory is formatted as a sequence of little-endian nibbles.
62 * That is, the byte at (addr >> (LOG2_TAG_GRANULE + 1)) contains two
63 * tags, with the tag at [3:0] for the lower addr and the tag at [7:4]
64 * for the higher addr.
66 * Here, resolve the physical address from the virtual address, and return
67 * a pointer to the corresponding tag byte. Exit with exception if the
68 * virtual address is not accessible for @ptr_access.
70 * The @ptr_size and @tag_size values may not have an obvious relation
71 * due to the alignment of @ptr, and the number of tag checks required.
73 * If there is no tag storage corresponding to @ptr, return NULL.
75 static uint8_t *allocation_tag_mem(CPUARMState
*env
, int ptr_mmu_idx
,
76 uint64_t ptr
, MMUAccessType ptr_access
,
77 int ptr_size
, MMUAccessType tag_access
,
78 int tag_size
, uintptr_t ra
)
80 #ifdef CONFIG_USER_ONLY
81 uint64_t clean_ptr
= useronly_clean_ptr(ptr
);
82 int flags
= page_get_flags(clean_ptr
);
86 if (!(flags
& (ptr_access
== MMU_DATA_STORE
? PAGE_WRITE
: PAGE_READ
))) {
88 arm_cpu_tlb_fill(env_cpu(env
), ptr
, ptr_size
, ptr_access
,
89 ptr_mmu_idx
, false, ra
);
90 g_assert_not_reached();
93 /* Require both MAP_ANON and PROT_MTE for the page. */
94 if (!(flags
& PAGE_ANON
) || !(flags
& PAGE_MTE
)) {
98 tags
= page_get_target_data(clean_ptr
);
100 size_t alloc_size
= TARGET_PAGE_SIZE
>> (LOG2_TAG_GRANULE
+ 1);
101 tags
= page_alloc_target_data(clean_ptr
, alloc_size
);
102 assert(tags
!= NULL
);
105 index
= extract32(ptr
, LOG2_TAG_GRANULE
+ 1,
106 TARGET_PAGE_BITS
- LOG2_TAG_GRANULE
- 1);
110 CPUIOTLBEntry
*iotlbentry
;
113 hwaddr ptr_paddr
, tag_paddr
, xlat
;
116 AddressSpace
*tag_as
;
120 * Probe the first byte of the virtual address. This raises an
121 * exception for inaccessible pages, and resolves the virtual address
122 * into the softmmu tlb.
124 * When RA == 0, this is for mte_probe1. The page is expected to be
125 * valid. Indicate to probe_access_flags no-fault, then assert that
126 * we received a valid page.
128 flags
= probe_access_flags(env
, ptr
, ptr_access
, ptr_mmu_idx
,
130 assert(!(flags
& TLB_INVALID_MASK
));
133 * Find the iotlbentry for ptr. This *must* be present in the TLB
134 * because we just found the mapping.
135 * TODO: Perhaps there should be a cputlb helper that returns a
136 * matching tlb entry + iotlb entry.
138 index
= tlb_index(env
, ptr_mmu_idx
, ptr
);
139 # ifdef CONFIG_DEBUG_TCG
141 CPUTLBEntry
*entry
= tlb_entry(env
, ptr_mmu_idx
, ptr
);
142 target_ulong comparator
= (ptr_access
== MMU_DATA_LOAD
144 : tlb_addr_write(entry
));
145 g_assert(tlb_hit(comparator
, ptr
));
148 iotlbentry
= &env_tlb(env
)->d
[ptr_mmu_idx
].iotlb
[index
];
150 /* If the virtual page MemAttr != Tagged, access unchecked. */
151 if (!arm_tlb_mte_tagged(&iotlbentry
->attrs
)) {
156 * If not backed by host ram, there is no tag storage: access unchecked.
157 * This is probably a guest os bug though, so log it.
159 if (unlikely(flags
& TLB_MMIO
)) {
160 qemu_log_mask(LOG_GUEST_ERROR
,
161 "Page @ 0x%" PRIx64
" indicates Tagged Normal memory "
162 "but is not backed by host ram\n", ptr
);
167 * The Normal memory access can extend to the next page. E.g. a single
168 * 8-byte access to the last byte of a page will check only the last
169 * tag on the first page.
170 * Any page access exception has priority over tag check exception.
172 in_page
= -(ptr
| TARGET_PAGE_MASK
);
173 if (unlikely(ptr_size
> in_page
)) {
175 flags
|= probe_access_flags(env
, ptr
+ in_page
, ptr_access
,
176 ptr_mmu_idx
, ra
== 0, &ignore
, ra
);
177 assert(!(flags
& TLB_INVALID_MASK
));
180 /* Any debug exception has priority over a tag check exception. */
181 if (unlikely(flags
& TLB_WATCHPOINT
)) {
182 int wp
= ptr_access
== MMU_DATA_LOAD
? BP_MEM_READ
: BP_MEM_WRITE
;
184 cpu_check_watchpoint(env_cpu(env
), ptr
, ptr_size
,
185 iotlbentry
->attrs
, wp
, ra
);
189 * Find the physical address within the normal mem space.
190 * The memory region lookup must succeed because TLB_MMIO was
191 * not set in the cputlb lookup above.
193 mr
= memory_region_from_host(host
, &ptr_ra
);
194 tcg_debug_assert(mr
!= NULL
);
195 tcg_debug_assert(memory_region_is_ram(mr
));
198 ptr_paddr
+= mr
->addr
;
202 /* Convert to the physical address in tag space. */
203 tag_paddr
= ptr_paddr
>> (LOG2_TAG_GRANULE
+ 1);
205 /* Look up the address in tag space. */
206 tag_asi
= iotlbentry
->attrs
.secure
? ARMASIdx_TagS
: ARMASIdx_TagNS
;
207 tag_as
= cpu_get_address_space(env_cpu(env
), tag_asi
);
208 mr
= address_space_translate(tag_as
, tag_paddr
, &xlat
, NULL
,
209 tag_access
== MMU_DATA_STORE
,
213 * Note that @mr will never be NULL. If there is nothing in the address
214 * space at @tag_paddr, the translation will return the unallocated memory
215 * region. For our purposes, the result must be ram.
217 if (unlikely(!memory_region_is_ram(mr
))) {
218 /* ??? Failure is a board configuration error. */
219 qemu_log_mask(LOG_UNIMP
,
220 "Tag Memory @ 0x%" HWADDR_PRIx
" not found for "
221 "Normal Memory @ 0x%" HWADDR_PRIx
"\n",
222 tag_paddr
, ptr_paddr
);
227 * Ensure the tag memory is dirty on write, for migration.
228 * Tag memory can never contain code or display memory (vga).
230 if (tag_access
== MMU_DATA_STORE
) {
231 ram_addr_t tag_ra
= memory_region_get_ram_addr(mr
) + xlat
;
232 cpu_physical_memory_set_dirty_flag(tag_ra
, DIRTY_MEMORY_MIGRATION
);
235 return memory_region_get_ram_ptr(mr
) + xlat
;
239 uint64_t HELPER(irg
)(CPUARMState
*env
, uint64_t rn
, uint64_t rm
)
241 uint16_t exclude
= extract32(rm
| env
->cp15
.gcr_el1
, 0, 16);
242 int rrnd
= extract32(env
->cp15
.gcr_el1
, 16, 1);
243 int start
= extract32(env
->cp15
.rgsr_el1
, 0, 4);
244 int seed
= extract32(env
->cp15
.rgsr_el1
, 8, 16);
248 * Our IMPDEF choice for GCR_EL1.RRND==1 is to continue to use the
249 * deterministic algorithm. Except that with RRND==1 the kernel is
250 * not required to have set RGSR_EL1.SEED != 0, which is required for
251 * the deterministic algorithm to function. So we force a non-zero
252 * SEED for that case.
254 if (unlikely(seed
== 0) && rrnd
) {
259 if (qemu_guest_getrandom(&two
, sizeof(two
), &err
) < 0) {
261 * Failed, for unknown reasons in the crypto subsystem.
262 * Best we can do is log the reason and use a constant seed.
264 qemu_log_mask(LOG_UNIMP
, "IRG: Crypto failure: %s\n",
265 error_get_pretty(err
));
274 for (i
= offset
= 0; i
< 4; ++i
) {
275 /* NextRandomTagBit */
276 int top
= (extract32(seed
, 5, 1) ^ extract32(seed
, 3, 1) ^
277 extract32(seed
, 2, 1) ^ extract32(seed
, 0, 1));
278 seed
= (top
<< 15) | (seed
>> 1);
281 rtag
= choose_nonexcluded_tag(start
, offset
, exclude
);
282 env
->cp15
.rgsr_el1
= rtag
| (seed
<< 8);
284 return address_with_allocation_tag(rn
, rtag
);
287 uint64_t HELPER(addsubg
)(CPUARMState
*env
, uint64_t ptr
,
288 int32_t offset
, uint32_t tag_offset
)
290 int start_tag
= allocation_tag_from_addr(ptr
);
291 uint16_t exclude
= extract32(env
->cp15
.gcr_el1
, 0, 16);
292 int rtag
= choose_nonexcluded_tag(start_tag
, tag_offset
, exclude
);
294 return address_with_allocation_tag(ptr
+ offset
, rtag
);
297 static int load_tag1(uint64_t ptr
, uint8_t *mem
)
299 int ofs
= extract32(ptr
, LOG2_TAG_GRANULE
, 1) * 4;
300 return extract32(*mem
, ofs
, 4);
303 uint64_t HELPER(ldg
)(CPUARMState
*env
, uint64_t ptr
, uint64_t xt
)
305 int mmu_idx
= cpu_mmu_index(env
, false);
309 /* Trap if accessing an invalid page. */
310 mem
= allocation_tag_mem(env
, mmu_idx
, ptr
, MMU_DATA_LOAD
, 1,
311 MMU_DATA_LOAD
, 1, GETPC());
313 /* Load if page supports tags. */
315 rtag
= load_tag1(ptr
, mem
);
318 return address_with_allocation_tag(xt
, rtag
);
321 static void check_tag_aligned(CPUARMState
*env
, uint64_t ptr
, uintptr_t ra
)
323 if (unlikely(!QEMU_IS_ALIGNED(ptr
, TAG_GRANULE
))) {
324 arm_cpu_do_unaligned_access(env_cpu(env
), ptr
, MMU_DATA_STORE
,
325 cpu_mmu_index(env
, false), ra
);
326 g_assert_not_reached();
330 /* For use in a non-parallel context, store to the given nibble. */
331 static void store_tag1(uint64_t ptr
, uint8_t *mem
, int tag
)
333 int ofs
= extract32(ptr
, LOG2_TAG_GRANULE
, 1) * 4;
334 *mem
= deposit32(*mem
, ofs
, 4, tag
);
337 /* For use in a parallel context, atomically store to the given nibble. */
338 static void store_tag1_parallel(uint64_t ptr
, uint8_t *mem
, int tag
)
340 int ofs
= extract32(ptr
, LOG2_TAG_GRANULE
, 1) * 4;
341 uint8_t old
= qatomic_read(mem
);
344 uint8_t new = deposit32(old
, ofs
, 4, tag
);
345 uint8_t cmp
= qatomic_cmpxchg(mem
, old
, new);
346 if (likely(cmp
== old
)) {
353 typedef void stg_store1(uint64_t, uint8_t *, int);
355 static inline void do_stg(CPUARMState
*env
, uint64_t ptr
, uint64_t xt
,
356 uintptr_t ra
, stg_store1 store1
)
358 int mmu_idx
= cpu_mmu_index(env
, false);
361 check_tag_aligned(env
, ptr
, ra
);
363 /* Trap if accessing an invalid page. */
364 mem
= allocation_tag_mem(env
, mmu_idx
, ptr
, MMU_DATA_STORE
, TAG_GRANULE
,
365 MMU_DATA_STORE
, 1, ra
);
367 /* Store if page supports tags. */
369 store1(ptr
, mem
, allocation_tag_from_addr(xt
));
373 void HELPER(stg
)(CPUARMState
*env
, uint64_t ptr
, uint64_t xt
)
375 do_stg(env
, ptr
, xt
, GETPC(), store_tag1
);
378 void HELPER(stg_parallel
)(CPUARMState
*env
, uint64_t ptr
, uint64_t xt
)
380 do_stg(env
, ptr
, xt
, GETPC(), store_tag1_parallel
);
383 void HELPER(stg_stub
)(CPUARMState
*env
, uint64_t ptr
)
385 int mmu_idx
= cpu_mmu_index(env
, false);
386 uintptr_t ra
= GETPC();
388 check_tag_aligned(env
, ptr
, ra
);
389 probe_write(env
, ptr
, TAG_GRANULE
, mmu_idx
, ra
);
392 static inline void do_st2g(CPUARMState
*env
, uint64_t ptr
, uint64_t xt
,
393 uintptr_t ra
, stg_store1 store1
)
395 int mmu_idx
= cpu_mmu_index(env
, false);
396 int tag
= allocation_tag_from_addr(xt
);
397 uint8_t *mem1
, *mem2
;
399 check_tag_aligned(env
, ptr
, ra
);
402 * Trap if accessing an invalid page(s).
403 * This takes priority over !allocation_tag_access_enabled.
405 if (ptr
& TAG_GRANULE
) {
406 /* Two stores unaligned mod TAG_GRANULE*2 -- modify two bytes. */
407 mem1
= allocation_tag_mem(env
, mmu_idx
, ptr
, MMU_DATA_STORE
,
408 TAG_GRANULE
, MMU_DATA_STORE
, 1, ra
);
409 mem2
= allocation_tag_mem(env
, mmu_idx
, ptr
+ TAG_GRANULE
,
410 MMU_DATA_STORE
, TAG_GRANULE
,
411 MMU_DATA_STORE
, 1, ra
);
413 /* Store if page(s) support tags. */
415 store1(TAG_GRANULE
, mem1
, tag
);
418 store1(0, mem2
, tag
);
421 /* Two stores aligned mod TAG_GRANULE*2 -- modify one byte. */
422 mem1
= allocation_tag_mem(env
, mmu_idx
, ptr
, MMU_DATA_STORE
,
423 2 * TAG_GRANULE
, MMU_DATA_STORE
, 1, ra
);
426 qatomic_set(mem1
, tag
);
431 void HELPER(st2g
)(CPUARMState
*env
, uint64_t ptr
, uint64_t xt
)
433 do_st2g(env
, ptr
, xt
, GETPC(), store_tag1
);
436 void HELPER(st2g_parallel
)(CPUARMState
*env
, uint64_t ptr
, uint64_t xt
)
438 do_st2g(env
, ptr
, xt
, GETPC(), store_tag1_parallel
);
441 void HELPER(st2g_stub
)(CPUARMState
*env
, uint64_t ptr
)
443 int mmu_idx
= cpu_mmu_index(env
, false);
444 uintptr_t ra
= GETPC();
445 int in_page
= -(ptr
| TARGET_PAGE_MASK
);
447 check_tag_aligned(env
, ptr
, ra
);
449 if (likely(in_page
>= 2 * TAG_GRANULE
)) {
450 probe_write(env
, ptr
, 2 * TAG_GRANULE
, mmu_idx
, ra
);
452 probe_write(env
, ptr
, TAG_GRANULE
, mmu_idx
, ra
);
453 probe_write(env
, ptr
+ TAG_GRANULE
, TAG_GRANULE
, mmu_idx
, ra
);
457 #define LDGM_STGM_SIZE (4 << GMID_EL1_BS)
459 uint64_t HELPER(ldgm
)(CPUARMState
*env
, uint64_t ptr
)
461 int mmu_idx
= cpu_mmu_index(env
, false);
462 uintptr_t ra
= GETPC();
465 ptr
= QEMU_ALIGN_DOWN(ptr
, LDGM_STGM_SIZE
);
467 /* Trap if accessing an invalid page. */
468 tag_mem
= allocation_tag_mem(env
, mmu_idx
, ptr
, MMU_DATA_LOAD
,
469 LDGM_STGM_SIZE
, MMU_DATA_LOAD
,
470 LDGM_STGM_SIZE
/ (2 * TAG_GRANULE
), ra
);
472 /* The tag is squashed to zero if the page does not support tags. */
477 QEMU_BUILD_BUG_ON(GMID_EL1_BS
!= 6);
479 * We are loading 64-bits worth of tags. The ordering of elements
480 * within the word corresponds to a 64-bit little-endian operation.
482 return ldq_le_p(tag_mem
);
485 void HELPER(stgm
)(CPUARMState
*env
, uint64_t ptr
, uint64_t val
)
487 int mmu_idx
= cpu_mmu_index(env
, false);
488 uintptr_t ra
= GETPC();
491 ptr
= QEMU_ALIGN_DOWN(ptr
, LDGM_STGM_SIZE
);
493 /* Trap if accessing an invalid page. */
494 tag_mem
= allocation_tag_mem(env
, mmu_idx
, ptr
, MMU_DATA_STORE
,
495 LDGM_STGM_SIZE
, MMU_DATA_LOAD
,
496 LDGM_STGM_SIZE
/ (2 * TAG_GRANULE
), ra
);
499 * Tag store only happens if the page support tags,
500 * and if the OS has enabled access to the tags.
506 QEMU_BUILD_BUG_ON(GMID_EL1_BS
!= 6);
508 * We are storing 64-bits worth of tags. The ordering of elements
509 * within the word corresponds to a 64-bit little-endian operation.
511 stq_le_p(tag_mem
, val
);
514 void HELPER(stzgm_tags
)(CPUARMState
*env
, uint64_t ptr
, uint64_t val
)
516 uintptr_t ra
= GETPC();
517 int mmu_idx
= cpu_mmu_index(env
, false);
518 int log2_dcz_bytes
, log2_tag_bytes
;
519 intptr_t dcz_bytes
, tag_bytes
;
523 * In arm_cpu_realizefn, we assert that dcz > LOG2_TAG_GRANULE+1,
524 * i.e. 32 bytes, which is an unreasonably small dcz anyway,
525 * to make sure that we can access one complete tag byte here.
527 log2_dcz_bytes
= env_archcpu(env
)->dcz_blocksize
+ 2;
528 log2_tag_bytes
= log2_dcz_bytes
- (LOG2_TAG_GRANULE
+ 1);
529 dcz_bytes
= (intptr_t)1 << log2_dcz_bytes
;
530 tag_bytes
= (intptr_t)1 << log2_tag_bytes
;
533 mem
= allocation_tag_mem(env
, mmu_idx
, ptr
, MMU_DATA_STORE
, dcz_bytes
,
534 MMU_DATA_STORE
, tag_bytes
, ra
);
536 int tag_pair
= (val
& 0xf) * 0x11;
537 memset(mem
, tag_pair
, tag_bytes
);
541 /* Record a tag check failure. */
542 static void mte_check_fail(CPUARMState
*env
, uint32_t desc
,
543 uint64_t dirty_ptr
, uintptr_t ra
)
545 int mmu_idx
= FIELD_EX32(desc
, MTEDESC
, MIDX
);
546 ARMMMUIdx arm_mmu_idx
= core_to_aa64_mmu_idx(mmu_idx
);
547 int el
, reg_el
, tcf
, select
, is_write
, syn
;
550 reg_el
= regime_el(env
, arm_mmu_idx
);
551 sctlr
= env
->cp15
.sctlr_el
[reg_el
];
553 el
= arm_current_el(env
);
555 tcf
= extract64(sctlr
, 38, 2);
557 tcf
= extract64(sctlr
, 40, 2);
563 * Tag check fail causes a synchronous exception.
565 * In restore_state_to_opc, we set the exception syndrome
566 * for the load or store operation. Unwind first so we
567 * may overwrite that with the syndrome for the tag check.
569 cpu_restore_state(env_cpu(env
), ra
, true);
570 env
->exception
.vaddress
= dirty_ptr
;
572 is_write
= FIELD_EX32(desc
, MTEDESC
, WRITE
);
573 syn
= syn_data_abort_no_iss(el
!= 0, 0, 0, 0, 0, is_write
, 0x11);
574 raise_exception(env
, EXCP_DATA_ABORT
, syn
, exception_target_el(env
));
575 /* noreturn, but fall through to the assert anyway */
579 * Tag check fail does not affect the PE.
580 * We eliminate this case by not setting MTE_ACTIVE
581 * in tb_flags, so that we never make this runtime call.
583 g_assert_not_reached();
586 /* Tag check fail causes asynchronous flag set. */
587 if (regime_has_2_ranges(arm_mmu_idx
)) {
588 select
= extract64(dirty_ptr
, 55, 1);
592 env
->cp15
.tfsr_el
[el
] |= 1 << select
;
593 #ifdef CONFIG_USER_ONLY
595 * Stand in for a timer irq, setting _TIF_MTE_ASYNC_FAULT,
596 * which then sends a SIGSEGV when the thread is next scheduled.
597 * This cpu will return to the main loop at the end of the TB,
598 * which is rather sooner than "normal". But the alternative
599 * is waiting until the next syscall.
601 qemu_cpu_kick(env_cpu(env
));
606 /* Case 3: Reserved. */
607 qemu_log_mask(LOG_GUEST_ERROR
,
608 "Tag check failure with SCTLR_EL%d.TCF%s "
609 "set to reserved value %d\n",
610 reg_el
, el
? "" : "0", tcf
);
616 * Perform an MTE checked access for a single logical or atomic access.
618 static bool mte_probe1_int(CPUARMState
*env
, uint32_t desc
, uint64_t ptr
,
619 uintptr_t ra
, int bit55
)
621 int mem_tag
, mmu_idx
, ptr_tag
, size
;
625 ptr_tag
= allocation_tag_from_addr(ptr
);
627 if (tcma_check(desc
, bit55
, ptr_tag
)) {
631 mmu_idx
= FIELD_EX32(desc
, MTEDESC
, MIDX
);
632 type
= FIELD_EX32(desc
, MTEDESC
, WRITE
) ? MMU_DATA_STORE
: MMU_DATA_LOAD
;
633 size
= FIELD_EX32(desc
, MTEDESC
, ESIZE
);
635 mem
= allocation_tag_mem(env
, mmu_idx
, ptr
, type
, size
,
636 MMU_DATA_LOAD
, 1, ra
);
641 mem_tag
= load_tag1(ptr
, mem
);
642 return ptr_tag
== mem_tag
;
646 * No-fault version of mte_check1, to be used by SVE for MemSingleNF.
647 * Returns false if the access is Checked and the check failed. This
648 * is only intended to probe the tag -- the validity of the page must
649 * be checked beforehand.
651 bool mte_probe1(CPUARMState
*env
, uint32_t desc
, uint64_t ptr
)
653 int bit55
= extract64(ptr
, 55, 1);
655 /* If TBI is disabled, the access is unchecked. */
656 if (unlikely(!tbi_check(desc
, bit55
))) {
660 return mte_probe1_int(env
, desc
, ptr
, 0, bit55
);
663 uint64_t mte_check1(CPUARMState
*env
, uint32_t desc
,
664 uint64_t ptr
, uintptr_t ra
)
666 int bit55
= extract64(ptr
, 55, 1);
668 /* If TBI is disabled, the access is unchecked, and ptr is not dirty. */
669 if (unlikely(!tbi_check(desc
, bit55
))) {
673 if (unlikely(!mte_probe1_int(env
, desc
, ptr
, ra
, bit55
))) {
674 mte_check_fail(env
, desc
, ptr
, ra
);
677 return useronly_clean_ptr(ptr
);
680 uint64_t HELPER(mte_check1
)(CPUARMState
*env
, uint32_t desc
, uint64_t ptr
)
682 return mte_check1(env
, desc
, ptr
, GETPC());
686 * Perform an MTE checked access for multiple logical accesses.
691 * @tag: tag memory to test
692 * @odd: true to begin testing at tags at odd nibble
693 * @cmp: the tag to compare against
694 * @count: number of tags to test
696 * Return the number of successful tests.
697 * Thus a return value < @count indicates a failure.
699 * A note about sizes: count is expected to be small.
701 * The most common use will be LDP/STP of two integer registers,
702 * which means 16 bytes of memory touching at most 2 tags, but
703 * often the access is aligned and thus just 1 tag.
705 * Using AdvSIMD LD/ST (multiple), one can access 64 bytes of memory,
706 * touching at most 5 tags. SVE LDR/STR (vector) with the default
707 * vector length is also 64 bytes; the maximum architectural length
708 * is 256 bytes touching at most 9 tags.
710 * The loop below uses 7 logical operations and 1 memory operation
711 * per tag pair. An implementation that loads an aligned word and
712 * uses masking to ignore adjacent tags requires 18 logical operations
713 * and thus does not begin to pay off until 6 tags.
714 * Which, according to the survey above, is unlikely to be common.
716 static int checkN(uint8_t *mem
, int odd
, int cmp
, int count
)
720 /* Replicate the test tag and compare. */
730 if (unlikely((diff
) & 0x0f)) {
739 if (unlikely((diff
) & 0xf0)) {
751 uint64_t mte_checkN(CPUARMState
*env
, uint32_t desc
,
752 uint64_t ptr
, uintptr_t ra
)
754 int mmu_idx
, ptr_tag
, bit55
;
755 uint64_t ptr_last
, ptr_end
, prev_page
, next_page
;
756 uint64_t tag_first
, tag_end
;
757 uint64_t tag_byte_first
, tag_byte_end
;
758 uint32_t esize
, total
, tag_count
, tag_size
, n
, c
;
759 uint8_t *mem1
, *mem2
;
762 bit55
= extract64(ptr
, 55, 1);
764 /* If TBI is disabled, the access is unchecked, and ptr is not dirty. */
765 if (unlikely(!tbi_check(desc
, bit55
))) {
769 ptr_tag
= allocation_tag_from_addr(ptr
);
771 if (tcma_check(desc
, bit55
, ptr_tag
)) {
775 mmu_idx
= FIELD_EX32(desc
, MTEDESC
, MIDX
);
776 type
= FIELD_EX32(desc
, MTEDESC
, WRITE
) ? MMU_DATA_STORE
: MMU_DATA_LOAD
;
777 esize
= FIELD_EX32(desc
, MTEDESC
, ESIZE
);
778 total
= FIELD_EX32(desc
, MTEDESC
, TSIZE
);
780 /* Find the addr of the end of the access, and of the last element. */
781 ptr_end
= ptr
+ total
;
782 ptr_last
= ptr_end
- esize
;
784 /* Round the bounds to the tag granule, and compute the number of tags. */
785 tag_first
= QEMU_ALIGN_DOWN(ptr
, TAG_GRANULE
);
786 tag_end
= QEMU_ALIGN_UP(ptr_last
, TAG_GRANULE
);
787 tag_count
= (tag_end
- tag_first
) / TAG_GRANULE
;
789 /* Round the bounds to twice the tag granule, and compute the bytes. */
790 tag_byte_first
= QEMU_ALIGN_DOWN(ptr
, 2 * TAG_GRANULE
);
791 tag_byte_end
= QEMU_ALIGN_UP(ptr_last
, 2 * TAG_GRANULE
);
793 /* Locate the page boundaries. */
794 prev_page
= ptr
& TARGET_PAGE_MASK
;
795 next_page
= prev_page
+ TARGET_PAGE_SIZE
;
797 if (likely(tag_end
- prev_page
<= TARGET_PAGE_SIZE
)) {
798 /* Memory access stays on one page. */
799 tag_size
= (tag_byte_end
- tag_byte_first
) / (2 * TAG_GRANULE
);
800 mem1
= allocation_tag_mem(env
, mmu_idx
, ptr
, type
, total
,
801 MMU_DATA_LOAD
, tag_size
, ra
);
805 /* Perform all of the comparisons. */
806 n
= checkN(mem1
, ptr
& TAG_GRANULE
, ptr_tag
, tag_count
);
808 /* Memory access crosses to next page. */
809 tag_size
= (next_page
- tag_byte_first
) / (2 * TAG_GRANULE
);
810 mem1
= allocation_tag_mem(env
, mmu_idx
, ptr
, type
, next_page
- ptr
,
811 MMU_DATA_LOAD
, tag_size
, ra
);
813 tag_size
= (tag_byte_end
- next_page
) / (2 * TAG_GRANULE
);
814 mem2
= allocation_tag_mem(env
, mmu_idx
, next_page
, type
,
816 MMU_DATA_LOAD
, tag_size
, ra
);
819 * Perform all of the comparisons.
820 * Note the possible but unlikely case of the operation spanning
821 * two pages that do not both have tagging enabled.
823 n
= c
= (next_page
- tag_first
) / TAG_GRANULE
;
825 n
= checkN(mem1
, ptr
& TAG_GRANULE
, ptr_tag
, c
);
831 n
+= checkN(mem2
, 0, ptr_tag
, tag_count
- c
);
836 * If we failed, we know which granule. Compute the element that
837 * is first in that granule, and signal failure on that element.
839 if (unlikely(n
< tag_count
)) {
842 fail_ofs
= tag_first
+ n
* TAG_GRANULE
- ptr
;
843 fail_ofs
= ROUND_UP(fail_ofs
, esize
);
844 mte_check_fail(env
, desc
, ptr
+ fail_ofs
, ra
);
848 return useronly_clean_ptr(ptr
);
851 uint64_t HELPER(mte_checkN
)(CPUARMState
*env
, uint32_t desc
, uint64_t ptr
)
853 return mte_checkN(env
, desc
, ptr
, GETPC());
857 * Perform an MTE checked access for DC_ZVA.
859 uint64_t HELPER(mte_check_zva
)(CPUARMState
*env
, uint32_t desc
, uint64_t ptr
)
861 uintptr_t ra
= GETPC();
862 int log2_dcz_bytes
, log2_tag_bytes
;
864 intptr_t dcz_bytes
, tag_bytes
, i
;
866 uint64_t ptr_tag
, mem_tag
, align_ptr
;
868 bit55
= extract64(ptr
, 55, 1);
870 /* If TBI is disabled, the access is unchecked, and ptr is not dirty. */
871 if (unlikely(!tbi_check(desc
, bit55
))) {
875 ptr_tag
= allocation_tag_from_addr(ptr
);
877 if (tcma_check(desc
, bit55
, ptr_tag
)) {
882 * In arm_cpu_realizefn, we asserted that dcz > LOG2_TAG_GRANULE+1,
883 * i.e. 32 bytes, which is an unreasonably small dcz anyway, to make
884 * sure that we can access one complete tag byte here.
886 log2_dcz_bytes
= env_archcpu(env
)->dcz_blocksize
+ 2;
887 log2_tag_bytes
= log2_dcz_bytes
- (LOG2_TAG_GRANULE
+ 1);
888 dcz_bytes
= (intptr_t)1 << log2_dcz_bytes
;
889 tag_bytes
= (intptr_t)1 << log2_tag_bytes
;
890 align_ptr
= ptr
& -dcz_bytes
;
893 * Trap if accessing an invalid page. DC_ZVA requires that we supply
894 * the original pointer for an invalid page. But watchpoints require
895 * that we probe the actual space. So do both.
897 mmu_idx
= FIELD_EX32(desc
, MTEDESC
, MIDX
);
898 (void) probe_write(env
, ptr
, 1, mmu_idx
, ra
);
899 mem
= allocation_tag_mem(env
, mmu_idx
, align_ptr
, MMU_DATA_STORE
,
900 dcz_bytes
, MMU_DATA_LOAD
, tag_bytes
, ra
);
906 * Unlike the reasoning for checkN, DC_ZVA is always aligned, and thus
907 * it is quite easy to perform all of the comparisons at once without
910 * The most common zva block size is 64; some of the thunderx cpus use
911 * a block size of 128. For user-only, aarch64_max_initfn will set the
912 * block size to 512. Fill out the other cases for future-proofing.
914 * In order to be able to find the first miscompare later, we want the
915 * tag bytes to be in little-endian order.
917 switch (log2_tag_bytes
) {
918 case 0: /* zva_blocksize 32 */
919 mem_tag
= *(uint8_t *)mem
;
922 case 1: /* zva_blocksize 64 */
923 mem_tag
= cpu_to_le16(*(uint16_t *)mem
);
926 case 2: /* zva_blocksize 128 */
927 mem_tag
= cpu_to_le32(*(uint32_t *)mem
);
928 ptr_tag
*= 0x11111111u
;
930 case 3: /* zva_blocksize 256 */
931 mem_tag
= cpu_to_le64(*(uint64_t *)mem
);
932 ptr_tag
*= 0x1111111111111111ull
;
935 default: /* zva_blocksize 512, 1024, 2048 */
936 ptr_tag
*= 0x1111111111111111ull
;
939 mem_tag
= cpu_to_le64(*(uint64_t *)(mem
+ i
));
940 if (unlikely(mem_tag
!= ptr_tag
)) {
944 align_ptr
+= 16 * TAG_GRANULE
;
945 } while (i
< tag_bytes
);
949 if (likely(mem_tag
== ptr_tag
)) {
954 /* Locate the first nibble that differs. */
955 i
= ctz64(mem_tag
^ ptr_tag
) >> 4;
956 mte_check_fail(env
, desc
, align_ptr
+ i
* TAG_GRANULE
, ra
);
959 return useronly_clean_ptr(ptr
);