spapr: split the IRQ allocation sequence
[qemu/rayw.git] / accel / tcg / cputlb.c
blob0a721bb9c40e28193afdd4fac59e42b9d77e5137
1 /*
2 * Common CPU TLB handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
21 #include "qemu/main-loop.h"
22 #include "cpu.h"
23 #include "exec/exec-all.h"
24 #include "exec/memory.h"
25 #include "exec/address-spaces.h"
26 #include "exec/cpu_ldst.h"
27 #include "exec/cputlb.h"
28 #include "exec/memory-internal.h"
29 #include "exec/ram_addr.h"
30 #include "tcg/tcg.h"
31 #include "qemu/error-report.h"
32 #include "exec/log.h"
33 #include "exec/helper-proto.h"
34 #include "qemu/atomic.h"
36 /* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
37 /* #define DEBUG_TLB */
38 /* #define DEBUG_TLB_LOG */
40 #ifdef DEBUG_TLB
41 # define DEBUG_TLB_GATE 1
42 # ifdef DEBUG_TLB_LOG
43 # define DEBUG_TLB_LOG_GATE 1
44 # else
45 # define DEBUG_TLB_LOG_GATE 0
46 # endif
47 #else
48 # define DEBUG_TLB_GATE 0
49 # define DEBUG_TLB_LOG_GATE 0
50 #endif
52 #define tlb_debug(fmt, ...) do { \
53 if (DEBUG_TLB_LOG_GATE) { \
54 qemu_log_mask(CPU_LOG_MMU, "%s: " fmt, __func__, \
55 ## __VA_ARGS__); \
56 } else if (DEBUG_TLB_GATE) { \
57 fprintf(stderr, "%s: " fmt, __func__, ## __VA_ARGS__); \
58 } \
59 } while (0)
61 #define assert_cpu_is_self(this_cpu) do { \
62 if (DEBUG_TLB_GATE) { \
63 g_assert(!cpu->created || qemu_cpu_is_self(cpu)); \
64 } \
65 } while (0)
67 /* run_on_cpu_data.target_ptr should always be big enough for a
68 * target_ulong even on 32 bit builds */
69 QEMU_BUILD_BUG_ON(sizeof(target_ulong) > sizeof(run_on_cpu_data));
71 /* We currently can't handle more than 16 bits in the MMUIDX bitmask.
73 QEMU_BUILD_BUG_ON(NB_MMU_MODES > 16);
74 #define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1)
76 /* flush_all_helper: run fn across all cpus
78 * If the wait flag is set then the src cpu's helper will be queued as
79 * "safe" work and the loop exited creating a synchronisation point
80 * where all queued work will be finished before execution starts
81 * again.
83 static void flush_all_helper(CPUState *src, run_on_cpu_func fn,
84 run_on_cpu_data d)
86 CPUState *cpu;
88 CPU_FOREACH(cpu) {
89 if (cpu != src) {
90 async_run_on_cpu(cpu, fn, d);
95 size_t tlb_flush_count(void)
97 CPUState *cpu;
98 size_t count = 0;
100 CPU_FOREACH(cpu) {
101 CPUArchState *env = cpu->env_ptr;
103 count += atomic_read(&env->tlb_flush_count);
105 return count;
108 /* This is OK because CPU architectures generally permit an
109 * implementation to drop entries from the TLB at any time, so
110 * flushing more entries than required is only an efficiency issue,
111 * not a correctness issue.
113 static void tlb_flush_nocheck(CPUState *cpu)
115 CPUArchState *env = cpu->env_ptr;
117 /* The QOM tests will trigger tlb_flushes without setting up TCG
118 * so we bug out here in that case.
120 if (!tcg_enabled()) {
121 return;
124 assert_cpu_is_self(cpu);
125 atomic_set(&env->tlb_flush_count, env->tlb_flush_count + 1);
126 tlb_debug("(count: %zu)\n", tlb_flush_count());
128 tb_lock();
130 memset(env->tlb_table, -1, sizeof(env->tlb_table));
131 memset(env->tlb_v_table, -1, sizeof(env->tlb_v_table));
132 cpu_tb_jmp_cache_clear(cpu);
134 env->vtlb_index = 0;
135 env->tlb_flush_addr = -1;
136 env->tlb_flush_mask = 0;
138 tb_unlock();
140 atomic_mb_set(&cpu->pending_tlb_flush, 0);
143 static void tlb_flush_global_async_work(CPUState *cpu, run_on_cpu_data data)
145 tlb_flush_nocheck(cpu);
148 void tlb_flush(CPUState *cpu)
150 if (cpu->created && !qemu_cpu_is_self(cpu)) {
151 if (atomic_mb_read(&cpu->pending_tlb_flush) != ALL_MMUIDX_BITS) {
152 atomic_mb_set(&cpu->pending_tlb_flush, ALL_MMUIDX_BITS);
153 async_run_on_cpu(cpu, tlb_flush_global_async_work,
154 RUN_ON_CPU_NULL);
156 } else {
157 tlb_flush_nocheck(cpu);
161 void tlb_flush_all_cpus(CPUState *src_cpu)
163 const run_on_cpu_func fn = tlb_flush_global_async_work;
164 flush_all_helper(src_cpu, fn, RUN_ON_CPU_NULL);
165 fn(src_cpu, RUN_ON_CPU_NULL);
168 void tlb_flush_all_cpus_synced(CPUState *src_cpu)
170 const run_on_cpu_func fn = tlb_flush_global_async_work;
171 flush_all_helper(src_cpu, fn, RUN_ON_CPU_NULL);
172 async_safe_run_on_cpu(src_cpu, fn, RUN_ON_CPU_NULL);
175 static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data)
177 CPUArchState *env = cpu->env_ptr;
178 unsigned long mmu_idx_bitmask = data.host_int;
179 int mmu_idx;
181 assert_cpu_is_self(cpu);
183 tb_lock();
185 tlb_debug("start: mmu_idx:0x%04lx\n", mmu_idx_bitmask);
187 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
189 if (test_bit(mmu_idx, &mmu_idx_bitmask)) {
190 tlb_debug("%d\n", mmu_idx);
192 memset(env->tlb_table[mmu_idx], -1, sizeof(env->tlb_table[0]));
193 memset(env->tlb_v_table[mmu_idx], -1, sizeof(env->tlb_v_table[0]));
197 cpu_tb_jmp_cache_clear(cpu);
199 tlb_debug("done\n");
201 tb_unlock();
204 void tlb_flush_by_mmuidx(CPUState *cpu, uint16_t idxmap)
206 tlb_debug("mmu_idx: 0x%" PRIx16 "\n", idxmap);
208 if (!qemu_cpu_is_self(cpu)) {
209 uint16_t pending_flushes = idxmap;
210 pending_flushes &= ~atomic_mb_read(&cpu->pending_tlb_flush);
212 if (pending_flushes) {
213 tlb_debug("reduced mmu_idx: 0x%" PRIx16 "\n", pending_flushes);
215 atomic_or(&cpu->pending_tlb_flush, pending_flushes);
216 async_run_on_cpu(cpu, tlb_flush_by_mmuidx_async_work,
217 RUN_ON_CPU_HOST_INT(pending_flushes));
219 } else {
220 tlb_flush_by_mmuidx_async_work(cpu,
221 RUN_ON_CPU_HOST_INT(idxmap));
225 void tlb_flush_by_mmuidx_all_cpus(CPUState *src_cpu, uint16_t idxmap)
227 const run_on_cpu_func fn = tlb_flush_by_mmuidx_async_work;
229 tlb_debug("mmu_idx: 0x%"PRIx16"\n", idxmap);
231 flush_all_helper(src_cpu, fn, RUN_ON_CPU_HOST_INT(idxmap));
232 fn(src_cpu, RUN_ON_CPU_HOST_INT(idxmap));
235 void tlb_flush_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
236 uint16_t idxmap)
238 const run_on_cpu_func fn = tlb_flush_by_mmuidx_async_work;
240 tlb_debug("mmu_idx: 0x%"PRIx16"\n", idxmap);
242 flush_all_helper(src_cpu, fn, RUN_ON_CPU_HOST_INT(idxmap));
243 async_safe_run_on_cpu(src_cpu, fn, RUN_ON_CPU_HOST_INT(idxmap));
248 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
250 if (addr == (tlb_entry->addr_read &
251 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
252 addr == (tlb_entry->addr_write &
253 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
254 addr == (tlb_entry->addr_code &
255 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
256 memset(tlb_entry, -1, sizeof(*tlb_entry));
260 static void tlb_flush_page_async_work(CPUState *cpu, run_on_cpu_data data)
262 CPUArchState *env = cpu->env_ptr;
263 target_ulong addr = (target_ulong) data.target_ptr;
264 int i;
265 int mmu_idx;
267 assert_cpu_is_self(cpu);
269 tlb_debug("page :" TARGET_FMT_lx "\n", addr);
271 /* Check if we need to flush due to large pages. */
272 if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
273 tlb_debug("forcing full flush ("
274 TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
275 env->tlb_flush_addr, env->tlb_flush_mask);
277 tlb_flush(cpu);
278 return;
281 addr &= TARGET_PAGE_MASK;
282 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
283 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
284 tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
287 /* check whether there are entries that need to be flushed in the vtlb */
288 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
289 int k;
290 for (k = 0; k < CPU_VTLB_SIZE; k++) {
291 tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], addr);
295 tb_flush_jmp_cache(cpu, addr);
298 void tlb_flush_page(CPUState *cpu, target_ulong addr)
300 tlb_debug("page :" TARGET_FMT_lx "\n", addr);
302 if (!qemu_cpu_is_self(cpu)) {
303 async_run_on_cpu(cpu, tlb_flush_page_async_work,
304 RUN_ON_CPU_TARGET_PTR(addr));
305 } else {
306 tlb_flush_page_async_work(cpu, RUN_ON_CPU_TARGET_PTR(addr));
310 /* As we are going to hijack the bottom bits of the page address for a
311 * mmuidx bit mask we need to fail to build if we can't do that
313 QEMU_BUILD_BUG_ON(NB_MMU_MODES > TARGET_PAGE_BITS_MIN);
315 static void tlb_flush_page_by_mmuidx_async_work(CPUState *cpu,
316 run_on_cpu_data data)
318 CPUArchState *env = cpu->env_ptr;
319 target_ulong addr_and_mmuidx = (target_ulong) data.target_ptr;
320 target_ulong addr = addr_and_mmuidx & TARGET_PAGE_MASK;
321 unsigned long mmu_idx_bitmap = addr_and_mmuidx & ALL_MMUIDX_BITS;
322 int page = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
323 int mmu_idx;
324 int i;
326 assert_cpu_is_self(cpu);
328 tlb_debug("page:%d addr:"TARGET_FMT_lx" mmu_idx:0x%lx\n",
329 page, addr, mmu_idx_bitmap);
331 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
332 if (test_bit(mmu_idx, &mmu_idx_bitmap)) {
333 tlb_flush_entry(&env->tlb_table[mmu_idx][page], addr);
335 /* check whether there are vltb entries that need to be flushed */
336 for (i = 0; i < CPU_VTLB_SIZE; i++) {
337 tlb_flush_entry(&env->tlb_v_table[mmu_idx][i], addr);
342 tb_flush_jmp_cache(cpu, addr);
345 static void tlb_check_page_and_flush_by_mmuidx_async_work(CPUState *cpu,
346 run_on_cpu_data data)
348 CPUArchState *env = cpu->env_ptr;
349 target_ulong addr_and_mmuidx = (target_ulong) data.target_ptr;
350 target_ulong addr = addr_and_mmuidx & TARGET_PAGE_MASK;
351 unsigned long mmu_idx_bitmap = addr_and_mmuidx & ALL_MMUIDX_BITS;
353 tlb_debug("addr:"TARGET_FMT_lx" mmu_idx: %04lx\n", addr, mmu_idx_bitmap);
355 /* Check if we need to flush due to large pages. */
356 if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
357 tlb_debug("forced full flush ("
358 TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
359 env->tlb_flush_addr, env->tlb_flush_mask);
361 tlb_flush_by_mmuidx_async_work(cpu,
362 RUN_ON_CPU_HOST_INT(mmu_idx_bitmap));
363 } else {
364 tlb_flush_page_by_mmuidx_async_work(cpu, data);
368 void tlb_flush_page_by_mmuidx(CPUState *cpu, target_ulong addr, uint16_t idxmap)
370 target_ulong addr_and_mmu_idx;
372 tlb_debug("addr: "TARGET_FMT_lx" mmu_idx:%" PRIx16 "\n", addr, idxmap);
374 /* This should already be page aligned */
375 addr_and_mmu_idx = addr & TARGET_PAGE_MASK;
376 addr_and_mmu_idx |= idxmap;
378 if (!qemu_cpu_is_self(cpu)) {
379 async_run_on_cpu(cpu, tlb_check_page_and_flush_by_mmuidx_async_work,
380 RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
381 } else {
382 tlb_check_page_and_flush_by_mmuidx_async_work(
383 cpu, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
387 void tlb_flush_page_by_mmuidx_all_cpus(CPUState *src_cpu, target_ulong addr,
388 uint16_t idxmap)
390 const run_on_cpu_func fn = tlb_check_page_and_flush_by_mmuidx_async_work;
391 target_ulong addr_and_mmu_idx;
393 tlb_debug("addr: "TARGET_FMT_lx" mmu_idx:%"PRIx16"\n", addr, idxmap);
395 /* This should already be page aligned */
396 addr_and_mmu_idx = addr & TARGET_PAGE_MASK;
397 addr_and_mmu_idx |= idxmap;
399 flush_all_helper(src_cpu, fn, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
400 fn(src_cpu, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
403 void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
404 target_ulong addr,
405 uint16_t idxmap)
407 const run_on_cpu_func fn = tlb_check_page_and_flush_by_mmuidx_async_work;
408 target_ulong addr_and_mmu_idx;
410 tlb_debug("addr: "TARGET_FMT_lx" mmu_idx:%"PRIx16"\n", addr, idxmap);
412 /* This should already be page aligned */
413 addr_and_mmu_idx = addr & TARGET_PAGE_MASK;
414 addr_and_mmu_idx |= idxmap;
416 flush_all_helper(src_cpu, fn, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
417 async_safe_run_on_cpu(src_cpu, fn, RUN_ON_CPU_TARGET_PTR(addr_and_mmu_idx));
420 void tlb_flush_page_all_cpus(CPUState *src, target_ulong addr)
422 const run_on_cpu_func fn = tlb_flush_page_async_work;
424 flush_all_helper(src, fn, RUN_ON_CPU_TARGET_PTR(addr));
425 fn(src, RUN_ON_CPU_TARGET_PTR(addr));
428 void tlb_flush_page_all_cpus_synced(CPUState *src,
429 target_ulong addr)
431 const run_on_cpu_func fn = tlb_flush_page_async_work;
433 flush_all_helper(src, fn, RUN_ON_CPU_TARGET_PTR(addr));
434 async_safe_run_on_cpu(src, fn, RUN_ON_CPU_TARGET_PTR(addr));
437 /* update the TLBs so that writes to code in the virtual page 'addr'
438 can be detected */
439 void tlb_protect_code(ram_addr_t ram_addr)
441 cpu_physical_memory_test_and_clear_dirty(ram_addr, TARGET_PAGE_SIZE,
442 DIRTY_MEMORY_CODE);
445 /* update the TLB so that writes in physical page 'phys_addr' are no longer
446 tested for self modifying code */
447 void tlb_unprotect_code(ram_addr_t ram_addr)
449 cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_CODE);
454 * Dirty write flag handling
456 * When the TCG code writes to a location it looks up the address in
457 * the TLB and uses that data to compute the final address. If any of
458 * the lower bits of the address are set then the slow path is forced.
459 * There are a number of reasons to do this but for normal RAM the
460 * most usual is detecting writes to code regions which may invalidate
461 * generated code.
463 * Because we want other vCPUs to respond to changes straight away we
464 * update the te->addr_write field atomically. If the TLB entry has
465 * been changed by the vCPU in the mean time we skip the update.
467 * As this function uses atomic accesses we also need to ensure
468 * updates to tlb_entries follow the same access rules. We don't need
469 * to worry about this for oversized guests as MTTCG is disabled for
470 * them.
473 static void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry, uintptr_t start,
474 uintptr_t length)
476 #if TCG_OVERSIZED_GUEST
477 uintptr_t addr = tlb_entry->addr_write;
479 if ((addr & (TLB_INVALID_MASK | TLB_MMIO | TLB_NOTDIRTY)) == 0) {
480 addr &= TARGET_PAGE_MASK;
481 addr += tlb_entry->addend;
482 if ((addr - start) < length) {
483 tlb_entry->addr_write |= TLB_NOTDIRTY;
486 #else
487 /* paired with atomic_mb_set in tlb_set_page_with_attrs */
488 uintptr_t orig_addr = atomic_mb_read(&tlb_entry->addr_write);
489 uintptr_t addr = orig_addr;
491 if ((addr & (TLB_INVALID_MASK | TLB_MMIO | TLB_NOTDIRTY)) == 0) {
492 addr &= TARGET_PAGE_MASK;
493 addr += atomic_read(&tlb_entry->addend);
494 if ((addr - start) < length) {
495 uintptr_t notdirty_addr = orig_addr | TLB_NOTDIRTY;
496 atomic_cmpxchg(&tlb_entry->addr_write, orig_addr, notdirty_addr);
499 #endif
502 /* For atomic correctness when running MTTCG we need to use the right
503 * primitives when copying entries */
504 static inline void copy_tlb_helper(CPUTLBEntry *d, CPUTLBEntry *s,
505 bool atomic_set)
507 #if TCG_OVERSIZED_GUEST
508 *d = *s;
509 #else
510 if (atomic_set) {
511 d->addr_read = s->addr_read;
512 d->addr_code = s->addr_code;
513 atomic_set(&d->addend, atomic_read(&s->addend));
514 /* Pairs with flag setting in tlb_reset_dirty_range */
515 atomic_mb_set(&d->addr_write, atomic_read(&s->addr_write));
516 } else {
517 d->addr_read = s->addr_read;
518 d->addr_write = atomic_read(&s->addr_write);
519 d->addr_code = s->addr_code;
520 d->addend = atomic_read(&s->addend);
522 #endif
525 /* This is a cross vCPU call (i.e. another vCPU resetting the flags of
526 * the target vCPU). As such care needs to be taken that we don't
527 * dangerously race with another vCPU update. The only thing actually
528 * updated is the target TLB entry ->addr_write flags.
530 void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length)
532 CPUArchState *env;
534 int mmu_idx;
536 env = cpu->env_ptr;
537 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
538 unsigned int i;
540 for (i = 0; i < CPU_TLB_SIZE; i++) {
541 tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
542 start1, length);
545 for (i = 0; i < CPU_VTLB_SIZE; i++) {
546 tlb_reset_dirty_range(&env->tlb_v_table[mmu_idx][i],
547 start1, length);
552 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
554 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY)) {
555 tlb_entry->addr_write = vaddr;
559 /* update the TLB corresponding to virtual page vaddr
560 so that it is no longer dirty */
561 void tlb_set_dirty(CPUState *cpu, target_ulong vaddr)
563 CPUArchState *env = cpu->env_ptr;
564 int i;
565 int mmu_idx;
567 assert_cpu_is_self(cpu);
569 vaddr &= TARGET_PAGE_MASK;
570 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
571 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
572 tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
575 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
576 int k;
577 for (k = 0; k < CPU_VTLB_SIZE; k++) {
578 tlb_set_dirty1(&env->tlb_v_table[mmu_idx][k], vaddr);
583 /* Our TLB does not support large pages, so remember the area covered by
584 large pages and trigger a full TLB flush if these are invalidated. */
585 static void tlb_add_large_page(CPUArchState *env, target_ulong vaddr,
586 target_ulong size)
588 target_ulong mask = ~(size - 1);
590 if (env->tlb_flush_addr == (target_ulong)-1) {
591 env->tlb_flush_addr = vaddr & mask;
592 env->tlb_flush_mask = mask;
593 return;
595 /* Extend the existing region to include the new page.
596 This is a compromise between unnecessary flushes and the cost
597 of maintaining a full variable size TLB. */
598 mask &= env->tlb_flush_mask;
599 while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
600 mask <<= 1;
602 env->tlb_flush_addr &= mask;
603 env->tlb_flush_mask = mask;
606 /* Add a new TLB entry. At most one entry for a given virtual address
607 * is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
608 * supplied size is only used by tlb_flush_page.
610 * Called from TCG-generated code, which is under an RCU read-side
611 * critical section.
613 void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
614 hwaddr paddr, MemTxAttrs attrs, int prot,
615 int mmu_idx, target_ulong size)
617 CPUArchState *env = cpu->env_ptr;
618 MemoryRegionSection *section;
619 unsigned int index;
620 target_ulong address;
621 target_ulong code_address;
622 uintptr_t addend;
623 CPUTLBEntry *te, *tv, tn;
624 hwaddr iotlb, xlat, sz;
625 unsigned vidx = env->vtlb_index++ % CPU_VTLB_SIZE;
626 int asidx = cpu_asidx_from_attrs(cpu, attrs);
628 assert_cpu_is_self(cpu);
629 assert(size >= TARGET_PAGE_SIZE);
630 if (size != TARGET_PAGE_SIZE) {
631 tlb_add_large_page(env, vaddr, size);
634 sz = size;
635 section = address_space_translate_for_iotlb(cpu, asidx, paddr, &xlat, &sz,
636 attrs, &prot);
637 assert(sz >= TARGET_PAGE_SIZE);
639 tlb_debug("vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
640 " prot=%x idx=%d\n",
641 vaddr, paddr, prot, mmu_idx);
643 address = vaddr;
644 if (!memory_region_is_ram(section->mr) && !memory_region_is_romd(section->mr)) {
645 /* IO memory case */
646 address |= TLB_MMIO;
647 addend = 0;
648 } else {
649 /* TLB_MMIO for rom/romd handled below */
650 addend = (uintptr_t)memory_region_get_ram_ptr(section->mr) + xlat;
653 code_address = address;
654 iotlb = memory_region_section_get_iotlb(cpu, section, vaddr, paddr, xlat,
655 prot, &address);
657 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
658 te = &env->tlb_table[mmu_idx][index];
659 /* do not discard the translation in te, evict it into a victim tlb */
660 tv = &env->tlb_v_table[mmu_idx][vidx];
662 /* addr_write can race with tlb_reset_dirty_range */
663 copy_tlb_helper(tv, te, true);
665 env->iotlb_v[mmu_idx][vidx] = env->iotlb[mmu_idx][index];
667 /* refill the tlb */
669 * At this point iotlb contains a physical section number in the lower
670 * TARGET_PAGE_BITS, and either
671 * + the ram_addr_t of the page base of the target RAM (if NOTDIRTY or ROM)
672 * + the offset within section->mr of the page base (otherwise)
673 * We subtract the vaddr (which is page aligned and thus won't
674 * disturb the low bits) to give an offset which can be added to the
675 * (non-page-aligned) vaddr of the eventual memory access to get
676 * the MemoryRegion offset for the access. Note that the vaddr we
677 * subtract here is that of the page base, and not the same as the
678 * vaddr we add back in io_readx()/io_writex()/get_page_addr_code().
680 env->iotlb[mmu_idx][index].addr = iotlb - vaddr;
681 env->iotlb[mmu_idx][index].attrs = attrs;
683 /* Now calculate the new entry */
684 tn.addend = addend - vaddr;
685 if (prot & PAGE_READ) {
686 tn.addr_read = address;
687 } else {
688 tn.addr_read = -1;
691 if (prot & PAGE_EXEC) {
692 tn.addr_code = code_address;
693 } else {
694 tn.addr_code = -1;
697 tn.addr_write = -1;
698 if (prot & PAGE_WRITE) {
699 if ((memory_region_is_ram(section->mr) && section->readonly)
700 || memory_region_is_romd(section->mr)) {
701 /* Write access calls the I/O callback. */
702 tn.addr_write = address | TLB_MMIO;
703 } else if (memory_region_is_ram(section->mr)
704 && cpu_physical_memory_is_clean(
705 memory_region_get_ram_addr(section->mr) + xlat)) {
706 tn.addr_write = address | TLB_NOTDIRTY;
707 } else {
708 tn.addr_write = address;
710 if (prot & PAGE_WRITE_INV) {
711 tn.addr_write |= TLB_INVALID_MASK;
715 /* Pairs with flag setting in tlb_reset_dirty_range */
716 copy_tlb_helper(te, &tn, true);
717 /* atomic_mb_set(&te->addr_write, write_address); */
720 /* Add a new TLB entry, but without specifying the memory
721 * transaction attributes to be used.
723 void tlb_set_page(CPUState *cpu, target_ulong vaddr,
724 hwaddr paddr, int prot,
725 int mmu_idx, target_ulong size)
727 tlb_set_page_with_attrs(cpu, vaddr, paddr, MEMTXATTRS_UNSPECIFIED,
728 prot, mmu_idx, size);
731 static void report_bad_exec(CPUState *cpu, target_ulong addr)
733 /* Accidentally executing outside RAM or ROM is quite common for
734 * several user-error situations, so report it in a way that
735 * makes it clear that this isn't a QEMU bug and provide suggestions
736 * about what a user could do to fix things.
738 error_report("Trying to execute code outside RAM or ROM at 0x"
739 TARGET_FMT_lx, addr);
740 error_printf("This usually means one of the following happened:\n\n"
741 "(1) You told QEMU to execute a kernel for the wrong machine "
742 "type, and it crashed on startup (eg trying to run a "
743 "raspberry pi kernel on a versatilepb QEMU machine)\n"
744 "(2) You didn't give QEMU a kernel or BIOS filename at all, "
745 "and QEMU executed a ROM full of no-op instructions until "
746 "it fell off the end\n"
747 "(3) Your guest kernel has a bug and crashed by jumping "
748 "off into nowhere\n\n"
749 "This is almost always one of the first two, so check your "
750 "command line and that you are using the right type of kernel "
751 "for this machine.\n"
752 "If you think option (3) is likely then you can try debugging "
753 "your guest with the -d debug options; in particular "
754 "-d guest_errors will cause the log to include a dump of the "
755 "guest register state at this point.\n\n"
756 "Execution cannot continue; stopping here.\n\n");
758 /* Report also to the logs, with more detail including register dump */
759 qemu_log_mask(LOG_GUEST_ERROR, "qemu: fatal: Trying to execute code "
760 "outside RAM or ROM at 0x" TARGET_FMT_lx "\n", addr);
761 log_cpu_state_mask(LOG_GUEST_ERROR, cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
764 static inline ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
766 ram_addr_t ram_addr;
768 ram_addr = qemu_ram_addr_from_host(ptr);
769 if (ram_addr == RAM_ADDR_INVALID) {
770 error_report("Bad ram pointer %p", ptr);
771 abort();
773 return ram_addr;
776 static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry,
777 int mmu_idx,
778 target_ulong addr, uintptr_t retaddr, int size)
780 CPUState *cpu = ENV_GET_CPU(env);
781 hwaddr mr_offset;
782 MemoryRegionSection *section;
783 MemoryRegion *mr;
784 uint64_t val;
785 bool locked = false;
786 MemTxResult r;
788 section = iotlb_to_section(cpu, iotlbentry->addr, iotlbentry->attrs);
789 mr = section->mr;
790 mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;
791 cpu->mem_io_pc = retaddr;
792 if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu->can_do_io) {
793 cpu_io_recompile(cpu, retaddr);
796 cpu->mem_io_vaddr = addr;
798 if (mr->global_locking && !qemu_mutex_iothread_locked()) {
799 qemu_mutex_lock_iothread();
800 locked = true;
802 r = memory_region_dispatch_read(mr, mr_offset,
803 &val, size, iotlbentry->attrs);
804 if (r != MEMTX_OK) {
805 hwaddr physaddr = mr_offset +
806 section->offset_within_address_space -
807 section->offset_within_region;
809 cpu_transaction_failed(cpu, physaddr, addr, size, MMU_DATA_LOAD,
810 mmu_idx, iotlbentry->attrs, r, retaddr);
812 if (locked) {
813 qemu_mutex_unlock_iothread();
816 return val;
819 static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry,
820 int mmu_idx,
821 uint64_t val, target_ulong addr,
822 uintptr_t retaddr, int size)
824 CPUState *cpu = ENV_GET_CPU(env);
825 hwaddr mr_offset;
826 MemoryRegionSection *section;
827 MemoryRegion *mr;
828 bool locked = false;
829 MemTxResult r;
831 section = iotlb_to_section(cpu, iotlbentry->addr, iotlbentry->attrs);
832 mr = section->mr;
833 mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;
834 if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu->can_do_io) {
835 cpu_io_recompile(cpu, retaddr);
837 cpu->mem_io_vaddr = addr;
838 cpu->mem_io_pc = retaddr;
840 if (mr->global_locking && !qemu_mutex_iothread_locked()) {
841 qemu_mutex_lock_iothread();
842 locked = true;
844 r = memory_region_dispatch_write(mr, mr_offset,
845 val, size, iotlbentry->attrs);
846 if (r != MEMTX_OK) {
847 hwaddr physaddr = mr_offset +
848 section->offset_within_address_space -
849 section->offset_within_region;
851 cpu_transaction_failed(cpu, physaddr, addr, size, MMU_DATA_STORE,
852 mmu_idx, iotlbentry->attrs, r, retaddr);
854 if (locked) {
855 qemu_mutex_unlock_iothread();
859 /* Return true if ADDR is present in the victim tlb, and has been copied
860 back to the main tlb. */
861 static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index,
862 size_t elt_ofs, target_ulong page)
864 size_t vidx;
865 for (vidx = 0; vidx < CPU_VTLB_SIZE; ++vidx) {
866 CPUTLBEntry *vtlb = &env->tlb_v_table[mmu_idx][vidx];
867 target_ulong cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs);
869 if (cmp == page) {
870 /* Found entry in victim tlb, swap tlb and iotlb. */
871 CPUTLBEntry tmptlb, *tlb = &env->tlb_table[mmu_idx][index];
873 copy_tlb_helper(&tmptlb, tlb, false);
874 copy_tlb_helper(tlb, vtlb, true);
875 copy_tlb_helper(vtlb, &tmptlb, true);
877 CPUIOTLBEntry tmpio, *io = &env->iotlb[mmu_idx][index];
878 CPUIOTLBEntry *vio = &env->iotlb_v[mmu_idx][vidx];
879 tmpio = *io; *io = *vio; *vio = tmpio;
880 return true;
883 return false;
886 /* Macro to call the above, with local variables from the use context. */
887 #define VICTIM_TLB_HIT(TY, ADDR) \
888 victim_tlb_hit(env, mmu_idx, index, offsetof(CPUTLBEntry, TY), \
889 (ADDR) & TARGET_PAGE_MASK)
891 /* NOTE: this function can trigger an exception */
892 /* NOTE2: the returned address is not exactly the physical address: it
893 * is actually a ram_addr_t (in system mode; the user mode emulation
894 * version of this function returns a guest virtual address).
896 tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr)
898 int mmu_idx, index;
899 void *p;
900 MemoryRegion *mr;
901 MemoryRegionSection *section;
902 CPUState *cpu = ENV_GET_CPU(env);
903 CPUIOTLBEntry *iotlbentry;
904 hwaddr physaddr, mr_offset;
906 index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
907 mmu_idx = cpu_mmu_index(env, true);
908 if (unlikely(env->tlb_table[mmu_idx][index].addr_code !=
909 (addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK)))) {
910 if (!VICTIM_TLB_HIT(addr_read, addr)) {
911 tlb_fill(ENV_GET_CPU(env), addr, 0, MMU_INST_FETCH, mmu_idx, 0);
914 iotlbentry = &env->iotlb[mmu_idx][index];
915 section = iotlb_to_section(cpu, iotlbentry->addr, iotlbentry->attrs);
916 mr = section->mr;
917 if (memory_region_is_unassigned(mr)) {
918 qemu_mutex_lock_iothread();
919 if (memory_region_request_mmio_ptr(mr, addr)) {
920 qemu_mutex_unlock_iothread();
921 /* A MemoryRegion is potentially added so re-run the
922 * get_page_addr_code.
924 return get_page_addr_code(env, addr);
926 qemu_mutex_unlock_iothread();
928 /* Give the new-style cpu_transaction_failed() hook first chance
929 * to handle this.
930 * This is not the ideal place to detect and generate CPU
931 * exceptions for instruction fetch failure (for instance
932 * we don't know the length of the access that the CPU would
933 * use, and it would be better to go ahead and try the access
934 * and use the MemTXResult it produced). However it is the
935 * simplest place we have currently available for the check.
937 mr_offset = (iotlbentry->addr & TARGET_PAGE_MASK) + addr;
938 physaddr = mr_offset +
939 section->offset_within_address_space -
940 section->offset_within_region;
941 cpu_transaction_failed(cpu, physaddr, addr, 0, MMU_INST_FETCH, mmu_idx,
942 iotlbentry->attrs, MEMTX_DECODE_ERROR, 0);
944 cpu_unassigned_access(cpu, addr, false, true, 0, 4);
945 /* The CPU's unassigned access hook might have longjumped out
946 * with an exception. If it didn't (or there was no hook) then
947 * we can't proceed further.
949 report_bad_exec(cpu, addr);
950 exit(1);
952 p = (void *)((uintptr_t)addr + env->tlb_table[mmu_idx][index].addend);
953 return qemu_ram_addr_from_host_nofail(p);
956 /* Probe for whether the specified guest write access is permitted.
957 * If it is not permitted then an exception will be taken in the same
958 * way as if this were a real write access (and we will not return).
959 * Otherwise the function will return, and there will be a valid
960 * entry in the TLB for this access.
962 void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx,
963 uintptr_t retaddr)
965 int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
966 target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
968 if ((addr & TARGET_PAGE_MASK)
969 != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
970 /* TLB entry is for a different page */
971 if (!VICTIM_TLB_HIT(addr_write, addr)) {
972 tlb_fill(ENV_GET_CPU(env), addr, size, MMU_DATA_STORE,
973 mmu_idx, retaddr);
978 /* Probe for a read-modify-write atomic operation. Do not allow unaligned
979 * operations, or io operations to proceed. Return the host address. */
980 static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
981 TCGMemOpIdx oi, uintptr_t retaddr,
982 NotDirtyInfo *ndi)
984 size_t mmu_idx = get_mmuidx(oi);
985 size_t index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
986 CPUTLBEntry *tlbe = &env->tlb_table[mmu_idx][index];
987 target_ulong tlb_addr = tlbe->addr_write;
988 TCGMemOp mop = get_memop(oi);
989 int a_bits = get_alignment_bits(mop);
990 int s_bits = mop & MO_SIZE;
991 void *hostaddr;
993 /* Adjust the given return address. */
994 retaddr -= GETPC_ADJ;
996 /* Enforce guest required alignment. */
997 if (unlikely(a_bits > 0 && (addr & ((1 << a_bits) - 1)))) {
998 /* ??? Maybe indicate atomic op to cpu_unaligned_access */
999 cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
1000 mmu_idx, retaddr);
1003 /* Enforce qemu required alignment. */
1004 if (unlikely(addr & ((1 << s_bits) - 1))) {
1005 /* We get here if guest alignment was not requested,
1006 or was not enforced by cpu_unaligned_access above.
1007 We might widen the access and emulate, but for now
1008 mark an exception and exit the cpu loop. */
1009 goto stop_the_world;
1012 /* Check TLB entry and enforce page permissions. */
1013 if ((addr & TARGET_PAGE_MASK)
1014 != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1015 if (!VICTIM_TLB_HIT(addr_write, addr)) {
1016 tlb_fill(ENV_GET_CPU(env), addr, 1 << s_bits, MMU_DATA_STORE,
1017 mmu_idx, retaddr);
1019 tlb_addr = tlbe->addr_write & ~TLB_INVALID_MASK;
1022 /* Notice an IO access */
1023 if (unlikely(tlb_addr & TLB_MMIO)) {
1024 /* There's really nothing that can be done to
1025 support this apart from stop-the-world. */
1026 goto stop_the_world;
1029 /* Let the guest notice RMW on a write-only page. */
1030 if (unlikely(tlbe->addr_read != (tlb_addr & ~TLB_NOTDIRTY))) {
1031 tlb_fill(ENV_GET_CPU(env), addr, 1 << s_bits, MMU_DATA_LOAD,
1032 mmu_idx, retaddr);
1033 /* Since we don't support reads and writes to different addresses,
1034 and we do have the proper page loaded for write, this shouldn't
1035 ever return. But just in case, handle via stop-the-world. */
1036 goto stop_the_world;
1039 hostaddr = (void *)((uintptr_t)addr + tlbe->addend);
1041 ndi->active = false;
1042 if (unlikely(tlb_addr & TLB_NOTDIRTY)) {
1043 ndi->active = true;
1044 memory_notdirty_write_prepare(ndi, ENV_GET_CPU(env), addr,
1045 qemu_ram_addr_from_host_nofail(hostaddr),
1046 1 << s_bits);
1049 return hostaddr;
1051 stop_the_world:
1052 cpu_loop_exit_atomic(ENV_GET_CPU(env), retaddr);
1055 #ifdef TARGET_WORDS_BIGENDIAN
1056 # define TGT_BE(X) (X)
1057 # define TGT_LE(X) BSWAP(X)
1058 #else
1059 # define TGT_BE(X) BSWAP(X)
1060 # define TGT_LE(X) (X)
1061 #endif
1063 #define MMUSUFFIX _mmu
1065 #define DATA_SIZE 1
1066 #include "softmmu_template.h"
1068 #define DATA_SIZE 2
1069 #include "softmmu_template.h"
1071 #define DATA_SIZE 4
1072 #include "softmmu_template.h"
1074 #define DATA_SIZE 8
1075 #include "softmmu_template.h"
1077 /* First set of helpers allows passing in of OI and RETADDR. This makes
1078 them callable from other helpers. */
1080 #define EXTRA_ARGS , TCGMemOpIdx oi, uintptr_t retaddr
1081 #define ATOMIC_NAME(X) \
1082 HELPER(glue(glue(glue(atomic_ ## X, SUFFIX), END), _mmu))
1083 #define ATOMIC_MMU_DECLS NotDirtyInfo ndi
1084 #define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, oi, retaddr, &ndi)
1085 #define ATOMIC_MMU_CLEANUP \
1086 do { \
1087 if (unlikely(ndi.active)) { \
1088 memory_notdirty_write_complete(&ndi); \
1090 } while (0)
1092 #define DATA_SIZE 1
1093 #include "atomic_template.h"
1095 #define DATA_SIZE 2
1096 #include "atomic_template.h"
1098 #define DATA_SIZE 4
1099 #include "atomic_template.h"
1101 #ifdef CONFIG_ATOMIC64
1102 #define DATA_SIZE 8
1103 #include "atomic_template.h"
1104 #endif
1106 #ifdef CONFIG_ATOMIC128
1107 #define DATA_SIZE 16
1108 #include "atomic_template.h"
1109 #endif
1111 /* Second set of helpers are directly callable from TCG as helpers. */
1113 #undef EXTRA_ARGS
1114 #undef ATOMIC_NAME
1115 #undef ATOMIC_MMU_LOOKUP
1116 #define EXTRA_ARGS , TCGMemOpIdx oi
1117 #define ATOMIC_NAME(X) HELPER(glue(glue(atomic_ ## X, SUFFIX), END))
1118 #define ATOMIC_MMU_LOOKUP atomic_mmu_lookup(env, addr, oi, GETPC(), &ndi)
1120 #define DATA_SIZE 1
1121 #include "atomic_template.h"
1123 #define DATA_SIZE 2
1124 #include "atomic_template.h"
1126 #define DATA_SIZE 4
1127 #include "atomic_template.h"
1129 #ifdef CONFIG_ATOMIC64
1130 #define DATA_SIZE 8
1131 #include "atomic_template.h"
1132 #endif
1134 /* Code access functions. */
1136 #undef MMUSUFFIX
1137 #define MMUSUFFIX _cmmu
1138 #undef GETPC
1139 #define GETPC() ((uintptr_t)0)
1140 #define SOFTMMU_CODE_ACCESS
1142 #define DATA_SIZE 1
1143 #include "softmmu_template.h"
1145 #define DATA_SIZE 2
1146 #include "softmmu_template.h"
1148 #define DATA_SIZE 4
1149 #include "softmmu_template.h"
1151 #define DATA_SIZE 8
1152 #include "softmmu_template.h"