2 * Common CPU TLB handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
22 #include "exec/exec-all.h"
23 #include "exec/memory.h"
24 #include "exec/address-spaces.h"
25 #include "exec/cpu_ldst.h"
27 #include "exec/cputlb.h"
29 #include "exec/memory-internal.h"
30 #include "exec/ram_addr.h"
34 //#define DEBUG_TLB_CHECK
40 * If flush_global is true (the usual case), flush all tlb entries.
41 * If flush_global is false, flush (at least) all tlb entries not
44 * Since QEMU doesn't currently implement a global/not-global flag
45 * for tlb entries, at the moment tlb_flush() will also flush all
46 * tlb entries in the flush_global == false case. This is OK because
47 * CPU architectures generally permit an implementation to drop
48 * entries from the TLB at any time, so flushing more entries than
49 * required is only an efficiency issue, not a correctness issue.
51 void tlb_flush(CPUState
*cpu
, int flush_global
)
53 CPUArchState
*env
= cpu
->env_ptr
;
55 #if defined(DEBUG_TLB)
56 printf("tlb_flush:\n");
58 /* must reset current TB so that interrupts cannot modify the
59 links while we are modifying them */
60 cpu
->current_tb
= NULL
;
62 memset(env
->tlb_table
, -1, sizeof(env
->tlb_table
));
63 memset(env
->tlb_v_table
, -1, sizeof(env
->tlb_v_table
));
64 memset(cpu
->tb_jmp_cache
, 0, sizeof(cpu
->tb_jmp_cache
));
67 env
->tlb_flush_addr
= -1;
68 env
->tlb_flush_mask
= 0;
72 static inline void v_tlb_flush_by_mmuidx(CPUState
*cpu
, va_list argp
)
74 CPUArchState
*env
= cpu
->env_ptr
;
76 #if defined(DEBUG_TLB)
77 printf("tlb_flush_by_mmuidx:");
79 /* must reset current TB so that interrupts cannot modify the
80 links while we are modifying them */
81 cpu
->current_tb
= NULL
;
84 int mmu_idx
= va_arg(argp
, int);
90 #if defined(DEBUG_TLB)
91 printf(" %d", mmu_idx
);
94 memset(env
->tlb_table
[mmu_idx
], -1, sizeof(env
->tlb_table
[0]));
95 memset(env
->tlb_v_table
[mmu_idx
], -1, sizeof(env
->tlb_v_table
[0]));
98 #if defined(DEBUG_TLB)
102 memset(cpu
->tb_jmp_cache
, 0, sizeof(cpu
->tb_jmp_cache
));
105 void tlb_flush_by_mmuidx(CPUState
*cpu
, ...)
109 v_tlb_flush_by_mmuidx(cpu
, argp
);
113 static inline void tlb_flush_entry(CPUTLBEntry
*tlb_entry
, target_ulong addr
)
115 if (addr
== (tlb_entry
->addr_read
&
116 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
117 addr
== (tlb_entry
->addr_write
&
118 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
119 addr
== (tlb_entry
->addr_code
&
120 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
))) {
121 memset(tlb_entry
, -1, sizeof(*tlb_entry
));
125 void tlb_flush_page(CPUState
*cpu
, target_ulong addr
)
127 CPUArchState
*env
= cpu
->env_ptr
;
131 #if defined(DEBUG_TLB)
132 printf("tlb_flush_page: " TARGET_FMT_lx
"\n", addr
);
134 /* Check if we need to flush due to large pages. */
135 if ((addr
& env
->tlb_flush_mask
) == env
->tlb_flush_addr
) {
136 #if defined(DEBUG_TLB)
137 printf("tlb_flush_page: forced full flush ("
138 TARGET_FMT_lx
"/" TARGET_FMT_lx
")\n",
139 env
->tlb_flush_addr
, env
->tlb_flush_mask
);
144 /* must reset current TB so that interrupts cannot modify the
145 links while we are modifying them */
146 cpu
->current_tb
= NULL
;
148 addr
&= TARGET_PAGE_MASK
;
149 i
= (addr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
150 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
151 tlb_flush_entry(&env
->tlb_table
[mmu_idx
][i
], addr
);
154 /* check whether there are entries that need to be flushed in the vtlb */
155 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
157 for (k
= 0; k
< CPU_VTLB_SIZE
; k
++) {
158 tlb_flush_entry(&env
->tlb_v_table
[mmu_idx
][k
], addr
);
162 tb_flush_jmp_cache(cpu
, addr
);
165 void tlb_flush_page_by_mmuidx(CPUState
*cpu
, target_ulong addr
, ...)
167 CPUArchState
*env
= cpu
->env_ptr
;
171 va_start(argp
, addr
);
173 #if defined(DEBUG_TLB)
174 printf("tlb_flush_page_by_mmu_idx: " TARGET_FMT_lx
, addr
);
176 /* Check if we need to flush due to large pages. */
177 if ((addr
& env
->tlb_flush_mask
) == env
->tlb_flush_addr
) {
178 #if defined(DEBUG_TLB)
179 printf(" forced full flush ("
180 TARGET_FMT_lx
"/" TARGET_FMT_lx
")\n",
181 env
->tlb_flush_addr
, env
->tlb_flush_mask
);
183 v_tlb_flush_by_mmuidx(cpu
, argp
);
187 /* must reset current TB so that interrupts cannot modify the
188 links while we are modifying them */
189 cpu
->current_tb
= NULL
;
191 addr
&= TARGET_PAGE_MASK
;
192 i
= (addr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
195 int mmu_idx
= va_arg(argp
, int);
201 #if defined(DEBUG_TLB)
202 printf(" %d", mmu_idx
);
205 tlb_flush_entry(&env
->tlb_table
[mmu_idx
][i
], addr
);
207 /* check whether there are vltb entries that need to be flushed */
208 for (k
= 0; k
< CPU_VTLB_SIZE
; k
++) {
209 tlb_flush_entry(&env
->tlb_v_table
[mmu_idx
][k
], addr
);
214 #if defined(DEBUG_TLB)
218 tb_flush_jmp_cache(cpu
, addr
);
221 /* update the TLBs so that writes to code in the virtual page 'addr'
223 void tlb_protect_code(ram_addr_t ram_addr
)
225 cpu_physical_memory_test_and_clear_dirty(ram_addr
, TARGET_PAGE_SIZE
,
229 /* update the TLB so that writes in physical page 'phys_addr' are no longer
230 tested for self modifying code */
231 void tlb_unprotect_code(ram_addr_t ram_addr
)
233 cpu_physical_memory_set_dirty_flag(ram_addr
, DIRTY_MEMORY_CODE
);
236 static bool tlb_is_dirty_ram(CPUTLBEntry
*tlbe
)
238 return (tlbe
->addr_write
& (TLB_INVALID_MASK
|TLB_MMIO
|TLB_NOTDIRTY
)) == 0;
241 void tlb_reset_dirty_range(CPUTLBEntry
*tlb_entry
, uintptr_t start
,
246 if (tlb_is_dirty_ram(tlb_entry
)) {
247 addr
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) + tlb_entry
->addend
;
248 if ((addr
- start
) < length
) {
249 tlb_entry
->addr_write
|= TLB_NOTDIRTY
;
254 static inline ram_addr_t
qemu_ram_addr_from_host_nofail(void *ptr
)
258 if (qemu_ram_addr_from_host(ptr
, &ram_addr
) == NULL
) {
259 fprintf(stderr
, "Bad ram pointer %p\n", ptr
);
265 void tlb_reset_dirty(CPUState
*cpu
, ram_addr_t start1
, ram_addr_t length
)
272 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
275 for (i
= 0; i
< CPU_TLB_SIZE
; i
++) {
276 tlb_reset_dirty_range(&env
->tlb_table
[mmu_idx
][i
],
280 for (i
= 0; i
< CPU_VTLB_SIZE
; i
++) {
281 tlb_reset_dirty_range(&env
->tlb_v_table
[mmu_idx
][i
],
287 static inline void tlb_set_dirty1(CPUTLBEntry
*tlb_entry
, target_ulong vaddr
)
289 if (tlb_entry
->addr_write
== (vaddr
| TLB_NOTDIRTY
)) {
290 tlb_entry
->addr_write
= vaddr
;
294 /* update the TLB corresponding to virtual page vaddr
295 so that it is no longer dirty */
296 void tlb_set_dirty(CPUState
*cpu
, target_ulong vaddr
)
298 CPUArchState
*env
= cpu
->env_ptr
;
302 vaddr
&= TARGET_PAGE_MASK
;
303 i
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
304 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
305 tlb_set_dirty1(&env
->tlb_table
[mmu_idx
][i
], vaddr
);
308 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
310 for (k
= 0; k
< CPU_VTLB_SIZE
; k
++) {
311 tlb_set_dirty1(&env
->tlb_v_table
[mmu_idx
][k
], vaddr
);
316 /* Our TLB does not support large pages, so remember the area covered by
317 large pages and trigger a full TLB flush if these are invalidated. */
318 static void tlb_add_large_page(CPUArchState
*env
, target_ulong vaddr
,
321 target_ulong mask
= ~(size
- 1);
323 if (env
->tlb_flush_addr
== (target_ulong
)-1) {
324 env
->tlb_flush_addr
= vaddr
& mask
;
325 env
->tlb_flush_mask
= mask
;
328 /* Extend the existing region to include the new page.
329 This is a compromise between unnecessary flushes and the cost
330 of maintaining a full variable size TLB. */
331 mask
&= env
->tlb_flush_mask
;
332 while (((env
->tlb_flush_addr
^ vaddr
) & mask
) != 0) {
335 env
->tlb_flush_addr
&= mask
;
336 env
->tlb_flush_mask
= mask
;
339 /* Add a new TLB entry. At most one entry for a given virtual address
340 * is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
341 * supplied size is only used by tlb_flush_page.
343 * Called from TCG-generated code, which is under an RCU read-side
346 void tlb_set_page_with_attrs(CPUState
*cpu
, target_ulong vaddr
,
347 hwaddr paddr
, MemTxAttrs attrs
, int prot
,
348 int mmu_idx
, target_ulong size
)
350 CPUArchState
*env
= cpu
->env_ptr
;
351 MemoryRegionSection
*section
;
353 target_ulong address
;
354 target_ulong code_address
;
357 hwaddr iotlb
, xlat
, sz
;
358 unsigned vidx
= env
->vtlb_index
++ % CPU_VTLB_SIZE
;
359 int asidx
= cpu_asidx_from_attrs(cpu
, attrs
);
361 assert(size
>= TARGET_PAGE_SIZE
);
362 if (size
!= TARGET_PAGE_SIZE
) {
363 tlb_add_large_page(env
, vaddr
, size
);
367 section
= address_space_translate_for_iotlb(cpu
, asidx
, paddr
, &xlat
, &sz
);
368 assert(sz
>= TARGET_PAGE_SIZE
);
370 #if defined(DEBUG_TLB)
371 qemu_log_mask(CPU_LOG_MMU
,
372 "tlb_set_page: vaddr=" TARGET_FMT_lx
" paddr=0x" TARGET_FMT_plx
374 vaddr
, paddr
, prot
, mmu_idx
);
378 if (!memory_region_is_ram(section
->mr
) && !memory_region_is_romd(section
->mr
)) {
383 /* TLB_MMIO for rom/romd handled below */
384 addend
= (uintptr_t)memory_region_get_ram_ptr(section
->mr
) + xlat
;
387 code_address
= address
;
388 iotlb
= memory_region_section_get_iotlb(cpu
, section
, vaddr
, paddr
, xlat
,
391 index
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
392 te
= &env
->tlb_table
[mmu_idx
][index
];
394 /* do not discard the translation in te, evict it into a victim tlb */
395 env
->tlb_v_table
[mmu_idx
][vidx
] = *te
;
396 env
->iotlb_v
[mmu_idx
][vidx
] = env
->iotlb
[mmu_idx
][index
];
399 env
->iotlb
[mmu_idx
][index
].addr
= iotlb
- vaddr
;
400 env
->iotlb
[mmu_idx
][index
].attrs
= attrs
;
401 te
->addend
= addend
- vaddr
;
402 if (prot
& PAGE_READ
) {
403 te
->addr_read
= address
;
408 if (prot
& PAGE_EXEC
) {
409 te
->addr_code
= code_address
;
413 if (prot
& PAGE_WRITE
) {
414 if ((memory_region_is_ram(section
->mr
) && section
->readonly
)
415 || memory_region_is_romd(section
->mr
)) {
416 /* Write access calls the I/O callback. */
417 te
->addr_write
= address
| TLB_MMIO
;
418 } else if (memory_region_is_ram(section
->mr
)
419 && cpu_physical_memory_is_clean(section
->mr
->ram_addr
421 te
->addr_write
= address
| TLB_NOTDIRTY
;
423 te
->addr_write
= address
;
430 /* Add a new TLB entry, but without specifying the memory
431 * transaction attributes to be used.
433 void tlb_set_page(CPUState
*cpu
, target_ulong vaddr
,
434 hwaddr paddr
, int prot
,
435 int mmu_idx
, target_ulong size
)
437 tlb_set_page_with_attrs(cpu
, vaddr
, paddr
, MEMTXATTRS_UNSPECIFIED
,
438 prot
, mmu_idx
, size
);
441 /* NOTE: this function can trigger an exception */
442 /* NOTE2: the returned address is not exactly the physical address: it
443 * is actually a ram_addr_t (in system mode; the user mode emulation
444 * version of this function returns a guest virtual address).
446 tb_page_addr_t
get_page_addr_code(CPUArchState
*env1
, target_ulong addr
)
448 int mmu_idx
, page_index
, pd
;
451 CPUState
*cpu
= ENV_GET_CPU(env1
);
452 CPUIOTLBEntry
*iotlbentry
;
454 page_index
= (addr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
455 mmu_idx
= cpu_mmu_index(env1
, true);
456 if (unlikely(env1
->tlb_table
[mmu_idx
][page_index
].addr_code
!=
457 (addr
& TARGET_PAGE_MASK
))) {
458 cpu_ldub_code(env1
, addr
);
460 iotlbentry
= &env1
->iotlb
[mmu_idx
][page_index
];
461 pd
= iotlbentry
->addr
& ~TARGET_PAGE_MASK
;
462 mr
= iotlb_to_region(cpu
, pd
, iotlbentry
->attrs
);
463 if (memory_region_is_unassigned(mr
)) {
464 CPUClass
*cc
= CPU_GET_CLASS(cpu
);
466 if (cc
->do_unassigned_access
) {
467 cc
->do_unassigned_access(cpu
, addr
, false, true, 0, 4);
469 cpu_abort(cpu
, "Trying to execute code outside RAM or ROM at 0x"
470 TARGET_FMT_lx
"\n", addr
);
473 p
= (void *)((uintptr_t)addr
+ env1
->tlb_table
[mmu_idx
][page_index
].addend
);
474 return qemu_ram_addr_from_host_nofail(p
);
477 #define MMUSUFFIX _mmu
480 #include "softmmu_template.h"
483 #include "softmmu_template.h"
486 #include "softmmu_template.h"
489 #include "softmmu_template.h"
492 #define MMUSUFFIX _cmmu
496 #define GETRA() ((uintptr_t)0)
497 #define SOFTMMU_CODE_ACCESS
500 #include "softmmu_template.h"
503 #include "softmmu_template.h"
506 #include "softmmu_template.h"
509 #include "softmmu_template.h"