2 * virtual page mapping and translated block handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
23 #include <sys/types.h>
27 #include "qemu-common.h"
35 #include "qemu-timer.h"
37 #include "exec-memory.h"
38 #if defined(CONFIG_USER_ONLY)
40 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
41 #include <sys/param.h>
42 #if __FreeBSD_version >= 700104
43 #define HAVE_KINFO_GETVMMAP
44 #define sigqueue sigqueue_freebsd /* avoid redefinition */
47 #include <machine/profile.h>
55 #else /* !CONFIG_USER_ONLY */
56 #include "xen-mapcache.h"
60 #define WANT_EXEC_OBSOLETE
61 #include "exec-obsolete.h"
63 //#define DEBUG_TB_INVALIDATE
66 //#define DEBUG_UNASSIGNED
68 /* make various TB consistency checks */
69 //#define DEBUG_TB_CHECK
70 //#define DEBUG_TLB_CHECK
72 //#define DEBUG_IOPORT
73 //#define DEBUG_SUBPAGE
75 #if !defined(CONFIG_USER_ONLY)
76 /* TB consistency checks only implemented for usermode emulation. */
80 #define SMC_BITMAP_USE_THRESHOLD 10
82 static TranslationBlock
*tbs
;
83 static int code_gen_max_blocks
;
84 TranslationBlock
*tb_phys_hash
[CODE_GEN_PHYS_HASH_SIZE
];
86 /* any access to the tbs or the page table must use this lock */
87 spinlock_t tb_lock
= SPIN_LOCK_UNLOCKED
;
89 #if defined(__arm__) || defined(__sparc_v9__)
90 /* The prologue must be reachable with a direct jump. ARM and Sparc64
91 have limited branch ranges (possibly also PPC) so place it in a
92 section close to code segment. */
93 #define code_gen_section \
94 __attribute__((__section__(".gen_code"))) \
95 __attribute__((aligned (32)))
97 /* Maximum alignment for Win32 is 16. */
98 #define code_gen_section \
99 __attribute__((aligned (16)))
101 #define code_gen_section \
102 __attribute__((aligned (32)))
105 uint8_t code_gen_prologue
[1024] code_gen_section
;
106 static uint8_t *code_gen_buffer
;
107 static unsigned long code_gen_buffer_size
;
108 /* threshold to flush the translated code buffer */
109 static unsigned long code_gen_buffer_max_size
;
110 static uint8_t *code_gen_ptr
;
112 #if !defined(CONFIG_USER_ONLY)
114 static int in_migration
;
116 RAMList ram_list
= { .blocks
= QLIST_HEAD_INITIALIZER(ram_list
.blocks
) };
118 static MemoryRegion
*system_memory
;
119 static MemoryRegion
*system_io
;
121 MemoryRegion io_mem_ram
, io_mem_rom
, io_mem_unassigned
, io_mem_notdirty
;
122 static MemoryRegion io_mem_subpage_ram
;
127 /* current CPU in the current thread. It is only valid inside
129 DEFINE_TLS(CPUState
*,cpu_single_env
);
130 /* 0 = Do not count executed instructions.
131 1 = Precise instruction counting.
132 2 = Adaptive rate instruction counting. */
135 typedef struct PageDesc
{
136 /* list of TBs intersecting this ram page */
137 TranslationBlock
*first_tb
;
138 /* in order to optimize self modifying code, we count the number
139 of lookups we do to a given page to use a bitmap */
140 unsigned int code_write_count
;
141 uint8_t *code_bitmap
;
142 #if defined(CONFIG_USER_ONLY)
147 /* In system mode we want L1_MAP to be based on ram offsets,
148 while in user mode we want it to be based on virtual addresses. */
149 #if !defined(CONFIG_USER_ONLY)
150 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
151 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
153 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
156 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
159 /* Size of the L2 (and L3, etc) page tables. */
161 #define L2_SIZE (1 << L2_BITS)
163 /* The bits remaining after N lower levels of page tables. */
164 #define P_L1_BITS_REM \
165 ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
166 #define V_L1_BITS_REM \
167 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
169 /* Size of the L1 page table. Avoid silly small sizes. */
170 #if P_L1_BITS_REM < 4
171 #define P_L1_BITS (P_L1_BITS_REM + L2_BITS)
173 #define P_L1_BITS P_L1_BITS_REM
176 #if V_L1_BITS_REM < 4
177 #define V_L1_BITS (V_L1_BITS_REM + L2_BITS)
179 #define V_L1_BITS V_L1_BITS_REM
182 #define P_L1_SIZE ((target_phys_addr_t)1 << P_L1_BITS)
183 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
185 #define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
186 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
188 unsigned long qemu_real_host_page_size
;
189 unsigned long qemu_host_page_size
;
190 unsigned long qemu_host_page_mask
;
192 /* This is a multi-level map on the virtual address space.
193 The bottom level has pointers to PageDesc. */
194 static void *l1_map
[V_L1_SIZE
];
196 #if !defined(CONFIG_USER_ONLY)
197 typedef struct PhysPageDesc
{
198 /* offset in host memory of the page + io_index in the low bits */
199 ram_addr_t phys_offset
;
200 ram_addr_t region_offset
;
203 /* This is a multi-level map on the physical address space.
204 The bottom level has pointers to PhysPageDesc. */
205 static void *l1_phys_map
[P_L1_SIZE
];
207 static void io_mem_init(void);
208 static void memory_map_init(void);
210 /* io memory support */
211 MemoryRegion
*io_mem_region
[IO_MEM_NB_ENTRIES
];
212 static char io_mem_used
[IO_MEM_NB_ENTRIES
];
213 static MemoryRegion io_mem_watch
;
218 static const char *logfilename
= "qemu.log";
220 static const char *logfilename
= "/tmp/qemu.log";
224 static int log_append
= 0;
227 #if !defined(CONFIG_USER_ONLY)
228 static int tlb_flush_count
;
230 static int tb_flush_count
;
231 static int tb_phys_invalidate_count
;
234 static void map_exec(void *addr
, long size
)
237 VirtualProtect(addr
, size
,
238 PAGE_EXECUTE_READWRITE
, &old_protect
);
242 static void map_exec(void *addr
, long size
)
244 unsigned long start
, end
, page_size
;
246 page_size
= getpagesize();
247 start
= (unsigned long)addr
;
248 start
&= ~(page_size
- 1);
250 end
= (unsigned long)addr
+ size
;
251 end
+= page_size
- 1;
252 end
&= ~(page_size
- 1);
254 mprotect((void *)start
, end
- start
,
255 PROT_READ
| PROT_WRITE
| PROT_EXEC
);
259 static void page_init(void)
261 /* NOTE: we can always suppose that qemu_host_page_size >=
265 SYSTEM_INFO system_info
;
267 GetSystemInfo(&system_info
);
268 qemu_real_host_page_size
= system_info
.dwPageSize
;
271 qemu_real_host_page_size
= getpagesize();
273 if (qemu_host_page_size
== 0)
274 qemu_host_page_size
= qemu_real_host_page_size
;
275 if (qemu_host_page_size
< TARGET_PAGE_SIZE
)
276 qemu_host_page_size
= TARGET_PAGE_SIZE
;
277 qemu_host_page_mask
= ~(qemu_host_page_size
- 1);
279 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
281 #ifdef HAVE_KINFO_GETVMMAP
282 struct kinfo_vmentry
*freep
;
285 freep
= kinfo_getvmmap(getpid(), &cnt
);
288 for (i
= 0; i
< cnt
; i
++) {
289 unsigned long startaddr
, endaddr
;
291 startaddr
= freep
[i
].kve_start
;
292 endaddr
= freep
[i
].kve_end
;
293 if (h2g_valid(startaddr
)) {
294 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
296 if (h2g_valid(endaddr
)) {
297 endaddr
= h2g(endaddr
);
298 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
300 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
302 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
313 last_brk
= (unsigned long)sbrk(0);
315 f
= fopen("/compat/linux/proc/self/maps", "r");
320 unsigned long startaddr
, endaddr
;
323 n
= fscanf (f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
325 if (n
== 2 && h2g_valid(startaddr
)) {
326 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
328 if (h2g_valid(endaddr
)) {
329 endaddr
= h2g(endaddr
);
333 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
345 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
351 #if defined(CONFIG_USER_ONLY)
352 /* We can't use g_malloc because it may recurse into a locked mutex. */
353 # define ALLOC(P, SIZE) \
355 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
356 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
359 # define ALLOC(P, SIZE) \
360 do { P = g_malloc0(SIZE); } while (0)
363 /* Level 1. Always allocated. */
364 lp
= l1_map
+ ((index
>> V_L1_SHIFT
) & (V_L1_SIZE
- 1));
367 for (i
= V_L1_SHIFT
/ L2_BITS
- 1; i
> 0; i
--) {
374 ALLOC(p
, sizeof(void *) * L2_SIZE
);
378 lp
= p
+ ((index
>> (i
* L2_BITS
)) & (L2_SIZE
- 1));
386 ALLOC(pd
, sizeof(PageDesc
) * L2_SIZE
);
392 return pd
+ (index
& (L2_SIZE
- 1));
395 static inline PageDesc
*page_find(tb_page_addr_t index
)
397 return page_find_alloc(index
, 0);
400 #if !defined(CONFIG_USER_ONLY)
401 static PhysPageDesc
*phys_page_find_alloc(target_phys_addr_t index
, int alloc
)
407 /* Level 1. Always allocated. */
408 lp
= l1_phys_map
+ ((index
>> P_L1_SHIFT
) & (P_L1_SIZE
- 1));
411 for (i
= P_L1_SHIFT
/ L2_BITS
- 1; i
> 0; i
--) {
417 *lp
= p
= g_malloc0(sizeof(void *) * L2_SIZE
);
419 lp
= p
+ ((index
>> (i
* L2_BITS
)) & (L2_SIZE
- 1));
425 int first_index
= index
& ~(L2_SIZE
- 1);
431 *lp
= pd
= g_malloc(sizeof(PhysPageDesc
) * L2_SIZE
);
433 for (i
= 0; i
< L2_SIZE
; i
++) {
434 pd
[i
].phys_offset
= io_mem_unassigned
.ram_addr
;
435 pd
[i
].region_offset
= (first_index
+ i
) << TARGET_PAGE_BITS
;
439 return pd
+ (index
& (L2_SIZE
- 1));
442 static inline PhysPageDesc
phys_page_find(target_phys_addr_t index
)
444 PhysPageDesc
*p
= phys_page_find_alloc(index
, 0);
449 return (PhysPageDesc
) {
450 .phys_offset
= io_mem_unassigned
.ram_addr
,
451 .region_offset
= index
<< TARGET_PAGE_BITS
,
456 static void tlb_protect_code(ram_addr_t ram_addr
);
457 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
459 #define mmap_lock() do { } while(0)
460 #define mmap_unlock() do { } while(0)
463 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
465 #if defined(CONFIG_USER_ONLY)
466 /* Currently it is not recommended to allocate big chunks of data in
467 user mode. It will change when a dedicated libc will be used */
468 #define USE_STATIC_CODE_GEN_BUFFER
471 #ifdef USE_STATIC_CODE_GEN_BUFFER
472 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
]
473 __attribute__((aligned (CODE_GEN_ALIGN
)));
476 static void code_gen_alloc(unsigned long tb_size
)
478 #ifdef USE_STATIC_CODE_GEN_BUFFER
479 code_gen_buffer
= static_code_gen_buffer
;
480 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
481 map_exec(code_gen_buffer
, code_gen_buffer_size
);
483 code_gen_buffer_size
= tb_size
;
484 if (code_gen_buffer_size
== 0) {
485 #if defined(CONFIG_USER_ONLY)
486 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
488 /* XXX: needs adjustments */
489 code_gen_buffer_size
= (unsigned long)(ram_size
/ 4);
492 if (code_gen_buffer_size
< MIN_CODE_GEN_BUFFER_SIZE
)
493 code_gen_buffer_size
= MIN_CODE_GEN_BUFFER_SIZE
;
494 /* The code gen buffer location may have constraints depending on
495 the host cpu and OS */
496 #if defined(__linux__)
501 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
502 #if defined(__x86_64__)
504 /* Cannot map more than that */
505 if (code_gen_buffer_size
> (800 * 1024 * 1024))
506 code_gen_buffer_size
= (800 * 1024 * 1024);
507 #elif defined(__sparc_v9__)
508 // Map the buffer below 2G, so we can use direct calls and branches
510 start
= (void *) 0x60000000UL
;
511 if (code_gen_buffer_size
> (512 * 1024 * 1024))
512 code_gen_buffer_size
= (512 * 1024 * 1024);
513 #elif defined(__arm__)
514 /* Keep the buffer no bigger than 16GB to branch between blocks */
515 if (code_gen_buffer_size
> 16 * 1024 * 1024)
516 code_gen_buffer_size
= 16 * 1024 * 1024;
517 #elif defined(__s390x__)
518 /* Map the buffer so that we can use direct calls and branches. */
519 /* We have a +- 4GB range on the branches; leave some slop. */
520 if (code_gen_buffer_size
> (3ul * 1024 * 1024 * 1024)) {
521 code_gen_buffer_size
= 3ul * 1024 * 1024 * 1024;
523 start
= (void *)0x90000000UL
;
525 code_gen_buffer
= mmap(start
, code_gen_buffer_size
,
526 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
528 if (code_gen_buffer
== MAP_FAILED
) {
529 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
533 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
534 || defined(__DragonFly__) || defined(__OpenBSD__) \
535 || defined(__NetBSD__)
539 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
540 #if defined(__x86_64__)
541 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
542 * 0x40000000 is free */
544 addr
= (void *)0x40000000;
545 /* Cannot map more than that */
546 if (code_gen_buffer_size
> (800 * 1024 * 1024))
547 code_gen_buffer_size
= (800 * 1024 * 1024);
548 #elif defined(__sparc_v9__)
549 // Map the buffer below 2G, so we can use direct calls and branches
551 addr
= (void *) 0x60000000UL
;
552 if (code_gen_buffer_size
> (512 * 1024 * 1024)) {
553 code_gen_buffer_size
= (512 * 1024 * 1024);
556 code_gen_buffer
= mmap(addr
, code_gen_buffer_size
,
557 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
559 if (code_gen_buffer
== MAP_FAILED
) {
560 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
565 code_gen_buffer
= g_malloc(code_gen_buffer_size
);
566 map_exec(code_gen_buffer
, code_gen_buffer_size
);
568 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
569 map_exec(code_gen_prologue
, sizeof(code_gen_prologue
));
570 code_gen_buffer_max_size
= code_gen_buffer_size
-
571 (TCG_MAX_OP_SIZE
* OPC_BUF_SIZE
);
572 code_gen_max_blocks
= code_gen_buffer_size
/ CODE_GEN_AVG_BLOCK_SIZE
;
573 tbs
= g_malloc(code_gen_max_blocks
* sizeof(TranslationBlock
));
576 /* Must be called before using the QEMU cpus. 'tb_size' is the size
577 (in bytes) allocated to the translation buffer. Zero means default
579 void tcg_exec_init(unsigned long tb_size
)
582 code_gen_alloc(tb_size
);
583 code_gen_ptr
= code_gen_buffer
;
585 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
586 /* There's no guest base to take into account, so go ahead and
587 initialize the prologue now. */
588 tcg_prologue_init(&tcg_ctx
);
592 bool tcg_enabled(void)
594 return code_gen_buffer
!= NULL
;
597 void cpu_exec_init_all(void)
599 #if !defined(CONFIG_USER_ONLY)
605 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
607 static int cpu_common_post_load(void *opaque
, int version_id
)
609 CPUState
*env
= opaque
;
611 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
612 version_id is increased. */
613 env
->interrupt_request
&= ~0x01;
619 static const VMStateDescription vmstate_cpu_common
= {
620 .name
= "cpu_common",
622 .minimum_version_id
= 1,
623 .minimum_version_id_old
= 1,
624 .post_load
= cpu_common_post_load
,
625 .fields
= (VMStateField
[]) {
626 VMSTATE_UINT32(halted
, CPUState
),
627 VMSTATE_UINT32(interrupt_request
, CPUState
),
628 VMSTATE_END_OF_LIST()
633 CPUState
*qemu_get_cpu(int cpu
)
635 CPUState
*env
= first_cpu
;
638 if (env
->cpu_index
== cpu
)
646 void cpu_exec_init(CPUState
*env
)
651 #if defined(CONFIG_USER_ONLY)
654 env
->next_cpu
= NULL
;
657 while (*penv
!= NULL
) {
658 penv
= &(*penv
)->next_cpu
;
661 env
->cpu_index
= cpu_index
;
663 QTAILQ_INIT(&env
->breakpoints
);
664 QTAILQ_INIT(&env
->watchpoints
);
665 #ifndef CONFIG_USER_ONLY
666 env
->thread_id
= qemu_get_thread_id();
669 #if defined(CONFIG_USER_ONLY)
672 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
673 vmstate_register(NULL
, cpu_index
, &vmstate_cpu_common
, env
);
674 register_savevm(NULL
, "cpu", cpu_index
, CPU_SAVE_VERSION
,
675 cpu_save
, cpu_load
, env
);
679 /* Allocate a new translation block. Flush the translation buffer if
680 too many translation blocks or too much generated code. */
681 static TranslationBlock
*tb_alloc(target_ulong pc
)
683 TranslationBlock
*tb
;
685 if (nb_tbs
>= code_gen_max_blocks
||
686 (code_gen_ptr
- code_gen_buffer
) >= code_gen_buffer_max_size
)
694 void tb_free(TranslationBlock
*tb
)
696 /* In practice this is mostly used for single use temporary TB
697 Ignore the hard cases and just back up if this TB happens to
698 be the last one generated. */
699 if (nb_tbs
> 0 && tb
== &tbs
[nb_tbs
- 1]) {
700 code_gen_ptr
= tb
->tc_ptr
;
705 static inline void invalidate_page_bitmap(PageDesc
*p
)
707 if (p
->code_bitmap
) {
708 g_free(p
->code_bitmap
);
709 p
->code_bitmap
= NULL
;
711 p
->code_write_count
= 0;
714 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
716 static void page_flush_tb_1 (int level
, void **lp
)
725 for (i
= 0; i
< L2_SIZE
; ++i
) {
726 pd
[i
].first_tb
= NULL
;
727 invalidate_page_bitmap(pd
+ i
);
731 for (i
= 0; i
< L2_SIZE
; ++i
) {
732 page_flush_tb_1 (level
- 1, pp
+ i
);
737 static void page_flush_tb(void)
740 for (i
= 0; i
< V_L1_SIZE
; i
++) {
741 page_flush_tb_1(V_L1_SHIFT
/ L2_BITS
- 1, l1_map
+ i
);
745 /* flush all the translation blocks */
746 /* XXX: tb_flush is currently not thread safe */
747 void tb_flush(CPUState
*env1
)
750 #if defined(DEBUG_FLUSH)
751 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
752 (unsigned long)(code_gen_ptr
- code_gen_buffer
),
754 ((unsigned long)(code_gen_ptr
- code_gen_buffer
)) / nb_tbs
: 0);
756 if ((unsigned long)(code_gen_ptr
- code_gen_buffer
) > code_gen_buffer_size
)
757 cpu_abort(env1
, "Internal error: code buffer overflow\n");
761 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
762 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
765 memset (tb_phys_hash
, 0, CODE_GEN_PHYS_HASH_SIZE
* sizeof (void *));
768 code_gen_ptr
= code_gen_buffer
;
769 /* XXX: flush processor icache at this point if cache flush is
774 #ifdef DEBUG_TB_CHECK
776 static void tb_invalidate_check(target_ulong address
)
778 TranslationBlock
*tb
;
780 address
&= TARGET_PAGE_MASK
;
781 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
782 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
783 if (!(address
+ TARGET_PAGE_SIZE
<= tb
->pc
||
784 address
>= tb
->pc
+ tb
->size
)) {
785 printf("ERROR invalidate: address=" TARGET_FMT_lx
786 " PC=%08lx size=%04x\n",
787 address
, (long)tb
->pc
, tb
->size
);
793 /* verify that all the pages have correct rights for code */
794 static void tb_page_check(void)
796 TranslationBlock
*tb
;
797 int i
, flags1
, flags2
;
799 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
800 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
801 flags1
= page_get_flags(tb
->pc
);
802 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
803 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
804 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
805 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
813 /* invalidate one TB */
814 static inline void tb_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
,
817 TranslationBlock
*tb1
;
821 *ptb
= *(TranslationBlock
**)((char *)tb1
+ next_offset
);
824 ptb
= (TranslationBlock
**)((char *)tb1
+ next_offset
);
828 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
830 TranslationBlock
*tb1
;
836 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
838 *ptb
= tb1
->page_next
[n1
];
841 ptb
= &tb1
->page_next
[n1
];
845 static inline void tb_jmp_remove(TranslationBlock
*tb
, int n
)
847 TranslationBlock
*tb1
, **ptb
;
850 ptb
= &tb
->jmp_next
[n
];
853 /* find tb(n) in circular list */
857 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
858 if (n1
== n
&& tb1
== tb
)
861 ptb
= &tb1
->jmp_first
;
863 ptb
= &tb1
->jmp_next
[n1
];
866 /* now we can suppress tb(n) from the list */
867 *ptb
= tb
->jmp_next
[n
];
869 tb
->jmp_next
[n
] = NULL
;
873 /* reset the jump entry 'n' of a TB so that it is not chained to
875 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
877 tb_set_jmp_target(tb
, n
, (unsigned long)(tb
->tc_ptr
+ tb
->tb_next_offset
[n
]));
880 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
885 tb_page_addr_t phys_pc
;
886 TranslationBlock
*tb1
, *tb2
;
888 /* remove the TB from the hash list */
889 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
890 h
= tb_phys_hash_func(phys_pc
);
891 tb_remove(&tb_phys_hash
[h
], tb
,
892 offsetof(TranslationBlock
, phys_hash_next
));
894 /* remove the TB from the page list */
895 if (tb
->page_addr
[0] != page_addr
) {
896 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
897 tb_page_remove(&p
->first_tb
, tb
);
898 invalidate_page_bitmap(p
);
900 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
901 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
902 tb_page_remove(&p
->first_tb
, tb
);
903 invalidate_page_bitmap(p
);
906 tb_invalidated_flag
= 1;
908 /* remove the TB from the hash list */
909 h
= tb_jmp_cache_hash_func(tb
->pc
);
910 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
911 if (env
->tb_jmp_cache
[h
] == tb
)
912 env
->tb_jmp_cache
[h
] = NULL
;
915 /* suppress this TB from the two jump lists */
916 tb_jmp_remove(tb
, 0);
917 tb_jmp_remove(tb
, 1);
919 /* suppress any remaining jumps to this TB */
925 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
926 tb2
= tb1
->jmp_next
[n1
];
927 tb_reset_jump(tb1
, n1
);
928 tb1
->jmp_next
[n1
] = NULL
;
931 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2); /* fail safe */
933 tb_phys_invalidate_count
++;
936 static inline void set_bits(uint8_t *tab
, int start
, int len
)
942 mask
= 0xff << (start
& 7);
943 if ((start
& ~7) == (end
& ~7)) {
945 mask
&= ~(0xff << (end
& 7));
950 start
= (start
+ 8) & ~7;
952 while (start
< end1
) {
957 mask
= ~(0xff << (end
& 7));
963 static void build_page_bitmap(PageDesc
*p
)
965 int n
, tb_start
, tb_end
;
966 TranslationBlock
*tb
;
968 p
->code_bitmap
= g_malloc0(TARGET_PAGE_SIZE
/ 8);
973 tb
= (TranslationBlock
*)((long)tb
& ~3);
974 /* NOTE: this is subtle as a TB may span two physical pages */
976 /* NOTE: tb_end may be after the end of the page, but
977 it is not a problem */
978 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
979 tb_end
= tb_start
+ tb
->size
;
980 if (tb_end
> TARGET_PAGE_SIZE
)
981 tb_end
= TARGET_PAGE_SIZE
;
984 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
986 set_bits(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
987 tb
= tb
->page_next
[n
];
991 TranslationBlock
*tb_gen_code(CPUState
*env
,
992 target_ulong pc
, target_ulong cs_base
,
993 int flags
, int cflags
)
995 TranslationBlock
*tb
;
997 tb_page_addr_t phys_pc
, phys_page2
;
998 target_ulong virt_page2
;
1001 phys_pc
= get_page_addr_code(env
, pc
);
1004 /* flush must be done */
1006 /* cannot fail at this point */
1008 /* Don't forget to invalidate previous TB info. */
1009 tb_invalidated_flag
= 1;
1011 tc_ptr
= code_gen_ptr
;
1012 tb
->tc_ptr
= tc_ptr
;
1013 tb
->cs_base
= cs_base
;
1015 tb
->cflags
= cflags
;
1016 cpu_gen_code(env
, tb
, &code_gen_size
);
1017 code_gen_ptr
= (void *)(((unsigned long)code_gen_ptr
+ code_gen_size
+ CODE_GEN_ALIGN
- 1) & ~(CODE_GEN_ALIGN
- 1));
1019 /* check next page if needed */
1020 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
1022 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
1023 phys_page2
= get_page_addr_code(env
, virt_page2
);
1025 tb_link_page(tb
, phys_pc
, phys_page2
);
1029 /* invalidate all TBs which intersect with the target physical page
1030 starting in range [start;end[. NOTE: start and end must refer to
1031 the same physical page. 'is_cpu_write_access' should be true if called
1032 from a real cpu write access: the virtual CPU will exit the current
1033 TB if code is modified inside this TB. */
1034 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
1035 int is_cpu_write_access
)
1037 TranslationBlock
*tb
, *tb_next
, *saved_tb
;
1038 CPUState
*env
= cpu_single_env
;
1039 tb_page_addr_t tb_start
, tb_end
;
1042 #ifdef TARGET_HAS_PRECISE_SMC
1043 int current_tb_not_found
= is_cpu_write_access
;
1044 TranslationBlock
*current_tb
= NULL
;
1045 int current_tb_modified
= 0;
1046 target_ulong current_pc
= 0;
1047 target_ulong current_cs_base
= 0;
1048 int current_flags
= 0;
1049 #endif /* TARGET_HAS_PRECISE_SMC */
1051 p
= page_find(start
>> TARGET_PAGE_BITS
);
1054 if (!p
->code_bitmap
&&
1055 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
&&
1056 is_cpu_write_access
) {
1057 /* build code bitmap */
1058 build_page_bitmap(p
);
1061 /* we remove all the TBs in the range [start, end[ */
1062 /* XXX: see if in some cases it could be faster to invalidate all the code */
1064 while (tb
!= NULL
) {
1066 tb
= (TranslationBlock
*)((long)tb
& ~3);
1067 tb_next
= tb
->page_next
[n
];
1068 /* NOTE: this is subtle as a TB may span two physical pages */
1070 /* NOTE: tb_end may be after the end of the page, but
1071 it is not a problem */
1072 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1073 tb_end
= tb_start
+ tb
->size
;
1075 tb_start
= tb
->page_addr
[1];
1076 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1078 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1079 #ifdef TARGET_HAS_PRECISE_SMC
1080 if (current_tb_not_found
) {
1081 current_tb_not_found
= 0;
1083 if (env
->mem_io_pc
) {
1084 /* now we have a real cpu fault */
1085 current_tb
= tb_find_pc(env
->mem_io_pc
);
1088 if (current_tb
== tb
&&
1089 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1090 /* If we are modifying the current TB, we must stop
1091 its execution. We could be more precise by checking
1092 that the modification is after the current PC, but it
1093 would require a specialized function to partially
1094 restore the CPU state */
1096 current_tb_modified
= 1;
1097 cpu_restore_state(current_tb
, env
, env
->mem_io_pc
);
1098 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1101 #endif /* TARGET_HAS_PRECISE_SMC */
1102 /* we need to do that to handle the case where a signal
1103 occurs while doing tb_phys_invalidate() */
1106 saved_tb
= env
->current_tb
;
1107 env
->current_tb
= NULL
;
1109 tb_phys_invalidate(tb
, -1);
1111 env
->current_tb
= saved_tb
;
1112 if (env
->interrupt_request
&& env
->current_tb
)
1113 cpu_interrupt(env
, env
->interrupt_request
);
1118 #if !defined(CONFIG_USER_ONLY)
1119 /* if no code remaining, no need to continue to use slow writes */
1121 invalidate_page_bitmap(p
);
1122 if (is_cpu_write_access
) {
1123 tlb_unprotect_code_phys(env
, start
, env
->mem_io_vaddr
);
1127 #ifdef TARGET_HAS_PRECISE_SMC
1128 if (current_tb_modified
) {
1129 /* we generate a block containing just the instruction
1130 modifying the memory. It will ensure that it cannot modify
1132 env
->current_tb
= NULL
;
1133 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1134 cpu_resume_from_signal(env
, NULL
);
1139 /* len must be <= 8 and start must be a multiple of len */
1140 static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start
, int len
)
1146 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1147 cpu_single_env
->mem_io_vaddr
, len
,
1148 cpu_single_env
->eip
,
1149 cpu_single_env
->eip
+ (long)cpu_single_env
->segs
[R_CS
].base
);
1152 p
= page_find(start
>> TARGET_PAGE_BITS
);
1155 if (p
->code_bitmap
) {
1156 offset
= start
& ~TARGET_PAGE_MASK
;
1157 b
= p
->code_bitmap
[offset
>> 3] >> (offset
& 7);
1158 if (b
& ((1 << len
) - 1))
1162 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1166 #if !defined(CONFIG_SOFTMMU)
1167 static void tb_invalidate_phys_page(tb_page_addr_t addr
,
1168 unsigned long pc
, void *puc
)
1170 TranslationBlock
*tb
;
1173 #ifdef TARGET_HAS_PRECISE_SMC
1174 TranslationBlock
*current_tb
= NULL
;
1175 CPUState
*env
= cpu_single_env
;
1176 int current_tb_modified
= 0;
1177 target_ulong current_pc
= 0;
1178 target_ulong current_cs_base
= 0;
1179 int current_flags
= 0;
1182 addr
&= TARGET_PAGE_MASK
;
1183 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1187 #ifdef TARGET_HAS_PRECISE_SMC
1188 if (tb
&& pc
!= 0) {
1189 current_tb
= tb_find_pc(pc
);
1192 while (tb
!= NULL
) {
1194 tb
= (TranslationBlock
*)((long)tb
& ~3);
1195 #ifdef TARGET_HAS_PRECISE_SMC
1196 if (current_tb
== tb
&&
1197 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1198 /* If we are modifying the current TB, we must stop
1199 its execution. We could be more precise by checking
1200 that the modification is after the current PC, but it
1201 would require a specialized function to partially
1202 restore the CPU state */
1204 current_tb_modified
= 1;
1205 cpu_restore_state(current_tb
, env
, pc
);
1206 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1209 #endif /* TARGET_HAS_PRECISE_SMC */
1210 tb_phys_invalidate(tb
, addr
);
1211 tb
= tb
->page_next
[n
];
1214 #ifdef TARGET_HAS_PRECISE_SMC
1215 if (current_tb_modified
) {
1216 /* we generate a block containing just the instruction
1217 modifying the memory. It will ensure that it cannot modify
1219 env
->current_tb
= NULL
;
1220 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1221 cpu_resume_from_signal(env
, puc
);
1227 /* add the tb in the target page and protect it if necessary */
1228 static inline void tb_alloc_page(TranslationBlock
*tb
,
1229 unsigned int n
, tb_page_addr_t page_addr
)
1232 #ifndef CONFIG_USER_ONLY
1233 bool page_already_protected
;
1236 tb
->page_addr
[n
] = page_addr
;
1237 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
, 1);
1238 tb
->page_next
[n
] = p
->first_tb
;
1239 #ifndef CONFIG_USER_ONLY
1240 page_already_protected
= p
->first_tb
!= NULL
;
1242 p
->first_tb
= (TranslationBlock
*)((long)tb
| n
);
1243 invalidate_page_bitmap(p
);
1245 #if defined(TARGET_HAS_SMC) || 1
1247 #if defined(CONFIG_USER_ONLY)
1248 if (p
->flags
& PAGE_WRITE
) {
1253 /* force the host page as non writable (writes will have a
1254 page fault + mprotect overhead) */
1255 page_addr
&= qemu_host_page_mask
;
1257 for(addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1258 addr
+= TARGET_PAGE_SIZE
) {
1260 p2
= page_find (addr
>> TARGET_PAGE_BITS
);
1264 p2
->flags
&= ~PAGE_WRITE
;
1266 mprotect(g2h(page_addr
), qemu_host_page_size
,
1267 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1268 #ifdef DEBUG_TB_INVALIDATE
1269 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1274 /* if some code is already present, then the pages are already
1275 protected. So we handle the case where only the first TB is
1276 allocated in a physical page */
1277 if (!page_already_protected
) {
1278 tlb_protect_code(page_addr
);
1282 #endif /* TARGET_HAS_SMC */
1285 /* add a new TB and link it to the physical page tables. phys_page2 is
1286 (-1) to indicate that only one page contains the TB. */
1287 void tb_link_page(TranslationBlock
*tb
,
1288 tb_page_addr_t phys_pc
, tb_page_addr_t phys_page2
)
1291 TranslationBlock
**ptb
;
1293 /* Grab the mmap lock to stop another thread invalidating this TB
1294 before we are done. */
1296 /* add in the physical hash table */
1297 h
= tb_phys_hash_func(phys_pc
);
1298 ptb
= &tb_phys_hash
[h
];
1299 tb
->phys_hash_next
= *ptb
;
1302 /* add in the page list */
1303 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1304 if (phys_page2
!= -1)
1305 tb_alloc_page(tb
, 1, phys_page2
);
1307 tb
->page_addr
[1] = -1;
1309 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2);
1310 tb
->jmp_next
[0] = NULL
;
1311 tb
->jmp_next
[1] = NULL
;
1313 /* init original jump addresses */
1314 if (tb
->tb_next_offset
[0] != 0xffff)
1315 tb_reset_jump(tb
, 0);
1316 if (tb
->tb_next_offset
[1] != 0xffff)
1317 tb_reset_jump(tb
, 1);
1319 #ifdef DEBUG_TB_CHECK
1325 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1326 tb[1].tc_ptr. Return NULL if not found */
1327 TranslationBlock
*tb_find_pc(unsigned long tc_ptr
)
1329 int m_min
, m_max
, m
;
1331 TranslationBlock
*tb
;
1335 if (tc_ptr
< (unsigned long)code_gen_buffer
||
1336 tc_ptr
>= (unsigned long)code_gen_ptr
)
1338 /* binary search (cf Knuth) */
1341 while (m_min
<= m_max
) {
1342 m
= (m_min
+ m_max
) >> 1;
1344 v
= (unsigned long)tb
->tc_ptr
;
1347 else if (tc_ptr
< v
) {
1356 static void tb_reset_jump_recursive(TranslationBlock
*tb
);
1358 static inline void tb_reset_jump_recursive2(TranslationBlock
*tb
, int n
)
1360 TranslationBlock
*tb1
, *tb_next
, **ptb
;
1363 tb1
= tb
->jmp_next
[n
];
1365 /* find head of list */
1368 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1371 tb1
= tb1
->jmp_next
[n1
];
1373 /* we are now sure now that tb jumps to tb1 */
1376 /* remove tb from the jmp_first list */
1377 ptb
= &tb_next
->jmp_first
;
1381 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1382 if (n1
== n
&& tb1
== tb
)
1384 ptb
= &tb1
->jmp_next
[n1
];
1386 *ptb
= tb
->jmp_next
[n
];
1387 tb
->jmp_next
[n
] = NULL
;
1389 /* suppress the jump to next tb in generated code */
1390 tb_reset_jump(tb
, n
);
1392 /* suppress jumps in the tb on which we could have jumped */
1393 tb_reset_jump_recursive(tb_next
);
1397 static void tb_reset_jump_recursive(TranslationBlock
*tb
)
1399 tb_reset_jump_recursive2(tb
, 0);
1400 tb_reset_jump_recursive2(tb
, 1);
1403 #if defined(TARGET_HAS_ICE)
1404 #if defined(CONFIG_USER_ONLY)
1405 static void breakpoint_invalidate(CPUState
*env
, target_ulong pc
)
1407 tb_invalidate_phys_page_range(pc
, pc
+ 1, 0);
1410 static void breakpoint_invalidate(CPUState
*env
, target_ulong pc
)
1412 target_phys_addr_t addr
;
1414 ram_addr_t ram_addr
;
1417 addr
= cpu_get_phys_page_debug(env
, pc
);
1418 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
1420 ram_addr
= (pd
& TARGET_PAGE_MASK
) | (pc
& ~TARGET_PAGE_MASK
);
1421 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1424 #endif /* TARGET_HAS_ICE */
1426 #if defined(CONFIG_USER_ONLY)
1427 void cpu_watchpoint_remove_all(CPUState
*env
, int mask
)
1432 int cpu_watchpoint_insert(CPUState
*env
, target_ulong addr
, target_ulong len
,
1433 int flags
, CPUWatchpoint
**watchpoint
)
1438 /* Add a watchpoint. */
1439 int cpu_watchpoint_insert(CPUState
*env
, target_ulong addr
, target_ulong len
,
1440 int flags
, CPUWatchpoint
**watchpoint
)
1442 target_ulong len_mask
= ~(len
- 1);
1445 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1446 if ((len
!= 1 && len
!= 2 && len
!= 4 && len
!= 8) || (addr
& ~len_mask
)) {
1447 fprintf(stderr
, "qemu: tried to set invalid watchpoint at "
1448 TARGET_FMT_lx
", len=" TARGET_FMT_lu
"\n", addr
, len
);
1451 wp
= g_malloc(sizeof(*wp
));
1454 wp
->len_mask
= len_mask
;
1457 /* keep all GDB-injected watchpoints in front */
1459 QTAILQ_INSERT_HEAD(&env
->watchpoints
, wp
, entry
);
1461 QTAILQ_INSERT_TAIL(&env
->watchpoints
, wp
, entry
);
1463 tlb_flush_page(env
, addr
);
1470 /* Remove a specific watchpoint. */
1471 int cpu_watchpoint_remove(CPUState
*env
, target_ulong addr
, target_ulong len
,
1474 target_ulong len_mask
= ~(len
- 1);
1477 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1478 if (addr
== wp
->vaddr
&& len_mask
== wp
->len_mask
1479 && flags
== (wp
->flags
& ~BP_WATCHPOINT_HIT
)) {
1480 cpu_watchpoint_remove_by_ref(env
, wp
);
1487 /* Remove a specific watchpoint by reference. */
1488 void cpu_watchpoint_remove_by_ref(CPUState
*env
, CPUWatchpoint
*watchpoint
)
1490 QTAILQ_REMOVE(&env
->watchpoints
, watchpoint
, entry
);
1492 tlb_flush_page(env
, watchpoint
->vaddr
);
1497 /* Remove all matching watchpoints. */
1498 void cpu_watchpoint_remove_all(CPUState
*env
, int mask
)
1500 CPUWatchpoint
*wp
, *next
;
1502 QTAILQ_FOREACH_SAFE(wp
, &env
->watchpoints
, entry
, next
) {
1503 if (wp
->flags
& mask
)
1504 cpu_watchpoint_remove_by_ref(env
, wp
);
1509 /* Add a breakpoint. */
1510 int cpu_breakpoint_insert(CPUState
*env
, target_ulong pc
, int flags
,
1511 CPUBreakpoint
**breakpoint
)
1513 #if defined(TARGET_HAS_ICE)
1516 bp
= g_malloc(sizeof(*bp
));
1521 /* keep all GDB-injected breakpoints in front */
1523 QTAILQ_INSERT_HEAD(&env
->breakpoints
, bp
, entry
);
1525 QTAILQ_INSERT_TAIL(&env
->breakpoints
, bp
, entry
);
1527 breakpoint_invalidate(env
, pc
);
1537 /* Remove a specific breakpoint. */
1538 int cpu_breakpoint_remove(CPUState
*env
, target_ulong pc
, int flags
)
1540 #if defined(TARGET_HAS_ICE)
1543 QTAILQ_FOREACH(bp
, &env
->breakpoints
, entry
) {
1544 if (bp
->pc
== pc
&& bp
->flags
== flags
) {
1545 cpu_breakpoint_remove_by_ref(env
, bp
);
1555 /* Remove a specific breakpoint by reference. */
1556 void cpu_breakpoint_remove_by_ref(CPUState
*env
, CPUBreakpoint
*breakpoint
)
1558 #if defined(TARGET_HAS_ICE)
1559 QTAILQ_REMOVE(&env
->breakpoints
, breakpoint
, entry
);
1561 breakpoint_invalidate(env
, breakpoint
->pc
);
1567 /* Remove all matching breakpoints. */
1568 void cpu_breakpoint_remove_all(CPUState
*env
, int mask
)
1570 #if defined(TARGET_HAS_ICE)
1571 CPUBreakpoint
*bp
, *next
;
1573 QTAILQ_FOREACH_SAFE(bp
, &env
->breakpoints
, entry
, next
) {
1574 if (bp
->flags
& mask
)
1575 cpu_breakpoint_remove_by_ref(env
, bp
);
1580 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1581 CPU loop after each instruction */
1582 void cpu_single_step(CPUState
*env
, int enabled
)
1584 #if defined(TARGET_HAS_ICE)
1585 if (env
->singlestep_enabled
!= enabled
) {
1586 env
->singlestep_enabled
= enabled
;
1588 kvm_update_guest_debug(env
, 0);
1590 /* must flush all the translated code to avoid inconsistencies */
1591 /* XXX: only flush what is necessary */
1598 /* enable or disable low levels log */
1599 void cpu_set_log(int log_flags
)
1601 loglevel
= log_flags
;
1602 if (loglevel
&& !logfile
) {
1603 logfile
= fopen(logfilename
, log_append
? "a" : "w");
1605 perror(logfilename
);
1608 #if !defined(CONFIG_SOFTMMU)
1609 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1611 static char logfile_buf
[4096];
1612 setvbuf(logfile
, logfile_buf
, _IOLBF
, sizeof(logfile_buf
));
1614 #elif defined(_WIN32)
1615 /* Win32 doesn't support line-buffering, so use unbuffered output. */
1616 setvbuf(logfile
, NULL
, _IONBF
, 0);
1618 setvbuf(logfile
, NULL
, _IOLBF
, 0);
1622 if (!loglevel
&& logfile
) {
1628 void cpu_set_log_filename(const char *filename
)
1630 logfilename
= strdup(filename
);
1635 cpu_set_log(loglevel
);
1638 static void cpu_unlink_tb(CPUState
*env
)
1640 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1641 problem and hope the cpu will stop of its own accord. For userspace
1642 emulation this often isn't actually as bad as it sounds. Often
1643 signals are used primarily to interrupt blocking syscalls. */
1644 TranslationBlock
*tb
;
1645 static spinlock_t interrupt_lock
= SPIN_LOCK_UNLOCKED
;
1647 spin_lock(&interrupt_lock
);
1648 tb
= env
->current_tb
;
1649 /* if the cpu is currently executing code, we must unlink it and
1650 all the potentially executing TB */
1652 env
->current_tb
= NULL
;
1653 tb_reset_jump_recursive(tb
);
1655 spin_unlock(&interrupt_lock
);
1658 #ifndef CONFIG_USER_ONLY
1659 /* mask must never be zero, except for A20 change call */
1660 static void tcg_handle_interrupt(CPUState
*env
, int mask
)
1664 old_mask
= env
->interrupt_request
;
1665 env
->interrupt_request
|= mask
;
1668 * If called from iothread context, wake the target cpu in
1671 if (!qemu_cpu_is_self(env
)) {
1677 env
->icount_decr
.u16
.high
= 0xffff;
1679 && (mask
& ~old_mask
) != 0) {
1680 cpu_abort(env
, "Raised interrupt while not in I/O function");
1687 CPUInterruptHandler cpu_interrupt_handler
= tcg_handle_interrupt
;
1689 #else /* CONFIG_USER_ONLY */
1691 void cpu_interrupt(CPUState
*env
, int mask
)
1693 env
->interrupt_request
|= mask
;
1696 #endif /* CONFIG_USER_ONLY */
1698 void cpu_reset_interrupt(CPUState
*env
, int mask
)
1700 env
->interrupt_request
&= ~mask
;
1703 void cpu_exit(CPUState
*env
)
1705 env
->exit_request
= 1;
1709 const CPULogItem cpu_log_items
[] = {
1710 { CPU_LOG_TB_OUT_ASM
, "out_asm",
1711 "show generated host assembly code for each compiled TB" },
1712 { CPU_LOG_TB_IN_ASM
, "in_asm",
1713 "show target assembly code for each compiled TB" },
1714 { CPU_LOG_TB_OP
, "op",
1715 "show micro ops for each compiled TB" },
1716 { CPU_LOG_TB_OP_OPT
, "op_opt",
1719 "before eflags optimization and "
1721 "after liveness analysis" },
1722 { CPU_LOG_INT
, "int",
1723 "show interrupts/exceptions in short format" },
1724 { CPU_LOG_EXEC
, "exec",
1725 "show trace before each executed TB (lots of logs)" },
1726 { CPU_LOG_TB_CPU
, "cpu",
1727 "show CPU state before block translation" },
1729 { CPU_LOG_PCALL
, "pcall",
1730 "show protected mode far calls/returns/exceptions" },
1731 { CPU_LOG_RESET
, "cpu_reset",
1732 "show CPU state before CPU resets" },
1735 { CPU_LOG_IOPORT
, "ioport",
1736 "show all i/o ports accesses" },
1741 static int cmp1(const char *s1
, int n
, const char *s2
)
1743 if (strlen(s2
) != n
)
1745 return memcmp(s1
, s2
, n
) == 0;
1748 /* takes a comma separated list of log masks. Return 0 if error. */
1749 int cpu_str_to_log_mask(const char *str
)
1751 const CPULogItem
*item
;
1758 p1
= strchr(p
, ',');
1761 if(cmp1(p
,p1
-p
,"all")) {
1762 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1766 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1767 if (cmp1(p
, p1
- p
, item
->name
))
1781 void cpu_abort(CPUState
*env
, const char *fmt
, ...)
1788 fprintf(stderr
, "qemu: fatal: ");
1789 vfprintf(stderr
, fmt
, ap
);
1790 fprintf(stderr
, "\n");
1792 cpu_dump_state(env
, stderr
, fprintf
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1794 cpu_dump_state(env
, stderr
, fprintf
, 0);
1796 if (qemu_log_enabled()) {
1797 qemu_log("qemu: fatal: ");
1798 qemu_log_vprintf(fmt
, ap2
);
1801 log_cpu_state(env
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1803 log_cpu_state(env
, 0);
1810 #if defined(CONFIG_USER_ONLY)
1812 struct sigaction act
;
1813 sigfillset(&act
.sa_mask
);
1814 act
.sa_handler
= SIG_DFL
;
1815 sigaction(SIGABRT
, &act
, NULL
);
1821 CPUState
*cpu_copy(CPUState
*env
)
1823 CPUState
*new_env
= cpu_init(env
->cpu_model_str
);
1824 CPUState
*next_cpu
= new_env
->next_cpu
;
1825 int cpu_index
= new_env
->cpu_index
;
1826 #if defined(TARGET_HAS_ICE)
1831 memcpy(new_env
, env
, sizeof(CPUState
));
1833 /* Preserve chaining and index. */
1834 new_env
->next_cpu
= next_cpu
;
1835 new_env
->cpu_index
= cpu_index
;
1837 /* Clone all break/watchpoints.
1838 Note: Once we support ptrace with hw-debug register access, make sure
1839 BP_CPU break/watchpoints are handled correctly on clone. */
1840 QTAILQ_INIT(&env
->breakpoints
);
1841 QTAILQ_INIT(&env
->watchpoints
);
1842 #if defined(TARGET_HAS_ICE)
1843 QTAILQ_FOREACH(bp
, &env
->breakpoints
, entry
) {
1844 cpu_breakpoint_insert(new_env
, bp
->pc
, bp
->flags
, NULL
);
1846 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1847 cpu_watchpoint_insert(new_env
, wp
->vaddr
, (~wp
->len_mask
) + 1,
1855 #if !defined(CONFIG_USER_ONLY)
1857 static inline void tlb_flush_jmp_cache(CPUState
*env
, target_ulong addr
)
1861 /* Discard jump cache entries for any tb which might potentially
1862 overlap the flushed page. */
1863 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1864 memset (&env
->tb_jmp_cache
[i
], 0,
1865 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1867 i
= tb_jmp_cache_hash_page(addr
);
1868 memset (&env
->tb_jmp_cache
[i
], 0,
1869 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1872 static CPUTLBEntry s_cputlb_empty_entry
= {
1879 /* NOTE: if flush_global is true, also flush global entries (not
1881 void tlb_flush(CPUState
*env
, int flush_global
)
1885 #if defined(DEBUG_TLB)
1886 printf("tlb_flush:\n");
1888 /* must reset current TB so that interrupts cannot modify the
1889 links while we are modifying them */
1890 env
->current_tb
= NULL
;
1892 for(i
= 0; i
< CPU_TLB_SIZE
; i
++) {
1894 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
1895 env
->tlb_table
[mmu_idx
][i
] = s_cputlb_empty_entry
;
1899 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
1901 env
->tlb_flush_addr
= -1;
1902 env
->tlb_flush_mask
= 0;
1906 static inline void tlb_flush_entry(CPUTLBEntry
*tlb_entry
, target_ulong addr
)
1908 if (addr
== (tlb_entry
->addr_read
&
1909 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1910 addr
== (tlb_entry
->addr_write
&
1911 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1912 addr
== (tlb_entry
->addr_code
&
1913 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
))) {
1914 *tlb_entry
= s_cputlb_empty_entry
;
1918 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
1923 #if defined(DEBUG_TLB)
1924 printf("tlb_flush_page: " TARGET_FMT_lx
"\n", addr
);
1926 /* Check if we need to flush due to large pages. */
1927 if ((addr
& env
->tlb_flush_mask
) == env
->tlb_flush_addr
) {
1928 #if defined(DEBUG_TLB)
1929 printf("tlb_flush_page: forced full flush ("
1930 TARGET_FMT_lx
"/" TARGET_FMT_lx
")\n",
1931 env
->tlb_flush_addr
, env
->tlb_flush_mask
);
1936 /* must reset current TB so that interrupts cannot modify the
1937 links while we are modifying them */
1938 env
->current_tb
= NULL
;
1940 addr
&= TARGET_PAGE_MASK
;
1941 i
= (addr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
1942 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++)
1943 tlb_flush_entry(&env
->tlb_table
[mmu_idx
][i
], addr
);
1945 tlb_flush_jmp_cache(env
, addr
);
1948 /* update the TLBs so that writes to code in the virtual page 'addr'
1950 static void tlb_protect_code(ram_addr_t ram_addr
)
1952 cpu_physical_memory_reset_dirty(ram_addr
,
1953 ram_addr
+ TARGET_PAGE_SIZE
,
1957 /* update the TLB so that writes in physical page 'phys_addr' are no longer
1958 tested for self modifying code */
1959 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
1962 cpu_physical_memory_set_dirty_flags(ram_addr
, CODE_DIRTY_FLAG
);
1965 static inline void tlb_reset_dirty_range(CPUTLBEntry
*tlb_entry
,
1966 unsigned long start
, unsigned long length
)
1969 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == io_mem_ram
.ram_addr
) {
1970 addr
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) + tlb_entry
->addend
;
1971 if ((addr
- start
) < length
) {
1972 tlb_entry
->addr_write
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) | TLB_NOTDIRTY
;
1977 /* Note: start and end must be within the same ram block. */
1978 void cpu_physical_memory_reset_dirty(ram_addr_t start
, ram_addr_t end
,
1982 unsigned long length
, start1
;
1985 start
&= TARGET_PAGE_MASK
;
1986 end
= TARGET_PAGE_ALIGN(end
);
1988 length
= end
- start
;
1991 cpu_physical_memory_mask_dirty_range(start
, length
, dirty_flags
);
1993 /* we modify the TLB cache so that the dirty bit will be set again
1994 when accessing the range */
1995 start1
= (unsigned long)qemu_safe_ram_ptr(start
);
1996 /* Check that we don't span multiple blocks - this breaks the
1997 address comparisons below. */
1998 if ((unsigned long)qemu_safe_ram_ptr(end
- 1) - start1
1999 != (end
- 1) - start
) {
2003 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
2005 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
2006 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
2007 tlb_reset_dirty_range(&env
->tlb_table
[mmu_idx
][i
],
2013 int cpu_physical_memory_set_dirty_tracking(int enable
)
2016 in_migration
= enable
;
2020 static inline void tlb_update_dirty(CPUTLBEntry
*tlb_entry
)
2022 ram_addr_t ram_addr
;
2025 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == io_mem_ram
.ram_addr
) {
2026 p
= (void *)(unsigned long)((tlb_entry
->addr_write
& TARGET_PAGE_MASK
)
2027 + tlb_entry
->addend
);
2028 ram_addr
= qemu_ram_addr_from_host_nofail(p
);
2029 if (!cpu_physical_memory_is_dirty(ram_addr
)) {
2030 tlb_entry
->addr_write
|= TLB_NOTDIRTY
;
2035 /* update the TLB according to the current state of the dirty bits */
2036 void cpu_tlb_update_dirty(CPUState
*env
)
2040 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
2041 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
2042 tlb_update_dirty(&env
->tlb_table
[mmu_idx
][i
]);
2046 static inline void tlb_set_dirty1(CPUTLBEntry
*tlb_entry
, target_ulong vaddr
)
2048 if (tlb_entry
->addr_write
== (vaddr
| TLB_NOTDIRTY
))
2049 tlb_entry
->addr_write
= vaddr
;
2052 /* update the TLB corresponding to virtual page vaddr
2053 so that it is no longer dirty */
2054 static inline void tlb_set_dirty(CPUState
*env
, target_ulong vaddr
)
2059 vaddr
&= TARGET_PAGE_MASK
;
2060 i
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
2061 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++)
2062 tlb_set_dirty1(&env
->tlb_table
[mmu_idx
][i
], vaddr
);
2065 /* Our TLB does not support large pages, so remember the area covered by
2066 large pages and trigger a full TLB flush if these are invalidated. */
2067 static void tlb_add_large_page(CPUState
*env
, target_ulong vaddr
,
2070 target_ulong mask
= ~(size
- 1);
2072 if (env
->tlb_flush_addr
== (target_ulong
)-1) {
2073 env
->tlb_flush_addr
= vaddr
& mask
;
2074 env
->tlb_flush_mask
= mask
;
2077 /* Extend the existing region to include the new page.
2078 This is a compromise between unnecessary flushes and the cost
2079 of maintaining a full variable size TLB. */
2080 mask
&= env
->tlb_flush_mask
;
2081 while (((env
->tlb_flush_addr
^ vaddr
) & mask
) != 0) {
2084 env
->tlb_flush_addr
&= mask
;
2085 env
->tlb_flush_mask
= mask
;
2088 static bool is_ram_rom(ram_addr_t pd
)
2090 pd
&= ~TARGET_PAGE_MASK
;
2091 return pd
== io_mem_ram
.ram_addr
|| pd
== io_mem_rom
.ram_addr
;
2094 static bool is_ram_rom_romd(ram_addr_t pd
)
2096 return is_ram_rom(pd
) || (pd
& IO_MEM_ROMD
);
2099 /* Add a new TLB entry. At most one entry for a given virtual address
2100 is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
2101 supplied size is only used by tlb_flush_page. */
2102 void tlb_set_page(CPUState
*env
, target_ulong vaddr
,
2103 target_phys_addr_t paddr
, int prot
,
2104 int mmu_idx
, target_ulong size
)
2109 target_ulong address
;
2110 target_ulong code_address
;
2111 unsigned long addend
;
2114 target_phys_addr_t iotlb
;
2116 assert(size
>= TARGET_PAGE_SIZE
);
2117 if (size
!= TARGET_PAGE_SIZE
) {
2118 tlb_add_large_page(env
, vaddr
, size
);
2120 p
= phys_page_find(paddr
>> TARGET_PAGE_BITS
);
2122 #if defined(DEBUG_TLB)
2123 printf("tlb_set_page: vaddr=" TARGET_FMT_lx
" paddr=0x" TARGET_FMT_plx
2124 " prot=%x idx=%d pd=0x%08lx\n",
2125 vaddr
, paddr
, prot
, mmu_idx
, pd
);
2129 if (!is_ram_rom_romd(pd
)) {
2130 /* IO memory case (romd handled later) */
2131 address
|= TLB_MMIO
;
2133 addend
= (unsigned long)qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
);
2134 if (is_ram_rom(pd
)) {
2136 iotlb
= pd
& TARGET_PAGE_MASK
;
2137 if ((pd
& ~TARGET_PAGE_MASK
) == io_mem_ram
.ram_addr
)
2138 iotlb
|= io_mem_notdirty
.ram_addr
;
2140 iotlb
|= io_mem_rom
.ram_addr
;
2142 /* IO handlers are currently passed a physical address.
2143 It would be nice to pass an offset from the base address
2144 of that region. This would avoid having to special case RAM,
2145 and avoid full address decoding in every device.
2146 We can't use the high bits of pd for this because
2147 IO_MEM_ROMD uses these as a ram address. */
2148 iotlb
= (pd
& ~TARGET_PAGE_MASK
);
2149 iotlb
+= p
.region_offset
;
2152 code_address
= address
;
2153 /* Make accesses to pages with watchpoints go via the
2154 watchpoint trap routines. */
2155 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
2156 if (vaddr
== (wp
->vaddr
& TARGET_PAGE_MASK
)) {
2157 /* Avoid trapping reads of pages with a write breakpoint. */
2158 if ((prot
& PAGE_WRITE
) || (wp
->flags
& BP_MEM_READ
)) {
2159 iotlb
= io_mem_watch
.ram_addr
+ paddr
;
2160 address
|= TLB_MMIO
;
2166 index
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
2167 env
->iotlb
[mmu_idx
][index
] = iotlb
- vaddr
;
2168 te
= &env
->tlb_table
[mmu_idx
][index
];
2169 te
->addend
= addend
- vaddr
;
2170 if (prot
& PAGE_READ
) {
2171 te
->addr_read
= address
;
2176 if (prot
& PAGE_EXEC
) {
2177 te
->addr_code
= code_address
;
2181 if (prot
& PAGE_WRITE
) {
2182 if ((pd
& ~TARGET_PAGE_MASK
) == io_mem_rom
.ram_addr
||
2183 (pd
& IO_MEM_ROMD
)) {
2184 /* Write access calls the I/O callback. */
2185 te
->addr_write
= address
| TLB_MMIO
;
2186 } else if ((pd
& ~TARGET_PAGE_MASK
) == io_mem_ram
.ram_addr
&&
2187 !cpu_physical_memory_is_dirty(pd
)) {
2188 te
->addr_write
= address
| TLB_NOTDIRTY
;
2190 te
->addr_write
= address
;
2193 te
->addr_write
= -1;
2199 void tlb_flush(CPUState
*env
, int flush_global
)
2203 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
2208 * Walks guest process memory "regions" one by one
2209 * and calls callback function 'fn' for each region.
2212 struct walk_memory_regions_data
2214 walk_memory_regions_fn fn
;
2216 unsigned long start
;
2220 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
2221 abi_ulong end
, int new_prot
)
2223 if (data
->start
!= -1ul) {
2224 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
2230 data
->start
= (new_prot
? end
: -1ul);
2231 data
->prot
= new_prot
;
2236 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
2237 abi_ulong base
, int level
, void **lp
)
2243 return walk_memory_regions_end(data
, base
, 0);
2248 for (i
= 0; i
< L2_SIZE
; ++i
) {
2249 int prot
= pd
[i
].flags
;
2251 pa
= base
| (i
<< TARGET_PAGE_BITS
);
2252 if (prot
!= data
->prot
) {
2253 rc
= walk_memory_regions_end(data
, pa
, prot
);
2261 for (i
= 0; i
< L2_SIZE
; ++i
) {
2262 pa
= base
| ((abi_ulong
)i
<<
2263 (TARGET_PAGE_BITS
+ L2_BITS
* level
));
2264 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
2274 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
2276 struct walk_memory_regions_data data
;
2284 for (i
= 0; i
< V_L1_SIZE
; i
++) {
2285 int rc
= walk_memory_regions_1(&data
, (abi_ulong
)i
<< V_L1_SHIFT
,
2286 V_L1_SHIFT
/ L2_BITS
- 1, l1_map
+ i
);
2292 return walk_memory_regions_end(&data
, 0, 0);
2295 static int dump_region(void *priv
, abi_ulong start
,
2296 abi_ulong end
, unsigned long prot
)
2298 FILE *f
= (FILE *)priv
;
2300 (void) fprintf(f
, TARGET_ABI_FMT_lx
"-"TARGET_ABI_FMT_lx
2301 " "TARGET_ABI_FMT_lx
" %c%c%c\n",
2302 start
, end
, end
- start
,
2303 ((prot
& PAGE_READ
) ? 'r' : '-'),
2304 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
2305 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
2310 /* dump memory mappings */
2311 void page_dump(FILE *f
)
2313 (void) fprintf(f
, "%-8s %-8s %-8s %s\n",
2314 "start", "end", "size", "prot");
2315 walk_memory_regions(f
, dump_region
);
2318 int page_get_flags(target_ulong address
)
2322 p
= page_find(address
>> TARGET_PAGE_BITS
);
2328 /* Modify the flags of a page and invalidate the code if necessary.
2329 The flag PAGE_WRITE_ORG is positioned automatically depending
2330 on PAGE_WRITE. The mmap_lock should already be held. */
2331 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
2333 target_ulong addr
, len
;
2335 /* This function should never be called with addresses outside the
2336 guest address space. If this assert fires, it probably indicates
2337 a missing call to h2g_valid. */
2338 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2339 assert(end
< ((abi_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2341 assert(start
< end
);
2343 start
= start
& TARGET_PAGE_MASK
;
2344 end
= TARGET_PAGE_ALIGN(end
);
2346 if (flags
& PAGE_WRITE
) {
2347 flags
|= PAGE_WRITE_ORG
;
2350 for (addr
= start
, len
= end
- start
;
2352 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2353 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2355 /* If the write protection bit is set, then we invalidate
2357 if (!(p
->flags
& PAGE_WRITE
) &&
2358 (flags
& PAGE_WRITE
) &&
2360 tb_invalidate_phys_page(addr
, 0, NULL
);
2366 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
2372 /* This function should never be called with addresses outside the
2373 guest address space. If this assert fires, it probably indicates
2374 a missing call to h2g_valid. */
2375 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2376 assert(start
< ((abi_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2382 if (start
+ len
- 1 < start
) {
2383 /* We've wrapped around. */
2387 end
= TARGET_PAGE_ALIGN(start
+len
); /* must do before we loose bits in the next step */
2388 start
= start
& TARGET_PAGE_MASK
;
2390 for (addr
= start
, len
= end
- start
;
2392 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2393 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2396 if( !(p
->flags
& PAGE_VALID
) )
2399 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
))
2401 if (flags
& PAGE_WRITE
) {
2402 if (!(p
->flags
& PAGE_WRITE_ORG
))
2404 /* unprotect the page if it was put read-only because it
2405 contains translated code */
2406 if (!(p
->flags
& PAGE_WRITE
)) {
2407 if (!page_unprotect(addr
, 0, NULL
))
2416 /* called from signal handler: invalidate the code and unprotect the
2417 page. Return TRUE if the fault was successfully handled. */
2418 int page_unprotect(target_ulong address
, unsigned long pc
, void *puc
)
2422 target_ulong host_start
, host_end
, addr
;
2424 /* Technically this isn't safe inside a signal handler. However we
2425 know this only ever happens in a synchronous SEGV handler, so in
2426 practice it seems to be ok. */
2429 p
= page_find(address
>> TARGET_PAGE_BITS
);
2435 /* if the page was really writable, then we change its
2436 protection back to writable */
2437 if ((p
->flags
& PAGE_WRITE_ORG
) && !(p
->flags
& PAGE_WRITE
)) {
2438 host_start
= address
& qemu_host_page_mask
;
2439 host_end
= host_start
+ qemu_host_page_size
;
2442 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2443 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2444 p
->flags
|= PAGE_WRITE
;
2447 /* and since the content will be modified, we must invalidate
2448 the corresponding translated code. */
2449 tb_invalidate_phys_page(addr
, pc
, puc
);
2450 #ifdef DEBUG_TB_CHECK
2451 tb_invalidate_check(addr
);
2454 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2464 static inline void tlb_set_dirty(CPUState
*env
,
2465 unsigned long addr
, target_ulong vaddr
)
2468 #endif /* defined(CONFIG_USER_ONLY) */
2470 #if !defined(CONFIG_USER_ONLY)
2472 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
2473 typedef struct subpage_t
{
2475 target_phys_addr_t base
;
2476 ram_addr_t sub_io_index
[TARGET_PAGE_SIZE
];
2477 ram_addr_t region_offset
[TARGET_PAGE_SIZE
];
2480 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2481 ram_addr_t memory
, ram_addr_t region_offset
);
2482 static subpage_t
*subpage_init (target_phys_addr_t base
, ram_addr_t
*phys
,
2483 ram_addr_t orig_memory
,
2484 ram_addr_t region_offset
);
2485 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2488 if (addr > start_addr) \
2491 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2492 if (start_addr2 > 0) \
2496 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2497 end_addr2 = TARGET_PAGE_SIZE - 1; \
2499 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2500 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2505 /* register physical memory.
2506 For RAM, 'size' must be a multiple of the target page size.
2507 If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2508 io memory page. The address used when calling the IO function is
2509 the offset from the start of the region, plus region_offset. Both
2510 start_addr and region_offset are rounded down to a page boundary
2511 before calculating this offset. This should not be a problem unless
2512 the low bits of start_addr and region_offset differ. */
2513 void cpu_register_physical_memory_log(MemoryRegionSection
*section
,
2514 bool readable
, bool readonly
)
2516 target_phys_addr_t start_addr
= section
->offset_within_address_space
;
2517 ram_addr_t size
= section
->size
;
2518 ram_addr_t phys_offset
= section
->mr
->ram_addr
;
2519 ram_addr_t region_offset
= section
->offset_within_region
;
2520 target_phys_addr_t addr
, end_addr
;
2523 ram_addr_t orig_size
= size
;
2526 if (memory_region_is_ram(section
->mr
)) {
2527 phys_offset
+= region_offset
;
2532 phys_offset
&= ~TARGET_PAGE_MASK
& ~IO_MEM_ROMD
;
2536 phys_offset
|= io_mem_rom
.ram_addr
;
2541 if (phys_offset
== io_mem_unassigned
.ram_addr
) {
2542 region_offset
= start_addr
;
2544 region_offset
&= TARGET_PAGE_MASK
;
2545 size
= (size
+ TARGET_PAGE_SIZE
- 1) & TARGET_PAGE_MASK
;
2546 end_addr
= start_addr
+ (target_phys_addr_t
)size
;
2550 p
= phys_page_find_alloc(addr
>> TARGET_PAGE_BITS
, 0);
2551 if (p
&& p
->phys_offset
!= io_mem_unassigned
.ram_addr
) {
2552 ram_addr_t orig_memory
= p
->phys_offset
;
2553 target_phys_addr_t start_addr2
, end_addr2
;
2554 int need_subpage
= 0;
2556 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
, end_addr2
,
2559 if (!(orig_memory
& IO_MEM_SUBPAGE
)) {
2560 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2561 &p
->phys_offset
, orig_memory
,
2565 = io_mem_region
[(orig_memory
& ~TARGET_PAGE_MASK
)
2567 subpage
= container_of(mr
, subpage_t
, iomem
);
2569 subpage_register(subpage
, start_addr2
, end_addr2
, phys_offset
,
2571 p
->region_offset
= 0;
2573 p
->phys_offset
= phys_offset
;
2574 p
->region_offset
= region_offset
;
2575 if (is_ram_rom_romd(phys_offset
))
2576 phys_offset
+= TARGET_PAGE_SIZE
;
2579 p
= phys_page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2580 p
->phys_offset
= phys_offset
;
2581 p
->region_offset
= region_offset
;
2582 if (is_ram_rom_romd(phys_offset
)) {
2583 phys_offset
+= TARGET_PAGE_SIZE
;
2585 target_phys_addr_t start_addr2
, end_addr2
;
2586 int need_subpage
= 0;
2588 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
,
2589 end_addr2
, need_subpage
);
2592 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2594 io_mem_unassigned
.ram_addr
,
2595 addr
& TARGET_PAGE_MASK
);
2596 subpage_register(subpage
, start_addr2
, end_addr2
,
2597 phys_offset
, region_offset
);
2598 p
->region_offset
= 0;
2602 region_offset
+= TARGET_PAGE_SIZE
;
2603 addr
+= TARGET_PAGE_SIZE
;
2604 } while (addr
!= end_addr
);
2606 /* since each CPU stores ram addresses in its TLB cache, we must
2607 reset the modified entries */
2609 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
2614 void qemu_register_coalesced_mmio(target_phys_addr_t addr
, ram_addr_t size
)
2617 kvm_coalesce_mmio_region(addr
, size
);
2620 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr
, ram_addr_t size
)
2623 kvm_uncoalesce_mmio_region(addr
, size
);
2626 void qemu_flush_coalesced_mmio_buffer(void)
2629 kvm_flush_coalesced_mmio_buffer();
2632 #if defined(__linux__) && !defined(TARGET_S390X)
2634 #include <sys/vfs.h>
2636 #define HUGETLBFS_MAGIC 0x958458f6
2638 static long gethugepagesize(const char *path
)
2644 ret
= statfs(path
, &fs
);
2645 } while (ret
!= 0 && errno
== EINTR
);
2652 if (fs
.f_type
!= HUGETLBFS_MAGIC
)
2653 fprintf(stderr
, "Warning: path not on HugeTLBFS: %s\n", path
);
2658 static void *file_ram_alloc(RAMBlock
*block
,
2668 unsigned long hpagesize
;
2670 hpagesize
= gethugepagesize(path
);
2675 if (memory
< hpagesize
) {
2679 if (kvm_enabled() && !kvm_has_sync_mmu()) {
2680 fprintf(stderr
, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
2684 if (asprintf(&filename
, "%s/qemu_back_mem.XXXXXX", path
) == -1) {
2688 fd
= mkstemp(filename
);
2690 perror("unable to create backing store for hugepages");
2697 memory
= (memory
+hpagesize
-1) & ~(hpagesize
-1);
2700 * ftruncate is not supported by hugetlbfs in older
2701 * hosts, so don't bother bailing out on errors.
2702 * If anything goes wrong with it under other filesystems,
2705 if (ftruncate(fd
, memory
))
2706 perror("ftruncate");
2709 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
2710 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
2711 * to sidestep this quirk.
2713 flags
= mem_prealloc
? MAP_POPULATE
| MAP_SHARED
: MAP_PRIVATE
;
2714 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
, flags
, fd
, 0);
2716 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
, MAP_PRIVATE
, fd
, 0);
2718 if (area
== MAP_FAILED
) {
2719 perror("file_ram_alloc: can't mmap RAM pages");
2728 static ram_addr_t
find_ram_offset(ram_addr_t size
)
2730 RAMBlock
*block
, *next_block
;
2731 ram_addr_t offset
= RAM_ADDR_MAX
, mingap
= RAM_ADDR_MAX
;
2733 if (QLIST_EMPTY(&ram_list
.blocks
))
2736 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2737 ram_addr_t end
, next
= RAM_ADDR_MAX
;
2739 end
= block
->offset
+ block
->length
;
2741 QLIST_FOREACH(next_block
, &ram_list
.blocks
, next
) {
2742 if (next_block
->offset
>= end
) {
2743 next
= MIN(next
, next_block
->offset
);
2746 if (next
- end
>= size
&& next
- end
< mingap
) {
2748 mingap
= next
- end
;
2752 if (offset
== RAM_ADDR_MAX
) {
2753 fprintf(stderr
, "Failed to find gap of requested size: %" PRIu64
"\n",
2761 static ram_addr_t
last_ram_offset(void)
2764 ram_addr_t last
= 0;
2766 QLIST_FOREACH(block
, &ram_list
.blocks
, next
)
2767 last
= MAX(last
, block
->offset
+ block
->length
);
2772 void qemu_ram_set_idstr(ram_addr_t addr
, const char *name
, DeviceState
*dev
)
2774 RAMBlock
*new_block
, *block
;
2777 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2778 if (block
->offset
== addr
) {
2784 assert(!new_block
->idstr
[0]);
2786 if (dev
&& dev
->parent_bus
&& dev
->parent_bus
->info
->get_dev_path
) {
2787 char *id
= dev
->parent_bus
->info
->get_dev_path(dev
);
2789 snprintf(new_block
->idstr
, sizeof(new_block
->idstr
), "%s/", id
);
2793 pstrcat(new_block
->idstr
, sizeof(new_block
->idstr
), name
);
2795 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2796 if (block
!= new_block
&& !strcmp(block
->idstr
, new_block
->idstr
)) {
2797 fprintf(stderr
, "RAMBlock \"%s\" already registered, abort!\n",
2804 ram_addr_t
qemu_ram_alloc_from_ptr(ram_addr_t size
, void *host
,
2807 RAMBlock
*new_block
;
2809 size
= TARGET_PAGE_ALIGN(size
);
2810 new_block
= g_malloc0(sizeof(*new_block
));
2813 new_block
->offset
= find_ram_offset(size
);
2815 new_block
->host
= host
;
2816 new_block
->flags
|= RAM_PREALLOC_MASK
;
2819 #if defined (__linux__) && !defined(TARGET_S390X)
2820 new_block
->host
= file_ram_alloc(new_block
, size
, mem_path
);
2821 if (!new_block
->host
) {
2822 new_block
->host
= qemu_vmalloc(size
);
2823 qemu_madvise(new_block
->host
, size
, QEMU_MADV_MERGEABLE
);
2826 fprintf(stderr
, "-mem-path option unsupported\n");
2830 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2831 /* S390 KVM requires the topmost vma of the RAM to be smaller than
2832 an system defined value, which is at least 256GB. Larger systems
2833 have larger values. We put the guest between the end of data
2834 segment (system break) and this value. We use 32GB as a base to
2835 have enough room for the system break to grow. */
2836 new_block
->host
= mmap((void*)0x800000000, size
,
2837 PROT_EXEC
|PROT_READ
|PROT_WRITE
,
2838 MAP_SHARED
| MAP_ANONYMOUS
| MAP_FIXED
, -1, 0);
2839 if (new_block
->host
== MAP_FAILED
) {
2840 fprintf(stderr
, "Allocating RAM failed\n");
2844 if (xen_enabled()) {
2845 xen_ram_alloc(new_block
->offset
, size
, mr
);
2847 new_block
->host
= qemu_vmalloc(size
);
2850 qemu_madvise(new_block
->host
, size
, QEMU_MADV_MERGEABLE
);
2853 new_block
->length
= size
;
2855 QLIST_INSERT_HEAD(&ram_list
.blocks
, new_block
, next
);
2857 ram_list
.phys_dirty
= g_realloc(ram_list
.phys_dirty
,
2858 last_ram_offset() >> TARGET_PAGE_BITS
);
2859 memset(ram_list
.phys_dirty
+ (new_block
->offset
>> TARGET_PAGE_BITS
),
2860 0xff, size
>> TARGET_PAGE_BITS
);
2863 kvm_setup_guest_memory(new_block
->host
, size
);
2865 return new_block
->offset
;
2868 ram_addr_t
qemu_ram_alloc(ram_addr_t size
, MemoryRegion
*mr
)
2870 return qemu_ram_alloc_from_ptr(size
, NULL
, mr
);
2873 void qemu_ram_free_from_ptr(ram_addr_t addr
)
2877 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2878 if (addr
== block
->offset
) {
2879 QLIST_REMOVE(block
, next
);
2886 void qemu_ram_free(ram_addr_t addr
)
2890 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2891 if (addr
== block
->offset
) {
2892 QLIST_REMOVE(block
, next
);
2893 if (block
->flags
& RAM_PREALLOC_MASK
) {
2895 } else if (mem_path
) {
2896 #if defined (__linux__) && !defined(TARGET_S390X)
2898 munmap(block
->host
, block
->length
);
2901 qemu_vfree(block
->host
);
2907 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2908 munmap(block
->host
, block
->length
);
2910 if (xen_enabled()) {
2911 xen_invalidate_map_cache_entry(block
->host
);
2913 qemu_vfree(block
->host
);
2925 void qemu_ram_remap(ram_addr_t addr
, ram_addr_t length
)
2932 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2933 offset
= addr
- block
->offset
;
2934 if (offset
< block
->length
) {
2935 vaddr
= block
->host
+ offset
;
2936 if (block
->flags
& RAM_PREALLOC_MASK
) {
2940 munmap(vaddr
, length
);
2942 #if defined(__linux__) && !defined(TARGET_S390X)
2945 flags
|= mem_prealloc
? MAP_POPULATE
| MAP_SHARED
:
2948 flags
|= MAP_PRIVATE
;
2950 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
2951 flags
, block
->fd
, offset
);
2953 flags
|= MAP_PRIVATE
| MAP_ANONYMOUS
;
2954 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
2961 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2962 flags
|= MAP_SHARED
| MAP_ANONYMOUS
;
2963 area
= mmap(vaddr
, length
, PROT_EXEC
|PROT_READ
|PROT_WRITE
,
2966 flags
|= MAP_PRIVATE
| MAP_ANONYMOUS
;
2967 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
2971 if (area
!= vaddr
) {
2972 fprintf(stderr
, "Could not remap addr: "
2973 RAM_ADDR_FMT
"@" RAM_ADDR_FMT
"\n",
2977 qemu_madvise(vaddr
, length
, QEMU_MADV_MERGEABLE
);
2983 #endif /* !_WIN32 */
2985 /* Return a host pointer to ram allocated with qemu_ram_alloc.
2986 With the exception of the softmmu code in this file, this should
2987 only be used for local memory (e.g. video ram) that the device owns,
2988 and knows it isn't going to access beyond the end of the block.
2990 It should not be used for general purpose DMA.
2991 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
2993 void *qemu_get_ram_ptr(ram_addr_t addr
)
2997 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2998 if (addr
- block
->offset
< block
->length
) {
2999 /* Move this entry to to start of the list. */
3000 if (block
!= QLIST_FIRST(&ram_list
.blocks
)) {
3001 QLIST_REMOVE(block
, next
);
3002 QLIST_INSERT_HEAD(&ram_list
.blocks
, block
, next
);
3004 if (xen_enabled()) {
3005 /* We need to check if the requested address is in the RAM
3006 * because we don't want to map the entire memory in QEMU.
3007 * In that case just map until the end of the page.
3009 if (block
->offset
== 0) {
3010 return xen_map_cache(addr
, 0, 0);
3011 } else if (block
->host
== NULL
) {
3013 xen_map_cache(block
->offset
, block
->length
, 1);
3016 return block
->host
+ (addr
- block
->offset
);
3020 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
3026 /* Return a host pointer to ram allocated with qemu_ram_alloc.
3027 * Same as qemu_get_ram_ptr but avoid reordering ramblocks.
3029 void *qemu_safe_ram_ptr(ram_addr_t addr
)
3033 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
3034 if (addr
- block
->offset
< block
->length
) {
3035 if (xen_enabled()) {
3036 /* We need to check if the requested address is in the RAM
3037 * because we don't want to map the entire memory in QEMU.
3038 * In that case just map until the end of the page.
3040 if (block
->offset
== 0) {
3041 return xen_map_cache(addr
, 0, 0);
3042 } else if (block
->host
== NULL
) {
3044 xen_map_cache(block
->offset
, block
->length
, 1);
3047 return block
->host
+ (addr
- block
->offset
);
3051 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
3057 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
3058 * but takes a size argument */
3059 void *qemu_ram_ptr_length(ram_addr_t addr
, ram_addr_t
*size
)
3064 if (xen_enabled()) {
3065 return xen_map_cache(addr
, *size
, 1);
3069 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
3070 if (addr
- block
->offset
< block
->length
) {
3071 if (addr
- block
->offset
+ *size
> block
->length
)
3072 *size
= block
->length
- addr
+ block
->offset
;
3073 return block
->host
+ (addr
- block
->offset
);
3077 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
3082 void qemu_put_ram_ptr(void *addr
)
3084 trace_qemu_put_ram_ptr(addr
);
3087 int qemu_ram_addr_from_host(void *ptr
, ram_addr_t
*ram_addr
)
3090 uint8_t *host
= ptr
;
3092 if (xen_enabled()) {
3093 *ram_addr
= xen_ram_addr_from_mapcache(ptr
);
3097 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
3098 /* This case append when the block is not mapped. */
3099 if (block
->host
== NULL
) {
3102 if (host
- block
->host
< block
->length
) {
3103 *ram_addr
= block
->offset
+ (host
- block
->host
);
3111 /* Some of the softmmu routines need to translate from a host pointer
3112 (typically a TLB entry) back to a ram offset. */
3113 ram_addr_t
qemu_ram_addr_from_host_nofail(void *ptr
)
3115 ram_addr_t ram_addr
;
3117 if (qemu_ram_addr_from_host(ptr
, &ram_addr
)) {
3118 fprintf(stderr
, "Bad ram pointer %p\n", ptr
);
3124 static uint64_t unassigned_mem_read(void *opaque
, target_phys_addr_t addr
,
3127 #ifdef DEBUG_UNASSIGNED
3128 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
3130 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3131 cpu_unassigned_access(cpu_single_env
, addr
, 0, 0, 0, size
);
3136 static void unassigned_mem_write(void *opaque
, target_phys_addr_t addr
,
3137 uint64_t val
, unsigned size
)
3139 #ifdef DEBUG_UNASSIGNED
3140 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%"PRIx64
"\n", addr
, val
);
3142 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3143 cpu_unassigned_access(cpu_single_env
, addr
, 1, 0, 0, size
);
3147 static const MemoryRegionOps unassigned_mem_ops
= {
3148 .read
= unassigned_mem_read
,
3149 .write
= unassigned_mem_write
,
3150 .endianness
= DEVICE_NATIVE_ENDIAN
,
3153 static uint64_t error_mem_read(void *opaque
, target_phys_addr_t addr
,
3159 static void error_mem_write(void *opaque
, target_phys_addr_t addr
,
3160 uint64_t value
, unsigned size
)
3165 static const MemoryRegionOps error_mem_ops
= {
3166 .read
= error_mem_read
,
3167 .write
= error_mem_write
,
3168 .endianness
= DEVICE_NATIVE_ENDIAN
,
3171 static const MemoryRegionOps rom_mem_ops
= {
3172 .read
= error_mem_read
,
3173 .write
= unassigned_mem_write
,
3174 .endianness
= DEVICE_NATIVE_ENDIAN
,
3177 static void notdirty_mem_write(void *opaque
, target_phys_addr_t ram_addr
,
3178 uint64_t val
, unsigned size
)
3181 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
3182 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
3183 #if !defined(CONFIG_USER_ONLY)
3184 tb_invalidate_phys_page_fast(ram_addr
, size
);
3185 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
3190 stb_p(qemu_get_ram_ptr(ram_addr
), val
);
3193 stw_p(qemu_get_ram_ptr(ram_addr
), val
);
3196 stl_p(qemu_get_ram_ptr(ram_addr
), val
);
3201 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
3202 cpu_physical_memory_set_dirty_flags(ram_addr
, dirty_flags
);
3203 /* we remove the notdirty callback only if the code has been
3205 if (dirty_flags
== 0xff)
3206 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
3209 static const MemoryRegionOps notdirty_mem_ops
= {
3210 .read
= error_mem_read
,
3211 .write
= notdirty_mem_write
,
3212 .endianness
= DEVICE_NATIVE_ENDIAN
,
3215 /* Generate a debug exception if a watchpoint has been hit. */
3216 static void check_watchpoint(int offset
, int len_mask
, int flags
)
3218 CPUState
*env
= cpu_single_env
;
3219 target_ulong pc
, cs_base
;
3220 TranslationBlock
*tb
;
3225 if (env
->watchpoint_hit
) {
3226 /* We re-entered the check after replacing the TB. Now raise
3227 * the debug interrupt so that is will trigger after the
3228 * current instruction. */
3229 cpu_interrupt(env
, CPU_INTERRUPT_DEBUG
);
3232 vaddr
= (env
->mem_io_vaddr
& TARGET_PAGE_MASK
) + offset
;
3233 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
3234 if ((vaddr
== (wp
->vaddr
& len_mask
) ||
3235 (vaddr
& wp
->len_mask
) == wp
->vaddr
) && (wp
->flags
& flags
)) {
3236 wp
->flags
|= BP_WATCHPOINT_HIT
;
3237 if (!env
->watchpoint_hit
) {
3238 env
->watchpoint_hit
= wp
;
3239 tb
= tb_find_pc(env
->mem_io_pc
);
3241 cpu_abort(env
, "check_watchpoint: could not find TB for "
3242 "pc=%p", (void *)env
->mem_io_pc
);
3244 cpu_restore_state(tb
, env
, env
->mem_io_pc
);
3245 tb_phys_invalidate(tb
, -1);
3246 if (wp
->flags
& BP_STOP_BEFORE_ACCESS
) {
3247 env
->exception_index
= EXCP_DEBUG
;
3249 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &cpu_flags
);
3250 tb_gen_code(env
, pc
, cs_base
, cpu_flags
, 1);
3252 cpu_resume_from_signal(env
, NULL
);
3255 wp
->flags
&= ~BP_WATCHPOINT_HIT
;
3260 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
3261 so these check for a hit then pass through to the normal out-of-line
3263 static uint64_t watch_mem_read(void *opaque
, target_phys_addr_t addr
,
3266 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~(size
- 1), BP_MEM_READ
);
3268 case 1: return ldub_phys(addr
);
3269 case 2: return lduw_phys(addr
);
3270 case 4: return ldl_phys(addr
);
3275 static void watch_mem_write(void *opaque
, target_phys_addr_t addr
,
3276 uint64_t val
, unsigned size
)
3278 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~(size
- 1), BP_MEM_WRITE
);
3280 case 1: stb_phys(addr
, val
);
3281 case 2: stw_phys(addr
, val
);
3282 case 4: stl_phys(addr
, val
);
3287 static const MemoryRegionOps watch_mem_ops
= {
3288 .read
= watch_mem_read
,
3289 .write
= watch_mem_write
,
3290 .endianness
= DEVICE_NATIVE_ENDIAN
,
3293 static uint64_t subpage_read(void *opaque
, target_phys_addr_t addr
,
3296 subpage_t
*mmio
= opaque
;
3297 unsigned int idx
= SUBPAGE_IDX(addr
);
3298 #if defined(DEBUG_SUBPAGE)
3299 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d\n", __func__
,
3300 mmio
, len
, addr
, idx
);
3303 addr
+= mmio
->region_offset
[idx
];
3304 idx
= mmio
->sub_io_index
[idx
];
3305 return io_mem_read(idx
, addr
, len
);
3308 static void subpage_write(void *opaque
, target_phys_addr_t addr
,
3309 uint64_t value
, unsigned len
)
3311 subpage_t
*mmio
= opaque
;
3312 unsigned int idx
= SUBPAGE_IDX(addr
);
3313 #if defined(DEBUG_SUBPAGE)
3314 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
3315 " idx %d value %"PRIx64
"\n",
3316 __func__
, mmio
, len
, addr
, idx
, value
);
3319 addr
+= mmio
->region_offset
[idx
];
3320 idx
= mmio
->sub_io_index
[idx
];
3321 io_mem_write(idx
, addr
, value
, len
);
3324 static const MemoryRegionOps subpage_ops
= {
3325 .read
= subpage_read
,
3326 .write
= subpage_write
,
3327 .endianness
= DEVICE_NATIVE_ENDIAN
,
3330 static uint64_t subpage_ram_read(void *opaque
, target_phys_addr_t addr
,
3333 ram_addr_t raddr
= addr
;
3334 void *ptr
= qemu_get_ram_ptr(raddr
);
3336 case 1: return ldub_p(ptr
);
3337 case 2: return lduw_p(ptr
);
3338 case 4: return ldl_p(ptr
);
3343 static void subpage_ram_write(void *opaque
, target_phys_addr_t addr
,
3344 uint64_t value
, unsigned size
)
3346 ram_addr_t raddr
= addr
;
3347 void *ptr
= qemu_get_ram_ptr(raddr
);
3349 case 1: return stb_p(ptr
, value
);
3350 case 2: return stw_p(ptr
, value
);
3351 case 4: return stl_p(ptr
, value
);
3356 static const MemoryRegionOps subpage_ram_ops
= {
3357 .read
= subpage_ram_read
,
3358 .write
= subpage_ram_write
,
3359 .endianness
= DEVICE_NATIVE_ENDIAN
,
3362 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
3363 ram_addr_t memory
, ram_addr_t region_offset
)
3367 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
3369 idx
= SUBPAGE_IDX(start
);
3370 eidx
= SUBPAGE_IDX(end
);
3371 #if defined(DEBUG_SUBPAGE)
3372 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__
,
3373 mmio
, start
, end
, idx
, eidx
, memory
);
3375 if ((memory
& ~TARGET_PAGE_MASK
) == io_mem_ram
.ram_addr
) {
3376 memory
= io_mem_subpage_ram
.ram_addr
;
3378 memory
= (memory
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3379 for (; idx
<= eidx
; idx
++) {
3380 mmio
->sub_io_index
[idx
] = memory
;
3381 mmio
->region_offset
[idx
] = region_offset
;
3387 static subpage_t
*subpage_init (target_phys_addr_t base
, ram_addr_t
*phys
,
3388 ram_addr_t orig_memory
,
3389 ram_addr_t region_offset
)
3394 mmio
= g_malloc0(sizeof(subpage_t
));
3397 memory_region_init_io(&mmio
->iomem
, &subpage_ops
, mmio
,
3398 "subpage", TARGET_PAGE_SIZE
);
3399 subpage_memory
= mmio
->iomem
.ram_addr
;
3400 #if defined(DEBUG_SUBPAGE)
3401 printf("%s: %p base " TARGET_FMT_plx
" len %08x %d\n", __func__
,
3402 mmio
, base
, TARGET_PAGE_SIZE
, subpage_memory
);
3404 *phys
= subpage_memory
| IO_MEM_SUBPAGE
;
3405 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
-1, orig_memory
, region_offset
);
3410 static int get_free_io_mem_idx(void)
3414 for (i
= 0; i
<IO_MEM_NB_ENTRIES
; i
++)
3415 if (!io_mem_used
[i
]) {
3419 fprintf(stderr
, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES
);
3423 /* mem_read and mem_write are arrays of functions containing the
3424 function to access byte (index 0), word (index 1) and dword (index
3425 2). Functions can be omitted with a NULL function pointer.
3426 If io_index is non zero, the corresponding io zone is
3427 modified. If it is zero, a new io zone is allocated. The return
3428 value can be used with cpu_register_physical_memory(). (-1) is
3429 returned if error. */
3430 static int cpu_register_io_memory_fixed(int io_index
, MemoryRegion
*mr
)
3432 if (io_index
<= 0) {
3433 io_index
= get_free_io_mem_idx();
3437 io_index
>>= IO_MEM_SHIFT
;
3438 if (io_index
>= IO_MEM_NB_ENTRIES
)
3442 io_mem_region
[io_index
] = mr
;
3444 return (io_index
<< IO_MEM_SHIFT
);
3447 int cpu_register_io_memory(MemoryRegion
*mr
)
3449 return cpu_register_io_memory_fixed(0, mr
);
3452 void cpu_unregister_io_memory(int io_table_address
)
3454 int io_index
= io_table_address
>> IO_MEM_SHIFT
;
3456 io_mem_region
[io_index
] = NULL
;
3457 io_mem_used
[io_index
] = 0;
3460 static void io_mem_init(void)
3464 /* Must be first: */
3465 memory_region_init_io(&io_mem_ram
, &error_mem_ops
, NULL
, "ram", UINT64_MAX
);
3466 assert(io_mem_ram
.ram_addr
== 0);
3467 memory_region_init_io(&io_mem_rom
, &rom_mem_ops
, NULL
, "rom", UINT64_MAX
);
3468 memory_region_init_io(&io_mem_unassigned
, &unassigned_mem_ops
, NULL
,
3469 "unassigned", UINT64_MAX
);
3470 memory_region_init_io(&io_mem_notdirty
, ¬dirty_mem_ops
, NULL
,
3471 "notdirty", UINT64_MAX
);
3472 memory_region_init_io(&io_mem_subpage_ram
, &subpage_ram_ops
, NULL
,
3473 "subpage-ram", UINT64_MAX
);
3477 memory_region_init_io(&io_mem_watch
, &watch_mem_ops
, NULL
,
3478 "watch", UINT64_MAX
);
3481 static void memory_map_init(void)
3483 system_memory
= g_malloc(sizeof(*system_memory
));
3484 memory_region_init(system_memory
, "system", INT64_MAX
);
3485 set_system_memory_map(system_memory
);
3487 system_io
= g_malloc(sizeof(*system_io
));
3488 memory_region_init(system_io
, "io", 65536);
3489 set_system_io_map(system_io
);
3492 MemoryRegion
*get_system_memory(void)
3494 return system_memory
;
3497 MemoryRegion
*get_system_io(void)
3502 #endif /* !defined(CONFIG_USER_ONLY) */
3504 /* physical memory access (slow version, mainly for debug) */
3505 #if defined(CONFIG_USER_ONLY)
3506 int cpu_memory_rw_debug(CPUState
*env
, target_ulong addr
,
3507 uint8_t *buf
, int len
, int is_write
)
3514 page
= addr
& TARGET_PAGE_MASK
;
3515 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3518 flags
= page_get_flags(page
);
3519 if (!(flags
& PAGE_VALID
))
3522 if (!(flags
& PAGE_WRITE
))
3524 /* XXX: this code should not depend on lock_user */
3525 if (!(p
= lock_user(VERIFY_WRITE
, addr
, l
, 0)))
3528 unlock_user(p
, addr
, l
);
3530 if (!(flags
& PAGE_READ
))
3532 /* XXX: this code should not depend on lock_user */
3533 if (!(p
= lock_user(VERIFY_READ
, addr
, l
, 1)))
3536 unlock_user(p
, addr
, 0);
3546 void cpu_physical_memory_rw(target_phys_addr_t addr
, uint8_t *buf
,
3547 int len
, int is_write
)
3552 target_phys_addr_t page
;
3557 page
= addr
& TARGET_PAGE_MASK
;
3558 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3561 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3565 if ((pd
& ~TARGET_PAGE_MASK
) != io_mem_ram
.ram_addr
) {
3566 target_phys_addr_t addr1
;
3567 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3568 addr1
= (addr
& ~TARGET_PAGE_MASK
) + p
.region_offset
;
3569 /* XXX: could force cpu_single_env to NULL to avoid
3571 if (l
>= 4 && ((addr1
& 3) == 0)) {
3572 /* 32 bit write access */
3574 io_mem_write(io_index
, addr1
, val
, 4);
3576 } else if (l
>= 2 && ((addr1
& 1) == 0)) {
3577 /* 16 bit write access */
3579 io_mem_write(io_index
, addr1
, val
, 2);
3582 /* 8 bit write access */
3584 io_mem_write(io_index
, addr1
, val
, 1);
3589 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3591 ptr
= qemu_get_ram_ptr(addr1
);
3592 memcpy(ptr
, buf
, l
);
3593 if (!cpu_physical_memory_is_dirty(addr1
)) {
3594 /* invalidate code */
3595 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
3597 cpu_physical_memory_set_dirty_flags(
3598 addr1
, (0xff & ~CODE_DIRTY_FLAG
));
3600 qemu_put_ram_ptr(ptr
);
3603 if (!is_ram_rom_romd(pd
)) {
3604 target_phys_addr_t addr1
;
3606 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3607 addr1
= (addr
& ~TARGET_PAGE_MASK
) + p
.region_offset
;
3608 if (l
>= 4 && ((addr1
& 3) == 0)) {
3609 /* 32 bit read access */
3610 val
= io_mem_read(io_index
, addr1
, 4);
3613 } else if (l
>= 2 && ((addr1
& 1) == 0)) {
3614 /* 16 bit read access */
3615 val
= io_mem_read(io_index
, addr1
, 2);
3619 /* 8 bit read access */
3620 val
= io_mem_read(io_index
, addr1
, 1);
3626 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
);
3627 memcpy(buf
, ptr
+ (addr
& ~TARGET_PAGE_MASK
), l
);
3628 qemu_put_ram_ptr(ptr
);
3637 /* used for ROM loading : can write in RAM and ROM */
3638 void cpu_physical_memory_write_rom(target_phys_addr_t addr
,
3639 const uint8_t *buf
, int len
)
3643 target_phys_addr_t page
;
3648 page
= addr
& TARGET_PAGE_MASK
;
3649 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3652 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3655 if (!is_ram_rom_romd(pd
)) {
3658 unsigned long addr1
;
3659 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3661 ptr
= qemu_get_ram_ptr(addr1
);
3662 memcpy(ptr
, buf
, l
);
3663 qemu_put_ram_ptr(ptr
);
3673 target_phys_addr_t addr
;
3674 target_phys_addr_t len
;
3677 static BounceBuffer bounce
;
3679 typedef struct MapClient
{
3681 void (*callback
)(void *opaque
);
3682 QLIST_ENTRY(MapClient
) link
;
3685 static QLIST_HEAD(map_client_list
, MapClient
) map_client_list
3686 = QLIST_HEAD_INITIALIZER(map_client_list
);
3688 void *cpu_register_map_client(void *opaque
, void (*callback
)(void *opaque
))
3690 MapClient
*client
= g_malloc(sizeof(*client
));
3692 client
->opaque
= opaque
;
3693 client
->callback
= callback
;
3694 QLIST_INSERT_HEAD(&map_client_list
, client
, link
);
3698 void cpu_unregister_map_client(void *_client
)
3700 MapClient
*client
= (MapClient
*)_client
;
3702 QLIST_REMOVE(client
, link
);
3706 static void cpu_notify_map_clients(void)
3710 while (!QLIST_EMPTY(&map_client_list
)) {
3711 client
= QLIST_FIRST(&map_client_list
);
3712 client
->callback(client
->opaque
);
3713 cpu_unregister_map_client(client
);
3717 /* Map a physical memory region into a host virtual address.
3718 * May map a subset of the requested range, given by and returned in *plen.
3719 * May return NULL if resources needed to perform the mapping are exhausted.
3720 * Use only for reads OR writes - not for read-modify-write operations.
3721 * Use cpu_register_map_client() to know when retrying the map operation is
3722 * likely to succeed.
3724 void *cpu_physical_memory_map(target_phys_addr_t addr
,
3725 target_phys_addr_t
*plen
,
3728 target_phys_addr_t len
= *plen
;
3729 target_phys_addr_t todo
= 0;
3731 target_phys_addr_t page
;
3734 ram_addr_t raddr
= RAM_ADDR_MAX
;
3739 page
= addr
& TARGET_PAGE_MASK
;
3740 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3743 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3746 if ((pd
& ~TARGET_PAGE_MASK
) != io_mem_ram
.ram_addr
) {
3747 if (todo
|| bounce
.buffer
) {
3750 bounce
.buffer
= qemu_memalign(TARGET_PAGE_SIZE
, TARGET_PAGE_SIZE
);
3754 cpu_physical_memory_read(addr
, bounce
.buffer
, l
);
3758 return bounce
.buffer
;
3761 raddr
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3769 ret
= qemu_ram_ptr_length(raddr
, &rlen
);
3774 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3775 * Will also mark the memory as dirty if is_write == 1. access_len gives
3776 * the amount of memory that was actually read or written by the caller.
3778 void cpu_physical_memory_unmap(void *buffer
, target_phys_addr_t len
,
3779 int is_write
, target_phys_addr_t access_len
)
3781 if (buffer
!= bounce
.buffer
) {
3783 ram_addr_t addr1
= qemu_ram_addr_from_host_nofail(buffer
);
3784 while (access_len
) {
3786 l
= TARGET_PAGE_SIZE
;
3789 if (!cpu_physical_memory_is_dirty(addr1
)) {
3790 /* invalidate code */
3791 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
3793 cpu_physical_memory_set_dirty_flags(
3794 addr1
, (0xff & ~CODE_DIRTY_FLAG
));
3800 if (xen_enabled()) {
3801 xen_invalidate_map_cache_entry(buffer
);
3806 cpu_physical_memory_write(bounce
.addr
, bounce
.buffer
, access_len
);
3808 qemu_vfree(bounce
.buffer
);
3809 bounce
.buffer
= NULL
;
3810 cpu_notify_map_clients();
3813 /* warning: addr must be aligned */
3814 static inline uint32_t ldl_phys_internal(target_phys_addr_t addr
,
3815 enum device_endian endian
)
3823 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3826 if (!is_ram_rom_romd(pd
)) {
3828 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3829 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
.region_offset
;
3830 val
= io_mem_read(io_index
, addr
, 4);
3831 #if defined(TARGET_WORDS_BIGENDIAN)
3832 if (endian
== DEVICE_LITTLE_ENDIAN
) {
3836 if (endian
== DEVICE_BIG_ENDIAN
) {
3842 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3843 (addr
& ~TARGET_PAGE_MASK
);
3845 case DEVICE_LITTLE_ENDIAN
:
3846 val
= ldl_le_p(ptr
);
3848 case DEVICE_BIG_ENDIAN
:
3849 val
= ldl_be_p(ptr
);
3859 uint32_t ldl_phys(target_phys_addr_t addr
)
3861 return ldl_phys_internal(addr
, DEVICE_NATIVE_ENDIAN
);
3864 uint32_t ldl_le_phys(target_phys_addr_t addr
)
3866 return ldl_phys_internal(addr
, DEVICE_LITTLE_ENDIAN
);
3869 uint32_t ldl_be_phys(target_phys_addr_t addr
)
3871 return ldl_phys_internal(addr
, DEVICE_BIG_ENDIAN
);
3874 /* warning: addr must be aligned */
3875 static inline uint64_t ldq_phys_internal(target_phys_addr_t addr
,
3876 enum device_endian endian
)
3884 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3887 if (!is_ram_rom_romd(pd
)) {
3889 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3890 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
.region_offset
;
3892 /* XXX This is broken when device endian != cpu endian.
3893 Fix and add "endian" variable check */
3894 #ifdef TARGET_WORDS_BIGENDIAN
3895 val
= io_mem_read(io_index
, addr
, 4) << 32;
3896 val
|= io_mem_read(io_index
, addr
+ 4, 4);
3898 val
= io_mem_read(io_index
, addr
, 4);
3899 val
|= io_mem_read(io_index
, addr
+ 4, 4) << 32;
3903 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3904 (addr
& ~TARGET_PAGE_MASK
);
3906 case DEVICE_LITTLE_ENDIAN
:
3907 val
= ldq_le_p(ptr
);
3909 case DEVICE_BIG_ENDIAN
:
3910 val
= ldq_be_p(ptr
);
3920 uint64_t ldq_phys(target_phys_addr_t addr
)
3922 return ldq_phys_internal(addr
, DEVICE_NATIVE_ENDIAN
);
3925 uint64_t ldq_le_phys(target_phys_addr_t addr
)
3927 return ldq_phys_internal(addr
, DEVICE_LITTLE_ENDIAN
);
3930 uint64_t ldq_be_phys(target_phys_addr_t addr
)
3932 return ldq_phys_internal(addr
, DEVICE_BIG_ENDIAN
);
3936 uint32_t ldub_phys(target_phys_addr_t addr
)
3939 cpu_physical_memory_read(addr
, &val
, 1);
3943 /* warning: addr must be aligned */
3944 static inline uint32_t lduw_phys_internal(target_phys_addr_t addr
,
3945 enum device_endian endian
)
3953 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3956 if (!is_ram_rom_romd(pd
)) {
3958 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3959 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
.region_offset
;
3960 val
= io_mem_read(io_index
, addr
, 2);
3961 #if defined(TARGET_WORDS_BIGENDIAN)
3962 if (endian
== DEVICE_LITTLE_ENDIAN
) {
3966 if (endian
== DEVICE_BIG_ENDIAN
) {
3972 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3973 (addr
& ~TARGET_PAGE_MASK
);
3975 case DEVICE_LITTLE_ENDIAN
:
3976 val
= lduw_le_p(ptr
);
3978 case DEVICE_BIG_ENDIAN
:
3979 val
= lduw_be_p(ptr
);
3989 uint32_t lduw_phys(target_phys_addr_t addr
)
3991 return lduw_phys_internal(addr
, DEVICE_NATIVE_ENDIAN
);
3994 uint32_t lduw_le_phys(target_phys_addr_t addr
)
3996 return lduw_phys_internal(addr
, DEVICE_LITTLE_ENDIAN
);
3999 uint32_t lduw_be_phys(target_phys_addr_t addr
)
4001 return lduw_phys_internal(addr
, DEVICE_BIG_ENDIAN
);
4004 /* warning: addr must be aligned. The ram page is not masked as dirty
4005 and the code inside is not invalidated. It is useful if the dirty
4006 bits are used to track modified PTEs */
4007 void stl_phys_notdirty(target_phys_addr_t addr
, uint32_t val
)
4014 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
4017 if ((pd
& ~TARGET_PAGE_MASK
) != io_mem_ram
.ram_addr
) {
4018 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
4019 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
.region_offset
;
4020 io_mem_write(io_index
, addr
, val
, 4);
4022 unsigned long addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
4023 ptr
= qemu_get_ram_ptr(addr1
);
4026 if (unlikely(in_migration
)) {
4027 if (!cpu_physical_memory_is_dirty(addr1
)) {
4028 /* invalidate code */
4029 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
4031 cpu_physical_memory_set_dirty_flags(
4032 addr1
, (0xff & ~CODE_DIRTY_FLAG
));
4038 void stq_phys_notdirty(target_phys_addr_t addr
, uint64_t val
)
4045 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
4048 if ((pd
& ~TARGET_PAGE_MASK
) != io_mem_ram
.ram_addr
) {
4049 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
4050 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
.region_offset
;
4051 #ifdef TARGET_WORDS_BIGENDIAN
4052 io_mem_write(io_index
, addr
, val
>> 32, 4);
4053 io_mem_write(io_index
, addr
+ 4, (uint32_t)val
, 4);
4055 io_mem_write(io_index
, addr
, (uint32_t)val
, 4);
4056 io_mem_write(io_index
, addr
+ 4, val
>> 32, 4);
4059 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
4060 (addr
& ~TARGET_PAGE_MASK
);
4065 /* warning: addr must be aligned */
4066 static inline void stl_phys_internal(target_phys_addr_t addr
, uint32_t val
,
4067 enum device_endian endian
)
4074 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
4077 if ((pd
& ~TARGET_PAGE_MASK
) != io_mem_ram
.ram_addr
) {
4078 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
4079 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
.region_offset
;
4080 #if defined(TARGET_WORDS_BIGENDIAN)
4081 if (endian
== DEVICE_LITTLE_ENDIAN
) {
4085 if (endian
== DEVICE_BIG_ENDIAN
) {
4089 io_mem_write(io_index
, addr
, val
, 4);
4091 unsigned long addr1
;
4092 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
4094 ptr
= qemu_get_ram_ptr(addr1
);
4096 case DEVICE_LITTLE_ENDIAN
:
4099 case DEVICE_BIG_ENDIAN
:
4106 if (!cpu_physical_memory_is_dirty(addr1
)) {
4107 /* invalidate code */
4108 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
4110 cpu_physical_memory_set_dirty_flags(addr1
,
4111 (0xff & ~CODE_DIRTY_FLAG
));
4116 void stl_phys(target_phys_addr_t addr
, uint32_t val
)
4118 stl_phys_internal(addr
, val
, DEVICE_NATIVE_ENDIAN
);
4121 void stl_le_phys(target_phys_addr_t addr
, uint32_t val
)
4123 stl_phys_internal(addr
, val
, DEVICE_LITTLE_ENDIAN
);
4126 void stl_be_phys(target_phys_addr_t addr
, uint32_t val
)
4128 stl_phys_internal(addr
, val
, DEVICE_BIG_ENDIAN
);
4132 void stb_phys(target_phys_addr_t addr
, uint32_t val
)
4135 cpu_physical_memory_write(addr
, &v
, 1);
4138 /* warning: addr must be aligned */
4139 static inline void stw_phys_internal(target_phys_addr_t addr
, uint32_t val
,
4140 enum device_endian endian
)
4147 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
4150 if ((pd
& ~TARGET_PAGE_MASK
) != io_mem_ram
.ram_addr
) {
4151 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
4152 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
.region_offset
;
4153 #if defined(TARGET_WORDS_BIGENDIAN)
4154 if (endian
== DEVICE_LITTLE_ENDIAN
) {
4158 if (endian
== DEVICE_BIG_ENDIAN
) {
4162 io_mem_write(io_index
, addr
, val
, 2);
4164 unsigned long addr1
;
4165 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
4167 ptr
= qemu_get_ram_ptr(addr1
);
4169 case DEVICE_LITTLE_ENDIAN
:
4172 case DEVICE_BIG_ENDIAN
:
4179 if (!cpu_physical_memory_is_dirty(addr1
)) {
4180 /* invalidate code */
4181 tb_invalidate_phys_page_range(addr1
, addr1
+ 2, 0);
4183 cpu_physical_memory_set_dirty_flags(addr1
,
4184 (0xff & ~CODE_DIRTY_FLAG
));
4189 void stw_phys(target_phys_addr_t addr
, uint32_t val
)
4191 stw_phys_internal(addr
, val
, DEVICE_NATIVE_ENDIAN
);
4194 void stw_le_phys(target_phys_addr_t addr
, uint32_t val
)
4196 stw_phys_internal(addr
, val
, DEVICE_LITTLE_ENDIAN
);
4199 void stw_be_phys(target_phys_addr_t addr
, uint32_t val
)
4201 stw_phys_internal(addr
, val
, DEVICE_BIG_ENDIAN
);
4205 void stq_phys(target_phys_addr_t addr
, uint64_t val
)
4208 cpu_physical_memory_write(addr
, &val
, 8);
4211 void stq_le_phys(target_phys_addr_t addr
, uint64_t val
)
4213 val
= cpu_to_le64(val
);
4214 cpu_physical_memory_write(addr
, &val
, 8);
4217 void stq_be_phys(target_phys_addr_t addr
, uint64_t val
)
4219 val
= cpu_to_be64(val
);
4220 cpu_physical_memory_write(addr
, &val
, 8);
4223 /* virtual memory access for debug (includes writing to ROM) */
4224 int cpu_memory_rw_debug(CPUState
*env
, target_ulong addr
,
4225 uint8_t *buf
, int len
, int is_write
)
4228 target_phys_addr_t phys_addr
;
4232 page
= addr
& TARGET_PAGE_MASK
;
4233 phys_addr
= cpu_get_phys_page_debug(env
, page
);
4234 /* if no physical page mapped, return an error */
4235 if (phys_addr
== -1)
4237 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
4240 phys_addr
+= (addr
& ~TARGET_PAGE_MASK
);
4242 cpu_physical_memory_write_rom(phys_addr
, buf
, l
);
4244 cpu_physical_memory_rw(phys_addr
, buf
, l
, is_write
);
4253 /* in deterministic execution mode, instructions doing device I/Os
4254 must be at the end of the TB */
4255 void cpu_io_recompile(CPUState
*env
, void *retaddr
)
4257 TranslationBlock
*tb
;
4259 target_ulong pc
, cs_base
;
4262 tb
= tb_find_pc((unsigned long)retaddr
);
4264 cpu_abort(env
, "cpu_io_recompile: could not find TB for pc=%p",
4267 n
= env
->icount_decr
.u16
.low
+ tb
->icount
;
4268 cpu_restore_state(tb
, env
, (unsigned long)retaddr
);
4269 /* Calculate how many instructions had been executed before the fault
4271 n
= n
- env
->icount_decr
.u16
.low
;
4272 /* Generate a new TB ending on the I/O insn. */
4274 /* On MIPS and SH, delay slot instructions can only be restarted if
4275 they were already the first instruction in the TB. If this is not
4276 the first instruction in a TB then re-execute the preceding
4278 #if defined(TARGET_MIPS)
4279 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
4280 env
->active_tc
.PC
-= 4;
4281 env
->icount_decr
.u16
.low
++;
4282 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
4284 #elif defined(TARGET_SH4)
4285 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
4288 env
->icount_decr
.u16
.low
++;
4289 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
4292 /* This should never happen. */
4293 if (n
> CF_COUNT_MASK
)
4294 cpu_abort(env
, "TB too big during recompile");
4296 cflags
= n
| CF_LAST_IO
;
4298 cs_base
= tb
->cs_base
;
4300 tb_phys_invalidate(tb
, -1);
4301 /* FIXME: In theory this could raise an exception. In practice
4302 we have already translated the block once so it's probably ok. */
4303 tb_gen_code(env
, pc
, cs_base
, flags
, cflags
);
4304 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
4305 the first in the TB) then we end up generating a whole new TB and
4306 repeating the fault, which is horribly inefficient.
4307 Better would be to execute just this insn uncached, or generate a
4309 cpu_resume_from_signal(env
, NULL
);
4312 #if !defined(CONFIG_USER_ONLY)
4314 void dump_exec_info(FILE *f
, fprintf_function cpu_fprintf
)
4316 int i
, target_code_size
, max_target_code_size
;
4317 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
4318 TranslationBlock
*tb
;
4320 target_code_size
= 0;
4321 max_target_code_size
= 0;
4323 direct_jmp_count
= 0;
4324 direct_jmp2_count
= 0;
4325 for(i
= 0; i
< nb_tbs
; i
++) {
4327 target_code_size
+= tb
->size
;
4328 if (tb
->size
> max_target_code_size
)
4329 max_target_code_size
= tb
->size
;
4330 if (tb
->page_addr
[1] != -1)
4332 if (tb
->tb_next_offset
[0] != 0xffff) {
4334 if (tb
->tb_next_offset
[1] != 0xffff) {
4335 direct_jmp2_count
++;
4339 /* XXX: avoid using doubles ? */
4340 cpu_fprintf(f
, "Translation buffer state:\n");
4341 cpu_fprintf(f
, "gen code size %td/%ld\n",
4342 code_gen_ptr
- code_gen_buffer
, code_gen_buffer_max_size
);
4343 cpu_fprintf(f
, "TB count %d/%d\n",
4344 nb_tbs
, code_gen_max_blocks
);
4345 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
4346 nb_tbs
? target_code_size
/ nb_tbs
: 0,
4347 max_target_code_size
);
4348 cpu_fprintf(f
, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
4349 nb_tbs
? (code_gen_ptr
- code_gen_buffer
) / nb_tbs
: 0,
4350 target_code_size
? (double) (code_gen_ptr
- code_gen_buffer
) / target_code_size
: 0);
4351 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n",
4353 nb_tbs
? (cross_page
* 100) / nb_tbs
: 0);
4354 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
4356 nb_tbs
? (direct_jmp_count
* 100) / nb_tbs
: 0,
4358 nb_tbs
? (direct_jmp2_count
* 100) / nb_tbs
: 0);
4359 cpu_fprintf(f
, "\nStatistics:\n");
4360 cpu_fprintf(f
, "TB flush count %d\n", tb_flush_count
);
4361 cpu_fprintf(f
, "TB invalidate count %d\n", tb_phys_invalidate_count
);
4362 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
4363 tcg_dump_info(f
, cpu_fprintf
);
4366 /* NOTE: this function can trigger an exception */
4367 /* NOTE2: the returned address is not exactly the physical address: it
4368 is the offset relative to phys_ram_base */
4369 tb_page_addr_t
get_page_addr_code(CPUState
*env1
, target_ulong addr
)
4371 int mmu_idx
, page_index
, pd
;
4374 page_index
= (addr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
4375 mmu_idx
= cpu_mmu_index(env1
);
4376 if (unlikely(env1
->tlb_table
[mmu_idx
][page_index
].addr_code
!=
4377 (addr
& TARGET_PAGE_MASK
))) {
4380 pd
= env1
->tlb_table
[mmu_idx
][page_index
].addr_code
& ~TARGET_PAGE_MASK
;
4381 if (pd
!= io_mem_ram
.ram_addr
&& pd
!= io_mem_rom
.ram_addr
4382 && !(pd
& IO_MEM_ROMD
)) {
4383 #if defined(TARGET_ALPHA) || defined(TARGET_MIPS) || defined(TARGET_SPARC)
4384 cpu_unassigned_access(env1
, addr
, 0, 1, 0, 4);
4386 cpu_abort(env1
, "Trying to execute code outside RAM or ROM at 0x" TARGET_FMT_lx
"\n", addr
);
4389 p
= (void *)((uintptr_t)addr
+ env1
->tlb_table
[mmu_idx
][page_index
].addend
);
4390 return qemu_ram_addr_from_host_nofail(p
);
4393 #define MMUSUFFIX _cmmu
4395 #define GETPC() NULL
4396 #define env cpu_single_env
4397 #define SOFTMMU_CODE_ACCESS
4400 #include "softmmu_template.h"
4403 #include "softmmu_template.h"
4406 #include "softmmu_template.h"
4409 #include "softmmu_template.h"