2 * virtual page mapping and translated block handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
23 #include <sys/types.h>
27 #include "qemu-common.h"
35 #include "qemu-timer.h"
37 #include "exec-memory.h"
38 #if defined(CONFIG_USER_ONLY)
40 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
41 #include <sys/param.h>
42 #if __FreeBSD_version >= 700104
43 #define HAVE_KINFO_GETVMMAP
44 #define sigqueue sigqueue_freebsd /* avoid redefinition */
47 #include <machine/profile.h>
55 #else /* !CONFIG_USER_ONLY */
56 #include "xen-mapcache.h"
60 #define WANT_EXEC_OBSOLETE
61 #include "exec-obsolete.h"
63 //#define DEBUG_TB_INVALIDATE
66 //#define DEBUG_UNASSIGNED
68 /* make various TB consistency checks */
69 //#define DEBUG_TB_CHECK
70 //#define DEBUG_TLB_CHECK
72 //#define DEBUG_IOPORT
73 //#define DEBUG_SUBPAGE
75 #if !defined(CONFIG_USER_ONLY)
76 /* TB consistency checks only implemented for usermode emulation. */
80 #define SMC_BITMAP_USE_THRESHOLD 10
82 static TranslationBlock
*tbs
;
83 static int code_gen_max_blocks
;
84 TranslationBlock
*tb_phys_hash
[CODE_GEN_PHYS_HASH_SIZE
];
86 /* any access to the tbs or the page table must use this lock */
87 spinlock_t tb_lock
= SPIN_LOCK_UNLOCKED
;
89 #if defined(__arm__) || defined(__sparc_v9__)
90 /* The prologue must be reachable with a direct jump. ARM and Sparc64
91 have limited branch ranges (possibly also PPC) so place it in a
92 section close to code segment. */
93 #define code_gen_section \
94 __attribute__((__section__(".gen_code"))) \
95 __attribute__((aligned (32)))
97 /* Maximum alignment for Win32 is 16. */
98 #define code_gen_section \
99 __attribute__((aligned (16)))
101 #define code_gen_section \
102 __attribute__((aligned (32)))
105 uint8_t code_gen_prologue
[1024] code_gen_section
;
106 static uint8_t *code_gen_buffer
;
107 static unsigned long code_gen_buffer_size
;
108 /* threshold to flush the translated code buffer */
109 static unsigned long code_gen_buffer_max_size
;
110 static uint8_t *code_gen_ptr
;
112 #if !defined(CONFIG_USER_ONLY)
114 static int in_migration
;
116 RAMList ram_list
= { .blocks
= QLIST_HEAD_INITIALIZER(ram_list
.blocks
) };
118 static MemoryRegion
*system_memory
;
119 static MemoryRegion
*system_io
;
121 MemoryRegion io_mem_ram
, io_mem_rom
, io_mem_unassigned
, io_mem_notdirty
;
122 static MemoryRegion io_mem_subpage_ram
;
127 /* current CPU in the current thread. It is only valid inside
129 DEFINE_TLS(CPUState
*,cpu_single_env
);
130 /* 0 = Do not count executed instructions.
131 1 = Precise instruction counting.
132 2 = Adaptive rate instruction counting. */
135 typedef struct PageDesc
{
136 /* list of TBs intersecting this ram page */
137 TranslationBlock
*first_tb
;
138 /* in order to optimize self modifying code, we count the number
139 of lookups we do to a given page to use a bitmap */
140 unsigned int code_write_count
;
141 uint8_t *code_bitmap
;
142 #if defined(CONFIG_USER_ONLY)
147 /* In system mode we want L1_MAP to be based on ram offsets,
148 while in user mode we want it to be based on virtual addresses. */
149 #if !defined(CONFIG_USER_ONLY)
150 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
151 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
153 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
156 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
159 /* Size of the L2 (and L3, etc) page tables. */
161 #define L2_SIZE (1 << L2_BITS)
163 #define P_L2_LEVELS \
164 (((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / L2_BITS) + 1)
166 /* The bits remaining after N lower levels of page tables. */
167 #define V_L1_BITS_REM \
168 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
170 #if V_L1_BITS_REM < 4
171 #define V_L1_BITS (V_L1_BITS_REM + L2_BITS)
173 #define V_L1_BITS V_L1_BITS_REM
176 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
178 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
180 unsigned long qemu_real_host_page_size
;
181 unsigned long qemu_host_page_size
;
182 unsigned long qemu_host_page_mask
;
184 /* This is a multi-level map on the virtual address space.
185 The bottom level has pointers to PageDesc. */
186 static void *l1_map
[V_L1_SIZE
];
188 #if !defined(CONFIG_USER_ONLY)
189 typedef struct PhysPageEntry PhysPageEntry
;
191 static MemoryRegionSection
*phys_sections
;
192 static unsigned phys_sections_nb
, phys_sections_nb_alloc
;
193 static uint16_t phys_section_unassigned
;
195 struct PhysPageEntry
{
197 uint16_t leaf
; /* index into phys_sections */
198 uint16_t node
; /* index into phys_map_nodes */
202 /* Simple allocator for PhysPageEntry nodes */
203 static PhysPageEntry (*phys_map_nodes
)[L2_SIZE
];
204 static unsigned phys_map_nodes_nb
, phys_map_nodes_nb_alloc
;
206 #define PHYS_MAP_NODE_NIL ((uint16_t)~0)
208 /* This is a multi-level map on the physical address space.
209 The bottom level has pointers to MemoryRegionSections. */
210 static PhysPageEntry phys_map
= { .u
.node
= PHYS_MAP_NODE_NIL
};
212 static void io_mem_init(void);
213 static void memory_map_init(void);
215 /* io memory support */
216 MemoryRegion
*io_mem_region
[IO_MEM_NB_ENTRIES
];
217 static char io_mem_used
[IO_MEM_NB_ENTRIES
];
218 static MemoryRegion io_mem_watch
;
223 static const char *logfilename
= "qemu.log";
225 static const char *logfilename
= "/tmp/qemu.log";
229 static int log_append
= 0;
232 #if !defined(CONFIG_USER_ONLY)
233 static int tlb_flush_count
;
235 static int tb_flush_count
;
236 static int tb_phys_invalidate_count
;
239 static void map_exec(void *addr
, long size
)
242 VirtualProtect(addr
, size
,
243 PAGE_EXECUTE_READWRITE
, &old_protect
);
247 static void map_exec(void *addr
, long size
)
249 unsigned long start
, end
, page_size
;
251 page_size
= getpagesize();
252 start
= (unsigned long)addr
;
253 start
&= ~(page_size
- 1);
255 end
= (unsigned long)addr
+ size
;
256 end
+= page_size
- 1;
257 end
&= ~(page_size
- 1);
259 mprotect((void *)start
, end
- start
,
260 PROT_READ
| PROT_WRITE
| PROT_EXEC
);
264 static void page_init(void)
266 /* NOTE: we can always suppose that qemu_host_page_size >=
270 SYSTEM_INFO system_info
;
272 GetSystemInfo(&system_info
);
273 qemu_real_host_page_size
= system_info
.dwPageSize
;
276 qemu_real_host_page_size
= getpagesize();
278 if (qemu_host_page_size
== 0)
279 qemu_host_page_size
= qemu_real_host_page_size
;
280 if (qemu_host_page_size
< TARGET_PAGE_SIZE
)
281 qemu_host_page_size
= TARGET_PAGE_SIZE
;
282 qemu_host_page_mask
= ~(qemu_host_page_size
- 1);
284 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
286 #ifdef HAVE_KINFO_GETVMMAP
287 struct kinfo_vmentry
*freep
;
290 freep
= kinfo_getvmmap(getpid(), &cnt
);
293 for (i
= 0; i
< cnt
; i
++) {
294 unsigned long startaddr
, endaddr
;
296 startaddr
= freep
[i
].kve_start
;
297 endaddr
= freep
[i
].kve_end
;
298 if (h2g_valid(startaddr
)) {
299 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
301 if (h2g_valid(endaddr
)) {
302 endaddr
= h2g(endaddr
);
303 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
305 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
307 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
318 last_brk
= (unsigned long)sbrk(0);
320 f
= fopen("/compat/linux/proc/self/maps", "r");
325 unsigned long startaddr
, endaddr
;
328 n
= fscanf (f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
330 if (n
== 2 && h2g_valid(startaddr
)) {
331 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
333 if (h2g_valid(endaddr
)) {
334 endaddr
= h2g(endaddr
);
338 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
350 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
356 #if defined(CONFIG_USER_ONLY)
357 /* We can't use g_malloc because it may recurse into a locked mutex. */
358 # define ALLOC(P, SIZE) \
360 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
361 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
364 # define ALLOC(P, SIZE) \
365 do { P = g_malloc0(SIZE); } while (0)
368 /* Level 1. Always allocated. */
369 lp
= l1_map
+ ((index
>> V_L1_SHIFT
) & (V_L1_SIZE
- 1));
372 for (i
= V_L1_SHIFT
/ L2_BITS
- 1; i
> 0; i
--) {
379 ALLOC(p
, sizeof(void *) * L2_SIZE
);
383 lp
= p
+ ((index
>> (i
* L2_BITS
)) & (L2_SIZE
- 1));
391 ALLOC(pd
, sizeof(PageDesc
) * L2_SIZE
);
397 return pd
+ (index
& (L2_SIZE
- 1));
400 static inline PageDesc
*page_find(tb_page_addr_t index
)
402 return page_find_alloc(index
, 0);
405 #if !defined(CONFIG_USER_ONLY)
407 static PhysPageEntry
*phys_map_node_alloc(uint16_t *ptr
)
412 /* Assign early to avoid the pointer being invalidated by g_renew() */
413 *ptr
= ret
= phys_map_nodes_nb
++;
414 assert(ret
!= PHYS_MAP_NODE_NIL
);
415 if (ret
== phys_map_nodes_nb_alloc
) {
416 typedef PhysPageEntry Node
[L2_SIZE
];
417 phys_map_nodes_nb_alloc
= MAX(phys_map_nodes_nb_alloc
* 2, 16);
418 phys_map_nodes
= g_renew(Node
, phys_map_nodes
,
419 phys_map_nodes_nb_alloc
);
421 for (i
= 0; i
< L2_SIZE
; ++i
) {
422 phys_map_nodes
[ret
][i
].u
.node
= PHYS_MAP_NODE_NIL
;
424 return phys_map_nodes
[ret
];
427 static void phys_map_nodes_reset(void)
429 phys_map_nodes_nb
= 0;
432 static uint16_t *phys_page_find_alloc(target_phys_addr_t index
, int alloc
)
434 PhysPageEntry
*lp
, *p
;
440 for (i
= P_L2_LEVELS
- 1; i
>= 0; i
--) {
441 if (lp
->u
.node
== PHYS_MAP_NODE_NIL
) {
445 p
= phys_map_node_alloc(&lp
->u
.node
);
447 for (j
= 0; j
< L2_SIZE
; j
++) {
448 p
[j
].u
.leaf
= phys_section_unassigned
;
452 p
= phys_map_nodes
[lp
->u
.node
];
454 lp
= &p
[(index
>> (i
* L2_BITS
)) & (L2_SIZE
- 1)];
460 static MemoryRegionSection
phys_page_find(target_phys_addr_t index
)
462 PhysPageEntry lp
= phys_map
;
465 MemoryRegionSection section
;
466 target_phys_addr_t delta
;
467 uint16_t s_index
= phys_section_unassigned
;
469 for (i
= P_L2_LEVELS
- 1; i
>= 0; i
--) {
470 if (lp
.u
.node
== PHYS_MAP_NODE_NIL
) {
473 p
= phys_map_nodes
[lp
.u
.node
];
474 lp
= p
[(index
>> (i
* L2_BITS
)) & (L2_SIZE
- 1)];
479 section
= phys_sections
[s_index
];
480 index
<<= TARGET_PAGE_BITS
;
481 assert(section
.offset_within_address_space
<= index
482 && index
<= section
.offset_within_address_space
+ section
.size
-1);
483 delta
= index
- section
.offset_within_address_space
;
484 section
.offset_within_address_space
+= delta
;
485 section
.offset_within_region
+= delta
;
486 section
.size
-= delta
;
490 static void tlb_protect_code(ram_addr_t ram_addr
);
491 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
493 #define mmap_lock() do { } while(0)
494 #define mmap_unlock() do { } while(0)
497 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
499 #if defined(CONFIG_USER_ONLY)
500 /* Currently it is not recommended to allocate big chunks of data in
501 user mode. It will change when a dedicated libc will be used */
502 #define USE_STATIC_CODE_GEN_BUFFER
505 #ifdef USE_STATIC_CODE_GEN_BUFFER
506 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
]
507 __attribute__((aligned (CODE_GEN_ALIGN
)));
510 static void code_gen_alloc(unsigned long tb_size
)
512 #ifdef USE_STATIC_CODE_GEN_BUFFER
513 code_gen_buffer
= static_code_gen_buffer
;
514 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
515 map_exec(code_gen_buffer
, code_gen_buffer_size
);
517 code_gen_buffer_size
= tb_size
;
518 if (code_gen_buffer_size
== 0) {
519 #if defined(CONFIG_USER_ONLY)
520 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
522 /* XXX: needs adjustments */
523 code_gen_buffer_size
= (unsigned long)(ram_size
/ 4);
526 if (code_gen_buffer_size
< MIN_CODE_GEN_BUFFER_SIZE
)
527 code_gen_buffer_size
= MIN_CODE_GEN_BUFFER_SIZE
;
528 /* The code gen buffer location may have constraints depending on
529 the host cpu and OS */
530 #if defined(__linux__)
535 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
536 #if defined(__x86_64__)
538 /* Cannot map more than that */
539 if (code_gen_buffer_size
> (800 * 1024 * 1024))
540 code_gen_buffer_size
= (800 * 1024 * 1024);
541 #elif defined(__sparc_v9__)
542 // Map the buffer below 2G, so we can use direct calls and branches
544 start
= (void *) 0x60000000UL
;
545 if (code_gen_buffer_size
> (512 * 1024 * 1024))
546 code_gen_buffer_size
= (512 * 1024 * 1024);
547 #elif defined(__arm__)
548 /* Keep the buffer no bigger than 16MB to branch between blocks */
549 if (code_gen_buffer_size
> 16 * 1024 * 1024)
550 code_gen_buffer_size
= 16 * 1024 * 1024;
551 #elif defined(__s390x__)
552 /* Map the buffer so that we can use direct calls and branches. */
553 /* We have a +- 4GB range on the branches; leave some slop. */
554 if (code_gen_buffer_size
> (3ul * 1024 * 1024 * 1024)) {
555 code_gen_buffer_size
= 3ul * 1024 * 1024 * 1024;
557 start
= (void *)0x90000000UL
;
559 code_gen_buffer
= mmap(start
, code_gen_buffer_size
,
560 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
562 if (code_gen_buffer
== MAP_FAILED
) {
563 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
567 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
568 || defined(__DragonFly__) || defined(__OpenBSD__) \
569 || defined(__NetBSD__)
573 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
574 #if defined(__x86_64__)
575 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
576 * 0x40000000 is free */
578 addr
= (void *)0x40000000;
579 /* Cannot map more than that */
580 if (code_gen_buffer_size
> (800 * 1024 * 1024))
581 code_gen_buffer_size
= (800 * 1024 * 1024);
582 #elif defined(__sparc_v9__)
583 // Map the buffer below 2G, so we can use direct calls and branches
585 addr
= (void *) 0x60000000UL
;
586 if (code_gen_buffer_size
> (512 * 1024 * 1024)) {
587 code_gen_buffer_size
= (512 * 1024 * 1024);
590 code_gen_buffer
= mmap(addr
, code_gen_buffer_size
,
591 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
593 if (code_gen_buffer
== MAP_FAILED
) {
594 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
599 code_gen_buffer
= g_malloc(code_gen_buffer_size
);
600 map_exec(code_gen_buffer
, code_gen_buffer_size
);
602 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
603 map_exec(code_gen_prologue
, sizeof(code_gen_prologue
));
604 code_gen_buffer_max_size
= code_gen_buffer_size
-
605 (TCG_MAX_OP_SIZE
* OPC_BUF_SIZE
);
606 code_gen_max_blocks
= code_gen_buffer_size
/ CODE_GEN_AVG_BLOCK_SIZE
;
607 tbs
= g_malloc(code_gen_max_blocks
* sizeof(TranslationBlock
));
610 /* Must be called before using the QEMU cpus. 'tb_size' is the size
611 (in bytes) allocated to the translation buffer. Zero means default
613 void tcg_exec_init(unsigned long tb_size
)
616 code_gen_alloc(tb_size
);
617 code_gen_ptr
= code_gen_buffer
;
619 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
620 /* There's no guest base to take into account, so go ahead and
621 initialize the prologue now. */
622 tcg_prologue_init(&tcg_ctx
);
626 bool tcg_enabled(void)
628 return code_gen_buffer
!= NULL
;
631 void cpu_exec_init_all(void)
633 #if !defined(CONFIG_USER_ONLY)
639 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
641 static int cpu_common_post_load(void *opaque
, int version_id
)
643 CPUState
*env
= opaque
;
645 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
646 version_id is increased. */
647 env
->interrupt_request
&= ~0x01;
653 static const VMStateDescription vmstate_cpu_common
= {
654 .name
= "cpu_common",
656 .minimum_version_id
= 1,
657 .minimum_version_id_old
= 1,
658 .post_load
= cpu_common_post_load
,
659 .fields
= (VMStateField
[]) {
660 VMSTATE_UINT32(halted
, CPUState
),
661 VMSTATE_UINT32(interrupt_request
, CPUState
),
662 VMSTATE_END_OF_LIST()
667 CPUState
*qemu_get_cpu(int cpu
)
669 CPUState
*env
= first_cpu
;
672 if (env
->cpu_index
== cpu
)
680 void cpu_exec_init(CPUState
*env
)
685 #if defined(CONFIG_USER_ONLY)
688 env
->next_cpu
= NULL
;
691 while (*penv
!= NULL
) {
692 penv
= &(*penv
)->next_cpu
;
695 env
->cpu_index
= cpu_index
;
697 QTAILQ_INIT(&env
->breakpoints
);
698 QTAILQ_INIT(&env
->watchpoints
);
699 #ifndef CONFIG_USER_ONLY
700 env
->thread_id
= qemu_get_thread_id();
703 #if defined(CONFIG_USER_ONLY)
706 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
707 vmstate_register(NULL
, cpu_index
, &vmstate_cpu_common
, env
);
708 register_savevm(NULL
, "cpu", cpu_index
, CPU_SAVE_VERSION
,
709 cpu_save
, cpu_load
, env
);
713 /* Allocate a new translation block. Flush the translation buffer if
714 too many translation blocks or too much generated code. */
715 static TranslationBlock
*tb_alloc(target_ulong pc
)
717 TranslationBlock
*tb
;
719 if (nb_tbs
>= code_gen_max_blocks
||
720 (code_gen_ptr
- code_gen_buffer
) >= code_gen_buffer_max_size
)
728 void tb_free(TranslationBlock
*tb
)
730 /* In practice this is mostly used for single use temporary TB
731 Ignore the hard cases and just back up if this TB happens to
732 be the last one generated. */
733 if (nb_tbs
> 0 && tb
== &tbs
[nb_tbs
- 1]) {
734 code_gen_ptr
= tb
->tc_ptr
;
739 static inline void invalidate_page_bitmap(PageDesc
*p
)
741 if (p
->code_bitmap
) {
742 g_free(p
->code_bitmap
);
743 p
->code_bitmap
= NULL
;
745 p
->code_write_count
= 0;
748 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
750 static void page_flush_tb_1 (int level
, void **lp
)
759 for (i
= 0; i
< L2_SIZE
; ++i
) {
760 pd
[i
].first_tb
= NULL
;
761 invalidate_page_bitmap(pd
+ i
);
765 for (i
= 0; i
< L2_SIZE
; ++i
) {
766 page_flush_tb_1 (level
- 1, pp
+ i
);
771 static void page_flush_tb(void)
774 for (i
= 0; i
< V_L1_SIZE
; i
++) {
775 page_flush_tb_1(V_L1_SHIFT
/ L2_BITS
- 1, l1_map
+ i
);
779 /* flush all the translation blocks */
780 /* XXX: tb_flush is currently not thread safe */
781 void tb_flush(CPUState
*env1
)
784 #if defined(DEBUG_FLUSH)
785 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
786 (unsigned long)(code_gen_ptr
- code_gen_buffer
),
788 ((unsigned long)(code_gen_ptr
- code_gen_buffer
)) / nb_tbs
: 0);
790 if ((unsigned long)(code_gen_ptr
- code_gen_buffer
) > code_gen_buffer_size
)
791 cpu_abort(env1
, "Internal error: code buffer overflow\n");
795 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
796 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
799 memset (tb_phys_hash
, 0, CODE_GEN_PHYS_HASH_SIZE
* sizeof (void *));
802 code_gen_ptr
= code_gen_buffer
;
803 /* XXX: flush processor icache at this point if cache flush is
808 #ifdef DEBUG_TB_CHECK
810 static void tb_invalidate_check(target_ulong address
)
812 TranslationBlock
*tb
;
814 address
&= TARGET_PAGE_MASK
;
815 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
816 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
817 if (!(address
+ TARGET_PAGE_SIZE
<= tb
->pc
||
818 address
>= tb
->pc
+ tb
->size
)) {
819 printf("ERROR invalidate: address=" TARGET_FMT_lx
820 " PC=%08lx size=%04x\n",
821 address
, (long)tb
->pc
, tb
->size
);
827 /* verify that all the pages have correct rights for code */
828 static void tb_page_check(void)
830 TranslationBlock
*tb
;
831 int i
, flags1
, flags2
;
833 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
834 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
835 flags1
= page_get_flags(tb
->pc
);
836 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
837 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
838 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
839 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
847 /* invalidate one TB */
848 static inline void tb_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
,
851 TranslationBlock
*tb1
;
855 *ptb
= *(TranslationBlock
**)((char *)tb1
+ next_offset
);
858 ptb
= (TranslationBlock
**)((char *)tb1
+ next_offset
);
862 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
864 TranslationBlock
*tb1
;
870 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
872 *ptb
= tb1
->page_next
[n1
];
875 ptb
= &tb1
->page_next
[n1
];
879 static inline void tb_jmp_remove(TranslationBlock
*tb
, int n
)
881 TranslationBlock
*tb1
, **ptb
;
884 ptb
= &tb
->jmp_next
[n
];
887 /* find tb(n) in circular list */
891 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
892 if (n1
== n
&& tb1
== tb
)
895 ptb
= &tb1
->jmp_first
;
897 ptb
= &tb1
->jmp_next
[n1
];
900 /* now we can suppress tb(n) from the list */
901 *ptb
= tb
->jmp_next
[n
];
903 tb
->jmp_next
[n
] = NULL
;
907 /* reset the jump entry 'n' of a TB so that it is not chained to
909 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
911 tb_set_jmp_target(tb
, n
, (unsigned long)(tb
->tc_ptr
+ tb
->tb_next_offset
[n
]));
914 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
919 tb_page_addr_t phys_pc
;
920 TranslationBlock
*tb1
, *tb2
;
922 /* remove the TB from the hash list */
923 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
924 h
= tb_phys_hash_func(phys_pc
);
925 tb_remove(&tb_phys_hash
[h
], tb
,
926 offsetof(TranslationBlock
, phys_hash_next
));
928 /* remove the TB from the page list */
929 if (tb
->page_addr
[0] != page_addr
) {
930 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
931 tb_page_remove(&p
->first_tb
, tb
);
932 invalidate_page_bitmap(p
);
934 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
935 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
936 tb_page_remove(&p
->first_tb
, tb
);
937 invalidate_page_bitmap(p
);
940 tb_invalidated_flag
= 1;
942 /* remove the TB from the hash list */
943 h
= tb_jmp_cache_hash_func(tb
->pc
);
944 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
945 if (env
->tb_jmp_cache
[h
] == tb
)
946 env
->tb_jmp_cache
[h
] = NULL
;
949 /* suppress this TB from the two jump lists */
950 tb_jmp_remove(tb
, 0);
951 tb_jmp_remove(tb
, 1);
953 /* suppress any remaining jumps to this TB */
959 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
960 tb2
= tb1
->jmp_next
[n1
];
961 tb_reset_jump(tb1
, n1
);
962 tb1
->jmp_next
[n1
] = NULL
;
965 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2); /* fail safe */
967 tb_phys_invalidate_count
++;
970 static inline void set_bits(uint8_t *tab
, int start
, int len
)
976 mask
= 0xff << (start
& 7);
977 if ((start
& ~7) == (end
& ~7)) {
979 mask
&= ~(0xff << (end
& 7));
984 start
= (start
+ 8) & ~7;
986 while (start
< end1
) {
991 mask
= ~(0xff << (end
& 7));
997 static void build_page_bitmap(PageDesc
*p
)
999 int n
, tb_start
, tb_end
;
1000 TranslationBlock
*tb
;
1002 p
->code_bitmap
= g_malloc0(TARGET_PAGE_SIZE
/ 8);
1005 while (tb
!= NULL
) {
1007 tb
= (TranslationBlock
*)((long)tb
& ~3);
1008 /* NOTE: this is subtle as a TB may span two physical pages */
1010 /* NOTE: tb_end may be after the end of the page, but
1011 it is not a problem */
1012 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
1013 tb_end
= tb_start
+ tb
->size
;
1014 if (tb_end
> TARGET_PAGE_SIZE
)
1015 tb_end
= TARGET_PAGE_SIZE
;
1018 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1020 set_bits(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
1021 tb
= tb
->page_next
[n
];
1025 TranslationBlock
*tb_gen_code(CPUState
*env
,
1026 target_ulong pc
, target_ulong cs_base
,
1027 int flags
, int cflags
)
1029 TranslationBlock
*tb
;
1031 tb_page_addr_t phys_pc
, phys_page2
;
1032 target_ulong virt_page2
;
1035 phys_pc
= get_page_addr_code(env
, pc
);
1038 /* flush must be done */
1040 /* cannot fail at this point */
1042 /* Don't forget to invalidate previous TB info. */
1043 tb_invalidated_flag
= 1;
1045 tc_ptr
= code_gen_ptr
;
1046 tb
->tc_ptr
= tc_ptr
;
1047 tb
->cs_base
= cs_base
;
1049 tb
->cflags
= cflags
;
1050 cpu_gen_code(env
, tb
, &code_gen_size
);
1051 code_gen_ptr
= (void *)(((unsigned long)code_gen_ptr
+ code_gen_size
+ CODE_GEN_ALIGN
- 1) & ~(CODE_GEN_ALIGN
- 1));
1053 /* check next page if needed */
1054 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
1056 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
1057 phys_page2
= get_page_addr_code(env
, virt_page2
);
1059 tb_link_page(tb
, phys_pc
, phys_page2
);
1063 /* invalidate all TBs which intersect with the target physical page
1064 starting in range [start;end[. NOTE: start and end must refer to
1065 the same physical page. 'is_cpu_write_access' should be true if called
1066 from a real cpu write access: the virtual CPU will exit the current
1067 TB if code is modified inside this TB. */
1068 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
1069 int is_cpu_write_access
)
1071 TranslationBlock
*tb
, *tb_next
, *saved_tb
;
1072 CPUState
*env
= cpu_single_env
;
1073 tb_page_addr_t tb_start
, tb_end
;
1076 #ifdef TARGET_HAS_PRECISE_SMC
1077 int current_tb_not_found
= is_cpu_write_access
;
1078 TranslationBlock
*current_tb
= NULL
;
1079 int current_tb_modified
= 0;
1080 target_ulong current_pc
= 0;
1081 target_ulong current_cs_base
= 0;
1082 int current_flags
= 0;
1083 #endif /* TARGET_HAS_PRECISE_SMC */
1085 p
= page_find(start
>> TARGET_PAGE_BITS
);
1088 if (!p
->code_bitmap
&&
1089 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
&&
1090 is_cpu_write_access
) {
1091 /* build code bitmap */
1092 build_page_bitmap(p
);
1095 /* we remove all the TBs in the range [start, end[ */
1096 /* XXX: see if in some cases it could be faster to invalidate all the code */
1098 while (tb
!= NULL
) {
1100 tb
= (TranslationBlock
*)((long)tb
& ~3);
1101 tb_next
= tb
->page_next
[n
];
1102 /* NOTE: this is subtle as a TB may span two physical pages */
1104 /* NOTE: tb_end may be after the end of the page, but
1105 it is not a problem */
1106 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1107 tb_end
= tb_start
+ tb
->size
;
1109 tb_start
= tb
->page_addr
[1];
1110 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1112 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1113 #ifdef TARGET_HAS_PRECISE_SMC
1114 if (current_tb_not_found
) {
1115 current_tb_not_found
= 0;
1117 if (env
->mem_io_pc
) {
1118 /* now we have a real cpu fault */
1119 current_tb
= tb_find_pc(env
->mem_io_pc
);
1122 if (current_tb
== tb
&&
1123 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1124 /* If we are modifying the current TB, we must stop
1125 its execution. We could be more precise by checking
1126 that the modification is after the current PC, but it
1127 would require a specialized function to partially
1128 restore the CPU state */
1130 current_tb_modified
= 1;
1131 cpu_restore_state(current_tb
, env
, env
->mem_io_pc
);
1132 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1135 #endif /* TARGET_HAS_PRECISE_SMC */
1136 /* we need to do that to handle the case where a signal
1137 occurs while doing tb_phys_invalidate() */
1140 saved_tb
= env
->current_tb
;
1141 env
->current_tb
= NULL
;
1143 tb_phys_invalidate(tb
, -1);
1145 env
->current_tb
= saved_tb
;
1146 if (env
->interrupt_request
&& env
->current_tb
)
1147 cpu_interrupt(env
, env
->interrupt_request
);
1152 #if !defined(CONFIG_USER_ONLY)
1153 /* if no code remaining, no need to continue to use slow writes */
1155 invalidate_page_bitmap(p
);
1156 if (is_cpu_write_access
) {
1157 tlb_unprotect_code_phys(env
, start
, env
->mem_io_vaddr
);
1161 #ifdef TARGET_HAS_PRECISE_SMC
1162 if (current_tb_modified
) {
1163 /* we generate a block containing just the instruction
1164 modifying the memory. It will ensure that it cannot modify
1166 env
->current_tb
= NULL
;
1167 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1168 cpu_resume_from_signal(env
, NULL
);
1173 /* len must be <= 8 and start must be a multiple of len */
1174 static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start
, int len
)
1180 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1181 cpu_single_env
->mem_io_vaddr
, len
,
1182 cpu_single_env
->eip
,
1183 cpu_single_env
->eip
+ (long)cpu_single_env
->segs
[R_CS
].base
);
1186 p
= page_find(start
>> TARGET_PAGE_BITS
);
1189 if (p
->code_bitmap
) {
1190 offset
= start
& ~TARGET_PAGE_MASK
;
1191 b
= p
->code_bitmap
[offset
>> 3] >> (offset
& 7);
1192 if (b
& ((1 << len
) - 1))
1196 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1200 #if !defined(CONFIG_SOFTMMU)
1201 static void tb_invalidate_phys_page(tb_page_addr_t addr
,
1202 unsigned long pc
, void *puc
)
1204 TranslationBlock
*tb
;
1207 #ifdef TARGET_HAS_PRECISE_SMC
1208 TranslationBlock
*current_tb
= NULL
;
1209 CPUState
*env
= cpu_single_env
;
1210 int current_tb_modified
= 0;
1211 target_ulong current_pc
= 0;
1212 target_ulong current_cs_base
= 0;
1213 int current_flags
= 0;
1216 addr
&= TARGET_PAGE_MASK
;
1217 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1221 #ifdef TARGET_HAS_PRECISE_SMC
1222 if (tb
&& pc
!= 0) {
1223 current_tb
= tb_find_pc(pc
);
1226 while (tb
!= NULL
) {
1228 tb
= (TranslationBlock
*)((long)tb
& ~3);
1229 #ifdef TARGET_HAS_PRECISE_SMC
1230 if (current_tb
== tb
&&
1231 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1232 /* If we are modifying the current TB, we must stop
1233 its execution. We could be more precise by checking
1234 that the modification is after the current PC, but it
1235 would require a specialized function to partially
1236 restore the CPU state */
1238 current_tb_modified
= 1;
1239 cpu_restore_state(current_tb
, env
, pc
);
1240 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1243 #endif /* TARGET_HAS_PRECISE_SMC */
1244 tb_phys_invalidate(tb
, addr
);
1245 tb
= tb
->page_next
[n
];
1248 #ifdef TARGET_HAS_PRECISE_SMC
1249 if (current_tb_modified
) {
1250 /* we generate a block containing just the instruction
1251 modifying the memory. It will ensure that it cannot modify
1253 env
->current_tb
= NULL
;
1254 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1255 cpu_resume_from_signal(env
, puc
);
1261 /* add the tb in the target page and protect it if necessary */
1262 static inline void tb_alloc_page(TranslationBlock
*tb
,
1263 unsigned int n
, tb_page_addr_t page_addr
)
1266 #ifndef CONFIG_USER_ONLY
1267 bool page_already_protected
;
1270 tb
->page_addr
[n
] = page_addr
;
1271 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
, 1);
1272 tb
->page_next
[n
] = p
->first_tb
;
1273 #ifndef CONFIG_USER_ONLY
1274 page_already_protected
= p
->first_tb
!= NULL
;
1276 p
->first_tb
= (TranslationBlock
*)((long)tb
| n
);
1277 invalidate_page_bitmap(p
);
1279 #if defined(TARGET_HAS_SMC) || 1
1281 #if defined(CONFIG_USER_ONLY)
1282 if (p
->flags
& PAGE_WRITE
) {
1287 /* force the host page as non writable (writes will have a
1288 page fault + mprotect overhead) */
1289 page_addr
&= qemu_host_page_mask
;
1291 for(addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1292 addr
+= TARGET_PAGE_SIZE
) {
1294 p2
= page_find (addr
>> TARGET_PAGE_BITS
);
1298 p2
->flags
&= ~PAGE_WRITE
;
1300 mprotect(g2h(page_addr
), qemu_host_page_size
,
1301 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1302 #ifdef DEBUG_TB_INVALIDATE
1303 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1308 /* if some code is already present, then the pages are already
1309 protected. So we handle the case where only the first TB is
1310 allocated in a physical page */
1311 if (!page_already_protected
) {
1312 tlb_protect_code(page_addr
);
1316 #endif /* TARGET_HAS_SMC */
1319 /* add a new TB and link it to the physical page tables. phys_page2 is
1320 (-1) to indicate that only one page contains the TB. */
1321 void tb_link_page(TranslationBlock
*tb
,
1322 tb_page_addr_t phys_pc
, tb_page_addr_t phys_page2
)
1325 TranslationBlock
**ptb
;
1327 /* Grab the mmap lock to stop another thread invalidating this TB
1328 before we are done. */
1330 /* add in the physical hash table */
1331 h
= tb_phys_hash_func(phys_pc
);
1332 ptb
= &tb_phys_hash
[h
];
1333 tb
->phys_hash_next
= *ptb
;
1336 /* add in the page list */
1337 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1338 if (phys_page2
!= -1)
1339 tb_alloc_page(tb
, 1, phys_page2
);
1341 tb
->page_addr
[1] = -1;
1343 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2);
1344 tb
->jmp_next
[0] = NULL
;
1345 tb
->jmp_next
[1] = NULL
;
1347 /* init original jump addresses */
1348 if (tb
->tb_next_offset
[0] != 0xffff)
1349 tb_reset_jump(tb
, 0);
1350 if (tb
->tb_next_offset
[1] != 0xffff)
1351 tb_reset_jump(tb
, 1);
1353 #ifdef DEBUG_TB_CHECK
1359 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1360 tb[1].tc_ptr. Return NULL if not found */
1361 TranslationBlock
*tb_find_pc(unsigned long tc_ptr
)
1363 int m_min
, m_max
, m
;
1365 TranslationBlock
*tb
;
1369 if (tc_ptr
< (unsigned long)code_gen_buffer
||
1370 tc_ptr
>= (unsigned long)code_gen_ptr
)
1372 /* binary search (cf Knuth) */
1375 while (m_min
<= m_max
) {
1376 m
= (m_min
+ m_max
) >> 1;
1378 v
= (unsigned long)tb
->tc_ptr
;
1381 else if (tc_ptr
< v
) {
1390 static void tb_reset_jump_recursive(TranslationBlock
*tb
);
1392 static inline void tb_reset_jump_recursive2(TranslationBlock
*tb
, int n
)
1394 TranslationBlock
*tb1
, *tb_next
, **ptb
;
1397 tb1
= tb
->jmp_next
[n
];
1399 /* find head of list */
1402 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1405 tb1
= tb1
->jmp_next
[n1
];
1407 /* we are now sure now that tb jumps to tb1 */
1410 /* remove tb from the jmp_first list */
1411 ptb
= &tb_next
->jmp_first
;
1415 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1416 if (n1
== n
&& tb1
== tb
)
1418 ptb
= &tb1
->jmp_next
[n1
];
1420 *ptb
= tb
->jmp_next
[n
];
1421 tb
->jmp_next
[n
] = NULL
;
1423 /* suppress the jump to next tb in generated code */
1424 tb_reset_jump(tb
, n
);
1426 /* suppress jumps in the tb on which we could have jumped */
1427 tb_reset_jump_recursive(tb_next
);
1431 static void tb_reset_jump_recursive(TranslationBlock
*tb
)
1433 tb_reset_jump_recursive2(tb
, 0);
1434 tb_reset_jump_recursive2(tb
, 1);
1437 #if defined(TARGET_HAS_ICE)
1438 #if defined(CONFIG_USER_ONLY)
1439 static void breakpoint_invalidate(CPUState
*env
, target_ulong pc
)
1441 tb_invalidate_phys_page_range(pc
, pc
+ 1, 0);
1444 static void breakpoint_invalidate(CPUState
*env
, target_ulong pc
)
1446 target_phys_addr_t addr
;
1447 ram_addr_t ram_addr
;
1448 MemoryRegionSection section
;
1450 addr
= cpu_get_phys_page_debug(env
, pc
);
1451 section
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
1452 if (!(memory_region_is_ram(section
.mr
)
1453 || (section
.mr
->rom_device
&& section
.mr
->readable
))) {
1456 ram_addr
= (memory_region_get_ram_addr(section
.mr
)
1457 + section
.offset_within_region
) & TARGET_PAGE_MASK
;
1458 ram_addr
|= (pc
& ~TARGET_PAGE_MASK
);
1459 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1462 #endif /* TARGET_HAS_ICE */
1464 #if defined(CONFIG_USER_ONLY)
1465 void cpu_watchpoint_remove_all(CPUState
*env
, int mask
)
1470 int cpu_watchpoint_insert(CPUState
*env
, target_ulong addr
, target_ulong len
,
1471 int flags
, CPUWatchpoint
**watchpoint
)
1476 /* Add a watchpoint. */
1477 int cpu_watchpoint_insert(CPUState
*env
, target_ulong addr
, target_ulong len
,
1478 int flags
, CPUWatchpoint
**watchpoint
)
1480 target_ulong len_mask
= ~(len
- 1);
1483 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1484 if ((len
!= 1 && len
!= 2 && len
!= 4 && len
!= 8) || (addr
& ~len_mask
)) {
1485 fprintf(stderr
, "qemu: tried to set invalid watchpoint at "
1486 TARGET_FMT_lx
", len=" TARGET_FMT_lu
"\n", addr
, len
);
1489 wp
= g_malloc(sizeof(*wp
));
1492 wp
->len_mask
= len_mask
;
1495 /* keep all GDB-injected watchpoints in front */
1497 QTAILQ_INSERT_HEAD(&env
->watchpoints
, wp
, entry
);
1499 QTAILQ_INSERT_TAIL(&env
->watchpoints
, wp
, entry
);
1501 tlb_flush_page(env
, addr
);
1508 /* Remove a specific watchpoint. */
1509 int cpu_watchpoint_remove(CPUState
*env
, target_ulong addr
, target_ulong len
,
1512 target_ulong len_mask
= ~(len
- 1);
1515 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1516 if (addr
== wp
->vaddr
&& len_mask
== wp
->len_mask
1517 && flags
== (wp
->flags
& ~BP_WATCHPOINT_HIT
)) {
1518 cpu_watchpoint_remove_by_ref(env
, wp
);
1525 /* Remove a specific watchpoint by reference. */
1526 void cpu_watchpoint_remove_by_ref(CPUState
*env
, CPUWatchpoint
*watchpoint
)
1528 QTAILQ_REMOVE(&env
->watchpoints
, watchpoint
, entry
);
1530 tlb_flush_page(env
, watchpoint
->vaddr
);
1535 /* Remove all matching watchpoints. */
1536 void cpu_watchpoint_remove_all(CPUState
*env
, int mask
)
1538 CPUWatchpoint
*wp
, *next
;
1540 QTAILQ_FOREACH_SAFE(wp
, &env
->watchpoints
, entry
, next
) {
1541 if (wp
->flags
& mask
)
1542 cpu_watchpoint_remove_by_ref(env
, wp
);
1547 /* Add a breakpoint. */
1548 int cpu_breakpoint_insert(CPUState
*env
, target_ulong pc
, int flags
,
1549 CPUBreakpoint
**breakpoint
)
1551 #if defined(TARGET_HAS_ICE)
1554 bp
= g_malloc(sizeof(*bp
));
1559 /* keep all GDB-injected breakpoints in front */
1561 QTAILQ_INSERT_HEAD(&env
->breakpoints
, bp
, entry
);
1563 QTAILQ_INSERT_TAIL(&env
->breakpoints
, bp
, entry
);
1565 breakpoint_invalidate(env
, pc
);
1575 /* Remove a specific breakpoint. */
1576 int cpu_breakpoint_remove(CPUState
*env
, target_ulong pc
, int flags
)
1578 #if defined(TARGET_HAS_ICE)
1581 QTAILQ_FOREACH(bp
, &env
->breakpoints
, entry
) {
1582 if (bp
->pc
== pc
&& bp
->flags
== flags
) {
1583 cpu_breakpoint_remove_by_ref(env
, bp
);
1593 /* Remove a specific breakpoint by reference. */
1594 void cpu_breakpoint_remove_by_ref(CPUState
*env
, CPUBreakpoint
*breakpoint
)
1596 #if defined(TARGET_HAS_ICE)
1597 QTAILQ_REMOVE(&env
->breakpoints
, breakpoint
, entry
);
1599 breakpoint_invalidate(env
, breakpoint
->pc
);
1605 /* Remove all matching breakpoints. */
1606 void cpu_breakpoint_remove_all(CPUState
*env
, int mask
)
1608 #if defined(TARGET_HAS_ICE)
1609 CPUBreakpoint
*bp
, *next
;
1611 QTAILQ_FOREACH_SAFE(bp
, &env
->breakpoints
, entry
, next
) {
1612 if (bp
->flags
& mask
)
1613 cpu_breakpoint_remove_by_ref(env
, bp
);
1618 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1619 CPU loop after each instruction */
1620 void cpu_single_step(CPUState
*env
, int enabled
)
1622 #if defined(TARGET_HAS_ICE)
1623 if (env
->singlestep_enabled
!= enabled
) {
1624 env
->singlestep_enabled
= enabled
;
1626 kvm_update_guest_debug(env
, 0);
1628 /* must flush all the translated code to avoid inconsistencies */
1629 /* XXX: only flush what is necessary */
1636 /* enable or disable low levels log */
1637 void cpu_set_log(int log_flags
)
1639 loglevel
= log_flags
;
1640 if (loglevel
&& !logfile
) {
1641 logfile
= fopen(logfilename
, log_append
? "a" : "w");
1643 perror(logfilename
);
1646 #if !defined(CONFIG_SOFTMMU)
1647 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1649 static char logfile_buf
[4096];
1650 setvbuf(logfile
, logfile_buf
, _IOLBF
, sizeof(logfile_buf
));
1652 #elif defined(_WIN32)
1653 /* Win32 doesn't support line-buffering, so use unbuffered output. */
1654 setvbuf(logfile
, NULL
, _IONBF
, 0);
1656 setvbuf(logfile
, NULL
, _IOLBF
, 0);
1660 if (!loglevel
&& logfile
) {
1666 void cpu_set_log_filename(const char *filename
)
1668 logfilename
= strdup(filename
);
1673 cpu_set_log(loglevel
);
1676 static void cpu_unlink_tb(CPUState
*env
)
1678 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1679 problem and hope the cpu will stop of its own accord. For userspace
1680 emulation this often isn't actually as bad as it sounds. Often
1681 signals are used primarily to interrupt blocking syscalls. */
1682 TranslationBlock
*tb
;
1683 static spinlock_t interrupt_lock
= SPIN_LOCK_UNLOCKED
;
1685 spin_lock(&interrupt_lock
);
1686 tb
= env
->current_tb
;
1687 /* if the cpu is currently executing code, we must unlink it and
1688 all the potentially executing TB */
1690 env
->current_tb
= NULL
;
1691 tb_reset_jump_recursive(tb
);
1693 spin_unlock(&interrupt_lock
);
1696 #ifndef CONFIG_USER_ONLY
1697 /* mask must never be zero, except for A20 change call */
1698 static void tcg_handle_interrupt(CPUState
*env
, int mask
)
1702 old_mask
= env
->interrupt_request
;
1703 env
->interrupt_request
|= mask
;
1706 * If called from iothread context, wake the target cpu in
1709 if (!qemu_cpu_is_self(env
)) {
1715 env
->icount_decr
.u16
.high
= 0xffff;
1717 && (mask
& ~old_mask
) != 0) {
1718 cpu_abort(env
, "Raised interrupt while not in I/O function");
1725 CPUInterruptHandler cpu_interrupt_handler
= tcg_handle_interrupt
;
1727 #else /* CONFIG_USER_ONLY */
1729 void cpu_interrupt(CPUState
*env
, int mask
)
1731 env
->interrupt_request
|= mask
;
1734 #endif /* CONFIG_USER_ONLY */
1736 void cpu_reset_interrupt(CPUState
*env
, int mask
)
1738 env
->interrupt_request
&= ~mask
;
1741 void cpu_exit(CPUState
*env
)
1743 env
->exit_request
= 1;
1747 const CPULogItem cpu_log_items
[] = {
1748 { CPU_LOG_TB_OUT_ASM
, "out_asm",
1749 "show generated host assembly code for each compiled TB" },
1750 { CPU_LOG_TB_IN_ASM
, "in_asm",
1751 "show target assembly code for each compiled TB" },
1752 { CPU_LOG_TB_OP
, "op",
1753 "show micro ops for each compiled TB" },
1754 { CPU_LOG_TB_OP_OPT
, "op_opt",
1757 "before eflags optimization and "
1759 "after liveness analysis" },
1760 { CPU_LOG_INT
, "int",
1761 "show interrupts/exceptions in short format" },
1762 { CPU_LOG_EXEC
, "exec",
1763 "show trace before each executed TB (lots of logs)" },
1764 { CPU_LOG_TB_CPU
, "cpu",
1765 "show CPU state before block translation" },
1767 { CPU_LOG_PCALL
, "pcall",
1768 "show protected mode far calls/returns/exceptions" },
1769 { CPU_LOG_RESET
, "cpu_reset",
1770 "show CPU state before CPU resets" },
1773 { CPU_LOG_IOPORT
, "ioport",
1774 "show all i/o ports accesses" },
1779 static int cmp1(const char *s1
, int n
, const char *s2
)
1781 if (strlen(s2
) != n
)
1783 return memcmp(s1
, s2
, n
) == 0;
1786 /* takes a comma separated list of log masks. Return 0 if error. */
1787 int cpu_str_to_log_mask(const char *str
)
1789 const CPULogItem
*item
;
1796 p1
= strchr(p
, ',');
1799 if(cmp1(p
,p1
-p
,"all")) {
1800 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1804 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1805 if (cmp1(p
, p1
- p
, item
->name
))
1819 void cpu_abort(CPUState
*env
, const char *fmt
, ...)
1826 fprintf(stderr
, "qemu: fatal: ");
1827 vfprintf(stderr
, fmt
, ap
);
1828 fprintf(stderr
, "\n");
1830 cpu_dump_state(env
, stderr
, fprintf
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1832 cpu_dump_state(env
, stderr
, fprintf
, 0);
1834 if (qemu_log_enabled()) {
1835 qemu_log("qemu: fatal: ");
1836 qemu_log_vprintf(fmt
, ap2
);
1839 log_cpu_state(env
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1841 log_cpu_state(env
, 0);
1848 #if defined(CONFIG_USER_ONLY)
1850 struct sigaction act
;
1851 sigfillset(&act
.sa_mask
);
1852 act
.sa_handler
= SIG_DFL
;
1853 sigaction(SIGABRT
, &act
, NULL
);
1859 CPUState
*cpu_copy(CPUState
*env
)
1861 CPUState
*new_env
= cpu_init(env
->cpu_model_str
);
1862 CPUState
*next_cpu
= new_env
->next_cpu
;
1863 int cpu_index
= new_env
->cpu_index
;
1864 #if defined(TARGET_HAS_ICE)
1869 memcpy(new_env
, env
, sizeof(CPUState
));
1871 /* Preserve chaining and index. */
1872 new_env
->next_cpu
= next_cpu
;
1873 new_env
->cpu_index
= cpu_index
;
1875 /* Clone all break/watchpoints.
1876 Note: Once we support ptrace with hw-debug register access, make sure
1877 BP_CPU break/watchpoints are handled correctly on clone. */
1878 QTAILQ_INIT(&env
->breakpoints
);
1879 QTAILQ_INIT(&env
->watchpoints
);
1880 #if defined(TARGET_HAS_ICE)
1881 QTAILQ_FOREACH(bp
, &env
->breakpoints
, entry
) {
1882 cpu_breakpoint_insert(new_env
, bp
->pc
, bp
->flags
, NULL
);
1884 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1885 cpu_watchpoint_insert(new_env
, wp
->vaddr
, (~wp
->len_mask
) + 1,
1893 #if !defined(CONFIG_USER_ONLY)
1895 static inline void tlb_flush_jmp_cache(CPUState
*env
, target_ulong addr
)
1899 /* Discard jump cache entries for any tb which might potentially
1900 overlap the flushed page. */
1901 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1902 memset (&env
->tb_jmp_cache
[i
], 0,
1903 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1905 i
= tb_jmp_cache_hash_page(addr
);
1906 memset (&env
->tb_jmp_cache
[i
], 0,
1907 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1910 static CPUTLBEntry s_cputlb_empty_entry
= {
1918 * If flush_global is true (the usual case), flush all tlb entries.
1919 * If flush_global is false, flush (at least) all tlb entries not
1922 * Since QEMU doesn't currently implement a global/not-global flag
1923 * for tlb entries, at the moment tlb_flush() will also flush all
1924 * tlb entries in the flush_global == false case. This is OK because
1925 * CPU architectures generally permit an implementation to drop
1926 * entries from the TLB at any time, so flushing more entries than
1927 * required is only an efficiency issue, not a correctness issue.
1929 void tlb_flush(CPUState
*env
, int flush_global
)
1933 #if defined(DEBUG_TLB)
1934 printf("tlb_flush:\n");
1936 /* must reset current TB so that interrupts cannot modify the
1937 links while we are modifying them */
1938 env
->current_tb
= NULL
;
1940 for(i
= 0; i
< CPU_TLB_SIZE
; i
++) {
1942 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
1943 env
->tlb_table
[mmu_idx
][i
] = s_cputlb_empty_entry
;
1947 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
1949 env
->tlb_flush_addr
= -1;
1950 env
->tlb_flush_mask
= 0;
1954 static inline void tlb_flush_entry(CPUTLBEntry
*tlb_entry
, target_ulong addr
)
1956 if (addr
== (tlb_entry
->addr_read
&
1957 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1958 addr
== (tlb_entry
->addr_write
&
1959 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1960 addr
== (tlb_entry
->addr_code
&
1961 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
))) {
1962 *tlb_entry
= s_cputlb_empty_entry
;
1966 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
1971 #if defined(DEBUG_TLB)
1972 printf("tlb_flush_page: " TARGET_FMT_lx
"\n", addr
);
1974 /* Check if we need to flush due to large pages. */
1975 if ((addr
& env
->tlb_flush_mask
) == env
->tlb_flush_addr
) {
1976 #if defined(DEBUG_TLB)
1977 printf("tlb_flush_page: forced full flush ("
1978 TARGET_FMT_lx
"/" TARGET_FMT_lx
")\n",
1979 env
->tlb_flush_addr
, env
->tlb_flush_mask
);
1984 /* must reset current TB so that interrupts cannot modify the
1985 links while we are modifying them */
1986 env
->current_tb
= NULL
;
1988 addr
&= TARGET_PAGE_MASK
;
1989 i
= (addr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
1990 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++)
1991 tlb_flush_entry(&env
->tlb_table
[mmu_idx
][i
], addr
);
1993 tlb_flush_jmp_cache(env
, addr
);
1996 /* update the TLBs so that writes to code in the virtual page 'addr'
1998 static void tlb_protect_code(ram_addr_t ram_addr
)
2000 cpu_physical_memory_reset_dirty(ram_addr
,
2001 ram_addr
+ TARGET_PAGE_SIZE
,
2005 /* update the TLB so that writes in physical page 'phys_addr' are no longer
2006 tested for self modifying code */
2007 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
2010 cpu_physical_memory_set_dirty_flags(ram_addr
, CODE_DIRTY_FLAG
);
2013 static inline void tlb_reset_dirty_range(CPUTLBEntry
*tlb_entry
,
2014 unsigned long start
, unsigned long length
)
2017 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == io_mem_ram
.ram_addr
) {
2018 addr
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) + tlb_entry
->addend
;
2019 if ((addr
- start
) < length
) {
2020 tlb_entry
->addr_write
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) | TLB_NOTDIRTY
;
2025 /* Note: start and end must be within the same ram block. */
2026 void cpu_physical_memory_reset_dirty(ram_addr_t start
, ram_addr_t end
,
2030 unsigned long length
, start1
;
2033 start
&= TARGET_PAGE_MASK
;
2034 end
= TARGET_PAGE_ALIGN(end
);
2036 length
= end
- start
;
2039 cpu_physical_memory_mask_dirty_range(start
, length
, dirty_flags
);
2041 /* we modify the TLB cache so that the dirty bit will be set again
2042 when accessing the range */
2043 start1
= (unsigned long)qemu_safe_ram_ptr(start
);
2044 /* Check that we don't span multiple blocks - this breaks the
2045 address comparisons below. */
2046 if ((unsigned long)qemu_safe_ram_ptr(end
- 1) - start1
2047 != (end
- 1) - start
) {
2051 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
2053 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
2054 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
2055 tlb_reset_dirty_range(&env
->tlb_table
[mmu_idx
][i
],
2061 int cpu_physical_memory_set_dirty_tracking(int enable
)
2064 in_migration
= enable
;
2068 static inline void tlb_update_dirty(CPUTLBEntry
*tlb_entry
)
2070 ram_addr_t ram_addr
;
2073 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == io_mem_ram
.ram_addr
) {
2074 p
= (void *)(unsigned long)((tlb_entry
->addr_write
& TARGET_PAGE_MASK
)
2075 + tlb_entry
->addend
);
2076 ram_addr
= qemu_ram_addr_from_host_nofail(p
);
2077 if (!cpu_physical_memory_is_dirty(ram_addr
)) {
2078 tlb_entry
->addr_write
|= TLB_NOTDIRTY
;
2083 /* update the TLB according to the current state of the dirty bits */
2084 void cpu_tlb_update_dirty(CPUState
*env
)
2088 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
2089 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
2090 tlb_update_dirty(&env
->tlb_table
[mmu_idx
][i
]);
2094 static inline void tlb_set_dirty1(CPUTLBEntry
*tlb_entry
, target_ulong vaddr
)
2096 if (tlb_entry
->addr_write
== (vaddr
| TLB_NOTDIRTY
))
2097 tlb_entry
->addr_write
= vaddr
;
2100 /* update the TLB corresponding to virtual page vaddr
2101 so that it is no longer dirty */
2102 static inline void tlb_set_dirty(CPUState
*env
, target_ulong vaddr
)
2107 vaddr
&= TARGET_PAGE_MASK
;
2108 i
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
2109 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++)
2110 tlb_set_dirty1(&env
->tlb_table
[mmu_idx
][i
], vaddr
);
2113 /* Our TLB does not support large pages, so remember the area covered by
2114 large pages and trigger a full TLB flush if these are invalidated. */
2115 static void tlb_add_large_page(CPUState
*env
, target_ulong vaddr
,
2118 target_ulong mask
= ~(size
- 1);
2120 if (env
->tlb_flush_addr
== (target_ulong
)-1) {
2121 env
->tlb_flush_addr
= vaddr
& mask
;
2122 env
->tlb_flush_mask
= mask
;
2125 /* Extend the existing region to include the new page.
2126 This is a compromise between unnecessary flushes and the cost
2127 of maintaining a full variable size TLB. */
2128 mask
&= env
->tlb_flush_mask
;
2129 while (((env
->tlb_flush_addr
^ vaddr
) & mask
) != 0) {
2132 env
->tlb_flush_addr
&= mask
;
2133 env
->tlb_flush_mask
= mask
;
2136 static bool is_ram_rom(MemoryRegionSection
*s
)
2138 return memory_region_is_ram(s
->mr
);
2141 static bool is_romd(MemoryRegionSection
*s
)
2143 MemoryRegion
*mr
= s
->mr
;
2145 return mr
->rom_device
&& mr
->readable
;
2148 static bool is_ram_rom_romd(MemoryRegionSection
*s
)
2150 return is_ram_rom(s
) || is_romd(s
);
2153 /* Add a new TLB entry. At most one entry for a given virtual address
2154 is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
2155 supplied size is only used by tlb_flush_page. */
2156 void tlb_set_page(CPUState
*env
, target_ulong vaddr
,
2157 target_phys_addr_t paddr
, int prot
,
2158 int mmu_idx
, target_ulong size
)
2160 MemoryRegionSection section
;
2162 target_ulong address
;
2163 target_ulong code_address
;
2164 unsigned long addend
;
2167 target_phys_addr_t iotlb
;
2169 assert(size
>= TARGET_PAGE_SIZE
);
2170 if (size
!= TARGET_PAGE_SIZE
) {
2171 tlb_add_large_page(env
, vaddr
, size
);
2173 section
= phys_page_find(paddr
>> TARGET_PAGE_BITS
);
2174 #if defined(DEBUG_TLB)
2175 printf("tlb_set_page: vaddr=" TARGET_FMT_lx
" paddr=0x" TARGET_FMT_plx
2176 " prot=%x idx=%d pd=0x%08lx\n",
2177 vaddr
, paddr
, prot
, mmu_idx
, pd
);
2181 if (!is_ram_rom_romd(§ion
)) {
2182 /* IO memory case (romd handled later) */
2183 address
|= TLB_MMIO
;
2185 if (is_ram_rom_romd(§ion
)) {
2186 addend
= (unsigned long)(memory_region_get_ram_ptr(section
.mr
)
2187 + section
.offset_within_region
);
2191 if (is_ram_rom(§ion
)) {
2193 iotlb
= (memory_region_get_ram_addr(section
.mr
)
2194 + section
.offset_within_region
) & TARGET_PAGE_MASK
;
2195 if (!section
.readonly
)
2196 iotlb
|= io_mem_notdirty
.ram_addr
;
2198 iotlb
|= io_mem_rom
.ram_addr
;
2200 /* IO handlers are currently passed a physical address.
2201 It would be nice to pass an offset from the base address
2202 of that region. This would avoid having to special case RAM,
2203 and avoid full address decoding in every device.
2204 We can't use the high bits of pd for this because
2205 IO_MEM_ROMD uses these as a ram address. */
2206 iotlb
= memory_region_get_ram_addr(section
.mr
) & ~TARGET_PAGE_MASK
;
2207 iotlb
+= section
.offset_within_region
;
2210 code_address
= address
;
2211 /* Make accesses to pages with watchpoints go via the
2212 watchpoint trap routines. */
2213 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
2214 if (vaddr
== (wp
->vaddr
& TARGET_PAGE_MASK
)) {
2215 /* Avoid trapping reads of pages with a write breakpoint. */
2216 if ((prot
& PAGE_WRITE
) || (wp
->flags
& BP_MEM_READ
)) {
2217 iotlb
= io_mem_watch
.ram_addr
+ paddr
;
2218 address
|= TLB_MMIO
;
2224 index
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
2225 env
->iotlb
[mmu_idx
][index
] = iotlb
- vaddr
;
2226 te
= &env
->tlb_table
[mmu_idx
][index
];
2227 te
->addend
= addend
- vaddr
;
2228 if (prot
& PAGE_READ
) {
2229 te
->addr_read
= address
;
2234 if (prot
& PAGE_EXEC
) {
2235 te
->addr_code
= code_address
;
2239 if (prot
& PAGE_WRITE
) {
2240 if ((memory_region_is_ram(section
.mr
) && section
.readonly
)
2241 || is_romd(§ion
)) {
2242 /* Write access calls the I/O callback. */
2243 te
->addr_write
= address
| TLB_MMIO
;
2244 } else if (memory_region_is_ram(section
.mr
)
2245 && !cpu_physical_memory_is_dirty(
2246 section
.mr
->ram_addr
2247 + section
.offset_within_region
)) {
2248 te
->addr_write
= address
| TLB_NOTDIRTY
;
2250 te
->addr_write
= address
;
2253 te
->addr_write
= -1;
2259 void tlb_flush(CPUState
*env
, int flush_global
)
2263 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
2268 * Walks guest process memory "regions" one by one
2269 * and calls callback function 'fn' for each region.
2272 struct walk_memory_regions_data
2274 walk_memory_regions_fn fn
;
2276 unsigned long start
;
2280 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
2281 abi_ulong end
, int new_prot
)
2283 if (data
->start
!= -1ul) {
2284 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
2290 data
->start
= (new_prot
? end
: -1ul);
2291 data
->prot
= new_prot
;
2296 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
2297 abi_ulong base
, int level
, void **lp
)
2303 return walk_memory_regions_end(data
, base
, 0);
2308 for (i
= 0; i
< L2_SIZE
; ++i
) {
2309 int prot
= pd
[i
].flags
;
2311 pa
= base
| (i
<< TARGET_PAGE_BITS
);
2312 if (prot
!= data
->prot
) {
2313 rc
= walk_memory_regions_end(data
, pa
, prot
);
2321 for (i
= 0; i
< L2_SIZE
; ++i
) {
2322 pa
= base
| ((abi_ulong
)i
<<
2323 (TARGET_PAGE_BITS
+ L2_BITS
* level
));
2324 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
2334 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
2336 struct walk_memory_regions_data data
;
2344 for (i
= 0; i
< V_L1_SIZE
; i
++) {
2345 int rc
= walk_memory_regions_1(&data
, (abi_ulong
)i
<< V_L1_SHIFT
,
2346 V_L1_SHIFT
/ L2_BITS
- 1, l1_map
+ i
);
2352 return walk_memory_regions_end(&data
, 0, 0);
2355 static int dump_region(void *priv
, abi_ulong start
,
2356 abi_ulong end
, unsigned long prot
)
2358 FILE *f
= (FILE *)priv
;
2360 (void) fprintf(f
, TARGET_ABI_FMT_lx
"-"TARGET_ABI_FMT_lx
2361 " "TARGET_ABI_FMT_lx
" %c%c%c\n",
2362 start
, end
, end
- start
,
2363 ((prot
& PAGE_READ
) ? 'r' : '-'),
2364 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
2365 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
2370 /* dump memory mappings */
2371 void page_dump(FILE *f
)
2373 (void) fprintf(f
, "%-8s %-8s %-8s %s\n",
2374 "start", "end", "size", "prot");
2375 walk_memory_regions(f
, dump_region
);
2378 int page_get_flags(target_ulong address
)
2382 p
= page_find(address
>> TARGET_PAGE_BITS
);
2388 /* Modify the flags of a page and invalidate the code if necessary.
2389 The flag PAGE_WRITE_ORG is positioned automatically depending
2390 on PAGE_WRITE. The mmap_lock should already be held. */
2391 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
2393 target_ulong addr
, len
;
2395 /* This function should never be called with addresses outside the
2396 guest address space. If this assert fires, it probably indicates
2397 a missing call to h2g_valid. */
2398 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2399 assert(end
< ((abi_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2401 assert(start
< end
);
2403 start
= start
& TARGET_PAGE_MASK
;
2404 end
= TARGET_PAGE_ALIGN(end
);
2406 if (flags
& PAGE_WRITE
) {
2407 flags
|= PAGE_WRITE_ORG
;
2410 for (addr
= start
, len
= end
- start
;
2412 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2413 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2415 /* If the write protection bit is set, then we invalidate
2417 if (!(p
->flags
& PAGE_WRITE
) &&
2418 (flags
& PAGE_WRITE
) &&
2420 tb_invalidate_phys_page(addr
, 0, NULL
);
2426 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
2432 /* This function should never be called with addresses outside the
2433 guest address space. If this assert fires, it probably indicates
2434 a missing call to h2g_valid. */
2435 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2436 assert(start
< ((abi_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2442 if (start
+ len
- 1 < start
) {
2443 /* We've wrapped around. */
2447 end
= TARGET_PAGE_ALIGN(start
+len
); /* must do before we loose bits in the next step */
2448 start
= start
& TARGET_PAGE_MASK
;
2450 for (addr
= start
, len
= end
- start
;
2452 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2453 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2456 if( !(p
->flags
& PAGE_VALID
) )
2459 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
))
2461 if (flags
& PAGE_WRITE
) {
2462 if (!(p
->flags
& PAGE_WRITE_ORG
))
2464 /* unprotect the page if it was put read-only because it
2465 contains translated code */
2466 if (!(p
->flags
& PAGE_WRITE
)) {
2467 if (!page_unprotect(addr
, 0, NULL
))
2476 /* called from signal handler: invalidate the code and unprotect the
2477 page. Return TRUE if the fault was successfully handled. */
2478 int page_unprotect(target_ulong address
, unsigned long pc
, void *puc
)
2482 target_ulong host_start
, host_end
, addr
;
2484 /* Technically this isn't safe inside a signal handler. However we
2485 know this only ever happens in a synchronous SEGV handler, so in
2486 practice it seems to be ok. */
2489 p
= page_find(address
>> TARGET_PAGE_BITS
);
2495 /* if the page was really writable, then we change its
2496 protection back to writable */
2497 if ((p
->flags
& PAGE_WRITE_ORG
) && !(p
->flags
& PAGE_WRITE
)) {
2498 host_start
= address
& qemu_host_page_mask
;
2499 host_end
= host_start
+ qemu_host_page_size
;
2502 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2503 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2504 p
->flags
|= PAGE_WRITE
;
2507 /* and since the content will be modified, we must invalidate
2508 the corresponding translated code. */
2509 tb_invalidate_phys_page(addr
, pc
, puc
);
2510 #ifdef DEBUG_TB_CHECK
2511 tb_invalidate_check(addr
);
2514 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2524 static inline void tlb_set_dirty(CPUState
*env
,
2525 unsigned long addr
, target_ulong vaddr
)
2528 #endif /* defined(CONFIG_USER_ONLY) */
2530 #if !defined(CONFIG_USER_ONLY)
2532 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
2533 typedef struct subpage_t
{
2535 target_phys_addr_t base
;
2536 uint16_t sub_section
[TARGET_PAGE_SIZE
];
2539 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2541 static subpage_t
*subpage_init (target_phys_addr_t base
, uint16_t *section
,
2542 uint16_t orig_section
);
2543 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2546 if (addr > start_addr) \
2549 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2550 if (start_addr2 > 0) \
2554 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2555 end_addr2 = TARGET_PAGE_SIZE - 1; \
2557 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2558 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2563 static void destroy_page_desc(uint16_t section_index
)
2565 MemoryRegionSection
*section
= &phys_sections
[section_index
];
2566 MemoryRegion
*mr
= section
->mr
;
2569 subpage_t
*subpage
= container_of(mr
, subpage_t
, iomem
);
2570 memory_region_destroy(&subpage
->iomem
);
2575 static void destroy_l2_mapping(PhysPageEntry
*lp
, unsigned level
)
2580 if (lp
->u
.node
== PHYS_MAP_NODE_NIL
) {
2584 p
= phys_map_nodes
[lp
->u
.node
];
2585 for (i
= 0; i
< L2_SIZE
; ++i
) {
2587 destroy_l2_mapping(&p
[i
], level
- 1);
2589 destroy_page_desc(p
[i
].u
.leaf
);
2592 lp
->u
.node
= PHYS_MAP_NODE_NIL
;
2595 static void destroy_all_mappings(void)
2597 destroy_l2_mapping(&phys_map
, P_L2_LEVELS
- 1);
2598 phys_map_nodes_reset();
2601 static uint16_t phys_section_add(MemoryRegionSection
*section
)
2603 if (phys_sections_nb
== phys_sections_nb_alloc
) {
2604 phys_sections_nb_alloc
= MAX(phys_sections_nb_alloc
* 2, 16);
2605 phys_sections
= g_renew(MemoryRegionSection
, phys_sections
,
2606 phys_sections_nb_alloc
);
2608 phys_sections
[phys_sections_nb
] = *section
;
2609 return phys_sections_nb
++;
2612 static void phys_sections_clear(void)
2614 phys_sections_nb
= 0;
2617 /* register physical memory.
2618 For RAM, 'size' must be a multiple of the target page size.
2619 If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2620 io memory page. The address used when calling the IO function is
2621 the offset from the start of the region, plus region_offset. Both
2622 start_addr and region_offset are rounded down to a page boundary
2623 before calculating this offset. This should not be a problem unless
2624 the low bits of start_addr and region_offset differ. */
2625 void cpu_register_physical_memory_log(MemoryRegionSection
*section
,
2628 target_phys_addr_t start_addr
= section
->offset_within_address_space
;
2629 ram_addr_t size
= section
->size
;
2630 target_phys_addr_t addr
, end_addr
;
2631 ram_addr_t orig_size
= size
;
2633 uint16_t section_index
= phys_section_add(section
);
2637 size
= (size
+ TARGET_PAGE_SIZE
- 1) & TARGET_PAGE_MASK
;
2638 end_addr
= start_addr
+ (target_phys_addr_t
)size
;
2642 uint16_t *p
= phys_page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2643 uint16_t orig_memory
= *p
;
2644 target_phys_addr_t start_addr2
, end_addr2
;
2645 int need_subpage
= 0;
2646 MemoryRegion
*mr
= phys_sections
[orig_memory
].mr
;
2648 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
, end_addr2
,
2651 if (!(mr
->subpage
)) {
2652 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2655 subpage
= container_of(mr
, subpage_t
, iomem
);
2657 subpage_register(subpage
, start_addr2
, end_addr2
,
2662 addr
+= TARGET_PAGE_SIZE
;
2663 } while (addr
!= end_addr
);
2666 void qemu_register_coalesced_mmio(target_phys_addr_t addr
, ram_addr_t size
)
2669 kvm_coalesce_mmio_region(addr
, size
);
2672 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr
, ram_addr_t size
)
2675 kvm_uncoalesce_mmio_region(addr
, size
);
2678 void qemu_flush_coalesced_mmio_buffer(void)
2681 kvm_flush_coalesced_mmio_buffer();
2684 #if defined(__linux__) && !defined(TARGET_S390X)
2686 #include <sys/vfs.h>
2688 #define HUGETLBFS_MAGIC 0x958458f6
2690 static long gethugepagesize(const char *path
)
2696 ret
= statfs(path
, &fs
);
2697 } while (ret
!= 0 && errno
== EINTR
);
2704 if (fs
.f_type
!= HUGETLBFS_MAGIC
)
2705 fprintf(stderr
, "Warning: path not on HugeTLBFS: %s\n", path
);
2710 static void *file_ram_alloc(RAMBlock
*block
,
2720 unsigned long hpagesize
;
2722 hpagesize
= gethugepagesize(path
);
2727 if (memory
< hpagesize
) {
2731 if (kvm_enabled() && !kvm_has_sync_mmu()) {
2732 fprintf(stderr
, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
2736 if (asprintf(&filename
, "%s/qemu_back_mem.XXXXXX", path
) == -1) {
2740 fd
= mkstemp(filename
);
2742 perror("unable to create backing store for hugepages");
2749 memory
= (memory
+hpagesize
-1) & ~(hpagesize
-1);
2752 * ftruncate is not supported by hugetlbfs in older
2753 * hosts, so don't bother bailing out on errors.
2754 * If anything goes wrong with it under other filesystems,
2757 if (ftruncate(fd
, memory
))
2758 perror("ftruncate");
2761 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
2762 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
2763 * to sidestep this quirk.
2765 flags
= mem_prealloc
? MAP_POPULATE
| MAP_SHARED
: MAP_PRIVATE
;
2766 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
, flags
, fd
, 0);
2768 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
, MAP_PRIVATE
, fd
, 0);
2770 if (area
== MAP_FAILED
) {
2771 perror("file_ram_alloc: can't mmap RAM pages");
2780 static ram_addr_t
find_ram_offset(ram_addr_t size
)
2782 RAMBlock
*block
, *next_block
;
2783 ram_addr_t offset
= RAM_ADDR_MAX
, mingap
= RAM_ADDR_MAX
;
2785 if (QLIST_EMPTY(&ram_list
.blocks
))
2788 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2789 ram_addr_t end
, next
= RAM_ADDR_MAX
;
2791 end
= block
->offset
+ block
->length
;
2793 QLIST_FOREACH(next_block
, &ram_list
.blocks
, next
) {
2794 if (next_block
->offset
>= end
) {
2795 next
= MIN(next
, next_block
->offset
);
2798 if (next
- end
>= size
&& next
- end
< mingap
) {
2800 mingap
= next
- end
;
2804 if (offset
== RAM_ADDR_MAX
) {
2805 fprintf(stderr
, "Failed to find gap of requested size: %" PRIu64
"\n",
2813 static ram_addr_t
last_ram_offset(void)
2816 ram_addr_t last
= 0;
2818 QLIST_FOREACH(block
, &ram_list
.blocks
, next
)
2819 last
= MAX(last
, block
->offset
+ block
->length
);
2824 void qemu_ram_set_idstr(ram_addr_t addr
, const char *name
, DeviceState
*dev
)
2826 RAMBlock
*new_block
, *block
;
2829 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2830 if (block
->offset
== addr
) {
2836 assert(!new_block
->idstr
[0]);
2838 if (dev
&& dev
->parent_bus
&& dev
->parent_bus
->info
->get_dev_path
) {
2839 char *id
= dev
->parent_bus
->info
->get_dev_path(dev
);
2841 snprintf(new_block
->idstr
, sizeof(new_block
->idstr
), "%s/", id
);
2845 pstrcat(new_block
->idstr
, sizeof(new_block
->idstr
), name
);
2847 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2848 if (block
!= new_block
&& !strcmp(block
->idstr
, new_block
->idstr
)) {
2849 fprintf(stderr
, "RAMBlock \"%s\" already registered, abort!\n",
2856 ram_addr_t
qemu_ram_alloc_from_ptr(ram_addr_t size
, void *host
,
2859 RAMBlock
*new_block
;
2861 size
= TARGET_PAGE_ALIGN(size
);
2862 new_block
= g_malloc0(sizeof(*new_block
));
2865 new_block
->offset
= find_ram_offset(size
);
2867 new_block
->host
= host
;
2868 new_block
->flags
|= RAM_PREALLOC_MASK
;
2871 #if defined (__linux__) && !defined(TARGET_S390X)
2872 new_block
->host
= file_ram_alloc(new_block
, size
, mem_path
);
2873 if (!new_block
->host
) {
2874 new_block
->host
= qemu_vmalloc(size
);
2875 qemu_madvise(new_block
->host
, size
, QEMU_MADV_MERGEABLE
);
2878 fprintf(stderr
, "-mem-path option unsupported\n");
2882 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2883 /* S390 KVM requires the topmost vma of the RAM to be smaller than
2884 an system defined value, which is at least 256GB. Larger systems
2885 have larger values. We put the guest between the end of data
2886 segment (system break) and this value. We use 32GB as a base to
2887 have enough room for the system break to grow. */
2888 new_block
->host
= mmap((void*)0x800000000, size
,
2889 PROT_EXEC
|PROT_READ
|PROT_WRITE
,
2890 MAP_SHARED
| MAP_ANONYMOUS
| MAP_FIXED
, -1, 0);
2891 if (new_block
->host
== MAP_FAILED
) {
2892 fprintf(stderr
, "Allocating RAM failed\n");
2896 if (xen_enabled()) {
2897 xen_ram_alloc(new_block
->offset
, size
, mr
);
2899 new_block
->host
= qemu_vmalloc(size
);
2902 qemu_madvise(new_block
->host
, size
, QEMU_MADV_MERGEABLE
);
2905 new_block
->length
= size
;
2907 QLIST_INSERT_HEAD(&ram_list
.blocks
, new_block
, next
);
2909 ram_list
.phys_dirty
= g_realloc(ram_list
.phys_dirty
,
2910 last_ram_offset() >> TARGET_PAGE_BITS
);
2911 memset(ram_list
.phys_dirty
+ (new_block
->offset
>> TARGET_PAGE_BITS
),
2912 0xff, size
>> TARGET_PAGE_BITS
);
2915 kvm_setup_guest_memory(new_block
->host
, size
);
2917 return new_block
->offset
;
2920 ram_addr_t
qemu_ram_alloc(ram_addr_t size
, MemoryRegion
*mr
)
2922 return qemu_ram_alloc_from_ptr(size
, NULL
, mr
);
2925 void qemu_ram_free_from_ptr(ram_addr_t addr
)
2929 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2930 if (addr
== block
->offset
) {
2931 QLIST_REMOVE(block
, next
);
2938 void qemu_ram_free(ram_addr_t addr
)
2942 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2943 if (addr
== block
->offset
) {
2944 QLIST_REMOVE(block
, next
);
2945 if (block
->flags
& RAM_PREALLOC_MASK
) {
2947 } else if (mem_path
) {
2948 #if defined (__linux__) && !defined(TARGET_S390X)
2950 munmap(block
->host
, block
->length
);
2953 qemu_vfree(block
->host
);
2959 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2960 munmap(block
->host
, block
->length
);
2962 if (xen_enabled()) {
2963 xen_invalidate_map_cache_entry(block
->host
);
2965 qemu_vfree(block
->host
);
2977 void qemu_ram_remap(ram_addr_t addr
, ram_addr_t length
)
2984 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2985 offset
= addr
- block
->offset
;
2986 if (offset
< block
->length
) {
2987 vaddr
= block
->host
+ offset
;
2988 if (block
->flags
& RAM_PREALLOC_MASK
) {
2992 munmap(vaddr
, length
);
2994 #if defined(__linux__) && !defined(TARGET_S390X)
2997 flags
|= mem_prealloc
? MAP_POPULATE
| MAP_SHARED
:
3000 flags
|= MAP_PRIVATE
;
3002 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
3003 flags
, block
->fd
, offset
);
3005 flags
|= MAP_PRIVATE
| MAP_ANONYMOUS
;
3006 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
3013 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
3014 flags
|= MAP_SHARED
| MAP_ANONYMOUS
;
3015 area
= mmap(vaddr
, length
, PROT_EXEC
|PROT_READ
|PROT_WRITE
,
3018 flags
|= MAP_PRIVATE
| MAP_ANONYMOUS
;
3019 area
= mmap(vaddr
, length
, PROT_READ
| PROT_WRITE
,
3023 if (area
!= vaddr
) {
3024 fprintf(stderr
, "Could not remap addr: "
3025 RAM_ADDR_FMT
"@" RAM_ADDR_FMT
"\n",
3029 qemu_madvise(vaddr
, length
, QEMU_MADV_MERGEABLE
);
3035 #endif /* !_WIN32 */
3037 /* Return a host pointer to ram allocated with qemu_ram_alloc.
3038 With the exception of the softmmu code in this file, this should
3039 only be used for local memory (e.g. video ram) that the device owns,
3040 and knows it isn't going to access beyond the end of the block.
3042 It should not be used for general purpose DMA.
3043 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
3045 void *qemu_get_ram_ptr(ram_addr_t addr
)
3049 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
3050 if (addr
- block
->offset
< block
->length
) {
3051 /* Move this entry to to start of the list. */
3052 if (block
!= QLIST_FIRST(&ram_list
.blocks
)) {
3053 QLIST_REMOVE(block
, next
);
3054 QLIST_INSERT_HEAD(&ram_list
.blocks
, block
, next
);
3056 if (xen_enabled()) {
3057 /* We need to check if the requested address is in the RAM
3058 * because we don't want to map the entire memory in QEMU.
3059 * In that case just map until the end of the page.
3061 if (block
->offset
== 0) {
3062 return xen_map_cache(addr
, 0, 0);
3063 } else if (block
->host
== NULL
) {
3065 xen_map_cache(block
->offset
, block
->length
, 1);
3068 return block
->host
+ (addr
- block
->offset
);
3072 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
3078 /* Return a host pointer to ram allocated with qemu_ram_alloc.
3079 * Same as qemu_get_ram_ptr but avoid reordering ramblocks.
3081 void *qemu_safe_ram_ptr(ram_addr_t addr
)
3085 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
3086 if (addr
- block
->offset
< block
->length
) {
3087 if (xen_enabled()) {
3088 /* We need to check if the requested address is in the RAM
3089 * because we don't want to map the entire memory in QEMU.
3090 * In that case just map until the end of the page.
3092 if (block
->offset
== 0) {
3093 return xen_map_cache(addr
, 0, 0);
3094 } else if (block
->host
== NULL
) {
3096 xen_map_cache(block
->offset
, block
->length
, 1);
3099 return block
->host
+ (addr
- block
->offset
);
3103 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
3109 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
3110 * but takes a size argument */
3111 void *qemu_ram_ptr_length(ram_addr_t addr
, ram_addr_t
*size
)
3116 if (xen_enabled()) {
3117 return xen_map_cache(addr
, *size
, 1);
3121 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
3122 if (addr
- block
->offset
< block
->length
) {
3123 if (addr
- block
->offset
+ *size
> block
->length
)
3124 *size
= block
->length
- addr
+ block
->offset
;
3125 return block
->host
+ (addr
- block
->offset
);
3129 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
3134 void qemu_put_ram_ptr(void *addr
)
3136 trace_qemu_put_ram_ptr(addr
);
3139 int qemu_ram_addr_from_host(void *ptr
, ram_addr_t
*ram_addr
)
3142 uint8_t *host
= ptr
;
3144 if (xen_enabled()) {
3145 *ram_addr
= xen_ram_addr_from_mapcache(ptr
);
3149 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
3150 /* This case append when the block is not mapped. */
3151 if (block
->host
== NULL
) {
3154 if (host
- block
->host
< block
->length
) {
3155 *ram_addr
= block
->offset
+ (host
- block
->host
);
3163 /* Some of the softmmu routines need to translate from a host pointer
3164 (typically a TLB entry) back to a ram offset. */
3165 ram_addr_t
qemu_ram_addr_from_host_nofail(void *ptr
)
3167 ram_addr_t ram_addr
;
3169 if (qemu_ram_addr_from_host(ptr
, &ram_addr
)) {
3170 fprintf(stderr
, "Bad ram pointer %p\n", ptr
);
3176 static uint64_t unassigned_mem_read(void *opaque
, target_phys_addr_t addr
,
3179 #ifdef DEBUG_UNASSIGNED
3180 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
3182 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3183 cpu_unassigned_access(cpu_single_env
, addr
, 0, 0, 0, size
);
3188 static void unassigned_mem_write(void *opaque
, target_phys_addr_t addr
,
3189 uint64_t val
, unsigned size
)
3191 #ifdef DEBUG_UNASSIGNED
3192 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%"PRIx64
"\n", addr
, val
);
3194 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3195 cpu_unassigned_access(cpu_single_env
, addr
, 1, 0, 0, size
);
3199 static const MemoryRegionOps unassigned_mem_ops
= {
3200 .read
= unassigned_mem_read
,
3201 .write
= unassigned_mem_write
,
3202 .endianness
= DEVICE_NATIVE_ENDIAN
,
3205 static uint64_t error_mem_read(void *opaque
, target_phys_addr_t addr
,
3211 static void error_mem_write(void *opaque
, target_phys_addr_t addr
,
3212 uint64_t value
, unsigned size
)
3217 static const MemoryRegionOps error_mem_ops
= {
3218 .read
= error_mem_read
,
3219 .write
= error_mem_write
,
3220 .endianness
= DEVICE_NATIVE_ENDIAN
,
3223 static const MemoryRegionOps rom_mem_ops
= {
3224 .read
= error_mem_read
,
3225 .write
= unassigned_mem_write
,
3226 .endianness
= DEVICE_NATIVE_ENDIAN
,
3229 static void notdirty_mem_write(void *opaque
, target_phys_addr_t ram_addr
,
3230 uint64_t val
, unsigned size
)
3233 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
3234 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
3235 #if !defined(CONFIG_USER_ONLY)
3236 tb_invalidate_phys_page_fast(ram_addr
, size
);
3237 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
3242 stb_p(qemu_get_ram_ptr(ram_addr
), val
);
3245 stw_p(qemu_get_ram_ptr(ram_addr
), val
);
3248 stl_p(qemu_get_ram_ptr(ram_addr
), val
);
3253 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
3254 cpu_physical_memory_set_dirty_flags(ram_addr
, dirty_flags
);
3255 /* we remove the notdirty callback only if the code has been
3257 if (dirty_flags
== 0xff)
3258 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
3261 static const MemoryRegionOps notdirty_mem_ops
= {
3262 .read
= error_mem_read
,
3263 .write
= notdirty_mem_write
,
3264 .endianness
= DEVICE_NATIVE_ENDIAN
,
3267 /* Generate a debug exception if a watchpoint has been hit. */
3268 static void check_watchpoint(int offset
, int len_mask
, int flags
)
3270 CPUState
*env
= cpu_single_env
;
3271 target_ulong pc
, cs_base
;
3272 TranslationBlock
*tb
;
3277 if (env
->watchpoint_hit
) {
3278 /* We re-entered the check after replacing the TB. Now raise
3279 * the debug interrupt so that is will trigger after the
3280 * current instruction. */
3281 cpu_interrupt(env
, CPU_INTERRUPT_DEBUG
);
3284 vaddr
= (env
->mem_io_vaddr
& TARGET_PAGE_MASK
) + offset
;
3285 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
3286 if ((vaddr
== (wp
->vaddr
& len_mask
) ||
3287 (vaddr
& wp
->len_mask
) == wp
->vaddr
) && (wp
->flags
& flags
)) {
3288 wp
->flags
|= BP_WATCHPOINT_HIT
;
3289 if (!env
->watchpoint_hit
) {
3290 env
->watchpoint_hit
= wp
;
3291 tb
= tb_find_pc(env
->mem_io_pc
);
3293 cpu_abort(env
, "check_watchpoint: could not find TB for "
3294 "pc=%p", (void *)env
->mem_io_pc
);
3296 cpu_restore_state(tb
, env
, env
->mem_io_pc
);
3297 tb_phys_invalidate(tb
, -1);
3298 if (wp
->flags
& BP_STOP_BEFORE_ACCESS
) {
3299 env
->exception_index
= EXCP_DEBUG
;
3301 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &cpu_flags
);
3302 tb_gen_code(env
, pc
, cs_base
, cpu_flags
, 1);
3304 cpu_resume_from_signal(env
, NULL
);
3307 wp
->flags
&= ~BP_WATCHPOINT_HIT
;
3312 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
3313 so these check for a hit then pass through to the normal out-of-line
3315 static uint64_t watch_mem_read(void *opaque
, target_phys_addr_t addr
,
3318 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~(size
- 1), BP_MEM_READ
);
3320 case 1: return ldub_phys(addr
);
3321 case 2: return lduw_phys(addr
);
3322 case 4: return ldl_phys(addr
);
3327 static void watch_mem_write(void *opaque
, target_phys_addr_t addr
,
3328 uint64_t val
, unsigned size
)
3330 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~(size
- 1), BP_MEM_WRITE
);
3332 case 1: stb_phys(addr
, val
);
3333 case 2: stw_phys(addr
, val
);
3334 case 4: stl_phys(addr
, val
);
3339 static const MemoryRegionOps watch_mem_ops
= {
3340 .read
= watch_mem_read
,
3341 .write
= watch_mem_write
,
3342 .endianness
= DEVICE_NATIVE_ENDIAN
,
3345 static uint64_t subpage_read(void *opaque
, target_phys_addr_t addr
,
3348 subpage_t
*mmio
= opaque
;
3349 unsigned int idx
= SUBPAGE_IDX(addr
);
3350 MemoryRegionSection
*section
;
3351 #if defined(DEBUG_SUBPAGE)
3352 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d\n", __func__
,
3353 mmio
, len
, addr
, idx
);
3356 section
= &phys_sections
[mmio
->sub_section
[idx
]];
3358 addr
-= section
->offset_within_address_space
;
3359 addr
+= section
->offset_within_region
;
3360 return io_mem_read(section
->mr
->ram_addr
, addr
, len
);
3363 static void subpage_write(void *opaque
, target_phys_addr_t addr
,
3364 uint64_t value
, unsigned len
)
3366 subpage_t
*mmio
= opaque
;
3367 unsigned int idx
= SUBPAGE_IDX(addr
);
3368 MemoryRegionSection
*section
;
3369 #if defined(DEBUG_SUBPAGE)
3370 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
3371 " idx %d value %"PRIx64
"\n",
3372 __func__
, mmio
, len
, addr
, idx
, value
);
3375 section
= &phys_sections
[mmio
->sub_section
[idx
]];
3377 addr
-= section
->offset_within_address_space
;
3378 addr
+= section
->offset_within_region
;
3379 io_mem_write(section
->mr
->ram_addr
, addr
, value
, len
);
3382 static const MemoryRegionOps subpage_ops
= {
3383 .read
= subpage_read
,
3384 .write
= subpage_write
,
3385 .endianness
= DEVICE_NATIVE_ENDIAN
,
3388 static uint64_t subpage_ram_read(void *opaque
, target_phys_addr_t addr
,
3391 ram_addr_t raddr
= addr
;
3392 void *ptr
= qemu_get_ram_ptr(raddr
);
3394 case 1: return ldub_p(ptr
);
3395 case 2: return lduw_p(ptr
);
3396 case 4: return ldl_p(ptr
);
3401 static void subpage_ram_write(void *opaque
, target_phys_addr_t addr
,
3402 uint64_t value
, unsigned size
)
3404 ram_addr_t raddr
= addr
;
3405 void *ptr
= qemu_get_ram_ptr(raddr
);
3407 case 1: return stb_p(ptr
, value
);
3408 case 2: return stw_p(ptr
, value
);
3409 case 4: return stl_p(ptr
, value
);
3414 static const MemoryRegionOps subpage_ram_ops
= {
3415 .read
= subpage_ram_read
,
3416 .write
= subpage_ram_write
,
3417 .endianness
= DEVICE_NATIVE_ENDIAN
,
3420 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
3425 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
3427 idx
= SUBPAGE_IDX(start
);
3428 eidx
= SUBPAGE_IDX(end
);
3429 #if defined(DEBUG_SUBPAGE)
3430 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__
,
3431 mmio
, start
, end
, idx
, eidx
, memory
);
3433 if (memory_region_is_ram(phys_sections
[section
].mr
)) {
3434 MemoryRegionSection new_section
= phys_sections
[section
];
3435 new_section
.mr
= &io_mem_subpage_ram
;
3436 section
= phys_section_add(&new_section
);
3438 for (; idx
<= eidx
; idx
++) {
3439 mmio
->sub_section
[idx
] = section
;
3445 static subpage_t
*subpage_init (target_phys_addr_t base
, uint16_t *section_ind
,
3446 uint16_t orig_section
)
3449 MemoryRegionSection section
= {
3450 .offset_within_address_space
= base
,
3451 .size
= TARGET_PAGE_SIZE
,
3454 mmio
= g_malloc0(sizeof(subpage_t
));
3457 memory_region_init_io(&mmio
->iomem
, &subpage_ops
, mmio
,
3458 "subpage", TARGET_PAGE_SIZE
);
3459 mmio
->iomem
.subpage
= true;
3460 section
.mr
= &mmio
->iomem
;
3461 #if defined(DEBUG_SUBPAGE)
3462 printf("%s: %p base " TARGET_FMT_plx
" len %08x %d\n", __func__
,
3463 mmio
, base
, TARGET_PAGE_SIZE
, subpage_memory
);
3465 *section_ind
= phys_section_add(§ion
);
3466 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
-1, orig_section
);
3471 static int get_free_io_mem_idx(void)
3475 for (i
= 0; i
<IO_MEM_NB_ENTRIES
; i
++)
3476 if (!io_mem_used
[i
]) {
3480 fprintf(stderr
, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES
);
3484 /* mem_read and mem_write are arrays of functions containing the
3485 function to access byte (index 0), word (index 1) and dword (index
3486 2). Functions can be omitted with a NULL function pointer.
3487 If io_index is non zero, the corresponding io zone is
3488 modified. If it is zero, a new io zone is allocated. The return
3489 value can be used with cpu_register_physical_memory(). (-1) is
3490 returned if error. */
3491 static int cpu_register_io_memory_fixed(int io_index
, MemoryRegion
*mr
)
3493 if (io_index
<= 0) {
3494 io_index
= get_free_io_mem_idx();
3498 if (io_index
>= IO_MEM_NB_ENTRIES
)
3502 io_mem_region
[io_index
] = mr
;
3507 int cpu_register_io_memory(MemoryRegion
*mr
)
3509 return cpu_register_io_memory_fixed(0, mr
);
3512 void cpu_unregister_io_memory(int io_index
)
3514 io_mem_region
[io_index
] = NULL
;
3515 io_mem_used
[io_index
] = 0;
3518 static uint16_t dummy_section(MemoryRegion
*mr
)
3520 MemoryRegionSection section
= {
3522 .offset_within_address_space
= 0,
3523 .offset_within_region
= 0,
3527 return phys_section_add(§ion
);
3530 static void io_mem_init(void)
3534 /* Must be first: */
3535 memory_region_init_io(&io_mem_ram
, &error_mem_ops
, NULL
, "ram", UINT64_MAX
);
3536 assert(io_mem_ram
.ram_addr
== 0);
3537 memory_region_init_io(&io_mem_rom
, &rom_mem_ops
, NULL
, "rom", UINT64_MAX
);
3538 memory_region_init_io(&io_mem_unassigned
, &unassigned_mem_ops
, NULL
,
3539 "unassigned", UINT64_MAX
);
3540 memory_region_init_io(&io_mem_notdirty
, ¬dirty_mem_ops
, NULL
,
3541 "notdirty", UINT64_MAX
);
3542 memory_region_init_io(&io_mem_subpage_ram
, &subpage_ram_ops
, NULL
,
3543 "subpage-ram", UINT64_MAX
);
3547 memory_region_init_io(&io_mem_watch
, &watch_mem_ops
, NULL
,
3548 "watch", UINT64_MAX
);
3551 static void core_begin(MemoryListener
*listener
)
3553 destroy_all_mappings();
3554 phys_sections_clear();
3555 phys_map
.u
.node
= PHYS_MAP_NODE_NIL
;
3556 phys_section_unassigned
= dummy_section(&io_mem_unassigned
);
3559 static void core_commit(MemoryListener
*listener
)
3563 /* since each CPU stores ram addresses in its TLB cache, we must
3564 reset the modified entries */
3566 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
3571 static void core_region_add(MemoryListener
*listener
,
3572 MemoryRegionSection
*section
)
3574 cpu_register_physical_memory_log(section
, section
->readonly
);
3577 static void core_region_del(MemoryListener
*listener
,
3578 MemoryRegionSection
*section
)
3582 static void core_region_nop(MemoryListener
*listener
,
3583 MemoryRegionSection
*section
)
3585 cpu_register_physical_memory_log(section
, section
->readonly
);
3588 static void core_log_start(MemoryListener
*listener
,
3589 MemoryRegionSection
*section
)
3593 static void core_log_stop(MemoryListener
*listener
,
3594 MemoryRegionSection
*section
)
3598 static void core_log_sync(MemoryListener
*listener
,
3599 MemoryRegionSection
*section
)
3603 static void core_log_global_start(MemoryListener
*listener
)
3605 cpu_physical_memory_set_dirty_tracking(1);
3608 static void core_log_global_stop(MemoryListener
*listener
)
3610 cpu_physical_memory_set_dirty_tracking(0);
3613 static void core_eventfd_add(MemoryListener
*listener
,
3614 MemoryRegionSection
*section
,
3615 bool match_data
, uint64_t data
, int fd
)
3619 static void core_eventfd_del(MemoryListener
*listener
,
3620 MemoryRegionSection
*section
,
3621 bool match_data
, uint64_t data
, int fd
)
3625 static void io_begin(MemoryListener
*listener
)
3629 static void io_commit(MemoryListener
*listener
)
3633 static void io_region_add(MemoryListener
*listener
,
3634 MemoryRegionSection
*section
)
3636 iorange_init(§ion
->mr
->iorange
, &memory_region_iorange_ops
,
3637 section
->offset_within_address_space
, section
->size
);
3638 ioport_register(§ion
->mr
->iorange
);
3641 static void io_region_del(MemoryListener
*listener
,
3642 MemoryRegionSection
*section
)
3644 isa_unassign_ioport(section
->offset_within_address_space
, section
->size
);
3647 static void io_region_nop(MemoryListener
*listener
,
3648 MemoryRegionSection
*section
)
3652 static void io_log_start(MemoryListener
*listener
,
3653 MemoryRegionSection
*section
)
3657 static void io_log_stop(MemoryListener
*listener
,
3658 MemoryRegionSection
*section
)
3662 static void io_log_sync(MemoryListener
*listener
,
3663 MemoryRegionSection
*section
)
3667 static void io_log_global_start(MemoryListener
*listener
)
3671 static void io_log_global_stop(MemoryListener
*listener
)
3675 static void io_eventfd_add(MemoryListener
*listener
,
3676 MemoryRegionSection
*section
,
3677 bool match_data
, uint64_t data
, int fd
)
3681 static void io_eventfd_del(MemoryListener
*listener
,
3682 MemoryRegionSection
*section
,
3683 bool match_data
, uint64_t data
, int fd
)
3687 static MemoryListener core_memory_listener
= {
3688 .begin
= core_begin
,
3689 .commit
= core_commit
,
3690 .region_add
= core_region_add
,
3691 .region_del
= core_region_del
,
3692 .region_nop
= core_region_nop
,
3693 .log_start
= core_log_start
,
3694 .log_stop
= core_log_stop
,
3695 .log_sync
= core_log_sync
,
3696 .log_global_start
= core_log_global_start
,
3697 .log_global_stop
= core_log_global_stop
,
3698 .eventfd_add
= core_eventfd_add
,
3699 .eventfd_del
= core_eventfd_del
,
3703 static MemoryListener io_memory_listener
= {
3705 .commit
= io_commit
,
3706 .region_add
= io_region_add
,
3707 .region_del
= io_region_del
,
3708 .region_nop
= io_region_nop
,
3709 .log_start
= io_log_start
,
3710 .log_stop
= io_log_stop
,
3711 .log_sync
= io_log_sync
,
3712 .log_global_start
= io_log_global_start
,
3713 .log_global_stop
= io_log_global_stop
,
3714 .eventfd_add
= io_eventfd_add
,
3715 .eventfd_del
= io_eventfd_del
,
3719 static void memory_map_init(void)
3721 system_memory
= g_malloc(sizeof(*system_memory
));
3722 memory_region_init(system_memory
, "system", INT64_MAX
);
3723 set_system_memory_map(system_memory
);
3725 system_io
= g_malloc(sizeof(*system_io
));
3726 memory_region_init(system_io
, "io", 65536);
3727 set_system_io_map(system_io
);
3729 memory_listener_register(&core_memory_listener
, system_memory
);
3730 memory_listener_register(&io_memory_listener
, system_io
);
3733 MemoryRegion
*get_system_memory(void)
3735 return system_memory
;
3738 MemoryRegion
*get_system_io(void)
3743 #endif /* !defined(CONFIG_USER_ONLY) */
3745 /* physical memory access (slow version, mainly for debug) */
3746 #if defined(CONFIG_USER_ONLY)
3747 int cpu_memory_rw_debug(CPUState
*env
, target_ulong addr
,
3748 uint8_t *buf
, int len
, int is_write
)
3755 page
= addr
& TARGET_PAGE_MASK
;
3756 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3759 flags
= page_get_flags(page
);
3760 if (!(flags
& PAGE_VALID
))
3763 if (!(flags
& PAGE_WRITE
))
3765 /* XXX: this code should not depend on lock_user */
3766 if (!(p
= lock_user(VERIFY_WRITE
, addr
, l
, 0)))
3769 unlock_user(p
, addr
, l
);
3771 if (!(flags
& PAGE_READ
))
3773 /* XXX: this code should not depend on lock_user */
3774 if (!(p
= lock_user(VERIFY_READ
, addr
, l
, 1)))
3777 unlock_user(p
, addr
, 0);
3787 void cpu_physical_memory_rw(target_phys_addr_t addr
, uint8_t *buf
,
3788 int len
, int is_write
)
3793 target_phys_addr_t page
;
3794 MemoryRegionSection section
;
3797 page
= addr
& TARGET_PAGE_MASK
;
3798 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3801 section
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3804 if (!memory_region_is_ram(section
.mr
)) {
3805 target_phys_addr_t addr1
;
3806 io_index
= memory_region_get_ram_addr(section
.mr
)
3807 & (IO_MEM_NB_ENTRIES
- 1);
3808 addr1
= (addr
& ~TARGET_PAGE_MASK
)
3809 + section
.offset_within_region
;
3810 /* XXX: could force cpu_single_env to NULL to avoid
3812 if (l
>= 4 && ((addr1
& 3) == 0)) {
3813 /* 32 bit write access */
3815 io_mem_write(io_index
, addr1
, val
, 4);
3817 } else if (l
>= 2 && ((addr1
& 1) == 0)) {
3818 /* 16 bit write access */
3820 io_mem_write(io_index
, addr1
, val
, 2);
3823 /* 8 bit write access */
3825 io_mem_write(io_index
, addr1
, val
, 1);
3828 } else if (!section
.readonly
) {
3830 addr1
= (memory_region_get_ram_addr(section
.mr
)
3831 + section
.offset_within_region
)
3832 | (addr
& ~TARGET_PAGE_MASK
);
3834 ptr
= qemu_get_ram_ptr(addr1
);
3835 memcpy(ptr
, buf
, l
);
3836 if (!cpu_physical_memory_is_dirty(addr1
)) {
3837 /* invalidate code */
3838 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
3840 cpu_physical_memory_set_dirty_flags(
3841 addr1
, (0xff & ~CODE_DIRTY_FLAG
));
3843 qemu_put_ram_ptr(ptr
);
3846 if (!is_ram_rom_romd(§ion
)) {
3847 target_phys_addr_t addr1
;
3849 io_index
= memory_region_get_ram_addr(section
.mr
)
3850 & (IO_MEM_NB_ENTRIES
- 1);
3851 addr1
= (addr
& ~TARGET_PAGE_MASK
)
3852 + section
.offset_within_region
;
3853 if (l
>= 4 && ((addr1
& 3) == 0)) {
3854 /* 32 bit read access */
3855 val
= io_mem_read(io_index
, addr1
, 4);
3858 } else if (l
>= 2 && ((addr1
& 1) == 0)) {
3859 /* 16 bit read access */
3860 val
= io_mem_read(io_index
, addr1
, 2);
3864 /* 8 bit read access */
3865 val
= io_mem_read(io_index
, addr1
, 1);
3871 ptr
= qemu_get_ram_ptr(section
.mr
->ram_addr
3872 + section
.offset_within_region
);
3873 memcpy(buf
, ptr
+ (addr
& ~TARGET_PAGE_MASK
), l
);
3874 qemu_put_ram_ptr(ptr
);
3883 /* used for ROM loading : can write in RAM and ROM */
3884 void cpu_physical_memory_write_rom(target_phys_addr_t addr
,
3885 const uint8_t *buf
, int len
)
3889 target_phys_addr_t page
;
3890 MemoryRegionSection section
;
3893 page
= addr
& TARGET_PAGE_MASK
;
3894 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3897 section
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3899 if (!is_ram_rom_romd(§ion
)) {
3902 unsigned long addr1
;
3903 addr1
= (memory_region_get_ram_addr(section
.mr
)
3904 + section
.offset_within_region
)
3905 + (addr
& ~TARGET_PAGE_MASK
);
3907 ptr
= qemu_get_ram_ptr(addr1
);
3908 memcpy(ptr
, buf
, l
);
3909 qemu_put_ram_ptr(ptr
);
3919 target_phys_addr_t addr
;
3920 target_phys_addr_t len
;
3923 static BounceBuffer bounce
;
3925 typedef struct MapClient
{
3927 void (*callback
)(void *opaque
);
3928 QLIST_ENTRY(MapClient
) link
;
3931 static QLIST_HEAD(map_client_list
, MapClient
) map_client_list
3932 = QLIST_HEAD_INITIALIZER(map_client_list
);
3934 void *cpu_register_map_client(void *opaque
, void (*callback
)(void *opaque
))
3936 MapClient
*client
= g_malloc(sizeof(*client
));
3938 client
->opaque
= opaque
;
3939 client
->callback
= callback
;
3940 QLIST_INSERT_HEAD(&map_client_list
, client
, link
);
3944 void cpu_unregister_map_client(void *_client
)
3946 MapClient
*client
= (MapClient
*)_client
;
3948 QLIST_REMOVE(client
, link
);
3952 static void cpu_notify_map_clients(void)
3956 while (!QLIST_EMPTY(&map_client_list
)) {
3957 client
= QLIST_FIRST(&map_client_list
);
3958 client
->callback(client
->opaque
);
3959 cpu_unregister_map_client(client
);
3963 /* Map a physical memory region into a host virtual address.
3964 * May map a subset of the requested range, given by and returned in *plen.
3965 * May return NULL if resources needed to perform the mapping are exhausted.
3966 * Use only for reads OR writes - not for read-modify-write operations.
3967 * Use cpu_register_map_client() to know when retrying the map operation is
3968 * likely to succeed.
3970 void *cpu_physical_memory_map(target_phys_addr_t addr
,
3971 target_phys_addr_t
*plen
,
3974 target_phys_addr_t len
= *plen
;
3975 target_phys_addr_t todo
= 0;
3977 target_phys_addr_t page
;
3978 MemoryRegionSection section
;
3979 ram_addr_t raddr
= RAM_ADDR_MAX
;
3984 page
= addr
& TARGET_PAGE_MASK
;
3985 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3988 section
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3990 if (!(memory_region_is_ram(section
.mr
) && !section
.readonly
)) {
3991 if (todo
|| bounce
.buffer
) {
3994 bounce
.buffer
= qemu_memalign(TARGET_PAGE_SIZE
, TARGET_PAGE_SIZE
);
3998 cpu_physical_memory_read(addr
, bounce
.buffer
, l
);
4002 return bounce
.buffer
;
4005 raddr
= memory_region_get_ram_addr(section
.mr
)
4006 + section
.offset_within_region
4007 + (addr
& ~TARGET_PAGE_MASK
);
4015 ret
= qemu_ram_ptr_length(raddr
, &rlen
);
4020 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
4021 * Will also mark the memory as dirty if is_write == 1. access_len gives
4022 * the amount of memory that was actually read or written by the caller.
4024 void cpu_physical_memory_unmap(void *buffer
, target_phys_addr_t len
,
4025 int is_write
, target_phys_addr_t access_len
)
4027 if (buffer
!= bounce
.buffer
) {
4029 ram_addr_t addr1
= qemu_ram_addr_from_host_nofail(buffer
);
4030 while (access_len
) {
4032 l
= TARGET_PAGE_SIZE
;
4035 if (!cpu_physical_memory_is_dirty(addr1
)) {
4036 /* invalidate code */
4037 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
4039 cpu_physical_memory_set_dirty_flags(
4040 addr1
, (0xff & ~CODE_DIRTY_FLAG
));
4046 if (xen_enabled()) {
4047 xen_invalidate_map_cache_entry(buffer
);
4052 cpu_physical_memory_write(bounce
.addr
, bounce
.buffer
, access_len
);
4054 qemu_vfree(bounce
.buffer
);
4055 bounce
.buffer
= NULL
;
4056 cpu_notify_map_clients();
4059 /* warning: addr must be aligned */
4060 static inline uint32_t ldl_phys_internal(target_phys_addr_t addr
,
4061 enum device_endian endian
)
4066 MemoryRegionSection section
;
4068 section
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
4070 if (!is_ram_rom_romd(§ion
)) {
4072 io_index
= memory_region_get_ram_addr(section
.mr
)
4073 & (IO_MEM_NB_ENTRIES
- 1);
4074 addr
= (addr
& ~TARGET_PAGE_MASK
) + section
.offset_within_region
;
4075 val
= io_mem_read(io_index
, addr
, 4);
4076 #if defined(TARGET_WORDS_BIGENDIAN)
4077 if (endian
== DEVICE_LITTLE_ENDIAN
) {
4081 if (endian
== DEVICE_BIG_ENDIAN
) {
4087 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(section
.mr
)
4089 + section
.offset_within_region
) +
4090 (addr
& ~TARGET_PAGE_MASK
);
4092 case DEVICE_LITTLE_ENDIAN
:
4093 val
= ldl_le_p(ptr
);
4095 case DEVICE_BIG_ENDIAN
:
4096 val
= ldl_be_p(ptr
);
4106 uint32_t ldl_phys(target_phys_addr_t addr
)
4108 return ldl_phys_internal(addr
, DEVICE_NATIVE_ENDIAN
);
4111 uint32_t ldl_le_phys(target_phys_addr_t addr
)
4113 return ldl_phys_internal(addr
, DEVICE_LITTLE_ENDIAN
);
4116 uint32_t ldl_be_phys(target_phys_addr_t addr
)
4118 return ldl_phys_internal(addr
, DEVICE_BIG_ENDIAN
);
4121 /* warning: addr must be aligned */
4122 static inline uint64_t ldq_phys_internal(target_phys_addr_t addr
,
4123 enum device_endian endian
)
4128 MemoryRegionSection section
;
4130 section
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
4132 if (!is_ram_rom_romd(§ion
)) {
4134 io_index
= memory_region_get_ram_addr(section
.mr
)
4135 & (IO_MEM_NB_ENTRIES
- 1);
4136 addr
= (addr
& ~TARGET_PAGE_MASK
) + section
.offset_within_region
;
4138 /* XXX This is broken when device endian != cpu endian.
4139 Fix and add "endian" variable check */
4140 #ifdef TARGET_WORDS_BIGENDIAN
4141 val
= io_mem_read(io_index
, addr
, 4) << 32;
4142 val
|= io_mem_read(io_index
, addr
+ 4, 4);
4144 val
= io_mem_read(io_index
, addr
, 4);
4145 val
|= io_mem_read(io_index
, addr
+ 4, 4) << 32;
4149 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(section
.mr
)
4151 + section
.offset_within_region
)
4152 + (addr
& ~TARGET_PAGE_MASK
);
4154 case DEVICE_LITTLE_ENDIAN
:
4155 val
= ldq_le_p(ptr
);
4157 case DEVICE_BIG_ENDIAN
:
4158 val
= ldq_be_p(ptr
);
4168 uint64_t ldq_phys(target_phys_addr_t addr
)
4170 return ldq_phys_internal(addr
, DEVICE_NATIVE_ENDIAN
);
4173 uint64_t ldq_le_phys(target_phys_addr_t addr
)
4175 return ldq_phys_internal(addr
, DEVICE_LITTLE_ENDIAN
);
4178 uint64_t ldq_be_phys(target_phys_addr_t addr
)
4180 return ldq_phys_internal(addr
, DEVICE_BIG_ENDIAN
);
4184 uint32_t ldub_phys(target_phys_addr_t addr
)
4187 cpu_physical_memory_read(addr
, &val
, 1);
4191 /* warning: addr must be aligned */
4192 static inline uint32_t lduw_phys_internal(target_phys_addr_t addr
,
4193 enum device_endian endian
)
4198 MemoryRegionSection section
;
4200 section
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
4202 if (!is_ram_rom_romd(§ion
)) {
4204 io_index
= memory_region_get_ram_addr(section
.mr
)
4205 & (IO_MEM_NB_ENTRIES
- 1);
4206 addr
= (addr
& ~TARGET_PAGE_MASK
) + section
.offset_within_region
;
4207 val
= io_mem_read(io_index
, addr
, 2);
4208 #if defined(TARGET_WORDS_BIGENDIAN)
4209 if (endian
== DEVICE_LITTLE_ENDIAN
) {
4213 if (endian
== DEVICE_BIG_ENDIAN
) {
4219 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(section
.mr
)
4221 + section
.offset_within_region
)
4222 + (addr
& ~TARGET_PAGE_MASK
);
4224 case DEVICE_LITTLE_ENDIAN
:
4225 val
= lduw_le_p(ptr
);
4227 case DEVICE_BIG_ENDIAN
:
4228 val
= lduw_be_p(ptr
);
4238 uint32_t lduw_phys(target_phys_addr_t addr
)
4240 return lduw_phys_internal(addr
, DEVICE_NATIVE_ENDIAN
);
4243 uint32_t lduw_le_phys(target_phys_addr_t addr
)
4245 return lduw_phys_internal(addr
, DEVICE_LITTLE_ENDIAN
);
4248 uint32_t lduw_be_phys(target_phys_addr_t addr
)
4250 return lduw_phys_internal(addr
, DEVICE_BIG_ENDIAN
);
4253 /* warning: addr must be aligned. The ram page is not masked as dirty
4254 and the code inside is not invalidated. It is useful if the dirty
4255 bits are used to track modified PTEs */
4256 void stl_phys_notdirty(target_phys_addr_t addr
, uint32_t val
)
4260 MemoryRegionSection section
;
4262 section
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
4264 if (!memory_region_is_ram(section
.mr
) || section
.readonly
) {
4265 if (memory_region_is_ram(section
.mr
)) {
4266 io_index
= io_mem_rom
.ram_addr
;
4268 io_index
= memory_region_get_ram_addr(section
.mr
);
4270 addr
= (addr
& ~TARGET_PAGE_MASK
) + section
.offset_within_region
;
4271 io_mem_write(io_index
, addr
, val
, 4);
4273 unsigned long addr1
= (memory_region_get_ram_addr(section
.mr
)
4275 + section
.offset_within_region
4276 + (addr
& ~TARGET_PAGE_MASK
);
4277 ptr
= qemu_get_ram_ptr(addr1
);
4280 if (unlikely(in_migration
)) {
4281 if (!cpu_physical_memory_is_dirty(addr1
)) {
4282 /* invalidate code */
4283 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
4285 cpu_physical_memory_set_dirty_flags(
4286 addr1
, (0xff & ~CODE_DIRTY_FLAG
));
4292 void stq_phys_notdirty(target_phys_addr_t addr
, uint64_t val
)
4296 MemoryRegionSection section
;
4298 section
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
4300 if (!memory_region_is_ram(section
.mr
) || section
.readonly
) {
4301 if (memory_region_is_ram(section
.mr
)) {
4302 io_index
= io_mem_rom
.ram_addr
;
4304 io_index
= memory_region_get_ram_addr(section
.mr
)
4305 & (IO_MEM_NB_ENTRIES
- 1);
4307 addr
= (addr
& ~TARGET_PAGE_MASK
) + section
.offset_within_region
;
4308 #ifdef TARGET_WORDS_BIGENDIAN
4309 io_mem_write(io_index
, addr
, val
>> 32, 4);
4310 io_mem_write(io_index
, addr
+ 4, (uint32_t)val
, 4);
4312 io_mem_write(io_index
, addr
, (uint32_t)val
, 4);
4313 io_mem_write(io_index
, addr
+ 4, val
>> 32, 4);
4316 ptr
= qemu_get_ram_ptr((memory_region_get_ram_addr(section
.mr
)
4318 + section
.offset_within_region
)
4319 + (addr
& ~TARGET_PAGE_MASK
);
4324 /* warning: addr must be aligned */
4325 static inline void stl_phys_internal(target_phys_addr_t addr
, uint32_t val
,
4326 enum device_endian endian
)
4330 MemoryRegionSection section
;
4332 section
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
4334 if (!memory_region_is_ram(section
.mr
) || section
.readonly
) {
4335 if (memory_region_is_ram(section
.mr
)) {
4336 io_index
= io_mem_rom
.ram_addr
;
4338 io_index
= memory_region_get_ram_addr(section
.mr
)
4339 & (IO_MEM_NB_ENTRIES
- 1);
4341 addr
= (addr
& ~TARGET_PAGE_MASK
) + section
.offset_within_region
;
4342 #if defined(TARGET_WORDS_BIGENDIAN)
4343 if (endian
== DEVICE_LITTLE_ENDIAN
) {
4347 if (endian
== DEVICE_BIG_ENDIAN
) {
4351 io_mem_write(io_index
, addr
, val
, 4);
4353 unsigned long addr1
;
4354 addr1
= (memory_region_get_ram_addr(section
.mr
) & TARGET_PAGE_MASK
)
4355 + section
.offset_within_region
4356 + (addr
& ~TARGET_PAGE_MASK
);
4358 ptr
= qemu_get_ram_ptr(addr1
);
4360 case DEVICE_LITTLE_ENDIAN
:
4363 case DEVICE_BIG_ENDIAN
:
4370 if (!cpu_physical_memory_is_dirty(addr1
)) {
4371 /* invalidate code */
4372 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
4374 cpu_physical_memory_set_dirty_flags(addr1
,
4375 (0xff & ~CODE_DIRTY_FLAG
));
4380 void stl_phys(target_phys_addr_t addr
, uint32_t val
)
4382 stl_phys_internal(addr
, val
, DEVICE_NATIVE_ENDIAN
);
4385 void stl_le_phys(target_phys_addr_t addr
, uint32_t val
)
4387 stl_phys_internal(addr
, val
, DEVICE_LITTLE_ENDIAN
);
4390 void stl_be_phys(target_phys_addr_t addr
, uint32_t val
)
4392 stl_phys_internal(addr
, val
, DEVICE_BIG_ENDIAN
);
4396 void stb_phys(target_phys_addr_t addr
, uint32_t val
)
4399 cpu_physical_memory_write(addr
, &v
, 1);
4402 /* warning: addr must be aligned */
4403 static inline void stw_phys_internal(target_phys_addr_t addr
, uint32_t val
,
4404 enum device_endian endian
)
4408 MemoryRegionSection section
;
4410 section
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
4412 if (!memory_region_is_ram(section
.mr
) || section
.readonly
) {
4413 if (memory_region_is_ram(section
.mr
)) {
4414 io_index
= io_mem_rom
.ram_addr
;
4416 io_index
= memory_region_get_ram_addr(section
.mr
)
4417 & (IO_MEM_NB_ENTRIES
- 1);
4419 addr
= (addr
& ~TARGET_PAGE_MASK
) + section
.offset_within_region
;
4420 #if defined(TARGET_WORDS_BIGENDIAN)
4421 if (endian
== DEVICE_LITTLE_ENDIAN
) {
4425 if (endian
== DEVICE_BIG_ENDIAN
) {
4429 io_mem_write(io_index
, addr
, val
, 2);
4431 unsigned long addr1
;
4432 addr1
= (memory_region_get_ram_addr(section
.mr
) & TARGET_PAGE_MASK
)
4433 + section
.offset_within_region
+ (addr
& ~TARGET_PAGE_MASK
);
4435 ptr
= qemu_get_ram_ptr(addr1
);
4437 case DEVICE_LITTLE_ENDIAN
:
4440 case DEVICE_BIG_ENDIAN
:
4447 if (!cpu_physical_memory_is_dirty(addr1
)) {
4448 /* invalidate code */
4449 tb_invalidate_phys_page_range(addr1
, addr1
+ 2, 0);
4451 cpu_physical_memory_set_dirty_flags(addr1
,
4452 (0xff & ~CODE_DIRTY_FLAG
));
4457 void stw_phys(target_phys_addr_t addr
, uint32_t val
)
4459 stw_phys_internal(addr
, val
, DEVICE_NATIVE_ENDIAN
);
4462 void stw_le_phys(target_phys_addr_t addr
, uint32_t val
)
4464 stw_phys_internal(addr
, val
, DEVICE_LITTLE_ENDIAN
);
4467 void stw_be_phys(target_phys_addr_t addr
, uint32_t val
)
4469 stw_phys_internal(addr
, val
, DEVICE_BIG_ENDIAN
);
4473 void stq_phys(target_phys_addr_t addr
, uint64_t val
)
4476 cpu_physical_memory_write(addr
, &val
, 8);
4479 void stq_le_phys(target_phys_addr_t addr
, uint64_t val
)
4481 val
= cpu_to_le64(val
);
4482 cpu_physical_memory_write(addr
, &val
, 8);
4485 void stq_be_phys(target_phys_addr_t addr
, uint64_t val
)
4487 val
= cpu_to_be64(val
);
4488 cpu_physical_memory_write(addr
, &val
, 8);
4491 /* virtual memory access for debug (includes writing to ROM) */
4492 int cpu_memory_rw_debug(CPUState
*env
, target_ulong addr
,
4493 uint8_t *buf
, int len
, int is_write
)
4496 target_phys_addr_t phys_addr
;
4500 page
= addr
& TARGET_PAGE_MASK
;
4501 phys_addr
= cpu_get_phys_page_debug(env
, page
);
4502 /* if no physical page mapped, return an error */
4503 if (phys_addr
== -1)
4505 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
4508 phys_addr
+= (addr
& ~TARGET_PAGE_MASK
);
4510 cpu_physical_memory_write_rom(phys_addr
, buf
, l
);
4512 cpu_physical_memory_rw(phys_addr
, buf
, l
, is_write
);
4521 /* in deterministic execution mode, instructions doing device I/Os
4522 must be at the end of the TB */
4523 void cpu_io_recompile(CPUState
*env
, void *retaddr
)
4525 TranslationBlock
*tb
;
4527 target_ulong pc
, cs_base
;
4530 tb
= tb_find_pc((unsigned long)retaddr
);
4532 cpu_abort(env
, "cpu_io_recompile: could not find TB for pc=%p",
4535 n
= env
->icount_decr
.u16
.low
+ tb
->icount
;
4536 cpu_restore_state(tb
, env
, (unsigned long)retaddr
);
4537 /* Calculate how many instructions had been executed before the fault
4539 n
= n
- env
->icount_decr
.u16
.low
;
4540 /* Generate a new TB ending on the I/O insn. */
4542 /* On MIPS and SH, delay slot instructions can only be restarted if
4543 they were already the first instruction in the TB. If this is not
4544 the first instruction in a TB then re-execute the preceding
4546 #if defined(TARGET_MIPS)
4547 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
4548 env
->active_tc
.PC
-= 4;
4549 env
->icount_decr
.u16
.low
++;
4550 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
4552 #elif defined(TARGET_SH4)
4553 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
4556 env
->icount_decr
.u16
.low
++;
4557 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
4560 /* This should never happen. */
4561 if (n
> CF_COUNT_MASK
)
4562 cpu_abort(env
, "TB too big during recompile");
4564 cflags
= n
| CF_LAST_IO
;
4566 cs_base
= tb
->cs_base
;
4568 tb_phys_invalidate(tb
, -1);
4569 /* FIXME: In theory this could raise an exception. In practice
4570 we have already translated the block once so it's probably ok. */
4571 tb_gen_code(env
, pc
, cs_base
, flags
, cflags
);
4572 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
4573 the first in the TB) then we end up generating a whole new TB and
4574 repeating the fault, which is horribly inefficient.
4575 Better would be to execute just this insn uncached, or generate a
4577 cpu_resume_from_signal(env
, NULL
);
4580 #if !defined(CONFIG_USER_ONLY)
4582 void dump_exec_info(FILE *f
, fprintf_function cpu_fprintf
)
4584 int i
, target_code_size
, max_target_code_size
;
4585 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
4586 TranslationBlock
*tb
;
4588 target_code_size
= 0;
4589 max_target_code_size
= 0;
4591 direct_jmp_count
= 0;
4592 direct_jmp2_count
= 0;
4593 for(i
= 0; i
< nb_tbs
; i
++) {
4595 target_code_size
+= tb
->size
;
4596 if (tb
->size
> max_target_code_size
)
4597 max_target_code_size
= tb
->size
;
4598 if (tb
->page_addr
[1] != -1)
4600 if (tb
->tb_next_offset
[0] != 0xffff) {
4602 if (tb
->tb_next_offset
[1] != 0xffff) {
4603 direct_jmp2_count
++;
4607 /* XXX: avoid using doubles ? */
4608 cpu_fprintf(f
, "Translation buffer state:\n");
4609 cpu_fprintf(f
, "gen code size %td/%ld\n",
4610 code_gen_ptr
- code_gen_buffer
, code_gen_buffer_max_size
);
4611 cpu_fprintf(f
, "TB count %d/%d\n",
4612 nb_tbs
, code_gen_max_blocks
);
4613 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
4614 nb_tbs
? target_code_size
/ nb_tbs
: 0,
4615 max_target_code_size
);
4616 cpu_fprintf(f
, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
4617 nb_tbs
? (code_gen_ptr
- code_gen_buffer
) / nb_tbs
: 0,
4618 target_code_size
? (double) (code_gen_ptr
- code_gen_buffer
) / target_code_size
: 0);
4619 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n",
4621 nb_tbs
? (cross_page
* 100) / nb_tbs
: 0);
4622 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
4624 nb_tbs
? (direct_jmp_count
* 100) / nb_tbs
: 0,
4626 nb_tbs
? (direct_jmp2_count
* 100) / nb_tbs
: 0);
4627 cpu_fprintf(f
, "\nStatistics:\n");
4628 cpu_fprintf(f
, "TB flush count %d\n", tb_flush_count
);
4629 cpu_fprintf(f
, "TB invalidate count %d\n", tb_phys_invalidate_count
);
4630 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
4631 tcg_dump_info(f
, cpu_fprintf
);
4634 /* NOTE: this function can trigger an exception */
4635 /* NOTE2: the returned address is not exactly the physical address: it
4636 is the offset relative to phys_ram_base */
4637 tb_page_addr_t
get_page_addr_code(CPUState
*env1
, target_ulong addr
)
4639 int mmu_idx
, page_index
, pd
;
4642 page_index
= (addr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
4643 mmu_idx
= cpu_mmu_index(env1
);
4644 if (unlikely(env1
->tlb_table
[mmu_idx
][page_index
].addr_code
!=
4645 (addr
& TARGET_PAGE_MASK
))) {
4648 pd
= env1
->tlb_table
[mmu_idx
][page_index
].addr_code
& ~TARGET_PAGE_MASK
;
4649 if (pd
!= io_mem_ram
.ram_addr
&& pd
!= io_mem_rom
.ram_addr
4650 && !io_mem_region
[pd
]->rom_device
) {
4651 #if defined(TARGET_ALPHA) || defined(TARGET_MIPS) || defined(TARGET_SPARC)
4652 cpu_unassigned_access(env1
, addr
, 0, 1, 0, 4);
4654 cpu_abort(env1
, "Trying to execute code outside RAM or ROM at 0x" TARGET_FMT_lx
"\n", addr
);
4657 p
= (void *)((uintptr_t)addr
+ env1
->tlb_table
[mmu_idx
][page_index
].addend
);
4658 return qemu_ram_addr_from_host_nofail(p
);
4662 * A helper function for the _utterly broken_ virtio device model to find out if
4663 * it's running on a big endian machine. Don't do this at home kids!
4665 bool virtio_is_big_endian(void);
4666 bool virtio_is_big_endian(void)
4668 #if defined(TARGET_WORDS_BIGENDIAN)
4675 #define MMUSUFFIX _cmmu
4677 #define GETPC() NULL
4678 #define env cpu_single_env
4679 #define SOFTMMU_CODE_ACCESS
4682 #include "softmmu_template.h"
4685 #include "softmmu_template.h"
4688 #include "softmmu_template.h"
4691 #include "softmmu_template.h"