2 * virtual page mapping and translated block handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
23 #include <sys/types.h>
36 #include "qemu-common.h"
37 #include "cache-utils.h"
39 #if !defined(TARGET_IA64)
47 #include "qemu-timer.h"
48 #if defined(CONFIG_USER_ONLY)
51 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
52 #include <sys/param.h>
53 #if __FreeBSD_version >= 700104
54 #define HAVE_KINFO_GETVMMAP
55 #define sigqueue sigqueue_freebsd /* avoid redefinition */
58 #include <machine/profile.h>
68 //#define DEBUG_TB_INVALIDATE
71 //#define DEBUG_UNASSIGNED
73 /* make various TB consistency checks */
74 //#define DEBUG_TB_CHECK
75 //#define DEBUG_TLB_CHECK
77 //#define DEBUG_IOPORT
78 //#define DEBUG_SUBPAGE
80 #if !defined(CONFIG_USER_ONLY)
81 /* TB consistency checks only implemented for usermode emulation. */
85 #define SMC_BITMAP_USE_THRESHOLD 10
87 static TranslationBlock
*tbs
;
88 int code_gen_max_blocks
;
89 TranslationBlock
*tb_phys_hash
[CODE_GEN_PHYS_HASH_SIZE
];
91 /* any access to the tbs or the page table must use this lock */
92 spinlock_t tb_lock
= SPIN_LOCK_UNLOCKED
;
94 #if defined(__arm__) || defined(__sparc_v9__)
95 /* The prologue must be reachable with a direct jump. ARM and Sparc64
96 have limited branch ranges (possibly also PPC) so place it in a
97 section close to code segment. */
98 #define code_gen_section \
99 __attribute__((__section__(".gen_code"))) \
100 __attribute__((aligned (32)))
101 #elif defined(_WIN32)
102 /* Maximum alignment for Win32 is 16. */
103 #define code_gen_section \
104 __attribute__((aligned (16)))
106 #define code_gen_section \
107 __attribute__((aligned (32)))
110 uint8_t code_gen_prologue
[1024] code_gen_section
;
111 static uint8_t *code_gen_buffer
;
112 static unsigned long code_gen_buffer_size
;
113 /* threshold to flush the translated code buffer */
114 static unsigned long code_gen_buffer_max_size
;
115 uint8_t *code_gen_ptr
;
117 #if !defined(CONFIG_USER_ONLY)
119 uint8_t *phys_ram_dirty
;
120 static int in_migration
;
122 typedef struct RAMBlock
{
126 struct RAMBlock
*next
;
129 static RAMBlock
*ram_blocks
;
130 /* TODO: When we implement (and use) ram deallocation (e.g. for hotplug)
131 then we can no longer assume contiguous ram offsets, and external uses
132 of this variable will break. */
133 ram_addr_t last_ram_offset
;
137 /* current CPU in the current thread. It is only valid inside
139 CPUState
*cpu_single_env
;
140 /* 0 = Do not count executed instructions.
141 1 = Precise instruction counting.
142 2 = Adaptive rate instruction counting. */
144 /* Current instruction counter. While executing translated code this may
145 include some instructions that have not yet been executed. */
148 typedef struct PageDesc
{
149 /* list of TBs intersecting this ram page */
150 TranslationBlock
*first_tb
;
151 /* in order to optimize self modifying code, we count the number
152 of lookups we do to a given page to use a bitmap */
153 unsigned int code_write_count
;
154 uint8_t *code_bitmap
;
155 #if defined(CONFIG_USER_ONLY)
160 /* In system mode we want L1_MAP to be based on ram offsets,
161 while in user mode we want it to be based on virtual addresses. */
162 #if !defined(CONFIG_USER_ONLY)
163 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
164 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
166 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
169 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
172 /* Size of the L2 (and L3, etc) page tables. */
174 #define L2_SIZE (1 << L2_BITS)
176 /* The bits remaining after N lower levels of page tables. */
177 #define P_L1_BITS_REM \
178 ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
179 #define V_L1_BITS_REM \
180 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
182 /* Size of the L1 page table. Avoid silly small sizes. */
183 #if P_L1_BITS_REM < 4
184 #define P_L1_BITS (P_L1_BITS_REM + L2_BITS)
186 #define P_L1_BITS P_L1_BITS_REM
189 #if V_L1_BITS_REM < 4
190 #define V_L1_BITS (V_L1_BITS_REM + L2_BITS)
192 #define V_L1_BITS V_L1_BITS_REM
195 #define P_L1_SIZE ((target_phys_addr_t)1 << P_L1_BITS)
196 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
198 #define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
199 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
201 unsigned long qemu_real_host_page_size
;
202 unsigned long qemu_host_page_bits
;
203 unsigned long qemu_host_page_size
;
204 unsigned long qemu_host_page_mask
;
206 /* This is a multi-level map on the virtual address space.
207 The bottom level has pointers to PageDesc. */
208 static void *l1_map
[V_L1_SIZE
];
210 #if !defined(CONFIG_USER_ONLY)
211 typedef struct PhysPageDesc
{
212 /* offset in host memory of the page + io_index in the low bits */
213 ram_addr_t phys_offset
;
214 ram_addr_t region_offset
;
217 /* This is a multi-level map on the physical address space.
218 The bottom level has pointers to PhysPageDesc. */
219 static void *l1_phys_map
[P_L1_SIZE
];
221 static void io_mem_init(void);
223 /* io memory support */
224 CPUWriteMemoryFunc
*io_mem_write
[IO_MEM_NB_ENTRIES
][4];
225 CPUReadMemoryFunc
*io_mem_read
[IO_MEM_NB_ENTRIES
][4];
226 void *io_mem_opaque
[IO_MEM_NB_ENTRIES
];
227 static char io_mem_used
[IO_MEM_NB_ENTRIES
];
228 static int io_mem_watch
;
233 static const char *logfilename
= "qemu.log";
235 static const char *logfilename
= "/tmp/qemu.log";
239 static int log_append
= 0;
242 #if !defined(CONFIG_USER_ONLY)
243 static int tlb_flush_count
;
245 static int tb_flush_count
;
246 static int tb_phys_invalidate_count
;
249 static void map_exec(void *addr
, long size
)
252 VirtualProtect(addr
, size
,
253 PAGE_EXECUTE_READWRITE
, &old_protect
);
257 static void map_exec(void *addr
, long size
)
259 unsigned long start
, end
, page_size
;
261 page_size
= getpagesize();
262 start
= (unsigned long)addr
;
263 start
&= ~(page_size
- 1);
265 end
= (unsigned long)addr
+ size
;
266 end
+= page_size
- 1;
267 end
&= ~(page_size
- 1);
269 mprotect((void *)start
, end
- start
,
270 PROT_READ
| PROT_WRITE
| PROT_EXEC
);
274 static void page_init(void)
276 /* NOTE: we can always suppose that qemu_host_page_size >=
280 SYSTEM_INFO system_info
;
282 GetSystemInfo(&system_info
);
283 qemu_real_host_page_size
= system_info
.dwPageSize
;
286 qemu_real_host_page_size
= getpagesize();
288 if (qemu_host_page_size
== 0)
289 qemu_host_page_size
= qemu_real_host_page_size
;
290 if (qemu_host_page_size
< TARGET_PAGE_SIZE
)
291 qemu_host_page_size
= TARGET_PAGE_SIZE
;
292 qemu_host_page_bits
= 0;
293 while ((1 << qemu_host_page_bits
) < qemu_host_page_size
)
294 qemu_host_page_bits
++;
295 qemu_host_page_mask
= ~(qemu_host_page_size
- 1);
297 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
299 #ifdef HAVE_KINFO_GETVMMAP
300 struct kinfo_vmentry
*freep
;
303 freep
= kinfo_getvmmap(getpid(), &cnt
);
306 for (i
= 0; i
< cnt
; i
++) {
307 unsigned long startaddr
, endaddr
;
309 startaddr
= freep
[i
].kve_start
;
310 endaddr
= freep
[i
].kve_end
;
311 if (h2g_valid(startaddr
)) {
312 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
314 if (h2g_valid(endaddr
)) {
315 endaddr
= h2g(endaddr
);
316 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
318 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
320 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
331 last_brk
= (unsigned long)sbrk(0);
333 f
= fopen("/compat/linux/proc/self/maps", "r");
338 unsigned long startaddr
, endaddr
;
341 n
= fscanf (f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
343 if (n
== 2 && h2g_valid(startaddr
)) {
344 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
346 if (h2g_valid(endaddr
)) {
347 endaddr
= h2g(endaddr
);
351 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
363 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
369 #if defined(CONFIG_USER_ONLY)
370 /* We can't use qemu_malloc because it may recurse into a locked mutex. */
371 # define ALLOC(P, SIZE) \
373 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
374 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
377 # define ALLOC(P, SIZE) \
378 do { P = qemu_mallocz(SIZE); } while (0)
381 /* Level 1. Always allocated. */
382 lp
= l1_map
+ ((index
>> V_L1_SHIFT
) & (V_L1_SIZE
- 1));
385 for (i
= V_L1_SHIFT
/ L2_BITS
- 1; i
> 0; i
--) {
392 ALLOC(p
, sizeof(void *) * L2_SIZE
);
396 lp
= p
+ ((index
>> (i
* L2_BITS
)) & (L2_SIZE
- 1));
404 ALLOC(pd
, sizeof(PageDesc
) * L2_SIZE
);
410 return pd
+ (index
& (L2_SIZE
- 1));
413 static inline PageDesc
*page_find(tb_page_addr_t index
)
415 return page_find_alloc(index
, 0);
418 #if !defined(CONFIG_USER_ONLY)
419 static PhysPageDesc
*phys_page_find_alloc(target_phys_addr_t index
, int alloc
)
425 /* Level 1. Always allocated. */
426 lp
= l1_phys_map
+ ((index
>> P_L1_SHIFT
) & (P_L1_SIZE
- 1));
429 for (i
= P_L1_SHIFT
/ L2_BITS
- 1; i
> 0; i
--) {
435 *lp
= p
= qemu_mallocz(sizeof(void *) * L2_SIZE
);
437 lp
= p
+ ((index
>> (i
* L2_BITS
)) & (L2_SIZE
- 1));
448 *lp
= pd
= qemu_malloc(sizeof(PhysPageDesc
) * L2_SIZE
);
450 for (i
= 0; i
< L2_SIZE
; i
++) {
451 pd
[i
].phys_offset
= IO_MEM_UNASSIGNED
;
452 pd
[i
].region_offset
= (index
+ i
) << TARGET_PAGE_BITS
;
456 return pd
+ (index
& (L2_SIZE
- 1));
459 static inline PhysPageDesc
*phys_page_find(target_phys_addr_t index
)
461 return phys_page_find_alloc(index
, 0);
464 static void tlb_protect_code(ram_addr_t ram_addr
);
465 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
467 #define mmap_lock() do { } while(0)
468 #define mmap_unlock() do { } while(0)
471 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
473 #if defined(CONFIG_USER_ONLY)
474 /* Currently it is not recommended to allocate big chunks of data in
475 user mode. It will change when a dedicated libc will be used */
476 #define USE_STATIC_CODE_GEN_BUFFER
479 #ifdef USE_STATIC_CODE_GEN_BUFFER
480 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
]
481 __attribute__((aligned (CODE_GEN_ALIGN
)));
484 static void code_gen_alloc(unsigned long tb_size
)
489 #ifdef USE_STATIC_CODE_GEN_BUFFER
490 code_gen_buffer
= static_code_gen_buffer
;
491 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
492 map_exec(code_gen_buffer
, code_gen_buffer_size
);
494 code_gen_buffer_size
= tb_size
;
495 if (code_gen_buffer_size
== 0) {
496 #if defined(CONFIG_USER_ONLY)
497 /* in user mode, phys_ram_size is not meaningful */
498 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
500 /* XXX: needs adjustments */
501 code_gen_buffer_size
= (unsigned long)(ram_size
/ 4);
504 if (code_gen_buffer_size
< MIN_CODE_GEN_BUFFER_SIZE
)
505 code_gen_buffer_size
= MIN_CODE_GEN_BUFFER_SIZE
;
506 /* The code gen buffer location may have constraints depending on
507 the host cpu and OS */
508 #if defined(__linux__)
513 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
514 #if defined(__x86_64__)
516 /* Cannot map more than that */
517 if (code_gen_buffer_size
> (800 * 1024 * 1024))
518 code_gen_buffer_size
= (800 * 1024 * 1024);
519 #elif defined(__sparc_v9__)
520 // Map the buffer below 2G, so we can use direct calls and branches
522 start
= (void *) 0x60000000UL
;
523 if (code_gen_buffer_size
> (512 * 1024 * 1024))
524 code_gen_buffer_size
= (512 * 1024 * 1024);
525 #elif defined(__arm__)
526 /* Map the buffer below 32M, so we can use direct calls and branches */
528 start
= (void *) 0x01000000UL
;
529 if (code_gen_buffer_size
> 16 * 1024 * 1024)
530 code_gen_buffer_size
= 16 * 1024 * 1024;
531 #elif defined(__s390x__)
532 /* Map the buffer so that we can use direct calls and branches. */
533 /* We have a +- 4GB range on the branches; leave some slop. */
534 if (code_gen_buffer_size
> (3ul * 1024 * 1024 * 1024)) {
535 code_gen_buffer_size
= 3ul * 1024 * 1024 * 1024;
537 start
= (void *)0x90000000UL
;
539 code_gen_buffer
= mmap(start
, code_gen_buffer_size
,
540 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
542 if (code_gen_buffer
== MAP_FAILED
) {
543 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
547 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__DragonFly__)
551 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
552 #if defined(__x86_64__)
553 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
554 * 0x40000000 is free */
556 addr
= (void *)0x40000000;
557 /* Cannot map more than that */
558 if (code_gen_buffer_size
> (800 * 1024 * 1024))
559 code_gen_buffer_size
= (800 * 1024 * 1024);
561 code_gen_buffer
= mmap(addr
, code_gen_buffer_size
,
562 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
564 if (code_gen_buffer
== MAP_FAILED
) {
565 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
570 code_gen_buffer
= qemu_malloc(code_gen_buffer_size
);
571 map_exec(code_gen_buffer
, code_gen_buffer_size
);
573 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
574 map_exec(code_gen_prologue
, sizeof(code_gen_prologue
));
575 code_gen_buffer_max_size
= code_gen_buffer_size
-
576 (TCG_MAX_OP_SIZE
* OPC_MAX_SIZE
);
577 code_gen_max_blocks
= code_gen_buffer_size
/ CODE_GEN_AVG_BLOCK_SIZE
;
578 tbs
= qemu_malloc(code_gen_max_blocks
* sizeof(TranslationBlock
));
581 /* Must be called before using the QEMU cpus. 'tb_size' is the size
582 (in bytes) allocated to the translation buffer. Zero means default
584 void cpu_exec_init_all(unsigned long tb_size
)
587 code_gen_alloc(tb_size
);
588 code_gen_ptr
= code_gen_buffer
;
590 #if !defined(CONFIG_USER_ONLY)
593 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
594 /* There's no guest base to take into account, so go ahead and
595 initialize the prologue now. */
596 tcg_prologue_init(&tcg_ctx
);
600 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
602 static int cpu_common_post_load(void *opaque
, int version_id
)
604 CPUState
*env
= opaque
;
606 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
607 version_id is increased. */
608 env
->interrupt_request
&= ~0x01;
614 static const VMStateDescription vmstate_cpu_common
= {
615 .name
= "cpu_common",
617 .minimum_version_id
= 1,
618 .minimum_version_id_old
= 1,
619 .post_load
= cpu_common_post_load
,
620 .fields
= (VMStateField
[]) {
621 VMSTATE_UINT32(halted
, CPUState
),
622 VMSTATE_UINT32(interrupt_request
, CPUState
),
623 VMSTATE_END_OF_LIST()
628 CPUState
*qemu_get_cpu(int cpu
)
630 CPUState
*env
= first_cpu
;
633 if (env
->cpu_index
== cpu
)
641 void cpu_exec_init(CPUState
*env
)
646 #if defined(CONFIG_USER_ONLY)
649 env
->next_cpu
= NULL
;
652 while (*penv
!= NULL
) {
653 penv
= &(*penv
)->next_cpu
;
656 env
->cpu_index
= cpu_index
;
658 QTAILQ_INIT(&env
->breakpoints
);
659 QTAILQ_INIT(&env
->watchpoints
);
661 env
->thread_id
= GetCurrentProcessId();
663 env
->thread_id
= getpid();
666 #if defined(CONFIG_USER_ONLY)
669 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
670 vmstate_register(cpu_index
, &vmstate_cpu_common
, env
);
671 register_savevm("cpu", cpu_index
, CPU_SAVE_VERSION
,
672 cpu_save
, cpu_load
, env
);
676 static inline void invalidate_page_bitmap(PageDesc
*p
)
678 if (p
->code_bitmap
) {
679 qemu_free(p
->code_bitmap
);
680 p
->code_bitmap
= NULL
;
682 p
->code_write_count
= 0;
685 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
687 static void page_flush_tb_1 (int level
, void **lp
)
696 for (i
= 0; i
< L2_SIZE
; ++i
) {
697 pd
[i
].first_tb
= NULL
;
698 invalidate_page_bitmap(pd
+ i
);
702 for (i
= 0; i
< L2_SIZE
; ++i
) {
703 page_flush_tb_1 (level
- 1, pp
+ i
);
708 static void page_flush_tb(void)
711 for (i
= 0; i
< V_L1_SIZE
; i
++) {
712 page_flush_tb_1(V_L1_SHIFT
/ L2_BITS
- 1, l1_map
+ i
);
716 /* flush all the translation blocks */
717 /* XXX: tb_flush is currently not thread safe */
718 void tb_flush(CPUState
*env1
)
721 #if defined(DEBUG_FLUSH)
722 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
723 (unsigned long)(code_gen_ptr
- code_gen_buffer
),
725 ((unsigned long)(code_gen_ptr
- code_gen_buffer
)) / nb_tbs
: 0);
727 if ((unsigned long)(code_gen_ptr
- code_gen_buffer
) > code_gen_buffer_size
)
728 cpu_abort(env1
, "Internal error: code buffer overflow\n");
732 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
733 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
736 memset (tb_phys_hash
, 0, CODE_GEN_PHYS_HASH_SIZE
* sizeof (void *));
739 code_gen_ptr
= code_gen_buffer
;
740 /* XXX: flush processor icache at this point if cache flush is
745 #ifdef DEBUG_TB_CHECK
747 static void tb_invalidate_check(target_ulong address
)
749 TranslationBlock
*tb
;
751 address
&= TARGET_PAGE_MASK
;
752 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
753 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
754 if (!(address
+ TARGET_PAGE_SIZE
<= tb
->pc
||
755 address
>= tb
->pc
+ tb
->size
)) {
756 printf("ERROR invalidate: address=" TARGET_FMT_lx
757 " PC=%08lx size=%04x\n",
758 address
, (long)tb
->pc
, tb
->size
);
764 /* verify that all the pages have correct rights for code */
765 static void tb_page_check(void)
767 TranslationBlock
*tb
;
768 int i
, flags1
, flags2
;
770 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
771 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
772 flags1
= page_get_flags(tb
->pc
);
773 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
774 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
775 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
776 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
784 /* invalidate one TB */
785 static inline void tb_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
,
788 TranslationBlock
*tb1
;
792 *ptb
= *(TranslationBlock
**)((char *)tb1
+ next_offset
);
795 ptb
= (TranslationBlock
**)((char *)tb1
+ next_offset
);
799 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
801 TranslationBlock
*tb1
;
807 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
809 *ptb
= tb1
->page_next
[n1
];
812 ptb
= &tb1
->page_next
[n1
];
816 static inline void tb_jmp_remove(TranslationBlock
*tb
, int n
)
818 TranslationBlock
*tb1
, **ptb
;
821 ptb
= &tb
->jmp_next
[n
];
824 /* find tb(n) in circular list */
828 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
829 if (n1
== n
&& tb1
== tb
)
832 ptb
= &tb1
->jmp_first
;
834 ptb
= &tb1
->jmp_next
[n1
];
837 /* now we can suppress tb(n) from the list */
838 *ptb
= tb
->jmp_next
[n
];
840 tb
->jmp_next
[n
] = NULL
;
844 /* reset the jump entry 'n' of a TB so that it is not chained to
846 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
848 tb_set_jmp_target(tb
, n
, (unsigned long)(tb
->tc_ptr
+ tb
->tb_next_offset
[n
]));
851 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
856 tb_page_addr_t phys_pc
;
857 TranslationBlock
*tb1
, *tb2
;
859 /* remove the TB from the hash list */
860 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
861 h
= tb_phys_hash_func(phys_pc
);
862 tb_remove(&tb_phys_hash
[h
], tb
,
863 offsetof(TranslationBlock
, phys_hash_next
));
865 /* remove the TB from the page list */
866 if (tb
->page_addr
[0] != page_addr
) {
867 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
868 tb_page_remove(&p
->first_tb
, tb
);
869 invalidate_page_bitmap(p
);
871 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
872 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
873 tb_page_remove(&p
->first_tb
, tb
);
874 invalidate_page_bitmap(p
);
877 tb_invalidated_flag
= 1;
879 /* remove the TB from the hash list */
880 h
= tb_jmp_cache_hash_func(tb
->pc
);
881 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
882 if (env
->tb_jmp_cache
[h
] == tb
)
883 env
->tb_jmp_cache
[h
] = NULL
;
886 /* suppress this TB from the two jump lists */
887 tb_jmp_remove(tb
, 0);
888 tb_jmp_remove(tb
, 1);
890 /* suppress any remaining jumps to this TB */
896 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
897 tb2
= tb1
->jmp_next
[n1
];
898 tb_reset_jump(tb1
, n1
);
899 tb1
->jmp_next
[n1
] = NULL
;
902 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2); /* fail safe */
904 tb_phys_invalidate_count
++;
907 static inline void set_bits(uint8_t *tab
, int start
, int len
)
913 mask
= 0xff << (start
& 7);
914 if ((start
& ~7) == (end
& ~7)) {
916 mask
&= ~(0xff << (end
& 7));
921 start
= (start
+ 8) & ~7;
923 while (start
< end1
) {
928 mask
= ~(0xff << (end
& 7));
934 static void build_page_bitmap(PageDesc
*p
)
936 int n
, tb_start
, tb_end
;
937 TranslationBlock
*tb
;
939 p
->code_bitmap
= qemu_mallocz(TARGET_PAGE_SIZE
/ 8);
944 tb
= (TranslationBlock
*)((long)tb
& ~3);
945 /* NOTE: this is subtle as a TB may span two physical pages */
947 /* NOTE: tb_end may be after the end of the page, but
948 it is not a problem */
949 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
950 tb_end
= tb_start
+ tb
->size
;
951 if (tb_end
> TARGET_PAGE_SIZE
)
952 tb_end
= TARGET_PAGE_SIZE
;
955 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
957 set_bits(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
958 tb
= tb
->page_next
[n
];
962 TranslationBlock
*tb_gen_code(CPUState
*env
,
963 target_ulong pc
, target_ulong cs_base
,
964 int flags
, int cflags
)
966 TranslationBlock
*tb
;
968 tb_page_addr_t phys_pc
, phys_page2
;
969 target_ulong virt_page2
;
972 phys_pc
= get_page_addr_code(env
, pc
);
975 /* flush must be done */
977 /* cannot fail at this point */
979 /* Don't forget to invalidate previous TB info. */
980 tb_invalidated_flag
= 1;
982 tc_ptr
= code_gen_ptr
;
984 tb
->cs_base
= cs_base
;
987 cpu_gen_code(env
, tb
, &code_gen_size
);
988 code_gen_ptr
= (void *)(((unsigned long)code_gen_ptr
+ code_gen_size
+ CODE_GEN_ALIGN
- 1) & ~(CODE_GEN_ALIGN
- 1));
990 /* check next page if needed */
991 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
993 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
994 phys_page2
= get_page_addr_code(env
, virt_page2
);
996 tb_link_page(tb
, phys_pc
, phys_page2
);
1000 /* invalidate all TBs which intersect with the target physical page
1001 starting in range [start;end[. NOTE: start and end must refer to
1002 the same physical page. 'is_cpu_write_access' should be true if called
1003 from a real cpu write access: the virtual CPU will exit the current
1004 TB if code is modified inside this TB. */
1005 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
1006 int is_cpu_write_access
)
1008 TranslationBlock
*tb
, *tb_next
, *saved_tb
;
1009 CPUState
*env
= cpu_single_env
;
1010 tb_page_addr_t tb_start
, tb_end
;
1013 #ifdef TARGET_HAS_PRECISE_SMC
1014 int current_tb_not_found
= is_cpu_write_access
;
1015 TranslationBlock
*current_tb
= NULL
;
1016 int current_tb_modified
= 0;
1017 target_ulong current_pc
= 0;
1018 target_ulong current_cs_base
= 0;
1019 int current_flags
= 0;
1020 #endif /* TARGET_HAS_PRECISE_SMC */
1022 p
= page_find(start
>> TARGET_PAGE_BITS
);
1025 if (!p
->code_bitmap
&&
1026 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
&&
1027 is_cpu_write_access
) {
1028 /* build code bitmap */
1029 build_page_bitmap(p
);
1032 /* we remove all the TBs in the range [start, end[ */
1033 /* XXX: see if in some cases it could be faster to invalidate all the code */
1035 while (tb
!= NULL
) {
1037 tb
= (TranslationBlock
*)((long)tb
& ~3);
1038 tb_next
= tb
->page_next
[n
];
1039 /* NOTE: this is subtle as a TB may span two physical pages */
1041 /* NOTE: tb_end may be after the end of the page, but
1042 it is not a problem */
1043 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1044 tb_end
= tb_start
+ tb
->size
;
1046 tb_start
= tb
->page_addr
[1];
1047 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1049 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1050 #ifdef TARGET_HAS_PRECISE_SMC
1051 if (current_tb_not_found
) {
1052 current_tb_not_found
= 0;
1054 if (env
->mem_io_pc
) {
1055 /* now we have a real cpu fault */
1056 current_tb
= tb_find_pc(env
->mem_io_pc
);
1059 if (current_tb
== tb
&&
1060 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1061 /* If we are modifying the current TB, we must stop
1062 its execution. We could be more precise by checking
1063 that the modification is after the current PC, but it
1064 would require a specialized function to partially
1065 restore the CPU state */
1067 current_tb_modified
= 1;
1068 cpu_restore_state(current_tb
, env
,
1069 env
->mem_io_pc
, NULL
);
1070 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1073 #endif /* TARGET_HAS_PRECISE_SMC */
1074 /* we need to do that to handle the case where a signal
1075 occurs while doing tb_phys_invalidate() */
1078 saved_tb
= env
->current_tb
;
1079 env
->current_tb
= NULL
;
1081 tb_phys_invalidate(tb
, -1);
1083 env
->current_tb
= saved_tb
;
1084 if (env
->interrupt_request
&& env
->current_tb
)
1085 cpu_interrupt(env
, env
->interrupt_request
);
1090 #if !defined(CONFIG_USER_ONLY)
1091 /* if no code remaining, no need to continue to use slow writes */
1093 invalidate_page_bitmap(p
);
1094 if (is_cpu_write_access
) {
1095 tlb_unprotect_code_phys(env
, start
, env
->mem_io_vaddr
);
1099 #ifdef TARGET_HAS_PRECISE_SMC
1100 if (current_tb_modified
) {
1101 /* we generate a block containing just the instruction
1102 modifying the memory. It will ensure that it cannot modify
1104 env
->current_tb
= NULL
;
1105 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1106 cpu_resume_from_signal(env
, NULL
);
1111 /* len must be <= 8 and start must be a multiple of len */
1112 static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start
, int len
)
1118 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1119 cpu_single_env
->mem_io_vaddr
, len
,
1120 cpu_single_env
->eip
,
1121 cpu_single_env
->eip
+ (long)cpu_single_env
->segs
[R_CS
].base
);
1124 p
= page_find(start
>> TARGET_PAGE_BITS
);
1127 if (p
->code_bitmap
) {
1128 offset
= start
& ~TARGET_PAGE_MASK
;
1129 b
= p
->code_bitmap
[offset
>> 3] >> (offset
& 7);
1130 if (b
& ((1 << len
) - 1))
1134 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1138 #if !defined(CONFIG_SOFTMMU)
1139 static void tb_invalidate_phys_page(tb_page_addr_t addr
,
1140 unsigned long pc
, void *puc
)
1142 TranslationBlock
*tb
;
1145 #ifdef TARGET_HAS_PRECISE_SMC
1146 TranslationBlock
*current_tb
= NULL
;
1147 CPUState
*env
= cpu_single_env
;
1148 int current_tb_modified
= 0;
1149 target_ulong current_pc
= 0;
1150 target_ulong current_cs_base
= 0;
1151 int current_flags
= 0;
1154 addr
&= TARGET_PAGE_MASK
;
1155 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1159 #ifdef TARGET_HAS_PRECISE_SMC
1160 if (tb
&& pc
!= 0) {
1161 current_tb
= tb_find_pc(pc
);
1164 while (tb
!= NULL
) {
1166 tb
= (TranslationBlock
*)((long)tb
& ~3);
1167 #ifdef TARGET_HAS_PRECISE_SMC
1168 if (current_tb
== tb
&&
1169 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1170 /* If we are modifying the current TB, we must stop
1171 its execution. We could be more precise by checking
1172 that the modification is after the current PC, but it
1173 would require a specialized function to partially
1174 restore the CPU state */
1176 current_tb_modified
= 1;
1177 cpu_restore_state(current_tb
, env
, pc
, puc
);
1178 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1181 #endif /* TARGET_HAS_PRECISE_SMC */
1182 tb_phys_invalidate(tb
, addr
);
1183 tb
= tb
->page_next
[n
];
1186 #ifdef TARGET_HAS_PRECISE_SMC
1187 if (current_tb_modified
) {
1188 /* we generate a block containing just the instruction
1189 modifying the memory. It will ensure that it cannot modify
1191 env
->current_tb
= NULL
;
1192 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1193 cpu_resume_from_signal(env
, puc
);
1199 /* add the tb in the target page and protect it if necessary */
1200 static inline void tb_alloc_page(TranslationBlock
*tb
,
1201 unsigned int n
, tb_page_addr_t page_addr
)
1204 TranslationBlock
*last_first_tb
;
1206 tb
->page_addr
[n
] = page_addr
;
1207 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
, 1);
1208 tb
->page_next
[n
] = p
->first_tb
;
1209 last_first_tb
= p
->first_tb
;
1210 p
->first_tb
= (TranslationBlock
*)((long)tb
| n
);
1211 invalidate_page_bitmap(p
);
1213 #if defined(TARGET_HAS_SMC) || 1
1215 #if defined(CONFIG_USER_ONLY)
1216 if (p
->flags
& PAGE_WRITE
) {
1221 /* force the host page as non writable (writes will have a
1222 page fault + mprotect overhead) */
1223 page_addr
&= qemu_host_page_mask
;
1225 for(addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1226 addr
+= TARGET_PAGE_SIZE
) {
1228 p2
= page_find (addr
>> TARGET_PAGE_BITS
);
1232 p2
->flags
&= ~PAGE_WRITE
;
1234 mprotect(g2h(page_addr
), qemu_host_page_size
,
1235 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1236 #ifdef DEBUG_TB_INVALIDATE
1237 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1242 /* if some code is already present, then the pages are already
1243 protected. So we handle the case where only the first TB is
1244 allocated in a physical page */
1245 if (!last_first_tb
) {
1246 tlb_protect_code(page_addr
);
1250 #endif /* TARGET_HAS_SMC */
1253 /* Allocate a new translation block. Flush the translation buffer if
1254 too many translation blocks or too much generated code. */
1255 TranslationBlock
*tb_alloc(target_ulong pc
)
1257 TranslationBlock
*tb
;
1259 if (nb_tbs
>= code_gen_max_blocks
||
1260 (code_gen_ptr
- code_gen_buffer
) >= code_gen_buffer_max_size
)
1262 tb
= &tbs
[nb_tbs
++];
1268 void tb_free(TranslationBlock
*tb
)
1270 /* In practice this is mostly used for single use temporary TB
1271 Ignore the hard cases and just back up if this TB happens to
1272 be the last one generated. */
1273 if (nb_tbs
> 0 && tb
== &tbs
[nb_tbs
- 1]) {
1274 code_gen_ptr
= tb
->tc_ptr
;
1279 /* add a new TB and link it to the physical page tables. phys_page2 is
1280 (-1) to indicate that only one page contains the TB. */
1281 void tb_link_page(TranslationBlock
*tb
,
1282 tb_page_addr_t phys_pc
, tb_page_addr_t phys_page2
)
1285 TranslationBlock
**ptb
;
1287 /* Grab the mmap lock to stop another thread invalidating this TB
1288 before we are done. */
1290 /* add in the physical hash table */
1291 h
= tb_phys_hash_func(phys_pc
);
1292 ptb
= &tb_phys_hash
[h
];
1293 tb
->phys_hash_next
= *ptb
;
1296 /* add in the page list */
1297 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1298 if (phys_page2
!= -1)
1299 tb_alloc_page(tb
, 1, phys_page2
);
1301 tb
->page_addr
[1] = -1;
1303 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2);
1304 tb
->jmp_next
[0] = NULL
;
1305 tb
->jmp_next
[1] = NULL
;
1307 /* init original jump addresses */
1308 if (tb
->tb_next_offset
[0] != 0xffff)
1309 tb_reset_jump(tb
, 0);
1310 if (tb
->tb_next_offset
[1] != 0xffff)
1311 tb_reset_jump(tb
, 1);
1313 #ifdef DEBUG_TB_CHECK
1319 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1320 tb[1].tc_ptr. Return NULL if not found */
1321 TranslationBlock
*tb_find_pc(unsigned long tc_ptr
)
1323 int m_min
, m_max
, m
;
1325 TranslationBlock
*tb
;
1329 if (tc_ptr
< (unsigned long)code_gen_buffer
||
1330 tc_ptr
>= (unsigned long)code_gen_ptr
)
1332 /* binary search (cf Knuth) */
1335 while (m_min
<= m_max
) {
1336 m
= (m_min
+ m_max
) >> 1;
1338 v
= (unsigned long)tb
->tc_ptr
;
1341 else if (tc_ptr
< v
) {
1350 static void tb_reset_jump_recursive(TranslationBlock
*tb
);
1352 static inline void tb_reset_jump_recursive2(TranslationBlock
*tb
, int n
)
1354 TranslationBlock
*tb1
, *tb_next
, **ptb
;
1357 tb1
= tb
->jmp_next
[n
];
1359 /* find head of list */
1362 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1365 tb1
= tb1
->jmp_next
[n1
];
1367 /* we are now sure now that tb jumps to tb1 */
1370 /* remove tb from the jmp_first list */
1371 ptb
= &tb_next
->jmp_first
;
1375 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1376 if (n1
== n
&& tb1
== tb
)
1378 ptb
= &tb1
->jmp_next
[n1
];
1380 *ptb
= tb
->jmp_next
[n
];
1381 tb
->jmp_next
[n
] = NULL
;
1383 /* suppress the jump to next tb in generated code */
1384 tb_reset_jump(tb
, n
);
1386 /* suppress jumps in the tb on which we could have jumped */
1387 tb_reset_jump_recursive(tb_next
);
1391 static void tb_reset_jump_recursive(TranslationBlock
*tb
)
1393 tb_reset_jump_recursive2(tb
, 0);
1394 tb_reset_jump_recursive2(tb
, 1);
1397 #if defined(TARGET_HAS_ICE)
1398 #if defined(CONFIG_USER_ONLY)
1399 static void breakpoint_invalidate(CPUState
*env
, target_ulong pc
)
1401 tb_invalidate_phys_page_range(pc
, pc
+ 1, 0);
1404 static void breakpoint_invalidate(CPUState
*env
, target_ulong pc
)
1406 target_phys_addr_t addr
;
1408 ram_addr_t ram_addr
;
1411 addr
= cpu_get_phys_page_debug(env
, pc
);
1412 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
1414 pd
= IO_MEM_UNASSIGNED
;
1416 pd
= p
->phys_offset
;
1418 ram_addr
= (pd
& TARGET_PAGE_MASK
) | (pc
& ~TARGET_PAGE_MASK
);
1419 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1422 #endif /* TARGET_HAS_ICE */
1424 #if defined(CONFIG_USER_ONLY)
1425 void cpu_watchpoint_remove_all(CPUState
*env
, int mask
)
1430 int cpu_watchpoint_insert(CPUState
*env
, target_ulong addr
, target_ulong len
,
1431 int flags
, CPUWatchpoint
**watchpoint
)
1436 /* Add a watchpoint. */
1437 int cpu_watchpoint_insert(CPUState
*env
, target_ulong addr
, target_ulong len
,
1438 int flags
, CPUWatchpoint
**watchpoint
)
1440 target_ulong len_mask
= ~(len
- 1);
1443 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1444 if ((len
!= 1 && len
!= 2 && len
!= 4 && len
!= 8) || (addr
& ~len_mask
)) {
1445 fprintf(stderr
, "qemu: tried to set invalid watchpoint at "
1446 TARGET_FMT_lx
", len=" TARGET_FMT_lu
"\n", addr
, len
);
1449 wp
= qemu_malloc(sizeof(*wp
));
1452 wp
->len_mask
= len_mask
;
1455 /* keep all GDB-injected watchpoints in front */
1457 QTAILQ_INSERT_HEAD(&env
->watchpoints
, wp
, entry
);
1459 QTAILQ_INSERT_TAIL(&env
->watchpoints
, wp
, entry
);
1461 tlb_flush_page(env
, addr
);
1468 /* Remove a specific watchpoint. */
1469 int cpu_watchpoint_remove(CPUState
*env
, target_ulong addr
, target_ulong len
,
1472 target_ulong len_mask
= ~(len
- 1);
1475 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1476 if (addr
== wp
->vaddr
&& len_mask
== wp
->len_mask
1477 && flags
== (wp
->flags
& ~BP_WATCHPOINT_HIT
)) {
1478 cpu_watchpoint_remove_by_ref(env
, wp
);
1485 /* Remove a specific watchpoint by reference. */
1486 void cpu_watchpoint_remove_by_ref(CPUState
*env
, CPUWatchpoint
*watchpoint
)
1488 QTAILQ_REMOVE(&env
->watchpoints
, watchpoint
, entry
);
1490 tlb_flush_page(env
, watchpoint
->vaddr
);
1492 qemu_free(watchpoint
);
1495 /* Remove all matching watchpoints. */
1496 void cpu_watchpoint_remove_all(CPUState
*env
, int mask
)
1498 CPUWatchpoint
*wp
, *next
;
1500 QTAILQ_FOREACH_SAFE(wp
, &env
->watchpoints
, entry
, next
) {
1501 if (wp
->flags
& mask
)
1502 cpu_watchpoint_remove_by_ref(env
, wp
);
1507 /* Add a breakpoint. */
1508 int cpu_breakpoint_insert(CPUState
*env
, target_ulong pc
, int flags
,
1509 CPUBreakpoint
**breakpoint
)
1511 #if defined(TARGET_HAS_ICE)
1514 bp
= qemu_malloc(sizeof(*bp
));
1519 /* keep all GDB-injected breakpoints in front */
1521 QTAILQ_INSERT_HEAD(&env
->breakpoints
, bp
, entry
);
1523 QTAILQ_INSERT_TAIL(&env
->breakpoints
, bp
, entry
);
1525 breakpoint_invalidate(env
, pc
);
1535 /* Remove a specific breakpoint. */
1536 int cpu_breakpoint_remove(CPUState
*env
, target_ulong pc
, int flags
)
1538 #if defined(TARGET_HAS_ICE)
1541 QTAILQ_FOREACH(bp
, &env
->breakpoints
, entry
) {
1542 if (bp
->pc
== pc
&& bp
->flags
== flags
) {
1543 cpu_breakpoint_remove_by_ref(env
, bp
);
1553 /* Remove a specific breakpoint by reference. */
1554 void cpu_breakpoint_remove_by_ref(CPUState
*env
, CPUBreakpoint
*breakpoint
)
1556 #if defined(TARGET_HAS_ICE)
1557 QTAILQ_REMOVE(&env
->breakpoints
, breakpoint
, entry
);
1559 breakpoint_invalidate(env
, breakpoint
->pc
);
1561 qemu_free(breakpoint
);
1565 /* Remove all matching breakpoints. */
1566 void cpu_breakpoint_remove_all(CPUState
*env
, int mask
)
1568 #if defined(TARGET_HAS_ICE)
1569 CPUBreakpoint
*bp
, *next
;
1571 QTAILQ_FOREACH_SAFE(bp
, &env
->breakpoints
, entry
, next
) {
1572 if (bp
->flags
& mask
)
1573 cpu_breakpoint_remove_by_ref(env
, bp
);
1578 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1579 CPU loop after each instruction */
1580 void cpu_single_step(CPUState
*env
, int enabled
)
1582 #if defined(TARGET_HAS_ICE)
1583 if (env
->singlestep_enabled
!= enabled
) {
1584 env
->singlestep_enabled
= enabled
;
1586 kvm_update_guest_debug(env
, 0);
1588 /* must flush all the translated code to avoid inconsistencies */
1589 /* XXX: only flush what is necessary */
1596 /* enable or disable low levels log */
1597 void cpu_set_log(int log_flags
)
1599 loglevel
= log_flags
;
1600 if (loglevel
&& !logfile
) {
1601 logfile
= fopen(logfilename
, log_append
? "a" : "w");
1603 perror(logfilename
);
1606 #if !defined(CONFIG_SOFTMMU)
1607 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1609 static char logfile_buf
[4096];
1610 setvbuf(logfile
, logfile_buf
, _IOLBF
, sizeof(logfile_buf
));
1612 #elif !defined(_WIN32)
1613 /* Win32 doesn't support line-buffering and requires size >= 2 */
1614 setvbuf(logfile
, NULL
, _IOLBF
, 0);
1618 if (!loglevel
&& logfile
) {
1624 void cpu_set_log_filename(const char *filename
)
1626 logfilename
= strdup(filename
);
1631 cpu_set_log(loglevel
);
1634 static void cpu_unlink_tb(CPUState
*env
)
1636 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1637 problem and hope the cpu will stop of its own accord. For userspace
1638 emulation this often isn't actually as bad as it sounds. Often
1639 signals are used primarily to interrupt blocking syscalls. */
1640 TranslationBlock
*tb
;
1641 static spinlock_t interrupt_lock
= SPIN_LOCK_UNLOCKED
;
1643 spin_lock(&interrupt_lock
);
1644 tb
= env
->current_tb
;
1645 /* if the cpu is currently executing code, we must unlink it and
1646 all the potentially executing TB */
1648 env
->current_tb
= NULL
;
1649 tb_reset_jump_recursive(tb
);
1651 spin_unlock(&interrupt_lock
);
1654 /* mask must never be zero, except for A20 change call */
1655 void cpu_interrupt(CPUState
*env
, int mask
)
1659 old_mask
= env
->interrupt_request
;
1660 env
->interrupt_request
|= mask
;
1661 if (kvm_enabled() && !kvm_irqchip_in_kernel())
1662 kvm_update_interrupt_request(env
);
1664 #ifndef CONFIG_USER_ONLY
1666 * If called from iothread context, wake the target cpu in
1669 if (!qemu_cpu_self(env
)) {
1676 env
->icount_decr
.u16
.high
= 0xffff;
1677 #ifndef CONFIG_USER_ONLY
1679 && (mask
& ~old_mask
) != 0) {
1680 cpu_abort(env
, "Raised interrupt while not in I/O function");
1688 void cpu_reset_interrupt(CPUState
*env
, int mask
)
1690 env
->interrupt_request
&= ~mask
;
1693 void cpu_exit(CPUState
*env
)
1695 env
->exit_request
= 1;
1699 const CPULogItem cpu_log_items
[] = {
1700 { CPU_LOG_TB_OUT_ASM
, "out_asm",
1701 "show generated host assembly code for each compiled TB" },
1702 { CPU_LOG_TB_IN_ASM
, "in_asm",
1703 "show target assembly code for each compiled TB" },
1704 { CPU_LOG_TB_OP
, "op",
1705 "show micro ops for each compiled TB" },
1706 { CPU_LOG_TB_OP_OPT
, "op_opt",
1709 "before eflags optimization and "
1711 "after liveness analysis" },
1712 { CPU_LOG_INT
, "int",
1713 "show interrupts/exceptions in short format" },
1714 { CPU_LOG_EXEC
, "exec",
1715 "show trace before each executed TB (lots of logs)" },
1716 { CPU_LOG_TB_CPU
, "cpu",
1717 "show CPU state before block translation" },
1719 { CPU_LOG_PCALL
, "pcall",
1720 "show protected mode far calls/returns/exceptions" },
1721 { CPU_LOG_RESET
, "cpu_reset",
1722 "show CPU state before CPU resets" },
1725 { CPU_LOG_IOPORT
, "ioport",
1726 "show all i/o ports accesses" },
1731 #ifndef CONFIG_USER_ONLY
1732 static QLIST_HEAD(memory_client_list
, CPUPhysMemoryClient
) memory_client_list
1733 = QLIST_HEAD_INITIALIZER(memory_client_list
);
1735 static void cpu_notify_set_memory(target_phys_addr_t start_addr
,
1737 ram_addr_t phys_offset
)
1739 CPUPhysMemoryClient
*client
;
1740 QLIST_FOREACH(client
, &memory_client_list
, list
) {
1741 client
->set_memory(client
, start_addr
, size
, phys_offset
);
1745 static int cpu_notify_sync_dirty_bitmap(target_phys_addr_t start
,
1746 target_phys_addr_t end
)
1748 CPUPhysMemoryClient
*client
;
1749 QLIST_FOREACH(client
, &memory_client_list
, list
) {
1750 int r
= client
->sync_dirty_bitmap(client
, start
, end
);
1757 static int cpu_notify_migration_log(int enable
)
1759 CPUPhysMemoryClient
*client
;
1760 QLIST_FOREACH(client
, &memory_client_list
, list
) {
1761 int r
= client
->migration_log(client
, enable
);
1768 static void phys_page_for_each_1(CPUPhysMemoryClient
*client
,
1769 int level
, void **lp
)
1777 PhysPageDesc
*pd
= *lp
;
1778 for (i
= 0; i
< L2_SIZE
; ++i
) {
1779 if (pd
[i
].phys_offset
!= IO_MEM_UNASSIGNED
) {
1780 client
->set_memory(client
, pd
[i
].region_offset
,
1781 TARGET_PAGE_SIZE
, pd
[i
].phys_offset
);
1786 for (i
= 0; i
< L2_SIZE
; ++i
) {
1787 phys_page_for_each_1(client
, level
- 1, pp
+ i
);
1792 static void phys_page_for_each(CPUPhysMemoryClient
*client
)
1795 for (i
= 0; i
< P_L1_SIZE
; ++i
) {
1796 phys_page_for_each_1(client
, P_L1_SHIFT
/ L2_BITS
- 1,
1801 void cpu_register_phys_memory_client(CPUPhysMemoryClient
*client
)
1803 QLIST_INSERT_HEAD(&memory_client_list
, client
, list
);
1804 phys_page_for_each(client
);
1807 void cpu_unregister_phys_memory_client(CPUPhysMemoryClient
*client
)
1809 QLIST_REMOVE(client
, list
);
1813 static int cmp1(const char *s1
, int n
, const char *s2
)
1815 if (strlen(s2
) != n
)
1817 return memcmp(s1
, s2
, n
) == 0;
1820 /* takes a comma separated list of log masks. Return 0 if error. */
1821 int cpu_str_to_log_mask(const char *str
)
1823 const CPULogItem
*item
;
1830 p1
= strchr(p
, ',');
1833 if(cmp1(p
,p1
-p
,"all")) {
1834 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1838 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1839 if (cmp1(p
, p1
- p
, item
->name
))
1853 void cpu_abort(CPUState
*env
, const char *fmt
, ...)
1860 fprintf(stderr
, "qemu: fatal: ");
1861 vfprintf(stderr
, fmt
, ap
);
1862 fprintf(stderr
, "\n");
1864 cpu_dump_state(env
, stderr
, fprintf
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1866 cpu_dump_state(env
, stderr
, fprintf
, 0);
1868 if (qemu_log_enabled()) {
1869 qemu_log("qemu: fatal: ");
1870 qemu_log_vprintf(fmt
, ap2
);
1873 log_cpu_state(env
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1875 log_cpu_state(env
, 0);
1882 #if defined(CONFIG_USER_ONLY)
1884 struct sigaction act
;
1885 sigfillset(&act
.sa_mask
);
1886 act
.sa_handler
= SIG_DFL
;
1887 sigaction(SIGABRT
, &act
, NULL
);
1893 CPUState
*cpu_copy(CPUState
*env
)
1895 CPUState
*new_env
= cpu_init(env
->cpu_model_str
);
1896 CPUState
*next_cpu
= new_env
->next_cpu
;
1897 int cpu_index
= new_env
->cpu_index
;
1898 #if defined(TARGET_HAS_ICE)
1903 memcpy(new_env
, env
, sizeof(CPUState
));
1905 /* Preserve chaining and index. */
1906 new_env
->next_cpu
= next_cpu
;
1907 new_env
->cpu_index
= cpu_index
;
1909 /* Clone all break/watchpoints.
1910 Note: Once we support ptrace with hw-debug register access, make sure
1911 BP_CPU break/watchpoints are handled correctly on clone. */
1912 QTAILQ_INIT(&env
->breakpoints
);
1913 QTAILQ_INIT(&env
->watchpoints
);
1914 #if defined(TARGET_HAS_ICE)
1915 QTAILQ_FOREACH(bp
, &env
->breakpoints
, entry
) {
1916 cpu_breakpoint_insert(new_env
, bp
->pc
, bp
->flags
, NULL
);
1918 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1919 cpu_watchpoint_insert(new_env
, wp
->vaddr
, (~wp
->len_mask
) + 1,
1927 #if !defined(CONFIG_USER_ONLY)
1929 static inline void tlb_flush_jmp_cache(CPUState
*env
, target_ulong addr
)
1933 /* Discard jump cache entries for any tb which might potentially
1934 overlap the flushed page. */
1935 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1936 memset (&env
->tb_jmp_cache
[i
], 0,
1937 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1939 i
= tb_jmp_cache_hash_page(addr
);
1940 memset (&env
->tb_jmp_cache
[i
], 0,
1941 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1944 static CPUTLBEntry s_cputlb_empty_entry
= {
1951 /* NOTE: if flush_global is true, also flush global entries (not
1953 void tlb_flush(CPUState
*env
, int flush_global
)
1957 #if defined(DEBUG_TLB)
1958 printf("tlb_flush:\n");
1960 /* must reset current TB so that interrupts cannot modify the
1961 links while we are modifying them */
1962 env
->current_tb
= NULL
;
1964 for(i
= 0; i
< CPU_TLB_SIZE
; i
++) {
1966 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
1967 env
->tlb_table
[mmu_idx
][i
] = s_cputlb_empty_entry
;
1971 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
1973 env
->tlb_flush_addr
= -1;
1974 env
->tlb_flush_mask
= 0;
1978 static inline void tlb_flush_entry(CPUTLBEntry
*tlb_entry
, target_ulong addr
)
1980 if (addr
== (tlb_entry
->addr_read
&
1981 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1982 addr
== (tlb_entry
->addr_write
&
1983 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1984 addr
== (tlb_entry
->addr_code
&
1985 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
))) {
1986 *tlb_entry
= s_cputlb_empty_entry
;
1990 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
1995 #if defined(DEBUG_TLB)
1996 printf("tlb_flush_page: " TARGET_FMT_lx
"\n", addr
);
1998 /* Check if we need to flush due to large pages. */
1999 if ((addr
& env
->tlb_flush_mask
) == env
->tlb_flush_addr
) {
2000 #if defined(DEBUG_TLB)
2001 printf("tlb_flush_page: forced full flush ("
2002 TARGET_FMT_lx
"/" TARGET_FMT_lx
")\n",
2003 env
->tlb_flush_addr
, env
->tlb_flush_mask
);
2008 /* must reset current TB so that interrupts cannot modify the
2009 links while we are modifying them */
2010 env
->current_tb
= NULL
;
2012 addr
&= TARGET_PAGE_MASK
;
2013 i
= (addr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
2014 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++)
2015 tlb_flush_entry(&env
->tlb_table
[mmu_idx
][i
], addr
);
2017 tlb_flush_jmp_cache(env
, addr
);
2020 /* update the TLBs so that writes to code in the virtual page 'addr'
2022 static void tlb_protect_code(ram_addr_t ram_addr
)
2024 cpu_physical_memory_reset_dirty(ram_addr
,
2025 ram_addr
+ TARGET_PAGE_SIZE
,
2029 /* update the TLB so that writes in physical page 'phys_addr' are no longer
2030 tested for self modifying code */
2031 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
2034 cpu_physical_memory_set_dirty_flags(ram_addr
, CODE_DIRTY_FLAG
);
2037 static inline void tlb_reset_dirty_range(CPUTLBEntry
*tlb_entry
,
2038 unsigned long start
, unsigned long length
)
2041 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
) {
2042 addr
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) + tlb_entry
->addend
;
2043 if ((addr
- start
) < length
) {
2044 tlb_entry
->addr_write
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) | TLB_NOTDIRTY
;
2049 /* Note: start and end must be within the same ram block. */
2050 void cpu_physical_memory_reset_dirty(ram_addr_t start
, ram_addr_t end
,
2054 unsigned long length
, start1
;
2057 start
&= TARGET_PAGE_MASK
;
2058 end
= TARGET_PAGE_ALIGN(end
);
2060 length
= end
- start
;
2063 cpu_physical_memory_mask_dirty_range(start
, length
, dirty_flags
);
2065 /* we modify the TLB cache so that the dirty bit will be set again
2066 when accessing the range */
2067 start1
= (unsigned long)qemu_get_ram_ptr(start
);
2068 /* Chek that we don't span multiple blocks - this breaks the
2069 address comparisons below. */
2070 if ((unsigned long)qemu_get_ram_ptr(end
- 1) - start1
2071 != (end
- 1) - start
) {
2075 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
2077 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
2078 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
2079 tlb_reset_dirty_range(&env
->tlb_table
[mmu_idx
][i
],
2085 int cpu_physical_memory_set_dirty_tracking(int enable
)
2088 in_migration
= enable
;
2089 ret
= cpu_notify_migration_log(!!enable
);
2093 int cpu_physical_memory_get_dirty_tracking(void)
2095 return in_migration
;
2098 int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr
,
2099 target_phys_addr_t end_addr
)
2103 ret
= cpu_notify_sync_dirty_bitmap(start_addr
, end_addr
);
2107 static inline void tlb_update_dirty(CPUTLBEntry
*tlb_entry
)
2109 ram_addr_t ram_addr
;
2112 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
) {
2113 p
= (void *)(unsigned long)((tlb_entry
->addr_write
& TARGET_PAGE_MASK
)
2114 + tlb_entry
->addend
);
2115 ram_addr
= qemu_ram_addr_from_host(p
);
2116 if (!cpu_physical_memory_is_dirty(ram_addr
)) {
2117 tlb_entry
->addr_write
|= TLB_NOTDIRTY
;
2122 /* update the TLB according to the current state of the dirty bits */
2123 void cpu_tlb_update_dirty(CPUState
*env
)
2127 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
2128 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
2129 tlb_update_dirty(&env
->tlb_table
[mmu_idx
][i
]);
2133 static inline void tlb_set_dirty1(CPUTLBEntry
*tlb_entry
, target_ulong vaddr
)
2135 if (tlb_entry
->addr_write
== (vaddr
| TLB_NOTDIRTY
))
2136 tlb_entry
->addr_write
= vaddr
;
2139 /* update the TLB corresponding to virtual page vaddr
2140 so that it is no longer dirty */
2141 static inline void tlb_set_dirty(CPUState
*env
, target_ulong vaddr
)
2146 vaddr
&= TARGET_PAGE_MASK
;
2147 i
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
2148 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++)
2149 tlb_set_dirty1(&env
->tlb_table
[mmu_idx
][i
], vaddr
);
2152 /* Our TLB does not support large pages, so remember the area covered by
2153 large pages and trigger a full TLB flush if these are invalidated. */
2154 static void tlb_add_large_page(CPUState
*env
, target_ulong vaddr
,
2157 target_ulong mask
= ~(size
- 1);
2159 if (env
->tlb_flush_addr
== (target_ulong
)-1) {
2160 env
->tlb_flush_addr
= vaddr
& mask
;
2161 env
->tlb_flush_mask
= mask
;
2164 /* Extend the existing region to include the new page.
2165 This is a compromise between unnecessary flushes and the cost
2166 of maintaining a full variable size TLB. */
2167 mask
&= env
->tlb_flush_mask
;
2168 while (((env
->tlb_flush_addr
^ vaddr
) & mask
) != 0) {
2171 env
->tlb_flush_addr
&= mask
;
2172 env
->tlb_flush_mask
= mask
;
2175 /* Add a new TLB entry. At most one entry for a given virtual address
2176 is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
2177 supplied size is only used by tlb_flush_page. */
2178 void tlb_set_page(CPUState
*env
, target_ulong vaddr
,
2179 target_phys_addr_t paddr
, int prot
,
2180 int mmu_idx
, target_ulong size
)
2185 target_ulong address
;
2186 target_ulong code_address
;
2187 unsigned long addend
;
2190 target_phys_addr_t iotlb
;
2192 assert(size
>= TARGET_PAGE_SIZE
);
2193 if (size
!= TARGET_PAGE_SIZE
) {
2194 tlb_add_large_page(env
, vaddr
, size
);
2196 p
= phys_page_find(paddr
>> TARGET_PAGE_BITS
);
2198 pd
= IO_MEM_UNASSIGNED
;
2200 pd
= p
->phys_offset
;
2202 #if defined(DEBUG_TLB)
2203 printf("tlb_set_page: vaddr=" TARGET_FMT_lx
" paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
2204 vaddr
, (int)paddr
, prot
, mmu_idx
, is_softmmu
, pd
);
2208 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&& !(pd
& IO_MEM_ROMD
)) {
2209 /* IO memory case (romd handled later) */
2210 address
|= TLB_MMIO
;
2212 addend
= (unsigned long)qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
);
2213 if ((pd
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
) {
2215 iotlb
= pd
& TARGET_PAGE_MASK
;
2216 if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
)
2217 iotlb
|= IO_MEM_NOTDIRTY
;
2219 iotlb
|= IO_MEM_ROM
;
2221 /* IO handlers are currently passed a physical address.
2222 It would be nice to pass an offset from the base address
2223 of that region. This would avoid having to special case RAM,
2224 and avoid full address decoding in every device.
2225 We can't use the high bits of pd for this because
2226 IO_MEM_ROMD uses these as a ram address. */
2227 iotlb
= (pd
& ~TARGET_PAGE_MASK
);
2229 iotlb
+= p
->region_offset
;
2235 code_address
= address
;
2236 /* Make accesses to pages with watchpoints go via the
2237 watchpoint trap routines. */
2238 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
2239 if (vaddr
== (wp
->vaddr
& TARGET_PAGE_MASK
)) {
2240 iotlb
= io_mem_watch
+ paddr
;
2241 /* TODO: The memory case can be optimized by not trapping
2242 reads of pages with a write breakpoint. */
2243 address
|= TLB_MMIO
;
2247 index
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
2248 env
->iotlb
[mmu_idx
][index
] = iotlb
- vaddr
;
2249 te
= &env
->tlb_table
[mmu_idx
][index
];
2250 te
->addend
= addend
- vaddr
;
2251 if (prot
& PAGE_READ
) {
2252 te
->addr_read
= address
;
2257 if (prot
& PAGE_EXEC
) {
2258 te
->addr_code
= code_address
;
2262 if (prot
& PAGE_WRITE
) {
2263 if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_ROM
||
2264 (pd
& IO_MEM_ROMD
)) {
2265 /* Write access calls the I/O callback. */
2266 te
->addr_write
= address
| TLB_MMIO
;
2267 } else if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
&&
2268 !cpu_physical_memory_is_dirty(pd
)) {
2269 te
->addr_write
= address
| TLB_NOTDIRTY
;
2271 te
->addr_write
= address
;
2274 te
->addr_write
= -1;
2280 void tlb_flush(CPUState
*env
, int flush_global
)
2284 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
2289 * Walks guest process memory "regions" one by one
2290 * and calls callback function 'fn' for each region.
2293 struct walk_memory_regions_data
2295 walk_memory_regions_fn fn
;
2297 unsigned long start
;
2301 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
2302 abi_ulong end
, int new_prot
)
2304 if (data
->start
!= -1ul) {
2305 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
2311 data
->start
= (new_prot
? end
: -1ul);
2312 data
->prot
= new_prot
;
2317 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
2318 abi_ulong base
, int level
, void **lp
)
2324 return walk_memory_regions_end(data
, base
, 0);
2329 for (i
= 0; i
< L2_SIZE
; ++i
) {
2330 int prot
= pd
[i
].flags
;
2332 pa
= base
| (i
<< TARGET_PAGE_BITS
);
2333 if (prot
!= data
->prot
) {
2334 rc
= walk_memory_regions_end(data
, pa
, prot
);
2342 for (i
= 0; i
< L2_SIZE
; ++i
) {
2343 pa
= base
| ((abi_ulong
)i
<<
2344 (TARGET_PAGE_BITS
+ L2_BITS
* level
));
2345 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
2355 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
2357 struct walk_memory_regions_data data
;
2365 for (i
= 0; i
< V_L1_SIZE
; i
++) {
2366 int rc
= walk_memory_regions_1(&data
, (abi_ulong
)i
<< V_L1_SHIFT
,
2367 V_L1_SHIFT
/ L2_BITS
- 1, l1_map
+ i
);
2373 return walk_memory_regions_end(&data
, 0, 0);
2376 static int dump_region(void *priv
, abi_ulong start
,
2377 abi_ulong end
, unsigned long prot
)
2379 FILE *f
= (FILE *)priv
;
2381 (void) fprintf(f
, TARGET_ABI_FMT_lx
"-"TARGET_ABI_FMT_lx
2382 " "TARGET_ABI_FMT_lx
" %c%c%c\n",
2383 start
, end
, end
- start
,
2384 ((prot
& PAGE_READ
) ? 'r' : '-'),
2385 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
2386 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
2391 /* dump memory mappings */
2392 void page_dump(FILE *f
)
2394 (void) fprintf(f
, "%-8s %-8s %-8s %s\n",
2395 "start", "end", "size", "prot");
2396 walk_memory_regions(f
, dump_region
);
2399 int page_get_flags(target_ulong address
)
2403 p
= page_find(address
>> TARGET_PAGE_BITS
);
2409 /* Modify the flags of a page and invalidate the code if necessary.
2410 The flag PAGE_WRITE_ORG is positioned automatically depending
2411 on PAGE_WRITE. The mmap_lock should already be held. */
2412 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
2414 target_ulong addr
, len
;
2416 /* This function should never be called with addresses outside the
2417 guest address space. If this assert fires, it probably indicates
2418 a missing call to h2g_valid. */
2419 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2420 assert(end
< ((abi_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2422 assert(start
< end
);
2424 start
= start
& TARGET_PAGE_MASK
;
2425 end
= TARGET_PAGE_ALIGN(end
);
2427 if (flags
& PAGE_WRITE
) {
2428 flags
|= PAGE_WRITE_ORG
;
2431 for (addr
= start
, len
= end
- start
;
2433 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2434 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2436 /* If the write protection bit is set, then we invalidate
2438 if (!(p
->flags
& PAGE_WRITE
) &&
2439 (flags
& PAGE_WRITE
) &&
2441 tb_invalidate_phys_page(addr
, 0, NULL
);
2447 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
2453 /* This function should never be called with addresses outside the
2454 guest address space. If this assert fires, it probably indicates
2455 a missing call to h2g_valid. */
2456 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2457 assert(start
< ((abi_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2463 if (start
+ len
- 1 < start
) {
2464 /* We've wrapped around. */
2468 end
= TARGET_PAGE_ALIGN(start
+len
); /* must do before we loose bits in the next step */
2469 start
= start
& TARGET_PAGE_MASK
;
2471 for (addr
= start
, len
= end
- start
;
2473 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2474 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2477 if( !(p
->flags
& PAGE_VALID
) )
2480 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
))
2482 if (flags
& PAGE_WRITE
) {
2483 if (!(p
->flags
& PAGE_WRITE_ORG
))
2485 /* unprotect the page if it was put read-only because it
2486 contains translated code */
2487 if (!(p
->flags
& PAGE_WRITE
)) {
2488 if (!page_unprotect(addr
, 0, NULL
))
2497 /* called from signal handler: invalidate the code and unprotect the
2498 page. Return TRUE if the fault was successfully handled. */
2499 int page_unprotect(target_ulong address
, unsigned long pc
, void *puc
)
2503 target_ulong host_start
, host_end
, addr
;
2505 /* Technically this isn't safe inside a signal handler. However we
2506 know this only ever happens in a synchronous SEGV handler, so in
2507 practice it seems to be ok. */
2510 p
= page_find(address
>> TARGET_PAGE_BITS
);
2516 /* if the page was really writable, then we change its
2517 protection back to writable */
2518 if ((p
->flags
& PAGE_WRITE_ORG
) && !(p
->flags
& PAGE_WRITE
)) {
2519 host_start
= address
& qemu_host_page_mask
;
2520 host_end
= host_start
+ qemu_host_page_size
;
2523 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2524 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2525 p
->flags
|= PAGE_WRITE
;
2528 /* and since the content will be modified, we must invalidate
2529 the corresponding translated code. */
2530 tb_invalidate_phys_page(addr
, pc
, puc
);
2531 #ifdef DEBUG_TB_CHECK
2532 tb_invalidate_check(addr
);
2535 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2545 static inline void tlb_set_dirty(CPUState
*env
,
2546 unsigned long addr
, target_ulong vaddr
)
2549 #endif /* defined(CONFIG_USER_ONLY) */
2551 #if !defined(CONFIG_USER_ONLY)
2553 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
2554 typedef struct subpage_t
{
2555 target_phys_addr_t base
;
2556 ram_addr_t sub_io_index
[TARGET_PAGE_SIZE
];
2557 ram_addr_t region_offset
[TARGET_PAGE_SIZE
];
2560 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2561 ram_addr_t memory
, ram_addr_t region_offset
);
2562 static subpage_t
*subpage_init (target_phys_addr_t base
, ram_addr_t
*phys
,
2563 ram_addr_t orig_memory
,
2564 ram_addr_t region_offset
);
2565 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2568 if (addr > start_addr) \
2571 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2572 if (start_addr2 > 0) \
2576 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2577 end_addr2 = TARGET_PAGE_SIZE - 1; \
2579 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2580 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2585 /* register physical memory.
2586 For RAM, 'size' must be a multiple of the target page size.
2587 If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2588 io memory page. The address used when calling the IO function is
2589 the offset from the start of the region, plus region_offset. Both
2590 start_addr and region_offset are rounded down to a page boundary
2591 before calculating this offset. This should not be a problem unless
2592 the low bits of start_addr and region_offset differ. */
2593 void cpu_register_physical_memory_offset(target_phys_addr_t start_addr
,
2595 ram_addr_t phys_offset
,
2596 ram_addr_t region_offset
)
2598 target_phys_addr_t addr
, end_addr
;
2601 ram_addr_t orig_size
= size
;
2604 cpu_notify_set_memory(start_addr
, size
, phys_offset
);
2606 if (phys_offset
== IO_MEM_UNASSIGNED
) {
2607 region_offset
= start_addr
;
2609 region_offset
&= TARGET_PAGE_MASK
;
2610 size
= (size
+ TARGET_PAGE_SIZE
- 1) & TARGET_PAGE_MASK
;
2611 end_addr
= start_addr
+ (target_phys_addr_t
)size
;
2612 for(addr
= start_addr
; addr
!= end_addr
; addr
+= TARGET_PAGE_SIZE
) {
2613 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2614 if (p
&& p
->phys_offset
!= IO_MEM_UNASSIGNED
) {
2615 ram_addr_t orig_memory
= p
->phys_offset
;
2616 target_phys_addr_t start_addr2
, end_addr2
;
2617 int need_subpage
= 0;
2619 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
, end_addr2
,
2622 if (!(orig_memory
& IO_MEM_SUBPAGE
)) {
2623 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2624 &p
->phys_offset
, orig_memory
,
2627 subpage
= io_mem_opaque
[(orig_memory
& ~TARGET_PAGE_MASK
)
2630 subpage_register(subpage
, start_addr2
, end_addr2
, phys_offset
,
2632 p
->region_offset
= 0;
2634 p
->phys_offset
= phys_offset
;
2635 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
||
2636 (phys_offset
& IO_MEM_ROMD
))
2637 phys_offset
+= TARGET_PAGE_SIZE
;
2640 p
= phys_page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2641 p
->phys_offset
= phys_offset
;
2642 p
->region_offset
= region_offset
;
2643 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
||
2644 (phys_offset
& IO_MEM_ROMD
)) {
2645 phys_offset
+= TARGET_PAGE_SIZE
;
2647 target_phys_addr_t start_addr2
, end_addr2
;
2648 int need_subpage
= 0;
2650 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
,
2651 end_addr2
, need_subpage
);
2654 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2655 &p
->phys_offset
, IO_MEM_UNASSIGNED
,
2656 addr
& TARGET_PAGE_MASK
);
2657 subpage_register(subpage
, start_addr2
, end_addr2
,
2658 phys_offset
, region_offset
);
2659 p
->region_offset
= 0;
2663 region_offset
+= TARGET_PAGE_SIZE
;
2666 /* since each CPU stores ram addresses in its TLB cache, we must
2667 reset the modified entries */
2669 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
2674 /* XXX: temporary until new memory mapping API */
2675 ram_addr_t
cpu_get_physical_page_desc(target_phys_addr_t addr
)
2679 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2681 return IO_MEM_UNASSIGNED
;
2682 return p
->phys_offset
;
2685 void qemu_register_coalesced_mmio(target_phys_addr_t addr
, ram_addr_t size
)
2688 kvm_coalesce_mmio_region(addr
, size
);
2691 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr
, ram_addr_t size
)
2694 kvm_uncoalesce_mmio_region(addr
, size
);
2697 void qemu_flush_coalesced_mmio_buffer(void)
2700 kvm_flush_coalesced_mmio_buffer();
2703 #if defined(__linux__) && !defined(TARGET_S390X)
2705 #include <sys/vfs.h>
2707 #define HUGETLBFS_MAGIC 0x958458f6
2709 static long gethugepagesize(const char *path
)
2715 ret
= statfs(path
, &fs
);
2716 } while (ret
!= 0 && errno
== EINTR
);
2723 if (fs
.f_type
!= HUGETLBFS_MAGIC
)
2724 fprintf(stderr
, "Warning: path not on HugeTLBFS: %s\n", path
);
2729 static void *file_ram_alloc(ram_addr_t memory
, const char *path
)
2737 unsigned long hpagesize
;
2739 hpagesize
= gethugepagesize(path
);
2744 if (memory
< hpagesize
) {
2748 if (kvm_enabled() && !kvm_has_sync_mmu()) {
2749 fprintf(stderr
, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
2753 if (asprintf(&filename
, "%s/qemu_back_mem.XXXXXX", path
) == -1) {
2757 fd
= mkstemp(filename
);
2759 perror("unable to create backing store for hugepages");
2766 memory
= (memory
+hpagesize
-1) & ~(hpagesize
-1);
2769 * ftruncate is not supported by hugetlbfs in older
2770 * hosts, so don't bother bailing out on errors.
2771 * If anything goes wrong with it under other filesystems,
2774 if (ftruncate(fd
, memory
))
2775 perror("ftruncate");
2778 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
2779 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
2780 * to sidestep this quirk.
2782 flags
= mem_prealloc
? MAP_POPULATE
| MAP_SHARED
: MAP_PRIVATE
;
2783 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
, flags
, fd
, 0);
2785 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
, MAP_PRIVATE
, fd
, 0);
2787 if (area
== MAP_FAILED
) {
2788 perror("file_ram_alloc: can't mmap RAM pages");
2796 ram_addr_t
qemu_ram_map(ram_addr_t size
, void *host
)
2798 RAMBlock
*new_block
;
2800 size
= TARGET_PAGE_ALIGN(size
);
2801 new_block
= qemu_malloc(sizeof(*new_block
));
2803 new_block
->host
= host
;
2805 new_block
->offset
= last_ram_offset
;
2806 new_block
->length
= size
;
2808 new_block
->next
= ram_blocks
;
2809 ram_blocks
= new_block
;
2811 phys_ram_dirty
= qemu_realloc(phys_ram_dirty
,
2812 (last_ram_offset
+ size
) >> TARGET_PAGE_BITS
);
2813 memset(phys_ram_dirty
+ (last_ram_offset
>> TARGET_PAGE_BITS
),
2814 0xff, size
>> TARGET_PAGE_BITS
);
2816 last_ram_offset
+= size
;
2819 kvm_setup_guest_memory(new_block
->host
, size
);
2821 return new_block
->offset
;
2824 ram_addr_t
qemu_ram_alloc(ram_addr_t size
)
2826 RAMBlock
*new_block
;
2828 size
= TARGET_PAGE_ALIGN(size
);
2829 new_block
= qemu_malloc(sizeof(*new_block
));
2832 #if defined (__linux__) && !defined(TARGET_S390X)
2833 new_block
->host
= file_ram_alloc(size
, mem_path
);
2834 if (!new_block
->host
) {
2835 new_block
->host
= qemu_vmalloc(size
);
2836 #ifdef MADV_MERGEABLE
2837 madvise(new_block
->host
, size
, MADV_MERGEABLE
);
2841 fprintf(stderr
, "-mem-path option unsupported\n");
2845 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2846 /* XXX S390 KVM requires the topmost vma of the RAM to be < 256GB */
2847 new_block
->host
= mmap((void*)0x1000000, size
,
2848 PROT_EXEC
|PROT_READ
|PROT_WRITE
,
2849 MAP_SHARED
| MAP_ANONYMOUS
, -1, 0);
2851 new_block
->host
= qemu_vmalloc(size
);
2853 #ifdef MADV_MERGEABLE
2854 madvise(new_block
->host
, size
, MADV_MERGEABLE
);
2857 new_block
->offset
= last_ram_offset
;
2858 new_block
->length
= size
;
2860 new_block
->next
= ram_blocks
;
2861 ram_blocks
= new_block
;
2863 phys_ram_dirty
= qemu_realloc(phys_ram_dirty
,
2864 (last_ram_offset
+ size
) >> TARGET_PAGE_BITS
);
2865 memset(phys_ram_dirty
+ (last_ram_offset
>> TARGET_PAGE_BITS
),
2866 0xff, size
>> TARGET_PAGE_BITS
);
2868 last_ram_offset
+= size
;
2871 kvm_setup_guest_memory(new_block
->host
, size
);
2873 return new_block
->offset
;
2876 void qemu_ram_free(ram_addr_t addr
)
2878 /* TODO: implement this. */
2881 /* Return a host pointer to ram allocated with qemu_ram_alloc.
2882 With the exception of the softmmu code in this file, this should
2883 only be used for local memory (e.g. video ram) that the device owns,
2884 and knows it isn't going to access beyond the end of the block.
2886 It should not be used for general purpose DMA.
2887 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
2889 void *qemu_get_ram_ptr(ram_addr_t addr
)
2896 prevp
= &ram_blocks
;
2898 while (block
&& (block
->offset
> addr
2899 || block
->offset
+ block
->length
<= addr
)) {
2901 prevp
= &prev
->next
;
2903 block
= block
->next
;
2906 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
2909 /* Move this entry to to start of the list. */
2911 prev
->next
= block
->next
;
2912 block
->next
= *prevp
;
2915 return block
->host
+ (addr
- block
->offset
);
2918 int do_qemu_ram_addr_from_host(void *ptr
, ram_addr_t
*ram_addr
)
2921 uint8_t *host
= ptr
;
2924 while (block
&& (block
->host
> host
2925 || block
->host
+ block
->length
<= host
)) {
2926 block
= block
->next
;
2930 *ram_addr
= block
->offset
+ (host
- block
->host
);
2934 /* Some of the softmmu routines need to translate from a host pointer
2935 (typically a TLB entry) back to a ram offset. */
2936 ram_addr_t
qemu_ram_addr_from_host(void *ptr
)
2938 ram_addr_t ram_addr
;
2940 if (do_qemu_ram_addr_from_host(ptr
, &ram_addr
)) {
2941 fprintf(stderr
, "Bad ram pointer %p\n", ptr
);
2947 static uint32_t unassigned_mem_readb(void *opaque
, target_phys_addr_t addr
)
2949 #ifdef DEBUG_UNASSIGNED
2950 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2952 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2953 do_unassigned_access(addr
, 0, 0, 0, 1);
2958 static uint32_t unassigned_mem_readw(void *opaque
, target_phys_addr_t addr
)
2960 #ifdef DEBUG_UNASSIGNED
2961 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2963 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2964 do_unassigned_access(addr
, 0, 0, 0, 2);
2969 static uint32_t unassigned_mem_readl(void *opaque
, target_phys_addr_t addr
)
2971 #ifdef DEBUG_UNASSIGNED
2972 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2974 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2975 do_unassigned_access(addr
, 0, 0, 0, 4);
2980 static void unassigned_mem_writeb(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2982 #ifdef DEBUG_UNASSIGNED
2983 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2985 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2986 do_unassigned_access(addr
, 1, 0, 0, 1);
2990 static void unassigned_mem_writew(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2992 #ifdef DEBUG_UNASSIGNED
2993 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2995 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2996 do_unassigned_access(addr
, 1, 0, 0, 2);
3000 static void unassigned_mem_writel(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
3002 #ifdef DEBUG_UNASSIGNED
3003 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
3005 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3006 do_unassigned_access(addr
, 1, 0, 0, 4);
3010 static CPUReadMemoryFunc
* const unassigned_mem_read
[3] = {
3011 unassigned_mem_readb
,
3012 unassigned_mem_readw
,
3013 unassigned_mem_readl
,
3016 static CPUWriteMemoryFunc
* const unassigned_mem_write
[3] = {
3017 unassigned_mem_writeb
,
3018 unassigned_mem_writew
,
3019 unassigned_mem_writel
,
3022 static void notdirty_mem_writeb(void *opaque
, target_phys_addr_t ram_addr
,
3026 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
3027 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
3028 #if !defined(CONFIG_USER_ONLY)
3029 tb_invalidate_phys_page_fast(ram_addr
, 1);
3030 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
3033 stb_p(qemu_get_ram_ptr(ram_addr
), val
);
3034 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
3035 cpu_physical_memory_set_dirty_flags(ram_addr
, dirty_flags
);
3036 /* we remove the notdirty callback only if the code has been
3038 if (dirty_flags
== 0xff)
3039 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
3042 static void notdirty_mem_writew(void *opaque
, target_phys_addr_t ram_addr
,
3046 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
3047 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
3048 #if !defined(CONFIG_USER_ONLY)
3049 tb_invalidate_phys_page_fast(ram_addr
, 2);
3050 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
3053 stw_p(qemu_get_ram_ptr(ram_addr
), val
);
3054 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
3055 cpu_physical_memory_set_dirty_flags(ram_addr
, dirty_flags
);
3056 /* we remove the notdirty callback only if the code has been
3058 if (dirty_flags
== 0xff)
3059 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
3062 static void notdirty_mem_writel(void *opaque
, target_phys_addr_t ram_addr
,
3066 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
3067 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
3068 #if !defined(CONFIG_USER_ONLY)
3069 tb_invalidate_phys_page_fast(ram_addr
, 4);
3070 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
3073 stl_p(qemu_get_ram_ptr(ram_addr
), val
);
3074 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
3075 cpu_physical_memory_set_dirty_flags(ram_addr
, dirty_flags
);
3076 /* we remove the notdirty callback only if the code has been
3078 if (dirty_flags
== 0xff)
3079 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
3082 static CPUReadMemoryFunc
* const error_mem_read
[3] = {
3083 NULL
, /* never used */
3084 NULL
, /* never used */
3085 NULL
, /* never used */
3088 static CPUWriteMemoryFunc
* const notdirty_mem_write
[3] = {
3089 notdirty_mem_writeb
,
3090 notdirty_mem_writew
,
3091 notdirty_mem_writel
,
3094 /* Generate a debug exception if a watchpoint has been hit. */
3095 static void check_watchpoint(int offset
, int len_mask
, int flags
)
3097 CPUState
*env
= cpu_single_env
;
3098 target_ulong pc
, cs_base
;
3099 TranslationBlock
*tb
;
3104 if (env
->watchpoint_hit
) {
3105 /* We re-entered the check after replacing the TB. Now raise
3106 * the debug interrupt so that is will trigger after the
3107 * current instruction. */
3108 cpu_interrupt(env
, CPU_INTERRUPT_DEBUG
);
3111 vaddr
= (env
->mem_io_vaddr
& TARGET_PAGE_MASK
) + offset
;
3112 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
3113 if ((vaddr
== (wp
->vaddr
& len_mask
) ||
3114 (vaddr
& wp
->len_mask
) == wp
->vaddr
) && (wp
->flags
& flags
)) {
3115 wp
->flags
|= BP_WATCHPOINT_HIT
;
3116 if (!env
->watchpoint_hit
) {
3117 env
->watchpoint_hit
= wp
;
3118 tb
= tb_find_pc(env
->mem_io_pc
);
3120 cpu_abort(env
, "check_watchpoint: could not find TB for "
3121 "pc=%p", (void *)env
->mem_io_pc
);
3123 cpu_restore_state(tb
, env
, env
->mem_io_pc
, NULL
);
3124 tb_phys_invalidate(tb
, -1);
3125 if (wp
->flags
& BP_STOP_BEFORE_ACCESS
) {
3126 env
->exception_index
= EXCP_DEBUG
;
3128 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &cpu_flags
);
3129 tb_gen_code(env
, pc
, cs_base
, cpu_flags
, 1);
3131 cpu_resume_from_signal(env
, NULL
);
3134 wp
->flags
&= ~BP_WATCHPOINT_HIT
;
3139 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
3140 so these check for a hit then pass through to the normal out-of-line
3142 static uint32_t watch_mem_readb(void *opaque
, target_phys_addr_t addr
)
3144 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x0, BP_MEM_READ
);
3145 return ldub_phys(addr
);
3148 static uint32_t watch_mem_readw(void *opaque
, target_phys_addr_t addr
)
3150 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x1, BP_MEM_READ
);
3151 return lduw_phys(addr
);
3154 static uint32_t watch_mem_readl(void *opaque
, target_phys_addr_t addr
)
3156 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x3, BP_MEM_READ
);
3157 return ldl_phys(addr
);
3160 static void watch_mem_writeb(void *opaque
, target_phys_addr_t addr
,
3163 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x0, BP_MEM_WRITE
);
3164 stb_phys(addr
, val
);
3167 static void watch_mem_writew(void *opaque
, target_phys_addr_t addr
,
3170 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x1, BP_MEM_WRITE
);
3171 stw_phys(addr
, val
);
3174 static void watch_mem_writel(void *opaque
, target_phys_addr_t addr
,
3177 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x3, BP_MEM_WRITE
);
3178 stl_phys(addr
, val
);
3181 static CPUReadMemoryFunc
* const watch_mem_read
[3] = {
3187 static CPUWriteMemoryFunc
* const watch_mem_write
[3] = {
3193 static inline uint32_t subpage_readlen (subpage_t
*mmio
,
3194 target_phys_addr_t addr
,
3197 unsigned int idx
= SUBPAGE_IDX(addr
);
3198 #if defined(DEBUG_SUBPAGE)
3199 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d\n", __func__
,
3200 mmio
, len
, addr
, idx
);
3203 addr
+= mmio
->region_offset
[idx
];
3204 idx
= mmio
->sub_io_index
[idx
];
3205 return io_mem_read
[idx
][len
](io_mem_opaque
[idx
], addr
);
3208 static inline void subpage_writelen (subpage_t
*mmio
, target_phys_addr_t addr
,
3209 uint32_t value
, unsigned int len
)
3211 unsigned int idx
= SUBPAGE_IDX(addr
);
3212 #if defined(DEBUG_SUBPAGE)
3213 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d value %08x\n",
3214 __func__
, mmio
, len
, addr
, idx
, value
);
3217 addr
+= mmio
->region_offset
[idx
];
3218 idx
= mmio
->sub_io_index
[idx
];
3219 io_mem_write
[idx
][len
](io_mem_opaque
[idx
], addr
, value
);
3222 static uint32_t subpage_readb (void *opaque
, target_phys_addr_t addr
)
3224 return subpage_readlen(opaque
, addr
, 0);
3227 static void subpage_writeb (void *opaque
, target_phys_addr_t addr
,
3230 subpage_writelen(opaque
, addr
, value
, 0);
3233 static uint32_t subpage_readw (void *opaque
, target_phys_addr_t addr
)
3235 return subpage_readlen(opaque
, addr
, 1);
3238 static void subpage_writew (void *opaque
, target_phys_addr_t addr
,
3241 subpage_writelen(opaque
, addr
, value
, 1);
3244 static uint32_t subpage_readl (void *opaque
, target_phys_addr_t addr
)
3246 return subpage_readlen(opaque
, addr
, 2);
3249 static void subpage_writel (void *opaque
, target_phys_addr_t addr
,
3252 subpage_writelen(opaque
, addr
, value
, 2);
3255 static CPUReadMemoryFunc
* const subpage_read
[] = {
3261 static CPUWriteMemoryFunc
* const subpage_write
[] = {
3267 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
3268 ram_addr_t memory
, ram_addr_t region_offset
)
3272 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
3274 idx
= SUBPAGE_IDX(start
);
3275 eidx
= SUBPAGE_IDX(end
);
3276 #if defined(DEBUG_SUBPAGE)
3277 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__
,
3278 mmio
, start
, end
, idx
, eidx
, memory
);
3280 memory
= (memory
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3281 for (; idx
<= eidx
; idx
++) {
3282 mmio
->sub_io_index
[idx
] = memory
;
3283 mmio
->region_offset
[idx
] = region_offset
;
3289 static subpage_t
*subpage_init (target_phys_addr_t base
, ram_addr_t
*phys
,
3290 ram_addr_t orig_memory
,
3291 ram_addr_t region_offset
)
3296 mmio
= qemu_mallocz(sizeof(subpage_t
));
3299 subpage_memory
= cpu_register_io_memory(subpage_read
, subpage_write
, mmio
);
3300 #if defined(DEBUG_SUBPAGE)
3301 printf("%s: %p base " TARGET_FMT_plx
" len %08x %d\n", __func__
,
3302 mmio
, base
, TARGET_PAGE_SIZE
, subpage_memory
);
3304 *phys
= subpage_memory
| IO_MEM_SUBPAGE
;
3305 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
-1, orig_memory
, region_offset
);
3310 static int get_free_io_mem_idx(void)
3314 for (i
= 0; i
<IO_MEM_NB_ENTRIES
; i
++)
3315 if (!io_mem_used
[i
]) {
3319 fprintf(stderr
, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES
);
3323 /* mem_read and mem_write are arrays of functions containing the
3324 function to access byte (index 0), word (index 1) and dword (index
3325 2). Functions can be omitted with a NULL function pointer.
3326 If io_index is non zero, the corresponding io zone is
3327 modified. If it is zero, a new io zone is allocated. The return
3328 value can be used with cpu_register_physical_memory(). (-1) is
3329 returned if error. */
3330 static int cpu_register_io_memory_fixed(int io_index
,
3331 CPUReadMemoryFunc
* const *mem_read
,
3332 CPUWriteMemoryFunc
* const *mem_write
,
3337 if (io_index
<= 0) {
3338 io_index
= get_free_io_mem_idx();
3342 io_index
>>= IO_MEM_SHIFT
;
3343 if (io_index
>= IO_MEM_NB_ENTRIES
)
3347 for (i
= 0; i
< 3; ++i
) {
3348 io_mem_read
[io_index
][i
]
3349 = (mem_read
[i
] ? mem_read
[i
] : unassigned_mem_read
[i
]);
3351 for (i
= 0; i
< 3; ++i
) {
3352 io_mem_write
[io_index
][i
]
3353 = (mem_write
[i
] ? mem_write
[i
] : unassigned_mem_write
[i
]);
3355 io_mem_opaque
[io_index
] = opaque
;
3357 return (io_index
<< IO_MEM_SHIFT
);
3360 int cpu_register_io_memory(CPUReadMemoryFunc
* const *mem_read
,
3361 CPUWriteMemoryFunc
* const *mem_write
,
3364 return cpu_register_io_memory_fixed(0, mem_read
, mem_write
, opaque
);
3367 void cpu_unregister_io_memory(int io_table_address
)
3370 int io_index
= io_table_address
>> IO_MEM_SHIFT
;
3372 for (i
=0;i
< 3; i
++) {
3373 io_mem_read
[io_index
][i
] = unassigned_mem_read
[i
];
3374 io_mem_write
[io_index
][i
] = unassigned_mem_write
[i
];
3376 io_mem_opaque
[io_index
] = NULL
;
3377 io_mem_used
[io_index
] = 0;
3380 static void io_mem_init(void)
3384 cpu_register_io_memory_fixed(IO_MEM_ROM
, error_mem_read
, unassigned_mem_write
, NULL
);
3385 cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED
, unassigned_mem_read
, unassigned_mem_write
, NULL
);
3386 cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY
, error_mem_read
, notdirty_mem_write
, NULL
);
3390 io_mem_watch
= cpu_register_io_memory(watch_mem_read
,
3391 watch_mem_write
, NULL
);
3394 #endif /* !defined(CONFIG_USER_ONLY) */
3396 /* physical memory access (slow version, mainly for debug) */
3397 #if defined(CONFIG_USER_ONLY)
3398 int cpu_memory_rw_debug(CPUState
*env
, target_ulong addr
,
3399 uint8_t *buf
, int len
, int is_write
)
3406 page
= addr
& TARGET_PAGE_MASK
;
3407 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3410 flags
= page_get_flags(page
);
3411 if (!(flags
& PAGE_VALID
))
3414 if (!(flags
& PAGE_WRITE
))
3416 /* XXX: this code should not depend on lock_user */
3417 if (!(p
= lock_user(VERIFY_WRITE
, addr
, l
, 0)))
3420 unlock_user(p
, addr
, l
);
3422 if (!(flags
& PAGE_READ
))
3424 /* XXX: this code should not depend on lock_user */
3425 if (!(p
= lock_user(VERIFY_READ
, addr
, l
, 1)))
3428 unlock_user(p
, addr
, 0);
3438 void cpu_physical_memory_rw(target_phys_addr_t addr
, uint8_t *buf
,
3439 int len
, int is_write
)
3444 target_phys_addr_t page
;
3449 page
= addr
& TARGET_PAGE_MASK
;
3450 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3453 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3455 pd
= IO_MEM_UNASSIGNED
;
3457 pd
= p
->phys_offset
;
3461 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3462 target_phys_addr_t addr1
= addr
;
3463 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3465 addr1
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3466 /* XXX: could force cpu_single_env to NULL to avoid
3468 if (l
>= 4 && ((addr1
& 3) == 0)) {
3469 /* 32 bit write access */
3471 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr1
, val
);
3473 } else if (l
>= 2 && ((addr1
& 1) == 0)) {
3474 /* 16 bit write access */
3476 io_mem_write
[io_index
][1](io_mem_opaque
[io_index
], addr1
, val
);
3479 /* 8 bit write access */
3481 io_mem_write
[io_index
][0](io_mem_opaque
[io_index
], addr1
, val
);
3485 unsigned long addr1
;
3486 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3488 ptr
= qemu_get_ram_ptr(addr1
);
3489 memcpy(ptr
, buf
, l
);
3490 if (!cpu_physical_memory_is_dirty(addr1
)) {
3491 /* invalidate code */
3492 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
3494 cpu_physical_memory_set_dirty_flags(
3495 addr1
, (0xff & ~CODE_DIRTY_FLAG
));
3497 /* qemu doesn't execute guest code directly, but kvm does
3498 therefore flush instruction caches */
3500 flush_icache_range((unsigned long)ptr
,
3501 ((unsigned long)ptr
)+l
);
3504 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3505 !(pd
& IO_MEM_ROMD
)) {
3506 target_phys_addr_t addr1
= addr
;
3508 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3510 addr1
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3511 if (l
>= 4 && ((addr1
& 3) == 0)) {
3512 /* 32 bit read access */
3513 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr1
);
3516 } else if (l
>= 2 && ((addr1
& 1) == 0)) {
3517 /* 16 bit read access */
3518 val
= io_mem_read
[io_index
][1](io_mem_opaque
[io_index
], addr1
);
3522 /* 8 bit read access */
3523 val
= io_mem_read
[io_index
][0](io_mem_opaque
[io_index
], addr1
);
3529 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3530 (addr
& ~TARGET_PAGE_MASK
);
3531 memcpy(buf
, ptr
, l
);
3540 /* used for ROM loading : can write in RAM and ROM */
3541 void cpu_physical_memory_write_rom(target_phys_addr_t addr
,
3542 const uint8_t *buf
, int len
)
3546 target_phys_addr_t page
;
3551 page
= addr
& TARGET_PAGE_MASK
;
3552 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3555 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3557 pd
= IO_MEM_UNASSIGNED
;
3559 pd
= p
->phys_offset
;
3562 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
&&
3563 (pd
& ~TARGET_PAGE_MASK
) != IO_MEM_ROM
&&
3564 !(pd
& IO_MEM_ROMD
)) {
3567 unsigned long addr1
;
3568 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3570 ptr
= qemu_get_ram_ptr(addr1
);
3571 memcpy(ptr
, buf
, l
);
3581 target_phys_addr_t addr
;
3582 target_phys_addr_t len
;
3585 static BounceBuffer bounce
;
3587 typedef struct MapClient
{
3589 void (*callback
)(void *opaque
);
3590 QLIST_ENTRY(MapClient
) link
;
3593 static QLIST_HEAD(map_client_list
, MapClient
) map_client_list
3594 = QLIST_HEAD_INITIALIZER(map_client_list
);
3596 void *cpu_register_map_client(void *opaque
, void (*callback
)(void *opaque
))
3598 MapClient
*client
= qemu_malloc(sizeof(*client
));
3600 client
->opaque
= opaque
;
3601 client
->callback
= callback
;
3602 QLIST_INSERT_HEAD(&map_client_list
, client
, link
);
3606 void cpu_unregister_map_client(void *_client
)
3608 MapClient
*client
= (MapClient
*)_client
;
3610 QLIST_REMOVE(client
, link
);
3614 static void cpu_notify_map_clients(void)
3618 while (!QLIST_EMPTY(&map_client_list
)) {
3619 client
= QLIST_FIRST(&map_client_list
);
3620 client
->callback(client
->opaque
);
3621 cpu_unregister_map_client(client
);
3625 /* Map a physical memory region into a host virtual address.
3626 * May map a subset of the requested range, given by and returned in *plen.
3627 * May return NULL if resources needed to perform the mapping are exhausted.
3628 * Use only for reads OR writes - not for read-modify-write operations.
3629 * Use cpu_register_map_client() to know when retrying the map operation is
3630 * likely to succeed.
3632 void *cpu_physical_memory_map(target_phys_addr_t addr
,
3633 target_phys_addr_t
*plen
,
3636 target_phys_addr_t len
= *plen
;
3637 target_phys_addr_t done
= 0;
3639 uint8_t *ret
= NULL
;
3641 target_phys_addr_t page
;
3644 unsigned long addr1
;
3647 page
= addr
& TARGET_PAGE_MASK
;
3648 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3651 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3653 pd
= IO_MEM_UNASSIGNED
;
3655 pd
= p
->phys_offset
;
3658 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3659 if (done
|| bounce
.buffer
) {
3662 bounce
.buffer
= qemu_memalign(TARGET_PAGE_SIZE
, TARGET_PAGE_SIZE
);
3666 cpu_physical_memory_rw(addr
, bounce
.buffer
, l
, 0);
3668 ptr
= bounce
.buffer
;
3670 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3671 ptr
= qemu_get_ram_ptr(addr1
);
3675 } else if (ret
+ done
!= ptr
) {
3687 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3688 * Will also mark the memory as dirty if is_write == 1. access_len gives
3689 * the amount of memory that was actually read or written by the caller.
3691 void cpu_physical_memory_unmap(void *buffer
, target_phys_addr_t len
,
3692 int is_write
, target_phys_addr_t access_len
)
3694 unsigned long flush_len
= (unsigned long)access_len
;
3696 if (buffer
!= bounce
.buffer
) {
3698 ram_addr_t addr1
= qemu_ram_addr_from_host(buffer
);
3699 while (access_len
) {
3701 l
= TARGET_PAGE_SIZE
;
3704 if (!cpu_physical_memory_is_dirty(addr1
)) {
3705 /* invalidate code */
3706 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
3708 cpu_physical_memory_set_dirty_flags(
3709 addr1
, (0xff & ~CODE_DIRTY_FLAG
));
3714 dma_flush_range((unsigned long)buffer
,
3715 (unsigned long)buffer
+ flush_len
);
3720 cpu_physical_memory_write(bounce
.addr
, bounce
.buffer
, access_len
);
3722 qemu_vfree(bounce
.buffer
);
3723 bounce
.buffer
= NULL
;
3724 cpu_notify_map_clients();
3727 /* warning: addr must be aligned */
3728 uint32_t ldl_phys(target_phys_addr_t addr
)
3736 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3738 pd
= IO_MEM_UNASSIGNED
;
3740 pd
= p
->phys_offset
;
3743 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3744 !(pd
& IO_MEM_ROMD
)) {
3746 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3748 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3749 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
3752 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3753 (addr
& ~TARGET_PAGE_MASK
);
3759 /* warning: addr must be aligned */
3760 uint64_t ldq_phys(target_phys_addr_t addr
)
3768 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3770 pd
= IO_MEM_UNASSIGNED
;
3772 pd
= p
->phys_offset
;
3775 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3776 !(pd
& IO_MEM_ROMD
)) {
3778 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3780 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3781 #ifdef TARGET_WORDS_BIGENDIAN
3782 val
= (uint64_t)io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
) << 32;
3783 val
|= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4);
3785 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
3786 val
|= (uint64_t)io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4) << 32;
3790 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3791 (addr
& ~TARGET_PAGE_MASK
);
3798 uint32_t ldub_phys(target_phys_addr_t addr
)
3801 cpu_physical_memory_read(addr
, &val
, 1);
3805 /* warning: addr must be aligned */
3806 uint32_t lduw_phys(target_phys_addr_t addr
)
3814 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3816 pd
= IO_MEM_UNASSIGNED
;
3818 pd
= p
->phys_offset
;
3821 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3822 !(pd
& IO_MEM_ROMD
)) {
3824 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3826 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3827 val
= io_mem_read
[io_index
][1](io_mem_opaque
[io_index
], addr
);
3830 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3831 (addr
& ~TARGET_PAGE_MASK
);
3837 /* warning: addr must be aligned. The ram page is not masked as dirty
3838 and the code inside is not invalidated. It is useful if the dirty
3839 bits are used to track modified PTEs */
3840 void stl_phys_notdirty(target_phys_addr_t addr
, uint32_t val
)
3847 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3849 pd
= IO_MEM_UNASSIGNED
;
3851 pd
= p
->phys_offset
;
3854 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3855 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3857 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3858 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3860 unsigned long addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3861 ptr
= qemu_get_ram_ptr(addr1
);
3864 if (unlikely(in_migration
)) {
3865 if (!cpu_physical_memory_is_dirty(addr1
)) {
3866 /* invalidate code */
3867 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
3869 cpu_physical_memory_set_dirty_flags(
3870 addr1
, (0xff & ~CODE_DIRTY_FLAG
));
3876 void stq_phys_notdirty(target_phys_addr_t addr
, uint64_t val
)
3883 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3885 pd
= IO_MEM_UNASSIGNED
;
3887 pd
= p
->phys_offset
;
3890 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3891 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3893 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3894 #ifdef TARGET_WORDS_BIGENDIAN
3895 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
>> 32);
3896 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4, val
);
3898 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3899 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4, val
>> 32);
3902 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3903 (addr
& ~TARGET_PAGE_MASK
);
3908 /* warning: addr must be aligned */
3909 void stl_phys(target_phys_addr_t addr
, uint32_t val
)
3916 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3918 pd
= IO_MEM_UNASSIGNED
;
3920 pd
= p
->phys_offset
;
3923 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3924 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3926 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3927 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3929 unsigned long addr1
;
3930 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3932 ptr
= qemu_get_ram_ptr(addr1
);
3934 if (!cpu_physical_memory_is_dirty(addr1
)) {
3935 /* invalidate code */
3936 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
3938 cpu_physical_memory_set_dirty_flags(addr1
,
3939 (0xff & ~CODE_DIRTY_FLAG
));
3945 void stb_phys(target_phys_addr_t addr
, uint32_t val
)
3948 cpu_physical_memory_write(addr
, &v
, 1);
3951 /* warning: addr must be aligned */
3952 void stw_phys(target_phys_addr_t addr
, uint32_t val
)
3959 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3961 pd
= IO_MEM_UNASSIGNED
;
3963 pd
= p
->phys_offset
;
3966 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3967 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3969 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3970 io_mem_write
[io_index
][1](io_mem_opaque
[io_index
], addr
, val
);
3972 unsigned long addr1
;
3973 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3975 ptr
= qemu_get_ram_ptr(addr1
);
3977 if (!cpu_physical_memory_is_dirty(addr1
)) {
3978 /* invalidate code */
3979 tb_invalidate_phys_page_range(addr1
, addr1
+ 2, 0);
3981 cpu_physical_memory_set_dirty_flags(addr1
,
3982 (0xff & ~CODE_DIRTY_FLAG
));
3988 void stq_phys(target_phys_addr_t addr
, uint64_t val
)
3991 cpu_physical_memory_write(addr
, (const uint8_t *)&val
, 8);
3994 /* virtual memory access for debug (includes writing to ROM) */
3995 int cpu_memory_rw_debug(CPUState
*env
, target_ulong addr
,
3996 uint8_t *buf
, int len
, int is_write
)
3999 target_phys_addr_t phys_addr
;
4003 page
= addr
& TARGET_PAGE_MASK
;
4004 phys_addr
= cpu_get_phys_page_debug(env
, page
);
4005 /* if no physical page mapped, return an error */
4006 if (phys_addr
== -1)
4008 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
4011 phys_addr
+= (addr
& ~TARGET_PAGE_MASK
);
4013 cpu_physical_memory_write_rom(phys_addr
, buf
, l
);
4015 cpu_physical_memory_rw(phys_addr
, buf
, l
, is_write
);
4024 /* in deterministic execution mode, instructions doing device I/Os
4025 must be at the end of the TB */
4026 void cpu_io_recompile(CPUState
*env
, void *retaddr
)
4028 TranslationBlock
*tb
;
4030 target_ulong pc
, cs_base
;
4033 tb
= tb_find_pc((unsigned long)retaddr
);
4035 cpu_abort(env
, "cpu_io_recompile: could not find TB for pc=%p",
4038 n
= env
->icount_decr
.u16
.low
+ tb
->icount
;
4039 cpu_restore_state(tb
, env
, (unsigned long)retaddr
, NULL
);
4040 /* Calculate how many instructions had been executed before the fault
4042 n
= n
- env
->icount_decr
.u16
.low
;
4043 /* Generate a new TB ending on the I/O insn. */
4045 /* On MIPS and SH, delay slot instructions can only be restarted if
4046 they were already the first instruction in the TB. If this is not
4047 the first instruction in a TB then re-execute the preceding
4049 #if defined(TARGET_MIPS)
4050 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
4051 env
->active_tc
.PC
-= 4;
4052 env
->icount_decr
.u16
.low
++;
4053 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
4055 #elif defined(TARGET_SH4)
4056 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
4059 env
->icount_decr
.u16
.low
++;
4060 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
4063 /* This should never happen. */
4064 if (n
> CF_COUNT_MASK
)
4065 cpu_abort(env
, "TB too big during recompile");
4067 cflags
= n
| CF_LAST_IO
;
4069 cs_base
= tb
->cs_base
;
4071 tb_phys_invalidate(tb
, -1);
4072 /* FIXME: In theory this could raise an exception. In practice
4073 we have already translated the block once so it's probably ok. */
4074 tb_gen_code(env
, pc
, cs_base
, flags
, cflags
);
4075 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
4076 the first in the TB) then we end up generating a whole new TB and
4077 repeating the fault, which is horribly inefficient.
4078 Better would be to execute just this insn uncached, or generate a
4080 cpu_resume_from_signal(env
, NULL
);
4083 #if !defined(CONFIG_USER_ONLY)
4085 void dump_exec_info(FILE *f
,
4086 int (*cpu_fprintf
)(FILE *f
, const char *fmt
, ...))
4088 int i
, target_code_size
, max_target_code_size
;
4089 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
4090 TranslationBlock
*tb
;
4092 target_code_size
= 0;
4093 max_target_code_size
= 0;
4095 direct_jmp_count
= 0;
4096 direct_jmp2_count
= 0;
4097 for(i
= 0; i
< nb_tbs
; i
++) {
4099 target_code_size
+= tb
->size
;
4100 if (tb
->size
> max_target_code_size
)
4101 max_target_code_size
= tb
->size
;
4102 if (tb
->page_addr
[1] != -1)
4104 if (tb
->tb_next_offset
[0] != 0xffff) {
4106 if (tb
->tb_next_offset
[1] != 0xffff) {
4107 direct_jmp2_count
++;
4111 /* XXX: avoid using doubles ? */
4112 cpu_fprintf(f
, "Translation buffer state:\n");
4113 cpu_fprintf(f
, "gen code size %ld/%ld\n",
4114 code_gen_ptr
- code_gen_buffer
, code_gen_buffer_max_size
);
4115 cpu_fprintf(f
, "TB count %d/%d\n",
4116 nb_tbs
, code_gen_max_blocks
);
4117 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
4118 nb_tbs
? target_code_size
/ nb_tbs
: 0,
4119 max_target_code_size
);
4120 cpu_fprintf(f
, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
4121 nb_tbs
? (code_gen_ptr
- code_gen_buffer
) / nb_tbs
: 0,
4122 target_code_size
? (double) (code_gen_ptr
- code_gen_buffer
) / target_code_size
: 0);
4123 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n",
4125 nb_tbs
? (cross_page
* 100) / nb_tbs
: 0);
4126 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
4128 nb_tbs
? (direct_jmp_count
* 100) / nb_tbs
: 0,
4130 nb_tbs
? (direct_jmp2_count
* 100) / nb_tbs
: 0);
4131 cpu_fprintf(f
, "\nStatistics:\n");
4132 cpu_fprintf(f
, "TB flush count %d\n", tb_flush_count
);
4133 cpu_fprintf(f
, "TB invalidate count %d\n", tb_phys_invalidate_count
);
4134 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
4135 #ifdef CONFIG_PROFILER
4136 tcg_dump_info(f
, cpu_fprintf
);
4140 #define MMUSUFFIX _cmmu
4141 #define GETPC() NULL
4142 #define env cpu_single_env
4143 #define SOFTMMU_CODE_ACCESS
4146 #include "softmmu_template.h"
4149 #include "softmmu_template.h"
4152 #include "softmmu_template.h"
4155 #include "softmmu_template.h"