2 * virtual page mapping and translated block handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
23 #include <sys/types.h>
27 #include "qemu-common.h"
30 #include "cache-utils.h"
32 #if !defined(TARGET_IA64)
41 #include "qemu-timer.h"
42 #if defined(CONFIG_USER_ONLY)
45 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
46 #include <sys/param.h>
47 #if __FreeBSD_version >= 700104
48 #define HAVE_KINFO_GETVMMAP
49 #define sigqueue sigqueue_freebsd /* avoid redefinition */
52 #include <machine/profile.h>
62 //#define DEBUG_TB_INVALIDATE
65 //#define DEBUG_UNASSIGNED
67 /* make various TB consistency checks */
68 //#define DEBUG_TB_CHECK
69 //#define DEBUG_TLB_CHECK
71 //#define DEBUG_IOPORT
72 //#define DEBUG_SUBPAGE
74 #if !defined(CONFIG_USER_ONLY)
75 /* TB consistency checks only implemented for usermode emulation. */
79 #define SMC_BITMAP_USE_THRESHOLD 10
81 static TranslationBlock
*tbs
;
82 static int code_gen_max_blocks
;
83 TranslationBlock
*tb_phys_hash
[CODE_GEN_PHYS_HASH_SIZE
];
85 /* any access to the tbs or the page table must use this lock */
86 spinlock_t tb_lock
= SPIN_LOCK_UNLOCKED
;
88 #if defined(__arm__) || defined(__sparc_v9__)
89 /* The prologue must be reachable with a direct jump. ARM and Sparc64
90 have limited branch ranges (possibly also PPC) so place it in a
91 section close to code segment. */
92 #define code_gen_section \
93 __attribute__((__section__(".gen_code"))) \
94 __attribute__((aligned (32)))
96 /* Maximum alignment for Win32 is 16. */
97 #define code_gen_section \
98 __attribute__((aligned (16)))
100 #define code_gen_section \
101 __attribute__((aligned (32)))
104 uint8_t code_gen_prologue
[1024] code_gen_section
;
105 static uint8_t *code_gen_buffer
;
106 static unsigned long code_gen_buffer_size
;
107 /* threshold to flush the translated code buffer */
108 static unsigned long code_gen_buffer_max_size
;
109 static uint8_t *code_gen_ptr
;
111 #if !defined(CONFIG_USER_ONLY)
113 static int in_migration
;
115 RAMList ram_list
= { .blocks
= QLIST_HEAD_INITIALIZER(ram_list
) };
119 /* current CPU in the current thread. It is only valid inside
121 CPUState
*cpu_single_env
;
122 /* 0 = Do not count executed instructions.
123 1 = Precise instruction counting.
124 2 = Adaptive rate instruction counting. */
126 /* Current instruction counter. While executing translated code this may
127 include some instructions that have not yet been executed. */
130 typedef struct PageDesc
{
131 /* list of TBs intersecting this ram page */
132 TranslationBlock
*first_tb
;
133 /* in order to optimize self modifying code, we count the number
134 of lookups we do to a given page to use a bitmap */
135 unsigned int code_write_count
;
136 uint8_t *code_bitmap
;
137 #if defined(CONFIG_USER_ONLY)
142 /* In system mode we want L1_MAP to be based on ram offsets,
143 while in user mode we want it to be based on virtual addresses. */
144 #if !defined(CONFIG_USER_ONLY)
145 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
146 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
148 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
151 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
154 /* Size of the L2 (and L3, etc) page tables. */
156 #define L2_SIZE (1 << L2_BITS)
158 /* The bits remaining after N lower levels of page tables. */
159 #define P_L1_BITS_REM \
160 ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
161 #define V_L1_BITS_REM \
162 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
164 /* Size of the L1 page table. Avoid silly small sizes. */
165 #if P_L1_BITS_REM < 4
166 #define P_L1_BITS (P_L1_BITS_REM + L2_BITS)
168 #define P_L1_BITS P_L1_BITS_REM
171 #if V_L1_BITS_REM < 4
172 #define V_L1_BITS (V_L1_BITS_REM + L2_BITS)
174 #define V_L1_BITS V_L1_BITS_REM
177 #define P_L1_SIZE ((target_phys_addr_t)1 << P_L1_BITS)
178 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
180 #define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
181 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
183 unsigned long qemu_real_host_page_size
;
184 unsigned long qemu_host_page_bits
;
185 unsigned long qemu_host_page_size
;
186 unsigned long qemu_host_page_mask
;
188 /* This is a multi-level map on the virtual address space.
189 The bottom level has pointers to PageDesc. */
190 static void *l1_map
[V_L1_SIZE
];
192 #if !defined(CONFIG_USER_ONLY)
193 typedef struct PhysPageDesc
{
194 /* offset in host memory of the page + io_index in the low bits */
195 ram_addr_t phys_offset
;
196 ram_addr_t region_offset
;
199 /* This is a multi-level map on the physical address space.
200 The bottom level has pointers to PhysPageDesc. */
201 static void *l1_phys_map
[P_L1_SIZE
];
203 static void io_mem_init(void);
205 /* io memory support */
206 CPUWriteMemoryFunc
*io_mem_write
[IO_MEM_NB_ENTRIES
][4];
207 CPUReadMemoryFunc
*io_mem_read
[IO_MEM_NB_ENTRIES
][4];
208 void *io_mem_opaque
[IO_MEM_NB_ENTRIES
];
209 static char io_mem_used
[IO_MEM_NB_ENTRIES
];
210 static int io_mem_watch
;
215 static const char *logfilename
= "qemu.log";
217 static const char *logfilename
= "/tmp/qemu.log";
221 static int log_append
= 0;
224 #if !defined(CONFIG_USER_ONLY)
225 static int tlb_flush_count
;
227 static int tb_flush_count
;
228 static int tb_phys_invalidate_count
;
231 static void map_exec(void *addr
, long size
)
234 VirtualProtect(addr
, size
,
235 PAGE_EXECUTE_READWRITE
, &old_protect
);
239 static void map_exec(void *addr
, long size
)
241 unsigned long start
, end
, page_size
;
243 page_size
= getpagesize();
244 start
= (unsigned long)addr
;
245 start
&= ~(page_size
- 1);
247 end
= (unsigned long)addr
+ size
;
248 end
+= page_size
- 1;
249 end
&= ~(page_size
- 1);
251 mprotect((void *)start
, end
- start
,
252 PROT_READ
| PROT_WRITE
| PROT_EXEC
);
256 static void page_init(void)
258 /* NOTE: we can always suppose that qemu_host_page_size >=
262 SYSTEM_INFO system_info
;
264 GetSystemInfo(&system_info
);
265 qemu_real_host_page_size
= system_info
.dwPageSize
;
268 qemu_real_host_page_size
= getpagesize();
270 if (qemu_host_page_size
== 0)
271 qemu_host_page_size
= qemu_real_host_page_size
;
272 if (qemu_host_page_size
< TARGET_PAGE_SIZE
)
273 qemu_host_page_size
= TARGET_PAGE_SIZE
;
274 qemu_host_page_bits
= 0;
275 while ((1 << qemu_host_page_bits
) < qemu_host_page_size
)
276 qemu_host_page_bits
++;
277 qemu_host_page_mask
= ~(qemu_host_page_size
- 1);
279 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
281 #ifdef HAVE_KINFO_GETVMMAP
282 struct kinfo_vmentry
*freep
;
285 freep
= kinfo_getvmmap(getpid(), &cnt
);
288 for (i
= 0; i
< cnt
; i
++) {
289 unsigned long startaddr
, endaddr
;
291 startaddr
= freep
[i
].kve_start
;
292 endaddr
= freep
[i
].kve_end
;
293 if (h2g_valid(startaddr
)) {
294 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
296 if (h2g_valid(endaddr
)) {
297 endaddr
= h2g(endaddr
);
298 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
300 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
302 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
313 last_brk
= (unsigned long)sbrk(0);
315 f
= fopen("/compat/linux/proc/self/maps", "r");
320 unsigned long startaddr
, endaddr
;
323 n
= fscanf (f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
325 if (n
== 2 && h2g_valid(startaddr
)) {
326 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
328 if (h2g_valid(endaddr
)) {
329 endaddr
= h2g(endaddr
);
333 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
345 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
351 #if defined(CONFIG_USER_ONLY)
352 /* We can't use qemu_malloc because it may recurse into a locked mutex. */
353 # define ALLOC(P, SIZE) \
355 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
356 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
359 # define ALLOC(P, SIZE) \
360 do { P = qemu_mallocz(SIZE); } while (0)
363 /* Level 1. Always allocated. */
364 lp
= l1_map
+ ((index
>> V_L1_SHIFT
) & (V_L1_SIZE
- 1));
367 for (i
= V_L1_SHIFT
/ L2_BITS
- 1; i
> 0; i
--) {
374 ALLOC(p
, sizeof(void *) * L2_SIZE
);
378 lp
= p
+ ((index
>> (i
* L2_BITS
)) & (L2_SIZE
- 1));
386 ALLOC(pd
, sizeof(PageDesc
) * L2_SIZE
);
392 return pd
+ (index
& (L2_SIZE
- 1));
395 static inline PageDesc
*page_find(tb_page_addr_t index
)
397 return page_find_alloc(index
, 0);
400 #if !defined(CONFIG_USER_ONLY)
401 static PhysPageDesc
*phys_page_find_alloc(target_phys_addr_t index
, int alloc
)
407 /* Level 1. Always allocated. */
408 lp
= l1_phys_map
+ ((index
>> P_L1_SHIFT
) & (P_L1_SIZE
- 1));
411 for (i
= P_L1_SHIFT
/ L2_BITS
- 1; i
> 0; i
--) {
417 *lp
= p
= qemu_mallocz(sizeof(void *) * L2_SIZE
);
419 lp
= p
+ ((index
>> (i
* L2_BITS
)) & (L2_SIZE
- 1));
430 *lp
= pd
= qemu_malloc(sizeof(PhysPageDesc
) * L2_SIZE
);
432 for (i
= 0; i
< L2_SIZE
; i
++) {
433 pd
[i
].phys_offset
= IO_MEM_UNASSIGNED
;
434 pd
[i
].region_offset
= (index
+ i
) << TARGET_PAGE_BITS
;
438 return pd
+ (index
& (L2_SIZE
- 1));
441 static inline PhysPageDesc
*phys_page_find(target_phys_addr_t index
)
443 return phys_page_find_alloc(index
, 0);
446 static void tlb_protect_code(ram_addr_t ram_addr
);
447 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
449 #define mmap_lock() do { } while(0)
450 #define mmap_unlock() do { } while(0)
453 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
455 #if defined(CONFIG_USER_ONLY)
456 /* Currently it is not recommended to allocate big chunks of data in
457 user mode. It will change when a dedicated libc will be used */
458 #define USE_STATIC_CODE_GEN_BUFFER
461 #ifdef USE_STATIC_CODE_GEN_BUFFER
462 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
]
463 __attribute__((aligned (CODE_GEN_ALIGN
)));
466 static void code_gen_alloc(unsigned long tb_size
)
471 #ifdef USE_STATIC_CODE_GEN_BUFFER
472 code_gen_buffer
= static_code_gen_buffer
;
473 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
474 map_exec(code_gen_buffer
, code_gen_buffer_size
);
476 code_gen_buffer_size
= tb_size
;
477 if (code_gen_buffer_size
== 0) {
478 #if defined(CONFIG_USER_ONLY)
479 /* in user mode, phys_ram_size is not meaningful */
480 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
482 /* XXX: needs adjustments */
483 code_gen_buffer_size
= (unsigned long)(ram_size
/ 4);
486 if (code_gen_buffer_size
< MIN_CODE_GEN_BUFFER_SIZE
)
487 code_gen_buffer_size
= MIN_CODE_GEN_BUFFER_SIZE
;
488 /* The code gen buffer location may have constraints depending on
489 the host cpu and OS */
490 #if defined(__linux__)
495 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
496 #if defined(__x86_64__)
498 /* Cannot map more than that */
499 if (code_gen_buffer_size
> (800 * 1024 * 1024))
500 code_gen_buffer_size
= (800 * 1024 * 1024);
501 #elif defined(__sparc_v9__)
502 // Map the buffer below 2G, so we can use direct calls and branches
504 start
= (void *) 0x60000000UL
;
505 if (code_gen_buffer_size
> (512 * 1024 * 1024))
506 code_gen_buffer_size
= (512 * 1024 * 1024);
507 #elif defined(__arm__)
508 /* Map the buffer below 32M, so we can use direct calls and branches */
510 start
= (void *) 0x01000000UL
;
511 if (code_gen_buffer_size
> 16 * 1024 * 1024)
512 code_gen_buffer_size
= 16 * 1024 * 1024;
513 #elif defined(__s390x__)
514 /* Map the buffer so that we can use direct calls and branches. */
515 /* We have a +- 4GB range on the branches; leave some slop. */
516 if (code_gen_buffer_size
> (3ul * 1024 * 1024 * 1024)) {
517 code_gen_buffer_size
= 3ul * 1024 * 1024 * 1024;
519 start
= (void *)0x90000000UL
;
521 code_gen_buffer
= mmap(start
, code_gen_buffer_size
,
522 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
524 if (code_gen_buffer
== MAP_FAILED
) {
525 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
529 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__DragonFly__)
533 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
534 #if defined(__x86_64__)
535 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
536 * 0x40000000 is free */
538 addr
= (void *)0x40000000;
539 /* Cannot map more than that */
540 if (code_gen_buffer_size
> (800 * 1024 * 1024))
541 code_gen_buffer_size
= (800 * 1024 * 1024);
543 code_gen_buffer
= mmap(addr
, code_gen_buffer_size
,
544 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
546 if (code_gen_buffer
== MAP_FAILED
) {
547 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
552 code_gen_buffer
= qemu_malloc(code_gen_buffer_size
);
553 map_exec(code_gen_buffer
, code_gen_buffer_size
);
555 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
556 map_exec(code_gen_prologue
, sizeof(code_gen_prologue
));
557 code_gen_buffer_max_size
= code_gen_buffer_size
-
558 (TCG_MAX_OP_SIZE
* OPC_MAX_SIZE
);
559 code_gen_max_blocks
= code_gen_buffer_size
/ CODE_GEN_AVG_BLOCK_SIZE
;
560 tbs
= qemu_malloc(code_gen_max_blocks
* sizeof(TranslationBlock
));
563 /* Must be called before using the QEMU cpus. 'tb_size' is the size
564 (in bytes) allocated to the translation buffer. Zero means default
566 void cpu_exec_init_all(unsigned long tb_size
)
569 code_gen_alloc(tb_size
);
570 code_gen_ptr
= code_gen_buffer
;
572 #if !defined(CONFIG_USER_ONLY)
575 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
576 /* There's no guest base to take into account, so go ahead and
577 initialize the prologue now. */
578 tcg_prologue_init(&tcg_ctx
);
582 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
584 static int cpu_common_post_load(void *opaque
, int version_id
)
586 CPUState
*env
= opaque
;
588 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
589 version_id is increased. */
590 env
->interrupt_request
&= ~0x01;
596 static const VMStateDescription vmstate_cpu_common
= {
597 .name
= "cpu_common",
599 .minimum_version_id
= 1,
600 .minimum_version_id_old
= 1,
601 .post_load
= cpu_common_post_load
,
602 .fields
= (VMStateField
[]) {
603 VMSTATE_UINT32(halted
, CPUState
),
604 VMSTATE_UINT32(interrupt_request
, CPUState
),
605 VMSTATE_END_OF_LIST()
610 CPUState
*qemu_get_cpu(int cpu
)
612 CPUState
*env
= first_cpu
;
615 if (env
->cpu_index
== cpu
)
623 void cpu_exec_init(CPUState
*env
)
628 #if defined(CONFIG_USER_ONLY)
631 env
->next_cpu
= NULL
;
634 while (*penv
!= NULL
) {
635 penv
= &(*penv
)->next_cpu
;
638 env
->cpu_index
= cpu_index
;
640 QTAILQ_INIT(&env
->breakpoints
);
641 QTAILQ_INIT(&env
->watchpoints
);
643 env
->thread_id
= GetCurrentProcessId();
645 env
->thread_id
= getpid();
648 #if defined(CONFIG_USER_ONLY)
651 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
652 vmstate_register(NULL
, cpu_index
, &vmstate_cpu_common
, env
);
653 register_savevm(NULL
, "cpu", cpu_index
, CPU_SAVE_VERSION
,
654 cpu_save
, cpu_load
, env
);
658 static inline void invalidate_page_bitmap(PageDesc
*p
)
660 if (p
->code_bitmap
) {
661 qemu_free(p
->code_bitmap
);
662 p
->code_bitmap
= NULL
;
664 p
->code_write_count
= 0;
667 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
669 static void page_flush_tb_1 (int level
, void **lp
)
678 for (i
= 0; i
< L2_SIZE
; ++i
) {
679 pd
[i
].first_tb
= NULL
;
680 invalidate_page_bitmap(pd
+ i
);
684 for (i
= 0; i
< L2_SIZE
; ++i
) {
685 page_flush_tb_1 (level
- 1, pp
+ i
);
690 static void page_flush_tb(void)
693 for (i
= 0; i
< V_L1_SIZE
; i
++) {
694 page_flush_tb_1(V_L1_SHIFT
/ L2_BITS
- 1, l1_map
+ i
);
698 /* flush all the translation blocks */
699 /* XXX: tb_flush is currently not thread safe */
700 void tb_flush(CPUState
*env1
)
703 #if defined(DEBUG_FLUSH)
704 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
705 (unsigned long)(code_gen_ptr
- code_gen_buffer
),
707 ((unsigned long)(code_gen_ptr
- code_gen_buffer
)) / nb_tbs
: 0);
709 if ((unsigned long)(code_gen_ptr
- code_gen_buffer
) > code_gen_buffer_size
)
710 cpu_abort(env1
, "Internal error: code buffer overflow\n");
714 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
715 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
718 memset (tb_phys_hash
, 0, CODE_GEN_PHYS_HASH_SIZE
* sizeof (void *));
721 code_gen_ptr
= code_gen_buffer
;
722 /* XXX: flush processor icache at this point if cache flush is
727 #ifdef DEBUG_TB_CHECK
729 static void tb_invalidate_check(target_ulong address
)
731 TranslationBlock
*tb
;
733 address
&= TARGET_PAGE_MASK
;
734 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
735 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
736 if (!(address
+ TARGET_PAGE_SIZE
<= tb
->pc
||
737 address
>= tb
->pc
+ tb
->size
)) {
738 printf("ERROR invalidate: address=" TARGET_FMT_lx
739 " PC=%08lx size=%04x\n",
740 address
, (long)tb
->pc
, tb
->size
);
746 /* verify that all the pages have correct rights for code */
747 static void tb_page_check(void)
749 TranslationBlock
*tb
;
750 int i
, flags1
, flags2
;
752 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
753 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
754 flags1
= page_get_flags(tb
->pc
);
755 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
756 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
757 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
758 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
766 /* invalidate one TB */
767 static inline void tb_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
,
770 TranslationBlock
*tb1
;
774 *ptb
= *(TranslationBlock
**)((char *)tb1
+ next_offset
);
777 ptb
= (TranslationBlock
**)((char *)tb1
+ next_offset
);
781 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
783 TranslationBlock
*tb1
;
789 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
791 *ptb
= tb1
->page_next
[n1
];
794 ptb
= &tb1
->page_next
[n1
];
798 static inline void tb_jmp_remove(TranslationBlock
*tb
, int n
)
800 TranslationBlock
*tb1
, **ptb
;
803 ptb
= &tb
->jmp_next
[n
];
806 /* find tb(n) in circular list */
810 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
811 if (n1
== n
&& tb1
== tb
)
814 ptb
= &tb1
->jmp_first
;
816 ptb
= &tb1
->jmp_next
[n1
];
819 /* now we can suppress tb(n) from the list */
820 *ptb
= tb
->jmp_next
[n
];
822 tb
->jmp_next
[n
] = NULL
;
826 /* reset the jump entry 'n' of a TB so that it is not chained to
828 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
830 tb_set_jmp_target(tb
, n
, (unsigned long)(tb
->tc_ptr
+ tb
->tb_next_offset
[n
]));
833 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
838 tb_page_addr_t phys_pc
;
839 TranslationBlock
*tb1
, *tb2
;
841 /* remove the TB from the hash list */
842 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
843 h
= tb_phys_hash_func(phys_pc
);
844 tb_remove(&tb_phys_hash
[h
], tb
,
845 offsetof(TranslationBlock
, phys_hash_next
));
847 /* remove the TB from the page list */
848 if (tb
->page_addr
[0] != page_addr
) {
849 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
850 tb_page_remove(&p
->first_tb
, tb
);
851 invalidate_page_bitmap(p
);
853 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
854 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
855 tb_page_remove(&p
->first_tb
, tb
);
856 invalidate_page_bitmap(p
);
859 tb_invalidated_flag
= 1;
861 /* remove the TB from the hash list */
862 h
= tb_jmp_cache_hash_func(tb
->pc
);
863 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
864 if (env
->tb_jmp_cache
[h
] == tb
)
865 env
->tb_jmp_cache
[h
] = NULL
;
868 /* suppress this TB from the two jump lists */
869 tb_jmp_remove(tb
, 0);
870 tb_jmp_remove(tb
, 1);
872 /* suppress any remaining jumps to this TB */
878 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
879 tb2
= tb1
->jmp_next
[n1
];
880 tb_reset_jump(tb1
, n1
);
881 tb1
->jmp_next
[n1
] = NULL
;
884 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2); /* fail safe */
886 tb_phys_invalidate_count
++;
889 static inline void set_bits(uint8_t *tab
, int start
, int len
)
895 mask
= 0xff << (start
& 7);
896 if ((start
& ~7) == (end
& ~7)) {
898 mask
&= ~(0xff << (end
& 7));
903 start
= (start
+ 8) & ~7;
905 while (start
< end1
) {
910 mask
= ~(0xff << (end
& 7));
916 static void build_page_bitmap(PageDesc
*p
)
918 int n
, tb_start
, tb_end
;
919 TranslationBlock
*tb
;
921 p
->code_bitmap
= qemu_mallocz(TARGET_PAGE_SIZE
/ 8);
926 tb
= (TranslationBlock
*)((long)tb
& ~3);
927 /* NOTE: this is subtle as a TB may span two physical pages */
929 /* NOTE: tb_end may be after the end of the page, but
930 it is not a problem */
931 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
932 tb_end
= tb_start
+ tb
->size
;
933 if (tb_end
> TARGET_PAGE_SIZE
)
934 tb_end
= TARGET_PAGE_SIZE
;
937 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
939 set_bits(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
940 tb
= tb
->page_next
[n
];
944 TranslationBlock
*tb_gen_code(CPUState
*env
,
945 target_ulong pc
, target_ulong cs_base
,
946 int flags
, int cflags
)
948 TranslationBlock
*tb
;
950 tb_page_addr_t phys_pc
, phys_page2
;
951 target_ulong virt_page2
;
954 phys_pc
= get_page_addr_code(env
, pc
);
957 /* flush must be done */
959 /* cannot fail at this point */
961 /* Don't forget to invalidate previous TB info. */
962 tb_invalidated_flag
= 1;
964 tc_ptr
= code_gen_ptr
;
966 tb
->cs_base
= cs_base
;
969 cpu_gen_code(env
, tb
, &code_gen_size
);
970 code_gen_ptr
= (void *)(((unsigned long)code_gen_ptr
+ code_gen_size
+ CODE_GEN_ALIGN
- 1) & ~(CODE_GEN_ALIGN
- 1));
972 /* check next page if needed */
973 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
975 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
976 phys_page2
= get_page_addr_code(env
, virt_page2
);
978 tb_link_page(tb
, phys_pc
, phys_page2
);
982 /* invalidate all TBs which intersect with the target physical page
983 starting in range [start;end[. NOTE: start and end must refer to
984 the same physical page. 'is_cpu_write_access' should be true if called
985 from a real cpu write access: the virtual CPU will exit the current
986 TB if code is modified inside this TB. */
987 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
988 int is_cpu_write_access
)
990 TranslationBlock
*tb
, *tb_next
, *saved_tb
;
991 CPUState
*env
= cpu_single_env
;
992 tb_page_addr_t tb_start
, tb_end
;
995 #ifdef TARGET_HAS_PRECISE_SMC
996 int current_tb_not_found
= is_cpu_write_access
;
997 TranslationBlock
*current_tb
= NULL
;
998 int current_tb_modified
= 0;
999 target_ulong current_pc
= 0;
1000 target_ulong current_cs_base
= 0;
1001 int current_flags
= 0;
1002 #endif /* TARGET_HAS_PRECISE_SMC */
1004 p
= page_find(start
>> TARGET_PAGE_BITS
);
1007 if (!p
->code_bitmap
&&
1008 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
&&
1009 is_cpu_write_access
) {
1010 /* build code bitmap */
1011 build_page_bitmap(p
);
1014 /* we remove all the TBs in the range [start, end[ */
1015 /* XXX: see if in some cases it could be faster to invalidate all the code */
1017 while (tb
!= NULL
) {
1019 tb
= (TranslationBlock
*)((long)tb
& ~3);
1020 tb_next
= tb
->page_next
[n
];
1021 /* NOTE: this is subtle as a TB may span two physical pages */
1023 /* NOTE: tb_end may be after the end of the page, but
1024 it is not a problem */
1025 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1026 tb_end
= tb_start
+ tb
->size
;
1028 tb_start
= tb
->page_addr
[1];
1029 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1031 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1032 #ifdef TARGET_HAS_PRECISE_SMC
1033 if (current_tb_not_found
) {
1034 current_tb_not_found
= 0;
1036 if (env
->mem_io_pc
) {
1037 /* now we have a real cpu fault */
1038 current_tb
= tb_find_pc(env
->mem_io_pc
);
1041 if (current_tb
== tb
&&
1042 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1043 /* If we are modifying the current TB, we must stop
1044 its execution. We could be more precise by checking
1045 that the modification is after the current PC, but it
1046 would require a specialized function to partially
1047 restore the CPU state */
1049 current_tb_modified
= 1;
1050 cpu_restore_state(current_tb
, env
,
1051 env
->mem_io_pc
, NULL
);
1052 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1055 #endif /* TARGET_HAS_PRECISE_SMC */
1056 /* we need to do that to handle the case where a signal
1057 occurs while doing tb_phys_invalidate() */
1060 saved_tb
= env
->current_tb
;
1061 env
->current_tb
= NULL
;
1063 tb_phys_invalidate(tb
, -1);
1065 env
->current_tb
= saved_tb
;
1066 if (env
->interrupt_request
&& env
->current_tb
)
1067 cpu_interrupt(env
, env
->interrupt_request
);
1072 #if !defined(CONFIG_USER_ONLY)
1073 /* if no code remaining, no need to continue to use slow writes */
1075 invalidate_page_bitmap(p
);
1076 if (is_cpu_write_access
) {
1077 tlb_unprotect_code_phys(env
, start
, env
->mem_io_vaddr
);
1081 #ifdef TARGET_HAS_PRECISE_SMC
1082 if (current_tb_modified
) {
1083 /* we generate a block containing just the instruction
1084 modifying the memory. It will ensure that it cannot modify
1086 env
->current_tb
= NULL
;
1087 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1088 cpu_resume_from_signal(env
, NULL
);
1093 /* len must be <= 8 and start must be a multiple of len */
1094 static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start
, int len
)
1100 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1101 cpu_single_env
->mem_io_vaddr
, len
,
1102 cpu_single_env
->eip
,
1103 cpu_single_env
->eip
+ (long)cpu_single_env
->segs
[R_CS
].base
);
1106 p
= page_find(start
>> TARGET_PAGE_BITS
);
1109 if (p
->code_bitmap
) {
1110 offset
= start
& ~TARGET_PAGE_MASK
;
1111 b
= p
->code_bitmap
[offset
>> 3] >> (offset
& 7);
1112 if (b
& ((1 << len
) - 1))
1116 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1120 #if !defined(CONFIG_SOFTMMU)
1121 static void tb_invalidate_phys_page(tb_page_addr_t addr
,
1122 unsigned long pc
, void *puc
)
1124 TranslationBlock
*tb
;
1127 #ifdef TARGET_HAS_PRECISE_SMC
1128 TranslationBlock
*current_tb
= NULL
;
1129 CPUState
*env
= cpu_single_env
;
1130 int current_tb_modified
= 0;
1131 target_ulong current_pc
= 0;
1132 target_ulong current_cs_base
= 0;
1133 int current_flags
= 0;
1136 addr
&= TARGET_PAGE_MASK
;
1137 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1141 #ifdef TARGET_HAS_PRECISE_SMC
1142 if (tb
&& pc
!= 0) {
1143 current_tb
= tb_find_pc(pc
);
1146 while (tb
!= NULL
) {
1148 tb
= (TranslationBlock
*)((long)tb
& ~3);
1149 #ifdef TARGET_HAS_PRECISE_SMC
1150 if (current_tb
== tb
&&
1151 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1152 /* If we are modifying the current TB, we must stop
1153 its execution. We could be more precise by checking
1154 that the modification is after the current PC, but it
1155 would require a specialized function to partially
1156 restore the CPU state */
1158 current_tb_modified
= 1;
1159 cpu_restore_state(current_tb
, env
, pc
, puc
);
1160 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1163 #endif /* TARGET_HAS_PRECISE_SMC */
1164 tb_phys_invalidate(tb
, addr
);
1165 tb
= tb
->page_next
[n
];
1168 #ifdef TARGET_HAS_PRECISE_SMC
1169 if (current_tb_modified
) {
1170 /* we generate a block containing just the instruction
1171 modifying the memory. It will ensure that it cannot modify
1173 env
->current_tb
= NULL
;
1174 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1175 cpu_resume_from_signal(env
, puc
);
1181 /* add the tb in the target page and protect it if necessary */
1182 static inline void tb_alloc_page(TranslationBlock
*tb
,
1183 unsigned int n
, tb_page_addr_t page_addr
)
1186 TranslationBlock
*last_first_tb
;
1188 tb
->page_addr
[n
] = page_addr
;
1189 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
, 1);
1190 tb
->page_next
[n
] = p
->first_tb
;
1191 last_first_tb
= p
->first_tb
;
1192 p
->first_tb
= (TranslationBlock
*)((long)tb
| n
);
1193 invalidate_page_bitmap(p
);
1195 #if defined(TARGET_HAS_SMC) || 1
1197 #if defined(CONFIG_USER_ONLY)
1198 if (p
->flags
& PAGE_WRITE
) {
1203 /* force the host page as non writable (writes will have a
1204 page fault + mprotect overhead) */
1205 page_addr
&= qemu_host_page_mask
;
1207 for(addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1208 addr
+= TARGET_PAGE_SIZE
) {
1210 p2
= page_find (addr
>> TARGET_PAGE_BITS
);
1214 p2
->flags
&= ~PAGE_WRITE
;
1216 mprotect(g2h(page_addr
), qemu_host_page_size
,
1217 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1218 #ifdef DEBUG_TB_INVALIDATE
1219 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1224 /* if some code is already present, then the pages are already
1225 protected. So we handle the case where only the first TB is
1226 allocated in a physical page */
1227 if (!last_first_tb
) {
1228 tlb_protect_code(page_addr
);
1232 #endif /* TARGET_HAS_SMC */
1235 /* Allocate a new translation block. Flush the translation buffer if
1236 too many translation blocks or too much generated code. */
1237 TranslationBlock
*tb_alloc(target_ulong pc
)
1239 TranslationBlock
*tb
;
1241 if (nb_tbs
>= code_gen_max_blocks
||
1242 (code_gen_ptr
- code_gen_buffer
) >= code_gen_buffer_max_size
)
1244 tb
= &tbs
[nb_tbs
++];
1250 void tb_free(TranslationBlock
*tb
)
1252 /* In practice this is mostly used for single use temporary TB
1253 Ignore the hard cases and just back up if this TB happens to
1254 be the last one generated. */
1255 if (nb_tbs
> 0 && tb
== &tbs
[nb_tbs
- 1]) {
1256 code_gen_ptr
= tb
->tc_ptr
;
1261 /* add a new TB and link it to the physical page tables. phys_page2 is
1262 (-1) to indicate that only one page contains the TB. */
1263 void tb_link_page(TranslationBlock
*tb
,
1264 tb_page_addr_t phys_pc
, tb_page_addr_t phys_page2
)
1267 TranslationBlock
**ptb
;
1269 /* Grab the mmap lock to stop another thread invalidating this TB
1270 before we are done. */
1272 /* add in the physical hash table */
1273 h
= tb_phys_hash_func(phys_pc
);
1274 ptb
= &tb_phys_hash
[h
];
1275 tb
->phys_hash_next
= *ptb
;
1278 /* add in the page list */
1279 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1280 if (phys_page2
!= -1)
1281 tb_alloc_page(tb
, 1, phys_page2
);
1283 tb
->page_addr
[1] = -1;
1285 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2);
1286 tb
->jmp_next
[0] = NULL
;
1287 tb
->jmp_next
[1] = NULL
;
1289 /* init original jump addresses */
1290 if (tb
->tb_next_offset
[0] != 0xffff)
1291 tb_reset_jump(tb
, 0);
1292 if (tb
->tb_next_offset
[1] != 0xffff)
1293 tb_reset_jump(tb
, 1);
1295 #ifdef DEBUG_TB_CHECK
1301 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1302 tb[1].tc_ptr. Return NULL if not found */
1303 TranslationBlock
*tb_find_pc(unsigned long tc_ptr
)
1305 int m_min
, m_max
, m
;
1307 TranslationBlock
*tb
;
1311 if (tc_ptr
< (unsigned long)code_gen_buffer
||
1312 tc_ptr
>= (unsigned long)code_gen_ptr
)
1314 /* binary search (cf Knuth) */
1317 while (m_min
<= m_max
) {
1318 m
= (m_min
+ m_max
) >> 1;
1320 v
= (unsigned long)tb
->tc_ptr
;
1323 else if (tc_ptr
< v
) {
1332 static void tb_reset_jump_recursive(TranslationBlock
*tb
);
1334 static inline void tb_reset_jump_recursive2(TranslationBlock
*tb
, int n
)
1336 TranslationBlock
*tb1
, *tb_next
, **ptb
;
1339 tb1
= tb
->jmp_next
[n
];
1341 /* find head of list */
1344 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1347 tb1
= tb1
->jmp_next
[n1
];
1349 /* we are now sure now that tb jumps to tb1 */
1352 /* remove tb from the jmp_first list */
1353 ptb
= &tb_next
->jmp_first
;
1357 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1358 if (n1
== n
&& tb1
== tb
)
1360 ptb
= &tb1
->jmp_next
[n1
];
1362 *ptb
= tb
->jmp_next
[n
];
1363 tb
->jmp_next
[n
] = NULL
;
1365 /* suppress the jump to next tb in generated code */
1366 tb_reset_jump(tb
, n
);
1368 /* suppress jumps in the tb on which we could have jumped */
1369 tb_reset_jump_recursive(tb_next
);
1373 static void tb_reset_jump_recursive(TranslationBlock
*tb
)
1375 tb_reset_jump_recursive2(tb
, 0);
1376 tb_reset_jump_recursive2(tb
, 1);
1379 #if defined(TARGET_HAS_ICE)
1380 #if defined(CONFIG_USER_ONLY)
1381 static void breakpoint_invalidate(CPUState
*env
, target_ulong pc
)
1383 tb_invalidate_phys_page_range(pc
, pc
+ 1, 0);
1386 static void breakpoint_invalidate(CPUState
*env
, target_ulong pc
)
1388 target_phys_addr_t addr
;
1390 ram_addr_t ram_addr
;
1393 addr
= cpu_get_phys_page_debug(env
, pc
);
1394 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
1396 pd
= IO_MEM_UNASSIGNED
;
1398 pd
= p
->phys_offset
;
1400 ram_addr
= (pd
& TARGET_PAGE_MASK
) | (pc
& ~TARGET_PAGE_MASK
);
1401 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1404 #endif /* TARGET_HAS_ICE */
1406 #if defined(CONFIG_USER_ONLY)
1407 void cpu_watchpoint_remove_all(CPUState
*env
, int mask
)
1412 int cpu_watchpoint_insert(CPUState
*env
, target_ulong addr
, target_ulong len
,
1413 int flags
, CPUWatchpoint
**watchpoint
)
1418 /* Add a watchpoint. */
1419 int cpu_watchpoint_insert(CPUState
*env
, target_ulong addr
, target_ulong len
,
1420 int flags
, CPUWatchpoint
**watchpoint
)
1422 target_ulong len_mask
= ~(len
- 1);
1425 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1426 if ((len
!= 1 && len
!= 2 && len
!= 4 && len
!= 8) || (addr
& ~len_mask
)) {
1427 fprintf(stderr
, "qemu: tried to set invalid watchpoint at "
1428 TARGET_FMT_lx
", len=" TARGET_FMT_lu
"\n", addr
, len
);
1431 wp
= qemu_malloc(sizeof(*wp
));
1434 wp
->len_mask
= len_mask
;
1437 /* keep all GDB-injected watchpoints in front */
1439 QTAILQ_INSERT_HEAD(&env
->watchpoints
, wp
, entry
);
1441 QTAILQ_INSERT_TAIL(&env
->watchpoints
, wp
, entry
);
1443 tlb_flush_page(env
, addr
);
1450 /* Remove a specific watchpoint. */
1451 int cpu_watchpoint_remove(CPUState
*env
, target_ulong addr
, target_ulong len
,
1454 target_ulong len_mask
= ~(len
- 1);
1457 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1458 if (addr
== wp
->vaddr
&& len_mask
== wp
->len_mask
1459 && flags
== (wp
->flags
& ~BP_WATCHPOINT_HIT
)) {
1460 cpu_watchpoint_remove_by_ref(env
, wp
);
1467 /* Remove a specific watchpoint by reference. */
1468 void cpu_watchpoint_remove_by_ref(CPUState
*env
, CPUWatchpoint
*watchpoint
)
1470 QTAILQ_REMOVE(&env
->watchpoints
, watchpoint
, entry
);
1472 tlb_flush_page(env
, watchpoint
->vaddr
);
1474 qemu_free(watchpoint
);
1477 /* Remove all matching watchpoints. */
1478 void cpu_watchpoint_remove_all(CPUState
*env
, int mask
)
1480 CPUWatchpoint
*wp
, *next
;
1482 QTAILQ_FOREACH_SAFE(wp
, &env
->watchpoints
, entry
, next
) {
1483 if (wp
->flags
& mask
)
1484 cpu_watchpoint_remove_by_ref(env
, wp
);
1489 /* Add a breakpoint. */
1490 int cpu_breakpoint_insert(CPUState
*env
, target_ulong pc
, int flags
,
1491 CPUBreakpoint
**breakpoint
)
1493 #if defined(TARGET_HAS_ICE)
1496 bp
= qemu_malloc(sizeof(*bp
));
1501 /* keep all GDB-injected breakpoints in front */
1503 QTAILQ_INSERT_HEAD(&env
->breakpoints
, bp
, entry
);
1505 QTAILQ_INSERT_TAIL(&env
->breakpoints
, bp
, entry
);
1507 breakpoint_invalidate(env
, pc
);
1517 /* Remove a specific breakpoint. */
1518 int cpu_breakpoint_remove(CPUState
*env
, target_ulong pc
, int flags
)
1520 #if defined(TARGET_HAS_ICE)
1523 QTAILQ_FOREACH(bp
, &env
->breakpoints
, entry
) {
1524 if (bp
->pc
== pc
&& bp
->flags
== flags
) {
1525 cpu_breakpoint_remove_by_ref(env
, bp
);
1535 /* Remove a specific breakpoint by reference. */
1536 void cpu_breakpoint_remove_by_ref(CPUState
*env
, CPUBreakpoint
*breakpoint
)
1538 #if defined(TARGET_HAS_ICE)
1539 QTAILQ_REMOVE(&env
->breakpoints
, breakpoint
, entry
);
1541 breakpoint_invalidate(env
, breakpoint
->pc
);
1543 qemu_free(breakpoint
);
1547 /* Remove all matching breakpoints. */
1548 void cpu_breakpoint_remove_all(CPUState
*env
, int mask
)
1550 #if defined(TARGET_HAS_ICE)
1551 CPUBreakpoint
*bp
, *next
;
1553 QTAILQ_FOREACH_SAFE(bp
, &env
->breakpoints
, entry
, next
) {
1554 if (bp
->flags
& mask
)
1555 cpu_breakpoint_remove_by_ref(env
, bp
);
1560 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1561 CPU loop after each instruction */
1562 void cpu_single_step(CPUState
*env
, int enabled
)
1564 #if defined(TARGET_HAS_ICE)
1565 if (env
->singlestep_enabled
!= enabled
) {
1566 env
->singlestep_enabled
= enabled
;
1568 kvm_update_guest_debug(env
, 0);
1570 /* must flush all the translated code to avoid inconsistencies */
1571 /* XXX: only flush what is necessary */
1578 /* enable or disable low levels log */
1579 void cpu_set_log(int log_flags
)
1581 loglevel
= log_flags
;
1582 if (loglevel
&& !logfile
) {
1583 logfile
= fopen(logfilename
, log_append
? "a" : "w");
1585 perror(logfilename
);
1588 #if !defined(CONFIG_SOFTMMU)
1589 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1591 static char logfile_buf
[4096];
1592 setvbuf(logfile
, logfile_buf
, _IOLBF
, sizeof(logfile_buf
));
1594 #elif !defined(_WIN32)
1595 /* Win32 doesn't support line-buffering and requires size >= 2 */
1596 setvbuf(logfile
, NULL
, _IOLBF
, 0);
1600 if (!loglevel
&& logfile
) {
1606 void cpu_set_log_filename(const char *filename
)
1608 logfilename
= strdup(filename
);
1613 cpu_set_log(loglevel
);
1616 static void cpu_unlink_tb(CPUState
*env
)
1618 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1619 problem and hope the cpu will stop of its own accord. For userspace
1620 emulation this often isn't actually as bad as it sounds. Often
1621 signals are used primarily to interrupt blocking syscalls. */
1622 TranslationBlock
*tb
;
1623 static spinlock_t interrupt_lock
= SPIN_LOCK_UNLOCKED
;
1625 spin_lock(&interrupt_lock
);
1626 tb
= env
->current_tb
;
1627 /* if the cpu is currently executing code, we must unlink it and
1628 all the potentially executing TB */
1630 env
->current_tb
= NULL
;
1631 tb_reset_jump_recursive(tb
);
1633 spin_unlock(&interrupt_lock
);
1636 /* mask must never be zero, except for A20 change call */
1637 void cpu_interrupt(CPUState
*env
, int mask
)
1641 old_mask
= env
->interrupt_request
;
1642 env
->interrupt_request
|= mask
;
1643 if (kvm_enabled() && !kvm_irqchip_in_kernel())
1644 kvm_update_interrupt_request(env
);
1646 #ifndef CONFIG_USER_ONLY
1648 * If called from iothread context, wake the target cpu in
1651 if (!qemu_cpu_self(env
)) {
1658 env
->icount_decr
.u16
.high
= 0xffff;
1659 #ifndef CONFIG_USER_ONLY
1661 && (mask
& ~old_mask
) != 0) {
1662 cpu_abort(env
, "Raised interrupt while not in I/O function");
1670 void cpu_reset_interrupt(CPUState
*env
, int mask
)
1672 env
->interrupt_request
&= ~mask
;
1675 void cpu_exit(CPUState
*env
)
1677 env
->exit_request
= 1;
1681 const CPULogItem cpu_log_items
[] = {
1682 { CPU_LOG_TB_OUT_ASM
, "out_asm",
1683 "show generated host assembly code for each compiled TB" },
1684 { CPU_LOG_TB_IN_ASM
, "in_asm",
1685 "show target assembly code for each compiled TB" },
1686 { CPU_LOG_TB_OP
, "op",
1687 "show micro ops for each compiled TB" },
1688 { CPU_LOG_TB_OP_OPT
, "op_opt",
1691 "before eflags optimization and "
1693 "after liveness analysis" },
1694 { CPU_LOG_INT
, "int",
1695 "show interrupts/exceptions in short format" },
1696 { CPU_LOG_EXEC
, "exec",
1697 "show trace before each executed TB (lots of logs)" },
1698 { CPU_LOG_TB_CPU
, "cpu",
1699 "show CPU state before block translation" },
1701 { CPU_LOG_PCALL
, "pcall",
1702 "show protected mode far calls/returns/exceptions" },
1703 { CPU_LOG_RESET
, "cpu_reset",
1704 "show CPU state before CPU resets" },
1707 { CPU_LOG_IOPORT
, "ioport",
1708 "show all i/o ports accesses" },
1713 #ifndef CONFIG_USER_ONLY
1714 static QLIST_HEAD(memory_client_list
, CPUPhysMemoryClient
) memory_client_list
1715 = QLIST_HEAD_INITIALIZER(memory_client_list
);
1717 static void cpu_notify_set_memory(target_phys_addr_t start_addr
,
1719 ram_addr_t phys_offset
)
1721 CPUPhysMemoryClient
*client
;
1722 QLIST_FOREACH(client
, &memory_client_list
, list
) {
1723 client
->set_memory(client
, start_addr
, size
, phys_offset
);
1727 static int cpu_notify_sync_dirty_bitmap(target_phys_addr_t start
,
1728 target_phys_addr_t end
)
1730 CPUPhysMemoryClient
*client
;
1731 QLIST_FOREACH(client
, &memory_client_list
, list
) {
1732 int r
= client
->sync_dirty_bitmap(client
, start
, end
);
1739 static int cpu_notify_migration_log(int enable
)
1741 CPUPhysMemoryClient
*client
;
1742 QLIST_FOREACH(client
, &memory_client_list
, list
) {
1743 int r
= client
->migration_log(client
, enable
);
1750 static void phys_page_for_each_1(CPUPhysMemoryClient
*client
,
1751 int level
, void **lp
)
1759 PhysPageDesc
*pd
= *lp
;
1760 for (i
= 0; i
< L2_SIZE
; ++i
) {
1761 if (pd
[i
].phys_offset
!= IO_MEM_UNASSIGNED
) {
1762 client
->set_memory(client
, pd
[i
].region_offset
,
1763 TARGET_PAGE_SIZE
, pd
[i
].phys_offset
);
1768 for (i
= 0; i
< L2_SIZE
; ++i
) {
1769 phys_page_for_each_1(client
, level
- 1, pp
+ i
);
1774 static void phys_page_for_each(CPUPhysMemoryClient
*client
)
1777 for (i
= 0; i
< P_L1_SIZE
; ++i
) {
1778 phys_page_for_each_1(client
, P_L1_SHIFT
/ L2_BITS
- 1,
1783 void cpu_register_phys_memory_client(CPUPhysMemoryClient
*client
)
1785 QLIST_INSERT_HEAD(&memory_client_list
, client
, list
);
1786 phys_page_for_each(client
);
1789 void cpu_unregister_phys_memory_client(CPUPhysMemoryClient
*client
)
1791 QLIST_REMOVE(client
, list
);
1795 static int cmp1(const char *s1
, int n
, const char *s2
)
1797 if (strlen(s2
) != n
)
1799 return memcmp(s1
, s2
, n
) == 0;
1802 /* takes a comma separated list of log masks. Return 0 if error. */
1803 int cpu_str_to_log_mask(const char *str
)
1805 const CPULogItem
*item
;
1812 p1
= strchr(p
, ',');
1815 if(cmp1(p
,p1
-p
,"all")) {
1816 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1820 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1821 if (cmp1(p
, p1
- p
, item
->name
))
1835 void cpu_abort(CPUState
*env
, const char *fmt
, ...)
1842 fprintf(stderr
, "qemu: fatal: ");
1843 vfprintf(stderr
, fmt
, ap
);
1844 fprintf(stderr
, "\n");
1846 cpu_dump_state(env
, stderr
, fprintf
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1848 cpu_dump_state(env
, stderr
, fprintf
, 0);
1850 if (qemu_log_enabled()) {
1851 qemu_log("qemu: fatal: ");
1852 qemu_log_vprintf(fmt
, ap2
);
1855 log_cpu_state(env
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1857 log_cpu_state(env
, 0);
1864 #if defined(CONFIG_USER_ONLY)
1866 struct sigaction act
;
1867 sigfillset(&act
.sa_mask
);
1868 act
.sa_handler
= SIG_DFL
;
1869 sigaction(SIGABRT
, &act
, NULL
);
1875 CPUState
*cpu_copy(CPUState
*env
)
1877 CPUState
*new_env
= cpu_init(env
->cpu_model_str
);
1878 CPUState
*next_cpu
= new_env
->next_cpu
;
1879 int cpu_index
= new_env
->cpu_index
;
1880 #if defined(TARGET_HAS_ICE)
1885 memcpy(new_env
, env
, sizeof(CPUState
));
1887 /* Preserve chaining and index. */
1888 new_env
->next_cpu
= next_cpu
;
1889 new_env
->cpu_index
= cpu_index
;
1891 /* Clone all break/watchpoints.
1892 Note: Once we support ptrace with hw-debug register access, make sure
1893 BP_CPU break/watchpoints are handled correctly on clone. */
1894 QTAILQ_INIT(&env
->breakpoints
);
1895 QTAILQ_INIT(&env
->watchpoints
);
1896 #if defined(TARGET_HAS_ICE)
1897 QTAILQ_FOREACH(bp
, &env
->breakpoints
, entry
) {
1898 cpu_breakpoint_insert(new_env
, bp
->pc
, bp
->flags
, NULL
);
1900 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1901 cpu_watchpoint_insert(new_env
, wp
->vaddr
, (~wp
->len_mask
) + 1,
1909 #if !defined(CONFIG_USER_ONLY)
1911 static inline void tlb_flush_jmp_cache(CPUState
*env
, target_ulong addr
)
1915 /* Discard jump cache entries for any tb which might potentially
1916 overlap the flushed page. */
1917 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1918 memset (&env
->tb_jmp_cache
[i
], 0,
1919 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1921 i
= tb_jmp_cache_hash_page(addr
);
1922 memset (&env
->tb_jmp_cache
[i
], 0,
1923 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1926 static CPUTLBEntry s_cputlb_empty_entry
= {
1933 /* NOTE: if flush_global is true, also flush global entries (not
1935 void tlb_flush(CPUState
*env
, int flush_global
)
1939 #if defined(DEBUG_TLB)
1940 printf("tlb_flush:\n");
1942 /* must reset current TB so that interrupts cannot modify the
1943 links while we are modifying them */
1944 env
->current_tb
= NULL
;
1946 for(i
= 0; i
< CPU_TLB_SIZE
; i
++) {
1948 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
1949 env
->tlb_table
[mmu_idx
][i
] = s_cputlb_empty_entry
;
1953 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
1955 env
->tlb_flush_addr
= -1;
1956 env
->tlb_flush_mask
= 0;
1960 static inline void tlb_flush_entry(CPUTLBEntry
*tlb_entry
, target_ulong addr
)
1962 if (addr
== (tlb_entry
->addr_read
&
1963 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1964 addr
== (tlb_entry
->addr_write
&
1965 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1966 addr
== (tlb_entry
->addr_code
&
1967 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
))) {
1968 *tlb_entry
= s_cputlb_empty_entry
;
1972 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
1977 #if defined(DEBUG_TLB)
1978 printf("tlb_flush_page: " TARGET_FMT_lx
"\n", addr
);
1980 /* Check if we need to flush due to large pages. */
1981 if ((addr
& env
->tlb_flush_mask
) == env
->tlb_flush_addr
) {
1982 #if defined(DEBUG_TLB)
1983 printf("tlb_flush_page: forced full flush ("
1984 TARGET_FMT_lx
"/" TARGET_FMT_lx
")\n",
1985 env
->tlb_flush_addr
, env
->tlb_flush_mask
);
1990 /* must reset current TB so that interrupts cannot modify the
1991 links while we are modifying them */
1992 env
->current_tb
= NULL
;
1994 addr
&= TARGET_PAGE_MASK
;
1995 i
= (addr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
1996 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++)
1997 tlb_flush_entry(&env
->tlb_table
[mmu_idx
][i
], addr
);
1999 tlb_flush_jmp_cache(env
, addr
);
2002 /* update the TLBs so that writes to code in the virtual page 'addr'
2004 static void tlb_protect_code(ram_addr_t ram_addr
)
2006 cpu_physical_memory_reset_dirty(ram_addr
,
2007 ram_addr
+ TARGET_PAGE_SIZE
,
2011 /* update the TLB so that writes in physical page 'phys_addr' are no longer
2012 tested for self modifying code */
2013 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
2016 cpu_physical_memory_set_dirty_flags(ram_addr
, CODE_DIRTY_FLAG
);
2019 static inline void tlb_reset_dirty_range(CPUTLBEntry
*tlb_entry
,
2020 unsigned long start
, unsigned long length
)
2023 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
) {
2024 addr
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) + tlb_entry
->addend
;
2025 if ((addr
- start
) < length
) {
2026 tlb_entry
->addr_write
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) | TLB_NOTDIRTY
;
2031 /* Note: start and end must be within the same ram block. */
2032 void cpu_physical_memory_reset_dirty(ram_addr_t start
, ram_addr_t end
,
2036 unsigned long length
, start1
;
2039 start
&= TARGET_PAGE_MASK
;
2040 end
= TARGET_PAGE_ALIGN(end
);
2042 length
= end
- start
;
2045 cpu_physical_memory_mask_dirty_range(start
, length
, dirty_flags
);
2047 /* we modify the TLB cache so that the dirty bit will be set again
2048 when accessing the range */
2049 start1
= (unsigned long)qemu_get_ram_ptr(start
);
2050 /* Chek that we don't span multiple blocks - this breaks the
2051 address comparisons below. */
2052 if ((unsigned long)qemu_get_ram_ptr(end
- 1) - start1
2053 != (end
- 1) - start
) {
2057 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
2059 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
2060 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
2061 tlb_reset_dirty_range(&env
->tlb_table
[mmu_idx
][i
],
2067 int cpu_physical_memory_set_dirty_tracking(int enable
)
2070 in_migration
= enable
;
2071 ret
= cpu_notify_migration_log(!!enable
);
2075 int cpu_physical_memory_get_dirty_tracking(void)
2077 return in_migration
;
2080 int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr
,
2081 target_phys_addr_t end_addr
)
2085 ret
= cpu_notify_sync_dirty_bitmap(start_addr
, end_addr
);
2089 static inline void tlb_update_dirty(CPUTLBEntry
*tlb_entry
)
2091 ram_addr_t ram_addr
;
2094 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
) {
2095 p
= (void *)(unsigned long)((tlb_entry
->addr_write
& TARGET_PAGE_MASK
)
2096 + tlb_entry
->addend
);
2097 ram_addr
= qemu_ram_addr_from_host_nofail(p
);
2098 if (!cpu_physical_memory_is_dirty(ram_addr
)) {
2099 tlb_entry
->addr_write
|= TLB_NOTDIRTY
;
2104 /* update the TLB according to the current state of the dirty bits */
2105 void cpu_tlb_update_dirty(CPUState
*env
)
2109 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
2110 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
2111 tlb_update_dirty(&env
->tlb_table
[mmu_idx
][i
]);
2115 static inline void tlb_set_dirty1(CPUTLBEntry
*tlb_entry
, target_ulong vaddr
)
2117 if (tlb_entry
->addr_write
== (vaddr
| TLB_NOTDIRTY
))
2118 tlb_entry
->addr_write
= vaddr
;
2121 /* update the TLB corresponding to virtual page vaddr
2122 so that it is no longer dirty */
2123 static inline void tlb_set_dirty(CPUState
*env
, target_ulong vaddr
)
2128 vaddr
&= TARGET_PAGE_MASK
;
2129 i
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
2130 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++)
2131 tlb_set_dirty1(&env
->tlb_table
[mmu_idx
][i
], vaddr
);
2134 /* Our TLB does not support large pages, so remember the area covered by
2135 large pages and trigger a full TLB flush if these are invalidated. */
2136 static void tlb_add_large_page(CPUState
*env
, target_ulong vaddr
,
2139 target_ulong mask
= ~(size
- 1);
2141 if (env
->tlb_flush_addr
== (target_ulong
)-1) {
2142 env
->tlb_flush_addr
= vaddr
& mask
;
2143 env
->tlb_flush_mask
= mask
;
2146 /* Extend the existing region to include the new page.
2147 This is a compromise between unnecessary flushes and the cost
2148 of maintaining a full variable size TLB. */
2149 mask
&= env
->tlb_flush_mask
;
2150 while (((env
->tlb_flush_addr
^ vaddr
) & mask
) != 0) {
2153 env
->tlb_flush_addr
&= mask
;
2154 env
->tlb_flush_mask
= mask
;
2157 /* Add a new TLB entry. At most one entry for a given virtual address
2158 is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
2159 supplied size is only used by tlb_flush_page. */
2160 void tlb_set_page(CPUState
*env
, target_ulong vaddr
,
2161 target_phys_addr_t paddr
, int prot
,
2162 int mmu_idx
, target_ulong size
)
2167 target_ulong address
;
2168 target_ulong code_address
;
2169 unsigned long addend
;
2172 target_phys_addr_t iotlb
;
2174 assert(size
>= TARGET_PAGE_SIZE
);
2175 if (size
!= TARGET_PAGE_SIZE
) {
2176 tlb_add_large_page(env
, vaddr
, size
);
2178 p
= phys_page_find(paddr
>> TARGET_PAGE_BITS
);
2180 pd
= IO_MEM_UNASSIGNED
;
2182 pd
= p
->phys_offset
;
2184 #if defined(DEBUG_TLB)
2185 printf("tlb_set_page: vaddr=" TARGET_FMT_lx
" paddr=0x" TARGET_FMT_plx
2186 " prot=%x idx=%d pd=0x%08lx\n",
2187 vaddr
, paddr
, prot
, mmu_idx
, pd
);
2191 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&& !(pd
& IO_MEM_ROMD
)) {
2192 /* IO memory case (romd handled later) */
2193 address
|= TLB_MMIO
;
2195 addend
= (unsigned long)qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
);
2196 if ((pd
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
) {
2198 iotlb
= pd
& TARGET_PAGE_MASK
;
2199 if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
)
2200 iotlb
|= IO_MEM_NOTDIRTY
;
2202 iotlb
|= IO_MEM_ROM
;
2204 /* IO handlers are currently passed a physical address.
2205 It would be nice to pass an offset from the base address
2206 of that region. This would avoid having to special case RAM,
2207 and avoid full address decoding in every device.
2208 We can't use the high bits of pd for this because
2209 IO_MEM_ROMD uses these as a ram address. */
2210 iotlb
= (pd
& ~TARGET_PAGE_MASK
);
2212 iotlb
+= p
->region_offset
;
2218 code_address
= address
;
2219 /* Make accesses to pages with watchpoints go via the
2220 watchpoint trap routines. */
2221 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
2222 if (vaddr
== (wp
->vaddr
& TARGET_PAGE_MASK
)) {
2223 /* Avoid trapping reads of pages with a write breakpoint. */
2224 if ((prot
& PAGE_WRITE
) || (wp
->flags
& BP_MEM_READ
)) {
2225 iotlb
= io_mem_watch
+ paddr
;
2226 address
|= TLB_MMIO
;
2232 index
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
2233 env
->iotlb
[mmu_idx
][index
] = iotlb
- vaddr
;
2234 te
= &env
->tlb_table
[mmu_idx
][index
];
2235 te
->addend
= addend
- vaddr
;
2236 if (prot
& PAGE_READ
) {
2237 te
->addr_read
= address
;
2242 if (prot
& PAGE_EXEC
) {
2243 te
->addr_code
= code_address
;
2247 if (prot
& PAGE_WRITE
) {
2248 if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_ROM
||
2249 (pd
& IO_MEM_ROMD
)) {
2250 /* Write access calls the I/O callback. */
2251 te
->addr_write
= address
| TLB_MMIO
;
2252 } else if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
&&
2253 !cpu_physical_memory_is_dirty(pd
)) {
2254 te
->addr_write
= address
| TLB_NOTDIRTY
;
2256 te
->addr_write
= address
;
2259 te
->addr_write
= -1;
2265 void tlb_flush(CPUState
*env
, int flush_global
)
2269 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
2274 * Walks guest process memory "regions" one by one
2275 * and calls callback function 'fn' for each region.
2278 struct walk_memory_regions_data
2280 walk_memory_regions_fn fn
;
2282 unsigned long start
;
2286 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
2287 abi_ulong end
, int new_prot
)
2289 if (data
->start
!= -1ul) {
2290 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
2296 data
->start
= (new_prot
? end
: -1ul);
2297 data
->prot
= new_prot
;
2302 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
2303 abi_ulong base
, int level
, void **lp
)
2309 return walk_memory_regions_end(data
, base
, 0);
2314 for (i
= 0; i
< L2_SIZE
; ++i
) {
2315 int prot
= pd
[i
].flags
;
2317 pa
= base
| (i
<< TARGET_PAGE_BITS
);
2318 if (prot
!= data
->prot
) {
2319 rc
= walk_memory_regions_end(data
, pa
, prot
);
2327 for (i
= 0; i
< L2_SIZE
; ++i
) {
2328 pa
= base
| ((abi_ulong
)i
<<
2329 (TARGET_PAGE_BITS
+ L2_BITS
* level
));
2330 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
2340 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
2342 struct walk_memory_regions_data data
;
2350 for (i
= 0; i
< V_L1_SIZE
; i
++) {
2351 int rc
= walk_memory_regions_1(&data
, (abi_ulong
)i
<< V_L1_SHIFT
,
2352 V_L1_SHIFT
/ L2_BITS
- 1, l1_map
+ i
);
2358 return walk_memory_regions_end(&data
, 0, 0);
2361 static int dump_region(void *priv
, abi_ulong start
,
2362 abi_ulong end
, unsigned long prot
)
2364 FILE *f
= (FILE *)priv
;
2366 (void) fprintf(f
, TARGET_ABI_FMT_lx
"-"TARGET_ABI_FMT_lx
2367 " "TARGET_ABI_FMT_lx
" %c%c%c\n",
2368 start
, end
, end
- start
,
2369 ((prot
& PAGE_READ
) ? 'r' : '-'),
2370 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
2371 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
2376 /* dump memory mappings */
2377 void page_dump(FILE *f
)
2379 (void) fprintf(f
, "%-8s %-8s %-8s %s\n",
2380 "start", "end", "size", "prot");
2381 walk_memory_regions(f
, dump_region
);
2384 int page_get_flags(target_ulong address
)
2388 p
= page_find(address
>> TARGET_PAGE_BITS
);
2394 /* Modify the flags of a page and invalidate the code if necessary.
2395 The flag PAGE_WRITE_ORG is positioned automatically depending
2396 on PAGE_WRITE. The mmap_lock should already be held. */
2397 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
2399 target_ulong addr
, len
;
2401 /* This function should never be called with addresses outside the
2402 guest address space. If this assert fires, it probably indicates
2403 a missing call to h2g_valid. */
2404 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2405 assert(end
< ((abi_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2407 assert(start
< end
);
2409 start
= start
& TARGET_PAGE_MASK
;
2410 end
= TARGET_PAGE_ALIGN(end
);
2412 if (flags
& PAGE_WRITE
) {
2413 flags
|= PAGE_WRITE_ORG
;
2416 for (addr
= start
, len
= end
- start
;
2418 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2419 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2421 /* If the write protection bit is set, then we invalidate
2423 if (!(p
->flags
& PAGE_WRITE
) &&
2424 (flags
& PAGE_WRITE
) &&
2426 tb_invalidate_phys_page(addr
, 0, NULL
);
2432 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
2438 /* This function should never be called with addresses outside the
2439 guest address space. If this assert fires, it probably indicates
2440 a missing call to h2g_valid. */
2441 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2442 assert(start
< ((abi_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2448 if (start
+ len
- 1 < start
) {
2449 /* We've wrapped around. */
2453 end
= TARGET_PAGE_ALIGN(start
+len
); /* must do before we loose bits in the next step */
2454 start
= start
& TARGET_PAGE_MASK
;
2456 for (addr
= start
, len
= end
- start
;
2458 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2459 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2462 if( !(p
->flags
& PAGE_VALID
) )
2465 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
))
2467 if (flags
& PAGE_WRITE
) {
2468 if (!(p
->flags
& PAGE_WRITE_ORG
))
2470 /* unprotect the page if it was put read-only because it
2471 contains translated code */
2472 if (!(p
->flags
& PAGE_WRITE
)) {
2473 if (!page_unprotect(addr
, 0, NULL
))
2482 /* called from signal handler: invalidate the code and unprotect the
2483 page. Return TRUE if the fault was successfully handled. */
2484 int page_unprotect(target_ulong address
, unsigned long pc
, void *puc
)
2488 target_ulong host_start
, host_end
, addr
;
2490 /* Technically this isn't safe inside a signal handler. However we
2491 know this only ever happens in a synchronous SEGV handler, so in
2492 practice it seems to be ok. */
2495 p
= page_find(address
>> TARGET_PAGE_BITS
);
2501 /* if the page was really writable, then we change its
2502 protection back to writable */
2503 if ((p
->flags
& PAGE_WRITE_ORG
) && !(p
->flags
& PAGE_WRITE
)) {
2504 host_start
= address
& qemu_host_page_mask
;
2505 host_end
= host_start
+ qemu_host_page_size
;
2508 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2509 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2510 p
->flags
|= PAGE_WRITE
;
2513 /* and since the content will be modified, we must invalidate
2514 the corresponding translated code. */
2515 tb_invalidate_phys_page(addr
, pc
, puc
);
2516 #ifdef DEBUG_TB_CHECK
2517 tb_invalidate_check(addr
);
2520 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2530 static inline void tlb_set_dirty(CPUState
*env
,
2531 unsigned long addr
, target_ulong vaddr
)
2534 #endif /* defined(CONFIG_USER_ONLY) */
2536 #if !defined(CONFIG_USER_ONLY)
2538 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
2539 typedef struct subpage_t
{
2540 target_phys_addr_t base
;
2541 ram_addr_t sub_io_index
[TARGET_PAGE_SIZE
];
2542 ram_addr_t region_offset
[TARGET_PAGE_SIZE
];
2545 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2546 ram_addr_t memory
, ram_addr_t region_offset
);
2547 static subpage_t
*subpage_init (target_phys_addr_t base
, ram_addr_t
*phys
,
2548 ram_addr_t orig_memory
,
2549 ram_addr_t region_offset
);
2550 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2553 if (addr > start_addr) \
2556 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2557 if (start_addr2 > 0) \
2561 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2562 end_addr2 = TARGET_PAGE_SIZE - 1; \
2564 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2565 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2570 /* register physical memory.
2571 For RAM, 'size' must be a multiple of the target page size.
2572 If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2573 io memory page. The address used when calling the IO function is
2574 the offset from the start of the region, plus region_offset. Both
2575 start_addr and region_offset are rounded down to a page boundary
2576 before calculating this offset. This should not be a problem unless
2577 the low bits of start_addr and region_offset differ. */
2578 void cpu_register_physical_memory_offset(target_phys_addr_t start_addr
,
2580 ram_addr_t phys_offset
,
2581 ram_addr_t region_offset
)
2583 target_phys_addr_t addr
, end_addr
;
2586 ram_addr_t orig_size
= size
;
2589 cpu_notify_set_memory(start_addr
, size
, phys_offset
);
2591 if (phys_offset
== IO_MEM_UNASSIGNED
) {
2592 region_offset
= start_addr
;
2594 region_offset
&= TARGET_PAGE_MASK
;
2595 size
= (size
+ TARGET_PAGE_SIZE
- 1) & TARGET_PAGE_MASK
;
2596 end_addr
= start_addr
+ (target_phys_addr_t
)size
;
2597 for(addr
= start_addr
; addr
!= end_addr
; addr
+= TARGET_PAGE_SIZE
) {
2598 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2599 if (p
&& p
->phys_offset
!= IO_MEM_UNASSIGNED
) {
2600 ram_addr_t orig_memory
= p
->phys_offset
;
2601 target_phys_addr_t start_addr2
, end_addr2
;
2602 int need_subpage
= 0;
2604 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
, end_addr2
,
2607 if (!(orig_memory
& IO_MEM_SUBPAGE
)) {
2608 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2609 &p
->phys_offset
, orig_memory
,
2612 subpage
= io_mem_opaque
[(orig_memory
& ~TARGET_PAGE_MASK
)
2615 subpage_register(subpage
, start_addr2
, end_addr2
, phys_offset
,
2617 p
->region_offset
= 0;
2619 p
->phys_offset
= phys_offset
;
2620 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
||
2621 (phys_offset
& IO_MEM_ROMD
))
2622 phys_offset
+= TARGET_PAGE_SIZE
;
2625 p
= phys_page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2626 p
->phys_offset
= phys_offset
;
2627 p
->region_offset
= region_offset
;
2628 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
||
2629 (phys_offset
& IO_MEM_ROMD
)) {
2630 phys_offset
+= TARGET_PAGE_SIZE
;
2632 target_phys_addr_t start_addr2
, end_addr2
;
2633 int need_subpage
= 0;
2635 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
,
2636 end_addr2
, need_subpage
);
2639 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2640 &p
->phys_offset
, IO_MEM_UNASSIGNED
,
2641 addr
& TARGET_PAGE_MASK
);
2642 subpage_register(subpage
, start_addr2
, end_addr2
,
2643 phys_offset
, region_offset
);
2644 p
->region_offset
= 0;
2648 region_offset
+= TARGET_PAGE_SIZE
;
2651 /* since each CPU stores ram addresses in its TLB cache, we must
2652 reset the modified entries */
2654 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
2659 /* XXX: temporary until new memory mapping API */
2660 ram_addr_t
cpu_get_physical_page_desc(target_phys_addr_t addr
)
2664 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2666 return IO_MEM_UNASSIGNED
;
2667 return p
->phys_offset
;
2670 void qemu_register_coalesced_mmio(target_phys_addr_t addr
, ram_addr_t size
)
2673 kvm_coalesce_mmio_region(addr
, size
);
2676 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr
, ram_addr_t size
)
2679 kvm_uncoalesce_mmio_region(addr
, size
);
2682 void qemu_flush_coalesced_mmio_buffer(void)
2685 kvm_flush_coalesced_mmio_buffer();
2688 #if defined(__linux__) && !defined(TARGET_S390X)
2690 #include <sys/vfs.h>
2692 #define HUGETLBFS_MAGIC 0x958458f6
2694 static long gethugepagesize(const char *path
)
2700 ret
= statfs(path
, &fs
);
2701 } while (ret
!= 0 && errno
== EINTR
);
2708 if (fs
.f_type
!= HUGETLBFS_MAGIC
)
2709 fprintf(stderr
, "Warning: path not on HugeTLBFS: %s\n", path
);
2714 static void *file_ram_alloc(RAMBlock
*block
,
2724 unsigned long hpagesize
;
2726 hpagesize
= gethugepagesize(path
);
2731 if (memory
< hpagesize
) {
2735 if (kvm_enabled() && !kvm_has_sync_mmu()) {
2736 fprintf(stderr
, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
2740 if (asprintf(&filename
, "%s/qemu_back_mem.XXXXXX", path
) == -1) {
2744 fd
= mkstemp(filename
);
2746 perror("unable to create backing store for hugepages");
2753 memory
= (memory
+hpagesize
-1) & ~(hpagesize
-1);
2756 * ftruncate is not supported by hugetlbfs in older
2757 * hosts, so don't bother bailing out on errors.
2758 * If anything goes wrong with it under other filesystems,
2761 if (ftruncate(fd
, memory
))
2762 perror("ftruncate");
2765 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
2766 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
2767 * to sidestep this quirk.
2769 flags
= mem_prealloc
? MAP_POPULATE
| MAP_SHARED
: MAP_PRIVATE
;
2770 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
, flags
, fd
, 0);
2772 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
, MAP_PRIVATE
, fd
, 0);
2774 if (area
== MAP_FAILED
) {
2775 perror("file_ram_alloc: can't mmap RAM pages");
2784 static ram_addr_t
find_ram_offset(ram_addr_t size
)
2786 RAMBlock
*block
, *next_block
;
2787 ram_addr_t offset
= 0, mingap
= ULONG_MAX
;
2789 if (QLIST_EMPTY(&ram_list
.blocks
))
2792 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2793 ram_addr_t end
, next
= ULONG_MAX
;
2795 end
= block
->offset
+ block
->length
;
2797 QLIST_FOREACH(next_block
, &ram_list
.blocks
, next
) {
2798 if (next_block
->offset
>= end
) {
2799 next
= MIN(next
, next_block
->offset
);
2802 if (next
- end
>= size
&& next
- end
< mingap
) {
2804 mingap
= next
- end
;
2810 static ram_addr_t
last_ram_offset(void)
2813 ram_addr_t last
= 0;
2815 QLIST_FOREACH(block
, &ram_list
.blocks
, next
)
2816 last
= MAX(last
, block
->offset
+ block
->length
);
2821 ram_addr_t
qemu_ram_alloc_from_ptr(DeviceState
*dev
, const char *name
,
2822 ram_addr_t size
, void *host
)
2824 RAMBlock
*new_block
, *block
;
2826 size
= TARGET_PAGE_ALIGN(size
);
2827 new_block
= qemu_mallocz(sizeof(*new_block
));
2829 if (dev
&& dev
->parent_bus
&& dev
->parent_bus
->info
->get_dev_path
) {
2830 char *id
= dev
->parent_bus
->info
->get_dev_path(dev
);
2832 snprintf(new_block
->idstr
, sizeof(new_block
->idstr
), "%s/", id
);
2836 pstrcat(new_block
->idstr
, sizeof(new_block
->idstr
), name
);
2838 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2839 if (!strcmp(block
->idstr
, new_block
->idstr
)) {
2840 fprintf(stderr
, "RAMBlock \"%s\" already registered, abort!\n",
2847 new_block
->host
= host
;
2850 #if defined (__linux__) && !defined(TARGET_S390X)
2851 new_block
->host
= file_ram_alloc(new_block
, size
, mem_path
);
2852 if (!new_block
->host
) {
2853 new_block
->host
= qemu_vmalloc(size
);
2854 qemu_madvise(new_block
->host
, size
, QEMU_MADV_MERGEABLE
);
2857 fprintf(stderr
, "-mem-path option unsupported\n");
2861 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2862 /* XXX S390 KVM requires the topmost vma of the RAM to be < 256GB */
2863 new_block
->host
= mmap((void*)0x1000000, size
,
2864 PROT_EXEC
|PROT_READ
|PROT_WRITE
,
2865 MAP_SHARED
| MAP_ANONYMOUS
, -1, 0);
2867 new_block
->host
= qemu_vmalloc(size
);
2869 qemu_madvise(new_block
->host
, size
, QEMU_MADV_MERGEABLE
);
2873 new_block
->offset
= find_ram_offset(size
);
2874 new_block
->length
= size
;
2876 QLIST_INSERT_HEAD(&ram_list
.blocks
, new_block
, next
);
2878 ram_list
.phys_dirty
= qemu_realloc(ram_list
.phys_dirty
,
2879 last_ram_offset() >> TARGET_PAGE_BITS
);
2880 memset(ram_list
.phys_dirty
+ (new_block
->offset
>> TARGET_PAGE_BITS
),
2881 0xff, size
>> TARGET_PAGE_BITS
);
2884 kvm_setup_guest_memory(new_block
->host
, size
);
2886 return new_block
->offset
;
2889 void qemu_ram_unmap(ram_addr_t addr
)
2893 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2894 if (addr
== block
->offset
) {
2895 QLIST_REMOVE(block
, next
);
2902 ram_addr_t
qemu_ram_alloc(DeviceState
*dev
, const char *name
, ram_addr_t size
)
2904 return qemu_ram_alloc_from_ptr(dev
, name
, size
, NULL
);
2907 void qemu_ram_free(ram_addr_t addr
)
2911 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2912 if (addr
== block
->offset
) {
2913 QLIST_REMOVE(block
, next
);
2915 #if defined (__linux__) && !defined(TARGET_S390X)
2917 munmap(block
->host
, block
->length
);
2920 qemu_vfree(block
->host
);
2924 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2925 munmap(block
->host
, block
->length
);
2927 qemu_vfree(block
->host
);
2937 /* Return a host pointer to ram allocated with qemu_ram_alloc.
2938 With the exception of the softmmu code in this file, this should
2939 only be used for local memory (e.g. video ram) that the device owns,
2940 and knows it isn't going to access beyond the end of the block.
2942 It should not be used for general purpose DMA.
2943 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
2945 void *qemu_get_ram_ptr(ram_addr_t addr
)
2949 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2950 if (addr
- block
->offset
< block
->length
) {
2951 QLIST_REMOVE(block
, next
);
2952 QLIST_INSERT_HEAD(&ram_list
.blocks
, block
, next
);
2953 return block
->host
+ (addr
- block
->offset
);
2957 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
2963 int qemu_ram_addr_from_host(void *ptr
, ram_addr_t
*ram_addr
)
2966 uint8_t *host
= ptr
;
2968 QLIST_FOREACH(block
, &ram_list
.blocks
, next
) {
2969 if (host
- block
->host
< block
->length
) {
2970 *ram_addr
= block
->offset
+ (host
- block
->host
);
2977 /* Some of the softmmu routines need to translate from a host pointer
2978 (typically a TLB entry) back to a ram offset. */
2979 ram_addr_t
qemu_ram_addr_from_host_nofail(void *ptr
)
2981 ram_addr_t ram_addr
;
2983 if (qemu_ram_addr_from_host(ptr
, &ram_addr
)) {
2984 fprintf(stderr
, "Bad ram pointer %p\n", ptr
);
2990 static uint32_t unassigned_mem_readb(void *opaque
, target_phys_addr_t addr
)
2992 #ifdef DEBUG_UNASSIGNED
2993 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2995 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2996 do_unassigned_access(addr
, 0, 0, 0, 1);
3001 static uint32_t unassigned_mem_readw(void *opaque
, target_phys_addr_t addr
)
3003 #ifdef DEBUG_UNASSIGNED
3004 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
3006 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3007 do_unassigned_access(addr
, 0, 0, 0, 2);
3012 static uint32_t unassigned_mem_readl(void *opaque
, target_phys_addr_t addr
)
3014 #ifdef DEBUG_UNASSIGNED
3015 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
3017 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3018 do_unassigned_access(addr
, 0, 0, 0, 4);
3023 static void unassigned_mem_writeb(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
3025 #ifdef DEBUG_UNASSIGNED
3026 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
3028 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3029 do_unassigned_access(addr
, 1, 0, 0, 1);
3033 static void unassigned_mem_writew(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
3035 #ifdef DEBUG_UNASSIGNED
3036 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
3038 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3039 do_unassigned_access(addr
, 1, 0, 0, 2);
3043 static void unassigned_mem_writel(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
3045 #ifdef DEBUG_UNASSIGNED
3046 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
3048 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3049 do_unassigned_access(addr
, 1, 0, 0, 4);
3053 static CPUReadMemoryFunc
* const unassigned_mem_read
[3] = {
3054 unassigned_mem_readb
,
3055 unassigned_mem_readw
,
3056 unassigned_mem_readl
,
3059 static CPUWriteMemoryFunc
* const unassigned_mem_write
[3] = {
3060 unassigned_mem_writeb
,
3061 unassigned_mem_writew
,
3062 unassigned_mem_writel
,
3065 static void notdirty_mem_writeb(void *opaque
, target_phys_addr_t ram_addr
,
3069 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
3070 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
3071 #if !defined(CONFIG_USER_ONLY)
3072 tb_invalidate_phys_page_fast(ram_addr
, 1);
3073 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
3076 stb_p(qemu_get_ram_ptr(ram_addr
), val
);
3077 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
3078 cpu_physical_memory_set_dirty_flags(ram_addr
, dirty_flags
);
3079 /* we remove the notdirty callback only if the code has been
3081 if (dirty_flags
== 0xff)
3082 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
3085 static void notdirty_mem_writew(void *opaque
, target_phys_addr_t ram_addr
,
3089 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
3090 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
3091 #if !defined(CONFIG_USER_ONLY)
3092 tb_invalidate_phys_page_fast(ram_addr
, 2);
3093 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
3096 stw_p(qemu_get_ram_ptr(ram_addr
), val
);
3097 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
3098 cpu_physical_memory_set_dirty_flags(ram_addr
, dirty_flags
);
3099 /* we remove the notdirty callback only if the code has been
3101 if (dirty_flags
== 0xff)
3102 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
3105 static void notdirty_mem_writel(void *opaque
, target_phys_addr_t ram_addr
,
3109 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
3110 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
3111 #if !defined(CONFIG_USER_ONLY)
3112 tb_invalidate_phys_page_fast(ram_addr
, 4);
3113 dirty_flags
= cpu_physical_memory_get_dirty_flags(ram_addr
);
3116 stl_p(qemu_get_ram_ptr(ram_addr
), val
);
3117 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
3118 cpu_physical_memory_set_dirty_flags(ram_addr
, dirty_flags
);
3119 /* we remove the notdirty callback only if the code has been
3121 if (dirty_flags
== 0xff)
3122 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
3125 static CPUReadMemoryFunc
* const error_mem_read
[3] = {
3126 NULL
, /* never used */
3127 NULL
, /* never used */
3128 NULL
, /* never used */
3131 static CPUWriteMemoryFunc
* const notdirty_mem_write
[3] = {
3132 notdirty_mem_writeb
,
3133 notdirty_mem_writew
,
3134 notdirty_mem_writel
,
3137 /* Generate a debug exception if a watchpoint has been hit. */
3138 static void check_watchpoint(int offset
, int len_mask
, int flags
)
3140 CPUState
*env
= cpu_single_env
;
3141 target_ulong pc
, cs_base
;
3142 TranslationBlock
*tb
;
3147 if (env
->watchpoint_hit
) {
3148 /* We re-entered the check after replacing the TB. Now raise
3149 * the debug interrupt so that is will trigger after the
3150 * current instruction. */
3151 cpu_interrupt(env
, CPU_INTERRUPT_DEBUG
);
3154 vaddr
= (env
->mem_io_vaddr
& TARGET_PAGE_MASK
) + offset
;
3155 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
3156 if ((vaddr
== (wp
->vaddr
& len_mask
) ||
3157 (vaddr
& wp
->len_mask
) == wp
->vaddr
) && (wp
->flags
& flags
)) {
3158 wp
->flags
|= BP_WATCHPOINT_HIT
;
3159 if (!env
->watchpoint_hit
) {
3160 env
->watchpoint_hit
= wp
;
3161 tb
= tb_find_pc(env
->mem_io_pc
);
3163 cpu_abort(env
, "check_watchpoint: could not find TB for "
3164 "pc=%p", (void *)env
->mem_io_pc
);
3166 cpu_restore_state(tb
, env
, env
->mem_io_pc
, NULL
);
3167 tb_phys_invalidate(tb
, -1);
3168 if (wp
->flags
& BP_STOP_BEFORE_ACCESS
) {
3169 env
->exception_index
= EXCP_DEBUG
;
3171 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &cpu_flags
);
3172 tb_gen_code(env
, pc
, cs_base
, cpu_flags
, 1);
3174 cpu_resume_from_signal(env
, NULL
);
3177 wp
->flags
&= ~BP_WATCHPOINT_HIT
;
3182 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
3183 so these check for a hit then pass through to the normal out-of-line
3185 static uint32_t watch_mem_readb(void *opaque
, target_phys_addr_t addr
)
3187 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x0, BP_MEM_READ
);
3188 return ldub_phys(addr
);
3191 static uint32_t watch_mem_readw(void *opaque
, target_phys_addr_t addr
)
3193 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x1, BP_MEM_READ
);
3194 return lduw_phys(addr
);
3197 static uint32_t watch_mem_readl(void *opaque
, target_phys_addr_t addr
)
3199 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x3, BP_MEM_READ
);
3200 return ldl_phys(addr
);
3203 static void watch_mem_writeb(void *opaque
, target_phys_addr_t addr
,
3206 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x0, BP_MEM_WRITE
);
3207 stb_phys(addr
, val
);
3210 static void watch_mem_writew(void *opaque
, target_phys_addr_t addr
,
3213 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x1, BP_MEM_WRITE
);
3214 stw_phys(addr
, val
);
3217 static void watch_mem_writel(void *opaque
, target_phys_addr_t addr
,
3220 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x3, BP_MEM_WRITE
);
3221 stl_phys(addr
, val
);
3224 static CPUReadMemoryFunc
* const watch_mem_read
[3] = {
3230 static CPUWriteMemoryFunc
* const watch_mem_write
[3] = {
3236 static inline uint32_t subpage_readlen (subpage_t
*mmio
,
3237 target_phys_addr_t addr
,
3240 unsigned int idx
= SUBPAGE_IDX(addr
);
3241 #if defined(DEBUG_SUBPAGE)
3242 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d\n", __func__
,
3243 mmio
, len
, addr
, idx
);
3246 addr
+= mmio
->region_offset
[idx
];
3247 idx
= mmio
->sub_io_index
[idx
];
3248 return io_mem_read
[idx
][len
](io_mem_opaque
[idx
], addr
);
3251 static inline void subpage_writelen (subpage_t
*mmio
, target_phys_addr_t addr
,
3252 uint32_t value
, unsigned int len
)
3254 unsigned int idx
= SUBPAGE_IDX(addr
);
3255 #if defined(DEBUG_SUBPAGE)
3256 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d value %08x\n",
3257 __func__
, mmio
, len
, addr
, idx
, value
);
3260 addr
+= mmio
->region_offset
[idx
];
3261 idx
= mmio
->sub_io_index
[idx
];
3262 io_mem_write
[idx
][len
](io_mem_opaque
[idx
], addr
, value
);
3265 static uint32_t subpage_readb (void *opaque
, target_phys_addr_t addr
)
3267 return subpage_readlen(opaque
, addr
, 0);
3270 static void subpage_writeb (void *opaque
, target_phys_addr_t addr
,
3273 subpage_writelen(opaque
, addr
, value
, 0);
3276 static uint32_t subpage_readw (void *opaque
, target_phys_addr_t addr
)
3278 return subpage_readlen(opaque
, addr
, 1);
3281 static void subpage_writew (void *opaque
, target_phys_addr_t addr
,
3284 subpage_writelen(opaque
, addr
, value
, 1);
3287 static uint32_t subpage_readl (void *opaque
, target_phys_addr_t addr
)
3289 return subpage_readlen(opaque
, addr
, 2);
3292 static void subpage_writel (void *opaque
, target_phys_addr_t addr
,
3295 subpage_writelen(opaque
, addr
, value
, 2);
3298 static CPUReadMemoryFunc
* const subpage_read
[] = {
3304 static CPUWriteMemoryFunc
* const subpage_write
[] = {
3310 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
3311 ram_addr_t memory
, ram_addr_t region_offset
)
3315 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
3317 idx
= SUBPAGE_IDX(start
);
3318 eidx
= SUBPAGE_IDX(end
);
3319 #if defined(DEBUG_SUBPAGE)
3320 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__
,
3321 mmio
, start
, end
, idx
, eidx
, memory
);
3323 if ((memory
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
)
3324 memory
= IO_MEM_UNASSIGNED
;
3325 memory
= (memory
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3326 for (; idx
<= eidx
; idx
++) {
3327 mmio
->sub_io_index
[idx
] = memory
;
3328 mmio
->region_offset
[idx
] = region_offset
;
3334 static subpage_t
*subpage_init (target_phys_addr_t base
, ram_addr_t
*phys
,
3335 ram_addr_t orig_memory
,
3336 ram_addr_t region_offset
)
3341 mmio
= qemu_mallocz(sizeof(subpage_t
));
3344 subpage_memory
= cpu_register_io_memory(subpage_read
, subpage_write
, mmio
);
3345 #if defined(DEBUG_SUBPAGE)
3346 printf("%s: %p base " TARGET_FMT_plx
" len %08x %d\n", __func__
,
3347 mmio
, base
, TARGET_PAGE_SIZE
, subpage_memory
);
3349 *phys
= subpage_memory
| IO_MEM_SUBPAGE
;
3350 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
-1, orig_memory
, region_offset
);
3355 static int get_free_io_mem_idx(void)
3359 for (i
= 0; i
<IO_MEM_NB_ENTRIES
; i
++)
3360 if (!io_mem_used
[i
]) {
3364 fprintf(stderr
, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES
);
3368 /* mem_read and mem_write are arrays of functions containing the
3369 function to access byte (index 0), word (index 1) and dword (index
3370 2). Functions can be omitted with a NULL function pointer.
3371 If io_index is non zero, the corresponding io zone is
3372 modified. If it is zero, a new io zone is allocated. The return
3373 value can be used with cpu_register_physical_memory(). (-1) is
3374 returned if error. */
3375 static int cpu_register_io_memory_fixed(int io_index
,
3376 CPUReadMemoryFunc
* const *mem_read
,
3377 CPUWriteMemoryFunc
* const *mem_write
,
3382 if (io_index
<= 0) {
3383 io_index
= get_free_io_mem_idx();
3387 io_index
>>= IO_MEM_SHIFT
;
3388 if (io_index
>= IO_MEM_NB_ENTRIES
)
3392 for (i
= 0; i
< 3; ++i
) {
3393 io_mem_read
[io_index
][i
]
3394 = (mem_read
[i
] ? mem_read
[i
] : unassigned_mem_read
[i
]);
3396 for (i
= 0; i
< 3; ++i
) {
3397 io_mem_write
[io_index
][i
]
3398 = (mem_write
[i
] ? mem_write
[i
] : unassigned_mem_write
[i
]);
3400 io_mem_opaque
[io_index
] = opaque
;
3402 return (io_index
<< IO_MEM_SHIFT
);
3405 int cpu_register_io_memory(CPUReadMemoryFunc
* const *mem_read
,
3406 CPUWriteMemoryFunc
* const *mem_write
,
3409 return cpu_register_io_memory_fixed(0, mem_read
, mem_write
, opaque
);
3412 void cpu_unregister_io_memory(int io_table_address
)
3415 int io_index
= io_table_address
>> IO_MEM_SHIFT
;
3417 for (i
=0;i
< 3; i
++) {
3418 io_mem_read
[io_index
][i
] = unassigned_mem_read
[i
];
3419 io_mem_write
[io_index
][i
] = unassigned_mem_write
[i
];
3421 io_mem_opaque
[io_index
] = NULL
;
3422 io_mem_used
[io_index
] = 0;
3425 static void io_mem_init(void)
3429 cpu_register_io_memory_fixed(IO_MEM_ROM
, error_mem_read
, unassigned_mem_write
, NULL
);
3430 cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED
, unassigned_mem_read
, unassigned_mem_write
, NULL
);
3431 cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY
, error_mem_read
, notdirty_mem_write
, NULL
);
3435 io_mem_watch
= cpu_register_io_memory(watch_mem_read
,
3436 watch_mem_write
, NULL
);
3439 #endif /* !defined(CONFIG_USER_ONLY) */
3441 /* physical memory access (slow version, mainly for debug) */
3442 #if defined(CONFIG_USER_ONLY)
3443 int cpu_memory_rw_debug(CPUState
*env
, target_ulong addr
,
3444 uint8_t *buf
, int len
, int is_write
)
3451 page
= addr
& TARGET_PAGE_MASK
;
3452 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3455 flags
= page_get_flags(page
);
3456 if (!(flags
& PAGE_VALID
))
3459 if (!(flags
& PAGE_WRITE
))
3461 /* XXX: this code should not depend on lock_user */
3462 if (!(p
= lock_user(VERIFY_WRITE
, addr
, l
, 0)))
3465 unlock_user(p
, addr
, l
);
3467 if (!(flags
& PAGE_READ
))
3469 /* XXX: this code should not depend on lock_user */
3470 if (!(p
= lock_user(VERIFY_READ
, addr
, l
, 1)))
3473 unlock_user(p
, addr
, 0);
3483 void cpu_physical_memory_rw(target_phys_addr_t addr
, uint8_t *buf
,
3484 int len
, int is_write
)
3489 target_phys_addr_t page
;
3494 page
= addr
& TARGET_PAGE_MASK
;
3495 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3498 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3500 pd
= IO_MEM_UNASSIGNED
;
3502 pd
= p
->phys_offset
;
3506 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3507 target_phys_addr_t addr1
= addr
;
3508 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3510 addr1
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3511 /* XXX: could force cpu_single_env to NULL to avoid
3513 if (l
>= 4 && ((addr1
& 3) == 0)) {
3514 /* 32 bit write access */
3516 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr1
, val
);
3518 } else if (l
>= 2 && ((addr1
& 1) == 0)) {
3519 /* 16 bit write access */
3521 io_mem_write
[io_index
][1](io_mem_opaque
[io_index
], addr1
, val
);
3524 /* 8 bit write access */
3526 io_mem_write
[io_index
][0](io_mem_opaque
[io_index
], addr1
, val
);
3530 unsigned long addr1
;
3531 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3533 ptr
= qemu_get_ram_ptr(addr1
);
3534 memcpy(ptr
, buf
, l
);
3535 if (!cpu_physical_memory_is_dirty(addr1
)) {
3536 /* invalidate code */
3537 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
3539 cpu_physical_memory_set_dirty_flags(
3540 addr1
, (0xff & ~CODE_DIRTY_FLAG
));
3542 /* qemu doesn't execute guest code directly, but kvm does
3543 therefore flush instruction caches */
3545 flush_icache_range((unsigned long)ptr
,
3546 ((unsigned long)ptr
)+l
);
3549 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3550 !(pd
& IO_MEM_ROMD
)) {
3551 target_phys_addr_t addr1
= addr
;
3553 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3555 addr1
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3556 if (l
>= 4 && ((addr1
& 3) == 0)) {
3557 /* 32 bit read access */
3558 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr1
);
3561 } else if (l
>= 2 && ((addr1
& 1) == 0)) {
3562 /* 16 bit read access */
3563 val
= io_mem_read
[io_index
][1](io_mem_opaque
[io_index
], addr1
);
3567 /* 8 bit read access */
3568 val
= io_mem_read
[io_index
][0](io_mem_opaque
[io_index
], addr1
);
3574 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3575 (addr
& ~TARGET_PAGE_MASK
);
3576 memcpy(buf
, ptr
, l
);
3585 /* used for ROM loading : can write in RAM and ROM */
3586 void cpu_physical_memory_write_rom(target_phys_addr_t addr
,
3587 const uint8_t *buf
, int len
)
3591 target_phys_addr_t page
;
3596 page
= addr
& TARGET_PAGE_MASK
;
3597 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3600 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3602 pd
= IO_MEM_UNASSIGNED
;
3604 pd
= p
->phys_offset
;
3607 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
&&
3608 (pd
& ~TARGET_PAGE_MASK
) != IO_MEM_ROM
&&
3609 !(pd
& IO_MEM_ROMD
)) {
3612 unsigned long addr1
;
3613 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3615 ptr
= qemu_get_ram_ptr(addr1
);
3616 memcpy(ptr
, buf
, l
);
3626 target_phys_addr_t addr
;
3627 target_phys_addr_t len
;
3630 static BounceBuffer bounce
;
3632 typedef struct MapClient
{
3634 void (*callback
)(void *opaque
);
3635 QLIST_ENTRY(MapClient
) link
;
3638 static QLIST_HEAD(map_client_list
, MapClient
) map_client_list
3639 = QLIST_HEAD_INITIALIZER(map_client_list
);
3641 void *cpu_register_map_client(void *opaque
, void (*callback
)(void *opaque
))
3643 MapClient
*client
= qemu_malloc(sizeof(*client
));
3645 client
->opaque
= opaque
;
3646 client
->callback
= callback
;
3647 QLIST_INSERT_HEAD(&map_client_list
, client
, link
);
3651 void cpu_unregister_map_client(void *_client
)
3653 MapClient
*client
= (MapClient
*)_client
;
3655 QLIST_REMOVE(client
, link
);
3659 static void cpu_notify_map_clients(void)
3663 while (!QLIST_EMPTY(&map_client_list
)) {
3664 client
= QLIST_FIRST(&map_client_list
);
3665 client
->callback(client
->opaque
);
3666 cpu_unregister_map_client(client
);
3670 /* Map a physical memory region into a host virtual address.
3671 * May map a subset of the requested range, given by and returned in *plen.
3672 * May return NULL if resources needed to perform the mapping are exhausted.
3673 * Use only for reads OR writes - not for read-modify-write operations.
3674 * Use cpu_register_map_client() to know when retrying the map operation is
3675 * likely to succeed.
3677 void *cpu_physical_memory_map(target_phys_addr_t addr
,
3678 target_phys_addr_t
*plen
,
3681 target_phys_addr_t len
= *plen
;
3682 target_phys_addr_t done
= 0;
3684 uint8_t *ret
= NULL
;
3686 target_phys_addr_t page
;
3689 unsigned long addr1
;
3692 page
= addr
& TARGET_PAGE_MASK
;
3693 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3696 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3698 pd
= IO_MEM_UNASSIGNED
;
3700 pd
= p
->phys_offset
;
3703 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3704 if (done
|| bounce
.buffer
) {
3707 bounce
.buffer
= qemu_memalign(TARGET_PAGE_SIZE
, TARGET_PAGE_SIZE
);
3711 cpu_physical_memory_rw(addr
, bounce
.buffer
, l
, 0);
3713 ptr
= bounce
.buffer
;
3715 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3716 ptr
= qemu_get_ram_ptr(addr1
);
3720 } else if (ret
+ done
!= ptr
) {
3732 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3733 * Will also mark the memory as dirty if is_write == 1. access_len gives
3734 * the amount of memory that was actually read or written by the caller.
3736 void cpu_physical_memory_unmap(void *buffer
, target_phys_addr_t len
,
3737 int is_write
, target_phys_addr_t access_len
)
3739 unsigned long flush_len
= (unsigned long)access_len
;
3741 if (buffer
!= bounce
.buffer
) {
3743 ram_addr_t addr1
= qemu_ram_addr_from_host_nofail(buffer
);
3744 while (access_len
) {
3746 l
= TARGET_PAGE_SIZE
;
3749 if (!cpu_physical_memory_is_dirty(addr1
)) {
3750 /* invalidate code */
3751 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
3753 cpu_physical_memory_set_dirty_flags(
3754 addr1
, (0xff & ~CODE_DIRTY_FLAG
));
3759 dma_flush_range((unsigned long)buffer
,
3760 (unsigned long)buffer
+ flush_len
);
3765 cpu_physical_memory_write(bounce
.addr
, bounce
.buffer
, access_len
);
3767 qemu_vfree(bounce
.buffer
);
3768 bounce
.buffer
= NULL
;
3769 cpu_notify_map_clients();
3772 /* warning: addr must be aligned */
3773 uint32_t ldl_phys(target_phys_addr_t addr
)
3781 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3783 pd
= IO_MEM_UNASSIGNED
;
3785 pd
= p
->phys_offset
;
3788 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3789 !(pd
& IO_MEM_ROMD
)) {
3791 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3793 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3794 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
3797 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3798 (addr
& ~TARGET_PAGE_MASK
);
3804 /* warning: addr must be aligned */
3805 uint64_t ldq_phys(target_phys_addr_t addr
)
3813 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3815 pd
= IO_MEM_UNASSIGNED
;
3817 pd
= p
->phys_offset
;
3820 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3821 !(pd
& IO_MEM_ROMD
)) {
3823 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3825 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3826 #ifdef TARGET_WORDS_BIGENDIAN
3827 val
= (uint64_t)io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
) << 32;
3828 val
|= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4);
3830 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
3831 val
|= (uint64_t)io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4) << 32;
3835 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3836 (addr
& ~TARGET_PAGE_MASK
);
3843 uint32_t ldub_phys(target_phys_addr_t addr
)
3846 cpu_physical_memory_read(addr
, &val
, 1);
3850 /* warning: addr must be aligned */
3851 uint32_t lduw_phys(target_phys_addr_t addr
)
3859 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3861 pd
= IO_MEM_UNASSIGNED
;
3863 pd
= p
->phys_offset
;
3866 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3867 !(pd
& IO_MEM_ROMD
)) {
3869 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3871 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3872 val
= io_mem_read
[io_index
][1](io_mem_opaque
[io_index
], addr
);
3875 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3876 (addr
& ~TARGET_PAGE_MASK
);
3882 /* warning: addr must be aligned. The ram page is not masked as dirty
3883 and the code inside is not invalidated. It is useful if the dirty
3884 bits are used to track modified PTEs */
3885 void stl_phys_notdirty(target_phys_addr_t addr
, uint32_t val
)
3892 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3894 pd
= IO_MEM_UNASSIGNED
;
3896 pd
= p
->phys_offset
;
3899 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3900 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3902 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3903 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3905 unsigned long addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3906 ptr
= qemu_get_ram_ptr(addr1
);
3909 if (unlikely(in_migration
)) {
3910 if (!cpu_physical_memory_is_dirty(addr1
)) {
3911 /* invalidate code */
3912 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
3914 cpu_physical_memory_set_dirty_flags(
3915 addr1
, (0xff & ~CODE_DIRTY_FLAG
));
3921 void stq_phys_notdirty(target_phys_addr_t addr
, uint64_t val
)
3928 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3930 pd
= IO_MEM_UNASSIGNED
;
3932 pd
= p
->phys_offset
;
3935 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3936 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3938 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3939 #ifdef TARGET_WORDS_BIGENDIAN
3940 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
>> 32);
3941 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4, val
);
3943 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3944 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4, val
>> 32);
3947 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3948 (addr
& ~TARGET_PAGE_MASK
);
3953 /* warning: addr must be aligned */
3954 void stl_phys(target_phys_addr_t addr
, uint32_t val
)
3961 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3963 pd
= IO_MEM_UNASSIGNED
;
3965 pd
= p
->phys_offset
;
3968 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3969 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3971 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3972 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3974 unsigned long addr1
;
3975 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3977 ptr
= qemu_get_ram_ptr(addr1
);
3979 if (!cpu_physical_memory_is_dirty(addr1
)) {
3980 /* invalidate code */
3981 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
3983 cpu_physical_memory_set_dirty_flags(addr1
,
3984 (0xff & ~CODE_DIRTY_FLAG
));
3990 void stb_phys(target_phys_addr_t addr
, uint32_t val
)
3993 cpu_physical_memory_write(addr
, &v
, 1);
3996 /* warning: addr must be aligned */
3997 void stw_phys(target_phys_addr_t addr
, uint32_t val
)
4004 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
4006 pd
= IO_MEM_UNASSIGNED
;
4008 pd
= p
->phys_offset
;
4011 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
4012 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
4014 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
4015 io_mem_write
[io_index
][1](io_mem_opaque
[io_index
], addr
, val
);
4017 unsigned long addr1
;
4018 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
4020 ptr
= qemu_get_ram_ptr(addr1
);
4022 if (!cpu_physical_memory_is_dirty(addr1
)) {
4023 /* invalidate code */
4024 tb_invalidate_phys_page_range(addr1
, addr1
+ 2, 0);
4026 cpu_physical_memory_set_dirty_flags(addr1
,
4027 (0xff & ~CODE_DIRTY_FLAG
));
4033 void stq_phys(target_phys_addr_t addr
, uint64_t val
)
4036 cpu_physical_memory_write(addr
, (const uint8_t *)&val
, 8);
4039 /* virtual memory access for debug (includes writing to ROM) */
4040 int cpu_memory_rw_debug(CPUState
*env
, target_ulong addr
,
4041 uint8_t *buf
, int len
, int is_write
)
4044 target_phys_addr_t phys_addr
;
4048 page
= addr
& TARGET_PAGE_MASK
;
4049 phys_addr
= cpu_get_phys_page_debug(env
, page
);
4050 /* if no physical page mapped, return an error */
4051 if (phys_addr
== -1)
4053 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
4056 phys_addr
+= (addr
& ~TARGET_PAGE_MASK
);
4058 cpu_physical_memory_write_rom(phys_addr
, buf
, l
);
4060 cpu_physical_memory_rw(phys_addr
, buf
, l
, is_write
);
4069 /* in deterministic execution mode, instructions doing device I/Os
4070 must be at the end of the TB */
4071 void cpu_io_recompile(CPUState
*env
, void *retaddr
)
4073 TranslationBlock
*tb
;
4075 target_ulong pc
, cs_base
;
4078 tb
= tb_find_pc((unsigned long)retaddr
);
4080 cpu_abort(env
, "cpu_io_recompile: could not find TB for pc=%p",
4083 n
= env
->icount_decr
.u16
.low
+ tb
->icount
;
4084 cpu_restore_state(tb
, env
, (unsigned long)retaddr
, NULL
);
4085 /* Calculate how many instructions had been executed before the fault
4087 n
= n
- env
->icount_decr
.u16
.low
;
4088 /* Generate a new TB ending on the I/O insn. */
4090 /* On MIPS and SH, delay slot instructions can only be restarted if
4091 they were already the first instruction in the TB. If this is not
4092 the first instruction in a TB then re-execute the preceding
4094 #if defined(TARGET_MIPS)
4095 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
4096 env
->active_tc
.PC
-= 4;
4097 env
->icount_decr
.u16
.low
++;
4098 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
4100 #elif defined(TARGET_SH4)
4101 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
4104 env
->icount_decr
.u16
.low
++;
4105 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
4108 /* This should never happen. */
4109 if (n
> CF_COUNT_MASK
)
4110 cpu_abort(env
, "TB too big during recompile");
4112 cflags
= n
| CF_LAST_IO
;
4114 cs_base
= tb
->cs_base
;
4116 tb_phys_invalidate(tb
, -1);
4117 /* FIXME: In theory this could raise an exception. In practice
4118 we have already translated the block once so it's probably ok. */
4119 tb_gen_code(env
, pc
, cs_base
, flags
, cflags
);
4120 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
4121 the first in the TB) then we end up generating a whole new TB and
4122 repeating the fault, which is horribly inefficient.
4123 Better would be to execute just this insn uncached, or generate a
4125 cpu_resume_from_signal(env
, NULL
);
4128 #if !defined(CONFIG_USER_ONLY)
4130 void dump_exec_info(FILE *f
, fprintf_function cpu_fprintf
)
4132 int i
, target_code_size
, max_target_code_size
;
4133 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
4134 TranslationBlock
*tb
;
4136 target_code_size
= 0;
4137 max_target_code_size
= 0;
4139 direct_jmp_count
= 0;
4140 direct_jmp2_count
= 0;
4141 for(i
= 0; i
< nb_tbs
; i
++) {
4143 target_code_size
+= tb
->size
;
4144 if (tb
->size
> max_target_code_size
)
4145 max_target_code_size
= tb
->size
;
4146 if (tb
->page_addr
[1] != -1)
4148 if (tb
->tb_next_offset
[0] != 0xffff) {
4150 if (tb
->tb_next_offset
[1] != 0xffff) {
4151 direct_jmp2_count
++;
4155 /* XXX: avoid using doubles ? */
4156 cpu_fprintf(f
, "Translation buffer state:\n");
4157 cpu_fprintf(f
, "gen code size %td/%ld\n",
4158 code_gen_ptr
- code_gen_buffer
, code_gen_buffer_max_size
);
4159 cpu_fprintf(f
, "TB count %d/%d\n",
4160 nb_tbs
, code_gen_max_blocks
);
4161 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
4162 nb_tbs
? target_code_size
/ nb_tbs
: 0,
4163 max_target_code_size
);
4164 cpu_fprintf(f
, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
4165 nb_tbs
? (code_gen_ptr
- code_gen_buffer
) / nb_tbs
: 0,
4166 target_code_size
? (double) (code_gen_ptr
- code_gen_buffer
) / target_code_size
: 0);
4167 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n",
4169 nb_tbs
? (cross_page
* 100) / nb_tbs
: 0);
4170 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
4172 nb_tbs
? (direct_jmp_count
* 100) / nb_tbs
: 0,
4174 nb_tbs
? (direct_jmp2_count
* 100) / nb_tbs
: 0);
4175 cpu_fprintf(f
, "\nStatistics:\n");
4176 cpu_fprintf(f
, "TB flush count %d\n", tb_flush_count
);
4177 cpu_fprintf(f
, "TB invalidate count %d\n", tb_phys_invalidate_count
);
4178 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
4179 #ifdef CONFIG_PROFILER
4180 tcg_dump_info(f
, cpu_fprintf
);
4184 #define MMUSUFFIX _cmmu
4185 #define GETPC() NULL
4186 #define env cpu_single_env
4187 #define SOFTMMU_CODE_ACCESS
4190 #include "softmmu_template.h"
4193 #include "softmmu_template.h"
4196 #include "softmmu_template.h"
4199 #include "softmmu_template.h"