2 * virtual page mapping and translated block handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
23 #include <sys/types.h>
36 #include "qemu-common.h"
41 #if defined(CONFIG_USER_ONLY)
45 //#define DEBUG_TB_INVALIDATE
48 //#define DEBUG_UNASSIGNED
50 /* make various TB consistency checks */
51 //#define DEBUG_TB_CHECK
52 //#define DEBUG_TLB_CHECK
54 //#define DEBUG_IOPORT
55 //#define DEBUG_SUBPAGE
57 #if !defined(CONFIG_USER_ONLY)
58 /* TB consistency checks only implemented for usermode emulation. */
62 #define SMC_BITMAP_USE_THRESHOLD 10
64 #if defined(TARGET_SPARC64)
65 #define TARGET_PHYS_ADDR_SPACE_BITS 41
66 #elif defined(TARGET_SPARC)
67 #define TARGET_PHYS_ADDR_SPACE_BITS 36
68 #elif defined(TARGET_ALPHA)
69 #define TARGET_PHYS_ADDR_SPACE_BITS 42
70 #define TARGET_VIRT_ADDR_SPACE_BITS 42
71 #elif defined(TARGET_PPC64)
72 #define TARGET_PHYS_ADDR_SPACE_BITS 42
73 #elif defined(TARGET_X86_64)
74 #define TARGET_PHYS_ADDR_SPACE_BITS 42
75 #elif defined(TARGET_I386)
76 #define TARGET_PHYS_ADDR_SPACE_BITS 36
78 #define TARGET_PHYS_ADDR_SPACE_BITS 32
81 static TranslationBlock
*tbs
;
82 int code_gen_max_blocks
;
83 TranslationBlock
*tb_phys_hash
[CODE_GEN_PHYS_HASH_SIZE
];
85 /* any access to the tbs or the page table must use this lock */
86 spinlock_t tb_lock
= SPIN_LOCK_UNLOCKED
;
88 #if defined(__arm__) || defined(__sparc_v9__)
89 /* The prologue must be reachable with a direct jump. ARM and Sparc64
90 have limited branch ranges (possibly also PPC) so place it in a
91 section close to code segment. */
92 #define code_gen_section \
93 __attribute__((__section__(".gen_code"))) \
94 __attribute__((aligned (32)))
96 /* Maximum alignment for Win32 is 16. */
97 #define code_gen_section \
98 __attribute__((aligned (16)))
100 #define code_gen_section \
101 __attribute__((aligned (32)))
104 uint8_t code_gen_prologue
[1024] code_gen_section
;
105 static uint8_t *code_gen_buffer
;
106 static unsigned long code_gen_buffer_size
;
107 /* threshold to flush the translated code buffer */
108 static unsigned long code_gen_buffer_max_size
;
109 uint8_t *code_gen_ptr
;
111 #if !defined(CONFIG_USER_ONLY)
113 uint8_t *phys_ram_dirty
;
114 static int in_migration
;
116 typedef struct RAMBlock
{
120 struct RAMBlock
*next
;
123 static RAMBlock
*ram_blocks
;
124 /* TODO: When we implement (and use) ram deallocation (e.g. for hotplug)
125 then we can no longer assume contiguous ram offsets, and external uses
126 of this variable will break. */
127 ram_addr_t last_ram_offset
;
131 /* current CPU in the current thread. It is only valid inside
133 CPUState
*cpu_single_env
;
134 /* 0 = Do not count executed instructions.
135 1 = Precise instruction counting.
136 2 = Adaptive rate instruction counting. */
138 /* Current instruction counter. While executing translated code this may
139 include some instructions that have not yet been executed. */
142 typedef struct PageDesc
{
143 /* list of TBs intersecting this ram page */
144 TranslationBlock
*first_tb
;
145 /* in order to optimize self modifying code, we count the number
146 of lookups we do to a given page to use a bitmap */
147 unsigned int code_write_count
;
148 uint8_t *code_bitmap
;
149 #if defined(CONFIG_USER_ONLY)
154 typedef struct PhysPageDesc
{
155 /* offset in host memory of the page + io_index in the low bits */
156 ram_addr_t phys_offset
;
157 ram_addr_t region_offset
;
161 #if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
162 /* XXX: this is a temporary hack for alpha target.
163 * In the future, this is to be replaced by a multi-level table
164 * to actually be able to handle the complete 64 bits address space.
166 #define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
168 #define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
171 #define L1_SIZE (1 << L1_BITS)
172 #define L2_SIZE (1 << L2_BITS)
174 unsigned long qemu_real_host_page_size
;
175 unsigned long qemu_host_page_bits
;
176 unsigned long qemu_host_page_size
;
177 unsigned long qemu_host_page_mask
;
179 /* XXX: for system emulation, it could just be an array */
180 static PageDesc
*l1_map
[L1_SIZE
];
181 static PhysPageDesc
**l1_phys_map
;
183 #if !defined(CONFIG_USER_ONLY)
184 static void io_mem_init(void);
186 /* io memory support */
187 CPUWriteMemoryFunc
*io_mem_write
[IO_MEM_NB_ENTRIES
][4];
188 CPUReadMemoryFunc
*io_mem_read
[IO_MEM_NB_ENTRIES
][4];
189 void *io_mem_opaque
[IO_MEM_NB_ENTRIES
];
190 static char io_mem_used
[IO_MEM_NB_ENTRIES
];
191 static int io_mem_watch
;
196 static const char *logfilename
= "qemu.log";
198 static const char *logfilename
= "/tmp/qemu.log";
202 static int log_append
= 0;
205 static int tlb_flush_count
;
206 static int tb_flush_count
;
207 static int tb_phys_invalidate_count
;
209 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
210 typedef struct subpage_t
{
211 target_phys_addr_t base
;
212 CPUReadMemoryFunc
* const *mem_read
[TARGET_PAGE_SIZE
][4];
213 CPUWriteMemoryFunc
* const *mem_write
[TARGET_PAGE_SIZE
][4];
214 void *opaque
[TARGET_PAGE_SIZE
][2][4];
215 ram_addr_t region_offset
[TARGET_PAGE_SIZE
][2][4];
219 static void map_exec(void *addr
, long size
)
222 VirtualProtect(addr
, size
,
223 PAGE_EXECUTE_READWRITE
, &old_protect
);
227 static void map_exec(void *addr
, long size
)
229 unsigned long start
, end
, page_size
;
231 page_size
= getpagesize();
232 start
= (unsigned long)addr
;
233 start
&= ~(page_size
- 1);
235 end
= (unsigned long)addr
+ size
;
236 end
+= page_size
- 1;
237 end
&= ~(page_size
- 1);
239 mprotect((void *)start
, end
- start
,
240 PROT_READ
| PROT_WRITE
| PROT_EXEC
);
244 static void page_init(void)
246 /* NOTE: we can always suppose that qemu_host_page_size >=
250 SYSTEM_INFO system_info
;
252 GetSystemInfo(&system_info
);
253 qemu_real_host_page_size
= system_info
.dwPageSize
;
256 qemu_real_host_page_size
= getpagesize();
258 if (qemu_host_page_size
== 0)
259 qemu_host_page_size
= qemu_real_host_page_size
;
260 if (qemu_host_page_size
< TARGET_PAGE_SIZE
)
261 qemu_host_page_size
= TARGET_PAGE_SIZE
;
262 qemu_host_page_bits
= 0;
263 while ((1 << qemu_host_page_bits
) < qemu_host_page_size
)
264 qemu_host_page_bits
++;
265 qemu_host_page_mask
= ~(qemu_host_page_size
- 1);
266 l1_phys_map
= qemu_vmalloc(L1_SIZE
* sizeof(void *));
267 memset(l1_phys_map
, 0, L1_SIZE
* sizeof(void *));
269 #if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
271 long long startaddr
, endaddr
;
276 last_brk
= (unsigned long)sbrk(0);
277 f
= fopen("/proc/self/maps", "r");
280 n
= fscanf (f
, "%llx-%llx %*[^\n]\n", &startaddr
, &endaddr
);
282 startaddr
= MIN(startaddr
,
283 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS
) - 1);
284 endaddr
= MIN(endaddr
,
285 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS
) - 1);
286 page_set_flags(startaddr
& TARGET_PAGE_MASK
,
287 TARGET_PAGE_ALIGN(endaddr
),
298 static inline PageDesc
**page_l1_map(target_ulong index
)
300 #if TARGET_LONG_BITS > 32
301 /* Host memory outside guest VM. For 32-bit targets we have already
302 excluded high addresses. */
303 if (index
> ((target_ulong
)L2_SIZE
* L1_SIZE
))
306 return &l1_map
[index
>> L2_BITS
];
309 static inline PageDesc
*page_find_alloc(target_ulong index
)
312 lp
= page_l1_map(index
);
318 /* allocate if not found */
319 #if defined(CONFIG_USER_ONLY)
320 size_t len
= sizeof(PageDesc
) * L2_SIZE
;
321 /* Don't use qemu_malloc because it may recurse. */
322 p
= mmap(NULL
, len
, PROT_READ
| PROT_WRITE
,
323 MAP_PRIVATE
| MAP_ANONYMOUS
, -1, 0);
326 unsigned long addr
= h2g(p
);
327 page_set_flags(addr
& TARGET_PAGE_MASK
,
328 TARGET_PAGE_ALIGN(addr
+ len
),
332 p
= qemu_mallocz(sizeof(PageDesc
) * L2_SIZE
);
336 return p
+ (index
& (L2_SIZE
- 1));
339 static inline PageDesc
*page_find(target_ulong index
)
342 lp
= page_l1_map(index
);
350 return p
+ (index
& (L2_SIZE
- 1));
353 static PhysPageDesc
*phys_page_find_alloc(target_phys_addr_t index
, int alloc
)
358 p
= (void **)l1_phys_map
;
359 #if TARGET_PHYS_ADDR_SPACE_BITS > 32
361 #if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
362 #error unsupported TARGET_PHYS_ADDR_SPACE_BITS
364 lp
= p
+ ((index
>> (L1_BITS
+ L2_BITS
)) & (L1_SIZE
- 1));
367 /* allocate if not found */
370 p
= qemu_vmalloc(sizeof(void *) * L1_SIZE
);
371 memset(p
, 0, sizeof(void *) * L1_SIZE
);
375 lp
= p
+ ((index
>> L2_BITS
) & (L1_SIZE
- 1));
379 /* allocate if not found */
382 pd
= qemu_vmalloc(sizeof(PhysPageDesc
) * L2_SIZE
);
384 for (i
= 0; i
< L2_SIZE
; i
++) {
385 pd
[i
].phys_offset
= IO_MEM_UNASSIGNED
;
386 pd
[i
].region_offset
= (index
+ i
) << TARGET_PAGE_BITS
;
389 return ((PhysPageDesc
*)pd
) + (index
& (L2_SIZE
- 1));
392 static inline PhysPageDesc
*phys_page_find(target_phys_addr_t index
)
394 return phys_page_find_alloc(index
, 0);
397 #if !defined(CONFIG_USER_ONLY)
398 static void tlb_protect_code(ram_addr_t ram_addr
);
399 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
401 #define mmap_lock() do { } while(0)
402 #define mmap_unlock() do { } while(0)
405 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
407 #if defined(CONFIG_USER_ONLY)
408 /* Currently it is not recommended to allocate big chunks of data in
409 user mode. It will change when a dedicated libc will be used */
410 #define USE_STATIC_CODE_GEN_BUFFER
413 #ifdef USE_STATIC_CODE_GEN_BUFFER
414 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
];
417 static void code_gen_alloc(unsigned long tb_size
)
419 #ifdef USE_STATIC_CODE_GEN_BUFFER
420 code_gen_buffer
= static_code_gen_buffer
;
421 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
422 map_exec(code_gen_buffer
, code_gen_buffer_size
);
424 code_gen_buffer_size
= tb_size
;
425 if (code_gen_buffer_size
== 0) {
426 #if defined(CONFIG_USER_ONLY)
427 /* in user mode, phys_ram_size is not meaningful */
428 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
430 /* XXX: needs adjustments */
431 code_gen_buffer_size
= (unsigned long)(ram_size
/ 4);
434 if (code_gen_buffer_size
< MIN_CODE_GEN_BUFFER_SIZE
)
435 code_gen_buffer_size
= MIN_CODE_GEN_BUFFER_SIZE
;
436 /* The code gen buffer location may have constraints depending on
437 the host cpu and OS */
438 #if defined(__linux__)
443 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
444 #if defined(__x86_64__)
446 /* Cannot map more than that */
447 if (code_gen_buffer_size
> (800 * 1024 * 1024))
448 code_gen_buffer_size
= (800 * 1024 * 1024);
449 #elif defined(__sparc_v9__)
450 // Map the buffer below 2G, so we can use direct calls and branches
452 start
= (void *) 0x60000000UL
;
453 if (code_gen_buffer_size
> (512 * 1024 * 1024))
454 code_gen_buffer_size
= (512 * 1024 * 1024);
455 #elif defined(__arm__)
456 /* Map the buffer below 32M, so we can use direct calls and branches */
458 start
= (void *) 0x01000000UL
;
459 if (code_gen_buffer_size
> 16 * 1024 * 1024)
460 code_gen_buffer_size
= 16 * 1024 * 1024;
462 code_gen_buffer
= mmap(start
, code_gen_buffer_size
,
463 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
465 if (code_gen_buffer
== MAP_FAILED
) {
466 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
470 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__DragonFly__)
474 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
475 #if defined(__x86_64__)
476 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
477 * 0x40000000 is free */
479 addr
= (void *)0x40000000;
480 /* Cannot map more than that */
481 if (code_gen_buffer_size
> (800 * 1024 * 1024))
482 code_gen_buffer_size
= (800 * 1024 * 1024);
484 code_gen_buffer
= mmap(addr
, code_gen_buffer_size
,
485 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
487 if (code_gen_buffer
== MAP_FAILED
) {
488 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
493 code_gen_buffer
= qemu_malloc(code_gen_buffer_size
);
494 map_exec(code_gen_buffer
, code_gen_buffer_size
);
496 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
497 map_exec(code_gen_prologue
, sizeof(code_gen_prologue
));
498 code_gen_buffer_max_size
= code_gen_buffer_size
-
499 code_gen_max_block_size();
500 code_gen_max_blocks
= code_gen_buffer_size
/ CODE_GEN_AVG_BLOCK_SIZE
;
501 tbs
= qemu_malloc(code_gen_max_blocks
* sizeof(TranslationBlock
));
504 /* Must be called before using the QEMU cpus. 'tb_size' is the size
505 (in bytes) allocated to the translation buffer. Zero means default
507 void cpu_exec_init_all(unsigned long tb_size
)
510 code_gen_alloc(tb_size
);
511 code_gen_ptr
= code_gen_buffer
;
513 #if !defined(CONFIG_USER_ONLY)
518 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
520 static void cpu_common_pre_save(void *opaque
)
522 CPUState
*env
= opaque
;
524 cpu_synchronize_state(env
);
527 static int cpu_common_pre_load(void *opaque
)
529 CPUState
*env
= opaque
;
531 cpu_synchronize_state(env
);
535 static int cpu_common_post_load(void *opaque
, int version_id
)
537 CPUState
*env
= opaque
;
539 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
540 version_id is increased. */
541 env
->interrupt_request
&= ~0x01;
547 static const VMStateDescription vmstate_cpu_common
= {
548 .name
= "cpu_common",
550 .minimum_version_id
= 1,
551 .minimum_version_id_old
= 1,
552 .pre_save
= cpu_common_pre_save
,
553 .pre_load
= cpu_common_pre_load
,
554 .post_load
= cpu_common_post_load
,
555 .fields
= (VMStateField
[]) {
556 VMSTATE_UINT32(halted
, CPUState
),
557 VMSTATE_UINT32(interrupt_request
, CPUState
),
558 VMSTATE_END_OF_LIST()
563 CPUState
*qemu_get_cpu(int cpu
)
565 CPUState
*env
= first_cpu
;
568 if (env
->cpu_index
== cpu
)
576 void cpu_exec_init(CPUState
*env
)
581 #if defined(CONFIG_USER_ONLY)
584 env
->next_cpu
= NULL
;
587 while (*penv
!= NULL
) {
588 penv
= &(*penv
)->next_cpu
;
591 env
->cpu_index
= cpu_index
;
593 QTAILQ_INIT(&env
->breakpoints
);
594 QTAILQ_INIT(&env
->watchpoints
);
596 #if defined(CONFIG_USER_ONLY)
599 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
600 vmstate_register(cpu_index
, &vmstate_cpu_common
, env
);
601 register_savevm("cpu", cpu_index
, CPU_SAVE_VERSION
,
602 cpu_save
, cpu_load
, env
);
606 static inline void invalidate_page_bitmap(PageDesc
*p
)
608 if (p
->code_bitmap
) {
609 qemu_free(p
->code_bitmap
);
610 p
->code_bitmap
= NULL
;
612 p
->code_write_count
= 0;
615 /* set to NULL all the 'first_tb' fields in all PageDescs */
616 static void page_flush_tb(void)
621 for(i
= 0; i
< L1_SIZE
; i
++) {
624 for(j
= 0; j
< L2_SIZE
; j
++) {
626 invalidate_page_bitmap(p
);
633 /* flush all the translation blocks */
634 /* XXX: tb_flush is currently not thread safe */
635 void tb_flush(CPUState
*env1
)
638 #if defined(DEBUG_FLUSH)
639 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
640 (unsigned long)(code_gen_ptr
- code_gen_buffer
),
642 ((unsigned long)(code_gen_ptr
- code_gen_buffer
)) / nb_tbs
: 0);
644 if ((unsigned long)(code_gen_ptr
- code_gen_buffer
) > code_gen_buffer_size
)
645 cpu_abort(env1
, "Internal error: code buffer overflow\n");
649 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
650 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
653 memset (tb_phys_hash
, 0, CODE_GEN_PHYS_HASH_SIZE
* sizeof (void *));
656 code_gen_ptr
= code_gen_buffer
;
657 /* XXX: flush processor icache at this point if cache flush is
662 #ifdef DEBUG_TB_CHECK
664 static void tb_invalidate_check(target_ulong address
)
666 TranslationBlock
*tb
;
668 address
&= TARGET_PAGE_MASK
;
669 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
670 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
671 if (!(address
+ TARGET_PAGE_SIZE
<= tb
->pc
||
672 address
>= tb
->pc
+ tb
->size
)) {
673 printf("ERROR invalidate: address=" TARGET_FMT_lx
674 " PC=%08lx size=%04x\n",
675 address
, (long)tb
->pc
, tb
->size
);
681 /* verify that all the pages have correct rights for code */
682 static void tb_page_check(void)
684 TranslationBlock
*tb
;
685 int i
, flags1
, flags2
;
687 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
688 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
689 flags1
= page_get_flags(tb
->pc
);
690 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
691 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
692 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
693 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
701 /* invalidate one TB */
702 static inline void tb_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
,
705 TranslationBlock
*tb1
;
709 *ptb
= *(TranslationBlock
**)((char *)tb1
+ next_offset
);
712 ptb
= (TranslationBlock
**)((char *)tb1
+ next_offset
);
716 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
718 TranslationBlock
*tb1
;
724 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
726 *ptb
= tb1
->page_next
[n1
];
729 ptb
= &tb1
->page_next
[n1
];
733 static inline void tb_jmp_remove(TranslationBlock
*tb
, int n
)
735 TranslationBlock
*tb1
, **ptb
;
738 ptb
= &tb
->jmp_next
[n
];
741 /* find tb(n) in circular list */
745 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
746 if (n1
== n
&& tb1
== tb
)
749 ptb
= &tb1
->jmp_first
;
751 ptb
= &tb1
->jmp_next
[n1
];
754 /* now we can suppress tb(n) from the list */
755 *ptb
= tb
->jmp_next
[n
];
757 tb
->jmp_next
[n
] = NULL
;
761 /* reset the jump entry 'n' of a TB so that it is not chained to
763 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
765 tb_set_jmp_target(tb
, n
, (unsigned long)(tb
->tc_ptr
+ tb
->tb_next_offset
[n
]));
768 void tb_phys_invalidate(TranslationBlock
*tb
, target_ulong page_addr
)
773 target_phys_addr_t phys_pc
;
774 TranslationBlock
*tb1
, *tb2
;
776 /* remove the TB from the hash list */
777 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
778 h
= tb_phys_hash_func(phys_pc
);
779 tb_remove(&tb_phys_hash
[h
], tb
,
780 offsetof(TranslationBlock
, phys_hash_next
));
782 /* remove the TB from the page list */
783 if (tb
->page_addr
[0] != page_addr
) {
784 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
785 tb_page_remove(&p
->first_tb
, tb
);
786 invalidate_page_bitmap(p
);
788 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
789 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
790 tb_page_remove(&p
->first_tb
, tb
);
791 invalidate_page_bitmap(p
);
794 tb_invalidated_flag
= 1;
796 /* remove the TB from the hash list */
797 h
= tb_jmp_cache_hash_func(tb
->pc
);
798 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
799 if (env
->tb_jmp_cache
[h
] == tb
)
800 env
->tb_jmp_cache
[h
] = NULL
;
803 /* suppress this TB from the two jump lists */
804 tb_jmp_remove(tb
, 0);
805 tb_jmp_remove(tb
, 1);
807 /* suppress any remaining jumps to this TB */
813 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
814 tb2
= tb1
->jmp_next
[n1
];
815 tb_reset_jump(tb1
, n1
);
816 tb1
->jmp_next
[n1
] = NULL
;
819 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2); /* fail safe */
821 tb_phys_invalidate_count
++;
824 static inline void set_bits(uint8_t *tab
, int start
, int len
)
830 mask
= 0xff << (start
& 7);
831 if ((start
& ~7) == (end
& ~7)) {
833 mask
&= ~(0xff << (end
& 7));
838 start
= (start
+ 8) & ~7;
840 while (start
< end1
) {
845 mask
= ~(0xff << (end
& 7));
851 static void build_page_bitmap(PageDesc
*p
)
853 int n
, tb_start
, tb_end
;
854 TranslationBlock
*tb
;
856 p
->code_bitmap
= qemu_mallocz(TARGET_PAGE_SIZE
/ 8);
861 tb
= (TranslationBlock
*)((long)tb
& ~3);
862 /* NOTE: this is subtle as a TB may span two physical pages */
864 /* NOTE: tb_end may be after the end of the page, but
865 it is not a problem */
866 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
867 tb_end
= tb_start
+ tb
->size
;
868 if (tb_end
> TARGET_PAGE_SIZE
)
869 tb_end
= TARGET_PAGE_SIZE
;
872 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
874 set_bits(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
875 tb
= tb
->page_next
[n
];
879 TranslationBlock
*tb_gen_code(CPUState
*env
,
880 target_ulong pc
, target_ulong cs_base
,
881 int flags
, int cflags
)
883 TranslationBlock
*tb
;
885 target_ulong phys_pc
, phys_page2
, virt_page2
;
888 phys_pc
= get_phys_addr_code(env
, pc
);
891 /* flush must be done */
893 /* cannot fail at this point */
895 /* Don't forget to invalidate previous TB info. */
896 tb_invalidated_flag
= 1;
898 tc_ptr
= code_gen_ptr
;
900 tb
->cs_base
= cs_base
;
903 cpu_gen_code(env
, tb
, &code_gen_size
);
904 code_gen_ptr
= (void *)(((unsigned long)code_gen_ptr
+ code_gen_size
+ CODE_GEN_ALIGN
- 1) & ~(CODE_GEN_ALIGN
- 1));
906 /* check next page if needed */
907 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
909 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
910 phys_page2
= get_phys_addr_code(env
, virt_page2
);
912 tb_link_phys(tb
, phys_pc
, phys_page2
);
916 /* invalidate all TBs which intersect with the target physical page
917 starting in range [start;end[. NOTE: start and end must refer to
918 the same physical page. 'is_cpu_write_access' should be true if called
919 from a real cpu write access: the virtual CPU will exit the current
920 TB if code is modified inside this TB. */
921 void tb_invalidate_phys_page_range(target_phys_addr_t start
, target_phys_addr_t end
,
922 int is_cpu_write_access
)
924 TranslationBlock
*tb
, *tb_next
, *saved_tb
;
925 CPUState
*env
= cpu_single_env
;
926 target_ulong tb_start
, tb_end
;
929 #ifdef TARGET_HAS_PRECISE_SMC
930 int current_tb_not_found
= is_cpu_write_access
;
931 TranslationBlock
*current_tb
= NULL
;
932 int current_tb_modified
= 0;
933 target_ulong current_pc
= 0;
934 target_ulong current_cs_base
= 0;
935 int current_flags
= 0;
936 #endif /* TARGET_HAS_PRECISE_SMC */
938 p
= page_find(start
>> TARGET_PAGE_BITS
);
941 if (!p
->code_bitmap
&&
942 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
&&
943 is_cpu_write_access
) {
944 /* build code bitmap */
945 build_page_bitmap(p
);
948 /* we remove all the TBs in the range [start, end[ */
949 /* XXX: see if in some cases it could be faster to invalidate all the code */
953 tb
= (TranslationBlock
*)((long)tb
& ~3);
954 tb_next
= tb
->page_next
[n
];
955 /* NOTE: this is subtle as a TB may span two physical pages */
957 /* NOTE: tb_end may be after the end of the page, but
958 it is not a problem */
959 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
960 tb_end
= tb_start
+ tb
->size
;
962 tb_start
= tb
->page_addr
[1];
963 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
965 if (!(tb_end
<= start
|| tb_start
>= end
)) {
966 #ifdef TARGET_HAS_PRECISE_SMC
967 if (current_tb_not_found
) {
968 current_tb_not_found
= 0;
970 if (env
->mem_io_pc
) {
971 /* now we have a real cpu fault */
972 current_tb
= tb_find_pc(env
->mem_io_pc
);
975 if (current_tb
== tb
&&
976 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
977 /* If we are modifying the current TB, we must stop
978 its execution. We could be more precise by checking
979 that the modification is after the current PC, but it
980 would require a specialized function to partially
981 restore the CPU state */
983 current_tb_modified
= 1;
984 cpu_restore_state(current_tb
, env
,
985 env
->mem_io_pc
, NULL
);
986 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
989 #endif /* TARGET_HAS_PRECISE_SMC */
990 /* we need to do that to handle the case where a signal
991 occurs while doing tb_phys_invalidate() */
994 saved_tb
= env
->current_tb
;
995 env
->current_tb
= NULL
;
997 tb_phys_invalidate(tb
, -1);
999 env
->current_tb
= saved_tb
;
1000 if (env
->interrupt_request
&& env
->current_tb
)
1001 cpu_interrupt(env
, env
->interrupt_request
);
1006 #if !defined(CONFIG_USER_ONLY)
1007 /* if no code remaining, no need to continue to use slow writes */
1009 invalidate_page_bitmap(p
);
1010 if (is_cpu_write_access
) {
1011 tlb_unprotect_code_phys(env
, start
, env
->mem_io_vaddr
);
1015 #ifdef TARGET_HAS_PRECISE_SMC
1016 if (current_tb_modified
) {
1017 /* we generate a block containing just the instruction
1018 modifying the memory. It will ensure that it cannot modify
1020 env
->current_tb
= NULL
;
1021 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1022 cpu_resume_from_signal(env
, NULL
);
1027 /* len must be <= 8 and start must be a multiple of len */
1028 static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start
, int len
)
1034 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1035 cpu_single_env
->mem_io_vaddr
, len
,
1036 cpu_single_env
->eip
,
1037 cpu_single_env
->eip
+ (long)cpu_single_env
->segs
[R_CS
].base
);
1040 p
= page_find(start
>> TARGET_PAGE_BITS
);
1043 if (p
->code_bitmap
) {
1044 offset
= start
& ~TARGET_PAGE_MASK
;
1045 b
= p
->code_bitmap
[offset
>> 3] >> (offset
& 7);
1046 if (b
& ((1 << len
) - 1))
1050 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1054 #if !defined(CONFIG_SOFTMMU)
1055 static void tb_invalidate_phys_page(target_phys_addr_t addr
,
1056 unsigned long pc
, void *puc
)
1058 TranslationBlock
*tb
;
1061 #ifdef TARGET_HAS_PRECISE_SMC
1062 TranslationBlock
*current_tb
= NULL
;
1063 CPUState
*env
= cpu_single_env
;
1064 int current_tb_modified
= 0;
1065 target_ulong current_pc
= 0;
1066 target_ulong current_cs_base
= 0;
1067 int current_flags
= 0;
1070 addr
&= TARGET_PAGE_MASK
;
1071 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1075 #ifdef TARGET_HAS_PRECISE_SMC
1076 if (tb
&& pc
!= 0) {
1077 current_tb
= tb_find_pc(pc
);
1080 while (tb
!= NULL
) {
1082 tb
= (TranslationBlock
*)((long)tb
& ~3);
1083 #ifdef TARGET_HAS_PRECISE_SMC
1084 if (current_tb
== tb
&&
1085 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1086 /* If we are modifying the current TB, we must stop
1087 its execution. We could be more precise by checking
1088 that the modification is after the current PC, but it
1089 would require a specialized function to partially
1090 restore the CPU state */
1092 current_tb_modified
= 1;
1093 cpu_restore_state(current_tb
, env
, pc
, puc
);
1094 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1097 #endif /* TARGET_HAS_PRECISE_SMC */
1098 tb_phys_invalidate(tb
, addr
);
1099 tb
= tb
->page_next
[n
];
1102 #ifdef TARGET_HAS_PRECISE_SMC
1103 if (current_tb_modified
) {
1104 /* we generate a block containing just the instruction
1105 modifying the memory. It will ensure that it cannot modify
1107 env
->current_tb
= NULL
;
1108 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1109 cpu_resume_from_signal(env
, puc
);
1115 /* add the tb in the target page and protect it if necessary */
1116 static inline void tb_alloc_page(TranslationBlock
*tb
,
1117 unsigned int n
, target_ulong page_addr
)
1120 TranslationBlock
*last_first_tb
;
1122 tb
->page_addr
[n
] = page_addr
;
1123 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
);
1124 tb
->page_next
[n
] = p
->first_tb
;
1125 last_first_tb
= p
->first_tb
;
1126 p
->first_tb
= (TranslationBlock
*)((long)tb
| n
);
1127 invalidate_page_bitmap(p
);
1129 #if defined(TARGET_HAS_SMC) || 1
1131 #if defined(CONFIG_USER_ONLY)
1132 if (p
->flags
& PAGE_WRITE
) {
1137 /* force the host page as non writable (writes will have a
1138 page fault + mprotect overhead) */
1139 page_addr
&= qemu_host_page_mask
;
1141 for(addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1142 addr
+= TARGET_PAGE_SIZE
) {
1144 p2
= page_find (addr
>> TARGET_PAGE_BITS
);
1148 p2
->flags
&= ~PAGE_WRITE
;
1149 page_get_flags(addr
);
1151 mprotect(g2h(page_addr
), qemu_host_page_size
,
1152 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1153 #ifdef DEBUG_TB_INVALIDATE
1154 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1159 /* if some code is already present, then the pages are already
1160 protected. So we handle the case where only the first TB is
1161 allocated in a physical page */
1162 if (!last_first_tb
) {
1163 tlb_protect_code(page_addr
);
1167 #endif /* TARGET_HAS_SMC */
1170 /* Allocate a new translation block. Flush the translation buffer if
1171 too many translation blocks or too much generated code. */
1172 TranslationBlock
*tb_alloc(target_ulong pc
)
1174 TranslationBlock
*tb
;
1176 if (nb_tbs
>= code_gen_max_blocks
||
1177 (code_gen_ptr
- code_gen_buffer
) >= code_gen_buffer_max_size
)
1179 tb
= &tbs
[nb_tbs
++];
1185 void tb_free(TranslationBlock
*tb
)
1187 /* In practice this is mostly used for single use temporary TB
1188 Ignore the hard cases and just back up if this TB happens to
1189 be the last one generated. */
1190 if (nb_tbs
> 0 && tb
== &tbs
[nb_tbs
- 1]) {
1191 code_gen_ptr
= tb
->tc_ptr
;
1196 /* add a new TB and link it to the physical page tables. phys_page2 is
1197 (-1) to indicate that only one page contains the TB. */
1198 void tb_link_phys(TranslationBlock
*tb
,
1199 target_ulong phys_pc
, target_ulong phys_page2
)
1202 TranslationBlock
**ptb
;
1204 /* Grab the mmap lock to stop another thread invalidating this TB
1205 before we are done. */
1207 /* add in the physical hash table */
1208 h
= tb_phys_hash_func(phys_pc
);
1209 ptb
= &tb_phys_hash
[h
];
1210 tb
->phys_hash_next
= *ptb
;
1213 /* add in the page list */
1214 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1215 if (phys_page2
!= -1)
1216 tb_alloc_page(tb
, 1, phys_page2
);
1218 tb
->page_addr
[1] = -1;
1220 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2);
1221 tb
->jmp_next
[0] = NULL
;
1222 tb
->jmp_next
[1] = NULL
;
1224 /* init original jump addresses */
1225 if (tb
->tb_next_offset
[0] != 0xffff)
1226 tb_reset_jump(tb
, 0);
1227 if (tb
->tb_next_offset
[1] != 0xffff)
1228 tb_reset_jump(tb
, 1);
1230 #ifdef DEBUG_TB_CHECK
1236 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1237 tb[1].tc_ptr. Return NULL if not found */
1238 TranslationBlock
*tb_find_pc(unsigned long tc_ptr
)
1240 int m_min
, m_max
, m
;
1242 TranslationBlock
*tb
;
1246 if (tc_ptr
< (unsigned long)code_gen_buffer
||
1247 tc_ptr
>= (unsigned long)code_gen_ptr
)
1249 /* binary search (cf Knuth) */
1252 while (m_min
<= m_max
) {
1253 m
= (m_min
+ m_max
) >> 1;
1255 v
= (unsigned long)tb
->tc_ptr
;
1258 else if (tc_ptr
< v
) {
1267 static void tb_reset_jump_recursive(TranslationBlock
*tb
);
1269 static inline void tb_reset_jump_recursive2(TranslationBlock
*tb
, int n
)
1271 TranslationBlock
*tb1
, *tb_next
, **ptb
;
1274 tb1
= tb
->jmp_next
[n
];
1276 /* find head of list */
1279 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1282 tb1
= tb1
->jmp_next
[n1
];
1284 /* we are now sure now that tb jumps to tb1 */
1287 /* remove tb from the jmp_first list */
1288 ptb
= &tb_next
->jmp_first
;
1292 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1293 if (n1
== n
&& tb1
== tb
)
1295 ptb
= &tb1
->jmp_next
[n1
];
1297 *ptb
= tb
->jmp_next
[n
];
1298 tb
->jmp_next
[n
] = NULL
;
1300 /* suppress the jump to next tb in generated code */
1301 tb_reset_jump(tb
, n
);
1303 /* suppress jumps in the tb on which we could have jumped */
1304 tb_reset_jump_recursive(tb_next
);
1308 static void tb_reset_jump_recursive(TranslationBlock
*tb
)
1310 tb_reset_jump_recursive2(tb
, 0);
1311 tb_reset_jump_recursive2(tb
, 1);
1314 #if defined(TARGET_HAS_ICE)
1315 static void breakpoint_invalidate(CPUState
*env
, target_ulong pc
)
1317 target_phys_addr_t addr
;
1319 ram_addr_t ram_addr
;
1322 addr
= cpu_get_phys_page_debug(env
, pc
);
1323 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
1325 pd
= IO_MEM_UNASSIGNED
;
1327 pd
= p
->phys_offset
;
1329 ram_addr
= (pd
& TARGET_PAGE_MASK
) | (pc
& ~TARGET_PAGE_MASK
);
1330 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1334 /* Add a watchpoint. */
1335 int cpu_watchpoint_insert(CPUState
*env
, target_ulong addr
, target_ulong len
,
1336 int flags
, CPUWatchpoint
**watchpoint
)
1338 target_ulong len_mask
= ~(len
- 1);
1341 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1342 if ((len
!= 1 && len
!= 2 && len
!= 4 && len
!= 8) || (addr
& ~len_mask
)) {
1343 fprintf(stderr
, "qemu: tried to set invalid watchpoint at "
1344 TARGET_FMT_lx
", len=" TARGET_FMT_lu
"\n", addr
, len
);
1347 wp
= qemu_malloc(sizeof(*wp
));
1350 wp
->len_mask
= len_mask
;
1353 /* keep all GDB-injected watchpoints in front */
1355 QTAILQ_INSERT_HEAD(&env
->watchpoints
, wp
, entry
);
1357 QTAILQ_INSERT_TAIL(&env
->watchpoints
, wp
, entry
);
1359 tlb_flush_page(env
, addr
);
1366 /* Remove a specific watchpoint. */
1367 int cpu_watchpoint_remove(CPUState
*env
, target_ulong addr
, target_ulong len
,
1370 target_ulong len_mask
= ~(len
- 1);
1373 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1374 if (addr
== wp
->vaddr
&& len_mask
== wp
->len_mask
1375 && flags
== (wp
->flags
& ~BP_WATCHPOINT_HIT
)) {
1376 cpu_watchpoint_remove_by_ref(env
, wp
);
1383 /* Remove a specific watchpoint by reference. */
1384 void cpu_watchpoint_remove_by_ref(CPUState
*env
, CPUWatchpoint
*watchpoint
)
1386 QTAILQ_REMOVE(&env
->watchpoints
, watchpoint
, entry
);
1388 tlb_flush_page(env
, watchpoint
->vaddr
);
1390 qemu_free(watchpoint
);
1393 /* Remove all matching watchpoints. */
1394 void cpu_watchpoint_remove_all(CPUState
*env
, int mask
)
1396 CPUWatchpoint
*wp
, *next
;
1398 QTAILQ_FOREACH_SAFE(wp
, &env
->watchpoints
, entry
, next
) {
1399 if (wp
->flags
& mask
)
1400 cpu_watchpoint_remove_by_ref(env
, wp
);
1404 /* Add a breakpoint. */
1405 int cpu_breakpoint_insert(CPUState
*env
, target_ulong pc
, int flags
,
1406 CPUBreakpoint
**breakpoint
)
1408 #if defined(TARGET_HAS_ICE)
1411 bp
= qemu_malloc(sizeof(*bp
));
1416 /* keep all GDB-injected breakpoints in front */
1418 QTAILQ_INSERT_HEAD(&env
->breakpoints
, bp
, entry
);
1420 QTAILQ_INSERT_TAIL(&env
->breakpoints
, bp
, entry
);
1422 breakpoint_invalidate(env
, pc
);
1432 /* Remove a specific breakpoint. */
1433 int cpu_breakpoint_remove(CPUState
*env
, target_ulong pc
, int flags
)
1435 #if defined(TARGET_HAS_ICE)
1438 QTAILQ_FOREACH(bp
, &env
->breakpoints
, entry
) {
1439 if (bp
->pc
== pc
&& bp
->flags
== flags
) {
1440 cpu_breakpoint_remove_by_ref(env
, bp
);
1450 /* Remove a specific breakpoint by reference. */
1451 void cpu_breakpoint_remove_by_ref(CPUState
*env
, CPUBreakpoint
*breakpoint
)
1453 #if defined(TARGET_HAS_ICE)
1454 QTAILQ_REMOVE(&env
->breakpoints
, breakpoint
, entry
);
1456 breakpoint_invalidate(env
, breakpoint
->pc
);
1458 qemu_free(breakpoint
);
1462 /* Remove all matching breakpoints. */
1463 void cpu_breakpoint_remove_all(CPUState
*env
, int mask
)
1465 #if defined(TARGET_HAS_ICE)
1466 CPUBreakpoint
*bp
, *next
;
1468 QTAILQ_FOREACH_SAFE(bp
, &env
->breakpoints
, entry
, next
) {
1469 if (bp
->flags
& mask
)
1470 cpu_breakpoint_remove_by_ref(env
, bp
);
1475 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1476 CPU loop after each instruction */
1477 void cpu_single_step(CPUState
*env
, int enabled
)
1479 #if defined(TARGET_HAS_ICE)
1480 if (env
->singlestep_enabled
!= enabled
) {
1481 env
->singlestep_enabled
= enabled
;
1483 kvm_update_guest_debug(env
, 0);
1485 /* must flush all the translated code to avoid inconsistencies */
1486 /* XXX: only flush what is necessary */
1493 /* enable or disable low levels log */
1494 void cpu_set_log(int log_flags
)
1496 loglevel
= log_flags
;
1497 if (loglevel
&& !logfile
) {
1498 logfile
= fopen(logfilename
, log_append
? "a" : "w");
1500 perror(logfilename
);
1503 #if !defined(CONFIG_SOFTMMU)
1504 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1506 static char logfile_buf
[4096];
1507 setvbuf(logfile
, logfile_buf
, _IOLBF
, sizeof(logfile_buf
));
1509 #elif !defined(_WIN32)
1510 /* Win32 doesn't support line-buffering and requires size >= 2 */
1511 setvbuf(logfile
, NULL
, _IOLBF
, 0);
1515 if (!loglevel
&& logfile
) {
1521 void cpu_set_log_filename(const char *filename
)
1523 logfilename
= strdup(filename
);
1528 cpu_set_log(loglevel
);
1531 static void cpu_unlink_tb(CPUState
*env
)
1533 #if defined(CONFIG_USE_NPTL)
1534 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1535 problem and hope the cpu will stop of its own accord. For userspace
1536 emulation this often isn't actually as bad as it sounds. Often
1537 signals are used primarily to interrupt blocking syscalls. */
1539 TranslationBlock
*tb
;
1540 static spinlock_t interrupt_lock
= SPIN_LOCK_UNLOCKED
;
1542 tb
= env
->current_tb
;
1543 /* if the cpu is currently executing code, we must unlink it and
1544 all the potentially executing TB */
1545 if (tb
&& !testandset(&interrupt_lock
)) {
1546 env
->current_tb
= NULL
;
1547 tb_reset_jump_recursive(tb
);
1548 resetlock(&interrupt_lock
);
1553 /* mask must never be zero, except for A20 change call */
1554 void cpu_interrupt(CPUState
*env
, int mask
)
1558 old_mask
= env
->interrupt_request
;
1559 env
->interrupt_request
|= mask
;
1561 #ifndef CONFIG_USER_ONLY
1563 * If called from iothread context, wake the target cpu in
1566 if (!qemu_cpu_self(env
)) {
1573 env
->icount_decr
.u16
.high
= 0xffff;
1574 #ifndef CONFIG_USER_ONLY
1576 && (mask
& ~old_mask
) != 0) {
1577 cpu_abort(env
, "Raised interrupt while not in I/O function");
1585 void cpu_reset_interrupt(CPUState
*env
, int mask
)
1587 env
->interrupt_request
&= ~mask
;
1590 void cpu_exit(CPUState
*env
)
1592 env
->exit_request
= 1;
1596 const CPULogItem cpu_log_items
[] = {
1597 { CPU_LOG_TB_OUT_ASM
, "out_asm",
1598 "show generated host assembly code for each compiled TB" },
1599 { CPU_LOG_TB_IN_ASM
, "in_asm",
1600 "show target assembly code for each compiled TB" },
1601 { CPU_LOG_TB_OP
, "op",
1602 "show micro ops for each compiled TB" },
1603 { CPU_LOG_TB_OP_OPT
, "op_opt",
1606 "before eflags optimization and "
1608 "after liveness analysis" },
1609 { CPU_LOG_INT
, "int",
1610 "show interrupts/exceptions in short format" },
1611 { CPU_LOG_EXEC
, "exec",
1612 "show trace before each executed TB (lots of logs)" },
1613 { CPU_LOG_TB_CPU
, "cpu",
1614 "show CPU state before block translation" },
1616 { CPU_LOG_PCALL
, "pcall",
1617 "show protected mode far calls/returns/exceptions" },
1618 { CPU_LOG_RESET
, "cpu_reset",
1619 "show CPU state before CPU resets" },
1622 { CPU_LOG_IOPORT
, "ioport",
1623 "show all i/o ports accesses" },
1628 static int cmp1(const char *s1
, int n
, const char *s2
)
1630 if (strlen(s2
) != n
)
1632 return memcmp(s1
, s2
, n
) == 0;
1635 /* takes a comma separated list of log masks. Return 0 if error. */
1636 int cpu_str_to_log_mask(const char *str
)
1638 const CPULogItem
*item
;
1645 p1
= strchr(p
, ',');
1648 if(cmp1(p
,p1
-p
,"all")) {
1649 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1653 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1654 if (cmp1(p
, p1
- p
, item
->name
))
1668 void cpu_abort(CPUState
*env
, const char *fmt
, ...)
1675 fprintf(stderr
, "qemu: fatal: ");
1676 vfprintf(stderr
, fmt
, ap
);
1677 fprintf(stderr
, "\n");
1679 cpu_dump_state(env
, stderr
, fprintf
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1681 cpu_dump_state(env
, stderr
, fprintf
, 0);
1683 if (qemu_log_enabled()) {
1684 qemu_log("qemu: fatal: ");
1685 qemu_log_vprintf(fmt
, ap2
);
1688 log_cpu_state(env
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1690 log_cpu_state(env
, 0);
1700 CPUState
*cpu_copy(CPUState
*env
)
1702 CPUState
*new_env
= cpu_init(env
->cpu_model_str
);
1703 CPUState
*next_cpu
= new_env
->next_cpu
;
1704 int cpu_index
= new_env
->cpu_index
;
1705 #if defined(TARGET_HAS_ICE)
1710 memcpy(new_env
, env
, sizeof(CPUState
));
1712 /* Preserve chaining and index. */
1713 new_env
->next_cpu
= next_cpu
;
1714 new_env
->cpu_index
= cpu_index
;
1716 /* Clone all break/watchpoints.
1717 Note: Once we support ptrace with hw-debug register access, make sure
1718 BP_CPU break/watchpoints are handled correctly on clone. */
1719 QTAILQ_INIT(&env
->breakpoints
);
1720 QTAILQ_INIT(&env
->watchpoints
);
1721 #if defined(TARGET_HAS_ICE)
1722 QTAILQ_FOREACH(bp
, &env
->breakpoints
, entry
) {
1723 cpu_breakpoint_insert(new_env
, bp
->pc
, bp
->flags
, NULL
);
1725 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1726 cpu_watchpoint_insert(new_env
, wp
->vaddr
, (~wp
->len_mask
) + 1,
1734 #if !defined(CONFIG_USER_ONLY)
1736 static inline void tlb_flush_jmp_cache(CPUState
*env
, target_ulong addr
)
1740 /* Discard jump cache entries for any tb which might potentially
1741 overlap the flushed page. */
1742 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1743 memset (&env
->tb_jmp_cache
[i
], 0,
1744 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1746 i
= tb_jmp_cache_hash_page(addr
);
1747 memset (&env
->tb_jmp_cache
[i
], 0,
1748 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1751 static CPUTLBEntry s_cputlb_empty_entry
= {
1758 /* NOTE: if flush_global is true, also flush global entries (not
1760 void tlb_flush(CPUState
*env
, int flush_global
)
1764 #if defined(DEBUG_TLB)
1765 printf("tlb_flush:\n");
1767 /* must reset current TB so that interrupts cannot modify the
1768 links while we are modifying them */
1769 env
->current_tb
= NULL
;
1771 for(i
= 0; i
< CPU_TLB_SIZE
; i
++) {
1773 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
1774 env
->tlb_table
[mmu_idx
][i
] = s_cputlb_empty_entry
;
1778 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
1783 static inline void tlb_flush_entry(CPUTLBEntry
*tlb_entry
, target_ulong addr
)
1785 if (addr
== (tlb_entry
->addr_read
&
1786 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1787 addr
== (tlb_entry
->addr_write
&
1788 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1789 addr
== (tlb_entry
->addr_code
&
1790 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
))) {
1791 *tlb_entry
= s_cputlb_empty_entry
;
1795 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
1800 #if defined(DEBUG_TLB)
1801 printf("tlb_flush_page: " TARGET_FMT_lx
"\n", addr
);
1803 /* must reset current TB so that interrupts cannot modify the
1804 links while we are modifying them */
1805 env
->current_tb
= NULL
;
1807 addr
&= TARGET_PAGE_MASK
;
1808 i
= (addr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
1809 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++)
1810 tlb_flush_entry(&env
->tlb_table
[mmu_idx
][i
], addr
);
1812 tlb_flush_jmp_cache(env
, addr
);
1815 /* update the TLBs so that writes to code in the virtual page 'addr'
1817 static void tlb_protect_code(ram_addr_t ram_addr
)
1819 cpu_physical_memory_reset_dirty(ram_addr
,
1820 ram_addr
+ TARGET_PAGE_SIZE
,
1824 /* update the TLB so that writes in physical page 'phys_addr' are no longer
1825 tested for self modifying code */
1826 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
1829 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] |= CODE_DIRTY_FLAG
;
1832 static inline void tlb_reset_dirty_range(CPUTLBEntry
*tlb_entry
,
1833 unsigned long start
, unsigned long length
)
1836 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
) {
1837 addr
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) + tlb_entry
->addend
;
1838 if ((addr
- start
) < length
) {
1839 tlb_entry
->addr_write
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) | TLB_NOTDIRTY
;
1844 /* Note: start and end must be within the same ram block. */
1845 void cpu_physical_memory_reset_dirty(ram_addr_t start
, ram_addr_t end
,
1849 unsigned long length
, start1
;
1853 start
&= TARGET_PAGE_MASK
;
1854 end
= TARGET_PAGE_ALIGN(end
);
1856 length
= end
- start
;
1859 len
= length
>> TARGET_PAGE_BITS
;
1860 mask
= ~dirty_flags
;
1861 p
= phys_ram_dirty
+ (start
>> TARGET_PAGE_BITS
);
1862 for(i
= 0; i
< len
; i
++)
1865 /* we modify the TLB cache so that the dirty bit will be set again
1866 when accessing the range */
1867 start1
= (unsigned long)qemu_get_ram_ptr(start
);
1868 /* Chek that we don't span multiple blocks - this breaks the
1869 address comparisons below. */
1870 if ((unsigned long)qemu_get_ram_ptr(end
- 1) - start1
1871 != (end
- 1) - start
) {
1875 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
1877 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
1878 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1879 tlb_reset_dirty_range(&env
->tlb_table
[mmu_idx
][i
],
1885 int cpu_physical_memory_set_dirty_tracking(int enable
)
1887 in_migration
= enable
;
1888 if (kvm_enabled()) {
1889 return kvm_set_migration_log(enable
);
1894 int cpu_physical_memory_get_dirty_tracking(void)
1896 return in_migration
;
1899 int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr
,
1900 target_phys_addr_t end_addr
)
1905 ret
= kvm_physical_sync_dirty_bitmap(start_addr
, end_addr
);
1909 static inline void tlb_update_dirty(CPUTLBEntry
*tlb_entry
)
1911 ram_addr_t ram_addr
;
1914 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
) {
1915 p
= (void *)(unsigned long)((tlb_entry
->addr_write
& TARGET_PAGE_MASK
)
1916 + tlb_entry
->addend
);
1917 ram_addr
= qemu_ram_addr_from_host(p
);
1918 if (!cpu_physical_memory_is_dirty(ram_addr
)) {
1919 tlb_entry
->addr_write
|= TLB_NOTDIRTY
;
1924 /* update the TLB according to the current state of the dirty bits */
1925 void cpu_tlb_update_dirty(CPUState
*env
)
1929 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
1930 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1931 tlb_update_dirty(&env
->tlb_table
[mmu_idx
][i
]);
1935 static inline void tlb_set_dirty1(CPUTLBEntry
*tlb_entry
, target_ulong vaddr
)
1937 if (tlb_entry
->addr_write
== (vaddr
| TLB_NOTDIRTY
))
1938 tlb_entry
->addr_write
= vaddr
;
1941 /* update the TLB corresponding to virtual page vaddr
1942 so that it is no longer dirty */
1943 static inline void tlb_set_dirty(CPUState
*env
, target_ulong vaddr
)
1948 vaddr
&= TARGET_PAGE_MASK
;
1949 i
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
1950 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++)
1951 tlb_set_dirty1(&env
->tlb_table
[mmu_idx
][i
], vaddr
);
1954 /* add a new TLB entry. At most one entry for a given virtual address
1955 is permitted. Return 0 if OK or 2 if the page could not be mapped
1956 (can only happen in non SOFTMMU mode for I/O pages or pages
1957 conflicting with the host address space). */
1958 int tlb_set_page_exec(CPUState
*env
, target_ulong vaddr
,
1959 target_phys_addr_t paddr
, int prot
,
1960 int mmu_idx
, int is_softmmu
)
1965 target_ulong address
;
1966 target_ulong code_address
;
1967 target_phys_addr_t addend
;
1971 target_phys_addr_t iotlb
;
1973 p
= phys_page_find(paddr
>> TARGET_PAGE_BITS
);
1975 pd
= IO_MEM_UNASSIGNED
;
1977 pd
= p
->phys_offset
;
1979 #if defined(DEBUG_TLB)
1980 printf("tlb_set_page: vaddr=" TARGET_FMT_lx
" paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
1981 vaddr
, (int)paddr
, prot
, mmu_idx
, is_softmmu
, pd
);
1986 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&& !(pd
& IO_MEM_ROMD
)) {
1987 /* IO memory case (romd handled later) */
1988 address
|= TLB_MMIO
;
1990 addend
= (unsigned long)qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
);
1991 if ((pd
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
) {
1993 iotlb
= pd
& TARGET_PAGE_MASK
;
1994 if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
)
1995 iotlb
|= IO_MEM_NOTDIRTY
;
1997 iotlb
|= IO_MEM_ROM
;
1999 /* IO handlers are currently passed a physical address.
2000 It would be nice to pass an offset from the base address
2001 of that region. This would avoid having to special case RAM,
2002 and avoid full address decoding in every device.
2003 We can't use the high bits of pd for this because
2004 IO_MEM_ROMD uses these as a ram address. */
2005 iotlb
= (pd
& ~TARGET_PAGE_MASK
);
2007 iotlb
+= p
->region_offset
;
2013 code_address
= address
;
2014 /* Make accesses to pages with watchpoints go via the
2015 watchpoint trap routines. */
2016 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
2017 if (vaddr
== (wp
->vaddr
& TARGET_PAGE_MASK
)) {
2018 iotlb
= io_mem_watch
+ paddr
;
2019 /* TODO: The memory case can be optimized by not trapping
2020 reads of pages with a write breakpoint. */
2021 address
|= TLB_MMIO
;
2025 index
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
2026 env
->iotlb
[mmu_idx
][index
] = iotlb
- vaddr
;
2027 te
= &env
->tlb_table
[mmu_idx
][index
];
2028 te
->addend
= addend
- vaddr
;
2029 if (prot
& PAGE_READ
) {
2030 te
->addr_read
= address
;
2035 if (prot
& PAGE_EXEC
) {
2036 te
->addr_code
= code_address
;
2040 if (prot
& PAGE_WRITE
) {
2041 if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_ROM
||
2042 (pd
& IO_MEM_ROMD
)) {
2043 /* Write access calls the I/O callback. */
2044 te
->addr_write
= address
| TLB_MMIO
;
2045 } else if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
&&
2046 !cpu_physical_memory_is_dirty(pd
)) {
2047 te
->addr_write
= address
| TLB_NOTDIRTY
;
2049 te
->addr_write
= address
;
2052 te
->addr_write
= -1;
2059 void tlb_flush(CPUState
*env
, int flush_global
)
2063 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
2067 int tlb_set_page_exec(CPUState
*env
, target_ulong vaddr
,
2068 target_phys_addr_t paddr
, int prot
,
2069 int mmu_idx
, int is_softmmu
)
2075 * Walks guest process memory "regions" one by one
2076 * and calls callback function 'fn' for each region.
2078 int walk_memory_regions(void *priv
,
2079 int (*fn
)(void *, unsigned long, unsigned long, unsigned long))
2081 unsigned long start
, end
;
2083 int i
, j
, prot
, prot1
;
2089 for (i
= 0; i
<= L1_SIZE
; i
++) {
2090 p
= (i
< L1_SIZE
) ? l1_map
[i
] : NULL
;
2091 for (j
= 0; j
< L2_SIZE
; j
++) {
2092 prot1
= (p
== NULL
) ? 0 : p
[j
].flags
;
2094 * "region" is one continuous chunk of memory
2095 * that has same protection flags set.
2097 if (prot1
!= prot
) {
2098 end
= (i
<< (32 - L1_BITS
)) | (j
<< TARGET_PAGE_BITS
);
2100 rc
= (*fn
)(priv
, start
, end
, prot
);
2101 /* callback can stop iteration by returning != 0 */
2118 static int dump_region(void *priv
, unsigned long start
,
2119 unsigned long end
, unsigned long prot
)
2121 FILE *f
= (FILE *)priv
;
2123 (void) fprintf(f
, "%08lx-%08lx %08lx %c%c%c\n",
2124 start
, end
, end
- start
,
2125 ((prot
& PAGE_READ
) ? 'r' : '-'),
2126 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
2127 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
2132 /* dump memory mappings */
2133 void page_dump(FILE *f
)
2135 (void) fprintf(f
, "%-8s %-8s %-8s %s\n",
2136 "start", "end", "size", "prot");
2137 walk_memory_regions(f
, dump_region
);
2140 int page_get_flags(target_ulong address
)
2144 p
= page_find(address
>> TARGET_PAGE_BITS
);
2150 /* modify the flags of a page and invalidate the code if
2151 necessary. The flag PAGE_WRITE_ORG is positioned automatically
2152 depending on PAGE_WRITE */
2153 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
2158 /* mmap_lock should already be held. */
2159 start
= start
& TARGET_PAGE_MASK
;
2160 end
= TARGET_PAGE_ALIGN(end
);
2161 if (flags
& PAGE_WRITE
)
2162 flags
|= PAGE_WRITE_ORG
;
2163 for(addr
= start
; addr
< end
; addr
+= TARGET_PAGE_SIZE
) {
2164 p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
);
2165 /* We may be called for host regions that are outside guest
2169 /* if the write protection is set, then we invalidate the code
2171 if (!(p
->flags
& PAGE_WRITE
) &&
2172 (flags
& PAGE_WRITE
) &&
2174 tb_invalidate_phys_page(addr
, 0, NULL
);
2180 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
2186 if (start
+ len
< start
)
2187 /* we've wrapped around */
2190 end
= TARGET_PAGE_ALIGN(start
+len
); /* must do before we loose bits in the next step */
2191 start
= start
& TARGET_PAGE_MASK
;
2193 for(addr
= start
; addr
< end
; addr
+= TARGET_PAGE_SIZE
) {
2194 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2197 if( !(p
->flags
& PAGE_VALID
) )
2200 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
))
2202 if (flags
& PAGE_WRITE
) {
2203 if (!(p
->flags
& PAGE_WRITE_ORG
))
2205 /* unprotect the page if it was put read-only because it
2206 contains translated code */
2207 if (!(p
->flags
& PAGE_WRITE
)) {
2208 if (!page_unprotect(addr
, 0, NULL
))
2217 /* called from signal handler: invalidate the code and unprotect the
2218 page. Return TRUE if the fault was successfully handled. */
2219 int page_unprotect(target_ulong address
, unsigned long pc
, void *puc
)
2221 unsigned int page_index
, prot
, pindex
;
2223 target_ulong host_start
, host_end
, addr
;
2225 /* Technically this isn't safe inside a signal handler. However we
2226 know this only ever happens in a synchronous SEGV handler, so in
2227 practice it seems to be ok. */
2230 host_start
= address
& qemu_host_page_mask
;
2231 page_index
= host_start
>> TARGET_PAGE_BITS
;
2232 p1
= page_find(page_index
);
2237 host_end
= host_start
+ qemu_host_page_size
;
2240 for(addr
= host_start
;addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2244 /* if the page was really writable, then we change its
2245 protection back to writable */
2246 if (prot
& PAGE_WRITE_ORG
) {
2247 pindex
= (address
- host_start
) >> TARGET_PAGE_BITS
;
2248 if (!(p1
[pindex
].flags
& PAGE_WRITE
)) {
2249 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2250 (prot
& PAGE_BITS
) | PAGE_WRITE
);
2251 p1
[pindex
].flags
|= PAGE_WRITE
;
2252 /* and since the content will be modified, we must invalidate
2253 the corresponding translated code. */
2254 tb_invalidate_phys_page(address
, pc
, puc
);
2255 #ifdef DEBUG_TB_CHECK
2256 tb_invalidate_check(address
);
2266 static inline void tlb_set_dirty(CPUState
*env
,
2267 unsigned long addr
, target_ulong vaddr
)
2270 #endif /* defined(CONFIG_USER_ONLY) */
2272 #if !defined(CONFIG_USER_ONLY)
2274 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2275 ram_addr_t memory
, ram_addr_t region_offset
);
2276 static void *subpage_init (target_phys_addr_t base
, ram_addr_t
*phys
,
2277 ram_addr_t orig_memory
, ram_addr_t region_offset
);
2278 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2281 if (addr > start_addr) \
2284 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2285 if (start_addr2 > 0) \
2289 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2290 end_addr2 = TARGET_PAGE_SIZE - 1; \
2292 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2293 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2298 /* register physical memory.
2299 For RAM, 'size' must be a multiple of the target page size.
2300 If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2301 io memory page. The address used when calling the IO function is
2302 the offset from the start of the region, plus region_offset. Both
2303 start_addr and region_offset are rounded down to a page boundary
2304 before calculating this offset. This should not be a problem unless
2305 the low bits of start_addr and region_offset differ. */
2306 void cpu_register_physical_memory_offset(target_phys_addr_t start_addr
,
2308 ram_addr_t phys_offset
,
2309 ram_addr_t region_offset
)
2311 target_phys_addr_t addr
, end_addr
;
2314 ram_addr_t orig_size
= size
;
2318 kvm_set_phys_mem(start_addr
, size
, phys_offset
);
2320 if (phys_offset
== IO_MEM_UNASSIGNED
) {
2321 region_offset
= start_addr
;
2323 region_offset
&= TARGET_PAGE_MASK
;
2324 size
= (size
+ TARGET_PAGE_SIZE
- 1) & TARGET_PAGE_MASK
;
2325 end_addr
= start_addr
+ (target_phys_addr_t
)size
;
2326 for(addr
= start_addr
; addr
!= end_addr
; addr
+= TARGET_PAGE_SIZE
) {
2327 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2328 if (p
&& p
->phys_offset
!= IO_MEM_UNASSIGNED
) {
2329 ram_addr_t orig_memory
= p
->phys_offset
;
2330 target_phys_addr_t start_addr2
, end_addr2
;
2331 int need_subpage
= 0;
2333 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
, end_addr2
,
2335 if (need_subpage
|| phys_offset
& IO_MEM_SUBWIDTH
) {
2336 if (!(orig_memory
& IO_MEM_SUBPAGE
)) {
2337 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2338 &p
->phys_offset
, orig_memory
,
2341 subpage
= io_mem_opaque
[(orig_memory
& ~TARGET_PAGE_MASK
)
2344 subpage_register(subpage
, start_addr2
, end_addr2
, phys_offset
,
2346 p
->region_offset
= 0;
2348 p
->phys_offset
= phys_offset
;
2349 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
||
2350 (phys_offset
& IO_MEM_ROMD
))
2351 phys_offset
+= TARGET_PAGE_SIZE
;
2354 p
= phys_page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2355 p
->phys_offset
= phys_offset
;
2356 p
->region_offset
= region_offset
;
2357 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
||
2358 (phys_offset
& IO_MEM_ROMD
)) {
2359 phys_offset
+= TARGET_PAGE_SIZE
;
2361 target_phys_addr_t start_addr2
, end_addr2
;
2362 int need_subpage
= 0;
2364 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
,
2365 end_addr2
, need_subpage
);
2367 if (need_subpage
|| phys_offset
& IO_MEM_SUBWIDTH
) {
2368 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2369 &p
->phys_offset
, IO_MEM_UNASSIGNED
,
2370 addr
& TARGET_PAGE_MASK
);
2371 subpage_register(subpage
, start_addr2
, end_addr2
,
2372 phys_offset
, region_offset
);
2373 p
->region_offset
= 0;
2377 region_offset
+= TARGET_PAGE_SIZE
;
2380 /* since each CPU stores ram addresses in its TLB cache, we must
2381 reset the modified entries */
2383 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
2388 /* XXX: temporary until new memory mapping API */
2389 ram_addr_t
cpu_get_physical_page_desc(target_phys_addr_t addr
)
2393 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2395 return IO_MEM_UNASSIGNED
;
2396 return p
->phys_offset
;
2399 void qemu_register_coalesced_mmio(target_phys_addr_t addr
, ram_addr_t size
)
2402 kvm_coalesce_mmio_region(addr
, size
);
2405 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr
, ram_addr_t size
)
2408 kvm_uncoalesce_mmio_region(addr
, size
);
2411 ram_addr_t
qemu_ram_alloc(ram_addr_t size
)
2413 RAMBlock
*new_block
;
2415 size
= TARGET_PAGE_ALIGN(size
);
2416 new_block
= qemu_malloc(sizeof(*new_block
));
2418 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2419 /* XXX S390 KVM requires the topmost vma of the RAM to be < 256GB */
2420 new_block
->host
= mmap((void*)0x1000000, size
, PROT_EXEC
|PROT_READ
|PROT_WRITE
,
2421 MAP_SHARED
| MAP_ANONYMOUS
, -1, 0);
2423 new_block
->host
= qemu_vmalloc(size
);
2425 #ifdef MADV_MERGEABLE
2426 madvise(new_block
->host
, size
, MADV_MERGEABLE
);
2428 new_block
->offset
= last_ram_offset
;
2429 new_block
->length
= size
;
2431 new_block
->next
= ram_blocks
;
2432 ram_blocks
= new_block
;
2434 phys_ram_dirty
= qemu_realloc(phys_ram_dirty
,
2435 (last_ram_offset
+ size
) >> TARGET_PAGE_BITS
);
2436 memset(phys_ram_dirty
+ (last_ram_offset
>> TARGET_PAGE_BITS
),
2437 0xff, size
>> TARGET_PAGE_BITS
);
2439 last_ram_offset
+= size
;
2442 kvm_setup_guest_memory(new_block
->host
, size
);
2444 return new_block
->offset
;
2447 void qemu_ram_free(ram_addr_t addr
)
2449 /* TODO: implement this. */
2452 /* Return a host pointer to ram allocated with qemu_ram_alloc.
2453 With the exception of the softmmu code in this file, this should
2454 only be used for local memory (e.g. video ram) that the device owns,
2455 and knows it isn't going to access beyond the end of the block.
2457 It should not be used for general purpose DMA.
2458 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
2460 void *qemu_get_ram_ptr(ram_addr_t addr
)
2467 prevp
= &ram_blocks
;
2469 while (block
&& (block
->offset
> addr
2470 || block
->offset
+ block
->length
<= addr
)) {
2472 prevp
= &prev
->next
;
2474 block
= block
->next
;
2477 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
2480 /* Move this entry to to start of the list. */
2482 prev
->next
= block
->next
;
2483 block
->next
= *prevp
;
2486 return block
->host
+ (addr
- block
->offset
);
2489 /* Some of the softmmu routines need to translate from a host pointer
2490 (typically a TLB entry) back to a ram offset. */
2491 ram_addr_t
qemu_ram_addr_from_host(void *ptr
)
2496 uint8_t *host
= ptr
;
2499 prevp
= &ram_blocks
;
2501 while (block
&& (block
->host
> host
2502 || block
->host
+ block
->length
<= host
)) {
2504 prevp
= &prev
->next
;
2506 block
= block
->next
;
2509 fprintf(stderr
, "Bad ram pointer %p\n", ptr
);
2512 return block
->offset
+ (host
- block
->host
);
2515 static uint32_t unassigned_mem_readb(void *opaque
, target_phys_addr_t addr
)
2517 #ifdef DEBUG_UNASSIGNED
2518 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2520 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2521 do_unassigned_access(addr
, 0, 0, 0, 1);
2526 static uint32_t unassigned_mem_readw(void *opaque
, target_phys_addr_t addr
)
2528 #ifdef DEBUG_UNASSIGNED
2529 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2531 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2532 do_unassigned_access(addr
, 0, 0, 0, 2);
2537 static uint32_t unassigned_mem_readl(void *opaque
, target_phys_addr_t addr
)
2539 #ifdef DEBUG_UNASSIGNED
2540 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2542 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2543 do_unassigned_access(addr
, 0, 0, 0, 4);
2548 static void unassigned_mem_writeb(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2550 #ifdef DEBUG_UNASSIGNED
2551 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2553 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2554 do_unassigned_access(addr
, 1, 0, 0, 1);
2558 static void unassigned_mem_writew(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2560 #ifdef DEBUG_UNASSIGNED
2561 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2563 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2564 do_unassigned_access(addr
, 1, 0, 0, 2);
2568 static void unassigned_mem_writel(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2570 #ifdef DEBUG_UNASSIGNED
2571 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2573 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2574 do_unassigned_access(addr
, 1, 0, 0, 4);
2578 static CPUReadMemoryFunc
* const unassigned_mem_read
[3] = {
2579 unassigned_mem_readb
,
2580 unassigned_mem_readw
,
2581 unassigned_mem_readl
,
2584 static CPUWriteMemoryFunc
* const unassigned_mem_write
[3] = {
2585 unassigned_mem_writeb
,
2586 unassigned_mem_writew
,
2587 unassigned_mem_writel
,
2590 static void notdirty_mem_writeb(void *opaque
, target_phys_addr_t ram_addr
,
2594 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2595 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
2596 #if !defined(CONFIG_USER_ONLY)
2597 tb_invalidate_phys_page_fast(ram_addr
, 1);
2598 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2601 stb_p(qemu_get_ram_ptr(ram_addr
), val
);
2602 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
2603 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
2604 /* we remove the notdirty callback only if the code has been
2606 if (dirty_flags
== 0xff)
2607 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
2610 static void notdirty_mem_writew(void *opaque
, target_phys_addr_t ram_addr
,
2614 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2615 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
2616 #if !defined(CONFIG_USER_ONLY)
2617 tb_invalidate_phys_page_fast(ram_addr
, 2);
2618 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2621 stw_p(qemu_get_ram_ptr(ram_addr
), val
);
2622 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
2623 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
2624 /* we remove the notdirty callback only if the code has been
2626 if (dirty_flags
== 0xff)
2627 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
2630 static void notdirty_mem_writel(void *opaque
, target_phys_addr_t ram_addr
,
2634 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2635 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
2636 #if !defined(CONFIG_USER_ONLY)
2637 tb_invalidate_phys_page_fast(ram_addr
, 4);
2638 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2641 stl_p(qemu_get_ram_ptr(ram_addr
), val
);
2642 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
2643 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
2644 /* we remove the notdirty callback only if the code has been
2646 if (dirty_flags
== 0xff)
2647 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
2650 static CPUReadMemoryFunc
* const error_mem_read
[3] = {
2651 NULL
, /* never used */
2652 NULL
, /* never used */
2653 NULL
, /* never used */
2656 static CPUWriteMemoryFunc
* const notdirty_mem_write
[3] = {
2657 notdirty_mem_writeb
,
2658 notdirty_mem_writew
,
2659 notdirty_mem_writel
,
2662 /* Generate a debug exception if a watchpoint has been hit. */
2663 static void check_watchpoint(int offset
, int len_mask
, int flags
)
2665 CPUState
*env
= cpu_single_env
;
2666 target_ulong pc
, cs_base
;
2667 TranslationBlock
*tb
;
2672 if (env
->watchpoint_hit
) {
2673 /* We re-entered the check after replacing the TB. Now raise
2674 * the debug interrupt so that is will trigger after the
2675 * current instruction. */
2676 cpu_interrupt(env
, CPU_INTERRUPT_DEBUG
);
2679 vaddr
= (env
->mem_io_vaddr
& TARGET_PAGE_MASK
) + offset
;
2680 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
2681 if ((vaddr
== (wp
->vaddr
& len_mask
) ||
2682 (vaddr
& wp
->len_mask
) == wp
->vaddr
) && (wp
->flags
& flags
)) {
2683 wp
->flags
|= BP_WATCHPOINT_HIT
;
2684 if (!env
->watchpoint_hit
) {
2685 env
->watchpoint_hit
= wp
;
2686 tb
= tb_find_pc(env
->mem_io_pc
);
2688 cpu_abort(env
, "check_watchpoint: could not find TB for "
2689 "pc=%p", (void *)env
->mem_io_pc
);
2691 cpu_restore_state(tb
, env
, env
->mem_io_pc
, NULL
);
2692 tb_phys_invalidate(tb
, -1);
2693 if (wp
->flags
& BP_STOP_BEFORE_ACCESS
) {
2694 env
->exception_index
= EXCP_DEBUG
;
2696 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &cpu_flags
);
2697 tb_gen_code(env
, pc
, cs_base
, cpu_flags
, 1);
2699 cpu_resume_from_signal(env
, NULL
);
2702 wp
->flags
&= ~BP_WATCHPOINT_HIT
;
2707 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2708 so these check for a hit then pass through to the normal out-of-line
2710 static uint32_t watch_mem_readb(void *opaque
, target_phys_addr_t addr
)
2712 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x0, BP_MEM_READ
);
2713 return ldub_phys(addr
);
2716 static uint32_t watch_mem_readw(void *opaque
, target_phys_addr_t addr
)
2718 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x1, BP_MEM_READ
);
2719 return lduw_phys(addr
);
2722 static uint32_t watch_mem_readl(void *opaque
, target_phys_addr_t addr
)
2724 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x3, BP_MEM_READ
);
2725 return ldl_phys(addr
);
2728 static void watch_mem_writeb(void *opaque
, target_phys_addr_t addr
,
2731 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x0, BP_MEM_WRITE
);
2732 stb_phys(addr
, val
);
2735 static void watch_mem_writew(void *opaque
, target_phys_addr_t addr
,
2738 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x1, BP_MEM_WRITE
);
2739 stw_phys(addr
, val
);
2742 static void watch_mem_writel(void *opaque
, target_phys_addr_t addr
,
2745 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x3, BP_MEM_WRITE
);
2746 stl_phys(addr
, val
);
2749 static CPUReadMemoryFunc
* const watch_mem_read
[3] = {
2755 static CPUWriteMemoryFunc
* const watch_mem_write
[3] = {
2761 static inline uint32_t subpage_readlen (subpage_t
*mmio
, target_phys_addr_t addr
,
2767 idx
= SUBPAGE_IDX(addr
);
2768 #if defined(DEBUG_SUBPAGE)
2769 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d\n", __func__
,
2770 mmio
, len
, addr
, idx
);
2772 ret
= (**mmio
->mem_read
[idx
][len
])(mmio
->opaque
[idx
][0][len
],
2773 addr
+ mmio
->region_offset
[idx
][0][len
]);
2778 static inline void subpage_writelen (subpage_t
*mmio
, target_phys_addr_t addr
,
2779 uint32_t value
, unsigned int len
)
2783 idx
= SUBPAGE_IDX(addr
);
2784 #if defined(DEBUG_SUBPAGE)
2785 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d value %08x\n", __func__
,
2786 mmio
, len
, addr
, idx
, value
);
2788 (**mmio
->mem_write
[idx
][len
])(mmio
->opaque
[idx
][1][len
],
2789 addr
+ mmio
->region_offset
[idx
][1][len
],
2793 static uint32_t subpage_readb (void *opaque
, target_phys_addr_t addr
)
2795 #if defined(DEBUG_SUBPAGE)
2796 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
2799 return subpage_readlen(opaque
, addr
, 0);
2802 static void subpage_writeb (void *opaque
, target_phys_addr_t addr
,
2805 #if defined(DEBUG_SUBPAGE)
2806 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
2808 subpage_writelen(opaque
, addr
, value
, 0);
2811 static uint32_t subpage_readw (void *opaque
, target_phys_addr_t addr
)
2813 #if defined(DEBUG_SUBPAGE)
2814 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
2817 return subpage_readlen(opaque
, addr
, 1);
2820 static void subpage_writew (void *opaque
, target_phys_addr_t addr
,
2823 #if defined(DEBUG_SUBPAGE)
2824 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
2826 subpage_writelen(opaque
, addr
, value
, 1);
2829 static uint32_t subpage_readl (void *opaque
, target_phys_addr_t addr
)
2831 #if defined(DEBUG_SUBPAGE)
2832 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
2835 return subpage_readlen(opaque
, addr
, 2);
2838 static void subpage_writel (void *opaque
,
2839 target_phys_addr_t addr
, uint32_t value
)
2841 #if defined(DEBUG_SUBPAGE)
2842 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
2844 subpage_writelen(opaque
, addr
, value
, 2);
2847 static CPUReadMemoryFunc
* const subpage_read
[] = {
2853 static CPUWriteMemoryFunc
* const subpage_write
[] = {
2859 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2860 ram_addr_t memory
, ram_addr_t region_offset
)
2865 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
2867 idx
= SUBPAGE_IDX(start
);
2868 eidx
= SUBPAGE_IDX(end
);
2869 #if defined(DEBUG_SUBPAGE)
2870 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__
,
2871 mmio
, start
, end
, idx
, eidx
, memory
);
2873 memory
>>= IO_MEM_SHIFT
;
2874 for (; idx
<= eidx
; idx
++) {
2875 for (i
= 0; i
< 4; i
++) {
2876 if (io_mem_read
[memory
][i
]) {
2877 mmio
->mem_read
[idx
][i
] = &io_mem_read
[memory
][i
];
2878 mmio
->opaque
[idx
][0][i
] = io_mem_opaque
[memory
];
2879 mmio
->region_offset
[idx
][0][i
] = region_offset
;
2881 if (io_mem_write
[memory
][i
]) {
2882 mmio
->mem_write
[idx
][i
] = &io_mem_write
[memory
][i
];
2883 mmio
->opaque
[idx
][1][i
] = io_mem_opaque
[memory
];
2884 mmio
->region_offset
[idx
][1][i
] = region_offset
;
2892 static void *subpage_init (target_phys_addr_t base
, ram_addr_t
*phys
,
2893 ram_addr_t orig_memory
, ram_addr_t region_offset
)
2898 mmio
= qemu_mallocz(sizeof(subpage_t
));
2901 subpage_memory
= cpu_register_io_memory(subpage_read
, subpage_write
, mmio
);
2902 #if defined(DEBUG_SUBPAGE)
2903 printf("%s: %p base " TARGET_FMT_plx
" len %08x %d\n", __func__
,
2904 mmio
, base
, TARGET_PAGE_SIZE
, subpage_memory
);
2906 *phys
= subpage_memory
| IO_MEM_SUBPAGE
;
2907 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
- 1, orig_memory
,
2913 static int get_free_io_mem_idx(void)
2917 for (i
= 0; i
<IO_MEM_NB_ENTRIES
; i
++)
2918 if (!io_mem_used
[i
]) {
2926 /* mem_read and mem_write are arrays of functions containing the
2927 function to access byte (index 0), word (index 1) and dword (index
2928 2). Functions can be omitted with a NULL function pointer.
2929 If io_index is non zero, the corresponding io zone is
2930 modified. If it is zero, a new io zone is allocated. The return
2931 value can be used with cpu_register_physical_memory(). (-1) is
2932 returned if error. */
2933 static int cpu_register_io_memory_fixed(int io_index
,
2934 CPUReadMemoryFunc
* const *mem_read
,
2935 CPUWriteMemoryFunc
* const *mem_write
,
2938 int i
, subwidth
= 0;
2940 if (io_index
<= 0) {
2941 io_index
= get_free_io_mem_idx();
2945 io_index
>>= IO_MEM_SHIFT
;
2946 if (io_index
>= IO_MEM_NB_ENTRIES
)
2950 for(i
= 0;i
< 3; i
++) {
2951 if (!mem_read
[i
] || !mem_write
[i
])
2952 subwidth
= IO_MEM_SUBWIDTH
;
2953 io_mem_read
[io_index
][i
] = mem_read
[i
];
2954 io_mem_write
[io_index
][i
] = mem_write
[i
];
2956 io_mem_opaque
[io_index
] = opaque
;
2957 return (io_index
<< IO_MEM_SHIFT
) | subwidth
;
2960 int cpu_register_io_memory(CPUReadMemoryFunc
* const *mem_read
,
2961 CPUWriteMemoryFunc
* const *mem_write
,
2964 return cpu_register_io_memory_fixed(0, mem_read
, mem_write
, opaque
);
2967 void cpu_unregister_io_memory(int io_table_address
)
2970 int io_index
= io_table_address
>> IO_MEM_SHIFT
;
2972 for (i
=0;i
< 3; i
++) {
2973 io_mem_read
[io_index
][i
] = unassigned_mem_read
[i
];
2974 io_mem_write
[io_index
][i
] = unassigned_mem_write
[i
];
2976 io_mem_opaque
[io_index
] = NULL
;
2977 io_mem_used
[io_index
] = 0;
2980 static void io_mem_init(void)
2984 cpu_register_io_memory_fixed(IO_MEM_ROM
, error_mem_read
, unassigned_mem_write
, NULL
);
2985 cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED
, unassigned_mem_read
, unassigned_mem_write
, NULL
);
2986 cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY
, error_mem_read
, notdirty_mem_write
, NULL
);
2990 io_mem_watch
= cpu_register_io_memory(watch_mem_read
,
2991 watch_mem_write
, NULL
);
2994 #endif /* !defined(CONFIG_USER_ONLY) */
2996 /* physical memory access (slow version, mainly for debug) */
2997 #if defined(CONFIG_USER_ONLY)
2998 void cpu_physical_memory_rw(target_phys_addr_t addr
, uint8_t *buf
,
2999 int len
, int is_write
)
3006 page
= addr
& TARGET_PAGE_MASK
;
3007 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3010 flags
= page_get_flags(page
);
3011 if (!(flags
& PAGE_VALID
))
3014 if (!(flags
& PAGE_WRITE
))
3016 /* XXX: this code should not depend on lock_user */
3017 if (!(p
= lock_user(VERIFY_WRITE
, addr
, l
, 0)))
3018 /* FIXME - should this return an error rather than just fail? */
3021 unlock_user(p
, addr
, l
);
3023 if (!(flags
& PAGE_READ
))
3025 /* XXX: this code should not depend on lock_user */
3026 if (!(p
= lock_user(VERIFY_READ
, addr
, l
, 1)))
3027 /* FIXME - should this return an error rather than just fail? */
3030 unlock_user(p
, addr
, 0);
3039 void cpu_physical_memory_rw(target_phys_addr_t addr
, uint8_t *buf
,
3040 int len
, int is_write
)
3045 target_phys_addr_t page
;
3050 page
= addr
& TARGET_PAGE_MASK
;
3051 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3054 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3056 pd
= IO_MEM_UNASSIGNED
;
3058 pd
= p
->phys_offset
;
3062 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3063 target_phys_addr_t addr1
= addr
;
3064 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3066 addr1
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3067 /* XXX: could force cpu_single_env to NULL to avoid
3069 if (l
>= 4 && ((addr1
& 3) == 0)) {
3070 /* 32 bit write access */
3072 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr1
, val
);
3074 } else if (l
>= 2 && ((addr1
& 1) == 0)) {
3075 /* 16 bit write access */
3077 io_mem_write
[io_index
][1](io_mem_opaque
[io_index
], addr1
, val
);
3080 /* 8 bit write access */
3082 io_mem_write
[io_index
][0](io_mem_opaque
[io_index
], addr1
, val
);
3086 unsigned long addr1
;
3087 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3089 ptr
= qemu_get_ram_ptr(addr1
);
3090 memcpy(ptr
, buf
, l
);
3091 if (!cpu_physical_memory_is_dirty(addr1
)) {
3092 /* invalidate code */
3093 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
3095 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
3096 (0xff & ~CODE_DIRTY_FLAG
);
3100 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3101 !(pd
& IO_MEM_ROMD
)) {
3102 target_phys_addr_t addr1
= addr
;
3104 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3106 addr1
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3107 if (l
>= 4 && ((addr1
& 3) == 0)) {
3108 /* 32 bit read access */
3109 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr1
);
3112 } else if (l
>= 2 && ((addr1
& 1) == 0)) {
3113 /* 16 bit read access */
3114 val
= io_mem_read
[io_index
][1](io_mem_opaque
[io_index
], addr1
);
3118 /* 8 bit read access */
3119 val
= io_mem_read
[io_index
][0](io_mem_opaque
[io_index
], addr1
);
3125 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3126 (addr
& ~TARGET_PAGE_MASK
);
3127 memcpy(buf
, ptr
, l
);
3136 /* used for ROM loading : can write in RAM and ROM */
3137 void cpu_physical_memory_write_rom(target_phys_addr_t addr
,
3138 const uint8_t *buf
, int len
)
3142 target_phys_addr_t page
;
3147 page
= addr
& TARGET_PAGE_MASK
;
3148 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3151 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3153 pd
= IO_MEM_UNASSIGNED
;
3155 pd
= p
->phys_offset
;
3158 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
&&
3159 (pd
& ~TARGET_PAGE_MASK
) != IO_MEM_ROM
&&
3160 !(pd
& IO_MEM_ROMD
)) {
3163 unsigned long addr1
;
3164 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3166 ptr
= qemu_get_ram_ptr(addr1
);
3167 memcpy(ptr
, buf
, l
);
3177 target_phys_addr_t addr
;
3178 target_phys_addr_t len
;
3181 static BounceBuffer bounce
;
3183 typedef struct MapClient
{
3185 void (*callback
)(void *opaque
);
3186 QLIST_ENTRY(MapClient
) link
;
3189 static QLIST_HEAD(map_client_list
, MapClient
) map_client_list
3190 = QLIST_HEAD_INITIALIZER(map_client_list
);
3192 void *cpu_register_map_client(void *opaque
, void (*callback
)(void *opaque
))
3194 MapClient
*client
= qemu_malloc(sizeof(*client
));
3196 client
->opaque
= opaque
;
3197 client
->callback
= callback
;
3198 QLIST_INSERT_HEAD(&map_client_list
, client
, link
);
3202 void cpu_unregister_map_client(void *_client
)
3204 MapClient
*client
= (MapClient
*)_client
;
3206 QLIST_REMOVE(client
, link
);
3210 static void cpu_notify_map_clients(void)
3214 while (!QLIST_EMPTY(&map_client_list
)) {
3215 client
= QLIST_FIRST(&map_client_list
);
3216 client
->callback(client
->opaque
);
3217 cpu_unregister_map_client(client
);
3221 /* Map a physical memory region into a host virtual address.
3222 * May map a subset of the requested range, given by and returned in *plen.
3223 * May return NULL if resources needed to perform the mapping are exhausted.
3224 * Use only for reads OR writes - not for read-modify-write operations.
3225 * Use cpu_register_map_client() to know when retrying the map operation is
3226 * likely to succeed.
3228 void *cpu_physical_memory_map(target_phys_addr_t addr
,
3229 target_phys_addr_t
*plen
,
3232 target_phys_addr_t len
= *plen
;
3233 target_phys_addr_t done
= 0;
3235 uint8_t *ret
= NULL
;
3237 target_phys_addr_t page
;
3240 unsigned long addr1
;
3243 page
= addr
& TARGET_PAGE_MASK
;
3244 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3247 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3249 pd
= IO_MEM_UNASSIGNED
;
3251 pd
= p
->phys_offset
;
3254 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3255 if (done
|| bounce
.buffer
) {
3258 bounce
.buffer
= qemu_memalign(TARGET_PAGE_SIZE
, TARGET_PAGE_SIZE
);
3262 cpu_physical_memory_rw(addr
, bounce
.buffer
, l
, 0);
3264 ptr
= bounce
.buffer
;
3266 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3267 ptr
= qemu_get_ram_ptr(addr1
);
3271 } else if (ret
+ done
!= ptr
) {
3283 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3284 * Will also mark the memory as dirty if is_write == 1. access_len gives
3285 * the amount of memory that was actually read or written by the caller.
3287 void cpu_physical_memory_unmap(void *buffer
, target_phys_addr_t len
,
3288 int is_write
, target_phys_addr_t access_len
)
3290 if (buffer
!= bounce
.buffer
) {
3292 ram_addr_t addr1
= qemu_ram_addr_from_host(buffer
);
3293 while (access_len
) {
3295 l
= TARGET_PAGE_SIZE
;
3298 if (!cpu_physical_memory_is_dirty(addr1
)) {
3299 /* invalidate code */
3300 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
3302 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
3303 (0xff & ~CODE_DIRTY_FLAG
);
3312 cpu_physical_memory_write(bounce
.addr
, bounce
.buffer
, access_len
);
3314 qemu_free(bounce
.buffer
);
3315 bounce
.buffer
= NULL
;
3316 cpu_notify_map_clients();
3319 /* warning: addr must be aligned */
3320 uint32_t ldl_phys(target_phys_addr_t addr
)
3328 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3330 pd
= IO_MEM_UNASSIGNED
;
3332 pd
= p
->phys_offset
;
3335 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3336 !(pd
& IO_MEM_ROMD
)) {
3338 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3340 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3341 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
3344 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3345 (addr
& ~TARGET_PAGE_MASK
);
3351 /* warning: addr must be aligned */
3352 uint64_t ldq_phys(target_phys_addr_t addr
)
3360 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3362 pd
= IO_MEM_UNASSIGNED
;
3364 pd
= p
->phys_offset
;
3367 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3368 !(pd
& IO_MEM_ROMD
)) {
3370 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3372 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3373 #ifdef TARGET_WORDS_BIGENDIAN
3374 val
= (uint64_t)io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
) << 32;
3375 val
|= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4);
3377 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
3378 val
|= (uint64_t)io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4) << 32;
3382 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3383 (addr
& ~TARGET_PAGE_MASK
);
3390 uint32_t ldub_phys(target_phys_addr_t addr
)
3393 cpu_physical_memory_read(addr
, &val
, 1);
3398 uint32_t lduw_phys(target_phys_addr_t addr
)
3401 cpu_physical_memory_read(addr
, (uint8_t *)&val
, 2);
3402 return tswap16(val
);
3405 /* warning: addr must be aligned. The ram page is not masked as dirty
3406 and the code inside is not invalidated. It is useful if the dirty
3407 bits are used to track modified PTEs */
3408 void stl_phys_notdirty(target_phys_addr_t addr
, uint32_t val
)
3415 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3417 pd
= IO_MEM_UNASSIGNED
;
3419 pd
= p
->phys_offset
;
3422 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3423 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3425 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3426 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3428 unsigned long addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3429 ptr
= qemu_get_ram_ptr(addr1
);
3432 if (unlikely(in_migration
)) {
3433 if (!cpu_physical_memory_is_dirty(addr1
)) {
3434 /* invalidate code */
3435 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
3437 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
3438 (0xff & ~CODE_DIRTY_FLAG
);
3444 void stq_phys_notdirty(target_phys_addr_t addr
, uint64_t val
)
3451 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3453 pd
= IO_MEM_UNASSIGNED
;
3455 pd
= p
->phys_offset
;
3458 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3459 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3461 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3462 #ifdef TARGET_WORDS_BIGENDIAN
3463 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
>> 32);
3464 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4, val
);
3466 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3467 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4, val
>> 32);
3470 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3471 (addr
& ~TARGET_PAGE_MASK
);
3476 /* warning: addr must be aligned */
3477 void stl_phys(target_phys_addr_t addr
, uint32_t val
)
3484 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3486 pd
= IO_MEM_UNASSIGNED
;
3488 pd
= p
->phys_offset
;
3491 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3492 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3494 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3495 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3497 unsigned long addr1
;
3498 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3500 ptr
= qemu_get_ram_ptr(addr1
);
3502 if (!cpu_physical_memory_is_dirty(addr1
)) {
3503 /* invalidate code */
3504 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
3506 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
3507 (0xff & ~CODE_DIRTY_FLAG
);
3513 void stb_phys(target_phys_addr_t addr
, uint32_t val
)
3516 cpu_physical_memory_write(addr
, &v
, 1);
3520 void stw_phys(target_phys_addr_t addr
, uint32_t val
)
3522 uint16_t v
= tswap16(val
);
3523 cpu_physical_memory_write(addr
, (const uint8_t *)&v
, 2);
3527 void stq_phys(target_phys_addr_t addr
, uint64_t val
)
3530 cpu_physical_memory_write(addr
, (const uint8_t *)&val
, 8);
3535 /* virtual memory access for debug (includes writing to ROM) */
3536 int cpu_memory_rw_debug(CPUState
*env
, target_ulong addr
,
3537 uint8_t *buf
, int len
, int is_write
)
3540 target_phys_addr_t phys_addr
;
3544 page
= addr
& TARGET_PAGE_MASK
;
3545 phys_addr
= cpu_get_phys_page_debug(env
, page
);
3546 /* if no physical page mapped, return an error */
3547 if (phys_addr
== -1)
3549 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3552 phys_addr
+= (addr
& ~TARGET_PAGE_MASK
);
3553 #if !defined(CONFIG_USER_ONLY)
3555 cpu_physical_memory_write_rom(phys_addr
, buf
, l
);
3558 cpu_physical_memory_rw(phys_addr
, buf
, l
, is_write
);
3566 /* in deterministic execution mode, instructions doing device I/Os
3567 must be at the end of the TB */
3568 void cpu_io_recompile(CPUState
*env
, void *retaddr
)
3570 TranslationBlock
*tb
;
3572 target_ulong pc
, cs_base
;
3575 tb
= tb_find_pc((unsigned long)retaddr
);
3577 cpu_abort(env
, "cpu_io_recompile: could not find TB for pc=%p",
3580 n
= env
->icount_decr
.u16
.low
+ tb
->icount
;
3581 cpu_restore_state(tb
, env
, (unsigned long)retaddr
, NULL
);
3582 /* Calculate how many instructions had been executed before the fault
3584 n
= n
- env
->icount_decr
.u16
.low
;
3585 /* Generate a new TB ending on the I/O insn. */
3587 /* On MIPS and SH, delay slot instructions can only be restarted if
3588 they were already the first instruction in the TB. If this is not
3589 the first instruction in a TB then re-execute the preceding
3591 #if defined(TARGET_MIPS)
3592 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
3593 env
->active_tc
.PC
-= 4;
3594 env
->icount_decr
.u16
.low
++;
3595 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
3597 #elif defined(TARGET_SH4)
3598 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
3601 env
->icount_decr
.u16
.low
++;
3602 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
3605 /* This should never happen. */
3606 if (n
> CF_COUNT_MASK
)
3607 cpu_abort(env
, "TB too big during recompile");
3609 cflags
= n
| CF_LAST_IO
;
3611 cs_base
= tb
->cs_base
;
3613 tb_phys_invalidate(tb
, -1);
3614 /* FIXME: In theory this could raise an exception. In practice
3615 we have already translated the block once so it's probably ok. */
3616 tb_gen_code(env
, pc
, cs_base
, flags
, cflags
);
3617 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
3618 the first in the TB) then we end up generating a whole new TB and
3619 repeating the fault, which is horribly inefficient.
3620 Better would be to execute just this insn uncached, or generate a
3622 cpu_resume_from_signal(env
, NULL
);
3625 void dump_exec_info(FILE *f
,
3626 int (*cpu_fprintf
)(FILE *f
, const char *fmt
, ...))
3628 int i
, target_code_size
, max_target_code_size
;
3629 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
3630 TranslationBlock
*tb
;
3632 target_code_size
= 0;
3633 max_target_code_size
= 0;
3635 direct_jmp_count
= 0;
3636 direct_jmp2_count
= 0;
3637 for(i
= 0; i
< nb_tbs
; i
++) {
3639 target_code_size
+= tb
->size
;
3640 if (tb
->size
> max_target_code_size
)
3641 max_target_code_size
= tb
->size
;
3642 if (tb
->page_addr
[1] != -1)
3644 if (tb
->tb_next_offset
[0] != 0xffff) {
3646 if (tb
->tb_next_offset
[1] != 0xffff) {
3647 direct_jmp2_count
++;
3651 /* XXX: avoid using doubles ? */
3652 cpu_fprintf(f
, "Translation buffer state:\n");
3653 cpu_fprintf(f
, "gen code size %ld/%ld\n",
3654 code_gen_ptr
- code_gen_buffer
, code_gen_buffer_max_size
);
3655 cpu_fprintf(f
, "TB count %d/%d\n",
3656 nb_tbs
, code_gen_max_blocks
);
3657 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
3658 nb_tbs
? target_code_size
/ nb_tbs
: 0,
3659 max_target_code_size
);
3660 cpu_fprintf(f
, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
3661 nb_tbs
? (code_gen_ptr
- code_gen_buffer
) / nb_tbs
: 0,
3662 target_code_size
? (double) (code_gen_ptr
- code_gen_buffer
) / target_code_size
: 0);
3663 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n",
3665 nb_tbs
? (cross_page
* 100) / nb_tbs
: 0);
3666 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
3668 nb_tbs
? (direct_jmp_count
* 100) / nb_tbs
: 0,
3670 nb_tbs
? (direct_jmp2_count
* 100) / nb_tbs
: 0);
3671 cpu_fprintf(f
, "\nStatistics:\n");
3672 cpu_fprintf(f
, "TB flush count %d\n", tb_flush_count
);
3673 cpu_fprintf(f
, "TB invalidate count %d\n", tb_phys_invalidate_count
);
3674 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
3675 tcg_dump_info(f
, cpu_fprintf
);
3678 #if !defined(CONFIG_USER_ONLY)
3680 #define MMUSUFFIX _cmmu
3681 #define GETPC() NULL
3682 #define env cpu_single_env
3683 #define SOFTMMU_CODE_ACCESS
3686 #include "softmmu_template.h"
3689 #include "softmmu_template.h"
3692 #include "softmmu_template.h"
3695 #include "softmmu_template.h"