2 * virtual page mapping and translated block handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
23 #include <sys/types.h>
36 #include "qemu-common.h"
37 #include "cache-utils.h"
39 #if !defined(TARGET_IA64)
47 #if defined(CONFIG_USER_ONLY)
52 //#define DEBUG_TB_INVALIDATE
55 //#define DEBUG_UNASSIGNED
57 /* make various TB consistency checks */
58 //#define DEBUG_TB_CHECK
59 //#define DEBUG_TLB_CHECK
61 //#define DEBUG_IOPORT
62 //#define DEBUG_SUBPAGE
64 #if !defined(CONFIG_USER_ONLY)
65 /* TB consistency checks only implemented for usermode emulation. */
69 #define SMC_BITMAP_USE_THRESHOLD 10
71 static TranslationBlock
*tbs
;
72 int code_gen_max_blocks
;
73 TranslationBlock
*tb_phys_hash
[CODE_GEN_PHYS_HASH_SIZE
];
75 /* any access to the tbs or the page table must use this lock */
76 spinlock_t tb_lock
= SPIN_LOCK_UNLOCKED
;
78 #if defined(__arm__) || defined(__sparc_v9__)
79 /* The prologue must be reachable with a direct jump. ARM and Sparc64
80 have limited branch ranges (possibly also PPC) so place it in a
81 section close to code segment. */
82 #define code_gen_section \
83 __attribute__((__section__(".gen_code"))) \
84 __attribute__((aligned (32)))
86 /* Maximum alignment for Win32 is 16. */
87 #define code_gen_section \
88 __attribute__((aligned (16)))
90 #define code_gen_section \
91 __attribute__((aligned (32)))
94 uint8_t code_gen_prologue
[1024] code_gen_section
;
95 static uint8_t *code_gen_buffer
;
96 static unsigned long code_gen_buffer_size
;
97 /* threshold to flush the translated code buffer */
98 static unsigned long code_gen_buffer_max_size
;
99 uint8_t *code_gen_ptr
;
101 #if !defined(CONFIG_USER_ONLY)
103 uint8_t *phys_ram_dirty
;
105 static int in_migration
;
107 typedef struct RAMBlock
{
111 struct RAMBlock
*next
;
114 static RAMBlock
*ram_blocks
;
115 /* TODO: When we implement (and use) ram deallocation (e.g. for hotplug)
116 then we can no longer assume contiguous ram offsets, and external uses
117 of this variable will break. */
118 ram_addr_t last_ram_offset
;
122 /* current CPU in the current thread. It is only valid inside
124 CPUState
*cpu_single_env
;
125 /* 0 = Do not count executed instructions.
126 1 = Precise instruction counting.
127 2 = Adaptive rate instruction counting. */
129 /* Current instruction counter. While executing translated code this may
130 include some instructions that have not yet been executed. */
133 typedef struct PageDesc
{
134 /* list of TBs intersecting this ram page */
135 TranslationBlock
*first_tb
;
136 /* in order to optimize self modifying code, we count the number
137 of lookups we do to a given page to use a bitmap */
138 unsigned int code_write_count
;
139 uint8_t *code_bitmap
;
140 #if defined(CONFIG_USER_ONLY)
145 /* In system mode we want L1_MAP to be based on ram offsets,
146 while in user mode we want it to be based on virtual addresses. */
147 #if !defined(CONFIG_USER_ONLY)
148 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
149 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
151 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
154 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
157 /* Size of the L2 (and L3, etc) page tables. */
159 #define L2_SIZE (1 << L2_BITS)
161 /* The bits remaining after N lower levels of page tables. */
162 #define P_L1_BITS_REM \
163 ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
164 #define V_L1_BITS_REM \
165 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
167 /* Size of the L1 page table. Avoid silly small sizes. */
168 #if P_L1_BITS_REM < 4
169 #define P_L1_BITS (P_L1_BITS_REM + L2_BITS)
171 #define P_L1_BITS P_L1_BITS_REM
174 #if V_L1_BITS_REM < 4
175 #define V_L1_BITS (V_L1_BITS_REM + L2_BITS)
177 #define V_L1_BITS V_L1_BITS_REM
180 #define P_L1_SIZE ((target_phys_addr_t)1 << P_L1_BITS)
181 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
183 #define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
184 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
186 unsigned long qemu_real_host_page_size
;
187 unsigned long qemu_host_page_bits
;
188 unsigned long qemu_host_page_size
;
189 unsigned long qemu_host_page_mask
;
191 /* This is a multi-level map on the virtual address space.
192 The bottom level has pointers to PageDesc. */
193 static void *l1_map
[V_L1_SIZE
];
195 #if !defined(CONFIG_USER_ONLY)
196 typedef struct PhysPageDesc
{
197 /* offset in host memory of the page + io_index in the low bits */
198 ram_addr_t phys_offset
;
199 ram_addr_t region_offset
;
202 /* This is a multi-level map on the physical address space.
203 The bottom level has pointers to PhysPageDesc. */
204 static void *l1_phys_map
[P_L1_SIZE
];
206 static void io_mem_init(void);
208 /* io memory support */
209 CPUWriteMemoryFunc
*io_mem_write
[IO_MEM_NB_ENTRIES
][4];
210 CPUReadMemoryFunc
*io_mem_read
[IO_MEM_NB_ENTRIES
][4];
211 void *io_mem_opaque
[IO_MEM_NB_ENTRIES
];
212 static char io_mem_used
[IO_MEM_NB_ENTRIES
];
213 static int io_mem_watch
;
218 static const char *logfilename
= "qemu.log";
220 static const char *logfilename
= "/tmp/qemu.log";
224 static int log_append
= 0;
227 #if !defined(CONFIG_USER_ONLY)
228 static int tlb_flush_count
;
230 static int tb_flush_count
;
231 static int tb_phys_invalidate_count
;
234 static void map_exec(void *addr
, long size
)
237 VirtualProtect(addr
, size
,
238 PAGE_EXECUTE_READWRITE
, &old_protect
);
242 static void map_exec(void *addr
, long size
)
244 unsigned long start
, end
, page_size
;
246 page_size
= getpagesize();
247 start
= (unsigned long)addr
;
248 start
&= ~(page_size
- 1);
250 end
= (unsigned long)addr
+ size
;
251 end
+= page_size
- 1;
252 end
&= ~(page_size
- 1);
254 mprotect((void *)start
, end
- start
,
255 PROT_READ
| PROT_WRITE
| PROT_EXEC
);
259 static void page_init(void)
261 /* NOTE: we can always suppose that qemu_host_page_size >=
265 SYSTEM_INFO system_info
;
267 GetSystemInfo(&system_info
);
268 qemu_real_host_page_size
= system_info
.dwPageSize
;
271 qemu_real_host_page_size
= getpagesize();
273 if (qemu_host_page_size
== 0)
274 qemu_host_page_size
= qemu_real_host_page_size
;
275 if (qemu_host_page_size
< TARGET_PAGE_SIZE
)
276 qemu_host_page_size
= TARGET_PAGE_SIZE
;
277 qemu_host_page_bits
= 0;
278 while ((1 << qemu_host_page_bits
) < qemu_host_page_size
)
279 qemu_host_page_bits
++;
280 qemu_host_page_mask
= ~(qemu_host_page_size
- 1);
282 #if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
286 last_brk
= (unsigned long)sbrk(0);
288 f
= fopen("/proc/self/maps", "r");
293 unsigned long startaddr
, endaddr
;
296 n
= fscanf (f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
298 if (n
== 2 && h2g_valid(startaddr
)) {
299 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
301 if (h2g_valid(endaddr
)) {
302 endaddr
= h2g(endaddr
);
306 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
317 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
323 #if defined(CONFIG_USER_ONLY)
324 /* We can't use qemu_malloc because it may recurse into a locked mutex.
325 Neither can we record the new pages we reserve while allocating a
326 given page because that may recurse into an unallocated page table
327 entry. Stuff the allocations we do make into a queue and process
328 them after having completed one entire page table allocation. */
330 unsigned long reserve
[2 * (V_L1_SHIFT
/ L2_BITS
)];
333 # define ALLOC(P, SIZE) \
335 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
336 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
337 if (h2g_valid(P)) { \
338 reserve[reserve_idx] = h2g(P); \
339 reserve[reserve_idx + 1] = SIZE; \
344 # define ALLOC(P, SIZE) \
345 do { P = qemu_mallocz(SIZE); } while (0)
348 /* Level 1. Always allocated. */
349 lp
= l1_map
+ ((index
>> V_L1_SHIFT
) & (V_L1_SIZE
- 1));
352 for (i
= V_L1_SHIFT
/ L2_BITS
- 1; i
> 0; i
--) {
359 ALLOC(p
, sizeof(void *) * L2_SIZE
);
363 lp
= p
+ ((index
>> (i
* L2_BITS
)) & (L2_SIZE
- 1));
371 ALLOC(pd
, sizeof(PageDesc
) * L2_SIZE
);
376 #if defined(CONFIG_USER_ONLY)
377 for (i
= 0; i
< reserve_idx
; i
+= 2) {
378 unsigned long addr
= reserve
[i
];
379 unsigned long len
= reserve
[i
+ 1];
381 page_set_flags(addr
& TARGET_PAGE_MASK
,
382 TARGET_PAGE_ALIGN(addr
+ len
),
387 return pd
+ (index
& (L2_SIZE
- 1));
390 static inline PageDesc
*page_find(tb_page_addr_t index
)
392 return page_find_alloc(index
, 0);
395 #if !defined(CONFIG_USER_ONLY)
396 static PhysPageDesc
*phys_page_find_alloc(target_phys_addr_t index
, int alloc
)
402 /* Level 1. Always allocated. */
403 lp
= l1_phys_map
+ ((index
>> P_L1_SHIFT
) & (P_L1_SIZE
- 1));
406 for (i
= P_L1_SHIFT
/ L2_BITS
- 1; i
> 0; i
--) {
412 *lp
= p
= qemu_mallocz(sizeof(void *) * L2_SIZE
);
414 lp
= p
+ ((index
>> (i
* L2_BITS
)) & (L2_SIZE
- 1));
425 *lp
= pd
= qemu_malloc(sizeof(PhysPageDesc
) * L2_SIZE
);
427 for (i
= 0; i
< L2_SIZE
; i
++) {
428 pd
[i
].phys_offset
= IO_MEM_UNASSIGNED
;
429 pd
[i
].region_offset
= (index
+ i
) << TARGET_PAGE_BITS
;
433 return pd
+ (index
& (L2_SIZE
- 1));
436 static inline PhysPageDesc
*phys_page_find(target_phys_addr_t index
)
438 return phys_page_find_alloc(index
, 0);
441 static void tlb_protect_code(ram_addr_t ram_addr
);
442 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
444 #define mmap_lock() do { } while(0)
445 #define mmap_unlock() do { } while(0)
448 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
450 #if defined(CONFIG_USER_ONLY)
451 /* Currently it is not recommended to allocate big chunks of data in
452 user mode. It will change when a dedicated libc will be used */
453 #define USE_STATIC_CODE_GEN_BUFFER
456 #ifdef USE_STATIC_CODE_GEN_BUFFER
457 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
];
460 static void code_gen_alloc(unsigned long tb_size
)
465 #ifdef USE_STATIC_CODE_GEN_BUFFER
466 code_gen_buffer
= static_code_gen_buffer
;
467 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
468 map_exec(code_gen_buffer
, code_gen_buffer_size
);
470 code_gen_buffer_size
= tb_size
;
471 if (code_gen_buffer_size
== 0) {
472 #if defined(CONFIG_USER_ONLY)
473 /* in user mode, phys_ram_size is not meaningful */
474 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
476 /* XXX: needs adjustments */
477 code_gen_buffer_size
= (unsigned long)(ram_size
/ 4);
480 if (code_gen_buffer_size
< MIN_CODE_GEN_BUFFER_SIZE
)
481 code_gen_buffer_size
= MIN_CODE_GEN_BUFFER_SIZE
;
482 /* The code gen buffer location may have constraints depending on
483 the host cpu and OS */
484 #if defined(__linux__)
489 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
490 #if defined(__x86_64__)
492 /* Cannot map more than that */
493 if (code_gen_buffer_size
> (800 * 1024 * 1024))
494 code_gen_buffer_size
= (800 * 1024 * 1024);
495 #elif defined(__sparc_v9__)
496 // Map the buffer below 2G, so we can use direct calls and branches
498 start
= (void *) 0x60000000UL
;
499 if (code_gen_buffer_size
> (512 * 1024 * 1024))
500 code_gen_buffer_size
= (512 * 1024 * 1024);
501 #elif defined(__arm__)
502 /* Map the buffer below 32M, so we can use direct calls and branches */
504 start
= (void *) 0x01000000UL
;
505 if (code_gen_buffer_size
> 16 * 1024 * 1024)
506 code_gen_buffer_size
= 16 * 1024 * 1024;
508 code_gen_buffer
= mmap(start
, code_gen_buffer_size
,
509 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
511 if (code_gen_buffer
== MAP_FAILED
) {
512 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
516 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__DragonFly__)
520 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
521 #if defined(__x86_64__)
522 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
523 * 0x40000000 is free */
525 addr
= (void *)0x40000000;
526 /* Cannot map more than that */
527 if (code_gen_buffer_size
> (800 * 1024 * 1024))
528 code_gen_buffer_size
= (800 * 1024 * 1024);
530 code_gen_buffer
= mmap(addr
, code_gen_buffer_size
,
531 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
533 if (code_gen_buffer
== MAP_FAILED
) {
534 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
539 code_gen_buffer
= qemu_malloc(code_gen_buffer_size
);
540 map_exec(code_gen_buffer
, code_gen_buffer_size
);
542 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
543 map_exec(code_gen_prologue
, sizeof(code_gen_prologue
));
544 code_gen_buffer_max_size
= code_gen_buffer_size
-
545 code_gen_max_block_size();
546 code_gen_max_blocks
= code_gen_buffer_size
/ CODE_GEN_AVG_BLOCK_SIZE
;
547 tbs
= qemu_malloc(code_gen_max_blocks
* sizeof(TranslationBlock
));
550 /* Must be called before using the QEMU cpus. 'tb_size' is the size
551 (in bytes) allocated to the translation buffer. Zero means default
553 void cpu_exec_init_all(unsigned long tb_size
)
556 code_gen_alloc(tb_size
);
557 code_gen_ptr
= code_gen_buffer
;
559 #if !defined(CONFIG_USER_ONLY)
564 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
566 static int cpu_common_post_load(void *opaque
, int version_id
)
568 CPUState
*env
= opaque
;
570 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
571 version_id is increased. */
572 env
->interrupt_request
&= ~0x01;
578 static const VMStateDescription vmstate_cpu_common
= {
579 .name
= "cpu_common",
581 .minimum_version_id
= 1,
582 .minimum_version_id_old
= 1,
583 .post_load
= cpu_common_post_load
,
584 .fields
= (VMStateField
[]) {
585 VMSTATE_UINT32(halted
, CPUState
),
586 VMSTATE_UINT32(interrupt_request
, CPUState
),
587 VMSTATE_END_OF_LIST()
592 CPUState
*qemu_get_cpu(int cpu
)
594 CPUState
*env
= first_cpu
;
597 if (env
->cpu_index
== cpu
)
605 void cpu_exec_init(CPUState
*env
)
610 #if defined(CONFIG_USER_ONLY)
613 env
->next_cpu
= NULL
;
616 while (*penv
!= NULL
) {
617 penv
= &(*penv
)->next_cpu
;
620 env
->cpu_index
= cpu_index
;
622 QTAILQ_INIT(&env
->breakpoints
);
623 QTAILQ_INIT(&env
->watchpoints
);
625 env
->thread_id
= GetCurrentProcessId();
627 env
->thread_id
= getpid();
630 #if defined(CONFIG_USER_ONLY)
633 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
634 vmstate_register(cpu_index
, &vmstate_cpu_common
, env
);
635 register_savevm("cpu", cpu_index
, CPU_SAVE_VERSION
,
636 cpu_save
, cpu_load
, env
);
640 static inline void invalidate_page_bitmap(PageDesc
*p
)
642 if (p
->code_bitmap
) {
643 qemu_free(p
->code_bitmap
);
644 p
->code_bitmap
= NULL
;
646 p
->code_write_count
= 0;
649 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
651 static void page_flush_tb_1 (int level
, void **lp
)
660 for (i
= 0; i
< L2_SIZE
; ++i
) {
661 pd
[i
].first_tb
= NULL
;
662 invalidate_page_bitmap(pd
+ i
);
666 for (i
= 0; i
< L2_SIZE
; ++i
) {
667 page_flush_tb_1 (level
- 1, pp
+ i
);
672 static void page_flush_tb(void)
675 for (i
= 0; i
< V_L1_SIZE
; i
++) {
676 page_flush_tb_1(V_L1_SHIFT
/ L2_BITS
- 1, l1_map
+ i
);
680 /* flush all the translation blocks */
681 /* XXX: tb_flush is currently not thread safe */
682 void tb_flush(CPUState
*env1
)
685 #if defined(DEBUG_FLUSH)
686 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
687 (unsigned long)(code_gen_ptr
- code_gen_buffer
),
689 ((unsigned long)(code_gen_ptr
- code_gen_buffer
)) / nb_tbs
: 0);
691 if ((unsigned long)(code_gen_ptr
- code_gen_buffer
) > code_gen_buffer_size
)
692 cpu_abort(env1
, "Internal error: code buffer overflow\n");
696 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
697 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
700 memset (tb_phys_hash
, 0, CODE_GEN_PHYS_HASH_SIZE
* sizeof (void *));
703 code_gen_ptr
= code_gen_buffer
;
704 /* XXX: flush processor icache at this point if cache flush is
709 #ifdef DEBUG_TB_CHECK
711 static void tb_invalidate_check(target_ulong address
)
713 TranslationBlock
*tb
;
715 address
&= TARGET_PAGE_MASK
;
716 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
717 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
718 if (!(address
+ TARGET_PAGE_SIZE
<= tb
->pc
||
719 address
>= tb
->pc
+ tb
->size
)) {
720 printf("ERROR invalidate: address=" TARGET_FMT_lx
721 " PC=%08lx size=%04x\n",
722 address
, (long)tb
->pc
, tb
->size
);
728 /* verify that all the pages have correct rights for code */
729 static void tb_page_check(void)
731 TranslationBlock
*tb
;
732 int i
, flags1
, flags2
;
734 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
735 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
736 flags1
= page_get_flags(tb
->pc
);
737 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
738 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
739 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
740 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
748 /* invalidate one TB */
749 static inline void tb_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
,
752 TranslationBlock
*tb1
;
756 *ptb
= *(TranslationBlock
**)((char *)tb1
+ next_offset
);
759 ptb
= (TranslationBlock
**)((char *)tb1
+ next_offset
);
763 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
765 TranslationBlock
*tb1
;
771 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
773 *ptb
= tb1
->page_next
[n1
];
776 ptb
= &tb1
->page_next
[n1
];
780 static inline void tb_jmp_remove(TranslationBlock
*tb
, int n
)
782 TranslationBlock
*tb1
, **ptb
;
785 ptb
= &tb
->jmp_next
[n
];
788 /* find tb(n) in circular list */
792 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
793 if (n1
== n
&& tb1
== tb
)
796 ptb
= &tb1
->jmp_first
;
798 ptb
= &tb1
->jmp_next
[n1
];
801 /* now we can suppress tb(n) from the list */
802 *ptb
= tb
->jmp_next
[n
];
804 tb
->jmp_next
[n
] = NULL
;
808 /* reset the jump entry 'n' of a TB so that it is not chained to
810 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
812 tb_set_jmp_target(tb
, n
, (unsigned long)(tb
->tc_ptr
+ tb
->tb_next_offset
[n
]));
815 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
820 tb_page_addr_t phys_pc
;
821 TranslationBlock
*tb1
, *tb2
;
823 /* remove the TB from the hash list */
824 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
825 h
= tb_phys_hash_func(phys_pc
);
826 tb_remove(&tb_phys_hash
[h
], tb
,
827 offsetof(TranslationBlock
, phys_hash_next
));
829 /* remove the TB from the page list */
830 if (tb
->page_addr
[0] != page_addr
) {
831 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
832 tb_page_remove(&p
->first_tb
, tb
);
833 invalidate_page_bitmap(p
);
835 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
836 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
837 tb_page_remove(&p
->first_tb
, tb
);
838 invalidate_page_bitmap(p
);
841 tb_invalidated_flag
= 1;
843 /* remove the TB from the hash list */
844 h
= tb_jmp_cache_hash_func(tb
->pc
);
845 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
846 if (env
->tb_jmp_cache
[h
] == tb
)
847 env
->tb_jmp_cache
[h
] = NULL
;
850 /* suppress this TB from the two jump lists */
851 tb_jmp_remove(tb
, 0);
852 tb_jmp_remove(tb
, 1);
854 /* suppress any remaining jumps to this TB */
860 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
861 tb2
= tb1
->jmp_next
[n1
];
862 tb_reset_jump(tb1
, n1
);
863 tb1
->jmp_next
[n1
] = NULL
;
866 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2); /* fail safe */
868 tb_phys_invalidate_count
++;
871 static inline void set_bits(uint8_t *tab
, int start
, int len
)
877 mask
= 0xff << (start
& 7);
878 if ((start
& ~7) == (end
& ~7)) {
880 mask
&= ~(0xff << (end
& 7));
885 start
= (start
+ 8) & ~7;
887 while (start
< end1
) {
892 mask
= ~(0xff << (end
& 7));
898 static void build_page_bitmap(PageDesc
*p
)
900 int n
, tb_start
, tb_end
;
901 TranslationBlock
*tb
;
903 p
->code_bitmap
= qemu_mallocz(TARGET_PAGE_SIZE
/ 8);
908 tb
= (TranslationBlock
*)((long)tb
& ~3);
909 /* NOTE: this is subtle as a TB may span two physical pages */
911 /* NOTE: tb_end may be after the end of the page, but
912 it is not a problem */
913 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
914 tb_end
= tb_start
+ tb
->size
;
915 if (tb_end
> TARGET_PAGE_SIZE
)
916 tb_end
= TARGET_PAGE_SIZE
;
919 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
921 set_bits(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
922 tb
= tb
->page_next
[n
];
926 TranslationBlock
*tb_gen_code(CPUState
*env
,
927 target_ulong pc
, target_ulong cs_base
,
928 int flags
, int cflags
)
930 TranslationBlock
*tb
;
932 tb_page_addr_t phys_pc
, phys_page2
;
933 target_ulong virt_page2
;
936 phys_pc
= get_page_addr_code(env
, pc
);
939 /* flush must be done */
941 /* cannot fail at this point */
943 /* Don't forget to invalidate previous TB info. */
944 tb_invalidated_flag
= 1;
946 tc_ptr
= code_gen_ptr
;
948 tb
->cs_base
= cs_base
;
951 cpu_gen_code(env
, tb
, &code_gen_size
);
952 code_gen_ptr
= (void *)(((unsigned long)code_gen_ptr
+ code_gen_size
+ CODE_GEN_ALIGN
- 1) & ~(CODE_GEN_ALIGN
- 1));
954 /* check next page if needed */
955 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
957 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
958 phys_page2
= get_page_addr_code(env
, virt_page2
);
960 tb_link_page(tb
, phys_pc
, phys_page2
);
964 /* invalidate all TBs which intersect with the target physical page
965 starting in range [start;end[. NOTE: start and end must refer to
966 the same physical page. 'is_cpu_write_access' should be true if called
967 from a real cpu write access: the virtual CPU will exit the current
968 TB if code is modified inside this TB. */
969 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
970 int is_cpu_write_access
)
972 TranslationBlock
*tb
, *tb_next
, *saved_tb
;
973 CPUState
*env
= cpu_single_env
;
974 tb_page_addr_t tb_start
, tb_end
;
977 #ifdef TARGET_HAS_PRECISE_SMC
978 int current_tb_not_found
= is_cpu_write_access
;
979 TranslationBlock
*current_tb
= NULL
;
980 int current_tb_modified
= 0;
981 target_ulong current_pc
= 0;
982 target_ulong current_cs_base
= 0;
983 int current_flags
= 0;
984 #endif /* TARGET_HAS_PRECISE_SMC */
986 p
= page_find(start
>> TARGET_PAGE_BITS
);
989 if (!p
->code_bitmap
&&
990 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
&&
991 is_cpu_write_access
) {
992 /* build code bitmap */
993 build_page_bitmap(p
);
996 /* we remove all the TBs in the range [start, end[ */
997 /* XXX: see if in some cases it could be faster to invalidate all the code */
1001 tb
= (TranslationBlock
*)((long)tb
& ~3);
1002 tb_next
= tb
->page_next
[n
];
1003 /* NOTE: this is subtle as a TB may span two physical pages */
1005 /* NOTE: tb_end may be after the end of the page, but
1006 it is not a problem */
1007 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1008 tb_end
= tb_start
+ tb
->size
;
1010 tb_start
= tb
->page_addr
[1];
1011 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1013 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1014 #ifdef TARGET_HAS_PRECISE_SMC
1015 if (current_tb_not_found
) {
1016 current_tb_not_found
= 0;
1018 if (env
->mem_io_pc
) {
1019 /* now we have a real cpu fault */
1020 current_tb
= tb_find_pc(env
->mem_io_pc
);
1023 if (current_tb
== tb
&&
1024 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1025 /* If we are modifying the current TB, we must stop
1026 its execution. We could be more precise by checking
1027 that the modification is after the current PC, but it
1028 would require a specialized function to partially
1029 restore the CPU state */
1031 current_tb_modified
= 1;
1032 cpu_restore_state(current_tb
, env
,
1033 env
->mem_io_pc
, NULL
);
1034 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1037 #endif /* TARGET_HAS_PRECISE_SMC */
1038 /* we need to do that to handle the case where a signal
1039 occurs while doing tb_phys_invalidate() */
1042 saved_tb
= env
->current_tb
;
1043 env
->current_tb
= NULL
;
1045 tb_phys_invalidate(tb
, -1);
1047 env
->current_tb
= saved_tb
;
1048 if (env
->interrupt_request
&& env
->current_tb
)
1049 cpu_interrupt(env
, env
->interrupt_request
);
1054 #if !defined(CONFIG_USER_ONLY)
1055 /* if no code remaining, no need to continue to use slow writes */
1057 invalidate_page_bitmap(p
);
1058 if (is_cpu_write_access
) {
1059 tlb_unprotect_code_phys(env
, start
, env
->mem_io_vaddr
);
1063 #ifdef TARGET_HAS_PRECISE_SMC
1064 if (current_tb_modified
) {
1065 /* we generate a block containing just the instruction
1066 modifying the memory. It will ensure that it cannot modify
1068 env
->current_tb
= NULL
;
1069 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1070 cpu_resume_from_signal(env
, NULL
);
1075 /* len must be <= 8 and start must be a multiple of len */
1076 static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start
, int len
)
1082 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1083 cpu_single_env
->mem_io_vaddr
, len
,
1084 cpu_single_env
->eip
,
1085 cpu_single_env
->eip
+ (long)cpu_single_env
->segs
[R_CS
].base
);
1088 p
= page_find(start
>> TARGET_PAGE_BITS
);
1091 if (p
->code_bitmap
) {
1092 offset
= start
& ~TARGET_PAGE_MASK
;
1093 b
= p
->code_bitmap
[offset
>> 3] >> (offset
& 7);
1094 if (b
& ((1 << len
) - 1))
1098 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1102 #if !defined(CONFIG_SOFTMMU)
1103 static void tb_invalidate_phys_page(tb_page_addr_t addr
,
1104 unsigned long pc
, void *puc
)
1106 TranslationBlock
*tb
;
1109 #ifdef TARGET_HAS_PRECISE_SMC
1110 TranslationBlock
*current_tb
= NULL
;
1111 CPUState
*env
= cpu_single_env
;
1112 int current_tb_modified
= 0;
1113 target_ulong current_pc
= 0;
1114 target_ulong current_cs_base
= 0;
1115 int current_flags
= 0;
1118 addr
&= TARGET_PAGE_MASK
;
1119 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1123 #ifdef TARGET_HAS_PRECISE_SMC
1124 if (tb
&& pc
!= 0) {
1125 current_tb
= tb_find_pc(pc
);
1128 while (tb
!= NULL
) {
1130 tb
= (TranslationBlock
*)((long)tb
& ~3);
1131 #ifdef TARGET_HAS_PRECISE_SMC
1132 if (current_tb
== tb
&&
1133 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1134 /* If we are modifying the current TB, we must stop
1135 its execution. We could be more precise by checking
1136 that the modification is after the current PC, but it
1137 would require a specialized function to partially
1138 restore the CPU state */
1140 current_tb_modified
= 1;
1141 cpu_restore_state(current_tb
, env
, pc
, puc
);
1142 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1145 #endif /* TARGET_HAS_PRECISE_SMC */
1146 tb_phys_invalidate(tb
, addr
);
1147 tb
= tb
->page_next
[n
];
1150 #ifdef TARGET_HAS_PRECISE_SMC
1151 if (current_tb_modified
) {
1152 /* we generate a block containing just the instruction
1153 modifying the memory. It will ensure that it cannot modify
1155 env
->current_tb
= NULL
;
1156 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1157 cpu_resume_from_signal(env
, puc
);
1163 /* add the tb in the target page and protect it if necessary */
1164 static inline void tb_alloc_page(TranslationBlock
*tb
,
1165 unsigned int n
, tb_page_addr_t page_addr
)
1168 TranslationBlock
*last_first_tb
;
1170 tb
->page_addr
[n
] = page_addr
;
1171 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
, 1);
1172 tb
->page_next
[n
] = p
->first_tb
;
1173 last_first_tb
= p
->first_tb
;
1174 p
->first_tb
= (TranslationBlock
*)((long)tb
| n
);
1175 invalidate_page_bitmap(p
);
1177 #if defined(TARGET_HAS_SMC) || 1
1179 #if defined(CONFIG_USER_ONLY)
1180 if (p
->flags
& PAGE_WRITE
) {
1185 /* force the host page as non writable (writes will have a
1186 page fault + mprotect overhead) */
1187 page_addr
&= qemu_host_page_mask
;
1189 for(addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1190 addr
+= TARGET_PAGE_SIZE
) {
1192 p2
= page_find (addr
>> TARGET_PAGE_BITS
);
1196 p2
->flags
&= ~PAGE_WRITE
;
1197 page_get_flags(addr
);
1199 mprotect(g2h(page_addr
), qemu_host_page_size
,
1200 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1201 #ifdef DEBUG_TB_INVALIDATE
1202 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1207 /* if some code is already present, then the pages are already
1208 protected. So we handle the case where only the first TB is
1209 allocated in a physical page */
1210 if (!last_first_tb
) {
1211 tlb_protect_code(page_addr
);
1215 #endif /* TARGET_HAS_SMC */
1218 /* Allocate a new translation block. Flush the translation buffer if
1219 too many translation blocks or too much generated code. */
1220 TranslationBlock
*tb_alloc(target_ulong pc
)
1222 TranslationBlock
*tb
;
1224 if (nb_tbs
>= code_gen_max_blocks
||
1225 (code_gen_ptr
- code_gen_buffer
) >= code_gen_buffer_max_size
)
1227 tb
= &tbs
[nb_tbs
++];
1233 void tb_free(TranslationBlock
*tb
)
1235 /* In practice this is mostly used for single use temporary TB
1236 Ignore the hard cases and just back up if this TB happens to
1237 be the last one generated. */
1238 if (nb_tbs
> 0 && tb
== &tbs
[nb_tbs
- 1]) {
1239 code_gen_ptr
= tb
->tc_ptr
;
1244 /* add a new TB and link it to the physical page tables. phys_page2 is
1245 (-1) to indicate that only one page contains the TB. */
1246 void tb_link_page(TranslationBlock
*tb
,
1247 tb_page_addr_t phys_pc
, tb_page_addr_t phys_page2
)
1250 TranslationBlock
**ptb
;
1252 /* Grab the mmap lock to stop another thread invalidating this TB
1253 before we are done. */
1255 /* add in the physical hash table */
1256 h
= tb_phys_hash_func(phys_pc
);
1257 ptb
= &tb_phys_hash
[h
];
1258 tb
->phys_hash_next
= *ptb
;
1261 /* add in the page list */
1262 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1263 if (phys_page2
!= -1)
1264 tb_alloc_page(tb
, 1, phys_page2
);
1266 tb
->page_addr
[1] = -1;
1268 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2);
1269 tb
->jmp_next
[0] = NULL
;
1270 tb
->jmp_next
[1] = NULL
;
1272 /* init original jump addresses */
1273 if (tb
->tb_next_offset
[0] != 0xffff)
1274 tb_reset_jump(tb
, 0);
1275 if (tb
->tb_next_offset
[1] != 0xffff)
1276 tb_reset_jump(tb
, 1);
1278 #ifdef DEBUG_TB_CHECK
1284 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1285 tb[1].tc_ptr. Return NULL if not found */
1286 TranslationBlock
*tb_find_pc(unsigned long tc_ptr
)
1288 int m_min
, m_max
, m
;
1290 TranslationBlock
*tb
;
1294 if (tc_ptr
< (unsigned long)code_gen_buffer
||
1295 tc_ptr
>= (unsigned long)code_gen_ptr
)
1297 /* binary search (cf Knuth) */
1300 while (m_min
<= m_max
) {
1301 m
= (m_min
+ m_max
) >> 1;
1303 v
= (unsigned long)tb
->tc_ptr
;
1306 else if (tc_ptr
< v
) {
1315 static void tb_reset_jump_recursive(TranslationBlock
*tb
);
1317 static inline void tb_reset_jump_recursive2(TranslationBlock
*tb
, int n
)
1319 TranslationBlock
*tb1
, *tb_next
, **ptb
;
1322 tb1
= tb
->jmp_next
[n
];
1324 /* find head of list */
1327 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1330 tb1
= tb1
->jmp_next
[n1
];
1332 /* we are now sure now that tb jumps to tb1 */
1335 /* remove tb from the jmp_first list */
1336 ptb
= &tb_next
->jmp_first
;
1340 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1341 if (n1
== n
&& tb1
== tb
)
1343 ptb
= &tb1
->jmp_next
[n1
];
1345 *ptb
= tb
->jmp_next
[n
];
1346 tb
->jmp_next
[n
] = NULL
;
1348 /* suppress the jump to next tb in generated code */
1349 tb_reset_jump(tb
, n
);
1351 /* suppress jumps in the tb on which we could have jumped */
1352 tb_reset_jump_recursive(tb_next
);
1356 static void tb_reset_jump_recursive(TranslationBlock
*tb
)
1358 tb_reset_jump_recursive2(tb
, 0);
1359 tb_reset_jump_recursive2(tb
, 1);
1362 #if defined(TARGET_HAS_ICE)
1363 #if defined(CONFIG_USER_ONLY)
1364 static void breakpoint_invalidate(CPUState
*env
, target_ulong pc
)
1366 tb_invalidate_phys_page_range(pc
, pc
+ 1, 0);
1369 static void breakpoint_invalidate(CPUState
*env
, target_ulong pc
)
1371 target_phys_addr_t addr
;
1373 ram_addr_t ram_addr
;
1376 addr
= cpu_get_phys_page_debug(env
, pc
);
1377 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
1379 pd
= IO_MEM_UNASSIGNED
;
1381 pd
= p
->phys_offset
;
1383 ram_addr
= (pd
& TARGET_PAGE_MASK
) | (pc
& ~TARGET_PAGE_MASK
);
1384 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1387 #endif /* TARGET_HAS_ICE */
1389 #if defined(CONFIG_USER_ONLY)
1390 void cpu_watchpoint_remove_all(CPUState
*env
, int mask
)
1395 int cpu_watchpoint_insert(CPUState
*env
, target_ulong addr
, target_ulong len
,
1396 int flags
, CPUWatchpoint
**watchpoint
)
1401 /* Add a watchpoint. */
1402 int cpu_watchpoint_insert(CPUState
*env
, target_ulong addr
, target_ulong len
,
1403 int flags
, CPUWatchpoint
**watchpoint
)
1405 target_ulong len_mask
= ~(len
- 1);
1408 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1409 if ((len
!= 1 && len
!= 2 && len
!= 4 && len
!= 8) || (addr
& ~len_mask
)) {
1410 fprintf(stderr
, "qemu: tried to set invalid watchpoint at "
1411 TARGET_FMT_lx
", len=" TARGET_FMT_lu
"\n", addr
, len
);
1414 wp
= qemu_malloc(sizeof(*wp
));
1417 wp
->len_mask
= len_mask
;
1420 /* keep all GDB-injected watchpoints in front */
1422 QTAILQ_INSERT_HEAD(&env
->watchpoints
, wp
, entry
);
1424 QTAILQ_INSERT_TAIL(&env
->watchpoints
, wp
, entry
);
1426 tlb_flush_page(env
, addr
);
1433 /* Remove a specific watchpoint. */
1434 int cpu_watchpoint_remove(CPUState
*env
, target_ulong addr
, target_ulong len
,
1437 target_ulong len_mask
= ~(len
- 1);
1440 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1441 if (addr
== wp
->vaddr
&& len_mask
== wp
->len_mask
1442 && flags
== (wp
->flags
& ~BP_WATCHPOINT_HIT
)) {
1443 cpu_watchpoint_remove_by_ref(env
, wp
);
1450 /* Remove a specific watchpoint by reference. */
1451 void cpu_watchpoint_remove_by_ref(CPUState
*env
, CPUWatchpoint
*watchpoint
)
1453 QTAILQ_REMOVE(&env
->watchpoints
, watchpoint
, entry
);
1455 tlb_flush_page(env
, watchpoint
->vaddr
);
1457 qemu_free(watchpoint
);
1460 /* Remove all matching watchpoints. */
1461 void cpu_watchpoint_remove_all(CPUState
*env
, int mask
)
1463 CPUWatchpoint
*wp
, *next
;
1465 QTAILQ_FOREACH_SAFE(wp
, &env
->watchpoints
, entry
, next
) {
1466 if (wp
->flags
& mask
)
1467 cpu_watchpoint_remove_by_ref(env
, wp
);
1472 /* Add a breakpoint. */
1473 int cpu_breakpoint_insert(CPUState
*env
, target_ulong pc
, int flags
,
1474 CPUBreakpoint
**breakpoint
)
1476 #if defined(TARGET_HAS_ICE)
1479 bp
= qemu_malloc(sizeof(*bp
));
1484 /* keep all GDB-injected breakpoints in front */
1486 QTAILQ_INSERT_HEAD(&env
->breakpoints
, bp
, entry
);
1488 QTAILQ_INSERT_TAIL(&env
->breakpoints
, bp
, entry
);
1490 breakpoint_invalidate(env
, pc
);
1500 /* Remove a specific breakpoint. */
1501 int cpu_breakpoint_remove(CPUState
*env
, target_ulong pc
, int flags
)
1503 #if defined(TARGET_HAS_ICE)
1506 QTAILQ_FOREACH(bp
, &env
->breakpoints
, entry
) {
1507 if (bp
->pc
== pc
&& bp
->flags
== flags
) {
1508 cpu_breakpoint_remove_by_ref(env
, bp
);
1518 /* Remove a specific breakpoint by reference. */
1519 void cpu_breakpoint_remove_by_ref(CPUState
*env
, CPUBreakpoint
*breakpoint
)
1521 #if defined(TARGET_HAS_ICE)
1522 QTAILQ_REMOVE(&env
->breakpoints
, breakpoint
, entry
);
1524 breakpoint_invalidate(env
, breakpoint
->pc
);
1526 qemu_free(breakpoint
);
1530 /* Remove all matching breakpoints. */
1531 void cpu_breakpoint_remove_all(CPUState
*env
, int mask
)
1533 #if defined(TARGET_HAS_ICE)
1534 CPUBreakpoint
*bp
, *next
;
1536 QTAILQ_FOREACH_SAFE(bp
, &env
->breakpoints
, entry
, next
) {
1537 if (bp
->flags
& mask
)
1538 cpu_breakpoint_remove_by_ref(env
, bp
);
1543 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1544 CPU loop after each instruction */
1545 void cpu_single_step(CPUState
*env
, int enabled
)
1547 #if defined(TARGET_HAS_ICE)
1548 if (env
->singlestep_enabled
!= enabled
) {
1549 env
->singlestep_enabled
= enabled
;
1551 kvm_update_guest_debug(env
, 0);
1553 /* must flush all the translated code to avoid inconsistencies */
1554 /* XXX: only flush what is necessary */
1561 /* enable or disable low levels log */
1562 void cpu_set_log(int log_flags
)
1564 loglevel
= log_flags
;
1565 if (loglevel
&& !logfile
) {
1566 logfile
= fopen(logfilename
, log_append
? "a" : "w");
1568 perror(logfilename
);
1571 #if !defined(CONFIG_SOFTMMU)
1572 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1574 static char logfile_buf
[4096];
1575 setvbuf(logfile
, logfile_buf
, _IOLBF
, sizeof(logfile_buf
));
1577 #elif !defined(_WIN32)
1578 /* Win32 doesn't support line-buffering and requires size >= 2 */
1579 setvbuf(logfile
, NULL
, _IOLBF
, 0);
1583 if (!loglevel
&& logfile
) {
1589 void cpu_set_log_filename(const char *filename
)
1591 logfilename
= strdup(filename
);
1596 cpu_set_log(loglevel
);
1599 static void cpu_unlink_tb(CPUState
*env
)
1601 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1602 problem and hope the cpu will stop of its own accord. For userspace
1603 emulation this often isn't actually as bad as it sounds. Often
1604 signals are used primarily to interrupt blocking syscalls. */
1605 TranslationBlock
*tb
;
1606 static spinlock_t interrupt_lock
= SPIN_LOCK_UNLOCKED
;
1608 spin_lock(&interrupt_lock
);
1609 tb
= env
->current_tb
;
1610 /* if the cpu is currently executing code, we must unlink it and
1611 all the potentially executing TB */
1613 env
->current_tb
= NULL
;
1614 tb_reset_jump_recursive(tb
);
1616 spin_unlock(&interrupt_lock
);
1619 /* mask must never be zero, except for A20 change call */
1620 void cpu_interrupt(CPUState
*env
, int mask
)
1624 old_mask
= env
->interrupt_request
;
1625 env
->interrupt_request
|= mask
;
1626 if (kvm_enabled() && !kvm_irqchip_in_kernel())
1627 kvm_update_interrupt_request(env
);
1629 #ifndef CONFIG_USER_ONLY
1631 * If called from iothread context, wake the target cpu in
1634 if (!qemu_cpu_self(env
)) {
1641 env
->icount_decr
.u16
.high
= 0xffff;
1642 #ifndef CONFIG_USER_ONLY
1644 && (mask
& ~old_mask
) != 0) {
1645 cpu_abort(env
, "Raised interrupt while not in I/O function");
1653 void cpu_reset_interrupt(CPUState
*env
, int mask
)
1655 env
->interrupt_request
&= ~mask
;
1658 void cpu_exit(CPUState
*env
)
1660 env
->exit_request
= 1;
1664 const CPULogItem cpu_log_items
[] = {
1665 { CPU_LOG_TB_OUT_ASM
, "out_asm",
1666 "show generated host assembly code for each compiled TB" },
1667 { CPU_LOG_TB_IN_ASM
, "in_asm",
1668 "show target assembly code for each compiled TB" },
1669 { CPU_LOG_TB_OP
, "op",
1670 "show micro ops for each compiled TB" },
1671 { CPU_LOG_TB_OP_OPT
, "op_opt",
1674 "before eflags optimization and "
1676 "after liveness analysis" },
1677 { CPU_LOG_INT
, "int",
1678 "show interrupts/exceptions in short format" },
1679 { CPU_LOG_EXEC
, "exec",
1680 "show trace before each executed TB (lots of logs)" },
1681 { CPU_LOG_TB_CPU
, "cpu",
1682 "show CPU state before block translation" },
1684 { CPU_LOG_PCALL
, "pcall",
1685 "show protected mode far calls/returns/exceptions" },
1686 { CPU_LOG_RESET
, "cpu_reset",
1687 "show CPU state before CPU resets" },
1690 { CPU_LOG_IOPORT
, "ioport",
1691 "show all i/o ports accesses" },
1696 #ifndef CONFIG_USER_ONLY
1697 static QLIST_HEAD(memory_client_list
, CPUPhysMemoryClient
) memory_client_list
1698 = QLIST_HEAD_INITIALIZER(memory_client_list
);
1700 static void cpu_notify_set_memory(target_phys_addr_t start_addr
,
1702 ram_addr_t phys_offset
)
1704 CPUPhysMemoryClient
*client
;
1705 QLIST_FOREACH(client
, &memory_client_list
, list
) {
1706 client
->set_memory(client
, start_addr
, size
, phys_offset
);
1710 static int cpu_notify_sync_dirty_bitmap(target_phys_addr_t start
,
1711 target_phys_addr_t end
)
1713 CPUPhysMemoryClient
*client
;
1714 QLIST_FOREACH(client
, &memory_client_list
, list
) {
1715 int r
= client
->sync_dirty_bitmap(client
, start
, end
);
1722 static int cpu_notify_migration_log(int enable
)
1724 CPUPhysMemoryClient
*client
;
1725 QLIST_FOREACH(client
, &memory_client_list
, list
) {
1726 int r
= client
->migration_log(client
, enable
);
1733 static void phys_page_for_each_1(CPUPhysMemoryClient
*client
,
1734 int level
, void **lp
)
1742 PhysPageDesc
*pd
= *lp
;
1743 for (i
= 0; i
< L2_SIZE
; ++i
) {
1744 if (pd
[i
].phys_offset
!= IO_MEM_UNASSIGNED
) {
1745 client
->set_memory(client
, pd
[i
].region_offset
,
1746 TARGET_PAGE_SIZE
, pd
[i
].phys_offset
);
1751 for (i
= 0; i
< L2_SIZE
; ++i
) {
1752 phys_page_for_each_1(client
, level
- 1, pp
+ i
);
1757 static void phys_page_for_each(CPUPhysMemoryClient
*client
)
1760 for (i
= 0; i
< P_L1_SIZE
; ++i
) {
1761 phys_page_for_each_1(client
, P_L1_SHIFT
/ L2_BITS
- 1,
1766 void cpu_register_phys_memory_client(CPUPhysMemoryClient
*client
)
1768 QLIST_INSERT_HEAD(&memory_client_list
, client
, list
);
1769 phys_page_for_each(client
);
1772 void cpu_unregister_phys_memory_client(CPUPhysMemoryClient
*client
)
1774 QLIST_REMOVE(client
, list
);
1778 static int cmp1(const char *s1
, int n
, const char *s2
)
1780 if (strlen(s2
) != n
)
1782 return memcmp(s1
, s2
, n
) == 0;
1785 /* takes a comma separated list of log masks. Return 0 if error. */
1786 int cpu_str_to_log_mask(const char *str
)
1788 const CPULogItem
*item
;
1795 p1
= strchr(p
, ',');
1798 if(cmp1(p
,p1
-p
,"all")) {
1799 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1803 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1804 if (cmp1(p
, p1
- p
, item
->name
))
1818 void cpu_abort(CPUState
*env
, const char *fmt
, ...)
1825 fprintf(stderr
, "qemu: fatal: ");
1826 vfprintf(stderr
, fmt
, ap
);
1827 fprintf(stderr
, "\n");
1829 cpu_dump_state(env
, stderr
, fprintf
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1831 cpu_dump_state(env
, stderr
, fprintf
, 0);
1833 if (qemu_log_enabled()) {
1834 qemu_log("qemu: fatal: ");
1835 qemu_log_vprintf(fmt
, ap2
);
1838 log_cpu_state(env
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1840 log_cpu_state(env
, 0);
1847 #if defined(CONFIG_USER_ONLY)
1849 struct sigaction act
;
1850 sigfillset(&act
.sa_mask
);
1851 act
.sa_handler
= SIG_DFL
;
1852 sigaction(SIGABRT
, &act
, NULL
);
1858 CPUState
*cpu_copy(CPUState
*env
)
1860 CPUState
*new_env
= cpu_init(env
->cpu_model_str
);
1861 CPUState
*next_cpu
= new_env
->next_cpu
;
1862 int cpu_index
= new_env
->cpu_index
;
1863 #if defined(TARGET_HAS_ICE)
1868 memcpy(new_env
, env
, sizeof(CPUState
));
1870 /* Preserve chaining and index. */
1871 new_env
->next_cpu
= next_cpu
;
1872 new_env
->cpu_index
= cpu_index
;
1874 /* Clone all break/watchpoints.
1875 Note: Once we support ptrace with hw-debug register access, make sure
1876 BP_CPU break/watchpoints are handled correctly on clone. */
1877 QTAILQ_INIT(&env
->breakpoints
);
1878 QTAILQ_INIT(&env
->watchpoints
);
1879 #if defined(TARGET_HAS_ICE)
1880 QTAILQ_FOREACH(bp
, &env
->breakpoints
, entry
) {
1881 cpu_breakpoint_insert(new_env
, bp
->pc
, bp
->flags
, NULL
);
1883 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1884 cpu_watchpoint_insert(new_env
, wp
->vaddr
, (~wp
->len_mask
) + 1,
1892 #if !defined(CONFIG_USER_ONLY)
1894 static inline void tlb_flush_jmp_cache(CPUState
*env
, target_ulong addr
)
1898 /* Discard jump cache entries for any tb which might potentially
1899 overlap the flushed page. */
1900 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1901 memset (&env
->tb_jmp_cache
[i
], 0,
1902 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1904 i
= tb_jmp_cache_hash_page(addr
);
1905 memset (&env
->tb_jmp_cache
[i
], 0,
1906 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1909 static CPUTLBEntry s_cputlb_empty_entry
= {
1916 /* NOTE: if flush_global is true, also flush global entries (not
1918 void tlb_flush(CPUState
*env
, int flush_global
)
1922 #if defined(DEBUG_TLB)
1923 printf("tlb_flush:\n");
1925 /* must reset current TB so that interrupts cannot modify the
1926 links while we are modifying them */
1927 env
->current_tb
= NULL
;
1929 for(i
= 0; i
< CPU_TLB_SIZE
; i
++) {
1931 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
1932 env
->tlb_table
[mmu_idx
][i
] = s_cputlb_empty_entry
;
1936 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
1941 static inline void tlb_flush_entry(CPUTLBEntry
*tlb_entry
, target_ulong addr
)
1943 if (addr
== (tlb_entry
->addr_read
&
1944 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1945 addr
== (tlb_entry
->addr_write
&
1946 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1947 addr
== (tlb_entry
->addr_code
&
1948 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
))) {
1949 *tlb_entry
= s_cputlb_empty_entry
;
1953 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
1958 #if defined(DEBUG_TLB)
1959 printf("tlb_flush_page: " TARGET_FMT_lx
"\n", addr
);
1961 /* must reset current TB so that interrupts cannot modify the
1962 links while we are modifying them */
1963 env
->current_tb
= NULL
;
1965 addr
&= TARGET_PAGE_MASK
;
1966 i
= (addr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
1967 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++)
1968 tlb_flush_entry(&env
->tlb_table
[mmu_idx
][i
], addr
);
1970 tlb_flush_jmp_cache(env
, addr
);
1973 /* update the TLBs so that writes to code in the virtual page 'addr'
1975 static void tlb_protect_code(ram_addr_t ram_addr
)
1977 cpu_physical_memory_reset_dirty(ram_addr
,
1978 ram_addr
+ TARGET_PAGE_SIZE
,
1982 /* update the TLB so that writes in physical page 'phys_addr' are no longer
1983 tested for self modifying code */
1984 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
1987 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] |= CODE_DIRTY_FLAG
;
1990 static inline void tlb_reset_dirty_range(CPUTLBEntry
*tlb_entry
,
1991 unsigned long start
, unsigned long length
)
1994 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
) {
1995 addr
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) + tlb_entry
->addend
;
1996 if ((addr
- start
) < length
) {
1997 tlb_entry
->addr_write
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) | TLB_NOTDIRTY
;
2002 /* Note: start and end must be within the same ram block. */
2003 void cpu_physical_memory_reset_dirty(ram_addr_t start
, ram_addr_t end
,
2007 unsigned long length
, start1
;
2011 start
&= TARGET_PAGE_MASK
;
2012 end
= TARGET_PAGE_ALIGN(end
);
2014 length
= end
- start
;
2017 len
= length
>> TARGET_PAGE_BITS
;
2018 mask
= ~dirty_flags
;
2019 p
= phys_ram_dirty
+ (start
>> TARGET_PAGE_BITS
);
2020 for(i
= 0; i
< len
; i
++)
2023 /* we modify the TLB cache so that the dirty bit will be set again
2024 when accessing the range */
2025 start1
= (unsigned long)qemu_get_ram_ptr(start
);
2026 /* Chek that we don't span multiple blocks - this breaks the
2027 address comparisons below. */
2028 if ((unsigned long)qemu_get_ram_ptr(end
- 1) - start1
2029 != (end
- 1) - start
) {
2033 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
2035 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
2036 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
2037 tlb_reset_dirty_range(&env
->tlb_table
[mmu_idx
][i
],
2043 int cpu_physical_memory_set_dirty_tracking(int enable
)
2046 in_migration
= enable
;
2047 ret
= cpu_notify_migration_log(!!enable
);
2051 int cpu_physical_memory_get_dirty_tracking(void)
2053 return in_migration
;
2056 int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr
,
2057 target_phys_addr_t end_addr
)
2061 ret
= cpu_notify_sync_dirty_bitmap(start_addr
, end_addr
);
2065 static inline void tlb_update_dirty(CPUTLBEntry
*tlb_entry
)
2067 ram_addr_t ram_addr
;
2070 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
) {
2071 p
= (void *)(unsigned long)((tlb_entry
->addr_write
& TARGET_PAGE_MASK
)
2072 + tlb_entry
->addend
);
2073 ram_addr
= qemu_ram_addr_from_host(p
);
2074 if (!cpu_physical_memory_is_dirty(ram_addr
)) {
2075 tlb_entry
->addr_write
|= TLB_NOTDIRTY
;
2080 /* update the TLB according to the current state of the dirty bits */
2081 void cpu_tlb_update_dirty(CPUState
*env
)
2085 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++) {
2086 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
2087 tlb_update_dirty(&env
->tlb_table
[mmu_idx
][i
]);
2091 static inline void tlb_set_dirty1(CPUTLBEntry
*tlb_entry
, target_ulong vaddr
)
2093 if (tlb_entry
->addr_write
== (vaddr
| TLB_NOTDIRTY
))
2094 tlb_entry
->addr_write
= vaddr
;
2097 /* update the TLB corresponding to virtual page vaddr
2098 so that it is no longer dirty */
2099 static inline void tlb_set_dirty(CPUState
*env
, target_ulong vaddr
)
2104 vaddr
&= TARGET_PAGE_MASK
;
2105 i
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
2106 for (mmu_idx
= 0; mmu_idx
< NB_MMU_MODES
; mmu_idx
++)
2107 tlb_set_dirty1(&env
->tlb_table
[mmu_idx
][i
], vaddr
);
2110 /* add a new TLB entry. At most one entry for a given virtual address
2111 is permitted. Return 0 if OK or 2 if the page could not be mapped
2112 (can only happen in non SOFTMMU mode for I/O pages or pages
2113 conflicting with the host address space). */
2114 int tlb_set_page_exec(CPUState
*env
, target_ulong vaddr
,
2115 target_phys_addr_t paddr
, int prot
,
2116 int mmu_idx
, int is_softmmu
)
2121 target_ulong address
;
2122 target_ulong code_address
;
2123 target_phys_addr_t addend
;
2127 target_phys_addr_t iotlb
;
2129 p
= phys_page_find(paddr
>> TARGET_PAGE_BITS
);
2131 pd
= IO_MEM_UNASSIGNED
;
2133 pd
= p
->phys_offset
;
2135 #if defined(DEBUG_TLB)
2136 printf("tlb_set_page: vaddr=" TARGET_FMT_lx
" paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
2137 vaddr
, (int)paddr
, prot
, mmu_idx
, is_softmmu
, pd
);
2142 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&& !(pd
& IO_MEM_ROMD
)) {
2143 /* IO memory case (romd handled later) */
2144 address
|= TLB_MMIO
;
2146 addend
= (unsigned long)qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
);
2147 if ((pd
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
) {
2149 iotlb
= pd
& TARGET_PAGE_MASK
;
2150 if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
)
2151 iotlb
|= IO_MEM_NOTDIRTY
;
2153 iotlb
|= IO_MEM_ROM
;
2155 /* IO handlers are currently passed a physical address.
2156 It would be nice to pass an offset from the base address
2157 of that region. This would avoid having to special case RAM,
2158 and avoid full address decoding in every device.
2159 We can't use the high bits of pd for this because
2160 IO_MEM_ROMD uses these as a ram address. */
2161 iotlb
= (pd
& ~TARGET_PAGE_MASK
);
2163 iotlb
+= p
->region_offset
;
2169 code_address
= address
;
2170 /* Make accesses to pages with watchpoints go via the
2171 watchpoint trap routines. */
2172 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
2173 if (vaddr
== (wp
->vaddr
& TARGET_PAGE_MASK
)) {
2174 iotlb
= io_mem_watch
+ paddr
;
2175 /* TODO: The memory case can be optimized by not trapping
2176 reads of pages with a write breakpoint. */
2177 address
|= TLB_MMIO
;
2181 index
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
2182 env
->iotlb
[mmu_idx
][index
] = iotlb
- vaddr
;
2183 te
= &env
->tlb_table
[mmu_idx
][index
];
2184 te
->addend
= addend
- vaddr
;
2185 if (prot
& PAGE_READ
) {
2186 te
->addr_read
= address
;
2191 if (prot
& PAGE_EXEC
) {
2192 te
->addr_code
= code_address
;
2196 if (prot
& PAGE_WRITE
) {
2197 if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_ROM
||
2198 (pd
& IO_MEM_ROMD
)) {
2199 /* Write access calls the I/O callback. */
2200 te
->addr_write
= address
| TLB_MMIO
;
2201 } else if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
&&
2202 !cpu_physical_memory_is_dirty(pd
)) {
2203 te
->addr_write
= address
| TLB_NOTDIRTY
;
2205 te
->addr_write
= address
;
2208 te
->addr_write
= -1;
2215 void tlb_flush(CPUState
*env
, int flush_global
)
2219 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
2224 * Walks guest process memory "regions" one by one
2225 * and calls callback function 'fn' for each region.
2228 struct walk_memory_regions_data
2230 walk_memory_regions_fn fn
;
2232 unsigned long start
;
2236 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
2237 abi_ulong end
, int new_prot
)
2239 if (data
->start
!= -1ul) {
2240 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
2246 data
->start
= (new_prot
? end
: -1ul);
2247 data
->prot
= new_prot
;
2252 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
2253 abi_ulong base
, int level
, void **lp
)
2259 return walk_memory_regions_end(data
, base
, 0);
2264 for (i
= 0; i
< L2_SIZE
; ++i
) {
2265 int prot
= pd
[i
].flags
;
2267 pa
= base
| (i
<< TARGET_PAGE_BITS
);
2268 if (prot
!= data
->prot
) {
2269 rc
= walk_memory_regions_end(data
, pa
, prot
);
2277 for (i
= 0; i
< L2_SIZE
; ++i
) {
2278 pa
= base
| ((abi_ulong
)i
<<
2279 (TARGET_PAGE_BITS
+ L2_BITS
* level
));
2280 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
2290 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
2292 struct walk_memory_regions_data data
;
2300 for (i
= 0; i
< V_L1_SIZE
; i
++) {
2301 int rc
= walk_memory_regions_1(&data
, (abi_ulong
)i
<< V_L1_SHIFT
,
2302 V_L1_SHIFT
/ L2_BITS
- 1, l1_map
+ i
);
2308 return walk_memory_regions_end(&data
, 0, 0);
2311 static int dump_region(void *priv
, abi_ulong start
,
2312 abi_ulong end
, unsigned long prot
)
2314 FILE *f
= (FILE *)priv
;
2316 (void) fprintf(f
, TARGET_ABI_FMT_lx
"-"TARGET_ABI_FMT_lx
2317 " "TARGET_ABI_FMT_lx
" %c%c%c\n",
2318 start
, end
, end
- start
,
2319 ((prot
& PAGE_READ
) ? 'r' : '-'),
2320 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
2321 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
2326 /* dump memory mappings */
2327 void page_dump(FILE *f
)
2329 (void) fprintf(f
, "%-8s %-8s %-8s %s\n",
2330 "start", "end", "size", "prot");
2331 walk_memory_regions(f
, dump_region
);
2334 int page_get_flags(target_ulong address
)
2338 p
= page_find(address
>> TARGET_PAGE_BITS
);
2344 /* Modify the flags of a page and invalidate the code if necessary.
2345 The flag PAGE_WRITE_ORG is positioned automatically depending
2346 on PAGE_WRITE. The mmap_lock should already be held. */
2347 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
2349 target_ulong addr
, len
;
2351 /* This function should never be called with addresses outside the
2352 guest address space. If this assert fires, it probably indicates
2353 a missing call to h2g_valid. */
2354 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2355 assert(end
< ((abi_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2357 assert(start
< end
);
2359 start
= start
& TARGET_PAGE_MASK
;
2360 end
= TARGET_PAGE_ALIGN(end
);
2362 if (flags
& PAGE_WRITE
) {
2363 flags
|= PAGE_WRITE_ORG
;
2366 for (addr
= start
, len
= end
- start
;
2368 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2369 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2371 /* If the write protection bit is set, then we invalidate
2373 if (!(p
->flags
& PAGE_WRITE
) &&
2374 (flags
& PAGE_WRITE
) &&
2376 tb_invalidate_phys_page(addr
, 0, NULL
);
2382 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
2388 /* This function should never be called with addresses outside the
2389 guest address space. If this assert fires, it probably indicates
2390 a missing call to h2g_valid. */
2391 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2392 assert(start
< ((abi_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2395 if (start
+ len
- 1 < start
) {
2396 /* We've wrapped around. */
2400 end
= TARGET_PAGE_ALIGN(start
+len
); /* must do before we loose bits in the next step */
2401 start
= start
& TARGET_PAGE_MASK
;
2403 for (addr
= start
, len
= end
- start
;
2405 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2406 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2409 if( !(p
->flags
& PAGE_VALID
) )
2412 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
))
2414 if (flags
& PAGE_WRITE
) {
2415 if (!(p
->flags
& PAGE_WRITE_ORG
))
2417 /* unprotect the page if it was put read-only because it
2418 contains translated code */
2419 if (!(p
->flags
& PAGE_WRITE
)) {
2420 if (!page_unprotect(addr
, 0, NULL
))
2429 /* called from signal handler: invalidate the code and unprotect the
2430 page. Return TRUE if the fault was successfully handled. */
2431 int page_unprotect(target_ulong address
, unsigned long pc
, void *puc
)
2433 unsigned int page_index
, prot
, pindex
;
2435 target_ulong host_start
, host_end
, addr
;
2437 /* Technically this isn't safe inside a signal handler. However we
2438 know this only ever happens in a synchronous SEGV handler, so in
2439 practice it seems to be ok. */
2442 host_start
= address
& qemu_host_page_mask
;
2443 page_index
= host_start
>> TARGET_PAGE_BITS
;
2444 p1
= page_find(page_index
);
2449 host_end
= host_start
+ qemu_host_page_size
;
2452 for(addr
= host_start
;addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2456 /* if the page was really writable, then we change its
2457 protection back to writable */
2458 if (prot
& PAGE_WRITE_ORG
) {
2459 pindex
= (address
- host_start
) >> TARGET_PAGE_BITS
;
2460 if (!(p1
[pindex
].flags
& PAGE_WRITE
)) {
2461 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2462 (prot
& PAGE_BITS
) | PAGE_WRITE
);
2463 p1
[pindex
].flags
|= PAGE_WRITE
;
2464 /* and since the content will be modified, we must invalidate
2465 the corresponding translated code. */
2466 tb_invalidate_phys_page(address
, pc
, puc
);
2467 #ifdef DEBUG_TB_CHECK
2468 tb_invalidate_check(address
);
2478 static inline void tlb_set_dirty(CPUState
*env
,
2479 unsigned long addr
, target_ulong vaddr
)
2482 #endif /* defined(CONFIG_USER_ONLY) */
2484 #if !defined(CONFIG_USER_ONLY)
2486 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
2487 typedef struct subpage_t
{
2488 target_phys_addr_t base
;
2489 CPUReadMemoryFunc
* const *mem_read
[TARGET_PAGE_SIZE
][4];
2490 CPUWriteMemoryFunc
* const *mem_write
[TARGET_PAGE_SIZE
][4];
2491 void *opaque
[TARGET_PAGE_SIZE
][2][4];
2492 ram_addr_t region_offset
[TARGET_PAGE_SIZE
][2][4];
2495 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2496 ram_addr_t memory
, ram_addr_t region_offset
);
2497 static void *subpage_init (target_phys_addr_t base
, ram_addr_t
*phys
,
2498 ram_addr_t orig_memory
, ram_addr_t region_offset
);
2499 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2502 if (addr > start_addr) \
2505 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2506 if (start_addr2 > 0) \
2510 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2511 end_addr2 = TARGET_PAGE_SIZE - 1; \
2513 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2514 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2519 /* register physical memory.
2520 For RAM, 'size' must be a multiple of the target page size.
2521 If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2522 io memory page. The address used when calling the IO function is
2523 the offset from the start of the region, plus region_offset. Both
2524 start_addr and region_offset are rounded down to a page boundary
2525 before calculating this offset. This should not be a problem unless
2526 the low bits of start_addr and region_offset differ. */
2527 void cpu_register_physical_memory_offset(target_phys_addr_t start_addr
,
2529 ram_addr_t phys_offset
,
2530 ram_addr_t region_offset
)
2532 target_phys_addr_t addr
, end_addr
;
2535 ram_addr_t orig_size
= size
;
2538 cpu_notify_set_memory(start_addr
, size
, phys_offset
);
2540 if (phys_offset
== IO_MEM_UNASSIGNED
) {
2541 region_offset
= start_addr
;
2543 region_offset
&= TARGET_PAGE_MASK
;
2544 size
= (size
+ TARGET_PAGE_SIZE
- 1) & TARGET_PAGE_MASK
;
2545 end_addr
= start_addr
+ (target_phys_addr_t
)size
;
2546 for(addr
= start_addr
; addr
!= end_addr
; addr
+= TARGET_PAGE_SIZE
) {
2547 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2548 if (p
&& p
->phys_offset
!= IO_MEM_UNASSIGNED
) {
2549 ram_addr_t orig_memory
= p
->phys_offset
;
2550 target_phys_addr_t start_addr2
, end_addr2
;
2551 int need_subpage
= 0;
2553 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
, end_addr2
,
2555 if (need_subpage
|| phys_offset
& IO_MEM_SUBWIDTH
) {
2556 if (!(orig_memory
& IO_MEM_SUBPAGE
)) {
2557 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2558 &p
->phys_offset
, orig_memory
,
2561 subpage
= io_mem_opaque
[(orig_memory
& ~TARGET_PAGE_MASK
)
2564 subpage_register(subpage
, start_addr2
, end_addr2
, phys_offset
,
2566 p
->region_offset
= 0;
2568 p
->phys_offset
= phys_offset
;
2569 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
||
2570 (phys_offset
& IO_MEM_ROMD
))
2571 phys_offset
+= TARGET_PAGE_SIZE
;
2574 p
= phys_page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2575 p
->phys_offset
= phys_offset
;
2576 p
->region_offset
= region_offset
;
2577 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
||
2578 (phys_offset
& IO_MEM_ROMD
)) {
2579 phys_offset
+= TARGET_PAGE_SIZE
;
2581 target_phys_addr_t start_addr2
, end_addr2
;
2582 int need_subpage
= 0;
2584 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
,
2585 end_addr2
, need_subpage
);
2587 if (need_subpage
|| phys_offset
& IO_MEM_SUBWIDTH
) {
2588 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2589 &p
->phys_offset
, IO_MEM_UNASSIGNED
,
2590 addr
& TARGET_PAGE_MASK
);
2591 subpage_register(subpage
, start_addr2
, end_addr2
,
2592 phys_offset
, region_offset
);
2593 p
->region_offset
= 0;
2597 region_offset
+= TARGET_PAGE_SIZE
;
2600 /* since each CPU stores ram addresses in its TLB cache, we must
2601 reset the modified entries */
2603 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
2608 /* XXX: temporary until new memory mapping API */
2609 ram_addr_t
cpu_get_physical_page_desc(target_phys_addr_t addr
)
2613 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2615 return IO_MEM_UNASSIGNED
;
2616 return p
->phys_offset
;
2619 void qemu_register_coalesced_mmio(target_phys_addr_t addr
, ram_addr_t size
)
2622 kvm_coalesce_mmio_region(addr
, size
);
2625 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr
, ram_addr_t size
)
2628 kvm_uncoalesce_mmio_region(addr
, size
);
2631 void qemu_flush_coalesced_mmio_buffer(void)
2634 kvm_flush_coalesced_mmio_buffer();
2637 #if defined(__linux__) && !defined(TARGET_S390X)
2639 #include <sys/vfs.h>
2641 #define HUGETLBFS_MAGIC 0x958458f6
2643 static long gethugepagesize(const char *path
)
2649 ret
= statfs(path
, &fs
);
2650 } while (ret
!= 0 && errno
== EINTR
);
2657 if (fs
.f_type
!= HUGETLBFS_MAGIC
)
2658 fprintf(stderr
, "Warning: path not on HugeTLBFS: %s\n", path
);
2663 static void *file_ram_alloc(ram_addr_t memory
, const char *path
)
2671 unsigned long hpagesize
;
2673 hpagesize
= gethugepagesize(path
);
2678 if (memory
< hpagesize
) {
2682 if (kvm_enabled() && !kvm_has_sync_mmu()) {
2683 fprintf(stderr
, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
2687 if (asprintf(&filename
, "%s/qemu_back_mem.XXXXXX", path
) == -1) {
2691 fd
= mkstemp(filename
);
2700 memory
= (memory
+hpagesize
-1) & ~(hpagesize
-1);
2703 * ftruncate is not supported by hugetlbfs in older
2704 * hosts, so don't bother bailing out on errors.
2705 * If anything goes wrong with it under other filesystems,
2708 if (ftruncate(fd
, memory
))
2709 perror("ftruncate");
2712 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
2713 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
2714 * to sidestep this quirk.
2716 flags
= mem_prealloc
? MAP_POPULATE
| MAP_SHARED
: MAP_PRIVATE
;
2717 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
, flags
, fd
, 0);
2719 area
= mmap(0, memory
, PROT_READ
| PROT_WRITE
, MAP_PRIVATE
, fd
, 0);
2721 if (area
== MAP_FAILED
) {
2722 perror("file_ram_alloc: can't mmap RAM pages");
2730 ram_addr_t
qemu_ram_alloc(ram_addr_t size
)
2732 RAMBlock
*new_block
;
2734 size
= TARGET_PAGE_ALIGN(size
);
2735 new_block
= qemu_malloc(sizeof(*new_block
));
2738 #if defined (__linux__) && !defined(TARGET_S390X)
2739 new_block
->host
= file_ram_alloc(size
, mem_path
);
2740 if (!new_block
->host
)
2743 fprintf(stderr
, "-mem-path option unsupported\n");
2747 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2748 /* XXX S390 KVM requires the topmost vma of the RAM to be < 256GB */
2749 new_block
->host
= mmap((void*)0x1000000, size
,
2750 PROT_EXEC
|PROT_READ
|PROT_WRITE
,
2751 MAP_SHARED
| MAP_ANONYMOUS
, -1, 0);
2753 new_block
->host
= qemu_vmalloc(size
);
2755 #ifdef MADV_MERGEABLE
2756 madvise(new_block
->host
, size
, MADV_MERGEABLE
);
2759 new_block
->offset
= last_ram_offset
;
2760 new_block
->length
= size
;
2762 new_block
->next
= ram_blocks
;
2763 ram_blocks
= new_block
;
2765 phys_ram_dirty
= qemu_realloc(phys_ram_dirty
,
2766 (last_ram_offset
+ size
) >> TARGET_PAGE_BITS
);
2767 memset(phys_ram_dirty
+ (last_ram_offset
>> TARGET_PAGE_BITS
),
2768 0xff, size
>> TARGET_PAGE_BITS
);
2770 last_ram_offset
+= size
;
2773 kvm_setup_guest_memory(new_block
->host
, size
);
2775 return new_block
->offset
;
2778 void qemu_ram_free(ram_addr_t addr
)
2780 /* TODO: implement this. */
2783 /* Return a host pointer to ram allocated with qemu_ram_alloc.
2784 With the exception of the softmmu code in this file, this should
2785 only be used for local memory (e.g. video ram) that the device owns,
2786 and knows it isn't going to access beyond the end of the block.
2788 It should not be used for general purpose DMA.
2789 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
2791 void *qemu_get_ram_ptr(ram_addr_t addr
)
2798 prevp
= &ram_blocks
;
2800 while (block
&& (block
->offset
> addr
2801 || block
->offset
+ block
->length
<= addr
)) {
2803 prevp
= &prev
->next
;
2805 block
= block
->next
;
2808 fprintf(stderr
, "Bad ram offset %" PRIx64
"\n", (uint64_t)addr
);
2811 /* Move this entry to to start of the list. */
2813 prev
->next
= block
->next
;
2814 block
->next
= *prevp
;
2817 return block
->host
+ (addr
- block
->offset
);
2820 int do_qemu_ram_addr_from_host(void *ptr
, ram_addr_t
*ram_addr
)
2824 uint8_t *host
= ptr
;
2828 while (block
&& (block
->host
> host
2829 || block
->host
+ block
->length
<= host
)) {
2831 block
= block
->next
;
2835 *ram_addr
= block
->offset
+ (host
- block
->host
);
2839 /* Some of the softmmu routines need to translate from a host pointer
2840 (typically a TLB entry) back to a ram offset. */
2841 ram_addr_t
qemu_ram_addr_from_host(void *ptr
)
2843 ram_addr_t ram_addr
;
2845 if (do_qemu_ram_addr_from_host(ptr
, &ram_addr
)) {
2846 fprintf(stderr
, "Bad ram pointer %p\n", ptr
);
2852 static uint32_t unassigned_mem_readb(void *opaque
, target_phys_addr_t addr
)
2854 #ifdef DEBUG_UNASSIGNED
2855 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2857 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2858 do_unassigned_access(addr
, 0, 0, 0, 1);
2863 static uint32_t unassigned_mem_readw(void *opaque
, target_phys_addr_t addr
)
2865 #ifdef DEBUG_UNASSIGNED
2866 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2868 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2869 do_unassigned_access(addr
, 0, 0, 0, 2);
2874 static uint32_t unassigned_mem_readl(void *opaque
, target_phys_addr_t addr
)
2876 #ifdef DEBUG_UNASSIGNED
2877 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2879 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2880 do_unassigned_access(addr
, 0, 0, 0, 4);
2885 static void unassigned_mem_writeb(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2887 #ifdef DEBUG_UNASSIGNED
2888 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2890 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2891 do_unassigned_access(addr
, 1, 0, 0, 1);
2895 static void unassigned_mem_writew(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2897 #ifdef DEBUG_UNASSIGNED
2898 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2900 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2901 do_unassigned_access(addr
, 1, 0, 0, 2);
2905 static void unassigned_mem_writel(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2907 #ifdef DEBUG_UNASSIGNED
2908 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2910 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2911 do_unassigned_access(addr
, 1, 0, 0, 4);
2915 static CPUReadMemoryFunc
* const unassigned_mem_read
[3] = {
2916 unassigned_mem_readb
,
2917 unassigned_mem_readw
,
2918 unassigned_mem_readl
,
2921 static CPUWriteMemoryFunc
* const unassigned_mem_write
[3] = {
2922 unassigned_mem_writeb
,
2923 unassigned_mem_writew
,
2924 unassigned_mem_writel
,
2927 static void notdirty_mem_writeb(void *opaque
, target_phys_addr_t ram_addr
,
2931 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2932 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
2933 #if !defined(CONFIG_USER_ONLY)
2934 tb_invalidate_phys_page_fast(ram_addr
, 1);
2935 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2938 stb_p(qemu_get_ram_ptr(ram_addr
), val
);
2939 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
2940 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
2941 /* we remove the notdirty callback only if the code has been
2943 if (dirty_flags
== 0xff)
2944 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
2947 static void notdirty_mem_writew(void *opaque
, target_phys_addr_t ram_addr
,
2951 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2952 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
2953 #if !defined(CONFIG_USER_ONLY)
2954 tb_invalidate_phys_page_fast(ram_addr
, 2);
2955 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2958 stw_p(qemu_get_ram_ptr(ram_addr
), val
);
2959 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
2960 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
2961 /* we remove the notdirty callback only if the code has been
2963 if (dirty_flags
== 0xff)
2964 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
2967 static void notdirty_mem_writel(void *opaque
, target_phys_addr_t ram_addr
,
2971 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2972 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
2973 #if !defined(CONFIG_USER_ONLY)
2974 tb_invalidate_phys_page_fast(ram_addr
, 4);
2975 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2978 stl_p(qemu_get_ram_ptr(ram_addr
), val
);
2979 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
2980 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
2981 /* we remove the notdirty callback only if the code has been
2983 if (dirty_flags
== 0xff)
2984 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
2987 static CPUReadMemoryFunc
* const error_mem_read
[3] = {
2988 NULL
, /* never used */
2989 NULL
, /* never used */
2990 NULL
, /* never used */
2993 static CPUWriteMemoryFunc
* const notdirty_mem_write
[3] = {
2994 notdirty_mem_writeb
,
2995 notdirty_mem_writew
,
2996 notdirty_mem_writel
,
2999 /* Generate a debug exception if a watchpoint has been hit. */
3000 static void check_watchpoint(int offset
, int len_mask
, int flags
)
3002 CPUState
*env
= cpu_single_env
;
3003 target_ulong pc
, cs_base
;
3004 TranslationBlock
*tb
;
3009 if (env
->watchpoint_hit
) {
3010 /* We re-entered the check after replacing the TB. Now raise
3011 * the debug interrupt so that is will trigger after the
3012 * current instruction. */
3013 cpu_interrupt(env
, CPU_INTERRUPT_DEBUG
);
3016 vaddr
= (env
->mem_io_vaddr
& TARGET_PAGE_MASK
) + offset
;
3017 QTAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
3018 if ((vaddr
== (wp
->vaddr
& len_mask
) ||
3019 (vaddr
& wp
->len_mask
) == wp
->vaddr
) && (wp
->flags
& flags
)) {
3020 wp
->flags
|= BP_WATCHPOINT_HIT
;
3021 if (!env
->watchpoint_hit
) {
3022 env
->watchpoint_hit
= wp
;
3023 tb
= tb_find_pc(env
->mem_io_pc
);
3025 cpu_abort(env
, "check_watchpoint: could not find TB for "
3026 "pc=%p", (void *)env
->mem_io_pc
);
3028 cpu_restore_state(tb
, env
, env
->mem_io_pc
, NULL
);
3029 tb_phys_invalidate(tb
, -1);
3030 if (wp
->flags
& BP_STOP_BEFORE_ACCESS
) {
3031 env
->exception_index
= EXCP_DEBUG
;
3033 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &cpu_flags
);
3034 tb_gen_code(env
, pc
, cs_base
, cpu_flags
, 1);
3036 cpu_resume_from_signal(env
, NULL
);
3039 wp
->flags
&= ~BP_WATCHPOINT_HIT
;
3044 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
3045 so these check for a hit then pass through to the normal out-of-line
3047 static uint32_t watch_mem_readb(void *opaque
, target_phys_addr_t addr
)
3049 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x0, BP_MEM_READ
);
3050 return ldub_phys(addr
);
3053 static uint32_t watch_mem_readw(void *opaque
, target_phys_addr_t addr
)
3055 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x1, BP_MEM_READ
);
3056 return lduw_phys(addr
);
3059 static uint32_t watch_mem_readl(void *opaque
, target_phys_addr_t addr
)
3061 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x3, BP_MEM_READ
);
3062 return ldl_phys(addr
);
3065 static void watch_mem_writeb(void *opaque
, target_phys_addr_t addr
,
3068 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x0, BP_MEM_WRITE
);
3069 stb_phys(addr
, val
);
3072 static void watch_mem_writew(void *opaque
, target_phys_addr_t addr
,
3075 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x1, BP_MEM_WRITE
);
3076 stw_phys(addr
, val
);
3079 static void watch_mem_writel(void *opaque
, target_phys_addr_t addr
,
3082 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x3, BP_MEM_WRITE
);
3083 stl_phys(addr
, val
);
3086 static CPUReadMemoryFunc
* const watch_mem_read
[3] = {
3092 static CPUWriteMemoryFunc
* const watch_mem_write
[3] = {
3098 static inline uint32_t subpage_readlen (subpage_t
*mmio
, target_phys_addr_t addr
,
3104 idx
= SUBPAGE_IDX(addr
);
3105 #if defined(DEBUG_SUBPAGE)
3106 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d\n", __func__
,
3107 mmio
, len
, addr
, idx
);
3109 ret
= (**mmio
->mem_read
[idx
][len
])(mmio
->opaque
[idx
][0][len
],
3110 addr
+ mmio
->region_offset
[idx
][0][len
]);
3115 static inline void subpage_writelen (subpage_t
*mmio
, target_phys_addr_t addr
,
3116 uint32_t value
, unsigned int len
)
3120 idx
= SUBPAGE_IDX(addr
);
3121 #if defined(DEBUG_SUBPAGE)
3122 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d value %08x\n", __func__
,
3123 mmio
, len
, addr
, idx
, value
);
3125 (**mmio
->mem_write
[idx
][len
])(mmio
->opaque
[idx
][1][len
],
3126 addr
+ mmio
->region_offset
[idx
][1][len
],
3130 static uint32_t subpage_readb (void *opaque
, target_phys_addr_t addr
)
3132 #if defined(DEBUG_SUBPAGE)
3133 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
3136 return subpage_readlen(opaque
, addr
, 0);
3139 static void subpage_writeb (void *opaque
, target_phys_addr_t addr
,
3142 #if defined(DEBUG_SUBPAGE)
3143 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
3145 subpage_writelen(opaque
, addr
, value
, 0);
3148 static uint32_t subpage_readw (void *opaque
, target_phys_addr_t addr
)
3150 #if defined(DEBUG_SUBPAGE)
3151 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
3154 return subpage_readlen(opaque
, addr
, 1);
3157 static void subpage_writew (void *opaque
, target_phys_addr_t addr
,
3160 #if defined(DEBUG_SUBPAGE)
3161 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
3163 subpage_writelen(opaque
, addr
, value
, 1);
3166 static uint32_t subpage_readl (void *opaque
, target_phys_addr_t addr
)
3168 #if defined(DEBUG_SUBPAGE)
3169 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
3172 return subpage_readlen(opaque
, addr
, 2);
3175 static void subpage_writel (void *opaque
,
3176 target_phys_addr_t addr
, uint32_t value
)
3178 #if defined(DEBUG_SUBPAGE)
3179 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
3181 subpage_writelen(opaque
, addr
, value
, 2);
3184 static CPUReadMemoryFunc
* const subpage_read
[] = {
3190 static CPUWriteMemoryFunc
* const subpage_write
[] = {
3196 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
3197 ram_addr_t memory
, ram_addr_t region_offset
)
3202 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
3204 idx
= SUBPAGE_IDX(start
);
3205 eidx
= SUBPAGE_IDX(end
);
3206 #if defined(DEBUG_SUBPAGE)
3207 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__
,
3208 mmio
, start
, end
, idx
, eidx
, memory
);
3210 memory
>>= IO_MEM_SHIFT
;
3211 for (; idx
<= eidx
; idx
++) {
3212 for (i
= 0; i
< 4; i
++) {
3213 if (io_mem_read
[memory
][i
]) {
3214 mmio
->mem_read
[idx
][i
] = &io_mem_read
[memory
][i
];
3215 mmio
->opaque
[idx
][0][i
] = io_mem_opaque
[memory
];
3216 mmio
->region_offset
[idx
][0][i
] = region_offset
;
3218 if (io_mem_write
[memory
][i
]) {
3219 mmio
->mem_write
[idx
][i
] = &io_mem_write
[memory
][i
];
3220 mmio
->opaque
[idx
][1][i
] = io_mem_opaque
[memory
];
3221 mmio
->region_offset
[idx
][1][i
] = region_offset
;
3229 static void *subpage_init (target_phys_addr_t base
, ram_addr_t
*phys
,
3230 ram_addr_t orig_memory
, ram_addr_t region_offset
)
3235 mmio
= qemu_mallocz(sizeof(subpage_t
));
3238 subpage_memory
= cpu_register_io_memory(subpage_read
, subpage_write
, mmio
);
3239 #if defined(DEBUG_SUBPAGE)
3240 printf("%s: %p base " TARGET_FMT_plx
" len %08x %d\n", __func__
,
3241 mmio
, base
, TARGET_PAGE_SIZE
, subpage_memory
);
3243 *phys
= subpage_memory
| IO_MEM_SUBPAGE
;
3244 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
- 1, orig_memory
,
3250 static int get_free_io_mem_idx(void)
3254 for (i
= 0; i
<IO_MEM_NB_ENTRIES
; i
++)
3255 if (!io_mem_used
[i
]) {
3259 fprintf(stderr
, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES
);
3263 /* mem_read and mem_write are arrays of functions containing the
3264 function to access byte (index 0), word (index 1) and dword (index
3265 2). Functions can be omitted with a NULL function pointer.
3266 If io_index is non zero, the corresponding io zone is
3267 modified. If it is zero, a new io zone is allocated. The return
3268 value can be used with cpu_register_physical_memory(). (-1) is
3269 returned if error. */
3270 static int cpu_register_io_memory_fixed(int io_index
,
3271 CPUReadMemoryFunc
* const *mem_read
,
3272 CPUWriteMemoryFunc
* const *mem_write
,
3275 int i
, subwidth
= 0;
3277 if (io_index
<= 0) {
3278 io_index
= get_free_io_mem_idx();
3282 io_index
>>= IO_MEM_SHIFT
;
3283 if (io_index
>= IO_MEM_NB_ENTRIES
)
3287 for(i
= 0;i
< 3; i
++) {
3288 if (!mem_read
[i
] || !mem_write
[i
])
3289 subwidth
= IO_MEM_SUBWIDTH
;
3290 io_mem_read
[io_index
][i
] = mem_read
[i
];
3291 io_mem_write
[io_index
][i
] = mem_write
[i
];
3293 io_mem_opaque
[io_index
] = opaque
;
3294 return (io_index
<< IO_MEM_SHIFT
) | subwidth
;
3297 int cpu_register_io_memory(CPUReadMemoryFunc
* const *mem_read
,
3298 CPUWriteMemoryFunc
* const *mem_write
,
3301 return cpu_register_io_memory_fixed(0, mem_read
, mem_write
, opaque
);
3304 void cpu_unregister_io_memory(int io_table_address
)
3307 int io_index
= io_table_address
>> IO_MEM_SHIFT
;
3309 for (i
=0;i
< 3; i
++) {
3310 io_mem_read
[io_index
][i
] = unassigned_mem_read
[i
];
3311 io_mem_write
[io_index
][i
] = unassigned_mem_write
[i
];
3313 io_mem_opaque
[io_index
] = NULL
;
3314 io_mem_used
[io_index
] = 0;
3317 static void io_mem_init(void)
3321 cpu_register_io_memory_fixed(IO_MEM_ROM
, error_mem_read
, unassigned_mem_write
, NULL
);
3322 cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED
, unassigned_mem_read
, unassigned_mem_write
, NULL
);
3323 cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY
, error_mem_read
, notdirty_mem_write
, NULL
);
3327 io_mem_watch
= cpu_register_io_memory(watch_mem_read
,
3328 watch_mem_write
, NULL
);
3331 #endif /* !defined(CONFIG_USER_ONLY) */
3333 /* physical memory access (slow version, mainly for debug) */
3334 #if defined(CONFIG_USER_ONLY)
3335 int cpu_memory_rw_debug(CPUState
*env
, target_ulong addr
,
3336 uint8_t *buf
, int len
, int is_write
)
3343 page
= addr
& TARGET_PAGE_MASK
;
3344 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3347 flags
= page_get_flags(page
);
3348 if (!(flags
& PAGE_VALID
))
3351 if (!(flags
& PAGE_WRITE
))
3353 /* XXX: this code should not depend on lock_user */
3354 if (!(p
= lock_user(VERIFY_WRITE
, addr
, l
, 0)))
3357 unlock_user(p
, addr
, l
);
3359 if (!(flags
& PAGE_READ
))
3361 /* XXX: this code should not depend on lock_user */
3362 if (!(p
= lock_user(VERIFY_READ
, addr
, l
, 1)))
3365 unlock_user(p
, addr
, 0);
3375 void cpu_physical_memory_rw(target_phys_addr_t addr
, uint8_t *buf
,
3376 int len
, int is_write
)
3381 target_phys_addr_t page
;
3386 page
= addr
& TARGET_PAGE_MASK
;
3387 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3390 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3392 pd
= IO_MEM_UNASSIGNED
;
3394 pd
= p
->phys_offset
;
3398 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3399 target_phys_addr_t addr1
= addr
;
3400 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3402 addr1
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3403 /* XXX: could force cpu_single_env to NULL to avoid
3405 if (l
>= 4 && ((addr1
& 3) == 0)) {
3406 /* 32 bit write access */
3408 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr1
, val
);
3410 } else if (l
>= 2 && ((addr1
& 1) == 0)) {
3411 /* 16 bit write access */
3413 io_mem_write
[io_index
][1](io_mem_opaque
[io_index
], addr1
, val
);
3416 /* 8 bit write access */
3418 io_mem_write
[io_index
][0](io_mem_opaque
[io_index
], addr1
, val
);
3422 unsigned long addr1
;
3423 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3425 ptr
= qemu_get_ram_ptr(addr1
);
3426 memcpy(ptr
, buf
, l
);
3427 if (!cpu_physical_memory_is_dirty(addr1
)) {
3428 /* invalidate code */
3429 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
3431 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
3432 (0xff & ~CODE_DIRTY_FLAG
);
3434 /* qemu doesn't execute guest code directly, but kvm does
3435 therefore flush instruction caches */
3437 flush_icache_range((unsigned long)ptr
,
3438 ((unsigned long)ptr
)+l
);
3441 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3442 !(pd
& IO_MEM_ROMD
)) {
3443 target_phys_addr_t addr1
= addr
;
3445 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3447 addr1
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3448 if (l
>= 4 && ((addr1
& 3) == 0)) {
3449 /* 32 bit read access */
3450 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr1
);
3453 } else if (l
>= 2 && ((addr1
& 1) == 0)) {
3454 /* 16 bit read access */
3455 val
= io_mem_read
[io_index
][1](io_mem_opaque
[io_index
], addr1
);
3459 /* 8 bit read access */
3460 val
= io_mem_read
[io_index
][0](io_mem_opaque
[io_index
], addr1
);
3466 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3467 (addr
& ~TARGET_PAGE_MASK
);
3468 memcpy(buf
, ptr
, l
);
3477 /* used for ROM loading : can write in RAM and ROM */
3478 void cpu_physical_memory_write_rom(target_phys_addr_t addr
,
3479 const uint8_t *buf
, int len
)
3483 target_phys_addr_t page
;
3488 page
= addr
& TARGET_PAGE_MASK
;
3489 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3492 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3494 pd
= IO_MEM_UNASSIGNED
;
3496 pd
= p
->phys_offset
;
3499 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
&&
3500 (pd
& ~TARGET_PAGE_MASK
) != IO_MEM_ROM
&&
3501 !(pd
& IO_MEM_ROMD
)) {
3504 unsigned long addr1
;
3505 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3507 ptr
= qemu_get_ram_ptr(addr1
);
3508 memcpy(ptr
, buf
, l
);
3518 target_phys_addr_t addr
;
3519 target_phys_addr_t len
;
3522 static BounceBuffer bounce
;
3524 typedef struct MapClient
{
3526 void (*callback
)(void *opaque
);
3527 QLIST_ENTRY(MapClient
) link
;
3530 static QLIST_HEAD(map_client_list
, MapClient
) map_client_list
3531 = QLIST_HEAD_INITIALIZER(map_client_list
);
3533 void *cpu_register_map_client(void *opaque
, void (*callback
)(void *opaque
))
3535 MapClient
*client
= qemu_malloc(sizeof(*client
));
3537 client
->opaque
= opaque
;
3538 client
->callback
= callback
;
3539 QLIST_INSERT_HEAD(&map_client_list
, client
, link
);
3543 void cpu_unregister_map_client(void *_client
)
3545 MapClient
*client
= (MapClient
*)_client
;
3547 QLIST_REMOVE(client
, link
);
3551 static void cpu_notify_map_clients(void)
3555 while (!QLIST_EMPTY(&map_client_list
)) {
3556 client
= QLIST_FIRST(&map_client_list
);
3557 client
->callback(client
->opaque
);
3558 cpu_unregister_map_client(client
);
3562 /* Map a physical memory region into a host virtual address.
3563 * May map a subset of the requested range, given by and returned in *plen.
3564 * May return NULL if resources needed to perform the mapping are exhausted.
3565 * Use only for reads OR writes - not for read-modify-write operations.
3566 * Use cpu_register_map_client() to know when retrying the map operation is
3567 * likely to succeed.
3569 void *cpu_physical_memory_map(target_phys_addr_t addr
,
3570 target_phys_addr_t
*plen
,
3573 target_phys_addr_t len
= *plen
;
3574 target_phys_addr_t done
= 0;
3576 uint8_t *ret
= NULL
;
3578 target_phys_addr_t page
;
3581 unsigned long addr1
;
3584 page
= addr
& TARGET_PAGE_MASK
;
3585 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3588 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3590 pd
= IO_MEM_UNASSIGNED
;
3592 pd
= p
->phys_offset
;
3595 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3596 if (done
|| bounce
.buffer
) {
3599 bounce
.buffer
= qemu_memalign(TARGET_PAGE_SIZE
, TARGET_PAGE_SIZE
);
3603 cpu_physical_memory_rw(addr
, bounce
.buffer
, l
, 0);
3605 ptr
= bounce
.buffer
;
3607 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3608 ptr
= qemu_get_ram_ptr(addr1
);
3612 } else if (ret
+ done
!= ptr
) {
3624 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3625 * Will also mark the memory as dirty if is_write == 1. access_len gives
3626 * the amount of memory that was actually read or written by the caller.
3628 void cpu_physical_memory_unmap(void *buffer
, target_phys_addr_t len
,
3629 int is_write
, target_phys_addr_t access_len
)
3631 unsigned long flush_len
= (unsigned long)access_len
;
3633 if (buffer
!= bounce
.buffer
) {
3635 ram_addr_t addr1
= qemu_ram_addr_from_host(buffer
);
3636 while (access_len
) {
3638 l
= TARGET_PAGE_SIZE
;
3641 if (!cpu_physical_memory_is_dirty(addr1
)) {
3642 /* invalidate code */
3643 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
3645 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
3646 (0xff & ~CODE_DIRTY_FLAG
);
3651 dma_flush_range((unsigned long)buffer
,
3652 (unsigned long)buffer
+ flush_len
);
3657 cpu_physical_memory_write(bounce
.addr
, bounce
.buffer
, access_len
);
3659 qemu_vfree(bounce
.buffer
);
3660 bounce
.buffer
= NULL
;
3661 cpu_notify_map_clients();
3664 /* warning: addr must be aligned */
3665 uint32_t ldl_phys(target_phys_addr_t addr
)
3673 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3675 pd
= IO_MEM_UNASSIGNED
;
3677 pd
= p
->phys_offset
;
3680 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3681 !(pd
& IO_MEM_ROMD
)) {
3683 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3685 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3686 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
3689 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3690 (addr
& ~TARGET_PAGE_MASK
);
3696 /* warning: addr must be aligned */
3697 uint64_t ldq_phys(target_phys_addr_t addr
)
3705 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3707 pd
= IO_MEM_UNASSIGNED
;
3709 pd
= p
->phys_offset
;
3712 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3713 !(pd
& IO_MEM_ROMD
)) {
3715 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3717 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3718 #ifdef TARGET_WORDS_BIGENDIAN
3719 val
= (uint64_t)io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
) << 32;
3720 val
|= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4);
3722 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
3723 val
|= (uint64_t)io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4) << 32;
3727 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3728 (addr
& ~TARGET_PAGE_MASK
);
3735 uint32_t ldub_phys(target_phys_addr_t addr
)
3738 cpu_physical_memory_read(addr
, &val
, 1);
3743 uint32_t lduw_phys(target_phys_addr_t addr
)
3746 cpu_physical_memory_read(addr
, (uint8_t *)&val
, 2);
3747 return tswap16(val
);
3750 /* warning: addr must be aligned. The ram page is not masked as dirty
3751 and the code inside is not invalidated. It is useful if the dirty
3752 bits are used to track modified PTEs */
3753 void stl_phys_notdirty(target_phys_addr_t addr
, uint32_t val
)
3760 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3762 pd
= IO_MEM_UNASSIGNED
;
3764 pd
= p
->phys_offset
;
3767 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3768 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3770 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3771 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3773 unsigned long addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3774 ptr
= qemu_get_ram_ptr(addr1
);
3777 if (unlikely(in_migration
)) {
3778 if (!cpu_physical_memory_is_dirty(addr1
)) {
3779 /* invalidate code */
3780 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
3782 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
3783 (0xff & ~CODE_DIRTY_FLAG
);
3789 void stq_phys_notdirty(target_phys_addr_t addr
, uint64_t val
)
3796 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3798 pd
= IO_MEM_UNASSIGNED
;
3800 pd
= p
->phys_offset
;
3803 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3804 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3806 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3807 #ifdef TARGET_WORDS_BIGENDIAN
3808 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
>> 32);
3809 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4, val
);
3811 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3812 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4, val
>> 32);
3815 ptr
= qemu_get_ram_ptr(pd
& TARGET_PAGE_MASK
) +
3816 (addr
& ~TARGET_PAGE_MASK
);
3821 /* warning: addr must be aligned */
3822 void stl_phys(target_phys_addr_t addr
, uint32_t val
)
3829 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3831 pd
= IO_MEM_UNASSIGNED
;
3833 pd
= p
->phys_offset
;
3836 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3837 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3839 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3840 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3842 unsigned long addr1
;
3843 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3845 ptr
= qemu_get_ram_ptr(addr1
);
3847 if (!cpu_physical_memory_is_dirty(addr1
)) {
3848 /* invalidate code */
3849 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
3851 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
3852 (0xff & ~CODE_DIRTY_FLAG
);
3858 void stb_phys(target_phys_addr_t addr
, uint32_t val
)
3861 cpu_physical_memory_write(addr
, &v
, 1);
3865 void stw_phys(target_phys_addr_t addr
, uint32_t val
)
3867 uint16_t v
= tswap16(val
);
3868 cpu_physical_memory_write(addr
, (const uint8_t *)&v
, 2);
3872 void stq_phys(target_phys_addr_t addr
, uint64_t val
)
3875 cpu_physical_memory_write(addr
, (const uint8_t *)&val
, 8);
3878 /* virtual memory access for debug (includes writing to ROM) */
3879 int cpu_memory_rw_debug(CPUState
*env
, target_ulong addr
,
3880 uint8_t *buf
, int len
, int is_write
)
3883 target_phys_addr_t phys_addr
;
3887 page
= addr
& TARGET_PAGE_MASK
;
3888 phys_addr
= cpu_get_phys_page_debug(env
, page
);
3889 /* if no physical page mapped, return an error */
3890 if (phys_addr
== -1)
3892 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3895 phys_addr
+= (addr
& ~TARGET_PAGE_MASK
);
3897 cpu_physical_memory_write_rom(phys_addr
, buf
, l
);
3899 cpu_physical_memory_rw(phys_addr
, buf
, l
, is_write
);
3908 /* in deterministic execution mode, instructions doing device I/Os
3909 must be at the end of the TB */
3910 void cpu_io_recompile(CPUState
*env
, void *retaddr
)
3912 TranslationBlock
*tb
;
3914 target_ulong pc
, cs_base
;
3917 tb
= tb_find_pc((unsigned long)retaddr
);
3919 cpu_abort(env
, "cpu_io_recompile: could not find TB for pc=%p",
3922 n
= env
->icount_decr
.u16
.low
+ tb
->icount
;
3923 cpu_restore_state(tb
, env
, (unsigned long)retaddr
, NULL
);
3924 /* Calculate how many instructions had been executed before the fault
3926 n
= n
- env
->icount_decr
.u16
.low
;
3927 /* Generate a new TB ending on the I/O insn. */
3929 /* On MIPS and SH, delay slot instructions can only be restarted if
3930 they were already the first instruction in the TB. If this is not
3931 the first instruction in a TB then re-execute the preceding
3933 #if defined(TARGET_MIPS)
3934 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
3935 env
->active_tc
.PC
-= 4;
3936 env
->icount_decr
.u16
.low
++;
3937 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
3939 #elif defined(TARGET_SH4)
3940 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
3943 env
->icount_decr
.u16
.low
++;
3944 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
3947 /* This should never happen. */
3948 if (n
> CF_COUNT_MASK
)
3949 cpu_abort(env
, "TB too big during recompile");
3951 cflags
= n
| CF_LAST_IO
;
3953 cs_base
= tb
->cs_base
;
3955 tb_phys_invalidate(tb
, -1);
3956 /* FIXME: In theory this could raise an exception. In practice
3957 we have already translated the block once so it's probably ok. */
3958 tb_gen_code(env
, pc
, cs_base
, flags
, cflags
);
3959 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
3960 the first in the TB) then we end up generating a whole new TB and
3961 repeating the fault, which is horribly inefficient.
3962 Better would be to execute just this insn uncached, or generate a
3964 cpu_resume_from_signal(env
, NULL
);
3967 #if !defined(CONFIG_USER_ONLY)
3969 void dump_exec_info(FILE *f
,
3970 int (*cpu_fprintf
)(FILE *f
, const char *fmt
, ...))
3972 int i
, target_code_size
, max_target_code_size
;
3973 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
3974 TranslationBlock
*tb
;
3976 target_code_size
= 0;
3977 max_target_code_size
= 0;
3979 direct_jmp_count
= 0;
3980 direct_jmp2_count
= 0;
3981 for(i
= 0; i
< nb_tbs
; i
++) {
3983 target_code_size
+= tb
->size
;
3984 if (tb
->size
> max_target_code_size
)
3985 max_target_code_size
= tb
->size
;
3986 if (tb
->page_addr
[1] != -1)
3988 if (tb
->tb_next_offset
[0] != 0xffff) {
3990 if (tb
->tb_next_offset
[1] != 0xffff) {
3991 direct_jmp2_count
++;
3995 /* XXX: avoid using doubles ? */
3996 cpu_fprintf(f
, "Translation buffer state:\n");
3997 cpu_fprintf(f
, "gen code size %ld/%ld\n",
3998 code_gen_ptr
- code_gen_buffer
, code_gen_buffer_max_size
);
3999 cpu_fprintf(f
, "TB count %d/%d\n",
4000 nb_tbs
, code_gen_max_blocks
);
4001 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
4002 nb_tbs
? target_code_size
/ nb_tbs
: 0,
4003 max_target_code_size
);
4004 cpu_fprintf(f
, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
4005 nb_tbs
? (code_gen_ptr
- code_gen_buffer
) / nb_tbs
: 0,
4006 target_code_size
? (double) (code_gen_ptr
- code_gen_buffer
) / target_code_size
: 0);
4007 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n",
4009 nb_tbs
? (cross_page
* 100) / nb_tbs
: 0);
4010 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
4012 nb_tbs
? (direct_jmp_count
* 100) / nb_tbs
: 0,
4014 nb_tbs
? (direct_jmp2_count
* 100) / nb_tbs
: 0);
4015 cpu_fprintf(f
, "\nStatistics:\n");
4016 cpu_fprintf(f
, "TB flush count %d\n", tb_flush_count
);
4017 cpu_fprintf(f
, "TB invalidate count %d\n", tb_phys_invalidate_count
);
4018 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
4019 #ifdef CONFIG_PROFILER
4020 tcg_dump_info(f
, cpu_fprintf
);
4024 #define MMUSUFFIX _cmmu
4025 #define GETPC() NULL
4026 #define env cpu_single_env
4027 #define SOFTMMU_CODE_ACCESS
4030 #include "softmmu_template.h"
4033 #include "softmmu_template.h"
4036 #include "softmmu_template.h"
4039 #include "softmmu_template.h"