?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
pflash_cfi02: Fix a typo in debug code (TARGET_FMT_pld -> TARGET_FMT_plx)
[qemu.git] / exec.c
blob8529390cb2d230df4ae8a4710ba43971644d0ee0
1 /*
2 * virtual page mapping and translated block handling
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "exec-all.h"
30 #include "tcg.h"
31 #include "hw/hw.h"
32 #include "hw/qdev.h"
33 #include "osdep.h"
34 #include "kvm.h"
35 #include "hw/xen.h"
36 #include "qemu-timer.h"
37 #if defined(CONFIG_USER_ONLY)
38 #include <qemu.h>
39 #include <signal.h>
40 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
41 #include <sys/param.h>
42 #if __FreeBSD_version >= 700104
43 #define HAVE_KINFO_GETVMMAP
44 #define sigqueue sigqueue_freebsd /* avoid redefinition */
45 #include <sys/time.h>
46 #include <sys/proc.h>
47 #include <machine/profile.h>
48 #define _KERNEL
49 #include <sys/user.h>
50 #undef _KERNEL
51 #undef sigqueue
52 #include <libutil.h>
53 #endif
54 #endif
55 #else /* !CONFIG_USER_ONLY */
56 #include "xen-mapcache.h"
57 #endif
58
59 //#define DEBUG_TB_INVALIDATE
60 //#define DEBUG_FLUSH
61 //#define DEBUG_TLB
62 //#define DEBUG_UNASSIGNED
63
64 /* make various TB consistency checks */
65 //#define DEBUG_TB_CHECK
66 //#define DEBUG_TLB_CHECK
67
68 //#define DEBUG_IOPORT
69 //#define DEBUG_SUBPAGE
70
71 #if !defined(CONFIG_USER_ONLY)
72 /* TB consistency checks only implemented for usermode emulation. */
73 #undef DEBUG_TB_CHECK
74 #endif
75
76 #define SMC_BITMAP_USE_THRESHOLD 10
77
78 static TranslationBlock *tbs;
79 static int code_gen_max_blocks;
80 TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
81 static int nb_tbs;
82 /* any access to the tbs or the page table must use this lock */
83 spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
84
85 #if defined(__arm__) || defined(__sparc_v9__)
86 /* The prologue must be reachable with a direct jump. ARM and Sparc64
87 have limited branch ranges (possibly also PPC) so place it in a
88 section close to code segment. */
89 #define code_gen_section \
90 __attribute__((__section__(".gen_code"))) \
91 __attribute__((aligned (32)))
92 #elif defined(_WIN32)
93 /* Maximum alignment for Win32 is 16. */
94 #define code_gen_section \
95 __attribute__((aligned (16)))
96 #else
97 #define code_gen_section \
98 __attribute__((aligned (32)))
99 #endif
100
101 uint8_t code_gen_prologue[1024] code_gen_section;
102 static uint8_t *code_gen_buffer;
103 static unsigned long code_gen_buffer_size;
104 /* threshold to flush the translated code buffer */
105 static unsigned long code_gen_buffer_max_size;
106 static uint8_t *code_gen_ptr;
107
108 #if !defined(CONFIG_USER_ONLY)
109 int phys_ram_fd;
110 static int in_migration;
111
112 RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list) };
113 #endif
114
115 CPUState *first_cpu;
116 /* current CPU in the current thread. It is only valid inside
117 cpu_exec() */
118 CPUState *cpu_single_env;
119 /* 0 = Do not count executed instructions.
120 1 = Precise instruction counting.
121 2 = Adaptive rate instruction counting. */
122 int use_icount = 0;
123 /* Current instruction counter. While executing translated code this may
124 include some instructions that have not yet been executed. */
125 int64_t qemu_icount;
126
127 typedef struct PageDesc {
128 /* list of TBs intersecting this ram page */
129 TranslationBlock *first_tb;
130 /* in order to optimize self modifying code, we count the number
131 of lookups we do to a given page to use a bitmap */
132 unsigned int code_write_count;
133 uint8_t *code_bitmap;
134 #if defined(CONFIG_USER_ONLY)
135 unsigned long flags;
136 #endif
137 } PageDesc;
138
139 /* In system mode we want L1_MAP to be based on ram offsets,
140 while in user mode we want it to be based on virtual addresses. */
141 #if !defined(CONFIG_USER_ONLY)
142 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
143 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
144 #else
145 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
146 #endif
147 #else
148 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
149 #endif
150
151 /* Size of the L2 (and L3, etc) page tables. */
152 #define L2_BITS 10
153 #define L2_SIZE (1 << L2_BITS)
154
155 /* The bits remaining after N lower levels of page tables. */
156 #define P_L1_BITS_REM \
157 ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
158 #define V_L1_BITS_REM \
159 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
160
161 /* Size of the L1 page table. Avoid silly small sizes. */
162 #if P_L1_BITS_REM < 4
163 #define P_L1_BITS (P_L1_BITS_REM + L2_BITS)
164 #else
165 #define P_L1_BITS P_L1_BITS_REM
166 #endif
167
168 #if V_L1_BITS_REM < 4
169 #define V_L1_BITS (V_L1_BITS_REM + L2_BITS)
170 #else
171 #define V_L1_BITS V_L1_BITS_REM
172 #endif
173
174 #define P_L1_SIZE ((target_phys_addr_t)1 << P_L1_BITS)
175 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
176
177 #define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
178 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
179
180 unsigned long qemu_real_host_page_size;
181 unsigned long qemu_host_page_bits;
182 unsigned long qemu_host_page_size;
183 unsigned long qemu_host_page_mask;
184
185 /* This is a multi-level map on the virtual address space.
186 The bottom level has pointers to PageDesc. */
187 static void *l1_map[V_L1_SIZE];
188
189 #if !defined(CONFIG_USER_ONLY)
190 typedef struct PhysPageDesc {
191 /* offset in host memory of the page + io_index in the low bits */
192 ram_addr_t phys_offset;
193 ram_addr_t region_offset;
194 } PhysPageDesc;
195
196 /* This is a multi-level map on the physical address space.
197 The bottom level has pointers to PhysPageDesc. */
198 static void *l1_phys_map[P_L1_SIZE];
199
200 static void io_mem_init(void);
201
202 /* io memory support */
203 CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
204 CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
205 void *io_mem_opaque[IO_MEM_NB_ENTRIES];
206 static char io_mem_used[IO_MEM_NB_ENTRIES];
207 static int io_mem_watch;
208 #endif
209
210 /* log support */
211 #ifdef WIN32
212 static const char *logfilename = "qemu.log";
213 #else
214 static const char *logfilename = "/tmp/qemu.log";
215 #endif
216 FILE *logfile;
217 int loglevel;
218 static int log_append = 0;
219
220 /* statistics */
221 #if !defined(CONFIG_USER_ONLY)
222 static int tlb_flush_count;
223 #endif
224 static int tb_flush_count;
225 static int tb_phys_invalidate_count;
226
227 #ifdef _WIN32
228 static void map_exec(void *addr, long size)
229 {
230 DWORD old_protect;
231 VirtualProtect(addr, size,
232 PAGE_EXECUTE_READWRITE, &old_protect);
233
234 }
235 #else
236 static void map_exec(void *addr, long size)
237 {
238 unsigned long start, end, page_size;
239
240 page_size = getpagesize();
241 start = (unsigned long)addr;
242 start &= ~(page_size - 1);
243
244 end = (unsigned long)addr + size;
245 end += page_size - 1;
246 end &= ~(page_size - 1);
247
248 mprotect((void *)start, end - start,
249 PROT_READ | PROT_WRITE | PROT_EXEC);
250 }
251 #endif
252
253 static void page_init(void)
254 {
255 /* NOTE: we can always suppose that qemu_host_page_size >=
256 TARGET_PAGE_SIZE */
257 #ifdef _WIN32
258 {
259 SYSTEM_INFO system_info;
260
261 GetSystemInfo(&system_info);
262 qemu_real_host_page_size = system_info.dwPageSize;
263 }
264 #else
265 qemu_real_host_page_size = getpagesize();
266 #endif
267 if (qemu_host_page_size == 0)
268 qemu_host_page_size = qemu_real_host_page_size;
269 if (qemu_host_page_size < TARGET_PAGE_SIZE)
270 qemu_host_page_size = TARGET_PAGE_SIZE;
271 qemu_host_page_bits = 0;
272 while ((1 << qemu_host_page_bits) < qemu_host_page_size)
273 qemu_host_page_bits++;
274 qemu_host_page_mask = ~(qemu_host_page_size - 1);
275
276 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
277 {
278 #ifdef HAVE_KINFO_GETVMMAP
279 struct kinfo_vmentry *freep;
280 int i, cnt;
281
282 freep = kinfo_getvmmap(getpid(), &cnt);
283 if (freep) {
284 mmap_lock();
285 for (i = 0; i < cnt; i++) {
286 unsigned long startaddr, endaddr;
287
288 startaddr = freep[i].kve_start;
289 endaddr = freep[i].kve_end;
290 if (h2g_valid(startaddr)) {
291 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
292
293 if (h2g_valid(endaddr)) {
294 endaddr = h2g(endaddr);
295 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
296 } else {
297 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
298 endaddr = ~0ul;
299 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
300 #endif
301 }
302 }
303 }
304 free(freep);
305 mmap_unlock();
306 }
307 #else
308 FILE *f;
309
310 last_brk = (unsigned long)sbrk(0);
311
312 f = fopen("/compat/linux/proc/self/maps", "r");
313 if (f) {
314 mmap_lock();
315
316 do {
317 unsigned long startaddr, endaddr;
318 int n;
319
320 n = fscanf (f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
321
322 if (n == 2 && h2g_valid(startaddr)) {
323 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
324
325 if (h2g_valid(endaddr)) {
326 endaddr = h2g(endaddr);
327 } else {
328 endaddr = ~0ul;
329 }
330 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
331 }
332 } while (!feof(f));
333
334 fclose(f);
335 mmap_unlock();
336 }
337 #endif
338 }
339 #endif
340 }
341
342 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
343 {
344 PageDesc *pd;
345 void **lp;
346 int i;
347
348 #if defined(CONFIG_USER_ONLY)
349 /* We can't use qemu_malloc because it may recurse into a locked mutex. */
350 # define ALLOC(P, SIZE) \
351 do { \
352 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
353 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
354 } while (0)
355 #else
356 # define ALLOC(P, SIZE) \
357 do { P = qemu_mallocz(SIZE); } while (0)
358 #endif
359
360 /* Level 1. Always allocated. */
361 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
362
363 /* Level 2..N-1. */
364 for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
365 void **p = *lp;
366
367 if (p == NULL) {
368 if (!alloc) {
369 return NULL;
370 }
371 ALLOC(p, sizeof(void *) * L2_SIZE);
372 *lp = p;
373 }
374
375 lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
376 }
377
378 pd = *lp;
379 if (pd == NULL) {
380 if (!alloc) {
381 return NULL;
382 }
383 ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
384 *lp = pd;
385 }
386
387 #undef ALLOC
388
389 return pd + (index & (L2_SIZE - 1));
390 }
391
392 static inline PageDesc *page_find(tb_page_addr_t index)
393 {
394 return page_find_alloc(index, 0);
395 }
396
397 #if !defined(CONFIG_USER_ONLY)
398 static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
399 {
400 PhysPageDesc *pd;
401 void **lp;
402 int i;
403
404 /* Level 1. Always allocated. */
405 lp = l1_phys_map + ((index >> P_L1_SHIFT) & (P_L1_SIZE - 1));
406
407 /* Level 2..N-1. */
408 for (i = P_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
409 void **p = *lp;
410 if (p == NULL) {
411 if (!alloc) {
412 return NULL;
413 }
414 *lp = p = qemu_mallocz(sizeof(void *) * L2_SIZE);
415 }
416 lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
417 }
418
419 pd = *lp;
420 if (pd == NULL) {
421 int i;
422
423 if (!alloc) {
424 return NULL;
425 }
426
427 *lp = pd = qemu_malloc(sizeof(PhysPageDesc) * L2_SIZE);
428
429 for (i = 0; i < L2_SIZE; i++) {
430 pd[i].phys_offset = IO_MEM_UNASSIGNED;
431 pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
432 }
433 }
434
435 return pd + (index & (L2_SIZE - 1));
436 }
437
438 static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
439 {
440 return phys_page_find_alloc(index, 0);
441 }
442
443 static void tlb_protect_code(ram_addr_t ram_addr);
444 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
445 target_ulong vaddr);
446 #define mmap_lock() do { } while(0)
447 #define mmap_unlock() do { } while(0)
448 #endif
449
450 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
451
452 #if defined(CONFIG_USER_ONLY)
453 /* Currently it is not recommended to allocate big chunks of data in
454 user mode. It will change when a dedicated libc will be used */
455 #define USE_STATIC_CODE_GEN_BUFFER
456 #endif
457
458 #ifdef USE_STATIC_CODE_GEN_BUFFER
459 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
460 __attribute__((aligned (CODE_GEN_ALIGN)));
461 #endif
462
463 static void code_gen_alloc(unsigned long tb_size)
464 {
465 #ifdef USE_STATIC_CODE_GEN_BUFFER
466 code_gen_buffer = static_code_gen_buffer;
467 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
468 map_exec(code_gen_buffer, code_gen_buffer_size);
469 #else
470 code_gen_buffer_size = tb_size;
471 if (code_gen_buffer_size == 0) {
472 #if defined(CONFIG_USER_ONLY)
473 /* in user mode, phys_ram_size is not meaningful */
474 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
475 #else
476 /* XXX: needs adjustments */
477 code_gen_buffer_size = (unsigned long)(ram_size / 4);
478 #endif
479 }
480 if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
481 code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
482 /* The code gen buffer location may have constraints depending on
483 the host cpu and OS */
484 #if defined(__linux__)
485 {
486 int flags;
487 void *start = NULL;
488
489 flags = MAP_PRIVATE | MAP_ANONYMOUS;
490 #if defined(__x86_64__)
491 flags |= MAP_32BIT;
492 /* Cannot map more than that */
493 if (code_gen_buffer_size > (800 * 1024 * 1024))
494 code_gen_buffer_size = (800 * 1024 * 1024);
495 #elif defined(__sparc_v9__)
496 // Map the buffer below 2G, so we can use direct calls and branches
497 flags |= MAP_FIXED;
498 start = (void *) 0x60000000UL;
499 if (code_gen_buffer_size > (512 * 1024 * 1024))
500 code_gen_buffer_size = (512 * 1024 * 1024);
501 #elif defined(__arm__)
502 /* Map the buffer below 32M, so we can use direct calls and branches */
503 flags |= MAP_FIXED;
504 start = (void *) 0x01000000UL;
505 if (code_gen_buffer_size > 16 * 1024 * 1024)
506 code_gen_buffer_size = 16 * 1024 * 1024;
507 #elif defined(__s390x__)
508 /* Map the buffer so that we can use direct calls and branches. */
509 /* We have a +- 4GB range on the branches; leave some slop. */
510 if (code_gen_buffer_size > (3ul * 1024 * 1024 * 1024)) {
511 code_gen_buffer_size = 3ul * 1024 * 1024 * 1024;
512 }
513 start = (void *)0x90000000UL;
514 #endif
515 code_gen_buffer = mmap(start, code_gen_buffer_size,
516 PROT_WRITE | PROT_READ | PROT_EXEC,
517 flags, -1, 0);
518 if (code_gen_buffer == MAP_FAILED) {
519 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
520 exit(1);
521 }
522 }
523 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
524 || defined(__DragonFly__) || defined(__OpenBSD__)
525 {
526 int flags;
527 void *addr = NULL;
528 flags = MAP_PRIVATE | MAP_ANONYMOUS;
529 #if defined(__x86_64__)
530 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
531 * 0x40000000 is free */
532 flags |= MAP_FIXED;
533 addr = (void *)0x40000000;
534 /* Cannot map more than that */
535 if (code_gen_buffer_size > (800 * 1024 * 1024))
536 code_gen_buffer_size = (800 * 1024 * 1024);
537 #elif defined(__sparc_v9__)
538 // Map the buffer below 2G, so we can use direct calls and branches
539 flags |= MAP_FIXED;
540 addr = (void *) 0x60000000UL;
541 if (code_gen_buffer_size > (512 * 1024 * 1024)) {
542 code_gen_buffer_size = (512 * 1024 * 1024);
543 }
544 #endif
545 code_gen_buffer = mmap(addr, code_gen_buffer_size,
546 PROT_WRITE | PROT_READ | PROT_EXEC,
547 flags, -1, 0);
548 if (code_gen_buffer == MAP_FAILED) {
549 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
550 exit(1);
551 }
552 }
553 #else
554 code_gen_buffer = qemu_malloc(code_gen_buffer_size);
555 map_exec(code_gen_buffer, code_gen_buffer_size);
556 #endif
557 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
558 map_exec(code_gen_prologue, sizeof(code_gen_prologue));
559 code_gen_buffer_max_size = code_gen_buffer_size -
560 (TCG_MAX_OP_SIZE * OPC_MAX_SIZE);
561 code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
562 tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
563 }
564
565 /* Must be called before using the QEMU cpus. 'tb_size' is the size
566 (in bytes) allocated to the translation buffer. Zero means default
567 size. */
568 void cpu_exec_init_all(unsigned long tb_size)
569 {
570 cpu_gen_init();
571 code_gen_alloc(tb_size);
572 code_gen_ptr = code_gen_buffer;
573 page_init();
574 #if !defined(CONFIG_USER_ONLY)
575 io_mem_init();
576 #endif
577 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
578 /* There's no guest base to take into account, so go ahead and
579 initialize the prologue now. */
580 tcg_prologue_init(&tcg_ctx);
581 #endif
582 }
583
584 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
585
586 static int cpu_common_post_load(void *opaque, int version_id)
587 {
588 CPUState *env = opaque;
589
590 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
591 version_id is increased. */
592 env->interrupt_request &= ~0x01;
593 tlb_flush(env, 1);
594
595 return 0;
596 }
597
598 static const VMStateDescription vmstate_cpu_common = {
599 .name = "cpu_common",
600 .version_id = 1,
601 .minimum_version_id = 1,
602 .minimum_version_id_old = 1,
603 .post_load = cpu_common_post_load,
604 .fields = (VMStateField []) {
605 VMSTATE_UINT32(halted, CPUState),
606 VMSTATE_UINT32(interrupt_request, CPUState),
607 VMSTATE_END_OF_LIST()
608 }
609 };
610 #endif
611
612 CPUState *qemu_get_cpu(int cpu)
613 {
614 CPUState *env = first_cpu;
615
616 while (env) {
617 if (env->cpu_index == cpu)
618 break;
619 env = env->next_cpu;
620 }
621
622 return env;
623 }
624
625 void cpu_exec_init(CPUState *env)
626 {
627 CPUState **penv;
628 int cpu_index;
629
630 #if defined(CONFIG_USER_ONLY)
631 cpu_list_lock();
632 #endif
633 env->next_cpu = NULL;
634 penv = &first_cpu;
635 cpu_index = 0;
636 while (*penv != NULL) {
637 penv = &(*penv)->next_cpu;
638 cpu_index++;
639 }
640 env->cpu_index = cpu_index;
641 env->numa_node = 0;
642 QTAILQ_INIT(&env->breakpoints);
643 QTAILQ_INIT(&env->watchpoints);
644 #ifndef CONFIG_USER_ONLY
645 env->thread_id = qemu_get_thread_id();
646 #endif
647 *penv = env;
648 #if defined(CONFIG_USER_ONLY)
649 cpu_list_unlock();
650 #endif
651 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
652 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, env);
653 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
654 cpu_save, cpu_load, env);
655 #endif
656 }
657
658 /* Allocate a new translation block. Flush the translation buffer if
659 too many translation blocks or too much generated code. */
660 static TranslationBlock *tb_alloc(target_ulong pc)
661 {
662 TranslationBlock *tb;
663
664 if (nb_tbs >= code_gen_max_blocks ||
665 (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
666 return NULL;
667 tb = &tbs[nb_tbs++];
668 tb->pc = pc;
669 tb->cflags = 0;
670 return tb;
671 }
672
673 void tb_free(TranslationBlock *tb)
674 {
675 /* In practice this is mostly used for single use temporary TB
676 Ignore the hard cases and just back up if this TB happens to
677 be the last one generated. */
678 if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
679 code_gen_ptr = tb->tc_ptr;
680 nb_tbs--;
681 }
682 }
683
684 static inline void invalidate_page_bitmap(PageDesc *p)
685 {
686 if (p->code_bitmap) {
687 qemu_free(p->code_bitmap);
688 p->code_bitmap = NULL;
689 }
690 p->code_write_count = 0;
691 }
692
693 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
694
695 static void page_flush_tb_1 (int level, void **lp)
696 {
697 int i;
698
699 if (*lp == NULL) {
700 return;
701 }
702 if (level == 0) {
703 PageDesc *pd = *lp;
704 for (i = 0; i < L2_SIZE; ++i) {
705 pd[i].first_tb = NULL;
706 invalidate_page_bitmap(pd + i);
707 }
708 } else {
709 void **pp = *lp;
710 for (i = 0; i < L2_SIZE; ++i) {
711 page_flush_tb_1 (level - 1, pp + i);
712 }
713 }
714 }
715
716 static void page_flush_tb(void)
717 {
718 int i;
719 for (i = 0; i < V_L1_SIZE; i++) {
720 page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
721 }
722 }
723
724 /* flush all the translation blocks */
725 /* XXX: tb_flush is currently not thread safe */
726 void tb_flush(CPUState *env1)
727 {
728 CPUState *env;
729 #if defined(DEBUG_FLUSH)
730 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
731 (unsigned long)(code_gen_ptr - code_gen_buffer),
732 nb_tbs, nb_tbs > 0 ?
733 ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
734 #endif
735 if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
736 cpu_abort(env1, "Internal error: code buffer overflow\n");
737
738 nb_tbs = 0;
739
740 for(env = first_cpu; env != NULL; env = env->next_cpu) {
741 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
742 }
743
744 memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
745 page_flush_tb();
746
747 code_gen_ptr = code_gen_buffer;
748 /* XXX: flush processor icache at this point if cache flush is
749 expensive */
750 tb_flush_count++;
751 }
752
753 #ifdef DEBUG_TB_CHECK
754
755 static void tb_invalidate_check(target_ulong address)
756 {
757 TranslationBlock *tb;
758 int i;
759 address &= TARGET_PAGE_MASK;
760 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
761 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
762 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
763 address >= tb->pc + tb->size)) {
764 printf("ERROR invalidate: address=" TARGET_FMT_lx
765 " PC=%08lx size=%04x\n",
766 address, (long)tb->pc, tb->size);
767 }
768 }
769 }
770 }
771
772 /* verify that all the pages have correct rights for code */
773 static void tb_page_check(void)
774 {
775 TranslationBlock *tb;
776 int i, flags1, flags2;
777
778 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
779 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
780 flags1 = page_get_flags(tb->pc);
781 flags2 = page_get_flags(tb->pc + tb->size - 1);
782 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
783 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
784 (long)tb->pc, tb->size, flags1, flags2);
785 }
786 }
787 }
788 }
789
790 #endif
791
792 /* invalidate one TB */
793 static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
794 int next_offset)
795 {
796 TranslationBlock *tb1;
797 for(;;) {
798 tb1 = *ptb;
799 if (tb1 == tb) {
800 *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
801 break;
802 }
803 ptb = (TranslationBlock **)((char *)tb1 + next_offset);
804 }
805 }
806
807 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
808 {
809 TranslationBlock *tb1;
810 unsigned int n1;
811
812 for(;;) {
813 tb1 = *ptb;
814 n1 = (long)tb1 & 3;
815 tb1 = (TranslationBlock *)((long)tb1 & ~3);
816 if (tb1 == tb) {
817 *ptb = tb1->page_next[n1];
818 break;
819 }
820 ptb = &tb1->page_next[n1];
821 }
822 }
823
824 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
825 {
826 TranslationBlock *tb1, **ptb;
827 unsigned int n1;
828
829 ptb = &tb->jmp_next[n];
830 tb1 = *ptb;
831 if (tb1) {
832 /* find tb(n) in circular list */
833 for(;;) {
834 tb1 = *ptb;
835 n1 = (long)tb1 & 3;
836 tb1 = (TranslationBlock *)((long)tb1 & ~3);
837 if (n1 == n && tb1 == tb)
838 break;
839 if (n1 == 2) {
840 ptb = &tb1->jmp_first;
841 } else {
842 ptb = &tb1->jmp_next[n1];
843 }
844 }
845 /* now we can suppress tb(n) from the list */
846 *ptb = tb->jmp_next[n];
847
848 tb->jmp_next[n] = NULL;
849 }
850 }
851
852 /* reset the jump entry 'n' of a TB so that it is not chained to
853 another TB */
854 static inline void tb_reset_jump(TranslationBlock *tb, int n)
855 {
856 tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
857 }
858
859 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
860 {
861 CPUState *env;
862 PageDesc *p;
863 unsigned int h, n1;
864 tb_page_addr_t phys_pc;
865 TranslationBlock *tb1, *tb2;
866
867 /* remove the TB from the hash list */
868 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
869 h = tb_phys_hash_func(phys_pc);
870 tb_remove(&tb_phys_hash[h], tb,
871 offsetof(TranslationBlock, phys_hash_next));
872
873 /* remove the TB from the page list */
874 if (tb->page_addr[0] != page_addr) {
875 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
876 tb_page_remove(&p->first_tb, tb);
877 invalidate_page_bitmap(p);
878 }
879 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
880 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
881 tb_page_remove(&p->first_tb, tb);
882 invalidate_page_bitmap(p);
883 }
884
885 tb_invalidated_flag = 1;
886
887 /* remove the TB from the hash list */
888 h = tb_jmp_cache_hash_func(tb->pc);
889 for(env = first_cpu; env != NULL; env = env->next_cpu) {
890 if (env->tb_jmp_cache[h] == tb)
891 env->tb_jmp_cache[h] = NULL;
892 }
893
894 /* suppress this TB from the two jump lists */
895 tb_jmp_remove(tb, 0);
896 tb_jmp_remove(tb, 1);
897
898 /* suppress any remaining jumps to this TB */
899 tb1 = tb->jmp_first;
900 for(;;) {
901 n1 = (long)tb1 & 3;
902 if (n1 == 2)
903 break;
904 tb1 = (TranslationBlock *)((long)tb1 & ~3);
905 tb2 = tb1->jmp_next[n1];
906 tb_reset_jump(tb1, n1);
907 tb1->jmp_next[n1] = NULL;
908 tb1 = tb2;
909 }
910 tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
911
912 tb_phys_invalidate_count++;
913 }
914
915 static inline void set_bits(uint8_t *tab, int start, int len)
916 {
917 int end, mask, end1;
918
919 end = start + len;
920 tab += start >> 3;
921 mask = 0xff << (start & 7);
922 if ((start & ~7) == (end & ~7)) {
923 if (start < end) {
924 mask &= ~(0xff << (end & 7));
925 *tab |= mask;
926 }
927 } else {
928 *tab++ |= mask;
929 start = (start + 8) & ~7;
930 end1 = end & ~7;
931 while (start < end1) {
932 *tab++ = 0xff;
933 start += 8;
934 }
935 if (start < end) {
936 mask = ~(0xff << (end & 7));
937 *tab |= mask;
938 }
939 }
940 }
941
942 static void build_page_bitmap(PageDesc *p)
943 {
944 int n, tb_start, tb_end;
945 TranslationBlock *tb;
946
947 p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
948
949 tb = p->first_tb;
950 while (tb != NULL) {
951 n = (long)tb & 3;
952 tb = (TranslationBlock *)((long)tb & ~3);
953 /* NOTE: this is subtle as a TB may span two physical pages */
954 if (n == 0) {
955 /* NOTE: tb_end may be after the end of the page, but
956 it is not a problem */
957 tb_start = tb->pc & ~TARGET_PAGE_MASK;
958 tb_end = tb_start + tb->size;
959 if (tb_end > TARGET_PAGE_SIZE)
960 tb_end = TARGET_PAGE_SIZE;
961 } else {
962 tb_start = 0;
963 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
964 }
965 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
966 tb = tb->page_next[n];
967 }
968 }
969
970 TranslationBlock *tb_gen_code(CPUState *env,
971 target_ulong pc, target_ulong cs_base,
972 int flags, int cflags)
973 {
974 TranslationBlock *tb;
975 uint8_t *tc_ptr;
976 tb_page_addr_t phys_pc, phys_page2;
977 target_ulong virt_page2;
978 int code_gen_size;
979
980 phys_pc = get_page_addr_code(env, pc);
981 tb = tb_alloc(pc);
982 if (!tb) {
983 /* flush must be done */
984 tb_flush(env);
985 /* cannot fail at this point */
986 tb = tb_alloc(pc);
987 /* Don't forget to invalidate previous TB info. */
988 tb_invalidated_flag = 1;
989 }
990 tc_ptr = code_gen_ptr;
991 tb->tc_ptr = tc_ptr;
992 tb->cs_base = cs_base;
993 tb->flags = flags;
994 tb->cflags = cflags;
995 cpu_gen_code(env, tb, &code_gen_size);
996 code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
997
998 /* check next page if needed */
999 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
1000 phys_page2 = -1;
1001 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
1002 phys_page2 = get_page_addr_code(env, virt_page2);
1003 }
1004 tb_link_page(tb, phys_pc, phys_page2);
1005 return tb;
1006 }
1007
1008 /* invalidate all TBs which intersect with the target physical page
1009 starting in range [start;end[. NOTE: start and end must refer to
1010 the same physical page. 'is_cpu_write_access' should be true if called
1011 from a real cpu write access: the virtual CPU will exit the current
1012 TB if code is modified inside this TB. */
1013 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
1014 int is_cpu_write_access)
1015 {
1016 TranslationBlock *tb, *tb_next, *saved_tb;
1017 CPUState *env = cpu_single_env;
1018 tb_page_addr_t tb_start, tb_end;
1019 PageDesc *p;
1020 int n;
1021 #ifdef TARGET_HAS_PRECISE_SMC
1022 int current_tb_not_found = is_cpu_write_access;
1023 TranslationBlock *current_tb = NULL;
1024 int current_tb_modified = 0;
1025 target_ulong current_pc = 0;
1026 target_ulong current_cs_base = 0;
1027 int current_flags = 0;
1028 #endif /* TARGET_HAS_PRECISE_SMC */
1029
1030 p = page_find(start >> TARGET_PAGE_BITS);
1031 if (!p)
1032 return;
1033 if (!p->code_bitmap &&
1034 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
1035 is_cpu_write_access) {
1036 /* build code bitmap */
1037 build_page_bitmap(p);
1038 }
1039
1040 /* we remove all the TBs in the range [start, end[ */
1041 /* XXX: see if in some cases it could be faster to invalidate all the code */
1042 tb = p->first_tb;
1043 while (tb != NULL) {
1044 n = (long)tb & 3;
1045 tb = (TranslationBlock *)((long)tb & ~3);
1046 tb_next = tb->page_next[n];
1047 /* NOTE: this is subtle as a TB may span two physical pages */
1048 if (n == 0) {
1049 /* NOTE: tb_end may be after the end of the page, but
1050 it is not a problem */
1051 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1052 tb_end = tb_start + tb->size;
1053 } else {
1054 tb_start = tb->page_addr[1];
1055 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1056 }
1057 if (!(tb_end <= start || tb_start >= end)) {
1058 #ifdef TARGET_HAS_PRECISE_SMC
1059 if (current_tb_not_found) {
1060 current_tb_not_found = 0;
1061 current_tb = NULL;
1062 if (env->mem_io_pc) {
1063 /* now we have a real cpu fault */
1064 current_tb = tb_find_pc(env->mem_io_pc);
1065 }
1066 }
1067 if (current_tb == tb &&
1068 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1069 /* If we are modifying the current TB, we must stop
1070 its execution. We could be more precise by checking
1071 that the modification is after the current PC, but it
1072 would require a specialized function to partially
1073 restore the CPU state */
1074
1075 current_tb_modified = 1;
1076 cpu_restore_state(current_tb, env, env->mem_io_pc);
1077 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1078 &current_flags);
1079 }
1080 #endif /* TARGET_HAS_PRECISE_SMC */
1081 /* we need to do that to handle the case where a signal
1082 occurs while doing tb_phys_invalidate() */
1083 saved_tb = NULL;
1084 if (env) {
1085 saved_tb = env->current_tb;
1086 env->current_tb = NULL;
1087 }
1088 tb_phys_invalidate(tb, -1);
1089 if (env) {
1090 env->current_tb = saved_tb;
1091 if (env->interrupt_request && env->current_tb)
1092 cpu_interrupt(env, env->interrupt_request);
1093 }
1094 }
1095 tb = tb_next;
1096 }
1097 #if !defined(CONFIG_USER_ONLY)
1098 /* if no code remaining, no need to continue to use slow writes */
1099 if (!p->first_tb) {
1100 invalidate_page_bitmap(p);
1101 if (is_cpu_write_access) {
1102 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1103 }
1104 }
1105 #endif
1106 #ifdef TARGET_HAS_PRECISE_SMC
1107 if (current_tb_modified) {
1108 /* we generate a block containing just the instruction
1109 modifying the memory. It will ensure that it cannot modify
1110 itself */
1111 env->current_tb = NULL;
1112 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1113 cpu_resume_from_signal(env, NULL);
1114 }
1115 #endif
1116 }
1117
1118 /* len must be <= 8 and start must be a multiple of len */
1119 static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1120 {
1121 PageDesc *p;
1122 int offset, b;
1123 #if 0
1124 if (1) {
1125 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1126 cpu_single_env->mem_io_vaddr, len,
1127 cpu_single_env->eip,
1128 cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1129 }
1130 #endif
1131 p = page_find(start >> TARGET_PAGE_BITS);
1132 if (!p)
1133 return;
1134 if (p->code_bitmap) {
1135 offset = start & ~TARGET_PAGE_MASK;
1136 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1137 if (b & ((1 << len) - 1))
1138 goto do_invalidate;
1139 } else {
1140 do_invalidate:
1141 tb_invalidate_phys_page_range(start, start + len, 1);
1142 }
1143 }
1144
1145 #if !defined(CONFIG_SOFTMMU)
1146 static void tb_invalidate_phys_page(tb_page_addr_t addr,
1147 unsigned long pc, void *puc)
1148 {
1149 TranslationBlock *tb;
1150 PageDesc *p;
1151 int n;
1152 #ifdef TARGET_HAS_PRECISE_SMC
1153 TranslationBlock *current_tb = NULL;
1154 CPUState *env = cpu_single_env;
1155 int current_tb_modified = 0;
1156 target_ulong current_pc = 0;
1157 target_ulong current_cs_base = 0;
1158 int current_flags = 0;
1159 #endif
1160
1161 addr &= TARGET_PAGE_MASK;
1162 p = page_find(addr >> TARGET_PAGE_BITS);
1163 if (!p)
1164 return;
1165 tb = p->first_tb;
1166 #ifdef TARGET_HAS_PRECISE_SMC
1167 if (tb && pc != 0) {
1168 current_tb = tb_find_pc(pc);
1169 }
1170 #endif
1171 while (tb != NULL) {
1172 n = (long)tb & 3;
1173 tb = (TranslationBlock *)((long)tb & ~3);
1174 #ifdef TARGET_HAS_PRECISE_SMC
1175 if (current_tb == tb &&
1176 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1177 /* If we are modifying the current TB, we must stop
1178 its execution. We could be more precise by checking
1179 that the modification is after the current PC, but it
1180 would require a specialized function to partially
1181 restore the CPU state */
1182
1183 current_tb_modified = 1;
1184 cpu_restore_state(current_tb, env, pc);
1185 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1186 &current_flags);
1187 }
1188 #endif /* TARGET_HAS_PRECISE_SMC */
1189 tb_phys_invalidate(tb, addr);
1190 tb = tb->page_next[n];
1191 }
1192 p->first_tb = NULL;
1193 #ifdef TARGET_HAS_PRECISE_SMC
1194 if (current_tb_modified) {
1195 /* we generate a block containing just the instruction
1196 modifying the memory. It will ensure that it cannot modify
1197 itself */
1198 env->current_tb = NULL;
1199 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1200 cpu_resume_from_signal(env, puc);
1201 }
1202 #endif
1203 }
1204 #endif
1205
1206 /* add the tb in the target page and protect it if necessary */
1207 static inline void tb_alloc_page(TranslationBlock *tb,
1208 unsigned int n, tb_page_addr_t page_addr)
1209 {
1210 PageDesc *p;
1211 TranslationBlock *last_first_tb;
1212
1213 tb->page_addr[n] = page_addr;
1214 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1215 tb->page_next[n] = p->first_tb;
1216 last_first_tb = p->first_tb;
1217 p->first_tb = (TranslationBlock *)((long)tb | n);
1218 invalidate_page_bitmap(p);
1219
1220 #if defined(TARGET_HAS_SMC) || 1
1221
1222 #if defined(CONFIG_USER_ONLY)
1223 if (p->flags & PAGE_WRITE) {
1224 target_ulong addr;
1225 PageDesc *p2;
1226 int prot;
1227
1228 /* force the host page as non writable (writes will have a
1229 page fault + mprotect overhead) */
1230 page_addr &= qemu_host_page_mask;
1231 prot = 0;
1232 for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1233 addr += TARGET_PAGE_SIZE) {
1234
1235 p2 = page_find (addr >> TARGET_PAGE_BITS);
1236 if (!p2)
1237 continue;
1238 prot |= p2->flags;
1239 p2->flags &= ~PAGE_WRITE;
1240 }
1241 mprotect(g2h(page_addr), qemu_host_page_size,
1242 (prot & PAGE_BITS) & ~PAGE_WRITE);
1243 #ifdef DEBUG_TB_INVALIDATE
1244 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1245 page_addr);
1246 #endif
1247 }
1248 #else
1249 /* if some code is already present, then the pages are already
1250 protected. So we handle the case where only the first TB is
1251 allocated in a physical page */
1252 if (!last_first_tb) {
1253 tlb_protect_code(page_addr);
1254 }
1255 #endif
1256
1257 #endif /* TARGET_HAS_SMC */
1258 }
1259
1260 /* add a new TB and link it to the physical page tables. phys_page2 is
1261 (-1) to indicate that only one page contains the TB. */
1262 void tb_link_page(TranslationBlock *tb,
1263 tb_page_addr_t phys_pc, tb_page_addr_t phys_page2)
1264 {
1265 unsigned int h;
1266 TranslationBlock **ptb;
1267
1268 /* Grab the mmap lock to stop another thread invalidating this TB
1269 before we are done. */
1270 mmap_lock();
1271 /* add in the physical hash table */
1272 h = tb_phys_hash_func(phys_pc);
1273 ptb = &tb_phys_hash[h];
1274 tb->phys_hash_next = *ptb;
1275 *ptb = tb;
1276
1277 /* add in the page list */
1278 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1279 if (phys_page2 != -1)
1280 tb_alloc_page(tb, 1, phys_page2);
1281 else
1282 tb->page_addr[1] = -1;
1283
1284 tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1285 tb->jmp_next[0] = NULL;
1286 tb->jmp_next[1] = NULL;
1287
1288 /* init original jump addresses */
1289 if (tb->tb_next_offset[0] != 0xffff)
1290 tb_reset_jump(tb, 0);
1291 if (tb->tb_next_offset[1] != 0xffff)
1292 tb_reset_jump(tb, 1);
1293
1294 #ifdef DEBUG_TB_CHECK
1295 tb_page_check();
1296 #endif
1297 mmap_unlock();
1298 }
1299
1300 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1301 tb[1].tc_ptr. Return NULL if not found */
1302 TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1303 {
1304 int m_min, m_max, m;
1305 unsigned long v;
1306 TranslationBlock *tb;
1307
1308 if (nb_tbs <= 0)
1309 return NULL;
1310 if (tc_ptr < (unsigned long)code_gen_buffer ||
1311 tc_ptr >= (unsigned long)code_gen_ptr)
1312 return NULL;
1313 /* binary search (cf Knuth) */
1314 m_min = 0;
1315 m_max = nb_tbs - 1;
1316 while (m_min <= m_max) {
1317 m = (m_min + m_max) >> 1;
1318 tb = &tbs[m];
1319 v = (unsigned long)tb->tc_ptr;
1320 if (v == tc_ptr)
1321 return tb;
1322 else if (tc_ptr < v) {
1323 m_max = m - 1;
1324 } else {
1325 m_min = m + 1;
1326 }
1327 }
1328 return &tbs[m_max];
1329 }
1330
1331 static void tb_reset_jump_recursive(TranslationBlock *tb);
1332
1333 static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1334 {
1335 TranslationBlock *tb1, *tb_next, **ptb;
1336 unsigned int n1;
1337
1338 tb1 = tb->jmp_next[n];
1339 if (tb1 != NULL) {
1340 /* find head of list */
1341 for(;;) {
1342 n1 = (long)tb1 & 3;
1343 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1344 if (n1 == 2)
1345 break;
1346 tb1 = tb1->jmp_next[n1];
1347 }
1348 /* we are now sure now that tb jumps to tb1 */
1349 tb_next = tb1;
1350
1351 /* remove tb from the jmp_first list */
1352 ptb = &tb_next->jmp_first;
1353 for(;;) {
1354 tb1 = *ptb;
1355 n1 = (long)tb1 & 3;
1356 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1357 if (n1 == n && tb1 == tb)
1358 break;
1359 ptb = &tb1->jmp_next[n1];
1360 }
1361 *ptb = tb->jmp_next[n];
1362 tb->jmp_next[n] = NULL;
1363
1364 /* suppress the jump to next tb in generated code */
1365 tb_reset_jump(tb, n);
1366
1367 /* suppress jumps in the tb on which we could have jumped */
1368 tb_reset_jump_recursive(tb_next);
1369 }
1370 }
1371
1372 static void tb_reset_jump_recursive(TranslationBlock *tb)
1373 {
1374 tb_reset_jump_recursive2(tb, 0);
1375 tb_reset_jump_recursive2(tb, 1);
1376 }
1377
1378 #if defined(TARGET_HAS_ICE)
1379 #if defined(CONFIG_USER_ONLY)
1380 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1381 {
1382 tb_invalidate_phys_page_range(pc, pc + 1, 0);
1383 }
1384 #else
1385 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1386 {
1387 target_phys_addr_t addr;
1388 target_ulong pd;
1389 ram_addr_t ram_addr;
1390 PhysPageDesc *p;
1391
1392 addr = cpu_get_phys_page_debug(env, pc);
1393 p = phys_page_find(addr >> TARGET_PAGE_BITS);
1394 if (!p) {
1395 pd = IO_MEM_UNASSIGNED;
1396 } else {
1397 pd = p->phys_offset;
1398 }
1399 ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1400 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1401 }
1402 #endif
1403 #endif /* TARGET_HAS_ICE */
1404
1405 #if defined(CONFIG_USER_ONLY)
1406 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1407
1408 {
1409 }
1410
1411 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1412 int flags, CPUWatchpoint **watchpoint)
1413 {
1414 return -ENOSYS;
1415 }
1416 #else
1417 /* Add a watchpoint. */
1418 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1419 int flags, CPUWatchpoint **watchpoint)
1420 {
1421 target_ulong len_mask = ~(len - 1);
1422 CPUWatchpoint *wp;
1423
1424 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1425 if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1426 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1427 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1428 return -EINVAL;
1429 }
1430 wp = qemu_malloc(sizeof(*wp));
1431
1432 wp->vaddr = addr;
1433 wp->len_mask = len_mask;
1434 wp->flags = flags;
1435
1436 /* keep all GDB-injected watchpoints in front */
1437 if (flags & BP_GDB)
1438 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1439 else
1440 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1441
1442 tlb_flush_page(env, addr);
1443
1444 if (watchpoint)
1445 *watchpoint = wp;
1446 return 0;
1447 }
1448
1449 /* Remove a specific watchpoint. */
1450 int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1451 int flags)
1452 {
1453 target_ulong len_mask = ~(len - 1);
1454 CPUWatchpoint *wp;
1455
1456 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1457 if (addr == wp->vaddr && len_mask == wp->len_mask
1458 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1459 cpu_watchpoint_remove_by_ref(env, wp);
1460 return 0;
1461 }
1462 }
1463 return -ENOENT;
1464 }
1465
1466 /* Remove a specific watchpoint by reference. */
1467 void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1468 {
1469 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1470
1471 tlb_flush_page(env, watchpoint->vaddr);
1472
1473 qemu_free(watchpoint);
1474 }
1475
1476 /* Remove all matching watchpoints. */
1477 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1478 {
1479 CPUWatchpoint *wp, *next;
1480
1481 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1482 if (wp->flags & mask)
1483 cpu_watchpoint_remove_by_ref(env, wp);
1484 }
1485 }
1486 #endif
1487
1488 /* Add a breakpoint. */
1489 int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1490 CPUBreakpoint **breakpoint)
1491 {
1492 #if defined(TARGET_HAS_ICE)
1493 CPUBreakpoint *bp;
1494
1495 bp = qemu_malloc(sizeof(*bp));
1496
1497 bp->pc = pc;
1498 bp->flags = flags;
1499
1500 /* keep all GDB-injected breakpoints in front */
1501 if (flags & BP_GDB)
1502 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1503 else
1504 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1505
1506 breakpoint_invalidate(env, pc);
1507
1508 if (breakpoint)
1509 *breakpoint = bp;
1510 return 0;
1511 #else
1512 return -ENOSYS;
1513 #endif
1514 }
1515
1516 /* Remove a specific breakpoint. */
1517 int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1518 {
1519 #if defined(TARGET_HAS_ICE)
1520 CPUBreakpoint *bp;
1521
1522 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1523 if (bp->pc == pc && bp->flags == flags) {
1524 cpu_breakpoint_remove_by_ref(env, bp);
1525 return 0;
1526 }
1527 }
1528 return -ENOENT;
1529 #else
1530 return -ENOSYS;
1531 #endif
1532 }
1533
1534 /* Remove a specific breakpoint by reference. */
1535 void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
1536 {
1537 #if defined(TARGET_HAS_ICE)
1538 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1539
1540 breakpoint_invalidate(env, breakpoint->pc);
1541
1542 qemu_free(breakpoint);
1543 #endif
1544 }
1545
1546 /* Remove all matching breakpoints. */
1547 void cpu_breakpoint_remove_all(CPUState *env, int mask)
1548 {
1549 #if defined(TARGET_HAS_ICE)
1550 CPUBreakpoint *bp, *next;
1551
1552 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1553 if (bp->flags & mask)
1554 cpu_breakpoint_remove_by_ref(env, bp);
1555 }
1556 #endif
1557 }
1558
1559 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1560 CPU loop after each instruction */
1561 void cpu_single_step(CPUState *env, int enabled)
1562 {
1563 #if defined(TARGET_HAS_ICE)
1564 if (env->singlestep_enabled != enabled) {
1565 env->singlestep_enabled = enabled;
1566 if (kvm_enabled())
1567 kvm_update_guest_debug(env, 0);
1568 else {
1569 /* must flush all the translated code to avoid inconsistencies */
1570 /* XXX: only flush what is necessary */
1571 tb_flush(env);
1572 }
1573 }
1574 #endif
1575 }
1576
1577 /* enable or disable low levels log */
1578 void cpu_set_log(int log_flags)
1579 {
1580 loglevel = log_flags;
1581 if (loglevel && !logfile) {
1582 logfile = fopen(logfilename, log_append ? "a" : "w");
1583 if (!logfile) {
1584 perror(logfilename);
1585 _exit(1);
1586 }
1587 #if !defined(CONFIG_SOFTMMU)
1588 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1589 {
1590 static char logfile_buf[4096];
1591 setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1592 }
1593 #elif !defined(_WIN32)
1594 /* Win32 doesn't support line-buffering and requires size >= 2 */
1595 setvbuf(logfile, NULL, _IOLBF, 0);
1596 #endif
1597 log_append = 1;
1598 }
1599 if (!loglevel && logfile) {
1600 fclose(logfile);
1601 logfile = NULL;
1602 }
1603 }
1604
1605 void cpu_set_log_filename(const char *filename)
1606 {
1607 logfilename = strdup(filename);
1608 if (logfile) {
1609 fclose(logfile);
1610 logfile = NULL;
1611 }
1612 cpu_set_log(loglevel);
1613 }
1614
1615 static void cpu_unlink_tb(CPUState *env)
1616 {
1617 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1618 problem and hope the cpu will stop of its own accord. For userspace
1619 emulation this often isn't actually as bad as it sounds. Often
1620 signals are used primarily to interrupt blocking syscalls. */
1621 TranslationBlock *tb;
1622 static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1623
1624 spin_lock(&interrupt_lock);
1625 tb = env->current_tb;
1626 /* if the cpu is currently executing code, we must unlink it and
1627 all the potentially executing TB */
1628 if (tb) {
1629 env->current_tb = NULL;
1630 tb_reset_jump_recursive(tb);
1631 }
1632 spin_unlock(&interrupt_lock);
1633 }
1634
1635 #ifndef CONFIG_USER_ONLY
1636 /* mask must never be zero, except for A20 change call */
1637 static void tcg_handle_interrupt(CPUState *env, int mask)
1638 {
1639 int old_mask;
1640
1641 old_mask = env->interrupt_request;
1642 env->interrupt_request |= mask;
1643
1644 /*
1645 * If called from iothread context, wake the target cpu in
1646 * case its halted.
1647 */
1648 if (!qemu_cpu_is_self(env)) {
1649 qemu_cpu_kick(env);
1650 return;
1651 }
1652
1653 if (use_icount) {
1654 env->icount_decr.u16.high = 0xffff;
1655 if (!can_do_io(env)
1656 && (mask & ~old_mask) != 0) {
1657 cpu_abort(env, "Raised interrupt while not in I/O function");
1658 }
1659 } else {
1660 cpu_unlink_tb(env);
1661 }
1662 }
1663
1664 CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
1665
1666 #else /* CONFIG_USER_ONLY */
1667
1668 void cpu_interrupt(CPUState *env, int mask)
1669 {
1670 env->interrupt_request |= mask;
1671 cpu_unlink_tb(env);
1672 }
1673 #endif /* CONFIG_USER_ONLY */
1674
1675 void cpu_reset_interrupt(CPUState *env, int mask)
1676 {
1677 env->interrupt_request &= ~mask;
1678 }
1679
1680 void cpu_exit(CPUState *env)
1681 {
1682 env->exit_request = 1;
1683 cpu_unlink_tb(env);
1684 }
1685
1686 const CPULogItem cpu_log_items[] = {
1687 { CPU_LOG_TB_OUT_ASM, "out_asm",
1688 "show generated host assembly code for each compiled TB" },
1689 { CPU_LOG_TB_IN_ASM, "in_asm",
1690 "show target assembly code for each compiled TB" },
1691 { CPU_LOG_TB_OP, "op",
1692 "show micro ops for each compiled TB" },
1693 { CPU_LOG_TB_OP_OPT, "op_opt",
1694 "show micro ops "
1695 #ifdef TARGET_I386
1696 "before eflags optimization and "
1697 #endif
1698 "after liveness analysis" },
1699 { CPU_LOG_INT, "int",
1700 "show interrupts/exceptions in short format" },
1701 { CPU_LOG_EXEC, "exec",
1702 "show trace before each executed TB (lots of logs)" },
1703 { CPU_LOG_TB_CPU, "cpu",
1704 "show CPU state before block translation" },
1705 #ifdef TARGET_I386
1706 { CPU_LOG_PCALL, "pcall",
1707 "show protected mode far calls/returns/exceptions" },
1708 { CPU_LOG_RESET, "cpu_reset",
1709 "show CPU state before CPU resets" },
1710 #endif
1711 #ifdef DEBUG_IOPORT
1712 { CPU_LOG_IOPORT, "ioport",
1713 "show all i/o ports accesses" },
1714 #endif
1715 { 0, NULL, NULL },
1716 };
1717
1718 #ifndef CONFIG_USER_ONLY
1719 static QLIST_HEAD(memory_client_list, CPUPhysMemoryClient) memory_client_list
1720 = QLIST_HEAD_INITIALIZER(memory_client_list);
1721
1722 static void cpu_notify_set_memory(target_phys_addr_t start_addr,
1723 ram_addr_t size,
1724 ram_addr_t phys_offset,
1725 bool log_dirty)
1726 {
1727 CPUPhysMemoryClient *client;
1728 QLIST_FOREACH(client, &memory_client_list, list) {
1729 client->set_memory(client, start_addr, size, phys_offset, log_dirty);
1730 }
1731 }
1732
1733 static int cpu_notify_sync_dirty_bitmap(target_phys_addr_t start,
1734 target_phys_addr_t end)
1735 {
1736 CPUPhysMemoryClient *client;
1737 QLIST_FOREACH(client, &memory_client_list, list) {
1738 int r = client->sync_dirty_bitmap(client, start, end);
1739 if (r < 0)
1740 return r;
1741 }
1742 return 0;
1743 }
1744
1745 static int cpu_notify_migration_log(int enable)
1746 {
1747 CPUPhysMemoryClient *client;
1748 QLIST_FOREACH(client, &memory_client_list, list) {
1749 int r = client->migration_log(client, enable);
1750 if (r < 0)
1751 return r;
1752 }
1753 return 0;
1754 }
1755
1756 /* The l1_phys_map provides the upper P_L1_BITs of the guest physical
1757 * address. Each intermediate table provides the next L2_BITs of guest
1758 * physical address space. The number of levels vary based on host and
1759 * guest configuration, making it efficient to build the final guest
1760 * physical address by seeding the L1 offset and shifting and adding in
1761 * each L2 offset as we recurse through them. */
1762 static void phys_page_for_each_1(CPUPhysMemoryClient *client,
1763 int level, void **lp, target_phys_addr_t addr)
1764 {
1765 int i;
1766
1767 if (*lp == NULL) {
1768 return;
1769 }
1770 if (level == 0) {
1771 PhysPageDesc *pd = *lp;
1772 addr <<= L2_BITS + TARGET_PAGE_BITS;
1773 for (i = 0; i < L2_SIZE; ++i) {
1774 if (pd[i].phys_offset != IO_MEM_UNASSIGNED) {
1775 client->set_memory(client, addr | i << TARGET_PAGE_BITS,
1776 TARGET_PAGE_SIZE, pd[i].phys_offset, false);
1777 }
1778 }
1779 } else {
1780 void **pp = *lp;
1781 for (i = 0; i < L2_SIZE; ++i) {
1782 phys_page_for_each_1(client, level - 1, pp + i,
1783 (addr << L2_BITS) | i);
1784 }
1785 }
1786 }
1787
1788 static void phys_page_for_each(CPUPhysMemoryClient *client)
1789 {
1790 int i;
1791 for (i = 0; i < P_L1_SIZE; ++i) {
1792 phys_page_for_each_1(client, P_L1_SHIFT / L2_BITS - 1,
1793 l1_phys_map + i, i);
1794 }
1795 }
1796
1797 void cpu_register_phys_memory_client(CPUPhysMemoryClient *client)
1798 {
1799 QLIST_INSERT_HEAD(&memory_client_list, client, list);
1800 phys_page_for_each(client);
1801 }
1802
1803 void cpu_unregister_phys_memory_client(CPUPhysMemoryClient *client)
1804 {
1805 QLIST_REMOVE(client, list);
1806 }
1807 #endif
1808
1809 static int cmp1(const char *s1, int n, const char *s2)
1810 {
1811 if (strlen(s2) != n)
1812 return 0;
1813 return memcmp(s1, s2, n) == 0;
1814 }
1815
1816 /* takes a comma separated list of log masks. Return 0 if error. */
1817 int cpu_str_to_log_mask(const char *str)
1818 {
1819 const CPULogItem *item;
1820 int mask;
1821 const char *p, *p1;
1822
1823 p = str;
1824 mask = 0;
1825 for(;;) {
1826 p1 = strchr(p, ',');
1827 if (!p1)
1828 p1 = p + strlen(p);
1829 if(cmp1(p,p1-p,"all")) {
1830 for(item = cpu_log_items; item->mask != 0; item++) {
1831 mask |= item->mask;
1832 }
1833 } else {
1834 for(item = cpu_log_items; item->mask != 0; item++) {
1835 if (cmp1(p, p1 - p, item->name))
1836 goto found;
1837 }
1838 return 0;
1839 }
1840 found:
1841 mask |= item->mask;
1842 if (*p1 != ',')
1843 break;
1844 p = p1 + 1;
1845 }
1846 return mask;
1847 }
1848
1849 void cpu_abort(CPUState *env, const char *fmt, ...)
1850 {
1851 va_list ap;
1852 va_list ap2;
1853
1854 va_start(ap, fmt);
1855 va_copy(ap2, ap);
1856 fprintf(stderr, "qemu: fatal: ");
1857 vfprintf(stderr, fmt, ap);
1858 fprintf(stderr, "\n");
1859 #ifdef TARGET_I386
1860 cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1861 #else
1862 cpu_dump_state(env, stderr, fprintf, 0);
1863 #endif
1864 if (qemu_log_enabled()) {
1865 qemu_log("qemu: fatal: ");
1866 qemu_log_vprintf(fmt, ap2);
1867 qemu_log("\n");
1868 #ifdef TARGET_I386
1869 log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1870 #else
1871 log_cpu_state(env, 0);
1872 #endif
1873 qemu_log_flush();
1874 qemu_log_close();
1875 }
1876 va_end(ap2);
1877 va_end(ap);
1878 #if defined(CONFIG_USER_ONLY)
1879 {
1880 struct sigaction act;
1881 sigfillset(&act.sa_mask);
1882 act.sa_handler = SIG_DFL;
1883 sigaction(SIGABRT, &act, NULL);
1884 }
1885 #endif
1886 abort();
1887 }
1888
1889 CPUState *cpu_copy(CPUState *env)
1890 {
1891 CPUState *new_env = cpu_init(env->cpu_model_str);
1892 CPUState *next_cpu = new_env->next_cpu;
1893 int cpu_index = new_env->cpu_index;
1894 #if defined(TARGET_HAS_ICE)
1895 CPUBreakpoint *bp;
1896 CPUWatchpoint *wp;
1897 #endif
1898
1899 memcpy(new_env, env, sizeof(CPUState));
1900
1901 /* Preserve chaining and index. */
1902 new_env->next_cpu = next_cpu;
1903 new_env->cpu_index = cpu_index;
1904
1905 /* Clone all break/watchpoints.
1906 Note: Once we support ptrace with hw-debug register access, make sure
1907 BP_CPU break/watchpoints are handled correctly on clone. */
1908 QTAILQ_INIT(&env->breakpoints);
1909 QTAILQ_INIT(&env->watchpoints);
1910 #if defined(TARGET_HAS_ICE)
1911 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1912 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1913 }
1914 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1915 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1916 wp->flags, NULL);
1917 }
1918 #endif
1919
1920 return new_env;
1921 }
1922
1923 #if !defined(CONFIG_USER_ONLY)
1924
1925 static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1926 {
1927 unsigned int i;
1928
1929 /* Discard jump cache entries for any tb which might potentially
1930 overlap the flushed page. */
1931 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1932 memset (&env->tb_jmp_cache[i], 0,
1933 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1934
1935 i = tb_jmp_cache_hash_page(addr);
1936 memset (&env->tb_jmp_cache[i], 0,
1937 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1938 }
1939
1940 static CPUTLBEntry s_cputlb_empty_entry = {
1941 .addr_read = -1,
1942 .addr_write = -1,
1943 .addr_code = -1,
1944 .addend = -1,
1945 };
1946
1947 /* NOTE: if flush_global is true, also flush global entries (not
1948 implemented yet) */
1949 void tlb_flush(CPUState *env, int flush_global)
1950 {
1951 int i;
1952
1953 #if defined(DEBUG_TLB)
1954 printf("tlb_flush:\n");
1955 #endif
1956 /* must reset current TB so that interrupts cannot modify the
1957 links while we are modifying them */
1958 env->current_tb = NULL;
1959
1960 for(i = 0; i < CPU_TLB_SIZE; i++) {
1961 int mmu_idx;
1962 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
1963 env->tlb_table[mmu_idx][i] = s_cputlb_empty_entry;
1964 }
1965 }
1966
1967 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1968
1969 env->tlb_flush_addr = -1;
1970 env->tlb_flush_mask = 0;
1971 tlb_flush_count++;
1972 }
1973
1974 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
1975 {
1976 if (addr == (tlb_entry->addr_read &
1977 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1978 addr == (tlb_entry->addr_write &
1979 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1980 addr == (tlb_entry->addr_code &
1981 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1982 *tlb_entry = s_cputlb_empty_entry;
1983 }
1984 }
1985
1986 void tlb_flush_page(CPUState *env, target_ulong addr)
1987 {
1988 int i;
1989 int mmu_idx;
1990
1991 #if defined(DEBUG_TLB)
1992 printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1993 #endif
1994 /* Check if we need to flush due to large pages. */
1995 if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
1996 #if defined(DEBUG_TLB)
1997 printf("tlb_flush_page: forced full flush ("
1998 TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
1999 env->tlb_flush_addr, env->tlb_flush_mask);
2000 #endif
2001 tlb_flush(env, 1);
2002 return;
2003 }
2004 /* must reset current TB so that interrupts cannot modify the
2005 links while we are modifying them */
2006 env->current_tb = NULL;
2007
2008 addr &= TARGET_PAGE_MASK;
2009 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2010 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2011 tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
2012
2013 tlb_flush_jmp_cache(env, addr);
2014 }
2015
2016 /* update the TLBs so that writes to code in the virtual page 'addr'
2017 can be detected */
2018 static void tlb_protect_code(ram_addr_t ram_addr)
2019 {
2020 cpu_physical_memory_reset_dirty(ram_addr,
2021 ram_addr + TARGET_PAGE_SIZE,
2022 CODE_DIRTY_FLAG);
2023 }
2024
2025 /* update the TLB so that writes in physical page 'phys_addr' are no longer
2026 tested for self modifying code */
2027 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
2028 target_ulong vaddr)
2029 {
2030 cpu_physical_memory_set_dirty_flags(ram_addr, CODE_DIRTY_FLAG);
2031 }
2032
2033 static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
2034 unsigned long start, unsigned long length)
2035 {
2036 unsigned long addr;
2037 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2038 addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
2039 if ((addr - start) < length) {
2040 tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
2041 }
2042 }
2043 }
2044
2045 /* Note: start and end must be within the same ram block. */
2046 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
2047 int dirty_flags)
2048 {
2049 CPUState *env;
2050 unsigned long length, start1;
2051 int i;
2052
2053 start &= TARGET_PAGE_MASK;
2054 end = TARGET_PAGE_ALIGN(end);
2055
2056 length = end - start;
2057 if (length == 0)
2058 return;
2059 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
2060
2061 /* we modify the TLB cache so that the dirty bit will be set again
2062 when accessing the range */
2063 start1 = (unsigned long)qemu_safe_ram_ptr(start);
2064 /* Check that we don't span multiple blocks - this breaks the
2065 address comparisons below. */
2066 if ((unsigned long)qemu_safe_ram_ptr(end - 1) - start1
2067 != (end - 1) - start) {
2068 abort();
2069 }
2070
2071 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2072 int mmu_idx;
2073 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2074 for(i = 0; i < CPU_TLB_SIZE; i++)
2075 tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
2076 start1, length);
2077 }
2078 }
2079 }
2080
2081 int cpu_physical_memory_set_dirty_tracking(int enable)
2082 {
2083 int ret = 0;
2084 in_migration = enable;
2085 ret = cpu_notify_migration_log(!!enable);
2086 return ret;
2087 }
2088
2089 int cpu_physical_memory_get_dirty_tracking(void)
2090 {
2091 return in_migration;
2092 }
2093
2094 int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
2095 target_phys_addr_t end_addr)
2096 {
2097 int ret;
2098
2099 ret = cpu_notify_sync_dirty_bitmap(start_addr, end_addr);
2100 return ret;
2101 }
2102
2103 int cpu_physical_log_start(target_phys_addr_t start_addr,
2104 ram_addr_t size)
2105 {
2106 CPUPhysMemoryClient *client;
2107 QLIST_FOREACH(client, &memory_client_list, list) {
2108 if (client->log_start) {
2109 int r = client->log_start(client, start_addr, size);
2110 if (r < 0) {
2111 return r;
2112 }
2113 }
2114 }
2115 return 0;
2116 }
2117
2118 int cpu_physical_log_stop(target_phys_addr_t start_addr,
2119 ram_addr_t size)
2120 {
2121 CPUPhysMemoryClient *client;
2122 QLIST_FOREACH(client, &memory_client_list, list) {
2123 if (client->log_stop) {
2124 int r = client->log_stop(client, start_addr, size);
2125 if (r < 0) {
2126 return r;
2127 }
2128 }
2129 }
2130 return 0;
2131 }
2132
2133 static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
2134 {
2135 ram_addr_t ram_addr;
2136 void *p;
2137
2138 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2139 p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
2140 + tlb_entry->addend);
2141 ram_addr = qemu_ram_addr_from_host_nofail(p);
2142 if (!cpu_physical_memory_is_dirty(ram_addr)) {
2143 tlb_entry->addr_write |= TLB_NOTDIRTY;
2144 }
2145 }
2146 }
2147
2148 /* update the TLB according to the current state of the dirty bits */
2149 void cpu_tlb_update_dirty(CPUState *env)
2150 {
2151 int i;
2152 int mmu_idx;
2153 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2154 for(i = 0; i < CPU_TLB_SIZE; i++)
2155 tlb_update_dirty(&env->tlb_table[mmu_idx][i]);
2156 }
2157 }
2158
2159 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
2160 {
2161 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
2162 tlb_entry->addr_write = vaddr;
2163 }
2164
2165 /* update the TLB corresponding to virtual page vaddr
2166 so that it is no longer dirty */
2167 static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
2168 {
2169 int i;
2170 int mmu_idx;
2171
2172 vaddr &= TARGET_PAGE_MASK;
2173 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2174 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2175 tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
2176 }
2177
2178 /* Our TLB does not support large pages, so remember the area covered by
2179 large pages and trigger a full TLB flush if these are invalidated. */
2180 static void tlb_add_large_page(CPUState *env, target_ulong vaddr,
2181 target_ulong size)
2182 {
2183 target_ulong mask = ~(size - 1);
2184
2185 if (env->tlb_flush_addr == (target_ulong)-1) {
2186 env->tlb_flush_addr = vaddr & mask;
2187 env->tlb_flush_mask = mask;
2188 return;
2189 }
2190 /* Extend the existing region to include the new page.
2191 This is a compromise between unnecessary flushes and the cost
2192 of maintaining a full variable size TLB. */
2193 mask &= env->tlb_flush_mask;
2194 while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
2195 mask <<= 1;
2196 }
2197 env->tlb_flush_addr &= mask;
2198 env->tlb_flush_mask = mask;
2199 }
2200
2201 /* Add a new TLB entry. At most one entry for a given virtual address
2202 is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
2203 supplied size is only used by tlb_flush_page. */
2204 void tlb_set_page(CPUState *env, target_ulong vaddr,
2205 target_phys_addr_t paddr, int prot,
2206 int mmu_idx, target_ulong size)
2207 {
2208 PhysPageDesc *p;
2209 unsigned long pd;
2210 unsigned int index;
2211 target_ulong address;
2212 target_ulong code_address;
2213 unsigned long addend;
2214 CPUTLBEntry *te;
2215 CPUWatchpoint *wp;
2216 target_phys_addr_t iotlb;
2217
2218 assert(size >= TARGET_PAGE_SIZE);
2219 if (size != TARGET_PAGE_SIZE) {
2220 tlb_add_large_page(env, vaddr, size);
2221 }
2222 p = phys_page_find(paddr >> TARGET_PAGE_BITS);
2223 if (!p) {
2224 pd = IO_MEM_UNASSIGNED;
2225 } else {
2226 pd = p->phys_offset;
2227 }
2228 #if defined(DEBUG_TLB)
2229 printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
2230 " prot=%x idx=%d pd=0x%08lx\n",
2231 vaddr, paddr, prot, mmu_idx, pd);
2232 #endif
2233
2234 address = vaddr;
2235 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
2236 /* IO memory case (romd handled later) */
2237 address |= TLB_MMIO;
2238 }
2239 addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
2240 if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
2241 /* Normal RAM. */
2242 iotlb = pd & TARGET_PAGE_MASK;
2243 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
2244 iotlb |= IO_MEM_NOTDIRTY;
2245 else
2246 iotlb |= IO_MEM_ROM;
2247 } else {
2248 /* IO handlers are currently passed a physical address.
2249 It would be nice to pass an offset from the base address
2250 of that region. This would avoid having to special case RAM,
2251 and avoid full address decoding in every device.
2252 We can't use the high bits of pd for this because
2253 IO_MEM_ROMD uses these as a ram address. */
2254 iotlb = (pd & ~TARGET_PAGE_MASK);
2255 if (p) {
2256 iotlb += p->region_offset;
2257 } else {
2258 iotlb += paddr;
2259 }
2260 }
2261
2262 code_address = address;
2263 /* Make accesses to pages with watchpoints go via the
2264 watchpoint trap routines. */
2265 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
2266 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2267 /* Avoid trapping reads of pages with a write breakpoint. */
2268 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
2269 iotlb = io_mem_watch + paddr;
2270 address |= TLB_MMIO;
2271 break;
2272 }
2273 }
2274 }
2275
2276 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2277 env->iotlb[mmu_idx][index] = iotlb - vaddr;
2278 te = &env->tlb_table[mmu_idx][index];
2279 te->addend = addend - vaddr;
2280 if (prot & PAGE_READ) {
2281 te->addr_read = address;
2282 } else {
2283 te->addr_read = -1;
2284 }
2285
2286 if (prot & PAGE_EXEC) {
2287 te->addr_code = code_address;
2288 } else {
2289 te->addr_code = -1;
2290 }
2291 if (prot & PAGE_WRITE) {
2292 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2293 (pd & IO_MEM_ROMD)) {
2294 /* Write access calls the I/O callback. */
2295 te->addr_write = address | TLB_MMIO;
2296 } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2297 !cpu_physical_memory_is_dirty(pd)) {
2298 te->addr_write = address | TLB_NOTDIRTY;
2299 } else {
2300 te->addr_write = address;
2301 }
2302 } else {
2303 te->addr_write = -1;
2304 }
2305 }
2306
2307 #else
2308
2309 void tlb_flush(CPUState *env, int flush_global)
2310 {
2311 }
2312
2313 void tlb_flush_page(CPUState *env, target_ulong addr)
2314 {
2315 }
2316
2317 /*
2318 * Walks guest process memory "regions" one by one
2319 * and calls callback function 'fn' for each region.
2320 */
2321
2322 struct walk_memory_regions_data
2323 {
2324 walk_memory_regions_fn fn;
2325 void *priv;
2326 unsigned long start;
2327 int prot;
2328 };
2329
2330 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
2331 abi_ulong end, int new_prot)
2332 {
2333 if (data->start != -1ul) {
2334 int rc = data->fn(data->priv, data->start, end, data->prot);
2335 if (rc != 0) {
2336 return rc;
2337 }
2338 }
2339
2340 data->start = (new_prot ? end : -1ul);
2341 data->prot = new_prot;
2342
2343 return 0;
2344 }
2345
2346 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
2347 abi_ulong base, int level, void **lp)
2348 {
2349 abi_ulong pa;
2350 int i, rc;
2351
2352 if (*lp == NULL) {
2353 return walk_memory_regions_end(data, base, 0);
2354 }
2355
2356 if (level == 0) {
2357 PageDesc *pd = *lp;
2358 for (i = 0; i < L2_SIZE; ++i) {
2359 int prot = pd[i].flags;
2360
2361 pa = base | (i << TARGET_PAGE_BITS);
2362 if (prot != data->prot) {
2363 rc = walk_memory_regions_end(data, pa, prot);
2364 if (rc != 0) {
2365 return rc;
2366 }
2367 }
2368 }
2369 } else {
2370 void **pp = *lp;
2371 for (i = 0; i < L2_SIZE; ++i) {
2372 pa = base | ((abi_ulong)i <<
2373 (TARGET_PAGE_BITS + L2_BITS * level));
2374 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
2375 if (rc != 0) {
2376 return rc;
2377 }
2378 }
2379 }
2380
2381 return 0;
2382 }
2383
2384 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
2385 {
2386 struct walk_memory_regions_data data;
2387 unsigned long i;
2388
2389 data.fn = fn;
2390 data.priv = priv;
2391 data.start = -1ul;
2392 data.prot = 0;
2393
2394 for (i = 0; i < V_L1_SIZE; i++) {
2395 int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
2396 V_L1_SHIFT / L2_BITS - 1, l1_map + i);
2397 if (rc != 0) {
2398 return rc;
2399 }
2400 }
2401
2402 return walk_memory_regions_end(&data, 0, 0);
2403 }
2404
2405 static int dump_region(void *priv, abi_ulong start,
2406 abi_ulong end, unsigned long prot)
2407 {
2408 FILE *f = (FILE *)priv;
2409
2410 (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
2411 " "TARGET_ABI_FMT_lx" %c%c%c\n",
2412 start, end, end - start,
2413 ((prot & PAGE_READ) ? 'r' : '-'),
2414 ((prot & PAGE_WRITE) ? 'w' : '-'),
2415 ((prot & PAGE_EXEC) ? 'x' : '-'));
2416
2417 return (0);
2418 }
2419
2420 /* dump memory mappings */
2421 void page_dump(FILE *f)
2422 {
2423 (void) fprintf(f, "%-8s %-8s %-8s %s\n",
2424 "start", "end", "size", "prot");
2425 walk_memory_regions(f, dump_region);
2426 }
2427
2428 int page_get_flags(target_ulong address)
2429 {
2430 PageDesc *p;
2431
2432 p = page_find(address >> TARGET_PAGE_BITS);
2433 if (!p)
2434 return 0;
2435 return p->flags;
2436 }
2437
2438 /* Modify the flags of a page and invalidate the code if necessary.
2439 The flag PAGE_WRITE_ORG is positioned automatically depending
2440 on PAGE_WRITE. The mmap_lock should already be held. */
2441 void page_set_flags(target_ulong start, target_ulong end, int flags)
2442 {
2443 target_ulong addr, len;
2444
2445 /* This function should never be called with addresses outside the
2446 guest address space. If this assert fires, it probably indicates
2447 a missing call to h2g_valid. */
2448 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2449 assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2450 #endif
2451 assert(start < end);
2452
2453 start = start & TARGET_PAGE_MASK;
2454 end = TARGET_PAGE_ALIGN(end);
2455
2456 if (flags & PAGE_WRITE) {
2457 flags |= PAGE_WRITE_ORG;
2458 }
2459
2460 for (addr = start, len = end - start;
2461 len != 0;
2462 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2463 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2464
2465 /* If the write protection bit is set, then we invalidate
2466 the code inside. */
2467 if (!(p->flags & PAGE_WRITE) &&
2468 (flags & PAGE_WRITE) &&
2469 p->first_tb) {
2470 tb_invalidate_phys_page(addr, 0, NULL);
2471 }
2472 p->flags = flags;
2473 }
2474 }
2475
2476 int page_check_range(target_ulong start, target_ulong len, int flags)
2477 {
2478 PageDesc *p;
2479 target_ulong end;
2480 target_ulong addr;
2481
2482 /* This function should never be called with addresses outside the
2483 guest address space. If this assert fires, it probably indicates
2484 a missing call to h2g_valid. */
2485 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2486 assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2487 #endif
2488
2489 if (len == 0) {
2490 return 0;
2491 }
2492 if (start + len - 1 < start) {
2493 /* We've wrapped around. */
2494 return -1;
2495 }
2496
2497 end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2498 start = start & TARGET_PAGE_MASK;
2499
2500 for (addr = start, len = end - start;
2501 len != 0;
2502 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2503 p = page_find(addr >> TARGET_PAGE_BITS);
2504 if( !p )
2505 return -1;
2506 if( !(p->flags & PAGE_VALID) )
2507 return -1;
2508
2509 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2510 return -1;
2511 if (flags & PAGE_WRITE) {
2512 if (!(p->flags & PAGE_WRITE_ORG))
2513 return -1;
2514 /* unprotect the page if it was put read-only because it
2515 contains translated code */
2516 if (!(p->flags & PAGE_WRITE)) {
2517 if (!page_unprotect(addr, 0, NULL))
2518 return -1;
2519 }
2520 return 0;
2521 }
2522 }
2523 return 0;
2524 }
2525
2526 /* called from signal handler: invalidate the code and unprotect the
2527 page. Return TRUE if the fault was successfully handled. */
2528 int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2529 {
2530 unsigned int prot;
2531 PageDesc *p;
2532 target_ulong host_start, host_end, addr;
2533
2534 /* Technically this isn't safe inside a signal handler. However we
2535 know this only ever happens in a synchronous SEGV handler, so in
2536 practice it seems to be ok. */
2537 mmap_lock();
2538
2539 p = page_find(address >> TARGET_PAGE_BITS);
2540 if (!p) {
2541 mmap_unlock();
2542 return 0;
2543 }
2544
2545 /* if the page was really writable, then we change its
2546 protection back to writable */
2547 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
2548 host_start = address & qemu_host_page_mask;
2549 host_end = host_start + qemu_host_page_size;
2550
2551 prot = 0;
2552 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
2553 p = page_find(addr >> TARGET_PAGE_BITS);
2554 p->flags |= PAGE_WRITE;
2555 prot |= p->flags;
2556
2557 /* and since the content will be modified, we must invalidate
2558 the corresponding translated code. */
2559 tb_invalidate_phys_page(addr, pc, puc);
2560 #ifdef DEBUG_TB_CHECK
2561 tb_invalidate_check(addr);
2562 #endif
2563 }
2564 mprotect((void *)g2h(host_start), qemu_host_page_size,
2565 prot & PAGE_BITS);
2566
2567 mmap_unlock();
2568 return 1;
2569 }
2570 mmap_unlock();
2571 return 0;
2572 }
2573
2574 static inline void tlb_set_dirty(CPUState *env,
2575 unsigned long addr, target_ulong vaddr)
2576 {
2577 }
2578 #endif /* defined(CONFIG_USER_ONLY) */
2579
2580 #if !defined(CONFIG_USER_ONLY)
2581
2582 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
2583 typedef struct subpage_t {
2584 target_phys_addr_t base;
2585 ram_addr_t sub_io_index[TARGET_PAGE_SIZE];
2586 ram_addr_t region_offset[TARGET_PAGE_SIZE];
2587 } subpage_t;
2588
2589 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2590 ram_addr_t memory, ram_addr_t region_offset);
2591 static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2592 ram_addr_t orig_memory,
2593 ram_addr_t region_offset);
2594 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2595 need_subpage) \
2596 do { \
2597 if (addr > start_addr) \
2598 start_addr2 = 0; \
2599 else { \
2600 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2601 if (start_addr2 > 0) \
2602 need_subpage = 1; \
2603 } \
2604 \
2605 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2606 end_addr2 = TARGET_PAGE_SIZE - 1; \
2607 else { \
2608 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2609 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2610 need_subpage = 1; \
2611 } \
2612 } while (0)
2613
2614 /* register physical memory.
2615 For RAM, 'size' must be a multiple of the target page size.
2616 If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2617 io memory page. The address used when calling the IO function is
2618 the offset from the start of the region, plus region_offset. Both
2619 start_addr and region_offset are rounded down to a page boundary
2620 before calculating this offset. This should not be a problem unless
2621 the low bits of start_addr and region_offset differ. */
2622 void cpu_register_physical_memory_log(target_phys_addr_t start_addr,
2623 ram_addr_t size,
2624 ram_addr_t phys_offset,
2625 ram_addr_t region_offset,
2626 bool log_dirty)
2627 {
2628 target_phys_addr_t addr, end_addr;
2629 PhysPageDesc *p;
2630 CPUState *env;
2631 ram_addr_t orig_size = size;
2632 subpage_t *subpage;
2633
2634 assert(size);
2635 cpu_notify_set_memory(start_addr, size, phys_offset, log_dirty);
2636
2637 if (phys_offset == IO_MEM_UNASSIGNED) {
2638 region_offset = start_addr;
2639 }
2640 region_offset &= TARGET_PAGE_MASK;
2641 size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2642 end_addr = start_addr + (target_phys_addr_t)size;
2643
2644 addr = start_addr;
2645 do {
2646 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2647 if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2648 ram_addr_t orig_memory = p->phys_offset;
2649 target_phys_addr_t start_addr2, end_addr2;
2650 int need_subpage = 0;
2651
2652 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2653 need_subpage);
2654 if (need_subpage) {
2655 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2656 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2657 &p->phys_offset, orig_memory,
2658 p->region_offset);
2659 } else {
2660 subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2661 >> IO_MEM_SHIFT];
2662 }
2663 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2664 region_offset);
2665 p->region_offset = 0;
2666 } else {
2667 p->phys_offset = phys_offset;
2668 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2669 (phys_offset & IO_MEM_ROMD))
2670 phys_offset += TARGET_PAGE_SIZE;
2671 }
2672 } else {
2673 p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2674 p->phys_offset = phys_offset;
2675 p->region_offset = region_offset;
2676 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2677 (phys_offset & IO_MEM_ROMD)) {
2678 phys_offset += TARGET_PAGE_SIZE;
2679 } else {
2680 target_phys_addr_t start_addr2, end_addr2;
2681 int need_subpage = 0;
2682
2683 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2684 end_addr2, need_subpage);
2685
2686 if (need_subpage) {
2687 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2688 &p->phys_offset, IO_MEM_UNASSIGNED,
2689 addr & TARGET_PAGE_MASK);
2690 subpage_register(subpage, start_addr2, end_addr2,
2691 phys_offset, region_offset);
2692 p->region_offset = 0;
2693 }
2694 }
2695 }
2696 region_offset += TARGET_PAGE_SIZE;
2697 addr += TARGET_PAGE_SIZE;
2698 } while (addr != end_addr);
2699
2700 /* since each CPU stores ram addresses in its TLB cache, we must
2701 reset the modified entries */
2702 /* XXX: slow ! */
2703 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2704 tlb_flush(env, 1);
2705 }
2706 }
2707
2708 /* XXX: temporary until new memory mapping API */
2709 ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2710 {
2711 PhysPageDesc *p;
2712
2713 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2714 if (!p)
2715 return IO_MEM_UNASSIGNED;
2716 return p->phys_offset;
2717 }
2718
2719 void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2720 {
2721 if (kvm_enabled())
2722 kvm_coalesce_mmio_region(addr, size);
2723 }
2724
2725 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2726 {
2727 if (kvm_enabled())
2728 kvm_uncoalesce_mmio_region(addr, size);
2729 }
2730
2731 void qemu_flush_coalesced_mmio_buffer(void)
2732 {
2733 if (kvm_enabled())
2734 kvm_flush_coalesced_mmio_buffer();
2735 }
2736
2737 #if defined(__linux__) && !defined(TARGET_S390X)
2738
2739 #include <sys/vfs.h>
2740
2741 #define HUGETLBFS_MAGIC 0x958458f6
2742
2743 static long gethugepagesize(const char *path)
2744 {
2745 struct statfs fs;
2746 int ret;
2747
2748 do {
2749 ret = statfs(path, &fs);
2750 } while (ret != 0 && errno == EINTR);
2751
2752 if (ret != 0) {
2753 perror(path);
2754 return 0;
2755 }
2756
2757 if (fs.f_type != HUGETLBFS_MAGIC)
2758 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
2759
2760 return fs.f_bsize;
2761 }
2762
2763 static void *file_ram_alloc(RAMBlock *block,
2764 ram_addr_t memory,
2765 const char *path)
2766 {
2767 char *filename;
2768 void *area;
2769 int fd;
2770 #ifdef MAP_POPULATE
2771 int flags;
2772 #endif
2773 unsigned long hpagesize;
2774
2775 hpagesize = gethugepagesize(path);
2776 if (!hpagesize) {
2777 return NULL;
2778 }
2779
2780 if (memory < hpagesize) {
2781 return NULL;
2782 }
2783
2784 if (kvm_enabled() && !kvm_has_sync_mmu()) {
2785 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
2786 return NULL;
2787 }
2788
2789 if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
2790 return NULL;
2791 }
2792
2793 fd = mkstemp(filename);
2794 if (fd < 0) {
2795 perror("unable to create backing store for hugepages");
2796 free(filename);
2797 return NULL;
2798 }
2799 unlink(filename);
2800 free(filename);
2801
2802 memory = (memory+hpagesize-1) & ~(hpagesize-1);
2803
2804 /*
2805 * ftruncate is not supported by hugetlbfs in older
2806 * hosts, so don't bother bailing out on errors.
2807 * If anything goes wrong with it under other filesystems,
2808 * mmap will fail.
2809 */
2810 if (ftruncate(fd, memory))
2811 perror("ftruncate");
2812
2813 #ifdef MAP_POPULATE
2814 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
2815 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
2816 * to sidestep this quirk.
2817 */
2818 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
2819 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
2820 #else
2821 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
2822 #endif
2823 if (area == MAP_FAILED) {
2824 perror("file_ram_alloc: can't mmap RAM pages");
2825 close(fd);
2826 return (NULL);
2827 }
2828 block->fd = fd;
2829 return area;
2830 }
2831 #endif
2832
2833 static ram_addr_t find_ram_offset(ram_addr_t size)
2834 {
2835 RAMBlock *block, *next_block;
2836 ram_addr_t offset = 0, mingap = ULONG_MAX;
2837
2838 if (QLIST_EMPTY(&ram_list.blocks))
2839 return 0;
2840
2841 QLIST_FOREACH(block, &ram_list.blocks, next) {
2842 ram_addr_t end, next = ULONG_MAX;
2843
2844 end = block->offset + block->length;
2845
2846 QLIST_FOREACH(next_block, &ram_list.blocks, next) {
2847 if (next_block->offset >= end) {
2848 next = MIN(next, next_block->offset);
2849 }
2850 }
2851 if (next - end >= size && next - end < mingap) {
2852 offset = end;
2853 mingap = next - end;
2854 }
2855 }
2856 return offset;
2857 }
2858
2859 static ram_addr_t last_ram_offset(void)
2860 {
2861 RAMBlock *block;
2862 ram_addr_t last = 0;
2863
2864 QLIST_FOREACH(block, &ram_list.blocks, next)
2865 last = MAX(last, block->offset + block->length);
2866
2867 return last;
2868 }
2869
2870 ram_addr_t qemu_ram_alloc_from_ptr(DeviceState *dev, const char *name,
2871 ram_addr_t size, void *host)
2872 {
2873 RAMBlock *new_block, *block;
2874
2875 size = TARGET_PAGE_ALIGN(size);
2876 new_block = qemu_mallocz(sizeof(*new_block));
2877
2878 if (dev && dev->parent_bus && dev->parent_bus->info->get_dev_path) {
2879 char *id = dev->parent_bus->info->get_dev_path(dev);
2880 if (id) {
2881 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
2882 qemu_free(id);
2883 }
2884 }
2885 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
2886
2887 QLIST_FOREACH(block, &ram_list.blocks, next) {
2888 if (!strcmp(block->idstr, new_block->idstr)) {
2889 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
2890 new_block->idstr);
2891 abort();
2892 }
2893 }
2894
2895 new_block->offset = find_ram_offset(size);
2896 if (host) {
2897 new_block->host = host;
2898 new_block->flags |= RAM_PREALLOC_MASK;
2899 } else {
2900 if (mem_path) {
2901 #if defined (__linux__) && !defined(TARGET_S390X)
2902 new_block->host = file_ram_alloc(new_block, size, mem_path);
2903 if (!new_block->host) {
2904 new_block->host = qemu_vmalloc(size);
2905 qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
2906 }
2907 #else
2908 fprintf(stderr, "-mem-path option unsupported\n");
2909 exit(1);
2910 #endif
2911 } else {
2912 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2913 /* S390 KVM requires the topmost vma of the RAM to be smaller than
2914 an system defined value, which is at least 256GB. Larger systems
2915 have larger values. We put the guest between the end of data
2916 segment (system break) and this value. We use 32GB as a base to
2917 have enough room for the system break to grow. */
2918 new_block->host = mmap((void*)0x800000000, size,
2919 PROT_EXEC|PROT_READ|PROT_WRITE,
2920 MAP_SHARED | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
2921 if (new_block->host == MAP_FAILED) {
2922 fprintf(stderr, "Allocating RAM failed\n");
2923 abort();
2924 }
2925 #else
2926 if (xen_mapcache_enabled()) {
2927 xen_ram_alloc(new_block->offset, size);
2928 } else {
2929 new_block->host = qemu_vmalloc(size);
2930 }
2931 #endif
2932 qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
2933 }
2934 }
2935 new_block->length = size;
2936
2937 QLIST_INSERT_HEAD(&ram_list.blocks, new_block, next);
2938
2939 ram_list.phys_dirty = qemu_realloc(ram_list.phys_dirty,
2940 last_ram_offset() >> TARGET_PAGE_BITS);
2941 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
2942 0xff, size >> TARGET_PAGE_BITS);
2943
2944 if (kvm_enabled())
2945 kvm_setup_guest_memory(new_block->host, size);
2946
2947 return new_block->offset;
2948 }
2949
2950 ram_addr_t qemu_ram_alloc(DeviceState *dev, const char *name, ram_addr_t size)
2951 {
2952 return qemu_ram_alloc_from_ptr(dev, name, size, NULL);
2953 }
2954
2955 void qemu_ram_free(ram_addr_t addr)
2956 {
2957 RAMBlock *block;
2958
2959 QLIST_FOREACH(block, &ram_list.blocks, next) {
2960 if (addr == block->offset) {
2961 QLIST_REMOVE(block, next);
2962 if (block->flags & RAM_PREALLOC_MASK) {
2963 ;
2964 } else if (mem_path) {
2965 #if defined (__linux__) && !defined(TARGET_S390X)
2966 if (block->fd) {
2967 munmap(block->host, block->length);
2968 close(block->fd);
2969 } else {
2970 qemu_vfree(block->host);
2971 }
2972 #else
2973 abort();
2974 #endif
2975 } else {
2976 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2977 munmap(block->host, block->length);
2978 #else
2979 if (xen_mapcache_enabled()) {
2980 qemu_invalidate_entry(block->host);
2981 } else {
2982 qemu_vfree(block->host);
2983 }
2984 #endif
2985 }
2986 qemu_free(block);
2987 return;
2988 }
2989 }
2990
2991 }
2992
2993 #ifndef _WIN32
2994 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
2995 {
2996 RAMBlock *block;
2997 ram_addr_t offset;
2998 int flags;
2999 void *area, *vaddr;
3000
3001 QLIST_FOREACH(block, &ram_list.blocks, next) {
3002 offset = addr - block->offset;
3003 if (offset < block->length) {
3004 vaddr = block->host + offset;
3005 if (block->flags & RAM_PREALLOC_MASK) {
3006 ;
3007 } else {
3008 flags = MAP_FIXED;
3009 munmap(vaddr, length);
3010 if (mem_path) {
3011 #if defined(__linux__) && !defined(TARGET_S390X)
3012 if (block->fd) {
3013 #ifdef MAP_POPULATE
3014 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
3015 MAP_PRIVATE;
3016 #else
3017 flags |= MAP_PRIVATE;
3018 #endif
3019 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
3020 flags, block->fd, offset);
3021 } else {
3022 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
3023 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
3024 flags, -1, 0);
3025 }
3026 #else
3027 abort();
3028 #endif
3029 } else {
3030 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
3031 flags |= MAP_SHARED | MAP_ANONYMOUS;
3032 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
3033 flags, -1, 0);
3034 #else
3035 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
3036 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
3037 flags, -1, 0);
3038 #endif
3039 }
3040 if (area != vaddr) {
3041 fprintf(stderr, "Could not remap addr: %lx@%lx\n",
3042 length, addr);
3043 exit(1);
3044 }
3045 qemu_madvise(vaddr, length, QEMU_MADV_MERGEABLE);
3046 }
3047 return;
3048 }
3049 }
3050 }
3051 #endif /* !_WIN32 */
3052
3053 /* Return a host pointer to ram allocated with qemu_ram_alloc.
3054 With the exception of the softmmu code in this file, this should
3055 only be used for local memory (e.g. video ram) that the device owns,
3056 and knows it isn't going to access beyond the end of the block.
3057
3058 It should not be used for general purpose DMA.
3059 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
3060 */
3061 void *qemu_get_ram_ptr(ram_addr_t addr)
3062 {
3063 RAMBlock *block;
3064
3065 QLIST_FOREACH(block, &ram_list.blocks, next) {
3066 if (addr - block->offset < block->length) {
3067 /* Move this entry to to start of the list. */
3068 if (block != QLIST_FIRST(&ram_list.blocks)) {
3069 QLIST_REMOVE(block, next);
3070 QLIST_INSERT_HEAD(&ram_list.blocks, block, next);
3071 }
3072 if (xen_mapcache_enabled()) {
3073 /* We need to check if the requested address is in the RAM
3074 * because we don't want to map the entire memory in QEMU.
3075 */
3076 if (block->offset == 0) {
3077 return qemu_map_cache(addr, 0, 1);
3078 } else if (block->host == NULL) {
3079 block->host = xen_map_block(block->offset, block->length);
3080 }
3081 }
3082 return block->host + (addr - block->offset);
3083 }
3084 }
3085
3086 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
3087 abort();
3088
3089 return NULL;
3090 }
3091
3092 /* Return a host pointer to ram allocated with qemu_ram_alloc.
3093 * Same as qemu_get_ram_ptr but avoid reordering ramblocks.
3094 */
3095 void *qemu_safe_ram_ptr(ram_addr_t addr)
3096 {
3097 RAMBlock *block;
3098
3099 QLIST_FOREACH(block, &ram_list.blocks, next) {
3100 if (addr - block->offset < block->length) {
3101 if (xen_mapcache_enabled()) {
3102 /* We need to check if the requested address is in the RAM
3103 * because we don't want to map the entire memory in QEMU.
3104 */
3105 if (block->offset == 0) {
3106 return qemu_map_cache(addr, 0, 1);
3107 } else if (block->host == NULL) {
3108 block->host = xen_map_block(block->offset, block->length);
3109 }
3110 }
3111 return block->host + (addr - block->offset);
3112 }
3113 }
3114
3115 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
3116 abort();
3117
3118 return NULL;
3119 }
3120
3121 void qemu_put_ram_ptr(void *addr)
3122 {
3123 trace_qemu_put_ram_ptr(addr);
3124
3125 if (xen_mapcache_enabled()) {
3126 RAMBlock *block;
3127
3128 QLIST_FOREACH(block, &ram_list.blocks, next) {
3129 if (addr == block->host) {
3130 break;
3131 }
3132 }
3133 if (block && block->host) {
3134 xen_unmap_block(block->host, block->length);
3135 block->host = NULL;
3136 } else {
3137 qemu_map_cache_unlock(addr);
3138 }
3139 }
3140 }
3141
3142 int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
3143 {
3144 RAMBlock *block;
3145 uint8_t *host = ptr;
3146
3147 QLIST_FOREACH(block, &ram_list.blocks, next) {
3148 /* This case append when the block is not mapped. */
3149 if (block->host == NULL) {
3150 continue;
3151 }
3152 if (host - block->host < block->length) {
3153 *ram_addr = block->offset + (host - block->host);
3154 return 0;
3155 }
3156 }
3157
3158 if (xen_mapcache_enabled()) {
3159 *ram_addr = qemu_ram_addr_from_mapcache(ptr);
3160 return 0;
3161 }
3162
3163 return -1;
3164 }
3165
3166 /* Some of the softmmu routines need to translate from a host pointer
3167 (typically a TLB entry) back to a ram offset. */
3168 ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
3169 {
3170 ram_addr_t ram_addr;
3171
3172 if (qemu_ram_addr_from_host(ptr, &ram_addr)) {
3173 fprintf(stderr, "Bad ram pointer %p\n", ptr);
3174 abort();
3175 }
3176 return ram_addr;
3177 }
3178
3179 static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
3180 {
3181 #ifdef DEBUG_UNASSIGNED
3182 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
3183 #endif
3184 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3185 do_unassigned_access(addr, 0, 0, 0, 1);
3186 #endif
3187 return 0;
3188 }
3189
3190 static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
3191 {
3192 #ifdef DEBUG_UNASSIGNED
3193 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
3194 #endif
3195 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3196 do_unassigned_access(addr, 0, 0, 0, 2);
3197 #endif
3198 return 0;
3199 }
3200
3201 static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
3202 {
3203 #ifdef DEBUG_UNASSIGNED
3204 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
3205 #endif
3206 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3207 do_unassigned_access(addr, 0, 0, 0, 4);
3208 #endif
3209 return 0;
3210 }
3211
3212 static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
3213 {
3214 #ifdef DEBUG_UNASSIGNED
3215 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3216 #endif
3217 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3218 do_unassigned_access(addr, 1, 0, 0, 1);
3219 #endif
3220 }
3221
3222 static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
3223 {
3224 #ifdef DEBUG_UNASSIGNED
3225 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3226 #endif
3227 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3228 do_unassigned_access(addr, 1, 0, 0, 2);
3229 #endif
3230 }
3231
3232 static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
3233 {
3234 #ifdef DEBUG_UNASSIGNED
3235 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3236 #endif
3237 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3238 do_unassigned_access(addr, 1, 0, 0, 4);
3239 #endif
3240 }
3241
3242 static CPUReadMemoryFunc * const unassigned_mem_read[3] = {
3243 unassigned_mem_readb,
3244 unassigned_mem_readw,
3245 unassigned_mem_readl,
3246 };
3247
3248 static CPUWriteMemoryFunc * const unassigned_mem_write[3] = {
3249 unassigned_mem_writeb,
3250 unassigned_mem_writew,
3251 unassigned_mem_writel,
3252 };
3253
3254 static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
3255 uint32_t val)
3256 {
3257 int dirty_flags;
3258 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3259 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3260 #if !defined(CONFIG_USER_ONLY)
3261 tb_invalidate_phys_page_fast(ram_addr, 1);
3262 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3263 #endif
3264 }
3265 stb_p(qemu_get_ram_ptr(ram_addr), val);
3266 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3267 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
3268 /* we remove the notdirty callback only if the code has been
3269 flushed */
3270 if (dirty_flags == 0xff)
3271 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3272 }
3273
3274 static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
3275 uint32_t val)
3276 {
3277 int dirty_flags;
3278 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3279 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3280 #if !defined(CONFIG_USER_ONLY)
3281 tb_invalidate_phys_page_fast(ram_addr, 2);
3282 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3283 #endif
3284 }
3285 stw_p(qemu_get_ram_ptr(ram_addr), val);
3286 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3287 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
3288 /* we remove the notdirty callback only if the code has been
3289 flushed */
3290 if (dirty_flags == 0xff)
3291 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3292 }
3293
3294 static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
3295 uint32_t val)
3296 {
3297 int dirty_flags;
3298 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3299 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3300 #if !defined(CONFIG_USER_ONLY)
3301 tb_invalidate_phys_page_fast(ram_addr, 4);
3302 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3303 #endif
3304 }
3305 stl_p(qemu_get_ram_ptr(ram_addr), val);
3306 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3307 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
3308 /* we remove the notdirty callback only if the code has been
3309 flushed */
3310 if (dirty_flags == 0xff)
3311 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3312 }
3313
3314 static CPUReadMemoryFunc * const error_mem_read[3] = {
3315 NULL, /* never used */
3316 NULL, /* never used */
3317 NULL, /* never used */
3318 };
3319
3320 static CPUWriteMemoryFunc * const notdirty_mem_write[3] = {
3321 notdirty_mem_writeb,
3322 notdirty_mem_writew,
3323 notdirty_mem_writel,
3324 };
3325
3326 /* Generate a debug exception if a watchpoint has been hit. */
3327 static void check_watchpoint(int offset, int len_mask, int flags)
3328 {
3329 CPUState *env = cpu_single_env;
3330 target_ulong pc, cs_base;
3331 TranslationBlock *tb;
3332 target_ulong vaddr;
3333 CPUWatchpoint *wp;
3334 int cpu_flags;
3335
3336 if (env->watchpoint_hit) {
3337 /* We re-entered the check after replacing the TB. Now raise
3338 * the debug interrupt so that is will trigger after the
3339 * current instruction. */
3340 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
3341 return;
3342 }
3343 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
3344 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
3345 if ((vaddr == (wp->vaddr & len_mask) ||
3346 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
3347 wp->flags |= BP_WATCHPOINT_HIT;
3348 if (!env->watchpoint_hit) {
3349 env->watchpoint_hit = wp;
3350 tb = tb_find_pc(env->mem_io_pc);
3351 if (!tb) {
3352 cpu_abort(env, "check_watchpoint: could not find TB for "
3353 "pc=%p", (void *)env->mem_io_pc);
3354 }
3355 cpu_restore_state(tb, env, env->mem_io_pc);
3356 tb_phys_invalidate(tb, -1);
3357 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
3358 env->exception_index = EXCP_DEBUG;
3359 } else {
3360 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
3361 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
3362 }
3363 cpu_resume_from_signal(env, NULL);
3364 }
3365 } else {
3366 wp->flags &= ~BP_WATCHPOINT_HIT;
3367 }
3368 }
3369 }
3370
3371 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
3372 so these check for a hit then pass through to the normal out-of-line
3373 phys routines. */
3374 static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
3375 {
3376 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
3377 return ldub_phys(addr);
3378 }
3379
3380 static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
3381 {
3382 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
3383 return lduw_phys(addr);
3384 }
3385
3386 static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
3387 {
3388 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
3389 return ldl_phys(addr);
3390 }
3391
3392 static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
3393 uint32_t val)
3394 {
3395 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
3396 stb_phys(addr, val);
3397 }
3398
3399 static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
3400 uint32_t val)
3401 {
3402 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
3403 stw_phys(addr, val);
3404 }
3405
3406 static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
3407 uint32_t val)
3408 {
3409 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
3410 stl_phys(addr, val);
3411 }
3412
3413 static CPUReadMemoryFunc * const watch_mem_read[3] = {
3414 watch_mem_readb,
3415 watch_mem_readw,
3416 watch_mem_readl,
3417 };
3418
3419 static CPUWriteMemoryFunc * const watch_mem_write[3] = {
3420 watch_mem_writeb,
3421 watch_mem_writew,
3422 watch_mem_writel,
3423 };
3424
3425 static inline uint32_t subpage_readlen (subpage_t *mmio,
3426 target_phys_addr_t addr,
3427 unsigned int len)
3428 {
3429 unsigned int idx = SUBPAGE_IDX(addr);
3430 #if defined(DEBUG_SUBPAGE)
3431 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
3432 mmio, len, addr, idx);
3433 #endif
3434
3435 addr += mmio->region_offset[idx];
3436 idx = mmio->sub_io_index[idx];
3437 return io_mem_read[idx][len](io_mem_opaque[idx], addr);
3438 }
3439
3440 static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
3441 uint32_t value, unsigned int len)
3442 {
3443 unsigned int idx = SUBPAGE_IDX(addr);
3444 #if defined(DEBUG_SUBPAGE)
3445 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n",
3446 __func__, mmio, len, addr, idx, value);
3447 #endif
3448
3449 addr += mmio->region_offset[idx];
3450 idx = mmio->sub_io_index[idx];
3451 io_mem_write[idx][len](io_mem_opaque[idx], addr, value);
3452 }
3453
3454 static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
3455 {
3456 return subpage_readlen(opaque, addr, 0);
3457 }
3458
3459 static void subpage_writeb (void *opaque, target_phys_addr_t addr,
3460 uint32_t value)
3461 {
3462 subpage_writelen(opaque, addr, value, 0);
3463 }
3464
3465 static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
3466 {
3467 return subpage_readlen(opaque, addr, 1);
3468 }
3469
3470 static void subpage_writew (void *opaque, target_phys_addr_t addr,
3471 uint32_t value)
3472 {
3473 subpage_writelen(opaque, addr, value, 1);
3474 }
3475
3476 static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
3477 {
3478 return subpage_readlen(opaque, addr, 2);
3479 }
3480
3481 static void subpage_writel (void *opaque, target_phys_addr_t addr,
3482 uint32_t value)
3483 {
3484 subpage_writelen(opaque, addr, value, 2);
3485 }
3486
3487 static CPUReadMemoryFunc * const subpage_read[] = {
3488 &subpage_readb,
3489 &subpage_readw,
3490 &subpage_readl,
3491 };
3492
3493 static CPUWriteMemoryFunc * const subpage_write[] = {
3494 &subpage_writeb,
3495 &subpage_writew,
3496 &subpage_writel,
3497 };
3498
3499 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
3500 ram_addr_t memory, ram_addr_t region_offset)
3501 {
3502 int idx, eidx;
3503
3504 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
3505 return -1;
3506 idx = SUBPAGE_IDX(start);
3507 eidx = SUBPAGE_IDX(end);
3508 #if defined(DEBUG_SUBPAGE)
3509 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
3510 mmio, start, end, idx, eidx, memory);
3511 #endif
3512 if ((memory & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
3513 memory = IO_MEM_UNASSIGNED;
3514 memory = (memory >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3515 for (; idx <= eidx; idx++) {
3516 mmio->sub_io_index[idx] = memory;
3517 mmio->region_offset[idx] = region_offset;
3518 }
3519
3520 return 0;
3521 }
3522
3523 static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
3524 ram_addr_t orig_memory,
3525 ram_addr_t region_offset)
3526 {
3527 subpage_t *mmio;
3528 int subpage_memory;
3529
3530 mmio = qemu_mallocz(sizeof(subpage_t));
3531
3532 mmio->base = base;
3533 subpage_memory = cpu_register_io_memory(subpage_read, subpage_write, mmio,
3534 DEVICE_NATIVE_ENDIAN);
3535 #if defined(DEBUG_SUBPAGE)
3536 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
3537 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
3538 #endif
3539 *phys = subpage_memory | IO_MEM_SUBPAGE;
3540 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, orig_memory, region_offset);
3541
3542 return mmio;
3543 }
3544
3545 static int get_free_io_mem_idx(void)
3546 {
3547 int i;
3548
3549 for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
3550 if (!io_mem_used[i]) {
3551 io_mem_used[i] = 1;
3552 return i;
3553 }
3554 fprintf(stderr, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES);
3555 return -1;
3556 }
3557
3558 /*
3559 * Usually, devices operate in little endian mode. There are devices out
3560 * there that operate in big endian too. Each device gets byte swapped
3561 * mmio if plugged onto a CPU that does the other endianness.
3562 *
3563 * CPU Device swap?
3564 *
3565 * little little no
3566 * little big yes
3567 * big little yes
3568 * big big no
3569 */
3570
3571 typedef struct SwapEndianContainer {
3572 CPUReadMemoryFunc *read[3];
3573 CPUWriteMemoryFunc *write[3];
3574 void *opaque;
3575 } SwapEndianContainer;
3576
3577 static uint32_t swapendian_mem_readb (void *opaque, target_phys_addr_t addr)
3578 {
3579 uint32_t val;
3580 SwapEndianContainer *c = opaque;
3581 val = c->read[0](c->opaque, addr);
3582 return val;
3583 }
3584
3585 static uint32_t swapendian_mem_readw(void *opaque, target_phys_addr_t addr)
3586 {
3587 uint32_t val;
3588 SwapEndianContainer *c = opaque;
3589 val = bswap16(c->read[1](c->opaque, addr));
3590 return val;
3591 }
3592
3593 static uint32_t swapendian_mem_readl(void *opaque, target_phys_addr_t addr)
3594 {
3595 uint32_t val;
3596 SwapEndianContainer *c = opaque;
3597 val = bswap32(c->read[2](c->opaque, addr));
3598 return val;
3599 }
3600
3601 static CPUReadMemoryFunc * const swapendian_readfn[3]={
3602 swapendian_mem_readb,
3603 swapendian_mem_readw,
3604 swapendian_mem_readl
3605 };
3606
3607 static void swapendian_mem_writeb(void *opaque, target_phys_addr_t addr,
3608 uint32_t val)
3609 {
3610 SwapEndianContainer *c = opaque;
3611 c->write[0](c->opaque, addr, val);
3612 }
3613
3614 static void swapendian_mem_writew(void *opaque, target_phys_addr_t addr,
3615 uint32_t val)
3616 {
3617 SwapEndianContainer *c = opaque;
3618 c->write[1](c->opaque, addr, bswap16(val));
3619 }
3620
3621 static void swapendian_mem_writel(void *opaque, target_phys_addr_t addr,
3622 uint32_t val)
3623 {
3624 SwapEndianContainer *c = opaque;
3625 c->write[2](c->opaque, addr, bswap32(val));
3626 }
3627
3628 static CPUWriteMemoryFunc * const swapendian_writefn[3]={
3629 swapendian_mem_writeb,
3630 swapendian_mem_writew,
3631 swapendian_mem_writel
3632 };
3633
3634 static void swapendian_init(int io_index)
3635 {
3636 SwapEndianContainer *c = qemu_malloc(sizeof(SwapEndianContainer));
3637 int i;
3638
3639 /* Swap mmio for big endian targets */
3640 c->opaque = io_mem_opaque[io_index];
3641 for (i = 0; i < 3; i++) {
3642 c->read[i] = io_mem_read[io_index][i];
3643 c->write[i] = io_mem_write[io_index][i];
3644
3645 io_mem_read[io_index][i] = swapendian_readfn[i];
3646 io_mem_write[io_index][i] = swapendian_writefn[i];
3647 }
3648 io_mem_opaque[io_index] = c;
3649 }
3650
3651 static void swapendian_del(int io_index)
3652 {
3653 if (io_mem_read[io_index][0] == swapendian_readfn[0]) {
3654 qemu_free(io_mem_opaque[io_index]);
3655 }
3656 }
3657
3658 /* mem_read and mem_write are arrays of functions containing the
3659 function to access byte (index 0), word (index 1) and dword (index
3660 2). Functions can be omitted with a NULL function pointer.
3661 If io_index is non zero, the corresponding io zone is
3662 modified. If it is zero, a new io zone is allocated. The return
3663 value can be used with cpu_register_physical_memory(). (-1) is
3664 returned if error. */
3665 static int cpu_register_io_memory_fixed(int io_index,
3666 CPUReadMemoryFunc * const *mem_read,
3667 CPUWriteMemoryFunc * const *mem_write,
3668 void *opaque, enum device_endian endian)
3669 {
3670 int i;
3671
3672 if (io_index <= 0) {
3673 io_index = get_free_io_mem_idx();
3674 if (io_index == -1)
3675 return io_index;
3676 } else {
3677 io_index >>= IO_MEM_SHIFT;
3678 if (io_index >= IO_MEM_NB_ENTRIES)
3679 return -1;
3680 }
3681
3682 for (i = 0; i < 3; ++i) {
3683 io_mem_read[io_index][i]
3684 = (mem_read[i] ? mem_read[i] : unassigned_mem_read[i]);
3685 }
3686 for (i = 0; i < 3; ++i) {
3687 io_mem_write[io_index][i]
3688 = (mem_write[i] ? mem_write[i] : unassigned_mem_write[i]);
3689 }
3690 io_mem_opaque[io_index] = opaque;
3691
3692 switch (endian) {
3693 case DEVICE_BIG_ENDIAN:
3694 #ifndef TARGET_WORDS_BIGENDIAN
3695 swapendian_init(io_index);
3696 #endif
3697 break;
3698 case DEVICE_LITTLE_ENDIAN:
3699 #ifdef TARGET_WORDS_BIGENDIAN
3700 swapendian_init(io_index);
3701 #endif
3702 break;
3703 case DEVICE_NATIVE_ENDIAN:
3704 default:
3705 break;
3706 }
3707
3708 return (io_index << IO_MEM_SHIFT);
3709 }
3710
3711 int cpu_register_io_memory(CPUReadMemoryFunc * const *mem_read,
3712 CPUWriteMemoryFunc * const *mem_write,
3713 void *opaque, enum device_endian endian)
3714 {
3715 return cpu_register_io_memory_fixed(0, mem_read, mem_write, opaque, endian);
3716 }
3717
3718 void cpu_unregister_io_memory(int io_table_address)
3719 {
3720 int i;
3721 int io_index = io_table_address >> IO_MEM_SHIFT;
3722
3723 swapendian_del(io_index);
3724
3725 for (i=0;i < 3; i++) {
3726 io_mem_read[io_index][i] = unassigned_mem_read[i];
3727 io_mem_write[io_index][i] = unassigned_mem_write[i];
3728 }
3729 io_mem_opaque[io_index] = NULL;
3730 io_mem_used[io_index] = 0;
3731 }
3732
3733 static void io_mem_init(void)
3734 {
3735 int i;
3736
3737 cpu_register_io_memory_fixed(IO_MEM_ROM, error_mem_read,
3738 unassigned_mem_write, NULL,
3739 DEVICE_NATIVE_ENDIAN);
3740 cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED, unassigned_mem_read,
3741 unassigned_mem_write, NULL,
3742 DEVICE_NATIVE_ENDIAN);
3743 cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY, error_mem_read,
3744 notdirty_mem_write, NULL,
3745 DEVICE_NATIVE_ENDIAN);
3746 for (i=0; i<5; i++)
3747 io_mem_used[i] = 1;
3748
3749 io_mem_watch = cpu_register_io_memory(watch_mem_read,
3750 watch_mem_write, NULL,
3751 DEVICE_NATIVE_ENDIAN);
3752 }
3753
3754 #endif /* !defined(CONFIG_USER_ONLY) */
3755
3756 /* physical memory access (slow version, mainly for debug) */
3757 #if defined(CONFIG_USER_ONLY)
3758 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3759 uint8_t *buf, int len, int is_write)
3760 {
3761 int l, flags;
3762 target_ulong page;
3763 void * p;
3764
3765 while (len > 0) {
3766 page = addr & TARGET_PAGE_MASK;
3767 l = (page + TARGET_PAGE_SIZE) - addr;
3768 if (l > len)
3769 l = len;
3770 flags = page_get_flags(page);
3771 if (!(flags & PAGE_VALID))
3772 return -1;
3773 if (is_write) {
3774 if (!(flags & PAGE_WRITE))
3775 return -1;
3776 /* XXX: this code should not depend on lock_user */
3777 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
3778 return -1;
3779 memcpy(p, buf, l);
3780 unlock_user(p, addr, l);
3781 } else {
3782 if (!(flags & PAGE_READ))
3783 return -1;
3784 /* XXX: this code should not depend on lock_user */
3785 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
3786 return -1;
3787 memcpy(buf, p, l);
3788 unlock_user(p, addr, 0);
3789 }
3790 len -= l;
3791 buf += l;
3792 addr += l;
3793 }
3794 return 0;
3795 }
3796
3797 #else
3798 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
3799 int len, int is_write)
3800 {
3801 int l, io_index;
3802 uint8_t *ptr;
3803 uint32_t val;
3804 target_phys_addr_t page;
3805 unsigned long pd;
3806 PhysPageDesc *p;
3807
3808 while (len > 0) {
3809 page = addr & TARGET_PAGE_MASK;
3810 l = (page + TARGET_PAGE_SIZE) - addr;
3811 if (l > len)
3812 l = len;
3813 p = phys_page_find(page >> TARGET_PAGE_BITS);
3814 if (!p) {
3815 pd = IO_MEM_UNASSIGNED;
3816 } else {
3817 pd = p->phys_offset;
3818 }
3819
3820 if (is_write) {
3821 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3822 target_phys_addr_t addr1 = addr;
3823 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3824 if (p)
3825 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3826 /* XXX: could force cpu_single_env to NULL to avoid
3827 potential bugs */
3828 if (l >= 4 && ((addr1 & 3) == 0)) {
3829 /* 32 bit write access */
3830 val = ldl_p(buf);
3831 io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
3832 l = 4;
3833 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3834 /* 16 bit write access */
3835 val = lduw_p(buf);
3836 io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
3837 l = 2;
3838 } else {
3839 /* 8 bit write access */
3840 val = ldub_p(buf);
3841 io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
3842 l = 1;
3843 }
3844 } else {
3845 unsigned long addr1;
3846 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3847 /* RAM case */
3848 ptr = qemu_get_ram_ptr(addr1);
3849 memcpy(ptr, buf, l);
3850 if (!cpu_physical_memory_is_dirty(addr1)) {
3851 /* invalidate code */
3852 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3853 /* set dirty bit */
3854 cpu_physical_memory_set_dirty_flags(
3855 addr1, (0xff & ~CODE_DIRTY_FLAG));
3856 }
3857 qemu_put_ram_ptr(ptr);
3858 }
3859 } else {
3860 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3861 !(pd & IO_MEM_ROMD)) {
3862 target_phys_addr_t addr1 = addr;
3863 /* I/O case */
3864 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3865 if (p)
3866 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3867 if (l >= 4 && ((addr1 & 3) == 0)) {
3868 /* 32 bit read access */
3869 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
3870 stl_p(buf, val);
3871 l = 4;
3872 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3873 /* 16 bit read access */
3874 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
3875 stw_p(buf, val);
3876 l = 2;
3877 } else {
3878 /* 8 bit read access */
3879 val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
3880 stb_p(buf, val);
3881 l = 1;
3882 }
3883 } else {
3884 /* RAM case */
3885 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
3886 memcpy(buf, ptr + (addr & ~TARGET_PAGE_MASK), l);
3887 qemu_put_ram_ptr(ptr);
3888 }
3889 }
3890 len -= l;
3891 buf += l;
3892 addr += l;
3893 }
3894 }
3895
3896 /* used for ROM loading : can write in RAM and ROM */
3897 void cpu_physical_memory_write_rom(target_phys_addr_t addr,
3898 const uint8_t *buf, int len)
3899 {
3900 int l;
3901 uint8_t *ptr;
3902 target_phys_addr_t page;
3903 unsigned long pd;
3904 PhysPageDesc *p;
3905
3906 while (len > 0) {
3907 page = addr & TARGET_PAGE_MASK;
3908 l = (page + TARGET_PAGE_SIZE) - addr;
3909 if (l > len)
3910 l = len;
3911 p = phys_page_find(page >> TARGET_PAGE_BITS);
3912 if (!p) {
3913 pd = IO_MEM_UNASSIGNED;
3914 } else {
3915 pd = p->phys_offset;
3916 }
3917
3918 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3919 (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3920 !(pd & IO_MEM_ROMD)) {
3921 /* do nothing */
3922 } else {
3923 unsigned long addr1;
3924 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3925 /* ROM/RAM case */
3926 ptr = qemu_get_ram_ptr(addr1);
3927 memcpy(ptr, buf, l);
3928 qemu_put_ram_ptr(ptr);
3929 }
3930 len -= l;
3931 buf += l;
3932 addr += l;
3933 }
3934 }
3935
3936 typedef struct {
3937 void *buffer;
3938 target_phys_addr_t addr;
3939 target_phys_addr_t len;
3940 } BounceBuffer;
3941
3942 static BounceBuffer bounce;
3943
3944 typedef struct MapClient {
3945 void *opaque;
3946 void (*callback)(void *opaque);
3947 QLIST_ENTRY(MapClient) link;
3948 } MapClient;
3949
3950 static QLIST_HEAD(map_client_list, MapClient) map_client_list
3951 = QLIST_HEAD_INITIALIZER(map_client_list);
3952
3953 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
3954 {
3955 MapClient *client = qemu_malloc(sizeof(*client));
3956
3957 client->opaque = opaque;
3958 client->callback = callback;
3959 QLIST_INSERT_HEAD(&map_client_list, client, link);
3960 return client;
3961 }
3962
3963 void cpu_unregister_map_client(void *_client)
3964 {
3965 MapClient *client = (MapClient *)_client;
3966
3967 QLIST_REMOVE(client, link);
3968 qemu_free(client);
3969 }
3970
3971 static void cpu_notify_map_clients(void)
3972 {
3973 MapClient *client;
3974
3975 while (!QLIST_EMPTY(&map_client_list)) {
3976 client = QLIST_FIRST(&map_client_list);
3977 client->callback(client->opaque);
3978 cpu_unregister_map_client(client);
3979 }
3980 }
3981
3982 /* Map a physical memory region into a host virtual address.
3983 * May map a subset of the requested range, given by and returned in *plen.
3984 * May return NULL if resources needed to perform the mapping are exhausted.
3985 * Use only for reads OR writes - not for read-modify-write operations.
3986 * Use cpu_register_map_client() to know when retrying the map operation is
3987 * likely to succeed.
3988 */
3989 void *cpu_physical_memory_map(target_phys_addr_t addr,
3990 target_phys_addr_t *plen,
3991 int is_write)
3992 {
3993 target_phys_addr_t len = *plen;
3994 target_phys_addr_t done = 0;
3995 int l;
3996 uint8_t *ret = NULL;
3997 uint8_t *ptr;
3998 target_phys_addr_t page;
3999 unsigned long pd;
4000 PhysPageDesc *p;
4001 unsigned long addr1;
4002
4003 while (len > 0) {
4004 page = addr & TARGET_PAGE_MASK;
4005 l = (page + TARGET_PAGE_SIZE) - addr;
4006 if (l > len)
4007 l = len;
4008 p = phys_page_find(page >> TARGET_PAGE_BITS);
4009 if (!p) {
4010 pd = IO_MEM_UNASSIGNED;
4011 } else {
4012 pd = p->phys_offset;
4013 }
4014
4015 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4016 if (done || bounce.buffer) {
4017 break;
4018 }
4019 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
4020 bounce.addr = addr;
4021 bounce.len = l;
4022 if (!is_write) {
4023 cpu_physical_memory_read(addr, bounce.buffer, l);
4024 }
4025 ptr = bounce.buffer;
4026 } else {
4027 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4028 ptr = qemu_get_ram_ptr(addr1);
4029 }
4030 if (!done) {
4031 ret = ptr;
4032 } else if (ret + done != ptr) {
4033 break;
4034 }
4035
4036 len -= l;
4037 addr += l;
4038 done += l;
4039 }
4040 *plen = done;
4041 return ret;
4042 }
4043
4044 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
4045 * Will also mark the memory as dirty if is_write == 1. access_len gives
4046 * the amount of memory that was actually read or written by the caller.
4047 */
4048 void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
4049 int is_write, target_phys_addr_t access_len)
4050 {
4051 if (buffer != bounce.buffer) {
4052 if (is_write) {
4053 ram_addr_t addr1 = qemu_ram_addr_from_host_nofail(buffer);
4054 while (access_len) {
4055 unsigned l;
4056 l = TARGET_PAGE_SIZE;
4057 if (l > access_len)
4058 l = access_len;
4059 if (!cpu_physical_memory_is_dirty(addr1)) {
4060 /* invalidate code */
4061 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
4062 /* set dirty bit */
4063 cpu_physical_memory_set_dirty_flags(
4064 addr1, (0xff & ~CODE_DIRTY_FLAG));
4065 }
4066 addr1 += l;
4067 access_len -= l;
4068 }
4069 }
4070 if (xen_mapcache_enabled()) {
4071 uint8_t *buffer1 = buffer;
4072 uint8_t *end_buffer = buffer + len;
4073
4074 while (buffer1 < end_buffer) {
4075 qemu_put_ram_ptr(buffer1);
4076 buffer1 += TARGET_PAGE_SIZE;
4077 }
4078 }
4079 return;
4080 }
4081 if (is_write) {
4082 cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
4083 }
4084 qemu_vfree(bounce.buffer);
4085 bounce.buffer = NULL;
4086 cpu_notify_map_clients();
4087 }
4088
4089 /* warning: addr must be aligned */
4090 uint32_t ldl_phys(target_phys_addr_t addr)
4091 {
4092 int io_index;
4093 uint8_t *ptr;
4094 uint32_t val;
4095 unsigned long pd;
4096 PhysPageDesc *p;
4097
4098 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4099 if (!p) {
4100 pd = IO_MEM_UNASSIGNED;
4101 } else {
4102 pd = p->phys_offset;
4103 }
4104
4105 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
4106 !(pd & IO_MEM_ROMD)) {
4107 /* I/O case */
4108 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4109 if (p)
4110 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4111 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
4112 } else {
4113 /* RAM case */
4114 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
4115 (addr & ~TARGET_PAGE_MASK);
4116 val = ldl_p(ptr);
4117 }
4118 return val;
4119 }
4120
4121 /* warning: addr must be aligned */
4122 uint64_t ldq_phys(target_phys_addr_t addr)
4123 {
4124 int io_index;
4125 uint8_t *ptr;
4126 uint64_t val;
4127 unsigned long pd;
4128 PhysPageDesc *p;
4129
4130 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4131 if (!p) {
4132 pd = IO_MEM_UNASSIGNED;
4133 } else {
4134 pd = p->phys_offset;
4135 }
4136
4137 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
4138 !(pd & IO_MEM_ROMD)) {
4139 /* I/O case */
4140 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4141 if (p)
4142 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4143 #ifdef TARGET_WORDS_BIGENDIAN
4144 val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
4145 val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
4146 #else
4147 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
4148 val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
4149 #endif
4150 } else {
4151 /* RAM case */
4152 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
4153 (addr & ~TARGET_PAGE_MASK);
4154 val = ldq_p(ptr);
4155 }
4156 return val;
4157 }
4158
4159 /* XXX: optimize */
4160 uint32_t ldub_phys(target_phys_addr_t addr)
4161 {
4162 uint8_t val;
4163 cpu_physical_memory_read(addr, &val, 1);
4164 return val;
4165 }
4166
4167 /* warning: addr must be aligned */
4168 uint32_t lduw_phys(target_phys_addr_t addr)
4169 {
4170 int io_index;
4171 uint8_t *ptr;
4172 uint64_t val;
4173 unsigned long pd;
4174 PhysPageDesc *p;
4175
4176 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4177 if (!p) {
4178 pd = IO_MEM_UNASSIGNED;
4179 } else {
4180 pd = p->phys_offset;
4181 }
4182
4183 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
4184 !(pd & IO_MEM_ROMD)) {
4185 /* I/O case */
4186 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4187 if (p)
4188 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4189 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr);
4190 } else {
4191 /* RAM case */
4192 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
4193 (addr & ~TARGET_PAGE_MASK);
4194 val = lduw_p(ptr);
4195 }
4196 return val;
4197 }
4198
4199 /* warning: addr must be aligned. The ram page is not masked as dirty
4200 and the code inside is not invalidated. It is useful if the dirty
4201 bits are used to track modified PTEs */
4202 void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
4203 {
4204 int io_index;
4205 uint8_t *ptr;
4206 unsigned long pd;
4207 PhysPageDesc *p;
4208
4209 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4210 if (!p) {
4211 pd = IO_MEM_UNASSIGNED;
4212 } else {
4213 pd = p->phys_offset;
4214 }
4215
4216 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4217 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4218 if (p)
4219 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4220 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4221 } else {
4222 unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4223 ptr = qemu_get_ram_ptr(addr1);
4224 stl_p(ptr, val);
4225
4226 if (unlikely(in_migration)) {
4227 if (!cpu_physical_memory_is_dirty(addr1)) {
4228 /* invalidate code */
4229 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
4230 /* set dirty bit */
4231 cpu_physical_memory_set_dirty_flags(
4232 addr1, (0xff & ~CODE_DIRTY_FLAG));
4233 }
4234 }
4235 }
4236 }
4237
4238 void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
4239 {
4240 int io_index;
4241 uint8_t *ptr;
4242 unsigned long pd;
4243 PhysPageDesc *p;
4244
4245 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4246 if (!p) {
4247 pd = IO_MEM_UNASSIGNED;
4248 } else {
4249 pd = p->phys_offset;
4250 }
4251
4252 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4253 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4254 if (p)
4255 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4256 #ifdef TARGET_WORDS_BIGENDIAN
4257 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
4258 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
4259 #else
4260 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4261 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
4262 #endif
4263 } else {
4264 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
4265 (addr & ~TARGET_PAGE_MASK);
4266 stq_p(ptr, val);
4267 }
4268 }
4269
4270 /* warning: addr must be aligned */
4271 void stl_phys(target_phys_addr_t addr, uint32_t val)
4272 {
4273 int io_index;
4274 uint8_t *ptr;
4275 unsigned long pd;
4276 PhysPageDesc *p;
4277
4278 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4279 if (!p) {
4280 pd = IO_MEM_UNASSIGNED;
4281 } else {
4282 pd = p->phys_offset;
4283 }
4284
4285 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4286 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4287 if (p)
4288 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4289 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4290 } else {
4291 unsigned long addr1;
4292 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4293 /* RAM case */
4294 ptr = qemu_get_ram_ptr(addr1);
4295 stl_p(ptr, val);
4296 if (!cpu_physical_memory_is_dirty(addr1)) {
4297 /* invalidate code */
4298 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
4299 /* set dirty bit */
4300 cpu_physical_memory_set_dirty_flags(addr1,
4301 (0xff & ~CODE_DIRTY_FLAG));
4302 }
4303 }
4304 }
4305
4306 /* XXX: optimize */
4307 void stb_phys(target_phys_addr_t addr, uint32_t val)
4308 {
4309 uint8_t v = val;
4310 cpu_physical_memory_write(addr, &v, 1);
4311 }
4312
4313 /* warning: addr must be aligned */
4314 void stw_phys(target_phys_addr_t addr, uint32_t val)
4315 {
4316 int io_index;
4317 uint8_t *ptr;
4318 unsigned long pd;
4319 PhysPageDesc *p;
4320
4321 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4322 if (!p) {
4323 pd = IO_MEM_UNASSIGNED;
4324 } else {
4325 pd = p->phys_offset;
4326 }
4327
4328 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4329 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4330 if (p)
4331 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4332 io_mem_write[io_index][1](io_mem_opaque[io_index], addr, val);
4333 } else {
4334 unsigned long addr1;
4335 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4336 /* RAM case */
4337 ptr = qemu_get_ram_ptr(addr1);
4338 stw_p(ptr, val);
4339 if (!cpu_physical_memory_is_dirty(addr1)) {
4340 /* invalidate code */
4341 tb_invalidate_phys_page_range(addr1, addr1 + 2, 0);
4342 /* set dirty bit */
4343 cpu_physical_memory_set_dirty_flags(addr1,
4344 (0xff & ~CODE_DIRTY_FLAG));
4345 }
4346 }
4347 }
4348
4349 /* XXX: optimize */
4350 void stq_phys(target_phys_addr_t addr, uint64_t val)
4351 {
4352 val = tswap64(val);
4353 cpu_physical_memory_write(addr, &val, 8);
4354 }
4355
4356 /* virtual memory access for debug (includes writing to ROM) */
4357 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
4358 uint8_t *buf, int len, int is_write)
4359 {
4360 int l;
4361 target_phys_addr_t phys_addr;
4362 target_ulong page;
4363
4364 while (len > 0) {
4365 page = addr & TARGET_PAGE_MASK;
4366 phys_addr = cpu_get_phys_page_debug(env, page);
4367 /* if no physical page mapped, return an error */
4368 if (phys_addr == -1)
4369 return -1;
4370 l = (page + TARGET_PAGE_SIZE) - addr;
4371 if (l > len)
4372 l = len;
4373 phys_addr += (addr & ~TARGET_PAGE_MASK);
4374 if (is_write)
4375 cpu_physical_memory_write_rom(phys_addr, buf, l);
4376 else
4377 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
4378 len -= l;
4379 buf += l;
4380 addr += l;
4381 }
4382 return 0;
4383 }
4384 #endif
4385
4386 /* in deterministic execution mode, instructions doing device I/Os
4387 must be at the end of the TB */
4388 void cpu_io_recompile(CPUState *env, void *retaddr)
4389 {
4390 TranslationBlock *tb;
4391 uint32_t n, cflags;
4392 target_ulong pc, cs_base;
4393 uint64_t flags;
4394
4395 tb = tb_find_pc((unsigned long)retaddr);
4396 if (!tb) {
4397 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
4398 retaddr);
4399 }
4400 n = env->icount_decr.u16.low + tb->icount;
4401 cpu_restore_state(tb, env, (unsigned long)retaddr);
4402 /* Calculate how many instructions had been executed before the fault
4403 occurred. */
4404 n = n - env->icount_decr.u16.low;
4405 /* Generate a new TB ending on the I/O insn. */
4406 n++;
4407 /* On MIPS and SH, delay slot instructions can only be restarted if
4408 they were already the first instruction in the TB. If this is not
4409 the first instruction in a TB then re-execute the preceding
4410 branch. */
4411 #if defined(TARGET_MIPS)
4412 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
4413 env->active_tc.PC -= 4;
4414 env->icount_decr.u16.low++;
4415 env->hflags &= ~MIPS_HFLAG_BMASK;
4416 }
4417 #elif defined(TARGET_SH4)
4418 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
4419 && n > 1) {
4420 env->pc -= 2;
4421 env->icount_decr.u16.low++;
4422 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
4423 }
4424 #endif
4425 /* This should never happen. */
4426 if (n > CF_COUNT_MASK)
4427 cpu_abort(env, "TB too big during recompile");
4428
4429 cflags = n | CF_LAST_IO;
4430 pc = tb->pc;
4431 cs_base = tb->cs_base;
4432 flags = tb->flags;
4433 tb_phys_invalidate(tb, -1);
4434 /* FIXME: In theory this could raise an exception. In practice
4435 we have already translated the block once so it's probably ok. */
4436 tb_gen_code(env, pc, cs_base, flags, cflags);
4437 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
4438 the first in the TB) then we end up generating a whole new TB and
4439 repeating the fault, which is horribly inefficient.
4440 Better would be to execute just this insn uncached, or generate a
4441 second new TB. */
4442 cpu_resume_from_signal(env, NULL);
4443 }
4444
4445 #if !defined(CONFIG_USER_ONLY)
4446
4447 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
4448 {
4449 int i, target_code_size, max_target_code_size;
4450 int direct_jmp_count, direct_jmp2_count, cross_page;
4451 TranslationBlock *tb;
4452
4453 target_code_size = 0;
4454 max_target_code_size = 0;
4455 cross_page = 0;
4456 direct_jmp_count = 0;
4457 direct_jmp2_count = 0;
4458 for(i = 0; i < nb_tbs; i++) {
4459 tb = &tbs[i];
4460 target_code_size += tb->size;
4461 if (tb->size > max_target_code_size)
4462 max_target_code_size = tb->size;
4463 if (tb->page_addr[1] != -1)
4464 cross_page++;
4465 if (tb->tb_next_offset[0] != 0xffff) {
4466 direct_jmp_count++;
4467 if (tb->tb_next_offset[1] != 0xffff) {
4468 direct_jmp2_count++;
4469 }
4470 }
4471 }
4472 /* XXX: avoid using doubles ? */
4473 cpu_fprintf(f, "Translation buffer state:\n");
4474 cpu_fprintf(f, "gen code size %td/%ld\n",
4475 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
4476 cpu_fprintf(f, "TB count %d/%d\n",
4477 nb_tbs, code_gen_max_blocks);
4478 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
4479 nb_tbs ? target_code_size / nb_tbs : 0,
4480 max_target_code_size);
4481 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
4482 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
4483 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
4484 cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
4485 cross_page,
4486 nb_tbs ? (cross_page * 100) / nb_tbs : 0);
4487 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
4488 direct_jmp_count,
4489 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
4490 direct_jmp2_count,
4491 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
4492 cpu_fprintf(f, "\nStatistics:\n");
4493 cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
4494 cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
4495 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
4496 tcg_dump_info(f, cpu_fprintf);
4497 }
4498
4499 #define MMUSUFFIX _cmmu
4500 #define GETPC() NULL
4501 #define env cpu_single_env
4502 #define SOFTMMU_CODE_ACCESS
4503
4504 #define SHIFT 0
4505 #include "softmmu_template.h"
4506
4507 #define SHIFT 1
4508 #include "softmmu_template.h"
4509
4510 #define SHIFT 2
4511 #include "softmmu_template.h"
4512
4513 #define SHIFT 3
4514 #include "softmmu_template.h"
4515
4516 #undef env
4517
4518 #endif
QEmu