?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge remote-tracking branch 'agraf/xen-next' into staging
[qemu.git] / exec.c
bloba6df2d61398ee33339fac4abac3cbfad36d0006b
1 /*
2 * virtual page mapping and translated block handling
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "exec-all.h"
30 #include "tcg.h"
31 #include "hw/hw.h"
32 #include "hw/qdev.h"
33 #include "osdep.h"
34 #include "kvm.h"
35 #include "hw/xen.h"
36 #include "qemu-timer.h"
37 #if defined(CONFIG_USER_ONLY)
38 #include <qemu.h>
39 #include <signal.h>
40 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
41 #include <sys/param.h>
42 #if __FreeBSD_version >= 700104
43 #define HAVE_KINFO_GETVMMAP
44 #define sigqueue sigqueue_freebsd /* avoid redefinition */
45 #include <sys/time.h>
46 #include <sys/proc.h>
47 #include <machine/profile.h>
48 #define _KERNEL
49 #include <sys/user.h>
50 #undef _KERNEL
51 #undef sigqueue
52 #include <libutil.h>
53 #endif
54 #endif
55 #else /* !CONFIG_USER_ONLY */
56 #include "xen-mapcache.h"
57 #endif
58
59 //#define DEBUG_TB_INVALIDATE
60 //#define DEBUG_FLUSH
61 //#define DEBUG_TLB
62 //#define DEBUG_UNASSIGNED
63
64 /* make various TB consistency checks */
65 //#define DEBUG_TB_CHECK
66 //#define DEBUG_TLB_CHECK
67
68 //#define DEBUG_IOPORT
69 //#define DEBUG_SUBPAGE
70
71 #if !defined(CONFIG_USER_ONLY)
72 /* TB consistency checks only implemented for usermode emulation. */
73 #undef DEBUG_TB_CHECK
74 #endif
75
76 #define SMC_BITMAP_USE_THRESHOLD 10
77
78 static TranslationBlock *tbs;
79 static int code_gen_max_blocks;
80 TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
81 static int nb_tbs;
82 /* any access to the tbs or the page table must use this lock */
83 spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
84
85 #if defined(__arm__) || defined(__sparc_v9__)
86 /* The prologue must be reachable with a direct jump. ARM and Sparc64
87 have limited branch ranges (possibly also PPC) so place it in a
88 section close to code segment. */
89 #define code_gen_section \
90 __attribute__((__section__(".gen_code"))) \
91 __attribute__((aligned (32)))
92 #elif defined(_WIN32)
93 /* Maximum alignment for Win32 is 16. */
94 #define code_gen_section \
95 __attribute__((aligned (16)))
96 #else
97 #define code_gen_section \
98 __attribute__((aligned (32)))
99 #endif
100
101 uint8_t code_gen_prologue[1024] code_gen_section;
102 static uint8_t *code_gen_buffer;
103 static unsigned long code_gen_buffer_size;
104 /* threshold to flush the translated code buffer */
105 static unsigned long code_gen_buffer_max_size;
106 static uint8_t *code_gen_ptr;
107
108 #if !defined(CONFIG_USER_ONLY)
109 int phys_ram_fd;
110 static int in_migration;
111
112 RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list) };
113 #endif
114
115 CPUState *first_cpu;
116 /* current CPU in the current thread. It is only valid inside
117 cpu_exec() */
118 CPUState *cpu_single_env;
119 /* 0 = Do not count executed instructions.
120 1 = Precise instruction counting.
121 2 = Adaptive rate instruction counting. */
122 int use_icount = 0;
123 /* Current instruction counter. While executing translated code this may
124 include some instructions that have not yet been executed. */
125 int64_t qemu_icount;
126
127 typedef struct PageDesc {
128 /* list of TBs intersecting this ram page */
129 TranslationBlock *first_tb;
130 /* in order to optimize self modifying code, we count the number
131 of lookups we do to a given page to use a bitmap */
132 unsigned int code_write_count;
133 uint8_t *code_bitmap;
134 #if defined(CONFIG_USER_ONLY)
135 unsigned long flags;
136 #endif
137 } PageDesc;
138
139 /* In system mode we want L1_MAP to be based on ram offsets,
140 while in user mode we want it to be based on virtual addresses. */
141 #if !defined(CONFIG_USER_ONLY)
142 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
143 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
144 #else
145 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
146 #endif
147 #else
148 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
149 #endif
150
151 /* Size of the L2 (and L3, etc) page tables. */
152 #define L2_BITS 10
153 #define L2_SIZE (1 << L2_BITS)
154
155 /* The bits remaining after N lower levels of page tables. */
156 #define P_L1_BITS_REM \
157 ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
158 #define V_L1_BITS_REM \
159 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
160
161 /* Size of the L1 page table. Avoid silly small sizes. */
162 #if P_L1_BITS_REM < 4
163 #define P_L1_BITS (P_L1_BITS_REM + L2_BITS)
164 #else
165 #define P_L1_BITS P_L1_BITS_REM
166 #endif
167
168 #if V_L1_BITS_REM < 4
169 #define V_L1_BITS (V_L1_BITS_REM + L2_BITS)
170 #else
171 #define V_L1_BITS V_L1_BITS_REM
172 #endif
173
174 #define P_L1_SIZE ((target_phys_addr_t)1 << P_L1_BITS)
175 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
176
177 #define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
178 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
179
180 unsigned long qemu_real_host_page_size;
181 unsigned long qemu_host_page_bits;
182 unsigned long qemu_host_page_size;
183 unsigned long qemu_host_page_mask;
184
185 /* This is a multi-level map on the virtual address space.
186 The bottom level has pointers to PageDesc. */
187 static void *l1_map[V_L1_SIZE];
188
189 #if !defined(CONFIG_USER_ONLY)
190 typedef struct PhysPageDesc {
191 /* offset in host memory of the page + io_index in the low bits */
192 ram_addr_t phys_offset;
193 ram_addr_t region_offset;
194 } PhysPageDesc;
195
196 /* This is a multi-level map on the physical address space.
197 The bottom level has pointers to PhysPageDesc. */
198 static void *l1_phys_map[P_L1_SIZE];
199
200 static void io_mem_init(void);
201
202 /* io memory support */
203 CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
204 CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
205 void *io_mem_opaque[IO_MEM_NB_ENTRIES];
206 static char io_mem_used[IO_MEM_NB_ENTRIES];
207 static int io_mem_watch;
208 #endif
209
210 /* log support */
211 #ifdef WIN32
212 static const char *logfilename = "qemu.log";
213 #else
214 static const char *logfilename = "/tmp/qemu.log";
215 #endif
216 FILE *logfile;
217 int loglevel;
218 static int log_append = 0;
219
220 /* statistics */
221 #if !defined(CONFIG_USER_ONLY)
222 static int tlb_flush_count;
223 #endif
224 static int tb_flush_count;
225 static int tb_phys_invalidate_count;
226
227 #ifdef _WIN32
228 static void map_exec(void *addr, long size)
229 {
230 DWORD old_protect;
231 VirtualProtect(addr, size,
232 PAGE_EXECUTE_READWRITE, &old_protect);
233
234 }
235 #else
236 static void map_exec(void *addr, long size)
237 {
238 unsigned long start, end, page_size;
239
240 page_size = getpagesize();
241 start = (unsigned long)addr;
242 start &= ~(page_size - 1);
243
244 end = (unsigned long)addr + size;
245 end += page_size - 1;
246 end &= ~(page_size - 1);
247
248 mprotect((void *)start, end - start,
249 PROT_READ | PROT_WRITE | PROT_EXEC);
250 }
251 #endif
252
253 static void page_init(void)
254 {
255 /* NOTE: we can always suppose that qemu_host_page_size >=
256 TARGET_PAGE_SIZE */
257 #ifdef _WIN32
258 {
259 SYSTEM_INFO system_info;
260
261 GetSystemInfo(&system_info);
262 qemu_real_host_page_size = system_info.dwPageSize;
263 }
264 #else
265 qemu_real_host_page_size = getpagesize();
266 #endif
267 if (qemu_host_page_size == 0)
268 qemu_host_page_size = qemu_real_host_page_size;
269 if (qemu_host_page_size < TARGET_PAGE_SIZE)
270 qemu_host_page_size = TARGET_PAGE_SIZE;
271 qemu_host_page_bits = 0;
272 while ((1 << qemu_host_page_bits) < qemu_host_page_size)
273 qemu_host_page_bits++;
274 qemu_host_page_mask = ~(qemu_host_page_size - 1);
275
276 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
277 {
278 #ifdef HAVE_KINFO_GETVMMAP
279 struct kinfo_vmentry *freep;
280 int i, cnt;
281
282 freep = kinfo_getvmmap(getpid(), &cnt);
283 if (freep) {
284 mmap_lock();
285 for (i = 0; i < cnt; i++) {
286 unsigned long startaddr, endaddr;
287
288 startaddr = freep[i].kve_start;
289 endaddr = freep[i].kve_end;
290 if (h2g_valid(startaddr)) {
291 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
292
293 if (h2g_valid(endaddr)) {
294 endaddr = h2g(endaddr);
295 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
296 } else {
297 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
298 endaddr = ~0ul;
299 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
300 #endif
301 }
302 }
303 }
304 free(freep);
305 mmap_unlock();
306 }
307 #else
308 FILE *f;
309
310 last_brk = (unsigned long)sbrk(0);
311
312 f = fopen("/compat/linux/proc/self/maps", "r");
313 if (f) {
314 mmap_lock();
315
316 do {
317 unsigned long startaddr, endaddr;
318 int n;
319
320 n = fscanf (f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
321
322 if (n == 2 && h2g_valid(startaddr)) {
323 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
324
325 if (h2g_valid(endaddr)) {
326 endaddr = h2g(endaddr);
327 } else {
328 endaddr = ~0ul;
329 }
330 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
331 }
332 } while (!feof(f));
333
334 fclose(f);
335 mmap_unlock();
336 }
337 #endif
338 }
339 #endif
340 }
341
342 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
343 {
344 PageDesc *pd;
345 void **lp;
346 int i;
347
348 #if defined(CONFIG_USER_ONLY)
349 /* We can't use qemu_malloc because it may recurse into a locked mutex. */
350 # define ALLOC(P, SIZE) \
351 do { \
352 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
353 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
354 } while (0)
355 #else
356 # define ALLOC(P, SIZE) \
357 do { P = qemu_mallocz(SIZE); } while (0)
358 #endif
359
360 /* Level 1. Always allocated. */
361 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
362
363 /* Level 2..N-1. */
364 for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
365 void **p = *lp;
366
367 if (p == NULL) {
368 if (!alloc) {
369 return NULL;
370 }
371 ALLOC(p, sizeof(void *) * L2_SIZE);
372 *lp = p;
373 }
374
375 lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
376 }
377
378 pd = *lp;
379 if (pd == NULL) {
380 if (!alloc) {
381 return NULL;
382 }
383 ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
384 *lp = pd;
385 }
386
387 #undef ALLOC
388
389 return pd + (index & (L2_SIZE - 1));
390 }
391
392 static inline PageDesc *page_find(tb_page_addr_t index)
393 {
394 return page_find_alloc(index, 0);
395 }
396
397 #if !defined(CONFIG_USER_ONLY)
398 static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
399 {
400 PhysPageDesc *pd;
401 void **lp;
402 int i;
403
404 /* Level 1. Always allocated. */
405 lp = l1_phys_map + ((index >> P_L1_SHIFT) & (P_L1_SIZE - 1));
406
407 /* Level 2..N-1. */
408 for (i = P_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
409 void **p = *lp;
410 if (p == NULL) {
411 if (!alloc) {
412 return NULL;
413 }
414 *lp = p = qemu_mallocz(sizeof(void *) * L2_SIZE);
415 }
416 lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
417 }
418
419 pd = *lp;
420 if (pd == NULL) {
421 int i;
422
423 if (!alloc) {
424 return NULL;
425 }
426
427 *lp = pd = qemu_malloc(sizeof(PhysPageDesc) * L2_SIZE);
428
429 for (i = 0; i < L2_SIZE; i++) {
430 pd[i].phys_offset = IO_MEM_UNASSIGNED;
431 pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
432 }
433 }
434
435 return pd + (index & (L2_SIZE - 1));
436 }
437
438 static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
439 {
440 return phys_page_find_alloc(index, 0);
441 }
442
443 static void tlb_protect_code(ram_addr_t ram_addr);
444 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
445 target_ulong vaddr);
446 #define mmap_lock() do { } while(0)
447 #define mmap_unlock() do { } while(0)
448 #endif
449
450 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
451
452 #if defined(CONFIG_USER_ONLY)
453 /* Currently it is not recommended to allocate big chunks of data in
454 user mode. It will change when a dedicated libc will be used */
455 #define USE_STATIC_CODE_GEN_BUFFER
456 #endif
457
458 #ifdef USE_STATIC_CODE_GEN_BUFFER
459 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
460 __attribute__((aligned (CODE_GEN_ALIGN)));
461 #endif
462
463 static void code_gen_alloc(unsigned long tb_size)
464 {
465 #ifdef USE_STATIC_CODE_GEN_BUFFER
466 code_gen_buffer = static_code_gen_buffer;
467 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
468 map_exec(code_gen_buffer, code_gen_buffer_size);
469 #else
470 code_gen_buffer_size = tb_size;
471 if (code_gen_buffer_size == 0) {
472 #if defined(CONFIG_USER_ONLY)
473 /* in user mode, phys_ram_size is not meaningful */
474 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
475 #else
476 /* XXX: needs adjustments */
477 code_gen_buffer_size = (unsigned long)(ram_size / 4);
478 #endif
479 }
480 if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
481 code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
482 /* The code gen buffer location may have constraints depending on
483 the host cpu and OS */
484 #if defined(__linux__)
485 {
486 int flags;
487 void *start = NULL;
488
489 flags = MAP_PRIVATE | MAP_ANONYMOUS;
490 #if defined(__x86_64__)
491 flags |= MAP_32BIT;
492 /* Cannot map more than that */
493 if (code_gen_buffer_size > (800 * 1024 * 1024))
494 code_gen_buffer_size = (800 * 1024 * 1024);
495 #elif defined(__sparc_v9__)
496 // Map the buffer below 2G, so we can use direct calls and branches
497 flags |= MAP_FIXED;
498 start = (void *) 0x60000000UL;
499 if (code_gen_buffer_size > (512 * 1024 * 1024))
500 code_gen_buffer_size = (512 * 1024 * 1024);
501 #elif defined(__arm__)
502 /* Map the buffer below 32M, so we can use direct calls and branches */
503 flags |= MAP_FIXED;
504 start = (void *) 0x01000000UL;
505 if (code_gen_buffer_size > 16 * 1024 * 1024)
506 code_gen_buffer_size = 16 * 1024 * 1024;
507 #elif defined(__s390x__)
508 /* Map the buffer so that we can use direct calls and branches. */
509 /* We have a +- 4GB range on the branches; leave some slop. */
510 if (code_gen_buffer_size > (3ul * 1024 * 1024 * 1024)) {
511 code_gen_buffer_size = 3ul * 1024 * 1024 * 1024;
512 }
513 start = (void *)0x90000000UL;
514 #endif
515 code_gen_buffer = mmap(start, code_gen_buffer_size,
516 PROT_WRITE | PROT_READ | PROT_EXEC,
517 flags, -1, 0);
518 if (code_gen_buffer == MAP_FAILED) {
519 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
520 exit(1);
521 }
522 }
523 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
524 || defined(__DragonFly__) || defined(__OpenBSD__)
525 {
526 int flags;
527 void *addr = NULL;
528 flags = MAP_PRIVATE | MAP_ANONYMOUS;
529 #if defined(__x86_64__)
530 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
531 * 0x40000000 is free */
532 flags |= MAP_FIXED;
533 addr = (void *)0x40000000;
534 /* Cannot map more than that */
535 if (code_gen_buffer_size > (800 * 1024 * 1024))
536 code_gen_buffer_size = (800 * 1024 * 1024);
537 #elif defined(__sparc_v9__)
538 // Map the buffer below 2G, so we can use direct calls and branches
539 flags |= MAP_FIXED;
540 addr = (void *) 0x60000000UL;
541 if (code_gen_buffer_size > (512 * 1024 * 1024)) {
542 code_gen_buffer_size = (512 * 1024 * 1024);
543 }
544 #endif
545 code_gen_buffer = mmap(addr, code_gen_buffer_size,
546 PROT_WRITE | PROT_READ | PROT_EXEC,
547 flags, -1, 0);
548 if (code_gen_buffer == MAP_FAILED) {
549 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
550 exit(1);
551 }
552 }
553 #else
554 code_gen_buffer = qemu_malloc(code_gen_buffer_size);
555 map_exec(code_gen_buffer, code_gen_buffer_size);
556 #endif
557 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
558 map_exec(code_gen_prologue, sizeof(code_gen_prologue));
559 code_gen_buffer_max_size = code_gen_buffer_size -
560 (TCG_MAX_OP_SIZE * OPC_MAX_SIZE);
561 code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
562 tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
563 }
564
565 /* Must be called before using the QEMU cpus. 'tb_size' is the size
566 (in bytes) allocated to the translation buffer. Zero means default
567 size. */
568 void cpu_exec_init_all(unsigned long tb_size)
569 {
570 cpu_gen_init();
571 code_gen_alloc(tb_size);
572 code_gen_ptr = code_gen_buffer;
573 page_init();
574 #if !defined(CONFIG_USER_ONLY)
575 io_mem_init();
576 #endif
577 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
578 /* There's no guest base to take into account, so go ahead and
579 initialize the prologue now. */
580 tcg_prologue_init(&tcg_ctx);
581 #endif
582 }
583
584 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
585
586 static int cpu_common_post_load(void *opaque, int version_id)
587 {
588 CPUState *env = opaque;
589
590 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
591 version_id is increased. */
592 env->interrupt_request &= ~0x01;
593 tlb_flush(env, 1);
594
595 return 0;
596 }
597
598 static const VMStateDescription vmstate_cpu_common = {
599 .name = "cpu_common",
600 .version_id = 1,
601 .minimum_version_id = 1,
602 .minimum_version_id_old = 1,
603 .post_load = cpu_common_post_load,
604 .fields = (VMStateField []) {
605 VMSTATE_UINT32(halted, CPUState),
606 VMSTATE_UINT32(interrupt_request, CPUState),
607 VMSTATE_END_OF_LIST()
608 }
609 };
610 #endif
611
612 CPUState *qemu_get_cpu(int cpu)
613 {
614 CPUState *env = first_cpu;
615
616 while (env) {
617 if (env->cpu_index == cpu)
618 break;
619 env = env->next_cpu;
620 }
621
622 return env;
623 }
624
625 void cpu_exec_init(CPUState *env)
626 {
627 CPUState **penv;
628 int cpu_index;
629
630 #if defined(CONFIG_USER_ONLY)
631 cpu_list_lock();
632 #endif
633 env->next_cpu = NULL;
634 penv = &first_cpu;
635 cpu_index = 0;
636 while (*penv != NULL) {
637 penv = &(*penv)->next_cpu;
638 cpu_index++;
639 }
640 env->cpu_index = cpu_index;
641 env->numa_node = 0;
642 QTAILQ_INIT(&env->breakpoints);
643 QTAILQ_INIT(&env->watchpoints);
644 #ifndef CONFIG_USER_ONLY
645 env->thread_id = qemu_get_thread_id();
646 #endif
647 *penv = env;
648 #if defined(CONFIG_USER_ONLY)
649 cpu_list_unlock();
650 #endif
651 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
652 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, env);
653 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
654 cpu_save, cpu_load, env);
655 #endif
656 }
657
658 /* Allocate a new translation block. Flush the translation buffer if
659 too many translation blocks or too much generated code. */
660 static TranslationBlock *tb_alloc(target_ulong pc)
661 {
662 TranslationBlock *tb;
663
664 if (nb_tbs >= code_gen_max_blocks ||
665 (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
666 return NULL;
667 tb = &tbs[nb_tbs++];
668 tb->pc = pc;
669 tb->cflags = 0;
670 return tb;
671 }
672
673 void tb_free(TranslationBlock *tb)
674 {
675 /* In practice this is mostly used for single use temporary TB
676 Ignore the hard cases and just back up if this TB happens to
677 be the last one generated. */
678 if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
679 code_gen_ptr = tb->tc_ptr;
680 nb_tbs--;
681 }
682 }
683
684 static inline void invalidate_page_bitmap(PageDesc *p)
685 {
686 if (p->code_bitmap) {
687 qemu_free(p->code_bitmap);
688 p->code_bitmap = NULL;
689 }
690 p->code_write_count = 0;
691 }
692
693 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
694
695 static void page_flush_tb_1 (int level, void **lp)
696 {
697 int i;
698
699 if (*lp == NULL) {
700 return;
701 }
702 if (level == 0) {
703 PageDesc *pd = *lp;
704 for (i = 0; i < L2_SIZE; ++i) {
705 pd[i].first_tb = NULL;
706 invalidate_page_bitmap(pd + i);
707 }
708 } else {
709 void **pp = *lp;
710 for (i = 0; i < L2_SIZE; ++i) {
711 page_flush_tb_1 (level - 1, pp + i);
712 }
713 }
714 }
715
716 static void page_flush_tb(void)
717 {
718 int i;
719 for (i = 0; i < V_L1_SIZE; i++) {
720 page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
721 }
722 }
723
724 /* flush all the translation blocks */
725 /* XXX: tb_flush is currently not thread safe */
726 void tb_flush(CPUState *env1)
727 {
728 CPUState *env;
729 #if defined(DEBUG_FLUSH)
730 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
731 (unsigned long)(code_gen_ptr - code_gen_buffer),
732 nb_tbs, nb_tbs > 0 ?
733 ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
734 #endif
735 if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
736 cpu_abort(env1, "Internal error: code buffer overflow\n");
737
738 nb_tbs = 0;
739
740 for(env = first_cpu; env != NULL; env = env->next_cpu) {
741 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
742 }
743
744 memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
745 page_flush_tb();
746
747 code_gen_ptr = code_gen_buffer;
748 /* XXX: flush processor icache at this point if cache flush is
749 expensive */
750 tb_flush_count++;
751 }
752
753 #ifdef DEBUG_TB_CHECK
754
755 static void tb_invalidate_check(target_ulong address)
756 {
757 TranslationBlock *tb;
758 int i;
759 address &= TARGET_PAGE_MASK;
760 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
761 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
762 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
763 address >= tb->pc + tb->size)) {
764 printf("ERROR invalidate: address=" TARGET_FMT_lx
765 " PC=%08lx size=%04x\n",
766 address, (long)tb->pc, tb->size);
767 }
768 }
769 }
770 }
771
772 /* verify that all the pages have correct rights for code */
773 static void tb_page_check(void)
774 {
775 TranslationBlock *tb;
776 int i, flags1, flags2;
777
778 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
779 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
780 flags1 = page_get_flags(tb->pc);
781 flags2 = page_get_flags(tb->pc + tb->size - 1);
782 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
783 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
784 (long)tb->pc, tb->size, flags1, flags2);
785 }
786 }
787 }
788 }
789
790 #endif
791
792 /* invalidate one TB */
793 static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
794 int next_offset)
795 {
796 TranslationBlock *tb1;
797 for(;;) {
798 tb1 = *ptb;
799 if (tb1 == tb) {
800 *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
801 break;
802 }
803 ptb = (TranslationBlock **)((char *)tb1 + next_offset);
804 }
805 }
806
807 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
808 {
809 TranslationBlock *tb1;
810 unsigned int n1;
811
812 for(;;) {
813 tb1 = *ptb;
814 n1 = (long)tb1 & 3;
815 tb1 = (TranslationBlock *)((long)tb1 & ~3);
816 if (tb1 == tb) {
817 *ptb = tb1->page_next[n1];
818 break;
819 }
820 ptb = &tb1->page_next[n1];
821 }
822 }
823
824 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
825 {
826 TranslationBlock *tb1, **ptb;
827 unsigned int n1;
828
829 ptb = &tb->jmp_next[n];
830 tb1 = *ptb;
831 if (tb1) {
832 /* find tb(n) in circular list */
833 for(;;) {
834 tb1 = *ptb;
835 n1 = (long)tb1 & 3;
836 tb1 = (TranslationBlock *)((long)tb1 & ~3);
837 if (n1 == n && tb1 == tb)
838 break;
839 if (n1 == 2) {
840 ptb = &tb1->jmp_first;
841 } else {
842 ptb = &tb1->jmp_next[n1];
843 }
844 }
845 /* now we can suppress tb(n) from the list */
846 *ptb = tb->jmp_next[n];
847
848 tb->jmp_next[n] = NULL;
849 }
850 }
851
852 /* reset the jump entry 'n' of a TB so that it is not chained to
853 another TB */
854 static inline void tb_reset_jump(TranslationBlock *tb, int n)
855 {
856 tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
857 }
858
859 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
860 {
861 CPUState *env;
862 PageDesc *p;
863 unsigned int h, n1;
864 tb_page_addr_t phys_pc;
865 TranslationBlock *tb1, *tb2;
866
867 /* remove the TB from the hash list */
868 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
869 h = tb_phys_hash_func(phys_pc);
870 tb_remove(&tb_phys_hash[h], tb,
871 offsetof(TranslationBlock, phys_hash_next));
872
873 /* remove the TB from the page list */
874 if (tb->page_addr[0] != page_addr) {
875 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
876 tb_page_remove(&p->first_tb, tb);
877 invalidate_page_bitmap(p);
878 }
879 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
880 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
881 tb_page_remove(&p->first_tb, tb);
882 invalidate_page_bitmap(p);
883 }
884
885 tb_invalidated_flag = 1;
886
887 /* remove the TB from the hash list */
888 h = tb_jmp_cache_hash_func(tb->pc);
889 for(env = first_cpu; env != NULL; env = env->next_cpu) {
890 if (env->tb_jmp_cache[h] == tb)
891 env->tb_jmp_cache[h] = NULL;
892 }
893
894 /* suppress this TB from the two jump lists */
895 tb_jmp_remove(tb, 0);
896 tb_jmp_remove(tb, 1);
897
898 /* suppress any remaining jumps to this TB */
899 tb1 = tb->jmp_first;
900 for(;;) {
901 n1 = (long)tb1 & 3;
902 if (n1 == 2)
903 break;
904 tb1 = (TranslationBlock *)((long)tb1 & ~3);
905 tb2 = tb1->jmp_next[n1];
906 tb_reset_jump(tb1, n1);
907 tb1->jmp_next[n1] = NULL;
908 tb1 = tb2;
909 }
910 tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
911
912 tb_phys_invalidate_count++;
913 }
914
915 static inline void set_bits(uint8_t *tab, int start, int len)
916 {
917 int end, mask, end1;
918
919 end = start + len;
920 tab += start >> 3;
921 mask = 0xff << (start & 7);
922 if ((start & ~7) == (end & ~7)) {
923 if (start < end) {
924 mask &= ~(0xff << (end & 7));
925 *tab |= mask;
926 }
927 } else {
928 *tab++ |= mask;
929 start = (start + 8) & ~7;
930 end1 = end & ~7;
931 while (start < end1) {
932 *tab++ = 0xff;
933 start += 8;
934 }
935 if (start < end) {
936 mask = ~(0xff << (end & 7));
937 *tab |= mask;
938 }
939 }
940 }
941
942 static void build_page_bitmap(PageDesc *p)
943 {
944 int n, tb_start, tb_end;
945 TranslationBlock *tb;
946
947 p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
948
949 tb = p->first_tb;
950 while (tb != NULL) {
951 n = (long)tb & 3;
952 tb = (TranslationBlock *)((long)tb & ~3);
953 /* NOTE: this is subtle as a TB may span two physical pages */
954 if (n == 0) {
955 /* NOTE: tb_end may be after the end of the page, but
956 it is not a problem */
957 tb_start = tb->pc & ~TARGET_PAGE_MASK;
958 tb_end = tb_start + tb->size;
959 if (tb_end > TARGET_PAGE_SIZE)
960 tb_end = TARGET_PAGE_SIZE;
961 } else {
962 tb_start = 0;
963 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
964 }
965 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
966 tb = tb->page_next[n];
967 }
968 }
969
970 TranslationBlock *tb_gen_code(CPUState *env,
971 target_ulong pc, target_ulong cs_base,
972 int flags, int cflags)
973 {
974 TranslationBlock *tb;
975 uint8_t *tc_ptr;
976 tb_page_addr_t phys_pc, phys_page2;
977 target_ulong virt_page2;
978 int code_gen_size;
979
980 phys_pc = get_page_addr_code(env, pc);
981 tb = tb_alloc(pc);
982 if (!tb) {
983 /* flush must be done */
984 tb_flush(env);
985 /* cannot fail at this point */
986 tb = tb_alloc(pc);
987 /* Don't forget to invalidate previous TB info. */
988 tb_invalidated_flag = 1;
989 }
990 tc_ptr = code_gen_ptr;
991 tb->tc_ptr = tc_ptr;
992 tb->cs_base = cs_base;
993 tb->flags = flags;
994 tb->cflags = cflags;
995 cpu_gen_code(env, tb, &code_gen_size);
996 code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
997
998 /* check next page if needed */
999 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
1000 phys_page2 = -1;
1001 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
1002 phys_page2 = get_page_addr_code(env, virt_page2);
1003 }
1004 tb_link_page(tb, phys_pc, phys_page2);
1005 return tb;
1006 }
1007
1008 /* invalidate all TBs which intersect with the target physical page
1009 starting in range [start;end[. NOTE: start and end must refer to
1010 the same physical page. 'is_cpu_write_access' should be true if called
1011 from a real cpu write access: the virtual CPU will exit the current
1012 TB if code is modified inside this TB. */
1013 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
1014 int is_cpu_write_access)
1015 {
1016 TranslationBlock *tb, *tb_next, *saved_tb;
1017 CPUState *env = cpu_single_env;
1018 tb_page_addr_t tb_start, tb_end;
1019 PageDesc *p;
1020 int n;
1021 #ifdef TARGET_HAS_PRECISE_SMC
1022 int current_tb_not_found = is_cpu_write_access;
1023 TranslationBlock *current_tb = NULL;
1024 int current_tb_modified = 0;
1025 target_ulong current_pc = 0;
1026 target_ulong current_cs_base = 0;
1027 int current_flags = 0;
1028 #endif /* TARGET_HAS_PRECISE_SMC */
1029
1030 p = page_find(start >> TARGET_PAGE_BITS);
1031 if (!p)
1032 return;
1033 if (!p->code_bitmap &&
1034 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
1035 is_cpu_write_access) {
1036 /* build code bitmap */
1037 build_page_bitmap(p);
1038 }
1039
1040 /* we remove all the TBs in the range [start, end[ */
1041 /* XXX: see if in some cases it could be faster to invalidate all the code */
1042 tb = p->first_tb;
1043 while (tb != NULL) {
1044 n = (long)tb & 3;
1045 tb = (TranslationBlock *)((long)tb & ~3);
1046 tb_next = tb->page_next[n];
1047 /* NOTE: this is subtle as a TB may span two physical pages */
1048 if (n == 0) {
1049 /* NOTE: tb_end may be after the end of the page, but
1050 it is not a problem */
1051 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1052 tb_end = tb_start + tb->size;
1053 } else {
1054 tb_start = tb->page_addr[1];
1055 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1056 }
1057 if (!(tb_end <= start || tb_start >= end)) {
1058 #ifdef TARGET_HAS_PRECISE_SMC
1059 if (current_tb_not_found) {
1060 current_tb_not_found = 0;
1061 current_tb = NULL;
1062 if (env->mem_io_pc) {
1063 /* now we have a real cpu fault */
1064 current_tb = tb_find_pc(env->mem_io_pc);
1065 }
1066 }
1067 if (current_tb == tb &&
1068 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1069 /* If we are modifying the current TB, we must stop
1070 its execution. We could be more precise by checking
1071 that the modification is after the current PC, but it
1072 would require a specialized function to partially
1073 restore the CPU state */
1074
1075 current_tb_modified = 1;
1076 cpu_restore_state(current_tb, env, env->mem_io_pc);
1077 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1078 &current_flags);
1079 }
1080 #endif /* TARGET_HAS_PRECISE_SMC */
1081 /* we need to do that to handle the case where a signal
1082 occurs while doing tb_phys_invalidate() */
1083 saved_tb = NULL;
1084 if (env) {
1085 saved_tb = env->current_tb;
1086 env->current_tb = NULL;
1087 }
1088 tb_phys_invalidate(tb, -1);
1089 if (env) {
1090 env->current_tb = saved_tb;
1091 if (env->interrupt_request && env->current_tb)
1092 cpu_interrupt(env, env->interrupt_request);
1093 }
1094 }
1095 tb = tb_next;
1096 }
1097 #if !defined(CONFIG_USER_ONLY)
1098 /* if no code remaining, no need to continue to use slow writes */
1099 if (!p->first_tb) {
1100 invalidate_page_bitmap(p);
1101 if (is_cpu_write_access) {
1102 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1103 }
1104 }
1105 #endif
1106 #ifdef TARGET_HAS_PRECISE_SMC
1107 if (current_tb_modified) {
1108 /* we generate a block containing just the instruction
1109 modifying the memory. It will ensure that it cannot modify
1110 itself */
1111 env->current_tb = NULL;
1112 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1113 cpu_resume_from_signal(env, NULL);
1114 }
1115 #endif
1116 }
1117
1118 /* len must be <= 8 and start must be a multiple of len */
1119 static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1120 {
1121 PageDesc *p;
1122 int offset, b;
1123 #if 0
1124 if (1) {
1125 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1126 cpu_single_env->mem_io_vaddr, len,
1127 cpu_single_env->eip,
1128 cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1129 }
1130 #endif
1131 p = page_find(start >> TARGET_PAGE_BITS);
1132 if (!p)
1133 return;
1134 if (p->code_bitmap) {
1135 offset = start & ~TARGET_PAGE_MASK;
1136 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1137 if (b & ((1 << len) - 1))
1138 goto do_invalidate;
1139 } else {
1140 do_invalidate:
1141 tb_invalidate_phys_page_range(start, start + len, 1);
1142 }
1143 }
1144
1145 #if !defined(CONFIG_SOFTMMU)
1146 static void tb_invalidate_phys_page(tb_page_addr_t addr,
1147 unsigned long pc, void *puc)
1148 {
1149 TranslationBlock *tb;
1150 PageDesc *p;
1151 int n;
1152 #ifdef TARGET_HAS_PRECISE_SMC
1153 TranslationBlock *current_tb = NULL;
1154 CPUState *env = cpu_single_env;
1155 int current_tb_modified = 0;
1156 target_ulong current_pc = 0;
1157 target_ulong current_cs_base = 0;
1158 int current_flags = 0;
1159 #endif
1160
1161 addr &= TARGET_PAGE_MASK;
1162 p = page_find(addr >> TARGET_PAGE_BITS);
1163 if (!p)
1164 return;
1165 tb = p->first_tb;
1166 #ifdef TARGET_HAS_PRECISE_SMC
1167 if (tb && pc != 0) {
1168 current_tb = tb_find_pc(pc);
1169 }
1170 #endif
1171 while (tb != NULL) {
1172 n = (long)tb & 3;
1173 tb = (TranslationBlock *)((long)tb & ~3);
1174 #ifdef TARGET_HAS_PRECISE_SMC
1175 if (current_tb == tb &&
1176 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1177 /* If we are modifying the current TB, we must stop
1178 its execution. We could be more precise by checking
1179 that the modification is after the current PC, but it
1180 would require a specialized function to partially
1181 restore the CPU state */
1182
1183 current_tb_modified = 1;
1184 cpu_restore_state(current_tb, env, pc);
1185 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1186 &current_flags);
1187 }
1188 #endif /* TARGET_HAS_PRECISE_SMC */
1189 tb_phys_invalidate(tb, addr);
1190 tb = tb->page_next[n];
1191 }
1192 p->first_tb = NULL;
1193 #ifdef TARGET_HAS_PRECISE_SMC
1194 if (current_tb_modified) {
1195 /* we generate a block containing just the instruction
1196 modifying the memory. It will ensure that it cannot modify
1197 itself */
1198 env->current_tb = NULL;
1199 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1200 cpu_resume_from_signal(env, puc);
1201 }
1202 #endif
1203 }
1204 #endif
1205
1206 /* add the tb in the target page and protect it if necessary */
1207 static inline void tb_alloc_page(TranslationBlock *tb,
1208 unsigned int n, tb_page_addr_t page_addr)
1209 {
1210 PageDesc *p;
1211 TranslationBlock *last_first_tb;
1212
1213 tb->page_addr[n] = page_addr;
1214 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1215 tb->page_next[n] = p->first_tb;
1216 last_first_tb = p->first_tb;
1217 p->first_tb = (TranslationBlock *)((long)tb | n);
1218 invalidate_page_bitmap(p);
1219
1220 #if defined(TARGET_HAS_SMC) || 1
1221
1222 #if defined(CONFIG_USER_ONLY)
1223 if (p->flags & PAGE_WRITE) {
1224 target_ulong addr;
1225 PageDesc *p2;
1226 int prot;
1227
1228 /* force the host page as non writable (writes will have a
1229 page fault + mprotect overhead) */
1230 page_addr &= qemu_host_page_mask;
1231 prot = 0;
1232 for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1233 addr += TARGET_PAGE_SIZE) {
1234
1235 p2 = page_find (addr >> TARGET_PAGE_BITS);
1236 if (!p2)
1237 continue;
1238 prot |= p2->flags;
1239 p2->flags &= ~PAGE_WRITE;
1240 }
1241 mprotect(g2h(page_addr), qemu_host_page_size,
1242 (prot & PAGE_BITS) & ~PAGE_WRITE);
1243 #ifdef DEBUG_TB_INVALIDATE
1244 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1245 page_addr);
1246 #endif
1247 }
1248 #else
1249 /* if some code is already present, then the pages are already
1250 protected. So we handle the case where only the first TB is
1251 allocated in a physical page */
1252 if (!last_first_tb) {
1253 tlb_protect_code(page_addr);
1254 }
1255 #endif
1256
1257 #endif /* TARGET_HAS_SMC */
1258 }
1259
1260 /* add a new TB and link it to the physical page tables. phys_page2 is
1261 (-1) to indicate that only one page contains the TB. */
1262 void tb_link_page(TranslationBlock *tb,
1263 tb_page_addr_t phys_pc, tb_page_addr_t phys_page2)
1264 {
1265 unsigned int h;
1266 TranslationBlock **ptb;
1267
1268 /* Grab the mmap lock to stop another thread invalidating this TB
1269 before we are done. */
1270 mmap_lock();
1271 /* add in the physical hash table */
1272 h = tb_phys_hash_func(phys_pc);
1273 ptb = &tb_phys_hash[h];
1274 tb->phys_hash_next = *ptb;
1275 *ptb = tb;
1276
1277 /* add in the page list */
1278 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1279 if (phys_page2 != -1)
1280 tb_alloc_page(tb, 1, phys_page2);
1281 else
1282 tb->page_addr[1] = -1;
1283
1284 tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1285 tb->jmp_next[0] = NULL;
1286 tb->jmp_next[1] = NULL;
1287
1288 /* init original jump addresses */
1289 if (tb->tb_next_offset[0] != 0xffff)
1290 tb_reset_jump(tb, 0);
1291 if (tb->tb_next_offset[1] != 0xffff)
1292 tb_reset_jump(tb, 1);
1293
1294 #ifdef DEBUG_TB_CHECK
1295 tb_page_check();
1296 #endif
1297 mmap_unlock();
1298 }
1299
1300 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1301 tb[1].tc_ptr. Return NULL if not found */
1302 TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1303 {
1304 int m_min, m_max, m;
1305 unsigned long v;
1306 TranslationBlock *tb;
1307
1308 if (nb_tbs <= 0)
1309 return NULL;
1310 if (tc_ptr < (unsigned long)code_gen_buffer ||
1311 tc_ptr >= (unsigned long)code_gen_ptr)
1312 return NULL;
1313 /* binary search (cf Knuth) */
1314 m_min = 0;
1315 m_max = nb_tbs - 1;
1316 while (m_min <= m_max) {
1317 m = (m_min + m_max) >> 1;
1318 tb = &tbs[m];
1319 v = (unsigned long)tb->tc_ptr;
1320 if (v == tc_ptr)
1321 return tb;
1322 else if (tc_ptr < v) {
1323 m_max = m - 1;
1324 } else {
1325 m_min = m + 1;
1326 }
1327 }
1328 return &tbs[m_max];
1329 }
1330
1331 static void tb_reset_jump_recursive(TranslationBlock *tb);
1332
1333 static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1334 {
1335 TranslationBlock *tb1, *tb_next, **ptb;
1336 unsigned int n1;
1337
1338 tb1 = tb->jmp_next[n];
1339 if (tb1 != NULL) {
1340 /* find head of list */
1341 for(;;) {
1342 n1 = (long)tb1 & 3;
1343 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1344 if (n1 == 2)
1345 break;
1346 tb1 = tb1->jmp_next[n1];
1347 }
1348 /* we are now sure now that tb jumps to tb1 */
1349 tb_next = tb1;
1350
1351 /* remove tb from the jmp_first list */
1352 ptb = &tb_next->jmp_first;
1353 for(;;) {
1354 tb1 = *ptb;
1355 n1 = (long)tb1 & 3;
1356 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1357 if (n1 == n && tb1 == tb)
1358 break;
1359 ptb = &tb1->jmp_next[n1];
1360 }
1361 *ptb = tb->jmp_next[n];
1362 tb->jmp_next[n] = NULL;
1363
1364 /* suppress the jump to next tb in generated code */
1365 tb_reset_jump(tb, n);
1366
1367 /* suppress jumps in the tb on which we could have jumped */
1368 tb_reset_jump_recursive(tb_next);
1369 }
1370 }
1371
1372 static void tb_reset_jump_recursive(TranslationBlock *tb)
1373 {
1374 tb_reset_jump_recursive2(tb, 0);
1375 tb_reset_jump_recursive2(tb, 1);
1376 }
1377
1378 #if defined(TARGET_HAS_ICE)
1379 #if defined(CONFIG_USER_ONLY)
1380 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1381 {
1382 tb_invalidate_phys_page_range(pc, pc + 1, 0);
1383 }
1384 #else
1385 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1386 {
1387 target_phys_addr_t addr;
1388 target_ulong pd;
1389 ram_addr_t ram_addr;
1390 PhysPageDesc *p;
1391
1392 addr = cpu_get_phys_page_debug(env, pc);
1393 p = phys_page_find(addr >> TARGET_PAGE_BITS);
1394 if (!p) {
1395 pd = IO_MEM_UNASSIGNED;
1396 } else {
1397 pd = p->phys_offset;
1398 }
1399 ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1400 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1401 }
1402 #endif
1403 #endif /* TARGET_HAS_ICE */
1404
1405 #if defined(CONFIG_USER_ONLY)
1406 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1407
1408 {
1409 }
1410
1411 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1412 int flags, CPUWatchpoint **watchpoint)
1413 {
1414 return -ENOSYS;
1415 }
1416 #else
1417 /* Add a watchpoint. */
1418 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1419 int flags, CPUWatchpoint **watchpoint)
1420 {
1421 target_ulong len_mask = ~(len - 1);
1422 CPUWatchpoint *wp;
1423
1424 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1425 if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1426 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1427 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1428 return -EINVAL;
1429 }
1430 wp = qemu_malloc(sizeof(*wp));
1431
1432 wp->vaddr = addr;
1433 wp->len_mask = len_mask;
1434 wp->flags = flags;
1435
1436 /* keep all GDB-injected watchpoints in front */
1437 if (flags & BP_GDB)
1438 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1439 else
1440 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1441
1442 tlb_flush_page(env, addr);
1443
1444 if (watchpoint)
1445 *watchpoint = wp;
1446 return 0;
1447 }
1448
1449 /* Remove a specific watchpoint. */
1450 int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1451 int flags)
1452 {
1453 target_ulong len_mask = ~(len - 1);
1454 CPUWatchpoint *wp;
1455
1456 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1457 if (addr == wp->vaddr && len_mask == wp->len_mask
1458 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1459 cpu_watchpoint_remove_by_ref(env, wp);
1460 return 0;
1461 }
1462 }
1463 return -ENOENT;
1464 }
1465
1466 /* Remove a specific watchpoint by reference. */
1467 void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1468 {
1469 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1470
1471 tlb_flush_page(env, watchpoint->vaddr);
1472
1473 qemu_free(watchpoint);
1474 }
1475
1476 /* Remove all matching watchpoints. */
1477 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1478 {
1479 CPUWatchpoint *wp, *next;
1480
1481 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1482 if (wp->flags & mask)
1483 cpu_watchpoint_remove_by_ref(env, wp);
1484 }
1485 }
1486 #endif
1487
1488 /* Add a breakpoint. */
1489 int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1490 CPUBreakpoint **breakpoint)
1491 {
1492 #if defined(TARGET_HAS_ICE)
1493 CPUBreakpoint *bp;
1494
1495 bp = qemu_malloc(sizeof(*bp));
1496
1497 bp->pc = pc;
1498 bp->flags = flags;
1499
1500 /* keep all GDB-injected breakpoints in front */
1501 if (flags & BP_GDB)
1502 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1503 else
1504 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1505
1506 breakpoint_invalidate(env, pc);
1507
1508 if (breakpoint)
1509 *breakpoint = bp;
1510 return 0;
1511 #else
1512 return -ENOSYS;
1513 #endif
1514 }
1515
1516 /* Remove a specific breakpoint. */
1517 int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1518 {
1519 #if defined(TARGET_HAS_ICE)
1520 CPUBreakpoint *bp;
1521
1522 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1523 if (bp->pc == pc && bp->flags == flags) {
1524 cpu_breakpoint_remove_by_ref(env, bp);
1525 return 0;
1526 }
1527 }
1528 return -ENOENT;
1529 #else
1530 return -ENOSYS;
1531 #endif
1532 }
1533
1534 /* Remove a specific breakpoint by reference. */
1535 void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
1536 {
1537 #if defined(TARGET_HAS_ICE)
1538 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1539
1540 breakpoint_invalidate(env, breakpoint->pc);
1541
1542 qemu_free(breakpoint);
1543 #endif
1544 }
1545
1546 /* Remove all matching breakpoints. */
1547 void cpu_breakpoint_remove_all(CPUState *env, int mask)
1548 {
1549 #if defined(TARGET_HAS_ICE)
1550 CPUBreakpoint *bp, *next;
1551
1552 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1553 if (bp->flags & mask)
1554 cpu_breakpoint_remove_by_ref(env, bp);
1555 }
1556 #endif
1557 }
1558
1559 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1560 CPU loop after each instruction */
1561 void cpu_single_step(CPUState *env, int enabled)
1562 {
1563 #if defined(TARGET_HAS_ICE)
1564 if (env->singlestep_enabled != enabled) {
1565 env->singlestep_enabled = enabled;
1566 if (kvm_enabled())
1567 kvm_update_guest_debug(env, 0);
1568 else {
1569 /* must flush all the translated code to avoid inconsistencies */
1570 /* XXX: only flush what is necessary */
1571 tb_flush(env);
1572 }
1573 }
1574 #endif
1575 }
1576
1577 /* enable or disable low levels log */
1578 void cpu_set_log(int log_flags)
1579 {
1580 loglevel = log_flags;
1581 if (loglevel && !logfile) {
1582 logfile = fopen(logfilename, log_append ? "a" : "w");
1583 if (!logfile) {
1584 perror(logfilename);
1585 _exit(1);
1586 }
1587 #if !defined(CONFIG_SOFTMMU)
1588 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1589 {
1590 static char logfile_buf[4096];
1591 setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1592 }
1593 #elif !defined(_WIN32)
1594 /* Win32 doesn't support line-buffering and requires size >= 2 */
1595 setvbuf(logfile, NULL, _IOLBF, 0);
1596 #endif
1597 log_append = 1;
1598 }
1599 if (!loglevel && logfile) {
1600 fclose(logfile);
1601 logfile = NULL;
1602 }
1603 }
1604
1605 void cpu_set_log_filename(const char *filename)
1606 {
1607 logfilename = strdup(filename);
1608 if (logfile) {
1609 fclose(logfile);
1610 logfile = NULL;
1611 }
1612 cpu_set_log(loglevel);
1613 }
1614
1615 static void cpu_unlink_tb(CPUState *env)
1616 {
1617 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1618 problem and hope the cpu will stop of its own accord. For userspace
1619 emulation this often isn't actually as bad as it sounds. Often
1620 signals are used primarily to interrupt blocking syscalls. */
1621 TranslationBlock *tb;
1622 static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1623
1624 spin_lock(&interrupt_lock);
1625 tb = env->current_tb;
1626 /* if the cpu is currently executing code, we must unlink it and
1627 all the potentially executing TB */
1628 if (tb) {
1629 env->current_tb = NULL;
1630 tb_reset_jump_recursive(tb);
1631 }
1632 spin_unlock(&interrupt_lock);
1633 }
1634
1635 #ifndef CONFIG_USER_ONLY
1636 /* mask must never be zero, except for A20 change call */
1637 static void tcg_handle_interrupt(CPUState *env, int mask)
1638 {
1639 int old_mask;
1640
1641 old_mask = env->interrupt_request;
1642 env->interrupt_request |= mask;
1643
1644 /*
1645 * If called from iothread context, wake the target cpu in
1646 * case its halted.
1647 */
1648 if (!qemu_cpu_is_self(env)) {
1649 qemu_cpu_kick(env);
1650 return;
1651 }
1652
1653 if (use_icount) {
1654 env->icount_decr.u16.high = 0xffff;
1655 if (!can_do_io(env)
1656 && (mask & ~old_mask) != 0) {
1657 cpu_abort(env, "Raised interrupt while not in I/O function");
1658 }
1659 } else {
1660 cpu_unlink_tb(env);
1661 }
1662 }
1663
1664 CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
1665
1666 #else /* CONFIG_USER_ONLY */
1667
1668 void cpu_interrupt(CPUState *env, int mask)
1669 {
1670 env->interrupt_request |= mask;
1671 cpu_unlink_tb(env);
1672 }
1673 #endif /* CONFIG_USER_ONLY */
1674
1675 void cpu_reset_interrupt(CPUState *env, int mask)
1676 {
1677 env->interrupt_request &= ~mask;
1678 }
1679
1680 void cpu_exit(CPUState *env)
1681 {
1682 env->exit_request = 1;
1683 cpu_unlink_tb(env);
1684 }
1685
1686 const CPULogItem cpu_log_items[] = {
1687 { CPU_LOG_TB_OUT_ASM, "out_asm",
1688 "show generated host assembly code for each compiled TB" },
1689 { CPU_LOG_TB_IN_ASM, "in_asm",
1690 "show target assembly code for each compiled TB" },
1691 { CPU_LOG_TB_OP, "op",
1692 "show micro ops for each compiled TB" },
1693 { CPU_LOG_TB_OP_OPT, "op_opt",
1694 "show micro ops "
1695 #ifdef TARGET_I386
1696 "before eflags optimization and "
1697 #endif
1698 "after liveness analysis" },
1699 { CPU_LOG_INT, "int",
1700 "show interrupts/exceptions in short format" },
1701 { CPU_LOG_EXEC, "exec",
1702 "show trace before each executed TB (lots of logs)" },
1703 { CPU_LOG_TB_CPU, "cpu",
1704 "show CPU state before block translation" },
1705 #ifdef TARGET_I386
1706 { CPU_LOG_PCALL, "pcall",
1707 "show protected mode far calls/returns/exceptions" },
1708 { CPU_LOG_RESET, "cpu_reset",
1709 "show CPU state before CPU resets" },
1710 #endif
1711 #ifdef DEBUG_IOPORT
1712 { CPU_LOG_IOPORT, "ioport",
1713 "show all i/o ports accesses" },
1714 #endif
1715 { 0, NULL, NULL },
1716 };
1717
1718 #ifndef CONFIG_USER_ONLY
1719 static QLIST_HEAD(memory_client_list, CPUPhysMemoryClient) memory_client_list
1720 = QLIST_HEAD_INITIALIZER(memory_client_list);
1721
1722 static void cpu_notify_set_memory(target_phys_addr_t start_addr,
1723 ram_addr_t size,
1724 ram_addr_t phys_offset,
1725 bool log_dirty)
1726 {
1727 CPUPhysMemoryClient *client;
1728 QLIST_FOREACH(client, &memory_client_list, list) {
1729 client->set_memory(client, start_addr, size, phys_offset, log_dirty);
1730 }
1731 }
1732
1733 static int cpu_notify_sync_dirty_bitmap(target_phys_addr_t start,
1734 target_phys_addr_t end)
1735 {
1736 CPUPhysMemoryClient *client;
1737 QLIST_FOREACH(client, &memory_client_list, list) {
1738 int r = client->sync_dirty_bitmap(client, start, end);
1739 if (r < 0)
1740 return r;
1741 }
1742 return 0;
1743 }
1744
1745 static int cpu_notify_migration_log(int enable)
1746 {
1747 CPUPhysMemoryClient *client;
1748 QLIST_FOREACH(client, &memory_client_list, list) {
1749 int r = client->migration_log(client, enable);
1750 if (r < 0)
1751 return r;
1752 }
1753 return 0;
1754 }
1755
1756 /* The l1_phys_map provides the upper P_L1_BITs of the guest physical
1757 * address. Each intermediate table provides the next L2_BITs of guest
1758 * physical address space. The number of levels vary based on host and
1759 * guest configuration, making it efficient to build the final guest
1760 * physical address by seeding the L1 offset and shifting and adding in
1761 * each L2 offset as we recurse through them. */
1762 static void phys_page_for_each_1(CPUPhysMemoryClient *client,
1763 int level, void **lp, target_phys_addr_t addr)
1764 {
1765 int i;
1766
1767 if (*lp == NULL) {
1768 return;
1769 }
1770 if (level == 0) {
1771 PhysPageDesc *pd = *lp;
1772 addr <<= L2_BITS + TARGET_PAGE_BITS;
1773 for (i = 0; i < L2_SIZE; ++i) {
1774 if (pd[i].phys_offset != IO_MEM_UNASSIGNED) {
1775 client->set_memory(client, addr | i << TARGET_PAGE_BITS,
1776 TARGET_PAGE_SIZE, pd[i].phys_offset, false);
1777 }
1778 }
1779 } else {
1780 void **pp = *lp;
1781 for (i = 0; i < L2_SIZE; ++i) {
1782 phys_page_for_each_1(client, level - 1, pp + i,
1783 (addr << L2_BITS) | i);
1784 }
1785 }
1786 }
1787
1788 static void phys_page_for_each(CPUPhysMemoryClient *client)
1789 {
1790 int i;
1791 for (i = 0; i < P_L1_SIZE; ++i) {
1792 phys_page_for_each_1(client, P_L1_SHIFT / L2_BITS - 1,
1793 l1_phys_map + i, i);
1794 }
1795 }
1796
1797 void cpu_register_phys_memory_client(CPUPhysMemoryClient *client)
1798 {
1799 QLIST_INSERT_HEAD(&memory_client_list, client, list);
1800 phys_page_for_each(client);
1801 }
1802
1803 void cpu_unregister_phys_memory_client(CPUPhysMemoryClient *client)
1804 {
1805 QLIST_REMOVE(client, list);
1806 }
1807 #endif
1808
1809 static int cmp1(const char *s1, int n, const char *s2)
1810 {
1811 if (strlen(s2) != n)
1812 return 0;
1813 return memcmp(s1, s2, n) == 0;
1814 }
1815
1816 /* takes a comma separated list of log masks. Return 0 if error. */
1817 int cpu_str_to_log_mask(const char *str)
1818 {
1819 const CPULogItem *item;
1820 int mask;
1821 const char *p, *p1;
1822
1823 p = str;
1824 mask = 0;
1825 for(;;) {
1826 p1 = strchr(p, ',');
1827 if (!p1)
1828 p1 = p + strlen(p);
1829 if(cmp1(p,p1-p,"all")) {
1830 for(item = cpu_log_items; item->mask != 0; item++) {
1831 mask |= item->mask;
1832 }
1833 } else {
1834 for(item = cpu_log_items; item->mask != 0; item++) {
1835 if (cmp1(p, p1 - p, item->name))
1836 goto found;
1837 }
1838 return 0;
1839 }
1840 found:
1841 mask |= item->mask;
1842 if (*p1 != ',')
1843 break;
1844 p = p1 + 1;
1845 }
1846 return mask;
1847 }
1848
1849 void cpu_abort(CPUState *env, const char *fmt, ...)
1850 {
1851 va_list ap;
1852 va_list ap2;
1853
1854 va_start(ap, fmt);
1855 va_copy(ap2, ap);
1856 fprintf(stderr, "qemu: fatal: ");
1857 vfprintf(stderr, fmt, ap);
1858 fprintf(stderr, "\n");
1859 #ifdef TARGET_I386
1860 cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1861 #else
1862 cpu_dump_state(env, stderr, fprintf, 0);
1863 #endif
1864 if (qemu_log_enabled()) {
1865 qemu_log("qemu: fatal: ");
1866 qemu_log_vprintf(fmt, ap2);
1867 qemu_log("\n");
1868 #ifdef TARGET_I386
1869 log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1870 #else
1871 log_cpu_state(env, 0);
1872 #endif
1873 qemu_log_flush();
1874 qemu_log_close();
1875 }
1876 va_end(ap2);
1877 va_end(ap);
1878 #if defined(CONFIG_USER_ONLY)
1879 {
1880 struct sigaction act;
1881 sigfillset(&act.sa_mask);
1882 act.sa_handler = SIG_DFL;
1883 sigaction(SIGABRT, &act, NULL);
1884 }
1885 #endif
1886 abort();
1887 }
1888
1889 CPUState *cpu_copy(CPUState *env)
1890 {
1891 CPUState *new_env = cpu_init(env->cpu_model_str);
1892 CPUState *next_cpu = new_env->next_cpu;
1893 int cpu_index = new_env->cpu_index;
1894 #if defined(TARGET_HAS_ICE)
1895 CPUBreakpoint *bp;
1896 CPUWatchpoint *wp;
1897 #endif
1898
1899 memcpy(new_env, env, sizeof(CPUState));
1900
1901 /* Preserve chaining and index. */
1902 new_env->next_cpu = next_cpu;
1903 new_env->cpu_index = cpu_index;
1904
1905 /* Clone all break/watchpoints.
1906 Note: Once we support ptrace with hw-debug register access, make sure
1907 BP_CPU break/watchpoints are handled correctly on clone. */
1908 QTAILQ_INIT(&env->breakpoints);
1909 QTAILQ_INIT(&env->watchpoints);
1910 #if defined(TARGET_HAS_ICE)
1911 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1912 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1913 }
1914 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1915 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1916 wp->flags, NULL);
1917 }
1918 #endif
1919
1920 return new_env;
1921 }
1922
1923 #if !defined(CONFIG_USER_ONLY)
1924
1925 static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1926 {
1927 unsigned int i;
1928
1929 /* Discard jump cache entries for any tb which might potentially
1930 overlap the flushed page. */
1931 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1932 memset (&env->tb_jmp_cache[i], 0,
1933 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1934
1935 i = tb_jmp_cache_hash_page(addr);
1936 memset (&env->tb_jmp_cache[i], 0,
1937 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1938 }
1939
1940 static CPUTLBEntry s_cputlb_empty_entry = {
1941 .addr_read = -1,
1942 .addr_write = -1,
1943 .addr_code = -1,
1944 .addend = -1,
1945 };
1946
1947 /* NOTE: if flush_global is true, also flush global entries (not
1948 implemented yet) */
1949 void tlb_flush(CPUState *env, int flush_global)
1950 {
1951 int i;
1952
1953 #if defined(DEBUG_TLB)
1954 printf("tlb_flush:\n");
1955 #endif
1956 /* must reset current TB so that interrupts cannot modify the
1957 links while we are modifying them */
1958 env->current_tb = NULL;
1959
1960 for(i = 0; i < CPU_TLB_SIZE; i++) {
1961 int mmu_idx;
1962 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
1963 env->tlb_table[mmu_idx][i] = s_cputlb_empty_entry;
1964 }
1965 }
1966
1967 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1968
1969 env->tlb_flush_addr = -1;
1970 env->tlb_flush_mask = 0;
1971 tlb_flush_count++;
1972 }
1973
1974 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
1975 {
1976 if (addr == (tlb_entry->addr_read &
1977 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1978 addr == (tlb_entry->addr_write &
1979 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1980 addr == (tlb_entry->addr_code &
1981 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1982 *tlb_entry = s_cputlb_empty_entry;
1983 }
1984 }
1985
1986 void tlb_flush_page(CPUState *env, target_ulong addr)
1987 {
1988 int i;
1989 int mmu_idx;
1990
1991 #if defined(DEBUG_TLB)
1992 printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1993 #endif
1994 /* Check if we need to flush due to large pages. */
1995 if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
1996 #if defined(DEBUG_TLB)
1997 printf("tlb_flush_page: forced full flush ("
1998 TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
1999 env->tlb_flush_addr, env->tlb_flush_mask);
2000 #endif
2001 tlb_flush(env, 1);
2002 return;
2003 }
2004 /* must reset current TB so that interrupts cannot modify the
2005 links while we are modifying them */
2006 env->current_tb = NULL;
2007
2008 addr &= TARGET_PAGE_MASK;
2009 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2010 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2011 tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
2012
2013 tlb_flush_jmp_cache(env, addr);
2014 }
2015
2016 /* update the TLBs so that writes to code in the virtual page 'addr'
2017 can be detected */
2018 static void tlb_protect_code(ram_addr_t ram_addr)
2019 {
2020 cpu_physical_memory_reset_dirty(ram_addr,
2021 ram_addr + TARGET_PAGE_SIZE,
2022 CODE_DIRTY_FLAG);
2023 }
2024
2025 /* update the TLB so that writes in physical page 'phys_addr' are no longer
2026 tested for self modifying code */
2027 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
2028 target_ulong vaddr)
2029 {
2030 cpu_physical_memory_set_dirty_flags(ram_addr, CODE_DIRTY_FLAG);
2031 }
2032
2033 static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
2034 unsigned long start, unsigned long length)
2035 {
2036 unsigned long addr;
2037 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2038 addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
2039 if ((addr - start) < length) {
2040 tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
2041 }
2042 }
2043 }
2044
2045 /* Note: start and end must be within the same ram block. */
2046 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
2047 int dirty_flags)
2048 {
2049 CPUState *env;
2050 unsigned long length, start1;
2051 int i;
2052
2053 start &= TARGET_PAGE_MASK;
2054 end = TARGET_PAGE_ALIGN(end);
2055
2056 length = end - start;
2057 if (length == 0)
2058 return;
2059 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
2060
2061 /* we modify the TLB cache so that the dirty bit will be set again
2062 when accessing the range */
2063 start1 = (unsigned long)qemu_safe_ram_ptr(start);
2064 /* Chek that we don't span multiple blocks - this breaks the
2065 address comparisons below. */
2066 if ((unsigned long)qemu_safe_ram_ptr(end - 1) - start1
2067 != (end - 1) - start) {
2068 abort();
2069 }
2070
2071 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2072 int mmu_idx;
2073 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2074 for(i = 0; i < CPU_TLB_SIZE; i++)
2075 tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
2076 start1, length);
2077 }
2078 }
2079 }
2080
2081 int cpu_physical_memory_set_dirty_tracking(int enable)
2082 {
2083 int ret = 0;
2084 in_migration = enable;
2085 ret = cpu_notify_migration_log(!!enable);
2086 return ret;
2087 }
2088
2089 int cpu_physical_memory_get_dirty_tracking(void)
2090 {
2091 return in_migration;
2092 }
2093
2094 int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
2095 target_phys_addr_t end_addr)
2096 {
2097 int ret;
2098
2099 ret = cpu_notify_sync_dirty_bitmap(start_addr, end_addr);
2100 return ret;
2101 }
2102
2103 int cpu_physical_log_start(target_phys_addr_t start_addr,
2104 ram_addr_t size)
2105 {
2106 CPUPhysMemoryClient *client;
2107 QLIST_FOREACH(client, &memory_client_list, list) {
2108 if (client->log_start) {
2109 int r = client->log_start(client, start_addr, size);
2110 if (r < 0) {
2111 return r;
2112 }
2113 }
2114 }
2115 return 0;
2116 }
2117
2118 int cpu_physical_log_stop(target_phys_addr_t start_addr,
2119 ram_addr_t size)
2120 {
2121 CPUPhysMemoryClient *client;
2122 QLIST_FOREACH(client, &memory_client_list, list) {
2123 if (client->log_stop) {
2124 int r = client->log_stop(client, start_addr, size);
2125 if (r < 0) {
2126 return r;
2127 }
2128 }
2129 }
2130 return 0;
2131 }
2132
2133 static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
2134 {
2135 ram_addr_t ram_addr;
2136 void *p;
2137
2138 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2139 p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
2140 + tlb_entry->addend);
2141 ram_addr = qemu_ram_addr_from_host_nofail(p);
2142 if (!cpu_physical_memory_is_dirty(ram_addr)) {
2143 tlb_entry->addr_write |= TLB_NOTDIRTY;
2144 }
2145 }
2146 }
2147
2148 /* update the TLB according to the current state of the dirty bits */
2149 void cpu_tlb_update_dirty(CPUState *env)
2150 {
2151 int i;
2152 int mmu_idx;
2153 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2154 for(i = 0; i < CPU_TLB_SIZE; i++)
2155 tlb_update_dirty(&env->tlb_table[mmu_idx][i]);
2156 }
2157 }
2158
2159 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
2160 {
2161 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
2162 tlb_entry->addr_write = vaddr;
2163 }
2164
2165 /* update the TLB corresponding to virtual page vaddr
2166 so that it is no longer dirty */
2167 static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
2168 {
2169 int i;
2170 int mmu_idx;
2171
2172 vaddr &= TARGET_PAGE_MASK;
2173 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2174 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2175 tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
2176 }
2177
2178 /* Our TLB does not support large pages, so remember the area covered by
2179 large pages and trigger a full TLB flush if these are invalidated. */
2180 static void tlb_add_large_page(CPUState *env, target_ulong vaddr,
2181 target_ulong size)
2182 {
2183 target_ulong mask = ~(size - 1);
2184
2185 if (env->tlb_flush_addr == (target_ulong)-1) {
2186 env->tlb_flush_addr = vaddr & mask;
2187 env->tlb_flush_mask = mask;
2188 return;
2189 }
2190 /* Extend the existing region to include the new page.
2191 This is a compromise between unnecessary flushes and the cost
2192 of maintaining a full variable size TLB. */
2193 mask &= env->tlb_flush_mask;
2194 while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
2195 mask <<= 1;
2196 }
2197 env->tlb_flush_addr &= mask;
2198 env->tlb_flush_mask = mask;
2199 }
2200
2201 /* Add a new TLB entry. At most one entry for a given virtual address
2202 is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
2203 supplied size is only used by tlb_flush_page. */
2204 void tlb_set_page(CPUState *env, target_ulong vaddr,
2205 target_phys_addr_t paddr, int prot,
2206 int mmu_idx, target_ulong size)
2207 {
2208 PhysPageDesc *p;
2209 unsigned long pd;
2210 unsigned int index;
2211 target_ulong address;
2212 target_ulong code_address;
2213 unsigned long addend;
2214 CPUTLBEntry *te;
2215 CPUWatchpoint *wp;
2216 target_phys_addr_t iotlb;
2217
2218 assert(size >= TARGET_PAGE_SIZE);
2219 if (size != TARGET_PAGE_SIZE) {
2220 tlb_add_large_page(env, vaddr, size);
2221 }
2222 p = phys_page_find(paddr >> TARGET_PAGE_BITS);
2223 if (!p) {
2224 pd = IO_MEM_UNASSIGNED;
2225 } else {
2226 pd = p->phys_offset;
2227 }
2228 #if defined(DEBUG_TLB)
2229 printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
2230 " prot=%x idx=%d pd=0x%08lx\n",
2231 vaddr, paddr, prot, mmu_idx, pd);
2232 #endif
2233
2234 address = vaddr;
2235 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
2236 /* IO memory case (romd handled later) */
2237 address |= TLB_MMIO;
2238 }
2239 addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
2240 if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
2241 /* Normal RAM. */
2242 iotlb = pd & TARGET_PAGE_MASK;
2243 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
2244 iotlb |= IO_MEM_NOTDIRTY;
2245 else
2246 iotlb |= IO_MEM_ROM;
2247 } else {
2248 /* IO handlers are currently passed a physical address.
2249 It would be nice to pass an offset from the base address
2250 of that region. This would avoid having to special case RAM,
2251 and avoid full address decoding in every device.
2252 We can't use the high bits of pd for this because
2253 IO_MEM_ROMD uses these as a ram address. */
2254 iotlb = (pd & ~TARGET_PAGE_MASK);
2255 if (p) {
2256 iotlb += p->region_offset;
2257 } else {
2258 iotlb += paddr;
2259 }
2260 }
2261
2262 code_address = address;
2263 /* Make accesses to pages with watchpoints go via the
2264 watchpoint trap routines. */
2265 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
2266 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2267 /* Avoid trapping reads of pages with a write breakpoint. */
2268 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
2269 iotlb = io_mem_watch + paddr;
2270 address |= TLB_MMIO;
2271 break;
2272 }
2273 }
2274 }
2275
2276 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2277 env->iotlb[mmu_idx][index] = iotlb - vaddr;
2278 te = &env->tlb_table[mmu_idx][index];
2279 te->addend = addend - vaddr;
2280 if (prot & PAGE_READ) {
2281 te->addr_read = address;
2282 } else {
2283 te->addr_read = -1;
2284 }
2285
2286 if (prot & PAGE_EXEC) {
2287 te->addr_code = code_address;
2288 } else {
2289 te->addr_code = -1;
2290 }
2291 if (prot & PAGE_WRITE) {
2292 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2293 (pd & IO_MEM_ROMD)) {
2294 /* Write access calls the I/O callback. */
2295 te->addr_write = address | TLB_MMIO;
2296 } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2297 !cpu_physical_memory_is_dirty(pd)) {
2298 te->addr_write = address | TLB_NOTDIRTY;
2299 } else {
2300 te->addr_write = address;
2301 }
2302 } else {
2303 te->addr_write = -1;
2304 }
2305 }
2306
2307 #else
2308
2309 void tlb_flush(CPUState *env, int flush_global)
2310 {
2311 }
2312
2313 void tlb_flush_page(CPUState *env, target_ulong addr)
2314 {
2315 }
2316
2317 /*
2318 * Walks guest process memory "regions" one by one
2319 * and calls callback function 'fn' for each region.
2320 */
2321
2322 struct walk_memory_regions_data
2323 {
2324 walk_memory_regions_fn fn;
2325 void *priv;
2326 unsigned long start;
2327 int prot;
2328 };
2329
2330 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
2331 abi_ulong end, int new_prot)
2332 {
2333 if (data->start != -1ul) {
2334 int rc = data->fn(data->priv, data->start, end, data->prot);
2335 if (rc != 0) {
2336 return rc;
2337 }
2338 }
2339
2340 data->start = (new_prot ? end : -1ul);
2341 data->prot = new_prot;
2342
2343 return 0;
2344 }
2345
2346 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
2347 abi_ulong base, int level, void **lp)
2348 {
2349 abi_ulong pa;
2350 int i, rc;
2351
2352 if (*lp == NULL) {
2353 return walk_memory_regions_end(data, base, 0);
2354 }
2355
2356 if (level == 0) {
2357 PageDesc *pd = *lp;
2358 for (i = 0; i < L2_SIZE; ++i) {
2359 int prot = pd[i].flags;
2360
2361 pa = base | (i << TARGET_PAGE_BITS);
2362 if (prot != data->prot) {
2363 rc = walk_memory_regions_end(data, pa, prot);
2364 if (rc != 0) {
2365 return rc;
2366 }
2367 }
2368 }
2369 } else {
2370 void **pp = *lp;
2371 for (i = 0; i < L2_SIZE; ++i) {
2372 pa = base | ((abi_ulong)i <<
2373 (TARGET_PAGE_BITS + L2_BITS * level));
2374 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
2375 if (rc != 0) {
2376 return rc;
2377 }
2378 }
2379 }
2380
2381 return 0;
2382 }
2383
2384 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
2385 {
2386 struct walk_memory_regions_data data;
2387 unsigned long i;
2388
2389 data.fn = fn;
2390 data.priv = priv;
2391 data.start = -1ul;
2392 data.prot = 0;
2393
2394 for (i = 0; i < V_L1_SIZE; i++) {
2395 int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
2396 V_L1_SHIFT / L2_BITS - 1, l1_map + i);
2397 if (rc != 0) {
2398 return rc;
2399 }
2400 }
2401
2402 return walk_memory_regions_end(&data, 0, 0);
2403 }
2404
2405 static int dump_region(void *priv, abi_ulong start,
2406 abi_ulong end, unsigned long prot)
2407 {
2408 FILE *f = (FILE *)priv;
2409
2410 (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
2411 " "TARGET_ABI_FMT_lx" %c%c%c\n",
2412 start, end, end - start,
2413 ((prot & PAGE_READ) ? 'r' : '-'),
2414 ((prot & PAGE_WRITE) ? 'w' : '-'),
2415 ((prot & PAGE_EXEC) ? 'x' : '-'));
2416
2417 return (0);
2418 }
2419
2420 /* dump memory mappings */
2421 void page_dump(FILE *f)
2422 {
2423 (void) fprintf(f, "%-8s %-8s %-8s %s\n",
2424 "start", "end", "size", "prot");
2425 walk_memory_regions(f, dump_region);
2426 }
2427
2428 int page_get_flags(target_ulong address)
2429 {
2430 PageDesc *p;
2431
2432 p = page_find(address >> TARGET_PAGE_BITS);
2433 if (!p)
2434 return 0;
2435 return p->flags;
2436 }
2437
2438 /* Modify the flags of a page and invalidate the code if necessary.
2439 The flag PAGE_WRITE_ORG is positioned automatically depending
2440 on PAGE_WRITE. The mmap_lock should already be held. */
2441 void page_set_flags(target_ulong start, target_ulong end, int flags)
2442 {
2443 target_ulong addr, len;
2444
2445 /* This function should never be called with addresses outside the
2446 guest address space. If this assert fires, it probably indicates
2447 a missing call to h2g_valid. */
2448 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2449 assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2450 #endif
2451 assert(start < end);
2452
2453 start = start & TARGET_PAGE_MASK;
2454 end = TARGET_PAGE_ALIGN(end);
2455
2456 if (flags & PAGE_WRITE) {
2457 flags |= PAGE_WRITE_ORG;
2458 }
2459
2460 for (addr = start, len = end - start;
2461 len != 0;
2462 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2463 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2464
2465 /* If the write protection bit is set, then we invalidate
2466 the code inside. */
2467 if (!(p->flags & PAGE_WRITE) &&
2468 (flags & PAGE_WRITE) &&
2469 p->first_tb) {
2470 tb_invalidate_phys_page(addr, 0, NULL);
2471 }
2472 p->flags = flags;
2473 }
2474 }
2475
2476 int page_check_range(target_ulong start, target_ulong len, int flags)
2477 {
2478 PageDesc *p;
2479 target_ulong end;
2480 target_ulong addr;
2481
2482 /* This function should never be called with addresses outside the
2483 guest address space. If this assert fires, it probably indicates
2484 a missing call to h2g_valid. */
2485 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2486 assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2487 #endif
2488
2489 if (len == 0) {
2490 return 0;
2491 }
2492 if (start + len - 1 < start) {
2493 /* We've wrapped around. */
2494 return -1;
2495 }
2496
2497 end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2498 start = start & TARGET_PAGE_MASK;
2499
2500 for (addr = start, len = end - start;
2501 len != 0;
2502 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2503 p = page_find(addr >> TARGET_PAGE_BITS);
2504 if( !p )
2505 return -1;
2506 if( !(p->flags & PAGE_VALID) )
2507 return -1;
2508
2509 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2510 return -1;
2511 if (flags & PAGE_WRITE) {
2512 if (!(p->flags & PAGE_WRITE_ORG))
2513 return -1;
2514 /* unprotect the page if it was put read-only because it
2515 contains translated code */
2516 if (!(p->flags & PAGE_WRITE)) {
2517 if (!page_unprotect(addr, 0, NULL))
2518 return -1;
2519 }
2520 return 0;
2521 }
2522 }
2523 return 0;
2524 }
2525
2526 /* called from signal handler: invalidate the code and unprotect the
2527 page. Return TRUE if the fault was successfully handled. */
2528 int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2529 {
2530 unsigned int prot;
2531 PageDesc *p;
2532 target_ulong host_start, host_end, addr;
2533
2534 /* Technically this isn't safe inside a signal handler. However we
2535 know this only ever happens in a synchronous SEGV handler, so in
2536 practice it seems to be ok. */
2537 mmap_lock();
2538
2539 p = page_find(address >> TARGET_PAGE_BITS);
2540 if (!p) {
2541 mmap_unlock();
2542 return 0;
2543 }
2544
2545 /* if the page was really writable, then we change its
2546 protection back to writable */
2547 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
2548 host_start = address & qemu_host_page_mask;
2549 host_end = host_start + qemu_host_page_size;
2550
2551 prot = 0;
2552 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
2553 p = page_find(addr >> TARGET_PAGE_BITS);
2554 p->flags |= PAGE_WRITE;
2555 prot |= p->flags;
2556
2557 /* and since the content will be modified, we must invalidate
2558 the corresponding translated code. */
2559 tb_invalidate_phys_page(addr, pc, puc);
2560 #ifdef DEBUG_TB_CHECK
2561 tb_invalidate_check(addr);
2562 #endif
2563 }
2564 mprotect((void *)g2h(host_start), qemu_host_page_size,
2565 prot & PAGE_BITS);
2566
2567 mmap_unlock();
2568 return 1;
2569 }
2570 mmap_unlock();
2571 return 0;
2572 }
2573
2574 static inline void tlb_set_dirty(CPUState *env,
2575 unsigned long addr, target_ulong vaddr)
2576 {
2577 }
2578 #endif /* defined(CONFIG_USER_ONLY) */
2579
2580 #if !defined(CONFIG_USER_ONLY)
2581
2582 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
2583 typedef struct subpage_t {
2584 target_phys_addr_t base;
2585 ram_addr_t sub_io_index[TARGET_PAGE_SIZE];
2586 ram_addr_t region_offset[TARGET_PAGE_SIZE];
2587 } subpage_t;
2588
2589 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2590 ram_addr_t memory, ram_addr_t region_offset);
2591 static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2592 ram_addr_t orig_memory,
2593 ram_addr_t region_offset);
2594 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2595 need_subpage) \
2596 do { \
2597 if (addr > start_addr) \
2598 start_addr2 = 0; \
2599 else { \
2600 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2601 if (start_addr2 > 0) \
2602 need_subpage = 1; \
2603 } \
2604 \
2605 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2606 end_addr2 = TARGET_PAGE_SIZE - 1; \
2607 else { \
2608 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2609 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2610 need_subpage = 1; \
2611 } \
2612 } while (0)
2613
2614 /* register physical memory.
2615 For RAM, 'size' must be a multiple of the target page size.
2616 If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2617 io memory page. The address used when calling the IO function is
2618 the offset from the start of the region, plus region_offset. Both
2619 start_addr and region_offset are rounded down to a page boundary
2620 before calculating this offset. This should not be a problem unless
2621 the low bits of start_addr and region_offset differ. */
2622 void cpu_register_physical_memory_log(target_phys_addr_t start_addr,
2623 ram_addr_t size,
2624 ram_addr_t phys_offset,
2625 ram_addr_t region_offset,
2626 bool log_dirty)
2627 {
2628 target_phys_addr_t addr, end_addr;
2629 PhysPageDesc *p;
2630 CPUState *env;
2631 ram_addr_t orig_size = size;
2632 subpage_t *subpage;
2633
2634 assert(size);
2635 cpu_notify_set_memory(start_addr, size, phys_offset, log_dirty);
2636
2637 if (phys_offset == IO_MEM_UNASSIGNED) {
2638 region_offset = start_addr;
2639 }
2640 region_offset &= TARGET_PAGE_MASK;
2641 size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2642 end_addr = start_addr + (target_phys_addr_t)size;
2643
2644 addr = start_addr;
2645 do {
2646 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2647 if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2648 ram_addr_t orig_memory = p->phys_offset;
2649 target_phys_addr_t start_addr2, end_addr2;
2650 int need_subpage = 0;
2651
2652 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2653 need_subpage);
2654 if (need_subpage) {
2655 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2656 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2657 &p->phys_offset, orig_memory,
2658 p->region_offset);
2659 } else {
2660 subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2661 >> IO_MEM_SHIFT];
2662 }
2663 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2664 region_offset);
2665 p->region_offset = 0;
2666 } else {
2667 p->phys_offset = phys_offset;
2668 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2669 (phys_offset & IO_MEM_ROMD))
2670 phys_offset += TARGET_PAGE_SIZE;
2671 }
2672 } else {
2673 p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2674 p->phys_offset = phys_offset;
2675 p->region_offset = region_offset;
2676 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2677 (phys_offset & IO_MEM_ROMD)) {
2678 phys_offset += TARGET_PAGE_SIZE;
2679 } else {
2680 target_phys_addr_t start_addr2, end_addr2;
2681 int need_subpage = 0;
2682
2683 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2684 end_addr2, need_subpage);
2685
2686 if (need_subpage) {
2687 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2688 &p->phys_offset, IO_MEM_UNASSIGNED,
2689 addr & TARGET_PAGE_MASK);
2690 subpage_register(subpage, start_addr2, end_addr2,
2691 phys_offset, region_offset);
2692 p->region_offset = 0;
2693 }
2694 }
2695 }
2696 region_offset += TARGET_PAGE_SIZE;
2697 addr += TARGET_PAGE_SIZE;
2698 } while (addr != end_addr);
2699
2700 /* since each CPU stores ram addresses in its TLB cache, we must
2701 reset the modified entries */
2702 /* XXX: slow ! */
2703 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2704 tlb_flush(env, 1);
2705 }
2706 }
2707
2708 /* XXX: temporary until new memory mapping API */
2709 ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2710 {
2711 PhysPageDesc *p;
2712
2713 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2714 if (!p)
2715 return IO_MEM_UNASSIGNED;
2716 return p->phys_offset;
2717 }
2718
2719 void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2720 {
2721 if (kvm_enabled())
2722 kvm_coalesce_mmio_region(addr, size);
2723 }
2724
2725 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2726 {
2727 if (kvm_enabled())
2728 kvm_uncoalesce_mmio_region(addr, size);
2729 }
2730
2731 void qemu_flush_coalesced_mmio_buffer(void)
2732 {
2733 if (kvm_enabled())
2734 kvm_flush_coalesced_mmio_buffer();
2735 }
2736
2737 #if defined(__linux__) && !defined(TARGET_S390X)
2738
2739 #include <sys/vfs.h>
2740
2741 #define HUGETLBFS_MAGIC 0x958458f6
2742
2743 static long gethugepagesize(const char *path)
2744 {
2745 struct statfs fs;
2746 int ret;
2747
2748 do {
2749 ret = statfs(path, &fs);
2750 } while (ret != 0 && errno == EINTR);
2751
2752 if (ret != 0) {
2753 perror(path);
2754 return 0;
2755 }
2756
2757 if (fs.f_type != HUGETLBFS_MAGIC)
2758 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
2759
2760 return fs.f_bsize;
2761 }
2762
2763 static void *file_ram_alloc(RAMBlock *block,
2764 ram_addr_t memory,
2765 const char *path)
2766 {
2767 char *filename;
2768 void *area;
2769 int fd;
2770 #ifdef MAP_POPULATE
2771 int flags;
2772 #endif
2773 unsigned long hpagesize;
2774
2775 hpagesize = gethugepagesize(path);
2776 if (!hpagesize) {
2777 return NULL;
2778 }
2779
2780 if (memory < hpagesize) {
2781 return NULL;
2782 }
2783
2784 if (kvm_enabled() && !kvm_has_sync_mmu()) {
2785 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
2786 return NULL;
2787 }
2788
2789 if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
2790 return NULL;
2791 }
2792
2793 fd = mkstemp(filename);
2794 if (fd < 0) {
2795 perror("unable to create backing store for hugepages");
2796 free(filename);
2797 return NULL;
2798 }
2799 unlink(filename);
2800 free(filename);
2801
2802 memory = (memory+hpagesize-1) & ~(hpagesize-1);
2803
2804 /*
2805 * ftruncate is not supported by hugetlbfs in older
2806 * hosts, so don't bother bailing out on errors.
2807 * If anything goes wrong with it under other filesystems,
2808 * mmap will fail.
2809 */
2810 if (ftruncate(fd, memory))
2811 perror("ftruncate");
2812
2813 #ifdef MAP_POPULATE
2814 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
2815 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
2816 * to sidestep this quirk.
2817 */
2818 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
2819 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
2820 #else
2821 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
2822 #endif
2823 if (area == MAP_FAILED) {
2824 perror("file_ram_alloc: can't mmap RAM pages");
2825 close(fd);
2826 return (NULL);
2827 }
2828 block->fd = fd;
2829 return area;
2830 }
2831 #endif
2832
2833 static ram_addr_t find_ram_offset(ram_addr_t size)
2834 {
2835 RAMBlock *block, *next_block;
2836 ram_addr_t offset = 0, mingap = ULONG_MAX;
2837
2838 if (QLIST_EMPTY(&ram_list.blocks))
2839 return 0;
2840
2841 QLIST_FOREACH(block, &ram_list.blocks, next) {
2842 ram_addr_t end, next = ULONG_MAX;
2843
2844 end = block->offset + block->length;
2845
2846 QLIST_FOREACH(next_block, &ram_list.blocks, next) {
2847 if (next_block->offset >= end) {
2848 next = MIN(next, next_block->offset);
2849 }
2850 }
2851 if (next - end >= size && next - end < mingap) {
2852 offset = end;
2853 mingap = next - end;
2854 }
2855 }
2856 return offset;
2857 }
2858
2859 static ram_addr_t last_ram_offset(void)
2860 {
2861 RAMBlock *block;
2862 ram_addr_t last = 0;
2863
2864 QLIST_FOREACH(block, &ram_list.blocks, next)
2865 last = MAX(last, block->offset + block->length);
2866
2867 return last;
2868 }
2869
2870 ram_addr_t qemu_ram_alloc_from_ptr(DeviceState *dev, const char *name,
2871 ram_addr_t size, void *host)
2872 {
2873 RAMBlock *new_block, *block;
2874
2875 size = TARGET_PAGE_ALIGN(size);
2876 new_block = qemu_mallocz(sizeof(*new_block));
2877
2878 if (dev && dev->parent_bus && dev->parent_bus->info->get_dev_path) {
2879 char *id = dev->parent_bus->info->get_dev_path(dev);
2880 if (id) {
2881 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
2882 qemu_free(id);
2883 }
2884 }
2885 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
2886
2887 QLIST_FOREACH(block, &ram_list.blocks, next) {
2888 if (!strcmp(block->idstr, new_block->idstr)) {
2889 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
2890 new_block->idstr);
2891 abort();
2892 }
2893 }
2894
2895 new_block->offset = find_ram_offset(size);
2896 if (host) {
2897 new_block->host = host;
2898 new_block->flags |= RAM_PREALLOC_MASK;
2899 } else {
2900 if (mem_path) {
2901 #if defined (__linux__) && !defined(TARGET_S390X)
2902 new_block->host = file_ram_alloc(new_block, size, mem_path);
2903 if (!new_block->host) {
2904 new_block->host = qemu_vmalloc(size);
2905 qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
2906 }
2907 #else
2908 fprintf(stderr, "-mem-path option unsupported\n");
2909 exit(1);
2910 #endif
2911 } else {
2912 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2913 /* XXX S390 KVM requires the topmost vma of the RAM to be < 256GB */
2914 new_block->host = mmap((void*)0x1000000, size,
2915 PROT_EXEC|PROT_READ|PROT_WRITE,
2916 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
2917 #else
2918 if (xen_mapcache_enabled()) {
2919 xen_ram_alloc(new_block->offset, size);
2920 } else {
2921 new_block->host = qemu_vmalloc(size);
2922 }
2923 #endif
2924 qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
2925 }
2926 }
2927 new_block->length = size;
2928
2929 QLIST_INSERT_HEAD(&ram_list.blocks, new_block, next);
2930
2931 ram_list.phys_dirty = qemu_realloc(ram_list.phys_dirty,
2932 last_ram_offset() >> TARGET_PAGE_BITS);
2933 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
2934 0xff, size >> TARGET_PAGE_BITS);
2935
2936 if (kvm_enabled())
2937 kvm_setup_guest_memory(new_block->host, size);
2938
2939 return new_block->offset;
2940 }
2941
2942 ram_addr_t qemu_ram_alloc(DeviceState *dev, const char *name, ram_addr_t size)
2943 {
2944 return qemu_ram_alloc_from_ptr(dev, name, size, NULL);
2945 }
2946
2947 void qemu_ram_free(ram_addr_t addr)
2948 {
2949 RAMBlock *block;
2950
2951 QLIST_FOREACH(block, &ram_list.blocks, next) {
2952 if (addr == block->offset) {
2953 QLIST_REMOVE(block, next);
2954 if (block->flags & RAM_PREALLOC_MASK) {
2955 ;
2956 } else if (mem_path) {
2957 #if defined (__linux__) && !defined(TARGET_S390X)
2958 if (block->fd) {
2959 munmap(block->host, block->length);
2960 close(block->fd);
2961 } else {
2962 qemu_vfree(block->host);
2963 }
2964 #else
2965 abort();
2966 #endif
2967 } else {
2968 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2969 munmap(block->host, block->length);
2970 #else
2971 if (xen_mapcache_enabled()) {
2972 qemu_invalidate_entry(block->host);
2973 } else {
2974 qemu_vfree(block->host);
2975 }
2976 #endif
2977 }
2978 qemu_free(block);
2979 return;
2980 }
2981 }
2982
2983 }
2984
2985 #ifndef _WIN32
2986 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
2987 {
2988 RAMBlock *block;
2989 ram_addr_t offset;
2990 int flags;
2991 void *area, *vaddr;
2992
2993 QLIST_FOREACH(block, &ram_list.blocks, next) {
2994 offset = addr - block->offset;
2995 if (offset < block->length) {
2996 vaddr = block->host + offset;
2997 if (block->flags & RAM_PREALLOC_MASK) {
2998 ;
2999 } else {
3000 flags = MAP_FIXED;
3001 munmap(vaddr, length);
3002 if (mem_path) {
3003 #if defined(__linux__) && !defined(TARGET_S390X)
3004 if (block->fd) {
3005 #ifdef MAP_POPULATE
3006 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
3007 MAP_PRIVATE;
3008 #else
3009 flags |= MAP_PRIVATE;
3010 #endif
3011 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
3012 flags, block->fd, offset);
3013 } else {
3014 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
3015 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
3016 flags, -1, 0);
3017 }
3018 #else
3019 abort();
3020 #endif
3021 } else {
3022 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
3023 flags |= MAP_SHARED | MAP_ANONYMOUS;
3024 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
3025 flags, -1, 0);
3026 #else
3027 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
3028 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
3029 flags, -1, 0);
3030 #endif
3031 }
3032 if (area != vaddr) {
3033 fprintf(stderr, "Could not remap addr: %lx@%lx\n",
3034 length, addr);
3035 exit(1);
3036 }
3037 qemu_madvise(vaddr, length, QEMU_MADV_MERGEABLE);
3038 }
3039 return;
3040 }
3041 }
3042 }
3043 #endif /* !_WIN32 */
3044
3045 /* Return a host pointer to ram allocated with qemu_ram_alloc.
3046 With the exception of the softmmu code in this file, this should
3047 only be used for local memory (e.g. video ram) that the device owns,
3048 and knows it isn't going to access beyond the end of the block.
3049
3050 It should not be used for general purpose DMA.
3051 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
3052 */
3053 void *qemu_get_ram_ptr(ram_addr_t addr)
3054 {
3055 RAMBlock *block;
3056
3057 QLIST_FOREACH(block, &ram_list.blocks, next) {
3058 if (addr - block->offset < block->length) {
3059 /* Move this entry to to start of the list. */
3060 if (block != QLIST_FIRST(&ram_list.blocks)) {
3061 QLIST_REMOVE(block, next);
3062 QLIST_INSERT_HEAD(&ram_list.blocks, block, next);
3063 }
3064 if (xen_mapcache_enabled()) {
3065 /* We need to check if the requested address is in the RAM
3066 * because we don't want to map the entire memory in QEMU.
3067 */
3068 if (block->offset == 0) {
3069 return qemu_map_cache(addr, 0, 1);
3070 } else if (block->host == NULL) {
3071 block->host = xen_map_block(block->offset, block->length);
3072 }
3073 }
3074 return block->host + (addr - block->offset);
3075 }
3076 }
3077
3078 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
3079 abort();
3080
3081 return NULL;
3082 }
3083
3084 /* Return a host pointer to ram allocated with qemu_ram_alloc.
3085 * Same as qemu_get_ram_ptr but avoid reordering ramblocks.
3086 */
3087 void *qemu_safe_ram_ptr(ram_addr_t addr)
3088 {
3089 RAMBlock *block;
3090
3091 QLIST_FOREACH(block, &ram_list.blocks, next) {
3092 if (addr - block->offset < block->length) {
3093 if (xen_mapcache_enabled()) {
3094 /* We need to check if the requested address is in the RAM
3095 * because we don't want to map the entire memory in QEMU.
3096 */
3097 if (block->offset == 0) {
3098 return qemu_map_cache(addr, 0, 1);
3099 } else if (block->host == NULL) {
3100 block->host = xen_map_block(block->offset, block->length);
3101 }
3102 }
3103 return block->host + (addr - block->offset);
3104 }
3105 }
3106
3107 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
3108 abort();
3109
3110 return NULL;
3111 }
3112
3113 void qemu_put_ram_ptr(void *addr)
3114 {
3115 trace_qemu_put_ram_ptr(addr);
3116
3117 if (xen_mapcache_enabled()) {
3118 RAMBlock *block;
3119
3120 QLIST_FOREACH(block, &ram_list.blocks, next) {
3121 if (addr == block->host) {
3122 break;
3123 }
3124 }
3125 if (block && block->host) {
3126 xen_unmap_block(block->host, block->length);
3127 block->host = NULL;
3128 } else {
3129 qemu_map_cache_unlock(addr);
3130 }
3131 }
3132 }
3133
3134 int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
3135 {
3136 RAMBlock *block;
3137 uint8_t *host = ptr;
3138
3139 QLIST_FOREACH(block, &ram_list.blocks, next) {
3140 /* This case append when the block is not mapped. */
3141 if (block->host == NULL) {
3142 continue;
3143 }
3144 if (host - block->host < block->length) {
3145 *ram_addr = block->offset + (host - block->host);
3146 return 0;
3147 }
3148 }
3149
3150 if (xen_mapcache_enabled()) {
3151 *ram_addr = qemu_ram_addr_from_mapcache(ptr);
3152 return 0;
3153 }
3154
3155 return -1;
3156 }
3157
3158 /* Some of the softmmu routines need to translate from a host pointer
3159 (typically a TLB entry) back to a ram offset. */
3160 ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
3161 {
3162 ram_addr_t ram_addr;
3163
3164 if (qemu_ram_addr_from_host(ptr, &ram_addr)) {
3165 fprintf(stderr, "Bad ram pointer %p\n", ptr);
3166 abort();
3167 }
3168 return ram_addr;
3169 }
3170
3171 static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
3172 {
3173 #ifdef DEBUG_UNASSIGNED
3174 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
3175 #endif
3176 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3177 do_unassigned_access(addr, 0, 0, 0, 1);
3178 #endif
3179 return 0;
3180 }
3181
3182 static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
3183 {
3184 #ifdef DEBUG_UNASSIGNED
3185 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
3186 #endif
3187 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3188 do_unassigned_access(addr, 0, 0, 0, 2);
3189 #endif
3190 return 0;
3191 }
3192
3193 static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
3194 {
3195 #ifdef DEBUG_UNASSIGNED
3196 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
3197 #endif
3198 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3199 do_unassigned_access(addr, 0, 0, 0, 4);
3200 #endif
3201 return 0;
3202 }
3203
3204 static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
3205 {
3206 #ifdef DEBUG_UNASSIGNED
3207 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3208 #endif
3209 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3210 do_unassigned_access(addr, 1, 0, 0, 1);
3211 #endif
3212 }
3213
3214 static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
3215 {
3216 #ifdef DEBUG_UNASSIGNED
3217 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3218 #endif
3219 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3220 do_unassigned_access(addr, 1, 0, 0, 2);
3221 #endif
3222 }
3223
3224 static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
3225 {
3226 #ifdef DEBUG_UNASSIGNED
3227 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3228 #endif
3229 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3230 do_unassigned_access(addr, 1, 0, 0, 4);
3231 #endif
3232 }
3233
3234 static CPUReadMemoryFunc * const unassigned_mem_read[3] = {
3235 unassigned_mem_readb,
3236 unassigned_mem_readw,
3237 unassigned_mem_readl,
3238 };
3239
3240 static CPUWriteMemoryFunc * const unassigned_mem_write[3] = {
3241 unassigned_mem_writeb,
3242 unassigned_mem_writew,
3243 unassigned_mem_writel,
3244 };
3245
3246 static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
3247 uint32_t val)
3248 {
3249 int dirty_flags;
3250 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3251 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3252 #if !defined(CONFIG_USER_ONLY)
3253 tb_invalidate_phys_page_fast(ram_addr, 1);
3254 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3255 #endif
3256 }
3257 stb_p(qemu_get_ram_ptr(ram_addr), val);
3258 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3259 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
3260 /* we remove the notdirty callback only if the code has been
3261 flushed */
3262 if (dirty_flags == 0xff)
3263 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3264 }
3265
3266 static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
3267 uint32_t val)
3268 {
3269 int dirty_flags;
3270 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3271 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3272 #if !defined(CONFIG_USER_ONLY)
3273 tb_invalidate_phys_page_fast(ram_addr, 2);
3274 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3275 #endif
3276 }
3277 stw_p(qemu_get_ram_ptr(ram_addr), val);
3278 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3279 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
3280 /* we remove the notdirty callback only if the code has been
3281 flushed */
3282 if (dirty_flags == 0xff)
3283 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3284 }
3285
3286 static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
3287 uint32_t val)
3288 {
3289 int dirty_flags;
3290 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3291 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3292 #if !defined(CONFIG_USER_ONLY)
3293 tb_invalidate_phys_page_fast(ram_addr, 4);
3294 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3295 #endif
3296 }
3297 stl_p(qemu_get_ram_ptr(ram_addr), val);
3298 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3299 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
3300 /* we remove the notdirty callback only if the code has been
3301 flushed */
3302 if (dirty_flags == 0xff)
3303 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3304 }
3305
3306 static CPUReadMemoryFunc * const error_mem_read[3] = {
3307 NULL, /* never used */
3308 NULL, /* never used */
3309 NULL, /* never used */
3310 };
3311
3312 static CPUWriteMemoryFunc * const notdirty_mem_write[3] = {
3313 notdirty_mem_writeb,
3314 notdirty_mem_writew,
3315 notdirty_mem_writel,
3316 };
3317
3318 /* Generate a debug exception if a watchpoint has been hit. */
3319 static void check_watchpoint(int offset, int len_mask, int flags)
3320 {
3321 CPUState *env = cpu_single_env;
3322 target_ulong pc, cs_base;
3323 TranslationBlock *tb;
3324 target_ulong vaddr;
3325 CPUWatchpoint *wp;
3326 int cpu_flags;
3327
3328 if (env->watchpoint_hit) {
3329 /* We re-entered the check after replacing the TB. Now raise
3330 * the debug interrupt so that is will trigger after the
3331 * current instruction. */
3332 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
3333 return;
3334 }
3335 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
3336 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
3337 if ((vaddr == (wp->vaddr & len_mask) ||
3338 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
3339 wp->flags |= BP_WATCHPOINT_HIT;
3340 if (!env->watchpoint_hit) {
3341 env->watchpoint_hit = wp;
3342 tb = tb_find_pc(env->mem_io_pc);
3343 if (!tb) {
3344 cpu_abort(env, "check_watchpoint: could not find TB for "
3345 "pc=%p", (void *)env->mem_io_pc);
3346 }
3347 cpu_restore_state(tb, env, env->mem_io_pc);
3348 tb_phys_invalidate(tb, -1);
3349 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
3350 env->exception_index = EXCP_DEBUG;
3351 } else {
3352 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
3353 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
3354 }
3355 cpu_resume_from_signal(env, NULL);
3356 }
3357 } else {
3358 wp->flags &= ~BP_WATCHPOINT_HIT;
3359 }
3360 }
3361 }
3362
3363 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
3364 so these check for a hit then pass through to the normal out-of-line
3365 phys routines. */
3366 static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
3367 {
3368 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
3369 return ldub_phys(addr);
3370 }
3371
3372 static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
3373 {
3374 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
3375 return lduw_phys(addr);
3376 }
3377
3378 static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
3379 {
3380 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
3381 return ldl_phys(addr);
3382 }
3383
3384 static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
3385 uint32_t val)
3386 {
3387 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
3388 stb_phys(addr, val);
3389 }
3390
3391 static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
3392 uint32_t val)
3393 {
3394 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
3395 stw_phys(addr, val);
3396 }
3397
3398 static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
3399 uint32_t val)
3400 {
3401 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
3402 stl_phys(addr, val);
3403 }
3404
3405 static CPUReadMemoryFunc * const watch_mem_read[3] = {
3406 watch_mem_readb,
3407 watch_mem_readw,
3408 watch_mem_readl,
3409 };
3410
3411 static CPUWriteMemoryFunc * const watch_mem_write[3] = {
3412 watch_mem_writeb,
3413 watch_mem_writew,
3414 watch_mem_writel,
3415 };
3416
3417 static inline uint32_t subpage_readlen (subpage_t *mmio,
3418 target_phys_addr_t addr,
3419 unsigned int len)
3420 {
3421 unsigned int idx = SUBPAGE_IDX(addr);
3422 #if defined(DEBUG_SUBPAGE)
3423 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
3424 mmio, len, addr, idx);
3425 #endif
3426
3427 addr += mmio->region_offset[idx];
3428 idx = mmio->sub_io_index[idx];
3429 return io_mem_read[idx][len](io_mem_opaque[idx], addr);
3430 }
3431
3432 static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
3433 uint32_t value, unsigned int len)
3434 {
3435 unsigned int idx = SUBPAGE_IDX(addr);
3436 #if defined(DEBUG_SUBPAGE)
3437 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n",
3438 __func__, mmio, len, addr, idx, value);
3439 #endif
3440
3441 addr += mmio->region_offset[idx];
3442 idx = mmio->sub_io_index[idx];
3443 io_mem_write[idx][len](io_mem_opaque[idx], addr, value);
3444 }
3445
3446 static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
3447 {
3448 return subpage_readlen(opaque, addr, 0);
3449 }
3450
3451 static void subpage_writeb (void *opaque, target_phys_addr_t addr,
3452 uint32_t value)
3453 {
3454 subpage_writelen(opaque, addr, value, 0);
3455 }
3456
3457 static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
3458 {
3459 return subpage_readlen(opaque, addr, 1);
3460 }
3461
3462 static void subpage_writew (void *opaque, target_phys_addr_t addr,
3463 uint32_t value)
3464 {
3465 subpage_writelen(opaque, addr, value, 1);
3466 }
3467
3468 static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
3469 {
3470 return subpage_readlen(opaque, addr, 2);
3471 }
3472
3473 static void subpage_writel (void *opaque, target_phys_addr_t addr,
3474 uint32_t value)
3475 {
3476 subpage_writelen(opaque, addr, value, 2);
3477 }
3478
3479 static CPUReadMemoryFunc * const subpage_read[] = {
3480 &subpage_readb,
3481 &subpage_readw,
3482 &subpage_readl,
3483 };
3484
3485 static CPUWriteMemoryFunc * const subpage_write[] = {
3486 &subpage_writeb,
3487 &subpage_writew,
3488 &subpage_writel,
3489 };
3490
3491 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
3492 ram_addr_t memory, ram_addr_t region_offset)
3493 {
3494 int idx, eidx;
3495
3496 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
3497 return -1;
3498 idx = SUBPAGE_IDX(start);
3499 eidx = SUBPAGE_IDX(end);
3500 #if defined(DEBUG_SUBPAGE)
3501 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
3502 mmio, start, end, idx, eidx, memory);
3503 #endif
3504 if ((memory & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
3505 memory = IO_MEM_UNASSIGNED;
3506 memory = (memory >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3507 for (; idx <= eidx; idx++) {
3508 mmio->sub_io_index[idx] = memory;
3509 mmio->region_offset[idx] = region_offset;
3510 }
3511
3512 return 0;
3513 }
3514
3515 static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
3516 ram_addr_t orig_memory,
3517 ram_addr_t region_offset)
3518 {
3519 subpage_t *mmio;
3520 int subpage_memory;
3521
3522 mmio = qemu_mallocz(sizeof(subpage_t));
3523
3524 mmio->base = base;
3525 subpage_memory = cpu_register_io_memory(subpage_read, subpage_write, mmio,
3526 DEVICE_NATIVE_ENDIAN);
3527 #if defined(DEBUG_SUBPAGE)
3528 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
3529 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
3530 #endif
3531 *phys = subpage_memory | IO_MEM_SUBPAGE;
3532 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, orig_memory, region_offset);
3533
3534 return mmio;
3535 }
3536
3537 static int get_free_io_mem_idx(void)
3538 {
3539 int i;
3540
3541 for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
3542 if (!io_mem_used[i]) {
3543 io_mem_used[i] = 1;
3544 return i;
3545 }
3546 fprintf(stderr, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES);
3547 return -1;
3548 }
3549
3550 /*
3551 * Usually, devices operate in little endian mode. There are devices out
3552 * there that operate in big endian too. Each device gets byte swapped
3553 * mmio if plugged onto a CPU that does the other endianness.
3554 *
3555 * CPU Device swap?
3556 *
3557 * little little no
3558 * little big yes
3559 * big little yes
3560 * big big no
3561 */
3562
3563 typedef struct SwapEndianContainer {
3564 CPUReadMemoryFunc *read[3];
3565 CPUWriteMemoryFunc *write[3];
3566 void *opaque;
3567 } SwapEndianContainer;
3568
3569 static uint32_t swapendian_mem_readb (void *opaque, target_phys_addr_t addr)
3570 {
3571 uint32_t val;
3572 SwapEndianContainer *c = opaque;
3573 val = c->read[0](c->opaque, addr);
3574 return val;
3575 }
3576
3577 static uint32_t swapendian_mem_readw(void *opaque, target_phys_addr_t addr)
3578 {
3579 uint32_t val;
3580 SwapEndianContainer *c = opaque;
3581 val = bswap16(c->read[1](c->opaque, addr));
3582 return val;
3583 }
3584
3585 static uint32_t swapendian_mem_readl(void *opaque, target_phys_addr_t addr)
3586 {
3587 uint32_t val;
3588 SwapEndianContainer *c = opaque;
3589 val = bswap32(c->read[2](c->opaque, addr));
3590 return val;
3591 }
3592
3593 static CPUReadMemoryFunc * const swapendian_readfn[3]={
3594 swapendian_mem_readb,
3595 swapendian_mem_readw,
3596 swapendian_mem_readl
3597 };
3598
3599 static void swapendian_mem_writeb(void *opaque, target_phys_addr_t addr,
3600 uint32_t val)
3601 {
3602 SwapEndianContainer *c = opaque;
3603 c->write[0](c->opaque, addr, val);
3604 }
3605
3606 static void swapendian_mem_writew(void *opaque, target_phys_addr_t addr,
3607 uint32_t val)
3608 {
3609 SwapEndianContainer *c = opaque;
3610 c->write[1](c->opaque, addr, bswap16(val));
3611 }
3612
3613 static void swapendian_mem_writel(void *opaque, target_phys_addr_t addr,
3614 uint32_t val)
3615 {
3616 SwapEndianContainer *c = opaque;
3617 c->write[2](c->opaque, addr, bswap32(val));
3618 }
3619
3620 static CPUWriteMemoryFunc * const swapendian_writefn[3]={
3621 swapendian_mem_writeb,
3622 swapendian_mem_writew,
3623 swapendian_mem_writel
3624 };
3625
3626 static void swapendian_init(int io_index)
3627 {
3628 SwapEndianContainer *c = qemu_malloc(sizeof(SwapEndianContainer));
3629 int i;
3630
3631 /* Swap mmio for big endian targets */
3632 c->opaque = io_mem_opaque[io_index];
3633 for (i = 0; i < 3; i++) {
3634 c->read[i] = io_mem_read[io_index][i];
3635 c->write[i] = io_mem_write[io_index][i];
3636
3637 io_mem_read[io_index][i] = swapendian_readfn[i];
3638 io_mem_write[io_index][i] = swapendian_writefn[i];
3639 }
3640 io_mem_opaque[io_index] = c;
3641 }
3642
3643 static void swapendian_del(int io_index)
3644 {
3645 if (io_mem_read[io_index][0] == swapendian_readfn[0]) {
3646 qemu_free(io_mem_opaque[io_index]);
3647 }
3648 }
3649
3650 /* mem_read and mem_write are arrays of functions containing the
3651 function to access byte (index 0), word (index 1) and dword (index
3652 2). Functions can be omitted with a NULL function pointer.
3653 If io_index is non zero, the corresponding io zone is
3654 modified. If it is zero, a new io zone is allocated. The return
3655 value can be used with cpu_register_physical_memory(). (-1) is
3656 returned if error. */
3657 static int cpu_register_io_memory_fixed(int io_index,
3658 CPUReadMemoryFunc * const *mem_read,
3659 CPUWriteMemoryFunc * const *mem_write,
3660 void *opaque, enum device_endian endian)
3661 {
3662 int i;
3663
3664 if (io_index <= 0) {
3665 io_index = get_free_io_mem_idx();
3666 if (io_index == -1)
3667 return io_index;
3668 } else {
3669 io_index >>= IO_MEM_SHIFT;
3670 if (io_index >= IO_MEM_NB_ENTRIES)
3671 return -1;
3672 }
3673
3674 for (i = 0; i < 3; ++i) {
3675 io_mem_read[io_index][i]
3676 = (mem_read[i] ? mem_read[i] : unassigned_mem_read[i]);
3677 }
3678 for (i = 0; i < 3; ++i) {
3679 io_mem_write[io_index][i]
3680 = (mem_write[i] ? mem_write[i] : unassigned_mem_write[i]);
3681 }
3682 io_mem_opaque[io_index] = opaque;
3683
3684 switch (endian) {
3685 case DEVICE_BIG_ENDIAN:
3686 #ifndef TARGET_WORDS_BIGENDIAN
3687 swapendian_init(io_index);
3688 #endif
3689 break;
3690 case DEVICE_LITTLE_ENDIAN:
3691 #ifdef TARGET_WORDS_BIGENDIAN
3692 swapendian_init(io_index);
3693 #endif
3694 break;
3695 case DEVICE_NATIVE_ENDIAN:
3696 default:
3697 break;
3698 }
3699
3700 return (io_index << IO_MEM_SHIFT);
3701 }
3702
3703 int cpu_register_io_memory(CPUReadMemoryFunc * const *mem_read,
3704 CPUWriteMemoryFunc * const *mem_write,
3705 void *opaque, enum device_endian endian)
3706 {
3707 return cpu_register_io_memory_fixed(0, mem_read, mem_write, opaque, endian);
3708 }
3709
3710 void cpu_unregister_io_memory(int io_table_address)
3711 {
3712 int i;
3713 int io_index = io_table_address >> IO_MEM_SHIFT;
3714
3715 swapendian_del(io_index);
3716
3717 for (i=0;i < 3; i++) {
3718 io_mem_read[io_index][i] = unassigned_mem_read[i];
3719 io_mem_write[io_index][i] = unassigned_mem_write[i];
3720 }
3721 io_mem_opaque[io_index] = NULL;
3722 io_mem_used[io_index] = 0;
3723 }
3724
3725 static void io_mem_init(void)
3726 {
3727 int i;
3728
3729 cpu_register_io_memory_fixed(IO_MEM_ROM, error_mem_read,
3730 unassigned_mem_write, NULL,
3731 DEVICE_NATIVE_ENDIAN);
3732 cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED, unassigned_mem_read,
3733 unassigned_mem_write, NULL,
3734 DEVICE_NATIVE_ENDIAN);
3735 cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY, error_mem_read,
3736 notdirty_mem_write, NULL,
3737 DEVICE_NATIVE_ENDIAN);
3738 for (i=0; i<5; i++)
3739 io_mem_used[i] = 1;
3740
3741 io_mem_watch = cpu_register_io_memory(watch_mem_read,
3742 watch_mem_write, NULL,
3743 DEVICE_NATIVE_ENDIAN);
3744 }
3745
3746 #endif /* !defined(CONFIG_USER_ONLY) */
3747
3748 /* physical memory access (slow version, mainly for debug) */
3749 #if defined(CONFIG_USER_ONLY)
3750 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3751 uint8_t *buf, int len, int is_write)
3752 {
3753 int l, flags;
3754 target_ulong page;
3755 void * p;
3756
3757 while (len > 0) {
3758 page = addr & TARGET_PAGE_MASK;
3759 l = (page + TARGET_PAGE_SIZE) - addr;
3760 if (l > len)
3761 l = len;
3762 flags = page_get_flags(page);
3763 if (!(flags & PAGE_VALID))
3764 return -1;
3765 if (is_write) {
3766 if (!(flags & PAGE_WRITE))
3767 return -1;
3768 /* XXX: this code should not depend on lock_user */
3769 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
3770 return -1;
3771 memcpy(p, buf, l);
3772 unlock_user(p, addr, l);
3773 } else {
3774 if (!(flags & PAGE_READ))
3775 return -1;
3776 /* XXX: this code should not depend on lock_user */
3777 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
3778 return -1;
3779 memcpy(buf, p, l);
3780 unlock_user(p, addr, 0);
3781 }
3782 len -= l;
3783 buf += l;
3784 addr += l;
3785 }
3786 return 0;
3787 }
3788
3789 #else
3790 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
3791 int len, int is_write)
3792 {
3793 int l, io_index;
3794 uint8_t *ptr;
3795 uint32_t val;
3796 target_phys_addr_t page;
3797 unsigned long pd;
3798 PhysPageDesc *p;
3799
3800 while (len > 0) {
3801 page = addr & TARGET_PAGE_MASK;
3802 l = (page + TARGET_PAGE_SIZE) - addr;
3803 if (l > len)
3804 l = len;
3805 p = phys_page_find(page >> TARGET_PAGE_BITS);
3806 if (!p) {
3807 pd = IO_MEM_UNASSIGNED;
3808 } else {
3809 pd = p->phys_offset;
3810 }
3811
3812 if (is_write) {
3813 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3814 target_phys_addr_t addr1 = addr;
3815 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3816 if (p)
3817 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3818 /* XXX: could force cpu_single_env to NULL to avoid
3819 potential bugs */
3820 if (l >= 4 && ((addr1 & 3) == 0)) {
3821 /* 32 bit write access */
3822 val = ldl_p(buf);
3823 io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
3824 l = 4;
3825 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3826 /* 16 bit write access */
3827 val = lduw_p(buf);
3828 io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
3829 l = 2;
3830 } else {
3831 /* 8 bit write access */
3832 val = ldub_p(buf);
3833 io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
3834 l = 1;
3835 }
3836 } else {
3837 unsigned long addr1;
3838 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3839 /* RAM case */
3840 ptr = qemu_get_ram_ptr(addr1);
3841 memcpy(ptr, buf, l);
3842 if (!cpu_physical_memory_is_dirty(addr1)) {
3843 /* invalidate code */
3844 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3845 /* set dirty bit */
3846 cpu_physical_memory_set_dirty_flags(
3847 addr1, (0xff & ~CODE_DIRTY_FLAG));
3848 }
3849 qemu_put_ram_ptr(ptr);
3850 }
3851 } else {
3852 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3853 !(pd & IO_MEM_ROMD)) {
3854 target_phys_addr_t addr1 = addr;
3855 /* I/O case */
3856 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3857 if (p)
3858 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3859 if (l >= 4 && ((addr1 & 3) == 0)) {
3860 /* 32 bit read access */
3861 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
3862 stl_p(buf, val);
3863 l = 4;
3864 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3865 /* 16 bit read access */
3866 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
3867 stw_p(buf, val);
3868 l = 2;
3869 } else {
3870 /* 8 bit read access */
3871 val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
3872 stb_p(buf, val);
3873 l = 1;
3874 }
3875 } else {
3876 /* RAM case */
3877 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
3878 memcpy(buf, ptr + (addr & ~TARGET_PAGE_MASK), l);
3879 qemu_put_ram_ptr(ptr);
3880 }
3881 }
3882 len -= l;
3883 buf += l;
3884 addr += l;
3885 }
3886 }
3887
3888 /* used for ROM loading : can write in RAM and ROM */
3889 void cpu_physical_memory_write_rom(target_phys_addr_t addr,
3890 const uint8_t *buf, int len)
3891 {
3892 int l;
3893 uint8_t *ptr;
3894 target_phys_addr_t page;
3895 unsigned long pd;
3896 PhysPageDesc *p;
3897
3898 while (len > 0) {
3899 page = addr & TARGET_PAGE_MASK;
3900 l = (page + TARGET_PAGE_SIZE) - addr;
3901 if (l > len)
3902 l = len;
3903 p = phys_page_find(page >> TARGET_PAGE_BITS);
3904 if (!p) {
3905 pd = IO_MEM_UNASSIGNED;
3906 } else {
3907 pd = p->phys_offset;
3908 }
3909
3910 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3911 (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3912 !(pd & IO_MEM_ROMD)) {
3913 /* do nothing */
3914 } else {
3915 unsigned long addr1;
3916 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3917 /* ROM/RAM case */
3918 ptr = qemu_get_ram_ptr(addr1);
3919 memcpy(ptr, buf, l);
3920 qemu_put_ram_ptr(ptr);
3921 }
3922 len -= l;
3923 buf += l;
3924 addr += l;
3925 }
3926 }
3927
3928 typedef struct {
3929 void *buffer;
3930 target_phys_addr_t addr;
3931 target_phys_addr_t len;
3932 } BounceBuffer;
3933
3934 static BounceBuffer bounce;
3935
3936 typedef struct MapClient {
3937 void *opaque;
3938 void (*callback)(void *opaque);
3939 QLIST_ENTRY(MapClient) link;
3940 } MapClient;
3941
3942 static QLIST_HEAD(map_client_list, MapClient) map_client_list
3943 = QLIST_HEAD_INITIALIZER(map_client_list);
3944
3945 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
3946 {
3947 MapClient *client = qemu_malloc(sizeof(*client));
3948
3949 client->opaque = opaque;
3950 client->callback = callback;
3951 QLIST_INSERT_HEAD(&map_client_list, client, link);
3952 return client;
3953 }
3954
3955 void cpu_unregister_map_client(void *_client)
3956 {
3957 MapClient *client = (MapClient *)_client;
3958
3959 QLIST_REMOVE(client, link);
3960 qemu_free(client);
3961 }
3962
3963 static void cpu_notify_map_clients(void)
3964 {
3965 MapClient *client;
3966
3967 while (!QLIST_EMPTY(&map_client_list)) {
3968 client = QLIST_FIRST(&map_client_list);
3969 client->callback(client->opaque);
3970 cpu_unregister_map_client(client);
3971 }
3972 }
3973
3974 /* Map a physical memory region into a host virtual address.
3975 * May map a subset of the requested range, given by and returned in *plen.
3976 * May return NULL if resources needed to perform the mapping are exhausted.
3977 * Use only for reads OR writes - not for read-modify-write operations.
3978 * Use cpu_register_map_client() to know when retrying the map operation is
3979 * likely to succeed.
3980 */
3981 void *cpu_physical_memory_map(target_phys_addr_t addr,
3982 target_phys_addr_t *plen,
3983 int is_write)
3984 {
3985 target_phys_addr_t len = *plen;
3986 target_phys_addr_t done = 0;
3987 int l;
3988 uint8_t *ret = NULL;
3989 uint8_t *ptr;
3990 target_phys_addr_t page;
3991 unsigned long pd;
3992 PhysPageDesc *p;
3993 unsigned long addr1;
3994
3995 while (len > 0) {
3996 page = addr & TARGET_PAGE_MASK;
3997 l = (page + TARGET_PAGE_SIZE) - addr;
3998 if (l > len)
3999 l = len;
4000 p = phys_page_find(page >> TARGET_PAGE_BITS);
4001 if (!p) {
4002 pd = IO_MEM_UNASSIGNED;
4003 } else {
4004 pd = p->phys_offset;
4005 }
4006
4007 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4008 if (done || bounce.buffer) {
4009 break;
4010 }
4011 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
4012 bounce.addr = addr;
4013 bounce.len = l;
4014 if (!is_write) {
4015 cpu_physical_memory_read(addr, bounce.buffer, l);
4016 }
4017 ptr = bounce.buffer;
4018 } else {
4019 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4020 ptr = qemu_get_ram_ptr(addr1);
4021 }
4022 if (!done) {
4023 ret = ptr;
4024 } else if (ret + done != ptr) {
4025 break;
4026 }
4027
4028 len -= l;
4029 addr += l;
4030 done += l;
4031 }
4032 *plen = done;
4033 return ret;
4034 }
4035
4036 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
4037 * Will also mark the memory as dirty if is_write == 1. access_len gives
4038 * the amount of memory that was actually read or written by the caller.
4039 */
4040 void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
4041 int is_write, target_phys_addr_t access_len)
4042 {
4043 if (buffer != bounce.buffer) {
4044 if (is_write) {
4045 ram_addr_t addr1 = qemu_ram_addr_from_host_nofail(buffer);
4046 while (access_len) {
4047 unsigned l;
4048 l = TARGET_PAGE_SIZE;
4049 if (l > access_len)
4050 l = access_len;
4051 if (!cpu_physical_memory_is_dirty(addr1)) {
4052 /* invalidate code */
4053 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
4054 /* set dirty bit */
4055 cpu_physical_memory_set_dirty_flags(
4056 addr1, (0xff & ~CODE_DIRTY_FLAG));
4057 }
4058 addr1 += l;
4059 access_len -= l;
4060 }
4061 }
4062 if (xen_mapcache_enabled()) {
4063 uint8_t *buffer1 = buffer;
4064 uint8_t *end_buffer = buffer + len;
4065
4066 while (buffer1 < end_buffer) {
4067 qemu_put_ram_ptr(buffer1);
4068 buffer1 += TARGET_PAGE_SIZE;
4069 }
4070 }
4071 return;
4072 }
4073 if (is_write) {
4074 cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
4075 }
4076 qemu_vfree(bounce.buffer);
4077 bounce.buffer = NULL;
4078 cpu_notify_map_clients();
4079 }
4080
4081 /* warning: addr must be aligned */
4082 uint32_t ldl_phys(target_phys_addr_t addr)
4083 {
4084 int io_index;
4085 uint8_t *ptr;
4086 uint32_t val;
4087 unsigned long pd;
4088 PhysPageDesc *p;
4089
4090 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4091 if (!p) {
4092 pd = IO_MEM_UNASSIGNED;
4093 } else {
4094 pd = p->phys_offset;
4095 }
4096
4097 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
4098 !(pd & IO_MEM_ROMD)) {
4099 /* I/O case */
4100 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4101 if (p)
4102 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4103 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
4104 } else {
4105 /* RAM case */
4106 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
4107 (addr & ~TARGET_PAGE_MASK);
4108 val = ldl_p(ptr);
4109 }
4110 return val;
4111 }
4112
4113 /* warning: addr must be aligned */
4114 uint64_t ldq_phys(target_phys_addr_t addr)
4115 {
4116 int io_index;
4117 uint8_t *ptr;
4118 uint64_t val;
4119 unsigned long pd;
4120 PhysPageDesc *p;
4121
4122 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4123 if (!p) {
4124 pd = IO_MEM_UNASSIGNED;
4125 } else {
4126 pd = p->phys_offset;
4127 }
4128
4129 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
4130 !(pd & IO_MEM_ROMD)) {
4131 /* I/O case */
4132 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4133 if (p)
4134 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4135 #ifdef TARGET_WORDS_BIGENDIAN
4136 val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
4137 val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
4138 #else
4139 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
4140 val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
4141 #endif
4142 } else {
4143 /* RAM case */
4144 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
4145 (addr & ~TARGET_PAGE_MASK);
4146 val = ldq_p(ptr);
4147 }
4148 return val;
4149 }
4150
4151 /* XXX: optimize */
4152 uint32_t ldub_phys(target_phys_addr_t addr)
4153 {
4154 uint8_t val;
4155 cpu_physical_memory_read(addr, &val, 1);
4156 return val;
4157 }
4158
4159 /* warning: addr must be aligned */
4160 uint32_t lduw_phys(target_phys_addr_t addr)
4161 {
4162 int io_index;
4163 uint8_t *ptr;
4164 uint64_t val;
4165 unsigned long pd;
4166 PhysPageDesc *p;
4167
4168 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4169 if (!p) {
4170 pd = IO_MEM_UNASSIGNED;
4171 } else {
4172 pd = p->phys_offset;
4173 }
4174
4175 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
4176 !(pd & IO_MEM_ROMD)) {
4177 /* I/O case */
4178 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4179 if (p)
4180 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4181 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr);
4182 } else {
4183 /* RAM case */
4184 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
4185 (addr & ~TARGET_PAGE_MASK);
4186 val = lduw_p(ptr);
4187 }
4188 return val;
4189 }
4190
4191 /* warning: addr must be aligned. The ram page is not masked as dirty
4192 and the code inside is not invalidated. It is useful if the dirty
4193 bits are used to track modified PTEs */
4194 void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
4195 {
4196 int io_index;
4197 uint8_t *ptr;
4198 unsigned long pd;
4199 PhysPageDesc *p;
4200
4201 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4202 if (!p) {
4203 pd = IO_MEM_UNASSIGNED;
4204 } else {
4205 pd = p->phys_offset;
4206 }
4207
4208 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4209 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4210 if (p)
4211 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4212 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4213 } else {
4214 unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4215 ptr = qemu_get_ram_ptr(addr1);
4216 stl_p(ptr, val);
4217
4218 if (unlikely(in_migration)) {
4219 if (!cpu_physical_memory_is_dirty(addr1)) {
4220 /* invalidate code */
4221 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
4222 /* set dirty bit */
4223 cpu_physical_memory_set_dirty_flags(
4224 addr1, (0xff & ~CODE_DIRTY_FLAG));
4225 }
4226 }
4227 }
4228 }
4229
4230 void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
4231 {
4232 int io_index;
4233 uint8_t *ptr;
4234 unsigned long pd;
4235 PhysPageDesc *p;
4236
4237 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4238 if (!p) {
4239 pd = IO_MEM_UNASSIGNED;
4240 } else {
4241 pd = p->phys_offset;
4242 }
4243
4244 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4245 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4246 if (p)
4247 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4248 #ifdef TARGET_WORDS_BIGENDIAN
4249 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
4250 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
4251 #else
4252 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4253 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
4254 #endif
4255 } else {
4256 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
4257 (addr & ~TARGET_PAGE_MASK);
4258 stq_p(ptr, val);
4259 }
4260 }
4261
4262 /* warning: addr must be aligned */
4263 void stl_phys(target_phys_addr_t addr, uint32_t val)
4264 {
4265 int io_index;
4266 uint8_t *ptr;
4267 unsigned long pd;
4268 PhysPageDesc *p;
4269
4270 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4271 if (!p) {
4272 pd = IO_MEM_UNASSIGNED;
4273 } else {
4274 pd = p->phys_offset;
4275 }
4276
4277 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4278 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4279 if (p)
4280 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4281 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4282 } else {
4283 unsigned long addr1;
4284 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4285 /* RAM case */
4286 ptr = qemu_get_ram_ptr(addr1);
4287 stl_p(ptr, val);
4288 if (!cpu_physical_memory_is_dirty(addr1)) {
4289 /* invalidate code */
4290 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
4291 /* set dirty bit */
4292 cpu_physical_memory_set_dirty_flags(addr1,
4293 (0xff & ~CODE_DIRTY_FLAG));
4294 }
4295 }
4296 }
4297
4298 /* XXX: optimize */
4299 void stb_phys(target_phys_addr_t addr, uint32_t val)
4300 {
4301 uint8_t v = val;
4302 cpu_physical_memory_write(addr, &v, 1);
4303 }
4304
4305 /* warning: addr must be aligned */
4306 void stw_phys(target_phys_addr_t addr, uint32_t val)
4307 {
4308 int io_index;
4309 uint8_t *ptr;
4310 unsigned long pd;
4311 PhysPageDesc *p;
4312
4313 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4314 if (!p) {
4315 pd = IO_MEM_UNASSIGNED;
4316 } else {
4317 pd = p->phys_offset;
4318 }
4319
4320 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4321 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4322 if (p)
4323 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4324 io_mem_write[io_index][1](io_mem_opaque[io_index], addr, val);
4325 } else {
4326 unsigned long addr1;
4327 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4328 /* RAM case */
4329 ptr = qemu_get_ram_ptr(addr1);
4330 stw_p(ptr, val);
4331 if (!cpu_physical_memory_is_dirty(addr1)) {
4332 /* invalidate code */
4333 tb_invalidate_phys_page_range(addr1, addr1 + 2, 0);
4334 /* set dirty bit */
4335 cpu_physical_memory_set_dirty_flags(addr1,
4336 (0xff & ~CODE_DIRTY_FLAG));
4337 }
4338 }
4339 }
4340
4341 /* XXX: optimize */
4342 void stq_phys(target_phys_addr_t addr, uint64_t val)
4343 {
4344 val = tswap64(val);
4345 cpu_physical_memory_write(addr, &val, 8);
4346 }
4347
4348 /* virtual memory access for debug (includes writing to ROM) */
4349 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
4350 uint8_t *buf, int len, int is_write)
4351 {
4352 int l;
4353 target_phys_addr_t phys_addr;
4354 target_ulong page;
4355
4356 while (len > 0) {
4357 page = addr & TARGET_PAGE_MASK;
4358 phys_addr = cpu_get_phys_page_debug(env, page);
4359 /* if no physical page mapped, return an error */
4360 if (phys_addr == -1)
4361 return -1;
4362 l = (page + TARGET_PAGE_SIZE) - addr;
4363 if (l > len)
4364 l = len;
4365 phys_addr += (addr & ~TARGET_PAGE_MASK);
4366 if (is_write)
4367 cpu_physical_memory_write_rom(phys_addr, buf, l);
4368 else
4369 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
4370 len -= l;
4371 buf += l;
4372 addr += l;
4373 }
4374 return 0;
4375 }
4376 #endif
4377
4378 /* in deterministic execution mode, instructions doing device I/Os
4379 must be at the end of the TB */
4380 void cpu_io_recompile(CPUState *env, void *retaddr)
4381 {
4382 TranslationBlock *tb;
4383 uint32_t n, cflags;
4384 target_ulong pc, cs_base;
4385 uint64_t flags;
4386
4387 tb = tb_find_pc((unsigned long)retaddr);
4388 if (!tb) {
4389 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
4390 retaddr);
4391 }
4392 n = env->icount_decr.u16.low + tb->icount;
4393 cpu_restore_state(tb, env, (unsigned long)retaddr);
4394 /* Calculate how many instructions had been executed before the fault
4395 occurred. */
4396 n = n - env->icount_decr.u16.low;
4397 /* Generate a new TB ending on the I/O insn. */
4398 n++;
4399 /* On MIPS and SH, delay slot instructions can only be restarted if
4400 they were already the first instruction in the TB. If this is not
4401 the first instruction in a TB then re-execute the preceding
4402 branch. */
4403 #if defined(TARGET_MIPS)
4404 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
4405 env->active_tc.PC -= 4;
4406 env->icount_decr.u16.low++;
4407 env->hflags &= ~MIPS_HFLAG_BMASK;
4408 }
4409 #elif defined(TARGET_SH4)
4410 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
4411 && n > 1) {
4412 env->pc -= 2;
4413 env->icount_decr.u16.low++;
4414 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
4415 }
4416 #endif
4417 /* This should never happen. */
4418 if (n > CF_COUNT_MASK)
4419 cpu_abort(env, "TB too big during recompile");
4420
4421 cflags = n | CF_LAST_IO;
4422 pc = tb->pc;
4423 cs_base = tb->cs_base;
4424 flags = tb->flags;
4425 tb_phys_invalidate(tb, -1);
4426 /* FIXME: In theory this could raise an exception. In practice
4427 we have already translated the block once so it's probably ok. */
4428 tb_gen_code(env, pc, cs_base, flags, cflags);
4429 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
4430 the first in the TB) then we end up generating a whole new TB and
4431 repeating the fault, which is horribly inefficient.
4432 Better would be to execute just this insn uncached, or generate a
4433 second new TB. */
4434 cpu_resume_from_signal(env, NULL);
4435 }
4436
4437 #if !defined(CONFIG_USER_ONLY)
4438
4439 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
4440 {
4441 int i, target_code_size, max_target_code_size;
4442 int direct_jmp_count, direct_jmp2_count, cross_page;
4443 TranslationBlock *tb;
4444
4445 target_code_size = 0;
4446 max_target_code_size = 0;
4447 cross_page = 0;
4448 direct_jmp_count = 0;
4449 direct_jmp2_count = 0;
4450 for(i = 0; i < nb_tbs; i++) {
4451 tb = &tbs[i];
4452 target_code_size += tb->size;
4453 if (tb->size > max_target_code_size)
4454 max_target_code_size = tb->size;
4455 if (tb->page_addr[1] != -1)
4456 cross_page++;
4457 if (tb->tb_next_offset[0] != 0xffff) {
4458 direct_jmp_count++;
4459 if (tb->tb_next_offset[1] != 0xffff) {
4460 direct_jmp2_count++;
4461 }
4462 }
4463 }
4464 /* XXX: avoid using doubles ? */
4465 cpu_fprintf(f, "Translation buffer state:\n");
4466 cpu_fprintf(f, "gen code size %td/%ld\n",
4467 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
4468 cpu_fprintf(f, "TB count %d/%d\n",
4469 nb_tbs, code_gen_max_blocks);
4470 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
4471 nb_tbs ? target_code_size / nb_tbs : 0,
4472 max_target_code_size);
4473 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
4474 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
4475 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
4476 cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
4477 cross_page,
4478 nb_tbs ? (cross_page * 100) / nb_tbs : 0);
4479 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
4480 direct_jmp_count,
4481 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
4482 direct_jmp2_count,
4483 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
4484 cpu_fprintf(f, "\nStatistics:\n");
4485 cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
4486 cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
4487 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
4488 tcg_dump_info(f, cpu_fprintf);
4489 }
4490
4491 #define MMUSUFFIX _cmmu
4492 #define GETPC() NULL
4493 #define env cpu_single_env
4494 #define SOFTMMU_CODE_ACCESS
4495
4496 #define SHIFT 0
4497 #include "softmmu_template.h"
4498
4499 #define SHIFT 1
4500 #include "softmmu_template.h"
4501
4502 #define SHIFT 2
4503 #include "softmmu_template.h"
4504
4505 #define SHIFT 3
4506 #include "softmmu_template.h"
4507
4508 #undef env
4509
4510 #endif
QEmu