?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
hw/9pfs: Add qdev.reset callback for virtio-9p-pci device
[qemu.git] / exec.c
blob6c206ff5b73d076b7bfddf9d95aacccabb5d8ae8
1 /*
2 * virtual page mapping and translated block handling
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "osdep.h"
33 #include "kvm.h"
34 #include "hw/xen.h"
35 #include "qemu-timer.h"
36 #include "memory.h"
37 #include "exec-memory.h"
38 #if defined(CONFIG_USER_ONLY)
39 #include <qemu.h>
40 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
41 #include <sys/param.h>
42 #if __FreeBSD_version >= 700104
43 #define HAVE_KINFO_GETVMMAP
44 #define sigqueue sigqueue_freebsd /* avoid redefinition */
45 #include <sys/time.h>
46 #include <sys/proc.h>
47 #include <machine/profile.h>
48 #define _KERNEL
49 #include <sys/user.h>
50 #undef _KERNEL
51 #undef sigqueue
52 #include <libutil.h>
53 #endif
54 #endif
55 #else /* !CONFIG_USER_ONLY */
56 #include "xen-mapcache.h"
57 #include "trace.h"
58 #endif
59
60 //#define DEBUG_TB_INVALIDATE
61 //#define DEBUG_FLUSH
62 //#define DEBUG_TLB
63 //#define DEBUG_UNASSIGNED
64
65 /* make various TB consistency checks */
66 //#define DEBUG_TB_CHECK
67 //#define DEBUG_TLB_CHECK
68
69 //#define DEBUG_IOPORT
70 //#define DEBUG_SUBPAGE
71
72 #if !defined(CONFIG_USER_ONLY)
73 /* TB consistency checks only implemented for usermode emulation. */
74 #undef DEBUG_TB_CHECK
75 #endif
76
77 #define SMC_BITMAP_USE_THRESHOLD 10
78
79 static TranslationBlock *tbs;
80 static int code_gen_max_blocks;
81 TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
82 static int nb_tbs;
83 /* any access to the tbs or the page table must use this lock */
84 spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
85
86 #if defined(__arm__) || defined(__sparc_v9__)
87 /* The prologue must be reachable with a direct jump. ARM and Sparc64
88 have limited branch ranges (possibly also PPC) so place it in a
89 section close to code segment. */
90 #define code_gen_section \
91 __attribute__((__section__(".gen_code"))) \
92 __attribute__((aligned (32)))
93 #elif defined(_WIN32)
94 /* Maximum alignment for Win32 is 16. */
95 #define code_gen_section \
96 __attribute__((aligned (16)))
97 #else
98 #define code_gen_section \
99 __attribute__((aligned (32)))
100 #endif
101
102 uint8_t code_gen_prologue[1024] code_gen_section;
103 static uint8_t *code_gen_buffer;
104 static unsigned long code_gen_buffer_size;
105 /* threshold to flush the translated code buffer */
106 static unsigned long code_gen_buffer_max_size;
107 static uint8_t *code_gen_ptr;
108
109 #if !defined(CONFIG_USER_ONLY)
110 int phys_ram_fd;
111 static int in_migration;
112
113 RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
114
115 static MemoryRegion *system_memory;
116 static MemoryRegion *system_io;
117
118 #endif
119
120 CPUState *first_cpu;
121 /* current CPU in the current thread. It is only valid inside
122 cpu_exec() */
123 DEFINE_TLS(CPUState *,cpu_single_env);
124 /* 0 = Do not count executed instructions.
125 1 = Precise instruction counting.
126 2 = Adaptive rate instruction counting. */
127 int use_icount = 0;
128
129 typedef struct PageDesc {
130 /* list of TBs intersecting this ram page */
131 TranslationBlock *first_tb;
132 /* in order to optimize self modifying code, we count the number
133 of lookups we do to a given page to use a bitmap */
134 unsigned int code_write_count;
135 uint8_t *code_bitmap;
136 #if defined(CONFIG_USER_ONLY)
137 unsigned long flags;
138 #endif
139 } PageDesc;
140
141 /* In system mode we want L1_MAP to be based on ram offsets,
142 while in user mode we want it to be based on virtual addresses. */
143 #if !defined(CONFIG_USER_ONLY)
144 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
145 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
146 #else
147 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
148 #endif
149 #else
150 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
151 #endif
152
153 /* Size of the L2 (and L3, etc) page tables. */
154 #define L2_BITS 10
155 #define L2_SIZE (1 << L2_BITS)
156
157 /* The bits remaining after N lower levels of page tables. */
158 #define P_L1_BITS_REM \
159 ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
160 #define V_L1_BITS_REM \
161 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
162
163 /* Size of the L1 page table. Avoid silly small sizes. */
164 #if P_L1_BITS_REM < 4
165 #define P_L1_BITS (P_L1_BITS_REM + L2_BITS)
166 #else
167 #define P_L1_BITS P_L1_BITS_REM
168 #endif
169
170 #if V_L1_BITS_REM < 4
171 #define V_L1_BITS (V_L1_BITS_REM + L2_BITS)
172 #else
173 #define V_L1_BITS V_L1_BITS_REM
174 #endif
175
176 #define P_L1_SIZE ((target_phys_addr_t)1 << P_L1_BITS)
177 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
178
179 #define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
180 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
181
182 unsigned long qemu_real_host_page_size;
183 unsigned long qemu_host_page_size;
184 unsigned long qemu_host_page_mask;
185
186 /* This is a multi-level map on the virtual address space.
187 The bottom level has pointers to PageDesc. */
188 static void *l1_map[V_L1_SIZE];
189
190 #if !defined(CONFIG_USER_ONLY)
191 typedef struct PhysPageDesc {
192 /* offset in host memory of the page + io_index in the low bits */
193 ram_addr_t phys_offset;
194 ram_addr_t region_offset;
195 } PhysPageDesc;
196
197 /* This is a multi-level map on the physical address space.
198 The bottom level has pointers to PhysPageDesc. */
199 static void *l1_phys_map[P_L1_SIZE];
200
201 static void io_mem_init(void);
202 static void memory_map_init(void);
203
204 /* io memory support */
205 CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
206 CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
207 void *io_mem_opaque[IO_MEM_NB_ENTRIES];
208 static char io_mem_used[IO_MEM_NB_ENTRIES];
209 static int io_mem_watch;
210 #endif
211
212 /* log support */
213 #ifdef WIN32
214 static const char *logfilename = "qemu.log";
215 #else
216 static const char *logfilename = "/tmp/qemu.log";
217 #endif
218 FILE *logfile;
219 int loglevel;
220 static int log_append = 0;
221
222 /* statistics */
223 #if !defined(CONFIG_USER_ONLY)
224 static int tlb_flush_count;
225 #endif
226 static int tb_flush_count;
227 static int tb_phys_invalidate_count;
228
229 #ifdef _WIN32
230 static void map_exec(void *addr, long size)
231 {
232 DWORD old_protect;
233 VirtualProtect(addr, size,
234 PAGE_EXECUTE_READWRITE, &old_protect);
235
236 }
237 #else
238 static void map_exec(void *addr, long size)
239 {
240 unsigned long start, end, page_size;
241
242 page_size = getpagesize();
243 start = (unsigned long)addr;
244 start &= ~(page_size - 1);
245
246 end = (unsigned long)addr + size;
247 end += page_size - 1;
248 end &= ~(page_size - 1);
249
250 mprotect((void *)start, end - start,
251 PROT_READ | PROT_WRITE | PROT_EXEC);
252 }
253 #endif
254
255 static void page_init(void)
256 {
257 /* NOTE: we can always suppose that qemu_host_page_size >=
258 TARGET_PAGE_SIZE */
259 #ifdef _WIN32
260 {
261 SYSTEM_INFO system_info;
262
263 GetSystemInfo(&system_info);
264 qemu_real_host_page_size = system_info.dwPageSize;
265 }
266 #else
267 qemu_real_host_page_size = getpagesize();
268 #endif
269 if (qemu_host_page_size == 0)
270 qemu_host_page_size = qemu_real_host_page_size;
271 if (qemu_host_page_size < TARGET_PAGE_SIZE)
272 qemu_host_page_size = TARGET_PAGE_SIZE;
273 qemu_host_page_mask = ~(qemu_host_page_size - 1);
274
275 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
276 {
277 #ifdef HAVE_KINFO_GETVMMAP
278 struct kinfo_vmentry *freep;
279 int i, cnt;
280
281 freep = kinfo_getvmmap(getpid(), &cnt);
282 if (freep) {
283 mmap_lock();
284 for (i = 0; i < cnt; i++) {
285 unsigned long startaddr, endaddr;
286
287 startaddr = freep[i].kve_start;
288 endaddr = freep[i].kve_end;
289 if (h2g_valid(startaddr)) {
290 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
291
292 if (h2g_valid(endaddr)) {
293 endaddr = h2g(endaddr);
294 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
295 } else {
296 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
297 endaddr = ~0ul;
298 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
299 #endif
300 }
301 }
302 }
303 free(freep);
304 mmap_unlock();
305 }
306 #else
307 FILE *f;
308
309 last_brk = (unsigned long)sbrk(0);
310
311 f = fopen("/compat/linux/proc/self/maps", "r");
312 if (f) {
313 mmap_lock();
314
315 do {
316 unsigned long startaddr, endaddr;
317 int n;
318
319 n = fscanf (f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
320
321 if (n == 2 && h2g_valid(startaddr)) {
322 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
323
324 if (h2g_valid(endaddr)) {
325 endaddr = h2g(endaddr);
326 } else {
327 endaddr = ~0ul;
328 }
329 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
330 }
331 } while (!feof(f));
332
333 fclose(f);
334 mmap_unlock();
335 }
336 #endif
337 }
338 #endif
339 }
340
341 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
342 {
343 PageDesc *pd;
344 void **lp;
345 int i;
346
347 #if defined(CONFIG_USER_ONLY)
348 /* We can't use g_malloc because it may recurse into a locked mutex. */
349 # define ALLOC(P, SIZE) \
350 do { \
351 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
352 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
353 } while (0)
354 #else
355 # define ALLOC(P, SIZE) \
356 do { P = g_malloc0(SIZE); } while (0)
357 #endif
358
359 /* Level 1. Always allocated. */
360 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
361
362 /* Level 2..N-1. */
363 for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
364 void **p = *lp;
365
366 if (p == NULL) {
367 if (!alloc) {
368 return NULL;
369 }
370 ALLOC(p, sizeof(void *) * L2_SIZE);
371 *lp = p;
372 }
373
374 lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
375 }
376
377 pd = *lp;
378 if (pd == NULL) {
379 if (!alloc) {
380 return NULL;
381 }
382 ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
383 *lp = pd;
384 }
385
386 #undef ALLOC
387
388 return pd + (index & (L2_SIZE - 1));
389 }
390
391 static inline PageDesc *page_find(tb_page_addr_t index)
392 {
393 return page_find_alloc(index, 0);
394 }
395
396 #if !defined(CONFIG_USER_ONLY)
397 static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
398 {
399 PhysPageDesc *pd;
400 void **lp;
401 int i;
402
403 /* Level 1. Always allocated. */
404 lp = l1_phys_map + ((index >> P_L1_SHIFT) & (P_L1_SIZE - 1));
405
406 /* Level 2..N-1. */
407 for (i = P_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
408 void **p = *lp;
409 if (p == NULL) {
410 if (!alloc) {
411 return NULL;
412 }
413 *lp = p = g_malloc0(sizeof(void *) * L2_SIZE);
414 }
415 lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
416 }
417
418 pd = *lp;
419 if (pd == NULL) {
420 int i;
421
422 if (!alloc) {
423 return NULL;
424 }
425
426 *lp = pd = g_malloc(sizeof(PhysPageDesc) * L2_SIZE);
427
428 for (i = 0; i < L2_SIZE; i++) {
429 pd[i].phys_offset = IO_MEM_UNASSIGNED;
430 pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
431 }
432 }
433
434 return pd + (index & (L2_SIZE - 1));
435 }
436
437 static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
438 {
439 return phys_page_find_alloc(index, 0);
440 }
441
442 static void tlb_protect_code(ram_addr_t ram_addr);
443 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
444 target_ulong vaddr);
445 #define mmap_lock() do { } while(0)
446 #define mmap_unlock() do { } while(0)
447 #endif
448
449 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
450
451 #if defined(CONFIG_USER_ONLY)
452 /* Currently it is not recommended to allocate big chunks of data in
453 user mode. It will change when a dedicated libc will be used */
454 #define USE_STATIC_CODE_GEN_BUFFER
455 #endif
456
457 #ifdef USE_STATIC_CODE_GEN_BUFFER
458 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
459 __attribute__((aligned (CODE_GEN_ALIGN)));
460 #endif
461
462 static void code_gen_alloc(unsigned long tb_size)
463 {
464 #ifdef USE_STATIC_CODE_GEN_BUFFER
465 code_gen_buffer = static_code_gen_buffer;
466 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
467 map_exec(code_gen_buffer, code_gen_buffer_size);
468 #else
469 code_gen_buffer_size = tb_size;
470 if (code_gen_buffer_size == 0) {
471 #if defined(CONFIG_USER_ONLY)
472 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
473 #else
474 /* XXX: needs adjustments */
475 code_gen_buffer_size = (unsigned long)(ram_size / 4);
476 #endif
477 }
478 if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
479 code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
480 /* The code gen buffer location may have constraints depending on
481 the host cpu and OS */
482 #if defined(__linux__)
483 {
484 int flags;
485 void *start = NULL;
486
487 flags = MAP_PRIVATE | MAP_ANONYMOUS;
488 #if defined(__x86_64__)
489 flags |= MAP_32BIT;
490 /* Cannot map more than that */
491 if (code_gen_buffer_size > (800 * 1024 * 1024))
492 code_gen_buffer_size = (800 * 1024 * 1024);
493 #elif defined(__sparc_v9__)
494 // Map the buffer below 2G, so we can use direct calls and branches
495 flags |= MAP_FIXED;
496 start = (void *) 0x60000000UL;
497 if (code_gen_buffer_size > (512 * 1024 * 1024))
498 code_gen_buffer_size = (512 * 1024 * 1024);
499 #elif defined(__arm__)
500 /* Map the buffer below 32M, so we can use direct calls and branches */
501 flags |= MAP_FIXED;
502 start = (void *) 0x01000000UL;
503 if (code_gen_buffer_size > 16 * 1024 * 1024)
504 code_gen_buffer_size = 16 * 1024 * 1024;
505 #elif defined(__s390x__)
506 /* Map the buffer so that we can use direct calls and branches. */
507 /* We have a +- 4GB range on the branches; leave some slop. */
508 if (code_gen_buffer_size > (3ul * 1024 * 1024 * 1024)) {
509 code_gen_buffer_size = 3ul * 1024 * 1024 * 1024;
510 }
511 start = (void *)0x90000000UL;
512 #endif
513 code_gen_buffer = mmap(start, code_gen_buffer_size,
514 PROT_WRITE | PROT_READ | PROT_EXEC,
515 flags, -1, 0);
516 if (code_gen_buffer == MAP_FAILED) {
517 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
518 exit(1);
519 }
520 }
521 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
522 || defined(__DragonFly__) || defined(__OpenBSD__) \
523 || defined(__NetBSD__)
524 {
525 int flags;
526 void *addr = NULL;
527 flags = MAP_PRIVATE | MAP_ANONYMOUS;
528 #if defined(__x86_64__)
529 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
530 * 0x40000000 is free */
531 flags |= MAP_FIXED;
532 addr = (void *)0x40000000;
533 /* Cannot map more than that */
534 if (code_gen_buffer_size > (800 * 1024 * 1024))
535 code_gen_buffer_size = (800 * 1024 * 1024);
536 #elif defined(__sparc_v9__)
537 // Map the buffer below 2G, so we can use direct calls and branches
538 flags |= MAP_FIXED;
539 addr = (void *) 0x60000000UL;
540 if (code_gen_buffer_size > (512 * 1024 * 1024)) {
541 code_gen_buffer_size = (512 * 1024 * 1024);
542 }
543 #endif
544 code_gen_buffer = mmap(addr, code_gen_buffer_size,
545 PROT_WRITE | PROT_READ | PROT_EXEC,
546 flags, -1, 0);
547 if (code_gen_buffer == MAP_FAILED) {
548 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
549 exit(1);
550 }
551 }
552 #else
553 code_gen_buffer = g_malloc(code_gen_buffer_size);
554 map_exec(code_gen_buffer, code_gen_buffer_size);
555 #endif
556 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
557 map_exec(code_gen_prologue, sizeof(code_gen_prologue));
558 code_gen_buffer_max_size = code_gen_buffer_size -
559 (TCG_MAX_OP_SIZE * OPC_BUF_SIZE);
560 code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
561 tbs = g_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
562 }
563
564 /* Must be called before using the QEMU cpus. 'tb_size' is the size
565 (in bytes) allocated to the translation buffer. Zero means default
566 size. */
567 void tcg_exec_init(unsigned long tb_size)
568 {
569 cpu_gen_init();
570 code_gen_alloc(tb_size);
571 code_gen_ptr = code_gen_buffer;
572 page_init();
573 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
574 /* There's no guest base to take into account, so go ahead and
575 initialize the prologue now. */
576 tcg_prologue_init(&tcg_ctx);
577 #endif
578 }
579
580 bool tcg_enabled(void)
581 {
582 return code_gen_buffer != NULL;
583 }
584
585 void cpu_exec_init_all(void)
586 {
587 #if !defined(CONFIG_USER_ONLY)
588 memory_map_init();
589 io_mem_init();
590 #endif
591 }
592
593 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
594
595 static int cpu_common_post_load(void *opaque, int version_id)
596 {
597 CPUState *env = opaque;
598
599 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
600 version_id is increased. */
601 env->interrupt_request &= ~0x01;
602 tlb_flush(env, 1);
603
604 return 0;
605 }
606
607 static const VMStateDescription vmstate_cpu_common = {
608 .name = "cpu_common",
609 .version_id = 1,
610 .minimum_version_id = 1,
611 .minimum_version_id_old = 1,
612 .post_load = cpu_common_post_load,
613 .fields = (VMStateField []) {
614 VMSTATE_UINT32(halted, CPUState),
615 VMSTATE_UINT32(interrupt_request, CPUState),
616 VMSTATE_END_OF_LIST()
617 }
618 };
619 #endif
620
621 CPUState *qemu_get_cpu(int cpu)
622 {
623 CPUState *env = first_cpu;
624
625 while (env) {
626 if (env->cpu_index == cpu)
627 break;
628 env = env->next_cpu;
629 }
630
631 return env;
632 }
633
634 void cpu_exec_init(CPUState *env)
635 {
636 CPUState **penv;
637 int cpu_index;
638
639 #if defined(CONFIG_USER_ONLY)
640 cpu_list_lock();
641 #endif
642 env->next_cpu = NULL;
643 penv = &first_cpu;
644 cpu_index = 0;
645 while (*penv != NULL) {
646 penv = &(*penv)->next_cpu;
647 cpu_index++;
648 }
649 env->cpu_index = cpu_index;
650 env->numa_node = 0;
651 QTAILQ_INIT(&env->breakpoints);
652 QTAILQ_INIT(&env->watchpoints);
653 #ifndef CONFIG_USER_ONLY
654 env->thread_id = qemu_get_thread_id();
655 #endif
656 *penv = env;
657 #if defined(CONFIG_USER_ONLY)
658 cpu_list_unlock();
659 #endif
660 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
661 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, env);
662 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
663 cpu_save, cpu_load, env);
664 #endif
665 }
666
667 /* Allocate a new translation block. Flush the translation buffer if
668 too many translation blocks or too much generated code. */
669 static TranslationBlock *tb_alloc(target_ulong pc)
670 {
671 TranslationBlock *tb;
672
673 if (nb_tbs >= code_gen_max_blocks ||
674 (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
675 return NULL;
676 tb = &tbs[nb_tbs++];
677 tb->pc = pc;
678 tb->cflags = 0;
679 return tb;
680 }
681
682 void tb_free(TranslationBlock *tb)
683 {
684 /* In practice this is mostly used for single use temporary TB
685 Ignore the hard cases and just back up if this TB happens to
686 be the last one generated. */
687 if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
688 code_gen_ptr = tb->tc_ptr;
689 nb_tbs--;
690 }
691 }
692
693 static inline void invalidate_page_bitmap(PageDesc *p)
694 {
695 if (p->code_bitmap) {
696 g_free(p->code_bitmap);
697 p->code_bitmap = NULL;
698 }
699 p->code_write_count = 0;
700 }
701
702 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
703
704 static void page_flush_tb_1 (int level, void **lp)
705 {
706 int i;
707
708 if (*lp == NULL) {
709 return;
710 }
711 if (level == 0) {
712 PageDesc *pd = *lp;
713 for (i = 0; i < L2_SIZE; ++i) {
714 pd[i].first_tb = NULL;
715 invalidate_page_bitmap(pd + i);
716 }
717 } else {
718 void **pp = *lp;
719 for (i = 0; i < L2_SIZE; ++i) {
720 page_flush_tb_1 (level - 1, pp + i);
721 }
722 }
723 }
724
725 static void page_flush_tb(void)
726 {
727 int i;
728 for (i = 0; i < V_L1_SIZE; i++) {
729 page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
730 }
731 }
732
733 /* flush all the translation blocks */
734 /* XXX: tb_flush is currently not thread safe */
735 void tb_flush(CPUState *env1)
736 {
737 CPUState *env;
738 #if defined(DEBUG_FLUSH)
739 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
740 (unsigned long)(code_gen_ptr - code_gen_buffer),
741 nb_tbs, nb_tbs > 0 ?
742 ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
743 #endif
744 if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
745 cpu_abort(env1, "Internal error: code buffer overflow\n");
746
747 nb_tbs = 0;
748
749 for(env = first_cpu; env != NULL; env = env->next_cpu) {
750 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
751 }
752
753 memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
754 page_flush_tb();
755
756 code_gen_ptr = code_gen_buffer;
757 /* XXX: flush processor icache at this point if cache flush is
758 expensive */
759 tb_flush_count++;
760 }
761
762 #ifdef DEBUG_TB_CHECK
763
764 static void tb_invalidate_check(target_ulong address)
765 {
766 TranslationBlock *tb;
767 int i;
768 address &= TARGET_PAGE_MASK;
769 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
770 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
771 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
772 address >= tb->pc + tb->size)) {
773 printf("ERROR invalidate: address=" TARGET_FMT_lx
774 " PC=%08lx size=%04x\n",
775 address, (long)tb->pc, tb->size);
776 }
777 }
778 }
779 }
780
781 /* verify that all the pages have correct rights for code */
782 static void tb_page_check(void)
783 {
784 TranslationBlock *tb;
785 int i, flags1, flags2;
786
787 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
788 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
789 flags1 = page_get_flags(tb->pc);
790 flags2 = page_get_flags(tb->pc + tb->size - 1);
791 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
792 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
793 (long)tb->pc, tb->size, flags1, flags2);
794 }
795 }
796 }
797 }
798
799 #endif
800
801 /* invalidate one TB */
802 static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
803 int next_offset)
804 {
805 TranslationBlock *tb1;
806 for(;;) {
807 tb1 = *ptb;
808 if (tb1 == tb) {
809 *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
810 break;
811 }
812 ptb = (TranslationBlock **)((char *)tb1 + next_offset);
813 }
814 }
815
816 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
817 {
818 TranslationBlock *tb1;
819 unsigned int n1;
820
821 for(;;) {
822 tb1 = *ptb;
823 n1 = (long)tb1 & 3;
824 tb1 = (TranslationBlock *)((long)tb1 & ~3);
825 if (tb1 == tb) {
826 *ptb = tb1->page_next[n1];
827 break;
828 }
829 ptb = &tb1->page_next[n1];
830 }
831 }
832
833 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
834 {
835 TranslationBlock *tb1, **ptb;
836 unsigned int n1;
837
838 ptb = &tb->jmp_next[n];
839 tb1 = *ptb;
840 if (tb1) {
841 /* find tb(n) in circular list */
842 for(;;) {
843 tb1 = *ptb;
844 n1 = (long)tb1 & 3;
845 tb1 = (TranslationBlock *)((long)tb1 & ~3);
846 if (n1 == n && tb1 == tb)
847 break;
848 if (n1 == 2) {
849 ptb = &tb1->jmp_first;
850 } else {
851 ptb = &tb1->jmp_next[n1];
852 }
853 }
854 /* now we can suppress tb(n) from the list */
855 *ptb = tb->jmp_next[n];
856
857 tb->jmp_next[n] = NULL;
858 }
859 }
860
861 /* reset the jump entry 'n' of a TB so that it is not chained to
862 another TB */
863 static inline void tb_reset_jump(TranslationBlock *tb, int n)
864 {
865 tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
866 }
867
868 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
869 {
870 CPUState *env;
871 PageDesc *p;
872 unsigned int h, n1;
873 tb_page_addr_t phys_pc;
874 TranslationBlock *tb1, *tb2;
875
876 /* remove the TB from the hash list */
877 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
878 h = tb_phys_hash_func(phys_pc);
879 tb_remove(&tb_phys_hash[h], tb,
880 offsetof(TranslationBlock, phys_hash_next));
881
882 /* remove the TB from the page list */
883 if (tb->page_addr[0] != page_addr) {
884 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
885 tb_page_remove(&p->first_tb, tb);
886 invalidate_page_bitmap(p);
887 }
888 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
889 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
890 tb_page_remove(&p->first_tb, tb);
891 invalidate_page_bitmap(p);
892 }
893
894 tb_invalidated_flag = 1;
895
896 /* remove the TB from the hash list */
897 h = tb_jmp_cache_hash_func(tb->pc);
898 for(env = first_cpu; env != NULL; env = env->next_cpu) {
899 if (env->tb_jmp_cache[h] == tb)
900 env->tb_jmp_cache[h] = NULL;
901 }
902
903 /* suppress this TB from the two jump lists */
904 tb_jmp_remove(tb, 0);
905 tb_jmp_remove(tb, 1);
906
907 /* suppress any remaining jumps to this TB */
908 tb1 = tb->jmp_first;
909 for(;;) {
910 n1 = (long)tb1 & 3;
911 if (n1 == 2)
912 break;
913 tb1 = (TranslationBlock *)((long)tb1 & ~3);
914 tb2 = tb1->jmp_next[n1];
915 tb_reset_jump(tb1, n1);
916 tb1->jmp_next[n1] = NULL;
917 tb1 = tb2;
918 }
919 tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
920
921 tb_phys_invalidate_count++;
922 }
923
924 static inline void set_bits(uint8_t *tab, int start, int len)
925 {
926 int end, mask, end1;
927
928 end = start + len;
929 tab += start >> 3;
930 mask = 0xff << (start & 7);
931 if ((start & ~7) == (end & ~7)) {
932 if (start < end) {
933 mask &= ~(0xff << (end & 7));
934 *tab |= mask;
935 }
936 } else {
937 *tab++ |= mask;
938 start = (start + 8) & ~7;
939 end1 = end & ~7;
940 while (start < end1) {
941 *tab++ = 0xff;
942 start += 8;
943 }
944 if (start < end) {
945 mask = ~(0xff << (end & 7));
946 *tab |= mask;
947 }
948 }
949 }
950
951 static void build_page_bitmap(PageDesc *p)
952 {
953 int n, tb_start, tb_end;
954 TranslationBlock *tb;
955
956 p->code_bitmap = g_malloc0(TARGET_PAGE_SIZE / 8);
957
958 tb = p->first_tb;
959 while (tb != NULL) {
960 n = (long)tb & 3;
961 tb = (TranslationBlock *)((long)tb & ~3);
962 /* NOTE: this is subtle as a TB may span two physical pages */
963 if (n == 0) {
964 /* NOTE: tb_end may be after the end of the page, but
965 it is not a problem */
966 tb_start = tb->pc & ~TARGET_PAGE_MASK;
967 tb_end = tb_start + tb->size;
968 if (tb_end > TARGET_PAGE_SIZE)
969 tb_end = TARGET_PAGE_SIZE;
970 } else {
971 tb_start = 0;
972 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
973 }
974 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
975 tb = tb->page_next[n];
976 }
977 }
978
979 TranslationBlock *tb_gen_code(CPUState *env,
980 target_ulong pc, target_ulong cs_base,
981 int flags, int cflags)
982 {
983 TranslationBlock *tb;
984 uint8_t *tc_ptr;
985 tb_page_addr_t phys_pc, phys_page2;
986 target_ulong virt_page2;
987 int code_gen_size;
988
989 phys_pc = get_page_addr_code(env, pc);
990 tb = tb_alloc(pc);
991 if (!tb) {
992 /* flush must be done */
993 tb_flush(env);
994 /* cannot fail at this point */
995 tb = tb_alloc(pc);
996 /* Don't forget to invalidate previous TB info. */
997 tb_invalidated_flag = 1;
998 }
999 tc_ptr = code_gen_ptr;
1000 tb->tc_ptr = tc_ptr;
1001 tb->cs_base = cs_base;
1002 tb->flags = flags;
1003 tb->cflags = cflags;
1004 cpu_gen_code(env, tb, &code_gen_size);
1005 code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
1006
1007 /* check next page if needed */
1008 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
1009 phys_page2 = -1;
1010 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
1011 phys_page2 = get_page_addr_code(env, virt_page2);
1012 }
1013 tb_link_page(tb, phys_pc, phys_page2);
1014 return tb;
1015 }
1016
1017 /* invalidate all TBs which intersect with the target physical page
1018 starting in range [start;end[. NOTE: start and end must refer to
1019 the same physical page. 'is_cpu_write_access' should be true if called
1020 from a real cpu write access: the virtual CPU will exit the current
1021 TB if code is modified inside this TB. */
1022 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
1023 int is_cpu_write_access)
1024 {
1025 TranslationBlock *tb, *tb_next, *saved_tb;
1026 CPUState *env = cpu_single_env;
1027 tb_page_addr_t tb_start, tb_end;
1028 PageDesc *p;
1029 int n;
1030 #ifdef TARGET_HAS_PRECISE_SMC
1031 int current_tb_not_found = is_cpu_write_access;
1032 TranslationBlock *current_tb = NULL;
1033 int current_tb_modified = 0;
1034 target_ulong current_pc = 0;
1035 target_ulong current_cs_base = 0;
1036 int current_flags = 0;
1037 #endif /* TARGET_HAS_PRECISE_SMC */
1038
1039 p = page_find(start >> TARGET_PAGE_BITS);
1040 if (!p)
1041 return;
1042 if (!p->code_bitmap &&
1043 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
1044 is_cpu_write_access) {
1045 /* build code bitmap */
1046 build_page_bitmap(p);
1047 }
1048
1049 /* we remove all the TBs in the range [start, end[ */
1050 /* XXX: see if in some cases it could be faster to invalidate all the code */
1051 tb = p->first_tb;
1052 while (tb != NULL) {
1053 n = (long)tb & 3;
1054 tb = (TranslationBlock *)((long)tb & ~3);
1055 tb_next = tb->page_next[n];
1056 /* NOTE: this is subtle as a TB may span two physical pages */
1057 if (n == 0) {
1058 /* NOTE: tb_end may be after the end of the page, but
1059 it is not a problem */
1060 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1061 tb_end = tb_start + tb->size;
1062 } else {
1063 tb_start = tb->page_addr[1];
1064 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1065 }
1066 if (!(tb_end <= start || tb_start >= end)) {
1067 #ifdef TARGET_HAS_PRECISE_SMC
1068 if (current_tb_not_found) {
1069 current_tb_not_found = 0;
1070 current_tb = NULL;
1071 if (env->mem_io_pc) {
1072 /* now we have a real cpu fault */
1073 current_tb = tb_find_pc(env->mem_io_pc);
1074 }
1075 }
1076 if (current_tb == tb &&
1077 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1078 /* If we are modifying the current TB, we must stop
1079 its execution. We could be more precise by checking
1080 that the modification is after the current PC, but it
1081 would require a specialized function to partially
1082 restore the CPU state */
1083
1084 current_tb_modified = 1;
1085 cpu_restore_state(current_tb, env, env->mem_io_pc);
1086 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1087 &current_flags);
1088 }
1089 #endif /* TARGET_HAS_PRECISE_SMC */
1090 /* we need to do that to handle the case where a signal
1091 occurs while doing tb_phys_invalidate() */
1092 saved_tb = NULL;
1093 if (env) {
1094 saved_tb = env->current_tb;
1095 env->current_tb = NULL;
1096 }
1097 tb_phys_invalidate(tb, -1);
1098 if (env) {
1099 env->current_tb = saved_tb;
1100 if (env->interrupt_request && env->current_tb)
1101 cpu_interrupt(env, env->interrupt_request);
1102 }
1103 }
1104 tb = tb_next;
1105 }
1106 #if !defined(CONFIG_USER_ONLY)
1107 /* if no code remaining, no need to continue to use slow writes */
1108 if (!p->first_tb) {
1109 invalidate_page_bitmap(p);
1110 if (is_cpu_write_access) {
1111 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1112 }
1113 }
1114 #endif
1115 #ifdef TARGET_HAS_PRECISE_SMC
1116 if (current_tb_modified) {
1117 /* we generate a block containing just the instruction
1118 modifying the memory. It will ensure that it cannot modify
1119 itself */
1120 env->current_tb = NULL;
1121 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1122 cpu_resume_from_signal(env, NULL);
1123 }
1124 #endif
1125 }
1126
1127 /* len must be <= 8 and start must be a multiple of len */
1128 static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1129 {
1130 PageDesc *p;
1131 int offset, b;
1132 #if 0
1133 if (1) {
1134 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1135 cpu_single_env->mem_io_vaddr, len,
1136 cpu_single_env->eip,
1137 cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1138 }
1139 #endif
1140 p = page_find(start >> TARGET_PAGE_BITS);
1141 if (!p)
1142 return;
1143 if (p->code_bitmap) {
1144 offset = start & ~TARGET_PAGE_MASK;
1145 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1146 if (b & ((1 << len) - 1))
1147 goto do_invalidate;
1148 } else {
1149 do_invalidate:
1150 tb_invalidate_phys_page_range(start, start + len, 1);
1151 }
1152 }
1153
1154 #if !defined(CONFIG_SOFTMMU)
1155 static void tb_invalidate_phys_page(tb_page_addr_t addr,
1156 unsigned long pc, void *puc)
1157 {
1158 TranslationBlock *tb;
1159 PageDesc *p;
1160 int n;
1161 #ifdef TARGET_HAS_PRECISE_SMC
1162 TranslationBlock *current_tb = NULL;
1163 CPUState *env = cpu_single_env;
1164 int current_tb_modified = 0;
1165 target_ulong current_pc = 0;
1166 target_ulong current_cs_base = 0;
1167 int current_flags = 0;
1168 #endif
1169
1170 addr &= TARGET_PAGE_MASK;
1171 p = page_find(addr >> TARGET_PAGE_BITS);
1172 if (!p)
1173 return;
1174 tb = p->first_tb;
1175 #ifdef TARGET_HAS_PRECISE_SMC
1176 if (tb && pc != 0) {
1177 current_tb = tb_find_pc(pc);
1178 }
1179 #endif
1180 while (tb != NULL) {
1181 n = (long)tb & 3;
1182 tb = (TranslationBlock *)((long)tb & ~3);
1183 #ifdef TARGET_HAS_PRECISE_SMC
1184 if (current_tb == tb &&
1185 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1186 /* If we are modifying the current TB, we must stop
1187 its execution. We could be more precise by checking
1188 that the modification is after the current PC, but it
1189 would require a specialized function to partially
1190 restore the CPU state */
1191
1192 current_tb_modified = 1;
1193 cpu_restore_state(current_tb, env, pc);
1194 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1195 &current_flags);
1196 }
1197 #endif /* TARGET_HAS_PRECISE_SMC */
1198 tb_phys_invalidate(tb, addr);
1199 tb = tb->page_next[n];
1200 }
1201 p->first_tb = NULL;
1202 #ifdef TARGET_HAS_PRECISE_SMC
1203 if (current_tb_modified) {
1204 /* we generate a block containing just the instruction
1205 modifying the memory. It will ensure that it cannot modify
1206 itself */
1207 env->current_tb = NULL;
1208 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1209 cpu_resume_from_signal(env, puc);
1210 }
1211 #endif
1212 }
1213 #endif
1214
1215 /* add the tb in the target page and protect it if necessary */
1216 static inline void tb_alloc_page(TranslationBlock *tb,
1217 unsigned int n, tb_page_addr_t page_addr)
1218 {
1219 PageDesc *p;
1220 #ifndef CONFIG_USER_ONLY
1221 bool page_already_protected;
1222 #endif
1223
1224 tb->page_addr[n] = page_addr;
1225 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1226 tb->page_next[n] = p->first_tb;
1227 #ifndef CONFIG_USER_ONLY
1228 page_already_protected = p->first_tb != NULL;
1229 #endif
1230 p->first_tb = (TranslationBlock *)((long)tb | n);
1231 invalidate_page_bitmap(p);
1232
1233 #if defined(TARGET_HAS_SMC) || 1
1234
1235 #if defined(CONFIG_USER_ONLY)
1236 if (p->flags & PAGE_WRITE) {
1237 target_ulong addr;
1238 PageDesc *p2;
1239 int prot;
1240
1241 /* force the host page as non writable (writes will have a
1242 page fault + mprotect overhead) */
1243 page_addr &= qemu_host_page_mask;
1244 prot = 0;
1245 for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1246 addr += TARGET_PAGE_SIZE) {
1247
1248 p2 = page_find (addr >> TARGET_PAGE_BITS);
1249 if (!p2)
1250 continue;
1251 prot |= p2->flags;
1252 p2->flags &= ~PAGE_WRITE;
1253 }
1254 mprotect(g2h(page_addr), qemu_host_page_size,
1255 (prot & PAGE_BITS) & ~PAGE_WRITE);
1256 #ifdef DEBUG_TB_INVALIDATE
1257 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1258 page_addr);
1259 #endif
1260 }
1261 #else
1262 /* if some code is already present, then the pages are already
1263 protected. So we handle the case where only the first TB is
1264 allocated in a physical page */
1265 if (!page_already_protected) {
1266 tlb_protect_code(page_addr);
1267 }
1268 #endif
1269
1270 #endif /* TARGET_HAS_SMC */
1271 }
1272
1273 /* add a new TB and link it to the physical page tables. phys_page2 is
1274 (-1) to indicate that only one page contains the TB. */
1275 void tb_link_page(TranslationBlock *tb,
1276 tb_page_addr_t phys_pc, tb_page_addr_t phys_page2)
1277 {
1278 unsigned int h;
1279 TranslationBlock **ptb;
1280
1281 /* Grab the mmap lock to stop another thread invalidating this TB
1282 before we are done. */
1283 mmap_lock();
1284 /* add in the physical hash table */
1285 h = tb_phys_hash_func(phys_pc);
1286 ptb = &tb_phys_hash[h];
1287 tb->phys_hash_next = *ptb;
1288 *ptb = tb;
1289
1290 /* add in the page list */
1291 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1292 if (phys_page2 != -1)
1293 tb_alloc_page(tb, 1, phys_page2);
1294 else
1295 tb->page_addr[1] = -1;
1296
1297 tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1298 tb->jmp_next[0] = NULL;
1299 tb->jmp_next[1] = NULL;
1300
1301 /* init original jump addresses */
1302 if (tb->tb_next_offset[0] != 0xffff)
1303 tb_reset_jump(tb, 0);
1304 if (tb->tb_next_offset[1] != 0xffff)
1305 tb_reset_jump(tb, 1);
1306
1307 #ifdef DEBUG_TB_CHECK
1308 tb_page_check();
1309 #endif
1310 mmap_unlock();
1311 }
1312
1313 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1314 tb[1].tc_ptr. Return NULL if not found */
1315 TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1316 {
1317 int m_min, m_max, m;
1318 unsigned long v;
1319 TranslationBlock *tb;
1320
1321 if (nb_tbs <= 0)
1322 return NULL;
1323 if (tc_ptr < (unsigned long)code_gen_buffer ||
1324 tc_ptr >= (unsigned long)code_gen_ptr)
1325 return NULL;
1326 /* binary search (cf Knuth) */
1327 m_min = 0;
1328 m_max = nb_tbs - 1;
1329 while (m_min <= m_max) {
1330 m = (m_min + m_max) >> 1;
1331 tb = &tbs[m];
1332 v = (unsigned long)tb->tc_ptr;
1333 if (v == tc_ptr)
1334 return tb;
1335 else if (tc_ptr < v) {
1336 m_max = m - 1;
1337 } else {
1338 m_min = m + 1;
1339 }
1340 }
1341 return &tbs[m_max];
1342 }
1343
1344 static void tb_reset_jump_recursive(TranslationBlock *tb);
1345
1346 static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1347 {
1348 TranslationBlock *tb1, *tb_next, **ptb;
1349 unsigned int n1;
1350
1351 tb1 = tb->jmp_next[n];
1352 if (tb1 != NULL) {
1353 /* find head of list */
1354 for(;;) {
1355 n1 = (long)tb1 & 3;
1356 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1357 if (n1 == 2)
1358 break;
1359 tb1 = tb1->jmp_next[n1];
1360 }
1361 /* we are now sure now that tb jumps to tb1 */
1362 tb_next = tb1;
1363
1364 /* remove tb from the jmp_first list */
1365 ptb = &tb_next->jmp_first;
1366 for(;;) {
1367 tb1 = *ptb;
1368 n1 = (long)tb1 & 3;
1369 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1370 if (n1 == n && tb1 == tb)
1371 break;
1372 ptb = &tb1->jmp_next[n1];
1373 }
1374 *ptb = tb->jmp_next[n];
1375 tb->jmp_next[n] = NULL;
1376
1377 /* suppress the jump to next tb in generated code */
1378 tb_reset_jump(tb, n);
1379
1380 /* suppress jumps in the tb on which we could have jumped */
1381 tb_reset_jump_recursive(tb_next);
1382 }
1383 }
1384
1385 static void tb_reset_jump_recursive(TranslationBlock *tb)
1386 {
1387 tb_reset_jump_recursive2(tb, 0);
1388 tb_reset_jump_recursive2(tb, 1);
1389 }
1390
1391 #if defined(TARGET_HAS_ICE)
1392 #if defined(CONFIG_USER_ONLY)
1393 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1394 {
1395 tb_invalidate_phys_page_range(pc, pc + 1, 0);
1396 }
1397 #else
1398 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1399 {
1400 target_phys_addr_t addr;
1401 target_ulong pd;
1402 ram_addr_t ram_addr;
1403 PhysPageDesc *p;
1404
1405 addr = cpu_get_phys_page_debug(env, pc);
1406 p = phys_page_find(addr >> TARGET_PAGE_BITS);
1407 if (!p) {
1408 pd = IO_MEM_UNASSIGNED;
1409 } else {
1410 pd = p->phys_offset;
1411 }
1412 ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1413 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1414 }
1415 #endif
1416 #endif /* TARGET_HAS_ICE */
1417
1418 #if defined(CONFIG_USER_ONLY)
1419 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1420
1421 {
1422 }
1423
1424 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1425 int flags, CPUWatchpoint **watchpoint)
1426 {
1427 return -ENOSYS;
1428 }
1429 #else
1430 /* Add a watchpoint. */
1431 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1432 int flags, CPUWatchpoint **watchpoint)
1433 {
1434 target_ulong len_mask = ~(len - 1);
1435 CPUWatchpoint *wp;
1436
1437 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1438 if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1439 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1440 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1441 return -EINVAL;
1442 }
1443 wp = g_malloc(sizeof(*wp));
1444
1445 wp->vaddr = addr;
1446 wp->len_mask = len_mask;
1447 wp->flags = flags;
1448
1449 /* keep all GDB-injected watchpoints in front */
1450 if (flags & BP_GDB)
1451 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1452 else
1453 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1454
1455 tlb_flush_page(env, addr);
1456
1457 if (watchpoint)
1458 *watchpoint = wp;
1459 return 0;
1460 }
1461
1462 /* Remove a specific watchpoint. */
1463 int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1464 int flags)
1465 {
1466 target_ulong len_mask = ~(len - 1);
1467 CPUWatchpoint *wp;
1468
1469 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1470 if (addr == wp->vaddr && len_mask == wp->len_mask
1471 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1472 cpu_watchpoint_remove_by_ref(env, wp);
1473 return 0;
1474 }
1475 }
1476 return -ENOENT;
1477 }
1478
1479 /* Remove a specific watchpoint by reference. */
1480 void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1481 {
1482 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1483
1484 tlb_flush_page(env, watchpoint->vaddr);
1485
1486 g_free(watchpoint);
1487 }
1488
1489 /* Remove all matching watchpoints. */
1490 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1491 {
1492 CPUWatchpoint *wp, *next;
1493
1494 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1495 if (wp->flags & mask)
1496 cpu_watchpoint_remove_by_ref(env, wp);
1497 }
1498 }
1499 #endif
1500
1501 /* Add a breakpoint. */
1502 int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1503 CPUBreakpoint **breakpoint)
1504 {
1505 #if defined(TARGET_HAS_ICE)
1506 CPUBreakpoint *bp;
1507
1508 bp = g_malloc(sizeof(*bp));
1509
1510 bp->pc = pc;
1511 bp->flags = flags;
1512
1513 /* keep all GDB-injected breakpoints in front */
1514 if (flags & BP_GDB)
1515 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1516 else
1517 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1518
1519 breakpoint_invalidate(env, pc);
1520
1521 if (breakpoint)
1522 *breakpoint = bp;
1523 return 0;
1524 #else
1525 return -ENOSYS;
1526 #endif
1527 }
1528
1529 /* Remove a specific breakpoint. */
1530 int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1531 {
1532 #if defined(TARGET_HAS_ICE)
1533 CPUBreakpoint *bp;
1534
1535 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1536 if (bp->pc == pc && bp->flags == flags) {
1537 cpu_breakpoint_remove_by_ref(env, bp);
1538 return 0;
1539 }
1540 }
1541 return -ENOENT;
1542 #else
1543 return -ENOSYS;
1544 #endif
1545 }
1546
1547 /* Remove a specific breakpoint by reference. */
1548 void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
1549 {
1550 #if defined(TARGET_HAS_ICE)
1551 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1552
1553 breakpoint_invalidate(env, breakpoint->pc);
1554
1555 g_free(breakpoint);
1556 #endif
1557 }
1558
1559 /* Remove all matching breakpoints. */
1560 void cpu_breakpoint_remove_all(CPUState *env, int mask)
1561 {
1562 #if defined(TARGET_HAS_ICE)
1563 CPUBreakpoint *bp, *next;
1564
1565 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1566 if (bp->flags & mask)
1567 cpu_breakpoint_remove_by_ref(env, bp);
1568 }
1569 #endif
1570 }
1571
1572 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1573 CPU loop after each instruction */
1574 void cpu_single_step(CPUState *env, int enabled)
1575 {
1576 #if defined(TARGET_HAS_ICE)
1577 if (env->singlestep_enabled != enabled) {
1578 env->singlestep_enabled = enabled;
1579 if (kvm_enabled())
1580 kvm_update_guest_debug(env, 0);
1581 else {
1582 /* must flush all the translated code to avoid inconsistencies */
1583 /* XXX: only flush what is necessary */
1584 tb_flush(env);
1585 }
1586 }
1587 #endif
1588 }
1589
1590 /* enable or disable low levels log */
1591 void cpu_set_log(int log_flags)
1592 {
1593 loglevel = log_flags;
1594 if (loglevel && !logfile) {
1595 logfile = fopen(logfilename, log_append ? "a" : "w");
1596 if (!logfile) {
1597 perror(logfilename);
1598 _exit(1);
1599 }
1600 #if !defined(CONFIG_SOFTMMU)
1601 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1602 {
1603 static char logfile_buf[4096];
1604 setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1605 }
1606 #elif !defined(_WIN32)
1607 /* Win32 doesn't support line-buffering and requires size >= 2 */
1608 setvbuf(logfile, NULL, _IOLBF, 0);
1609 #endif
1610 log_append = 1;
1611 }
1612 if (!loglevel && logfile) {
1613 fclose(logfile);
1614 logfile = NULL;
1615 }
1616 }
1617
1618 void cpu_set_log_filename(const char *filename)
1619 {
1620 logfilename = strdup(filename);
1621 if (logfile) {
1622 fclose(logfile);
1623 logfile = NULL;
1624 }
1625 cpu_set_log(loglevel);
1626 }
1627
1628 static void cpu_unlink_tb(CPUState *env)
1629 {
1630 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1631 problem and hope the cpu will stop of its own accord. For userspace
1632 emulation this often isn't actually as bad as it sounds. Often
1633 signals are used primarily to interrupt blocking syscalls. */
1634 TranslationBlock *tb;
1635 static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1636
1637 spin_lock(&interrupt_lock);
1638 tb = env->current_tb;
1639 /* if the cpu is currently executing code, we must unlink it and
1640 all the potentially executing TB */
1641 if (tb) {
1642 env->current_tb = NULL;
1643 tb_reset_jump_recursive(tb);
1644 }
1645 spin_unlock(&interrupt_lock);
1646 }
1647
1648 #ifndef CONFIG_USER_ONLY
1649 /* mask must never be zero, except for A20 change call */
1650 static void tcg_handle_interrupt(CPUState *env, int mask)
1651 {
1652 int old_mask;
1653
1654 old_mask = env->interrupt_request;
1655 env->interrupt_request |= mask;
1656
1657 /*
1658 * If called from iothread context, wake the target cpu in
1659 * case its halted.
1660 */
1661 if (!qemu_cpu_is_self(env)) {
1662 qemu_cpu_kick(env);
1663 return;
1664 }
1665
1666 if (use_icount) {
1667 env->icount_decr.u16.high = 0xffff;
1668 if (!can_do_io(env)
1669 && (mask & ~old_mask) != 0) {
1670 cpu_abort(env, "Raised interrupt while not in I/O function");
1671 }
1672 } else {
1673 cpu_unlink_tb(env);
1674 }
1675 }
1676
1677 CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
1678
1679 #else /* CONFIG_USER_ONLY */
1680
1681 void cpu_interrupt(CPUState *env, int mask)
1682 {
1683 env->interrupt_request |= mask;
1684 cpu_unlink_tb(env);
1685 }
1686 #endif /* CONFIG_USER_ONLY */
1687
1688 void cpu_reset_interrupt(CPUState *env, int mask)
1689 {
1690 env->interrupt_request &= ~mask;
1691 }
1692
1693 void cpu_exit(CPUState *env)
1694 {
1695 env->exit_request = 1;
1696 cpu_unlink_tb(env);
1697 }
1698
1699 const CPULogItem cpu_log_items[] = {
1700 { CPU_LOG_TB_OUT_ASM, "out_asm",
1701 "show generated host assembly code for each compiled TB" },
1702 { CPU_LOG_TB_IN_ASM, "in_asm",
1703 "show target assembly code for each compiled TB" },
1704 { CPU_LOG_TB_OP, "op",
1705 "show micro ops for each compiled TB" },
1706 { CPU_LOG_TB_OP_OPT, "op_opt",
1707 "show micro ops "
1708 #ifdef TARGET_I386
1709 "before eflags optimization and "
1710 #endif
1711 "after liveness analysis" },
1712 { CPU_LOG_INT, "int",
1713 "show interrupts/exceptions in short format" },
1714 { CPU_LOG_EXEC, "exec",
1715 "show trace before each executed TB (lots of logs)" },
1716 { CPU_LOG_TB_CPU, "cpu",
1717 "show CPU state before block translation" },
1718 #ifdef TARGET_I386
1719 { CPU_LOG_PCALL, "pcall",
1720 "show protected mode far calls/returns/exceptions" },
1721 { CPU_LOG_RESET, "cpu_reset",
1722 "show CPU state before CPU resets" },
1723 #endif
1724 #ifdef DEBUG_IOPORT
1725 { CPU_LOG_IOPORT, "ioport",
1726 "show all i/o ports accesses" },
1727 #endif
1728 { 0, NULL, NULL },
1729 };
1730
1731 #ifndef CONFIG_USER_ONLY
1732 static QLIST_HEAD(memory_client_list, CPUPhysMemoryClient) memory_client_list
1733 = QLIST_HEAD_INITIALIZER(memory_client_list);
1734
1735 static void cpu_notify_set_memory(target_phys_addr_t start_addr,
1736 ram_addr_t size,
1737 ram_addr_t phys_offset,
1738 bool log_dirty)
1739 {
1740 CPUPhysMemoryClient *client;
1741 QLIST_FOREACH(client, &memory_client_list, list) {
1742 client->set_memory(client, start_addr, size, phys_offset, log_dirty);
1743 }
1744 }
1745
1746 static int cpu_notify_sync_dirty_bitmap(target_phys_addr_t start,
1747 target_phys_addr_t end)
1748 {
1749 CPUPhysMemoryClient *client;
1750 QLIST_FOREACH(client, &memory_client_list, list) {
1751 int r = client->sync_dirty_bitmap(client, start, end);
1752 if (r < 0)
1753 return r;
1754 }
1755 return 0;
1756 }
1757
1758 static int cpu_notify_migration_log(int enable)
1759 {
1760 CPUPhysMemoryClient *client;
1761 QLIST_FOREACH(client, &memory_client_list, list) {
1762 int r = client->migration_log(client, enable);
1763 if (r < 0)
1764 return r;
1765 }
1766 return 0;
1767 }
1768
1769 struct last_map {
1770 target_phys_addr_t start_addr;
1771 ram_addr_t size;
1772 ram_addr_t phys_offset;
1773 };
1774
1775 /* The l1_phys_map provides the upper P_L1_BITs of the guest physical
1776 * address. Each intermediate table provides the next L2_BITs of guest
1777 * physical address space. The number of levels vary based on host and
1778 * guest configuration, making it efficient to build the final guest
1779 * physical address by seeding the L1 offset and shifting and adding in
1780 * each L2 offset as we recurse through them. */
1781 static void phys_page_for_each_1(CPUPhysMemoryClient *client, int level,
1782 void **lp, target_phys_addr_t addr,
1783 struct last_map *map)
1784 {
1785 int i;
1786
1787 if (*lp == NULL) {
1788 return;
1789 }
1790 if (level == 0) {
1791 PhysPageDesc *pd = *lp;
1792 addr <<= L2_BITS + TARGET_PAGE_BITS;
1793 for (i = 0; i < L2_SIZE; ++i) {
1794 if (pd[i].phys_offset != IO_MEM_UNASSIGNED) {
1795 target_phys_addr_t start_addr = addr | i << TARGET_PAGE_BITS;
1796
1797 if (map->size &&
1798 start_addr == map->start_addr + map->size &&
1799 pd[i].phys_offset == map->phys_offset + map->size) {
1800
1801 map->size += TARGET_PAGE_SIZE;
1802 continue;
1803 } else if (map->size) {
1804 client->set_memory(client, map->start_addr,
1805 map->size, map->phys_offset, false);
1806 }
1807
1808 map->start_addr = start_addr;
1809 map->size = TARGET_PAGE_SIZE;
1810 map->phys_offset = pd[i].phys_offset;
1811 }
1812 }
1813 } else {
1814 void **pp = *lp;
1815 for (i = 0; i < L2_SIZE; ++i) {
1816 phys_page_for_each_1(client, level - 1, pp + i,
1817 (addr << L2_BITS) | i, map);
1818 }
1819 }
1820 }
1821
1822 static void phys_page_for_each(CPUPhysMemoryClient *client)
1823 {
1824 int i;
1825 struct last_map map = { };
1826
1827 for (i = 0; i < P_L1_SIZE; ++i) {
1828 phys_page_for_each_1(client, P_L1_SHIFT / L2_BITS - 1,
1829 l1_phys_map + i, i, &map);
1830 }
1831 if (map.size) {
1832 client->set_memory(client, map.start_addr, map.size, map.phys_offset,
1833 false);
1834 }
1835 }
1836
1837 void cpu_register_phys_memory_client(CPUPhysMemoryClient *client)
1838 {
1839 QLIST_INSERT_HEAD(&memory_client_list, client, list);
1840 phys_page_for_each(client);
1841 }
1842
1843 void cpu_unregister_phys_memory_client(CPUPhysMemoryClient *client)
1844 {
1845 QLIST_REMOVE(client, list);
1846 }
1847 #endif
1848
1849 static int cmp1(const char *s1, int n, const char *s2)
1850 {
1851 if (strlen(s2) != n)
1852 return 0;
1853 return memcmp(s1, s2, n) == 0;
1854 }
1855
1856 /* takes a comma separated list of log masks. Return 0 if error. */
1857 int cpu_str_to_log_mask(const char *str)
1858 {
1859 const CPULogItem *item;
1860 int mask;
1861 const char *p, *p1;
1862
1863 p = str;
1864 mask = 0;
1865 for(;;) {
1866 p1 = strchr(p, ',');
1867 if (!p1)
1868 p1 = p + strlen(p);
1869 if(cmp1(p,p1-p,"all")) {
1870 for(item = cpu_log_items; item->mask != 0; item++) {
1871 mask |= item->mask;
1872 }
1873 } else {
1874 for(item = cpu_log_items; item->mask != 0; item++) {
1875 if (cmp1(p, p1 - p, item->name))
1876 goto found;
1877 }
1878 return 0;
1879 }
1880 found:
1881 mask |= item->mask;
1882 if (*p1 != ',')
1883 break;
1884 p = p1 + 1;
1885 }
1886 return mask;
1887 }
1888
1889 void cpu_abort(CPUState *env, const char *fmt, ...)
1890 {
1891 va_list ap;
1892 va_list ap2;
1893
1894 va_start(ap, fmt);
1895 va_copy(ap2, ap);
1896 fprintf(stderr, "qemu: fatal: ");
1897 vfprintf(stderr, fmt, ap);
1898 fprintf(stderr, "\n");
1899 #ifdef TARGET_I386
1900 cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1901 #else
1902 cpu_dump_state(env, stderr, fprintf, 0);
1903 #endif
1904 if (qemu_log_enabled()) {
1905 qemu_log("qemu: fatal: ");
1906 qemu_log_vprintf(fmt, ap2);
1907 qemu_log("\n");
1908 #ifdef TARGET_I386
1909 log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1910 #else
1911 log_cpu_state(env, 0);
1912 #endif
1913 qemu_log_flush();
1914 qemu_log_close();
1915 }
1916 va_end(ap2);
1917 va_end(ap);
1918 #if defined(CONFIG_USER_ONLY)
1919 {
1920 struct sigaction act;
1921 sigfillset(&act.sa_mask);
1922 act.sa_handler = SIG_DFL;
1923 sigaction(SIGABRT, &act, NULL);
1924 }
1925 #endif
1926 abort();
1927 }
1928
1929 CPUState *cpu_copy(CPUState *env)
1930 {
1931 CPUState *new_env = cpu_init(env->cpu_model_str);
1932 CPUState *next_cpu = new_env->next_cpu;
1933 int cpu_index = new_env->cpu_index;
1934 #if defined(TARGET_HAS_ICE)
1935 CPUBreakpoint *bp;
1936 CPUWatchpoint *wp;
1937 #endif
1938
1939 memcpy(new_env, env, sizeof(CPUState));
1940
1941 /* Preserve chaining and index. */
1942 new_env->next_cpu = next_cpu;
1943 new_env->cpu_index = cpu_index;
1944
1945 /* Clone all break/watchpoints.
1946 Note: Once we support ptrace with hw-debug register access, make sure
1947 BP_CPU break/watchpoints are handled correctly on clone. */
1948 QTAILQ_INIT(&env->breakpoints);
1949 QTAILQ_INIT(&env->watchpoints);
1950 #if defined(TARGET_HAS_ICE)
1951 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1952 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1953 }
1954 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1955 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1956 wp->flags, NULL);
1957 }
1958 #endif
1959
1960 return new_env;
1961 }
1962
1963 #if !defined(CONFIG_USER_ONLY)
1964
1965 static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1966 {
1967 unsigned int i;
1968
1969 /* Discard jump cache entries for any tb which might potentially
1970 overlap the flushed page. */
1971 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1972 memset (&env->tb_jmp_cache[i], 0,
1973 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1974
1975 i = tb_jmp_cache_hash_page(addr);
1976 memset (&env->tb_jmp_cache[i], 0,
1977 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1978 }
1979
1980 static CPUTLBEntry s_cputlb_empty_entry = {
1981 .addr_read = -1,
1982 .addr_write = -1,
1983 .addr_code = -1,
1984 .addend = -1,
1985 };
1986
1987 /* NOTE: if flush_global is true, also flush global entries (not
1988 implemented yet) */
1989 void tlb_flush(CPUState *env, int flush_global)
1990 {
1991 int i;
1992
1993 #if defined(DEBUG_TLB)
1994 printf("tlb_flush:\n");
1995 #endif
1996 /* must reset current TB so that interrupts cannot modify the
1997 links while we are modifying them */
1998 env->current_tb = NULL;
1999
2000 for(i = 0; i < CPU_TLB_SIZE; i++) {
2001 int mmu_idx;
2002 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2003 env->tlb_table[mmu_idx][i] = s_cputlb_empty_entry;
2004 }
2005 }
2006
2007 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
2008
2009 env->tlb_flush_addr = -1;
2010 env->tlb_flush_mask = 0;
2011 tlb_flush_count++;
2012 }
2013
2014 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
2015 {
2016 if (addr == (tlb_entry->addr_read &
2017 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
2018 addr == (tlb_entry->addr_write &
2019 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
2020 addr == (tlb_entry->addr_code &
2021 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
2022 *tlb_entry = s_cputlb_empty_entry;
2023 }
2024 }
2025
2026 void tlb_flush_page(CPUState *env, target_ulong addr)
2027 {
2028 int i;
2029 int mmu_idx;
2030
2031 #if defined(DEBUG_TLB)
2032 printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
2033 #endif
2034 /* Check if we need to flush due to large pages. */
2035 if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
2036 #if defined(DEBUG_TLB)
2037 printf("tlb_flush_page: forced full flush ("
2038 TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
2039 env->tlb_flush_addr, env->tlb_flush_mask);
2040 #endif
2041 tlb_flush(env, 1);
2042 return;
2043 }
2044 /* must reset current TB so that interrupts cannot modify the
2045 links while we are modifying them */
2046 env->current_tb = NULL;
2047
2048 addr &= TARGET_PAGE_MASK;
2049 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2050 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2051 tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
2052
2053 tlb_flush_jmp_cache(env, addr);
2054 }
2055
2056 /* update the TLBs so that writes to code in the virtual page 'addr'
2057 can be detected */
2058 static void tlb_protect_code(ram_addr_t ram_addr)
2059 {
2060 cpu_physical_memory_reset_dirty(ram_addr,
2061 ram_addr + TARGET_PAGE_SIZE,
2062 CODE_DIRTY_FLAG);
2063 }
2064
2065 /* update the TLB so that writes in physical page 'phys_addr' are no longer
2066 tested for self modifying code */
2067 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
2068 target_ulong vaddr)
2069 {
2070 cpu_physical_memory_set_dirty_flags(ram_addr, CODE_DIRTY_FLAG);
2071 }
2072
2073 static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
2074 unsigned long start, unsigned long length)
2075 {
2076 unsigned long addr;
2077 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2078 addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
2079 if ((addr - start) < length) {
2080 tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
2081 }
2082 }
2083 }
2084
2085 /* Note: start and end must be within the same ram block. */
2086 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
2087 int dirty_flags)
2088 {
2089 CPUState *env;
2090 unsigned long length, start1;
2091 int i;
2092
2093 start &= TARGET_PAGE_MASK;
2094 end = TARGET_PAGE_ALIGN(end);
2095
2096 length = end - start;
2097 if (length == 0)
2098 return;
2099 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
2100
2101 /* we modify the TLB cache so that the dirty bit will be set again
2102 when accessing the range */
2103 start1 = (unsigned long)qemu_safe_ram_ptr(start);
2104 /* Check that we don't span multiple blocks - this breaks the
2105 address comparisons below. */
2106 if ((unsigned long)qemu_safe_ram_ptr(end - 1) - start1
2107 != (end - 1) - start) {
2108 abort();
2109 }
2110
2111 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2112 int mmu_idx;
2113 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2114 for(i = 0; i < CPU_TLB_SIZE; i++)
2115 tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
2116 start1, length);
2117 }
2118 }
2119 }
2120
2121 int cpu_physical_memory_set_dirty_tracking(int enable)
2122 {
2123 int ret = 0;
2124 in_migration = enable;
2125 ret = cpu_notify_migration_log(!!enable);
2126 return ret;
2127 }
2128
2129 int cpu_physical_memory_get_dirty_tracking(void)
2130 {
2131 return in_migration;
2132 }
2133
2134 int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
2135 target_phys_addr_t end_addr)
2136 {
2137 int ret;
2138
2139 ret = cpu_notify_sync_dirty_bitmap(start_addr, end_addr);
2140 return ret;
2141 }
2142
2143 int cpu_physical_log_start(target_phys_addr_t start_addr,
2144 ram_addr_t size)
2145 {
2146 CPUPhysMemoryClient *client;
2147 QLIST_FOREACH(client, &memory_client_list, list) {
2148 if (client->log_start) {
2149 int r = client->log_start(client, start_addr, size);
2150 if (r < 0) {
2151 return r;
2152 }
2153 }
2154 }
2155 return 0;
2156 }
2157
2158 int cpu_physical_log_stop(target_phys_addr_t start_addr,
2159 ram_addr_t size)
2160 {
2161 CPUPhysMemoryClient *client;
2162 QLIST_FOREACH(client, &memory_client_list, list) {
2163 if (client->log_stop) {
2164 int r = client->log_stop(client, start_addr, size);
2165 if (r < 0) {
2166 return r;
2167 }
2168 }
2169 }
2170 return 0;
2171 }
2172
2173 static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
2174 {
2175 ram_addr_t ram_addr;
2176 void *p;
2177
2178 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2179 p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
2180 + tlb_entry->addend);
2181 ram_addr = qemu_ram_addr_from_host_nofail(p);
2182 if (!cpu_physical_memory_is_dirty(ram_addr)) {
2183 tlb_entry->addr_write |= TLB_NOTDIRTY;
2184 }
2185 }
2186 }
2187
2188 /* update the TLB according to the current state of the dirty bits */
2189 void cpu_tlb_update_dirty(CPUState *env)
2190 {
2191 int i;
2192 int mmu_idx;
2193 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2194 for(i = 0; i < CPU_TLB_SIZE; i++)
2195 tlb_update_dirty(&env->tlb_table[mmu_idx][i]);
2196 }
2197 }
2198
2199 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
2200 {
2201 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
2202 tlb_entry->addr_write = vaddr;
2203 }
2204
2205 /* update the TLB corresponding to virtual page vaddr
2206 so that it is no longer dirty */
2207 static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
2208 {
2209 int i;
2210 int mmu_idx;
2211
2212 vaddr &= TARGET_PAGE_MASK;
2213 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2214 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2215 tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
2216 }
2217
2218 /* Our TLB does not support large pages, so remember the area covered by
2219 large pages and trigger a full TLB flush if these are invalidated. */
2220 static void tlb_add_large_page(CPUState *env, target_ulong vaddr,
2221 target_ulong size)
2222 {
2223 target_ulong mask = ~(size - 1);
2224
2225 if (env->tlb_flush_addr == (target_ulong)-1) {
2226 env->tlb_flush_addr = vaddr & mask;
2227 env->tlb_flush_mask = mask;
2228 return;
2229 }
2230 /* Extend the existing region to include the new page.
2231 This is a compromise between unnecessary flushes and the cost
2232 of maintaining a full variable size TLB. */
2233 mask &= env->tlb_flush_mask;
2234 while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
2235 mask <<= 1;
2236 }
2237 env->tlb_flush_addr &= mask;
2238 env->tlb_flush_mask = mask;
2239 }
2240
2241 /* Add a new TLB entry. At most one entry for a given virtual address
2242 is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
2243 supplied size is only used by tlb_flush_page. */
2244 void tlb_set_page(CPUState *env, target_ulong vaddr,
2245 target_phys_addr_t paddr, int prot,
2246 int mmu_idx, target_ulong size)
2247 {
2248 PhysPageDesc *p;
2249 unsigned long pd;
2250 unsigned int index;
2251 target_ulong address;
2252 target_ulong code_address;
2253 unsigned long addend;
2254 CPUTLBEntry *te;
2255 CPUWatchpoint *wp;
2256 target_phys_addr_t iotlb;
2257
2258 assert(size >= TARGET_PAGE_SIZE);
2259 if (size != TARGET_PAGE_SIZE) {
2260 tlb_add_large_page(env, vaddr, size);
2261 }
2262 p = phys_page_find(paddr >> TARGET_PAGE_BITS);
2263 if (!p) {
2264 pd = IO_MEM_UNASSIGNED;
2265 } else {
2266 pd = p->phys_offset;
2267 }
2268 #if defined(DEBUG_TLB)
2269 printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
2270 " prot=%x idx=%d pd=0x%08lx\n",
2271 vaddr, paddr, prot, mmu_idx, pd);
2272 #endif
2273
2274 address = vaddr;
2275 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
2276 /* IO memory case (romd handled later) */
2277 address |= TLB_MMIO;
2278 }
2279 addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
2280 if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
2281 /* Normal RAM. */
2282 iotlb = pd & TARGET_PAGE_MASK;
2283 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
2284 iotlb |= IO_MEM_NOTDIRTY;
2285 else
2286 iotlb |= IO_MEM_ROM;
2287 } else {
2288 /* IO handlers are currently passed a physical address.
2289 It would be nice to pass an offset from the base address
2290 of that region. This would avoid having to special case RAM,
2291 and avoid full address decoding in every device.
2292 We can't use the high bits of pd for this because
2293 IO_MEM_ROMD uses these as a ram address. */
2294 iotlb = (pd & ~TARGET_PAGE_MASK);
2295 if (p) {
2296 iotlb += p->region_offset;
2297 } else {
2298 iotlb += paddr;
2299 }
2300 }
2301
2302 code_address = address;
2303 /* Make accesses to pages with watchpoints go via the
2304 watchpoint trap routines. */
2305 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
2306 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2307 /* Avoid trapping reads of pages with a write breakpoint. */
2308 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
2309 iotlb = io_mem_watch + paddr;
2310 address |= TLB_MMIO;
2311 break;
2312 }
2313 }
2314 }
2315
2316 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2317 env->iotlb[mmu_idx][index] = iotlb - vaddr;
2318 te = &env->tlb_table[mmu_idx][index];
2319 te->addend = addend - vaddr;
2320 if (prot & PAGE_READ) {
2321 te->addr_read = address;
2322 } else {
2323 te->addr_read = -1;
2324 }
2325
2326 if (prot & PAGE_EXEC) {
2327 te->addr_code = code_address;
2328 } else {
2329 te->addr_code = -1;
2330 }
2331 if (prot & PAGE_WRITE) {
2332 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2333 (pd & IO_MEM_ROMD)) {
2334 /* Write access calls the I/O callback. */
2335 te->addr_write = address | TLB_MMIO;
2336 } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2337 !cpu_physical_memory_is_dirty(pd)) {
2338 te->addr_write = address | TLB_NOTDIRTY;
2339 } else {
2340 te->addr_write = address;
2341 }
2342 } else {
2343 te->addr_write = -1;
2344 }
2345 }
2346
2347 #else
2348
2349 void tlb_flush(CPUState *env, int flush_global)
2350 {
2351 }
2352
2353 void tlb_flush_page(CPUState *env, target_ulong addr)
2354 {
2355 }
2356
2357 /*
2358 * Walks guest process memory "regions" one by one
2359 * and calls callback function 'fn' for each region.
2360 */
2361
2362 struct walk_memory_regions_data
2363 {
2364 walk_memory_regions_fn fn;
2365 void *priv;
2366 unsigned long start;
2367 int prot;
2368 };
2369
2370 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
2371 abi_ulong end, int new_prot)
2372 {
2373 if (data->start != -1ul) {
2374 int rc = data->fn(data->priv, data->start, end, data->prot);
2375 if (rc != 0) {
2376 return rc;
2377 }
2378 }
2379
2380 data->start = (new_prot ? end : -1ul);
2381 data->prot = new_prot;
2382
2383 return 0;
2384 }
2385
2386 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
2387 abi_ulong base, int level, void **lp)
2388 {
2389 abi_ulong pa;
2390 int i, rc;
2391
2392 if (*lp == NULL) {
2393 return walk_memory_regions_end(data, base, 0);
2394 }
2395
2396 if (level == 0) {
2397 PageDesc *pd = *lp;
2398 for (i = 0; i < L2_SIZE; ++i) {
2399 int prot = pd[i].flags;
2400
2401 pa = base | (i << TARGET_PAGE_BITS);
2402 if (prot != data->prot) {
2403 rc = walk_memory_regions_end(data, pa, prot);
2404 if (rc != 0) {
2405 return rc;
2406 }
2407 }
2408 }
2409 } else {
2410 void **pp = *lp;
2411 for (i = 0; i < L2_SIZE; ++i) {
2412 pa = base | ((abi_ulong)i <<
2413 (TARGET_PAGE_BITS + L2_BITS * level));
2414 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
2415 if (rc != 0) {
2416 return rc;
2417 }
2418 }
2419 }
2420
2421 return 0;
2422 }
2423
2424 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
2425 {
2426 struct walk_memory_regions_data data;
2427 unsigned long i;
2428
2429 data.fn = fn;
2430 data.priv = priv;
2431 data.start = -1ul;
2432 data.prot = 0;
2433
2434 for (i = 0; i < V_L1_SIZE; i++) {
2435 int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
2436 V_L1_SHIFT / L2_BITS - 1, l1_map + i);
2437 if (rc != 0) {
2438 return rc;
2439 }
2440 }
2441
2442 return walk_memory_regions_end(&data, 0, 0);
2443 }
2444
2445 static int dump_region(void *priv, abi_ulong start,
2446 abi_ulong end, unsigned long prot)
2447 {
2448 FILE *f = (FILE *)priv;
2449
2450 (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
2451 " "TARGET_ABI_FMT_lx" %c%c%c\n",
2452 start, end, end - start,
2453 ((prot & PAGE_READ) ? 'r' : '-'),
2454 ((prot & PAGE_WRITE) ? 'w' : '-'),
2455 ((prot & PAGE_EXEC) ? 'x' : '-'));
2456
2457 return (0);
2458 }
2459
2460 /* dump memory mappings */
2461 void page_dump(FILE *f)
2462 {
2463 (void) fprintf(f, "%-8s %-8s %-8s %s\n",
2464 "start", "end", "size", "prot");
2465 walk_memory_regions(f, dump_region);
2466 }
2467
2468 int page_get_flags(target_ulong address)
2469 {
2470 PageDesc *p;
2471
2472 p = page_find(address >> TARGET_PAGE_BITS);
2473 if (!p)
2474 return 0;
2475 return p->flags;
2476 }
2477
2478 /* Modify the flags of a page and invalidate the code if necessary.
2479 The flag PAGE_WRITE_ORG is positioned automatically depending
2480 on PAGE_WRITE. The mmap_lock should already be held. */
2481 void page_set_flags(target_ulong start, target_ulong end, int flags)
2482 {
2483 target_ulong addr, len;
2484
2485 /* This function should never be called with addresses outside the
2486 guest address space. If this assert fires, it probably indicates
2487 a missing call to h2g_valid. */
2488 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2489 assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2490 #endif
2491 assert(start < end);
2492
2493 start = start & TARGET_PAGE_MASK;
2494 end = TARGET_PAGE_ALIGN(end);
2495
2496 if (flags & PAGE_WRITE) {
2497 flags |= PAGE_WRITE_ORG;
2498 }
2499
2500 for (addr = start, len = end - start;
2501 len != 0;
2502 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2503 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2504
2505 /* If the write protection bit is set, then we invalidate
2506 the code inside. */
2507 if (!(p->flags & PAGE_WRITE) &&
2508 (flags & PAGE_WRITE) &&
2509 p->first_tb) {
2510 tb_invalidate_phys_page(addr, 0, NULL);
2511 }
2512 p->flags = flags;
2513 }
2514 }
2515
2516 int page_check_range(target_ulong start, target_ulong len, int flags)
2517 {
2518 PageDesc *p;
2519 target_ulong end;
2520 target_ulong addr;
2521
2522 /* This function should never be called with addresses outside the
2523 guest address space. If this assert fires, it probably indicates
2524 a missing call to h2g_valid. */
2525 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2526 assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2527 #endif
2528
2529 if (len == 0) {
2530 return 0;
2531 }
2532 if (start + len - 1 < start) {
2533 /* We've wrapped around. */
2534 return -1;
2535 }
2536
2537 end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2538 start = start & TARGET_PAGE_MASK;
2539
2540 for (addr = start, len = end - start;
2541 len != 0;
2542 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2543 p = page_find(addr >> TARGET_PAGE_BITS);
2544 if( !p )
2545 return -1;
2546 if( !(p->flags & PAGE_VALID) )
2547 return -1;
2548
2549 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2550 return -1;
2551 if (flags & PAGE_WRITE) {
2552 if (!(p->flags & PAGE_WRITE_ORG))
2553 return -1;
2554 /* unprotect the page if it was put read-only because it
2555 contains translated code */
2556 if (!(p->flags & PAGE_WRITE)) {
2557 if (!page_unprotect(addr, 0, NULL))
2558 return -1;
2559 }
2560 return 0;
2561 }
2562 }
2563 return 0;
2564 }
2565
2566 /* called from signal handler: invalidate the code and unprotect the
2567 page. Return TRUE if the fault was successfully handled. */
2568 int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2569 {
2570 unsigned int prot;
2571 PageDesc *p;
2572 target_ulong host_start, host_end, addr;
2573
2574 /* Technically this isn't safe inside a signal handler. However we
2575 know this only ever happens in a synchronous SEGV handler, so in
2576 practice it seems to be ok. */
2577 mmap_lock();
2578
2579 p = page_find(address >> TARGET_PAGE_BITS);
2580 if (!p) {
2581 mmap_unlock();
2582 return 0;
2583 }
2584
2585 /* if the page was really writable, then we change its
2586 protection back to writable */
2587 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
2588 host_start = address & qemu_host_page_mask;
2589 host_end = host_start + qemu_host_page_size;
2590
2591 prot = 0;
2592 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
2593 p = page_find(addr >> TARGET_PAGE_BITS);
2594 p->flags |= PAGE_WRITE;
2595 prot |= p->flags;
2596
2597 /* and since the content will be modified, we must invalidate
2598 the corresponding translated code. */
2599 tb_invalidate_phys_page(addr, pc, puc);
2600 #ifdef DEBUG_TB_CHECK
2601 tb_invalidate_check(addr);
2602 #endif
2603 }
2604 mprotect((void *)g2h(host_start), qemu_host_page_size,
2605 prot & PAGE_BITS);
2606
2607 mmap_unlock();
2608 return 1;
2609 }
2610 mmap_unlock();
2611 return 0;
2612 }
2613
2614 static inline void tlb_set_dirty(CPUState *env,
2615 unsigned long addr, target_ulong vaddr)
2616 {
2617 }
2618 #endif /* defined(CONFIG_USER_ONLY) */
2619
2620 #if !defined(CONFIG_USER_ONLY)
2621
2622 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
2623 typedef struct subpage_t {
2624 target_phys_addr_t base;
2625 ram_addr_t sub_io_index[TARGET_PAGE_SIZE];
2626 ram_addr_t region_offset[TARGET_PAGE_SIZE];
2627 } subpage_t;
2628
2629 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2630 ram_addr_t memory, ram_addr_t region_offset);
2631 static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2632 ram_addr_t orig_memory,
2633 ram_addr_t region_offset);
2634 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2635 need_subpage) \
2636 do { \
2637 if (addr > start_addr) \
2638 start_addr2 = 0; \
2639 else { \
2640 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2641 if (start_addr2 > 0) \
2642 need_subpage = 1; \
2643 } \
2644 \
2645 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2646 end_addr2 = TARGET_PAGE_SIZE - 1; \
2647 else { \
2648 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2649 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2650 need_subpage = 1; \
2651 } \
2652 } while (0)
2653
2654 /* register physical memory.
2655 For RAM, 'size' must be a multiple of the target page size.
2656 If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2657 io memory page. The address used when calling the IO function is
2658 the offset from the start of the region, plus region_offset. Both
2659 start_addr and region_offset are rounded down to a page boundary
2660 before calculating this offset. This should not be a problem unless
2661 the low bits of start_addr and region_offset differ. */
2662 void cpu_register_physical_memory_log(target_phys_addr_t start_addr,
2663 ram_addr_t size,
2664 ram_addr_t phys_offset,
2665 ram_addr_t region_offset,
2666 bool log_dirty)
2667 {
2668 target_phys_addr_t addr, end_addr;
2669 PhysPageDesc *p;
2670 CPUState *env;
2671 ram_addr_t orig_size = size;
2672 subpage_t *subpage;
2673
2674 assert(size);
2675 cpu_notify_set_memory(start_addr, size, phys_offset, log_dirty);
2676
2677 if (phys_offset == IO_MEM_UNASSIGNED) {
2678 region_offset = start_addr;
2679 }
2680 region_offset &= TARGET_PAGE_MASK;
2681 size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2682 end_addr = start_addr + (target_phys_addr_t)size;
2683
2684 addr = start_addr;
2685 do {
2686 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2687 if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2688 ram_addr_t orig_memory = p->phys_offset;
2689 target_phys_addr_t start_addr2, end_addr2;
2690 int need_subpage = 0;
2691
2692 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2693 need_subpage);
2694 if (need_subpage) {
2695 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2696 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2697 &p->phys_offset, orig_memory,
2698 p->region_offset);
2699 } else {
2700 subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2701 >> IO_MEM_SHIFT];
2702 }
2703 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2704 region_offset);
2705 p->region_offset = 0;
2706 } else {
2707 p->phys_offset = phys_offset;
2708 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2709 (phys_offset & IO_MEM_ROMD))
2710 phys_offset += TARGET_PAGE_SIZE;
2711 }
2712 } else {
2713 p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2714 p->phys_offset = phys_offset;
2715 p->region_offset = region_offset;
2716 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2717 (phys_offset & IO_MEM_ROMD)) {
2718 phys_offset += TARGET_PAGE_SIZE;
2719 } else {
2720 target_phys_addr_t start_addr2, end_addr2;
2721 int need_subpage = 0;
2722
2723 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2724 end_addr2, need_subpage);
2725
2726 if (need_subpage) {
2727 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2728 &p->phys_offset, IO_MEM_UNASSIGNED,
2729 addr & TARGET_PAGE_MASK);
2730 subpage_register(subpage, start_addr2, end_addr2,
2731 phys_offset, region_offset);
2732 p->region_offset = 0;
2733 }
2734 }
2735 }
2736 region_offset += TARGET_PAGE_SIZE;
2737 addr += TARGET_PAGE_SIZE;
2738 } while (addr != end_addr);
2739
2740 /* since each CPU stores ram addresses in its TLB cache, we must
2741 reset the modified entries */
2742 /* XXX: slow ! */
2743 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2744 tlb_flush(env, 1);
2745 }
2746 }
2747
2748 /* XXX: temporary until new memory mapping API */
2749 ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2750 {
2751 PhysPageDesc *p;
2752
2753 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2754 if (!p)
2755 return IO_MEM_UNASSIGNED;
2756 return p->phys_offset;
2757 }
2758
2759 void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2760 {
2761 if (kvm_enabled())
2762 kvm_coalesce_mmio_region(addr, size);
2763 }
2764
2765 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2766 {
2767 if (kvm_enabled())
2768 kvm_uncoalesce_mmio_region(addr, size);
2769 }
2770
2771 void qemu_flush_coalesced_mmio_buffer(void)
2772 {
2773 if (kvm_enabled())
2774 kvm_flush_coalesced_mmio_buffer();
2775 }
2776
2777 #if defined(__linux__) && !defined(TARGET_S390X)
2778
2779 #include <sys/vfs.h>
2780
2781 #define HUGETLBFS_MAGIC 0x958458f6
2782
2783 static long gethugepagesize(const char *path)
2784 {
2785 struct statfs fs;
2786 int ret;
2787
2788 do {
2789 ret = statfs(path, &fs);
2790 } while (ret != 0 && errno == EINTR);
2791
2792 if (ret != 0) {
2793 perror(path);
2794 return 0;
2795 }
2796
2797 if (fs.f_type != HUGETLBFS_MAGIC)
2798 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
2799
2800 return fs.f_bsize;
2801 }
2802
2803 static void *file_ram_alloc(RAMBlock *block,
2804 ram_addr_t memory,
2805 const char *path)
2806 {
2807 char *filename;
2808 void *area;
2809 int fd;
2810 #ifdef MAP_POPULATE
2811 int flags;
2812 #endif
2813 unsigned long hpagesize;
2814
2815 hpagesize = gethugepagesize(path);
2816 if (!hpagesize) {
2817 return NULL;
2818 }
2819
2820 if (memory < hpagesize) {
2821 return NULL;
2822 }
2823
2824 if (kvm_enabled() && !kvm_has_sync_mmu()) {
2825 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
2826 return NULL;
2827 }
2828
2829 if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
2830 return NULL;
2831 }
2832
2833 fd = mkstemp(filename);
2834 if (fd < 0) {
2835 perror("unable to create backing store for hugepages");
2836 free(filename);
2837 return NULL;
2838 }
2839 unlink(filename);
2840 free(filename);
2841
2842 memory = (memory+hpagesize-1) & ~(hpagesize-1);
2843
2844 /*
2845 * ftruncate is not supported by hugetlbfs in older
2846 * hosts, so don't bother bailing out on errors.
2847 * If anything goes wrong with it under other filesystems,
2848 * mmap will fail.
2849 */
2850 if (ftruncate(fd, memory))
2851 perror("ftruncate");
2852
2853 #ifdef MAP_POPULATE
2854 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
2855 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
2856 * to sidestep this quirk.
2857 */
2858 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
2859 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
2860 #else
2861 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
2862 #endif
2863 if (area == MAP_FAILED) {
2864 perror("file_ram_alloc: can't mmap RAM pages");
2865 close(fd);
2866 return (NULL);
2867 }
2868 block->fd = fd;
2869 return area;
2870 }
2871 #endif
2872
2873 static ram_addr_t find_ram_offset(ram_addr_t size)
2874 {
2875 RAMBlock *block, *next_block;
2876 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
2877
2878 if (QLIST_EMPTY(&ram_list.blocks))
2879 return 0;
2880
2881 QLIST_FOREACH(block, &ram_list.blocks, next) {
2882 ram_addr_t end, next = RAM_ADDR_MAX;
2883
2884 end = block->offset + block->length;
2885
2886 QLIST_FOREACH(next_block, &ram_list.blocks, next) {
2887 if (next_block->offset >= end) {
2888 next = MIN(next, next_block->offset);
2889 }
2890 }
2891 if (next - end >= size && next - end < mingap) {
2892 offset = end;
2893 mingap = next - end;
2894 }
2895 }
2896
2897 if (offset == RAM_ADDR_MAX) {
2898 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
2899 (uint64_t)size);
2900 abort();
2901 }
2902
2903 return offset;
2904 }
2905
2906 static ram_addr_t last_ram_offset(void)
2907 {
2908 RAMBlock *block;
2909 ram_addr_t last = 0;
2910
2911 QLIST_FOREACH(block, &ram_list.blocks, next)
2912 last = MAX(last, block->offset + block->length);
2913
2914 return last;
2915 }
2916
2917 ram_addr_t qemu_ram_alloc_from_ptr(DeviceState *dev, const char *name,
2918 ram_addr_t size, void *host)
2919 {
2920 RAMBlock *new_block, *block;
2921
2922 size = TARGET_PAGE_ALIGN(size);
2923 new_block = g_malloc0(sizeof(*new_block));
2924
2925 if (dev && dev->parent_bus && dev->parent_bus->info->get_dev_path) {
2926 char *id = dev->parent_bus->info->get_dev_path(dev);
2927 if (id) {
2928 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
2929 g_free(id);
2930 }
2931 }
2932 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
2933
2934 QLIST_FOREACH(block, &ram_list.blocks, next) {
2935 if (!strcmp(block->idstr, new_block->idstr)) {
2936 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
2937 new_block->idstr);
2938 abort();
2939 }
2940 }
2941
2942 new_block->offset = find_ram_offset(size);
2943 if (host) {
2944 new_block->host = host;
2945 new_block->flags |= RAM_PREALLOC_MASK;
2946 } else {
2947 if (mem_path) {
2948 #if defined (__linux__) && !defined(TARGET_S390X)
2949 new_block->host = file_ram_alloc(new_block, size, mem_path);
2950 if (!new_block->host) {
2951 new_block->host = qemu_vmalloc(size);
2952 qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
2953 }
2954 #else
2955 fprintf(stderr, "-mem-path option unsupported\n");
2956 exit(1);
2957 #endif
2958 } else {
2959 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2960 /* S390 KVM requires the topmost vma of the RAM to be smaller than
2961 an system defined value, which is at least 256GB. Larger systems
2962 have larger values. We put the guest between the end of data
2963 segment (system break) and this value. We use 32GB as a base to
2964 have enough room for the system break to grow. */
2965 new_block->host = mmap((void*)0x800000000, size,
2966 PROT_EXEC|PROT_READ|PROT_WRITE,
2967 MAP_SHARED | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
2968 if (new_block->host == MAP_FAILED) {
2969 fprintf(stderr, "Allocating RAM failed\n");
2970 abort();
2971 }
2972 #else
2973 if (xen_enabled()) {
2974 xen_ram_alloc(new_block->offset, size);
2975 } else {
2976 new_block->host = qemu_vmalloc(size);
2977 }
2978 #endif
2979 qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
2980 }
2981 }
2982 new_block->length = size;
2983
2984 QLIST_INSERT_HEAD(&ram_list.blocks, new_block, next);
2985
2986 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
2987 last_ram_offset() >> TARGET_PAGE_BITS);
2988 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
2989 0xff, size >> TARGET_PAGE_BITS);
2990
2991 if (kvm_enabled())
2992 kvm_setup_guest_memory(new_block->host, size);
2993
2994 return new_block->offset;
2995 }
2996
2997 ram_addr_t qemu_ram_alloc(DeviceState *dev, const char *name, ram_addr_t size)
2998 {
2999 return qemu_ram_alloc_from_ptr(dev, name, size, NULL);
3000 }
3001
3002 void qemu_ram_free_from_ptr(ram_addr_t addr)
3003 {
3004 RAMBlock *block;
3005
3006 QLIST_FOREACH(block, &ram_list.blocks, next) {
3007 if (addr == block->offset) {
3008 QLIST_REMOVE(block, next);
3009 g_free(block);
3010 return;
3011 }
3012 }
3013 }
3014
3015 void qemu_ram_free(ram_addr_t addr)
3016 {
3017 RAMBlock *block;
3018
3019 QLIST_FOREACH(block, &ram_list.blocks, next) {
3020 if (addr == block->offset) {
3021 QLIST_REMOVE(block, next);
3022 if (block->flags & RAM_PREALLOC_MASK) {
3023 ;
3024 } else if (mem_path) {
3025 #if defined (__linux__) && !defined(TARGET_S390X)
3026 if (block->fd) {
3027 munmap(block->host, block->length);
3028 close(block->fd);
3029 } else {
3030 qemu_vfree(block->host);
3031 }
3032 #else
3033 abort();
3034 #endif
3035 } else {
3036 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
3037 munmap(block->host, block->length);
3038 #else
3039 if (xen_enabled()) {
3040 xen_invalidate_map_cache_entry(block->host);
3041 } else {
3042 qemu_vfree(block->host);
3043 }
3044 #endif
3045 }
3046 g_free(block);
3047 return;
3048 }
3049 }
3050
3051 }
3052
3053 #ifndef _WIN32
3054 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
3055 {
3056 RAMBlock *block;
3057 ram_addr_t offset;
3058 int flags;
3059 void *area, *vaddr;
3060
3061 QLIST_FOREACH(block, &ram_list.blocks, next) {
3062 offset = addr - block->offset;
3063 if (offset < block->length) {
3064 vaddr = block->host + offset;
3065 if (block->flags & RAM_PREALLOC_MASK) {
3066 ;
3067 } else {
3068 flags = MAP_FIXED;
3069 munmap(vaddr, length);
3070 if (mem_path) {
3071 #if defined(__linux__) && !defined(TARGET_S390X)
3072 if (block->fd) {
3073 #ifdef MAP_POPULATE
3074 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
3075 MAP_PRIVATE;
3076 #else
3077 flags |= MAP_PRIVATE;
3078 #endif
3079 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
3080 flags, block->fd, offset);
3081 } else {
3082 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
3083 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
3084 flags, -1, 0);
3085 }
3086 #else
3087 abort();
3088 #endif
3089 } else {
3090 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
3091 flags |= MAP_SHARED | MAP_ANONYMOUS;
3092 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
3093 flags, -1, 0);
3094 #else
3095 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
3096 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
3097 flags, -1, 0);
3098 #endif
3099 }
3100 if (area != vaddr) {
3101 fprintf(stderr, "Could not remap addr: "
3102 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
3103 length, addr);
3104 exit(1);
3105 }
3106 qemu_madvise(vaddr, length, QEMU_MADV_MERGEABLE);
3107 }
3108 return;
3109 }
3110 }
3111 }
3112 #endif /* !_WIN32 */
3113
3114 /* Return a host pointer to ram allocated with qemu_ram_alloc.
3115 With the exception of the softmmu code in this file, this should
3116 only be used for local memory (e.g. video ram) that the device owns,
3117 and knows it isn't going to access beyond the end of the block.
3118
3119 It should not be used for general purpose DMA.
3120 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
3121 */
3122 void *qemu_get_ram_ptr(ram_addr_t addr)
3123 {
3124 RAMBlock *block;
3125
3126 QLIST_FOREACH(block, &ram_list.blocks, next) {
3127 if (addr - block->offset < block->length) {
3128 /* Move this entry to to start of the list. */
3129 if (block != QLIST_FIRST(&ram_list.blocks)) {
3130 QLIST_REMOVE(block, next);
3131 QLIST_INSERT_HEAD(&ram_list.blocks, block, next);
3132 }
3133 if (xen_enabled()) {
3134 /* We need to check if the requested address is in the RAM
3135 * because we don't want to map the entire memory in QEMU.
3136 * In that case just map until the end of the page.
3137 */
3138 if (block->offset == 0) {
3139 return xen_map_cache(addr, 0, 0);
3140 } else if (block->host == NULL) {
3141 block->host =
3142 xen_map_cache(block->offset, block->length, 1);
3143 }
3144 }
3145 return block->host + (addr - block->offset);
3146 }
3147 }
3148
3149 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
3150 abort();
3151
3152 return NULL;
3153 }
3154
3155 /* Return a host pointer to ram allocated with qemu_ram_alloc.
3156 * Same as qemu_get_ram_ptr but avoid reordering ramblocks.
3157 */
3158 void *qemu_safe_ram_ptr(ram_addr_t addr)
3159 {
3160 RAMBlock *block;
3161
3162 QLIST_FOREACH(block, &ram_list.blocks, next) {
3163 if (addr - block->offset < block->length) {
3164 if (xen_enabled()) {
3165 /* We need to check if the requested address is in the RAM
3166 * because we don't want to map the entire memory in QEMU.
3167 * In that case just map until the end of the page.
3168 */
3169 if (block->offset == 0) {
3170 return xen_map_cache(addr, 0, 0);
3171 } else if (block->host == NULL) {
3172 block->host =
3173 xen_map_cache(block->offset, block->length, 1);
3174 }
3175 }
3176 return block->host + (addr - block->offset);
3177 }
3178 }
3179
3180 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
3181 abort();
3182
3183 return NULL;
3184 }
3185
3186 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
3187 * but takes a size argument */
3188 void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
3189 {
3190 if (*size == 0) {
3191 return NULL;
3192 }
3193 if (xen_enabled()) {
3194 return xen_map_cache(addr, *size, 1);
3195 } else {
3196 RAMBlock *block;
3197
3198 QLIST_FOREACH(block, &ram_list.blocks, next) {
3199 if (addr - block->offset < block->length) {
3200 if (addr - block->offset + *size > block->length)
3201 *size = block->length - addr + block->offset;
3202 return block->host + (addr - block->offset);
3203 }
3204 }
3205
3206 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
3207 abort();
3208 }
3209 }
3210
3211 void qemu_put_ram_ptr(void *addr)
3212 {
3213 trace_qemu_put_ram_ptr(addr);
3214 }
3215
3216 int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
3217 {
3218 RAMBlock *block;
3219 uint8_t *host = ptr;
3220
3221 if (xen_enabled()) {
3222 *ram_addr = xen_ram_addr_from_mapcache(ptr);
3223 return 0;
3224 }
3225
3226 QLIST_FOREACH(block, &ram_list.blocks, next) {
3227 /* This case append when the block is not mapped. */
3228 if (block->host == NULL) {
3229 continue;
3230 }
3231 if (host - block->host < block->length) {
3232 *ram_addr = block->offset + (host - block->host);
3233 return 0;
3234 }
3235 }
3236
3237 return -1;
3238 }
3239
3240 /* Some of the softmmu routines need to translate from a host pointer
3241 (typically a TLB entry) back to a ram offset. */
3242 ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
3243 {
3244 ram_addr_t ram_addr;
3245
3246 if (qemu_ram_addr_from_host(ptr, &ram_addr)) {
3247 fprintf(stderr, "Bad ram pointer %p\n", ptr);
3248 abort();
3249 }
3250 return ram_addr;
3251 }
3252
3253 static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
3254 {
3255 #ifdef DEBUG_UNASSIGNED
3256 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
3257 #endif
3258 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3259 cpu_unassigned_access(cpu_single_env, addr, 0, 0, 0, 1);
3260 #endif
3261 return 0;
3262 }
3263
3264 static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
3265 {
3266 #ifdef DEBUG_UNASSIGNED
3267 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
3268 #endif
3269 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3270 cpu_unassigned_access(cpu_single_env, addr, 0, 0, 0, 2);
3271 #endif
3272 return 0;
3273 }
3274
3275 static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
3276 {
3277 #ifdef DEBUG_UNASSIGNED
3278 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
3279 #endif
3280 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3281 cpu_unassigned_access(cpu_single_env, addr, 0, 0, 0, 4);
3282 #endif
3283 return 0;
3284 }
3285
3286 static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
3287 {
3288 #ifdef DEBUG_UNASSIGNED
3289 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3290 #endif
3291 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3292 cpu_unassigned_access(cpu_single_env, addr, 1, 0, 0, 1);
3293 #endif
3294 }
3295
3296 static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
3297 {
3298 #ifdef DEBUG_UNASSIGNED
3299 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3300 #endif
3301 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3302 cpu_unassigned_access(cpu_single_env, addr, 1, 0, 0, 2);
3303 #endif
3304 }
3305
3306 static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
3307 {
3308 #ifdef DEBUG_UNASSIGNED
3309 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3310 #endif
3311 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3312 cpu_unassigned_access(cpu_single_env, addr, 1, 0, 0, 4);
3313 #endif
3314 }
3315
3316 static CPUReadMemoryFunc * const unassigned_mem_read[3] = {
3317 unassigned_mem_readb,
3318 unassigned_mem_readw,
3319 unassigned_mem_readl,
3320 };
3321
3322 static CPUWriteMemoryFunc * const unassigned_mem_write[3] = {
3323 unassigned_mem_writeb,
3324 unassigned_mem_writew,
3325 unassigned_mem_writel,
3326 };
3327
3328 static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
3329 uint32_t val)
3330 {
3331 int dirty_flags;
3332 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3333 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3334 #if !defined(CONFIG_USER_ONLY)
3335 tb_invalidate_phys_page_fast(ram_addr, 1);
3336 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3337 #endif
3338 }
3339 stb_p(qemu_get_ram_ptr(ram_addr), val);
3340 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3341 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
3342 /* we remove the notdirty callback only if the code has been
3343 flushed */
3344 if (dirty_flags == 0xff)
3345 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3346 }
3347
3348 static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
3349 uint32_t val)
3350 {
3351 int dirty_flags;
3352 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3353 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3354 #if !defined(CONFIG_USER_ONLY)
3355 tb_invalidate_phys_page_fast(ram_addr, 2);
3356 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3357 #endif
3358 }
3359 stw_p(qemu_get_ram_ptr(ram_addr), val);
3360 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3361 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
3362 /* we remove the notdirty callback only if the code has been
3363 flushed */
3364 if (dirty_flags == 0xff)
3365 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3366 }
3367
3368 static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
3369 uint32_t val)
3370 {
3371 int dirty_flags;
3372 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3373 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3374 #if !defined(CONFIG_USER_ONLY)
3375 tb_invalidate_phys_page_fast(ram_addr, 4);
3376 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3377 #endif
3378 }
3379 stl_p(qemu_get_ram_ptr(ram_addr), val);
3380 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3381 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
3382 /* we remove the notdirty callback only if the code has been
3383 flushed */
3384 if (dirty_flags == 0xff)
3385 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3386 }
3387
3388 static CPUReadMemoryFunc * const error_mem_read[3] = {
3389 NULL, /* never used */
3390 NULL, /* never used */
3391 NULL, /* never used */
3392 };
3393
3394 static CPUWriteMemoryFunc * const notdirty_mem_write[3] = {
3395 notdirty_mem_writeb,
3396 notdirty_mem_writew,
3397 notdirty_mem_writel,
3398 };
3399
3400 /* Generate a debug exception if a watchpoint has been hit. */
3401 static void check_watchpoint(int offset, int len_mask, int flags)
3402 {
3403 CPUState *env = cpu_single_env;
3404 target_ulong pc, cs_base;
3405 TranslationBlock *tb;
3406 target_ulong vaddr;
3407 CPUWatchpoint *wp;
3408 int cpu_flags;
3409
3410 if (env->watchpoint_hit) {
3411 /* We re-entered the check after replacing the TB. Now raise
3412 * the debug interrupt so that is will trigger after the
3413 * current instruction. */
3414 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
3415 return;
3416 }
3417 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
3418 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
3419 if ((vaddr == (wp->vaddr & len_mask) ||
3420 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
3421 wp->flags |= BP_WATCHPOINT_HIT;
3422 if (!env->watchpoint_hit) {
3423 env->watchpoint_hit = wp;
3424 tb = tb_find_pc(env->mem_io_pc);
3425 if (!tb) {
3426 cpu_abort(env, "check_watchpoint: could not find TB for "
3427 "pc=%p", (void *)env->mem_io_pc);
3428 }
3429 cpu_restore_state(tb, env, env->mem_io_pc);
3430 tb_phys_invalidate(tb, -1);
3431 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
3432 env->exception_index = EXCP_DEBUG;
3433 } else {
3434 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
3435 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
3436 }
3437 cpu_resume_from_signal(env, NULL);
3438 }
3439 } else {
3440 wp->flags &= ~BP_WATCHPOINT_HIT;
3441 }
3442 }
3443 }
3444
3445 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
3446 so these check for a hit then pass through to the normal out-of-line
3447 phys routines. */
3448 static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
3449 {
3450 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
3451 return ldub_phys(addr);
3452 }
3453
3454 static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
3455 {
3456 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
3457 return lduw_phys(addr);
3458 }
3459
3460 static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
3461 {
3462 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
3463 return ldl_phys(addr);
3464 }
3465
3466 static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
3467 uint32_t val)
3468 {
3469 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
3470 stb_phys(addr, val);
3471 }
3472
3473 static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
3474 uint32_t val)
3475 {
3476 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
3477 stw_phys(addr, val);
3478 }
3479
3480 static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
3481 uint32_t val)
3482 {
3483 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
3484 stl_phys(addr, val);
3485 }
3486
3487 static CPUReadMemoryFunc * const watch_mem_read[3] = {
3488 watch_mem_readb,
3489 watch_mem_readw,
3490 watch_mem_readl,
3491 };
3492
3493 static CPUWriteMemoryFunc * const watch_mem_write[3] = {
3494 watch_mem_writeb,
3495 watch_mem_writew,
3496 watch_mem_writel,
3497 };
3498
3499 static inline uint32_t subpage_readlen (subpage_t *mmio,
3500 target_phys_addr_t addr,
3501 unsigned int len)
3502 {
3503 unsigned int idx = SUBPAGE_IDX(addr);
3504 #if defined(DEBUG_SUBPAGE)
3505 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
3506 mmio, len, addr, idx);
3507 #endif
3508
3509 addr += mmio->region_offset[idx];
3510 idx = mmio->sub_io_index[idx];
3511 return io_mem_read[idx][len](io_mem_opaque[idx], addr);
3512 }
3513
3514 static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
3515 uint32_t value, unsigned int len)
3516 {
3517 unsigned int idx = SUBPAGE_IDX(addr);
3518 #if defined(DEBUG_SUBPAGE)
3519 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n",
3520 __func__, mmio, len, addr, idx, value);
3521 #endif
3522
3523 addr += mmio->region_offset[idx];
3524 idx = mmio->sub_io_index[idx];
3525 io_mem_write[idx][len](io_mem_opaque[idx], addr, value);
3526 }
3527
3528 static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
3529 {
3530 return subpage_readlen(opaque, addr, 0);
3531 }
3532
3533 static void subpage_writeb (void *opaque, target_phys_addr_t addr,
3534 uint32_t value)
3535 {
3536 subpage_writelen(opaque, addr, value, 0);
3537 }
3538
3539 static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
3540 {
3541 return subpage_readlen(opaque, addr, 1);
3542 }
3543
3544 static void subpage_writew (void *opaque, target_phys_addr_t addr,
3545 uint32_t value)
3546 {
3547 subpage_writelen(opaque, addr, value, 1);
3548 }
3549
3550 static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
3551 {
3552 return subpage_readlen(opaque, addr, 2);
3553 }
3554
3555 static void subpage_writel (void *opaque, target_phys_addr_t addr,
3556 uint32_t value)
3557 {
3558 subpage_writelen(opaque, addr, value, 2);
3559 }
3560
3561 static CPUReadMemoryFunc * const subpage_read[] = {
3562 &subpage_readb,
3563 &subpage_readw,
3564 &subpage_readl,
3565 };
3566
3567 static CPUWriteMemoryFunc * const subpage_write[] = {
3568 &subpage_writeb,
3569 &subpage_writew,
3570 &subpage_writel,
3571 };
3572
3573 static uint32_t subpage_ram_readb(void *opaque, target_phys_addr_t addr)
3574 {
3575 ram_addr_t raddr = addr;
3576 void *ptr = qemu_get_ram_ptr(raddr);
3577 return ldub_p(ptr);
3578 }
3579
3580 static void subpage_ram_writeb(void *opaque, target_phys_addr_t addr,
3581 uint32_t value)
3582 {
3583 ram_addr_t raddr = addr;
3584 void *ptr = qemu_get_ram_ptr(raddr);
3585 stb_p(ptr, value);
3586 }
3587
3588 static uint32_t subpage_ram_readw(void *opaque, target_phys_addr_t addr)
3589 {
3590 ram_addr_t raddr = addr;
3591 void *ptr = qemu_get_ram_ptr(raddr);
3592 return lduw_p(ptr);
3593 }
3594
3595 static void subpage_ram_writew(void *opaque, target_phys_addr_t addr,
3596 uint32_t value)
3597 {
3598 ram_addr_t raddr = addr;
3599 void *ptr = qemu_get_ram_ptr(raddr);
3600 stw_p(ptr, value);
3601 }
3602
3603 static uint32_t subpage_ram_readl(void *opaque, target_phys_addr_t addr)
3604 {
3605 ram_addr_t raddr = addr;
3606 void *ptr = qemu_get_ram_ptr(raddr);
3607 return ldl_p(ptr);
3608 }
3609
3610 static void subpage_ram_writel(void *opaque, target_phys_addr_t addr,
3611 uint32_t value)
3612 {
3613 ram_addr_t raddr = addr;
3614 void *ptr = qemu_get_ram_ptr(raddr);
3615 stl_p(ptr, value);
3616 }
3617
3618 static CPUReadMemoryFunc * const subpage_ram_read[] = {
3619 &subpage_ram_readb,
3620 &subpage_ram_readw,
3621 &subpage_ram_readl,
3622 };
3623
3624 static CPUWriteMemoryFunc * const subpage_ram_write[] = {
3625 &subpage_ram_writeb,
3626 &subpage_ram_writew,
3627 &subpage_ram_writel,
3628 };
3629
3630 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
3631 ram_addr_t memory, ram_addr_t region_offset)
3632 {
3633 int idx, eidx;
3634
3635 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
3636 return -1;
3637 idx = SUBPAGE_IDX(start);
3638 eidx = SUBPAGE_IDX(end);
3639 #if defined(DEBUG_SUBPAGE)
3640 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
3641 mmio, start, end, idx, eidx, memory);
3642 #endif
3643 if ((memory & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
3644 memory = IO_MEM_SUBPAGE_RAM;
3645 }
3646 memory = (memory >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3647 for (; idx <= eidx; idx++) {
3648 mmio->sub_io_index[idx] = memory;
3649 mmio->region_offset[idx] = region_offset;
3650 }
3651
3652 return 0;
3653 }
3654
3655 static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
3656 ram_addr_t orig_memory,
3657 ram_addr_t region_offset)
3658 {
3659 subpage_t *mmio;
3660 int subpage_memory;
3661
3662 mmio = g_malloc0(sizeof(subpage_t));
3663
3664 mmio->base = base;
3665 subpage_memory = cpu_register_io_memory(subpage_read, subpage_write, mmio,
3666 DEVICE_NATIVE_ENDIAN);
3667 #if defined(DEBUG_SUBPAGE)
3668 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
3669 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
3670 #endif
3671 *phys = subpage_memory | IO_MEM_SUBPAGE;
3672 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, orig_memory, region_offset);
3673
3674 return mmio;
3675 }
3676
3677 static int get_free_io_mem_idx(void)
3678 {
3679 int i;
3680
3681 for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
3682 if (!io_mem_used[i]) {
3683 io_mem_used[i] = 1;
3684 return i;
3685 }
3686 fprintf(stderr, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES);
3687 return -1;
3688 }
3689
3690 /*
3691 * Usually, devices operate in little endian mode. There are devices out
3692 * there that operate in big endian too. Each device gets byte swapped
3693 * mmio if plugged onto a CPU that does the other endianness.
3694 *
3695 * CPU Device swap?
3696 *
3697 * little little no
3698 * little big yes
3699 * big little yes
3700 * big big no
3701 */
3702
3703 typedef struct SwapEndianContainer {
3704 CPUReadMemoryFunc *read[3];
3705 CPUWriteMemoryFunc *write[3];
3706 void *opaque;
3707 } SwapEndianContainer;
3708
3709 static uint32_t swapendian_mem_readb (void *opaque, target_phys_addr_t addr)
3710 {
3711 uint32_t val;
3712 SwapEndianContainer *c = opaque;
3713 val = c->read[0](c->opaque, addr);
3714 return val;
3715 }
3716
3717 static uint32_t swapendian_mem_readw(void *opaque, target_phys_addr_t addr)
3718 {
3719 uint32_t val;
3720 SwapEndianContainer *c = opaque;
3721 val = bswap16(c->read[1](c->opaque, addr));
3722 return val;
3723 }
3724
3725 static uint32_t swapendian_mem_readl(void *opaque, target_phys_addr_t addr)
3726 {
3727 uint32_t val;
3728 SwapEndianContainer *c = opaque;
3729 val = bswap32(c->read[2](c->opaque, addr));
3730 return val;
3731 }
3732
3733 static CPUReadMemoryFunc * const swapendian_readfn[3]={
3734 swapendian_mem_readb,
3735 swapendian_mem_readw,
3736 swapendian_mem_readl
3737 };
3738
3739 static void swapendian_mem_writeb(void *opaque, target_phys_addr_t addr,
3740 uint32_t val)
3741 {
3742 SwapEndianContainer *c = opaque;
3743 c->write[0](c->opaque, addr, val);
3744 }
3745
3746 static void swapendian_mem_writew(void *opaque, target_phys_addr_t addr,
3747 uint32_t val)
3748 {
3749 SwapEndianContainer *c = opaque;
3750 c->write[1](c->opaque, addr, bswap16(val));
3751 }
3752
3753 static void swapendian_mem_writel(void *opaque, target_phys_addr_t addr,
3754 uint32_t val)
3755 {
3756 SwapEndianContainer *c = opaque;
3757 c->write[2](c->opaque, addr, bswap32(val));
3758 }
3759
3760 static CPUWriteMemoryFunc * const swapendian_writefn[3]={
3761 swapendian_mem_writeb,
3762 swapendian_mem_writew,
3763 swapendian_mem_writel
3764 };
3765
3766 static void swapendian_init(int io_index)
3767 {
3768 SwapEndianContainer *c = g_malloc(sizeof(SwapEndianContainer));
3769 int i;
3770
3771 /* Swap mmio for big endian targets */
3772 c->opaque = io_mem_opaque[io_index];
3773 for (i = 0; i < 3; i++) {
3774 c->read[i] = io_mem_read[io_index][i];
3775 c->write[i] = io_mem_write[io_index][i];
3776
3777 io_mem_read[io_index][i] = swapendian_readfn[i];
3778 io_mem_write[io_index][i] = swapendian_writefn[i];
3779 }
3780 io_mem_opaque[io_index] = c;
3781 }
3782
3783 static void swapendian_del(int io_index)
3784 {
3785 if (io_mem_read[io_index][0] == swapendian_readfn[0]) {
3786 g_free(io_mem_opaque[io_index]);
3787 }
3788 }
3789
3790 /* mem_read and mem_write are arrays of functions containing the
3791 function to access byte (index 0), word (index 1) and dword (index
3792 2). Functions can be omitted with a NULL function pointer.
3793 If io_index is non zero, the corresponding io zone is
3794 modified. If it is zero, a new io zone is allocated. The return
3795 value can be used with cpu_register_physical_memory(). (-1) is
3796 returned if error. */
3797 static int cpu_register_io_memory_fixed(int io_index,
3798 CPUReadMemoryFunc * const *mem_read,
3799 CPUWriteMemoryFunc * const *mem_write,
3800 void *opaque, enum device_endian endian)
3801 {
3802 int i;
3803
3804 if (io_index <= 0) {
3805 io_index = get_free_io_mem_idx();
3806 if (io_index == -1)
3807 return io_index;
3808 } else {
3809 io_index >>= IO_MEM_SHIFT;
3810 if (io_index >= IO_MEM_NB_ENTRIES)
3811 return -1;
3812 }
3813
3814 for (i = 0; i < 3; ++i) {
3815 io_mem_read[io_index][i]
3816 = (mem_read[i] ? mem_read[i] : unassigned_mem_read[i]);
3817 }
3818 for (i = 0; i < 3; ++i) {
3819 io_mem_write[io_index][i]
3820 = (mem_write[i] ? mem_write[i] : unassigned_mem_write[i]);
3821 }
3822 io_mem_opaque[io_index] = opaque;
3823
3824 switch (endian) {
3825 case DEVICE_BIG_ENDIAN:
3826 #ifndef TARGET_WORDS_BIGENDIAN
3827 swapendian_init(io_index);
3828 #endif
3829 break;
3830 case DEVICE_LITTLE_ENDIAN:
3831 #ifdef TARGET_WORDS_BIGENDIAN
3832 swapendian_init(io_index);
3833 #endif
3834 break;
3835 case DEVICE_NATIVE_ENDIAN:
3836 default:
3837 break;
3838 }
3839
3840 return (io_index << IO_MEM_SHIFT);
3841 }
3842
3843 int cpu_register_io_memory(CPUReadMemoryFunc * const *mem_read,
3844 CPUWriteMemoryFunc * const *mem_write,
3845 void *opaque, enum device_endian endian)
3846 {
3847 return cpu_register_io_memory_fixed(0, mem_read, mem_write, opaque, endian);
3848 }
3849
3850 void cpu_unregister_io_memory(int io_table_address)
3851 {
3852 int i;
3853 int io_index = io_table_address >> IO_MEM_SHIFT;
3854
3855 swapendian_del(io_index);
3856
3857 for (i=0;i < 3; i++) {
3858 io_mem_read[io_index][i] = unassigned_mem_read[i];
3859 io_mem_write[io_index][i] = unassigned_mem_write[i];
3860 }
3861 io_mem_opaque[io_index] = NULL;
3862 io_mem_used[io_index] = 0;
3863 }
3864
3865 static void io_mem_init(void)
3866 {
3867 int i;
3868
3869 cpu_register_io_memory_fixed(IO_MEM_ROM, error_mem_read,
3870 unassigned_mem_write, NULL,
3871 DEVICE_NATIVE_ENDIAN);
3872 cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED, unassigned_mem_read,
3873 unassigned_mem_write, NULL,
3874 DEVICE_NATIVE_ENDIAN);
3875 cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY, error_mem_read,
3876 notdirty_mem_write, NULL,
3877 DEVICE_NATIVE_ENDIAN);
3878 cpu_register_io_memory_fixed(IO_MEM_SUBPAGE_RAM, subpage_ram_read,
3879 subpage_ram_write, NULL,
3880 DEVICE_NATIVE_ENDIAN);
3881 for (i=0; i<5; i++)
3882 io_mem_used[i] = 1;
3883
3884 io_mem_watch = cpu_register_io_memory(watch_mem_read,
3885 watch_mem_write, NULL,
3886 DEVICE_NATIVE_ENDIAN);
3887 }
3888
3889 static void memory_map_init(void)
3890 {
3891 system_memory = g_malloc(sizeof(*system_memory));
3892 memory_region_init(system_memory, "system", INT64_MAX);
3893 set_system_memory_map(system_memory);
3894
3895 system_io = g_malloc(sizeof(*system_io));
3896 memory_region_init(system_io, "io", 65536);
3897 set_system_io_map(system_io);
3898 }
3899
3900 MemoryRegion *get_system_memory(void)
3901 {
3902 return system_memory;
3903 }
3904
3905 MemoryRegion *get_system_io(void)
3906 {
3907 return system_io;
3908 }
3909
3910 #endif /* !defined(CONFIG_USER_ONLY) */
3911
3912 /* physical memory access (slow version, mainly for debug) */
3913 #if defined(CONFIG_USER_ONLY)
3914 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3915 uint8_t *buf, int len, int is_write)
3916 {
3917 int l, flags;
3918 target_ulong page;
3919 void * p;
3920
3921 while (len > 0) {
3922 page = addr & TARGET_PAGE_MASK;
3923 l = (page + TARGET_PAGE_SIZE) - addr;
3924 if (l > len)
3925 l = len;
3926 flags = page_get_flags(page);
3927 if (!(flags & PAGE_VALID))
3928 return -1;
3929 if (is_write) {
3930 if (!(flags & PAGE_WRITE))
3931 return -1;
3932 /* XXX: this code should not depend on lock_user */
3933 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
3934 return -1;
3935 memcpy(p, buf, l);
3936 unlock_user(p, addr, l);
3937 } else {
3938 if (!(flags & PAGE_READ))
3939 return -1;
3940 /* XXX: this code should not depend on lock_user */
3941 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
3942 return -1;
3943 memcpy(buf, p, l);
3944 unlock_user(p, addr, 0);
3945 }
3946 len -= l;
3947 buf += l;
3948 addr += l;
3949 }
3950 return 0;
3951 }
3952
3953 #else
3954 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
3955 int len, int is_write)
3956 {
3957 int l, io_index;
3958 uint8_t *ptr;
3959 uint32_t val;
3960 target_phys_addr_t page;
3961 ram_addr_t pd;
3962 PhysPageDesc *p;
3963
3964 while (len > 0) {
3965 page = addr & TARGET_PAGE_MASK;
3966 l = (page + TARGET_PAGE_SIZE) - addr;
3967 if (l > len)
3968 l = len;
3969 p = phys_page_find(page >> TARGET_PAGE_BITS);
3970 if (!p) {
3971 pd = IO_MEM_UNASSIGNED;
3972 } else {
3973 pd = p->phys_offset;
3974 }
3975
3976 if (is_write) {
3977 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3978 target_phys_addr_t addr1 = addr;
3979 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3980 if (p)
3981 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3982 /* XXX: could force cpu_single_env to NULL to avoid
3983 potential bugs */
3984 if (l >= 4 && ((addr1 & 3) == 0)) {
3985 /* 32 bit write access */
3986 val = ldl_p(buf);
3987 io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
3988 l = 4;
3989 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3990 /* 16 bit write access */
3991 val = lduw_p(buf);
3992 io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
3993 l = 2;
3994 } else {
3995 /* 8 bit write access */
3996 val = ldub_p(buf);
3997 io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
3998 l = 1;
3999 }
4000 } else {
4001 ram_addr_t addr1;
4002 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4003 /* RAM case */
4004 ptr = qemu_get_ram_ptr(addr1);
4005 memcpy(ptr, buf, l);
4006 if (!cpu_physical_memory_is_dirty(addr1)) {
4007 /* invalidate code */
4008 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
4009 /* set dirty bit */
4010 cpu_physical_memory_set_dirty_flags(
4011 addr1, (0xff & ~CODE_DIRTY_FLAG));
4012 }
4013 qemu_put_ram_ptr(ptr);
4014 }
4015 } else {
4016 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
4017 !(pd & IO_MEM_ROMD)) {
4018 target_phys_addr_t addr1 = addr;
4019 /* I/O case */
4020 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4021 if (p)
4022 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4023 if (l >= 4 && ((addr1 & 3) == 0)) {
4024 /* 32 bit read access */
4025 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
4026 stl_p(buf, val);
4027 l = 4;
4028 } else if (l >= 2 && ((addr1 & 1) == 0)) {
4029 /* 16 bit read access */
4030 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
4031 stw_p(buf, val);
4032 l = 2;
4033 } else {
4034 /* 8 bit read access */
4035 val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
4036 stb_p(buf, val);
4037 l = 1;
4038 }
4039 } else {
4040 /* RAM case */
4041 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
4042 memcpy(buf, ptr + (addr & ~TARGET_PAGE_MASK), l);
4043 qemu_put_ram_ptr(ptr);
4044 }
4045 }
4046 len -= l;
4047 buf += l;
4048 addr += l;
4049 }
4050 }
4051
4052 /* used for ROM loading : can write in RAM and ROM */
4053 void cpu_physical_memory_write_rom(target_phys_addr_t addr,
4054 const uint8_t *buf, int len)
4055 {
4056 int l;
4057 uint8_t *ptr;
4058 target_phys_addr_t page;
4059 unsigned long pd;
4060 PhysPageDesc *p;
4061
4062 while (len > 0) {
4063 page = addr & TARGET_PAGE_MASK;
4064 l = (page + TARGET_PAGE_SIZE) - addr;
4065 if (l > len)
4066 l = len;
4067 p = phys_page_find(page >> TARGET_PAGE_BITS);
4068 if (!p) {
4069 pd = IO_MEM_UNASSIGNED;
4070 } else {
4071 pd = p->phys_offset;
4072 }
4073
4074 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
4075 (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
4076 !(pd & IO_MEM_ROMD)) {
4077 /* do nothing */
4078 } else {
4079 unsigned long addr1;
4080 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4081 /* ROM/RAM case */
4082 ptr = qemu_get_ram_ptr(addr1);
4083 memcpy(ptr, buf, l);
4084 qemu_put_ram_ptr(ptr);
4085 }
4086 len -= l;
4087 buf += l;
4088 addr += l;
4089 }
4090 }
4091
4092 typedef struct {
4093 void *buffer;
4094 target_phys_addr_t addr;
4095 target_phys_addr_t len;
4096 } BounceBuffer;
4097
4098 static BounceBuffer bounce;
4099
4100 typedef struct MapClient {
4101 void *opaque;
4102 void (*callback)(void *opaque);
4103 QLIST_ENTRY(MapClient) link;
4104 } MapClient;
4105
4106 static QLIST_HEAD(map_client_list, MapClient) map_client_list
4107 = QLIST_HEAD_INITIALIZER(map_client_list);
4108
4109 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
4110 {
4111 MapClient *client = g_malloc(sizeof(*client));
4112
4113 client->opaque = opaque;
4114 client->callback = callback;
4115 QLIST_INSERT_HEAD(&map_client_list, client, link);
4116 return client;
4117 }
4118
4119 void cpu_unregister_map_client(void *_client)
4120 {
4121 MapClient *client = (MapClient *)_client;
4122
4123 QLIST_REMOVE(client, link);
4124 g_free(client);
4125 }
4126
4127 static void cpu_notify_map_clients(void)
4128 {
4129 MapClient *client;
4130
4131 while (!QLIST_EMPTY(&map_client_list)) {
4132 client = QLIST_FIRST(&map_client_list);
4133 client->callback(client->opaque);
4134 cpu_unregister_map_client(client);
4135 }
4136 }
4137
4138 /* Map a physical memory region into a host virtual address.
4139 * May map a subset of the requested range, given by and returned in *plen.
4140 * May return NULL if resources needed to perform the mapping are exhausted.
4141 * Use only for reads OR writes - not for read-modify-write operations.
4142 * Use cpu_register_map_client() to know when retrying the map operation is
4143 * likely to succeed.
4144 */
4145 void *cpu_physical_memory_map(target_phys_addr_t addr,
4146 target_phys_addr_t *plen,
4147 int is_write)
4148 {
4149 target_phys_addr_t len = *plen;
4150 target_phys_addr_t todo = 0;
4151 int l;
4152 target_phys_addr_t page;
4153 unsigned long pd;
4154 PhysPageDesc *p;
4155 ram_addr_t raddr = RAM_ADDR_MAX;
4156 ram_addr_t rlen;
4157 void *ret;
4158
4159 while (len > 0) {
4160 page = addr & TARGET_PAGE_MASK;
4161 l = (page + TARGET_PAGE_SIZE) - addr;
4162 if (l > len)
4163 l = len;
4164 p = phys_page_find(page >> TARGET_PAGE_BITS);
4165 if (!p) {
4166 pd = IO_MEM_UNASSIGNED;
4167 } else {
4168 pd = p->phys_offset;
4169 }
4170
4171 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4172 if (todo || bounce.buffer) {
4173 break;
4174 }
4175 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
4176 bounce.addr = addr;
4177 bounce.len = l;
4178 if (!is_write) {
4179 cpu_physical_memory_read(addr, bounce.buffer, l);
4180 }
4181
4182 *plen = l;
4183 return bounce.buffer;
4184 }
4185 if (!todo) {
4186 raddr = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4187 }
4188
4189 len -= l;
4190 addr += l;
4191 todo += l;
4192 }
4193 rlen = todo;
4194 ret = qemu_ram_ptr_length(raddr, &rlen);
4195 *plen = rlen;
4196 return ret;
4197 }
4198
4199 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
4200 * Will also mark the memory as dirty if is_write == 1. access_len gives
4201 * the amount of memory that was actually read or written by the caller.
4202 */
4203 void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
4204 int is_write, target_phys_addr_t access_len)
4205 {
4206 if (buffer != bounce.buffer) {
4207 if (is_write) {
4208 ram_addr_t addr1 = qemu_ram_addr_from_host_nofail(buffer);
4209 while (access_len) {
4210 unsigned l;
4211 l = TARGET_PAGE_SIZE;
4212 if (l > access_len)
4213 l = access_len;
4214 if (!cpu_physical_memory_is_dirty(addr1)) {
4215 /* invalidate code */
4216 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
4217 /* set dirty bit */
4218 cpu_physical_memory_set_dirty_flags(
4219 addr1, (0xff & ~CODE_DIRTY_FLAG));
4220 }
4221 addr1 += l;
4222 access_len -= l;
4223 }
4224 }
4225 if (xen_enabled()) {
4226 xen_invalidate_map_cache_entry(buffer);
4227 }
4228 return;
4229 }
4230 if (is_write) {
4231 cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
4232 }
4233 qemu_vfree(bounce.buffer);
4234 bounce.buffer = NULL;
4235 cpu_notify_map_clients();
4236 }
4237
4238 /* warning: addr must be aligned */
4239 static inline uint32_t ldl_phys_internal(target_phys_addr_t addr,
4240 enum device_endian endian)
4241 {
4242 int io_index;
4243 uint8_t *ptr;
4244 uint32_t val;
4245 unsigned long pd;
4246 PhysPageDesc *p;
4247
4248 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4249 if (!p) {
4250 pd = IO_MEM_UNASSIGNED;
4251 } else {
4252 pd = p->phys_offset;
4253 }
4254
4255 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
4256 !(pd & IO_MEM_ROMD)) {
4257 /* I/O case */
4258 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4259 if (p)
4260 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4261 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
4262 #if defined(TARGET_WORDS_BIGENDIAN)
4263 if (endian == DEVICE_LITTLE_ENDIAN) {
4264 val = bswap32(val);
4265 }
4266 #else
4267 if (endian == DEVICE_BIG_ENDIAN) {
4268 val = bswap32(val);
4269 }
4270 #endif
4271 } else {
4272 /* RAM case */
4273 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
4274 (addr & ~TARGET_PAGE_MASK);
4275 switch (endian) {
4276 case DEVICE_LITTLE_ENDIAN:
4277 val = ldl_le_p(ptr);
4278 break;
4279 case DEVICE_BIG_ENDIAN:
4280 val = ldl_be_p(ptr);
4281 break;
4282 default:
4283 val = ldl_p(ptr);
4284 break;
4285 }
4286 }
4287 return val;
4288 }
4289
4290 uint32_t ldl_phys(target_phys_addr_t addr)
4291 {
4292 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
4293 }
4294
4295 uint32_t ldl_le_phys(target_phys_addr_t addr)
4296 {
4297 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
4298 }
4299
4300 uint32_t ldl_be_phys(target_phys_addr_t addr)
4301 {
4302 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
4303 }
4304
4305 /* warning: addr must be aligned */
4306 static inline uint64_t ldq_phys_internal(target_phys_addr_t addr,
4307 enum device_endian endian)
4308 {
4309 int io_index;
4310 uint8_t *ptr;
4311 uint64_t val;
4312 unsigned long pd;
4313 PhysPageDesc *p;
4314
4315 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4316 if (!p) {
4317 pd = IO_MEM_UNASSIGNED;
4318 } else {
4319 pd = p->phys_offset;
4320 }
4321
4322 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
4323 !(pd & IO_MEM_ROMD)) {
4324 /* I/O case */
4325 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4326 if (p)
4327 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4328
4329 /* XXX This is broken when device endian != cpu endian.
4330 Fix and add "endian" variable check */
4331 #ifdef TARGET_WORDS_BIGENDIAN
4332 val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
4333 val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
4334 #else
4335 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
4336 val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
4337 #endif
4338 } else {
4339 /* RAM case */
4340 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
4341 (addr & ~TARGET_PAGE_MASK);
4342 switch (endian) {
4343 case DEVICE_LITTLE_ENDIAN:
4344 val = ldq_le_p(ptr);
4345 break;
4346 case DEVICE_BIG_ENDIAN:
4347 val = ldq_be_p(ptr);
4348 break;
4349 default:
4350 val = ldq_p(ptr);
4351 break;
4352 }
4353 }
4354 return val;
4355 }
4356
4357 uint64_t ldq_phys(target_phys_addr_t addr)
4358 {
4359 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
4360 }
4361
4362 uint64_t ldq_le_phys(target_phys_addr_t addr)
4363 {
4364 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
4365 }
4366
4367 uint64_t ldq_be_phys(target_phys_addr_t addr)
4368 {
4369 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
4370 }
4371
4372 /* XXX: optimize */
4373 uint32_t ldub_phys(target_phys_addr_t addr)
4374 {
4375 uint8_t val;
4376 cpu_physical_memory_read(addr, &val, 1);
4377 return val;
4378 }
4379
4380 /* warning: addr must be aligned */
4381 static inline uint32_t lduw_phys_internal(target_phys_addr_t addr,
4382 enum device_endian endian)
4383 {
4384 int io_index;
4385 uint8_t *ptr;
4386 uint64_t val;
4387 unsigned long pd;
4388 PhysPageDesc *p;
4389
4390 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4391 if (!p) {
4392 pd = IO_MEM_UNASSIGNED;
4393 } else {
4394 pd = p->phys_offset;
4395 }
4396
4397 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
4398 !(pd & IO_MEM_ROMD)) {
4399 /* I/O case */
4400 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4401 if (p)
4402 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4403 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr);
4404 #if defined(TARGET_WORDS_BIGENDIAN)
4405 if (endian == DEVICE_LITTLE_ENDIAN) {
4406 val = bswap16(val);
4407 }
4408 #else
4409 if (endian == DEVICE_BIG_ENDIAN) {
4410 val = bswap16(val);
4411 }
4412 #endif
4413 } else {
4414 /* RAM case */
4415 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
4416 (addr & ~TARGET_PAGE_MASK);
4417 switch (endian) {
4418 case DEVICE_LITTLE_ENDIAN:
4419 val = lduw_le_p(ptr);
4420 break;
4421 case DEVICE_BIG_ENDIAN:
4422 val = lduw_be_p(ptr);
4423 break;
4424 default:
4425 val = lduw_p(ptr);
4426 break;
4427 }
4428 }
4429 return val;
4430 }
4431
4432 uint32_t lduw_phys(target_phys_addr_t addr)
4433 {
4434 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
4435 }
4436
4437 uint32_t lduw_le_phys(target_phys_addr_t addr)
4438 {
4439 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
4440 }
4441
4442 uint32_t lduw_be_phys(target_phys_addr_t addr)
4443 {
4444 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
4445 }
4446
4447 /* warning: addr must be aligned. The ram page is not masked as dirty
4448 and the code inside is not invalidated. It is useful if the dirty
4449 bits are used to track modified PTEs */
4450 void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
4451 {
4452 int io_index;
4453 uint8_t *ptr;
4454 unsigned long pd;
4455 PhysPageDesc *p;
4456
4457 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4458 if (!p) {
4459 pd = IO_MEM_UNASSIGNED;
4460 } else {
4461 pd = p->phys_offset;
4462 }
4463
4464 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4465 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4466 if (p)
4467 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4468 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4469 } else {
4470 unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4471 ptr = qemu_get_ram_ptr(addr1);
4472 stl_p(ptr, val);
4473
4474 if (unlikely(in_migration)) {
4475 if (!cpu_physical_memory_is_dirty(addr1)) {
4476 /* invalidate code */
4477 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
4478 /* set dirty bit */
4479 cpu_physical_memory_set_dirty_flags(
4480 addr1, (0xff & ~CODE_DIRTY_FLAG));
4481 }
4482 }
4483 }
4484 }
4485
4486 void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
4487 {
4488 int io_index;
4489 uint8_t *ptr;
4490 unsigned long pd;
4491 PhysPageDesc *p;
4492
4493 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4494 if (!p) {
4495 pd = IO_MEM_UNASSIGNED;
4496 } else {
4497 pd = p->phys_offset;
4498 }
4499
4500 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4501 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4502 if (p)
4503 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4504 #ifdef TARGET_WORDS_BIGENDIAN
4505 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
4506 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
4507 #else
4508 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4509 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
4510 #endif
4511 } else {
4512 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
4513 (addr & ~TARGET_PAGE_MASK);
4514 stq_p(ptr, val);
4515 }
4516 }
4517
4518 /* warning: addr must be aligned */
4519 static inline void stl_phys_internal(target_phys_addr_t addr, uint32_t val,
4520 enum device_endian endian)
4521 {
4522 int io_index;
4523 uint8_t *ptr;
4524 unsigned long pd;
4525 PhysPageDesc *p;
4526
4527 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4528 if (!p) {
4529 pd = IO_MEM_UNASSIGNED;
4530 } else {
4531 pd = p->phys_offset;
4532 }
4533
4534 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4535 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4536 if (p)
4537 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4538 #if defined(TARGET_WORDS_BIGENDIAN)
4539 if (endian == DEVICE_LITTLE_ENDIAN) {
4540 val = bswap32(val);
4541 }
4542 #else
4543 if (endian == DEVICE_BIG_ENDIAN) {
4544 val = bswap32(val);
4545 }
4546 #endif
4547 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4548 } else {
4549 unsigned long addr1;
4550 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4551 /* RAM case */
4552 ptr = qemu_get_ram_ptr(addr1);
4553 switch (endian) {
4554 case DEVICE_LITTLE_ENDIAN:
4555 stl_le_p(ptr, val);
4556 break;
4557 case DEVICE_BIG_ENDIAN:
4558 stl_be_p(ptr, val);
4559 break;
4560 default:
4561 stl_p(ptr, val);
4562 break;
4563 }
4564 if (!cpu_physical_memory_is_dirty(addr1)) {
4565 /* invalidate code */
4566 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
4567 /* set dirty bit */
4568 cpu_physical_memory_set_dirty_flags(addr1,
4569 (0xff & ~CODE_DIRTY_FLAG));
4570 }
4571 }
4572 }
4573
4574 void stl_phys(target_phys_addr_t addr, uint32_t val)
4575 {
4576 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
4577 }
4578
4579 void stl_le_phys(target_phys_addr_t addr, uint32_t val)
4580 {
4581 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
4582 }
4583
4584 void stl_be_phys(target_phys_addr_t addr, uint32_t val)
4585 {
4586 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
4587 }
4588
4589 /* XXX: optimize */
4590 void stb_phys(target_phys_addr_t addr, uint32_t val)
4591 {
4592 uint8_t v = val;
4593 cpu_physical_memory_write(addr, &v, 1);
4594 }
4595
4596 /* warning: addr must be aligned */
4597 static inline void stw_phys_internal(target_phys_addr_t addr, uint32_t val,
4598 enum device_endian endian)
4599 {
4600 int io_index;
4601 uint8_t *ptr;
4602 unsigned long pd;
4603 PhysPageDesc *p;
4604
4605 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4606 if (!p) {
4607 pd = IO_MEM_UNASSIGNED;
4608 } else {
4609 pd = p->phys_offset;
4610 }
4611
4612 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4613 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4614 if (p)
4615 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4616 #if defined(TARGET_WORDS_BIGENDIAN)
4617 if (endian == DEVICE_LITTLE_ENDIAN) {
4618 val = bswap16(val);
4619 }
4620 #else
4621 if (endian == DEVICE_BIG_ENDIAN) {
4622 val = bswap16(val);
4623 }
4624 #endif
4625 io_mem_write[io_index][1](io_mem_opaque[io_index], addr, val);
4626 } else {
4627 unsigned long addr1;
4628 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4629 /* RAM case */
4630 ptr = qemu_get_ram_ptr(addr1);
4631 switch (endian) {
4632 case DEVICE_LITTLE_ENDIAN:
4633 stw_le_p(ptr, val);
4634 break;
4635 case DEVICE_BIG_ENDIAN:
4636 stw_be_p(ptr, val);
4637 break;
4638 default:
4639 stw_p(ptr, val);
4640 break;
4641 }
4642 if (!cpu_physical_memory_is_dirty(addr1)) {
4643 /* invalidate code */
4644 tb_invalidate_phys_page_range(addr1, addr1 + 2, 0);
4645 /* set dirty bit */
4646 cpu_physical_memory_set_dirty_flags(addr1,
4647 (0xff & ~CODE_DIRTY_FLAG));
4648 }
4649 }
4650 }
4651
4652 void stw_phys(target_phys_addr_t addr, uint32_t val)
4653 {
4654 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
4655 }
4656
4657 void stw_le_phys(target_phys_addr_t addr, uint32_t val)
4658 {
4659 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
4660 }
4661
4662 void stw_be_phys(target_phys_addr_t addr, uint32_t val)
4663 {
4664 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
4665 }
4666
4667 /* XXX: optimize */
4668 void stq_phys(target_phys_addr_t addr, uint64_t val)
4669 {
4670 val = tswap64(val);
4671 cpu_physical_memory_write(addr, &val, 8);
4672 }
4673
4674 void stq_le_phys(target_phys_addr_t addr, uint64_t val)
4675 {
4676 val = cpu_to_le64(val);
4677 cpu_physical_memory_write(addr, &val, 8);
4678 }
4679
4680 void stq_be_phys(target_phys_addr_t addr, uint64_t val)
4681 {
4682 val = cpu_to_be64(val);
4683 cpu_physical_memory_write(addr, &val, 8);
4684 }
4685
4686 /* virtual memory access for debug (includes writing to ROM) */
4687 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
4688 uint8_t *buf, int len, int is_write)
4689 {
4690 int l;
4691 target_phys_addr_t phys_addr;
4692 target_ulong page;
4693
4694 while (len > 0) {
4695 page = addr & TARGET_PAGE_MASK;
4696 phys_addr = cpu_get_phys_page_debug(env, page);
4697 /* if no physical page mapped, return an error */
4698 if (phys_addr == -1)
4699 return -1;
4700 l = (page + TARGET_PAGE_SIZE) - addr;
4701 if (l > len)
4702 l = len;
4703 phys_addr += (addr & ~TARGET_PAGE_MASK);
4704 if (is_write)
4705 cpu_physical_memory_write_rom(phys_addr, buf, l);
4706 else
4707 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
4708 len -= l;
4709 buf += l;
4710 addr += l;
4711 }
4712 return 0;
4713 }
4714 #endif
4715
4716 /* in deterministic execution mode, instructions doing device I/Os
4717 must be at the end of the TB */
4718 void cpu_io_recompile(CPUState *env, void *retaddr)
4719 {
4720 TranslationBlock *tb;
4721 uint32_t n, cflags;
4722 target_ulong pc, cs_base;
4723 uint64_t flags;
4724
4725 tb = tb_find_pc((unsigned long)retaddr);
4726 if (!tb) {
4727 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
4728 retaddr);
4729 }
4730 n = env->icount_decr.u16.low + tb->icount;
4731 cpu_restore_state(tb, env, (unsigned long)retaddr);
4732 /* Calculate how many instructions had been executed before the fault
4733 occurred. */
4734 n = n - env->icount_decr.u16.low;
4735 /* Generate a new TB ending on the I/O insn. */
4736 n++;
4737 /* On MIPS and SH, delay slot instructions can only be restarted if
4738 they were already the first instruction in the TB. If this is not
4739 the first instruction in a TB then re-execute the preceding
4740 branch. */
4741 #if defined(TARGET_MIPS)
4742 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
4743 env->active_tc.PC -= 4;
4744 env->icount_decr.u16.low++;
4745 env->hflags &= ~MIPS_HFLAG_BMASK;
4746 }
4747 #elif defined(TARGET_SH4)
4748 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
4749 && n > 1) {
4750 env->pc -= 2;
4751 env->icount_decr.u16.low++;
4752 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
4753 }
4754 #endif
4755 /* This should never happen. */
4756 if (n > CF_COUNT_MASK)
4757 cpu_abort(env, "TB too big during recompile");
4758
4759 cflags = n | CF_LAST_IO;
4760 pc = tb->pc;
4761 cs_base = tb->cs_base;
4762 flags = tb->flags;
4763 tb_phys_invalidate(tb, -1);
4764 /* FIXME: In theory this could raise an exception. In practice
4765 we have already translated the block once so it's probably ok. */
4766 tb_gen_code(env, pc, cs_base, flags, cflags);
4767 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
4768 the first in the TB) then we end up generating a whole new TB and
4769 repeating the fault, which is horribly inefficient.
4770 Better would be to execute just this insn uncached, or generate a
4771 second new TB. */
4772 cpu_resume_from_signal(env, NULL);
4773 }
4774
4775 #if !defined(CONFIG_USER_ONLY)
4776
4777 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
4778 {
4779 int i, target_code_size, max_target_code_size;
4780 int direct_jmp_count, direct_jmp2_count, cross_page;
4781 TranslationBlock *tb;
4782
4783 target_code_size = 0;
4784 max_target_code_size = 0;
4785 cross_page = 0;
4786 direct_jmp_count = 0;
4787 direct_jmp2_count = 0;
4788 for(i = 0; i < nb_tbs; i++) {
4789 tb = &tbs[i];
4790 target_code_size += tb->size;
4791 if (tb->size > max_target_code_size)
4792 max_target_code_size = tb->size;
4793 if (tb->page_addr[1] != -1)
4794 cross_page++;
4795 if (tb->tb_next_offset[0] != 0xffff) {
4796 direct_jmp_count++;
4797 if (tb->tb_next_offset[1] != 0xffff) {
4798 direct_jmp2_count++;
4799 }
4800 }
4801 }
4802 /* XXX: avoid using doubles ? */
4803 cpu_fprintf(f, "Translation buffer state:\n");
4804 cpu_fprintf(f, "gen code size %td/%ld\n",
4805 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
4806 cpu_fprintf(f, "TB count %d/%d\n",
4807 nb_tbs, code_gen_max_blocks);
4808 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
4809 nb_tbs ? target_code_size / nb_tbs : 0,
4810 max_target_code_size);
4811 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
4812 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
4813 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
4814 cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
4815 cross_page,
4816 nb_tbs ? (cross_page * 100) / nb_tbs : 0);
4817 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
4818 direct_jmp_count,
4819 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
4820 direct_jmp2_count,
4821 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
4822 cpu_fprintf(f, "\nStatistics:\n");
4823 cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
4824 cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
4825 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
4826 tcg_dump_info(f, cpu_fprintf);
4827 }
4828
4829 #define MMUSUFFIX _cmmu
4830 #undef GETPC
4831 #define GETPC() NULL
4832 #define env cpu_single_env
4833 #define SOFTMMU_CODE_ACCESS
4834
4835 #define SHIFT 0
4836 #include "softmmu_template.h"
4837
4838 #define SHIFT 1
4839 #include "softmmu_template.h"
4840
4841 #define SHIFT 2
4842 #include "softmmu_template.h"
4843
4844 #define SHIFT 3
4845 #include "softmmu_template.h"
4846
4847 #undef env
4848
4849 #endif
QEmu