ossaudio: add endianness support for VoiceIn
[qemu/stefanha.git] / exec.c
blob49c28b160e0f76698348d65e16206d8b0a9f8066
1 /*
2 * virtual page mapping and translated block handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "exec-all.h"
30 #include "tcg.h"
31 #include "hw/hw.h"
32 #include "hw/qdev.h"
33 #include "osdep.h"
34 #include "kvm.h"
35 #include "qemu-timer.h"
36 #if defined(CONFIG_USER_ONLY)
37 #include <qemu.h>
38 #include <signal.h>
39 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
40 #include <sys/param.h>
41 #if __FreeBSD_version >= 700104
42 #define HAVE_KINFO_GETVMMAP
43 #define sigqueue sigqueue_freebsd /* avoid redefinition */
44 #include <sys/time.h>
45 #include <sys/proc.h>
46 #include <machine/profile.h>
47 #define _KERNEL
48 #include <sys/user.h>
49 #undef _KERNEL
50 #undef sigqueue
51 #include <libutil.h>
52 #endif
53 #endif
54 #endif
56 //#define DEBUG_TB_INVALIDATE
57 //#define DEBUG_FLUSH
58 //#define DEBUG_TLB
59 //#define DEBUG_UNASSIGNED
61 /* make various TB consistency checks */
62 //#define DEBUG_TB_CHECK
63 //#define DEBUG_TLB_CHECK
65 //#define DEBUG_IOPORT
66 //#define DEBUG_SUBPAGE
68 #if !defined(CONFIG_USER_ONLY)
69 /* TB consistency checks only implemented for usermode emulation. */
70 #undef DEBUG_TB_CHECK
71 #endif
73 #define SMC_BITMAP_USE_THRESHOLD 10
75 static TranslationBlock *tbs;
76 static int code_gen_max_blocks;
77 TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
78 static int nb_tbs;
79 /* any access to the tbs or the page table must use this lock */
80 spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
82 #if defined(__arm__) || defined(__sparc_v9__)
83 /* The prologue must be reachable with a direct jump. ARM and Sparc64
84 have limited branch ranges (possibly also PPC) so place it in a
85 section close to code segment. */
86 #define code_gen_section \
87 __attribute__((__section__(".gen_code"))) \
88 __attribute__((aligned (32)))
89 #elif defined(_WIN32)
90 /* Maximum alignment for Win32 is 16. */
91 #define code_gen_section \
92 __attribute__((aligned (16)))
93 #else
94 #define code_gen_section \
95 __attribute__((aligned (32)))
96 #endif
98 uint8_t code_gen_prologue[1024] code_gen_section;
99 static uint8_t *code_gen_buffer;
100 static unsigned long code_gen_buffer_size;
101 /* threshold to flush the translated code buffer */
102 static unsigned long code_gen_buffer_max_size;
103 static uint8_t *code_gen_ptr;
105 #if !defined(CONFIG_USER_ONLY)
106 int phys_ram_fd;
107 static int in_migration;
109 RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list) };
110 #endif
112 CPUState *first_cpu;
113 /* current CPU in the current thread. It is only valid inside
114 cpu_exec() */
115 CPUState *cpu_single_env;
116 /* 0 = Do not count executed instructions.
117 1 = Precise instruction counting.
118 2 = Adaptive rate instruction counting. */
119 int use_icount = 0;
120 /* Current instruction counter. While executing translated code this may
121 include some instructions that have not yet been executed. */
122 int64_t qemu_icount;
124 typedef struct PageDesc {
125 /* list of TBs intersecting this ram page */
126 TranslationBlock *first_tb;
127 /* in order to optimize self modifying code, we count the number
128 of lookups we do to a given page to use a bitmap */
129 unsigned int code_write_count;
130 uint8_t *code_bitmap;
131 #if defined(CONFIG_USER_ONLY)
132 unsigned long flags;
133 #endif
134 } PageDesc;
136 /* In system mode we want L1_MAP to be based on ram offsets,
137 while in user mode we want it to be based on virtual addresses. */
138 #if !defined(CONFIG_USER_ONLY)
139 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
140 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
141 #else
142 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
143 #endif
144 #else
145 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
146 #endif
148 /* Size of the L2 (and L3, etc) page tables. */
149 #define L2_BITS 10
150 #define L2_SIZE (1 << L2_BITS)
152 /* The bits remaining after N lower levels of page tables. */
153 #define P_L1_BITS_REM \
154 ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
155 #define V_L1_BITS_REM \
156 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
158 /* Size of the L1 page table. Avoid silly small sizes. */
159 #if P_L1_BITS_REM < 4
160 #define P_L1_BITS (P_L1_BITS_REM + L2_BITS)
161 #else
162 #define P_L1_BITS P_L1_BITS_REM
163 #endif
165 #if V_L1_BITS_REM < 4
166 #define V_L1_BITS (V_L1_BITS_REM + L2_BITS)
167 #else
168 #define V_L1_BITS V_L1_BITS_REM
169 #endif
171 #define P_L1_SIZE ((target_phys_addr_t)1 << P_L1_BITS)
172 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
174 #define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
175 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
177 unsigned long qemu_real_host_page_size;
178 unsigned long qemu_host_page_bits;
179 unsigned long qemu_host_page_size;
180 unsigned long qemu_host_page_mask;
182 /* This is a multi-level map on the virtual address space.
183 The bottom level has pointers to PageDesc. */
184 static void *l1_map[V_L1_SIZE];
186 #if !defined(CONFIG_USER_ONLY)
187 typedef struct PhysPageDesc {
188 /* offset in host memory of the page + io_index in the low bits */
189 ram_addr_t phys_offset;
190 ram_addr_t region_offset;
191 } PhysPageDesc;
193 /* This is a multi-level map on the physical address space.
194 The bottom level has pointers to PhysPageDesc. */
195 static void *l1_phys_map[P_L1_SIZE];
197 static void io_mem_init(void);
199 /* io memory support */
200 CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
201 CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
202 void *io_mem_opaque[IO_MEM_NB_ENTRIES];
203 static char io_mem_used[IO_MEM_NB_ENTRIES];
204 static int io_mem_watch;
205 #endif
207 /* log support */
208 #ifdef WIN32
209 static const char *logfilename = "qemu.log";
210 #else
211 static const char *logfilename = "/tmp/qemu.log";
212 #endif
213 FILE *logfile;
214 int loglevel;
215 static int log_append = 0;
217 /* statistics */
218 #if !defined(CONFIG_USER_ONLY)
219 static int tlb_flush_count;
220 #endif
221 static int tb_flush_count;
222 static int tb_phys_invalidate_count;
224 #ifdef _WIN32
225 static void map_exec(void *addr, long size)
227 DWORD old_protect;
228 VirtualProtect(addr, size,
229 PAGE_EXECUTE_READWRITE, &old_protect);
232 #else
233 static void map_exec(void *addr, long size)
235 unsigned long start, end, page_size;
237 page_size = getpagesize();
238 start = (unsigned long)addr;
239 start &= ~(page_size - 1);
241 end = (unsigned long)addr + size;
242 end += page_size - 1;
243 end &= ~(page_size - 1);
245 mprotect((void *)start, end - start,
246 PROT_READ | PROT_WRITE | PROT_EXEC);
248 #endif
250 static void page_init(void)
252 /* NOTE: we can always suppose that qemu_host_page_size >=
253 TARGET_PAGE_SIZE */
254 #ifdef _WIN32
256 SYSTEM_INFO system_info;
258 GetSystemInfo(&system_info);
259 qemu_real_host_page_size = system_info.dwPageSize;
261 #else
262 qemu_real_host_page_size = getpagesize();
263 #endif
264 if (qemu_host_page_size == 0)
265 qemu_host_page_size = qemu_real_host_page_size;
266 if (qemu_host_page_size < TARGET_PAGE_SIZE)
267 qemu_host_page_size = TARGET_PAGE_SIZE;
268 qemu_host_page_bits = 0;
269 while ((1 << qemu_host_page_bits) < qemu_host_page_size)
270 qemu_host_page_bits++;
271 qemu_host_page_mask = ~(qemu_host_page_size - 1);
273 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
275 #ifdef HAVE_KINFO_GETVMMAP
276 struct kinfo_vmentry *freep;
277 int i, cnt;
279 freep = kinfo_getvmmap(getpid(), &cnt);
280 if (freep) {
281 mmap_lock();
282 for (i = 0; i < cnt; i++) {
283 unsigned long startaddr, endaddr;
285 startaddr = freep[i].kve_start;
286 endaddr = freep[i].kve_end;
287 if (h2g_valid(startaddr)) {
288 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
290 if (h2g_valid(endaddr)) {
291 endaddr = h2g(endaddr);
292 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
293 } else {
294 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
295 endaddr = ~0ul;
296 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
297 #endif
301 free(freep);
302 mmap_unlock();
304 #else
305 FILE *f;
307 last_brk = (unsigned long)sbrk(0);
309 f = fopen("/compat/linux/proc/self/maps", "r");
310 if (f) {
311 mmap_lock();
313 do {
314 unsigned long startaddr, endaddr;
315 int n;
317 n = fscanf (f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
319 if (n == 2 && h2g_valid(startaddr)) {
320 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
322 if (h2g_valid(endaddr)) {
323 endaddr = h2g(endaddr);
324 } else {
325 endaddr = ~0ul;
327 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
329 } while (!feof(f));
331 fclose(f);
332 mmap_unlock();
334 #endif
336 #endif
339 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
341 PageDesc *pd;
342 void **lp;
343 int i;
345 #if defined(CONFIG_USER_ONLY)
346 /* We can't use qemu_malloc because it may recurse into a locked mutex. */
347 # define ALLOC(P, SIZE) \
348 do { \
349 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
350 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
351 } while (0)
352 #else
353 # define ALLOC(P, SIZE) \
354 do { P = qemu_mallocz(SIZE); } while (0)
355 #endif
357 /* Level 1. Always allocated. */
358 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
360 /* Level 2..N-1. */
361 for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
362 void **p = *lp;
364 if (p == NULL) {
365 if (!alloc) {
366 return NULL;
368 ALLOC(p, sizeof(void *) * L2_SIZE);
369 *lp = p;
372 lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
375 pd = *lp;
376 if (pd == NULL) {
377 if (!alloc) {
378 return NULL;
380 ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
381 *lp = pd;
384 #undef ALLOC
386 return pd + (index & (L2_SIZE - 1));
389 static inline PageDesc *page_find(tb_page_addr_t index)
391 return page_find_alloc(index, 0);
394 #if !defined(CONFIG_USER_ONLY)
395 static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
397 PhysPageDesc *pd;
398 void **lp;
399 int i;
401 /* Level 1. Always allocated. */
402 lp = l1_phys_map + ((index >> P_L1_SHIFT) & (P_L1_SIZE - 1));
404 /* Level 2..N-1. */
405 for (i = P_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
406 void **p = *lp;
407 if (p == NULL) {
408 if (!alloc) {
409 return NULL;
411 *lp = p = qemu_mallocz(sizeof(void *) * L2_SIZE);
413 lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
416 pd = *lp;
417 if (pd == NULL) {
418 int i;
420 if (!alloc) {
421 return NULL;
424 *lp = pd = qemu_malloc(sizeof(PhysPageDesc) * L2_SIZE);
426 for (i = 0; i < L2_SIZE; i++) {
427 pd[i].phys_offset = IO_MEM_UNASSIGNED;
428 pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
432 return pd + (index & (L2_SIZE - 1));
435 static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
437 return phys_page_find_alloc(index, 0);
440 static void tlb_protect_code(ram_addr_t ram_addr);
441 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
442 target_ulong vaddr);
443 #define mmap_lock() do { } while(0)
444 #define mmap_unlock() do { } while(0)
445 #endif
447 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
449 #if defined(CONFIG_USER_ONLY)
450 /* Currently it is not recommended to allocate big chunks of data in
451 user mode. It will change when a dedicated libc will be used */
452 #define USE_STATIC_CODE_GEN_BUFFER
453 #endif
455 #ifdef USE_STATIC_CODE_GEN_BUFFER
456 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
457 __attribute__((aligned (CODE_GEN_ALIGN)));
458 #endif
460 static void code_gen_alloc(unsigned long tb_size)
462 #ifdef USE_STATIC_CODE_GEN_BUFFER
463 code_gen_buffer = static_code_gen_buffer;
464 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
465 map_exec(code_gen_buffer, code_gen_buffer_size);
466 #else
467 code_gen_buffer_size = tb_size;
468 if (code_gen_buffer_size == 0) {
469 #if defined(CONFIG_USER_ONLY)
470 /* in user mode, phys_ram_size is not meaningful */
471 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
472 #else
473 /* XXX: needs adjustments */
474 code_gen_buffer_size = (unsigned long)(ram_size / 4);
475 #endif
477 if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
478 code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
479 /* The code gen buffer location may have constraints depending on
480 the host cpu and OS */
481 #if defined(__linux__)
483 int flags;
484 void *start = NULL;
486 flags = MAP_PRIVATE | MAP_ANONYMOUS;
487 #if defined(__x86_64__)
488 flags |= MAP_32BIT;
489 /* Cannot map more than that */
490 if (code_gen_buffer_size > (800 * 1024 * 1024))
491 code_gen_buffer_size = (800 * 1024 * 1024);
492 #elif defined(__sparc_v9__)
493 // Map the buffer below 2G, so we can use direct calls and branches
494 flags |= MAP_FIXED;
495 start = (void *) 0x60000000UL;
496 if (code_gen_buffer_size > (512 * 1024 * 1024))
497 code_gen_buffer_size = (512 * 1024 * 1024);
498 #elif defined(__arm__)
499 /* Map the buffer below 32M, so we can use direct calls and branches */
500 flags |= MAP_FIXED;
501 start = (void *) 0x01000000UL;
502 if (code_gen_buffer_size > 16 * 1024 * 1024)
503 code_gen_buffer_size = 16 * 1024 * 1024;
504 #elif defined(__s390x__)
505 /* Map the buffer so that we can use direct calls and branches. */
506 /* We have a +- 4GB range on the branches; leave some slop. */
507 if (code_gen_buffer_size > (3ul * 1024 * 1024 * 1024)) {
508 code_gen_buffer_size = 3ul * 1024 * 1024 * 1024;
510 start = (void *)0x90000000UL;
511 #endif
512 code_gen_buffer = mmap(start, code_gen_buffer_size,
513 PROT_WRITE | PROT_READ | PROT_EXEC,
514 flags, -1, 0);
515 if (code_gen_buffer == MAP_FAILED) {
516 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
517 exit(1);
520 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
521 || defined(__DragonFly__) || defined(__OpenBSD__)
523 int flags;
524 void *addr = NULL;
525 flags = MAP_PRIVATE | MAP_ANONYMOUS;
526 #if defined(__x86_64__)
527 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
528 * 0x40000000 is free */
529 flags |= MAP_FIXED;
530 addr = (void *)0x40000000;
531 /* Cannot map more than that */
532 if (code_gen_buffer_size > (800 * 1024 * 1024))
533 code_gen_buffer_size = (800 * 1024 * 1024);
534 #endif
535 code_gen_buffer = mmap(addr, code_gen_buffer_size,
536 PROT_WRITE | PROT_READ | PROT_EXEC,
537 flags, -1, 0);
538 if (code_gen_buffer == MAP_FAILED) {
539 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
540 exit(1);
543 #else
544 code_gen_buffer = qemu_malloc(code_gen_buffer_size);
545 map_exec(code_gen_buffer, code_gen_buffer_size);
546 #endif
547 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
548 map_exec(code_gen_prologue, sizeof(code_gen_prologue));
549 code_gen_buffer_max_size = code_gen_buffer_size -
550 (TCG_MAX_OP_SIZE * OPC_MAX_SIZE);
551 code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
552 tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
555 /* Must be called before using the QEMU cpus. 'tb_size' is the size
556 (in bytes) allocated to the translation buffer. Zero means default
557 size. */
558 void cpu_exec_init_all(unsigned long tb_size)
560 cpu_gen_init();
561 code_gen_alloc(tb_size);
562 code_gen_ptr = code_gen_buffer;
563 page_init();
564 #if !defined(CONFIG_USER_ONLY)
565 io_mem_init();
566 #endif
567 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
568 /* There's no guest base to take into account, so go ahead and
569 initialize the prologue now. */
570 tcg_prologue_init(&tcg_ctx);
571 #endif
574 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
576 static int cpu_common_post_load(void *opaque, int version_id)
578 CPUState *env = opaque;
580 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
581 version_id is increased. */
582 env->interrupt_request &= ~0x01;
583 tlb_flush(env, 1);
585 return 0;
588 static const VMStateDescription vmstate_cpu_common = {
589 .name = "cpu_common",
590 .version_id = 1,
591 .minimum_version_id = 1,
592 .minimum_version_id_old = 1,
593 .post_load = cpu_common_post_load,
594 .fields = (VMStateField []) {
595 VMSTATE_UINT32(halted, CPUState),
596 VMSTATE_UINT32(interrupt_request, CPUState),
597 VMSTATE_END_OF_LIST()
600 #endif
602 CPUState *qemu_get_cpu(int cpu)
604 CPUState *env = first_cpu;
606 while (env) {
607 if (env->cpu_index == cpu)
608 break;
609 env = env->next_cpu;
612 return env;
615 void cpu_exec_init(CPUState *env)
617 CPUState **penv;
618 int cpu_index;
620 #if defined(CONFIG_USER_ONLY)
621 cpu_list_lock();
622 #endif
623 env->next_cpu = NULL;
624 penv = &first_cpu;
625 cpu_index = 0;
626 while (*penv != NULL) {
627 penv = &(*penv)->next_cpu;
628 cpu_index++;
630 env->cpu_index = cpu_index;
631 env->numa_node = 0;
632 QTAILQ_INIT(&env->breakpoints);
633 QTAILQ_INIT(&env->watchpoints);
634 *penv = env;
635 #if defined(CONFIG_USER_ONLY)
636 cpu_list_unlock();
637 #endif
638 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
639 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, env);
640 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
641 cpu_save, cpu_load, env);
642 #endif
645 static inline void invalidate_page_bitmap(PageDesc *p)
647 if (p->code_bitmap) {
648 qemu_free(p->code_bitmap);
649 p->code_bitmap = NULL;
651 p->code_write_count = 0;
654 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
656 static void page_flush_tb_1 (int level, void **lp)
658 int i;
660 if (*lp == NULL) {
661 return;
663 if (level == 0) {
664 PageDesc *pd = *lp;
665 for (i = 0; i < L2_SIZE; ++i) {
666 pd[i].first_tb = NULL;
667 invalidate_page_bitmap(pd + i);
669 } else {
670 void **pp = *lp;
671 for (i = 0; i < L2_SIZE; ++i) {
672 page_flush_tb_1 (level - 1, pp + i);
677 static void page_flush_tb(void)
679 int i;
680 for (i = 0; i < V_L1_SIZE; i++) {
681 page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
685 /* flush all the translation blocks */
686 /* XXX: tb_flush is currently not thread safe */
687 void tb_flush(CPUState *env1)
689 CPUState *env;
690 #if defined(DEBUG_FLUSH)
691 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
692 (unsigned long)(code_gen_ptr - code_gen_buffer),
693 nb_tbs, nb_tbs > 0 ?
694 ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
695 #endif
696 if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
697 cpu_abort(env1, "Internal error: code buffer overflow\n");
699 nb_tbs = 0;
701 for(env = first_cpu; env != NULL; env = env->next_cpu) {
702 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
705 memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
706 page_flush_tb();
708 code_gen_ptr = code_gen_buffer;
709 /* XXX: flush processor icache at this point if cache flush is
710 expensive */
711 tb_flush_count++;
714 #ifdef DEBUG_TB_CHECK
716 static void tb_invalidate_check(target_ulong address)
718 TranslationBlock *tb;
719 int i;
720 address &= TARGET_PAGE_MASK;
721 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
722 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
723 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
724 address >= tb->pc + tb->size)) {
725 printf("ERROR invalidate: address=" TARGET_FMT_lx
726 " PC=%08lx size=%04x\n",
727 address, (long)tb->pc, tb->size);
733 /* verify that all the pages have correct rights for code */
734 static void tb_page_check(void)
736 TranslationBlock *tb;
737 int i, flags1, flags2;
739 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
740 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
741 flags1 = page_get_flags(tb->pc);
742 flags2 = page_get_flags(tb->pc + tb->size - 1);
743 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
744 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
745 (long)tb->pc, tb->size, flags1, flags2);
751 #endif
753 /* invalidate one TB */
754 static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
755 int next_offset)
757 TranslationBlock *tb1;
758 for(;;) {
759 tb1 = *ptb;
760 if (tb1 == tb) {
761 *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
762 break;
764 ptb = (TranslationBlock **)((char *)tb1 + next_offset);
768 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
770 TranslationBlock *tb1;
771 unsigned int n1;
773 for(;;) {
774 tb1 = *ptb;
775 n1 = (long)tb1 & 3;
776 tb1 = (TranslationBlock *)((long)tb1 & ~3);
777 if (tb1 == tb) {
778 *ptb = tb1->page_next[n1];
779 break;
781 ptb = &tb1->page_next[n1];
785 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
787 TranslationBlock *tb1, **ptb;
788 unsigned int n1;
790 ptb = &tb->jmp_next[n];
791 tb1 = *ptb;
792 if (tb1) {
793 /* find tb(n) in circular list */
794 for(;;) {
795 tb1 = *ptb;
796 n1 = (long)tb1 & 3;
797 tb1 = (TranslationBlock *)((long)tb1 & ~3);
798 if (n1 == n && tb1 == tb)
799 break;
800 if (n1 == 2) {
801 ptb = &tb1->jmp_first;
802 } else {
803 ptb = &tb1->jmp_next[n1];
806 /* now we can suppress tb(n) from the list */
807 *ptb = tb->jmp_next[n];
809 tb->jmp_next[n] = NULL;
813 /* reset the jump entry 'n' of a TB so that it is not chained to
814 another TB */
815 static inline void tb_reset_jump(TranslationBlock *tb, int n)
817 tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
820 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
822 CPUState *env;
823 PageDesc *p;
824 unsigned int h, n1;
825 tb_page_addr_t phys_pc;
826 TranslationBlock *tb1, *tb2;
828 /* remove the TB from the hash list */
829 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
830 h = tb_phys_hash_func(phys_pc);
831 tb_remove(&tb_phys_hash[h], tb,
832 offsetof(TranslationBlock, phys_hash_next));
834 /* remove the TB from the page list */
835 if (tb->page_addr[0] != page_addr) {
836 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
837 tb_page_remove(&p->first_tb, tb);
838 invalidate_page_bitmap(p);
840 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
841 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
842 tb_page_remove(&p->first_tb, tb);
843 invalidate_page_bitmap(p);
846 tb_invalidated_flag = 1;
848 /* remove the TB from the hash list */
849 h = tb_jmp_cache_hash_func(tb->pc);
850 for(env = first_cpu; env != NULL; env = env->next_cpu) {
851 if (env->tb_jmp_cache[h] == tb)
852 env->tb_jmp_cache[h] = NULL;
855 /* suppress this TB from the two jump lists */
856 tb_jmp_remove(tb, 0);
857 tb_jmp_remove(tb, 1);
859 /* suppress any remaining jumps to this TB */
860 tb1 = tb->jmp_first;
861 for(;;) {
862 n1 = (long)tb1 & 3;
863 if (n1 == 2)
864 break;
865 tb1 = (TranslationBlock *)((long)tb1 & ~3);
866 tb2 = tb1->jmp_next[n1];
867 tb_reset_jump(tb1, n1);
868 tb1->jmp_next[n1] = NULL;
869 tb1 = tb2;
871 tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
873 tb_phys_invalidate_count++;
876 static inline void set_bits(uint8_t *tab, int start, int len)
878 int end, mask, end1;
880 end = start + len;
881 tab += start >> 3;
882 mask = 0xff << (start & 7);
883 if ((start & ~7) == (end & ~7)) {
884 if (start < end) {
885 mask &= ~(0xff << (end & 7));
886 *tab |= mask;
888 } else {
889 *tab++ |= mask;
890 start = (start + 8) & ~7;
891 end1 = end & ~7;
892 while (start < end1) {
893 *tab++ = 0xff;
894 start += 8;
896 if (start < end) {
897 mask = ~(0xff << (end & 7));
898 *tab |= mask;
903 static void build_page_bitmap(PageDesc *p)
905 int n, tb_start, tb_end;
906 TranslationBlock *tb;
908 p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
910 tb = p->first_tb;
911 while (tb != NULL) {
912 n = (long)tb & 3;
913 tb = (TranslationBlock *)((long)tb & ~3);
914 /* NOTE: this is subtle as a TB may span two physical pages */
915 if (n == 0) {
916 /* NOTE: tb_end may be after the end of the page, but
917 it is not a problem */
918 tb_start = tb->pc & ~TARGET_PAGE_MASK;
919 tb_end = tb_start + tb->size;
920 if (tb_end > TARGET_PAGE_SIZE)
921 tb_end = TARGET_PAGE_SIZE;
922 } else {
923 tb_start = 0;
924 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
926 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
927 tb = tb->page_next[n];
931 TranslationBlock *tb_gen_code(CPUState *env,
932 target_ulong pc, target_ulong cs_base,
933 int flags, int cflags)
935 TranslationBlock *tb;
936 uint8_t *tc_ptr;
937 tb_page_addr_t phys_pc, phys_page2;
938 target_ulong virt_page2;
939 int code_gen_size;
941 phys_pc = get_page_addr_code(env, pc);
942 tb = tb_alloc(pc);
943 if (!tb) {
944 /* flush must be done */
945 tb_flush(env);
946 /* cannot fail at this point */
947 tb = tb_alloc(pc);
948 /* Don't forget to invalidate previous TB info. */
949 tb_invalidated_flag = 1;
951 tc_ptr = code_gen_ptr;
952 tb->tc_ptr = tc_ptr;
953 tb->cs_base = cs_base;
954 tb->flags = flags;
955 tb->cflags = cflags;
956 cpu_gen_code(env, tb, &code_gen_size);
957 code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
959 /* check next page if needed */
960 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
961 phys_page2 = -1;
962 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
963 phys_page2 = get_page_addr_code(env, virt_page2);
965 tb_link_page(tb, phys_pc, phys_page2);
966 return tb;
969 /* invalidate all TBs which intersect with the target physical page
970 starting in range [start;end[. NOTE: start and end must refer to
971 the same physical page. 'is_cpu_write_access' should be true if called
972 from a real cpu write access: the virtual CPU will exit the current
973 TB if code is modified inside this TB. */
974 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
975 int is_cpu_write_access)
977 TranslationBlock *tb, *tb_next, *saved_tb;
978 CPUState *env = cpu_single_env;
979 tb_page_addr_t tb_start, tb_end;
980 PageDesc *p;
981 int n;
982 #ifdef TARGET_HAS_PRECISE_SMC
983 int current_tb_not_found = is_cpu_write_access;
984 TranslationBlock *current_tb = NULL;
985 int current_tb_modified = 0;
986 target_ulong current_pc = 0;
987 target_ulong current_cs_base = 0;
988 int current_flags = 0;
989 #endif /* TARGET_HAS_PRECISE_SMC */
991 p = page_find(start >> TARGET_PAGE_BITS);
992 if (!p)
993 return;
994 if (!p->code_bitmap &&
995 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
996 is_cpu_write_access) {
997 /* build code bitmap */
998 build_page_bitmap(p);
1001 /* we remove all the TBs in the range [start, end[ */
1002 /* XXX: see if in some cases it could be faster to invalidate all the code */
1003 tb = p->first_tb;
1004 while (tb != NULL) {
1005 n = (long)tb & 3;
1006 tb = (TranslationBlock *)((long)tb & ~3);
1007 tb_next = tb->page_next[n];
1008 /* NOTE: this is subtle as a TB may span two physical pages */
1009 if (n == 0) {
1010 /* NOTE: tb_end may be after the end of the page, but
1011 it is not a problem */
1012 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1013 tb_end = tb_start + tb->size;
1014 } else {
1015 tb_start = tb->page_addr[1];
1016 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1018 if (!(tb_end <= start || tb_start >= end)) {
1019 #ifdef TARGET_HAS_PRECISE_SMC
1020 if (current_tb_not_found) {
1021 current_tb_not_found = 0;
1022 current_tb = NULL;
1023 if (env->mem_io_pc) {
1024 /* now we have a real cpu fault */
1025 current_tb = tb_find_pc(env->mem_io_pc);
1028 if (current_tb == tb &&
1029 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1030 /* If we are modifying the current TB, we must stop
1031 its execution. We could be more precise by checking
1032 that the modification is after the current PC, but it
1033 would require a specialized function to partially
1034 restore the CPU state */
1036 current_tb_modified = 1;
1037 cpu_restore_state(current_tb, env,
1038 env->mem_io_pc, NULL);
1039 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1040 &current_flags);
1042 #endif /* TARGET_HAS_PRECISE_SMC */
1043 /* we need to do that to handle the case where a signal
1044 occurs while doing tb_phys_invalidate() */
1045 saved_tb = NULL;
1046 if (env) {
1047 saved_tb = env->current_tb;
1048 env->current_tb = NULL;
1050 tb_phys_invalidate(tb, -1);
1051 if (env) {
1052 env->current_tb = saved_tb;
1053 if (env->interrupt_request && env->current_tb)
1054 cpu_interrupt(env, env->interrupt_request);
1057 tb = tb_next;
1059 #if !defined(CONFIG_USER_ONLY)
1060 /* if no code remaining, no need to continue to use slow writes */
1061 if (!p->first_tb) {
1062 invalidate_page_bitmap(p);
1063 if (is_cpu_write_access) {
1064 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1067 #endif
1068 #ifdef TARGET_HAS_PRECISE_SMC
1069 if (current_tb_modified) {
1070 /* we generate a block containing just the instruction
1071 modifying the memory. It will ensure that it cannot modify
1072 itself */
1073 env->current_tb = NULL;
1074 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1075 cpu_resume_from_signal(env, NULL);
1077 #endif
1080 /* len must be <= 8 and start must be a multiple of len */
1081 static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1083 PageDesc *p;
1084 int offset, b;
1085 #if 0
1086 if (1) {
1087 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1088 cpu_single_env->mem_io_vaddr, len,
1089 cpu_single_env->eip,
1090 cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1092 #endif
1093 p = page_find(start >> TARGET_PAGE_BITS);
1094 if (!p)
1095 return;
1096 if (p->code_bitmap) {
1097 offset = start & ~TARGET_PAGE_MASK;
1098 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1099 if (b & ((1 << len) - 1))
1100 goto do_invalidate;
1101 } else {
1102 do_invalidate:
1103 tb_invalidate_phys_page_range(start, start + len, 1);
1107 #if !defined(CONFIG_SOFTMMU)
1108 static void tb_invalidate_phys_page(tb_page_addr_t addr,
1109 unsigned long pc, void *puc)
1111 TranslationBlock *tb;
1112 PageDesc *p;
1113 int n;
1114 #ifdef TARGET_HAS_PRECISE_SMC
1115 TranslationBlock *current_tb = NULL;
1116 CPUState *env = cpu_single_env;
1117 int current_tb_modified = 0;
1118 target_ulong current_pc = 0;
1119 target_ulong current_cs_base = 0;
1120 int current_flags = 0;
1121 #endif
1123 addr &= TARGET_PAGE_MASK;
1124 p = page_find(addr >> TARGET_PAGE_BITS);
1125 if (!p)
1126 return;
1127 tb = p->first_tb;
1128 #ifdef TARGET_HAS_PRECISE_SMC
1129 if (tb && pc != 0) {
1130 current_tb = tb_find_pc(pc);
1132 #endif
1133 while (tb != NULL) {
1134 n = (long)tb & 3;
1135 tb = (TranslationBlock *)((long)tb & ~3);
1136 #ifdef TARGET_HAS_PRECISE_SMC
1137 if (current_tb == tb &&
1138 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1139 /* If we are modifying the current TB, we must stop
1140 its execution. We could be more precise by checking
1141 that the modification is after the current PC, but it
1142 would require a specialized function to partially
1143 restore the CPU state */
1145 current_tb_modified = 1;
1146 cpu_restore_state(current_tb, env, pc, puc);
1147 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1148 &current_flags);
1150 #endif /* TARGET_HAS_PRECISE_SMC */
1151 tb_phys_invalidate(tb, addr);
1152 tb = tb->page_next[n];
1154 p->first_tb = NULL;
1155 #ifdef TARGET_HAS_PRECISE_SMC
1156 if (current_tb_modified) {
1157 /* we generate a block containing just the instruction
1158 modifying the memory. It will ensure that it cannot modify
1159 itself */
1160 env->current_tb = NULL;
1161 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1162 cpu_resume_from_signal(env, puc);
1164 #endif
1166 #endif
1168 /* add the tb in the target page and protect it if necessary */
1169 static inline void tb_alloc_page(TranslationBlock *tb,
1170 unsigned int n, tb_page_addr_t page_addr)
1172 PageDesc *p;
1173 TranslationBlock *last_first_tb;
1175 tb->page_addr[n] = page_addr;
1176 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1177 tb->page_next[n] = p->first_tb;
1178 last_first_tb = p->first_tb;
1179 p->first_tb = (TranslationBlock *)((long)tb | n);
1180 invalidate_page_bitmap(p);
1182 #if defined(TARGET_HAS_SMC) || 1
1184 #if defined(CONFIG_USER_ONLY)
1185 if (p->flags & PAGE_WRITE) {
1186 target_ulong addr;
1187 PageDesc *p2;
1188 int prot;
1190 /* force the host page as non writable (writes will have a
1191 page fault + mprotect overhead) */
1192 page_addr &= qemu_host_page_mask;
1193 prot = 0;
1194 for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1195 addr += TARGET_PAGE_SIZE) {
1197 p2 = page_find (addr >> TARGET_PAGE_BITS);
1198 if (!p2)
1199 continue;
1200 prot |= p2->flags;
1201 p2->flags &= ~PAGE_WRITE;
1203 mprotect(g2h(page_addr), qemu_host_page_size,
1204 (prot & PAGE_BITS) & ~PAGE_WRITE);
1205 #ifdef DEBUG_TB_INVALIDATE
1206 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1207 page_addr);
1208 #endif
1210 #else
1211 /* if some code is already present, then the pages are already
1212 protected. So we handle the case where only the first TB is
1213 allocated in a physical page */
1214 if (!last_first_tb) {
1215 tlb_protect_code(page_addr);
1217 #endif
1219 #endif /* TARGET_HAS_SMC */
1222 /* Allocate a new translation block. Flush the translation buffer if
1223 too many translation blocks or too much generated code. */
1224 TranslationBlock *tb_alloc(target_ulong pc)
1226 TranslationBlock *tb;
1228 if (nb_tbs >= code_gen_max_blocks ||
1229 (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
1230 return NULL;
1231 tb = &tbs[nb_tbs++];
1232 tb->pc = pc;
1233 tb->cflags = 0;
1234 return tb;
1237 void tb_free(TranslationBlock *tb)
1239 /* In practice this is mostly used for single use temporary TB
1240 Ignore the hard cases and just back up if this TB happens to
1241 be the last one generated. */
1242 if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
1243 code_gen_ptr = tb->tc_ptr;
1244 nb_tbs--;
1248 /* add a new TB and link it to the physical page tables. phys_page2 is
1249 (-1) to indicate that only one page contains the TB. */
1250 void tb_link_page(TranslationBlock *tb,
1251 tb_page_addr_t phys_pc, tb_page_addr_t phys_page2)
1253 unsigned int h;
1254 TranslationBlock **ptb;
1256 /* Grab the mmap lock to stop another thread invalidating this TB
1257 before we are done. */
1258 mmap_lock();
1259 /* add in the physical hash table */
1260 h = tb_phys_hash_func(phys_pc);
1261 ptb = &tb_phys_hash[h];
1262 tb->phys_hash_next = *ptb;
1263 *ptb = tb;
1265 /* add in the page list */
1266 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1267 if (phys_page2 != -1)
1268 tb_alloc_page(tb, 1, phys_page2);
1269 else
1270 tb->page_addr[1] = -1;
1272 tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1273 tb->jmp_next[0] = NULL;
1274 tb->jmp_next[1] = NULL;
1276 /* init original jump addresses */
1277 if (tb->tb_next_offset[0] != 0xffff)
1278 tb_reset_jump(tb, 0);
1279 if (tb->tb_next_offset[1] != 0xffff)
1280 tb_reset_jump(tb, 1);
1282 #ifdef DEBUG_TB_CHECK
1283 tb_page_check();
1284 #endif
1285 mmap_unlock();
1288 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1289 tb[1].tc_ptr. Return NULL if not found */
1290 TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1292 int m_min, m_max, m;
1293 unsigned long v;
1294 TranslationBlock *tb;
1296 if (nb_tbs <= 0)
1297 return NULL;
1298 if (tc_ptr < (unsigned long)code_gen_buffer ||
1299 tc_ptr >= (unsigned long)code_gen_ptr)
1300 return NULL;
1301 /* binary search (cf Knuth) */
1302 m_min = 0;
1303 m_max = nb_tbs - 1;
1304 while (m_min <= m_max) {
1305 m = (m_min + m_max) >> 1;
1306 tb = &tbs[m];
1307 v = (unsigned long)tb->tc_ptr;
1308 if (v == tc_ptr)
1309 return tb;
1310 else if (tc_ptr < v) {
1311 m_max = m - 1;
1312 } else {
1313 m_min = m + 1;
1316 return &tbs[m_max];
1319 static void tb_reset_jump_recursive(TranslationBlock *tb);
1321 static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1323 TranslationBlock *tb1, *tb_next, **ptb;
1324 unsigned int n1;
1326 tb1 = tb->jmp_next[n];
1327 if (tb1 != NULL) {
1328 /* find head of list */
1329 for(;;) {
1330 n1 = (long)tb1 & 3;
1331 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1332 if (n1 == 2)
1333 break;
1334 tb1 = tb1->jmp_next[n1];
1336 /* we are now sure now that tb jumps to tb1 */
1337 tb_next = tb1;
1339 /* remove tb from the jmp_first list */
1340 ptb = &tb_next->jmp_first;
1341 for(;;) {
1342 tb1 = *ptb;
1343 n1 = (long)tb1 & 3;
1344 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1345 if (n1 == n && tb1 == tb)
1346 break;
1347 ptb = &tb1->jmp_next[n1];
1349 *ptb = tb->jmp_next[n];
1350 tb->jmp_next[n] = NULL;
1352 /* suppress the jump to next tb in generated code */
1353 tb_reset_jump(tb, n);
1355 /* suppress jumps in the tb on which we could have jumped */
1356 tb_reset_jump_recursive(tb_next);
1360 static void tb_reset_jump_recursive(TranslationBlock *tb)
1362 tb_reset_jump_recursive2(tb, 0);
1363 tb_reset_jump_recursive2(tb, 1);
1366 #if defined(TARGET_HAS_ICE)
1367 #if defined(CONFIG_USER_ONLY)
1368 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1370 tb_invalidate_phys_page_range(pc, pc + 1, 0);
1372 #else
1373 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1375 target_phys_addr_t addr;
1376 target_ulong pd;
1377 ram_addr_t ram_addr;
1378 PhysPageDesc *p;
1380 addr = cpu_get_phys_page_debug(env, pc);
1381 p = phys_page_find(addr >> TARGET_PAGE_BITS);
1382 if (!p) {
1383 pd = IO_MEM_UNASSIGNED;
1384 } else {
1385 pd = p->phys_offset;
1387 ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1388 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1390 #endif
1391 #endif /* TARGET_HAS_ICE */
1393 #if defined(CONFIG_USER_ONLY)
1394 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1399 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1400 int flags, CPUWatchpoint **watchpoint)
1402 return -ENOSYS;
1404 #else
1405 /* Add a watchpoint. */
1406 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1407 int flags, CPUWatchpoint **watchpoint)
1409 target_ulong len_mask = ~(len - 1);
1410 CPUWatchpoint *wp;
1412 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1413 if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1414 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1415 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1416 return -EINVAL;
1418 wp = qemu_malloc(sizeof(*wp));
1420 wp->vaddr = addr;
1421 wp->len_mask = len_mask;
1422 wp->flags = flags;
1424 /* keep all GDB-injected watchpoints in front */
1425 if (flags & BP_GDB)
1426 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1427 else
1428 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1430 tlb_flush_page(env, addr);
1432 if (watchpoint)
1433 *watchpoint = wp;
1434 return 0;
1437 /* Remove a specific watchpoint. */
1438 int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1439 int flags)
1441 target_ulong len_mask = ~(len - 1);
1442 CPUWatchpoint *wp;
1444 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1445 if (addr == wp->vaddr && len_mask == wp->len_mask
1446 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1447 cpu_watchpoint_remove_by_ref(env, wp);
1448 return 0;
1451 return -ENOENT;
1454 /* Remove a specific watchpoint by reference. */
1455 void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1457 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1459 tlb_flush_page(env, watchpoint->vaddr);
1461 qemu_free(watchpoint);
1464 /* Remove all matching watchpoints. */
1465 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1467 CPUWatchpoint *wp, *next;
1469 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1470 if (wp->flags & mask)
1471 cpu_watchpoint_remove_by_ref(env, wp);
1474 #endif
1476 /* Add a breakpoint. */
1477 int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1478 CPUBreakpoint **breakpoint)
1480 #if defined(TARGET_HAS_ICE)
1481 CPUBreakpoint *bp;
1483 bp = qemu_malloc(sizeof(*bp));
1485 bp->pc = pc;
1486 bp->flags = flags;
1488 /* keep all GDB-injected breakpoints in front */
1489 if (flags & BP_GDB)
1490 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1491 else
1492 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1494 breakpoint_invalidate(env, pc);
1496 if (breakpoint)
1497 *breakpoint = bp;
1498 return 0;
1499 #else
1500 return -ENOSYS;
1501 #endif
1504 /* Remove a specific breakpoint. */
1505 int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1507 #if defined(TARGET_HAS_ICE)
1508 CPUBreakpoint *bp;
1510 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1511 if (bp->pc == pc && bp->flags == flags) {
1512 cpu_breakpoint_remove_by_ref(env, bp);
1513 return 0;
1516 return -ENOENT;
1517 #else
1518 return -ENOSYS;
1519 #endif
1522 /* Remove a specific breakpoint by reference. */
1523 void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
1525 #if defined(TARGET_HAS_ICE)
1526 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1528 breakpoint_invalidate(env, breakpoint->pc);
1530 qemu_free(breakpoint);
1531 #endif
1534 /* Remove all matching breakpoints. */
1535 void cpu_breakpoint_remove_all(CPUState *env, int mask)
1537 #if defined(TARGET_HAS_ICE)
1538 CPUBreakpoint *bp, *next;
1540 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1541 if (bp->flags & mask)
1542 cpu_breakpoint_remove_by_ref(env, bp);
1544 #endif
1547 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1548 CPU loop after each instruction */
1549 void cpu_single_step(CPUState *env, int enabled)
1551 #if defined(TARGET_HAS_ICE)
1552 if (env->singlestep_enabled != enabled) {
1553 env->singlestep_enabled = enabled;
1554 if (kvm_enabled())
1555 kvm_update_guest_debug(env, 0);
1556 else {
1557 /* must flush all the translated code to avoid inconsistencies */
1558 /* XXX: only flush what is necessary */
1559 tb_flush(env);
1562 #endif
1565 /* enable or disable low levels log */
1566 void cpu_set_log(int log_flags)
1568 loglevel = log_flags;
1569 if (loglevel && !logfile) {
1570 logfile = fopen(logfilename, log_append ? "a" : "w");
1571 if (!logfile) {
1572 perror(logfilename);
1573 _exit(1);
1575 #if !defined(CONFIG_SOFTMMU)
1576 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1578 static char logfile_buf[4096];
1579 setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1581 #elif !defined(_WIN32)
1582 /* Win32 doesn't support line-buffering and requires size >= 2 */
1583 setvbuf(logfile, NULL, _IOLBF, 0);
1584 #endif
1585 log_append = 1;
1587 if (!loglevel && logfile) {
1588 fclose(logfile);
1589 logfile = NULL;
1593 void cpu_set_log_filename(const char *filename)
1595 logfilename = strdup(filename);
1596 if (logfile) {
1597 fclose(logfile);
1598 logfile = NULL;
1600 cpu_set_log(loglevel);
1603 static void cpu_unlink_tb(CPUState *env)
1605 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1606 problem and hope the cpu will stop of its own accord. For userspace
1607 emulation this often isn't actually as bad as it sounds. Often
1608 signals are used primarily to interrupt blocking syscalls. */
1609 TranslationBlock *tb;
1610 static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1612 spin_lock(&interrupt_lock);
1613 tb = env->current_tb;
1614 /* if the cpu is currently executing code, we must unlink it and
1615 all the potentially executing TB */
1616 if (tb) {
1617 env->current_tb = NULL;
1618 tb_reset_jump_recursive(tb);
1620 spin_unlock(&interrupt_lock);
1623 /* mask must never be zero, except for A20 change call */
1624 void cpu_interrupt(CPUState *env, int mask)
1626 int old_mask;
1628 old_mask = env->interrupt_request;
1629 env->interrupt_request |= mask;
1631 #ifndef CONFIG_USER_ONLY
1633 * If called from iothread context, wake the target cpu in
1634 * case its halted.
1636 if (!qemu_cpu_self(env)) {
1637 qemu_cpu_kick(env);
1638 return;
1640 #endif
1642 if (use_icount) {
1643 env->icount_decr.u16.high = 0xffff;
1644 #ifndef CONFIG_USER_ONLY
1645 if (!can_do_io(env)
1646 && (mask & ~old_mask) != 0) {
1647 cpu_abort(env, "Raised interrupt while not in I/O function");
1649 #endif
1650 } else {
1651 cpu_unlink_tb(env);
1655 void cpu_reset_interrupt(CPUState *env, int mask)
1657 env->interrupt_request &= ~mask;
1660 void cpu_exit(CPUState *env)
1662 env->exit_request = 1;
1663 cpu_unlink_tb(env);
1666 const CPULogItem cpu_log_items[] = {
1667 { CPU_LOG_TB_OUT_ASM, "out_asm",
1668 "show generated host assembly code for each compiled TB" },
1669 { CPU_LOG_TB_IN_ASM, "in_asm",
1670 "show target assembly code for each compiled TB" },
1671 { CPU_LOG_TB_OP, "op",
1672 "show micro ops for each compiled TB" },
1673 { CPU_LOG_TB_OP_OPT, "op_opt",
1674 "show micro ops "
1675 #ifdef TARGET_I386
1676 "before eflags optimization and "
1677 #endif
1678 "after liveness analysis" },
1679 { CPU_LOG_INT, "int",
1680 "show interrupts/exceptions in short format" },
1681 { CPU_LOG_EXEC, "exec",
1682 "show trace before each executed TB (lots of logs)" },
1683 { CPU_LOG_TB_CPU, "cpu",
1684 "show CPU state before block translation" },
1685 #ifdef TARGET_I386
1686 { CPU_LOG_PCALL, "pcall",
1687 "show protected mode far calls/returns/exceptions" },
1688 { CPU_LOG_RESET, "cpu_reset",
1689 "show CPU state before CPU resets" },
1690 #endif
1691 #ifdef DEBUG_IOPORT
1692 { CPU_LOG_IOPORT, "ioport",
1693 "show all i/o ports accesses" },
1694 #endif
1695 { 0, NULL, NULL },
1698 #ifndef CONFIG_USER_ONLY
1699 static QLIST_HEAD(memory_client_list, CPUPhysMemoryClient) memory_client_list
1700 = QLIST_HEAD_INITIALIZER(memory_client_list);
1702 static void cpu_notify_set_memory(target_phys_addr_t start_addr,
1703 ram_addr_t size,
1704 ram_addr_t phys_offset)
1706 CPUPhysMemoryClient *client;
1707 QLIST_FOREACH(client, &memory_client_list, list) {
1708 client->set_memory(client, start_addr, size, phys_offset);
1712 static int cpu_notify_sync_dirty_bitmap(target_phys_addr_t start,
1713 target_phys_addr_t end)
1715 CPUPhysMemoryClient *client;
1716 QLIST_FOREACH(client, &memory_client_list, list) {
1717 int r = client->sync_dirty_bitmap(client, start, end);
1718 if (r < 0)
1719 return r;
1721 return 0;
1724 static int cpu_notify_migration_log(int enable)
1726 CPUPhysMemoryClient *client;
1727 QLIST_FOREACH(client, &memory_client_list, list) {
1728 int r = client->migration_log(client, enable);
1729 if (r < 0)
1730 return r;
1732 return 0;
1735 static void phys_page_for_each_1(CPUPhysMemoryClient *client,
1736 int level, void **lp)
1738 int i;
1740 if (*lp == NULL) {
1741 return;
1743 if (level == 0) {
1744 PhysPageDesc *pd = *lp;
1745 for (i = 0; i < L2_SIZE; ++i) {
1746 if (pd[i].phys_offset != IO_MEM_UNASSIGNED) {
1747 client->set_memory(client, pd[i].region_offset,
1748 TARGET_PAGE_SIZE, pd[i].phys_offset);
1751 } else {
1752 void **pp = *lp;
1753 for (i = 0; i < L2_SIZE; ++i) {
1754 phys_page_for_each_1(client, level - 1, pp + i);
1759 static void phys_page_for_each(CPUPhysMemoryClient *client)
1761 int i;
1762 for (i = 0; i < P_L1_SIZE; ++i) {
1763 phys_page_for_each_1(client, P_L1_SHIFT / L2_BITS - 1,
1764 l1_phys_map + 1);
1768 void cpu_register_phys_memory_client(CPUPhysMemoryClient *client)
1770 QLIST_INSERT_HEAD(&memory_client_list, client, list);
1771 phys_page_for_each(client);
1774 void cpu_unregister_phys_memory_client(CPUPhysMemoryClient *client)
1776 QLIST_REMOVE(client, list);
1778 #endif
1780 static int cmp1(const char *s1, int n, const char *s2)
1782 if (strlen(s2) != n)
1783 return 0;
1784 return memcmp(s1, s2, n) == 0;
1787 /* takes a comma separated list of log masks. Return 0 if error. */
1788 int cpu_str_to_log_mask(const char *str)
1790 const CPULogItem *item;
1791 int mask;
1792 const char *p, *p1;
1794 p = str;
1795 mask = 0;
1796 for(;;) {
1797 p1 = strchr(p, ',');
1798 if (!p1)
1799 p1 = p + strlen(p);
1800 if(cmp1(p,p1-p,"all")) {
1801 for(item = cpu_log_items; item->mask != 0; item++) {
1802 mask |= item->mask;
1804 } else {
1805 for(item = cpu_log_items; item->mask != 0; item++) {
1806 if (cmp1(p, p1 - p, item->name))
1807 goto found;
1809 return 0;
1811 found:
1812 mask |= item->mask;
1813 if (*p1 != ',')
1814 break;
1815 p = p1 + 1;
1817 return mask;
1820 void cpu_abort(CPUState *env, const char *fmt, ...)
1822 va_list ap;
1823 va_list ap2;
1825 va_start(ap, fmt);
1826 va_copy(ap2, ap);
1827 fprintf(stderr, "qemu: fatal: ");
1828 vfprintf(stderr, fmt, ap);
1829 fprintf(stderr, "\n");
1830 #ifdef TARGET_I386
1831 cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1832 #else
1833 cpu_dump_state(env, stderr, fprintf, 0);
1834 #endif
1835 if (qemu_log_enabled()) {
1836 qemu_log("qemu: fatal: ");
1837 qemu_log_vprintf(fmt, ap2);
1838 qemu_log("\n");
1839 #ifdef TARGET_I386
1840 log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1841 #else
1842 log_cpu_state(env, 0);
1843 #endif
1844 qemu_log_flush();
1845 qemu_log_close();
1847 va_end(ap2);
1848 va_end(ap);
1849 #if defined(CONFIG_USER_ONLY)
1851 struct sigaction act;
1852 sigfillset(&act.sa_mask);
1853 act.sa_handler = SIG_DFL;
1854 sigaction(SIGABRT, &act, NULL);
1856 #endif
1857 abort();
1860 CPUState *cpu_copy(CPUState *env)
1862 CPUState *new_env = cpu_init(env->cpu_model_str);
1863 CPUState *next_cpu = new_env->next_cpu;
1864 int cpu_index = new_env->cpu_index;
1865 #if defined(TARGET_HAS_ICE)
1866 CPUBreakpoint *bp;
1867 CPUWatchpoint *wp;
1868 #endif
1870 memcpy(new_env, env, sizeof(CPUState));
1872 /* Preserve chaining and index. */
1873 new_env->next_cpu = next_cpu;
1874 new_env->cpu_index = cpu_index;
1876 /* Clone all break/watchpoints.
1877 Note: Once we support ptrace with hw-debug register access, make sure
1878 BP_CPU break/watchpoints are handled correctly on clone. */
1879 QTAILQ_INIT(&env->breakpoints);
1880 QTAILQ_INIT(&env->watchpoints);
1881 #if defined(TARGET_HAS_ICE)
1882 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1883 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1885 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1886 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1887 wp->flags, NULL);
1889 #endif
1891 return new_env;
1894 #if !defined(CONFIG_USER_ONLY)
1896 static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1898 unsigned int i;
1900 /* Discard jump cache entries for any tb which might potentially
1901 overlap the flushed page. */
1902 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1903 memset (&env->tb_jmp_cache[i], 0,
1904 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1906 i = tb_jmp_cache_hash_page(addr);
1907 memset (&env->tb_jmp_cache[i], 0,
1908 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1911 static CPUTLBEntry s_cputlb_empty_entry = {
1912 .addr_read = -1,
1913 .addr_write = -1,
1914 .addr_code = -1,
1915 .addend = -1,
1918 /* NOTE: if flush_global is true, also flush global entries (not
1919 implemented yet) */
1920 void tlb_flush(CPUState *env, int flush_global)
1922 int i;
1924 #if defined(DEBUG_TLB)
1925 printf("tlb_flush:\n");
1926 #endif
1927 /* must reset current TB so that interrupts cannot modify the
1928 links while we are modifying them */
1929 env->current_tb = NULL;
1931 for(i = 0; i < CPU_TLB_SIZE; i++) {
1932 int mmu_idx;
1933 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
1934 env->tlb_table[mmu_idx][i] = s_cputlb_empty_entry;
1938 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1940 env->tlb_flush_addr = -1;
1941 env->tlb_flush_mask = 0;
1942 tlb_flush_count++;
1945 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
1947 if (addr == (tlb_entry->addr_read &
1948 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1949 addr == (tlb_entry->addr_write &
1950 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1951 addr == (tlb_entry->addr_code &
1952 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1953 *tlb_entry = s_cputlb_empty_entry;
1957 void tlb_flush_page(CPUState *env, target_ulong addr)
1959 int i;
1960 int mmu_idx;
1962 #if defined(DEBUG_TLB)
1963 printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1964 #endif
1965 /* Check if we need to flush due to large pages. */
1966 if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
1967 #if defined(DEBUG_TLB)
1968 printf("tlb_flush_page: forced full flush ("
1969 TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
1970 env->tlb_flush_addr, env->tlb_flush_mask);
1971 #endif
1972 tlb_flush(env, 1);
1973 return;
1975 /* must reset current TB so that interrupts cannot modify the
1976 links while we are modifying them */
1977 env->current_tb = NULL;
1979 addr &= TARGET_PAGE_MASK;
1980 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1981 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
1982 tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
1984 tlb_flush_jmp_cache(env, addr);
1987 /* update the TLBs so that writes to code in the virtual page 'addr'
1988 can be detected */
1989 static void tlb_protect_code(ram_addr_t ram_addr)
1991 cpu_physical_memory_reset_dirty(ram_addr,
1992 ram_addr + TARGET_PAGE_SIZE,
1993 CODE_DIRTY_FLAG);
1996 /* update the TLB so that writes in physical page 'phys_addr' are no longer
1997 tested for self modifying code */
1998 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
1999 target_ulong vaddr)
2001 cpu_physical_memory_set_dirty_flags(ram_addr, CODE_DIRTY_FLAG);
2004 static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
2005 unsigned long start, unsigned long length)
2007 unsigned long addr;
2008 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2009 addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
2010 if ((addr - start) < length) {
2011 tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
2016 /* Note: start and end must be within the same ram block. */
2017 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
2018 int dirty_flags)
2020 CPUState *env;
2021 unsigned long length, start1;
2022 int i;
2024 start &= TARGET_PAGE_MASK;
2025 end = TARGET_PAGE_ALIGN(end);
2027 length = end - start;
2028 if (length == 0)
2029 return;
2030 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
2032 /* we modify the TLB cache so that the dirty bit will be set again
2033 when accessing the range */
2034 start1 = (unsigned long)qemu_safe_ram_ptr(start);
2035 /* Chek that we don't span multiple blocks - this breaks the
2036 address comparisons below. */
2037 if ((unsigned long)qemu_safe_ram_ptr(end - 1) - start1
2038 != (end - 1) - start) {
2039 abort();
2042 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2043 int mmu_idx;
2044 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2045 for(i = 0; i < CPU_TLB_SIZE; i++)
2046 tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
2047 start1, length);
2052 int cpu_physical_memory_set_dirty_tracking(int enable)
2054 int ret = 0;
2055 in_migration = enable;
2056 ret = cpu_notify_migration_log(!!enable);
2057 return ret;
2060 int cpu_physical_memory_get_dirty_tracking(void)
2062 return in_migration;
2065 int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
2066 target_phys_addr_t end_addr)
2068 int ret;
2070 ret = cpu_notify_sync_dirty_bitmap(start_addr, end_addr);
2071 return ret;
2074 static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
2076 ram_addr_t ram_addr;
2077 void *p;
2079 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2080 p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
2081 + tlb_entry->addend);
2082 ram_addr = qemu_ram_addr_from_host_nofail(p);
2083 if (!cpu_physical_memory_is_dirty(ram_addr)) {
2084 tlb_entry->addr_write |= TLB_NOTDIRTY;
2089 /* update the TLB according to the current state of the dirty bits */
2090 void cpu_tlb_update_dirty(CPUState *env)
2092 int i;
2093 int mmu_idx;
2094 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2095 for(i = 0; i < CPU_TLB_SIZE; i++)
2096 tlb_update_dirty(&env->tlb_table[mmu_idx][i]);
2100 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
2102 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
2103 tlb_entry->addr_write = vaddr;
2106 /* update the TLB corresponding to virtual page vaddr
2107 so that it is no longer dirty */
2108 static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
2110 int i;
2111 int mmu_idx;
2113 vaddr &= TARGET_PAGE_MASK;
2114 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2115 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2116 tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
2119 /* Our TLB does not support large pages, so remember the area covered by
2120 large pages and trigger a full TLB flush if these are invalidated. */
2121 static void tlb_add_large_page(CPUState *env, target_ulong vaddr,
2122 target_ulong size)
2124 target_ulong mask = ~(size - 1);
2126 if (env->tlb_flush_addr == (target_ulong)-1) {
2127 env->tlb_flush_addr = vaddr & mask;
2128 env->tlb_flush_mask = mask;
2129 return;
2131 /* Extend the existing region to include the new page.
2132 This is a compromise between unnecessary flushes and the cost
2133 of maintaining a full variable size TLB. */
2134 mask &= env->tlb_flush_mask;
2135 while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
2136 mask <<= 1;
2138 env->tlb_flush_addr &= mask;
2139 env->tlb_flush_mask = mask;
2142 /* Add a new TLB entry. At most one entry for a given virtual address
2143 is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
2144 supplied size is only used by tlb_flush_page. */
2145 void tlb_set_page(CPUState *env, target_ulong vaddr,
2146 target_phys_addr_t paddr, int prot,
2147 int mmu_idx, target_ulong size)
2149 PhysPageDesc *p;
2150 unsigned long pd;
2151 unsigned int index;
2152 target_ulong address;
2153 target_ulong code_address;
2154 unsigned long addend;
2155 CPUTLBEntry *te;
2156 CPUWatchpoint *wp;
2157 target_phys_addr_t iotlb;
2159 assert(size >= TARGET_PAGE_SIZE);
2160 if (size != TARGET_PAGE_SIZE) {
2161 tlb_add_large_page(env, vaddr, size);
2163 p = phys_page_find(paddr >> TARGET_PAGE_BITS);
2164 if (!p) {
2165 pd = IO_MEM_UNASSIGNED;
2166 } else {
2167 pd = p->phys_offset;
2169 #if defined(DEBUG_TLB)
2170 printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
2171 " prot=%x idx=%d pd=0x%08lx\n",
2172 vaddr, paddr, prot, mmu_idx, pd);
2173 #endif
2175 address = vaddr;
2176 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
2177 /* IO memory case (romd handled later) */
2178 address |= TLB_MMIO;
2180 addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
2181 if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
2182 /* Normal RAM. */
2183 iotlb = pd & TARGET_PAGE_MASK;
2184 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
2185 iotlb |= IO_MEM_NOTDIRTY;
2186 else
2187 iotlb |= IO_MEM_ROM;
2188 } else {
2189 /* IO handlers are currently passed a physical address.
2190 It would be nice to pass an offset from the base address
2191 of that region. This would avoid having to special case RAM,
2192 and avoid full address decoding in every device.
2193 We can't use the high bits of pd for this because
2194 IO_MEM_ROMD uses these as a ram address. */
2195 iotlb = (pd & ~TARGET_PAGE_MASK);
2196 if (p) {
2197 iotlb += p->region_offset;
2198 } else {
2199 iotlb += paddr;
2203 code_address = address;
2204 /* Make accesses to pages with watchpoints go via the
2205 watchpoint trap routines. */
2206 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
2207 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2208 /* Avoid trapping reads of pages with a write breakpoint. */
2209 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
2210 iotlb = io_mem_watch + paddr;
2211 address |= TLB_MMIO;
2212 break;
2217 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2218 env->iotlb[mmu_idx][index] = iotlb - vaddr;
2219 te = &env->tlb_table[mmu_idx][index];
2220 te->addend = addend - vaddr;
2221 if (prot & PAGE_READ) {
2222 te->addr_read = address;
2223 } else {
2224 te->addr_read = -1;
2227 if (prot & PAGE_EXEC) {
2228 te->addr_code = code_address;
2229 } else {
2230 te->addr_code = -1;
2232 if (prot & PAGE_WRITE) {
2233 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2234 (pd & IO_MEM_ROMD)) {
2235 /* Write access calls the I/O callback. */
2236 te->addr_write = address | TLB_MMIO;
2237 } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2238 !cpu_physical_memory_is_dirty(pd)) {
2239 te->addr_write = address | TLB_NOTDIRTY;
2240 } else {
2241 te->addr_write = address;
2243 } else {
2244 te->addr_write = -1;
2248 #else
2250 void tlb_flush(CPUState *env, int flush_global)
2254 void tlb_flush_page(CPUState *env, target_ulong addr)
2259 * Walks guest process memory "regions" one by one
2260 * and calls callback function 'fn' for each region.
2263 struct walk_memory_regions_data
2265 walk_memory_regions_fn fn;
2266 void *priv;
2267 unsigned long start;
2268 int prot;
2271 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
2272 abi_ulong end, int new_prot)
2274 if (data->start != -1ul) {
2275 int rc = data->fn(data->priv, data->start, end, data->prot);
2276 if (rc != 0) {
2277 return rc;
2281 data->start = (new_prot ? end : -1ul);
2282 data->prot = new_prot;
2284 return 0;
2287 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
2288 abi_ulong base, int level, void **lp)
2290 abi_ulong pa;
2291 int i, rc;
2293 if (*lp == NULL) {
2294 return walk_memory_regions_end(data, base, 0);
2297 if (level == 0) {
2298 PageDesc *pd = *lp;
2299 for (i = 0; i < L2_SIZE; ++i) {
2300 int prot = pd[i].flags;
2302 pa = base | (i << TARGET_PAGE_BITS);
2303 if (prot != data->prot) {
2304 rc = walk_memory_regions_end(data, pa, prot);
2305 if (rc != 0) {
2306 return rc;
2310 } else {
2311 void **pp = *lp;
2312 for (i = 0; i < L2_SIZE; ++i) {
2313 pa = base | ((abi_ulong)i <<
2314 (TARGET_PAGE_BITS + L2_BITS * level));
2315 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
2316 if (rc != 0) {
2317 return rc;
2322 return 0;
2325 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
2327 struct walk_memory_regions_data data;
2328 unsigned long i;
2330 data.fn = fn;
2331 data.priv = priv;
2332 data.start = -1ul;
2333 data.prot = 0;
2335 for (i = 0; i < V_L1_SIZE; i++) {
2336 int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
2337 V_L1_SHIFT / L2_BITS - 1, l1_map + i);
2338 if (rc != 0) {
2339 return rc;
2343 return walk_memory_regions_end(&data, 0, 0);
2346 static int dump_region(void *priv, abi_ulong start,
2347 abi_ulong end, unsigned long prot)
2349 FILE *f = (FILE *)priv;
2351 (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
2352 " "TARGET_ABI_FMT_lx" %c%c%c\n",
2353 start, end, end - start,
2354 ((prot & PAGE_READ) ? 'r' : '-'),
2355 ((prot & PAGE_WRITE) ? 'w' : '-'),
2356 ((prot & PAGE_EXEC) ? 'x' : '-'));
2358 return (0);
2361 /* dump memory mappings */
2362 void page_dump(FILE *f)
2364 (void) fprintf(f, "%-8s %-8s %-8s %s\n",
2365 "start", "end", "size", "prot");
2366 walk_memory_regions(f, dump_region);
2369 int page_get_flags(target_ulong address)
2371 PageDesc *p;
2373 p = page_find(address >> TARGET_PAGE_BITS);
2374 if (!p)
2375 return 0;
2376 return p->flags;
2379 /* Modify the flags of a page and invalidate the code if necessary.
2380 The flag PAGE_WRITE_ORG is positioned automatically depending
2381 on PAGE_WRITE. The mmap_lock should already be held. */
2382 void page_set_flags(target_ulong start, target_ulong end, int flags)
2384 target_ulong addr, len;
2386 /* This function should never be called with addresses outside the
2387 guest address space. If this assert fires, it probably indicates
2388 a missing call to h2g_valid. */
2389 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2390 assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2391 #endif
2392 assert(start < end);
2394 start = start & TARGET_PAGE_MASK;
2395 end = TARGET_PAGE_ALIGN(end);
2397 if (flags & PAGE_WRITE) {
2398 flags |= PAGE_WRITE_ORG;
2401 for (addr = start, len = end - start;
2402 len != 0;
2403 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2404 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2406 /* If the write protection bit is set, then we invalidate
2407 the code inside. */
2408 if (!(p->flags & PAGE_WRITE) &&
2409 (flags & PAGE_WRITE) &&
2410 p->first_tb) {
2411 tb_invalidate_phys_page(addr, 0, NULL);
2413 p->flags = flags;
2417 int page_check_range(target_ulong start, target_ulong len, int flags)
2419 PageDesc *p;
2420 target_ulong end;
2421 target_ulong addr;
2423 /* This function should never be called with addresses outside the
2424 guest address space. If this assert fires, it probably indicates
2425 a missing call to h2g_valid. */
2426 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2427 assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2428 #endif
2430 if (len == 0) {
2431 return 0;
2433 if (start + len - 1 < start) {
2434 /* We've wrapped around. */
2435 return -1;
2438 end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2439 start = start & TARGET_PAGE_MASK;
2441 for (addr = start, len = end - start;
2442 len != 0;
2443 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2444 p = page_find(addr >> TARGET_PAGE_BITS);
2445 if( !p )
2446 return -1;
2447 if( !(p->flags & PAGE_VALID) )
2448 return -1;
2450 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2451 return -1;
2452 if (flags & PAGE_WRITE) {
2453 if (!(p->flags & PAGE_WRITE_ORG))
2454 return -1;
2455 /* unprotect the page if it was put read-only because it
2456 contains translated code */
2457 if (!(p->flags & PAGE_WRITE)) {
2458 if (!page_unprotect(addr, 0, NULL))
2459 return -1;
2461 return 0;
2464 return 0;
2467 /* called from signal handler: invalidate the code and unprotect the
2468 page. Return TRUE if the fault was successfully handled. */
2469 int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2471 unsigned int prot;
2472 PageDesc *p;
2473 target_ulong host_start, host_end, addr;
2475 /* Technically this isn't safe inside a signal handler. However we
2476 know this only ever happens in a synchronous SEGV handler, so in
2477 practice it seems to be ok. */
2478 mmap_lock();
2480 p = page_find(address >> TARGET_PAGE_BITS);
2481 if (!p) {
2482 mmap_unlock();
2483 return 0;
2486 /* if the page was really writable, then we change its
2487 protection back to writable */
2488 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
2489 host_start = address & qemu_host_page_mask;
2490 host_end = host_start + qemu_host_page_size;
2492 prot = 0;
2493 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
2494 p = page_find(addr >> TARGET_PAGE_BITS);
2495 p->flags |= PAGE_WRITE;
2496 prot |= p->flags;
2498 /* and since the content will be modified, we must invalidate
2499 the corresponding translated code. */
2500 tb_invalidate_phys_page(addr, pc, puc);
2501 #ifdef DEBUG_TB_CHECK
2502 tb_invalidate_check(addr);
2503 #endif
2505 mprotect((void *)g2h(host_start), qemu_host_page_size,
2506 prot & PAGE_BITS);
2508 mmap_unlock();
2509 return 1;
2511 mmap_unlock();
2512 return 0;
2515 static inline void tlb_set_dirty(CPUState *env,
2516 unsigned long addr, target_ulong vaddr)
2519 #endif /* defined(CONFIG_USER_ONLY) */
2521 #if !defined(CONFIG_USER_ONLY)
2523 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
2524 typedef struct subpage_t {
2525 target_phys_addr_t base;
2526 ram_addr_t sub_io_index[TARGET_PAGE_SIZE];
2527 ram_addr_t region_offset[TARGET_PAGE_SIZE];
2528 } subpage_t;
2530 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2531 ram_addr_t memory, ram_addr_t region_offset);
2532 static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2533 ram_addr_t orig_memory,
2534 ram_addr_t region_offset);
2535 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2536 need_subpage) \
2537 do { \
2538 if (addr > start_addr) \
2539 start_addr2 = 0; \
2540 else { \
2541 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2542 if (start_addr2 > 0) \
2543 need_subpage = 1; \
2546 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2547 end_addr2 = TARGET_PAGE_SIZE - 1; \
2548 else { \
2549 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2550 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2551 need_subpage = 1; \
2553 } while (0)
2555 /* register physical memory.
2556 For RAM, 'size' must be a multiple of the target page size.
2557 If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2558 io memory page. The address used when calling the IO function is
2559 the offset from the start of the region, plus region_offset. Both
2560 start_addr and region_offset are rounded down to a page boundary
2561 before calculating this offset. This should not be a problem unless
2562 the low bits of start_addr and region_offset differ. */
2563 void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
2564 ram_addr_t size,
2565 ram_addr_t phys_offset,
2566 ram_addr_t region_offset)
2568 target_phys_addr_t addr, end_addr;
2569 PhysPageDesc *p;
2570 CPUState *env;
2571 ram_addr_t orig_size = size;
2572 subpage_t *subpage;
2574 cpu_notify_set_memory(start_addr, size, phys_offset);
2576 if (phys_offset == IO_MEM_UNASSIGNED) {
2577 region_offset = start_addr;
2579 region_offset &= TARGET_PAGE_MASK;
2580 size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2581 end_addr = start_addr + (target_phys_addr_t)size;
2582 for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2583 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2584 if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2585 ram_addr_t orig_memory = p->phys_offset;
2586 target_phys_addr_t start_addr2, end_addr2;
2587 int need_subpage = 0;
2589 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2590 need_subpage);
2591 if (need_subpage) {
2592 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2593 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2594 &p->phys_offset, orig_memory,
2595 p->region_offset);
2596 } else {
2597 subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2598 >> IO_MEM_SHIFT];
2600 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2601 region_offset);
2602 p->region_offset = 0;
2603 } else {
2604 p->phys_offset = phys_offset;
2605 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2606 (phys_offset & IO_MEM_ROMD))
2607 phys_offset += TARGET_PAGE_SIZE;
2609 } else {
2610 p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2611 p->phys_offset = phys_offset;
2612 p->region_offset = region_offset;
2613 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2614 (phys_offset & IO_MEM_ROMD)) {
2615 phys_offset += TARGET_PAGE_SIZE;
2616 } else {
2617 target_phys_addr_t start_addr2, end_addr2;
2618 int need_subpage = 0;
2620 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2621 end_addr2, need_subpage);
2623 if (need_subpage) {
2624 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2625 &p->phys_offset, IO_MEM_UNASSIGNED,
2626 addr & TARGET_PAGE_MASK);
2627 subpage_register(subpage, start_addr2, end_addr2,
2628 phys_offset, region_offset);
2629 p->region_offset = 0;
2633 region_offset += TARGET_PAGE_SIZE;
2636 /* since each CPU stores ram addresses in its TLB cache, we must
2637 reset the modified entries */
2638 /* XXX: slow ! */
2639 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2640 tlb_flush(env, 1);
2644 /* XXX: temporary until new memory mapping API */
2645 ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2647 PhysPageDesc *p;
2649 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2650 if (!p)
2651 return IO_MEM_UNASSIGNED;
2652 return p->phys_offset;
2655 void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2657 if (kvm_enabled())
2658 kvm_coalesce_mmio_region(addr, size);
2661 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2663 if (kvm_enabled())
2664 kvm_uncoalesce_mmio_region(addr, size);
2667 void qemu_flush_coalesced_mmio_buffer(void)
2669 if (kvm_enabled())
2670 kvm_flush_coalesced_mmio_buffer();
2673 #if defined(__linux__) && !defined(TARGET_S390X)
2675 #include <sys/vfs.h>
2677 #define HUGETLBFS_MAGIC 0x958458f6
2679 static long gethugepagesize(const char *path)
2681 struct statfs fs;
2682 int ret;
2684 do {
2685 ret = statfs(path, &fs);
2686 } while (ret != 0 && errno == EINTR);
2688 if (ret != 0) {
2689 perror(path);
2690 return 0;
2693 if (fs.f_type != HUGETLBFS_MAGIC)
2694 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
2696 return fs.f_bsize;
2699 static void *file_ram_alloc(RAMBlock *block,
2700 ram_addr_t memory,
2701 const char *path)
2703 char *filename;
2704 void *area;
2705 int fd;
2706 #ifdef MAP_POPULATE
2707 int flags;
2708 #endif
2709 unsigned long hpagesize;
2711 hpagesize = gethugepagesize(path);
2712 if (!hpagesize) {
2713 return NULL;
2716 if (memory < hpagesize) {
2717 return NULL;
2720 if (kvm_enabled() && !kvm_has_sync_mmu()) {
2721 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
2722 return NULL;
2725 if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
2726 return NULL;
2729 fd = mkstemp(filename);
2730 if (fd < 0) {
2731 perror("unable to create backing store for hugepages");
2732 free(filename);
2733 return NULL;
2735 unlink(filename);
2736 free(filename);
2738 memory = (memory+hpagesize-1) & ~(hpagesize-1);
2741 * ftruncate is not supported by hugetlbfs in older
2742 * hosts, so don't bother bailing out on errors.
2743 * If anything goes wrong with it under other filesystems,
2744 * mmap will fail.
2746 if (ftruncate(fd, memory))
2747 perror("ftruncate");
2749 #ifdef MAP_POPULATE
2750 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
2751 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
2752 * to sidestep this quirk.
2754 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
2755 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
2756 #else
2757 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
2758 #endif
2759 if (area == MAP_FAILED) {
2760 perror("file_ram_alloc: can't mmap RAM pages");
2761 close(fd);
2762 return (NULL);
2764 block->fd = fd;
2765 return area;
2767 #endif
2769 static ram_addr_t find_ram_offset(ram_addr_t size)
2771 RAMBlock *block, *next_block;
2772 ram_addr_t offset = 0, mingap = ULONG_MAX;
2774 if (QLIST_EMPTY(&ram_list.blocks))
2775 return 0;
2777 QLIST_FOREACH(block, &ram_list.blocks, next) {
2778 ram_addr_t end, next = ULONG_MAX;
2780 end = block->offset + block->length;
2782 QLIST_FOREACH(next_block, &ram_list.blocks, next) {
2783 if (next_block->offset >= end) {
2784 next = MIN(next, next_block->offset);
2787 if (next - end >= size && next - end < mingap) {
2788 offset = end;
2789 mingap = next - end;
2792 return offset;
2795 static ram_addr_t last_ram_offset(void)
2797 RAMBlock *block;
2798 ram_addr_t last = 0;
2800 QLIST_FOREACH(block, &ram_list.blocks, next)
2801 last = MAX(last, block->offset + block->length);
2803 return last;
2806 ram_addr_t qemu_ram_alloc_from_ptr(DeviceState *dev, const char *name,
2807 ram_addr_t size, void *host)
2809 RAMBlock *new_block, *block;
2811 size = TARGET_PAGE_ALIGN(size);
2812 new_block = qemu_mallocz(sizeof(*new_block));
2814 if (dev && dev->parent_bus && dev->parent_bus->info->get_dev_path) {
2815 char *id = dev->parent_bus->info->get_dev_path(dev);
2816 if (id) {
2817 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
2818 qemu_free(id);
2821 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
2823 QLIST_FOREACH(block, &ram_list.blocks, next) {
2824 if (!strcmp(block->idstr, new_block->idstr)) {
2825 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
2826 new_block->idstr);
2827 abort();
2831 if (host) {
2832 new_block->host = host;
2833 } else {
2834 if (mem_path) {
2835 #if defined (__linux__) && !defined(TARGET_S390X)
2836 new_block->host = file_ram_alloc(new_block, size, mem_path);
2837 if (!new_block->host) {
2838 new_block->host = qemu_vmalloc(size);
2839 qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
2841 #else
2842 fprintf(stderr, "-mem-path option unsupported\n");
2843 exit(1);
2844 #endif
2845 } else {
2846 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2847 /* XXX S390 KVM requires the topmost vma of the RAM to be < 256GB */
2848 new_block->host = mmap((void*)0x1000000, size,
2849 PROT_EXEC|PROT_READ|PROT_WRITE,
2850 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
2851 #else
2852 new_block->host = qemu_vmalloc(size);
2853 #endif
2854 qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
2858 new_block->offset = find_ram_offset(size);
2859 new_block->length = size;
2861 QLIST_INSERT_HEAD(&ram_list.blocks, new_block, next);
2863 ram_list.phys_dirty = qemu_realloc(ram_list.phys_dirty,
2864 last_ram_offset() >> TARGET_PAGE_BITS);
2865 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
2866 0xff, size >> TARGET_PAGE_BITS);
2868 if (kvm_enabled())
2869 kvm_setup_guest_memory(new_block->host, size);
2871 return new_block->offset;
2874 ram_addr_t qemu_ram_alloc(DeviceState *dev, const char *name, ram_addr_t size)
2876 return qemu_ram_alloc_from_ptr(dev, name, size, NULL);
2879 void qemu_ram_free(ram_addr_t addr)
2881 RAMBlock *block;
2883 QLIST_FOREACH(block, &ram_list.blocks, next) {
2884 if (addr == block->offset) {
2885 QLIST_REMOVE(block, next);
2886 if (mem_path) {
2887 #if defined (__linux__) && !defined(TARGET_S390X)
2888 if (block->fd) {
2889 munmap(block->host, block->length);
2890 close(block->fd);
2891 } else {
2892 qemu_vfree(block->host);
2894 #endif
2895 } else {
2896 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2897 munmap(block->host, block->length);
2898 #else
2899 qemu_vfree(block->host);
2900 #endif
2902 qemu_free(block);
2903 return;
2909 /* Return a host pointer to ram allocated with qemu_ram_alloc.
2910 With the exception of the softmmu code in this file, this should
2911 only be used for local memory (e.g. video ram) that the device owns,
2912 and knows it isn't going to access beyond the end of the block.
2914 It should not be used for general purpose DMA.
2915 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
2917 void *qemu_get_ram_ptr(ram_addr_t addr)
2919 RAMBlock *block;
2921 QLIST_FOREACH(block, &ram_list.blocks, next) {
2922 if (addr - block->offset < block->length) {
2923 QLIST_REMOVE(block, next);
2924 QLIST_INSERT_HEAD(&ram_list.blocks, block, next);
2925 return block->host + (addr - block->offset);
2929 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
2930 abort();
2932 return NULL;
2935 /* Return a host pointer to ram allocated with qemu_ram_alloc.
2936 * Same as qemu_get_ram_ptr but avoid reordering ramblocks.
2938 void *qemu_safe_ram_ptr(ram_addr_t addr)
2940 RAMBlock *block;
2942 QLIST_FOREACH(block, &ram_list.blocks, next) {
2943 if (addr - block->offset < block->length) {
2944 return block->host + (addr - block->offset);
2948 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
2949 abort();
2951 return NULL;
2954 int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
2956 RAMBlock *block;
2957 uint8_t *host = ptr;
2959 QLIST_FOREACH(block, &ram_list.blocks, next) {
2960 if (host - block->host < block->length) {
2961 *ram_addr = block->offset + (host - block->host);
2962 return 0;
2965 return -1;
2968 /* Some of the softmmu routines need to translate from a host pointer
2969 (typically a TLB entry) back to a ram offset. */
2970 ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
2972 ram_addr_t ram_addr;
2974 if (qemu_ram_addr_from_host(ptr, &ram_addr)) {
2975 fprintf(stderr, "Bad ram pointer %p\n", ptr);
2976 abort();
2978 return ram_addr;
2981 static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2983 #ifdef DEBUG_UNASSIGNED
2984 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2985 #endif
2986 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2987 do_unassigned_access(addr, 0, 0, 0, 1);
2988 #endif
2989 return 0;
2992 static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2994 #ifdef DEBUG_UNASSIGNED
2995 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2996 #endif
2997 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2998 do_unassigned_access(addr, 0, 0, 0, 2);
2999 #endif
3000 return 0;
3003 static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
3005 #ifdef DEBUG_UNASSIGNED
3006 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
3007 #endif
3008 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3009 do_unassigned_access(addr, 0, 0, 0, 4);
3010 #endif
3011 return 0;
3014 static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
3016 #ifdef DEBUG_UNASSIGNED
3017 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3018 #endif
3019 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3020 do_unassigned_access(addr, 1, 0, 0, 1);
3021 #endif
3024 static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
3026 #ifdef DEBUG_UNASSIGNED
3027 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3028 #endif
3029 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3030 do_unassigned_access(addr, 1, 0, 0, 2);
3031 #endif
3034 static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
3036 #ifdef DEBUG_UNASSIGNED
3037 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3038 #endif
3039 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3040 do_unassigned_access(addr, 1, 0, 0, 4);
3041 #endif
3044 static CPUReadMemoryFunc * const unassigned_mem_read[3] = {
3045 unassigned_mem_readb,
3046 unassigned_mem_readw,
3047 unassigned_mem_readl,
3050 static CPUWriteMemoryFunc * const unassigned_mem_write[3] = {
3051 unassigned_mem_writeb,
3052 unassigned_mem_writew,
3053 unassigned_mem_writel,
3056 static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
3057 uint32_t val)
3059 int dirty_flags;
3060 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3061 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3062 #if !defined(CONFIG_USER_ONLY)
3063 tb_invalidate_phys_page_fast(ram_addr, 1);
3064 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3065 #endif
3067 stb_p(qemu_get_ram_ptr(ram_addr), val);
3068 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3069 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
3070 /* we remove the notdirty callback only if the code has been
3071 flushed */
3072 if (dirty_flags == 0xff)
3073 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3076 static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
3077 uint32_t val)
3079 int dirty_flags;
3080 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3081 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3082 #if !defined(CONFIG_USER_ONLY)
3083 tb_invalidate_phys_page_fast(ram_addr, 2);
3084 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3085 #endif
3087 stw_p(qemu_get_ram_ptr(ram_addr), val);
3088 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3089 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
3090 /* we remove the notdirty callback only if the code has been
3091 flushed */
3092 if (dirty_flags == 0xff)
3093 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3096 static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
3097 uint32_t val)
3099 int dirty_flags;
3100 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3101 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3102 #if !defined(CONFIG_USER_ONLY)
3103 tb_invalidate_phys_page_fast(ram_addr, 4);
3104 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3105 #endif
3107 stl_p(qemu_get_ram_ptr(ram_addr), val);
3108 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3109 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
3110 /* we remove the notdirty callback only if the code has been
3111 flushed */
3112 if (dirty_flags == 0xff)
3113 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3116 static CPUReadMemoryFunc * const error_mem_read[3] = {
3117 NULL, /* never used */
3118 NULL, /* never used */
3119 NULL, /* never used */
3122 static CPUWriteMemoryFunc * const notdirty_mem_write[3] = {
3123 notdirty_mem_writeb,
3124 notdirty_mem_writew,
3125 notdirty_mem_writel,
3128 /* Generate a debug exception if a watchpoint has been hit. */
3129 static void check_watchpoint(int offset, int len_mask, int flags)
3131 CPUState *env = cpu_single_env;
3132 target_ulong pc, cs_base;
3133 TranslationBlock *tb;
3134 target_ulong vaddr;
3135 CPUWatchpoint *wp;
3136 int cpu_flags;
3138 if (env->watchpoint_hit) {
3139 /* We re-entered the check after replacing the TB. Now raise
3140 * the debug interrupt so that is will trigger after the
3141 * current instruction. */
3142 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
3143 return;
3145 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
3146 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
3147 if ((vaddr == (wp->vaddr & len_mask) ||
3148 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
3149 wp->flags |= BP_WATCHPOINT_HIT;
3150 if (!env->watchpoint_hit) {
3151 env->watchpoint_hit = wp;
3152 tb = tb_find_pc(env->mem_io_pc);
3153 if (!tb) {
3154 cpu_abort(env, "check_watchpoint: could not find TB for "
3155 "pc=%p", (void *)env->mem_io_pc);
3157 cpu_restore_state(tb, env, env->mem_io_pc, NULL);
3158 tb_phys_invalidate(tb, -1);
3159 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
3160 env->exception_index = EXCP_DEBUG;
3161 } else {
3162 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
3163 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
3165 cpu_resume_from_signal(env, NULL);
3167 } else {
3168 wp->flags &= ~BP_WATCHPOINT_HIT;
3173 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
3174 so these check for a hit then pass through to the normal out-of-line
3175 phys routines. */
3176 static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
3178 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
3179 return ldub_phys(addr);
3182 static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
3184 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
3185 return lduw_phys(addr);
3188 static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
3190 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
3191 return ldl_phys(addr);
3194 static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
3195 uint32_t val)
3197 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
3198 stb_phys(addr, val);
3201 static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
3202 uint32_t val)
3204 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
3205 stw_phys(addr, val);
3208 static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
3209 uint32_t val)
3211 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
3212 stl_phys(addr, val);
3215 static CPUReadMemoryFunc * const watch_mem_read[3] = {
3216 watch_mem_readb,
3217 watch_mem_readw,
3218 watch_mem_readl,
3221 static CPUWriteMemoryFunc * const watch_mem_write[3] = {
3222 watch_mem_writeb,
3223 watch_mem_writew,
3224 watch_mem_writel,
3227 static inline uint32_t subpage_readlen (subpage_t *mmio,
3228 target_phys_addr_t addr,
3229 unsigned int len)
3231 unsigned int idx = SUBPAGE_IDX(addr);
3232 #if defined(DEBUG_SUBPAGE)
3233 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
3234 mmio, len, addr, idx);
3235 #endif
3237 addr += mmio->region_offset[idx];
3238 idx = mmio->sub_io_index[idx];
3239 return io_mem_read[idx][len](io_mem_opaque[idx], addr);
3242 static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
3243 uint32_t value, unsigned int len)
3245 unsigned int idx = SUBPAGE_IDX(addr);
3246 #if defined(DEBUG_SUBPAGE)
3247 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n",
3248 __func__, mmio, len, addr, idx, value);
3249 #endif
3251 addr += mmio->region_offset[idx];
3252 idx = mmio->sub_io_index[idx];
3253 io_mem_write[idx][len](io_mem_opaque[idx], addr, value);
3256 static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
3258 return subpage_readlen(opaque, addr, 0);
3261 static void subpage_writeb (void *opaque, target_phys_addr_t addr,
3262 uint32_t value)
3264 subpage_writelen(opaque, addr, value, 0);
3267 static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
3269 return subpage_readlen(opaque, addr, 1);
3272 static void subpage_writew (void *opaque, target_phys_addr_t addr,
3273 uint32_t value)
3275 subpage_writelen(opaque, addr, value, 1);
3278 static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
3280 return subpage_readlen(opaque, addr, 2);
3283 static void subpage_writel (void *opaque, target_phys_addr_t addr,
3284 uint32_t value)
3286 subpage_writelen(opaque, addr, value, 2);
3289 static CPUReadMemoryFunc * const subpage_read[] = {
3290 &subpage_readb,
3291 &subpage_readw,
3292 &subpage_readl,
3295 static CPUWriteMemoryFunc * const subpage_write[] = {
3296 &subpage_writeb,
3297 &subpage_writew,
3298 &subpage_writel,
3301 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
3302 ram_addr_t memory, ram_addr_t region_offset)
3304 int idx, eidx;
3306 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
3307 return -1;
3308 idx = SUBPAGE_IDX(start);
3309 eidx = SUBPAGE_IDX(end);
3310 #if defined(DEBUG_SUBPAGE)
3311 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
3312 mmio, start, end, idx, eidx, memory);
3313 #endif
3314 if ((memory & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
3315 memory = IO_MEM_UNASSIGNED;
3316 memory = (memory >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3317 for (; idx <= eidx; idx++) {
3318 mmio->sub_io_index[idx] = memory;
3319 mmio->region_offset[idx] = region_offset;
3322 return 0;
3325 static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
3326 ram_addr_t orig_memory,
3327 ram_addr_t region_offset)
3329 subpage_t *mmio;
3330 int subpage_memory;
3332 mmio = qemu_mallocz(sizeof(subpage_t));
3334 mmio->base = base;
3335 subpage_memory = cpu_register_io_memory(subpage_read, subpage_write, mmio,
3336 DEVICE_NATIVE_ENDIAN);
3337 #if defined(DEBUG_SUBPAGE)
3338 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
3339 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
3340 #endif
3341 *phys = subpage_memory | IO_MEM_SUBPAGE;
3342 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, orig_memory, region_offset);
3344 return mmio;
3347 static int get_free_io_mem_idx(void)
3349 int i;
3351 for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
3352 if (!io_mem_used[i]) {
3353 io_mem_used[i] = 1;
3354 return i;
3356 fprintf(stderr, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES);
3357 return -1;
3361 * Usually, devices operate in little endian mode. There are devices out
3362 * there that operate in big endian too. Each device gets byte swapped
3363 * mmio if plugged onto a CPU that does the other endianness.
3365 * CPU Device swap?
3367 * little little no
3368 * little big yes
3369 * big little yes
3370 * big big no
3373 typedef struct SwapEndianContainer {
3374 CPUReadMemoryFunc *read[3];
3375 CPUWriteMemoryFunc *write[3];
3376 void *opaque;
3377 } SwapEndianContainer;
3379 static uint32_t swapendian_mem_readb (void *opaque, target_phys_addr_t addr)
3381 uint32_t val;
3382 SwapEndianContainer *c = opaque;
3383 val = c->read[0](c->opaque, addr);
3384 return val;
3387 static uint32_t swapendian_mem_readw(void *opaque, target_phys_addr_t addr)
3389 uint32_t val;
3390 SwapEndianContainer *c = opaque;
3391 val = bswap16(c->read[1](c->opaque, addr));
3392 return val;
3395 static uint32_t swapendian_mem_readl(void *opaque, target_phys_addr_t addr)
3397 uint32_t val;
3398 SwapEndianContainer *c = opaque;
3399 val = bswap32(c->read[2](c->opaque, addr));
3400 return val;
3403 static CPUReadMemoryFunc * const swapendian_readfn[3]={
3404 swapendian_mem_readb,
3405 swapendian_mem_readw,
3406 swapendian_mem_readl
3409 static void swapendian_mem_writeb(void *opaque, target_phys_addr_t addr,
3410 uint32_t val)
3412 SwapEndianContainer *c = opaque;
3413 c->write[0](c->opaque, addr, val);
3416 static void swapendian_mem_writew(void *opaque, target_phys_addr_t addr,
3417 uint32_t val)
3419 SwapEndianContainer *c = opaque;
3420 c->write[1](c->opaque, addr, bswap16(val));
3423 static void swapendian_mem_writel(void *opaque, target_phys_addr_t addr,
3424 uint32_t val)
3426 SwapEndianContainer *c = opaque;
3427 c->write[2](c->opaque, addr, bswap32(val));
3430 static CPUWriteMemoryFunc * const swapendian_writefn[3]={
3431 swapendian_mem_writeb,
3432 swapendian_mem_writew,
3433 swapendian_mem_writel
3436 static void swapendian_init(int io_index)
3438 SwapEndianContainer *c = qemu_malloc(sizeof(SwapEndianContainer));
3439 int i;
3441 /* Swap mmio for big endian targets */
3442 c->opaque = io_mem_opaque[io_index];
3443 for (i = 0; i < 3; i++) {
3444 c->read[i] = io_mem_read[io_index][i];
3445 c->write[i] = io_mem_write[io_index][i];
3447 io_mem_read[io_index][i] = swapendian_readfn[i];
3448 io_mem_write[io_index][i] = swapendian_writefn[i];
3450 io_mem_opaque[io_index] = c;
3453 static void swapendian_del(int io_index)
3455 if (io_mem_read[io_index][0] == swapendian_readfn[0]) {
3456 qemu_free(io_mem_opaque[io_index]);
3460 /* mem_read and mem_write are arrays of functions containing the
3461 function to access byte (index 0), word (index 1) and dword (index
3462 2). Functions can be omitted with a NULL function pointer.
3463 If io_index is non zero, the corresponding io zone is
3464 modified. If it is zero, a new io zone is allocated. The return
3465 value can be used with cpu_register_physical_memory(). (-1) is
3466 returned if error. */
3467 static int cpu_register_io_memory_fixed(int io_index,
3468 CPUReadMemoryFunc * const *mem_read,
3469 CPUWriteMemoryFunc * const *mem_write,
3470 void *opaque, enum device_endian endian)
3472 int i;
3474 if (io_index <= 0) {
3475 io_index = get_free_io_mem_idx();
3476 if (io_index == -1)
3477 return io_index;
3478 } else {
3479 io_index >>= IO_MEM_SHIFT;
3480 if (io_index >= IO_MEM_NB_ENTRIES)
3481 return -1;
3484 for (i = 0; i < 3; ++i) {
3485 io_mem_read[io_index][i]
3486 = (mem_read[i] ? mem_read[i] : unassigned_mem_read[i]);
3488 for (i = 0; i < 3; ++i) {
3489 io_mem_write[io_index][i]
3490 = (mem_write[i] ? mem_write[i] : unassigned_mem_write[i]);
3492 io_mem_opaque[io_index] = opaque;
3494 switch (endian) {
3495 case DEVICE_BIG_ENDIAN:
3496 #ifndef TARGET_WORDS_BIGENDIAN
3497 swapendian_init(io_index);
3498 #endif
3499 break;
3500 case DEVICE_LITTLE_ENDIAN:
3501 #ifdef TARGET_WORDS_BIGENDIAN
3502 swapendian_init(io_index);
3503 #endif
3504 break;
3505 case DEVICE_NATIVE_ENDIAN:
3506 default:
3507 break;
3510 return (io_index << IO_MEM_SHIFT);
3513 int cpu_register_io_memory(CPUReadMemoryFunc * const *mem_read,
3514 CPUWriteMemoryFunc * const *mem_write,
3515 void *opaque, enum device_endian endian)
3517 return cpu_register_io_memory_fixed(0, mem_read, mem_write, opaque, endian);
3520 void cpu_unregister_io_memory(int io_table_address)
3522 int i;
3523 int io_index = io_table_address >> IO_MEM_SHIFT;
3525 swapendian_del(io_index);
3527 for (i=0;i < 3; i++) {
3528 io_mem_read[io_index][i] = unassigned_mem_read[i];
3529 io_mem_write[io_index][i] = unassigned_mem_write[i];
3531 io_mem_opaque[io_index] = NULL;
3532 io_mem_used[io_index] = 0;
3535 static void io_mem_init(void)
3537 int i;
3539 cpu_register_io_memory_fixed(IO_MEM_ROM, error_mem_read,
3540 unassigned_mem_write, NULL,
3541 DEVICE_NATIVE_ENDIAN);
3542 cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED, unassigned_mem_read,
3543 unassigned_mem_write, NULL,
3544 DEVICE_NATIVE_ENDIAN);
3545 cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY, error_mem_read,
3546 notdirty_mem_write, NULL,
3547 DEVICE_NATIVE_ENDIAN);
3548 for (i=0; i<5; i++)
3549 io_mem_used[i] = 1;
3551 io_mem_watch = cpu_register_io_memory(watch_mem_read,
3552 watch_mem_write, NULL,
3553 DEVICE_NATIVE_ENDIAN);
3556 #endif /* !defined(CONFIG_USER_ONLY) */
3558 /* physical memory access (slow version, mainly for debug) */
3559 #if defined(CONFIG_USER_ONLY)
3560 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3561 uint8_t *buf, int len, int is_write)
3563 int l, flags;
3564 target_ulong page;
3565 void * p;
3567 while (len > 0) {
3568 page = addr & TARGET_PAGE_MASK;
3569 l = (page + TARGET_PAGE_SIZE) - addr;
3570 if (l > len)
3571 l = len;
3572 flags = page_get_flags(page);
3573 if (!(flags & PAGE_VALID))
3574 return -1;
3575 if (is_write) {
3576 if (!(flags & PAGE_WRITE))
3577 return -1;
3578 /* XXX: this code should not depend on lock_user */
3579 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
3580 return -1;
3581 memcpy(p, buf, l);
3582 unlock_user(p, addr, l);
3583 } else {
3584 if (!(flags & PAGE_READ))
3585 return -1;
3586 /* XXX: this code should not depend on lock_user */
3587 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
3588 return -1;
3589 memcpy(buf, p, l);
3590 unlock_user(p, addr, 0);
3592 len -= l;
3593 buf += l;
3594 addr += l;
3596 return 0;
3599 #else
3600 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
3601 int len, int is_write)
3603 int l, io_index;
3604 uint8_t *ptr;
3605 uint32_t val;
3606 target_phys_addr_t page;
3607 unsigned long pd;
3608 PhysPageDesc *p;
3610 while (len > 0) {
3611 page = addr & TARGET_PAGE_MASK;
3612 l = (page + TARGET_PAGE_SIZE) - addr;
3613 if (l > len)
3614 l = len;
3615 p = phys_page_find(page >> TARGET_PAGE_BITS);
3616 if (!p) {
3617 pd = IO_MEM_UNASSIGNED;
3618 } else {
3619 pd = p->phys_offset;
3622 if (is_write) {
3623 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3624 target_phys_addr_t addr1 = addr;
3625 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3626 if (p)
3627 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3628 /* XXX: could force cpu_single_env to NULL to avoid
3629 potential bugs */
3630 if (l >= 4 && ((addr1 & 3) == 0)) {
3631 /* 32 bit write access */
3632 val = ldl_p(buf);
3633 io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
3634 l = 4;
3635 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3636 /* 16 bit write access */
3637 val = lduw_p(buf);
3638 io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
3639 l = 2;
3640 } else {
3641 /* 8 bit write access */
3642 val = ldub_p(buf);
3643 io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
3644 l = 1;
3646 } else {
3647 unsigned long addr1;
3648 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3649 /* RAM case */
3650 ptr = qemu_get_ram_ptr(addr1);
3651 memcpy(ptr, buf, l);
3652 if (!cpu_physical_memory_is_dirty(addr1)) {
3653 /* invalidate code */
3654 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3655 /* set dirty bit */
3656 cpu_physical_memory_set_dirty_flags(
3657 addr1, (0xff & ~CODE_DIRTY_FLAG));
3660 } else {
3661 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3662 !(pd & IO_MEM_ROMD)) {
3663 target_phys_addr_t addr1 = addr;
3664 /* I/O case */
3665 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3666 if (p)
3667 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3668 if (l >= 4 && ((addr1 & 3) == 0)) {
3669 /* 32 bit read access */
3670 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
3671 stl_p(buf, val);
3672 l = 4;
3673 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3674 /* 16 bit read access */
3675 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
3676 stw_p(buf, val);
3677 l = 2;
3678 } else {
3679 /* 8 bit read access */
3680 val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
3681 stb_p(buf, val);
3682 l = 1;
3684 } else {
3685 /* RAM case */
3686 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3687 (addr & ~TARGET_PAGE_MASK);
3688 memcpy(buf, ptr, l);
3691 len -= l;
3692 buf += l;
3693 addr += l;
3697 /* used for ROM loading : can write in RAM and ROM */
3698 void cpu_physical_memory_write_rom(target_phys_addr_t addr,
3699 const uint8_t *buf, int len)
3701 int l;
3702 uint8_t *ptr;
3703 target_phys_addr_t page;
3704 unsigned long pd;
3705 PhysPageDesc *p;
3707 while (len > 0) {
3708 page = addr & TARGET_PAGE_MASK;
3709 l = (page + TARGET_PAGE_SIZE) - addr;
3710 if (l > len)
3711 l = len;
3712 p = phys_page_find(page >> TARGET_PAGE_BITS);
3713 if (!p) {
3714 pd = IO_MEM_UNASSIGNED;
3715 } else {
3716 pd = p->phys_offset;
3719 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3720 (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3721 !(pd & IO_MEM_ROMD)) {
3722 /* do nothing */
3723 } else {
3724 unsigned long addr1;
3725 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3726 /* ROM/RAM case */
3727 ptr = qemu_get_ram_ptr(addr1);
3728 memcpy(ptr, buf, l);
3730 len -= l;
3731 buf += l;
3732 addr += l;
3736 typedef struct {
3737 void *buffer;
3738 target_phys_addr_t addr;
3739 target_phys_addr_t len;
3740 } BounceBuffer;
3742 static BounceBuffer bounce;
3744 typedef struct MapClient {
3745 void *opaque;
3746 void (*callback)(void *opaque);
3747 QLIST_ENTRY(MapClient) link;
3748 } MapClient;
3750 static QLIST_HEAD(map_client_list, MapClient) map_client_list
3751 = QLIST_HEAD_INITIALIZER(map_client_list);
3753 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
3755 MapClient *client = qemu_malloc(sizeof(*client));
3757 client->opaque = opaque;
3758 client->callback = callback;
3759 QLIST_INSERT_HEAD(&map_client_list, client, link);
3760 return client;
3763 void cpu_unregister_map_client(void *_client)
3765 MapClient *client = (MapClient *)_client;
3767 QLIST_REMOVE(client, link);
3768 qemu_free(client);
3771 static void cpu_notify_map_clients(void)
3773 MapClient *client;
3775 while (!QLIST_EMPTY(&map_client_list)) {
3776 client = QLIST_FIRST(&map_client_list);
3777 client->callback(client->opaque);
3778 cpu_unregister_map_client(client);
3782 /* Map a physical memory region into a host virtual address.
3783 * May map a subset of the requested range, given by and returned in *plen.
3784 * May return NULL if resources needed to perform the mapping are exhausted.
3785 * Use only for reads OR writes - not for read-modify-write operations.
3786 * Use cpu_register_map_client() to know when retrying the map operation is
3787 * likely to succeed.
3789 void *cpu_physical_memory_map(target_phys_addr_t addr,
3790 target_phys_addr_t *plen,
3791 int is_write)
3793 target_phys_addr_t len = *plen;
3794 target_phys_addr_t done = 0;
3795 int l;
3796 uint8_t *ret = NULL;
3797 uint8_t *ptr;
3798 target_phys_addr_t page;
3799 unsigned long pd;
3800 PhysPageDesc *p;
3801 unsigned long addr1;
3803 while (len > 0) {
3804 page = addr & TARGET_PAGE_MASK;
3805 l = (page + TARGET_PAGE_SIZE) - addr;
3806 if (l > len)
3807 l = len;
3808 p = phys_page_find(page >> TARGET_PAGE_BITS);
3809 if (!p) {
3810 pd = IO_MEM_UNASSIGNED;
3811 } else {
3812 pd = p->phys_offset;
3815 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3816 if (done || bounce.buffer) {
3817 break;
3819 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
3820 bounce.addr = addr;
3821 bounce.len = l;
3822 if (!is_write) {
3823 cpu_physical_memory_rw(addr, bounce.buffer, l, 0);
3825 ptr = bounce.buffer;
3826 } else {
3827 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3828 ptr = qemu_get_ram_ptr(addr1);
3830 if (!done) {
3831 ret = ptr;
3832 } else if (ret + done != ptr) {
3833 break;
3836 len -= l;
3837 addr += l;
3838 done += l;
3840 *plen = done;
3841 return ret;
3844 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3845 * Will also mark the memory as dirty if is_write == 1. access_len gives
3846 * the amount of memory that was actually read or written by the caller.
3848 void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
3849 int is_write, target_phys_addr_t access_len)
3851 if (buffer != bounce.buffer) {
3852 if (is_write) {
3853 ram_addr_t addr1 = qemu_ram_addr_from_host_nofail(buffer);
3854 while (access_len) {
3855 unsigned l;
3856 l = TARGET_PAGE_SIZE;
3857 if (l > access_len)
3858 l = access_len;
3859 if (!cpu_physical_memory_is_dirty(addr1)) {
3860 /* invalidate code */
3861 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3862 /* set dirty bit */
3863 cpu_physical_memory_set_dirty_flags(
3864 addr1, (0xff & ~CODE_DIRTY_FLAG));
3866 addr1 += l;
3867 access_len -= l;
3870 return;
3872 if (is_write) {
3873 cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
3875 qemu_vfree(bounce.buffer);
3876 bounce.buffer = NULL;
3877 cpu_notify_map_clients();
3880 /* warning: addr must be aligned */
3881 uint32_t ldl_phys(target_phys_addr_t addr)
3883 int io_index;
3884 uint8_t *ptr;
3885 uint32_t val;
3886 unsigned long pd;
3887 PhysPageDesc *p;
3889 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3890 if (!p) {
3891 pd = IO_MEM_UNASSIGNED;
3892 } else {
3893 pd = p->phys_offset;
3896 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3897 !(pd & IO_MEM_ROMD)) {
3898 /* I/O case */
3899 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3900 if (p)
3901 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3902 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3903 } else {
3904 /* RAM case */
3905 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3906 (addr & ~TARGET_PAGE_MASK);
3907 val = ldl_p(ptr);
3909 return val;
3912 /* warning: addr must be aligned */
3913 uint64_t ldq_phys(target_phys_addr_t addr)
3915 int io_index;
3916 uint8_t *ptr;
3917 uint64_t val;
3918 unsigned long pd;
3919 PhysPageDesc *p;
3921 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3922 if (!p) {
3923 pd = IO_MEM_UNASSIGNED;
3924 } else {
3925 pd = p->phys_offset;
3928 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3929 !(pd & IO_MEM_ROMD)) {
3930 /* I/O case */
3931 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3932 if (p)
3933 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3934 #ifdef TARGET_WORDS_BIGENDIAN
3935 val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
3936 val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
3937 #else
3938 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3939 val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
3940 #endif
3941 } else {
3942 /* RAM case */
3943 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3944 (addr & ~TARGET_PAGE_MASK);
3945 val = ldq_p(ptr);
3947 return val;
3950 /* XXX: optimize */
3951 uint32_t ldub_phys(target_phys_addr_t addr)
3953 uint8_t val;
3954 cpu_physical_memory_read(addr, &val, 1);
3955 return val;
3958 /* warning: addr must be aligned */
3959 uint32_t lduw_phys(target_phys_addr_t addr)
3961 int io_index;
3962 uint8_t *ptr;
3963 uint64_t val;
3964 unsigned long pd;
3965 PhysPageDesc *p;
3967 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3968 if (!p) {
3969 pd = IO_MEM_UNASSIGNED;
3970 } else {
3971 pd = p->phys_offset;
3974 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3975 !(pd & IO_MEM_ROMD)) {
3976 /* I/O case */
3977 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3978 if (p)
3979 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3980 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr);
3981 } else {
3982 /* RAM case */
3983 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3984 (addr & ~TARGET_PAGE_MASK);
3985 val = lduw_p(ptr);
3987 return val;
3990 /* warning: addr must be aligned. The ram page is not masked as dirty
3991 and the code inside is not invalidated. It is useful if the dirty
3992 bits are used to track modified PTEs */
3993 void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
3995 int io_index;
3996 uint8_t *ptr;
3997 unsigned long pd;
3998 PhysPageDesc *p;
4000 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4001 if (!p) {
4002 pd = IO_MEM_UNASSIGNED;
4003 } else {
4004 pd = p->phys_offset;
4007 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4008 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4009 if (p)
4010 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4011 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4012 } else {
4013 unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4014 ptr = qemu_get_ram_ptr(addr1);
4015 stl_p(ptr, val);
4017 if (unlikely(in_migration)) {
4018 if (!cpu_physical_memory_is_dirty(addr1)) {
4019 /* invalidate code */
4020 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
4021 /* set dirty bit */
4022 cpu_physical_memory_set_dirty_flags(
4023 addr1, (0xff & ~CODE_DIRTY_FLAG));
4029 void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
4031 int io_index;
4032 uint8_t *ptr;
4033 unsigned long pd;
4034 PhysPageDesc *p;
4036 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4037 if (!p) {
4038 pd = IO_MEM_UNASSIGNED;
4039 } else {
4040 pd = p->phys_offset;
4043 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4044 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4045 if (p)
4046 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4047 #ifdef TARGET_WORDS_BIGENDIAN
4048 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
4049 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
4050 #else
4051 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4052 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
4053 #endif
4054 } else {
4055 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
4056 (addr & ~TARGET_PAGE_MASK);
4057 stq_p(ptr, val);
4061 /* warning: addr must be aligned */
4062 void stl_phys(target_phys_addr_t addr, uint32_t val)
4064 int io_index;
4065 uint8_t *ptr;
4066 unsigned long pd;
4067 PhysPageDesc *p;
4069 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4070 if (!p) {
4071 pd = IO_MEM_UNASSIGNED;
4072 } else {
4073 pd = p->phys_offset;
4076 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4077 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4078 if (p)
4079 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4080 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4081 } else {
4082 unsigned long addr1;
4083 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4084 /* RAM case */
4085 ptr = qemu_get_ram_ptr(addr1);
4086 stl_p(ptr, val);
4087 if (!cpu_physical_memory_is_dirty(addr1)) {
4088 /* invalidate code */
4089 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
4090 /* set dirty bit */
4091 cpu_physical_memory_set_dirty_flags(addr1,
4092 (0xff & ~CODE_DIRTY_FLAG));
4097 /* XXX: optimize */
4098 void stb_phys(target_phys_addr_t addr, uint32_t val)
4100 uint8_t v = val;
4101 cpu_physical_memory_write(addr, &v, 1);
4104 /* warning: addr must be aligned */
4105 void stw_phys(target_phys_addr_t addr, uint32_t val)
4107 int io_index;
4108 uint8_t *ptr;
4109 unsigned long pd;
4110 PhysPageDesc *p;
4112 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4113 if (!p) {
4114 pd = IO_MEM_UNASSIGNED;
4115 } else {
4116 pd = p->phys_offset;
4119 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4120 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4121 if (p)
4122 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4123 io_mem_write[io_index][1](io_mem_opaque[io_index], addr, val);
4124 } else {
4125 unsigned long addr1;
4126 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4127 /* RAM case */
4128 ptr = qemu_get_ram_ptr(addr1);
4129 stw_p(ptr, val);
4130 if (!cpu_physical_memory_is_dirty(addr1)) {
4131 /* invalidate code */
4132 tb_invalidate_phys_page_range(addr1, addr1 + 2, 0);
4133 /* set dirty bit */
4134 cpu_physical_memory_set_dirty_flags(addr1,
4135 (0xff & ~CODE_DIRTY_FLAG));
4140 /* XXX: optimize */
4141 void stq_phys(target_phys_addr_t addr, uint64_t val)
4143 val = tswap64(val);
4144 cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
4147 /* virtual memory access for debug (includes writing to ROM) */
4148 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
4149 uint8_t *buf, int len, int is_write)
4151 int l;
4152 target_phys_addr_t phys_addr;
4153 target_ulong page;
4155 while (len > 0) {
4156 page = addr & TARGET_PAGE_MASK;
4157 phys_addr = cpu_get_phys_page_debug(env, page);
4158 /* if no physical page mapped, return an error */
4159 if (phys_addr == -1)
4160 return -1;
4161 l = (page + TARGET_PAGE_SIZE) - addr;
4162 if (l > len)
4163 l = len;
4164 phys_addr += (addr & ~TARGET_PAGE_MASK);
4165 if (is_write)
4166 cpu_physical_memory_write_rom(phys_addr, buf, l);
4167 else
4168 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
4169 len -= l;
4170 buf += l;
4171 addr += l;
4173 return 0;
4175 #endif
4177 /* in deterministic execution mode, instructions doing device I/Os
4178 must be at the end of the TB */
4179 void cpu_io_recompile(CPUState *env, void *retaddr)
4181 TranslationBlock *tb;
4182 uint32_t n, cflags;
4183 target_ulong pc, cs_base;
4184 uint64_t flags;
4186 tb = tb_find_pc((unsigned long)retaddr);
4187 if (!tb) {
4188 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
4189 retaddr);
4191 n = env->icount_decr.u16.low + tb->icount;
4192 cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
4193 /* Calculate how many instructions had been executed before the fault
4194 occurred. */
4195 n = n - env->icount_decr.u16.low;
4196 /* Generate a new TB ending on the I/O insn. */
4197 n++;
4198 /* On MIPS and SH, delay slot instructions can only be restarted if
4199 they were already the first instruction in the TB. If this is not
4200 the first instruction in a TB then re-execute the preceding
4201 branch. */
4202 #if defined(TARGET_MIPS)
4203 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
4204 env->active_tc.PC -= 4;
4205 env->icount_decr.u16.low++;
4206 env->hflags &= ~MIPS_HFLAG_BMASK;
4208 #elif defined(TARGET_SH4)
4209 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
4210 && n > 1) {
4211 env->pc -= 2;
4212 env->icount_decr.u16.low++;
4213 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
4215 #endif
4216 /* This should never happen. */
4217 if (n > CF_COUNT_MASK)
4218 cpu_abort(env, "TB too big during recompile");
4220 cflags = n | CF_LAST_IO;
4221 pc = tb->pc;
4222 cs_base = tb->cs_base;
4223 flags = tb->flags;
4224 tb_phys_invalidate(tb, -1);
4225 /* FIXME: In theory this could raise an exception. In practice
4226 we have already translated the block once so it's probably ok. */
4227 tb_gen_code(env, pc, cs_base, flags, cflags);
4228 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
4229 the first in the TB) then we end up generating a whole new TB and
4230 repeating the fault, which is horribly inefficient.
4231 Better would be to execute just this insn uncached, or generate a
4232 second new TB. */
4233 cpu_resume_from_signal(env, NULL);
4236 #if !defined(CONFIG_USER_ONLY)
4238 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
4240 int i, target_code_size, max_target_code_size;
4241 int direct_jmp_count, direct_jmp2_count, cross_page;
4242 TranslationBlock *tb;
4244 target_code_size = 0;
4245 max_target_code_size = 0;
4246 cross_page = 0;
4247 direct_jmp_count = 0;
4248 direct_jmp2_count = 0;
4249 for(i = 0; i < nb_tbs; i++) {
4250 tb = &tbs[i];
4251 target_code_size += tb->size;
4252 if (tb->size > max_target_code_size)
4253 max_target_code_size = tb->size;
4254 if (tb->page_addr[1] != -1)
4255 cross_page++;
4256 if (tb->tb_next_offset[0] != 0xffff) {
4257 direct_jmp_count++;
4258 if (tb->tb_next_offset[1] != 0xffff) {
4259 direct_jmp2_count++;
4263 /* XXX: avoid using doubles ? */
4264 cpu_fprintf(f, "Translation buffer state:\n");
4265 cpu_fprintf(f, "gen code size %td/%ld\n",
4266 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
4267 cpu_fprintf(f, "TB count %d/%d\n",
4268 nb_tbs, code_gen_max_blocks);
4269 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
4270 nb_tbs ? target_code_size / nb_tbs : 0,
4271 max_target_code_size);
4272 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
4273 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
4274 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
4275 cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
4276 cross_page,
4277 nb_tbs ? (cross_page * 100) / nb_tbs : 0);
4278 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
4279 direct_jmp_count,
4280 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
4281 direct_jmp2_count,
4282 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
4283 cpu_fprintf(f, "\nStatistics:\n");
4284 cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
4285 cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
4286 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
4287 tcg_dump_info(f, cpu_fprintf);
4290 #define MMUSUFFIX _cmmu
4291 #define GETPC() NULL
4292 #define env cpu_single_env
4293 #define SOFTMMU_CODE_ACCESS
4295 #define SHIFT 0
4296 #include "softmmu_template.h"
4298 #define SHIFT 1
4299 #include "softmmu_template.h"
4301 #define SHIFT 2
4302 #include "softmmu_template.h"
4304 #define SHIFT 3
4305 #include "softmmu_template.h"
4307 #undef env
4309 #endif