?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
test: switch 64-bit smp startup to real APIC
[qemu-kvm/amd-iommu.git] / exec.c
blobdd8881cfe62c95061ae908303952ca60268c13b2
1 /*
2 * virtual page mapping and translated block handling
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26 #include <stdlib.h>
27 #include <stdio.h>
28 #include <stdarg.h>
29 #include <string.h>
30 #include <errno.h>
31 #include <unistd.h>
32 #include <inttypes.h>
33
34 #include "cpu.h"
35 #include "exec-all.h"
36 #include "qemu-common.h"
37 #include "cache-utils.h"
38
39 #if !defined(TARGET_IA64)
40 #include "tcg.h"
41 #endif
42 #include "qemu-kvm.h"
43
44 #include "hw/hw.h"
45 #include "osdep.h"
46 #include "kvm.h"
47 #if defined(CONFIG_USER_ONLY)
48 #include <qemu.h>
49 #endif
50
51 //#define DEBUG_TB_INVALIDATE
52 //#define DEBUG_FLUSH
53 //#define DEBUG_TLB
54 //#define DEBUG_UNASSIGNED
55
56 /* make various TB consistency checks */
57 //#define DEBUG_TB_CHECK
58 //#define DEBUG_TLB_CHECK
59
60 //#define DEBUG_IOPORT
61 //#define DEBUG_SUBPAGE
62
63 #if !defined(CONFIG_USER_ONLY)
64 /* TB consistency checks only implemented for usermode emulation. */
65 #undef DEBUG_TB_CHECK
66 #endif
67
68 #define SMC_BITMAP_USE_THRESHOLD 10
69
70 #if defined(TARGET_SPARC64)
71 #define TARGET_PHYS_ADDR_SPACE_BITS 41
72 #elif defined(TARGET_SPARC)
73 #define TARGET_PHYS_ADDR_SPACE_BITS 36
74 #elif defined(TARGET_ALPHA)
75 #define TARGET_PHYS_ADDR_SPACE_BITS 42
76 #define TARGET_VIRT_ADDR_SPACE_BITS 42
77 #elif defined(TARGET_PPC64)
78 #define TARGET_PHYS_ADDR_SPACE_BITS 42
79 #elif defined(TARGET_X86_64)
80 #define TARGET_PHYS_ADDR_SPACE_BITS 42
81 #elif defined(TARGET_I386)
82 #define TARGET_PHYS_ADDR_SPACE_BITS 36
83 #elif defined(TARGET_IA64)
84 #define TARGET_PHYS_ADDR_SPACE_BITS 36
85 #else
86 #define TARGET_PHYS_ADDR_SPACE_BITS 32
87 #endif
88
89 static TranslationBlock *tbs;
90 int code_gen_max_blocks;
91 TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
92 static int nb_tbs;
93 /* any access to the tbs or the page table must use this lock */
94 spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
95
96 #if defined(__arm__) || defined(__sparc_v9__)
97 /* The prologue must be reachable with a direct jump. ARM and Sparc64
98 have limited branch ranges (possibly also PPC) so place it in a
99 section close to code segment. */
100 #define code_gen_section \
101 __attribute__((__section__(".gen_code"))) \
102 __attribute__((aligned (32)))
103 #elif defined(_WIN32)
104 /* Maximum alignment for Win32 is 16. */
105 #define code_gen_section \
106 __attribute__((aligned (16)))
107 #else
108 #define code_gen_section \
109 __attribute__((aligned (32)))
110 #endif
111
112 uint8_t code_gen_prologue[1024] code_gen_section;
113 static uint8_t *code_gen_buffer;
114 static unsigned long code_gen_buffer_size;
115 /* threshold to flush the translated code buffer */
116 static unsigned long code_gen_buffer_max_size;
117 uint8_t *code_gen_ptr;
118
119 #if !defined(CONFIG_USER_ONLY)
120 int phys_ram_fd;
121 uint8_t *phys_ram_dirty;
122 uint8_t *bios_mem;
123 static int in_migration;
124
125 typedef struct RAMBlock {
126 uint8_t *host;
127 ram_addr_t offset;
128 ram_addr_t length;
129 struct RAMBlock *next;
130 } RAMBlock;
131
132 static RAMBlock *ram_blocks;
133 /* TODO: When we implement (and use) ram deallocation (e.g. for hotplug)
134 then we can no longer assume contiguous ram offsets, and external uses
135 of this variable will break. */
136 ram_addr_t last_ram_offset;
137 #endif
138
139 CPUState *first_cpu;
140 /* current CPU in the current thread. It is only valid inside
141 cpu_exec() */
142 CPUState *cpu_single_env;
143 /* 0 = Do not count executed instructions.
144 1 = Precise instruction counting.
145 2 = Adaptive rate instruction counting. */
146 int use_icount = 0;
147 /* Current instruction counter. While executing translated code this may
148 include some instructions that have not yet been executed. */
149 int64_t qemu_icount;
150
151 typedef struct PageDesc {
152 /* list of TBs intersecting this ram page */
153 TranslationBlock *first_tb;
154 /* in order to optimize self modifying code, we count the number
155 of lookups we do to a given page to use a bitmap */
156 unsigned int code_write_count;
157 uint8_t *code_bitmap;
158 #if defined(CONFIG_USER_ONLY)
159 unsigned long flags;
160 #endif
161 } PageDesc;
162
163 typedef struct PhysPageDesc {
164 /* offset in host memory of the page + io_index in the low bits */
165 ram_addr_t phys_offset;
166 ram_addr_t region_offset;
167 } PhysPageDesc;
168
169 #define L2_BITS 10
170 #if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
171 /* XXX: this is a temporary hack for alpha target.
172 * In the future, this is to be replaced by a multi-level table
173 * to actually be able to handle the complete 64 bits address space.
174 */
175 #define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
176 #else
177 #define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
178 #endif
179
180 #define L1_SIZE (1 << L1_BITS)
181 #define L2_SIZE (1 << L2_BITS)
182
183 unsigned long qemu_real_host_page_size;
184 unsigned long qemu_host_page_bits;
185 unsigned long qemu_host_page_size;
186 unsigned long qemu_host_page_mask;
187
188 /* XXX: for system emulation, it could just be an array */
189 static PageDesc *l1_map[L1_SIZE];
190 static PhysPageDesc **l1_phys_map;
191
192 #if !defined(CONFIG_USER_ONLY)
193 static void io_mem_init(void);
194
195 /* io memory support */
196 CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
197 CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
198 void *io_mem_opaque[IO_MEM_NB_ENTRIES];
199 static char io_mem_used[IO_MEM_NB_ENTRIES];
200 static int io_mem_watch;
201 #endif
202
203 /* log support */
204 static const char *logfilename = "/tmp/qemu.log";
205 FILE *logfile;
206 int loglevel;
207 static int log_append = 0;
208
209 /* statistics */
210 static int tlb_flush_count;
211 static int tb_flush_count;
212 static int tb_phys_invalidate_count;
213
214 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
215 typedef struct subpage_t {
216 target_phys_addr_t base;
217 CPUReadMemoryFunc * const *mem_read[TARGET_PAGE_SIZE][4];
218 CPUWriteMemoryFunc * const *mem_write[TARGET_PAGE_SIZE][4];
219 void *opaque[TARGET_PAGE_SIZE][2][4];
220 ram_addr_t region_offset[TARGET_PAGE_SIZE][2][4];
221 } subpage_t;
222
223 #ifdef _WIN32
224 static void map_exec(void *addr, long size)
225 {
226 DWORD old_protect;
227 VirtualProtect(addr, size,
228 PAGE_EXECUTE_READWRITE, &old_protect);
229
230 }
231 #else
232 static void map_exec(void *addr, long size)
233 {
234 unsigned long start, end, page_size;
235
236 page_size = getpagesize();
237 start = (unsigned long)addr;
238 start &= ~(page_size - 1);
239
240 end = (unsigned long)addr + size;
241 end += page_size - 1;
242 end &= ~(page_size - 1);
243
244 mprotect((void *)start, end - start,
245 PROT_READ | PROT_WRITE | PROT_EXEC);
246 }
247 #endif
248
249 static void page_init(void)
250 {
251 /* NOTE: we can always suppose that qemu_host_page_size >=
252 TARGET_PAGE_SIZE */
253 #ifdef _WIN32
254 {
255 SYSTEM_INFO system_info;
256
257 GetSystemInfo(&system_info);
258 qemu_real_host_page_size = system_info.dwPageSize;
259 }
260 #else
261 qemu_real_host_page_size = getpagesize();
262 #endif
263 if (qemu_host_page_size == 0)
264 qemu_host_page_size = qemu_real_host_page_size;
265 if (qemu_host_page_size < TARGET_PAGE_SIZE)
266 qemu_host_page_size = TARGET_PAGE_SIZE;
267 qemu_host_page_bits = 0;
268 while ((1 << qemu_host_page_bits) < qemu_host_page_size)
269 qemu_host_page_bits++;
270 qemu_host_page_mask = ~(qemu_host_page_size - 1);
271 l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));
272 memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));
273
274 #if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
275 {
276 long long startaddr, endaddr;
277 FILE *f;
278 int n;
279
280 mmap_lock();
281 last_brk = (unsigned long)sbrk(0);
282 f = fopen("/proc/self/maps", "r");
283 if (f) {
284 do {
285 n = fscanf (f, "%llx-%llx %*[^\n]\n", &startaddr, &endaddr);
286 if (n == 2) {
287 startaddr = MIN(startaddr,
288 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
289 endaddr = MIN(endaddr,
290 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
291 page_set_flags(startaddr & TARGET_PAGE_MASK,
292 TARGET_PAGE_ALIGN(endaddr),
293 PAGE_RESERVED);
294 }
295 } while (!feof(f));
296 fclose(f);
297 }
298 mmap_unlock();
299 }
300 #endif
301 }
302
303 static inline PageDesc **page_l1_map(target_ulong index)
304 {
305 #if TARGET_LONG_BITS > 32
306 /* Host memory outside guest VM. For 32-bit targets we have already
307 excluded high addresses. */
308 if (index > ((target_ulong)L2_SIZE * L1_SIZE))
309 return NULL;
310 #endif
311 return &l1_map[index >> L2_BITS];
312 }
313
314 static inline PageDesc *page_find_alloc(target_ulong index)
315 {
316 PageDesc **lp, *p;
317 lp = page_l1_map(index);
318 if (!lp)
319 return NULL;
320
321 p = *lp;
322 if (!p) {
323 /* allocate if not found */
324 #if defined(CONFIG_USER_ONLY)
325 size_t len = sizeof(PageDesc) * L2_SIZE;
326 /* Don't use qemu_malloc because it may recurse. */
327 p = mmap(NULL, len, PROT_READ | PROT_WRITE,
328 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
329 *lp = p;
330 if (h2g_valid(p)) {
331 unsigned long addr = h2g(p);
332 page_set_flags(addr & TARGET_PAGE_MASK,
333 TARGET_PAGE_ALIGN(addr + len),
334 PAGE_RESERVED);
335 }
336 #else
337 p = qemu_mallocz(sizeof(PageDesc) * L2_SIZE);
338 *lp = p;
339 #endif
340 }
341 return p + (index & (L2_SIZE - 1));
342 }
343
344 static inline PageDesc *page_find(target_ulong index)
345 {
346 PageDesc **lp, *p;
347 lp = page_l1_map(index);
348 if (!lp)
349 return NULL;
350
351 p = *lp;
352 if (!p) {
353 return NULL;
354 }
355 return p + (index & (L2_SIZE - 1));
356 }
357
358 static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
359 {
360 void **lp, **p;
361 PhysPageDesc *pd;
362
363 p = (void **)l1_phys_map;
364 #if TARGET_PHYS_ADDR_SPACE_BITS > 32
365
366 #if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
367 #error unsupported TARGET_PHYS_ADDR_SPACE_BITS
368 #endif
369 lp = p + ((index >> (L1_BITS + L2_BITS)) & (L1_SIZE - 1));
370 p = *lp;
371 if (!p) {
372 /* allocate if not found */
373 if (!alloc)
374 return NULL;
375 p = qemu_vmalloc(sizeof(void *) * L1_SIZE);
376 memset(p, 0, sizeof(void *) * L1_SIZE);
377 *lp = p;
378 }
379 #endif
380 lp = p + ((index >> L2_BITS) & (L1_SIZE - 1));
381 pd = *lp;
382 if (!pd) {
383 int i;
384 /* allocate if not found */
385 if (!alloc)
386 return NULL;
387 pd = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);
388 *lp = pd;
389 for (i = 0; i < L2_SIZE; i++) {
390 pd[i].phys_offset = IO_MEM_UNASSIGNED;
391 pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
392 }
393 }
394 return ((PhysPageDesc *)pd) + (index & (L2_SIZE - 1));
395 }
396
397 static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
398 {
399 return phys_page_find_alloc(index, 0);
400 }
401
402 #if !defined(CONFIG_USER_ONLY)
403 static void tlb_protect_code(ram_addr_t ram_addr);
404 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
405 target_ulong vaddr);
406 #define mmap_lock() do { } while(0)
407 #define mmap_unlock() do { } while(0)
408 #endif
409
410 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
411
412 #if defined(CONFIG_USER_ONLY)
413 /* Currently it is not recommended to allocate big chunks of data in
414 user mode. It will change when a dedicated libc will be used */
415 #define USE_STATIC_CODE_GEN_BUFFER
416 #endif
417
418 #ifdef USE_STATIC_CODE_GEN_BUFFER
419 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE];
420 #endif
421
422 static void code_gen_alloc(unsigned long tb_size)
423 {
424 if (kvm_enabled())
425 return;
426
427 #ifdef USE_STATIC_CODE_GEN_BUFFER
428 code_gen_buffer = static_code_gen_buffer;
429 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
430 map_exec(code_gen_buffer, code_gen_buffer_size);
431 #else
432 code_gen_buffer_size = tb_size;
433 if (code_gen_buffer_size == 0) {
434 #if defined(CONFIG_USER_ONLY)
435 /* in user mode, phys_ram_size is not meaningful */
436 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
437 #else
438 /* XXX: needs adjustments */
439 code_gen_buffer_size = (unsigned long)(ram_size / 4);
440 #endif
441 }
442 if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
443 code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
444 /* The code gen buffer location may have constraints depending on
445 the host cpu and OS */
446 #if defined(__linux__)
447 {
448 int flags;
449 void *start = NULL;
450
451 flags = MAP_PRIVATE | MAP_ANONYMOUS;
452 #if defined(__x86_64__)
453 flags |= MAP_32BIT;
454 /* Cannot map more than that */
455 if (code_gen_buffer_size > (800 * 1024 * 1024))
456 code_gen_buffer_size = (800 * 1024 * 1024);
457 #elif defined(__sparc_v9__)
458 // Map the buffer below 2G, so we can use direct calls and branches
459 flags |= MAP_FIXED;
460 start = (void *) 0x60000000UL;
461 if (code_gen_buffer_size > (512 * 1024 * 1024))
462 code_gen_buffer_size = (512 * 1024 * 1024);
463 #elif defined(__arm__)
464 /* Map the buffer below 32M, so we can use direct calls and branches */
465 flags |= MAP_FIXED;
466 start = (void *) 0x01000000UL;
467 if (code_gen_buffer_size > 16 * 1024 * 1024)
468 code_gen_buffer_size = 16 * 1024 * 1024;
469 #endif
470 code_gen_buffer = mmap(start, code_gen_buffer_size,
471 PROT_WRITE | PROT_READ | PROT_EXEC,
472 flags, -1, 0);
473 if (code_gen_buffer == MAP_FAILED) {
474 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
475 exit(1);
476 }
477 }
478 #elif defined(__FreeBSD__) || defined(__DragonFly__)
479 {
480 int flags;
481 void *addr = NULL;
482 flags = MAP_PRIVATE | MAP_ANONYMOUS;
483 #if defined(__x86_64__)
484 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
485 * 0x40000000 is free */
486 flags |= MAP_FIXED;
487 addr = (void *)0x40000000;
488 /* Cannot map more than that */
489 if (code_gen_buffer_size > (800 * 1024 * 1024))
490 code_gen_buffer_size = (800 * 1024 * 1024);
491 #endif
492 code_gen_buffer = mmap(addr, code_gen_buffer_size,
493 PROT_WRITE | PROT_READ | PROT_EXEC,
494 flags, -1, 0);
495 if (code_gen_buffer == MAP_FAILED) {
496 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
497 exit(1);
498 }
499 }
500 #else
501 code_gen_buffer = qemu_malloc(code_gen_buffer_size);
502 map_exec(code_gen_buffer, code_gen_buffer_size);
503 #endif
504 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
505 map_exec(code_gen_prologue, sizeof(code_gen_prologue));
506 code_gen_buffer_max_size = code_gen_buffer_size -
507 code_gen_max_block_size();
508 code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
509 tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
510 }
511
512 /* Must be called before using the QEMU cpus. 'tb_size' is the size
513 (in bytes) allocated to the translation buffer. Zero means default
514 size. */
515 void cpu_exec_init_all(unsigned long tb_size)
516 {
517 cpu_gen_init();
518 code_gen_alloc(tb_size);
519 code_gen_ptr = code_gen_buffer;
520 page_init();
521 #if !defined(CONFIG_USER_ONLY)
522 io_mem_init();
523 #endif
524 }
525
526 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
527
528 static void cpu_common_pre_save(const void *opaque)
529 {
530 CPUState *env = (void *)opaque;
531
532 cpu_synchronize_state(env);
533 }
534
535 static int cpu_common_pre_load(void *opaque)
536 {
537 CPUState *env = opaque;
538
539 cpu_synchronize_state(env);
540 return 0;
541 }
542
543 static int cpu_common_post_load(void *opaque)
544 {
545 CPUState *env = opaque;
546
547 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
548 version_id is increased. */
549 env->interrupt_request &= ~0x01;
550 tlb_flush(env, 1);
551
552 return 0;
553 }
554
555 static const VMStateDescription vmstate_cpu_common = {
556 .name = "cpu_common",
557 .version_id = 1,
558 .minimum_version_id = 1,
559 .minimum_version_id_old = 1,
560 .pre_save = cpu_common_pre_save,
561 .pre_load = cpu_common_pre_load,
562 .post_load = cpu_common_post_load,
563 .fields = (VMStateField []) {
564 VMSTATE_UINT32(halted, CPUState),
565 VMSTATE_UINT32(interrupt_request, CPUState),
566 VMSTATE_END_OF_LIST()
567 }
568 };
569 #endif
570
571 CPUState *qemu_get_cpu(int cpu)
572 {
573 CPUState *env = first_cpu;
574
575 while (env) {
576 if (env->cpu_index == cpu)
577 break;
578 env = env->next_cpu;
579 }
580
581 return env;
582 }
583
584 void cpu_exec_init(CPUState *env)
585 {
586 CPUState **penv;
587 int cpu_index;
588
589 #if defined(CONFIG_USER_ONLY)
590 cpu_list_lock();
591 #endif
592 env->next_cpu = NULL;
593 penv = &first_cpu;
594 cpu_index = 0;
595 while (*penv != NULL) {
596 penv = &(*penv)->next_cpu;
597 cpu_index++;
598 }
599 env->cpu_index = cpu_index;
600 env->numa_node = 0;
601 QTAILQ_INIT(&env->breakpoints);
602 QTAILQ_INIT(&env->watchpoints);
603 #ifdef __WIN32
604 env->thread_id = GetCurrentProcessId();
605 #else
606 env->thread_id = getpid();
607 #endif
608 *penv = env;
609 #if defined(CONFIG_USER_ONLY)
610 cpu_list_unlock();
611 #endif
612 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
613 vmstate_register(cpu_index, &vmstate_cpu_common, env);
614 register_savevm("cpu", cpu_index, CPU_SAVE_VERSION,
615 cpu_save, cpu_load, env);
616 #endif
617 }
618
619 static inline void invalidate_page_bitmap(PageDesc *p)
620 {
621 if (p->code_bitmap) {
622 qemu_free(p->code_bitmap);
623 p->code_bitmap = NULL;
624 }
625 p->code_write_count = 0;
626 }
627
628 /* set to NULL all the 'first_tb' fields in all PageDescs */
629 static void page_flush_tb(void)
630 {
631 int i, j;
632 PageDesc *p;
633
634 for(i = 0; i < L1_SIZE; i++) {
635 p = l1_map[i];
636 if (p) {
637 for(j = 0; j < L2_SIZE; j++) {
638 p->first_tb = NULL;
639 invalidate_page_bitmap(p);
640 p++;
641 }
642 }
643 }
644 }
645
646 /* flush all the translation blocks */
647 /* XXX: tb_flush is currently not thread safe */
648 void tb_flush(CPUState *env1)
649 {
650 CPUState *env;
651 #if defined(DEBUG_FLUSH)
652 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
653 (unsigned long)(code_gen_ptr - code_gen_buffer),
654 nb_tbs, nb_tbs > 0 ?
655 ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
656 #endif
657 if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
658 cpu_abort(env1, "Internal error: code buffer overflow\n");
659
660 nb_tbs = 0;
661
662 for(env = first_cpu; env != NULL; env = env->next_cpu) {
663 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
664 }
665
666 memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
667 page_flush_tb();
668
669 code_gen_ptr = code_gen_buffer;
670 /* XXX: flush processor icache at this point if cache flush is
671 expensive */
672 tb_flush_count++;
673 }
674
675 #ifdef DEBUG_TB_CHECK
676
677 static void tb_invalidate_check(target_ulong address)
678 {
679 TranslationBlock *tb;
680 int i;
681 address &= TARGET_PAGE_MASK;
682 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
683 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
684 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
685 address >= tb->pc + tb->size)) {
686 printf("ERROR invalidate: address=" TARGET_FMT_lx
687 " PC=%08lx size=%04x\n",
688 address, (long)tb->pc, tb->size);
689 }
690 }
691 }
692 }
693
694 /* verify that all the pages have correct rights for code */
695 static void tb_page_check(void)
696 {
697 TranslationBlock *tb;
698 int i, flags1, flags2;
699
700 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
701 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
702 flags1 = page_get_flags(tb->pc);
703 flags2 = page_get_flags(tb->pc + tb->size - 1);
704 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
705 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
706 (long)tb->pc, tb->size, flags1, flags2);
707 }
708 }
709 }
710 }
711
712 #endif
713
714 /* invalidate one TB */
715 static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
716 int next_offset)
717 {
718 TranslationBlock *tb1;
719 for(;;) {
720 tb1 = *ptb;
721 if (tb1 == tb) {
722 *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
723 break;
724 }
725 ptb = (TranslationBlock **)((char *)tb1 + next_offset);
726 }
727 }
728
729 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
730 {
731 TranslationBlock *tb1;
732 unsigned int n1;
733
734 for(;;) {
735 tb1 = *ptb;
736 n1 = (long)tb1 & 3;
737 tb1 = (TranslationBlock *)((long)tb1 & ~3);
738 if (tb1 == tb) {
739 *ptb = tb1->page_next[n1];
740 break;
741 }
742 ptb = &tb1->page_next[n1];
743 }
744 }
745
746 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
747 {
748 TranslationBlock *tb1, **ptb;
749 unsigned int n1;
750
751 ptb = &tb->jmp_next[n];
752 tb1 = *ptb;
753 if (tb1) {
754 /* find tb(n) in circular list */
755 for(;;) {
756 tb1 = *ptb;
757 n1 = (long)tb1 & 3;
758 tb1 = (TranslationBlock *)((long)tb1 & ~3);
759 if (n1 == n && tb1 == tb)
760 break;
761 if (n1 == 2) {
762 ptb = &tb1->jmp_first;
763 } else {
764 ptb = &tb1->jmp_next[n1];
765 }
766 }
767 /* now we can suppress tb(n) from the list */
768 *ptb = tb->jmp_next[n];
769
770 tb->jmp_next[n] = NULL;
771 }
772 }
773
774 /* reset the jump entry 'n' of a TB so that it is not chained to
775 another TB */
776 static inline void tb_reset_jump(TranslationBlock *tb, int n)
777 {
778 tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
779 }
780
781 void tb_phys_invalidate(TranslationBlock *tb, target_ulong page_addr)
782 {
783 CPUState *env;
784 PageDesc *p;
785 unsigned int h, n1;
786 target_phys_addr_t phys_pc;
787 TranslationBlock *tb1, *tb2;
788
789 /* remove the TB from the hash list */
790 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
791 h = tb_phys_hash_func(phys_pc);
792 tb_remove(&tb_phys_hash[h], tb,
793 offsetof(TranslationBlock, phys_hash_next));
794
795 /* remove the TB from the page list */
796 if (tb->page_addr[0] != page_addr) {
797 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
798 tb_page_remove(&p->first_tb, tb);
799 invalidate_page_bitmap(p);
800 }
801 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
802 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
803 tb_page_remove(&p->first_tb, tb);
804 invalidate_page_bitmap(p);
805 }
806
807 tb_invalidated_flag = 1;
808
809 /* remove the TB from the hash list */
810 h = tb_jmp_cache_hash_func(tb->pc);
811 for(env = first_cpu; env != NULL; env = env->next_cpu) {
812 if (env->tb_jmp_cache[h] == tb)
813 env->tb_jmp_cache[h] = NULL;
814 }
815
816 /* suppress this TB from the two jump lists */
817 tb_jmp_remove(tb, 0);
818 tb_jmp_remove(tb, 1);
819
820 /* suppress any remaining jumps to this TB */
821 tb1 = tb->jmp_first;
822 for(;;) {
823 n1 = (long)tb1 & 3;
824 if (n1 == 2)
825 break;
826 tb1 = (TranslationBlock *)((long)tb1 & ~3);
827 tb2 = tb1->jmp_next[n1];
828 tb_reset_jump(tb1, n1);
829 tb1->jmp_next[n1] = NULL;
830 tb1 = tb2;
831 }
832 tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
833
834 tb_phys_invalidate_count++;
835 }
836
837 static inline void set_bits(uint8_t *tab, int start, int len)
838 {
839 int end, mask, end1;
840
841 end = start + len;
842 tab += start >> 3;
843 mask = 0xff << (start & 7);
844 if ((start & ~7) == (end & ~7)) {
845 if (start < end) {
846 mask &= ~(0xff << (end & 7));
847 *tab |= mask;
848 }
849 } else {
850 *tab++ |= mask;
851 start = (start + 8) & ~7;
852 end1 = end & ~7;
853 while (start < end1) {
854 *tab++ = 0xff;
855 start += 8;
856 }
857 if (start < end) {
858 mask = ~(0xff << (end & 7));
859 *tab |= mask;
860 }
861 }
862 }
863
864 static void build_page_bitmap(PageDesc *p)
865 {
866 int n, tb_start, tb_end;
867 TranslationBlock *tb;
868
869 p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
870
871 tb = p->first_tb;
872 while (tb != NULL) {
873 n = (long)tb & 3;
874 tb = (TranslationBlock *)((long)tb & ~3);
875 /* NOTE: this is subtle as a TB may span two physical pages */
876 if (n == 0) {
877 /* NOTE: tb_end may be after the end of the page, but
878 it is not a problem */
879 tb_start = tb->pc & ~TARGET_PAGE_MASK;
880 tb_end = tb_start + tb->size;
881 if (tb_end > TARGET_PAGE_SIZE)
882 tb_end = TARGET_PAGE_SIZE;
883 } else {
884 tb_start = 0;
885 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
886 }
887 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
888 tb = tb->page_next[n];
889 }
890 }
891
892 TranslationBlock *tb_gen_code(CPUState *env,
893 target_ulong pc, target_ulong cs_base,
894 int flags, int cflags)
895 {
896 TranslationBlock *tb;
897 uint8_t *tc_ptr;
898 target_ulong phys_pc, phys_page2, virt_page2;
899 int code_gen_size;
900
901 phys_pc = get_phys_addr_code(env, pc);
902 tb = tb_alloc(pc);
903 if (!tb) {
904 /* flush must be done */
905 tb_flush(env);
906 /* cannot fail at this point */
907 tb = tb_alloc(pc);
908 /* Don't forget to invalidate previous TB info. */
909 tb_invalidated_flag = 1;
910 }
911 tc_ptr = code_gen_ptr;
912 tb->tc_ptr = tc_ptr;
913 tb->cs_base = cs_base;
914 tb->flags = flags;
915 tb->cflags = cflags;
916 cpu_gen_code(env, tb, &code_gen_size);
917 code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
918
919 /* check next page if needed */
920 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
921 phys_page2 = -1;
922 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
923 phys_page2 = get_phys_addr_code(env, virt_page2);
924 }
925 tb_link_phys(tb, phys_pc, phys_page2);
926 return tb;
927 }
928
929 /* invalidate all TBs which intersect with the target physical page
930 starting in range [start;end[. NOTE: start and end must refer to
931 the same physical page. 'is_cpu_write_access' should be true if called
932 from a real cpu write access: the virtual CPU will exit the current
933 TB if code is modified inside this TB. */
934 void tb_invalidate_phys_page_range(target_phys_addr_t start, target_phys_addr_t end,
935 int is_cpu_write_access)
936 {
937 TranslationBlock *tb, *tb_next, *saved_tb;
938 CPUState *env = cpu_single_env;
939 target_ulong tb_start, tb_end;
940 PageDesc *p;
941 int n;
942 #ifdef TARGET_HAS_PRECISE_SMC
943 int current_tb_not_found = is_cpu_write_access;
944 TranslationBlock *current_tb = NULL;
945 int current_tb_modified = 0;
946 target_ulong current_pc = 0;
947 target_ulong current_cs_base = 0;
948 int current_flags = 0;
949 #endif /* TARGET_HAS_PRECISE_SMC */
950
951 p = page_find(start >> TARGET_PAGE_BITS);
952 if (!p)
953 return;
954 if (!p->code_bitmap &&
955 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
956 is_cpu_write_access) {
957 /* build code bitmap */
958 build_page_bitmap(p);
959 }
960
961 /* we remove all the TBs in the range [start, end[ */
962 /* XXX: see if in some cases it could be faster to invalidate all the code */
963 tb = p->first_tb;
964 while (tb != NULL) {
965 n = (long)tb & 3;
966 tb = (TranslationBlock *)((long)tb & ~3);
967 tb_next = tb->page_next[n];
968 /* NOTE: this is subtle as a TB may span two physical pages */
969 if (n == 0) {
970 /* NOTE: tb_end may be after the end of the page, but
971 it is not a problem */
972 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
973 tb_end = tb_start + tb->size;
974 } else {
975 tb_start = tb->page_addr[1];
976 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
977 }
978 if (!(tb_end <= start || tb_start >= end)) {
979 #ifdef TARGET_HAS_PRECISE_SMC
980 if (current_tb_not_found) {
981 current_tb_not_found = 0;
982 current_tb = NULL;
983 if (env->mem_io_pc) {
984 /* now we have a real cpu fault */
985 current_tb = tb_find_pc(env->mem_io_pc);
986 }
987 }
988 if (current_tb == tb &&
989 (current_tb->cflags & CF_COUNT_MASK) != 1) {
990 /* If we are modifying the current TB, we must stop
991 its execution. We could be more precise by checking
992 that the modification is after the current PC, but it
993 would require a specialized function to partially
994 restore the CPU state */
995
996 current_tb_modified = 1;
997 cpu_restore_state(current_tb, env,
998 env->mem_io_pc, NULL);
999 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1000 &current_flags);
1001 }
1002 #endif /* TARGET_HAS_PRECISE_SMC */
1003 /* we need to do that to handle the case where a signal
1004 occurs while doing tb_phys_invalidate() */
1005 saved_tb = NULL;
1006 if (env) {
1007 saved_tb = env->current_tb;
1008 env->current_tb = NULL;
1009 }
1010 tb_phys_invalidate(tb, -1);
1011 if (env) {
1012 env->current_tb = saved_tb;
1013 if (env->interrupt_request && env->current_tb)
1014 cpu_interrupt(env, env->interrupt_request);
1015 }
1016 }
1017 tb = tb_next;
1018 }
1019 #if !defined(CONFIG_USER_ONLY)
1020 /* if no code remaining, no need to continue to use slow writes */
1021 if (!p->first_tb) {
1022 invalidate_page_bitmap(p);
1023 if (is_cpu_write_access) {
1024 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1025 }
1026 }
1027 #endif
1028 #ifdef TARGET_HAS_PRECISE_SMC
1029 if (current_tb_modified) {
1030 /* we generate a block containing just the instruction
1031 modifying the memory. It will ensure that it cannot modify
1032 itself */
1033 env->current_tb = NULL;
1034 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1035 cpu_resume_from_signal(env, NULL);
1036 }
1037 #endif
1038 }
1039
1040 /* len must be <= 8 and start must be a multiple of len */
1041 static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start, int len)
1042 {
1043 PageDesc *p;
1044 int offset, b;
1045 #if 0
1046 if (1) {
1047 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1048 cpu_single_env->mem_io_vaddr, len,
1049 cpu_single_env->eip,
1050 cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1051 }
1052 #endif
1053 p = page_find(start >> TARGET_PAGE_BITS);
1054 if (!p)
1055 return;
1056 if (p->code_bitmap) {
1057 offset = start & ~TARGET_PAGE_MASK;
1058 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1059 if (b & ((1 << len) - 1))
1060 goto do_invalidate;
1061 } else {
1062 do_invalidate:
1063 tb_invalidate_phys_page_range(start, start + len, 1);
1064 }
1065 }
1066
1067 #if !defined(CONFIG_SOFTMMU)
1068 static void tb_invalidate_phys_page(target_phys_addr_t addr,
1069 unsigned long pc, void *puc)
1070 {
1071 TranslationBlock *tb;
1072 PageDesc *p;
1073 int n;
1074 #ifdef TARGET_HAS_PRECISE_SMC
1075 TranslationBlock *current_tb = NULL;
1076 CPUState *env = cpu_single_env;
1077 int current_tb_modified = 0;
1078 target_ulong current_pc = 0;
1079 target_ulong current_cs_base = 0;
1080 int current_flags = 0;
1081 #endif
1082
1083 addr &= TARGET_PAGE_MASK;
1084 p = page_find(addr >> TARGET_PAGE_BITS);
1085 if (!p)
1086 return;
1087 tb = p->first_tb;
1088 #ifdef TARGET_HAS_PRECISE_SMC
1089 if (tb && pc != 0) {
1090 current_tb = tb_find_pc(pc);
1091 }
1092 #endif
1093 while (tb != NULL) {
1094 n = (long)tb & 3;
1095 tb = (TranslationBlock *)((long)tb & ~3);
1096 #ifdef TARGET_HAS_PRECISE_SMC
1097 if (current_tb == tb &&
1098 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1099 /* If we are modifying the current TB, we must stop
1100 its execution. We could be more precise by checking
1101 that the modification is after the current PC, but it
1102 would require a specialized function to partially
1103 restore the CPU state */
1104
1105 current_tb_modified = 1;
1106 cpu_restore_state(current_tb, env, pc, puc);
1107 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1108 &current_flags);
1109 }
1110 #endif /* TARGET_HAS_PRECISE_SMC */
1111 tb_phys_invalidate(tb, addr);
1112 tb = tb->page_next[n];
1113 }
1114 p->first_tb = NULL;
1115 #ifdef TARGET_HAS_PRECISE_SMC
1116 if (current_tb_modified) {
1117 /* we generate a block containing just the instruction
1118 modifying the memory. It will ensure that it cannot modify
1119 itself */
1120 env->current_tb = NULL;
1121 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1122 cpu_resume_from_signal(env, puc);
1123 }
1124 #endif
1125 }
1126 #endif
1127
1128 /* add the tb in the target page and protect it if necessary */
1129 static inline void tb_alloc_page(TranslationBlock *tb,
1130 unsigned int n, target_ulong page_addr)
1131 {
1132 PageDesc *p;
1133 TranslationBlock *last_first_tb;
1134
1135 tb->page_addr[n] = page_addr;
1136 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS);
1137 tb->page_next[n] = p->first_tb;
1138 last_first_tb = p->first_tb;
1139 p->first_tb = (TranslationBlock *)((long)tb | n);
1140 invalidate_page_bitmap(p);
1141
1142 #if defined(TARGET_HAS_SMC) || 1
1143
1144 #if defined(CONFIG_USER_ONLY)
1145 if (p->flags & PAGE_WRITE) {
1146 target_ulong addr;
1147 PageDesc *p2;
1148 int prot;
1149
1150 /* force the host page as non writable (writes will have a
1151 page fault + mprotect overhead) */
1152 page_addr &= qemu_host_page_mask;
1153 prot = 0;
1154 for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1155 addr += TARGET_PAGE_SIZE) {
1156
1157 p2 = page_find (addr >> TARGET_PAGE_BITS);
1158 if (!p2)
1159 continue;
1160 prot |= p2->flags;
1161 p2->flags &= ~PAGE_WRITE;
1162 page_get_flags(addr);
1163 }
1164 mprotect(g2h(page_addr), qemu_host_page_size,
1165 (prot & PAGE_BITS) & ~PAGE_WRITE);
1166 #ifdef DEBUG_TB_INVALIDATE
1167 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1168 page_addr);
1169 #endif
1170 }
1171 #else
1172 /* if some code is already present, then the pages are already
1173 protected. So we handle the case where only the first TB is
1174 allocated in a physical page */
1175 if (!last_first_tb) {
1176 tlb_protect_code(page_addr);
1177 }
1178 #endif
1179
1180 #endif /* TARGET_HAS_SMC */
1181 }
1182
1183 /* Allocate a new translation block. Flush the translation buffer if
1184 too many translation blocks or too much generated code. */
1185 TranslationBlock *tb_alloc(target_ulong pc)
1186 {
1187 TranslationBlock *tb;
1188
1189 if (nb_tbs >= code_gen_max_blocks ||
1190 (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
1191 return NULL;
1192 tb = &tbs[nb_tbs++];
1193 tb->pc = pc;
1194 tb->cflags = 0;
1195 return tb;
1196 }
1197
1198 void tb_free(TranslationBlock *tb)
1199 {
1200 /* In practice this is mostly used for single use temporary TB
1201 Ignore the hard cases and just back up if this TB happens to
1202 be the last one generated. */
1203 if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
1204 code_gen_ptr = tb->tc_ptr;
1205 nb_tbs--;
1206 }
1207 }
1208
1209 /* add a new TB and link it to the physical page tables. phys_page2 is
1210 (-1) to indicate that only one page contains the TB. */
1211 void tb_link_phys(TranslationBlock *tb,
1212 target_ulong phys_pc, target_ulong phys_page2)
1213 {
1214 unsigned int h;
1215 TranslationBlock **ptb;
1216
1217 /* Grab the mmap lock to stop another thread invalidating this TB
1218 before we are done. */
1219 mmap_lock();
1220 /* add in the physical hash table */
1221 h = tb_phys_hash_func(phys_pc);
1222 ptb = &tb_phys_hash[h];
1223 tb->phys_hash_next = *ptb;
1224 *ptb = tb;
1225
1226 /* add in the page list */
1227 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1228 if (phys_page2 != -1)
1229 tb_alloc_page(tb, 1, phys_page2);
1230 else
1231 tb->page_addr[1] = -1;
1232
1233 tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1234 tb->jmp_next[0] = NULL;
1235 tb->jmp_next[1] = NULL;
1236
1237 /* init original jump addresses */
1238 if (tb->tb_next_offset[0] != 0xffff)
1239 tb_reset_jump(tb, 0);
1240 if (tb->tb_next_offset[1] != 0xffff)
1241 tb_reset_jump(tb, 1);
1242
1243 #ifdef DEBUG_TB_CHECK
1244 tb_page_check();
1245 #endif
1246 mmap_unlock();
1247 }
1248
1249 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1250 tb[1].tc_ptr. Return NULL if not found */
1251 TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1252 {
1253 int m_min, m_max, m;
1254 unsigned long v;
1255 TranslationBlock *tb;
1256
1257 if (nb_tbs <= 0)
1258 return NULL;
1259 if (tc_ptr < (unsigned long)code_gen_buffer ||
1260 tc_ptr >= (unsigned long)code_gen_ptr)
1261 return NULL;
1262 /* binary search (cf Knuth) */
1263 m_min = 0;
1264 m_max = nb_tbs - 1;
1265 while (m_min <= m_max) {
1266 m = (m_min + m_max) >> 1;
1267 tb = &tbs[m];
1268 v = (unsigned long)tb->tc_ptr;
1269 if (v == tc_ptr)
1270 return tb;
1271 else if (tc_ptr < v) {
1272 m_max = m - 1;
1273 } else {
1274 m_min = m + 1;
1275 }
1276 }
1277 return &tbs[m_max];
1278 }
1279
1280 static void tb_reset_jump_recursive(TranslationBlock *tb);
1281
1282 static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1283 {
1284 TranslationBlock *tb1, *tb_next, **ptb;
1285 unsigned int n1;
1286
1287 tb1 = tb->jmp_next[n];
1288 if (tb1 != NULL) {
1289 /* find head of list */
1290 for(;;) {
1291 n1 = (long)tb1 & 3;
1292 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1293 if (n1 == 2)
1294 break;
1295 tb1 = tb1->jmp_next[n1];
1296 }
1297 /* we are now sure now that tb jumps to tb1 */
1298 tb_next = tb1;
1299
1300 /* remove tb from the jmp_first list */
1301 ptb = &tb_next->jmp_first;
1302 for(;;) {
1303 tb1 = *ptb;
1304 n1 = (long)tb1 & 3;
1305 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1306 if (n1 == n && tb1 == tb)
1307 break;
1308 ptb = &tb1->jmp_next[n1];
1309 }
1310 *ptb = tb->jmp_next[n];
1311 tb->jmp_next[n] = NULL;
1312
1313 /* suppress the jump to next tb in generated code */
1314 tb_reset_jump(tb, n);
1315
1316 /* suppress jumps in the tb on which we could have jumped */
1317 tb_reset_jump_recursive(tb_next);
1318 }
1319 }
1320
1321 static void tb_reset_jump_recursive(TranslationBlock *tb)
1322 {
1323 tb_reset_jump_recursive2(tb, 0);
1324 tb_reset_jump_recursive2(tb, 1);
1325 }
1326
1327 #if defined(TARGET_HAS_ICE)
1328 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1329 {
1330 target_phys_addr_t addr;
1331 target_ulong pd;
1332 ram_addr_t ram_addr;
1333 PhysPageDesc *p;
1334
1335 addr = cpu_get_phys_page_debug(env, pc);
1336 p = phys_page_find(addr >> TARGET_PAGE_BITS);
1337 if (!p) {
1338 pd = IO_MEM_UNASSIGNED;
1339 } else {
1340 pd = p->phys_offset;
1341 }
1342 ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1343 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1344 }
1345 #endif
1346
1347 /* Add a watchpoint. */
1348 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1349 int flags, CPUWatchpoint **watchpoint)
1350 {
1351 target_ulong len_mask = ~(len - 1);
1352 CPUWatchpoint *wp;
1353
1354 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1355 if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1356 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1357 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1358 return -EINVAL;
1359 }
1360 wp = qemu_malloc(sizeof(*wp));
1361
1362 wp->vaddr = addr;
1363 wp->len_mask = len_mask;
1364 wp->flags = flags;
1365
1366 /* keep all GDB-injected watchpoints in front */
1367 if (flags & BP_GDB)
1368 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1369 else
1370 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1371
1372 tlb_flush_page(env, addr);
1373
1374 if (watchpoint)
1375 *watchpoint = wp;
1376 return 0;
1377 }
1378
1379 /* Remove a specific watchpoint. */
1380 int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1381 int flags)
1382 {
1383 target_ulong len_mask = ~(len - 1);
1384 CPUWatchpoint *wp;
1385
1386 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1387 if (addr == wp->vaddr && len_mask == wp->len_mask
1388 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1389 cpu_watchpoint_remove_by_ref(env, wp);
1390 return 0;
1391 }
1392 }
1393 return -ENOENT;
1394 }
1395
1396 /* Remove a specific watchpoint by reference. */
1397 void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1398 {
1399 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1400
1401 tlb_flush_page(env, watchpoint->vaddr);
1402
1403 qemu_free(watchpoint);
1404 }
1405
1406 /* Remove all matching watchpoints. */
1407 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1408 {
1409 CPUWatchpoint *wp, *next;
1410
1411 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1412 if (wp->flags & mask)
1413 cpu_watchpoint_remove_by_ref(env, wp);
1414 }
1415 }
1416
1417 /* Add a breakpoint. */
1418 int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1419 CPUBreakpoint **breakpoint)
1420 {
1421 #if defined(TARGET_HAS_ICE)
1422 CPUBreakpoint *bp;
1423
1424 bp = qemu_malloc(sizeof(*bp));
1425
1426 bp->pc = pc;
1427 bp->flags = flags;
1428
1429 /* keep all GDB-injected breakpoints in front */
1430 if (flags & BP_GDB)
1431 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1432 else
1433 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1434
1435 breakpoint_invalidate(env, pc);
1436
1437 if (breakpoint)
1438 *breakpoint = bp;
1439 return 0;
1440 #else
1441 return -ENOSYS;
1442 #endif
1443 }
1444
1445 /* Remove a specific breakpoint. */
1446 int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1447 {
1448 #if defined(TARGET_HAS_ICE)
1449 CPUBreakpoint *bp;
1450
1451 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1452 if (bp->pc == pc && bp->flags == flags) {
1453 cpu_breakpoint_remove_by_ref(env, bp);
1454 return 0;
1455 }
1456 }
1457 return -ENOENT;
1458 #else
1459 return -ENOSYS;
1460 #endif
1461 }
1462
1463 /* Remove a specific breakpoint by reference. */
1464 void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
1465 {
1466 #if defined(TARGET_HAS_ICE)
1467 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1468
1469 breakpoint_invalidate(env, breakpoint->pc);
1470
1471 qemu_free(breakpoint);
1472 #endif
1473 }
1474
1475 /* Remove all matching breakpoints. */
1476 void cpu_breakpoint_remove_all(CPUState *env, int mask)
1477 {
1478 #if defined(TARGET_HAS_ICE)
1479 CPUBreakpoint *bp, *next;
1480
1481 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1482 if (bp->flags & mask)
1483 cpu_breakpoint_remove_by_ref(env, bp);
1484 }
1485 #endif
1486 }
1487
1488 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1489 CPU loop after each instruction */
1490 void cpu_single_step(CPUState *env, int enabled)
1491 {
1492 #if defined(TARGET_HAS_ICE)
1493 if (env->singlestep_enabled != enabled) {
1494 env->singlestep_enabled = enabled;
1495 if (kvm_enabled())
1496 kvm_update_guest_debug(env, 0);
1497 else {
1498 /* must flush all the translated code to avoid inconsistencies */
1499 /* XXX: only flush what is necessary */
1500 tb_flush(env);
1501 }
1502 }
1503 #endif
1504 }
1505
1506 /* enable or disable low levels log */
1507 void cpu_set_log(int log_flags)
1508 {
1509 loglevel = log_flags;
1510 if (loglevel && !logfile) {
1511 logfile = fopen(logfilename, log_append ? "a" : "w");
1512 if (!logfile) {
1513 perror(logfilename);
1514 _exit(1);
1515 }
1516 #if !defined(CONFIG_SOFTMMU)
1517 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1518 {
1519 static char logfile_buf[4096];
1520 setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1521 }
1522 #elif !defined(_WIN32)
1523 /* Win32 doesn't support line-buffering and requires size >= 2 */
1524 setvbuf(logfile, NULL, _IOLBF, 0);
1525 #endif
1526 log_append = 1;
1527 }
1528 if (!loglevel && logfile) {
1529 fclose(logfile);
1530 logfile = NULL;
1531 }
1532 }
1533
1534 void cpu_set_log_filename(const char *filename)
1535 {
1536 logfilename = strdup(filename);
1537 if (logfile) {
1538 fclose(logfile);
1539 logfile = NULL;
1540 }
1541 cpu_set_log(loglevel);
1542 }
1543
1544 static void cpu_unlink_tb(CPUState *env)
1545 {
1546 #if defined(CONFIG_USE_NPTL)
1547 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1548 problem and hope the cpu will stop of its own accord. For userspace
1549 emulation this often isn't actually as bad as it sounds. Often
1550 signals are used primarily to interrupt blocking syscalls. */
1551 #else
1552 TranslationBlock *tb;
1553 static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1554
1555 tb = env->current_tb;
1556 /* if the cpu is currently executing code, we must unlink it and
1557 all the potentially executing TB */
1558 if (tb && !testandset(&interrupt_lock)) {
1559 env->current_tb = NULL;
1560 tb_reset_jump_recursive(tb);
1561 resetlock(&interrupt_lock);
1562 }
1563 #endif
1564 }
1565
1566 /* mask must never be zero, except for A20 change call */
1567 void cpu_interrupt(CPUState *env, int mask)
1568 {
1569 int old_mask;
1570
1571 old_mask = env->interrupt_request;
1572 env->interrupt_request |= mask;
1573 if (kvm_enabled() && !qemu_kvm_irqchip_in_kernel())
1574 kvm_update_interrupt_request(env);
1575
1576 #ifndef CONFIG_USER_ONLY
1577 /*
1578 * If called from iothread context, wake the target cpu in
1579 * case its halted.
1580 */
1581 if (!qemu_cpu_self(env)) {
1582 qemu_cpu_kick(env);
1583 return;
1584 }
1585 #endif
1586
1587 if (use_icount) {
1588 env->icount_decr.u16.high = 0xffff;
1589 #ifndef CONFIG_USER_ONLY
1590 if (!can_do_io(env)
1591 && (mask & ~old_mask) != 0) {
1592 cpu_abort(env, "Raised interrupt while not in I/O function");
1593 }
1594 #endif
1595 } else {
1596 cpu_unlink_tb(env);
1597 }
1598 }
1599
1600 void cpu_reset_interrupt(CPUState *env, int mask)
1601 {
1602 env->interrupt_request &= ~mask;
1603 }
1604
1605 void cpu_exit(CPUState *env)
1606 {
1607 env->exit_request = 1;
1608 cpu_unlink_tb(env);
1609 }
1610
1611 const CPULogItem cpu_log_items[] = {
1612 { CPU_LOG_TB_OUT_ASM, "out_asm",
1613 "show generated host assembly code for each compiled TB" },
1614 { CPU_LOG_TB_IN_ASM, "in_asm",
1615 "show target assembly code for each compiled TB" },
1616 { CPU_LOG_TB_OP, "op",
1617 "show micro ops for each compiled TB" },
1618 { CPU_LOG_TB_OP_OPT, "op_opt",
1619 "show micro ops "
1620 #ifdef TARGET_I386
1621 "before eflags optimization and "
1622 #endif
1623 "after liveness analysis" },
1624 { CPU_LOG_INT, "int",
1625 "show interrupts/exceptions in short format" },
1626 { CPU_LOG_EXEC, "exec",
1627 "show trace before each executed TB (lots of logs)" },
1628 { CPU_LOG_TB_CPU, "cpu",
1629 "show CPU state before block translation" },
1630 #ifdef TARGET_I386
1631 { CPU_LOG_PCALL, "pcall",
1632 "show protected mode far calls/returns/exceptions" },
1633 { CPU_LOG_RESET, "cpu_reset",
1634 "show CPU state before CPU resets" },
1635 #endif
1636 #ifdef DEBUG_IOPORT
1637 { CPU_LOG_IOPORT, "ioport",
1638 "show all i/o ports accesses" },
1639 #endif
1640 { 0, NULL, NULL },
1641 };
1642
1643 static int cmp1(const char *s1, int n, const char *s2)
1644 {
1645 if (strlen(s2) != n)
1646 return 0;
1647 return memcmp(s1, s2, n) == 0;
1648 }
1649
1650 /* takes a comma separated list of log masks. Return 0 if error. */
1651 int cpu_str_to_log_mask(const char *str)
1652 {
1653 const CPULogItem *item;
1654 int mask;
1655 const char *p, *p1;
1656
1657 p = str;
1658 mask = 0;
1659 for(;;) {
1660 p1 = strchr(p, ',');
1661 if (!p1)
1662 p1 = p + strlen(p);
1663 if(cmp1(p,p1-p,"all")) {
1664 for(item = cpu_log_items; item->mask != 0; item++) {
1665 mask |= item->mask;
1666 }
1667 } else {
1668 for(item = cpu_log_items; item->mask != 0; item++) {
1669 if (cmp1(p, p1 - p, item->name))
1670 goto found;
1671 }
1672 return 0;
1673 }
1674 found:
1675 mask |= item->mask;
1676 if (*p1 != ',')
1677 break;
1678 p = p1 + 1;
1679 }
1680 return mask;
1681 }
1682
1683 void cpu_abort(CPUState *env, const char *fmt, ...)
1684 {
1685 va_list ap;
1686 va_list ap2;
1687
1688 va_start(ap, fmt);
1689 va_copy(ap2, ap);
1690 fprintf(stderr, "qemu: fatal: ");
1691 vfprintf(stderr, fmt, ap);
1692 fprintf(stderr, "\n");
1693 #ifdef TARGET_I386
1694 cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1695 #else
1696 cpu_dump_state(env, stderr, fprintf, 0);
1697 #endif
1698 if (qemu_log_enabled()) {
1699 qemu_log("qemu: fatal: ");
1700 qemu_log_vprintf(fmt, ap2);
1701 qemu_log("\n");
1702 #ifdef TARGET_I386
1703 log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1704 #else
1705 log_cpu_state(env, 0);
1706 #endif
1707 qemu_log_flush();
1708 qemu_log_close();
1709 }
1710 va_end(ap2);
1711 va_end(ap);
1712 abort();
1713 }
1714
1715 CPUState *cpu_copy(CPUState *env)
1716 {
1717 CPUState *new_env = cpu_init(env->cpu_model_str);
1718 CPUState *next_cpu = new_env->next_cpu;
1719 int cpu_index = new_env->cpu_index;
1720 #if defined(TARGET_HAS_ICE)
1721 CPUBreakpoint *bp;
1722 CPUWatchpoint *wp;
1723 #endif
1724
1725 memcpy(new_env, env, sizeof(CPUState));
1726
1727 /* Preserve chaining and index. */
1728 new_env->next_cpu = next_cpu;
1729 new_env->cpu_index = cpu_index;
1730
1731 /* Clone all break/watchpoints.
1732 Note: Once we support ptrace with hw-debug register access, make sure
1733 BP_CPU break/watchpoints are handled correctly on clone. */
1734 QTAILQ_INIT(&env->breakpoints);
1735 QTAILQ_INIT(&env->watchpoints);
1736 #if defined(TARGET_HAS_ICE)
1737 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1738 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1739 }
1740 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1741 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1742 wp->flags, NULL);
1743 }
1744 #endif
1745
1746 return new_env;
1747 }
1748
1749 #if !defined(CONFIG_USER_ONLY)
1750
1751 static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1752 {
1753 unsigned int i;
1754
1755 /* Discard jump cache entries for any tb which might potentially
1756 overlap the flushed page. */
1757 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1758 memset (&env->tb_jmp_cache[i], 0,
1759 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1760
1761 i = tb_jmp_cache_hash_page(addr);
1762 memset (&env->tb_jmp_cache[i], 0,
1763 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1764 }
1765
1766 static CPUTLBEntry s_cputlb_empty_entry = {
1767 .addr_read = -1,
1768 .addr_write = -1,
1769 .addr_code = -1,
1770 .addend = -1,
1771 };
1772
1773 /* NOTE: if flush_global is true, also flush global entries (not
1774 implemented yet) */
1775 void tlb_flush(CPUState *env, int flush_global)
1776 {
1777 int i;
1778
1779 #if defined(DEBUG_TLB)
1780 printf("tlb_flush:\n");
1781 #endif
1782 /* must reset current TB so that interrupts cannot modify the
1783 links while we are modifying them */
1784 env->current_tb = NULL;
1785
1786 for(i = 0; i < CPU_TLB_SIZE; i++) {
1787 int mmu_idx;
1788 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
1789 env->tlb_table[mmu_idx][i] = s_cputlb_empty_entry;
1790 }
1791 }
1792
1793 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1794
1795 tlb_flush_count++;
1796 }
1797
1798 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
1799 {
1800 if (addr == (tlb_entry->addr_read &
1801 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1802 addr == (tlb_entry->addr_write &
1803 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1804 addr == (tlb_entry->addr_code &
1805 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1806 *tlb_entry = s_cputlb_empty_entry;
1807 }
1808 }
1809
1810 void tlb_flush_page(CPUState *env, target_ulong addr)
1811 {
1812 int i;
1813 int mmu_idx;
1814
1815 #if defined(DEBUG_TLB)
1816 printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1817 #endif
1818 /* must reset current TB so that interrupts cannot modify the
1819 links while we are modifying them */
1820 env->current_tb = NULL;
1821
1822 addr &= TARGET_PAGE_MASK;
1823 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1824 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
1825 tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
1826
1827 tlb_flush_jmp_cache(env, addr);
1828 }
1829
1830 /* update the TLBs so that writes to code in the virtual page 'addr'
1831 can be detected */
1832 static void tlb_protect_code(ram_addr_t ram_addr)
1833 {
1834 cpu_physical_memory_reset_dirty(ram_addr,
1835 ram_addr + TARGET_PAGE_SIZE,
1836 CODE_DIRTY_FLAG);
1837 }
1838
1839 /* update the TLB so that writes in physical page 'phys_addr' are no longer
1840 tested for self modifying code */
1841 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
1842 target_ulong vaddr)
1843 {
1844 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] |= CODE_DIRTY_FLAG;
1845 }
1846
1847 static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
1848 unsigned long start, unsigned long length)
1849 {
1850 unsigned long addr;
1851 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1852 addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
1853 if ((addr - start) < length) {
1854 tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
1855 }
1856 }
1857 }
1858
1859 /* Note: start and end must be within the same ram block. */
1860 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
1861 int dirty_flags)
1862 {
1863 CPUState *env;
1864 unsigned long length, start1;
1865 int i, mask, len;
1866 uint8_t *p;
1867
1868 start &= TARGET_PAGE_MASK;
1869 end = TARGET_PAGE_ALIGN(end);
1870
1871 length = end - start;
1872 if (length == 0)
1873 return;
1874 len = length >> TARGET_PAGE_BITS;
1875 mask = ~dirty_flags;
1876 p = phys_ram_dirty + (start >> TARGET_PAGE_BITS);
1877 for(i = 0; i < len; i++)
1878 p[i] &= mask;
1879
1880 /* we modify the TLB cache so that the dirty bit will be set again
1881 when accessing the range */
1882 start1 = (unsigned long)qemu_get_ram_ptr(start);
1883 /* Chek that we don't span multiple blocks - this breaks the
1884 address comparisons below. */
1885 if ((unsigned long)qemu_get_ram_ptr(end - 1) - start1
1886 != (end - 1) - start) {
1887 abort();
1888 }
1889
1890 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1891 int mmu_idx;
1892 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
1893 for(i = 0; i < CPU_TLB_SIZE; i++)
1894 tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
1895 start1, length);
1896 }
1897 }
1898 }
1899
1900 int cpu_physical_memory_set_dirty_tracking(int enable)
1901 {
1902 if (kvm_enabled()) {
1903 return kvm_set_migration_log(enable);
1904 }
1905 return 0;
1906 }
1907
1908 int cpu_physical_memory_get_dirty_tracking(void)
1909 {
1910 return in_migration;
1911 }
1912
1913 int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
1914 target_phys_addr_t end_addr)
1915 {
1916 int ret = 0;
1917
1918 if (kvm_enabled())
1919 ret = kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
1920 return ret;
1921 }
1922
1923 static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
1924 {
1925 ram_addr_t ram_addr;
1926 void *p;
1927
1928 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1929 p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
1930 + tlb_entry->addend);
1931 ram_addr = qemu_ram_addr_from_host(p);
1932 if (!cpu_physical_memory_is_dirty(ram_addr)) {
1933 tlb_entry->addr_write |= TLB_NOTDIRTY;
1934 }
1935 }
1936 }
1937
1938 /* update the TLB according to the current state of the dirty bits */
1939 void cpu_tlb_update_dirty(CPUState *env)
1940 {
1941 int i;
1942 int mmu_idx;
1943 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
1944 for(i = 0; i < CPU_TLB_SIZE; i++)
1945 tlb_update_dirty(&env->tlb_table[mmu_idx][i]);
1946 }
1947 }
1948
1949 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
1950 {
1951 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
1952 tlb_entry->addr_write = vaddr;
1953 }
1954
1955 /* update the TLB corresponding to virtual page vaddr
1956 so that it is no longer dirty */
1957 static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
1958 {
1959 int i;
1960 int mmu_idx;
1961
1962 vaddr &= TARGET_PAGE_MASK;
1963 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1964 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
1965 tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
1966 }
1967
1968 /* add a new TLB entry. At most one entry for a given virtual address
1969 is permitted. Return 0 if OK or 2 if the page could not be mapped
1970 (can only happen in non SOFTMMU mode for I/O pages or pages
1971 conflicting with the host address space). */
1972 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
1973 target_phys_addr_t paddr, int prot,
1974 int mmu_idx, int is_softmmu)
1975 {
1976 PhysPageDesc *p;
1977 unsigned long pd;
1978 unsigned int index;
1979 target_ulong address;
1980 target_ulong code_address;
1981 target_phys_addr_t addend;
1982 int ret;
1983 CPUTLBEntry *te;
1984 CPUWatchpoint *wp;
1985 target_phys_addr_t iotlb;
1986
1987 p = phys_page_find(paddr >> TARGET_PAGE_BITS);
1988 if (!p) {
1989 pd = IO_MEM_UNASSIGNED;
1990 } else {
1991 pd = p->phys_offset;
1992 }
1993 #if defined(DEBUG_TLB)
1994 printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
1995 vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
1996 #endif
1997
1998 ret = 0;
1999 address = vaddr;
2000 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
2001 /* IO memory case (romd handled later) */
2002 address |= TLB_MMIO;
2003 }
2004 addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
2005 if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
2006 /* Normal RAM. */
2007 iotlb = pd & TARGET_PAGE_MASK;
2008 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
2009 iotlb |= IO_MEM_NOTDIRTY;
2010 else
2011 iotlb |= IO_MEM_ROM;
2012 } else {
2013 /* IO handlers are currently passed a physical address.
2014 It would be nice to pass an offset from the base address
2015 of that region. This would avoid having to special case RAM,
2016 and avoid full address decoding in every device.
2017 We can't use the high bits of pd for this because
2018 IO_MEM_ROMD uses these as a ram address. */
2019 iotlb = (pd & ~TARGET_PAGE_MASK);
2020 if (p) {
2021 iotlb += p->region_offset;
2022 } else {
2023 iotlb += paddr;
2024 }
2025 }
2026
2027 code_address = address;
2028 /* Make accesses to pages with watchpoints go via the
2029 watchpoint trap routines. */
2030 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
2031 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2032 iotlb = io_mem_watch + paddr;
2033 /* TODO: The memory case can be optimized by not trapping
2034 reads of pages with a write breakpoint. */
2035 address |= TLB_MMIO;
2036 }
2037 }
2038
2039 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2040 env->iotlb[mmu_idx][index] = iotlb - vaddr;
2041 te = &env->tlb_table[mmu_idx][index];
2042 te->addend = addend - vaddr;
2043 if (prot & PAGE_READ) {
2044 te->addr_read = address;
2045 } else {
2046 te->addr_read = -1;
2047 }
2048
2049 if (prot & PAGE_EXEC) {
2050 te->addr_code = code_address;
2051 } else {
2052 te->addr_code = -1;
2053 }
2054 if (prot & PAGE_WRITE) {
2055 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2056 (pd & IO_MEM_ROMD)) {
2057 /* Write access calls the I/O callback. */
2058 te->addr_write = address | TLB_MMIO;
2059 } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2060 !cpu_physical_memory_is_dirty(pd)) {
2061 te->addr_write = address | TLB_NOTDIRTY;
2062 } else {
2063 te->addr_write = address;
2064 }
2065 } else {
2066 te->addr_write = -1;
2067 }
2068 return ret;
2069 }
2070
2071 #else
2072
2073 void tlb_flush(CPUState *env, int flush_global)
2074 {
2075 }
2076
2077 void tlb_flush_page(CPUState *env, target_ulong addr)
2078 {
2079 }
2080
2081 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
2082 target_phys_addr_t paddr, int prot,
2083 int mmu_idx, int is_softmmu)
2084 {
2085 return 0;
2086 }
2087
2088 /*
2089 * Walks guest process memory "regions" one by one
2090 * and calls callback function 'fn' for each region.
2091 */
2092 int walk_memory_regions(void *priv,
2093 int (*fn)(void *, unsigned long, unsigned long, unsigned long))
2094 {
2095 unsigned long start, end;
2096 PageDesc *p = NULL;
2097 int i, j, prot, prot1;
2098 int rc = 0;
2099
2100 start = end = -1;
2101 prot = 0;
2102
2103 for (i = 0; i <= L1_SIZE; i++) {
2104 p = (i < L1_SIZE) ? l1_map[i] : NULL;
2105 for (j = 0; j < L2_SIZE; j++) {
2106 prot1 = (p == NULL) ? 0 : p[j].flags;
2107 /*
2108 * "region" is one continuous chunk of memory
2109 * that has same protection flags set.
2110 */
2111 if (prot1 != prot) {
2112 end = (i << (32 - L1_BITS)) | (j << TARGET_PAGE_BITS);
2113 if (start != -1) {
2114 rc = (*fn)(priv, start, end, prot);
2115 /* callback can stop iteration by returning != 0 */
2116 if (rc != 0)
2117 return (rc);
2118 }
2119 if (prot1 != 0)
2120 start = end;
2121 else
2122 start = -1;
2123 prot = prot1;
2124 }
2125 if (p == NULL)
2126 break;
2127 }
2128 }
2129 return (rc);
2130 }
2131
2132 static int dump_region(void *priv, unsigned long start,
2133 unsigned long end, unsigned long prot)
2134 {
2135 FILE *f = (FILE *)priv;
2136
2137 (void) fprintf(f, "%08lx-%08lx %08lx %c%c%c\n",
2138 start, end, end - start,
2139 ((prot & PAGE_READ) ? 'r' : '-'),
2140 ((prot & PAGE_WRITE) ? 'w' : '-'),
2141 ((prot & PAGE_EXEC) ? 'x' : '-'));
2142
2143 return (0);
2144 }
2145
2146 /* dump memory mappings */
2147 void page_dump(FILE *f)
2148 {
2149 (void) fprintf(f, "%-8s %-8s %-8s %s\n",
2150 "start", "end", "size", "prot");
2151 walk_memory_regions(f, dump_region);
2152 }
2153
2154 int page_get_flags(target_ulong address)
2155 {
2156 PageDesc *p;
2157
2158 p = page_find(address >> TARGET_PAGE_BITS);
2159 if (!p)
2160 return 0;
2161 return p->flags;
2162 }
2163
2164 /* modify the flags of a page and invalidate the code if
2165 necessary. The flag PAGE_WRITE_ORG is positioned automatically
2166 depending on PAGE_WRITE */
2167 void page_set_flags(target_ulong start, target_ulong end, int flags)
2168 {
2169 PageDesc *p;
2170 target_ulong addr;
2171
2172 /* mmap_lock should already be held. */
2173 start = start & TARGET_PAGE_MASK;
2174 end = TARGET_PAGE_ALIGN(end);
2175 if (flags & PAGE_WRITE)
2176 flags |= PAGE_WRITE_ORG;
2177 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2178 p = page_find_alloc(addr >> TARGET_PAGE_BITS);
2179 /* We may be called for host regions that are outside guest
2180 address space. */
2181 if (!p)
2182 return;
2183 /* if the write protection is set, then we invalidate the code
2184 inside */
2185 if (!(p->flags & PAGE_WRITE) &&
2186 (flags & PAGE_WRITE) &&
2187 p->first_tb) {
2188 tb_invalidate_phys_page(addr, 0, NULL);
2189 }
2190 p->flags = flags;
2191 }
2192 }
2193
2194 int page_check_range(target_ulong start, target_ulong len, int flags)
2195 {
2196 PageDesc *p;
2197 target_ulong end;
2198 target_ulong addr;
2199
2200 if (start + len < start)
2201 /* we've wrapped around */
2202 return -1;
2203
2204 end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2205 start = start & TARGET_PAGE_MASK;
2206
2207 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2208 p = page_find(addr >> TARGET_PAGE_BITS);
2209 if( !p )
2210 return -1;
2211 if( !(p->flags & PAGE_VALID) )
2212 return -1;
2213
2214 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2215 return -1;
2216 if (flags & PAGE_WRITE) {
2217 if (!(p->flags & PAGE_WRITE_ORG))
2218 return -1;
2219 /* unprotect the page if it was put read-only because it
2220 contains translated code */
2221 if (!(p->flags & PAGE_WRITE)) {
2222 if (!page_unprotect(addr, 0, NULL))
2223 return -1;
2224 }
2225 return 0;
2226 }
2227 }
2228 return 0;
2229 }
2230
2231 /* called from signal handler: invalidate the code and unprotect the
2232 page. Return TRUE if the fault was successfully handled. */
2233 int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2234 {
2235 unsigned int page_index, prot, pindex;
2236 PageDesc *p, *p1;
2237 target_ulong host_start, host_end, addr;
2238
2239 /* Technically this isn't safe inside a signal handler. However we
2240 know this only ever happens in a synchronous SEGV handler, so in
2241 practice it seems to be ok. */
2242 mmap_lock();
2243
2244 host_start = address & qemu_host_page_mask;
2245 page_index = host_start >> TARGET_PAGE_BITS;
2246 p1 = page_find(page_index);
2247 if (!p1) {
2248 mmap_unlock();
2249 return 0;
2250 }
2251 host_end = host_start + qemu_host_page_size;
2252 p = p1;
2253 prot = 0;
2254 for(addr = host_start;addr < host_end; addr += TARGET_PAGE_SIZE) {
2255 prot |= p->flags;
2256 p++;
2257 }
2258 /* if the page was really writable, then we change its
2259 protection back to writable */
2260 if (prot & PAGE_WRITE_ORG) {
2261 pindex = (address - host_start) >> TARGET_PAGE_BITS;
2262 if (!(p1[pindex].flags & PAGE_WRITE)) {
2263 mprotect((void *)g2h(host_start), qemu_host_page_size,
2264 (prot & PAGE_BITS) | PAGE_WRITE);
2265 p1[pindex].flags |= PAGE_WRITE;
2266 /* and since the content will be modified, we must invalidate
2267 the corresponding translated code. */
2268 tb_invalidate_phys_page(address, pc, puc);
2269 #ifdef DEBUG_TB_CHECK
2270 tb_invalidate_check(address);
2271 #endif
2272 mmap_unlock();
2273 return 1;
2274 }
2275 }
2276 mmap_unlock();
2277 return 0;
2278 }
2279
2280 static inline void tlb_set_dirty(CPUState *env,
2281 unsigned long addr, target_ulong vaddr)
2282 {
2283 }
2284 #endif /* defined(CONFIG_USER_ONLY) */
2285
2286 #if !defined(CONFIG_USER_ONLY)
2287
2288 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2289 ram_addr_t memory, ram_addr_t region_offset);
2290 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2291 ram_addr_t orig_memory, ram_addr_t region_offset);
2292 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2293 need_subpage) \
2294 do { \
2295 if (addr > start_addr) \
2296 start_addr2 = 0; \
2297 else { \
2298 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2299 if (start_addr2 > 0) \
2300 need_subpage = 1; \
2301 } \
2302 \
2303 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2304 end_addr2 = TARGET_PAGE_SIZE - 1; \
2305 else { \
2306 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2307 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2308 need_subpage = 1; \
2309 } \
2310 } while (0)
2311
2312 /* register physical memory. 'size' must be a multiple of the target
2313 page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2314 io memory page. The address used when calling the IO function is
2315 the offset from the start of the region, plus region_offset. Both
2316 start_addr and region_offset are rounded down to a page boundary
2317 before calculating this offset. This should not be a problem unless
2318 the low bits of start_addr and region_offset differ. */
2319 void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
2320 ram_addr_t size,
2321 ram_addr_t phys_offset,
2322 ram_addr_t region_offset)
2323 {
2324 target_phys_addr_t addr, end_addr;
2325 PhysPageDesc *p;
2326 CPUState *env;
2327 ram_addr_t orig_size = size;
2328 void *subpage;
2329
2330 if (kvm_enabled())
2331 kvm_set_phys_mem(start_addr, size, phys_offset);
2332
2333 if (phys_offset == IO_MEM_UNASSIGNED) {
2334 region_offset = start_addr;
2335 }
2336 region_offset &= TARGET_PAGE_MASK;
2337 size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2338 end_addr = start_addr + (target_phys_addr_t)size;
2339 for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2340 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2341 if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2342 ram_addr_t orig_memory = p->phys_offset;
2343 target_phys_addr_t start_addr2, end_addr2;
2344 int need_subpage = 0;
2345
2346 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2347 need_subpage);
2348 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2349 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2350 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2351 &p->phys_offset, orig_memory,
2352 p->region_offset);
2353 } else {
2354 subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2355 >> IO_MEM_SHIFT];
2356 }
2357 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2358 region_offset);
2359 p->region_offset = 0;
2360 } else {
2361 p->phys_offset = phys_offset;
2362 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2363 (phys_offset & IO_MEM_ROMD))
2364 phys_offset += TARGET_PAGE_SIZE;
2365 }
2366 } else {
2367 p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2368 p->phys_offset = phys_offset;
2369 p->region_offset = region_offset;
2370 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2371 (phys_offset & IO_MEM_ROMD)) {
2372 phys_offset += TARGET_PAGE_SIZE;
2373 } else {
2374 target_phys_addr_t start_addr2, end_addr2;
2375 int need_subpage = 0;
2376
2377 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2378 end_addr2, need_subpage);
2379
2380 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2381 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2382 &p->phys_offset, IO_MEM_UNASSIGNED,
2383 addr & TARGET_PAGE_MASK);
2384 subpage_register(subpage, start_addr2, end_addr2,
2385 phys_offset, region_offset);
2386 p->region_offset = 0;
2387 }
2388 }
2389 }
2390 region_offset += TARGET_PAGE_SIZE;
2391 }
2392
2393 /* since each CPU stores ram addresses in its TLB cache, we must
2394 reset the modified entries */
2395 /* XXX: slow ! */
2396 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2397 tlb_flush(env, 1);
2398 }
2399 }
2400
2401 /* XXX: temporary until new memory mapping API */
2402 ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2403 {
2404 PhysPageDesc *p;
2405
2406 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2407 if (!p)
2408 return IO_MEM_UNASSIGNED;
2409 return p->phys_offset;
2410 }
2411
2412 void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2413 {
2414 if (kvm_enabled())
2415 kvm_coalesce_mmio_region(addr, size);
2416 }
2417
2418 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2419 {
2420 if (kvm_enabled())
2421 kvm_uncoalesce_mmio_region(addr, size);
2422 }
2423
2424 #ifdef __linux__
2425
2426 #include <sys/vfs.h>
2427
2428 #define HUGETLBFS_MAGIC 0x958458f6
2429
2430 static long gethugepagesize(const char *path)
2431 {
2432 struct statfs fs;
2433 int ret;
2434
2435 do {
2436 ret = statfs(path, &fs);
2437 } while (ret != 0 && errno == EINTR);
2438
2439 if (ret != 0) {
2440 perror("statfs");
2441 return 0;
2442 }
2443
2444 if (fs.f_type != HUGETLBFS_MAGIC)
2445 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
2446
2447 return fs.f_bsize;
2448 }
2449
2450 static void *file_ram_alloc(ram_addr_t memory, const char *path)
2451 {
2452 char *filename;
2453 void *area;
2454 int fd;
2455 #ifdef MAP_POPULATE
2456 int flags;
2457 #endif
2458 unsigned long hpagesize;
2459 extern int mem_prealloc;
2460
2461 if (!path) {
2462 return NULL;
2463 }
2464
2465 hpagesize = gethugepagesize(path);
2466 if (!hpagesize) {
2467 return NULL;
2468 }
2469
2470 if (memory < hpagesize) {
2471 return NULL;
2472 }
2473
2474 if (kvm_enabled() && !kvm_has_sync_mmu()) {
2475 fprintf(stderr, "host lacks mmu notifiers, disabling --mem-path\n");
2476 return NULL;
2477 }
2478
2479 if (asprintf(&filename, "%s/kvm.XXXXXX", path) == -1) {
2480 return NULL;
2481 }
2482
2483 fd = mkstemp(filename);
2484 if (fd < 0) {
2485 perror("mkstemp");
2486 free(filename);
2487 return NULL;
2488 }
2489 unlink(filename);
2490 free(filename);
2491
2492 memory = (memory+hpagesize-1) & ~(hpagesize-1);
2493
2494 /*
2495 * ftruncate is not supported by hugetlbfs in older
2496 * hosts, so don't bother checking for errors.
2497 * If anything goes wrong with it under other filesystems,
2498 * mmap will fail.
2499 */
2500 ftruncate(fd, memory);
2501
2502 #ifdef MAP_POPULATE
2503 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
2504 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
2505 * to sidestep this quirk.
2506 */
2507 flags = mem_prealloc ? MAP_POPULATE|MAP_SHARED : MAP_PRIVATE;
2508 area = mmap(0, memory, PROT_READ|PROT_WRITE, flags, fd, 0);
2509 #else
2510 area = mmap(0, memory, PROT_READ|PROT_WRITE, MAP_PRIVATE, fd, 0);
2511 #endif
2512 if (area == MAP_FAILED) {
2513 perror("alloc_mem_area: can't mmap hugetlbfs pages");
2514 close(fd);
2515 return (NULL);
2516 }
2517 return area;
2518 }
2519
2520 #else
2521
2522 static void *file_ram_alloc(ram_addr_t memory, const char *path)
2523 {
2524 return NULL;
2525 }
2526
2527 #endif
2528
2529 extern const char *mem_path;
2530
2531 ram_addr_t qemu_ram_alloc(ram_addr_t size)
2532 {
2533 RAMBlock *new_block;
2534
2535 size = TARGET_PAGE_ALIGN(size);
2536 new_block = qemu_malloc(sizeof(*new_block));
2537
2538 new_block->host = file_ram_alloc(size, mem_path);
2539 if (!new_block->host) {
2540 new_block->host = qemu_vmalloc(size);
2541 }
2542 new_block->offset = last_ram_offset;
2543 new_block->length = size;
2544
2545 new_block->next = ram_blocks;
2546 ram_blocks = new_block;
2547
2548 phys_ram_dirty = qemu_realloc(phys_ram_dirty,
2549 (last_ram_offset + size) >> TARGET_PAGE_BITS);
2550 memset(phys_ram_dirty + (last_ram_offset >> TARGET_PAGE_BITS),
2551 0xff, size >> TARGET_PAGE_BITS);
2552
2553 last_ram_offset += size;
2554
2555 if (kvm_enabled())
2556 kvm_setup_guest_memory(new_block->host, size);
2557
2558 return new_block->offset;
2559 }
2560
2561 void qemu_ram_free(ram_addr_t addr)
2562 {
2563 /* TODO: implement this. */
2564 }
2565
2566 /* Return a host pointer to ram allocated with qemu_ram_alloc.
2567 With the exception of the softmmu code in this file, this should
2568 only be used for local memory (e.g. video ram) that the device owns,
2569 and knows it isn't going to access beyond the end of the block.
2570
2571 It should not be used for general purpose DMA.
2572 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
2573 */
2574 void *qemu_get_ram_ptr(ram_addr_t addr)
2575 {
2576 RAMBlock *prev;
2577 RAMBlock **prevp;
2578 RAMBlock *block;
2579
2580 prev = NULL;
2581 prevp = &ram_blocks;
2582 block = ram_blocks;
2583 while (block && (block->offset > addr
2584 || block->offset + block->length <= addr)) {
2585 if (prev)
2586 prevp = &prev->next;
2587 prev = block;
2588 block = block->next;
2589 }
2590 if (!block) {
2591 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
2592 abort();
2593 }
2594 /* Move this entry to to start of the list. */
2595 if (prev) {
2596 prev->next = block->next;
2597 block->next = *prevp;
2598 *prevp = block;
2599 }
2600 return block->host + (addr - block->offset);
2601 }
2602
2603 /* Some of the softmmu routines need to translate from a host pointer
2604 (typically a TLB entry) back to a ram offset. */
2605 ram_addr_t qemu_ram_addr_from_host(void *ptr)
2606 {
2607 RAMBlock *prev;
2608 RAMBlock **prevp;
2609 RAMBlock *block;
2610 uint8_t *host = ptr;
2611
2612 prev = NULL;
2613 prevp = &ram_blocks;
2614 block = ram_blocks;
2615 while (block && (block->host > host
2616 || block->host + block->length <= host)) {
2617 if (prev)
2618 prevp = &prev->next;
2619 prev = block;
2620 block = block->next;
2621 }
2622 if (!block) {
2623 fprintf(stderr, "Bad ram pointer %p\n", ptr);
2624 abort();
2625 }
2626 return block->offset + (host - block->host);
2627 }
2628
2629 static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2630 {
2631 #ifdef DEBUG_UNASSIGNED
2632 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2633 #endif
2634 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2635 do_unassigned_access(addr, 0, 0, 0, 1);
2636 #endif
2637 return 0;
2638 }
2639
2640 static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2641 {
2642 #ifdef DEBUG_UNASSIGNED
2643 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2644 #endif
2645 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2646 do_unassigned_access(addr, 0, 0, 0, 2);
2647 #endif
2648 return 0;
2649 }
2650
2651 static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
2652 {
2653 #ifdef DEBUG_UNASSIGNED
2654 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2655 #endif
2656 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2657 do_unassigned_access(addr, 0, 0, 0, 4);
2658 #endif
2659 return 0;
2660 }
2661
2662 static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
2663 {
2664 #ifdef DEBUG_UNASSIGNED
2665 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2666 #endif
2667 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2668 do_unassigned_access(addr, 1, 0, 0, 1);
2669 #endif
2670 }
2671
2672 static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
2673 {
2674 #ifdef DEBUG_UNASSIGNED
2675 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2676 #endif
2677 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2678 do_unassigned_access(addr, 1, 0, 0, 2);
2679 #endif
2680 }
2681
2682 static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
2683 {
2684 #ifdef DEBUG_UNASSIGNED
2685 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2686 #endif
2687 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2688 do_unassigned_access(addr, 1, 0, 0, 4);
2689 #endif
2690 }
2691
2692 static CPUReadMemoryFunc * const unassigned_mem_read[3] = {
2693 unassigned_mem_readb,
2694 unassigned_mem_readw,
2695 unassigned_mem_readl,
2696 };
2697
2698 static CPUWriteMemoryFunc * const unassigned_mem_write[3] = {
2699 unassigned_mem_writeb,
2700 unassigned_mem_writew,
2701 unassigned_mem_writel,
2702 };
2703
2704 static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
2705 uint32_t val)
2706 {
2707 int dirty_flags;
2708 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2709 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2710 #if !defined(CONFIG_USER_ONLY)
2711 tb_invalidate_phys_page_fast(ram_addr, 1);
2712 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2713 #endif
2714 }
2715 stb_p(qemu_get_ram_ptr(ram_addr), val);
2716 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2717 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2718 /* we remove the notdirty callback only if the code has been
2719 flushed */
2720 if (dirty_flags == 0xff)
2721 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2722 }
2723
2724 static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
2725 uint32_t val)
2726 {
2727 int dirty_flags;
2728 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2729 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2730 #if !defined(CONFIG_USER_ONLY)
2731 tb_invalidate_phys_page_fast(ram_addr, 2);
2732 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2733 #endif
2734 }
2735 stw_p(qemu_get_ram_ptr(ram_addr), val);
2736 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2737 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2738 /* we remove the notdirty callback only if the code has been
2739 flushed */
2740 if (dirty_flags == 0xff)
2741 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2742 }
2743
2744 static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
2745 uint32_t val)
2746 {
2747 int dirty_flags;
2748 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2749 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2750 #if !defined(CONFIG_USER_ONLY)
2751 tb_invalidate_phys_page_fast(ram_addr, 4);
2752 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2753 #endif
2754 }
2755 stl_p(qemu_get_ram_ptr(ram_addr), val);
2756 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2757 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2758 /* we remove the notdirty callback only if the code has been
2759 flushed */
2760 if (dirty_flags == 0xff)
2761 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2762 }
2763
2764 static CPUReadMemoryFunc * const error_mem_read[3] = {
2765 NULL, /* never used */
2766 NULL, /* never used */
2767 NULL, /* never used */
2768 };
2769
2770 static CPUWriteMemoryFunc * const notdirty_mem_write[3] = {
2771 notdirty_mem_writeb,
2772 notdirty_mem_writew,
2773 notdirty_mem_writel,
2774 };
2775
2776 /* Generate a debug exception if a watchpoint has been hit. */
2777 static void check_watchpoint(int offset, int len_mask, int flags)
2778 {
2779 CPUState *env = cpu_single_env;
2780 target_ulong pc, cs_base;
2781 TranslationBlock *tb;
2782 target_ulong vaddr;
2783 CPUWatchpoint *wp;
2784 int cpu_flags;
2785
2786 if (env->watchpoint_hit) {
2787 /* We re-entered the check after replacing the TB. Now raise
2788 * the debug interrupt so that is will trigger after the
2789 * current instruction. */
2790 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
2791 return;
2792 }
2793 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
2794 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
2795 if ((vaddr == (wp->vaddr & len_mask) ||
2796 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
2797 wp->flags |= BP_WATCHPOINT_HIT;
2798 if (!env->watchpoint_hit) {
2799 env->watchpoint_hit = wp;
2800 tb = tb_find_pc(env->mem_io_pc);
2801 if (!tb) {
2802 cpu_abort(env, "check_watchpoint: could not find TB for "
2803 "pc=%p", (void *)env->mem_io_pc);
2804 }
2805 cpu_restore_state(tb, env, env->mem_io_pc, NULL);
2806 tb_phys_invalidate(tb, -1);
2807 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
2808 env->exception_index = EXCP_DEBUG;
2809 } else {
2810 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
2811 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
2812 }
2813 cpu_resume_from_signal(env, NULL);
2814 }
2815 } else {
2816 wp->flags &= ~BP_WATCHPOINT_HIT;
2817 }
2818 }
2819 }
2820
2821 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2822 so these check for a hit then pass through to the normal out-of-line
2823 phys routines. */
2824 static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
2825 {
2826 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
2827 return ldub_phys(addr);
2828 }
2829
2830 static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
2831 {
2832 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
2833 return lduw_phys(addr);
2834 }
2835
2836 static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
2837 {
2838 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
2839 return ldl_phys(addr);
2840 }
2841
2842 static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
2843 uint32_t val)
2844 {
2845 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
2846 stb_phys(addr, val);
2847 }
2848
2849 static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
2850 uint32_t val)
2851 {
2852 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
2853 stw_phys(addr, val);
2854 }
2855
2856 static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
2857 uint32_t val)
2858 {
2859 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
2860 stl_phys(addr, val);
2861 }
2862
2863 static CPUReadMemoryFunc * const watch_mem_read[3] = {
2864 watch_mem_readb,
2865 watch_mem_readw,
2866 watch_mem_readl,
2867 };
2868
2869 static CPUWriteMemoryFunc * const watch_mem_write[3] = {
2870 watch_mem_writeb,
2871 watch_mem_writew,
2872 watch_mem_writel,
2873 };
2874
2875 static inline uint32_t subpage_readlen (subpage_t *mmio, target_phys_addr_t addr,
2876 unsigned int len)
2877 {
2878 uint32_t ret;
2879 unsigned int idx;
2880
2881 idx = SUBPAGE_IDX(addr);
2882 #if defined(DEBUG_SUBPAGE)
2883 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
2884 mmio, len, addr, idx);
2885 #endif
2886 ret = (**mmio->mem_read[idx][len])(mmio->opaque[idx][0][len],
2887 addr + mmio->region_offset[idx][0][len]);
2888
2889 return ret;
2890 }
2891
2892 static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
2893 uint32_t value, unsigned int len)
2894 {
2895 unsigned int idx;
2896
2897 idx = SUBPAGE_IDX(addr);
2898 #if defined(DEBUG_SUBPAGE)
2899 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n", __func__,
2900 mmio, len, addr, idx, value);
2901 #endif
2902 (**mmio->mem_write[idx][len])(mmio->opaque[idx][1][len],
2903 addr + mmio->region_offset[idx][1][len],
2904 value);
2905 }
2906
2907 static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
2908 {
2909 #if defined(DEBUG_SUBPAGE)
2910 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2911 #endif
2912
2913 return subpage_readlen(opaque, addr, 0);
2914 }
2915
2916 static void subpage_writeb (void *opaque, target_phys_addr_t addr,
2917 uint32_t value)
2918 {
2919 #if defined(DEBUG_SUBPAGE)
2920 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2921 #endif
2922 subpage_writelen(opaque, addr, value, 0);
2923 }
2924
2925 static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
2926 {
2927 #if defined(DEBUG_SUBPAGE)
2928 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2929 #endif
2930
2931 return subpage_readlen(opaque, addr, 1);
2932 }
2933
2934 static void subpage_writew (void *opaque, target_phys_addr_t addr,
2935 uint32_t value)
2936 {
2937 #if defined(DEBUG_SUBPAGE)
2938 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2939 #endif
2940 subpage_writelen(opaque, addr, value, 1);
2941 }
2942
2943 static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
2944 {
2945 #if defined(DEBUG_SUBPAGE)
2946 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2947 #endif
2948
2949 return subpage_readlen(opaque, addr, 2);
2950 }
2951
2952 static void subpage_writel (void *opaque,
2953 target_phys_addr_t addr, uint32_t value)
2954 {
2955 #if defined(DEBUG_SUBPAGE)
2956 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2957 #endif
2958 subpage_writelen(opaque, addr, value, 2);
2959 }
2960
2961 static CPUReadMemoryFunc * const subpage_read[] = {
2962 &subpage_readb,
2963 &subpage_readw,
2964 &subpage_readl,
2965 };
2966
2967 static CPUWriteMemoryFunc * const subpage_write[] = {
2968 &subpage_writeb,
2969 &subpage_writew,
2970 &subpage_writel,
2971 };
2972
2973 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2974 ram_addr_t memory, ram_addr_t region_offset)
2975 {
2976 int idx, eidx;
2977 unsigned int i;
2978
2979 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2980 return -1;
2981 idx = SUBPAGE_IDX(start);
2982 eidx = SUBPAGE_IDX(end);
2983 #if defined(DEBUG_SUBPAGE)
2984 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
2985 mmio, start, end, idx, eidx, memory);
2986 #endif
2987 memory >>= IO_MEM_SHIFT;
2988 for (; idx <= eidx; idx++) {
2989 for (i = 0; i < 4; i++) {
2990 if (io_mem_read[memory][i]) {
2991 mmio->mem_read[idx][i] = &io_mem_read[memory][i];
2992 mmio->opaque[idx][0][i] = io_mem_opaque[memory];
2993 mmio->region_offset[idx][0][i] = region_offset;
2994 }
2995 if (io_mem_write[memory][i]) {
2996 mmio->mem_write[idx][i] = &io_mem_write[memory][i];
2997 mmio->opaque[idx][1][i] = io_mem_opaque[memory];
2998 mmio->region_offset[idx][1][i] = region_offset;
2999 }
3000 }
3001 }
3002
3003 return 0;
3004 }
3005
3006 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
3007 ram_addr_t orig_memory, ram_addr_t region_offset)
3008 {
3009 subpage_t *mmio;
3010 int subpage_memory;
3011
3012 mmio = qemu_mallocz(sizeof(subpage_t));
3013
3014 mmio->base = base;
3015 subpage_memory = cpu_register_io_memory(subpage_read, subpage_write, mmio);
3016 #if defined(DEBUG_SUBPAGE)
3017 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
3018 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
3019 #endif
3020 *phys = subpage_memory | IO_MEM_SUBPAGE;
3021 subpage_register(mmio, 0, TARGET_PAGE_SIZE - 1, orig_memory,
3022 region_offset);
3023
3024 return mmio;
3025 }
3026
3027 static int get_free_io_mem_idx(void)
3028 {
3029 int i;
3030
3031 for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
3032 if (!io_mem_used[i]) {
3033 io_mem_used[i] = 1;
3034 return i;
3035 }
3036
3037 return -1;
3038 }
3039
3040 /* mem_read and mem_write are arrays of functions containing the
3041 function to access byte (index 0), word (index 1) and dword (index
3042 2). Functions can be omitted with a NULL function pointer.
3043 If io_index is non zero, the corresponding io zone is
3044 modified. If it is zero, a new io zone is allocated. The return
3045 value can be used with cpu_register_physical_memory(). (-1) is
3046 returned if error. */
3047 static int cpu_register_io_memory_fixed(int io_index,
3048 CPUReadMemoryFunc * const *mem_read,
3049 CPUWriteMemoryFunc * const *mem_write,
3050 void *opaque)
3051 {
3052 int i, subwidth = 0;
3053
3054 if (io_index <= 0) {
3055 io_index = get_free_io_mem_idx();
3056 if (io_index == -1)
3057 return io_index;
3058 } else {
3059 io_index >>= IO_MEM_SHIFT;
3060 if (io_index >= IO_MEM_NB_ENTRIES)
3061 return -1;
3062 }
3063
3064 for(i = 0;i < 3; i++) {
3065 if (!mem_read[i] || !mem_write[i])
3066 subwidth = IO_MEM_SUBWIDTH;
3067 io_mem_read[io_index][i] = mem_read[i];
3068 io_mem_write[io_index][i] = mem_write[i];
3069 }
3070 io_mem_opaque[io_index] = opaque;
3071 return (io_index << IO_MEM_SHIFT) | subwidth;
3072 }
3073
3074 int cpu_register_io_memory(CPUReadMemoryFunc * const *mem_read,
3075 CPUWriteMemoryFunc * const *mem_write,
3076 void *opaque)
3077 {
3078 return cpu_register_io_memory_fixed(0, mem_read, mem_write, opaque);
3079 }
3080
3081 void cpu_unregister_io_memory(int io_table_address)
3082 {
3083 int i;
3084 int io_index = io_table_address >> IO_MEM_SHIFT;
3085
3086 for (i=0;i < 3; i++) {
3087 io_mem_read[io_index][i] = unassigned_mem_read[i];
3088 io_mem_write[io_index][i] = unassigned_mem_write[i];
3089 }
3090 io_mem_opaque[io_index] = NULL;
3091 io_mem_used[io_index] = 0;
3092 }
3093
3094 static void io_mem_init(void)
3095 {
3096 int i;
3097
3098 cpu_register_io_memory_fixed(IO_MEM_ROM, error_mem_read, unassigned_mem_write, NULL);
3099 cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED, unassigned_mem_read, unassigned_mem_write, NULL);
3100 cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY, error_mem_read, notdirty_mem_write, NULL);
3101 for (i=0; i<5; i++)
3102 io_mem_used[i] = 1;
3103
3104 io_mem_watch = cpu_register_io_memory(watch_mem_read,
3105 watch_mem_write, NULL);
3106 }
3107
3108 #endif /* !defined(CONFIG_USER_ONLY) */
3109
3110 /* physical memory access (slow version, mainly for debug) */
3111 #if defined(CONFIG_USER_ONLY)
3112 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
3113 int len, int is_write)
3114 {
3115 int l, flags;
3116 target_ulong page;
3117 void * p;
3118
3119 while (len > 0) {
3120 page = addr & TARGET_PAGE_MASK;
3121 l = (page + TARGET_PAGE_SIZE) - addr;
3122 if (l > len)
3123 l = len;
3124 flags = page_get_flags(page);
3125 if (!(flags & PAGE_VALID))
3126 return;
3127 if (is_write) {
3128 if (!(flags & PAGE_WRITE))
3129 return;
3130 /* XXX: this code should not depend on lock_user */
3131 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
3132 /* FIXME - should this return an error rather than just fail? */
3133 return;
3134 memcpy(p, buf, l);
3135 unlock_user(p, addr, l);
3136 } else {
3137 if (!(flags & PAGE_READ))
3138 return;
3139 /* XXX: this code should not depend on lock_user */
3140 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
3141 /* FIXME - should this return an error rather than just fail? */
3142 return;
3143 memcpy(buf, p, l);
3144 unlock_user(p, addr, 0);
3145 }
3146 len -= l;
3147 buf += l;
3148 addr += l;
3149 }
3150 }
3151
3152 #else
3153 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
3154 int len, int is_write)
3155 {
3156 int l, io_index;
3157 uint8_t *ptr;
3158 uint32_t val;
3159 target_phys_addr_t page;
3160 unsigned long pd;
3161 PhysPageDesc *p;
3162
3163 while (len > 0) {
3164 page = addr & TARGET_PAGE_MASK;
3165 l = (page + TARGET_PAGE_SIZE) - addr;
3166 if (l > len)
3167 l = len;
3168 p = phys_page_find(page >> TARGET_PAGE_BITS);
3169 if (!p) {
3170 pd = IO_MEM_UNASSIGNED;
3171 } else {
3172 pd = p->phys_offset;
3173 }
3174
3175 if (is_write) {
3176 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3177 target_phys_addr_t addr1 = addr;
3178 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3179 if (p)
3180 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3181 /* XXX: could force cpu_single_env to NULL to avoid
3182 potential bugs */
3183 if (l >= 4 && ((addr1 & 3) == 0)) {
3184 /* 32 bit write access */
3185 val = ldl_p(buf);
3186 io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
3187 l = 4;
3188 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3189 /* 16 bit write access */
3190 val = lduw_p(buf);
3191 io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
3192 l = 2;
3193 } else {
3194 /* 8 bit write access */
3195 val = ldub_p(buf);
3196 io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
3197 l = 1;
3198 }
3199 } else {
3200 unsigned long addr1;
3201 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3202 /* RAM case */
3203 ptr = qemu_get_ram_ptr(addr1);
3204 memcpy(ptr, buf, l);
3205 if (!cpu_physical_memory_is_dirty(addr1)) {
3206 /* invalidate code */
3207 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3208 /* set dirty bit */
3209 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3210 (0xff & ~CODE_DIRTY_FLAG);
3211 }
3212 /* qemu doesn't execute guest code directly, but kvm does
3213 therefore flush instruction caches */
3214 if (kvm_enabled())
3215 flush_icache_range((unsigned long)ptr,
3216 ((unsigned long)ptr)+l);
3217 }
3218 } else {
3219 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3220 !(pd & IO_MEM_ROMD)) {
3221 target_phys_addr_t addr1 = addr;
3222 /* I/O case */
3223 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3224 if (p)
3225 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3226 if (l >= 4 && ((addr1 & 3) == 0)) {
3227 /* 32 bit read access */
3228 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
3229 stl_p(buf, val);
3230 l = 4;
3231 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3232 /* 16 bit read access */
3233 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
3234 stw_p(buf, val);
3235 l = 2;
3236 } else {
3237 /* 8 bit read access */
3238 val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
3239 stb_p(buf, val);
3240 l = 1;
3241 }
3242 } else {
3243 /* RAM case */
3244 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3245 (addr & ~TARGET_PAGE_MASK);
3246 memcpy(buf, ptr, l);
3247 }
3248 }
3249 len -= l;
3250 buf += l;
3251 addr += l;
3252 }
3253 }
3254
3255 /* used for ROM loading : can write in RAM and ROM */
3256 void cpu_physical_memory_write_rom(target_phys_addr_t addr,
3257 const uint8_t *buf, int len)
3258 {
3259 int l;
3260 uint8_t *ptr;
3261 target_phys_addr_t page;
3262 unsigned long pd;
3263 PhysPageDesc *p;
3264
3265 while (len > 0) {
3266 page = addr & TARGET_PAGE_MASK;
3267 l = (page + TARGET_PAGE_SIZE) - addr;
3268 if (l > len)
3269 l = len;
3270 p = phys_page_find(page >> TARGET_PAGE_BITS);
3271 if (!p) {
3272 pd = IO_MEM_UNASSIGNED;
3273 } else {
3274 pd = p->phys_offset;
3275 }
3276
3277 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3278 (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3279 !(pd & IO_MEM_ROMD)) {
3280 /* do nothing */
3281 } else {
3282 unsigned long addr1;
3283 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3284 /* ROM/RAM case */
3285 ptr = qemu_get_ram_ptr(addr1);
3286 memcpy(ptr, buf, l);
3287 }
3288 len -= l;
3289 buf += l;
3290 addr += l;
3291 }
3292 }
3293
3294 typedef struct {
3295 void *buffer;
3296 target_phys_addr_t addr;
3297 target_phys_addr_t len;
3298 } BounceBuffer;
3299
3300 static BounceBuffer bounce;
3301
3302 typedef struct MapClient {
3303 void *opaque;
3304 void (*callback)(void *opaque);
3305 QLIST_ENTRY(MapClient) link;
3306 } MapClient;
3307
3308 static QLIST_HEAD(map_client_list, MapClient) map_client_list
3309 = QLIST_HEAD_INITIALIZER(map_client_list);
3310
3311 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
3312 {
3313 MapClient *client = qemu_malloc(sizeof(*client));
3314
3315 client->opaque = opaque;
3316 client->callback = callback;
3317 QLIST_INSERT_HEAD(&map_client_list, client, link);
3318 return client;
3319 }
3320
3321 void cpu_unregister_map_client(void *_client)
3322 {
3323 MapClient *client = (MapClient *)_client;
3324
3325 QLIST_REMOVE(client, link);
3326 qemu_free(client);
3327 }
3328
3329 static void cpu_notify_map_clients(void)
3330 {
3331 MapClient *client;
3332
3333 while (!QLIST_EMPTY(&map_client_list)) {
3334 client = QLIST_FIRST(&map_client_list);
3335 client->callback(client->opaque);
3336 cpu_unregister_map_client(client);
3337 }
3338 }
3339
3340 /* Map a physical memory region into a host virtual address.
3341 * May map a subset of the requested range, given by and returned in *plen.
3342 * May return NULL if resources needed to perform the mapping are exhausted.
3343 * Use only for reads OR writes - not for read-modify-write operations.
3344 * Use cpu_register_map_client() to know when retrying the map operation is
3345 * likely to succeed.
3346 */
3347 void *cpu_physical_memory_map(target_phys_addr_t addr,
3348 target_phys_addr_t *plen,
3349 int is_write)
3350 {
3351 target_phys_addr_t len = *plen;
3352 target_phys_addr_t done = 0;
3353 int l;
3354 uint8_t *ret = NULL;
3355 uint8_t *ptr;
3356 target_phys_addr_t page;
3357 unsigned long pd;
3358 PhysPageDesc *p;
3359 unsigned long addr1;
3360
3361 while (len > 0) {
3362 page = addr & TARGET_PAGE_MASK;
3363 l = (page + TARGET_PAGE_SIZE) - addr;
3364 if (l > len)
3365 l = len;
3366 p = phys_page_find(page >> TARGET_PAGE_BITS);
3367 if (!p) {
3368 pd = IO_MEM_UNASSIGNED;
3369 } else {
3370 pd = p->phys_offset;
3371 }
3372
3373 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3374 if (done || bounce.buffer) {
3375 break;
3376 }
3377 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
3378 bounce.addr = addr;
3379 bounce.len = l;
3380 if (!is_write) {
3381 cpu_physical_memory_rw(addr, bounce.buffer, l, 0);
3382 }
3383 ptr = bounce.buffer;
3384 } else {
3385 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3386 ptr = qemu_get_ram_ptr(addr1);
3387 }
3388 if (!done) {
3389 ret = ptr;
3390 } else if (ret + done != ptr) {
3391 break;
3392 }
3393
3394 len -= l;
3395 addr += l;
3396 done += l;
3397 }
3398 *plen = done;
3399 return ret;
3400 }
3401
3402 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3403 * Will also mark the memory as dirty if is_write == 1. access_len gives
3404 * the amount of memory that was actually read or written by the caller.
3405 */
3406 void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
3407 int is_write, target_phys_addr_t access_len)
3408 {
3409 unsigned long flush_len = (unsigned long)access_len;
3410
3411 if (buffer != bounce.buffer) {
3412 if (is_write) {
3413 ram_addr_t addr1 = qemu_ram_addr_from_host(buffer);
3414 while (access_len) {
3415 unsigned l;
3416 l = TARGET_PAGE_SIZE;
3417 if (l > access_len)
3418 l = access_len;
3419 if (!cpu_physical_memory_is_dirty(addr1)) {
3420 /* invalidate code */
3421 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3422 /* set dirty bit */
3423 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3424 (0xff & ~CODE_DIRTY_FLAG);
3425 }
3426 addr1 += l;
3427 access_len -= l;
3428 }
3429 dma_flush_range((unsigned long)buffer,
3430 (unsigned long)buffer + flush_len);
3431 }
3432 return;
3433 }
3434 if (is_write) {
3435 cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
3436 }
3437 qemu_free(bounce.buffer);
3438 bounce.buffer = NULL;
3439 cpu_notify_map_clients();
3440 }
3441
3442 /* warning: addr must be aligned */
3443 uint32_t ldl_phys(target_phys_addr_t addr)
3444 {
3445 int io_index;
3446 uint8_t *ptr;
3447 uint32_t val;
3448 unsigned long pd;
3449 PhysPageDesc *p;
3450
3451 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3452 if (!p) {
3453 pd = IO_MEM_UNASSIGNED;
3454 } else {
3455 pd = p->phys_offset;
3456 }
3457
3458 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3459 !(pd & IO_MEM_ROMD)) {
3460 /* I/O case */
3461 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3462 if (p)
3463 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3464 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3465 } else {
3466 /* RAM case */
3467 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3468 (addr & ~TARGET_PAGE_MASK);
3469 val = ldl_p(ptr);
3470 }
3471 return val;
3472 }
3473
3474 /* warning: addr must be aligned */
3475 uint64_t ldq_phys(target_phys_addr_t addr)
3476 {
3477 int io_index;
3478 uint8_t *ptr;
3479 uint64_t val;
3480 unsigned long pd;
3481 PhysPageDesc *p;
3482
3483 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3484 if (!p) {
3485 pd = IO_MEM_UNASSIGNED;
3486 } else {
3487 pd = p->phys_offset;
3488 }
3489
3490 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3491 !(pd & IO_MEM_ROMD)) {
3492 /* I/O case */
3493 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3494 if (p)
3495 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3496 #ifdef TARGET_WORDS_BIGENDIAN
3497 val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
3498 val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
3499 #else
3500 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3501 val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
3502 #endif
3503 } else {
3504 /* RAM case */
3505 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3506 (addr & ~TARGET_PAGE_MASK);
3507 val = ldq_p(ptr);
3508 }
3509 return val;
3510 }
3511
3512 /* XXX: optimize */
3513 uint32_t ldub_phys(target_phys_addr_t addr)
3514 {
3515 uint8_t val;
3516 cpu_physical_memory_read(addr, &val, 1);
3517 return val;
3518 }
3519
3520 /* XXX: optimize */
3521 uint32_t lduw_phys(target_phys_addr_t addr)
3522 {
3523 uint16_t val;
3524 cpu_physical_memory_read(addr, (uint8_t *)&val, 2);
3525 return tswap16(val);
3526 }
3527
3528 /* warning: addr must be aligned. The ram page is not masked as dirty
3529 and the code inside is not invalidated. It is useful if the dirty
3530 bits are used to track modified PTEs */
3531 void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
3532 {
3533 int io_index;
3534 uint8_t *ptr;
3535 unsigned long pd;
3536 PhysPageDesc *p;
3537
3538 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3539 if (!p) {
3540 pd = IO_MEM_UNASSIGNED;
3541 } else {
3542 pd = p->phys_offset;
3543 }
3544
3545 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3546 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3547 if (p)
3548 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3549 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3550 } else {
3551 unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3552 ptr = qemu_get_ram_ptr(addr1);
3553 stl_p(ptr, val);
3554
3555 if (unlikely(in_migration)) {
3556 if (!cpu_physical_memory_is_dirty(addr1)) {
3557 /* invalidate code */
3558 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3559 /* set dirty bit */
3560 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3561 (0xff & ~CODE_DIRTY_FLAG);
3562 }
3563 }
3564 }
3565 }
3566
3567 void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
3568 {
3569 int io_index;
3570 uint8_t *ptr;
3571 unsigned long pd;
3572 PhysPageDesc *p;
3573
3574 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3575 if (!p) {
3576 pd = IO_MEM_UNASSIGNED;
3577 } else {
3578 pd = p->phys_offset;
3579 }
3580
3581 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3582 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3583 if (p)
3584 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3585 #ifdef TARGET_WORDS_BIGENDIAN
3586 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
3587 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
3588 #else
3589 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3590 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
3591 #endif
3592 } else {
3593 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3594 (addr & ~TARGET_PAGE_MASK);
3595 stq_p(ptr, val);
3596 }
3597 }
3598
3599 /* warning: addr must be aligned */
3600 void stl_phys(target_phys_addr_t addr, uint32_t val)
3601 {
3602 int io_index;
3603 uint8_t *ptr;
3604 unsigned long pd;
3605 PhysPageDesc *p;
3606
3607 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3608 if (!p) {
3609 pd = IO_MEM_UNASSIGNED;
3610 } else {
3611 pd = p->phys_offset;
3612 }
3613
3614 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3615 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3616 if (p)
3617 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3618 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3619 } else {
3620 unsigned long addr1;
3621 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3622 /* RAM case */
3623 ptr = qemu_get_ram_ptr(addr1);
3624 stl_p(ptr, val);
3625 if (!cpu_physical_memory_is_dirty(addr1)) {
3626 /* invalidate code */
3627 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3628 /* set dirty bit */
3629 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3630 (0xff & ~CODE_DIRTY_FLAG);
3631 }
3632 }
3633 }
3634
3635 /* XXX: optimize */
3636 void stb_phys(target_phys_addr_t addr, uint32_t val)
3637 {
3638 uint8_t v = val;
3639 cpu_physical_memory_write(addr, &v, 1);
3640 }
3641
3642 /* XXX: optimize */
3643 void stw_phys(target_phys_addr_t addr, uint32_t val)
3644 {
3645 uint16_t v = tswap16(val);
3646 cpu_physical_memory_write(addr, (const uint8_t *)&v, 2);
3647 }
3648
3649 /* XXX: optimize */
3650 void stq_phys(target_phys_addr_t addr, uint64_t val)
3651 {
3652 val = tswap64(val);
3653 cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
3654 }
3655
3656 #endif
3657
3658 /* virtual memory access for debug (includes writing to ROM) */
3659 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3660 uint8_t *buf, int len, int is_write)
3661 {
3662 int l;
3663 target_phys_addr_t phys_addr;
3664 target_ulong page;
3665
3666 while (len > 0) {
3667 page = addr & TARGET_PAGE_MASK;
3668 phys_addr = cpu_get_phys_page_debug(env, page);
3669 /* if no physical page mapped, return an error */
3670 if (phys_addr == -1)
3671 return -1;
3672 l = (page + TARGET_PAGE_SIZE) - addr;
3673 if (l > len)
3674 l = len;
3675 phys_addr += (addr & ~TARGET_PAGE_MASK);
3676 #if !defined(CONFIG_USER_ONLY)
3677 if (is_write)
3678 cpu_physical_memory_write_rom(phys_addr, buf, l);
3679 else
3680 #endif
3681 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
3682 len -= l;
3683 buf += l;
3684 addr += l;
3685 }
3686 return 0;
3687 }
3688
3689 /* in deterministic execution mode, instructions doing device I/Os
3690 must be at the end of the TB */
3691 void cpu_io_recompile(CPUState *env, void *retaddr)
3692 {
3693 TranslationBlock *tb;
3694 uint32_t n, cflags;
3695 target_ulong pc, cs_base;
3696 uint64_t flags;
3697
3698 tb = tb_find_pc((unsigned long)retaddr);
3699 if (!tb) {
3700 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
3701 retaddr);
3702 }
3703 n = env->icount_decr.u16.low + tb->icount;
3704 cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
3705 /* Calculate how many instructions had been executed before the fault
3706 occurred. */
3707 n = n - env->icount_decr.u16.low;
3708 /* Generate a new TB ending on the I/O insn. */
3709 n++;
3710 /* On MIPS and SH, delay slot instructions can only be restarted if
3711 they were already the first instruction in the TB. If this is not
3712 the first instruction in a TB then re-execute the preceding
3713 branch. */
3714 #if defined(TARGET_MIPS)
3715 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
3716 env->active_tc.PC -= 4;
3717 env->icount_decr.u16.low++;
3718 env->hflags &= ~MIPS_HFLAG_BMASK;
3719 }
3720 #elif defined(TARGET_SH4)
3721 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
3722 && n > 1) {
3723 env->pc -= 2;
3724 env->icount_decr.u16.low++;
3725 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
3726 }
3727 #endif
3728 /* This should never happen. */
3729 if (n > CF_COUNT_MASK)
3730 cpu_abort(env, "TB too big during recompile");
3731
3732 cflags = n | CF_LAST_IO;
3733 pc = tb->pc;
3734 cs_base = tb->cs_base;
3735 flags = tb->flags;
3736 tb_phys_invalidate(tb, -1);
3737 /* FIXME: In theory this could raise an exception. In practice
3738 we have already translated the block once so it's probably ok. */
3739 tb_gen_code(env, pc, cs_base, flags, cflags);
3740 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
3741 the first in the TB) then we end up generating a whole new TB and
3742 repeating the fault, which is horribly inefficient.
3743 Better would be to execute just this insn uncached, or generate a
3744 second new TB. */
3745 cpu_resume_from_signal(env, NULL);
3746 }
3747
3748 void dump_exec_info(FILE *f,
3749 int (*cpu_fprintf)(FILE *f, const char *fmt, ...))
3750 {
3751 int i, target_code_size, max_target_code_size;
3752 int direct_jmp_count, direct_jmp2_count, cross_page;
3753 TranslationBlock *tb;
3754
3755 target_code_size = 0;
3756 max_target_code_size = 0;
3757 cross_page = 0;
3758 direct_jmp_count = 0;
3759 direct_jmp2_count = 0;
3760 for(i = 0; i < nb_tbs; i++) {
3761 tb = &tbs[i];
3762 target_code_size += tb->size;
3763 if (tb->size > max_target_code_size)
3764 max_target_code_size = tb->size;
3765 if (tb->page_addr[1] != -1)
3766 cross_page++;
3767 if (tb->tb_next_offset[0] != 0xffff) {
3768 direct_jmp_count++;
3769 if (tb->tb_next_offset[1] != 0xffff) {
3770 direct_jmp2_count++;
3771 }
3772 }
3773 }
3774 /* XXX: avoid using doubles ? */
3775 cpu_fprintf(f, "Translation buffer state:\n");
3776 cpu_fprintf(f, "gen code size %ld/%ld\n",
3777 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
3778 cpu_fprintf(f, "TB count %d/%d\n",
3779 nb_tbs, code_gen_max_blocks);
3780 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
3781 nb_tbs ? target_code_size / nb_tbs : 0,
3782 max_target_code_size);
3783 cpu_fprintf(f, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
3784 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
3785 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
3786 cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
3787 cross_page,
3788 nb_tbs ? (cross_page * 100) / nb_tbs : 0);
3789 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
3790 direct_jmp_count,
3791 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
3792 direct_jmp2_count,
3793 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
3794 cpu_fprintf(f, "\nStatistics:\n");
3795 cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
3796 cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
3797 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
3798 tcg_dump_info(f, cpu_fprintf);
3799 }
3800
3801 #if !defined(CONFIG_USER_ONLY)
3802
3803 #define MMUSUFFIX _cmmu
3804 #define GETPC() NULL
3805 #define env cpu_single_env
3806 #define SOFTMMU_CODE_ACCESS
3807
3808 #define SHIFT 0
3809 #include "softmmu_template.h"
3810
3811 #define SHIFT 1
3812 #include "softmmu_template.h"
3813
3814 #define SHIFT 2
3815 #include "softmmu_template.h"
3816
3817 #define SHIFT 3
3818 #include "softmmu_template.h"
3819
3820 #undef env
3821
3822 #endif
AMD IOMMU emulation for KVM