?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
kvm: bios: resolve memory device roll over reporting issues with >32G guests
[qemu-kvm/fedora.git] / exec.c
blob8f85f5801bd3dfd078575158e118e1ad4004ec52
1 /*
2 * virtual page mapping and translated block handling
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 */
20 #include "config.h"
21 #ifdef _WIN32
22 #define WIN32_LEAN_AND_MEAN
23 #include <windows.h>
24 #else
25 #include <sys/types.h>
26 #include <sys/mman.h>
27 #endif
28 #include <stdlib.h>
29 #include <stdio.h>
30 #include <stdarg.h>
31 #include <string.h>
32 #include <errno.h>
33 #include <unistd.h>
34 #include <inttypes.h>
35
36 #include "cpu.h"
37 #include "exec-all.h"
38 #include "qemu-common.h"
39
40 #if !defined(TARGET_IA64)
41 #include "tcg.h"
42 #endif
43 #include "qemu-kvm.h"
44
45 #include "hw/hw.h"
46 #include "osdep.h"
47 #include "kvm.h"
48 #if defined(CONFIG_USER_ONLY)
49 #include <qemu.h>
50 #endif
51
52 //#define DEBUG_TB_INVALIDATE
53 //#define DEBUG_FLUSH
54 //#define DEBUG_TLB
55 //#define DEBUG_UNASSIGNED
56
57 /* make various TB consistency checks */
58 //#define DEBUG_TB_CHECK
59 //#define DEBUG_TLB_CHECK
60
61 //#define DEBUG_IOPORT
62 //#define DEBUG_SUBPAGE
63
64 #if !defined(CONFIG_USER_ONLY)
65 /* TB consistency checks only implemented for usermode emulation. */
66 #undef DEBUG_TB_CHECK
67 #endif
68
69 #define SMC_BITMAP_USE_THRESHOLD 10
70
71 #define MMAP_AREA_START 0x00000000
72 #define MMAP_AREA_END 0xa8000000
73
74 #if defined(TARGET_SPARC64)
75 #define TARGET_PHYS_ADDR_SPACE_BITS 41
76 #elif defined(TARGET_SPARC)
77 #define TARGET_PHYS_ADDR_SPACE_BITS 36
78 #elif defined(TARGET_ALPHA)
79 #define TARGET_PHYS_ADDR_SPACE_BITS 42
80 #define TARGET_VIRT_ADDR_SPACE_BITS 42
81 #elif defined(TARGET_PPC64)
82 #define TARGET_PHYS_ADDR_SPACE_BITS 42
83 #elif defined(TARGET_X86_64) && !defined(USE_KQEMU)
84 #define TARGET_PHYS_ADDR_SPACE_BITS 42
85 #elif defined(TARGET_I386) && !defined(USE_KQEMU)
86 #define TARGET_PHYS_ADDR_SPACE_BITS 36
87 #elif defined(TARGET_IA64)
88 #define TARGET_PHYS_ADDR_SPACE_BITS 36
89 #else
90 /* Note: for compatibility with kqemu, we use 32 bits for x86_64 */
91 #define TARGET_PHYS_ADDR_SPACE_BITS 32
92 #endif
93
94 static TranslationBlock *tbs;
95 int code_gen_max_blocks;
96 TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
97 static int nb_tbs;
98 /* any access to the tbs or the page table must use this lock */
99 spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
100
101 #if defined(__arm__) || defined(__sparc_v9__)
102 /* The prologue must be reachable with a direct jump. ARM and Sparc64
103 have limited branch ranges (possibly also PPC) so place it in a
104 section close to code segment. */
105 #define code_gen_section \
106 __attribute__((__section__(".gen_code"))) \
107 __attribute__((aligned (32)))
108 #else
109 #define code_gen_section \
110 __attribute__((aligned (32)))
111 #endif
112
113 uint8_t code_gen_prologue[1024] code_gen_section;
114 static uint8_t *code_gen_buffer;
115 static unsigned long code_gen_buffer_size;
116 /* threshold to flush the translated code buffer */
117 static unsigned long code_gen_buffer_max_size;
118 uint8_t *code_gen_ptr;
119
120 #if !defined(CONFIG_USER_ONLY)
121 ram_addr_t phys_ram_size;
122 int phys_ram_fd;
123 uint8_t *phys_ram_base;
124 uint8_t *phys_ram_dirty;
125 uint8_t *bios_mem;
126 static int in_migration;
127 static ram_addr_t phys_ram_alloc_offset = 0;
128 #endif
129
130 CPUState *first_cpu;
131 /* current CPU in the current thread. It is only valid inside
132 cpu_exec() */
133 CPUState *cpu_single_env;
134 /* 0 = Do not count executed instructions.
135 1 = Precise instruction counting.
136 2 = Adaptive rate instruction counting. */
137 int use_icount = 0;
138 /* Current instruction counter. While executing translated code this may
139 include some instructions that have not yet been executed. */
140 int64_t qemu_icount;
141
142 typedef struct PageDesc {
143 /* list of TBs intersecting this ram page */
144 TranslationBlock *first_tb;
145 /* in order to optimize self modifying code, we count the number
146 of lookups we do to a given page to use a bitmap */
147 unsigned int code_write_count;
148 uint8_t *code_bitmap;
149 #if defined(CONFIG_USER_ONLY)
150 unsigned long flags;
151 #endif
152 } PageDesc;
153
154 typedef struct PhysPageDesc {
155 /* offset in host memory of the page + io_index in the low bits */
156 ram_addr_t phys_offset;
157 } PhysPageDesc;
158
159 #define L2_BITS 10
160 #if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
161 /* XXX: this is a temporary hack for alpha target.
162 * In the future, this is to be replaced by a multi-level table
163 * to actually be able to handle the complete 64 bits address space.
164 */
165 #define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
166 #else
167 #define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
168 #endif
169
170 #define L1_SIZE (1 << L1_BITS)
171 #define L2_SIZE (1 << L2_BITS)
172
173 unsigned long qemu_real_host_page_size;
174 unsigned long qemu_host_page_bits;
175 unsigned long qemu_host_page_size;
176 unsigned long qemu_host_page_mask;
177
178 /* XXX: for system emulation, it could just be an array */
179 static PageDesc *l1_map[L1_SIZE];
180 static PhysPageDesc **l1_phys_map;
181
182 #if !defined(CONFIG_USER_ONLY)
183 static void io_mem_init(void);
184
185 /* io memory support */
186 CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
187 CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
188 void *io_mem_opaque[IO_MEM_NB_ENTRIES];
189 char io_mem_used[IO_MEM_NB_ENTRIES];
190 static int io_mem_watch;
191 #endif
192
193 /* log support */
194 static const char *logfilename = "/tmp/qemu.log";
195 FILE *logfile;
196 int loglevel;
197 static int log_append = 0;
198
199 /* statistics */
200 static int tlb_flush_count;
201 static int tb_flush_count;
202 static int tb_phys_invalidate_count;
203
204 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
205 typedef struct subpage_t {
206 target_phys_addr_t base;
207 CPUReadMemoryFunc **mem_read[TARGET_PAGE_SIZE][4];
208 CPUWriteMemoryFunc **mem_write[TARGET_PAGE_SIZE][4];
209 void *opaque[TARGET_PAGE_SIZE][2][4];
210 } subpage_t;
211
212 #ifdef _WIN32
213 static void map_exec(void *addr, long size)
214 {
215 DWORD old_protect;
216 VirtualProtect(addr, size,
217 PAGE_EXECUTE_READWRITE, &old_protect);
218
219 }
220 #else
221 static void map_exec(void *addr, long size)
222 {
223 unsigned long start, end, page_size;
224
225 page_size = getpagesize();
226 start = (unsigned long)addr;
227 start &= ~(page_size - 1);
228
229 end = (unsigned long)addr + size;
230 end += page_size - 1;
231 end &= ~(page_size - 1);
232
233 mprotect((void *)start, end - start,
234 PROT_READ | PROT_WRITE | PROT_EXEC);
235 }
236 #endif
237
238 static void page_init(void)
239 {
240 /* NOTE: we can always suppose that qemu_host_page_size >=
241 TARGET_PAGE_SIZE */
242 #ifdef _WIN32
243 {
244 SYSTEM_INFO system_info;
245
246 GetSystemInfo(&system_info);
247 qemu_real_host_page_size = system_info.dwPageSize;
248 }
249 #else
250 qemu_real_host_page_size = getpagesize();
251 #endif
252 if (qemu_host_page_size == 0)
253 qemu_host_page_size = qemu_real_host_page_size;
254 if (qemu_host_page_size < TARGET_PAGE_SIZE)
255 qemu_host_page_size = TARGET_PAGE_SIZE;
256 qemu_host_page_bits = 0;
257 while ((1 << qemu_host_page_bits) < qemu_host_page_size)
258 qemu_host_page_bits++;
259 qemu_host_page_mask = ~(qemu_host_page_size - 1);
260 l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));
261 memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));
262
263 #if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
264 {
265 long long startaddr, endaddr;
266 FILE *f;
267 int n;
268
269 mmap_lock();
270 last_brk = (unsigned long)sbrk(0);
271 f = fopen("/proc/self/maps", "r");
272 if (f) {
273 do {
274 n = fscanf (f, "%llx-%llx %*[^\n]\n", &startaddr, &endaddr);
275 if (n == 2) {
276 startaddr = MIN(startaddr,
277 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
278 endaddr = MIN(endaddr,
279 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
280 page_set_flags(startaddr & TARGET_PAGE_MASK,
281 TARGET_PAGE_ALIGN(endaddr),
282 PAGE_RESERVED);
283 }
284 } while (!feof(f));
285 fclose(f);
286 }
287 mmap_unlock();
288 }
289 #endif
290 }
291
292 static inline PageDesc **page_l1_map(target_ulong index)
293 {
294 #if TARGET_LONG_BITS > 32
295 /* Host memory outside guest VM. For 32-bit targets we have already
296 excluded high addresses. */
297 if (index > ((target_ulong)L2_SIZE * L1_SIZE))
298 return NULL;
299 #endif
300 return &l1_map[index >> L2_BITS];
301 }
302
303 static inline PageDesc *page_find_alloc(target_ulong index)
304 {
305 PageDesc **lp, *p;
306 lp = page_l1_map(index);
307 if (!lp)
308 return NULL;
309
310 p = *lp;
311 if (!p) {
312 /* allocate if not found */
313 #if defined(CONFIG_USER_ONLY)
314 unsigned long addr;
315 size_t len = sizeof(PageDesc) * L2_SIZE;
316 /* Don't use qemu_malloc because it may recurse. */
317 p = mmap(0, len, PROT_READ | PROT_WRITE,
318 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
319 *lp = p;
320 addr = h2g(p);
321 if (addr == (target_ulong)addr) {
322 page_set_flags(addr & TARGET_PAGE_MASK,
323 TARGET_PAGE_ALIGN(addr + len),
324 PAGE_RESERVED);
325 }
326 #else
327 p = qemu_mallocz(sizeof(PageDesc) * L2_SIZE);
328 *lp = p;
329 #endif
330 }
331 return p + (index & (L2_SIZE - 1));
332 }
333
334 static inline PageDesc *page_find(target_ulong index)
335 {
336 PageDesc **lp, *p;
337 lp = page_l1_map(index);
338 if (!lp)
339 return NULL;
340
341 p = *lp;
342 if (!p)
343 return 0;
344 return p + (index & (L2_SIZE - 1));
345 }
346
347 static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
348 {
349 void **lp, **p;
350 PhysPageDesc *pd;
351
352 p = (void **)l1_phys_map;
353 #if TARGET_PHYS_ADDR_SPACE_BITS > 32
354
355 #if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
356 #error unsupported TARGET_PHYS_ADDR_SPACE_BITS
357 #endif
358 lp = p + ((index >> (L1_BITS + L2_BITS)) & (L1_SIZE - 1));
359 p = *lp;
360 if (!p) {
361 /* allocate if not found */
362 if (!alloc)
363 return NULL;
364 p = qemu_vmalloc(sizeof(void *) * L1_SIZE);
365 memset(p, 0, sizeof(void *) * L1_SIZE);
366 *lp = p;
367 }
368 #endif
369 lp = p + ((index >> L2_BITS) & (L1_SIZE - 1));
370 pd = *lp;
371 if (!pd) {
372 int i;
373 /* allocate if not found */
374 if (!alloc)
375 return NULL;
376 pd = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);
377 *lp = pd;
378 for (i = 0; i < L2_SIZE; i++)
379 pd[i].phys_offset = IO_MEM_UNASSIGNED;
380 }
381 return ((PhysPageDesc *)pd) + (index & (L2_SIZE - 1));
382 }
383
384 static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
385 {
386 return phys_page_find_alloc(index, 0);
387 }
388
389 #if !defined(CONFIG_USER_ONLY)
390 static void tlb_protect_code(ram_addr_t ram_addr);
391 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
392 target_ulong vaddr);
393 #define mmap_lock() do { } while(0)
394 #define mmap_unlock() do { } while(0)
395 #endif
396
397 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
398
399 #if defined(CONFIG_USER_ONLY)
400 /* Currently it is not recommanded to allocate big chunks of data in
401 user mode. It will change when a dedicated libc will be used */
402 #define USE_STATIC_CODE_GEN_BUFFER
403 #endif
404
405 #ifdef USE_STATIC_CODE_GEN_BUFFER
406 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE];
407 #endif
408
409 static void code_gen_alloc(unsigned long tb_size)
410 {
411 if (kvm_enabled())
412 return;
413
414 #ifdef USE_STATIC_CODE_GEN_BUFFER
415 code_gen_buffer = static_code_gen_buffer;
416 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
417 map_exec(code_gen_buffer, code_gen_buffer_size);
418 #else
419 code_gen_buffer_size = tb_size;
420 if (code_gen_buffer_size == 0) {
421 #if defined(CONFIG_USER_ONLY)
422 /* in user mode, phys_ram_size is not meaningful */
423 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
424 #else
425 /* XXX: needs ajustments */
426 code_gen_buffer_size = (unsigned long)(phys_ram_size / 4);
427 #endif
428 }
429 if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
430 code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
431 /* The code gen buffer location may have constraints depending on
432 the host cpu and OS */
433 #if defined(__linux__)
434 {
435 int flags;
436 void *start = NULL;
437
438 flags = MAP_PRIVATE | MAP_ANONYMOUS;
439 #if defined(__x86_64__)
440 flags |= MAP_32BIT;
441 /* Cannot map more than that */
442 if (code_gen_buffer_size > (800 * 1024 * 1024))
443 code_gen_buffer_size = (800 * 1024 * 1024);
444 #elif defined(__sparc_v9__)
445 // Map the buffer below 2G, so we can use direct calls and branches
446 flags |= MAP_FIXED;
447 start = (void *) 0x60000000UL;
448 if (code_gen_buffer_size > (512 * 1024 * 1024))
449 code_gen_buffer_size = (512 * 1024 * 1024);
450 #endif
451 code_gen_buffer = mmap(start, code_gen_buffer_size,
452 PROT_WRITE | PROT_READ | PROT_EXEC,
453 flags, -1, 0);
454 if (code_gen_buffer == MAP_FAILED) {
455 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
456 exit(1);
457 }
458 }
459 #elif defined(__FreeBSD__)
460 {
461 int flags;
462 void *addr = NULL;
463 flags = MAP_PRIVATE | MAP_ANONYMOUS;
464 #if defined(__x86_64__)
465 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
466 * 0x40000000 is free */
467 flags |= MAP_FIXED;
468 addr = (void *)0x40000000;
469 /* Cannot map more than that */
470 if (code_gen_buffer_size > (800 * 1024 * 1024))
471 code_gen_buffer_size = (800 * 1024 * 1024);
472 #endif
473 code_gen_buffer = mmap(addr, code_gen_buffer_size,
474 PROT_WRITE | PROT_READ | PROT_EXEC,
475 flags, -1, 0);
476 if (code_gen_buffer == MAP_FAILED) {
477 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
478 exit(1);
479 }
480 }
481 #else
482 code_gen_buffer = qemu_malloc(code_gen_buffer_size);
483 if (!code_gen_buffer) {
484 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
485 exit(1);
486 }
487 map_exec(code_gen_buffer, code_gen_buffer_size);
488 #endif
489 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
490 map_exec(code_gen_prologue, sizeof(code_gen_prologue));
491 code_gen_buffer_max_size = code_gen_buffer_size -
492 code_gen_max_block_size();
493 code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
494 tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
495 }
496
497 /* Must be called before using the QEMU cpus. 'tb_size' is the size
498 (in bytes) allocated to the translation buffer. Zero means default
499 size. */
500 void cpu_exec_init_all(unsigned long tb_size)
501 {
502 cpu_gen_init();
503 code_gen_alloc(tb_size);
504 code_gen_ptr = code_gen_buffer;
505 page_init();
506 #if !defined(CONFIG_USER_ONLY)
507 io_mem_init();
508 #endif
509 }
510
511 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
512
513 #define CPU_COMMON_SAVE_VERSION 1
514
515 static void cpu_common_save(QEMUFile *f, void *opaque)
516 {
517 CPUState *env = opaque;
518
519 qemu_put_be32s(f, &env->halted);
520 qemu_put_be32s(f, &env->interrupt_request);
521 }
522
523 static int cpu_common_load(QEMUFile *f, void *opaque, int version_id)
524 {
525 CPUState *env = opaque;
526
527 if (version_id != CPU_COMMON_SAVE_VERSION)
528 return -EINVAL;
529
530 qemu_get_be32s(f, &env->halted);
531 qemu_get_be32s(f, &env->interrupt_request);
532 tlb_flush(env, 1);
533
534 return 0;
535 }
536 #endif
537
538 void cpu_exec_init(CPUState *env)
539 {
540 CPUState **penv;
541 int cpu_index;
542
543 env->next_cpu = NULL;
544 penv = &first_cpu;
545 cpu_index = 0;
546 while (*penv != NULL) {
547 penv = (CPUState **)&(*penv)->next_cpu;
548 cpu_index++;
549 }
550 env->cpu_index = cpu_index;
551 env->nb_watchpoints = 0;
552 #ifdef __WIN32
553 env->thread_id = GetCurrentProcessId();
554 #else
555 env->thread_id = getpid();
556 #endif
557 *penv = env;
558 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
559 register_savevm("cpu_common", cpu_index, CPU_COMMON_SAVE_VERSION,
560 cpu_common_save, cpu_common_load, env);
561 register_savevm("cpu", cpu_index, CPU_SAVE_VERSION,
562 cpu_save, cpu_load, env);
563 #endif
564 }
565
566 static inline void invalidate_page_bitmap(PageDesc *p)
567 {
568 if (p->code_bitmap) {
569 qemu_free(p->code_bitmap);
570 p->code_bitmap = NULL;
571 }
572 p->code_write_count = 0;
573 }
574
575 /* set to NULL all the 'first_tb' fields in all PageDescs */
576 static void page_flush_tb(void)
577 {
578 int i, j;
579 PageDesc *p;
580
581 for(i = 0; i < L1_SIZE; i++) {
582 p = l1_map[i];
583 if (p) {
584 for(j = 0; j < L2_SIZE; j++) {
585 p->first_tb = NULL;
586 invalidate_page_bitmap(p);
587 p++;
588 }
589 }
590 }
591 }
592
593 /* flush all the translation blocks */
594 /* XXX: tb_flush is currently not thread safe */
595 void tb_flush(CPUState *env1)
596 {
597 CPUState *env;
598 #if defined(DEBUG_FLUSH)
599 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
600 (unsigned long)(code_gen_ptr - code_gen_buffer),
601 nb_tbs, nb_tbs > 0 ?
602 ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
603 #endif
604 if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
605 cpu_abort(env1, "Internal error: code buffer overflow\n");
606
607 nb_tbs = 0;
608
609 for(env = first_cpu; env != NULL; env = env->next_cpu) {
610 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
611 }
612
613 memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
614 page_flush_tb();
615
616 code_gen_ptr = code_gen_buffer;
617 /* XXX: flush processor icache at this point if cache flush is
618 expensive */
619 tb_flush_count++;
620 }
621
622 #ifdef DEBUG_TB_CHECK
623
624 static void tb_invalidate_check(target_ulong address)
625 {
626 TranslationBlock *tb;
627 int i;
628 address &= TARGET_PAGE_MASK;
629 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
630 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
631 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
632 address >= tb->pc + tb->size)) {
633 printf("ERROR invalidate: address=%08lx PC=%08lx size=%04x\n",
634 address, (long)tb->pc, tb->size);
635 }
636 }
637 }
638 }
639
640 /* verify that all the pages have correct rights for code */
641 static void tb_page_check(void)
642 {
643 TranslationBlock *tb;
644 int i, flags1, flags2;
645
646 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
647 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
648 flags1 = page_get_flags(tb->pc);
649 flags2 = page_get_flags(tb->pc + tb->size - 1);
650 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
651 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
652 (long)tb->pc, tb->size, flags1, flags2);
653 }
654 }
655 }
656 }
657
658 static void tb_jmp_check(TranslationBlock *tb)
659 {
660 TranslationBlock *tb1;
661 unsigned int n1;
662
663 /* suppress any remaining jumps to this TB */
664 tb1 = tb->jmp_first;
665 for(;;) {
666 n1 = (long)tb1 & 3;
667 tb1 = (TranslationBlock *)((long)tb1 & ~3);
668 if (n1 == 2)
669 break;
670 tb1 = tb1->jmp_next[n1];
671 }
672 /* check end of list */
673 if (tb1 != tb) {
674 printf("ERROR: jmp_list from 0x%08lx\n", (long)tb);
675 }
676 }
677
678 #endif
679
680 /* invalidate one TB */
681 static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
682 int next_offset)
683 {
684 TranslationBlock *tb1;
685 for(;;) {
686 tb1 = *ptb;
687 if (tb1 == tb) {
688 *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
689 break;
690 }
691 ptb = (TranslationBlock **)((char *)tb1 + next_offset);
692 }
693 }
694
695 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
696 {
697 TranslationBlock *tb1;
698 unsigned int n1;
699
700 for(;;) {
701 tb1 = *ptb;
702 n1 = (long)tb1 & 3;
703 tb1 = (TranslationBlock *)((long)tb1 & ~3);
704 if (tb1 == tb) {
705 *ptb = tb1->page_next[n1];
706 break;
707 }
708 ptb = &tb1->page_next[n1];
709 }
710 }
711
712 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
713 {
714 TranslationBlock *tb1, **ptb;
715 unsigned int n1;
716
717 ptb = &tb->jmp_next[n];
718 tb1 = *ptb;
719 if (tb1) {
720 /* find tb(n) in circular list */
721 for(;;) {
722 tb1 = *ptb;
723 n1 = (long)tb1 & 3;
724 tb1 = (TranslationBlock *)((long)tb1 & ~3);
725 if (n1 == n && tb1 == tb)
726 break;
727 if (n1 == 2) {
728 ptb = &tb1->jmp_first;
729 } else {
730 ptb = &tb1->jmp_next[n1];
731 }
732 }
733 /* now we can suppress tb(n) from the list */
734 *ptb = tb->jmp_next[n];
735
736 tb->jmp_next[n] = NULL;
737 }
738 }
739
740 /* reset the jump entry 'n' of a TB so that it is not chained to
741 another TB */
742 static inline void tb_reset_jump(TranslationBlock *tb, int n)
743 {
744 tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
745 }
746
747 void tb_phys_invalidate(TranslationBlock *tb, target_ulong page_addr)
748 {
749 CPUState *env;
750 PageDesc *p;
751 unsigned int h, n1;
752 target_phys_addr_t phys_pc;
753 TranslationBlock *tb1, *tb2;
754
755 /* remove the TB from the hash list */
756 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
757 h = tb_phys_hash_func(phys_pc);
758 tb_remove(&tb_phys_hash[h], tb,
759 offsetof(TranslationBlock, phys_hash_next));
760
761 /* remove the TB from the page list */
762 if (tb->page_addr[0] != page_addr) {
763 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
764 tb_page_remove(&p->first_tb, tb);
765 invalidate_page_bitmap(p);
766 }
767 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
768 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
769 tb_page_remove(&p->first_tb, tb);
770 invalidate_page_bitmap(p);
771 }
772
773 tb_invalidated_flag = 1;
774
775 /* remove the TB from the hash list */
776 h = tb_jmp_cache_hash_func(tb->pc);
777 for(env = first_cpu; env != NULL; env = env->next_cpu) {
778 if (env->tb_jmp_cache[h] == tb)
779 env->tb_jmp_cache[h] = NULL;
780 }
781
782 /* suppress this TB from the two jump lists */
783 tb_jmp_remove(tb, 0);
784 tb_jmp_remove(tb, 1);
785
786 /* suppress any remaining jumps to this TB */
787 tb1 = tb->jmp_first;
788 for(;;) {
789 n1 = (long)tb1 & 3;
790 if (n1 == 2)
791 break;
792 tb1 = (TranslationBlock *)((long)tb1 & ~3);
793 tb2 = tb1->jmp_next[n1];
794 tb_reset_jump(tb1, n1);
795 tb1->jmp_next[n1] = NULL;
796 tb1 = tb2;
797 }
798 tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
799
800 tb_phys_invalidate_count++;
801 }
802
803 static inline void set_bits(uint8_t *tab, int start, int len)
804 {
805 int end, mask, end1;
806
807 end = start + len;
808 tab += start >> 3;
809 mask = 0xff << (start & 7);
810 if ((start & ~7) == (end & ~7)) {
811 if (start < end) {
812 mask &= ~(0xff << (end & 7));
813 *tab |= mask;
814 }
815 } else {
816 *tab++ |= mask;
817 start = (start + 8) & ~7;
818 end1 = end & ~7;
819 while (start < end1) {
820 *tab++ = 0xff;
821 start += 8;
822 }
823 if (start < end) {
824 mask = ~(0xff << (end & 7));
825 *tab |= mask;
826 }
827 }
828 }
829
830 static void build_page_bitmap(PageDesc *p)
831 {
832 int n, tb_start, tb_end;
833 TranslationBlock *tb;
834
835 p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
836 if (!p->code_bitmap)
837 return;
838
839 tb = p->first_tb;
840 while (tb != NULL) {
841 n = (long)tb & 3;
842 tb = (TranslationBlock *)((long)tb & ~3);
843 /* NOTE: this is subtle as a TB may span two physical pages */
844 if (n == 0) {
845 /* NOTE: tb_end may be after the end of the page, but
846 it is not a problem */
847 tb_start = tb->pc & ~TARGET_PAGE_MASK;
848 tb_end = tb_start + tb->size;
849 if (tb_end > TARGET_PAGE_SIZE)
850 tb_end = TARGET_PAGE_SIZE;
851 } else {
852 tb_start = 0;
853 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
854 }
855 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
856 tb = tb->page_next[n];
857 }
858 }
859
860 TranslationBlock *tb_gen_code(CPUState *env,
861 target_ulong pc, target_ulong cs_base,
862 int flags, int cflags)
863 {
864 TranslationBlock *tb;
865 uint8_t *tc_ptr;
866 target_ulong phys_pc, phys_page2, virt_page2;
867 int code_gen_size;
868
869 phys_pc = get_phys_addr_code(env, pc);
870 tb = tb_alloc(pc);
871 if (!tb) {
872 /* flush must be done */
873 tb_flush(env);
874 /* cannot fail at this point */
875 tb = tb_alloc(pc);
876 /* Don't forget to invalidate previous TB info. */
877 tb_invalidated_flag = 1;
878 }
879 tc_ptr = code_gen_ptr;
880 tb->tc_ptr = tc_ptr;
881 tb->cs_base = cs_base;
882 tb->flags = flags;
883 tb->cflags = cflags;
884 cpu_gen_code(env, tb, &code_gen_size);
885 code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
886
887 /* check next page if needed */
888 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
889 phys_page2 = -1;
890 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
891 phys_page2 = get_phys_addr_code(env, virt_page2);
892 }
893 tb_link_phys(tb, phys_pc, phys_page2);
894 return tb;
895 }
896
897 /* invalidate all TBs which intersect with the target physical page
898 starting in range [start;end[. NOTE: start and end must refer to
899 the same physical page. 'is_cpu_write_access' should be true if called
900 from a real cpu write access: the virtual CPU will exit the current
901 TB if code is modified inside this TB. */
902 void tb_invalidate_phys_page_range(target_phys_addr_t start, target_phys_addr_t end,
903 int is_cpu_write_access)
904 {
905 int n, current_tb_modified, current_tb_not_found, current_flags;
906 CPUState *env = cpu_single_env;
907 PageDesc *p;
908 TranslationBlock *tb, *tb_next, *current_tb, *saved_tb;
909 target_ulong tb_start, tb_end;
910 target_ulong current_pc, current_cs_base;
911
912 p = page_find(start >> TARGET_PAGE_BITS);
913 if (!p)
914 return;
915 if (!p->code_bitmap &&
916 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
917 is_cpu_write_access) {
918 /* build code bitmap */
919 build_page_bitmap(p);
920 }
921
922 /* we remove all the TBs in the range [start, end[ */
923 /* XXX: see if in some cases it could be faster to invalidate all the code */
924 current_tb_not_found = is_cpu_write_access;
925 current_tb_modified = 0;
926 current_tb = NULL; /* avoid warning */
927 current_pc = 0; /* avoid warning */
928 current_cs_base = 0; /* avoid warning */
929 current_flags = 0; /* avoid warning */
930 tb = p->first_tb;
931 while (tb != NULL) {
932 n = (long)tb & 3;
933 tb = (TranslationBlock *)((long)tb & ~3);
934 tb_next = tb->page_next[n];
935 /* NOTE: this is subtle as a TB may span two physical pages */
936 if (n == 0) {
937 /* NOTE: tb_end may be after the end of the page, but
938 it is not a problem */
939 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
940 tb_end = tb_start + tb->size;
941 } else {
942 tb_start = tb->page_addr[1];
943 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
944 }
945 if (!(tb_end <= start || tb_start >= end)) {
946 #ifdef TARGET_HAS_PRECISE_SMC
947 if (current_tb_not_found) {
948 current_tb_not_found = 0;
949 current_tb = NULL;
950 if (env->mem_io_pc) {
951 /* now we have a real cpu fault */
952 current_tb = tb_find_pc(env->mem_io_pc);
953 }
954 }
955 if (current_tb == tb &&
956 (current_tb->cflags & CF_COUNT_MASK) != 1) {
957 /* If we are modifying the current TB, we must stop
958 its execution. We could be more precise by checking
959 that the modification is after the current PC, but it
960 would require a specialized function to partially
961 restore the CPU state */
962
963 current_tb_modified = 1;
964 cpu_restore_state(current_tb, env,
965 env->mem_io_pc, NULL);
966 #if defined(TARGET_I386)
967 current_flags = env->hflags;
968 current_flags |= (env->eflags & (IOPL_MASK | TF_MASK | VM_MASK));
969 current_cs_base = (target_ulong)env->segs[R_CS].base;
970 current_pc = current_cs_base + env->eip;
971 #else
972 #error unsupported CPU
973 #endif
974 }
975 #endif /* TARGET_HAS_PRECISE_SMC */
976 /* we need to do that to handle the case where a signal
977 occurs while doing tb_phys_invalidate() */
978 saved_tb = NULL;
979 if (env) {
980 saved_tb = env->current_tb;
981 env->current_tb = NULL;
982 }
983 tb_phys_invalidate(tb, -1);
984 if (env) {
985 env->current_tb = saved_tb;
986 if (env->interrupt_request && env->current_tb)
987 cpu_interrupt(env, env->interrupt_request);
988 }
989 }
990 tb = tb_next;
991 }
992 #if !defined(CONFIG_USER_ONLY)
993 /* if no code remaining, no need to continue to use slow writes */
994 if (!p->first_tb) {
995 invalidate_page_bitmap(p);
996 if (is_cpu_write_access) {
997 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
998 }
999 }
1000 #endif
1001 #ifdef TARGET_HAS_PRECISE_SMC
1002 if (current_tb_modified) {
1003 /* we generate a block containing just the instruction
1004 modifying the memory. It will ensure that it cannot modify
1005 itself */
1006 env->current_tb = NULL;
1007 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1008 cpu_resume_from_signal(env, NULL);
1009 }
1010 #endif
1011 }
1012
1013 /* len must be <= 8 and start must be a multiple of len */
1014 static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start, int len)
1015 {
1016 PageDesc *p;
1017 int offset, b;
1018 #if 0
1019 if (1) {
1020 if (loglevel) {
1021 fprintf(logfile, "modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1022 cpu_single_env->mem_io_vaddr, len,
1023 cpu_single_env->eip,
1024 cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1025 }
1026 }
1027 #endif
1028 p = page_find(start >> TARGET_PAGE_BITS);
1029 if (!p)
1030 return;
1031 if (p->code_bitmap) {
1032 offset = start & ~TARGET_PAGE_MASK;
1033 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1034 if (b & ((1 << len) - 1))
1035 goto do_invalidate;
1036 } else {
1037 do_invalidate:
1038 tb_invalidate_phys_page_range(start, start + len, 1);
1039 }
1040 }
1041
1042 #if !defined(CONFIG_SOFTMMU)
1043 static void tb_invalidate_phys_page(target_phys_addr_t addr,
1044 unsigned long pc, void *puc)
1045 {
1046 int n, current_flags, current_tb_modified;
1047 target_ulong current_pc, current_cs_base;
1048 PageDesc *p;
1049 TranslationBlock *tb, *current_tb;
1050 #ifdef TARGET_HAS_PRECISE_SMC
1051 CPUState *env = cpu_single_env;
1052 #endif
1053
1054 addr &= TARGET_PAGE_MASK;
1055 p = page_find(addr >> TARGET_PAGE_BITS);
1056 if (!p)
1057 return;
1058 tb = p->first_tb;
1059 current_tb_modified = 0;
1060 current_tb = NULL;
1061 current_pc = 0; /* avoid warning */
1062 current_cs_base = 0; /* avoid warning */
1063 current_flags = 0; /* avoid warning */
1064 #ifdef TARGET_HAS_PRECISE_SMC
1065 if (tb && pc != 0) {
1066 current_tb = tb_find_pc(pc);
1067 }
1068 #endif
1069 while (tb != NULL) {
1070 n = (long)tb & 3;
1071 tb = (TranslationBlock *)((long)tb & ~3);
1072 #ifdef TARGET_HAS_PRECISE_SMC
1073 if (current_tb == tb &&
1074 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1075 /* If we are modifying the current TB, we must stop
1076 its execution. We could be more precise by checking
1077 that the modification is after the current PC, but it
1078 would require a specialized function to partially
1079 restore the CPU state */
1080
1081 current_tb_modified = 1;
1082 cpu_restore_state(current_tb, env, pc, puc);
1083 #if defined(TARGET_I386)
1084 current_flags = env->hflags;
1085 current_flags |= (env->eflags & (IOPL_MASK | TF_MASK | VM_MASK));
1086 current_cs_base = (target_ulong)env->segs[R_CS].base;
1087 current_pc = current_cs_base + env->eip;
1088 #else
1089 #error unsupported CPU
1090 #endif
1091 }
1092 #endif /* TARGET_HAS_PRECISE_SMC */
1093 tb_phys_invalidate(tb, addr);
1094 tb = tb->page_next[n];
1095 }
1096 p->first_tb = NULL;
1097 #ifdef TARGET_HAS_PRECISE_SMC
1098 if (current_tb_modified) {
1099 /* we generate a block containing just the instruction
1100 modifying the memory. It will ensure that it cannot modify
1101 itself */
1102 env->current_tb = NULL;
1103 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1104 cpu_resume_from_signal(env, puc);
1105 }
1106 #endif
1107 }
1108 #endif
1109
1110 /* add the tb in the target page and protect it if necessary */
1111 static inline void tb_alloc_page(TranslationBlock *tb,
1112 unsigned int n, target_ulong page_addr)
1113 {
1114 PageDesc *p;
1115 TranslationBlock *last_first_tb;
1116
1117 tb->page_addr[n] = page_addr;
1118 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS);
1119 tb->page_next[n] = p->first_tb;
1120 last_first_tb = p->first_tb;
1121 p->first_tb = (TranslationBlock *)((long)tb | n);
1122 invalidate_page_bitmap(p);
1123
1124 #if defined(TARGET_HAS_SMC) || 1
1125
1126 #if defined(CONFIG_USER_ONLY)
1127 if (p->flags & PAGE_WRITE) {
1128 target_ulong addr;
1129 PageDesc *p2;
1130 int prot;
1131
1132 /* force the host page as non writable (writes will have a
1133 page fault + mprotect overhead) */
1134 page_addr &= qemu_host_page_mask;
1135 prot = 0;
1136 for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1137 addr += TARGET_PAGE_SIZE) {
1138
1139 p2 = page_find (addr >> TARGET_PAGE_BITS);
1140 if (!p2)
1141 continue;
1142 prot |= p2->flags;
1143 p2->flags &= ~PAGE_WRITE;
1144 page_get_flags(addr);
1145 }
1146 mprotect(g2h(page_addr), qemu_host_page_size,
1147 (prot & PAGE_BITS) & ~PAGE_WRITE);
1148 #ifdef DEBUG_TB_INVALIDATE
1149 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1150 page_addr);
1151 #endif
1152 }
1153 #else
1154 /* if some code is already present, then the pages are already
1155 protected. So we handle the case where only the first TB is
1156 allocated in a physical page */
1157 if (!last_first_tb) {
1158 tlb_protect_code(page_addr);
1159 }
1160 #endif
1161
1162 #endif /* TARGET_HAS_SMC */
1163 }
1164
1165 /* Allocate a new translation block. Flush the translation buffer if
1166 too many translation blocks or too much generated code. */
1167 TranslationBlock *tb_alloc(target_ulong pc)
1168 {
1169 TranslationBlock *tb;
1170
1171 if (nb_tbs >= code_gen_max_blocks ||
1172 (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
1173 return NULL;
1174 tb = &tbs[nb_tbs++];
1175 tb->pc = pc;
1176 tb->cflags = 0;
1177 return tb;
1178 }
1179
1180 void tb_free(TranslationBlock *tb)
1181 {
1182 /* In practice this is mostly used for single use temporary TB
1183 Ignore the hard cases and just back up if this TB happens to
1184 be the last one generated. */
1185 if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
1186 code_gen_ptr = tb->tc_ptr;
1187 nb_tbs--;
1188 }
1189 }
1190
1191 /* add a new TB and link it to the physical page tables. phys_page2 is
1192 (-1) to indicate that only one page contains the TB. */
1193 void tb_link_phys(TranslationBlock *tb,
1194 target_ulong phys_pc, target_ulong phys_page2)
1195 {
1196 unsigned int h;
1197 TranslationBlock **ptb;
1198
1199 /* Grab the mmap lock to stop another thread invalidating this TB
1200 before we are done. */
1201 mmap_lock();
1202 /* add in the physical hash table */
1203 h = tb_phys_hash_func(phys_pc);
1204 ptb = &tb_phys_hash[h];
1205 tb->phys_hash_next = *ptb;
1206 *ptb = tb;
1207
1208 /* add in the page list */
1209 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1210 if (phys_page2 != -1)
1211 tb_alloc_page(tb, 1, phys_page2);
1212 else
1213 tb->page_addr[1] = -1;
1214
1215 tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1216 tb->jmp_next[0] = NULL;
1217 tb->jmp_next[1] = NULL;
1218
1219 /* init original jump addresses */
1220 if (tb->tb_next_offset[0] != 0xffff)
1221 tb_reset_jump(tb, 0);
1222 if (tb->tb_next_offset[1] != 0xffff)
1223 tb_reset_jump(tb, 1);
1224
1225 #ifdef DEBUG_TB_CHECK
1226 tb_page_check();
1227 #endif
1228 mmap_unlock();
1229 }
1230
1231 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1232 tb[1].tc_ptr. Return NULL if not found */
1233 TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1234 {
1235 int m_min, m_max, m;
1236 unsigned long v;
1237 TranslationBlock *tb;
1238
1239 if (nb_tbs <= 0)
1240 return NULL;
1241 if (tc_ptr < (unsigned long)code_gen_buffer ||
1242 tc_ptr >= (unsigned long)code_gen_ptr)
1243 return NULL;
1244 /* binary search (cf Knuth) */
1245 m_min = 0;
1246 m_max = nb_tbs - 1;
1247 while (m_min <= m_max) {
1248 m = (m_min + m_max) >> 1;
1249 tb = &tbs[m];
1250 v = (unsigned long)tb->tc_ptr;
1251 if (v == tc_ptr)
1252 return tb;
1253 else if (tc_ptr < v) {
1254 m_max = m - 1;
1255 } else {
1256 m_min = m + 1;
1257 }
1258 }
1259 return &tbs[m_max];
1260 }
1261
1262 static void tb_reset_jump_recursive(TranslationBlock *tb);
1263
1264 static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1265 {
1266 TranslationBlock *tb1, *tb_next, **ptb;
1267 unsigned int n1;
1268
1269 tb1 = tb->jmp_next[n];
1270 if (tb1 != NULL) {
1271 /* find head of list */
1272 for(;;) {
1273 n1 = (long)tb1 & 3;
1274 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1275 if (n1 == 2)
1276 break;
1277 tb1 = tb1->jmp_next[n1];
1278 }
1279 /* we are now sure now that tb jumps to tb1 */
1280 tb_next = tb1;
1281
1282 /* remove tb from the jmp_first list */
1283 ptb = &tb_next->jmp_first;
1284 for(;;) {
1285 tb1 = *ptb;
1286 n1 = (long)tb1 & 3;
1287 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1288 if (n1 == n && tb1 == tb)
1289 break;
1290 ptb = &tb1->jmp_next[n1];
1291 }
1292 *ptb = tb->jmp_next[n];
1293 tb->jmp_next[n] = NULL;
1294
1295 /* suppress the jump to next tb in generated code */
1296 tb_reset_jump(tb, n);
1297
1298 /* suppress jumps in the tb on which we could have jumped */
1299 tb_reset_jump_recursive(tb_next);
1300 }
1301 }
1302
1303 static void tb_reset_jump_recursive(TranslationBlock *tb)
1304 {
1305 tb_reset_jump_recursive2(tb, 0);
1306 tb_reset_jump_recursive2(tb, 1);
1307 }
1308
1309 #if defined(TARGET_HAS_ICE)
1310 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1311 {
1312 target_phys_addr_t addr;
1313 target_ulong pd;
1314 ram_addr_t ram_addr;
1315 PhysPageDesc *p;
1316
1317 addr = cpu_get_phys_page_debug(env, pc);
1318 p = phys_page_find(addr >> TARGET_PAGE_BITS);
1319 if (!p) {
1320 pd = IO_MEM_UNASSIGNED;
1321 } else {
1322 pd = p->phys_offset;
1323 }
1324 ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1325 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1326 }
1327 #endif
1328
1329 /* Add a watchpoint. */
1330 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, int type)
1331 {
1332 int i;
1333
1334 for (i = 0; i < env->nb_watchpoints; i++) {
1335 if (addr == env->watchpoint[i].vaddr)
1336 return 0;
1337 }
1338 if (env->nb_watchpoints >= MAX_WATCHPOINTS)
1339 return -1;
1340
1341 i = env->nb_watchpoints++;
1342 env->watchpoint[i].vaddr = addr;
1343 env->watchpoint[i].type = type;
1344 tlb_flush_page(env, addr);
1345 /* FIXME: This flush is needed because of the hack to make memory ops
1346 terminate the TB. It can be removed once the proper IO trap and
1347 re-execute bits are in. */
1348 tb_flush(env);
1349 return i;
1350 }
1351
1352 /* Remove a watchpoint. */
1353 int cpu_watchpoint_remove(CPUState *env, target_ulong addr)
1354 {
1355 int i;
1356
1357 for (i = 0; i < env->nb_watchpoints; i++) {
1358 if (addr == env->watchpoint[i].vaddr) {
1359 env->nb_watchpoints--;
1360 env->watchpoint[i] = env->watchpoint[env->nb_watchpoints];
1361 tlb_flush_page(env, addr);
1362 return 0;
1363 }
1364 }
1365 return -1;
1366 }
1367
1368 /* Remove all watchpoints. */
1369 void cpu_watchpoint_remove_all(CPUState *env) {
1370 int i;
1371
1372 for (i = 0; i < env->nb_watchpoints; i++) {
1373 tlb_flush_page(env, env->watchpoint[i].vaddr);
1374 }
1375 env->nb_watchpoints = 0;
1376 }
1377
1378 /* add a breakpoint. EXCP_DEBUG is returned by the CPU loop if a
1379 breakpoint is reached */
1380 int cpu_breakpoint_insert(CPUState *env, target_ulong pc)
1381 {
1382 #if defined(TARGET_HAS_ICE)
1383 int i;
1384
1385 for(i = 0; i < env->nb_breakpoints; i++) {
1386 if (env->breakpoints[i] == pc)
1387 return 0;
1388 }
1389
1390 if (env->nb_breakpoints >= MAX_BREAKPOINTS)
1391 return -1;
1392 env->breakpoints[env->nb_breakpoints++] = pc;
1393
1394 if (kvm_enabled())
1395 kvm_update_debugger(env);
1396
1397 breakpoint_invalidate(env, pc);
1398 return 0;
1399 #else
1400 return -1;
1401 #endif
1402 }
1403
1404 /* remove all breakpoints */
1405 void cpu_breakpoint_remove_all(CPUState *env) {
1406 #if defined(TARGET_HAS_ICE)
1407 int i;
1408 for(i = 0; i < env->nb_breakpoints; i++) {
1409 breakpoint_invalidate(env, env->breakpoints[i]);
1410 }
1411 env->nb_breakpoints = 0;
1412 #endif
1413 }
1414
1415 /* remove a breakpoint */
1416 int cpu_breakpoint_remove(CPUState *env, target_ulong pc)
1417 {
1418 #if defined(TARGET_HAS_ICE)
1419 int i;
1420 for(i = 0; i < env->nb_breakpoints; i++) {
1421 if (env->breakpoints[i] == pc)
1422 goto found;
1423 }
1424 return -1;
1425 found:
1426 env->nb_breakpoints--;
1427 if (i < env->nb_breakpoints)
1428 env->breakpoints[i] = env->breakpoints[env->nb_breakpoints];
1429
1430 if (kvm_enabled())
1431 kvm_update_debugger(env);
1432
1433 breakpoint_invalidate(env, pc);
1434 return 0;
1435 #else
1436 return -1;
1437 #endif
1438 }
1439
1440 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1441 CPU loop after each instruction */
1442 void cpu_single_step(CPUState *env, int enabled)
1443 {
1444 #if defined(TARGET_HAS_ICE)
1445 if (env->singlestep_enabled != enabled) {
1446 env->singlestep_enabled = enabled;
1447 /* must flush all the translated code to avoid inconsistancies */
1448 /* XXX: only flush what is necessary */
1449 tb_flush(env);
1450 }
1451 if (kvm_enabled())
1452 kvm_update_debugger(env);
1453 #endif
1454 }
1455
1456 /* enable or disable low levels log */
1457 void cpu_set_log(int log_flags)
1458 {
1459 loglevel = log_flags;
1460 if (loglevel && !logfile) {
1461 logfile = fopen(logfilename, log_append ? "a" : "w");
1462 if (!logfile) {
1463 perror(logfilename);
1464 _exit(1);
1465 }
1466 #if !defined(CONFIG_SOFTMMU)
1467 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1468 {
1469 static char logfile_buf[4096];
1470 setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1471 }
1472 #else
1473 setvbuf(logfile, NULL, _IOLBF, 0);
1474 #endif
1475 log_append = 1;
1476 }
1477 if (!loglevel && logfile) {
1478 fclose(logfile);
1479 logfile = NULL;
1480 }
1481 }
1482
1483 void cpu_set_log_filename(const char *filename)
1484 {
1485 logfilename = strdup(filename);
1486 if (logfile) {
1487 fclose(logfile);
1488 logfile = NULL;
1489 }
1490 cpu_set_log(loglevel);
1491 }
1492
1493 /* mask must never be zero, except for A20 change call */
1494 void cpu_interrupt(CPUState *env, int mask)
1495 {
1496 #if !defined(USE_NPTL)
1497 TranslationBlock *tb;
1498 static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1499 #endif
1500 int old_mask;
1501
1502 old_mask = env->interrupt_request;
1503 /* FIXME: This is probably not threadsafe. A different thread could
1504 be in the middle of a read-modify-write operation. */
1505 env->interrupt_request |= mask;
1506 if (kvm_enabled() && !qemu_kvm_irqchip_in_kernel())
1507 kvm_update_interrupt_request(env);
1508 #if defined(USE_NPTL)
1509 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1510 problem and hope the cpu will stop of its own accord. For userspace
1511 emulation this often isn't actually as bad as it sounds. Often
1512 signals are used primarily to interrupt blocking syscalls. */
1513 #else
1514 if (use_icount) {
1515 env->icount_decr.u16.high = 0xffff;
1516 #ifndef CONFIG_USER_ONLY
1517 /* CPU_INTERRUPT_EXIT isn't a real interrupt. It just means
1518 an async event happened and we need to process it. */
1519 if (!can_do_io(env)
1520 && (mask & ~(old_mask | CPU_INTERRUPT_EXIT)) != 0) {
1521 cpu_abort(env, "Raised interrupt while not in I/O function");
1522 }
1523 #endif
1524 } else {
1525 tb = env->current_tb;
1526 /* if the cpu is currently executing code, we must unlink it and
1527 all the potentially executing TB */
1528 if (tb && !testandset(&interrupt_lock)) {
1529 env->current_tb = NULL;
1530 tb_reset_jump_recursive(tb);
1531 resetlock(&interrupt_lock);
1532 }
1533 }
1534 #endif
1535 }
1536
1537 void cpu_reset_interrupt(CPUState *env, int mask)
1538 {
1539 env->interrupt_request &= ~mask;
1540 }
1541
1542 const CPULogItem cpu_log_items[] = {
1543 { CPU_LOG_TB_OUT_ASM, "out_asm",
1544 "show generated host assembly code for each compiled TB" },
1545 { CPU_LOG_TB_IN_ASM, "in_asm",
1546 "show target assembly code for each compiled TB" },
1547 { CPU_LOG_TB_OP, "op",
1548 "show micro ops for each compiled TB" },
1549 { CPU_LOG_TB_OP_OPT, "op_opt",
1550 "show micro ops "
1551 #ifdef TARGET_I386
1552 "before eflags optimization and "
1553 #endif
1554 "after liveness analysis" },
1555 { CPU_LOG_INT, "int",
1556 "show interrupts/exceptions in short format" },
1557 { CPU_LOG_EXEC, "exec",
1558 "show trace before each executed TB (lots of logs)" },
1559 { CPU_LOG_TB_CPU, "cpu",
1560 "show CPU state before block translation" },
1561 #ifdef TARGET_I386
1562 { CPU_LOG_PCALL, "pcall",
1563 "show protected mode far calls/returns/exceptions" },
1564 #endif
1565 #ifdef DEBUG_IOPORT
1566 { CPU_LOG_IOPORT, "ioport",
1567 "show all i/o ports accesses" },
1568 #endif
1569 { 0, NULL, NULL },
1570 };
1571
1572 static int cmp1(const char *s1, int n, const char *s2)
1573 {
1574 if (strlen(s2) != n)
1575 return 0;
1576 return memcmp(s1, s2, n) == 0;
1577 }
1578
1579 /* takes a comma separated list of log masks. Return 0 if error. */
1580 int cpu_str_to_log_mask(const char *str)
1581 {
1582 const CPULogItem *item;
1583 int mask;
1584 const char *p, *p1;
1585
1586 p = str;
1587 mask = 0;
1588 for(;;) {
1589 p1 = strchr(p, ',');
1590 if (!p1)
1591 p1 = p + strlen(p);
1592 if(cmp1(p,p1-p,"all")) {
1593 for(item = cpu_log_items; item->mask != 0; item++) {
1594 mask |= item->mask;
1595 }
1596 } else {
1597 for(item = cpu_log_items; item->mask != 0; item++) {
1598 if (cmp1(p, p1 - p, item->name))
1599 goto found;
1600 }
1601 return 0;
1602 }
1603 found:
1604 mask |= item->mask;
1605 if (*p1 != ',')
1606 break;
1607 p = p1 + 1;
1608 }
1609 return mask;
1610 }
1611
1612 void cpu_abort(CPUState *env, const char *fmt, ...)
1613 {
1614 va_list ap;
1615 va_list ap2;
1616
1617 va_start(ap, fmt);
1618 va_copy(ap2, ap);
1619 fprintf(stderr, "qemu: fatal: ");
1620 vfprintf(stderr, fmt, ap);
1621 fprintf(stderr, "\n");
1622 #ifdef TARGET_I386
1623 cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1624 #else
1625 cpu_dump_state(env, stderr, fprintf, 0);
1626 #endif
1627 if (logfile) {
1628 fprintf(logfile, "qemu: fatal: ");
1629 vfprintf(logfile, fmt, ap2);
1630 fprintf(logfile, "\n");
1631 #ifdef TARGET_I386
1632 cpu_dump_state(env, logfile, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1633 #else
1634 cpu_dump_state(env, logfile, fprintf, 0);
1635 #endif
1636 fflush(logfile);
1637 fclose(logfile);
1638 }
1639 va_end(ap2);
1640 va_end(ap);
1641 abort();
1642 }
1643
1644 CPUState *cpu_copy(CPUState *env)
1645 {
1646 CPUState *new_env = cpu_init(env->cpu_model_str);
1647 /* preserve chaining and index */
1648 CPUState *next_cpu = new_env->next_cpu;
1649 int cpu_index = new_env->cpu_index;
1650 memcpy(new_env, env, sizeof(CPUState));
1651 new_env->next_cpu = next_cpu;
1652 new_env->cpu_index = cpu_index;
1653 return new_env;
1654 }
1655
1656 #if !defined(CONFIG_USER_ONLY)
1657
1658 static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1659 {
1660 unsigned int i;
1661
1662 /* Discard jump cache entries for any tb which might potentially
1663 overlap the flushed page. */
1664 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1665 memset (&env->tb_jmp_cache[i], 0,
1666 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1667
1668 i = tb_jmp_cache_hash_page(addr);
1669 memset (&env->tb_jmp_cache[i], 0,
1670 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1671 }
1672
1673 /* NOTE: if flush_global is true, also flush global entries (not
1674 implemented yet) */
1675 void tlb_flush(CPUState *env, int flush_global)
1676 {
1677 int i;
1678
1679 #if defined(DEBUG_TLB)
1680 printf("tlb_flush:\n");
1681 #endif
1682 /* must reset current TB so that interrupts cannot modify the
1683 links while we are modifying them */
1684 env->current_tb = NULL;
1685
1686 for(i = 0; i < CPU_TLB_SIZE; i++) {
1687 env->tlb_table[0][i].addr_read = -1;
1688 env->tlb_table[0][i].addr_write = -1;
1689 env->tlb_table[0][i].addr_code = -1;
1690 env->tlb_table[1][i].addr_read = -1;
1691 env->tlb_table[1][i].addr_write = -1;
1692 env->tlb_table[1][i].addr_code = -1;
1693 #if (NB_MMU_MODES >= 3)
1694 env->tlb_table[2][i].addr_read = -1;
1695 env->tlb_table[2][i].addr_write = -1;
1696 env->tlb_table[2][i].addr_code = -1;
1697 #if (NB_MMU_MODES == 4)
1698 env->tlb_table[3][i].addr_read = -1;
1699 env->tlb_table[3][i].addr_write = -1;
1700 env->tlb_table[3][i].addr_code = -1;
1701 #endif
1702 #endif
1703 }
1704
1705 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1706
1707 #ifdef USE_KQEMU
1708 if (env->kqemu_enabled) {
1709 kqemu_flush(env, flush_global);
1710 }
1711 #endif
1712 tlb_flush_count++;
1713 }
1714
1715 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
1716 {
1717 if (addr == (tlb_entry->addr_read &
1718 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1719 addr == (tlb_entry->addr_write &
1720 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1721 addr == (tlb_entry->addr_code &
1722 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1723 tlb_entry->addr_read = -1;
1724 tlb_entry->addr_write = -1;
1725 tlb_entry->addr_code = -1;
1726 }
1727 }
1728
1729 void tlb_flush_page(CPUState *env, target_ulong addr)
1730 {
1731 int i;
1732
1733 #if defined(DEBUG_TLB)
1734 printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1735 #endif
1736 /* must reset current TB so that interrupts cannot modify the
1737 links while we are modifying them */
1738 env->current_tb = NULL;
1739
1740 addr &= TARGET_PAGE_MASK;
1741 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1742 tlb_flush_entry(&env->tlb_table[0][i], addr);
1743 tlb_flush_entry(&env->tlb_table[1][i], addr);
1744 #if (NB_MMU_MODES >= 3)
1745 tlb_flush_entry(&env->tlb_table[2][i], addr);
1746 #if (NB_MMU_MODES == 4)
1747 tlb_flush_entry(&env->tlb_table[3][i], addr);
1748 #endif
1749 #endif
1750
1751 tlb_flush_jmp_cache(env, addr);
1752
1753 #ifdef USE_KQEMU
1754 if (env->kqemu_enabled) {
1755 kqemu_flush_page(env, addr);
1756 }
1757 #endif
1758 }
1759
1760 /* update the TLBs so that writes to code in the virtual page 'addr'
1761 can be detected */
1762 static void tlb_protect_code(ram_addr_t ram_addr)
1763 {
1764 cpu_physical_memory_reset_dirty(ram_addr,
1765 ram_addr + TARGET_PAGE_SIZE,
1766 CODE_DIRTY_FLAG);
1767 }
1768
1769 /* update the TLB so that writes in physical page 'phys_addr' are no longer
1770 tested for self modifying code */
1771 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
1772 target_ulong vaddr)
1773 {
1774 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] |= CODE_DIRTY_FLAG;
1775 }
1776
1777 static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
1778 unsigned long start, unsigned long length)
1779 {
1780 unsigned long addr;
1781 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1782 addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
1783 if ((addr - start) < length) {
1784 tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
1785 }
1786 }
1787 }
1788
1789 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
1790 int dirty_flags)
1791 {
1792 CPUState *env;
1793 unsigned long length, start1;
1794 int i, mask, len;
1795 uint8_t *p;
1796
1797 start &= TARGET_PAGE_MASK;
1798 end = TARGET_PAGE_ALIGN(end);
1799
1800 length = end - start;
1801 if (length == 0)
1802 return;
1803 len = length >> TARGET_PAGE_BITS;
1804 #ifdef USE_KQEMU
1805 /* XXX: should not depend on cpu context */
1806 env = first_cpu;
1807 if (env->kqemu_enabled) {
1808 ram_addr_t addr;
1809 addr = start;
1810 for(i = 0; i < len; i++) {
1811 kqemu_set_notdirty(env, addr);
1812 addr += TARGET_PAGE_SIZE;
1813 }
1814 }
1815 #endif
1816 mask = ~dirty_flags;
1817 p = phys_ram_dirty + (start >> TARGET_PAGE_BITS);
1818 for(i = 0; i < len; i++)
1819 p[i] &= mask;
1820
1821 /* we modify the TLB cache so that the dirty bit will be set again
1822 when accessing the range */
1823 start1 = start + (unsigned long)phys_ram_base;
1824 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1825 for(i = 0; i < CPU_TLB_SIZE; i++)
1826 tlb_reset_dirty_range(&env->tlb_table[0][i], start1, length);
1827 for(i = 0; i < CPU_TLB_SIZE; i++)
1828 tlb_reset_dirty_range(&env->tlb_table[1][i], start1, length);
1829 #if (NB_MMU_MODES >= 3)
1830 for(i = 0; i < CPU_TLB_SIZE; i++)
1831 tlb_reset_dirty_range(&env->tlb_table[2][i], start1, length);
1832 #if (NB_MMU_MODES == 4)
1833 for(i = 0; i < CPU_TLB_SIZE; i++)
1834 tlb_reset_dirty_range(&env->tlb_table[3][i], start1, length);
1835 #endif
1836 #endif
1837 }
1838 }
1839
1840 int cpu_physical_memory_set_dirty_tracking(int enable)
1841 {
1842 int r=0;
1843
1844 if (kvm_enabled())
1845 r = kvm_physical_memory_set_dirty_tracking(enable);
1846 in_migration = enable;
1847 return r;
1848 }
1849
1850 int cpu_physical_memory_get_dirty_tracking(void)
1851 {
1852 return in_migration;
1853 }
1854
1855 static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
1856 {
1857 ram_addr_t ram_addr;
1858
1859 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1860 ram_addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) +
1861 tlb_entry->addend - (unsigned long)phys_ram_base;
1862 if (!cpu_physical_memory_is_dirty(ram_addr)) {
1863 tlb_entry->addr_write |= TLB_NOTDIRTY;
1864 }
1865 }
1866 }
1867
1868 /* update the TLB according to the current state of the dirty bits */
1869 void cpu_tlb_update_dirty(CPUState *env)
1870 {
1871 int i;
1872 for(i = 0; i < CPU_TLB_SIZE; i++)
1873 tlb_update_dirty(&env->tlb_table[0][i]);
1874 for(i = 0; i < CPU_TLB_SIZE; i++)
1875 tlb_update_dirty(&env->tlb_table[1][i]);
1876 #if (NB_MMU_MODES >= 3)
1877 for(i = 0; i < CPU_TLB_SIZE; i++)
1878 tlb_update_dirty(&env->tlb_table[2][i]);
1879 #if (NB_MMU_MODES == 4)
1880 for(i = 0; i < CPU_TLB_SIZE; i++)
1881 tlb_update_dirty(&env->tlb_table[3][i]);
1882 #endif
1883 #endif
1884 }
1885
1886 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
1887 {
1888 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
1889 tlb_entry->addr_write = vaddr;
1890 }
1891
1892 /* update the TLB corresponding to virtual page vaddr
1893 so that it is no longer dirty */
1894 static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
1895 {
1896 int i;
1897
1898 vaddr &= TARGET_PAGE_MASK;
1899 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1900 tlb_set_dirty1(&env->tlb_table[0][i], vaddr);
1901 tlb_set_dirty1(&env->tlb_table[1][i], vaddr);
1902 #if (NB_MMU_MODES >= 3)
1903 tlb_set_dirty1(&env->tlb_table[2][i], vaddr);
1904 #if (NB_MMU_MODES == 4)
1905 tlb_set_dirty1(&env->tlb_table[3][i], vaddr);
1906 #endif
1907 #endif
1908 }
1909
1910 /* add a new TLB entry. At most one entry for a given virtual address
1911 is permitted. Return 0 if OK or 2 if the page could not be mapped
1912 (can only happen in non SOFTMMU mode for I/O pages or pages
1913 conflicting with the host address space). */
1914 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
1915 target_phys_addr_t paddr, int prot,
1916 int mmu_idx, int is_softmmu)
1917 {
1918 PhysPageDesc *p;
1919 unsigned long pd;
1920 unsigned int index;
1921 target_ulong address;
1922 target_ulong code_address;
1923 target_phys_addr_t addend;
1924 int ret;
1925 CPUTLBEntry *te;
1926 int i;
1927 target_phys_addr_t iotlb;
1928
1929 p = phys_page_find(paddr >> TARGET_PAGE_BITS);
1930 if (!p) {
1931 pd = IO_MEM_UNASSIGNED;
1932 } else {
1933 pd = p->phys_offset;
1934 }
1935 #if defined(DEBUG_TLB)
1936 printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
1937 vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
1938 #endif
1939
1940 ret = 0;
1941 address = vaddr;
1942 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
1943 /* IO memory case (romd handled later) */
1944 address |= TLB_MMIO;
1945 }
1946 addend = (unsigned long)phys_ram_base + (pd & TARGET_PAGE_MASK);
1947 if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
1948 /* Normal RAM. */
1949 iotlb = pd & TARGET_PAGE_MASK;
1950 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
1951 iotlb |= IO_MEM_NOTDIRTY;
1952 else
1953 iotlb |= IO_MEM_ROM;
1954 } else {
1955 /* IO handlers are currently passed a phsical address.
1956 It would be nice to pass an offset from the base address
1957 of that region. This would avoid having to special case RAM,
1958 and avoid full address decoding in every device.
1959 We can't use the high bits of pd for this because
1960 IO_MEM_ROMD uses these as a ram address. */
1961 iotlb = (pd & ~TARGET_PAGE_MASK) + paddr;
1962 }
1963
1964 code_address = address;
1965 /* Make accesses to pages with watchpoints go via the
1966 watchpoint trap routines. */
1967 for (i = 0; i < env->nb_watchpoints; i++) {
1968 if (vaddr == (env->watchpoint[i].vaddr & TARGET_PAGE_MASK)) {
1969 iotlb = io_mem_watch + paddr;
1970 /* TODO: The memory case can be optimized by not trapping
1971 reads of pages with a write breakpoint. */
1972 address |= TLB_MMIO;
1973 }
1974 }
1975
1976 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1977 env->iotlb[mmu_idx][index] = iotlb - vaddr;
1978 te = &env->tlb_table[mmu_idx][index];
1979 te->addend = addend - vaddr;
1980 if (prot & PAGE_READ) {
1981 te->addr_read = address;
1982 } else {
1983 te->addr_read = -1;
1984 }
1985
1986 if (prot & PAGE_EXEC) {
1987 te->addr_code = code_address;
1988 } else {
1989 te->addr_code = -1;
1990 }
1991 if (prot & PAGE_WRITE) {
1992 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
1993 (pd & IO_MEM_ROMD)) {
1994 /* Write access calls the I/O callback. */
1995 te->addr_write = address | TLB_MMIO;
1996 } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
1997 !cpu_physical_memory_is_dirty(pd)) {
1998 te->addr_write = address | TLB_NOTDIRTY;
1999 } else {
2000 te->addr_write = address;
2001 }
2002 } else {
2003 te->addr_write = -1;
2004 }
2005 return ret;
2006 }
2007
2008 #else
2009
2010 void tlb_flush(CPUState *env, int flush_global)
2011 {
2012 }
2013
2014 void tlb_flush_page(CPUState *env, target_ulong addr)
2015 {
2016 }
2017
2018 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
2019 target_phys_addr_t paddr, int prot,
2020 int mmu_idx, int is_softmmu)
2021 {
2022 return 0;
2023 }
2024
2025 /* dump memory mappings */
2026 void page_dump(FILE *f)
2027 {
2028 unsigned long start, end;
2029 int i, j, prot, prot1;
2030 PageDesc *p;
2031
2032 fprintf(f, "%-8s %-8s %-8s %s\n",
2033 "start", "end", "size", "prot");
2034 start = -1;
2035 end = -1;
2036 prot = 0;
2037 for(i = 0; i <= L1_SIZE; i++) {
2038 if (i < L1_SIZE)
2039 p = l1_map[i];
2040 else
2041 p = NULL;
2042 for(j = 0;j < L2_SIZE; j++) {
2043 if (!p)
2044 prot1 = 0;
2045 else
2046 prot1 = p[j].flags;
2047 if (prot1 != prot) {
2048 end = (i << (32 - L1_BITS)) | (j << TARGET_PAGE_BITS);
2049 if (start != -1) {
2050 fprintf(f, "%08lx-%08lx %08lx %c%c%c\n",
2051 start, end, end - start,
2052 prot & PAGE_READ ? 'r' : '-',
2053 prot & PAGE_WRITE ? 'w' : '-',
2054 prot & PAGE_EXEC ? 'x' : '-');
2055 }
2056 if (prot1 != 0)
2057 start = end;
2058 else
2059 start = -1;
2060 prot = prot1;
2061 }
2062 if (!p)
2063 break;
2064 }
2065 }
2066 }
2067
2068 int page_get_flags(target_ulong address)
2069 {
2070 PageDesc *p;
2071
2072 p = page_find(address >> TARGET_PAGE_BITS);
2073 if (!p)
2074 return 0;
2075 return p->flags;
2076 }
2077
2078 /* modify the flags of a page and invalidate the code if
2079 necessary. The flag PAGE_WRITE_ORG is positionned automatically
2080 depending on PAGE_WRITE */
2081 void page_set_flags(target_ulong start, target_ulong end, int flags)
2082 {
2083 PageDesc *p;
2084 target_ulong addr;
2085
2086 /* mmap_lock should already be held. */
2087 start = start & TARGET_PAGE_MASK;
2088 end = TARGET_PAGE_ALIGN(end);
2089 if (flags & PAGE_WRITE)
2090 flags |= PAGE_WRITE_ORG;
2091 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2092 p = page_find_alloc(addr >> TARGET_PAGE_BITS);
2093 /* We may be called for host regions that are outside guest
2094 address space. */
2095 if (!p)
2096 return;
2097 /* if the write protection is set, then we invalidate the code
2098 inside */
2099 if (!(p->flags & PAGE_WRITE) &&
2100 (flags & PAGE_WRITE) &&
2101 p->first_tb) {
2102 tb_invalidate_phys_page(addr, 0, NULL);
2103 }
2104 p->flags = flags;
2105 }
2106 }
2107
2108 int page_check_range(target_ulong start, target_ulong len, int flags)
2109 {
2110 PageDesc *p;
2111 target_ulong end;
2112 target_ulong addr;
2113
2114 if (start + len < start)
2115 /* we've wrapped around */
2116 return -1;
2117
2118 end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2119 start = start & TARGET_PAGE_MASK;
2120
2121 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2122 p = page_find(addr >> TARGET_PAGE_BITS);
2123 if( !p )
2124 return -1;
2125 if( !(p->flags & PAGE_VALID) )
2126 return -1;
2127
2128 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2129 return -1;
2130 if (flags & PAGE_WRITE) {
2131 if (!(p->flags & PAGE_WRITE_ORG))
2132 return -1;
2133 /* unprotect the page if it was put read-only because it
2134 contains translated code */
2135 if (!(p->flags & PAGE_WRITE)) {
2136 if (!page_unprotect(addr, 0, NULL))
2137 return -1;
2138 }
2139 return 0;
2140 }
2141 }
2142 return 0;
2143 }
2144
2145 /* called from signal handler: invalidate the code and unprotect the
2146 page. Return TRUE if the fault was succesfully handled. */
2147 int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2148 {
2149 unsigned int page_index, prot, pindex;
2150 PageDesc *p, *p1;
2151 target_ulong host_start, host_end, addr;
2152
2153 /* Technically this isn't safe inside a signal handler. However we
2154 know this only ever happens in a synchronous SEGV handler, so in
2155 practice it seems to be ok. */
2156 mmap_lock();
2157
2158 host_start = address & qemu_host_page_mask;
2159 page_index = host_start >> TARGET_PAGE_BITS;
2160 p1 = page_find(page_index);
2161 if (!p1) {
2162 mmap_unlock();
2163 return 0;
2164 }
2165 host_end = host_start + qemu_host_page_size;
2166 p = p1;
2167 prot = 0;
2168 for(addr = host_start;addr < host_end; addr += TARGET_PAGE_SIZE) {
2169 prot |= p->flags;
2170 p++;
2171 }
2172 /* if the page was really writable, then we change its
2173 protection back to writable */
2174 if (prot & PAGE_WRITE_ORG) {
2175 pindex = (address - host_start) >> TARGET_PAGE_BITS;
2176 if (!(p1[pindex].flags & PAGE_WRITE)) {
2177 mprotect((void *)g2h(host_start), qemu_host_page_size,
2178 (prot & PAGE_BITS) | PAGE_WRITE);
2179 p1[pindex].flags |= PAGE_WRITE;
2180 /* and since the content will be modified, we must invalidate
2181 the corresponding translated code. */
2182 tb_invalidate_phys_page(address, pc, puc);
2183 #ifdef DEBUG_TB_CHECK
2184 tb_invalidate_check(address);
2185 #endif
2186 mmap_unlock();
2187 return 1;
2188 }
2189 }
2190 mmap_unlock();
2191 return 0;
2192 }
2193
2194 static inline void tlb_set_dirty(CPUState *env,
2195 unsigned long addr, target_ulong vaddr)
2196 {
2197 }
2198 #endif /* defined(CONFIG_USER_ONLY) */
2199
2200 #if !defined(CONFIG_USER_ONLY)
2201 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2202 ram_addr_t memory);
2203 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2204 ram_addr_t orig_memory);
2205 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2206 need_subpage) \
2207 do { \
2208 if (addr > start_addr) \
2209 start_addr2 = 0; \
2210 else { \
2211 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2212 if (start_addr2 > 0) \
2213 need_subpage = 1; \
2214 } \
2215 \
2216 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2217 end_addr2 = TARGET_PAGE_SIZE - 1; \
2218 else { \
2219 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2220 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2221 need_subpage = 1; \
2222 } \
2223 } while (0)
2224
2225 /* register physical memory. 'size' must be a multiple of the target
2226 page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2227 io memory page */
2228 void cpu_register_physical_memory(target_phys_addr_t start_addr,
2229 ram_addr_t size,
2230 ram_addr_t phys_offset)
2231 {
2232 target_phys_addr_t addr, end_addr;
2233 PhysPageDesc *p;
2234 CPUState *env;
2235 ram_addr_t orig_size = size;
2236 void *subpage;
2237
2238 #ifdef USE_KQEMU
2239 /* XXX: should not depend on cpu context */
2240 env = first_cpu;
2241 if (env->kqemu_enabled) {
2242 kqemu_set_phys_mem(start_addr, size, phys_offset);
2243 }
2244 #endif
2245 if (kvm_enabled())
2246 kvm_set_phys_mem(start_addr, size, phys_offset);
2247
2248 size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2249 end_addr = start_addr + (target_phys_addr_t)size;
2250 for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2251 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2252 if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2253 ram_addr_t orig_memory = p->phys_offset;
2254 target_phys_addr_t start_addr2, end_addr2;
2255 int need_subpage = 0;
2256
2257 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2258 need_subpage);
2259 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2260 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2261 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2262 &p->phys_offset, orig_memory);
2263 } else {
2264 subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2265 >> IO_MEM_SHIFT];
2266 }
2267 subpage_register(subpage, start_addr2, end_addr2, phys_offset);
2268 } else {
2269 p->phys_offset = phys_offset;
2270 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2271 (phys_offset & IO_MEM_ROMD))
2272 phys_offset += TARGET_PAGE_SIZE;
2273 }
2274 } else {
2275 p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2276 p->phys_offset = phys_offset;
2277 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2278 (phys_offset & IO_MEM_ROMD))
2279 phys_offset += TARGET_PAGE_SIZE;
2280 else {
2281 target_phys_addr_t start_addr2, end_addr2;
2282 int need_subpage = 0;
2283
2284 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2285 end_addr2, need_subpage);
2286
2287 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2288 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2289 &p->phys_offset, IO_MEM_UNASSIGNED);
2290 subpage_register(subpage, start_addr2, end_addr2,
2291 phys_offset);
2292 }
2293 }
2294 }
2295 }
2296
2297 /* since each CPU stores ram addresses in its TLB cache, we must
2298 reset the modified entries */
2299 /* XXX: slow ! */
2300 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2301 tlb_flush(env, 1);
2302 }
2303 }
2304
2305 /* XXX: temporary until new memory mapping API */
2306 ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2307 {
2308 PhysPageDesc *p;
2309
2310 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2311 if (!p)
2312 return IO_MEM_UNASSIGNED;
2313 return p->phys_offset;
2314 }
2315
2316 /* XXX: better than nothing */
2317 ram_addr_t qemu_ram_alloc(ram_addr_t size)
2318 {
2319 ram_addr_t addr;
2320 if ((phys_ram_alloc_offset + size) > phys_ram_size) {
2321 fprintf(stderr, "Not enough memory (requested_size = %" PRIu64 ", max memory = %" PRIu64 ")\n",
2322 (uint64_t)size, (uint64_t)phys_ram_size);
2323 abort();
2324 }
2325 addr = phys_ram_alloc_offset;
2326 phys_ram_alloc_offset = TARGET_PAGE_ALIGN(phys_ram_alloc_offset + size);
2327 return addr;
2328 }
2329
2330 void qemu_ram_free(ram_addr_t addr)
2331 {
2332 }
2333
2334 static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2335 {
2336 #ifdef DEBUG_UNASSIGNED
2337 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2338 #endif
2339 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2340 do_unassigned_access(addr, 0, 0, 0, 1);
2341 #endif
2342 return 0;
2343 }
2344
2345 static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2346 {
2347 #ifdef DEBUG_UNASSIGNED
2348 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2349 #endif
2350 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2351 do_unassigned_access(addr, 0, 0, 0, 2);
2352 #endif
2353 return 0;
2354 }
2355
2356 static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
2357 {
2358 #ifdef DEBUG_UNASSIGNED
2359 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2360 #endif
2361 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2362 do_unassigned_access(addr, 0, 0, 0, 4);
2363 #endif
2364 return 0;
2365 }
2366
2367 static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
2368 {
2369 #ifdef DEBUG_UNASSIGNED
2370 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2371 #endif
2372 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2373 do_unassigned_access(addr, 1, 0, 0, 1);
2374 #endif
2375 }
2376
2377 static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
2378 {
2379 #ifdef DEBUG_UNASSIGNED
2380 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2381 #endif
2382 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2383 do_unassigned_access(addr, 1, 0, 0, 2);
2384 #endif
2385 }
2386
2387 static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
2388 {
2389 #ifdef DEBUG_UNASSIGNED
2390 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2391 #endif
2392 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2393 do_unassigned_access(addr, 1, 0, 0, 4);
2394 #endif
2395 }
2396
2397 static CPUReadMemoryFunc *unassigned_mem_read[3] = {
2398 unassigned_mem_readb,
2399 unassigned_mem_readw,
2400 unassigned_mem_readl,
2401 };
2402
2403 static CPUWriteMemoryFunc *unassigned_mem_write[3] = {
2404 unassigned_mem_writeb,
2405 unassigned_mem_writew,
2406 unassigned_mem_writel,
2407 };
2408
2409 static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
2410 uint32_t val)
2411 {
2412 int dirty_flags;
2413 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2414 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2415 #if !defined(CONFIG_USER_ONLY)
2416 tb_invalidate_phys_page_fast(ram_addr, 1);
2417 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2418 #endif
2419 }
2420 stb_p(phys_ram_base + ram_addr, val);
2421 #ifdef USE_KQEMU
2422 if (cpu_single_env->kqemu_enabled &&
2423 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2424 kqemu_modify_page(cpu_single_env, ram_addr);
2425 #endif
2426 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2427 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2428 /* we remove the notdirty callback only if the code has been
2429 flushed */
2430 if (dirty_flags == 0xff)
2431 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2432 }
2433
2434 static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
2435 uint32_t val)
2436 {
2437 int dirty_flags;
2438 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2439 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2440 #if !defined(CONFIG_USER_ONLY)
2441 tb_invalidate_phys_page_fast(ram_addr, 2);
2442 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2443 #endif
2444 }
2445 stw_p(phys_ram_base + ram_addr, val);
2446 #ifdef USE_KQEMU
2447 if (cpu_single_env->kqemu_enabled &&
2448 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2449 kqemu_modify_page(cpu_single_env, ram_addr);
2450 #endif
2451 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2452 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2453 /* we remove the notdirty callback only if the code has been
2454 flushed */
2455 if (dirty_flags == 0xff)
2456 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2457 }
2458
2459 static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
2460 uint32_t val)
2461 {
2462 int dirty_flags;
2463 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2464 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2465 #if !defined(CONFIG_USER_ONLY)
2466 tb_invalidate_phys_page_fast(ram_addr, 4);
2467 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2468 #endif
2469 }
2470 stl_p(phys_ram_base + ram_addr, val);
2471 #ifdef USE_KQEMU
2472 if (cpu_single_env->kqemu_enabled &&
2473 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2474 kqemu_modify_page(cpu_single_env, ram_addr);
2475 #endif
2476 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2477 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2478 /* we remove the notdirty callback only if the code has been
2479 flushed */
2480 if (dirty_flags == 0xff)
2481 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2482 }
2483
2484 static CPUReadMemoryFunc *error_mem_read[3] = {
2485 NULL, /* never used */
2486 NULL, /* never used */
2487 NULL, /* never used */
2488 };
2489
2490 static CPUWriteMemoryFunc *notdirty_mem_write[3] = {
2491 notdirty_mem_writeb,
2492 notdirty_mem_writew,
2493 notdirty_mem_writel,
2494 };
2495
2496 /* Generate a debug exception if a watchpoint has been hit. */
2497 static void check_watchpoint(int offset, int flags)
2498 {
2499 CPUState *env = cpu_single_env;
2500 target_ulong vaddr;
2501 int i;
2502
2503 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
2504 for (i = 0; i < env->nb_watchpoints; i++) {
2505 if (vaddr == env->watchpoint[i].vaddr
2506 && (env->watchpoint[i].type & flags)) {
2507 env->watchpoint_hit = i + 1;
2508 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
2509 break;
2510 }
2511 }
2512 }
2513
2514 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2515 so these check for a hit then pass through to the normal out-of-line
2516 phys routines. */
2517 static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
2518 {
2519 check_watchpoint(addr & ~TARGET_PAGE_MASK, PAGE_READ);
2520 return ldub_phys(addr);
2521 }
2522
2523 static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
2524 {
2525 check_watchpoint(addr & ~TARGET_PAGE_MASK, PAGE_READ);
2526 return lduw_phys(addr);
2527 }
2528
2529 static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
2530 {
2531 check_watchpoint(addr & ~TARGET_PAGE_MASK, PAGE_READ);
2532 return ldl_phys(addr);
2533 }
2534
2535 static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
2536 uint32_t val)
2537 {
2538 check_watchpoint(addr & ~TARGET_PAGE_MASK, PAGE_WRITE);
2539 stb_phys(addr, val);
2540 }
2541
2542 static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
2543 uint32_t val)
2544 {
2545 check_watchpoint(addr & ~TARGET_PAGE_MASK, PAGE_WRITE);
2546 stw_phys(addr, val);
2547 }
2548
2549 static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
2550 uint32_t val)
2551 {
2552 check_watchpoint(addr & ~TARGET_PAGE_MASK, PAGE_WRITE);
2553 stl_phys(addr, val);
2554 }
2555
2556 static CPUReadMemoryFunc *watch_mem_read[3] = {
2557 watch_mem_readb,
2558 watch_mem_readw,
2559 watch_mem_readl,
2560 };
2561
2562 static CPUWriteMemoryFunc *watch_mem_write[3] = {
2563 watch_mem_writeb,
2564 watch_mem_writew,
2565 watch_mem_writel,
2566 };
2567
2568 static inline uint32_t subpage_readlen (subpage_t *mmio, target_phys_addr_t addr,
2569 unsigned int len)
2570 {
2571 uint32_t ret;
2572 unsigned int idx;
2573
2574 idx = SUBPAGE_IDX(addr - mmio->base);
2575 #if defined(DEBUG_SUBPAGE)
2576 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
2577 mmio, len, addr, idx);
2578 #endif
2579 ret = (**mmio->mem_read[idx][len])(mmio->opaque[idx][0][len], addr);
2580
2581 return ret;
2582 }
2583
2584 static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
2585 uint32_t value, unsigned int len)
2586 {
2587 unsigned int idx;
2588
2589 idx = SUBPAGE_IDX(addr - mmio->base);
2590 #if defined(DEBUG_SUBPAGE)
2591 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n", __func__,
2592 mmio, len, addr, idx, value);
2593 #endif
2594 (**mmio->mem_write[idx][len])(mmio->opaque[idx][1][len], addr, value);
2595 }
2596
2597 static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
2598 {
2599 #if defined(DEBUG_SUBPAGE)
2600 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2601 #endif
2602
2603 return subpage_readlen(opaque, addr, 0);
2604 }
2605
2606 static void subpage_writeb (void *opaque, target_phys_addr_t addr,
2607 uint32_t value)
2608 {
2609 #if defined(DEBUG_SUBPAGE)
2610 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2611 #endif
2612 subpage_writelen(opaque, addr, value, 0);
2613 }
2614
2615 static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
2616 {
2617 #if defined(DEBUG_SUBPAGE)
2618 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2619 #endif
2620
2621 return subpage_readlen(opaque, addr, 1);
2622 }
2623
2624 static void subpage_writew (void *opaque, target_phys_addr_t addr,
2625 uint32_t value)
2626 {
2627 #if defined(DEBUG_SUBPAGE)
2628 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2629 #endif
2630 subpage_writelen(opaque, addr, value, 1);
2631 }
2632
2633 static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
2634 {
2635 #if defined(DEBUG_SUBPAGE)
2636 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2637 #endif
2638
2639 return subpage_readlen(opaque, addr, 2);
2640 }
2641
2642 static void subpage_writel (void *opaque,
2643 target_phys_addr_t addr, uint32_t value)
2644 {
2645 #if defined(DEBUG_SUBPAGE)
2646 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2647 #endif
2648 subpage_writelen(opaque, addr, value, 2);
2649 }
2650
2651 static CPUReadMemoryFunc *subpage_read[] = {
2652 &subpage_readb,
2653 &subpage_readw,
2654 &subpage_readl,
2655 };
2656
2657 static CPUWriteMemoryFunc *subpage_write[] = {
2658 &subpage_writeb,
2659 &subpage_writew,
2660 &subpage_writel,
2661 };
2662
2663 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2664 ram_addr_t memory)
2665 {
2666 int idx, eidx;
2667 unsigned int i;
2668
2669 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2670 return -1;
2671 idx = SUBPAGE_IDX(start);
2672 eidx = SUBPAGE_IDX(end);
2673 #if defined(DEBUG_SUBPAGE)
2674 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %d\n", __func__,
2675 mmio, start, end, idx, eidx, memory);
2676 #endif
2677 memory >>= IO_MEM_SHIFT;
2678 for (; idx <= eidx; idx++) {
2679 for (i = 0; i < 4; i++) {
2680 if (io_mem_read[memory][i]) {
2681 mmio->mem_read[idx][i] = &io_mem_read[memory][i];
2682 mmio->opaque[idx][0][i] = io_mem_opaque[memory];
2683 }
2684 if (io_mem_write[memory][i]) {
2685 mmio->mem_write[idx][i] = &io_mem_write[memory][i];
2686 mmio->opaque[idx][1][i] = io_mem_opaque[memory];
2687 }
2688 }
2689 }
2690
2691 return 0;
2692 }
2693
2694 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2695 ram_addr_t orig_memory)
2696 {
2697 subpage_t *mmio;
2698 int subpage_memory;
2699
2700 mmio = qemu_mallocz(sizeof(subpage_t));
2701 if (mmio != NULL) {
2702 mmio->base = base;
2703 subpage_memory = cpu_register_io_memory(0, subpage_read, subpage_write, mmio);
2704 #if defined(DEBUG_SUBPAGE)
2705 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
2706 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
2707 #endif
2708 *phys = subpage_memory | IO_MEM_SUBPAGE;
2709 subpage_register(mmio, 0, TARGET_PAGE_SIZE - 1, orig_memory);
2710 }
2711
2712 return mmio;
2713 }
2714
2715 static int get_free_io_mem_idx(void)
2716 {
2717 int i;
2718
2719 for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
2720 if (!io_mem_used[i]) {
2721 io_mem_used[i] = 1;
2722 return i;
2723 }
2724
2725 return -1;
2726 }
2727
2728 static void io_mem_init(void)
2729 {
2730 int i;
2731
2732 cpu_register_io_memory(IO_MEM_ROM >> IO_MEM_SHIFT, error_mem_read, unassigned_mem_write, NULL);
2733 cpu_register_io_memory(IO_MEM_UNASSIGNED >> IO_MEM_SHIFT, unassigned_mem_read, unassigned_mem_write, NULL);
2734 cpu_register_io_memory(IO_MEM_NOTDIRTY >> IO_MEM_SHIFT, error_mem_read, notdirty_mem_write, NULL);
2735 for (i=0; i<5; i++)
2736 io_mem_used[i] = 1;
2737
2738 io_mem_watch = cpu_register_io_memory(0, watch_mem_read,
2739 watch_mem_write, NULL);
2740 /* alloc dirty bits array */
2741 phys_ram_dirty = qemu_vmalloc(phys_ram_size >> TARGET_PAGE_BITS);
2742 memset(phys_ram_dirty, 0xff, phys_ram_size >> TARGET_PAGE_BITS);
2743 }
2744
2745 /* mem_read and mem_write are arrays of functions containing the
2746 function to access byte (index 0), word (index 1) and dword (index
2747 2). Functions can be omitted with a NULL function pointer. The
2748 registered functions may be modified dynamically later.
2749 If io_index is non zero, the corresponding io zone is
2750 modified. If it is zero, a new io zone is allocated. The return
2751 value can be used with cpu_register_physical_memory(). (-1) is
2752 returned if error. */
2753 int cpu_register_io_memory(int io_index,
2754 CPUReadMemoryFunc **mem_read,
2755 CPUWriteMemoryFunc **mem_write,
2756 void *opaque)
2757 {
2758 int i, subwidth = 0;
2759
2760 if (io_index <= 0) {
2761 io_index = get_free_io_mem_idx();
2762 if (io_index == -1)
2763 return io_index;
2764 } else {
2765 if (io_index >= IO_MEM_NB_ENTRIES)
2766 return -1;
2767 }
2768
2769 for(i = 0;i < 3; i++) {
2770 if (!mem_read[i] || !mem_write[i])
2771 subwidth = IO_MEM_SUBWIDTH;
2772 io_mem_read[io_index][i] = mem_read[i];
2773 io_mem_write[io_index][i] = mem_write[i];
2774 }
2775 io_mem_opaque[io_index] = opaque;
2776 return (io_index << IO_MEM_SHIFT) | subwidth;
2777 }
2778
2779 void cpu_unregister_io_memory(int io_table_address)
2780 {
2781 int i;
2782 int io_index = io_table_address >> IO_MEM_SHIFT;
2783
2784 for (i=0;i < 3; i++) {
2785 io_mem_read[io_index][i] = unassigned_mem_read[i];
2786 io_mem_write[io_index][i] = unassigned_mem_write[i];
2787 }
2788 io_mem_opaque[io_index] = NULL;
2789 io_mem_used[io_index] = 0;
2790 }
2791
2792 CPUWriteMemoryFunc **cpu_get_io_memory_write(int io_index)
2793 {
2794 return io_mem_write[io_index >> IO_MEM_SHIFT];
2795 }
2796
2797 CPUReadMemoryFunc **cpu_get_io_memory_read(int io_index)
2798 {
2799 return io_mem_read[io_index >> IO_MEM_SHIFT];
2800 }
2801
2802 #endif /* !defined(CONFIG_USER_ONLY) */
2803
2804 /* physical memory access (slow version, mainly for debug) */
2805 #if defined(CONFIG_USER_ONLY)
2806 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
2807 int len, int is_write)
2808 {
2809 int l, flags;
2810 target_ulong page;
2811 void * p;
2812
2813 while (len > 0) {
2814 page = addr & TARGET_PAGE_MASK;
2815 l = (page + TARGET_PAGE_SIZE) - addr;
2816 if (l > len)
2817 l = len;
2818 flags = page_get_flags(page);
2819 if (!(flags & PAGE_VALID))
2820 return;
2821 if (is_write) {
2822 if (!(flags & PAGE_WRITE))
2823 return;
2824 /* XXX: this code should not depend on lock_user */
2825 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
2826 /* FIXME - should this return an error rather than just fail? */
2827 return;
2828 memcpy(p, buf, l);
2829 unlock_user(p, addr, l);
2830 } else {
2831 if (!(flags & PAGE_READ))
2832 return;
2833 /* XXX: this code should not depend on lock_user */
2834 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
2835 /* FIXME - should this return an error rather than just fail? */
2836 return;
2837 memcpy(buf, p, l);
2838 unlock_user(p, addr, 0);
2839 }
2840 len -= l;
2841 buf += l;
2842 addr += l;
2843 }
2844 }
2845
2846 #else
2847 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
2848 int len, int is_write)
2849 {
2850 int l, io_index;
2851 uint8_t *ptr;
2852 uint32_t val;
2853 target_phys_addr_t page;
2854 unsigned long pd;
2855 PhysPageDesc *p;
2856
2857 while (len > 0) {
2858 page = addr & TARGET_PAGE_MASK;
2859 l = (page + TARGET_PAGE_SIZE) - addr;
2860 if (l > len)
2861 l = len;
2862 p = phys_page_find(page >> TARGET_PAGE_BITS);
2863 if (!p) {
2864 pd = IO_MEM_UNASSIGNED;
2865 } else {
2866 pd = p->phys_offset;
2867 }
2868
2869 if (is_write) {
2870 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
2871 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
2872 /* XXX: could force cpu_single_env to NULL to avoid
2873 potential bugs */
2874 if (l >= 4 && ((addr & 3) == 0)) {
2875 /* 32 bit write access */
2876 val = ldl_p(buf);
2877 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
2878 l = 4;
2879 } else if (l >= 2 && ((addr & 1) == 0)) {
2880 /* 16 bit write access */
2881 val = lduw_p(buf);
2882 io_mem_write[io_index][1](io_mem_opaque[io_index], addr, val);
2883 l = 2;
2884 } else {
2885 /* 8 bit write access */
2886 val = ldub_p(buf);
2887 io_mem_write[io_index][0](io_mem_opaque[io_index], addr, val);
2888 l = 1;
2889 }
2890 } else {
2891 unsigned long addr1;
2892 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
2893 /* RAM case */
2894 ptr = phys_ram_base + addr1;
2895 memcpy(ptr, buf, l);
2896 if (!cpu_physical_memory_is_dirty(addr1)) {
2897 /* invalidate code */
2898 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
2899 /* set dirty bit */
2900 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
2901 (0xff & ~CODE_DIRTY_FLAG);
2902 }
2903 /* qemu doesn't execute guest code directly, but kvm does
2904 therefore fluch instruction caches */
2905 if (kvm_enabled())
2906 flush_icache_range((unsigned long)ptr,
2907 ((unsigned long)ptr)+l);
2908 }
2909 } else {
2910 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
2911 !(pd & IO_MEM_ROMD)) {
2912 /* I/O case */
2913 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
2914 if (l >= 4 && ((addr & 3) == 0)) {
2915 /* 32 bit read access */
2916 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
2917 stl_p(buf, val);
2918 l = 4;
2919 } else if (l >= 2 && ((addr & 1) == 0)) {
2920 /* 16 bit read access */
2921 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr);
2922 stw_p(buf, val);
2923 l = 2;
2924 } else {
2925 /* 8 bit read access */
2926 val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr);
2927 stb_p(buf, val);
2928 l = 1;
2929 }
2930 } else {
2931 /* RAM case */
2932 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
2933 (addr & ~TARGET_PAGE_MASK);
2934 memcpy(buf, ptr, l);
2935 }
2936 }
2937 len -= l;
2938 buf += l;
2939 addr += l;
2940 }
2941 }
2942
2943 /* used for ROM loading : can write in RAM and ROM */
2944 void cpu_physical_memory_write_rom(target_phys_addr_t addr,
2945 const uint8_t *buf, int len)
2946 {
2947 int l;
2948 uint8_t *ptr;
2949 target_phys_addr_t page;
2950 unsigned long pd;
2951 PhysPageDesc *p;
2952
2953 while (len > 0) {
2954 page = addr & TARGET_PAGE_MASK;
2955 l = (page + TARGET_PAGE_SIZE) - addr;
2956 if (l > len)
2957 l = len;
2958 p = phys_page_find(page >> TARGET_PAGE_BITS);
2959 if (!p) {
2960 pd = IO_MEM_UNASSIGNED;
2961 } else {
2962 pd = p->phys_offset;
2963 }
2964
2965 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
2966 (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
2967 !(pd & IO_MEM_ROMD)) {
2968 /* do nothing */
2969 } else {
2970 unsigned long addr1;
2971 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
2972 /* ROM/RAM case */
2973 ptr = phys_ram_base + addr1;
2974 memcpy(ptr, buf, l);
2975 }
2976 len -= l;
2977 buf += l;
2978 addr += l;
2979 }
2980 }
2981
2982
2983 /* warning: addr must be aligned */
2984 uint32_t ldl_phys(target_phys_addr_t addr)
2985 {
2986 int io_index;
2987 uint8_t *ptr;
2988 uint32_t val;
2989 unsigned long pd;
2990 PhysPageDesc *p;
2991
2992 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2993 if (!p) {
2994 pd = IO_MEM_UNASSIGNED;
2995 } else {
2996 pd = p->phys_offset;
2997 }
2998
2999 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3000 !(pd & IO_MEM_ROMD)) {
3001 /* I/O case */
3002 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3003 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3004 } else {
3005 /* RAM case */
3006 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3007 (addr & ~TARGET_PAGE_MASK);
3008 val = ldl_p(ptr);
3009 }
3010 return val;
3011 }
3012
3013 /* warning: addr must be aligned */
3014 uint64_t ldq_phys(target_phys_addr_t addr)
3015 {
3016 int io_index;
3017 uint8_t *ptr;
3018 uint64_t val;
3019 unsigned long pd;
3020 PhysPageDesc *p;
3021
3022 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3023 if (!p) {
3024 pd = IO_MEM_UNASSIGNED;
3025 } else {
3026 pd = p->phys_offset;
3027 }
3028
3029 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3030 !(pd & IO_MEM_ROMD)) {
3031 /* I/O case */
3032 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3033 #ifdef TARGET_WORDS_BIGENDIAN
3034 val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
3035 val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
3036 #else
3037 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3038 val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
3039 #endif
3040 } else {
3041 /* RAM case */
3042 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3043 (addr & ~TARGET_PAGE_MASK);
3044 val = ldq_p(ptr);
3045 }
3046 return val;
3047 }
3048
3049 /* XXX: optimize */
3050 uint32_t ldub_phys(target_phys_addr_t addr)
3051 {
3052 uint8_t val;
3053 cpu_physical_memory_read(addr, &val, 1);
3054 return val;
3055 }
3056
3057 /* XXX: optimize */
3058 uint32_t lduw_phys(target_phys_addr_t addr)
3059 {
3060 uint16_t val;
3061 cpu_physical_memory_read(addr, (uint8_t *)&val, 2);
3062 return tswap16(val);
3063 }
3064
3065 #ifdef __GNUC__
3066 #define likely(x) __builtin_expect(!!(x), 1)
3067 #define unlikely(x) __builtin_expect(!!(x), 0)
3068 #else
3069 #define likely(x) x
3070 #define unlikely(x) x
3071 #endif
3072
3073 /* warning: addr must be aligned. The ram page is not masked as dirty
3074 and the code inside is not invalidated. It is useful if the dirty
3075 bits are used to track modified PTEs */
3076 void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
3077 {
3078 int io_index;
3079 uint8_t *ptr;
3080 unsigned long pd;
3081 PhysPageDesc *p;
3082
3083 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3084 if (!p) {
3085 pd = IO_MEM_UNASSIGNED;
3086 } else {
3087 pd = p->phys_offset;
3088 }
3089
3090 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3091 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3092 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3093 } else {
3094 unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3095 ptr = phys_ram_base + addr1;
3096 stl_p(ptr, val);
3097
3098 if (unlikely(in_migration)) {
3099 if (!cpu_physical_memory_is_dirty(addr1)) {
3100 /* invalidate code */
3101 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3102 /* set dirty bit */
3103 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3104 (0xff & ~CODE_DIRTY_FLAG);
3105 }
3106 }
3107 }
3108 }
3109
3110 void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
3111 {
3112 int io_index;
3113 uint8_t *ptr;
3114 unsigned long pd;
3115 PhysPageDesc *p;
3116
3117 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3118 if (!p) {
3119 pd = IO_MEM_UNASSIGNED;
3120 } else {
3121 pd = p->phys_offset;
3122 }
3123
3124 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3125 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3126 #ifdef TARGET_WORDS_BIGENDIAN
3127 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
3128 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
3129 #else
3130 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3131 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
3132 #endif
3133 } else {
3134 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3135 (addr & ~TARGET_PAGE_MASK);
3136 stq_p(ptr, val);
3137 }
3138 }
3139
3140 /* warning: addr must be aligned */
3141 void stl_phys(target_phys_addr_t addr, uint32_t val)
3142 {
3143 int io_index;
3144 uint8_t *ptr;
3145 unsigned long pd;
3146 PhysPageDesc *p;
3147
3148 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3149 if (!p) {
3150 pd = IO_MEM_UNASSIGNED;
3151 } else {
3152 pd = p->phys_offset;
3153 }
3154
3155 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3156 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3157 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3158 } else {
3159 unsigned long addr1;
3160 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3161 /* RAM case */
3162 ptr = phys_ram_base + addr1;
3163 stl_p(ptr, val);
3164 if (!cpu_physical_memory_is_dirty(addr1)) {
3165 /* invalidate code */
3166 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3167 /* set dirty bit */
3168 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3169 (0xff & ~CODE_DIRTY_FLAG);
3170 }
3171 }
3172 }
3173
3174 /* XXX: optimize */
3175 void stb_phys(target_phys_addr_t addr, uint32_t val)
3176 {
3177 uint8_t v = val;
3178 cpu_physical_memory_write(addr, &v, 1);
3179 }
3180
3181 /* XXX: optimize */
3182 void stw_phys(target_phys_addr_t addr, uint32_t val)
3183 {
3184 uint16_t v = tswap16(val);
3185 cpu_physical_memory_write(addr, (const uint8_t *)&v, 2);
3186 }
3187
3188 /* XXX: optimize */
3189 void stq_phys(target_phys_addr_t addr, uint64_t val)
3190 {
3191 val = tswap64(val);
3192 cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
3193 }
3194
3195 #endif
3196
3197 /* virtual memory access for debug */
3198 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3199 uint8_t *buf, int len, int is_write)
3200 {
3201 int l;
3202 target_phys_addr_t phys_addr;
3203 target_ulong page;
3204
3205 while (len > 0) {
3206 page = addr & TARGET_PAGE_MASK;
3207 phys_addr = cpu_get_phys_page_debug(env, page);
3208 /* if no physical page mapped, return an error */
3209 if (phys_addr == -1)
3210 return -1;
3211 l = (page + TARGET_PAGE_SIZE) - addr;
3212 if (l > len)
3213 l = len;
3214 cpu_physical_memory_rw(phys_addr + (addr & ~TARGET_PAGE_MASK),
3215 buf, l, is_write);
3216 len -= l;
3217 buf += l;
3218 addr += l;
3219 }
3220 return 0;
3221 }
3222
3223 /* in deterministic execution mode, instructions doing device I/Os
3224 must be at the end of the TB */
3225 void cpu_io_recompile(CPUState *env, void *retaddr)
3226 {
3227 TranslationBlock *tb;
3228 uint32_t n, cflags;
3229 target_ulong pc, cs_base;
3230 uint64_t flags;
3231
3232 tb = tb_find_pc((unsigned long)retaddr);
3233 if (!tb) {
3234 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
3235 retaddr);
3236 }
3237 n = env->icount_decr.u16.low + tb->icount;
3238 cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
3239 /* Calculate how many instructions had been executed before the fault
3240 occurred. */
3241 n = n - env->icount_decr.u16.low;
3242 /* Generate a new TB ending on the I/O insn. */
3243 n++;
3244 /* On MIPS and SH, delay slot instructions can only be restarted if
3245 they were already the first instruction in the TB. If this is not
3246 the first instruction in a TB then re-execute the preceding
3247 branch. */
3248 #if defined(TARGET_MIPS)
3249 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
3250 env->active_tc.PC -= 4;
3251 env->icount_decr.u16.low++;
3252 env->hflags &= ~MIPS_HFLAG_BMASK;
3253 }
3254 #elif defined(TARGET_SH4)
3255 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
3256 && n > 1) {
3257 env->pc -= 2;
3258 env->icount_decr.u16.low++;
3259 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
3260 }
3261 #endif
3262 /* This should never happen. */
3263 if (n > CF_COUNT_MASK)
3264 cpu_abort(env, "TB too big during recompile");
3265
3266 cflags = n | CF_LAST_IO;
3267 pc = tb->pc;
3268 cs_base = tb->cs_base;
3269 flags = tb->flags;
3270 tb_phys_invalidate(tb, -1);
3271 /* FIXME: In theory this could raise an exception. In practice
3272 we have already translated the block once so it's probably ok. */
3273 tb_gen_code(env, pc, cs_base, flags, cflags);
3274 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
3275 the first in the TB) then we end up generating a whole new TB and
3276 repeating the fault, which is horribly inefficient.
3277 Better would be to execute just this insn uncached, or generate a
3278 second new TB. */
3279 cpu_resume_from_signal(env, NULL);
3280 }
3281
3282 void dump_exec_info(FILE *f,
3283 int (*cpu_fprintf)(FILE *f, const char *fmt, ...))
3284 {
3285 int i, target_code_size, max_target_code_size;
3286 int direct_jmp_count, direct_jmp2_count, cross_page;
3287 TranslationBlock *tb;
3288
3289 target_code_size = 0;
3290 max_target_code_size = 0;
3291 cross_page = 0;
3292 direct_jmp_count = 0;
3293 direct_jmp2_count = 0;
3294 for(i = 0; i < nb_tbs; i++) {
3295 tb = &tbs[i];
3296 target_code_size += tb->size;
3297 if (tb->size > max_target_code_size)
3298 max_target_code_size = tb->size;
3299 if (tb->page_addr[1] != -1)
3300 cross_page++;
3301 if (tb->tb_next_offset[0] != 0xffff) {
3302 direct_jmp_count++;
3303 if (tb->tb_next_offset[1] != 0xffff) {
3304 direct_jmp2_count++;
3305 }
3306 }
3307 }
3308 /* XXX: avoid using doubles ? */
3309 cpu_fprintf(f, "Translation buffer state:\n");
3310 cpu_fprintf(f, "gen code size %ld/%ld\n",
3311 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
3312 cpu_fprintf(f, "TB count %d/%d\n",
3313 nb_tbs, code_gen_max_blocks);
3314 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
3315 nb_tbs ? target_code_size / nb_tbs : 0,
3316 max_target_code_size);
3317 cpu_fprintf(f, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
3318 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
3319 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
3320 cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
3321 cross_page,
3322 nb_tbs ? (cross_page * 100) / nb_tbs : 0);
3323 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
3324 direct_jmp_count,
3325 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
3326 direct_jmp2_count,
3327 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
3328 cpu_fprintf(f, "\nStatistics:\n");
3329 cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
3330 cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
3331 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
3332 tcg_dump_info(f, cpu_fprintf);
3333 }
3334
3335 #if !defined(CONFIG_USER_ONLY)
3336
3337 #define MMUSUFFIX _cmmu
3338 #define GETPC() NULL
3339 #define env cpu_single_env
3340 #define SOFTMMU_CODE_ACCESS
3341
3342 #define SHIFT 0
3343 #include "softmmu_template.h"
3344
3345 #define SHIFT 1
3346 #include "softmmu_template.h"
3347
3348 #define SHIFT 2
3349 #include "softmmu_template.h"
3350
3351 #define SHIFT 3
3352 #include "softmmu_template.h"
3353
3354 #undef env
3355
3356 #endif
Patches shipped by Fedora