?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
kvm: user: set $PROCESSOR from configure --arch=<arch>-<processor>
[qemu-kvm/fedora.git] / exec.c
blob51a7f9f62e639510302c2bdff187c322278bfcab
1 /*
2 * virtual page mapping and translated block handling
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 */
20 #include "config.h"
21 #ifdef _WIN32
22 #define WIN32_LEAN_AND_MEAN
23 #include <windows.h>
24 #else
25 #include <sys/types.h>
26 #include <sys/mman.h>
27 #endif
28 #include <stdlib.h>
29 #include <stdio.h>
30 #include <stdarg.h>
31 #include <string.h>
32 #include <errno.h>
33 #include <unistd.h>
34 #include <inttypes.h>
35
36 #include "cpu.h"
37 #include "exec-all.h"
38 #include "qemu-common.h"
39
40 #if !defined(TARGET_IA64)
41 #include "tcg.h"
42 #endif
43 #include "qemu-kvm.h"
44
45 #include "hw/hw.h"
46 #include "osdep.h"
47 #if defined(CONFIG_USER_ONLY)
48 #include <qemu.h>
49 #endif
50
51 //#define DEBUG_TB_INVALIDATE
52 //#define DEBUG_FLUSH
53 //#define DEBUG_TLB
54 //#define DEBUG_UNASSIGNED
55
56 /* make various TB consistency checks */
57 //#define DEBUG_TB_CHECK
58 //#define DEBUG_TLB_CHECK
59
60 //#define DEBUG_IOPORT
61 //#define DEBUG_SUBPAGE
62
63 #if !defined(CONFIG_USER_ONLY)
64 /* TB consistency checks only implemented for usermode emulation. */
65 #undef DEBUG_TB_CHECK
66 #endif
67
68 #define SMC_BITMAP_USE_THRESHOLD 10
69
70 #define MMAP_AREA_START 0x00000000
71 #define MMAP_AREA_END 0xa8000000
72
73 #if defined(TARGET_SPARC64)
74 #define TARGET_PHYS_ADDR_SPACE_BITS 41
75 #elif defined(TARGET_SPARC)
76 #define TARGET_PHYS_ADDR_SPACE_BITS 36
77 #elif defined(TARGET_ALPHA)
78 #define TARGET_PHYS_ADDR_SPACE_BITS 42
79 #define TARGET_VIRT_ADDR_SPACE_BITS 42
80 #elif defined(TARGET_PPC64)
81 #define TARGET_PHYS_ADDR_SPACE_BITS 42
82 #elif defined(TARGET_X86_64) && !defined(USE_KQEMU)
83 #define TARGET_PHYS_ADDR_SPACE_BITS 42
84 #elif defined(TARGET_I386) && !defined(USE_KQEMU)
85 #define TARGET_PHYS_ADDR_SPACE_BITS 36
86 #elif defined(TARGET_IA64)
87 #define TARGET_PHYS_ADDR_SPACE_BITS 36
88 #else
89 /* Note: for compatibility with kqemu, we use 32 bits for x86_64 */
90 #define TARGET_PHYS_ADDR_SPACE_BITS 32
91 #endif
92
93 static TranslationBlock *tbs;
94 int code_gen_max_blocks;
95 TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
96 static int nb_tbs;
97 /* any access to the tbs or the page table must use this lock */
98 spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
99
100 #if defined(__arm__) || defined(__sparc_v9__)
101 /* The prologue must be reachable with a direct jump. ARM and Sparc64
102 have limited branch ranges (possibly also PPC) so place it in a
103 section close to code segment. */
104 #define code_gen_section \
105 __attribute__((__section__(".gen_code"))) \
106 __attribute__((aligned (32)))
107 #else
108 #define code_gen_section \
109 __attribute__((aligned (32)))
110 #endif
111
112 uint8_t code_gen_prologue[1024] code_gen_section;
113 static uint8_t *code_gen_buffer;
114 static unsigned long code_gen_buffer_size;
115 /* threshold to flush the translated code buffer */
116 static unsigned long code_gen_buffer_max_size;
117 uint8_t *code_gen_ptr;
118
119 #if !defined(CONFIG_USER_ONLY)
120 ram_addr_t phys_ram_size;
121 int phys_ram_fd;
122 uint8_t *phys_ram_base;
123 uint8_t *phys_ram_dirty;
124 uint8_t *bios_mem;
125 static int in_migration;
126 static ram_addr_t phys_ram_alloc_offset = 0;
127 #endif
128
129 CPUState *first_cpu;
130 /* current CPU in the current thread. It is only valid inside
131 cpu_exec() */
132 CPUState *cpu_single_env;
133 /* 0 = Do not count executed instructions.
134 1 = Precise instruction counting.
135 2 = Adaptive rate instruction counting. */
136 int use_icount = 0;
137 /* Current instruction counter. While executing translated code this may
138 include some instructions that have not yet been executed. */
139 int64_t qemu_icount;
140
141 typedef struct PageDesc {
142 /* list of TBs intersecting this ram page */
143 TranslationBlock *first_tb;
144 /* in order to optimize self modifying code, we count the number
145 of lookups we do to a given page to use a bitmap */
146 unsigned int code_write_count;
147 uint8_t *code_bitmap;
148 #if defined(CONFIG_USER_ONLY)
149 unsigned long flags;
150 #endif
151 } PageDesc;
152
153 typedef struct PhysPageDesc {
154 /* offset in host memory of the page + io_index in the low bits */
155 ram_addr_t phys_offset;
156 } PhysPageDesc;
157
158 #define L2_BITS 10
159 #if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
160 /* XXX: this is a temporary hack for alpha target.
161 * In the future, this is to be replaced by a multi-level table
162 * to actually be able to handle the complete 64 bits address space.
163 */
164 #define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
165 #else
166 #define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
167 #endif
168
169 #define L1_SIZE (1 << L1_BITS)
170 #define L2_SIZE (1 << L2_BITS)
171
172 unsigned long qemu_real_host_page_size;
173 unsigned long qemu_host_page_bits;
174 unsigned long qemu_host_page_size;
175 unsigned long qemu_host_page_mask;
176
177 /* XXX: for system emulation, it could just be an array */
178 static PageDesc *l1_map[L1_SIZE];
179 static PhysPageDesc **l1_phys_map;
180
181 #if !defined(CONFIG_USER_ONLY)
182 static void io_mem_init(void);
183
184 /* io memory support */
185 CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
186 CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
187 void *io_mem_opaque[IO_MEM_NB_ENTRIES];
188 char io_mem_used[IO_MEM_NB_ENTRIES];
189 static int io_mem_watch;
190 #endif
191
192 /* log support */
193 static const char *logfilename = "/tmp/qemu.log";
194 FILE *logfile;
195 int loglevel;
196 static int log_append = 0;
197
198 /* statistics */
199 static int tlb_flush_count;
200 static int tb_flush_count;
201 static int tb_phys_invalidate_count;
202
203 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
204 typedef struct subpage_t {
205 target_phys_addr_t base;
206 CPUReadMemoryFunc **mem_read[TARGET_PAGE_SIZE][4];
207 CPUWriteMemoryFunc **mem_write[TARGET_PAGE_SIZE][4];
208 void *opaque[TARGET_PAGE_SIZE][2][4];
209 } subpage_t;
210
211 #ifdef _WIN32
212 static void map_exec(void *addr, long size)
213 {
214 DWORD old_protect;
215 VirtualProtect(addr, size,
216 PAGE_EXECUTE_READWRITE, &old_protect);
217
218 }
219 #else
220 static void map_exec(void *addr, long size)
221 {
222 unsigned long start, end, page_size;
223
224 page_size = getpagesize();
225 start = (unsigned long)addr;
226 start &= ~(page_size - 1);
227
228 end = (unsigned long)addr + size;
229 end += page_size - 1;
230 end &= ~(page_size - 1);
231
232 mprotect((void *)start, end - start,
233 PROT_READ | PROT_WRITE | PROT_EXEC);
234 }
235 #endif
236
237 static void page_init(void)
238 {
239 /* NOTE: we can always suppose that qemu_host_page_size >=
240 TARGET_PAGE_SIZE */
241 #ifdef _WIN32
242 {
243 SYSTEM_INFO system_info;
244 DWORD old_protect;
245
246 GetSystemInfo(&system_info);
247 qemu_real_host_page_size = system_info.dwPageSize;
248 }
249 #else
250 qemu_real_host_page_size = getpagesize();
251 #endif
252 if (qemu_host_page_size == 0)
253 qemu_host_page_size = qemu_real_host_page_size;
254 if (qemu_host_page_size < TARGET_PAGE_SIZE)
255 qemu_host_page_size = TARGET_PAGE_SIZE;
256 qemu_host_page_bits = 0;
257 while ((1 << qemu_host_page_bits) < qemu_host_page_size)
258 qemu_host_page_bits++;
259 qemu_host_page_mask = ~(qemu_host_page_size - 1);
260 l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));
261 memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));
262
263 #if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
264 {
265 long long startaddr, endaddr;
266 FILE *f;
267 int n;
268
269 mmap_lock();
270 last_brk = (unsigned long)sbrk(0);
271 f = fopen("/proc/self/maps", "r");
272 if (f) {
273 do {
274 n = fscanf (f, "%llx-%llx %*[^\n]\n", &startaddr, &endaddr);
275 if (n == 2) {
276 startaddr = MIN(startaddr,
277 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
278 endaddr = MIN(endaddr,
279 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
280 page_set_flags(startaddr & TARGET_PAGE_MASK,
281 TARGET_PAGE_ALIGN(endaddr),
282 PAGE_RESERVED);
283 }
284 } while (!feof(f));
285 fclose(f);
286 }
287 mmap_unlock();
288 }
289 #endif
290 }
291
292 static inline PageDesc **page_l1_map(target_ulong index)
293 {
294 #if TARGET_LONG_BITS > 32
295 /* Host memory outside guest VM. For 32-bit targets we have already
296 excluded high addresses. */
297 if (index > ((target_ulong)L2_SIZE * L1_SIZE))
298 return NULL;
299 #endif
300 return &l1_map[index >> L2_BITS];
301 }
302
303 static inline PageDesc *page_find_alloc(target_ulong index)
304 {
305 PageDesc **lp, *p;
306 lp = page_l1_map(index);
307 if (!lp)
308 return NULL;
309
310 p = *lp;
311 if (!p) {
312 /* allocate if not found */
313 #if defined(CONFIG_USER_ONLY)
314 unsigned long addr;
315 size_t len = sizeof(PageDesc) * L2_SIZE;
316 /* Don't use qemu_malloc because it may recurse. */
317 p = mmap(0, len, PROT_READ | PROT_WRITE,
318 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
319 *lp = p;
320 addr = h2g(p);
321 if (addr == (target_ulong)addr) {
322 page_set_flags(addr & TARGET_PAGE_MASK,
323 TARGET_PAGE_ALIGN(addr + len),
324 PAGE_RESERVED);
325 }
326 #else
327 p = qemu_mallocz(sizeof(PageDesc) * L2_SIZE);
328 *lp = p;
329 #endif
330 }
331 return p + (index & (L2_SIZE - 1));
332 }
333
334 static inline PageDesc *page_find(target_ulong index)
335 {
336 PageDesc **lp, *p;
337 lp = page_l1_map(index);
338 if (!lp)
339 return NULL;
340
341 p = *lp;
342 if (!p)
343 return 0;
344 return p + (index & (L2_SIZE - 1));
345 }
346
347 static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
348 {
349 void **lp, **p;
350 PhysPageDesc *pd;
351
352 p = (void **)l1_phys_map;
353 #if TARGET_PHYS_ADDR_SPACE_BITS > 32
354
355 #if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
356 #error unsupported TARGET_PHYS_ADDR_SPACE_BITS
357 #endif
358 lp = p + ((index >> (L1_BITS + L2_BITS)) & (L1_SIZE - 1));
359 p = *lp;
360 if (!p) {
361 /* allocate if not found */
362 if (!alloc)
363 return NULL;
364 p = qemu_vmalloc(sizeof(void *) * L1_SIZE);
365 memset(p, 0, sizeof(void *) * L1_SIZE);
366 *lp = p;
367 }
368 #endif
369 lp = p + ((index >> L2_BITS) & (L1_SIZE - 1));
370 pd = *lp;
371 if (!pd) {
372 int i;
373 /* allocate if not found */
374 if (!alloc)
375 return NULL;
376 pd = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);
377 *lp = pd;
378 for (i = 0; i < L2_SIZE; i++)
379 pd[i].phys_offset = IO_MEM_UNASSIGNED;
380 }
381 return ((PhysPageDesc *)pd) + (index & (L2_SIZE - 1));
382 }
383
384 static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
385 {
386 return phys_page_find_alloc(index, 0);
387 }
388
389 #if !defined(CONFIG_USER_ONLY)
390 static void tlb_protect_code(ram_addr_t ram_addr);
391 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
392 target_ulong vaddr);
393 #define mmap_lock() do { } while(0)
394 #define mmap_unlock() do { } while(0)
395 #endif
396
397 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
398
399 #if defined(CONFIG_USER_ONLY)
400 /* Currently it is not recommanded to allocate big chunks of data in
401 user mode. It will change when a dedicated libc will be used */
402 #define USE_STATIC_CODE_GEN_BUFFER
403 #endif
404
405 #ifdef USE_STATIC_CODE_GEN_BUFFER
406 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE];
407 #endif
408
409 static void code_gen_alloc(unsigned long tb_size)
410 {
411 if (kvm_enabled())
412 return;
413
414 #ifdef USE_STATIC_CODE_GEN_BUFFER
415 code_gen_buffer = static_code_gen_buffer;
416 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
417 map_exec(code_gen_buffer, code_gen_buffer_size);
418 #else
419 code_gen_buffer_size = tb_size;
420 if (code_gen_buffer_size == 0) {
421 #if defined(CONFIG_USER_ONLY)
422 /* in user mode, phys_ram_size is not meaningful */
423 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
424 #else
425 /* XXX: needs ajustments */
426 code_gen_buffer_size = (unsigned long)(phys_ram_size / 4);
427 #endif
428 }
429 if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
430 code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
431 /* The code gen buffer location may have constraints depending on
432 the host cpu and OS */
433 #if defined(__linux__)
434 {
435 int flags;
436 void *start = NULL;
437
438 flags = MAP_PRIVATE | MAP_ANONYMOUS;
439 #if defined(__x86_64__)
440 flags |= MAP_32BIT;
441 /* Cannot map more than that */
442 if (code_gen_buffer_size > (800 * 1024 * 1024))
443 code_gen_buffer_size = (800 * 1024 * 1024);
444 #elif defined(__sparc_v9__)
445 // Map the buffer below 2G, so we can use direct calls and branches
446 flags |= MAP_FIXED;
447 start = (void *) 0x60000000UL;
448 if (code_gen_buffer_size > (512 * 1024 * 1024))
449 code_gen_buffer_size = (512 * 1024 * 1024);
450 #endif
451 code_gen_buffer = mmap(start, code_gen_buffer_size,
452 PROT_WRITE | PROT_READ | PROT_EXEC,
453 flags, -1, 0);
454 if (code_gen_buffer == MAP_FAILED) {
455 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
456 exit(1);
457 }
458 }
459 #elif defined(__FreeBSD__)
460 {
461 int flags;
462 void *addr = NULL;
463 flags = MAP_PRIVATE | MAP_ANONYMOUS;
464 #if defined(__x86_64__)
465 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
466 * 0x40000000 is free */
467 flags |= MAP_FIXED;
468 addr = (void *)0x40000000;
469 /* Cannot map more than that */
470 if (code_gen_buffer_size > (800 * 1024 * 1024))
471 code_gen_buffer_size = (800 * 1024 * 1024);
472 #endif
473 code_gen_buffer = mmap(addr, code_gen_buffer_size,
474 PROT_WRITE | PROT_READ | PROT_EXEC,
475 flags, -1, 0);
476 if (code_gen_buffer == MAP_FAILED) {
477 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
478 exit(1);
479 }
480 }
481 #else
482 code_gen_buffer = qemu_malloc(code_gen_buffer_size);
483 if (!code_gen_buffer) {
484 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
485 exit(1);
486 }
487 map_exec(code_gen_buffer, code_gen_buffer_size);
488 #endif
489 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
490 map_exec(code_gen_prologue, sizeof(code_gen_prologue));
491 code_gen_buffer_max_size = code_gen_buffer_size -
492 code_gen_max_block_size();
493 code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
494 tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
495 }
496
497 /* Must be called before using the QEMU cpus. 'tb_size' is the size
498 (in bytes) allocated to the translation buffer. Zero means default
499 size. */
500 void cpu_exec_init_all(unsigned long tb_size)
501 {
502 cpu_gen_init();
503 code_gen_alloc(tb_size);
504 code_gen_ptr = code_gen_buffer;
505 page_init();
506 #if !defined(CONFIG_USER_ONLY)
507 io_mem_init();
508 #endif
509 }
510
511 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
512
513 #define CPU_COMMON_SAVE_VERSION 1
514
515 static void cpu_common_save(QEMUFile *f, void *opaque)
516 {
517 CPUState *env = opaque;
518
519 qemu_put_be32s(f, &env->halted);
520 qemu_put_be32s(f, &env->interrupt_request);
521 }
522
523 static int cpu_common_load(QEMUFile *f, void *opaque, int version_id)
524 {
525 CPUState *env = opaque;
526
527 if (version_id != CPU_COMMON_SAVE_VERSION)
528 return -EINVAL;
529
530 qemu_get_be32s(f, &env->halted);
531 qemu_get_be32s(f, &env->interrupt_request);
532 tlb_flush(env, 1);
533
534 return 0;
535 }
536 #endif
537
538 void cpu_exec_init(CPUState *env)
539 {
540 CPUState **penv;
541 int cpu_index;
542
543 env->next_cpu = NULL;
544 penv = &first_cpu;
545 cpu_index = 0;
546 while (*penv != NULL) {
547 penv = (CPUState **)&(*penv)->next_cpu;
548 cpu_index++;
549 }
550 env->cpu_index = cpu_index;
551 env->nb_watchpoints = 0;
552 #ifdef __WIN32
553 env->thread_id = GetCurrentProcessId();
554 #else
555 env->thread_id = getpid();
556 #endif
557 *penv = env;
558 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
559 register_savevm("cpu_common", cpu_index, CPU_COMMON_SAVE_VERSION,
560 cpu_common_save, cpu_common_load, env);
561 register_savevm("cpu", cpu_index, CPU_SAVE_VERSION,
562 cpu_save, cpu_load, env);
563 #endif
564 }
565
566 static inline void invalidate_page_bitmap(PageDesc *p)
567 {
568 if (p->code_bitmap) {
569 qemu_free(p->code_bitmap);
570 p->code_bitmap = NULL;
571 }
572 p->code_write_count = 0;
573 }
574
575 /* set to NULL all the 'first_tb' fields in all PageDescs */
576 static void page_flush_tb(void)
577 {
578 int i, j;
579 PageDesc *p;
580
581 for(i = 0; i < L1_SIZE; i++) {
582 p = l1_map[i];
583 if (p) {
584 for(j = 0; j < L2_SIZE; j++) {
585 p->first_tb = NULL;
586 invalidate_page_bitmap(p);
587 p++;
588 }
589 }
590 }
591 }
592
593 /* flush all the translation blocks */
594 /* XXX: tb_flush is currently not thread safe */
595 void tb_flush(CPUState *env1)
596 {
597 CPUState *env;
598 #if defined(DEBUG_FLUSH)
599 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
600 (unsigned long)(code_gen_ptr - code_gen_buffer),
601 nb_tbs, nb_tbs > 0 ?
602 ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
603 #endif
604 if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
605 cpu_abort(env1, "Internal error: code buffer overflow\n");
606
607 nb_tbs = 0;
608
609 for(env = first_cpu; env != NULL; env = env->next_cpu) {
610 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
611 }
612
613 memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
614 page_flush_tb();
615
616 code_gen_ptr = code_gen_buffer;
617 /* XXX: flush processor icache at this point if cache flush is
618 expensive */
619 tb_flush_count++;
620 }
621
622 #ifdef DEBUG_TB_CHECK
623
624 static void tb_invalidate_check(target_ulong address)
625 {
626 TranslationBlock *tb;
627 int i;
628 address &= TARGET_PAGE_MASK;
629 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
630 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
631 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
632 address >= tb->pc + tb->size)) {
633 printf("ERROR invalidate: address=%08lx PC=%08lx size=%04x\n",
634 address, (long)tb->pc, tb->size);
635 }
636 }
637 }
638 }
639
640 /* verify that all the pages have correct rights for code */
641 static void tb_page_check(void)
642 {
643 TranslationBlock *tb;
644 int i, flags1, flags2;
645
646 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
647 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
648 flags1 = page_get_flags(tb->pc);
649 flags2 = page_get_flags(tb->pc + tb->size - 1);
650 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
651 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
652 (long)tb->pc, tb->size, flags1, flags2);
653 }
654 }
655 }
656 }
657
658 static void tb_jmp_check(TranslationBlock *tb)
659 {
660 TranslationBlock *tb1;
661 unsigned int n1;
662
663 /* suppress any remaining jumps to this TB */
664 tb1 = tb->jmp_first;
665 for(;;) {
666 n1 = (long)tb1 & 3;
667 tb1 = (TranslationBlock *)((long)tb1 & ~3);
668 if (n1 == 2)
669 break;
670 tb1 = tb1->jmp_next[n1];
671 }
672 /* check end of list */
673 if (tb1 != tb) {
674 printf("ERROR: jmp_list from 0x%08lx\n", (long)tb);
675 }
676 }
677
678 #endif
679
680 /* invalidate one TB */
681 static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
682 int next_offset)
683 {
684 TranslationBlock *tb1;
685 for(;;) {
686 tb1 = *ptb;
687 if (tb1 == tb) {
688 *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
689 break;
690 }
691 ptb = (TranslationBlock **)((char *)tb1 + next_offset);
692 }
693 }
694
695 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
696 {
697 TranslationBlock *tb1;
698 unsigned int n1;
699
700 for(;;) {
701 tb1 = *ptb;
702 n1 = (long)tb1 & 3;
703 tb1 = (TranslationBlock *)((long)tb1 & ~3);
704 if (tb1 == tb) {
705 *ptb = tb1->page_next[n1];
706 break;
707 }
708 ptb = &tb1->page_next[n1];
709 }
710 }
711
712 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
713 {
714 TranslationBlock *tb1, **ptb;
715 unsigned int n1;
716
717 ptb = &tb->jmp_next[n];
718 tb1 = *ptb;
719 if (tb1) {
720 /* find tb(n) in circular list */
721 for(;;) {
722 tb1 = *ptb;
723 n1 = (long)tb1 & 3;
724 tb1 = (TranslationBlock *)((long)tb1 & ~3);
725 if (n1 == n && tb1 == tb)
726 break;
727 if (n1 == 2) {
728 ptb = &tb1->jmp_first;
729 } else {
730 ptb = &tb1->jmp_next[n1];
731 }
732 }
733 /* now we can suppress tb(n) from the list */
734 *ptb = tb->jmp_next[n];
735
736 tb->jmp_next[n] = NULL;
737 }
738 }
739
740 /* reset the jump entry 'n' of a TB so that it is not chained to
741 another TB */
742 static inline void tb_reset_jump(TranslationBlock *tb, int n)
743 {
744 tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
745 }
746
747 void tb_phys_invalidate(TranslationBlock *tb, target_ulong page_addr)
748 {
749 CPUState *env;
750 PageDesc *p;
751 unsigned int h, n1;
752 target_phys_addr_t phys_pc;
753 TranslationBlock *tb1, *tb2;
754
755 /* remove the TB from the hash list */
756 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
757 h = tb_phys_hash_func(phys_pc);
758 tb_remove(&tb_phys_hash[h], tb,
759 offsetof(TranslationBlock, phys_hash_next));
760
761 /* remove the TB from the page list */
762 if (tb->page_addr[0] != page_addr) {
763 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
764 tb_page_remove(&p->first_tb, tb);
765 invalidate_page_bitmap(p);
766 }
767 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
768 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
769 tb_page_remove(&p->first_tb, tb);
770 invalidate_page_bitmap(p);
771 }
772
773 tb_invalidated_flag = 1;
774
775 /* remove the TB from the hash list */
776 h = tb_jmp_cache_hash_func(tb->pc);
777 for(env = first_cpu; env != NULL; env = env->next_cpu) {
778 if (env->tb_jmp_cache[h] == tb)
779 env->tb_jmp_cache[h] = NULL;
780 }
781
782 /* suppress this TB from the two jump lists */
783 tb_jmp_remove(tb, 0);
784 tb_jmp_remove(tb, 1);
785
786 /* suppress any remaining jumps to this TB */
787 tb1 = tb->jmp_first;
788 for(;;) {
789 n1 = (long)tb1 & 3;
790 if (n1 == 2)
791 break;
792 tb1 = (TranslationBlock *)((long)tb1 & ~3);
793 tb2 = tb1->jmp_next[n1];
794 tb_reset_jump(tb1, n1);
795 tb1->jmp_next[n1] = NULL;
796 tb1 = tb2;
797 }
798 tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
799
800 tb_phys_invalidate_count++;
801 }
802
803 static inline void set_bits(uint8_t *tab, int start, int len)
804 {
805 int end, mask, end1;
806
807 end = start + len;
808 tab += start >> 3;
809 mask = 0xff << (start & 7);
810 if ((start & ~7) == (end & ~7)) {
811 if (start < end) {
812 mask &= ~(0xff << (end & 7));
813 *tab |= mask;
814 }
815 } else {
816 *tab++ |= mask;
817 start = (start + 8) & ~7;
818 end1 = end & ~7;
819 while (start < end1) {
820 *tab++ = 0xff;
821 start += 8;
822 }
823 if (start < end) {
824 mask = ~(0xff << (end & 7));
825 *tab |= mask;
826 }
827 }
828 }
829
830 static void build_page_bitmap(PageDesc *p)
831 {
832 int n, tb_start, tb_end;
833 TranslationBlock *tb;
834
835 p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
836 if (!p->code_bitmap)
837 return;
838
839 tb = p->first_tb;
840 while (tb != NULL) {
841 n = (long)tb & 3;
842 tb = (TranslationBlock *)((long)tb & ~3);
843 /* NOTE: this is subtle as a TB may span two physical pages */
844 if (n == 0) {
845 /* NOTE: tb_end may be after the end of the page, but
846 it is not a problem */
847 tb_start = tb->pc & ~TARGET_PAGE_MASK;
848 tb_end = tb_start + tb->size;
849 if (tb_end > TARGET_PAGE_SIZE)
850 tb_end = TARGET_PAGE_SIZE;
851 } else {
852 tb_start = 0;
853 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
854 }
855 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
856 tb = tb->page_next[n];
857 }
858 }
859
860 TranslationBlock *tb_gen_code(CPUState *env,
861 target_ulong pc, target_ulong cs_base,
862 int flags, int cflags)
863 {
864 TranslationBlock *tb;
865 uint8_t *tc_ptr;
866 target_ulong phys_pc, phys_page2, virt_page2;
867 int code_gen_size;
868
869 phys_pc = get_phys_addr_code(env, pc);
870 tb = tb_alloc(pc);
871 if (!tb) {
872 /* flush must be done */
873 tb_flush(env);
874 /* cannot fail at this point */
875 tb = tb_alloc(pc);
876 /* Don't forget to invalidate previous TB info. */
877 tb_invalidated_flag = 1;
878 }
879 tc_ptr = code_gen_ptr;
880 tb->tc_ptr = tc_ptr;
881 tb->cs_base = cs_base;
882 tb->flags = flags;
883 tb->cflags = cflags;
884 cpu_gen_code(env, tb, &code_gen_size);
885 code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
886
887 /* check next page if needed */
888 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
889 phys_page2 = -1;
890 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
891 phys_page2 = get_phys_addr_code(env, virt_page2);
892 }
893 tb_link_phys(tb, phys_pc, phys_page2);
894 return tb;
895 }
896
897 /* invalidate all TBs which intersect with the target physical page
898 starting in range [start;end[. NOTE: start and end must refer to
899 the same physical page. 'is_cpu_write_access' should be true if called
900 from a real cpu write access: the virtual CPU will exit the current
901 TB if code is modified inside this TB. */
902 void tb_invalidate_phys_page_range(target_phys_addr_t start, target_phys_addr_t end,
903 int is_cpu_write_access)
904 {
905 int n, current_tb_modified, current_tb_not_found, current_flags;
906 CPUState *env = cpu_single_env;
907 PageDesc *p;
908 TranslationBlock *tb, *tb_next, *current_tb, *saved_tb;
909 target_ulong tb_start, tb_end;
910 target_ulong current_pc, current_cs_base;
911
912 p = page_find(start >> TARGET_PAGE_BITS);
913 if (!p)
914 return;
915 if (!p->code_bitmap &&
916 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
917 is_cpu_write_access) {
918 /* build code bitmap */
919 build_page_bitmap(p);
920 }
921
922 /* we remove all the TBs in the range [start, end[ */
923 /* XXX: see if in some cases it could be faster to invalidate all the code */
924 current_tb_not_found = is_cpu_write_access;
925 current_tb_modified = 0;
926 current_tb = NULL; /* avoid warning */
927 current_pc = 0; /* avoid warning */
928 current_cs_base = 0; /* avoid warning */
929 current_flags = 0; /* avoid warning */
930 tb = p->first_tb;
931 while (tb != NULL) {
932 n = (long)tb & 3;
933 tb = (TranslationBlock *)((long)tb & ~3);
934 tb_next = tb->page_next[n];
935 /* NOTE: this is subtle as a TB may span two physical pages */
936 if (n == 0) {
937 /* NOTE: tb_end may be after the end of the page, but
938 it is not a problem */
939 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
940 tb_end = tb_start + tb->size;
941 } else {
942 tb_start = tb->page_addr[1];
943 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
944 }
945 if (!(tb_end <= start || tb_start >= end)) {
946 #ifdef TARGET_HAS_PRECISE_SMC
947 if (current_tb_not_found) {
948 current_tb_not_found = 0;
949 current_tb = NULL;
950 if (env->mem_io_pc) {
951 /* now we have a real cpu fault */
952 current_tb = tb_find_pc(env->mem_io_pc);
953 }
954 }
955 if (current_tb == tb &&
956 (current_tb->cflags & CF_COUNT_MASK) != 1) {
957 /* If we are modifying the current TB, we must stop
958 its execution. We could be more precise by checking
959 that the modification is after the current PC, but it
960 would require a specialized function to partially
961 restore the CPU state */
962
963 current_tb_modified = 1;
964 cpu_restore_state(current_tb, env,
965 env->mem_io_pc, NULL);
966 #if defined(TARGET_I386)
967 current_flags = env->hflags;
968 current_flags |= (env->eflags & (IOPL_MASK | TF_MASK | VM_MASK));
969 current_cs_base = (target_ulong)env->segs[R_CS].base;
970 current_pc = current_cs_base + env->eip;
971 #else
972 #error unsupported CPU
973 #endif
974 }
975 #endif /* TARGET_HAS_PRECISE_SMC */
976 /* we need to do that to handle the case where a signal
977 occurs while doing tb_phys_invalidate() */
978 saved_tb = NULL;
979 if (env) {
980 saved_tb = env->current_tb;
981 env->current_tb = NULL;
982 }
983 tb_phys_invalidate(tb, -1);
984 if (env) {
985 env->current_tb = saved_tb;
986 if (env->interrupt_request && env->current_tb)
987 cpu_interrupt(env, env->interrupt_request);
988 }
989 }
990 tb = tb_next;
991 }
992 #if !defined(CONFIG_USER_ONLY)
993 /* if no code remaining, no need to continue to use slow writes */
994 if (!p->first_tb) {
995 invalidate_page_bitmap(p);
996 if (is_cpu_write_access) {
997 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
998 }
999 }
1000 #endif
1001 #ifdef TARGET_HAS_PRECISE_SMC
1002 if (current_tb_modified) {
1003 /* we generate a block containing just the instruction
1004 modifying the memory. It will ensure that it cannot modify
1005 itself */
1006 env->current_tb = NULL;
1007 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1008 cpu_resume_from_signal(env, NULL);
1009 }
1010 #endif
1011 }
1012
1013 /* len must be <= 8 and start must be a multiple of len */
1014 static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start, int len)
1015 {
1016 PageDesc *p;
1017 int offset, b;
1018 #if 0
1019 if (1) {
1020 if (loglevel) {
1021 fprintf(logfile, "modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1022 cpu_single_env->mem_io_vaddr, len,
1023 cpu_single_env->eip,
1024 cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1025 }
1026 }
1027 #endif
1028 p = page_find(start >> TARGET_PAGE_BITS);
1029 if (!p)
1030 return;
1031 if (p->code_bitmap) {
1032 offset = start & ~TARGET_PAGE_MASK;
1033 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1034 if (b & ((1 << len) - 1))
1035 goto do_invalidate;
1036 } else {
1037 do_invalidate:
1038 tb_invalidate_phys_page_range(start, start + len, 1);
1039 }
1040 }
1041
1042 #if !defined(CONFIG_SOFTMMU)
1043 static void tb_invalidate_phys_page(target_phys_addr_t addr,
1044 unsigned long pc, void *puc)
1045 {
1046 int n, current_flags, current_tb_modified;
1047 target_ulong current_pc, current_cs_base;
1048 PageDesc *p;
1049 TranslationBlock *tb, *current_tb;
1050 #ifdef TARGET_HAS_PRECISE_SMC
1051 CPUState *env = cpu_single_env;
1052 #endif
1053
1054 addr &= TARGET_PAGE_MASK;
1055 p = page_find(addr >> TARGET_PAGE_BITS);
1056 if (!p)
1057 return;
1058 tb = p->first_tb;
1059 current_tb_modified = 0;
1060 current_tb = NULL;
1061 current_pc = 0; /* avoid warning */
1062 current_cs_base = 0; /* avoid warning */
1063 current_flags = 0; /* avoid warning */
1064 #ifdef TARGET_HAS_PRECISE_SMC
1065 if (tb && pc != 0) {
1066 current_tb = tb_find_pc(pc);
1067 }
1068 #endif
1069 while (tb != NULL) {
1070 n = (long)tb & 3;
1071 tb = (TranslationBlock *)((long)tb & ~3);
1072 #ifdef TARGET_HAS_PRECISE_SMC
1073 if (current_tb == tb &&
1074 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1075 /* If we are modifying the current TB, we must stop
1076 its execution. We could be more precise by checking
1077 that the modification is after the current PC, but it
1078 would require a specialized function to partially
1079 restore the CPU state */
1080
1081 current_tb_modified = 1;
1082 cpu_restore_state(current_tb, env, pc, puc);
1083 #if defined(TARGET_I386)
1084 current_flags = env->hflags;
1085 current_flags |= (env->eflags & (IOPL_MASK | TF_MASK | VM_MASK));
1086 current_cs_base = (target_ulong)env->segs[R_CS].base;
1087 current_pc = current_cs_base + env->eip;
1088 #else
1089 #error unsupported CPU
1090 #endif
1091 }
1092 #endif /* TARGET_HAS_PRECISE_SMC */
1093 tb_phys_invalidate(tb, addr);
1094 tb = tb->page_next[n];
1095 }
1096 p->first_tb = NULL;
1097 #ifdef TARGET_HAS_PRECISE_SMC
1098 if (current_tb_modified) {
1099 /* we generate a block containing just the instruction
1100 modifying the memory. It will ensure that it cannot modify
1101 itself */
1102 env->current_tb = NULL;
1103 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1104 cpu_resume_from_signal(env, puc);
1105 }
1106 #endif
1107 }
1108 #endif
1109
1110 /* add the tb in the target page and protect it if necessary */
1111 static inline void tb_alloc_page(TranslationBlock *tb,
1112 unsigned int n, target_ulong page_addr)
1113 {
1114 PageDesc *p;
1115 TranslationBlock *last_first_tb;
1116
1117 tb->page_addr[n] = page_addr;
1118 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS);
1119 tb->page_next[n] = p->first_tb;
1120 last_first_tb = p->first_tb;
1121 p->first_tb = (TranslationBlock *)((long)tb | n);
1122 invalidate_page_bitmap(p);
1123
1124 #if defined(TARGET_HAS_SMC) || 1
1125
1126 #if defined(CONFIG_USER_ONLY)
1127 if (p->flags & PAGE_WRITE) {
1128 target_ulong addr;
1129 PageDesc *p2;
1130 int prot;
1131
1132 /* force the host page as non writable (writes will have a
1133 page fault + mprotect overhead) */
1134 page_addr &= qemu_host_page_mask;
1135 prot = 0;
1136 for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1137 addr += TARGET_PAGE_SIZE) {
1138
1139 p2 = page_find (addr >> TARGET_PAGE_BITS);
1140 if (!p2)
1141 continue;
1142 prot |= p2->flags;
1143 p2->flags &= ~PAGE_WRITE;
1144 page_get_flags(addr);
1145 }
1146 mprotect(g2h(page_addr), qemu_host_page_size,
1147 (prot & PAGE_BITS) & ~PAGE_WRITE);
1148 #ifdef DEBUG_TB_INVALIDATE
1149 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1150 page_addr);
1151 #endif
1152 }
1153 #else
1154 /* if some code is already present, then the pages are already
1155 protected. So we handle the case where only the first TB is
1156 allocated in a physical page */
1157 if (!last_first_tb) {
1158 tlb_protect_code(page_addr);
1159 }
1160 #endif
1161
1162 #endif /* TARGET_HAS_SMC */
1163 }
1164
1165 /* Allocate a new translation block. Flush the translation buffer if
1166 too many translation blocks or too much generated code. */
1167 TranslationBlock *tb_alloc(target_ulong pc)
1168 {
1169 TranslationBlock *tb;
1170
1171 if (nb_tbs >= code_gen_max_blocks ||
1172 (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
1173 return NULL;
1174 tb = &tbs[nb_tbs++];
1175 tb->pc = pc;
1176 tb->cflags = 0;
1177 return tb;
1178 }
1179
1180 void tb_free(TranslationBlock *tb)
1181 {
1182 /* In practice this is mostly used for single use temporary TB
1183 Ignore the hard cases and just back up if this TB happens to
1184 be the last one generated. */
1185 if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
1186 code_gen_ptr = tb->tc_ptr;
1187 nb_tbs--;
1188 }
1189 }
1190
1191 /* add a new TB and link it to the physical page tables. phys_page2 is
1192 (-1) to indicate that only one page contains the TB. */
1193 void tb_link_phys(TranslationBlock *tb,
1194 target_ulong phys_pc, target_ulong phys_page2)
1195 {
1196 unsigned int h;
1197 TranslationBlock **ptb;
1198
1199 /* Grab the mmap lock to stop another thread invalidating this TB
1200 before we are done. */
1201 mmap_lock();
1202 /* add in the physical hash table */
1203 h = tb_phys_hash_func(phys_pc);
1204 ptb = &tb_phys_hash[h];
1205 tb->phys_hash_next = *ptb;
1206 *ptb = tb;
1207
1208 /* add in the page list */
1209 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1210 if (phys_page2 != -1)
1211 tb_alloc_page(tb, 1, phys_page2);
1212 else
1213 tb->page_addr[1] = -1;
1214
1215 tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1216 tb->jmp_next[0] = NULL;
1217 tb->jmp_next[1] = NULL;
1218
1219 /* init original jump addresses */
1220 if (tb->tb_next_offset[0] != 0xffff)
1221 tb_reset_jump(tb, 0);
1222 if (tb->tb_next_offset[1] != 0xffff)
1223 tb_reset_jump(tb, 1);
1224
1225 #ifdef DEBUG_TB_CHECK
1226 tb_page_check();
1227 #endif
1228 mmap_unlock();
1229 }
1230
1231 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1232 tb[1].tc_ptr. Return NULL if not found */
1233 TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1234 {
1235 int m_min, m_max, m;
1236 unsigned long v;
1237 TranslationBlock *tb;
1238
1239 if (nb_tbs <= 0)
1240 return NULL;
1241 if (tc_ptr < (unsigned long)code_gen_buffer ||
1242 tc_ptr >= (unsigned long)code_gen_ptr)
1243 return NULL;
1244 /* binary search (cf Knuth) */
1245 m_min = 0;
1246 m_max = nb_tbs - 1;
1247 while (m_min <= m_max) {
1248 m = (m_min + m_max) >> 1;
1249 tb = &tbs[m];
1250 v = (unsigned long)tb->tc_ptr;
1251 if (v == tc_ptr)
1252 return tb;
1253 else if (tc_ptr < v) {
1254 m_max = m - 1;
1255 } else {
1256 m_min = m + 1;
1257 }
1258 }
1259 return &tbs[m_max];
1260 }
1261
1262 static void tb_reset_jump_recursive(TranslationBlock *tb);
1263
1264 static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1265 {
1266 TranslationBlock *tb1, *tb_next, **ptb;
1267 unsigned int n1;
1268
1269 tb1 = tb->jmp_next[n];
1270 if (tb1 != NULL) {
1271 /* find head of list */
1272 for(;;) {
1273 n1 = (long)tb1 & 3;
1274 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1275 if (n1 == 2)
1276 break;
1277 tb1 = tb1->jmp_next[n1];
1278 }
1279 /* we are now sure now that tb jumps to tb1 */
1280 tb_next = tb1;
1281
1282 /* remove tb from the jmp_first list */
1283 ptb = &tb_next->jmp_first;
1284 for(;;) {
1285 tb1 = *ptb;
1286 n1 = (long)tb1 & 3;
1287 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1288 if (n1 == n && tb1 == tb)
1289 break;
1290 ptb = &tb1->jmp_next[n1];
1291 }
1292 *ptb = tb->jmp_next[n];
1293 tb->jmp_next[n] = NULL;
1294
1295 /* suppress the jump to next tb in generated code */
1296 tb_reset_jump(tb, n);
1297
1298 /* suppress jumps in the tb on which we could have jumped */
1299 tb_reset_jump_recursive(tb_next);
1300 }
1301 }
1302
1303 static void tb_reset_jump_recursive(TranslationBlock *tb)
1304 {
1305 tb_reset_jump_recursive2(tb, 0);
1306 tb_reset_jump_recursive2(tb, 1);
1307 }
1308
1309 #if defined(TARGET_HAS_ICE)
1310 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1311 {
1312 target_phys_addr_t addr;
1313 target_ulong pd;
1314 ram_addr_t ram_addr;
1315 PhysPageDesc *p;
1316
1317 addr = cpu_get_phys_page_debug(env, pc);
1318 p = phys_page_find(addr >> TARGET_PAGE_BITS);
1319 if (!p) {
1320 pd = IO_MEM_UNASSIGNED;
1321 } else {
1322 pd = p->phys_offset;
1323 }
1324 ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1325 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1326 }
1327 #endif
1328
1329 /* Add a watchpoint. */
1330 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, int type)
1331 {
1332 int i;
1333
1334 for (i = 0; i < env->nb_watchpoints; i++) {
1335 if (addr == env->watchpoint[i].vaddr)
1336 return 0;
1337 }
1338 if (env->nb_watchpoints >= MAX_WATCHPOINTS)
1339 return -1;
1340
1341 i = env->nb_watchpoints++;
1342 env->watchpoint[i].vaddr = addr;
1343 env->watchpoint[i].type = type;
1344 tlb_flush_page(env, addr);
1345 /* FIXME: This flush is needed because of the hack to make memory ops
1346 terminate the TB. It can be removed once the proper IO trap and
1347 re-execute bits are in. */
1348 tb_flush(env);
1349 return i;
1350 }
1351
1352 /* Remove a watchpoint. */
1353 int cpu_watchpoint_remove(CPUState *env, target_ulong addr)
1354 {
1355 int i;
1356
1357 for (i = 0; i < env->nb_watchpoints; i++) {
1358 if (addr == env->watchpoint[i].vaddr) {
1359 env->nb_watchpoints--;
1360 env->watchpoint[i] = env->watchpoint[env->nb_watchpoints];
1361 tlb_flush_page(env, addr);
1362 return 0;
1363 }
1364 }
1365 return -1;
1366 }
1367
1368 /* Remove all watchpoints. */
1369 void cpu_watchpoint_remove_all(CPUState *env) {
1370 int i;
1371
1372 for (i = 0; i < env->nb_watchpoints; i++) {
1373 tlb_flush_page(env, env->watchpoint[i].vaddr);
1374 }
1375 env->nb_watchpoints = 0;
1376 }
1377
1378 /* add a breakpoint. EXCP_DEBUG is returned by the CPU loop if a
1379 breakpoint is reached */
1380 int cpu_breakpoint_insert(CPUState *env, target_ulong pc)
1381 {
1382 #if defined(TARGET_HAS_ICE)
1383 int i;
1384
1385 for(i = 0; i < env->nb_breakpoints; i++) {
1386 if (env->breakpoints[i] == pc)
1387 return 0;
1388 }
1389
1390 if (env->nb_breakpoints >= MAX_BREAKPOINTS)
1391 return -1;
1392 env->breakpoints[env->nb_breakpoints++] = pc;
1393
1394 if (kvm_enabled())
1395 kvm_update_debugger(env);
1396
1397 breakpoint_invalidate(env, pc);
1398 return 0;
1399 #else
1400 return -1;
1401 #endif
1402 }
1403
1404 /* remove all breakpoints */
1405 void cpu_breakpoint_remove_all(CPUState *env) {
1406 #if defined(TARGET_HAS_ICE)
1407 int i;
1408 for(i = 0; i < env->nb_breakpoints; i++) {
1409 breakpoint_invalidate(env, env->breakpoints[i]);
1410 }
1411 env->nb_breakpoints = 0;
1412 #endif
1413 }
1414
1415 /* remove a breakpoint */
1416 int cpu_breakpoint_remove(CPUState *env, target_ulong pc)
1417 {
1418 #if defined(TARGET_HAS_ICE)
1419 int i;
1420 for(i = 0; i < env->nb_breakpoints; i++) {
1421 if (env->breakpoints[i] == pc)
1422 goto found;
1423 }
1424 return -1;
1425 found:
1426 env->nb_breakpoints--;
1427 if (i < env->nb_breakpoints)
1428 env->breakpoints[i] = env->breakpoints[env->nb_breakpoints];
1429
1430 if (kvm_enabled())
1431 kvm_update_debugger(env);
1432
1433 breakpoint_invalidate(env, pc);
1434 return 0;
1435 #else
1436 return -1;
1437 #endif
1438 }
1439
1440 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1441 CPU loop after each instruction */
1442 void cpu_single_step(CPUState *env, int enabled)
1443 {
1444 #if defined(TARGET_HAS_ICE)
1445 if (env->singlestep_enabled != enabled) {
1446 env->singlestep_enabled = enabled;
1447 /* must flush all the translated code to avoid inconsistancies */
1448 /* XXX: only flush what is necessary */
1449 tb_flush(env);
1450 }
1451 if (kvm_enabled())
1452 kvm_update_debugger(env);
1453 #endif
1454 }
1455
1456 /* enable or disable low levels log */
1457 void cpu_set_log(int log_flags)
1458 {
1459 loglevel = log_flags;
1460 if (loglevel && !logfile) {
1461 logfile = fopen(logfilename, log_append ? "a" : "w");
1462 if (!logfile) {
1463 perror(logfilename);
1464 _exit(1);
1465 }
1466 #if !defined(CONFIG_SOFTMMU)
1467 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1468 {
1469 static char logfile_buf[4096];
1470 setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1471 }
1472 #else
1473 setvbuf(logfile, NULL, _IOLBF, 0);
1474 #endif
1475 log_append = 1;
1476 }
1477 if (!loglevel && logfile) {
1478 fclose(logfile);
1479 logfile = NULL;
1480 }
1481 }
1482
1483 void cpu_set_log_filename(const char *filename)
1484 {
1485 logfilename = strdup(filename);
1486 if (logfile) {
1487 fclose(logfile);
1488 logfile = NULL;
1489 }
1490 cpu_set_log(loglevel);
1491 }
1492
1493 /* mask must never be zero, except for A20 change call */
1494 void cpu_interrupt(CPUState *env, int mask)
1495 {
1496 #if !defined(USE_NPTL)
1497 TranslationBlock *tb;
1498 static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1499 #endif
1500 int old_mask;
1501
1502 old_mask = env->interrupt_request;
1503 /* FIXME: This is probably not threadsafe. A different thread could
1504 be in the middle of a read-modify-write operation. */
1505 env->interrupt_request |= mask;
1506 if (kvm_enabled() && !qemu_kvm_irqchip_in_kernel())
1507 kvm_update_interrupt_request(env);
1508 #if defined(USE_NPTL)
1509 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1510 problem and hope the cpu will stop of its own accord. For userspace
1511 emulation this often isn't actually as bad as it sounds. Often
1512 signals are used primarily to interrupt blocking syscalls. */
1513 #else
1514 if (use_icount) {
1515 env->icount_decr.u16.high = 0xffff;
1516 #ifndef CONFIG_USER_ONLY
1517 /* CPU_INTERRUPT_EXIT isn't a real interrupt. It just means
1518 an async event happened and we need to process it. */
1519 if (!can_do_io(env)
1520 && (mask & ~(old_mask | CPU_INTERRUPT_EXIT)) != 0) {
1521 cpu_abort(env, "Raised interrupt while not in I/O function");
1522 }
1523 #endif
1524 } else {
1525 tb = env->current_tb;
1526 /* if the cpu is currently executing code, we must unlink it and
1527 all the potentially executing TB */
1528 if (tb && !testandset(&interrupt_lock)) {
1529 env->current_tb = NULL;
1530 tb_reset_jump_recursive(tb);
1531 resetlock(&interrupt_lock);
1532 }
1533 }
1534 #endif
1535 }
1536
1537 void cpu_reset_interrupt(CPUState *env, int mask)
1538 {
1539 env->interrupt_request &= ~mask;
1540 }
1541
1542 const CPULogItem cpu_log_items[] = {
1543 { CPU_LOG_TB_OUT_ASM, "out_asm",
1544 "show generated host assembly code for each compiled TB" },
1545 { CPU_LOG_TB_IN_ASM, "in_asm",
1546 "show target assembly code for each compiled TB" },
1547 { CPU_LOG_TB_OP, "op",
1548 "show micro ops for each compiled TB" },
1549 { CPU_LOG_TB_OP_OPT, "op_opt",
1550 "show micro ops "
1551 #ifdef TARGET_I386
1552 "before eflags optimization and "
1553 #endif
1554 "after liveness analysis" },
1555 { CPU_LOG_INT, "int",
1556 "show interrupts/exceptions in short format" },
1557 { CPU_LOG_EXEC, "exec",
1558 "show trace before each executed TB (lots of logs)" },
1559 { CPU_LOG_TB_CPU, "cpu",
1560 "show CPU state before block translation" },
1561 #ifdef TARGET_I386
1562 { CPU_LOG_PCALL, "pcall",
1563 "show protected mode far calls/returns/exceptions" },
1564 #endif
1565 #ifdef DEBUG_IOPORT
1566 { CPU_LOG_IOPORT, "ioport",
1567 "show all i/o ports accesses" },
1568 #endif
1569 { 0, NULL, NULL },
1570 };
1571
1572 static int cmp1(const char *s1, int n, const char *s2)
1573 {
1574 if (strlen(s2) != n)
1575 return 0;
1576 return memcmp(s1, s2, n) == 0;
1577 }
1578
1579 /* takes a comma separated list of log masks. Return 0 if error. */
1580 int cpu_str_to_log_mask(const char *str)
1581 {
1582 const CPULogItem *item;
1583 int mask;
1584 const char *p, *p1;
1585
1586 p = str;
1587 mask = 0;
1588 for(;;) {
1589 p1 = strchr(p, ',');
1590 if (!p1)
1591 p1 = p + strlen(p);
1592 if(cmp1(p,p1-p,"all")) {
1593 for(item = cpu_log_items; item->mask != 0; item++) {
1594 mask |= item->mask;
1595 }
1596 } else {
1597 for(item = cpu_log_items; item->mask != 0; item++) {
1598 if (cmp1(p, p1 - p, item->name))
1599 goto found;
1600 }
1601 return 0;
1602 }
1603 found:
1604 mask |= item->mask;
1605 if (*p1 != ',')
1606 break;
1607 p = p1 + 1;
1608 }
1609 return mask;
1610 }
1611
1612 void cpu_abort(CPUState *env, const char *fmt, ...)
1613 {
1614 va_list ap;
1615 va_list ap2;
1616
1617 va_start(ap, fmt);
1618 va_copy(ap2, ap);
1619 fprintf(stderr, "qemu: fatal: ");
1620 vfprintf(stderr, fmt, ap);
1621 fprintf(stderr, "\n");
1622 #ifdef TARGET_I386
1623 cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1624 #else
1625 cpu_dump_state(env, stderr, fprintf, 0);
1626 #endif
1627 if (logfile) {
1628 fprintf(logfile, "qemu: fatal: ");
1629 vfprintf(logfile, fmt, ap2);
1630 fprintf(logfile, "\n");
1631 #ifdef TARGET_I386
1632 cpu_dump_state(env, logfile, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1633 #else
1634 cpu_dump_state(env, logfile, fprintf, 0);
1635 #endif
1636 fflush(logfile);
1637 fclose(logfile);
1638 }
1639 va_end(ap2);
1640 va_end(ap);
1641 abort();
1642 }
1643
1644 CPUState *cpu_copy(CPUState *env)
1645 {
1646 CPUState *new_env = cpu_init(env->cpu_model_str);
1647 /* preserve chaining and index */
1648 CPUState *next_cpu = new_env->next_cpu;
1649 int cpu_index = new_env->cpu_index;
1650 memcpy(new_env, env, sizeof(CPUState));
1651 new_env->next_cpu = next_cpu;
1652 new_env->cpu_index = cpu_index;
1653 return new_env;
1654 }
1655
1656 #if !defined(CONFIG_USER_ONLY)
1657
1658 static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1659 {
1660 unsigned int i;
1661
1662 /* Discard jump cache entries for any tb which might potentially
1663 overlap the flushed page. */
1664 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1665 memset (&env->tb_jmp_cache[i], 0,
1666 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1667
1668 i = tb_jmp_cache_hash_page(addr);
1669 memset (&env->tb_jmp_cache[i], 0,
1670 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1671 }
1672
1673 /* NOTE: if flush_global is true, also flush global entries (not
1674 implemented yet) */
1675 void tlb_flush(CPUState *env, int flush_global)
1676 {
1677 int i;
1678
1679 #if defined(DEBUG_TLB)
1680 printf("tlb_flush:\n");
1681 #endif
1682 /* must reset current TB so that interrupts cannot modify the
1683 links while we are modifying them */
1684 env->current_tb = NULL;
1685
1686 for(i = 0; i < CPU_TLB_SIZE; i++) {
1687 env->tlb_table[0][i].addr_read = -1;
1688 env->tlb_table[0][i].addr_write = -1;
1689 env->tlb_table[0][i].addr_code = -1;
1690 env->tlb_table[1][i].addr_read = -1;
1691 env->tlb_table[1][i].addr_write = -1;
1692 env->tlb_table[1][i].addr_code = -1;
1693 #if (NB_MMU_MODES >= 3)
1694 env->tlb_table[2][i].addr_read = -1;
1695 env->tlb_table[2][i].addr_write = -1;
1696 env->tlb_table[2][i].addr_code = -1;
1697 #if (NB_MMU_MODES == 4)
1698 env->tlb_table[3][i].addr_read = -1;
1699 env->tlb_table[3][i].addr_write = -1;
1700 env->tlb_table[3][i].addr_code = -1;
1701 #endif
1702 #endif
1703 }
1704
1705 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1706
1707 #ifdef USE_KQEMU
1708 if (env->kqemu_enabled) {
1709 kqemu_flush(env, flush_global);
1710 }
1711 #endif
1712 tlb_flush_count++;
1713 }
1714
1715 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
1716 {
1717 if (addr == (tlb_entry->addr_read &
1718 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1719 addr == (tlb_entry->addr_write &
1720 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1721 addr == (tlb_entry->addr_code &
1722 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1723 tlb_entry->addr_read = -1;
1724 tlb_entry->addr_write = -1;
1725 tlb_entry->addr_code = -1;
1726 }
1727 }
1728
1729 void tlb_flush_page(CPUState *env, target_ulong addr)
1730 {
1731 int i;
1732
1733 #if defined(DEBUG_TLB)
1734 printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1735 #endif
1736 /* must reset current TB so that interrupts cannot modify the
1737 links while we are modifying them */
1738 env->current_tb = NULL;
1739
1740 addr &= TARGET_PAGE_MASK;
1741 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1742 tlb_flush_entry(&env->tlb_table[0][i], addr);
1743 tlb_flush_entry(&env->tlb_table[1][i], addr);
1744 #if (NB_MMU_MODES >= 3)
1745 tlb_flush_entry(&env->tlb_table[2][i], addr);
1746 #if (NB_MMU_MODES == 4)
1747 tlb_flush_entry(&env->tlb_table[3][i], addr);
1748 #endif
1749 #endif
1750
1751 tlb_flush_jmp_cache(env, addr);
1752
1753 #ifdef USE_KQEMU
1754 if (env->kqemu_enabled) {
1755 kqemu_flush_page(env, addr);
1756 }
1757 #endif
1758 }
1759
1760 /* update the TLBs so that writes to code in the virtual page 'addr'
1761 can be detected */
1762 static void tlb_protect_code(ram_addr_t ram_addr)
1763 {
1764 cpu_physical_memory_reset_dirty(ram_addr,
1765 ram_addr + TARGET_PAGE_SIZE,
1766 CODE_DIRTY_FLAG);
1767 }
1768
1769 /* update the TLB so that writes in physical page 'phys_addr' are no longer
1770 tested for self modifying code */
1771 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
1772 target_ulong vaddr)
1773 {
1774 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] |= CODE_DIRTY_FLAG;
1775 }
1776
1777 static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
1778 unsigned long start, unsigned long length)
1779 {
1780 unsigned long addr;
1781 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1782 addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
1783 if ((addr - start) < length) {
1784 tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
1785 }
1786 }
1787 }
1788
1789 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
1790 int dirty_flags)
1791 {
1792 CPUState *env;
1793 unsigned long length, start1;
1794 int i, mask, len;
1795 uint8_t *p;
1796
1797 start &= TARGET_PAGE_MASK;
1798 end = TARGET_PAGE_ALIGN(end);
1799
1800 length = end - start;
1801 if (length == 0)
1802 return;
1803 len = length >> TARGET_PAGE_BITS;
1804 #ifdef USE_KQEMU
1805 /* XXX: should not depend on cpu context */
1806 env = first_cpu;
1807 if (env->kqemu_enabled) {
1808 ram_addr_t addr;
1809 addr = start;
1810 for(i = 0; i < len; i++) {
1811 kqemu_set_notdirty(env, addr);
1812 addr += TARGET_PAGE_SIZE;
1813 }
1814 }
1815 #endif
1816 mask = ~dirty_flags;
1817 p = phys_ram_dirty + (start >> TARGET_PAGE_BITS);
1818 for(i = 0; i < len; i++)
1819 p[i] &= mask;
1820
1821 /* we modify the TLB cache so that the dirty bit will be set again
1822 when accessing the range */
1823 start1 = start + (unsigned long)phys_ram_base;
1824 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1825 for(i = 0; i < CPU_TLB_SIZE; i++)
1826 tlb_reset_dirty_range(&env->tlb_table[0][i], start1, length);
1827 for(i = 0; i < CPU_TLB_SIZE; i++)
1828 tlb_reset_dirty_range(&env->tlb_table[1][i], start1, length);
1829 #if (NB_MMU_MODES >= 3)
1830 for(i = 0; i < CPU_TLB_SIZE; i++)
1831 tlb_reset_dirty_range(&env->tlb_table[2][i], start1, length);
1832 #if (NB_MMU_MODES == 4)
1833 for(i = 0; i < CPU_TLB_SIZE; i++)
1834 tlb_reset_dirty_range(&env->tlb_table[3][i], start1, length);
1835 #endif
1836 #endif
1837 }
1838 }
1839
1840 int cpu_physical_memory_set_dirty_tracking(int enable)
1841 {
1842 int r=0;
1843
1844 if (kvm_enabled())
1845 r = kvm_physical_memory_set_dirty_tracking(enable);
1846 in_migration = enable;
1847 return r;
1848 }
1849
1850 int cpu_physical_memory_get_dirty_tracking(void)
1851 {
1852 return in_migration;
1853 }
1854
1855 static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
1856 {
1857 ram_addr_t ram_addr;
1858
1859 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1860 ram_addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) +
1861 tlb_entry->addend - (unsigned long)phys_ram_base;
1862 if (!cpu_physical_memory_is_dirty(ram_addr)) {
1863 tlb_entry->addr_write |= TLB_NOTDIRTY;
1864 }
1865 }
1866 }
1867
1868 /* update the TLB according to the current state of the dirty bits */
1869 void cpu_tlb_update_dirty(CPUState *env)
1870 {
1871 int i;
1872 for(i = 0; i < CPU_TLB_SIZE; i++)
1873 tlb_update_dirty(&env->tlb_table[0][i]);
1874 for(i = 0; i < CPU_TLB_SIZE; i++)
1875 tlb_update_dirty(&env->tlb_table[1][i]);
1876 #if (NB_MMU_MODES >= 3)
1877 for(i = 0; i < CPU_TLB_SIZE; i++)
1878 tlb_update_dirty(&env->tlb_table[2][i]);
1879 #if (NB_MMU_MODES == 4)
1880 for(i = 0; i < CPU_TLB_SIZE; i++)
1881 tlb_update_dirty(&env->tlb_table[3][i]);
1882 #endif
1883 #endif
1884 }
1885
1886 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
1887 {
1888 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
1889 tlb_entry->addr_write = vaddr;
1890 }
1891
1892 /* update the TLB corresponding to virtual page vaddr
1893 so that it is no longer dirty */
1894 static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
1895 {
1896 int i;
1897
1898 vaddr &= TARGET_PAGE_MASK;
1899 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1900 tlb_set_dirty1(&env->tlb_table[0][i], vaddr);
1901 tlb_set_dirty1(&env->tlb_table[1][i], vaddr);
1902 #if (NB_MMU_MODES >= 3)
1903 tlb_set_dirty1(&env->tlb_table[2][i], vaddr);
1904 #if (NB_MMU_MODES == 4)
1905 tlb_set_dirty1(&env->tlb_table[3][i], vaddr);
1906 #endif
1907 #endif
1908 }
1909
1910 /* add a new TLB entry. At most one entry for a given virtual address
1911 is permitted. Return 0 if OK or 2 if the page could not be mapped
1912 (can only happen in non SOFTMMU mode for I/O pages or pages
1913 conflicting with the host address space). */
1914 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
1915 target_phys_addr_t paddr, int prot,
1916 int mmu_idx, int is_softmmu)
1917 {
1918 PhysPageDesc *p;
1919 unsigned long pd;
1920 unsigned int index;
1921 target_ulong address;
1922 target_ulong code_address;
1923 target_phys_addr_t addend;
1924 int ret;
1925 CPUTLBEntry *te;
1926 int i;
1927 target_phys_addr_t iotlb;
1928
1929 p = phys_page_find(paddr >> TARGET_PAGE_BITS);
1930 if (!p) {
1931 pd = IO_MEM_UNASSIGNED;
1932 } else {
1933 pd = p->phys_offset;
1934 }
1935 #if defined(DEBUG_TLB)
1936 printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
1937 vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
1938 #endif
1939
1940 ret = 0;
1941 address = vaddr;
1942 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
1943 /* IO memory case (romd handled later) */
1944 address |= TLB_MMIO;
1945 }
1946 addend = (unsigned long)phys_ram_base + (pd & TARGET_PAGE_MASK);
1947 if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
1948 /* Normal RAM. */
1949 iotlb = pd & TARGET_PAGE_MASK;
1950 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
1951 iotlb |= IO_MEM_NOTDIRTY;
1952 else
1953 iotlb |= IO_MEM_ROM;
1954 } else {
1955 /* IO handlers are currently passed a phsical address.
1956 It would be nice to pass an offset from the base address
1957 of that region. This would avoid having to special case RAM,
1958 and avoid full address decoding in every device.
1959 We can't use the high bits of pd for this because
1960 IO_MEM_ROMD uses these as a ram address. */
1961 iotlb = (pd & ~TARGET_PAGE_MASK) + paddr;
1962 }
1963
1964 code_address = address;
1965 /* Make accesses to pages with watchpoints go via the
1966 watchpoint trap routines. */
1967 for (i = 0; i < env->nb_watchpoints; i++) {
1968 if (vaddr == (env->watchpoint[i].vaddr & TARGET_PAGE_MASK)) {
1969 iotlb = io_mem_watch + paddr;
1970 /* TODO: The memory case can be optimized by not trapping
1971 reads of pages with a write breakpoint. */
1972 address |= TLB_MMIO;
1973 }
1974 }
1975
1976 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1977 env->iotlb[mmu_idx][index] = iotlb - vaddr;
1978 te = &env->tlb_table[mmu_idx][index];
1979 te->addend = addend - vaddr;
1980 if (prot & PAGE_READ) {
1981 te->addr_read = address;
1982 } else {
1983 te->addr_read = -1;
1984 }
1985
1986 if (prot & PAGE_EXEC) {
1987 te->addr_code = code_address;
1988 } else {
1989 te->addr_code = -1;
1990 }
1991 if (prot & PAGE_WRITE) {
1992 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
1993 (pd & IO_MEM_ROMD)) {
1994 /* Write access calls the I/O callback. */
1995 te->addr_write = address | TLB_MMIO;
1996 } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
1997 !cpu_physical_memory_is_dirty(pd)) {
1998 te->addr_write = address | TLB_NOTDIRTY;
1999 } else {
2000 te->addr_write = address;
2001 }
2002 } else {
2003 te->addr_write = -1;
2004 }
2005 return ret;
2006 }
2007
2008 #else
2009
2010 void tlb_flush(CPUState *env, int flush_global)
2011 {
2012 }
2013
2014 void tlb_flush_page(CPUState *env, target_ulong addr)
2015 {
2016 }
2017
2018 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
2019 target_phys_addr_t paddr, int prot,
2020 int mmu_idx, int is_softmmu)
2021 {
2022 return 0;
2023 }
2024
2025 /* dump memory mappings */
2026 void page_dump(FILE *f)
2027 {
2028 unsigned long start, end;
2029 int i, j, prot, prot1;
2030 PageDesc *p;
2031
2032 fprintf(f, "%-8s %-8s %-8s %s\n",
2033 "start", "end", "size", "prot");
2034 start = -1;
2035 end = -1;
2036 prot = 0;
2037 for(i = 0; i <= L1_SIZE; i++) {
2038 if (i < L1_SIZE)
2039 p = l1_map[i];
2040 else
2041 p = NULL;
2042 for(j = 0;j < L2_SIZE; j++) {
2043 if (!p)
2044 prot1 = 0;
2045 else
2046 prot1 = p[j].flags;
2047 if (prot1 != prot) {
2048 end = (i << (32 - L1_BITS)) | (j << TARGET_PAGE_BITS);
2049 if (start != -1) {
2050 fprintf(f, "%08lx-%08lx %08lx %c%c%c\n",
2051 start, end, end - start,
2052 prot & PAGE_READ ? 'r' : '-',
2053 prot & PAGE_WRITE ? 'w' : '-',
2054 prot & PAGE_EXEC ? 'x' : '-');
2055 }
2056 if (prot1 != 0)
2057 start = end;
2058 else
2059 start = -1;
2060 prot = prot1;
2061 }
2062 if (!p)
2063 break;
2064 }
2065 }
2066 }
2067
2068 int page_get_flags(target_ulong address)
2069 {
2070 PageDesc *p;
2071
2072 p = page_find(address >> TARGET_PAGE_BITS);
2073 if (!p)
2074 return 0;
2075 return p->flags;
2076 }
2077
2078 /* modify the flags of a page and invalidate the code if
2079 necessary. The flag PAGE_WRITE_ORG is positionned automatically
2080 depending on PAGE_WRITE */
2081 void page_set_flags(target_ulong start, target_ulong end, int flags)
2082 {
2083 PageDesc *p;
2084 target_ulong addr;
2085
2086 /* mmap_lock should already be held. */
2087 start = start & TARGET_PAGE_MASK;
2088 end = TARGET_PAGE_ALIGN(end);
2089 if (flags & PAGE_WRITE)
2090 flags |= PAGE_WRITE_ORG;
2091 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2092 p = page_find_alloc(addr >> TARGET_PAGE_BITS);
2093 /* We may be called for host regions that are outside guest
2094 address space. */
2095 if (!p)
2096 return;
2097 /* if the write protection is set, then we invalidate the code
2098 inside */
2099 if (!(p->flags & PAGE_WRITE) &&
2100 (flags & PAGE_WRITE) &&
2101 p->first_tb) {
2102 tb_invalidate_phys_page(addr, 0, NULL);
2103 }
2104 p->flags = flags;
2105 }
2106 }
2107
2108 int page_check_range(target_ulong start, target_ulong len, int flags)
2109 {
2110 PageDesc *p;
2111 target_ulong end;
2112 target_ulong addr;
2113
2114 end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2115 start = start & TARGET_PAGE_MASK;
2116
2117 if( end < start )
2118 /* we've wrapped around */
2119 return -1;
2120 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2121 p = page_find(addr >> TARGET_PAGE_BITS);
2122 if( !p )
2123 return -1;
2124 if( !(p->flags & PAGE_VALID) )
2125 return -1;
2126
2127 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2128 return -1;
2129 if (flags & PAGE_WRITE) {
2130 if (!(p->flags & PAGE_WRITE_ORG))
2131 return -1;
2132 /* unprotect the page if it was put read-only because it
2133 contains translated code */
2134 if (!(p->flags & PAGE_WRITE)) {
2135 if (!page_unprotect(addr, 0, NULL))
2136 return -1;
2137 }
2138 return 0;
2139 }
2140 }
2141 return 0;
2142 }
2143
2144 /* called from signal handler: invalidate the code and unprotect the
2145 page. Return TRUE if the fault was succesfully handled. */
2146 int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2147 {
2148 unsigned int page_index, prot, pindex;
2149 PageDesc *p, *p1;
2150 target_ulong host_start, host_end, addr;
2151
2152 /* Technically this isn't safe inside a signal handler. However we
2153 know this only ever happens in a synchronous SEGV handler, so in
2154 practice it seems to be ok. */
2155 mmap_lock();
2156
2157 host_start = address & qemu_host_page_mask;
2158 page_index = host_start >> TARGET_PAGE_BITS;
2159 p1 = page_find(page_index);
2160 if (!p1) {
2161 mmap_unlock();
2162 return 0;
2163 }
2164 host_end = host_start + qemu_host_page_size;
2165 p = p1;
2166 prot = 0;
2167 for(addr = host_start;addr < host_end; addr += TARGET_PAGE_SIZE) {
2168 prot |= p->flags;
2169 p++;
2170 }
2171 /* if the page was really writable, then we change its
2172 protection back to writable */
2173 if (prot & PAGE_WRITE_ORG) {
2174 pindex = (address - host_start) >> TARGET_PAGE_BITS;
2175 if (!(p1[pindex].flags & PAGE_WRITE)) {
2176 mprotect((void *)g2h(host_start), qemu_host_page_size,
2177 (prot & PAGE_BITS) | PAGE_WRITE);
2178 p1[pindex].flags |= PAGE_WRITE;
2179 /* and since the content will be modified, we must invalidate
2180 the corresponding translated code. */
2181 tb_invalidate_phys_page(address, pc, puc);
2182 #ifdef DEBUG_TB_CHECK
2183 tb_invalidate_check(address);
2184 #endif
2185 mmap_unlock();
2186 return 1;
2187 }
2188 }
2189 mmap_unlock();
2190 return 0;
2191 }
2192
2193 static inline void tlb_set_dirty(CPUState *env,
2194 unsigned long addr, target_ulong vaddr)
2195 {
2196 }
2197 #endif /* defined(CONFIG_USER_ONLY) */
2198
2199 #if !defined(CONFIG_USER_ONLY)
2200 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2201 ram_addr_t memory);
2202 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2203 ram_addr_t orig_memory);
2204 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2205 need_subpage) \
2206 do { \
2207 if (addr > start_addr) \
2208 start_addr2 = 0; \
2209 else { \
2210 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2211 if (start_addr2 > 0) \
2212 need_subpage = 1; \
2213 } \
2214 \
2215 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2216 end_addr2 = TARGET_PAGE_SIZE - 1; \
2217 else { \
2218 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2219 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2220 need_subpage = 1; \
2221 } \
2222 } while (0)
2223
2224 /* register physical memory. 'size' must be a multiple of the target
2225 page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2226 io memory page */
2227 void cpu_register_physical_memory(target_phys_addr_t start_addr,
2228 ram_addr_t size,
2229 ram_addr_t phys_offset)
2230 {
2231 target_phys_addr_t addr, end_addr;
2232 PhysPageDesc *p;
2233 CPUState *env;
2234 ram_addr_t orig_size = size;
2235 void *subpage;
2236
2237 #ifdef USE_KQEMU
2238 /* XXX: should not depend on cpu context */
2239 env = first_cpu;
2240 if (env->kqemu_enabled) {
2241 kqemu_set_phys_mem(start_addr, size, phys_offset);
2242 }
2243 #endif
2244 #ifdef USE_KVM
2245 if (kvm_enabled())
2246 kvm_cpu_register_physical_memory(start_addr, size, phys_offset);
2247 #endif
2248
2249 size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2250 end_addr = start_addr + (target_phys_addr_t)size;
2251 for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2252 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2253 if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2254 ram_addr_t orig_memory = p->phys_offset;
2255 target_phys_addr_t start_addr2, end_addr2;
2256 int need_subpage = 0;
2257
2258 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2259 need_subpage);
2260 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2261 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2262 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2263 &p->phys_offset, orig_memory);
2264 } else {
2265 subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2266 >> IO_MEM_SHIFT];
2267 }
2268 subpage_register(subpage, start_addr2, end_addr2, phys_offset);
2269 } else {
2270 p->phys_offset = phys_offset;
2271 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2272 (phys_offset & IO_MEM_ROMD))
2273 phys_offset += TARGET_PAGE_SIZE;
2274 }
2275 } else {
2276 p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2277 p->phys_offset = phys_offset;
2278 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2279 (phys_offset & IO_MEM_ROMD))
2280 phys_offset += TARGET_PAGE_SIZE;
2281 else {
2282 target_phys_addr_t start_addr2, end_addr2;
2283 int need_subpage = 0;
2284
2285 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2286 end_addr2, need_subpage);
2287
2288 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2289 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2290 &p->phys_offset, IO_MEM_UNASSIGNED);
2291 subpage_register(subpage, start_addr2, end_addr2,
2292 phys_offset);
2293 }
2294 }
2295 }
2296 }
2297
2298 /* since each CPU stores ram addresses in its TLB cache, we must
2299 reset the modified entries */
2300 /* XXX: slow ! */
2301 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2302 tlb_flush(env, 1);
2303 }
2304 }
2305
2306 /* XXX: temporary until new memory mapping API */
2307 ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2308 {
2309 PhysPageDesc *p;
2310
2311 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2312 if (!p)
2313 return IO_MEM_UNASSIGNED;
2314 return p->phys_offset;
2315 }
2316
2317 /* XXX: better than nothing */
2318 ram_addr_t qemu_ram_alloc(ram_addr_t size)
2319 {
2320 ram_addr_t addr;
2321 if ((phys_ram_alloc_offset + size) > phys_ram_size) {
2322 fprintf(stderr, "Not enough memory (requested_size = %" PRIu64 ", max memory = %" PRIu64 ")\n",
2323 (uint64_t)size, (uint64_t)phys_ram_size);
2324 abort();
2325 }
2326 addr = phys_ram_alloc_offset;
2327 phys_ram_alloc_offset = TARGET_PAGE_ALIGN(phys_ram_alloc_offset + size);
2328 return addr;
2329 }
2330
2331 void qemu_ram_free(ram_addr_t addr)
2332 {
2333 }
2334
2335 static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2336 {
2337 #ifdef DEBUG_UNASSIGNED
2338 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2339 #endif
2340 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2341 do_unassigned_access(addr, 0, 0, 0, 1);
2342 #endif
2343 return 0;
2344 }
2345
2346 static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2347 {
2348 #ifdef DEBUG_UNASSIGNED
2349 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2350 #endif
2351 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2352 do_unassigned_access(addr, 0, 0, 0, 2);
2353 #endif
2354 return 0;
2355 }
2356
2357 static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
2358 {
2359 #ifdef DEBUG_UNASSIGNED
2360 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2361 #endif
2362 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2363 do_unassigned_access(addr, 0, 0, 0, 4);
2364 #endif
2365 return 0;
2366 }
2367
2368 static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
2369 {
2370 #ifdef DEBUG_UNASSIGNED
2371 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2372 #endif
2373 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2374 do_unassigned_access(addr, 1, 0, 0, 1);
2375 #endif
2376 }
2377
2378 static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
2379 {
2380 #ifdef DEBUG_UNASSIGNED
2381 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2382 #endif
2383 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2384 do_unassigned_access(addr, 1, 0, 0, 2);
2385 #endif
2386 }
2387
2388 static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
2389 {
2390 #ifdef DEBUG_UNASSIGNED
2391 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2392 #endif
2393 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2394 do_unassigned_access(addr, 1, 0, 0, 4);
2395 #endif
2396 }
2397
2398 static CPUReadMemoryFunc *unassigned_mem_read[3] = {
2399 unassigned_mem_readb,
2400 unassigned_mem_readw,
2401 unassigned_mem_readl,
2402 };
2403
2404 static CPUWriteMemoryFunc *unassigned_mem_write[3] = {
2405 unassigned_mem_writeb,
2406 unassigned_mem_writew,
2407 unassigned_mem_writel,
2408 };
2409
2410 static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
2411 uint32_t val)
2412 {
2413 int dirty_flags;
2414 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2415 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2416 #if !defined(CONFIG_USER_ONLY)
2417 tb_invalidate_phys_page_fast(ram_addr, 1);
2418 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2419 #endif
2420 }
2421 stb_p(phys_ram_base + ram_addr, val);
2422 #ifdef USE_KQEMU
2423 if (cpu_single_env->kqemu_enabled &&
2424 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2425 kqemu_modify_page(cpu_single_env, ram_addr);
2426 #endif
2427 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2428 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2429 /* we remove the notdirty callback only if the code has been
2430 flushed */
2431 if (dirty_flags == 0xff)
2432 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2433 }
2434
2435 static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
2436 uint32_t val)
2437 {
2438 int dirty_flags;
2439 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2440 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2441 #if !defined(CONFIG_USER_ONLY)
2442 tb_invalidate_phys_page_fast(ram_addr, 2);
2443 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2444 #endif
2445 }
2446 stw_p(phys_ram_base + ram_addr, val);
2447 #ifdef USE_KQEMU
2448 if (cpu_single_env->kqemu_enabled &&
2449 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2450 kqemu_modify_page(cpu_single_env, ram_addr);
2451 #endif
2452 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2453 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2454 /* we remove the notdirty callback only if the code has been
2455 flushed */
2456 if (dirty_flags == 0xff)
2457 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2458 }
2459
2460 static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
2461 uint32_t val)
2462 {
2463 int dirty_flags;
2464 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2465 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2466 #if !defined(CONFIG_USER_ONLY)
2467 tb_invalidate_phys_page_fast(ram_addr, 4);
2468 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2469 #endif
2470 }
2471 stl_p(phys_ram_base + ram_addr, val);
2472 #ifdef USE_KQEMU
2473 if (cpu_single_env->kqemu_enabled &&
2474 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2475 kqemu_modify_page(cpu_single_env, ram_addr);
2476 #endif
2477 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2478 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2479 /* we remove the notdirty callback only if the code has been
2480 flushed */
2481 if (dirty_flags == 0xff)
2482 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2483 }
2484
2485 static CPUReadMemoryFunc *error_mem_read[3] = {
2486 NULL, /* never used */
2487 NULL, /* never used */
2488 NULL, /* never used */
2489 };
2490
2491 static CPUWriteMemoryFunc *notdirty_mem_write[3] = {
2492 notdirty_mem_writeb,
2493 notdirty_mem_writew,
2494 notdirty_mem_writel,
2495 };
2496
2497 /* Generate a debug exception if a watchpoint has been hit. */
2498 static void check_watchpoint(int offset, int flags)
2499 {
2500 CPUState *env = cpu_single_env;
2501 target_ulong vaddr;
2502 int i;
2503
2504 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
2505 for (i = 0; i < env->nb_watchpoints; i++) {
2506 if (vaddr == env->watchpoint[i].vaddr
2507 && (env->watchpoint[i].type & flags)) {
2508 env->watchpoint_hit = i + 1;
2509 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
2510 break;
2511 }
2512 }
2513 }
2514
2515 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2516 so these check for a hit then pass through to the normal out-of-line
2517 phys routines. */
2518 static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
2519 {
2520 check_watchpoint(addr & ~TARGET_PAGE_MASK, PAGE_READ);
2521 return ldub_phys(addr);
2522 }
2523
2524 static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
2525 {
2526 check_watchpoint(addr & ~TARGET_PAGE_MASK, PAGE_READ);
2527 return lduw_phys(addr);
2528 }
2529
2530 static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
2531 {
2532 check_watchpoint(addr & ~TARGET_PAGE_MASK, PAGE_READ);
2533 return ldl_phys(addr);
2534 }
2535
2536 static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
2537 uint32_t val)
2538 {
2539 check_watchpoint(addr & ~TARGET_PAGE_MASK, PAGE_WRITE);
2540 stb_phys(addr, val);
2541 }
2542
2543 static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
2544 uint32_t val)
2545 {
2546 check_watchpoint(addr & ~TARGET_PAGE_MASK, PAGE_WRITE);
2547 stw_phys(addr, val);
2548 }
2549
2550 static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
2551 uint32_t val)
2552 {
2553 check_watchpoint(addr & ~TARGET_PAGE_MASK, PAGE_WRITE);
2554 stl_phys(addr, val);
2555 }
2556
2557 static CPUReadMemoryFunc *watch_mem_read[3] = {
2558 watch_mem_readb,
2559 watch_mem_readw,
2560 watch_mem_readl,
2561 };
2562
2563 static CPUWriteMemoryFunc *watch_mem_write[3] = {
2564 watch_mem_writeb,
2565 watch_mem_writew,
2566 watch_mem_writel,
2567 };
2568
2569 static inline uint32_t subpage_readlen (subpage_t *mmio, target_phys_addr_t addr,
2570 unsigned int len)
2571 {
2572 uint32_t ret;
2573 unsigned int idx;
2574
2575 idx = SUBPAGE_IDX(addr - mmio->base);
2576 #if defined(DEBUG_SUBPAGE)
2577 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
2578 mmio, len, addr, idx);
2579 #endif
2580 ret = (**mmio->mem_read[idx][len])(mmio->opaque[idx][0][len], addr);
2581
2582 return ret;
2583 }
2584
2585 static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
2586 uint32_t value, unsigned int len)
2587 {
2588 unsigned int idx;
2589
2590 idx = SUBPAGE_IDX(addr - mmio->base);
2591 #if defined(DEBUG_SUBPAGE)
2592 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n", __func__,
2593 mmio, len, addr, idx, value);
2594 #endif
2595 (**mmio->mem_write[idx][len])(mmio->opaque[idx][1][len], addr, value);
2596 }
2597
2598 static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
2599 {
2600 #if defined(DEBUG_SUBPAGE)
2601 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2602 #endif
2603
2604 return subpage_readlen(opaque, addr, 0);
2605 }
2606
2607 static void subpage_writeb (void *opaque, target_phys_addr_t addr,
2608 uint32_t value)
2609 {
2610 #if defined(DEBUG_SUBPAGE)
2611 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2612 #endif
2613 subpage_writelen(opaque, addr, value, 0);
2614 }
2615
2616 static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
2617 {
2618 #if defined(DEBUG_SUBPAGE)
2619 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2620 #endif
2621
2622 return subpage_readlen(opaque, addr, 1);
2623 }
2624
2625 static void subpage_writew (void *opaque, target_phys_addr_t addr,
2626 uint32_t value)
2627 {
2628 #if defined(DEBUG_SUBPAGE)
2629 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2630 #endif
2631 subpage_writelen(opaque, addr, value, 1);
2632 }
2633
2634 static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
2635 {
2636 #if defined(DEBUG_SUBPAGE)
2637 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2638 #endif
2639
2640 return subpage_readlen(opaque, addr, 2);
2641 }
2642
2643 static void subpage_writel (void *opaque,
2644 target_phys_addr_t addr, uint32_t value)
2645 {
2646 #if defined(DEBUG_SUBPAGE)
2647 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2648 #endif
2649 subpage_writelen(opaque, addr, value, 2);
2650 }
2651
2652 static CPUReadMemoryFunc *subpage_read[] = {
2653 &subpage_readb,
2654 &subpage_readw,
2655 &subpage_readl,
2656 };
2657
2658 static CPUWriteMemoryFunc *subpage_write[] = {
2659 &subpage_writeb,
2660 &subpage_writew,
2661 &subpage_writel,
2662 };
2663
2664 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2665 ram_addr_t memory)
2666 {
2667 int idx, eidx;
2668 unsigned int i;
2669
2670 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2671 return -1;
2672 idx = SUBPAGE_IDX(start);
2673 eidx = SUBPAGE_IDX(end);
2674 #if defined(DEBUG_SUBPAGE)
2675 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %d\n", __func__,
2676 mmio, start, end, idx, eidx, memory);
2677 #endif
2678 memory >>= IO_MEM_SHIFT;
2679 for (; idx <= eidx; idx++) {
2680 for (i = 0; i < 4; i++) {
2681 if (io_mem_read[memory][i]) {
2682 mmio->mem_read[idx][i] = &io_mem_read[memory][i];
2683 mmio->opaque[idx][0][i] = io_mem_opaque[memory];
2684 }
2685 if (io_mem_write[memory][i]) {
2686 mmio->mem_write[idx][i] = &io_mem_write[memory][i];
2687 mmio->opaque[idx][1][i] = io_mem_opaque[memory];
2688 }
2689 }
2690 }
2691
2692 return 0;
2693 }
2694
2695 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2696 ram_addr_t orig_memory)
2697 {
2698 subpage_t *mmio;
2699 int subpage_memory;
2700
2701 mmio = qemu_mallocz(sizeof(subpage_t));
2702 if (mmio != NULL) {
2703 mmio->base = base;
2704 subpage_memory = cpu_register_io_memory(0, subpage_read, subpage_write, mmio);
2705 #if defined(DEBUG_SUBPAGE)
2706 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
2707 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
2708 #endif
2709 *phys = subpage_memory | IO_MEM_SUBPAGE;
2710 subpage_register(mmio, 0, TARGET_PAGE_SIZE - 1, orig_memory);
2711 }
2712
2713 return mmio;
2714 }
2715
2716 static int get_free_io_mem_idx(void)
2717 {
2718 int i;
2719
2720 for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
2721 if (!io_mem_used[i]) {
2722 io_mem_used[i] = 1;
2723 return i;
2724 }
2725
2726 return -1;
2727 }
2728
2729 static void io_mem_init(void)
2730 {
2731 int i;
2732
2733 cpu_register_io_memory(IO_MEM_ROM >> IO_MEM_SHIFT, error_mem_read, unassigned_mem_write, NULL);
2734 cpu_register_io_memory(IO_MEM_UNASSIGNED >> IO_MEM_SHIFT, unassigned_mem_read, unassigned_mem_write, NULL);
2735 cpu_register_io_memory(IO_MEM_NOTDIRTY >> IO_MEM_SHIFT, error_mem_read, notdirty_mem_write, NULL);
2736 for (i=0; i<5; i++)
2737 io_mem_used[i] = 1;
2738
2739 io_mem_watch = cpu_register_io_memory(0, watch_mem_read,
2740 watch_mem_write, NULL);
2741 /* alloc dirty bits array */
2742 phys_ram_dirty = qemu_vmalloc(phys_ram_size >> TARGET_PAGE_BITS);
2743 memset(phys_ram_dirty, 0xff, phys_ram_size >> TARGET_PAGE_BITS);
2744 }
2745
2746 /* mem_read and mem_write are arrays of functions containing the
2747 function to access byte (index 0), word (index 1) and dword (index
2748 2). Functions can be omitted with a NULL function pointer. The
2749 registered functions may be modified dynamically later.
2750 If io_index is non zero, the corresponding io zone is
2751 modified. If it is zero, a new io zone is allocated. The return
2752 value can be used with cpu_register_physical_memory(). (-1) is
2753 returned if error. */
2754 int cpu_register_io_memory(int io_index,
2755 CPUReadMemoryFunc **mem_read,
2756 CPUWriteMemoryFunc **mem_write,
2757 void *opaque)
2758 {
2759 int i, subwidth = 0;
2760
2761 if (io_index <= 0) {
2762 io_index = get_free_io_mem_idx();
2763 if (io_index == -1)
2764 return io_index;
2765 } else {
2766 if (io_index >= IO_MEM_NB_ENTRIES)
2767 return -1;
2768 }
2769
2770 for(i = 0;i < 3; i++) {
2771 if (!mem_read[i] || !mem_write[i])
2772 subwidth = IO_MEM_SUBWIDTH;
2773 io_mem_read[io_index][i] = mem_read[i];
2774 io_mem_write[io_index][i] = mem_write[i];
2775 }
2776 io_mem_opaque[io_index] = opaque;
2777 return (io_index << IO_MEM_SHIFT) | subwidth;
2778 }
2779
2780 void cpu_unregister_io_memory(int io_table_address)
2781 {
2782 int i;
2783 int io_index = io_table_address >> IO_MEM_SHIFT;
2784
2785 for (i=0;i < 3; i++) {
2786 io_mem_read[io_index][i] = unassigned_mem_read[i];
2787 io_mem_write[io_index][i] = unassigned_mem_write[i];
2788 }
2789 io_mem_opaque[io_index] = NULL;
2790 io_mem_used[io_index] = 0;
2791 }
2792
2793 CPUWriteMemoryFunc **cpu_get_io_memory_write(int io_index)
2794 {
2795 return io_mem_write[io_index >> IO_MEM_SHIFT];
2796 }
2797
2798 CPUReadMemoryFunc **cpu_get_io_memory_read(int io_index)
2799 {
2800 return io_mem_read[io_index >> IO_MEM_SHIFT];
2801 }
2802
2803 #endif /* !defined(CONFIG_USER_ONLY) */
2804
2805 /* physical memory access (slow version, mainly for debug) */
2806 #if defined(CONFIG_USER_ONLY)
2807 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
2808 int len, int is_write)
2809 {
2810 int l, flags;
2811 target_ulong page;
2812 void * p;
2813
2814 while (len > 0) {
2815 page = addr & TARGET_PAGE_MASK;
2816 l = (page + TARGET_PAGE_SIZE) - addr;
2817 if (l > len)
2818 l = len;
2819 flags = page_get_flags(page);
2820 if (!(flags & PAGE_VALID))
2821 return;
2822 if (is_write) {
2823 if (!(flags & PAGE_WRITE))
2824 return;
2825 /* XXX: this code should not depend on lock_user */
2826 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
2827 /* FIXME - should this return an error rather than just fail? */
2828 return;
2829 memcpy(p, buf, l);
2830 unlock_user(p, addr, l);
2831 } else {
2832 if (!(flags & PAGE_READ))
2833 return;
2834 /* XXX: this code should not depend on lock_user */
2835 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
2836 /* FIXME - should this return an error rather than just fail? */
2837 return;
2838 memcpy(buf, p, l);
2839 unlock_user(p, addr, 0);
2840 }
2841 len -= l;
2842 buf += l;
2843 addr += l;
2844 }
2845 }
2846
2847 #else
2848 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
2849 int len, int is_write)
2850 {
2851 int l, io_index;
2852 uint8_t *ptr;
2853 uint32_t val;
2854 target_phys_addr_t page;
2855 unsigned long pd;
2856 PhysPageDesc *p;
2857
2858 while (len > 0) {
2859 page = addr & TARGET_PAGE_MASK;
2860 l = (page + TARGET_PAGE_SIZE) - addr;
2861 if (l > len)
2862 l = len;
2863 p = phys_page_find(page >> TARGET_PAGE_BITS);
2864 if (!p) {
2865 pd = IO_MEM_UNASSIGNED;
2866 } else {
2867 pd = p->phys_offset;
2868 }
2869
2870 if (is_write) {
2871 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
2872 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
2873 /* XXX: could force cpu_single_env to NULL to avoid
2874 potential bugs */
2875 if (l >= 4 && ((addr & 3) == 0)) {
2876 /* 32 bit write access */
2877 val = ldl_p(buf);
2878 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
2879 l = 4;
2880 } else if (l >= 2 && ((addr & 1) == 0)) {
2881 /* 16 bit write access */
2882 val = lduw_p(buf);
2883 io_mem_write[io_index][1](io_mem_opaque[io_index], addr, val);
2884 l = 2;
2885 } else {
2886 /* 8 bit write access */
2887 val = ldub_p(buf);
2888 io_mem_write[io_index][0](io_mem_opaque[io_index], addr, val);
2889 l = 1;
2890 }
2891 } else {
2892 unsigned long addr1;
2893 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
2894 /* RAM case */
2895 ptr = phys_ram_base + addr1;
2896 memcpy(ptr, buf, l);
2897 if (!cpu_physical_memory_is_dirty(addr1)) {
2898 /* invalidate code */
2899 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
2900 /* set dirty bit */
2901 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
2902 (0xff & ~CODE_DIRTY_FLAG);
2903 }
2904 /* qemu doesn't execute guest code directly, but kvm does
2905 therefore fluch instruction caches */
2906 if (kvm_enabled())
2907 flush_icache_range((unsigned long)ptr,
2908 ((unsigned long)ptr)+l);
2909 }
2910 } else {
2911 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
2912 !(pd & IO_MEM_ROMD)) {
2913 /* I/O case */
2914 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
2915 if (l >= 4 && ((addr & 3) == 0)) {
2916 /* 32 bit read access */
2917 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
2918 stl_p(buf, val);
2919 l = 4;
2920 } else if (l >= 2 && ((addr & 1) == 0)) {
2921 /* 16 bit read access */
2922 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr);
2923 stw_p(buf, val);
2924 l = 2;
2925 } else {
2926 /* 8 bit read access */
2927 val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr);
2928 stb_p(buf, val);
2929 l = 1;
2930 }
2931 } else {
2932 /* RAM case */
2933 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
2934 (addr & ~TARGET_PAGE_MASK);
2935 memcpy(buf, ptr, l);
2936 }
2937 }
2938 len -= l;
2939 buf += l;
2940 addr += l;
2941 }
2942 }
2943
2944 /* used for ROM loading : can write in RAM and ROM */
2945 void cpu_physical_memory_write_rom(target_phys_addr_t addr,
2946 const uint8_t *buf, int len)
2947 {
2948 int l;
2949 uint8_t *ptr;
2950 target_phys_addr_t page;
2951 unsigned long pd;
2952 PhysPageDesc *p;
2953
2954 while (len > 0) {
2955 page = addr & TARGET_PAGE_MASK;
2956 l = (page + TARGET_PAGE_SIZE) - addr;
2957 if (l > len)
2958 l = len;
2959 p = phys_page_find(page >> TARGET_PAGE_BITS);
2960 if (!p) {
2961 pd = IO_MEM_UNASSIGNED;
2962 } else {
2963 pd = p->phys_offset;
2964 }
2965
2966 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
2967 (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
2968 !(pd & IO_MEM_ROMD)) {
2969 /* do nothing */
2970 } else {
2971 unsigned long addr1;
2972 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
2973 /* ROM/RAM case */
2974 ptr = phys_ram_base + addr1;
2975 memcpy(ptr, buf, l);
2976 }
2977 len -= l;
2978 buf += l;
2979 addr += l;
2980 }
2981 }
2982
2983
2984 /* warning: addr must be aligned */
2985 uint32_t ldl_phys(target_phys_addr_t addr)
2986 {
2987 int io_index;
2988 uint8_t *ptr;
2989 uint32_t val;
2990 unsigned long pd;
2991 PhysPageDesc *p;
2992
2993 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2994 if (!p) {
2995 pd = IO_MEM_UNASSIGNED;
2996 } else {
2997 pd = p->phys_offset;
2998 }
2999
3000 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3001 !(pd & IO_MEM_ROMD)) {
3002 /* I/O case */
3003 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3004 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3005 } else {
3006 /* RAM case */
3007 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3008 (addr & ~TARGET_PAGE_MASK);
3009 val = ldl_p(ptr);
3010 }
3011 return val;
3012 }
3013
3014 /* warning: addr must be aligned */
3015 uint64_t ldq_phys(target_phys_addr_t addr)
3016 {
3017 int io_index;
3018 uint8_t *ptr;
3019 uint64_t val;
3020 unsigned long pd;
3021 PhysPageDesc *p;
3022
3023 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3024 if (!p) {
3025 pd = IO_MEM_UNASSIGNED;
3026 } else {
3027 pd = p->phys_offset;
3028 }
3029
3030 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3031 !(pd & IO_MEM_ROMD)) {
3032 /* I/O case */
3033 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3034 #ifdef TARGET_WORDS_BIGENDIAN
3035 val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
3036 val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
3037 #else
3038 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3039 val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
3040 #endif
3041 } else {
3042 /* RAM case */
3043 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3044 (addr & ~TARGET_PAGE_MASK);
3045 val = ldq_p(ptr);
3046 }
3047 return val;
3048 }
3049
3050 /* XXX: optimize */
3051 uint32_t ldub_phys(target_phys_addr_t addr)
3052 {
3053 uint8_t val;
3054 cpu_physical_memory_read(addr, &val, 1);
3055 return val;
3056 }
3057
3058 /* XXX: optimize */
3059 uint32_t lduw_phys(target_phys_addr_t addr)
3060 {
3061 uint16_t val;
3062 cpu_physical_memory_read(addr, (uint8_t *)&val, 2);
3063 return tswap16(val);
3064 }
3065
3066 #ifdef __GNUC__
3067 #define likely(x) __builtin_expect(!!(x), 1)
3068 #define unlikely(x) __builtin_expect(!!(x), 0)
3069 #else
3070 #define likely(x) x
3071 #define unlikely(x) x
3072 #endif
3073
3074 /* warning: addr must be aligned. The ram page is not masked as dirty
3075 and the code inside is not invalidated. It is useful if the dirty
3076 bits are used to track modified PTEs */
3077 void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
3078 {
3079 int io_index;
3080 uint8_t *ptr;
3081 unsigned long pd;
3082 PhysPageDesc *p;
3083
3084 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3085 if (!p) {
3086 pd = IO_MEM_UNASSIGNED;
3087 } else {
3088 pd = p->phys_offset;
3089 }
3090
3091 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3092 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3093 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3094 } else {
3095 unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3096 ptr = phys_ram_base + addr1;
3097 stl_p(ptr, val);
3098
3099 if (unlikely(in_migration)) {
3100 if (!cpu_physical_memory_is_dirty(addr1)) {
3101 /* invalidate code */
3102 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3103 /* set dirty bit */
3104 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3105 (0xff & ~CODE_DIRTY_FLAG);
3106 }
3107 }
3108 }
3109 }
3110
3111 void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
3112 {
3113 int io_index;
3114 uint8_t *ptr;
3115 unsigned long pd;
3116 PhysPageDesc *p;
3117
3118 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3119 if (!p) {
3120 pd = IO_MEM_UNASSIGNED;
3121 } else {
3122 pd = p->phys_offset;
3123 }
3124
3125 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3126 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3127 #ifdef TARGET_WORDS_BIGENDIAN
3128 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
3129 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
3130 #else
3131 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3132 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
3133 #endif
3134 } else {
3135 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3136 (addr & ~TARGET_PAGE_MASK);
3137 stq_p(ptr, val);
3138 }
3139 }
3140
3141 /* warning: addr must be aligned */
3142 void stl_phys(target_phys_addr_t addr, uint32_t val)
3143 {
3144 int io_index;
3145 uint8_t *ptr;
3146 unsigned long pd;
3147 PhysPageDesc *p;
3148
3149 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3150 if (!p) {
3151 pd = IO_MEM_UNASSIGNED;
3152 } else {
3153 pd = p->phys_offset;
3154 }
3155
3156 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3157 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3158 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3159 } else {
3160 unsigned long addr1;
3161 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3162 /* RAM case */
3163 ptr = phys_ram_base + addr1;
3164 stl_p(ptr, val);
3165 if (!cpu_physical_memory_is_dirty(addr1)) {
3166 /* invalidate code */
3167 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3168 /* set dirty bit */
3169 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3170 (0xff & ~CODE_DIRTY_FLAG);
3171 }
3172 }
3173 }
3174
3175 /* XXX: optimize */
3176 void stb_phys(target_phys_addr_t addr, uint32_t val)
3177 {
3178 uint8_t v = val;
3179 cpu_physical_memory_write(addr, &v, 1);
3180 }
3181
3182 /* XXX: optimize */
3183 void stw_phys(target_phys_addr_t addr, uint32_t val)
3184 {
3185 uint16_t v = tswap16(val);
3186 cpu_physical_memory_write(addr, (const uint8_t *)&v, 2);
3187 }
3188
3189 /* XXX: optimize */
3190 void stq_phys(target_phys_addr_t addr, uint64_t val)
3191 {
3192 val = tswap64(val);
3193 cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
3194 }
3195
3196 #endif
3197
3198 /* virtual memory access for debug */
3199 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3200 uint8_t *buf, int len, int is_write)
3201 {
3202 int l;
3203 target_phys_addr_t phys_addr;
3204 target_ulong page;
3205
3206 while (len > 0) {
3207 page = addr & TARGET_PAGE_MASK;
3208 phys_addr = cpu_get_phys_page_debug(env, page);
3209 /* if no physical page mapped, return an error */
3210 if (phys_addr == -1)
3211 return -1;
3212 l = (page + TARGET_PAGE_SIZE) - addr;
3213 if (l > len)
3214 l = len;
3215 cpu_physical_memory_rw(phys_addr + (addr & ~TARGET_PAGE_MASK),
3216 buf, l, is_write);
3217 len -= l;
3218 buf += l;
3219 addr += l;
3220 }
3221 return 0;
3222 }
3223
3224 /* in deterministic execution mode, instructions doing device I/Os
3225 must be at the end of the TB */
3226 void cpu_io_recompile(CPUState *env, void *retaddr)
3227 {
3228 TranslationBlock *tb;
3229 uint32_t n, cflags;
3230 target_ulong pc, cs_base;
3231 uint64_t flags;
3232
3233 tb = tb_find_pc((unsigned long)retaddr);
3234 if (!tb) {
3235 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
3236 retaddr);
3237 }
3238 n = env->icount_decr.u16.low + tb->icount;
3239 cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
3240 /* Calculate how many instructions had been executed before the fault
3241 occurred. */
3242 n = n - env->icount_decr.u16.low;
3243 /* Generate a new TB ending on the I/O insn. */
3244 n++;
3245 /* On MIPS and SH, delay slot instructions can only be restarted if
3246 they were already the first instruction in the TB. If this is not
3247 the first instruction in a TB then re-execute the preceding
3248 branch. */
3249 #if defined(TARGET_MIPS)
3250 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
3251 env->active_tc.PC -= 4;
3252 env->icount_decr.u16.low++;
3253 env->hflags &= ~MIPS_HFLAG_BMASK;
3254 }
3255 #elif defined(TARGET_SH4)
3256 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
3257 && n > 1) {
3258 env->pc -= 2;
3259 env->icount_decr.u16.low++;
3260 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
3261 }
3262 #endif
3263 /* This should never happen. */
3264 if (n > CF_COUNT_MASK)
3265 cpu_abort(env, "TB too big during recompile");
3266
3267 cflags = n | CF_LAST_IO;
3268 pc = tb->pc;
3269 cs_base = tb->cs_base;
3270 flags = tb->flags;
3271 tb_phys_invalidate(tb, -1);
3272 /* FIXME: In theory this could raise an exception. In practice
3273 we have already translated the block once so it's probably ok. */
3274 tb_gen_code(env, pc, cs_base, flags, cflags);
3275 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
3276 the first in the TB) then we end up generating a whole new TB and
3277 repeating the fault, which is horribly inefficient.
3278 Better would be to execute just this insn uncached, or generate a
3279 second new TB. */
3280 cpu_resume_from_signal(env, NULL);
3281 }
3282
3283 void dump_exec_info(FILE *f,
3284 int (*cpu_fprintf)(FILE *f, const char *fmt, ...))
3285 {
3286 int i, target_code_size, max_target_code_size;
3287 int direct_jmp_count, direct_jmp2_count, cross_page;
3288 TranslationBlock *tb;
3289
3290 target_code_size = 0;
3291 max_target_code_size = 0;
3292 cross_page = 0;
3293 direct_jmp_count = 0;
3294 direct_jmp2_count = 0;
3295 for(i = 0; i < nb_tbs; i++) {
3296 tb = &tbs[i];
3297 target_code_size += tb->size;
3298 if (tb->size > max_target_code_size)
3299 max_target_code_size = tb->size;
3300 if (tb->page_addr[1] != -1)
3301 cross_page++;
3302 if (tb->tb_next_offset[0] != 0xffff) {
3303 direct_jmp_count++;
3304 if (tb->tb_next_offset[1] != 0xffff) {
3305 direct_jmp2_count++;
3306 }
3307 }
3308 }
3309 /* XXX: avoid using doubles ? */
3310 cpu_fprintf(f, "Translation buffer state:\n");
3311 cpu_fprintf(f, "gen code size %ld/%ld\n",
3312 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
3313 cpu_fprintf(f, "TB count %d/%d\n",
3314 nb_tbs, code_gen_max_blocks);
3315 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
3316 nb_tbs ? target_code_size / nb_tbs : 0,
3317 max_target_code_size);
3318 cpu_fprintf(f, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
3319 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
3320 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
3321 cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
3322 cross_page,
3323 nb_tbs ? (cross_page * 100) / nb_tbs : 0);
3324 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
3325 direct_jmp_count,
3326 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
3327 direct_jmp2_count,
3328 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
3329 cpu_fprintf(f, "\nStatistics:\n");
3330 cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
3331 cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
3332 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
3333 tcg_dump_info(f, cpu_fprintf);
3334 }
3335
3336 #if !defined(CONFIG_USER_ONLY)
3337
3338 #define MMUSUFFIX _cmmu
3339 #define GETPC() NULL
3340 #define env cpu_single_env
3341 #define SOFTMMU_CODE_ACCESS
3342
3343 #define SHIFT 0
3344 #include "softmmu_template.h"
3345
3346 #define SHIFT 1
3347 #include "softmmu_template.h"
3348
3349 #define SHIFT 2
3350 #include "softmmu_template.h"
3351
3352 #define SHIFT 3
3353 #include "softmmu_template.h"
3354
3355 #undef env
3356
3357 #endif
Patches shipped by Fedora